diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 67c5bcf6e1..f97a066c66 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,6 +7,10 @@ # See LICENSE in the root of the software repository for the full text of the License. # ----------------------------------------------------------------------------------------------------------- +# This self-contained probe intentionally carries generated binaries, Chinese +# source annotations, and performance outputs for offline comparison. +exclude: ^tests/atomic_probe/lazy_lamda_sample/ + repos: # Common @@ -23,7 +27,10 @@ repos: entry: python tests/lint/check_english_only.py language: python language_version: python3 - exclude: ^(3rdparty/|docs/zh-cn/|README\.zh-CN\.md) + # The atomic-probe tree contains standalone probes, usage guides and + # investigation records that are intentionally maintained in Chinese + # for the A5 device-performance workflow. + exclude: ^(3rdparty/|docs/zh-cn/|README\.zh-CN\.md|tests/atomic_probe/) - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 @@ -39,7 +46,10 @@ repos: hooks: - id: clang-format types_or: [c++, c] - exclude: ^3rdparty/ + # This self-contained probe preserves its hand-reviewed CCEC layout; + # formatting a touched legacy header rewrites thousands of unrelated + # lines and obscures the protocol delta under test. + exclude: ^(3rdparty/|tests/atomic_probe/pa_scheduler/) - repo: local hooks: diff --git a/conftest.py b/conftest.py index 938fe06f44..51d144e046 100644 --- a/conftest.py +++ b/conftest.py @@ -149,6 +149,55 @@ def pytest_addoption(parser): help="Enable L2 swimlane. Bare flag=level 4 (full). " "1=AICore timing, 2=+dispatch/fanout, 3=+sched phases, 4=+orch phases", ) + parser.addoption( + "--fdwic-tensormap", + action="store", + choices=["private", "shared"], + default="private", + help="Select the compile-time TensorMap artifact family for the a5/a5sim " + "fully_distributed_within_core runtime. The default is private.", + ) + parser.addoption( + "--fdwic-profile", + action="store", + choices=[ + "none", + "perf-clock", + "perf-clock-kernel", + "submit-pmu-none", + "submit-pmu-arg-build", + "submit-pmu-empty-bracket", + "submit-pmu-materialize", + "submit-pmu-claim", + "submit-pmu-register", + "submit-pmu-submit-transition", + "submit-pmu-efdrain-control", + "submit-pmu-prepare-map", + "submit-pmu-fanin", + "submit-pmu-winner-build-control", + "submit-pmu-alloc-complete-control", + "submit-pmu-loser-replay", + ], + default="none", + help="Select a private fully_distributed_within_core evidence build. " + "perf-clock keeps only the first/last Submit device clock per core; " + "perf-clock-kernel additionally aggregates linked-kernel time/calls inside that per-core window; " + "submit-pmu-none keeps one full Submit-sequence scalar/I-cache PMU window per core; " + "submit-pmu-arg-build attributes the Claim-to-Materialize eager-build interval; " + "submit-pmu-empty-bracket calibrates the adjacent begin/end observer cost at Claim.end; " + "submit-pmu-materialize attributes the current Materialize business span; " + "submit-pmu-claim attributes the current Claim business span; " + "submit-pmu-register attributes the RegisterOutputs call body; " + "submit-pmu-submit-transition attributes adjacent Submit gaps; " + "submit-pmu-efdrain-control attributes EfDrain scalar control while excluding linked Kernel calls; " + "submit-pmu-prepare-map attributes the dist_submit_prepare_map call body; " + "submit-pmu-fanin attributes the dynamic Kernel-winner Fanin span; " + "submit-pmu-winner-build-control attributes scalar control inside the complete WinnerBuild time boundary " + "while excluding linked Kernel calls; " + "submit-pmu-alloc-complete-control attributes scalar control inside the complete AllocComplete time boundary " + "while excluding linked Kernel calls; " + "submit-pmu-loser-replay attributes the real Kernel-loser drain_block_won call body.", + ) parser.addoption( "--use-example-exec-time", action="store_true", @@ -433,6 +482,85 @@ def _configure_sanitizer(config): ) +def _configure_fdwic_profile(config): + """Validate and publish the private real-A5 FDWIC evidence profile.""" + fdwic_profile = config.getoption("--fdwic-profile", default="none") + if fdwic_profile == "none": + os.environ.pop("PTO_FDWIC_PROFILE", None) + return + if fdwic_profile not in { + "perf-clock", + "perf-clock-kernel", + "submit-pmu-none", + "submit-pmu-arg-build", + "submit-pmu-empty-bracket", + "submit-pmu-materialize", + "submit-pmu-claim", + "submit-pmu-register", + "submit-pmu-submit-transition", + "submit-pmu-efdrain-control", + "submit-pmu-prepare-map", + "submit-pmu-fanin", + "submit-pmu-winner-build-control", + "submit-pmu-alloc-complete-control", + "submit-pmu-loser-replay", + }: + raise pytest.UsageError(f"unsupported --fdwic-profile {fdwic_profile!r}") + + platform = config.getoption("--platform", default=None) + runtime = config.getoption("--runtime", default=None) + level = config.getoption("--level", default=None) + if platform != "a5": + raise pytest.UsageError(f"--fdwic-profile {fdwic_profile} requires --platform a5") + if runtime not in {None, "fully_distributed_within_core"}: + raise pytest.UsageError(f"--fdwic-profile {fdwic_profile} only supports runtime fully_distributed_within_core") + if level not in {None, 2}: + raise pytest.UsageError(f"--fdwic-profile {fdwic_profile} only supports SceneTest level 2") + if config.getoption("--rounds", default=1) != 1: + raise pytest.UsageError( + f"--fdwic-profile {fdwic_profile} requires --rounds 1 because its per-case artifact is single-run" + ) + conflicting = [] + for option, label in ( + ("--enable-l2-swimlane", "--enable-l2-swimlane"), + ("--dump-args", "--dump-args"), + ("--enable-pmu", "--enable-pmu"), + ("--enable-dep-gen", "--enable-dep-gen"), + ("--enable-scope-stats", "--enable-scope-stats"), + ("--enable-device-log-timing", "--enable-device-log-timing"), + ("--enable-swimlane-overhead", "--enable-swimlane-overhead"), + ("--use-example-exec-time", "--use-example-exec-time"), + ): + if config.getoption(option, default=0): + conflicting.append(label) + if conflicting: + raise pytest.UsageError( + f"--fdwic-profile {fdwic_profile} must run without other diagnostics: " + ", ".join(conflicting) + ) + os.environ["PTO_FDWIC_PROFILE"] = fdwic_profile + + +def _configure_fdwic_tensormap(config): + """Validate and publish the explicit FDWIC TensorMap artifact family.""" + mode = config.getoption("--fdwic-tensormap", default="private") + if mode == "private": + os.environ.pop("PTO_FDWIC_TENSORMAP_MODE", None) + return + if mode != "shared": + raise pytest.UsageError(f"unsupported --fdwic-tensormap {mode!r}") + + platform = config.getoption("--platform", default=None) + runtime = config.getoption("--runtime", default=None) + level = config.getoption("--level", default=None) + if platform not in {"a5", "a5sim"}: + raise pytest.UsageError(f"--fdwic-tensormap {mode} requires --platform a5 or a5sim") + if runtime not in {None, "fully_distributed_within_core"}: + raise pytest.UsageError(f"--fdwic-tensormap {mode} only supports runtime fully_distributed_within_core") + if level not in {None, 2}: + raise pytest.UsageError(f"--fdwic-tensormap {mode} only supports SceneTest level 2") + os.environ["PTO_FDWIC_TENSORMAP_MODE"] = mode + + def pytest_configure(config): """Register custom markers and apply global config.""" config.addinivalue_line("markers", "platforms(list): supported platforms for standalone ST functions") @@ -445,6 +573,8 @@ def pytest_configure(config): ) _configure_sanitizer(config) + _configure_fdwic_tensormap(config) + _configure_fdwic_profile(config) # Configure logging unconditionally (not only when --log-level is passed) so # simpler's own WARNINGs — e.g. the device-log-timing "no device log written" @@ -613,6 +743,65 @@ def sort_key(item): items.sort(key=sort_key) + fdwic_profile = config.getoption("--fdwic-profile", default="none") + if fdwic_profile in { + "perf-clock", + "perf-clock-kernel", + "submit-pmu-none", + "submit-pmu-arg-build", + "submit-pmu-empty-bracket", + "submit-pmu-materialize", + "submit-pmu-claim", + "submit-pmu-register", + "submit-pmu-submit-transition", + "submit-pmu-efdrain-control", + "submit-pmu-prepare-map", + "submit-pmu-fanin", + "submit-pmu-winner-build-control", + "submit-pmu-alloc-complete-control", + "submit-pmu-loser-replay", + }: + incompatible = [] + for item in items: + if any(m.name == "skip" for m in item.iter_markers()): + continue + cls = getattr(item, "cls", None) + if cls is None: + incompatible.append(item.nodeid) + continue + if getattr(cls, "_st_level", None) != 2 or getattr(cls, "_st_runtime", None) != ( + "fully_distributed_within_core" + ): + incompatible.append(item.nodeid) + if incompatible: + sample = ", ".join(incompatible[:3]) + more = "" if len(incompatible) <= 3 else f" (+{len(incompatible) - 3} more)" + raise pytest.UsageError( + f"--fdwic-profile {fdwic_profile} only accepts level-2 fully_distributed_within_core tests; " + f"incompatible item(s): {sample}{more}" + ) + + fdwic_tensormap = config.getoption("--fdwic-tensormap", default="private") + if fdwic_tensormap == "shared": + incompatible = [] + for item in items: + if any(m.name == "skip" for m in item.iter_markers()): + continue + cls = getattr(item, "cls", None) + if ( + cls is None + or getattr(cls, "_st_level", None) != 2 + or getattr(cls, "_st_runtime", None) != "fully_distributed_within_core" + ): + incompatible.append(item.nodeid) + if incompatible: + sample = ", ".join(incompatible[:3]) + more = "" if len(incompatible) <= 3 else f" (+{len(incompatible) - 3} more)" + raise pytest.UsageError( + "--fdwic-tensormap shared only accepts level-2 fully_distributed_within_core tests; " + f"incompatible item(s): {sample}{more}" + ) + # L3 perf collection is not supported yet: a single L3 case forks N chip-processes # that all write l2_swimlane_records_.json to the same directory with # second-precision timestamps, so they trample each other. Block the @@ -1267,6 +1456,29 @@ def _l2_poisoned(): return set() +def _fdwic_worker_build_config(cls, platform, runtime): + """Prepare mode-aware FDWIC worker arguments and pool identity.""" + if runtime != "fully_distributed_within_core" or platform not in {"a5", "a5sim"}: + return {}, "" + + from simpler_setup.scene_test import ( # noqa: PLC0415 + _fdwic_tensormap_mode, + get_aicore_path_override, + ) + + cache_key = (cls.__qualname__, platform, runtime) + cls.compile_chip_callable(platform) + tensormap_mode = _fdwic_tensormap_mode() + kwargs = {"fdwic_tensormap_mode": tensormap_mode} + pool_token = f"{tensormap_mode}:" + aicore_override = get_aicore_path_override(cache_key) + if aicore_override is not None: + aicore_override = aicore_override.resolve() + kwargs["aicore_path_override"] = aicore_override + pool_token += str(aicore_override) + return kwargs, pool_token + + @pytest.fixture() def st_worker(request, st_platform, device_pool, _l2_worker_pool, _l2_poisoned): """Per-test Worker. @@ -1297,18 +1509,7 @@ def st_worker(request, st_platform, device_pool, _l2_worker_pool, _l2_poisoned): from simpler.worker import Worker # noqa: PLC0415 - kwargs = {} - aicore_pool_token = "" - if runtime == "fully_distributed_within_core" and st_platform in {"a5", "a5sim"}: - from simpler_setup.scene_test import get_aicore_path_override # noqa: PLC0415 - - cache_key = (cls.__qualname__, st_platform, runtime) - cls.compile_chip_callable(st_platform) - aicore_override = get_aicore_path_override(cache_key) - if aicore_override is not None: - aicore_override = aicore_override.resolve() - kwargs["aicore_path_override"] = aicore_override - aicore_pool_token = str(aicore_override) + kwargs, aicore_pool_token = _fdwic_worker_build_config(cls, st_platform, runtime) # L2 share: reuse any Worker already created for this runtime image in # the current process. Under xdist, each worker process is sliced to a diff --git a/docs/dfx/l2-swimlane-profiling.md b/docs/dfx/l2-swimlane-profiling.md index 420ed83136..d445432f41 100644 --- a/docs/dfx/l2-swimlane-profiling.md +++ b/docs/dfx/l2-swimlane-profiling.md @@ -127,9 +127,10 @@ runs): ```text / -├── l2_swimlane_records.json # raw runtime output -├── name_map_.json # optional func_id → name mapping -└── merged_swimlane.json # Perfetto trace (added by converter) +├── l2_swimlane_records.json # raw runtime output +├── name_map_.json # optional func_id → name mapping +├── merged_swimlane.json # Perfetto trace (added by converter) +└── swimlane_exclusive_analysis.json # FDWIC schema-v4 only ``` Filenames are fixed (no per-file timestamp) — the directory is the @@ -190,6 +191,70 @@ join key between `aicore_tasks` and `aicpu_tasks` is canonical producer of `task_token_raw`; AICPU only stamps the dispatch / finish timestamps and the per-core join token. +#### Fully-distributed-within-core schema-v4 + +The A5 fully-distributed-within-core (FDWIC) executor adds a strict +per-scalar-lane hierarchy at every enabled collection level. It does +not infer task kind from the task ID: `Submit.aux` is the source of +truth (`0` = kernel, `1` = allocation), and `Submit.flags & 1` records +the winner state. + +The exact exclusive child order is: + +| Submit path | Exclusive children in timestamp order | +| ----------- | ------------------------------------- | +| Kernel winner | `EfDrain`, `Materialize`, `PrepareMap`, `Claim`, `Fanin`, `Register`, `WinnerBuild` | +| Kernel loser | `EfDrain`, `Materialize`, `PrepareMap`, `Claim`, `Register`, `LoserReplay` | +| Alloc winner | `EfDrain`, `Materialize`, `PrepareMap`, `Register`, `Claim`, `AllocComplete` | +| Alloc loser | `EfDrain`, `Materialize`, `PrepareMap`, `Register`, `Claim` | + +The `Submit` timestamp starts after `dist_submit_begin()` and ends +before publishing the Submit record and returning from the API. Its +residual therefore includes unmarked control and intermediate trace +record writes. Exact closure describes the instrumented observation +window; it is not a claim that instrumentation has zero performance +impact. + +`LoserReplay` measures the production kernel-loser +`drain_block_won()` call. An allocation loser has no corresponding +action, so its Claim-to-Submit-end suffix remains a measured residual; +the tooling does not create a synthetic phase. `DrainWon`, `Atomic`, +`ClockBaseline`, `Commit`, and `RingBp` are nested observations and are +therefore reported as non-additive overlays. + +Each core emits exactly one adjacent pair of top-level parents: +`OrchestrationReplay` followed by `FinalDrain`. This worker-completion +window starts after the startup barrier and ends +before clock baselines, trace flush, and finish publication. Those +observation/lifecycle operations are deliberately outside the additive +business partition. + +The analyzer validates the following identities with raw integer cycles +before any cycle-to-µs conversion: + +```text +Submit = exclusive Submit children + SubmitResidual +SubmitEnvelope = SubmitUnion + BetweenSubmitResidual +OrchestrationReplay = Setup + SubmitUnion + BetweenSubmitResidual + Tail +FinalDrain = KernelUnion + FinalDrainResidual +WorkerCompletion = OrchestrationReplay + FinalDrain +``` + +Production orchestration can execute ready kernels while tensor-data +access waits between Submit calls. Such kernels are valid inside the +orchestration residual. Inside a Submit they are valid only in +`EfDrain`, `WinnerBuild`, or `AllocComplete`; a Kernel in Submit +residual or crossing a partition boundary is invalid. The report keeps +the cross-core wall-clock makespan separate from aggregate per-core +work, because summing core cycles is not elapsed wall time. + +For schema-v4 input, `swimlane_converter` also writes +`swimlane_exclusive_analysis.json`. The merged trace contains explicit +`submit_residual`, `submit_tail_gap`, and +`between_submit_residual` spans derived from the validated raw records. +This schema change adds no I-cache/PMU metric; existing level-4 atomic +records remain overlays. + #### Reader output (µs domain) After `read_perf_data()` joins the streams and converts to diff --git a/docs/fully_distributed_within_core.md b/docs/fully_distributed_within_core.md index fbc4418977..d539c26170 100644 --- a/docs/fully_distributed_within_core.md +++ b/docs/fully_distributed_within_core.md @@ -1112,12 +1112,56 @@ anchor 推送最后的多核子任务。这不是 per-task 串行阻塞，只发 ### 11.1 Claim 原子性 + 两条流的无跳过（原“Claim 原子性”“每 anchor 类型 claim 计数器”） **原语：单条 `atomic_fetch_max`。** 一个类型为 `T` 的核到达任务 `N` 时执行 -`old = atomic_fetch_max(cursor[T], N)`（`cursor[T]` 为 GM 上一个 64 位字），**`old < N` 即胜出**， +`old = atomic_fetch_max(cursor[T], N)`（`cursor[T]` 为 GM 上一个字），**`old < N` 即胜出**， 否则 `N` 已被认领。单原子、无循环。若硬件无 `fetch_max`，等价 CAS 回路： `do { c = load(cursor[T]); if (N <= c) return LOST; } while (!CAS(cursor[T], c, N)); return WON;` 内存序取 **acq-rel**（release 发布胜利，acquire 观察既有认领）。所有权判定只依赖 cursor 本身； 真正的产出数据另由完成标志同步（§11.5）。 +#### 11.1.1 A5 onboard 实现：硬件 `atomicMax` 直接维护全局 cursor + +**关键结论（推翻旧 §16.2/C1 假设）：A5（`dav_3510`）拥有可用的、核间一致的 GM 硬件原子。** +CANN `dav_3510` 的 `kernel_operator_atomic_impl.h` 暴露一组作用于 `__gm__` 地址的原子内建： +`atomicAdd` / `atomicMax` / `atomicMin`（支持 `int32_t/uint32_t/int64_t/uint64_t/float`）、 +`atomicCAS` / `atomicExch`（`uint32_t/uint64_t`）。CANN 文档 [AtomicMax](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/910beta3/API/ascendcopapi/atlasascendc_api_07_00261.html) +明确给出**三核并发 `AtomicMax` 得到正确结果、且各核拿到本次原子操作前的旧值**的示例——即这是**内存级、核间序列化**的真原子，**不需要 uncacheable 内存别名**（本机 double page table 不可用，见 §16.2 修订）。因此 §11.1 中"若硬件无 `fetch_max`"的 CAS 回退分支在 A5 上**不再需要**。 + +**global task cursor（全局，跨核共享）** —— 每类型一个（或分片，见 §7.2）GM 上的 `int32_t`，仅由 +硬件 `atomicMax` 维护。认领即一条原子： + +```cpp +// T ∈ {cube, vector, alloc}；cursor[T] 为 GM int32_t，初值 -1 +int32_t old = atomicMax(&cursor[T], N); // 硬件原子：内存级、跨核序列化；返回操作前旧值 +bool won = (old < N); // old < N ⇒ 本核把 cursor 从 old 推进到 N，独占 N +``` + +- **无 CAS 回路、无单独的 `load`**：`atomicMax` 一次完成"读旧值 + 取大发布"，返回值即判定依据。 + 这从根本上绕开了"先 `load` 读到 cacheable 陈旧副本再 CAS"的隐患——旧路径的 `claim()` 用 + `coherent_load` 起头，在真机 cacheable GM 上会读到本核的陈旧缓存值（见下"coherent 读"）。 +- **恰一胜者且无跳过**：`atomicMax` 的单调性与 §11.1 正文一致——每个 `T` id 恰被一个核置位， + cursor 只在 `T` 子序列上单调跃进，不跳过任何 `T` id。 + +**local task cursor（每核私有）** —— `local_current_task_index`（实现里的 `self->local_index`）是 +本核 replay submit 流时到达的任务 id，**纯每核变量、非共享、不加任何原子**，随核走位自增。它与 +global cursor 的唯一交互就是上面那条 `atomicMax(&cursor[T], local_index)`。 + +**cursor 的"coherent 读"（非认领场景）** —— 少数地方需要**读**（而非推进）global cursor：run-ahead +节流（§6.1，比较本核 `local_index` 与最慢核进度）、以及诊断打印。真机 cacheable GM 上普通 `load` +读到的是本核缓存副本（可能陈旧）。用一条**幂等原子**做一致读，避免陈旧： + +```cpp +int32_t cur = atomicMax(&cursor[T], INT32_MIN); // 恒不推进（任何真值 ≥ INT32_MIN），返回内存中真值 +``` + +即"以 `INT32_MIN` 取大"永不改变 cursor，却经原子单元读回内存里的当前真值。cursor 初值 `-1`、 +运行期只增，故 `INT32_MIN` 是安全的 no-op 下界。（等价替代：读前对该 cache line 做 `dcci` 失效再普通 +`load`；二者皆可，优先用幂等原子，省一次 cacheline 失效且与写路径同一原子单元、序更清晰。） + +> 适用范围：本节把**全局 cursor 的认领与读**在 A5 上定死为硬件 `atomicMax`。其余跨核共享量 +> （完成前沿 `frontier`、启动/回放 barrier `started_count`/`replay_done`、完成标志环 `flags[]`、 +> `block.won` 的 `remaining`/`state`）遵循**同一原则**——RMW 走硬件原子（`atomicMax`/`atomicAdd`/ +> `atomicCAS`）、纯读走幂等原子或 `dcci` 失效——实现细节与落地顺序见 §16.4 修订后的阶段计划。 + **恰一胜者且无跳过（取代“claim 计数器”）。** 每个 `T` 核按 id 递增顺序遇到 `T` 任务，`cursor[T]` 只会取到真实的 `T` 任务 id 值。在任何核尝试第 `k` 个 `T` 任务 `t_k` 之前，它必先尝试过 `t_{k-1}` （于是其时 `cursor[T] ≥ t_{k-1}`）；而 `cursor[T]` 的相邻取值之间没有别的 `T` id，故它只能从 @@ -1225,7 +1269,1263 @@ anchor 推送最后的多核子任务。这不是 per-task 串行阻塞，只发 动态配对方案（跨 block 均衡 MIX 工作；亦即 §3.2 讨论并暂不采用的“block 内先到先得代发布”等 思路的归宿）**仅在未来核解除该硬件绑定时**才需要，届时再行设计，**本节不予裁定**。 -## 12. 相关文档 +## 12. TensorMap 构建与 Private / Shared 双模式（统一 ring-per-bucket） + +本章新增一个**正交的运行模式开关**：TensorMap（§4、§8.2、§9）既可以保持"每核全量复制"形态 +（private），也可以改为"全局共享一份"（shared）。二者由**命令行开关**在运行启动时一次性选定， +贯穿整次运行不再切换。本章先回顾 TensorMap 如何被构建，再**论证两种模式统一采用同一套 +ring-per-bucket 数据结构**（§12.3 的 链表 vs ring 性能分析），随后定义两种模式仅有的差异、重点分析 +**shared 模式下无缓存一致性平台上的数据一致性问题**并给出解决方案，最后比较两种模式的性能、 +**论证 `auto` 定容的安全性与实现**（§12.7.2），并说明顶层入口（TensorMap 指针）如何被改造为支持双模。 + +> **规范约定（本章基准）**：**private 与 shared 两种模式统一采用 ring-per-bucket（每桶一个有界环形 +> 缓冲，§12.7.1）作为唯一数据结构**；二者仅在"副本数 / 谁 insert / 并发纪律 / 回收阈值"四点上分叉 +> （对比见 §12.3.2）。之所以统一：§12.3 的分析表明 **ring 在 private 与 shared 下均不劣于链表、且多数 +> 维度更优**（§6.4 的 O(N) 收益来自"回收窗口 + 哈希分桶"而非"链表"这一存储形态，ring 完整继承之， +> 另在局部性/内存/回收常数上取胜）。早期为 shared 考虑的"桶内链表 + 空闲链"方案因空闲链 ABA / +> `next` 悬挂指针 / use-after-recycle 等一致性陷阱**已被否决**（§12.7 作对照基线保留）；§6.4 为 private +> 实现的链表结构在本章分析下**亦被 ring 取代**（其回收窗口/哈希/winner-only 等洞见原样继承）。ring 的 +> 容量 `CAP`、溢出报错与命令行参数见 §12.7.2；`auto` 定容的安全性论证见 §12.7.2.3。 + +### 12.1 TensorMap 的构建回顾 + +TensorMap 把一个 tensor 区域 `[lo, hi)` 映射到其 **producer 任务 id**。它的构建规则在 §4 已定， +此处重述为可被两种模式共用的"构建原语"： + +- **查（lookup）**：给定一个 `INPUT`/`INOUT` tensor 区间，找到与之重叠、且 producer id **最大** + （最新）的条目，作为该 fan-in 的 producer。`INOUT` 两侧都参与：先查（消费旧版本）再插 + （产出新版本）。 +- **插（insert）**：给定一个 `OUTPUT` 或 `INOUT` tensor 区间，以**本任务 id** `N` 作为 producer + 登记一条新条目。 + +§6.4 曾为 **private 模式**把 `DistTensorMap` 物理结构定为"按 buffer 基址哈希分桶 + 桶内链表 + 按 +生产者任务的 entry 链 + 空闲链表 + lazy invalidation + `cleanup_retired` 按任务回收"，且 **insert 总是 +挂新条目**（不做就地替换），`lookup` 返回重叠者中 producer **最大**的那个。**但本章 §12.3 的性能分析 +表明该链表结构应被 ring-per-bucket 取代**——不仅 shared 必须用 ring（§12.7），private 用 ring 也全面 +不劣于链表且更省更快。故**两种模式统一采用 ring-per-bucket**；§6.4 的回收窗口（`N−H`）、哈希分桶、 +"多版本共存、不就地替换"、winner-only fan-in 等语义与洞见**原样继承**到 ring 实现（§12.5/§12.6）。 + +两种模式的差异不在于"构建原语"或"数据结构"本身（统一为 ring），而只在于：**insert/lookup 作用在 +哪一份 map 上、由谁来执行、以及跨核可见性如何保证**（四点差异见 §12.3.2）。 + +### 12.2 命令行开关 + +新增一个启动期开关（环境变量与 CLI 同义，沿用 §6.3 的 `--bind` 风格）： + +``` +--tensormap-mode {private|shared} # 等价环境变量 PTO_DIST_TENSORMAP_MODE +--tensormap-ring-cap {N|auto} # 等价环境变量 PTO_DIST_TENSORMAP_RING_CAP；private/shared 均生效 +``` + +- `private`（**默认**）：每核一份全量复制 map，数据结构为 ring-per-bucket（每核私有、单线程纪律，§12.3.2）。 +- `shared`：全核共享**唯一一份** TensorMap，数据结构同为 ring-per-bucket（并发纪律 + per-slot `seq`，§12.7.1）。 +- `--tensormap-ring-cap`：**两种模式均生效**（两者都用 ring），设定每桶 ring 的定长槽数 `CAP`（2 的幂）。 + 默认 `auto` = 由依赖跨度 `H`（private）或 `Δ+H`（shared）与该桶静态区域分布推导（§12.7.2）。**`auto` + 在两种模式下都能给出可证充分的容量**（§12.7.2.3）。 + +开关在 runtime 初始化阶段被读取一次，据此构造对应形态的 TensorMap 句柄（§12.9）并选择对应的 +insert/lookup 实现。**运行期不切换**，避免中途一致性灾难。所有 per-core 编排循环（§6）通过同一 +组抽象 API（`tm_insert` / `tm_lookup`）访问 map，由句柄分发到 private 或 shared 的 ring 实现——上层 +伪代码不变。`--tensormap-ring-cap` 影响两种模式每桶 ring 的分配尺寸，完整论证（含 `auto` 安全性） +见 §12.7.2。 + +### 12.3 为何统一到 ring-per-bucket：链表 vs ring 性能分析 + +**关键澄清（先破一个误解）。** §6.4 的 O(N) 收益**来自"按 H 窗口回收 + 哈希分桶"，与"链表 vs 数组" +这一存储形态无关**。ring 完整保留同一个回收窗口（private 用确定性 `N−H`，shared 用全局前沿 `R`） +与同一套哈希分桶，只把"桶内链表 + 空闲链"换成"桶内定长环 + 游标"。因此 **ring 继承 §6.4 的全部渐进 +收益（仍是 O(N)）**，差异只在**常数因子**（局部性、内存、回收开销）与**并发友好度**上——而这些都对 +ring 有利。 + +#### 12.3.1 逐维度对比（同一模式下，链表 vs ring） + +| 维度 | 链表（原 §6.4） | ring-per-bucket | 谁更优 | +| ---- | ---- | ---- | ---- | +| **lookup 访存** | 指针跳转，节点随机布局，每跳可能一次 cache miss | 连续数组扫描，硬件预取友好，触达 cache line 更少 | **ring** | +| **insert** | 从空闲链取节点 + 挂两条链（bucket 头 + 生产者链）指针操作 | `slots[tail%CAP]=e; tail++` 一次连续写 | **ring** | +| **回收** | 沿生产者任务链逐节点摘除并归还空闲链 | `head++` 游标自增，不摘链、不归还 | **ring** | +| **每 entry 内存** | payload + ~3 指针（bucket next / 生产者链 next / freelist next，≈24B） | payload only（下标隐式，无指针） | **ring** | +| **容量弹性** | 不定长，随空闲链弹性伸缩，无"满"概念 | 定长 `CAP`，需定容；满则反压/报错 | **链表**（唯一劣势，auto 可证充分定容化解，§12.7.2.3） | +| **并发（shared）** | 空闲链 CAS / ABA / use-after-recycle（§12.7 最难处） | per-slot `seq` + 游标，**无空闲链、无 next 指针** | **ring（决定性）** | +| **并发（private）** | 无（每核私有），但仍付指针/空闲链簿记 | 无，且退化为**纯整数 head/tail**（无 `seq`、无原子） | **ring**（更省） | +| **渐进复杂度** | O(N)（H 窗口回收之效） | O(N)（同一回收窗口） | 平 | +| **确定性/回收阈值** | private `N−H` | private `N−H`，shared `R`（§12.7.1） | 平 | +| **实现/验证面** | private 链表、shared ring → **两套结构** | 两模式**同一套 ring** | **ring（工程）** | + +**private 专门分析。** private 无跨核并发，ring 在此**退化为最简形态**：`head`/`tail` 是**普通整数** +（无需 `seq`、无需原子、无 ABA），insert = "写槽 + `tail++`"，lookup = "从 `tail-1` 向 `head` 连续扫"， +回收 = "`while slots[head%CAP].producer_id ≤ N−H: head++`"。相比 §6.4 链表，它**同为 O(N)**，但 +①lookup 连续扫描（链表是指针跳转，局部性差）；②每 entry 省约 3 指针 + 整条空闲链；③回收从"走生产者 +链 + 归还空闲池"简化为"游标自增"。**唯一代价**是定长 `CAP`——但 private 回收阈值是**确定性 `N−H`**、 +任务图**静态已知**，故每桶存活槽数上界**可在构建期精确算出**，`auto` 能给出**可证不溢出**的 `CAP` +（§12.7.2.3）。故 private 下 ring **全面不劣于链表、且更省更快**。 + +**shared 专门分析。** 已由 §12.7 定论：链表空闲链在无缓存一致性平台上引出 ABA / use-after-recycle +两大最难陷阱，ring 用"游标自增回收 + per-slot `seq`"直接消去（§12.7.1）。故 shared **只能是 ring**。 + +**结论。** ring 在 **private 与 shared 下均不劣于链表**：private 赢在常数因子（局部性/内存/回收）且退化 +到无原子最简形态，shared 则**只有** ring 可行；再加**只需维护/验证一套数据结构**的工程收益。因此 +本章**弃用 §6.4 的 private 链表，两模式统一为 ring**（§6.4 的回收窗口、哈希分桶、winner-only fan-in +等洞见原样继承到 ring）。 + +#### 12.3.2 统一之后：两种模式仅存的四点差异 + +统一到 ring 后，private 与 shared **数据结构完全相同**，仅在下列四点分叉（其余——哈希分桶、多版本 +追加、时序过滤 lookup、`N−H`/`R` 回收窗口——完全共用）： + +| 分叉点 | private | shared | +| ---- | ---- | ---- | +| **副本数** | 每核一份（全量复制） | 全局唯一一份 | +| **谁 insert** | **所有核**都 insert（各写自己副本，保持各核一致） | **仅 winner** insert（每任务恰好一核） | +| **并发纪律** | 无：`head`/`tail` 为普通整数，无 `seq`、无原子、无 invalidate | per-slot `seq` acq-rel + `reserve`/`head` 原子 + writeback/invalidate（§12.7.1） | +| **回收阈值** | 确定性 `N−H`，每核本地推进（§6.4 语义） | 全局 `R = min_progress−H−1`，协作推进（§12.7.1/§9.5） | + +- **一致性**：private 每核只读写自己的副本，**无跨核可见性问题**（producer 数据可见性仍由完成标志环 + §11.5 保证）；shared 是并发单副本，一致性由 §12.7.1 的 `seq`/acq-rel/游标纪律保证。 +- **代价权衡**：private 内存 = `核数 × 单份`、每核为全部任务付 insert 地板；shared 内存 `1×`、insert + 仅 winner，但引入跨核 invalidate 流量与热桶 `reserve` 竞争。完整取舍见 §12.8。 +- **lookup**：两模式都是"连续扫描 + 时序过滤取最新合法"（§12.6）；private 少了 per-slot `seq` 校验与 + invalidate（纯本地读）。 + +### 12.4 shared_tensormap —— 单副本 + winner-only insert + +> **核心观察**：claim race（§2）使走得最快的核（winner）在任务 id 序列上**领先**于其它核。winner +> 先构建并执行靠前的任务，因此**它的 TensorMap 进度也领先**——它刚 insert 的条目，正是落后核稍后 +> lookup 时所需要的。于是存在一种可能：**让 winner 把它 insert 的条目直接发布给所有核共享**， +> 落后核无需自己 insert、直接查这份共享 map 即可。 + +shared 模式据此重新划分职责： + +- **形态**：全核共享**唯一一份** TensorMap，物理上驻留在一块全局可寻址的 GM 区域，组织为 + **ring-per-bucket**——按 buffer 基址哈希分桶，**每个桶是一个定长 `CAP` 槽的有界环**（`RingBucket`， + §12.7.1），只有 `head`（回收游标）/ `tail`（发布游标）/ `reserve`（MPSC 抢槽游标）三个原子字， + 外加每槽一个 `seq` 代戳。**没有链表节点、没有 `next` 指针、没有空闲链。** +- **谁构建（insert）**：**仅 winner 做 insert**。败者与 follower 在走位到任务 `N` 时**不再** insert—— + 因为 winner 的 insert 已经（或即将）对全核可见，落后核重放 insert 既冗余又会与 winner 抢同一份 + map。insert = `k = fetch_add(reserve, 1)` 抢一个确定下标 `k % CAP` → 写槽字段 → writeback → + release-store 该槽 `seq`（§12.7.1）。这一改动直接消除了 §6.3/§6.4 中"每核为全部任务 insert"的地板 + 开销，是 shared 模式在多核下的主要性能收益来源（§12.8）。 +- **谁查（lookup）**：任何核在赢得任务 `N`、需要解析 fan-in 时，都查这一份共享 ring：从 `reserve` + 往 `head` 方向扫连续槽（无指针跳转），对每槽 acquire-read `seq` 校验有效后读字段，应用 §12.6 + 时序过滤取合法者中 producer 最大（§12.6）。 +- **回收**：由全局 reclaim 前沿 `R`（基于**各核进度最小值**，§9.5）驱动，**回收即游标自增**——当 + `slots[head % CAP].producer_id ≤ R` 时 `head++`，既不摘链也不归还空闲池（§12.7.1）。 +- **溢出**：`fetch_add(reserve)` 后若 `reserve − head > CAP`（ring 满）则 winner **不覆写**、走 + 反压/报错路径（§12.7.2），绝不静默丢条目。 + +shared 模式随即带来三个必须解决的问题：(A) INOUT 重写导致同一区域存在多个 producer 版本，如何 +在共享 ring 中表达（§12.5）；(B) 落后核 lookup 时如何避免看到"未来 producer"（§12.6）；(C) AI 核 +**无缓存一致性**，共享 ring 的跨核数据一致性如何保证（§12.7）。下面三节逐一展开，**均以 ring 为准**。 + +### 12.5 INOUT 重写：多版本以 ring 槽共存（append，不摘链、不替换） + +INOUT tensor 既消费旧版本又产出新版本（§4）。在 claim race 下，同一区域可能被多个 winner 先后 +以 INOUT 方式写入，从而**同一区域在共享 ring 中存在多个 producer 版本**。private 模式里这不是问题 +（每核自己 insert，lookup 取最新即可）；shared 模式下，若试图"用新 producer **替换**旧槽"， +会丢失旧版本——而落后核此刻可能仍需要旧版本作为它的 fan-in（它的"现在"还没到新 producer）。 + +**规则（共享 ring 的多版本追加）：** + +1. **绝不就地替换 producer。** 当某区域的 producer 被更新（例如 INOUT 重写），**不**修改任何已发布 + 槽的字段（§12.7.1 的"发布即不可变"），而是**追加一条新槽**，其 producer id = winner 的任务 id `N`。 +2. **追加落在 ring 尾部。** `k = fetch_add(reserve, 1)` 抢一个确定下标 `k % CAP`，写入 + `{producer_id=N, region, ...}`，writeback 后 release-store 该槽 `seq = k + lap*CAP`（§12.7.1）。 + 旧槽**原样保留**在环内，仍携带它更老的 producer id，直到 `head` 越过它被回收。 +3. **lookup 取"最新可见且合法"者。** 从 `reserve` 往 `head` 方向扫环内有效槽（seq 校验通过），在所有 + 与查询区间重叠者中，选 producer id 最大、但又满足下节 §12.6 时序合法性的那一个。 + +如此，同一区域的多版本 producer 在共享 ring 中以**按 append 次序（≈producer id 升序）排列的连续槽** +共存，旧版本随 reclaim 前沿 `R` 推进、`head++` 而被回收（§12.7.1）。相比被否决的链表方案，ring 用 +"下标 + `seq`"取代"`next` 指针 + 桶头替换"，多版本共存无需任何指针操作，扫描局部性更好。 + +### 12.6 跳过"未来 producer"：以本地任务索引为时序过滤 + +> **问题**：winner 走得快，它 insert 的条目 producer id 可能**大于**某个落后核当前的 +> `local_current_task_index`。若落后核在解析自己位于 id `N` 的任务的 fan-in 时，查到了一个 +> producer id `P > N`，那它就**引用了一个属于它自己未来的任务**——该未来任务可能尚未执行、其 +> 完成标志未置位、其输出数据尚不可读，于是消费者会错误地阻塞等待一个"未来 producer"，或更糟， +> 读到未完成的数据。这本质上是"把 winner 的时钟强加给落后核"。 + +**规则（时序过滤）：** 任何核在 lookup 时，**跳过 producer id ≥ 自身 `local_current_task_index` +的条目**。设本核当前走位到任务 `N`（即 `local_current_task_index == N`），则只接受 producer id +`< N` 的条目作为合法 fan-in。ring 版 lookup 扫描的是**连续槽下标**（无指针跳转），每槽先 acquire-read +`seq` 确认有效（§12.7.1 防 ABA）再读字段： + +```text +lookup(region, N): # N = 本核 local_current_task_index + best = NONE + b = bucket_of(region) # 定位桶（该桶的 RingBucket） + hi = acquire_load(b.reserve) # 已抢到的最高下标（发布上界） + lo = acquire_load(b.head) # 回收游标（最旧仍存活槽） + for k in range(hi-1, lo-1, -1): # 从最新 append 往最旧扫连续槽 + s = &b.slots[k % CAP] + if acquire_load(s.seq) != k # ★per-slot seq 校验：槽已被复用/未发布 → 跳过 + continue # （非一致平台先 invalidate 该 slot cache line） + if s.producer_id < N # ★时序过滤：跳过"未来 producer" + and overlaps(s.region, region): + if best == NONE or s.producer_id > best.producer_id: + best = snapshot(s) # 取合法者中最新；拷出快照，不缓存槽指针 + return best # 可能返回 NONE（尚无合法 producer） +``` + +> ring 扫描按下标从 `reserve-1` 递减到 `head`，因追加近似按 producer id 升序，故先遇到者即较新； +> 也可一旦命中一个 `producer_id < N` 的重叠槽就提前返回（该方向上它已是最大合法者）。`seq != k` +> 表示该槽尚未发布或已被后续 lap 复用（§12.7.1），一律跳过；**全程不跨调用缓存 `head`/槽指针**。 +> +> **注（落地优化）**：上面每槽 `acquire_load(s.seq)` 是**通用 MPSC** 设计所需（`reserve` 只是抢槽游标、 +> 非发布水位）。实际实现采用**单一串行追加者**（§12.10(1)），`tail` 成为真正的发布水位，故 reader 只需对 +> `tail` 做**一次** acquire、其下各槽 `seq` 改 relaxed 读即可——把"每槽一次 acquire"摊薄为"每 lookup 一次 +> acquire"，详见 §12.10(4)。 + +**为什么用 `local_current_task_index` 而不是全局前沿 `F`。** 时序合法性是**每核本地的时钟**概念： +"我还没走到 id `P`，就不该把 `P` 当作我的 producer"。各核的 `local_current_task_index` 严格单调地 +跟着自己的走位推进，是本核"当前时刻"的权威；而 `F` 是全局完成前沿，与"我是否已到达 `P`"无关。 +用本地索引作阈值，确保每个核只引用**自己时间线上的过去**。 + +**返回 NONE 的处置。** 若整个环内没有任何 producer id `< N` 的重叠有效槽（winner 还没 append 到此 +区域、或本核是该区域的第一个 producer），则该 fan-in 解析为"无 producer"——即本任务的该输入是 +图的外部输入（host 提供的初始 tensor），不需要等待完成标志。这与 private 模式下"查不到 = 外部 +输入"的语义一致，只是 shared 模式下"查不到"还可能是"winner 尚未发布"——但二者对消费者行为相同 +（都不等任何 producer），且当本核确实是该区域首任 producer 时为正确；当本核并非首任、只是 winner +尚未发布时，见 §12.7 末尾的"发布保证"。 + +### 12.7 无缓存一致性下的数据一致性分析（核心） + +> **本节定位**：shared 模式的**规范数据结构是 ring-per-bucket（§12.7.1）**，其一致性方案见 §12.7.1、 +> 容量与溢出见 §12.7.2。本节 §12.7 先分析"无硬件一致性平台上共享 map"的**通用难点**，并以最初设想 +> 的**"哈希桶 + 链表 + 空闲链"方案为对照基线**说明"为何不用链表"——这些难点（尤其空闲链 ABA 与 +> `next` 悬挂指针）正是 ring 设计要规避的。通用的无一致性纪律（release/acquire 发布、writeback + +> invalidate）对 ring 同样适用；ring 的具体落地见 §12.7.1。**链表方案不作为实现，仅作动机保留。** + +这是 shared 模式最困难的部分。**AI 核之间没有硬件缓存一致性**（§11.5 已就此为完成标志专门处理）。 +若把共享 TensorMap 实现成一块被多核并发读写的复杂数据结构（哈希桶 + 链表 + 空闲链），其一致性不能 +想当然——下面逐条剖析，正是这些陷阱促成了 §12.7.1 的 ring 决策。 + +**问题一：仅 invalidate 新插入条目的数据，足够吗？** + +不够。winner insert 一条新 entry 时，若只把自己写的这条 entry 的 cache line invalidate/flush 到 +GM，其它核仍可能基于**陈旧的桶 head 指针**导航——它们根本看不到新 entry 存在。一致性故障点至少 +有四处，而非一处： + +| 故障点 | 现象 | 仅 invalidate 新条目能解决吗 | +| ------ | ---- | ---- | +| (a) 桶 head 指针 | 其它核 cache 里仍是旧 head，永不到达新 entry | **否**——head 在另一条 cache line 上 | +| (b) 新 entry 的字段（producer id / region / next） | 其它核读到新 entry 但字段为旧值/撕裂 | 部分——需写回 + 读侧 invalidate | +| (c) 旧 entry 的字段 | 旧 entry 被 winner 保留不修改，但若被回收复用则字段被改写 | **否**——见问题二/三 | +| (d) 空闲链 / entry 池复用 | 一个被回收的 entry 被重新分配、改写，而某核仍持有旧指针在读它 | **否**——经典的 use-after-recycle | + +**问题二：其它核是否会使用"过时的 tensormap 数据结构"？** + +会，且有两种"过时"： + +1. **结构性过时（miss 新 head）**：落后核 cache 里的桶 head 是旧值，于是它遍历的是**旧链表前缀**， + 完全错过 winner 新挂的 head 条目。后果：lookup 漏掉最新 producer，退而取到次新的合法 producer + （§12.5 多版本链表使次新仍可用），**语义上仍正确**，但可能不是最新的过去版本——这在 INOUT + 场景下意味着消费了一个较旧版本的数据（见下文"数据正确性"）。 +2. **悬挂指针过时（use-after-recycle）**：落后核正遍历链表到 entry `e`，此时 reclaim 把 `e` 回收 + 并分配给另一个 winner 改写。落后核继续读 `e.next` / `e.producer_id`，读到**新写入者的内容**， + 指向完全无关的区域/任务 → 错误依赖，可能挂死或读错数据。 + +**问题三：数据正确性（不只是元数据）。** + +TensorMap 只是元数据；真正的产出数据在 GM 堆，由 §11.5 的完成标志 + writeback/invalidate 保证可见。 +§9.3 的**确定性 bump 分配**确保**每个 producer 写到自己的独立地址**（INOUT 的新版本也是新地址， +**非就地覆写**），因此消费者一旦选定 producer `P` 并 acquire 到 `flag(P)=true`，从 `addr(P)` 读到的 +必是 `P` 的产出，不会被未来 producer 覆写。所以 shared 模式下的**数据正确性仍由 §11.5 兜底**， +shared 模式新增的风险只在**元数据**层：选错了 producer（或读到回收后的垃圾 entry），会引用错误的 +`addr(P)` / 错误的完成标志位。 + +**（对照基线）链表方案的解决方案——仅说明其复杂度，非本设计实现。** 若坚持用"哈希桶 + 链表 + +空闲链"，需把它当作"无硬件一致性下的并发发布数据结构"来设计，沿用 §11.5 的发布/观察纪律并补齐 +回收纪律，至少需以下五条。**读者可略过细节，只需记住：其中第 3、4 条（reclaim 前沿驱动回收、空闲链 +无锁栈 + 版本防 ABA）是最易出错的部分——ring 设计（§12.7.1）通过取消空闲链与 `next` 指针，直接 +消去了它们。** 五条如下： + +1. **桶 head 为原子、acq-rel 发布。** `bucket_head` 用一个原子字（64 位指针 + 版本 tag，见下）。 + winner：先写回新 entry 的全部字段与 `next`（§11.5 writeback），再对 `bucket_head` 做 + **release-store**；reader：对 `bucket_head` 做 **acquire-load**（非一致平台先 invalidate 该 cache + line），拿到 head 后再 invalidate 对应 entry 的 cache line 读其字段。这解决问题一 (a)(b)。 +2. **entry 一经发布即不可变（immutable after publish）。** 一条 entry 被 head 指向、对其它核可见 + 后，其 `producer_id` / `region` / `next` **永不再被改写**。可变的只有 head 指针与 entry 在空闲链 + 中的 `freelist_next`（且二者用同一原子字的不同位/不同字段，发布期与空闲期互斥）。这把"读 entry + 字段"从并发读写降为并发只读，消除字段撕裂。 +3. **回收仅由 reclaim 前沿 `R` 驱动，且 `R` 基于各核进度最小值。** §9.5 已定义 `heap_reclaim_frontier` + 由"完成前沿 + 各核进度最小值"推导。shared 模式的 TensorMap 复用同一前沿：仅当某 entry 的 + `producer_id ≤ R` 时才允许回收。由"依赖跨度 `H` + 各核进度 ≥ 完成前沿"可知，任何核在 lookup 时 + 能引用的 producer id 下界 = `其 local_index − H` ≥ `R`（因为最慢核的 `local_index` 也已超过 + `R + H`，否则 `R` 不会推进到此），故**任何活着的 lookup 都不会触及已回收 entry**——问题二(2) 的 + use-after-recycle 在不变式下不可能发生。 +4. **空闲链为无锁 Treiber 栈 + 版本指针防 ABA。** 多 winner 并发 pop 空 entry、reclaim 并发 push， + 用 CAS + 指针带版本号（`head{ptr, tag}`）杜绝"同地址被多次回收再分配"造成的 ABA。这是 shared + 模式新增的唯一热点原子（除既有 cursor/F 外）；可仿 §6.6 按 `bucket_index % G` 分片以降竞争。 +5. **lookup 全程不缓存 head / entry 指针。** 每次进入 `tm_lookup` 都重新 acquire-load `bucket_head` + （invalidate 后读），遍历过程中对每条 entry 的字段读取都遵循 acquire/invalidate。**禁止跨调用 + 缓存 head 或 entry 指针**——结构性过时（问题二(1)）的根因正是缓存了旧 head；强制每次重读，让 + "最新 head"在 lookup 入口处对齐到当前前沿。 + +**关于"结构性过时取到次新版本"的最终判据。** 即便有上述全套方案，落后核在某一刻仍可能读到一个 +尚未被 winner 发布的最新 entry 之前的旧 head——但这等价于"winner 尚未发布该版本"。此时落后核 +取到的是**次新的合法 producer** `P_old < N`。由 §9.3（独立地址）+ §11.5（flag(P_old) 可见即数据 +可见），`P_old` 的输出是完整且正确的旧版本。**只要该消费者对"必须消费最新版本"没有强要求**， +这就是可接受的弱一致（最终一致）语义——落后核消费了一个稍旧的版本。若某任务的语义要求它必须 +消费"恰好最近的前任 producer"（典型如严格 in-place 累加序列），则需在任务图层面保证该前任已完成 +且其 entry 已发布——这由 §11.5 的 flag 依赖链天然保证：消费者在 acquire `flag(P)` 后才读数据， +而 `P` 的 entry 由 `P` 的 winner 在置 flag **之前**就已 insert 并 release-head 发布（insert 发生在 +build 阶段、flag 置位在 execute 完成 阶段，二者顺序固定）。因此"前任已发布 entry"是"前任已完成" +的必要前置，**不会出现"前任已完成但 entry 未发布"**。综上，shared 模式在上述五条方案下达成正确的 +元数据一致性，数据正确性沿用 §9.3 + §11.5。 + +**发布保证（回应 §12.6 末尾"winner 尚未发布"）。** 当落后核 lookup 返回 NONE 时，除"本核是该区域 +首任 producer / 外部输入"外，另一可能是"前任 winner 已认领但尚未 insert"。但 insert 发生在 winner +build 该任务的早期、远早于其 execute 完成与 flag 置位；落后核若需消费该前任，必先 acquire +`flag(P)`——而 `flag(P)` 置位晚于 `insert`。故"落后核看到 flag(P) 但看不到 entry"在 acq-rel 纪律下 +不可能。NONE 即真正无前任，安全。 + +### 12.7.1 规范数据结构：ring-per-bucket（两种模式统一的实现基准） + +> **本节是 private 与 shared 两种模式共用的规范实现。** §12.7 的五件套方案是**针对"桶内链表 + 空闲链" +> 这一（被否决的）数据结构**给出的。链表方案里**最容易出错**的不是桶 head 的 acq-rel，而是**空闲链**： +> 多 winner 并发 pop、reclaim 并发 push、指针 ABA、use-after-recycle——这些才是"处理一致性容易出错"的 +> 根源。因此两种模式**统一改用同一数据结构，让这些陷阱根本不出现**：**每桶一个有界环（ring-per-bucket）**。 +> +> **两模式共用同一 ring，仅并发纪律不同（§12.3.2）。** 下文的 `seq` / `reserve` / acq-rel / writeback / +> invalidate 是 **shared 模式**（并发单副本）所需；**private 模式**每核私有、单线程访问自己的副本， +> **退化为最简形态**：`head`/`tail` 是普通整数，insert = "写槽 + `tail++`"，回收阈值用确定性 `N−H` +> 取代全局 `R`，**无需 `seq`、无需 `reserve` 原子、无需 invalidate**。即 private = "把下文所有并发纪律 +> 关掉"的 ring。 + +**思路。** TensorMap 的访问模式恰好是"**增量追加（append）+ 按前沿回收（evict from front）**"： +winner 不断往一个 bucket 里 insert 新条目，旧条目随 reclaim 前沿 `R` 推进被回收。这正是 **ring +（有界环形缓冲）** 的天然工作模式——**每个 bucket 一个 ring**，只有两个游标： + +```text +struct RingBucket { + Entry slots[CAP]; // 定长槽数组 + atom head; // 回收游标：slots[head % CAP] 是最旧仍存活条目 + atom tail; // 发布游标：下一个 append 落在 slots[tail % CAP] + atom reserve; // 预定游标：MPSC 下 winner 用 fetch_add 抢槽位 + // 每个 slot 内含一个 seq 字（见下） +}; +``` + +- **append（winner insert）**：`k = fetch_add(reserve, 1)` → 写 `slots[k % CAP]` 的字段 → + writeback → release-store 该 slot 的 `seq`（标记"本 lap 已发布"）。tail 由"已连续填充前缀"推进 + （或等价地，reader 直接用 per-slot `seq` 判定有效性，无需单一 tail）。 +- **evict（reclaim）**：当 `slots[head % CAP].producer_id ≤ R` 时 `head++`。回收 = **游标自增**， + 不摘链、不归还空闲池。 +- **lookup**：从 `reserve`（或 tail）往 `head` 方向扫 `slots[k % CAP]`，对每个 slot 先 acquire-read + 其 `seq` 确认有效，再读字段，应用 §12.6 时序过滤与重叠判定，取合法者中 producer id 最大。 + +**为什么这能避开"最容易出错的部分"。** + +| §12.7 链表方案的陷阱 | ring 方案的处置 | +| ---- | ---- | +| **空闲链 CAS 栈（ABA / use-after-recycle / 竞争）** | **彻底消失**——没有空闲链，"回收"只是 `head++`，"分配"只是 `fetch_add(reserve)` 抢一个确定下标 | +| **`next` 指针的跨核读（每跳一条远程 cache line，且 next 本身可能被回收）** | **彻底消失**——槽是连续数组，下标由 `k % CAP` 算出，无指针跳转；扫描局部性好，invalidate 目标地址确定 | +| **回收时"摘链"可能摘掉某核正持有的节点** | **不可能**——回收只动 `head` 游标，不动任何槽内容；被回收的槽在被 `reserve` 再次追上之前不会被改写 | +| **桶 head 指针的发布/观察** | 改为 **per-slot `seq` 字**（见下），把"head 可见性"问题局部化到"单个 slot 的发布可见性"，模式更标准、更易推理 | + +**新增的、但更标准的要求。** + +1. **per-slot `seq` 防 ABA（关键）。** ring 是定长的，slot `k % CAP` 会被反复复用。若读者刚读完 lap `L` + 的 slot `k`、被挂起，此时槽被回收并写入了 lap `L+1` 的新条目，读者醒来若只凭"slot `k` 有数据"就会 + 把 lap `L+1` 的内容当成 lap `L` 的来用——经典 ABA。解法是 bounded-queue 标准技巧：每个 slot 带 + `seq`，发布时写 `seq = k + L*CAP`（每复用一次 `+CAP`），读者记下自己期望的 `seq` 值，acquire-read + `seq` **等于**期望值才认为该槽有效。lap 切换后 `seq` 不等 → 读者识别为"槽已被复用，停止扫描" + （因为它的目标旧条目已不可达）。这把 ABA 从"指针级、需带版本号的 Treiber 栈"降为"整数比较"，简单 + 且可局部推理。 +2. **定长容量 `CAP` 须按 H 窗口定。** 每个 bucket 的存活条目数上界 = "落进该桶、producer id 在 + `(R, 最新]` 内的不同版本数"，受依赖跨度 `H`（§11.4）封顶。取 `CAP ≈ (H × 桶均条目数) × 安全系数` + 即可。**溢出**时 winner **不覆写**、走反压/报错路径。这比链表的"无界增长 + 空闲链"更省内存、更可 + 预测，代价是需要正确估 `CAP`。**`CAP` 的完整取值分析、溢出报错设计与是否引入命令行参数见 §12.7.2。** +3. **MPSC 预定游标 `reserve` 是新的热点原子。** 同一桶的多个 winner 用 `fetch_add(reserve)` 抢槽， + 是 per-bucket 的 CAS 热点（类似 §6.5 的 cursor）。缓解：`reserve` 与 `head` 落在同一 cache line 会 + 伪共享，需分开对齐；热桶可按 §6.6 思路分片（同一 bucket 拆 `G` 个 sub-ring，按 producer id 取模）。 + +**能否连 `seq` 也省掉？** 严格条件下可以。若 (i) AI 核**无 OS 抢占**（lookup 在有界本地时间内完成， +不会"读到一半被挂起很久"）且 (ii) reclaim 不变式严格成立——"slot 被回收复用"要求其 +`producer_id ≤ R`，而任何 lookup 能引用的 producer id `> R`（由 `R = min_progress − H − 1` 与读者 +`local_index ≥ min_progress` 推出，§12.7 不变式）——则读者**永远不会**触及一个正被复用的 slot， +`seq` 可省。但**仿真在 host 线程上跑、会被抢占**，且 defense-in-depth 更稳，故**推荐保留 `seq`**； +`seq` 成本仅每槽一个整数 + 一次 acquire 比较，远低于链表的空闲链 CAS。 + +**结论。** 把 bucket 从链表换成 ring，**消掉了 shared 模式里最易错的空闲链与 next 指针**，把一致性 +问题收敛到"per-slot acq-rel 发布 + `seq` 防 ABA + `head` 由 `R` 推进"这一套**有界队列标准模式**， +推理局部、实现成熟。回收从"摘链 + 归还空闲池"简化为"游标自增"，append 从"建节点 + CAS 挂头"简化 +为"fetch_add 抢槽 + 写 slot + 发布 seq"。这正是用**数据结构的简化**换**一致性论证的简化**——代价是 +定长 `CAP` 与新的 `reserve` 热点，二者都可调/可分片。**后续 §12.8/§12.9 的 shared 实现默认采用 +ring-per-bucket。** + +### 12.7.2 Ring 容量 `CAP`、溢出报错与命令行参数（两种模式） + +ring 是**定长**的，这把"map 无界增长"换成了"必须正确定容"。**统一到 ring 后，private 与 shared 都需 +定容**，但二者的活跃窗口不同（private 窄、shared 宽），且 private 的窗口是**确定性静态可算**的。本节 +回答三个工程问题：(1) 两模式 `CAP` 各取多大；(2) 满环（溢出）如何检测与报错，绝不静默丢条目或覆写； +(3) 是否/如何把 `CAP` 暴露为命令行参数、**`auto` 是否安全、如何实现**（§12.7.2.3，本章重点之一）。 + +#### 12.7.2.1 `CAP` 如何定：活跃版本窗口 + 安全系数 + +单个 bucket 在任意时刻的**存活槽数**（`tail − head`，shared 下为 `reserve − head`）有明确上界，可据此 +定容。**两模式的窗口不同：** + +- **private 的存活窗口 = `H`（更窄、确定性）。** private 每核用确定性阈值 `N−H` 回收自己的副本：走位到 + `N` 时，`producer_id ≤ N−H` 的槽已被回收，故存活槽的 producer id 落在 `(N−H, N]`，**窗口恰为 `H`**。 + 它**不含** run-ahead `Δ`——因为每核只对自己的单一进度 `N` 回收，无跨核进度差。 +- **shared 的存活窗口 = `Δ + H`（更宽）。** shared 用全局前沿 `R = min_progress − H − 1` 回收共享副本 + （§12.7.1）。最快核可领先最慢核达 `Δ`（run-ahead 上界，§11.1），故存活槽 producer id 落在 + `(R, 最新 append]`，跨度 = `最新 − R ≤ Δ + H`。 +- **落进单桶的比例来自哈希。** 全局活跃版本总数 ≤ `窗口 × 每任务平均输出条目数`（private 窗口=`H`、 + shared 窗口=`Δ+H`）；按 `B` 个桶哈希，**单桶期望存活槽 ≈ 全局活跃版本 / `B`**。哈希非理想均匀，需 + 留倾斜裕度（`auto` 如何在构建期精确取代"倾斜裕度估计"见 §12.7.2.3）。 +- **取值公式（建议下界）**： + +```text +# W = H (private) 或 Δ+H (shared) +CAP = ceil_pow2( W * avg_outputs_per_task / B * skew_factor ) +``` + + 其中 `skew_factor`（经验 2~4）覆盖哈希不均与 INOUT 多版本堆积；`ceil_pow2` 向上取到 2 的幂，使 + `k % CAP` 退化为位与、`seq = k + lap*CAP` 的 lap 递增用移位。`Δ`、`H`、`B` 均已是 §11 的现有常量， + `CAP` 与 `W`（完成标志环窗口，§11.3）**同源**，可一并标定。 +- **估偏的代价（非对称）**：估**大**只浪费 GM（每桶多几个槽 × `B` 桶，线性且可控）；估**小**在 shared 下 + 会在热桶频繁触发反压 stall，甚至（若窗口真的不足）**死锁**——因为 winner 等 `head` 前进、而 `head` + 前进又依赖更慢核推进 `R`。**private 下估小更严重**：private 回收已用最紧的确定性 `N−H`、无更慢核可 + 等，故一旦某桶在 `H` 窗口内溢出即为**真正的配置错误**，无法靠等待自解，必须直接报错（§12.7.2.2）。 + 故**宁可略微估大**——好在 `auto` 能在构建期把两模式的窗口占用**精确算出**、不必"估"（§12.7.2.3）。 + +#### 12.7.2.2 溢出检测与报错设计（绝不静默覆写） + +ring 满的语义必须是**显式失败或可恢复反压**，不能像无界链表那样"总能再挂一个"。**shared** 分两类 +处置（A/B）；**private** 无跨核等待余地，溢出直接走 B 的确定性报错（见 B 末）。 + +**A. 可恢复反压（shared 默认，热路径）。** winner 在 `k = fetch_add(reserve, 1)` 后、写槽**之前**先检查 +`k − load_acquire(head) >= CAP`。若成立即表示环满： + +1. **不写槽、不发布 seq**，并把 `reserve` 回退（`fetch_sub(reserve, 1)`，或采用"先探测后提交"两段式 + 抢槽以避免回退竞争）； +2. 进入**有界自旋 + backoff**，周期性推进本核可推进的 `F`/`R`（§11.4 协作式回收），给 `head` 前进创造 + 条件； +3. `head` 前进后重试 `insert`。这与 §11 的**堆反压语义一致**（满则等待，不丢数据）。 + +**B. 不可恢复 → 结构化报错（诊断路径）。** 若反压自旋超过阈值 `T_stall`（如按最坏依赖链估算的上界 +的数倍）仍无法推进，判定为**容量配置错误或依赖跨度估计错误**，触发一次**确定性、可定位**的运行时 +错误，而非挂死或 UB： + +```text +FATAL[tensormap-ring-overflow] + bucket = # 哪个桶溢出 + cap = # 当前容量 + live = reserve - head # 溢出时的存活槽数 + head/R = / # 回收游标与全局 reclaim 前沿 + slowest = core @ local_index # 拖住 R 的最慢核（定位反压根因） + hint = "raise --tensormap-ring-cap or check H/Δ estimate; \ + possible deadlock if a producer never completes" +``` + + 报错要点：**(i)** 指明**是哪个桶**、当前 `CAP`、溢出时 `live` 值，便于直接调参；**(ii)** 打印**最慢核** + 及其 `local_index`，区分"真溢出（窗口不足）"与"某 producer 卡死导致 `R` 不前进"（后者是别处 bug， + ring 只是最先撞墙的地方）；**(iii)** 给出可操作建议（调大 `--tensormap-ring-cap` 或复核 `H`/`Δ`）。 + 该错误应是**确定性**的（同输入必在同一 bucket 触发），便于复现与回归。 + + **private 的溢出更简单直接**：private 无 `reserve`/`R`，也无跨核可等；`tail − head` 触到 `CAP` 即 + 刻判定为配置错误，**立即**抛同款 FATAL（`slowest`/`R` 字段留空，`head` 用本核 `N−H`）。但在 `auto` + 下 private **可证不溢出**（§12.7.2.3），此路仅在用户手动把 `--tensormap-ring-cap` 设得过小时触发。 + +**C. 调试增益（可选）。** debug build 下额外维护每桶 `high_watermark = max(reserve − head)`，运行 +结束打印各桶水位分布，用于**离线标定 `CAP`**：水位远低于 `CAP` 说明可调小省内存，逼近 `CAP` 说明 +需调大或该桶是热点（考虑 §12.7.1 的 sub-ring 分片）。 + +#### 12.7.2.3 `auto` 模式是否安全、如何实现（本章重点） + +统一到 ring 后，定容的安全性是新引入的**唯一**风险（链表无"满"概念）。核心问题：**默认的 `auto` 定容 +安全吗？** 结论： + +> **`auto` 在 private 下可证 100% 安全（永不溢出）；在 shared 下在"按最坏单桶占用定容"时同样可证不 +> 溢出，退一步即便估紧也绝不静默损坏（溢出→反压/确定性 FATAL）。** 关键在于：**决定 `CAP` 的两个量 +> ——(1) 每桶落入的静态区域集合、(2) 回收窗口宽度——都在构建期已知或有硬上界**，因此 `auto` 不是 +> "猜"，而是**在构建期精确计算**。 + +**为何安全：两个决定量都是已知/有界的。** + +1. **区域集合是静态的。** SPMD 下每个核 replay **同一条确定性 submit 流**（§2/§6.4），全部任务的输出 + region 及其 `bucket_of(region)` 哈希**在构建期即完全确定**。因此"哪些 producer 落进哪个桶"不是运行 + 期随机量，而是可枚举的静态事实——哈希倾斜**不需要"估"**，可直接数出每桶的真实占用。 +2. **回收窗口有硬上界。** private 窗口 = `H`（确定性 `N−H`）；shared 窗口 = `Δ+H`，其中 `Δ` 是私有环 + run-ahead 的**配置上界**（§11.1）、`H` 是依赖跨度**契约上界**（§11.4）。二者都不是无界运行期量。 + +两点合起来：**每桶存活槽数的最大值 = "在任意长度为 `W` 的 producer-id 滑动窗口内、哈希到该桶的输出 +region 条数"的最大值**（`W=H` 或 `Δ+H`）。这是一个**可在构建期精确算出**的确定值，不含任何运行期 +不确定性（private 完全确定；shared 唯一的运行期量是进度差，而它被 `Δ` 硬封顶）。 + +**`auto` 的实现（构建期精确定容，非启发式）。** + +```text +compute_auto_cap(task_graph, mode): + W = H if mode == Private # 确定性窗口 + (Δ + H) if mode == Shared # Δ 为 run-ahead 硬上界 + per_bucket_max = array[B] of 0 + live = sliding_multiset() # 以 producer-id 为键的滑动窗口 + for N in 0 .. num_tasks-1: # 按确定性 submit 顺序扫全图 + for r in outputs(task[N]): + b = bucket_of(r) + live.add(b, producer_id=N) + live.evict(producer_id <= N - W) # 精确模拟 §12.7.1 的 head 回收 + for b in 0 .. B-1: + per_bucket_max[b] = max(per_bucket_max[b], live.count(b)) + cap = ceil_pow2( max_over_b(per_bucket_max[b]) * safety ) # safety ∈ {1(可证), 小裕度} + return cap +``` + +- 该过程**只依赖静态任务图**（host build-once 或各核构建期均可跑），复杂度 O(任务数 × 每任务输出数)， + 一次性、与执行无关。它**精确复刻** §12.7.1 的 `head` 回收语义（private 按 `N−W`、shared 按窗口 + `Δ+H`），因此算出的 `per_bucket_max` 就是运行期真实峰值的**上确界**。 +- **private**：`safety = 1` 即已**可证不溢出**（窗口确定、无进度差、无并发追加）——`auto` 给出的 `CAP` + 就是精确峰值。这就是"private 下 `auto` 100% 安全"的证明。 +- **shared**：以 `Δ` 硬上界代入窗口后，`per_bucket_max` 是**最坏进度差下**的峰值上界；取 `safety = 1` + 即得**可证不溢出**的 `CAP`（代价是按最坏 `Δ` 偏保守、略费内存）。若要更省内存，可取更小的有效 + `Δ_eff < Δ`（按实测/期望进度差）作为"紧档"，此时不再可证、但 §12.7.2.2 的反压 + FATAL 兜底保证 + **绝不静默损坏**。默认 `auto` 采用**可证档（`safety=1`、`Δ` 满值）**，安全优先。 +- **全局单值 vs 每桶**：`auto` 天然算出**每桶**峰值，可直接支持"每桶独立 `CAP`"（最省内存）；一期为 + 实现简洁可取 `CAP = max_over_b`（全局单值），后续再切每桶。 + +**是否需要命令行参数——需要，但默认 `auto`。** + +- **为何仍保留 `--tensormap-ring-cap`**：`auto` 依赖 `Δ`/`H` 的取值正确；若用户想**收紧内存**（接受 shared + 下的紧档风险）或**排障时放大**容量，需一个**免重编译**的覆盖出口。与 §11 把 `W`/`Δ` 做成可配置一脉相承。 +- **为何默认 `auto` 而非必填**：`auto` 既开箱即用又（private 可证 / shared 可证档）安全，强制用户填值 + 徒增负担且易填错。 +- **两模式均生效**（统一 ring 后 private 也是 ring）：取值**向上取 2 的幂**；设定值 **< `auto` 算出的 + 可证峰值**时，private **启动期直接拒绝并报错**（因其必然溢出、无法自解），shared 则**告警并允许** + （用户显式选择紧档，运行期由 §12.7.2.2 兜底）。 + +> 一句话：`auto` **不是估，是构建期按静态图 + 硬上界窗口精确算**——**private 可证永不溢出**，**shared +> 取可证档同样不溢出**、紧档也绝不静默损坏；命令行 `--tensormap-ring-cap`（默认 `auto`，两模式生效） +> 仅作收紧内存/排障的覆盖出口。 + +### 12.7.3 run-ahead 上界 `Δ_max`：负载均衡旋钮（两种模式）+ 平台默认值 + +**动机。** 认领用**单调 `fetch_max` 全局游标**（§11.1）——谁先 replay 到任务 `N` 谁就 win。若各核推进速率 +不均（真机上偶发的慢核；仿真里 host 过订阅导致的调度倾斜），跑得快的核会把游标一路推到前沿，**抢走一长段 +连续 id**，把落后的核饿死（落后核 replay 到那些 id 时游标已越过 → 只能 skip、领不到活）。§6 的 execute-first +只是**软**减速；要**硬**封顶就需要一个 run-ahead 上界。 + +**机制（`dist_runahead_throttle`，两模式通用）。** 每核在推进到任务 `N` 前，先把自己的 replay 走位发布到 +`gd->core_progress[core]`，然后**等待**直到 `N − min(core_progress) ≤ Δ_max`（`min` 取全体核的最慢走位）。 +等待是**协作式**的：每圈调用 `drain_block_won()` + `drain_phase_b()` 清偿本核欠下的完成事件，仅在无就绪任务时 +`SPIN_WAIT_HINT`（仿真上即 `sched_yield`）。**无死锁**：最慢核的 `N − min == 0` 永不被节流 → 持续推进 → 抬升 +`min` → 释放所有超前核。**它只改变"谁执行"，绝不改变确定性 replay / 依赖图**——`PTO_DIST_DEPSIG` 实测跨 `Δ_max` +取值、跨 private/shared **逐位一致**（a5 `Balanced9` `sig=33b150b1…` edges=270；a2a3 `Case0` `sig=8a877fd1…` +edges=750）。与 §12.7.2 shared 环的追加反压同源（后者另在 `tm_shared_claim_append` 里按 ring `cap` 限追加前沿）。 + +**平台默认值（按核数派生 ⇒ 每平台自动合理）。** `gd->runahead_max = 2 × num_workers`： +`num_workers` 在 a2/a3 为 24 AIC + 48 AIV = 72（默认 144），在 a5 为 36 AIC + 72 AIV = 108（默认 216）。 +**下界约束 `Δ_max ≥ num_workers`**：否则窗口装不下"每核一个在飞任务"，健康并行会被节流成**空闲核**（实测 +`Δ_max=8 ≪ 108` 时 shared 出现 10–24 个空闲核、makespan 翻倍）；`2×` 给流水线留裕度，又能把失控核封在 +"领先 2×核数"以内。`PTO_DIST_RUNAHEAD=N` 覆盖，`0` 关闭节流。 + +**均衡验证（a5sim，`PTO_DIST_FAKE_EXEC_NS` 给每个 kernel 等长耗时）。** +- **非过订阅（`Balanced9`，9 worker ≈ 8 物理核）＋默认 `Δ_max=18`**：GEMM CV **2.4%**、`max/mean` **1.02×**、 + **0 空闲核**；ADD CV 15%、`max/mean` 1.20×——引擎+旋钮+默认值在忠实并行下**均衡极佳**。 +- **满核过订阅（`FullCore36`，108 线程 / 8 物理核）**：任何 `Δ_max` 都无法均衡。根因是**过订阅 + 单调游标**： + 被 OS 调度到的线程抢光窗口内全部认领并推高游标，`sched_yield` 不能可靠把 CPU 让给被饿死的 107 个线程；落后 + 线程追上时只能 skip。这是**仿真伪影**（host 只有 8 个真并行核），**非引擎缺陷**——真机上 108 个核真并行、 + 速率近似，`Δ_max` 只在偶发慢核时兜底。故满核仿真泳道的不均衡应按过订阅解读，均衡结论以 `Balanced9` 这类 + ≈1:1 配置为准。 + +### 12.8 两种模式的性能分析（均为 ring-per-bucket） + +> 两种模式**数据结构相同（ring-per-bucket）**，下表比较的是"每核私有 ring"与"全局共享 ring"这两种 +> 用法在内存/insert/同步上的取舍（§12.3.2 的四点差异所致），**不是**链表 vs ring（后者见 §12.3.1）。 + +| 维度 | private（每核私有 ring） | shared（全局共享 ring） | +| ---- | ---- | ---- | +| **map 内存占用** | `O(核数 × 单份 ring)` | `O(单份 ring)` —— **省 `核数` 倍** | +| **每任务 insert 工作量** | **每核都 insert 全部任务**（SPMD 冗余，§6.4 的"地板"） | **仅 winner insert**（每任务恰好一核） | +| **每任务 lookup 工作量** | 本地 ring 连续扫描，**零跨核、无 `seq` 校验**；winner-only | 共享 ring，需 invalidate/acquire per-slot `seq` + 字段；同为连续数组扫描 | +| **跨核同步/原子** | **无**（`head`/`tail` 为普通整数） | per-bucket `reserve`/`head` 原子 + per-slot `seq` acq-rel（可分片） | +| **缓存一致性流量** | 无 | 随核数增长（更多 reader invalidate slot；热桶 `reserve` 竞争） | +| **ABA / 回收风险** | 无（单线程，回收 = `head++`） | ring 复用 ABA 由 per-slot `seq` 化解；回收 = `head++`，无 use-after-recycle | +| **`CAP` 定容窗口** | `H`（更窄），`auto` **可证不溢出**（§12.7.2.3） | `Δ+H`（更宽），`auto` 可证档不溢出 | +| **确定性 / golden 不变** | 各核 ring 内容严格一致 | map 内容由 winner 发布顺序决定，**弱确定**（最终一致） | +| **随核数 scale 的编排地板** | §6.2/§6.3 实测：随核数近线性上升（SPMD 重放 + 每 insert 地板） | 去掉每 insert 地板 → **有望显著 flattening** §6.2 曲线 | + +**定性结论。** + +- **shared 的主胜场**：在**多核**（大 `block_dim`）场景下，把"每核为全部任务 insert"的 SPMD 地板 + 砍成"每任务仅一核 insert"，并把 map 内存从 `核数×` 降到 `1×`。这恰好对症 §6.2/§6.3 暴露的 + "编排墙钟随核数近线性增长"与 §6.5 的 cursor CAS 竞争之外的另一条地板——理论上 shared 模式可把 + §6.2 的 1→13 约 2× 的增长曲线明显压平（insert 工作总量从 `核数 × 任务数` 降到 `任务数`）。 +- **shared 的主代价**：lookup 引入跨核 cache line 读 + invalidate 流量，且热桶 `reserve` 成为新的 + 原子热点（类似 §6.5 的 cursor）。在**少核**或桶内存活槽多（需扫多条远程 line）时，lookup 延迟可能 + 吃掉 insert 省下的红利。 +- **private 的主胜场**：**少核**或**内存充裕**场景下，零跨核协调、零 ABA、严格确定、lookup 纯本地 + 连续扫描（无 `seq` 校验、无 invalidate）。与 §6.4 的 O(N) 回收窗口、winner-only fan-in、§6.6 cursor + 分片等优化完全兼容（这些语义已在 ring 上继承）。 +- **建议**：`private` 作为安全默认；`shared` 作为**大核数 / 大任务图**下的可选加速档，需在目标平台 + 实测 §12.7 的 invalidate 流量与热桶 `reserve` 竞争是否可接受。二者正交于 cursor 分片（§6.6）、 + winner-only fan-in（§6.4）等其它优化，可叠加。 + +> 一句话：**两模式同用 ring-per-bucket**（§12.7.1），差异仅在"每核私有 / 全局共享"（§12.3.2）； +> **少核求快用 private**（零跨核、纯整数游标、`auto` 可证不溢出），**多核求省用 shared**（内存 `1×`、 +> insert 仅 winner，正确性靠"per-slot `seq` acq-rel + 发布即不可变 + `R` 驱动 `head++` + lookup 不缓存 +> 游标"，数据正确性仍由 §9.3 独立地址 + §11.5 完成标志兜底）。 + +#### 12.8.1 实测 overhead（TensorMap 操作计数，确定性、跨平台） + +墙钟对比在仿真主机上不可用（`device_wall_us` 对单次冷跑报 0；且线程数 > 物理核时所有核忙等自旋， +makespan 被 OS 时间片放大到 ~33 ms/task，两模式**同等**放大，把编排逻辑差异完全淹没——实测 6 核 +private 67.41 s vs shared 67.43 s，差 <0.03%，无法区分）。因此改用**确定性的 TensorMap 操作计数** +（env `PTO_DIST_OVERHEAD=1`，引擎在 DONE 打印 `[dist] TMOPS ...`；不依赖时钟、不受 SIGBUS/超额订阅 +影响）作为 overhead 度量。BGEMM `Case0`（72 核 / 500 matmul-add / D=1000 个不同输出 region）实测： + +| 指标 | private | shared | 说明 | +| ---- | ---- | ---- | ---- | +| **inserts**（map 写入次数） | **72000** | **1000** | private = `核数 × D`（每核为全部任务 insert，SPMD 地板）；shared = `D`（每 region 仅首达核 append 一次）→ **shared 少 `核数`(72)× 写入**，且内存 `1×` vs `72×` | +| **lookups**（fan-in 解析次数） | 3000 | 3000 | 两模式相同（均 winner-only 解析同一批边，§6.4） | +| **scans**（lookup 扫描的槽数） | 9332 | 43689 | shared **多 ~4.7×**：全局环在 `Δ+H` 窗口内堆积**所有核**的 append（比 private 每副本的 `H` 窗口更深），且**每扫一槽多付一次 `seq` 原子 acquire + 跨核读**——这是 shared 的并发税 | + +**交叉验证（`runtime_overhead_test`，3 核 / tasks=100 / skip-exec）** —— 换一条完全不同的编排流、 +核数从 72 降到 3，`insert` 缩减比仍**精确等于核数**，佐证该规律是结构性的、与具体模型无关： + +| 指标 | private | shared | 比值 | 说明 | +| ---- | ---- | ---- | ---- | ---- | +| **inserts** | 600 | **200** | **3× = 核数** | 与 72 核例同构：private=`核数×D`，shared=`D` | +| **lookups** | 600 | 600 | 1× | 完全一致 | +| **scans** | 1581 | 1906 | ~1.2× | 该流 `Δ+H` 窗口浅，并发税小于 72 核例的 4.7× | + +**结论（量化 §12.8 的定性判断）：** +- **shared 的胜场 = insert**：写入次数从 `核数×D` 砍到 `D`（此例 **72→1**，即 72× 减少），且随核数**线性**扩大；map 内存同步从 `72×` 降到 `1×`。这正是压平 §6.2 "编排墙钟随核数近线性增长"的那条地板。 +- **shared 的代价 = lookup 扫得更深**：扫描槽数 ~4.7×。根因是**核间进度 skew**——private 每核一份副本、各自只装本核 `[N−H, N)` 的 `H` 深窗口；shared 是单一全局环，必须同时覆盖**最慢核的尾**到**最快核的头**，即 `Δ+H` 深（`Δ` = 快慢核回放领先量，§12.7.1）。多出的深度就是被 coalesce 进同一个环的核间 skew。 +- **但这个扩大是有界的，不会无限增长**：`Δ` 被 run-ahead 节流（§12.7.2）按 `Δ_max` 封顶，故 shared 窗口恒 ≤ `Δ_max + H`，与核数**无关**；而 insert 红利随核数**线性**增长。两者一减一增 ⇒ **核数越多，shared 越划算**：insert 节省无上限地随核数放大，scan 深度却被 `Δ_max` 钉死。极端地令 `Δ_max→0`（lockstep）时 shared 窗口退化为 `H`、scan 与 private 完全一致，代价是牺牲 run-ahead 并行度——所以深度是"用并行度换来的可调量"，而非失控项。 +- **单次 scan 的原子代价已摊薄至接近 private**：曾经每扫一槽一次 `seq` acquire，现已优化为**每次 lookup 仅一次 acquire**（快照 tail 后对其下所有槽用 relaxed 读，正确性依据见 §12.7.1）。残留的**跨核 cache line coherence 读**不可消除——那是"单份共享"换取"省 N× 写入 + N× 内存"的固有对价（private 靠复制 N 份让读永远 L1 命中）。 +- **权衡**：故 **大核数 / insert 密集 → shared 明确胜出（如 72 核）**；**少核 / lookup 密集 / 内存充裕 → private**。此结论与墙钟无关，纯由确定性操作计数得出。 + +**为什么不用墙钟 us/task（macOS 仿真的深入排查记录）。** 曾尝试用 `[dist] OVERHEAD` 的 +`makespan_us / tasks` 得到 us/task，结果稳定在 ~33–67 ms/task（比历史 0.5–5 µs/task 大 4 个数量级）， +排查结论如下，供后来者免于重复踩坑： + +1. **`SPIN_WAIT_HINT` 本已让核**：仿真构建通过 `common/platform/sim/aicpu/spin_hint.h` 展开为 + `yield + sched_yield()`（并非 no-op）。追加 macOS 专用 `nanosleep` 强制让核后重测，us/task **完全不变** + → 放大**不是**忙等自旋造成的超额订阅。 +2. **replay 阶段本身就 ~13 s**：新增 `[dist] OVERHEAD ... replay_us[min/avg/max]`（仅 `orch_func` + 回放耗时，排除 drain 循环）后发现 `replay_us ≈ busy_us ≈ makespan`，且**最快核**的 replay 也有 12.8 s。 + 即跨核等待发生在**回放内部**的堆回收 / 完成前沿（frontier）背压——每次跨核交接被 OS 调度按 + ~ms 量级计时，任务再多也是**线性**累加，故与任务数成正比（tasks=100→13.5 s，tasks=500→67 s， + 恒为 ~67 ms/task），并非固定超时。TMOPS 全程仅 ~1.6 k 次 scan，证实这 13 s 与 TensorMap 计算无关。 +3. **口径不同**：历史 0.5–5 µs/task 来自**设备周期剖析**（`PTO2_ORCH_PROFILING` 的 `avg/task`，计编排 + 代码消耗的设备周期），与主机墙钟不是同一度量；在 3 核 macOS 仿真上无法用墙钟复现到 µs。 + +因此 **overhead 对比以上表 TMOPS 为准**（确定、跨平台、可复现）。若确需可比的 µs：(a) 在 Linux 多核机上 +重跑墙钟（`sched_yield` 在 Linux 真让核、跨核交接是微秒级），或 (b) 用设备周期剖析构建并把周期计数埋进 +`dist_engine.cpp` 编排热路径（排除自旋段）。`replay_us` 字段保留在 `PTO_DIST_OVERHEAD=1` 输出中，供在 +低交接延迟平台上直接得到 replay/task。 + +### 12.9 顶层入口：TensorMap 指针的双模化设计 + +为支持两种模式共存于同一份代码、由命令行开关选定，runtime 的**顶层 TensorMap 入口**需被改造为 +一个**间接句柄**，而非硬编码的"每核 array"。 + +**现状（private 硬编码）。** 编排循环里直接持有"每核私有 map"（原为链表 `DistTensorMap` 实例， +§6.4；本章统一后为每核私有 ring），`tm_insert` / `tm_lookup` 直接作用于它。这把"每核私有"写死在入口处。 + +**改造。** 引入一个抽象句柄 `TensorMapHandle`，封装**同一 ring 数据结构的两种用法**并提供统一 API： + +```cpp +enum class TensorMapMode { Private, Shared }; + +// 两模式共用同一 ring 结构（§12.7.1）；仅并发纪律 / 副本数不同（§12.3.2） +struct RingTensorMap; // ring-per-bucket；Shared 用 seq/reserve/acq-rel，Private 关闭并发纪律 + +struct TensorMapHandle { + TensorMapMode mode; + RingTensorMap* map; // private：指向本核私有 ring（每核各一份，单线程纪律） + // shared ：指向全局唯一 ring（并发纪律 + per-slot seq，§12.7.1） +}; + +// 统一入口，由 mode 分发（Private 走无原子快路径，Shared 走并发路径）： +void tm_insert (TensorMapHandle& h, const TensorRegion& r, task_id_t producer); +Entry* tm_lookup (TensorMapHandle& h, const TensorRegion& r, task_id_t local_index); +``` + +**初始化（按开关二选一，同一结构不同参数）。** + +```text +init(runtime_args): + mode = parse_tensormap_mode(args.tensormap_mode) # private | shared + cap = resolve_ring_cap(args.tensormap_ring_cap, mode) # auto → compute_auto_cap(graph, mode) (§12.7.2.3) + if mode == Private: + # 每核构造一份私有 ring（单线程纪律：head/tail 普通整数，无 seq/reserve） + for each core c: c.tensormap = { mode:Private, map: new RingTensorMap(cap, concurrent=false) } + else: # Shared + # 全局构造唯一一份共享 ring（并发纪律：seq + reserve + acq-rel） + shared = new RingTensorMap(cap, concurrent=true, pool=global_pool) + for each core c: c.tensormap = { mode:Shared, map: shared } +``` + +**调用点改动（§6 伪代码中的 `update_tensormap(task)`）。** 唯一变化是 insert 改为**仅 winner 调用** +（shared 模式下），lookup 仍由 winner 在解析 fan-in 时调用。这通过在 `update_tensormap` 内部按 +`mode` 分发实现，**上层 §6 循环伪代码不动**： + +```text +update_tensormap(task, won, N, mode): + if mode == Private: + # 无条件 insert（胜者、败者、follower 都做），保持各核 ring 副本一致 + for t in task.inputs: tm_lookup (h, t, N) # fan-in（winner-only 仍由调用方门控） + for t in task.outputs: tm_insert (h, t, N) # 私有 ring：写槽 + tail++（无原子/seq） + else: # Shared + # 仅 winner 做 insert；lookup 仍为 winner 专属 + if won: + for t in task.inputs: tm_lookup (h, t, N) # 内含 §12.6 时序过滤 + §12.7.1 per-slot seq 校验 + for t in task.outputs: tm_insert(h, t, N) # 内含 §12.5 ring 追加 + §12.7.1 fetch_add 抢槽 + seq 发布 +``` + +> 注意 `tm_insert`/`tm_lookup` 在 shared 用法里隐含 §12.7.1 的 writeback/invalidate、per-slot `seq` +> acq-rel 与 `reserve`/`head` 游标纪律，对上层透明；private 用法走**同一 ring 的无原子快路径**（普通 +> 整数 `head`/`tail`、纯本地连续扫描）。两条路径共享 ring 的桶/槽/哈希/多版本逻辑，仅并发纪律有别， +> 不互相污染热路径。 + +**回收侧的双模（同一 ring、不同阈值）。** private 每核按确定性阈值 `N − H` 推进自己的 `head`（继承 +§6.4 的回收窗口语义，改为游标自增）；shared 用全局 reclaim 前沿 `R`（§9.5）驱动——仅当 +`slots[head % CAP].producer_id ≤ R` 时 `head++`。两者回收都是**游标自增、不归还任何空闲池**（§12.7.1）； +`R` 由 §11.4 协作式 `F` 推进后派生，可由任一核在推进 `F` 时顺带推进各 bucket 的 `head`（按 bucket +独立、低频，竞争小）。 + +如此，**命令行开关只决定 `TensorMapHandle` 的初始化参数（副本数 / 是否开并发纪律 / `CAP`）与 +`update_tensormap` 的分发分支**，§6 主循环、§9 堆管理、§11 完成标志环均不改。两模式共用一套 +ring-per-bucket 实现（§12.7.1），§6.4 的回收窗口/哈希/winner-only 等语义原样继承。这把"双模"的改动面 +收敛到"同一 ring 的两种纪律配置" + insert/lookup 分发，满足"基于命令行设置支持两种方案"的设计要求。 + +### 12.10 落地实现与验证（a2a3 `dist_engine.cpp`） + +§12.4–§12.9 给出的是 shared 的**通用设计**（MPSC `reserve` 抢槽 + winner-only insert + 最终一致）。 +实际落地在 `src/a2a3/runtime/fully_distributed_within_core/runtime/dist_engine.cpp` 时，为**保证 shared +与 private 逐位一致的计算结果**（验收目标），做了两处**更强**的选择，并新增一个反压旋钮。三者都不改变 +§12.1–§12.3 的统一 ring 结论，只是把"弱一致 + MPSC"收紧为"强一致 + 串行追加"。 + +**(1) 顺序化追加定序器（取代 MPSC winner-only insert）。** shared 用一个全局原子 `tm_insert_next` +把**所有追加严格串行化到 task-id 顺序**：任务 `N` 只由"第一个走位到 `N` 的核"追加一次，且必须在 +`0..N−1` 全部追加之后（`CAS(insert_next, N, BUSY)` 抢占 → 写槽 → `store(N+1)`）。因每个核都按 id 顺序 +replay、且离开任务 `K` 前必已确保 `K` 已被追加（定序器阻塞它），故**任何核解析任务 `N` 的 fan-in 时， +全局 ring 的内容恰好等于 private 副本此刻应有的内容**。这把 §12.4 的"最终一致（winner 可能尚未发布 → +lookup 命中次新/NONE）"收紧为**强一致**，代价是追加环节全局串行（但追加本身极廉价，且 execute 仍 +乱序重叠）。因**只有单一追加者**，§12.7.1 的 MPSC `reserve` 不再需要：`tail` 由当前唯一追加者写， +`head`/`tail` 为原子、per-slot `seq` 仍保留以防读者在 host 线程被抢占时撞上槽复用。 + +**(2) lookup 双过滤 `producer ∈ [N−H, N)`。** 在 §12.6 时序过滤（`< N`，跳过未来 producer）之外，**再加 +`≥ N−H` 的下界**——精确对齐 private 的 `alive_floor = N−H`。这样即便共享 ring 里同时存在快核追加的 +"未来条目"（`≥N`）与尚未回收的"陈旧条目"（` 其 id`），`min_progress` 上升即释放前沿——**无死锁，只节流 +前沿**。默认 `Δ_max = 3·cap/4 − H − 1`（令窗口 ≤ ~¾ cap，留哈希倾斜裕度）；命令行 `--runahead N`（→ +`PTO_DIST_RUNAHEAD=N`）覆盖，`0` 关闭（溢出回退为 FATAL）。这与 §11.4 的堆反压同源（满则等待、不丢数据）。 +> 注：`PTO_DIST_RUNAHEAD` 现同时驱动**两模式通用的均衡节流** `gd->runahead_max`（§12.7.3，submit/alloc +> 入口按 `N − min_progress ≤ Δ_max` 等待，默认 `2×num_workers`）；shared 下它一并覆盖此处的追加前沿上界。 +> 二者机制同构（都读 `core_progress[]` 的 `min`），前者管**认领走位**均衡、后者兼管**共享环窗口**不溢出。 + +**节流时 worker 不空转，而是协作式 drain（不是 park 线程）。** 被节流的前沿核**不阻塞、不睡眠**， +而是在等待窗口的每一圈都调用 `drain_block_won()`（把发到本 lane 的 block.won 存款拉进空闲 slot）+ +`drain_phase_b()`（扫描本核私有 ring，把 **fan-in 已就绪** 的 slot 立即 `execute` 并发布完成标志、释放 +slot）。**仅当**本核确实没有任何就绪任务可执行（`drain_phase_b` 返回 0）才 `SPIN_WAIT_HINT` 轻自旋 + +看门狗。这正是"worker 去检查窗口里的任务是否满足执行条件并 drain"的行为——而且它 drain 出的完成标志 +会**解锁其它核**依赖这些数据的任务，让最慢核得以推进 replay、抬升 `min_progress`，从而释放本核的节流。 +因此这是一条**协作式、无死锁**的等待：前沿核用等待时间替系统清偿它自己欠下的完成事件。 + +**为何 per-core task slot 不能替代该节流（回答"worker 的 task slot 本身是否已限制窗口"）。** 每核有 +一个 **`kPrivateSlots = 4` 的私有执行 ring**（`kWonReserve = 2` 预留给 follower，故自领任务实际在 +`occupied_count ≥ 2` 时即反压）。**但它约束的是"本核已认领、尚未执行"的任务数（执行窗口），不是 +replay 走位的 run-ahead `Δ`。** 关键区别：一个核 **replay 全部任务**（shared 下还参与全部任务的顺序 +追加），却**只对自己 win 的任务占用 slot**；对没抢到的任务，它只做 lookup/append 后**径直走过、不占 +slot、不反压**。因此在 skip-exec（执行 0 成本）下，自领 slot 瞬间 drain 空、执行反压从不触发，走位游标 +可一路冲到终点 → `Δ` 爆炸 → shared 的 `Δ+H` 窗口溢出（这正是先前观察到的 FATAL）。**在真实执行下**， +自领 slot 的执行反压只提供**软性、间接**的减速（核必须等自己认领的任务算完才能继续），能压低但**不能 +封顶** `Δ`，且强度取决于认领比例与依赖结构。故 task slot 无法安全地界定 tensormap 窗口——必须有独立的 +run-ahead 硬上界 `Δ_max`。二者约束**正交**：task-slot ring 限"owned-in-flight"（§3.1 完成侧）， +run-ahead 限"replay 前沿领先量"（§12.7.1 tensormap 侧）。 + +**(4) lookup 的 acquire 摊薄：每次 lookup 一次 acquire（而非每槽一次）。** §12.7.1 的通用 MPSC 伪码里 +每扫一槽都要 `acquire_load(s.seq)`（因 `reserve` 只是抢槽游标、非发布水位，无法保证其下每槽已发布）。 +但落地实现是**单一串行追加者**（上文 (1)）：`tail` 是**真正的发布水位**——追加者先 `store(seq=k, release)` +再 `store(tail=k+1, release)`，且跨核的 `append(k)→append(k+1)` 由 `tm_insert_next` 的 release/acquire 链 +串起、`tail` 单调。故 reader **只需对 `tail` 做一次 acquire-load**，即与"所有 `< tail` 的追加"建立 +happens-before，其下每一槽的字段与 `seq` 都已可见；扫描各槽时 `seq` 改用 **relaxed 读**，仅作 ABA 护栏 +（防扫描期间 `head` 并发回收把物理槽复用出去）。这把 §12.8.1 里"每扫一槽一次原子 acquire"降到"每次 +lookup 一次 acquire"，使 shared 的单槽扫描逼近 private 的纯本地读；残留的**跨核 cache line coherence 读** +不可消除（单份共享的固有对价，private 靠复制 N 份规避）。**验证**：改动后 6 核（`sig=358074ac…`）与 24 核 +（`sig=c01c01e9…`）的 `PTO_DIST_DEPSIG` 依旧与 private **逐位一致**，private 侧签名不变（`8a877fd1…`）。 + +**命令行 / 环境变量。** + +| 变量 | 作用 | +| ---- | ---- | +| `PTO_DIST_TENSORMAP_MODE={private\|shared}` | 选择模式（默认 `private`）；运行期一次性读取，不中途切换 | +| `PTO_DIST_TENSORMAP_RING_CAP={N\|auto}` | 每桶 ring 深度（2 的幂，`auto` 由 `H` 派生，两模式生效） | +| `PTO_DIST_RUNAHEAD=N` | run-ahead 上界 `Δ_max`（**两模式通用**的负载均衡旋钮，§12.7.3）：任一核的 replay 走位最多领先最慢核 `N` 个任务。`0` 关闭节流；shared 下还同时覆盖前沿追加反压上界（`0` ⇒ ring 溢出回退确定性 FATAL）。默认按平台核数派生（见 §12.7.3） | +| `PTO_DIST_DEPSIG=1` | 打印依赖图签名（见下），供 private/shared 一致性验证 | +| `PTO_DIST_OVERHEAD=1` | 打印 `[dist] TMOPS`（inserts/lookups/scans 计数，§12.8.1）+ `[dist] OVERHEAD`（makespan/busy/replay 墙钟；macOS 仿真上被跨核调度延迟主导、仅供参考，理由见 §12.8.1） | + +**正确性验证（依赖图签名，免疫浮点噪声）。** 因 BGEMM/PagedAttention 的 `C += A@B` 等**浮点累加顺序 +随调度变化**，数值 `max_diff` **逐次运行本就波动**（private 自身即 0.0091↔0.0093），无法作逐位判据。故 +引入 `PTO_DIST_DEPSIG`：对每条已解析的 fan-in 边 `(consumer, producer)` 做 **XOR 累加**——与调度顺序 +无关，只取决于边的**集合**，是免疫浮点噪声的正确性判据。实测（build/lib，a2a3sim）： + +| 用例 | 规模 | private 签名 | shared 签名 | +| ---- | ---- | ---- | ---- | +| 差分 UT `test_dist_tensormap_ring.cpp` | 268 万次查询（含"快核超前/滞后回收"越窗条件） | 参考=private | **== private** | +| BGEMM `Case0` | 72 核 / 500 任务 | `8a877fd1e0e02bb1` (750 边) | **相同** | +| PagedAttention `CaseSmall1` | 27 核 | `a7db56ff3de5afa6` (15 边) | **相同** | +| runtime_overhead | 12 核 / 600 · 1200 · 4000 任务 | `c01c…`·`3969…`·`4286b6f704b38748` | **相同** | + +结论：**shared 与 private 解析出完全相同的依赖图**（跨 72 并发核、含 4000 任务大图），二者数值差异纯属 +浮点累加顺序噪声（private 自身也有），非正确性差异。run-ahead 反压使 4000 任务的 skip-exec 大图从 +"`Δ+H` 溢出 FATAL"转为顺利完成且签名一致。 + +## 13. 进程全局变量的跨平台处理（`global_data` 段 + base 指针寻址） + +### 13.1 问题：CCEC 不支持进程全局变量 + +在常规 pthread 编程里，进程拥有的 file-scope 全局变量**对进程内所有线程自动可见、自动共享**。本分布式 +运行时正是**依赖这一点**在 a2a3sim 上工作的：仿真平台把每个 AI 核实现为**同一进程内的一条 host 线程**， +共享同一地址空间，于是像 `DistGlobal g_dist`（含全局 task-id 定序器 `tm_insert_next`、shared TensorMap、 +claim cursor、完成标志环、frontier/vend、block.won 投递表 …）这样**唯一一份 file-scope 全局对象**天然被 +所有核共享，语义正确。 + +**但真实硬件（a5 等）不成立。** AICore 由 **CCEC 编译**，其生成的核上程序**不支持进程全局变量** +（无可共享的 `.data`/`.bss` 进程段）：file-scope 全局要么不可用，要么**每核各一份、互不共享**——而我们 +恰恰需要**跨核共享**的编排状态。因此必须改造这块"进程全局空间"的表达与访问方式。 + +**关键分工（决定了改造边界）：** + +| 阶段 | 运行在 | 能力 | +| ---- | ---- | ---- | +| `dist_engine_register()`（每次运行一次） | **AICPU（ARM A55）** | 有 `malloc`、有正常进程全局；可分配并初始化共享段 | +| `dist_core_main()` 及其被调用链（replay/claim/execute/drain、ops 回调） | **AICore（CCEC）** | **无进程全局**；只能靠传入的参数/寄存器寻址共享内存 | + +即：**AICPU 负责"分配 + 初始化"共享段，AICore 只能"经指针访问"它**。当前代码里 AICore 直接引用 +`g_dist.*` / `g_self` / `g_tm_*` 等 file-scope 全局（`dist_engine.cpp` 内约 180 处），这些正是在 a5 上失效的点。 + +### 13.2 设计：运行时分配 `global_data` 段，base 指针经 worker 参数下发 + +**核心方案（同一套代码在 sim 与 HW 上都走这条路）：** + +1. **分配（AICPU / register）**：启动时把**所有需跨核共享的全局状态收拢进一个结构体 `DistGlobal`**， + 在一块**全局可寻址内存（GM）**上分配其唯一实例，并完成初始化（cursor=-1、flags=0、topology、模式旋钮…）。 +2. **下发 base（经已有的 worker 参数）**：把段基址存入 `Runtime::dist.global_data_base`（新增字段）。 + `runtime`（一个共享 GM 上的 `Runtime*`）**本就作为参数**传给每个 worker： + `core_main(runtime, core_idx, core_type)`。故**任何核都能从参数链取得 base**，无需任何 file-scope 符号。 +3. **访问（AICore）**：`DistGlobal* gd = (DistGlobal*)runtime->dist.global_data_base;`，其后所有 + `g_dist.X` → `gd->X`。这**天然编译成 `base + 字段偏移` 的访存**——正是用户要求的"base 指针 + 各变量 + 偏移量算出实际地址"的方案，且**不产生任何进程全局符号**，CCEC 安全。 +4. **偏移量来自 `offsetof`，不写死魔数**：字段地址 = `base + offsetof(DistGlobal, field)`，由编译器在编译期 + 给出。工程上**直接用"类型化结构指针 + 成员访问"（`gd->field`）即可**——编译器自动发射 base+offset 访存， + 既杜绝手算偏移出错，又满足 CCEC 无全局约束。§13.4 给出完整字段清单及其 `offsetof` 语义。 + +```text +┌─ AICPU (A55, register) ────────────────┐ ┌─ AICore #k (CCEC, core_main) ──────────┐ +│ gd = alloc_global_segment(sizeof(DistGlobal)) │ │ gd = (DistGlobal*)runtime->dist.global_data_base │ +│ init gd->cursors/flags/topology/... │ base │ self = &gd->cores[core_id()] │ +│ runtime->dist.global_data_base = gd ──┼───────▶│ ... gd->frontier / gd->shared_map ... │ +│ store-release; publish dist.go=1 │ │ (base + offsetof 访存，无全局符号) │ +└────────────────────────────────────────┘ └────────────────────────────────────────┘ +``` + +### 13.3 方案 B：AICore 执行路径上零 file-scope 符号（`gd`/`self` 全程参数穿引） + +> **为什么不是"平台化 file-scope 指针"（方案 A）**：曾考虑用 `#if` 把 `g_gd`/`g_self` 在 sim 上编成 +> `thread_local`、在 HW 上编成"每核各一份的普通指针"。但这依赖"CCEC 允许每核私有的**可写 file-scope 静态 +> 存储**"这一前提；一旦 CCEC 完全禁止可写静态存储，方案 A 即失效。**方案 B 更强硬也更可移植：AICore 的 +> 功能路径上不出现任何 file-scope / thread_local 符号，`gd`（段基址）与 `self`（当前核）一律作为参数穿引。** + +**这正是集中式运行时早已在用的套路**：`Runtime` / `PTO2Runtime` 本身就是"AICore 经一个发下来的 base 指针 +访问的 GM 结构"，其内部再以指针字段（如 `aicore_mailbox`、`sm_handle`）指向 arena 里的其它子区。方案 B +照搬这一模式，把 `DistGlobal` 段基址经 AICore **本就读取**的运行时对象下发： + +| 入口 | 拿到什么 | 如何取 `gd` / `self` | +| ---- | ---- | ---- | +| `dist_core_main(runtime, core_idx, …)`（每核 worker 主函数） | `Runtime*`（arg） | `gd = (DistGlobal*)runtime->dist.global_data_base;` `self = &gd->cores[core_idx];` | +| ops 回调（`dist_submit_impl` / `dist_alloc_tensors` / `dist_get/set_tensor_data` / `dist_is_fatal` / `dist_report_fatal`） | 仅 `PTO2Runtime* rt` | `gd = (DistGlobal*)rt->dist_global;` `self = &gd->cores[pto_core_id()];`（见下） | +| 其余 helper（绝大多数持有 `self`） | `DistCore* self`（arg） | 顶部 `DistGlobal* gd = self->gd;`（`DistCore` 的回指字段） | +| 无 `self` 的自由函数（`fatal_set`/`set_fatal`/`advance_frontier`/`watchdog`/`dep_sig_add`/`tm_shared_*`/`alloc_won_slot`） | — | 显式加 `DistGlobal* gd` 形参 | + +**ops 回调的回收 seam（新增 `PTO2Runtime::dist_global` 字段）**：回调只收到 `rt`，故在 `rt` 上加一个 +`void* dist_global` 字段（register 时写入段基址），回调即 `pto_gd(rt) = (DistGlobal*)rt->dist_global`—— +与 `rt` 已有的 `aicore_mailbox`/`sm_handle` 指针字段同构，不引入任何进程全局。 + +**"我是哪个核" `pto_core_id()`（编译期分支，段里唯一的 per-core 数据）：** + +| 平台 | 实现 | +| ---- | ---- | +| a2a3sim | `thread_local int32_t`（在 `dist_core_main` 入口用 `pto_set_core_id(core_idx)` 写入）。注意：这是 **seam 内部**的一个 thread_local **整数**，不是 file-scope 的 `self` 指针；且**此分支在 HW 上不编译**。 | +| a5 / CCEC | 读**硬件 per-core id 寄存器**（block/core index，见 [simt-launch.md](simt-launch.md)），无任何存储。 | + +为把改动局部化，`DistCore` 增加一个**回指指针 `DistGlobal* gd`**（register 时写好）：凡持有 `self` 的函数 +即可 `self->gd->…`；只有少数无 `self` 的自由函数/回调需显式取 `gd`。这样"段化"后**没有任何一处功能代码 +依赖 file-scope 可写符号**。 + +### 13.3.1 平台抽象 seam（GM 分配 / 缓存一致 / core-id） + +方案 B 把所有平台差异收敛到 `dist_engine.cpp` 顶部匿名命名空间里的四个内联 seam（`#if DIST_SIM_HOST_CLOCK` +分支；`DIST_SIM_HOST_CLOCK==0` 当且仅当 CCEC/HW）。功能代码只调它们，不再出现任何平台条件： + +| seam | 运行在 | a2a3sim | a5 / CCEC（TODO 联调） | +| ---- | ---- | ---- | ---- | +| `pto_gm_alloc(bytes)` / `pto_gm_free(p)` | AICPU | host `malloc`/`free` | GM 分配器：返回**所有核可寻址且一致**的一块 GM | +| `pto_gm_publish(base, bytes)` | AICPU | 空操作（同地址空间 + register 处的 release 栅栏已足够） | 对 `[base, base+bytes)` 做 flush/invalidate，使各核读到已初始化态 | +| `pto_core_id()` / `pto_set_core_id(id)` | AICore | `thread_local int32_t`（入口写入 `core_idx`） | 读硬件 core-id 寄存器；`set` 为空 | + +- **分配只发生在 AICPU**（register），AICPU 有 `malloc`/GM 分配器与进程全局，故段句柄用一个**进程静态指针** + 持有（跨 run 复用、每 run 重置）——"无全局"约束只针对 AICore。 +- **一致性**：sim 单地址空间 + register 末尾的 `atomic_thread_fence(release)` 即够；HW 由 `pto_gm_publish` + 在 worker 观测到 `dist.go` 之前把段刷出。 +- `pto_core_id()` 是**段内唯一的 per-core 数据**，也是方案 B 对硬件的唯一新增要求。 + +### 13.4 完整全局变量清单与段内布局 + +**(A) 收拢进 `DistGlobal` 段的共享状态**（跨核共享，必须迁移；字段地址 = `base + offsetof(DistGlobal, ·)`）： + +| # | 变量 / 字段 | 类型 | 用途 | 现状 | +| - | ---- | ---- | ---- | ---- | +| 1 | `cube_cursor[4]` / `vector_cursor[4]` / `alloc_cursor[4]` | `PaddedCursor`（cacheline 对齐） | 分类型 claim 高水位（cursor sharding，§6.6） | `g_dist` 内 | +| 2 | `flags[kFlagCap]` | `atomic[65536]` | 每任务完成标志环（§11.5） | `g_dist` 内 | +| 3 | `frontier` / `H` / `vend[kFlagCap]` | `atomic` / `int32_t` / `atomic[65536]` | 完成前沿 F、依赖跨度、累计虚拟堆字节（§9.5/§11.4） | `g_dist` 内 | +| 4 | `heap_base` / `heap_size` | `uint8_t*` / `size_t` | 确定性 GM 输出堆环 | `g_dist` 内 | +| 5 | `orch_func` / `orch_args` / `rt` / `runtime` | 指针 | 编排入口/参数/运行时回指 | `g_dist` 内 | +| 6 | `fatal` | `atomic` | 全局致命标志 | `g_dist` 内 | +| 7 | `num_workers` / `num_blocks` / `layout[]` / `blocks[]` | 标量 / `CoreLayout[]` / `BlockWon[]` | 物理拓扑 + block.won 投递表（§3.1） | `g_dist` 内 | +| 8 | `replay_done` / `started_count` | `atomic` | tail-idle 计数 + 启动栅栏（§7） | `g_dist` 内 | +| 9 | `shared_map` / `tm_insert_next` / `core_progress[]` | `SharedTensorMap` / `atomic` / `atomic[]` | shared 模式全局环 + 追加定序器 + 各核进度（§12） | `g_dist` 内 | +| 10 | `cores[RUNTIME_MAX_WORKER]` | `DistCore[]` | 各核私有状态（private map、task slot 环、outpool…）；**新增回指 `gd`** | `g_dist` 内 | +| 11 | `tm_shared` / `ring_cap` / `tm_runahead_max` / `runahead_max` | `bool`/`int32_t` | 模式/容量/run-ahead 旋钮（register 设定、全程只读）；`runahead_max` 为两模式通用的均衡上界 `Δ_max`（§12.7.3，默认 `2×num_workers`） | 现为独立 `g_*`，**并入段** | +| 12 | `dep_sig` / `dep_edges` | `atomic` | 依赖图签名（验证用，§12.10） | 现为独立 `g_*`，**并入段** | + +**(B) 仿真专属、已被 `#if DIST_SIM_HOST_CLOCK` 排除于 HW 之外**（HW 构建里根本不编译，无需迁移，但为 +"同源"整洁仍建议并入段）：`overhead_on`、`skip_exec`、`trace_on`/`trace_epoch_ns`/`trace_reserve`、 +`tm_inserts`/`lookups`/`scans`、`orch_t0_min…replay_sum`/`orch_recorded`。 +> `DIST_SIM_HOST_CLOCK == 0` 当且仅当 `__CCE_AICORE__ || __DAV_C220__ || __CCE_KT_TEST__`（即 HW/CCEC）， +> 故这批开销/追踪计数在 a5 上不存在，天然规避。 + +**(C) 只读常量与函数指针表**： +- `g_dist_ops`（`const PTO2RuntimeOps`，ops 函数指针表）：只读常量数据，`rt->ops = &g_dist_ops` 取址。 + 只读常量不是"可变进程全局"，但 CCEC 下 const-data 的放置需按平台约定（常量区/GM）；实现时以 + `constexpr`/只读 GM 常量登记，或在 register（AICPU 侧）把表拷入段的一个 `ops` 字段并让 `rt->ops` 指向它。 +- `kFlagCap`/`kRingBuckets`/`kBucketCapMax`/`kHDefault`/`kPrivateSlots` 等 `constexpr`：编译期常量，无存储，**无需迁移**。 +- `g_self`（原 `thread_local`）：方案 B 已删除，由 `pto_dist_self(rt)`（`rt->dist_global` + `pto_core_id()`）取代，见 §13.3。 + +### 13.5 逐点访问迁移映射（约 180 处） + +| 原引用（file-scope 全局） | 迁移后（base+offset 访问） | 备注 | +| ---- | ---- | ---- | +| `g_dist.`（约 140 处） | `gd->`（每函数顶部 `gd = self->gd` / 形参 / 局部） | 方案 B：`gd` 全程参数穿引，无 file-scope 符号 | +| `DistCore* self = g_self;`（ops 回调，4 处） | `DistCore* self = pto_dist_self(rt);` | `gd=(DistGlobal*)rt->dist_global`（新字段）+ `pto_core_id()` | +| `g_tm_shared`（13） | `gd->tm_shared` | 只读旋钮 | +| `g_dist_ring_cap`（7） | `gd->ring_cap` | 只读旋钮 | +| `g_tm_runahead_max`（7） | `gd->tm_runahead_max` | 只读旋钮 | +| `g_dep_sig` / `g_dep_edges`（10） | `gd->dep_sig` / `gd->dep_edges` | 验证累加器 | +| `g_tm_inserts/lookups/scans`（15） | `gd->tm_inserts/...` | 仅 `DIST_SIM_HOST_CLOCK` 下增量 | +| `g_orch_*`（sim-only） | `gd->orch_*` | `#if DIST_SIM_HOST_CLOCK` 内 | + +**访问约定（方案 B，减小改动面）：** 全文 `g_dist.` → `gd->` 后，在每个用到 `gd` 的函数补一个来源：持有 +`self` 的函数顶部 `DistGlobal* gd = self->gd;`；`dist_core_main` / `register` 用局部 `gd`；无 `self` 的自由函数 +加 `DistGlobal* gd` 形参；ops 回调用 `pto_gd(rt)` / `pto_dist_self(rt)`。**编译器即是清单**——改完签名后一次 +构建会精确列出所有"缺 `gd`"的函数，逐一补齐即可。 + +### 13.6 平台统一与验证策略 + +- **同一套源码、同一条路径**：sim 与 HW **都**分配段、都经 `runtime->dist.global_data_base` 访问。sim + 上段就是 host 堆的一块（`malloc`），HW 上是 GM 分配；差异仅在**分配器**与**core-id/self seam**两个被 + `#if` 隔离的点。sim 上不需要该方案，但**故意也走它**——用来在上 a5 之前，先在 a2a3sim 证明改造正确。 +- **验证判据（复用现有）**： + 1. 差分 UT `test_dist_tensormap_ring.cpp` 仍通过； + 2. `PTO_DIST_DEPSIG` 下 private == shared 签名不变（6/24/72 核）； + 3. BGEMM / PagedAttention golden 与改造前一致。 + 三者全绿即证明"段化 + base 寻址"未改变任何功能语义。 + +### 13.7 落地实现计划（方案 B，分阶段，可逐步在 a2a3sim 验证） + +1. **平台 seam**：加 §13.3.1 的 `pto_gm_alloc/free` / `pto_gm_publish` / `pto_core_id/set_core_id`（`#if DIST_SIM_HOST_CLOCK`）。 +2. **段结构收拢**：把 §13.4(A)(B) 的独立 `g_*` 并入 `DistGlobal`；`DistCore` 增 `DistGlobal* gd` 回指。 +3. **下发通道**：`Runtime::DistHandoff` 增 `volatile uint64_t global_data_base`（给 `dist_core_main`）； + `PTO2Runtime` 增 `void* dist_global`（给 ops 回调）；`shared/runtime.cpp` 初始化 `global_data_base=0`。 +4. **参数穿引**：全文 `g_dist.`→`gd->`；每函数补 `gd` 来源（`self->gd` / 形参 / 局部 / `pto_gd(rt)`）； + `g_self`→`pto_dist_self(rt)`。删除所有 file-scope `g_gd`/`g_self`。 +5. **register / core_main**：register 经 `pto_gm_alloc` 分配、初始化、`pto_gm_publish`，写两个下发字段与 + `cores[].gd`；`dist_core_main` 入口取 `gd`、`pto_set_core_id(core_idx)`。 +6. **构建 + 验证 a2a3sim**：DEPSIG（private==shared）、TMOPS、BGEMM golden 全绿。 +7. **（后续）a5 落地**：实现 `pto_gm_alloc/publish` 的 GM 版与 `pto_core_id()` 的寄存器版，打通 CCEC 分支。 + +### 13.8 落地实现与验证（方案 B，已完成，a2a3sim） + +第 1–6 步已实现于 `dist_engine.cpp` / `pto_runtime2.h` / `runtime.h` / `shared/runtime.cpp`，要点： + +- **平台 seam**（§13.3.1）落在 `dist_engine.cpp` 顶部匿名命名空间；功能代码只调 seam，零平台条件。 +- `DistGlobal` 收拢全部功能性共享状态（含 `tm_shared`/`ring_cap`/`tm_runahead_max`/`dep_sig`/`dep_edges`）； + `DistCore` 增回指 `DistGlobal* gd`。 +- **下发**：`Runtime::DistHandoff` 增 `global_data_base`，`PTO2Runtime` 增 `dist_global`。 + `dist_engine_register`（AICPU）经 `pto_gm_alloc` + placement-new 分配段（进程静态句柄，跨 run 复用）， + `pto_gm_publish` 刷出，写 `runtime->dist.global_data_base`、`rt->dist_global`、所有 `cores[i].gd`。 +- **零 file-scope 符号**：`g_gd`/`g_self` 已彻底删除。约 140 处 `g_dist.*`→`gd->`，`gd` 一律来自 + `self->gd` / 形参 / 局部 / `pto_gd(rt)`；4 处回调的 `self` 改由 `pto_dist_self(rt)`（`rt->dist_global` + + `pto_core_id()`）回收；`fatal_set`/`set_fatal`/`advance_frontier`/`watchdog`/`dep_sig_add`/`tm_shared_*`/ + `alloc_won_slot` 增 `gd` 形参。 +- **诊断隔离**：`dist_dump_state`（SIGUSR1/watchdog 信号处理器，签名 `void(int)`）与 `dist_engine_dump_trace` + （arg-less swimlane 导出）是 host/sim 专属调试器（`fprintf`/`chrono`，env 门控，不编到 CCEC），故用一个 + **AICPU 侧诊断句柄 `s_dump_gd`**（register 时写）访问——**不在功能路径上**，功能路径仍零符号。 +- TMOPS 诊断计数仍为 `#if DIST_SIM_HOST_CLOCK` 专属，`TMOP_COUNT()` 宏在 HW 上编空。 + +**验证（a2a3sim，与迁移前判据一致）：** + +| 判据 | 结果 | +| ---- | ---- | +| DEPSIG private vs shared（runtime_overhead 12 blocks / 36 核 / 240 任务） | 均 `edabb0ba8876d2cf`（360 边）——**逐位一致** | +| TMOPS（36 核） | private `inserts=17280` vs shared `inserts=480`（≈单环追加），`lookups` 均 `1440`——**插入下沉比符合预期** | +| kernels-enabled exec（4 blocks / 48 任务） | 正常完成，无 FATAL/abort，产出 `OVERHEAD` 指标 | +| 构建 | a2a3sim（AICPU/AICORE/HOST/SIM_CONTEXT）全部 `Build complete!` | + +> a5 硬件落地（第 7 步）尚待：实现 `pto_gm_alloc`/`pto_gm_publish` 的 GM 版与 `pto_core_id()` 的寄存器版， +> 打通 CCEC 分支。sim 上"零 file-scope 符号 + 参数穿引 + base 寻址"已验证功能正确，为 a5 联调的基线。 + +## 14. 跨核缓存一致性抽象（A5 落地契约） + +§13 解决了"进程全局变量"，但那只是把共享状态**放到哪里**的问题。真正让 SPMD 引擎能在 A5 上跑通的，是 +**跨核可见性**：a2a3sim 的每个"核"是同进程 host 线程，天然缓存一致，`std::atomic` 的 `memory_order` 足以 +保证一个核写、另一个核读的可见性。**但 A5 的 AICore 之间没有硬件缓存一致**——一个核写入的字，另一个核在 +它**刷出到 HBM（`dcci … CACHELINE_OUT`）**且读者**失效本地陈旧副本（`dcci …`）**之前都看不到，这正是 +a5 `aicore_executor.cpp` 全程手动做的事（第 66–194 行的 `dcci`/`OUT_OF_ORDER_STORE_BARRIER`）。因此在 HW 上 +`std::atomic` 是**必要但不充分**的：它只排序本核访问，从不跨核发布。 + +### 14.1 设计：`Coherent` —— `std::atomic` 的一致性替身 + +在 `dist_engine.cpp` 顶部匿名命名空间引入一层 seam 与一个替身类型 `Coherent`： + +- **一致性原语（`#if DIST_SIM_HOST_CLOCK`）**：`pto_dcci_inval(p,n)` / `pto_dcci_flush(p,n)` / + `pto_shared_fence(order)`。sim 上 `inval`/`flush` 为空（同地址空间 + 下面的 `std::atomic` 序即足够）， + HW 上分别发失效 / 刷出 + 屏障（TODO a5）。 +- **`Coherent`**：只含一个 `std::atomic a` 成员（**size/alignment 与 `std::atomic` 相同**，故 + `DistGlobal` 布局与 `sizeof` 不变）。它的每个方法都镜像调用点用到的 `std::atomic` 操作——读前 `inval`、 + 写后 `flush`： + +| 方法 | 语义 | +| ---- | ---- | +| `load(order)` | `dcci_inval` → `a.load(order)` | +| `store(v, order)` | `a.store(v, order)` → `dcci_flush` | +| `compare_exchange_weak/strong(…)`、`fetch_add/sub/xor(…)` | `dcci_inval` → RMW → `dcci_flush` | + +**关键收益**：迁移只改**字段声明**（`std::atomic` → `Coherent`），约 80 处调用点的 +`.load(order)/.store/.cas/.fetch_*` 语法**原样不动**。sim 上 `Coherent` 就是原来的 `std::atomic` 操作 +（`dcci` 编译为空）——**行为逐位不变、零成本**；HW 上则在同一处集中发 `dcci`。 + +### 14.2 已段化为 `Coherent` 的跨核共享状态 + +`flags[]`、`frontier`、`vend[]`、`fatal`、`replay_done`、`started_count`、`tm_insert_next`、 +`core_progress[]`、`dep_sig`/`dep_edges`（`DistGlobal`）；`cube/vector/alloc_cursor[].v`（`PaddedCursor`）； +`SharedTensorMap` 的 `seq`/`head[]`/`tail[]`；`block.won` 的 `state`/`remaining`/`drained[]`/`any_pub`。 +两处显式发布点的 `std::atomic_thread_fence(release)` 改为 `pto_shared_fence(release)`。 +> 仿真专属诊断计数（`g_tm_*`/`g_orch_*`，`#if DIST_SIM_HOST_CLOCK` 内）仍为普通 `std::atomic`——不在 HW 路径上。 + +### 14.3 A5 落地契约与遗留风险 + +- **落地点收敛为三处**：`pto_dcci_inval` / `pto_dcci_flush` / `pto_shared_fence` 的 HW 实现，加上 §13.3.1 的 + `pto_gm_alloc/publish` 与 `pto_core_id()`。功能代码不再有任何平台条件。 +- **粒度**：当前按 `sizeof(atomic)`（≤8B，单 cacheline 内）失效/刷出。`flags[]` 是逐字节环，相邻 flag 可能 + 共享 cacheline——HW 上 `dcci` 以 cacheline 为单位，需确认"失效邻居未刷出的写"不会丢数据（必要时 flag 按 + cacheline 对齐，或改用 per-core flag 分片）。 +- **跨核 RMW 原子性（最大遗留风险）**：`claim()` 的 `fetch_max`-式 CAS、`remaining.fetch_sub`、`state`/ + `drained` 的 CAS 依赖**核间真原子**。`Coherent` 的 "inval + `std::atomic` RMW + flush" 在 sim 正确， + 但 HW 上若无核间原子单元 / LL-SC，则**不是真原子**——认领仲裁可能需改设计（HW 原子单元，或退回 AICPU 仲裁）。 + `Coherent` 把这一决策**集中到了一处**，但并未消除它。 +- **`__gm__` 地址空间**：`Coherent` 目前是普通指针成员；CCEC 上段与其内部访问需 `__gm__` 限定，属后续 HW 化工作。 + +### 14.4 验证（a2a3sim，与迁移前逐位一致） + +| 判据 | 结果 | +| ---- | ---- | +| DEPSIG private vs shared（12 blocks / 36 核 / 240 任务） | 均 `edabb0ba8876d2cf`（360 边）——**与引入 `Coherent` 前逐位相同** | +| TMOPS（36 核） | private `inserts=17280` vs shared `inserts=480`，`lookups` 均 `1440`——不变 | +| kernels-enabled exec（4 blocks / 48 任务） | 正常完成，无 FATAL/abort | +| 构建 | a2a3sim 全部 `Build complete!` | + +> 结论：`Coherent` 建立了 HW 缓存一致性的**单一接缝**且对 sim 零影响。A5 联调时只需实现三处 `dcci`/屏障 +> 原语，并单独攻克 §14.3 的跨核 RMW 原子性与 `__gm__` 化——这两项是 SPMD 引擎上 A5 的下一个主攻方向。 + +## 15. a5sim 落地 SPMD 引擎（上真机前的基线） + +上真 A5 硬件前，先让 **a5sim** 跑通同一套去中心化 SPMD 引擎（`dist_engine.cpp`）。a5sim 与 a2a3sim 共用同一 +host 线程执行模型（AICore 也是同进程 `std::thread`，非独立 CCE 地址空间），故 `DIST_SIM_HOST_CLOCK=1`，§14 的 +`Coherent`/`dcci` 编译为空——本阶段验证的是**引擎结构能在 a5 代码树 / 类型上跑通**，真正的 HW 一致性路径要到 +onboard CCEC 构建才激活。 + +### 15.1 共享单一源（消除双份维护） + +`dist_engine.{cpp,h}` 从 `src/a2a3/.../runtime/` 迁到 **`src/common/runtime/fully_distributed_within_core/`**， +a2a3 与 a5 的 `build_config.py` 各自把该公共目录加入 **AICPU** 的 `include_dirs` + `source_dirs`（`DIST_COMMON`）。 +CMake 递归 GLOB 该目录、编译器用**各 arch 自己的 `-I`** 解析 `runtime.h`/`pto_runtime2.h` 等——**同一份源、按 arch 编译**， +零重复。dist_engine 只编进 AICPU `.so`（`dist_core_main` 经函数指针在 AICore 线程上执行），与 a2a3 一致。 + +- **可移植接缝**：a2a3 的 `LocalContext` 用 `block_idx`/`block_num`，a5 用 `s_block_idx`/`s_block_num`（避开 CCE + 内建符号冲突）。共享源用**检测惯用法重载** `dist_set_local_block()` 设值——优先选 `s_` 字段，否则回退无前缀， + 两 arch 同源编译，**不改任一 `intrinsic.h`**。 + +### 15.2 a5 侧接线 + +- **类型对齐**：a5 `runtime.h` 的 `DistHandoff` 补 `global_data_base`，`Runtime` 补 `use_example_exec_time_`/ + `example_exec_time_ns_[]`；`pto_runtime2.h` 补 `dist_global`；`shared/runtime.cpp` 构造函数初始化上述字段。 +- **AICPU**（`aicpu_executor.cpp`）：把编排直调 `(*p_func)(orch_args)` 换成 **dist handoff**——`dist_engine_register` + 写 `core_main_fn`、置 `dist.go`、等 `done_count==num_workers`、`dist_engine_dump_trace`、恢复 `rt->ops`。 +- **AICore**（`aicore_executor.cpp`）：保留 a5 的 phase1–3 握手与 teardown（EXIT/EXITED 协议），把 `DATA_MAIN_BASE` + 轮询主循环换成 **等 `dist.go` → 调 `core_main_fn`**（SPMD 入口）。 + +### 15.3 关键修复：scheduler 的 0-task 完成判据 + +SPMD 下编排/调度/执行全在核上完成，**没有任务下发到 AICPU 共享内存**，故 `on_orchestration_done(total_tasks=0)`。 +a2a3 的 `handle_orchestrator_exit` 对此有专门分支（`completed_tasks_ >= task_count` 即 `0>=0` → 立即 `completed_`）， +而 **a5 版多了 `task_count > 0 &&` 门**，导致 SPMD 路径永不完成——engine 已跑完（9 核 START→DONE、DEPSIG 已出、 +"all workers finished"），但 AICPU scheduler 线程在 `completed=0/0` **空转不退出**，整个 run 挂死。修复：a5 +`scheduler_cold_path.cpp::handle_orchestrator_exit` 去掉 `task_count > 0 &&` 门，与 a2a3 对齐。 + +### 15.4 验证（a5sim） + +移植的用例置于 **`tests/st/a5/fully_distributed_within_core/`**，用 `--platform a5sim` 运行： + +| 用例 | 结果 | +| ---- | ---- | +| `vector_example`（AIV-only，5 任务 DAG，golden 精确 f=47） | golden PASSED；DEPSIG private==shared `0d7fa3a297fced17`（6 边） | +| `mix_coown`（MIX 1C+2V 协同拥有 + 消费者，12 组） | golden PASSED（rtol/atol 1e-3）；DEPSIG `300391b07de72c6f`（12 边） | + +> a2a3sim 回归：迁源 + 可移植接缝后，paged_attention DEPSIG private==shared 仍 `a7db56ff3de5afa6`（15 边），无回归。 +> 遗留（上真 A5）：§14.3 的跨核 `dcci`/屏障 HW 实现、跨核 RMW 真原子性、`__gm__` 化，以及物理→逻辑 `pto_core_id()`。 + +## 16. A5 onboard（真机）落地设计与阶段计划 + +§15 让 SPMD 引擎在 **a5sim** 跑通,但 a5sim 与 a5 onboard 是两个世界:sim 是同进程 host 线程(单一地址空间、天然缓存一致、`thread_local`),onboard 是 AICPU(aarch64)+ 多个 AICore(CCEC)的**异构、非一致缓存、分地址空间**真机。对 a5 onboard 代码库的调查暴露了两个**根本阻塞**,它们决定了 onboard 落地不是"填 seam",而是**重构 + 一个悬而未决的硬件设计决策**。 + +> 注:本机无 CCEC 工具链(`build_runtimes.py --list` 仅 a2a3sim/a5sim),onboard 无法在此编译验证;下述实现均需在 CI/真机上做 CCEC 编译与硬件验证。 + +### 16.1 阻塞 1（架构）：热路径编在了错误的处理器上 + +onboard a5 各 target 的编译器(`runtime_compiler.py::_init_a5`):`aicore`→**CCEC**;`aicpu`→**aarch64-g++**;`host`→g++。 +`dist_engine.cpp` 现在只在 **aicpu** 的 `source_dirs`,故由 **aarch64-g++** 编译。sim 上"AICore worker"是同进程 host 线程,靠函数指针调用 `dist_core_main` 没问题;但**真机 AICore 是独立 CCEC 处理器,无法执行 aarch64 编出来的函数**。 + +推论: +- onboard AICPU 编译单元里 `__CCE_AICORE__` 未定义 → `DIST_SIM_HOST_CLOCK=1` → §13/§14 的 **HW seam `#else` 分支永不编译**(死代码)。单填 seam 不产生任何真机效果。 +- `dist_core_main` 及其调用链(claim、per-core TensorMap、`Coherent`、execute)**必须 CCEC 编译进 AICore 二进制**;只有 `dist_engine_register`(GM 段分配 + 发布)留在 AICPU。 + +**所需重构**:把引擎拆成两个编译单元 —— AICPU 侧(`register`/GM 分配/发布,aarch64)与 AICore 侧(`dist_core_main` 热路径,CCEC),平台 seam 抽到 CCEC 会编译的 header;`dist.core_main_fn` 在真机改为指向 **AICore 二进制内**的入口(而非 AICPU .so 内的函数指针)。同时需确认引擎热路径不使用 CCEC 不支持的 STL(`std::vector/string/chrono` 已被 `#if` 关闭,但需 CCEC 实编确认无残留)。 + +### 16.2 阻塞 2（硬件能力）：~~a5 AICore 无跨核原子 RMW~~ —— 已解除：A5 有硬件 GM 原子 + +> **本节结论已修订（2026-07）。** 早前基于对旧代码注释（`// No hardware fetch_max on the target`） +> 的判断，认定"a5 AICore 无跨核原子 RMW"，并把它列为总闸 blocker。**这个判断是错的。** +> A5（`dav_3510`）实测拥有可用、核间一致的 GM 硬件原子（见下）。claim 竞争可用一条硬件 +> `atomicMax` 直接实现，**无需** AICPU 仲裁 / 静态分派 / 软件锁，也**无需** uncacheable 内存。 + +**证据。** CANN `dav_3510` 头 `asc/impl/basic_api/dav_3510/kernel_operator_atomic_impl.h` 暴露： +`atomicAdd`/`atomicMax`/`atomicMin`（`int32_t/uint32_t/int64_t/uint64_t/float`）、 +`atomicCAS`/`atomicExch`（`uint32_t/uint64_t`），均作用于 `__gm__` 地址。CANN 文档 +[AtomicMax](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/910beta3/API/ascendcopapi/atlasascendc_api_07_00261.html) +给出三核并发 `AtomicMax` 结果正确、各核返回操作前旧值的示例——**内存级、核间序列化的真原子**。 +`dist_engine.cpp` 现有的 `dist_atomic_cas`/`dist_atomic_add` 已经在用 `atomicCAS`/`atomicAdd`， +说明 RMW 写侧本就走硬件原子。 + +**真正的真机 bug（已定位）在读侧，不在原子侧。** `coherent_load` 的 onboard 分支是**普通 +`load`（无 `dcci` 失效）**，注释假设"uncacheable → 普通 load 即一致"。但本机 double page table +不可用（§16.2 附注/探针），段实际是 **cacheable**，普通 load 读到的是本核**陈旧缓存副本**。 +后果：`atomicAdd`/`atomicMax` 的写在内存里是对的，但用 `coherent_load` 观察这些量的核看不到 +别人的更新 → drain barrier（`replay_done`）永远读不到 `num_workers` → 挂死（实测 9 核只读到 8）。 + +**修复方向（取代旧的 (0)/(1)/(2)/(3) 候选）：** + +1. **claim / 全局 cursor**：改用单条硬件 `atomicMax`（§11.1.1）。认领 `old = atomicMax(&cursor[T], N); won = old 附注（uncacheable 探针，2026-07）：`halMemCtl(CTRL_TYPE_GET_DOUBLE_PGTABLE_OFFSET)` 在本机 +> 返回 `rc=0` 但 `offset=0`（`CTRL_TYPE_GET_DCACHE_ADDR` 亦返回 0），即驱动确认**本设备无 +> cacheable/uncacheable 双页表别名**。故不能靠"把段搬到 uncacheable 别名"来获得一致性—— +> 必须走上面的硬件原子 + `dcci` 读一致路线。 + +### 16.3 seam → a5 现成原语映射（重构后填入 CCEC 单元） + +| seam(§13/§14) | a5 onboard 落点 | +| ---- | ---- | +| `pto_gm_alloc/free`(AICPU) | host `rtMalloc(RT_MEMORY_HBM)` 预分配 pooled 段(`memory_allocator.cpp`),或 AICPU `halMemAlloc`(`device_malloc.cpp`);段基址经 `runtime->dist.global_data_base` 下发 | +| `pto_gm_publish`(AICPU) | `cache_flush_range`(`dc cvac`+`dsb sy`,`cache_ops.cpp`)刷 `[base,base+bytes)` | +| `pto_dcci_inval`(AICore) | CANN `dcci(p, SINGLE_CACHE_LINE)`(无 `CACHELINE_OUT`,失效本地) | +| `pto_dcci_flush`(AICore) | CANN `dcci(p, SINGLE_CACHE_LINE, CACHELINE_OUT)`(写回 HBM) | +| `pto_shared_fence`(AICore) | `dsb(DSB_DDR)` + `pipe_barrier(PIPE_ALL)`(参见 `pto_async_kernel_api.h`) | +| `pto_core_id()`(AICore) | 逻辑 worker index = `dist_core_main` 的 `core_idx`(= launch `s_block_idx`),**由入口参数携带**,无需读寄存器;物理 id(`get_coreid()&0x0FFF`)仅用于诊断 | + +> 注意 §14.3 粒度风险:`flags[]` 逐字节环,`dcci` 以 cacheline 为单位——需让 flag 按 cacheline 对齐或改 per-core 分片,否则"失效邻居未刷出的写"会丢数据。 + +### 16.4 阶段计划（每阶段带验证闸） + +0. **P0 决策 —— 已完成。** claim 机制定为 **A5 硬件 `atomicMax`**（§11.1.1）：A5(`dav_3510`)有 + 核间一致的 GM 硬件原子，无需 AICPU 仲裁 / 静态分派 / uncacheable 别名。**闸已过**：HW 原子能力 + 结论（`kernel_operator_atomic_impl.h` + CANN 三核 `AtomicMax` 示例）+ 选定方案。 +1. **P1 结构拆分(可 sim 验证)**:seam 抽到独立 header;引擎拆 AICPU/AICore 两单元;a5 `build_config` 把热路径加入 `aicore` `source_dirs`、`register` 留 `aicpu`。**闸**:a5sim + a2a3sim 仍 DEPSIG private==shared、golden 通过(结构不回归)。 +2. **P2 CCEC 编译打通**:HW seam 分支填入(§16.3);`__gm__` 化引擎内 GM 指针;确认无 CCEC 不支持的 STL。**闸**:CCEC 编译 aicore 二进制成功(CI)。 +3. **P3 claim + 读一致性落地**： + - claim 用单条 `atomicMax`（`dist_atomic_max`，已落 `dist_engine.cpp` 的 `#if !DIST_SIM_HOST_CLOCK` claim 分支）； + - global cursor 纯读改幂等 `atomicMax(&cursor, INT32_MIN)`（节流 §6.1 / 诊断）； + - `coherent_load` 的 onboard 分支补 `dcci` 失效再 load（`frontier`/`started_count`/`replay_done`/`flags[]`/`block.won`），RMW 保持 `atomicAdd`/`atomicCAS`。 + **闸**:单核→多核真机小用例(vector_example / mix_coown) barrier 收敛、golden + 无重复认领/丢更新。 +4. **P4 一致性 & 压测**:flag cacheline 对齐(C4，独立于原子能力)、makespan/负载均衡对比 sim。**闸**:真机 paged_attention 等 DEPSIG 与 sim 一致、性能达标。 + +## 17. Open Challenges（A5 onboard 未决项） + +以下是把 SPMD 引擎从 a5sim 推进到 **a5 真机** 尚未解决的问题清单(§16 为对应设计)。分三档:**BLOCKER**(不解决无法在真机跑)、**CORRECTNESS**(能编但真机结果可能错)、**ENG/PROC**(工程或流程)。 + +| # | 挑战 | 档位 | 现状 / 根因 | 待决策 or 落地方向 | 验证依赖 | +| - | ---- | ---- | ---- | ---- | ---- | +| C1 | ~~跨核原子 RMW 缺失~~ **已解除** → 改为**读侧一致性** | ~~BLOCKER~~ CORRECTNESS | **修订**：A5(`dav_3510`)有硬件 GM 原子 `atomicAdd/Max/Min/CAS/Exch`(内存级、核间一致,CANN 文档三核 `AtomicMax` 示例佐证),claim 可用单条 `atomicMax` 实现(§11.1.1)。真机 bug 实为 `coherent_load` 的 onboard 分支是无 `dcci` 的普通 load(误设 uncacheable),读到陈旧缓存 → barrier 挂死。本机 double page table 不可用(探针 `offset=0`),不能靠 uncacheable 别名 | claim/cursor 用 `atomicMax`;cursor 纯读用幂等 `atomicMax(x,INT32_MIN)`;其余 `coherent_load` onboard 分支加 `dcci` 失效再 load(§11.1.1/§16.2) | 真机多核无重复认领 + barrier 收敛 | +| C2 | ~~热路径处理器错位~~ **结构拆分已完成** | ~~BLOCKER~~ 已落地 | **修订**：引擎已拆 AICPU(`register`,`SIMPLER_DIST_AICPU_ONLY`)/AICore(`dist_core_main`,`SIMPLER_DIST_AICORE_ONLY`)两单元；a5 `build_config` 把 `DIST_COMMON` 加入 `aicore` `source_dirs`；`aicore_execute` onboard 分支**直接调用** `dist_core_main`(非函数指针)。CCEC 已把 `dist_engine.cpp` 编成 `dist_engine_aic.o`/`dist_engine_aiv.o` 并链入 `aicore_kernel.o`(797KB)。**剩余的真机故障是 C10(ops 表函数指针重定位)，不再是编译错位** | 已完成；后续问题见 C10 | a5sim/a2a3sim 不回归(a5sim 通过；a2a3sim 见 C11);CCEC 编译成功✅ | +| C3 | **CCEC STL 支持不确定** | ENG(潜在 BLOCKER) | 引擎含 `std::vector/string/chrono`(HW 上已 `#if` 关闭),但需 CCEC 实编确认热路径无残留 STL / 异常 / RTTI 依赖 | 热路径去 STL 化;必要时改固定容量数组 | CCEC 编译通过 | +| C4 | **flag 环 cacheline 粒度 / false sharing** | CORRECTNESS | `flags[]` 逐字节环;`dcci` 以 cacheline 为单位失效/刷出 → "失效邻居未刷出的写"会丢数据 | flag 按 cacheline 对齐,或改 per-core 分片(§14.3 / §16.3 注) | 真机多核压测 | +| C5 | **计数缺配对 dcci** | CORRECTNESS | `runtime->dist.done_count` 的 `__atomic_add_fetch`(`aicore_executor.cpp`)无配对 dcci → 真机跨核语义不可靠(同属 C1 家族) | 计数改经确定的可见性协议(单写者 / dcci 配对 / 归约) | 真机 done_count 收敛 | +| C6 | **`__gm__` 地址空间化** | ENG | sim 上 `__gm__` 为空宏;真机 CCEC 上引擎内所有 GM 指针/段访问需 `__gm__` 限定 | 随 C2 重构给 `DistGlobal`/`DistCore`/段指针加 `__gm__` | CCEC 编译通过 | +| C7 | **GM 段分配 + 发布** | ENG | seam `pto_gm_alloc/publish` 真机分支为 TODO | AICPU 侧 `rtMalloc(RT_MEMORY_HBM)`/`halMemAlloc` + `cache_flush_range` 发布(§16.3) | 真机各核可读到初始化后的段 | +| C8 | **core-id 映射**(低风险,记录在案) | ENG | 逻辑 worker index 已由 `dist_core_main(core_idx)` 入口参数携带(= launch `s_block_idx`),无需读寄存器;物理 id 仅诊断用 | 沿用入口参数;`pto_core_id()` HW 分支返回该入参 | — | +| C9 | **验证缺口:无本地 CCEC/真机** | PROC | ~~本机无 CCEC~~ **修订**：本机 CCEC 可用(`cann-9.1.T500/bin/ccec`)，`build_runtimes --platforms a5` 能出 CCEC aicore 二进制；真机经 `task-submit --device auto` 跑 `vector_example`。仍无本地 npu-smi(缺 `libsecurec.so`，onboard-arch-precheck 需手动确认 a5) | 继续走 `task-submit` + `_run_probe.sh`(自动重编+抓 device log crumbs) | 已可用✅ | +| C10 | ~~ops 表函数指针 incore 重定位~~ **已解除** | ~~BLOCKER~~ 已落地 | **修订**：真机开 `dist_ops_refresh_aicore`+ops 后 `gd->rt->ops->submit_task` 经 `runtime->dist.global_data_base + offsetof(ops)` 的**整数偏移自解析**已可用；`dist_submit_impl` 被正常进入。之前"9 核全 271"的真因是 **ABI 分歧(C13)+`pto_core_id()` 桩(C14)** 联合，与 ops 重定位无关。修 C14 后 9 核 `local_index=5`(各核提交 5 任务)、`core_id` 各不相同(0-8) | 已完成 | 编排能提交任务✅ | +| C11 | **`a2a3sim` 编译回归** | ENG | 共享 `dist_engine.cpp` 引用 `runtime->dist.seg_base/seg_size`、`TensorRef::raw_addr()` —— 这些**仅 a5 定义**(前序会话加的 a5-onboard 段交付逻辑)，a2a3 的 runtime.h 无 → a2a3sim 编译失败 | 给这些 a5-only 字段访问加 `#ifdef`/特征检测隔离，或把段交付逻辑下沉到 a5-only 编译单元 | a2a3sim `Build complete!` | +| C13 | ~~`DistGlobal` ABI 分歧~~ **已解除** | ~~BLOCKER~~ 已落地 | `PTO2_PROFILING` 宏在 AICPU(g++)/AICore(CCEC)不一致 → `L2TaskArgs orch_args_gm` 尺寸差 160B → AICore 读 `gd->rt`=0 → MPU 271。**修复**：把 `orch_args_gm` 移到 `DistGlobal` 末尾，profiling 相关字段不再影响前段 offset | 已完成 | AICore 读到正确 `gd->rt`✅ | +| C14 | ~~编排未在核上执行(`pto_core_id` 桩 + 编排调度错位)~~ **已解除** | ~~BLOCKER~~ 已落地 | 两处联合：(a) `pto_core_id()` HW 分支是恒返回 0 的桩 → 所有核自认 core 0 → SPMD 崩塌；(b) `dist_core_main` onboard 分支**直调弱符号** `aicpu_orchestration_entry`(仅弱声明→链接器解析到镜像基址=空操作)，而真正的编排是 CCEC 编成的**独立 PC-relative blob**(`_compile_orchestration_dist_blob`)，入口在 `gd->orch_func`。**修复**：(a) `pto_set_core_id/pto_core_id` 用 `[[block_local]] static` 存/取入参 `core_idx`；(b) onboard 分支改与 sim 一致——先 `gd->orch_bind_func(gd->rt)` 绑定 blob 自身 `g_current_runtime`，再经 `gd->orch_func(*gd->orch_args)` 调用。移除 `DIST_DIAG_SKIP_ORCH` | 已完成 | 9 核 `local_index=5`、`core_id`=0..8✅ | +| C15 | ~~drain 死锁(flag false-sharing)~~ **已解除** | ~~BLOCKER~~ 已落地 | C14 解除后 `drain` 开始执行 kernel，但 2 核卡 `crumb 33`(非 40/41 → kernel 不挂死、无 DMA 故障)、`done_count=7/9`、507000 超时。诊断证实：卡住核 `occupied_count=1`、`ring_empty=0`，其私有环 slot 的 fanin 生产者 flag 永不置位——**`flags[2]=0` 但结构对称的 `flags[1]=1`**(t1/t2 都是 c+标量、只依赖 t0)。真因 = **C4 false-sharing**：`flags[]` 是 `Coherent` 稠密环，t0..t4 五个 flag 共用一条 64B cacheline，`coherent_store`+dcci **按 cacheline 刷出会把相邻核刚写的 flag clobber 回 0** | flag 改 `int32_t` + 置位走**内存级 `atomicMax(&flags[i],1)`**(`dist_set_flag`)——直写真实 HBM 字、不回写共享行，相邻置位互不 clobber；读侧仍 `coherent_load` dcci 失效 | **9/9 完成、无 507000/异常✅**(仍差数值，见 C16) | +| C16 | **中间 `TensorCreateInfo` 栈局部地址空间损坏(现总闸)** | BLOCKER | C15 解除后 9 核全部完成、无死锁，但 `f` golden `max_diff≈31~51`。**真因已确诊(非缓存一致性)**：上板 dump 逐张量地址显示中间张量间距仅 `0x400`(1024B=256 float)而非 `0x10000`(65536B)——`buffer_size_bytes()` 用了 `ndims=0`(ALIGN_UP(4,1024)=1024)。进一步 dump `dist_submit_impl` 读到的 create_info：`@0x107c78 ndims=0 shape0=0`。根因 = **用户编排里 `TensorCreateInfo inter_ci` 是 CCEC AICore 的栈局部变量**(栈在 local/workspace 空间，`-cce-aicore-stack-size`)，而 `TensorRef` 在 CCEC 用**整数地址 seam**存指针(`addr_=reinterpret_cast(&inter_ci)`)——整数 round-trip **丢失 local 地址空间限定**，重建为 generic 指针后指向错误位置→读出 ndims=0→中间输出被分配成 4B→严重重叠(每张量覆盖后续 63 个,只剩首 256 元素正确)。外部张量(ext_a/ext_f)正常是因为它们 GM 常驻,地址可扁平 round-trip。注释"on-core 一切 GM-resident"的假设**对用户栈 create_info 不成立** | `TensorRef` 对 OUTPUT **按值内嵌** `TensorCreateInfo`(在 `add_output` 时用仍带正确空间限定的指针拷贝,不再存整数地址),`create_info()` 返回内嵌成员(经 `args` 引用做成员访问,空间正确)。host/CCEC 一致内嵌以保 `L2TaskArgs`(orch_args_gm)布局不变 | 真机 `f` golden 通过 | + +> 依赖关系:**C1** claim 机制已定(硬件 `atomicMax`)；**C2** 结构拆分完成；**C10/C13/C14/C15 已解除**——编排提交任务、SPMD id 正确、**drain 无死锁、9/9 完成**。**当前总闸转为 C16**(中间张量数据跨核可见性)——最后一步数值正确性。C16 解除后收 C5(done_count)/C11(a2a3 隔离)。落地顺序见 §16.4 的 P0→P4 阶段闸。 + +### 17.1 本轮上板定位记录（2026-07，MPU 271 收敛） + +真机 `vector_example`(device auto，9 workers = 3 AIC + 6 AIV)三次迭代的证据链： + +1. **已修两处跨核陈旧指针隐患**（`dist_core_main_impl` / `drain_block_won` / `has_pending_won`）： + - `self->gd`（`gd->cores[i].gd`）由 AICPU 写、AICore 用**裸指针 load**读（`Coherent` 只包原子，不含此回指针），cacheable HBM 上可能读到上一轮 segment base 的陈旧值 → `gd->blocks[self->block_id]` 野地址。**修复**：在 `dist_core_main_impl` 用本核已验证的 local `gd` 就地 `self->gd = gd`（同核写读，天然一致）。 + - `block_id` 越界防护：`drain_block_won`/`has_pending_won` 进入前判 `0 <= block_id < num_blocks`。 +2. **故障随 ops/orch 开关移动**（关键 bisection）： + - `DIST_DIAG_SKIP_OPS=1 && SKIP_ORCH=1`：故障在 **drain 循环**（crumb 30），**7/9 存活**。 + - `DIST_DIAG_SKIP_OPS=0 && SKIP_ORCH=0`：故障前移到 **ops 表/编排回放**（crumb 26-27），**9/9 全 271**，pc 收敛到 `0x2600`/`0x2624`。 + - 结论：真正的元凶在 **`dist_ops_refresh_aicore` + `rt->ops` 调用路径**（C10），与 cursor 认领、读一致性无关。 +3. **本轮正确性改动**（cursor `atomicMax`、`coherent_load/store` 补 dcci、`self->gd`、`block_id` 防护）都是必要且正确的，把故障从 drain 路径清掉；但它们不是 C10 的成因。 +4. **复现命令**：`task-submit --device auto --max-time 260 --run "bash tests/st/a5/fully_distributed_within_core/vector_example/_run_probe.sh"`；device log crumbs 在 `~/ascend/log/debug/device-*/`，`progress=[...]` 按 core_idx、`dbg=[...]` 为各核读到的诊断包。诊断开关在 `dist_engine.cpp` 顶部 `DIST_DIAG_SKIP_ORCH/BIND/OPS`（当前全 0 = 完整路径）。 + +### 17.2 编排提交打通里程碑（2026-07，C13/C14 解除） + +`vector_example`（device auto，9 workers）关键突破的证据链： + +1. **C13（ABI 分歧）** 修复后，AICore 不再 271，能读到正确 `gd->rt`；但 `local_index` 全 0（无任务提交）。 +2. **`pto_core_id()` 是桩**：HW 分支恒返回 0（带 TODO）→ 所有核自认 core 0，`dist_submit_impl`/`dist_is_fatal` 全走 core 0 视角，SPMD 崩塌。改用 `[[block_local]] static int32_t` 存 `dist_core_main(core_idx)` 入参后修复。 +3. **编排调度错位（C14 核心）**：onboard `dist_core_main` 直调 `aicpu_orchestration_entry`——但该符号在通用运行时镜像里只有**弱声明**，链接器解析到镜像基址 `0x…be00000`（探针 `dbg99` bits0-31 实测），等于**空操作**；真正的编排是 CCEC 编成的**独立 PC-relative blob**（`kernel_compiler._compile_orchestration_dist_blob`），入口存 `gd->orch_func`（探针实测 `0x…0001d04c`，落在 DevMalloc 上传区）。`DIST_DIAG_SKIP_ORCH=1` 又把直调整个跳过（双重失效）。 +4. **修复**：onboard 分支改与 sim 一致，先 `gd->orch_bind_func(gd->rt)` 绑定 blob 自身 `g_current_runtime`，再 `gd->orch_func(*gd->orch_args)`；删除 `DIST_DIAG_SKIP_ORCH`。（§16.1 的"编排编进镜像/直调符号"方案构建侧尚未接通，故沿用 blob 指针调度——与 kernel blob 一致。） +5. **结果**：9 核 `dbg=[0x50k0001d04c]` 解码 = `local_index=5`（每核提交 5 任务）+ `core_id`=0..8（各不相同）。**编排首次在核上真正提交任务、SPMD id 正确**。随后暴露 C15（kernel 执行/DMA MTE 异常），成为新总闸。 + +## 18. 相关文档 | 文档 | 关联性 | | ---- | ------ | diff --git a/examples/a5/fully_distributed_within_core/paged_attention_unroll/kernels/orchestration/paged_attention_orch.cpp b/examples/a5/fully_distributed_within_core/paged_attention_unroll/kernels/orchestration/paged_attention_orch.cpp index a58c776b39..0dff85555d 100644 --- a/examples/a5/fully_distributed_within_core/paged_attention_unroll/kernels/orchestration/paged_attention_orch.cpp +++ b/examples/a5/fully_distributed_within_core/paged_attention_unroll/kernels/orchestration/paged_attention_orch.cpp @@ -94,6 +94,13 @@ aicpu_orchestration_entry(const L2TaskArgs &orch_args) { // Read dimensions from tensor metadata // query: shape=[batch, num_heads, head_dim] uint64_t batch = orch_args.tensor(0).ref().shapes[0]; +#if PTO_FDWIC_PERF_CLOCK || PTO_FDWIC_SUBMIT_PMU + // perf-clock 与 submit-pmu-none 当前只服务与 Case1 同构的 PA：每个 batch 恰好一次 + // Alloc 和一组 QK/SF/PV/UP，共 5 次 Submit。Case2/3 的 block 分组数 + // 不同；若误用该诊断构建，最终实际 count 会超过 expected，host 必须 + // fail closed，不能把中途第 5*batch 次 Submit 冒充为末次。 + rt_perf_clock_expect_submits(static_cast(5 * batch)); +#endif uint64_t num_heads = orch_args.tensor(0).ref().shapes[1]; uint64_t head_dim = orch_args.tensor(0).ref().shapes[2]; DataType data_type = orch_args.tensor(0).ref().dtype; @@ -172,15 +179,22 @@ aicpu_orchestration_entry(const L2TaskArgs &orch_args) { prof_view_count += 2; CYCLE_COUNT_LAP(prof_tensor_view); #endif - // alloc_tensors() and the four kernel submits synchronously - // consume their Arg. One scope-local container is sufficient; - // reset() repopulates its active slots for each next call. + // Compete-first helpers synchronously run Claim before invoking + // the eager argument builder, then consume params in Finish. + // Every worker still builds the complete argument list; only + // the Claim/argument-construction order changes. L0TaskArgs params; CYCLE_COUNT_LAP(prof_param_setup); - params.add_output(tile2d_ci); - params.add_output(scalar_ci); - params.add_output(scalar_ci); - TaskOutputTensors alloc_outs = alloc_tensors(params); + TaskOutputTensors alloc_outs = alloc_tensors_compete_first( + params, + [&](L0TaskArgs &submit_args) PTO_DEVICE_FUNC { + CYCLE_COUNT_LAP(prof_submit_task); + submit_args.add_output(tile2d_ci); + submit_args.add_output(scalar_ci); + submit_args.add_output(scalar_ci); + CYCLE_COUNT_LAP(prof_param_setup); + } + ); __gm__ const Tensor &oi = alloc_outs.get_ref(0); __gm__ const Tensor &li_update = alloc_outs.get_ref(1); __gm__ const Tensor &mi_update = alloc_outs.get_ref(2); @@ -207,12 +221,17 @@ aicpu_orchestration_entry(const L2TaskArgs &orch_args) { CYCLE_COUNT_LAP(prof_make_tensor); #endif - params.reset(); - params.add_input(qi, key_cache, block_table); - params.add_output(sij_buf_ci); - params.add_scalar(n_blocks, b_idx * block_num + bn); - CYCLE_COUNT_LAP(prof_param_setup); - TaskOutputTensors qk_outs = rt_submit_aic_task(FUNC_QK_MATMUL, params); + TaskOutputTensors qk_outs = rt_submit_aic_task_compete_first( + FUNC_QK_MATMUL, params, + [&](L0TaskArgs &submit_args) PTO_DEVICE_FUNC { + CYCLE_COUNT_LAP(prof_submit_task); + submit_args.reset(); + submit_args.add_input(qi, key_cache, block_table); + submit_args.add_output(sij_buf_ci); + submit_args.add_scalar(n_blocks, b_idx * block_num + bn); + CYCLE_COUNT_LAP(prof_param_setup); + } + ); __gm__ const Tensor &sij_buf = qk_outs.get_ref(0); #ifdef ENABLE_PROFILING prof_submit_count++; @@ -229,12 +248,17 @@ aicpu_orchestration_entry(const L2TaskArgs &orch_args) { CYCLE_COUNT_LAP(prof_make_tensor); #endif - params.reset(); - params.add_input(sij_buf); - params.add_output(pij_buf_ci, scalar_ci, scalar_ci); - params.add_scalar(scale_value, n_blocks, valid_len_last); - CYCLE_COUNT_LAP(prof_param_setup); - TaskOutputTensors sf_outs = rt_submit_aiv_task(FUNC_SOFTMAX_PREPARE, params); + TaskOutputTensors sf_outs = rt_submit_aiv_task_compete_first( + FUNC_SOFTMAX_PREPARE, params, + [&](L0TaskArgs &submit_args) PTO_DEVICE_FUNC { + CYCLE_COUNT_LAP(prof_submit_task); + submit_args.reset(); + submit_args.add_input(sij_buf); + submit_args.add_output(pij_buf_ci, scalar_ci, scalar_ci); + submit_args.add_scalar(scale_value, n_blocks, valid_len_last); + CYCLE_COUNT_LAP(prof_param_setup); + } + ); __gm__ const Tensor &pij_buf = sf_outs.get_ref(0); __gm__ const Tensor &mi = sf_outs.get_ref(1); __gm__ const Tensor &li = sf_outs.get_ref(2); @@ -244,12 +268,17 @@ aicpu_orchestration_entry(const L2TaskArgs &orch_args) { #endif // === Task 3: SplitK PV matmul (accumulated P @ V) === - params.reset(); - params.add_input(pij_buf, value_cache, block_table); - params.add_output(tile2d_ci); - params.add_scalar(n_blocks, b_idx * block_num + bn); - CYCLE_COUNT_LAP(prof_param_setup); - TaskOutputTensors pv_outs = rt_submit_aic_task(FUNC_PV_MATMUL, params); + TaskOutputTensors pv_outs = rt_submit_aic_task_compete_first( + FUNC_PV_MATMUL, params, + [&](L0TaskArgs &submit_args) PTO_DEVICE_FUNC { + CYCLE_COUNT_LAP(prof_submit_task); + submit_args.reset(); + submit_args.add_input(pij_buf, value_cache, block_table); + submit_args.add_output(tile2d_ci); + submit_args.add_scalar(n_blocks, b_idx * block_num + bn); + CYCLE_COUNT_LAP(prof_param_setup); + } + ); __gm__ const Tensor &oi_new = pv_outs.get_ref(0); #ifdef ENABLE_PROFILING prof_submit_count++; @@ -259,13 +288,19 @@ aicpu_orchestration_entry(const L2TaskArgs &orch_args) { // === Task 4: Online update (per-group) === uint64_t is_first = (bn == 0) ? 1 : 0; uint64_t is_last = (bn + n_blocks >= bn_this_batch) ? 1 : 0; - - params.reset(); - params.add_input(mi, li, oi_new); - params.add_inout(mi_update, li_update, oi, out_view); - params.add_scalar(is_first, is_last); CYCLE_COUNT_LAP(prof_param_setup); - rt_submit_aiv_task(FUNC_ONLINE_UPDATE, params); + + rt_submit_aiv_task_compete_first( + FUNC_ONLINE_UPDATE, params, + [&](L0TaskArgs &submit_args) PTO_DEVICE_FUNC { + CYCLE_COUNT_LAP(prof_submit_task); + submit_args.reset(); + submit_args.add_input(mi, li, oi_new); + submit_args.add_inout(mi_update, li_update, oi, out_view); + submit_args.add_scalar(is_first, is_last); + CYCLE_COUNT_LAP(prof_param_setup); + } + ); #ifdef ENABLE_PROFILING prof_submit_count++; CYCLE_COUNT_LAP(prof_submit_task); diff --git a/examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py b/examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py index 9bebfd9be2..aa1e70bf3f 100644 --- a/examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py +++ b/examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py @@ -65,6 +65,22 @@ class TestPagedAttentionUnroll(SceneTestCase): } CASES = [ + { + "name": "CaseB1", + "platforms": ["a5"], + "config": {"aicpu_thread_num": 4}, + "manual": True, + "params": { + "batch": 1, + "num_heads": 16, + "kv_head_num": 1, + "head_dim": 128, + "block_size": 128, + "context_len": 8192, + "max_model_len": 32768, + "dtype": "bfloat16", + }, + }, { "name": "Case1", "platforms": ["a5sim", "a5"], diff --git a/examples/a5/fully_distributed_within_core/simple_orch_smoke/kernels/orchestration/simple_orch.cpp b/examples/a5/fully_distributed_within_core/simple_orch_smoke/kernels/orchestration/simple_orch.cpp index b2206d4ba2..898231debb 100644 --- a/examples/a5/fully_distributed_within_core/simple_orch_smoke/kernels/orchestration/simple_orch.cpp +++ b/examples/a5/fully_distributed_within_core/simple_orch_smoke/kernels/orchestration/simple_orch.cpp @@ -37,6 +37,23 @@ __attribute__((visibility("default"), weak)) PTO_DEVICE_FUNC void aicpu_orchestr uint64_t mixed = orch_args.scalar(2); for (uint64_t i = 0; i < n; i++) { + if (mixed == 3) { + MixedKernels mk; + mk.aic_kernel_id = FUNC_MARK_AIC; + mk.aiv0_kernel_id = FUNC_MARK_AIV; + L0TaskArgs eager_args; + rt_submit_task_compete_first( + mk, eager_args, + [&](L0TaskArgs &submit_args) PTO_DEVICE_FUNC { + submit_args.add_input(input); + submit_args.add_inout(output); + submit_args.add_scalar(n); + submit_args.add_scalar(delta); + submit_args.add_scalar(i); + } + ); + continue; + } L0TaskArgs args; args.add_input(input); args.add_inout(output); diff --git a/examples/a5/fully_distributed_within_core/simple_orch_smoke/test_simple_orch_smoke.py b/examples/a5/fully_distributed_within_core/simple_orch_smoke/test_simple_orch_smoke.py index a4e45fd914..642c1dce55 100644 --- a/examples/a5/fully_distributed_within_core/simple_orch_smoke/test_simple_orch_smoke.py +++ b/examples/a5/fully_distributed_within_core/simple_orch_smoke/test_simple_orch_smoke.py @@ -97,6 +97,12 @@ class TestSimpleOrchSmoke(SceneTestCase): "config": {"aicpu_thread_num": 4, "block_dim": 36}, "params": {"n": 3, "delta": 17, "mixed": 1}, }, + { + "name": "A5SimBd36CompeteFirstMixedDelta47", + "platforms": ["a5sim"], + "config": {"aicpu_thread_num": 4, "block_dim": 36}, + "params": {"n": 3, "delta": 47, "mixed": 3}, + }, { "name": "A5SimBd36RepeatedMixedDelta29", "platforms": ["a5sim"], diff --git a/python/simpler/worker.py b/python/simpler/worker.py index 79cf036d4d..c58f6b1e76 100644 --- a/python/simpler/worker.py +++ b/python/simpler/worker.py @@ -1875,7 +1875,10 @@ def _init_level2(self) -> None: runtime = self._config["runtime"] device_id = self._config.get("device_id", 0) - builder = RuntimeBuilder(platform) + builder = RuntimeBuilder( + platform, + fdwic_tensormap_mode=self._config.get("fdwic_tensormap_mode"), + ) binaries = builder.get_binaries(runtime) aicore_path_override = self._config.get("aicore_path_override") if aicore_path_override: @@ -1911,7 +1914,10 @@ def _init_hierarchical(self) -> None: platform = self._config["platform"] runtime = self._config["runtime"] - builder = RuntimeBuilder(platform) + builder = RuntimeBuilder( + platform, + fdwic_tensormap_mode=self._config.get("fdwic_tensormap_mode"), + ) binaries = builder.get_binaries(runtime) # Stash the full RuntimeBinaries so forked chip children can diff --git a/simpler_setup/build_runtimes.py b/simpler_setup/build_runtimes.py index 4c77c06375..f5223654b3 100644 --- a/simpler_setup/build_runtimes.py +++ b/simpler_setup/build_runtimes.py @@ -11,11 +11,16 @@ Detects available toolchains and builds all runtime binaries using persistent build directories (build/cache/) for incremental compilation. -Final binaries are placed in build/lib/{arch}/{variant}/{runtime}/. +Final binaries are placed in build/lib/{arch}/{variant}/{runtime}/. A5 FDWIC +uses one additional private/shared mode directory below the runtime. Usage: python simpler_setup/build_runtimes.py # auto-detect platforms python simpler_setup/build_runtimes.py --platforms a2a3sim # build specific platform + python simpler_setup/build_runtimes.py --platforms a5sim \ + --fdwic-tensormap shared # build shared FDWIC baseline + python simpler_setup/build_runtimes.py --platforms a5sim \ + --fdwic-tensormap private --fdwic-tensormap shared # build both FDWIC baselines python simpler_setup/build_runtimes.py --list # list buildable platforms """ @@ -42,6 +47,63 @@ logger = logging.getLogger(__name__) +_FDWIC_RUNTIME = "fully_distributed_within_core" +_FDWIC_TENSORMAP_MODES = ("private", "shared") + + +def _is_a5_fdwic(platform: str, runtime_name: str) -> bool: + """Return whether this task has the selectable TensorMap implementation.""" + return parse_platform(platform)[0] == "a5" and runtime_name == _FDWIC_RUNTIME + + +def _normalize_fdwic_tensormap_modes(modes: Optional[list[str]]) -> tuple[str, ...]: + """Validate and de-duplicate explicitly requested FDWIC build modes. + + The install/pre-build entry point always defaults to ``private`` so wheel + contents do not depend on ambient shell state. ``shared`` is built only + through the explicit CLI/API request. + """ + requested = ["private"] if modes is None else modes + if isinstance(requested, str): + raise ValueError("fdwic_tensormap_modes must be a list of 'private'/'shared' values") + + normalized: list[str] = [] + for raw_mode in requested: + if not isinstance(raw_mode, str) or raw_mode.strip().lower() not in _FDWIC_TENSORMAP_MODES: + choices = ", ".join(_FDWIC_TENSORMAP_MODES) + raise ValueError(f"Invalid FDWIC TensorMap mode {raw_mode!r}; expected one of: {choices}") + mode = raw_mode.strip().lower() + if mode not in normalized: + normalized.append(mode) + if not normalized: + raise ValueError("fdwic_tensormap_modes must contain at least one mode") + return tuple(normalized) + + +def _collect_runtime_build_tasks( + platforms: list[str], + fdwic_tensormap_modes: Optional[list[str]], +) -> list[tuple[str, str, str]]: + """Collect builds while applying TensorMap mode only to A5 FDWIC.""" + tasks: list[tuple[str, str, str]] = [] + selected_fdwic_modes: Optional[tuple[str, ...]] = None + for platform in platforms: + arch, _ = parse_platform(platform) + runtimes = discover_runtimes(arch) + + if not runtimes: + logger.warning(f" {platform}: no runtimes found, skipping") + continue + + for runtime_name in runtimes: + if arch == "a5" and runtime_name == _FDWIC_RUNTIME: + if selected_fdwic_modes is None: + selected_fdwic_modes = _normalize_fdwic_tensormap_modes(fdwic_tensormap_modes) + tasks.extend((platform, runtime_name, mode) for mode in selected_fdwic_modes) + else: + tasks.append((platform, runtime_name, "private")) + return tasks + def detect_buildable_platforms() -> list: """Detect which platforms can be built with available toolchains. @@ -78,6 +140,7 @@ def build_all( clone_protocol: str = "ssh", sanitizer: str = "none", pto_isa_commit: Optional[str] = None, + fdwic_tensormap_modes: Optional[list[str]] = None, ) -> None: """Build all runtime variants for the given platforms. @@ -94,6 +157,10 @@ def build_all( pto_isa_commit: optional pto-isa commit override for the onboard a2a3 host build. None preserves the original behavior and uses the current checkout HEAD. + fdwic_tensormap_modes: FDWIC baseline modes to pre-build. ``None`` + defaults to ``private``. Passing ``["private", "shared"]`` builds + both isolated artifact families. This selection applies only to the A5 + ``fully_distributed_within_core`` runtime. """ # Override default paths to respect CLI args RuntimeBuilder._LIB_DIR = lib_dir @@ -143,7 +210,7 @@ def build_all( if platforms: logger.info("Building simpler_log (process-global)...") try: - RuntimeBuilder(platform=platforms[0]).ensure_simpler_log(build=True) + RuntimeBuilder(platform=platforms[0], fdwic_tensormap_mode="private").ensure_simpler_log(build=True) except Exception as e: logger.error(f"Failed to build simpler_log: {e}") raise @@ -152,42 +219,36 @@ def build_all( if sim_platforms: logger.info("Building cpu_sim_context (process-global)...") try: - RuntimeBuilder(platform=sim_platforms[0]).ensure_sim_context(build=True) + RuntimeBuilder(platform=sim_platforms[0], fdwic_tensormap_mode="private").ensure_sim_context(build=True) except Exception as e: logger.error(f"Failed to build cpu_sim_context: {e}") raise - # Collect all (platform, runtime_name) tasks to run in parallel - tasks: list[tuple[str, str]] = [] - for platform in platforms: - arch, _ = parse_platform(platform) - runtimes = discover_runtimes(arch) - - if not runtimes: - logger.warning(f" {platform}: no runtimes found, skipping") - continue + # Collect all (platform, runtime_name, FDWIC mode) tasks to run in + # parallel. Non-FDWIC tasks always receive an explicit private identity, + # so a shell-wide shared setting cannot leak into an unrelated builder. + tasks = _collect_runtime_build_tasks(platforms, fdwic_tensormap_modes) - for runtime_name in runtimes: - tasks.append((platform, runtime_name)) - - def _build_runtime(platform: str, runtime_name: str) -> None: + def _build_runtime(platform: str, runtime_name: str, fdwic_tensormap_mode: str) -> None: try: - builder = RuntimeBuilder(platform=platform) + builder = RuntimeBuilder(platform=platform, fdwic_tensormap_mode=fdwic_tensormap_mode) except (ValueError, FileNotFoundError) as e: logger.warning(f" {platform}: cannot initialize builder: {e}") return - logger.info(f" Building {platform}/{runtime_name}...") + mode_suffix = f"/{fdwic_tensormap_mode}" if _is_a5_fdwic(platform, runtime_name) else "" + logger.info(f" Building {platform}/{runtime_name}{mode_suffix}...") builder.get_binaries(runtime_name, build=True) with ThreadPoolExecutor(max_workers=len(tasks) or 1) as executor: - futures = {executor.submit(_build_runtime, p, r): (p, r) for p, r in tasks} + futures = {executor.submit(_build_runtime, p, r, m): (p, r, m) for p, r, m in tasks} for future in as_completed(futures): - platform, runtime_name = futures[future] + platform, runtime_name, fdwic_tensormap_mode = futures[future] try: future.result() except Exception as e: - logger.error(f" Failed to build {platform}/{runtime_name}: {e}") + mode_suffix = f"/{fdwic_tensormap_mode}" if _is_a5_fdwic(platform, runtime_name) else "" + logger.error(f" Failed to build {platform}/{runtime_name}{mode_suffix}: {e}") executor.shutdown(wait=True, cancel_futures=True) raise @@ -253,6 +314,18 @@ def main(): "a2a3 host_runtime. Default/latest: use the current checkout HEAD." ), ) + parser.add_argument( + "--fdwic-tensormap", + dest="fdwic_tensormap_modes", + action="append", + choices=_FDWIC_TENSORMAP_MODES, + default=None, + help=( + "FDWIC TensorMap baseline to pre-build (private or shared). " + "Repeat the option to build both. If omitted, build private. " + "The choice applies only to A5 fully_distributed_within_core." + ), + ) args = parser.parse_args() logging.basicConfig( @@ -279,6 +352,7 @@ def main(): clone_protocol=args.clone_protocol, sanitizer=args.sanitizer, pto_isa_commit=args.pto_isa_commit, + fdwic_tensormap_modes=args.fdwic_tensormap_modes, ) diff --git a/simpler_setup/fdwic_build_config.py b/simpler_setup/fdwic_build_config.py new file mode 100644 index 0000000000..3a2bfcc764 --- /dev/null +++ b/simpler_setup/fdwic_build_config.py @@ -0,0 +1,18 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Single-source Python build constants for the A5 FDWIC TensorMap.""" + +FDWIC_TENSORMAP_RING_CAP_DEFINITION = "PTO_FDWIC_TENSORMAP_RING_CAP" +FDWIC_TENSORMAP_RING_CAP = 128 + + +def fdwic_tensormap_ring_cap_definition() -> str: + """Return the compile definition shared by all FDWIC artifact families.""" + + return f"{FDWIC_TENSORMAP_RING_CAP_DEFINITION}={FDWIC_TENSORMAP_RING_CAP}" diff --git a/simpler_setup/kernel_compiler.py b/simpler_setup/kernel_compiler.py index 2ea17b66e4..641391e5a3 100644 --- a/simpler_setup/kernel_compiler.py +++ b/simpler_setup/kernel_compiler.py @@ -9,6 +9,7 @@ import importlib.util import logging import os +import re import subprocess import sys import tempfile @@ -28,6 +29,8 @@ logger = logging.getLogger(__name__) +_COMPILE_DEFINITION_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + class KernelCompiler: """ @@ -388,6 +391,7 @@ def compile_orchestration( source_path: str, extra_include_dirs: Optional[list[str]] = None, build_dir: Optional[str] = None, + compile_definitions: Optional[list[str]] = None, ) -> bytes: """Compile an orchestration function for the given runtime. @@ -400,6 +404,9 @@ def compile_orchestration( source_path: Path to orchestration source file (.cpp) extra_include_dirs: Additional include directories (merged with the runtime/platform include dirs) + build_dir: Optional directory for the compiler output + compile_definitions: Optional preprocessor definitions. Each item + is forwarded as one ``-D`` argument. Returns: Binary contents of the compiled orchestration .so file @@ -443,8 +450,28 @@ def compile_orchestration( extra_include_dirs=include_dirs, extra_sources=orch_sources or None, build_dir=build_dir, + compile_definitions=compile_definitions, ) + @staticmethod + def _compile_definition_flags(compile_definitions: Optional[list[str]]) -> list[str]: + """Validate per-callable macros and return direct compiler arguments.""" + if compile_definitions is None: + return [] + + flags: list[str] = [] + for definition in compile_definitions: + if not isinstance(definition, str) or not definition or definition != definition.strip(): + raise ValueError("compile definitions must be non-empty strings without surrounding whitespace") + if "\0" in definition: + raise ValueError(f"invalid compile definition: {definition!r}") + + name = definition.split("=", 1)[0] + if not _COMPILE_DEFINITION_NAME_RE.fullmatch(name): + raise ValueError(f"invalid compile definition name: {name!r}") + flags.append(f"-D{definition}") + return flags + def _compile_orchestration_shared_lib( self, source_path: str, @@ -452,6 +479,7 @@ def _compile_orchestration_shared_lib( extra_include_dirs: Optional[list[str]] = None, extra_sources: Optional[list[str]] = None, build_dir: Optional[str] = None, + compile_definitions: Optional[list[str]] = None, ) -> bytes: """Compile an orchestration function to a shared library (.so). @@ -462,6 +490,9 @@ def _compile_orchestration_shared_lib( toolchain: Resolved toolchain object (GxxToolchain or Aarch64GxxToolchain) extra_include_dirs: Additional include directories extra_sources: Additional source files to compile into the SO + build_dir: Optional directory for the compiler output + compile_definitions: Optional preprocessor definitions. Each item + is forwarded as one ``-D`` argument. Returns: Binary contents of the compiled .so file @@ -477,6 +508,7 @@ def _compile_orchestration_shared_lib( cmd = [toolchain.cxx_path] + toolchain.get_compile_flags() cmd += self._sanitizer_flags(toolchain) + cmd += self._compile_definition_flags(compile_definitions) # Force a deterministic ELF GNU Build-ID into every orchestration .so. # The host-side DeviceRunner reads `.note.gnu.build-id` to detect when diff --git a/simpler_setup/runtime_builder.py b/simpler_setup/runtime_builder.py index dc37c0f926..6bbb238bb6 100644 --- a/simpler_setup/runtime_builder.py +++ b/simpler_setup/runtime_builder.py @@ -19,6 +19,7 @@ from typing import Optional from .environment import PROJECT_ROOT +from .fdwic_build_config import fdwic_tensormap_ring_cap_definition from .platform_info import TARGETS, load_build_config, parse_platform from .runtime_compiler import RuntimeCompiler @@ -26,6 +27,10 @@ _GIT_COMMIT_FILE = ".git_commit" _SOURCE_STATE_VERSION = "source-v2" +_FDWIC_RUNTIME = "fully_distributed_within_core" +_FDWIC_TENSORMAP_MODE_ENV = "PTO_FDWIC_TENSORMAP_MODE" +_FDWIC_TENSORMAP_MODES = frozenset({"private", "shared"}) +_FDWIC_SHARED_MAP_DEFINITION = "PTO_FDWIC_SHARED_MAP" _SOURCE_FINGERPRINT_SUFFIXES = { ".c", ".cc", @@ -148,15 +153,31 @@ class RuntimeBuilder: _COMPDB_RUNTIME = "tensormap_and_ringbuffer" _COMPDB_VARIANT = "onboard" - def __init__(self, platform: str = "a2a3"): + def __init__(self, platform: str = "a2a3", fdwic_tensormap_mode: Optional[str] = None): """ Initialize RuntimeBuilder with platform selection. Args: platform: Target platform ("a2a3", "a2a3sim", "a5", or "a5sim") + fdwic_tensormap_mode: TensorMap implementation selected for the + A5 fully_distributed_within_core runtime. ``None`` reads + ``PTO_FDWIC_TENSORMAP_MODE`` and falls back to ``private``. """ self.platform = platform self._arch, self._variant = parse_platform(platform) + selected_mode = ( + os.environ.get(_FDWIC_TENSORMAP_MODE_ENV, "private") + if fdwic_tensormap_mode is None + else fdwic_tensormap_mode + ) + if not isinstance(selected_mode, str) or selected_mode.strip().lower() not in _FDWIC_TENSORMAP_MODES: + choices = ", ".join(sorted(_FDWIC_TENSORMAP_MODES)) + raise ValueError(f"Invalid FDWIC TensorMap mode {selected_mode!r}; expected one of: {choices}") + self.fdwic_tensormap_mode = selected_mode.strip().lower() + if self.fdwic_tensormap_mode == "shared" and self._arch != "a5": + raise ValueError( + "FDWIC TensorMap mode 'shared' is only valid for the a5/a5sim fully_distributed_within_core runtime" + ) runtime_root = PROJECT_ROOT self.runtime_root = runtime_root @@ -187,6 +208,76 @@ def _validate_runtime(self, name: str) -> None: f"Note: Different platforms may support different runtimes. " f"Check {self.runtime_dir} for available implementations." ) + if self.fdwic_tensormap_mode == "shared" and not self._is_fdwic_runtime(name): + raise ValueError( + "FDWIC TensorMap mode 'shared' is only valid for the a5/a5sim " + "fully_distributed_within_core runtime; " + f"got platform={self.platform!r}, runtime={name!r}" + ) + + def _is_fdwic_runtime(self, name: str) -> bool: + """Return whether ``name`` is the A5 runtime with selectable TensorMap.""" + return self._arch == "a5" and name == _FDWIC_RUNTIME + + def _runtime_artifact_dir(self, root: Path, name: str) -> Path: + """Return the cache/output root for one runtime build identity.""" + path = root / self._arch / self._variant / name + if self._is_fdwic_runtime(name): + path /= self.fdwic_tensormap_mode + return path + + @staticmethod + def _merge_compile_definitions(*definition_groups: Optional[list[str]]) -> list[str]: + """Merge CMake definitions while rejecting conflicting macro values.""" + merged: list[str] = [] + by_name: dict[str, str] = {} + for definitions in definition_groups: + for raw_definition in definitions or []: + if not isinstance(raw_definition, str) or not raw_definition.strip(): + raise ValueError(f"Invalid compile definition: {raw_definition!r}") + definition = raw_definition.strip() + name = definition.split("=", 1)[0].strip() + if not name: + raise ValueError(f"Invalid compile definition: {raw_definition!r}") + previous = by_name.get(name) + if previous is not None: + if previous != definition: + raise ValueError(f"Conflicting compile definitions for {name}: {previous!r} and {definition!r}") + continue + by_name[name] = definition + merged.append(definition) + return merged + + def effective_compile_definitions( + self, + name: str, + compile_definitions: Optional[list[str]] = None, + ) -> list[str]: + """Return target definitions for ``name`` including its build mode.""" + mode_definitions: list[str] = [] + if self._is_fdwic_runtime(name): + shared_value = 1 if self.fdwic_tensormap_mode == "shared" else 0 + mode_definitions.append(f"{_FDWIC_SHARED_MAP_DEFINITION}={shared_value}") + mode_definitions.append(fdwic_tensormap_ring_cap_definition()) + return self._merge_compile_definitions(mode_definitions, compile_definitions) + + @staticmethod + def _source_state( + current_commit: str, + fingerprint_paths: list[Path], + compile_definitions: list[str], + ) -> str: + """Build a cache identity from revision, contents, and CMake macros.""" + definitions_state = hashlib.sha256(repr(compile_definitions).encode("utf-8")).hexdigest() + return ( + _SOURCE_STATE_VERSION + + ":" + + current_commit + + ":" + + _source_fingerprint(fingerprint_paths) + + ":" + + definitions_state + ) def _resolve_target_dirs(self, config_dir: Path, build_config: dict, target: str): """Resolve include and source dirs for a target from build_config.""" @@ -288,7 +379,7 @@ def get_binaries(self, name: str, build: bool = False) -> RuntimeBinaries: self._validate_runtime(name) arch, variant = self._arch, self._variant - output_dir = self._LIB_DIR / arch / variant / name + output_dir = self._runtime_artifact_dir(self._LIB_DIR, name) # Per-arch shared destination for libsimpler_aicpu_dispatcher.so. The # dispatcher has no runtime-specific code, so all runtimes on a given # arch reuse the same SO instead of carrying a copy each (~50 KB × N). @@ -305,12 +396,21 @@ def get_binaries(self, name: str, build: bool = False) -> RuntimeBinaries: compiler = self._runtime_compiler current_commit = _get_git_head(PROJECT_ROOT) + effective_compile_definitions = self.effective_compile_definitions(name) def _compile_target(target: str) -> Path: include_dirs, source_dirs = self._resolve_target_dirs(config_dir, build_config, target) # compile() adds a {target}/ subdirectory inside build_dir - cache_dir = self._CACHE_DIR / arch / variant / name + cache_dir = self._runtime_artifact_dir(self._CACHE_DIR, name) cache_dir.mkdir(parents=True, exist_ok=True) + current_state = current_commit + if self._is_fdwic_runtime(name): + fingerprint_paths = [*(Path(p) for p in include_dirs), *(Path(p) for p in source_dirs)] + current_state = self._source_state( + current_commit, + fingerprint_paths, + effective_compile_definitions, + ) # File lock to prevent concurrent cmake runs in the same build dir. # Each target gets its own lock so host/aicpu/aicore build in parallel, @@ -318,7 +418,10 @@ def _compile_target(target: str) -> Path: lock_path = cache_dir / f".{target}.lock" with open(lock_path, "w") as lock_fd: fcntl.flock(lock_fd, fcntl.LOCK_EX) - _invalidate_cache_if_stale(cache_dir / target, current_commit) + _invalidate_cache_if_stale(cache_dir / target, current_state) + compile_kwargs = {} + if effective_compile_definitions: + compile_kwargs["compile_definitions"] = effective_compile_definitions return compiler.compile( # type: ignore[return-value] target, include_dirs, @@ -326,6 +429,7 @@ def _compile_target(target: str) -> Path: build_dir=str(cache_dir), output_dir=output_dir, dispatcher_dest=dispatcher_staging_dir if target == "aicpu" else None, + **compile_kwargs, ) logger.info("Compiling AICore, AICPU, Host in parallel...") @@ -369,15 +473,19 @@ def build_aicore_with_extra_sources( extra_sources: list[Path], cache_key: str, pto_isa_root: Optional[str] = None, + compile_definitions: Optional[list[str]] = None, ) -> Path: """Build a per-callable AICore image with additional source files. This is used by fully_distributed_within_core while moving per-example orchestration into the AICore image. It intentionally stages the result outside the baseline runtime output directory so `build_runtimes.py` - remains a per-runtime prebuild. + remains a per-runtime prebuild. ``compile_definitions`` is part of both + the CMake command and cache fingerprint, so evidence profiles cannot + silently reuse an image built with different preprocessor gates. """ self._validate_runtime(name) + effective_compile_definitions = self.effective_compile_definitions(name, compile_definitions) config_path = self._runtimes[name] config_dir = config_path.parent build_config = load_build_config(config_path) @@ -400,15 +508,16 @@ def build_aicore_with_extra_sources( pto_pto_include = os.path.join(pto_isa_root, "include", "pto") include_dirs.extend([pto_include, pto_pto_include]) - arch, variant = self._arch, self._variant - cache_dir = self._CACHE_DIR / arch / variant / name / "aicore-extra" / cache_key - output_dir = self._LIB_DIR / arch / variant / name / "aicore-extra" / cache_key + runtime_cache_dir = self._runtime_artifact_dir(self._CACHE_DIR, name) + runtime_output_dir = self._runtime_artifact_dir(self._LIB_DIR, name) + cache_dir = runtime_cache_dir / "aicore-extra" / cache_key + output_dir = runtime_output_dir / "aicore-extra" / cache_key cache_dir.mkdir(parents=True, exist_ok=True) output_dir.mkdir(parents=True, exist_ok=True) current_commit = _get_git_head(PROJECT_ROOT) fingerprint_paths = [*(Path(p) for p in include_dirs), *(Path(p) for p in source_dirs), *extra_sources] - current_state = _SOURCE_STATE_VERSION + ":" + current_commit + ":" + _source_fingerprint(fingerprint_paths) + current_state = self._source_state(current_commit, fingerprint_paths, effective_compile_definitions) lock_path = cache_dir / ".aicore-extra.lock" with open(lock_path, "w") as lock_fd: fcntl.flock(lock_fd, fcntl.LOCK_EX) @@ -420,6 +529,7 @@ def build_aicore_with_extra_sources( build_dir=str(cache_dir), output_dir=output_dir, source_files=[str(p) for p in extra_sources], + compile_definitions=effective_compile_definitions, ) def _resolve_dispatcher_path(self) -> Optional[Path]: @@ -520,8 +630,9 @@ def _place_compile_commands(self, runtime_name: str) -> None: """ arch, variant = self._arch, self._variant entries = [] + runtime_cache_dir = self._runtime_artifact_dir(self._CACHE_DIR, runtime_name) for target in TARGETS: - cc = self._CACHE_DIR / arch / variant / runtime_name / target / "compile_commands.json" + cc = runtime_cache_dir / target / "compile_commands.json" if cc.exists(): try: entries.extend(json.loads(cc.read_text())) diff --git a/simpler_setup/runtime_compiler.py b/simpler_setup/runtime_compiler.py index c6a9c70fe2..04e290e62e 100644 --- a/simpler_setup/runtime_compiler.py +++ b/simpler_setup/runtime_compiler.py @@ -46,8 +46,9 @@ def gen_cmake_args( source_dirs: list[str], sanitizers: str = "", source_files: Optional[list[str]] = None, + compile_definitions: Optional[list[str]] = None, ) -> list[str]: - """Generate CMake arguments list from toolchain args + custom directories.""" + """Generate CMake arguments from toolchain, source and per-build definitions.""" inc = ";".join(os.path.abspath(d) for d in include_dirs) src = ";".join(os.path.abspath(d) for d in source_dirs) args = self.toolchain.get_cmake_args() + [ @@ -57,6 +58,9 @@ def gen_cmake_args( if source_files: files = ";".join(os.path.abspath(f) for f in source_files) args.append(f"-DCUSTOM_SOURCE_FILES={files}") + if compile_definitions: + definitions = ";".join(compile_definitions) + args.append(f"-DCUSTOM_COMPILE_DEFINITIONS={definitions}") # Sanitizers only apply to host-compiled targets — device toolchains # (ccec, aarch64 cross) run on the NPU and can't carry a host sanitizer # runtime. cmake/sanitizers.cmake reads both defines. @@ -231,6 +235,7 @@ def compile( output_dir: Optional[Union[str, Path]] = None, dispatcher_dest: Optional[Union[str, Path]] = None, source_files: Optional[list[str]] = None, + compile_definitions: Optional[list[str]] = None, ) -> Union[bytes, Path]: """ Compile binary for the specified target platform. @@ -248,6 +253,9 @@ def compile( When None, the dispatcher SO is not exported. Used by runtime_builder to share one dispatcher SO across all runtimes for a given arch. + source_files: Additional translation units outside source_dirs. + compile_definitions: Definitions forwarded to every translation unit + of this CMake target, without a leading ``-D``. Returns: If output_dir is set: Path to the compiled binary in output_dir. @@ -272,6 +280,7 @@ def compile( source_dirs, sanitizers=self._sanitizers, source_files=source_files, + compile_definitions=compile_definitions, ) cmake_source_dir = target.get_root_dir() binary_name = target.get_binary_name() diff --git a/simpler_setup/scene_test.py b/simpler_setup/scene_test.py index fbe8466f21..991fae394f 100644 --- a/simpler_setup/scene_test.py +++ b/simpler_setup/scene_test.py @@ -25,20 +25,215 @@ import gc import hashlib import inspect +import json import logging import os +import subprocess import sys from contextlib import contextmanager from pathlib import Path -from typing import Any, NamedTuple +from typing import TYPE_CHECKING, Any, NamedTuple +from .fdwic_build_config import fdwic_tensormap_ring_cap_definition from .log_config import DEFAULT_LOG_LEVEL, LOG_LEVEL_CHOICES, configure_logging from .pto_isa import ensure_pto_isa_root +if TYPE_CHECKING: + from .tools.fdwic_submit_pmu_report import SubmitPmuBuildIdentity + logger = logging.getLogger(__name__) -_compile_cache: dict[tuple[str, str, str], object] = {} -_aicore_override_cache: dict[tuple[str, str, str], Path] = {} +_compile_cache: dict[tuple[Any, ...], object] = {} +_aicore_override_cache: dict[tuple[Any, ...], Path] = {} +_fdwic_build_identity_cache: dict[tuple[Any, ...], SubmitPmuBuildIdentity] = {} + +_FDWIC_TENSORMAP_MODE_ENV = "PTO_FDWIC_TENSORMAP_MODE" +_FDWIC_TENSORMAP_PRIVATE = "private" +_FDWIC_TENSORMAP_SHARED = "shared" +_FDWIC_TENSORMAP_MODES = frozenset({_FDWIC_TENSORMAP_PRIVATE, _FDWIC_TENSORMAP_SHARED}) +_FDWIC_PROFILE_ENV = "PTO_FDWIC_PROFILE" +_FDWIC_PROFILE_NONE = "none" +_FDWIC_PROFILE_PERF_CLOCK = "perf-clock" +_FDWIC_PROFILE_PERF_CLOCK_KERNEL = "perf-clock-kernel" +_FDWIC_PROFILE_SUBMIT_PMU_NONE = "submit-pmu-none" +_FDWIC_PROFILE_SUBMIT_PMU_ARG_BUILD = "submit-pmu-arg-build" +_FDWIC_PROFILE_SUBMIT_PMU_EMPTY_BRACKET = "submit-pmu-empty-bracket" +_FDWIC_PROFILE_SUBMIT_PMU_MATERIALIZE = "submit-pmu-materialize" +_FDWIC_PROFILE_SUBMIT_PMU_CLAIM = "submit-pmu-claim" +_FDWIC_PROFILE_SUBMIT_PMU_REGISTER = "submit-pmu-register" +_FDWIC_PROFILE_SUBMIT_PMU_SUBMIT_TRANSITION = "submit-pmu-submit-transition" +_FDWIC_PROFILE_SUBMIT_PMU_EFDRAIN_CONTROL = "submit-pmu-efdrain-control" +_FDWIC_PROFILE_SUBMIT_PMU_PREPARE_MAP = "submit-pmu-prepare-map" +_FDWIC_PROFILE_SUBMIT_PMU_FANIN = "submit-pmu-fanin" +_FDWIC_PROFILE_SUBMIT_PMU_WINNER_BUILD_CONTROL = "submit-pmu-winner-build-control" +_FDWIC_PROFILE_SUBMIT_PMU_ALLOC_COMPLETE_CONTROL = "submit-pmu-alloc-complete-control" +_FDWIC_PROFILE_SUBMIT_PMU_LOSER_REPLAY = "submit-pmu-loser-replay" +_FDWIC_PERF_CLOCK_ARTIFACTS = { + _FDWIC_PROFILE_PERF_CLOCK: ("fdwic_perf_clock_summary.json", "fdwic-perf-clock-v1"), + _FDWIC_PROFILE_PERF_CLOCK_KERNEL: ( + "fdwic_perf_clock_kernel_summary.json", + "fdwic-perf-clock-kernel-v1", + ), +} +_FDWIC_SUBMIT_PMU_PHASE_PROFILES = frozenset( + { + _FDWIC_PROFILE_SUBMIT_PMU_ARG_BUILD, + _FDWIC_PROFILE_SUBMIT_PMU_EMPTY_BRACKET, + _FDWIC_PROFILE_SUBMIT_PMU_MATERIALIZE, + _FDWIC_PROFILE_SUBMIT_PMU_CLAIM, + _FDWIC_PROFILE_SUBMIT_PMU_REGISTER, + _FDWIC_PROFILE_SUBMIT_PMU_SUBMIT_TRANSITION, + _FDWIC_PROFILE_SUBMIT_PMU_EFDRAIN_CONTROL, + _FDWIC_PROFILE_SUBMIT_PMU_PREPARE_MAP, + _FDWIC_PROFILE_SUBMIT_PMU_FANIN, + _FDWIC_PROFILE_SUBMIT_PMU_WINNER_BUILD_CONTROL, + _FDWIC_PROFILE_SUBMIT_PMU_ALLOC_COMPLETE_CONTROL, + _FDWIC_PROFILE_SUBMIT_PMU_LOSER_REPLAY, + } +) +_FDWIC_SUBMIT_PMU_PROFILES = frozenset({_FDWIC_PROFILE_SUBMIT_PMU_NONE, *_FDWIC_SUBMIT_PMU_PHASE_PROFILES}) +_FDWIC_PERF_CLOCK_PROFILES = frozenset(_FDWIC_PERF_CLOCK_ARTIFACTS) +_FDWIC_ISOLATED_PROFILES = frozenset({*_FDWIC_PERF_CLOCK_PROFILES, *_FDWIC_SUBMIT_PMU_PROFILES}) +_FDWIC_PROFILES = frozenset({_FDWIC_PROFILE_NONE, *_FDWIC_ISOLATED_PROFILES}) + + +def _fdwic_tensormap_mode() -> str: + mode = os.environ.get(_FDWIC_TENSORMAP_MODE_ENV, _FDWIC_TENSORMAP_PRIVATE) or _FDWIC_TENSORMAP_PRIVATE + if mode not in _FDWIC_TENSORMAP_MODES: + raise ValueError(f"Unsupported {_FDWIC_TENSORMAP_MODE_ENV}={mode!r}") + return mode + + +def _validate_fdwic_tensormap_test_classes(mode: str, classes) -> None: + """Keep standalone shared selection on the same L2 FDWIC scope as pytest.""" + if mode != _FDWIC_TENSORMAP_SHARED: + return + incompatible = sorted( + cls.__name__ + for cls in classes + if getattr(cls, "_st_level", None) != 2 or getattr(cls, "_st_runtime", None) != "fully_distributed_within_core" + ) + if incompatible: + raise ValueError( + "--fdwic-tensormap shared only accepts level-2 " + "fully_distributed_within_core tests; incompatible class(es): " + ", ".join(incompatible) + ) + + +def _fdwic_tensormap_compile_definitions(platform: str, runtime: str) -> list[str] | None: + """Return the explicit FDWIC mode macro for a mode-aware translation unit.""" + mode = _fdwic_tensormap_mode() + is_fdwic = platform in {"a5", "a5sim"} and runtime == "fully_distributed_within_core" + if mode == _FDWIC_TENSORMAP_SHARED and not is_fdwic: + raise ValueError( + f"{_FDWIC_TENSORMAP_MODE_ENV}=shared is only supported by " + "the a5/a5sim fully_distributed_within_core runtime" + ) + if not is_fdwic: + return None + return [ + f"PTO_FDWIC_SHARED_MAP={1 if mode == _FDWIC_TENSORMAP_SHARED else 0}", + fdwic_tensormap_ring_cap_definition(), + ] + + +def _fdwic_profile() -> str: + profile = os.environ.get(_FDWIC_PROFILE_ENV, _FDWIC_PROFILE_NONE) or _FDWIC_PROFILE_NONE + if profile not in _FDWIC_PROFILES: + raise ValueError(f"Unsupported {_FDWIC_PROFILE_ENV}={profile!r}") + return profile + + +def _fdwic_compile_definitions(profile: str) -> list[str] | None: + """Return the ABI-preserving compile gates for one private FDWIC image.""" + if profile == _FDWIC_PROFILE_PERF_CLOCK: + return ["PTO_FDWIC_PERF_CLOCK=1", "PTO_FDWIC_TRACE_ENABLED=0"] + if profile == _FDWIC_PROFILE_PERF_CLOCK_KERNEL: + return [ + "PTO_FDWIC_PERF_CLOCK=1", + "PTO_FDWIC_PERF_CLOCK_KERNEL=1", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + if profile == _FDWIC_PROFILE_SUBMIT_PMU_NONE: + return ["PTO_FDWIC_SUBMIT_PMU=1", "PTO_FDWIC_TRACE_ENABLED=0"] + if profile == _FDWIC_PROFILE_SUBMIT_PMU_ARG_BUILD: + return [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=1", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + if profile == _FDWIC_PROFILE_SUBMIT_PMU_EMPTY_BRACKET: + return [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=2", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + if profile == _FDWIC_PROFILE_SUBMIT_PMU_MATERIALIZE: + return [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=3", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + if profile == _FDWIC_PROFILE_SUBMIT_PMU_CLAIM: + return [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=4", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + if profile == _FDWIC_PROFILE_SUBMIT_PMU_REGISTER: + return [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=5", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + if profile == _FDWIC_PROFILE_SUBMIT_PMU_SUBMIT_TRANSITION: + return [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=6", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + if profile == _FDWIC_PROFILE_SUBMIT_PMU_EFDRAIN_CONTROL: + return [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=7", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + if profile == _FDWIC_PROFILE_SUBMIT_PMU_PREPARE_MAP: + return [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=8", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + if profile == _FDWIC_PROFILE_SUBMIT_PMU_FANIN: + return [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=9", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + if profile == _FDWIC_PROFILE_SUBMIT_PMU_WINNER_BUILD_CONTROL: + return [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=10", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + if profile == _FDWIC_PROFILE_SUBMIT_PMU_ALLOC_COMPLETE_CONTROL: + return [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=11", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + if profile == _FDWIC_PROFILE_SUBMIT_PMU_LOSER_REPLAY: + return [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=12", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + return None + + +def _profiled_cache_key(cache_key) -> tuple[Any, ...]: + base = cache_key if isinstance(cache_key, tuple) else (cache_key,) + return (*base, _fdwic_tensormap_mode(), _fdwic_profile()) def clear_compile_cache() -> None: @@ -54,6 +249,7 @@ def clear_compile_cache() -> None: """ _compile_cache.clear() _aicore_override_cache.clear() + _fdwic_build_identity_cache.clear() gc.collect() @@ -117,7 +313,179 @@ def _write_aicore_incore_wrapper(cache_key: str, incores: list[dict]) -> Path | def get_aicore_path_override(cache_key) -> Path | None: - return _aicore_override_cache.get(cache_key) + return _aicore_override_cache.get(_profiled_cache_key(cache_key)) + + +def _fdwic_elf_symbol_rows(binary: Path) -> list[tuple[str, str, str]]: + """Return ``(kind, section, name)`` rows from one final FDWIC ELF.""" + try: + result = subprocess.run( + ["readelf", "-Ws", "-W", str(binary)], + check=False, + capture_output=True, + text=True, + ) + except FileNotFoundError as exc: + raise RuntimeError("readelf is required to validate the FDWIC ELF") from exc + if result.returncode != 0: + raise RuntimeError(f"readelf failed for FDWIC ELF {binary}: {result.stderr.strip()}") + + symbol_rows = [] + for line in result.stdout.splitlines(): + fields = line.split() + if len(fields) < 8 or not fields[0].endswith(":"): + continue + symbol_rows.append((fields[3], fields[6], fields[7])) + return symbol_rows + + +def _assert_fdwic_perf_clock_elf(binary: Path, profile: str = _FDWIC_PROFILE_PERF_CLOCK) -> None: + """Prove the final CCEC image excludes FDWIC trace/atomic and platform PMU code.""" + if profile not in _FDWIC_PERF_CLOCK_PROFILES: + raise ValueError(f"Unsupported perf-clock ELF profile {profile!r}") + symbol_rows = _fdwic_elf_symbol_rows(binary) + + required = ["dist_perf_clock_expect_submits"] + if profile == _FDWIC_PROFILE_PERF_CLOCK_KERNEL: + required.append("dist_perf_clock_kernel_profile_marker") + forbidden = [ + "fdwic_atomic_poll_boundary_slow", + "fdwic_swimlane_detail_record_atomic", + "g_fdwic_swimlane_", + "g_fdwic_atomic_", + "g_fdwic_poll_", + "set_aicore_profiling_flag", + "get_aicore_profiling_flag", + "set_l2_swimlane_aicore_head_slot", + "get_l2_swimlane_aicore_head", + "set_aicore_pmu_ring", + "get_aicore_pmu_ring", + "set_aicore_pmu_reg_base", + "get_aicore_pmu_reg_base", + "dist_submit_pmu_expect_submits", + "fdwic_submit_pmu_read_counters", + "set_fdwic_submit_pmu_reg_base", + "get_fdwic_submit_pmu_reg_base", + "g_fdwic_submit_pmu_", + ] + if profile == _FDWIC_PROFILE_PERF_CLOCK: + forbidden.extend(("dist_perf_clock_kernel_profile_marker", "g_fdwic_perf_clock_kernel_")) + missing = [ + symbol + for symbol in required + if not any(kind == "FUNC" and ndx != "UND" and symbol in name for kind, ndx, name in symbol_rows) + ] + present = [symbol for symbol in forbidden if any(symbol in name for _kind, _ndx, name in symbol_rows)] + if missing or present: + details = [] + if missing: + details.append(f"missing defined perf-clock marker(s): {', '.join(missing)}") + if present: + details.append(f"profiling symbol(s) still present: {', '.join(present)}") + raise RuntimeError(f"Invalid perf-clock AICore image {binary}: {'; '.join(details)}") + + +def _assert_fdwic_submit_pmu_elf(binary: Path, profile: str) -> None: + """Prove one submit-PMU image contains exactly its selected observer.""" + symbol_rows = _fdwic_elf_symbol_rows(binary) + + phase_readers = ( + "fdwic_submit_pmu_phase_read_shadow_counters", + "fdwic_submit_pmu_phase_read_scalar_shadow", + "fdwic_submit_pmu_phase_read_total_shadow", + ) + required = ["dist_submit_pmu_expect_submits", "fdwic_submit_pmu_read_counters"] + if profile in _FDWIC_SUBMIT_PMU_PHASE_PROFILES: + required.extend(phase_readers) + forbidden = ( + "dist_perf_clock_expect_submits", + "g_fdwic_perf_clock_", + "fdwic_atomic_poll_boundary_slow", + "fdwic_swimlane_detail_record_atomic", + "g_fdwic_swimlane_", + "g_fdwic_atomic_", + "g_fdwic_poll_", + "set_aicore_profiling_flag", + "get_aicore_profiling_flag", + "set_l2_swimlane_aicore_head_slot", + "get_l2_swimlane_aicore_head", + "set_aicore_pmu_ring", + "get_aicore_pmu_ring", + "set_aicore_pmu_reg_base", + "get_aicore_pmu_reg_base", + "pmu_aicore_record_task", + ) + if profile == _FDWIC_PROFILE_SUBMIT_PMU_NONE: + forbidden = (*forbidden, *phase_readers) + missing = [ + symbol + for symbol in required + if not any(kind == "FUNC" and ndx != "UND" and symbol in name for kind, ndx, name in symbol_rows) + ] + present = [symbol for symbol in forbidden if any(symbol in name for _kind, _ndx, name in symbol_rows)] + if missing or present: + details = [] + if missing: + details.append(f"missing defined submit-pmu marker(s): {', '.join(missing)}") + if present: + details.append(f"unrelated profiling symbol(s) still present: {', '.join(present)}") + raise RuntimeError(f"Invalid {profile} AICore image {binary}: {'; '.join(details)}") + + +def _assert_fdwic_submit_pmu_host_elf(binary: Path, profile: str) -> None: + """Reject a stale host runtime before it can launch a profiled device case.""" + if profile not in _FDWIC_SUBMIT_PMU_PROFILES: + raise ValueError(f"Unsupported submit-PMU host profile {profile!r}") + symbol_rows = _fdwic_elf_symbol_rows(binary) + required = ( + "fdwic_submit_pmu_host_init", + "fdwic_submit_pmu_host_export", + "fdwic_submit_pmu_host_finalize", + ) + missing = [ + symbol + for symbol in required + if not any(kind == "FUNC" and ndx != "UND" and symbol in name for kind, ndx, name in symbol_rows) + ] + try: + image = Path(binary).read_bytes() + except OSError as exc: + raise RuntimeError(f"Cannot read submit-PMU host runtime {binary}: {exc}") from exc + profile_marker = profile.encode("utf-8") + b"\0" + if missing or profile_marker not in image: + details = [] + if missing: + details.append(f"missing defined host hook(s): {', '.join(missing)}") + if profile_marker not in image: + details.append(f"missing exact profile marker {profile!r}; rebuild the a5 FDWIC host runtime") + raise RuntimeError(f"Invalid {profile} host runtime {binary}: {'; '.join(details)}") + + +def _assert_fdwic_swimlane_elf(binary: Path) -> None: + """Prove the normal real-A5 FDWIC image carries the merged phase/atomic observer.""" + symbol_rows = _fdwic_elf_symbol_rows(binary) + required = ("fdwic_atomic_poll_boundary_slow", "fdwic_swimlane_detail_record_atomic") + forbidden = ( + "dist_perf_clock_expect_submits", + "dist_submit_pmu_expect_submits", + "fdwic_submit_pmu_read_counters", + "set_fdwic_submit_pmu_reg_base", + "get_fdwic_submit_pmu_reg_base", + "g_fdwic_submit_pmu_", + ) + missing = [ + symbol + for symbol in required + if not any(kind == "FUNC" and ndx != "UND" and symbol in name for kind, ndx, name in symbol_rows) + ] + present = [symbol for symbol in forbidden if any(symbol in name for _kind, _ndx, name in symbol_rows)] + if missing or present: + details = [] + if missing: + details.append(f"missing defined swimlane observer(s): {', '.join(missing)}") + if present: + details.append(f"isolated-profile symbol(s) leaked into normal image: {', '.join(present)}") + raise RuntimeError(f"Invalid FDWIC swimlane AICore image {binary}: {'; '.join(details)}") def maybe_build_aicore_override( @@ -128,6 +496,9 @@ def maybe_build_aicore_override( incores: list[dict], pto_isa_root: str | None = None, ) -> Path | None: + profile = _fdwic_profile() + if profile in _FDWIC_ISOLATED_PROFILES and (platform != "a5" or runtime != "fully_distributed_within_core"): + raise ValueError(f"{profile} is only supported by the real a5 fully_distributed_within_core runtime") if platform not in {"a5", "a5sim"} or runtime != "fully_distributed_within_core": return None @@ -141,8 +512,44 @@ def maybe_build_aicore_override( if wrapper_path is not None: source_paths.append(wrapper_path) key = _aicore_extra_cache_key(cache_key, source_paths) - builder = RuntimeBuilder(platform) - return builder.build_aicore_with_extra_sources(runtime, source_paths, key, pto_isa_root=pto_isa_root) + # Keep PTO2_PROFILING at its normal value because it also owns the public + # Arg layout. Each isolated evidence profile independently removes the dist + # swimlane/atomic path without changing orchestration/incore ABI. + compile_definitions = _fdwic_compile_definitions(profile) + tensormap_mode = _fdwic_tensormap_mode() + builder = RuntimeBuilder(platform, fdwic_tensormap_mode=tensormap_mode) + binary = builder.build_aicore_with_extra_sources( + runtime, + source_paths, + key, + pto_isa_root=pto_isa_root, + compile_definitions=compile_definitions, + ) + if profile in _FDWIC_PERF_CLOCK_PROFILES: + _assert_fdwic_perf_clock_elf(binary, profile) + elif profile in _FDWIC_SUBMIT_PMU_PROFILES: + _assert_fdwic_submit_pmu_elf(binary, profile) + from .tools.fdwic_submit_pmu_report import capture_build_identity # noqa: PLC0415 + + baseline_binaries = builder.get_binaries(runtime) + host_runtime = baseline_binaries.host_path + aicpu_runtime = baseline_binaries.aicpu_path + _assert_fdwic_submit_pmu_host_elf(host_runtime, profile) + output_key_dir = Path(binary).resolve().parent + aicore_build_dir = builder._CACHE_DIR / output_key_dir.relative_to(builder._LIB_DIR) / "aicore" + _fdwic_build_identity_cache[cache_key] = capture_build_identity( + profile=profile, + profiled_cache_key=cache_key, + aicore_extra_cache_key=key, + compile_definitions=builder.effective_compile_definitions(runtime, compile_definitions), + aicore_kernel=binary, + aicore_build_dir=aicore_build_dir, + host_runtime=host_runtime, + aicpu_runtime=aicpu_runtime, + ) + elif platform == "a5" and runtime == "fully_distributed_within_core": + _assert_fdwic_swimlane_elf(binary) + return binary # --------------------------------------------------------------------------- @@ -819,6 +1226,8 @@ def _run_swimlane_converter( input_path: Path | None = None, func_names_path: Path | None = None, enable_overhead: bool = False, + *, + strict_fdwic_v4: bool = False, ) -> None: """Invoke the bundled swimlane converter as a subprocess. @@ -846,8 +1255,23 @@ def _run_swimlane_converter( result = subprocess.run(cmd, check=True, capture_output=True, text=True) if result.stdout: logger.info(result.stdout) + if strict_fdwic_v4: + if input_path is None: + raise RuntimeError("strict FDWIC schema-v4 conversion requires an explicit raw input path") + required_outputs = ( + input_path.parent / "merged_swimlane.json", + input_path.parent / "swimlane_exclusive_analysis.json", + ) + missing_outputs = [str(path) for path in required_outputs if not path.is_file() or path.stat().st_size == 0] + if missing_outputs: + raise RuntimeError( + "FDWIC schema-v4 conversion did not publish required artifact(s): " + ", ".join(missing_outputs) + ) logger.info("Swimlane JSON generation completed") except subprocess.CalledProcessError as e: + if strict_fdwic_v4: + details = e.stderr.strip() or e.stdout.strip() or str(e) + raise RuntimeError(f"FDWIC schema-v4 conversion/closure validation failed: {details}") from e logger.warning(f"Failed to generate swimlane JSON: {e}") if e.stdout: logger.debug(f"stdout: {e.stdout}") @@ -864,6 +1288,8 @@ def _convert_case_swimlane( output_prefix: Path, callable_spec: dict | None = None, enable_overhead: bool = False, + *, + strict_fdwic_v4: bool = False, ) -> None: """Post-case: invoke the swimlane converter on the perf file the runtime just wrote into ``/l2_swimlane_records.json``. No diff/rename @@ -874,6 +1300,8 @@ def _convert_case_swimlane( logger = logging.getLogger(__name__) perf_file = output_prefix / "l2_swimlane_records.json" if not perf_file.exists(): + if strict_fdwic_v4: + raise RuntimeError(f"[{case_label}] required FDWIC schema-v4 raw artifact was not produced: {perf_file}") logger.warning(f"[{case_label}] {perf_file} not produced; skipping conversion") return @@ -884,7 +1312,12 @@ def _convert_case_swimlane( safe_label = _sanitize_for_filename(case_label) func_names_path = _dump_name_map(mapping, output_prefix / f"name_map_{safe_label}.json") - _run_swimlane_converter(input_path=perf_file, func_names_path=func_names_path, enable_overhead=enable_overhead) + _run_swimlane_converter( + input_path=perf_file, + func_names_path=func_names_path, + enable_overhead=enable_overhead, + strict_fdwic_v4=strict_fdwic_v4, + ) def _run_deps_viewer( @@ -970,6 +1403,197 @@ def _plot_case_scope_stats(case_label: str, output_prefix: Path) -> None: sys.path.remove(str(tools_dir)) +def _render_case_fdwic_submit_pmu(case_label: str, output_prefix: Path, build_identity: SubmitPmuBuildIdentity) -> Path: + """Strictly validate the real-PA Submit-PMU raw and publish its HTML.""" + from .tools.fdwic_submit_pmu_report import ( # noqa: PLC0415 + DEFAULT_INPUT_NAME, + DEFAULT_OUTPUT_NAME, + write_report_with_provenance, + ) + + raw = output_prefix / DEFAULT_INPUT_NAME + if not raw.is_file() or raw.stat().st_size == 0: + raise RuntimeError(f"[{case_label}] submit-PMU did not publish a non-empty {raw}") + report = output_prefix / DEFAULT_OUTPUT_NAME + published_report, provenance = write_report_with_provenance(raw, build_identity, report) + if published_report != report: + raise RuntimeError(f"[{case_label}] submit-PMU published an unexpected report path {published_report}") + if not report.is_file() or report.stat().st_size == 0: + raise RuntimeError(f"[{case_label}] submit-PMU did not publish a non-empty {report}") + if not provenance.is_file() or provenance.stat().st_size == 0: + raise RuntimeError(f"[{case_label}] submit-PMU did not publish a non-empty {provenance}") + return report + + +def _validate_case_fdwic_perf_clock( # noqa: PLR0912 -- fail-closed artifact contract is intentionally explicit + case_label: str, output_prefix: Path, profile: str +) -> Path: + """Validate the exact artifact contract of one successful perf-clock case.""" + try: + output_name, schema = _FDWIC_PERF_CLOCK_ARTIFACTS[profile] + except KeyError as exc: + raise ValueError(f"Unsupported perf-clock artifact profile {profile!r}") from exc + + artifact = output_prefix / output_name + if not artifact.is_file() or artifact.stat().st_size == 0: + raise RuntimeError(f"[{case_label}] {profile} did not publish a non-empty {artifact}") + try: + payload = json.loads(artifact.read_text()) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError(f"[{case_label}] {artifact} is not valid JSON: {exc}") from exc + if not isinstance(payload, dict): + raise RuntimeError(f"[{case_label}] {artifact} root must be a JSON object") + + expected_scalars = { + "schema": schema, + "mode": profile, + "num_cores": 96, + "aic_cores": 32, + "aiv_cores": 64, + } + mismatches = [ + f"{name}={payload.get(name)!r}, expected {expected!r}" + for name, expected in expected_scalars.items() + if payload.get(name) != expected + ] + if mismatches: + raise RuntimeError(f"[{case_label}] invalid {profile} artifact contract: {'; '.join(mismatches)}") + + expected_submits = payload.get("expected_submits_per_core") + cores = payload.get("cores") + if type(expected_submits) is not int or expected_submits <= 0: + raise RuntimeError(f"[{case_label}] {profile} expected_submits_per_core must be a positive integer") + if not isinstance(cores, list) or len(cores) != 96 or not all(isinstance(core, dict) for core in cores): + raise RuntimeError(f"[{case_label}] {profile} cores must contain exactly 96 objects") + + core_ids = [core.get("core_id") for core in cores] + core_types = [core.get("core_type") for core in cores] + if set(core_ids) != set(range(96)) or len(core_ids) != len(set(core_ids)): + raise RuntimeError(f"[{case_label}] {profile} core_id topology is not exactly 0..95") + if core_types.count("aic") != 32 or core_types.count("aiv") != 64: + raise RuntimeError(f"[{case_label}] {profile} core_type topology is not 32 AIC + 64 AIV") + + for core in cores: + if core.get("submit_count") != expected_submits: + raise RuntimeError( + f"[{case_label}] {profile} core {core.get('core_id')} submit_count does not match " + "expected_submits_per_core" + ) + start = core.get("first_submit_start") + end = core.get("last_submit_end") + elapsed = core.get("elapsed_ticks") + ordered_window = ( + type(start) is int + and type(end) is int + and type(elapsed) is int + and start > 0 + and end - start == elapsed + and (end > start if profile == _FDWIC_PROFILE_PERF_CLOCK_KERNEL else end >= start) + ) + if not ordered_window: + raise RuntimeError(f"[{case_label}] {profile} core {core.get('core_id')} elapsed tick closure failed") + + global_start = min(core["first_submit_start"] for core in cores) + global_end = max(core["last_submit_end"] for core in cores) + if ( + payload.get("global_first_submit_start") != global_start + or payload.get("global_last_submit_end") != global_end + or payload.get("global_submit_span_ticks") != global_end - global_start + ): + raise RuntimeError(f"[{case_label}] {profile} global Submit tick closure failed") + + kernel_ticks: list[int] = [] + kernel_calls: list[int] = [] + residual_ticks: list[int] = [] + if profile == _FDWIC_PROFILE_PERF_CLOCK_KERNEL: + kernel_ticks = [core.get("kernel_elapsed_ticks") for core in cores] + kernel_calls = [core.get("kernel_calls") for core in cores] + residual_ticks = [core.get("non_kernel_residual_ticks") for core in cores] + if not all(type(value) is int and value >= 0 for value in (*kernel_ticks, *kernel_calls, *residual_ticks)): + raise RuntimeError(f"[{case_label}] {profile} per-core Kernel aggregates must be non-negative integers") + + groups = payload.get("groups") + if not isinstance(groups, dict): + raise RuntimeError(f"[{case_label}] {profile} groups must be a JSON object") + for core_type in ("aic", "aiv"): + group = groups.get(core_type) + typed_cores = [core for core in cores if core["core_type"] == core_type] + elapsed_values = [core["elapsed_ticks"] for core in typed_cores] + if not isinstance(group, dict): + raise RuntimeError(f"[{case_label}] {profile} groups.{core_type} must be a JSON object") + expected_group = ( + { + "min_ticks": min(elapsed_values), + "max_ticks": max(elapsed_values), + } + if profile == _FDWIC_PROFILE_PERF_CLOCK + else { + "cores": len(typed_cores), + "elapsed_min_ticks": min(elapsed_values), + "elapsed_max_ticks": max(elapsed_values), + "elapsed_sum_ticks": sum(elapsed_values), + "kernel_min_ticks": min(core["kernel_elapsed_ticks"] for core in typed_cores), + "kernel_max_ticks": max(core["kernel_elapsed_ticks"] for core in typed_cores), + "kernel_sum_ticks": sum(core["kernel_elapsed_ticks"] for core in typed_cores), + "kernel_calls_min": min(core["kernel_calls"] for core in typed_cores), + "kernel_calls_max": max(core["kernel_calls"] for core in typed_cores), + "kernel_calls_sum": sum(core["kernel_calls"] for core in typed_cores), + "residual_min_ticks": min(core["non_kernel_residual_ticks"] for core in typed_cores), + "residual_max_ticks": max(core["non_kernel_residual_ticks"] for core in typed_cores), + "residual_sum_ticks": sum(core["non_kernel_residual_ticks"] for core in typed_cores), + } + ) + group_mismatches = [ + f"{name}={group.get(name)!r}, recomputed {expected}" + for name, expected in expected_group.items() + if type(group.get(name)) is not int or group.get(name) != expected + ] + if group_mismatches: + raise RuntimeError( + f"[{case_label}] invalid {profile} groups.{core_type} integer aggregates: {'; '.join(group_mismatches)}" + ) + + if profile == _FDWIC_PROFILE_PERF_CLOCK_KERNEL: + if any( + kernel + residual != core["elapsed_ticks"] + for core, kernel, residual in zip(cores, kernel_ticks, residual_ticks, strict=True) + ): + raise RuntimeError(f"[{case_label}] {profile} per-core Kernel/residual tick closure failed") + if any((calls == 0) != (ticks == 0) for calls, ticks in zip(kernel_calls, kernel_ticks, strict=True)): + raise RuntimeError(f"[{case_label}] {profile} per-core Kernel call/tick presence closure failed") + exact_aggregates = { + "kernel_calls": sum(kernel_calls), + "kernel_elapsed_ticks_sum": sum(kernel_ticks), + "non_kernel_residual_ticks_sum": sum(residual_ticks), + } + aggregate_mismatches = [ + f"{name}={payload.get(name)!r}, recomputed {expected}" + for name, expected in exact_aggregates.items() + if payload.get(name) != expected + ] + if aggregate_mismatches: + raise RuntimeError( + f"[{case_label}] invalid {profile} integer aggregates: {'; '.join(aggregate_mismatches)}" + ) + min_calls = payload.get("min_kernel_calls_in_window") + max_calls = payload.get("max_kernel_calls_in_window") + batches, remainder = divmod(expected_submits, 5) + aic_calls = sum(core["kernel_calls"] for core in cores if core["core_type"] == "aic") + aiv_calls = sum(core["kernel_calls"] for core in cores if core["core_type"] == "aiv") + if ( + remainder != 0 + or type(min_calls) is not int + or type(max_calls) is not int + or min_calls != batches + or max_calls != 4 * batches + or not batches <= aic_calls <= 2 * batches + or not 0 <= aiv_calls <= 2 * batches + or not batches <= exact_aggregates["kernel_calls"] <= 4 * batches + ): + raise RuntimeError(f"[{case_label}] {profile} global Kernel call range closure failed") + return artifact + + def _format_case_context(cls_name: str, case: dict, worker) -> str: platform = worker._config.get("platform", "") config = case.get("config", {}) @@ -1002,7 +1626,27 @@ def run_class_cases( # noqa: PLR0913 -- shared layer-5 entry; kwargs mirror CLI """ cls_name = type(cls_inst).__name__ callable_spec = getattr(type(cls_inst), "CALLABLE", None) - diagnostics_on = enable_l2_swimlane or enable_dump_args or enable_pmu or enable_dep_gen or enable_scope_stats + fdwic_profile = _fdwic_profile() + isolated_profile_on = fdwic_profile in _FDWIC_ISOLATED_PROFILES + submit_pmu_build_identity = None + if fdwic_profile in _FDWIC_SUBMIT_PMU_PROFILES: + platform = worker._config.get("platform") + runtime = getattr(type(cls_inst), "_st_runtime", None) + build_identity_key = _profiled_cache_key((type(cls_inst).__qualname__, platform, runtime)) + submit_pmu_build_identity = _fdwic_build_identity_cache.get(build_identity_key) + if submit_pmu_build_identity is None: + raise RuntimeError( + "Submit-PMU build identity is missing for " + f"profiled cache key {build_identity_key!r}; refusing to run without build provenance" + ) + diagnostics_on = ( + enable_l2_swimlane + or enable_dump_args + or enable_pmu + or enable_dep_gen + or enable_scope_stats + or isolated_profile_on + ) # device-log timing wraps each case here (not inside _run_and_validate*), # the same way swimlane conversion does — _run_and_validate_l2 is overridden # by some SceneTestCase subclasses, so threading a kwarg through it would @@ -1028,6 +1672,7 @@ def run_class_cases( # noqa: PLR0913 -- shared layer-5 entry; kwargs mirror CLI prefix = _build_output_prefix(case_label) if diagnostics_on else Path("") dlt_baseline = _snapshot_time() if dlt_on else None dlt_offsets = _snapshot_log_offsets(_get_device_log_dir(dlt_device_id)) if dlt_on else None + case_succeeded = False try: cls_inst._run_and_validate( worker, @@ -1043,21 +1688,33 @@ def run_class_cases( # noqa: PLR0913 -- shared layer-5 entry; kwargs mirror CLI enable_scope_stats=enable_scope_stats, output_prefix=str(prefix) if diagnostics_on else "", ) + case_succeeded = True except BaseException as exc: exc.add_note(f"SceneTest case context: {case_context}") raise finally: if enable_l2_swimlane: + strict_fdwic_v4 = ( + case_succeeded + and enable_l2_swimlane == 4 + and getattr(cls_inst, "_st_runtime", None) == "fully_distributed_within_core" + and worker._config.get("platform") == "a5" + ) _convert_case_swimlane( case_label, prefix, callable_spec=callable_spec, enable_overhead=enable_swimlane_overhead, + strict_fdwic_v4=strict_fdwic_v4, ) if enable_dep_gen: _graph_case_dep_gen(case_label, prefix, callable_spec=callable_spec) if enable_scope_stats: _plot_case_scope_stats(case_label, prefix) + if case_succeeded and fdwic_profile in _FDWIC_SUBMIT_PMU_PROFILES: + _render_case_fdwic_submit_pmu(case_label, prefix, submit_pmu_build_identity) + if case_succeeded and fdwic_profile in _FDWIC_PERF_CLOCK_PROFILES: + _validate_case_fdwic_perf_clock(case_label, prefix, fdwic_profile) if dlt_baseline is not None: _print_device_log_timing(dlt_device_id, dlt_baseline, dlt_offsets, rounds) @@ -1082,6 +1739,7 @@ def _compare_outputs(test_args, golden_args, output_names, rtol, atol): def _compile_chip_callable_from_spec(spec, platform, runtime, cache_key): """Compile a chip entry spec (orchestration + incores) -> ChipCallable. Session-cached.""" + cache_key = _profiled_cache_key(cache_key) if cache_key in _compile_cache: return _compile_cache[cache_key] @@ -1098,7 +1756,11 @@ def _compile_chip_callable_from_spec(spec, platform, runtime, cache_key): kc = KernelCompiler(platform=platform) is_sim = platform.endswith("sim") - orch_binary = kc.compile_orchestration(runtime, orch["source"]) + orch_binary = kc.compile_orchestration( + runtime, + orch["source"], + compile_definitions=_fdwic_tensormap_compile_definitions(platform, runtime), + ) inc_dirs = kc.get_orchestration_include_dirs(runtime) kernel_binaries = [] @@ -1214,7 +1876,7 @@ def _create_worker(cls, platform, device_id=0): cache_key = (cls.__qualname__, platform, cls._st_runtime) cls.compile_chip_callable(platform) aicore_override = get_aicore_path_override(cache_key) - kwargs = {} + kwargs = {"fdwic_tensormap_mode": _fdwic_tensormap_mode()} if aicore_override is not None: kwargs["aicore_path_override"] = aicore_override w = Worker(level=2, device_id=device_id, platform=platform, runtime=cls._st_runtime, **kwargs) @@ -1667,6 +2329,13 @@ def run_module(module_name): # noqa: PLR0912, PLR0915 -- CLI parsing + dispatch parser = argparse.ArgumentParser() parser.add_argument("-p", "--platform", required=True) + parser.add_argument( + "--fdwic-tensormap", + choices=["private", "shared"], + default="private", + help="Select the compile-time TensorMap artifact family for the " + "a5/a5sim fully_distributed_within_core runtime.", + ) parser.add_argument( "-d", "--device", @@ -1801,6 +2470,12 @@ def run_module(module_name): # noqa: PLR0912, PLR0915 -- CLI parsing + dispatch ) args = parser.parse_args() configure_logging(args.log_level) + if args.fdwic_tensormap == "shared": + if args.platform not in {"a5", "a5sim"}: + parser.error("--fdwic-tensormap shared requires -p a5 or a5sim") + os.environ[_FDWIC_TENSORMAP_MODE_ENV] = _FDWIC_TENSORMAP_SHARED + else: + os.environ.pop(_FDWIC_TENSORMAP_MODE_ENV, None) # Match the per-test kernel/orchestration compile to the runtime's # sanitizer, and require the runtime preloaded — same as conftest, since @@ -1900,6 +2575,12 @@ def run_module(module_name): # noqa: PLR0912, PLR0915 -- CLI parsing + dispatch for cls, case in selected: selected_by_cls.setdefault(cls, []).append(case) + try: + _validate_fdwic_tensormap_test_classes(args.fdwic_tensormap, selected_by_cls) + except ValueError as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(2) + # L3 profiling not supported yet (multi-chip-process filename collision). # Mirror the pytest-side guard so standalone users get the same early-fail. if args.enable_l2_swimlane: @@ -2000,6 +2681,8 @@ def _dispatch_test_phases_standalone(module_name, selected_by_cls, args): # noq script = os.path.abspath(getattr(module, "__file__", sys.argv[0])) common = ["-p", args.platform, "--manual", args.manual, "--log-level", args.log_level] + if args.fdwic_tensormap != "private": + common += ["--fdwic-tensormap", args.fdwic_tensormap] if args.sanitizer != "none": common += ["--sanitizer", args.sanitizer] if args.rounds != 1: diff --git a/simpler_setup/tools/README.md b/simpler_setup/tools/README.md index 781a4a2d6d..77da1dd3eb 100644 --- a/simpler_setup/tools/README.md +++ b/simpler_setup/tools/README.md @@ -10,6 +10,7 @@ no repo checkout required. ## Tool list - **[swimlane_converter](#swimlane_converter)** — perf JSON → Chrome Trace Event (Perfetto) +- **[fdwic_swimlane_exclusive_analyzer](#fdwic_swimlane_exclusive_analyzer)** — strict FDWIC schema-v4 closure report - **[sched_overhead_analysis](#sched_overhead_analysis)** — scheduler overhead / Tail OH breakdown - **[device_log_timing](#device_log_timing)** — Total / Orch / Sched from a CANN device log (no swimlane JSON) - **[dump_viewer](#dump_viewer)** — inspect / export args dumps (see [docs/tensor-dump.md](../../docs/dfx/tensor-dump.md) for full workflow) @@ -94,7 +95,25 @@ A statistics summary grouped by function (printed to the console), including Exe - **Head/Tail OH**: scheduling head/tail overhead - **Exec_%**: Exec / Latency percentage (kernel utilization) -#### 3. Scheduler Overhead Deep-Dive +#### 3. FDWIC Exclusive Analysis + +When the input is an A5 fully-distributed-within-core schema-v4 +capture, the converter also writes +`swimlane_exclusive_analysis.json` next to the raw input. The report +validates the physical core topology, common contiguous Submit stream, +parent/child containment, exclusive phase order, zero producer drops, +and exact integer-cycle closure. It reports cross-core wall-clock +makespans separately from summed per-core work and keeps nested +`DrainWon`/atomic observations out of additive totals. + +The report can also be regenerated directly: + +```bash +python -m simpler_setup.tools.fdwic_swimlane_exclusive_analyzer \ + outputs/_/l2_swimlane_records.json +``` + +#### 4. Scheduler Overhead Deep-Dive `swimlane_converter` no longer runs the deep-dive inline — it needs the task DAG (`deps.json`) from a *separate* `--enable-dep-gen` run, which can't be produced @@ -119,7 +138,39 @@ After the test passes, the tool will: 1. Auto-detect the latest `l2_swimlane_records_*.json` in outputs/ 2. Load function names from the kernel_config.py specified via `-k` 3. Produce `merged_swimlane_*.json` for visualization -4. Print the task statistics and scheduler overhead deep-dive report to the console +4. Produce `swimlane_exclusive_analysis.json` for FDWIC schema-v4 captures +5. Print the task statistics and scheduler overhead deep-dive report to the console + +--- + +## fdwic_swimlane_exclusive_analyzer + +Validate and summarize the production FDWIC schema-v4 hierarchy using +raw integer cycles. The accepted Submit sequences follow the production +Kernel/Alloc and winner/loser paths; task type is read from `Submit.aux` +and is never inferred from task-ID arithmetic. Compete-first paths start with +`EfDrain -> Claim -> Materialize -> PrepareMap`; a kernel winner then uses +`Fanin -> Register -> WinnerBuild`, a kernel loser uses +`Register -> LoserReplay`, an Alloc winner uses +`Register -> AllocComplete`, and an Alloc loser ends after `Register`. +The synchronous eager callback that constructs arguments is represented by +the existing `Claim.end -> Materialize.begin` residual; it adds no raw phase +or field. The still-supported one-shot APIs retain their strict +Materialize-first Kernel/Alloc sequences; the validator accepts these two +live API families but no arbitrary phase permutation. A Kernel may be nested in +`EfDrain`, `WinnerBuild`, `AllocComplete`, an orchestration residual, or +FinalDrain. It may not occupy a Submit residual or cross a partition +boundary. + +The analyzer fails closed on incomplete topology, missing parents, +producer drops, orphan/overlapping children, or a non-closing partition. +Its output includes per-core and per-role metrics, residual boundary +breakdowns, kernel containment counts, and non-additive overlay counts. + +```bash +python -m simpler_setup.tools.fdwic_swimlane_exclusive_analyzer INPUT \ + --output OUTPUT +``` --- diff --git a/simpler_setup/tools/fdwic_submit_pmu_report.py b/simpler_setup/tools/fdwic_submit_pmu_report.py new file mode 100644 index 0000000000..85337c75cc --- /dev/null +++ b/simpler_setup/tools/fdwic_submit_pmu_report.py @@ -0,0 +1,2675 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Validate a production FDWIC Submit-PMU capture and render its I-cache report.""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import json +import math +import os +import re +import subprocess +import sys +import tempfile +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +if __package__: + from ..fdwic_build_config import fdwic_tensormap_ring_cap_definition +else: + # 保留使用文档中的直接脚本入口；该入口的 sys.path 默认只包含 + # simpler_setup/tools，直接加入上一层读取统一构建常量，避免触发 + # simpler_setup 包初始化及其编译环境依赖。 + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from fdwic_build_config import fdwic_tensormap_ring_cap_definition # type: ignore[no-redef] # noqa: E402 + +SCHEMA_NAME = "fdwic-submit-pmu-v3" +DEFAULT_INPUT_NAME = "fdwic_submit_pmu_raw.json" +DEFAULT_OUTPUT_NAME = "fdwic_submit_pmu_report.html" +PROVENANCE_SCHEMA_NAME = "fdwic-submit-pmu-provenance-v2" +DEFAULT_PROVENANCE_NAME = "fdwic_submit_pmu_provenance.json" + +EXPECTED_CORES = 96 +EXPECTED_AIC_CORES = 32 +EXPECTED_AIV_CORES = 64 +PHYSICAL_CORES = 108 +PHYSICAL_CORES_PER_DIE = 54 +AIC_CORES_PER_DIE = 18 +COMMON_REQUIRED_STATUS_MASK = (1 << 19) - 1 +NONE_REQUIRED_STATUS_MASK = (1 << 21) - 1 +PHASE_REQUIRED_STATUS_MASK = (1 << 10) - 1 +PROGRAMMABLE_COUNTER_RISK_THRESHOLD = 0x3FFFFFFF + +NONE_CAPTURE_MODE = "submit-pmu-none" +ARG_BUILD_CAPTURE_MODE = "submit-pmu-arg-build" +EMPTY_BRACKET_CAPTURE_MODE = "submit-pmu-empty-bracket" +MATERIALIZE_CAPTURE_MODE = "submit-pmu-materialize" +CLAIM_CAPTURE_MODE = "submit-pmu-claim" +REGISTER_CAPTURE_MODE = "submit-pmu-register" +SUBMIT_TRANSITION_CAPTURE_MODE = "submit-pmu-submit-transition" +EFDRAIN_CONTROL_CAPTURE_MODE = "submit-pmu-efdrain-control" +PREPARE_MAP_CAPTURE_MODE = "submit-pmu-prepare-map" +FANIN_CAPTURE_MODE = "submit-pmu-fanin" +WINNER_BUILD_CAPTURE_MODE = "submit-pmu-winner-build-control" +ALLOC_COMPLETE_CAPTURE_MODE = "submit-pmu-alloc-complete-control" +LOSER_REPLAY_CAPTURE_MODE = "submit-pmu-loser-replay" +ARG_BUILD_PHASE_ID = 1 +EMPTY_BRACKET_PHASE_ID = 2 +MATERIALIZE_PHASE_ID = 3 +CLAIM_PHASE_ID = 4 +REGISTER_PHASE_ID = 5 +SUBMIT_TRANSITION_PHASE_ID = 6 +EFDRAIN_CONTROL_PHASE_ID = 7 +PREPARE_MAP_PHASE_ID = 8 +FANIN_PHASE_ID = 9 +WINNER_BUILD_PHASE_ID = 10 +ALLOC_COMPLETE_PHASE_ID = 11 +LOSER_REPLAY_PHASE_ID = 12 +PHASE_PMU_OBSERVATION = { + "total_cycles": "running_read_clear_observed_with_software_reconstructed_whole", + "scalar_busy": "cnt3_running_read_clear_observed_bounded_by_cnt2_primary", + "non_scalar_busy": "per_core_phase_total_cycles_observed_minus_phase_scalar_busy_observed", + "sys_cnt_role": "boundary_diagnostic_only_not_primary_phase_timing", +} +PHASE_CONFIG_BY_MODE = { + ARG_BUILD_CAPTURE_MODE: { + "id": ARG_BUILD_PHASE_ID, + "name": "arg-build", + "boundary": "claim_end_to_materialize_begin", + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + }, + EMPTY_BRACKET_CAPTURE_MODE: { + "id": EMPTY_BRACKET_PHASE_ID, + "name": "empty-bracket", + "boundary": "claim_end_adjacent_empty_bracket", + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_empty_bracket_calibration", + "time_semantics": "boundary_diagnostic_outer_sys_cnt_around_adjacent_observer_pair", + }, + MATERIALIZE_CAPTURE_MODE: { + "id": MATERIALIZE_PHASE_ID, + "name": "materialize", + "boundary": "materialize_begin_to_materialize_end", + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + }, + CLAIM_CAPTURE_MODE: { + "id": CLAIM_PHASE_ID, + "name": "claim", + "boundary": "claim_begin_to_claim_end", + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + }, + REGISTER_CAPTURE_MODE: { + "id": REGISTER_PHASE_ID, + "name": "register", + "boundary": "register_outputs_call_entry_to_return", + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + }, + SUBMIT_TRANSITION_CAPTURE_MODE: { + "id": SUBMIT_TRANSITION_PHASE_ID, + "name": "submit-transition", + "boundary": "previous_submit_end_to_next_submit_begin", + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + }, + EFDRAIN_CONTROL_CAPTURE_MODE: { + "id": EFDRAIN_CONTROL_PHASE_ID, + "name": "efdrain-control", + "boundary": "efdrain_begin_to_end_excluding_linked_kernel_calls", + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "discontinuous_running_read_clear_excluding_linked_kernel_calls", + "time_semantics": "boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls", + }, + PREPARE_MAP_CAPTURE_MODE: { + "id": PREPARE_MAP_PHASE_ID, + "name": "prepare-map", + "boundary": "dist_submit_prepare_map_call_entry_to_return", + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + }, + FANIN_CAPTURE_MODE: { + "id": FANIN_PHASE_ID, + "name": "fanin", + "boundary": "fanin_begin_to_fanin_end", + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + }, + WINNER_BUILD_CAPTURE_MODE: { + "id": WINNER_BUILD_PHASE_ID, + "name": "winner-build-control", + "boundary": "winner_build_begin_to_end_excluding_linked_kernel_calls", + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "discontinuous_running_read_clear_excluding_linked_kernel_calls", + "time_semantics": "boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls", + }, + ALLOC_COMPLETE_CAPTURE_MODE: { + "id": ALLOC_COMPLETE_PHASE_ID, + "name": "alloc-complete-control", + "boundary": "alloc_complete_begin_to_end_excluding_linked_kernel_calls", + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "discontinuous_running_read_clear_excluding_linked_kernel_calls", + "time_semantics": "boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls", + }, + LOSER_REPLAY_CAPTURE_MODE: { + "id": LOSER_REPLAY_PHASE_ID, + "name": "loser-replay", + "boundary": "register_end_to_drain_block_won_return", + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + }, +} +DYNAMIC_PHASE_CAPTURE_MODES = frozenset( + {FANIN_CAPTURE_MODE, WINNER_BUILD_CAPTURE_MODE, ALLOC_COMPLETE_CAPTURE_MODE, LOSER_REPLAY_CAPTURE_MODE} +) +FIXED_ROLE_DYNAMIC_PHASE_CAPTURE_MODES = frozenset( + {FANIN_CAPTURE_MODE, WINNER_BUILD_CAPTURE_MODE, LOSER_REPLAY_CAPTURE_MODE} +) +KERNEL_EXCLUDING_PHASE_CAPTURE_MODES = frozenset(PHASE_CONFIG_BY_MODE) + + +def _expected_compile_definitions(profile: str, tensormap_mode: str = "private") -> tuple[str, ...]: + if tensormap_mode not in {"private", "shared"}: + _fail(f"unsupported FDWIC TensorMap mode {tensormap_mode!r}") + definitions = [ + f"PTO_FDWIC_SHARED_MAP={1 if tensormap_mode == 'shared' else 0}", + fdwic_tensormap_ring_cap_definition(), + "PTO_FDWIC_SUBMIT_PMU=1", + ] + if profile != NONE_CAPTURE_MODE: + definitions.append(f"PTO_FDWIC_SUBMIT_PMU_PHASE_ID={PHASE_CONFIG_BY_MODE[profile]['id']}") + definitions.append("PTO_FDWIC_TRACE_ENABLED=0") + return tuple(definitions) + + +def _expected_core_status_mask(mode: str) -> int: + return NONE_REQUIRED_STATUS_MASK if mode == NONE_CAPTURE_MODE else COMMON_REQUIRED_STATUS_MASK + + +SUPPORTED_CAPTURE_MODES = {NONE_CAPTURE_MODE, *PHASE_CONFIG_BY_MODE} +PHASE_RECORD_FIELDS = ( + "phase_id", + "phase_elapsed_ticks", + "phase_total_cycles_observed", + "phase_scalar_busy_observed", + "phase_icache_requests_observed", + "phase_icache_misses_observed", + "phase_begin_reads", + "phase_end_reads", + "phase_excluded_kernel_calls", + "phase_status", +) +DEPRECATED_PHASE_BOUND_FIELDS = ( + "phase_icache_requests_lower_bound", + "phase_icache_misses_lower_bound", +) + +EXPECTED_SELECTORS = { + "cnt0_vector_busy": 0x501, + "cnt1_cube_busy": 0x301, + "cnt2_scalar_busy": 0x001, + "cnt3_shadow_scalar_busy": 0x001, + "cnt5_shadow_icache_miss": 0x035, + "cnt6_primary_icache_request": 0x034, + "cnt7_primary_icache_miss": 0x035, + "cnt8_shadow_icache_request": 0x034, +} +EXPECTED_LINKED_KERNEL_EXCLUSION = { + "enabled": True, + "boundary": "dist_aicore_call_slot_kernel_entry_to_return", + "gate_semantics": "metrics_prof_stop_before_call_and_start_after_return", + "time_denominator": "scalar_submit_elapsed_ticks", + "wall_tick_semantics": "first_submit_start_to_last_submit_end_closure_only", +} +EXPECTED_RETURN_READY_ATOMIC_EXCLUSION = { + "enabled": True, + "classification": "result_used_atomic_only", + "time_boundary": "sys_cnt_before_atomic_to_result_dependent_sys_cnt_after_return", + "counter_semantics": "pmu_counters_include_atomic_instruction_events", + "time_denominator_effect": "subtract_return_ready_atomic_elapsed", +} +EXPECTED_PMU_CYCLES_PER_NS = {"all": 1.649844, "aic": 1.650062, "aiv": 1.649731} +METRICS = ("total_cycles", "scalar_busy", "icache_requests", "icache_misses") +GROUP_NAMES = ("all", "aic", "aiv") +PHASE_RECORDING_REFERENCE_METRICS = ( + ( + "pmu_total_cycles", + "cycles", + "phase_total_cycles_observed", + "total_cycles", + ), + ( + "scalar_busy_cycles", + "cycles", + "phase_scalar_busy_observed", + "scalar_busy", + ), + ( + "non_scalar_busy_cycles", + "cycles", + "phase_non_scalar_busy_cycles", + "non_scalar_busy_cycles", + ), + ( + "icache_requests", + "events", + "phase_icache_requests_observed", + "icache_requests", + ), + ( + "icache_misses", + "events", + "phase_icache_misses_observed", + "icache_misses", + ), +) +PHASE_CALIBRATION_STABLE_CONFIGURATION_FIELDS = ( + "num_cores", + "aic_cores", + "aiv_cores", + "sys_counter_tick_ns", + "selectors", + "linked_kernel_exclusion", + "return_ready_atomic_exclusion", + "counter_width_bits", + "programmable_counter_risk_threshold", + "pmu_cycles_per_ns", +) + + +@dataclass(frozen=True) +class SubmitPmuCapture: + """A capture that passed all producer/consumer closure checks.""" + + input_path: Path + raw_size: int + raw_sha256: str + data: dict[str, Any] + records: tuple[dict[str, Any], ...] + groups: dict[str, tuple[dict[str, Any], ...]] + summary: dict[str, dict[str, Any]] + phase_summary: dict[str, dict[str, Any]] | None + + +@dataclass(frozen=True) +class BuildArtifactIdentity: + """Frozen identity of one executable artifact used by the diagnostic run.""" + + path: Path + sha256: str + size_bytes: int + text_sha256: str + text_size_bytes: int + + +@dataclass(frozen=True) +class SubmitPmuBuildIdentity: + """Build-time identity captured before the A5 case starts.""" + + profile: str + tensormap_mode: str + profiled_cache_key: tuple[str, ...] + aicore_extra_cache_key: str + compile_definitions: tuple[str, ...] + source_state: str + source_state_path: Path + artifacts: tuple[tuple[str, BuildArtifactIdentity], ...] + + +def _fail(message: str) -> None: + raise ValueError(message) + + +_TEXT_SECTION_PATTERN = re.compile( + r"^\s*\[\s*\d+\]\s+\.text\s+\S+\s+[0-9a-fA-F]+\s+" + r"([0-9a-fA-F]+)\s+([0-9a-fA-F]+)\s", + re.MULTILINE, +) +_HEX_16_PATTERN = re.compile(r"^[0-9a-f]{16}$") +_HEX_40_PATTERN = re.compile(r"^[0-9a-f]{40}$") +_HEX_64_PATTERN = re.compile(r"^[0-9a-f]{64}$") + + +def _inspect_build_artifact(path: Path | str) -> BuildArtifactIdentity: + artifact = Path(path).resolve() + if not artifact.is_file(): + _fail(f"build artifact does not exist: {artifact}") + data = artifact.read_bytes() + try: + result = subprocess.run( + ["readelf", "-SW", str(artifact)], + check=False, + capture_output=True, + text=True, + ) + except FileNotFoundError as exc: + raise ValueError("readelf is required to inspect Submit-PMU build provenance") from exc + if result.returncode != 0: + _fail(f"readelf failed for build artifact {artifact}: {result.stderr.strip()}") + match = _TEXT_SECTION_PATTERN.search(result.stdout) + if match is None: + _fail(f"build artifact has no literal .text section: {artifact}") + text_offset, text_size = (int(value, 16) for value in match.groups()) + text_end = text_offset + text_size + if text_size <= 0 or text_end > len(data): + _fail(f"build artifact has an invalid .text range: {artifact}") + text_data = data[text_offset:text_end] + return BuildArtifactIdentity( + path=artifact, + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + text_sha256=hashlib.sha256(text_data).hexdigest(), + text_size_bytes=text_size, + ) + + +def _parse_source_state(source_state: str) -> tuple[str, str, str, str]: + fields = source_state.split(":") + if ( + len(fields) != 4 + or fields[0] != "source-v2" + or _HEX_40_PATTERN.fullmatch(fields[1]) is None + or _HEX_64_PATTERN.fullmatch(fields[2]) is None + or _HEX_64_PATTERN.fullmatch(fields[3]) is None + ): + _fail("Submit-PMU source state must be source-v2:::") + return fields[0], fields[1], fields[2], fields[3] + + +def capture_build_identity( + *, + profile: str, + profiled_cache_key: Sequence[Any], + aicore_extra_cache_key: str, + compile_definitions: Sequence[str], + aicore_kernel: Path | str, + aicore_build_dir: Path | str, + host_runtime: Path | str, + aicpu_runtime: Path | str, +) -> SubmitPmuBuildIdentity: + """Freeze the exact diagnostic ELF/SO identity before running a case.""" + + if profile not in {NONE_CAPTURE_MODE, *PHASE_CONFIG_BY_MODE}: + _fail(f"unsupported Submit-PMU provenance profile {profile!r}") + if _HEX_16_PATTERN.fullmatch(aicore_extra_cache_key) is None: + _fail("AICore extra cache key must contain exactly 16 lowercase hex digits") + cache_key = tuple(str(value) for value in profiled_cache_key) + if len(cache_key) < 2 or cache_key[-1] != profile: + _fail("profiled cache key must end with TensorMap mode and the selected Submit-PMU profile") + tensormap_mode = cache_key[-2] + if tensormap_mode not in {"private", "shared"}: + _fail("profiled cache key must carry private/shared immediately before the selected profile") + definitions = tuple(compile_definitions) + if definitions != _expected_compile_definitions(profile, tensormap_mode): + _fail("Submit-PMU provenance compile definitions do not match the selected profile") + kernel = Path(aicore_kernel).resolve() + build_dir = Path(aicore_build_dir).resolve() + host = Path(host_runtime).resolve() + aicpu = Path(aicpu_runtime).resolve() + stamp = build_dir / ".git_commit" + if not stamp.is_file(): + _fail(f"Submit-PMU AICore source-state stamp is missing: {stamp}") + source_state = stamp.read_text(encoding="utf-8").strip() + _version, _git_head, _source_fingerprint, definitions_sha256 = _parse_source_state(source_state) + expected_definitions_sha256 = hashlib.sha256(repr(list(definitions)).encode("utf-8")).hexdigest() + if definitions_sha256 != expected_definitions_sha256: + _fail("Submit-PMU source-state definition hash does not match the selected compile definitions") + if kernel.parent.name != aicore_extra_cache_key or build_dir.parent.name != aicore_extra_cache_key: + _fail("Submit-PMU AICore output/build paths do not match the selected extra cache key") + + runtime_name = "fully_distributed_within_core" + host_family = host.parent + aicpu_family = aicpu.parent + kernel_family = kernel.parents[2] if len(kernel.parents) > 2 else Path() + build_family = build_dir.parents[2] if len(build_dir.parents) > 2 else Path() + family_tail = (runtime_name, tensormap_mode) + if ( + host_family.parts[-2:] != family_tail + or aicpu_family != host_family + or kernel_family.parts[-2:] != family_tail + or build_family.parts[-2:] != family_tail + or kernel.parent.parent.name != "aicore-extra" + or build_dir.parent.parent.name != "aicore-extra" + or kernel_family.parts[-4:-2] != host_family.parts[-4:-2] + or build_family.parts[-4:-2] != host_family.parts[-4:-2] + ): + _fail( + "Submit-PMU Host/AICPU/AICore artifacts do not belong to the selected " + f"{tensormap_mode} FDWIC artifact family" + ) + + artifact_paths = ( + ("aicore_kernel", kernel), + ("aic_combined", build_dir / "aicore_aic_combined.o"), + ("aiv_combined", build_dir / "aicore_aiv_combined.o"), + ("host_runtime", host), + ("aicpu_runtime", aicpu), + ) + artifacts = tuple((name, _inspect_build_artifact(path)) for name, path in artifact_paths) + return SubmitPmuBuildIdentity( + profile=profile, + tensormap_mode=tensormap_mode, + profiled_cache_key=cache_key, + aicore_extra_cache_key=aicore_extra_cache_key, + compile_definitions=definitions, + source_state=source_state, + source_state_path=stamp, + artifacts=artifacts, + ) + + +def _object(value: Any, path: str) -> dict[str, Any]: + if not isinstance(value, dict): + _fail(f"{path} must be an object") + return value + + +def _array(value: Any, path: str) -> list[Any]: + if not isinstance(value, list): + _fail(f"{path} must be an array") + return value + + +def _integer(value: Any, path: str, *, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + _fail(f"{path} must be an integer >= {minimum}") + return value + + +def _number(value: Any, path: str, *, positive: bool = False) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + _fail(f"{path} must be a finite number") + result = float(value) + if not math.isfinite(result) or (positive and result <= 0): + qualifier = "positive " if positive else "" + _fail(f"{path} must be a finite {qualifier}number") + return result + + +def _same_json_value(actual: Any, expected: Any) -> bool: + if isinstance(expected, dict): + return ( + isinstance(actual, dict) + and actual.keys() == expected.keys() + and all(_same_json_value(actual[key], value) for key, value in expected.items()) + ) + if isinstance(expected, bool): + return isinstance(actual, bool) and actual is expected + if isinstance(expected, int): + return isinstance(actual, int) and not isinstance(actual, bool) and actual == expected + return type(actual) is type(expected) and actual == expected + + +def _require_equal(actual: Any, expected: Any, path: str) -> None: + if not _same_json_value(actual, expected): + _fail(f"{path} must equal {expected!r}, got {actual!r}") + + +def _expected_phase_calls(mode: str, expected_submits: int) -> int: + if mode == SUBMIT_TRANSITION_CAPTURE_MODE: + if expected_submits <= 1: + _fail("submit-pmu-submit-transition requires at least two submits per core") + return expected_submits - 1 + return expected_submits + + +def _expected_dynamic_phase_calls(mode: str, expected_submits: int) -> dict[str, int]: + if expected_submits % 5: + _fail(f"{mode} requires expected_submits_per_core divisible by 5") + batches = expected_submits // 5 + if mode in {FANIN_CAPTURE_MODE, WINNER_BUILD_CAPTURE_MODE}: + return {"all": 4 * batches, "aic": 2 * batches, "aiv": 2 * batches} + if mode == ALLOC_COMPLETE_CAPTURE_MODE: + return {"all": batches} + if mode == LOSER_REPLAY_CAPTURE_MODE: + return { + "all": (4 * EXPECTED_CORES - 4) * batches, + "aic": (4 * EXPECTED_AIC_CORES - 2) * batches, + "aiv": (4 * EXPECTED_AIV_CORES - 2) * batches, + } + _fail(f"{mode} has no dynamic phase call formula") + + +def _dynamic_phase_max_calls_per_core(mode: str, expected_submits: int) -> int: + batches = expected_submits // 5 + if mode == LOSER_REPLAY_CAPTURE_MODE: + return 4 * batches + return 2 * batches if mode in FIXED_ROLE_DYNAMIC_PHASE_CAPTURE_MODES else batches + + +def _phase_business_calls(record: Mapping[str, Any], mode: str) -> int: + """Return outer business calls, excluding pause/resume boundary pairs.""" + + begin_reads = int(record["phase_begin_reads"]) + if mode not in KERNEL_EXCLUDING_PHASE_CAPTURE_MODES: + return begin_reads + return begin_reads - int(record["phase_excluded_kernel_calls"]) + + +def _validate_capture_header(data: dict[str, Any]) -> tuple[str, dict[str, Any], int, dict[str, float]]: + _require_equal(data.get("schema"), SCHEMA_NAME, "schema") + capture = _object(data.get("capture"), "capture") + mode = capture.get("mode") + if not isinstance(mode, str) or mode not in SUPPORTED_CAPTURE_MODES: + _fail(f"capture.mode must be one of {sorted(SUPPORTED_CAPTURE_MODES)!r}, got {mode!r}") + _require_equal( + capture.get("window_scope"), + "per_core_first_submit_begin_to_last_submit_end", + "capture.window_scope", + ) + _require_equal(capture.get("accepted"), True, "capture.accepted") + _require_equal(capture.get("owner_restore_passed"), True, "capture.owner_restore_passed") + + configuration = _object(data.get("configuration"), "configuration") + _require_equal(configuration.get("num_cores"), EXPECTED_CORES, "configuration.num_cores") + _require_equal(configuration.get("aic_cores"), EXPECTED_AIC_CORES, "configuration.aic_cores") + _require_equal(configuration.get("aiv_cores"), EXPECTED_AIV_CORES, "configuration.aiv_cores") + expected_submits = _integer( + configuration.get("expected_submits_per_core"), + "configuration.expected_submits_per_core", + minimum=1, + ) + if mode == NONE_CAPTURE_MODE: + if "phase" in configuration: + _fail(f"configuration must not contain phase in {NONE_CAPTURE_MODE}") + else: + if mode in DYNAMIC_PHASE_CAPTURE_MODES: + call_shape = "dynamic_balanced" if mode in FIXED_ROLE_DYNAMIC_PHASE_CAPTURE_MODES else "dynamic_global" + expected_phase = { + **PHASE_CONFIG_BY_MODE[mode], + "call_shape": call_shape, + "expected_calls": _expected_dynamic_phase_calls(mode, expected_submits), + "pmu_observation": PHASE_PMU_OBSERVATION, + } + else: + expected_phase = { + **PHASE_CONFIG_BY_MODE[mode], + "expected_calls_per_core": _expected_phase_calls(mode, expected_submits), + "pmu_observation": PHASE_PMU_OBSERVATION, + } + _require_equal( + configuration.get("phase"), + expected_phase, + "configuration.phase", + ) + _require_equal(configuration.get("sys_counter_tick_ns"), 1, "configuration.sys_counter_tick_ns") + _require_equal(configuration.get("selectors"), EXPECTED_SELECTORS, "configuration.selectors") + _require_equal( + configuration.get("status_required_mask"), + _expected_core_status_mask(mode), + "configuration.status_required_mask", + ) + _require_equal( + configuration.get("linked_kernel_exclusion"), + EXPECTED_LINKED_KERNEL_EXCLUSION, + "configuration.linked_kernel_exclusion", + ) + _require_equal( + configuration.get("return_ready_atomic_exclusion"), + EXPECTED_RETURN_READY_ATOMIC_EXCLUSION, + "configuration.return_ready_atomic_exclusion", + ) + _require_equal( + configuration.get("counter_width_bits"), + {"total": 64, "programmable": 32}, + "configuration.counter_width_bits", + ) + _require_equal( + configuration.get("programmable_counter_risk_threshold"), + PROGRAMMABLE_COUNTER_RISK_THRESHOLD, + "configuration.programmable_counter_risk_threshold", + ) + frequency_data = _object(configuration.get("pmu_cycles_per_ns"), "configuration.pmu_cycles_per_ns") + _require_equal(frequency_data, EXPECTED_PMU_CYCLES_PER_NS, "configuration.pmu_cycles_per_ns") + frequencies = { + name: _number(frequency_data.get(name), f"configuration.pmu_cycles_per_ns.{name}", positive=True) + for name in GROUP_NAMES + } + return mode, configuration, expected_submits, frequencies + + +def _validate_phase_elapsed_shape( + *, + prefix: str, + dynamic_calls: bool, + begin_reads: int, + phase_elapsed: int, + phase_total_observed: int, + phase_scalar_observed: int, + phase_requests_observed: int, + phase_misses_observed: int, +) -> None: + if begin_reads > 0 and phase_total_observed == 0: + _fail(f"{prefix} non-empty phase must have positive phase_total_cycles_observed") + if not dynamic_calls: + if phase_elapsed == 0: + _fail(f"{prefix}.phase_elapsed_ticks must be an integer >= 1") + return + if begin_reads == 0: + observed_values = ( + phase_elapsed, + phase_total_observed, + phase_scalar_observed, + phase_requests_observed, + phase_misses_observed, + ) + if any(observed_values): + _fail(f"{prefix} zero-call dynamic phase must have zero elapsed/total/scalar/request/miss") + elif phase_elapsed == 0: + _fail(f"{prefix} non-empty dynamic phase must have positive elapsed") + + +def _validate_phase_read_shape( + *, + record: Mapping[str, Any], + prefix: str, + mode: str, + dynamic_calls: bool, + expected_calls: int | None, + begin_reads: int, + end_reads: int, +) -> int: + kernel_excluding = mode in KERNEL_EXCLUDING_PHASE_CAPTURE_MODES + if kernel_excluding: + excluded_kernel_calls = _integer( + record.get("phase_excluded_kernel_calls"), + f"{prefix}.phase_excluded_kernel_calls", + ) + else: + if "phase_excluded_kernel_calls" in record: + _fail( + f"{prefix}.phase_excluded_kernel_calls is only valid in " + f"{sorted(KERNEL_EXCLUDING_PHASE_CAPTURE_MODES)!r}" + ) + excluded_kernel_calls = 0 + if begin_reads < excluded_kernel_calls or end_reads < excluded_kernel_calls: + _fail(f"{prefix} excluded Kernel calls exceed phase begin/end reads") + business_begin_calls = begin_reads - excluded_kernel_calls + business_end_calls = end_reads - excluded_kernel_calls + if dynamic_calls: + if business_begin_calls != business_end_calls: + if excluded_kernel_calls: + _fail( + f"{prefix} phase begin/end reads must remain balanced after subtracting " + f"{excluded_kernel_calls} excluded Kernel calls" + ) + _fail(f"{prefix} dynamic phase begin/end reads must be balanced") + return business_begin_calls + + expected_reads = expected_calls + excluded_kernel_calls + if begin_reads != expected_reads or end_reads != expected_reads: + if excluded_kernel_calls: + _fail( + f"{prefix} phase begin/end reads must both equal expected calls " + f"{expected_calls} + excluded Kernel calls {excluded_kernel_calls} = {expected_reads}" + ) + _fail(f"{prefix} phase begin/end reads must both equal {expected_reads}") + return business_begin_calls + + +def _validate_phase_record( + record: dict[str, Any], + prefix: str, + mode: str, + expected_calls: int | None, + scalar_submit_elapsed: int, + expected_phase_id: int, +) -> None: + _require_equal(record.get("phase_id"), expected_phase_id, f"{prefix}.phase_id") + phase_elapsed = _integer(record.get("phase_elapsed_ticks"), f"{prefix}.phase_elapsed_ticks") + phase_total_observed = _integer( + record.get("phase_total_cycles_observed"), + f"{prefix}.phase_total_cycles_observed", + ) + phase_scalar_observed = _integer( + record.get("phase_scalar_busy_observed"), + f"{prefix}.phase_scalar_busy_observed", + ) + phase_requests_observed = _integer( + record.get("phase_icache_requests_observed"), + f"{prefix}.phase_icache_requests_observed", + ) + phase_misses_observed = _integer( + record.get("phase_icache_misses_observed"), + f"{prefix}.phase_icache_misses_observed", + ) + begin_reads = _integer(record.get("phase_begin_reads"), f"{prefix}.phase_begin_reads") + end_reads = _integer(record.get("phase_end_reads"), f"{prefix}.phase_end_reads") + dynamic_calls = mode in DYNAMIC_PHASE_CAPTURE_MODES + business_begin_calls = _validate_phase_read_shape( + record=record, + prefix=prefix, + mode=mode, + dynamic_calls=dynamic_calls, + expected_calls=expected_calls, + begin_reads=begin_reads, + end_reads=end_reads, + ) + _validate_phase_elapsed_shape( + prefix=prefix, + dynamic_calls=dynamic_calls, + begin_reads=business_begin_calls, + phase_elapsed=phase_elapsed, + phase_total_observed=phase_total_observed, + phase_scalar_observed=phase_scalar_observed, + phase_requests_observed=phase_requests_observed, + phase_misses_observed=phase_misses_observed, + ) + if phase_elapsed > scalar_submit_elapsed: + _fail(f"{prefix}.phase_elapsed_ticks exceeds scalar_submit_elapsed_ticks") + if phase_scalar_observed > phase_total_observed: + _fail(f"{prefix}.phase_scalar_busy_observed exceeds phase_total_cycles_observed") + if phase_total_observed > record["total_cycles"]: + _fail(f"{prefix}.phase_total_cycles_observed exceeds total_cycles") + if phase_scalar_observed > record["shadow_scalar_busy"]: + _fail(f"{prefix}.phase_scalar_busy_observed exceeds shadow_scalar_busy") + if phase_requests_observed > record["shadow_icache_requests"]: + _fail(f"{prefix}.phase_icache_requests_observed exceeds shadow_icache_requests") + if phase_misses_observed > record["shadow_icache_misses"]: + _fail(f"{prefix}.phase_icache_misses_observed exceeds shadow_icache_misses") + if phase_scalar_observed >= PROGRAMMABLE_COUNTER_RISK_THRESHOLD: + _fail(f"{prefix}.phase_scalar_busy_observed reaches the risk threshold 0x3fffffff") + + phase_status = _integer(record.get("phase_status"), f"{prefix}.phase_status") + if phase_status != PHASE_REQUIRED_STATUS_MASK: + _fail(f"{prefix}.phase_status must equal 0x{PHASE_REQUIRED_STATUS_MASK:x}, got 0x{phase_status:x}") + + +def _validate_shadow_counters( + record: Mapping[str, Any], + prefix: str, + mode: str, + scalar: int, + requests: int, + misses: int, +) -> tuple[int, ...]: + """Validate counters used only to close the primary running-read reconstruction.""" + + if mode == NONE_CAPTURE_MODE: + if any(field in record for field in ("shadow_scalar_busy", "shadow_icache_requests", "shadow_icache_misses")): + _fail(f"{prefix} must not publish redundant shadow counters in {NONE_CAPTURE_MODE}") + return () + + shadow_scalar = _integer( + record.get("shadow_scalar_busy"), + f"{prefix}.shadow_scalar_busy", + minimum=1, + ) + shadow_requests = _integer(record.get("shadow_icache_requests"), f"{prefix}.shadow_icache_requests") + shadow_misses = _integer(record.get("shadow_icache_misses"), f"{prefix}.shadow_icache_misses") + if shadow_scalar > scalar: + _fail(f"{prefix}.shadow_scalar_busy exceeds scalar_busy") + if shadow_misses > shadow_requests: + _fail(f"{prefix}.shadow_icache_misses exceeds shadow_icache_requests") + if shadow_requests > requests: + _fail(f"{prefix}.shadow_icache_requests exceeds icache_requests") + if shadow_misses > misses: + _fail(f"{prefix}.shadow_icache_misses exceeds icache_misses") + return (shadow_scalar, shadow_requests, shadow_misses) + + +def _validate_record( + record_data: Any, + logical_core_id: int, + expected_submits: int, + mode: str, +) -> dict[str, Any]: + record = _object(record_data, f"records[{logical_core_id}]") + prefix = f"records[{logical_core_id}]" + _require_equal(record.get("logical_core_id"), logical_core_id, f"{prefix}.logical_core_id") + _integer(record.get("physical_core_id"), f"{prefix}.physical_core_id") + if record.get("role") not in {"aic", "aiv"}: + _fail(f"{prefix}.role must be 'aic' or 'aiv'") + _integer(record.get("block_id"), f"{prefix}.block_id") + _integer(record.get("lane"), f"{prefix}.lane") + + submit_count = _integer(record.get("submit_count"), f"{prefix}.submit_count") + record_expected_submits = _integer(record.get("expected_submit_count"), f"{prefix}.expected_submit_count") + if submit_count != expected_submits or record_expected_submits != expected_submits: + _fail(f"{prefix} submit_count does not close at {expected_submits}") + + start = _integer(record.get("first_submit_start_tick"), f"{prefix}.first_submit_start_tick", minimum=1) + end = _integer(record.get("last_submit_end_tick"), f"{prefix}.last_submit_end_tick", minimum=1) + elapsed = _integer(record.get("submit_elapsed_ticks"), f"{prefix}.submit_elapsed_ticks", minimum=1) + if end < start or elapsed != end - start: + _fail(f"{prefix} Submit tick window is not closed") + scalar_elapsed = _integer( + record.get("scalar_submit_elapsed_ticks"), + f"{prefix}.scalar_submit_elapsed_ticks", + minimum=1, + ) + if scalar_elapsed > elapsed: + _fail(f"{prefix}.scalar_submit_elapsed_ticks exceeds submit_elapsed_ticks") + + total = _integer(record.get("total_cycles"), f"{prefix}.total_cycles", minimum=1) + scalar = _integer(record.get("scalar_busy"), f"{prefix}.scalar_busy") + requests = _integer(record.get("icache_requests"), f"{prefix}.icache_requests", minimum=1) + misses = _integer(record.get("icache_misses"), f"{prefix}.icache_misses") + if scalar > total: + _fail(f"{prefix}.scalar_busy exceeds total_cycles") + if misses > requests: + _fail(f"{prefix}.icache_misses exceeds icache_requests") + shadow_programmable = _validate_shadow_counters(record, prefix, mode, scalar, requests, misses) + if mode == NONE_CAPTURE_MODE: + if any(field in record for field in (*PHASE_RECORD_FIELDS, *DEPRECATED_PHASE_BOUND_FIELDS)): + _fail(f"{prefix} must not contain phase fields in {NONE_CAPTURE_MODE}") + else: + if any(field in record for field in DEPRECATED_PHASE_BOUND_FIELDS): + _fail(f"{prefix} must use observed phase fields, not lower/upper-bound names") + expected_calls = None if mode in DYNAMIC_PHASE_CAPTURE_MODES else _expected_phase_calls(mode, expected_submits) + _validate_phase_record( + record, + prefix, + mode, + expected_calls, + scalar_elapsed, + PHASE_CONFIG_BY_MODE[mode]["id"], + ) + programmable = (scalar, requests, misses, *shadow_programmable) + if any(value >= PROGRAMMABLE_COUNTER_RISK_THRESHOLD for value in programmable): + _fail(f"{prefix} programmable counter reaches the risk threshold 0x3fffffff") + + status = _integer(record.get("status"), f"{prefix}.status") + expected_status = _expected_core_status_mask(mode) + if status != expected_status: + _fail(f"{prefix}.status must equal 0x{expected_status:x}, got 0x{status:x}") + return record + + +def _validate_logical_layout(records: Sequence[dict[str, Any]]) -> None: + aic_ordinal = 0 + aiv_ordinal = 0 + for logical_core_id, record in enumerate(records): + if record["role"] == "aic": + expected = ("aic", aic_ordinal, 0) + aic_ordinal += 1 + else: + expected = ("aiv", aiv_ordinal // 2, 1 + aiv_ordinal % 2) + aiv_ordinal += 1 + actual = (record["role"], record["block_id"], record["lane"]) + if actual != expected: + _fail(f"records[{logical_core_id}] logical role/block/lane must equal {expected!r}, got {actual!r}") + + +def _expected_aiv_physical_ids(aic_physical_id: int) -> tuple[int, int]: + die_base = aic_physical_id // PHYSICAL_CORES_PER_DIE * PHYSICAL_CORES_PER_DIE + local_id = aic_physical_id % PHYSICAL_CORES_PER_DIE + return die_base + AIC_CORES_PER_DIE + 2 * local_id, die_base + AIC_CORES_PER_DIE + 2 * local_id + 1 + + +def _validate_topology(records: Sequence[dict[str, Any]]) -> set[int]: + physical_ids = [record["physical_core_id"] for record in records] + if len(set(physical_ids)) != EXPECTED_CORES: + _fail("physical_core_id values must be unique across all 96 records") + if any(physical_id >= PHYSICAL_CORES for physical_id in physical_ids): + _fail("physical_core_id must be in the A5 range [0, 108)") + + aic_physical_ids = { + record["physical_core_id"] + for record in records + if record["physical_core_id"] % PHYSICAL_CORES_PER_DIE < AIC_CORES_PER_DIE + } + reported_aic_ids = {record["physical_core_id"] for record in records if record["role"] == "aic"} + if reported_aic_ids != aic_physical_ids: + _fail("record roles must match the A5 physical AIC/AIV roles") + physical_id_set = set(physical_ids) + complete_triplets = 0 + for aic_physical_id in aic_physical_ids: + aiv1, aiv2 = _expected_aiv_physical_ids(aic_physical_id) + if aiv1 in physical_id_set and aiv2 in physical_id_set: + complete_triplets += 1 + if complete_triplets != EXPECTED_AIC_CORES: + _fail("records must form exactly 32 complete 1:2 mixed triplets") + return physical_id_set + + +def _bitmap_physical_ids(words_data: Any) -> set[int]: + words = _array(words_data, "owner.configured_bitmap_words") + if len(words) != 4: + _fail("owner.configured_bitmap_words must contain exactly four uint32 words") + result: set[int] = set() + for word_index, value in enumerate(words): + word = _integer(value, f"owner.configured_bitmap_words[{word_index}]") + if word > 0xFFFFFFFF: + _fail(f"owner.configured_bitmap_words[{word_index}] exceeds uint32") + for bit in range(32): + if word & (1 << bit): + result.add(word_index * 32 + bit) + if any(physical_id >= PHYSICAL_CORES for physical_id in result): + _fail("owner.configured_bitmap_words contains an out-of-range physical core") + return result + + +def _validate_owner(data: dict[str, Any], physical_ids: set[int]) -> None: + owner = _object(data.get("owner"), "owner") + if owner.get("restore_passed") is not True: + _fail("owner.restore_passed must be true") + required = { + "configure_passed": True, + "configured_count": EXPECTED_CORES, + "configured_aic": EXPECTED_AIC_CORES, + "configured_aiv": EXPECTED_AIV_CORES, + "restored_count": EXPECTED_CORES, + "active_after_restore": 0, + "restore_failures": 0, + "complete_mixed_triplets": EXPECTED_AIC_CORES, + } + for field, expected in required.items(): + _require_equal(owner.get(field), expected, f"owner.{field}") + bitmap_ids = _bitmap_physical_ids(owner.get("configured_bitmap_words")) + if bitmap_ids != physical_ids: + _fail("owner.configured_bitmap_words must exactly match all record physical_core_id values") + + +def _validate_window(data: dict[str, Any], records: Sequence[dict[str, Any]]) -> None: + window = _object(data.get("window"), "window") + first_tick = min(record["first_submit_start_tick"] for record in records) + last_tick = max(record["last_submit_end_tick"] for record in records) + span_ticks = last_tick - first_tick + _require_equal(window.get("global_first_submit_start_tick"), first_tick, "window.global_first_submit_start_tick") + _require_equal(window.get("global_last_submit_end_tick"), last_tick, "window.global_last_submit_end_tick") + _require_equal(window.get("global_submit_span_ticks"), span_ticks, "window.global_submit_span_ticks") + span_us = _number(window.get("global_submit_span_us"), "window.global_submit_span_us") + if not math.isclose(span_us, span_ticks / 1_000, rel_tol=1e-12, abs_tol=1e-9): + _fail("window.global_submit_span_us does not match the raw Submit ticks") + + +def _value_summary(values: Sequence[int | float]) -> dict[str, int | float]: + """Summarize already-derived per-core values without changing their aggregation order.""" + + total = sum(values) + return {"sum": total, "min": min(values), "mean": total / len(values), "max": max(values)} + + +def _metric_summary(records: Sequence[dict[str, Any]], metric: str) -> dict[str, int | float]: + return _value_summary([record[metric] for record in records]) + + +def _group_summary(records: Sequence[dict[str, Any]]) -> dict[str, Any]: + metrics = {metric: _metric_summary(records, metric) for metric in METRICS} + total_cycles = metrics["total_cycles"]["sum"] + scalar_busy = metrics["scalar_busy"]["sum"] + requests = metrics["icache_requests"]["sum"] + misses = metrics["icache_misses"]["sum"] + submit_elapsed_ticks = _metric_summary(records, "submit_elapsed_ticks") + scalar_submit_elapsed_ticks = _metric_summary(records, "scalar_submit_elapsed_ticks") + scalar_denominator_excluded_wall_ticks = _value_summary( + [record["submit_elapsed_ticks"] - record["scalar_submit_elapsed_ticks"] for record in records] + ) + non_scalar_busy_cycles = _value_summary([record["total_cycles"] - record["scalar_busy"] for record in records]) + return { + "cores": len(records), + **metrics, + "submit_elapsed_ticks": submit_elapsed_ticks, + "scalar_submit_elapsed_ticks": scalar_submit_elapsed_ticks, + "scalar_denominator_excluded_wall_ticks": scalar_denominator_excluded_wall_ticks, + "non_scalar_busy_cycles": non_scalar_busy_cycles, + "scalar_window_share_of_wall": scalar_submit_elapsed_ticks["sum"] / submit_elapsed_ticks["sum"], + "scalar_busy_share": scalar_busy / total_cycles, + "icache_miss_rate": misses / requests if requests else 0.0, + } + + +def _phase_group_summary(records: Sequence[dict[str, Any]], mode: str) -> dict[str, Any]: + submit_elapsed = _metric_summary(records, "submit_elapsed_ticks") + scalar_submit_elapsed = _metric_summary(records, "scalar_submit_elapsed_ticks") + phase_elapsed = _metric_summary(records, "phase_elapsed_ticks") + phase_total_observed = _metric_summary(records, "phase_total_cycles_observed") + phase_scalar_observed = _metric_summary(records, "phase_scalar_busy_observed") + phase_non_scalar = _value_summary( + [record["phase_total_cycles_observed"] - record["phase_scalar_busy_observed"] for record in records] + ) + shadow_scalar_loss = _value_summary([record["scalar_busy"] - record["shadow_scalar_busy"] for record in records]) + requests_observed = _metric_summary(records, "phase_icache_requests_observed") + misses_observed = _metric_summary(records, "phase_icache_misses_observed") + + requests_observed_plus_gap_values = [ + record["phase_icache_requests_observed"] + record["icache_requests"] - record["shadow_icache_requests"] + for record in records + ] + misses_observed_plus_gap_values = [ + record["phase_icache_misses_observed"] + record["icache_misses"] - record["shadow_icache_misses"] + for record in records + ] + requests_observed_plus_gap = { + "sum": sum(requests_observed_plus_gap_values), + "min": min(requests_observed_plus_gap_values), + "mean": sum(requests_observed_plus_gap_values) / len(requests_observed_plus_gap_values), + "max": max(requests_observed_plus_gap_values), + } + misses_observed_plus_gap = { + "sum": sum(misses_observed_plus_gap_values), + "min": min(misses_observed_plus_gap_values), + "mean": sum(misses_observed_plus_gap_values) / len(misses_observed_plus_gap_values), + "max": max(misses_observed_plus_gap_values), + } + primary_requests = sum(record["icache_requests"] for record in records) + primary_misses = sum(record["icache_misses"] for record in records) + primary_total = sum(record["total_cycles"] for record in records) + primary_scalar = sum(record["scalar_busy"] for record in records) + phase_calls_per_core = [_phase_business_calls(record, mode) for record in records] + phase_calls = sum(phase_calls_per_core) + summary = { + "cores": len(records), + "submit_elapsed_ticks": submit_elapsed, + "scalar_submit_elapsed_ticks": scalar_submit_elapsed, + "primary_icache_requests": primary_requests, + "primary_icache_misses": primary_misses, + "phase_elapsed_ticks": phase_elapsed, + "phase_total_cycles_observed": phase_total_observed, + "phase_scalar_busy_observed": phase_scalar_observed, + "phase_non_scalar_busy_cycles": phase_non_scalar, + "shadow_scalar_loss": shadow_scalar_loss, + "phase_icache_requests_observed": requests_observed, + "phase_icache_requests_observed_plus_capture_gap": requests_observed_plus_gap, + "phase_icache_misses_observed": misses_observed, + "phase_icache_misses_observed_plus_capture_gap": misses_observed_plus_gap, + "phase_total_share_of_pmu_total": phase_total_observed["sum"] / primary_total, + "phase_scalar_share_of_whole_scalar": ( + phase_scalar_observed["sum"] / primary_scalar if primary_scalar else 0.0 + ), + "phase_scalar_busy_share_of_phase_total": ( + phase_scalar_observed["sum"] / phase_total_observed["sum"] if phase_total_observed["sum"] else 0.0 + ), + "phase_request_observed_share_of_primary": requests_observed["sum"] / primary_requests, + "phase_request_observed_plus_capture_gap_share_of_primary": ( + requests_observed_plus_gap["sum"] / primary_requests + ), + "phase_miss_observed_share_of_primary": misses_observed["sum"] / primary_misses if primary_misses else 0.0, + "phase_miss_observed_plus_capture_gap_share_of_primary": ( + misses_observed_plus_gap["sum"] / primary_misses if primary_misses else 0.0 + ), + "phase_total_cycles_observed_per_call": (phase_total_observed["sum"] / phase_calls if phase_calls else None), + "phase_scalar_busy_observed_per_call": (phase_scalar_observed["sum"] / phase_calls if phase_calls else None), + "phase_non_scalar_busy_cycles_per_call": phase_non_scalar["sum"] / phase_calls if phase_calls else None, + "phase_icache_requests_observed_per_call": requests_observed["sum"] / phase_calls if phase_calls else None, + "phase_icache_misses_observed_per_call": misses_observed["sum"] / phase_calls if phase_calls else None, + "phase_begin_reads": sum(record["phase_begin_reads"] for record in records), + "phase_end_reads": sum(record["phase_end_reads"] for record in records), + "phase_business_calls": phase_calls, + "phase_calls_per_core": _value_summary(phase_calls_per_core), + "phase_zero_call_cores": sum(calls == 0 for calls in phase_calls_per_core), + } + if mode in KERNEL_EXCLUDING_PHASE_CAPTURE_MODES: + summary["phase_excluded_kernel_calls"] = sum(record["phase_excluded_kernel_calls"] for record in records) + return summary + + +def _validate_summary_number(actual: Any, expected: int | float, path: str) -> None: + if isinstance(expected, int): + if isinstance(actual, bool) or not isinstance(actual, int) or actual != expected: + _fail(f"{path} does not match the host recomputation: expected {expected!r}, got {actual!r}") + return + value = _number(actual, path) + # The production C++ emitter serializes floating summaries with 12 + # significant digits. Integer sum/min/max remain exact; this tolerance is + # only for the mean and weighted ratios reconstructed from those integers. + if not math.isclose(value, expected, rel_tol=1e-10, abs_tol=1e-12): + _fail(f"{path} does not match the host recomputation: expected {expected!r}, got {actual!r}") + + +def _validate_host_summary(data: dict[str, Any], computed: Mapping[str, dict[str, Any]]) -> None: + supplied = _object(data.get("summary"), "summary") + for group_name in GROUP_NAMES: + supplied_group = _object(supplied.get(group_name), f"summary.{group_name}") + expected_group = computed[group_name] + _validate_summary_number(supplied_group.get("cores"), expected_group["cores"], f"summary.{group_name}.cores") + for metric in METRICS: + supplied_metric = _object(supplied_group.get(metric), f"summary.{group_name}.{metric}") + for statistic in ("sum", "min", "mean", "max"): + _validate_summary_number( + supplied_metric.get(statistic), + expected_group[metric][statistic], + f"summary.{group_name}.{metric}.{statistic}", + ) + for ratio in ("scalar_busy_share", "icache_miss_rate"): + _validate_summary_number( + supplied_group.get(ratio), + expected_group[ratio], + f"summary.{group_name}.{ratio}", + ) + + +def _validate_producer_summary(data: dict[str, Any], mode: str) -> None: + validation = _object(data.get("validation"), "validation") + kernel_exclusion_validation = "phase_kernel_exclusion_closed_records" + dynamic_call_validation = "phase_global_call_count_closed" + required = { + "passed": True, + "trusted_records": EXPECTED_CORES, + "unique_physical_core_ids": EXPECTED_CORES, + "aic_records": EXPECTED_AIC_CORES, + "aiv_records": EXPECTED_AIV_CORES, + "mixed_triplets": EXPECTED_AIC_CORES, + "owner_bitmap_member_records": EXPECTED_CORES, + "status_match_records": EXPECTED_CORES, + "selector_match_records": EXPECTED_CORES, + "window_started_records": EXPECTED_CORES, + "window_stopped_records": EXPECTED_CORES, + "submit_count_closed_records": EXPECTED_CORES, + "scalar_le_total_records": EXPECTED_CORES, + "icache_miss_le_request_records": EXPECTED_CORES, + "counter_below_risk_threshold_records": EXPECTED_CORES, + "linked_kernel_gate_closed_records": EXPECTED_CORES, + "scalar_submit_elapsed_valid_records": EXPECTED_CORES, + "vector_busy_zero_records": EXPECTED_CORES, + "cube_busy_zero_records": EXPECTED_CORES, + "return_ready_atomic_time_valid_records": EXPECTED_CORES, + } + if mode == NONE_CAPTURE_MODE: + required.update( + { + "shadow_icache_primary_match_records": EXPECTED_CORES, + "shadow_scalar_primary_match_records": EXPECTED_CORES, + } + ) + else: + if "phase_time_bounded_records" in validation: + _fail("validation must use phase_time_within_submit_records, not phase_time_bounded_records") + required.update( + { + "phase_boundary_closed_records": EXPECTED_CORES, + "phase_shape_match_records": EXPECTED_CORES, + "phase_icache_values_ordered_records": EXPECTED_CORES, + "phase_pmu_values_ordered_records": EXPECTED_CORES, + "phase_counter_reconstruction_valid_records": EXPECTED_CORES, + "phase_time_within_submit_records": EXPECTED_CORES, + "shadow_icache_primary_bounded_records": EXPECTED_CORES, + "shadow_scalar_primary_bounded_records": EXPECTED_CORES, + } + ) + if mode in KERNEL_EXCLUDING_PHASE_CAPTURE_MODES: + required[kernel_exclusion_validation] = EXPECTED_CORES + elif kernel_exclusion_validation in validation: + _fail( + f"validation.{kernel_exclusion_validation} is only valid in " + f"{sorted(KERNEL_EXCLUDING_PHASE_CAPTURE_MODES)!r}" + ) + if mode in DYNAMIC_PHASE_CAPTURE_MODES: + required[dynamic_call_validation] = True + elif dynamic_call_validation in validation: + _fail(f"validation.{dynamic_call_validation} is only valid in dynamic phase profiles") + for field, expected in required.items(): + _require_equal(validation.get(field), expected, f"validation.{field}") + + +def _validate_dynamic_phase_call_totals( + records: Sequence[dict[str, Any]], + mode: str, + expected_submits: int, +) -> None: + if mode not in DYNAMIC_PHASE_CAPTURE_MODES: + return + expected = _expected_dynamic_phase_calls(mode, expected_submits) + actual = { + "all": sum(_phase_business_calls(record, mode) for record in records), + "aic": sum(_phase_business_calls(record, mode) for record in records if record["role"] == "aic"), + "aiv": sum(_phase_business_calls(record, mode) for record in records if record["role"] == "aiv"), + } + max_calls_per_core = _dynamic_phase_max_calls_per_core(mode, expected_submits) + for logical_core_id, record in enumerate(records): + business_calls = _phase_business_calls(record, mode) + if business_calls > max_calls_per_core: + _fail( + f"records[{logical_core_id}] business phase calls exceed dynamic per-core maximum {max_calls_per_core}" + ) + if actual["all"] != expected["all"] or actual["aic"] + actual["aiv"] != actual["all"]: + _fail(f"dynamic phase call totals: global call total must equal {expected['all']}, got {actual!r}") + if mode in FIXED_ROLE_DYNAMIC_PHASE_CAPTURE_MODES and actual != expected: + _fail(f"dynamic phase call totals must equal {expected!r}, got {actual!r}") + + +def load_capture(input_path: Path | str) -> SubmitPmuCapture: + """Read and strictly validate the fixed production Submit-PMU artifact.""" + + path = Path(input_path) + if path.name != DEFAULT_INPUT_NAME: + _fail(f"Submit-PMU input filename must be {DEFAULT_INPUT_NAME!r}") + raw_bytes = path.read_bytes() + decoded = json.loads(raw_bytes) + data = _object(decoded, "root") + mode, _, expected_submits, _ = _validate_capture_header(data) + + record_data = _array(data.get("records"), "records") + if len(record_data) != EXPECTED_CORES: + _fail("records must contain exactly 96 cores") + records = tuple( + _validate_record(value, logical_id, expected_submits, mode) for logical_id, value in enumerate(record_data) + ) + role_counts = Counter(record["role"] for record in records) + if role_counts != Counter({"aic": EXPECTED_AIC_CORES, "aiv": EXPECTED_AIV_CORES}): + _fail("records must contain exactly 32 AIC and 64 AIV cores") + + physical_ids = _validate_topology(records) + _validate_logical_layout(records) + _validate_owner(data, physical_ids) + _validate_window(data, records) + _validate_producer_summary(data, mode) + _validate_dynamic_phase_call_totals(records, mode, expected_submits) + + groups = { + "all": records, + "aic": tuple(record for record in records if record["role"] == "aic"), + "aiv": tuple(record for record in records if record["role"] == "aiv"), + } + summary = {name: _group_summary(group) for name, group in groups.items()} + phase_summary = ( + None + if mode == NONE_CAPTURE_MODE + else {name: _phase_group_summary(group, mode) for name, group in groups.items()} + ) + _validate_host_summary(data, summary) + return SubmitPmuCapture( + input_path=path, + raw_size=len(raw_bytes), + raw_sha256=hashlib.sha256(raw_bytes).hexdigest(), + data=data, + records=records, + groups=groups, + summary=summary, + phase_summary=phase_summary, + ) + + +def _recording_reference_metric( + *, + unit: str, + phase_field: str, + whole_field: str, + target_group: Mapping[str, Any], + target_whole: Mapping[str, Any], + empty_group: Mapping[str, Any], + target_record_pairs: int, + empty_record_pairs: int, +) -> dict[str, Any]: + raw_phase_sum = int(target_group[phase_field]["sum"]) + raw_whole_sum = int(target_whole[whole_field]["sum"]) + empty_cost_per_record_pair = int(empty_group[phase_field]["sum"]) / empty_record_pairs + recording_cost_estimate_sum = empty_cost_per_record_pair * target_record_pairs + reference_sum = raw_phase_sum - recording_cost_estimate_sum + return { + "unit": unit, + "phase_field": phase_field, + "whole_field": whole_field, + "raw_phase_observed_sum": raw_phase_sum, + "raw_whole_sum": raw_whole_sum, + "raw_phase_observed_ratio_to_raw_whole": (raw_phase_sum / raw_whole_sum if raw_whole_sum else None), + "empty_cost_per_record_pair": empty_cost_per_record_pair, + "recording_cost_estimate_sum": recording_cost_estimate_sum, + "recording_cost_estimate_share_of_raw_phase": ( + recording_cost_estimate_sum / raw_phase_sum if raw_phase_sum else None + ), + "after_recording_cost_reference_sum": reference_sum, + "after_recording_cost_reference_ratio_to_raw_whole": (reference_sum / raw_whole_sum if raw_whole_sum else None), + } + + +def _validate_phase_recording_cost_inputs( + target_capture: SubmitPmuCapture, + empty_capture: SubmitPmuCapture, +) -> None: + target_mode = str(target_capture.data["capture"]["mode"]) + empty_mode = str(empty_capture.data["capture"]["mode"]) + if target_mode in {NONE_CAPTURE_MODE, EMPTY_BRACKET_CAPTURE_MODE} or target_capture.phase_summary is None: + _fail("recording-cost reference target must be one business phase capture") + if empty_mode != EMPTY_BRACKET_CAPTURE_MODE or empty_capture.phase_summary is None: + _fail(f"recording-cost calibration must use {EMPTY_BRACKET_CAPTURE_MODE}") + + target_configuration = target_capture.data["configuration"] + empty_configuration = empty_capture.data["configuration"] + if target_configuration["expected_submits_per_core"] != empty_configuration["expected_submits_per_core"]: + _fail("recording-cost target and empty-bracket expected submits do not match") + for field in PHASE_CALIBRATION_STABLE_CONFIGURATION_FIELDS: + if target_configuration[field] != empty_configuration[field]: + _fail(f"recording-cost target and empty-bracket configuration.{field} do not match") + if target_configuration["phase"]["pmu_observation"] != empty_configuration["phase"]["pmu_observation"]: + _fail("recording-cost target and empty-bracket phase PMU observation semantics do not match") + if target_capture.data["capture"]["window_scope"] != empty_capture.data["capture"]["window_scope"]: + _fail("recording-cost target and empty-bracket window scopes do not match") + + target_topology = tuple( + ( + record["logical_core_id"], + record["physical_core_id"], + record["role"], + record["block_id"], + record["lane"], + ) + for record in target_capture.records + ) + empty_topology = tuple( + ( + record["logical_core_id"], + record["physical_core_id"], + record["role"], + record["block_id"], + record["lane"], + ) + for record in empty_capture.records + ) + if target_topology != empty_topology: + _fail("recording-cost target and empty-bracket core topology do not match") + + +def build_phase_recording_cost_reference( + target_capture: SubmitPmuCapture, + empty_capture: SubmitPmuCapture, +) -> dict[str, Any]: + """Estimate phase-local recording work without modifying the raw whole denominator. + + The empty-bracket capture measures only code executed inside one phase + begin/end recording pair. It therefore supports a reference value for the + phase observed numerator, but it does not measure all recording work in the + whole Submit window. The raw whole denominator is deliberately left intact. + """ + + _validate_phase_recording_cost_inputs(target_capture, empty_capture) + assert target_capture.phase_summary is not None + assert empty_capture.phase_summary is not None + + groups: dict[str, Any] = {} + for group_name in ("aic", "aiv"): + target_group = target_capture.phase_summary[group_name] + empty_group = empty_capture.phase_summary[group_name] + target_record_pairs = int(target_group["phase_end_reads"]) + empty_record_pairs = int(empty_group["phase_end_reads"]) + if empty_record_pairs <= 0: + _fail(f"empty-bracket {group_name} group has no complete begin/end record pair") + metrics = { + metric_name: _recording_reference_metric( + unit=unit, + phase_field=phase_field, + whole_field=whole_field, + target_group=target_group, + target_whole=target_capture.summary[group_name], + empty_group=empty_group, + target_record_pairs=target_record_pairs, + empty_record_pairs=empty_record_pairs, + ) + for metric_name, unit, phase_field, whole_field in PHASE_RECORDING_REFERENCE_METRICS + } + groups[group_name] = { + "cores": int(target_capture.summary[group_name]["cores"]), + "target_record_pairs": target_record_pairs, + "empty_record_pairs": empty_record_pairs, + "metrics": metrics, + } + + target_all = target_capture.phase_summary["all"] + empty_all = empty_capture.phase_summary["all"] + target_all_pairs = int(target_all["phase_end_reads"]) + empty_all_pairs = int(empty_all["phase_end_reads"]) + role_target_pairs = sum(int(groups[role]["target_record_pairs"]) for role in ("aic", "aiv")) + role_empty_pairs = sum(int(groups[role]["empty_record_pairs"]) for role in ("aic", "aiv")) + if target_all_pairs != role_target_pairs: + _fail("target ALL record-pair count does not equal AIC plus AIV") + if empty_all_pairs != role_empty_pairs: + _fail("empty-bracket ALL record-pair count does not equal AIC plus AIV") + + all_metrics: dict[str, Any] = {} + for metric_name, unit, phase_field, whole_field in PHASE_RECORDING_REFERENCE_METRICS: + role_metrics = [groups[role]["metrics"][metric_name] for role in ("aic", "aiv")] + raw_phase_sum = sum(int(metric["raw_phase_observed_sum"]) for metric in role_metrics) + raw_whole_sum = sum(int(metric["raw_whole_sum"]) for metric in role_metrics) + direct_raw_phase_sum = int(target_all[phase_field]["sum"]) + direct_raw_whole_sum = int(target_capture.summary["all"][whole_field]["sum"]) + if raw_phase_sum != direct_raw_phase_sum: + _fail(f"{metric_name} target ALL phase sum does not equal AIC plus AIV") + if raw_whole_sum != direct_raw_whole_sum: + _fail(f"{metric_name} target ALL whole sum does not equal AIC plus AIV") + recording_cost_estimate_sum = sum(float(metric["recording_cost_estimate_sum"]) for metric in role_metrics) + reference_sum = raw_phase_sum - recording_cost_estimate_sum + all_metrics[metric_name] = { + "unit": unit, + "phase_field": phase_field, + "whole_field": whole_field, + "raw_phase_observed_sum": raw_phase_sum, + "raw_whole_sum": raw_whole_sum, + "raw_phase_observed_ratio_to_raw_whole": (raw_phase_sum / raw_whole_sum if raw_whole_sum else None), + "empty_cost_per_record_pair": ( + recording_cost_estimate_sum / target_all_pairs if target_all_pairs else None + ), + "recording_cost_estimate_sum": recording_cost_estimate_sum, + "recording_cost_estimate_share_of_raw_phase": ( + recording_cost_estimate_sum / raw_phase_sum if raw_phase_sum else None + ), + "after_recording_cost_reference_sum": reference_sum, + "after_recording_cost_reference_ratio_to_raw_whole": ( + reference_sum / raw_whole_sum if raw_whole_sum else None + ), + } + groups["all"] = { + "cores": int(target_capture.summary["all"]["cores"]), + "target_record_pairs": target_all_pairs, + "empty_record_pairs": empty_all_pairs, + "metrics": all_metrics, + "all_values_are_aic_aiv_sums": True, + } + + total_metrics = groups["all"]["metrics"] + for field in ( + "raw_phase_observed_sum", + "raw_whole_sum", + "recording_cost_estimate_sum", + "after_recording_cost_reference_sum", + ): + total_value = float(total_metrics["pmu_total_cycles"][field]) + component_value = float(total_metrics["scalar_busy_cycles"][field]) + float( + total_metrics["non_scalar_busy_cycles"][field] + ) + if not math.isclose(total_value, component_value, rel_tol=1e-12, abs_tol=1e-9): + _fail(f"recording-cost reference total does not equal scalar plus non-scalar for {field}") + + return { + "kind": "phase-local-recording-cost-reference", + "target_capture_mode": str(target_capture.data["capture"]["mode"]), + "target_raw_path": str(target_capture.input_path.resolve()), + "target_raw_sha256": target_capture.raw_sha256, + "calibration_capture_mode": EMPTY_BRACKET_CAPTURE_MODE, + "calibration_raw_path": str(empty_capture.input_path.resolve()), + "calibration_raw_sha256": empty_capture.raw_sha256, + "exact_correction": False, + "raw_whole_denominator_is_unchanged": True, + "whole_recording_cost_is_not_measured": True, + "groups": groups, + } + + +def _assert_capture_raw_unchanged(capture: SubmitPmuCapture) -> bytes: + """Re-read one raw and prove it still equals the snapshot accepted by the loader.""" + + raw_bytes = capture.input_path.read_bytes() + if len(raw_bytes) != capture.raw_size or hashlib.sha256(raw_bytes).hexdigest() != capture.raw_sha256: + _fail("Submit-PMU raw changed after the validated capture snapshot") + return raw_bytes + + +def _artifact_payload(identity: BuildArtifactIdentity) -> dict[str, Any]: + return { + "path": str(identity.path), + "sha256": identity.sha256, + "size_bytes": identity.size_bytes, + "text": {"sha256": identity.text_sha256, "size_bytes": identity.text_size_bytes}, + } + + +def _build_provenance_payload( + capture: SubmitPmuCapture, + identity: SubmitPmuBuildIdentity, +) -> dict[str, Any]: + if capture.data["capture"]["mode"] != identity.profile: + _fail("Submit-PMU build identity profile does not match the raw capture mode") + source_version, git_head, source_fingerprint, definitions_sha256 = _parse_source_state(identity.source_state) + stamp = identity.source_state_path + if not stamp.is_file() or stamp.read_text(encoding="utf-8").strip() != identity.source_state: + _fail("Submit-PMU AICore source-state stamp changed after the build identity was frozen") + + artifacts: dict[str, Any] = {} + for name, frozen in identity.artifacts: + current = _inspect_build_artifact(frozen.path) + if current != frozen: + _fail(f"Submit-PMU build artifact changed after identity freeze: {name}") + artifacts[name] = _artifact_payload(frozen) + return { + "schema": PROVENANCE_SCHEMA_NAME, + "binding": { + "raw_name": capture.input_path.name, + "raw_size": capture.raw_size, + "raw_sha256": capture.raw_sha256, + "capture_mode": identity.profile, + }, + "build": { + "profile": identity.profile, + "tensormap_mode": identity.tensormap_mode, + "profiled_cache_key": list(identity.profiled_cache_key), + "aicore_extra_cache_key": identity.aicore_extra_cache_key, + "compile_definitions": list(identity.compile_definitions), + "source_state": identity.source_state, + "source_state_path": str(identity.source_state_path), + "source_state_version": source_version, + "git_head": git_head, + "source_fingerprint": source_fingerprint, + "definitions_sha256": definitions_sha256, + }, + "artifacts": artifacts, + } + + +def _json_document(data: Mapping[str, Any]) -> str: + return json.dumps(data, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + + +def _exact_keys(data: Mapping[str, Any], expected: set[str], path: str) -> None: + actual = set(data) + if actual != expected: + _fail( + f"{path} fields do not match the fixed provenance schema: expected {sorted(expected)}, got {sorted(actual)}" + ) + + +def _validate_provenance_artifact(value: Any, path: str) -> None: + artifact = _object(value, path) + _exact_keys(artifact, {"path", "sha256", "size_bytes", "text"}, path) + if not isinstance(artifact["path"], str) or not artifact["path"]: + _fail(f"{path}.path must be a non-empty string") + if not isinstance(artifact["sha256"], str) or _HEX_64_PATTERN.fullmatch(artifact["sha256"]) is None: + _fail(f"{path}.sha256 must contain 64 lowercase hex digits") + _integer(artifact["size_bytes"], f"{path}.size_bytes", minimum=1) + text = _object(artifact["text"], f"{path}.text") + _exact_keys(text, {"sha256", "size_bytes"}, f"{path}.text") + if not isinstance(text["sha256"], str) or _HEX_64_PATTERN.fullmatch(text["sha256"]) is None: + _fail(f"{path}.text.sha256 must contain 64 lowercase hex digits") + _integer(text["size_bytes"], f"{path}.text.size_bytes", minimum=1) + if text["size_bytes"] > artifact["size_bytes"]: + _fail(f"{path}.text.size_bytes exceeds the whole artifact size") + + +def _validate_provenance_data(data: dict[str, Any], capture: SubmitPmuCapture) -> dict[str, Any]: + """Validate one decoded sidecar against a single accepted raw snapshot.""" + + _exact_keys(data, {"schema", "binding", "build", "artifacts"}, "provenance") + if data["schema"] != PROVENANCE_SCHEMA_NAME: + _fail(f"provenance.schema must equal {PROVENANCE_SCHEMA_NAME!r}") + + binding = _object(data["binding"], "provenance.binding") + _exact_keys(binding, {"raw_name", "raw_size", "raw_sha256", "capture_mode"}, "provenance.binding") + expected_binding = { + "raw_name": capture.input_path.name, + "raw_size": capture.raw_size, + "raw_sha256": capture.raw_sha256, + "capture_mode": capture.data["capture"]["mode"], + } + if binding != expected_binding: + _fail("provenance.binding does not match the validated Submit-PMU raw") + + build = _object(data["build"], "provenance.build") + _exact_keys( + build, + { + "profile", + "tensormap_mode", + "profiled_cache_key", + "aicore_extra_cache_key", + "compile_definitions", + "source_state", + "source_state_path", + "source_state_version", + "git_head", + "source_fingerprint", + "definitions_sha256", + }, + "provenance.build", + ) + if not isinstance(build["source_state"], str): + _fail("provenance.build.source_state must be a string") + if not isinstance(build["source_state_path"], str) or not build["source_state_path"]: + _fail("provenance.build.source_state_path must be a non-empty string") + source_version, git_head, source_fingerprint, definitions_sha256 = _parse_source_state(build["source_state"]) + if ( + build["profile"] != binding["capture_mode"] + or build["source_state_version"] != source_version + or build["git_head"] != git_head + or build["source_fingerprint"] != source_fingerprint + or build["definitions_sha256"] != definitions_sha256 + ): + _fail("provenance.build does not close against its profile/source state") + cache_key = build["profiled_cache_key"] + if not isinstance(cache_key, list) or not cache_key or not all(isinstance(value, str) for value in cache_key): + _fail("provenance.build.profiled_cache_key must be a non-empty string array") + if ( + build["tensormap_mode"] not in {"private", "shared"} + or len(cache_key) < 2 + or cache_key[-2:] != [build["tensormap_mode"], build["profile"]] + ): + _fail("provenance.build.profiled_cache_key must end with TensorMap mode and selected profile") + if ( + not isinstance(build["aicore_extra_cache_key"], str) + or _HEX_16_PATTERN.fullmatch(build["aicore_extra_cache_key"]) is None + ): + _fail("provenance.build.aicore_extra_cache_key must contain 16 lowercase hex digits") + definitions = build["compile_definitions"] + if not isinstance(definitions, list) or not definitions or not all(isinstance(value, str) for value in definitions): + _fail("provenance.build.compile_definitions must be a non-empty string array") + if tuple(definitions) != _expected_compile_definitions(build["profile"], build["tensormap_mode"]): + _fail("provenance.build compile definitions do not match the selected profile") + if hashlib.sha256(repr(definitions).encode("utf-8")).hexdigest() != definitions_sha256: + _fail("provenance.build compile definitions do not match definitions_sha256") + + artifacts = _object(data["artifacts"], "provenance.artifacts") + expected_artifacts = { + "aicore_kernel", + "aic_combined", + "aiv_combined", + "host_runtime", + "aicpu_runtime", + } + _exact_keys(artifacts, expected_artifacts, "provenance.artifacts") + for name in sorted(expected_artifacts): + _validate_provenance_artifact(artifacts[name], f"provenance.artifacts.{name}") + return data + + +def load_provenance( + provenance_path: Path | str, + capture: SubmitPmuCapture, +) -> tuple[dict[str, Any], str]: + """Validate a provenance sidecar and its immutable binding to one raw capture.""" + + path = Path(provenance_path) + if path.name != DEFAULT_PROVENANCE_NAME: + _fail(f"Submit-PMU provenance filename must be {DEFAULT_PROVENANCE_NAME!r}") + raw_bytes = path.read_bytes() + data = _validate_provenance_data(_object(json.loads(raw_bytes), "provenance"), capture) + return data, hashlib.sha256(raw_bytes).hexdigest() + + +def _validate_recording_reference_provenance( + target: Mapping[str, Any], + calibration: Mapping[str, Any], +) -> dict[str, Any]: + """Bind one empty-bracket estimate to the same source revision and scenario.""" + + target_build = target["build"] + calibration_build = calibration["build"] + if target_build["git_head"] != calibration_build["git_head"]: + _fail("recording-cost target and empty-bracket provenance git heads do not match") + target_key = tuple(str(value) for value in target_build["profiled_cache_key"]) + calibration_key = tuple(str(value) for value in calibration_build["profiled_cache_key"]) + if len(target_key) < 2 or len(calibration_key) < 2 or target_key[:-1] != calibration_key[:-1]: + _fail("recording-cost target and empty-bracket provenance scenarios do not match") + if target_build["source_state_version"] != calibration_build["source_state_version"]: + _fail("recording-cost target and empty-bracket source-state versions do not match") + return { + "verified": True, + "git_head": str(target_build["git_head"]), + "source_state_version": str(target_build["source_state_version"]), + "profiled_cache_key_prefix": list(target_key[:-1]), + } + + +def _format_integer(value: int | float) -> str: + return f"{int(value):,}" + + +def _format_number(value: int | float, digits: int = 3) -> str: + return f"{float(value):,.{digits}f}" + + +def _format_optional_percent(value: float | None) -> str: + return "N/A" if value is None else f"{value:.3%}" + + +def _metric_range(summary: Mapping[str, int | float], *, digits: int = 1) -> str: + return ( + f"均值 {_format_number(summary['mean'], digits)}；" + f"最小 {_format_integer(summary['min'])}；最大 {_format_integer(summary['max'])}" + ) + + +def _metric_extrema(summary: Mapping[str, int | float]) -> str: + """Format only the per-core minimum and maximum for I-cache counters.""" + + return f"最小 {_format_integer(summary['min'])}；最大 {_format_integer(summary['max'])}" + + +def _scaled_metric_range( + summary: Mapping[str, int | float], + scale: float, + *, + digits: int = 3, +) -> str: + """Format mean/min/max after a monotonic unit conversion.""" + + return ( + f"均值 {_format_number(float(summary['mean']) * scale, digits)}；" + f"最小 {_format_number(float(summary['min']) * scale, digits)}；" + f"最大 {_format_number(float(summary['max']) * scale, digits)}" + ) + + +def _scaled_metric_extrema( + summary: Mapping[str, int | float], + scale: float, + *, + digits: int = 3, +) -> str: + """Format only scaled per-core extrema for an I-cache-derived quantity.""" + + return ( + f"最小 {_format_number(float(summary['min']) * scale, digits)}；" + f"最大 {_format_number(float(summary['max']) * scale, digits)}" + ) + + +def _group_card(name: str, summary: Mapping[str, Any], cycles_per_ns: float, miss_penalty_ns: float) -> str: + wall_elapsed = summary["submit_elapsed_ticks"] + scalar_elapsed = summary["scalar_submit_elapsed_ticks"] + excluded_wall = summary["scalar_denominator_excluded_wall_ticks"] + total = summary["total_cycles"] + scalar = summary["scalar_busy"] + non_scalar_busy = summary["non_scalar_busy_cycles"] + requests = summary["icache_requests"] + misses = summary["icache_misses"] + cycles_to_us = 1.0 / cycles_per_ns / 1_000 + title = {"all": "ALL", "aic": "AIC", "aiv": "AIV"}[name] + return f""" +
+

{title} · {summary["cores"]} 核

+
+
Submit PMU total/core
+
{_metric_range(total)} cycles
+ 等效时间 {_scaled_metric_range(total, cycles_to_us)} µs + （按 {cycles_per_ns:.6f} cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
{_metric_range(scalar)} cycles
+ 等效时间 {_scaled_metric_range(scalar, cycles_to_us)} µs； + 加权占比 {float(summary["scalar_busy_share"]):.3%} +
+
非 Scalar-busy 残余/core
+
{_metric_range(non_scalar_busy)} cycles
+ 逐核先算 total−scalar；等效时间 + {_scaled_metric_range(non_scalar_busy, cycles_to_us)} µs +
+
SYS gate 边界诊断/core
+
{_metric_range(scalar_elapsed)} raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
{_scaled_metric_range(wall_elapsed, 1 / 1_000)} µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
{_metric_range(excluded_wall)} raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + {float(summary["scalar_window_share_of_wall"]):.3%} +
+
Primary I-cache request/core
{_metric_extrema(requests)}
+
Primary I-cache miss/core
{_metric_extrema(misses)}
+
加权 miss rate
{float(summary["icache_miss_rate"]):.3%}（Σmiss/Σrequest）
+
{miss_penalty_ns:g} ns 直觉量尺/core
+
{_scaled_metric_extrema(misses, miss_penalty_ns / 1_000)} µs
+
+
+ """ + + +def _chart_svg(records: Sequence[dict[str, Any]], metric: str, title: str) -> str: + width, height = 1_080, 240 + left, right, top, bottom = 64, 20, 32, 42 + plot_width = width - left - right + plot_height = height - top - bottom + values = [float(record[metric]) for record in records] + maximum = max(values) or 1.0 + points = [] + for record, value in zip(records, values): + x = left + record["physical_core_id"] / (PHYSICAL_CORES - 1) * plot_width + y = top + (1.0 - value / maximum) * plot_height + color = "#2563eb" if record["role"] == "aic" else "#059669" + tooltip = html.escape( + f"physical={record['physical_core_id']} logical={record['logical_core_id']} " + f"role={record['role'].upper()} {metric}={int(value)}" + ) + points.append(f'{tooltip}') + return f""" +
+
{html.escape(title)}
+ + + + {_format_number(maximum, 0)} + 0 + physical 0 + physical 107 + {"".join(points)} + +
+ """ + + +def _per_core_rows(records: Sequence[dict[str, Any]], miss_penalty_ns: float) -> str: + rows = [] + for record in sorted(records, key=lambda item: item["physical_core_id"]): + request = record["icache_requests"] + miss = record["icache_misses"] + miss_rate = miss / request if request else 0.0 + rows.append( + "" + f"{record['physical_core_id']}{record['logical_core_id']}" + f"{record['role'].upper()}{record['block_id']}{record['lane']}" + f"{_format_integer(record['scalar_submit_elapsed_ticks'])}" + f"{_format_number(record['submit_elapsed_ticks'] / 1_000)}" + f"{_format_integer(record['total_cycles'])}{_format_integer(record['scalar_busy'])}" + f"{_format_integer(request)}{_format_integer(miss)}{miss_rate:.3%}" + f"{_format_number(miss * miss_penalty_ns / 1_000)}" + "" + ) + return "".join(rows) + + +def _phase_observed_cell( + observed: Mapping[str, int | float], + observed_plus_capture_gap: Mapping[str, int | float], + primary: int, + observed_share: float, + observed_plus_capture_gap_share: float, + recording_reference: Mapping[str, Any] | None = None, +) -> str: + capture_gap = int(observed_plus_capture_gap["sum"]) - int(observed["sum"]) + if recording_reference is not None: + return ( + "扣除记录代码开销估算后的参考值 " + f"{_format_number(recording_reference['after_recording_cost_reference_sum'])}
" + f"原始 observed {_format_integer(observed['sum'])} − 记录代码开销估算 " + f"{_format_number(recording_reference['recording_cost_estimate_sum'])}
" + f"参考值 / 原始整窗 " + f"{_format_optional_percent(recording_reference['after_recording_cost_reference_ratio_to_raw_whole'])}；" + f"原始 observed / 原始整窗 " + f"{_format_optional_percent(recording_reference['raw_phase_observed_ratio_to_raw_whole'])}
" + f"记录代码开销估算 / 原始 observed " + f"{_format_optional_percent(recording_reference['recording_cost_estimate_share_of_raw_phase'])}
" + f"原始逐核 {_metric_extrema(observed)}；原始整窗 {_format_integer(primary)}
" + f"原始 capture gap +{_format_integer(capture_gap)}；加 gap 后 " + f"{_format_integer(observed_plus_capture_gap['sum'])}" + f"（{observed_plus_capture_gap_share:.3%}）" + ) + return ( + f"原始 observed {_format_integer(observed['sum'])}
" + f"原始逐核 {_metric_extrema(observed)}
" + f"原始整窗 {_format_integer(primary)}；原始 observed / 原始整窗 " + f"{observed_share:.3%}（非业务占比）
" + f"原始 capture gap +{_format_integer(capture_gap)}；加 gap 后 " + f"{_format_integer(observed_plus_capture_gap['sum'])}（{observed_plus_capture_gap_share:.3%}）" + ) + + +def _phase_cycle_cell( + observed: Mapping[str, int | float], + cycles_per_ns: float, + per_call: float | None, + recording_reference: Mapping[str, Any] | None = None, +) -> str: + """Render one role-calibrated phase PMU quantity without using SYS ticks as time.""" + + elapsed_us = float(observed["sum"]) / cycles_per_ns / 1_000 + per_call_text = "—" if per_call is None else f"原始 {_format_number(per_call)} cycles/call" + if recording_reference is not None: + reference_sum = float(recording_reference["after_recording_cost_reference_sum"]) + estimate_sum = float(recording_reference["recording_cost_estimate_sum"]) + return ( + "扣除记录代码开销估算后的参考值 Σ " + f"{_format_number(reference_sum)} cycles
" + f"≈ {_format_number(reference_sum / cycles_per_ns / 1_000)} µs" + f"（{cycles_per_ns:.6f} cycles/ns）
" + f"原始 observed Σ {_format_integer(observed['sum'])} cycles" + f"（≈ {_format_number(elapsed_us)} µs）− 记录代码开销估算 " + f"{_format_number(estimate_sum)} cycles
" + f"记录代码开销估算 / 原始 observed " + f"{_format_optional_percent(recording_reference['recording_cost_estimate_share_of_raw_phase'])}；" + f"原始逐核 {_metric_extrema(observed)}；{per_call_text}" + ) + return ( + f"原始 observed Σ {_format_integer(observed['sum'])} cycles
" + f"≈ {_format_number(elapsed_us)} µs（{cycles_per_ns:.6f} cycles/ns）
" + f"原始逐核 {_metric_extrema(observed)}；{per_call_text}" + ) + + +def _phase_recording_note( + phase: Mapping[str, Any], + recording_cost_reference: Mapping[str, Any] | None, +) -> str: + if phase["id"] == EMPTY_BRACKET_PHASE_ID: + return """ +

empty-bracket 用于估算每次 begin/end 紧邻执行的记录代码开销， + 不是业务 phase。 + phase PMU total/scalar 是紧邻 begin/end 对本身带来的计数开销；request/miss observed + 仍只覆盖两次 shadow read-clear 之间。SYS tick 只用来核验边界是否闭合，不参与阶段主时间换算。 + 这些结果只能用于估算进入 phase observed 的局部记录代码，不能估算完整 whole + 窗口中的全部记录开销。

+ """ + if recording_cost_reference is None: + return """ +

本报告没有提供 empty-bracket 校准输入。 + 下表只能展示原始 phase observed 与本 ELF raw 整窗的比例；记录代码本身进入了 phase + observed，因此这些 raw 比例不能解释为业务阶段占比。若要生成扣除局部记录代码开销估算后的 + 参考值，请用 --calibration-input 指定对应的 empty-bracket raw。

+ """ + + calibration_path = html.escape(str(recording_cost_reference["calibration_raw_path"])) + calibration_sha = html.escape(str(recording_cost_reference["calibration_raw_sha256"])) + provenance_binding = recording_cost_reference.get("provenance_binding") + if provenance_binding and provenance_binding["verified"]: + provenance_note = ( + "两份 provenance 已核验为同一场景、同一 Git revision " + f"{html.escape(str(provenance_binding['git_head'])[:12])}。" + ) + else: + provenance_note = ( + "两份 raw 均未提供 provenance sidecar；当前只校验了 raw 配置、窗口语义和核拓扑，" + "没有证明它们来自同一代码 revision。" + ) + return f""" +

本报告使用 empty-bracket 估算进入 phase observed + 的局部记录代码开销。 + AIC/AIV 分别按“空区间每组记录开销 × 本阶段实际 begin/end 记录组数”计算，ALL + 再由两类核相加。参考值只从 phase observed 分子扣除该估算；原始 whole 分母保持不变， + 因为空区间没有测量 whole 窗口中的全部记录开销。所以下表的参考比例不是完整业务 + 阶段占完整调度窗口的比例，也不是精确校正。若参考值为负，表示记录代码开销估算已经 + 超过原始 observed；这只能说明该信号低于当前校准的分辨能力，不能截成零或解释成负的 + 业务事件。校准 raw：{calibration_path} · + SHA-256 {calibration_sha}。{provenance_note}

+ """ + + +def _phase_table_headers( + recording_cost_reference: Mapping[str, Any] | None, +) -> tuple[str, str, str]: + if recording_cost_reference is None: + return ( + "原始阶段关系（非业务占比）", + "Request raw observed（总数 / 逐核 min–max / 原始整窗）", + "Miss raw observed（总数 / 逐核 min–max / 原始整窗）", + ) + return ( + "阶段参考关系（raw 同格保留）", + "Request（参考主值 / raw 明细）", + "Miss（参考主值 / raw 明细）", + ) + + +def _phase_overview( + capture: SubmitPmuCapture, + recording_cost_reference: Mapping[str, Any] | None = None, +) -> str: + if capture.phase_summary is None: + return "" + + phase = capture.data["configuration"]["phase"] + phase_name = html.escape(phase["name"]) + phase_boundary = html.escape(phase["boundary"]) + counter_semantics = html.escape(phase["counter_semantics"]) + time_semantics = html.escape(phase["time_semantics"]) + if "expected_calls_per_core" in phase: + call_shape_text = f"expected_calls_per_core={phase['expected_calls_per_core']}" + else: + expected = phase["expected_calls"] + if phase["call_shape"] == "dynamic_global": + call_shape_text = f"call_shape=dynamic_global · expected_calls=ALL {expected['all']}（角色不锁定）" + else: + call_shape_text = ( + f"call_shape={phase['call_shape']} · expected_calls=" + f"ALL {expected['all']} / AIC {expected['aic']} / AIV {expected['aiv']}" + ) + calibration_note = _phase_recording_note(phase, recording_cost_reference) + control_exclusion_note = "" + if capture.data["capture"]["mode"] in KERNEL_EXCLUDING_PHASE_CAPTURE_MODES: + control_exclusion_note = """ +

本阶段若跨过 linked Kernel，会同时门控 SYS 时间与 PMU counter， + 因而 total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段。 + result-used return-ready atomic 依赖区间只从 SYS 边界累计值扣除，PMU counter + 仍含 atomic 指令事件；source-issue atomic + 保留在时间和计数口径内。没有 linked Kernel 的阶段不触发 PMU 停/开表； + 每次调用的分母仍是外层业务调用次数，不是 Begin/End 读数，也不是排除的 Kernel 调用数。

+ """ + relationship_header, request_header, miss_header = _phase_table_headers(recording_cost_reference) + + group_cells: dict[str, dict[str, str]] = {} + group_headers: dict[str, str] = {} + frequencies = capture.data["configuration"]["pmu_cycles_per_ns"] + for group_name in GROUP_NAMES: + group = capture.phase_summary[group_name] + reference_group = None if recording_cost_reference is None else recording_cost_reference["groups"][group_name] + reference_metrics = None if reference_group is None else reference_group["metrics"] + title = {"all": "ALL", "aic": "AIC", "aiv": "AIV"}[group_name] + group_headers[group_name] = f"{title}{group['cores']} 核" + cycles_per_ns = float(frequencies[group_name]) + total_observed = _phase_cycle_cell( + group["phase_total_cycles_observed"], + cycles_per_ns, + group["phase_total_cycles_observed_per_call"], + None if reference_metrics is None else reference_metrics["pmu_total_cycles"], + ) + scalar_observed = _phase_cycle_cell( + group["phase_scalar_busy_observed"], + cycles_per_ns, + group["phase_scalar_busy_observed_per_call"], + None if reference_metrics is None else reference_metrics["scalar_busy_cycles"], + ) + non_scalar = _phase_cycle_cell( + group["phase_non_scalar_busy_cycles"], + cycles_per_ns, + group["phase_non_scalar_busy_cycles_per_call"], + None if reference_metrics is None else reference_metrics["non_scalar_busy_cycles"], + ) + request_observed = _phase_observed_cell( + group["phase_icache_requests_observed"], + group["phase_icache_requests_observed_plus_capture_gap"], + group["primary_icache_requests"], + group["phase_request_observed_share_of_primary"], + group["phase_request_observed_plus_capture_gap_share_of_primary"], + None if reference_metrics is None else reference_metrics["icache_requests"], + ) + miss_observed = _phase_observed_cell( + group["phase_icache_misses_observed"], + group["phase_icache_misses_observed_plus_capture_gap"], + group["primary_icache_misses"], + group["phase_miss_observed_share_of_primary"], + group["phase_miss_observed_plus_capture_gap_share_of_primary"], + None if reference_metrics is None else reference_metrics["icache_misses"], + ) + reads = ( + "Begin / End：" + f"{_format_integer(group['phase_begin_reads'])} / " + f"{_format_integer(group['phase_end_reads'])}" + ) + if capture.data["capture"]["mode"] in KERNEL_EXCLUDING_PHASE_CAPTURE_MODES: + reads += ( + "
业务调用 " + f"{_format_integer(group['phase_business_calls'])} 次；排除 linked Kernel 调用 " + f"{_format_integer(group['phase_excluded_kernel_calls'])} 次" + ) + if capture.data["capture"]["mode"] in DYNAMIC_PHASE_CAPTURE_MODES: + calls = group["phase_calls_per_core"] + reads += ( + f"
逐核 {_format_integer(calls['min'])}–{_format_integer(calls['max'])}；" + f"零调用核 {_format_integer(group['phase_zero_call_cores'])}" + ) + if reference_metrics is None: + relationships = ( + "以下均为原始 observed，不能作为业务占比
" + f"原始 Scalar / 原始 Phase total " + f"{group['phase_scalar_busy_share_of_phase_total']:.3%}
" + f"原始 Non-scalar / 原始 Phase total " + f"{1.0 - group['phase_scalar_busy_share_of_phase_total']:.3%}
" + f"原始 Phase total / 原始 whole total " + f"{group['phase_total_share_of_pmu_total']:.3%}
" + f"原始 Phase scalar / 原始 whole scalar " + f"{group['phase_scalar_share_of_whole_scalar']:.3%}
" + f"whole scalar−shadow scalar：Σ " + f"{_format_integer(group['shadow_scalar_loss']['sum'])} cycles；逐核 " + f"{_metric_extrema(group['shadow_scalar_loss'])}" + ) + else: + total_reference = reference_metrics["pmu_total_cycles"] + scalar_reference = reference_metrics["scalar_busy_cycles"] + non_scalar_reference = reference_metrics["non_scalar_busy_cycles"] + total_reference_sum = float(total_reference["after_recording_cost_reference_sum"]) + scalar_reference_sum = float(scalar_reference["after_recording_cost_reference_sum"]) + non_scalar_reference_sum = float(non_scalar_reference["after_recording_cost_reference_sum"]) + scalar_of_reference_total = scalar_reference_sum / total_reference_sum if total_reference_sum else None + non_scalar_of_reference_total = ( + non_scalar_reference_sum / total_reference_sum if total_reference_sum else None + ) + relationships = ( + "扣除局部记录代码开销估算后的参考比例
" + f"参考 Scalar / 参考 Phase total " + f"{_format_optional_percent(scalar_of_reference_total)}
" + f"参考 Non-scalar / 参考 Phase total " + f"{_format_optional_percent(non_scalar_of_reference_total)}
" + f"参考 Phase total / 原始 whole total " + f"{_format_optional_percent(total_reference['after_recording_cost_reference_ratio_to_raw_whole'])}" + f"；原始 observed {group['phase_total_share_of_pmu_total']:.3%}
" + f"参考 Phase scalar / 原始 whole scalar " + f"{_format_optional_percent(scalar_reference['after_recording_cost_reference_ratio_to_raw_whole'])}" + f"；原始 observed {group['phase_scalar_share_of_whole_scalar']:.3%}
" + f"whole scalar−shadow scalar：Σ " + f"{_format_integer(group['shadow_scalar_loss']['sum'])} cycles；逐核 " + f"{_metric_extrema(group['shadow_scalar_loss'])}" + ) + sys_diagnostic = ( + f"Σ {_format_integer(group['phase_elapsed_ticks']['sum'])} raw ticks
" + f"逐核 {_metric_extrema(group['phase_elapsed_ticks'])}；仅边界诊断" + ) + group_cells[group_name] = { + "total": total_observed, + "scalar": scalar_observed, + "non_scalar": non_scalar, + "relationships": relationships, + "requests": request_observed, + "misses": miss_observed, + "sys": f"{sys_diagnostic}
{reads}", + } + + phase_rows = ( + ("Phase PMU total", "total"), + ("Phase scalar busy", "scalar"), + ("非 Scalar-busy 残余", "non_scalar"), + (relationship_header, "relationships"), + (request_header, "requests"), + (miss_header, "misses"), + ("SYS 边界诊断 / Begin-End", "sys"), + ) + rows = "".join( + "" + f"{html.escape(label)}" + + "".join(f"{group_cells[group_name][cell_key]}" for group_name in GROUP_NAMES) + + "" + for label, cell_key in phase_rows + ) + headers = "".join(f"{group_headers[group_name]}" for group_name in GROUP_NAMES) + return f""" +
+

{phase_name} 阶段观察（phase_id={phase["id"]}）

+

边界 {phase_boundary} · 计数语义 {counter_semantics} · + 时间语义 {time_semantics} · + {html.escape(call_shape_text)}

+ {calibration_note} + {control_exclusion_note} +

阶段原始 PMU 观测来自 running read-clear 的 total cycles，并按 AIC/AIV + 各自校准频率换算；提供 empty-bracket 时，页面主参考值再扣除局部记录代码开销估算。 + phase_elapsed_ticks 只是 SYS 边界闭合诊断，绝不按 1 GHz 当作阶段时间。 + Scalar busy 与 non-scalar residual（逐核先算 total−scalar）共同解释阶段 PMU total； + 未校准时只能展示 raw observed；提供 empty-bracket 后，记录代码开销估算只从 phase + observed 分子扣除，原始 whole 分母保持不变。当前数据无法得到完整业务阶段占完整调度 + 窗口的精确比例。 + linked vector/cube Kernel 从 SYS 与 PMU counter 一并门控排除；result-used return-ready atomic + 依赖区间只从 SYS 边界累计值扣除，I-cache/PMU counter 仍含其指令事件；source-issue atomic + 保留。request/miss 百分比的分母是同一 ELF、同一次采集的 Submit 整窗 primary。 + observed_plus_capture_gap = observed + (primary − shadow)； + 边界读数和插桩 bookkeeping 会进入 sample，因此“观测值”和“加全窗 capture gap”都不是原业务 + 事件数的数学上下界。不同 ELF 不能直接相减后冒充业务真值；提供 empty-bracket 时只按上方 + 明示的公式估算局部记录代码开销。request/miss 的逐核 min/max 是每核累计整个 phase + 的极值，不是逐调用极值。

+
+ + {headers} + + {rows} +
指标
+
+ """ + + +def _provenance_overview(provenance: Mapping[str, Any] | None, provenance_sha256: str | None) -> str: + if provenance is None: + return "" + build = provenance["build"] + artifacts = provenance["artifacts"] + labels = { + "aicore_kernel": "AICore final", + "aic_combined": "AIC combined", + "aiv_combined": "AIV combined", + "host_runtime": "Host runtime", + "aicpu_runtime": "Inner AICPU runtime", + } + rows = [] + for name in ("aicore_kernel", "aic_combined", "aiv_combined", "host_runtime", "aicpu_runtime"): + artifact = artifacts[name] + rows.append( + "" + f"{labels[name]}" + f"{_format_integer(artifact['size_bytes'])}" + f"{artifact['sha256']}" + f"{_format_integer(artifact['text']['size_bytes'])}" + f"{artifact['text']['sha256']}" + f"{html.escape(artifact['path'])}" + "" + ) + definitions = "
".join(f"{html.escape(value)}" for value in build["compile_definitions"]) + cache_key = " / ".join(html.escape(value) for value in build["profiled_cache_key"]) + return f""" +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
{provenance_sha256}
+
Profile / extra cache
{html.escape(build["profile"])} / + {build["aicore_extra_cache_key"]}
+
TensorMap mode
{html.escape(build["tensormap_mode"])}
+
Profiled cache key
{cache_key}
+
Source state
{build["source_state"]}
+
Source-state stamp
{html.escape(build["source_state_path"])}
+
Compile definitions
{definitions}
+
+
+ + + {"".join(rows)} +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
+
+ """ + + +def _document( + capture: SubmitPmuCapture, + miss_penalty_ns: float, + provenance: Mapping[str, Any] | None = None, + provenance_sha256: str | None = None, + recording_cost_reference: Mapping[str, Any] | None = None, +) -> str: + configuration = capture.data["configuration"] + frequencies = configuration["pmu_cycles_per_ns"] + window = capture.data["window"] + cards = "".join( + _group_card(name, capture.summary[name], float(frequencies[name]), miss_penalty_ns) for name in GROUP_NAMES + ) + charts = "".join( + ( + _chart_svg(capture.records, "scalar_submit_elapsed_ticks", "SYS gate boundary-diagnostic ticks/core"), + _chart_svg(capture.records, "total_cycles", "PMU total cycles/core"), + _chart_svg(capture.records, "scalar_busy", "Scalar busy cycles/core"), + _chart_svg(capture.records, "icache_requests", "Primary I-cache requests/core"), + _chart_svg(capture.records, "icache_misses", "Primary I-cache misses/core"), + ) + ) + rows = _per_core_rows(capture.records, miss_penalty_ns) + phase_overview = _phase_overview(capture, recording_cost_reference) + provenance_overview = _provenance_overview(provenance, provenance_sha256) + shadow_note = ( + "phase 模式的 I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和，" + "只用于标记全窗 capture gap；scalar shadow 同样用于核验 phase 读清重建，并显示其相对 " + "CNT2 primary 的 capture loss，不作为第二份性能数据展示。" + if capture.phase_summary is not None + else "none 模式不重复发布 shadow 原值；逐核 primary/shadow 同值由必选 status 闭环。" + ) + source_name = html.escape(capture.input_path.name) + return f""" + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 {source_name} · schema {SCHEMA_NAME} · 32 AIC + 64 AIV

+ {phase_overview} + {provenance_overview} +
+
原始全局 Submit 时间范围（墙钟） + {_format_number(window["global_submit_span_us"])} µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + {configuration["expected_submits_per_core"]}96 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-256{capture.raw_sha256}
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + {miss_penalty_ns:.3f} ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×{miss_penalty_ns:g} ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
{cards}
+

逐物理核分布

+

AICAIV

+
{charts}
+

逐核原始主计数

+

报告展示 primary request/miss；{shadow_note}

+
+ + + + + + + {rows} +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate{miss_penalty_ns:g}ns量尺 µs
+
+ +""" + + +def _clean_html_document(document: str) -> str: + """Normalize generated HTML so published artifacts never retain indentation-only lines.""" + + return "\n".join(line.rstrip() for line in document.splitlines()) + "\n" + + +def render_report( + input_path: Path | str, + *, + miss_penalty_ns: float = 90.0, + calibration_input_path: Path | str | None = None, +) -> str: + """Return a self-contained HTML report after validating and recomputing the raw capture.""" + + penalty = _number(miss_penalty_ns, "miss_penalty_ns", positive=True) + capture = load_capture(input_path) + provenance_path = capture.input_path.with_name(DEFAULT_PROVENANCE_NAME) + provenance = None + provenance_sha256 = None + if provenance_path.is_file(): + provenance, provenance_sha256 = load_provenance(provenance_path, capture) + + calibration_capture = None + recording_cost_reference = None + if calibration_input_path is not None: + calibration_capture = load_capture(calibration_input_path) + recording_cost_reference = build_phase_recording_cost_reference(capture, calibration_capture) + calibration_provenance_path = calibration_capture.input_path.with_name(DEFAULT_PROVENANCE_NAME) + if provenance_path.is_file() != calibration_provenance_path.is_file(): + _fail( + "recording-cost target and empty-bracket must either both provide provenance sidecars or both omit them" + ) + if provenance is None: + recording_cost_reference["provenance_binding"] = { + "verified": False, + "reason": "both raw inputs omit provenance sidecars", + } + else: + calibration_provenance, calibration_provenance_sha256 = load_provenance( + calibration_provenance_path, + calibration_capture, + ) + recording_cost_reference["provenance_binding"] = { + **_validate_recording_reference_provenance(provenance, calibration_provenance), + "target_provenance_sha256": provenance_sha256, + "calibration_provenance_sha256": calibration_provenance_sha256, + } + document = _document( + capture, + penalty, + provenance, + provenance_sha256, + recording_cost_reference, + ) + _assert_capture_raw_unchanged(capture) + if calibration_capture is not None: + _assert_capture_raw_unchanged(calibration_capture) + return _clean_html_document(document) + + +def _atomic_write_text(output_file: Path, document: str) -> None: + output_file.parent.mkdir(parents=True, exist_ok=True) + temporary = output_file.with_name(f"{output_file.name}.tmp") + temporary.unlink(missing_ok=True) + try: + with temporary.open("x", encoding="utf-8") as stream: + stream.write(document) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, output_file) + finally: + temporary.unlink(missing_ok=True) + + +def _stage_text(output_file: Path, document: str, *, purpose: str) -> Path: + """Durably stage text beside its final path without making it official.""" + + output_file.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{output_file.name}.{purpose}.", + suffix=".tmp", + dir=output_file.parent, + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(document) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + temporary.unlink(missing_ok=True) + raise + return temporary + + +def _restore_file_snapshot(output_file: Path, previous: bytes | None) -> None: + """Restore exactly the file state observed before a paired publication.""" + + if previous is None: + output_file.unlink(missing_ok=True) + return + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{output_file.name}.rollback.", + suffix=".tmp", + dir=output_file.parent, + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(previous) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, output_file) + finally: + temporary.unlink(missing_ok=True) + + +def _publish_provenance_report_pair( + *, + capture: SubmitPmuCapture, + raw_snapshot: bytes, + provenance_file: Path, + provenance_document: str, + output_file: Path, + report_document: str, +) -> None: + """Publish the sidecar/report pair or restore the exact previous pair.""" + + raw_parent = capture.input_path.resolve().parent + if provenance_file.resolve().parent != raw_parent or output_file.resolve().parent != raw_parent: + _fail("Submit-PMU raw, provenance and report must share one output directory") + if provenance_file == output_file: + _fail("Submit-PMU provenance and report paths must be distinct") + + previous = { + provenance_file: provenance_file.read_bytes() if provenance_file.is_file() else None, + output_file: output_file.read_bytes() if output_file.is_file() else None, + } + staged: list[Path] = [] + published = False + try: + staged_provenance = _stage_text(provenance_file, provenance_document, purpose="pending") + staged.append(staged_provenance) + staged_report = _stage_text(output_file, report_document, purpose="pending") + staged.append(staged_report) + if _assert_capture_raw_unchanged(capture) != raw_snapshot: + _fail("Submit-PMU raw changed while preparing provenance/report artifacts") + + os.replace(staged_provenance, provenance_file) + published = True + os.replace(staged_report, output_file) + if _assert_capture_raw_unchanged(capture) != raw_snapshot: + _fail("Submit-PMU raw changed while publishing provenance/report artifacts") + except BaseException as original_error: + rollback_errors: list[str] = [] + if published: + for final_path, old_bytes in previous.items(): + try: + _restore_file_snapshot(final_path, old_bytes) + except BaseException as rollback_error: # pragma: no cover - catastrophic filesystem failure + rollback_errors.append(f"{final_path}: {rollback_error}") + if rollback_errors: + raise RuntimeError( + "Submit-PMU paired publication failed and rollback was incomplete: " + "; ".join(rollback_errors) + ) from original_error + raise + finally: + for temporary in staged: + temporary.unlink(missing_ok=True) + + +def write_report( + input_path: Path | str, + output_path: Path | str | None = None, + *, + miss_penalty_ns: float = 90.0, + calibration_input_path: Path | str | None = None, +) -> Path: + """Validate first, then atomically publish the fixed-name HTML artifact.""" + + input_file = Path(input_path) + output_file = Path(output_path) if output_path is not None else input_file.with_name(DEFAULT_OUTPUT_NAME) + if output_file.name != DEFAULT_OUTPUT_NAME: + _fail(f"Submit-PMU output filename must be {DEFAULT_OUTPUT_NAME!r}") + document = render_report( + input_file, + miss_penalty_ns=miss_penalty_ns, + calibration_input_path=calibration_input_path, + ) + _atomic_write_text(output_file, document) + return output_file + + +def write_report_with_provenance( + input_path: Path | str, + identity: SubmitPmuBuildIdentity, + output_path: Path | str | None = None, + provenance_path: Path | str | None = None, + *, + miss_penalty_ns: float = 90.0, +) -> tuple[Path, Path]: + """Publish provenance and HTML while preserving the C++ producer's raw bytes.""" + + penalty = _number(miss_penalty_ns, "miss_penalty_ns", positive=True) + capture = load_capture(input_path) + output_file = Path(output_path) if output_path is not None else capture.input_path.with_name(DEFAULT_OUTPUT_NAME) + provenance_file = ( + Path(provenance_path) if provenance_path is not None else capture.input_path.with_name(DEFAULT_PROVENANCE_NAME) + ) + if output_file.name != DEFAULT_OUTPUT_NAME: + _fail(f"Submit-PMU output filename must be {DEFAULT_OUTPUT_NAME!r}") + if provenance_file.name != DEFAULT_PROVENANCE_NAME: + _fail(f"Submit-PMU provenance filename must be {DEFAULT_PROVENANCE_NAME!r}") + + raw_snapshot = _assert_capture_raw_unchanged(capture) + payload = _build_provenance_payload(capture, identity) + provenance_document = _json_document(payload) + validated_provenance = _validate_provenance_data( + _object(json.loads(provenance_document), "provenance"), + capture, + ) + provenance_sha256 = hashlib.sha256(provenance_document.encode("utf-8")).hexdigest() + document = _clean_html_document(_document(capture, penalty, validated_provenance, provenance_sha256)) + _publish_provenance_report_pair( + capture=capture, + raw_snapshot=raw_snapshot, + provenance_file=provenance_file, + provenance_document=provenance_document, + output_file=output_file, + report_document=document, + ) + return output_file, provenance_file + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", nargs="?", type=Path, default=Path(DEFAULT_INPUT_NAME), help=DEFAULT_INPUT_NAME) + parser.add_argument( + "-o", + "--output", + type=Path, + help=f"output path; basename must be {DEFAULT_OUTPUT_NAME} (default: beside input)", + ) + parser.add_argument( + "--miss-penalty-ns", + type=float, + default=90.0, + help="intuitive miss-cost scale only; it is not wall-clock loss (default: 90)", + ) + parser.add_argument( + "--calibration-input", + type=Path, + help=( + f"{EMPTY_BRACKET_CAPTURE_MODE} {DEFAULT_INPUT_NAME}; estimates only phase-local " + "recording work and leaves the raw whole denominator unchanged" + ), + ) + args = parser.parse_args(argv) + try: + output = write_report( + args.input, + args.output, + miss_penalty_ns=args.miss_penalty_ns, + calibration_input_path=args.calibration_input, + ) + except (OSError, ValueError) as error: + parser.exit(2, f"error: {error}\n") + print(output) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/simpler_setup/tools/fdwic_submit_span_overview.py b/simpler_setup/tools/fdwic_submit_span_overview.py new file mode 100644 index 0000000000..17c3f7278b --- /dev/null +++ b/simpler_setup/tools/fdwic_submit_span_overview.py @@ -0,0 +1,2513 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the LICENSE file. +# ----------------------------------------------------------------------------------------------------------- +"""Build one provenance-aware FDWIC Submit span overview from independent evidence ELFs.""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import json +import math +import os +import sys +import tempfile +from collections import Counter +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +try: + from .fdwic_submit_pmu_report import ( + ALLOC_COMPLETE_CAPTURE_MODE, + ARG_BUILD_CAPTURE_MODE, + CLAIM_CAPTURE_MODE, + DEFAULT_INPUT_NAME, + DEFAULT_PROVENANCE_NAME, + EFDRAIN_CONTROL_CAPTURE_MODE, + EMPTY_BRACKET_CAPTURE_MODE, + FANIN_CAPTURE_MODE, + LOSER_REPLAY_CAPTURE_MODE, + MATERIALIZE_CAPTURE_MODE, + NONE_CAPTURE_MODE, + PREPARE_MAP_CAPTURE_MODE, + REGISTER_CAPTURE_MODE, + SUBMIT_TRANSITION_CAPTURE_MODE, + WINNER_BUILD_CAPTURE_MODE, + SubmitPmuCapture, + build_phase_recording_cost_reference, + load_capture, + load_provenance, + ) + from .fdwic_swimlane_exclusive_analyzer import CORE_METRICS, REPORT_SCHEMA_VERSION, analyze_capture +except ImportError: + from fdwic_submit_pmu_report import ( # type: ignore[no-redef] + ALLOC_COMPLETE_CAPTURE_MODE, + ARG_BUILD_CAPTURE_MODE, + CLAIM_CAPTURE_MODE, + DEFAULT_INPUT_NAME, + DEFAULT_PROVENANCE_NAME, + EFDRAIN_CONTROL_CAPTURE_MODE, + EMPTY_BRACKET_CAPTURE_MODE, + FANIN_CAPTURE_MODE, + LOSER_REPLAY_CAPTURE_MODE, + MATERIALIZE_CAPTURE_MODE, + NONE_CAPTURE_MODE, + PREPARE_MAP_CAPTURE_MODE, + REGISTER_CAPTURE_MODE, + SUBMIT_TRANSITION_CAPTURE_MODE, + WINNER_BUILD_CAPTURE_MODE, + SubmitPmuCapture, + build_phase_recording_cost_reference, + load_capture, + load_provenance, + ) + from fdwic_swimlane_exclusive_analyzer import ( # type: ignore[no-redef] + CORE_METRICS, + REPORT_SCHEMA_VERSION, + analyze_capture, + ) + + +OVERVIEW_SCHEMA = "fdwic-submit-span-overview-v5" +DEFAULT_JSON_NAME = "fdwic_submit_span_overview.json" +DEFAULT_HTML_NAME = "fdwic_submit_span_overview.html" +OUTPUT_LOCK_NAME = ".fdwic_submit_span_overview.lock" + +PMU_MODE_ORDER = ( + NONE_CAPTURE_MODE, + EFDRAIN_CONTROL_CAPTURE_MODE, + CLAIM_CAPTURE_MODE, + ARG_BUILD_CAPTURE_MODE, + MATERIALIZE_CAPTURE_MODE, + PREPARE_MAP_CAPTURE_MODE, + FANIN_CAPTURE_MODE, + REGISTER_CAPTURE_MODE, + WINNER_BUILD_CAPTURE_MODE, + ALLOC_COMPLETE_CAPTURE_MODE, + LOSER_REPLAY_CAPTURE_MODE, + SUBMIT_TRANSITION_CAPTURE_MODE, + EMPTY_BRACKET_CAPTURE_MODE, +) +EXPECTED_PMU_MODES = frozenset(PMU_MODE_ORDER) + +PHASE_PRESENTATION = { + EFDRAIN_CONTROL_CAPTURE_MODE: ("EfDrain Scalar control", "EfDrain", "control-only"), + CLAIM_CAPTURE_MODE: ("Claim", "Claim", "same-business-boundary"), + ARG_BUILD_CAPTURE_MODE: ("ArgBuild", "Claim→Materialize internal residual", "residual-boundary"), + MATERIALIZE_CAPTURE_MODE: ("Materialize", "Materialize", "same-business-boundary"), + PREPARE_MAP_CAPTURE_MODE: ("PrepareMap", "PrepareMap", "same-business-boundary"), + FANIN_CAPTURE_MODE: ("Fanin", "Fanin", "same-business-boundary"), + REGISTER_CAPTURE_MODE: ("Register", "Register", "same-business-boundary"), + WINNER_BUILD_CAPTURE_MODE: ("WinnerBuild Scalar control", "WinnerBuild", "control-only"), + ALLOC_COMPLETE_CAPTURE_MODE: ("AllocComplete Scalar control", "AllocComplete", "control-only"), + LOSER_REPLAY_CAPTURE_MODE: ("LoserReplay", "LoserReplay", "same-business-boundary"), + SUBMIT_TRANSITION_CAPTURE_MODE: ( + "SubmitTransition", + "BetweenSubmitResidual", + "adjacent-submit-boundary", + ), + EMPTY_BRACKET_CAPTURE_MODE: ("EmptyBracket", "—", "observer-calibration"), +} + +# 这里只连接泳道聚合行与语义明确的 PMU phase。ArgBuild 仅对应 +# Claim→Materialize 子段，不能冒充整个 SubmitInternalResidual，因此故意不映射。 +PARTITION_PMU_MODE_BY_METRIC = { + "between_submit_residual": SUBMIT_TRANSITION_CAPTURE_MODE, + "efdrain": EFDRAIN_CONTROL_CAPTURE_MODE, + "efdrain_control": EFDRAIN_CONTROL_CAPTURE_MODE, + "materialize": MATERIALIZE_CAPTURE_MODE, + "prepare_map": PREPARE_MAP_CAPTURE_MODE, + "claim": CLAIM_CAPTURE_MODE, + "fanin": FANIN_CAPTURE_MODE, + "register": REGISTER_CAPTURE_MODE, + "winner_build": WINNER_BUILD_CAPTURE_MODE, + "alloc_complete": ALLOC_COMPLETE_CAPTURE_MODE, + "loser_replay": LOSER_REPLAY_CAPTURE_MODE, +} + +# SubmitUnion 的均值合计除 9 个直接对应表格行的 phase 外，还包含精确覆盖 +# Claim.end→Materialize.begin 子段的 ArgBuild。SubmitTransition 属于相邻 Submit 间隙， +# 严格位于 SubmitUnion 之外，因此不进入这里的分子。 +SUBMIT_UNION_PMU_MODES = ( + EFDRAIN_CONTROL_CAPTURE_MODE, + CLAIM_CAPTURE_MODE, + ARG_BUILD_CAPTURE_MODE, + MATERIALIZE_CAPTURE_MODE, + PREPARE_MAP_CAPTURE_MODE, + FANIN_CAPTURE_MODE, + REGISTER_CAPTURE_MODE, + WINNER_BUILD_CAPTURE_MODE, + ALLOC_COMPLETE_CAPTURE_MODE, + LOSER_REPLAY_CAPTURE_MODE, +) + +SYNTHETIC_PHASE_SUM_METRICS = ( + ( + "pmu_total_cycles", + "PMU total", + "phase_total_cycles_observed", + "pmu_total_cycles", + "cycles", + ), + ( + "scalar_busy_cycles", + "Scalar busy", + "phase_scalar_busy_observed", + "scalar_busy_cycles", + "cycles", + ), + ( + "non_scalar_busy_cycles", + "非 Scalar-busy 残余", + "phase_non_scalar_busy_cycles", + "non_scalar_busy_cycles", + "cycles", + ), + ( + "icache_requests", + "I-cache request", + "phase_icache_requests_observed", + "primary_icache_requests", + "events", + ), + ( + "icache_misses", + "I-cache miss", + "phase_icache_misses_observed", + "primary_icache_misses", + "events", + ), +) + +SUBMIT_PHASES = ( + ("EfDrain", "efdrain"), + ("Materialize", "materialize"), + ("PrepareMap", "prepare_map"), + ("Claim", "claim"), + ("Fanin", "fanin"), + ("Register", "register"), + ("WinnerBuild", "winner_build"), + ("AllocComplete", "alloc_complete"), + ("LoserReplay", "loser_replay"), +) + + +def _fail(message: str) -> None: + raise ValueError(message) + + +def _sha256_file(path: Path) -> tuple[int, str]: + digest = hashlib.sha256() + size = 0 + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + size += len(chunk) + digest.update(chunk) + return size, digest.hexdigest() + + +def _require_int(value: Any, path: str, *, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + _fail(f"{path} must be an integer >= {minimum}") + return value + + +def _require_number(value: Any, path: str, *, minimum: float = 0.0) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < minimum: + _fail(f"{path} must be a finite number >= {minimum}") + return float(value) + + +def _require_mapping(value: Any, path: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + _fail(f"{path} must be an object") + return value + + +def _validate_swimlane_analysis(data: Mapping[str, Any]) -> tuple[int, int]: # noqa: PLR0912 + """Reject a partial or non-closing schema-v4 analysis before summarizing it.""" + + if data.get("schema_version") != REPORT_SCHEMA_VERSION: + _fail(f"swimlane analysis schema_version must equal {REPORT_SCHEMA_VERSION}") + validation = _require_mapping(data.get("validation"), "swimlane.validation") + required_validation = { + "status": "PASS", + "dropped_records": 0, + "physical_topology_complete": True, + "task_stream_contiguous_and_equal_per_core": True, + "orchestration_parent_exactly_one_per_core": True, + "final_drain_parent_exactly_one_per_core": True, + "parent_boundaries_adjacent": True, + "exclusive_children_non_overlapping": True, + "all_integer_cycle_closures_exact": True, + } + for field, expected in required_validation.items(): + if validation.get(field) != expected: + _fail(f"swimlane.validation.{field} must equal {expected!r}") + + capture = _require_mapping(data.get("capture"), "swimlane.capture") + if capture.get("trace_schema_version") != 4: + _fail("swimlane.capture.trace_schema_version must equal 4") + frequency_hz = _require_int(capture.get("clock_freq_hz"), "swimlane.capture.clock_freq_hz", minimum=1) + core_count = _require_int(capture.get("core_count"), "swimlane.capture.core_count", minimum=1) + if core_count != 96: + _fail("swimlane capture must contain exactly 96 logical cores") + task_count = _require_int(capture.get("task_count_per_core"), "swimlane.capture.task_count_per_core", minimum=1) + + aggregate = _require_mapping(data.get("aggregate_core_work"), "swimlane.aggregate_core_work") + metrics = _require_mapping(aggregate.get("metrics_cycles"), "swimlane.aggregate_core_work.metrics_cycles") + aggregate_values = { + metric: _require_int(metrics.get(metric), f"swimlane.aggregate_core_work.metrics_cycles.{metric}") + for metric in CORE_METRICS + } + closures = _require_mapping(aggregate.get("closure"), "swimlane.aggregate_core_work.closure") + required_closures = { + "submit_partition", + "submit_envelope", + "efdrain_partition", + "orchestration_replay", + "final_drain", + "worker_completion", + } + if set(closures) != required_closures: + _fail("swimlane aggregate closure set is incomplete or unexpected") + for name, closure_value in closures.items(): + closure = _require_mapping(closure_value, f"swimlane.aggregate_core_work.closure.{name}") + if closure.get("exact") is not True: + _fail(f"swimlane aggregate closure {name!r} is not exact") + + per_core_value = data.get("per_core") + if not isinstance(per_core_value, list) or len(per_core_value) != core_count: + _fail("swimlane.per_core must contain exactly 96 records") + roles: Counter[str] = Counter() + core_ids: set[int] = set() + recomputed = {metric: 0 for metric in CORE_METRICS} + for index, core_value in enumerate(per_core_value): + core = _require_mapping(core_value, f"swimlane.per_core[{index}]") + core_id = _require_int(core.get("core_id"), f"swimlane.per_core[{index}].core_id") + core_ids.add(core_id) + role = core.get("role") + if role not in {"aic", "aiv"}: + _fail(f"swimlane.per_core[{index}].role must be 'aic' or 'aiv'") + roles[str(role)] += 1 + if core.get("submit_count") != task_count: + _fail(f"swimlane.per_core[{index}].submit_count does not match task_count_per_core") + core_metrics = _require_mapping(core.get("metrics_cycles"), f"swimlane.per_core[{index}].metrics_cycles") + for metric in CORE_METRICS: + recomputed[metric] += _require_int( + core_metrics.get(metric), f"swimlane.per_core[{index}].metrics_cycles.{metric}" + ) + if core_ids != set(range(core_count)): + _fail("swimlane logical core IDs must be contiguous 0..95") + if roles != Counter({"aic": 32, "aiv": 64}): + _fail("swimlane topology must contain exactly 32 AIC and 64 AIV cores") + if recomputed != aggregate_values: + _fail("swimlane aggregate metrics do not equal the sum of per-core metrics") + + residual = _require_mapping(data.get("residual_breakdown"), "swimlane.residual_breakdown") + internal = _require_mapping(residual.get("submit_internal_residual"), "swimlane residual internal") + tail = _require_mapping(residual.get("submit_tail_residual"), "swimlane residual tail") + if ( + _require_int(internal.get("total_cycles"), "swimlane residual internal total") + + _require_int(tail.get("total_cycles"), "swimlane residual tail total") + != aggregate_values["submit_residual"] + ): + _fail("swimlane internal + tail residual does not close to submit_residual") + + overlays = _require_mapping(data.get("overlays"), "swimlane.overlays") + for name, overlay_value in overlays.items(): + overlay = _require_mapping(overlay_value, f"swimlane.overlays.{name}") + if overlay.get("included_in_additive_totals") is not False: + _fail(f"swimlane overlay {name!r} must remain non-additive") + + makespan = _require_mapping(data.get("global_submit_makespan"), "swimlane.global_submit_makespan") + makespan_cycles = _require_int(makespan.get("duration_cycles"), "swimlane global Submit duration", minimum=1) + makespan_us = _require_number(makespan.get("duration_us"), "swimlane global Submit duration_us") + expected_us = makespan_cycles * 1_000_000 / frequency_hz + if not math.isclose(makespan_us, expected_us, rel_tol=1e-12, abs_tol=1e-9): + _fail("swimlane global Submit duration_us does not match its SYS counter frequency") + return frequency_hz, task_count + + +def _ticks_to_us(ticks: int | float, frequency_hz: int) -> float: + return float(ticks) * 1_000_000 / frequency_hz + + +def _metric_range(data: Mapping[str, Any]) -> dict[str, int]: + return {"min": int(data["min"]), "max": int(data["max"])} + + +def _role_metric_range( + per_core: Sequence[Mapping[str, Any]], metric: str, role: str, frequency_hz: int +) -> dict[str, int | float]: + values = [int(core["metrics_cycles"][metric]) for core in per_core if core["role"] == role] + return { + "min_ticks": min(values), + "max_ticks": max(values), + "min_us": _ticks_to_us(min(values), frequency_hz), + "max_us": _ticks_to_us(max(values), frequency_hz), + } + + +def _swimlane_metric_row( + label: str, + metric: str, + metrics: Mapping[str, int], + denominator: int, + per_core: Sequence[Mapping[str, Any]], + frequency_hz: int, +) -> dict[str, Any]: + ticks = int(metrics[metric]) + return { + "label": label, + "metric": metric, + "core_time_ticks": ticks, + "core_time_us": _ticks_to_us(ticks, frequency_hz), + "share": ticks / denominator if denominator else 0.0, + "aic_per_core": _role_metric_range(per_core, metric, "aic", frequency_hz), + "aiv_per_core": _role_metric_range(per_core, metric, "aiv", frequency_hz), + } + + +def _residual_row( + label: str, + metric: str, + ticks: int, + denominator: int, + frequency_hz: int, +) -> dict[str, Any]: + return { + "label": label, + "metric": metric, + "core_time_ticks": ticks, + "core_time_us": _ticks_to_us(ticks, frequency_hz), + "share": ticks / denominator if denominator else 0.0, + "aic_per_core": None, + "aiv_per_core": None, + } + + +def _partition(name: str, parent_metric: str, parent_ticks: int, rows: Sequence[dict[str, Any]]) -> dict[str, Any]: + children_ticks = sum(int(row["core_time_ticks"]) for row in rows) + if children_ticks != parent_ticks: + _fail(f"overview partition {name!r} does not close: parent={parent_ticks} children={children_ticks}") + return { + "name": name, + "parent_metric": parent_metric, + "parent_core_time_ticks": parent_ticks, + "children_core_time_ticks": children_ticks, + "exact": True, + "rows": list(rows), + } + + +def _summarize_swimlane( + analysis: Mapping[str, Any], raw_path: Path, raw_size: int, raw_sha256: str, frequency_hz: int +) -> dict[str, Any]: + aggregate = analysis["aggregate_core_work"]["metrics_cycles"] + metrics = {name: int(value) for name, value in aggregate.items()} + per_core = analysis["per_core"] + residual = analysis["residual_breakdown"] + + envelope_rows = [ + _swimlane_metric_row( + "SubmitUnion", "submit_union", metrics, metrics["submit_envelope"], per_core, frequency_hz + ), + _swimlane_metric_row( + "BetweenSubmitResidual", + "between_submit_residual", + metrics, + metrics["submit_envelope"], + per_core, + frequency_hz, + ), + ] + internal_ticks = int(residual["submit_internal_residual"]["total_cycles"]) + tail_ticks = int(residual["submit_tail_residual"]["total_cycles"]) + submit_rows = [ + *( + _swimlane_metric_row(label, metric, metrics, metrics["submit_union"], per_core, frequency_hz) + for label, metric in SUBMIT_PHASES + ), + _residual_row( + "SubmitInternalResidual", + "submit_internal_residual", + internal_ticks, + metrics["submit_union"], + frequency_hz, + ), + _residual_row( + "SubmitTailResidual", + "submit_tail_residual", + tail_ticks, + metrics["submit_union"], + frequency_hz, + ), + ] + efdrain_rows = [ + _swimlane_metric_row( + "EfDrainKernelUnion", + "efdrain_kernel_union", + metrics, + metrics["efdrain"], + per_core, + frequency_hz, + ), + _swimlane_metric_row("EfDrainControl", "efdrain_control", metrics, metrics["efdrain"], per_core, frequency_hz), + ] + orchestration_rows = [ + _swimlane_metric_row( + "OrchestrationSetup", + "orchestration_setup", + metrics, + metrics["orchestration_replay"], + per_core, + frequency_hz, + ), + _swimlane_metric_row( + "SubmitUnion", "submit_union", metrics, metrics["orchestration_replay"], per_core, frequency_hz + ), + _swimlane_metric_row( + "BetweenSubmitResidual", + "between_submit_residual", + metrics, + metrics["orchestration_replay"], + per_core, + frequency_hz, + ), + _swimlane_metric_row( + "OrchestrationTail", + "orchestration_tail", + metrics, + metrics["orchestration_replay"], + per_core, + frequency_hz, + ), + ] + final_drain_rows = [ + _swimlane_metric_row( + "FinalDrainKernelUnion", + "final_drain_kernel_union", + metrics, + metrics["final_drain"], + per_core, + frequency_hz, + ), + _swimlane_metric_row( + "FinalDrainResidual", + "final_drain_residual", + metrics, + metrics["final_drain"], + per_core, + frequency_hz, + ), + ] + worker_rows = [ + _swimlane_metric_row( + "OrchestrationReplay", + "orchestration_replay", + metrics, + metrics["worker_completion"], + per_core, + frequency_hz, + ), + _swimlane_metric_row( + "FinalDrain", "final_drain", metrics, metrics["worker_completion"], per_core, frequency_hz + ), + ] + + residual_segments = {} + for name in ("submit_internal_residual", "submit_tail_residual", "between_submit_residual"): + item = residual[name] + residual_segments[name] = { + "total_cycles": int(item["total_cycles"]), + "segments": [ + { + "boundary": str(segment["boundary"]), + "event_count": int(segment["event_count"]), + "cycles": int(segment["cycles"]), + "aic_cycles": int(segment["aic_cycles"]), + "aiv_cycles": int(segment["aiv_cycles"]), + } + for segment in item["segments"] + ], + } + + return { + "source": { + "raw_path": str(raw_path.resolve()), + "raw_size": raw_size, + "raw_sha256": raw_sha256, + "build_provenance_available": False, + "identity_limit": ( + "schema-v4 raw was strictly reanalyzed, but this capture has no build-provenance sidecar" + ), + }, + "core_count": len(per_core), + "sys_counter_frequency_hz": frequency_hz, + "sys_counter_semantics": "SYS counter ticks; not PMU cycles", + "global_submit_makespan": analysis["global_submit_makespan"], + "global_worker_completion_makespan": analysis["global_worker_completion_makespan"], + "submit_envelope_partition": _partition( + "SubmitEnvelope", "submit_envelope", metrics["submit_envelope"], envelope_rows + ), + "submit_union_partition": _partition("SubmitUnion", "submit_union", metrics["submit_union"], submit_rows), + "efdrain_partition": _partition("EfDrain", "efdrain", metrics["efdrain"], efdrain_rows), + "orchestration_partition": _partition( + "OrchestrationReplay", "orchestration_replay", metrics["orchestration_replay"], orchestration_rows + ), + "final_drain_partition": _partition("FinalDrain", "final_drain", metrics["final_drain"], final_drain_rows), + "worker_completion_partition": _partition( + "WorkerCompletion", "worker_completion", metrics["worker_completion"], worker_rows + ), + "submit_residual_per_core": { + "aic": _role_metric_range(per_core, "submit_residual", "aic", frequency_hz), + "aiv": _role_metric_range(per_core, "submit_residual", "aiv", frequency_hz), + }, + "residual_segments": residual_segments, + "kernel_containment": analysis["kernel_containment"], + "overlays": analysis["overlays"], + } + + +def _group_denominator(summary: Mapping[str, Any]) -> dict[str, Any]: + return { + "cores": int(summary["cores"]), + "scalar_submit_elapsed_ticks": { + "sum": int(summary["scalar_submit_elapsed_ticks"]["sum"]), + **_metric_range(summary["scalar_submit_elapsed_ticks"]), + }, + "pmu_total_cycles": { + "sum": int(summary["total_cycles"]["sum"]), + **_metric_range(summary["total_cycles"]), + }, + "scalar_busy_cycles": { + "sum": int(summary["scalar_busy"]["sum"]), + **_metric_range(summary["scalar_busy"]), + }, + "non_scalar_busy_cycles": { + "sum": int(summary["non_scalar_busy_cycles"]["sum"]), + **_metric_range(summary["non_scalar_busy_cycles"]), + }, + "primary_icache_requests": { + "sum": int(summary["icache_requests"]["sum"]), + **_metric_range(summary["icache_requests"]), + }, + "primary_icache_misses": { + "sum": int(summary["icache_misses"]["sum"]), + **_metric_range(summary["icache_misses"]), + }, + } + + +def _phase_group_summary(summary: Mapping[str, Any]) -> dict[str, Any]: + return { + "cores": int(summary["cores"]), + "phase_total_cycles_observed": { + "sum": int(summary["phase_total_cycles_observed"]["sum"]), + **_metric_range(summary["phase_total_cycles_observed"]), + }, + "phase_scalar_busy_observed": { + "sum": int(summary["phase_scalar_busy_observed"]["sum"]), + **_metric_range(summary["phase_scalar_busy_observed"]), + }, + "phase_non_scalar_busy_cycles": { + "sum": int(summary["phase_non_scalar_busy_cycles"]["sum"]), + **_metric_range(summary["phase_non_scalar_busy_cycles"]), + }, + "shadow_scalar_loss": { + "sum": int(summary["shadow_scalar_loss"]["sum"]), + **_metric_range(summary["shadow_scalar_loss"]), + }, + "phase_elapsed_ticks": { + "sum": int(summary["phase_elapsed_ticks"]["sum"]), + **_metric_range(summary["phase_elapsed_ticks"]), + }, + "phase_icache_requests_observed": { + "sum": int(summary["phase_icache_requests_observed"]["sum"]), + **_metric_range(summary["phase_icache_requests_observed"]), + }, + "phase_icache_misses_observed": { + "sum": int(summary["phase_icache_misses_observed"]["sum"]), + **_metric_range(summary["phase_icache_misses_observed"]), + }, + "phase_total_share_of_pmu_total": float(summary["phase_total_share_of_pmu_total"]), + "phase_scalar_share_of_whole_scalar": float(summary["phase_scalar_share_of_whole_scalar"]), + "phase_scalar_busy_share_of_phase_total": float(summary["phase_scalar_busy_share_of_phase_total"]), + "phase_request_observed_share_of_primary": float(summary["phase_request_observed_share_of_primary"]), + "phase_miss_observed_share_of_primary": float(summary["phase_miss_observed_share_of_primary"]), + "phase_business_calls": int(summary["phase_business_calls"]), + # 每个 end read 对应一组完整的 begin/end 记录边界。跨 linked Kernel + # 的 pause/resume 会额外产生一组，因此它比外层业务调用次数更适合 + # 缩放 empty-bracket 测得的记录代码开销。 + "phase_record_pairs": int(summary["phase_end_reads"]), + "phase_calls_per_core": _metric_range(summary["phase_calls_per_core"]), + "phase_zero_call_cores": int(summary["phase_zero_call_cores"]), + "phase_excluded_kernel_calls": int(summary.get("phase_excluded_kernel_calls", 0)), + } + + +def _summarize_pmu_capture( + capture: SubmitPmuCapture, + provenance: Mapping[str, Any], + provenance_sha256: str, + provenance_path: Path, +) -> dict[str, Any]: + mode = str(capture.data["capture"]["mode"]) + source = { + "raw_path": str(capture.input_path.resolve()), + "raw_size": capture.raw_size, + "raw_sha256": capture.raw_sha256, + "provenance_path": str(provenance_path.resolve()), + "provenance_sha256": provenance_sha256, + "html_report_path": str(capture.input_path.with_name("fdwic_submit_pmu_report.html").resolve()), + "git_head": str(provenance["build"]["git_head"]), + "source_fingerprint": str(provenance["build"]["source_fingerprint"]), + "aicore_kernel_sha256": str(provenance["artifacts"]["aicore_kernel"]["sha256"]), + "aicore_kernel_text_sha256": str(provenance["artifacts"]["aicore_kernel"]["text"]["sha256"]), + } + frequency_data = capture.data["configuration"]["pmu_cycles_per_ns"] + result = { + "capture_mode": mode, + "global_submit_span_us": float(capture.data["window"]["global_submit_span_us"]), + "expected_submits_per_core": int(capture.data["configuration"]["expected_submits_per_core"]), + "sys_counter_tick_ns": int(capture.data["configuration"]["sys_counter_tick_ns"]), + "pmu_cycles_per_ns": {name: float(frequency_data[name]) for name in ("all", "aic", "aiv")}, + "denominators": {name: _group_denominator(capture.summary[name]) for name in ("all", "aic", "aiv")}, + "measurement_scopes": { + "scalar_submit_elapsed_ticks": ( + "sum of gate-running SYS counter segments inside the first/last Submit closure, then subtract " + "result-used return-ready atomic waiting" + ), + "pmu_total_scalar_busy_and_primary_icache": ( + "per-core PMU gate nested inside the first/last Submit SYS counter boundary; linked Kernel causes " + "paired gate pauses, while atomic instruction events remain" + ), + "phase_pmu": ( + "running read-clear PMU total and scalar-busy observations; non-scalar-busy is derived per core " + "as phase total minus phase scalar-busy" + ), + "phase_sys_counter": "boundary diagnostic only; not the primary phase timing source", + "global_submit_span_us": "cross-core first/last Submit closure only", + }, + "source": source, + } + if capture.phase_summary is None: + result.update( + { + "kind": "whole-window", + "label": "PMU whole gate 与 SYS 边界诊断", + "boundary_reference": "SubmitEnvelope boundary only", + "mapping": "whole-window-boundary", + } + ) + return result + + label, boundary_reference, mapping = PHASE_PRESENTATION[mode] + phase = capture.data["configuration"]["phase"] + result.update( + { + "kind": "calibration" if mode == EMPTY_BRACKET_CAPTURE_MODE else "phase", + "label": label, + "boundary_reference": boundary_reference, + "mapping": mapping, + "phase": { + "id": int(phase["id"]), + "name": str(phase["name"]), + "boundary": str(phase["boundary"]), + "counter_semantics": str(phase["counter_semantics"]), + "time_semantics": str(phase["time_semantics"]), + "groups": {name: _phase_group_summary(capture.phase_summary[name]) for name in ("all", "aic", "aiv")}, + }, + } + ) + return result + + +def _synthetic_phase_sum_vs_none( + phase_profiles: Sequence[Mapping[str, Any]], + whole_window: Mapping[str, Any], +) -> dict[str, Any]: + """Estimate the phase recorder's contribution without presenting it as an exact correction.""" + + def metric_payload( + *, + label: str, + unit: str, + phase_field: str, + whole_field: str, + raw_sum: int, + none_sum: int, + cores: int, + empty_cost_per_record_pair: float, + recording_cost_estimate_sum: float, + ) -> dict[str, Any]: + raw_mean = raw_sum / cores + none_mean = none_sum / cores + recording_mean = recording_cost_estimate_sum / cores + reference_sum = raw_sum - recording_cost_estimate_sum + reference_mean = reference_sum / cores + return { + "label": label, + "unit": unit, + "raw_observed_sum": raw_sum, + "raw_observed_mean": raw_mean, + "submit_none_sum": none_sum, + "submit_none_mean": none_mean, + "raw_observed_ratio_to_submit_none": raw_mean / none_mean if none_mean else None, + "empty_cost_per_record_pair": empty_cost_per_record_pair, + "recording_cost_estimate_sum": recording_cost_estimate_sum, + "recording_cost_estimate_mean": recording_mean, + "recording_cost_estimate_share_of_raw": recording_mean / raw_mean if raw_mean else None, + "recording_cost_estimate_ratio_to_submit_none": ( + recording_mean / none_mean if none_mean else None + ), + "after_recording_cost_reference_sum": reference_sum, + "after_recording_cost_reference_mean": reference_mean, + "after_recording_cost_reference_ratio_to_submit_none": ( + reference_mean / none_mean if none_mean else None + ), + "phase_field": phase_field, + "submit_none_field": whole_field, + } + + groups: dict[str, Any] = {} + for group_name in ("all", "aic", "aiv"): + denominator = whole_window["denominators"][group_name] + cores = int(denominator["cores"]) + phase_business_calls = sum( + int(profile["phase"]["groups"][group_name]["phase_business_calls"]) for profile in phase_profiles + ) + phase_record_pairs = sum( + int(profile["phase"]["groups"][group_name]["phase_record_pairs"]) for profile in phase_profiles + ) + calibration_pair_counts = { + int(profile["recording_cost_reference"]["groups"][group_name]["empty_record_pairs"]) + for profile in phase_profiles + } + if len(calibration_pair_counts) != 1: + _fail(f"phase profiles do not share one {group_name} empty-bracket calibration pair count") + calibration_pairs = next(iter(calibration_pair_counts)) + group_metrics = {} + for metric_name, label, phase_field, whole_field, unit in SYNTHETIC_PHASE_SUM_METRICS: + profile_metrics = [ + profile["recording_cost_reference"]["groups"][group_name]["metrics"][metric_name] + for profile in phase_profiles + ] + raw_sum = sum(int(metric["raw_phase_observed_sum"]) for metric in profile_metrics) + recording_sum = sum(float(metric["recording_cost_estimate_sum"]) for metric in profile_metrics) + direct_reference_sum = sum( + float(metric["after_recording_cost_reference_sum"]) for metric in profile_metrics + ) + none_sum = int(denominator[whole_field]["sum"]) + empty_cost_per_pair = recording_sum / phase_record_pairs if phase_record_pairs else 0.0 + metric = metric_payload( + label=label, + unit=unit, + phase_field=phase_field, + whole_field=whole_field, + raw_sum=raw_sum, + none_sum=none_sum, + cores=cores, + empty_cost_per_record_pair=empty_cost_per_pair, + recording_cost_estimate_sum=recording_sum, + ) + if not math.isclose( + float(metric["after_recording_cost_reference_sum"]), + direct_reference_sum, + rel_tol=1e-12, + abs_tol=1e-6, + ): + _fail(f"{metric_name} aggregate reference does not equal the phase-profile references") + group_metrics[metric_name] = metric + groups[group_name] = { + "cores": cores, + "phase_business_calls": phase_business_calls, + "phase_business_calls_per_core": phase_business_calls / cores, + "phase_record_pairs": phase_record_pairs, + "phase_record_pairs_per_core": phase_record_pairs / cores, + "empty_calibration_record_pairs": calibration_pairs, + "metrics": group_metrics, + } + + if int(groups["all"]["cores"]) != int(groups["aic"]["cores"]) + int(groups["aiv"]["cores"]): + _fail("ALL core count does not equal AIC plus AIV core counts") + if int(groups["all"]["phase_business_calls"]) != int(groups["aic"]["phase_business_calls"]) + int( + groups["aiv"]["phase_business_calls"] + ): + _fail("ALL business calls do not equal AIC plus AIV") + if int(groups["all"]["phase_record_pairs"]) != int(groups["aic"]["phase_record_pairs"]) + int( + groups["aiv"]["phase_record_pairs"] + ): + _fail("ALL record pairs do not equal AIC plus AIV") + if int(groups["all"]["empty_calibration_record_pairs"]) != int( + groups["aic"]["empty_calibration_record_pairs"] + ) + int(groups["aiv"]["empty_calibration_record_pairs"]): + _fail("ALL empty-bracket record pairs do not equal AIC plus AIV") + groups["all"]["all_values_are_aic_aiv_weighted"] = True + groups["all"]["empty_cost_per_record_pair_is_phase_role_weighted_effective_rate"] = True + for metric_name, *_ in SYNTHETIC_PHASE_SUM_METRICS: + all_metric = groups["all"]["metrics"][metric_name] + for field in ( + "raw_observed_sum", + "submit_none_sum", + "recording_cost_estimate_sum", + "after_recording_cost_reference_sum", + ): + role_sum = sum(float(groups[role]["metrics"][metric_name][field]) for role in ("aic", "aiv")) + if not math.isclose(float(all_metric[field]), role_sum, rel_tol=1e-12, abs_tol=1e-6): + _fail(f"{metric_name} {field} ALL does not equal AIC plus AIV") + + return { + "kind": "cross-elf-recording-cost-decomposition", + "phase_profile_count": len(phase_profiles), + "included_profiles": [str(profile["capture_mode"]) for profile in phase_profiles], + "excluded_profiles": [NONE_CAPTURE_MODE, EMPTY_BRACKET_CAPTURE_MODE], + "empty_bracket_excluded_from_raw_phase_sum": True, + "empty_bracket_used_as_recording_cost_calibration": True, + "formal_partition_closure": False, + "exact_recording_cost_correction": False, + "pmu_cycles_per_ns": { + name: float(whole_window["pmu_cycles_per_ns"][name]) for name in ("all", "aic", "aiv") + }, + "semantics": ( + "raw observed sums independent phase ELFs; the recording-cost estimate multiplies each role's " + "empty-bracket cost per begin/end record pair by that role's selected phase record-pair count; " + "the post-subtraction value is only a reference because call-site context, code layout, " + "coverage gaps/overlap and run variance remain mixed" + ), + "groups": groups, + } + + +def _coverage_matrix(swimlane: Mapping[str, Any]) -> list[dict[str, str]]: + rows = [ + { + "swimlane_region": "WorkerCompletion", + "pmu_profile": "—", + "coverage": "not-covered", + "note": "含 OrchestrationReplay 与 FinalDrain；没有独立 PMU 父窗", + }, + { + "swimlane_region": "OrchestrationReplay", + "pmu_profile": "—", + "coverage": "not-covered", + "note": "PMU gate 嵌在首末 Submit SYS closure 内，不等同该父区间", + }, + { + "swimlane_region": "OrchestrationSetup", + "pmu_profile": "—", + "coverage": "not-covered", + "note": "当前没有独立 PMU phase", + }, + { + "swimlane_region": "SubmitEnvelope", + "pmu_profile": NONE_CAPTURE_MODE, + "coverage": "boundary-only", + "note": "PMU Scalar 分母另行排除 linked Kernel 与 return-ready atomic 时间", + }, + { + "swimlane_region": "SubmitUnion", + "pmu_profile": "—", + "coverage": "independent-components-only", + "note": "各 phase 是独立 ELF，不允许求和成 SubmitUnion", + }, + { + "swimlane_region": "EfDrain", + "pmu_profile": EFDRAIN_CONTROL_CAPTURE_MODE, + "coverage": "control-only", + "note": "泳道父 span 含 Kernel；PMU 只看排除 Kernel 后的 Scalar control", + }, + { + "swimlane_region": "EfDrainKernelUnion", + "pmu_profile": "—", + "coverage": "intentionally-excluded", + "note": "目标是纯 Scalar 归因，linked Kernel 从 PMU gate 排除", + }, + { + "swimlane_region": "EfDrainControl", + "pmu_profile": EFDRAIN_CONTROL_CAPTURE_MODE, + "coverage": "control-only", + "note": "与泳道 EfDrain 扣除 Kernel union 后的控制语义对应", + }, + { + "swimlane_region": "Materialize", + "pmu_profile": MATERIALIZE_CAPTURE_MODE, + "coverage": "same-business-boundary", + "note": "独立 ELF，只可用自己的分母", + }, + { + "swimlane_region": "PrepareMap", + "pmu_profile": PREPARE_MAP_CAPTURE_MODE, + "coverage": "same-business-boundary", + "note": "独立 ELF，只可用自己的分母", + }, + { + "swimlane_region": "Claim", + "pmu_profile": CLAIM_CAPTURE_MODE, + "coverage": "same-business-boundary", + "note": "独立 ELF，只可用自己的分母", + }, + { + "swimlane_region": "Fanin", + "pmu_profile": FANIN_CAPTURE_MODE, + "coverage": "same-business-boundary", + "note": "动态 winner 调用区间", + }, + { + "swimlane_region": "Register", + "pmu_profile": REGISTER_CAPTURE_MODE, + "coverage": "same-business-boundary", + "note": "独立 ELF，只可用自己的分母", + }, + { + "swimlane_region": "WinnerBuild", + "pmu_profile": WINNER_BUILD_CAPTURE_MODE, + "coverage": "control-only", + "note": "泳道父 span 可能含 Kernel；PMU 排除 linked Kernel", + }, + { + "swimlane_region": "AllocComplete", + "pmu_profile": ALLOC_COMPLETE_CAPTURE_MODE, + "coverage": "control-only", + "note": "泳道父 span 可能含 Kernel；PMU 排除 linked Kernel", + }, + { + "swimlane_region": "LoserReplay", + "pmu_profile": LOSER_REPLAY_CAPTURE_MODE, + "coverage": "same-business-boundary", + "note": "真实路径不执行 linked Kernel", + }, + { + "swimlane_region": "BetweenSubmitResidual", + "pmu_profile": SUBMIT_TRANSITION_CAPTURE_MODE, + "coverage": "adjacent-submit-boundary", + "note": "聚合每核相邻 Submit 的 N-1 个间隙", + }, + { + "swimlane_region": "OrchestrationTail", + "pmu_profile": "—", + "coverage": "not-covered", + "note": "当前没有独立 PMU phase", + }, + { + "swimlane_region": "FinalDrain", + "pmu_profile": "—", + "coverage": "outside-submit-pmu", + "note": "发生在末次 Submit 之后，不进入当前 PMU 窗", + }, + { + "swimlane_region": "FinalDrainKernelUnion", + "pmu_profile": "—", + "coverage": "outside-submit-pmu", + "note": "末次 Submit 后的 linked Kernel，不属于纯 Submit Scalar 分母", + }, + { + "swimlane_region": "FinalDrainResidual", + "pmu_profile": "—", + "coverage": "outside-submit-pmu", + "note": "末次 Submit 后的控制残余", + }, + { + "swimlane_region": "SubmitResidual", + "pmu_profile": ARG_BUILD_CAPTURE_MODE, + "coverage": "partial", + "note": "仅 Claim→Materialize internal residual 有同边界 PMU；tail 未覆盖", + }, + { + "swimlane_region": "SubmitInternalResidual", + "pmu_profile": ARG_BUILD_CAPTURE_MODE, + "coverage": "partial", + "note": "逐 segment 见下列动态覆盖项", + }, + { + "swimlane_region": "SubmitTailResidual", + "pmu_profile": "—", + "coverage": "not-covered", + "note": "逐 segment 列出但当前没有独立 PMU phase", + }, + ] + internal_segments = swimlane["residual_segments"]["submit_internal_residual"]["segments"] + for segment in internal_segments: + is_arg_build = segment["boundary"] == "Claim->Materialize" + rows.append( + { + "swimlane_region": f"SubmitInternalResidual/{segment['boundary']}", + "pmu_profile": ARG_BUILD_CAPTURE_MODE if is_arg_build else "—", + "coverage": "residual-boundary" if is_arg_build else "not-covered", + "note": "ArgBuild 同边界" if is_arg_build else "当前没有独立 PMU phase", + } + ) + for segment in swimlane["residual_segments"]["submit_tail_residual"]["segments"]: + rows.append( + { + "swimlane_region": f"SubmitTailResidual/{segment['boundary']}", + "pmu_profile": "—", + "coverage": "not-covered", + "note": "当前没有独立 PMU phase", + } + ) + for overlay_name in swimlane["overlays"]: + is_atomic = overlay_name == "Atomic" + rows.append( + { + "swimlane_region": f"{overlay_name} overlay", + "pmu_profile": "—", + "coverage": "swimlane-overlay", + "note": ( + "PMU 时间只扣 result-used return-ready 等待；counter 仍含 atomic 指令事件" + if is_atomic + else "非加和泳道 overlay；当前没有独立 PMU phase" + ), + } + ) + rows.append( + { + "swimlane_region": "观察器校准", + "pmu_profile": EMPTY_BRACKET_CAPTURE_MODE, + "coverage": "observer-calibration", + "note": "不是业务 span，不进入任何业务分布", + } + ) + return rows + + +def _assert_pmu_sources_unchanged(sources: Sequence[Mapping[str, Any]]) -> None: + for source in sources: + raw_path = Path(str(source["raw_path"])) + raw_size, raw_sha = _sha256_file(raw_path) + if raw_size != source["raw_size"] or raw_sha != source["raw_sha256"]: + _fail(f"Submit-PMU raw changed while building overview: {raw_path}") + provenance_path = Path(str(source["provenance_path"])) + _, provenance_sha = _sha256_file(provenance_path) + if provenance_sha != source["provenance_sha256"]: + _fail(f"Submit-PMU provenance changed while building overview: {provenance_path}") + + +def build_overview(swimlane_raw: Path | str, pmu_dirs: Sequence[Path | str]) -> dict[str, Any]: + """Strictly reload all source artifacts and return a compact machine-readable overview.""" + + raw_path = Path(swimlane_raw) + if raw_path.name != "l2_swimlane_records.json": + _fail("swimlane input must be the producer l2_swimlane_records.json, not merged_swimlane.json") + raw_size, raw_sha256 = _sha256_file(raw_path) + analysis = analyze_capture(raw_path) + if _sha256_file(raw_path) != (raw_size, raw_sha256): + _fail("swimlane raw changed while schema-v4 analysis was running") + frequency_hz, task_count = _validate_swimlane_analysis(analysis) + swimlane = _summarize_swimlane(analysis, raw_path, raw_size, raw_sha256, frequency_hz) + + if len(pmu_dirs) != len(EXPECTED_PMU_MODES): + _fail(f"overview requires exactly {len(EXPECTED_PMU_MODES)} Submit-PMU directories") + captures: dict[str, dict[str, Any]] = {} + capture_objects: dict[str, SubmitPmuCapture] = {} + scenario_keys: set[tuple[str, ...]] = set() + for raw_dir_value in pmu_dirs: + raw_dir = Path(raw_dir_value) + raw_file = raw_dir / DEFAULT_INPUT_NAME if raw_dir.is_dir() else raw_dir + if raw_file.name != DEFAULT_INPUT_NAME: + _fail(f"Submit-PMU input must be a directory or {DEFAULT_INPUT_NAME}") + provenance_file = raw_file.with_name(DEFAULT_PROVENANCE_NAME) + if not provenance_file.is_file(): + _fail(f"missing Submit-PMU provenance: {provenance_file}") + capture = load_capture(raw_file) + provenance, provenance_sha256 = load_provenance(provenance_file, capture) + mode = str(capture.data["capture"]["mode"]) + if mode in captures: + _fail(f"duplicate Submit-PMU capture mode: {mode}") + if mode not in EXPECTED_PMU_MODES: + _fail(f"unsupported Submit-PMU capture mode: {mode}") + if int(capture.data["configuration"]["expected_submits_per_core"]) != task_count: + _fail(f"Submit-PMU {mode} task count does not match the swimlane capture") + profile_key = tuple(str(value) for value in provenance["build"]["profiled_cache_key"]) + scenario_keys.add(profile_key[:-1]) + capture_objects[mode] = capture + captures[mode] = _summarize_pmu_capture(capture, provenance, provenance_sha256, provenance_file) + + missing = EXPECTED_PMU_MODES - captures.keys() + if missing: + _fail(f"missing Submit-PMU capture modes: {sorted(missing)!r}") + if len(scenario_keys) != 1: + _fail("Submit-PMU provenance entries do not describe one common test/platform/scene") + empty_capture = capture_objects[EMPTY_BRACKET_CAPTURE_MODE] + empty_git_head = str(captures[EMPTY_BRACKET_CAPTURE_MODE]["source"]["git_head"]) + for mode, summarized in captures.items(): + if summarized["kind"] != "phase": + continue + if str(summarized["source"]["git_head"]) != empty_git_head: + _fail( + f"Submit-PMU {mode} and {EMPTY_BRACKET_CAPTURE_MODE} provenance git heads do not match" + ) + summarized["recording_cost_reference"] = build_phase_recording_cost_reference( + capture_objects[mode], + empty_capture, + ) + ordered = [captures[mode] for mode in PMU_MODE_ORDER] + sources = [capture["source"] for capture in ordered] + _assert_pmu_sources_unchanged(sources) + + git_heads = sorted({str(source["git_head"]) for source in sources}) + aicore_hashes = sorted({str(source["aicore_kernel_sha256"]) for source in sources}) + phase_profiles = [item for item in ordered if item["kind"] == "phase"] + synthetic_phase_sum = _synthetic_phase_sum_vs_none( + phase_profiles, + captures[NONE_CAPTURE_MODE], + ) + payload = { + "schema": OVERVIEW_SCHEMA, + "semantics": { + "evidence_chains_are_independent": True, + "cross_elf_absolute_subtraction_allowed": False, + "cross_elf_phase_shares_additive": False, + "cross_elf_synthetic_phase_sum_is_exact_overhead": False, + "empty_bracket_recording_cost_is_exact_correction": False, + "swimlane_percentages": "same swimlane ELF aggregate core-work partitions", + "pmu_percentages": ( + "the main phase reference numerator is raw phase observed minus the AIC/AIV-weighted " + "empty-bracket local-recording estimate; its denominator remains that target PMU ELF's " + "raw whole-window value, and the raw phase/raw whole ratio is retained as secondary evidence" + ), + "phase_reference_is_exact_business_share": False, + "phase_reference_provenance": ( + "target and empty-bracket provenance must share one scenario and git revision; the estimate " + "still crosses separately compiled ELFs and does not measure whole-window recording work" + ), + "swimlane_sys_counter": "capture frequency is a SYS counter conversion, not the 1.65 GHz PMU cycle rate", + "sys_boundary_diagnostic": ( + "linked vector/cube Kernel and result-used return-ready atomic waiting are excluded; " + "this SYS counter value is not the primary phase timing source" + ), + "pmu_counter": ( + "PMU whole gate excludes linked vector/cube Kernel; atomic instruction and observation events " + "remain included" + ), + "pmu_whole_gate": ( + "PMU total/scalar-busy/primary I-cache use a per-core gate nested inside the first/last Submit " + "SYS counter closure; scalar_submit_elapsed_ticks sums gate-running SYS segments and then " + "subtracts result-used return-ready atomic waiting" + ), + "phase_observed": ( + "phase PMU total/scalar use running read-clear observations; non-scalar-busy is derived per core " + "as total minus scalar; SYS ticks diagnose boundary closure only" + ), + "synthetic_phase_sum_vs_none": ( + "a non-closing four-way comparison of raw observations, the empty-bracket-scaled recording-cost " + "estimate, the post-subtraction reference, and submit-pmu-none; each AIC/AIV role is scaled by its " + "actual begin/end record-pair count, and the result is not exact performance overhead" + ), + }, + "validation": { + "status": "PASS", + "status_scope": "each evidence chain passed its own strict validation", + "swimlane_schema_v4_reanalyzed": True, + "swimlane_raw_unchanged": True, + "swimlane_build_provenance_available": False, + "swimlane_to_pmu_identity_bound": False, + "swimlane_to_pmu_alignment": "96-core topology and expected Submit count only", + "pmu_profiles_complete": True, + "pmu_profile_count": len(ordered), + "pmu_raw_and_provenance_unchanged": True, + "expected_submits_per_core": task_count, + "scenario_key": list(next(iter(scenario_keys))), + "git_head_uniform": len(git_heads) == 1, + "git_heads": git_heads, + "aicore_kernel_hash_count": len(aicore_hashes), + }, + "swimlane_elf": swimlane, + "submit_pmu_elfs": { + "whole_window": captures[NONE_CAPTURE_MODE], + "phase_profiles": phase_profiles, + "calibration": captures[EMPTY_BRACKET_CAPTURE_MODE], + "synthetic_phase_sum_vs_none": synthetic_phase_sum, + }, + "coverage_matrix": _coverage_matrix(swimlane), + } + return payload + + +def _fmt_int(value: int | float) -> str: + return f"{int(value):,}" + + +def _fmt_us(value: int | float) -> str: + return f"{float(value):,.3f} µs" + + +def _fmt_pct(value: int | float) -> str: + return f"{float(value):.3%}" + + +def _range_text(data: Mapping[str, Any], *, unit: str = "") -> str: + suffix = f" {unit}" if unit else "" + return f"{_fmt_int(data['min'])}–{_fmt_int(data['max'])}{suffix}" + + +def _cycles_to_us(cycles: int | float, cycles_per_ns: float) -> float: + return float(cycles) / cycles_per_ns / 1_000 + + +def _cycle_range_text(data: Mapping[str, Any], cycles_per_ns: float) -> str: + return ( + f"{_range_text(data, unit='cycles')}" + f"（{_cycles_to_us(data['min'], cycles_per_ns):.3f}–" + f"{_cycles_to_us(data['max'], cycles_per_ns):.3f} µs）" + ) + + +def _same_elf_ratio(numerator: int, denominator: int, share: float) -> str: + if not denominator: + return "N/A（本 ELF 分母为 0）" + return f"{_fmt_int(numerator)} / {_fmt_int(denominator)} = {_fmt_pct(share)}" + + +def _per_core_mean(metric: Mapping[str, Any], cores: int) -> float: + if cores <= 0: + _fail("per-core mean requires a positive core count") + return float(metric["sum"]) / cores + + +def _partition_pmu_ratio_cells( + row: Mapping[str, Any], + phase_by_metric: Mapping[str, Mapping[str, Any]], + *, + use_per_core_means: bool = False, +) -> tuple[str, str, str]: + metric = str(row["metric"]) + phase = phase_by_metric.get(metric) + if phase is None: + return "", '—', '—' + + capture_mode = str(phase["capture_mode"]) + mapping = str(phase["mapping"]) + group = phase["recording_cost_reference"]["groups"]["all"] + cores = int(group["cores"]) + + def ratio_cell(metric_name: str) -> tuple[str, float | None, float | None]: + metric_data = group["metrics"][metric_name] + reference_share = metric_data["after_recording_cost_reference_ratio_to_raw_whole"] + raw_share = metric_data["raw_phase_observed_ratio_to_raw_whole"] + estimate_share = metric_data["recording_cost_estimate_share_of_raw_phase"] + scale = cores if use_per_core_means else 1 + reference_value = float(metric_data["after_recording_cost_reference_sum"]) / scale + raw_value = float(metric_data["raw_phase_observed_sum"]) / scale + whole_value = float(metric_data["raw_whole_sum"]) / scale + unit = "每核均值" if use_per_core_means else "Σ" + reference_text = _ratio_value_text(reference_share) + raw_text = _ratio_value_text(raw_share) + estimate_text = _ratio_value_text(estimate_share) + if estimate_share is None: + sensitivity = "敏感度 N/A" + elif float(estimate_share) >= 0.85: + sensitivity = "高度依赖校准" + elif float(estimate_share) >= 0.70: + sensitivity = "较高依赖校准" + elif float(estimate_share) >= 0.40: + sensitivity = "中等依赖校准" + else: + sensitivity = "较低依赖校准" + title = ( + f"{capture_mode}：参考 {unit} {reference_value:,.3f} / raw whole " + f"{whole_value:,.3f} = {reference_text}；raw {unit} {raw_value:,.3f} / " + f"{whole_value:,.3f} = {raw_text}" + ) + cell = ( + f'' + f"{reference_text}" + "扣局部记录估算后的阶段值 / raw 整窗，仅参考" + f"raw {raw_text} · 记录估算/raw {estimate_text}" + f"{sensitivity} · {html.escape(mapping)}" + ) + return cell, reference_share, raw_share + + total_cell, total_reference_share, total_raw_share = ratio_cell("pmu_total_cycles") + scalar_cell, scalar_reference_share, scalar_raw_share = ratio_cell("scalar_busy_cycles") + row_attributes = ( + f' data-pmu-profile="{html.escape(capture_mode)}" data-pmu-mapping="{html.escape(mapping)}"' + f' data-pmu-reference-share="{html.escape(_ratio_value_text(total_reference_share))}"' + f' data-pmu-raw-share="{html.escape(_ratio_value_text(total_raw_share))}"' + f' data-scalar-reference-share="{html.escape(_ratio_value_text(scalar_reference_share))}"' + f' data-scalar-raw-share="{html.escape(_ratio_value_text(scalar_raw_share))}"' + ) + return row_attributes, total_cell, scalar_cell + + +def _partition_mean_diagnostic( + partition: Mapping[str, Any], + phase_by_metric: Mapping[str, Mapping[str, Any]], + phase_profiles: Sequence[Mapping[str, Any]], + whole_window: Mapping[str, Any], + core_count: int, +) -> dict[str, Any]: + mapped_rows = [] + unmapped_labels = [] + for row in partition["rows"]: + phase = phase_by_metric.get(str(row["metric"])) + if phase is None: + unmapped_labels.append(str(row["label"])) + continue + mapped_rows.append(row) + + profiles = list(phase_profiles) + if not profiles: + _fail(f"partition {partition['name']!r} has no PMU profiles for mean comparison") + profile_modes = [str(profile["capture_mode"]) for profile in profiles] + if len(profile_modes) != len(set(profile_modes)): + _fail(f"partition {partition['name']!r} mean comparison contains duplicate PMU profiles") + mapped_modes = {str(phase_by_metric[str(row["metric"])]["capture_mode"]) for row in mapped_rows} + if not mapped_modes <= set(profile_modes): + _fail(f"partition {partition['name']!r} mean comparison omits a directly mapped PMU profile") + decomposition = _synthetic_phase_sum_vs_none(profiles, whole_window) + + return { + "profile_count": len(profiles), + "profiles": profile_modes, + "direct_mapped_profile_count": len(mapped_modes), + "unmapped_labels": unmapped_labels, + "partition_row_count": len(partition["rows"]), + "partition_core_time_mean_us": sum(float(row["core_time_us"]) for row in partition["rows"]) / core_count, + "pmu_cycles_per_ns": { + name: float(whole_window["pmu_cycles_per_ns"][name]) for name in ("all", "aic", "aiv") + }, + "decomposition": decomposition, + } + + +def _mean_metric_text(value: float, unit: str) -> str: + suffix = "cycles/core" if unit == "cycles" else "events/core" + return f"{value:,.3f} {suffix}" + + +def _ratio_value_text(value: float | None) -> str: + return "N/A" if value is None else _fmt_pct(value) + + +def _mean_equivalent_time(value: float, unit: str, cycles_per_ns: float) -> str: + if unit != "cycles": + return "" + return f"按 {cycles_per_ns:.6f} cycles/ns：≈ {_cycles_to_us(value, cycles_per_ns):,.3f} µs/core" + + +def _partition_mean_comparison_rows(diagnostic: Mapping[str, Any]) -> str: + decomposition = diagnostic["decomposition"] + groups = decomposition["groups"] + all_group = groups["all"] + aic_group = groups["aic"] + aiv_group = groups["aiv"] + all_metrics = all_group["metrics"] + aic_metrics = aic_group["metrics"] + aiv_metrics = aiv_group["metrics"] + frequencies = diagnostic["pmu_cycles_per_ns"] + raw_cards = [ + ( + "
" + "取数范围" + f"{int(diagnostic['profile_count'])} 个 phase 每核均值之和" + f"{int(diagnostic['direct_mapped_profile_count'])} 个直接表格行 + " + "ArgBuild 精确子段" + "SubmitTransition 位于 SubmitUnion 外，不进入分子" + "
" + ) + ] + recording_cards = [ + ( + "
" + "估算方法空区间每次记录开销 × 本组记录次数" + f"ALL {float(all_group['phase_record_pairs_per_core']):,.3f} 组/core · " + f"AIC {float(aic_group['phase_record_pairs_per_core']):,.3f} · " + f"AIV {float(aiv_group['phase_record_pairs_per_core']):,.3f}" + "linked Kernel pause/resume 产生的额外记录也已计入" + "
" + ) + ] + reference_cards = [ + ( + "
" + "参考值口径原始观测 − 上述记录开销估算" + "允许为负；负值表示估算或运行波动已超过 raw" + "它不是“真实业务值”，也不是精确校正结果" + "
" + ) + ] + none_cards = [ + ( + "
" + "对照范围首个 Submit begin → 末个 Submit end" + "包含 BetweenSubmitResidual / SubmitTransition" + "独立 ELF；不是与分子完全同范围的一次闭合" + "
" + ) + ] + for metric_name, label, _phase_field, _whole_field, unit in SYNTHETIC_PHASE_SUM_METRICS: + metric = all_metrics[metric_name] + aic_metric = aic_metrics[metric_name] + aiv_metric = aiv_metrics[metric_name] + raw_cards.append( + "
" + f"{html.escape(label)} raw observed" + f"{_mean_metric_text(metric['raw_observed_mean'], unit)}" + f"{_mean_equivalent_time(metric['raw_observed_mean'], unit, frequencies['all'])}" + f"raw / none：{_ratio_value_text(metric['raw_observed_ratio_to_submit_none'])}" + f"AIC {_ratio_value_text(aic_metric['raw_observed_ratio_to_submit_none'])} · " + f"AIV {_ratio_value_text(aiv_metric['raw_observed_ratio_to_submit_none'])}" + "
" + ) + recording_cards.append( + "
" + f"{html.escape(label)} 记录开销估算" + f"{_mean_metric_text(metric['recording_cost_estimate_mean'], unit)}" + f"{_mean_equivalent_time(metric['recording_cost_estimate_mean'], unit, frequencies['all'])}" + f"占 raw：{_ratio_value_text(metric['recording_cost_estimate_share_of_raw'])}；" + f"相对 none：{_ratio_value_text(metric['recording_cost_estimate_ratio_to_submit_none'])}" + f"AIC {_ratio_value_text(aic_metric['recording_cost_estimate_share_of_raw'])} · " + f"AIV {_ratio_value_text(aiv_metric['recording_cost_estimate_share_of_raw'])}" + "
" + ) + reference_cards.append( + "
" + f"{html.escape(label)} 扣除后参考值" + f"{_mean_metric_text(metric['after_recording_cost_reference_mean'], unit)}" + f"{_mean_equivalent_time(metric['after_recording_cost_reference_mean'], unit, frequencies['all'])}" + f"参考值 / none：" + f"{_ratio_value_text(metric['after_recording_cost_reference_ratio_to_submit_none'])}" + f"AIC {_ratio_value_text(aic_metric['after_recording_cost_reference_ratio_to_submit_none'])} · " + f"AIV {_ratio_value_text(aiv_metric['after_recording_cost_reference_ratio_to_submit_none'])}" + "
" + ) + none_cards.append( + "
" + f"{html.escape(label)} 每核均值" + f"{_mean_metric_text(metric['submit_none_mean'], unit)}" + f"{_mean_equivalent_time(metric['submit_none_mean'], unit, frequencies['all'])}" + "对照基准：100%" + f"AIC {_mean_metric_text(aic_metric['submit_none_mean'], unit)} · " + f"AIV {_mean_metric_text(aiv_metric['submit_none_mean'], unit)}" + "
" + ) + + unmapped = "、".join(html.escape(label) for label in diagnostic["unmapped_labels"]) + profile_count = int(diagnostic["profile_count"]) + return ( + '' + "SubmitUnion 平均每核时间合计" + f"{_fmt_us(diagnostic['partition_core_time_mean_us'])}" + f"{int(diagnostic['partition_row_count'])} 个分段每核均值之和" + "100.000%" + '同一泳道 ELF 的严格分区闭合；' + "不与下方独立 PMU ELF 的时间直接相减。" + f'' + f"原始分段观测合计{profile_count} 个独立 phase ELF" + '' + f'
{"".join(raw_cards)}
' + f'

合计 9 个直接映射行和 ArgBuild 的 Claim→Materialize 精确子段；' + f"{unmapped} 的其余 residual 不进入分子。" + "raw 含分段记录代码自身开销，不能直接拿来解释业务耗时。

" + "" + '' + "空区间估算的记录代码开销按 AIC/AIV 分开缩放后加权" + '' + f'
{"".join(recording_cards)}
' + '

empty-bracket 只在固定调用点测量一对紧邻 begin/end；这里按每个 phase 实际' + " begin/end 记录组数缩放。调用点、I-cache 布局和运行轮次不同，所以只能作为估算。

" + "" + '' + "扣除上述估算后的参考值raw − 记录开销估算" + '' + f'
{"".join(reference_cards)}
' + '

该行用于判断此前 PMU/Scalar 合计膨胀主要来自哪里；它不是同一 ELF 的闭合，' + "不能当成已精确恢复的业务值。

" + "" + '' + f"{NONE_CAPTURE_MODE}每核均值对照" + '' + f'
{"".join(none_cards)}
' + "" + ) + + +def _partition_html( + partition: Mapping[str, Any], + phase_by_metric: Mapping[str, Mapping[str, Any]], + *, + show_ranges: bool = True, + mean_core_count: int | None = None, + comparison_whole_window: Mapping[str, Any] | None = None, + comparison_phase_profiles: Sequence[Mapping[str, Any]] | None = None, +) -> str: + colors = ("#2563eb", "#0d9488", "#7c3aed", "#d97706", "#dc2626", "#0891b2", "#65a30d") + bars = [] + rows = [] + for index, row in enumerate(partition["rows"]): + color = colors[index % len(colors)] + share = float(row["share"]) + bars.append( + f'' + ) + if show_ranges and row["aic_per_core"] is not None: + aic = f"{row['aic_per_core']['min_us']:.3f}–{row['aic_per_core']['max_us']:.3f} µs" + aiv = f"{row['aiv_per_core']['min_us']:.3f}–{row['aiv_per_core']['max_us']:.3f} µs" + else: + aic = aiv = "—" + row_attributes, pmu_total_cell, scalar_busy_cell = _partition_pmu_ratio_cells( + row, + phase_by_metric, + use_per_core_means=mean_core_count is not None, + ) + core_time_us = float(row["core_time_us"]) + share_cell = f"{_fmt_pct(share)}" + if mean_core_count is not None: + core_time_us /= mean_core_count + parent_mean_us = sum(float(item["core_time_us"]) for item in partition["rows"]) / mean_core_count + mean_share = core_time_us / parent_mean_us if parent_mean_us else 0.0 + if not math.isclose(mean_share, share, rel_tol=1e-12, abs_tol=1e-12): + _fail(f"partition {partition['name']!r} mean share disagrees with aggregate share") + share_cell = ( + f'' + f"{_fmt_pct(mean_share)}" + ) + rows.append( + f'' + f'{html.escape(row["label"])}' + f"{_fmt_us(core_time_us)}{share_cell}" + f"{pmu_total_cell}{scalar_busy_cell}" + f"{aic}{aiv}" + "" + ) + comparison_rows = "" + if comparison_whole_window is not None: + if mean_core_count is None: + _fail("partition mean comparison requires mean_core_count") + if comparison_phase_profiles is None: + _fail("partition mean comparison requires explicit phase profiles") + diagnostic = _partition_mean_diagnostic( + partition, + phase_by_metric, + comparison_phase_profiles, + comparison_whole_window, + mean_core_count, + ) + comparison_rows = _partition_mean_comparison_rows(diagnostic) + core_time_header = "平均每核时间" if mean_core_count is not None else "Σ core-time" + swimlane_share_header = ( + "泳道每核均值 / 同父区间每核均值" if mean_core_count is not None else "泳道同父区间" + ) + pmu_share_header = ( + "扣局部记录估算后的 Phase PMU 每核均值 / raw whole PMU 每核均值" + if mean_core_count is not None + else "扣局部记录估算后的 Phase PMU / raw whole PMU" + ) + scalar_share_header = ( + "扣局部记录估算后的 Phase scalar 每核均值 / raw whole scalar 每核均值" + if mean_core_count is not None + else "扣局部记录估算后的 Phase scalar / raw whole scalar" + ) + comparison_header = "均值占比对照" if mean_core_count is not None else "占比对照" + return f""" +
+

{html.escape(str(partition["name"]))}

+
{"".join(bars)}
+
+ + + + + + + + + + + + + + {"".join(rows)}{comparison_rows} +
区域{core_time_header}{comparison_header}AIC 每核 min–maxAIV 每核 min–max
{swimlane_share_header}PMU 局部记录扣除参考
{pmu_share_header}；raw 比例同格保留		Scalar 局部记录扣除参考
{scalar_share_header}；raw 比例同格保留
+
+ """ + + +def _synthetic_phase_sum_html(diagnostic: Mapping[str, Any]) -> str: + all_group = diagnostic["groups"]["all"] + aic_group = diagnostic["groups"]["aic"] + aiv_group = diagnostic["groups"]["aiv"] + all_metrics = all_group["metrics"] + aic_metrics = aic_group["metrics"] + aiv_metrics = aiv_group["metrics"] + chart_limit = 6.0 + baseline_position = 1.0 / chart_limit * 100 + chart_rows = [] + table_rows = [] + for metric_name, label, _phase_field, _whole_field, unit in SYNTHETIC_PHASE_SUM_METRICS: + metric = all_metrics[metric_name] + raw_ratio = metric["raw_observed_ratio_to_submit_none"] + recording_ratio = metric["recording_cost_estimate_ratio_to_submit_none"] + reference_ratio = metric["after_recording_cost_reference_ratio_to_submit_none"] + recording_width = ( + 0.0 if recording_ratio is None else min(max(float(recording_ratio), 0.0), chart_limit) / chart_limit * 100 + ) + remaining_limit = max(0.0, 100.0 - recording_width) + reference_width = ( + 0.0 + if reference_ratio is None + else min(max(float(reference_ratio), 0.0) / chart_limit * 100, remaining_limit) + ) + overflow = raw_ratio is not None and float(raw_ratio) > chart_limit + overflow_text = "图形封顶 600%" if overflow else "" + negative_text = ( + "扣除后参考值为负，图中不画负向部分；精确值见下表" + if reference_ratio is not None and float(reference_ratio) < 0 + else "" + ) + title = ( + f"{label}：raw {_ratio_value_text(raw_ratio)}；记录开销估算 " + f"{_ratio_value_text(recording_ratio)}；扣除后参考值 {_ratio_value_text(reference_ratio)}" + ) + chart_rows.append( + '
' + f"{html.escape(label)}" + f'
' + f'' + f'' + f'
' + f"raw {_ratio_value_text(raw_ratio)}" + f"记录估算 {_ratio_value_text(recording_ratio)} + " + f"扣除后 {_ratio_value_text(reference_ratio)}{overflow_text}{negative_text}" + "
" + ) + + def mean_cell(field: str) -> str: + value = float(metric[field]) + time_text = ( + f"≈ {_cycles_to_us(value, float(diagnostic['pmu_cycles_per_ns']['all'])):,.3f} µs/core" + if unit == "cycles" + else "" + ) + return f"{_mean_metric_text(value, unit)}{time_text}" + + aic_raw = _ratio_value_text(aic_metrics[metric_name]["raw_observed_ratio_to_submit_none"]) + aic_recording = _ratio_value_text( + aic_metrics[metric_name]["recording_cost_estimate_ratio_to_submit_none"] + ) + aic_reference = _ratio_value_text( + aic_metrics[metric_name]["after_recording_cost_reference_ratio_to_submit_none"] + ) + aiv_raw = _ratio_value_text(aiv_metrics[metric_name]["raw_observed_ratio_to_submit_none"]) + aiv_recording = _ratio_value_text( + aiv_metrics[metric_name]["recording_cost_estimate_ratio_to_submit_none"] + ) + aiv_reference = _ratio_value_text( + aiv_metrics[metric_name]["after_recording_cost_reference_ratio_to_submit_none"] + ) + table_rows.append( + "" + f"{html.escape(label)}" + f"{mean_cell('raw_observed_mean')}" + f"raw / none {_ratio_value_text(raw_ratio)}" + f"{mean_cell('recording_cost_estimate_mean')}" + f"占 raw {_ratio_value_text(metric['recording_cost_estimate_share_of_raw'])}" + f"{mean_cell('after_recording_cost_reference_mean')}" + f"参考值 / none {_ratio_value_text(reference_ratio)}" + f"{mean_cell('submit_none_mean')}基线 100%" + "" + f"AIC：raw {aic_raw} / 记录估算 {aic_recording} / 扣除后 {aic_reference}" + f"AIV：raw {aiv_raw} / 记录估算 {aiv_recording} / 扣除后 {aiv_reference}" + "" + "" + ) + profile_count = int(diagnostic["phase_profile_count"]) + return f""" +
+

{profile_count} 个业务分段的记录开销拆分（含 SubmitTransition）

+

raw 合计包含分段记录代码自身开销，不能直接拿它和 + {NONE_CAPTURE_MODE} 比出“业务变慢了多少”。 + 本节另外用 empty-bracket 测得“一组 begin/end 记录代码”的开销，再按每个 phase 实际记录组数估算； + AIC 与 AIV 分开计算后才合成 ALL。扣除后的数值只供定位量级，不是精确还原的业务值。

+

下图以 none 为 100%（黑色竖线），橙色为“空区间估算的记录代码开销”， + 蓝色为“扣除上述估算后的参考值”；二者代数相加为 raw。横轴统一为 0–600%，超过部分只在图上封顶。 + empty-bracket 的调用点、代码布局及采集轮次与各 phase 不同，所以该估算不能跨 ELF 当作精确扣除。 + I-cache raw 使用 phase observed，none 使用 primary。

+
{"".join(chart_rows)}
+
+ + + {"".join(table_rows)} +
指标原始观测合计记录代码开销估算扣除后参考值submit-pmu-noneAIC / AIV 分角色比例
+

本组合计 {float(all_group['phase_record_pairs_per_core']):,.3f} 组记录/core； + AIC {float(aic_group['phase_record_pairs_per_core']):,.3f}，AIV + {float(aiv_group['phase_record_pairs_per_core']):,.3f}。记录组数包含 linked Kernel + pause/resume 产生的额外 begin/end；动态阶段的零调用核仍按完整 32/64 核总体计入均值。

+
+ """ + + +def _phase_metric_html( + label: str, + metric: Mapping[str, Any], + denominator: int, + share: float, + aic: Mapping[str, Any], + aiv: Mapping[str, Any], + recording_reference: Mapping[str, Any] | None = None, +) -> str: + ratio = _same_elf_ratio(int(metric["sum"]), denominator, share) + if recording_reference is not None: + reference_sum = float(recording_reference["after_recording_cost_reference_sum"]) + recording_sum = float(recording_reference["recording_cost_estimate_sum"]) + reference_ratio = _ratio_value_text( + recording_reference["after_recording_cost_reference_ratio_to_raw_whole"] + ) + raw_ratio = _ratio_value_text(recording_reference["raw_phase_observed_ratio_to_raw_whole"]) + recording_share = _ratio_value_text( + recording_reference["recording_cost_estimate_share_of_raw_phase"] + ) + return f""" +
+ {html.escape(label)} + 扣局部记录估算参考 {reference_sum:,.3f} + 参考值 / raw 整窗：{reference_ratio} + raw observed {_fmt_int(metric["sum"])} / raw 整窗 {_fmt_int(denominator)} + = {raw_ratio} + raw observed − 记录代码开销估算 {recording_sum:,.3f}； + 估算/raw {recording_share} + raw AIC 每核 {_range_text(aic)}；raw AIV 每核 {_range_text(aiv)} +
+ """ + return f""" +
+ {html.escape(label)} + raw observed {_fmt_int(metric["sum"])} + raw observed / raw 整窗：{ratio} + raw AIC 每核 {_range_text(aic)}；raw AIV 每核 {_range_text(aiv)} +
+ """ + + +def _phase_cycle_metric_html( + label: str, + metric: Mapping[str, Any], + aic: Mapping[str, Any], + aiv: Mapping[str, Any], + frequencies: Mapping[str, float], + relationships: Sequence[str], + *, + note: str = "", + recording_reference: Mapping[str, Any] | None = None, +) -> str: + relationship_html = "".join(f"{html.escape(value)}" for value in relationships) + note_html = f"{html.escape(note)}" if note else "" + if recording_reference is not None: + reference_sum = float(recording_reference["after_recording_cost_reference_sum"]) + recording_sum = float(recording_reference["recording_cost_estimate_sum"]) + reference_ratio = _ratio_value_text( + recording_reference["after_recording_cost_reference_ratio_to_raw_whole"] + ) + raw_ratio = _ratio_value_text(recording_reference["raw_phase_observed_ratio_to_raw_whole"]) + recording_share = _ratio_value_text( + recording_reference["recording_cost_estimate_share_of_raw_phase"] + ) + return f""" +
+ {html.escape(label)} + 扣局部记录估算参考 Σ {reference_sum:,.3f} cycles · ≈ + {_cycles_to_us(reference_sum, frequencies["all"]):.3f} µs + 参考值 / raw 整窗：{reference_ratio}；raw observed / raw 整窗： + {raw_ratio} + raw observed Σ {_fmt_int(metric["sum"])} cycles − 记录代码开销估算 + {recording_sum:,.3f} cycles；估算/raw {recording_share} + {relationship_html} + raw AIC 每核 {_cycle_range_text(aic, frequencies["aic"])} + raw AIV 每核 {_cycle_range_text(aiv, frequencies["aiv"])} + {note_html} +
+ """ + return f""" +
+ {html.escape(label)} + raw observed Σ {_fmt_int(metric["sum"])} cycles · ≈ + {_cycles_to_us(metric["sum"], frequencies["all"]):.3f} µs + {relationship_html} + raw AIC 每核 {_cycle_range_text(aic, frequencies["aic"])} + raw AIV 每核 {_cycle_range_text(aiv, frequencies["aiv"])} + {note_html} +
+ """ + + +def _phase_card_html(item: Mapping[str, Any]) -> str: + groups = item["phase"]["groups"] + all_group = groups["all"] + aic = groups["aic"] + aiv = groups["aiv"] + source = item["source"] + calls = ( + f"{_fmt_int(all_group['phase_business_calls'])} / " + f"{_fmt_int(aic['phase_business_calls'])} / {_fmt_int(aiv['phase_business_calls'])}" + ) + calls_per_core = ( + f"AIC {_range_text(aic['phase_calls_per_core'])}（零调用核 {aic['phase_zero_call_cores']}）；" + f"AIV {_range_text(aiv['phase_calls_per_core'])}（零调用核 {aiv['phase_zero_call_cores']}）" + ) + total_metric = all_group["phase_total_cycles_observed"] + total_aic = aic["phase_total_cycles_observed"] + total_aiv = aiv["phase_total_cycles_observed"] + scalar_metric = all_group["phase_scalar_busy_observed"] + scalar_aic = aic["phase_scalar_busy_observed"] + scalar_aiv = aiv["phase_scalar_busy_observed"] + residual_metric = all_group["phase_non_scalar_busy_cycles"] + residual_aic = aic["phase_non_scalar_busy_cycles"] + residual_aiv = aiv["phase_non_scalar_busy_cycles"] + request_metric = all_group["phase_icache_requests_observed"] + request_aic = aic["phase_icache_requests_observed"] + request_aiv = aiv["phase_icache_requests_observed"] + miss_metric = all_group["phase_icache_misses_observed"] + miss_aic = aic["phase_icache_misses_observed"] + miss_aiv = aiv["phase_icache_misses_observed"] + denominators = item["denominators"]["all"] + frequencies = item["pmu_cycles_per_ns"] + kernel_calls = all_group["phase_excluded_kernel_calls"] + recording_metrics = ( + item["recording_cost_reference"]["groups"]["all"]["metrics"] + if "recording_cost_reference" in item + else None + ) + total_reference = None if recording_metrics is None else recording_metrics["pmu_total_cycles"] + scalar_reference = None if recording_metrics is None else recording_metrics["scalar_busy_cycles"] + residual_reference = None if recording_metrics is None else recording_metrics["non_scalar_busy_cycles"] + request_reference = None if recording_metrics is None else recording_metrics["icache_requests"] + miss_reference = None if recording_metrics is None else recording_metrics["icache_misses"] + total_html = _phase_cycle_metric_html( + "Phase PMU total", + total_metric, + total_aic, + total_aiv, + frequencies, + ( + "raw Phase total / raw whole total：" + + _same_elf_ratio( + int(total_metric["sum"]), + int(denominators["pmu_total_cycles"]["sum"]), + all_group["phase_total_share_of_pmu_total"], + ), + ), + recording_reference=total_reference, + ) + if total_reference is None or scalar_reference is None: + scalar_relationship = ( + "raw Scalar / raw Phase total：" + + _same_elf_ratio( + int(scalar_metric["sum"]), + int(total_metric["sum"]), + all_group["phase_scalar_busy_share_of_phase_total"], + ) + ) + else: + reference_total_sum = float(total_reference["after_recording_cost_reference_sum"]) + reference_scalar_sum = float(scalar_reference["after_recording_cost_reference_sum"]) + reference_scalar_share = ( + reference_scalar_sum / reference_total_sum if reference_total_sum else None + ) + scalar_relationship = ( + "参考 Scalar / 参考 Phase total：" + + _ratio_value_text(reference_scalar_share) + + "；raw " + + _fmt_pct(all_group["phase_scalar_busy_share_of_phase_total"]) + ) + scalar_html = _phase_cycle_metric_html( + "Phase scalar busy", + scalar_metric, + scalar_aic, + scalar_aiv, + frequencies, + ( + scalar_relationship, + "raw Phase scalar / raw whole scalar：" + + _same_elf_ratio( + int(scalar_metric["sum"]), + int(denominators["scalar_busy_cycles"]["sum"]), + all_group["phase_scalar_share_of_whole_scalar"], + ), + ), + recording_reference=scalar_reference, + ) + residual_share = ( + float(residual_metric["sum"]) / float(total_metric["sum"]) if total_metric["sum"] else 0.0 + ) + if residual_reference is None or total_reference is None: + residual_relationship = ( + "raw residual / raw Phase total：" + + _same_elf_ratio( + int(residual_metric["sum"]), + int(total_metric["sum"]), + residual_share, + ) + ) + else: + reference_total_sum = float(total_reference["after_recording_cost_reference_sum"]) + reference_residual_sum = float(residual_reference["after_recording_cost_reference_sum"]) + reference_residual_share = ( + reference_residual_sum / reference_total_sum if reference_total_sum else None + ) + residual_relationship = ( + "参考 residual / 参考 Phase total：" + + _ratio_value_text(reference_residual_share) + + "；raw " + + _fmt_pct(residual_share) + ) + residual_html = _phase_cycle_metric_html( + "非 Scalar-busy 残余", + residual_metric, + residual_aic, + residual_aiv, + frequencies, + (f"逐核先算 total−scalar；{residual_relationship}",), + note="它不是“空闲时间”，可能包含等待、I-cache、atomic、观察器及其他非 scalar-busy 周期。", + recording_reference=residual_reference, + ) + request_html = _phase_metric_html( + "I-cache request observed", + request_metric, + denominators["primary_icache_requests"]["sum"], + all_group["phase_request_observed_share_of_primary"], + request_aic, + request_aiv, + request_reference, + ) + miss_html = _phase_metric_html( + "I-cache miss observed", + miss_metric, + denominators["primary_icache_misses"]["sum"], + all_group["phase_miss_observed_share_of_primary"], + miss_aic, + miss_aiv, + miss_reference, + ) + return f""" +
+
+

{html.escape(str(item["label"]))}

+ {html.escape(str(item["capture_mode"]))}
+ {html.escape(str(item["mapping"]))} +
+

{html.escape(str(item["phase"]["boundary"]))} · calls ALL/AIC/AIV {calls}
+ calls/core {calls_per_core}

+
+ {total_html} + {scalar_html} + {residual_html} + {request_html} + {miss_html} +
+

SYS 边界诊断： + Σ {_fmt_int(all_group["phase_elapsed_ticks"]["sum"])} raw ticks； + AIC 每核 {_range_text(aic["phase_elapsed_ticks"], unit="ticks")}； + AIV 每核 {_range_text(aiv["phase_elapsed_ticks"], unit="ticks")}。 + 只核验 phase 边界闭合，不作为阶段主时间或占比分母。

+

Scalar shadow 诊断： + whole CNT2 scalar − CNT3 shadow scalar Σ {_fmt_int(all_group["shadow_scalar_loss"]["sum"])} cycles； + AIC 每核 {_range_text(aic["shadow_scalar_loss"], unit="cycles")}； + AIV 每核 {_range_text(aiv["shadow_scalar_loss"], unit="cycles")}。

+

分母只来自本 ELF；excluded linked Kernel calls={_fmt_int(kernel_calls)}； + PMU 等效时间分别按 ALL/AIC/AIV + {frequencies["all"]:.6f}/{frequencies["aic"]:.6f}/{frequencies["aiv"]:.6f} cycles/ns 换算； + global Submit closure={item["global_submit_span_us"]:.3f} µs（不作 phase 分母）。

+

git {html.escape(source["git_head"][:12])} · AICore ELF + {html.escape(source["aicore_kernel_sha256"][:12])} · + 单份报告

+
+ """ + + +def _whole_window_html(item: Mapping[str, Any]) -> str: + groups = item["denominators"] + all_group = groups["all"] + aic = groups["aic"] + aiv = groups["aiv"] + + frequencies = item["pmu_cycles_per_ns"] + + def card(label: str, key: str, unit: str = "", *, cycles: bool = False, note: str = "") -> str: + if cycles: + all_value = ( + f"Σ {_fmt_int(all_group[key]['sum'])} cycles · ≈ " + f"{_cycles_to_us(all_group[key]['sum'], frequencies['all']):.3f} µs" + ) + aic_value = _cycle_range_text(aic[key], frequencies["aic"]) + aiv_value = _cycle_range_text(aiv[key], frequencies["aiv"]) + else: + all_value = _fmt_int(all_group[key]["sum"]) + aic_value = _range_text(aic[key], unit=unit) + aiv_value = _range_text(aiv[key], unit=unit) + note_html = f"{html.escape(note)}" if note else "" + return f""" +
+ {html.escape(label)}{all_value} + AIC 每核 {aic_value} + AIV 每核 {aiv_value} + {note_html} +
""" + + source = item["source"] + total_sum = int(all_group["pmu_total_cycles"]["sum"]) + scalar_sum = int(all_group["scalar_busy_cycles"]["sum"]) + residual_sum = int(all_group["non_scalar_busy_cycles"]["sum"]) + sys_diagnostic_card = card( + "SYS 边界闭合诊断（raw ticks）", + "scalar_submit_elapsed_ticks", + "ticks", + note="只核验首末 Submit 与门控边界；不参与 PMU 1.65 GHz 时间换算或阶段比例。", + ) + total_card = card("PMU total cycles", "pmu_total_cycles", cycles=True) + scalar_card = card( + "Scalar busy cycles", + "scalar_busy_cycles", + cycles=True, + note="Scalar / PMU total：" + + _same_elf_ratio(scalar_sum, total_sum, scalar_sum / total_sum if total_sum else 0.0), + ) + residual_card = card( + "非 Scalar-busy 残余", + "non_scalar_busy_cycles", + cycles=True, + note="逐核先算 total−scalar；残余 / PMU total：" + + _same_elf_ratio(residual_sum, total_sum, residual_sum / total_sum if total_sum else 0.0) + + "。不是空闲时间。", + ) + return f""" +
+

{NONE_CAPTURE_MODE}：PMU whole gate 与 SYS 边界诊断

+

global 首末 Submit 仅作闭合：{item["global_submit_span_us"]:.3f} µs； + 下列数值是 96 核各自累计，不是墙钟。每核先读首个 Submit start tick，再启动 PMU；末次 Submit + 先停止 PMU，再读 end tick，所以 PMU gate 嵌在首末 SYS closure 内。Scalar elapsed 累计 gate-running + SYS 段，再扣 linked Kernel 与 return-ready atomic 等待；PMU cycle 与 SYS tick 口径不同，不能直接相减。

+
+ {sys_diagnostic_card} + {total_card} + {scalar_card} + {residual_card} + {card("Primary I-cache request", "primary_icache_requests")} + {card("Primary I-cache miss", "primary_icache_misses")} +
+

PMU 等效时间分别按 ALL/AIC/AIV + {frequencies["all"]:.6f}/{frequencies["aic"]:.6f}/{frequencies["aiv"]:.6f} cycles/ns 换算。

+

git {html.escape(source["git_head"][:12])} · AICore ELF + {html.escape(source["aicore_kernel_sha256"][:12])} · + 单份报告

+
+ """ + + +def render_overview(payload: Mapping[str, Any]) -> str: + """Render a self-contained report without inventing a cross-ELF denominator.""" + + swimlane = payload["swimlane_elf"] + pmu = payload["submit_pmu_elfs"] + validation = payload["validation"] + phase_by_mode = {str(item["capture_mode"]): item for item in pmu["phase_profiles"]} + phase_by_metric = {} + for metric, mode in PARTITION_PMU_MODE_BY_METRIC.items(): + if mode not in phase_by_mode: + _fail(f"partition PMU mapping requires missing capture mode {mode!r}") + phase_by_metric[metric] = phase_by_mode[mode] + submit_union_core_count = int(swimlane["core_count"]) + none_core_count = int(pmu["whole_window"]["denominators"]["all"]["cores"]) + if submit_union_core_count != none_core_count: + _fail("SubmitUnion swimlane and submit-pmu-none must describe the same core count") + submit_union_phase_profiles = [phase_by_mode[mode] for mode in SUBMIT_UNION_PMU_MODES] + phase_cards = "".join(_phase_card_html(item) for item in pmu["phase_profiles"]) + calibration = _phase_card_html(pmu["calibration"]) + coverage_rows = "".join( + "" + f"{html.escape(row['swimlane_region'])}" + f"{html.escape(row['pmu_profile'])}" + f"{html.escape(row['coverage'])}{html.escape(row['note'])}" + "" + for row in payload["coverage_matrix"] + ) + residual_rows_list = [] + for name, value in swimlane["residual_segments"].items(): + for segment in value["segments"]: + duration_us = segment["cycles"] / swimlane["sys_counter_frequency_hz"] * 1_000_000 + residual_rows_list.append( + "" + f"{html.escape(name)}/{html.escape(segment['boundary'])}" + f"{_fmt_int(segment['event_count'])}{_fmt_us(duration_us)}" + f"{_fmt_int(segment['aic_cycles'])}" + f"{_fmt_int(segment['aiv_cycles'])}" + "" + ) + residual_rows = "".join(residual_rows_list) + overlay_rows = "".join( + "" + f"{html.escape(name)}" + f"{_fmt_int(value['event_count'])}" + f"{_fmt_int(value['aggregate_duration_cycles'])}" + "否" + for name, value in swimlane["overlays"].items() + ) + containment_rows = "".join( + f"{html.escape(name)}{_fmt_int(value)}" + for name, value in sorted(swimlane["kernel_containment"].items()) + ) + head_list = "、".join(html.escape(head[:12]) for head in validation["git_heads"]) + document = f""" + + + + + FDWIC Submit 全 span 证据汇总 + + +
+

FDWIC Submit 全 span 证据汇总

+

PASS · 96 核 · 每核 {validation["expected_submits_per_core"]:,} 个 Submit · + 泳道 raw 重新执行 schema-v4 严格闭合 · 13/13 PMU raw + provenance 闭合。 + PASS 只表示两条证据链各自通过门禁，不表示它们已绑定到同一 ELF。

+
这里有两条互不混算的证据链。 + 泳道图的百分比只在同一个泳道 ELF 的排他时间树中相加；每个 Submit-PMU profile 都是独立 ELF， + phase PMU total、Scalar busy、非 Scalar-busy 残余、request 和 miss 只能使用本 ELF 自己的分母。 + 不同 ELF 的绝对值不能直接相减，各 PMU 行的占比也不能相加成 100%。业务 phase 的主显示值 + 先从 raw observed 分子中扣除 empty-bracket 估算的局部记录代码开销，再除以本 ELF 的 raw whole； + raw observed / raw whole 同格保留。empty-bracket 没有测到 whole 窗口中的全部记录工作，因此这个 + 主显示值只是更接近业务量级的参考值，不是“纯业务阶段 / 纯业务整窗”的精确占比。
+ +
+

口径与来源

+

泳道 SYS counter 频率：{swimlane["sys_counter_frequency_hz"]:,} Hz；它用于时间戳换算， + 不是约 1.65 GHz 的 PMU cycle 频率。泳道全局 Submit 墙钟范围： + {swimlane["global_submit_makespan"]["duration_us"]:.3f} µs。

+

SYS 边界诊断排除了 linked Vector/Cube Kernel 和 result-used return-ready atomic 的等待区间； + PMU total、Scalar busy 与 primary I-cache 来自嵌在首末 Submit SYS closure 内且遇 linked Kernel + 会暂停的 PMU gate，但 PMU counter 仍保留 atomic 指令及等待事件。阶段原始 PMU 观测使用 + running read-clear total；Scalar busy 独立观测，非 Scalar-busy 残余逐核按 total−scalar 得到。 + 页面再用 empty-bracket 估算进入 phase 分子的局部记录代码并给出参考值；SYS phase tick 只核验 + 边界闭合。PMU/I-cache counter 仍含 atomic 指令及观察代码事件，局部 observed 不是零插桩 + 函数体的数学上下界。

+

本批 PMU 来自 {len(validation["git_heads"])} 组 revision：{head_list}； + 每个业务 phase 与 empty-bracket 必须来自同一场景和同一 revision，才允许计算局部记录开销参考值； + 其他跨 ELF 数值仍只作明示的数量级对照。泳道 raw 没有 build provenance，页面只证明 raw SHA + 与 schema-v4 重算闭合， + 不伪称已证明当时 ELF 身份；泳道与 PMU 之间仅对齐 96 核拓扑和每核 Submit 数。

+
+ +

泳道 ELF：同一份业务时间分布

+

泳道分区展示原始业务 elapsed，不等同于纯 Scalar 时间。Kernel 是父 span 内的嵌套事件； + 当前 analyzer 能精确拆开 EfDrain 与 FinalDrain；其他 containment（本轮包括 WinnerBuild）只有事件数、 + 没有独立 Kernel union 时长。因此纯 Scalar 归因只看下方相应 PMU control ELF，不从泳道父 span 猜减。

+

分区表的 PMU 与 scalar 两列来自对应 phase 自己的独立 ELF。主值的分子是 + “raw phase observed − 按 AIC/AIV 分别估算的局部记录代码开销”，分母仍是该 ELF 的 raw whole + PMU total 或 raw whole scalar；raw 比例作为次要信息保留。它们并非泳道父区间，也不是 + submit-pmu-none，所以各行不得相加；control-only 只代表排除 linked + Kernel 后的控制路径；没有等价 phase 的行显示“—”。SubmitUnion 表专门改用每核均值： + 时间列是平均每核时间，三个占比也分别由同口径的每核均值相除；表尾四层拆分只作跨 ELF + 数量级诊断。

+ {_partition_html(swimlane["submit_envelope_partition"], phase_by_metric)} + {_partition_html( + swimlane["submit_union_partition"], + phase_by_metric, + mean_core_count=submit_union_core_count, + comparison_whole_window=pmu["whole_window"], + comparison_phase_profiles=submit_union_phase_profiles, + )} + {_partition_html(swimlane["efdrain_partition"], phase_by_metric)} + +
外围 Worker / Orchestration / FinalDrain 闭合 + {_partition_html(swimlane["worker_completion_partition"], phase_by_metric)} + {_partition_html(swimlane["orchestration_partition"], phase_by_metric)} + {_partition_html(swimlane["final_drain_partition"], phase_by_metric)} +
+ +

泳道 residual 的业务边界

+
+ + {residual_rows}
边界次数Σ core-timeAIC ticksAIV ticks
+ +

泳道 Kernel containment 与非加和 overlay

+
+
+ {containment_rows}
Kernel 归属事件数
+
+ + {overlay_rows}
Overlay事件数累计 ticks进入分区
+
+ +

Submit-PMU ELF：whole gate 与 SYS 边界诊断

+ {_whole_window_html(pmu["whole_window"])} + + {_synthetic_phase_sum_html(pmu["synthetic_phase_sum_vs_none"])} + +

Submit-PMU ELF：各阶段独立归因

+

每张业务阶段卡都把“扣局部记录代码开销估算后的参考值”放在主位置，并同时列出 raw observed、 + 记录代码开销估算及 raw whole 分母。参考比例仍以该卡所属 ELF 的 raw whole 为分母； + empty-bracket 没有测到整窗全部观察代码，因此不能把它解释成精确业务占比。各阶段来自不同 ELF， + 也不能求和后与 submit-pmu-none 做正式判等；上方合计只用于判断数量级。若某项参考值 + 为负，表示空区间记录估算已经超过 raw observed，只说明该信号低于当前校准分辨能力；页面不截成 + 零，也不把它解释成负的业务事件。

+
{phase_cards}
+ +

观察器校准（不是业务阶段）

+
{calibration}
+ +

泳道区域与 PMU 覆盖矩阵

+
+ {coverage_rows}
泳道区域PMU profile关系说明
+ +

机器可读数据见同目录 {DEFAULT_JSON_NAME}。所有逐核分布只呈现 min–max； + phase extrema 是每核累计完整 phase 的极值，不是单次调用极值。

+
+""" + return "\n".join(line.rstrip() for line in document.splitlines()) + "\n" + + +def _stage_text(path: Path, document: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".pending", dir=path.parent) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(document) + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + temporary.unlink(missing_ok=True) + raise + return temporary + + +def _restore(path: Path, snapshot: bytes | None) -> None: + if snapshot is None: + path.unlink(missing_ok=True) + return + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".restore", dir=path.parent) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(snapshot) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def _publish_pair(json_path: Path, json_document: str, html_path: Path, html_document: str, overwrite: bool) -> None: + if not overwrite and (json_path.exists() or html_path.exists()): + _fail("overview output already exists; pass --overwrite to replace the pair") + previous = { + json_path: json_path.read_bytes() if json_path.is_file() else None, + html_path: html_path.read_bytes() if html_path.is_file() else None, + } + staged_json = _stage_text(json_path, json_document) + try: + staged_html = _stage_text(html_path, html_document) + except BaseException: + staged_json.unlink(missing_ok=True) + raise + published: list[Path] = [] + try: + os.replace(staged_json, json_path) + published.append(json_path) + os.replace(staged_html, html_path) + published.append(html_path) + except BaseException as error: + restore_errors = [] + for path in published: + try: + _restore(path, previous[path]) + except BaseException as restore_error: # pragma: no cover - catastrophic filesystem failure + restore_errors.append(f"{path}: {restore_error}") + if restore_errors: + message = "overview publication failed and rollback was incomplete: " + "; ".join(restore_errors) + raise RuntimeError(message) from error + raise + finally: + staged_json.unlink(missing_ok=True) + staged_html.unlink(missing_ok=True) + + +def _acquire_output_lock(output_dir: Path) -> Path: + output_dir.mkdir(parents=True, exist_ok=True) + lock_path = output_dir / OUTPUT_LOCK_NAME + try: + descriptor = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + except FileExistsError as error: + raise ValueError( + f"another overview publication owns {lock_path}; remove a stale lock only after confirming no writer" + ) from error + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(f"pid={os.getpid()}\n") + stream.flush() + os.fsync(stream.fileno()) + return lock_path + + +def write_overview( + swimlane_raw: Path | str, + pmu_dirs: Sequence[Path | str], + output_dir: Path | str, + *, + overwrite: bool = False, +) -> tuple[Path, Path]: + """Build and publish the fixed JSON/HTML pair with an output lock and rollback.""" + + payload = build_overview(swimlane_raw, pmu_dirs) + json_document = json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + html_document = render_overview(payload) + + swimlane_source = payload["swimlane_elf"]["source"] + swimlane_path = Path(str(swimlane_source["raw_path"])) + if _sha256_file(swimlane_path) != (swimlane_source["raw_size"], swimlane_source["raw_sha256"]): + _fail("swimlane raw changed before overview publication") + pmu_sources = [payload["submit_pmu_elfs"]["whole_window"]["source"]] + pmu_sources.extend(item["source"] for item in payload["submit_pmu_elfs"]["phase_profiles"]) + pmu_sources.append(payload["submit_pmu_elfs"]["calibration"]["source"]) + _assert_pmu_sources_unchanged(pmu_sources) + + output = Path(output_dir) + json_path = output / DEFAULT_JSON_NAME + html_path = output / DEFAULT_HTML_NAME + lock_path = _acquire_output_lock(output) + try: + _publish_pair(json_path, json_document, html_path, html_document, overwrite) + finally: + lock_path.unlink(missing_ok=True) + return json_path, html_path + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--swimlane-raw", required=True, type=Path, help="producer l2_swimlane_records.json") + parser.add_argument( + "--pmu-dir", + required=True, + action="append", + type=Path, + help=f"one directory containing {DEFAULT_INPUT_NAME} and {DEFAULT_PROVENANCE_NAME}; repeat exactly 13 times", + ) + parser.add_argument("--output-dir", required=True, type=Path, help="directory for the fixed JSON/HTML pair") + parser.add_argument("--overwrite", action="store_true", help="replace an existing output pair") + args = parser.parse_args(argv) + try: + json_path, html_path = write_overview( + args.swimlane_raw, + args.pmu_dir, + args.output_dir, + overwrite=args.overwrite, + ) + except (OSError, ValueError) as error: + parser.exit(2, f"error: {error}\n") + print(json_path) + print(html_path) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/simpler_setup/tools/fdwic_swimlane_exclusive_analyzer.py b/simpler_setup/tools/fdwic_swimlane_exclusive_analyzer.py new file mode 100644 index 0000000000..f65c5e7eda --- /dev/null +++ b/simpler_setup/tools/fdwic_swimlane_exclusive_analyzer.py @@ -0,0 +1,572 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Generate a strict integer-cycle exclusive report for production FDWIC schema-v4.""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import sys +import tempfile +from collections import Counter +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +try: + from .fdwic_swimlane_schema import ( + KERNEL_EXECUTION_CHILD_PHASES, + OVERLAY_PHASES, + V4_EXCLUSIVE_SUBMIT_PHASES, + Event, + FdwicV4Model, + find_containing_event, + validate_and_partition_v4, + ) +except ImportError: + from fdwic_swimlane_schema import ( # type: ignore[no-redef] + KERNEL_EXECUTION_CHILD_PHASES, + OVERLAY_PHASES, + V4_EXCLUSIVE_SUBMIT_PHASES, + Event, + FdwicV4Model, + find_containing_event, + validate_and_partition_v4, + ) + + +REPORT_SCHEMA_VERSION = 1 +PHASE_TO_METRIC = { + "EfDrain": "efdrain", + "Materialize": "materialize", + "PrepareMap": "prepare_map", + "Claim": "claim", + "Fanin": "fanin", + "Register": "register", + "WinnerBuild": "winner_build", + "AllocComplete": "alloc_complete", + "LoserReplay": "loser_replay", +} +SUBMIT_PARTITION_METRICS = (*PHASE_TO_METRIC.values(), "submit_residual") +CORE_METRICS = ( + "submit_envelope", + "submit_union", + "between_submit_residual", + *SUBMIT_PARTITION_METRICS, + "efdrain_kernel_union", + "efdrain_control", + "orchestration_replay", + "orchestration_setup", + "orchestration_tail", + "final_drain", + "final_drain_kernel_union", + "final_drain_residual", + "worker_completion", +) + + +def _contains(parent: Event, child: Event) -> bool: + return parent.start_cycle <= child.start_cycle and child.end_cycle <= parent.end_cycle + + +def _interval_union_cycles(intervals: Sequence[tuple[int, int]]) -> int: + if not intervals: + return 0 + ordered = sorted(intervals) + total = 0 + current_start, current_end = ordered[0] + for start, end in ordered[1:]: + if start > current_end: + total += current_end - current_start + current_start, current_end = start, end + else: + current_end = max(current_end, end) + return total + current_end - current_start + + +def _median(values: Sequence[int]) -> int | float: + ordered = sorted(values) + middle = len(ordered) // 2 + if len(ordered) % 2: + return ordered[middle] + total = ordered[middle - 1] + ordered[middle] + return total // 2 if total % 2 == 0 else total / 2 + + +def _distribution(values: Sequence[int]) -> dict[str, int | float]: + ordered = sorted(values) + p95_index = math.ceil(0.95 * len(ordered)) - 1 + return { + "median_cycles": _median(ordered), + "p95_cycles": ordered[p95_index], + "max_cycles": ordered[-1], + } + + +def _classify_kernels(model: FdwicV4Model) -> tuple[dict[int, list[Event]], dict[int, list[Event]], Counter]: + """Assign each Kernel to the smallest exclusive or top-level residual container.""" + + children_by_core: dict[int, list[Event]] = {} + submits_by_core: dict[int, list[Event]] = {} + for core in model.cores: + children_by_core[core.core_id] = sorted( + [child for partition in core.submits for child in partition.children], + key=lambda event: (event.start_cycle, event.end_cycle, event.row_index), + ) + submits_by_core[core.core_id] = [partition.submit for partition in core.submits] + child_starts = {core_id: [event.start_cycle for event in events] for core_id, events in children_by_core.items()} + submit_starts = {core_id: [event.start_cycle for event in events] for core_id, events in submits_by_core.items()} + + kernels_by_child: dict[int, list[Event]] = {} + kernels_by_final_drain: dict[int, list[Event]] = {core.core_id: [] for core in model.cores} + counts: Counter = Counter() + for kernel in model.kernels: + core = model.cores[kernel.core_id] + child = find_containing_event( + kernel, + children_by_core[kernel.core_id], + child_starts[kernel.core_id], + "exclusive child", + ) + if child is not None: + kernels_by_child.setdefault(child.row_index, []).append(kernel) + counts[f"inside_{PHASE_TO_METRIC[child.phase]}_events"] += 1 + continue + submit = find_containing_event( + kernel, + submits_by_core[kernel.core_id], + submit_starts[kernel.core_id], + "Submit", + ) + if submit is not None: + counts["inside_submit_residual_events"] += 1 + elif _contains(core.orchestration, kernel): + # Production orchestration may call tensor-data access helpers, + # which drain executable work between Submit calls. This is valid + # production behavior absent from the standalone probe. + counts["inside_orchestration_residual_events"] += 1 + else: + kernels_by_final_drain[kernel.core_id].append(kernel) + counts["inside_final_drain_events"] += 1 + counts["total_events"] = len(model.kernels) + return kernels_by_child, kernels_by_final_drain, counts + + +def _new_segment() -> dict[str, int]: + return {"event_count": 0, "cycles": 0, "aic_cycles": 0, "aiv_cycles": 0} + + +def _add_segment(segments: dict[str, dict[str, int]], key: str, cycles: int, role: str) -> None: + if cycles <= 0: + return + segment = segments.setdefault(key, _new_segment()) + segment["event_count"] += 1 + segment["cycles"] += cycles + segment[f"{role}_cycles"] += cycles + + +def _ordered_segments(segments: dict[str, dict[str, int]]) -> list[dict[str, Any]]: + return [ + {"boundary": key, **values} + for key, values in sorted(segments.items(), key=lambda item: (-item[1]["cycles"], item[0])) + ] + + +def analyze_data( # noqa: PLR0912, PLR0915 + data: dict[str, Any], input_path: Path | None = None +) -> dict[str, Any]: + """Analyze the validated reader output without converting cycles to float first.""" + + if int(data.get("trace_schema_version", 0)) != 4: + raise ValueError("exclusive FDWIC analysis requires trace_schema_version=4") + events = data.get("fdwic_events") or [] + model = validate_and_partition_v4(events, int(data.get("num_cores", 0)), data.get("core_types") or []) + frequency_hz = int(data.get("clock_freq_hz", 0)) + if frequency_hz <= 0: + raise ValueError("exclusive FDWIC analysis requires a positive clock frequency") + + kernels_by_child, kernels_by_final_drain, kernel_counts = _classify_kernels(model) + internal_segments: dict[str, dict[str, int]] = {} + tail_segments: dict[str, dict[str, int]] = {} + between_segments: dict[str, dict[str, int]] = {} + internal_total = 0 + tail_total = 0 + between_total = 0 + per_core = [] + + for core in model.cores: + metrics = {name: 0 for name in CORE_METRICS} + for partition in core.submits: + submit = partition.submit + child_cycles = 0 + cursor = submit.start_cycle + previous_phase = "SubmitBegin" + for child in partition.children: + duration = child.duration + metrics[PHASE_TO_METRIC[child.phase]] += duration + child_cycles += duration + gap = child.start_cycle - cursor + _add_segment(internal_segments, f"{previous_phase}->{child.phase}", gap, core.role) + internal_total += gap + cursor = child.end_cycle + previous_phase = child.phase + if child.phase == "EfDrain": + kernel_union = _interval_union_cycles( + [(kernel.start_cycle, kernel.end_cycle) for kernel in kernels_by_child.get(child.row_index, [])] + ) + metrics["efdrain_kernel_union"] += kernel_union + metrics["efdrain_control"] += child.duration - kernel_union + tail_gap = submit.end_cycle - cursor + _add_segment(tail_segments, f"{previous_phase}->SubmitEnd", tail_gap, core.role) + tail_total += tail_gap + + residual = submit.duration - child_cycles + if residual < 0: + raise ValueError(f"core {core.core_id} task {submit.task_id} Submit partition is negative") + metrics["submit_union"] += submit.duration + metrics["submit_residual"] += residual + + for previous, current in zip(core.submits, core.submits[1:]): + gap = current.submit.start_cycle - previous.submit.end_cycle + previous_kind = "alloc" if previous.submit.auxiliary else "kernel" + current_kind = "alloc" if current.submit.auxiliary else "kernel" + _add_segment(between_segments, f"{previous_kind}->{current_kind}", gap, core.role) + between_total += gap + + if core.submits: + first_start = core.submits[0].submit.start_cycle + last_end = core.submits[-1].submit.end_cycle + metrics["submit_envelope"] = last_end - first_start + metrics["between_submit_residual"] = sum( + current.submit.start_cycle - previous.submit.end_cycle + for previous, current in zip(core.submits, core.submits[1:]) + ) + metrics["orchestration_setup"] = first_start - core.orchestration.start_cycle + metrics["orchestration_tail"] = core.orchestration.end_cycle - last_end + else: + first_start = None + last_end = None + metrics["orchestration_setup"] = core.orchestration.duration + + metrics["orchestration_replay"] = core.orchestration.duration + final_kernel_union = _interval_union_cycles( + [(kernel.start_cycle, kernel.end_cycle) for kernel in kernels_by_final_drain[core.core_id]] + ) + metrics["final_drain"] = core.final_drain.duration + metrics["final_drain_kernel_union"] = final_kernel_union + metrics["final_drain_residual"] = core.final_drain.duration - final_kernel_union + metrics["worker_completion"] = core.final_drain.end_cycle - core.orchestration.start_cycle + + if sum(metrics[name] for name in SUBMIT_PARTITION_METRICS) != metrics["submit_union"]: + raise ValueError(f"core {core.core_id} aggregate Submit partition does not close") + if metrics["submit_union"] + metrics["between_submit_residual"] != metrics["submit_envelope"]: + raise ValueError(f"core {core.core_id} Submit envelope does not close") + if metrics["efdrain_kernel_union"] + metrics["efdrain_control"] != metrics["efdrain"]: + raise ValueError(f"core {core.core_id} EfDrain partition does not close") + orchestration_sum = ( + metrics["orchestration_setup"] + + metrics["submit_union"] + + metrics["between_submit_residual"] + + metrics["orchestration_tail"] + ) + if orchestration_sum != metrics["orchestration_replay"]: + raise ValueError(f"core {core.core_id} OrchestrationReplay partition does not close") + if metrics["final_drain_kernel_union"] + metrics["final_drain_residual"] != metrics["final_drain"]: + raise ValueError(f"core {core.core_id} FinalDrain partition does not close") + if metrics["orchestration_replay"] + metrics["final_drain"] != metrics["worker_completion"]: + raise ValueError(f"core {core.core_id} WorkerCompletion partition does not close") + + per_core.append( + { + "core_id": core.core_id, + "block_id": core.block_id, + "lane": core.lane, + "role": core.role, + "submit_count": len(core.submits), + "first_submit_start_cycle": first_start, + "last_submit_end_cycle": last_end, + "worker_completion_start_cycle": core.orchestration.start_cycle, + "worker_completion_end_cycle": core.final_drain.end_cycle, + "metrics_cycles": metrics, + } + ) + + aggregate_metrics = {metric: sum(core["metrics_cycles"][metric] for core in per_core) for metric in CORE_METRICS} + submit_partition_sum = sum(aggregate_metrics[name] for name in SUBMIT_PARTITION_METRICS) + orchestration_partition_sum = ( + aggregate_metrics["orchestration_setup"] + + aggregate_metrics["submit_union"] + + aggregate_metrics["between_submit_residual"] + + aggregate_metrics["orchestration_tail"] + ) + final_drain_partition_sum = ( + aggregate_metrics["final_drain_kernel_union"] + aggregate_metrics["final_drain_residual"] + ) + worker_completion_sum = aggregate_metrics["orchestration_replay"] + aggregate_metrics["final_drain"] + if internal_total + tail_total != aggregate_metrics["submit_residual"]: + raise AssertionError("Submit internal + tail residual breakdown does not close") + if between_total != aggregate_metrics["between_submit_residual"]: + raise AssertionError("between-Submit residual breakdown does not close") + + role_statistics = {} + for role in ("aic", "aiv"): + role_cores = [core for core in per_core if core["role"] == role] + if not role_cores: + continue + role_statistics[role] = { + "core_count": len(role_cores), + "metrics": { + metric: _distribution([core["metrics_cycles"][metric] for core in role_cores]) + for metric in CORE_METRICS + }, + } + + all_submits = [partition.submit for core in model.cores for partition in core.submits] + worker_start = min(core.orchestration.start_cycle for core in model.cores) + worker_end = max(core.final_drain.end_cycle for core in model.cores) + global_submit_makespan = None + if all_submits: + submit_start = min(submit.start_cycle for submit in all_submits) + submit_end = max(submit.end_cycle for submit in all_submits) + global_submit_makespan = { + "start_cycle": submit_start, + "end_cycle": submit_end, + "duration_cycles": submit_end - submit_start, + "duration_us": (submit_end - submit_start) * 1_000_000 / frequency_hz, + "semantics": "cross-core wall-clock Submit envelope; not aggregate core-work", + } + + overlays = { + phase: { + **model.overlay_statistics[phase], + "included_in_additive_totals": False, + } + for phase in OVERLAY_PHASES + } + residual_breakdown = { + "submit_internal_residual": { + "total_cycles": internal_total, + "segments": _ordered_segments(internal_segments), + }, + "submit_tail_residual": { + "total_cycles": tail_total, + "segments": _ordered_segments(tail_segments), + }, + "between_submit_residual": { + "total_cycles": between_total, + "segments": _ordered_segments(between_segments), + }, + } + if aggregate_metrics["submit_union"]: + residual_breakdown["submit_internal_residual"]["share_of_submit_union"] = ( + internal_total / aggregate_metrics["submit_union"] + ) + residual_breakdown["submit_tail_residual"]["share_of_submit_union"] = ( + tail_total / aggregate_metrics["submit_union"] + ) + if aggregate_metrics["submit_envelope"]: + residual_breakdown["between_submit_residual"]["share_of_submit_envelope"] = ( + between_total / aggregate_metrics["submit_envelope"] + ) + + return { + "schema_version": REPORT_SCHEMA_VERSION, + "input": str(input_path) if input_path is not None else None, + "capture": { + "trace_schema_version": 4, + "clock_freq_hz": frequency_hz, + "core_count": len(model.cores), + "task_count_per_core": len(model.task_ids), + "event_count": model.event_count, + }, + "validation": { + "status": "PASS", + "dropped_records": 0, + "physical_topology_complete": True, + "task_stream_contiguous_and_equal_per_core": True, + "orchestration_parent_exactly_one_per_core": True, + "final_drain_parent_exactly_one_per_core": True, + "parent_boundaries_adjacent": True, + "legacy_lap_records": 0, + "exclusive_children_non_overlapping": True, + "all_integer_cycle_closures_exact": True, + }, + "semantics": { + "cycle_arithmetic": "raw_integer_cycles", + "exclusive_submit_children": sorted(V4_EXCLUSIVE_SUBMIT_PHASES), + "kernel_execution_submit_children": sorted(KERNEL_EXECUTION_CHILD_PHASES), + "kernel_execution_top_level_residual": "OrchestrationReplay outside Submit, or FinalDrain", + "submit_boundary": ( + "starts after dist_submit_begin and ends at the final timestamp before Submit record publication/return" + ), + "submit_residual": "Submit minus its non-overlapping exclusive children", + "submit_residual_contents": ( + "on compete-first paths, the Claim-to-Materialize gap includes Claim-record publication, synchronous " + "eager callback argument construction, and call handoff; one-shot paths retain their Materialize-first " + "order; other gaps are unmarked control/record overhead; Kernel execution is forbidden" + ), + "orchestration_setup": ( + "OrchestrationReplay.begin to first Submit.begin; includes setup and first dist_submit_begin" + ), + "between_submit_residual": ( + "one Submit.end to the next Submit.begin; includes record/return, orchestration work, " + "and next dist_submit_begin" + ), + "orchestration_tail": ( + "last Submit.end to OrchestrationReplay.end; includes final record/return and orchestration epilogue" + ), + "worker_completion_boundary": ( + "post-startup OrchestrationReplay.begin through FinalDrain.end; excludes startup and post-window " + "clock baselines, trace flush, and finish publication" + ), + "orchestration_children": [ + "OrchestrationSetup", + "SubmitUnion", + "BetweenSubmitResidual", + "OrchestrationTail", + ], + "final_drain_children": ["KernelUnion", "FinalDrainResidual"], + "worker_completion_children": ["OrchestrationReplay", "FinalDrain"], + "legacy_lap_phases_forbidden": ["Alloc", "Build", "Replay"], + "drain_won": "real nested BlockWon action; retained as a non-additive overlay", + "loser_replay": "real kernel-loser drain_block_won call; exclusive Submit child", + "alloc_loser_tail": "no fabricated action; remains Submit tail residual", + "overlays_are_additive": False, + "p95_method": "nearest_rank", + }, + "global_worker_completion_makespan": { + "start_cycle": worker_start, + "end_cycle": worker_end, + "duration_cycles": worker_end - worker_start, + "duration_us": (worker_end - worker_start) * 1_000_000 / frequency_hz, + "semantics": "cross-core wall-clock envelope; not aggregate core-work", + }, + "global_submit_makespan": global_submit_makespan, + "aggregate_core_work": { + "metrics_cycles": aggregate_metrics, + "closure": { + "submit_partition": { + "parent_cycles": aggregate_metrics["submit_union"], + "children_plus_residual_cycles": submit_partition_sum, + "exact": submit_partition_sum == aggregate_metrics["submit_union"], + }, + "submit_envelope": { + "parent_cycles": aggregate_metrics["submit_envelope"], + "submit_union_plus_between_cycles": ( + aggregate_metrics["submit_union"] + aggregate_metrics["between_submit_residual"] + ), + "exact": True, + }, + "efdrain_partition": { + "parent_cycles": aggregate_metrics["efdrain"], + "kernel_union_plus_control_cycles": ( + aggregate_metrics["efdrain_kernel_union"] + aggregate_metrics["efdrain_control"] + ), + "exact": True, + }, + "orchestration_replay": { + "parent_cycles": aggregate_metrics["orchestration_replay"], + "setup_submit_union_between_tail_cycles": orchestration_partition_sum, + "exact": orchestration_partition_sum == aggregate_metrics["orchestration_replay"], + }, + "final_drain": { + "parent_cycles": aggregate_metrics["final_drain"], + "kernel_union_plus_residual_cycles": final_drain_partition_sum, + "exact": final_drain_partition_sum == aggregate_metrics["final_drain"], + }, + "worker_completion": { + "parent_cycles": aggregate_metrics["worker_completion"], + "orchestration_plus_final_drain_cycles": worker_completion_sum, + "exact": worker_completion_sum == aggregate_metrics["worker_completion"], + }, + }, + "semantics": "sum of per-core cycles; not wall-clock duration", + }, + "residual_breakdown": residual_breakdown, + "per_role_core_statistics": role_statistics, + "kernel_containment": dict(kernel_counts), + "overlays": overlays, + "per_core": per_core, + } + + +def analyze_capture(input_path: Path) -> dict[str, Any]: + try: + from .swimlane_converter import read_perf_data # noqa: PLC0415 + except ImportError: + from swimlane_converter import read_perf_data # type: ignore[no-redef] # noqa: PLC0415 + + input_path = Path(input_path) + return analyze_data(read_perf_data(input_path), input_path) + + +def write_analysis_data(data: dict[str, Any], input_path: Path, output_path: Path) -> Path: + output_path = Path(output_path) + input_path = Path(input_path) + if input_path.resolve() == output_path.resolve(): + raise ValueError("exclusive analysis output must differ from the raw input") + document = json.dumps(analyze_data(data, input_path), ensure_ascii=False, indent=2) + "\n" + output_path.parent.mkdir(parents=True, exist_ok=True) + temporary_name: str | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix=f".{output_path.name}.", + suffix=".tmp", + dir=output_path.parent, + delete=False, + ) as temporary: + temporary.write(document) + temporary.flush() + os.fsync(temporary.fileno()) + os.fchmod(temporary.fileno(), 0o644) + temporary_name = temporary.name + os.replace(temporary_name, output_path) + finally: + if temporary_name is not None: + Path(temporary_name).unlink(missing_ok=True) + return output_path + + +def write_analysis(input_path: Path, output_path: Path) -> Path: + try: + from .swimlane_converter import read_perf_data # noqa: PLC0415 + except ImportError: + from swimlane_converter import read_perf_data # type: ignore[no-redef] # noqa: PLC0415 + + input_path = Path(input_path) + return write_analysis_data(read_perf_data(input_path), input_path, output_path) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path, help="schema-v4 l2_swimlane_records.json") + parser.add_argument( + "-o", + "--output", + type=Path, + help="output path (default: /swimlane_exclusive_analysis.json)", + ) + arguments = parser.parse_args(argv) + output = arguments.output or arguments.input.parent / "swimlane_exclusive_analysis.json" + try: + write_analysis(arguments.input, output) + except (OSError, ValueError) as error: + print(f"exclusive FDWIC swimlane analysis failed: {error}", file=sys.stderr) + return 1 + print(f"[SWIMLANE-EXCLUSIVE] input={arguments.input} output={output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/simpler_setup/tools/fdwic_swimlane_schema.py b/simpler_setup/tools/fdwic_swimlane_schema.py new file mode 100644 index 0000000000..ecc7217ab6 --- /dev/null +++ b/simpler_setup/tools/fdwic_swimlane_schema.py @@ -0,0 +1,470 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Shared schema-v4 semantics for production FDWIC swimlane tooling.""" + +from __future__ import annotations + +import bisect +from collections import Counter, defaultdict +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from typing import Any + +LEGACY_LAP_PHASES = frozenset({"Alloc", "Build", "Replay"}) +V4_PHASES = frozenset( + { + "OrchestrationReplay", + "FinalDrain", + "WinnerBuild", + "AllocComplete", + "LoserReplay", + } +) +V4_EXCLUSIVE_SUBMIT_PHASES = frozenset( + { + "EfDrain", + "Materialize", + "PrepareMap", + "Claim", + "Fanin", + "Register", + "WinnerBuild", + "AllocComplete", + "LoserReplay", + } +) +KERNEL_EXECUTION_CHILD_PHASES = frozenset({"EfDrain", "WinnerBuild", "AllocComplete"}) +REQUIRED_SUBMIT_PHASES = ("EfDrain", "Materialize", "PrepareMap", "Claim", "Register") +OVERLAY_PHASES = ("Atomic", "ClockBaseline", "Commit", "RingBp", "DrainWon") + +_MODEL_PHASES = V4_EXCLUSIVE_SUBMIT_PHASES | { + "Submit", + "OrchestrationReplay", + "FinalDrain", + "Kernel", +} + + +@dataclass(frozen=True, slots=True) +class Event: + """Integer-cycle view of one already row-validated FDWIC event.""" + + row_index: int + core_id: int + block_id: int + lane: int + task_id: int + function_id: int + phase: str + start_cycle: int + end_cycle: int + flags: int + auxiliary: int + + @classmethod + def from_mapping(cls, row_index: int, event: dict[str, Any]) -> Event: + return cls( + row_index=row_index, + core_id=int(event["core_id"]), + block_id=int(event["block_id"]), + lane=int(event["lane"]), + task_id=int(event["task_id"]), + function_id=int(event["func_id"]), + phase=str(event["phase"]), + start_cycle=int(event["start_cycles"]), + end_cycle=int(event["end_cycles"]), + flags=int(event["flags"]), + auxiliary=int(event["aux"]), + ) + + @property + def duration(self) -> int: + return self.end_cycle - self.start_cycle + + @property + def lane_key(self) -> tuple[int, int]: + return self.core_id, self.lane + + +@dataclass(frozen=True, slots=True) +class SubmitPartition: + submit: Event + children: tuple[Event, ...] + + +@dataclass(frozen=True, slots=True) +class CorePartition: + core_id: int + block_id: int + lane: int + role: str + orchestration: Event + final_drain: Event + submits: tuple[SubmitPartition, ...] + + +@dataclass(frozen=True, slots=True) +class FdwicV4Model: + cores: tuple[CorePartition, ...] + kernels: tuple[Event, ...] + overlay_statistics: dict[str, dict[str, int]] + event_count: int + task_ids: tuple[int, ...] + + +@dataclass(frozen=True, slots=True) +class ResidualSpan: + core_id: int + block_id: int + lane: int + start_cycle: int + end_cycle: int + name: str + + +def _contains(parent: Event, child: Event) -> bool: + return parent.start_cycle <= child.start_cycle and child.end_cycle <= parent.end_cycle + + +def _overlaps(left: Event, right: Event) -> bool: + return max(left.start_cycle, right.start_cycle) < min(left.end_cycle, right.end_cycle) + + +def find_containing_event( + event: Event, + parents: Sequence[Event], + starts: Sequence[int], + container_name: str, +) -> Event | None: + """Find the containing interval and reject a partial boundary crossing.""" + + candidate = bisect.bisect_right(starts, event.start_cycle) - 1 + if candidate >= 0: + parent = parents[candidate] + if _contains(parent, event): + return parent + if _overlaps(parent, event): + raise ValueError(f"row {event.row_index} {event.phase} crosses {container_name} row {parent.row_index}") + next_candidate = candidate + 1 + if next_candidate < len(parents) and _overlaps(parents[next_candidate], event): + raise ValueError( + f"row {event.row_index} {event.phase} crosses {container_name} row {parents[next_candidate].row_index}" + ) + return None + + +def _expected_layout(core_types: Sequence[str]) -> tuple[tuple[int, int, str], ...]: + """Mirror the host's dynamic AIC + two-AIV-per-block topology contract.""" + + layout: list[tuple[int, int, str] | None] = [None] * len(core_types) + aic_count = 0 + for core_id, role in enumerate(core_types): + if role == "aic": + layout[core_id] = (aic_count, 0, role) + aic_count += 1 + elif role != "aiv": + raise ValueError(f"metadata.core_types[{core_id}] has invalid role {role!r}") + if aic_count == 0: + raise ValueError("schema-v4 FDWIC topology has no AIC core") + + aiv_ordinal = 0 + for core_id, role in enumerate(core_types): + if role != "aiv": + continue + block_id, lane = divmod(aiv_ordinal, 2) + if block_id >= aic_count: + raise ValueError( + f"schema-v4 AIV core {core_id} cannot map to an AIC block: aic={aic_count} aiv_ordinal={aiv_ordinal}" + ) + layout[core_id] = (block_id, lane + 1, role) + aiv_ordinal += 1 + if aiv_ordinal != 2 * aic_count: + raise ValueError( + f"schema-v4 FDWIC topology requires exactly two AIV cores per AIC: aic={aic_count} aiv={aiv_ordinal}" + ) + return tuple(item for item in layout if item is not None) + + +def _validate_submit_semantics(partition: SubmitPartition) -> None: # noqa: PLR0912 + submit = partition.submit + children = partition.children + counts = Counter(child.phase for child in children) + for phase in REQUIRED_SUBMIT_PHASES: + if counts[phase] != 1: + raise ValueError( + f"core {submit.core_id} task {submit.task_id} requires exactly one {phase}, got {counts[phase]}" + ) + if any(count > 1 for count in counts.values()): + duplicates = sorted(phase for phase, count in counts.items() if count > 1) + raise ValueError(f"core {submit.core_id} task {submit.task_id} has duplicate exclusive phases {duplicates}") + + is_winner = bool(submit.flags & 1) + is_alloc = bool(submit.auxiliary) + expected_sequences: list[list[str]] + if is_alloc: + compete_first = ["EfDrain", "Claim", "Materialize", "PrepareMap", "Register"] + one_shot = ["EfDrain", "Materialize", "PrepareMap", "Register", "Claim"] + if is_winner: + compete_first.append("AllocComplete") + one_shot.append("AllocComplete") + else: + compete_first = ["EfDrain", "Claim", "Materialize", "PrepareMap"] + one_shot = ["EfDrain", "Materialize", "PrepareMap", "Claim"] + if is_winner: + compete_first.append("Fanin") + one_shot.append("Fanin") + compete_first.append("Register") + one_shot.append("Register") + tail = "WinnerBuild" if is_winner else "LoserReplay" + compete_first.append(tail) + one_shot.append(tail) + expected_sequences = [compete_first, one_shot] + actual_sequence = [child.phase for child in children] + if actual_sequence not in expected_sequences: + raise ValueError( + f"core {submit.core_id} task {submit.task_id} has invalid exclusive sequence: " + f"expected_one_of={expected_sequences} actual={actual_sequence}" + ) + + claim = next(child for child in children if child.phase == "Claim") + claim_won = bool(claim.flags & 1) + claim_attempted = bool(claim.flags & 2) + if claim_won and not claim_attempted: + raise ValueError(f"core {submit.core_id} task {submit.task_id} Claim won without an attempt") + if claim_won != is_winner or bool(claim.auxiliary) != is_alloc: + raise ValueError(f"core {submit.core_id} task {submit.task_id} Submit/Claim semantics disagree") + + if submit.function_id != claim.function_id: + raise ValueError(f"core {submit.core_id} task {submit.task_id} Submit/Claim function IDs disagree") + if is_alloc: + if submit.function_id != -1: + raise ValueError(f"core {submit.core_id} alloc task {submit.task_id} must use function_id=-1") + elif is_winner: + if submit.function_id < 0: + raise ValueError(f"core {submit.core_id} kernel winner task {submit.task_id} lacks a function ID") + for child in children: + if child.phase in {"Fanin", "Register", "WinnerBuild"} and child.function_id != submit.function_id: + raise ValueError( + f"core {submit.core_id} task {submit.task_id} {child.phase} function ID disagrees with Submit" + ) + elif submit.function_id != -1: + raise ValueError(f"core {submit.core_id} kernel loser task {submit.task_id} must use function_id=-1") + + +def validate_and_partition_v4( # noqa: PLR0912 + fdwic_events: Sequence[dict[str, Any]], + num_cores: int, + core_types: Sequence[str], +) -> FdwicV4Model: + """Validate production schema-v4 hierarchy and return its shared partition model.""" + + if num_cores <= 0 or len(core_types) != num_cores: + raise ValueError( + "schema-v4 requires metadata.num_cores to match metadata.core_types: " + f"num_cores={num_cores} core_types={len(core_types)}" + ) + expected_layout = _expected_layout(core_types) + parents: dict[int, dict[str, list[Event]]] = { + core_id: {"OrchestrationReplay": [], "FinalDrain": []} for core_id in range(num_cores) + } + submits_by_core: dict[int, list[Event]] = defaultdict(list) + children_by_key: dict[tuple[int, int], list[Event]] = defaultdict(list) + kernels: list[Event] = [] + overlay_statistics = {phase: {"event_count": 0, "aggregate_duration_cycles": 0} for phase in OVERLAY_PHASES} + + for row_index, raw_event in enumerate(fdwic_events): + core_id = int(raw_event["core_id"]) + if not 0 <= core_id < num_cores: + raise ValueError(f"fdwic event {row_index} has invalid core_id {core_id}") + block_id, lane, _role = expected_layout[core_id] + if int(raw_event["block_id"]) != block_id or int(raw_event["lane"]) != lane: + raise ValueError( + f"fdwic event {row_index} has invalid physical identity for core {core_id}: " + f"block/lane={raw_event['block_id']}/{raw_event['lane']} expected={block_id}/{lane}" + ) + phase = str(raw_event["phase"]) + duration = int(raw_event["end_cycles"]) - int(raw_event["start_cycles"]) + if phase in overlay_statistics: + overlay_statistics[phase]["event_count"] += 1 + overlay_statistics[phase]["aggregate_duration_cycles"] += duration + if phase not in _MODEL_PHASES: + continue + + event = Event.from_mapping(row_index, raw_event) + if phase in ("OrchestrationReplay", "FinalDrain"): + parents[core_id][phase].append(event) + elif phase == "Submit": + submits_by_core[core_id].append(event) + elif phase in V4_EXCLUSIVE_SUBMIT_PHASES: + children_by_key[(core_id, event.task_id)].append(event) + elif phase == "Kernel": + kernels.append(event) + + core_partitions: list[CorePartition] = [] + reference_task_ids: tuple[int, ...] | None = None + consumed_child_keys: set[tuple[int, int]] = set() + for core_id in range(num_cores): + block_id, lane, role = expected_layout[core_id] + orchestration_rows = parents[core_id]["OrchestrationReplay"] + final_drain_rows = parents[core_id]["FinalDrain"] + if len(orchestration_rows) != 1 or len(final_drain_rows) != 1: + raise ValueError( + f"core {core_id} requires exactly one OrchestrationReplay and FinalDrain: " + f"orchestration={len(orchestration_rows)} final_drain={len(final_drain_rows)}" + ) + orchestration = orchestration_rows[0] + final_drain = final_drain_rows[0] + if orchestration.duration < 0 or final_drain.duration < 0: + raise ValueError(f"core {core_id} has a negative schema-v4 parent duration") + if orchestration.end_cycle != final_drain.start_cycle: + raise ValueError(f"core {core_id} OrchestrationReplay.end must equal FinalDrain.start") + + submits = sorted( + submits_by_core.get(core_id, []), + key=lambda event: (event.start_cycle, event.end_cycle, event.row_index), + ) + for previous, current in zip(submits, submits[1:]): + if _overlaps(previous, current): + raise ValueError( + f"core {core_id} has overlapping Submit rows {previous.row_index} and {current.row_index}" + ) + task_ids = tuple(submit.task_id for submit in submits) + if len(task_ids) != len(set(task_ids)): + raise ValueError(f"core {core_id} has duplicate Submit task IDs") + if task_ids and task_ids != tuple(range(task_ids[-1] + 1)): + raise ValueError(f"core {core_id} Submit task IDs are not contiguous 0..N-1: {task_ids}") + if reference_task_ids is None: + reference_task_ids = task_ids + elif task_ids != reference_task_ids: + raise ValueError(f"core {core_id} Submit task IDs do not match the common replay stream") + + submit_partitions: list[SubmitPartition] = [] + for submit in submits: + if submit.duration <= 0 or not _contains(orchestration, submit): + raise ValueError( + f"core {core_id} task {submit.task_id} Submit must be positive and contained by OrchestrationReplay" + ) + key = (core_id, submit.task_id) + consumed_child_keys.add(key) + children = sorted( + children_by_key.get(key, []), + key=lambda event: (event.start_cycle, event.end_cycle, event.row_index), + ) + for child in children: + if not _contains(submit, child): + raise ValueError( + f"row {child.row_index} {child.phase} is outside core {core_id} task {submit.task_id} Submit" + ) + for previous, current in zip(children, children[1:]): + if _overlaps(previous, current): + raise ValueError( + f"core {core_id} task {submit.task_id} has overlapping exclusive children " + f"rows {previous.row_index} and {current.row_index}" + ) + partition = SubmitPartition(submit=submit, children=tuple(children)) + _validate_submit_semantics(partition) + submit_partitions.append(partition) + + core_partitions.append( + CorePartition( + core_id=core_id, + block_id=block_id, + lane=lane, + role=role, + orchestration=orchestration, + final_drain=final_drain, + submits=tuple(submit_partitions), + ) + ) + + orphan_child_keys = set(children_by_key) - consumed_child_keys + if orphan_child_keys: + raise ValueError(f"schema-v4 exclusive children have no matching Submit: {sorted(orphan_child_keys)[:8]}") + + exclusive_by_core = { + core.core_id: sorted( + [child for partition in core.submits for child in partition.children], + key=lambda event: (event.start_cycle, event.end_cycle, event.row_index), + ) + for core in core_partitions + } + submits_by_core = {core.core_id: [partition.submit for partition in core.submits] for core in core_partitions} + exclusive_starts_by_core = { + core_id: [event.start_cycle for event in events] for core_id, events in exclusive_by_core.items() + } + submit_starts_by_core = { + core_id: [event.start_cycle for event in events] for core_id, events in submits_by_core.items() + } + for kernel in kernels: + core = core_partitions[kernel.core_id] + inside_orchestration = _contains(core.orchestration, kernel) + inside_final_drain = _contains(core.final_drain, kernel) + if inside_orchestration == inside_final_drain: + raise ValueError(f"row {kernel.row_index} Kernel must be contained by exactly one top-level parent") + exclusive = exclusive_by_core[kernel.core_id] + child = find_containing_event( + kernel, + exclusive, + exclusive_starts_by_core[kernel.core_id], + "exclusive child", + ) + submits = submits_by_core[kernel.core_id] + submit = find_containing_event( + kernel, + submits, + submit_starts_by_core[kernel.core_id], + "Submit", + ) + if inside_orchestration and child is not None and child.phase not in KERNEL_EXECUTION_CHILD_PHASES: + raise ValueError(f"row {kernel.row_index} Kernel has unsupported exclusive container {child.phase}") + if inside_orchestration and submit is not None and child is None: + raise ValueError(f"row {kernel.row_index} Kernel is inside Submit residual") + + return FdwicV4Model( + cores=tuple(core_partitions), + kernels=tuple(kernels), + overlay_statistics=overlay_statistics, + event_count=len(fdwic_events), + task_ids=reference_task_ids or (), + ) + + +def iter_v4_residual_spans(model: FdwicV4Model) -> Iterator[ResidualSpan]: + """Yield the exact complement of exclusive children and adjacent Submits.""" + + for core in model.cores: + for previous, current in zip(core.submits, core.submits[1:]): + start = previous.submit.end_cycle + end = current.submit.start_cycle + if end > start: + yield ResidualSpan(core.core_id, core.block_id, core.lane, start, end, "between_submit_residual") + + for partition in core.submits: + submit = partition.submit + cursor = submit.start_cycle + for child in partition.children: + if child.start_cycle > cursor: + yield ResidualSpan( + core.core_id, + core.block_id, + core.lane, + cursor, + child.start_cycle, + "submit_residual", + ) + cursor = child.end_cycle + if submit.end_cycle > cursor: + yield ResidualSpan( + core.core_id, + core.block_id, + core.lane, + cursor, + submit.end_cycle, + "submit_tail_gap", + ) diff --git a/simpler_setup/tools/swimlane_converter.py b/simpler_setup/tools/swimlane_converter.py index e25d7f2df8..3aa94b002c 100644 --- a/simpler_setup/tools/swimlane_converter.py +++ b/simpler_setup/tools/swimlane_converter.py @@ -31,6 +31,157 @@ from pathlib import Path from typing import Any +try: + from .fdwic_swimlane_exclusive_analyzer import write_analysis_data + from .fdwic_swimlane_schema import ( + LEGACY_LAP_PHASES, + V4_PHASES, + iter_v4_residual_spans, + validate_and_partition_v4, + ) +except ImportError: + # Preserve direct-script use from the tools directory. + from fdwic_swimlane_exclusive_analyzer import write_analysis_data # type: ignore[no-redef] + from fdwic_swimlane_schema import ( # type: ignore[no-redef] + LEGACY_LAP_PHASES, + V4_PHASES, + iter_v4_residual_spans, + validate_and_partition_v4, + ) + +_FDWIC_PHASE_NAMES = { + "Kernel": "kernel", + "Alloc": "alloc", + "Build": "build", + "DrainWon": "drain_won", + "Replay": "replay", + "RingBp": "ringbp", + "EfDrain": "efdrain", + "Commit": "commit", + "Submit": "submit", + "Materialize": "materialize", + "PrepareMap": "prepare_map", + "Claim": "claim", + "Fanin": "fanin", + "Register": "register", + "Atomic": "atomic", + "ClockBaseline": "clock_baseline", + "OrchestrationReplay": "orchestration_replay", + "FinalDrain": "final_drain", + "WinnerBuild": "winner_build", + "AllocComplete": "alloc_complete", + "LoserReplay": "loser_replay", +} + +# IDs 0..14 are the standalone PA ABI. Real PA only appends IDs so archived +# captures and the standalone calibration keep the same names. +_FDWIC_ATOMIC_SITE_NAMES = { + 0: "startup_increment", + 1: "startup_poll", + 2: "fatal_poll", + 3: "fatal_set", + 4: "claim_max", + 5: "fanin_flag_load", + 6: "completion_vend_exchange", + 7: "completion_flag_exchange", + 8: "frontier_initial_load", + 9: "frontier_flag_load", + 10: "frontier_max", + 11: "heap_frontier_load", + 12: "heap_vend_load", + 13: "replay_done_increment", + 14: "replay_done_poll", + 15: "won_slot_claim_max", + 16: "won_remaining_exchange", + 17: "won_lane_reset_exchange", + 18: "won_lane_deposit_exchange", + 19: "won_state_publish_exchange", + 20: "won_any_publish_exchange", + 21: "won_any_load", + 22: "won_state_load", + 23: "won_lane_claim_exchange", + 24: "won_lane_release_exchange", + 25: "won_remaining_fetch_sub", + 26: "won_state_clear_exchange", + 27: "won_drained_load", +} + +_FDWIC_ATOMIC_OP_NAMES = { + 0: "load", + 1: "exchange", + 2: "fetch_add", + 3: "fetch_max", + 4: "fetch_sub", +} + +_FDWIC_ATOMIC_SITE_OP_IDS = { + 0: 2, + 1: 0, + 2: 0, + 3: 1, + 4: 3, + 5: 0, + 6: 1, + 7: 1, + 8: 0, + 9: 0, + 10: 3, + 11: 0, + 12: 0, + 13: 2, + 14: 0, + 15: 3, + 16: 1, + 17: 1, + 18: 1, + 19: 1, + 20: 1, + 21: 0, + 22: 0, + 23: 1, + 24: 1, + 25: 4, + 26: 1, + 27: 0, +} + +# These sites issue the atomic for its side effect and do not consume the old +# value. All other production sites consume the result. A v3 converter must +# mirror this ABI instead of trusting a self-consistent summary around a +# malformed direct Atomic row. +_FDWIC_ATOMIC_RESULT_UNUSED_SITE_IDS = {0, 3, 6, 7, 13, 16, 17, 18, 19, 20, 24, 26} + +_FDWIC_POLL_BATCH_SITE_OP_IDS = {1: 0, 2: 0, 5: 0, 11: 0, 12: 0, 14: 0, 21: 0, 22: 0, 23: 1, 27: 0} + + +def _validate_fdwic_v4_phase_fields(row_index, task_id, func_id, phase, flags, aux): + """Mirror the production host's non-atomic schema-v4 row contract.""" + + if phase in {"Kernel", "Commit"}: + valid = flags <= 1 and aux == 0 + elif phase == "DrainWon": + valid = flags == 1 and aux < 4 + elif phase == "RingBp": + valid = flags == 0 and aux <= 1 + elif phase == "Submit": + valid = task_id >= 0 and flags <= 1 and aux <= 1 + elif phase in {"Materialize", "PrepareMap", "Register"}: + valid = task_id >= 0 and flags == 0 and aux <= 1 + elif phase == "Fanin": + valid = task_id >= 0 and flags == 0 and aux <= 16 + elif phase in {"EfDrain", "WinnerBuild", "AllocComplete", "LoserReplay"}: + valid = task_id >= 0 and flags == 0 and aux == 0 + elif phase in {"OrchestrationReplay", "FinalDrain"}: + valid = task_id == -1 and func_id == -1 and flags == 0 and aux == 0 + else: + # Claim, Atomic, and ClockBaseline have dedicated validation below. + return + if not valid: + raise ValueError( + f"fdwic_events[{row_index}] has invalid schema-v4 {phase} fields: " + f"task={task_id} func={func_id} flags=0x{flags:x} aux={aux}" + ) + def _func_id_to_letter(func_id): """Map a non-negative integer func_id to a numeric+letter label. @@ -70,7 +221,15 @@ def _task_display_name(func_id, func_id_to_name, tdisp): return f"func_{_func_id_to_letter(func_id)}({tdisp})" -def _append_fdwic_dist_engine_events(events, fdwic_events, func_id_to_name=None): +def _append_fdwic_dist_engine_events( # noqa: PLR0912, PLR0915 + events, + fdwic_events, + func_id_to_name=None, + trace_schema_version=1, + clock_freq_hz=0, + num_cores=0, + core_types=None, +): """Append fully_distributed_within_core AICore-runtime spans. The visual shape follows the a2a3 dist_engine swimlane where it applies on @@ -84,28 +243,33 @@ def lane_name(lane): return {0: "AIC", 1: "AIV0", 2: "AIV1"}.get(int(lane), "?") def phase_name(phase): - return { - "Kernel": "kernel", - "Alloc": "alloc", - "Build": "build", - "DrainWon": "drain_won", - "Replay": "replay", - "RingBp": "ringbp", - "EfDrain": "efdrain", - "Commit": "commit", - "Submit": "submit", - "Materialize": "materialize", - "PrepareMap": "prepare_map", - "Claim": "claim", - "Fanin": "fanin", - "Register": "register", - }.get(str(phase), str(phase).lower()) + return _FDWIC_PHASE_NAMES.get(str(phase), str(phase).lower()) def kernel_name(func_id): if func_id_to_name: return func_id_to_name.get(str(func_id), func_id_to_name.get(func_id, f"f{func_id}")) return f"f{func_id}" + v4_model = None + residual_factor = 0.0 + cycle_time_us = {} + if trace_schema_version == 4: + if clock_freq_hz <= 0: + raise ValueError("schema-v4 FDWIC conversion requires a positive clock frequency") + if not num_cores: + core_ids = {int(event["core_id"]) for event in fdwic_events} + num_cores = max(core_ids, default=-1) + 1 + if core_types is None: + inferred_roles = {} + for event in fdwic_events: + inferred_roles.setdefault(int(event["core_id"]), "aic" if int(event["lane"]) == 0 else "aiv") + core_types = [inferred_roles.get(core_id, "unknown") for core_id in range(num_cores)] + v4_model = validate_and_partition_v4(fdwic_events, int(num_cores), list(core_types)) + residual_factor = 1_000_000.0 / float(clock_freq_hz) + for event in fdwic_events: + cycle_time_us.setdefault(int(event["start_cycles"]), float(event["start_time_us"])) + cycle_time_us.setdefault(int(event["end_cycles"]), float(event["end_time_us"])) + blocks = sorted({int(e["block_id"]) for e in fdwic_events if int(e["block_id"]) >= 0}) core_by_block_lane = {} for e in fdwic_events: @@ -139,37 +303,185 @@ def kernel_name(func_id): } ) + legacy_claim_max_spans = defaultdict(list) + has_atomic_trace = False + if trace_schema_version == 1: + for event in fdwic_events: + if phase_name(event["phase"]) != "atomic": + continue + has_atomic_trace = True + if int(event["aux"]) == 4: + key = (int(event["core_id"]), int(event["block_id"]), int(event["lane"]), int(event["task_id"])) + legacy_claim_max_spans[key].append((int(event["start_cycles"]), int(event["end_cycles"]))) + for e in fdwic_events: phase = phase_name(e["phase"]) func_id = int(e["func_id"]) task_id = int(e["task_id"]) lane = int(e["lane"]) - if phase == "kernel" and func_id >= 0: + flags = int(e["flags"]) + aux = int(e["aux"]) + if phase == "claim": + claim_won = bool(flags & 0x1) + if trace_schema_version >= 2: + claim_attempted = bool(flags & 0x2) + claim_attempted_source = "raw_flag" + elif has_atomic_trace: + key = (int(e["core_id"]), int(e["block_id"]), lane, task_id) + matched_claim_max = any( + atomic_start >= int(e["start_cycles"]) and atomic_end <= int(e["end_cycles"]) + for atomic_start, atomic_end in legacy_claim_max_spans.get(key, []) + ) + claim_attempted = True if matched_claim_max else None + claim_attempted_source = ( + "contained_claim_max" if matched_claim_max else "unknown_v1_without_matching_claim_max" + ) + else: + claim_attempted = None + claim_attempted_source = "unknown_v1_without_atomic_trace" + if claim_attempted is False: + name = f"claim.not_attempted#{task_id}" + elif claim_attempted is True: + name = f"claim.{'won' if claim_won else 'lost'}#{task_id}" + else: + name = f"claim#{task_id}" + tid = lane + elif phase == "atomic": + atomic_site_id = aux + atomic_op_id = flags & 0xF + atomic_site = _FDWIC_ATOMIC_SITE_NAMES.get(atomic_site_id, f"site_{atomic_site_id}") + atomic_op = _FDWIC_ATOMIC_OP_NAMES.get(atomic_op_id, f"op_{atomic_op_id}") + atomic_poll_batch = trace_schema_version >= 3 and bool(flags & (1 << 7)) + if atomic_poll_batch: + atomic_call_count = (flags >> 8) & 0xFFFFFF + name = f"atomic.poll_batch.{atomic_site}.{atomic_op}×{atomic_call_count}" + else: + atomic_boundary_tag = "return_ready" if flags & (1 << 6) else "source_issue" + name = f"atomic.{atomic_boundary_tag}.{atomic_site}.{atomic_op}#{task_id}" + tid = lane + elif phase == "clock_baseline": + name = "clock.atomic_return_dependency_hook" if flags & 1 else "clock.consecutive_sys_cnt_reads" + tid = lane + elif phase == "kernel" and func_id >= 0: name = f"{kernel_name(func_id)}#{task_id}" tid = lane + 3 elif phase == "commit": name = f"{phase}#{task_id}" tid = lane + 3 + elif phase in ("orchestration_replay", "final_drain"): + name = phase + tid = lane else: name = f"{phase}#{task_id}" tid = lane - events.append( - { - "ph": "X", - "name": name, - "pid": int(e["block_id"]), - "tid": tid, - "ts": round(float(e["start_time_us"]), 3), - "dur": round(float(e["duration_us"]), 3), - "args": { + event = { + "ph": "X", + "name": name, + "pid": int(e["block_id"]), + "tid": tid, + "ts": round(float(e["start_time_us"]), 3), + "dur": round(float(e["duration_us"]), 3), + "args": { + "phase": phase, + "task_id": task_id, + "func_id": func_id, + "core": int(e["core_id"]), + "mc": flags & 1, + }, + } + if phase == "atomic": + if atomic_poll_batch: + event["args"] = { + "phase": "atomic_poll_batch", + "task_id": task_id, + "func_id": func_id, + "core": int(e["core_id"]), + "site": atomic_site, + "site_id": atomic_site_id, + "op": atomic_op, + "op_id": atomic_op_id, + "call_count": atomic_call_count, + "poll_window_cycles": int(e["end_cycles"]) - int(e["start_cycles"]), + "estimate_formula": "call_count * calibrated_atomic_cost", + "is_poll_batch": True, + "batch_semantics": ( + "idempotent_failed_exchange_retries" if atomic_site_id == 23 else "observation_load_calls" + ), + "duration_semantics": "logical_poll_episode_envelope_not_single_atomic_latency", + "may_contain_interleaved_direct_atomics": True, + "flags": flags, + "execution_unit": "scalar", + } + event["cat"] = "atomic.poll_batch" + else: + event["args"] = { "phase": phase, "task_id": task_id, "func_id": func_id, "core": int(e["core_id"]), - "mc": int(e["flags"]) & 1, - }, + "site": atomic_site, + "site_id": atomic_site_id, + "op": atomic_op, + "op_id": atomic_op_id, + "call_count": 1, + "cycles": int(e["end_cycles"]) - int(e["start_cycles"]), + "result_used": bool(flags & (1 << 4)), + "return_ready_observed": bool(flags & (1 << 6)), + "completion_boundary": "return_value_ready" if flags & (1 << 6) else "source_issue_bracket", + "flags": flags, + "execution_unit": "scalar", + } + event["cat"] = f"atomic.{atomic_boundary_tag}" + if atomic_op_id == 0: + event["args"]["value_zero"] = bool(flags & (1 << 5)) + if atomic_op_id == 3: + event["args"]["retries"] = (flags >> 8) & 0xFFFFFF + elif phase == "claim": + event["args"] = { + "phase": phase, + "task_id": task_id, + "func_id": func_id, + "core": int(e["core_id"]), + "claim_attempted": claim_attempted, + "claim_won": claim_won, + "claim_attempted_source": claim_attempted_source, + "claim_path": "alloc" if aux == 1 else "kernel", + "execution_unit": "scalar", + "flags": flags, } - ) + event["cat"] = "scalar_scheduler" + elif phase == "clock_baseline": + dependency_hook = bool(flags & 1) + event["args"] = { + "phase": phase, + "core": int(e["core_id"]), + "ticks": int(e["end_cycles"]) - int(e["start_cycles"]), + "clock_freq_hz": int(clock_freq_hz), + "definition": ("atomic-return-dependency-hook" if dependency_hook else "consecutive-sys-cnt-reads"), + "dependency_applied": bool(flags & 2) if dependency_hook else False, + "execution_unit": "scalar", + } + event["cat"] = "scalar_clock" + if trace_schema_version == 4: + # Schema-v4 keeps the ten-column raw file as the authority. The + # merged trace carries only the fields Perfetto needs; phase/task + # identity is already encoded in the event name. + event.pop("args", None) + event.pop("cat", None) + events.append(event) + + if v4_model is not None: + for residual in iter_v4_residual_spans(v4_model): + events.append( + { + "ph": "X", + "name": residual.name, + "pid": residual.block_id, + "tid": residual.lane, + "ts": round(cycle_time_us[residual.start_cycle], 3), + "dur": round((residual.end_cycle - residual.start_cycle) * residual_factor, 3), + } + ) def normalize_pto2_task_id_int(v): @@ -217,6 +529,7 @@ def read_perf_data(filepath): # noqa: PLR0912, PLR0915 "l2_swimlane_level": <1..4>, "metadata": { "clock_freq_hz": , + "trace_schema_version": <1|2|3|4>, # optional; default 1 "num_cores": , "core_types": ["aic"|"aiv", ...], # indexed by core_id "core_to_thread": [, ...] # optional (level >= 3) @@ -274,6 +587,10 @@ def read_perf_data(filepath): # noqa: PLR0912, PLR0915 clock_freq_hz = int(metadata.get("clock_freq_hz") or 0) if clock_freq_hz <= 0: raise ValueError(f"metadata missing/zero clock_freq_hz: {clock_freq_hz}") + trace_schema_version = int(metadata.get("trace_schema_version", 1)) + if trace_schema_version not in (1, 2, 3, 4): + raise ValueError(f"Unsupported metadata.trace_schema_version: {trace_schema_version} (expected 1, 2, 3, or 4)") + num_cores = int(metadata.get("num_cores") or 0) core_types = list(metadata.get("core_types") or []) core_to_thread = list(metadata.get("core_to_thread") or []) @@ -282,6 +599,16 @@ def read_perf_data(filepath): # noqa: PLR0912, PLR0915 sched_phases_raw = data.get("aicpu_scheduler_phases") or [] orch_phases_raw = data.get("aicpu_orchestrator_phases") or [] fdwic_rows = data.get("fdwic_events") or [] + fdwic_summary = metadata.get("fdwic_summary") + if trace_schema_version == 3 and level != 4: + raise ValueError("metadata.trace_schema_version=3 requires l2_swimlane_level=4") + if trace_schema_version >= 3 and (num_cores <= 0 or len(core_types) != num_cores): + raise ValueError( + f"metadata.trace_schema_version>=3 requires num_cores matching core_types: " + f"num_cores={num_cores} core_types={len(core_types)}" + ) + if trace_schema_version == 4 and not fdwic_rows: + raise ValueError("metadata.trace_schema_version=4 requires non-empty fdwic_events") # AICore lookup keyed by (core_id, reg_task_id). Two dispatches of the # same PTO2 task_token_raw to the same core (SPMD over-subscription, MIX @@ -465,28 +792,175 @@ def _phase_us(pr): aicpu_orchestrator_phases.append(converted) fdwic_events = [] - for row in fdwic_rows: + observed_summary = { + "records": len(fdwic_rows), + "atomic_records": 0, + "clock_baseline_records": 0, + "atomic_calls": 0, + "batched_poll_calls": 0, + "poll_batch_records": 0, + "dropped_records": 0, + } + v3_clock_rows = defaultdict(lambda: {"plain": 0, "dependency": 0, "return_ready": None}) + v3_result_used_direct_rows = [] + for row_index, row in enumerate(fdwic_rows): + if not isinstance(row, (list, tuple)) or len(row) != 10: + raise ValueError(f"fdwic_events[{row_index}] must contain exactly 10 columns") core_id, block_id, lane, task_id, func_id, phase, start_cycles, end_cycles, flags, aux = row - start_us = _to_us(int(start_cycles)) - end_us = _to_us(int(end_cycles)) + core_id = int(core_id) + block_id = int(block_id) + lane = int(lane) + task_id = int(task_id) + func_id = int(func_id) + start_cycles = int(start_cycles) + end_cycles = int(end_cycles) + flags = int(flags) + aux = int(aux) + phase = str(phase) + if trace_schema_version >= 3: + if not 0 <= core_id < num_cores: + raise ValueError(f"fdwic_events[{row_index}] has invalid core_id {core_id}") + if task_id < -1 or func_id < -1 or aux < 0 or phase not in _FDWIC_PHASE_NAMES: + raise ValueError( + f"fdwic_events[{row_index}] has invalid base fields: " + f"task={task_id} func={func_id} phase={phase!r} aux={aux}" + ) + if not (0 <= start_cycles <= end_cycles <= 0xFFFFFFFFFFFFFFFF): + raise ValueError(f"fdwic_events[{row_index}] has invalid cycle range {start_cycles}..{end_cycles}") + if not 0 <= flags <= 0xFFFFFFFF: + raise ValueError(f"fdwic_events[{row_index}] has invalid uint32 flags {flags}") + if trace_schema_version == 4 and phase in LEGACY_LAP_PHASES: + raise ValueError(f"fdwic_events[{row_index}] schema-v4 forbids legacy lap phase {phase!r}") + if trace_schema_version < 4 and phase in V4_PHASES: + raise ValueError(f"fdwic_events[{row_index}] phase {phase!r} requires trace_schema_version=4") + if trace_schema_version == 4: + _validate_fdwic_v4_phase_fields(row_index, task_id, func_id, phase, flags, aux) + if phase == "Claim" and trace_schema_version >= 2: + if flags & ~0x3 or (flags & 0x1 and not flags & 0x2): + raise ValueError(f"fdwic_events[{row_index}] has invalid Claim flags 0x{flags:x}") + if phase == "Atomic" and flags & (1 << 7): + call_count = (flags >> 8) & 0xFFFFFF + if ( + trace_schema_version < 3 + or (trace_schema_version == 4 and level != 4) + or call_count == 0 + or not flags & (1 << 4) + or _FDWIC_POLL_BATCH_SITE_OP_IDS.get(aux) != (flags & 0xF) + or flags & ((1 << 5) | (1 << 6)) + or task_id != -1 + or func_id != -1 + ): + raise ValueError(f"fdwic_events[{row_index}] has invalid Atomic PollBatch flags 0x{flags:x}") + elif phase == "Atomic" and trace_schema_version in (3, 4): + if level != 4: + raise ValueError(f"fdwic_events[{row_index}] Atomic requires l2_swimlane_level=4") + op = flags & 0xF + result_used = bool(flags & (1 << 4)) + value_zero = bool(flags & (1 << 5)) + return_ready = bool(flags & (1 << 6)) + payload = flags >> 8 + expected_result_used = aux in _FDWIC_ATOMIC_SITE_OP_IDS and aux not in _FDWIC_ATOMIC_RESULT_UNUSED_SITE_IDS + if ( + _FDWIC_ATOMIC_SITE_OP_IDS.get(aux) != op + or result_used != expected_result_used + or (return_ready and not result_used) + or (value_zero and op != 0) + or (payload and op != 3) + or func_id != -1 + ): + raise ValueError(f"fdwic_events[{row_index}] has invalid direct Atomic site={aux} flags=0x{flags:x}") + if result_used: + v3_result_used_direct_rows.append((row_index, core_id, return_ready)) + if phase == "ClockBaseline" and trace_schema_version in (3, 4): + if level != 4: + raise ValueError(f"fdwic_events[{row_index}] ClockBaseline requires l2_swimlane_level=4") + dependency = bool(flags & 0x1) + dependency_applied = bool(flags & 0x2) + if flags & ~0x3 or (dependency_applied and not dependency) or task_id != -1 or func_id != -1 or aux != 0: + raise ValueError(f"fdwic_events[{row_index}] has invalid ClockBaseline flags=0x{flags:x} aux={aux}") + clock_state = v3_clock_rows[core_id] + if dependency: + clock_state["dependency"] += 1 + clock_state["return_ready"] = dependency_applied + else: + clock_state["plain"] += 1 + if phase == "Claim" and trace_schema_version >= 3 and aux > 1: + raise ValueError(f"fdwic_events[{row_index}] has invalid Claim aux {aux}") + if phase == "Atomic": + observed_summary["atomic_records"] += 1 + if flags & (1 << 7): + call_count = (flags >> 8) & 0xFFFFFF + observed_summary["atomic_calls"] += call_count + observed_summary["batched_poll_calls"] += call_count + observed_summary["poll_batch_records"] += 1 + else: + observed_summary["atomic_calls"] += 1 + elif phase == "ClockBaseline": + observed_summary["clock_baseline_records"] += 1 + start_us = _to_us(start_cycles) + end_us = _to_us(end_cycles) fdwic_events.append( { - "core_id": int(core_id), - "block_id": int(block_id), - "lane": int(lane), - "task_id": int(task_id), - "func_id": int(func_id), - "phase": str(phase), + "core_id": core_id, + "block_id": block_id, + "lane": lane, + "task_id": task_id, + "func_id": func_id, + "phase": phase, + "start_cycles": start_cycles, + "end_cycles": end_cycles, "start_time_us": start_us, "end_time_us": end_us, "duration_us": end_us - start_us, - "flags": int(flags), - "aux": int(aux), + "flags": flags, + "aux": aux, } ) + if trace_schema_version in (3, 4) and level == 4: + for core_id in range(num_cores): + clock_state = v3_clock_rows[core_id] + if clock_state["plain"] != 1 or clock_state["dependency"] != 1: + raise ValueError( + f"core {core_id} requires exactly one plain and one dependency ClockBaseline: " + f"plain={clock_state['plain']} dependency={clock_state['dependency']}" + ) + for row_index, core_id, return_ready in v3_result_used_direct_rows: + expected_return_ready = bool(v3_clock_rows[core_id]["return_ready"]) + if return_ready != expected_return_ready: + raise ValueError( + f"fdwic_events[{row_index}] direct Atomic return_ready={return_ready} does not match " + f"core {core_id} ClockBaseline dependency_applied={expected_return_ready}" + ) + if trace_schema_version >= 3: + if not isinstance(fdwic_summary, dict): + raise ValueError("metadata.fdwic_summary is required for trace_schema_version>=3") + required_summary = { + "records": observed_summary["records"], + "atomic_records": observed_summary["atomic_records"], + "clock_baseline_records": observed_summary["clock_baseline_records"], + "atomic_calls": observed_summary["atomic_calls"], + "batched_poll_calls": observed_summary["batched_poll_calls"], + "poll_batch_records": observed_summary["poll_batch_records"], + "dropped_records": 0, + } + for key, observed_value in required_summary.items(): + try: + producer_value = int(fdwic_summary[key]) + except (KeyError, TypeError, ValueError) as exc: + raise ValueError(f"metadata.fdwic_summary.{key} is missing or invalid") from exc + if producer_value != observed_value: + raise ValueError( + f"metadata.fdwic_summary.{key}={producer_value} does not match raw value {observed_value}" + ) + + if trace_schema_version == 4: + validate_and_partition_v4(fdwic_events, num_cores, core_types) + out = { "l2_swimlane_level": level, + "clock_freq_hz": clock_freq_hz, + "trace_schema_version": trace_schema_version, "tasks": tasks, } if aicpu_scheduler_phases: @@ -497,6 +971,12 @@ def _phase_us(pr): out["core_to_thread"] = core_to_thread if fdwic_events: out["fdwic_events"] = fdwic_events + if trace_schema_version >= 3: + assert isinstance(fdwic_summary, dict) + out["fdwic_summary"] = dict(fdwic_summary) + if trace_schema_version == 4: + out["num_cores"] = num_cores + out["core_types"] = core_types return out @@ -998,6 +1478,10 @@ def generate_chrome_trace_json( # noqa: PLR0912, PLR0913, PLR0915 deps_kernel_map=None, emit_overhead=False, fdwic_events=None, + trace_schema_version=1, + clock_freq_hz=0, + fdwic_num_cores=0, + fdwic_core_types=None, ): """Generate Chrome Trace Event Format JSON from task data. @@ -1013,6 +1497,9 @@ def generate_chrome_trace_json( # noqa: PLR0912, PLR0913, PLR0915 scheduler_phases: Optional list of per-thread phase record lists (l2_swimlane_level >= 3) orchestrator_phases: Optional list of per-task orchestrator phase records (l2_swimlane_level >= 4) core_to_thread: Optional list mapping core_id (index) to scheduler thread index (-1 = unassigned) + trace_schema_version: FDWIC raw record schema (1 legacy, 2 explicit Claim flags, + 3 exact-count Atomic PollBatch rows, 4 exclusive hierarchy) + clock_freq_hz: Raw FDWIC cycle-counter frequency used by ClockBaseline event arguments Generates processes in the trace: - pid=4 "Worker View": start_time_us to end_time_us (kernel execution) @@ -1057,7 +1544,15 @@ def generate_chrome_trace_json( # noqa: PLR0912, PLR0913, PLR0915 events = [] if fdwic_events and not tasks: - _append_fdwic_dist_engine_events(events, fdwic_events, func_id_to_name) + _append_fdwic_dist_engine_events( + events, + fdwic_events, + func_id_to_name, + trace_schema_version=trace_schema_version, + clock_freq_hz=clock_freq_hz, + num_cores=fdwic_num_cores, + core_types=fdwic_core_types, + ) with open(output_path, "w") as f: json.dump({"displayTimeUnit": "ns", "traceEvents": events}, f, indent=2) if verbose: @@ -2485,7 +2980,15 @@ def main(): deps_kernel_map=deps_kernel_map, emit_overhead=args.overhead, fdwic_events=data.get("fdwic_events"), + trace_schema_version=data.get("trace_schema_version", 1), + clock_freq_hz=data.get("clock_freq_hz", 0), + fdwic_num_cores=data.get("num_cores", 0), + fdwic_core_types=data.get("core_types"), ) + exclusive_output = None + if data.get("trace_schema_version") == 4 and data.get("fdwic_events"): + exclusive_output = input_path.parent / "swimlane_exclusive_analysis.json" + write_analysis_data(data, input_path, exclusive_output) if args.overhead and deps_edges is None: print( "Warning: --overhead needs deps.json for task readiness; no deps found, " @@ -2496,6 +2999,8 @@ def main(): print("\n✓ Conversion complete") print(f" Input: {input_path}") print(f" Output: {output_path}") + if exclusive_output is not None: + print(f" Exclusive analysis: {exclusive_output}") print(f"\nTo visualize: Open https://ui.perfetto.dev/ and drag in {output_path}") print_task_statistics(data["tasks"], func_names) diff --git a/src/a5/platform/include/aicore/fdwic_submit_pmu_state.h b/src/a5/platform/include/aicore/fdwic_submit_pmu_state.h new file mode 100644 index 0000000000..ed61ef5097 --- /dev/null +++ b/src/a5/platform/include/aicore/fdwic_submit_pmu_state.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include + +// submit-pmu-none 只需要当前物理子核的 PMU MMIO 基址。独立接口避免把 +// generic PMU ring、普通泳道和总 profiling flag 带入诊断 ELF。 +extern __aicore__ void set_fdwic_submit_pmu_reg_base(uint64_t reg_base); +extern __aicore__ uint64_t get_fdwic_submit_pmu_reg_base(); diff --git a/src/a5/platform/onboard/aicore/CMakeLists.txt b/src/a5/platform/onboard/aicore/CMakeLists.txt index 81b4a22a4a..a2ddf42be8 100644 --- a/src/a5/platform/onboard/aicore/CMakeLists.txt +++ b/src/a5/platform/onboard/aicore/CMakeLists.txt @@ -42,6 +42,16 @@ if(DEFINED CUSTOM_SOURCE_FILES) list(APPEND ALL_SOURCES "${SRC_FILE}") endforeach() endif() + +# Per-callable diagnostic profiles pass compile definitions here. Keep them on +# every CCEC translation unit (platform entry, dist engine, orchestration and +# linked incore wrapper), otherwise one TU can silently retain profiling code. +set(CMAKE_CUSTOM_COMPILE_DEFINITION_FLAGS "") +if(DEFINED CUSTOM_COMPILE_DEFINITIONS) + foreach(DEFINITION ${CUSTOM_COMPILE_DEFINITIONS}) + list(APPEND CMAKE_CUSTOM_COMPILE_DEFINITION_FLAGS "-D${DEFINITION}") + endforeach() +endif() list(LENGTH ALL_SOURCES NUM_SOURCES) message(STATUS "AICore kernel: ${NUM_SOURCES} source files") message(VERBOSE "AICore kernel sources: ${ALL_SOURCES}") @@ -84,6 +94,7 @@ foreach(SRC_FILE ${ALL_SOURCES}) add_custom_command( OUTPUT ${OBJ_AIC} COMMAND ${BISHENG_CC} ${AICORE_FLAGS} ${CMAKE_CUSTOM_INCLUDE_DIR_FLAGS} --cce-aicore-arch=dav-c310-cube + ${CMAKE_CUSTOM_COMPILE_DEFINITION_FLAGS} -o ${OBJ_AIC} ${SRC_FILE} DEPENDS ${SRC_FILE} COMMENT "Compiling ${SRC_NAME} for AIC" @@ -93,6 +104,7 @@ foreach(SRC_FILE ${ALL_SOURCES}) add_custom_command( OUTPUT ${OBJ_AIV} COMMAND ${BISHENG_CC} ${AICORE_FLAGS} ${CMAKE_CUSTOM_INCLUDE_DIR_FLAGS} --cce-aicore-arch=dav-c310-vec + ${CMAKE_CUSTOM_COMPILE_DEFINITION_FLAGS} -o ${OBJ_AIV} ${SRC_FILE} DEPENDS ${SRC_FILE} COMMENT "Compiling ${SRC_NAME} for AIV" diff --git a/src/a5/platform/onboard/aicore/kernel.cpp b/src/a5/platform/onboard/aicore/kernel.cpp index 57f0ffcfe3..16a1ae9e02 100644 --- a/src/a5/platform/onboard/aicore/kernel.cpp +++ b/src/a5/platform/onboard/aicore/kernel.cpp @@ -12,12 +12,20 @@ * Minimal AICore Kernel */ #include "aicore/aicore.h" +#if defined(PTO_FDWIC_SUBMIT_PMU) && PTO_FDWIC_SUBMIT_PMU +#include "aicore/fdwic_submit_pmu_state.h" +#elif !defined(PTO_FDWIC_PERF_CLOCK) || !PTO_FDWIC_PERF_CLOCK #include "aicore/aicore_profiling_state.h" +#endif #include "common/core_type.h" #include "common/kernel_args.h" +#if !defined(PTO_FDWIC_PERF_CLOCK) || !PTO_FDWIC_PERF_CLOCK +#if !defined(PTO_FDWIC_SUBMIT_PMU) || !PTO_FDWIC_SUBMIT_PMU #include "common/l2_swimlane_profiling.h" -#include "common/platform_config.h" #include "common/pmu_profiling.h" +#endif +#endif +#include "common/platform_config.h" #include "simt_anchor.h" class Runtime; @@ -37,6 +45,16 @@ class Runtime; [[block_local]] int block_idx; [[block_local]] CoreType core_type; +#if defined(PTO_FDWIC_SUBMIT_PMU) && PTO_FDWIC_SUBMIT_PMU +[[block_local]] static uint64_t s_fdwic_submit_pmu_reg_base; + +__attribute__((weak)) __aicore__ void set_fdwic_submit_pmu_reg_base(uint64_t reg_base) { + s_fdwic_submit_pmu_reg_base = reg_base; +} +__attribute__((weak)) __aicore__ uint64_t get_fdwic_submit_pmu_reg_base() { + return s_fdwic_submit_pmu_reg_base; +} +#elif !defined(PTO_FDWIC_PERF_CLOCK) || !PTO_FDWIC_PERF_CLOCK // Per-core profiling state. Populated once by KERNEL_ENTRY from KernelArgs; // read by aicore_execute and profiling helpers via the getters below. This // mirrors the AICPU-side set_l2_swimlane_enabled / set_pmu_enabled pattern, @@ -78,6 +96,7 @@ __attribute__((weak)) __aicore__ __gm__ PmuAicoreRing *get_aicore_pmu_ring() { r __attribute__((weak)) __aicore__ void set_aicore_pmu_reg_base(uint64_t reg_base) { s_aicore_pmu_reg_base = reg_base; } __attribute__((weak)) __aicore__ uint64_t get_aicore_pmu_reg_base() { return s_aicore_pmu_reg_base; } +#endif extern __aicore__ void aicore_execute(__gm__ Runtime *runtime, int block_idx, CoreType core_type); @@ -111,6 +130,14 @@ extern "C" __global__ __aicore__ void KERNEL_ENTRY(aicore_kernel)(__gm__ KernelA core_type = CoreType::AIC; #endif +#if defined(PTO_FDWIC_SUBMIT_PMU) && PTO_FDWIC_SUBMIT_PMU + // 独立整窗 PMU 构建不打开 generic PROFILING_FLAG_PMU，也不创建逐 task + // ring。这里仅从 host 已发布的物理寄存器表解析本核 MMIO 基址。 + __gm__ uint64_t *regs_array = reinterpret_cast<__gm__ uint64_t *>(k_args->regs); + set_fdwic_submit_pmu_reg_base( + regs_array == nullptr ? 0 : regs_array[get_physical_core_id()] + ); +#elif !defined(PTO_FDWIC_PERF_CLOCK) || !PTO_FDWIC_PERF_CLOCK // Publish per-core profiling state into platform-owned slots before the // executor runs. AICore reads via get_aicore_*() — never touches Handshake // for profiling. The PMU MMIO base is resolved here from @@ -154,6 +181,7 @@ extern "C" __global__ __aicore__ void KERNEL_ENTRY(aicore_kernel)(__gm__ KernelA set_aicore_pmu_ring(nullptr); set_aicore_pmu_reg_base(0); } +#endif #ifdef __DAV_VEC__ // SIMT classification anchor (AIV only). Never executes — diff --git a/src/a5/platform/onboard/aicpu/CMakeLists.txt b/src/a5/platform/onboard/aicpu/CMakeLists.txt index fca9c6b782..a24ac505f9 100644 --- a/src/a5/platform/onboard/aicpu/CMakeLists.txt +++ b/src/a5/platform/onboard/aicpu/CMakeLists.txt @@ -84,6 +84,13 @@ target_compile_options(aicpu_kernel $<$:-std=gnu11> ) +# Runtime ABI feature gates belong only to the runtime-bearing inner AICPU +# image. The dispatcher is process-global and deliberately remains mode +# neutral. +if(DEFINED CUSTOM_COMPILE_DEFINITIONS) + target_compile_definitions(aicpu_kernel PRIVATE ${CUSTOM_COMPILE_DEFINITIONS}) +endif() + target_include_directories(aicpu_kernel PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/src/a5/platform/onboard/host/CMakeLists.txt b/src/a5/platform/onboard/host/CMakeLists.txt index 192975b92e..b69a9670ef 100644 --- a/src/a5/platform/onboard/host/CMakeLists.txt +++ b/src/a5/platform/onboard/host/CMakeLists.txt @@ -95,6 +95,9 @@ target_compile_options(host_runtime # Platform name baked into the shared get_platform() impl in # src/common/platform/shared/host/platform_compile_info.cpp. target_compile_definitions(host_runtime PRIVATE SIMPLER_PLATFORM_NAME="a5") +if(DEFINED CUSTOM_COMPILE_DEFINITIONS) + target_compile_definitions(host_runtime PRIVATE ${CUSTOM_COMPILE_DEFINITIONS}) +endif() # Include directories - always include local headers target_include_directories(host_runtime diff --git a/src/a5/platform/onboard/host/device_runner.cpp b/src/a5/platform/onboard/host/device_runner.cpp index 13ee9de0cc..7f354361db 100644 --- a/src/a5/platform/onboard/host/device_runner.cpp +++ b/src/a5/platform/onboard/host/device_runner.cpp @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -56,9 +57,17 @@ extern "C" __attribute__((weak, visibility("hidden"))) int dep_gen_replay_emit_d } extern "C" __attribute__((weak)) int -fdwic_swimlane_host_init(Runtime *runtime, int num_cores, int enabled, const char *output_prefix); +fdwic_swimlane_host_init(Runtime *runtime, int num_cores, int level, const char *output_prefix); extern "C" __attribute__((weak)) int fdwic_swimlane_host_export(Runtime *runtime); extern "C" __attribute__((weak)) void fdwic_swimlane_host_finalize(Runtime *runtime); +extern "C" __attribute__((weak)) int +fdwic_perf_clock_host_init(Runtime *runtime, int num_cores, const char *output_prefix); +extern "C" __attribute__((weak)) int fdwic_perf_clock_host_export(Runtime *runtime); +extern "C" __attribute__((weak)) void fdwic_perf_clock_host_finalize(Runtime *runtime); +extern "C" __attribute__((weak)) int +fdwic_submit_pmu_host_init(Runtime *runtime, int num_cores, const char *output_prefix); +extern "C" __attribute__((weak)) int fdwic_submit_pmu_host_export(Runtime *runtime); +extern "C" __attribute__((weak)) void fdwic_submit_pmu_host_finalize(Runtime *runtime); // ============================================================================= // DeviceRunner Implementation @@ -158,11 +167,9 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { // failure cascade into a single fast, self-explanatory error; the runner is // then recovered at finalize. if (device_unusable_) { - LOG_ERROR( - "DeviceRunner marked unusable by a prior AICore failure; refusing to run. " - "A soft reset does not clear the poison on a5; finalize() will force-reset " - "the card so the next Worker on it inits clean." - ); + LOG_ERROR("DeviceRunner marked unusable by a prior AICore failure; refusing to run. " + "A soft reset does not clear the poison on a5; finalize() will force-reset " + "the card so the next Worker on it inits clean."); return -1; } if (validate_launch_aicpu_num(launch_aicpu_num) != 0) return -1; @@ -270,8 +277,85 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { enable_l2_swimlane_ = original_enable_l2_swimlane; }); bool fdwic_swimlane_active = false; + auto fdwic_swimlane_cleanup = RAIIScopeGuard([&runtime, &fdwic_swimlane_active]() { + if (fdwic_swimlane_active && fdwic_swimlane_host_finalize != nullptr) { + fdwic_swimlane_host_finalize(&runtime); + fdwic_swimlane_active = false; + } + }); + bool fdwic_perf_clock_active = false; + auto fdwic_perf_clock_cleanup = RAIIScopeGuard([&runtime, &fdwic_perf_clock_active]() { + if (fdwic_perf_clock_active && fdwic_perf_clock_host_finalize != nullptr) { + fdwic_perf_clock_host_finalize(&runtime); + fdwic_perf_clock_active = false; + } + }); + bool fdwic_submit_pmu_active = false; + auto fdwic_submit_pmu_cleanup = RAIIScopeGuard([&runtime, &fdwic_submit_pmu_active]() { + if (fdwic_submit_pmu_active && fdwic_submit_pmu_host_finalize != nullptr) { + fdwic_submit_pmu_host_finalize(&runtime); + fdwic_submit_pmu_active = false; + } + }); + const char *fdwic_profile = std::getenv("PTO_FDWIC_PROFILE"); + const bool fdwic_perf_clock_requested = + fdwic_profile != nullptr && + (std::strcmp(fdwic_profile, "perf-clock") == 0 || std::strcmp(fdwic_profile, "perf-clock-kernel") == 0); + const bool fdwic_submit_pmu_requested = + fdwic_profile != nullptr && std::strncmp(fdwic_profile, "submit-pmu-", 11) == 0; + if (fdwic_perf_clock_requested) { + if (enable_profiling_flag != PROFILING_FLAG_NONE) { + LOG_ERROR("fdwic perf-clock cannot be combined with another runtime diagnostic"); + return -1; + } + if (fdwic_perf_clock_host_init == nullptr || fdwic_perf_clock_host_export == nullptr || + fdwic_perf_clock_host_finalize == nullptr) { + LOG_ERROR("fdwic perf-clock was requested but the selected runtime does not provide its host hooks"); + return -1; + } + rc = fdwic_perf_clock_host_init(&runtime, num_aicore, output_prefix_.c_str()); + if (rc <= 0) { + LOG_ERROR("fdwic perf-clock init failed: %d", rc); + return rc < 0 ? rc : -1; + } + fdwic_perf_clock_active = true; + // perf-clock is an independent header-only evidence path. Generic + // L2/PMU pointers and the umbrella launch flag must remain empty. + kernel_args_.args.enable_profiling_flag = PROFILING_FLAG_NONE; + kernel_args_.args.l2_swimlane_data_base = 0; + kernel_args_.args.l2_swimlane_aicore_rotation_table = 0; + kernel_args_.args.aicore_pmu_ring_addrs = 0; + kernel_args_.args.pmu_data_base = 0; + } + if (fdwic_submit_pmu_requested) { + if (enable_profiling_flag != PROFILING_FLAG_NONE) { + LOG_ERROR("fdwic submit-PMU profile cannot be combined with another runtime diagnostic"); + return -1; + } + if (fdwic_submit_pmu_host_init == nullptr || fdwic_submit_pmu_host_export == nullptr || + fdwic_submit_pmu_host_finalize == nullptr) { + LOG_ERROR("fdwic submit-PMU profile was requested but the selected runtime does not provide its host hooks" + ); + return -1; + } + rc = fdwic_submit_pmu_host_init(&runtime, num_aicore, output_prefix_.c_str()); + if (rc <= 0) { + LOG_ERROR("fdwic submit-PMU profile init failed: %d", rc); + return rc < 0 ? rc : -1; + } + fdwic_submit_pmu_active = true; + // 该 profile 只复用 KernelArgs::regs 解析本核 MMIO 地址。所有通用 + // collector 位与地址必须为空，避免把逐 task ring 混入整窗证据。 + kernel_args_.args.enable_profiling_flag = PROFILING_FLAG_NONE; + kernel_args_.args.l2_swimlane_data_base = 0; + kernel_args_.args.l2_swimlane_aicore_rotation_table = 0; + kernel_args_.args.aicore_pmu_ring_addrs = 0; + kernel_args_.args.pmu_data_base = 0; + } if (enable_l2_swimlane_ && fdwic_swimlane_host_init != nullptr) { - rc = fdwic_swimlane_host_init(&runtime, num_aicore, 1, output_prefix_.c_str()); + rc = fdwic_swimlane_host_init( + &runtime, num_aicore, static_cast(l2_swimlane_level_), output_prefix_.c_str() + ); if (rc < 0) { LOG_ERROR("fdwic swimlane init failed: %d", rc); return rc; @@ -279,6 +363,14 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { if (rc > 0) { fdwic_swimlane_active = true; enable_l2_swimlane_ = false; + // The FDWIC runtime owns its trace buffer and does not use the + // platform-generic L2 collector. Keep the launch bit and generic + // pointers consistent with that routing decision; these fields + // may otherwise retain a prior run's collector addresses. + CLEAR_PROFILING_FLAG(enable_profiling_flag, PROFILING_FLAG_L2_SWIMLANE); + kernel_args_.args.enable_profiling_flag = enable_profiling_flag; + kernel_args_.args.l2_swimlane_data_base = 0; + kernel_args_.args.l2_swimlane_aicore_rotation_table = 0; } } if (enable_l2_swimlane_) { @@ -325,12 +417,8 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { // Cleanup guard for early returns: stops all started collectors so // their mgmt + poll threads exit cleanly. stop() is idempotent and a // no-op on collectors that never started. - auto perf_cleanup = RAIIScopeGuard([this, &runtime, &fdwic_swimlane_active]() { + auto perf_cleanup = RAIIScopeGuard([this]() { finalize_collectors(); - if (fdwic_swimlane_active && fdwic_swimlane_host_finalize != nullptr) { - fdwic_swimlane_host_finalize(&runtime); - fdwic_swimlane_active = false; - } }); LOG_INFO_V0("=== Initialize runtime args ==="); @@ -351,12 +439,10 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { dep_gen_collector_.start(thread_factory); } - // workers[i].core_type is written by the AICore kernel during its - // AICPU<->AICore handshake (aicore_executor.cpp), launched further below, - // so the values read here reflect the most recent prior run's handshake - // still resident in device memory (unset on the first run of a freshly- - // loaded runtime). Publish the table to the L2 swimlane collector so the - // AICORE_TIMING (level=1) host emit path can label lanes ("aic"/"aiv"). + // prepare_runtime_for_launch 已按本轮 block_dim 写入 host launch-plan： + // 前 block_dim 个 worker 为 AIC，其余为 AIV。设备握手随后会在 device + // Runtime 中确认实际角色，但不会回写这里的 host Runtime。将当前计划表 + // 交给 L2 collector，用于 AICORE_TIMING (level=1) host 输出的 lane 标签。 if (enable_l2_swimlane_ && l2_swimlane_collector_.is_initialized()) { std::vector core_types(num_aicore); for (int i = 0; i < num_aicore; i++) { @@ -433,7 +519,22 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { // still exports exactly once below. teardown_shared_collectors_after_run(); if (fdwic_swimlane_active && fdwic_swimlane_host_export != nullptr) { - fdwic_swimlane_host_export(&runtime); + const int export_rc = fdwic_swimlane_host_export(&runtime); + if (export_rc != 0) { + LOG_ERROR("fdwic swimlane export failed after runtime error: %d", export_rc); + } + } + if (fdwic_perf_clock_active) { + const int export_rc = fdwic_perf_clock_host_export(&runtime); + if (export_rc != 0) { + LOG_ERROR("fdwic perf-clock export failed after runtime error: %d", export_rc); + } + } + if (fdwic_submit_pmu_active) { + const int export_rc = fdwic_submit_pmu_host_export(&runtime); + if (export_rc != 0) { + LOG_ERROR("fdwic submit-PMU export rejected after runtime error: %d", export_rc); + } } return rc; } @@ -441,8 +542,26 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { read_device_wall_ns(); teardown_shared_collectors_after_run(); + int fdwic_swimlane_export_rc = 0; if (fdwic_swimlane_active && fdwic_swimlane_host_export != nullptr) { - fdwic_swimlane_host_export(&runtime); + fdwic_swimlane_export_rc = fdwic_swimlane_host_export(&runtime); + if (fdwic_swimlane_export_rc != 0) { + LOG_ERROR("fdwic swimlane export failed: %d", fdwic_swimlane_export_rc); + } + } + int fdwic_perf_clock_export_rc = 0; + if (fdwic_perf_clock_active) { + fdwic_perf_clock_export_rc = fdwic_perf_clock_host_export(&runtime); + if (fdwic_perf_clock_export_rc != 0) { + LOG_ERROR("fdwic perf-clock export failed: %d", fdwic_perf_clock_export_rc); + } + } + int fdwic_submit_pmu_export_rc = 0; + if (fdwic_submit_pmu_active) { + fdwic_submit_pmu_export_rc = fdwic_submit_pmu_host_export(&runtime); + if (fdwic_submit_pmu_export_rc != 0) { + LOG_ERROR("fdwic submit-PMU export failed: %d", fdwic_submit_pmu_export_rc); + } } // a5-specific dep_gen teardown: stop + reconcile + replay emit. @@ -461,7 +580,9 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { // Print handshake results (reads from device memory, must be before free) print_handshake_results(); - return 0; + if (fdwic_swimlane_export_rc != 0) return fdwic_swimlane_export_rc; + if (fdwic_perf_clock_export_rc != 0) return fdwic_perf_clock_export_rc; + return fdwic_submit_pmu_export_rc; } void DeviceRunner::recover_device_or_mark_unusable(int aicore_rc) { diff --git a/src/a5/platform/sim/aicore/CMakeLists.txt b/src/a5/platform/sim/aicore/CMakeLists.txt index 0330fdb5c8..9ea689a82e 100644 --- a/src/a5/platform/sim/aicore/CMakeLists.txt +++ b/src/a5/platform/sim/aicore/CMakeLists.txt @@ -75,6 +75,9 @@ target_compile_options(aicore_kernel # Define __CPU_SIM for CPU simulation backend. # This enables CPU stubs for PTO instructions (TLOAD, TSTORE, TADD, etc.). target_compile_definitions(aicore_kernel PRIVATE __CPU_SIM) +if(DEFINED CUSTOM_COMPILE_DEFINITIONS) + target_compile_definitions(aicore_kernel PRIVATE ${CUSTOM_COMPILE_DEFINITIONS}) +endif() # Include directories target_include_directories(aicore_kernel diff --git a/src/a5/platform/sim/aicpu/CMakeLists.txt b/src/a5/platform/sim/aicpu/CMakeLists.txt index 7a852f00ba..f0eb5338eb 100644 --- a/src/a5/platform/sim/aicpu/CMakeLists.txt +++ b/src/a5/platform/sim/aicpu/CMakeLists.txt @@ -88,6 +88,10 @@ target_compile_options(aicpu_kernel $<$:-std=gnu11> ) +if(DEFINED CUSTOM_COMPILE_DEFINITIONS) + target_compile_definitions(aicpu_kernel PRIVATE ${CUSTOM_COMPILE_DEFINITIONS}) +endif() + # Include directories target_include_directories(aicpu_kernel PRIVATE diff --git a/src/a5/platform/sim/host/CMakeLists.txt b/src/a5/platform/sim/host/CMakeLists.txt index aefeac7099..455b076fab 100644 --- a/src/a5/platform/sim/host/CMakeLists.txt +++ b/src/a5/platform/sim/host/CMakeLists.txt @@ -95,6 +95,9 @@ target_compile_options(host_runtime # Platform name baked into the shared get_platform() impl in # src/common/platform/shared/host/platform_compile_info.cpp. target_compile_definitions(host_runtime PRIVATE SIMPLER_PLATFORM_NAME="a5sim" SIMPLER_AICORE_LINKED_ORCH=1) +if(DEFINED CUSTOM_COMPILE_DEFINITIONS) + target_compile_definitions(host_runtime PRIVATE ${CUSTOM_COMPILE_DEFINITIONS}) +endif() # Include directories target_include_directories(host_runtime diff --git a/src/a5/platform/sim/host/device_runner.cpp b/src/a5/platform/sim/host/device_runner.cpp index d377772f1f..a2c2d633d6 100644 --- a/src/a5/platform/sim/host/device_runner.cpp +++ b/src/a5/platform/sim/host/device_runner.cpp @@ -56,7 +56,7 @@ extern "C" __attribute__((weak, visibility("hidden"))) int dep_gen_replay_emit_d } extern "C" __attribute__((weak)) int -fdwic_swimlane_host_init(Runtime *runtime, int num_cores, int enabled, const char *output_prefix); +fdwic_swimlane_host_init(Runtime *runtime, int num_cores, int level, const char *output_prefix); extern "C" __attribute__((weak)) int fdwic_swimlane_host_export(Runtime *runtime); extern "C" __attribute__((weak)) void fdwic_swimlane_host_finalize(Runtime *runtime); @@ -262,8 +262,16 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { enable_l2_swimlane_ = original_enable_l2_swimlane; }); bool fdwic_swimlane_active = false; + auto fdwic_swimlane_cleanup = RAIIScopeGuard([&runtime, &fdwic_swimlane_active]() { + if (fdwic_swimlane_active && fdwic_swimlane_host_finalize != nullptr) { + fdwic_swimlane_host_finalize(&runtime); + fdwic_swimlane_active = false; + } + }); if (enable_l2_swimlane_ && fdwic_swimlane_host_init != nullptr) { - rc = fdwic_swimlane_host_init(&runtime, num_aicore, 1, output_prefix_.c_str()); + rc = fdwic_swimlane_host_init( + &runtime, num_aicore, static_cast(l2_swimlane_level_), output_prefix_.c_str() + ); if (rc < 0) { LOG_ERROR("fdwic swimlane init failed: %d", rc); return rc; @@ -271,6 +279,14 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { if (rc > 0) { fdwic_swimlane_active = true; enable_l2_swimlane_ = false; + // The FDWIC runtime owns its trace buffer and does not use the + // platform-generic L2 collector. Keep the launch bit and generic + // pointers consistent with that routing decision; these fields + // may otherwise retain a prior run's collector addresses. + CLEAR_PROFILING_FLAG(enable_profiling_flag, PROFILING_FLAG_L2_SWIMLANE); + kernel_args_.enable_profiling_flag = enable_profiling_flag; + kernel_args_.l2_swimlane_data_base = 0; + kernel_args_.l2_swimlane_aicore_rotation_table = 0; } } @@ -325,12 +341,8 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { // Cleanup guard for early returns: stops all started collectors so their // mgmt + poll threads exit cleanly. stop() is idempotent and a no-op on // collectors that never started. - auto perf_cleanup = RAIIScopeGuard([this, &runtime, &fdwic_swimlane_active]() { + auto perf_cleanup = RAIIScopeGuard([this]() { stop_collectors(); - if (fdwic_swimlane_active && fdwic_swimlane_host_finalize != nullptr) { - fdwic_swimlane_host_finalize(&runtime); - fdwic_swimlane_active = false; - } }); // Allocate simulated register blocks for all AICore cores. Uses sparse @@ -472,7 +484,10 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { if (runtime_rc != 0) { LOG_ERROR("AICPU execution failed with rc=%d", runtime_rc); if (fdwic_swimlane_active && fdwic_swimlane_host_export != nullptr) { - fdwic_swimlane_host_export(&runtime); + const int export_rc = fdwic_swimlane_host_export(&runtime); + if (export_rc != 0) { + LOG_ERROR("fdwic swimlane export failed after runtime error: %d", export_rc); + } } return runtime_rc; } @@ -485,8 +500,12 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { l2_swimlane_collector_.reconcile_counters(); l2_swimlane_collector_.export_swimlane_json(); } + int fdwic_swimlane_export_rc = 0; if (fdwic_swimlane_active && fdwic_swimlane_host_export != nullptr) { - fdwic_swimlane_host_export(&runtime); + fdwic_swimlane_export_rc = fdwic_swimlane_host_export(&runtime); + if (fdwic_swimlane_export_rc != 0) { + LOG_ERROR("fdwic swimlane export failed: %d", fdwic_swimlane_export_rc); + } } if (enable_dump_tensor_) { @@ -530,7 +549,7 @@ int DeviceRunner::run(Runtime &runtime, int block_dim, int launch_aicpu_num) { aicore_so_path_.clear(); } - return 0; + return fdwic_swimlane_export_rc; } void DeviceRunner::unload_executor_binaries() { diff --git a/src/a5/runtime/fully_distributed_within_core/aicore/aicore_executor.cpp b/src/a5/runtime/fully_distributed_within_core/aicore/aicore_executor.cpp index bed5b64be7..5ab0edc46d 100644 --- a/src/a5/runtime/fully_distributed_within_core/aicore/aicore_executor.cpp +++ b/src/a5/runtime/fully_distributed_within_core/aicore/aicore_executor.cpp @@ -18,6 +18,7 @@ #include "common/pmu_profiling.h" #include "pto2_dispatch_payload.h" #include "runtime.h" +#include "dist_engine/aicore/primitive.h" PTO_DEVICE_FUNC void dist_core_main(__gm__ Runtime *runtime, int core_idx, int core_type_int); @@ -70,6 +71,14 @@ __aicore__ __attribute__((always_inline)) static void execute_task(__gm__ PTO2Di * @param core_type Core type (AIC or AIV) */ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, int s_block_idx, CoreType core_type) { + // The identity line is the stable Runtime prefix shared by all three + // images. Read it before interpreting mode-dependent state, but keep the + // existing handshake alive even on mismatch so AICPU can issue DIST_ABORT + // and complete teardown without a device hang. + dcci(&runtime->fdwic_build_identity, SINGLE_CACHE_LINE); + const bool fdwic_build_identity_ok = + fdwic_build_identity_matches(runtime->fdwic_build_identity, static_cast(sizeof(Runtime))); + __gm__ Handshake *my_hank = (__gm__ Handshake *)(&runtime->workers[s_block_idx]); // Phase 1: Wait for AICPU initialization signal @@ -104,11 +113,34 @@ __aicore__ __attribute__((weak)) void aicore_execute(__gm__ Runtime *runtime, in // task claim/build/execute, and completion flag publication internally; // there is no per-task register handshake. // =========================================================================== - while (my_hank->aicpu_ready != AICPU_READY_DIST_RUN) { + while (my_hank->aicpu_ready != AICPU_READY_DIST_RUN && my_hank->aicpu_ready != AICPU_READY_DIST_ABORT) { dcci(my_hank, SINGLE_CACHE_LINE); SPIN_WAIT_HINT(); } - dist_core_main(runtime, s_block_idx, static_cast(core_type)); + if (my_hank->aicpu_ready == AICPU_READY_DIST_RUN) { + if (fdwic_build_identity_ok && kFdwicCompiledBackendReady) { + dist_core_main(runtime, s_block_idx, static_cast(core_type)); + } else { + // All workers make the same image-level decision, so one elected + // core publishes the global error. This avoids a 96-core RMW race + // on the identity line. The cold-path flush includes a completion + // barrier; block 0 publishes FIN only after the error is visible, + // and AICPU waits for every worker's FIN before reading it. + if (s_block_idx == 0) { +#if defined(__CCE_AICORE__) + runtime->fdwic_build_identity.error_bits = + runtime->fdwic_build_identity.error_bits | FdwicBuildErrorAicoreMismatch; +#else + __atomic_fetch_or( + &runtime->fdwic_build_identity.error_bits, static_cast(FdwicBuildErrorAicoreMismatch), + __ATOMIC_RELEASE + ); +#endif + dist_aicore_flush_region(&runtime->fdwic_build_identity, sizeof(runtime->fdwic_build_identity)); + } + write_reg(RegId::COND, MAKE_FIN_VALUE(0)); + } + } // Teardown: wait for the AICPU EXIT signal on DATA_MAIN_BASE and ack. while (true) { uint32_t reg_val = static_cast(read_reg(RegId::DATA_MAIN_BASE)); diff --git a/src/a5/runtime/fully_distributed_within_core/aicpu/aicpu_executor.cpp b/src/a5/runtime/fully_distributed_within_core/aicpu/aicpu_executor.cpp index c352a90cd3..8a24e0c86e 100644 --- a/src/a5/runtime/fully_distributed_within_core/aicpu/aicpu_executor.cpp +++ b/src/a5/runtime/fully_distributed_within_core/aicpu/aicpu_executor.cpp @@ -20,6 +20,7 @@ // Runtime headers #include "pto_runtime2.h" #include "dist_engine/dist_engine.h" +#include "dist_engine/aicpu/submit_pmu_owner.h" // Performance profiling headers #include "aicpu/l2_swimlane_collector_aicpu.h" @@ -56,6 +57,10 @@ struct AicpuExecutor { std::atomic finished_count_{0}; std::atomic runtime_done_{false}; + // Orchestrator owns the distributed run, then publishes one common status + // before runtime_done_. Every AICPU block returns the same runtime failure + // instead of depending on launcher-specific multi-block aggregation. + std::atomic run_status_{0}; // ===== Core lifecycle context ===== PTO2DispatchPayload payload_per_core_[RUNTIME_MAX_WORKER][2]; @@ -110,6 +115,7 @@ int32_t AicpuExecutor::init(Runtime *runtime) { finished_count_.store(0, std::memory_order_release); runtime_done_.store(false, std::memory_order_release); + run_status_.store(0, std::memory_order_release); init_done_.store(true, std::memory_order_release); LOG_INFO_V0("AicpuExecutor: Init complete"); @@ -225,123 +231,235 @@ int32_t AicpuExecutor::run(Runtime *runtime) { if (thread_idx >= sched_thread_num_) { #if PTO2_PROFILING uint64_t orch_cycle_start = 0; + bool orch_cycle_started = false; #endif // AICPU setup thread: initialize the shared distributed state, start // AICore workers, and wait for their on-core orchestration replay. { LOG_INFO_V0("Thread %d: Preparing shared dist state", thread_idx); - - // Build the entry-arg once per run. The AICore-side dist engine - // consumes it when replaying the linked orchestration entry. - runtime->dist.orch_args.create_from_chip_args(runtime->get_orch_args()); - const ChipStorageTaskArgs &orch_args = runtime->get_orch_args(); - runtime->dist.ccec_orch_tensor_count = orch_args.tensor_count(); - runtime->dist.ccec_orch_scalar_count = orch_args.scalar_count(); - for (int32_t i = 0; i < orch_args.tensor_count() && i < CHIP_MAX_TENSOR_ARGS; i++) { - Tensor::copy(runtime->dist.ccec_orch_tensors[i], orch_args.tensor(i)); - } - for (int32_t i = 0; i < orch_args.scalar_count() && i < CHIP_MAX_SCALAR_ARGS; i++) { - runtime->dist.ccec_orch_scalars[i] = orch_args.scalar(i); - } - cache_flush_range(runtime->dist.ccec_orch_tensors, sizeof(runtime->dist.ccec_orch_tensors)); - cache_flush_range(runtime->dist.ccec_orch_scalars, sizeof(runtime->dist.ccec_orch_scalars)); - cache_flush_range(const_cast(&runtime->dist.ccec_orch_tensor_count), 2 * sizeof(int32_t)); - - // rt is bound to *this* run's memory and must be reattached every - // run. - const ChipStorageTaskArgs &args = runtime->get_orch_args(); - int32_t arg_count = args.tensor_count() + args.scalar_count(); - LOG_INFO_V0("Thread %d: arg_count=%d", thread_idx, arg_count); - for (int32_t i = 0; i < args.tensor_count() && i < 20; i++) { - const Tensor &t = args.tensor(i); - LOG_INFO_V0( - "Thread %d: orch_args[%d] = TENSOR(data=0x%lx, ndims=%u, dtype=%u)", thread_idx, i, - static_cast(t.buffer.addr), t.ndims, static_cast(t.dtype) + const int32_t num_workers = runtime->worker_count; + + // Validate the stable Runtime prefix before reading DistHandoff or + // PTO2Runtime state. A mismatch means the Host/AICPU images came + // from different TensorMap artifact families. The shared backend + // is intentionally fail-closed until its real implementation is + // connected in the next stage. + cache_invalidate_range(&runtime->fdwic_build_identity, sizeof(runtime->fdwic_build_identity)); + bool dispatch_dist_run = + fdwic_build_identity_matches(runtime->fdwic_build_identity, static_cast(sizeof(Runtime))); + if (!dispatch_dist_run) { + LOG_ERROR( + "Thread %d: FDWIC build identity mismatch " + "(magic=0x%" PRIx64 ", abi=%u, mode=%u, ring_cap=%u, runtime_bytes=%u, dist_layout=%u; " + "expected abi=%u, mode=%u, ring_cap=%u, runtime_bytes=%zu, dist_layout=%u)", + thread_idx, static_cast(runtime->fdwic_build_identity.magic), + runtime->fdwic_build_identity.abi_version, runtime->fdwic_build_identity.tensor_map_mode, + runtime->fdwic_build_identity.tensor_map_ring_cap, runtime->fdwic_build_identity.runtime_bytes, + runtime->fdwic_build_identity.dist_global_layout_version, kFdwicBuildAbiVersion, + static_cast(kFdwicCompiledTensorMapMode), kFdwicTensorMapRingCap, sizeof(Runtime), + kFdwicDistGlobalLayoutVersion ); - } - for (int32_t i = 0; i < args.scalar_count() && (args.tensor_count() + i) < 20; i++) { - LOG_INFO_V0( - "Thread %d: orch_args[%d] = SCALAR(0x%lx)", thread_idx, args.tensor_count() + i, - static_cast(args.scalar(i)) + __atomic_fetch_or( + &runtime->fdwic_build_identity.error_bits, static_cast(FdwicBuildErrorAicpuMismatch), + __ATOMIC_RELEASE + ); + run_rc = -1; + } else if (!kFdwicCompiledBackendReady) { + LOG_ERROR( + "Thread %d: FDWIC shared TensorMap artifact is ABI-valid but its runtime backend " + "is not connected yet; aborting before Submit", + thread_idx + ); + __atomic_fetch_or( + &runtime->fdwic_build_identity.error_bits, static_cast(FdwicBuildErrorBackendUnavailable), + __ATOMIC_RELEASE ); + dispatch_dist_run = false; + run_rc = -1; } + cache_flush_range(&runtime->fdwic_build_identity, sizeof(runtime->fdwic_build_identity)); - // Host has pre-populated the runtime header and uploaded it into - // the pooled runtime_arena buffer. The distributed path reads the - // PTO2Runtime header directly from that arena. - void *prebuilt_arena = runtime->get_prebuilt_arena_base(); - size_t off_runtime = runtime->get_prebuilt_runtime_offset(); - if (prebuilt_arena == nullptr) { - LOG_ERROR("Thread %d: prebuilt_arena_base is null", thread_idx); - runtime_done_.store(true, std::memory_order_release); - return -1; - } - rt = reinterpret_cast(static_cast(prebuilt_arena) + off_runtime); + if (!dispatch_dist_run) { + for (int32_t i = 0; i < num_workers; i++) { + runtime->workers[i].aicpu_ready = AICPU_READY_DIST_ABORT; + cache_flush_range(const_cast(&runtime->workers[i].aicpu_ready), sizeof(uint32_t)); + } + } else { + // Build the entry-arg once per run. The AICore-side dist engine + // consumes it when replaying the linked orchestration entry. + runtime->dist.orch_args.create_from_chip_args(runtime->get_orch_args()); + const ChipStorageTaskArgs &orch_args = runtime->get_orch_args(); + runtime->dist.ccec_orch_tensor_count = orch_args.tensor_count(); + runtime->dist.ccec_orch_scalar_count = orch_args.scalar_count(); + for (int32_t i = 0; i < orch_args.tensor_count() && i < CHIP_MAX_TENSOR_ARGS; i++) { + Tensor::copy(runtime->dist.ccec_orch_tensors[i], orch_args.tensor(i)); + } + for (int32_t i = 0; i < orch_args.scalar_count() && i < CHIP_MAX_SCALAR_ARGS; i++) { + runtime->dist.ccec_orch_scalars[i] = orch_args.scalar(i); + } + cache_flush_range(runtime->dist.ccec_orch_tensors, sizeof(runtime->dist.ccec_orch_tensors)); + cache_flush_range(runtime->dist.ccec_orch_scalars, sizeof(runtime->dist.ccec_orch_scalars)); + cache_flush_range( + const_cast(&runtime->dist.ccec_orch_tensor_count), 2 * sizeof(int32_t) + ); -#if PTO2_PROFILING - if (get_l2_swimlane_level() >= L2SwimlaneLevel::ORCH_PHASES) { - l2_swimlane_aicpu_set_orch_thread_idx(thread_idx); - } -#endif + // rt is bound to *this* run's memory and must be reattached every + // run. + const ChipStorageTaskArgs &args = runtime->get_orch_args(); + int32_t arg_count = args.tensor_count() + args.scalar_count(); + LOG_INFO_V0("Thread %d: arg_count=%d", thread_idx, arg_count); + for (int32_t i = 0; i < args.tensor_count() && i < 20; i++) { + const Tensor &t = args.tensor(i); + LOG_INFO_V0( + "Thread %d: orch_args[%d] = TENSOR(data=0x%lx, ndims=%u, dtype=%u)", thread_idx, i, + static_cast(t.buffer.addr), t.ndims, static_cast(t.dtype) + ); + } + for (int32_t i = 0; i < args.scalar_count() && (args.tensor_count() + i) < 20; i++) { + LOG_INFO_V0( + "Thread %d: orch_args[%d] = SCALAR(0x%lx)", thread_idx, args.tensor_count() + i, + static_cast(args.scalar(i)) + ); + } -#if PTO2_PROFILING - orch_cycle_start = get_sys_cnt_aicpu(); -#endif - // ---- fully_distributed_within_core handoff ---- - // Instead of running orchestration here, wire the distributed engine - // (resets cursors/flags/heap) and wake the AICore worker threads, - // which call their own linked dist_core_main entry, replay - // orchestration in SPMD fashion, and execute the tasks they win. - // This AICPU thread then waits for all workers. - // See runtime/dist_engine.* and docs/fully_distributed_within_core.md. - { - const int32_t num_workers = runtime->worker_count; - dist_engine_register(rt, &runtime->dist.orch_args, num_workers, runtime); - runtime->dist.num_workers = num_workers; - __atomic_store_n(&runtime->dist.done_count, 0, __ATOMIC_RELEASE); - cache_flush_range(rt->dist_global, dist_engine_global_state_size()); - cache_flush_range(&runtime->dist, sizeof(runtime->dist)); - cache_flush_range(const_cast(&runtime->dist.num_workers), sizeof(int32_t)); - cache_flush_range(const_cast(&runtime->dist.done_count), sizeof(int64_t)); - uint64_t *regs = reinterpret_cast(get_platform_regs()); - if (regs == nullptr) { - LOG_ERROR("Thread %d: platform regs unavailable", thread_idx); + // Host has pre-populated the runtime header and uploaded it into + // the pooled runtime_arena buffer. The distributed path reads the + // PTO2Runtime header directly from that arena. + void *prebuilt_arena = runtime->get_prebuilt_arena_base(); + size_t off_runtime = runtime->get_prebuilt_runtime_offset(); + if (prebuilt_arena == nullptr) { + LOG_ERROR("Thread %d: prebuilt_arena_base is null", thread_idx); runtime_done_.store(true, std::memory_order_release); return -1; } - for (int32_t i = 0; i < num_workers; i++) { - const uint32_t physical_core_id = runtime->workers[i].physical_core_id; - write_reg(regs[physical_core_id], RegId::COND, AICORE_IDLE_VALUE); - } - for (int32_t i = 0; i < num_workers; i++) { - runtime->workers[i].aicpu_ready = AICPU_READY_DIST_RUN; - cache_flush_range(const_cast(&runtime->workers[i].aicpu_ready), sizeof(uint32_t)); + rt = reinterpret_cast(static_cast(prebuilt_arena) + off_runtime); + +#if PTO2_PROFILING + if (get_l2_swimlane_level() >= L2SwimlaneLevel::ORCH_PHASES) { + l2_swimlane_aicpu_set_orch_thread_idx(thread_idx); } - while (true) { - if (__atomic_load_n(&runtime->dist.done_count, __ATOMIC_ACQUIRE) >= num_workers) break; - bool all_cond_done = true; +#endif + +#if PTO2_PROFILING + orch_cycle_start = get_sys_cnt_aicpu(); + orch_cycle_started = true; +#endif + // ---- fully_distributed_within_core handoff ---- + // Instead of running orchestration here, wire the distributed engine + // (resets cursors/flags/heap) and wake the AICore worker threads, + // which call their own linked dist_core_main entry, replay + // orchestration in SPMD fashion, and execute the tasks they win. + // This AICPU thread then waits for all workers. + // See runtime/dist_engine.* and docs/fully_distributed_within_core.md. + { + const int32_t register_rc = + dist_engine_register(rt, &runtime->dist.orch_args, num_workers, runtime); + if (register_rc != 0) { + LOG_ERROR( + "Thread %d: distributed runtime configuration failed with status %d", thread_idx, + register_rc + ); + dispatch_dist_run = false; + run_rc = register_rc; + } + runtime->dist.num_workers = num_workers; + __atomic_store_n(&runtime->dist.done_count, 0, __ATOMIC_RELEASE); + if (dispatch_dist_run) { + cache_flush_range(rt->dist_global, dist_engine_global_state_size()); + } + cache_flush_range(&runtime->dist, sizeof(runtime->dist)); + cache_flush_range(const_cast(&runtime->dist.num_workers), sizeof(int32_t)); + cache_flush_range(const_cast(&runtime->dist.done_count), sizeof(int64_t)); + FdwicSubmitPmuHeader *submit_pmu_header = nullptr; + const bool submit_pmu_requested = + dispatch_dist_run && fdwic_submit_pmu_requested(runtime, &submit_pmu_header); + if (submit_pmu_requested && fdwic_submit_pmu_owner_configure(runtime, submit_pmu_header) != 0) { + LOG_ERROR( + "Thread %d: FDWIC submit-PMU owner configure failed; aborting dist replay", thread_idx + ); + // Configure 已经执行一次回滚；这里再做一次幂等恢复，优先 + // 避免失败诊断给后续本地设备运行留下 PMU 配置。 + (void)fdwic_submit_pmu_owner_restore(submit_pmu_header); + dispatch_dist_run = false; + run_rc = -1; + } + uint64_t *regs = reinterpret_cast(get_platform_regs()); + if (regs == nullptr) { + LOG_ERROR("Thread %d: platform regs unavailable", thread_idx); + runtime_done_.store(true, std::memory_order_release); + return -1; + } for (int32_t i = 0; i < num_workers; i++) { const uint32_t physical_core_id = runtime->workers[i].physical_core_id; - const uint64_t cond = read_reg(regs[physical_core_id], RegId::COND); - const bool done = cond == MAKE_FIN_VALUE(0); - all_cond_done = all_cond_done && done; + write_reg(regs[physical_core_id], RegId::COND, AICORE_IDLE_VALUE); } - if (all_cond_done) { - __atomic_store_n(&runtime->dist.done_count, num_workers, __ATOMIC_RELEASE); - break; + for (int32_t i = 0; i < num_workers; i++) { + runtime->workers[i].aicpu_ready = + dispatch_dist_run ? AICPU_READY_DIST_RUN : AICPU_READY_DIST_ABORT; + cache_flush_range( + const_cast(&runtime->workers[i].aicpu_ready), sizeof(uint32_t) + ); + } + if (dispatch_dist_run) { + while (true) { + if (__atomic_load_n(&runtime->dist.done_count, __ATOMIC_ACQUIRE) >= num_workers) break; + bool all_cond_done = true; + for (int32_t i = 0; i < num_workers; i++) { + const uint32_t physical_core_id = runtime->workers[i].physical_core_id; + const uint64_t cond = read_reg(regs[physical_core_id], RegId::COND); + const bool done = cond == MAKE_FIN_VALUE(0); + all_cond_done = all_cond_done && done; + } + if (all_cond_done) { + __atomic_store_n(&runtime->dist.done_count, num_workers, __ATOMIC_RELEASE); + break; + } + SPIN_WAIT_HINT(); + } + if (submit_pmu_requested) { + // 每个 worker 在发布 done 前已经 stop/read/flush 自己的 + // 整窗记录及可选 phase sidecar。所有 worker 完成后才 + // 恢复共享 PMU 配置。 + const int restore_rc = fdwic_submit_pmu_owner_restore(submit_pmu_header); + if (restore_rc != 0) { + // ownership bitmap 保留失败槽，允许一次幂等重试； + // 即使重试恢复，首次失败也会留在 header，正式 raw + // 仍被 host 拒绝。 + (void)fdwic_submit_pmu_owner_restore(submit_pmu_header); + run_rc = -1; + } + } + cache_invalidate_range(&runtime->fdwic_build_identity, sizeof(runtime->fdwic_build_identity)); + if ((runtime->fdwic_build_identity.error_bits & FdwicBuildErrorAicoreMismatch) != 0) { + LOG_ERROR( + "Thread %d: at least one AICore rejected the FDWIC build identity; " + "failing the run after clean worker completion", + thread_idx + ); + run_rc = -1; + } + const int32_t dist_status = dist_engine_runtime_status(rt); + if (dist_status != 0) { + LOG_ERROR( + "Thread %d: distributed AICore run failed with status %d", thread_idx, dist_status + ); + run_rc = dist_status; + } } - SPIN_WAIT_HINT(); } } + run_status_.store(run_rc, std::memory_order_release); runtime_done_.store(true, std::memory_order_release); } #if PTO2_PROFILING - uint64_t orch_end_ts = get_sys_cnt_aicpu(); - LOG_INFO_V9( - "Thread %d: orch_start=%" PRIu64 " orch_end=%" PRIu64 " orch_cost=%.3fus", thread_idx, - static_cast(orch_cycle_start), static_cast(orch_end_ts), - cycles_to_us(orch_end_ts - orch_cycle_start) - ); + if (orch_cycle_started) { + uint64_t orch_end_ts = get_sys_cnt_aicpu(); + LOG_INFO_V9( + "Thread %d: orch_start=%" PRIu64 " orch_end=%" PRIu64 " orch_cost=%.3fus", thread_idx, + static_cast(orch_cycle_start), static_cast(orch_end_ts), + cycles_to_us(orch_end_ts - orch_cycle_start) + ); + } #endif LOG_INFO_V0("Thread %d: Orchestrator completed", thread_idx); } else { @@ -350,6 +468,18 @@ int32_t AicpuExecutor::run(Runtime *runtime) { } } + const int32_t distributed_status = run_status_.load(std::memory_order_acquire); + if (distributed_status != 0) run_rc = distributed_status; + + // AICPU launches multiple threads, but the orchestrator alone owns the + // distributed handoff. Mirror its ABI/backend failure onto every AICPU + // return value instead of relying on undocumented multi-block return-code + // aggregation in the platform launcher. + cache_invalidate_range(&runtime->fdwic_build_identity, sizeof(runtime->fdwic_build_identity)); + if (__atomic_load_n(&runtime->fdwic_build_identity.error_bits, __ATOMIC_ACQUIRE) != FdwicBuildErrorNone) { + run_rc = -1; + } + // Shutdown is gated by runtime_done_: AICores only enter the EXIT wait // after completing the distributed replay, so sending EXIT earlier would // either race execution or time out while workers are still inside Phase 4. @@ -378,6 +508,7 @@ void AicpuExecutor::deinit(Runtime *runtime) { finished_count_.store(0, std::memory_order_release); runtime_done_.store(false, std::memory_order_release); + run_status_.store(0, std::memory_order_release); aicpu_thread_num_ = 0; sched_thread_num_ = 0; diff --git a/src/a5/runtime/fully_distributed_within_core/common/pto_runtime_status.h b/src/a5/runtime/fully_distributed_within_core/common/pto_runtime_status.h index e663ef4775..fd5b8fcb74 100644 --- a/src/a5/runtime/fully_distributed_within_core/common/pto_runtime_status.h +++ b/src/a5/runtime/fully_distributed_within_core/common/pto_runtime_status.h @@ -31,7 +31,11 @@ #define PTO2_ERROR_REQUIRE_SYNC_START_INVALID 7 #define PTO2_ERROR_TENSOR_WAIT_TIMEOUT 8 #define PTO2_ERROR_EXPLICIT_ORCH_FATAL 9 -#define PTO2_ERROR_SCOPE_TASKS_OVERFLOW 10 // scope_tasks buffer saturated (all rings full) +#define PTO2_ERROR_SCOPE_TASKS_OVERFLOW 10 // scope_tasks buffer saturated (all rings full) +#define PTO2_ERROR_TENSORMAP_CAPACITY 11 // TensorMap cannot retain another live producer region +#define PTO2_ERROR_DIST_CONFIG_INVALID 12 // Invalid fully-distributed runtime configuration +#define PTO2_ERROR_TENSORMAP_PROTOCOL 13 // Shared TensorMap rejected the task before publishing its entries +#define PTO2_ERROR_TENSORMAP_PARTIAL_PUBLISH 14 // Shared TensorMap failed after publishing some task state // Scheduler errors (100+): detected in scheduler threads #define PTO2_ERROR_SCHEDULER_TIMEOUT 100 diff --git a/src/a5/runtime/fully_distributed_within_core/docs/profiling_levels.md b/src/a5/runtime/fully_distributed_within_core/docs/profiling_levels.md index ec9d81e8f1..a78ecc8933 100644 --- a/src/a5/runtime/fully_distributed_within_core/docs/profiling_levels.md +++ b/src/a5/runtime/fully_distributed_within_core/docs/profiling_levels.md @@ -232,6 +232,56 @@ header just like on onboard. Bare `--enable-l2-swimlane` = level 4 (backward compatible). +### FDWIC schema-v4 partition semantics + +When FDWIC swimlane collection is enabled, every scalar core records +two adjacent top-level business windows: +`OrchestrationReplay` and `FinalDrain`. Each `Submit` inside the first +window has non-overlapping children with a path-specific order. Compete-first +paths use: + +- Kernel winner: `EfDrain` → `Claim` → `Materialize` → `PrepareMap` → + `Fanin` → `Register` → `WinnerBuild`. +- Kernel loser: `EfDrain` → `Claim` → `Materialize` → `PrepareMap` → + `Register` → `LoserReplay`. +- Alloc winner: `EfDrain` → `Claim` → `Materialize` → `PrepareMap` → + `Register` → `AllocComplete`. +- Alloc loser: `EfDrain` → `Claim` → `Materialize` → `PrepareMap` → + `Register`; its remaining tail is a residual because + production performs no loser action there. + +The still-supported one-shot APIs keep their original strict order: + +- Kernel: `EfDrain` → `Materialize` → `PrepareMap` → `Claim` → + optional `Fanin` → `Register` → winner/loser tail. +- Alloc: `EfDrain` → `Materialize` → `PrepareMap` → `Register` → + `Claim` → optional `AllocComplete`. + +The `Submit` record encodes winner state in `flags & 1` and task kind +in `aux`; tooling must not derive either from task-ID patterns. +`LoserReplay` is the real kernel-loser `drain_block_won()` call. +`DrainWon` remains a real nested observation, but it and other overlay +phases must not be added to an exclusive parent total. + +The Submit window starts after `dist_submit_begin()` and ends before +the final Submit-record write and API return. Residual time can include +unmarked control and trace-record publication overhead. In the Claim-first +path, the existing `Claim.end` to `Materialize.begin` residual includes +Claim-record publication, synchronous eager callback argument construction, +and the handoff into the finish path. No callback phase, timestamp, or raw +field is added. Exact closure is therefore an integrity property of the +instrumented capture rather than proof of zero observer effect. Schema-v4 +tooling validates both live API families against their exact sequence; it +does not accept arbitrary phase permutations. + +The Python converter validates schema-v4 and emits +`swimlane_exclusive_analysis.json`, whose integer-cycle closures cover +Submit, the inter-Submit envelope, orchestration, final drain, and the +complete worker window. The report distinguishes summed per-core work +from cross-core elapsed makespan. This partition work does not add an +I-cache/PMU metric; level-4 atomic observations remain independent +overlays. + ### Level gating in AICPU code Use the strongly-typed `L2SwimlaneLevel` enum so each gate names the diff --git a/src/a5/runtime/fully_distributed_within_core/host/fdwic_submit_pmu.cpp b/src/a5/runtime/fully_distributed_within_core/host/fdwic_submit_pmu.cpp new file mode 100644 index 0000000000..b777bd1412 --- /dev/null +++ b/src/a5/runtime/fully_distributed_within_core/host/fdwic_submit_pmu.cpp @@ -0,0 +1,689 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/platform_config.h" +#include "common/unified_log.h" +#include "dist_engine/common/submit_pmu_types.h" +#include "runtime.h" + +extern "C" void fdwic_swimlane_host_finalize(Runtime *runtime); + +namespace { + +constexpr double kPmuCyclesPerNsAll = 1.649844; +constexpr double kPmuCyclesPerNsAic = 1.650062; +constexpr double kPmuCyclesPerNsAiv = 1.649731; + +struct SubmitPmuProfile { + const char *name; + uint16_t mode; + FdwicSubmitPmuPhase phase; + size_t bytes; + const char *phase_name; + const char *phase_boundary; + const char *counter_semantics; + const char *time_semantics; +}; + +constexpr SubmitPmuProfile kSubmitPmuProfiles[] = { + {"submit-pmu-none", kFdwicSubmitPmuModeNone, FdwicSubmitPmuPhase::None, kFdwicSubmitPmuNoneBytes, nullptr, nullptr, + nullptr, nullptr}, + {"submit-pmu-arg-build", kFdwicSubmitPmuModeArgBuild, FdwicSubmitPmuPhase::ArgBuild, kFdwicSubmitPmuPhaseBytes, + "arg-build", "claim_end_to_materialize_begin", "running_read_clear_observed_bracket", + "boundary_diagnostic_sys_cnt_between_observers"}, + {"submit-pmu-empty-bracket", kFdwicSubmitPmuModeEmptyBracket, FdwicSubmitPmuPhase::EmptyBracket, + kFdwicSubmitPmuPhaseBytes, "empty-bracket", "claim_end_adjacent_empty_bracket", + "running_read_clear_empty_bracket_calibration", "boundary_diagnostic_outer_sys_cnt_around_adjacent_observer_pair"}, + {"submit-pmu-materialize", kFdwicSubmitPmuModeMaterialize, FdwicSubmitPmuPhase::Materialize, + kFdwicSubmitPmuPhaseBytes, "materialize", "materialize_begin_to_materialize_end", + "running_read_clear_observed_bracket", "boundary_diagnostic_sys_cnt_between_observers"}, + {"submit-pmu-claim", kFdwicSubmitPmuModeClaim, FdwicSubmitPmuPhase::Claim, kFdwicSubmitPmuPhaseBytes, "claim", + "claim_begin_to_claim_end", "running_read_clear_observed_bracket", + "boundary_diagnostic_sys_cnt_between_observers"}, + {"submit-pmu-register", kFdwicSubmitPmuModeRegister, FdwicSubmitPmuPhase::Register, kFdwicSubmitPmuPhaseBytes, + "register", "register_outputs_call_entry_to_return", "running_read_clear_observed_bracket", + "boundary_diagnostic_sys_cnt_between_observers"}, + {"submit-pmu-submit-transition", kFdwicSubmitPmuModeSubmitTransition, FdwicSubmitPmuPhase::SubmitTransition, + kFdwicSubmitPmuPhaseBytes, "submit-transition", "previous_submit_end_to_next_submit_begin", + "running_read_clear_observed_bracket", "boundary_diagnostic_sys_cnt_between_observers"}, + {"submit-pmu-efdrain-control", kFdwicSubmitPmuModeEfDrainControl, FdwicSubmitPmuPhase::EfDrainControl, + kFdwicSubmitPmuPhaseBytes, "efdrain-control", "efdrain_begin_to_end_excluding_linked_kernel_calls", + "discontinuous_running_read_clear_excluding_linked_kernel_calls", + "boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls"}, + {"submit-pmu-prepare-map", kFdwicSubmitPmuModePrepareMap, FdwicSubmitPmuPhase::PrepareMap, + kFdwicSubmitPmuPhaseBytes, "prepare-map", "dist_submit_prepare_map_call_entry_to_return", + "running_read_clear_observed_bracket", "boundary_diagnostic_sys_cnt_between_observers"}, + {"submit-pmu-fanin", kFdwicSubmitPmuModeFanin, FdwicSubmitPmuPhase::Fanin, kFdwicSubmitPmuPhaseBytes, "fanin", + "fanin_begin_to_fanin_end", "running_read_clear_observed_bracket", + "boundary_diagnostic_sys_cnt_between_observers"}, + {"submit-pmu-winner-build-control", kFdwicSubmitPmuModeWinnerBuild, FdwicSubmitPmuPhase::WinnerBuild, + kFdwicSubmitPmuPhaseBytes, "winner-build-control", "winner_build_begin_to_end_excluding_linked_kernel_calls", + "discontinuous_running_read_clear_excluding_linked_kernel_calls", + "boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls"}, + {"submit-pmu-alloc-complete-control", kFdwicSubmitPmuModeAllocComplete, FdwicSubmitPmuPhase::AllocComplete, + kFdwicSubmitPmuPhaseBytes, "alloc-complete-control", "alloc_complete_begin_to_end_excluding_linked_kernel_calls", + "discontinuous_running_read_clear_excluding_linked_kernel_calls", + "boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls"}, + {"submit-pmu-loser-replay", kFdwicSubmitPmuModeLoserReplay, FdwicSubmitPmuPhase::LoserReplay, + kFdwicSubmitPmuPhaseBytes, "loser-replay", "register_end_to_drain_block_won_return", + "running_read_clear_observed_bracket", "boundary_diagnostic_sys_cnt_between_observers"}, +}; + +const SubmitPmuProfile *requested_profile() { + const char *name = std::getenv("PTO_FDWIC_PROFILE"); + if (name == nullptr) return nullptr; + for (const SubmitPmuProfile &profile : kSubmitPmuProfiles) { + if (std::strcmp(name, profile.name) == 0) return &profile; + } + return nullptr; +} + +const FdwicSubmitPmuPhaseCoreData *phase_records(const FdwicSubmitPmuHeader &header) { + return reinterpret_cast( + reinterpret_cast(&header) + sizeof(FdwicSubmitPmuHeader) + ); +} + +std::string raw_path(const char *prefix) { + const std::string dir = prefix == nullptr || prefix[0] == '\0' ? "." : prefix; + return dir.back() == '/' ? dir + "fdwic_submit_pmu_raw.json" : dir + "/fdwic_submit_pmu_raw.json"; +} + +bool prepare_directory(const std::string &path) { + struct stat info {}; + if (stat(path.c_str(), &info) == 0) return S_ISDIR(info.st_mode); + return errno == ENOENT && mkdir(path.c_str(), 0755) == 0; +} + +bool remove_if_present(const std::string &path) { return std::remove(path.c_str()) == 0 || errno == ENOENT; } + +const char *role_name(CoreType role) { return role == CoreType::AIC ? "aic" : "aiv"; } + +bool physical_is_aic(uint32_t physical_id) { + return physical_id < kFdwicSubmitPmuPhysicalSubcores && physical_id % 54U < 18U; +} + +uint32_t bitmap_count(const uint32_t words[kFdwicSubmitPmuBitmapWords]) { + uint32_t count = 0; + for (uint32_t i = 0; i < kFdwicSubmitPmuBitmapWords; ++i) { + count += static_cast(__builtin_popcount(words[i])); + } + return count; +} + +bool bitmap_contains(const uint32_t words[kFdwicSubmitPmuBitmapWords], uint32_t physical_id) { + return physical_id < kFdwicSubmitPmuPhysicalSubcores && + (words[physical_id / 32U] & (1U << (physical_id % 32U))) != 0; +} + +struct MetricStats { + uint64_t sum{0}; + uint64_t min{std::numeric_limits::max()}; + uint64_t max{0}; + + void add(uint64_t value) { + sum += value; + min = std::min(min, value); + max = std::max(max, value); + } +}; + +struct GroupStats { + uint32_t cores{0}; + MetricStats total; + MetricStats scalar_elapsed; + MetricStats scalar; + MetricStats requests; + MetricStats misses; + + void add(const FdwicSubmitPmuCoreData &core) { + ++cores; + total.add(core.total_cycles); + scalar_elapsed.add(core.scalar_submit_elapsed_ticks); + scalar.add(core.scalar_busy); + requests.add(core.icache_requests); + misses.add(core.icache_misses); + } +}; + +void write_metric(std::ofstream &out, const char *name, const MetricStats &metric, uint32_t cores, bool comma) { + out << " \"" << name << "\": {\"sum\": " << metric.sum << ", \"min\": " << metric.min + << ", \"mean\": " << static_cast(metric.sum) / cores << ", \"max\": " << metric.max << "}" + << (comma ? "," : "") << "\n"; +} + +void write_group(std::ofstream &out, const char *name, const GroupStats &group, bool comma) { + out << " \"" << name << "\": {\n"; + out << " \"cores\": " << group.cores << ",\n"; + write_metric(out, "total_cycles", group.total, group.cores, true); + write_metric(out, "scalar_submit_elapsed_ticks", group.scalar_elapsed, group.cores, true); + write_metric(out, "scalar_busy", group.scalar, group.cores, true); + write_metric(out, "icache_requests", group.requests, group.cores, true); + write_metric(out, "icache_misses", group.misses, group.cores, true); + out << " \"scalar_busy_share\": " + << static_cast(group.scalar.sum) / static_cast(group.total.sum) << ",\n"; + out << " \"icache_miss_rate\": " + << static_cast(group.misses.sum) / static_cast(group.requests.sum) << "\n"; + out << " }" << (comma ? "," : "") << "\n"; +} + +struct ValidatedData { + uint32_t expected_submits{0}; + uint64_t phase_calls_all{0}; + uint64_t phase_calls_aic{0}; + uint64_t phase_calls_aiv{0}; + uint64_t global_start{std::numeric_limits::max()}; + uint64_t global_end{0}; + uint32_t bitmap_words[kFdwicSubmitPmuBitmapWords]{}; + GroupStats all; + GroupStats aic; + GroupStats aiv; +}; + +bool validate( + Runtime *runtime, const FdwicSubmitPmuHeader &header, const SubmitPmuProfile &profile, ValidatedData &data +) { + const bool fixed_header = + header.magic == kFdwicSubmitPmuMagic && header.version == kFdwicSubmitPmuVersion && + header.mode == profile.mode && header.header_bytes == profile.bytes && + header.record_bytes == sizeof(FdwicSubmitPmuCoreData) && header.num_cores == kFdwicSubmitPmuExpectedCores && + header.expected_aic == kFdwicSubmitPmuExpectedAic && header.expected_aiv == kFdwicSubmitPmuExpectedAiv && + header.sys_cnt_freq_hz == PLATFORM_PROF_SYS_CNT_FREQ && header.selectors[0] == kFdwicSubmitPmuCnt2ScalarBusy && + header.selectors[1] == kFdwicSubmitPmuCnt3ShadowScalarBusy && + header.selectors[2] == kFdwicSubmitPmuCnt5ShadowIcacheMiss && + header.selectors[3] == kFdwicSubmitPmuCnt6IcacheRequest && + header.selectors[4] == kFdwicSubmitPmuCnt7IcacheMiss && + header.selectors[5] == kFdwicSubmitPmuCnt8ShadowIcacheRequest && runtime != nullptr && + runtime->worker_count == static_cast(kFdwicSubmitPmuExpectedCores) && + runtime->fdwic_swimlane_bytes_ == profile.bytes && + runtime->dist.swimlane_base == runtime->fdwic_swimlane_dev_base_ && runtime->dist.swimlane_level == 0 && + runtime->dist.swimlane_records_per_core == 0; + if (!fixed_header) { + LOG_ERROR("fdwic submit-PMU fixed header/state validation failed"); + return false; + } + + uint32_t owner_words[kFdwicSubmitPmuBitmapWords]{}; + for (uint32_t word = 0; word < kFdwicSubmitPmuBitmapWords; ++word) { + owner_words[word] = header.configured_bitmap_words[word]; + data.bitmap_words[word] = owner_words[word]; + } + const bool owner_valid = + header.owner_status == kFdwicSubmitPmuRequiredOwnerStatus && + header.configured_count == kFdwicSubmitPmuExpectedCores && + header.restored_count == kFdwicSubmitPmuExpectedCores && header.configured_aic == kFdwicSubmitPmuExpectedAic && + header.configured_aiv == kFdwicSubmitPmuExpectedAiv && + header.complete_mixed_triplets == kFdwicSubmitPmuExpectedAic && header.restore_failures == 0 && + header.active_after_restore == 0 && bitmap_count(owner_words) == kFdwicSubmitPmuExpectedCores && + header.first_failure_field == static_cast(FdwicSubmitPmuOwnerField::None); + if (!owner_valid) { + LOG_ERROR( + "fdwic submit-PMU owner closure failed: status=0x%x configured=%u restored=%u active=%u failures=%u", + header.owner_status, header.configured_count, header.restored_count, header.active_after_restore, + header.restore_failures + ); + return false; + } + + const bool phase_mode = fdwic_submit_pmu_mode_has_phase(profile.mode); + const FdwicSubmitPmuPhaseCoreData *phases = phase_mode ? phase_records(header) : nullptr; + bool physical_seen[kFdwicSubmitPmuPhysicalSubcores]{}; + uint32_t aic_count = 0; + uint32_t aiv_count = 0; + for (uint32_t logical = 0; logical < kFdwicSubmitPmuExpectedCores; ++logical) { + const FdwicSubmitPmuCoreData &core = header.cores[logical]; + const CoreType role = runtime->workers[logical].core_type; + const uint32_t expected_block = role == CoreType::AIC ? aic_count : aiv_count / 2U; + const uint32_t expected_lane = role == CoreType::AIC ? 0U : 1U + aiv_count % 2U; + if (role == CoreType::AIC) { + ++aic_count; + } else if (role == CoreType::AIV) { + ++aiv_count; + } else { + LOG_ERROR("fdwic submit-PMU logical core %u has invalid role", logical); + return false; + } + const uint32_t physical = core.physical_core_id; + const bool physical_in_range = physical < kFdwicSubmitPmuPhysicalSubcores; + const bool physical_duplicate = physical_in_range && physical_seen[physical]; + const bool owner_configured = physical_in_range && bitmap_contains(owner_words, physical); + const bool role_matches_physical = physical_in_range && physical_is_aic(physical) == (role == CoreType::AIC); + const bool identity_valid = core.logical_core_id == logical && physical_in_range && !physical_duplicate && + role_matches_physical && core.block_id == expected_block && + core.lane == expected_lane && owner_configured; + const bool count_valid = core.expected_submit_count != 0 && core.submit_count == core.expected_submit_count && + (data.expected_submits == 0 || core.expected_submit_count == data.expected_submits); + const bool wall_window_valid = + core.first_submit_start_tick != 0 && core.last_submit_end_tick >= core.first_submit_start_tick; + const uint64_t wall_elapsed_ticks = + wall_window_valid ? core.last_submit_end_tick - core.first_submit_start_tick : 0; + const bool scalar_elapsed_valid = + core.scalar_submit_elapsed_ticks != 0 && core.scalar_submit_elapsed_ticks <= wall_elapsed_ticks; + const bool window_valid = wall_window_valid && core.total_cycles != 0; + const bool counters_valid = core.scalar_busy <= core.total_cycles && core.icache_requests != 0 && + core.icache_misses <= core.icache_requests && + core.scalar_busy < kFdwicSubmitPmuCounterRiskThreshold && + core.icache_requests < kFdwicSubmitPmuCounterRiskThreshold && + core.icache_misses < kFdwicSubmitPmuCounterRiskThreshold; + const uint32_t required_core_status = fdwic_submit_pmu_required_core_status(profile.phase); + const bool status_valid = core.status == required_core_status; + if (!identity_valid || !count_valid || !window_valid || !scalar_elapsed_valid || !counters_valid || + !status_valid) { + LOG_ERROR( + "fdwic submit-PMU core %u closure failed: physical=%u block=%u lane=%u count=%u/%u " + "ticks=%llu..%llu scalar/wall=%llu/%llu total=%llu scalar_busy=%u req=%u miss=%u " + "status=0x%x/0x%x gates(identity/count/window/scalar/counters/status)=%u/%u/%u/%u/%u/%u " + "expected_block/lane=%u/%u role=%d physical_in_range/duplicate/bitmap=%u/%u/%u", + logical, physical, core.block_id, core.lane, core.submit_count, core.expected_submit_count, + static_cast(core.first_submit_start_tick), + static_cast(core.last_submit_end_tick), + static_cast(core.scalar_submit_elapsed_ticks), + static_cast(wall_elapsed_ticks), static_cast(core.total_cycles), + core.scalar_busy, core.icache_requests, core.icache_misses, core.status, required_core_status, + identity_valid, count_valid, window_valid, scalar_elapsed_valid, counters_valid, status_valid, + expected_block, expected_lane, static_cast(role), physical_in_range, physical_duplicate, + owner_configured + ); + return false; + } + if (phase_mode) { + const FdwicSubmitPmuPhaseCoreData &phase = phases[logical]; + const uint32_t expected_phase_calls = + fdwic_submit_pmu_expected_phase_calls(profile.phase, core.expected_submit_count); + const bool dynamic_calls = fdwic_submit_pmu_phase_has_dynamic_calls(profile.phase); + const uint32_t excluded_kernel_calls = phase.excluded_kernel_calls; + const uint32_t phase_shadow_scalar = phase.shadow_scalar_busy; + const uint32_t phase_shadow_requests = phase.shadow_icache_requests; + const uint32_t phase_shadow_misses = phase.shadow_icache_misses; + const uint64_t expected_boundary_reads = fdwic_submit_pmu_expected_phase_boundary_reads( + profile.phase, core.expected_submit_count, excluded_kernel_calls + ); + const bool outer_reads_valid = + phase.phase_begin_reads >= excluded_kernel_calls && phase.phase_end_reads >= excluded_kernel_calls; + const uint32_t outer_begin_reads = + outer_reads_valid ? phase.phase_begin_reads - excluded_kernel_calls : UINT32_MAX; + const uint32_t outer_end_reads = + outer_reads_valid ? phase.phase_end_reads - excluded_kernel_calls : UINT32_MAX; + const bool boundary_shape_valid = + dynamic_calls ? + outer_reads_valid && outer_begin_reads == outer_end_reads : + expected_phase_calls != 0 && outer_begin_reads == expected_phase_calls && + outer_end_reads == expected_phase_calls && phase.phase_begin_reads == expected_boundary_reads && + phase.phase_end_reads == expected_boundary_reads; + const bool dynamic_count_valid = + !dynamic_calls || + outer_begin_reads <= + fdwic_submit_pmu_dynamic_calls_max_per_core(profile.phase, core.expected_submit_count); + const bool zero_call_dynamic = dynamic_calls && outer_begin_reads == 0; + const bool elapsed_valid = + zero_call_dynamic ? + phase.phase_elapsed_ticks == 0 : + phase.phase_elapsed_ticks != 0 && phase.phase_elapsed_ticks <= core.scalar_submit_elapsed_ticks; + const bool zero_values_valid = + !zero_call_dynamic || + (phase.phase_total_cycles_observed == 0 && phase.phase_scalar_busy_observed == 0 && + phase.phase_icache_requests_observed == 0 && phase.phase_icache_misses_observed == 0); + const bool shadow_icache_primary_bounded = phase_shadow_misses <= phase_shadow_requests && + phase_shadow_requests <= core.icache_requests && + phase_shadow_misses <= core.icache_misses; + // CNT3 在 running phase 中会被边界反复 read-clear，同周期递增可能形成少量 + // shadow loss，因此这里只允许 shadow<=CNT2 primary；none 的单次尾读仍由 + // core status 要求逐核精确相等。 + const bool shadow_scalar_primary_bounded = phase_shadow_scalar <= core.scalar_busy; + // phase mode 的 core.total_cycles 是所有 TOTAL read-clear chunk 的软件重建 + // whole。局部 total/scalar 只在同一 ELF 内作 observed 配对，不与其他 + // phase 或 submit-pmu-none 相加、相减。 + const bool phase_pmu_values_ordered = + phase.phase_total_cycles_observed <= core.total_cycles && + phase.phase_scalar_busy_observed <= phase.phase_total_cycles_observed && + phase.phase_scalar_busy_observed <= phase_shadow_scalar; + const bool phase_valid = phase.phase_id == static_cast(profile.phase) && boundary_shape_valid && + dynamic_count_valid && elapsed_valid && zero_values_valid && + (!zero_call_dynamic ? phase.phase_total_cycles_observed != 0 : true) && + phase.phase_icache_misses_observed <= phase.phase_icache_requests_observed && + phase.phase_icache_requests_observed <= phase_shadow_requests && + phase.phase_icache_misses_observed <= phase_shadow_misses && + shadow_icache_primary_bounded && shadow_scalar_primary_bounded && + phase_pmu_values_ordered && + phase_shadow_scalar < kFdwicSubmitPmuCounterRiskThreshold && + phase_shadow_requests < kFdwicSubmitPmuCounterRiskThreshold && + phase_shadow_misses < kFdwicSubmitPmuCounterRiskThreshold && + phase.status == kFdwicSubmitPmuRequiredPhaseStatus; + if (!phase_valid) { + LOG_ERROR( + "fdwic submit-PMU phase core %u closure failed: id=%u reads=%u/%u outer=%u/%u " + "expected_outer/boundary=%u/%llu excluded_kernel=%u scalar/wall=%llu/%llu " + "phase_ticks=%llu phase_pmu(total/scalar)=%llu/%u shadow/primary_scalar=%u/%u " + "observed_icache=%llu/%llu shadow_icache=%u/%u primary_icache=%u/%u status=0x%x", + logical, phase.phase_id, phase.phase_begin_reads, phase.phase_end_reads, outer_begin_reads, + outer_end_reads, expected_phase_calls, static_cast(expected_boundary_reads), + excluded_kernel_calls, static_cast(core.scalar_submit_elapsed_ticks), + static_cast(wall_elapsed_ticks), + static_cast(phase.phase_elapsed_ticks), + static_cast(phase.phase_total_cycles_observed), + phase.phase_scalar_busy_observed, phase_shadow_scalar, core.scalar_busy, + static_cast(phase.phase_icache_requests_observed), + static_cast(phase.phase_icache_misses_observed), phase_shadow_requests, + phase_shadow_misses, core.icache_requests, core.icache_misses, phase.status + ); + return false; + } + if (dynamic_calls) { + data.phase_calls_all += outer_begin_reads; + (role == CoreType::AIC ? data.phase_calls_aic : data.phase_calls_aiv) += outer_begin_reads; + } + } + physical_seen[physical] = true; + if (data.expected_submits == 0) data.expected_submits = core.expected_submit_count; + data.global_start = std::min(data.global_start, core.first_submit_start_tick); + data.global_end = std::max(data.global_end, core.last_submit_end_tick); + data.all.add(core); + (role == CoreType::AIC ? data.aic : data.aiv).add(core); + } + if (aic_count != kFdwicSubmitPmuExpectedAic || aiv_count != kFdwicSubmitPmuExpectedAiv || + data.global_start == std::numeric_limits::max() || data.global_end < data.global_start) { + LOG_ERROR("fdwic submit-PMU topology/window aggregate closure failed"); + return false; + } + if (fdwic_submit_pmu_phase_has_dynamic_calls(profile.phase)) { + const uint64_t expected_all = fdwic_submit_pmu_expected_dynamic_calls_all(profile.phase, data.expected_submits); + const bool role_calls_fixed = fdwic_submit_pmu_dynamic_calls_have_fixed_roles(profile.phase); + const uint64_t expected_aic = + role_calls_fixed ? fdwic_submit_pmu_expected_dynamic_calls_aic(profile.phase, data.expected_submits) : 0U; + const uint64_t expected_aiv = + role_calls_fixed ? fdwic_submit_pmu_expected_dynamic_calls_aiv(profile.phase, data.expected_submits) : 0U; + const bool global_closed = expected_all != 0 && data.phase_calls_all == expected_all && + data.phase_calls_aic + data.phase_calls_aiv == data.phase_calls_all; + const bool roles_closed = + !role_calls_fixed || (data.phase_calls_aic == expected_aic && data.phase_calls_aiv == expected_aiv); + if (!global_closed || !roles_closed) { + LOG_ERROR( + "fdwic submit-PMU dynamic phase call closure failed: actual(all/aic/aiv)=%llu/%llu/%llu " + "expected_all=%llu fixed_roles=%u expected_aic/aiv=%llu/%llu", + static_cast(data.phase_calls_all), + static_cast(data.phase_calls_aic), + static_cast(data.phase_calls_aiv), static_cast(expected_all), + static_cast(role_calls_fixed), static_cast(expected_aic), + static_cast(expected_aiv) + ); + return false; + } + } + + uint32_t complete_triplets = 0; + for (uint32_t physical = 0; physical < kFdwicSubmitPmuPhysicalSubcores; ++physical) { + if (!physical_seen[physical] || !physical_is_aic(physical)) continue; + const uint32_t die_base = physical / 54U * 54U; + const uint32_t aic_local = physical % 54U; + const uint32_t aiv0 = die_base + 18U + 2U * aic_local; + const uint32_t aiv1 = aiv0 + 1U; + if (aiv1 >= kFdwicSubmitPmuPhysicalSubcores || !physical_seen[aiv0] || !physical_seen[aiv1]) { + LOG_ERROR("fdwic submit-PMU physical triplet is incomplete for AIC %u", physical); + return false; + } + ++complete_triplets; + } + if (complete_triplets != kFdwicSubmitPmuExpectedAic || complete_triplets != header.complete_mixed_triplets) { + LOG_ERROR("fdwic submit-PMU physical triplet count mismatch: %u", complete_triplets); + return false; + } + return true; +} + +} // namespace + +extern "C" int fdwic_submit_pmu_host_init(Runtime *runtime, int num_cores, const char *output_prefix) { + const SubmitPmuProfile *profile = requested_profile(); + if (profile == nullptr) return 0; + if (runtime == nullptr || num_cores != static_cast(kFdwicSubmitPmuExpectedCores) || + runtime->worker_count != static_cast(kFdwicSubmitPmuExpectedCores)) { + return -1; + } + const bool storage_empty = runtime->fdwic_swimlane_host_shadow_ == nullptr && + runtime->fdwic_swimlane_dev_allocation_ == 0 && runtime->fdwic_swimlane_dev_base_ == 0 && + runtime->fdwic_swimlane_bytes_ == 0 && runtime->dist.swimlane_base == 0 && + runtime->dist.swimlane_level == 0 && runtime->dist.swimlane_records_per_core == 0; + if (!storage_empty || runtime->host_api.device_malloc == nullptr || runtime->host_api.device_free == nullptr || + runtime->host_api.copy_to_device == nullptr || runtime->host_api.copy_from_device == nullptr) { + return -1; + } + const std::string prefix = output_prefix == nullptr || output_prefix[0] == '\0' ? "." : output_prefix; + if (prefix.size() >= sizeof(runtime->fdwic_swimlane_output_prefix_) || !prepare_directory(prefix)) return -1; + const std::string path = raw_path(prefix.c_str()); + if (!remove_if_present(path) || !remove_if_present(path + ".tmp")) return -1; + + void *shadow = std::aligned_alloc(64, profile->bytes); + if (shadow == nullptr) return -1; + std::memset(shadow, 0, profile->bytes); + auto *header = reinterpret_cast(shadow); + header->magic = kFdwicSubmitPmuMagic; + header->version = kFdwicSubmitPmuVersion; + header->mode = profile->mode; + header->header_bytes = static_cast(profile->bytes); + header->record_bytes = sizeof(FdwicSubmitPmuCoreData); + header->num_cores = kFdwicSubmitPmuExpectedCores; + header->expected_aic = kFdwicSubmitPmuExpectedAic; + header->expected_aiv = kFdwicSubmitPmuExpectedAiv; + header->sys_cnt_freq_hz = PLATFORM_PROF_SYS_CNT_FREQ; + header->selectors[0] = kFdwicSubmitPmuCnt2ScalarBusy; + header->selectors[1] = kFdwicSubmitPmuCnt3ShadowScalarBusy; + header->selectors[2] = kFdwicSubmitPmuCnt5ShadowIcacheMiss; + header->selectors[3] = kFdwicSubmitPmuCnt6IcacheRequest; + header->selectors[4] = kFdwicSubmitPmuCnt7IcacheMiss; + header->selectors[5] = kFdwicSubmitPmuCnt8ShadowIcacheRequest; + + constexpr uintptr_t kAlignment = 64; + void *allocation = runtime->host_api.device_malloc(profile->bytes + kAlignment - 1); + if (allocation == nullptr) { + std::free(shadow); + return -1; + } + const uintptr_t base = (reinterpret_cast(allocation) + kAlignment - 1) & ~(kAlignment - 1); + if (runtime->host_api.copy_to_device(reinterpret_cast(base), shadow, profile->bytes) != 0) { + runtime->host_api.device_free(allocation); + std::free(shadow); + return -1; + } + runtime->fdwic_swimlane_host_shadow_ = shadow; + runtime->fdwic_swimlane_dev_allocation_ = reinterpret_cast(allocation); + runtime->fdwic_swimlane_dev_base_ = base; + runtime->fdwic_swimlane_bytes_ = profile->bytes; + runtime->fdwic_swimlane_num_cores_ = kFdwicSubmitPmuExpectedCores; + runtime->fdwic_swimlane_records_per_core_ = 0; + std::memcpy(runtime->fdwic_swimlane_output_prefix_, prefix.c_str(), prefix.size() + 1); + runtime->dist.swimlane_base = base; + runtime->dist.swimlane_level = 0; + runtime->dist.swimlane_records_per_core = 0; + return 1; +} + +extern "C" int fdwic_submit_pmu_host_export(Runtime *runtime) { + const SubmitPmuProfile *profile = requested_profile(); + if (profile == nullptr || runtime == nullptr || runtime->fdwic_swimlane_host_shadow_ == nullptr || + runtime->fdwic_swimlane_dev_base_ == 0) { + return 0; + } + if (runtime->host_api.copy_from_device( + runtime->fdwic_swimlane_host_shadow_, reinterpret_cast(runtime->fdwic_swimlane_dev_base_), + profile->bytes + ) != 0) { + LOG_ERROR("fdwic submit-PMU D2H copy failed"); + return -1; + } + const auto &header = *reinterpret_cast(runtime->fdwic_swimlane_host_shadow_); + ValidatedData data; + if (!validate(runtime, header, *profile, data)) return -1; + + const std::string path = raw_path(runtime->fdwic_swimlane_output_prefix_); + const std::string temporary = path + ".tmp"; + if (!remove_if_present(temporary)) return -1; + std::ofstream out(temporary, std::ios::out | std::ios::trunc); + if (!out.is_open()) return -1; + out << std::setprecision(12); + const uint64_t global_span = data.global_end - data.global_start; + const bool phase_mode = fdwic_submit_pmu_mode_has_phase(profile->mode); + const FdwicSubmitPmuPhaseCoreData *phases = phase_mode ? phase_records(header) : nullptr; + out << "{\n"; + out << " \"schema\": \"fdwic-submit-pmu-v3\",\n"; + out << " \"capture\": {\"mode\": \"" << profile->name + << "\", " + "\"window_scope\": \"per_core_first_submit_begin_to_last_submit_end\", " + "\"accepted\": true, \"owner_restore_passed\": true},\n"; + out << " \"configuration\": {\n"; + out << " \"num_cores\": 96, \"aic_cores\": 32, \"aiv_cores\": 64,\n"; + out << " \"expected_submits_per_core\": " << data.expected_submits << ",\n"; + out << " \"sys_counter_tick_ns\": 1,\n"; + out << " \"selectors\": {\"cnt0_vector_busy\": " << kFdwicSubmitPmuCnt0VectorBusy + << ", \"cnt1_cube_busy\": " << kFdwicSubmitPmuCnt1CubeBusy + << ", \"cnt2_scalar_busy\": " << kFdwicSubmitPmuCnt2ScalarBusy + << ", \"cnt3_shadow_scalar_busy\": " << kFdwicSubmitPmuCnt3ShadowScalarBusy + << ", \"cnt5_shadow_icache_miss\": " << kFdwicSubmitPmuCnt5ShadowIcacheMiss + << ", \"cnt6_primary_icache_request\": " << kFdwicSubmitPmuCnt6IcacheRequest + << ", \"cnt7_primary_icache_miss\": " << kFdwicSubmitPmuCnt7IcacheMiss + << ", \"cnt8_shadow_icache_request\": " << kFdwicSubmitPmuCnt8ShadowIcacheRequest << "},\n"; + out << " \"status_required_mask\": " << fdwic_submit_pmu_required_core_status(profile->phase) << ",\n"; + out << " \"linked_kernel_exclusion\": {\"enabled\": true, " + "\"boundary\": \"dist_aicore_call_slot_kernel_entry_to_return\", " + "\"gate_semantics\": \"metrics_prof_stop_before_call_and_start_after_return\", " + "\"time_denominator\": \"scalar_submit_elapsed_ticks\", " + "\"wall_tick_semantics\": \"first_submit_start_to_last_submit_end_closure_only\"},\n"; + out << " \"return_ready_atomic_exclusion\": {\"enabled\": true, " + "\"classification\": \"result_used_atomic_only\", " + "\"time_boundary\": \"sys_cnt_before_atomic_to_result_dependent_sys_cnt_after_return\", " + "\"counter_semantics\": \"pmu_counters_include_atomic_instruction_events\", " + "\"time_denominator_effect\": \"subtract_return_ready_atomic_elapsed\"},\n"; + out << " \"counter_width_bits\": {\"total\": 64, \"programmable\": 32},\n"; + out << " \"programmable_counter_risk_threshold\": " << kFdwicSubmitPmuCounterRiskThreshold << ",\n"; + if (phase_mode) { + out << " \"phase\": {\"id\": " << static_cast(profile->phase) << ", \"name\": \"" + << profile->phase_name << "\", " << "\"boundary\": \"" << profile->phase_boundary << "\", "; + if (fdwic_submit_pmu_phase_has_dynamic_calls(profile->phase)) { + const uint64_t expected_all = + fdwic_submit_pmu_expected_dynamic_calls_all(profile->phase, data.expected_submits); + if (fdwic_submit_pmu_dynamic_calls_have_fixed_roles(profile->phase)) { + const uint64_t expected_aic = + fdwic_submit_pmu_expected_dynamic_calls_aic(profile->phase, data.expected_submits); + const uint64_t expected_aiv = + fdwic_submit_pmu_expected_dynamic_calls_aiv(profile->phase, data.expected_submits); + out << "\"call_shape\": \"dynamic_balanced\", \"expected_calls\": {\"all\": " << expected_all + << ", \"aic\": " << expected_aic << ", \"aiv\": " << expected_aiv << "}, "; + } else { + out << "\"call_shape\": \"dynamic_global\", \"expected_calls\": {\"all\": " << expected_all << "}, "; + } + } else { + const uint32_t expected_phase_calls = + fdwic_submit_pmu_expected_phase_calls(profile->phase, data.expected_submits); + out << "\"expected_calls_per_core\": " << expected_phase_calls << ", "; + } + out << "\"status_required_mask\": " << kFdwicSubmitPmuRequiredPhaseStatus << ", \"counter_semantics\": \"" + << profile->counter_semantics << "\", " << "\"time_semantics\": \"" << profile->time_semantics + << "\", \"pmu_observation\": {" + "\"total_cycles\": \"running_read_clear_observed_with_software_reconstructed_whole\", " + "\"scalar_busy\": \"cnt3_running_read_clear_observed_bounded_by_cnt2_primary\", " + "\"non_scalar_busy\": \"per_core_phase_total_cycles_observed_minus_phase_scalar_busy_observed\", " + "\"sys_cnt_role\": \"boundary_diagnostic_only_not_primary_phase_timing\"}},\n"; + } + out << " \"pmu_cycles_per_ns\": {\"all\": " << kPmuCyclesPerNsAll << ", \"aic\": " << kPmuCyclesPerNsAic + << ", \"aiv\": " << kPmuCyclesPerNsAiv << "}\n"; + out << " },\n"; + out << " \"owner\": {\"configure_passed\": true, \"restore_passed\": true, " + "\"configured_count\": 96, \"configured_aic\": 32, \"configured_aiv\": 64, " + "\"restored_count\": 96, \"active_after_restore\": 0, \"restore_failures\": 0, " + "\"configured_bitmap_words\": ["; + for (uint32_t word = 0; word < kFdwicSubmitPmuBitmapWords; ++word) { + if (word != 0) out << ", "; + out << data.bitmap_words[word]; + } + out << "], \"complete_mixed_triplets\": 32},\n"; + out << " \"window\": {\"global_first_submit_start_tick\": " << data.global_start + << ", \"global_last_submit_end_tick\": " << data.global_end << ", \"global_submit_span_ticks\": " << global_span + << ", \"global_submit_span_us\": " << static_cast(global_span) / 1000.0 << "},\n"; + out << " \"records\": [\n"; + for (uint32_t logical = 0; logical < kFdwicSubmitPmuExpectedCores; ++logical) { + const auto &core = header.cores[logical]; + const char *role = role_name(runtime->workers[logical].core_type); + out << " {\"logical_core_id\": " << core.logical_core_id + << ", \"physical_core_id\": " << core.physical_core_id << ", \"role\": \"" << role + << "\", \"block_id\": " << core.block_id << ", \"lane\": " << core.lane + << ", \"submit_count\": " << core.submit_count + << ", \"expected_submit_count\": " << core.expected_submit_count + << ", \"first_submit_start_tick\": " << core.first_submit_start_tick + << ", \"last_submit_end_tick\": " << core.last_submit_end_tick + << ", \"submit_elapsed_ticks\": " << core.last_submit_end_tick - core.first_submit_start_tick + << ", \"scalar_submit_elapsed_ticks\": " << core.scalar_submit_elapsed_ticks + << ", \"total_cycles\": " << core.total_cycles << ", \"scalar_busy\": " << core.scalar_busy + << ", \"icache_requests\": " << core.icache_requests << ", \"icache_misses\": " << core.icache_misses + << ", \"status\": " << core.status; + if (phase_mode) { + const FdwicSubmitPmuPhaseCoreData &phase = phases[logical]; + out << ", \"phase_id\": " << phase.phase_id << ", \"phase_elapsed_ticks\": " << phase.phase_elapsed_ticks + << ", \"phase_total_cycles_observed\": " << phase.phase_total_cycles_observed + << ", \"phase_scalar_busy_observed\": " << phase.phase_scalar_busy_observed + << ", \"phase_icache_requests_observed\": " << phase.phase_icache_requests_observed + << ", \"phase_icache_misses_observed\": " << phase.phase_icache_misses_observed + << ", \"phase_begin_reads\": " << phase.phase_begin_reads + << ", \"phase_end_reads\": " << phase.phase_end_reads << ", \"phase_status\": " << phase.status + << ", \"phase_excluded_kernel_calls\": " << phase.excluded_kernel_calls + << ", \"shadow_scalar_busy\": " << phase.shadow_scalar_busy + << ", \"shadow_icache_requests\": " << phase.shadow_icache_requests + << ", \"shadow_icache_misses\": " << phase.shadow_icache_misses; + } + out << "}" << (logical + 1U == kFdwicSubmitPmuExpectedCores ? "\n" : ",\n"); + } + out << " ],\n"; + out << " \"validation\": {\"passed\": true, \"trusted_records\": 96, " + "\"unique_physical_core_ids\": 96, \"aic_records\": 32, \"aiv_records\": 64, " + "\"mixed_triplets\": 32, \"owner_bitmap_member_records\": 96, \"status_match_records\": 96, " + "\"selector_match_records\": 96, \"window_started_records\": 96, " + "\"window_stopped_records\": 96, \"submit_count_closed_records\": 96, " + "\"scalar_le_total_records\": 96, \"linked_kernel_gate_closed_records\": 96, " + "\"scalar_submit_elapsed_valid_records\": 96, \"vector_busy_zero_records\": 96, " + "\"cube_busy_zero_records\": 96, \"return_ready_atomic_time_valid_records\": 96, "; + if (phase_mode) { + out << "\"shadow_icache_primary_bounded_records\": 96, " + "\"shadow_scalar_primary_bounded_records\": 96, \"phase_boundary_closed_records\": 96, " + "\"phase_shape_match_records\": 96, \"phase_icache_values_ordered_records\": 96, " + "\"phase_pmu_values_ordered_records\": 96, \"phase_counter_reconstruction_valid_records\": 96, " + "\"phase_time_within_submit_records\": 96, \"phase_kernel_exclusion_closed_records\": 96, "; + if (fdwic_submit_pmu_phase_has_dynamic_calls(profile->phase)) { + out << "\"phase_global_call_count_closed\": true, "; + } + } else { + out << "\"shadow_icache_primary_match_records\": 96, \"shadow_scalar_primary_match_records\": 96, "; + } + out << "\"icache_miss_le_request_records\": 96, \"counter_below_risk_threshold_records\": 96},\n"; + out << " \"summary\": {\n"; + write_group(out, "all", data.all, true); + write_group(out, "aic", data.aic, true); + write_group(out, "aiv", data.aiv, false); + out << " }\n"; + out << "}\n"; + out.close(); + if (!out || std::rename(temporary.c_str(), path.c_str()) != 0) { + std::remove(temporary.c_str()); + return -1; + } + LOG_INFO_V0("fdwic submit-PMU raw written to %s", path.c_str()); + return 0; +} + +extern "C" void fdwic_submit_pmu_host_finalize(Runtime *runtime) { fdwic_swimlane_host_finalize(runtime); } diff --git a/src/a5/runtime/fully_distributed_within_core/host/fdwic_swimlane.cpp b/src/a5/runtime/fully_distributed_within_core/host/fdwic_swimlane.cpp index ea720fb1de..0b3bdb6682 100644 --- a/src/a5/runtime/fully_distributed_within_core/host/fdwic_swimlane.cpp +++ b/src/a5/runtime/fully_distributed_within_core/host/fdwic_swimlane.cpp @@ -11,13 +11,16 @@ #include "runtime.h" +#include #include #include +#include #include #include #include #include +#include #include #include "common/platform_config.h" @@ -26,6 +29,27 @@ namespace { +constexpr uint32_t kFdwicSwimlaneMaxLevel = kFdwicAtomicSwimlaneLevel; +constexpr int32_t kFdwicSwimlanePhaseCount = static_cast(FdwicSwimlanePhase::Count); + +struct TraceSummary { + uint64_t records = 0; + uint64_t atomic_records = 0; + uint64_t clock_baseline_records = 0; + uint64_t atomic_calls = 0; + uint64_t poll_calls = 0; + uint64_t poll_batch_records = 0; + uint64_t dropped_records = 0; +}; + +bool is_cpu_sim_trace() { +#if defined(SIMPLER_PLATFORM_NAME) + return std::strcmp(SIMPLER_PLATFORM_NAME, "a5sim") == 0; +#else + return false; +#endif +} + const char *phase_name(int32_t phase) { switch (static_cast(phase)) { case FdwicSwimlanePhase::Kernel: @@ -56,17 +80,485 @@ const char *phase_name(int32_t phase) { return "Fanin"; case FdwicSwimlanePhase::Register: return "Register"; + case FdwicSwimlanePhase::Atomic: + return "Atomic"; + case FdwicSwimlanePhase::ClockBaseline: + return "ClockBaseline"; + case FdwicSwimlanePhase::OrchestrationReplay: + return "OrchestrationReplay"; + case FdwicSwimlanePhase::FinalDrain: + return "FinalDrain"; + case FdwicSwimlanePhase::WinnerBuild: + return "WinnerBuild"; + case FdwicSwimlanePhase::AllocComplete: + return "AllocComplete"; + case FdwicSwimlanePhase::LoserReplay: + return "LoserReplay"; + case FdwicSwimlanePhase::Count: + break; } return "Unknown"; } -const char *core_type_name(uint32_t core_idx) { return (core_idx % 3 == 0) ? "aic" : "aiv"; } +const char *atomic_site_name(uint32_t site) { + static constexpr const char *names[] = { + "StartupIncrement", + "StartupPoll", + "FatalPoll", + "FatalSet", + "ClaimMax", + "FaninFlagLoad", + "CompletionVendExchange", + "CompletionFlagExchange", + "FrontierInitialLoad", + "FrontierFlagLoad", + "FrontierMax", + "HeapFrontierLoad", + "HeapVendLoad", + "ReplayDoneIncrement", + "ReplayDonePoll", + "WonSlotClaimMax", + "WonRemainingExchange", + "WonLaneResetExchange", + "WonLaneDepositExchange", + "WonStatePublishExchange", + "WonAnyPublishExchange", + "WonAnyLoad", + "WonStateLoad", + "WonLaneClaimExchange", + "WonLaneReleaseExchange", + "WonRemainingFetchSub", + "WonStateClearExchange", + "WonDrainedLoad", + }; + static_assert( + sizeof(names) / sizeof(names[0]) == static_cast(FdwicAtomicSite::Count), + "atomic site name table must match the raw ABI" + ); + return site < sizeof(names) / sizeof(names[0]) ? names[site] : "Unknown"; +} + +const char *atomic_op_name(uint32_t op) { + static constexpr const char *names[] = {"Load", "Exchange", "FetchAdd", "FetchMax", "FetchSub"}; + return op < sizeof(names) / sizeof(names[0]) ? names[op] : "Unknown"; +} + +const char *core_type_name(CoreType core_type) { + switch (core_type) { + case CoreType::AIC: + return "aic"; + case CoreType::AIV: + return "aiv"; + } + return "unknown"; +} + +bool build_expected_core_layout( + const Runtime *runtime, uint32_t num_cores, int32_t expected_blocks[RUNTIME_MAX_WORKER], + int32_t expected_lanes[RUNTIME_MAX_WORKER] +) { + uint32_t aic_count = 0; + for (uint32_t core = 0; core < num_cores; ++core) { + const CoreType core_type = runtime->workers[core].core_type; + if (core_type == CoreType::AIC) { + expected_blocks[core] = static_cast(aic_count++); + expected_lanes[core] = 0; + } else if (core_type != CoreType::AIV) { + LOG_ERROR("fdwic swimlane core %u has invalid core type %d", core, static_cast(core_type)); + return false; + } + } + if (aic_count == 0) { + LOG_ERROR("fdwic swimlane topology has no AIC workers"); + return false; + } + + uint32_t aiv_ordinal = 0; + for (uint32_t core = 0; core < num_cores; ++core) { + if (runtime->workers[core].core_type != CoreType::AIV) continue; + const uint32_t block = aiv_ordinal / 2; + if (block >= aic_count) { + LOG_ERROR( + "fdwic swimlane AIV worker %u cannot map to an AIC block: aic=%u aiv_ordinal=%u", core, aic_count, + aiv_ordinal + ); + return false; + } + expected_blocks[core] = static_cast(block); + expected_lanes[core] = 1 + static_cast(aiv_ordinal % 2); + ++aiv_ordinal; + } + if (aiv_ordinal != 2 * aic_count) { + LOG_ERROR( + "fdwic swimlane topology must contain two AIV workers per AIC: aic=%u aiv=%u", aic_count, aiv_ordinal + ); + return false; + } + return true; +} + +bool atomic_record_schema_valid(const FdwicSwimlaneRecord &record) { + if (record.aux >= static_cast(FdwicAtomicSite::Count)) return false; + const auto site = static_cast(record.aux); + const uint32_t op = record.flags & kFdwicAtomicOpMask; + if (op != static_cast(fdwic_atomic_site_op(site))) return false; + + const bool result_used = (record.flags & kFdwicAtomicResultUsed) != 0; + const bool return_ready = (record.flags & kFdwicAtomicReturnReady) != 0; + const bool value_zero = (record.flags & kFdwicAtomicValueZero) != 0; + const bool poll_batch = (record.flags & kFdwicAtomicPollBatch) != 0; + const uint32_t payload = record.flags >> kFdwicAtomicRetriesShift; + if (poll_batch) { + return fdwic_atomic_site_is_poll_batchable(site) && result_used && !return_ready && !value_zero && + payload > 0 && record.task_id == -1 && record.func_id == -1; + } + const bool expected_return_ready = result_used && !is_cpu_sim_trace(); + if (result_used != fdwic_atomic_site_result_used(site) || return_ready != expected_return_ready) return false; + if (value_zero && op != static_cast(FdwicAtomicOp::Load)) return false; + if (payload != 0 && op != static_cast(FdwicAtomicOp::FetchMax)) return false; + return record.func_id == -1; +} + +uint32_t atomic_record_call_count(const FdwicSwimlaneRecord &record) { + return (record.flags & kFdwicAtomicPollBatch) != 0 ? record.flags >> kFdwicAtomicPollCountShift : 1U; +} + +bool claim_record_schema_valid(const FdwicSwimlaneRecord &record) { + if ((record.flags & ~(kFdwicClaimWon | kFdwicClaimAttempted)) != 0) return false; + if ((record.flags & kFdwicClaimWon) != 0 && (record.flags & kFdwicClaimAttempted) == 0) return false; + return record.aux <= 1; +} + +bool clock_record_schema_valid(const FdwicSwimlaneRecord &record) { + if ((record.flags & ~(kFdwicClockAtomicDependency | kFdwicClockAtomicDependencyApplied)) != 0) return false; + if ((record.flags & kFdwicClockAtomicDependencyApplied) != 0 && (record.flags & kFdwicClockAtomicDependency) == 0) { + return false; + } + const bool dependency = (record.flags & kFdwicClockAtomicDependency) != 0; + const bool dependency_applied = (record.flags & kFdwicClockAtomicDependencyApplied) != 0; + if (dependency_applied != (dependency && !is_cpu_sim_trace())) return false; + return record.task_id == -1 && record.func_id == -1 && record.aux == 0; +} + +bool ordinary_record_schema_valid(const FdwicSwimlaneRecord &record) { + switch (static_cast(record.phase)) { + case FdwicSwimlanePhase::Kernel: + case FdwicSwimlanePhase::Commit: + return record.flags <= 1 && record.aux == 0; + case FdwicSwimlanePhase::DrainWon: + return record.flags == 1 && record.aux < 4; + case FdwicSwimlanePhase::RingBp: + return record.flags == 0 && record.aux <= 1; + case FdwicSwimlanePhase::Submit: + return record.flags <= 1 && record.aux <= 1; + case FdwicSwimlanePhase::Materialize: + case FdwicSwimlanePhase::PrepareMap: + case FdwicSwimlanePhase::Register: + return record.flags == 0 && record.aux <= 1; + case FdwicSwimlanePhase::Fanin: + return record.flags == 0 && record.aux <= 16; + case FdwicSwimlanePhase::Alloc: + case FdwicSwimlanePhase::Build: + case FdwicSwimlanePhase::Replay: + // Schema-v4 reserves the legacy IDs for archived raw files but never + // accepts newly produced overlapping lap records. + return false; + case FdwicSwimlanePhase::EfDrain: + return record.flags == 0 && record.aux == 0; + case FdwicSwimlanePhase::WinnerBuild: + case FdwicSwimlanePhase::AllocComplete: + case FdwicSwimlanePhase::LoserReplay: + return record.flags == 0 && record.aux == 0; + case FdwicSwimlanePhase::OrchestrationReplay: + case FdwicSwimlanePhase::FinalDrain: + return record.task_id == -1 && record.func_id == -1 && record.flags == 0 && record.aux == 0; + case FdwicSwimlanePhase::Claim: + case FdwicSwimlanePhase::Atomic: + case FdwicSwimlanePhase::ClockBaseline: + return true; + case FdwicSwimlanePhase::Count: + return false; + } + return false; +} + +bool validate_header_and_counts( + const Runtime *runtime, const FdwicSwimlaneHeader *header, uint32_t level, TraceSummary &summary, + uint32_t &max_core_records +) { + const uint32_t expected_cores = runtime->fdwic_swimlane_num_cores_; + const uint32_t expected_capacity = runtime->fdwic_swimlane_records_per_core_; + const uint64_t expected_bytes = sizeof(FdwicSwimlaneHeader) + static_cast(expected_cores) * + expected_capacity * sizeof(FdwicSwimlaneRecord); + const bool header_valid = + header->magic == kFdwicSwimlaneMagic && header->version == kFdwicSwimlaneVersion && expected_cores > 0 && + expected_cores <= RUNTIME_MAX_WORKER && header->num_cores == expected_cores && + runtime->worker_count == static_cast(expected_cores) && expected_capacity > 0 && + header->records_per_core == expected_capacity && header->freq_hz == PLATFORM_PROF_SYS_CNT_FREQ && + runtime->fdwic_swimlane_bytes_ == expected_bytes && runtime->dist.swimlane_level == level && + runtime->dist.swimlane_base == runtime->fdwic_swimlane_dev_base_ && + runtime->dist.swimlane_records_per_core == expected_capacity; + if (!header_valid) { + LOG_ERROR( + "fdwic swimlane invalid header/state: magic=0x%08x version=%u cores=%u/%u worker_count=%d " + "capacity=%u/%u freq=%llu bytes=%llu/%llu", + header->magic, header->version, header->num_cores, expected_cores, runtime->worker_count, + header->records_per_core, expected_capacity, static_cast(header->freq_hz), + static_cast(runtime->fdwic_swimlane_bytes_), + static_cast(expected_bytes) + ); + return false; + } + + for (uint32_t core = 0; core < expected_cores; ++core) { + const FdwicSwimlaneCoreState &core_state = header->cores[core]; + summary.records += core_state.count; + summary.atomic_calls += core_state.atomic_calls; + summary.poll_calls += core_state.poll_calls; + summary.poll_batch_records += core_state.poll_batch_records; + summary.dropped_records += core_state.dropped; + if (core_state.count > max_core_records) max_core_records = core_state.count; + if (core_state.count > expected_capacity || core_state.dropped != 0) { + LOG_ERROR( + "fdwic swimlane core %u is incomplete: count=%u capacity=%u dropped=%u atomic_calls=%u", core, + core_state.count, expected_capacity, core_state.dropped, core_state.atomic_calls + ); + return false; + } + if (level < kFdwicAtomicSwimlaneLevel && + (core_state.atomic_calls != 0 || core_state.poll_calls != 0 || core_state.poll_batch_records != 0)) { + LOG_ERROR( + "fdwic swimlane level-%u core %u unexpectedly reports atomic counters: calls=%u poll_calls=%u " + "poll_batches=%u", + level, core, core_state.atomic_calls, core_state.poll_calls, core_state.poll_batch_records + ); + return false; + } + if (level >= kFdwicAtomicSwimlaneLevel) { + if (core_state.poll_calls > core_state.atomic_calls || + (core_state.poll_calls == 0) != (core_state.poll_batch_records == 0)) { + LOG_ERROR( + "fdwic swimlane level-4 core %u has invalid poll counters: calls=%u poll_calls=%u " + "poll_batches=%u", + core, core_state.atomic_calls, core_state.poll_calls, core_state.poll_batch_records + ); + return false; + } + const uint64_t atomic_records = + static_cast(core_state.atomic_calls) - core_state.poll_calls + core_state.poll_batch_records; + if (core_state.count < 2 || atomic_records > core_state.count - 2) { + LOG_ERROR( + "fdwic swimlane level-4 core %u cannot close physical rows: count=%u atomic_records=%llu " + "atomic_calls=%u poll_calls=%u poll_batches=%u", + core, core_state.count, static_cast(atomic_records), core_state.atomic_calls, + core_state.poll_calls, core_state.poll_batch_records + ); + return false; + } + summary.atomic_records += atomic_records; + } + } + if (level >= kFdwicAtomicSwimlaneLevel) { + summary.clock_baseline_records = 2 * static_cast(expected_cores); + } + return true; +} + +bool validate_and_write_core( + const FdwicSwimlaneHeader *header, const FdwicSwimlaneRecord *records, uint32_t core, int32_t expected_block, + int32_t expected_lane, uint32_t level, std::ofstream &out, bool &first, TraceSummary &observed +) { + const FdwicSwimlaneCoreState &core_state = header->cores[core]; + if (core_state.core_idx != static_cast(core) || core_state.block_id != expected_block || + core_state.lane != expected_lane) { + LOG_ERROR( + "fdwic swimlane invalid worker identity: worker=%u core=%d block=%d/%d lane=%d/%d", core, + core_state.core_idx, core_state.block_id, expected_block, core_state.lane, expected_lane + ); + return false; + } + uint32_t core_atomic_records = 0; + uint64_t core_atomic_calls = 0; + uint64_t core_poll_calls = 0; + uint32_t core_poll_batch_records = 0; + uint32_t core_clock_records = 0; + uint32_t core_plain_clock_records = 0; + uint32_t core_dependency_clock_records = 0; + uint32_t core_orchestration_records = 0; + uint32_t core_final_drain_records = 0; + for (uint32_t index = 0; index < core_state.count; ++index) { + const FdwicSwimlaneRecord &record = records[index]; + const bool base_valid = record.end_cycle >= record.start_cycle && record.phase < kFdwicSwimlanePhaseCount && + record.task_id >= -1 && record.func_id >= -1; + bool schema_valid = base_valid; + if (record.phase == static_cast(FdwicSwimlanePhase::Atomic)) { + schema_valid = schema_valid && atomic_record_schema_valid(record); + ++core_atomic_records; + const uint32_t call_count = atomic_record_call_count(record); + core_atomic_calls += call_count; + if ((record.flags & kFdwicAtomicPollBatch) != 0) { + core_poll_calls += call_count; + ++core_poll_batch_records; + } + } else if (record.phase == static_cast(FdwicSwimlanePhase::Claim)) { + schema_valid = schema_valid && claim_record_schema_valid(record); + } else if (record.phase == static_cast(FdwicSwimlanePhase::ClockBaseline)) { + schema_valid = schema_valid && clock_record_schema_valid(record); + ++core_clock_records; + if ((record.flags & kFdwicClockAtomicDependency) != 0) { + ++core_dependency_clock_records; + } else { + ++core_plain_clock_records; + } + } else { + schema_valid = schema_valid && ordinary_record_schema_valid(record); + if (record.phase == static_cast(FdwicSwimlanePhase::OrchestrationReplay)) { + ++core_orchestration_records; + } else if (record.phase == static_cast(FdwicSwimlanePhase::FinalDrain)) { + ++core_final_drain_records; + } + } + if (!schema_valid) { + const uint32_t op = record.flags & kFdwicAtomicOpMask; + LOG_ERROR( + "fdwic swimlane invalid record: worker=%u index=%u core=%d block=%d lane=%d phase=%d(%s) " + "task=%d func=%d start=%llu end=%llu flags=0x%08x aux=%u site=%s op=%s", + core, index, core_state.core_idx, core_state.block_id, core_state.lane, record.phase, + phase_name(record.phase), record.task_id, record.func_id, + static_cast(record.start_cycle), static_cast(record.end_cycle), + record.flags, record.aux, atomic_site_name(record.aux), atomic_op_name(op) + ); + return false; + } + if (!first) out << ","; + out << "\n [" << core_state.core_idx << ", " << core_state.block_id << ", " << core_state.lane << ", " + << record.task_id << ", " << record.func_id << ", \"" << phase_name(record.phase) << "\", " + << record.start_cycle << ", " << record.end_cycle << ", " << record.flags << ", " << record.aux << "]"; + first = false; + } + + if (core_orchestration_records != 1 || core_final_drain_records != 1) { + LOG_ERROR( + "fdwic swimlane schema-v4 parent closure failed on core %u: orchestration=%u final_drain=%u", core, + core_orchestration_records, core_final_drain_records + ); + return false; + } + + if (level >= kFdwicAtomicSwimlaneLevel) { + if (core_atomic_records != static_cast(core_state.atomic_calls) - core_state.poll_calls + + core_state.poll_batch_records || + core_atomic_calls != core_state.atomic_calls || core_poll_calls != core_state.poll_calls || + core_poll_batch_records != core_state.poll_batch_records || core_clock_records != 2 || + core_plain_clock_records != 1 || core_dependency_clock_records != 1) { + LOG_ERROR( + "fdwic swimlane level-4 closure failed on core %u: atomic_records=%u atomic_calls=%llu/%u " + "poll_calls=%llu/%u poll_batches=%u/%u clock=%u plain_clock=%u dependency_clock=%u", + core, core_atomic_records, static_cast(core_atomic_calls), core_state.atomic_calls, + static_cast(core_poll_calls), core_state.poll_calls, core_poll_batch_records, + core_state.poll_batch_records, core_clock_records, core_plain_clock_records, + core_dependency_clock_records + ); + return false; + } + } else if (core_atomic_records != 0 || core_state.atomic_calls != 0 || core_state.poll_calls != 0 || + core_state.poll_batch_records != 0 || core_clock_records != 0) { + LOG_ERROR( + "fdwic swimlane level-%u contains level-4 records on core %u: atomic_records=%u atomic_calls=%u " + "poll_calls=%u poll_batches=%u clock=%u", + level, core, core_atomic_records, core_state.atomic_calls, core_state.poll_calls, + core_state.poll_batch_records, core_clock_records + ); + return false; + } + observed.records += core_state.count; + observed.atomic_records += core_atomic_records; + observed.clock_baseline_records += core_clock_records; + observed.atomic_calls += core_atomic_calls; + observed.poll_calls += core_poll_calls; + observed.poll_batch_records += core_poll_batch_records; + observed.dropped_records += core_state.dropped; + return true; +} + +std::string output_path_from_prefix(const std::string &prefix) { + if (!prefix.empty() && prefix.back() == '/') return prefix + "l2_swimlane_records.json"; + return prefix + "/l2_swimlane_records.json"; +} std::string output_path(const Runtime *runtime) { - std::string base = runtime->fdwic_swimlane_output_prefix_; - if (base.empty()) base = "."; - mkdir(base.c_str(), 0755); - return base + "/l2_swimlane_records.json"; + return output_path_from_prefix(runtime->fdwic_swimlane_output_prefix_); +} + +std::string perf_clock_output_path_from_prefix(const std::string &prefix) { + const char *mode = std::getenv("PTO_FDWIC_PROFILE"); + const bool kernel = mode != nullptr && std::strcmp(mode, "perf-clock-kernel") == 0; + const char *name = kernel ? "fdwic_perf_clock_kernel_summary.json" : "fdwic_perf_clock_summary.json"; + if (!prefix.empty() && prefix.back() == '/') return prefix + name; + return prefix + "/" + name; +} + +bool perf_clock_kernel_requested() { + const char *mode = std::getenv("PTO_FDWIC_PROFILE"); + return mode != nullptr && std::strcmp(mode, "perf-clock-kernel") == 0; +} + +bool perf_clock_requested() { + const char *mode = std::getenv("PTO_FDWIC_PROFILE"); + return mode != nullptr && (std::strcmp(mode, "perf-clock") == 0 || std::strcmp(mode, "perf-clock-kernel") == 0); +} + +struct PerfClockGroupAggregate { + uint32_t cores = 0; + uint64_t elapsed_sum = 0; + uint64_t elapsed_min = std::numeric_limits::max(); + uint64_t elapsed_max = 0; + uint64_t kernel_sum = 0; + uint64_t kernel_min = std::numeric_limits::max(); + uint64_t kernel_max = 0; + uint64_t residual_sum = 0; + uint64_t residual_min = std::numeric_limits::max(); + uint64_t residual_max = 0; + uint64_t kernel_calls_sum = 0; + uint32_t kernel_calls_min = std::numeric_limits::max(); + uint32_t kernel_calls_max = 0; + + void add(uint64_t elapsed, uint64_t kernel, uint32_t calls) { + const uint64_t residual = elapsed - kernel; + ++cores; + elapsed_sum += elapsed; + elapsed_min = std::min(elapsed_min, elapsed); + elapsed_max = std::max(elapsed_max, elapsed); + kernel_sum += kernel; + kernel_min = std::min(kernel_min, kernel); + kernel_max = std::max(kernel_max, kernel); + residual_sum += residual; + residual_min = std::min(residual_min, residual); + residual_max = std::max(residual_max, residual); + kernel_calls_sum += calls; + kernel_calls_min = std::min(kernel_calls_min, calls); + kernel_calls_max = std::max(kernel_calls_max, calls); + } +}; + +bool prepare_output_directory(const std::string &prefix) { + struct stat info{}; + if (stat(prefix.c_str(), &info) == 0) { + if (S_ISDIR(info.st_mode)) return true; + LOG_ERROR("fdwic swimlane output prefix is not a directory: %s", prefix.c_str()); + return false; + } + if (errno != ENOENT || mkdir(prefix.c_str(), 0755) != 0) { + LOG_ERROR("cannot create fdwic swimlane output directory %s: %s", prefix.c_str(), std::strerror(errno)); + return false; + } + return true; +} + +bool remove_output_if_present(const std::string &path) { + if (std::remove(path.c_str()) == 0 || errno == ENOENT) return true; + LOG_ERROR("cannot remove stale fdwic swimlane output %s: %s", path.c_str(), std::strerror(errno)); + return false; } bool should_print_trace_export() { @@ -79,30 +571,51 @@ bool should_print_trace_export() { } // namespace -extern "C" int fdwic_swimlane_host_init(Runtime *runtime, int num_cores, int enabled, const char *output_prefix) { +extern "C" int fdwic_swimlane_host_init(Runtime *runtime, int num_cores, int level, const char *output_prefix) { if (runtime == nullptr) return -1; - runtime->dist.swimlane_enabled = 0; + runtime->dist.swimlane_level = 0; runtime->dist.swimlane_base = 0; runtime->dist.swimlane_records_per_core = 0; runtime->fdwic_swimlane_host_shadow_ = nullptr; + runtime->fdwic_swimlane_dev_allocation_ = 0; runtime->fdwic_swimlane_dev_base_ = 0; runtime->fdwic_swimlane_bytes_ = 0; runtime->fdwic_swimlane_num_cores_ = 0; runtime->fdwic_swimlane_records_per_core_ = 0; runtime->fdwic_swimlane_output_prefix_[0] = '\0'; - if (!enabled) return 0; - if (num_cores <= 0 || num_cores > 108) return -1; + if (level < 0 || level > static_cast(kFdwicSwimlaneMaxLevel)) { + LOG_ERROR("fdwic swimlane level %d is outside [0, %u]", level, kFdwicSwimlaneMaxLevel); + return -1; + } + if (level == 0) return 0; + if (num_cores <= 0 || num_cores > RUNTIME_MAX_WORKER) return -1; if (runtime->host_api.device_malloc == nullptr || runtime->host_api.device_free == nullptr || runtime->host_api.copy_to_device == nullptr || runtime->host_api.copy_from_device == nullptr) { return -1; } - const uint32_t records_per_core = kFdwicSwimlaneDefaultRecordsPerCore; + const std::string exact_output_prefix = output_prefix == nullptr || output_prefix[0] == '\0' ? "." : output_prefix; + if (exact_output_prefix.size() >= sizeof(runtime->fdwic_swimlane_output_prefix_)) { + LOG_ERROR("fdwic swimlane output prefix is too long: %zu bytes", exact_output_prefix.size()); + return -1; + } + if (!prepare_output_directory(exact_output_prefix)) return -1; + const std::string path = output_path_from_prefix(exact_output_prefix); + if (!remove_output_if_present(path) || !remove_output_if_present(path + ".tmp")) return -1; + + const uint32_t records_per_core = level >= static_cast(kFdwicAtomicSwimlaneLevel) ? + kFdwicAtomicSwimlaneRecordsPerCore : + kFdwicSwimlaneDefaultRecordsPerCore; const uint64_t bytes = sizeof(FdwicSwimlaneHeader) + static_cast(num_cores) * records_per_core * sizeof(FdwicSwimlaneRecord); - void *host_shadow = std::malloc(static_cast(bytes)); + constexpr uint64_t kDeviceAlignment = 64; + if (bytes > std::numeric_limits::max() - (kDeviceAlignment - 1)) { + LOG_ERROR("fdwic swimlane allocation is too large: %llu bytes", static_cast(bytes)); + return -1; + } + void *host_shadow = std::aligned_alloc(64, sizeof(FdwicSwimlaneHeader)); if (host_shadow == nullptr) return -1; - std::memset(host_shadow, 0, static_cast(bytes)); + std::memset(host_shadow, 0, sizeof(FdwicSwimlaneHeader)); FdwicSwimlaneHeader *header = reinterpret_cast(host_shadow); header->magic = kFdwicSwimlaneMagic; header->version = kFdwicSwimlaneVersion; @@ -110,31 +623,30 @@ extern "C" int fdwic_swimlane_host_init(Runtime *runtime, int num_cores, int ena header->records_per_core = records_per_core; header->freq_hz = PLATFORM_PROF_SYS_CNT_FREQ; - void *dev = runtime->host_api.device_malloc(static_cast(bytes)); - if (dev == nullptr) { + void *dev_allocation = runtime->host_api.device_malloc(static_cast(bytes + (kDeviceAlignment - 1))); + if (dev_allocation == nullptr) { std::free(host_shadow); return -1; } - if (runtime->host_api.copy_to_device(dev, host_shadow, static_cast(bytes)) != 0) { - runtime->host_api.device_free(dev); + const uintptr_t dev_base = + (reinterpret_cast(dev_allocation) + (kDeviceAlignment - 1)) & ~(kDeviceAlignment - 1); + void *dev = reinterpret_cast(dev_base); + if (runtime->host_api.copy_to_device(dev, host_shadow, sizeof(FdwicSwimlaneHeader)) != 0) { + runtime->host_api.device_free(dev_allocation); std::free(host_shadow); return -1; } runtime->fdwic_swimlane_host_shadow_ = host_shadow; - runtime->fdwic_swimlane_dev_base_ = reinterpret_cast(dev); + runtime->fdwic_swimlane_dev_allocation_ = reinterpret_cast(dev_allocation); + runtime->fdwic_swimlane_dev_base_ = dev_base; runtime->fdwic_swimlane_bytes_ = bytes; runtime->fdwic_swimlane_num_cores_ = static_cast(num_cores); runtime->fdwic_swimlane_records_per_core_ = records_per_core; - if (output_prefix != nullptr) { - std::strncpy( - runtime->fdwic_swimlane_output_prefix_, output_prefix, sizeof(runtime->fdwic_swimlane_output_prefix_) - 1 - ); - runtime->fdwic_swimlane_output_prefix_[sizeof(runtime->fdwic_swimlane_output_prefix_) - 1] = '\0'; - } + std::memcpy(runtime->fdwic_swimlane_output_prefix_, exact_output_prefix.c_str(), exact_output_prefix.size() + 1); runtime->dist.swimlane_base = runtime->fdwic_swimlane_dev_base_; runtime->dist.swimlane_records_per_core = records_per_core; - runtime->dist.swimlane_enabled = 1; + runtime->dist.swimlane_level = static_cast(level); return 1; } @@ -143,35 +655,102 @@ extern "C" int fdwic_swimlane_host_export(Runtime *runtime) { runtime->fdwic_swimlane_dev_base_ == 0) { return 0; } + const std::string path = output_path(runtime); + const std::string temporary_path = path + ".tmp"; + if (!remove_output_if_present(temporary_path)) return -1; + void *dev = reinterpret_cast(runtime->fdwic_swimlane_dev_base_); - if (runtime->host_api.copy_from_device( - runtime->fdwic_swimlane_host_shadow_, dev, static_cast(runtime->fdwic_swimlane_bytes_) - ) != 0) { - LOG_ERROR("fdwic swimlane D2H copy failed"); + if (runtime->host_api.copy_from_device(runtime->fdwic_swimlane_host_shadow_, dev, sizeof(FdwicSwimlaneHeader)) != + 0) { + LOG_ERROR("fdwic swimlane header D2H copy failed"); + std::remove(temporary_path.c_str()); return -1; } FdwicSwimlaneHeader *header = reinterpret_cast(runtime->fdwic_swimlane_host_shadow_); - if (header->magic != kFdwicSwimlaneMagic || header->version != kFdwicSwimlaneVersion) { - LOG_ERROR("fdwic swimlane header mismatch"); + const uint32_t level = runtime->dist.swimlane_level; + if (level == 0 || level > kFdwicSwimlaneMaxLevel) { + LOG_ERROR("fdwic swimlane export has invalid level %u", level); + std::remove(temporary_path.c_str()); return -1; } - auto *records = reinterpret_cast( - static_cast(runtime->fdwic_swimlane_host_shadow_) + sizeof(FdwicSwimlaneHeader) - ); - std::ofstream out(output_path(runtime)); - if (!out.is_open()) return -1; + TraceSummary summary; + uint32_t max_core_records = 0; + if (!validate_header_and_counts(runtime, header, level, summary, max_core_records)) { + std::remove(temporary_path.c_str()); + return -1; + } + + int32_t expected_blocks[RUNTIME_MAX_WORKER] = {}; + int32_t expected_lanes[RUNTIME_MAX_WORKER] = {}; + if (!build_expected_core_layout(runtime, header->num_cores, expected_blocks, expected_lanes)) { + std::remove(temporary_path.c_str()); + return -1; + } + + FdwicSwimlaneRecord *scratch = nullptr; + if (max_core_records != 0) { + const size_t scratch_bytes = static_cast(max_core_records) * sizeof(FdwicSwimlaneRecord); + scratch = static_cast(std::aligned_alloc(alignof(FdwicSwimlaneRecord), scratch_bytes)); + if (scratch == nullptr) { + LOG_ERROR("cannot allocate fdwic swimlane per-core scratch: %zu bytes", scratch_bytes); + std::remove(temporary_path.c_str()); + return -1; + } + } + + std::ofstream out(temporary_path, std::ios::out | std::ios::trunc); + if (!out.is_open()) { + LOG_ERROR("cannot open fdwic swimlane temporary output %s: %s", temporary_path.c_str(), std::strerror(errno)); + std::free(scratch); + std::remove(temporary_path.c_str()); + return -1; + } + auto fail_export = [&out, &scratch, &temporary_path]() { + out.close(); + std::free(scratch); + scratch = nullptr; + std::remove(temporary_path.c_str()); + return -1; + }; + out << "{\n"; - out << " \"l2_swimlane_level\": 1,\n"; + out << " \"l2_swimlane_level\": " << level << ",\n"; out << " \"metadata\": {\n"; out << " \"clock_freq_hz\": " << header->freq_hz << ",\n"; out << " \"num_cores\": " << header->num_cores << ",\n"; + out << " \"trace_schema_version\": " << kFdwicSwimlaneTraceSchemaVersion << ",\n"; + out << " \"raw_trace_version\": " << header->version << ",\n"; + out << " \"records_per_core\": " << header->records_per_core << ",\n"; + out << " \"record_size_bytes\": " << sizeof(FdwicSwimlaneRecord) << ",\n"; + out << " \"device_trace_bytes\": " << runtime->fdwic_swimlane_bytes_ << ",\n"; out << " \"core_types\": ["; for (uint32_t c = 0; c < header->num_cores; c++) { if (c > 0) out << ", "; - out << "\"" << core_type_name(c) << "\""; + out << "\"" << core_type_name(runtime->workers[c].core_type) << "\""; + } + out << "],\n"; + out << " \"atomic_site_names\": ["; + for (uint32_t site = 0; site < static_cast(FdwicAtomicSite::Count); ++site) { + if (site > 0) out << ", "; + out << "\"" << atomic_site_name(site) << "\""; } - out << "]\n"; + out << "],\n"; + out << " \"atomic_op_names\": ["; + for (uint32_t op = 0; op <= static_cast(FdwicAtomicOp::FetchSub); ++op) { + if (op > 0) out << ", "; + out << "\"" << atomic_op_name(op) << "\""; + } + out << "],\n"; + out << " \"fdwic_summary\": {\n"; + out << " \"records\": " << summary.records << ",\n"; + out << " \"atomic_records\": " << summary.atomic_records << ",\n"; + out << " \"clock_baseline_records\": " << summary.clock_baseline_records << ",\n"; + out << " \"atomic_calls\": " << summary.atomic_calls << ",\n"; + out << " \"batched_poll_calls\": " << summary.poll_calls << ",\n"; + out << " \"poll_batch_records\": " << summary.poll_batch_records << ",\n"; + out << " \"dropped_records\": " << summary.dropped_records << "\n"; + out << " }\n"; out << " },\n"; out << " \"aicore_tasks\": [],\n"; out << " \"aicpu_tasks\": [],\n"; @@ -179,39 +758,471 @@ extern "C" int fdwic_swimlane_host_export(Runtime *runtime) { out << " \"aicpu_orchestrator_phases\": [],\n"; out << " \"fdwic_events\": ["; bool first = true; + TraceSummary observed; + const uint64_t records_base = runtime->fdwic_swimlane_dev_base_ + sizeof(FdwicSwimlaneHeader); for (uint32_t c = 0; c < header->num_cores; c++) { - const uint32_t count = - header->cores[c].count < header->records_per_core ? header->cores[c].count : header->records_per_core; - for (uint32_t i = 0; i < count; i++) { - const FdwicSwimlaneRecord &r = records[static_cast(c) * header->records_per_core + i]; - if (r.end_cycle < r.start_cycle) continue; - if (!first) out << ","; - out << "\n [" << r.core_idx << ", " << r.block_id << ", " << r.lane << ", " << r.task_id << ", " - << r.func_id << ", \"" << phase_name(r.phase) << "\", " << r.start_cycle << ", " << r.end_cycle << ", " - << r.flags << ", " << r.aux << "]"; - first = false; + const uint32_t count = header->cores[c].count; + if (count != 0) { + const uint64_t core_offset = + static_cast(c) * header->records_per_core * sizeof(FdwicSwimlaneRecord); + const void *core_records_dev = reinterpret_cast(records_base + core_offset); + const size_t core_bytes = static_cast(count) * sizeof(FdwicSwimlaneRecord); + if (runtime->host_api.copy_from_device(scratch, core_records_dev, core_bytes) != 0) { + LOG_ERROR("fdwic swimlane core %u D2H copy failed: records=%u bytes=%zu", c, count, core_bytes); + return fail_export(); + } + } + if (!validate_and_write_core( + header, scratch, c, expected_blocks[c], expected_lanes[c], level, out, first, observed + )) { + return fail_export(); + } + if (!out) { + LOG_ERROR("failed while writing fdwic swimlane core %u to %s", c, temporary_path.c_str()); + return fail_export(); } } + const bool summary_closed = + observed.records == summary.records && observed.atomic_records == summary.atomic_records && + observed.clock_baseline_records == summary.clock_baseline_records && + observed.atomic_calls == summary.atomic_calls && observed.poll_calls == summary.poll_calls && + observed.poll_batch_records == summary.poll_batch_records && + observed.dropped_records == summary.dropped_records; + if (!summary_closed) { + LOG_ERROR( + "fdwic swimlane summary closure failed: records=%llu/%llu atomic=%llu/%llu clock=%llu/%llu " + "calls=%llu/%llu poll_calls=%llu/%llu poll_batches=%llu/%llu dropped=%llu/%llu", + static_cast(observed.records), static_cast(summary.records), + static_cast(observed.atomic_records), + static_cast(summary.atomic_records), + static_cast(observed.clock_baseline_records), + static_cast(summary.clock_baseline_records), + static_cast(observed.atomic_calls), + static_cast(summary.atomic_calls), static_cast(observed.poll_calls), + static_cast(summary.poll_calls), + static_cast(observed.poll_batch_records), + static_cast(summary.poll_batch_records), + static_cast(observed.dropped_records), + static_cast(summary.dropped_records) + ); + return fail_export(); + } if (!first) out << "\n "; out << "]\n}\n"; + out.close(); + std::free(scratch); + scratch = nullptr; + if (!out) { + LOG_ERROR("failed while writing fdwic swimlane output %s", temporary_path.c_str()); + std::remove(temporary_path.c_str()); + return -1; + } + if (std::rename(temporary_path.c_str(), path.c_str()) != 0) { + LOG_ERROR("cannot finalize fdwic swimlane output %s: %s", path.c_str(), std::strerror(errno)); + std::remove(temporary_path.c_str()); + return -1; + } if (should_print_trace_export()) { - const std::string path = output_path(runtime); - LOG_INFO_V0("fdwic swimlane trace written to %s", path.c_str()); + LOG_INFO_V0( + "fdwic swimlane trace written to %s: records=%llu atomic=%llu clock=%llu calls=%llu poll_calls=%llu " + "poll_batches=%llu dropped=%llu", + path.c_str(), static_cast(summary.records), + static_cast(summary.atomic_records), + static_cast(summary.clock_baseline_records), + static_cast(summary.atomic_calls), static_cast(summary.poll_calls), + static_cast(summary.poll_batch_records), + static_cast(summary.dropped_records) + ); } return 0; } extern "C" void fdwic_swimlane_host_finalize(Runtime *runtime) { if (runtime == nullptr) return; - if (runtime->fdwic_swimlane_dev_base_ != 0 && runtime->host_api.device_free != nullptr) { - runtime->host_api.device_free(reinterpret_cast(runtime->fdwic_swimlane_dev_base_)); + if (runtime->fdwic_swimlane_dev_allocation_ != 0 && runtime->host_api.device_free != nullptr) { + runtime->host_api.device_free(reinterpret_cast(runtime->fdwic_swimlane_dev_allocation_)); } if (runtime->fdwic_swimlane_host_shadow_ != nullptr) { std::free(runtime->fdwic_swimlane_host_shadow_); } runtime->fdwic_swimlane_host_shadow_ = nullptr; + runtime->fdwic_swimlane_dev_allocation_ = 0; runtime->fdwic_swimlane_dev_base_ = 0; runtime->fdwic_swimlane_bytes_ = 0; - runtime->dist.swimlane_enabled = 0; + runtime->fdwic_swimlane_num_cores_ = 0; + runtime->fdwic_swimlane_records_per_core_ = 0; + runtime->dist.swimlane_level = 0; runtime->dist.swimlane_base = 0; + runtime->dist.swimlane_records_per_core = 0; +} + +extern "C" int fdwic_perf_clock_host_init(Runtime *runtime, int num_cores, const char *output_prefix) { + if (!perf_clock_requested()) return 0; + if (runtime == nullptr) return -1; + const bool storage_empty = + runtime->fdwic_swimlane_host_shadow_ == nullptr && runtime->fdwic_swimlane_dev_allocation_ == 0 && + runtime->fdwic_swimlane_dev_base_ == 0 && runtime->fdwic_swimlane_bytes_ == 0 && + runtime->fdwic_swimlane_num_cores_ == 0 && runtime->fdwic_swimlane_records_per_core_ == 0 && + runtime->dist.swimlane_base == 0 && runtime->dist.swimlane_level == 0 && + runtime->dist.swimlane_records_per_core == 0; + if (!storage_empty) { + LOG_ERROR("fdwic perf-clock found non-empty diagnostic storage; refusing to overwrite a live allocation"); + return -1; + } + runtime->fdwic_swimlane_output_prefix_[0] = '\0'; + // 当前证据链只服务真实 PA 的 32 AIC + 64 AIV 全核 Case1/B1。 + // 其他拓扑直接拒绝，避免把部分核数据包装成“每核基线”。 + constexpr int kExpectedAic = 32; + constexpr int kExpectedAiv = 64; + constexpr int kExpectedCores = kExpectedAic + kExpectedAiv; + if (num_cores != kExpectedCores || runtime->worker_count != kExpectedCores) { + LOG_ERROR( + "fdwic perf-clock requires 96 workers (32 AIC + 64 AIV): num_cores=%d worker_count=%d", num_cores, + runtime->worker_count + ); + return -1; + } + if (runtime->host_api.device_malloc == nullptr || runtime->host_api.device_free == nullptr || + runtime->host_api.copy_to_device == nullptr || runtime->host_api.copy_from_device == nullptr) { + return -1; + } + + const std::string prefix = output_prefix == nullptr || output_prefix[0] == '\0' ? "." : output_prefix; + if (prefix.size() >= sizeof(runtime->fdwic_swimlane_output_prefix_)) { + LOG_ERROR("fdwic perf-clock output prefix is too long: %zu bytes", prefix.size()); + return -1; + } + if (!prepare_output_directory(prefix)) return -1; + const std::string path = perf_clock_output_path_from_prefix(prefix); + if (!remove_output_if_present(path) || !remove_output_if_present(path + ".tmp")) return -1; + + constexpr uint64_t kDeviceAlignment = 64; + constexpr uint64_t bytes = sizeof(FdwicSwimlaneHeader); + void *host_shadow = std::aligned_alloc(64, sizeof(FdwicSwimlaneHeader)); + if (host_shadow == nullptr) return -1; + std::memset(host_shadow, 0, sizeof(FdwicSwimlaneHeader)); + auto *header = reinterpret_cast(host_shadow); + header->magic = kFdwicSwimlaneMagic; + header->version = kFdwicSwimlaneVersion; + header->num_cores = kExpectedCores; + header->records_per_core = 0; + header->freq_hz = PLATFORM_PROF_SYS_CNT_FREQ; + + void *dev_allocation = runtime->host_api.device_malloc(static_cast(bytes + (kDeviceAlignment - 1))); + if (dev_allocation == nullptr) { + std::free(host_shadow); + return -1; + } + const uintptr_t dev_base = + (reinterpret_cast(dev_allocation) + (kDeviceAlignment - 1)) & ~(kDeviceAlignment - 1); + if (runtime->host_api.copy_to_device(reinterpret_cast(dev_base), host_shadow, sizeof(FdwicSwimlaneHeader)) != + 0) { + runtime->host_api.device_free(dev_allocation); + std::free(host_shadow); + return -1; + } + + runtime->fdwic_swimlane_host_shadow_ = host_shadow; + runtime->fdwic_swimlane_dev_allocation_ = reinterpret_cast(dev_allocation); + runtime->fdwic_swimlane_dev_base_ = dev_base; + runtime->fdwic_swimlane_bytes_ = bytes; + runtime->fdwic_swimlane_num_cores_ = kExpectedCores; + runtime->fdwic_swimlane_records_per_core_ = 0; + std::memcpy(runtime->fdwic_swimlane_output_prefix_, prefix.c_str(), prefix.size() + 1); + // 只借用已有 handoff 地址传输固定 header；level/records 保持 0，明确 + // 表示这不是 1..4 任一级泳道。 + runtime->dist.swimlane_base = dev_base; + runtime->dist.swimlane_level = 0; + runtime->dist.swimlane_records_per_core = 0; + return 1; } + +extern "C" int fdwic_perf_clock_host_export(Runtime *runtime) { + if (runtime == nullptr || runtime->fdwic_swimlane_host_shadow_ == nullptr || + runtime->fdwic_swimlane_dev_base_ == 0) { + return 0; + } + if (runtime->host_api.copy_from_device( + runtime->fdwic_swimlane_host_shadow_, reinterpret_cast(runtime->fdwic_swimlane_dev_base_), + sizeof(FdwicSwimlaneHeader) + ) != 0) { + LOG_ERROR("fdwic perf-clock header D2H copy failed"); + return -1; + } + + const auto *header = reinterpret_cast(runtime->fdwic_swimlane_host_shadow_); + constexpr uint32_t kExpectedAic = 32; + constexpr uint32_t kExpectedAiv = 64; + constexpr uint32_t kExpectedCores = kExpectedAic + kExpectedAiv; + const bool header_valid = + header->magic == kFdwicSwimlaneMagic && header->version == kFdwicSwimlaneVersion && + header->num_cores == kExpectedCores && header->records_per_core == 0 && + header->freq_hz == PLATFORM_PROF_SYS_CNT_FREQ && runtime->worker_count == static_cast(kExpectedCores) && + runtime->fdwic_swimlane_bytes_ == sizeof(FdwicSwimlaneHeader) && runtime->dist.swimlane_level == 0 && + runtime->dist.swimlane_records_per_core == 0 && + runtime->dist.swimlane_base == runtime->fdwic_swimlane_dev_base_; + if (!header_valid) { + LOG_ERROR( + "fdwic perf-clock invalid header/state: magic=0x%08x version=%u cores=%u records=%u freq=%llu bytes=%llu", + header->magic, header->version, header->num_cores, header->records_per_core, + static_cast(header->freq_hz), + static_cast(runtime->fdwic_swimlane_bytes_) + ); + return -1; + } + + const bool kernel_profile = perf_clock_kernel_requested(); + const char *profile_name = kernel_profile ? "perf-clock-kernel" : "perf-clock"; + int32_t expected_blocks[RUNTIME_MAX_WORKER] = {}; + int32_t expected_lanes[RUNTIME_MAX_WORKER] = {}; + if (!build_expected_core_layout(runtime, header->num_cores, expected_blocks, expected_lanes)) return -1; + + uint32_t expected_submits = 0; + uint64_t global_start = std::numeric_limits::max(); + uint64_t global_end = 0; + PerfClockGroupAggregate aic; + PerfClockGroupAggregate aiv; + + for (uint32_t core_id = 0; core_id < header->num_cores; ++core_id) { + const FdwicSwimlaneCoreState &core = header->cores[core_id]; + uint64_t first_submit_start = 0; + uint64_t last_submit_end = 0; + uint64_t kernel_elapsed_ticks = 0; + uint32_t submit_count = 0; + uint32_t expected_submit_count = 0; + uint32_t kernel_calls = 0; + uint32_t stored_mode = 0; + uint32_t final_seen = 0; + bool diagnostic_fields_valid = false; + if (kernel_profile) { + const FdwicPerfClockKernelCoreData &clock = core.perf_clock_kernel; + first_submit_start = clock.first_submit_start; + last_submit_end = clock.last_submit_end; + submit_count = clock.submit_count; + expected_submit_count = clock.expected_submit_count; + kernel_elapsed_ticks = clock.kernel_elapsed_ticks; + kernel_calls = core.count; + stored_mode = core.poll_batch_records; + final_seen = last_submit_end != 0 ? 1U : 0U; + const bool call_time_shape = + (kernel_calls == 0 && kernel_elapsed_ticks == 0) || (kernel_calls != 0 && kernel_elapsed_ticks != 0); + diagnostic_fields_valid = core.dropped == 0 && core.atomic_calls == 0 && core.poll_calls == 0 && + stored_mode == kFdwicPerfClockKernelMode && call_time_shape; + } else { + const FdwicPerfClockCoreData &clock = core.perf_clock; + first_submit_start = clock.first_submit_start; + last_submit_end = clock.last_submit_end; + submit_count = clock.submit_count; + expected_submit_count = clock.expected_submit_count; + stored_mode = clock.mode; + final_seen = clock.final_seen; + diagnostic_fields_valid = core.count == 0 && core.dropped == 0 && core.atomic_calls == 0 && + core.poll_calls == 0 && core.poll_batch_records == 0 && + stored_mode == kFdwicPerfClockMode && final_seen == 1; + } + + const bool identity_valid = core.core_idx == static_cast(core_id) && + core.block_id == expected_blocks[core_id] && core.lane == expected_lanes[core_id]; + const bool window_order_valid = + kernel_profile ? last_submit_end > first_submit_start : last_submit_end >= first_submit_start; + const bool clock_valid = expected_submit_count != 0 && submit_count == expected_submit_count && + first_submit_start != 0 && window_order_valid && + kernel_elapsed_ticks <= last_submit_end - first_submit_start; + if (!identity_valid || !diagnostic_fields_valid || !clock_valid) { + LOG_ERROR( + "fdwic %s core %u failed closure: core=%d block=%d/%d lane=%d/%d count=%u/%u " + "start=%llu end=%llu mode=%u final=%u kernel_calls=%u kernel_ticks=%llu status=%u reserved=%u/%u", + profile_name, core_id, core.core_idx, core.block_id, expected_blocks[core_id], core.lane, + expected_lanes[core_id], submit_count, expected_submit_count, + static_cast(first_submit_start), static_cast(last_submit_end), + stored_mode, final_seen, kernel_calls, static_cast(kernel_elapsed_ticks), + core.dropped, core.atomic_calls, core.poll_calls + ); + return -1; + } + if (expected_submits == 0) expected_submits = expected_submit_count; + if (expected_submit_count != expected_submits) { + LOG_ERROR( + "fdwic %s expected Submit count differs across cores: core=%u expected=%u reference=%u", profile_name, + core_id, expected_submit_count, expected_submits + ); + return -1; + } + global_start = std::min(global_start, first_submit_start); + global_end = std::max(global_end, last_submit_end); + const uint64_t elapsed = last_submit_end - first_submit_start; + if (runtime->workers[core_id].core_type == CoreType::AIC) { + aic.add(elapsed, kernel_elapsed_ticks, kernel_calls); + } else if (runtime->workers[core_id].core_type == CoreType::AIV) { + aiv.add(elapsed, kernel_elapsed_ticks, kernel_calls); + } else { + LOG_ERROR("fdwic %s core %u has invalid core type", profile_name, core_id); + return -1; + } + } + if (aic.cores != kExpectedAic || aiv.cores != kExpectedAiv || + global_start == std::numeric_limits::max() || global_end < global_start) { + LOG_ERROR( + "fdwic %s topology/global closure failed: AIC=%u/%u AIV=%u/%u start=%llu end=%llu", profile_name, aic.cores, + kExpectedAic, aiv.cores, kExpectedAiv, static_cast(global_start), + static_cast(global_end) + ); + return -1; + } + + uint64_t min_kernel_calls = 0; + uint64_t max_kernel_calls = 0; + if (kernel_profile) { + if (aic.elapsed_sum == 0 || aiv.elapsed_sum == 0) { + LOG_ERROR( + "fdwic perf-clock-kernel requires non-zero role elapsed sums: AIC=%llu AIV=%llu", + static_cast(aic.elapsed_sum), static_cast(aiv.elapsed_sum) + ); + return -1; + } + if (expected_submits % 5U != 0) { + LOG_ERROR("fdwic perf-clock-kernel requires 5*batch Submit shape: expected=%u", expected_submits); + return -1; + } + const uint64_t batches = expected_submits / 5U; + const uint64_t max_role_calls = 2U * batches; + min_kernel_calls = batches; + max_kernel_calls = 4U * batches; + // 当前 PA 每个 batch 的 QK 无 fanin，最迟会在后继 SF Submit 的 EfDrain + // 执行，因此 AIC 至少应观测到 batches 次；其余 Kernel 可能落入 + // FinalDrain，只能校验角色/总调用数上界，不能强求 4*batches 等式。 + if (aic.kernel_calls_sum < batches || aic.kernel_calls_sum > max_role_calls || + aiv.kernel_calls_sum > max_role_calls || aic.kernel_calls_sum + aiv.kernel_calls_sum > max_kernel_calls) { + LOG_ERROR( + "fdwic perf-clock-kernel call closure failed: AIC=%llu/[%llu,%llu] AIV=%llu/[0,%llu] ALL=%llu/[" + "%llu,%llu]", + static_cast(aic.kernel_calls_sum), static_cast(batches), + static_cast(max_role_calls), static_cast(aiv.kernel_calls_sum), + static_cast(max_role_calls), + static_cast(aic.kernel_calls_sum + aiv.kernel_calls_sum), + static_cast(batches), static_cast(max_kernel_calls) + ); + return -1; + } + } + + const std::string path = perf_clock_output_path_from_prefix(runtime->fdwic_swimlane_output_prefix_); + const std::string temporary_path = path + ".tmp"; + if (!remove_output_if_present(temporary_path)) return -1; + std::ofstream out(temporary_path, std::ios::out | std::ios::trunc); + if (!out.is_open()) { + LOG_ERROR("cannot open fdwic perf-clock output %s: %s", temporary_path.c_str(), std::strerror(errno)); + return -1; + } + const uint64_t global_elapsed = global_end - global_start; + out << "{\n"; + out << " \"schema\": \"" << (kernel_profile ? "fdwic-perf-clock-kernel-v1" : "fdwic-perf-clock-v1") << "\",\n"; + out << " \"mode\": \"" << profile_name << "\",\n"; + out << " \"clock_freq_hz\": " << header->freq_hz << ",\n"; + out << " \"device_header_bytes\": " << sizeof(FdwicSwimlaneHeader) << ",\n"; + out << " \"num_cores\": " << header->num_cores << ",\n"; + out << " \"aic_cores\": " << aic.cores << ",\n"; + out << " \"aiv_cores\": " << aiv.cores << ",\n"; + out << " \"expected_submits_per_core\": " << expected_submits << ",\n"; + out << " \"global_first_submit_start\": " << global_start << ",\n"; + out << " \"global_last_submit_end\": " << global_end << ",\n"; + out << " \"global_submit_span_ticks\": " << global_elapsed << ",\n"; + out << " \"global_submit_span_us\": " + << static_cast(global_elapsed) * 1000000.0 / static_cast(header->freq_hz) << ",\n"; + if (kernel_profile) { + const uint64_t elapsed_sum = aic.elapsed_sum + aiv.elapsed_sum; + const uint64_t kernel_sum = aic.kernel_sum + aiv.kernel_sum; + out << " \"kernel_boundary\": \"linked_kernel_call_within_per_core_submit_window\",\n"; + out << " \"kernel_calls\": " << aic.kernel_calls_sum + aiv.kernel_calls_sum << ",\n"; + out << " \"min_kernel_calls_in_window\": " << min_kernel_calls << ",\n"; + out << " \"max_kernel_calls_in_window\": " << max_kernel_calls << ",\n"; + out << " \"kernel_elapsed_ticks_sum\": " << kernel_sum << ",\n"; + out << " \"non_kernel_residual_ticks_sum\": " << elapsed_sum - kernel_sum << ",\n"; + out << " \"kernel_core_time_share\": " << static_cast(kernel_sum) / elapsed_sum << ",\n"; + } + out << " \"groups\": {\n"; + if (kernel_profile) { + out << " \"aic\": {\"cores\": " << aic.cores << ", \"elapsed_min_ticks\": " << aic.elapsed_min + << ", \"elapsed_max_ticks\": " << aic.elapsed_max << ", \"elapsed_sum_ticks\": " << aic.elapsed_sum + << ", \"elapsed_mean_ticks\": " << static_cast(aic.elapsed_sum) / aic.cores + << ", \"kernel_min_ticks\": " << aic.kernel_min << ", \"kernel_max_ticks\": " << aic.kernel_max + << ", \"kernel_sum_ticks\": " << aic.kernel_sum + << ", \"kernel_mean_ticks\": " << static_cast(aic.kernel_sum) / aic.cores + << ", \"kernel_calls_min\": " << aic.kernel_calls_min << ", \"kernel_calls_max\": " << aic.kernel_calls_max + << ", \"kernel_calls_sum\": " << aic.kernel_calls_sum << ", \"residual_min_ticks\": " << aic.residual_min + << ", \"residual_max_ticks\": " << aic.residual_max << ", \"residual_sum_ticks\": " << aic.residual_sum + << ", \"residual_mean_ticks\": " << static_cast(aic.residual_sum) / aic.cores + << ", \"kernel_core_time_share\": " << static_cast(aic.kernel_sum) / aic.elapsed_sum << "},\n"; + out << " \"aiv\": {\"cores\": " << aiv.cores << ", \"elapsed_min_ticks\": " << aiv.elapsed_min + << ", \"elapsed_max_ticks\": " << aiv.elapsed_max << ", \"elapsed_sum_ticks\": " << aiv.elapsed_sum + << ", \"elapsed_mean_ticks\": " << static_cast(aiv.elapsed_sum) / aiv.cores + << ", \"kernel_min_ticks\": " << aiv.kernel_min << ", \"kernel_max_ticks\": " << aiv.kernel_max + << ", \"kernel_sum_ticks\": " << aiv.kernel_sum + << ", \"kernel_mean_ticks\": " << static_cast(aiv.kernel_sum) / aiv.cores + << ", \"kernel_calls_min\": " << aiv.kernel_calls_min << ", \"kernel_calls_max\": " << aiv.kernel_calls_max + << ", \"kernel_calls_sum\": " << aiv.kernel_calls_sum << ", \"residual_min_ticks\": " << aiv.residual_min + << ", \"residual_max_ticks\": " << aiv.residual_max << ", \"residual_sum_ticks\": " << aiv.residual_sum + << ", \"residual_mean_ticks\": " << static_cast(aiv.residual_sum) / aiv.cores + << ", \"kernel_core_time_share\": " << static_cast(aiv.kernel_sum) / aiv.elapsed_sum << "}\n"; + } else { + out << " \"aic\": {\"min_ticks\": " << aic.elapsed_min << ", \"max_ticks\": " << aic.elapsed_max + << ", \"mean_ticks\": " << static_cast(aic.elapsed_sum) / aic.cores << "},\n"; + out << " \"aiv\": {\"min_ticks\": " << aiv.elapsed_min << ", \"max_ticks\": " << aiv.elapsed_max + << ", \"mean_ticks\": " << static_cast(aiv.elapsed_sum) / aiv.cores << "}\n"; + } + out << " },\n"; + out << " \"cores\": [\n"; + for (uint32_t core_id = 0; core_id < header->num_cores; ++core_id) { + const FdwicSwimlaneCoreState &core = header->cores[core_id]; + const uint64_t first_submit_start = + kernel_profile ? core.perf_clock_kernel.first_submit_start : core.perf_clock.first_submit_start; + const uint64_t last_submit_end = + kernel_profile ? core.perf_clock_kernel.last_submit_end : core.perf_clock.last_submit_end; + const uint32_t submit_count = + kernel_profile ? core.perf_clock_kernel.submit_count : core.perf_clock.submit_count; + const uint64_t elapsed = last_submit_end - first_submit_start; + out << " {\"core_id\": " << core_id << ", \"core_type\": \"" + << core_type_name(runtime->workers[core_id].core_type) << "\", \"block_id\": " << core.block_id + << ", \"lane\": " << core.lane << ", \"submit_count\": " << submit_count + << ", \"first_submit_start\": " << first_submit_start << ", \"last_submit_end\": " << last_submit_end + << ", \"elapsed_ticks\": " << elapsed; + if (kernel_profile) { + const uint64_t kernel_elapsed_ticks = core.perf_clock_kernel.kernel_elapsed_ticks; + out << ", \"kernel_elapsed_ticks\": " << kernel_elapsed_ticks << ", \"kernel_calls\": " << core.count + << ", \"non_kernel_residual_ticks\": " << elapsed - kernel_elapsed_ticks; + } + out << "}" << (core_id + 1 == header->num_cores ? "\n" : ",\n"); + } + out << " ]\n}\n"; + out.close(); + if (!out) { + LOG_ERROR("failed while writing fdwic perf-clock output %s", temporary_path.c_str()); + std::remove(temporary_path.c_str()); + return -1; + } + if (std::rename(temporary_path.c_str(), path.c_str()) != 0) { + LOG_ERROR("cannot finalize fdwic perf-clock output %s: %s", path.c_str(), std::strerror(errno)); + std::remove(temporary_path.c_str()); + return -1; + } + if (kernel_profile) { + LOG_INFO_V0( + "fdwic perf-clock-kernel written to %s: cores=%u AIC=%u AIV=%u submits/core=%u span=%.3fus " + "kernel_calls=%llu/[%llu,%llu] kernel_ticks=%llu", + path.c_str(), header->num_cores, aic.cores, aiv.cores, expected_submits, + static_cast(global_elapsed) * 1000000.0 / static_cast(header->freq_hz), + static_cast(aic.kernel_calls_sum + aiv.kernel_calls_sum), + static_cast(min_kernel_calls), static_cast(max_kernel_calls), + static_cast(aic.kernel_sum + aiv.kernel_sum) + ); + } else { + LOG_INFO_V0( + "fdwic perf-clock written to %s: cores=%u AIC=%u AIV=%u submits/core=%u span=%.3fus", path.c_str(), + header->num_cores, aic.cores, aiv.cores, expected_submits, + static_cast(global_elapsed) * 1000000.0 / static_cast(header->freq_hz) + ); + } + return 0; +} + +extern "C" void fdwic_perf_clock_host_finalize(Runtime *runtime) { fdwic_swimlane_host_finalize(runtime); } diff --git a/src/a5/runtime/fully_distributed_within_core/host/runtime_maker.cpp b/src/a5/runtime/fully_distributed_within_core/host/runtime_maker.cpp index 341e16bc78..f8299d7fae 100644 --- a/src/a5/runtime/fully_distributed_within_core/host/runtime_maker.cpp +++ b/src/a5/runtime/fully_distributed_within_core/host/runtime_maker.cpp @@ -119,6 +119,16 @@ extern "C" int bind_callable_to_runtime_impl( LOG_ERROR("Runtime pointer is null"); return -1; } + if (!fdwic_build_identity_matches(runtime->fdwic_build_identity, static_cast(sizeof(Runtime)))) { + LOG_ERROR( + "FDWIC host Runtime build identity is corrupt or stale " + "(abi=%u, mode=%u, ring_cap=%u, runtime_bytes=%u)", + runtime->fdwic_build_identity.abi_version, runtime->fdwic_build_identity.tensor_map_mode, + runtime->fdwic_build_identity.tensor_map_ring_cap, runtime->fdwic_build_identity.runtime_bytes + ); + return -1; + } + runtime->fdwic_build_identity.error_bits = FdwicBuildErrorNone; if (orch_args == nullptr) { LOG_ERROR("orch_args pointer is null"); return -1; diff --git a/src/a5/runtime/fully_distributed_within_core/orchestration/pto_orchestration_api.h b/src/a5/runtime/fully_distributed_within_core/orchestration/pto_orchestration_api.h index f774041c2a..64aa3e0b82 100644 --- a/src/a5/runtime/fully_distributed_within_core/orchestration/pto_orchestration_api.h +++ b/src/a5/runtime/fully_distributed_within_core/orchestration/pto_orchestration_api.h @@ -53,6 +53,53 @@ PTO_DEVICE_FUNC inline TaskOutputTensors rt_submit_aiv_task(int32_t kernel_id, c return rt_submit_task(mk, args); } +/** + * Compete-first eager wrappers. + * + * `args` is owned by the caller and is deliberately not reset here. The + * callback runs synchronously after EfDrain/Claim and before Finish; neither + * the closure nor an internal thunk is retained by the runtime. Device + * orchestration must therefore give its lambda an AICore-callable operator + * (for example, append `__aicore__` in a CCEC source). The callback must not + * submit another task or mutate the `MixedKernels` object used by the matching + * Begin/Finish pair. + */ +template +PTO_DEVICE_FUNC inline TaskOutputTensors alloc_tensors_compete_first(L0TaskArgs &args, BuildArgs &&build_args) { + if (dist_is_fatal_query()) return TaskOutputTensors{}; + const DistCompeteFirstTicket ticket = dist_alloc_compete_first_begin(nullptr); + if (ticket.ready != 0) build_args(args); + return dist_alloc_compete_first_finish(nullptr, ticket, args); +} + +template +PTO_DEVICE_FUNC inline TaskOutputTensors rt_submit_task_compete_first( + const MixedKernels &mixed_kernels, L0TaskArgs &args, BuildArgs &&build_args +) { + if (dist_is_fatal_query()) return TaskOutputTensors{}; + const DistCompeteFirstTicket ticket = dist_submit_compete_first_begin(nullptr, mixed_kernels); + if (ticket.ready != 0) build_args(args); + return dist_submit_compete_first_finish(nullptr, mixed_kernels, ticket, args); +} + +template +PTO_DEVICE_FUNC inline TaskOutputTensors rt_submit_aic_task_compete_first( + int32_t kernel_id, L0TaskArgs &args, BuildArgs &&build_args +) { + MixedKernels mk; + mk.aic_kernel_id = kernel_id; + return rt_submit_task_compete_first(mk, args, static_cast(build_args)); +} + +template +PTO_DEVICE_FUNC inline TaskOutputTensors rt_submit_aiv_task_compete_first( + int32_t kernel_id, L0TaskArgs &args, BuildArgs &&build_args +) { + MixedKernels mk; + mk.aiv0_kernel_id = kernel_id; + return rt_submit_task_compete_first(mk, args, static_cast(build_args)); +} + PTO_DEVICE_FUNC inline TaskOutputTensors rt_submit_dummy_task(const L0TaskArgs &args) { if (dist_is_fatal_query()) return TaskOutputTensors{}; return dist_submit_dummy_impl(nullptr, args); @@ -72,6 +119,16 @@ PTO_DEVICE_FUNC inline void rt_orchestration_done() { dist_orchestration_done_im PTO_DEVICE_FUNC inline bool rt_is_fatal() { return dist_is_fatal_query(); } +PTO_DEVICE_FUNC inline void rt_perf_clock_expect_submits(uint32_t expected_submits) { +#if PTO_FDWIC_PERF_CLOCK + dist_perf_clock_expect_submits(expected_submits); +#elif PTO_FDWIC_SUBMIT_PMU + dist_submit_pmu_expect_submits(expected_submits); +#else + (void)expected_submits; +#endif +} + #define rt_report_fatal(code, fmt, ...) \ do { \ dist_report_fatal_msg((code), __FUNCTION__, (fmt)); \ diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/api_glue.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/api_glue.h index e9605aa704..0ace58d68c 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/api_glue.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/api_glue.h @@ -18,7 +18,7 @@ DIST_API_ATTR PTO_DEVICE_FUNC bool dist_is_fatal_query() { #if defined(__CCE_AICORE__) return false; #else - return fatal_set(); + return fdwic_trace_is_fatal(); #endif } @@ -43,6 +43,22 @@ DIST_API_ATTR PTO_DEVICE_FUNC void dist_scope_end_impl(PTO2Runtime *) {} DIST_API_ATTR PTO_DEVICE_FUNC void dist_orchestration_done_impl(PTO2Runtime *) {} DIST_API_ATTR PTO_DEVICE_FUNC void dist_scope_set_site_impl(const char *, int) {} +#if PTO_FDWIC_PERF_CLOCK +DIST_API_ATTR PTO_DEVICE_FUNC void dist_perf_clock_expect_submits(uint32_t expected_submits) { + fdwic_perf_clock_expect_submits(expected_submits); +} +#if PTO_FDWIC_PERF_CLOCK_KERNEL +// 只作为最终 ELF 的构建身份标记，不进入热路径。 +DIST_API_ATTR PTO_DEVICE_FUNC uint32_t dist_perf_clock_kernel_profile_marker() { return kFdwicPerfClockKernelMode; } +#endif +#endif + +#if PTO_FDWIC_SUBMIT_PMU +DIST_API_ATTR PTO_DEVICE_FUNC void dist_submit_pmu_expect_submits(uint32_t expected_submits) { + fdwic_submit_pmu_expect_submits(expected_submits); +} +#endif + DIST_API_ATTR PTO_DEVICE_FUNC TaskOutputTensors dist_submit_dummy_impl(PTO2Runtime *, const L0TaskArgs &) { return TaskOutputTensors{}; } diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/core_main.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/core_main.h index eb0af23a47..3b1b0e11e5 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/core_main.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/core_main.h @@ -16,22 +16,45 @@ DIST_API_ATTR PTO_DEVICE_FUNC void dist_core_main(__gm__ Runtime *runtime, int c __gm__ DistCore *self = dist_aicore_attach_worker(runtime, core_idx, core_type_int); if (self == nullptr) return; g_fdwic_joint_submit_seen = false; + fdwic_perf_clock_attach(runtime, self); + fdwic_submit_pmu_attach(runtime, self); fdwic_swimlane_attach(runtime); trace_reset_core(self); - if (!fatal_set()) { - atomic_fetch_add(g_dist.started_count, 1); + if (!fdwic_trace_is_fatal()) { + (void)fdwic_trace_atomic_fetch_add( + -1, FdwicAtomicSite::StartupIncrement, g_dist.started_count, 1, /*result_used=*/false + ); uint64_t wd_start = 0; - while (atomic_load(g_dist.started_count) < g_dist.num_workers && !fatal_set()) { + const uint32_t startup_poll_region = fdwic_atomic_poll_region_begin( + fdwic_atomic_site_mask(FdwicAtomicSite::StartupPoll) | fdwic_atomic_site_mask(FdwicAtomicSite::FatalPoll) + ); + while (fdwic_trace_atomic_load(-1, FdwicAtomicSite::StartupPoll, g_dist.started_count) < g_dist.num_workers && + !fdwic_trace_is_fatal()) { SPIN_WAIT_HINT(); watchdog(wd_start); } + fdwic_atomic_poll_region_end(startup_poll_region); } - TRACE_LAP_RESET(self); // origin for the first lap span (post-barrier, pre-replay) + // Schema-v4 observes the complete worker business window as two adjacent + // parents. Reuse the orchestration end as the final-drain start so their + // aggregate closes exactly in integer SYS_CNT cycles. + TRACE_TIMESTAMP(orchestration_begin); dist_submit_replay_orch(runtime); - - dist_submit_drain_to_completion(self); + TRACE_TIMESTAMP(orchestration_end); + // 失败运行不再等待已经失去依赖闭包的 task ring。容量错误由每个 + // private replica 在相同逻辑 Submit 上确定性发现，worker 直接完成， + // AICPU 汇总全局 error_code 后向 Host 返回非零。 + if (!fdwic_trace_is_fatal()) dist_submit_drain_to_completion(self); + TRACE_TIMESTAMP(final_drain_end); + // Publish both parent records after the measured work. Their own GM writes + // therefore belong to neither business interval. + TRACE_SPAN_RECORD(orchestration_begin, orchestration_end, self, -1, -1, TracePhase::OrchestrationReplay, 0, 0); + TRACE_SPAN_RECORD(orchestration_end, final_drain_end, self, -1, -1, TracePhase::FinalDrain, 0, 0); + fdwic_swimlane_record_clock_baselines(self, core_idx); TRACE_FLUSH_CORE(self); + fdwic_perf_clock_flush(self); + fdwic_submit_pmu_flush(self); dist_aicore_finish_worker(runtime); } diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/core_state.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/core_state.h index faf311d784..aab95f3aa4 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/core_state.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/core_state.h @@ -20,7 +20,7 @@ PTO_DEVICE_FUNC void dist_core_reset(__gm__ DistCore &self, CoreType r, int32_t self.sub_block_id = (lane_id == LANE_AIV1) ? 1 : 0; self.local_index = 0; self.heap_next = 0; - dist_tensor_map_reset(self.map); + dist_tensor_map_reset_worker(self); self.occupied_count = 0; self.owned_total = 0; self.swimlane_last_cycle = 0; diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/dist_engine.cpp b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/dist_engine.cpp index 9887be57cc..1acc3d9642 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/dist_engine.cpp +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/dist_engine.cpp @@ -45,6 +45,8 @@ extern "C" PTO_DEVICE_FUNC void aicpu_orchestration_entry(const L2TaskArgs &orch #endif #include "dist_engine/aicore/primitive.h" // NOLINT(build/include_subdir) +#include "dist_engine/common/perf_clock.h" // NOLINT(build/include_subdir) +#include "dist_engine/common/submit_pmu.h" // NOLINT(build/include_subdir) #include "dist_engine/aicore/api_glue.h" // NOLINT(build/include_subdir) #include "dist_engine/aicore/log.h" // NOLINT(build/include_subdir) #include "dist_engine/aicore/tensor_map.h" // NOLINT(build/include_subdir) diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/private_tensor_map.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/private_tensor_map.h new file mode 100644 index 0000000000..7b8a5b7229 --- /dev/null +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/private_tensor_map.h @@ -0,0 +1,156 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include "dist_engine/aicore/tensor_map_common.h" + +namespace { + +PTO_DEVICE_FUNC inline uint64_t +dist_private_tensor_map_load_head(__gm__ const DistTensorMap &self, uint32_t bucket) { +#if PTO_FDWIC_TENSORMAP_RING_CAP < 128 + if (bucket >= kMapBaseControlBuckets) { + return self.extra_bucket_heads[bucket - kMapBaseControlBuckets]; + } +#endif + return self.bucket_heads[bucket]; +} + +PTO_DEVICE_FUNC inline void +dist_private_tensor_map_store_head(__gm__ DistTensorMap &self, uint32_t bucket, uint64_t value) { +#if PTO_FDWIC_TENSORMAP_RING_CAP < 128 + if (bucket >= kMapBaseControlBuckets) { + self.extra_bucket_heads[bucket - kMapBaseControlBuckets] = value; + return; + } +#endif + self.bucket_heads[bucket] = value; +} + +PTO_DEVICE_FUNC inline uint64_t +dist_private_tensor_map_load_tail(__gm__ const DistTensorMap &self, uint32_t bucket) { +#if PTO_FDWIC_TENSORMAP_RING_CAP < 128 + if (bucket >= kMapBaseControlBuckets) { + return self.extra_bucket_tails[bucket - kMapBaseControlBuckets]; + } +#endif + return self.bucket_tails[bucket]; +} + +PTO_DEVICE_FUNC inline void +dist_private_tensor_map_store_tail(__gm__ DistTensorMap &self, uint32_t bucket, uint64_t value) { +#if PTO_FDWIC_TENSORMAP_RING_CAP < 128 + if (bucket >= kMapBaseControlBuckets) { + self.extra_bucket_tails[bucket - kMapBaseControlBuckets] = value; + return; + } +#endif + self.bucket_tails[bucket] = value; +} + +PTO_DEVICE_FUNC inline void dist_private_tensor_map_reset(__gm__ DistTensorMap &self) { + self.alive_floor = 0; + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + dist_private_tensor_map_store_head(self, bucket, 0); + dist_private_tensor_map_store_tail(self, bucket, 0); + } +} + +PTO_DEVICE_FUNC inline uint32_t dist_private_tensor_map_hash(uint64_t addr) { return dist_tensor_map_hash(addr); } + +template +PTO_DEVICE_FUNC inline void +dist_private_tensor_map_byte_range(const TensorRef &t, uint64_t &addr, uint64_t &lo, uint64_t &hi) { + dist_tensor_map_byte_range(t, addr, lo, hi); +} + +PTO_DEVICE_FUNC inline uint32_t dist_private_tensor_map_slot_index(uint32_t bucket, uint64_t cursor) { + return dist_tensor_map_slot_index(bucket, cursor); +} + +PTO_DEVICE_FUNC inline void dist_private_tensor_map_retire_bucket( + __gm__ DistTensorMap &self, uint32_t bucket, uint64_t &head, uint64_t &tail +) { + head = dist_private_tensor_map_load_head(self, bucket); + tail = dist_private_tensor_map_load_tail(self, bucket); + const uint64_t original_head = head; + while (head < tail) { + __gm__ const MapEntry &entry = self.entries[dist_private_tensor_map_slot_index(bucket, head)]; + if (entry.producer >= self.alive_floor) { + break; + } + ++head; + } + if (head != original_head) { + dist_private_tensor_map_store_head(self, bucket, head); + } +} + +PTO_DEVICE_FUNC inline void +dist_private_tensor_map_advance_retire(__gm__ DistTensorMap &self, int32_t task_id, int32_t history) { + const int32_t new_floor = task_id - history; + if (new_floor > self.alive_floor) { + self.alive_floor = new_floor; + } +} + +template +PTO_DEVICE_FUNC inline bool dist_private_tensor_map_insert( + __gm__ DistTensorMap &self, const TensorRef &t, int32_t producer +) { + uint64_t addr, lo, hi; + dist_private_tensor_map_byte_range(t, addr, lo, hi); + const uint32_t bucket = dist_private_tensor_map_hash(addr); + uint64_t head, tail; + dist_private_tensor_map_retire_bucket(self, bucket, head, tail); + if (tail - head >= kMapBucketCapacity) { + return false; + } + + // private replay 的 task_id 来自单调递增的 DistCore::local_index，同一 + // task 的多个 OUTPUT 可以相等，因此同桶 producer 必须单调不降。 + // 桶头 lazy-retire 正是建立在这一合同上；CCEC 中 always_assert + // 为零成本，CPU/A5sim 则在所有构建类型拒绝绕过 Submit 的逆序调用。 + always_assert( + tail == head || + self.entries[dist_private_tensor_map_slot_index(bucket, tail - 1)].producer <= producer + ); + __gm__ MapEntry &entry = self.entries[dist_private_tensor_map_slot_index(bucket, tail)]; + entry.buf_addr = addr; + entry.lo = lo; + entry.hi = hi; + entry.producer = producer; + dist_private_tensor_map_store_tail(self, bucket, tail + 1); + return true; +} + +template +PTO_DEVICE_FUNC inline int32_t dist_private_tensor_map_lookup(__gm__ DistTensorMap &self, const TensorRef &t) { + uint64_t addr, lo, hi; + dist_private_tensor_map_byte_range(t, addr, lo, hi); + const uint32_t bucket = dist_private_tensor_map_hash(addr); + uint64_t head, tail; + dist_private_tensor_map_retire_bucket(self, bucket, head, tail); + int32_t best = -1; + for (uint64_t cursor = head; cursor < tail; ++cursor) { + __gm__ const MapEntry &entry = self.entries[dist_private_tensor_map_slot_index(bucket, cursor)]; + if (entry.producer < self.alive_floor) { + continue; + } + if (entry.buf_addr == addr && lo < entry.hi && entry.lo < hi && entry.producer > best) { + best = entry.producer; + } + } + return best; +} + +} // namespace diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/shared_tensor_map.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/shared_tensor_map.h new file mode 100644 index 0000000000..3d83187ac7 --- /dev/null +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/shared_tensor_map.h @@ -0,0 +1,465 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include "dist_engine/common/state.h" +#include "dist_engine/aicore/primitive.h" +#include "dist_engine/aicore/tensor_map_common.h" +#include "dist_engine/common/atomic.h" + +namespace { + +// Preflight must distinguish capacity exhaustion from corrupted shared state. +// Capacity exhaustion cannot resolve by waiting under exact-turn and must map +// to the structured TensorMap capacity fatal; protocol errors require their +// own convergence path. +enum class DistSharedTensorMapAppendCheck : uint32_t { + Ready = 0, + CapacityBlocked = 1, + ProtocolError = 2, +}; + +// The task adapter must distinguish rejection before this call publishes +// current-task data from failure after acquiring slot ownership. The latter +// cannot be rolled back or retried: preserve the evidence and fail-stop the +// run. ProtocolError does not claim the whole sidecar is clean; a competing +// writer may already have corrupted shared control state. +enum class DistSharedTensorMapEntryPublishResult : uint32_t { + Published = 0, + ProtocolError = 1, + PartialPublish = 2, +}; + +enum class DistSharedTensorMapTaskPublishResult : uint32_t { + Committed = 0, + CapacityBlocked = 1, + ProtocolError = 2, + PartialPublish = 3, +}; + +PTO_DEVICE_FUNC inline uint32_t dist_shared_tensor_map_slot_index(uint32_t bucket, uint64_t cursor) { + return dist_tensor_map_slot_index(bucket, cursor); +} + +template +PTO_DEVICE_FUNC inline SharedTensorMapValue +dist_shared_tensor_map_make_value(const TensorRef &tensor, int32_t producer) { + SharedTensorMapValue value{}; + dist_tensor_map_byte_range(tensor, value.buf_addr, value.lo, value.hi); + value.producer = producer; + value.reserved = 0; + return value; +} + +// Ops is the only seam between standalone tests and production AICore +// primitives. The state machine does not access g_dist or depend on +// Submit/fatal/wait, so it can be verified before the backend gate is opened. +template +PTO_DEVICE_FUNC inline bool dist_shared_tensor_map_read_slot_impl( + __gm__ SharedTensorMapState &map, uint32_t bucket, uint64_t cursor, SharedTensorMapValue &snapshot +) { + if (bucket >= kMapBuckets || cursor > static_cast(INT64_MAX)) { + return false; + } + __gm__ SharedTensorMapSlot &slot = map.slots[dist_shared_tensor_map_slot_index(bucket, cursor)]; + const int64_t expected = static_cast(cursor); + if (Ops::Load(&slot.sequence.v) != expected) { + return false; + } + Ops::InvalidateRegion(&slot.payload, sizeof(slot.payload)); + snapshot.buf_addr = slot.payload.value.buf_addr; + snapshot.lo = slot.payload.value.lo; + snapshot.hi = slot.payload.value.hi; + snapshot.producer = slot.payload.value.producer; + snapshot.reserved = slot.payload.value.reserved; + if (Ops::Load(&slot.sequence.v) != expected) { + return false; + } + return snapshot.producer >= 0 && snapshot.reserved == 0 && snapshot.lo < snapshot.hi; +} + +// The authoritative lookup window is [max(0, N-H), N). A -1 result can mean a +// normal miss or a sequence/cursor protocol failure, so callers must also +// inspect protocol_ok. +template +PTO_DEVICE_FUNC inline int32_t dist_shared_tensor_map_lookup_region_impl( + __gm__ SharedTensorMapState &map, const SharedTensorMapValue &query, int32_t current_task, int32_t history, + bool &protocol_ok +) { + protocol_ok = false; + if (current_task < 0 || current_task >= kFlagCap || history < 0 || query.lo >= query.hi) { + return -1; + } + const uint32_t bucket = dist_tensor_map_hash(query.buf_addr); + const int64_t head = Ops::Load(&map.buckets[bucket].head.v); + const int64_t tail = Ops::Load(&map.buckets[bucket].tail.v); + if (head < 0 || tail < head || static_cast(tail - head) > kMapBucketCapacity) { + return -1; + } + + const int32_t lower = current_task > history ? current_task - history : 0; + int32_t best = -1; + for (uint64_t cursor = static_cast(head); cursor < static_cast(tail); ++cursor) { + SharedTensorMapValue candidate{}; + if (!dist_shared_tensor_map_read_slot_impl(map, bucket, cursor, candidate)) { + return -1; + } + if (candidate.producer >= lower && candidate.producer < current_task && + dist_tensor_map_regions_overlap(candidate, query.buf_addr, query.lo, query.hi) && + candidate.producer > best) { + best = candidate.producer; + } + } + protocol_ok = true; + return best; +} + +template +PTO_DEVICE_FUNC inline int32_t dist_shared_tensor_map_lookup_tensor_impl( + __gm__ SharedTensorMapState &map, const TensorRef &tensor, int32_t current_task, int32_t history, bool &protocol_ok +) { + const SharedTensorMapValue query = dist_shared_tensor_map_make_value(tensor, -1); + return dist_shared_tensor_map_lookup_region_impl(map, query, current_task, history, protocol_ok); +} + +// Only the unique winner observing committed_tasks==N may retire or append. +// Concurrent unordered writers touching one bucket are outside this ordered +// single-appender protocol. +template +PTO_DEVICE_FUNC inline bool +dist_shared_tensor_map_retire_bucket_impl(__gm__ SharedTensorMapState &map, uint32_t bucket, int64_t reclaim_upto) { + if (bucket >= kMapBuckets || reclaim_upto < -1) { + return false; + } + __gm__ SharedTensorMapBucketState &controls = map.buckets[bucket]; + const int64_t original_head = Ops::Load(&controls.head.v); + const int64_t tail = Ops::Load(&controls.tail.v); + if (original_head < 0 || tail < original_head || static_cast(tail - original_head) > kMapBucketCapacity) { + return false; + } + + int64_t head = original_head; + while (head < tail) { + SharedTensorMapValue value{}; + if (!dist_shared_tensor_map_read_slot_impl(map, bucket, static_cast(head), value)) { + return false; + } + if (value.producer > reclaim_upto) { + break; + } + ++head; + } + if (head == original_head) { + return true; + } + const int64_t observed = Ops::CompareExchange(&controls.head.v, original_head, head); + return observed == original_head; +} + +// Winner N has completed its lookups, and later tasks can read no earlier than +// N-H, so the inclusive reclaim boundary is N-H-1. Use int64_t to avoid +// underflow at the boundary. +PTO_DEVICE_FUNC inline bool +dist_shared_tensor_map_compute_reclaim(int32_t current_task, int32_t history, int64_t &candidate) { + if (current_task < 0 || history < 0) { + return false; + } + candidate = static_cast(current_task) - static_cast(history) - 1; + if (candidate < -1) { + candidate = -1; + } + return true; +} + +template +PTO_DEVICE_FUNC inline bool +dist_shared_tensor_map_has_exact_turn_impl(__gm__ SharedTensorMapState &map, int32_t current_task) { + // Tasks and flags share the kFlagCap bounded protocol. Reject an invalid + // task before changing reclaim, head, or any slot instead of waiting until + // publish_commit. + return current_task >= 0 && current_task < kFlagCap && Ops::Load(&map.committed_tasks.v) == current_task; +} + +template +PTO_DEVICE_FUNC inline bool dist_shared_tensor_map_refresh_reclaim_impl( + __gm__ SharedTensorMapState &map, int32_t current_task, int32_t history, int64_t &reclaim_upto +) { + if (!dist_shared_tensor_map_has_exact_turn_impl(map, current_task)) { + return false; + } + int64_t candidate = -1; + if (!dist_shared_tensor_map_compute_reclaim(current_task, history, candidate)) { + return false; + } + const int64_t current = Ops::Load(&map.reclaim_upto.v); + if (current < -1 || candidate < current) { + return false; + } + if (candidate == current) { + reclaim_upto = current; + return true; + } + const int64_t observed = Ops::CompareExchange(&map.reclaim_upto.v, current, candidate); + if (observed != current) { + return false; + } + reclaim_upto = candidate; + return true; +} + +PTO_DEVICE_FUNC inline uint32_t +dist_shared_tensor_map_earlier_entries_in_bucket(const SharedTensorMapValue *entries, uint32_t index, uint32_t bucket) { + uint32_t earlier = 0; + for (uint32_t previous = 0; previous < index; ++previous) { + if (dist_tensor_map_hash(entries[previous].buf_addr) == bucket) { + ++earlier; + } + } + return earlier; +} + +// Validate capacity, cursors, and old sequences for the whole task before +// writing any slot. A failed check may retain a proven monotonic reclaim/head +// advance, but must not publish this task's payload, sequence, tail, or commit. +template +PTO_DEVICE_FUNC inline DistSharedTensorMapAppendCheck dist_shared_tensor_map_check_task_append_impl( + __gm__ SharedTensorMapState &map, const SharedTensorMapValue *entries, uint32_t count, int32_t current_task, + int64_t reclaim_upto, uint64_t *planned_cursors = nullptr +) { + if (current_task < 0 || count > MAX_TENSOR_ARGS || (count != 0 && entries == nullptr) || + !dist_shared_tensor_map_has_exact_turn_impl(map, current_task) || + Ops::Load(&map.reclaim_upto.v) != reclaim_upto) { + return DistSharedTensorMapAppendCheck::ProtocolError; + } + for (uint32_t index = 0; index < count; ++index) { + const SharedTensorMapValue &entry = entries[index]; + if (entry.producer != current_task || entry.reserved != 0 || entry.lo >= entry.hi) { + return DistSharedTensorMapAppendCheck::ProtocolError; + } + const uint32_t bucket = dist_tensor_map_hash(entry.buf_addr); + if (!dist_shared_tensor_map_retire_bucket_impl(map, bucket, reclaim_upto)) { + return DistSharedTensorMapAppendCheck::ProtocolError; + } + const int64_t head = Ops::Load(&map.buckets[bucket].head.v); + const int64_t tail = Ops::Load(&map.buckets[bucket].tail.v); + if (head < 0 || tail < head || static_cast(tail - head) > kMapBucketCapacity) { + return DistSharedTensorMapAppendCheck::ProtocolError; + } + const uint32_t earlier = dist_shared_tensor_map_earlier_entries_in_bucket(entries, index, bucket); + const uint64_t occupied = static_cast(tail - head) + earlier; + if (occupied >= kMapBucketCapacity) { + return DistSharedTensorMapAppendCheck::CapacityBlocked; + } + const uint64_t cursor = static_cast(tail) + earlier; + // Append eventually publishes tail=cursor+1, so INT64_MAX is invalid. + if (cursor >= static_cast(INT64_MAX)) { + return DistSharedTensorMapAppendCheck::ProtocolError; + } + __gm__ SharedTensorMapSlot &slot = map.slots[dist_shared_tensor_map_slot_index(bucket, cursor)]; + const int64_t expected_old = cursor < kMapBucketCapacity ? kSharedTensorMapInvalidSequence : + static_cast(cursor - kMapBucketCapacity); + if (Ops::Load(&slot.sequence.v) != expected_old) { + return DistSharedTensorMapAppendCheck::ProtocolError; + } + if (planned_cursors != nullptr) { + planned_cursors[index] = cursor; + } + } + return DistSharedTensorMapAppendCheck::Ready; +} + +template +PTO_DEVICE_FUNC inline DistSharedTensorMapEntryPublishResult dist_shared_tensor_map_publish_prepared_entry_impl( + __gm__ SharedTensorMapState &map, const SharedTensorMapValue &entry, int32_t current_task, uint64_t planned_cursor +) { + if (entry.producer != current_task || entry.reserved != 0 || entry.lo >= entry.hi || + planned_cursor >= static_cast(INT64_MAX)) { + return DistSharedTensorMapEntryPublishResult::ProtocolError; + } + const uint32_t bucket = dist_tensor_map_hash(entry.buf_addr); + __gm__ SharedTensorMapBucketState &controls = map.buckets[bucket]; + const int64_t head = Ops::Load(&controls.head.v); + const int64_t tail = Ops::Load(&controls.tail.v); + if (head < 0 || tail < head || static_cast(tail) != planned_cursor || + static_cast(tail - head) >= kMapBucketCapacity) { + return DistSharedTensorMapEntryPublishResult::ProtocolError; + } + __gm__ SharedTensorMapSlot &slot = map.slots[dist_shared_tensor_map_slot_index(bucket, planned_cursor)]; + const int64_t expected_old = planned_cursor < kMapBucketCapacity ? + kSharedTensorMapInvalidSequence : + static_cast(planned_cursor - kMapBucketCapacity); + // WRITING is the exclusive ownership state. A later CAS failure after a + // successful claim means the exact-turn protocol is broken. Preserve the + // evidence for fatal convergence; never roll an overwritten payload back + // to the previous lap sequence. + const int64_t before_claim = Ops::CompareExchange(&slot.sequence.v, expected_old, kSharedTensorMapWritingSequence); + if (before_claim != expected_old) { + return DistSharedTensorMapEntryPublishResult::ProtocolError; + } + + Ops::InvalidateRegion(&slot.payload, sizeof(slot.payload)); + slot.payload.value.buf_addr = entry.buf_addr; + slot.payload.value.lo = entry.lo; + slot.payload.value.hi = entry.hi; + slot.payload.value.producer = entry.producer; + slot.payload.value.reserved = 0; + // Padding carries no protocol state; avoid scalar stores for padding. + Ops::FlushRegion(&slot.payload, sizeof(slot.payload)); + + const int64_t before_publish = + Ops::CompareExchange(&slot.sequence.v, kSharedTensorMapWritingSequence, static_cast(planned_cursor)); + if (before_publish != kSharedTensorMapWritingSequence) { + return DistSharedTensorMapEntryPublishResult::PartialPublish; + } + const int64_t previous_tail = Ops::CompareExchange( + &controls.tail.v, static_cast(planned_cursor), static_cast(planned_cursor) + 1 + ); + return previous_tail == static_cast(planned_cursor) ? + DistSharedTensorMapEntryPublishResult::Published : + DistSharedTensorMapEntryPublishResult::PartialPublish; +} + +template +PTO_DEVICE_FUNC inline bool +dist_shared_tensor_map_publish_commit_impl(__gm__ SharedTensorMapState &map, int32_t current_task) { + if (current_task < 0 || current_task >= kFlagCap) { + return false; + } + const int64_t previous = + Ops::CompareExchange(&map.committed_tasks.v, current_task, static_cast(current_task) + 1); + return previous == current_task; +} + +template +PTO_DEVICE_FUNC inline DistSharedTensorMapTaskPublishResult dist_shared_tensor_map_publish_prepared_task_impl( + __gm__ SharedTensorMapState &map, const SharedTensorMapValue *entries, const uint64_t *planned_cursors, + uint32_t count, int32_t current_task +) { + if (count > MAX_TENSOR_ARGS || (count != 0 && (entries == nullptr || planned_cursors == nullptr)) || + !dist_shared_tensor_map_has_exact_turn_impl(map, current_task)) { + return DistSharedTensorMapTaskPublishResult::ProtocolError; + } + for (uint32_t index = 0; index < count; ++index) { + const DistSharedTensorMapEntryPublishResult result = dist_shared_tensor_map_publish_prepared_entry_impl( + map, entries[index], current_task, planned_cursors[index] + ); + if (result == DistSharedTensorMapEntryPublishResult::Published) { + continue; + } + if (result == DistSharedTensorMapEntryPublishResult::PartialPublish || index != 0) { + return DistSharedTensorMapTaskPublishResult::PartialPublish; + } + return DistSharedTensorMapTaskPublishResult::ProtocolError; + } + if (!dist_shared_tensor_map_publish_commit_impl(map, current_task)) { + return count == 0 ? DistSharedTensorMapTaskPublishResult::ProtocolError : + DistSharedTensorMapTaskPublishResult::PartialPublish; + } + return DistSharedTensorMapTaskPublishResult::Committed; +} + +// Callers must complete lookup before entering this function. It only handles +// reclaim, whole-task preflight, ordered publication, and the final commit. +// A zero-entry task still advances the contiguous task frontier. Any partial +// publication after preflight is unrecoverable and must reach Submit without +// being downgraded to an ordinary protocol rejection. +template +PTO_DEVICE_FUNC inline DistSharedTensorMapTaskPublishResult dist_shared_tensor_map_publish_task_impl( + __gm__ SharedTensorMapState &map, const SharedTensorMapValue *entries, uint32_t count, int32_t current_task, + int32_t history +) { + if (current_task < 0 || current_task >= kFlagCap || history < 0 || count > MAX_TENSOR_ARGS || + (count != 0 && entries == nullptr)) { + return DistSharedTensorMapTaskPublishResult::ProtocolError; + } + for (uint32_t index = 0; index < count; ++index) { + const SharedTensorMapValue &entry = entries[index]; + if (entry.producer != current_task || entry.reserved != 0 || entry.lo >= entry.hi) { + return DistSharedTensorMapTaskPublishResult::ProtocolError; + } + } + + int64_t reclaim_upto = -2; + if (!dist_shared_tensor_map_refresh_reclaim_impl(map, current_task, history, reclaim_upto)) { + return DistSharedTensorMapTaskPublishResult::ProtocolError; + } + uint64_t planned_cursors[MAX_TENSOR_ARGS]; + const DistSharedTensorMapAppendCheck check = dist_shared_tensor_map_check_task_append_impl( + map, entries, count, current_task, reclaim_upto, planned_cursors + ); + if (check == DistSharedTensorMapAppendCheck::CapacityBlocked) { + return DistSharedTensorMapTaskPublishResult::CapacityBlocked; + } + if (check != DistSharedTensorMapAppendCheck::Ready) { + return DistSharedTensorMapTaskPublishResult::ProtocolError; + } + + return dist_shared_tensor_map_publish_prepared_task_impl(map, entries, planned_cursors, count, current_task); +} + +struct DistSharedTensorMapAicoreOps { + PTO_DEVICE_FUNC static int64_t Load(__gm__ volatile int64_t *address) { return atomic_load(*address); } + + PTO_DEVICE_FUNC static int64_t + CompareExchange(__gm__ volatile int64_t *address, int64_t expected, int64_t desired) { + return atomic_compare_exchange(*address, expected, desired); + } + + PTO_DEVICE_FUNC static void InvalidateRegion(__gm__ const void *address, uint64_t bytes) { + dist_aicore_invalidate_region(address, bytes); + } + + PTO_DEVICE_FUNC static void FlushRegion(__gm__ void *address, uint64_t bytes) { + dist_aicore_flush_region(address, bytes); + } +}; + +// These concrete wrappers force Host, CPU-sim, and CCEC to instantiate the +// same production primitives. The shared Submit facade calls them, while the +// top-level backend gate remains closed until the remaining concurrency and +// device-visibility gates pass. +PTO_DEVICE_FUNC inline int32_t dist_shared_tensor_map_lookup_region( + __gm__ SharedTensorMapState &map, const SharedTensorMapValue &query, int32_t current_task, int32_t history, + bool &protocol_ok +) { + return dist_shared_tensor_map_lookup_region_impl( + map, query, current_task, history, protocol_ok + ); +} + +template +PTO_DEVICE_FUNC inline int32_t dist_shared_tensor_map_lookup_tensor( + __gm__ SharedTensorMapState &map, const TensorRef &tensor, int32_t current_task, int32_t history, bool &protocol_ok +) { + return dist_shared_tensor_map_lookup_tensor_impl( + map, tensor, current_task, history, protocol_ok + ); +} + +PTO_DEVICE_FUNC inline bool dist_shared_tensor_map_refresh_reclaim( + __gm__ SharedTensorMapState &map, int32_t current_task, int32_t history, int64_t &reclaim_upto +) { + return dist_shared_tensor_map_refresh_reclaim_impl( + map, current_task, history, reclaim_upto + ); +} + +PTO_DEVICE_FUNC inline DistSharedTensorMapTaskPublishResult dist_shared_tensor_map_publish_task( + __gm__ SharedTensorMapState &map, const SharedTensorMapValue *entries, uint32_t count, int32_t current_task, + int32_t history +) { + return dist_shared_tensor_map_publish_task_impl( + map, entries, count, current_task, history + ); +} + +} // namespace diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/submit_core.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/submit_core.h index fa14f2bc49..30056b3458 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/submit_core.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/submit_core.h @@ -35,46 +35,76 @@ namespace { PTO_DEVICE_FUNC void publish_task_flag(int32_t task_id) { if (task_id < 0 || task_id >= kFlagCap) return; __gm__ DistTaskCell &cell = task_cell(task_id); - atomic_exchange(cell.flag, int64_t{1}, __ATOMIC_RELEASE); + (void)fdwic_trace_atomic_exchange( + task_id, FdwicAtomicSite::CompletionFlagExchange, cell.flag, int64_t{1}, /*result_used=*/false, __ATOMIC_RELEASE + ); } -PTO_DEVICE_FUNC bool task_flag_ready(int32_t task_id, int memorder) { +PTO_DEVICE_FUNC bool task_flag_ready(int32_t task_id, int memorder, FdwicAtomicSite site) { if (task_id < 0 || task_id >= kFlagCap) return false; __gm__ DistTaskCell &cell = task_cell(task_id); - return atomic_load(cell.flag, memorder) != 0; + return fdwic_trace_atomic_load(task_id, site, cell.flag, /*result_used=*/true, memorder) != 0; } PTO_DEVICE_FUNC void store_task_vend(int32_t task_id, uint64_t vend) { if (task_id < 0 || task_id >= kFlagCap) return; __gm__ DistTaskCell &cell = task_cell(task_id); - atomic_exchange(cell.vend, vend, __ATOMIC_RELAXED); + (void)fdwic_trace_atomic_exchange( + task_id, FdwicAtomicSite::CompletionVendExchange, cell.vend, vend, /*result_used=*/false, __ATOMIC_RELAXED + ); } -PTO_DEVICE_FUNC void store_won_remaining(__gm__ WonSlot &w, int32_t count) { - atomic_exchange(w.remaining.v, static_cast(count), __ATOMIC_RELAXED); +PTO_DEVICE_FUNC void store_won_remaining(__gm__ WonSlot &w, int32_t count, int32_t task_id) { + (void)fdwic_trace_atomic_exchange( + task_id, FdwicAtomicSite::WonRemainingExchange, w.remaining.v, static_cast(count), + /*result_used=*/false, __ATOMIC_RELAXED + ); } -PTO_DEVICE_FUNC void reset_won_lane(__gm__ WonSlot &w, int32_t lane) { - atomic_exchange(w.drained[lane].v, kDrainedClaimed); +PTO_DEVICE_FUNC void reset_won_lane(__gm__ WonSlot &w, int32_t lane, int32_t task_id) { + (void)fdwic_trace_atomic_exchange( + task_id, FdwicAtomicSite::WonLaneResetExchange, w.drained[lane].v, kDrainedClaimed, + /*result_used=*/false + ); w.lane[lane].present = false; } PTO_DEVICE_FUNC bool claim_won_lane(__gm__ WonSlot &w, int32_t lane) { - return atomic_exchange(w.drained[lane].v, kDrainedClaimed) == kDrainedFree; + return fdwic_trace_atomic_exchange( + -1, FdwicAtomicSite::WonLaneClaimExchange, w.drained[lane].v, kDrainedClaimed, + /*result_used=*/true + ) == kDrainedFree; } -PTO_DEVICE_FUNC void publish_won_slot(__gm__ WonSlot &w) { atomic_exchange(w.state.v, kWonStatePublished); } +PTO_DEVICE_FUNC void publish_won_slot(__gm__ WonSlot &w, int32_t task_id) { + (void)fdwic_trace_atomic_exchange( + task_id, FdwicAtomicSite::WonStatePublishExchange, w.state.v, kWonStatePublished, + /*result_used=*/false + ); +} -PTO_DEVICE_FUNC bool decrement_won_remaining_is_last(__gm__ WonSlot &w) { - return atomic_fetch_sub(w.remaining.v, 1) == 1; +PTO_DEVICE_FUNC bool decrement_won_remaining_is_last(__gm__ WonSlot &w, int32_t task_id) { + return fdwic_trace_atomic_fetch_sub( + task_id, FdwicAtomicSite::WonRemainingFetchSub, w.remaining.v, 1, /*result_used=*/true + ) == 1; } -PTO_DEVICE_FUNC void clear_won_slot_state(__gm__ WonSlot &w) { atomic_exchange(w.state.v, kWonStateFree); } +PTO_DEVICE_FUNC void clear_won_slot_state(__gm__ WonSlot &w, int32_t task_id) { + (void)fdwic_trace_atomic_exchange( + task_id, FdwicAtomicSite::WonStateClearExchange, w.state.v, kWonStateFree, /*result_used=*/false + ); +} -PTO_DEVICE_FUNC int64_t load_frontier_for_advance() { return atomic_load(g_dist.frontier); } +PTO_DEVICE_FUNC int64_t load_frontier_for_advance() { + // The first load reads global scan state, not a particular task cell. + return fdwic_trace_atomic_load(-1, FdwicAtomicSite::FrontierInitialLoad, g_dist.frontier); +} PTO_DEVICE_FUNC bool try_advance_frontier_to(int64_t &frontier, int64_t next) { - const int64_t old = atomic_fetch_max(g_dist.frontier, next); + // Pair this update with the immediately preceding next-task flag load. + const int64_t old = fdwic_trace_atomic_fetch_max( + static_cast(next), FdwicAtomicSite::FrontierMax, g_dist.frontier, next, /*result_used=*/true + ); frontier = old > next ? old : next; return next > old; } @@ -87,7 +117,7 @@ PTO_DEVICE_FUNC void advance_frontier() { while (true) { const int64_t next = f + 1; if (next >= kFlagCap) break; - if (!task_flag_ready(static_cast(next), __ATOMIC_ACQUIRE)) break; + if (!task_flag_ready(static_cast(next), __ATOMIC_ACQUIRE, FdwicAtomicSite::FrontierFlagLoad)) break; try_advance_frontier_to(f, next); } } @@ -129,17 +159,25 @@ PTO_DEVICE_FUNC void execute_slot([[maybe_unused]] __gm__ DistCore *self, __gm__ ); } #else +#if PTO_FDWIC_PERF_CLOCK_KERNEL + const uint64_t perf_clock_kernel_begin = fdwic_perf_clock_kernel_begin(); +#endif TRACE_SPAN_BEGIN(kernel_trace); + const uint32_t submit_pmu_kernel_token = fdwic_submit_pmu_linked_kernel_pause(); dist_aicore_call_slot_kernel(s); + fdwic_submit_pmu_linked_kernel_resume(submit_pmu_kernel_token); TRACE_SPAN_END( kernel_trace, self, s.task_id, s.func_id, TracePhase::Kernel, static_cast(s.is_multicore ? 1 : 0), 0 ); +#if PTO_FDWIC_PERF_CLOCK_KERNEL + fdwic_perf_clock_kernel_end(perf_clock_kernel_begin); +#endif #endif store_barrier(); if (s.is_multicore) { __gm__ WonSlot &w = g_dist.blocks[s.won_block].slots[s.won_slot]; - if (decrement_won_remaining_is_last(w)) { - clear_won_slot_state(w); + if (decrement_won_remaining_is_last(w, s.task_id)) { + clear_won_slot_state(w, s.task_id); complete_executed_task(self, s.task_id); } } else { @@ -158,7 +196,7 @@ PTO_DEVICE_FUNC int32_t drain_phase_b(__gm__ DistCore *self) { if (!s.occupied || !s.built) continue; bool ready = true; for (int32_t f = 0; f < s.fanin_count; f++) { - if (!task_flag_ready(s.fanin[f], __ATOMIC_ACQUIRE)) { + if (!task_flag_ready(s.fanin[f], __ATOMIC_ACQUIRE, FdwicAtomicSite::FaninFlagLoad)) { ready = false; break; } @@ -240,11 +278,11 @@ PTO_DEVICE_FUNC bool drain_block_won(__gm__ DistCore *self) { if (!g_fdwic_joint_submit_seen) return false; if (self == nullptr || self->lane == LANE_AIC || self->lane == LANE_NONE) return false; __gm__ BlockWon &bw = g_dist.blocks[self->block_id]; - if (atomic_load(bw.any_pub) == 0) return false; + if (fdwic_trace_atomic_load(-1, FdwicAtomicSite::WonAnyLoad, bw.any_pub) == 0) return false; bool drained = false; for (int32_t i = 0; i < kPrivateSlots; i++) { __gm__ WonSlot &w = bw.slots[i]; - if (atomic_load(w.state.v) != kWonStatePublished) continue; + if (fdwic_trace_atomic_load(-1, FdwicAtomicSite::WonStateLoad, w.state.v) != kWonStatePublished) continue; #if defined(__CCE_AICORE__) dist_aicore_invalidate_region(&w.lane[self->lane].present, sizeof(w.lane[self->lane].present)); #endif @@ -252,7 +290,10 @@ PTO_DEVICE_FUNC bool drain_block_won(__gm__ DistCore *self) { if (!claim_won_lane(w, self->lane)) continue; int32_t si = alloc_ring_slot(self); if (si < 0) { - atomic_exchange(w.drained[self->lane].v, kDrainedFree); + (void)fdwic_trace_atomic_exchange( + -1, FdwicAtomicSite::WonLaneReleaseExchange, w.drained[self->lane].v, kDrainedFree, + /*result_used=*/false + ); return drained; } #if defined(__CCE_AICORE__) @@ -280,28 +321,20 @@ PTO_DEVICE_FUNC bool has_pending_won(__gm__ DistCore *self) { if (!g_fdwic_joint_submit_seen) return false; if (self == nullptr || self->lane == LANE_AIC || self->lane == LANE_NONE) return false; __gm__ BlockWon &bw = g_dist.blocks[self->block_id]; - if (atomic_load(bw.any_pub) == 0) return false; + if (fdwic_trace_atomic_load(-1, FdwicAtomicSite::WonAnyLoad, bw.any_pub) == 0) return false; for (int32_t i = 0; i < kPrivateSlots; i++) { __gm__ WonSlot &w = bw.slots[i]; - if (atomic_load(w.state.v) != kWonStatePublished) continue; + if (fdwic_trace_atomic_load(-1, FdwicAtomicSite::WonStateLoad, w.state.v) != kWonStatePublished) continue; #if defined(__CCE_AICORE__) dist_aicore_invalidate_region(&w.lane[self->lane].present, sizeof(w.lane[self->lane].present)); #endif if (!w.lane[self->lane].present) continue; - if (atomic_load(w.drained[self->lane].v) == kDrainedFree) return true; + if (fdwic_trace_atomic_load(-1, FdwicAtomicSite::WonDrainedLoad, w.drained[self->lane].v) == kDrainedFree) + return true; } return false; } -PTO_DEVICE_FUNC void dist_submit_execute_first(__gm__ DistCore *self) { - TRACE_LAP_RESET(self); - if (!fatal_set()) { - drain_block_won(self); - drain_phase_b(self); - } - TRACE_LAP(self, self->local_index, -1, TracePhase::EfDrain); -} - enum class DistSubmitKind : int32_t { Kernel = 0, Alloc = 1, @@ -325,9 +358,10 @@ struct DistSubmitCtx { int32_t joint_block; int32_t joint_slot; int32_t joint_count; + bool claim_attempted; }; -PTO_DEVICE_FUNC void dist_submit_begin(__gm__ DistCore *self, const L0TaskArgs &args, DistSubmitCtx &ctx) { +PTO_DEVICE_FUNC void dist_submit_begin(__gm__ DistCore *self, DistSubmitCtx &ctx) { ctx.self = self != nullptr ? self : g_self; if (ctx.self == nullptr) { ctx.task_id = kFlagCap; @@ -337,8 +371,10 @@ PTO_DEVICE_FUNC void dist_submit_begin(__gm__ DistCore *self, const L0TaskArgs & ctx.payload = &ctx.self->task_payloads[ctx.task_id & kTaskPayloadMask]; } ctx.result.set_task_id(PTO2TaskId::make(0, static_cast(ctx.task_id))); - ctx.tensor_count = args.tensor_count(); - ctx.scalar_count = args.scalar_count(); + // The compete-first Begin deliberately has no L0TaskArgs. Finish fills + // these two counts after the synchronous caller-side argument callback. + ctx.tensor_count = 0; + ctx.scalar_count = 0; ctx.register_mask = 0; ctx.output_bytes = 0; ctx.fanin_count = 0; @@ -349,11 +385,21 @@ PTO_DEVICE_FUNC void dist_submit_begin(__gm__ DistCore *self, const L0TaskArgs & ctx.joint_block = -1; ctx.joint_slot = -1; ctx.joint_count = 0; + ctx.claim_attempted = false; +} + +PTO_DEVICE_FUNC void dist_submit_begin(__gm__ DistCore *self, const L0TaskArgs &args, DistSubmitCtx &ctx) { + dist_submit_begin(self, ctx); + ctx.tensor_count = args.tensor_count(); + ctx.scalar_count = args.scalar_count(); } PTO_DEVICE_FUNC bool dist_submit_check_task_cap(const DistSubmitCtx &ctx, DistSubmitKind kind) { if (ctx.task_id < kFlagCap) return true; - set_fatal(); + // Begin uses post-increment. Clamp the cold failure path to the sentinel so + // invalid orchestration cannot wrap local_index by issuing more Submits. + if (ctx.self != nullptr) ctx.self->local_index = kFlagCap; + fdwic_trace_set_fatal(ctx.task_id); if (kind == DistSubmitKind::Alloc) { DIST_ERRF("[dist_engine] alloc task id %d exceeds kFlagCap %d\n", ctx.task_id, kFlagCap); } else { @@ -365,6 +411,15 @@ PTO_DEVICE_FUNC bool dist_submit_check_task_cap(const DistSubmitCtx &ctx, DistSu return false; } +PTO_DEVICE_FUNC bool dist_submit_tensor_uses_manual_dependency(const L0TaskArgs &args, int32_t index) { +#if defined(__CCE_AICORE__) + return args.tensor(index).tensor_from_gm() ? args.tensor(index).gm_ref().manual_dep : + args.tensor(index).ref().manual_dep; +#else + return args.tensor(index).ref().manual_dep; +#endif +} + PTO_DEVICE_FUNC uint32_t calculate_output_layout(const L0TaskArgs &args, DistOutputLayout &layout, uint32_t ®ister_mask) { layout.total_output_size = 0; @@ -372,7 +427,10 @@ calculate_output_layout(const L0TaskArgs &args, DistOutputLayout &layout, uint32 uint32_t output_mask = 0; for (int32_t i = 0; i < args.tensor_count(); i++) { const TensorArgType tag = args.tag(i); - if (tag == TensorArgType::INOUT || tag == TensorArgType::OUTPUT_EXISTING) register_mask |= 1u << i; + if ((tag == TensorArgType::INOUT || tag == TensorArgType::OUTPUT_EXISTING) && + !dist_submit_tensor_uses_manual_dependency(args, i)) { + register_mask |= 1u << i; + } if (tag != TensorArgType::OUTPUT) continue; output_mask |= 1u << i; layout.buffer_sizes[i] = TensorCreateInfo::buffer_size_bytes(args.tensor(i).create_info()); @@ -393,7 +451,7 @@ PTO_DEVICE_FUNC bool dist_submit_materialize_args(const L0TaskArgs &args, DistSu uint64_t task_base = PTO2_ALIGN_UP(ctx.self->heap_next, PTO2_PACKED_OUTPUT_ALIGN); if (total > 0 && g_dist.heap_base != nullptr) { if (total > ring) { - set_fatal(); + fdwic_trace_set_fatal(ctx.task_id); if (kind == DistSubmitKind::Alloc) { DIST_ERRF( "[dist_engine] alloc task %d outputs %llu B exceed heap ring %zu B\n", ctx.task_id, @@ -417,7 +475,7 @@ PTO_DEVICE_FUNC bool dist_submit_materialize_args(const L0TaskArgs &args, DistSu if ((output_mask & 1u) == 0) continue; const auto &ci = args.tensor(i).create_info(); if (g_dist.heap_base == nullptr) { - set_fatal(); + fdwic_trace_set_fatal(ctx.task_id); if (kind == DistSubmitKind::Alloc) { DIST_ERRF("[dist_engine] GM output heap not allocated at alloc %d\n", ctx.task_id); } else { @@ -497,7 +555,7 @@ PTO_DEVICE_FUNC void build_ring_slot_from_submit( PTO_DEVICE_FUNC void dist_submit_prepare_map(__gm__ DistCore *self, int32_t task_id) { if (self == nullptr) return; - dist_tensor_map_advance_retire(self->map, task_id, g_dist.H); + dist_tensor_map_prepare_task(*self, task_id, g_dist.H); } PTO_DEVICE_FUNC void dist_submit_add_fanin(int32_t fanin[], int32_t &fanin_count, int32_t producer) { @@ -507,7 +565,12 @@ PTO_DEVICE_FUNC void dist_submit_add_fanin(int32_t fanin[], int32_t &fanin_count if (fanin_count < kMaxFanin) fanin[fanin_count++] = producer; } +#if PTO_FDWIC_SHARED_MAP +PTO_DEVICE_FUNC bool +dist_submit_collect_fanin(const L0TaskArgs &args, const DistSubmitCtx &ctx, int32_t fanin[], int32_t &fanin_count) { +#else PTO_DEVICE_FUNC int32_t dist_submit_collect_fanin(const L0TaskArgs &args, const DistSubmitCtx &ctx, int32_t fanin[]) { +#endif int32_t fc = 0; for (int32_t i = 0; i < ctx.tensor_count; i++) { const TensorArgType tag = args.tag(i); @@ -519,7 +582,15 @@ PTO_DEVICE_FUNC int32_t dist_submit_collect_fanin(const L0TaskArgs &args, const dist_submit_add_fanin(fanin, fc, static_cast(owner_raw & 0xFFFFFFFFu)); } if (tag != TensorArgType::INPUT && tag != TensorArgType::INOUT) continue; - const int32_t p = dist_tensor_map_lookup(ctx.self->map, args.tensor(i).gm_ref()); + if (args.tensor(i).gm_ref().manual_dep) continue; +#if PTO_FDWIC_SHARED_MAP + int32_t p = -1; + if (!dist_tensor_map_lookup_for_submit_winner(*ctx.self, args.tensor(i).gm_ref(), ctx.task_id, p)) { + return false; + } +#else + const int32_t p = dist_tensor_map_lookup_for_task(*ctx.self, args.tensor(i).gm_ref(), ctx.task_id); +#endif dist_submit_add_fanin(fanin, fc, p); } else { const Tensor &t = args.tensor(i).ref(); @@ -528,7 +599,13 @@ PTO_DEVICE_FUNC int32_t dist_submit_collect_fanin(const L0TaskArgs &args, const dist_submit_add_fanin(fanin, fc, static_cast(owner_raw & 0xFFFFFFFFu)); } if (tag != TensorArgType::INPUT && tag != TensorArgType::INOUT) continue; - const int32_t p = dist_tensor_map_lookup(ctx.self->map, t); + if (t.manual_dep) continue; +#if PTO_FDWIC_SHARED_MAP + int32_t p = -1; + if (!dist_tensor_map_lookup_for_submit_winner(*ctx.self, t, ctx.task_id, p)) return false; +#else + const int32_t p = dist_tensor_map_lookup_for_task(*ctx.self, t, ctx.task_id); +#endif dist_submit_add_fanin(fanin, fc, p); } #else @@ -537,47 +614,151 @@ PTO_DEVICE_FUNC int32_t dist_submit_collect_fanin(const L0TaskArgs &args, const if (owner_raw != UINT64_MAX) dist_submit_add_fanin(fanin, fc, static_cast(owner_raw & 0xFFFFFFFFu)); if (tag != TensorArgType::INPUT && tag != TensorArgType::INOUT) continue; if (t.manual_dep) continue; - const int32_t p = dist_tensor_map_lookup(ctx.self->map, t); +#if PTO_FDWIC_SHARED_MAP + int32_t p = -1; + if (!dist_tensor_map_lookup_for_submit_winner(*ctx.self, t, ctx.task_id, p)) return false; +#else + const int32_t p = dist_tensor_map_lookup_for_task(*ctx.self, t, ctx.task_id); +#endif dist_submit_add_fanin(fanin, fc, p); #endif } +#if PTO_FDWIC_SHARED_MAP + fanin_count = fc; + return true; +#else return fc; +#endif } -PTO_DEVICE_FUNC void dist_submit_insert_existing_tensor(DistSubmitCtx &ctx, const L0TaskArgs &args, int32_t i) { +#if !PTO_FDWIC_SHARED_MAP +PTO_DEVICE_FUNC bool dist_submit_insert_existing_tensor(DistSubmitCtx &ctx, const L0TaskArgs &args, int32_t i) { #if defined(__CCE_AICORE__) if (args.tensor(i).tensor_from_gm()) { - dist_tensor_map_insert(ctx.self->map, args.tensor(i).gm_ref(), ctx.task_id); + return dist_tensor_map_insert_for_task(*ctx.self, args.tensor(i).gm_ref(), ctx.task_id, ctx.won); } else { - dist_tensor_map_insert(ctx.self->map, args.tensor(i).ref(), ctx.task_id); + return dist_tensor_map_insert_for_task(*ctx.self, args.tensor(i).ref(), ctx.task_id, ctx.won); } #else - dist_tensor_map_insert(ctx.self->map, args.tensor(i).ref(), ctx.task_id); + return dist_tensor_map_insert_for_task(*ctx.self, args.tensor(i).ref(), ctx.task_id, ctx.won); #endif } -PTO_DEVICE_FUNC void dist_submit_register_outputs(DistSubmitCtx &ctx, const L0TaskArgs &args, bool include_existing) { - if (!include_existing) return; +PTO_DEVICE_FUNC void dist_submit_latch_tensor_map_failure(DistSubmitCtx &ctx) { + // Reuse the existing task-cap gate as this worker's failure latch without + // adding state or GM reads to normal Submit. Register has already aborted + // the current task; later Begin calls start at kFlagCap and cannot Claim or + // Build. + if (ctx.self != nullptr) ctx.self->local_index = kFlagCap; + set_fatal_code(PTO2_ERROR_TENSORMAP_CAPACITY); +} +#endif + +#if PTO_FDWIC_SHARED_MAP +PTO_DEVICE_FUNC int32_t dist_submit_shared_tensor_map_error_code(DistSharedTensorMapTaskPublishResult result) { + switch (result) { + case DistSharedTensorMapTaskPublishResult::Committed: + return PTO2_ERROR_NONE; + case DistSharedTensorMapTaskPublishResult::CapacityBlocked: + return PTO2_ERROR_TENSORMAP_CAPACITY; + case DistSharedTensorMapTaskPublishResult::PartialPublish: + return PTO2_ERROR_TENSORMAP_PARTIAL_PUBLISH; + case DistSharedTensorMapTaskPublishResult::ProtocolError: + default: + return PTO2_ERROR_TENSORMAP_PROTOCOL; + } +} + +PTO_DEVICE_FUNC bool +dist_submit_handle_shared_tensor_map_result(DistSubmitCtx &ctx, DistSharedTensorMapTaskPublishResult result) { + if (result == DistSharedTensorMapTaskPublishResult::Committed) { + return true; + } + if (ctx.self != nullptr) { + ctx.self->local_index = kFlagCap; + } + set_fatal_code(dist_submit_shared_tensor_map_error_code(result)); + return false; +} +#endif + +PTO_DEVICE_FUNC bool dist_submit_register_outputs(DistSubmitCtx &ctx, const L0TaskArgs &args, bool include_existing) { +#if PTO_FDWIC_SHARED_MAP + if (!include_existing || !ctx.won) return true; + SharedTensorMapValue entries[MAX_TENSOR_ARGS]; + uint32_t count = 0; + uint32_t register_mask = ctx.register_mask; + for (int32_t i = 0; register_mask != 0; i++, register_mask >>= 1) { + if ((register_mask & 1u) == 0) continue; +#if defined(__CCE_AICORE__) + entries[count++] = args.tensor(i).tensor_from_gm() ? + dist_shared_tensor_map_make_value(args.tensor(i).gm_ref(), ctx.task_id) : + dist_shared_tensor_map_make_value(args.tensor(i).ref(), ctx.task_id); +#else + entries[count++] = dist_shared_tensor_map_make_value(args.tensor(i).ref(), ctx.task_id); +#endif + } + return dist_submit_handle_shared_tensor_map_result( + ctx, dist_tensor_map_publish_shared_task(entries, count, ctx.task_id) + ); +#else + if (!include_existing) return true; uint32_t register_mask = ctx.register_mask; for (int32_t i = 0; register_mask != 0; i++, register_mask >>= 1) { - if ((register_mask & 1u) != 0) dist_submit_insert_existing_tensor(ctx, args, i); + if ((register_mask & 1u) == 0) continue; + if (!dist_submit_insert_existing_tensor(ctx, args, i)) { + dist_submit_latch_tensor_map_failure(ctx); + return false; + } } + return true; +#endif } +#if PTO_FDWIC_SHARED_MAP +PTO_DEVICE_FUNC bool dist_submit_commit_empty_shared_tensor_map_task(DistSubmitCtx &ctx) { + if (!ctx.won) return true; + return dist_submit_handle_shared_tensor_map_result( + ctx, dist_tensor_map_publish_shared_task(nullptr, 0, ctx.task_id) + ); +} +#endif + PTO_DEVICE_FUNC bool dist_submit_materialize_and_prepare_map( - __gm__ DistCore *self, const L0TaskArgs &args, DistSubmitCtx &ctx, DistSubmitKind kind + __gm__ DistCore *self, const L0TaskArgs &args, DistSubmitCtx &ctx, DistSubmitKind kind, uint64_t materialize_begin, + uint64_t &prepare_map_end ) { - TRACE_LAP_RESET(self); if (!dist_submit_check_task_cap(ctx, kind)) return false; - TRACE_SPAN_BEGIN(materialize_trace); if (!dist_submit_materialize_args(args, ctx, kind)) return false; - TRACE_SPAN_END(materialize_trace, self, ctx.task_id, -1, TracePhase::Materialize, 0, static_cast(kind)); + // Match the business boundary immediately before the swimlane + // materialize_end timestamp. Failure paths do not forge an end marker and + // are rejected by phase-shape and balance validation. + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(materialize_end); + TRACE_SPAN_RECORD( + materialize_begin, materialize_end, self, ctx.task_id, -1, TracePhase::Materialize, 0, + static_cast(kind) + ); #if !defined(__CCE_AICORE__) - if (fatal_set()) return false; + if (fdwic_trace_is_fatal(ctx.task_id)) return false; #endif - TRACE_SPAN_BEGIN(prepare_map_trace); + // Share the business-call boundary with swimlane PrepareMap. The PMU + // variant accumulates only dist_submit_prepare_map(), excluding trace + // publication from the preceding phase. + fdwic_submit_pmu_phase_begin(); dist_submit_prepare_map(self, ctx.task_id); - TRACE_SPAN_END(prepare_map_trace, self, ctx.task_id, -1, TracePhase::PrepareMap, 0, static_cast(kind)); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(prepare_map_finish); + // A compete-first Kernel winner starts swimlane Fanin at the PrepareMap end + // boundary. The legacy path has not claimed yet, so ctx.won remains false. + if (kind == DistSubmitKind::Kernel && ctx.won) { + fdwic_submit_pmu_phase_begin(); + } + TRACE_SPAN_RECORD( + materialize_end, prepare_map_finish, self, ctx.task_id, -1, TracePhase::PrepareMap, 0, + static_cast(kind) + ); + prepare_map_end = prepare_map_finish; return true; } diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/submit_helpers.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/submit_helpers.h index b454b377fd..6f7f24b9b6 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/submit_helpers.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/submit_helpers.h @@ -17,14 +17,18 @@ namespace { PTO_DEVICE_FUNC bool claim(__gm__ volatile int64_t &cursor, int32_t N) { if (N < 0 || N >= kFlagCap) return false; - const int64_t old = atomic_fetch_max(cursor, static_cast(N)); + const int64_t old = fdwic_trace_atomic_fetch_max( + N, FdwicAtomicSite::ClaimMax, cursor, static_cast(N), /*result_used=*/true + ); return N > old; } -PTO_DEVICE_FUNC uint64_t load_task_vend(int32_t task_id) { - if (task_id < 0) return 0; - __gm__ DistTaskCell &cell = task_cell(task_id); - return atomic_load(cell.vend, __ATOMIC_RELAXED); +PTO_DEVICE_FUNC uint64_t load_task_vend(int32_t current_task_id, int32_t vend_task_id) { + if (vend_task_id < 0) return 0; + __gm__ DistTaskCell &cell = task_cell(vend_task_id); + return fdwic_trace_atomic_load( + current_task_id, FdwicAtomicSite::HeapVendLoad, cell.vend, /*result_used=*/true, __ATOMIC_RELAXED + ); } // Resolve a kernel id to its executable address (CoreCallable::resolved_addr()). @@ -47,38 +51,45 @@ PTO_DEVICE_FUNC void populate_won_slot_from_submit( Runtime *runtime, const L0TaskArgs &args, const DistSubmitCtx &ctx, FaninArrPtr fanin, int32_t fc ) { w.meta.task_id = task_id; -#define POPULATE_WON_LANE_FROM_SUBMIT(L) \ - do { \ - if ((L) == own_lane || !lane_active(M, (L))) break; \ - __gm__ BuiltSubtask &b = w.lane[(L)]; \ - b.present = true; \ - b.func_id = kernel_id_for_lane(mixed, (L)); \ - b.function_bin_addr = runtime != nullptr ? resolve_kernel_addr(runtime, b.func_id) : 0; \ - b.tensor_count = ctx.tensor_count; \ - b.scalar_count = ctx.scalar_count; \ - for (int32_t i = 0; i < ctx.tensor_count; i++) \ - dist_submit_copy_arg_tensor(b.tensors[i], args, ctx, i); \ - for (int32_t j = 0; j < ctx.scalar_count; j++) \ - b.scalars[j] = args.scalar(j); \ - b.fanin_count = fc; \ - for (int32_t k = 0; k < fc; k++) \ - b.fanin[k] = fanin[k]; \ - b.sub_block_id = ((L) == LANE_AIV1) ? 1 : 0; \ - atomic_exchange(w.drained[(L)].v, kDrainedFree); \ +#define POPULATE_WON_LANE_FROM_SUBMIT(L) \ + do { \ + if ((L) == own_lane || !lane_active(M, (L))) break; \ + __gm__ BuiltSubtask &b = w.lane[(L)]; \ + b.present = true; \ + b.func_id = kernel_id_for_lane(mixed, (L)); \ + b.function_bin_addr = runtime != nullptr ? resolve_kernel_addr(runtime, b.func_id) : 0; \ + b.tensor_count = ctx.tensor_count; \ + b.scalar_count = ctx.scalar_count; \ + for (int32_t i = 0; i < ctx.tensor_count; i++) \ + dist_submit_copy_arg_tensor(b.tensors[i], args, ctx, i); \ + for (int32_t j = 0; j < ctx.scalar_count; j++) \ + b.scalars[j] = args.scalar(j); \ + b.fanin_count = fc; \ + for (int32_t k = 0; k < fc; k++) \ + b.fanin[k] = fanin[k]; \ + b.sub_block_id = ((L) == LANE_AIV1) ? 1 : 0; \ + (void)fdwic_trace_atomic_exchange( \ + task_id, FdwicAtomicSite::WonLaneDepositExchange, w.drained[(L)].v, kDrainedFree, /*result_used=*/false \ + ); \ } while (0) - reset_won_lane(w, LANE_AIC); - reset_won_lane(w, LANE_AIV0); - reset_won_lane(w, LANE_AIV1); + reset_won_lane(w, LANE_AIC, task_id); + reset_won_lane(w, LANE_AIV0, task_id); + reset_won_lane(w, LANE_AIV1, task_id); POPULATE_WON_LANE_FROM_SUBMIT(LANE_AIC); POPULATE_WON_LANE_FROM_SUBMIT(LANE_AIV0); POPULATE_WON_LANE_FROM_SUBMIT(LANE_AIV1); #undef POPULATE_WON_LANE_FROM_SUBMIT } -PTO_DEVICE_FUNC int32_t alloc_won_slot(int32_t block) { +PTO_DEVICE_FUNC int32_t alloc_won_slot(int32_t block, int32_t task_id) { __gm__ BlockWon &bw = g_dist.blocks[block]; for (int32_t i = 0; i < kPrivateSlots; i++) { - if (atomic_fetch_max(bw.slots[i].state.v, kWonStateClaimed) == kWonStateFree) return i; + if (fdwic_trace_atomic_fetch_max( + task_id, FdwicAtomicSite::WonSlotClaimMax, bw.slots[i].state.v, kWonStateClaimed, + /*result_used=*/true + ) == kWonStateFree) { + return i; + } } return -1; } diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/submit_runtime.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/submit_runtime.h index 97f75eb968..1c82980407 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/submit_runtime.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/submit_runtime.h @@ -42,6 +42,7 @@ PTO_DEVICE_FUNC bool dist_submit_claim_kernel(const MixedKernels &mixed, DistSub const int32_t anchor_lane = anchor_lane_for_mask(M); if (!dist_submit_self_is_lane(ctx.self, block, anchor_lane)) return false; __gm__ PaddedCursor *cursors = anchor_lane == LANE_AIC ? g_dist.cube_cursor : g_dist.vector_cursor; + ctx.claim_attempted = true; ctx.won = claim(cursors[ctx.task_id % kCursorShards].v, ctx.task_id); if (!ctx.won) return false; ctx.kernel_id = kernel_id_for_lane(mixed, anchor_lane); @@ -50,6 +51,7 @@ PTO_DEVICE_FUNC bool dist_submit_claim_kernel(const MixedKernels &mixed, DistSub } if (lane_active(M, LANE_AIC)) { if (ctx.self->role != CoreType::AIC) return false; + ctx.claim_attempted = true; ctx.won = claim(g_dist.cube_cursor[ctx.task_id % kCursorShards].v, ctx.task_id); if (!ctx.won) return false; ctx.kernel_id = mixed.aic_kernel_id; @@ -57,6 +59,7 @@ PTO_DEVICE_FUNC bool dist_submit_claim_kernel(const MixedKernels &mixed, DistSub } if (lane_active(M, LANE_AIV0) || lane_active(M, LANE_AIV1)) { if (ctx.self->role != CoreType::AIV) return false; + ctx.claim_attempted = true; ctx.won = claim(g_dist.vector_cursor[ctx.task_id % kCursorShards].v, ctx.task_id); if (!ctx.won) return false; const int32_t own_lane = lane_active(M, LANE_AIV0) ? LANE_AIV0 : LANE_AIV1; @@ -69,6 +72,7 @@ PTO_DEVICE_FUNC bool dist_submit_claim_kernel(const MixedKernels &mixed, DistSub PTO_DEVICE_FUNC bool dist_submit_claim_alloc(DistSubmitCtx &ctx) { ctx.kernel_id = INVALID_KERNEL_ID; if (ctx.self == nullptr || ctx.task_id < 0 || ctx.task_id >= kFlagCap) return false; + ctx.claim_attempted = true; ctx.won = claim(g_dist.alloc_cursor[ctx.task_id % kCursorShards].v, ctx.task_id); return ctx.won; } @@ -90,18 +94,51 @@ PTO_DEVICE_FUNC __gm__ RingSlot *dist_submit_alloc_slot(__gm__ DistCore *self) { return &slot; } +#if PTO_FDWIC_SHARED_MAP +PTO_DEVICE_FUNC bool dist_submit_wait_slot_capacity(__gm__ DistCore *self, int32_t task_id) { +#else PTO_DEVICE_FUNC void dist_submit_wait_slot_capacity(__gm__ DistCore *self, int32_t task_id) { - if (self == nullptr) return; +#endif + if (self == nullptr) { +#if PTO_FDWIC_SHARED_MAP + return false; +#else + return; +#endif + } bool waited = false; TRACE_SPAN_BEGIN(ring_bp_trace); + const uint32_t slot_poll_region = fdwic_atomic_poll_region_begin( + fdwic_atomic_site_mask(FdwicAtomicSite::FaninFlagLoad) | +#if PTO_FDWIC_SHARED_MAP + fdwic_atomic_site_mask(FdwicAtomicSite::FatalPoll) | +#endif + fdwic_atomic_block_won_poll_mask() + ); while (self->occupied_count >= kPrivateSlots - kWonReserve) { waited = true; +#if PTO_FDWIC_SHARED_MAP + // Another winner may terminate the shared run while this worker is + // blocked behind slots whose fan-in can no longer complete. Consume + // fatal only on the existing backpressure loop, so the no-wait winner + // path gains no extra atomic load. + if (fdwic_trace_is_fatal(task_id)) { + fdwic_atomic_poll_region_end(slot_poll_region); + TRACE_SPAN_END(ring_bp_trace, self, task_id, -1, TracePhase::RingBp, 0, 0); + self->local_index = kFlagCap; + return false; + } +#endif drain_block_won(self); if (drain_phase_b(self) == 0) SPIN_WAIT_HINT(); } + fdwic_atomic_poll_region_end(slot_poll_region); if (waited) { TRACE_SPAN_END(ring_bp_trace, self, task_id, -1, TracePhase::RingBp, 0, 0); } +#if PTO_FDWIC_SHARED_MAP + return true; +#endif } PTO_DEVICE_FUNC bool dist_submit_wait_heap_capacity(DistSubmitCtx &ctx, DistSubmitKind kind) { @@ -109,18 +146,38 @@ PTO_DEVICE_FUNC bool dist_submit_wait_heap_capacity(DistSubmitCtx &ctx, DistSubm const size_t ring = g_dist.heap_size; bool waited = false; TRACE_SPAN_BEGIN(heap_bp_trace); - while (!fatal_set()) { - const int32_t f = static_cast(atomic_load(g_dist.frontier)); + bool heap_poll_region_active = false; + uint32_t heap_poll_region = 0; + while (!fdwic_trace_is_fatal(ctx.task_id)) { + // Before the logical heap completes its first lap, no physical address + // has been reused. Keep the fatal check but skip frontier/vend loads. + if (ctx.self->heap_next <= ring) { + if (heap_poll_region_active) fdwic_atomic_poll_region_end(heap_poll_region); + return true; + } + if (!heap_poll_region_active) { + heap_poll_region = fdwic_atomic_poll_region_begin( + fdwic_atomic_site_mask(FdwicAtomicSite::FatalPoll) | + fdwic_atomic_site_mask(FdwicAtomicSite::HeapFrontierLoad) | + fdwic_atomic_site_mask(FdwicAtomicSite::HeapVendLoad) | + fdwic_atomic_site_mask(FdwicAtomicSite::FaninFlagLoad) | fdwic_atomic_block_won_poll_mask() + ); + heap_poll_region_active = true; + } + const int32_t f = static_cast(fdwic_trace_atomic_load( + ctx.task_id, FdwicAtomicSite::HeapFrontierLoad, g_dist.frontier, /*result_used=*/true + )); const int32_t R = f - g_dist.H; - const uint64_t vstart_live = load_task_vend(R); + const uint64_t vstart_live = load_task_vend(ctx.task_id, R); if (ctx.self->heap_next - vstart_live <= ring) { + fdwic_atomic_poll_region_end(heap_poll_region); if (waited) { TRACE_SPAN_END(heap_bp_trace, ctx.self, ctx.task_id, -1, TracePhase::RingBp, 0, 1); } return true; } if (f >= ctx.task_id - 1) { - set_fatal(); + fdwic_trace_set_fatal(ctx.task_id); if (kind == DistSubmitKind::Alloc) { DIST_ERRF( "[dist_engine] heap ring %zu B too small for H=%d window at alloc %d (live=%llu B)\n", ring, @@ -133,18 +190,41 @@ PTO_DEVICE_FUNC bool dist_submit_wait_heap_capacity(DistSubmitCtx &ctx, DistSubm ring, g_dist.H, ctx.task_id, (unsigned long long)(ctx.self->heap_next - vstart_live) ); } + fdwic_atomic_poll_region_end(heap_poll_region); return false; } waited = true; drain_block_won(ctx.self); if (drain_phase_b(ctx.self) == 0) SPIN_WAIT_HINT(); } + if (heap_poll_region_active) fdwic_atomic_poll_region_end(heap_poll_region); if (waited) { TRACE_SPAN_END(heap_bp_trace, ctx.self, ctx.task_id, -1, TracePhase::RingBp, 0, 1); } return false; } +#if PTO_FDWIC_SHARED_MAP +PTO_DEVICE_FUNC bool dist_submit_wait_shared_tensor_map_turn(DistSubmitCtx &ctx) { + if (ctx.self == nullptr || !ctx.won) { + return dist_submit_handle_shared_tensor_map_result(ctx, DistSharedTensorMapTaskPublishResult::ProtocolError); + } + while (!fatal_set()) { + const int64_t next_task = dist_tensor_map_next_publish_task(); + if (next_task == ctx.task_id) return true; + if (next_task < 0 || next_task > ctx.task_id) { + return dist_submit_handle_shared_tensor_map_result( + ctx, DistSharedTensorMapTaskPublishResult::ProtocolError + ); + } + drain_block_won(ctx.self); + if (drain_phase_b(ctx.self) == 0) SPIN_WAIT_HINT(); + } + ctx.self->local_index = kFlagCap; + return false; +} +#endif + PTO_DEVICE_FUNC void publish_joint_deposits(DistSubmitCtx &ctx, const MixedKernels &mixed, const L0TaskArgs &args) { if (!ctx.joint) return; __gm__ WonSlot &w = g_dist.blocks[ctx.joint_block].slots[ctx.joint_slot]; @@ -162,18 +242,26 @@ PTO_DEVICE_FUNC void publish_joint_deposits(DistSubmitCtx &ctx, const MixedKerne dist_aicore_flush_region(&w.meta, sizeof(w.meta)); dist_aicore_flush_region(w.lane, sizeof(w.lane)); #endif - store_won_remaining(w, ctx.joint_count); - publish_won_slot(w); - atomic_exchange(g_dist.blocks[ctx.joint_block].any_pub, 1); + store_won_remaining(w, ctx.joint_count, ctx.task_id); + publish_won_slot(w, ctx.task_id); + (void)fdwic_trace_atomic_exchange( + ctx.task_id, FdwicAtomicSite::WonAnyPublishExchange, g_dist.blocks[ctx.joint_block].any_pub, int32_t{1}, + /*result_used=*/false + ); } -PTO_DEVICE_FUNC int32_t wait_alloc_won_slot(__gm__ DistCore *self, int32_t block) { - int32_t won_slot = alloc_won_slot(block); - while (won_slot < 0 && !fatal_set()) { +PTO_DEVICE_FUNC int32_t wait_alloc_won_slot(__gm__ DistCore *self, int32_t block, int32_t task_id) { + int32_t won_slot = alloc_won_slot(block, task_id); + const uint32_t won_slot_poll_region = fdwic_atomic_poll_region_begin( + fdwic_atomic_site_mask(FdwicAtomicSite::FatalPoll) | fdwic_atomic_site_mask(FdwicAtomicSite::FaninFlagLoad) | + fdwic_atomic_block_won_poll_mask() + ); + while (won_slot < 0 && !fdwic_trace_is_fatal(task_id)) { drain_block_won(self); if (drain_phase_b(self) == 0) SPIN_WAIT_HINT(); - won_slot = alloc_won_slot(block); + won_slot = alloc_won_slot(block, task_id); } + fdwic_atomic_poll_region_end(won_slot_poll_region); return won_slot; } @@ -191,10 +279,14 @@ PTO_DEVICE_FUNC bool dist_submit_build_winner_slot(DistSubmitCtx &ctx, const L0T PTO_DEVICE_FUNC void dist_submit_build_winner_task(DistSubmitCtx &ctx, const MixedKernels &mixed, const L0TaskArgs &args) { if (ctx.self == nullptr) return; +#if PTO_FDWIC_SHARED_MAP + if (!dist_submit_wait_slot_capacity(ctx.self, ctx.task_id)) return; +#else dist_submit_wait_slot_capacity(ctx.self, ctx.task_id); +#endif if (!dist_submit_wait_heap_capacity(ctx, DistSubmitKind::Kernel)) return; if (ctx.joint && ctx.joint_slot < 0) { - ctx.joint_slot = wait_alloc_won_slot(ctx.self, ctx.joint_block); + ctx.joint_slot = wait_alloc_won_slot(ctx.self, ctx.joint_block, ctx.task_id); if (ctx.joint_slot < 0) return; } __gm__ RingSlot *slot = dist_submit_alloc_slot(ctx.self); @@ -211,20 +303,246 @@ PTO_DEVICE_FUNC void dist_submit_complete_alloc(DistSubmitCtx &ctx) { } } +PTO_DEVICE_FUNC DistCompeteFirstTicket +dist_submit_make_ticket(const DistSubmitCtx &ctx, DistSubmitKind kind, uint64_t submit_begin, bool ready) { + DistCompeteFirstTicket ticket; + ticket.submit_begin = submit_begin; + ticket.task_id = ctx.task_id; + ticket.kernel_id = ctx.kernel_id; + ticket.joint_block = ctx.joint_block; + ticket.joint_count = ctx.joint_count; + ticket.won = static_cast(ctx.won); + ticket.joint = static_cast(ctx.joint); + ticket.joint_init = static_cast(ctx.joint_init); + ticket.claim_attempted = static_cast(ctx.claim_attempted); + ticket.ready = static_cast(ready); + ticket.kind = static_cast( + kind == DistSubmitKind::Alloc ? DistCompeteFirstKind::Alloc : DistCompeteFirstKind::Kernel + ); + ticket.reserved = 0; + return ticket; +} + +PTO_DEVICE_FUNC void +dist_submit_restore_from_ticket(const DistCompeteFirstTicket &ticket, const L0TaskArgs &args, DistSubmitCtx &ctx) { + ctx.self = g_self; + ctx.task_id = ticket.task_id; + ctx.payload = ctx.self != nullptr && ctx.task_id >= 0 && ctx.task_id < kFlagCap ? + &ctx.self->task_payloads[ctx.task_id & kTaskPayloadMask] : + nullptr; + ctx.result.set_task_id(PTO2TaskId::make(0, static_cast(ctx.task_id))); + ctx.tensor_count = args.tensor_count(); + ctx.scalar_count = args.scalar_count(); + ctx.register_mask = 0; + ctx.output_bytes = 0; + ctx.fanin_count = 0; + ctx.kernel_id = ticket.kernel_id; + ctx.won = ticket.won != 0; + ctx.joint = ticket.joint != 0; + ctx.joint_init = ticket.joint_init != 0; + ctx.joint_block = ticket.joint_block; + // Begin cannot reserve a WonSlot because that can block and requires the + // materialized task. The winner allocates it later in the existing tail. + ctx.joint_slot = -1; + ctx.joint_count = ticket.joint_count; + ctx.claim_attempted = ticket.claim_attempted != 0; +} + +PTO_DEVICE_FUNC bool dist_submit_validate_ticket( + const DistCompeteFirstTicket &ticket, DistSubmitKind expected_kind, const DistSubmitCtx &ctx +) { + const uint8_t expected = static_cast( + expected_kind == DistSubmitKind::Alloc ? DistCompeteFirstKind::Alloc : DistCompeteFirstKind::Kernel + ); + const bool sequence_ok = ctx.self != nullptr && ticket.task_id >= 0 && ticket.task_id < kFlagCap && + ctx.self->local_index == ticket.task_id + 1; + const bool fields_ok = ticket.ready != 0 && ticket.kind == expected && ticket.reserved == 0 && ticket.won <= 1 && + ticket.joint <= 1 && ticket.joint_init <= 1 && ticket.claim_attempted <= 1; + if (__builtin_expect(sequence_ok && fields_ok, 1)) return true; + // A callback must be synchronous and may not submit another task before + // its matching Finish. Treat a stale/malformed ticket as a protocol error + // instead of reconstructing a context for the wrong per-core payload. + fdwic_trace_set_fatal(ticket.task_id); + return false; +} + +PTO_DEVICE_FUNC TaskOutputTensors dist_submit_finish_kernel_tail( + DistSubmitCtx &ctx, const MixedKernels &mixed, const L0TaskArgs &args, uint64_t tail_begin, uint64_t submit_begin +) { + uint64_t register_begin = tail_begin; + if (ctx.won) { +#if PTO_FDWIC_SHARED_MAP + if (!dist_submit_wait_shared_tensor_map_turn(ctx)) return ctx.result; + const bool fanin_ok = dist_submit_collect_fanin(args, ctx, ctx.fanin, ctx.fanin_count); +#else + ctx.fanin_count = dist_submit_collect_fanin(args, ctx, ctx.fanin); +#endif + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(fanin_end); + TRACE_SPAN_RECORD( + tail_begin, fanin_end, ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::Fanin, 0, + static_cast(ctx.fanin_count) + ); + register_begin = fanin_end; +#if PTO_FDWIC_SHARED_MAP + if (__builtin_expect(!fanin_ok, 0)) { + (void)dist_submit_handle_shared_tensor_map_result(ctx, DistSharedTensorMapTaskPublishResult::ProtocolError); + return ctx.result; + } +#endif + } + // The Register PMU window covers only the real RegisterOutputs call, not + // the preceding Fanin/Claim record publication or caller transition. + fdwic_submit_pmu_phase_begin(); + const bool register_ok = dist_submit_register_outputs(ctx, args, /*include_existing=*/true); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(register_end); + if (__builtin_expect(!register_ok, 0)) { + // The current task has claimed but must not Build or publish. The + // worker task-cap sentinel blocks later tasks before Claim. Leave the + // outer Submit/perf/PMU window open so integrity validation rejects the + // failed raw trace; the Register span still identifies the failure. + TRACE_SPAN_RECORD( + register_begin, register_end, ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::Register, 0, 1 + ); + return ctx.result; + } + if (ctx.won) { + fdwic_submit_pmu_phase_begin(); + } else { + // Share Register.end with the swimlane and cover only the loser's real + // drain_block_won() call. + fdwic_submit_pmu_phase_begin(); + } + TRACE_SPAN_RECORD(register_begin, register_end, ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::Register, 0, 1); + if (__builtin_expect(ctx.won, 0)) { + dist_submit_build_winner_task(ctx, mixed, args); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(winner_build_end); + TRACE_SPAN_RECORD( + register_end, winner_build_end, ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::WinnerBuild, 0, 0 + ); + } else { + // Production losers perform real BlockWon progress. This is not the + // empty loser path used by the standalone single-lane probe. + drain_block_won(ctx.self); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(loser_replay_end); + TRACE_SPAN_RECORD( + register_end, loser_replay_end, ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::LoserReplay, 0, 0 + ); + } + TRACE_TIMESTAMP(submit_end); + fdwic_perf_clock_submit_end(ctx.task_id); + fdwic_submit_pmu_submit_end(ctx.task_id); + TRACE_SPAN_RECORD( + submit_begin, submit_end, ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::Submit, + static_cast(ctx.won), 0 + ); + return ctx.result; +} + +PTO_DEVICE_FUNC TaskOutputTensors +dist_submit_finish_alloc_tail(DistSubmitCtx &ctx, uint64_t completion_begin, uint64_t submit_begin) { + if (__builtin_expect(ctx.won, 0)) { +#if PTO_FDWIC_SHARED_MAP + if (!dist_submit_wait_shared_tensor_map_turn(ctx)) return ctx.result; + if (!dist_submit_commit_empty_shared_tensor_map_task(ctx)) return ctx.result; +#endif + dist_submit_complete_alloc(ctx); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(alloc_complete_end); + TRACE_SPAN_RECORD( + completion_begin, alloc_complete_end, ctx.self, ctx.task_id, -1, TracePhase::AllocComplete, 0, 0 + ); + } + // Alloc losers have no corresponding replay action. Their final suffix is + // intentionally left as an offline Submit residual, not a fake phase. + TRACE_TIMESTAMP(submit_end); + fdwic_perf_clock_submit_end(ctx.task_id); + fdwic_submit_pmu_submit_end(ctx.task_id); + TRACE_SPAN_RECORD( + submit_begin, submit_end, ctx.self, ctx.task_id, -1, TracePhase::Submit, static_cast(ctx.won), 1 + ); + return ctx.result; +} + #include "dist_engine/aicore/run_state.h" +PTO_DEVICE_FUNC inline void dist_final_barrier_publish(__gm__ volatile int64_t &value) { + (void)fdwic_trace_atomic_fetch_add( + -1, FdwicAtomicSite::ReplayDoneIncrement, value, 1, /*result_used=*/false + ); +} + +PTO_DEVICE_FUNC inline bool +dist_final_barrier_progress(__gm__ DistCore *self, bool &leaf_forwarded, bool &root_released, bool &leaf_released) { + const int32_t group = self->block_id % kFinalBarrierGroups; + __gm__ FinalBarrierArrival &leaf_arrival = g_dist.final_barrier.leaf_arrivals[group]; + const bool leaf_leader = self->lane == LANE_AIC && self->block_id == group; + if (leaf_leader && !leaf_forwarded && + fdwic_trace_atomic_load(-1, FdwicAtomicSite::ReplayDonePoll, leaf_arrival.v) >= leaf_arrival.expected) { + dist_final_barrier_publish(g_dist.final_barrier.root_arrival.v); + leaf_forwarded = true; + } + + const bool root_leader = leaf_leader && group == 0; + if (root_leader && !root_released && + fdwic_trace_atomic_load(-1, FdwicAtomicSite::ReplayDonePoll, g_dist.final_barrier.root_arrival.v) >= + g_dist.final_barrier.root_arrival.expected) { + dist_final_barrier_publish(g_dist.final_barrier.root_release.v); + root_released = true; + } + + if (leaf_leader && leaf_forwarded && !leaf_released && + fdwic_trace_atomic_load(-1, FdwicAtomicSite::ReplayDonePoll, g_dist.final_barrier.root_release.v) >= 1) { + dist_final_barrier_publish(g_dist.final_barrier.leaf_releases[group].v); + leaf_released = true; + } + return fdwic_trace_atomic_load(-1, FdwicAtomicSite::ReplayDonePoll, g_dist.final_barrier.leaf_releases[group].v) >= + 1; +} + PTO_DEVICE_FUNC void dist_submit_drain_to_completion(__gm__ DistCore *self) { if (self == nullptr) return; - atomic_fetch_add(g_dist.replay_done, 1); + const int32_t final_group = self->block_id % kFinalBarrierGroups; + dist_final_barrier_publish(g_dist.final_barrier.leaf_arrivals[final_group].v); + const uint32_t final_poll_region = fdwic_atomic_poll_region_begin( + fdwic_atomic_site_mask(FdwicAtomicSite::ReplayDonePoll) | + fdwic_atomic_site_mask(FdwicAtomicSite::FaninFlagLoad) | +#if PTO_FDWIC_SHARED_MAP + fdwic_atomic_site_mask(FdwicAtomicSite::FatalPoll) | +#endif + fdwic_atomic_block_won_poll_mask() + ); + bool leaf_forwarded = false; + bool root_released = false; + bool leaf_released = false; + bool global_release_observed = false; while (true) { drain_block_won(self); const int32_t freed = drain_phase_b(self); - const bool all_replayed = atomic_load(g_dist.replay_done) >= g_dist.num_workers; + if (!global_release_observed) { + global_release_observed = dist_final_barrier_progress(self, leaf_forwarded, root_released, leaf_released); + } const bool ring_empty = self->occupied_count == 0; const bool pending = has_pending_won(self); - if (all_replayed && ring_empty && !pending) break; - if (freed == 0) SPIN_WAIT_HINT(); + if (global_release_observed && ring_empty && !pending) break; + if (freed == 0) { +#if PTO_FDWIC_SHARED_MAP + // A worker that reaches FinalDrain before a remote fatal cannot + // wait for the missing worker's barrier arrival: core_main skips + // FinalDrain after observing fatal. Poll only on an idle drain + // iteration so successful progress does not pay an extra load. + if (fdwic_trace_is_fatal()) { + self->local_index = kFlagCap; + break; + } +#endif + SPIN_WAIT_HINT(); + } } + fdwic_atomic_poll_region_end(final_poll_region); } PTO_DEVICE_FUNC void dist_submit_replay_orch(__gm__ Runtime *runtime) { @@ -252,7 +570,7 @@ PTO_DEVICE_FUNC void dist_submit_replay_orch(__gm__ Runtime *runtime) { aicpu_orchestration_entry(local_args); #else (void)runtime; - if (g_dist.orch_args != nullptr && !fatal_set()) { + if (g_dist.orch_args != nullptr && !fdwic_trace_is_fatal()) { aicpu_orchestration_entry(*g_dist.orch_args); } #endif @@ -268,62 +586,180 @@ dist_submit_impl(PTO2Runtime *, const MixedKernels &mixed, const L0TaskArgs &arg if (__builtin_popcount(active.core_mask()) >= 2) g_fdwic_joint_submit_seen = true; DistSubmitCtx ctx; dist_submit_begin(nullptr, args, ctx); - TRACE_SPAN_BEGIN(submit_trace); - TRACE_LAP_RESET(ctx.self); + TRACE_TIMESTAMP(submit_begin); + fdwic_perf_clock_submit_begin(ctx.task_id); + fdwic_submit_pmu_submit_begin(ctx.task_id); + fdwic_submit_pmu_phase_begin(); drain_block_won(ctx.self); drain_phase_b(ctx.self); - TRACE_LAP(ctx.self, ctx.task_id, -1, TracePhase::EfDrain); - if (!dist_submit_materialize_and_prepare_map(ctx.self, args, ctx, DistSubmitKind::Kernel)) return ctx.result; - TRACE_SPAN_BEGIN(claim_trace); - const bool is_winner = dist_submit_claim(DistSubmitKind::Kernel, &mixed, ctx); - TRACE_SPAN_END( - claim_trace, ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::Claim, static_cast(is_winner), 0 - ); - if (is_winner) { - TRACE_SPAN_BEGIN(fanin_trace); - ctx.fanin_count = dist_submit_collect_fanin(args, ctx, ctx.fanin); - TRACE_SPAN_END( - fanin_trace, ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::Fanin, 0, - static_cast(ctx.fanin_count) - ); - } - TRACE_SPAN_BEGIN(register_trace); - dist_submit_register_outputs(ctx, args, /*include_existing=*/true); - TRACE_SPAN_END(register_trace, ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::Register, 0, 1); - if (is_winner) { - TRACE_LAP(ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::Build); - dist_submit_build_winner_task(ctx, mixed, args); - } else { - TRACE_LAP(ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::Replay); - drain_block_won(ctx.self); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(efdrain_end); + // The legacy API reuses efdrain_end as Materialize.start and opens the + // selected phase at the same source boundary. Swimlane records are + // compiled out of the submit-PMU ELF. + fdwic_submit_pmu_phase_begin(); + TRACE_SPAN_RECORD(submit_begin, efdrain_end, ctx.self, ctx.task_id, -1, TracePhase::EfDrain, 0, 0); + uint64_t prepare_map_end = efdrain_end; + if (!dist_submit_materialize_and_prepare_map( + ctx.self, args, ctx, DistSubmitKind::Kernel, efdrain_end, prepare_map_end + )) { + return ctx.result; } - TRACE_SPAN_END( - submit_trace, ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::Submit, static_cast(is_winner), 0 - ); - return ctx.result; + const uint64_t claim_begin = prepare_map_end; + fdwic_submit_pmu_phase_begin(); + const bool is_winner = dist_submit_claim(DistSubmitKind::Kernel, &mixed, ctx); + const uint32_t claim_flags = (is_winner ? kFdwicClaimWon : 0U) | (ctx.claim_attempted ? kFdwicClaimAttempted : 0U); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(claim_end); + // A legacy Kernel winner starts swimlane Fanin at Claim.end. + if (ctx.won) fdwic_submit_pmu_phase_begin(); + TRACE_SPAN_RECORD(claim_begin, claim_end, ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::Claim, claim_flags, 0); + return dist_submit_finish_kernel_tail(ctx, mixed, args, claim_end, submit_begin); } DIST_API_ATTR PTO_DEVICE_FUNC TaskOutputTensors dist_alloc_tensors(PTO2Runtime *, const L0TaskArgs &args) { DistSubmitCtx ctx; dist_submit_begin(nullptr, args, ctx); - TRACE_SPAN_BEGIN(submit_trace); - TRACE_LAP_RESET(ctx.self); + TRACE_TIMESTAMP(submit_begin); + fdwic_perf_clock_submit_begin(ctx.task_id); + fdwic_submit_pmu_submit_begin(ctx.task_id); + fdwic_submit_pmu_phase_begin(); drain_block_won(ctx.self); drain_phase_b(ctx.self); - TRACE_LAP(ctx.self, ctx.task_id, -1, TracePhase::EfDrain); - if (!dist_submit_materialize_and_prepare_map(ctx.self, args, ctx, DistSubmitKind::Alloc)) return ctx.result; - TRACE_SPAN_BEGIN(register_trace); - dist_submit_register_outputs(ctx, args, /*include_existing=*/false); - TRACE_SPAN_END(register_trace, ctx.self, ctx.task_id, -1, TracePhase::Register, 0, 0); - TRACE_SPAN_BEGIN(claim_trace); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(efdrain_end); + fdwic_submit_pmu_phase_begin(); + TRACE_SPAN_RECORD(submit_begin, efdrain_end, ctx.self, ctx.task_id, -1, TracePhase::EfDrain, 0, 0); + uint64_t prepare_map_end = efdrain_end; + if (!dist_submit_materialize_and_prepare_map( + ctx.self, args, ctx, DistSubmitKind::Alloc, efdrain_end, prepare_map_end + )) { + return ctx.result; + } + const uint64_t register_begin = prepare_map_end; + fdwic_submit_pmu_phase_begin(); + (void)dist_submit_register_outputs(ctx, args, /*include_existing=*/false); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(register_end); + TRACE_SPAN_RECORD(register_begin, register_end, ctx.self, ctx.task_id, -1, TracePhase::Register, 0, 0); + const uint64_t claim_begin = register_end; + fdwic_submit_pmu_phase_begin(); const bool is_winner = dist_submit_claim(DistSubmitKind::Alloc, nullptr, ctx); - TRACE_SPAN_END(claim_trace, ctx.self, ctx.task_id, -1, TracePhase::Claim, static_cast(is_winner), 1); - if (is_winner) { - dist_submit_complete_alloc(ctx); - TRACE_LAP(ctx.self, ctx.task_id, -1, TracePhase::Alloc); - } else { - TRACE_LAP(ctx.self, ctx.task_id, -1, TracePhase::Replay); + const uint32_t claim_flags = (is_winner ? kFdwicClaimWon : 0U) | (ctx.claim_attempted ? kFdwicClaimAttempted : 0U); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(claim_end); + // AllocComplete shares Claim.end with the swimlane and opens only for a + // real Alloc winner. + if (ctx.won) fdwic_submit_pmu_phase_begin(); + TRACE_SPAN_RECORD(claim_begin, claim_end, ctx.self, ctx.task_id, -1, TracePhase::Claim, claim_flags, 1); + return dist_submit_finish_alloc_tail(ctx, claim_end, submit_begin); +} + +DIST_API_ATTR PTO_DEVICE_FUNC DistCompeteFirstTicket +dist_submit_compete_first_begin(PTO2Runtime *, const MixedKernels &mixed) { + const ActiveMask active = mixed.to_active_mask(); + if (__builtin_popcount(active.core_mask()) >= 2) g_fdwic_joint_submit_seen = true; + + DistSubmitCtx ctx; + dist_submit_begin(nullptr, ctx); + TRACE_TIMESTAMP(submit_begin); + fdwic_perf_clock_submit_begin(ctx.task_id); + fdwic_submit_pmu_submit_begin(ctx.task_id); + fdwic_submit_pmu_phase_begin(); + drain_block_won(ctx.self); + drain_phase_b(ctx.self); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(efdrain_end); + TRACE_SPAN_RECORD(submit_begin, efdrain_end, ctx.self, ctx.task_id, -1, TracePhase::EfDrain, 0, 0); + + const uint64_t claim_begin = efdrain_end; + fdwic_submit_pmu_phase_begin(); + const bool ready = dist_submit_check_task_cap(ctx, DistSubmitKind::Kernel); + const bool is_winner = ready && dist_submit_claim(DistSubmitKind::Kernel, &mixed, ctx); + const uint32_t claim_flags = (is_winner ? kFdwicClaimWon : 0U) | (ctx.claim_attempted ? kFdwicClaimAttempted : 0U); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(claim_end); + // Use the same boundary as swimlane Claim.end. The local PMU spans the + // Begin return and synchronous eager argument callback up to Materialize + // in the matching Finish. + fdwic_submit_pmu_phase_begin(); + // The calibration ELF executes an adjacent begin/end pair at this same + // Claim.end call site. Other ELFs compile it to an empty wrapper. This + // window measures observer overhead only. + fdwic_submit_pmu_empty_bracket_calibrate(); + TRACE_SPAN_RECORD(claim_begin, claim_end, ctx.self, ctx.task_id, ctx.kernel_id, TracePhase::Claim, claim_flags, 0); + return dist_submit_make_ticket(ctx, DistSubmitKind::Kernel, submit_begin, ready); +} + +DIST_API_ATTR PTO_DEVICE_FUNC TaskOutputTensors dist_submit_compete_first_finish( + PTO2Runtime *, const MixedKernels &mixed, const DistCompeteFirstTicket &ticket, const L0TaskArgs &args +) { + DistSubmitCtx ctx; + dist_submit_restore_from_ticket(ticket, args, ctx); + if (!dist_submit_validate_ticket(ticket, DistSubmitKind::Kernel, ctx)) return ctx.result; + + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(materialize_begin); + // The compete-first selected phase shares the current swimlane + // Materialize.begin business boundary. The helper end excludes PrepareMap. + fdwic_submit_pmu_phase_begin(); + uint64_t prepare_map_end = materialize_begin; + if (!dist_submit_materialize_and_prepare_map( + ctx.self, args, ctx, DistSubmitKind::Kernel, materialize_begin, prepare_map_end + )) { + return ctx.result; } - TRACE_SPAN_END(submit_trace, ctx.self, ctx.task_id, -1, TracePhase::Submit, static_cast(is_winner), 1); - return ctx.result; + return dist_submit_finish_kernel_tail(ctx, mixed, args, prepare_map_end, ticket.submit_begin); +} + +DIST_API_ATTR PTO_DEVICE_FUNC DistCompeteFirstTicket dist_alloc_compete_first_begin(PTO2Runtime *) { + DistSubmitCtx ctx; + dist_submit_begin(nullptr, ctx); + TRACE_TIMESTAMP(submit_begin); + fdwic_perf_clock_submit_begin(ctx.task_id); + fdwic_submit_pmu_submit_begin(ctx.task_id); + fdwic_submit_pmu_phase_begin(); + drain_block_won(ctx.self); + drain_phase_b(ctx.self); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(efdrain_end); + TRACE_SPAN_RECORD(submit_begin, efdrain_end, ctx.self, ctx.task_id, -1, TracePhase::EfDrain, 0, 0); + + const uint64_t claim_begin = efdrain_end; + fdwic_submit_pmu_phase_begin(); + const bool ready = dist_submit_check_task_cap(ctx, DistSubmitKind::Alloc); + const bool is_winner = ready && dist_submit_claim(DistSubmitKind::Alloc, nullptr, ctx); + const uint32_t claim_flags = (is_winner ? kFdwicClaimWon : 0U) | (ctx.claim_attempted ? kFdwicClaimAttempted : 0U); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(claim_end); + fdwic_submit_pmu_phase_begin(); + fdwic_submit_pmu_empty_bracket_calibrate(); + TRACE_SPAN_RECORD(claim_begin, claim_end, ctx.self, ctx.task_id, -1, TracePhase::Claim, claim_flags, 1); + return dist_submit_make_ticket(ctx, DistSubmitKind::Alloc, submit_begin, ready); +} + +DIST_API_ATTR PTO_DEVICE_FUNC TaskOutputTensors +dist_alloc_compete_first_finish(PTO2Runtime *, const DistCompeteFirstTicket &ticket, const L0TaskArgs &args) { + DistSubmitCtx ctx; + dist_submit_restore_from_ticket(ticket, args, ctx); + if (!dist_submit_validate_ticket(ticket, DistSubmitKind::Alloc, ctx)) return ctx.result; + + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(materialize_begin); + fdwic_submit_pmu_phase_begin(); + uint64_t prepare_map_end = materialize_begin; + if (!dist_submit_materialize_and_prepare_map( + ctx.self, args, ctx, DistSubmitKind::Alloc, materialize_begin, prepare_map_end + )) { + return ctx.result; + } + fdwic_submit_pmu_phase_begin(); + (void)dist_submit_register_outputs(ctx, args, /*include_existing=*/false); + fdwic_submit_pmu_phase_end(); + TRACE_TIMESTAMP(register_end); + // Compete-first AllocComplete shares Register.end with the swimlane. The + // common tail closes it after dist_submit_complete_alloc() returns. + if (ctx.won) fdwic_submit_pmu_phase_begin(); + TRACE_SPAN_RECORD(prepare_map_end, register_end, ctx.self, ctx.task_id, -1, TracePhase::Register, 0, 0); + return dist_submit_finish_alloc_tail(ctx, register_end, ticket.submit_begin); } diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/tensor_data_access.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/tensor_data_access.h index 403bb43d8b..64ae982439 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/tensor_data_access.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/tensor_data_access.h @@ -19,17 +19,22 @@ namespace { // AICore sim orchestration replay shares the submit runtime path, so scalar // reads/writes must drain the worker's own queue until the producer is complete. PTO_DEVICE_FUNC void wait_producer_ready(DistCore *self, const Tensor &t) { - const int32_t p = dist_tensor_map_lookup(self->map, t); + const int32_t p = dist_tensor_map_lookup_for_task(*self, t, self->local_index); if (p < 0) return; uint64_t wd = 0; - while (!fatal_set()) { - if (task_flag_ready(p, __ATOMIC_ACQUIRE)) break; + const uint32_t producer_poll_region = fdwic_atomic_poll_region_begin( + fdwic_atomic_site_mask(FdwicAtomicSite::FatalPoll) | fdwic_atomic_site_mask(FdwicAtomicSite::FaninFlagLoad) | + fdwic_atomic_block_won_poll_mask() + ); + while (!fdwic_trace_is_fatal(p)) { + if (task_flag_ready(p, __ATOMIC_ACQUIRE, FdwicAtomicSite::FaninFlagLoad)) break; drain_block_won(self); if (drain_phase_b(self) == 0) { SPIN_WAIT_HINT(); watchdog(wd); } } + fdwic_atomic_poll_region_end(producer_poll_region); } #endif @@ -48,6 +53,11 @@ DIST_API_ATTR PTO_DEVICE_FUNC uint64_t dist_get_tensor_data_impl(PTO2Runtime *, const Tensor &tensor, uint32_t ndims, const uint32_t indices[]) { if (tensor.buffer.addr == 0) return 0; wait_tensor_data_access_ready(tensor); +#if PTO_FDWIC_SHARED_MAP && !defined(__CCE_AICORE__) + // Shared CPU-sim scalar access has no Claim/exact-turn proof. Its map + // facade latches a protocol error, so the underlying buffer must not be read. + if (fatal_set()) return 0; +#endif return dist_read_tensor_scalar_raw(tensor, ndims, indices); } @@ -56,5 +66,9 @@ DIST_API_ATTR PTO_DEVICE_FUNC void dist_set_tensor_data_impl( ) { if (tensor.buffer.addr == 0) return; wait_tensor_data_access_ready(tensor); +#if PTO_FDWIC_SHARED_MAP && !defined(__CCE_AICORE__) + // Match the read path: fail before mutating tensor storage. + if (fatal_set()) return; +#endif dist_write_tensor_scalar_raw(tensor, ndims, indices, value); } diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/tensor_map.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/tensor_map.h index b51c67da12..3b5caf6def 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/tensor_map.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/tensor_map.h @@ -11,115 +11,90 @@ #pragma once +#include "dist_engine/aicore/private_tensor_map.h" +#if PTO_FDWIC_SHARED_MAP +#include "dist_engine/aicore/shared_tensor_map.h" +#endif + namespace { -PTO_DEVICE_FUNC void dist_tensor_map_reset(__gm__ DistTensorMap &self) { - self.free_head = -1; - self.high_water = 0; - self.alive_floor = 0; - self.cleaned_upto = 0; - for (int32_t i = 0; i < kMapBuckets; i++) - self.buckets[i] = -1; - for (int32_t i = 0; i < kTaskWindow; i++) - self.task_heads[i] = -1; +/* + * Submit uses this facade instead of reading DistCore::map directly. Both + * modes share hash/range/slot semantics, while ownership and publication + * discipline remain compile-time choices: + * + * - private resets and retires one map per worker; every worker publishes its + * task outputs into its own copy; + * - shared is reset once by AICPU; only the exact-turn Submit winner reads or + * publishes the global map. CPU-sim scalar data access has no Claim/turn + * proof and remains explicitly unsupported under the shared identity. + * + * Keeping the split here prevents loser paths and worker reset from + * accidentally touching the shared single copy. + */ +PTO_DEVICE_FUNC void dist_tensor_map_reset_worker(__gm__ DistCore &worker) { +#if PTO_FDWIC_SHARED_MAP + (void)worker; +#else + dist_private_tensor_map_reset(worker.map); +#endif } -PTO_DEVICE_FUNC uint32_t dist_tensor_map_hash(uint64_t addr) { - addr *= 0x9E3779B97F4A7C15ULL; - return static_cast(addr >> (64 - kMapBucketShift)); +PTO_DEVICE_FUNC void dist_tensor_map_prepare_task(__gm__ DistCore &worker, int32_t task_id, int32_t history) { +#if PTO_FDWIC_SHARED_MAP + (void)worker; + (void)task_id; + (void)history; +#else + dist_private_tensor_map_advance_retire(worker.map, task_id, history); +#endif } +#if PTO_FDWIC_SHARED_MAP template -PTO_DEVICE_FUNC void dist_tensor_map_byte_range(const TensorRef &t, uint64_t &addr, uint64_t &lo, uint64_t &hi) { - const uint64_t esz = get_element_size(t.dtype); - addr = t.buffer.addr; - lo = t.start_offset * esz; - uint64_t ext; - if (t.is_contiguous) { - ext = 1; - for (uint32_t i = 0; i < t.ndims; i++) - ext *= t.shapes[i]; - } else { - ext = t.extent_elem_cache; - } - hi = (t.start_offset + ext) * esz; +PTO_DEVICE_FUNC bool dist_tensor_map_lookup_for_submit_winner( + __gm__ DistCore &worker, const TensorRef &tensor, int32_t consumer_task_id, int32_t &producer +) { + (void)worker; + bool protocol_ok = false; + producer = + dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, tensor, consumer_task_id, g_dist.H, protocol_ok); + return protocol_ok; } +#endif -PTO_DEVICE_FUNC int32_t dist_tensor_map_alloc_slot(__gm__ DistTensorMap &self) { - if (self.free_head >= 0) { - const int32_t s = self.free_head; - self.free_head = self.entries[s].next_in_bucket; - return s; - } - if (self.high_water < kMapCap) return self.high_water++; +template +PTO_DEVICE_FUNC int32_t +dist_tensor_map_lookup_for_task(__gm__ DistCore &worker, const TensorRef &tensor, int32_t consumer_task_id) { +#if PTO_FDWIC_SHARED_MAP + (void)worker; + (void)tensor; + (void)consumer_task_id; + set_fatal_code(PTO2_ERROR_TENSORMAP_PROTOCOL); return -1; +#else + (void)consumer_task_id; + return dist_private_tensor_map_lookup(worker.map, tensor); +#endif } -PTO_DEVICE_FUNC void dist_tensor_map_free_entry(__gm__ DistTensorMap &self, int32_t idx) { - __gm__ MapEntry &e = self.entries[idx]; - if (e.prev_in_bucket < 0) self.buckets[e.bucket] = e.next_in_bucket; - else self.entries[e.prev_in_bucket].next_in_bucket = e.next_in_bucket; - if (e.next_in_bucket >= 0) self.entries[e.next_in_bucket].prev_in_bucket = e.prev_in_bucket; - e.bucket = -1; - e.next_in_bucket = self.free_head; - self.free_head = idx; +#if PTO_FDWIC_SHARED_MAP +PTO_DEVICE_FUNC int64_t dist_tensor_map_next_publish_task() { + return DistSharedTensorMapAicoreOps::Load(&g_dist.shared_tensor_map.committed_tasks.v); } -PTO_DEVICE_FUNC void dist_tensor_map_advance_retire(__gm__ DistTensorMap &self, int32_t N, int32_t H) { - const int32_t new_floor = N - H; - if (new_floor <= self.cleaned_upto) { - if (new_floor > self.alive_floor) self.alive_floor = new_floor; - return; - } - for (int32_t id = self.cleaned_upto; id < new_floor; id++) { - int32_t cur = self.task_heads[id & kTaskWindowMask]; - while (cur >= 0) { - const int32_t nxt = self.entries[cur].next_in_task; - debug_assert(self.entries[cur].producer == id); - dist_tensor_map_free_entry(self, cur); - cur = nxt; - } - self.task_heads[id & kTaskWindowMask] = -1; - } - self.cleaned_upto = new_floor; - self.alive_floor = new_floor; +PTO_DEVICE_FUNC DistSharedTensorMapTaskPublishResult +dist_tensor_map_publish_shared_task(const SharedTensorMapValue *entries, uint32_t count, int32_t producer_task_id) { + return dist_shared_tensor_map_publish_task(g_dist.shared_tensor_map, entries, count, producer_task_id, g_dist.H); } - -template -PTO_DEVICE_FUNC void dist_tensor_map_insert(__gm__ DistTensorMap &self, const TensorRef &t, int32_t producer) { - uint64_t addr, lo, hi; - dist_tensor_map_byte_range(t, addr, lo, hi); - const int32_t s = dist_tensor_map_alloc_slot(self); - if (s < 0) return; - const uint32_t b = dist_tensor_map_hash(addr); - __gm__ MapEntry &e = self.entries[s]; - e.buf_addr = addr; - e.lo = lo; - e.hi = hi; - e.producer = producer; - e.bucket = static_cast(b); - e.prev_in_bucket = -1; - e.next_in_bucket = self.buckets[b]; - if (self.buckets[b] >= 0) self.entries[self.buckets[b]].prev_in_bucket = s; - self.buckets[b] = s; - const int32_t slot = producer & kTaskWindowMask; - e.next_in_task = self.task_heads[slot]; - self.task_heads[slot] = s; -} - +#else template -PTO_DEVICE_FUNC int32_t dist_tensor_map_lookup(__gm__ const DistTensorMap &self, const TensorRef &t) { - uint64_t addr, lo, hi; - dist_tensor_map_byte_range(t, addr, lo, hi); - int32_t best = -1; - for (int32_t cur = self.buckets[dist_tensor_map_hash(addr)]; cur >= 0; cur = self.entries[cur].next_in_bucket) { - __gm__ const MapEntry &e = self.entries[cur]; - if (e.producer < self.alive_floor) continue; - if (e.buf_addr == addr && lo < e.hi && e.lo < hi) { - if (e.producer > best) best = e.producer; - } - } - return best; +PTO_DEVICE_FUNC bool dist_tensor_map_insert_for_task( + __gm__ DistCore &worker, const TensorRef &tensor, int32_t producer_task_id, bool task_won +) { + (void)task_won; + return dist_private_tensor_map_insert(worker.map, tensor, producer_task_id); } +#endif } // namespace diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/tensor_map_common.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/tensor_map_common.h new file mode 100644 index 0000000000..3fecce877b --- /dev/null +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/tensor_map_common.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +namespace { + +// private/shared 只在副本所有权和并发发布纪律上分叉；hash、Tensor 的逻辑 +// byte range、连续分桶下标与半开区间重叠必须使用同一份实现，才能保证两种 +// backend 生成的依赖具有可比性。 +PTO_DEVICE_FUNC inline uint32_t dist_tensor_map_hash(uint64_t addr) { +#if PTO_FDWIC_TENSORMAP_RING_CAP == 16384 + (void)addr; + return 0; +#else + addr *= 0x9E3779B97F4A7C15ULL; + return static_cast(addr >> (64U - kMapBucketShift)) & kMapBucketMask; +#endif +} + +template +PTO_DEVICE_FUNC inline void +dist_tensor_map_byte_range(const TensorRef &tensor, uint64_t &addr, uint64_t &lo, uint64_t &hi) { + const uint64_t element_bytes = get_element_size(tensor.dtype); + addr = tensor.buffer.addr; + lo = tensor.start_offset * element_bytes; + uint64_t extent; + if (tensor.is_contiguous) { + extent = 1; + for (uint32_t dimension = 0; dimension < tensor.ndims; ++dimension) { + extent *= tensor.shapes[dimension]; + } + } else { + extent = tensor.extent_elem_cache; + } + hi = (tensor.start_offset + extent) * element_bytes; +} + +PTO_DEVICE_FUNC inline uint32_t dist_tensor_map_slot_index(uint32_t bucket, uint64_t cursor) { + return bucket * kMapBucketCapacity + (static_cast(cursor) & kMapBucketSlotMask); +} + +template +PTO_DEVICE_FUNC inline bool dist_tensor_map_regions_overlap( + const RegionValue &left, uint64_t right_addr, uint64_t right_lo, uint64_t right_hi +) { + return left.buf_addr == right_addr && right_lo < left.hi && left.lo < right_hi; +} + +} // namespace diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/config_parse.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/config_parse.h new file mode 100644 index 0000000000..0e022783fb --- /dev/null +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/config_parse.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include + +// PTO_DIST_H 只接受非空 ASCII 十进制数字串，并完整解析到给定闭区间。 +// 不借用 strtol 的空白/正负号语法，避免运维输入看似合法却改变保留窗口。 +inline bool dist_parse_history_window(const char *text, int32_t max_inclusive, int32_t &value) { + if (text == nullptr || *text == '\0' || max_inclusive < 0) return false; + int32_t parsed = 0; + for (const char *cursor = text; *cursor != '\0'; ++cursor) { + if (*cursor < '0' || *cursor > '9') return false; + const int32_t digit = static_cast(*cursor - '0'); + const int64_t next = static_cast(parsed) * 10 + digit; + if (next > max_inclusive) return false; + parsed = static_cast(next); + } + value = parsed; + return true; +} diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/control_plane.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/control_plane.h index a4e402ff3a..22f70c5fae 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/control_plane.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/control_plane.h @@ -13,26 +13,37 @@ // configuration reads and signal-handler installation; AICore images never // include this file. -void dist_engine_register(PTO2Runtime *rt, const L2TaskArgs *orch_args, int num_workers, Runtime *runtime) { +#include "dist_engine/aicpu/shared_tensor_map_init.h" + +int32_t dist_engine_register(PTO2Runtime *rt, const L2TaskArgs *orch_args, int num_workers, Runtime *runtime) { + if (runtime != nullptr) runtime->dist.shared_addr = 0; if (rt == nullptr || rt->dist_global == nullptr || rt->gm_heap == nullptr || rt->gm_heap_size == 0) { DIST_ERRF("[dist_engine] missing host-allocated runtime state\n"); - if (runtime != nullptr) runtime->dist.shared_addr = 0; - return; + return runtime_status_from_error_codes(PTO2_ERROR_INVALID_ARGS, PTO2_ERROR_NONE); + } + if (runtime == nullptr || num_workers <= 0 || num_workers > RUNTIME_MAX_WORKER) { + DIST_ERRF("[dist_engine] invalid runtime/worker configuration: runtime=%p workers=%d\n", runtime, num_workers); + return runtime_status_from_error_codes(PTO2_ERROR_DIST_CONFIG_INVALID, PTO2_ERROR_NONE); + } + int32_t configured_history = kHDefault; + if (const char *e = getenv("PTO_DIST_H")) { + if (!dist_parse_history_window(e, kTaskWindow - 2, configured_history)) { + DIST_ERRF( + "[dist_engine] invalid PTO_DIST_H='%s'; expected ASCII decimal digits in [0, %d]\n", e, kTaskWindow - 2 + ); + return runtime_status_from_error_codes(PTO2_ERROR_DIST_CONFIG_INVALID, PTO2_ERROR_NONE); + } } + g_dist_ptr = reinterpret_cast(rt->dist_global); g_dist.heap_base = static_cast(rt->gm_heap); g_dist.heap_size = rt->gm_heap_size; - // Dependency-span bound H (R = F - H). Env override for graphs with longer - // heap spans; default kHDefault. - g_dist.H = kHDefault; - if (const char *e = getenv("PTO_DIST_H")) { - const long h = std::strtol(e, nullptr, 10); - if (h >= 0) g_dist.H = static_cast(h); - } - // The producer map recycles a task's entry-head slot kTaskWindow tasks later; - // cleanup retires a task once it leaves the H span, so H must stay below the - // window (with margin) or a slot could be reused before its task is cleaned. - always_assert(g_dist.H < kTaskWindow - 1); + atomic_exchange(g_dist.error_code, int32_t{PTO2_ERROR_NONE}, __ATOMIC_RELAXED); + atomic_exchange(g_dist.fatal, int32_t{0}, __ATOMIC_RELAXED); + // Dependency-span bound H (R = F - H). Validation happens before touching + // the shared arena, so an invalid run cannot leave partially initialized + // control state for a later invocation. + g_dist.H = configured_history; #if DIST_SIM_HOST_CLOCK // Overhead-isolation gate (skip incore kernel calls, keep all bookkeeping). g_skip_exec = (getenv("PTO_DIST_SKIP_EXEC") != nullptr); @@ -46,9 +57,23 @@ void dist_engine_register(PTO2Runtime *rt, const L2TaskArgs *orch_args, int num_ atomic_exchange(g_dist.frontier, int64_t{-1}, __ATOMIC_RELAXED); for (int32_t i = 0; i < kFlagCap; i++) reset_task_cell(i); - atomic_exchange(g_dist.fatal, int32_t{0}, __ATOMIC_RELAXED); atomic_exchange(g_dist.replay_done, int64_t{0}, __ATOMIC_RELAXED); atomic_exchange(g_dist.started_count, int64_t{0}, __ATOMIC_RELAXED); + for (int32_t group = 0; group < kFinalBarrierGroups; group++) { + atomic_exchange(g_dist.final_barrier.leaf_arrivals[group].v, int64_t{0}, __ATOMIC_RELAXED); + g_dist.final_barrier.leaf_arrivals[group].expected = 0; + atomic_exchange(g_dist.final_barrier.leaf_releases[group].v, int64_t{0}, __ATOMIC_RELAXED); + } + atomic_exchange(g_dist.final_barrier.root_arrival.v, int64_t{0}, __ATOMIC_RELAXED); + g_dist.final_barrier.root_arrival.expected = 0; + atomic_exchange(g_dist.final_barrier.root_release.v, int64_t{0}, __ATOMIC_RELAXED); +#if PTO_FDWIC_SHARED_MAP + // The upper backend-ready gate still rejects normal shared execution before + // the first Submit. Keep one-time initialization on the real AICPU setup + // path so enabling the backend needs no second reset path; worker reset must + // never clear this global single copy concurrently. + dist_shared_tensor_map_reset(g_dist.shared_tensor_map); +#endif g_dist.orch_args = orch_args; g_dist.rt = rt; g_dist.runtime = runtime; @@ -80,6 +105,17 @@ void dist_engine_register(PTO2Runtime *rt, const L2TaskArgs *orch_args, int num_ atomic_exchange(g_dist.blocks[b].slots[s].state.v, int64_t{0}, __ATOMIC_RELAXED); } } + int32_t active_final_groups = 0; + for (int32_t i = 0; i < num_workers && i < RUNTIME_MAX_WORKER; i++) { + const int32_t block_id = g_dist.layout[i].block_id; + if (block_id < 0) continue; + const int32_t group = block_id % kFinalBarrierGroups; + ++g_dist.final_barrier.leaf_arrivals[group].expected; + } + for (int32_t group = 0; group < kFinalBarrierGroups; group++) { + if (g_dist.final_barrier.leaf_arrivals[group].expected != 0) ++active_final_groups; + } + g_dist.final_barrier.root_arrival.expected = active_final_groups; #if DIST_SIM_HOST_CLOCK fprintf( @@ -107,5 +143,18 @@ void dist_engine_register(PTO2Runtime *rt, const L2TaskArgs *orch_args, int num_ // Publish all of the above before AICPU wakes workers through their // per-core handshake flags. store_barrier(); - return; + return 0; +} + +int32_t dist_engine_runtime_status(PTO2Runtime *rt) { + if (rt == nullptr || rt->dist_global == nullptr) { + return runtime_status_from_error_codes(PTO2_ERROR_INVALID_ARGS, PTO2_ERROR_NONE); + } + DistGlobal *state = reinterpret_cast(rt->dist_global); + cache_invalidate_range(const_cast(&state->fatal), kCacheLine); + const int32_t fatal = __atomic_load_n(&state->fatal, __ATOMIC_ACQUIRE); + if (fatal == 0) return 0; + int32_t code = __atomic_load_n(&state->error_code, __ATOMIC_ACQUIRE); + if (code == PTO2_ERROR_NONE) code = PTO2_ERROR_EXPLICIT_ORCH_FATAL; + return runtime_status_from_error_codes(code, PTO2_ERROR_NONE); } diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/dist_engine.cpp b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/dist_engine.cpp index 2dd0a0f42f..ab8f70792a 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/dist_engine.cpp +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/dist_engine.cpp @@ -11,6 +11,7 @@ #include "spin_hint.h" +#include "aicpu/platform_regs.h" #include "pto_runtime2.h" #include "runtime.h" @@ -22,4 +23,5 @@ #include "dist_engine/common/sim_control.h" // NOLINT(build/include_subdir) #include "dist_engine/aicpu/log.h" // NOLINT(build/include_subdir) +#include "dist_engine/aicpu/config_parse.h" // NOLINT(build/include_subdir) #include "dist_engine/aicpu/control_plane.h" // NOLINT(build/include_subdir) diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/shared_tensor_map_init.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/shared_tensor_map_init.h new file mode 100644 index 0000000000..e77ef909e1 --- /dev/null +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/shared_tensor_map_init.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include "dist_engine/common/atomic.h" +#include "dist_engine/common/state.h" + +namespace { + +// shared sidecar 只能由 AICPU setup thread 在唤醒 worker 前初始化一次。 +// payload 无需清零：reader 只接受与 absolute cursor 相等的非负 seq； +// -1 和 WRITING 都不可读，reset 统一恢复为 -1，且随后 AICPU 会把整个 +// DistGlobal flush 到 GM。显式重置所有控制字可以支持同一 arena 的重复 run， +// 也避免把 slot 0 的合法 seq=0 与零填充状态混为一谈。 +inline void dist_shared_tensor_map_reset(SharedTensorMapState &state) { + atomic_exchange(state.committed_tasks.v, int64_t{kSharedTensorMapInitialCommit}, __ATOMIC_RELAXED); + atomic_exchange(state.reclaim_upto.v, int64_t{kSharedTensorMapInitialReclaim}, __ATOMIC_RELAXED); + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + atomic_exchange(state.buckets[bucket].head.v, int64_t{0}, __ATOMIC_RELAXED); + atomic_exchange(state.buckets[bucket].tail.v, int64_t{0}, __ATOMIC_RELAXED); + } + for (int32_t slot = 0; slot < kMapCap; ++slot) { + atomic_exchange(state.slots[slot].sequence.v, int64_t{kSharedTensorMapInvalidSequence}, __ATOMIC_RELAXED); + } +} + +} // namespace diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/submit_pmu_owner.cpp b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/submit_pmu_owner.cpp new file mode 100644 index 0000000000..df75560e48 --- /dev/null +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/submit_pmu_owner.cpp @@ -0,0 +1,647 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "dist_engine/aicpu/submit_pmu_owner.h" + +#include +#include + +#include "aicpu/platform_regs.h" +#include "common/core_type.h" +#include "common/platform_config.h" +#include "common/unified_log.h" +#include "runtime.h" + +namespace { + +constexpr uint32_t kCounterCount = 10U; +constexpr uint32_t kPhysicalSubcoresPerDie = 54U; +constexpr uint32_t kAicPerDie = 18U; +constexpr uint32_t kDies = kFdwicSubmitPmuPhysicalSubcores / kPhysicalSubcoresPerDie; +constexpr uint32_t kUnsetCore = UINT32_MAX; + +// submit-PMU 保留 CNT2/CNT6/CNT7 作为不在窗口中途读取的权威计数，并用 +// CNT3/CNT8/CNT5 复制同一 scalar/request/miss 事件。none 要求副本与 +// primary 逐核精确相等；running phase 则重建 shadow whole 并要求不超过 +// primary。CNT3 原来的 MTE1 busy 没有进入 submit-PMU raw，因此替换它不会 +// 丢失既有对外指标。 +constexpr uint32_t kConfiguredSelectors[kCounterCount] = { + 0x501U, // CNT0: vector busy + 0x301U, // CNT1: cube busy + 0x001U, // CNT2: scalar busy + 0x001U, // CNT3: shadow scalar busy + 0x202U, // CNT4: MTE2 busy + 0x035U, // CNT5: shadow I-cache miss + 0x034U, // CNT6: primary I-cache request + 0x035U, // CNT7: primary I-cache miss + 0x034U, // CNT8: shadow I-cache request + 0x000U, // CNT9: DAV3510 正式未使用；本机已反证不能承载 miss +}; + +static_assert(kConfiguredSelectors[2] == kFdwicSubmitPmuCnt2ScalarBusy, "CNT2 selector contract changed"); +static_assert(kConfiguredSelectors[3] == kFdwicSubmitPmuCnt3ShadowScalarBusy, "CNT3 shadow selector contract changed"); +static_assert(kConfiguredSelectors[5] == kFdwicSubmitPmuCnt5ShadowIcacheMiss, "CNT5 selector contract changed"); +static_assert(kConfiguredSelectors[6] == kFdwicSubmitPmuCnt6IcacheRequest, "CNT6 selector contract changed"); +static_assert(kConfiguredSelectors[7] == kFdwicSubmitPmuCnt7IcacheMiss, "CNT7 selector contract changed"); +static_assert(kConfiguredSelectors[8] == kFdwicSubmitPmuCnt8ShadowIcacheRequest, "CNT8 selector contract changed"); +static_assert(kFdwicSubmitPmuPhysicalSubcores == kDies * kPhysicalSubcoresPerDie, "physical topology changed"); + +// owner 改写的寄存器原值恰好占一条 cacheline。事件 counter 和 TOTAL 是 +// read-to-clear，旧计数无法保存，也不属于可恢复配置。 +struct alignas(64) SavedRegisters { + uint32_t ctrl0; + uint32_t ctrl1; + uint32_t selectors[kCounterCount]; + uint32_t start_low; + uint32_t start_high; + uint32_t stop_low; + uint32_t stop_high; +}; + +static_assert(sizeof(SavedRegisters) == 64U, "saved PMU configuration must occupy one cacheline"); + +// 该状态只存在于本次 AICPU runtime SO 内，不属于 host/device ABI。saved[] +// 按物理子核编号索引；owned_bitmap 是唯一所有权真相，配置或恢复失败后 +// 不得覆盖对应 saved 槽。 +struct OwnerState { + SavedRegisters saved[kFdwicSubmitPmuPhysicalSubcores]; + uint32_t owned_bitmap[kFdwicSubmitPmuBitmapWords]; + uint32_t configured_bitmap[kFdwicSubmitPmuBitmapWords]; + uint32_t configured_count; + uint32_t configured_aic; + uint32_t configured_aiv; + uint32_t restored_count; + uint32_t restore_failures; + bool configured; +}; + +OwnerState g_owner; + +inline void FullSystemBarrier() { +#if defined(__aarch64__) + __asm__ volatile("dsb sy" ::: "memory"); +#else + // submit-pmu 只允许真实 A5 使用；该分支仅保证同一源码可参与 a5sim 构建。 + __asm__ volatile("" ::: "memory"); +#endif +} + +inline bool IsAicPhysicalId(uint32_t physical_id) { + return physical_id < kFdwicSubmitPmuPhysicalSubcores && (physical_id % kPhysicalSubcoresPerDie) < kAicPerDie; +} + +inline bool BitmapContains(const uint32_t *bitmap, uint32_t physical_id) { + return physical_id < kFdwicSubmitPmuPhysicalSubcores && + (bitmap[physical_id / 32U] & (1U << (physical_id % 32U))) != 0U; +} + +inline void BitmapSet(uint32_t *bitmap, uint32_t physical_id) { + bitmap[physical_id / 32U] |= 1U << (physical_id % 32U); +} + +inline void BitmapClear(uint32_t *bitmap, uint32_t physical_id) { + bitmap[physical_id / 32U] &= ~(1U << (physical_id % 32U)); +} + +uint32_t BitmapCount(const uint32_t *bitmap) { + uint32_t count = 0U; + for (uint32_t physical_id = 0; physical_id < kFdwicSubmitPmuPhysicalSubcores; ++physical_id) { + count += BitmapContains(bitmap, physical_id) ? 1U : 0U; + } + return count; +} + +constexpr FdwicSubmitPmuOwnerField SelectorField(uint32_t counter) { + return static_cast(static_cast(FdwicSubmitPmuOwnerField::Selector0) + counter); +} + +void FlushOwnerState(FdwicSubmitPmuHeader *header) { + if (header == nullptr) return; + cache_flush_range( + const_cast(&header->owner_status), + offsetof(FdwicSubmitPmuHeader, cores) - offsetof(FdwicSubmitPmuHeader, owner_status) + ); +} + +void ResetHeaderOwnerState(FdwicSubmitPmuHeader *header) { + header->owner_status = kFdwicSubmitPmuOwnerRequested; + header->configured_count = 0U; + header->restored_count = 0U; + header->configured_aic = 0U; + header->configured_aiv = 0U; + header->complete_mixed_triplets = 0U; + header->restore_failures = 0U; + header->active_after_restore = 0U; + for (uint32_t word = 0; word < kFdwicSubmitPmuBitmapWords; ++word) { + header->configured_bitmap_words[word] = 0U; + } + header->first_failure_core = kUnsetCore; + header->first_failure_field = static_cast(FdwicSubmitPmuOwnerField::None); + header->first_failure_observed = 0U; + header->first_failure_expected = 0U; + FlushOwnerState(header); +} + +void RecordFirstFailure( + FdwicSubmitPmuHeader *header, uint32_t physical_id, FdwicSubmitPmuOwnerField field, uint32_t observed, + uint32_t expected +) { + if (header == nullptr || header->first_failure_core != kUnsetCore) return; + header->first_failure_core = physical_id; + header->first_failure_field = static_cast(field); + header->first_failure_observed = observed; + header->first_failure_expected = expected; +} + +void PublishOwnerProgress(FdwicSubmitPmuHeader *header) { + if (header == nullptr) return; + header->configured_count = g_owner.configured_count; + header->configured_aic = g_owner.configured_aic; + header->configured_aiv = g_owner.configured_aiv; + header->restored_count = g_owner.restored_count; + header->restore_failures = g_owner.restore_failures; + header->active_after_restore = BitmapCount(g_owner.owned_bitmap); + for (uint32_t word = 0; word < kFdwicSubmitPmuBitmapWords; ++word) { + // 这是本次成功配置集合的永久快照，不是恢复过程中逐步清零的 + // ownership bitmap；host 在 Restore 后仍需据此核对96条worker记录。 + header->configured_bitmap_words[word] = g_owner.configured_bitmap[word]; + } + FlushOwnerState(header); +} + +void ResetOwnerState() { + for (uint32_t word = 0; word < kFdwicSubmitPmuBitmapWords; ++word) { + g_owner.owned_bitmap[word] = 0U; + g_owner.configured_bitmap[word] = 0U; + } + g_owner.configured_count = 0U; + g_owner.configured_aic = 0U; + g_owner.configured_aiv = 0U; + g_owner.restored_count = 0U; + g_owner.restore_failures = 0U; + g_owner.configured = false; +} + +void SaveOne(uint64_t reg_base, uint32_t physical_id) { + SavedRegisters &saved = g_owner.saved[physical_id]; + saved.ctrl0 = static_cast(read_reg(reg_base, RegId::PMU_CTRL_0)); + saved.ctrl1 = static_cast(read_reg(reg_base, RegId::PMU_CTRL_1)); + for (uint32_t counter = 0; counter < kCounterCount; ++counter) { + saved.selectors[counter] = static_cast(read_reg(reg_base, reg_index(RegId::PMU_CNT0_IDX, counter))); + } + saved.start_low = static_cast(read_reg(reg_base, RegId::PMU_START_CYC0)); + saved.start_high = static_cast(read_reg(reg_base, RegId::PMU_START_CYC1)); + saved.stop_low = static_cast(read_reg(reg_base, RegId::PMU_STOP_CYC0)); + saved.stop_high = static_cast(read_reg(reg_base, RegId::PMU_STOP_CYC1)); +} + +void ConfigureOne(uint64_t reg_base) { + // 所有写之前，调用方已经保存原值并置 ownership bit。DSB 只存在于 + // AICPU owner 冷路径，不进入任何 Submit 或 AICore scalar 观察窗口。 + write_reg(reg_base, RegId::PMU_CTRL_0, 0U); + write_reg(reg_base, RegId::PMU_CTRL_1, 0U); + for (uint32_t counter = 0; counter < kCounterCount; ++counter) { + write_reg(reg_base, reg_index(RegId::PMU_CNT0_IDX, counter), kConfiguredSelectors[counter]); + } + + // DAV3510 的事件 counter 和 64-bit TOTAL 都是 read-to-clear。此处清除 + // owner 配置前的残值；旧计数无法也不应伪装成可恢复状态。 + for (uint32_t counter = 0; counter < kCounterCount; ++counter) { + (void)read_reg(reg_base, reg_index(RegId::PMU_CNT0, counter)); + } + (void)read_reg(reg_base, RegId::PMU_CNT_TOTAL0); + (void)read_reg(reg_base, RegId::PMU_CNT_TOTAL1); + + write_reg(reg_base, RegId::PMU_START_CYC0, 0U); + write_reg(reg_base, RegId::PMU_START_CYC1, 0U); + write_reg(reg_base, RegId::PMU_STOP_CYC0, UINT32_MAX); + write_reg(reg_base, RegId::PMU_STOP_CYC1, UINT32_MAX); + write_reg(reg_base, RegId::PMU_CTRL_0, REG_MMIO_PMU_CTRL_0_ENABLE_VAL); + write_reg(reg_base, RegId::PMU_CTRL_1, REG_MMIO_PMU_CTRL_1_ENABLE_VAL); + FullSystemBarrier(); +} + +bool CheckRegister( + uint64_t reg_base, RegId reg, uint32_t expected, FdwicSubmitPmuOwnerField field, + FdwicSubmitPmuOwnerField *failed_field, uint32_t *observed +) { + const uint32_t actual = static_cast(read_reg(reg_base, reg)); + if (actual == expected) return true; + *failed_field = field; + *observed = actual; + return false; +} + +bool ConfigurationMatches( + uint64_t reg_base, FdwicSubmitPmuOwnerField *failed_field, uint32_t *observed, uint32_t *expected +) { + if (!CheckRegister( + reg_base, RegId::PMU_CTRL_0, REG_MMIO_PMU_CTRL_0_ENABLE_VAL, FdwicSubmitPmuOwnerField::Ctrl0, failed_field, + observed + )) { + *expected = REG_MMIO_PMU_CTRL_0_ENABLE_VAL; + return false; + } + if (!CheckRegister( + reg_base, RegId::PMU_CTRL_1, REG_MMIO_PMU_CTRL_1_ENABLE_VAL, FdwicSubmitPmuOwnerField::Ctrl1, failed_field, + observed + )) { + *expected = REG_MMIO_PMU_CTRL_1_ENABLE_VAL; + return false; + } + for (uint32_t counter = 0; counter < kCounterCount; ++counter) { + if (!CheckRegister( + reg_base, reg_index(RegId::PMU_CNT0_IDX, counter), kConfiguredSelectors[counter], + SelectorField(counter), failed_field, observed + )) { + *expected = kConfiguredSelectors[counter]; + return false; + } + } + if (!CheckRegister( + reg_base, RegId::PMU_START_CYC0, 0U, FdwicSubmitPmuOwnerField::StartLow, failed_field, observed + )) { + *expected = 0U; + return false; + } + if (!CheckRegister( + reg_base, RegId::PMU_START_CYC1, 0U, FdwicSubmitPmuOwnerField::StartHigh, failed_field, observed + )) { + *expected = 0U; + return false; + } + if (!CheckRegister( + reg_base, RegId::PMU_STOP_CYC0, UINT32_MAX, FdwicSubmitPmuOwnerField::StopLow, failed_field, observed + )) { + *expected = UINT32_MAX; + return false; + } + if (!CheckRegister( + reg_base, RegId::PMU_STOP_CYC1, UINT32_MAX, FdwicSubmitPmuOwnerField::StopHigh, failed_field, observed + )) { + *expected = UINT32_MAX; + return false; + } + return true; +} + +void WriteSavedConfiguration(uint64_t reg_base, uint32_t physical_id) { + const SavedRegisters &saved = g_owner.saved[physical_id]; + write_reg(reg_base, RegId::PMU_CTRL_0, 0U); + write_reg(reg_base, RegId::PMU_CTRL_1, 0U); + for (uint32_t counter = 0; counter < kCounterCount; ++counter) { + write_reg(reg_base, reg_index(RegId::PMU_CNT0_IDX, counter), saved.selectors[counter]); + } + write_reg(reg_base, RegId::PMU_START_CYC0, saved.start_low); + write_reg(reg_base, RegId::PMU_START_CYC1, saved.start_high); + write_reg(reg_base, RegId::PMU_STOP_CYC0, saved.stop_low); + write_reg(reg_base, RegId::PMU_STOP_CYC1, saved.stop_high); + // CTRL 最后恢复，避免 selector/range 尚未恢复时短暂重新开启旧配置。 + write_reg(reg_base, RegId::PMU_CTRL_0, saved.ctrl0); + write_reg(reg_base, RegId::PMU_CTRL_1, saved.ctrl1); + FullSystemBarrier(); +} + +bool SavedConfigurationMatches( + uint64_t reg_base, uint32_t physical_id, FdwicSubmitPmuOwnerField *failed_field, uint32_t *observed, + uint32_t *expected +) { + const SavedRegisters &saved = g_owner.saved[physical_id]; + if (!CheckRegister( + reg_base, RegId::PMU_CTRL_0, saved.ctrl0, FdwicSubmitPmuOwnerField::Ctrl0, failed_field, observed + )) { + *expected = saved.ctrl0; + return false; + } + if (!CheckRegister( + reg_base, RegId::PMU_CTRL_1, saved.ctrl1, FdwicSubmitPmuOwnerField::Ctrl1, failed_field, observed + )) { + *expected = saved.ctrl1; + return false; + } + for (uint32_t counter = 0; counter < kCounterCount; ++counter) { + if (!CheckRegister( + reg_base, reg_index(RegId::PMU_CNT0_IDX, counter), saved.selectors[counter], SelectorField(counter), + failed_field, observed + )) { + *expected = saved.selectors[counter]; + return false; + } + } + if (!CheckRegister( + reg_base, RegId::PMU_START_CYC0, saved.start_low, FdwicSubmitPmuOwnerField::StartLow, failed_field, observed + )) { + *expected = saved.start_low; + return false; + } + if (!CheckRegister( + reg_base, RegId::PMU_START_CYC1, saved.start_high, FdwicSubmitPmuOwnerField::StartHigh, failed_field, + observed + )) { + *expected = saved.start_high; + return false; + } + if (!CheckRegister( + reg_base, RegId::PMU_STOP_CYC0, saved.stop_low, FdwicSubmitPmuOwnerField::StopLow, failed_field, observed + )) { + *expected = saved.stop_low; + return false; + } + if (!CheckRegister( + reg_base, RegId::PMU_STOP_CYC1, saved.stop_high, FdwicSubmitPmuOwnerField::StopHigh, failed_field, observed + )) { + *expected = saved.stop_high; + return false; + } + return true; +} + +bool RestoreOne(uint64_t *register_bases, uint32_t physical_id, FdwicSubmitPmuHeader *header) { + const uint64_t reg_base = register_bases == nullptr ? 0U : register_bases[physical_id]; + if (reg_base == 0U) { + RecordFirstFailure(header, physical_id, FdwicSubmitPmuOwnerField::RegisterBase, 0U, 1U); + ++g_owner.restore_failures; + return false; + } + + WriteSavedConfiguration(reg_base, physical_id); + FdwicSubmitPmuOwnerField failed_field = FdwicSubmitPmuOwnerField::None; + uint32_t observed = 0U; + uint32_t expected = 0U; + if (!SavedConfigurationMatches(reg_base, physical_id, &failed_field, &observed, &expected)) { + RecordFirstFailure(header, physical_id, failed_field, observed, expected); + ++g_owner.restore_failures; + return false; + } + return true; +} + +bool RestoreOwned(FdwicSubmitPmuHeader *header) { + uint64_t *register_bases = reinterpret_cast(get_platform_regs()); + bool all_restored = true; + for (uint32_t next = kFdwicSubmitPmuPhysicalSubcores; next != 0U; --next) { + const uint32_t physical_id = next - 1U; + if (!BitmapContains(g_owner.owned_bitmap, physical_id)) continue; + if (RestoreOne(register_bases, physical_id, header)) { + BitmapClear(g_owner.owned_bitmap, physical_id); + ++g_owner.restored_count; + } else { + all_restored = false; + } + } + return all_restored && BitmapCount(g_owner.owned_bitmap) == 0U; +} + +bool HeaderConfigurationMatches(const FdwicSubmitPmuHeader &header) { + const uint32_t expected_selectors[6] = { + kFdwicSubmitPmuCnt2ScalarBusy, kFdwicSubmitPmuCnt3ShadowScalarBusy, kFdwicSubmitPmuCnt5ShadowIcacheMiss, + kFdwicSubmitPmuCnt6IcacheRequest, kFdwicSubmitPmuCnt7IcacheMiss, kFdwicSubmitPmuCnt8ShadowIcacheRequest, + }; + const bool mode_valid = header.mode == kFdwicSubmitPmuModeNone || fdwic_submit_pmu_mode_has_phase(header.mode); + if (header.magic != kFdwicSubmitPmuMagic || header.version != kFdwicSubmitPmuVersion || !mode_valid || + header.header_bytes != fdwic_submit_pmu_bytes_for_mode(header.mode) || + header.record_bytes != sizeof(FdwicSubmitPmuCoreData) || header.num_cores != kFdwicSubmitPmuExpectedCores || + header.expected_aic != kFdwicSubmitPmuExpectedAic || header.expected_aiv != kFdwicSubmitPmuExpectedAiv || + header.sys_cnt_freq_hz != PLATFORM_PROF_SYS_CNT_FREQ) { + return false; + } + for (uint32_t index = 0; index < 6U; ++index) { + if (header.selectors[index] != expected_selectors[index]) return false; + } + return true; +} + +bool ValidateActiveTopology( + Runtime *runtime, uint32_t *active_bitmap, uint32_t *aic_count, uint32_t *aiv_count, uint32_t *triplet_count, + uint32_t *failed_core, uint32_t *observed, uint32_t *expected +) { + if (runtime == nullptr || runtime->worker_count != static_cast(kFdwicSubmitPmuExpectedCores)) { + *failed_core = kUnsetCore; + *observed = runtime == nullptr ? 0U : static_cast(runtime->worker_count); + *expected = kFdwicSubmitPmuExpectedCores; + return false; + } + + uint64_t *register_bases = reinterpret_cast(get_platform_regs()); + if (register_bases == nullptr) { + *failed_core = kUnsetCore; + *observed = 0U; + *expected = 1U; + return false; + } + + *aic_count = 0U; + *aiv_count = 0U; + *triplet_count = 0U; + for (uint32_t word = 0; word < kFdwicSubmitPmuBitmapWords; ++word) + active_bitmap[word] = 0U; + + for (uint32_t logical_id = 0; logical_id < kFdwicSubmitPmuExpectedCores; ++logical_id) { + const Handshake &worker = runtime->workers[logical_id]; + const uint32_t physical_id = worker.physical_core_id; + const CoreType role = worker.core_type; + if (physical_id >= kFdwicSubmitPmuPhysicalSubcores || BitmapContains(active_bitmap, physical_id)) { + *failed_core = physical_id; + *observed = physical_id >= kFdwicSubmitPmuPhysicalSubcores ? physical_id : 2U; + *expected = physical_id >= kFdwicSubmitPmuPhysicalSubcores ? kFdwicSubmitPmuPhysicalSubcores - 1U : 1U; + return false; + } + if (register_bases[physical_id] == 0U) { + *failed_core = physical_id; + *observed = 0U; + *expected = 1U; + return false; + } + + const bool physical_aic = IsAicPhysicalId(physical_id); + const bool reported_aic = role == CoreType::AIC; + const bool role_valid = reported_aic || role == CoreType::AIV; + if (!role_valid || physical_aic != reported_aic) { + *failed_core = physical_id; + *observed = static_cast(role); + *expected = static_cast(physical_aic ? CoreType::AIC : CoreType::AIV); + return false; + } + BitmapSet(active_bitmap, physical_id); + *aic_count += reported_aic ? 1U : 0U; + *aiv_count += reported_aic ? 0U : 1U; + } + + if (*aic_count != kFdwicSubmitPmuExpectedAic || *aiv_count != kFdwicSubmitPmuExpectedAiv) { + *failed_core = kUnsetCore; + *observed = (*aic_count << 16U) | *aiv_count; + *expected = (kFdwicSubmitPmuExpectedAic << 16U) | kFdwicSubmitPmuExpectedAiv; + return false; + } + + uint32_t broken_triplets = 0U; + for (uint32_t die = 0; die < kDies; ++die) { + const uint32_t die_base = die * kPhysicalSubcoresPerDie; + for (uint32_t local = 0; local < kAicPerDie; ++local) { + const bool aic = BitmapContains(active_bitmap, die_base + local); + const bool aiv0 = BitmapContains(active_bitmap, die_base + kAicPerDie + local * 2U); + const bool aiv1 = BitmapContains(active_bitmap, die_base + kAicPerDie + local * 2U + 1U); + if (aic == aiv0 && aic == aiv1) { + *triplet_count += aic ? 1U : 0U; + } else { + ++broken_triplets; + } + } + } + if (*triplet_count != kFdwicSubmitPmuExpectedAic || broken_triplets != 0U) { + *failed_core = kUnsetCore; + *observed = (*triplet_count << 16U) | broken_triplets; + *expected = kFdwicSubmitPmuExpectedAic << 16U; + return false; + } + return true; +} + +} // namespace + +bool fdwic_submit_pmu_requested(Runtime *runtime, FdwicSubmitPmuHeader **header_out) { + if (header_out != nullptr) *header_out = nullptr; + if (runtime == nullptr || runtime->dist.swimlane_base == 0U || runtime->dist.swimlane_level != 0U || + runtime->dist.swimlane_records_per_core != 0U) { + return false; + } + + auto *header = reinterpret_cast(runtime->dist.swimlane_base); + if ((reinterpret_cast(header) & (alignof(FdwicSubmitPmuHeader) - 1U)) != 0U) return false; + // Host H2D 不维护 AICPU L1。这里只失效 host 配置与 owner 两条 cacheline， + // 不触碰随后由96个AICore独占写入的结果区。 + cache_invalidate_range(header, offsetof(FdwicSubmitPmuHeader, cores)); + if (!HeaderConfigurationMatches(*header)) return false; + if (header_out != nullptr) *header_out = header; + return true; +} + +int fdwic_submit_pmu_owner_configure(Runtime *runtime, FdwicSubmitPmuHeader *header) { + if (runtime == nullptr || header == nullptr) return -1; + + ResetHeaderOwnerState(header); + + // 上一次 Restore 若留下 bit，saved[] 仍是唯一原值，绝不能被新一轮 + // Configure 覆盖。先按当前完整 MMIO 表幂等重试；失败则拒绝新会话。 + const bool retried_stale_owner = BitmapCount(g_owner.owned_bitmap) != 0U; + if (retried_stale_owner) { + if (!RestoreOwned(header)) { + header->owner_status = + kFdwicSubmitPmuOwnerRequested | kFdwicSubmitPmuOwnerRestoreAttempted | kFdwicSubmitPmuOwnerAborted; + PublishOwnerProgress(header); + return -1; + } + } + + ResetOwnerState(); + // stale owner 已恢复时，上一会话的恢复计数与配置历史不能混进本轮。 + if (retried_stale_owner) ResetHeaderOwnerState(header); + + uint32_t active_bitmap[kFdwicSubmitPmuBitmapWords] = {}; + uint32_t aic_count = 0U; + uint32_t aiv_count = 0U; + uint32_t triplet_count = 0U; + uint32_t failed_core = kUnsetCore; + uint32_t observed = 0U; + uint32_t expected = 0U; + if (!ValidateActiveTopology( + runtime, active_bitmap, &aic_count, &aiv_count, &triplet_count, &failed_core, &observed, &expected + )) { + RecordFirstFailure(header, failed_core, FdwicSubmitPmuOwnerField::Topology, observed, expected); + header->owner_status |= kFdwicSubmitPmuOwnerAborted; + FlushOwnerState(header); + return -1; + } + header->owner_status |= kFdwicSubmitPmuOwnerTopologyValid; + header->complete_mixed_triplets = triplet_count; + FlushOwnerState(header); + + uint64_t *register_bases = reinterpret_cast(get_platform_regs()); + for (uint32_t physical_id = 0; physical_id < kFdwicSubmitPmuPhysicalSubcores; ++physical_id) { + if (!BitmapContains(active_bitmap, physical_id)) continue; + const uint64_t reg_base = register_bases[physical_id]; + + SaveOne(reg_base, physical_id); + // ownership bit 必须先于本槽第一条 MMIO 写。即使配置和当场恢复都 + // 失败，该 bit 仍保留，使后续 Restore 可以继续使用未被覆盖的 saved[]。 + BitmapSet(g_owner.owned_bitmap, physical_id); + ConfigureOne(reg_base); + + FdwicSubmitPmuOwnerField failed_field = FdwicSubmitPmuOwnerField::None; + observed = 0U; + expected = 0U; + if (!ConfigurationMatches(reg_base, &failed_field, &observed, &expected)) { + RecordFirstFailure(header, physical_id, failed_field, observed, expected); + const bool rollback_ok = RestoreOwned(header); + header->owner_status |= kFdwicSubmitPmuOwnerRestoreAttempted | kFdwicSubmitPmuOwnerAborted; + if (rollback_ok) header->owner_status |= kFdwicSubmitPmuOwnerRestored; + PublishOwnerProgress(header); + return -1; + } + + BitmapSet(g_owner.configured_bitmap, physical_id); + ++g_owner.configured_count; + if (IsAicPhysicalId(physical_id)) { + ++g_owner.configured_aic; + } else { + ++g_owner.configured_aiv; + } + } + + const bool counts_match = g_owner.configured_count == kFdwicSubmitPmuExpectedCores && + g_owner.configured_aic == kFdwicSubmitPmuExpectedAic && + g_owner.configured_aiv == kFdwicSubmitPmuExpectedAiv && + BitmapCount(g_owner.owned_bitmap) == kFdwicSubmitPmuExpectedCores; + if (!counts_match) { + RecordFirstFailure( + header, kUnsetCore, FdwicSubmitPmuOwnerField::State, g_owner.configured_count, kFdwicSubmitPmuExpectedCores + ); + const bool rollback_ok = RestoreOwned(header); + header->owner_status |= kFdwicSubmitPmuOwnerRestoreAttempted | kFdwicSubmitPmuOwnerAborted; + if (rollback_ok) header->owner_status |= kFdwicSubmitPmuOwnerRestored; + PublishOwnerProgress(header); + return -1; + } + + g_owner.configured = true; + header->owner_status |= kFdwicSubmitPmuOwnerConfigured | kFdwicSubmitPmuOwnerConfigReadbackValid; + PublishOwnerProgress(header); + LOG_INFO_V0( + "FDWIC submit-PMU owner configured %u physical subcores (%u AIC + %u AIV)", g_owner.configured_count, + g_owner.configured_aic, g_owner.configured_aiv + ); + return 0; +} + +int fdwic_submit_pmu_owner_restore(FdwicSubmitPmuHeader *header) { + if (header != nullptr) header->owner_status |= kFdwicSubmitPmuOwnerRestoreAttempted; + + // bitmap 为空时仍把 Restore 视为幂等成功；configured_count/bitmap 快照 + // 保留给 host 做 worker membership 校验。 + const bool restored = RestoreOwned(header); + if (restored) { + g_owner.configured = false; + if (header != nullptr) header->owner_status |= kFdwicSubmitPmuOwnerRestored; + } else if (header != nullptr) { + header->owner_status |= kFdwicSubmitPmuOwnerAborted; + } + PublishOwnerProgress(header); + + if (!restored) { + LOG_ERROR( + "FDWIC submit-PMU owner restore incomplete: active=%u failures=%u", BitmapCount(g_owner.owned_bitmap), + g_owner.restore_failures + ); + return -1; + } + LOG_INFO_V0("FDWIC submit-PMU owner restored %u physical subcores", g_owner.restored_count); + return 0; +} diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/submit_pmu_owner.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/submit_pmu_owner.h new file mode 100644 index 0000000000..027f2aa1ed --- /dev/null +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicpu/submit_pmu_owner.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef FDWIC_DIST_ENGINE_AICPU_SUBMIT_PMU_OWNER_H_ +#define FDWIC_DIST_ENGINE_AICPU_SUBMIT_PMU_OWNER_H_ + +#include "dist_engine/common/submit_pmu_types.h" + +class Runtime; + +/** + * 判断本轮是否携带合法的 submit-PMU 共享头。 + * + * submit-pmu 与 perf-clock 一样复用 Runtime::dist.swimlane_base，但使用独立 + * magic/version/mode 判型，并要求普通泳道 level/capacity 都为零。返回 true + * 时，header_out 指向已经完成 AICPU cache invalidate 的共享头。 + */ +bool fdwic_submit_pmu_requested(Runtime *runtime, FdwicSubmitPmuHeader **header_out); + +/** + * 在真实 AICore 握手完成后，为本轮 96 个活跃物理子核取得 PMU 所有权。 + * + * 函数先完整校验 32 AIC + 64 AIV 拓扑，再逐物理核保存、配置、读回。 + * 任一配置失败都会逆序回滚已经取得所有权的槽；返回非零时不得放行业务 + * AICore 进入 dist_core_main。 + */ +int fdwic_submit_pmu_owner_configure(Runtime *runtime, FdwicSubmitPmuHeader *header); + +/** + * 恢复 owner 尚未释放的全部物理子核。 + * + * 恢复以 AICPU 私有 ownership bitmap 为准，按物理 id 107 -> 0 逆序执行； + * 只有 16 个配置寄存器全部读回原值后才释放对应 bit。失败 bit 会保留， + * 因而调用方可以幂等重试。 + */ +int fdwic_submit_pmu_owner_restore(FdwicSubmitPmuHeader *header); + +#endif // FDWIC_DIST_ENGINE_AICPU_SUBMIT_PMU_OWNER_H_ diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/atomic.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/atomic.h index a67d1d4352..515e7c227b 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/atomic.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/atomic.h @@ -84,4 +84,25 @@ PTO_DEVICE_FUNC inline T atomic_fetch_max(__gm__ volatile T &value, T desired, i #endif } +// Compare `value` with expected and replace it with desired on equality. +// Returns the value observed before the operation, matching A5 atomicCAS. +template +PTO_DEVICE_FUNC inline T atomic_compare_exchange( + __gm__ volatile T &value, T expected, T desired, int success_memorder = __ATOMIC_ACQ_REL, + int failure_memorder = __ATOMIC_ACQUIRE +) { +#if defined(__CCE_AICORE__) + (void)success_memorder; + (void)failure_memorder; + __gm__ T *addr = const_cast<__gm__ T *>(&value); + return atomicCAS(addr, expected, desired); +#else + T observed = expected; + (void)__atomic_compare_exchange_n( + &value, &observed, desired, /*weak=*/false, success_memorder, failure_memorder + ); + return observed; +#endif +} + } // namespace diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/debug_dump.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/debug_dump.h index 3fafd61fcf..01faf0c91b 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/debug_dump.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/debug_dump.h @@ -19,9 +19,11 @@ void dist_dump_state(int) { fprintf(stderr, "\n===== DIST STATE DUMP =====\n"); fprintf( - stderr, "frontier=%ld H=%d ring=%zuB replay_done=%ld/%d num_blocks=%d fatal=%d\n", + stderr, "frontier=%ld H=%d ring=%zuB final_root=%ld/%d release=%ld num_blocks=%d fatal=%d\n", static_cast(atomic_load(g_dist.frontier, __ATOMIC_RELAXED)), g_dist.H, g_dist.heap_size, - static_cast(atomic_load(g_dist.replay_done, __ATOMIC_RELAXED)), g_dist.num_workers, g_dist.num_blocks, + static_cast(atomic_load(g_dist.final_barrier.root_arrival.v, __ATOMIC_RELAXED)), + g_dist.final_barrier.root_arrival.expected, + static_cast(atomic_load(g_dist.final_barrier.root_release.v, __ATOMIC_RELAXED)), g_dist.num_blocks, atomic_load(g_dist.fatal, __ATOMIC_RELAXED) ); fprintf(stderr, "cube_cursor[%d]=", kCursorShards); diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/perf_clock.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/perf_clock.h new file mode 100644 index 0000000000..7b0ae1d1d6 --- /dev/null +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/perf_clock.h @@ -0,0 +1,145 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include "dist_engine/common/target.h" +#include "dist_engine/common/swimlane_types.h" +#include "dist_engine/common/worker_state.h" +#include "dist_engine/aicore/primitive.h" + +namespace { + +#if PTO_FDWIC_PERF_CLOCK + +PTO_DEVICE_FUNC inline void fdwic_perf_clock_attach(__gm__ Runtime *runtime, __gm__ DistCore *self) { + g_fdwic_perf_clock_core = nullptr; + g_fdwic_perf_clock_first_submit = 0; + g_fdwic_perf_clock_last_submit = 0; + g_fdwic_perf_clock_expected_submits = 0; +#if PTO_FDWIC_PERF_CLOCK_KERNEL + g_fdwic_perf_clock_kernel_ticks = 0; + g_fdwic_perf_clock_kernel_calls = 0; + g_fdwic_perf_clock_kernel_status = 0; +#endif + if (runtime == nullptr || self == nullptr) return; +#if defined(__CCE_AICORE__) + dist_aicore_invalidate_region(const_cast<__gm__ uint64_t *>(&runtime->dist.swimlane_base), 64); +#endif + const uint64_t base = runtime->dist.swimlane_base; + if (base == 0 || self->core_idx < 0 || self->core_idx >= runtime->dist.num_workers) return; + __gm__ auto *header = reinterpret_cast<__gm__ FdwicSwimlaneHeader *>(base); + g_fdwic_perf_clock_core = &header->cores[self->core_idx]; +} + +PTO_DEVICE_FUNC inline void fdwic_perf_clock_expect_submits(uint32_t expected_submits) { + // 该接口在首个 Submit 之前由 PA orchestration 调用一次。最终 host 会用 + // DistCore::local_index 与 expected/final_seen 做闭合校验，不静默修正。 + g_fdwic_perf_clock_expected_submits = expected_submits; +} + +PTO_DEVICE_FUNC inline void fdwic_perf_clock_submit_begin(int32_t task_id) { + // task_id 本来就是本核严格递增的 Submit 序号；直接复用，避免另做一份 + // 1280 次 block-local increment/store。 + if (task_id == 0) { + g_fdwic_perf_clock_first_submit = get_sys_cnt_aicore(); + } +} + +PTO_DEVICE_FUNC inline void fdwic_perf_clock_submit_end(int32_t task_id) { + if (task_id >= 0 && static_cast(task_id + 1) == g_fdwic_perf_clock_expected_submits && + g_fdwic_perf_clock_expected_submits != 0) { + g_fdwic_perf_clock_last_submit = get_sys_cnt_aicore(); + } +} + +#if PTO_FDWIC_PERF_CLOCK_KERNEL + +constexpr uint32_t kFdwicPerfClockKernelTickOrderError = 1U << 0; +constexpr uint32_t kFdwicPerfClockKernelTickOverflow = 1U << 1; +constexpr uint32_t kFdwicPerfClockKernelCallOverflow = 1U << 2; + +// 只在本核首个 Submit 已进入、末个 Submit 尚未返回时打开 Kernel 子窗。 +// 因此最后一个 Submit 之后的 FinalDrain Kernel 不会混入逐核整数闭合。 +PTO_DEVICE_FUNC inline uint64_t fdwic_perf_clock_kernel_begin() { + if (g_fdwic_perf_clock_first_submit == 0 || g_fdwic_perf_clock_last_submit != 0 || + g_fdwic_perf_clock_expected_submits == 0 || g_fdwic_perf_clock_kernel_status != 0) { + return 0; + } + return get_sys_cnt_aicore(); +} + +PTO_DEVICE_FUNC inline void fdwic_perf_clock_kernel_end(uint64_t begin_tick) { + if (begin_tick == 0) return; + const uint64_t end_tick = get_sys_cnt_aicore(); + if (end_tick < begin_tick) { + g_fdwic_perf_clock_kernel_status |= kFdwicPerfClockKernelTickOrderError; + return; + } + const uint64_t delta = end_tick - begin_tick; + if (g_fdwic_perf_clock_kernel_ticks > UINT64_MAX - delta) { + g_fdwic_perf_clock_kernel_status |= kFdwicPerfClockKernelTickOverflow; + return; + } + if (g_fdwic_perf_clock_kernel_calls == UINT32_MAX) { + g_fdwic_perf_clock_kernel_status |= kFdwicPerfClockKernelCallOverflow; + return; + } + g_fdwic_perf_clock_kernel_ticks += delta; + ++g_fdwic_perf_clock_kernel_calls; +} + +#endif + +PTO_DEVICE_FUNC inline void fdwic_perf_clock_flush(__gm__ DistCore *self) { + __gm__ FdwicSwimlaneCoreState *core = g_fdwic_perf_clock_core; + if (core == nullptr || self == nullptr) return; + core->core_idx = self->core_idx; + core->block_id = self->block_id; + core->lane = self->lane; +#if PTO_FDWIC_PERF_CLOCK_KERNEL + core->count = g_fdwic_perf_clock_kernel_calls; + core->dropped = g_fdwic_perf_clock_kernel_status; + core->atomic_calls = 0; + core->poll_calls = 0; + core->poll_batch_records = kFdwicPerfClockKernelMode; + core->perf_clock_kernel.first_submit_start = g_fdwic_perf_clock_first_submit; + core->perf_clock_kernel.last_submit_end = g_fdwic_perf_clock_last_submit; + core->perf_clock_kernel.submit_count = static_cast(self->local_index); + core->perf_clock_kernel.expected_submit_count = g_fdwic_perf_clock_expected_submits; + core->perf_clock_kernel.kernel_elapsed_ticks = g_fdwic_perf_clock_kernel_ticks; +#else + core->count = 0; + core->dropped = 0; + core->atomic_calls = 0; + core->poll_calls = 0; + core->poll_batch_records = 0; + core->perf_clock.first_submit_start = g_fdwic_perf_clock_first_submit; + core->perf_clock.last_submit_end = g_fdwic_perf_clock_last_submit; + core->perf_clock.submit_count = static_cast(self->local_index); + core->perf_clock.expected_submit_count = g_fdwic_perf_clock_expected_submits; + core->perf_clock.mode = kFdwicPerfClockMode; + core->perf_clock.final_seen = g_fdwic_perf_clock_last_submit != 0 ? 1U : 0U; +#endif + dist_aicore_flush_region(core, sizeof(FdwicSwimlaneCoreState)); +} + +#else + +PTO_DEVICE_FUNC inline void fdwic_perf_clock_attach(__gm__ Runtime *, __gm__ DistCore *) {} +PTO_DEVICE_FUNC inline void fdwic_perf_clock_expect_submits(uint32_t) {} +PTO_DEVICE_FUNC inline void fdwic_perf_clock_submit_begin(int32_t) {} +PTO_DEVICE_FUNC inline void fdwic_perf_clock_submit_end(int32_t) {} +PTO_DEVICE_FUNC inline void fdwic_perf_clock_flush(__gm__ DistCore *) {} + +#endif + +} // namespace diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/runtime_state.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/runtime_state.h index b838c0ad2a..275fed3fb3 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/runtime_state.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/runtime_state.h @@ -14,6 +14,7 @@ #include "dist_engine/common/atomic.h" #include "dist_engine/common/state.h" #include "dist_engine/common/worker_state.h" +#include "pto_runtime_status.h" #if DIST_SIM_HOST_CLOCK #include @@ -40,6 +41,17 @@ inline uint64_t dist_now_ns() { PTO_DEVICE_FUNC inline bool fatal_set() { return atomic_load(g_dist.fatal) != 0; } PTO_DEVICE_FUNC inline void set_fatal() { atomic_exchange(g_dist.fatal, 1); } +// 运行时错误使用“首个非零错误码获胜”的合同。先发布错误码，再发布 fatal， +// AICPU 在观察 fatal 后即可取得与本次失败对应的稳定 code。 +PTO_DEVICE_FUNC inline void set_fatal_code(int32_t code) { + if (code == PTO2_ERROR_NONE) code = PTO2_ERROR_EXPLICIT_ORCH_FATAL; + (void)atomic_compare_exchange( + g_dist.error_code, int32_t{PTO2_ERROR_NONE}, code, __ATOMIC_RELEASE, __ATOMIC_RELAXED + ); + store_barrier(); + (void)atomic_exchange(g_dist.fatal, int32_t{1}, __ATOMIC_RELEASE); +} + PTO_DEVICE_FUNC inline void watchdog([[maybe_unused]] uint64_t &start_ns) { #if DIST_SIM_HOST_CLOCK static const long budget_s = []() -> long { diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/state.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/state.h index b6734ef7b7..40b20f56b5 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/state.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/state.h @@ -18,6 +18,7 @@ #include "dist_engine/dist_engine.h" #include "common/core_type.h" +#include "fdwic_build_identity.h" #include "pto2_dispatch_payload.h" #include "pto_constants.h" #include "pto_submit_types.h" @@ -37,10 +38,26 @@ constexpr int32_t kPrivateSlots = 4; constexpr int32_t kWonReserve = 2; constexpr int32_t kMaxFanin = 16; constexpr int32_t kMapCap = 16384; +constexpr uint32_t kMapBucketCapacity = kFdwicTensorMapRingCap; +constexpr uint32_t kMapBuckets = kFdwicTensorMapRingBuckets; +constexpr uint32_t kMapBaseControlBuckets = 128; +constexpr uint32_t kMapBucketSlotMask = kMapBucketCapacity - 1; +constexpr uint32_t kMapBucketMask = kMapBuckets - 1; +constexpr size_t kMapControlBytes = 32768; constexpr int32_t kFlagCap = 1 << 16; constexpr int32_t kTaskPayloadSlots = 2048; constexpr int32_t kTaskPayloadMask = kTaskPayloadSlots - 1; static_assert((kTaskPayloadSlots & kTaskPayloadMask) == 0, "task payload slots must be a power of two"); +static_assert(kMapBucketCapacity * kMapBuckets == static_cast(kMapCap)); +static_assert((kMapBucketCapacity & kMapBucketSlotMask) == 0, "TensorMap bucket capacity must be a power of two"); +static_assert((kMapBuckets & kMapBucketMask) == 0, "TensorMap bucket count must be a power of two"); +static_assert(kMapBuckets <= 512, "TensorMap control area only reserves up to 512 buckets"); + +constexpr uint32_t dist_constexpr_log2(uint32_t value) { + return value <= 1U ? 0U : 1U + dist_constexpr_log2(value >> 1U); +} + +constexpr uint32_t kMapBucketShift = dist_constexpr_log2(kMapBuckets); struct DistTaskPayload { Tensor tensors[MAX_TENSOR_ARGS]; @@ -61,26 +78,69 @@ struct MapEntry { uint64_t lo; uint64_t hi; int32_t producer; - int32_t bucket; - int32_t next_in_bucket; - int32_t prev_in_bucket; - int32_t next_in_task; + uint32_t payload_abi_reserved; + // private ring 的 bucket/slot 由连续下标隐式给出，不再保存链指针。 + // 末 16B 为后续布局演进预留；shared 发布协议不会借用 private 热槽。 + uint8_t abi_reserved[16]; }; +static_assert(sizeof(MapEntry) == 48, "FDWIC MapEntry ABI size changed"); +static_assert(alignof(MapEntry) == 8, "FDWIC MapEntry ABI alignment changed"); +static_assert(offsetof(MapEntry, producer) == 24, "FDWIC MapEntry producer offset changed"); +static_assert(offsetof(MapEntry, abi_reserved) == 32, "FDWIC MapEntry reserve offset changed"); -constexpr int32_t kMapBuckets = 1 << 13; -constexpr int32_t kMapBucketShift = 13; constexpr int32_t kTaskWindow = 1 << 10; constexpr int32_t kTaskWindowMask = kTaskWindow - 1; struct DistTensorMap { MapEntry entries[kMapCap]; - int32_t buckets[kMapBuckets]; - int32_t task_heads[kTaskWindow]; - int32_t free_head; - int32_t high_water; + // 默认 CAP=128 时，前 128 个 head/tail 保持连续固定位置。CAP=32/64 + // 的额外桶游标从原 32KiB bucket 区域内部切出，所有模式的 map 总尺寸 + // 和 DistCore 后续字段偏移保持不动。 + uint64_t bucket_heads[kMapBaseControlBuckets]; + uint64_t bucket_tails[kMapBaseControlBuckets]; +#if PTO_FDWIC_TENSORMAP_RING_CAP < 128 + uint64_t extra_bucket_heads[kMapBuckets - kMapBaseControlBuckets]; + uint64_t extra_bucket_tails[kMapBuckets - kMapBaseControlBuckets]; + uint8_t control_abi_reserved[kMapControlBytes - 2 * sizeof(uint64_t) * kMapBuckets]; +#else + uint8_t control_abi_reserved[ + kMapControlBytes - 2 * sizeof(uint64_t) * kMapBaseControlBuckets + ]; +#endif + // 旧 task-head/free-list 区只保留物理 ABI，不在默认热路径维护全局 live + // 计数。每桶容量由 tail-head 当场判断，auto CAP 由后续静态 planner 负责。 + uint8_t task_window_abi_reserved[sizeof(int32_t) * kTaskWindow]; + uint32_t tail_abi_reserved0; + uint32_t tail_abi_reserved1; int32_t alive_floor; - int32_t cleaned_upto; + int32_t tail_abi_reserved2; }; +static_assert(sizeof(DistTensorMap) == 823312, "FDWIC TensorMap must preserve the DistCore ABI"); +static_assert(alignof(DistTensorMap) == 8, "FDWIC TensorMap alignment changed"); +static_assert(offsetof(DistTensorMap, bucket_heads) == 786432, "FDWIC TensorMap head offset changed"); +static_assert(offsetof(DistTensorMap, bucket_tails) == 787456, "FDWIC TensorMap tail offset changed"); +#if PTO_FDWIC_TENSORMAP_RING_CAP < 128 +static_assert( + offsetof(DistTensorMap, extra_bucket_heads) == 788480, "FDWIC TensorMap extra-head offset changed" +); +static_assert( + offsetof(DistTensorMap, extra_bucket_tails) == + 788480 + sizeof(uint64_t) * (kMapBuckets - kMapBaseControlBuckets), + "FDWIC TensorMap extra-tail offset changed" +); +#endif +static_assert( + offsetof(DistTensorMap, task_window_abi_reserved) == 819200, + "FDWIC TensorMap task-window reserve offset changed" +); +static_assert( + offsetof(DistTensorMap, control_abi_reserved) + sizeof(DistTensorMap::control_abi_reserved) == 819200, + "FDWIC TensorMap control area size changed" +); +static_assert(offsetof(DistTensorMap, tail_abi_reserved0) == 823296, "FDWIC TensorMap tail offset changed"); +static_assert(offsetof(DistTensorMap, tail_abi_reserved1) == 823300, "FDWIC TensorMap tail offset changed"); +static_assert(offsetof(DistTensorMap, alive_floor) == 823304, "FDWIC TensorMap alive-floor offset changed"); +static_assert(offsetof(DistTensorMap, tail_abi_reserved2) == 823308, "FDWIC TensorMap tail offset changed"); enum class TracePhase : int32_t { Kernel = 0, @@ -97,6 +157,13 @@ enum class TracePhase : int32_t { Claim = 11, Fanin = 12, Register = 13, + Atomic = 14, + ClockBaseline = 15, + OrchestrationReplay = 16, + FinalDrain = 17, + WinnerBuild = 18, + AllocComplete = 19, + LoserReplay = 20, }; struct RingSlot { @@ -218,10 +285,24 @@ struct DistCore { uint8_t task_payloads_pad[16]; DistTaskPayload task_payloads[kTaskPayloadSlots]; }; +static_assert(offsetof(DistCore, map) == 32, "FDWIC DistCore TensorMap offset changed"); +static_assert(offsetof(DistCore, slots_pad) == 823344, "FDWIC DistCore slot padding offset changed"); +static_assert(offsetof(DistCore, slots) == 823360, "FDWIC DistCore ring-slot offset changed"); +static_assert(offsetof(DistCore, occupied_count) == 842656, "FDWIC DistCore occupancy offset changed"); +static_assert(offsetof(DistCore, owned_total) == 842660, "FDWIC DistCore owned-total offset changed"); +static_assert( + offsetof(DistCore, swimlane_last_cycle) == 842664, "FDWIC DistCore swimlane-clock offset changed" +); +static_assert( + offsetof(DistCore, task_payloads_pad) == 842672, "FDWIC DistCore payload padding offset changed" +); +static_assert(offsetof(DistCore, task_payloads) == 842688, "FDWIC DistCore task-payload offset changed"); +static_assert(sizeof(DistCore) == 9231296, "FDWIC DistCore ABI size changed"); static_assert(offsetof(DistCore, slots) % 64 == 0, "DistCore slots must be cacheline-aligned"); static_assert(offsetof(DistCore, task_payloads) % 64 == 0, "DistCore task_payloads must be cacheline-aligned"); constexpr int32_t kCursorShards = 4; +constexpr int32_t kFinalBarrierGroups = 16; constexpr size_t kCacheLine = 64; static_assert(PTO2_PACKED_OUTPUT_ALIGN >= kCacheLine); static_assert((PTO2_PACKED_OUTPUT_ALIGN % kCacheLine) == 0); @@ -232,6 +313,97 @@ struct PaddedCursor { }; static_assert(sizeof(PaddedCursor) == kCacheLine, "PaddedCursor must occupy one cacheline"); +// shared TensorMap 与 private 共用 bucket/CAP/hash/逻辑 region 语义，但不 +// 借用 private MapEntry 的 16B ABI reserve。专属 32B value 的 reserved +// 必须由 writer 写 0、reader 校验 0，协议字段边界与 standalone 一致。 +struct SharedTensorMapValue { + uint64_t buf_addr; + uint64_t lo; + uint64_t hi; + int32_t producer; + uint32_t reserved; +}; +static_assert(sizeof(SharedTensorMapValue) == 32, "shared TensorMap logical value size changed"); +static_assert(offsetof(SharedTensorMapValue, buf_addr) == 0, "shared TensorMap buffer offset changed"); +static_assert(offsetof(SharedTensorMapValue, lo) == 8, "shared TensorMap lower-bound offset changed"); +static_assert(offsetof(SharedTensorMapValue, hi) == 16, "shared TensorMap upper-bound offset changed"); +static_assert(offsetof(SharedTensorMapValue, producer) == 24, "shared TensorMap producer offset changed"); +static_assert(offsetof(SharedTensorMapValue, reserved) == 28, "shared TensorMap reserve offset changed"); + +// payload 与发布 seq 必须分处独占 cache line。A5 的 atomic seq 访问和 +// 普通 payload cache writeback 若落在同一行，可能互相覆盖；两行分离后， +// writer 可按“payload flush -> seq publish”建立明确的跨核可见性边界。 +struct alignas(kCacheLine) SharedTensorMapPayloadLine { + SharedTensorMapValue value; + uint8_t pad[kCacheLine - sizeof(SharedTensorMapValue)]; +}; +static_assert(sizeof(SharedTensorMapPayloadLine) == kCacheLine, "shared TensorMap payload must occupy one cacheline"); +static_assert(offsetof(SharedTensorMapPayloadLine, value) == 0, "shared TensorMap payload value offset changed"); + +struct alignas(kCacheLine) SharedTensorMapSequenceLine { + volatile int64_t v; + uint8_t pad[kCacheLine - sizeof(int64_t)]; +}; +static_assert(offsetof(SharedTensorMapSequenceLine, v) == 0, "shared TensorMap sequence value offset changed"); +static_assert(sizeof(SharedTensorMapSequenceLine) == kCacheLine, "shared TensorMap sequence must occupy one cacheline"); + +struct alignas(kCacheLine) SharedTensorMapSlot { + SharedTensorMapPayloadLine payload; + SharedTensorMapSequenceLine sequence; +}; +static_assert(sizeof(SharedTensorMapSlot) == 2 * kCacheLine, "shared TensorMap slot size changed"); +static_assert( + offsetof(SharedTensorMapSlot, sequence) == kCacheLine, + "shared TensorMap sequence must not share the payload cacheline" +); + +constexpr int64_t kSharedTensorMapInvalidSequence = -1; +// writer 必须先用 CAS 从旧 lap seq 取得 WRITING 所有权，再修改 payload。 +// 不能复用 -1：首圈 expected_old 本来就是 -1，CAS(-1,-1) 无法排除非法 +// 双 writer。有效 absolute cursor 恒非负，因此 INT64_MIN 可作为独立哨兵。 +constexpr int64_t kSharedTensorMapWritingSequence = (-9223372036854775807LL - 1); +constexpr int64_t kSharedTensorMapInitialCommit = 0; +constexpr int64_t kSharedTensorMapInitialReclaim = -1; + +struct alignas(kCacheLine) SharedTensorMapBucketState { + PaddedCursor head; + PaddedCursor tail; +}; +static_assert(sizeof(SharedTensorMapBucketState) == 2 * kCacheLine, "shared TensorMap bucket controls changed"); +static_assert( + offsetof(SharedTensorMapBucketState, tail) == kCacheLine, + "shared TensorMap head and tail must not share a cacheline" +); + +struct alignas(kCacheLine) SharedTensorMapState { + // committed_tasks 是下一个允许发布的 task id；即使任务没有 region， + // ordered commit 也必须从 N 推进到 N+1。 + PaddedCursor committed_tasks; + // 已可回收的最大 producer id，初值 -1。只有 exact-turn winner 会 + // 访问 map，因此在完成 task N lookup 后可直接用 N-H-1 单调推进； + // loser 不读 map，也不需要 per-core progress。 + PaddedCursor reclaim_upto; + // 每桶 head/tail 各占一行且彼此相邻。第一版采用 task-id 有序单 + // 追加者，因而不需要 MPSC reserve 游标或全局 free-list。 + SharedTensorMapBucketState buckets[kMapBuckets]; + // 与 private 完全相同的连续分桶下标： + // bucket * CAP + (absolute_cursor & (CAP - 1))。 + SharedTensorMapSlot slots[kMapCap]; +}; +static_assert(offsetof(SharedTensorMapState, committed_tasks) == 0); +static_assert(offsetof(SharedTensorMapState, reclaim_upto) == kCacheLine); +static_assert(offsetof(SharedTensorMapState, buckets) == 2 * kCacheLine); +static_assert( + offsetof(SharedTensorMapState, slots) == 2 * kCacheLine + sizeof(SharedTensorMapBucketState) * kMapBuckets, + "shared TensorMap slots must immediately follow bucket controls" +); +static_assert(sizeof(SharedTensorMapState) % kCacheLine == 0); +#if PTO_FDWIC_TENSORMAP_RING_CAP == 128 +static_assert(offsetof(SharedTensorMapState, buckets) == 128); +static_assert(offsetof(SharedTensorMapState, slots) == 16512); +static_assert(sizeof(SharedTensorMapState) == 2113664); +#endif + struct DistTaskCell { volatile int64_t flag; volatile uint64_t vend; @@ -239,6 +411,27 @@ struct DistTaskCell { }; static_assert(sizeof(DistTaskCell) == kCacheLine); +struct alignas(kCacheLine) FinalBarrierArrival { + volatile int64_t v; + volatile int32_t expected; + uint8_t pad[kCacheLine - sizeof(int64_t) - sizeof(int32_t)]; +}; +static_assert(sizeof(FinalBarrierArrival) == kCacheLine, "final barrier arrival must occupy one cacheline"); + +struct alignas(kCacheLine) FinalBarrierRelease { + volatile int64_t v; + uint8_t pad[kCacheLine - sizeof(int64_t)]; +}; +static_assert(sizeof(FinalBarrierRelease) == kCacheLine, "final barrier release must occupy one cacheline"); + +struct alignas(kCacheLine) FinalBarrierState { + FinalBarrierArrival leaf_arrivals[kFinalBarrierGroups]; + FinalBarrierRelease leaf_releases[kFinalBarrierGroups]; + FinalBarrierArrival root_arrival; + FinalBarrierRelease root_release; +}; +static_assert(sizeof(FinalBarrierState) == 34 * kCacheLine, "final barrier state size changed"); + struct DistGlobal { PaddedCursor cube_cursor[kCursorShards]; PaddedCursor vector_cursor[kCursorShards]; @@ -259,7 +452,10 @@ struct DistGlobal { uint8_t fatal_pad[24]; volatile int32_t fatal; - uint8_t fatal_tail_pad[kCacheLine - sizeof(int32_t)]; + // 首个非零运行时错误码获胜；fatal 仍保留原 offset，后续字段也不移动。 + // AICPU 在所有 worker 完成后失效并读取整条 cache line。 + volatile int32_t error_code; + uint8_t fatal_tail_pad[kCacheLine - 2 * sizeof(int32_t)]; int32_t num_workers; int32_t num_blocks; @@ -267,6 +463,8 @@ struct DistGlobal { uint8_t blocks_pad[24]; BlockWon blocks[kDistRuntimeMaxWorker]; + // Retained in place for the existing DistGlobal hot-field ABI. Final + // completion now uses final_barrier after cores instead of this flat line. volatile int64_t replay_done; uint8_t replay_done_pad[kCacheLine - sizeof(int64_t)]; @@ -274,15 +472,52 @@ struct DistGlobal { uint8_t started_count_pad[kCacheLine - sizeof(int64_t)]; DistCore cores[kDistRuntimeMaxWorker]; + + // Keep all existing hot-field and DistCore offsets stable. Only the tail + // grows for the fixed two-level G=16 final barrier. + FinalBarrierState final_barrier; +#if PTO_FDWIC_SHARED_MAP + // shared 专属 sidecar 只追加在旧 DistGlobal 尾部；private artifact 不 + // 实例化这 2MiB 状态，且所有旧热字段与 per-core map offset 均不移动。 + SharedTensorMapState shared_tensor_map; +#endif }; static_assert(offsetof(DistGlobal, frontier) % 64 == 0, "DistGlobal frontier must be cacheline-aligned"); static_assert(offsetof(DistGlobal, tasks) % 64 == 0, "DistGlobal tasks must be cacheline-aligned"); static_assert(offsetof(DistGlobal, fatal) % 64 == 0, "DistGlobal fatal must be cacheline-aligned"); +static_assert( + offsetof(DistGlobal, error_code) == offsetof(DistGlobal, fatal) + sizeof(int32_t), + "DistGlobal runtime error must share the fatal cacheline" +); static_assert(offsetof(DistGlobal, blocks) % 64 == 0, "DistGlobal blocks must be cacheline-aligned"); static_assert(offsetof(DistGlobal, replay_done) % 64 == 0, "DistGlobal replay_done must be cacheline-aligned"); static_assert(offsetof(DistGlobal, started_count) % 64 == 0, "DistGlobal started_count must be cacheline-aligned"); static_assert(offsetof(DistGlobal, cores) % 64 == 0, "DistGlobal cores must be cacheline-aligned"); +static_assert( + offsetof(DistGlobal, final_barrier) % 64 == 0, "DistGlobal final barrier must be cacheline-aligned" +); +// 68f51451 已冻结的 private DistGlobal 尾边界。shared sidecar 只能从该 +// offset 追加，不能把 mode-specific 字段插进旧热布局。 +constexpr size_t kFdwicSharedTensorMapOffset = 1007026048; +static_assert( + offsetof(DistGlobal, final_barrier) + sizeof(FinalBarrierState) == kFdwicSharedTensorMapOffset, + "FDWIC legacy DistGlobal tail moved" +); +#if PTO_FDWIC_SHARED_MAP +static_assert( + offsetof(DistGlobal, shared_tensor_map) == kFdwicSharedTensorMapOffset, + "shared TensorMap sidecar must append after the frozen DistGlobal tail" +); +static_assert( + sizeof(DistGlobal) == kFdwicSharedTensorMapOffset + sizeof(SharedTensorMapState), + "shared DistGlobal may only grow by its TensorMap sidecar" +); +#else +static_assert( + sizeof(DistGlobal) == kFdwicSharedTensorMapOffset, "private DistGlobal size changed while adding shared sidecar" +); +#endif static_assert(sizeof(DistGlobal) <= kDistEngineGlobalStateSize, "DistGlobal exceeds the reserved runtime arena size"); static_assert( alignof(DistGlobal) <= kDistEngineGlobalStateAlign, "DistGlobal exceeds the reserved runtime arena align" diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/submit_pmu.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/submit_pmu.h new file mode 100644 index 0000000000..783c8454bc --- /dev/null +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/submit_pmu.h @@ -0,0 +1,822 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include "dist_engine/common/target.h" + +#if PTO_FDWIC_SUBMIT_PMU + +#include "aicore/aicore.h" +#include "aicore/fdwic_submit_pmu_state.h" +#include "common/platform_config.h" +#include "dist_engine/aicore/primitive.h" +#include "dist_engine/common/submit_pmu_types.h" +#include "dist_engine/common/worker_state.h" + +namespace { + +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 0 +constexpr FdwicSubmitPmuPhase kFdwicSubmitPmuCompiledPhase = FdwicSubmitPmuPhase::None; +constexpr uint16_t kFdwicSubmitPmuCompiledMode = kFdwicSubmitPmuModeNone; +constexpr size_t kFdwicSubmitPmuCompiledBytes = kFdwicSubmitPmuNoneBytes; +#elif PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 1 +constexpr FdwicSubmitPmuPhase kFdwicSubmitPmuCompiledPhase = FdwicSubmitPmuPhase::ArgBuild; +constexpr uint16_t kFdwicSubmitPmuCompiledMode = kFdwicSubmitPmuModeArgBuild; +constexpr size_t kFdwicSubmitPmuCompiledBytes = kFdwicSubmitPmuPhaseBytes; +#elif PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 2 +constexpr FdwicSubmitPmuPhase kFdwicSubmitPmuCompiledPhase = FdwicSubmitPmuPhase::EmptyBracket; +constexpr uint16_t kFdwicSubmitPmuCompiledMode = kFdwicSubmitPmuModeEmptyBracket; +constexpr size_t kFdwicSubmitPmuCompiledBytes = kFdwicSubmitPmuPhaseBytes; +#elif PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 3 +constexpr FdwicSubmitPmuPhase kFdwicSubmitPmuCompiledPhase = FdwicSubmitPmuPhase::Materialize; +constexpr uint16_t kFdwicSubmitPmuCompiledMode = kFdwicSubmitPmuModeMaterialize; +constexpr size_t kFdwicSubmitPmuCompiledBytes = kFdwicSubmitPmuPhaseBytes; +#elif PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 4 +constexpr FdwicSubmitPmuPhase kFdwicSubmitPmuCompiledPhase = FdwicSubmitPmuPhase::Claim; +constexpr uint16_t kFdwicSubmitPmuCompiledMode = kFdwicSubmitPmuModeClaim; +constexpr size_t kFdwicSubmitPmuCompiledBytes = kFdwicSubmitPmuPhaseBytes; +#elif PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 5 +constexpr FdwicSubmitPmuPhase kFdwicSubmitPmuCompiledPhase = FdwicSubmitPmuPhase::Register; +constexpr uint16_t kFdwicSubmitPmuCompiledMode = kFdwicSubmitPmuModeRegister; +constexpr size_t kFdwicSubmitPmuCompiledBytes = kFdwicSubmitPmuPhaseBytes; +#elif PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 6 +constexpr FdwicSubmitPmuPhase kFdwicSubmitPmuCompiledPhase = FdwicSubmitPmuPhase::SubmitTransition; +constexpr uint16_t kFdwicSubmitPmuCompiledMode = kFdwicSubmitPmuModeSubmitTransition; +constexpr size_t kFdwicSubmitPmuCompiledBytes = kFdwicSubmitPmuPhaseBytes; +#elif PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 7 +constexpr FdwicSubmitPmuPhase kFdwicSubmitPmuCompiledPhase = FdwicSubmitPmuPhase::EfDrainControl; +constexpr uint16_t kFdwicSubmitPmuCompiledMode = kFdwicSubmitPmuModeEfDrainControl; +constexpr size_t kFdwicSubmitPmuCompiledBytes = kFdwicSubmitPmuPhaseBytes; +#elif PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 8 +constexpr FdwicSubmitPmuPhase kFdwicSubmitPmuCompiledPhase = FdwicSubmitPmuPhase::PrepareMap; +constexpr uint16_t kFdwicSubmitPmuCompiledMode = kFdwicSubmitPmuModePrepareMap; +constexpr size_t kFdwicSubmitPmuCompiledBytes = kFdwicSubmitPmuPhaseBytes; +#elif PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 9 +constexpr FdwicSubmitPmuPhase kFdwicSubmitPmuCompiledPhase = FdwicSubmitPmuPhase::Fanin; +constexpr uint16_t kFdwicSubmitPmuCompiledMode = kFdwicSubmitPmuModeFanin; +constexpr size_t kFdwicSubmitPmuCompiledBytes = kFdwicSubmitPmuPhaseBytes; +#elif PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 10 +constexpr FdwicSubmitPmuPhase kFdwicSubmitPmuCompiledPhase = FdwicSubmitPmuPhase::WinnerBuild; +constexpr uint16_t kFdwicSubmitPmuCompiledMode = kFdwicSubmitPmuModeWinnerBuild; +constexpr size_t kFdwicSubmitPmuCompiledBytes = kFdwicSubmitPmuPhaseBytes; +#elif PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 11 +constexpr FdwicSubmitPmuPhase kFdwicSubmitPmuCompiledPhase = FdwicSubmitPmuPhase::AllocComplete; +constexpr uint16_t kFdwicSubmitPmuCompiledMode = kFdwicSubmitPmuModeAllocComplete; +constexpr size_t kFdwicSubmitPmuCompiledBytes = kFdwicSubmitPmuPhaseBytes; +#elif PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 12 +constexpr FdwicSubmitPmuPhase kFdwicSubmitPmuCompiledPhase = FdwicSubmitPmuPhase::LoserReplay; +constexpr uint16_t kFdwicSubmitPmuCompiledMode = kFdwicSubmitPmuModeLoserReplay; +constexpr size_t kFdwicSubmitPmuCompiledBytes = kFdwicSubmitPmuPhaseBytes; +#else +#error "invalid real FDWIC submit-PMU phase" +#endif + +template +PTO_DEVICE_FUNC inline uint32_t fdwic_submit_pmu_ld(uint64_t reg_base) { + int32_t *block = reinterpret_cast(reg_base + BlockOffset); + return static_cast(ld_dev(block, static_cast(RegisterOffset - BlockOffset))); +} + +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_clear_counters(uint64_t reg_base) { + constexpr uint32_t kCounterBlock = REG_MMIO_PMU_CTRL_0_OFFSET; + (void)fdwic_submit_pmu_ld(reg_base); + (void)fdwic_submit_pmu_ld(reg_base); + (void)fdwic_submit_pmu_ld(reg_base); + (void)fdwic_submit_pmu_ld(reg_base); + (void)fdwic_submit_pmu_ld(reg_base); + (void)fdwic_submit_pmu_ld(reg_base); + (void)fdwic_submit_pmu_ld(reg_base); + (void)fdwic_submit_pmu_ld(reg_base); + (void)fdwic_submit_pmu_ld(reg_base); + (void)fdwic_submit_pmu_ld(reg_base); + (void)fdwic_submit_pmu_ld(reg_base); + (void)fdwic_submit_pmu_ld(reg_base); +} + +PTO_DEVICE_FUNC inline uint32_t fdwic_submit_pmu_selector_status(uint64_t reg_base) { + constexpr uint32_t kSelectorBlock = REG_MMIO_PMU_CTRL_1_OFFSET; + uint32_t status = 0; + if (fdwic_submit_pmu_ld(reg_base) == kFdwicSubmitPmuCnt0VectorBusy) { + status |= kFdwicSubmitPmuCnt0SelectorValid; + } + if (fdwic_submit_pmu_ld(reg_base) == kFdwicSubmitPmuCnt1CubeBusy) { + status |= kFdwicSubmitPmuCnt1SelectorValid; + } + if (fdwic_submit_pmu_ld(reg_base) == kFdwicSubmitPmuCnt2ScalarBusy) { + status |= kFdwicSubmitPmuCnt2SelectorValid; + } + if (fdwic_submit_pmu_ld(reg_base) == + kFdwicSubmitPmuCnt3ShadowScalarBusy) { + status |= kFdwicSubmitPmuCnt3SelectorValid; + } + if (fdwic_submit_pmu_ld(reg_base) == + kFdwicSubmitPmuCnt5ShadowIcacheMiss) { + status |= kFdwicSubmitPmuCnt5SelectorValid; + } + if (fdwic_submit_pmu_ld(reg_base) == + kFdwicSubmitPmuCnt6IcacheRequest) { + status |= kFdwicSubmitPmuCnt6SelectorValid; + } + if (fdwic_submit_pmu_ld(reg_base) == kFdwicSubmitPmuCnt7IcacheMiss) { + status |= kFdwicSubmitPmuCnt7SelectorValid; + } + if (fdwic_submit_pmu_ld(reg_base) == + kFdwicSubmitPmuCnt8ShadowIcacheRequest) { + status |= kFdwicSubmitPmuCnt8SelectorValid; + } + return status; +} + +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 +struct FdwicSubmitPmuIcacheShadowSnapshot { + uint32_t requests; + uint32_t misses; +}; + +struct FdwicSubmitPmuTotalShadowSnapshot { + uint32_t low; + uint32_t high; +}; + +// 三类 shadow 都在 PMU gate 运行时 read-to-clear。没有 DSB/PIPE_ALL， +// 相邻寄存器也不是同一时刻的原子快照；因此 raw 明确使用 observed 命名。 +// 这些 noinline 符号同时是 phase ELF 门禁，none 中不得出现。 +PTO_DEVICE_FUNC __attribute__((noinline)) FdwicSubmitPmuIcacheShadowSnapshot +fdwic_submit_pmu_phase_read_shadow_counters() { + constexpr uint32_t kCounterBlock = REG_MMIO_PMU_CTRL_0_OFFSET; + return FdwicSubmitPmuIcacheShadowSnapshot{ + fdwic_submit_pmu_ld(g_fdwic_submit_pmu_reg_base), + fdwic_submit_pmu_ld(g_fdwic_submit_pmu_reg_base), + }; +} + +PTO_DEVICE_FUNC __attribute__((noinline)) uint32_t fdwic_submit_pmu_phase_read_scalar_shadow() { + constexpr uint32_t kCounterBlock = REG_MMIO_PMU_CTRL_0_OFFSET; + return fdwic_submit_pmu_ld(g_fdwic_submit_pmu_reg_base); +} + +PTO_DEVICE_FUNC __attribute__((noinline)) FdwicSubmitPmuTotalShadowSnapshot fdwic_submit_pmu_phase_read_total_shadow() { + constexpr uint32_t kCounterBlock = REG_MMIO_PMU_CTRL_0_OFFSET; + return FdwicSubmitPmuTotalShadowSnapshot{ + fdwic_submit_pmu_ld(g_fdwic_submit_pmu_reg_base), + fdwic_submit_pmu_ld(g_fdwic_submit_pmu_reg_base), + }; +} + +PTO_DEVICE_FUNC inline bool +fdwic_submit_pmu_add_total_shadow(const FdwicSubmitPmuTotalShadowSnapshot &sample, bool include_in_phase) { + FdwicSubmitPmuPhaseAccumulator &phase = g_fdwic_submit_pmu_phase; + // 当前约 5 ms 窗和更短的 phase chunk 不应跨越 32-bit。要求 high==0 + // 既规避 low/high 顺序读取的 rollover 歧义，也让运行中重建失败可见。 + if (sample.high != 0 || phase.shadow_total_cycles > UINT64_MAX - sample.low || + (include_in_phase && phase.phase_total_cycles > UINT64_MAX - sample.low)) { + phase.counter_error = true; + return false; + } + phase.shadow_total_cycles += sample.low; + if (include_in_phase) phase.phase_total_cycles += sample.low; + return true; +} + +PTO_DEVICE_FUNC inline bool fdwic_submit_pmu_add_scalar_shadow(uint32_t sample, bool include_in_phase) { + FdwicSubmitPmuPhaseAccumulator &phase = g_fdwic_submit_pmu_phase; + if (phase.shadow_scalar_busy > UINT64_MAX - sample || + (include_in_phase && phase.phase_scalar_busy > UINT64_MAX - sample)) { + phase.counter_error = true; + return false; + } + phase.shadow_scalar_busy += sample; + if (include_in_phase) phase.phase_scalar_busy += sample; + return true; +} + +PTO_DEVICE_FUNC inline bool +fdwic_submit_pmu_add_icache_shadow(const FdwicSubmitPmuIcacheShadowSnapshot &sample, bool include_in_phase) { + FdwicSubmitPmuPhaseAccumulator &phase = g_fdwic_submit_pmu_phase; + if (phase.shadow_requests > UINT64_MAX - sample.requests || phase.shadow_misses > UINT64_MAX - sample.misses) { + phase.counter_error = true; + return false; + } + phase.shadow_requests += sample.requests; + phase.shadow_misses += sample.misses; + if (include_in_phase) { + if (phase.phase_requests > UINT64_MAX - sample.requests || phase.phase_misses > UINT64_MAX - sample.misses) { + phase.counter_error = true; + return false; + } + phase.phase_requests += sample.requests; + phase.phase_misses += sample.misses; + } + return true; +} +#endif + +// noinline 是构建门禁的一部分：最终 ELF 必须能证明完整 Submit 计数读取 +// 存在，同时普通泳道/perf-clock ELF 必须不含该符号。 +PTO_DEVICE_FUNC __attribute__((noinline)) void fdwic_submit_pmu_read_counters() { + const uint64_t reg_base = g_fdwic_submit_pmu_reg_base; + if (reg_base == 0) return; + constexpr uint32_t kCounterBlock = REG_MMIO_PMU_CTRL_0_OFFSET; + const uint32_t vector_busy = fdwic_submit_pmu_ld(reg_base); + const uint32_t cube_busy = fdwic_submit_pmu_ld(reg_base); + if (vector_busy == 0) g_fdwic_submit_pmu_status |= kFdwicSubmitPmuVectorBusyZero; + if (cube_busy == 0) g_fdwic_submit_pmu_status |= kFdwicSubmitPmuCubeBusyZero; + g_fdwic_submit_pmu_scalar_busy = fdwic_submit_pmu_ld(reg_base); + g_fdwic_submit_pmu_icache_requests = fdwic_submit_pmu_ld(reg_base); + g_fdwic_submit_pmu_icache_misses = fdwic_submit_pmu_ld(reg_base); + // primary 必须先读且窗口中从不 read-clear。none 随后直接读取一次 + // shadow；phase 则在这里补最后一个 tail segment，软件重建完整 shadow。 +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 0 + const uint32_t shadow_scalar_busy = fdwic_submit_pmu_ld(reg_base); + const uint32_t shadow_requests = fdwic_submit_pmu_ld(reg_base); + const uint32_t shadow_misses = fdwic_submit_pmu_ld(reg_base); + if (shadow_requests == g_fdwic_submit_pmu_icache_requests && shadow_misses == g_fdwic_submit_pmu_icache_misses) { + g_fdwic_submit_pmu_status |= kFdwicSubmitPmuNoneIcacheShadowPrimaryMatch; + } + if (shadow_scalar_busy == g_fdwic_submit_pmu_scalar_busy) { + g_fdwic_submit_pmu_status |= kFdwicSubmitPmuNoneScalarShadowPrimaryMatch; + } + const uint64_t low = fdwic_submit_pmu_ld(reg_base); + const uint64_t high = fdwic_submit_pmu_ld(reg_base); + g_fdwic_submit_pmu_total_cycles = low | (high << 32); +#else + // gate 已停止；仍沿 phase end 的 I-cache -> scalar -> TOTAL 反序补尾。 + const FdwicSubmitPmuIcacheShadowSnapshot icache_tail = fdwic_submit_pmu_phase_read_shadow_counters(); + const uint32_t scalar_tail = fdwic_submit_pmu_phase_read_scalar_shadow(); + const FdwicSubmitPmuTotalShadowSnapshot total_tail = fdwic_submit_pmu_phase_read_total_shadow(); + FdwicSubmitPmuPhaseAccumulator &phase = g_fdwic_submit_pmu_phase; + if (fdwic_submit_pmu_add_icache_shadow(icache_tail, /*include_in_phase=*/false)) { + phase.status |= kFdwicSubmitPmuPhaseIcacheTailRead; + } else { + phase.boundary_error = true; + } + if (fdwic_submit_pmu_add_scalar_shadow(scalar_tail, /*include_in_phase=*/false)) { + phase.status |= kFdwicSubmitPmuPhaseScalarTailRead; + } else { + phase.boundary_error = true; + } + if (fdwic_submit_pmu_add_total_shadow(total_tail, /*include_in_phase=*/false)) { + phase.status |= kFdwicSubmitPmuPhaseTotalTailRead; + } else { + phase.boundary_error = true; + } + g_fdwic_submit_pmu_total_cycles = phase.shadow_total_cycles; +#endif +} + +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_reset_local() { + g_fdwic_submit_pmu_core = nullptr; + g_fdwic_submit_pmu_phase_core = nullptr; + g_fdwic_submit_pmu_reg_base = 0; + g_fdwic_submit_pmu_start_tick = 0; + g_fdwic_submit_pmu_end_tick = 0; + g_fdwic_submit_pmu_scalar_elapsed_ticks = 0; + g_fdwic_submit_pmu_scalar_segment_begin_tick = 0; + g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks = 0; + g_fdwic_submit_pmu_return_ready_atomic_begin_tick = 0; + g_fdwic_submit_pmu_total_cycles = 0; + g_fdwic_submit_pmu_scalar_busy = 0; + g_fdwic_submit_pmu_icache_requests = 0; + g_fdwic_submit_pmu_icache_misses = 0; + g_fdwic_submit_pmu_expected_submits = 0; + g_fdwic_submit_pmu_status = 0; + g_fdwic_submit_pmu_started = false; + g_fdwic_submit_pmu_stopped = false; + g_fdwic_submit_pmu_gate_running = false; + g_fdwic_submit_pmu_gate_error = false; + g_fdwic_submit_pmu_return_ready_atomic_active = false; + g_fdwic_submit_pmu_return_ready_atomic_phase_armed = false; + g_fdwic_submit_pmu_return_ready_atomic_seen = false; + g_fdwic_submit_pmu_return_ready_atomic_time_error = false; + g_fdwic_submit_pmu_phase.shadow_total_cycles = 0; + g_fdwic_submit_pmu_phase.phase_total_cycles = 0; + g_fdwic_submit_pmu_phase.shadow_scalar_busy = 0; + g_fdwic_submit_pmu_phase.phase_scalar_busy = 0; + g_fdwic_submit_pmu_phase.shadow_requests = 0; + g_fdwic_submit_pmu_phase.shadow_misses = 0; + g_fdwic_submit_pmu_phase.phase_requests = 0; + g_fdwic_submit_pmu_phase.phase_misses = 0; + g_fdwic_submit_pmu_phase.phase_elapsed_ticks = 0; + g_fdwic_submit_pmu_phase.phase_begin_tick = 0; + g_fdwic_submit_pmu_phase.phase_excluded_atomic_ticks = 0; + g_fdwic_submit_pmu_phase.begin_reads = 0; + g_fdwic_submit_pmu_phase.end_reads = 0; + g_fdwic_submit_pmu_phase.status = 0; + g_fdwic_submit_pmu_phase.armed = false; + g_fdwic_submit_pmu_phase.boundary_error = false; + g_fdwic_submit_pmu_phase.counter_error = false; +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 + g_fdwic_submit_pmu_excluded_kernel_calls = 0; +#endif +} + +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_attach(__gm__ Runtime *runtime, __gm__ DistCore *self) { + fdwic_submit_pmu_reset_local(); + if (runtime == nullptr || self == nullptr) return; + g_fdwic_submit_pmu_status = kFdwicSubmitPmuRequested; + dist_aicore_invalidate_region(const_cast<__gm__ uint64_t *>(&runtime->dist.swimlane_base), 64); + const uint64_t base = runtime->dist.swimlane_base; + if (base == 0 || self->core_idx < 0 || self->core_idx >= static_cast(kFdwicSubmitPmuExpectedCores)) { + return; + } + __gm__ auto *header = reinterpret_cast<__gm__ FdwicSubmitPmuHeader *>(base); + dist_aicore_invalidate_region(header, 128); + if (header->magic != kFdwicSubmitPmuMagic || header->version != kFdwicSubmitPmuVersion || + header->mode != kFdwicSubmitPmuCompiledMode || header->header_bytes != kFdwicSubmitPmuCompiledBytes || + header->record_bytes != sizeof(FdwicSubmitPmuCoreData) || header->num_cores != kFdwicSubmitPmuExpectedCores) { + return; + } + g_fdwic_submit_pmu_core = &header->cores[self->core_idx]; +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 + g_fdwic_submit_pmu_phase_core = + &reinterpret_cast<__gm__ FdwicSubmitPmuPhaseCoreData *>(base + sizeof(FdwicSubmitPmuHeader))[self->core_idx]; + g_fdwic_submit_pmu_phase.status = kFdwicSubmitPmuPhaseRequested; +#endif + g_fdwic_submit_pmu_reg_base = get_fdwic_submit_pmu_reg_base(); + if (g_fdwic_submit_pmu_reg_base == 0) return; + g_fdwic_submit_pmu_status |= kFdwicSubmitPmuRegMapped; + const uint32_t physical_core_id = get_physical_core_id(); + if (physical_core_id >= kFdwicSubmitPmuPhysicalSubcores) return; + g_fdwic_submit_pmu_status |= kFdwicSubmitPmuPhysicalIdValid; + + // Owner 在唤醒 worker 前已经完成 MMIO 配置。先冻结本核 AICore CTRL gate， + // 再验证 selector 并读清 owner 配置到首个 Submit 之间的冷路径计数。 + bisheng::cce::metrics_prof_stop(); + g_fdwic_submit_pmu_status |= fdwic_submit_pmu_selector_status(g_fdwic_submit_pmu_reg_base); + fdwic_submit_pmu_clear_counters(g_fdwic_submit_pmu_reg_base); +} + +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_expect_submits(uint32_t expected_submits) { + g_fdwic_submit_pmu_expected_submits = expected_submits; +} + +template +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_phase_begin() { +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 + if constexpr (Phase == kFdwicSubmitPmuCompiledPhase) { + FdwicSubmitPmuPhaseAccumulator &phase = g_fdwic_submit_pmu_phase; + if (!g_fdwic_submit_pmu_started || g_fdwic_submit_pmu_stopped || !g_fdwic_submit_pmu_gate_running || + g_fdwic_submit_pmu_phase_core == nullptr || g_fdwic_submit_pmu_reg_base == 0 || phase.armed || + g_fdwic_submit_pmu_return_ready_atomic_active) { + phase.boundary_error = true; + if (g_fdwic_submit_pmu_return_ready_atomic_active) { + g_fdwic_submit_pmu_return_ready_atomic_time_error = true; + } + return; + } + // begin 使用 TOTAL -> scalar -> I-cache。三者均为运行中 read-clear； + // 由外向内的顺序使随后观测到的 scalar 窗包含在 total 窗内。 + const FdwicSubmitPmuTotalShadowSnapshot total = fdwic_submit_pmu_phase_read_total_shadow(); + const uint32_t scalar = fdwic_submit_pmu_phase_read_scalar_shadow(); + const FdwicSubmitPmuIcacheShadowSnapshot icache = fdwic_submit_pmu_phase_read_shadow_counters(); + if (!fdwic_submit_pmu_add_total_shadow(total, /*include_in_phase=*/false) || + !fdwic_submit_pmu_add_scalar_shadow(scalar, /*include_in_phase=*/false) || + !fdwic_submit_pmu_add_icache_shadow(icache, /*include_in_phase=*/false)) { + phase.boundary_error = true; + return; + } + ++phase.begin_reads; + phase.armed = true; + phase.phase_excluded_atomic_ticks = 0; + // 起点位于 begin read-clear/bookkeeping 之后。empty-bracket wrapper + // 会在外层另行覆盖为完整 begin/end 对的经验耗时。 + phase.phase_begin_tick = get_sys_cnt_aicore(); + } +#endif +} + +template +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_phase_end() { +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 + if constexpr (Phase == kFdwicSubmitPmuCompiledPhase) { + const uint64_t phase_end_tick = get_sys_cnt_aicore(); + FdwicSubmitPmuPhaseAccumulator &phase = g_fdwic_submit_pmu_phase; + if (!g_fdwic_submit_pmu_started || g_fdwic_submit_pmu_stopped || !g_fdwic_submit_pmu_gate_running || + g_fdwic_submit_pmu_phase_core == nullptr || g_fdwic_submit_pmu_reg_base == 0 || !phase.armed || + phase_end_tick < phase.phase_begin_tick || g_fdwic_submit_pmu_return_ready_atomic_active) { + phase.boundary_error = true; + if (g_fdwic_submit_pmu_return_ready_atomic_active) { + g_fdwic_submit_pmu_return_ready_atomic_time_error = true; + } + return; + } + const uint64_t raw_phase_ticks = phase_end_tick - phase.phase_begin_tick; + if (phase.phase_excluded_atomic_ticks > raw_phase_ticks || + phase.phase_elapsed_ticks > UINT64_MAX - (raw_phase_ticks - phase.phase_excluded_atomic_ticks)) { + phase.boundary_error = true; + g_fdwic_submit_pmu_return_ready_atomic_time_error = true; + return; + } + phase.phase_elapsed_ticks += raw_phase_ticks - phase.phase_excluded_atomic_ticks; + // 终点位于 read-clear 之前，SYS 时间不包含 end 读取开销。PMU end + // 使用 I-cache -> scalar -> TOTAL，与 begin 反序；scalar 观测窗因而 + // 包含在 total 观测窗内。边界读取及少量 bookkeeping 会进入 PMU + // observed，不能把它解释成原业务计数的数学下界。 + const FdwicSubmitPmuIcacheShadowSnapshot icache = fdwic_submit_pmu_phase_read_shadow_counters(); + const uint32_t scalar = fdwic_submit_pmu_phase_read_scalar_shadow(); + const FdwicSubmitPmuTotalShadowSnapshot total = fdwic_submit_pmu_phase_read_total_shadow(); + if (!fdwic_submit_pmu_add_icache_shadow(icache, /*include_in_phase=*/true) || + !fdwic_submit_pmu_add_scalar_shadow(scalar, /*include_in_phase=*/true) || + !fdwic_submit_pmu_add_total_shadow(total, /*include_in_phase=*/true)) { + phase.boundary_error = true; + return; + } + ++phase.end_reads; + phase.armed = false; + phase.phase_begin_tick = 0; + phase.phase_excluded_atomic_ticks = 0; + } +#endif +} + +constexpr uint32_t kFdwicSubmitPmuKernelTokenWhole = 1U << 0; +constexpr uint32_t kFdwicSubmitPmuKernelTokenPhase = 1U << 1; + +// scalar 分母只累计 gate 真正运行的离散片段。调用点位于 stop 之前、start +// 之后，因此 linked Kernel 和两侧 PIPE_ALL 门控成本都不进入该 SYS_CNT 值。 +PTO_DEVICE_FUNC inline bool fdwic_submit_pmu_close_scalar_segment() { + const uint64_t segment_end = get_sys_cnt_aicore(); + if (!g_fdwic_submit_pmu_gate_running || g_fdwic_submit_pmu_scalar_segment_begin_tick == 0 || + segment_end < g_fdwic_submit_pmu_scalar_segment_begin_tick) { + g_fdwic_submit_pmu_gate_error = true; + g_fdwic_submit_pmu_scalar_segment_begin_tick = 0; + g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks = 0; + return false; + } + const uint64_t raw_segment_ticks = segment_end - g_fdwic_submit_pmu_scalar_segment_begin_tick; + if (g_fdwic_submit_pmu_return_ready_atomic_active || + g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks > raw_segment_ticks || + g_fdwic_submit_pmu_scalar_elapsed_ticks > + UINT64_MAX - (raw_segment_ticks - g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks)) { + g_fdwic_submit_pmu_return_ready_atomic_time_error = true; + g_fdwic_submit_pmu_scalar_segment_begin_tick = 0; + g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks = 0; + return false; + } + g_fdwic_submit_pmu_scalar_elapsed_ticks += + raw_segment_ticks - g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks; + g_fdwic_submit_pmu_scalar_segment_begin_tick = 0; + g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks = 0; + return true; +} + +// return-ready atomic 不停 PMU gate：begin 只在有效 scalar segment 内保存 +// SYS_CNT，end 由 atomic wrapper 在消费返回值的数据依赖 SYS_CNT 后回填。 +// 这样只从时间分母扣除本地完成等待，不引入 PIPE_ALL，也不声称跨核可见。 +constexpr uint32_t kFdwicSubmitPmuReturnReadyAtomicToken = 1U; + +PTO_DEVICE_FUNC inline uint32_t fdwic_submit_pmu_return_ready_atomic_begin() { + if (!g_fdwic_submit_pmu_started || g_fdwic_submit_pmu_stopped || !g_fdwic_submit_pmu_gate_running) return 0; + // 嵌套 wrapper 复用外层 bracket：内层拿到 token=0，end(0) 也完全 + // no-op。只有没有外层可覆盖却缺失有效 scalar segment 才算异常。 + if (g_fdwic_submit_pmu_return_ready_atomic_active) return 0; + if (g_fdwic_submit_pmu_scalar_segment_begin_tick == 0) { + g_fdwic_submit_pmu_return_ready_atomic_time_error = true; + return 0; + } + g_fdwic_submit_pmu_return_ready_atomic_begin_tick = get_sys_cnt_aicore(); + g_fdwic_submit_pmu_return_ready_atomic_active = true; +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 + g_fdwic_submit_pmu_return_ready_atomic_phase_armed = g_fdwic_submit_pmu_phase.armed; +#else + g_fdwic_submit_pmu_return_ready_atomic_phase_armed = false; +#endif + return kFdwicSubmitPmuReturnReadyAtomicToken; +} + +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_return_ready_atomic_end(uint32_t token, uint64_t dependency_end_tick) { + if (token == 0) return; + if (token != kFdwicSubmitPmuReturnReadyAtomicToken || !g_fdwic_submit_pmu_started || g_fdwic_submit_pmu_stopped || + !g_fdwic_submit_pmu_gate_running || !g_fdwic_submit_pmu_return_ready_atomic_active || + dependency_end_tick < g_fdwic_submit_pmu_return_ready_atomic_begin_tick) { + g_fdwic_submit_pmu_return_ready_atomic_time_error = true; + g_fdwic_submit_pmu_return_ready_atomic_active = false; + g_fdwic_submit_pmu_return_ready_atomic_begin_tick = 0; + g_fdwic_submit_pmu_return_ready_atomic_phase_armed = false; + return; + } + + const uint64_t elapsed = dependency_end_tick - g_fdwic_submit_pmu_return_ready_atomic_begin_tick; + bool valid = g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks <= UINT64_MAX - elapsed; +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 + FdwicSubmitPmuPhaseAccumulator &phase = g_fdwic_submit_pmu_phase; + valid = valid && phase.armed == g_fdwic_submit_pmu_return_ready_atomic_phase_armed; + if (valid && phase.armed) { + valid = phase.phase_excluded_atomic_ticks <= UINT64_MAX - elapsed; + } +#endif + if (!valid) { + g_fdwic_submit_pmu_return_ready_atomic_time_error = true; +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 + phase.boundary_error = true; +#endif + } else { + g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks += elapsed; +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 + if (phase.armed) phase.phase_excluded_atomic_ticks += elapsed; +#endif + g_fdwic_submit_pmu_return_ready_atomic_seen = true; + } + g_fdwic_submit_pmu_return_ready_atomic_active = false; + g_fdwic_submit_pmu_return_ready_atomic_begin_tick = 0; + g_fdwic_submit_pmu_return_ready_atomic_phase_armed = false; +} + +// execute_slot() 也会被背压等待和 FinalDrain 调用。窗口外返回 0，不触碰 +// PMU；窗口内每个真实 linked Kernel 都先暂停 whole gate。若选中 phase 正在 +// armed，则先将其闭合并把额外边界数编码进 token，供紧邻调用后的 resume 恢复。 +PTO_DEVICE_FUNC inline uint32_t fdwic_submit_pmu_linked_kernel_pause() { + if (!g_fdwic_submit_pmu_started || g_fdwic_submit_pmu_stopped) return 0; + if (!g_fdwic_submit_pmu_gate_running) { + // 不允许嵌套 pause 或丢失 resume 后继续操作 gate。外层若仍持有 token， + // 保持关窗可确保 Kernel 不污染计数；本轮最终由 core status 拒绝。 + g_fdwic_submit_pmu_gate_error = true; + return 0; + } + + uint32_t token = kFdwicSubmitPmuKernelTokenWhole; +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 + FdwicSubmitPmuPhaseAccumulator &phase = g_fdwic_submit_pmu_phase; + if (phase.armed) { + const uint32_t old_end_reads = phase.end_reads; + fdwic_submit_pmu_phase_end(); + if (phase.boundary_error || phase.armed || phase.end_reads != old_end_reads + 1U || + g_fdwic_submit_pmu_excluded_kernel_calls == UINT32_MAX) { + phase.boundary_error = true; + } else { + ++g_fdwic_submit_pmu_excluded_kernel_calls; + token |= kFdwicSubmitPmuKernelTokenPhase; + } + } +#endif + + (void)fdwic_submit_pmu_close_scalar_segment(); + bisheng::cce::metrics_prof_stop(); + g_fdwic_submit_pmu_gate_running = false; + return token; +} + +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_linked_kernel_resume(uint32_t token) { + if (token == 0) return; + if ((token & kFdwicSubmitPmuKernelTokenWhole) == 0 || !g_fdwic_submit_pmu_started || g_fdwic_submit_pmu_stopped || + g_fdwic_submit_pmu_gate_running) { + g_fdwic_submit_pmu_gate_error = true; + return; + } + + bisheng::cce::metrics_prof_start(); + g_fdwic_submit_pmu_gate_running = true; + g_fdwic_submit_pmu_scalar_segment_begin_tick = get_sys_cnt_aicore(); + g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks = 0; + +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 + if ((token & kFdwicSubmitPmuKernelTokenPhase) != 0) { + FdwicSubmitPmuPhaseAccumulator &phase = g_fdwic_submit_pmu_phase; + if (phase.armed) { + phase.boundary_error = true; + return; + } + const uint32_t old_begin_reads = phase.begin_reads; + fdwic_submit_pmu_phase_begin(); + if (phase.boundary_error || !phase.armed || phase.begin_reads != old_begin_reads + 1U) { + phase.boundary_error = true; + } + } +#else + if ((token & kFdwicSubmitPmuKernelTokenPhase) != 0) g_fdwic_submit_pmu_gate_error = true; +#endif +} + +// 在 Claim.end 调用点量化一对原样 running begin/end observer。外层 SYS_CNT +// 位于两次 shadow read-clear 之外，因此不会主动进入局部 request/miss +// observed；它自身仍是计时边界底噪，结果只能作经验量尺。 +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_empty_bracket_calibrate() { +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 2 + FdwicSubmitPmuPhaseAccumulator &phase = g_fdwic_submit_pmu_phase; + const uint64_t old_elapsed = phase.phase_elapsed_ticks; + const uint32_t old_begin_reads = phase.begin_reads; + const uint32_t old_end_reads = phase.end_reads; + const uint64_t outer_begin = get_sys_cnt_aicore(); + fdwic_submit_pmu_phase_begin(); + fdwic_submit_pmu_phase_end(); + const uint64_t outer_end = get_sys_cnt_aicore(); + if (phase.boundary_error || phase.armed || phase.begin_reads != old_begin_reads + 1U || + phase.end_reads != old_end_reads + 1U || outer_end < outer_begin || + old_elapsed > UINT64_MAX - (outer_end - outer_begin)) { + phase.boundary_error = true; + return; + } + phase.phase_elapsed_ticks = old_elapsed + outer_end - outer_begin; +#endif +} + +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_submit_begin(int32_t task_id) { + constexpr uint32_t kReadyMask = ((1U << 8) - 1U) | kFdwicSubmitPmuCnt0SelectorValid | + kFdwicSubmitPmuCnt1SelectorValid | kFdwicSubmitPmuCnt3SelectorValid; +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 6 + // 非首个 Submit 已完成 dist_submit_begin()，在统一 begin hook 关闭 + // 上一次 tail 打开的跨 Submit 区间。 + if (task_id > 0 && static_cast(task_id) < g_fdwic_submit_pmu_expected_submits) { + fdwic_submit_pmu_phase_end(); + } +#endif + if (task_id != 0 || g_fdwic_submit_pmu_core == nullptr || g_fdwic_submit_pmu_expected_submits == 0 || + (g_fdwic_submit_pmu_status & kReadyMask) != kReadyMask) { + return; + } + // SYS_CNT 包围的是同一业务挂点；PMU gate 的 PIPE_ALL 成本位于 PMU + // window 外，不能把该 tick 区间与另一 ELF 的绝对时间直接相减。 + g_fdwic_submit_pmu_start_tick = get_sys_cnt_aicore(); + bisheng::cce::metrics_prof_start(); + g_fdwic_submit_pmu_gate_running = true; + g_fdwic_submit_pmu_scalar_segment_begin_tick = get_sys_cnt_aicore(); + g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks = 0; + g_fdwic_submit_pmu_started = true; + g_fdwic_submit_pmu_status |= kFdwicSubmitPmuWindowStarted; +} + +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_submit_end(int32_t task_id) { + if (!g_fdwic_submit_pmu_started || g_fdwic_submit_pmu_stopped || task_id < 0) { + return; + } + const uint32_t submit_ordinal = static_cast(task_id) + 1U; +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID == 6 + // 每个非末次 Submit 在统一 end hook 打开 transition；末次只负责关闭 + // 整窗，避免制造一个没有后继 Submit 的悬空区间。 + if (submit_ordinal < g_fdwic_submit_pmu_expected_submits) { + fdwic_submit_pmu_phase_begin(); + return; + } +#endif + if (submit_ordinal != g_fdwic_submit_pmu_expected_submits) return; + const bool gate_was_running = g_fdwic_submit_pmu_gate_running; + const bool scalar_segment_closed = gate_was_running && fdwic_submit_pmu_close_scalar_segment(); + if (gate_was_running) { + bisheng::cce::metrics_prof_stop(); + } else { + g_fdwic_submit_pmu_gate_error = true; + } + g_fdwic_submit_pmu_gate_running = false; + g_fdwic_submit_pmu_end_tick = get_sys_cnt_aicore(); + fdwic_submit_pmu_read_counters(); +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 + FdwicSubmitPmuPhaseAccumulator &phase = g_fdwic_submit_pmu_phase; + if (!phase.boundary_error && !phase.armed && phase.begin_reads == phase.end_reads) { + phase.status |= kFdwicSubmitPmuPhaseBoundaryBalanced; + } + const uint64_t expected_boundary_reads = fdwic_submit_pmu_expected_phase_boundary_reads( + kFdwicSubmitPmuCompiledPhase, g_fdwic_submit_pmu_expected_submits, g_fdwic_submit_pmu_excluded_kernel_calls + ); + const bool dynamic_calls = fdwic_submit_pmu_phase_has_dynamic_calls(kFdwicSubmitPmuCompiledPhase); + const bool outer_reads_valid = phase.begin_reads >= g_fdwic_submit_pmu_excluded_kernel_calls && + phase.end_reads >= g_fdwic_submit_pmu_excluded_kernel_calls; + const uint64_t outer_begin_calls = + outer_reads_valid ? phase.begin_reads - g_fdwic_submit_pmu_excluded_kernel_calls : UINT64_MAX; + const uint64_t outer_end_calls = + outer_reads_valid ? phase.end_reads - g_fdwic_submit_pmu_excluded_kernel_calls : UINT64_MAX; + const bool shape_valid = + dynamic_calls ? outer_reads_valid && outer_begin_calls == outer_end_calls && + outer_begin_calls <= fdwic_submit_pmu_dynamic_calls_max_per_core( + kFdwicSubmitPmuCompiledPhase, g_fdwic_submit_pmu_expected_submits + ) : + phase.begin_reads == expected_boundary_reads && phase.end_reads == expected_boundary_reads; + if (shape_valid) { + phase.status |= kFdwicSubmitPmuPhaseShapeValid; + } + const uint32_t excluded_kernel_calls = g_fdwic_submit_pmu_excluded_kernel_calls; + const bool zero_call_dynamic = fdwic_submit_pmu_phase_has_dynamic_calls(kFdwicSubmitPmuCompiledPhase) && + phase.begin_reads == excluded_kernel_calls && + phase.end_reads == excluded_kernel_calls; + if (phase.phase_requests <= phase.shadow_requests && phase.phase_misses <= phase.shadow_misses && + phase.shadow_misses <= phase.shadow_requests && phase.shadow_requests <= g_fdwic_submit_pmu_icache_requests && + phase.shadow_misses <= g_fdwic_submit_pmu_icache_misses) { + phase.status |= kFdwicSubmitPmuPhaseIcacheValuesOrdered; + } + if (phase.phase_scalar_busy <= phase.shadow_scalar_busy && + phase.shadow_scalar_busy <= g_fdwic_submit_pmu_scalar_busy && + phase.phase_total_cycles <= phase.shadow_total_cycles && phase.phase_scalar_busy <= phase.phase_total_cycles && + phase.shadow_scalar_busy <= phase.shadow_total_cycles && + g_fdwic_submit_pmu_scalar_busy <= g_fdwic_submit_pmu_total_cycles) { + phase.status |= kFdwicSubmitPmuPhasePmuValuesOrdered; + } + const bool phase_activity_valid = zero_call_dynamic ? + (phase.phase_total_cycles == 0 && phase.phase_scalar_busy == 0 && + phase.phase_requests == 0 && phase.phase_misses == 0) : + phase.phase_total_cycles != 0; + if (!phase.counter_error && phase_activity_valid && phase.shadow_total_cycles == g_fdwic_submit_pmu_total_cycles && + phase.shadow_scalar_busy != 0 && phase.shadow_scalar_busy <= UINT32_MAX && + phase.shadow_requests <= UINT32_MAX && phase.shadow_misses <= UINT32_MAX && + g_fdwic_submit_pmu_scalar_busy < kFdwicSubmitPmuCounterRiskThreshold && + g_fdwic_submit_pmu_icache_requests < kFdwicSubmitPmuCounterRiskThreshold && + g_fdwic_submit_pmu_icache_misses < kFdwicSubmitPmuCounterRiskThreshold) { + phase.status |= kFdwicSubmitPmuPhaseCounterReconstructionValid; + } + if (!g_fdwic_submit_pmu_return_ready_atomic_time_error && + ((zero_call_dynamic && phase.phase_elapsed_ticks == 0) || + (!zero_call_dynamic && phase.phase_elapsed_ticks != 0 && + phase.phase_elapsed_ticks <= g_fdwic_submit_pmu_scalar_elapsed_ticks))) { + phase.status |= kFdwicSubmitPmuPhaseTimeValid; + } +#endif + g_fdwic_submit_pmu_stopped = true; + g_fdwic_submit_pmu_status |= kFdwicSubmitPmuWindowStopped; + if (!g_fdwic_submit_pmu_gate_error && gate_was_running) { + g_fdwic_submit_pmu_status |= kFdwicSubmitPmuLinkedKernelGateBalanced; + } + if (!g_fdwic_submit_pmu_gate_error && !g_fdwic_submit_pmu_return_ready_atomic_time_error && scalar_segment_closed && + g_fdwic_submit_pmu_scalar_elapsed_ticks != 0 && g_fdwic_submit_pmu_end_tick >= g_fdwic_submit_pmu_start_tick && + g_fdwic_submit_pmu_scalar_elapsed_ticks <= g_fdwic_submit_pmu_end_tick - g_fdwic_submit_pmu_start_tick) { + g_fdwic_submit_pmu_status |= kFdwicSubmitPmuScalarElapsedValid; + } + bool return_ready_atomic_time_closed = + g_fdwic_submit_pmu_return_ready_atomic_seen && !g_fdwic_submit_pmu_return_ready_atomic_time_error && + !g_fdwic_submit_pmu_return_ready_atomic_active && g_fdwic_submit_pmu_return_ready_atomic_begin_tick == 0 && + g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks == 0; +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 + return_ready_atomic_time_closed = + return_ready_atomic_time_closed && g_fdwic_submit_pmu_phase.phase_excluded_atomic_ticks == 0; +#endif + if (return_ready_atomic_time_closed) { + g_fdwic_submit_pmu_status |= kFdwicSubmitPmuReturnReadyAtomicTimeValid; + } + if (g_fdwic_submit_pmu_total_cycles != 0) { + g_fdwic_submit_pmu_status |= kFdwicSubmitPmuTotalNonzero; + } +} + +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_flush(__gm__ DistCore *self) { + if (g_fdwic_submit_pmu_started && !g_fdwic_submit_pmu_stopped && g_fdwic_submit_pmu_gate_running) { + // 只负责关闭遗留 gate，故意不伪造 WindowStopped。host 会因闭合失败 + // 拒绝正式 raw；FinalDrain 也不会被包装成有效 Submit 样本。 + bisheng::cce::metrics_prof_stop(); + g_fdwic_submit_pmu_gate_running = false; + } + __gm__ FdwicSubmitPmuCoreData *core = g_fdwic_submit_pmu_core; + if (core == nullptr || self == nullptr) return; + core->first_submit_start_tick = g_fdwic_submit_pmu_start_tick; + core->last_submit_end_tick = g_fdwic_submit_pmu_end_tick; + core->total_cycles = g_fdwic_submit_pmu_total_cycles; + core->scalar_submit_elapsed_ticks = g_fdwic_submit_pmu_scalar_elapsed_ticks; + core->scalar_busy = g_fdwic_submit_pmu_scalar_busy; + core->icache_requests = g_fdwic_submit_pmu_icache_requests; + core->icache_misses = g_fdwic_submit_pmu_icache_misses; + core->submit_count = static_cast(self->local_index); + core->expected_submit_count = g_fdwic_submit_pmu_expected_submits; + core->logical_core_id = static_cast(self->core_idx); + core->physical_core_id = static_cast(get_physical_core_id()); + core->block_id = static_cast(self->block_id); + core->lane = static_cast(self->lane); + core->status = g_fdwic_submit_pmu_status; + dist_aicore_flush_region(core, sizeof(FdwicSubmitPmuCoreData)); +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 + __gm__ FdwicSubmitPmuPhaseCoreData *phase_core = g_fdwic_submit_pmu_phase_core; + if (phase_core == nullptr) return; + const FdwicSubmitPmuPhaseAccumulator &phase = g_fdwic_submit_pmu_phase; + phase_core->phase_elapsed_ticks = phase.phase_elapsed_ticks; + phase_core->phase_total_cycles_observed = phase.phase_total_cycles; + phase_core->phase_icache_requests_observed = phase.phase_requests; + phase_core->phase_icache_misses_observed = phase.phase_misses; + phase_core->phase_scalar_busy_observed = static_cast(phase.phase_scalar_busy); + phase_core->shadow_scalar_busy = static_cast(phase.shadow_scalar_busy); + phase_core->shadow_icache_requests = static_cast(phase.shadow_requests); + phase_core->shadow_icache_misses = static_cast(phase.shadow_misses); + phase_core->phase_id = static_cast(kFdwicSubmitPmuCompiledPhase); + phase_core->status = static_cast(phase.status); + phase_core->phase_begin_reads = phase.begin_reads; + phase_core->phase_end_reads = phase.end_reads; + phase_core->excluded_kernel_calls = g_fdwic_submit_pmu_excluded_kernel_calls; + dist_aicore_flush_region(phase_core, sizeof(FdwicSubmitPmuPhaseCoreData)); +#endif +} + +} // namespace + +#else + +namespace { +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_attach(__gm__ Runtime *, __gm__ DistCore *) {} +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_expect_submits(uint32_t) {} +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_submit_begin(int32_t) {} +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_submit_end(int32_t) {} +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_flush(__gm__ DistCore *) {} +template +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_phase_begin() {} +template +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_phase_end() {} +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_empty_bracket_calibrate() {} +PTO_DEVICE_FUNC inline uint32_t fdwic_submit_pmu_linked_kernel_pause() { return 0; } +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_linked_kernel_resume(uint32_t) {} +PTO_DEVICE_FUNC inline uint32_t fdwic_submit_pmu_return_ready_atomic_begin() { return 0; } +PTO_DEVICE_FUNC inline void fdwic_submit_pmu_return_ready_atomic_end(uint32_t, uint64_t) {} +} // namespace + +#endif diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/submit_pmu_types.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/submit_pmu_types.h new file mode 100644 index 0000000000..3072c6de42 --- /dev/null +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/submit_pmu_types.h @@ -0,0 +1,428 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include +#include + +#include "data_type.h" + +// 真实 A5 PA 的 submit-PMU 不复用普通泳道的逐事件 record，也不复用通用 +// PMU 的逐 kernel task ring。none 每核只发布一个 64B 整窗结果；局部阶段 +// 在相同整窗结果之后追加一个 64B sidecar，仍然没有逐事件记录。 +constexpr uint32_t kFdwicSubmitPmuMagic = 0x554d5053U; // little-endian "SPMU" +// v2 将整窗口径收敛为 scalar 代码时间：所有 linked vector/cube Kernel +// 在 gate 之外执行，result-used atomic 的 return-ready 依赖区间再从 +// SYS_CNT 累计值中扣除；PMU counter 仍保留 atomic 指令事件。v3 在局部 +// phase 中用 CNT3 复制 CNT2 scalar-busy，并对 CNT3 与 TOTAL 做 running +// read-clear 软件重建，从而在同一 phase 窗口内比较 PMU total/scalar。 +constexpr uint16_t kFdwicSubmitPmuVersion = 3; +constexpr uint16_t kFdwicSubmitPmuModeNone = 1; +constexpr uint16_t kFdwicSubmitPmuModeArgBuild = 2; +constexpr uint16_t kFdwicSubmitPmuModeEmptyBracket = 3; +constexpr uint16_t kFdwicSubmitPmuModeMaterialize = 4; +constexpr uint16_t kFdwicSubmitPmuModeClaim = 5; +constexpr uint16_t kFdwicSubmitPmuModeRegister = 6; +constexpr uint16_t kFdwicSubmitPmuModeSubmitTransition = 7; +constexpr uint16_t kFdwicSubmitPmuModeEfDrainControl = 8; +constexpr uint16_t kFdwicSubmitPmuModePrepareMap = 9; +constexpr uint16_t kFdwicSubmitPmuModeFanin = 10; +constexpr uint16_t kFdwicSubmitPmuModeWinnerBuild = 11; +constexpr uint16_t kFdwicSubmitPmuModeAllocComplete = 12; +constexpr uint16_t kFdwicSubmitPmuModeLoserReplay = 13; +constexpr uint32_t kFdwicSubmitPmuExpectedAic = 32; +constexpr uint32_t kFdwicSubmitPmuExpectedAiv = 64; +constexpr uint32_t kFdwicSubmitPmuExpectedCores = kFdwicSubmitPmuExpectedAic + kFdwicSubmitPmuExpectedAiv; +constexpr uint32_t kFdwicSubmitPmuPhysicalSubcores = 108; +constexpr uint32_t kFdwicSubmitPmuBitmapWords = 4; +constexpr uint32_t kFdwicSubmitPmuCounterRiskThreshold = 0x3fffffffU; + +enum class FdwicSubmitPmuPhase : uint16_t { + None = 0, + // compete-first Claim 完成到匹配 Finish 的 Materialize 入口；包含同步 + // eager callback 构参、Begin 返回和 Finish 重入。 + ArgBuild = 1, + // Claim.end 同一调用点的紧邻 begin/end；只提供 running bracket 自身 + // 引入的空区间经验观察指纹，不代表任何业务 phase 或数学最小值。 + EmptyBracket = 2, + // 当前泳道 Materialize.begin 到 Materialize.end：task-cap 检查与 + // dist_submit_materialize_args 主体；每个 Submit 固定调用一次。 + Materialize = 3, + // 当前泳道 Claim.begin 到 Claim.end：compete-first 还包含 Claim 前的 + // task-cap 检查；每个 Submit 固定调用一次。 + Claim = 4, + // dist_submit_register_outputs() 调用入口到返回。刻意排除前一阶段 + // record 发布和 caller 衔接；每个 Submit 固定调用一次。 + Register = 5, + // 上一次 Submit 返回前到下一次 dist_submit_begin() 完成。首个 Submit + // 没有前驱、末个 Submit 没有后继，因此每核固定 expected_submits - 1 次。 + SubmitTransition = 6, + // 每次 Submit 开头的 EfDrain 控制段。execute_slot() 中的真实 linked-kernel + // 调用通过成对 pause/resume 排除，barrier、完成发布和 frontier 等 scalar + // 后处理仍保留在控制段内。 + EfDrainControl = 7, + // dist_submit_prepare_map() 调用入口到返回；与泳道 PrepareMap 的业务 + // 主体边界一致，每个 Submit 固定调用一次。 + PrepareMap = 8, + // Kernel winner 的 dist_submit_collect_fanin() 调用体。winner 分布由 + // 多核竞争决定，因此只要求逐核边界平衡与全局调用数闭合。 + Fanin = 9, + // Kernel winner 的 WinnerBuild 完整业务边界内的 scalar control；等待 + // 期间回收执行的 linked Kernel 通过成对 pause/resume 从计数和时间中排除。 + WinnerBuild = 10, + // Alloc winner 的 AllocComplete 完整业务边界内的 scalar control；HeapGuard + // 慢路径回收执行的 linked Kernel 同样通过成对 pause/resume 排除。 + AllocComplete = 11, + // Kernel loser 的真实 drain_block_won() 调用体；每个 worker 都回放四个 + // Kernel Submit，winner 之外的调用均进入该阶段。 + LoserReplay = 12, + Count = 13, +}; + +constexpr uint16_t fdwic_submit_pmu_mode_for_phase(FdwicSubmitPmuPhase phase) { + switch (phase) { + case FdwicSubmitPmuPhase::None: + return kFdwicSubmitPmuModeNone; + case FdwicSubmitPmuPhase::ArgBuild: + return kFdwicSubmitPmuModeArgBuild; + case FdwicSubmitPmuPhase::EmptyBracket: + return kFdwicSubmitPmuModeEmptyBracket; + case FdwicSubmitPmuPhase::Materialize: + return kFdwicSubmitPmuModeMaterialize; + case FdwicSubmitPmuPhase::Claim: + return kFdwicSubmitPmuModeClaim; + case FdwicSubmitPmuPhase::Register: + return kFdwicSubmitPmuModeRegister; + case FdwicSubmitPmuPhase::SubmitTransition: + return kFdwicSubmitPmuModeSubmitTransition; + case FdwicSubmitPmuPhase::EfDrainControl: + return kFdwicSubmitPmuModeEfDrainControl; + case FdwicSubmitPmuPhase::PrepareMap: + return kFdwicSubmitPmuModePrepareMap; + case FdwicSubmitPmuPhase::Fanin: + return kFdwicSubmitPmuModeFanin; + case FdwicSubmitPmuPhase::WinnerBuild: + return kFdwicSubmitPmuModeWinnerBuild; + case FdwicSubmitPmuPhase::AllocComplete: + return kFdwicSubmitPmuModeAllocComplete; + case FdwicSubmitPmuPhase::LoserReplay: + return kFdwicSubmitPmuModeLoserReplay; + case FdwicSubmitPmuPhase::Count: + break; + } + return 0; +} + +PTO_DEVICE_FUNC constexpr bool fdwic_submit_pmu_phase_has_dynamic_calls(FdwicSubmitPmuPhase phase) { + return phase == FdwicSubmitPmuPhase::Fanin || phase == FdwicSubmitPmuPhase::WinnerBuild || + phase == FdwicSubmitPmuPhase::AllocComplete || phase == FdwicSubmitPmuPhase::LoserReplay; +} + +// Fanin/WinnerBuild 的 AIC/AIV 数量由 PA 的四个 Kernel 角色固定；LoserReplay +// 是 96 核回放总数扣除这些 winner 后的角色补集。AllocComplete 只有全局 B 次 +// 可由协议确定，winner 落在哪类核取决于多核竞争，不能伪造角色公式。 +constexpr bool fdwic_submit_pmu_dynamic_calls_have_fixed_roles(FdwicSubmitPmuPhase phase) { + return phase == FdwicSubmitPmuPhase::Fanin || phase == FdwicSubmitPmuPhase::WinnerBuild || + phase == FdwicSubmitPmuPhase::LoserReplay; +} + +PTO_DEVICE_FUNC constexpr bool fdwic_submit_pmu_phase_excludes_linked_kernel(FdwicSubmitPmuPhase phase) { + // 当前每个 phase ELF 只编译一个选中阶段。无论 linked Kernel 将来落入 + // 哪个阶段，begin/end 都统一扣除 pause/resume 生成的附加边界。 + return phase != FdwicSubmitPmuPhase::None; +} + +// 当前 PA 每个 batch 固定提交四个 Kernel task 和一个 Alloc task。动态 +// winner/loser phase 不伪造“每核固定次数”，只在 96 核聚合后按业务形状闭合。 +constexpr uint32_t fdwic_submit_pmu_batch_count(uint32_t expected_submits) { + return expected_submits != 0 && expected_submits % 5U == 0 ? expected_submits / 5U : 0U; +} + +constexpr uint64_t fdwic_submit_pmu_expected_dynamic_calls_all(FdwicSubmitPmuPhase phase, uint32_t expected_submits) { + const uint64_t batches = fdwic_submit_pmu_batch_count(expected_submits); + if (phase == FdwicSubmitPmuPhase::Fanin || phase == FdwicSubmitPmuPhase::WinnerBuild) return 4U * batches; + if (phase == FdwicSubmitPmuPhase::LoserReplay) { + return (4U * kFdwicSubmitPmuExpectedCores - 4U) * batches; + } + return phase == FdwicSubmitPmuPhase::AllocComplete ? batches : 0U; +} + +constexpr uint64_t fdwic_submit_pmu_expected_dynamic_calls_aic(FdwicSubmitPmuPhase phase, uint32_t expected_submits) { + const uint64_t batches = fdwic_submit_pmu_batch_count(expected_submits); + if (phase == FdwicSubmitPmuPhase::Fanin || phase == FdwicSubmitPmuPhase::WinnerBuild) return 2U * batches; + if (phase == FdwicSubmitPmuPhase::LoserReplay) { + return (4U * kFdwicSubmitPmuExpectedAic - 2U) * batches; + } + return 0U; +} + +constexpr uint64_t fdwic_submit_pmu_expected_dynamic_calls_aiv(FdwicSubmitPmuPhase phase, uint32_t expected_submits) { + const uint64_t batches = fdwic_submit_pmu_batch_count(expected_submits); + if (phase == FdwicSubmitPmuPhase::Fanin || phase == FdwicSubmitPmuPhase::WinnerBuild) return 2U * batches; + if (phase == FdwicSubmitPmuPhase::LoserReplay) { + return (4U * kFdwicSubmitPmuExpectedAiv - 2U) * batches; + } + return 0U; +} + +PTO_DEVICE_FUNC constexpr uint32_t +fdwic_submit_pmu_dynamic_calls_max_per_core(FdwicSubmitPmuPhase phase, uint32_t expected_submits) { + const uint32_t batches = expected_submits != 0 && expected_submits % 5U == 0 ? expected_submits / 5U : 0U; + if (phase == FdwicSubmitPmuPhase::Fanin || phase == FdwicSubmitPmuPhase::WinnerBuild) return 2U * batches; + if (phase == FdwicSubmitPmuPhase::LoserReplay) return 4U * batches; + return phase == FdwicSubmitPmuPhase::AllocComplete ? batches : 0U; +} + +PTO_DEVICE_FUNC constexpr uint32_t +fdwic_submit_pmu_expected_phase_calls(FdwicSubmitPmuPhase phase, uint32_t expected_submits) { + if (phase == FdwicSubmitPmuPhase::None) return 0; + if (fdwic_submit_pmu_phase_has_dynamic_calls(phase)) return 0; + if (phase == FdwicSubmitPmuPhase::SubmitTransition) { + return expected_submits == 0 ? 0 : expected_submits - 1U; + } + return expected_submits; +} + +PTO_DEVICE_FUNC constexpr uint64_t fdwic_submit_pmu_expected_phase_boundary_reads( + FdwicSubmitPmuPhase phase, uint32_t expected_submits, uint32_t excluded_kernel_calls +) { + const uint64_t outer_calls = fdwic_submit_pmu_expected_phase_calls(phase, expected_submits); + return outer_calls + (fdwic_submit_pmu_phase_excludes_linked_kernel(phase) ? excluded_kernel_calls : 0U); +} + +constexpr bool fdwic_submit_pmu_mode_has_phase(uint16_t mode) { + return mode == kFdwicSubmitPmuModeArgBuild || mode == kFdwicSubmitPmuModeEmptyBracket || + mode == kFdwicSubmitPmuModeMaterialize || mode == kFdwicSubmitPmuModeClaim || + mode == kFdwicSubmitPmuModeRegister || mode == kFdwicSubmitPmuModeSubmitTransition || + mode == kFdwicSubmitPmuModeEfDrainControl || mode == kFdwicSubmitPmuModePrepareMap || + mode == kFdwicSubmitPmuModeFanin || mode == kFdwicSubmitPmuModeWinnerBuild || + mode == kFdwicSubmitPmuModeAllocComplete || mode == kFdwicSubmitPmuModeLoserReplay; +} + +// A5 PIPE_UTIL 事件布局。CNT6/CNT7 是权威值；CNT8/CNT5 使用相同事件作 +// 同窗副本。none 构建没有中途 read-clear，故两组必须逐核精确相等。 +constexpr uint32_t kFdwicSubmitPmuCnt0VectorBusy = 0x501U; +constexpr uint32_t kFdwicSubmitPmuCnt1CubeBusy = 0x301U; +constexpr uint32_t kFdwicSubmitPmuCnt2ScalarBusy = 0x001U; +constexpr uint32_t kFdwicSubmitPmuCnt3ShadowScalarBusy = 0x001U; +constexpr uint32_t kFdwicSubmitPmuCnt5ShadowIcacheMiss = 0x035U; +constexpr uint32_t kFdwicSubmitPmuCnt6IcacheRequest = 0x034U; +constexpr uint32_t kFdwicSubmitPmuCnt7IcacheMiss = 0x035U; +constexpr uint32_t kFdwicSubmitPmuCnt8ShadowIcacheRequest = 0x034U; + +enum FdwicSubmitPmuCoreStatus : uint32_t { + kFdwicSubmitPmuRequested = 1U << 0, + kFdwicSubmitPmuRegMapped = 1U << 1, + kFdwicSubmitPmuPhysicalIdValid = 1U << 2, + kFdwicSubmitPmuCnt2SelectorValid = 1U << 3, + kFdwicSubmitPmuCnt5SelectorValid = 1U << 4, + kFdwicSubmitPmuCnt6SelectorValid = 1U << 5, + kFdwicSubmitPmuCnt7SelectorValid = 1U << 6, + kFdwicSubmitPmuCnt8SelectorValid = 1U << 7, + kFdwicSubmitPmuWindowStarted = 1U << 8, + kFdwicSubmitPmuWindowStopped = 1U << 9, + kFdwicSubmitPmuTotalNonzero = 1U << 10, + kFdwicSubmitPmuCnt0SelectorValid = 1U << 11, + kFdwicSubmitPmuCnt1SelectorValid = 1U << 12, + kFdwicSubmitPmuLinkedKernelGateBalanced = 1U << 13, + kFdwicSubmitPmuScalarElapsedValid = 1U << 14, + kFdwicSubmitPmuVectorBusyZero = 1U << 15, + kFdwicSubmitPmuCubeBusyZero = 1U << 16, + // 所有 result-used atomic 的 return-ready 依赖区间都已成对闭合，且已 + // 从 scalar SYS_CNT 分母和命中的 phase 时间中扣除。该位不改变 PMU + // gate，避免 PIPE_ALL 反过来改写 atomic 热路径。 + kFdwicSubmitPmuReturnReadyAtomicTimeValid = 1U << 17, + kFdwicSubmitPmuCnt3SelectorValid = 1U << 18, + // none 没有 phase sidecar，故用独立状态位证明一次性读取的 CNT8/CNT5 + // 与 CNT6/CNT7 权威整窗逐核精确相等。 + kFdwicSubmitPmuNoneIcacheShadowPrimaryMatch = 1U << 19, + // none 不做中途 read-clear，CNT3/CNT2 的同事件计数必须逐核精确相等。 + kFdwicSubmitPmuNoneScalarShadowPrimaryMatch = 1U << 20, +}; +constexpr uint32_t kFdwicSubmitPmuRequiredCoreStatus = (1U << 19) - 1U; +constexpr uint32_t kFdwicSubmitPmuRequiredNoneCoreStatus = (1U << 21) - 1U; + +PTO_DEVICE_FUNC constexpr uint32_t fdwic_submit_pmu_required_core_status(FdwicSubmitPmuPhase phase) { + return phase == FdwicSubmitPmuPhase::None ? kFdwicSubmitPmuRequiredNoneCoreStatus : + kFdwicSubmitPmuRequiredCoreStatus; +} + +enum FdwicSubmitPmuPhaseStatus : uint32_t { + kFdwicSubmitPmuPhaseRequested = 1U << 0, + kFdwicSubmitPmuPhaseBoundaryBalanced = 1U << 1, + kFdwicSubmitPmuPhaseShapeValid = 1U << 2, + kFdwicSubmitPmuPhaseIcacheValuesOrdered = 1U << 3, + kFdwicSubmitPmuPhaseTimeValid = 1U << 4, + kFdwicSubmitPmuPhaseIcacheTailRead = 1U << 5, + kFdwicSubmitPmuPhaseScalarTailRead = 1U << 6, + kFdwicSubmitPmuPhaseTotalTailRead = 1U << 7, + kFdwicSubmitPmuPhasePmuValuesOrdered = 1U << 8, + kFdwicSubmitPmuPhaseCounterReconstructionValid = 1U << 9, +}; +constexpr uint32_t kFdwicSubmitPmuRequiredPhaseStatus = (1U << 10) - 1U; + +enum FdwicSubmitPmuOwnerStatus : uint32_t { + kFdwicSubmitPmuOwnerRequested = 1U << 0, + kFdwicSubmitPmuOwnerTopologyValid = 1U << 1, + kFdwicSubmitPmuOwnerConfigured = 1U << 2, + kFdwicSubmitPmuOwnerConfigReadbackValid = 1U << 3, + kFdwicSubmitPmuOwnerRestoreAttempted = 1U << 4, + kFdwicSubmitPmuOwnerRestored = 1U << 5, + kFdwicSubmitPmuOwnerAborted = 1U << 31, +}; +constexpr uint32_t kFdwicSubmitPmuRequiredOwnerStatus = + kFdwicSubmitPmuOwnerRequested | kFdwicSubmitPmuOwnerTopologyValid | kFdwicSubmitPmuOwnerConfigured | + kFdwicSubmitPmuOwnerConfigReadbackValid | kFdwicSubmitPmuOwnerRestoreAttempted | kFdwicSubmitPmuOwnerRestored; + +// 失败字段只用于 owner 冷路径诊断；正式 raw 只有全部字段闭合后才发布。 +enum class FdwicSubmitPmuOwnerField : uint32_t { + None = 0, + State, + Topology, + RegisterBase, + Ctrl0, + Ctrl1, + Selector0, + StartLow = 16, + StartHigh, + StopLow, + StopHigh, +}; + +struct FdwicSubmitPmuCoreData { + uint64_t first_submit_start_tick; + uint64_t last_submit_end_tick; + // none 是 stop 后一次性读取的 TOTAL；局部 phase 是所有 running + // read-clear chunk 与 stop 后 tail 的软件重建 observed whole。 + uint64_t total_cycles; + // 多段 SYS_CNT 累计，只覆盖 PMU gate 开启的 scalar 调度区间；linked + // Kernel、两侧 metrics_prof_stop/start 边界及 result-used atomic 的 + // return-ready 依赖区间均不进入该时间分母。 + uint64_t scalar_submit_elapsed_ticks; + uint32_t scalar_busy; + uint32_t icache_requests; + uint32_t icache_misses; + uint32_t submit_count; + uint32_t expected_submit_count; + uint16_t logical_core_id; + uint16_t physical_core_id; + uint16_t block_id; + uint16_t lane; + uint32_t status; +} __attribute__((aligned(64))); + +static_assert(sizeof(FdwicSubmitPmuCoreData) == 64, "submit-PMU core record must occupy one cacheline"); +static_assert( + offsetof(FdwicSubmitPmuCoreData, scalar_submit_elapsed_ticks) == 24, "submit-PMU scalar elapsed offset changed" +); +static_assert(offsetof(FdwicSubmitPmuCoreData, status) == 60, "submit-PMU status offset changed"); + +// phase sidecar 只在局部阶段 mode 中分配。每个 worker 独占一条 cacheline， +// 避免相邻 worker 发布结果时产生伪共享。CNT2/6/7 的整窗 primary 仍保存在 +// FdwicSubmitPmuCoreData；这里的 total/scalar/request/miss 是 running +// read-clear 观测值。begin/end 两侧的少量 bookkeeping 也会进入该样本， +// 因此它们只能用于同一 phase ELF 内的配对比较。 +struct FdwicSubmitPmuPhaseCoreData { + uint64_t phase_elapsed_ticks; + uint64_t phase_total_cycles_observed; + uint64_t phase_icache_requests_observed; + uint64_t phase_icache_misses_observed; + uint32_t phase_scalar_busy_observed; + uint32_t shadow_scalar_busy; + uint32_t shadow_icache_requests; + uint32_t shadow_icache_misses; + uint16_t phase_id; + uint16_t status; + uint32_t phase_begin_reads; + uint32_t phase_end_reads; + uint32_t excluded_kernel_calls; +} __attribute__((aligned(64))); + +static_assert(sizeof(FdwicSubmitPmuPhaseCoreData) == 64, "submit-PMU phase record must occupy one cacheline"); +static_assert( + offsetof(FdwicSubmitPmuPhaseCoreData, phase_total_cycles_observed) == 8, "submit-PMU phase total offset changed" +); +static_assert( + offsetof(FdwicSubmitPmuPhaseCoreData, phase_scalar_busy_observed) == 32, "submit-PMU phase scalar offset changed" +); +static_assert(offsetof(FdwicSubmitPmuPhaseCoreData, status) == 50, "submit-PMU phase status offset changed"); +static_assert( + offsetof(FdwicSubmitPmuPhaseCoreData, excluded_kernel_calls) == 60, + "submit-PMU phase excluded-kernel offset changed" +); + +struct FdwicSubmitPmuPhaseAccumulator { + uint64_t shadow_total_cycles; + uint64_t phase_total_cycles; + uint64_t shadow_scalar_busy; + uint64_t phase_scalar_busy; + uint64_t shadow_requests; + uint64_t shadow_misses; + uint64_t phase_requests; + uint64_t phase_misses; + uint64_t phase_elapsed_ticks; + uint64_t phase_begin_tick; + // 当前一次 phase invocation 内已经闭合的 return-ready atomic 等待。 + // phase_end() 从原始 SYS_CNT delta 中扣除后立即清零。 + uint64_t phase_excluded_atomic_ticks; + uint32_t begin_reads; + uint32_t end_reads; + uint32_t status; + bool armed; + bool boundary_error; + bool counter_error; +}; + +struct FdwicSubmitPmuHeader { + // Host 初始化的只读配置 cacheline。 + uint32_t magic; + uint16_t version; + uint16_t mode; + uint32_t header_bytes; + uint32_t record_bytes; + uint32_t num_cores; + uint32_t expected_aic; + uint32_t expected_aiv; + uint64_t sys_cnt_freq_hz; + // 依次为 CNT2 primary scalar、CNT3 shadow scalar、CNT5 shadow miss、 + // CNT6 primary request、CNT7 primary miss、CNT8 shadow request。 + uint32_t selectors[6]; + + // AICPU owner 独占写入的状态 cacheline。 + volatile uint32_t owner_status; + volatile uint32_t configured_count; + volatile uint32_t restored_count; + volatile uint32_t configured_aic; + volatile uint32_t configured_aiv; + volatile uint32_t complete_mixed_triplets; + volatile uint32_t restore_failures; + volatile uint32_t active_after_restore; + volatile uint32_t configured_bitmap_words[kFdwicSubmitPmuBitmapWords]; + volatile uint32_t first_failure_core; + volatile uint32_t first_failure_field; + volatile uint32_t first_failure_observed; + volatile uint32_t first_failure_expected; + + FdwicSubmitPmuCoreData cores[kFdwicSubmitPmuExpectedCores]; +} __attribute__((aligned(64))); + +static_assert(offsetof(FdwicSubmitPmuHeader, owner_status) == 64, "owner state must occupy cacheline two"); +static_assert(offsetof(FdwicSubmitPmuHeader, cores) == 128, "per-core records must start after two cachelines"); +static_assert(sizeof(FdwicSubmitPmuHeader) == 128 + 64 * kFdwicSubmitPmuExpectedCores, "submit-PMU ABI size changed"); + +constexpr size_t kFdwicSubmitPmuNoneBytes = sizeof(FdwicSubmitPmuHeader); +constexpr size_t kFdwicSubmitPmuPhaseBytes = + sizeof(FdwicSubmitPmuHeader) + sizeof(FdwicSubmitPmuPhaseCoreData) * kFdwicSubmitPmuExpectedCores; + +constexpr size_t fdwic_submit_pmu_bytes_for_mode(uint16_t mode) { + return fdwic_submit_pmu_mode_has_phase(mode) ? kFdwicSubmitPmuPhaseBytes : kFdwicSubmitPmuNoneBytes; +} diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/swimlane.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/swimlane.h index 2a8ba2266d..8e4ba7a5a2 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/swimlane.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/swimlane.h @@ -16,6 +16,7 @@ #include "dist_engine/common/worker_state.h" #include "dist_engine/common/atomic.h" #include "dist_engine/aicore/primitive.h" +#include "dist_engine/common/submit_pmu.h" #if defined(__CCE_AICORE__) || defined(__CPU_SIM) #include "inner_kernel.h" @@ -23,6 +24,28 @@ namespace { +template +PTO_DEVICE_FUNC inline uint64_t fdwic_atomic_result_ready_tick(T value) { +#if defined(__CCE_AICORE__) + static_assert(sizeof(T) == 4 || sizeof(T) == 8, "atomic dependency expects a scalar result"); + uint64_t cycle = 0; + // 在读取 SYS_CNT 的同一汇编块中消费 atomic 返回值，形成局部 + // return-ready 边界；这不是跨核可见性屏障，也不会引入 DSB。 + asm volatile("MOV %0, %0\n" + "MOV %1, SYS_CNT\n" + : "+l"(value), "=&l"(cycle)); + return cycle; +#elif defined(__CPU_SIM) + (void)value; + return get_sys_cnt_aicore(); +#else + (void)value; + return 0; +#endif +} + +#if DIST_TRACE_ENABLED + PTO_DEVICE_FUNC inline uint64_t fdwic_swimlane_detail_now() { #if defined(__CCE_AICORE__) || defined(__CPU_SIM) return get_sys_cnt_aicore(); @@ -31,24 +54,51 @@ PTO_DEVICE_FUNC inline uint64_t fdwic_swimlane_detail_now() { #endif } -PTO_DEVICE_FUNC inline bool fdwic_swimlane_enabled() { return g_fdwic_swimlane_enabled; } +PTO_DEVICE_FUNC inline bool fdwic_swimlane_enabled() { return g_fdwic_swimlane_level != 0; } + +PTO_DEVICE_FUNC inline bool fdwic_atomic_swimlane_enabled() { + return g_fdwic_swimlane_level >= kFdwicAtomicSwimlaneLevel; +} + +PTO_DEVICE_FUNC inline bool fdwic_atomic_return_ready_observed() { +#if defined(__CCE_AICORE__) + return true; +#else + // CPU simulation validates the scheduler and raw schema, not A5 atomic + // completion timing. Do not claim a hardware return-ready boundary there. + return false; +#endif +} PTO_DEVICE_FUNC inline void fdwic_swimlane_attach(__gm__ Runtime *runtime) { - g_fdwic_swimlane_enabled = false; + g_fdwic_swimlane_level = 0; g_fdwic_swimlane_header = nullptr; g_fdwic_swimlane_core = nullptr; g_fdwic_swimlane_records = nullptr; g_fdwic_swimlane_records_per_core = 0; + g_fdwic_atomic_poll_burst.active_mask = 0; + g_fdwic_atomic_poll_burst.enabled_mask = 0; + g_fdwic_atomic_calls = 0; + g_fdwic_poll_calls = 0; + g_fdwic_poll_batch_records = 0; + g_fdwic_atomic_counter_overflow = false; if (runtime == nullptr) return; #if defined(__CCE_AICORE__) dist_aicore_invalidate_region(const_cast<__gm__ uint64_t *>(&runtime->dist.swimlane_base), 64); #endif const uint64_t base = runtime->dist.swimlane_base; + const uint32_t level = runtime->dist.swimlane_level; const uint32_t records_per_core = runtime->dist.swimlane_records_per_core; - if (runtime->dist.swimlane_enabled == 0 || base == 0 || records_per_core == 0) return; + if (level == 0 || base == 0 || records_per_core == 0) return; g_fdwic_swimlane_header = reinterpret_cast<__gm__ FdwicSwimlaneHeader *>(base); +#if defined(__CCE_AICORE__) + // The host initializes this cache line before launching the kernel. Drop a + // possibly stale line left by a previous allocation at the same GM address. + // Do not invalidate the whole header: other cores update their own states. + dist_aicore_invalidate_region(g_fdwic_swimlane_header, 64); +#endif g_fdwic_swimlane_records_per_core = records_per_core; - g_fdwic_swimlane_enabled = true; + g_fdwic_swimlane_level = level; } PTO_DEVICE_FUNC inline __gm__ FdwicSwimlaneRecord *fdwic_swimlane_detail_records(__gm__ FdwicSwimlaneHeader *header) { @@ -63,65 +113,443 @@ PTO_DEVICE_FUNC inline void fdwic_swimlane_reset_core(__gm__ DistCore *self) { if (header == nullptr) return; if (self->core_idx < 0 || self->core_idx >= static_cast(header->num_cores)) return; g_fdwic_swimlane_core = &header->cores[self->core_idx]; - g_fdwic_swimlane_records = &fdwic_swimlane_detail_records( - header - )[static_cast(self->core_idx) * g_fdwic_swimlane_records_per_core]; + __gm__ FdwicSwimlaneRecord *records = fdwic_swimlane_detail_records(header); + const uint64_t record_offset = static_cast(self->core_idx) * g_fdwic_swimlane_records_per_core; + g_fdwic_swimlane_records = records + record_offset; g_fdwic_swimlane_core->count = 0; g_fdwic_swimlane_core->dropped = 0; + g_fdwic_swimlane_core->atomic_calls = 0; + g_fdwic_swimlane_core->poll_calls = 0; + g_fdwic_swimlane_core->poll_batch_records = 0; + g_fdwic_swimlane_core->core_idx = self->core_idx; + g_fdwic_swimlane_core->block_id = self->block_id; + g_fdwic_swimlane_core->lane = self->lane; + g_fdwic_atomic_poll_burst.active_mask = 0; + g_fdwic_atomic_poll_burst.enabled_mask = 0; + g_fdwic_atomic_calls = 0; + g_fdwic_poll_calls = 0; + g_fdwic_poll_batch_records = 0; + g_fdwic_atomic_counter_overflow = false; } -PTO_DEVICE_FUNC inline void fdwic_swimlane_flush_core(__gm__ DistCore *self) { - if (!fdwic_swimlane_enabled() || self == nullptr) return; +PTO_DEVICE_FUNC inline bool fdwic_swimlane_detail_write_record( + __gm__ DistCore *self, int32_t task_id, int32_t func_id, FdwicSwimlanePhase phase, uint64_t start_cycle, + uint64_t end_cycle, uint32_t flags, uint32_t aux +) { + if (!fdwic_swimlane_enabled() || self == nullptr) return false; const uint32_t records_per_core = g_fdwic_swimlane_records_per_core; __gm__ FdwicSwimlaneCoreState *core = g_fdwic_swimlane_core; - if (core == nullptr || g_fdwic_swimlane_records == nullptr || records_per_core == 0) return; - const uint32_t count = core->count < records_per_core ? core->count : records_per_core; - if (count > 0) { - dist_aicore_flush_region(g_fdwic_swimlane_records, static_cast(count) * sizeof(FdwicSwimlaneRecord)); + if (core == nullptr || g_fdwic_swimlane_records == nullptr || records_per_core == 0) return false; + const uint32_t slot = core->count; + if (slot >= records_per_core) { + if (core->dropped != UINT32_MAX) core->dropped = core->dropped + 1; + return false; } - dist_aicore_flush_region(core, sizeof(FdwicSwimlaneCoreState)); + __gm__ FdwicSwimlaneRecord *record = &g_fdwic_swimlane_records[slot]; + record->start_cycle = start_cycle; + record->end_cycle = end_cycle; + record->task_id = task_id; + record->func_id = func_id; + record->flags = flags; + record->phase = static_cast(phase); + record->aux = static_cast(aux); + core->count = slot + 1; + return true; } PTO_DEVICE_FUNC inline void fdwic_swimlane_detail_record( __gm__ DistCore *self, int32_t task_id, int32_t func_id, FdwicSwimlanePhase phase, uint64_t start_cycle, uint64_t end_cycle, uint32_t flags = 0, uint32_t aux = 0 ) { + (void)fdwic_swimlane_detail_write_record(self, task_id, func_id, phase, start_cycle, end_cycle, flags, aux); +} + +PTO_DEVICE_FUNC inline uint32_t fdwic_atomic_trace_flags( + FdwicAtomicOp op, bool result_used, bool return_ready, bool value_zero = false, uint64_t retries = 0 +) { + constexpr uint64_t kMaxRetries = (1ULL << (32 - kFdwicAtomicRetriesShift)) - 1; + const uint32_t encoded_retries = static_cast(retries > kMaxRetries ? kMaxRetries : retries); + return static_cast(op) | (result_used ? kFdwicAtomicResultUsed : 0U) | + (value_zero ? kFdwicAtomicValueZero : 0U) | (return_ready ? kFdwicAtomicReturnReady : 0U) | + (encoded_retries << kFdwicAtomicRetriesShift); +} + +PTO_DEVICE_FUNC inline uint32_t fdwic_atomic_poll_trace_flags(FdwicAtomicSite site, uint32_t call_count) { + return static_cast(fdwic_atomic_site_op(site)) | kFdwicAtomicResultUsed | kFdwicAtomicPollBatch | + (call_count << kFdwicAtomicPollCountShift); +} + +PTO_DEVICE_FUNC inline uint32_t fdwic_atomic_site_mask(FdwicAtomicSite site) { + return 1U << static_cast(site); +} + +PTO_DEVICE_FUNC inline uint32_t fdwic_atomic_block_won_poll_mask() { + return fdwic_atomic_site_mask(FdwicAtomicSite::WonAnyLoad) | fdwic_atomic_site_mask(FdwicAtomicSite::WonStateLoad) | + fdwic_atomic_site_mask(FdwicAtomicSite::WonLaneClaimExchange) | + fdwic_atomic_site_mask(FdwicAtomicSite::WonDrainedLoad); +} + +PTO_DEVICE_FUNC inline void fdwic_swimlane_count_atomic_call(bool poll_batch) { + if (g_fdwic_atomic_calls == UINT32_MAX) { + g_fdwic_atomic_counter_overflow = true; + return; + } + g_fdwic_atomic_calls++; + if (!poll_batch) return; + if (g_fdwic_poll_calls == UINT32_MAX) { + g_fdwic_atomic_counter_overflow = true; + return; + } + g_fdwic_poll_calls++; +} + +// 调用方已经在边界处取得 end_cycle；十类 PollBatch 的遍历与落盘只在 +// level-4 且确有活动批次时需要。把慢体共享起来，避免它被每个 phase/lap +// 边界重复内联，同时不让 level-1 快路径承担函数调用。 +PTO_DEVICE_FUNC __attribute__((noinline)) void fdwic_atomic_poll_boundary_slow(uint64_t end_cycle) { + __gm__ DistCore *self = g_self; + if (self == nullptr || g_fdwic_swimlane_core == nullptr) return; + const uint32_t active_mask = g_fdwic_atomic_poll_burst.active_mask; + for (uint32_t batch_index = 0; batch_index < kFdwicAtomicPollBatchSiteCount; ++batch_index) { + const uint32_t bit = 1U << batch_index; + if ((active_mask & bit) == 0) continue; + const uint32_t call_count = g_fdwic_atomic_poll_burst.call_count[batch_index]; + if (call_count == 0 || call_count > kFdwicAtomicPollCountMax) { + g_fdwic_atomic_counter_overflow = true; + continue; + } + const FdwicAtomicSite site = fdwic_atomic_poll_batch_site(batch_index); + const uint32_t site_index = static_cast(site); + const bool written = fdwic_swimlane_detail_write_record( + self, -1, -1, FdwicSwimlanePhase::Atomic, g_fdwic_atomic_poll_burst.start_cycle[batch_index], end_cycle, + fdwic_atomic_poll_trace_flags(site, call_count), site_index + ); + if (written) { + if (g_fdwic_poll_batch_records == UINT32_MAX) { + g_fdwic_atomic_counter_overflow = true; + } else { + g_fdwic_poll_batch_records++; + } + } + g_fdwic_atomic_poll_burst.call_count[batch_index] = 0; + } + g_fdwic_atomic_poll_burst.active_mask = 0; +} + +PTO_DEVICE_FUNC inline void fdwic_atomic_poll_boundary_at(uint64_t end_cycle) { + if (!fdwic_atomic_swimlane_enabled() || g_fdwic_atomic_poll_burst.active_mask == 0) return; + fdwic_atomic_poll_boundary_slow(end_cycle); +} + +PTO_DEVICE_FUNC inline void fdwic_atomic_poll_boundary() { + if (g_fdwic_atomic_poll_burst.active_mask == 0) return; + fdwic_atomic_poll_boundary_at(fdwic_swimlane_detail_now()); +} + +PTO_DEVICE_FUNC inline uint32_t fdwic_atomic_poll_region_begin(uint32_t site_mask) { + const uint32_t previous_mask = g_fdwic_atomic_poll_burst.enabled_mask; + if (!fdwic_atomic_swimlane_enabled()) return previous_mask; + fdwic_atomic_poll_boundary(); + g_fdwic_atomic_poll_burst.enabled_mask = previous_mask | site_mask; + return previous_mask; +} + +PTO_DEVICE_FUNC inline void fdwic_atomic_poll_region_end(uint32_t previous_mask) { + if (!fdwic_atomic_swimlane_enabled()) return; + fdwic_atomic_poll_boundary(); + g_fdwic_atomic_poll_burst.enabled_mask = previous_mask; +} + +PTO_DEVICE_FUNC inline bool fdwic_atomic_poll_batch_enabled(FdwicAtomicSite site, FdwicAtomicOp actual_op) { + if (!fdwic_atomic_site_is_poll_batchable(site) || fdwic_atomic_site_op(site) != actual_op) return false; + return (g_fdwic_atomic_poll_burst.enabled_mask & fdwic_atomic_site_mask(site)) != 0; +} + +PTO_DEVICE_FUNC inline void fdwic_swimlane_accumulate_poll_call(FdwicAtomicSite site, uint64_t start_cycle) { + const int32_t batch_index = fdwic_atomic_poll_batch_index(site); + if (batch_index < 0) { + g_fdwic_atomic_counter_overflow = true; + return; + } + const uint32_t bit = 1U << static_cast(batch_index); + if ((g_fdwic_atomic_poll_burst.active_mask & bit) == 0) { + g_fdwic_atomic_poll_burst.start_cycle[batch_index] = start_cycle; + g_fdwic_atomic_poll_burst.call_count[batch_index] = 0; + g_fdwic_atomic_poll_burst.active_mask |= bit; + } + uint32_t &call_count = g_fdwic_atomic_poll_burst.call_count[batch_index]; + call_count++; + if (call_count == kFdwicAtomicPollCountMax) fdwic_atomic_poll_boundary(); +} + +PTO_DEVICE_FUNC inline void fdwic_swimlane_flush_core(__gm__ DistCore *self) { if (!fdwic_swimlane_enabled() || self == nullptr) return; - const int32_t core_idx = self->core_idx; + fdwic_atomic_poll_boundary(); const uint32_t records_per_core = g_fdwic_swimlane_records_per_core; __gm__ FdwicSwimlaneCoreState *core = g_fdwic_swimlane_core; if (core == nullptr || g_fdwic_swimlane_records == nullptr || records_per_core == 0) return; - uint32_t slot = core->count; - if (slot >= records_per_core) { - core->dropped = core->dropped + 1; - return; + core->atomic_calls = g_fdwic_atomic_calls; + core->poll_calls = g_fdwic_poll_calls; + core->poll_batch_records = g_fdwic_poll_batch_records; + if (g_fdwic_atomic_counter_overflow && core->dropped != UINT32_MAX) core->dropped = core->dropped + 1; + const uint32_t count = core->count < records_per_core ? core->count : records_per_core; + if (count > 0) { + dist_aicore_flush_region(g_fdwic_swimlane_records, static_cast(count) * sizeof(FdwicSwimlaneRecord)); } - __gm__ FdwicSwimlaneRecord *record = &g_fdwic_swimlane_records[slot]; - record->start_cycle = start_cycle; - record->end_cycle = end_cycle; - record->task_id = task_id; - record->func_id = func_id; - record->phase = static_cast(phase); - record->lane = self->lane; - record->block_id = self->block_id; - record->core_idx = core_idx; - record->flags = flags; - record->aux = aux; - core->count = slot + 1; + dist_aicore_flush_region(core, sizeof(FdwicSwimlaneCoreState)); +} + +// Atomic 记录落盘是 level-4 诊断冷路径。保持它为单一设备函数，避免完整的 +// GM 边界检查与记录写入被复制到每一个 atomic 调用点；level-1 快路径在各 +// wrapper 的首个分支已经返回，不会承担这里的 call/ret。 +PTO_DEVICE_FUNC __attribute__((noinline)) void fdwic_swimlane_detail_record_atomic( + int32_t task_id, FdwicAtomicSite site, FdwicAtomicOp op, uint64_t start_cycle, uint64_t end_cycle, bool result_used, + bool return_ready, bool value_zero = false, uint64_t retries = 0 +) { + __gm__ DistCore *self = g_self; + if (!fdwic_atomic_swimlane_enabled() || self == nullptr || g_fdwic_swimlane_core == nullptr) return; + fdwic_swimlane_detail_record( + self, task_id, -1, FdwicSwimlanePhase::Atomic, start_cycle, end_cycle, + fdwic_atomic_trace_flags(op, result_used, return_ready, value_zero, retries), static_cast(site) + ); +} + +// Direct atomics remain one row per source call but do not split an active +// PollBatch. A batch is a logical wait-region window and may contain these +// interleaved rows; only its call_count, not its duration, represents atomic +// work. Region/phase/lap/final boundaries still close every active batch. + +template +PTO_DEVICE_FUNC inline T fdwic_trace_atomic_load( + int32_t task_id, FdwicAtomicSite site, __gm__ volatile T &value, bool result_used = true, + int memorder = __ATOMIC_ACQUIRE +) { + if (!fdwic_atomic_swimlane_enabled()) return atomic_load(value, memorder); + const bool poll_batch = result_used && fdwic_atomic_poll_batch_enabled(site, FdwicAtomicOp::Load); + const int32_t batch_index = poll_batch ? fdwic_atomic_poll_batch_index(site) : -1; + const bool first_in_batch = + poll_batch && (g_fdwic_atomic_poll_burst.active_mask & (1U << static_cast(batch_index))) == 0; + const uint64_t begin = !poll_batch || first_in_batch ? fdwic_swimlane_detail_now() : 0; + const T old = atomic_load(value, memorder); + if (poll_batch) { + fdwic_swimlane_count_atomic_call(true); + fdwic_swimlane_accumulate_poll_call(site, begin); + return old; + } + const bool return_ready = result_used && fdwic_atomic_return_ready_observed(); + const uint64_t end = result_used ? fdwic_atomic_result_ready_tick(old) : fdwic_swimlane_detail_now(); + // Keep tracing bookkeeping outside the measured direct-atomic boundary. + fdwic_swimlane_count_atomic_call(false); + fdwic_swimlane_detail_record_atomic( + task_id, site, FdwicAtomicOp::Load, begin, end, result_used, return_ready, old == static_cast(0) + ); + return old; +} + +template +PTO_DEVICE_FUNC inline T fdwic_trace_atomic_exchange( + int32_t task_id, FdwicAtomicSite site, __gm__ volatile T &value, V desired, bool result_used = false, + int memorder = __ATOMIC_ACQ_REL +) { + if (!fdwic_atomic_swimlane_enabled()) return atomic_exchange(value, desired, memorder); + const T desired_value = static_cast(desired); + const bool failed_claim_batch_enabled = result_used && site == FdwicAtomicSite::WonLaneClaimExchange && + desired_value == static_cast(kDrainedClaimed) && + fdwic_atomic_poll_batch_enabled(site, FdwicAtomicOp::Exchange); + const uint64_t begin = fdwic_swimlane_detail_now(); + const T old = atomic_exchange(value, desired, memorder); + const bool failed_claim_batch = failed_claim_batch_enabled && old == desired_value; + if (failed_claim_batch) { + fdwic_swimlane_count_atomic_call(true); + fdwic_swimlane_accumulate_poll_call(site, begin); + return old; + } + const bool return_ready = result_used && fdwic_atomic_return_ready_observed(); + const uint64_t end = result_used ? fdwic_atomic_result_ready_tick(old) : fdwic_swimlane_detail_now(); + fdwic_swimlane_count_atomic_call(false); + // Close failed retries at the successful transition's issue boundary. The + // successful claim itself remains an exact direct row. Capture its end + // first so batch-record writes are not charged to the direct span. + if (failed_claim_batch_enabled) fdwic_atomic_poll_boundary_at(begin); + fdwic_swimlane_detail_record_atomic(task_id, site, FdwicAtomicOp::Exchange, begin, end, result_used, return_ready); + return old; +} + +template +PTO_DEVICE_FUNC inline T fdwic_trace_atomic_fetch_add( + int32_t task_id, FdwicAtomicSite site, __gm__ volatile T &value, T delta, bool result_used = false, + int memorder = __ATOMIC_ACQ_REL +) { + if (!fdwic_atomic_swimlane_enabled()) return atomic_fetch_add(value, delta, memorder); + const uint64_t begin = fdwic_swimlane_detail_now(); + const T old = atomic_fetch_add(value, delta, memorder); + const bool return_ready = result_used && fdwic_atomic_return_ready_observed(); + const uint64_t end = result_used ? fdwic_atomic_result_ready_tick(old) : fdwic_swimlane_detail_now(); + fdwic_swimlane_count_atomic_call(false); + fdwic_swimlane_detail_record_atomic(task_id, site, FdwicAtomicOp::FetchAdd, begin, end, result_used, return_ready); + return old; +} + +template +PTO_DEVICE_FUNC inline T fdwic_trace_atomic_fetch_sub( + int32_t task_id, FdwicAtomicSite site, __gm__ volatile T &value, T delta, bool result_used = false, + int memorder = __ATOMIC_ACQ_REL +) { + if (!fdwic_atomic_swimlane_enabled()) return atomic_fetch_sub(value, delta, memorder); + const uint64_t begin = fdwic_swimlane_detail_now(); + const T old = atomic_fetch_sub(value, delta, memorder); + const bool return_ready = result_used && fdwic_atomic_return_ready_observed(); + const uint64_t end = result_used ? fdwic_atomic_result_ready_tick(old) : fdwic_swimlane_detail_now(); + fdwic_swimlane_count_atomic_call(false); + fdwic_swimlane_detail_record_atomic(task_id, site, FdwicAtomicOp::FetchSub, begin, end, result_used, return_ready); + return old; +} + +template +PTO_DEVICE_FUNC inline T fdwic_trace_atomic_fetch_max( + int32_t task_id, FdwicAtomicSite site, __gm__ volatile T &value, T desired, bool result_used = true, + int memorder = __ATOMIC_ACQ_REL +) { + if (!fdwic_atomic_swimlane_enabled()) return atomic_fetch_max(value, desired, memorder); + const uint64_t begin = fdwic_swimlane_detail_now(); + const T old = atomic_fetch_max(value, desired, memorder); + const bool return_ready = result_used && fdwic_atomic_return_ready_observed(); + const uint64_t end = result_used ? fdwic_atomic_result_ready_tick(old) : fdwic_swimlane_detail_now(); + fdwic_swimlane_count_atomic_call(false); + fdwic_swimlane_detail_record_atomic(task_id, site, FdwicAtomicOp::FetchMax, begin, end, result_used, return_ready); + return old; +} + +PTO_DEVICE_FUNC inline bool fdwic_trace_is_fatal(int32_t task_id = -1) { + return fdwic_trace_atomic_load(task_id, FdwicAtomicSite::FatalPoll, g_dist.fatal) != 0; +} + +PTO_DEVICE_FUNC inline void fdwic_trace_set_fatal(int32_t task_id = -1) { + const int32_t previous = fdwic_trace_atomic_exchange( + task_id, FdwicAtomicSite::FatalSet, g_dist.fatal, int32_t{1}, /*result_used=*/false + ); + (void)previous; +} + +PTO_DEVICE_FUNC inline void fdwic_swimlane_record_clock_baselines(__gm__ DistCore *self, int32_t dependency_value) { + if (!fdwic_atomic_swimlane_enabled() || self == nullptr) return; + fdwic_atomic_poll_boundary(); + const uint64_t clock_begin = fdwic_swimlane_detail_now(); + const uint64_t clock_end = fdwic_swimlane_detail_now(); + fdwic_swimlane_detail_record(self, -1, -1, FdwicSwimlanePhase::ClockBaseline, clock_begin, clock_end); + const uint64_t dependency_begin = fdwic_swimlane_detail_now(); + const uint64_t dependency_end = fdwic_atomic_result_ready_tick(dependency_value); + const uint32_t flags = + kFdwicClockAtomicDependency | (fdwic_atomic_return_ready_observed() ? kFdwicClockAtomicDependencyApplied : 0U); + fdwic_swimlane_detail_record( + self, -1, -1, FdwicSwimlanePhase::ClockBaseline, dependency_begin, dependency_end, flags + ); } PTO_DEVICE_FUNC inline void fdwic_swimlane_lap_reset(__gm__ DistCore *self) { if (self == nullptr) return; - self->swimlane_last_cycle = fdwic_swimlane_detail_now(); + const uint64_t cycle = fdwic_swimlane_detail_now(); + fdwic_atomic_poll_boundary_at(cycle); + self->swimlane_last_cycle = cycle; } PTO_DEVICE_FUNC inline void fdwic_swimlane_lap(__gm__ DistCore *self, int32_t task_id, int32_t func_id, FdwicSwimlanePhase phase) { if (self == nullptr) return; const uint64_t end_cycle = fdwic_swimlane_detail_now(); + fdwic_atomic_poll_boundary_at(end_cycle); const uint64_t start_cycle = self->swimlane_last_cycle; fdwic_swimlane_detail_record(self, task_id, func_id, phase, start_cycle, end_cycle); self->swimlane_last_cycle = end_cycle; } +#else + +// 无泳道构建必须在编译期回到原始 atomic，而不是只让运行时 level=0。 +// 这样 perf-clock / submit-pmu ELF 不携带 record 分支、轮询聚合状态或 +// atomic 观察慢体，避免诊断代码布局反过来污染权威性能基线。 +PTO_DEVICE_FUNC inline void fdwic_swimlane_attach(__gm__ Runtime *) {} + +PTO_DEVICE_FUNC constexpr uint32_t fdwic_atomic_site_mask(FdwicAtomicSite site) { + return 1U << static_cast(site); +} + +PTO_DEVICE_FUNC inline uint32_t fdwic_atomic_block_won_poll_mask() { return 0; } +PTO_DEVICE_FUNC inline uint32_t fdwic_atomic_poll_region_begin(uint32_t) { return 0; } +PTO_DEVICE_FUNC inline void fdwic_atomic_poll_region_end(uint32_t) {} +PTO_DEVICE_FUNC inline void fdwic_atomic_poll_boundary() {} + +template +PTO_DEVICE_FUNC inline T fdwic_trace_atomic_load( + int32_t, FdwicAtomicSite site, __gm__ volatile T &value, bool result_used = true, int memorder = __ATOMIC_ACQUIRE +) { + const bool observe_return_ready = result_used && fdwic_atomic_site_result_used(site); + const uint32_t token = observe_return_ready ? fdwic_submit_pmu_return_ready_atomic_begin() : 0; + const T old = atomic_load(value, memorder); + if (token != 0) fdwic_submit_pmu_return_ready_atomic_end(token, fdwic_atomic_result_ready_tick(old)); + return old; +} + +template +PTO_DEVICE_FUNC inline T fdwic_trace_atomic_exchange( + int32_t, FdwicAtomicSite site, __gm__ volatile T &value, V desired, bool result_used = false, + int memorder = __ATOMIC_ACQ_REL +) { + const bool observe_return_ready = result_used && fdwic_atomic_site_result_used(site); + const uint32_t token = observe_return_ready ? fdwic_submit_pmu_return_ready_atomic_begin() : 0; + const T old = atomic_exchange(value, static_cast(desired), memorder); + if (token != 0) fdwic_submit_pmu_return_ready_atomic_end(token, fdwic_atomic_result_ready_tick(old)); + return old; +} + +template +PTO_DEVICE_FUNC inline T fdwic_trace_atomic_fetch_add( + int32_t, FdwicAtomicSite site, __gm__ volatile T &value, T delta, bool result_used = false, + int memorder = __ATOMIC_ACQ_REL +) { + const bool observe_return_ready = result_used && fdwic_atomic_site_result_used(site); + const uint32_t token = observe_return_ready ? fdwic_submit_pmu_return_ready_atomic_begin() : 0; + const T old = atomic_fetch_add(value, delta, memorder); + if (token != 0) fdwic_submit_pmu_return_ready_atomic_end(token, fdwic_atomic_result_ready_tick(old)); + return old; +} + +template +PTO_DEVICE_FUNC inline T fdwic_trace_atomic_fetch_sub( + int32_t, FdwicAtomicSite site, __gm__ volatile T &value, T delta, bool result_used = false, + int memorder = __ATOMIC_ACQ_REL +) { + const bool observe_return_ready = result_used && fdwic_atomic_site_result_used(site); + const uint32_t token = observe_return_ready ? fdwic_submit_pmu_return_ready_atomic_begin() : 0; + const T old = atomic_fetch_sub(value, delta, memorder); + if (token != 0) fdwic_submit_pmu_return_ready_atomic_end(token, fdwic_atomic_result_ready_tick(old)); + return old; +} + +template +PTO_DEVICE_FUNC inline T fdwic_trace_atomic_fetch_max( + int32_t, FdwicAtomicSite site, __gm__ volatile T &value, T desired, bool result_used = true, + int memorder = __ATOMIC_ACQ_REL +) { + const bool observe_return_ready = result_used && fdwic_atomic_site_result_used(site); + const uint32_t token = observe_return_ready ? fdwic_submit_pmu_return_ready_atomic_begin() : 0; + const T old = atomic_fetch_max(value, desired, memorder); + if (token != 0) fdwic_submit_pmu_return_ready_atomic_end(token, fdwic_atomic_result_ready_tick(old)); + return old; +} + +PTO_DEVICE_FUNC inline bool fdwic_trace_is_fatal(int32_t = -1) { + return fdwic_trace_atomic_load(-1, FdwicAtomicSite::FatalPoll, g_dist.fatal) != 0; +} + +PTO_DEVICE_FUNC inline void fdwic_trace_set_fatal(int32_t = -1) { + (void)fdwic_trace_atomic_exchange( + -1, FdwicAtomicSite::FatalSet, g_dist.fatal, int32_t{1}, /*result_used=*/false, __ATOMIC_ACQ_REL + ); +} + +PTO_DEVICE_FUNC inline void fdwic_swimlane_record_clock_baselines(__gm__ DistCore *, int32_t) {} + +#endif + } // namespace diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/swimlane_types.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/swimlane_types.h index dddfc3779a..60a8340017 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/swimlane_types.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/swimlane_types.h @@ -11,11 +11,25 @@ #pragma once +#include #include +#include "data_type.h" + constexpr uint32_t kFdwicSwimlaneMagic = 0x4653574Cu; // FSWL -constexpr uint32_t kFdwicSwimlaneVersion = 1; +constexpr uint32_t kFdwicSwimlaneVersion = 4; +constexpr uint32_t kFdwicSwimlaneTraceSchemaVersion = 4; constexpr uint32_t kFdwicSwimlaneDefaultRecordsPerCore = 1u << 16; +// Eligible wait-region atomic calls are aggregated into exact-count batches at +// level 4. Reuse the existing 64K partition instead of reserving hundreds of +// thousands of rows per worker for individual spin iterations. +constexpr uint32_t kFdwicAtomicSwimlaneRecordsPerCore = kFdwicSwimlaneDefaultRecordsPerCore; +constexpr uint32_t kFdwicAtomicSwimlaneLevel = 4; +constexpr uint32_t kFdwicPerfClockMode = 1; +constexpr uint32_t kFdwicPerfClockKernelMode = 2; +static_assert( + kFdwicAtomicSwimlaneRecordsPerCore % 2 == 0, "32B record partitions must keep every worker base on a 64B boundary" +); enum class FdwicSwimlanePhase : int32_t { Kernel = 0, @@ -32,15 +46,315 @@ enum class FdwicSwimlanePhase : int32_t { Claim = 11, Fanin = 12, Register = 13, + Atomic = 14, + ClockBaseline = 15, + // Schema-v4 parent intervals and true Submit-tail actions. The legacy + // Alloc/Build/Replay IDs stay reserved for archived captures but are no + // longer emitted by the production runtime. + OrchestrationReplay = 16, + FinalDrain = 17, + WinnerBuild = 18, + AllocComplete = 19, + // Unlike the single-lane standalone probe, a production kernel loser + // really calls drain_block_won(); keep that work as an exclusive child. + LoserReplay = 20, + Count = 21, +}; + +// Atomic/ClockBaseline extend the existing ten-column FDWIC raw record ABI. +// The first fifteen sites intentionally keep the standalone PA probe's stable +// numbering; production-only BlockWon sites are appended and must not reorder +// those existing values. +enum class FdwicAtomicSite : uint32_t { + StartupIncrement = 0, + StartupPoll = 1, + FatalPoll = 2, + FatalSet = 3, + ClaimMax = 4, + FaninFlagLoad = 5, + CompletionVendExchange = 6, + CompletionFlagExchange = 7, + FrontierInitialLoad = 8, + FrontierFlagLoad = 9, + FrontierMax = 10, + HeapFrontierLoad = 11, + HeapVendLoad = 12, + ReplayDoneIncrement = 13, + ReplayDonePoll = 14, + WonSlotClaimMax = 15, + WonRemainingExchange = 16, + WonLaneResetExchange = 17, + WonLaneDepositExchange = 18, + WonStatePublishExchange = 19, + WonAnyPublishExchange = 20, + WonAnyLoad = 21, + WonStateLoad = 22, + WonLaneClaimExchange = 23, + WonLaneReleaseExchange = 24, + WonRemainingFetchSub = 25, + WonStateClearExchange = 26, + WonDrainedLoad = 27, + Count = 28, }; +enum class FdwicAtomicOp : uint32_t { + Load = 0, + Exchange = 1, + FetchAdd = 2, + FetchMax = 3, + FetchSub = 4, +}; + +static_assert( + static_cast(FdwicAtomicSite::Count) <= (1U << 16), "atomic site id must fit the compact record" +); +static_assert( + static_cast(FdwicAtomicSite::Count) <= 32, "atomic site id must fit the 32-bit poll-region site mask" +); + +constexpr uint32_t kFdwicAtomicOpMask = 0x0fU; +constexpr uint32_t kFdwicAtomicResultUsed = 1U << 4; +constexpr uint32_t kFdwicAtomicValueZero = 1U << 5; +constexpr uint32_t kFdwicAtomicReturnReady = 1U << 6; +constexpr uint32_t kFdwicAtomicPollBatch = 1U << 7; +constexpr uint32_t kFdwicAtomicRetriesShift = 8; +constexpr uint32_t kFdwicAtomicPollCountShift = 8; +constexpr uint32_t kFdwicAtomicPollCountMax = (1U << (32 - kFdwicAtomicPollCountShift)) - 1; + +constexpr uint32_t kFdwicClockAtomicDependency = 1U << 0; +constexpr uint32_t kFdwicClockAtomicDependencyApplied = 1U << 1; + +constexpr uint32_t kFdwicClaimWon = 1U << 0; +constexpr uint32_t kFdwicClaimAttempted = 1U << 1; + +PTO_DEVICE_FUNC constexpr FdwicAtomicOp fdwic_atomic_site_op(FdwicAtomicSite site) { + switch (site) { + case FdwicAtomicSite::StartupIncrement: + case FdwicAtomicSite::ReplayDoneIncrement: + return FdwicAtomicOp::FetchAdd; + case FdwicAtomicSite::FatalSet: + case FdwicAtomicSite::CompletionVendExchange: + case FdwicAtomicSite::CompletionFlagExchange: + case FdwicAtomicSite::WonRemainingExchange: + case FdwicAtomicSite::WonLaneResetExchange: + case FdwicAtomicSite::WonLaneDepositExchange: + case FdwicAtomicSite::WonStatePublishExchange: + case FdwicAtomicSite::WonAnyPublishExchange: + case FdwicAtomicSite::WonLaneClaimExchange: + case FdwicAtomicSite::WonLaneReleaseExchange: + case FdwicAtomicSite::WonStateClearExchange: + return FdwicAtomicOp::Exchange; + case FdwicAtomicSite::ClaimMax: + case FdwicAtomicSite::FrontierMax: + case FdwicAtomicSite::WonSlotClaimMax: + return FdwicAtomicOp::FetchMax; + case FdwicAtomicSite::WonRemainingFetchSub: + return FdwicAtomicOp::FetchSub; + default: + return FdwicAtomicOp::Load; + } +} + +PTO_DEVICE_FUNC constexpr bool fdwic_atomic_site_result_used(FdwicAtomicSite site) { + switch (site) { + case FdwicAtomicSite::StartupPoll: + case FdwicAtomicSite::FatalPoll: + case FdwicAtomicSite::ClaimMax: + case FdwicAtomicSite::FaninFlagLoad: + case FdwicAtomicSite::FrontierInitialLoad: + case FdwicAtomicSite::FrontierFlagLoad: + case FdwicAtomicSite::FrontierMax: + case FdwicAtomicSite::HeapFrontierLoad: + case FdwicAtomicSite::HeapVendLoad: + case FdwicAtomicSite::ReplayDonePoll: + case FdwicAtomicSite::WonSlotClaimMax: + case FdwicAtomicSite::WonAnyLoad: + case FdwicAtomicSite::WonStateLoad: + case FdwicAtomicSite::WonLaneClaimExchange: + case FdwicAtomicSite::WonRemainingFetchSub: + case FdwicAtomicSite::WonDrainedLoad: + return true; + case FdwicAtomicSite::StartupIncrement: + case FdwicAtomicSite::FatalSet: + case FdwicAtomicSite::CompletionVendExchange: + case FdwicAtomicSite::CompletionFlagExchange: + case FdwicAtomicSite::ReplayDoneIncrement: + case FdwicAtomicSite::WonRemainingExchange: + case FdwicAtomicSite::WonLaneResetExchange: + case FdwicAtomicSite::WonLaneDepositExchange: + case FdwicAtomicSite::WonStatePublishExchange: + case FdwicAtomicSite::WonAnyPublishExchange: + case FdwicAtomicSite::WonLaneReleaseExchange: + case FdwicAtomicSite::WonStateClearExchange: + case FdwicAtomicSite::Count: + return false; + } + return false; +} + +constexpr uint32_t kFdwicAtomicReturnReadySiteCount = 16; +constexpr uint32_t kFdwicAtomicSourceIssueSiteCount = 12; +static_assert( + kFdwicAtomicReturnReadySiteCount + kFdwicAtomicSourceIssueSiteCount == + static_cast(FdwicAtomicSite::Count), + "every atomic site must have one explicit completion classification" +); + +// Observation loads used by explicit scheduler wait regions are batchable. +// WonLaneClaimExchange is the sole RMW exception: only its idempotent failed +// retries (old value already kDrainedClaimed) are batched, while the successful +// state transition remains a one-call record. +PTO_DEVICE_FUNC constexpr bool fdwic_atomic_site_is_poll_batchable(FdwicAtomicSite site) { + switch (site) { + case FdwicAtomicSite::StartupPoll: + case FdwicAtomicSite::FatalPoll: + case FdwicAtomicSite::FaninFlagLoad: + case FdwicAtomicSite::HeapFrontierLoad: + case FdwicAtomicSite::HeapVendLoad: + case FdwicAtomicSite::ReplayDonePoll: + case FdwicAtomicSite::WonAnyLoad: + case FdwicAtomicSite::WonStateLoad: + case FdwicAtomicSite::WonLaneClaimExchange: + case FdwicAtomicSite::WonDrainedLoad: + return true; + default: + return false; + } +} + +constexpr uint32_t kFdwicAtomicPollBatchSiteCount = 10; +static_assert(kFdwicAtomicPollBatchSiteCount <= 32, "poll-batch sites must fit the 32-bit active mask"); + +PTO_DEVICE_FUNC constexpr int32_t fdwic_atomic_poll_batch_index(FdwicAtomicSite site) { + switch (site) { + case FdwicAtomicSite::StartupPoll: + return 0; + case FdwicAtomicSite::FatalPoll: + return 1; + case FdwicAtomicSite::FaninFlagLoad: + return 2; + case FdwicAtomicSite::HeapFrontierLoad: + return 3; + case FdwicAtomicSite::HeapVendLoad: + return 4; + case FdwicAtomicSite::ReplayDonePoll: + return 5; + case FdwicAtomicSite::WonAnyLoad: + return 6; + case FdwicAtomicSite::WonStateLoad: + return 7; + case FdwicAtomicSite::WonDrainedLoad: + return 8; + case FdwicAtomicSite::WonLaneClaimExchange: + return 9; + default: + return -1; + } +} + +PTO_DEVICE_FUNC constexpr FdwicAtomicSite fdwic_atomic_poll_batch_site(uint32_t index) { + switch (index) { + case 0: + return FdwicAtomicSite::StartupPoll; + case 1: + return FdwicAtomicSite::FatalPoll; + case 2: + return FdwicAtomicSite::FaninFlagLoad; + case 3: + return FdwicAtomicSite::HeapFrontierLoad; + case 4: + return FdwicAtomicSite::HeapVendLoad; + case 5: + return FdwicAtomicSite::ReplayDonePoll; + case 6: + return FdwicAtomicSite::WonAnyLoad; + case 7: + return FdwicAtomicSite::WonStateLoad; + case 8: + return FdwicAtomicSite::WonDrainedLoad; + case 9: + return FdwicAtomicSite::WonLaneClaimExchange; + default: + return FdwicAtomicSite::Count; + } +} + +struct FdwicAtomicPollBurst { + uint64_t start_cycle[kFdwicAtomicPollBatchSiteCount]; + uint32_t call_count[kFdwicAtomicPollBatchSiteCount]; + uint32_t active_mask; + uint32_t enabled_mask; +}; + +// perf-clock 只复用每核固定 64B 状态中的既有 32B pad，不分配逐事件 +// record。expected_submit_count 由 PA orchestration 明确声明；设备与 host +// 都用它校验最后一个 Submit 的边界，而不是把任意一次 Submit 当作末次。 +struct FdwicPerfClockCoreData { + uint64_t first_submit_start; + uint64_t last_submit_end; + uint32_t submit_count; + uint32_t expected_submit_count; + uint32_t mode; + uint32_t final_seen; +}; + +static_assert(sizeof(FdwicPerfClockCoreData) == 32, "perf-clock data must fit the existing core-state pad"); +static_assert(offsetof(FdwicPerfClockCoreData, first_submit_start) == 0, "perf-clock start offset changed"); +static_assert(offsetof(FdwicPerfClockCoreData, last_submit_end) == 8, "perf-clock end offset changed"); +static_assert(offsetof(FdwicPerfClockCoreData, submit_count) == 16, "perf-clock count offset changed"); +static_assert(offsetof(FdwicPerfClockCoreData, expected_submit_count) == 20, "perf-clock expected offset changed"); + +// perf-clock-kernel 与普通 perf-clock 共用同一个 32B tail，但把末尾 8B +// 用于逐核 Kernel 累计时间。构建 profile 和外层 mode 字段负责区分两种 +// 解释，避免扩大每核 64B cacheline 或增加多核共享 sidecar。 +struct FdwicPerfClockKernelCoreData { + uint64_t first_submit_start; + uint64_t last_submit_end; + uint32_t submit_count; + uint32_t expected_submit_count; + uint64_t kernel_elapsed_ticks; +}; + +static_assert( + sizeof(FdwicPerfClockKernelCoreData) == 32, "perf-clock-kernel data must fit the existing core-state tail" +); +static_assert( + offsetof(FdwicPerfClockKernelCoreData, kernel_elapsed_ticks) == 24, "perf-clock-kernel elapsed offset changed" +); + struct FdwicSwimlaneCoreState { + // perf-clock-kernel 关闭 trace 后复用前 20B：count=Kernel 调用数， + // dropped=聚合错误状态，atomic_calls/poll_calls=0， + // poll_batch_records=kFdwicPerfClockKernelMode。普通 perf-clock 仍要求 + // 五个字段全零；两种解释不会静默混用。 volatile uint32_t count; volatile uint32_t dropped; - uint32_t pad[14]; + // Exact number of source-level atomic wrapper calls made by this worker + // while level-4 tracing was active. Poll batches contribute their encoded + // call_count rather than one call per Atomic record. + volatile uint32_t atomic_calls; + // Calls represented by PollBatch rows and the physical number of those + // rows. The host derives Atomic rows as + // atomic_calls - poll_calls + poll_batch_records, then verifies the raw. + volatile uint32_t poll_calls; + volatile uint32_t poll_batch_records; + // Topology is invariant within a worker partition. Store it once here + // instead of repeating the same 12 bytes in every record. + volatile int32_t core_idx; + volatile int32_t block_id; + volatile int32_t lane; + union { + uint32_t pad[8]; + FdwicPerfClockCoreData perf_clock; + FdwicPerfClockKernelCoreData perf_clock_kernel; + }; } __attribute__((aligned(64))); static_assert(sizeof(FdwicSwimlaneCoreState) == 64, "FdwicSwimlaneCoreState must occupy one cacheline"); +static_assert(offsetof(FdwicSwimlaneCoreState, perf_clock) == 32, "perf-clock must reuse the existing 32B tail"); +static_assert( + offsetof(FdwicSwimlaneCoreState, perf_clock_kernel) == 32, "perf-clock-kernel must reuse the existing 32B tail" +); struct FdwicSwimlaneHeader { uint32_t magic; @@ -52,18 +366,17 @@ struct FdwicSwimlaneHeader { } __attribute__((aligned(64))); static_assert(sizeof(FdwicSwimlaneHeader) % 64 == 0, "FdwicSwimlaneHeader must be cacheline aligned"); +static_assert(offsetof(FdwicSwimlaneHeader, cores) == 64, "per-core state must start at the second cacheline"); struct FdwicSwimlaneRecord { uint64_t start_cycle; uint64_t end_cycle; int32_t task_id; int32_t func_id; - int32_t phase; - int32_t lane; - int32_t block_id; - int32_t core_idx; uint32_t flags; - uint32_t aux; -} __attribute__((aligned(64))); + uint16_t phase; + uint16_t aux; +} __attribute__((aligned(32))); -static_assert(sizeof(FdwicSwimlaneRecord) == 64, "FdwicSwimlaneRecord must occupy one cacheline"); +static_assert(sizeof(FdwicSwimlaneRecord) == 32, "FdwicSwimlaneRecord must occupy half a cacheline"); +static_assert(alignof(FdwicSwimlaneRecord) == 32, "FdwicSwimlaneRecord alignment changed"); diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/target.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/target.h index 7c4652e603..fbca6a1ddf 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/target.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/target.h @@ -21,7 +21,51 @@ #define PTO2_PROFILING 1 #endif -#if defined(PTO2_PROFILING) && PTO2_PROFILING +#ifndef PTO_FDWIC_PERF_CLOCK +#define PTO_FDWIC_PERF_CLOCK 0 +#endif + +// perf-clock-kernel 是独立的低容量诊断 ELF：保留首末 Submit 时钟， +// 并额外累计落在同一逐核窗口内的 linked-kernel 时间与调用次数。 +#ifndef PTO_FDWIC_PERF_CLOCK_KERNEL +#define PTO_FDWIC_PERF_CLOCK_KERNEL 0 +#endif + +#ifndef PTO_FDWIC_SUBMIT_PMU +#define PTO_FDWIC_SUBMIT_PMU 0 +#endif + +// 每个 submit-PMU 诊断 ELF 只允许编译一个局部阶段。0 表示整窗 none； +// 其余编号由 submit_pmu_types.h 的 FdwicSubmitPmuPhase 定义。 +#ifndef PTO_FDWIC_SUBMIT_PMU_PHASE_ID +#define PTO_FDWIC_SUBMIT_PMU_PHASE_ID 0 +#endif + +#ifndef PTO_FDWIC_TRACE_ENABLED +#define PTO_FDWIC_TRACE_ENABLED PTO2_PROFILING +#endif + +#if PTO_FDWIC_PERF_CLOCK && PTO_FDWIC_TRACE_ENABLED +#error "PTO_FDWIC_PERF_CLOCK requires PTO_FDWIC_TRACE_ENABLED=0" +#endif + +#if PTO_FDWIC_PERF_CLOCK_KERNEL && !PTO_FDWIC_PERF_CLOCK +#error "PTO_FDWIC_PERF_CLOCK_KERNEL requires PTO_FDWIC_PERF_CLOCK=1" +#endif + +#if PTO_FDWIC_SUBMIT_PMU && PTO_FDWIC_TRACE_ENABLED +#error "PTO_FDWIC_SUBMIT_PMU requires PTO_FDWIC_TRACE_ENABLED=0" +#endif + +#if PTO_FDWIC_SUBMIT_PMU && PTO_FDWIC_PERF_CLOCK +#error "PTO_FDWIC_SUBMIT_PMU and PTO_FDWIC_PERF_CLOCK are mutually exclusive" +#endif + +#if !PTO_FDWIC_SUBMIT_PMU && PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 +#error "PTO_FDWIC_SUBMIT_PMU_PHASE_ID requires PTO_FDWIC_SUBMIT_PMU=1" +#endif + +#if PTO_FDWIC_TRACE_ENABLED #define DIST_TRACE_ENABLED 1 #else #define DIST_TRACE_ENABLED 0 diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/trace.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/trace.h index 893f01c9bd..0b57f797ed 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/trace.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/trace.h @@ -50,6 +50,20 @@ PTO_DEVICE_FUNC inline FdwicSwimlanePhase trace_phase_to_swimlane_phase(TracePha return FdwicSwimlanePhase::Fanin; case TracePhase::Register: return FdwicSwimlanePhase::Register; + case TracePhase::Atomic: + return FdwicSwimlanePhase::Atomic; + case TracePhase::ClockBaseline: + return FdwicSwimlanePhase::ClockBaseline; + case TracePhase::OrchestrationReplay: + return FdwicSwimlanePhase::OrchestrationReplay; + case TracePhase::FinalDrain: + return FdwicSwimlanePhase::FinalDrain; + case TracePhase::WinnerBuild: + return FdwicSwimlanePhase::WinnerBuild; + case TracePhase::AllocComplete: + return FdwicSwimlanePhase::AllocComplete; + case TracePhase::LoserReplay: + return FdwicSwimlanePhase::LoserReplay; } return FdwicSwimlanePhase::Kernel; } @@ -85,13 +99,21 @@ PTO_DEVICE_FUNC inline void trace_span_impl( PTO_DEVICE_FUNC inline void trace_instant_impl(__gm__ DistCore *self, int32_t task_id, int32_t func_id, TracePhase phase, uint32_t flags = 0) { const uint64_t cycle = fdwic_swimlane_detail_now(); + fdwic_atomic_poll_boundary_at(cycle); trace_span_impl(self, task_id, func_id, phase, cycle, cycle, flags, 0); } -PTO_DEVICE_FUNC inline uint64_t trace_span_begin_impl() { return fdwic_swimlane_detail_now(); } +PTO_DEVICE_FUNC inline uint64_t trace_span_begin_impl() { + const uint64_t cycle = fdwic_swimlane_detail_now(); + fdwic_atomic_poll_boundary_at(cycle); + return cycle; +} #define TRACE_LAP(self, task_id, func_id, phase) trace_lap_impl((self), (task_id), (func_id), (phase)) #define TRACE_LAP_RESET(self) trace_lap_reset_impl((self)) +#define TRACE_TIMESTAMP(name) const uint64_t name = trace_span_begin_impl() +#define TRACE_SPAN_RECORD(start, end, self, task_id, func_id, phase, flags, aux) \ + trace_span_impl((self), (task_id), (func_id), (phase), (start), (end), (flags), (aux)) #define TRACE_SPAN_BEGIN(name) const uint64_t name = trace_span_begin_impl() #define TRACE_SPAN_END(name, self, task_id, func_id, phase, flags, aux) \ trace_span_impl((self), (task_id), (func_id), (phase), (name), trace_span_begin_impl(), (flags), (aux)) @@ -105,6 +127,12 @@ PTO_DEVICE_FUNC inline void trace_reset_core(__gm__ DistCore *) {} #define TRACE_LAP(self, task_id, func_id, phase) ((void)0) #define TRACE_LAP_RESET(self) ((void)0) +#define TRACE_TIMESTAMP(name) const uint64_t name = 0 +#define TRACE_SPAN_RECORD(start, end, self, task_id, func_id, phase, flags, aux) \ + do { \ + (void)(start); \ + (void)(end); \ + } while (0) #define TRACE_SPAN_BEGIN(name) ((void)0) #define TRACE_SPAN_END(name, self, task_id, func_id, phase, flags, aux) ((void)0) #define TRACE_INSTANT(self, task_id, func_id, phase, flags) ((void)0) diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/worker_state.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/worker_state.h index c32657e575..d341d68b53 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/worker_state.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/worker_state.h @@ -12,6 +12,7 @@ #pragma once #include "dist_engine/common/target.h" +#include "dist_engine/common/submit_pmu_types.h" #include "dist_engine/common/swimlane_types.h" // g_dist / g_self storage. The AICPU build owns the BSS DistGlobal and @@ -30,32 +31,174 @@ [[block_local]] static int32_t g_ccec_ordinal; [[block_local]] static bool g_ccec_valid_worker; [[block_local]] static bool g_fdwic_joint_submit_seen; -[[block_local]] static bool g_fdwic_swimlane_enabled; +#if DIST_TRACE_ENABLED +[[block_local]] static uint32_t g_fdwic_swimlane_level; [[block_local]] static __gm__ FdwicSwimlaneHeader *g_fdwic_swimlane_header; [[block_local]] static __gm__ FdwicSwimlaneCoreState *g_fdwic_swimlane_core; [[block_local]] static __gm__ FdwicSwimlaneRecord *g_fdwic_swimlane_records; [[block_local]] static uint32_t g_fdwic_swimlane_records_per_core; +[[block_local]] static FdwicAtomicPollBurst g_fdwic_atomic_poll_burst; +[[block_local]] static uint32_t g_fdwic_atomic_calls; +[[block_local]] static uint32_t g_fdwic_poll_calls; +[[block_local]] static uint32_t g_fdwic_poll_batch_records; +[[block_local]] static bool g_fdwic_atomic_counter_overflow; +#endif +#if PTO_FDWIC_PERF_CLOCK +[[block_local]] static __gm__ FdwicSwimlaneCoreState *g_fdwic_perf_clock_core; +[[block_local]] static uint64_t g_fdwic_perf_clock_first_submit; +[[block_local]] static uint64_t g_fdwic_perf_clock_last_submit; +[[block_local]] static uint32_t g_fdwic_perf_clock_expected_submits; +#if PTO_FDWIC_PERF_CLOCK_KERNEL +[[block_local]] static uint64_t g_fdwic_perf_clock_kernel_ticks; +[[block_local]] static uint32_t g_fdwic_perf_clock_kernel_calls; +[[block_local]] static uint32_t g_fdwic_perf_clock_kernel_status; +#endif +#endif +#if PTO_FDWIC_SUBMIT_PMU +[[block_local]] static __gm__ FdwicSubmitPmuCoreData *g_fdwic_submit_pmu_core; +[[block_local]] static __gm__ FdwicSubmitPmuPhaseCoreData *g_fdwic_submit_pmu_phase_core; +[[block_local]] static uint64_t g_fdwic_submit_pmu_reg_base; +[[block_local]] static uint64_t g_fdwic_submit_pmu_start_tick; +[[block_local]] static uint64_t g_fdwic_submit_pmu_end_tick; +[[block_local]] static uint64_t g_fdwic_submit_pmu_scalar_elapsed_ticks; +[[block_local]] static uint64_t g_fdwic_submit_pmu_scalar_segment_begin_tick; +[[block_local]] static uint64_t g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks; +[[block_local]] static uint64_t g_fdwic_submit_pmu_return_ready_atomic_begin_tick; +[[block_local]] static uint64_t g_fdwic_submit_pmu_total_cycles; +[[block_local]] static uint32_t g_fdwic_submit_pmu_scalar_busy; +[[block_local]] static uint32_t g_fdwic_submit_pmu_icache_requests; +[[block_local]] static uint32_t g_fdwic_submit_pmu_icache_misses; +[[block_local]] static uint32_t g_fdwic_submit_pmu_expected_submits; +[[block_local]] static uint32_t g_fdwic_submit_pmu_status; +[[block_local]] static bool g_fdwic_submit_pmu_started; +[[block_local]] static bool g_fdwic_submit_pmu_stopped; +[[block_local]] static bool g_fdwic_submit_pmu_gate_running; +[[block_local]] static bool g_fdwic_submit_pmu_gate_error; +[[block_local]] static bool g_fdwic_submit_pmu_return_ready_atomic_active; +[[block_local]] static bool g_fdwic_submit_pmu_return_ready_atomic_phase_armed; +[[block_local]] static bool g_fdwic_submit_pmu_return_ready_atomic_seen; +[[block_local]] static bool g_fdwic_submit_pmu_return_ready_atomic_time_error; +[[block_local]] static FdwicSubmitPmuPhaseAccumulator g_fdwic_submit_pmu_phase; +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 +// 每个局部 phase 独占的排除计数；none 不分配 phase sidecar，也不需要该状态。 +[[block_local]] static uint32_t g_fdwic_submit_pmu_excluded_kernel_calls; +#endif +#endif #define g_dist (*g_dist_ptr) #elif defined(__CPU_SIM) static DistGlobal g_dist_fallback; static DistGlobal *g_dist_ptr = nullptr; thread_local DistCore *g_self = nullptr; thread_local bool g_fdwic_joint_submit_seen = false; -thread_local bool g_fdwic_swimlane_enabled = false; +#if DIST_TRACE_ENABLED +thread_local uint32_t g_fdwic_swimlane_level = 0; thread_local FdwicSwimlaneHeader *g_fdwic_swimlane_header = nullptr; thread_local FdwicSwimlaneCoreState *g_fdwic_swimlane_core = nullptr; thread_local FdwicSwimlaneRecord *g_fdwic_swimlane_records = nullptr; thread_local uint32_t g_fdwic_swimlane_records_per_core = 0; +thread_local FdwicAtomicPollBurst g_fdwic_atomic_poll_burst = {}; +thread_local uint32_t g_fdwic_atomic_calls = 0; +thread_local uint32_t g_fdwic_poll_calls = 0; +thread_local uint32_t g_fdwic_poll_batch_records = 0; +thread_local bool g_fdwic_atomic_counter_overflow = false; +#endif +#if PTO_FDWIC_PERF_CLOCK +thread_local FdwicSwimlaneCoreState *g_fdwic_perf_clock_core = nullptr; +thread_local uint64_t g_fdwic_perf_clock_first_submit = 0; +thread_local uint64_t g_fdwic_perf_clock_last_submit = 0; +thread_local uint32_t g_fdwic_perf_clock_expected_submits = 0; +#if PTO_FDWIC_PERF_CLOCK_KERNEL +thread_local uint64_t g_fdwic_perf_clock_kernel_ticks = 0; +thread_local uint32_t g_fdwic_perf_clock_kernel_calls = 0; +thread_local uint32_t g_fdwic_perf_clock_kernel_status = 0; +#endif +#endif +#if PTO_FDWIC_SUBMIT_PMU +thread_local FdwicSubmitPmuCoreData *g_fdwic_submit_pmu_core = nullptr; +thread_local FdwicSubmitPmuPhaseCoreData *g_fdwic_submit_pmu_phase_core = nullptr; +thread_local uint64_t g_fdwic_submit_pmu_reg_base = 0; +thread_local uint64_t g_fdwic_submit_pmu_start_tick = 0; +thread_local uint64_t g_fdwic_submit_pmu_end_tick = 0; +thread_local uint64_t g_fdwic_submit_pmu_scalar_elapsed_ticks = 0; +thread_local uint64_t g_fdwic_submit_pmu_scalar_segment_begin_tick = 0; +thread_local uint64_t g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks = 0; +thread_local uint64_t g_fdwic_submit_pmu_return_ready_atomic_begin_tick = 0; +thread_local uint64_t g_fdwic_submit_pmu_total_cycles = 0; +thread_local uint32_t g_fdwic_submit_pmu_scalar_busy = 0; +thread_local uint32_t g_fdwic_submit_pmu_icache_requests = 0; +thread_local uint32_t g_fdwic_submit_pmu_icache_misses = 0; +thread_local uint32_t g_fdwic_submit_pmu_expected_submits = 0; +thread_local uint32_t g_fdwic_submit_pmu_status = 0; +thread_local bool g_fdwic_submit_pmu_started = false; +thread_local bool g_fdwic_submit_pmu_stopped = false; +thread_local bool g_fdwic_submit_pmu_gate_running = false; +thread_local bool g_fdwic_submit_pmu_gate_error = false; +thread_local bool g_fdwic_submit_pmu_return_ready_atomic_active = false; +thread_local bool g_fdwic_submit_pmu_return_ready_atomic_phase_armed = false; +thread_local bool g_fdwic_submit_pmu_return_ready_atomic_seen = false; +thread_local bool g_fdwic_submit_pmu_return_ready_atomic_time_error = false; +thread_local FdwicSubmitPmuPhaseAccumulator g_fdwic_submit_pmu_phase = {}; +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 +thread_local uint32_t g_fdwic_submit_pmu_excluded_kernel_calls = 0; +#endif +#endif #define g_dist (*g_dist_ptr) #else static DistGlobal g_dist_fallback; static DistGlobal *g_dist_ptr = &g_dist_fallback; thread_local DistCore *g_self = nullptr; thread_local bool g_fdwic_joint_submit_seen = false; -thread_local bool g_fdwic_swimlane_enabled = false; +#if DIST_TRACE_ENABLED +thread_local uint32_t g_fdwic_swimlane_level = 0; thread_local FdwicSwimlaneHeader *g_fdwic_swimlane_header = nullptr; thread_local FdwicSwimlaneCoreState *g_fdwic_swimlane_core = nullptr; thread_local FdwicSwimlaneRecord *g_fdwic_swimlane_records = nullptr; thread_local uint32_t g_fdwic_swimlane_records_per_core = 0; +thread_local FdwicAtomicPollBurst g_fdwic_atomic_poll_burst = {}; +thread_local uint32_t g_fdwic_atomic_calls = 0; +thread_local uint32_t g_fdwic_poll_calls = 0; +thread_local uint32_t g_fdwic_poll_batch_records = 0; +thread_local bool g_fdwic_atomic_counter_overflow = false; +#endif +#if PTO_FDWIC_PERF_CLOCK +thread_local FdwicSwimlaneCoreState *g_fdwic_perf_clock_core = nullptr; +thread_local uint64_t g_fdwic_perf_clock_first_submit = 0; +thread_local uint64_t g_fdwic_perf_clock_last_submit = 0; +thread_local uint32_t g_fdwic_perf_clock_expected_submits = 0; +#if PTO_FDWIC_PERF_CLOCK_KERNEL +thread_local uint64_t g_fdwic_perf_clock_kernel_ticks = 0; +thread_local uint32_t g_fdwic_perf_clock_kernel_calls = 0; +thread_local uint32_t g_fdwic_perf_clock_kernel_status = 0; +#endif +#endif +#if PTO_FDWIC_SUBMIT_PMU +thread_local FdwicSubmitPmuCoreData *g_fdwic_submit_pmu_core = nullptr; +thread_local FdwicSubmitPmuPhaseCoreData *g_fdwic_submit_pmu_phase_core = nullptr; +thread_local uint64_t g_fdwic_submit_pmu_reg_base = 0; +thread_local uint64_t g_fdwic_submit_pmu_start_tick = 0; +thread_local uint64_t g_fdwic_submit_pmu_end_tick = 0; +thread_local uint64_t g_fdwic_submit_pmu_scalar_elapsed_ticks = 0; +thread_local uint64_t g_fdwic_submit_pmu_scalar_segment_begin_tick = 0; +thread_local uint64_t g_fdwic_submit_pmu_scalar_segment_excluded_atomic_ticks = 0; +thread_local uint64_t g_fdwic_submit_pmu_return_ready_atomic_begin_tick = 0; +thread_local uint64_t g_fdwic_submit_pmu_total_cycles = 0; +thread_local uint32_t g_fdwic_submit_pmu_scalar_busy = 0; +thread_local uint32_t g_fdwic_submit_pmu_icache_requests = 0; +thread_local uint32_t g_fdwic_submit_pmu_icache_misses = 0; +thread_local uint32_t g_fdwic_submit_pmu_expected_submits = 0; +thread_local uint32_t g_fdwic_submit_pmu_status = 0; +thread_local bool g_fdwic_submit_pmu_started = false; +thread_local bool g_fdwic_submit_pmu_stopped = false; +thread_local bool g_fdwic_submit_pmu_gate_running = false; +thread_local bool g_fdwic_submit_pmu_gate_error = false; +thread_local bool g_fdwic_submit_pmu_return_ready_atomic_active = false; +thread_local bool g_fdwic_submit_pmu_return_ready_atomic_phase_armed = false; +thread_local bool g_fdwic_submit_pmu_return_ready_atomic_seen = false; +thread_local bool g_fdwic_submit_pmu_return_ready_atomic_time_error = false; +thread_local FdwicSubmitPmuPhaseAccumulator g_fdwic_submit_pmu_phase = {}; +#if PTO_FDWIC_SUBMIT_PMU_PHASE_ID != 0 +thread_local uint32_t g_fdwic_submit_pmu_excluded_kernel_calls = 0; +#endif +#endif #define g_dist (*g_dist_ptr) #endif diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/dist_engine.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/dist_engine.h index 1e97acf140..52f6d1833d 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/dist_engine.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/dist_engine.h @@ -26,6 +26,7 @@ #pragma once #include +#include struct PTO2Runtime; struct L2TaskArgs; @@ -51,4 +52,8 @@ inline constexpr size_t dist_engine_global_state_align() { return kDistEngineGlo * * AICore workers call dist_engine_api.h symbols directly on sim and onboard. */ -void dist_engine_register(PTO2Runtime *rt, const L2TaskArgs *orch_args, int num_workers, Runtime *runtime); +int32_t dist_engine_register(PTO2Runtime *rt, const L2TaskArgs *orch_args, int num_workers, Runtime *runtime); + +// AICPU 在所有 worker 完成后读取 AICore 发布的 fatal cache line，并将 +// 首个运行时错误转换成公共负状态码。正常运行返回 0。 +int32_t dist_engine_runtime_status(PTO2Runtime *rt); diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/dist_engine_api.h b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/dist_engine_api.h index 99c9bbeec2..cec5a8be65 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/dist_engine_api.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/dist_engine_api.h @@ -46,6 +46,43 @@ struct PTO2Runtime; +/** + * Fixed-size hand-off between the two synchronous halves of a compete-first + * submit. Begin never retains an argument-builder closure: orchestration + * receives this POD, builds its caller-owned L0TaskArgs, then immediately + * passes both objects to Finish. + * + * Byte fields are intentional. Besides keeping the cross-TU ABI explicit, + * they avoid relying on the target compiler's bool layout. `reserved` must + * remain zero and is checked by Finish together with `kind` and the per-core + * task sequence. + */ +enum class DistCompeteFirstKind : uint8_t { + Kernel = 0, + Alloc = 1, +}; + +struct DistCompeteFirstTicket { + uint64_t submit_begin; + int32_t task_id; + int32_t kernel_id; + int32_t joint_block; + int32_t joint_count; + uint8_t won; + uint8_t joint; + uint8_t joint_init; + uint8_t claim_attempted; + uint8_t ready; + uint8_t kind; + uint16_t reserved; +}; + +static_assert(sizeof(DistCompeteFirstTicket) == 32, "compete-first ticket ABI must remain 32 bytes"); +static_assert(offsetof(DistCompeteFirstTicket, submit_begin) == 0, "compete-first timestamp ABI mismatch"); +static_assert(offsetof(DistCompeteFirstTicket, task_id) == 8, "compete-first task ABI mismatch"); +static_assert(offsetof(DistCompeteFirstTicket, won) == 24, "compete-first state ABI mismatch"); +static_assert(offsetof(DistCompeteFirstTicket, reserved) == 30, "compete-first reserved ABI mismatch"); + // Task submission and allocation. Host/sim definitions use the per-core g_self // stashed by dist_core_main / thread_local sim. CCEC definitions use the same // materialize/map/fanin/register stages, then dispatch through the current @@ -53,6 +90,25 @@ struct PTO2Runtime; PTO_DEVICE_FUNC TaskOutputTensors dist_submit_impl(PTO2Runtime *rt, const MixedKernels &mixed, const L0TaskArgs &args); PTO_DEVICE_FUNC TaskOutputTensors dist_alloc_tensors(PTO2Runtime *rt, const L0TaskArgs &args); +// Explicit compete-first eager path. Begin performs argument-independent +// progress and Claim; Finish consumes the synchronously built args. The same +// MixedKernels object must remain unchanged until Finish returns. The old +// one-shot APIs above remain available unchanged for all existing examples. +PTO_DEVICE_FUNC DistCompeteFirstTicket +dist_submit_compete_first_begin(PTO2Runtime *rt, const MixedKernels &mixed); +PTO_DEVICE_FUNC TaskOutputTensors dist_submit_compete_first_finish( + PTO2Runtime *rt, const MixedKernels &mixed, const DistCompeteFirstTicket &ticket, const L0TaskArgs &args +); +PTO_DEVICE_FUNC DistCompeteFirstTicket dist_alloc_compete_first_begin(PTO2Runtime *rt); +PTO_DEVICE_FUNC TaskOutputTensors dist_alloc_compete_first_finish( + PTO2Runtime *rt, const DistCompeteFirstTicket &ticket, const L0TaskArgs &args +); + +// perf-clock 专用构建由具体 orchestration 显式声明本核应重放的 Submit +// 总数。普通构建中该接口编译为空操作，不改变公开 submit ABI。 +PTO_DEVICE_FUNC void dist_perf_clock_expect_submits(uint32_t expected_submits); +PTO_DEVICE_FUNC void dist_submit_pmu_expect_submits(uint32_t expected_submits); + // Fatal-state helpers. dist_engine.cpp already exposes fatal_set() / // set_fatal(); these are the CCEC-safe wrappers orchestration reaches. PTO_DEVICE_FUNC bool dist_is_fatal_query(); diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/fdwic_build_identity.h b/src/a5/runtime/fully_distributed_within_core/runtime/fdwic_build_identity.h new file mode 100644 index 0000000000..8b9f45c15b --- /dev/null +++ b/src/a5/runtime/fully_distributed_within_core/runtime/fdwic_build_identity.h @@ -0,0 +1,119 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#pragma once + +#include +#include + +// Private and shared are separate compile-time artifacts, not a hot-path +// runtime switch. The default preserves private behavior; the build must pass +// the same 0/1 identity to all three images. +#ifndef PTO_FDWIC_SHARED_MAP +#define PTO_FDWIC_SHARED_MAP 0 +#endif + +#if PTO_FDWIC_SHARED_MAP != 0 && PTO_FDWIC_SHARED_MAP != 1 +#error "PTO_FDWIC_SHARED_MAP must be 0 (private) or 1 (shared)" +#endif + +#ifndef PTO_FDWIC_TENSORMAP_RING_CAP +#define PTO_FDWIC_TENSORMAP_RING_CAP 128 +#endif + +#if PTO_FDWIC_TENSORMAP_RING_CAP < 32 || PTO_FDWIC_TENSORMAP_RING_CAP > 16384 +#error "PTO_FDWIC_TENSORMAP_RING_CAP must be in [32, 16384]" +#endif + +#if (PTO_FDWIC_TENSORMAP_RING_CAP & (PTO_FDWIC_TENSORMAP_RING_CAP - 1)) != 0 +#error "PTO_FDWIC_TENSORMAP_RING_CAP must be a power of two" +#endif + +#if (16384 % PTO_FDWIC_TENSORMAP_RING_CAP) != 0 +#error "PTO_FDWIC_TENSORMAP_RING_CAP must divide the fixed 16K physical slot pool" +#endif + +enum class FdwicTensorMapMode : uint32_t { + Private = 0, + Shared = 1, +}; + +inline constexpr uint64_t kFdwicBuildIdentityMagic = 0x46445749434d4150ULL; // "FDWICMAP" +inline constexpr uint32_t kFdwicBuildAbiVersion = 4; +inline constexpr uint32_t kFdwicDistGlobalLayoutVersion = 4; +inline constexpr FdwicTensorMapMode kFdwicCompiledTensorMapMode = static_cast(PTO_FDWIC_SHARED_MAP); +inline constexpr uint32_t kFdwicTensorMapRingCap = PTO_FDWIC_TENSORMAP_RING_CAP; +inline constexpr uint32_t kFdwicTensorMapRingBuckets = 16384U / kFdwicTensorMapRingCap; + +// Basic shared Submit transactions are wired, but multi-worker convergence, +// PA region-intent, visibility, and device gates are not complete. Shared +// images may compile and run ABI/integration gates, but normal execution must +// still stop before the first Submit and may never fall back to private map +// semantics. +inline constexpr bool kFdwicCompiledBackendReady = PTO_FDWIC_SHARED_MAP == 0; + +enum FdwicBuildError : uint32_t { + FdwicBuildErrorNone = 0, + FdwicBuildErrorAicpuMismatch = 1U << 0, + FdwicBuildErrorAicoreMismatch = 1U << 1, + FdwicBuildErrorBackendUnavailable = 1U << 2, +}; + +// This cache line must remain Runtime's first field. Host, AICPU, and AICore +// read it before interpreting mode-dependent state. Future shared layout +// changes may only append or modify later state, never move this stable prefix. +struct alignas(64) FdwicBuildIdentity { + uint64_t magic; + uint32_t abi_version; + uint32_t tensor_map_mode; + uint32_t runtime_bytes; + uint32_t dist_global_layout_version; + volatile uint32_t error_bits; + uint32_t tensor_map_ring_cap; + uint32_t reserved[8]; +}; + +static_assert(sizeof(FdwicBuildIdentity) == 64, "FDWIC build identity must occupy exactly one cache line"); +static_assert(alignof(FdwicBuildIdentity) == 64, "FDWIC build identity must be cache-line aligned"); +// The first three control fields are already shared by v1 Host/AICPU/AICore. +// New identity fields may only consume old reserved words and must not move +// error_bits, or a mismatched image could publish failure at an unseen offset. +static_assert(offsetof(FdwicBuildIdentity, runtime_bytes) == 16, "FDWIC runtime-size identity offset changed"); +static_assert( + offsetof(FdwicBuildIdentity, dist_global_layout_version) == 20, "FDWIC dist-layout identity offset changed" +); +static_assert(offsetof(FdwicBuildIdentity, error_bits) == 24, "FDWIC cross-image error-bit offset changed"); +static_assert(offsetof(FdwicBuildIdentity, tensor_map_ring_cap) == 28, "FDWIC ring-cap identity offset changed"); + +inline FdwicBuildIdentity fdwic_make_build_identity(uint32_t runtime_bytes) { + return { + kFdwicBuildIdentityMagic, + kFdwicBuildAbiVersion, + static_cast(kFdwicCompiledTensorMapMode), + runtime_bytes, + kFdwicDistGlobalLayoutVersion, + FdwicBuildErrorNone, + kFdwicTensorMapRingCap, + {}, + }; +} + +#if defined(__CCE_AICORE__) +__aicore__ inline bool +fdwic_build_identity_matches(__gm__ const volatile FdwicBuildIdentity &identity, uint32_t expected_runtime_bytes) { +#else +inline bool fdwic_build_identity_matches(const volatile FdwicBuildIdentity &identity, uint32_t expected_runtime_bytes) { +#endif + return identity.magic == kFdwicBuildIdentityMagic && identity.abi_version == kFdwicBuildAbiVersion && + identity.tensor_map_mode == static_cast(kFdwicCompiledTensorMapMode) && + identity.tensor_map_ring_cap == kFdwicTensorMapRingCap && identity.runtime_bytes == expected_runtime_bytes && + identity.dist_global_layout_version == kFdwicDistGlobalLayoutVersion; +} diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/runtime.h b/src/a5/runtime/fully_distributed_within_core/runtime/runtime.h index 5a35156947..35a4854815 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/runtime.h +++ b/src/a5/runtime/fully_distributed_within_core/runtime/runtime.h @@ -38,6 +38,7 @@ #include "common/core_type.h" #include "dist_engine/common/swimlane_types.h" #include "common/platform_config.h" +#include "fdwic_build_identity.h" #include "pto2_dispatch_payload.h" #include "pto_types.h" #include "task_args.h" @@ -58,6 +59,7 @@ constexpr uint32_t AICPU_READY_NONE = 0; constexpr uint32_t AICPU_READY_HANDSHAKE = 1; constexpr uint32_t AICPU_READY_DIST_RUN = 2; +constexpr uint32_t AICPU_READY_DIST_ABORT = 3; /** * Handshake Structure - Shared between Host, AICPU, and AICore @@ -70,6 +72,7 @@ constexpr uint32_t AICPU_READY_DIST_RUN = 2; * 2. Acknowledgment: AICore sets aicore_done=core_id+1 * 3. Dist Run: AICPU sets aicpu_ready=AICPU_READY_DIST_RUN after publishing * shared runtime state + * (or AICPU_READY_DIST_ABORT when a cold-path diagnostic setup failed) * 4. Task Completion: AICore writes FIN to COND; AICPU observes completion * 5. Shutdown: AICPU sets control=1, AICore exits * @@ -201,6 +204,15 @@ struct Task { */ class Runtime { public: + // Stable three-image ABI/control prefix. The identity must remain first; + // workers, worker_count and the AICPU launch fields through + // aicpu_launch_count must keep the same order and layout in every + // private/shared image. AICore must finish the common handshake even when + // the identity mismatches, otherwise AICPU cannot issue DIST_ABORT/EXIT. + // Mode-dependent state therefore belongs after this common prefix (the + // shared TensorMap itself lives behind dist.shared_addr). + FdwicBuildIdentity fdwic_build_identity; + // Handshake buffers for AICPU-AICore communication Handshake workers[RUNTIME_MAX_WORKER]; // Worker (AICore) handshake buffers int worker_count; // Number of active workers @@ -254,7 +266,9 @@ class Runtime { volatile uint64_t shared_addr; volatile int32_t num_workers; // number of AICore workers participating volatile uint64_t swimlane_base; - volatile uint32_t swimlane_enabled; + // Existing L2 swimlane perf level (0..4). FDWIC keeps phase spans at + // levels 1..3 and adds source-level atomic spans at level 4. + volatile uint32_t swimlane_level; volatile uint32_t swimlane_records_per_core; Tensor ccec_orch_tensors[CHIP_MAX_TENSOR_ARGS]; uint64_t ccec_orch_scalars[CHIP_MAX_SCALAR_ARGS]; @@ -265,6 +279,7 @@ class Runtime { } dist; void *fdwic_swimlane_host_shadow_; + uint64_t fdwic_swimlane_dev_allocation_; uint64_t fdwic_swimlane_dev_base_; uint64_t fdwic_swimlane_bytes_; uint32_t fdwic_swimlane_num_cores_; diff --git a/src/a5/runtime/fully_distributed_within_core/runtime/shared/runtime.cpp b/src/a5/runtime/fully_distributed_within_core/runtime/shared/runtime.cpp index e27554f89b..9fafcb032e 100644 --- a/src/a5/runtime/fully_distributed_within_core/runtime/shared/runtime.cpp +++ b/src/a5/runtime/fully_distributed_within_core/runtime/shared/runtime.cpp @@ -28,6 +28,8 @@ Runtime::Runtime() { // NOTE: host_api is initialized in InitRuntime() (host-only code) // because the CApi functions don't exist when compiled for device. + fdwic_build_identity = fdwic_make_build_identity(static_cast(sizeof(Runtime))); + // Initialize handshake buffers memset(workers, 0, sizeof(workers)); worker_count = 0; @@ -40,13 +42,14 @@ Runtime::Runtime() { dist.shared_addr = 0; dist.num_workers = 0; dist.swimlane_base = 0; - dist.swimlane_enabled = 0; + dist.swimlane_level = 0; dist.swimlane_records_per_core = 0; dist.orch_args.reset(); dist.ccec_orch_tensor_count = 0; dist.ccec_orch_scalar_count = 0; dist.done_count = 0; fdwic_swimlane_host_shadow_ = nullptr; + fdwic_swimlane_dev_allocation_ = 0; fdwic_swimlane_dev_base_ = 0; fdwic_swimlane_bytes_ = 0; fdwic_swimlane_num_cores_ = 0; diff --git a/tests/TENSORMAP_SCHEDULING_OVERVIEW.md b/tests/TENSORMAP_SCHEDULING_OVERVIEW.md index 4025a9d921..6b1931a475 100644 --- a/tests/TENSORMAP_SCHEDULING_OVERVIEW.md +++ b/tests/TENSORMAP_SCHEDULING_OVERVIEW.md @@ -392,21 +392,25 @@ AICPU 调用 `dist_engine_register()` 时，会为本轮运行写入或重置： - `orch_args`：本轮 orchestration 的输入参数入口； - 三组 claim cursor：重置为 `-1`； - task cell、frontier、fatal：重置为未完成状态； -- `started_count/replay_done`：重置为 0。 +- `started_count/replay_done`：重置为 0；`replay_done` 仅保留原有 ABI 位置； +- 固定 G=16 final 树：重置 leaf/root arrival 和 release，并根据本轮 + `layout[]` 写入每个活跃叶组的 worker 数与活跃组数。 这些数据不是每个 task 都重新生成，但它们是每次算子运行的动态上下文。例如同一个程序 用不同 worker 数运行时，`layout[]` 和 `num_blocks` 会随本轮资源重新推导。 worker 进入 `dist_core_main()` 后，先原子增加 `started_count`。所有 worker 到齐后才开始 重放，避免一部分核已经提交很远、另一部分核尚未启动。每个 worker 重放完 orchestration -后增加 `replay_done`，但仍会继续 drain 已经排队的 slot；只有全部 worker 重放结束且本核 -没有待执行 slot/`block.won` 时，才退出。 +后原子增加 `final_barrier.leaf_arrivals[block_id % 16]`。每组的静态 AIC 代表在本组 +到齐后向 root 转发一次；root 收到全部活跃组后发布全局 release，各组代表 +再发布本组 release。等待过程中 worker 仍继续 drain 已经排队的 slot；只有观察到 +本组的全局 release，且本核没有待执行 slot/`block.won` 时，才退出。 因此： ```text started_count 表示“多少 worker 已进入本轮” -replay_done 表示“多少 worker 已生成完全部逻辑 task” +final leaf/root/release 表示“所有 worker 是否已生成完全部逻辑 task” task flags 表示“具体 kernel task 是否执行完成” ``` @@ -1597,7 +1601,7 @@ dist_core_main(runtime, core_idx, core_type) 3. reset 本核 `DistCore`； 4. 参与 `started_count` 启动屏障； 5. 重放完整 orchestration； -6. 发布 `replay_done`； +6. 到达固定 G=16 final 树，并等待全局 release； 7. drain 本核和 `block.won` 的剩余任务； 8. 发布 worker done； 9. 返回 AICore executor，等待 AICPU EXIT。 @@ -1934,14 +1938,14 @@ AICore: 进入 dist_core_main | `frontier` | 从 0 开始连续完成到哪个 task？ | | `WonSlot.remaining` | 某 MIX task 还有几个 lane 未完成？ | | `DistCore.occupied_count` | 本核还有几个 private slot？ | -| `replay_done` | 有多少 worker 已生成完全部 task？ | +| final leaf/root/release | 所有 worker 是否已生成完全部 task？ | | `Runtime::dist.done_count`/COND | 有多少 worker 已退出 dist engine？ | | `runtime_done_` | AICPU setup thread 是否结束等待？ | | host stream sync | 整个 device operation 是否已结束？ | -例如 `replay_done == num_workers` 只表示所有核都走完 orchestration 源码，不能立刻退出； -private ring 中可能仍有等待 producer 的任务。每核还要 drain 到 ring 空且没有待收取的 -`block.won`。 +例如观察到 final leaf release 只表示所有核都走完 orchestration 源码，不能立刻 +退出；private ring 中可能仍有等待 producer 的任务。每核还要 drain 到 ring 空且 +没有待收取的 `block.won`。 ### 17.3 Task completion 的发布顺序 @@ -2160,11 +2164,11 @@ completion 和 AICPU handoff 都属于这一类。 - 等某个 producer：看 task flag； - 回收连续历史：看 frontier； -- 判断所有核生成完图：看 replay_done； +- 判断所有核生成完图：看 final 树的本组 release； - 判断本核可退出：还要 ring 空、无 pending won； - 判断 host 可回收本轮：等 stream 完成。 -拿 `replay_done` 代替 task completion，或拿 frontier 代替最慢 core progress，都会在并发 +拿 final release 代替 task completion，或拿 frontier 代替最慢 core progress，都会在并发 速度变化后出错。 ### 20.5 所有有界结构都需要反压或失败语义 @@ -2242,7 +2246,7 @@ post-kernel flush 前必须证明不会写回陈旧控制 cacheline，也不能 | `block.won` | MIX follower 不执行 | state/payload/drained/remaining | | kernel call | 进入 slot 后 hang/mismatch | func id、args ABI、输入可见性 | | completion | kernel 返回但后继不跑 | data flush、flag、frontier | -| drain/exit | task 完成但 host timeout | replay_done、ring、COND/EXIT | +| drain/exit | task 完成但 host timeout | final leaf/root/release、ring、COND/EXIT | | validate | device 对、host 错 | D2H 和 top-level direction | ### 21.2 不同改动从哪里下手 diff --git a/tests/atomic_probe/A5_FDWIC_PAGED_ATTENTION_REPRO.md b/tests/atomic_probe/A5_FDWIC_PAGED_ATTENTION_REPRO.md deleted file mode 100644 index 278c3b8275..0000000000 --- a/tests/atomic_probe/A5_FDWIC_PAGED_ATTENTION_REPRO.md +++ /dev/null @@ -1,818 +0,0 @@ -# A5 FDWIC Paged Attention 安装与复现指南 - -## 1. 目标、边界与已验证结论 - -本文记录在真实 A5 开发板上安装用户态依赖，并复现以下 Case1 的完整过程： - -~~~text -examples/a5/fully_distributed_within_core/paged_attention_unroll/ -test_paged_attention_unroll.py -~~~ - -范围严格限定为： - -- 平台仅为 A5Sim 和 A5； -- runtime 仅为 fully_distributed_within_core； -- Case 仅为 Case1； -- Python 始终使用 $HOME/.venv； -- CANN 优先且固定使用 9.1 weekly 20260708； -- 性能口径是全局第一个 Submit 开始到最后一个 Submit 结束。 - -本文不覆盖其他测试目录、其他 runtime、A2/A3、L3 或整段 device wall time。 -A5Sim 用于功能和调度流程验证；5.6 ms 基线只从真实 A5 生成的 -l2_swimlane_records.json 中读取。 - -### 已验证环境 - -| 项目 | 本次验证值 | -| --- | --- | -| 验证日期 | 2026-07-17 | -| 芯片 | Ascend950PR_958b | -| 设备 | /dev/davinci0 | -| Driver | 7.0.t9.0.B798，ascendhal 7.35.23 | -| CANN | 9.1.0 weekly 20260708 | -| CCEC | clang 15.0.5 | -| AICPU 交叉编译器 | Do-Compiler 7.3.0 | -| A5 计算核 | 32 CUBE + 64 VECTOR，共 96 条 swimlane | -| AICPU 用户池 | 5，OCCUPY 掩码 0x3e | -| Python | 3.12.3 | -| PyTorch | 2.6.0+cpu | -| pytest | 7.4.4 | -| GCC 15 | 15.0.1，Ubuntu 15-20250404-0ubuntu1 | -| PTO-ISA | ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8 | -| simpler 分支 | fdwic-swimlane-deps | -| simpler 实测基准 HEAD | 52ca4f5eba343c2f7b7a3a743e575cb9308d128f | - -系统的 /etc/os-release 标签为 Ubuntu 20.04.6，但实际 -getconf GNU_LIBC_VERSION 输出 glibc 2.39。判断 GCC 15 二进制兼容性时， -应以实际 glibc 和 ldd 结果为准，不能只看发行版标签。 - -### 已验证结果 - -| 检查项 | 结果 | -| --- | --- | -| A5Sim Case1 | PASSED，约 71.62 s | -| A5 Case1 正确性 | PASSED | -| A5 swimlane Case1 | PASSED，pytest 约 85.48 s | -| 每核 Submit | 1280 个，task id 为 0 到 1279 | -| 全局首个至末个 Submit | 5.642245 ms | -| 排除 task 0 分配后的 kernel Submit | 5.635263 ms | -| 每核 Submit span 中位数 | 5.5725575 ms | -| 历史参考值 | 5.577570 ms，commit dbbf621ac2d1cf162d0807e170c042212d067e51 | - -因此，用户关注的约 5.6 ms 基线已经复现。pytest wall time 和日志中的整段 -device wall time 不属于这一性能口径。 - -### 必须包含的源码状态 - -复现使用的仓库版本必须同时包含以下三处源码调整： - -1. Case1、Case2、Case3 不再硬编码 block_dim=36，只保留 - aicpu_thread_num=4，由 A5 平台自动解析实际 block 数； -2. 旧 Driver 的 HAL 和 DSMI 都不支持 CPU_TOPO、返回 65534 时， - 允许经过双重校验的 flat OCCUPY 回退； -3. flat 回退只有在 OCCUPY 的 popcount 与 - ACL_DEV_ATTR_AICPU_CORE_NUM 完全相等时才接受，否则保持失败关闭。 - -对应文件为： - -~~~text -examples/a5/fully_distributed_within_core/paged_attention_unroll/ -test_paged_attention_unroll.py -src/a5/platform/onboard/host/aicpu_topology_probe.cpp -src/a5/platform/onboard/host/aicpu_topology_probe.h -~~~ - -在仓库根目录执行以下检查。第一条应无输出，第二条应命中： - -~~~bash -TEST_FILE=examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py - -if rg -n '"block_dim"[[:space:]]*:[[:space:]]*36' "$TEST_FILE"; then - echo "ERROR: 当前 revision 仍硬编码 block_dim=36" - exit 1 -fi - -rg -n 'ACL_DEV_ATTR_AICPU_CORE_NUM|flat OCCUPY fallback' \ - src/a5/platform/onboard/host/aicpu_topology_probe.cpp -~~~ - -## 2. 系统与设备前置检查 - -以下命令都以普通用户执行，不需要 sudo。Driver 和 firmware 是板端系统级 -前置条件，本文只校验，不覆盖安装或升级。 - -先进入已经下载好的 simpler 仓库： - -~~~bash -cd /path/to/simpler -export REPO_ROOT="$(git rev-parse --show-toplevel)" -cd "$REPO_ROOT" - -git rev-parse HEAD -git status --short -~~~ - -记录 HEAD 和工作区差异。若源码调整尚未提交，迁移环境时必须连同差异一起带走； -只有原始 HEAD 不能代表完整复现版本。 - -检查主机、Driver 和设备节点： - -~~~bash -uname -m -getconf GNU_LIBC_VERSION -grep -E '^(Version|ascendhal_version|timestamp)=' \ - /usr/local/Ascend/driver/version.info - -test -c /dev/davinci0 -test -r /dev/davinci0 -test -w /dev/davinci0 -ls -l /dev/davinci0 -~~~ - -本次预期为 x86_64、glibc 2.39、Driver 7.0.t9.0.B798，并且当前用户对 -/dev/davinci0 可读写。任何一项失败时先修复系统权限或 Driver，不要用 Python -代码绕过。 - -检查安装过程会使用的基础工具： - -~~~bash -for tool in bash git python3 rg sha256sum tar dpkg-deb; do - command -v "$tool" || { - echo "ERROR: missing tool: $tool" - exit 1 - } -done -~~~ - -当前 A5 EVB 没有 npu-smi，也没有 task-submit。这不等于设备不可用； -本次通过 Driver 版本文件、设备节点和实际 ACL 调用完成了验证。如果另一个环境 -提供设备预约工具，应先按该环境规则独占设备，再执行上板命令。 - -## 3. 安装 CANN 9.1 与用户级 GCC 15 - -### 安装 CANN 9.1 - -只使用以下两个 9.1 安装包，不要混入同目录下的 9.2 包： - -| 安装包 | 字节数 | SHA-256 | -| --- | ---: | --- | -| Ascend-cann-toolkit_9.1.0~weekly.20260708.01_linux-x86_64.run | 1543071133 | 947165d939e83e4e73c14498e19b5ed69dd0de49bd9b5d71e04765bfa0c09313 | -| Ascend-cann-950-ops_9.1.0~weekly.20260708.01_linux-x86_64.run | 2669342311 | 9b5df71c1ca9a855f65027fb37c3fcd352ca607e997277aacff71508e36b8b91 | - -先校验文件： - -~~~bash -TOOLKIT="$HOME/cann/Ascend-cann-toolkit_9.1.0~weekly.20260708.01_linux-x86_64.run" -OPS="$HOME/cann/Ascend-cann-950-ops_9.1.0~weekly.20260708.01_linux-x86_64.run" - -test -f "$TOOLKIT" -test -f "$OPS" - -printf '%s %s\n' \ - 947165d939e83e4e73c14498e19b5ed69dd0de49bd9b5d71e04765bfa0c09313 \ - "$TOOLKIT" | sha256sum -c - -printf '%s %s\n' \ - 9b5df71c1ca9a855f65027fb37c3fcd352ca607e997277aacff71508e36b8b91 \ - "$OPS" | sha256sum -c - -~~~ - -安装包当前没有 executable bit，因此显式交给 bash。先按组织要求完成软件许可 -确认，再使用 quiet 非交互安装： - -~~~bash -export CANN_INSTALL_ROOT="$HOME/Ascend/cann-9.1.0-weekly-20260708" -mkdir -p "$CANN_INSTALL_ROOT" - -bash "$TOOLKIT" \ - --full \ - --quiet \ - --install-path="$CANN_INSTALL_ROOT" - -bash "$OPS" \ - --full \ - --quiet \ - --install-path="$CANN_INSTALL_ROOT" -~~~ - -本次成功安装未使用 --force。只有安装器明确报告兼容性问题，且已经核实 -Driver/CANN 匹配关系时，才考虑该参数。 - -立即验证安装，不依赖 .bashrc： - -~~~bash -test -f "$CANN_INSTALL_ROOT/cann/set_env.sh" -source "$CANN_INSTALL_ROOT/cann/set_env.sh" - -test "$ASCEND_HOME_PATH" = \ - "$CANN_INSTALL_ROOT/cann-9.1.0" -test -x "$ASCEND_HOME_PATH/bin/ccec" -test -x \ - "$ASCEND_HOME_PATH/tools/hcc/bin/aarch64-target-linux-gnu-g++" - -grep -E '^(Version|timestamp)=' \ - "$ASCEND_HOME_PATH/opp/version.info" -"$ASCEND_HOME_PATH/bin/ccec" --version | head -"$ASCEND_HOME_PATH/tools/hcc/bin/aarch64-target-linux-gnu-g++" \ - --version | head -n 1 -~~~ - -预期 OPP Version 为 9.1.0，timestamp 为 20260708_000326093。 - -### 安装用户级 GCC 15 - -A5Sim 的 incore kernel 由仓库中的 Gxx15Toolchain 直接调用 g++-15， -所以仅有系统 g++ 不够。普通 host runtime 默认仍可使用系统 gcc/g++； -不要为了这一用例全局改写 CC 和 CXX。 - -本次验证使用从另一台已验证环境复制的 Ubuntu Plucky 解包目录： - -~~~text -$HOME/.local/gcc-15/root -~~~ - -这是用户态解包，不是 dpkg -i。精确源码包版本可在 Ubuntu Launchpad 的 -gcc-15 15-20250404-0ubuntu1 页面核对： - - - -推荐直接从已验证环境打包并传输完整 root 目录： - -~~~bash -# 在已验证的源环境执行 -cd "$HOME/.local/gcc-15" -tar -czf "$HOME/gcc-15-plucky-20250404-root.tar.gz" root -cd "$HOME" -sha256sum gcc-15-plucky-20250404-root.tar.gz \ - > gcc-15-plucky-20250404-root.tar.gz.sha256 - -# 将归档及其 SHA-256 传到目标环境后执行 -mkdir -p "$HOME/.local/gcc-15" -cp /path/to/gcc-15-plucky-20250404-root.tar.gz "$HOME/" -cp /path/to/gcc-15-plucky-20250404-root.tar.gz.sha256 "$HOME/" -cd "$HOME" -sha256sum -c gcc-15-plucky-20250404-root.tar.gz.sha256 -tar -xzf gcc-15-plucky-20250404-root.tar.gz \ - -C "$HOME/.local/gcc-15" -~~~ - -如果使用原始 deb 重建目录，应准备同一版本的以下包，并逐个用 -dpkg-deb -x 解到同一个 GCC15_ROOT： - -~~~text -cpp-15 -cpp-15-x86-64-linux-gnu -g++-15 -g++-15-x86-64-linux-gnu -gcc-15 -gcc-15-base -gcc-15-x86-64-linux-gnu -libasan8 -libatomic1 -libcc1-0 -libgcc-15-dev -libgcc-s1 -libgomp1 -libhwasan0 -libitm1 -liblsan0 -libquadmath0 -libstdc++-15-dev -libstdc++6 -libtsan2 -libubsan1 -~~~ - -~~~bash -export GCC15_ROOT="$HOME/.local/gcc-15/root" -mkdir -p "$GCC15_ROOT" - -for deb in "$HOME/cann/gcc-15-plucky-debs"/*.deb; do - dpkg-deb -f "$deb" Package Version - test "$(dpkg-deb -f "$deb" Version)" = \ - "15-20250404-0ubuntu1" - dpkg-deb -x "$deb" "$GCC15_ROOT" -done -~~~ - -不要把其他 Plucky 系统包或 libc6 一并放入该目录。当前编译器二进制要求 -GLIBC_2.38，目标主机实际 glibc 必须满足要求。若 ldd 显示 not found，应先 -补齐与主机兼容的 libisl、libmpc、libmpfr、libgmp、zlib、libzstd 或 binutils， -不要盲目混用另一发行版的 libc。 - -对复制结果做内容检查： - -~~~bash -export GCC15_ROOT="$HOME/.local/gcc-15/root" - -printf '%s %s\n' \ - db5b698ddfbbefa3978b76c0f9dd7504bd82136db461a05320b74743a8933ec9 \ - "$GCC15_ROOT/usr/bin/x86_64-linux-gnu-g++-15" \ - | sha256sum -c - -printf '%s %s\n' \ - 34ffcc0db386d0d654c29464b84c57a8218b650dfd3b801720b778eacdca7a9e \ - "$GCC15_ROOT/usr/libexec/gcc/x86_64-linux-gnu/15/cc1plus" \ - | sha256sum -c - -printf '%s %s\n' \ - 9fb7d85e8aa687d1d8b27d5c189f3d938579a29e75af9d4651db4d33218fb401 \ - "$GCC15_ROOT/usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.34" \ - | sha256sum -c - -~~~ - -### 写入用户 .bashrc - -$HOME/.bashrc 是文件，不是目录。把以下内容追加到文件末尾；这里的 HOME -就是当前普通用户的 home，不是系统 /root： - -~~~bash -# Ascend CANN user installation. -if [ -f "$HOME/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh" ]; then - source "$HOME/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh" -fi - -# User-local GCC 15 (Ubuntu 25.04 Plucky packages). -export GCC15_ROOT="$HOME/.local/gcc-15/root" -if [ -x "$GCC15_ROOT/usr/bin/g++-15" ]; then - export PATH="$GCC15_ROOT/usr/bin:$PATH" - if [ -n "$LD_LIBRARY_PATH" ]; then - export LD_LIBRARY_PATH="$GCC15_ROOT/usr/lib/x86_64-linux-gnu:$GCC15_ROOT/usr/lib/gcc/x86_64-linux-gnu/15:$LD_LIBRARY_PATH" - else - export LD_LIBRARY_PATH="$GCC15_ROOT/usr/lib/x86_64-linux-gnu:$GCC15_ROOT/usr/lib/gcc/x86_64-linux-gnu/15" - fi -fi - -# User Python environment. -if [ -f "$HOME/.venv/bin/activate" ]; then - source "$HOME/.venv/bin/activate" -fi -~~~ - -自动激活 venv 会影响所有新开的交互 shell，这是本次用户要求的行为。CI、 -cron 或非交互脚本仍应显式 source 对应环境。 - -保存后打开新的交互 shell，或执行 exec bash，再验证： - -~~~bash -command -v ccec -command -v g++-15 -g++-15 --version | head -n 1 -g++-15 -print-prog-name=cc1plus - -ldd "$(g++-15 -print-prog-name=cc1plus)" | \ - grep 'not found' && exit 1 || true - -printf '#include \nint main(){std::cout << "gcc15-ok\\n";}\n' | - g++-15 -x c++ -std=c++23 - -o /tmp/gcc15-smoke -/tmp/gcc15-smoke -~~~ - -预期版本首行为： - -~~~text -g++-15 (Ubuntu 15-20250404-0ubuntu1) 15.0.1 20250404 (experimental) -~~~ - -## 4. Python、PTO-ISA 与精确构建 - -### 创建用户 Python 环境 - -本次使用 $HOME/.venv，而不是仓库内的 .venv。首次创建： - -~~~bash -python3 -m venv --system-site-packages "$HOME/.venv" -source "$HOME/.venv/bin/activate" - -python --version -python -m pip --version -~~~ - ---system-site-packages 与当前板端部署一致，使已安装的 torch 2.6.0+cpu -可见。目标用例直接 import torch 来生成输入和 golden，因此 PyTorch 必需； -它不直接 import torch_npu。 - -安装本次用到的 Python 和构建依赖： - -~~~bash -python -m pip install \ - scikit-build-core==1.0.3 \ - nanobind==2.13.0 \ - cmake==4.4.0 \ - cloudpickle==3.1.2 \ - pytest==7.4.4 \ - pytest-xdist==3.8.0 \ - pytest-timeout==2.4.0 \ - ruff==0.14.8 -~~~ - -若系统 site-packages 中没有 torch，再从当前环境认可的 wheel 源安装 -torch 2.6.0+cpu；不要未经确认改成最新版本。能访问 PyTorch 官方 CPU -wheel 源时可执行： - -~~~bash -python -c 'import torch; print(torch.__version__)' || \ - python -m pip install \ - --index-url https://download.pytorch.org/whl/cpu \ - torch==2.6.0 - -python -c ' -import pytest -import torch -print("python:", __import__("sys").version.split()[0]) -print("pytest:", pytest.__version__) -print("torch:", torch.__version__) -' -~~~ - -### 固定 PTO-ISA - -~~~bash -cd "$REPO_ROOT" -export PTO_ISA_COMMIT=ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8 -export PTO_ISA_ROOT="$REPO_ROOT/build/pto-isa" - -if [ ! -d "$PTO_ISA_ROOT/.git" ]; then - git clone https://github.com/hw-native-sys/pto-isa.git \ - "$PTO_ISA_ROOT" -fi - -git -C "$PTO_ISA_ROOT" fetch origin "$PTO_ISA_COMMIT" -git -C "$PTO_ISA_ROOT" checkout --detach "$PTO_ISA_COMMIT" -test "$(git -C "$PTO_ISA_ROOT" rev-parse HEAD)" = \ - "$PTO_ISA_COMMIT" -~~~ - -### 只构建 Python binding - -直接执行 pip install -e . 会触发顶层 ALL target，并自动枚举当前可构建的 -所有平台和 runtime。为保持本文边界，先只构建 _task_interface： - -~~~bash -cd "$REPO_ROOT" -source "$HOME/.venv/bin/activate" -source "$HOME/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh" - -# 移除这个 venv 中可能残留的旧 simpler/editable import hook。 -# 只删除 Python 安装记录，不删除当前源码树或 build 产物。 -if python -m pip show simpler >/dev/null 2>&1; then - python -m pip uninstall -y simpler -fi - -cmake -S . -B build/python-bindings \ - -DCMAKE_BUILD_TYPE=Release \ - -DPython_EXECUTABLE="$(command -v python)" \ - -Dnanobind_DIR="$(python -c \ - 'import nanobind; print(nanobind.cmake_dir())')" - -cmake --build build/python-bindings \ - --target _task_interface \ - --parallel "$(nproc)" - -if [ -n "$PYTHONPATH" ]; then - export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/python:$PYTHONPATH" -else - export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/python" -fi -python -c ' -from pathlib import Path -import simpler -import _task_interface -print("simpler:", simpler.__file__) -print("_task_interface:", _task_interface.__file__) -assert Path(_task_interface.__file__).resolve().parent == \ - Path("python").resolve() -' -~~~ - -不要把项目专用 PYTHONPATH 永久写入全局 .bashrc。每次进入本仓工作时设置， -或在测试命令所在 shell 中保持以上 export 即可。若不移除旧 editable -安装，它注册的 import hook 可能优先加载 site-packages 中的旧 binding， -使刚构建的源码树产物没有真正被测试。 - -### 只构建目标 runtime - -当前 build_runtimes.py 的 --platforms 只能限制平台，不能限制 runtime。 -使用 RuntimeBuilder 的现有接口，精确构建 A5Sim/A5 的 -fully_distributed_within_core 及它们必需的共享 helper： - -~~~bash -cd "$REPO_ROOT" -source "$HOME/.venv/bin/activate" -source "$HOME/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh" -if [ -n "$PYTHONPATH" ]; then - export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/python:$PYTHONPATH" -else - export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/python" -fi - -python - <<'PY' -from simpler_setup.runtime_builder import RuntimeBuilder - -runtime = "fully_distributed_within_core" -for platform in ("a5sim", "a5"): - print(f"building {platform}/{runtime}") - binaries = RuntimeBuilder(platform).get_binaries(runtime, build=True) - print(binaries) -PY -~~~ - -验证目标产物： - -~~~bash -test -f \ - build/lib/a5/sim/fully_distributed_within_core/libhost_runtime.so -test -f \ - build/lib/a5/sim/fully_distributed_within_core/libaicpu_kernel.so -test -f \ - build/lib/a5/sim/fully_distributed_within_core/libaicore_kernel.so - -test -f \ - build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so -test -f \ - build/lib/a5/onboard/fully_distributed_within_core/libaicpu_kernel.so -test -f \ - build/lib/a5/onboard/fully_distributed_within_core/aicore_kernel.o - -test -f build/lib/a5/dispatcher/libsimpler_aicpu_dispatcher.so -test -f build/lib/libsimpler_log.so -test -f build/lib/libcpu_sim_context.so -~~~ - -编译器职责如下： - -| 目标 | 编译器 | -| --- | --- | -| A5Sim incore kernel | 用户级 g++-15 | -| A5Sim host/runtime helper | 系统 gcc/g++ | -| A5 AICore kernel | CANN 9.1 ccec | -| A5 AICPU 目标 | CANN 9.1 AArch64 交叉编译器 | -| A5 host 目标 | 系统 gcc/g++ | - -因此，CMake cache 中看到系统 g++ 不代表 GCC 15 被绕过；A5Sim incore -kernel 是后续由 KernelCompiler 直接调用 g++-15 编译的。 - -## 5. 执行 A5Sim 与真实 A5 - -每个新 shell 先执行统一准备： - -~~~bash -cd "$REPO_ROOT" -source "$HOME/.venv/bin/activate" -source "$HOME/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh" - -if [ -n "$PYTHONPATH" ]; then - export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/python:$PYTHONPATH" -else - export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/python" -fi -export PTO_ISA_ROOT="$REPO_ROOT/build/pto-isa" -export PTO_ISA_COMMIT=ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8 - -TEST_FILE=examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py - -test "$(command -v python)" = "$HOME/.venv/bin/python" -test "$(git -C "$PTO_ISA_ROOT" rev-parse HEAD)" = \ - "$PTO_ISA_COMMIT" -~~~ - -### 运行 A5Sim Case1 - -~~~bash -python -m pytest "$TEST_FILE" \ - --platform a5sim \ - --case Case1 \ - --enable-l2-swimlane \ - --use-example-exec-time \ - --clone-protocol https \ - --pto-isa-commit "$PTO_ISA_COMMIT" \ - --pto-session-timeout 1200 \ - --require-pto-isa \ - -s -v -~~~ - ---use-example-exec-time 仅适用于 fully_distributed_within_core 的 sim。 -它不能用于真实 A5 命令。 - -### 运行 A5 正确性 smoke - -确认没有其他进程占用 device 0 后执行： - -~~~bash -test -r /dev/davinci0 -test -w /dev/davinci0 - -python -m pytest "$TEST_FILE" \ - --platform a5 \ - --device 0 \ - --case Case1 \ - --clone-protocol https \ - --pto-isa-commit "$PTO_ISA_COMMIT" \ - --pto-session-timeout 1200 \ - --require-pto-isa \ - -s -v -~~~ - -### 运行 A5 swimlane 性能复现 - -~~~bash -python -m pytest "$TEST_FILE" \ - --platform a5 \ - --device 0 \ - --case Case1 \ - --enable-l2-swimlane 4 \ - --clone-protocol https \ - --pto-isa-commit "$PTO_ISA_COMMIT" \ - --pto-session-timeout 1200 \ - --require-pto-isa \ - -s -v -~~~ - -本次实测生成： - -~~~text -outputs/TestPagedAttentionUnroll_Case1_20260717_023809/ -l2_swimlane_records.json -~~~ - -新的复现会生成不同时间戳目录。trace 约几十 MiB，pytest 结束后再读取， -不要用 pytest wall time 代替 Submit 指标。 - -### 提取首个到末个 Submit - -以下脚本自动选择最新 Case1 trace，校验 96 个 core、每核 1280 个 Submit -以及完整 task id，并输出用户关注的全局 span： - -~~~bash -python - <<'PY' -import json -import statistics -from collections import defaultdict -from pathlib import Path - -traces = list( - Path("outputs").glob( - "TestPagedAttentionUnroll_Case1_*/l2_swimlane_records.json" - ) -) -if not traces: - raise SystemExit("no Case1 l2_swimlane_records.json found") - -trace = max(traces, key=lambda path: path.stat().st_mtime) -with trace.open() as stream: - data = json.load(stream) - -hz = int(data["metadata"]["clock_freq_hz"]) -submits = [row for row in data["fdwic_events"] if row[5] == "Submit"] -if not submits: - raise SystemExit("trace contains no Submit events") - -by_core = defaultdict(list) -for row in submits: - by_core[int(row[0])].append(row) - -assert int(data["metadata"]["num_cores"]) == 96 -assert len(by_core) == 96 -for core, rows in by_core.items(): - task_ids = sorted(int(row[3]) for row in rows) - assert len(rows) == 1280, (core, len(rows)) - assert task_ids == list(range(1280)), core - -first_cycle = min(int(row[6]) for row in submits) -last_cycle = max(int(row[7]) for row in submits) -first_to_last_ms = (last_cycle - first_cycle) * 1000 / hz - -kernel_first_cycle = min( - int(row[6]) for row in submits if int(row[3]) == 1 -) -kernel_to_last_ms = (last_cycle - kernel_first_cycle) * 1000 / hz - -per_core_ms = [] -for rows in by_core.values(): - start = min(int(row[6]) for row in rows) - end = max(int(row[7]) for row in rows) - per_core_ms.append((end - start) * 1000 / hz) - -first_row = min(submits, key=lambda row: int(row[6])) -last_row = max(submits, key=lambda row: int(row[7])) -assert int(first_row[3]) == 0 -assert int(last_row[3]) == 1279 - -print("trace:", trace) -print("clock_freq_hz:", hz) -print("cores:", len(by_core)) -print("submits_per_core:", len(next(iter(by_core.values())))) -print(f"first_to_last_submit_ms: {first_to_last_ms:.6f}") -print(f"task1_to_last_submit_ms: {kernel_to_last_ms:.6f}") -print(f"per_core_median_ms: {statistics.median(per_core_ms):.7f}") -print(f"per_core_max_ms: {max(per_core_ms):.6f}") -PY -~~~ - -本次预期输出的关键值： - -~~~text -clock_freq_hz: 1000000000 -cores: 96 -submits_per_core: 1280 -first_to_last_submit_ms: 5.642245 -task1_to_last_submit_ms: 5.635263 -per_core_median_ms: 5.5725575 -per_core_max_ms: 5.641331 -~~~ - -不同运行允许有小幅抖动。验收重点是正确性 PASSED、事件完整，并且 -first_to_last_submit_ms 仍位于约 5.6 ms 的基线附近。 - -## 6. 验收清单与故障定位 - -### 最终验收 - -- CANN 安装包 SHA-256 与本文一致； -- ASCEND_HOME_PATH 指向用户目录下的 CANN 9.1； -- command -v python 为 $HOME/.venv/bin/python； -- command -v g++-15 指向 $HOME/.local/gcc-15/root； -- PTO-ISA HEAD 为固定 commit； -- 源码中没有 block_dim=36； -- 只构建 A5Sim/A5 的 fully_distributed_within_core； -- A5Sim Case1 PASSED； -- A5 Case1 PASSED； -- trace 为 96 core，每核 1280 个 Submit； -- 全局首末 Submit 约为 5.6 ms。 - -### 常见问题 - -**找不到 g++-15** - -确认 GCC15_ROOT、PATH 和 LD_LIBRARY_PATH 已生效，并重新打开交互 shell。 -A5Sim incore kernel 必须能直接执行 g++-15。 - -**找不到 ccec 或 AArch64 交叉编译器** - -重新 source 用户 CANN 9.1 的 cann/set_env.sh，并检查 -ASCEND_HOME_PATH。不要回退到同目录的 CANN 9.2。 - -**提示 pre-built runtime binaries not found** - -重新执行“只构建目标 runtime”中的 RuntimeBuilder 片段。不要改用会自动 -枚举所有 runtime 的顶层构建。 - -**CPU_TOPO 的 HAL/DSMI 返回 65534** - -这是当前旧 Driver 的已知能力差异。只有日志同时表明 OCCUPY popcount 与 -ACL AICPU count 一致，并出现 using flat OCCUPY fallback 时才可继续。 -本次预期是 mask=0x3e、count=5。若出现 flat fallback rejected，停止运行， -不要删除校验或强行构造 CPU 列表。 - -**仍然使用 block_dim=36** - -说明源码 revision 不完整。切换到同时包含本文三处源码调整的 revision， -再增量重建 A5 目标 runtime。 - -**PTO-ISA clone 超时或 commit 不一致** - -先在 build/pto-isa 中独立完成 fetch 和 detached checkout，再运行 pytest。 ---require-pto-isa 会让错误尽早暴露，不能去掉 pin 后继续跑未知版本。 - -**import torch 失败** - -确认 venv 是用 --system-site-packages 创建，或从环境认可的 wheel 源安装 -torch 2.6.0。该用例需要 torch，但不因这一点要求直接调用 torch_npu。 - -**出现 torch_npu library owner permission mismatch warning** - -当前系统 site-packages 可能在 import 阶段报告某个 torch_npu 共享库 owner -不匹配。目标用例不直接使用 torch_npu；若 torch、simpler 均可导入且测试 -PASSED，该 warning 不影响本次结论。不要以普通用户修改系统共享库的 owner； -若它升级为 import error，再交由系统环境维护者处理。 - -**/dev/davinci0 无权限或设备忙** - -由系统管理员修复用户组/ACL，或等待当前任务释放设备。不要 sudo 运行 pytest， -否则会绕开用户 venv、HOME 和 CANN 安装路径。 - -**结果显示 70 到 80 ms** - -这通常是整段 device wall time，不是本文指标。必须读取 -l2_swimlane_records.json 的 fdwic_events，并按本文章节计算 Submit span。 - -### 建议保存的复现证据 - -每次正式复现至少保留： - -~~~bash -git rev-parse HEAD -git status --short -git diff -- \ - examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py \ - src/a5/platform/onboard/host/aicpu_topology_probe.cpp \ - src/a5/platform/onboard/host/aicpu_topology_probe.h - -python --version -python -c 'import torch; print(torch.__version__)' -g++-15 --version | head -n 1 -grep -E '^(Version|timestamp)=' \ - "$ASCEND_HOME_PATH/opp/version.info" -git -C "$PTO_ISA_ROOT" rev-parse HEAD -~~~ - -同时归档 pytest 完整日志和对应的 l2_swimlane_records.json。这样可以区分 -代码变化、工具链变化、设备占用和真实性能回归。 diff --git "a/tests/atomic_probe/AICore\345\265\214\345\245\227Lambda\346\215\225\350\216\267\344\270\216\346\250\241\346\235\277\350\260\203\347\224\250\351\252\214\350\257\201.md" "b/tests/atomic_probe/AICore\345\265\214\345\245\227Lambda\346\215\225\350\216\267\344\270\216\346\250\241\346\235\277\350\260\203\347\224\250\351\252\214\350\257\201.md" index 68adce5af7..824d40c10b 100644 --- "a/tests/atomic_probe/AICore\345\265\214\345\245\227Lambda\346\215\225\350\216\267\344\270\216\346\250\241\346\235\277\350\260\203\347\224\250\351\252\214\350\257\201.md" +++ "b/tests/atomic_probe/AICore\345\265\214\345\245\227Lambda\346\215\225\350\216\267\344\270\216\346\250\241\346\235\277\350\260\203\347\224\250\351\252\214\350\257\201.md" @@ -1,52 +1,66 @@ -# AICore 嵌套 Lambda、捕获与 caller 栈地址验证 +# AICore 嵌套 Lambda、caller 栈地址与 inline 调用边界验证 > 最后验证日期：2026-07-17 > -> 验证仓库：`simpler`，基线 HEAD `52ca4f5eba343c2f7b7a3a743e575cb9308d128f` +> 验证基线 HEAD：`3a3de54db0a900e489a5a5496cbee1dc5b76be7a` > > 测试目录：`tests/atomic_probe` > -> 验证平台：A5 `Ascend950PR_9599`、CANN 9.1.0 +> 验证平台：A5 device 0、CANN 9.1.0 ## 1. 文档目的和最终结论 -本文是一份可以独立阅读和执行的复现手册。读者不需要先了解此前的排查过程， -按本文即可完成以下验证： +本文验证四件事： 1. CPU、AscendC 和纯 CCEC 是否支持嵌套 lambda、捕获和模板调用； -2. 是否能在 A5 上复现 caller 栈地址经过 runtime 调用后使用时的异常； -3. “删除独立 context，把 capture 写入现有 `L0TaskArgs`，由 runtime TU - 直接读取”的数据驱动方案是否可行； -4. CPU 对等实现是否具有相同功能，以及是否存在明显生命周期或未定义行为。 - -本次结果为： - -- CPU、AscendC AIV、纯 CCEC AIV 的嵌套 lambda/捕获/模板基线全部通过； -- 纯 CCEC AIC 的 `weak-context-materialize-0` 在最终测试版本上 5/5 次触发 - `507015 AICore exception`； -- 同一版探针中的 `args-runtime-read` 在 A5 上 11/11 次通过，累计 - 704 轮、2816 次 submit； -- CPU caller 与 AIC 共用同一份 runtime TU 源码，功能验证通过；GCC - `-O0/-O2/-O3` 各 20/20 次通过，ASan+UBSan 通过。 - -必须保留的结论边界：业务用例是 1/2 条无业务地址物化仍失败，第 3 条 -`block_table` 地址物化后通过；当前纯 CCEC 探针是 0 条失败、1/2/3 条通过。 -因此本文已经复现“caller 栈地址传递对 HiIPU 最终 codegen 敏感”的核心现象， -但不是与业务阈值和机器码完全相同的最小复现。 - -当前设备状态也必须单独记录：最后一次负向复核之后，后续正向检查在 -`aclInit(nullptr)` 阶段返回 500000；用户已确认当前 NPU 设备不存在。本文此前 -的 A5 数据是在设备仍存在时取得，现在不能继续上板验证。没有芯片或驱动侧 -故障日志，不能仅凭时间先后断言是本 probe 导致设备物理掉线。 +2. caller 栈上的 `Tensor` 地址经过未内联 submit 调用后，是否会触发 + AICore 异常； +3. 异常是否由“链接两个 `.o`”或 `ld.lld` 跨对象重定位直接导致； +4. runtime submit 全 inline，以及把 capture 写入既有 `L0TaskArgs`， + 是否可作为当前工具链的规避形态。 -## 2. 最短复现路径 +本轮得到的核心矩阵如下： + +| AIC 构建形态 | submit 形态 | 语义变体 | 本轮结果 | +| ------------ | ----------- | -------- | -------- | +| 原始双 `.o` | 外部 submit | m0 | 5/5 507015 | +| 双 `.o`，第二个 `.text=0` | 全 inline | m0 | 5/5 PASS | +| 单 `.o` | 全 inline | m0 | 5/5 PASS | +| 单 `.o` | 仅 weak-context submit noinline | m0 | 5/5 507015 | +| 同一 noinline fixture | 仅 weak-context submit noinline | m1 | 3/3 PASS | +| 同一 noinline fixture | strong 路径保持 inline | strong | 3/3 PASS | + +因此可以排除“只要链接两个 `.o` 就会失败”。单 `.o` 保留 +`nested_probe_submit_weak_context` 未内联时仍然失败，而双 `.o` +在第二个对象没有代码、submit 已完全内联时通过。 + +当前最小、可重复的触发边界是： -如果只想最快确认结果，执行本节即可。后续章节解释原理、源码和每个测试的 -完整判定规则。 +```text +weak-context-materialize-0 ++ nested_probe_submit_weak_context 未内联 ++ caller 栈地址跨该调用边界 +-> 507015 AICore exception +``` -### 2.1 设置路径 +全 inline 后 m0 通过；同一 noinline 产物增加一次无业务地址物化后也通过。 +这仍然是对 HiIPU 最终 codegen 形态的收敛，不是对某个具体后端 pass 或 +某条机器指令的根因定位。 -将前两行替换为本机实际路径： +需要严格保留以下结论边界： + +- 不是 C++ lambda、捕获或模板语言语义不受支持； +- 没有证据表明 `ld.lld` 链接两个对象本身有错误； +- 不能把 weak、noinline 或零地址物化任一单因素写成充分根因； +- all-inline 是已验证规避，不是 CCEC 编译器修复； +- `args-runtime-read` 仍是避免独立 caller context 的数据驱动方案， + 与“submit 全 inline”是两个不同维度。 + +## 2. 最短复现路径 + +以下命令从仓库根目录执行。 + +### 2.1 环境 ```bash export REPO=/path/to/simpler @@ -58,19 +72,18 @@ export PTO_ISA_ROOT="$CANN_ROOT/x86_64-linux" export ATOMIC_PROBE_DEVICE=0 ``` -确认关键文件和工具存在： +确认工具和源码存在： ```bash test -x "$ASCEND_HOME_PATH/bin/ccec" -test -x "$ASCEND_HOME_PATH/bin/bisheng" test -x "$ASCEND_HOME_PATH/bin/ld.lld" test -f "$PTO_ISA_ROOT/include/pto/common/kernel_meta.hpp" test -f tests/atomic_probe/ccec/nested_lambda_cross_tu.cpp -test -f tests/atomic_probe/ccec/nested_lambda_cross_tu_runtime.cpp -test -f tests/atomic_probe/cpu/nested_lambda_args_runtime_read.cpp +test -f tests/atomic_probe/ccec/nested_lambda_cross_tu_api.h +test -f tests/atomic_probe/ccec/nested_lambda_only_weak_submit_noinline.cpp ``` -### 2.2 CPU 对等功能验证 +### 2.2 CPU 语义和 inline runtime-read ```bash tests/atomic_probe/run_nested_lambda.sh cpu @@ -80,837 +93,474 @@ tests/atomic_probe/run_nested_lambda.sh cpu ```text [ASSERT] CPU nested capture/template semantics PASS -[VALUES] rounds=64 mismatches=0 checksum=0x3e6cd1b792bff0e0 L0TaskArgs=1024B +[VALUES] rounds=64 mismatches=0 checksum=0x3e6cd1b792bff0e0 [ASSERT] CPU L0TaskArgs args-runtime-read semantics PASS [SUMMARY] semantic_failures=0 ``` -### 2.3 编译 AIC 双 TU 探针 +CPU 和 AIC 都从 `nested_lambda_cross_tu_api.h` 使用同一份 inline +runtime 实现。CPU 不再链接独立 runtime TU。 + +### 2.3 默认单对象、全 inline 正向组 + +构建： ```bash tests/atomic_probe/ccec/run_all.sh nested_lambda_cross_tu build ``` -命令必须以 0 退出，并生成： +构建过程会检查： ```text -tests/atomic_probe/ccec/build/nested_lambda_cross_tu_kernel_caller_aic.o -tests/atomic_probe/ccec/build/nested_lambda_cross_tu_kernel_runtime_aic.o -tests/atomic_probe/ccec/build/nested_lambda_cross_tu_kernel.o -tests/atomic_probe/ccec/build/nested_lambda_cross_tu_host +[ASSERT] CCEC caller-capture runtime symbol shape PASS +[VALUES] aic_input_objects=1 runtime_text=n/a submit_symbols=none ``` -### 2.4 运行可落地方案组 +运行默认正向组： ```bash -ATOMIC_PROBE_MODE=args-runtime-read \ - tests/atomic_probe/ccec/run_all.sh nested_lambda_cross_tu run +tests/atomic_probe/ccec/run_all.sh nested_lambda_cross_tu run ``` -关键期望输出和退出码： +固定顺序和期望结果： ```text -[ASSERT] CCEC AIC cross-TU ABI variant=args-runtime-read PASS -[VALUES] rounds=64 mismatches=0 dispatches=0 materializations=3 \ -checksum=0x3e6cd1b792bff0e0 L0TaskArgs=1024B -[SUMMARY] semantic_failures=0 +args-runtime-read PASS +weak-context-materialize-0 PASS +run_failures=0 ``` -退出码必须为 0。 - -### 2.5 运行故障对照组 - -该组会故意触发 AICore 异常。必须使用独立进程，并遵守所在环境的设备独占和 -异常恢复规则。 +也可以单独运行 m0： ```bash -set +e ATOMIC_PROBE_MODE=weak-context-materialize-0 \ tests/atomic_probe/ccec/run_all.sh nested_lambda_cross_tu run -bad_rc=$? -set -e -echo "bad_rc=$bad_rc" ``` -本次受影响编译器上的期望结果： +严格 oracle 为： ```text -ACL error 507015 from aclrtSynchronizeStream(stream) ... -CCEC [nested_lambda_cross_tu_kernel] failed runs: 1 -=== Done. run_failures=1 === -bad_rc=1 -``` - -此处退出码 1 是“故障复现成功”，不是 launcher 构建失败。如果该组返回 0， -说明当前编译器或当前代码布局没有命中此复现边界，不能据此认为方案组失败。 - -## 3. 被验证的问题是什么 - -### 3.1 原始业务现象 - -业务中的 private-lazy 调用会在 orchestration 栈上构造多个 `Tensor`，再把 -这些对象的地址放入一个 caller context，经过 runtime 调用后继续使用： - -```text -orchestration caller - -> runtime submit - -> dispatcher/recipe bind - -> 使用 caller 栈上的 Tensor* -``` - -已有业务证据为： - -- 优化后的 LLVM IR 正确写入三个 `Tensor *` context 字段； -- orchestration CFA 为 `reg93 + 1952`，栈帧只有 1952B； -- 32 KiB 和 64 KiB 栈配置都失败，所以不是普通栈容量不足； -- 不参与业务的地址写会改变 PASS/FAIL； -- 去掉 `-cce-aicore-addr-transform` 后业务仍失败，不能归罪于单一 pass； -- 同 TU、可内联 dispatcher 仍失败，说明跨 TU 外部回调不是根因； -- 业务原发日志包含 code 264：scalar 访问使用了无效 GM 地址；外层最终表现为 - `507018 AICPU exception`。 - -### 3.2 当前最合理的出错原理 - -坏路径可以简化为： - -```text -reg93 栈 - ├─ Tensor first - ├─ Tensor second - ├─ Tensor third - ├─ CallerContext {&first, &second, &third, salt} - └─ L0TaskArgs args - │ - └─ runtime(site_id, &context, &args) - │ - └─ 再解引用 context 中的 Tensor* +rounds=64 +mismatches=0 +dispatches=128 +materializations=0 +checksum=0x3e6cd1b792bff0e0 +L0TaskArgs=1024B +进程退出码=0 ``` -LLVM IR 已经正确，栈容量也足够。可疑区间位于优化后 LLVM IR 到 HiIPU 最终 -机器码之间，包括栈地址物化、指令选择、活跃区间、调度和寄存器分配。 -`&Tensor` 来自 `reg93 + offset`；如果其地址寄存器在调用边界被错误复用、 -覆盖或以错误偏移传递，runtime 后续解引用就会访问无效地址。 - -无业务用途的 `ptrtoint/store` 会改变地址的活跃区间和寄存器选择，所以即使 -LLVM 业务语义完全不变，最终机器码也可能从失败形态切换到通过形态。这是 -Heisenbug 诊断特征，不是合法修复方式。 - -目前尚未证明是某一个具体 CCEC pass。本文只把问题边界收敛到 HiIPU 后端的 -地址物化/调度/寄存器分配组合，不声称已经完成编译器根因定位。 +### 2.4 双对象数量控制 -### 3.3 数据驱动方案为什么可能绕开问题 +该 fixture 仍向 `ld.lld` 传入两个 AIC 对象，但第二个对象的 +`.text` 大小必须为零，且所有 submit 已进入 caller 对象并被内联。 -方案组不再构造独立 `CallerContext`： +```bash +tests/atomic_probe/ccec/run_all.sh \ + nested_lambda_inline_plus_empty_runtime build -```text -caller 栈上的 Tensor 地址 - -> 写入既有 L0TaskArgs 固定 slot - -> 跨调用只传稳定使用的 args_ptr - -> runtime TU 直接读取 slot - -> submit 返回前 add_input 并完成物化 +tests/atomic_probe/ccec/run_all.sh \ + nested_lambda_inline_plus_empty_runtime run ``` -它减少了一条独立 caller 栈指针传参链，并迫使三个地址在调用前形成明确的 -内存表示。它是对易错 codegen 形态的规避，不是对 CCEC 后端的修复。 - -## 4. 测试分层与调用结构 - -### 4.1 第一层：嵌套 lambda/捕获/模板语言基线 - -共享测试形态为： +构建期望： ```text -自由函数模板 Submit(outer_lambda) - -> outer lambda 按引用捕获 caller 状态 - -> SubmitBuilder::AddInput(inner_lambda) - -> SubmitBuilder::AddOutput(inner_lambda) - -> SubmitBuilder::AddScalar(inner_lambda) +[VALUES] aic_input_objects=2 runtime_text=000000 submit_symbols=none ``` -具体覆盖： - -- outer lambda：`[&]`； -- input lambda：`[&]`，修改引用捕获； -- output lambda：`[outer_local, &output_calls]`，混合按值/按引用捕获； -- scalar lambda：按值和按引用混合捕获； -- 自由函数模板 `Submit`； -- 成员函数模板 `AddInput/AddOutput/AddScalar`； -- AICore 版本的 lambda 显式标注 `__aicore__`。 - -这一层分别由 CPU、AscendC AIV 和纯 CCEC AIV 实现。它只能回答“语言和 -单 TU 语义是否支持”，不能单独复现 caller 栈地址问题。 +运行期望：m0 PASS、退出码 0。该控制证明两个链接输入对象本身不足以触发 +异常。 -### 4.2 第二层：caller capture 传输 A/B +### 2.5 only-weak-submit-noinline 故障控制 -第二层直接复用仓库中的真实类型： +该 fixture 仍只编译一个有代码的 AIC 对象，但仅保留 +`nested_probe_submit_weak_context` 为 noinline。其他 runtime submit、 +digest、consume 和 control 均保持 `always_inline`。 -```text -L0TaskArgs -Tensor -TaskOutputTensors +```bash +tests/atomic_probe/ccec/run_all.sh \ + nested_lambda_only_weak_submit_noinline build ``` -没有另造简化容器，也没有链接完整 Simpler runtime。探针只复用类型定义， -用单独的 `nested_lambda_cross_tu_runtime.cpp` 模拟待测 runtime 调用边界。 -ACL 只负责加载 raw AICore ELF、启动 kernel 和回读结果，不参与被测 ABI。 - -每个变体执行： +构建期望： ```text -64 轮 - 每轮在 caller 栈上新建 3 个 Tensor - 复用同一个 L0TaskArgs - 第 1 次 submit：待测 lazy/capture 路径 - 第 2~4 次 submit：控制路径 +[VALUES] aic_input_objects=1 runtime_text=n/a +submit_symbols=*nested_probe_submit_weak_context* ``` -这保留了“同一容器四次 submit”和 caller 栈对象反复创建/销毁的关键形态。 - -### 4.3 `L0TaskArgs` recipe slot 布局 - -测试专用布局为： - -| slot | 内容 | 读取者 | -| ---: | --- | --- | -| scalar 8 | `&first` | runtime 或 dispatcher | -| scalar 9 | `&second` | runtime 或 dispatcher | -| scalar 10 | `&third` | runtime 或 dispatcher | -| scalar 11 | `salt` | runtime 或 dispatcher | -| scalar 0 | runtime 计算结果 | caller oracle | -| scalar 5 | dispatcher 次数 | caller oracle | -| scalar 6 | dispatcher 缺失标志 | caller oracle | - -8~11 是本探针的 recipe/诊断 slot，不属于生产 ABI。真实落地必须为 recipe -定义正式布局、容量和版本，不能直接把这些编号当成业务规范。 - -### 4.4 七个 AIC 变体 - -所有变体都是独立 global AIC kernel，地址物化数量在编译期固定，不是运行时 -分支。数字 entry 必须唯一。 - -| entry | 运行参数 | 独立 context | capture 读取位置 | dispatcher | 地址写 | 本次 A5 | -| ---: | --- | --- | --- | --- | ---: | --- | -| 0 | `weak-context-materialize-0` | 有 | caller dispatcher | weak | 0 | 507015 | -| 1 | `weak-context-materialize-1` | 有 | caller dispatcher | weak | 1 | PASS | -| 2 | `weak-context-materialize-2` | 有 | caller dispatcher | weak | 2 | PASS | -| 3 | `weak-context-materialize-3` | 有 | caller dispatcher | weak | 3 | PASS | -| 4 | `weak-args-storage` | 无 | caller dispatcher | weak | 3 个实际 capture | PASS | -| 5 | `strong-context` | 有 | caller dispatcher | strong | 0 | PASS | -| 6 | `args-runtime-read` | 无 | runtime TU 直接读取 | 无回调 | 3 个实际 capture | PASS | - -四个 `materialize-N` 组中的 scalar 8~10 不参与业务读取，只用于改变地址物化。 -`weak-args-storage` 中这些 slot 是实际 capture，但 runtime 仍回调 weak -dispatcher。`args-runtime-read` 才是最终待验证方案：没有第二个 context， -也没有反向回调。 - -### 4.5 精确 oracle - -语言基线固定 `seed=0x120`： - -| 字段 | 期望值 | -| --- | ---: | -| outer/input/output/scalar 调用次数 | 各 1 | -| `reference_state` | 300 | -| `input.value` | 591 | -| `output.value` | 323 | -| `scalar` | 337 | -| `combined` | 1251 | - -caller capture 组要求： - -- `completed_rounds == 64`； -- `mismatches == 0`； -- 64×4 次 submit 的 checksum 为 `0x3e6cd1b792bff0e0`； -- 回调组 dispatcher 次数为 128；`args-runtime-read` 为 0； -- 地址写数量和 variant echo 与 entry 完全一致； -- `sizeof(L0TaskArgs) == 1024`，小于 32 KiB 且按 64B 对齐。 - -## 5. 源码清单和复用关系 - -所有测试代码位于 `simpler/tests/atomic_probe`。 - -### 5.1 语言基线 - -| 文件 | 作用 | -| --- | --- | -| `nested_lambda_probe.h` | 三端共用的 builder、模板、字段布局和 oracle | -| `cpu/nested_lambda.cpp` | 标准 C++17 语言对照 | -| `ascendc/nested_lambda.asc` | AscendC AIV kernel 和 ACL host | -| `ccec/nested_lambda.cpp` | 纯 CCEC AIV kernel，无 AscendC API | -| `ccec/nested_lambda_host.cpp` | 纯 CCEC raw ELF launcher 和校验 | +先运行通过控制： -### 5.2 caller capture 与 CPU 对等用例 - -| 文件 | 作用 | -| --- | --- | -| `ccec/nested_lambda_cross_tu.cpp` | AIC caller、dispatcher、七个 kernel entry | -| `ccec/nested_lambda_cross_tu_runtime.cpp` | 独立 runtime TU；含回调组和直接读取组 | -| `ccec/nested_lambda_cross_tu_api.h` | caller/runtime 共用函数签名 | -| `ccec/nested_lambda_cross_tu_layout.h` | 变体、entry、结果字段和 checksum oracle | -| `ccec/nested_lambda_cross_tu_host.cpp` | 按 entry 启动 raw ELF 并回读校验 | -| `cpu/nested_lambda_args_runtime_read.cpp` | CPU caller 对等实现 | -| `run_nested_lambda.sh` | CPU、AscendC、CCEC 统一入口 | -| `ccec/run_all.sh` | CCEC 编译、链接和独立进程矩阵 runner | -| `test_atomic_probe.py` | CPU 和 A5 pytest 入口 | - -CPU 用例没有复制 runtime 实现。g++ 把 -`cpu/nested_lambda_args_runtime_read.cpp` 与同一份 -`ccec/nested_lambda_cross_tu_runtime.cpp` 作为两个 TU 编译，再通过 -`--gc-sections` 裁掉 CPU 未调用的 dispatcher 控制函数。CPU 和 AIC 实际调用 -的是同一个 `nested_probe_submit_args_runtime_read()` 源码函数。 - -## 6. 环境与构建前置条件 - -### 6.1 CPU - -最低要求： +```bash +ATOMIC_PROBE_MODE=strong-context \ + tests/atomic_probe/ccec/run_all.sh \ + nested_lambda_only_weak_submit_noinline run -```text -bash -git -g++，支持 C++17 +ATOMIC_PROBE_MODE=weak-context-materialize-1 \ + tests/atomic_probe/ccec/run_all.sh \ + nested_lambda_only_weak_submit_noinline run ``` -CPU 功能测试不需要 CANN runtime，也不访问设备，但会包含仓库现有的 -`L0TaskArgs/Tensor` 头文件。 - -### 6.2 A5/CANN - -要求： - -```text -CANN 9.1.0 -ccec -bisheng -ld.lld -g++ -libascendcl.so -PTO kernel_meta.hpp -至少 1 个可用 A5 device -``` +两条命令都必须退出 0。 -推荐检查： +最后运行故障控制： ```bash -echo "ASCEND_HOME_PATH=$ASCEND_HOME_PATH" -"$ASCEND_HOME_PATH/bin/ccec" --version | head -n 3 -"$ASCEND_HOME_PATH/bin/bisheng" --version | head -n 1 -g++ --version | head -n 1 -test -f "$ASCEND_HOME_PATH/x86_64-linux/lib64/libascendcl.so" -test -f "$PTO_ISA_ROOT/include/pto/common/kernel_meta.hpp" +set +e +ATOMIC_PROBE_MODE=weak-context-materialize-0 \ + tests/atomic_probe/ccec/run_all.sh \ + nested_lambda_only_weak_submit_noinline run +bad_rc=$? +set -e +echo "bad_rc=$bad_rc" ``` -### 6.3 负向用例执行约束 - -故障组会触发 AICore exception，必须一次只运行一个变体，并在独立 host 进程 -中启动。host 强制要求 ` ` 两个参数；runner 也为每个变体 -创建独立进程，避免一个异常污染其他控制组的判定。 - -早期两组交替实验中，故障进程退出后方案组可以立即再次通过；但最后一次 -负向复核后，稍后的正向检查在 `aclInit(nullptr)` 就返回 500000，用户随后 -确认 NPU 设备已经不存在。因此不能把“进程退出即可恢复”写成通用结论；没有 -故障恢复条件时不要执行负向组。当前环境不再执行任何硬件命令。 - -### 6.4 本探针与 32K/64K 配置的关系 - -本探针不是 `aclInit` 栈配置测试：host 使用 `aclInit(nullptr)`，不读取额外 -ACL JSON。CCEC runner 的 AIC 编译参数包含： +受影响 CCEC 上的期望结果： ```text --O3 -g -x cce -std=c++17 --cce-aicore-only --mllvm -cce-aicore-stack-size=0x8000 --mllvm -cce-aicore-function-stack-size=0x8000 --mllvm -cce-aicore-record-overflow=false --mllvm -cce-aicore-addr-transform --mllvm -cce-aicore-dcci-insert-for-scalar=false --mllvm -cce-aicore-dcci-before-kernel-end=false ---cce-aicore-arch=dav-c310-cube +ACL error 507015 from aclrtSynchronizeStream(stream) ... +CCEC [...] failed runs: 1 +bad_rc=1 ``` -`0x8000` 是 32 KiB 上限，但实际 orchestration CFA 只有 1952B。业务侧已经 -独立验证 32K/64K 都失败，因此本文不再通过增大栈来解释或掩盖问题。 - -## 7. 逐项测试用例 - -以下命令除特别说明外，都从 `simpler` 仓库根目录执行。 +这里的退出码 1 表示故障控制命中，不是构建失败。该组必须在独立 host +进程中运行，并放在所有通过控制之后。 -### TC-CPU-01：CPU 语言基线与 runtime-read 对等实现 +## 3. 被验证的问题 -目的： +### 3.1 原始调用形态 -- 验证标准 C++17 的嵌套 lambda/捕获/模板语义； -- 验证真实 `L0TaskArgs/Tensor/TaskOutputTensors`； -- 用与 AIC 相同的 runtime TU 验证数据驱动 recipe 功能。 +业务中的 private-lazy 路径会在 orchestration 栈上构造多个 `Tensor`， +再把对象地址写入 caller context： -命令： - -```bash -tests/atomic_probe/run_nested_lambda.sh cpu +```text +orchestration caller + -> caller 栈上的 Tensor first/second/third + -> CallerContext {&first, &second, &third, salt} + -> submit(context, args) + -> dispatcher 解引用 context 中的 Tensor* ``` -判定：两个 `[ASSERT]` 均为 PASS，两个 summary 均为 0；runtime-read 的 -rounds、mismatch、checksum 和 `L0TaskArgs` 大小必须与第 2.2 节一致。 +已有证据包括： -pytest： +- 优化后的 LLVM IR 正确写入三个 `Tensor *`； +- orchestration CFA 为 `reg93 + 1952`，不是普通栈容量不足； +- 32 KiB 和 64 KiB 栈配置都失败； +- 无业务用途的地址写会改变 PASS/FAIL； +- 去掉 `-cce-aicore-addr-transform` 后业务仍失败； +- 业务原发日志包含无效 GM 地址访问。 -```bash -export PYTHONPATH=python -.venv/bin/python -m pytest \ - tests/atomic_probe/test_atomic_probe.py::test_cpu_nested_lambda_compiler_probe \ - -q -s -``` - -本次结果：`1 passed in 1.73s`。 +### 3.2 本轮新增的判别证据 -### TC-CPU-02：CPU 优化级别和 sanitizer +本轮把“对象数量”和“函数调用边界”拆开： -目的：排除该 CPU 实现仅在某个优化级别偶然通过，以及明显的越界、 -use-after-scope 或未定义行为。 - -下面的命令与本次实测一致： +```text +两个对象 + 空 runtime + submit 全 inline + -> PASS -```bash -INCLUDES=( - -Itests/atomic_probe/ccec - -Isrc/a5/platform/onboard/aicore - -Isrc/a5/platform/include - -Isrc/common/platform/include - -Isrc/common/task_interface - -Isrc/common/log/include - -Isrc/common - -Isrc/a5/runtime/fully_distributed_within_core/runtime - -Isrc/a5/runtime/fully_distributed_within_core/common - -Isrc/a5/runtime/fully_distributed_within_core/orchestration - -Isrc/a5/runtime -) -SOURCES=( - tests/atomic_probe/cpu/nested_lambda_args_runtime_read.cpp - tests/atomic_probe/ccec/nested_lambda_cross_tu_runtime.cpp -) - -for opt in 0 2 3; do - out="/tmp/cpu_args_runtime_read_O${opt}" - g++ "-O${opt}" -std=c++17 -Wall -Wextra -Werror -ffunction-sections \ - "${INCLUDES[@]}" "${SOURCES[@]}" -Wl,--gc-sections -o "$out" - for i in $(seq 1 20); do "$out" >/dev/null; done - echo "g++ -O${opt}: 20/20 PASS" -done - -out=/tmp/cpu_args_runtime_read_sanitize -g++ -O2 -std=c++17 -Wall -Wextra -Werror -ffunction-sections \ - -fsanitize=address,undefined -fno-omit-frame-pointer \ - "${INCLUDES[@]}" "${SOURCES[@]}" -Wl,--gc-sections -o "$out" -ASAN_OPTIONS=detect_leaks=1 UBSAN_OPTIONS=halt_on_error=1 "$out" +一个对象 + weak-context submit noinline + -> 507015 ``` -本次 GCC 13.3.0 结果：O0/O2/O3 各 20/20 PASS；ASan+UBSan PASS。本机没有 -`clang++`，所以未完成第二种 CPU 后端交叉验证。 +因此多对象链接不是必要条件，未内联 submit 才是当前最小 fixture 中的 +必要构建形态。它仍不是单独的充分解释，因为： -### TC-ASCENDC-01：AscendC AIV 语言基线 +- 同一 noinline fixture 的 m1 通过； +- 同一 fixture 的 strong 路径通过； +- 全 inline 的 weak m0 通过。 -目的：验证 `bisheng -xasc` 对设备 lambda、捕获和模板调用的支持。 +最合理的范围仍是优化 LLVM IR 到 HiIPU 最终机器码之间的地址物化、 +活跃区间、调度和寄存器分配组合。 -编译并上板： +当前安装的 `llvm-objdump` 能读取 `elf64-hiipu` 符号和 DWARF，但指令 +反汇编显示 ``。没有可用 fault PC，因此本文不声称已经定位 +具体机器指令。 -```bash -ATOMIC_PROBE_DEVICE=0 tests/atomic_probe/run_nested_lambda.sh ascendc -``` +### 3.3 两种规避形态 -仅编译、不访问设备： +全 inline 形态： -```bash -bisheng -xasc tests/atomic_probe/ascendc/nested_lambda.asc \ - --npu-arch=dav-3510 \ - -o /tmp/nested_lambda_ascendc +```text +caller 栈地址 + -> inline submit/helper + -> 不跨 nested_probe_submit_weak_context 函数边界 + -> dispatcher/consume ``` -上板期望： +数据驱动形态： ```text -[ASSERT] AscendC nested capture/template semantics PASS -[ASSERT] AscendC ACL cleanup PASS -[SUMMARY] semantic_failures=0 +caller 栈地址 + -> 写入既有 L0TaskArgs 固定 slot + -> inline args-runtime-read 直接读取 + -> submit 返回前完成 add_input 和结果物化 ``` -### TC-CCEC-AIV-01：纯 CCEC AIV 语言基线 +两者都在当前工具链上通过。生产实现仍必须保证 submit 返回前完成复制或 +物化，不能让异步阶段继续保存 caller 栈裸地址。 -目的：不包含 AscendC header，只用 CCEC 和 lowercase builtin 验证同一语言 -形态。 +## 4. 测试结构和 oracle -编译并上板： +### 4.1 语言语义层 -```bash -ATOMIC_PROBE_DEVICE=0 \ - tests/atomic_probe/ccec/run_all.sh nested_lambda -``` +CPU、AscendC AIV 和纯 CCEC AIV 共同覆盖： -拆分执行： +- outer lambda 按引用捕获； +- nested lambda 按值、按引用和混合捕获； +- 自由函数模板 `Submit`； +- 成员函数模板 `AddInput/AddOutput/AddScalar`； +- AICore lambda 的 `__aicore__` 标注。 -```bash -tests/atomic_probe/ccec/run_all.sh nested_lambda build -ATOMIC_PROBE_DEVICE=0 \ - tests/atomic_probe/ccec/run_all.sh nested_lambda run -``` +固定 `seed=0x120` 的期望值为： -期望： +| 字段 | 期望值 | +| ---- | -----: | +| outer/input/output/scalar 调用次数 | 各 1 | +| `reference_state` | 300 | +| `input.value` | 591 | +| `output.value` | 323 | +| `scalar` | 337 | +| `combined` | 1251 | -```text -[ASSERT] CCEC nested capture/template semantics PASS -[ASSERT] CCEC ACL cleanup PASS -[SUMMARY] semantic_failures=0 -``` +### 4.2 caller-capture 语义层 -### TC-CCEC-AIC-01：双 TU build-only 和 ELF 结构 +每个变体执行 64 轮，每轮执行 4 次 submit： -目的：确认 caller/runtime 分开编译、weak/strong 符号和七个入口都真实保留。 +- 第一次为待测 caller-capture 或 args-runtime-read 路径； +- 后三次为固定 control 路径； +- 每轮重新创建三个 caller 栈 `Tensor`； +- 同一轮复用一个 `L0TaskArgs`。 -构建： +精确 oracle： -```bash -tests/atomic_probe/ccec/run_all.sh nested_lambda_cross_tu build -``` +| 字段 | 期望 | +| ---- | ---: | +| completed rounds | 64 | +| mismatches | 0 | +| submits | 256 | +| checksum | `0x3e6cd1b792bff0e0` | +| `sizeof(L0TaskArgs)` | 1024B | +| callback dispatcher 次数 | 128 | +| args-runtime-read dispatcher 次数 | 0 | -静态核对： +`L0TaskArgs` 的测试 slot： -```bash -BUILD=tests/atomic_probe/ccec/build +| slot | 内容 | +| ---: | ---- | +| scalar 8 | `&first` | +| scalar 9 | `&second` | +| scalar 10 | `&third` | +| scalar 11 | `salt` | +| scalar 0 | 计算结果 | +| scalar 5 | dispatcher 次数 | +| scalar 6 | dispatcher 缺失标志 | -readelf -S -W "$BUILD/nested_lambda_cross_tu_kernel.o" \ - | rg '\.ascend\.meta\.' +这些编号仅属于探针，不是生产 ABI。 -readelf -Ws -W "$BUILD/nested_lambda_cross_tu_kernel_runtime_aic.o" \ - | rg 'nested_probe_(weak_.*dispatch|strong_context_dispatch)' +### 4.3 七个语义变体 -readelf -Ws -W "$BUILD/nested_lambda_cross_tu_kernel.o" \ - | rg 'nested_probe_orchestration|nested_lambda_cross_tu_.*mix_aic' +| entry | 运行参数 | 独立 context | 地址物化 | dispatcher | +| ----: | -------- | ------------ | -------: | ---------- | +| 0 | `weak-context-materialize-0` | 有 | 0 | weak | +| 1 | `weak-context-materialize-1` | 有 | 1 | weak | +| 2 | `weak-context-materialize-2` | 有 | 2 | weak | +| 3 | `weak-context-materialize-3` | 有 | 3 | weak | +| 4 | `weak-args-storage` | 无 | 3 | weak | +| 5 | `strong-context` | 有 | 0 | strong | +| 6 | `args-runtime-read` | 无 | 3 | 无回调 | -"$ASCEND_HOME_PATH/bin/llvm-objdump" --dwarf=frames \ - "$BUILD/nested_lambda_cross_tu_kernel.o" \ - | rg 'reg93 \+1952' -``` +变体本身不再绑定固定 PASS/FAIL。结果必须同时写明构建 fixture。尤其 m0 +在 all-inline fixture 中通过，在 only-weak-submit-noinline fixture 中失败。 -期望： +### 4.4 三个持久构建 fixture -- 七个 `.ascend.meta.*` section； -- runtime object 中两个 dispatcher 是 `WEAK UND`，strong dispatcher 是 - `GLOBAL UND`； -- context orchestration text 为 1432/1436/1440/1444B； -- `args-runtime-read` orchestration text 为 1484B； -- 七个 orchestration 的 CFA 均为 `reg93 + 1952`； -- 七个 global wrapper 均为 128B。 +| target | AIC 输入对象 | submit FUNC 符号 | 默认运行 | +| ------ | -----------: | ---------------- | -------- | +| `nested_lambda_cross_tu` | 1 | 0 | args-runtime-read、m0 | +| `nested_lambda_inline_plus_empty_runtime` | 2 | 0 | m0 | +| `nested_lambda_only_weak_submit_noinline` | 1 | 仅 weak-context submit | strong、m1、m0 | -### TC-CCEC-AIC-02：数据驱动方案正向验证 +第三个 target 的 m0 放在最后；预期 runner 退出 1。默认 pytest 不运行该 +故障组。 -目的：验证没有独立 context、没有反向 dispatcher 的 runtime-read 路径。 +## 5. 源码和测试入口 -```bash -ATOMIC_PROBE_MODE=args-runtime-read \ -ATOMIC_PROBE_DEVICE=0 \ - tests/atomic_probe/ccec/run_all.sh nested_lambda_cross_tu run -``` +| 文件 | 作用 | +| ---- | ---- | +| `nested_lambda_probe.h` | 三端共享的语言语义和 oracle | +| `cpu/nested_lambda.cpp` | 标准 C++17 语言对照 | +| `cpu/nested_lambda_args_runtime_read.cpp` | CPU inline runtime-read 对照 | +| `ccec/nested_lambda_cross_tu.cpp` | AIC caller、dispatcher 和七个入口 | +| `ccec/nested_lambda_cross_tu_api.h` | inline runtime 实现和 caller context | +| `ccec/nested_lambda_cross_tu_runtime.cpp` | `.text=0` 的第二对象控制 | +| `ccec/nested_lambda_inline_plus_empty_runtime.cpp` | 双对象数量控制 wrapper | +| `ccec/nested_lambda_only_weak_submit_noinline.cpp` | only-weak-submit-noinline wrapper | +| `ccec/nested_lambda_cross_tu_layout.h` | 变体、字段和精确 oracle | +| `ccec/nested_lambda_cross_tu_host.cpp` | raw ELF launcher 和结果校验 | +| `ccec/run_all.sh` | 三种 fixture 的构建、ELF 断言和运行 | +| `run_nested_lambda.sh` | CPU、AscendC、CCEC 统一入口 | +| `test_atomic_probe.py` | CPU 和默认 A5 正向 pytest | -严格判定： +文件名中的 `cross_tu` 是历史命名。当前默认 target 是单个有代码的 AIC +输入对象，不能再根据文件名推断构建形态。 -```text -rounds=64 -mismatches=0 -dispatches=0 -materializations=3 -checksum=0x3e6cd1b792bff0e0 -L0TaskArgs=1024B -semantic_failures=0 -进程退出码=0 -``` +## 6. 自动化测试 -pytest 会重新 build 再运行方案组： +### 6.1 CPU pytest ```bash export PYTHONPATH=python -ATOMIC_PROBE_DEVICE=0 .venv/bin/python -m pytest \ - tests/atomic_probe/test_atomic_probe.py::test_a5_ccec_nested_lambda_args_runtime_read \ +.venv/bin/python -m pytest \ + tests/atomic_probe/test_atomic_probe.py::test_cpu_nested_lambda_compiler_probe \ -q -s ``` -### TC-CCEC-AIC-03：独立 context 故障复现 - -目的：证明同一编译器和同一测试版本仍能触发 caller-context 异常，避免因为 -所有变体都通过而错误宣称方案有效。 +### 6.2 A5 默认正向 pytest ```bash -set +e -ATOMIC_PROBE_MODE=weak-context-materialize-0 \ -ATOMIC_PROBE_DEVICE=0 \ - tests/atomic_probe/ccec/run_all.sh nested_lambda_cross_tu run -rc=$? -set -e -echo "rc=$rc" -``` - -本机期望：`aclrtSynchronizeStream` 返回 507015，runner 和最终命令退出码都为 -1。由于异常发生在同步阶段，host 的普通成功清理路径不会执行；必须依赖独立 -进程和平台认可的异常恢复流程。 - -最新一次文档复核确实再次得到 507015；之后的正向恢复检查在 -`aclInit(nullptr)` 返回 500000，当前 NPU 已不再存在。只有具备平台恢复能力 -时才应继续执行本负向用例。 - -### TC-CCEC-AIC-04：完整七变体矩阵 - -runner 会把每个变体放在独立进程中，并把故障组放到最后，防止一个 AICore -exception 遮住其他控制组。 - -```bash -unset ATOMIC_PROBE_MODE -set +e -ATOMIC_PROBE_DEVICE=0 \ - tests/atomic_probe/ccec/run_all.sh nested_lambda_cross_tu run -matrix_rc=$? -set -e -echo "matrix_rc=$matrix_rc" -``` - -固定运行顺序： - -```text -strong-context PASS -args-runtime-read PASS -weak-args-storage PASS -weak-context-materialize-3 PASS -weak-context-materialize-2 PASS -weak-context-materialize-1 PASS -weak-context-materialize-0 507015 +export PYTHONPATH=python +.venv/bin/python -m pytest \ + tests/atomic_probe/test_atomic_probe.py::test_a5_ccec_nested_lambda_call_boundary_controls \ + --platform a5 --device 0 \ + -q -s ``` -本机 `matrix_rc=1` 是期望结果。该显式诊断 probe 不在 `ccec/run_all.sh` 默认 -cache-line suite 中。 +该 pytest 会： -### TC-CCEC-AIC-05：方案组稳定性 +1. build 单对象 all-inline target，并验证 submit FUNC 符号为零； +2. 先运行 args-runtime-read，要求 PASS； +3. 再运行 m0，要求 PASS。 -先完成 build-only，再用独立进程重复运行： +它不会运行 only-weak-submit-noinline m0，避免默认 CI 故意制造 AICore +exception。双对象数量控制和 noinline 故障控制使用第 2.4、2.5 节的显式 +命令。 -```bash -for i in $(seq 1 10); do - echo "=== args-runtime-read iteration=$i/10 ===" - ATOMIC_PROBE_MODE=args-runtime-read \ - ATOMIC_PROBE_DEVICE=0 \ - tests/atomic_probe/ccec/run_all.sh nested_lambda_cross_tu run -done -``` - -要求 10/10 进程退出码为 0，且每次 checksum 完全一致。正式稳定性批次为 -10/10；文档复核又成功运行 1 次，因此累计记录为 11/11、704 轮、2816 次 -submit。 - -可选的交替 A/B： +### 6.3 完整语言入口 ```bash -for cycle in 1 2; do - set +e - ATOMIC_PROBE_MODE=weak-context-materialize-0 \ - ATOMIC_PROBE_DEVICE=0 \ - tests/atomic_probe/ccec/run_all.sh nested_lambda_cross_tu run - bad_rc=$? - set -e - - if [ "$bad_rc" -eq 0 ]; then - echo "bad control unexpectedly passed" >&2 - exit 1 - fi - - # 若所在平台要求 reset，请在这里执行平台批准的恢复流程。 - ATOMIC_PROBE_MODE=args-runtime-read \ - ATOMIC_PROBE_DEVICE=0 \ - tests/atomic_probe/ccec/run_all.sh nested_lambda_cross_tu run -done +tests/atomic_probe/run_nested_lambda.sh all ``` -早期两组交替均为“context 507015，随后方案组立即 PASS”。这只证明当时两次 -可以恢复，不覆盖最后一次设备消失的状态。 +`all` 依次运行 CPU、AscendC、纯 CCEC AIV 和默认 AIC caller-capture +正向组。 -## 8. 实测环境和结果记录 +## 7. 本轮实测结果 -### 8.1 软件和设备 +软件环境： ```text -Repo HEAD: 52ca4f5eba343c2f7b7a3a743e575cb9308d128f -Device: Ascend950PR_9599 -Short SoC: Ascend950 -CANN: 9.1.0 -ccec: clang 15.0.5, build 2026-06-10T11:29:46+08:00 +Repo HEAD: 3a3de54db0a900e489a5a5496cbee1dc5b76be7a +CCEC: clang 15.0.5, build 2026-07-07T20:35:46+08:00 GCC: 13.3.0 -AscendC arch: dav-3510 -CCEC AIV arch: dav-c310-vec -CCEC AIC arch: dav-c310-cube +AIC arch: dav-c310-cube +Device: A5 device 0 ``` -### 8.2 A5 矩阵 +持久 fixture 结果： -| 变体 | 次数 | 结果 | rounds | mismatches | dispatcher | checksum | -| --- | ---: | --- | ---: | ---: | ---: | --- | -| `args-runtime-read` | 11 | 11/11 PASS | 64/次 | 0 | 0 | 固定正确 | -| `weak-context-materialize-0` | 5 | 5/5 507015 | 未回读 | 未回读 | 未回读 | 未回读 | -| 其他五个控制组 | 各至少 1 | PASS | 64 | 0 | 128 | 固定正确 | +| fixture / variant | 次数 | 结果 | +| ----------------- | ---: | ---- | +| 单对象 all-inline / m0 | 5 | 5/5 PASS | +| 双对象、空 runtime / m0 | 5 | 5/5 PASS | +| only-weak-submit-noinline / m1 | 3 | 3/3 PASS | +| only-weak-submit-noinline / strong | 3 | 3/3 PASS | +| only-weak-submit-noinline / m0 | 5 | 5/5 507015 | +| 单对象 all-inline / args-runtime-read | 1 | PASS | +| CPU 语言语义 | 1 | PASS | +| CPU inline args-runtime-read | 1 | PASS | -故障组在 `aclrtSynchronizeStream` 就返回，因此不能伪造 rounds/mismatch 等设备 -结果；表中明确写“未回读”。 +此外，本轮从当前 HEAD 临时恢复原始 external runtime 源码，重新生成双 +`.o` 产物；m0 为 5/5 507015。该临时产物只用于确认历史基线，不是当前 +runner 的持久 target。 -最新状态：第 5 次负向命中后，后续 `args-runtime-read` 没有进入 kernel， -而是在 `aclInit(nullptr)` 返回 500000。用户确认当前 NPU 设备已经不存在。 -因此 A5 数据到此冻结，后续仅做离线审查。 +故障组在 `aclrtSynchronizeStream` 返回 507015，不能回读 rounds、 +mismatch 或 checksum。表中不为故障组伪造设备结果。 -### 8.3 最终 build 产物哈希 +## 8. 如何解释结果 -哈希用于确认当前机器上的复现产物，不应假设不同绝对源码路径或不同工具版本 -一定生成同一哈希。 +| 观察 | 可以支持 | 不能推出 | +| ---- | -------- | -------- | +| 双对象空 runtime 通过 | 两个链接输入不足以触发 | `ld.lld` 已被全面证明无缺陷 | +| 单对象 noinline m0 失败 | 多对象链接不是必要条件 | 任意 noinline 调用都会失败 | +| all-inline m0 通过 | inline 可规避当前形态 | inline 修复了 CCEC 后端 | +| noinline m1 通过 | 地址物化会改变最终 codegen | 增加一次 store 是生产修复 | +| 同 fixture strong 通过 | strong 路径是有效控制 | weak 单独就是根因 | +| args-runtime-read 通过 | 数据驱动方案机制可行 | 真实 PA 业务已经完成修复 | -```text -caller AIC object: -2412dd11bf0d8b07d8c9f9fd08179cc4821e95335616f314a2f55b4b710f52b0 +507015 在 CANN 中表示 AICore exception。业务经过 AICPU 外层时可能报告 +507018；错误层级与调用路径不同，不能只凭错误码断言 fault PC 相同。 -runtime AIC object: -ec365301d026dbefec93d562102f7b85b44c72aa8ade2cea4956eb6e3f09470e +## 9. 生产落地约束 -linked raw AICore ELF: -74167ad081799b93daa29a7b974746339cc4a3cd488dd2463514097696fb900b +1. submit 返回前必须复制或物化 caller 栈数据； +2. 不得让异步阶段保存 caller 栈裸地址； +3. all-inline 必须检查最终 ELF 不含 `nested_probe_submit_*` FUNC 符号； +4. inline 会改变代码体积，真实业务必须检查指令空间和性能； +5. recipe slot 必须定义正式布局、容量、版本和边界检查； +6. 真实 PA 必须保留 eager、原 lazy 和候选方案三组 A/B； +7. 无业务地址写只能用于诊断，不能作为正式修复； +8. only-weak-submit-noinline m0 只属于显式诊断，不进入默认 CI。 -host launcher: -未记录。host 入口已收紧为必须显式指定单个变体，修改后未重新执行 build-only。 -``` - -## 9. 如何解释结果 +## 10. 常见问题 -| 观察 | 可以支持的结论 | 不能推出的结论 | -| --- | --- | --- | -| CPU/AscendC/CCEC AIV 全通过 | 前端和单 TU 语言语义支持 | 真实跨调用 ABI 一定正确 | -| context 组 507015、方案组通过 | 数据驱动方案避开当前易错形态 | 已定位某个具体 CCEC pass | -| 地址写数量改变 PASS/FAIL | 最终 codegen 对地址活跃区间敏感 | “加一条/三条 store”是合法修复 | -| strong 通过、weak 失败 | weak 会影响当前最小代码布局 | weak 是业务根因；同 TU 实验已反证 | -| 所有七组都通过 | 当前工具链/布局未命中 probe | 原业务问题不存在 | -| `args-runtime-read` 失败 | 当前方案在该工具链下不可用或测试有回归 | 一定与原业务是同一 fault PC | +### 10.1 build 目录中混入旧产物 -### 9.1 507015 与 507018 +每个 fixture 使用独立 kernel、caller object 和 host 文件名。不要拿 +`nested_lambda_cross_tu_host` 启动 noinline kernel，也不要只看 build +目录中是否残留历史 runtime object。 -CANN 安装头文件 `include/acl/error_codes/rt_error_codes.h` 定义： +以 runner 的构建输出为准： ```text -507015 = ACL_ERROR_RT_AICORE_EXCEPTION -507018 = ACL_ERROR_RT_AICPU_EXCEPTION +aic_input_objects=... +runtime_text=... +submit_symbols=... ``` -本文探针由 ACL 直接启动 AIC kernel，所以原发异常在 host 侧表现为 507015。 -业务经过 AICPU 外层时报告 507018。错误层级符合调用路径差异，但仅凭两个 -错误码不能证明 fault PC 完全相同。 - -### 9.2 CPU 是否也有栈地址物化缺陷 - -CPU 同样需要物化栈地址，但当前 x86-64 GCC 后端在这条双 TU 调用链上没有 -观察到缺陷：多优化级别和 sanitizer 都通过。这支持问题是 CCEC HiIPU 后端 -特有，而不是通用 C++ 语义错误。 - -这不是对所有 CPU 编译器的普遍证明。本机没有 `clang++`，也没有覆盖所有 -代码布局。如果 runtime 在 submit 返回后仍保存并异步解引用 caller 栈地址， -CPU 也会发生 use-after-return；那是生命周期错误，不是地址物化缺陷。 - -## 10. 方案落地约束 - -当前结果证明“机制可行”，不等于真实 PA 业务修复已经完成。生产改造至少要 -满足： +### 10.2 noinline 完整运行退出 1 -1. runtime 必须在 submit 返回前把 recipe/capture 复制或物化到自己的稳定 - 存储，不能把 caller 栈裸地址留给异步阶段； -2. recipe slot 必须定义正式布局、容量、版本和边界检查； -3. 不能让 recipe slot 与正常 `scalar_count`、tensor slot 或后续 ABI 演进冲突； -4. 必须在真实 PA 上保留 eager control、原 lazy bad 和数据驱动 recipe 三组 - A/B，并检查 golden； -5. 必须覆盖真实的 64 轮、同容器四次 submit、多进程稳定性和异常恢复； -6. 无业务地址写只能用于诊断，不能作为正式修复提交； -7. 负向 507015 probe 只留在显式诊断矩阵，默认 CI 只运行正向 - `args-runtime-read`。 - -## 11. 常见问题与排查 - -### 11.1 `ASCEND_HOME_PATH` 或 `PTO_ISA_ROOT` 未设置 - -重新执行： +以下命令默认按 strong、m1、m0 顺序运行： ```bash -source "$CANN_ROOT/set_env.sh" -export PTO_ISA_ROOT="$CANN_ROOT/x86_64-linux" -``` - -不要把 `PTO_ISA_ROOT` 指到不含 -`include/pto/common/kernel_meta.hpp` 的目录。 - -### 11.2 `aclrtBinaryGetFunction` 返回 107000 - -raw AICore ELF 的多入口不要使用带 `_mix_aic` 的完整符号名查找。本文已经按 -CANN runtime 头文件的定义处理：`funcEntry` 是 kernel 名中的数字后缀，七个 -入口使用唯一的 `0..6`，host 调用 `aclrtBinaryGetFunctionByEntry`。 - -当前正确命名示例： - -```text -nested_lambda_cross_tu_ctx_m0_0_mix_aic -nested_lambda_cross_tu_ctx_m1_1_mix_aic -... -nested_lambda_cross_tu_runtime_args_6_mix_aic -``` - -本机官方依据： - -```text -$ASCEND_HOME_PATH/x86_64-linux/pkg_inc/runtime/runtime/rts/rts_kernel.h -funcEntry: the suffix number; kernel_foo_123 -> 123 +tests/atomic_probe/ccec/run_all.sh \ + nested_lambda_only_weak_submit_noinline run ``` -### 11.3 完整矩阵退出码为 1 - -先检查是否只有最后的 `weak-context-materialize-0` 返回 507015。如果前六组 -都是 PASS，那么退出码 1 是预期诊断结果。若 build、入口获取或方案组也失败， -则不是预期结果。 - -### 11.4 CCEC 报 `tensor.h` 的 unused variable warning - -当前构建会从仓库已有 `tensor.h` 报一个 `buffer_elems` unused warning；本次 -编译仍成功。不要把 warning 当作本 probe 的 semantic failure，也不要为了 -本测试批量修改无关生产头文件。 - -### 11.5 故障组后设备不可用 - -停止继续运行，执行所在平台批准的设备 reset/recovery，再先跑 -`strong-context` 或 `args-runtime-read` 控制组。本文不提供未经本机文档验证的 -reset 命令。本文最终一次复核后正处于该状态：`aclInit=500000`，NPU 已不再 -存在/暴露，所以没有继续执行任何硬件命令。 +如果前两组 PASS，最后 m0 返回 507015，则最终退出 1 是预期诊断结果。 -### 11.6 故障组意外通过 +### 10.3 m0 意外通过或失败 先确认： -- 使用的是 `dav-c310-cube`，不是 AIV； -- caller/runtime 确实分开编译； -- 没有复用旧 build； -- CCEC 版本和 flags 与第 6.4 节一致； -- entry 0 启动的是 `weak-context-materialize-0`； -- source tree 包含本文列出的测试版本。 - -重新执行 build-only 后再运行。如果仍通过，应记录为“当前环境未复现”，不能 -人为添加无关代码强迫其失败。 - -## 12. 最终复现检查清单 - -完成下列检查即可认为复现记录完整： - -- [ ] 记录 repo HEAD、CANN/ccec/GCC 版本和设备型号； -- [ ] CPU 嵌套 lambda 基线 PASS； -- [ ] CPU runtime-read 对等实现 PASS； -- [ ] AscendC AIV 基线 PASS； -- [ ] 纯 CCEC AIV 基线 PASS； -- [ ] AIC caller/runtime 双 TU build PASS； -- [ ] 七个 metadata entry 和 weak/strong 符号符合预期； -- [ ] orchestration CFA 为 `reg93 + 1952`； -- [ ] `args-runtime-read` 为 PASS、checksum 精确匹配； -- [ ] `weak-context-materialize-0` 在受影响环境返回 507015； -- [ ] 明确记录完整矩阵退出码 1 是预期负向命中； -- [ ] 异常后按平台规范完成恢复； -- [ ] 没有把地址物化条数或 weak 属性误写成最终根因； -- [ ] 没有把机制验证误写成真实业务修复完成。 +- 使用了正确 fixture 的 kernel； +- all-inline target 没有 submit FUNC 符号； +- noinline target 只保留 weak-context submit； +- CCEC arch 为 `dav-c310-cube`； +- caller orchestration 和七个 metadata entry 仍存在； +- 没有复用另一 fixture 的 host/kernel 组合。 + +若 noinline m0 仍通过，应记录“当前工具链未命中”，不能添加无关代码强迫 +失败。 + +### 10.4 `tensor.h` unused warning + +CCEC 当前会报告 `buffer_elems` 未使用。它不属于本 probe 的 semantic +failure，不应为了该测试修改无关生产头文件。 + +## 11. 最终检查清单 + +- [ ] CPU 嵌套 lambda 语义 PASS； +- [ ] CPU inline args-runtime-read PASS； +- [ ] 默认 AIC target 只有一个输入对象； +- [ ] 默认 ELF 没有 `nested_probe_submit_*` FUNC 符号； +- [ ] 默认 all-inline m0 为 5/5 PASS； +- [ ] 双对象控制的第二个对象 `.text=0`； +- [ ] 双对象控制 m0 为 5/5 PASS； +- [ ] noinline target 只保留 weak-context submit； +- [ ] noinline m1 为 3/3 PASS； +- [ ] 同 fixture strong 为 3/3 PASS； +- [ ] noinline m0 为 5/5 507015； +- [ ] args-runtime-read checksum 精确匹配； +- [ ] `sizeof(L0TaskArgs) == 1024`； +- [ ] 没有把 `.o` 数量、`ld.lld`、weak 或某个 pass 写成已定位根因； +- [ ] 默认 pytest 不运行故障组。 diff --git a/tests/atomic_probe/ATOMIC_USAGE_GUIDE.md b/tests/atomic_probe/ATOMIC_USAGE_GUIDE.md index 87a8334f15..0ef53d87eb 100644 --- a/tests/atomic_probe/ATOMIC_USAGE_GUIDE.md +++ b/tests/atomic_probe/ATOMIC_USAGE_GUIDE.md @@ -27,7 +27,9 @@ load/store 和 atomic 的代码。API 定义来自本机 CANN 头文件，行为 stale dirty line，另一个核更新该目标后，前者再对这条 64B line 执行 DCCI，仍可能用旧快照覆盖 已经成功完成的 atomic 更新。 4. 基于上述实测，本仓采用保守布局规则：atomic 控制字与被 DCCI 的 data 分 cacheline；关键 atomic - 默认一变量独占一条 64B line；其整条 line 按 atomic-only 管理，不混入普通 scalar store/DCCI。 + 默认一变量独占一条 64B line；其整条 line 按 atomic-only 管理，不混入普通 scalar store。 + 当前专项用例中，对仅经 atomic 访问的独占 line 执行 SINGLE OUT DCCI 后新值 `100/100` 保留， + 但该 DCCI 没有业务必要性，默认仍不对 atomic 控制 line 执行 DCCI。 5. 所有跨核共享的可变控制状态和交权动作都必须使用 atomic 或已有明确契约的硬件同步原语，包括 `lock/state/phase/ready/done/remaining/refcount/cursor`。普通 scalar store、`st_dev`、DCCI、DSB 都不能替代 atomic 交权。该规则不要求业务 payload 的每个 word 都执行 atomic：payload 可以在 @@ -183,6 +185,11 @@ HCCL 的 `FlushDataCache` helper 和本仓 fdwic 的 cache-region helper 都采 作为本机实际用法的佐证，不能代替 API/ISA 契约。反过来也要注意：DSB 只等待发指令核的相关 memory access，不提供跨核会合或 coherence，更不能修复一个本来就会覆盖其他核新值的 stale dirty DCCI。 +这不表示每一条 DCCI 指令后都必须机械追加一条 DSB。连续发出多条 DCCI、期间没有动作依赖其完成时， +可以在依赖边界前统一执行一次 DSB。本手册的 TaskCell 专项探针固定使用 +`SINGLE_CACHE_LINE + CACHELINE_OUT + DSB_ALL`，是为了让随后发布的 `phase=2` 精确表示“DCCI 已经 +完成”；若去掉 DSB，后续观察只能说明 DCCI 已发射或正在处理，不能用于判定完成后的目标值。 + ## 3. `st_dev` / bypass load-store ### 3.1 API @@ -310,7 +317,10 @@ line 内的 atomic target 仍会被旧快照覆盖。它没有直接测试“loc - 不能在存在 ENTIRE DCCI 的并发区域中仅依赖地址分-line隔离。 这些规则用于规避已经实测的 stale dirty writeback 风险，不表示当前用例已经证明它们对所有 -atomic 类型和所有硬件时序都是必要且充分条件。clean atomic-only line 上执行 DCCI 也没有专项用例。 +atomic 类型和所有硬件时序都是必要且充分条件。现在已有“仅经 atomic 访问的独占 line”专项用例： +CAS 发布的新值先被另一 AIV 看见，再由发布核执行 SINGLE OUT DCCI + DSB，当前 A5 上 `100/100` +保持新值且整行 guard 完整。该结果排除了这一精确时序下的覆盖，不能外推为其他 selector、atomic +类型、普通访问混入或 ENTIRE DCCI 的通用保证；工程默认仍不对 atomic-only line 执行无必要的 DCCI。 默认建议： @@ -323,6 +333,53 @@ atomic 类型和所有硬件时序都是必要且充分条件。clean atomic-onl `AtomicExch` 的三组路径均为 `0/4000`，这是该特定模式的支持证据，不是所有 atomic 类型和时序的 通用证明。 +### 4.4 `TaskCell::deps_prepared` 五场景 AIV 专项实测 + +2026-07-28 在当前 A5 device 0、CANN 9.1 上执行了 CCEC AIV-only 专项用例。runner 显式关闭 scalar +自动 DCCI 和 kernel-end DCCI；一次 kernel 只启动两个 AIV，五个场景各执行 100 轮。每个 +`(scenario, trial)` 使用一份从未被 device 访问过的 640B 独立存储，其中被测 line、三个握手 +atomic line、角色领取 line、结果和 guard 均按 64B 隔离。每轮由两个 AIV 动态领取 writer/reader +角色。写入侧使用 `ordinary/CAS -> compiler barrier -> DSB -> compiler barrier -> phase=1` 闭合 +发布边界；reader 在 DCCI 后的第一次目标访问固定为 `atomicAdd(address, 0)`，随后才允许整行 +`ld_dev` 快照。当前 shell 未找到 `npu-smi` 和 `task-submit`，本次按用户指定直接使用 device 0， +没有外部资源隔离；本文记录的是逐字段正确性结果，不使用这组运行推导性能。最终版本独立完整运行 +两次，每次每场景 100 轮且汇总完全一致；下表列出单次运行结果，对应合计 200 轮也没有新增失败。 + +| 场景 | writer 路径 | DCCI 前 reader 看见新值 | DCCI 后 reader 看见新值 | 完整快照/host 新值 | 精确判定 | +|---|---|---:|---:|---:|---| +| 0：TaskCell 共线 atomic | ordinary 构造含旧 `deps` 的 dirty TaskCell；CAS 发布新值；再 DCCI TaskCell line | 100/100 | 0/100 | 0/100 | 新值被完整旧快照覆盖 100/100 | +| 1：TaskCell 共线普通写 | ordinary 构造 TaskCell 并直接写新 `deps`；再 DCCI TaskCell line | 0/100 | 100/100 | 100/100 | 100/100 通过 | +| 2：`deps` 独占行 atomic | CAS 发布新值；对仅经 atomic 访问的独占 line 做 DCCI | 100/100 | 100/100 | 100/100 | 100/100 通过 | +| 3：`deps` 独占行普通写 | ordinary 写新值；再 DCCI 独占 line | 0/100 | 100/100 | 100/100 | 100/100 通过 | +| 4：`deps` 独占行普通写且无 DCCI | ordinary 写新值；不做 DCCI | 0/100 | 0/100 | 0/100 | 100/100 单调合法 | + +场景 0 的“覆盖”不是只看最终 word：每轮都要求 CAS 返回旧值、远端确实先看见新值、DCCI 后远端 +第一次读取回到旧值、完整 64B TaskCell 的其余字段保持 writer 构造值，且独立 guard 全部正确。 +这证明问题来自同一条 ordinary dirty 快照后续写回，而不是 atomic 没有成功。 + +场景 1 和 3 说明当前精确时序下普通 scalar 写在 DCCI 前不会被远端 atomic poll 看见、在 +DCCI+DSB 后会被看见。场景 4 的 `0/100` 只是本次“不做 DCCI”对照的观测，不是“普通写永远不可见” +的架构保证；自然 writeback/eviction 可能改变其他运行的观察值，因此判定器允许旧值到新值的单调 +变化，只拒绝第三值、撕裂和新值回退。 + +场景 2 说明把 `deps_prepared` 独立为 atomic-only cacheline 后，本次 SINGLE OUT DCCI 没有冲掉 +atomic 新值。它与场景 0 的差别不是“atomic 写法不同”，而是该独占 line 从未形成 ordinary dirty +旧快照。仍应优先省掉 atomic 控制 line 上无必要的 DCCI，而不是把这 100 轮结果扩大为任意 DCCI +组合都可用。 + +复现命令： + +```bash +PTO_ISA_ROOT="$PWD/build/pto-isa" \ + tests/atomic_probe/ccec/run_all.sh taskcell_atomic_dcci +``` + +用例入口： + +- `ccec/taskcell_atomic_dcci.cpp`：五类设备时序、有限握手和精确取证； +- `ccec/taskcell_atomic_dcci_host.cpp`：100 轮初始化、全字段判定和汇总； +- `ccec/taskcell_atomic_dcci_shared.h`：64B 布局、常量和 host/device 共用结构。 + ## 5. 推荐的 cacheline 所有权协议 ### 5.1 内存布局 @@ -405,6 +462,10 @@ repeated `st_dev` 终值问题，也不能使 `st_dev` 写路径重新可用。 | 两 AIV repeated `st_dev`，各写独占 line | 高频或低频复现终值错误 | 分 line 不能修复；业务写路径仍禁用 | | 两 AIV `AtomicExch`，同/分 line | 当前三路径均 `0/4000` | 可作已测 atomic 对照，仍遵守 atomic-only line | | 多个纯 atomic word 共 line | AtomicExch 特定压力未复现问题 | 可审计后使用；关键变量仍建议独占 line | +| dirty TaskCell 内 `deps` 先 CAS、后对同 line DCCI | 远端先见新值，随后 `100/100` 被旧 dirty 快照覆盖 | 禁止这种同-line混用 | +| 独占 atomic-only `deps` line 先 CAS、后 SINGLE OUT DCCI | 当前 `100/100` 保持新值且 guard 完整 | 排除本精确时序的覆盖；默认仍省掉无必要 DCCI | +| ordinary `deps` 写后 SINGLE OUT DCCI | 共线与独占行均由 DCCI 前 `0/100` 变为后 `100/100` | ordinary payload 必须按所有权协议发布 | +| ordinary `deps` 写且无 DCCI | 当前窗口和最终快照均 `0/100` 可见 | 只作负对照，不外推“永不自然写回” | ## 7. DSB 与跨核同步 @@ -486,6 +547,8 @@ CCEC runner 对全部 probe 显式关闭；AscendC runner 当前只对 `mb8_dcci - repeated `st_dev` 分-line独立压力：`ascendc/st_dev_separate_line_stress.asc`、 `ccec/st_dev_separate_line_stress.cpp` - AtomicExch 同构对照：`ascendc/atomic_exch_same_line.asc`、`ccec/atomic_exch_same_line.cpp` +- TaskCell `deps_prepared` 五场景：`ccec/taskcell_atomic_dcci.cpp`、 + `ccec/taskcell_atomic_dcci_host.cpp` 本机 CANN 定义依据： diff --git a/tests/atomic_probe/a5_fdwic_atomic_swimlane_repo.md b/tests/atomic_probe/a5_fdwic_atomic_swimlane_repo.md new file mode 100644 index 0000000000..90e8b8d762 --- /dev/null +++ b/tests/atomic_probe/a5_fdwic_atomic_swimlane_repo.md @@ -0,0 +1,958 @@ +# A5 FDWIC Paged Attention 安装与复现指南 + +## 1. 目标、边界与已验证结论 + +本文记录在真实 A5 开发板上安装用户态依赖，并复现以下 Case1 的完整过程： + +~~~text +examples/a5/fully_distributed_within_core/paged_attention_unroll/ +test_paged_attention_unroll.py +~~~ + +范围严格限定为： + +- 平台仅为 A5Sim 和 A5； +- runtime 仅为 fully_distributed_within_core； +- Case 仅为 Case1； +- Python 始终使用 $HOME/.venv； +- CANN 优先且固定使用 9.1 weekly 20260708； +- 性能口径是全局第一个 Submit 开始到最后一个 Submit 结束。 + +本文不覆盖其他测试目录、其他 runtime、A2/A3、L3 或整段 device wall time。A5Sim 用于功能和调度流程验证；5.6 ms 基线只从真实 A5 生成的 l2_swimlane_records.json 中读取。直接 atomic 逐调用记录和等待区 poll 精确计数只在 `--enable-l2-swimlane 4` 开启；它们用于定位真实 PA 的 scalar atomic 分布，不能替代 level 1 到 3 的 phase-only 性能基线。 + +### 已验证环境 + +| 项目 | 本次验证值 | +| --- | --- | +| 验证日期 | 2026-07-17（phase 基线）、2026-07-18（level-4 atomic） | +| 芯片 | Ascend950PR_958b | +| 设备 | /dev/davinci0 | +| Driver | 7.0.t9.0.B798，ascendhal 7.35.23 | +| CANN | 9.1.0 weekly 20260708 | +| CCEC | clang 15.0.5 | +| AICPU 交叉编译器 | Do-Compiler 7.3.0 | +| A5 计算核 | 32 CUBE + 64 VECTOR，共 96 条 swimlane | +| AICPU 用户池 | 5，OCCUPY 掩码 0x3e | +| Python | 3.12.3 | +| PyTorch | 2.6.0+cpu | +| pytest | 7.4.4 | +| GCC 15 | 15.0.1，Ubuntu 15-20250404-0ubuntu1 | +| PTO-ISA | ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8 | +| simpler 分支 | real-pa-atomic-swimlane | +| 历史 phase 实测 HEAD | 52ca4f5eba343c2f7b7a3a743e575cb9308d128f | +| schema-v3 迁移基线 HEAD | 5274945b（迁移改动基于此展开） | + +系统的 /etc/os-release 标签为 Ubuntu 20.04.6，但实际 getconf GNU_LIBC_VERSION 输出 glibc 2.39。判断 GCC 15 二进制兼容性时，应以实际 glibc 和 ldd 结果为准，不能只看发行版标签。 + +### 已验证结果 + +| 检查项 | 结果 | +| --- | --- | +| A5Sim Case1 | PASSED，约 71.62 s | +| A5 Case1 正确性 | PASSED | +| A5 level-1 phase swimlane Case1 | PASSED，pytest 约 85.48 s | +| A5 level-4 atomic swimlane Case1 | PASSED，pytest 81.90 s，96 核 schema v3 闭合 | +| 每核 Submit | 1280 个，task id 为 0 到 1279 | +| 全局首个至末个 Submit | 5.642245 ms | +| 排除 task 0 分配后的 kernel Submit | 5.635263 ms | +| 每核 Submit span 中位数 | 5.5725575 ms | +| 历史参考值 | 5.577570 ms，commit dbbf621ac2d1cf162d0807e170c042212d067e51 | + +因此，用户关注的约 5.6 ms 基线已经复现。pytest wall time 和日志中的整段 device wall time 不属于这一性能口径。 + +### 必须包含的源码状态 + +复现使用的仓库版本必须同时包含以下三处源码调整： + +1. Case1、Case2、Case3 不再硬编码 block_dim=36，只保留 aicpu_thread_num=4，由 A5 平台自动解析实际 block 数； +2. 旧 Driver 的 HAL 和 DSMI 都不支持 CPU_TOPO、返回 65534 时，允许经过双重校验的 flat OCCUPY 回退； +3. flat 回退只有在 OCCUPY 的 popcount 与 ACL_DEV_ATTR_AICPU_CORE_NUM 完全相等时才接受，否则保持失败关闭。 + +对应文件为： + +~~~text +examples/a5/fully_distributed_within_core/paged_attention_unroll/ +test_paged_attention_unroll.py +src/a5/platform/onboard/host/aicpu_topology_probe.cpp +src/a5/platform/onboard/host/aicpu_topology_probe.h +~~~ + +在仓库根目录执行以下检查。第一条应无输出，第二条应命中： + +~~~bash +TEST_FILE=examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py + +if rg -n '"block_dim"[[:space:]]*:[[:space:]]*36' "$TEST_FILE"; then + echo "ERROR: 当前 revision 仍硬编码 block_dim=36" + exit 1 +fi + +rg -n 'ACL_DEV_ATTR_AICPU_CORE_NUM|flat OCCUPY fallback' \ + src/a5/platform/onboard/host/aicpu_topology_probe.cpp +~~~ + +## 2. 系统与设备前置检查 + +以下命令都以普通用户执行，不需要 sudo。Driver 和 firmware 是板端系统级前置条件，本文只校验，不覆盖安装或升级。 + +先进入已经下载好的 simpler 仓库： + +~~~bash +cd /path/to/simpler +export REPO_ROOT="$(git rev-parse --show-toplevel)" +cd "$REPO_ROOT" + +git rev-parse HEAD +git status --short +~~~ + +记录 HEAD 和工作区差异。若源码调整尚未提交，迁移环境时必须连同差异一起带走；只有原始 HEAD 不能代表完整复现版本。 + +检查主机、Driver 和设备节点： + +~~~bash +uname -m +getconf GNU_LIBC_VERSION +grep -E '^(Version|ascendhal_version|timestamp)=' \ + /usr/local/Ascend/driver/version.info + +test -c /dev/davinci0 +test -r /dev/davinci0 +test -w /dev/davinci0 +ls -l /dev/davinci0 +~~~ + +本次预期为 x86_64、glibc 2.39、Driver 7.0.t9.0.B798，并且当前用户对 /dev/davinci0 可读写。任何一项失败时先修复系统权限或 Driver，不要用 Python 代码绕过。 + +检查安装过程会使用的基础工具： + +~~~bash +for tool in bash git python3 rg sha256sum tar dpkg-deb; do + command -v "$tool" || { + echo "ERROR: missing tool: $tool" + exit 1 + } +done +~~~ + +当前 A5 EVB 没有 npu-smi，也没有 task-submit。这不等于设备不可用；本次通过 Driver 版本文件、设备节点和实际 ACL 调用完成了验证。如果另一个环境提供设备预约工具，应先按该环境规则独占设备，再执行上板命令。 + +## 3. 安装 CANN 9.1 与用户级 GCC 15 + +### 安装 CANN 9.1 + +只使用以下两个 9.1 安装包，不要混入同目录下的 9.2 包： + +| 安装包 | 字节数 | SHA-256 | +| --- | ---: | --- | +| Ascend-cann-toolkit_9.1.0~weekly.20260708.01_linux-x86_64.run | 1543071133 | 947165d939e83e4e73c14498e19b5ed69dd0de49bd9b5d71e04765bfa0c09313 | +| Ascend-cann-950-ops_9.1.0~weekly.20260708.01_linux-x86_64.run | 2669342311 | 9b5df71c1ca9a855f65027fb37c3fcd352ca607e997277aacff71508e36b8b91 | + +先校验文件： + +~~~bash +TOOLKIT="$HOME/cann/Ascend-cann-toolkit_9.1.0~weekly.20260708.01_linux-x86_64.run" +OPS="$HOME/cann/Ascend-cann-950-ops_9.1.0~weekly.20260708.01_linux-x86_64.run" + +test -f "$TOOLKIT" +test -f "$OPS" + +printf '%s %s\n' \ + 947165d939e83e4e73c14498e19b5ed69dd0de49bd9b5d71e04765bfa0c09313 \ + "$TOOLKIT" | sha256sum -c - +printf '%s %s\n' \ + 9b5df71c1ca9a855f65027fb37c3fcd352ca607e997277aacff71508e36b8b91 \ + "$OPS" | sha256sum -c - +~~~ + +安装包当前没有 executable bit，因此显式交给 bash。先按组织要求完成软件许可确认，再使用 quiet 非交互安装： + +~~~bash +export CANN_INSTALL_ROOT="$HOME/Ascend/cann-9.1.0-weekly-20260708" +mkdir -p "$CANN_INSTALL_ROOT" + +bash "$TOOLKIT" \ + --full \ + --quiet \ + --install-path="$CANN_INSTALL_ROOT" + +bash "$OPS" \ + --full \ + --quiet \ + --install-path="$CANN_INSTALL_ROOT" +~~~ + +本次成功安装未使用 --force。只有安装器明确报告兼容性问题，且已经核实 Driver/CANN 匹配关系时，才考虑该参数。 + +立即验证安装，不依赖 .bashrc： + +~~~bash +test -f "$CANN_INSTALL_ROOT/cann/set_env.sh" +source "$CANN_INSTALL_ROOT/cann/set_env.sh" + +test "$ASCEND_HOME_PATH" = \ + "$CANN_INSTALL_ROOT/cann-9.1.0" +test -x "$ASCEND_HOME_PATH/bin/ccec" +test -x \ + "$ASCEND_HOME_PATH/tools/hcc/bin/aarch64-target-linux-gnu-g++" + +grep -E '^(Version|timestamp)=' \ + "$ASCEND_HOME_PATH/opp/version.info" +"$ASCEND_HOME_PATH/bin/ccec" --version | head +"$ASCEND_HOME_PATH/tools/hcc/bin/aarch64-target-linux-gnu-g++" \ + --version | head -n 1 +~~~ + +预期 OPP Version 为 9.1.0，timestamp 为 20260708_000326093。 + +### 安装用户级 GCC 15 + +A5Sim 的 incore kernel 由仓库中的 Gxx15Toolchain 直接调用 g++-15，所以仅有系统 g++ 不够。普通 host runtime 默认仍可使用系统 gcc/g++；不要为了这一用例全局改写 CC 和 CXX。 + +本次验证使用从另一台已验证环境复制的 Ubuntu Plucky 解包目录： + +~~~text +$HOME/.local/gcc-15/root +~~~ + +这是用户态解包，不是 dpkg -i。精确源码包版本可在 Ubuntu Launchpad 的 gcc-15 15-20250404-0ubuntu1 页面核对： + + + +推荐直接从已验证环境打包并传输完整 root 目录： + +~~~bash +# 在已验证的源环境执行 +cd "$HOME/.local/gcc-15" +tar -czf "$HOME/gcc-15-plucky-20250404-root.tar.gz" root +cd "$HOME" +sha256sum gcc-15-plucky-20250404-root.tar.gz \ + > gcc-15-plucky-20250404-root.tar.gz.sha256 + +# 将归档及其 SHA-256 传到目标环境后执行 +mkdir -p "$HOME/.local/gcc-15" +cp /path/to/gcc-15-plucky-20250404-root.tar.gz "$HOME/" +cp /path/to/gcc-15-plucky-20250404-root.tar.gz.sha256 "$HOME/" +cd "$HOME" +sha256sum -c gcc-15-plucky-20250404-root.tar.gz.sha256 +tar -xzf gcc-15-plucky-20250404-root.tar.gz \ + -C "$HOME/.local/gcc-15" +~~~ + +如果使用原始 deb 重建目录，应准备同一版本的以下包，并逐个用 dpkg-deb -x 解到同一个 GCC15_ROOT： + +~~~text +cpp-15 +cpp-15-x86-64-linux-gnu +g++-15 +g++-15-x86-64-linux-gnu +gcc-15 +gcc-15-base +gcc-15-x86-64-linux-gnu +libasan8 +libatomic1 +libcc1-0 +libgcc-15-dev +libgcc-s1 +libgomp1 +libhwasan0 +libitm1 +liblsan0 +libquadmath0 +libstdc++-15-dev +libstdc++6 +libtsan2 +libubsan1 +~~~ + +~~~bash +export GCC15_ROOT="$HOME/.local/gcc-15/root" +mkdir -p "$GCC15_ROOT" + +for deb in "$HOME/cann/gcc-15-plucky-debs"/*.deb; do + dpkg-deb -f "$deb" Package Version + test "$(dpkg-deb -f "$deb" Version)" = \ + "15-20250404-0ubuntu1" + dpkg-deb -x "$deb" "$GCC15_ROOT" +done +~~~ + +不要把其他 Plucky 系统包或 libc6 一并放入该目录。当前编译器二进制要求 GLIBC_2.38，目标主机实际 glibc 必须满足要求。若 ldd 显示 not found，应先补齐与主机兼容的 libisl、libmpc、libmpfr、libgmp、zlib、libzstd 或 binutils，不要盲目混用另一发行版的 libc。 + +对复制结果做内容检查： + +~~~bash +export GCC15_ROOT="$HOME/.local/gcc-15/root" + +printf '%s %s\n' \ + db5b698ddfbbefa3978b76c0f9dd7504bd82136db461a05320b74743a8933ec9 \ + "$GCC15_ROOT/usr/bin/x86_64-linux-gnu-g++-15" \ + | sha256sum -c - +printf '%s %s\n' \ + 34ffcc0db386d0d654c29464b84c57a8218b650dfd3b801720b778eacdca7a9e \ + "$GCC15_ROOT/usr/libexec/gcc/x86_64-linux-gnu/15/cc1plus" \ + | sha256sum -c - +printf '%s %s\n' \ + 9fb7d85e8aa687d1d8b27d5c189f3d938579a29e75af9d4651db4d33218fb401 \ + "$GCC15_ROOT/usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.34" \ + | sha256sum -c - +~~~ + +### 写入用户 .bashrc + +$HOME/.bashrc 是文件，不是目录。把以下内容追加到文件末尾；这里的 HOME 就是当前普通用户的 home，不是系统 /root： + +~~~bash +# Ascend CANN user installation. +if [ -f "$HOME/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh" ]; then + source "$HOME/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh" +fi + +# User-local GCC 15 (Ubuntu 25.04 Plucky packages). +export GCC15_ROOT="$HOME/.local/gcc-15/root" +if [ -x "$GCC15_ROOT/usr/bin/g++-15" ]; then + export PATH="$GCC15_ROOT/usr/bin:$PATH" + if [ -n "$LD_LIBRARY_PATH" ]; then + export LD_LIBRARY_PATH="$GCC15_ROOT/usr/lib/x86_64-linux-gnu:$GCC15_ROOT/usr/lib/gcc/x86_64-linux-gnu/15:$LD_LIBRARY_PATH" + else + export LD_LIBRARY_PATH="$GCC15_ROOT/usr/lib/x86_64-linux-gnu:$GCC15_ROOT/usr/lib/gcc/x86_64-linux-gnu/15" + fi +fi + +# User Python environment. +if [ -f "$HOME/.venv/bin/activate" ]; then + source "$HOME/.venv/bin/activate" +fi +~~~ + +自动激活 venv 会影响所有新开的交互 shell，这是本次用户要求的行为。CI、cron 或非交互脚本仍应显式 source 对应环境。 + +保存后打开新的交互 shell，或执行 exec bash，再验证： + +~~~bash +command -v ccec +command -v g++-15 +g++-15 --version | head -n 1 +g++-15 -print-prog-name=cc1plus + +ldd "$(g++-15 -print-prog-name=cc1plus)" | \ + grep 'not found' && exit 1 || true + +printf '#include \nint main(){std::cout << "gcc15-ok\\n";}\n' | + g++-15 -x c++ -std=c++23 - -o /tmp/gcc15-smoke +/tmp/gcc15-smoke +~~~ + +预期版本首行为： + +~~~text +g++-15 (Ubuntu 15-20250404-0ubuntu1) 15.0.1 20250404 (experimental) +~~~ + +## 4. Python、PTO-ISA 与精确构建 + +### 创建用户 Python 环境 + +本次使用 $HOME/.venv，而不是仓库内的 .venv。首次创建： + +~~~bash +python3 -m venv --system-site-packages "$HOME/.venv" +source "$HOME/.venv/bin/activate" + +python --version +python -m pip --version +~~~ + +--system-site-packages 与当前板端部署一致，使已安装的 torch 2.6.0+cpu 可见。目标用例直接 import torch 来生成输入和 golden，因此 PyTorch 必需；它不直接 import torch_npu。 + +安装本次用到的 Python 和构建依赖： + +~~~bash +python -m pip install \ + scikit-build-core==1.0.3 \ + nanobind==2.13.0 \ + cmake==4.4.0 \ + cloudpickle==3.1.2 \ + pytest==7.4.4 \ + pytest-xdist==3.8.0 \ + pytest-timeout==2.4.0 \ + ruff==0.14.8 +~~~ + +若系统 site-packages 中没有 torch，再从当前环境认可的 wheel 源安装 torch 2.6.0+cpu；不要未经确认改成最新版本。能访问 PyTorch 官方 CPU wheel 源时可执行： + +~~~bash +python -c 'import torch; print(torch.__version__)' || \ + python -m pip install \ + --index-url https://download.pytorch.org/whl/cpu \ + torch==2.6.0 + +python -c ' +import pytest +import torch +print("python:", __import__("sys").version.split()[0]) +print("pytest:", pytest.__version__) +print("torch:", torch.__version__) +' +~~~ + +### 固定 PTO-ISA + +~~~bash +cd "$REPO_ROOT" +export PTO_ISA_COMMIT=ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8 +export PTO_ISA_ROOT="$REPO_ROOT/build/pto-isa" + +if [ ! -d "$PTO_ISA_ROOT/.git" ]; then + git clone https://github.com/hw-native-sys/pto-isa.git \ + "$PTO_ISA_ROOT" +fi + +git -C "$PTO_ISA_ROOT" fetch origin "$PTO_ISA_COMMIT" +git -C "$PTO_ISA_ROOT" checkout --detach "$PTO_ISA_COMMIT" +test "$(git -C "$PTO_ISA_ROOT" rev-parse HEAD)" = \ + "$PTO_ISA_COMMIT" +~~~ + +### 只构建 Python binding + +直接执行 pip install -e . 会触发顶层 ALL target，并自动枚举当前可构建的所有平台和 runtime。为保持本文边界，先只构建 _task_interface： + +~~~bash +cd "$REPO_ROOT" +source "$HOME/.venv/bin/activate" +source "$HOME/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh" + +# 移除这个 venv 中可能残留的旧 simpler/editable import hook。 +# 只删除 Python 安装记录，不删除当前源码树或 build 产物。 +if python -m pip show simpler >/dev/null 2>&1; then + python -m pip uninstall -y simpler +fi + +cmake -S . -B build/python-bindings \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython_EXECUTABLE="$(command -v python)" \ + -Dnanobind_DIR="$(python -c \ + 'import nanobind; print(nanobind.cmake_dir())')" + +cmake --build build/python-bindings \ + --target _task_interface \ + --parallel "$(nproc)" + +if [ -n "$PYTHONPATH" ]; then + export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/python:$PYTHONPATH" +else + export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/python" +fi +python -c ' +from pathlib import Path +import simpler +import _task_interface +print("simpler:", simpler.__file__) +print("_task_interface:", _task_interface.__file__) +assert Path(_task_interface.__file__).resolve().parent == \ + Path("python").resolve() +' +~~~ + +不要把项目专用 PYTHONPATH 永久写入全局 .bashrc。每次进入本仓工作时设置，或在测试命令所在 shell 中保持以上 export 即可。若不移除旧 editable 安装，它注册的 import hook 可能优先加载 site-packages 中的旧 binding，使刚构建的源码树产物没有真正被测试。 + +### 只构建目标 runtime + +当前 build_runtimes.py 的 --platforms 只能限制平台，不能限制 runtime。使用 RuntimeBuilder 的现有接口，精确构建 A5Sim/A5 的 fully_distributed_within_core 及它们必需的共享 helper： + +~~~bash +cd "$REPO_ROOT" +source "$HOME/.venv/bin/activate" +source "$HOME/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh" +if [ -n "$PYTHONPATH" ]; then + export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/python:$PYTHONPATH" +else + export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/python" +fi + +python - <<'PY' +from simpler_setup.runtime_builder import RuntimeBuilder + +runtime = "fully_distributed_within_core" +for platform in ("a5sim", "a5"): + print(f"building {platform}/{runtime}/private") + binaries = RuntimeBuilder( + platform, + fdwic_tensormap_mode="private", + ).get_binaries(runtime, build=True) + print(binaries) +PY +~~~ + +`private` 是默认和当前生产行为。若只验证 shared artifact family 的 +构建/ABI 隔离，可把参数改成 `shared`；在 shared backend 真正接入之前， +该模式会在 0 次 Submit 前明确退出，不能作为业务用例成功运行。 + +安装期预构建入口也支持显式选择，而且默认固定为 private，不读取 shell +中的 `PTO_FDWIC_TENSORMAP_MODE`： + +~~~bash +python simpler_setup/build_runtimes.py --platforms a5sim a5 \ + --fdwic-tensormap private + +# 需要同时准备两个彼此隔离的 artifact family 时重复该参数。 +python simpler_setup/build_runtimes.py --platforms a5sim a5 \ + --fdwic-tensormap private \ + --fdwic-tensormap shared +~~~ + +验证目标产物： + +~~~bash +test -f \ + build/lib/a5/sim/fully_distributed_within_core/private/libhost_runtime.so +test -f \ + build/lib/a5/sim/fully_distributed_within_core/private/libaicpu_kernel.so +test -f \ + build/lib/a5/sim/fully_distributed_within_core/private/libaicore_kernel.so + +test -f \ + build/lib/a5/onboard/fully_distributed_within_core/private/libhost_runtime.so +test -f \ + build/lib/a5/onboard/fully_distributed_within_core/private/libaicpu_kernel.so +test -f \ + build/lib/a5/onboard/fully_distributed_within_core/private/aicore_kernel.o + +test -f build/lib/a5/dispatcher/libsimpler_aicpu_dispatcher.so +test -f build/lib/libsimpler_log.so +test -f build/lib/libcpu_sim_context.so +~~~ + +编译器职责如下： + +| 目标 | 编译器 | +| --- | --- | +| A5Sim incore kernel | 用户级 g++-15 | +| A5Sim host/runtime helper | 系统 gcc/g++ | +| A5 AICore kernel | CANN 9.1 ccec | +| A5 AICPU 目标 | CANN 9.1 AArch64 交叉编译器 | +| A5 host 目标 | 系统 gcc/g++ | + +因此，CMake cache 中看到系统 g++ 不代表 GCC 15 被绕过；A5Sim incore kernel 是后续由 KernelCompiler 直接调用 g++-15 编译的。 + +## 5. 执行 A5Sim 与真实 A5 + +每个新 shell 先执行统一准备： + +~~~bash +cd "$REPO_ROOT" +source "$HOME/.venv/bin/activate" +source "$HOME/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh" + +if [ -n "$PYTHONPATH" ]; then + export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/python:$PYTHONPATH" +else + export PYTHONPATH="$REPO_ROOT:$REPO_ROOT/python" +fi +export PTO_ISA_ROOT="$REPO_ROOT/build/pto-isa" +export PTO_ISA_COMMIT=ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8 + +TEST_FILE=examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py + +test "$(command -v python)" = "$HOME/.venv/bin/python" +test "$(git -C "$PTO_ISA_ROOT" rev-parse HEAD)" = \ + "$PTO_ISA_COMMIT" +~~~ + +### 运行 A5Sim Case1 + +~~~bash +python -m pytest "$TEST_FILE" \ + --platform a5sim \ + --case Case1 \ + --enable-l2-swimlane 4 \ + --use-example-exec-time \ + --clone-protocol https \ + --pto-isa-commit "$PTO_ISA_COMMIT" \ + --pto-session-timeout 1200 \ + --require-pto-isa \ + -s -v +~~~ + +--use-example-exec-time 仅适用于 fully_distributed_within_core 的 sim。它不能用于真实 A5 命令。这里显式写出的 level 4 与裸参数 `--enable-l2-swimlane` 等价；A5Sim 用于检查 schema v3、记录结构、加权计数闭合和转换结果，不提供真实 A5 atomic 完成时间。A5Sim 的直接 Atomic 只标记模拟执行的源码包围边界，PollBatch 只标记模拟调度中的 poll 窗口；两者都不能当作真实硬件时延。 + +### 运行 A5 正确性 smoke + +确认没有其他进程占用 device 0 后执行： + +~~~bash +test -r /dev/davinci0 +test -w /dev/davinci0 + +python -m pytest "$TEST_FILE" \ + --platform a5 \ + --device 0 \ + --case Case1 \ + --clone-protocol https \ + --pto-isa-commit "$PTO_ISA_COMMIT" \ + --pto-session-timeout 1200 \ + --require-pto-isa \ + -s -v +~~~ + +### 运行 A5 phase 泳道性能复现 + +~~~bash +python -m pytest "$TEST_FILE" \ + --platform a5 \ + --device 0 \ + --case Case1 \ + --enable-l2-swimlane 1 \ + --clone-protocol https \ + --pto-isa-commit "$PTO_ISA_COMMIT" \ + --pto-session-timeout 1200 \ + --require-pto-isa \ + -s -v +~~~ + +level 1 到 3 保留已有 FDWIC phase 记录，不增加 Atomic 和 `ClockBaseline`。上面的 level 1 与历史 trace 的有效 level 一致，适合继续复现本文约 5.6 ms 的 Submit 性能口径。它们继续导出 schema 1 的 legacy Claim flags；level 4 导出 schema v3，其中 Claim 使用 `attempted/won` 三态，Atomic 同时支持逐调用直接记录和精确计数 PollBatch。源码 atomic 调用点现在统一经过 level 判断，因此不能声称新旧二进制指令级完全相同；做前后性能比较时，应使用同一版二进制分别采 level 1 phase 基线和 level 4 诊断样本。 + +### 运行 A5 level-4 atomic 泳道 + +需要观察真实 PA 的直接 atomic 调用并统计等待区 poll 调用时，单独运行 level 4： + +~~~bash +python -m pytest "$TEST_FILE" \ + --platform a5 \ + --device 0 \ + --case Case1 \ + --enable-l2-swimlane 4 \ + --clone-protocol https \ + --pto-isa-commit "$PTO_ISA_COMMIT" \ + --pto-session-timeout 1200 \ + --require-pto-isa \ + -s -v +~~~ + +level 4 在原有 phase 上增加真实 FDWIC PA 的 Atomic 记录：直接 atomic 保持一次源码调用一条记录；显式等待区内允许合并的 observation load，以及唯一一种已确认的幂等失败 exchange 重试，则用带精确调用次数的 PollBatch 表示。最终 drain 之后，每个 AIC/AIV 还会写两条 `ClockBaseline`。该模式会增加记录写、改变代码布局，也可能改变多核到达和轮询次数，所以它是诊断样本，不是本文 5.6 ms 无 atomic 插桩基线。 + +### raw、merged 产物与重新转换 + +A5Sim 和 A5 的 Case1 都使用 SceneTest 的同一输出规则： + +~~~text +outputs/TestPagedAttentionUnroll_Case1_/ +├── l2_swimlane_records.json +├── merged_swimlane.json +└── swimlane_exclusive_analysis.json +~~~ + +其中 `l2_swimlane_records.json` 是保留原始 cycle 的 raw 文件；pytest 在 case 结束时调用仓内共享转换器，生成可直接载入 Perfetto 的 `merged_swimlane.json`。当前真实 A5 FDWIC level-4 还必须生成严格父子/Kernel/整数闭合报告 `swimlane_exclusive_analysis.json`；raw 缺失、转换失败或任一加工件缺失/为空都会使成功的 pytest 用例失败。若设备执行本身已经失败，则仍保留设备侧原始异常，不用离线转换错误覆盖根因。历史 phase 基线 raw 与本次真实 A5 level-4 raw 分别为： + +~~~text +outputs/TestPagedAttentionUnroll_Case1_20260717_023809/l2_swimlane_records.json +outputs/TestPagedAttentionUnroll_Case1_20260718_161520/l2_swimlane_records.json +~~~ + +新的复现会生成不同时间戳目录。若需要重新转换已有 raw，执行： + +~~~bash +python -m simpler_setup.tools.swimlane_converter \ + outputs/TestPagedAttentionUnroll_Case1_YYYYMMDD_HHMMSS/l2_swimlane_records.json \ + -v +~~~ + +未指定 `-o` 时，转换器仍写到 raw 同目录的 `merged_swimlane.json`。上面的历史 phase-only raw 约几十 MiB；level 4 的 JSON 大小取决于直接 Atomic、PollBatch 和 phase 的实际记录条数，不再随每一次连续 poll 线性增长。pytest 结束后再读取，不要用 pytest wall time 代替 Submit 指标。 + +### level-4 atomic 数据契约 + +当前生产端导出的 level 4 raw，其 `metadata.trace_schema_version` 必须为 4；共享 converter 仍保留对历史 schema-v3 的读取兼容，但只有 schema-v4 生成当前排他闭合报告。转换后的 `Atomic` 和 `ClockBaseline` 都画在对应 AIC、AIV0 或 AIV1 的原 scalar lane；它们不是与 scalar 并行的伪子轨。Kernel 仍画在独立的 `AIC/AIV·kernel` 轨。 + +直接 Atomic 的名称明确给出终点语义： + +~~~text +atomic.return_ready..# +atomic.source_issue..# +~~~ + +真实 A5 上，返回值会被后续逻辑消费的直接调用使用 `return_ready`，表示返回旧值已可被本核 scalar 使用；返回值被丢弃的发布型直接调用使用 `source_issue`，只表示源码发射包围区间。两者都不表示跨核可见时刻，也不能直接称为 atomic retire 延迟。每条直接 Atomic 的 `args.call_count` 固定为 1。结束 cycle 在 atomic 返回后立即采样，本地调用计数更新、PollBatch 落盘和 direct 记录写入都发生在结束 cycle 之后，不能混入该条 direct span。 + +以下九类 observation load 只有在对应的显式 scheduler 等待区内才允许聚合： + +- 通用等待：`startup_poll`、`fatal_poll`、`fanin_flag_load`、`heap_frontier_load`、`heap_vend_load`、`replay_done_poll`； +- BlockWon 等待：`won_any_load`、`won_state_load`、`won_drained_load`。 + +BlockWon 三类 load 只在 slot-capacity、heap slow path、won-slot、sim producer-ready、final-drain 这些已有外层等待区中聚合；普通 Submit 中的一次性或 opportunistic BlockWon 扫描仍是直接 Atomic。 + +唯一允许聚合的 RMW 是 `won_lane_claim_exchange`，并且必须同时满足：位于上述显式等待区、写入 `claimed(1)`、返回旧值也是 `claimed(1)`。这表示一次 1→1、没有改变协议状态的失败 claim 重试。返回 `free(0)` 的成功 claim 始终逐条记录；`won_slot_claim_max`、release、`fetch_sub`、state clear 以及其他 RMW 也全部逐条记录。 + +每个聚合记录转换为： + +~~~text +atomic.poll_batch..× +~~~ + +其 `args.call_count` 是该等待区内实际执行的源码 atomic wrapper 调用次数，不是采样值；`task_id=-1`、`func_id=-1` 表示它归属于 scheduler 等待区而非某个任务。一个等待区可同时累积多个 site，因此不同 site 的 PollBatch 时间窗可以重叠；等待区内的直接 Atomic 也可能与该窗口交错。merged 用 `batch_semantics=observation_load_calls` 或 `idempotent_failed_exchange_retries` 区分两种计数，并显式写出 `may_contain_interleaved_direct_atomics=true`。 + +PollBatch 的 `duration`/`poll_window_cycles` 只是 logical poll episode 的包络：它既不是纯 poll 时间或独占 scalar 时间，也不是其中任一次 atomic 的延迟，更不是 `call_count` 次 atomic 串行延迟之和。raw 中的物理相邻顺序也不等于严格时间顺序；分析应以 cycle 字段为准，不能用 PollBatch duration 计算单次 atomic 的 median 或 p95。 + +schema v3 用 Atomic flags 的 bit 7 标识 PollBatch，bits 8..31 保存无符号 24 bit `call_count`；低 4 bit 必须是该 site 的实际 op：九类 observation 是 `load(0)`，`won_lane_claim_exchange` 是 `exchange(1)`。bit 4 表示返回值被消费，bit 5/6 在 PollBatch 中必须为 0。bit 7 为 0 的直接 Atomic 保留原有 flags 语义，不能把其高位按 poll 次数解析。 + +site 0 到 14 与 standalone PA 的稳定编号完全一致；真实 FDWIC PA 只在末尾追加 15 到 27，覆盖生产实现的 BlockWon 路径和 `FetchSub`，没有重排已有编号： + +| `site_id` | Perfetto `site` | `op` | 真实 PA 路径 | +| --------: | --------------- | ---- | ------------ | +| 0 | `startup_increment` | `fetch_add` | 启动屏障到达计数 | +| 1 | `startup_poll` | `load` | 启动屏障轮询 | +| 2 | `fatal_poll` | `load` | fatal 状态检查 | +| 3 | `fatal_set` | `exchange` | fatal 状态发布 | +| 4 | `claim_max` | `fetch_max` | Submit lane Claim | +| 5 | `fanin_flag_load` | `load` | fanin 依赖 flag | +| 6 | `completion_vend_exchange` | `exchange` | completion vend 发布 | +| 7 | `completion_flag_exchange` | `exchange` | completion flag 发布 | +| 8 | `frontier_initial_load` | `load` | completion frontier 首次读取 | +| 9 | `frontier_flag_load` | `load` | frontier 扫描 flag | +| 10 | `frontier_max` | `fetch_max` | frontier 推进 | +| 11 | `heap_frontier_load` | `load` | HeapGuard frontier | +| 12 | `heap_vend_load` | `load` | HeapGuard vend | +| 13 | `replay_done_increment` | `fetch_add` | final 树的 leaf/root arrival 和 release 发布 | +| 14 | `replay_done_poll` | `load` | 最终 drain 轮询 leaf/root arrival 和 release | +| 15 | `won_slot_claim_max` | `fetch_max` | BlockWon slot 认领 | +| 16 | `won_remaining_exchange` | `exchange` | BlockWon remaining 初始化 | +| 17 | `won_lane_reset_exchange` | `exchange` | BlockWon lane 重置 | +| 18 | `won_lane_deposit_exchange` | `exchange` | BlockWon lane 完成发布 | +| 19 | `won_state_publish_exchange` | `exchange` | BlockWon state 发布 | +| 20 | `won_any_publish_exchange` | `exchange` | BlockWon any 发布 | +| 21 | `won_any_load` | `load` | BlockWon any 读取 | +| 22 | `won_state_load` | `load` | BlockWon state 读取 | +| 23 | `won_lane_claim_exchange` | `exchange` | BlockWon lane claim | +| 24 | `won_lane_release_exchange` | `exchange` | BlockWon lane release | +| 25 | `won_remaining_fetch_sub` | `fetch_sub` | BlockWon remaining 递减并判断最后一个 lane | +| 26 | `won_state_clear_exchange` | `exchange` | BlockWon state 清理 | +| 27 | `won_drained_load` | `load` | BlockWon drained 检查 | + +上表定义的是本文覆盖的真实 FDWIC PA / A5 hot path 可记录调用点集合，不包含 CPU sim watchdog/debug 诊断原子，也不表示 Case1 每轮一定出现全部 28 类事件。Case1 的单 lane 图通常不进入 BlockWon 动态路径；某个 BlockWon site 计数为零不能单独判定为漏插桩。`fetch_sub` 的 op id 为 4，不能按 standalone 旧版只有 Load、Exchange、FetchAdd、FetchMax 四类 op 的假设解析。 + +frontier 扫描沿用 standalone 的任务归因：读取全局扫描起点的 `frontier_initial_load` 记为 `task_id=-1`；随后读取 `next` 完成 flag 的 `frontier_flag_load` 与推进同一个 `next` 的 `frontier_max` 都记为 `task_id=next`。因此两条扫描事件可以按 core、task_id 配对，不归到触发本轮 completion 的另一个任务上。 + +raw 的 `metadata.trace_schema_version=3` 中，Claim 的 flags 明确记录 `attempted` 和 `won`，merged 中对应三种互斥状态： + +| Claim 状态 | `attempted` | `won` | Perfetto 名称 | Case1 预期数量（`N=metadata.num_cores`） | +| ---------- | ----------: | ----: | ------------- | -------------: | +| 未参与该 lane 的 Claim | 0 | 0 | `claim.not_attempted` | `N*512` | +| 已尝试但失败 | 1 | 0 | `claim.lost` | `N*768-1280` | +| 已尝试且获胜 | 1 | 1 | `claim.won` | `1280` | + +三者合计 `N*1280` 条 Claim，其中实际执行 `claim_max.fetch_max` 的数量为 `N*768`。历史 96 核样本对应 49,152 / 72,448 / 1,280；108 核 A5Sim 对应 55,296 / 81,664 / 1,280。`attempted=0, won=1` 是非法组合，不应出现在有效导出中。 + +host 在发布 raw 前按核执行闭合校验： + +- `count` 不得超过该核分区容量，禁止截断后继续导出； +- `dropped` 必须为 0； +- 每条直接 Atomic 计为一次调用；每条 PollBatch 按 flags 高 24 bit 编码的 `call_count` 加权，且 `call_count` 必须大于 0； +- `atomic_calls = atomic_records - poll_batch_records + batched_poll_calls`； +- 所有 PollBatch 的 `call_count` 之和必须等于 `batched_poll_calls`，物理 PollBatch 条数必须等于 `poll_batch_records`； +- level 4 每核必须恰有两条 `ClockBaseline`，总数必须为 `2*N`； +- level 1 到 3 不应出现 `Atomic` 或 `ClockBaseline`。 + +level 1 到 4 都使用每核 65,536 条（64K）分区。设备二进制 record version 3 不再逐条重复 `core_idx/block_id/lane`，而是在每核 state 中保存并校验一次，导出十列 raw JSON 时再补回；每条物理记录为 32 byte，JSON 列格式不变。仅计 record 分区时，96 个 worker 约占 192 MiB，108 个 worker 约占 216 MiB，另有很小的 header；raw metadata 中的 `records_per_core`、`record_size_bytes` 和 `device_trace_bytes` 给出本次运行的精确配置。host 只初始化 header，导出时先读取 header/core 计数，再按核搬运实际 `count*32` byte 的有效记录，不常驻完整设备镜像。 + +PollBatch 让大量连续轮询按等待区和 site 合并，同时保留精确调用次数，因此不再需要为每一次 poll 预留物理记录。若单个 batch 达到高 24 bit 可表示的最大次数，实现会先落盘并开启下一条 batch，不会饱和后丢失计数。物理记录容量仍不是理论无界；任何容量溢出都必须明确失败，不能截断后发布，也不能只放宽 `dropped` 校验。设备侧边界 GTest 直接从 `0xFFFFFE` 累加到 `0xFFFFFF`，确认第一条立即落盘；随后第 `0x1000000` 次调用以 `call_count=1` 重开第二条，并验证两条之和精确等于 `0x1000000`。 + +`batched_poll_calls` 包含上述九类 observation load 和幂等失败 exchange 重试，是本次启用 schema v3 插桩后真实执行的精确调用数。插桩本身会改变代码布局、核间到达时序和轮询节奏，因此不同插桩方案下的调用次数不能当作固定 workload 常量直接比较；计数换算和前后对比应使用同一观察模式。 + +2026-07-18 最终 A5Sim 结构验证得到以下闭合结果；它们用于证明记录规模和计数契约，不表示真实 A5 atomic 时延，也不参与后文按 160 ns 对真机计数所做的归因估算。ReuseStress 只是十类规则与 BlockWon 路径的结构压力验收，不扩大本文只复现 PA Case1 性能的范围： + +| 样本 | raw 大小 | 总记录 | 逻辑 `atomic_calls` | 物理 Atomic | PollBatch | 单核记录峰值 | dropped | +| ---- | -------: | -----: | ---------------------: | ----------: | --------: | ------------: | ------: | +| Case1 `20260718_152435` | 124,547,744 B（约 118.8 MiB） | 1,464,594 | 187,860,395 | 493,301 | 959 | 16,357 | 0 | +| BlockWon ReuseStress `20260718_154229` | 2,055,624 B（约 1.96 MiB） | 24,442 | 243,357,709 | 10,505 | 572 | 519 | 0 | + +Case1 中 `batched_poll_calls=187,368,053`，满足 `187,860,395 - 187,368,053 + 959 = 493,301`；108 核共有 216 条 `ClockBaseline`。ReuseStress 中 `batched_poll_calls=243,347,776`，满足 `243,357,709 - 243,347,776 + 572 = 10,505`；其中 22 条 site 23 PollBatch 精确表示 625,394 次幂等失败 exchange 重试，另有 348 条 site 23 直接记录。这里的直接记录包含成功 claim、等待区外调用或其他非聚合情形，不能全部等同为成功次数。 + +同一版 level-4 代码随后在真实 A5 device 0 上执行 Case1，得到以下实际 PA 记录： + +| 样本 | raw 大小 | merged 大小 | 核拓扑 | 总记录 | 逻辑 `atomic_calls` | 物理 Atomic | PollBatch | 单核记录峰值 | dropped | +| ---- | -------: | ----------: | ------ | -----: | ---------------------: | ----------: | --------: | ------------: | ------: | +| A5 Case1 `20260718_161520` | 77,128,944 B（约 73.6 MiB） | 333,581,552 B（约 318.1 MiB） | 32 AIC + 64 AIV | 973,430 | 115,200 | 110,006 | 340 | 10,751 | 0 | + +该真机样本中 `batched_poll_calls=5,534`，满足 `115,200 - 5,534 + 340 = 110,006`；96 核共有 192 条 `ClockBaseline`。340 条 PollBatch 分布在 `StartupPoll`、`FatalPoll`、`FaninFlagLoad` 和 `ReplayDonePoll`，精确表示 5,534 次等待区调用。Case1 没有动态进入 BlockWon，因此没有出现 21/22/23/27 类 batch；这不影响十类 allowlist 的实现和独立 BlockWon ReuseStress 覆盖。直接事件中 106,914 条使用真机 `return_ready` 边界，与 `result_used` 数量一致，可与 A5Sim 的 `source_issue` 边界明确区分。 + +若只为形成直观的 scalar 归因依据，暂统一使用 160 ns/次，则这份真实 A5 Case1 的全核累计估计为 `115,200 × 160 ns = 18.432 ms core-work`。它是跨 96 核求和后的工作量，不是 PA 墙钟耗时；不能把 18.432 ms 与约 5.6 ms Submit 包络直接相加。 + +任一条件不满足，export 返回非零；若 runtime 本身成功，该错误继续传播为本次运行失败，不能把旧文件或不完整文件当作有效样本。共享 converter 会按 raw 行重新计算 `records`、`atomic_records`、`clock_baseline_records`、`atomic_calls`、`batched_poll_calls` 和 `poll_batch_records`，逐项核对 `metadata.fdwic_summary`；`dropped_records` 无法从已导出的有效行反推，因此 converter 要求 producer summary 明确给出 0。正式分析还应确认 raw 与 merged 的物理 Atomic 条数相等，并以 `atomic_calls` 而不是物理 Atomic 条数表示源码调用总数。 + +两条 `ClockBaseline` 分别是连续两次 `SYS_CNT` 读取，以及 atomic 返回值依赖 hook 的固定路径。它们用于观察计时分辨率和 hook 本身的分布，不是可以从每条 Atomic 机械相减的校正常数。 + +### atomic 性能解释边界 + +单条直接 Atomic 是某一 AIC/AIV scalar lane 上的本地 span。可以按 `core_type/site/op` 查看直接事件数、中位数、p95、最大值，也可以比较同一核上某个直接 site 的累计分布；这些数据适合回答“哪类直接 atomic 常见、哪类本核返回等待长”。PollBatch 只适合统计对应 site/op 的调用次数和等待 episode 分布；其窗口可能包含交错的直接 atomic，duration 不能混入单次 atomic 延迟的 median/p95。 + +若已有独立 atomic probe 给出的标定开销，可把 schema v3 的精确调用数换算为直观的 scalar 工作量估计： + +~~~text +estimated_atomic_core_work_ns = Σ(event.call_count × calibrated_atomic_cost_ns(site, op)) +~~~ + +直接 Atomic 的 `call_count=1`，PollBatch 使用其精确计数。若暂时对所有 site/op 统一采用约 160 ns 的单次标定值，公式简化为 `estimated_atomic_core_work_ns ≈ atomic_calls × 160 ns`；若只归因某个 scalar 阶段，则只对属于该阶段的事件求和。该数值是所有核累计的 scalar core-work 估计，不是 Submit wall time，也不是从 PollBatch duration 反推的单次硬件时延。 + +不能把所有核的 Atomic span 简单求和后称为 Submit 墙钟开销：不同核并行执行，大量 span 相互重叠；Atomic 还嵌套在 Claim、Replay、Submit 等外层 phase 中，外层和内层也不能再次相加。全核求和只能解释为带观察的 aggregate core-work。若要判断对墙钟时间的影响，应结合关键 scalar lane、全局最早 Submit 到最晚 Submit 的包络，并对优化前后使用相同观察模式；最终端到端收益仍用关闭 atomic 诊断的独立运行确认。 + +### 提取首个到末个 Submit + +先把 `TRACE` 指向 level 1 到 3 的 phase-only Case1 raw。以下脚本拒绝 level 4，避免误把 Atomic 插桩后的诊断时间当成约 5.6 ms 基线；随后校验 96 个 core、每核 1280 个 Submit 以及完整 task id，并输出用户关注的全局 span： + +~~~bash +export TRACE=outputs/TestPagedAttentionUnroll_Case1_YYYYMMDD_HHMMSS/l2_swimlane_records.json + +python - <<'PY' +import json +import os +import statistics +from collections import defaultdict +from pathlib import Path + +trace = Path(os.environ["TRACE"]) +if not trace.is_file(): + raise SystemExit(f"trace does not exist: {trace}") +with trace.open() as stream: + data = json.load(stream) + +level = int(data["l2_swimlane_level"]) +assert level in (1, 2, 3), f"phase baseline requires level 1..3, got {level}" +hz = int(data["metadata"]["clock_freq_hz"]) +submits = [row for row in data["fdwic_events"] if row[5] == "Submit"] +if not submits: + raise SystemExit("trace contains no Submit events") + +by_core = defaultdict(list) +for row in submits: + by_core[int(row[0])].append(row) + +assert int(data["metadata"]["num_cores"]) == 96 +assert len(by_core) == 96 +for core, rows in by_core.items(): + task_ids = sorted(int(row[3]) for row in rows) + assert len(rows) == 1280, (core, len(rows)) + assert task_ids == list(range(1280)), core + +first_cycle = min(int(row[6]) for row in submits) +last_cycle = max(int(row[7]) for row in submits) +first_to_last_ms = (last_cycle - first_cycle) * 1000 / hz + +kernel_first_cycle = min( + int(row[6]) for row in submits if int(row[3]) == 1 +) +kernel_to_last_ms = (last_cycle - kernel_first_cycle) * 1000 / hz + +per_core_ms = [] +for rows in by_core.values(): + start = min(int(row[6]) for row in rows) + end = max(int(row[7]) for row in rows) + per_core_ms.append((end - start) * 1000 / hz) + +first_row = min(submits, key=lambda row: int(row[6])) +last_row = max(submits, key=lambda row: int(row[7])) +assert int(first_row[3]) == 0 +assert int(last_row[3]) == 1279 + +print("trace:", trace) +print("clock_freq_hz:", hz) +print("cores:", len(by_core)) +print("submits_per_core:", len(next(iter(by_core.values())))) +print(f"first_to_last_submit_ms: {first_to_last_ms:.6f}") +print(f"task1_to_last_submit_ms: {kernel_to_last_ms:.6f}") +print(f"per_core_median_ms: {statistics.median(per_core_ms):.7f}") +print(f"per_core_max_ms: {max(per_core_ms):.6f}") +PY +~~~ + +本次预期输出的关键值： + +~~~text +clock_freq_hz: 1000000000 +cores: 96 +submits_per_core: 1280 +first_to_last_submit_ms: 5.642245 +task1_to_last_submit_ms: 5.635263 +per_core_median_ms: 5.5725575 +per_core_max_ms: 5.641331 +~~~ + +不同运行允许有小幅抖动。验收重点是正确性 PASSED、事件完整，并且 first_to_last_submit_ms 仍位于约 5.6 ms 的基线附近。 + +## 6. 验收清单与故障定位 + +### 最终验收 + +- CANN 安装包 SHA-256 与本文一致； +- ASCEND_HOME_PATH 指向用户目录下的 CANN 9.1； +- command -v python 为 $HOME/.venv/bin/python； +- command -v g++-15 指向 $HOME/.local/gcc-15/root； +- PTO-ISA HEAD 为固定 commit； +- 源码中没有 block_dim=36； +- 只构建 A5Sim/A5 的 fully_distributed_within_core； +- A5Sim Case1 PASSED； +- A5 Case1 PASSED； +- trace 为 96 core，每核 1280 个 Submit； +- 全局首末 Submit 约为 5.6 ms。 + +### 常见问题 + +**找不到 g++-15** + +确认 GCC15_ROOT、PATH 和 LD_LIBRARY_PATH 已生效，并重新打开交互 shell。A5Sim incore kernel 必须能直接执行 g++-15。 + +**找不到 ccec 或 AArch64 交叉编译器** + +重新 source 用户 CANN 9.1 的 cann/set_env.sh，并检查 ASCEND_HOME_PATH。不要回退到同目录的 CANN 9.2。 + +**提示 pre-built runtime binaries not found** + +重新执行“只构建目标 runtime”中的 RuntimeBuilder 片段。不要改用会自动枚举所有 runtime 的顶层构建。 + +**CPU_TOPO 的 HAL/DSMI 返回 65534** + +这是当前旧 Driver 的已知能力差异。只有日志同时表明 OCCUPY popcount 与 ACL AICPU count 一致，并出现 using flat OCCUPY fallback 时才可继续。本次预期是 mask=0x3e、count=5。若出现 flat fallback rejected，停止运行，不要删除校验或强行构造 CPU 列表。 + +**仍然使用 block_dim=36** + +说明源码 revision 不完整。切换到同时包含本文三处源码调整的 revision，再增量重建 A5 目标 runtime。 + +**PTO-ISA clone 超时或 commit 不一致** + +先在 build/pto-isa 中独立完成 fetch 和 detached checkout，再运行 pytest。--require-pto-isa 会让错误尽早暴露，不能去掉 pin 后继续跑未知版本。 + +**import torch 失败** + +确认 venv 是用 --system-site-packages 创建，或从环境认可的 wheel 源安装 torch 2.6.0。该用例需要 torch，但不因这一点要求直接调用 torch_npu。 + +**出现 torch_npu library owner permission mismatch warning** + +当前系统 site-packages 可能在 import 阶段报告某个 torch_npu 共享库 owner 不匹配。目标用例不直接使用 torch_npu；若 torch、simpler 均可导入且测试 PASSED，该 warning 不影响本次结论。不要以普通用户修改系统共享库的 owner；若它升级为 import error，再交由系统环境维护者处理。 + +**/dev/davinci0 无权限或设备忙** + +由系统管理员修复用户组/ACL，或等待当前任务释放设备。不要 sudo 运行 pytest，否则会绕开用户 venv、HOME 和 CANN 安装路径。 + +**结果显示 70 到 80 ms** + +这通常是整段 device wall time，不是本文指标。必须读取 l2_swimlane_records.json 的 fdwic_events，并按本文章节计算 Submit span。 + +### 建议保存的复现证据 + +每次正式复现至少保留： + +~~~bash +git rev-parse HEAD +git status --short +git diff -- \ + examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py \ + src/a5/platform/onboard/host/aicpu_topology_probe.cpp \ + src/a5/platform/onboard/host/aicpu_topology_probe.h + +python --version +python -c 'import torch; print(torch.__version__)' +g++-15 --version | head -n 1 +grep -E '^(Version|timestamp)=' \ + "$ASCEND_HOME_PATH/opp/version.info" +git -C "$PTO_ISA_ROOT" rev-parse HEAD +~~~ + +同时归档 pytest 完整日志和对应的 l2_swimlane_records.json。这样可以区分代码变化、工具链变化、设备占用和真实性能回归。 diff --git a/tests/atomic_probe/ascendc/cache_preload_probe.asc b/tests/atomic_probe/ascendc/cache_preload_probe.asc new file mode 100644 index 0000000000..8c8f38e928 --- /dev/null +++ b/tests/atomic_probe/ascendc/cache_preload_probe.asc @@ -0,0 +1,687 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +// AscendC counterpart of ccec/cache_preload.cpp. Both probes share exactly +// the same mode/ABI/checksum definitions, cold-state construction, independent +// gap and raw SYS_CNT measurement. This file intentionally uses the public +// AscendC cache APIs: +// +// AscendC::DataCachePreload(GlobalTensor, byte_offset) +// AscendC::ICachePreLoad(units_from_current_pc) +// AscendC::GetICachePreloadStatus() +// +// Preload remains only a performance hint. It is not DCCI, memory ordering, +// cross-core publication, or synchronization. + +#include "../cache_preload_shared.h" +#include "../probe_host.h" +#include "kernel_operator.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace AscendC; + +#ifndef SINGLE_CACHE_LINE +#define SINGLE_CACHE_LINE 0 +#endif + +namespace { + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadOrderedCycle() { + uint64_t cycle = 0; + asm volatile("MOV %0, SYS_CNT\n" : "=&l"(cycle) : : "memory"); + return cycle; +} + +__aicore__ __attribute__((always_inline)) inline uint64_t CycleBeforeValue(uint64_t &value) { + uint64_t cycle = 0; + asm volatile("MOV %1, SYS_CNT\n" + "MOV %0, %0\n" + : "+l"(value), "=&l"(cycle) + : + : "memory"); + return cycle; +} + +__aicore__ __attribute__((always_inline)) inline uint64_t CycleAfterValue(uint64_t &value) { + uint64_t cycle = 0; + asm volatile("MOV %0, %0\n" + "MOV %1, SYS_CNT\n" + : "+l"(value), "=&l"(cycle) + : + : "memory"); + return cycle; +} + +__aicore__ __attribute__((always_inline)) inline uint64_t CycleBeforeStore(uint64_t &address, uint64_t &value) { + uint64_t cycle = 0; + asm volatile("MOV %2, SYS_CNT\n" + "MOV %0, %0\n" + "MOV %1, %1\n" + : "+l"(address), "+l"(value), "=&l"(cycle) + : + : "memory"); + return cycle; +} + +__aicore__ __attribute__((always_inline)) inline uint64_t CycleAfterStore(uint64_t &address, uint64_t &value) { + uint64_t cycle = 0; + asm volatile("MOV %0, %0\n" + "MOV %1, %1\n" + "MOV %2, SYS_CNT\n" + : "+l"(address), "+l"(value), "=&l"(cycle) + : + : "memory"); + return cycle; +} + +__aicore__ __attribute__((always_inline)) inline uint64_t OpaqueIdentity(uint64_t value) { + asm volatile("MOV %0, %0\n" : "+l"(value) : : "memory"); + return value; +} + +__aicore__ __attribute__((always_inline)) inline void PublishResult( + __gm__ cache_preload::ProbeResult *result, uint64_t value, uint64_t preparation_checksum, uint64_t gap_checksum, + uint64_t issue_ticks, uint64_t access_or_work_ticks, uint64_t store_flush_ticks, uint64_t total_ticks, + uint64_t immediate_status, uint64_t final_status, uint64_t polls, uint32_t mode, uint32_t target_word, + uint32_t gap_rounds +) { + WriteGmByPassDCache(&result->value, value); + WriteGmByPassDCache(&result->preparation_checksum, preparation_checksum); + WriteGmByPassDCache(&result->gap_checksum, gap_checksum); + WriteGmByPassDCache(&result->issue_ticks, issue_ticks); + WriteGmByPassDCache(&result->access_or_work_ticks, access_or_work_ticks); + WriteGmByPassDCache(&result->store_flush_ticks, store_flush_ticks); + WriteGmByPassDCache(&result->total_ticks, total_ticks); + WriteGmByPassDCache(&result->icache_immediate_status, immediate_status); + WriteGmByPassDCache(&result->icache_final_status, final_status); + WriteGmByPassDCache(&result->icache_polls, polls); + WriteGmByPassDCache(&result->mode_echo, static_cast(mode)); + WriteGmByPassDCache(&result->target_word_echo, static_cast(target_word)); + WriteGmByPassDCache(&result->gap_rounds_echo, static_cast(gap_rounds)); +} + +} // namespace + +extern "C" __aicore__ __attribute__((noinline, used)) uint64_t +cache_preload_ascendc_gap(uint64_t seed, uint32_t gap_rounds) { + return cache_preload::GapOracle(seed, gap_rounds); +} + +#define CACHE_PRELOAD_ASC_NOPS_1() asm volatile("nop"); +#define CACHE_PRELOAD_ASC_NOPS_2() \ + CACHE_PRELOAD_ASC_NOPS_1() \ + CACHE_PRELOAD_ASC_NOPS_1() +#define CACHE_PRELOAD_ASC_NOPS_4() \ + CACHE_PRELOAD_ASC_NOPS_2() \ + CACHE_PRELOAD_ASC_NOPS_2() +#define CACHE_PRELOAD_ASC_NOPS_8() \ + CACHE_PRELOAD_ASC_NOPS_4() \ + CACHE_PRELOAD_ASC_NOPS_4() +#define CACHE_PRELOAD_ASC_NOPS_16() \ + CACHE_PRELOAD_ASC_NOPS_8() \ + CACHE_PRELOAD_ASC_NOPS_8() +#define CACHE_PRELOAD_ASC_NOPS_32() \ + CACHE_PRELOAD_ASC_NOPS_16() \ + CACHE_PRELOAD_ASC_NOPS_16() +#define CACHE_PRELOAD_ASC_NOPS_64() \ + CACHE_PRELOAD_ASC_NOPS_32() \ + CACHE_PRELOAD_ASC_NOPS_32() +#define CACHE_PRELOAD_ASC_NOPS_128() \ + CACHE_PRELOAD_ASC_NOPS_64() \ + CACHE_PRELOAD_ASC_NOPS_64() +#define CACHE_PRELOAD_ASC_NOPS_256() \ + CACHE_PRELOAD_ASC_NOPS_128() \ + CACHE_PRELOAD_ASC_NOPS_128() +#define CACHE_PRELOAD_ASC_NOPS_512() \ + CACHE_PRELOAD_ASC_NOPS_256() \ + CACHE_PRELOAD_ASC_NOPS_256() +#define CACHE_PRELOAD_ASC_NOPS_1024() \ + CACHE_PRELOAD_ASC_NOPS_512() \ + CACHE_PRELOAD_ASC_NOPS_512() +#define CACHE_PRELOAD_ASC_NOPS_2048() \ + CACHE_PRELOAD_ASC_NOPS_1024() \ + CACHE_PRELOAD_ASC_NOPS_1024() +#define CACHE_PRELOAD_ASC_NOPS_4096() \ + CACHE_PRELOAD_ASC_NOPS_2048() \ + CACHE_PRELOAD_ASC_NOPS_2048() +#define CACHE_PRELOAD_ASC_NOPS_8192() \ + CACHE_PRELOAD_ASC_NOPS_4096() \ + CACHE_PRELOAD_ASC_NOPS_4096() + +extern "C" __aicore__ __attribute__((noinline, used, aligned(128))) uint64_t +cache_preload_ascendc_icache_evictor(uint64_t seed) { + CACHE_PRELOAD_ASC_NOPS_8192() + return cache_preload::ICacheEvictorOracle(seed); +} + +extern "C" __aicore__ __attribute__((noinline, used, aligned(128))) void cache_preload_ascendc_icache_path( + uint32_t mode_value, uint64_t seed, uint32_t target_word, uint32_t gap_rounds, uint64_t preparation_checksum, + __gm__ cache_preload::ProbeResult *result +) { + const uint64_t total_begin = ReadOrderedCycle(); + uint64_t issue_ticks = 0; + uint64_t immediate_status = 0; + uint64_t final_status = 0; + uint64_t polls = 0; + + const bool use_preload = mode_value == static_cast(cache_preload::Mode::ICacheCurrentPcAsync) || + mode_value == static_cast(cache_preload::Mode::ICacheCurrentPcWait); + if (use_preload) { + const uint64_t issue_begin = ReadOrderedCycle(); + ICachePreLoad(static_cast(cache_preload::kICachePreloadUnits)); + const uint64_t issue_end = ReadOrderedCycle(); + issue_ticks = issue_end - issue_begin; + + immediate_status = static_cast(GetICachePreloadStatus()); + final_status = immediate_status; + if (mode_value == static_cast(cache_preload::Mode::ICacheCurrentPcWait)) { + while (final_status != 0 && polls < cache_preload::kICachePollLimit) { + CACHE_PRELOAD_ASC_NOPS_16() + final_status = static_cast(GetICachePreloadStatus()); + ++polls; + } + } + } + + const uint64_t gap_checksum = cache_preload_ascendc_gap(seed, gap_rounds); + if (use_preload) { + final_status = static_cast(GetICachePreloadStatus()); + } + + uint64_t work_value = seed ^ gap_checksum; + const uint64_t work_begin = CycleBeforeValue(work_value); + CACHE_PRELOAD_ASC_NOPS_1024() + work_value = cache_preload::ICacheTargetOracle(work_value); + const uint64_t work_end = CycleAfterValue(work_value); + + PublishResult( + result, work_value, preparation_checksum, gap_checksum, issue_ticks, work_end - work_begin, 0, + work_end - total_begin, immediate_status, final_status, polls, mode_value, target_word, gap_rounds + ); +} + +extern "C" __schedmode__(1) __global__ + __mix__(0, 1) void cache_preload_ascendc_probe(__gm__ cache_preload::ProbeState *state) { + if (GetBlockIdx() != 0) return; + + dcci(&state->control, SINGLE_CACHE_LINE); + dsb(DSB_ALL); + + const uint32_t mode_value = state->control.mode; + const uint32_t target_word = state->control.target_word; + const uint32_t gap_rounds = state->control.gap_rounds; + const uint64_t seed = state->control.seed; + __gm__ cache_preload::ProbeResult *result = &state->result; + + if (mode_value == static_cast(cache_preload::Mode::DCacheBaseline) || + mode_value == static_cast(cache_preload::Mode::DCachePreload)) { + __gm__ uint64_t *target = &state->data[target_word]; + dcci(target, SINGLE_CACHE_LINE); + dsb(DSB_ALL); + + GlobalTensor data; + data.SetGlobalBuffer(state->data, cache_preload::kDataWords); + + const uint64_t total_begin = ReadOrderedCycle(); + uint64_t issue_ticks = 0; + if (mode_value == static_cast(cache_preload::Mode::DCachePreload)) { + const uint64_t issue_begin = ReadOrderedCycle(); + const int64_t byte_offset = static_cast(target_word) * sizeof(uint64_t); + DataCachePreload(data, byte_offset); + const uint64_t issue_end = ReadOrderedCycle(); + issue_ticks = issue_end - issue_begin; + } + + const uint64_t gap_checksum = cache_preload_ascendc_gap(seed, gap_rounds); + const uint64_t opaque_gap = OpaqueIdentity(gap_checksum); + const uint64_t address_delta = opaque_gap - gap_checksum; + uint64_t target_address = reinterpret_cast(target) + address_delta; + const uint64_t access_begin = CycleBeforeValue(target_address); + volatile __gm__ uint64_t *volatile_target = reinterpret_cast(target_address); + uint64_t value = *volatile_target; + const uint64_t access_end = CycleAfterValue(value); + + PublishResult( + result, value, 0, gap_checksum, issue_ticks, access_end - access_begin, 0, access_end - total_begin, 0, 0, + 0, mode_value, target_word, gap_rounds + ); + return; + } + + if (mode_value == static_cast(cache_preload::Mode::DCacheStoreBaseline) || + mode_value == static_cast(cache_preload::Mode::DCacheStorePreload) || + mode_value == static_cast(cache_preload::Mode::DCachePublishBaseline) || + mode_value == static_cast(cache_preload::Mode::DCachePublishPreload)) { + __gm__ uint64_t *target = &state->data[target_word]; + + // Host resets the target before every write launch. Establish the + // same cold, clean DCache state outside the timed interval. + dcci(target, SINGLE_CACHE_LINE); + dsb(DSB_ALL); + + GlobalTensor data; + data.SetGlobalBuffer(state->data, cache_preload::kDataWords); + + const bool use_preload = mode_value == static_cast(cache_preload::Mode::DCacheStorePreload) || + mode_value == static_cast(cache_preload::Mode::DCachePublishPreload); + const bool publish_to_gm = mode_value == static_cast(cache_preload::Mode::DCachePublishBaseline) || + mode_value == static_cast(cache_preload::Mode::DCachePublishPreload); + + const uint64_t total_begin = ReadOrderedCycle(); + uint64_t issue_ticks = 0; + if (use_preload) { + const uint64_t issue_begin = ReadOrderedCycle(); + const int64_t byte_offset = static_cast(target_word) * sizeof(uint64_t); + DataCachePreload(data, byte_offset); + const uint64_t issue_end = ReadOrderedCycle(); + issue_ticks = issue_end - issue_begin; + } + + const uint64_t gap_checksum = cache_preload_ascendc_gap(seed, gap_rounds); + const uint64_t opaque_gap = OpaqueIdentity(gap_checksum); + const uint64_t address_delta = opaque_gap - gap_checksum; + uint64_t target_address = reinterpret_cast(target) + address_delta; + uint64_t store_value = cache_preload::WriteValue(target_word, seed, gap_checksum); + volatile __gm__ uint64_t *volatile_target = reinterpret_cast(target_address); + + const uint64_t store_begin = CycleBeforeStore(target_address, store_value); + *volatile_target = store_value; + const uint64_t store_end = CycleAfterStore(target_address, store_value); + + uint64_t total_end = store_end; + uint64_t store_flush_ticks = 0; + if (publish_to_gm) { + dcci(target, SINGLE_CACHE_LINE, CACHELINE_OUT); + dsb(DSB_ALL); + total_end = ReadOrderedCycle(); + store_flush_ticks = total_end - store_begin; + } + + const uint64_t local_value = *volatile_target; + if (!publish_to_gm) { + // Cleanup is intentionally after total_end and therefore excluded + // from the store-only business interval. + dcci(target, SINGLE_CACHE_LINE, CACHELINE_OUT); + dsb(DSB_ALL); + } + const uint64_t gm_value = ReadGmByPassDCache(target); + + PublishResult( + result, gm_value, local_value, gap_checksum, issue_ticks, store_end - store_begin, store_flush_ticks, + total_end - total_begin, 0, 0, 0, mode_value, target_word, gap_rounds + ); + return; + } + + if (mode_value == static_cast(cache_preload::Mode::ICacheColdBaseline) || + mode_value == static_cast(cache_preload::Mode::ICacheCurrentPcAsync) || + mode_value == static_cast(cache_preload::Mode::ICacheCurrentPcWait)) { + const uint64_t preparation_checksum = cache_preload_ascendc_icache_evictor(seed); + cache_preload_ascendc_icache_path(mode_value, seed, target_word, gap_rounds, preparation_checksum, result); + } +} + +namespace { + +struct Sample { + cache_preload::ProbeResult result{}; + uint64_t target_value = 0; +}; + +bool Check(aclError error, const char *expression) { + return atomic_probe::CheckAcl(error, expression, __FILE__, __LINE__); +} + +const char *ModeName(cache_preload::Mode mode) { + switch (mode) { + case cache_preload::Mode::DCacheBaseline: + return "dcache-baseline"; + case cache_preload::Mode::DCachePreload: + return "dcache-preload"; + case cache_preload::Mode::DCacheStoreBaseline: + return "dstore-only-baseline"; + case cache_preload::Mode::DCacheStorePreload: + return "dstore-only-preload"; + case cache_preload::Mode::DCachePublishBaseline: + return "dpublish-gm-baseline"; + case cache_preload::Mode::DCachePublishPreload: + return "dpublish-gm-preload"; + case cache_preload::Mode::ICacheColdBaseline: + return "icache-cold"; + case cache_preload::Mode::ICacheCurrentPcAsync: + return "icache-current-pc"; + case cache_preload::Mode::ICacheCurrentPcWait: + return "icache-wait"; + case cache_preload::Mode::Count: + break; + } + return "unknown"; +} + +bool IsDCacheReadMode(cache_preload::Mode mode) { + return mode == cache_preload::Mode::DCacheBaseline || mode == cache_preload::Mode::DCachePreload; +} + +bool IsDCacheStoreMode(cache_preload::Mode mode) { + return mode == cache_preload::Mode::DCacheStoreBaseline || mode == cache_preload::Mode::DCacheStorePreload; +} + +bool IsDCachePublishMode(cache_preload::Mode mode) { + return mode == cache_preload::Mode::DCachePublishBaseline || mode == cache_preload::Mode::DCachePublishPreload; +} + +bool IsDCacheWriteMode(cache_preload::Mode mode) { return IsDCacheStoreMode(mode) || IsDCachePublishMode(mode); } + +uint64_t Median(std::vector values) { + std::sort(values.begin(), values.end()); + return values[values.size() / 2U]; +} + +bool RunOne( + aclrtStream stream, cache_preload::ProbeState *state_device, cache_preload::Mode mode, uint32_t target_word, + uint64_t seed, Sample *sample +) { + cache_preload::ProbeControl control{}; + control.mode = static_cast(mode); + control.target_word = target_word; + control.gap_rounds = cache_preload::kGapRounds; + control.seed = seed; + cache_preload::ProbeResult zero{}; + + auto *state_bytes = reinterpret_cast(state_device); + void *control_device = state_bytes + offsetof(cache_preload::ProbeState, control); + void *result_device = state_bytes + offsetof(cache_preload::ProbeState, result); + void *target_device = + state_bytes + offsetof(cache_preload::ProbeState, data) + static_cast(target_word) * sizeof(uint64_t); + if (IsDCacheWriteMode(mode)) { + const uint64_t initial_value = cache_preload::DataValue(target_word); + if (!Check( + aclrtMemcpy( + target_device, sizeof(initial_value), &initial_value, sizeof(initial_value), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D write target reset)" + )) { + return false; + } + } + if (!Check( + aclrtMemcpy(control_device, sizeof(control), &control, sizeof(control), ACL_MEMCPY_HOST_TO_DEVICE), + "aclrtMemcpy(H2D control)" + ) || + !Check( + aclrtMemcpy(result_device, sizeof(zero), &zero, sizeof(zero), ACL_MEMCPY_HOST_TO_DEVICE), + "aclrtMemcpy(H2D result reset)" + )) { + return false; + } + + cache_preload_ascendc_probe<<<1, nullptr, stream>>>(state_device); + if (!Check(aclrtSynchronizeStream(stream), "aclrtSynchronizeStream") || + !Check( + aclrtMemcpy( + &sample->result, sizeof(sample->result), result_device, sizeof(sample->result), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H result)" + )) { + return false; + } + if (IsDCacheWriteMode(mode) && !Check( + aclrtMemcpy( + &sample->target_value, sizeof(sample->target_value), target_device, + sizeof(sample->target_value), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H write target)" + )) { + return false; + } + return true; +} + +bool ValidateSample( + const Sample &sample, cache_preload::Mode mode, uint32_t target_word, uint64_t seed, std::string *reason +) { + const cache_preload::ProbeResult &result = sample.result; + if (result.mode_echo != static_cast(mode)) { + *reason = "mode-echo"; + return false; + } + if (result.target_word_echo != target_word) { + *reason = "target-word-echo"; + return false; + } + if (result.gap_rounds_echo != cache_preload::kGapRounds) { + *reason = "gap-rounds-echo"; + return false; + } + const uint64_t expected_gap = cache_preload::GapOracle(seed, cache_preload::kGapRounds); + if (result.gap_checksum != expected_gap) { + *reason = "gap-checksum"; + return false; + } + + if (IsDCacheReadMode(mode)) { + if (result.value != cache_preload::DataValue(target_word)) { + *reason = "dcache-load-value"; + return false; + } + if (result.preparation_checksum != 0 || result.icache_immediate_status != 0 || + result.icache_final_status != 0 || result.icache_polls != 0 || result.store_flush_ticks != 0) { + *reason = "dcache-unexpected-icache-fields"; + return false; + } + } else if (IsDCacheWriteMode(mode)) { + const uint64_t expected_value = cache_preload::WriteValue(target_word, seed, expected_gap); + if (result.preparation_checksum != expected_value) { + *reason = "dcache-writer-local-value"; + return false; + } + if (result.value != expected_value) { + *reason = "dcache-bypass-value-after-cleanup"; + return false; + } + if (sample.target_value != expected_value) { + *reason = "dcache-host-visible-value"; + return false; + } + if ((IsDCacheStoreMode(mode) && result.store_flush_ticks != 0) || + (IsDCachePublishMode(mode) && result.store_flush_ticks == 0)) { + *reason = "dcache-store-flush-window"; + return false; + } + if (result.icache_immediate_status != 0 || result.icache_final_status != 0 || result.icache_polls != 0) { + *reason = "dcache-write-unexpected-icache-fields"; + return false; + } + } else { + if (result.preparation_checksum != cache_preload::ICacheEvictorOracle(seed)) { + *reason = "icache-evictor-checksum"; + return false; + } + if (result.value != cache_preload::ICacheTargetOracle(seed ^ expected_gap)) { + *reason = "icache-target-checksum"; + return false; + } + if (mode == cache_preload::Mode::ICacheCurrentPcWait && + (result.icache_final_status != 0 || result.icache_polls >= cache_preload::kICachePollLimit)) { + *reason = "icache-wait-did-not-reach-idle"; + return false; + } + if (result.store_flush_ticks != 0) { + *reason = "icache-unexpected-store-flush"; + return false; + } + } + + if (result.access_or_work_ticks == 0 || result.total_ticks == 0) { + *reason = "zero-timing-window"; + return false; + } + return true; +} + +void PrintSummary(cache_preload::Mode mode, const std::vector &samples) { + std::vector issue; + std::vector access_or_work; + std::vector store_flush; + std::vector total; + std::vector polls; + uint32_t immediate_busy = 0; + uint32_t final_busy = 0; + for (const Sample &sample : samples) { + issue.push_back(sample.result.issue_ticks); + access_or_work.push_back(sample.result.access_or_work_ticks); + store_flush.push_back(sample.result.store_flush_ticks); + total.push_back(sample.result.total_ticks); + polls.push_back(sample.result.icache_polls); + immediate_busy += sample.result.icache_immediate_status != 0 ? 1U : 0U; + final_busy += sample.result.icache_final_status != 0 ? 1U : 0U; + } + std::printf( + "%-21s issue=%6llu access/work=%6llu store->gm=%6llu total=%6llu " + "polls=%5llu immediate_busy=%u/%zu final_busy=%u/%zu\n", + ModeName(mode), static_cast(Median(std::move(issue))), + static_cast(Median(std::move(access_or_work))), + static_cast(Median(std::move(store_flush))), + static_cast(Median(std::move(total))), + static_cast(Median(std::move(polls))), immediate_busy, samples.size(), final_busy, + samples.size() + ); +} + +} // namespace + +int32_t main() { + const int32_t device = atomic_probe::DeviceId(); + if (device < 0) return EXIT_FAILURE; + if (!Check(aclInit(nullptr), "aclInit") || !Check(aclrtSetDevice(device), "aclrtSetDevice")) { + return EXIT_FAILURE; + } + + aclrtStream stream = nullptr; + if (!Check(aclrtCreateStream(&stream), "aclrtCreateStream")) { + return EXIT_FAILURE; + } + + cache_preload::ProbeState *state_device = nullptr; + if (!Check( + aclrtMalloc( + reinterpret_cast(&state_device), sizeof(cache_preload::ProbeState), ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(probe state)" + )) { + return EXIT_FAILURE; + } + + auto initial = std::make_unique(); + for (uint32_t index = 0; index < cache_preload::kDataWords; ++index) { + initial->data[index] = cache_preload::DataValue(index); + } + if (!Check( + aclrtMemcpy(state_device, sizeof(*initial), initial.get(), sizeof(*initial), ACL_MEMCPY_HOST_TO_DEVICE), + "aclrtMemcpy(H2D initial state)" + )) { + return EXIT_FAILURE; + } + + constexpr std::array kModes = { + cache_preload::Mode::DCacheBaseline, cache_preload::Mode::DCachePreload, + cache_preload::Mode::DCacheStoreBaseline, cache_preload::Mode::DCacheStorePreload, + cache_preload::Mode::DCachePublishBaseline, cache_preload::Mode::DCachePublishPreload, + cache_preload::Mode::ICacheColdBaseline, cache_preload::Mode::ICacheCurrentPcAsync, + cache_preload::Mode::ICacheCurrentPcWait, + }; + std::array, 9> samples; + bool launch_ok = true; + bool semantic_ok = true; + for (uint32_t sample_index = 0; sample_index < cache_preload::kSamples; ++sample_index) { + const uint32_t target_word = cache_preload::kTargetStartWord + sample_index * cache_preload::kTargetStrideWords; + const uint64_t seed = 0x123456789abcdef0ULL ^ (static_cast(sample_index) * 0x9e3779b97f4a7c15ULL); + + for (const cache_preload::Mode mode : kModes) { + Sample sample; + launch_ok = RunOne(stream, state_device, mode, target_word, seed, &sample); + if (!launch_ok) break; + + std::string reason; + if (!ValidateSample(sample, mode, target_word, seed, &reason)) { + std::fprintf( + stderr, + "[MISMATCH] sample=%u mode=%s reason=%s " + "value=0x%llx target=0x%llx issue=%llu access/work=%llu " + "store->gm=%llu total=%llu immediate=%llu final=%llu polls=%llu\n", + sample_index, ModeName(mode), reason.c_str(), static_cast(sample.result.value), + static_cast(sample.target_value), + static_cast(sample.result.issue_ticks), + static_cast(sample.result.access_or_work_ticks), + static_cast(sample.result.store_flush_ticks), + static_cast(sample.result.total_ticks), + static_cast(sample.result.icache_immediate_status), + static_cast(sample.result.icache_final_status), + static_cast(sample.result.icache_polls) + ); + semantic_ok = false; + } + samples[static_cast(mode)].push_back(sample); + } + if (!launch_ok) break; + } + + atomic_probe::Result result; + result.Expect(launch_ok, "AscendC cache preload launches and result copies"); + result.Expect(semantic_ok, "AscendC cache preload values, write publication, and status contract"); + + std::printf( + "=== A5 AscendC Cache Preload Usage Probe ===\n" + "samples_per_mode=%u gap_rounds=%u icache_preload_units=%u\n" + "All timing values are raw SYS_CNT deltas; they are " + "observations, not API guarantees.\n", + cache_preload::kSamples, cache_preload::kGapRounds, cache_preload::kICachePreloadUnits + ); + for (const cache_preload::Mode mode : kModes) { + const auto &mode_samples = samples[static_cast(mode)]; + if (!mode_samples.empty()) { + PrintSummary(mode, mode_samples); + } + } + + bool cleanup_ok = true; + cleanup_ok &= Check(aclrtFree(state_device), "aclrtFree(probe state)"); + cleanup_ok &= Check(aclrtDestroyStream(stream), "aclrtDestroyStream"); + cleanup_ok &= Check(aclrtResetDevice(device), "aclrtResetDevice"); + cleanup_ok &= Check(aclFinalize(), "aclFinalize"); + result.Expect(cleanup_ok, "AscendC cache preload cleanup"); + return result.ExitCode(); +} + +#undef CACHE_PRELOAD_ASC_NOPS_8192 +#undef CACHE_PRELOAD_ASC_NOPS_4096 +#undef CACHE_PRELOAD_ASC_NOPS_2048 +#undef CACHE_PRELOAD_ASC_NOPS_1024 +#undef CACHE_PRELOAD_ASC_NOPS_512 +#undef CACHE_PRELOAD_ASC_NOPS_256 +#undef CACHE_PRELOAD_ASC_NOPS_128 +#undef CACHE_PRELOAD_ASC_NOPS_64 +#undef CACHE_PRELOAD_ASC_NOPS_32 +#undef CACHE_PRELOAD_ASC_NOPS_16 +#undef CACHE_PRELOAD_ASC_NOPS_8 +#undef CACHE_PRELOAD_ASC_NOPS_4 +#undef CACHE_PRELOAD_ASC_NOPS_2 +#undef CACHE_PRELOAD_ASC_NOPS_1 diff --git a/tests/atomic_probe/ascendc/run_cache_preload.sh b/tests/atomic_probe/ascendc/run_cache_preload.sh new file mode 100755 index 0000000000..eec5e6ab10 --- /dev/null +++ b/tests/atomic_probe/ascendc/run_cache_preload.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +# Build and run the AscendC counterpart of the cache-preload probe. The +# dedicated entry point adds final embedded-AICore ELF size/alignment checks +# that the generic _run_asc_probe.sh intentionally does not perform. +# +# Usage: +# ./run_cache_preload.sh +# ./run_cache_preload.sh build +# ./run_cache_preload.sh run +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BUILD_DIR="$SCRIPT_DIR/build/cache_preload" +ACTION="${1:-all}" +if [[ "$ACTION" != "all" && "$ACTION" != "build" && "$ACTION" != "run" ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ $# -gt 1 ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then + echo "ASCEND_HOME_PATH is not set; source the local CANN environment first." >&2 + exit 1 +fi + +BISHENG="${BISHENG:-$ASCEND_HOME_PATH/bin/bisheng}" +OBJCOPY_BIN="${OBJCOPY:-objcopy}" +READELF_BIN="${READELF:-readelf}" +SOURCE="$SCRIPT_DIR/cache_preload_probe.asc" +EXECUTABLE="$BUILD_DIR/cache_preload_probe" +AICORE_ELF="$BUILD_DIR/cache_preload_aicore.o" + +if [[ ! -x "$BISHENG" ]]; then + echo "bisheng is missing: $BISHENG" >&2 + exit 1 +fi +if ! command -v "$OBJCOPY_BIN" >/dev/null 2>&1; then + echo "objcopy is required: $OBJCOPY_BIN" >&2 + exit 1 +fi +if ! command -v "$READELF_BIN" >/dev/null 2>&1; then + echo "readelf is required: $READELF_BIN" >&2 + exit 1 +fi + +symbol_address_hex="" +symbol_size="" +read_unique_function_symbol() { + local elf="$1" + local name="$2" + local -a matches=() + mapfile -t matches < <( + "$READELF_BIN" --symbols --wide "$elf" | + awk -v symbol="$name" '$4 == "FUNC" && $8 == symbol {print $2 " " $3}' + ) + if [[ ${#matches[@]} -ne 1 ]]; then + echo "Expected exactly one FUNC symbol named $name; found ${#matches[@]}" >&2 + exit 1 + fi + read -r symbol_address_hex symbol_size <<<"${matches[0]}" + if [[ ! "$symbol_address_hex" =~ ^[0-9a-fA-F]+$ || + ! "$symbol_size" =~ ^[0-9]+$ ]]; then + echo "Cannot parse ELF symbol $name: ${matches[0]}" >&2 + exit 1 + fi +} + +extract_and_check_aicore() { + if [[ ! -x "$EXECUTABLE" ]]; then + echo "AscendC executable is missing: $EXECUTABLE" >&2 + exit 1 + fi + "$OBJCOPY_BIN" \ + --dump-section ".aicore_binary=$AICORE_ELF" \ + "$EXECUTABLE" + if [[ ! -s "$AICORE_ELF" ]]; then + echo "Embedded .aicore_binary extraction failed: $AICORE_ELF" >&2 + exit 1 + fi + + local symbols sections entry="cache_preload_ascendc_probe" + symbols="$("$READELF_BIN" --symbols --wide "$AICORE_ELF")" + sections="$("$READELF_BIN" --sections --wide "$AICORE_ELF")" + if [[ "$symbols" != *" $entry"* || + "$sections" != *".ascend.meta.$entry"* ]]; then + echo "Missing AIV entry or metadata for $entry" >&2 + exit 1 + fi + + read_unique_function_symbol \ + "$AICORE_ELF" "cache_preload_ascendc_icache_path.vector" + local path_address_hex="$symbol_address_hex" + local path_size="$symbol_size" + read_unique_function_symbol \ + "$AICORE_ELF" "cache_preload_ascendc_icache_evictor.vector" + local evictor_address_hex="$symbol_address_hex" + local evictor_size="$symbol_size" + read_unique_function_symbol \ + "$AICORE_ELF" "cache_preload_ascendc_gap.vector" + + local path_address=$((16#$path_address_hex)) + local evictor_address=$((16#$evictor_address_hex)) + local path_end=$((path_address + path_size)) + local evictor_end=$((evictor_address + evictor_size)) + if ((path_size < 4096)); then + echo "AscendC current-PC path is smaller than 4096B: $path_size" >&2 + exit 1 + fi + if ((evictor_size < 32768)); then + echo "AscendC ICache evictor is smaller than 32768B: $evictor_size" >&2 + exit 1 + fi + if ((path_address % 128 != 0 || evictor_address % 128 != 0)); then + printf 'AscendC ICache symbols are not 128B-aligned: path=0x%x evictor=0x%x\n' \ + "$path_address" "$evictor_address" >&2 + exit 1 + fi + if ! ((path_end <= evictor_address || evictor_end <= path_address)); then + printf 'AscendC path/evictor overlap: path=[0x%x,0x%x) evictor=[0x%x,0x%x)\n' \ + "$path_address" "$path_end" "$evictor_address" "$evictor_end" >&2 + exit 1 + fi + printf '[CHECK] AscendC AIV entry/meta PASS; current-PC path=0x%x/%uB; ' \ + "$path_address" "$path_size" + printf 'evictor=0x%x/%uB; alignment=128B ranges=non-overlap\n' \ + "$evictor_address" "$evictor_size" +} + +build_probe() { + mkdir -p "$BUILD_DIR" + echo "[BUILD] AscendC cache preload probe (dav-3510)" + "$BISHENG" -xasc "$SOURCE" --npu-arch=dav-3510 \ + -o "$EXECUTABLE" + extract_and_check_aicore + echo "[BUILD] complete: $BUILD_DIR" +} + +run_probe() { + extract_and_check_aicore + echo "[RUN] device=${ATOMIC_PROBE_DEVICE:-${TASK_DEVICE:-0}}" + timeout "${CACHE_PRELOAD_TIMEOUT:-120}" "$EXECUTABLE" +} + +echo "=== AscendC Cache Preload Probe ===" +echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" +echo "git_head=$(git -C "$SCRIPT_DIR" rev-parse HEAD 2>/dev/null || echo unknown)" +echo "ascend_home=$ASCEND_HOME_PATH" +echo "bisheng=$($BISHENG --version 2>&1 | head -n 1)" +echo "action=$ACTION" + +if [[ "$ACTION" == "all" || "$ACTION" == "build" ]]; then + build_probe +fi +if [[ "$ACTION" == "all" || "$ACTION" == "run" ]]; then + run_probe +fi diff --git a/tests/atomic_probe/cache_preload_shared.h b/tests/atomic_probe/cache_preload_shared.h new file mode 100644 index 0000000000..b31e33dbfe --- /dev/null +++ b/tests/atomic_probe/cache_preload_shared.h @@ -0,0 +1,132 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef TESTS_ATOMIC_PROBE_CACHE_PRELOAD_SHARED_H_ +#define TESTS_ATOMIC_PROBE_CACHE_PRELOAD_SHARED_H_ + +#include +#include + +namespace cache_preload { + +#if defined(__ENABLE_ASC_LANG__) +#define CACHE_PRELOAD_SHARED_FN __host__ __aicore__ +#elif defined(__CCE_AICORE__) +#define CACHE_PRELOAD_SHARED_FN __aicore__ +#else +#define CACHE_PRELOAD_SHARED_FN +#endif + +constexpr uint32_t kDataWords = 8192; +constexpr uint32_t kSamples = 9; +constexpr uint32_t kTargetStartWord = 512; +constexpr uint32_t kTargetStrideWords = 64; +constexpr uint32_t kGapRounds = 64; +constexpr uint32_t kICachePreloadUnits = 2; +constexpr uint32_t kICachePollLimit = 4096; +constexpr uint32_t kICacheTargetNops = 1024; +constexpr uint32_t kICacheEvictorNops = 8192; + +enum class Mode : uint32_t { + DCacheBaseline = 0, + DCachePreload = 1, + DCacheStoreBaseline = 2, + DCacheStorePreload = 3, + DCachePublishBaseline = 4, + DCachePublishPreload = 5, + ICacheColdBaseline = 6, + ICacheCurrentPcAsync = 7, + ICacheCurrentPcWait = 8, + Count = 9, +}; + +CACHE_PRELOAD_SHARED_FN constexpr uint64_t DataValue(uint32_t index) { + return 0xa500000000000000ULL ^ (static_cast(index) * 0x9e3779b97f4a7c15ULL); +} + +CACHE_PRELOAD_SHARED_FN constexpr uint64_t WriteValue(uint32_t index, uint64_t seed, uint64_t gap_checksum) { + return 0x5a00000000000000ULL ^ (static_cast(index) * 0xd6e8feb86659fd93ULL) ^ seed ^ gap_checksum; +} + +// This intentionally remains a runtime loop in each device kernel: gap_rounds is +// read from GM and passed into a noinline function. It models useful scalar +// work that can overlap a preload request without sequentially walking the +// upcoming ICache target region. +CACHE_PRELOAD_SHARED_FN inline uint64_t GapOracle(uint64_t seed, uint32_t gap_rounds) { + uint64_t value = seed ^ 0xd1b54a32d192ed03ULL; + for (uint32_t round = 0; round < gap_rounds; ++round) { + value ^= value >> (round % 17U + 1U); + value *= 0x9e3779b97f4a7c15ULL + static_cast(round) * 2ULL; + value += (value << (round % 7U + 1U)) ^ (0x94d049bb133111ebULL + static_cast(round)); + } + return value; +} + +CACHE_PRELOAD_SHARED_FN constexpr uint64_t ICacheTargetOracle(uint64_t value) { + value ^= value >> 29U; + value *= 0x9e3779b185ebca87ULL; + value += 0xa0761d6478bd642fULL; + value ^= value >> 31U; + return value; +} + +CACHE_PRELOAD_SHARED_FN constexpr uint64_t ICacheEvictorOracle(uint64_t value) { + value ^= 0xe7037ed1a0b428dbULL; + value ^= value >> 23U; + value *= 0x8ebc6af09c88c6e3ULL; + value ^= value >> 27U; + return value; +} + +struct alignas(64) ProbeControl { + uint32_t mode; + uint32_t target_word; + uint32_t gap_rounds; + uint32_t reserved0; + uint64_t seed; + uint64_t reserved[5]; +}; + +struct alignas(64) ProbeResult { + uint64_t value; + uint64_t preparation_checksum; + uint64_t gap_checksum; + uint64_t issue_ticks; + uint64_t access_or_work_ticks; + uint64_t store_flush_ticks; + uint64_t total_ticks; + uint64_t icache_immediate_status; + + uint64_t icache_final_status; + uint64_t icache_polls; + uint64_t mode_echo; + uint64_t target_word_echo; + uint64_t gap_rounds_echo; + uint64_t reserved[3]; +}; + +struct alignas(64) ProbeState { + ProbeControl control; + ProbeResult result; + alignas(64) uint64_t data[kDataWords]; +}; + +static_assert(sizeof(ProbeControl) == 64, "control must occupy one cache line"); +static_assert(sizeof(ProbeResult) == 128, "result must occupy two cache lines"); +static_assert(offsetof(ProbeState, result) == 64, "result must start on its own cache line"); +static_assert(offsetof(ProbeState, data) == 192, "data must start on its own cache line"); +static_assert(sizeof(ProbeState) == 192 + kDataWords * sizeof(uint64_t), "probe ABI changed"); + +#undef CACHE_PRELOAD_SHARED_FN + +} // namespace cache_preload + +#endif // TESTS_ATOMIC_PROBE_CACHE_PRELOAD_SHARED_H_ diff --git a/tests/atomic_probe/cache_preload_usage_guide.md b/tests/atomic_probe/cache_preload_usage_guide.md new file mode 100644 index 0000000000..5c9b8719eb --- /dev/null +++ b/tests/atomic_probe/cache_preload_usage_guide.md @@ -0,0 +1,848 @@ +# A5 CCEC/AscendC ICache/DCache Preload 用法与真机验证 + +## 1. 当前交付边界 + +本目录现在提供四套可独立构建和运行的用例： + +- CCEC 用例直接调用编译器 intrinsic，不包含 `kernel_operator.h`； +- AscendC 的被测 preload 调用使用公开 `kernel_operator.h` API；冷态构造、发布边界和 + 校验还显式使用 `dcci`、`dsb` 与 bypass load/store； +- 上述两者复用同一套 mode、host/device ABI、输入、checksum oracle、冷态构造和 + raw `SYS_CNT` 统计口径，以便区分“API 用法差异”和“测试模型差异”； +- 第三套是纯 CCEC 的 `1:2` mixed 持续写探针，用与 PA 泳道相同的 32 B + 七字段 record、独占 cacheline 和最终逐行发布口径，专门回答单条 store + microprobe 没有覆盖的连续写问题； +- 第四套是纯 CCEC 的 PA shared 物理模型，不接入调度主流程，只复刻当前 + `writer_history` 的 40 B destination footprint、1/3 个 `TensorDesc` 的 128/384 B 发布、 + descriptor 的 invalidate 后普通读取，以及 current-PC ICache preload + 跨函数与同目标两种放置方式。 + +用户口语中的 `icache_pretch`、`dcache_pretch` 不是当前 CANN 头文件中的符号。正确检索词是： + +| 目的 | CCEC 接口 | AscendC 接口 | 说明 | +| --- | --- | --- | --- | +| 从当前 PC 预取后续指令 | `icache_preload(len)` | `ICachePreLoad(len)` | `len` 在 Atlas 350 上以 2 KiB 为单位 | +| 查询 ICache preload 状态 | `get_icache_prl_st()` | `GetICachePreloadStatus()` | `0=idle`，`1=busy` | +| 从 GM 预取数据到 DCache | `dc_preload(base, byteOffset)` | `DataCachePreload(tensor, byteOffset)` | offset 单位为 byte | +| 指定 ICache 地址的底层接口 | `preload(addr, len)` | 无需在业务中直接使用 | 本探针只测试 current-PC 接口 | + +本次 A5 真机验证观察到： + +- CCEC DCache 普通 GM load 中位数为 `316 -> 5`，AscendC 为 `295 -> 5` raw ticks； +- store-only 场景只计到普通 store 被接受：CCEC、AscendC 均为 `2 -> 2` + raw ticks，完整区间没有收益； +- 新增的持续顺序写探针复刻泳道记录的物理形态：96 核各写 120 KiB、每条 + 32 B、每 cacheline 两条记录；提前 16 条 cacheline 的 `dc_preload` + 使最慢核 store 发射窗口下降 `40.436%`，包含最终逐行 + `CACHELINE_OUT + DSB` 的总窗口下降 `30.460%`； +- publish-to-GM 场景包含 `CACHELINE_OUT + DSB`：CCEC 的 `store->GM` 为 + `512 -> 206`，AscendC 为 `343 -> 121` raw ticks； +- CCEC ICache 同一物理指令区中位数为 `975 -> 484`，AscendC 为 + `1085 -> 743` raw ticks； +- PA shared 物理模型中，128 B/384 B descriptor 发布即使没有额外 overlap + gap，96 核关键核总窗口也分别稳定下降约 `9%`/`25%`；保留约 725 raw ticks + 独立 gap 后分别下降约 `22%`/`28%`； +- 同一 4,644 B ICache 目标区中，在与目标不相邻的 caller 发起 preload 时关键核 + 总窗口仅变化约 `-1%~-2%`，在目标函数内部发起时两轮均下降约 `41%`； +- 两套实现的两个 ICache preload mode 都在发起后立即得到 `9/9 busy`，独立 gap + 之后均得到 `9/9 idle`； +- 立即轮询等待没有降低目标工作区间；两次有效运行中完整区间的微小差异方向并不 + 一致，不能解释为 wait 收益。 + +因此当前证据必须按场景表述：单条 cold-line store 被 Scalar/store queue 接受的 +窗口没有观察到收益；这不能外推为持续写也无收益。持续顺序写超过每核 DCache +容量后，提前预取未来 cacheline 可以减少 store-side 停顿，但没有降低最终 +`DCCI + DSB` 发布段。这些仍是 microprobe 的真机观察，不是 PA Submit 的固定收益 +承诺；真实泳道路径还需要带编译开关的端到端 A/B。 + +## 2. 文件与运行方式 + +| 文件 | 用途 | +| --- | --- | +| `ccec/cache_preload.cpp` | 纯 CCEC AIV kernel，直接调用 preload intrinsic | +| `cache_preload_shared.h` | CCEC/AscendC/host 共用 ABI、模式和 checksum oracle | +| `ccec/cache_preload_host.cpp` | 真机 launcher、逐样本校验和中位数汇总 | +| `ccec/run_cache_preload.sh` | CCEC 独立构建、最终 ELF 门禁和定向运行 | +| `ascendc/cache_preload_probe.asc` | AscendC AIV kernel、host launcher 和校验 | +| `ascendc/run_cache_preload.sh` | AscendC 独立构建、嵌入 AICore ELF 门禁和定向运行 | +| `trace_write_preload_shared.h` | 持续泳道写探针的 32 B record、控制块和结果 ABI | +| `ccec/trace_write_preload.cpp` | 1:2 mixed CCEC kernel；复刻七字段顺序写与最终逐行发布 | +| `ccec/trace_write_preload_host.cpp` | 96 核/单核策略矩阵、payload 全量回读和容量拐点校验 | +| `ccec/run_trace_write_preload.sh` | 持续泳道写探针的独立构建、mixed ELF 门禁和运行入口 | +| `shared_preload_model_shared.h` | PA shared 物理粒度、控制块、结果和 payload oracle | +| `ccec/shared_preload_model.cpp` | 1:2 mixed CCEC kernel；发布/消费与 ICache 放置模型 | +| `ccec/shared_preload_model_host.cpp` | 交错策略矩阵、96 核结果和 payload 全量校验 | +| `ccec/run_shared_preload_model.sh` | shared 模型构建、最终 ICache 布局门禁和 A5 运行入口 | + +从仓库根目录运行： + +```bash +tests/atomic_probe/ccec/run_cache_preload.sh +tests/atomic_probe/ascendc/run_cache_preload.sh +tests/atomic_probe/ccec/run_trace_write_preload.sh +tests/atomic_probe/ccec/run_shared_preload_model.sh +``` + +三套脚本都可以拆开： + +```bash +tests/atomic_probe/ccec/run_cache_preload.sh build +tests/atomic_probe/ccec/run_cache_preload.sh run + +tests/atomic_probe/ascendc/run_cache_preload.sh build +tests/atomic_probe/ascendc/run_cache_preload.sh run + +tests/atomic_probe/ccec/run_trace_write_preload.sh build +tests/atomic_probe/ccec/run_trace_write_preload.sh run + +tests/atomic_probe/ccec/run_shared_preload_model.sh build +tests/atomic_probe/ccec/run_shared_preload_model.sh run +``` + +四个脚本都是独立入口，没有修改 `ccec/run_all.sh`，也不会误跑目录内其他探针。 +AscendC runner 会从可执行文件提取 `.aicore_binary`，再对实际执行的 `.vector` +符号做尺寸、对齐和地址不重叠检查，不能用 host ELF 的表面尺寸替代该门禁。持续写 +runner 则检查最终 ELF 同时存在 AIC/AIV 入口及其 `1:2` mixed metadata，32 个物理 +block 实际形成 32 AIC + 64 AIV。shared 模型 runner 还要求 4 KiB 以上的 ICache +目标区、32 KiB 以上的 evictor 均在最终 ELF 中保留且按 128 B 对齐，并证明远端 +caller 的 4 KiB forward preload 窗口不覆盖目标函数。 + +## 3. 已查证的接口语义 + +### 3.1 `dc_preload` + +CCEC 业务参考形态： + +```cpp +__gm__ uint8_t *metadata_bytes = /* GM base */; +const int64_t byte_offset = + static_cast(entry_index) * entry_stride_bytes; + +dc_preload( + reinterpret_cast<__gm__ uint64_t *>(metadata_bytes), + byte_offset); + +// 必须放已有且与目标 load 无数据依赖的工作，创造隐藏 miss 的窗口。 +DoIndependentWork(); + +const uint64_t value = + *reinterpret_cast( + metadata_bytes + byte_offset); +``` + +AscendC 对等形态： + +```cpp +AscendC::GlobalTensor metadata; +metadata.SetGlobalBuffer(metadata_base, metadata_word_count); + +const int64_t byte_offset = + static_cast(entry_index) * entry_stride_bytes; +AscendC::DataCachePreload(metadata, byte_offset); + +DoIndependentWork(); +const uint64_t value = + *reinterpret_cast( + reinterpret_cast<__gm__ uint8_t *>(metadata_base) + byte_offset); +``` + +当前 CANN 9.1 weekly 的 `dav_3510/kernel_operator_cache_impl.h` 也把上层 +`DataCachePreload` 直接下沉为： + +```cpp +dc_preload(src, cacheOffset); +``` + +官方接口契约确认： + +- 源地址按 `uint64_t` GM tensor/指针描述； +- `cacheOffset` 单位是 byte，上层 API 支持 `int16_t`/`int64_t`； +- 作用是从特定 GM 地址预加载到 data cache； +- 频繁调用可能造成保留站拥塞，此时指令可能被当作 NOP，并阻塞 Scalar 流水。 + +官方契约只说明“把 GM 数据预加载到 DCache”，没有承诺后续 store 或写回一定加速。 +因此不能在每个短小 load/store 前机械调用。只有“地址能提前确定、后面有独立工作、 +目标很可能被消费”的点才值得尝试；写场景必须再区分 store-only 和发布到 GM。 + +当前 CANN ListTensor 内部实现有“从 miss offset 预取 256 Bytes”的注释，但公开接口文档没有把 +256 B 定义为跨版本契约。业务正确性不能依赖固定预取范围。 + +官方文档： +[DataCachePreload](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/900/API/ascendcopapi/atlasascendc_api_07_0176.html)。 + +### 3.2 普通 store 与发布到 GM + +普通 scalar store、把 dirty DCache line 发布到 GM 是两个不同业务边界： + +```cpp +*volatile_target = value; +const uint64_t store_only_end = ReadSysCnt(); + +// 只有业务要求本次把 dirty line 刷到 GM 时，才把下面两步纳入关键路径。 +dcci(target, SINGLE_CACHE_LINE, CACHELINE_OUT); +dsb(DSB_ALL); +const uint64_t publish_to_gm_end = ReadSysCnt(); +``` + +当前探针据此提供四个独立 mode： + +| mode | preload | 被测终点 | +| --- | --- | --- | +| `dstore-only-baseline` | 无 | 普通 volatile store 后 | +| `dstore-only-preload` | 有 | 普通 volatile store 后 | +| `dpublish-gm-baseline` | 无 | store 后的 `CACHELINE_OUT + DSB` 完成后 | +| `dpublish-gm-preload` | 有 | store 后的 `CACHELINE_OUT + DSB` 完成后 | + +store-only 不插 `PipeBarrier()`。官方明确说明 Scalar 流水间顺序由硬件自动 +保证，调用 `PipeBarrier()` 会触发硬件错误。store 后的 `SYS_CNT` 本身是后续 +Scalar 指令，因此这里测的是 store 在 Scalar 顺序中的指令窗口，不是 dirty line 已经 +写回 GM。若业务语义是 `store + DSB`、但不做 DCCI，那是第三种边界，当前两场景探针 +没有把它混入 store-only。官方文档： +[PipeBarrier](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/900beta2/API/ascendcopapi/atlasascendc_api_07_0271.html)。 + +store-only mode 为了校验与清理，也会在计时终点之后执行本核普通回读、 +`CACHELINE_OUT + DSB`、bypass read 和 host D2H。这些动作不计入 store-only 的 +`access/work` 或 `total`，因此后续 host 能看到新值不能被误解成“store-only 被测区间 +已经完成 GM 发布”。 + +官方 `CACHELINE_OUT` 契约是让 Data Cache 与 Global Memory 保持一致；preload 本身不 +clean dirty line，也不提供发布或同步语义。官方文档： +[DataCacheCleanAndInvalid](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/900/API/ascendcopapi/atlasascendc_api_07_0177.html)。 + +### 3.3 `icache_preload` + +当前 Bisheng CCEC 头文件中的实现是： + +```cpp +inline void icache_preload(int64_t len) +{ + preload(reinterpret_cast(get_pc()), len); +} +``` + +也就是说，它不是“按 C++ 函数名预取”，而是从调用点当前 PC 所在的指令地址开始预取。 +两套发起方式分别为： + +```cpp +icache_preload(2); // CCEC +AscendC::ICachePreLoad(2); // AscendC；业务只选与当前实现对应的一行 + +// 最好执行会跳到别处的小段已有工作，让当前 PC 后方代码在后台预取。 +DoIndependentControlWork(); + +RunUpcomingSequentialHotPath(); +``` + +调用点应位于即将执行的顺序热点代码之前。若调用后马上跳到无关且很远的分支，预取到的代码可能不被 +消费；若调用太晚，miss 已经发生，也无法隐藏延迟。 + +官方文档对 Atlas 350 说明： + +- `len`/`preFetchLen` 单位为 2 KiB； +- 取值应小于 `ICache size / 2 KiB`； +- 文档列出的 AIC/AIV ICache 大小分别是 32 KiB/16 KiB。 + +本轮两套实现都使用 2 units，即 4 KiB。最终 ELF 中，CCEC current-PC 被测函数为 +4,804 B，AscendC 实际执行的 `.vector` 被测函数为 4,820 B。 + +当前 CANN OPP 的 arch35 存量代码也采用相同模式。例如 +`mat_mul_v3/arch35/mat_mul_stream_k_kernel.h` 在 AIV 分支调用 `ICachePreLoad(2)` 后直接进入后续 +处理，没有立即轮询；这与“发起后继续可重叠工作”的用法一致。 + +官方文档： +[ICachePreLoad](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/900/API/ascendcopapi/atlasascendc_api_07_0276.html)。 + +### 3.4 `get_icache_prl_st` + +两套查询方式分别为： + +```cpp +const int64_t ccec_status = get_icache_prl_st(); +const int64_t ascendc_status = AscendC::GetICachePreloadStatus(); +``` + +返回值： + +- `0`：idle； +- `1`：busy。 + +两套探针都保留了 wait 对照，下面展示 AscendC 写法；CCEC 只需替换成对应的 +小写 intrinsic： + +```cpp +AscendC::ICachePreLoad(2); +while (AscendC::GetICachePreloadStatus() != 0) { + asm volatile("nop"); +} +RunUpcomingSequentialHotPath(); +``` + +这只是状态契约验证，不是推荐的性能写法。为了隐藏 miss，通常应让其他有用工作和预取重叠；立即轮询 +会把异步行为重新串行化，并增加 Scalar 指令。 + +官方上层状态文档： +[GetICachePreloadStatus](https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/900/API/ascendcopapi/atlasascendc_api_07_0277.html)。 + +## 4. 真机探针如何避免错误归因 + +### 4.1 DCache 冷读对照 + +每个样本使用相隔 512 B 的新目标地址，baseline 与 preload 执行： + +1. 在计时区间外对目标行执行单行 DCCI invalidation 和完成等待； +2. baseline 不发起 hint；CCEC preload 调用 + `dc_preload(state->data, targetWord * sizeof(uint64_t))`，AscendC 调用 + `DataCachePreload(data, targetWord * sizeof(uint64_t))`； +3. 两条路径调用完全相同的 noinline 64-round Scalar gap； +4. gap 的 checksum 通过 opaque register dependency 约束目标地址计算； +5. 执行普通 volatile GM load； +6. host 校验 load 值、gap checksum、mode/offset echo。 + +DCCI 只用于建立 probe 的冷态起点，位于测量区间之外；它不是 preload 业务写法的一部分。 + +### 4.2 DCache 写对照 + +四个写 mode 在每次 launch 前都由 host 把目标 word 恢复成同一初值。kernel 随后在 +计时区间外对目标 64 B line 执行 DCCI 和 DSB，确保 baseline/preload 都从 clean、 +cold line 开始。目标 word 位于 cacheline 首地址，不存在跨 line 的 DCCI 歧义。 + +两类场景复用相同地址、写入值和 64-round gap： + +1. `dstore-only-*` 的 `access/work` 只包围普通 volatile store，`total` 从可选 + preload 发起前开始、在 store 后结束； +2. `dpublish-gm-*` 的 `access/work` 仍只包围 store，`store->GM` 从 store 前开始、 + 在 `CACHELINE_OUT + DSB` 后结束，`total` 也在该发布序列后结束； +3. preload mode 都在 gap 前发起 `dc_preload`/`DataCachePreload`，baseline 不发起； +4. 写入值依赖 gap checksum，目标地址也通过 opaque dependency 依赖 gap，防止 O3 + 把 store 提前； +5. 计时结束后，本核普通回读、bypass read 与 host D2H 必须三者都等于预期新值。 + +第 5 点只做正确性门禁。尤其对 store-only mode，bypass/host 校验发生在计时结束后的 +清理性 `CACHELINE_OUT + DSB` 之后，不属于被测业务路径。 + +### 4.3 ICache 冷态与同一物理代码区 + +为了避免“源码写了很多 NOP，但最终 ELF 并没有形成足够指令 footprint”的假测试，构建脚本直接检查 +链接后实际 AIV ELF： + +| 实现 | 符号 | 最终地址 | 最终大小 | 门禁 | +| --- | --- | ---: | ---: | --- | +| CCEC | `cache_preload_icache_evictor` | `0x100` | 32,836 B | ≥32 KiB、128 B 对齐 | +| CCEC | `cache_preload_icache_path` | `0x8180` | 4,804 B | ≥4 KiB、128 B 对齐 | +| AscendC | `cache_preload_ascendc_icache_evictor.vector` | `0x9580` | 32,836 B | ≥32 KiB、128 B 对齐 | +| AscendC | `cache_preload_ascendc_icache_path.vector` | `0x11600` | 4,820 B | ≥4 KiB、128 B 对齐 | + +每套实现内部的 evictor/path 地址范围互不重叠。每个 ICache 样本先完整执行该实现的 +evictor，再从同一个 current-PC path 进入以下三种动态路径： + +- `icache-cold`：不发起 preload； +- `icache-current-pc`：调用各自 current-PC preload API 后进入 noinline gap； +- `icache-wait`：发起后先轮询 idle，再调用同一 gap。 + +三条路径最终汇合到同一份 1,024 条 volatile NOP 指令区，而不是各自复制一份 target。host 同时复算： + +- evictor checksum； +- gap checksum； +- target checksum； +- preload immediate/final status； +- mode、target 和 gap-round echo。 + +### 4.4 计时边界 + +两套实现的 `SYS_CNT` 读取和被测值都放在同一个 inline asm 数据依赖中，防止 O3 +把普通 load、store 或纯 Scalar checksum 移出计时区间。这个 dependency 只约束 +编译器，不是 DSB、DCCI 或跨核同步。 + +所有结果都报告 raw `SYS_CNT` delta。本轮没有把 raw tick 按 1 GHz、1.65 GHz 或其他假设频率换算 +成 ns。 + +### 4.5 持续泳道写与 DCache 容量对照 + +单条 store 的 `2 -> 2` 只能回答“一个 ordinary store 何时能继续执行后续 Scalar +指令”，不能覆盖以下持续写行为： + +- 连续触及新的 cold cacheline； +- dirty line 超出 DCache 容量后逐步被替换； +- store queue、DCache miss 和写回流量形成反压； +- 最后对完整有效区间逐行执行 `DCCI(CACHELINE_OUT)`，再以一次 DSB 收口。 + +因此持续写探针不复用单 word mode，而是建立与当前 PA trace 相同的物理 record： + +```cpp +struct alignas(32) TraceRecord { + uint64_t start_cycle; + uint64_t end_cycle; + int32_t task_id; + int32_t function_id; + uint32_t flags; + uint16_t phase; + uint16_t auxiliary; +}; +static_assert(sizeof(TraceRecord) == 32); +``` + +每条 record 分别执行 7 个 ordinary scalar GM 字段 store；两个连续 record 恰好填满 +一条 64 B cacheline。每核拥有独立且 64 B 对齐的 128 KiB 区间，不存在跨核同 line +竞争。被测窗口分成： + +1. `issue`：从第一条 record 写入前到最后一条 ordinary store 之后； +2. `flush`：对全部有效 cacheline 逐行 `CACHELINE_OUT`，再执行 DSB； +3. `total = issue + flush`。 + +preload 策略写成 `dN-cM`：`dN` 表示预取当前写位置之后 N 条 cacheline，`cM` +表示每 M 条 cacheline 发起一次 hint。例如 `d16-c1` 是每写一条 line 时，为未来 +第 16 条 line 发起一次 `dc_preload`。预取指令本身包含在 `issue` 窗口内，不能靠 +移出计时区间制造收益。每个策略的首轮还会把所有有效 record 从 GM 全量回读，逐字段 +验证预期值；每次 launch 都校验 96 个 mixed worker 的结果或未参与者零值。 + +同一探针还提供随机依赖 pointer-cycle 容量扫描。每条 cacheline 只保存下一条 line +编号，下一次 load 地址依赖上一次结果，从而避免把顺序硬件预取误认为 DCache 容量。 +每个 working set 先 cold 遍历一次，再立即重复遍历 8 次；这里只用复用延迟的容量 +拐点判断有效 resident set，不把单次 raw tick 直接换算为 ns。 + +## 5. 2026-07-29 A5 实测结果 + +环境： + +| 项目 | 值 | +| --- | --- | +| Git 基线 | `ad52c018e323511b9f3a9dfe17c95aadbf9ddab9` | +| CANN | `9.1.0-weekly-20260708` | +| Bisheng/CCEC | clang 15.0.5，构建时间 `2026-07-07T20:35:46+08:00` | +| CCEC arch | `dav-c310-vec` | +| AscendC arch | `--npu-arch=dav-3510`，统计实际执行的 AIV `.vector` 符号 | +| device | `0` | +| 模式数 | 9：DCache read 2、store-only 2、publish-to-GM 2、ICache 3 | +| 每种模式 | 9 个样本，中位数汇总 | +| 独立 gap | 64 rounds | +| ICache preload | 2 units，即 4 KiB | + +同一次最终复测的 CCEC 原始结果： + +| 模式 | issue | access/work | store->GM | total | polls | immediate busy | final busy | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| `dcache-baseline` | 0 | 316 | 0 | 1052 | 0 | 0/9 | 0/9 | +| `dcache-preload` | 3 | 5 | 0 | 742 | 0 | 0/9 | 0/9 | +| `dstore-only-baseline` | 0 | 2 | 0 | 741 | 0 | 0/9 | 0/9 | +| `dstore-only-preload` | 2 | 2 | 0 | 742 | 0 | 0/9 | 0/9 | +| `dpublish-gm-baseline` | 0 | 2 | 512 | 1252 | 0 | 0/9 | 0/9 | +| `dpublish-gm-preload` | 1 | 2 | 206 | 942 | 0 | 0/9 | 0/9 | +| `icache-cold` | 0 | 975 | 0 | 1892 | 0 | 0/9 | 0/9 | +| `icache-current-pc` | 2 | 484 | 0 | 1423 | 0 | 9/9 | 0/9 | +| `icache-wait` | 2 | 484 | 0 | 1439 | 2 | 9/9 | 0/9 | + +同一次最终复测的 AscendC 原始结果： + +| 模式 | issue | access/work | store->GM | total | polls | immediate busy | final busy | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| `dcache-baseline` | 0 | 295 | 0 | 1147 | 0 | 0/9 | 0/9 | +| `dcache-preload` | 2 | 5 | 0 | 739 | 0 | 0/9 | 0/9 | +| `dstore-only-baseline` | 0 | 2 | 0 | 745 | 0 | 0/9 | 0/9 | +| `dstore-only-preload` | 4 | 2 | 0 | 749 | 0 | 0/9 | 0/9 | +| `dpublish-gm-baseline` | 0 | 2 | 343 | 1083 | 0 | 0/9 | 0/9 | +| `dpublish-gm-preload` | 3 | 2 | 121 | 863 | 0 | 0/9 | 0/9 | +| `icache-cold` | 0 | 1085 | 0 | 1912 | 0 | 0/9 | 0/9 | +| `icache-current-pc` | 3 | 743 | 0 | 1602 | 0 | 9/9 | 0/9 | +| `icache-wait` | 3 | 743 | 0 | 1596 | 1 | 9/9 | 0/9 | + +两套实现的 kernel launch、D2H 回读、checksum/echo/status 校验和 cleanup 均为 +PASS。 + +同一逻辑在最终重建前还有一次有效运行。写场景两次中位数如下，用于区分稳定方向和 +绝对值波动： + +| 实现/指标 | 前一次 baseline -> preload | 最终 baseline -> preload | 判断 | +| --- | ---: | ---: | --- | +| CCEC store-only access | `2 -> 1` | `2 -> 2` | 没有稳定下降 | +| AscendC store-only access | `2 -> 2` | `2 -> 2` | 没有下降 | +| CCEC publish `store->GM` | `319 -> 110` | `512 -> 206` | 两次均明显下降 | +| AscendC publish `store->GM` | `526 -> 213` | `343 -> 121` | 两次均明显下降 | + +### 5.1 可以下的结论 + +**接口和真机事实：** + +- CCEC 三个 intrinsic 与 AscendC 三个公开 API 在当前 A5 编译栈均可编译、可运行； +- 两套实现发起 ICache preload 后立即观察到 busy，64-round gap 结束时均已 idle； +- baseline/preload 使用同一 gap，写 mode 在每次 launch 前恢复同一目标初值， + ICache 三种模式使用同一物理目标指令区； +- DCache 普通读值、写者本核普通回读、清理后的 bypass read、host D2H 和所有 + ICache checksum 都保持正确； +- AscendC 构建会同时产生不同 helper 变体，ELF 门禁检查的是本用例真正执行的 + `.vector` 符号。 + +**本探针内的性能观察：** + +- CCEC DCache load 为 `316 -> 5`，完整区间为 `1052 -> 742` raw ticks； +- AscendC DCache load 为 `295 -> 5`，完整区间为 `1147 -> 739` raw ticks； +- 最终 store-only 的 store 指令窗口两套均为 `2 -> 2`；完整区间分别为 + `741 -> 742`、`745 -> 749`，没有净收益； +- publish-to-GM 的 `store->GM` 在 CCEC 中为 `512 -> 206`、AscendC 中为 + `343 -> 121`；完整区间分别为 `1252 -> 942`、`1083 -> 863`； +- CCEC ICache work 为 `975 -> 484`，完整区间为 `1892 -> 1423` raw ticks； +- AscendC ICache work 为 `1085 -> 743`，完整区间为 `1912 -> 1602` raw ticks； +- wait 相对 async 没有降低 target work。两次 AscendC 完整区间差值分别为 + `+16`、`-6` raw ticks，方向反转；CCEC 分别为 `+53`、`+16`，只能判为小开销或波动， + 不能证明 wait 有收益。 + +两套单 word 实现的变化方向一致：提前 preload 对普通读和本探针的 publish-to-GM +序列有收益，但对单条 store-only 没有可见净收益。本轮数据也不支持 ICache +preload 后立即等待。这里不能替代 5.3 节对持续顺序写的独立结论。 + +publish-to-GM 的下降与“预先把 line 带入 DCache，随后普通 store 和 dirty-line clean +不再从 cold line 起步”这一解释相符，但探针没有直接观测内部 write-allocate/store +queue 状态，因此这里只能作为机制推测，不能写成已证硬件原因。 + +CCEC 与 AscendC 的绝对 tick 不应彼此直接做 API 成本归因。两者的 wrapper、kernel +入口和最终 `.text` 地址/大小不同，ICache 冷态本身也对代码布局敏感。这里的对等关系是 +测试模型、语义校验和 A/B 方向对等，不是要求两个二进制得到相同 tick。 + +### 5.2 不能下的结论 + +- 不能把任一 microprobe 的下降幅度外推成 PA Submit 的固定收益； +- 不能只用单 word publish-to-GM 的收益证明 store-only、bypass store、atomic、整 + line 写或多次连续写也会受益；持续顺序写的证据来自 5.3 节独立探针； +- 不能假设每次 preload 都会执行；官方明确允许 DCache hint 在拥塞时按 NOP 处理； +- 不能把 preload 当成数据一致性、发布、内存顺序或跨核同步； +- 不能把 raw tick 未经计数器频率校准直接换成时间； +- 不能把 CCEC/AscendC 绝对值之差只归因于 API wrapper； +- 不能只看 ICache miss 降低而忽略新增指令和 `.text` 布局变化。 + +### 5.3 持续顺序写的补充实验 + +运行环境和证据身份： + +| 项目 | 值 | +| --- | --- | +| 实验日期 | `2026-07-29` | +| 运行时 Git HEAD | `f0903f6f68743606e70bba983923aec4a476f3b6` | +| 分支 | `fdwic-swimlane-exclusive`，跟踪 `origin/fdwic-swimlane-deps` | +| CANN/CCEC | `9.1.0-weekly-20260708` / clang 15.0.5 | +| 最终 mixed kernel SHA256 | `b98c6c912d3533d19705e23e13c9b298a5915a62651f9ec02d19949a4d8a34c6` | +| topology | 32 个物理 block，`1 AIC : 2 AIV`，总计 96 worker | +| 主工作集 | 每核 120 KiB，即 3,840 records / 1,920 cachelines | +| 计时单位 | raw `SYS_CNT`；百分比不依赖频率换算 | +| 设备隔离 | 当前 shell 没有 `task-submit`/`npu-smi`；按已有用户授权在 device 0 未加锁直跑 | + +第一轮用 9 个交错样本扫描预取距离和发起密度。下表的 `critical` 是每次 launch +先取 96 核最大值、再跨样本取中位数；它不会把所有核的 duration 相加。 + +| 策略 | critical issue | critical total | 相对 baseline total | +| --- | ---: | ---: | ---: | +| baseline | 85,903 | 114,491 | 0 | +| `d1-c1` | 89,498 | 117,812 | +2.901% | +| `d2-c1` | 67,650 | 95,821 | -16.307% | +| `d4-c1` | 54,502 | 83,081 | -27.435% | +| `d8-c1` | 53,912 | 82,248 | -28.162% | +| `d16-c1` | 51,097 | 79,561 | **-30.509%** | +| `d4-c4` | 63,771 | 92,264 | -19.414% | +| `d8-c4` | 64,811 | 93,471 | -18.359% | +| `d16-c4` | 67,469 | 95,979 | -16.169% | + +`d1-c1` 说明“调用了 preload”不等于会加速：只提前一条 line 时，hint 成本已经进入 +关键路径，却没有留下足够的重叠窗口。`d16-c1` 是本次已测试集合中的最优点，不是 +所有代码布局、记录密度或硬件负载下的固定参数。 + +随后将 baseline 与 `d16-c1` 交错运行 21 个确认样本： + +| 96 核、每核 120 KiB | baseline | `d16-c1` | 变化 | +| --- | ---: | ---: | ---: | +| critical issue | 85,888 | 51,158 | **-40.436%** | +| 每核 issue 中位数 | 83,552 | 49,589 | -40.649% | +| 每核最终 flush 中位数 | 27,787 | 27,735 | -0.187% | +| critical total | 114,519 | 79,636 | **-30.460%** | +| 设备阶段完整 span | 115,171 | 80,254 | -30.318% | +| host launch + stream sync 中位数 | 132.268 us | 99.988 us | -24.405% | + +最终 flush 段基本不变，把收益边界定位在 ordinary store 发射阶段，而不是 +`DCCI(CACHELINE_OUT)+DSB`。这与“提前把未来 cold line 带入本核 DCache，持续写时 +减少 store-side 等待/反压”的机制解释一致；本探针没有采集 DCache miss 或 store +queue PMU，因此不能把该内部原因写成直接观测事实。 + +工作集和核类型对照同样保持相同方向： + +| 对照 | critical issue 变化 | critical total 变化 | +| --- | ---: | ---: | +| 96 核、每核 96 KiB，13 样本 | `69,141 -> 40,696`，-41.141% | `96,873 -> 68,251`，-29.546% | +| 单 AIC、每核 120 KiB，13 样本 | `84,695 -> 50,163`，-40.772% | `112,232 -> 77,630`，-30.831% | +| 单 AIV、每核 120 KiB，13 样本 | `85,329 -> 50,491`，-40.828% | `113,024 -> 78,114`，-30.887% | + +每个策略首个样本都全量回读有效 record 并校验七个字段；所有 launch 的 mixed +topology、结果 echo、GM publication、pointer cycle 和 cleanup 均为 PASS。 + +随机依赖容量扫描得到： + +| working set | AIC reuse ticks/load | AIV reuse ticks/load | +| --- | ---: | ---: | +| 4 KiB | 246.148 | 246.133 | +| 8 KiB | 246.103 | 246.100 | +| 12 KiB | 246.083 | 246.087 | +| 16 KiB | 246.076 | 246.078 | +| 20 KiB | 295.791 | 296.135 | +| 24 KiB | 328.494 | 328.778 | +| 32 KiB | 329.346 | 331.048 | +| 64 KiB | 328.449 | 329.654 | + +16 KiB 以内复用延迟稳定，20 KiB 开始退化，24 KiB 后已经接近本探针约 +`325-331 ticks/load` 的 cold traversal。因此本机 A5 的 AIC/AIV Scalar DCache 在该 +随机依赖访问模型下都呈现约 **16 KiB 有效容量拐点**。这是实测 effective resident +set，不等价于对所有地址映射、关联冲突和未来 SKU 声明精确架构容量。 + +这组实验修正了“纯写不能 preload”的过度结论： + +- 单条 ordinary store 没有收益，只代表其立即接受窗口； +- 大于 DCache 容量的持续、独占、顺序 cold-line 写可以从足够提前的 preload 获益； +- preload 不减少最终发布成本，也不能替代 DCCI/DSB； +- 当前约 30% 是隔离的紧凑连续写模型收益，不能直接写成 PA Submit 收益。真实 + TraceWriter 仍需以编译开关接入相同策略，并用同一业务输入做 level 4 端到端 A/B。 + +### 5.4 PA shared 路径的定向模型 + +#### 5.4.1 先从最新泳道确定真实对象 + +本节只参考用户指定的最新 shared 捕获： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + pa_scheduler_shared_swimlane_20260729_151323_2641728/ccec/ + merged_swimlane.json + swimlane_exclusive_analysis.json +``` + +该捕获是 schema-v5、96 核、shared TensorMap、1 GHz trace clock、Case1 +real-compute。全局 Submit makespan 为 `2,794.331 us`。下面的 Materialize、 +Register 等数值若标为 aggregate core-work，都是 96 核各自 duration 的求和， +不能与 `2,794.331 us` 墙钟直接相减。 + +| 最新泳道项目 | aggregate core-work | 每 task 均值 | 业务语义 | +| --- | ---: | ---: | --- | +| Materialize | 11,559,850 | 9,031.133 | 构造当前 task 参数、writer delta，并发布 fresh output | +| output publication | 6,194,703 | 4,839.612 | 预检、writer reserve、descriptor copy/flush、barrier、published | +| 其中 descriptor copy | 2,067,321 | 1,615.095 | `TensorDesc` 从 task payload 复制到独占 shared-output cell | +| 其中 descriptor flush | 572,004 | 446.878 | `FlushRegion`；不含后续全部 atomic/residual | +| output publication residual | 3,555,378 | 2,777.639 | 预检、FetchMax、StoreBarrier、published Exchange 等 | +| Register | 83,051,504 | 64,883.988 | 等前驱 insert 完成，再发布 writer metadata 并交棒 | +| 其中 predecessor wait | 80,687,004 | 63,036.722 | 串行 insert-turn 等待 | +| 其中 writer metadata | 1,868,256 | 1,459.575 | ordinary/symbol writer 元数据发布 | +| 其中 insert completion | 496,244 | 387.691 | 向后继发布本 task 已完成插入 | +| Winner Build | 7,381,717 | 5,766.966 | 组装执行 slot；其中包含 shared descriptor invalidate/copy | + +`merged_swimlane.json` 还能把物理粒度锁得更精确： + +| 对象 | 次数 | 大小/line | 同类事件均值 | +| --- | ---: | ---: | ---: | +| 零 output task | 256 | 0 | output publication `0.140930 us` | +| 一个 `TensorDesc` 的 output task | 512 | 128 B / 2 lines | copy `1.211871 us`；完整 publication `3.633336 us` | +| 三个 `TensorDesc` 的 output task | 512 | 384 B / 6 lines | copy `2.825551 us`；完整 publication `8.395229 us` | +| output descriptor clean-out | 512 + 512 | 2 lines / 6 lines | `0.340762 us` / `0.776129 us` | +| winner 读取 shared descriptor | 2,048 | 每次 128 B / 2 lines | invalidate overlay `0.032448 us`；copy 未单独打点 | +| 三 symbol writer history 发布 | 256 | 40 B / 1 line | clean-out overlay `0.204328 us` | +| writer history 读取 | 768 | 1 line | invalidate overlay `0.100422 us` | + +这里的零 output 数量是 `1,280 - 512 - 512` 的事件闭合结果。DCCI overlay +只包 DCCI/DSB 自身，不能拿 `0.032448 us` 解释后续 128 B 普通读取的总成本。 + +对应源码语义为： + +- `PublishSharedTaskOutputs()` 对 `shared_outputs[task_id]` 的独占 cell 做整批 + `CopyGmTensor()`，随后保留 `FlushRegion -> StoreBarrier -> published Exchange`； +- `PopulateSlotPayloadImpl()` 在观察到 producer 已发布后，对每个 + `SharedOutputRef` 执行 descriptor invalidate，再立即普通复制到 winner slot； +- 三 symbol UP writer 写本 task 独占的 40 B history，clean-out 后才用 + `last_writer` CAS 发布； +- `published`、`last_writer`、`deps_prepared` 和 insert turn 是 atomic + 控制线，不属于普通 DCache load/store 优化对象。 + +捕获 metadata 没有保存 kernel SHA256，旁边的 `build/` 也是可变目录，因此不能把 +某个后来重建的 ELF 冒充该捕获的精确二进制。分析期间相邻 shared swimlane ELF 的 +两次检查均显示 `.text` 大于 270 KiB，AIC/AIV orchestration 单函数约 +76～82 KiB；这只证明当前代码存在 ICache 压力的结构条件，不证明本次 +`2,794.331 us` 的瓶颈已经由 ICache miss 主导。 + +#### 5.4.2 探针如何保持与 shared 业务对等 + +新增 `shared_preload_model` 不 include PA 调度主流程，也没有改动 shared +生产代码。它只固定下面的物理模型： + +1. `publish`： + source 先在本核变为 hot，destination 保持 cold；可选对 destination 每条 + cacheline 发起 `dc_preload`，再逐 byte volatile copy，最后始终执行原有语义的 + `DCCI(CACHELINE_OUT) + DSB`； +2. `consume`： + destination 先变为 hot；计时窗内始终先对 source 执行 + `DCCI invalidate + DSB`，之后才可选 preload，再逐 byte复制；用于 host + 校验的 destination clean-out 放在计时窗外； +3. 大小只取当前业务真实出现的 40 B、128 B 和 384 B； +4. `gap_rounds=0` 不放额外独立业务，只保留相同函数/时钟括号； + `gap_rounds=64` 在 preload 与 copy 之间放运行时 noinline 标量工作，本次 + DCache 两轮中位数约 724～729 raw ticks； +5. 每种 DCache A/B 各 11 个交错样本、每个样本 96 worker；`critical` 先取每次 + launch 的最慢核，再跨样本取中位数； +6. 每次 launch 都恢复完整 source/destination，回读 96 核 payload 逐 byte + 校验；preload 不能参与正确性； +7. ICache 三种 mode 共用同一 4,644 B target；32,836 B evictor 先构造 cold + 状态。最终 ELF 门禁确认 172 B caller 的 forward 4 KiB preload 窗口不覆盖 + target；每种 mode 各 13 个交错样本、64 个 AIV。 + +其中 128/384 B output publish 与 128 B consume 都按当前 `CopyGmTensor()` 的 +GM-to-GM 逐 byte volatile copy 建模。40 B history case 只锁定“一个独占 +destination cacheline 被写后 clean-out”的物理问题；真实 history 是把 header +和三个 atomic 结果从标量寄存器写入 GM，不执行 40 B GM-to-GM copy。因此 history +百分比只能证明 destination preload 值得做业务 A/B，不能当成该 helper 的预计降幅。 + +publish 中 source-hot 用来模拟同一 winner 刚完成 payload materialization； +consume 中 destination-hot 用来隔离 shared source 的冷读。这两个 residency +条件是明确的测试假设，最新泳道本身没有 DCache residency PMU，不能声称每个真实 +task 都满足。后续业务 A/B 必须覆盖真实 slot 复用和核间调度状态。 + +最终探针身份如下： + +| 项目 | 值 | +| --- | --- | +| 日期 | `2026-07-29` | +| Git HEAD | `0f51a06fab5e6316f0cc8aa7dd5ae0c140140636` | +| CANN/CCEC | `9.1.0-weekly-20260708` / clang 15.0.5 | +| mixed kernel SHA256 | `947dce0e240316827d0718b6dbcd3451f7d120fc313c579b8ab38264b4b3cae9` | +| topology | DCache：32 AIC + 64 AIV；ICache：64 AIV | +| 计时 | raw `SYS_CNT`；下表百分比不依赖频率换算 | +| 设备隔离 | shell 无 `task-submit`/`npu-smi`；按已有用户授权在 device 0 未加锁运行 | +| 完整有效运行 | 两轮；payload、oracle、status、topology、cleanup 全部 PASS | + +#### 5.4.3 DCache 结果 + +下表给出第一轮代表性 raw 值，并用“复测变化”列给出第二轮关键核 total 的变化； +两轮都来自删除 timed-copy checksum 后的最终代码。 + +| shared 模型 | overlap | 第一轮 critical total baseline → preload | 第一轮变化 | 第二轮变化 | +| --- | --- | ---: | ---: | ---: | +| history-line write model，40 B | 无额外 gap | `784 -> 759` | -3.189% | -3.258% | +| history-line write model，40 B | 64-round gap | `1,490 -> 1,142` | -23.356% | -23.518% | +| 1-desc publish，128 B | 无额外 gap | `1,275 -> 1,153` | -9.569% | -8.830% | +| 1-desc publish，128 B | 64-round gap | `1,964 -> 1,542` | -21.487% | -22.150% | +| 3-desc publish，384 B | 无额外 gap | `3,496 -> 2,612` | -25.286% | -25.561% | +| 3-desc publish，384 B | 64-round gap | `4,209 -> 3,026` | -28.106% | -28.681% | +| 1-desc consume，128 B | 无额外 gap | `964 -> 841` | -12.759% | -12.369% | +| 1-desc consume，128 B | 64-round gap | `1,666 -> 1,224` | -26.531% | -26.221% | + +当前生产 consume 是逐个 128 B descriptor invalidate/copy，所以表中的 128 B +是直接对等项。探针也测了 384 B 批量 consume：无 gap 两轮 critical total +分别下降 `31.699%`、`32.816%`，有 gap 分别下降 `35.014%`、`35.196%`； +它只用于评估未来“先处理多个引用、再批量 copy”的可能性，不是当前源码已有动作。 + +三条可以直接成立的观察： + +- output/history destination 都是当前 task 独占，预取不会引入同地址 writer + 竞争；多 line copy 即使没有合成 gap，后续 line 也能获得前面 copy 提供的自然 + lead，因此 384 B 比 40 B 的无-gap 收益稳定得多； +- preload 降低的是 ordinary copy/写入部分。128 B gap 模型的 publish 中位数 + 两轮分别保持 `256 -> 256`、`257 -> 257` raw ticks；384 B 同样基本不变， + 没有证据表明 DCCI/DSB 被加速； +- consume 的 preload 必须位于既有 invalidate **之后**。在 invalidate 之前 + preload 随后会被失效，且无论放在哪里都不能替代 producer publication、 + DCCI、DSB 或 atomic ready 检查。 + +因此 DCache 有真实的 shared 接入候选，但优先级不同： + +| 候选位置 | 可行性 | 原因与边界 | +| --- | --- | --- | +| Materialize 的 fresh output destination | 高 | 512 个 128 B 和 512 个 384 B 任务直接命中模型；cell 独占，地址和 output count 已知，保留原 flush/barrier/published | +| Register 的 40 B writer-history destination | 中 | cell 独占，可在三组 published/last-writer atomic 检查前发 hint；但 Register 的 64.884 us/task 主要是 63.037 us predecessor wait，不能把 history microprobe 当成 Register 总收益 | +| Winner Build 的 128 B descriptor source | 中 | 无-gap 模型已有稳定下降；更长 lead 需要把当前“invalidate 后立即 copy”改成 prepass/分批处理，必须另做语义和端到端验证 | +| fanin 的 writer-history source | 低 | 单 line 且只在 future-writer 慢路使用；最新图中 invalidate overlay 很小，普通 scan 又没有单独边界 | +| `published`/`last_writer`/insert turn 等 atomic line | 不建议 | 当前访问是 atomic/bypass 同步协议，不是普通 DCache cold load；preload 不能提供新鲜度或顺序 | + +#### 5.4.4 ICache 结果 + +| placement | 第一轮 critical work | 第一轮 critical total | 第二轮 critical total | 结论 | +| --- | ---: | ---: | ---: | --- | +| 远端 caller current-PC | `1,559 -> 1,493`，-4.233% | `2,520 -> 2,487`，-1.310% | `2,559 -> 2,499`，-2.345% | 目标不在 preload 窗口，差异只能当布局/波动，不能宣称有效预取 | +| target 内 current-PC | `1,559 -> 421`，-72.996% | `2,520 -> 1,485`，-41.071% | `2,559 -> 1,510`，-40.993% | 对同一 cold 顺序目标区有稳定收益 | + +两轮 caller/target preload 都是发起后 `832/832 immediate busy`、独立 gap 后 +`0/832 final busy`。这说明“hint 完成”与“hint 覆盖了即将执行的目标”是两件事： +caller 的请求也完整结束，但由于最终 ELF 已证明其 forward 窗口不含 target， +没有得到 target 内放置的收益。 + +对 shared 主流程只能得出条件性建议： + +- 当前超大 orchestration `.text` 使 ICache 优化值得继续查，但最新泳道没有 PMU， + 还不能断言 Materialize/Register 的长时间就是 ICache miss； +- `icache_preload(2)` 必须放在最终 linked ELF 中即将顺序执行的目标块内部或紧邻 + 前方。源码上“调用关系接近”不够，跨 noinline helper、冷失败块和分支重排都可能 + 让 current-PC 窗口指错位置； +- 不应在热路径新增 status 轮询。已有基础探针和本模型都没有证明等待 hint 完成 + 能优于让真实独立工作与其重叠； +- 后续 session 若试接入，必须保存 baseline/preload 两个最终 ELF 的目标符号地址、 + `.text` 大小与反汇编边界，并用 submit-PMU 或等价配对证据确认 miss 下降，再看 + trace-free Submit 墙钟是否下降。 + +#### 5.4.5 本轮不能外推的内容 + +- 不能把上述 `-9%`、`-28%` 或 `-41%` 直接乘到 PA Submit；microprobe 每核从 + cold 状态启动，而真实 task 在不同核、不同时间交错，cache residency 不同； +- 不能把 6,194,703 output-publication core-work 减去某个 probe 百分比后，称为 + `2,794.331 us` wall-clock 的预计收益；前者是各核求和，后者是跨核 envelope； +- 不能把 Register metadata 的改善等同于去掉 predecessor serialization。最多只可能 + 让当前 task 更早交棒，真实链式收益必须由完整泳道重测； +- 不能用 ICache target microprobe 证明当前 shared 已经发生同等 miss，也不能只看 + `.text` 大就决定加 hint； +- 不能删、移动或弱化任何 DCCI、DSB、StoreBarrier、published、last_writer、 + deps_prepared 或 insert-turn 协议来换取 microbenchmark 数字。 + +## 6. Preload 不能替代什么 + +| 业务要求 | 应使用的机制 | Preload 不满足的原因 | +| --- | --- | --- | +| 让其他核看见最新 GM 写入 | 已定义的 DCCI/发布协议 | preload 不 clean 写回，也不建立 happens-before | +| 等待前序流水完成 | pipe flag、barrier 或既有同步协议 | preload 不是完成屏障 | +| 跨核启动/结束汇聚 | `SyncAll` 或 atomic barrier | preload 没有参与者和 release 语义 | +| 保证目标一定进 cache | 普通 load/取指的正确性路径 | preload 是可丢弃的性能 hint | +| 修复错误的代码布局 | `.text`、分支与热点工作集优化 | preload 只能覆盖有限地址窗口 | + +尤其不能把 `dc_preload` 混入 tensormap/symbol 发布协议来替代 DCCI，也不能由 +`get_icache_prl_st()==0` 或 `GetICachePreloadStatus()==0` 推导任何 GM 数据已经跨核可见。 + +## 7. 接入业务的验收标准 + +只在同时满足以下条件时建议保留业务改动： + +1. PMU、泳道或稳定 A/B 已确认目标区间存在 ICache miss 或 GM Scalar 冷读； +2. DCache 地址能提前确定，或 ICache 调用点确实位于即将执行的顺序热点代码前； +3. 发起后存在真实独立工作，不是 preload 后立即 load/立即轮询； +4. 删除 preload 后业务仍完全正确； +5. 同输入、同环境、同构建口径下，Submit/算子端到端性能稳定改善； +6. 同时检查 Scalar busy、I-cache miss、`.text` 大小/布局和其他阶段是否回退； +7. 默认保留 raw `SYS_CNT`，只有频率已经校准时才换算时间； +8. 若收益处于正常波动，或 miss 降低但端到端时间不降，则删除该 hint。 + +写路径还必须先确认真实关键路径： + +- 只要求 ordinary store 被本核接受时，应看 `dstore-only-*` 的 `access/work` 和 + `total`；不能拿 publish 数据替它证明收益； +- 要求 dirty line 在本次发布到 GM 时，应看 `dpublish-gm-*` 的 `store->GM` 和 + `total`，且业务仍必须保留既有 DCCI/DSB； +- 单次 store 探针不覆盖连续写把 store queue 压满后的吞吐，也不覆盖 bypass store、 + atomic 或多核同 line 写；连续、独占、顺序 cacheline 写应使用本目录的 + `trace_write_preload` 探针，但它仍不覆盖跨核同 line 竞争。 + +ICache A/B 要特别检查 `.text` 扰动。新增 preload、状态读取或诊断打点都会改变指令数和分支地址； +不能用还包含其他代码差异的两个 ELF，把全部变化归因给 preload。 + +## 8. 查证来源 + +本次本地 CANN 9.1 weekly 查证路径如下。`impl/` 是内部实现，只用于核对当前版本下沉关系，业务代码 +不应直接 include： + +```text +tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_aicore_functions.h +tools/bisheng_compiler/lib/clang/15.0.5/include/cce_aicore_intrinsics.h +compiler/asc/impl/basic_api/dav_3510/kernel_operator_cache_impl.h +compiler/asc/impl/basic_api/dav_3510/kernel_operator_list_tensor_impl.h +x86_64-linux/asc/include/c_api/cache_ctrl/cache_ctrl.h +opp/built-in/op_impl/ai_core/tbe/impl/ops_transformer/ascendc/flash_attn/flash_attn.cpp +opp/built-in/op_impl/ai_core/tbe/impl/ops_nn/ascendc/mat_mul_v3/arch35/ +``` + +其中 CANN OPP 的 `flash_attn.cpp` 已有 raw `dc_preload` 的生产实现示例； +`mat_mul_v3/arch35/` 已有 AscendC `ICachePreLoad(2)` 的存量用例；Bisheng 头文件则 +直接给出 `icache_preload(len) -> preload(get_pc(), len)` 的 CCEC 关系。 diff --git a/tests/atomic_probe/ccec/atomic_scalar_pmu.cpp b/tests/atomic_probe/ccec/atomic_scalar_pmu.cpp new file mode 100644 index 0000000000..7214e86cb7 --- /dev/null +++ b/tests/atomic_probe/ccec/atomic_scalar_pmu.cpp @@ -0,0 +1,179 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +// 测试目标：用单个 AIV、无竞争、无轮询的固定次数窗口，精确区分 +// atomicAdd 等待返回的时间是否计入 PMU scalar busy。Host/AICPU 在 launch 前已把 +// PMU slot0/1/2 配置为 scalar busy(0x1)、I-cache request(0x34)、I-cache miss(0x35)。 +// +// 每次 kernel 的完整时序： +// 1. 先关闭 PMU，再对 host 写入的 control 单独 cache line 做 DCCI + DSB，避免复用 +// ProbeState 时沿用上一次 kernel 的旧 control。Atomic target 不做普通 load/store +// 或 DCCI，始终保持为一条独占 cache line 的 raw atomic 目标。 +// 2. 按 physical core id 从 host 传入的寄存器基址表取本 AIV PMU base，然后用 +// read-to-clear 清空所有 counter。 +// 3. 同一个 get_sys_cnt 时间窗内执行 metrics_prof_start/stop；三种 mode 只替换 +// gate 内部的固定 rounds 工作负载： +// EMPTY：不做工作，测 gate 与计时固有开销； +// SCALAR_CONTROL：只在 scalar 寄存器中执行与 atomic 相同的数据依赖递推； +// DEPENDENT_ATOMIC_ADD：对独占 target 执行 atomicAdd，下一轮 addend 由上一轮 +// atomicAdd 的返回值计算，不允许多条 atomic 并行隐藏单条等待。 +// 4. 关闭 PMU 后才读 total/scalar/request/miss，最后仅用 st_dev 把结果发布到 +// result 独占 cache line，并用 DSB 收口。 +// +// CONTROL 与 ATOMIC 共用可由 host 精确复算的递推（uint64_t 模 2^64）： +// value=seed, delta=1, checksum=0; +// 每轮 old=value/atomicAdd(target, delta) 的返回值，checksum+=old， +// delta=1+(old&1)；CONTROL 另执行 value+=本轮 delta。 +// 解读时应对多个 rounds 取斜率并扣除 EMPTY/CONTROL：若 atomic 的 total 斜率显著 +// 增长而 scalar 斜率不同比例增长，atomic 等待不属于 scalar busy；若两者同比例增长， +// 则等待被计入 scalar busy。 + +#include "atomic_scalar_pmu_shared.h" +#include "ccec_utils.h" + +CCEC_PROBE_KERNEL_META(atomic_scalar_pmu); + +namespace { + +constexpr uint32_t kPmuPhysicalSubcores = 108; +constexpr uint64_t kPmuCtrl0Offset = 0x4200ULL; + +__aicore__ __attribute__((always_inline)) inline int32_t *PmuCounterBase(uint64_t register_base) +{ + // 以 PMU_CTRL_0(0x4200) 为基准后，所有 counter 都落在 ld_dev 的 12-bit immediate 范围内。 + return reinterpret_cast(register_base + kPmuCtrl0Offset); +} + +__aicore__ __attribute__((always_inline)) inline void ClearPmuCounters(uint64_t register_base) +{ + int32_t *base = PmuCounterBase(register_base); + // A5 PMU counter 是 read-to-clear；显式展开保证每个 ld_dev offset 为编译期常量。 + (void)ld_dev(base, 0x10); + (void)ld_dev(base, 0x18); + (void)ld_dev(base, 0x20); + (void)ld_dev(base, 0x28); + (void)ld_dev(base, 0x30); + (void)ld_dev(base, 0x38); + (void)ld_dev(base, 0x40); + (void)ld_dev(base, 0x48); + (void)ld_dev(base, 0x50); + (void)ld_dev(base, 0x54); + (void)ld_dev(base, 0x60); + (void)ld_dev(base, 0x64); +} + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadPmuScalar(uint64_t register_base) +{ + return static_cast(ld_dev(PmuCounterBase(register_base), 0x10)); +} + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadPmuIcacheRequest(uint64_t register_base) +{ + return static_cast(ld_dev(PmuCounterBase(register_base), 0x18)); +} + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadPmuIcacheMiss(uint64_t register_base) +{ + return static_cast(ld_dev(PmuCounterBase(register_base), 0x20)); +} + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadPmuTotal(uint64_t register_base) +{ + int32_t *base = PmuCounterBase(register_base); + const uint64_t low = static_cast(ld_dev(base, 0x60)); + const uint64_t high = static_cast(ld_dev(base, 0x64)); + return low | (high << 32); +} + +__aicore__ __attribute__((always_inline)) inline void Publish64(__gm__ uint64_t *address, uint64_t value) +{ + __builtin_cce_st_dev(value, address, 0); +} + +} // namespace + +extern "C" __global__ __aicore__ void KERNEL_ENTRY(atomic_scalar_pmu)( + __gm__ atomic_scalar_pmu::ProbeState *state) +{ + using atomic_scalar_pmu::Mode; + + // task-based profiler 可能在入口前已打开 PMU；先关闭，确保 control DCCI 和准备阶段不入窗。 + bisheng::cce::metrics_prof_stop(); + dcci(&state->control, SINGLE_CACHE_LINE); + dsb(DSB_ALL); + + const uint32_t mode_value = state->control.mode; + const uint32_t rounds = state->control.rounds; + const uint64_t seed = state->control.seed; + const uint32_t physical_core_id = static_cast(get_coreid()) & 0x0fffU; + + uint64_t register_base = 0; + if (state->control.pmu_register_bases != 0 && physical_core_id < kPmuPhysicalSubcores) { + __gm__ uint64_t *register_bases = + reinterpret_cast<__gm__ uint64_t *>(state->control.pmu_register_bases); + register_base = register_bases[physical_core_id]; + } + if (register_base != 0) { + ClearPmuCounters(register_base); + } + + uint64_t checksum = 0; + uint64_t delta = 1; + uint64_t scalar_value = seed; + + const uint64_t sys_begin = static_cast(get_sys_cnt()); + bisheng::cce::metrics_prof_start(); + + if (mode_value == static_cast(Mode::ScalarControl)) { + for (uint32_t round = 0; round < rounds; ++round) { + const uint64_t old = scalar_value; + scalar_value += delta; + checksum += old; + delta = 1 + (old & 1U); + } + } else if (mode_value == static_cast(Mode::DependentAtomicAdd)) { + __gm__ uint64_t *target = const_cast<__gm__ uint64_t *>(&state->target.value); + for (uint32_t round = 0; round < rounds; ++round) { + // delta 直接依赖上一轮 old；除第一轮外，后一条 atomic 必须等前一条返回。 + const uint64_t old = atomicAdd(target, delta); + checksum += old; + delta = 1 + (old & 1U); + } + } + // Empty 和非法 mode 都保持空窗；host 仅会发布 enum 中的三个合法值。 + + bisheng::cce::metrics_prof_stop(); + const uint64_t sys_end = static_cast(get_sys_cnt()); + const uint64_t ctrl_after_stop = static_cast(get_ctrl()); + + uint64_t pmu_total = 0; + uint64_t pmu_scalar = 0; + uint64_t pmu_icache_request = 0; + uint64_t pmu_icache_miss = 0; + if (register_base != 0) { + // counter 为 read-to-clear，每项只读一次，且必须在 stop 后执行。 + pmu_scalar = ReadPmuScalar(register_base); + pmu_icache_request = ReadPmuIcacheRequest(register_base); + pmu_icache_miss = ReadPmuIcacheMiss(register_base); + pmu_total = ReadPmuTotal(register_base); + } + + __gm__ atomic_scalar_pmu::ProbeResult *result = &state->result; + Publish64(&result->sys_cycles, sys_end - sys_begin); + Publish64(&result->pmu_total_cycles, pmu_total); + Publish64(&result->pmu_scalar_busy, pmu_scalar); + Publish64(&result->pmu_icache_request, pmu_icache_request); + Publish64(&result->pmu_icache_miss, pmu_icache_miss); + Publish64(&result->checksum, checksum); + Publish64(&result->pmu_ctrl_after_stop, ctrl_after_stop); + Publish64(&result->physical_core_id, physical_core_id); + dsb(DSB_ALL); +} diff --git a/tests/atomic_probe/ccec/atomic_scalar_pmu_host.cpp b/tests/atomic_probe/ccec/atomic_scalar_pmu_host.cpp new file mode 100644 index 0000000000..8c44ac4b66 --- /dev/null +++ b/tests/atomic_probe/ccec/atomic_scalar_pmu_host.cpp @@ -0,0 +1,654 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +// 单 AIV、无轮询、固定操作数的 atomic/标量 PMU 对照 host。 +// 每个 rounds 都重复 EMPTY -> SCALAR_CONTROL -> DEPENDENT_ATOMIC_ADD： +// 1. EMPTY 给出同位置 PMU gate/read 的固定成本； +// 2. SCALAR_CONTROL 执行与 atomic 路径完全相同的返回值递推和 checksum； +// 3. DEPENDENT_ATOMIC_ADD 让下一条 atomicAdd 的 addend 依赖上一条返回值。 +// 因而 (ATOMIC-CONTROL)/rounds 直接回答 atomic 等待周期落在 PMU total、scalar busy +// 中的哪一项，而不混入多核竞争、轮询次数变化或未消费返回值的并行发射。 + +#include "atomic_scalar_pmu_shared.h" +#include "pmu_probe_host_support.h" +#include "../probe_host.h" + +#include "aicpu_loader/host/load_aicpu_op.h" +#include "common/kernel_args.h" +#include "driver/ascend_hal.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr uint32_t kPhysicalAicoreCount = 36; +constexpr uint32_t kSubcoresPerAicore = 3; +constexpr uint32_t kPhysicalSubcoreCount = kPhysicalAicoreCount * kSubcoresPerAicore; +constexpr uint32_t kAicorePerDie = 18; +constexpr uint32_t kSubcoresPerDie = kAicorePerDie * kSubcoresPerAicore; +constexpr uint32_t kAivBaseInDie = kAicorePerDie; +constexpr uint64_t kSubcoreStride = 0x100000ULL; +constexpr uint32_t kAicoreMapBytes = 0x300000U; + +static_assert( + kPhysicalSubcoreCount == atomic_probe::pmu::kPmuPhysicalSubcores, "PMU table size mismatch" +); + +bool CheckAcl(aclError error, const char *label) { + if (error == ACL_SUCCESS) return true; + std::fprintf(stderr, "ACL error %d: %s\n", static_cast(error), label); + return false; +} + +std::vector ReadBinary(const std::string &path) { + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) return {}; + const std::streamsize size = file.tellg(); + if (size <= 0) return {}; + std::vector bytes(static_cast(size)); + file.seekg(0, std::ios::beg); + if (!file.read(bytes.data(), size)) return {}; + return bytes; +} + +std::string ArtifactBesideKernel(const std::string &kernel_path, const char *name) { + const size_t slash = kernel_path.find_last_of('/'); + return slash == std::string::npos ? std::string(name) : kernel_path.substr(0, slash + 1) + name; +} + +class PmuResources { +public: + using MapFn = drvError_t (*)(unsigned int, struct res_map_info *, unsigned long *, unsigned int *); + using UnmapFn = drvError_t (*)(unsigned int, struct res_map_info *); + + ~PmuResources() { RestoreAndUnmap(); } + + bool Initialize(uint32_t device) { + device_ = device; + map_ = reinterpret_cast(dlsym(RTLD_DEFAULT, "halResMap")); + unmap_ = reinterpret_cast(dlsym(RTLD_DEFAULT, "halResUnmap")); + if (map_ == nullptr || unmap_ == nullptr) { + hal_handle_ = dlopen("libascend_hal.so", RTLD_NOW | RTLD_GLOBAL); + if (hal_handle_ != nullptr) { + map_ = reinterpret_cast(dlsym(hal_handle_, "halResMap")); + unmap_ = reinterpret_cast(dlsym(hal_handle_, "halResUnmap")); + } + } + if (map_ == nullptr || unmap_ == nullptr) { + std::fprintf(stderr, "Cannot resolve halResMap/halResUnmap.\n"); + return false; + } + + for (uint32_t physical = 0; physical < kPhysicalAicoreCount; ++physical) { + res_map_info &info = map_info_[physical]; + std::memset(&info, 0, sizeof(info)); + info.target_proc_type = PROCESS_CP1; + info.res_type = RES_AICORE; + info.res_id = physical; + unsigned long map_address = 0; + unsigned int map_length = kAicoreMapBytes; + const drvError_t error = map_(device_, &info, &map_address, &map_length); + if (error != 0 || map_address == 0 || map_length < kAicoreMapBytes) { + std::fprintf( + stderr, "halResMap failed: physical=%u error=%d address=0x%lx length=%u\n", physical, + static_cast(error), map_address, map_length + ); + Unmap(); + return false; + } + ++mapped_count_; + const uint64_t base = static_cast(map_address); + const uint32_t die = physical / kAicorePerDie; + const uint32_t local = physical % kAicorePerDie; + const uint32_t die_base = die * kSubcoresPerDie; + register_bases_[die_base + local] = base; + const uint32_t aiv0 = die_base + kAivBaseInDie + local * 2; + register_bases_[aiv0] = base + kSubcoreStride; + register_bases_[aiv0 + 1] = base + 2 * kSubcoreStride; + } + return true; + } + + void RestoreAndUnmap() { + Unmap(); + if (hal_handle_ != nullptr) { + dlclose(hal_handle_); + hal_handle_ = nullptr; + } + } + + const std::array &RegisterBases() const { return register_bases_; } + +private: + void Unmap() { + while (mapped_count_ != 0) { + --mapped_count_; + const drvError_t error = unmap_(device_, &map_info_[mapped_count_]); + if (error != 0) { + std::fprintf( + stderr, "halResUnmap failed: physical=%u error=%d\n", mapped_count_, + static_cast(error) + ); + } + } + } + + uint32_t device_ = 0; + uint32_t mapped_count_ = 0; + void *hal_handle_ = nullptr; + MapFn map_ = nullptr; + UnmapFn unmap_ = nullptr; + std::array map_info_{}; + std::array register_bases_{}; +}; + +bool RunPmuCommand( + host::LoadAicpuOp &loader, aclrtStream stream, KernelArgs *kernel_args, void *control_device, + atomic_probe::pmu::PmuControl *control, atomic_probe::pmu::PmuCommand command +) { + control->command = static_cast(command); + control->status = atomic_probe::pmu::kPmuStatusPending; + kernel_args->enable_profiling_flag = static_cast(command); + const int launch_error = loader.LaunchBuiltInOp(stream, kernel_args, 1, host::KernelNames::RunName); + if (launch_error != 0) { + std::fprintf(stderr, "AICPU PMU helper launch failed: %d\n", launch_error); + return false; + } + if (!CheckAcl(aclrtSynchronizeStream(stream), "aclrtSynchronizeStream(PMU helper)") || + !CheckAcl( + aclrtMemcpy(control, sizeof(*control), control_device, sizeof(*control), ACL_MEMCPY_DEVICE_TO_HOST), + "aclrtMemcpy(D2H PMU control)" + )) { + return false; + } + const uint32_t bitmap_count = atomic_probe::pmu::CountPmuConfiguredSubcores(*control); + const bool expected_state = command == atomic_probe::pmu::PmuCommand::Configure + ? control->configured == 1 && control->processed_subcores == control->expected_subcores && + bitmap_count == control->expected_subcores && + control->skipped_subcores + bitmap_count == atomic_probe::pmu::kPmuPhysicalSubcores + : control->configured == 0 && control->processed_subcores == 0 && bitmap_count == 0; + if (control->status != 0 || !expected_state) { + const auto failed_field = + static_cast(control->first_failed_field); + std::fprintf( + stderr, + "PMU helper failed: command=%u status=%d configured=%u processed=%u bitmap_count=%u " + "expected_subcores=%u skipped=%u failed_index=%u failed_field=%s(%u) " + "observed=0x%x expected=0x%x\n", + control->command, static_cast(control->status), control->configured, + control->processed_subcores, bitmap_count, control->expected_subcores, + control->skipped_subcores, control->first_failed_index, + atomic_probe::pmu::PmuRegisterFieldName(failed_field), control->first_failed_field, + control->first_failed_observed, control->first_failed_expected + ); + return false; + } + if (command == atomic_probe::pmu::PmuCommand::Configure) { + const auto failed_field = + static_cast(control->first_failed_field); + std::printf( + "[PMU_OWNER] configured=%u bitmap_count=%u expected=%u skipped=%u " + "first_skipped_index=%u first_skipped_field=%s(%u) observed=0x%x expected_value=0x%x\n", + control->processed_subcores, bitmap_count, control->expected_subcores, + control->skipped_subcores, control->first_failed_index, + atomic_probe::pmu::PmuRegisterFieldName(failed_field), control->first_failed_field, + control->first_failed_observed, control->first_failed_expected + ); + std::printf( + "[PMU_OWNER] configured_bitmap=%08x:%08x:%08x:%08x\n", + control->configured_bitmap[3], control->configured_bitmap[2], + control->configured_bitmap[1], control->configured_bitmap[0] + ); + } + return true; +} + +const char *ModeName(atomic_scalar_pmu::Mode mode) { + switch (mode) { + case atomic_scalar_pmu::Mode::Empty: return "EMPTY"; + case atomic_scalar_pmu::Mode::ScalarControl: return "SCALAR_CONTROL"; + case atomic_scalar_pmu::Mode::DependentAtomicAdd: return "DEPENDENT_ATOMIC_ADD"; + default: return "UNKNOWN"; + } +} + +bool ParseUint64(const char *text, uint64_t maximum, uint64_t *value) { + if (text == nullptr || text[0] == '\0') return false; + errno = 0; + char *end = nullptr; + const unsigned long long parsed = std::strtoull(text, &end, 0); + if (errno != 0 || end == text || *end != '\0' || parsed > maximum) return false; + *value = static_cast(parsed); + return true; +} + +uint32_t RepeatsFromEnv() { + const char *raw = std::getenv("ATOMIC_SCALAR_PMU_REPEATS"); + if (raw == nullptr || raw[0] == '\0') return 7; + uint64_t value = 0; + if (!ParseUint64(raw, 100, &value) || value == 0) { + std::fprintf(stderr, "ATOMIC_SCALAR_PMU_REPEATS must be in 1..100: %s\n", raw); + return 0; + } + return static_cast(value); +} + +uint64_t SeedFromEnv(bool *ok) { + const char *raw = std::getenv("ATOMIC_SCALAR_PMU_SEED"); + if (raw == nullptr || raw[0] == '\0') return 0x1234ULL; + uint64_t value = 0; + const bool parsed = ParseUint64(raw, std::numeric_limits::max(), &value); + *ok &= parsed; + if (!parsed) std::fprintf(stderr, "Invalid ATOMIC_SCALAR_PMU_SEED: %s\n", raw); + return value; +} + +std::vector RoundsFromEnv(bool *ok) { + const char *raw = std::getenv("ATOMIC_SCALAR_PMU_ROUNDS"); + if (raw == nullptr || raw[0] == '\0') return {0, 1, 4, 16, 64, 256, 1024, 4096, 8192}; + std::vector rounds; + const std::string input(raw); + size_t begin = 0; + while (begin <= input.size()) { + const size_t comma = input.find(',', begin); + const std::string token = input.substr(begin, comma == std::string::npos ? comma : comma - begin); + uint64_t value = 0; + if (!ParseUint64(token.c_str(), 1000000, &value)) { + std::fprintf(stderr, "Invalid ATOMIC_SCALAR_PMU_ROUNDS item: %s\n", token.c_str()); + *ok = false; + return {}; + } + rounds.push_back(static_cast(value)); + if (comma == std::string::npos) break; + begin = comma + 1; + } + return rounds; +} + +struct Oracle { + uint64_t final_value; + uint64_t checksum; +}; + +Oracle Simulate(uint64_t seed, uint32_t rounds) { + uint64_t value = seed; + uint64_t delta = 1; + uint64_t checksum = 0; + for (uint32_t round = 0; round < rounds; ++round) { + const uint64_t old = value; + value += delta; + checksum += old; + delta = 1 + (old & 1ULL); + } + return {value, checksum}; +} + +struct Sample { + atomic_scalar_pmu::ProbeResult result{}; + uint64_t final_value = 0; +}; + +bool ValidateSample( + const Sample &sample, atomic_scalar_pmu::Mode mode, uint32_t rounds, uint64_t seed, + const atomic_probe::pmu::PmuControl &pmu_control, std::string *reason +) { + const Oracle oracle = Simulate(seed, rounds); + const uint64_t expected_checksum = mode == atomic_scalar_pmu::Mode::Empty ? 0 : oracle.checksum; + const uint64_t expected_final = mode == atomic_scalar_pmu::Mode::DependentAtomicAdd ? oracle.final_value : seed; + if (sample.result.checksum != expected_checksum) { + *reason = "checksum"; + return false; + } + if (sample.final_value != expected_final) { + *reason = "target-final"; + return false; + } + if (sample.result.physical_core_id >= kPhysicalSubcoreCount) { + *reason = "physical-core-id"; + return false; + } + if (!atomic_probe::pmu::IsPmuSubcoreConfigured( + pmu_control, static_cast(sample.result.physical_core_id) + )) { + *reason = "physical-core-not-in-configured-bitmap"; + return false; + } + if ((sample.result.pmu_ctrl_after_stop & 1ULL) != 0) { + *reason = "pmu-gate-still-enabled"; + return false; + } + if (sample.result.sys_cycles == 0 || sample.result.pmu_total_cycles == 0) { + *reason = "zero-cycle-window"; + return false; + } + if (sample.result.pmu_icache_miss > sample.result.pmu_icache_request) { + *reason = "icache-miss-exceeds-request"; + return false; + } + return true; +} + +bool RunOne( + aclrtFuncHandle function, aclrtStream stream, void *state_device, uint64_t pmu_register_bases, + atomic_scalar_pmu::Mode mode, uint32_t rounds, uint32_t repeat, uint64_t seed, + const atomic_probe::pmu::PmuControl &pmu_control, Sample *sample +) { + atomic_scalar_pmu::ProbeState state{}; + state.control.pmu_register_bases = pmu_register_bases; + state.control.mode = static_cast(mode); + state.control.rounds = rounds; + state.control.seed = seed; + state.target.value = seed; + if (!CheckAcl( + aclrtMemcpy( + state_device, sizeof(state), &state, sizeof(state), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D probe state)" + )) { + return false; + } + + struct KernelArgs { + uint64_t state_pointer; + } args{reinterpret_cast(state_device)}; + static_assert(sizeof(KernelArgs) == sizeof(uint64_t), "unexpected CCEC kernel argument ABI"); + if (!CheckAcl( + aclrtLaunchKernelWithHostArgs(function, 1, stream, nullptr, &args, sizeof(args), nullptr, 0), + "aclrtLaunchKernelWithHostArgs" + ) || + !CheckAcl(aclrtSynchronizeStream(stream), "aclrtSynchronizeStream(AIV probe)") || + !CheckAcl( + aclrtMemcpy(&state, sizeof(state), state_device, sizeof(state), ACL_MEMCPY_DEVICE_TO_HOST), + "aclrtMemcpy(D2H probe state)" + )) { + return false; + } + + sample->result = state.result; + sample->final_value = state.target.value; + std::string reason; + const bool semantic_ok = ValidateSample(*sample, mode, rounds, seed, pmu_control, &reason); + std::printf( + "[RAW] repeat=%u rounds=%u mode=%s sys_cycles=%llu total=%llu scalar=%llu " + "icache_req=%llu icache_miss=%llu checksum=%llu final=%llu physical=%llu ctrl=0x%llx status=%s%s%s\n", + repeat, rounds, ModeName(mode), static_cast(sample->result.sys_cycles), + static_cast(sample->result.pmu_total_cycles), + static_cast(sample->result.pmu_scalar_busy), + static_cast(sample->result.pmu_icache_request), + static_cast(sample->result.pmu_icache_miss), + static_cast(sample->result.checksum), + static_cast(sample->final_value), + static_cast(sample->result.physical_core_id), + static_cast(sample->result.pmu_ctrl_after_stop), semantic_ok ? "PASS" : "FAIL", + semantic_ok ? "" : " reason=", semantic_ok ? "" : reason.c_str() + ); + return semantic_ok; +} + +uint64_t Median(std::vector values) { + std::sort(values.begin(), values.end()); + const size_t middle = values.size() / 2; + if ((values.size() & 1U) != 0) return values[middle]; + return values[middle - 1] + (values[middle] - values[middle - 1]) / 2; +} + +double Median(std::vector values) { + std::sort(values.begin(), values.end()); + const size_t middle = values.size() / 2; + return (values.size() & 1U) != 0 ? values[middle] : (values[middle - 1] + values[middle]) / 2.0; +} + +using CounterMember = uint64_t atomic_scalar_pmu::ProbeResult::*; + +double PairedDeltaPerOperation( + const std::vector &minuend, const std::vector &subtrahend, + CounterMember member, uint32_t rounds +) { + std::vector deltas; + for (size_t index = 0; index < minuend.size(); ++index) { + deltas.push_back( + (static_cast(minuend[index].result.*member) - + static_cast(subtrahend[index].result.*member)) / + rounds + ); + } + return Median(std::move(deltas)); +} + +void PrintRoundSummary( + uint32_t rounds, const std::array, 3> &samples +) { + struct Metric { + const char *name; + CounterMember member; + }; + constexpr Metric metrics[] = { + {"sys_cycles", &atomic_scalar_pmu::ProbeResult::sys_cycles}, + {"total", &atomic_scalar_pmu::ProbeResult::pmu_total_cycles}, + {"scalar", &atomic_scalar_pmu::ProbeResult::pmu_scalar_busy}, + {"icache_req", &atomic_scalar_pmu::ProbeResult::pmu_icache_request}, + {"icache_miss", &atomic_scalar_pmu::ProbeResult::pmu_icache_miss}, + }; + for (uint32_t mode_index = 0; mode_index < static_cast(atomic_scalar_pmu::Mode::Count); + ++mode_index) { + const auto mode = static_cast(mode_index); + std::printf("[MEDIAN] rounds=%u mode=%s", rounds, ModeName(mode)); + for (const Metric &metric : metrics) { + std::vector values; + for (const Sample &sample : samples[mode_index]) values.push_back(sample.result.*(metric.member)); + std::printf(" %s=%llu", metric.name, static_cast(Median(std::move(values)))); + } + std::printf("\n"); + } + if (rounds == 0) return; + + const auto &empty = samples[static_cast(atomic_scalar_pmu::Mode::Empty)]; + const auto &control = samples[static_cast(atomic_scalar_pmu::Mode::ScalarControl)]; + const auto &atomic = samples[static_cast(atomic_scalar_pmu::Mode::DependentAtomicAdd)]; + for (const Metric &metric : metrics) { + const double control_minus_empty = + PairedDeltaPerOperation(control, empty, metric.member, rounds); + const double atomic_minus_control = + PairedDeltaPerOperation(atomic, control, metric.member, rounds); + std::printf( + "[DELTA_PER_OP] rounds=%u metric=%s control_minus_empty=%.6f atomic_minus_control=%.6f\n", rounds, + metric.name, control_minus_empty, atomic_minus_control + ); + } + + const double atomic_sys_ns = PairedDeltaPerOperation( + atomic, control, &atomic_scalar_pmu::ProbeResult::sys_cycles, rounds + ); + const double atomic_total_cycles = PairedDeltaPerOperation( + atomic, control, &atomic_scalar_pmu::ProbeResult::pmu_total_cycles, rounds + ); + const double atomic_scalar_cycles = PairedDeltaPerOperation( + atomic, control, &atomic_scalar_pmu::ProbeResult::pmu_scalar_busy, rounds + ); + const double scalar_share = atomic_total_cycles == 0.0 ? 0.0 : atomic_scalar_cycles / atomic_total_cycles; + std::printf( + "[ATOMIC_CLASSIFICATION] rounds=%u completion_ns_per_op=%.6f " + "pmu_total_cycles_per_op=%.6f scalar_busy_cycles_per_op=%.6f scalar_share=%.9f\n", + rounds, atomic_sys_ns, atomic_total_cycles, atomic_scalar_cycles, scalar_share + ); +} + +} // namespace + +int main(int argc, char **argv) { + const std::string kernel_path = argc > 1 ? argv[1] : "./atomic_scalar_pmu_kernel.o"; + if (argc > 2) { + std::fprintf(stderr, "Usage: %s [atomic_scalar_pmu_kernel.o]\n", argv[0]); + return EXIT_FAILURE; + } + const uint32_t repeats = RepeatsFromEnv(); + bool options_ok = repeats != 0; + const uint64_t seed = SeedFromEnv(&options_ok); + const std::vector round_values = RoundsFromEnv(&options_ok); + if (!options_ok || round_values.empty()) return EXIT_FAILURE; + + const int32_t device = atomic_probe::DeviceId(); + if (device < 0) return EXIT_FAILURE; + const std::vector kernel_data = ReadBinary(kernel_path); + if (kernel_data.empty()) { + std::fprintf(stderr, "Cannot read kernel binary: %s\n", kernel_path.c_str()); + return EXIT_FAILURE; + } + + if (!CheckAcl(aclInit(nullptr), "aclInit") || !CheckAcl(aclrtSetDevice(device), "aclrtSetDevice")) { + return EXIT_FAILURE; + } + aclrtStream stream = nullptr; + if (!CheckAcl(aclrtCreateStream(&stream), "aclrtCreateStream")) return EXIT_FAILURE; + atomic_probe::pmu::ActiveSubcoreLimits active_limits; + if (!atomic_probe::pmu::QueryActiveSubcoreLimits(stream, &active_limits)) return EXIT_FAILURE; + + aclrtBinHandle binary_handle = nullptr; + if (!CheckAcl( + atomic_probe::LoadAicoreBinaryFromData(kernel_data.data(), kernel_data.size(), &binary_handle), + "LoadAicoreBinaryFromData" + )) { + return EXIT_FAILURE; + } + aclrtFuncHandle function = nullptr; + if (!CheckAcl(aclrtBinaryGetFunctionByEntry(binary_handle, 0, &function), "aclrtBinaryGetFunctionByEntry")) { + return EXIT_FAILURE; + } + + void *state_device = nullptr; + void *pmu_regs_device = nullptr; + void *pmu_control_device = nullptr; + if (!CheckAcl( + aclrtMalloc(&state_device, sizeof(atomic_scalar_pmu::ProbeState), ACL_MEM_MALLOC_NORMAL_ONLY), + "aclrtMalloc(probe state)" + )) { + return EXIT_FAILURE; + } + PmuResources pmu_resources; + if (!pmu_resources.Initialize(static_cast(device))) return EXIT_FAILURE; + const size_t pmu_regs_bytes = sizeof(pmu_resources.RegisterBases()); + if (!CheckAcl(aclrtMalloc(&pmu_regs_device, pmu_regs_bytes, ACL_MEM_MALLOC_NORMAL_ONLY), "aclrtMalloc(PMU regs)") || + !CheckAcl( + aclrtMemcpy( + pmu_regs_device, pmu_regs_bytes, pmu_resources.RegisterBases().data(), pmu_regs_bytes, + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D PMU regs)" + ) || + !CheckAcl( + aclrtMalloc(&pmu_control_device, sizeof(atomic_probe::pmu::PmuControl), ACL_MEM_MALLOC_NORMAL_ONLY), + "aclrtMalloc(PMU control)" + )) { + return EXIT_FAILURE; + } + + atomic_probe::pmu::PmuControl pmu_control{}; + pmu_control.magic = atomic_probe::pmu::kPmuControlMagic; + pmu_control.version = atomic_probe::pmu::kPmuControlVersion; + pmu_control.expected_subcores = active_limits.total; + if (!CheckAcl( + aclrtMemcpy( + pmu_control_device, sizeof(pmu_control), &pmu_control, sizeof(pmu_control), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D initial PMU control)" + )) { + return EXIT_FAILURE; + } + + const std::string dispatcher_path = ArtifactBesideKernel(kernel_path, "libsimpler_aicpu_dispatcher.so"); + const std::string helper_path = ArtifactBesideKernel(kernel_path, "libatomic_scalar_pmu_aicpu.so"); + const std::vector dispatcher_data = ReadBinary(dispatcher_path); + const std::vector helper_data = ReadBinary(helper_path); + if (dispatcher_data.empty() || helper_data.empty()) { + std::fprintf(stderr, "Cannot read PMU artifacts: %s %s\n", dispatcher_path.c_str(), helper_path.c_str()); + return EXIT_FAILURE; + } + host::LoadAicpuOp pmu_loader; + if (pmu_loader.BootstrapDispatcher( + dispatcher_data.data(), dispatcher_data.size(), helper_data.data(), helper_data.size(), stream, device + ) != 0 || + pmu_loader.Init() != 0) { + std::fprintf(stderr, "Cannot initialize PMU AICPU helper.\n"); + return EXIT_FAILURE; + } + KernelArgs pmu_kernel_args{}; + pmu_kernel_args.runtime_args = reinterpret_cast(pmu_control_device); + pmu_kernel_args.regs = reinterpret_cast(pmu_regs_device); + pmu_kernel_args.device_id = static_cast(device); + if (!RunPmuCommand( + pmu_loader, stream, &pmu_kernel_args, pmu_control_device, &pmu_control, + atomic_probe::pmu::PmuCommand::Configure + )) { + return EXIT_FAILURE; + } + + std::printf( + "=== Single-AIV dependent atomicAdd scalar-busy PMU probe ===\n" + "device=%d repeats=%u seed=0x%llx events=total,scalar_busy(0x1),icache_req(0x34),icache_miss(0x35)\n", + device, repeats, static_cast(seed) + ); + bool all_passed = true; + for (const uint32_t rounds : round_values) { + std::array, 3> samples; + for (uint32_t repeat = 1; repeat <= repeats; ++repeat) { + for (uint32_t mode_index = 0; mode_index < static_cast(atomic_scalar_pmu::Mode::Count); + ++mode_index) { + Sample sample; + const auto mode = static_cast(mode_index); + const bool passed = RunOne( + function, stream, state_device, reinterpret_cast(pmu_regs_device), mode, rounds, + repeat, seed, pmu_control, &sample + ); + all_passed &= passed; + samples[mode_index].push_back(sample); + if (!passed) break; + } + if (!all_passed) break; + } + if (!all_passed) break; + PrintRoundSummary(rounds, samples); + } + + const bool restored = RunPmuCommand( + pmu_loader, stream, &pmu_kernel_args, pmu_control_device, &pmu_control, + atomic_probe::pmu::PmuCommand::Restore + ); + bool cleanup_ok = restored; + pmu_loader.Finalize(); + pmu_resources.RestoreAndUnmap(); + cleanup_ok &= CheckAcl(aclrtFree(pmu_control_device), "aclrtFree(PMU control)"); + cleanup_ok &= CheckAcl(aclrtFree(pmu_regs_device), "aclrtFree(PMU regs)"); + cleanup_ok &= CheckAcl(aclrtFree(state_device), "aclrtFree(probe state)"); + cleanup_ok &= CheckAcl(aclrtBinaryUnLoad(binary_handle), "aclrtBinaryUnLoad"); + cleanup_ok &= CheckAcl(aclrtDestroyStream(stream), "aclrtDestroyStream"); + cleanup_ok &= CheckAcl(aclrtResetDevice(device), "aclrtResetDevice"); + cleanup_ok &= CheckAcl(aclFinalize(), "aclFinalize"); + std::printf( + "[SUMMARY] semantic_status=%s pmu_restore_and_cleanup=%s\n", all_passed ? "PASS" : "FAIL", + cleanup_ok ? "PASS" : "FAIL" + ); + return all_passed && cleanup_ok ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/atomic_probe/ccec/atomic_scalar_pmu_shared.h b/tests/atomic_probe/ccec/atomic_scalar_pmu_shared.h new file mode 100644 index 0000000000..415449058e --- /dev/null +++ b/tests/atomic_probe/ccec/atomic_scalar_pmu_shared.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef TESTS_ATOMIC_PROBE_CCEC_ATOMIC_SCALAR_PMU_SHARED_H_ +#define TESTS_ATOMIC_PROBE_CCEC_ATOMIC_SCALAR_PMU_SHARED_H_ + +#include +#include + +namespace atomic_scalar_pmu { + +// 三条路径使用同一份 kernel 和同一个 PMU 读数协议，只替换 gate 内的固定次数工作负载： +// EMPTY 量 gate/read 固有开销；SCALAR_CONTROL 量与 atomic 路径相同的标量递推； +// DEPENDENT_ATOMIC_ADD 让后一条 atomicAdd 的加数依赖前一条返回值，避免多条 atomic 并行掩盖等待时间。 +enum class Mode : uint32_t { + Empty = 0, + ScalarControl = 1, + DependentAtomicAdd = 2, + Count = 3, +}; + +// Host launch 前只写本 cache line；kernel 在测量窗口中只读。PMU MMIO base 表 +// 按 get_coreid() 的低 12 bit 索引，布局与公共 pmu_probe AICPU helper 完全一致。 +struct alignas(64) ProbeControl { + uint64_t pmu_register_bases; + uint32_t mode; + uint32_t rounds; + uint64_t seed; + uint64_t reserved[5]; +}; + +// Atomic 目标独占 cache line，排除 result/control 的普通 GM 写或 DCCI 对原子值的影响。 +struct alignas(64) AtomicTarget { + volatile uint64_t value; + uint64_t reserved[7]; +}; + +// 单 AIV 独占写结果 cache line。sys_cycles 是 get_sys_cnt() 前后差；其余四项是同一 +// gate 窗口内的 PMU total、scalar busy、I-cache request 和 I-cache miss 原始计数。 +struct alignas(64) ProbeResult { + uint64_t sys_cycles; + uint64_t pmu_total_cycles; + uint64_t pmu_scalar_busy; + uint64_t pmu_icache_request; + uint64_t pmu_icache_miss; + uint64_t checksum; + uint64_t pmu_ctrl_after_stop; + uint64_t physical_core_id; +}; + +struct alignas(64) ProbeState { + ProbeControl control; + AtomicTarget target; + ProbeResult result; +}; + +static_assert(sizeof(ProbeControl) == 64, "probe control must occupy one cache line"); +static_assert(sizeof(AtomicTarget) == 64, "atomic target must occupy one cache line"); +static_assert(sizeof(ProbeResult) == 64, "probe result must occupy one cache line"); +static_assert(offsetof(ProbeState, target) == 64, "atomic target must start on its own cache line"); +static_assert(offsetof(ProbeState, result) == 128, "probe result must start on its own cache line"); +static_assert(sizeof(ProbeState) == 192, "probe state ABI changed unexpectedly"); + +} // namespace atomic_scalar_pmu + +#endif // TESTS_ATOMIC_PROBE_CCEC_ATOMIC_SCALAR_PMU_SHARED_H_ diff --git a/tests/atomic_probe/ccec/cache_preload.cpp b/tests/atomic_probe/ccec/cache_preload.cpp new file mode 100644 index 0000000000..01e86dec8e --- /dev/null +++ b/tests/atomic_probe/ccec/cache_preload.cpp @@ -0,0 +1,329 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +// Pure CCEC A5 cache-preload usage probe. This deliberately uses the compiler +// interfaces directly: +// +// dc_preload(__gm__ uint64_t *base, int64_t byte_offset) +// icache_preload(int64_t units_from_current_pc) +// get_icache_prl_st() +// +// Preload is only a performance hint. It is not DCCI, a memory-ordering +// primitive, cross-core publication, or synchronization. + +#include "../cache_preload_shared.h" +#include "ccec_utils.h" + +CCEC_PROBE_KERNEL_META(cache_preload); + +namespace { + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadOrderedCycle() { + uint64_t cycle = 0; + asm volatile("MOV %0, SYS_CNT\n" : "=&l"(cycle) : : "memory"); + return cycle; +} + +// The tied operand makes the value under test a compiler dependency of the +// SYS_CNT boundary. These helpers are measurement machinery, not a hardware +// barrier or a cache-coherence operation. +__aicore__ __attribute__((always_inline)) inline uint64_t CycleBeforeValue(uint64_t &value) { + uint64_t cycle = 0; + asm volatile("MOV %1, SYS_CNT\n" + "MOV %0, %0\n" + : "+l"(value), "=&l"(cycle) + : + : "memory"); + return cycle; +} + +__aicore__ __attribute__((always_inline)) inline uint64_t CycleAfterValue(uint64_t &value) { + uint64_t cycle = 0; + asm volatile("MOV %0, %0\n" + "MOV %1, SYS_CNT\n" + : "+l"(value), "=&l"(cycle) + : + : "memory"); + return cycle; +} + +__aicore__ __attribute__((always_inline)) inline uint64_t CycleBeforeStore(uint64_t &address, uint64_t &value) { + uint64_t cycle = 0; + asm volatile("MOV %2, SYS_CNT\n" + "MOV %0, %0\n" + "MOV %1, %1\n" + : "+l"(address), "+l"(value), "=&l"(cycle) + : + : "memory"); + return cycle; +} + +__aicore__ __attribute__((always_inline)) inline uint64_t CycleAfterStore(uint64_t &address, uint64_t &value) { + uint64_t cycle = 0; + asm volatile("MOV %0, %0\n" + "MOV %1, %1\n" + "MOV %2, SYS_CNT\n" + : "+l"(address), "+l"(value), "=&l"(cycle) + : + : "memory"); + return cycle; +} + +__aicore__ __attribute__((always_inline)) inline uint64_t OpaqueIdentity(uint64_t value) { + asm volatile("MOV %0, %0\n" : "+l"(value) : : "memory"); + return value; +} + +__aicore__ __attribute__((always_inline)) inline void PublishResult( + __gm__ cache_preload::ProbeResult *result, uint64_t value, uint64_t preparation_checksum, uint64_t gap_checksum, + uint64_t issue_ticks, uint64_t access_or_work_ticks, uint64_t store_flush_ticks, uint64_t total_ticks, + uint64_t immediate_status, uint64_t final_status, uint64_t polls, uint32_t mode, uint32_t target_word, + uint32_t gap_rounds +) { + st_dev_b64(&result->value, value); + st_dev_b64(&result->preparation_checksum, preparation_checksum); + st_dev_b64(&result->gap_checksum, gap_checksum); + st_dev_b64(&result->issue_ticks, issue_ticks); + st_dev_b64(&result->access_or_work_ticks, access_or_work_ticks); + st_dev_b64(&result->store_flush_ticks, store_flush_ticks); + st_dev_b64(&result->total_ticks, total_ticks); + st_dev_b64(&result->icache_immediate_status, immediate_status); + st_dev_b64(&result->icache_final_status, final_status); + st_dev_b64(&result->icache_polls, polls); + st_dev_b64(&result->mode_echo, static_cast(mode)); + st_dev_b64(&result->target_word_echo, static_cast(target_word)); + st_dev_b64(&result->gap_rounds_echo, static_cast(gap_rounds)); +} + +} // namespace + +// Keep the independent gap in a separate small function. While this function +// runs, current-PC ICache preload can work on the sequential code that follows +// its call site in cache_preload_icache_path. +extern "C" __aicore__ __attribute__((noinline, used)) uint64_t cache_preload_gap(uint64_t seed, uint32_t gap_rounds) { + return cache_preload::GapOracle(seed, gap_rounds); +} + +#define CACHE_PRELOAD_NOPS_1() asm volatile("nop"); +#define CACHE_PRELOAD_NOPS_2() CACHE_PRELOAD_NOPS_1() CACHE_PRELOAD_NOPS_1() +#define CACHE_PRELOAD_NOPS_4() CACHE_PRELOAD_NOPS_2() CACHE_PRELOAD_NOPS_2() +#define CACHE_PRELOAD_NOPS_8() CACHE_PRELOAD_NOPS_4() CACHE_PRELOAD_NOPS_4() +#define CACHE_PRELOAD_NOPS_16() CACHE_PRELOAD_NOPS_8() CACHE_PRELOAD_NOPS_8() +#define CACHE_PRELOAD_NOPS_32() CACHE_PRELOAD_NOPS_16() CACHE_PRELOAD_NOPS_16() +#define CACHE_PRELOAD_NOPS_64() CACHE_PRELOAD_NOPS_32() CACHE_PRELOAD_NOPS_32() +#define CACHE_PRELOAD_NOPS_128() CACHE_PRELOAD_NOPS_64() CACHE_PRELOAD_NOPS_64() +#define CACHE_PRELOAD_NOPS_256() CACHE_PRELOAD_NOPS_128() CACHE_PRELOAD_NOPS_128() +#define CACHE_PRELOAD_NOPS_512() CACHE_PRELOAD_NOPS_256() CACHE_PRELOAD_NOPS_256() +#define CACHE_PRELOAD_NOPS_1024() CACHE_PRELOAD_NOPS_512() CACHE_PRELOAD_NOPS_512() +#define CACHE_PRELOAD_NOPS_2048() CACHE_PRELOAD_NOPS_1024() CACHE_PRELOAD_NOPS_1024() +#define CACHE_PRELOAD_NOPS_4096() CACHE_PRELOAD_NOPS_2048() CACHE_PRELOAD_NOPS_2048() +#define CACHE_PRELOAD_NOPS_8192() CACHE_PRELOAD_NOPS_4096() CACHE_PRELOAD_NOPS_4096() + +// The evictor is deliberately larger than the documented 16 KiB AIV ICache. +// The build script validates its final linked symbol size and non-overlap with +// the measured current-PC path. +extern "C" __aicore__ __attribute__((noinline, used, aligned(128))) uint64_t cache_preload_icache_evictor(uint64_t seed +) { + CACHE_PRELOAD_NOPS_8192() + return cache_preload::ICacheEvictorOracle(seed); +} + +// Baseline/async/wait share this exact physical NOP region. icache_preload(2) +// uses the current PC, then the noinline gap provides an overlap window before +// execution reaches the upcoming sequential region. +extern "C" __aicore__ __attribute__((noinline, used, aligned(128))) void cache_preload_icache_path( + uint32_t mode_value, uint64_t seed, uint32_t target_word, uint32_t gap_rounds, uint64_t preparation_checksum, + __gm__ cache_preload::ProbeResult *result +) { + const uint64_t total_begin = ReadOrderedCycle(); + uint64_t issue_ticks = 0; + uint64_t immediate_status = 0; + uint64_t final_status = 0; + uint64_t polls = 0; + + const bool use_preload = mode_value == static_cast(cache_preload::Mode::ICacheCurrentPcAsync) || + mode_value == static_cast(cache_preload::Mode::ICacheCurrentPcWait); + if (use_preload) { + const uint64_t issue_begin = ReadOrderedCycle(); + icache_preload(static_cast(cache_preload::kICachePreloadUnits)); + const uint64_t issue_end = ReadOrderedCycle(); + issue_ticks = issue_end - issue_begin; + + immediate_status = static_cast(get_icache_prl_st()); + final_status = immediate_status; + if (mode_value == static_cast(cache_preload::Mode::ICacheCurrentPcWait)) { + while (final_status != 0 && polls < cache_preload::kICachePollLimit) { + CACHE_PRELOAD_NOPS_16() + final_status = static_cast(get_icache_prl_st()); + ++polls; + } + } + } + + const uint64_t gap_checksum = cache_preload_gap(seed, gap_rounds); + if (use_preload) { + final_status = static_cast(get_icache_prl_st()); + } + + uint64_t work_value = seed ^ gap_checksum; + const uint64_t work_begin = CycleBeforeValue(work_value); + CACHE_PRELOAD_NOPS_1024() + work_value = cache_preload::ICacheTargetOracle(work_value); + const uint64_t work_end = CycleAfterValue(work_value); + + PublishResult( + result, work_value, preparation_checksum, gap_checksum, issue_ticks, work_end - work_begin, 0, + work_end - total_begin, immediate_status, final_status, polls, mode_value, target_word, gap_rounds + ); +} + +extern "C" __global__ __aicore__ void KERNEL_ENTRY(cache_preload)(__gm__ cache_preload::ProbeState *state) { + if (get_block_idx() != 0) return; + + // Host rewrites only this line for each launch. Invalidate and complete it + // before consuming the runtime mode/offset/seed. + dcci(&state->control, SINGLE_CACHE_LINE); + dsb(DSB_ALL); + + const uint32_t mode_value = state->control.mode; + const uint32_t target_word = state->control.target_word; + const uint32_t gap_rounds = state->control.gap_rounds; + const uint64_t seed = state->control.seed; + __gm__ cache_preload::ProbeResult *result = &state->result; + + if (mode_value == static_cast(cache_preload::Mode::DCacheBaseline) || + mode_value == static_cast(cache_preload::Mode::DCachePreload)) { + __gm__ uint64_t *target = &state->data[target_word]; + + // Establish a deterministic cold local-DCache line outside the timed + // interval. This DCCI is measurement setup, not part of preload usage. + dcci(target, SINGLE_CACHE_LINE); + dsb(DSB_ALL); + + const uint64_t total_begin = ReadOrderedCycle(); + uint64_t issue_ticks = 0; + if (mode_value == static_cast(cache_preload::Mode::DCachePreload)) { + const uint64_t issue_begin = ReadOrderedCycle(); + const int64_t byte_offset = static_cast(target_word) * sizeof(uint64_t); + dc_preload(state->data, byte_offset); + const uint64_t issue_end = ReadOrderedCycle(); + issue_ticks = issue_end - issue_begin; + } + + const uint64_t gap_checksum = cache_preload_gap(seed, gap_rounds); + + // OpaqueIdentity is MOV-at-runtime, so the address delta is exactly + // zero on hardware while still creating a compiler dependency from + // the independent gap to the ordinary target load. + const uint64_t opaque_gap = OpaqueIdentity(gap_checksum); + const uint64_t address_delta = opaque_gap - gap_checksum; + uint64_t target_address = reinterpret_cast(target) + address_delta; + const uint64_t access_begin = CycleBeforeValue(target_address); + volatile __gm__ uint64_t *volatile_target = reinterpret_cast(target_address); + uint64_t value = *volatile_target; + const uint64_t access_end = CycleAfterValue(value); + + PublishResult( + result, value, 0, gap_checksum, issue_ticks, access_end - access_begin, 0, access_end - total_begin, 0, 0, + 0, mode_value, target_word, gap_rounds + ); + return; + } + + if (mode_value == static_cast(cache_preload::Mode::DCacheStoreBaseline) || + mode_value == static_cast(cache_preload::Mode::DCacheStorePreload) || + mode_value == static_cast(cache_preload::Mode::DCachePublishBaseline) || + mode_value == static_cast(cache_preload::Mode::DCachePublishPreload)) { + __gm__ uint64_t *target = &state->data[target_word]; + + // Both write scenarios start from the same cold, clean cache line. + // Host resets the target word before every launch. This DCCI/DSB pair + // is setup outside the timed interval. + dcci(target, SINGLE_CACHE_LINE); + dsb(DSB_ALL); + + const bool use_preload = mode_value == static_cast(cache_preload::Mode::DCacheStorePreload) || + mode_value == static_cast(cache_preload::Mode::DCachePublishPreload); + const bool publish_to_gm = mode_value == static_cast(cache_preload::Mode::DCachePublishBaseline) || + mode_value == static_cast(cache_preload::Mode::DCachePublishPreload); + + const uint64_t total_begin = ReadOrderedCycle(); + uint64_t issue_ticks = 0; + if (use_preload) { + const uint64_t issue_begin = ReadOrderedCycle(); + const int64_t byte_offset = static_cast(target_word) * sizeof(uint64_t); + dc_preload(state->data, byte_offset); + const uint64_t issue_end = ReadOrderedCycle(); + issue_ticks = issue_end - issue_begin; + } + + const uint64_t gap_checksum = cache_preload_gap(seed, gap_rounds); + const uint64_t opaque_gap = OpaqueIdentity(gap_checksum); + const uint64_t address_delta = opaque_gap - gap_checksum; + uint64_t target_address = reinterpret_cast(target) + address_delta; + uint64_t store_value = cache_preload::WriteValue(target_word, seed, gap_checksum); + volatile __gm__ uint64_t *volatile_target = reinterpret_cast(target_address); + + const uint64_t store_begin = CycleBeforeStore(target_address, store_value); + *volatile_target = store_value; + const uint64_t store_end = CycleAfterStore(target_address, store_value); + + uint64_t total_end = store_end; + uint64_t store_flush_ticks = 0; + if (publish_to_gm) { + // This is part of the publish-to-GM scenario only. The official + // CACHELINE_OUT contract makes the DCache line consistent with + // GM; DSB completes that measured publication sequence. + dcci(target, SINGLE_CACHE_LINE, CACHELINE_OUT); + dsb(DSB_ALL); + total_end = ReadOrderedCycle(); + store_flush_ticks = total_end - store_begin; + } + + // The ordinary read validates the writer core's cached value. For the + // store-only scenario, publication is deliberately deferred until + // after total_end and is probe cleanup, not measured business work. + const uint64_t local_value = *volatile_target; + if (!publish_to_gm) { + dcci(target, SINGLE_CACHE_LINE, CACHELINE_OUT); + dsb(DSB_ALL); + } + const uint64_t gm_value = ld_dev_b64(target); + + PublishResult( + result, gm_value, local_value, gap_checksum, issue_ticks, store_end - store_begin, store_flush_ticks, + total_end - total_begin, 0, 0, 0, mode_value, target_word, gap_rounds + ); + return; + } + + if (mode_value == static_cast(cache_preload::Mode::ICacheColdBaseline) || + mode_value == static_cast(cache_preload::Mode::ICacheCurrentPcAsync) || + mode_value == static_cast(cache_preload::Mode::ICacheCurrentPcWait)) { + const uint64_t preparation_checksum = cache_preload_icache_evictor(seed); + cache_preload_icache_path(mode_value, seed, target_word, gap_rounds, preparation_checksum, result); + } +} + +#undef CACHE_PRELOAD_NOPS_8192 +#undef CACHE_PRELOAD_NOPS_4096 +#undef CACHE_PRELOAD_NOPS_2048 +#undef CACHE_PRELOAD_NOPS_1024 +#undef CACHE_PRELOAD_NOPS_512 +#undef CACHE_PRELOAD_NOPS_256 +#undef CACHE_PRELOAD_NOPS_128 +#undef CACHE_PRELOAD_NOPS_64 +#undef CACHE_PRELOAD_NOPS_32 +#undef CACHE_PRELOAD_NOPS_16 +#undef CACHE_PRELOAD_NOPS_8 +#undef CACHE_PRELOAD_NOPS_4 +#undef CACHE_PRELOAD_NOPS_2 +#undef CACHE_PRELOAD_NOPS_1 diff --git a/tests/atomic_probe/ccec/cache_preload_host.cpp b/tests/atomic_probe/ccec/cache_preload_host.cpp new file mode 100644 index 0000000000..ebc37748ef --- /dev/null +++ b/tests/atomic_probe/ccec/cache_preload_host.cpp @@ -0,0 +1,400 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "../probe_host.h" +#include "../cache_preload_shared.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct KernelArgs { + uint64_t state_pointer; +}; + +struct Sample { + cache_preload::ProbeResult result{}; + uint64_t target_value = 0; +}; + +static_assert(sizeof(KernelArgs) == sizeof(uint64_t), "unexpected CCEC kernel argument ABI"); + +bool Check(aclError error, const char *expression) { + return atomic_probe::CheckAcl(error, expression, __FILE__, __LINE__); +} + +std::vector ReadBinary(const std::string &path) { + std::ifstream stream(path, std::ios::binary | std::ios::ate); + if (!stream) return {}; + const std::streamsize size = stream.tellg(); + if (size <= 0) return {}; + stream.seekg(0); + std::vector data(static_cast(size)); + if (!stream.read(data.data(), size)) return {}; + return data; +} + +const char *ModeName(cache_preload::Mode mode) { + switch (mode) { + case cache_preload::Mode::DCacheBaseline: + return "dcache-baseline"; + case cache_preload::Mode::DCachePreload: + return "dcache-preload"; + case cache_preload::Mode::DCacheStoreBaseline: + return "dstore-only-baseline"; + case cache_preload::Mode::DCacheStorePreload: + return "dstore-only-preload"; + case cache_preload::Mode::DCachePublishBaseline: + return "dpublish-gm-baseline"; + case cache_preload::Mode::DCachePublishPreload: + return "dpublish-gm-preload"; + case cache_preload::Mode::ICacheColdBaseline: + return "icache-cold"; + case cache_preload::Mode::ICacheCurrentPcAsync: + return "icache-current-pc"; + case cache_preload::Mode::ICacheCurrentPcWait: + return "icache-wait"; + case cache_preload::Mode::Count: + break; + } + return "unknown"; +} + +bool IsDCacheReadMode(cache_preload::Mode mode) { + return mode == cache_preload::Mode::DCacheBaseline || mode == cache_preload::Mode::DCachePreload; +} + +bool IsDCacheStoreMode(cache_preload::Mode mode) { + return mode == cache_preload::Mode::DCacheStoreBaseline || mode == cache_preload::Mode::DCacheStorePreload; +} + +bool IsDCachePublishMode(cache_preload::Mode mode) { + return mode == cache_preload::Mode::DCachePublishBaseline || mode == cache_preload::Mode::DCachePublishPreload; +} + +bool IsDCacheWriteMode(cache_preload::Mode mode) { return IsDCacheStoreMode(mode) || IsDCachePublishMode(mode); } + +uint64_t Median(std::vector values) { + std::sort(values.begin(), values.end()); + return values[values.size() / 2U]; +} + +bool RunOne( + aclrtFuncHandle function, aclrtStream stream, void *state_device, cache_preload::Mode mode, uint32_t target_word, + uint64_t seed, Sample *sample +) { + cache_preload::ProbeControl control{}; + control.mode = static_cast(mode); + control.target_word = target_word; + control.gap_rounds = cache_preload::kGapRounds; + control.seed = seed; + cache_preload::ProbeResult zero{}; + + auto *state_bytes = reinterpret_cast(state_device); + void *control_device = state_bytes + offsetof(cache_preload::ProbeState, control); + void *result_device = state_bytes + offsetof(cache_preload::ProbeState, result); + void *target_device = + state_bytes + offsetof(cache_preload::ProbeState, data) + static_cast(target_word) * sizeof(uint64_t); + if (IsDCacheWriteMode(mode)) { + const uint64_t initial_value = cache_preload::DataValue(target_word); + if (!Check( + aclrtMemcpy( + target_device, sizeof(initial_value), &initial_value, sizeof(initial_value), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D write target reset)" + )) { + return false; + } + } + if (!Check( + aclrtMemcpy(control_device, sizeof(control), &control, sizeof(control), ACL_MEMCPY_HOST_TO_DEVICE), + "aclrtMemcpy(H2D control)" + ) || + !Check( + aclrtMemcpy(result_device, sizeof(zero), &zero, sizeof(zero), ACL_MEMCPY_HOST_TO_DEVICE), + "aclrtMemcpy(H2D result reset)" + )) { + return false; + } + + KernelArgs args{reinterpret_cast(state_device)}; + if (!Check( + aclrtLaunchKernelWithHostArgs(function, 1, stream, nullptr, &args, sizeof(args), nullptr, 0), + "aclrtLaunchKernelWithHostArgs" + ) || + !Check(aclrtSynchronizeStream(stream), "aclrtSynchronizeStream") || + !Check( + aclrtMemcpy( + &sample->result, sizeof(sample->result), result_device, sizeof(sample->result), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H result)" + )) { + return false; + } + if (IsDCacheWriteMode(mode) && !Check( + aclrtMemcpy( + &sample->target_value, sizeof(sample->target_value), target_device, + sizeof(sample->target_value), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H write target)" + )) { + return false; + } + return true; +} + +bool ValidateSample( + const Sample &sample, cache_preload::Mode mode, uint32_t target_word, uint64_t seed, std::string *reason +) { + const cache_preload::ProbeResult &result = sample.result; + if (result.mode_echo != static_cast(mode)) { + *reason = "mode-echo"; + return false; + } + if (result.target_word_echo != target_word) { + *reason = "target-word-echo"; + return false; + } + if (result.gap_rounds_echo != cache_preload::kGapRounds) { + *reason = "gap-rounds-echo"; + return false; + } + const uint64_t expected_gap = cache_preload::GapOracle(seed, cache_preload::kGapRounds); + if (result.gap_checksum != expected_gap) { + *reason = "gap-checksum"; + return false; + } + + if (IsDCacheReadMode(mode)) { + if (result.value != cache_preload::DataValue(target_word)) { + *reason = "dcache-load-value"; + return false; + } + if (result.preparation_checksum != 0 || result.icache_immediate_status != 0 || + result.icache_final_status != 0 || result.icache_polls != 0 || result.store_flush_ticks != 0) { + *reason = "dcache-unexpected-icache-fields"; + return false; + } + } else if (IsDCacheWriteMode(mode)) { + const uint64_t expected_value = cache_preload::WriteValue(target_word, seed, expected_gap); + if (result.preparation_checksum != expected_value) { + *reason = "dcache-writer-local-value"; + return false; + } + if (result.value != expected_value) { + *reason = "dcache-bypass-value-after-cleanup"; + return false; + } + if (sample.target_value != expected_value) { + *reason = "dcache-host-visible-value"; + return false; + } + if ((IsDCacheStoreMode(mode) && result.store_flush_ticks != 0) || + (IsDCachePublishMode(mode) && result.store_flush_ticks == 0)) { + *reason = "dcache-store-flush-window"; + return false; + } + if (result.icache_immediate_status != 0 || result.icache_final_status != 0 || result.icache_polls != 0) { + *reason = "dcache-write-unexpected-icache-fields"; + return false; + } + } else { + if (result.preparation_checksum != cache_preload::ICacheEvictorOracle(seed)) { + *reason = "icache-evictor-checksum"; + return false; + } + if (result.value != cache_preload::ICacheTargetOracle(seed ^ expected_gap)) { + *reason = "icache-target-checksum"; + return false; + } + if (mode == cache_preload::Mode::ICacheCurrentPcWait && + (result.icache_final_status != 0 || result.icache_polls >= cache_preload::kICachePollLimit)) { + *reason = "icache-wait-did-not-reach-idle"; + return false; + } + if (result.store_flush_ticks != 0) { + *reason = "icache-unexpected-store-flush"; + return false; + } + } + + if (result.access_or_work_ticks == 0 || result.total_ticks == 0) { + *reason = "zero-timing-window"; + return false; + } + return true; +} + +void PrintSummary(cache_preload::Mode mode, const std::vector &samples) { + std::vector issue; + std::vector access_or_work; + std::vector store_flush; + std::vector total; + std::vector polls; + uint32_t immediate_busy = 0; + uint32_t final_busy = 0; + for (const Sample &sample : samples) { + issue.push_back(sample.result.issue_ticks); + access_or_work.push_back(sample.result.access_or_work_ticks); + store_flush.push_back(sample.result.store_flush_ticks); + total.push_back(sample.result.total_ticks); + polls.push_back(sample.result.icache_polls); + immediate_busy += sample.result.icache_immediate_status != 0 ? 1U : 0U; + final_busy += sample.result.icache_final_status != 0 ? 1U : 0U; + } + std::printf( + "%-21s issue=%6llu access/work=%6llu store->gm=%6llu total=%6llu polls=%5llu " + "immediate_busy=%u/%zu final_busy=%u/%zu\n", + ModeName(mode), static_cast(Median(std::move(issue))), + static_cast(Median(std::move(access_or_work))), + static_cast(Median(std::move(store_flush))), + static_cast(Median(std::move(total))), + static_cast(Median(std::move(polls))), immediate_busy, samples.size(), final_busy, + samples.size() + ); +} + +} // namespace + +int main(int argc, char **argv) { + const std::string kernel_path = argc > 1 ? argv[1] : "./cache_preload_kernel.o"; + if (argc > 2) { + std::fprintf(stderr, "Usage: %s [cache_preload_kernel.o]\n", argv[0]); + return EXIT_FAILURE; + } + + const std::vector kernel_data = ReadBinary(kernel_path); + if (kernel_data.empty()) { + std::fprintf(stderr, "Cannot read kernel binary: %s\n", kernel_path.c_str()); + return EXIT_FAILURE; + } + const int32_t device = atomic_probe::DeviceId(); + if (device < 0) return EXIT_FAILURE; + + if (!Check(aclInit(nullptr), "aclInit") || !Check(aclrtSetDevice(device), "aclrtSetDevice")) { + return EXIT_FAILURE; + } + aclrtStream stream = nullptr; + if (!Check(aclrtCreateStream(&stream), "aclrtCreateStream")) { + return EXIT_FAILURE; + } + + aclrtBinHandle binary_handle = nullptr; + if (!Check( + atomic_probe::LoadAicoreBinaryFromData(kernel_data.data(), kernel_data.size(), &binary_handle), + "LoadAicoreBinaryFromData" + )) { + return EXIT_FAILURE; + } + aclrtFuncHandle function = nullptr; + if (!Check(aclrtBinaryGetFunctionByEntry(binary_handle, 0, &function), "aclrtBinaryGetFunctionByEntry")) { + return EXIT_FAILURE; + } + + void *state_device = nullptr; + if (!Check( + aclrtMalloc(&state_device, sizeof(cache_preload::ProbeState), ACL_MEM_MALLOC_HUGE_FIRST), + "aclrtMalloc(probe state)" + )) { + return EXIT_FAILURE; + } + + auto initial = std::make_unique(); + for (uint32_t index = 0; index < cache_preload::kDataWords; ++index) { + initial->data[index] = cache_preload::DataValue(index); + } + if (!Check( + aclrtMemcpy(state_device, sizeof(*initial), initial.get(), sizeof(*initial), ACL_MEMCPY_HOST_TO_DEVICE), + "aclrtMemcpy(H2D initial state)" + )) { + return EXIT_FAILURE; + } + + constexpr std::array kModes = { + cache_preload::Mode::DCacheBaseline, cache_preload::Mode::DCachePreload, + cache_preload::Mode::DCacheStoreBaseline, cache_preload::Mode::DCacheStorePreload, + cache_preload::Mode::DCachePublishBaseline, cache_preload::Mode::DCachePublishPreload, + cache_preload::Mode::ICacheColdBaseline, cache_preload::Mode::ICacheCurrentPcAsync, + cache_preload::Mode::ICacheCurrentPcWait, + }; + std::array, 9> samples; + bool launch_ok = true; + bool semantic_ok = true; + for (uint32_t sample_index = 0; sample_index < cache_preload::kSamples; ++sample_index) { + const uint32_t target_word = cache_preload::kTargetStartWord + sample_index * cache_preload::kTargetStrideWords; + const uint64_t seed = 0x123456789abcdef0ULL ^ (static_cast(sample_index) * 0x9e3779b97f4a7c15ULL); + + for (const cache_preload::Mode mode : kModes) { + Sample sample; + launch_ok = RunOne(function, stream, state_device, mode, target_word, seed, &sample); + if (!launch_ok) break; + + std::string reason; + if (!ValidateSample(sample, mode, target_word, seed, &reason)) { + std::fprintf( + stderr, + "[MISMATCH] sample=%u mode=%s reason=%s value=0x%llx " + "target=0x%llx issue=%llu access/work=%llu store->gm=%llu total=%llu immediate=%llu " + "final=%llu polls=%llu\n", + sample_index, ModeName(mode), reason.c_str(), static_cast(sample.result.value), + static_cast(sample.target_value), + static_cast(sample.result.issue_ticks), + static_cast(sample.result.access_or_work_ticks), + static_cast(sample.result.store_flush_ticks), + static_cast(sample.result.total_ticks), + static_cast(sample.result.icache_immediate_status), + static_cast(sample.result.icache_final_status), + static_cast(sample.result.icache_polls) + ); + semantic_ok = false; + } + samples[static_cast(mode)].push_back(sample); + } + if (!launch_ok) break; + } + + atomic_probe::Result result; + result.Expect(launch_ok, "CCEC cache preload launches and result copies"); + result.Expect(semantic_ok, "CCEC cache preload values, write publication, and status contract"); + + std::printf( + "=== A5 pure CCEC Cache Preload Usage Probe ===\n" + "kernel=%s bytes=%zu samples_per_mode=%u gap_rounds=%u " + "icache_preload_units=%u\n" + "All timing values are raw SYS_CNT deltas; they are observations, " + "not API guarantees.\n", + kernel_path.c_str(), kernel_data.size(), cache_preload::kSamples, cache_preload::kGapRounds, + cache_preload::kICachePreloadUnits + ); + for (const cache_preload::Mode mode : kModes) { + const auto &mode_samples = samples[static_cast(mode)]; + if (!mode_samples.empty()) PrintSummary(mode, mode_samples); + } + + bool cleanup_ok = true; + cleanup_ok &= Check(aclrtFree(state_device), "aclrtFree(probe state)"); + cleanup_ok &= Check(aclrtBinaryUnLoad(binary_handle), "aclrtBinaryUnLoad"); + cleanup_ok &= Check(aclrtDestroyStream(stream), "aclrtDestroyStream"); + cleanup_ok &= Check(aclrtResetDevice(device), "aclrtResetDevice"); + cleanup_ok &= Check(aclFinalize(), "aclFinalize"); + result.Expect(cleanup_ok, "CCEC cache preload cleanup"); + return result.ExitCode(); +} diff --git a/tests/atomic_probe/ccec/icache_scalar_pmu.cpp b/tests/atomic_probe/ccec/icache_scalar_pmu.cpp new file mode 100644 index 0000000000..2a2ddb7e6f --- /dev/null +++ b/tests/atomic_probe/ccec/icache_scalar_pmu.cpp @@ -0,0 +1,226 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +// 测试目标：用单个 AIV 精确核实“等待 I-cache miss 回填的周期是否计入 PMU +// scalar busy”。Host/AICPU 在 launch 前把 PMU slot0/1/2 配置为 scalar +// busy(0x1)、I-cache request(0x34)、I-cache miss(0x35)。本用例不含 atomic、 +// GM 轮询、Vector/Cube/MTE 计算，因此 WARM/COLD 的差值只来自 target 的取指状态。 +// +// 每次 kernel 的完整时序： +// 1. 关闭 PMU；对 host 写入的 control 独立 cache line 执行 DCCI + DSB；读取 +// mode、seed、PMU MMIO base，并用 read-to-clear 清空所有 PMU counter。 +// 2. 在 PMU 窗口外准备 I-cache： +// WARM：先调用一次被测 noinline target，使其约 8 KiB 指令体进入 I-cache； +// COLD：调用约 32 KiB 的 noinline evictor，以超过 16 KiB 容量的顺序 +// 指令流替换 I-cache 内容。 +// 两条准备路径都返回可精确复算的 checksum，确保调用不能被编译器删除。 +// 3. 两条路径在分支后汇合。get_sys_cnt 后打开 PMU，窗口内只从同一个调用点 +// 调用同一个 noinline target 一次，然后立即关闭 PMU。两种 mode 的窗内动态 +// 指令完全相同，仅 target 调用前的 I-cache 冷热状态不同。 +// 4. PMU 关闭后才读取 total/scalar/request/miss；最后仅用 st_dev 发布结果， +// 并用 DSB 收口。Host 必须同时校验 target checksum、准备 checksum、mode echo、 +// physical core id 和 PMU gate 状态。 +// +// 预期与判读： +// - COLD 的 I-cache miss 必须显著高于 WARM，先证明冷热对照确实成立； +// - 以 COLD-WARM 扣除同一 target 的固定执行成本。若 total 增量与 scalar busy +// 增量近似相同，miss 回填等待计入 scalar busy；若 total 显著增加而 scalar +// busy 不同比例增加，则该等待形成 scalar-busy gap。 +// - 构建后还必须按最终 ELF 符号大小核实 target >= 8 KiB、evictor >= 32 KiB； +// 否则本用例只能算源码意图，不能算有效的 I-cache 驱逐实验。 + +#include "icache_scalar_pmu_shared.h" +#include "ccec_utils.h" + +CCEC_PROBE_KERNEL_META(icache_scalar_pmu); + +namespace { + +constexpr uint32_t kPmuPhysicalSubcores = 108; +constexpr uint64_t kPmuCtrl0Offset = 0x4200ULL; + +__aicore__ __attribute__((always_inline)) inline int32_t *PmuCounterBase(uint64_t register_base) +{ + // 以 PMU_CTRL_0(0x4200) 为基准后，所有 counter 都落在 ld_dev 的 12-bit immediate 范围内。 + return reinterpret_cast(register_base + kPmuCtrl0Offset); +} + +__aicore__ __attribute__((always_inline)) inline void ClearPmuCounters(uint64_t register_base) +{ + int32_t *base = PmuCounterBase(register_base); + // A5 PMU counter 是 read-to-clear；逐项显式展开，保持 ld_dev offset 为编译期常量。 + (void)ld_dev(base, 0x10); + (void)ld_dev(base, 0x18); + (void)ld_dev(base, 0x20); + (void)ld_dev(base, 0x28); + (void)ld_dev(base, 0x30); + (void)ld_dev(base, 0x38); + (void)ld_dev(base, 0x40); + (void)ld_dev(base, 0x48); + (void)ld_dev(base, 0x50); + (void)ld_dev(base, 0x54); + (void)ld_dev(base, 0x60); + (void)ld_dev(base, 0x64); +} + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadPmuScalar(uint64_t register_base) +{ + return static_cast(ld_dev(PmuCounterBase(register_base), 0x10)); +} + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadPmuIcacheRequest(uint64_t register_base) +{ + return static_cast(ld_dev(PmuCounterBase(register_base), 0x18)); +} + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadPmuIcacheMiss(uint64_t register_base) +{ + return static_cast(ld_dev(PmuCounterBase(register_base), 0x20)); +} + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadPmuTotal(uint64_t register_base) +{ + int32_t *base = PmuCounterBase(register_base); + const uint64_t low = static_cast(ld_dev(base, 0x60)); + const uint64_t high = static_cast(ld_dev(base, 0x64)); + return low | (high << 32); +} + +__aicore__ __attribute__((always_inline)) inline void Publish64(__gm__ uint64_t *address, uint64_t value) +{ + __builtin_cce_st_dev(value, address, 0); +} + +} // namespace + +// 递归宏最终展开成固定数量的独立 volatile NOP。这里不用运行时循环，是为了让 +// target/evictor 的静态指令 footprint 本身达到指定大小，而不是反复执行一个热循环。 +#define ICACHE_PMU_NOPS_1() asm volatile("nop"); +#define ICACHE_PMU_NOPS_2() ICACHE_PMU_NOPS_1() ICACHE_PMU_NOPS_1() +#define ICACHE_PMU_NOPS_4() ICACHE_PMU_NOPS_2() ICACHE_PMU_NOPS_2() +#define ICACHE_PMU_NOPS_8() ICACHE_PMU_NOPS_4() ICACHE_PMU_NOPS_4() +#define ICACHE_PMU_NOPS_16() ICACHE_PMU_NOPS_8() ICACHE_PMU_NOPS_8() +#define ICACHE_PMU_NOPS_32() ICACHE_PMU_NOPS_16() ICACHE_PMU_NOPS_16() +#define ICACHE_PMU_NOPS_64() ICACHE_PMU_NOPS_32() ICACHE_PMU_NOPS_32() +#define ICACHE_PMU_NOPS_128() ICACHE_PMU_NOPS_64() ICACHE_PMU_NOPS_64() +#define ICACHE_PMU_NOPS_256() ICACHE_PMU_NOPS_128() ICACHE_PMU_NOPS_128() +#define ICACHE_PMU_NOPS_512() ICACHE_PMU_NOPS_256() ICACHE_PMU_NOPS_256() +#define ICACHE_PMU_NOPS_1024() ICACHE_PMU_NOPS_512() ICACHE_PMU_NOPS_512() +#define ICACHE_PMU_NOPS_2048() ICACHE_PMU_NOPS_1024() ICACHE_PMU_NOPS_1024() +#define ICACHE_PMU_NOPS_4096() ICACHE_PMU_NOPS_2048() ICACHE_PMU_NOPS_2048() +#define ICACHE_PMU_NOPS_8192() ICACHE_PMU_NOPS_4096() ICACHE_PMU_NOPS_4096() + +// 保持外部可见、used、noinline：WARM 预热调用和 gate 内测量调用必须指向同一符号， +// 不能被内联成两份物理代码。ELF 检查还会验证本函数的最终符号大小至少为 8 KiB。 +extern "C" __aicore__ __attribute__((noinline, used, aligned(128))) uint64_t +icache_scalar_pmu_target(uint64_t seed) +{ + ICACHE_PMU_NOPS_2048() + return icache_scalar_pmu::TargetOracle(seed); +} + +// evictor 同样保持外部可见、used、noinline；8192 条 volatile NOP 形成约 32 KiB +// 顺序指令流。返回值由 host 复算，额外证明 COLD 准备调用确实完成。 +extern "C" __aicore__ __attribute__((noinline, used, aligned(128))) uint64_t +icache_scalar_pmu_evictor(uint64_t seed) +{ + ICACHE_PMU_NOPS_8192() + return icache_scalar_pmu::EvictorOracle(seed); +} + +// 把 mode 分支封装在另一个 noinline 函数内：kernel 本体在准备调用返回以后没有 +// WARM/COLD 控制流，防止 O3 对共同测量尾部做 tail duplication，进而在 gate 内 +// 生成两个物理调用点。该函数本身完全位于 PMU start 之前。 +extern "C" __aicore__ __attribute__((noinline, used)) uint64_t +icache_scalar_pmu_prepare(uint32_t mode_value, uint64_t seed) +{ + if (mode_value == static_cast(icache_scalar_pmu::Mode::WarmTarget)) { + return icache_scalar_pmu_target(seed); + } + if (mode_value == static_cast(icache_scalar_pmu::Mode::ColdTarget)) { + return icache_scalar_pmu_evictor(seed); + } + return 0; +} + +extern "C" __global__ __aicore__ void KERNEL_ENTRY(icache_scalar_pmu)( + __gm__ icache_scalar_pmu::ProbeState *state) +{ + // task-based profiler 可能在入口前已打开 PMU；先关闭，确保准备阶段绝不入窗。 + bisheng::cce::metrics_prof_stop(); + dcci(&state->control, SINGLE_CACHE_LINE); + dsb(DSB_ALL); + + const uint32_t mode_value = state->control.mode; + const uint64_t seed = state->control.seed; + const uint32_t physical_core_id = static_cast(get_coreid()) & 0x0fffU; + + uint64_t register_base = 0; + if (state->control.pmu_register_bases != 0 && physical_core_id < kPmuPhysicalSubcores) { + __gm__ uint64_t *register_bases = + reinterpret_cast<__gm__ uint64_t *>(state->control.pmu_register_bases); + register_base = register_bases[physical_core_id]; + } + if (register_base != 0) { + ClearPmuCounters(register_base); + } + + const uint64_t preparation_checksum = icache_scalar_pmu_prepare(mode_value, seed); + + // mode 分支在这里结束。以下测量窗口对 WARM/COLD 是同一个静态调用点和同一 + // target 符号，动态指令序列不再依赖 mode。 + const uint64_t sys_begin = static_cast(get_sys_cnt()); + bisheng::cce::metrics_prof_start(); + const uint64_t target_checksum = icache_scalar_pmu_target(seed); + bisheng::cce::metrics_prof_stop(); + const uint64_t sys_end = static_cast(get_sys_cnt()); + const uint64_t ctrl_after_stop = static_cast(get_ctrl()); + + uint64_t pmu_total = 0; + uint64_t pmu_scalar = 0; + uint64_t pmu_icache_request = 0; + uint64_t pmu_icache_miss = 0; + if (register_base != 0) { + // counter 为 read-to-clear，每项只读一次，且必须在 stop 后执行。 + pmu_scalar = ReadPmuScalar(register_base); + pmu_icache_request = ReadPmuIcacheRequest(register_base); + pmu_icache_miss = ReadPmuIcacheMiss(register_base); + pmu_total = ReadPmuTotal(register_base); + } + + __gm__ icache_scalar_pmu::ProbeResult *result = &state->result; + Publish64(&result->sys_cycles, sys_end - sys_begin); + Publish64(&result->pmu_total_cycles, pmu_total); + Publish64(&result->pmu_scalar_busy, pmu_scalar); + Publish64(&result->pmu_icache_request, pmu_icache_request); + Publish64(&result->pmu_icache_miss, pmu_icache_miss); + Publish64(&result->target_checksum, target_checksum); + Publish64(&result->preparation_checksum, preparation_checksum); + Publish64(&result->pmu_ctrl_after_stop, ctrl_after_stop); + Publish64(&result->physical_core_id, physical_core_id); + Publish64(&result->mode_echo, mode_value); + dsb(DSB_ALL); +} + +#undef ICACHE_PMU_NOPS_8192 +#undef ICACHE_PMU_NOPS_4096 +#undef ICACHE_PMU_NOPS_2048 +#undef ICACHE_PMU_NOPS_1024 +#undef ICACHE_PMU_NOPS_512 +#undef ICACHE_PMU_NOPS_256 +#undef ICACHE_PMU_NOPS_128 +#undef ICACHE_PMU_NOPS_64 +#undef ICACHE_PMU_NOPS_32 +#undef ICACHE_PMU_NOPS_16 +#undef ICACHE_PMU_NOPS_8 +#undef ICACHE_PMU_NOPS_4 +#undef ICACHE_PMU_NOPS_2 +#undef ICACHE_PMU_NOPS_1 diff --git a/tests/atomic_probe/ccec/icache_scalar_pmu_host.cpp b/tests/atomic_probe/ccec/icache_scalar_pmu_host.cpp new file mode 100644 index 0000000000..6b881dcf42 --- /dev/null +++ b/tests/atomic_probe/ccec/icache_scalar_pmu_host.cpp @@ -0,0 +1,402 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +// 单 AIV、同一 target 调用点的 WARM/COLD I-cache PMU 配对 host。 +// 每一对样本使用同一个 seed；WARM 在窗外预热 target，COLD 在窗外执行 32 KiB +// evictor。Host 逐样本复算两种 checksum，并要求配对样本落在同一物理 AIV。 +// 最终只报告 COLD-WARM 的原始差值和比例，不在代码中预设“miss 是否计入 +// scalar busy”的结论。 + +#include "icache_scalar_pmu_shared.h" +#include "pmu_probe_host_support.h" +#include "../probe_host.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using atomic_probe::pmu::CheckAcl; +using atomic_probe::pmu::ReadBinary; + +constexpr uint32_t kMinimumRepeats = 11; +constexpr uint32_t kMaximumRepeats = 101; +constexpr uint64_t kRepeatSeedStride = 0x9e3779b97f4a7c15ULL; + +const char *ModeName(icache_scalar_pmu::Mode mode) +{ + switch (mode) { + case icache_scalar_pmu::Mode::WarmTarget: return "WARM_TARGET"; + case icache_scalar_pmu::Mode::ColdTarget: return "COLD_TARGET"; + default: return "UNKNOWN"; + } +} + +bool ParseUint64(const char *text, uint64_t maximum, uint64_t *value) +{ + if (text == nullptr || text[0] == '\0') return false; + errno = 0; + char *end = nullptr; + const unsigned long long parsed = std::strtoull(text, &end, 0); + if (errno != 0 || end == text || *end != '\0' || parsed > maximum) return false; + *value = static_cast(parsed); + return true; +} + +uint32_t RepeatsFromEnv() +{ + const char *raw = std::getenv("ICACHE_SCALAR_PMU_REPEATS"); + if (raw == nullptr || raw[0] == '\0') return kMinimumRepeats; + uint64_t value = 0; + if (!ParseUint64(raw, kMaximumRepeats, &value) || value < kMinimumRepeats) { + std::fprintf( + stderr, "ICACHE_SCALAR_PMU_REPEATS must be in %u..%u: %s\n", kMinimumRepeats, + kMaximumRepeats, raw + ); + return 0; + } + return static_cast(value); +} + +uint64_t SeedFromEnv(bool *ok) +{ + const char *raw = std::getenv("ICACHE_SCALAR_PMU_SEED"); + if (raw == nullptr || raw[0] == '\0') return 0x123456789abcdef0ULL; + uint64_t value = 0; + const bool parsed = ParseUint64(raw, std::numeric_limits::max(), &value); + *ok &= parsed; + if (!parsed) std::fprintf(stderr, "Invalid ICACHE_SCALAR_PMU_SEED: %s\n", raw); + return value; +} + +struct Sample { + icache_scalar_pmu::ProbeResult result{}; +}; + +bool ValidateSample( + const Sample &sample, icache_scalar_pmu::Mode mode, uint64_t seed, std::string *reason +) +{ + const uint64_t expected_preparation = mode == icache_scalar_pmu::Mode::WarmTarget + ? icache_scalar_pmu::TargetOracle(seed) + : icache_scalar_pmu::EvictorOracle(seed); + if (sample.result.target_checksum != icache_scalar_pmu::TargetOracle(seed)) { + *reason = "target-checksum"; + return false; + } + if (sample.result.preparation_checksum != expected_preparation) { + *reason = "preparation-checksum"; + return false; + } + if (sample.result.mode_echo != static_cast(mode)) { + *reason = "mode-echo"; + return false; + } + if (sample.result.physical_core_id >= atomic_probe::pmu::kPhysicalSubcoreCount) { + *reason = "physical-core-id"; + return false; + } + if ((sample.result.pmu_ctrl_after_stop & 1ULL) != 0) { + *reason = "pmu-gate-still-enabled"; + return false; + } + if (sample.result.sys_cycles == 0 || sample.result.pmu_total_cycles == 0) { + *reason = "zero-cycle-window"; + return false; + } + if (sample.result.pmu_icache_miss > sample.result.pmu_icache_request) { + *reason = "icache-miss-exceeds-request"; + return false; + } + return true; +} + +bool RunOne( + aclrtFuncHandle function, aclrtStream stream, void *state_device, uint64_t pmu_register_bases, + icache_scalar_pmu::Mode mode, uint32_t repeat, uint64_t seed, Sample *sample +) +{ + icache_scalar_pmu::ProbeState state{}; + state.control.pmu_register_bases = pmu_register_bases; + state.control.mode = static_cast(mode); + state.control.seed = seed; + if (!CheckAcl( + aclrtMemcpy(state_device, sizeof(state), &state, sizeof(state), ACL_MEMCPY_HOST_TO_DEVICE), + "aclrtMemcpy(H2D probe state)" + )) { + return false; + } + + struct KernelArgs { + uint64_t state_pointer; + } args{reinterpret_cast(state_device)}; + static_assert(sizeof(KernelArgs) == sizeof(uint64_t), "unexpected CCEC kernel argument ABI"); + if (!CheckAcl( + aclrtLaunchKernelWithHostArgs(function, 1, stream, nullptr, &args, sizeof(args), nullptr, 0), + "aclrtLaunchKernelWithHostArgs" + ) || + !CheckAcl(aclrtSynchronizeStream(stream), "aclrtSynchronizeStream(AIV probe)") || + !CheckAcl( + aclrtMemcpy(&state, sizeof(state), state_device, sizeof(state), ACL_MEMCPY_DEVICE_TO_HOST), + "aclrtMemcpy(D2H probe state)" + )) { + return false; + } + + sample->result = state.result; + std::string reason; + const bool semantic_ok = ValidateSample(*sample, mode, seed, &reason); + std::printf( + "[RAW] repeat=%u mode=%s seed=0x%llx sys_cycles=%llu total=%llu scalar=%llu " + "icache_req=%llu icache_miss=%llu target=0x%llx preparation=0x%llx " + "physical=%llu ctrl=0x%llx status=%s%s%s\n", + repeat, ModeName(mode), static_cast(seed), + static_cast(sample->result.sys_cycles), + static_cast(sample->result.pmu_total_cycles), + static_cast(sample->result.pmu_scalar_busy), + static_cast(sample->result.pmu_icache_request), + static_cast(sample->result.pmu_icache_miss), + static_cast(sample->result.target_checksum), + static_cast(sample->result.preparation_checksum), + static_cast(sample->result.physical_core_id), + static_cast(sample->result.pmu_ctrl_after_stop), semantic_ok ? "PASS" : "FAIL", + semantic_ok ? "" : " reason=", semantic_ok ? "" : reason.c_str() + ); + return semantic_ok; +} + +uint64_t Median(std::vector values) +{ + std::sort(values.begin(), values.end()); + const size_t middle = values.size() / 2; + if ((values.size() & 1U) != 0) return values[middle]; + return values[middle - 1] + (values[middle] - values[middle - 1]) / 2; +} + +double Median(std::vector values) +{ + std::sort(values.begin(), values.end()); + const size_t middle = values.size() / 2; + return (values.size() & 1U) != 0 ? values[middle] : (values[middle - 1] + values[middle]) / 2.0; +} + +using CounterMember = uint64_t icache_scalar_pmu::ProbeResult::*; + +double PairedDelta( + const std::vector &cold, const std::vector &warm, CounterMember member +) +{ + std::vector deltas; + deltas.reserve(cold.size()); + for (size_t index = 0; index < cold.size(); ++index) { + deltas.push_back( + static_cast(cold[index].result.*member) - + static_cast(warm[index].result.*member) + ); + } + return Median(std::move(deltas)); +} + +struct Metric { + const char *name; + CounterMember member; +}; + +constexpr Metric kMetrics[] = { + {"sys_cycles", &icache_scalar_pmu::ProbeResult::sys_cycles}, + {"total", &icache_scalar_pmu::ProbeResult::pmu_total_cycles}, + {"scalar", &icache_scalar_pmu::ProbeResult::pmu_scalar_busy}, + {"icache_req", &icache_scalar_pmu::ProbeResult::pmu_icache_request}, + {"icache_miss", &icache_scalar_pmu::ProbeResult::pmu_icache_miss}, +}; + +void PrintSummary(const std::array, 2> &samples) +{ + for (uint32_t mode_index = 0; mode_index < static_cast(icache_scalar_pmu::Mode::Count); + ++mode_index) { + const auto mode = static_cast(mode_index); + std::printf("[MEDIAN] mode=%s", ModeName(mode)); + for (const Metric &metric : kMetrics) { + std::vector values; + values.reserve(samples[mode_index].size()); + for (const Sample &sample : samples[mode_index]) values.push_back(sample.result.*(metric.member)); + std::printf(" %s=%llu", metric.name, static_cast(Median(std::move(values)))); + } + std::printf("\n"); + } + + const auto &warm = samples[static_cast(icache_scalar_pmu::Mode::WarmTarget)]; + const auto &cold = samples[static_cast(icache_scalar_pmu::Mode::ColdTarget)]; + for (const Metric &metric : kMetrics) { + std::printf( + "[PAIRED_DELTA] metric=%s cold_minus_warm=%.6f\n", metric.name, + PairedDelta(cold, warm, metric.member) + ); + } + + const double sys_delta = PairedDelta(cold, warm, &icache_scalar_pmu::ProbeResult::sys_cycles); + const double total_delta = PairedDelta(cold, warm, &icache_scalar_pmu::ProbeResult::pmu_total_cycles); + const double scalar_delta = PairedDelta(cold, warm, &icache_scalar_pmu::ProbeResult::pmu_scalar_busy); + const double request_delta = PairedDelta(cold, warm, &icache_scalar_pmu::ProbeResult::pmu_icache_request); + const double miss_delta = PairedDelta(cold, warm, &icache_scalar_pmu::ProbeResult::pmu_icache_miss); + const double scalar_share = total_delta == 0.0 ? 0.0 : scalar_delta / total_delta; + const double scalar_gap = total_delta - scalar_delta; + const double total_per_miss = miss_delta == 0.0 ? 0.0 : total_delta / miss_delta; + const double scalar_per_miss = miss_delta == 0.0 ? 0.0 : scalar_delta / miss_delta; + const double gap_per_miss = miss_delta == 0.0 ? 0.0 : scalar_gap / miss_delta; + std::printf( + "[ICACHE_CLASSIFICATION] completion_delta_sys_cycles=%.6f pmu_total_delta_cycles=%.6f " + "scalar_busy_delta_cycles=%.6f request_delta=%.6f miss_delta=%.6f " + "scalar_share_of_total_delta=%.9f scalar_gap_cycles=%.6f " + "total_cycles_per_extra_miss=%.6f scalar_cycles_per_extra_miss=%.6f " + "gap_cycles_per_extra_miss=%.6f\n", + sys_delta, total_delta, scalar_delta, request_delta, miss_delta, scalar_share, scalar_gap, + total_per_miss, scalar_per_miss, gap_per_miss + ); +} + +} // namespace + +int main(int argc, char **argv) +{ + const std::string kernel_path = argc > 1 ? argv[1] : "./icache_scalar_pmu_kernel.o"; + if (argc > 2) { + std::fprintf(stderr, "Usage: %s [icache_scalar_pmu_kernel.o]\n", argv[0]); + return EXIT_FAILURE; + } + const uint32_t repeats = RepeatsFromEnv(); + bool options_ok = repeats != 0; + const uint64_t base_seed = SeedFromEnv(&options_ok); + if (!options_ok) return EXIT_FAILURE; + + const int32_t device = atomic_probe::DeviceId(); + if (device < 0) return EXIT_FAILURE; + const std::vector kernel_data = ReadBinary(kernel_path); + if (kernel_data.empty()) { + std::fprintf(stderr, "Cannot read kernel binary: %s\n", kernel_path.c_str()); + return EXIT_FAILURE; + } + + if (!CheckAcl(aclInit(nullptr), "aclInit") || !CheckAcl(aclrtSetDevice(device), "aclrtSetDevice")) { + return EXIT_FAILURE; + } + aclrtStream stream = nullptr; + if (!CheckAcl(aclrtCreateStream(&stream), "aclrtCreateStream")) return EXIT_FAILURE; + + aclrtBinHandle binary_handle = nullptr; + if (!CheckAcl( + atomic_probe::LoadAicoreBinaryFromData(kernel_data.data(), kernel_data.size(), &binary_handle), + "LoadAicoreBinaryFromData" + )) { + return EXIT_FAILURE; + } + aclrtFuncHandle function = nullptr; + if (!CheckAcl(aclrtBinaryGetFunctionByEntry(binary_handle, 0, &function), "aclrtBinaryGetFunctionByEntry")) { + return EXIT_FAILURE; + } + + void *state_device = nullptr; + if (!CheckAcl( + aclrtMalloc(&state_device, sizeof(icache_scalar_pmu::ProbeState), ACL_MEM_MALLOC_NORMAL_ONLY), + "aclrtMalloc(probe state)" + )) { + return EXIT_FAILURE; + } + + atomic_probe::pmu::PmuSession pmu_session; + if (!pmu_session.Initialize( + static_cast(device), stream, kernel_path, "libicache_scalar_pmu_aicpu.so" + ) || + !pmu_session.Configure()) { + (void)pmu_session.Finalize(); + return EXIT_FAILURE; + } + + std::printf( + "=== Single-AIV paired WARM/COLD I-cache scalar-busy PMU probe ===\n" + "device=%d pairs=%u base_seed=0x%llx " + "events=total,scalar_busy(0x1),icache_req(0x34),icache_miss(0x35)\n", + device, repeats, static_cast(base_seed) + ); + + std::array, 2> samples; + for (auto &mode_samples : samples) mode_samples.reserve(repeats); + bool all_passed = true; + for (uint32_t repeat = 1; repeat <= repeats; ++repeat) { + const uint64_t seed = base_seed + static_cast(repeat - 1) * kRepeatSeedStride; + const bool warm_first = (repeat & 1U) != 0; + const std::array order = warm_first + ? std::array{ + icache_scalar_pmu::Mode::WarmTarget, icache_scalar_pmu::Mode::ColdTarget} + : std::array{ + icache_scalar_pmu::Mode::ColdTarget, icache_scalar_pmu::Mode::WarmTarget}; + + std::array pair; + bool pair_samples_ok = true; + for (const icache_scalar_pmu::Mode mode : order) { + const uint32_t mode_index = static_cast(mode); + pair_samples_ok &= RunOne( + function, stream, state_device, pmu_session.RegisterBasesDeviceAddress(), mode, repeat, seed, + &pair[mode_index] + ); + if (!pair_samples_ok) break; + } + if (!pair_samples_ok) { + all_passed = false; + break; + } + + const Sample &warm = pair[static_cast(icache_scalar_pmu::Mode::WarmTarget)]; + const Sample &cold = pair[static_cast(icache_scalar_pmu::Mode::ColdTarget)]; + const bool same_physical_core = warm.result.physical_core_id == cold.result.physical_core_id; + const bool cold_has_more_misses = cold.result.pmu_icache_miss > warm.result.pmu_icache_miss; + std::printf( + "[PAIR] repeat=%u order=%s physical_warm=%llu physical_cold=%llu " + "miss_warm=%llu miss_cold=%llu same_physical=%s cold_gt_warm_miss=%s status=%s\n", + repeat, warm_first ? "WARM,COLD" : "COLD,WARM", + static_cast(warm.result.physical_core_id), + static_cast(cold.result.physical_core_id), + static_cast(warm.result.pmu_icache_miss), + static_cast(cold.result.pmu_icache_miss), same_physical_core ? "PASS" : "FAIL", + cold_has_more_misses ? "PASS" : "FAIL", + same_physical_core && cold_has_more_misses ? "PASS" : "FAIL" + ); + if (!same_physical_core || !cold_has_more_misses) { + all_passed = false; + break; + } + samples[static_cast(icache_scalar_pmu::Mode::WarmTarget)].push_back(warm); + samples[static_cast(icache_scalar_pmu::Mode::ColdTarget)].push_back(cold); + } + + if (all_passed) PrintSummary(samples); + + bool cleanup_ok = pmu_session.Finalize(); + cleanup_ok &= CheckAcl(aclrtFree(state_device), "aclrtFree(probe state)"); + cleanup_ok &= CheckAcl(aclrtBinaryUnLoad(binary_handle), "aclrtBinaryUnLoad"); + cleanup_ok &= CheckAcl(aclrtDestroyStream(stream), "aclrtDestroyStream"); + cleanup_ok &= CheckAcl(aclrtResetDevice(device), "aclrtResetDevice"); + cleanup_ok &= CheckAcl(aclFinalize(), "aclFinalize"); + std::printf( + "[SUMMARY] completed_pairs=%zu requested_pairs=%u semantic_status=%s " + "pmu_restore_and_cleanup=%s\n", + samples[0].size(), repeats, all_passed ? "PASS" : "FAIL", cleanup_ok ? "PASS" : "FAIL" + ); + return all_passed && cleanup_ok ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/atomic_probe/ccec/icache_scalar_pmu_shared.h b/tests/atomic_probe/ccec/icache_scalar_pmu_shared.h new file mode 100644 index 0000000000..43e8d98243 --- /dev/null +++ b/tests/atomic_probe/ccec/icache_scalar_pmu_shared.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef TESTS_ATOMIC_PROBE_CCEC_ICACHE_SCALAR_PMU_SHARED_H_ +#define TESTS_ATOMIC_PROBE_CCEC_ICACHE_SCALAR_PMU_SHARED_H_ + +#include +#include + +namespace icache_scalar_pmu { + +#if defined(__CCE_AICORE__) +#define ICACHE_SCALAR_PMU_SHARED_FN __aicore__ +#else +#define ICACHE_SCALAR_PMU_SHARED_FN +#endif + +// 两条路径的 PMU 窗口内都只调用一次、且调用同一个 noinline target： +// WARM 在窗外先调用一次 target；COLD 在窗外执行大于 16 KiB I-cache 容量的 evictor。 +// 模式分支本身不进入 PMU gate，因而不会改变窗内 target 的动态指令序列。 +enum class Mode : uint32_t { + WarmTarget = 0, + ColdTarget = 1, + Count = 2, +}; + +// A5 scalar I-cache 容量为 16 KiB。AICore 标量指令为 4B；target 用 2048 条 +// volatile NOP 形成约 8 KiB 指令体，evictor 用 8192 条形成约 32 KiB 顺序指令流。 +// 构建脚本仍应从最终 ELF 的符号大小复核这两个下界，不能只相信源码常量。 +constexpr uint32_t kTargetNopCount = 2048; +constexpr uint32_t kEvictorNopCount = 8192; +constexpr uint64_t kTargetXor = 0xd6e8feb86659fd93ULL; +constexpr uint64_t kTargetMultiplier = 0x9e3779b185ebca87ULL; +constexpr uint64_t kTargetAddend = 0xa0761d6478bd642fULL; +constexpr uint64_t kEvictorXor = 0xe7037ed1a0b428dbULL; +constexpr uint64_t kEvictorMultiplier = 0x8ebc6af09c88c6e3ULL; + +// Host 与 device 共用完全相同的无符号递推；uint64_t 溢出按模 2^64 定义。 +// checksum 只负责证明 target/evictor 的调用确实发生，不参与 PMU 窗口分类。 +ICACHE_SCALAR_PMU_SHARED_FN constexpr uint64_t TargetOracle(uint64_t seed) +{ + uint64_t value = seed ^ kTargetXor; + value ^= value >> 29; + value *= kTargetMultiplier; + value += kTargetAddend; + value ^= value >> 31; + return value; +} + +ICACHE_SCALAR_PMU_SHARED_FN constexpr uint64_t EvictorOracle(uint64_t seed) +{ + uint64_t value = seed ^ kEvictorXor; + value ^= value >> 23; + value *= kEvictorMultiplier; + value ^= value >> 27; + return value; +} + +// Host launch 前只写本 cache line；kernel 在 PMU 窗口外 DCCI 后读取。 +// pmu_register_bases 按 get_coreid() 的低 12 bit 索引，两个 scalar PMU 探针 +// 共用同一份 108 physical sub-core MMIO base 表协议。 +struct alignas(64) ProbeControl { + uint64_t pmu_register_bases; + uint32_t mode; + uint32_t reserved0; + uint64_t seed; + uint64_t reserved[5]; +}; + +// 前五项是同一 target 单次调用窗口内的时间和 PMU 原始计数。 +// target_checksum 在两种 mode 下都必须等于 TargetOracle(seed)：这是窗内动态 +// 工作负载一致的功能 oracle。preparation_checksum 在 WARM 下也等于 TargetOracle， +// 在 COLD 下等于 EvictorOracle，用来证明对应的窗外准备路径没有被编译器删除。 +// 两条 cache line 均由唯一 AIV 用 st_dev 发布，kernel 末尾统一 DSB。 +struct alignas(64) ProbeResult { + uint64_t sys_cycles; + uint64_t pmu_total_cycles; + uint64_t pmu_scalar_busy; + uint64_t pmu_icache_request; + uint64_t pmu_icache_miss; + uint64_t target_checksum; + uint64_t preparation_checksum; + uint64_t pmu_ctrl_after_stop; + + uint64_t physical_core_id; + uint64_t mode_echo; + uint64_t reserved[6]; +}; + +struct alignas(64) ProbeState { + ProbeControl control; + ProbeResult result; +}; + +static_assert(sizeof(ProbeControl) == 64, "probe control must occupy one cache line"); +static_assert(sizeof(ProbeResult) == 128, "probe result must occupy two cache lines"); +static_assert(offsetof(ProbeState, result) == 64, "probe result must start on its own cache line"); +static_assert(sizeof(ProbeState) == 192, "probe state ABI changed unexpectedly"); + +#undef ICACHE_SCALAR_PMU_SHARED_FN + +} // namespace icache_scalar_pmu + +#endif // TESTS_ATOMIC_PROBE_CCEC_ICACHE_SCALAR_PMU_SHARED_H_ diff --git a/tests/atomic_probe/ccec/nested_lambda_cross_tu.cpp b/tests/atomic_probe/ccec/nested_lambda_cross_tu.cpp index dbdbd955b5..0d8caae84b 100644 --- a/tests/atomic_probe/ccec/nested_lambda_cross_tu.cpp +++ b/tests/atomic_probe/ccec/nested_lambda_cross_tu.cpp @@ -28,10 +28,9 @@ using nested_lambda_cross_tu_probe::Field; using nested_lambda_cross_tu_probe::Variant; constexpr int32_t kSiteId = 7; +static_assert(sizeof(L0TaskArgs) == nested_lambda_cross_tu_probe::kExpectedL0TaskArgsBytes); -PTO_DEVICE_FUNC void BindContext( - int32_t site_id, int32_t phase, uint64_t caller_context, L0TaskArgs *args) -{ +PTO_DEVICE_FUNC void BindContext(int32_t site_id, int32_t phase, uint64_t caller_context, L0TaskArgs *args) { if (site_id != kSiteId) return; const auto *context = reinterpret_cast(caller_context); if (phase == static_cast(DispatchPhase::Prepare)) { @@ -41,23 +40,21 @@ PTO_DEVICE_FUNC void BindContext( args->add_input(*context->first, *context->second, *context->third); } -PTO_DEVICE_FUNC void InitTensor(Tensor &tensor, uint32_t round, uint32_t tensor_index) -{ +PTO_DEVICE_FUNC void InitTensor(Tensor &tensor, uint32_t round, uint32_t tensor_index) { tensor.buffer.addr = nested_lambda_cross_tu_probe::TensorAddress(round, tensor_index); tensor.start_offset = nested_lambda_cross_tu_probe::TensorOffset(round, tensor_index); tensor.version = nested_lambda_cross_tu_probe::TensorVersion(round, tensor_index); tensor.shapes[0] = nested_lambda_cross_tu_probe::TensorShape(round, tensor_index); } -PTO_DEVICE_FUNC void StoreField(__gm__ uint32_t *storage, Field field, uint32_t value) -{ +PTO_DEVICE_FUNC void StoreField(__gm__ uint32_t *storage, Field field, uint32_t value) { st_dev_b32(&storage[nested_lambda_cross_tu_probe::FieldIndex(field)], value); } PTO_DEVICE_FUNC void StoreResults( __gm__ uint32_t *storage, Variant variant, uint32_t completed_rounds, uint32_t mismatches, - uint32_t dispatcher_calls, uint32_t materializations, uint64_t checksum) -{ + uint32_t dispatcher_calls, uint32_t materializations, uint64_t checksum +) { StoreField(storage, Field::CompletedRounds, completed_rounds); StoreField(storage, Field::MismatchCount, mismatches); StoreField(storage, Field::DispatcherCalls, dispatcher_calls); @@ -70,8 +67,7 @@ PTO_DEVICE_FUNC void StoreResults( } template -PTO_DEVICE_FUNC TaskOutputTensors SubmitContext(uint64_t caller_context, L0TaskArgs *args) -{ +PTO_DEVICE_FUNC TaskOutputTensors SubmitContext(uint64_t caller_context, L0TaskArgs *args) { if constexpr (StrongDispatcher) { return nested_probe_submit_strong_context(kSiteId, caller_context, args); } @@ -79,8 +75,7 @@ PTO_DEVICE_FUNC TaskOutputTensors SubmitContext(uint64_t caller_context, L0TaskA } template -PTO_DEVICE_FUNC void RunContextVariant(__gm__ uint32_t *storage, Variant variant) -{ +PTO_DEVICE_FUNC void RunContextVariant(__gm__ uint32_t *storage, Variant variant) { L0TaskArgs args; uint32_t completed_rounds = 0; uint32_t mismatches = 0; @@ -129,8 +124,7 @@ PTO_DEVICE_FUNC void RunContextVariant(__gm__ uint32_t *storage, Variant variant } template -PTO_DEVICE_FUNC void RunArgsStorageVariant(__gm__ uint32_t *storage, Variant variant) -{ +PTO_DEVICE_FUNC void RunArgsStorageVariant(__gm__ uint32_t *storage, Variant variant) { L0TaskArgs args; uint32_t completed_rounds = 0; uint32_t mismatches = 0; @@ -178,21 +172,18 @@ PTO_DEVICE_FUNC void RunArgsStorageVariant(__gm__ uint32_t *storage, Variant var completed_rounds++; } - StoreResults( - storage, variant, completed_rounds, mismatches, dispatcher_calls, 3, checksum); + StoreResults(storage, variant, completed_rounds, mismatches, dispatcher_calls, 3, checksum); } -} // namespace +} // namespace -extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_weak_context_dispatch( - int32_t site_id, int32_t phase, uint64_t caller_context, L0TaskArgs *args) -{ +extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void +nested_probe_weak_context_dispatch(int32_t site_id, int32_t phase, uint64_t caller_context, L0TaskArgs *args) { BindContext(site_id, phase, caller_context, args); } -extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_weak_args_dispatch( - int32_t site_id, int32_t phase, L0TaskArgs *args) -{ +extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void +nested_probe_weak_args_dispatch(int32_t site_id, int32_t phase, L0TaskArgs *args) { if (site_id != kSiteId) return; if (phase == static_cast(DispatchPhase::Prepare)) { args->scalar(4) = args->scalar(11); @@ -204,9 +195,8 @@ extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_weak_args_dis args->add_input(*first, *second, *third); } -extern "C" PTO_DEVICE_FUNC void nested_probe_strong_context_dispatch( - int32_t site_id, int32_t phase, uint64_t caller_context, L0TaskArgs *args) -{ +extern "C" PTO_DEVICE_FUNC void +nested_probe_strong_context_dispatch(int32_t site_id, int32_t phase, uint64_t caller_context, L0TaskArgs *args) { BindContext(site_id, phase, caller_context, args); } @@ -214,79 +204,59 @@ extern "C" PTO_DEVICE_FUNC void nested_probe_strong_context_dispatch( // aicpu_orchestration_entry. Putting it directly in a __global__ wrapper would // move its locals from the orchestration function stack to the kernel stack and // miss the suspected reg93 stack-address materialization path. -extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_orchestration_ctx_m0( - __gm__ uint32_t *storage) -{ +extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_orchestration_ctx_m0(__gm__ uint32_t *storage) { RunContextVariant<0, false>(storage, Variant::WeakContextMaterialize0); } -extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_orchestration_ctx_m1( - __gm__ uint32_t *storage) -{ +extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_orchestration_ctx_m1(__gm__ uint32_t *storage) { RunContextVariant<1, false>(storage, Variant::WeakContextMaterialize1); } -extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_orchestration_ctx_m2( - __gm__ uint32_t *storage) -{ +extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_orchestration_ctx_m2(__gm__ uint32_t *storage) { RunContextVariant<2, false>(storage, Variant::WeakContextMaterialize2); } -extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_orchestration_ctx_m3( - __gm__ uint32_t *storage) -{ +extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_orchestration_ctx_m3(__gm__ uint32_t *storage) { RunContextVariant<3, false>(storage, Variant::WeakContextMaterialize3); } -extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_orchestration_args( - __gm__ uint32_t *storage) -{ +extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_orchestration_args(__gm__ uint32_t *storage) { RunArgsStorageVariant(storage, Variant::WeakArgsStorage); } -extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_orchestration_strong( - __gm__ uint32_t *storage) -{ +extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_orchestration_strong(__gm__ uint32_t *storage) { RunContextVariant<0, true>(storage, Variant::StrongContext); } -extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void nested_probe_orchestration_runtime_args( - __gm__ uint32_t *storage) -{ +extern "C" __attribute__((weak)) PTO_DEVICE_FUNC void +nested_probe_orchestration_runtime_args(__gm__ uint32_t *storage) { RunArgsStorageVariant(storage, Variant::ArgsRuntimeRead); } -extern "C" __global__ __aicore__ void nested_lambda_cross_tu_ctx_m0_0_mix_aic(__gm__ uint32_t *storage) -{ +extern "C" __global__ __aicore__ void nested_lambda_cross_tu_ctx_m0_0_mix_aic(__gm__ uint32_t *storage) { if (get_block_idx() == 0) nested_probe_orchestration_ctx_m0(storage); } -extern "C" __global__ __aicore__ void nested_lambda_cross_tu_ctx_m1_1_mix_aic(__gm__ uint32_t *storage) -{ +extern "C" __global__ __aicore__ void nested_lambda_cross_tu_ctx_m1_1_mix_aic(__gm__ uint32_t *storage) { if (get_block_idx() == 0) nested_probe_orchestration_ctx_m1(storage); } -extern "C" __global__ __aicore__ void nested_lambda_cross_tu_ctx_m2_2_mix_aic(__gm__ uint32_t *storage) -{ +extern "C" __global__ __aicore__ void nested_lambda_cross_tu_ctx_m2_2_mix_aic(__gm__ uint32_t *storage) { if (get_block_idx() == 0) nested_probe_orchestration_ctx_m2(storage); } -extern "C" __global__ __aicore__ void nested_lambda_cross_tu_ctx_m3_3_mix_aic(__gm__ uint32_t *storage) -{ +extern "C" __global__ __aicore__ void nested_lambda_cross_tu_ctx_m3_3_mix_aic(__gm__ uint32_t *storage) { if (get_block_idx() == 0) nested_probe_orchestration_ctx_m3(storage); } -extern "C" __global__ __aicore__ void nested_lambda_cross_tu_args_4_mix_aic(__gm__ uint32_t *storage) -{ +extern "C" __global__ __aicore__ void nested_lambda_cross_tu_args_4_mix_aic(__gm__ uint32_t *storage) { if (get_block_idx() == 0) nested_probe_orchestration_args(storage); } -extern "C" __global__ __aicore__ void nested_lambda_cross_tu_strong_5_mix_aic(__gm__ uint32_t *storage) -{ +extern "C" __global__ __aicore__ void nested_lambda_cross_tu_strong_5_mix_aic(__gm__ uint32_t *storage) { if (get_block_idx() == 0) nested_probe_orchestration_strong(storage); } -extern "C" __global__ __aicore__ void nested_lambda_cross_tu_runtime_args_6_mix_aic(__gm__ uint32_t *storage) -{ +extern "C" __global__ __aicore__ void nested_lambda_cross_tu_runtime_args_6_mix_aic(__gm__ uint32_t *storage) { if (get_block_idx() == 0) nested_probe_orchestration_runtime_args(storage); } diff --git a/tests/atomic_probe/ccec/nested_lambda_cross_tu_api.h b/tests/atomic_probe/ccec/nested_lambda_cross_tu_api.h index a4db3dc4c9..48ef4606ff 100644 --- a/tests/atomic_probe/ccec/nested_lambda_cross_tu_api.h +++ b/tests/atomic_probe/ccec/nested_lambda_cross_tu_api.h @@ -11,6 +11,7 @@ #ifndef TESTS_ATOMIC_PROBE_CCEC_NESTED_LAMBDA_CROSS_TU_API_H #define TESTS_ATOMIC_PROBE_CCEC_NESTED_LAMBDA_CROSS_TU_API_H +#include "nested_lambda_cross_tu_layout.h" #include "pto_types.h" #include @@ -29,15 +30,108 @@ struct CallerContext { uint64_t salt; }; -} // namespace nested_lambda_cross_tu_probe +} // namespace nested_lambda_cross_tu_probe -PTO_DEVICE_FUNC TaskOutputTensors nested_probe_submit_weak_context( - int32_t site_id, uint64_t caller_context, L0TaskArgs *args); -PTO_DEVICE_FUNC TaskOutputTensors nested_probe_submit_weak_args( - int32_t site_id, L0TaskArgs *args); -PTO_DEVICE_FUNC TaskOutputTensors nested_probe_submit_strong_context( - int32_t site_id, uint64_t caller_context, L0TaskArgs *args); -PTO_DEVICE_FUNC TaskOutputTensors nested_probe_submit_control(L0TaskArgs *args); -PTO_DEVICE_FUNC TaskOutputTensors nested_probe_submit_args_runtime_read(L0TaskArgs *args); +extern "C" PTO_DEVICE_FUNC void +nested_probe_weak_context_dispatch(int32_t site_id, int32_t phase, uint64_t caller_context, L0TaskArgs *args) + __attribute__((weak)); +extern "C" PTO_DEVICE_FUNC void nested_probe_weak_args_dispatch(int32_t site_id, int32_t phase, L0TaskArgs *args) + __attribute__((weak)); +extern "C" PTO_DEVICE_FUNC void +nested_probe_strong_context_dispatch(int32_t site_id, int32_t phase, uint64_t caller_context, L0TaskArgs *args); -#endif // TESTS_ATOMIC_PROBE_CCEC_NESTED_LAMBDA_CROSS_TU_API_H +#define NESTED_PROBE_RUNTIME_INLINE PTO_DEVICE_FUNC __attribute__((always_inline)) inline + +#if defined(NESTED_PROBE_WEAK_CONTEXT_NOINLINE) +#define NESTED_PROBE_WEAK_CONTEXT_FUNC PTO_DEVICE_FUNC __attribute__((noinline)) +#else +#define NESTED_PROBE_WEAK_CONTEXT_FUNC NESTED_PROBE_RUNTIME_INLINE +#endif + +namespace nested_lambda_cross_tu_probe::detail { + +NESTED_PROBE_RUNTIME_INLINE uint64_t TensorDigest(const Tensor &tensor) { + return tensor.buffer.addr + tensor.start_offset * 17ULL + + static_cast(static_cast(tensor.version)) * 257ULL + tensor.shapes[0] * 65537ULL; +} + +NESTED_PROBE_RUNTIME_INLINE void ConsumeBoundArguments(L0TaskArgs *args) { + const Tensor &first = args->tensor(0).ref(); + const Tensor &second = args->tensor(1).ref(); + const Tensor &third = args->tensor(2).ref(); + args->scalar(0) = + TensorDigest(first) * 3ULL + TensorDigest(second) * 5ULL + TensorDigest(third) * 7ULL + args->scalar(4); + args->scalar(5) = 2; + args->scalar(6) = 0; +} + +} // namespace nested_lambda_cross_tu_probe::detail + +NESTED_PROBE_WEAK_CONTEXT_FUNC TaskOutputTensors +nested_probe_submit_weak_context(int32_t site_id, uint64_t caller_context, L0TaskArgs *args) { + TaskOutputTensors outputs; + if (nested_probe_weak_context_dispatch == nullptr) { + args->scalar(6) = 1; + return outputs; + } + nested_probe_weak_context_dispatch( + site_id, static_cast(nested_lambda_cross_tu_probe::DispatchPhase::Prepare), caller_context, args + ); + nested_probe_weak_context_dispatch( + site_id, static_cast(nested_lambda_cross_tu_probe::DispatchPhase::WinnerBind), caller_context, args + ); + nested_lambda_cross_tu_probe::detail::ConsumeBoundArguments(args); + return outputs; +} + +NESTED_PROBE_RUNTIME_INLINE TaskOutputTensors nested_probe_submit_weak_args(int32_t site_id, L0TaskArgs *args) { + TaskOutputTensors outputs; + if (nested_probe_weak_args_dispatch == nullptr) { + args->scalar(6) = 1; + return outputs; + } + nested_probe_weak_args_dispatch( + site_id, static_cast(nested_lambda_cross_tu_probe::DispatchPhase::Prepare), args + ); + nested_probe_weak_args_dispatch( + site_id, static_cast(nested_lambda_cross_tu_probe::DispatchPhase::WinnerBind), args + ); + nested_lambda_cross_tu_probe::detail::ConsumeBoundArguments(args); + return outputs; +} + +NESTED_PROBE_RUNTIME_INLINE TaskOutputTensors +nested_probe_submit_strong_context(int32_t site_id, uint64_t caller_context, L0TaskArgs *args) { + TaskOutputTensors outputs; + nested_probe_strong_context_dispatch( + site_id, static_cast(nested_lambda_cross_tu_probe::DispatchPhase::Prepare), caller_context, args + ); + nested_probe_strong_context_dispatch( + site_id, static_cast(nested_lambda_cross_tu_probe::DispatchPhase::WinnerBind), caller_context, args + ); + nested_lambda_cross_tu_probe::detail::ConsumeBoundArguments(args); + return outputs; +} + +NESTED_PROBE_RUNTIME_INLINE TaskOutputTensors nested_probe_submit_control(L0TaskArgs *args) { + TaskOutputTensors outputs; + args->scalar(0) ^= nested_lambda_cross_tu_probe::kControlXor; + return outputs; +} + +NESTED_PROBE_RUNTIME_INLINE TaskOutputTensors nested_probe_submit_args_runtime_read(L0TaskArgs *args) { + TaskOutputTensors outputs; + const auto *first = reinterpret_cast(args->scalar(8)); + const auto *second = reinterpret_cast(args->scalar(9)); + const auto *third = reinterpret_cast(args->scalar(10)); + args->scalar(4) = args->scalar(11); + args->add_input(*first, *second, *third); + nested_lambda_cross_tu_probe::detail::ConsumeBoundArguments(args); + args->scalar(5) = 0; + return outputs; +} + +#undef NESTED_PROBE_RUNTIME_INLINE +#undef NESTED_PROBE_WEAK_CONTEXT_FUNC + +#endif // TESTS_ATOMIC_PROBE_CCEC_NESTED_LAMBDA_CROSS_TU_API_H diff --git a/tests/atomic_probe/ccec/nested_lambda_cross_tu_host.cpp b/tests/atomic_probe/ccec/nested_lambda_cross_tu_host.cpp index cac501d91e..b789d28dce 100644 --- a/tests/atomic_probe/ccec/nested_lambda_cross_tu_host.cpp +++ b/tests/atomic_probe/ccec/nested_lambda_cross_tu_host.cpp @@ -26,13 +26,11 @@ struct KernelArgs { uint64_t storage_pointer; }; -uint32_t ReadField(const std::vector &storage, Field field) -{ +uint32_t ReadField(const std::vector &storage, Field field) { return storage[nested_lambda_cross_tu_probe::FieldIndex(field)]; } -bool ParseVariant(const char *text, Variant *variant) -{ +bool ParseVariant(const char *text, Variant *variant) { for (uint32_t raw = 0; raw < static_cast(Variant::Count); raw++) { const Variant candidate = static_cast(raw); if (std::strcmp(text, nested_lambda_cross_tu_probe::VariantName(candidate)) == 0) { @@ -43,39 +41,38 @@ bool ParseVariant(const char *text, Variant *variant) return false; } -void Validate(const std::vector &storage, Variant variant, atomic_probe::Result &result) -{ +void Validate(const std::vector &storage, Variant variant, atomic_probe::Result &result) { const uint64_t checksum = static_cast(ReadField(storage, Field::ChecksumLow)) | - (static_cast(ReadField(storage, Field::ChecksumHigh)) << 32); + (static_cast(ReadField(storage, Field::ChecksumHigh)) << 32); const uint32_t args_bytes = ReadField(storage, Field::L0TaskArgsBytes); bool exact = true; exact &= ReadField(storage, Field::CompletedRounds) == nested_lambda_cross_tu_probe::kRounds; exact &= ReadField(storage, Field::MismatchCount) == 0; - exact &= ReadField(storage, Field::DispatcherCalls) == - nested_lambda_cross_tu_probe::ExpectedDispatcherCalls(variant); + exact &= + ReadField(storage, Field::DispatcherCalls) == nested_lambda_cross_tu_probe::ExpectedDispatcherCalls(variant); exact &= ReadField(storage, Field::AddressMaterializations) == - nested_lambda_cross_tu_probe::ExpectedMaterializations(variant); + nested_lambda_cross_tu_probe::ExpectedMaterializations(variant); exact &= checksum == nested_lambda_cross_tu_probe::ExpectedTotalChecksum(); - exact &= args_bytes > 0 && args_bytes < 32U * 1024U && args_bytes % 64U == 0; + exact &= args_bytes == nested_lambda_cross_tu_probe::kExpectedL0TaskArgsBytes; exact &= ReadField(storage, Field::VariantEcho) == static_cast(variant); char label[120]; std::snprintf( - label, sizeof(label), "CCEC AIC cross-TU ABI variant=%s", - nested_lambda_cross_tu_probe::VariantName(variant)); + label, sizeof(label), "CCEC AIC caller-capture variant=%s", nested_lambda_cross_tu_probe::VariantName(variant) + ); result.Expect(exact, label); std::printf( "[VALUES] rounds=%u mismatches=%u dispatches=%u materializations=%u " "checksum=0x%016llx L0TaskArgs=%uB\n", ReadField(storage, Field::CompletedRounds), ReadField(storage, Field::MismatchCount), ReadField(storage, Field::DispatcherCalls), ReadField(storage, Field::AddressMaterializations), - static_cast(checksum), args_bytes); + static_cast(checksum), args_bytes + ); } -} // namespace +} // namespace -int main(int argc, char *argv[]) -{ +int main(int argc, char *argv[]) { if (argc != 3) { std::fprintf(stderr, "Usage: %s \n", argv[0]); return EXIT_FAILURE; @@ -83,7 +80,7 @@ int main(int argc, char *argv[]) const char *kernel_path = argv[1]; Variant variant = Variant::Count; if (!ParseVariant(argv[2], &variant)) { - std::fprintf(stderr, "Unknown cross-TU variant: %s\n", argv[2]); + std::fprintf(stderr, "Unknown caller-capture variant: %s\n", argv[2]); return EXIT_FAILURE; } @@ -110,32 +107,35 @@ int main(int argc, char *argv[]) void *storage_device = nullptr; PROBE_ACL_CHECK(aclrtMalloc(&storage_device, storage_bytes, ACL_MEM_MALLOC_HUGE_FIRST)); - std::printf("=== Pure CCEC AIC Caller-Capture Transport Probe ===\n"); + std::printf("=== Pure CCEC AIC Caller-Capture Call-Boundary Probe ===\n"); atomic_probe::Result result; aclrtFuncHandle function_handle = nullptr; PROBE_ACL_CHECK(aclrtBinaryGetFunctionByEntry( - binary_handle, nested_lambda_cross_tu_probe::KernelEntry(variant), &function_handle)); + binary_handle, nested_lambda_cross_tu_probe::KernelEntry(variant), &function_handle + )); std::vector storage(nested_lambda_cross_tu_probe::kStorageWords, 0); - PROBE_ACL_CHECK(aclrtMemcpy( - storage_device, storage_bytes, storage.data(), storage_bytes, ACL_MEMCPY_HOST_TO_DEVICE)); + PROBE_ACL_CHECK( + aclrtMemcpy(storage_device, storage_bytes, storage.data(), storage_bytes, ACL_MEMCPY_HOST_TO_DEVICE) + ); KernelArgs args{reinterpret_cast(storage_device)}; - PROBE_ACL_CHECK(aclrtLaunchKernelWithHostArgs( - function_handle, 1, stream, nullptr, &args, sizeof(args), nullptr, 0)); + PROBE_ACL_CHECK( + aclrtLaunchKernelWithHostArgs(function_handle, 1, stream, nullptr, &args, sizeof(args), nullptr, 0) + ); PROBE_ACL_CHECK(aclrtSynchronizeStream(stream)); - PROBE_ACL_CHECK(aclrtMemcpy( - storage.data(), storage_bytes, storage_device, storage_bytes, ACL_MEMCPY_DEVICE_TO_HOST)); + PROBE_ACL_CHECK( + aclrtMemcpy(storage.data(), storage_bytes, storage_device, storage_bytes, ACL_MEMCPY_DEVICE_TO_HOST) + ); Validate(storage, variant, result); bool cleanup_ok = true; + cleanup_ok &= atomic_probe::CheckAcl(aclrtFree(storage_device), "aclrtFree(storage_device)", __FILE__, __LINE__); cleanup_ok &= atomic_probe::CheckAcl( - aclrtFree(storage_device), "aclrtFree(storage_device)", __FILE__, __LINE__); - cleanup_ok &= atomic_probe::CheckAcl( - aclrtBinaryUnLoad(binary_handle), "aclrtBinaryUnLoad(binary_handle)", __FILE__, __LINE__); - cleanup_ok &= atomic_probe::CheckAcl( - aclrtDestroyStream(stream), "aclrtDestroyStream(stream)", __FILE__, __LINE__); - cleanup_ok &= atomic_probe::CheckAcl( - aclrtResetDevice(device_id), "aclrtResetDevice(device_id)", __FILE__, __LINE__); + aclrtBinaryUnLoad(binary_handle), "aclrtBinaryUnLoad(binary_handle)", __FILE__, __LINE__ + ); + cleanup_ok &= atomic_probe::CheckAcl(aclrtDestroyStream(stream), "aclrtDestroyStream(stream)", __FILE__, __LINE__); + cleanup_ok &= + atomic_probe::CheckAcl(aclrtResetDevice(device_id), "aclrtResetDevice(device_id)", __FILE__, __LINE__); cleanup_ok &= atomic_probe::CheckAcl(aclFinalize(), "aclFinalize()", __FILE__, __LINE__); - result.Expect(cleanup_ok, "CCEC cross-TU ACL cleanup"); + result.Expect(cleanup_ok, "CCEC caller-capture ACL cleanup"); return result.ExitCode(); } diff --git a/tests/atomic_probe/ccec/nested_lambda_cross_tu_layout.h b/tests/atomic_probe/ccec/nested_lambda_cross_tu_layout.h index 1713b0111a..a7c4e420d0 100644 --- a/tests/atomic_probe/ccec/nested_lambda_cross_tu_layout.h +++ b/tests/atomic_probe/ccec/nested_lambda_cross_tu_layout.h @@ -46,66 +46,52 @@ enum class Field : uint32_t { constexpr uint32_t kRounds = 64; constexpr uint32_t kSubmitsPerRound = 4; +constexpr uint32_t kExpectedL0TaskArgsBytes = 1024; constexpr uint32_t kCacheLineWords = 16; constexpr uint32_t kStorageWords = static_cast(Field::Count) * kCacheLineWords; constexpr uint64_t kControlXor = 0x6A09E667F3BCC909ULL; -CROSS_TU_DEVICE constexpr uint32_t FieldIndex(Field field) -{ - return static_cast(field) * kCacheLineWords; -} +CROSS_TU_DEVICE constexpr uint32_t FieldIndex(Field field) { return static_cast(field) * kCacheLineWords; } -CROSS_TU_DEVICE constexpr uint64_t TensorAddress(uint32_t round, uint32_t tensor_index) -{ - return 0x100000000ULL + static_cast(round) * 0x1000ULL + - static_cast(tensor_index) * 0x100ULL + 0x55ULL; +CROSS_TU_DEVICE constexpr uint64_t TensorAddress(uint32_t round, uint32_t tensor_index) { + return 0x100000000ULL + static_cast(round) * 0x1000ULL + static_cast(tensor_index) * 0x100ULL + + 0x55ULL; } -CROSS_TU_DEVICE constexpr uint64_t TensorOffset(uint32_t round, uint32_t tensor_index) -{ +CROSS_TU_DEVICE constexpr uint64_t TensorOffset(uint32_t round, uint32_t tensor_index) { return static_cast(round) * 8ULL + tensor_index; } -CROSS_TU_DEVICE constexpr int32_t TensorVersion(uint32_t round, uint32_t tensor_index) -{ +CROSS_TU_DEVICE constexpr int32_t TensorVersion(uint32_t round, uint32_t tensor_index) { return static_cast(100U + round * 3U + tensor_index); } -CROSS_TU_DEVICE constexpr uint32_t TensorShape(uint32_t round, uint32_t tensor_index) -{ +CROSS_TU_DEVICE constexpr uint32_t TensorShape(uint32_t round, uint32_t tensor_index) { return 17U + round + tensor_index; } -CROSS_TU_DEVICE constexpr uint64_t ContextSalt(uint32_t round) -{ - return 0xBADC000000000000ULL + round; -} +CROSS_TU_DEVICE constexpr uint64_t ContextSalt(uint32_t round) { return 0xBADC000000000000ULL + round; } -CROSS_TU_DEVICE constexpr uint64_t TensorDigest(uint32_t round, uint32_t tensor_index) -{ +CROSS_TU_DEVICE constexpr uint64_t TensorDigest(uint32_t round, uint32_t tensor_index) { return TensorAddress(round, tensor_index) + TensorOffset(round, tensor_index) * 17ULL + - static_cast(static_cast(TensorVersion(round, tensor_index))) * 257ULL + - TensorShape(round, tensor_index) * 65537ULL; + static_cast(static_cast(TensorVersion(round, tensor_index))) * 257ULL + + TensorShape(round, tensor_index) * 65537ULL; } -CROSS_TU_DEVICE constexpr uint64_t ExpectedLazyDigest(uint32_t round) -{ - return TensorDigest(round, 0) * 3ULL + TensorDigest(round, 1) * 5ULL + - TensorDigest(round, 2) * 7ULL + ContextSalt(round); +CROSS_TU_DEVICE constexpr uint64_t ExpectedLazyDigest(uint32_t round) { + return TensorDigest(round, 0) * 3ULL + TensorDigest(round, 1) * 5ULL + TensorDigest(round, 2) * 7ULL + + ContextSalt(round); } -CROSS_TU_DEVICE constexpr uint64_t ControlInput(uint32_t round, uint32_t submit_index) -{ +CROSS_TU_DEVICE constexpr uint64_t ControlInput(uint32_t round, uint32_t submit_index) { return 0xC000000000000000ULL + static_cast(round) * kSubmitsPerRound + submit_index; } -CROSS_TU_DEVICE constexpr uint64_t ExpectedControlResult(uint32_t round, uint32_t submit_index) -{ +CROSS_TU_DEVICE constexpr uint64_t ExpectedControlResult(uint32_t round, uint32_t submit_index) { return ControlInput(round, submit_index) ^ kControlXor; } -CROSS_TU_DEVICE constexpr uint64_t ExpectedTotalChecksum() -{ +CROSS_TU_DEVICE constexpr uint64_t ExpectedTotalChecksum() { uint64_t checksum = 0; for (uint32_t round = 0; round < kRounds; round++) { checksum += ExpectedLazyDigest(round); @@ -116,8 +102,7 @@ CROSS_TU_DEVICE constexpr uint64_t ExpectedTotalChecksum() return checksum; } -constexpr uint32_t ExpectedMaterializations(Variant variant) -{ +constexpr uint32_t ExpectedMaterializations(Variant variant) { switch (variant) { case Variant::WeakContextMaterialize0: case Variant::StrongContext: @@ -136,8 +121,7 @@ constexpr uint32_t ExpectedMaterializations(Variant variant) return 0; } -constexpr const char *VariantName(Variant variant) -{ +constexpr const char *VariantName(Variant variant) { switch (variant) { case Variant::WeakContextMaterialize0: return "weak-context-materialize-0"; @@ -159,8 +143,7 @@ constexpr const char *VariantName(Variant variant) return "invalid"; } -constexpr const char *KernelName(Variant variant) -{ +constexpr const char *KernelName(Variant variant) { switch (variant) { case Variant::WeakContextMaterialize0: return "nested_lambda_cross_tu_ctx_m0_0_mix_aic"; @@ -184,18 +167,14 @@ constexpr const char *KernelName(Variant variant) // CANN defines funcEntry as the numeric suffix in kernel_foo_. // Keep entries unique inside this multi-kernel raw ELF. -constexpr uint64_t KernelEntry(Variant variant) -{ - return static_cast(variant); -} +constexpr uint64_t KernelEntry(Variant variant) { return static_cast(variant); } -constexpr uint32_t ExpectedDispatcherCalls(Variant variant) -{ +constexpr uint32_t ExpectedDispatcherCalls(Variant variant) { return variant == Variant::ArgsRuntimeRead ? 0 : kRounds * 2; } #undef CROSS_TU_DEVICE -} // namespace nested_lambda_cross_tu_probe +} // namespace nested_lambda_cross_tu_probe -#endif // TESTS_ATOMIC_PROBE_CCEC_NESTED_LAMBDA_CROSS_TU_LAYOUT_H +#endif // TESTS_ATOMIC_PROBE_CCEC_NESTED_LAMBDA_CROSS_TU_LAYOUT_H diff --git a/tests/atomic_probe/ccec/nested_lambda_cross_tu_runtime.cpp b/tests/atomic_probe/ccec/nested_lambda_cross_tu_runtime.cpp index 249c3c93bb..53ea6e332d 100644 --- a/tests/atomic_probe/ccec/nested_lambda_cross_tu_runtime.cpp +++ b/tests/atomic_probe/ccec/nested_lambda_cross_tu_runtime.cpp @@ -9,97 +9,7 @@ * ----------------------------------------------------------------------------------------------------------- */ #include "nested_lambda_cross_tu_api.h" -#include "nested_lambda_cross_tu_layout.h" -extern "C" PTO_DEVICE_FUNC void nested_probe_weak_context_dispatch( - int32_t site_id, int32_t phase, uint64_t caller_context, L0TaskArgs *args) __attribute__((weak)); -extern "C" PTO_DEVICE_FUNC void nested_probe_weak_args_dispatch( - int32_t site_id, int32_t phase, L0TaskArgs *args) __attribute__((weak)); -extern "C" PTO_DEVICE_FUNC void nested_probe_strong_context_dispatch( - int32_t site_id, int32_t phase, uint64_t caller_context, L0TaskArgs *args); - -namespace { - -PTO_DEVICE_FUNC uint64_t TensorDigest(const Tensor &tensor) -{ - return tensor.buffer.addr + tensor.start_offset * 17ULL + - static_cast(static_cast(tensor.version)) * 257ULL + - tensor.shapes[0] * 65537ULL; -} - -PTO_DEVICE_FUNC void ConsumeBoundArguments(L0TaskArgs *args) -{ - const Tensor &first = args->tensor(0).ref(); - const Tensor &second = args->tensor(1).ref(); - const Tensor &third = args->tensor(2).ref(); - args->scalar(0) = TensorDigest(first) * 3ULL + TensorDigest(second) * 5ULL + - TensorDigest(third) * 7ULL + args->scalar(4); - args->scalar(5) = 2; - args->scalar(6) = 0; -} - -} // namespace - -PTO_DEVICE_FUNC TaskOutputTensors nested_probe_submit_weak_context( - int32_t site_id, uint64_t caller_context, L0TaskArgs *args) -{ - TaskOutputTensors outputs; - if (nested_probe_weak_context_dispatch == nullptr) { - args->scalar(6) = 1; - return outputs; - } - nested_probe_weak_context_dispatch( - site_id, static_cast(nested_lambda_cross_tu_probe::DispatchPhase::Prepare), caller_context, args); - nested_probe_weak_context_dispatch( - site_id, static_cast(nested_lambda_cross_tu_probe::DispatchPhase::WinnerBind), caller_context, args); - ConsumeBoundArguments(args); - return outputs; -} - -PTO_DEVICE_FUNC TaskOutputTensors nested_probe_submit_weak_args( - int32_t site_id, L0TaskArgs *args) -{ - TaskOutputTensors outputs; - if (nested_probe_weak_args_dispatch == nullptr) { - args->scalar(6) = 1; - return outputs; - } - nested_probe_weak_args_dispatch( - site_id, static_cast(nested_lambda_cross_tu_probe::DispatchPhase::Prepare), args); - nested_probe_weak_args_dispatch( - site_id, static_cast(nested_lambda_cross_tu_probe::DispatchPhase::WinnerBind), args); - ConsumeBoundArguments(args); - return outputs; -} - -PTO_DEVICE_FUNC TaskOutputTensors nested_probe_submit_strong_context( - int32_t site_id, uint64_t caller_context, L0TaskArgs *args) -{ - TaskOutputTensors outputs; - nested_probe_strong_context_dispatch( - site_id, static_cast(nested_lambda_cross_tu_probe::DispatchPhase::Prepare), caller_context, args); - nested_probe_strong_context_dispatch( - site_id, static_cast(nested_lambda_cross_tu_probe::DispatchPhase::WinnerBind), caller_context, args); - ConsumeBoundArguments(args); - return outputs; -} - -PTO_DEVICE_FUNC TaskOutputTensors nested_probe_submit_control(L0TaskArgs *args) -{ - TaskOutputTensors outputs; - args->scalar(0) ^= nested_lambda_cross_tu_probe::kControlXor; - return outputs; -} - -PTO_DEVICE_FUNC TaskOutputTensors nested_probe_submit_args_runtime_read(L0TaskArgs *args) -{ - TaskOutputTensors outputs; - const auto *first = reinterpret_cast(args->scalar(8)); - const auto *second = reinterpret_cast(args->scalar(9)); - const auto *third = reinterpret_cast(args->scalar(10)); - args->scalar(4) = args->scalar(11); - args->add_input(*first, *second, *third); - ConsumeBoundArguments(args); - args->scalar(5) = 0; - return outputs; -} +// This translation unit intentionally emits no runtime submit functions. It is +// linked only by the two-object control to separate object count from an actual +// cross-object call boundary. diff --git a/tests/atomic_probe/ccec/nested_lambda_inline_plus_empty_runtime.cpp b/tests/atomic_probe/ccec/nested_lambda_inline_plus_empty_runtime.cpp new file mode 100644 index 0000000000..434b5f1d6a --- /dev/null +++ b/tests/atomic_probe/ccec/nested_lambda_inline_plus_empty_runtime.cpp @@ -0,0 +1,11 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#include "nested_lambda_cross_tu.cpp" diff --git a/tests/atomic_probe/ccec/nested_lambda_only_weak_submit_noinline.cpp b/tests/atomic_probe/ccec/nested_lambda_only_weak_submit_noinline.cpp new file mode 100644 index 0000000000..98ab040a89 --- /dev/null +++ b/tests/atomic_probe/ccec/nested_lambda_only_weak_submit_noinline.cpp @@ -0,0 +1,12 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#define NESTED_PROBE_WEAK_CONTEXT_NOINLINE +#include "nested_lambda_cross_tu.cpp" diff --git a/tests/atomic_probe/ccec/pmu_probe_aicpu.cpp b/tests/atomic_probe/ccec/pmu_probe_aicpu.cpp new file mode 100644 index 0000000000..6f01a985bb --- /dev/null +++ b/tests/atomic_probe/ccec/pmu_probe_aicpu.cpp @@ -0,0 +1,312 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "pmu_probe_control.h" + +#include "aicpu/platform_regs.h" +#include "common/kernel_args.h" + +#include +#include + +namespace { + +using atomic_probe::pmu::PmuControl; +using atomic_probe::pmu::PmuRegisterField; + +void FlushControl(const PmuControl *control) +{ + const uintptr_t begin = reinterpret_cast(control); + const uintptr_t end = begin + sizeof(*control); + for (uintptr_t address = begin; address < end; address += 64) { + __asm__ volatile("dc cvac, %0" : : "r"(address) : "memory"); + } + __asm__ volatile("dsb sy" ::: "memory"); + __asm__ volatile("isb" ::: "memory"); +} + +void ResetFailureDiagnostic(PmuControl *control) +{ + control->first_failed_index = atomic_probe::pmu::kPmuDiagnosticUnset; + control->first_failed_field = static_cast(PmuRegisterField::None); + control->first_failed_observed = 0; + control->first_failed_expected = 0; +} + +void RecordFirstFailure( + PmuControl *control, uint32_t index, PmuRegisterField field, uint32_t observed, uint32_t expected +) +{ + if (control->first_failed_index != atomic_probe::pmu::kPmuDiagnosticUnset) return; + control->first_failed_index = index; + control->first_failed_field = static_cast(field); + control->first_failed_observed = observed; + control->first_failed_expected = expected; +} + +bool CheckProbeRegister( + uint64_t base, RegId reg, uint32_t expected, PmuRegisterField field, + PmuRegisterField *failed_field, uint32_t *observed +) +{ + const uint32_t actual = static_cast(read_reg(base, reg)); + if (actual == expected) return true; + *failed_field = field; + *observed = actual; + return false; +} + +bool ProbeConfigurationMatches( + uint64_t base, PmuRegisterField *failed_field, uint32_t *observed, uint32_t *expected +) +{ + struct RegisterExpectation { + RegId reg; + uint32_t value; + PmuRegisterField field; + }; + const RegisterExpectation expectations[] = { + {RegId::PMU_CTRL_0, REG_MMIO_PMU_CTRL_0_ENABLE_VAL, PmuRegisterField::Ctrl0}, + {RegId::PMU_CTRL_1, REG_MMIO_PMU_CTRL_1_ENABLE_VAL, PmuRegisterField::Ctrl1}, + {RegId::PMU_CNT0_IDX, 0x1U, PmuRegisterField::Selector0}, + {RegId::PMU_CNT1_IDX, 0x34U, PmuRegisterField::Selector1}, + {RegId::PMU_CNT2_IDX, 0x35U, PmuRegisterField::Selector2}, + {RegId::PMU_START_CYC0, 0U, PmuRegisterField::StartCycleLow}, + {RegId::PMU_START_CYC1, 0U, PmuRegisterField::StartCycleHigh}, + {RegId::PMU_STOP_CYC0, 0xffffffffU, PmuRegisterField::StopCycleLow}, + {RegId::PMU_STOP_CYC1, 0xffffffffU, PmuRegisterField::StopCycleHigh}, + }; + for (const RegisterExpectation &expectation : expectations) { + if (!CheckProbeRegister( + base, expectation.reg, expectation.value, expectation.field, failed_field, observed + )) { + *expected = expectation.value; + return false; + } + } + return true; +} + +bool SavedConfigurationMatches( + const PmuControl *control, uint64_t base, uint32_t index, + PmuRegisterField *failed_field, uint32_t *observed, uint32_t *expected +) +{ + struct RegisterExpectation { + RegId reg; + uint32_t value; + PmuRegisterField field; + }; + const RegisterExpectation expectations[] = { + {RegId::PMU_CTRL_0, control->saved_ctrl0[index], PmuRegisterField::Ctrl0}, + {RegId::PMU_CTRL_1, control->saved_ctrl1[index], PmuRegisterField::Ctrl1}, + {RegId::PMU_CNT0_IDX, control->saved_selector0[index], PmuRegisterField::Selector0}, + {RegId::PMU_CNT1_IDX, control->saved_selector1[index], PmuRegisterField::Selector1}, + {RegId::PMU_CNT2_IDX, control->saved_selector2[index], PmuRegisterField::Selector2}, + {RegId::PMU_START_CYC0, control->saved_start_cycle_low[index], PmuRegisterField::StartCycleLow}, + {RegId::PMU_START_CYC1, control->saved_start_cycle_high[index], PmuRegisterField::StartCycleHigh}, + {RegId::PMU_STOP_CYC0, control->saved_stop_cycle_low[index], PmuRegisterField::StopCycleLow}, + {RegId::PMU_STOP_CYC1, control->saved_stop_cycle_high[index], PmuRegisterField::StopCycleHigh}, + }; + for (const RegisterExpectation &expectation : expectations) { + if (!CheckProbeRegister( + base, expectation.reg, expectation.value, expectation.field, failed_field, observed + )) { + *expected = expectation.value; + return false; + } + } + return true; +} + +bool RestoreOne( + PmuControl *control, uint64_t base, uint32_t index, + PmuRegisterField *failed_field, uint32_t *observed, uint32_t *expected +) +{ + write_reg(base, RegId::PMU_CTRL_0, 0); + write_reg(base, RegId::PMU_CTRL_1, 0); + write_reg(base, RegId::PMU_CNT0_IDX, control->saved_selector0[index]); + write_reg(base, RegId::PMU_CNT1_IDX, control->saved_selector1[index]); + write_reg(base, RegId::PMU_CNT2_IDX, control->saved_selector2[index]); + write_reg(base, RegId::PMU_START_CYC0, control->saved_start_cycle_low[index]); + write_reg(base, RegId::PMU_START_CYC1, control->saved_start_cycle_high[index]); + write_reg(base, RegId::PMU_STOP_CYC0, control->saved_stop_cycle_low[index]); + write_reg(base, RegId::PMU_STOP_CYC1, control->saved_stop_cycle_high[index]); + write_reg(base, RegId::PMU_CTRL_0, control->saved_ctrl0[index]); + write_reg(base, RegId::PMU_CTRL_1, control->saved_ctrl1[index]); + return SavedConfigurationMatches(control, base, index, failed_field, observed, expected); +} + +void SaveOne(PmuControl *control, uint64_t base, uint32_t index) +{ + control->saved_ctrl0[index] = static_cast(read_reg(base, RegId::PMU_CTRL_0)); + control->saved_ctrl1[index] = static_cast(read_reg(base, RegId::PMU_CTRL_1)); + control->saved_selector0[index] = static_cast(read_reg(base, RegId::PMU_CNT0_IDX)); + control->saved_selector1[index] = static_cast(read_reg(base, RegId::PMU_CNT1_IDX)); + control->saved_selector2[index] = static_cast(read_reg(base, RegId::PMU_CNT2_IDX)); + control->saved_start_cycle_low[index] = static_cast(read_reg(base, RegId::PMU_START_CYC0)); + control->saved_start_cycle_high[index] = static_cast(read_reg(base, RegId::PMU_START_CYC1)); + control->saved_stop_cycle_low[index] = static_cast(read_reg(base, RegId::PMU_STOP_CYC0)); + control->saved_stop_cycle_high[index] = static_cast(read_reg(base, RegId::PMU_STOP_CYC1)); +} + +void ConfigureOne(uint64_t base) +{ + // 先冻结框架，再配置 Custom 三事件及完整计数周期；最后启用 + // GLB_PMU_EN | USER_PMU_MODE_EN | SAMPLE_PMU_MODE_EN。 + write_reg(base, RegId::PMU_CTRL_0, 0); + write_reg(base, RegId::PMU_CTRL_1, 0); + write_reg(base, RegId::PMU_CNT0_IDX, 0x1U); + write_reg(base, RegId::PMU_CNT1_IDX, 0x34U); + write_reg(base, RegId::PMU_CNT2_IDX, 0x35U); + for (int counter = 0; counter < 10; ++counter) { + (void)read_reg(base, reg_index(RegId::PMU_CNT0, counter)); + } + (void)read_reg(base, RegId::PMU_CNT_TOTAL0); + (void)read_reg(base, RegId::PMU_CNT_TOTAL1); + write_reg(base, RegId::PMU_START_CYC0, 0); + write_reg(base, RegId::PMU_START_CYC1, 0); + write_reg(base, RegId::PMU_STOP_CYC0, 0xffffffffU); + write_reg(base, RegId::PMU_STOP_CYC1, 0xffffffffU); + write_reg(base, RegId::PMU_CTRL_0, REG_MMIO_PMU_CTRL_0_ENABLE_VAL); + write_reg(base, RegId::PMU_CTRL_1, REG_MMIO_PMU_CTRL_1_ENABLE_VAL); +} + +bool RestoreConfiguredBitmap(PmuControl *control, const uint64_t *register_bases) +{ + bool all_restored = true; + for (uint32_t next = atomic_probe::pmu::kPmuPhysicalSubcores; next != 0; --next) { + const uint32_t index = next - 1; + if (!atomic_probe::pmu::IsPmuSubcoreConfigured(*control, index)) continue; + PmuRegisterField failed_field = PmuRegisterField::None; + uint32_t observed = 0; + uint32_t expected = 0; + if (RestoreOne( + control, register_bases[index], index, &failed_field, &observed, &expected + )) { + atomic_probe::pmu::ClearPmuSubcoreConfigured(control, index); + if (control->processed_subcores != 0) --control->processed_subcores; + } else { + RecordFirstFailure(control, index, failed_field, observed, expected); + all_restored = false; + } + } + return all_restored && control->processed_subcores == 0 && + atomic_probe::pmu::CountPmuConfiguredSubcores(*control) == 0; +} + +int Configure(PmuControl *control, const uint64_t *register_bases) +{ + if (control->configured != 0) return -10; + control->processed_subcores = 0; + control->skipped_subcores = 0; + for (uint32_t word = 0; word < atomic_probe::pmu::kPmuBitmapWords; ++word) { + control->configured_bitmap[word] = 0; + } + ResetFailureDiagnostic(control); + for (uint32_t index = 0; index < atomic_probe::pmu::kPmuPhysicalSubcores; ++index) { + const uint64_t base = register_bases[index]; + if (base == 0) { + RecordFirstFailure(control, index, PmuRegisterField::RegisterBase, 0, 1); + ++control->skipped_subcores; + continue; + } + + SaveOne(control, base, index); + ConfigureOne(base); + PmuRegisterField failed_field = PmuRegisterField::None; + uint32_t observed = 0; + uint32_t expected = 0; + if (!ProbeConfigurationMatches(base, &failed_field, &observed, &expected)) { + RecordFirstFailure(control, index, failed_field, observed, expected); + ++control->skipped_subcores; + PmuRegisterField restore_failed_field = PmuRegisterField::None; + uint32_t restore_observed = 0; + uint32_t restore_expected = 0; + if (!RestoreOne( + control, base, index, &restore_failed_field, &restore_observed, &restore_expected + )) { + // 该项没有进入成功 bitmap，无法在后续 Restore 命令中重试； + // 先回滚此前成功项，再用独立状态区分“探测失败且现场恢复失败”。 + const bool rollback_ok = RestoreConfiguredBitmap(control, register_bases); + control->configured = control->processed_subcores == 0 ? 0U : 1U; + return rollback_ok ? -13 : -14; + } + continue; + } + atomic_probe::pmu::SetPmuSubcoreConfigured(control, index); + ++control->processed_subcores; + } + + if (control->processed_subcores != control->expected_subcores || + atomic_probe::pmu::CountPmuConfiguredSubcores(*control) != control->expected_subcores) { + RecordFirstFailure( + control, atomic_probe::pmu::kPmuPhysicalSubcores, PmuRegisterField::ConfiguredCount, + control->processed_subcores, control->expected_subcores + ); + const bool rollback_ok = RestoreConfiguredBitmap(control, register_bases); + control->configured = control->processed_subcores == 0 ? 0U : 1U; + return rollback_ok ? -12 : -14; + } + control->configured = 1; + return 0; +} + +int Restore(PmuControl *control, const uint64_t *register_bases) +{ + if (control->configured == 0 || control->processed_subcores == 0 || + control->processed_subcores != atomic_probe::pmu::CountPmuConfiguredSubcores(*control)) { + return -20; + } + ResetFailureDiagnostic(control); + const bool all_restored = RestoreConfiguredBitmap(control, register_bases); + control->configured = all_restored ? 0U : 1U; + return all_restored ? 0 : -21; +} + +} // namespace + +extern "C" __attribute__((visibility("default"))) int simpler_aicpu_exec(void *argument) +{ + if (argument == nullptr) return -1; + auto *kernel_args = reinterpret_cast(argument); + auto *control = reinterpret_cast(kernel_args->runtime_args); + auto *register_bases = reinterpret_cast(kernel_args->regs); + if (control == nullptr || register_bases == nullptr) return -2; + + // command 位于每次 launch 都由 CANN 重新复制的 inline KernelArgs 中； + // PmuControl 初始化后只由 AICPU 写，因此这里不依赖 EL0 cache invalidate。 + control->command = kernel_args->enable_profiling_flag; + control->status = atomic_probe::pmu::kPmuStatusPending; + if (control->magic != atomic_probe::pmu::kPmuControlMagic || + control->version != atomic_probe::pmu::kPmuControlVersion || + control->expected_subcores == 0 || + control->expected_subcores > atomic_probe::pmu::kPmuPhysicalSubcores) { + control->status = -3; + FlushControl(control); + return 0; + } + + int status = -4; + const auto command = static_cast(kernel_args->enable_profiling_flag); + if (command == atomic_probe::pmu::PmuCommand::Configure) { + status = Configure(control, register_bases); + } else if (command == atomic_probe::pmu::PmuCommand::Restore) { + status = Restore(control, register_bases); + } + control->status = status; + FlushControl(control); + // AICPU entry 的非零返回会被 runtime 升格为 stream 异常，host 从而无法 + // D2H 读取上面的精确状态。协议级成败统一由 control->status 传递， + // entry 只报告“命令已执行并已发布状态”。 + return 0; +} diff --git a/tests/atomic_probe/ccec/pmu_probe_control.h b/tests/atomic_probe/ccec/pmu_probe_control.h new file mode 100644 index 0000000000..24258870c9 --- /dev/null +++ b/tests/atomic_probe/ccec/pmu_probe_control.h @@ -0,0 +1,131 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef TESTS_ATOMIC_PROBE_CCEC_PMU_PROBE_CONTROL_H_ +#define TESTS_ATOMIC_PROBE_CCEC_PMU_PROBE_CONTROL_H_ + +#include +#include + +namespace atomic_probe::pmu { + +constexpr uint32_t kPmuControlMagic = 0x504d5551U; // "PMUQ" +constexpr uint32_t kPmuControlVersion = 2; +constexpr uint32_t kPmuPhysicalSubcores = 108; +constexpr uint32_t kPmuBitmapWords = (kPmuPhysicalSubcores + 31U) / 32U; +constexpr int32_t kPmuStatusPending = 0x7fffffff; +constexpr uint32_t kPmuDiagnosticUnset = 0xffffffffU; + +enum class PmuRegisterField : uint32_t { + None = 0, + RegisterBase, + Ctrl0, + Ctrl1, + Selector0, + Selector1, + Selector2, + StartCycleLow, + StartCycleHigh, + StopCycleLow, + StopCycleHigh, + ConfiguredCount, +}; + +enum class PmuCommand : uint32_t { + Configure = 1, + Restore = 2, +}; + +// Host 与单线程 AICPU helper 共享的 PMU 所有权记录。helper 在 Configure +// 阶段保存被改寄存器，在 Restore 阶段先关 PMU、恢复 selector/range，最后恢复 CTRL。 +// counter 本身是 read-to-clear，旧计数内容无法恢复，因此 PMU session 必须独占。 +struct alignas(64) PmuControl { + uint32_t magic; + uint32_t version; + uint32_t command; + volatile int32_t status; + uint32_t configured; + // v2 中这是 bitmap 内成功配置的数量，不再表示 0..N 连续前缀。 + uint32_t processed_subcores; + // 由 host 从同一 stream 的 cube/vector resource limit 求和后填入。 + uint32_t expected_subcores; + uint32_t configured_bitmap[kPmuBitmapWords]; + uint32_t first_failed_index; + uint32_t first_failed_field; + uint32_t first_failed_observed; + uint32_t first_failed_expected; + uint32_t skipped_subcores; + + uint32_t saved_ctrl0[kPmuPhysicalSubcores]; + uint32_t saved_ctrl1[kPmuPhysicalSubcores]; + uint32_t saved_selector0[kPmuPhysicalSubcores]; + uint32_t saved_selector1[kPmuPhysicalSubcores]; + uint32_t saved_selector2[kPmuPhysicalSubcores]; + uint32_t saved_start_cycle_low[kPmuPhysicalSubcores]; + uint32_t saved_start_cycle_high[kPmuPhysicalSubcores]; + uint32_t saved_stop_cycle_low[kPmuPhysicalSubcores]; + uint32_t saved_stop_cycle_high[kPmuPhysicalSubcores]; +}; + +static_assert(offsetof(PmuControl, saved_ctrl0) == 64, "PMU control header must occupy one cache line"); +static_assert(sizeof(PmuControl) % 64 == 0, "PMU control must use complete cache lines"); + +inline bool IsPmuSubcoreConfigured(const PmuControl &control, uint32_t index) +{ + return index < kPmuPhysicalSubcores && + (control.configured_bitmap[index / 32U] & (1U << (index % 32U))) != 0; +} + +inline void SetPmuSubcoreConfigured(PmuControl *control, uint32_t index) +{ + if (index < kPmuPhysicalSubcores) { + control->configured_bitmap[index / 32U] |= 1U << (index % 32U); + } +} + +inline void ClearPmuSubcoreConfigured(PmuControl *control, uint32_t index) +{ + if (index < kPmuPhysicalSubcores) { + control->configured_bitmap[index / 32U] &= ~(1U << (index % 32U)); + } +} + +inline uint32_t CountPmuConfiguredSubcores(const PmuControl &control) +{ + uint32_t count = 0; + for (uint32_t index = 0; index < kPmuPhysicalSubcores; ++index) { + count += IsPmuSubcoreConfigured(control, index) ? 1U : 0U; + } + return count; +} + +inline const char *PmuRegisterFieldName(PmuRegisterField field) +{ + switch (field) { + case PmuRegisterField::None: return "none"; + case PmuRegisterField::RegisterBase: return "register-base"; + case PmuRegisterField::Ctrl0: return "ctrl0"; + case PmuRegisterField::Ctrl1: return "ctrl1"; + case PmuRegisterField::Selector0: return "selector0"; + case PmuRegisterField::Selector1: return "selector1"; + case PmuRegisterField::Selector2: return "selector2"; + case PmuRegisterField::StartCycleLow: return "start-cycle-low"; + case PmuRegisterField::StartCycleHigh: return "start-cycle-high"; + case PmuRegisterField::StopCycleLow: return "stop-cycle-low"; + case PmuRegisterField::StopCycleHigh: return "stop-cycle-high"; + case PmuRegisterField::ConfiguredCount: return "configured-count"; + default: return "unknown"; + } +} + +} // namespace atomic_probe::pmu + +#endif // TESTS_ATOMIC_PROBE_CCEC_PMU_PROBE_CONTROL_H_ diff --git a/tests/atomic_probe/ccec/pmu_probe_host_support.h b/tests/atomic_probe/ccec/pmu_probe_host_support.h new file mode 100644 index 0000000000..f2a0b088ec --- /dev/null +++ b/tests/atomic_probe/ccec/pmu_probe_host_support.h @@ -0,0 +1,388 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef TESTS_ATOMIC_PROBE_CCEC_PMU_PROBE_HOST_SUPPORT_H_ +#define TESTS_ATOMIC_PROBE_CCEC_PMU_PROBE_HOST_SUPPORT_H_ + +#include "pmu_probe_control.h" + +#include "acl/acl.h" +#include "aicpu_loader/host/load_aicpu_op.h" +#include "common/kernel_args.h" +#include "driver/ascend_hal.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace atomic_probe::pmu { + +inline bool CheckAcl(aclError error, const char *label) +{ + if (error == ACL_SUCCESS) return true; + std::fprintf(stderr, "ACL error %d: %s\n", static_cast(error), label); + return false; +} + +inline std::vector ReadBinary(const std::string &path) +{ + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) return {}; + const std::streamsize size = file.tellg(); + if (size <= 0) return {}; + std::vector bytes(static_cast(size)); + file.seekg(0, std::ios::beg); + if (!file.read(bytes.data(), size)) return {}; + return bytes; +} + +inline std::string ArtifactBesideKernel(const std::string &kernel_path, const char *name) +{ + const size_t slash = kernel_path.find_last_of('/'); + return slash == std::string::npos ? std::string(name) : kernel_path.substr(0, slash + 1) + name; +} + +struct ActiveSubcoreLimits { + uint32_t cube = 0; + uint32_t vector = 0; + uint32_t total = 0; +}; + +// PMU MMIO 表覆盖芯片的 108 个物理槽，但当前 stream 只会调度 ACL +// resource limit 报告的活跃 cube/vector 子核。A5 当前典型值为 32+64=96； +// 不把 96 写死，以免把 stream 分区变化误报成寄存器配置失败。 +inline bool QueryActiveSubcoreLimits(aclrtStream stream, ActiveSubcoreLimits *limits) +{ + if (stream == nullptr || limits == nullptr) { + std::fprintf(stderr, "Cannot query PMU active subcores with a null stream/result.\n"); + return false; + } + uint32_t cube = 0; + uint32_t vector = 0; + const aclError cube_error = aclrtGetStreamResLimit(stream, ACL_RT_DEV_RES_CUBE_CORE, &cube); + const aclError vector_error = aclrtGetStreamResLimit(stream, ACL_RT_DEV_RES_VECTOR_CORE, &vector); + const uint64_t total = static_cast(cube) + vector; + constexpr uint32_t kMaximumCubeSubcores = kPmuPhysicalSubcores / 3U; + constexpr uint32_t kMaximumVectorSubcores = kPmuPhysicalSubcores - kMaximumCubeSubcores; + if (cube_error != ACL_SUCCESS || vector_error != ACL_SUCCESS || cube == 0 || vector == 0 || + cube > kMaximumCubeSubcores || vector > kMaximumVectorSubcores || total > kPmuPhysicalSubcores) { + std::fprintf( + stderr, + "Cannot determine stream PMU subcores: cube_error=%d vector_error=%d " + "cube=%u vector=%u total=%llu physical_slots=%u\n", + static_cast(cube_error), static_cast(vector_error), cube, vector, + static_cast(total), kPmuPhysicalSubcores + ); + return false; + } + limits->cube = cube; + limits->vector = vector; + limits->total = static_cast(total); + std::printf( + "[PMU_OWNER] stream_active cube=%u vector=%u total=%u physical_slots=%u\n", + limits->cube, limits->vector, limits->total, kPmuPhysicalSubcores + ); + return true; +} + +constexpr uint32_t kPhysicalAicoreCount = 36; +constexpr uint32_t kSubcoresPerAicore = 3; +constexpr uint32_t kPhysicalSubcoreCount = kPhysicalAicoreCount * kSubcoresPerAicore; +constexpr uint32_t kAicorePerDie = 18; +constexpr uint32_t kSubcoresPerDie = kAicorePerDie * kSubcoresPerAicore; +constexpr uint32_t kAivBaseInDie = kAicorePerDie; +constexpr uint64_t kSubcoreStride = 0x100000ULL; +constexpr uint32_t kAicoreMapBytes = 0x300000U; + +static_assert(kPhysicalSubcoreCount == kPmuPhysicalSubcores, "PMU table size mismatch"); + +// halResMap 只负责把 36 个物理 AICore 展开成 AIC/AIV 共 108 项 MMIO base。 +// 它不负责恢复 PMU selector/CTRL；恢复必须先由 PmuSession::Restore 完成。 +class RegisterMappings { +public: + using MapFn = drvError_t (*)(unsigned int, struct res_map_info *, unsigned long *, unsigned int *); + using UnmapFn = drvError_t (*)(unsigned int, struct res_map_info *); + + ~RegisterMappings() { Release(); } + + bool Initialize(uint32_t device) + { + device_ = device; + map_ = reinterpret_cast(dlsym(RTLD_DEFAULT, "halResMap")); + unmap_ = reinterpret_cast(dlsym(RTLD_DEFAULT, "halResUnmap")); + if (map_ == nullptr || unmap_ == nullptr) { + hal_handle_ = dlopen("libascend_hal.so", RTLD_NOW | RTLD_GLOBAL); + if (hal_handle_ != nullptr) { + map_ = reinterpret_cast(dlsym(hal_handle_, "halResMap")); + unmap_ = reinterpret_cast(dlsym(hal_handle_, "halResUnmap")); + } + } + if (map_ == nullptr || unmap_ == nullptr) { + std::fprintf(stderr, "Cannot resolve halResMap/halResUnmap.\n"); + return false; + } + + for (uint32_t physical = 0; physical < kPhysicalAicoreCount; ++physical) { + res_map_info &info = map_info_[physical]; + std::memset(&info, 0, sizeof(info)); + info.target_proc_type = PROCESS_CP1; + info.res_type = RES_AICORE; + info.res_id = physical; + unsigned long map_address = 0; + unsigned int map_length = kAicoreMapBytes; + const drvError_t error = map_(device_, &info, &map_address, &map_length); + if (error != 0 || map_address == 0 || map_length < kAicoreMapBytes) { + std::fprintf( + stderr, "halResMap failed: physical=%u error=%d address=0x%lx length=%u\n", physical, + static_cast(error), map_address, map_length + ); + Release(); + return false; + } + ++mapped_count_; + const uint64_t base = static_cast(map_address); + const uint32_t die = physical / kAicorePerDie; + const uint32_t local = physical % kAicorePerDie; + const uint32_t die_base = die * kSubcoresPerDie; + register_bases_[die_base + local] = base; + const uint32_t aiv0 = die_base + kAivBaseInDie + local * 2; + register_bases_[aiv0] = base + kSubcoreStride; + register_bases_[aiv0 + 1] = base + 2 * kSubcoreStride; + } + return true; + } + + void Release() + { + while (mapped_count_ != 0) { + --mapped_count_; + const drvError_t error = unmap_(device_, &map_info_[mapped_count_]); + if (error != 0) { + std::fprintf( + stderr, "halResUnmap failed: physical=%u error=%d\n", mapped_count_, + static_cast(error) + ); + } + } + if (hal_handle_ != nullptr) { + dlclose(hal_handle_); + hal_handle_ = nullptr; + } + } + + const std::array &RegisterBases() const { return register_bases_; } + +private: + uint32_t device_ = 0; + uint32_t mapped_count_ = 0; + void *hal_handle_ = nullptr; + MapFn map_ = nullptr; + UnmapFn unmap_ = nullptr; + std::array map_info_{}; + std::array register_bases_{}; +}; + +// 两个 scalar PMU probe 共用唯一配置会话：selector/range 只配置一次，所有 +// 单 AIV 样本完成后统一恢复。command 始终经 inline KernelArgs 传给 AICPU； +// Configure/Restore 之间绝不由 host 再 H2D 覆盖 PmuControl cache line。 +class PmuSession { +public: + bool Initialize( + uint32_t device, aclrtStream stream, const std::string &kernel_path, const char *helper_name + ) + { + stream_ = stream; + device_ = device; + if (!QueryActiveSubcoreLimits(stream, &active_limits_)) return false; + if (!mappings_.Initialize(device)) return false; + + const size_t register_bytes = sizeof(mappings_.RegisterBases()); + if (!CheckAcl( + aclrtMalloc(®ister_bases_device_, register_bytes, ACL_MEM_MALLOC_NORMAL_ONLY), + "aclrtMalloc(PMU regs)" + ) || + !CheckAcl( + aclrtMemcpy( + register_bases_device_, register_bytes, mappings_.RegisterBases().data(), register_bytes, + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D PMU regs)" + ) || + !CheckAcl( + aclrtMalloc(&control_device_, sizeof(PmuControl), ACL_MEM_MALLOC_NORMAL_ONLY), + "aclrtMalloc(PMU control)" + )) { + return false; + } + + control_.magic = kPmuControlMagic; + control_.version = kPmuControlVersion; + control_.expected_subcores = active_limits_.total; + if (!CheckAcl( + aclrtMemcpy( + control_device_, sizeof(control_), &control_, sizeof(control_), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D initial PMU control)" + )) { + return false; + } + + const std::string dispatcher_path = ArtifactBesideKernel(kernel_path, "libsimpler_aicpu_dispatcher.so"); + const std::string helper_path = ArtifactBesideKernel(kernel_path, helper_name); + const std::vector dispatcher_data = ReadBinary(dispatcher_path); + const std::vector helper_data = ReadBinary(helper_path); + if (dispatcher_data.empty() || helper_data.empty()) { + std::fprintf(stderr, "Cannot read PMU artifacts: %s %s\n", dispatcher_path.c_str(), helper_path.c_str()); + return false; + } + if (loader_.BootstrapDispatcher( + dispatcher_data.data(), dispatcher_data.size(), helper_data.data(), helper_data.size(), stream, + static_cast(device) + ) != 0 || + loader_.Init() != 0) { + std::fprintf(stderr, "Cannot initialize PMU AICPU helper.\n"); + return false; + } + kernel_args_.runtime_args = reinterpret_cast(control_device_); + kernel_args_.regs = reinterpret_cast(register_bases_device_); + kernel_args_.device_id = device; + initialized_ = true; + return true; + } + + bool Configure() + { + if (!initialized_ || configured_) return false; + const bool command_ok = RunCommand(PmuCommand::Configure); + // 若 Configure 报错且设备侧回滚也失败，bitmap 仍保存待恢复项； + // 不能因为业务状态失败就丢失 Restore 所有权。 + configured_ = control_.configured != 0; + return command_ok; + } + + bool Restore() + { + if (!configured_) return true; + if (!RunCommand(PmuCommand::Restore)) return false; + configured_ = false; + return true; + } + + bool Finalize() + { + bool ok = Restore(); + loader_.Finalize(); + if (control_device_ != nullptr) { + ok &= CheckAcl(aclrtFree(control_device_), "aclrtFree(PMU control)"); + control_device_ = nullptr; + } + if (register_bases_device_ != nullptr) { + ok &= CheckAcl(aclrtFree(register_bases_device_), "aclrtFree(PMU regs)"); + register_bases_device_ = nullptr; + } + mappings_.Release(); + initialized_ = false; + return ok; + } + + uint64_t RegisterBasesDeviceAddress() const + { + return reinterpret_cast(register_bases_device_); + } + + bool IsConfiguredSubcore(uint32_t index) const + { + return configured_ && IsPmuSubcoreConfigured(control_, index); + } + +private: + bool RunCommand(PmuCommand command) + { + control_.command = static_cast(command); + control_.status = kPmuStatusPending; + kernel_args_.enable_profiling_flag = static_cast(command); + const int launch_error = loader_.LaunchBuiltInOp( + stream_, &kernel_args_, 1, host::KernelNames::RunName + ); + if (launch_error != 0) { + std::fprintf(stderr, "AICPU PMU helper launch failed: %d\n", launch_error); + return false; + } + if (!CheckAcl(aclrtSynchronizeStream(stream_), "aclrtSynchronizeStream(PMU helper)") || + !CheckAcl( + aclrtMemcpy( + &control_, sizeof(control_), control_device_, sizeof(control_), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H PMU control)" + )) { + return false; + } + const uint32_t bitmap_count = CountPmuConfiguredSubcores(control_); + const bool expected_state = command == PmuCommand::Configure + ? control_.configured == 1 && control_.processed_subcores == control_.expected_subcores && + bitmap_count == control_.expected_subcores && + control_.skipped_subcores + bitmap_count == kPmuPhysicalSubcores + : control_.configured == 0 && control_.processed_subcores == 0 && bitmap_count == 0; + if (control_.status != 0 || !expected_state) { + const auto failed_field = static_cast(control_.first_failed_field); + std::fprintf( + stderr, + "PMU helper failed: command=%u status=%d configured=%u processed=%u bitmap_count=%u " + "expected_subcores=%u skipped=%u failed_index=%u failed_field=%s(%u) " + "observed=0x%x expected=0x%x\n", + control_.command, static_cast(control_.status), control_.configured, + control_.processed_subcores, bitmap_count, control_.expected_subcores, + control_.skipped_subcores, control_.first_failed_index, + PmuRegisterFieldName(failed_field), control_.first_failed_field, + control_.first_failed_observed, control_.first_failed_expected + ); + return false; + } + if (command == PmuCommand::Configure) { + const auto failed_field = static_cast(control_.first_failed_field); + std::printf( + "[PMU_OWNER] configured=%u bitmap_count=%u expected=%u skipped=%u " + "first_skipped_index=%u first_skipped_field=%s(%u) observed=0x%x expected_value=0x%x\n", + control_.processed_subcores, bitmap_count, control_.expected_subcores, + control_.skipped_subcores, control_.first_failed_index, + PmuRegisterFieldName(failed_field), control_.first_failed_field, + control_.first_failed_observed, control_.first_failed_expected + ); + std::printf( + "[PMU_OWNER] configured_bitmap=%08x:%08x:%08x:%08x\n", + control_.configured_bitmap[3], control_.configured_bitmap[2], + control_.configured_bitmap[1], control_.configured_bitmap[0] + ); + } + return true; + } + + uint32_t device_ = 0; + aclrtStream stream_ = nullptr; + RegisterMappings mappings_; + host::LoadAicpuOp loader_; + void *register_bases_device_ = nullptr; + void *control_device_ = nullptr; + PmuControl control_{}; + KernelArgs kernel_args_{}; + ActiveSubcoreLimits active_limits_{}; + bool initialized_ = false; + bool configured_ = false; +}; + +} // namespace atomic_probe::pmu + +#endif // TESTS_ATOMIC_PROBE_CCEC_PMU_PROBE_HOST_SUPPORT_H_ diff --git a/tests/atomic_probe/ccec/run_all.sh b/tests/atomic_probe/ccec/run_all.sh index f33d931099..a49ee94bec 100755 --- a/tests/atomic_probe/ccec/run_all.sh +++ b/tests/atomic_probe/ccec/run_all.sh @@ -1,13 +1,22 @@ #!/bin/bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- # Build & run ALL ccec atomic probes on A5 onboard hardware. # # For each probe: compiles the kernel .cpp with ccec -x cce, links it into # an AICore binary with ld.lld, then compiles the host launcher with g++ and -# runs it. Probes are AIV-only except the explicit cross-TU compiler/ABI probe, -# which targets AIC to match the affected orchestration build. +# runs it. Probes are AIV-only except the explicit caller-capture compiler +# probes, which target AIC to match the affected orchestration build. # -# All kernels are pure-CCEC (ccec_utils.h + lowercase builtins); no -# kernel_operator.h, no AscendC APIs. +# All kernels are pure CCEC; no kernel_operator.h or AscendC APIs. The shared +# TensorMap visibility probe additionally includes and calls the production +# FDWIC CCEC headers instead of copying their state machine. # # Usage: # ./run_all.sh # build + run all @@ -60,6 +69,7 @@ RUN_TIMEOUT="${ATOMIC_PROBE_TIMEOUT:-120}" PROBES=( "nested_lambda.cpp:nested_lambda_kernel.o:nested_lambda_host.cpp:nested_lambda_host" "atomic_cas_probe.cpp:atomic_cas_kernel.o:atomic_cas_host.cpp:atomic_cas_host" + "st_dev_ld_dev_sync.cpp:st_dev_ld_dev_sync_kernel.o:st_dev_ld_dev_sync_host.cpp:st_dev_ld_dev_sync_host" "entire_flush_clobber.cpp:entire_flush_clobber_kernel.o:entire_flush_clobber_host.cpp:entire_flush_clobber_host" "bypass_dcache_ccec.cpp:bypass_dcache_kernel.o:bypass_dcache_ccec_host.cpp:bypass_dcache_ccec_host" "dcci_clean_clobber.cpp:dcci_clean_kernel.o:dcci_clean_clobber_host.cpp:dcci_clean_host" @@ -73,13 +83,17 @@ PROBES=( "st_dev_single_core_stress.cpp:st_dev_single_core_stress_kernel.o:st_dev_single_core_stress_host.cpp:st_dev_single_core_stress_host" "ld_dev_fanout_publish.cpp:ld_dev_fanout_publish_kernel.o:ld_dev_fanout_publish_host.cpp:ld_dev_fanout_publish_host" "cacheline_matrix.cpp:cacheline_matrix_kernel.o:cacheline_matrix_host.cpp:cacheline_matrix_host" + "taskcell_atomic_dcci.cpp:taskcell_atomic_dcci_kernel.o:taskcell_atomic_dcci_host.cpp:taskcell_atomic_dcci_host" + "shared_tensor_map_visibility_probe.cpp:shared_tensor_map_visibility_probe_kernel.o:shared_tensor_map_visibility_probe_host.cpp:shared_tensor_map_visibility_probe_host" ) -# This compiler-regression probe can intentionally trigger an AICore exception -# on affected CCEC builds, so it is selectable by name but is not part of the -# default cache-line suite. +# These caller-capture build-shape probes are selectable by name but are not +# part of the default cache-line suite. The noinline target can intentionally +# trigger an AICore exception on affected CCEC builds. MANUAL_PROBES=( "nested_lambda_cross_tu.cpp:nested_lambda_cross_tu_kernel.o:nested_lambda_cross_tu_host.cpp:nested_lambda_cross_tu_host" + "nested_lambda_inline_plus_empty_runtime.cpp:nested_lambda_inline_plus_empty_runtime_kernel.o:nested_lambda_cross_tu_host.cpp:nested_lambda_inline_plus_empty_runtime_host" + "nested_lambda_only_weak_submit_noinline.cpp:nested_lambda_only_weak_submit_noinline_kernel.o:nested_lambda_cross_tu_host.cpp:nested_lambda_only_weak_submit_noinline_host" ) REQUESTED="${1:-all}" @@ -122,12 +136,43 @@ build_one() { tag="$(basename "$ko" .o)" local probe_flags=(-DCCEC_SYNC_AIV_ONLY) + local kernel_inc_flags=("${INC_FLAGS[@]}") + local host_inc_flags=() + local host_definition_flags=() if [[ "$ks" == "cacheline_matrix.cpp" ]]; then probe_flags+=(-DCCEC_MATRIX_AIV_ONLY) fi + if [[ "$ks" == "shared_tensor_map_visibility_probe.cpp" ]]; then + local repo_root + repo_root="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)" + local fdwic_inc_flags=( + -I"$repo_root/src/a5/platform/onboard/aicore" + -I"$repo_root/src/a5/platform/include" + -I"$repo_root/src/common/platform/include" + -I"$repo_root/src/common/task_interface" + -I"$repo_root/src/common/log/include" + -I"$repo_root/src/common" + -I"$repo_root/src/a5/runtime/fully_distributed_within_core/runtime" + -I"$repo_root/src/a5/runtime/fully_distributed_within_core/common" + -I"$repo_root/src/a5/runtime/fully_distributed_within_core/orchestration" + -I"$repo_root/src/a5/runtime" + ) + kernel_inc_flags+=("${fdwic_inc_flags[@]}") + host_inc_flags+=("${fdwic_inc_flags[@]}") + probe_flags+=( + -DPTO_FDWIC_SHARED_MAP=1 + -DPTO_FDWIC_TENSORMAP_RING_CAP=128 + ) + host_definition_flags+=( + -DPTO_FDWIC_SHARED_MAP=1 + -DPTO_FDWIC_TENSORMAP_RING_CAP=128 + ) + fi local kernel_objects=() - if [[ "$ks" == "nested_lambda_cross_tu.cpp" ]]; then + if [[ "$ks" == "nested_lambda_cross_tu.cpp" || + "$ks" == "nested_lambda_inline_plus_empty_runtime.cpp" || + "$ks" == "nested_lambda_only_weak_submit_noinline.cpp" ]]; then local repo_root repo_root="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)" local cross_tu_inc_flags=( @@ -147,18 +192,37 @@ build_one() { "$CCEC" "${CCEC_FLAGS[@]}" --cce-aicore-arch=dav-c310-cube \ "${cross_tu_inc_flags[@]}" \ -o "$BUILD_DIR/${tag}_caller_aic.o" "$SCRIPT_DIR/$ks" - echo "=== [$tag] Compiling runtime TU for AIC (dav-c310-cube) ===" - "$CCEC" "${CCEC_FLAGS[@]}" --cce-aicore-arch=dav-c310-cube \ - "${cross_tu_inc_flags[@]}" \ - -o "$BUILD_DIR/${tag}_runtime_aic.o" "$SCRIPT_DIR/nested_lambda_cross_tu_runtime.cpp" - kernel_objects+=( - "$BUILD_DIR/${tag}_caller_aic.o" - "$BUILD_DIR/${tag}_runtime_aic.o" - ) + kernel_objects+=("$BUILD_DIR/${tag}_caller_aic.o") + if [[ "$ks" == "nested_lambda_inline_plus_empty_runtime.cpp" ]]; then + echo "=== [$tag] Compiling empty runtime TU for AIC (dav-c310-cube) ===" + "$CCEC" "${CCEC_FLAGS[@]}" --cce-aicore-arch=dav-c310-cube \ + "${cross_tu_inc_flags[@]}" \ + -o "$BUILD_DIR/${tag}_runtime_empty_aic.o" \ + "$SCRIPT_DIR/nested_lambda_cross_tu_runtime.cpp" + kernel_objects+=("$BUILD_DIR/${tag}_runtime_empty_aic.o") + fi + local expected_objects=1 + if [[ "$ks" == "nested_lambda_inline_plus_empty_runtime.cpp" ]]; then + expected_objects=2 + local section_table runtime_text_size + if ! section_table="$(readelf -S -W "$BUILD_DIR/${tag}_runtime_empty_aic.o")"; then + echo "Error: failed to read empty runtime object sections" >&2 + exit 1 + fi + runtime_text_size="$(awk '$3 == ".text" {print $7}' <<< "$section_table")" + if [[ "$runtime_text_size" != "000000" ]]; then + echo "Error: runtime object control must have an empty .text section" >&2 + exit 1 + fi + fi + if [[ "${#kernel_objects[@]}" -ne "$expected_objects" ]]; then + echo "Error: unexpected caller-capture AIC input object count" >&2 + exit 1 + fi else echo "=== [$tag] Compiling AIV-only (dav-c310-vec) ===" "$CCEC" "${CCEC_FLAGS[@]}" --cce-aicore-arch=dav-c310-vec \ - "${probe_flags[@]}" "${INC_FLAGS[@]}" \ + "${probe_flags[@]}" "${kernel_inc_flags[@]}" \ -o "$BUILD_DIR/${tag}_vec.o" "$SCRIPT_DIR/$ks" kernel_objects+=("$BUILD_DIR/${tag}_vec.o") fi @@ -167,9 +231,39 @@ build_one() { "$LD" -m aicorelinux -Ttext=0 -static --allow-multiple-definition \ -o "$BUILD_DIR/$ko" "${kernel_objects[@]}" + if [[ "$ks" == "nested_lambda_cross_tu.cpp" || + "$ks" == "nested_lambda_inline_plus_empty_runtime.cpp" || + "$ks" == "nested_lambda_only_weak_submit_noinline.cpp" ]]; then + local symbol_table submit_symbols + if ! symbol_table="$(readelf -Ws -W "$BUILD_DIR/$ko")"; then + echo "Error: failed to read caller-capture symbol table" >&2 + exit 1 + fi + submit_symbols="$( + awk '$4 == "FUNC" && $8 ~ /nested_probe_submit_/ {sub(/\$local$/, "", $8); print $8}' \ + <<< "$symbol_table" | + sort -u + )" + if [[ "$ks" == "nested_lambda_only_weak_submit_noinline.cpp" ]]; then + if [[ "$(printf '%s\n' "$submit_symbols" | sed '/^$/d' | wc -l)" -ne 1 || + "$submit_symbols" != *nested_probe_submit_weak_context* ]]; then + echo "Error: noinline control must retain only nested_probe_submit_weak_context" >&2 + exit 1 + fi + elif [[ -n "$submit_symbols" ]]; then + echo "Error: inline caller-capture probe retained runtime submit symbols" >&2 + exit 1 + fi + echo "[ASSERT] CCEC caller-capture runtime symbol shape PASS" + echo "[VALUES] aic_input_objects=${#kernel_objects[@]}" \ + "runtime_text=${runtime_text_size:-n/a} submit_symbols=${submit_symbols:-none}" + fi + echo "=== [$tag] Compiling host ===" g++ -O2 -std=c++17 \ -I"$ASCEND_HOME_PATH/include" \ + "${host_definition_flags[@]}" \ + "${host_inc_flags[@]}" \ "$SCRIPT_DIR/$hs" \ -L"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ -Wl,-rpath,"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ @@ -185,24 +279,27 @@ run_one() { local probe_failures=0 tag="$(basename "$ko" .o)" echo "=== Running [$tag] ===" - if [[ "$tag" == "nested_lambda_cross_tu_kernel" ]]; then + if [[ "$tag" == "nested_lambda_cross_tu_kernel" || + "$tag" == "nested_lambda_inline_plus_empty_runtime_kernel" || + "$tag" == "nested_lambda_only_weak_submit_noinline_kernel" ]]; then if [[ -n "${ATOMIC_PROBE_MODE:-}" ]]; then probe_modes=("$ATOMIC_PROBE_MODE") - else - # Run each variant in its own process: an expected AICore exception - # in a bad compiler variant must not hide the remaining controls. + elif [[ "$tag" == "nested_lambda_only_weak_submit_noinline_kernel" ]]; then probe_modes=( strong-context - args-runtime-read - weak-args-storage - weak-context-materialize-3 - weak-context-materialize-2 weak-context-materialize-1 weak-context-materialize-0 ) + elif [[ "$tag" == "nested_lambda_inline_plus_empty_runtime_kernel" ]]; then + probe_modes=(weak-context-materialize-0) + else + probe_modes=( + args-runtime-read + weak-context-materialize-0 + ) fi for probe_mode in "${probe_modes[@]}"; do - echo "--- CCEC cross-TU variant=$probe_mode ---" + echo "--- CCEC caller-capture variant=$probe_mode ---" if ! timeout "$RUN_TIMEOUT" \ "$BUILD_DIR/$hb" "$BUILD_DIR/$ko" "$probe_mode"; then probe_failures=$((probe_failures + 1)) @@ -370,7 +467,9 @@ export LD_LIBRARY_PATH="$ASCEND_HOME_PATH/x86_64-linux/lib64:${LD_LIBRARY_PATH:- selected=0 suite_run_failures=0 entries=("${PROBES[@]}") -if [[ "$SELECT" == "nested_lambda_cross_tu" ]]; then +if [[ "$SELECT" == "nested_lambda_cross_tu" || + "$SELECT" == "nested_lambda_inline_plus_empty_runtime" || + "$SELECT" == "nested_lambda_only_weak_submit_noinline" ]]; then entries+=("${MANUAL_PROBES[@]}") fi for entry in "${entries[@]}"; do diff --git a/tests/atomic_probe/ccec/run_atomic_scalar_pmu.sh b/tests/atomic_probe/ccec/run_atomic_scalar_pmu.sh new file mode 100755 index 0000000000..3db203cf76 --- /dev/null +++ b/tests/atomic_probe/ccec/run_atomic_scalar_pmu.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +# 独立构建并运行单 AIV atomic/scalar PMU 对照；不修改 run_all.sh，也不下载 PTO_ISA。 +# 用法： +# ./run_atomic_scalar_pmu.sh # build + run +# ./run_atomic_scalar_pmu.sh build # 仅构建 +# ./run_atomic_scalar_pmu.sh run # 仅运行已有产物 +# 可选环境：ATOMIC_PROBE_DEVICE、ATOMIC_SCALAR_PMU_REPEATS、 +# ATOMIC_SCALAR_PMU_ROUNDS（逗号分隔）、ATOMIC_SCALAR_PMU_SEED。 +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)" +BUILD_DIR="$SCRIPT_DIR/build/atomic_scalar_pmu" +ACTION="${1:-all}" +if [[ "$ACTION" != "all" && "$ACTION" != "build" && "$ACTION" != "run" ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ $# -gt 1 ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then + echo "ASCEND_HOME_PATH is not set; source the local CANN environment first." >&2 + exit 1 +fi + +CCEC="$ASCEND_HOME_PATH/bin/ccec" +LD="$ASCEND_HOME_PATH/bin/ld.lld" +HCC="$ASCEND_HOME_PATH/tools/hcc/bin/aarch64-target-linux-gnu-g++" +CXX_BIN="${CXX:-g++}" +READELF_BIN="${READELF:-readelf}" +# 优先复用调用者指定的本机 PTO_ISA；未指定时使用 CANN 安装包自带的 metadata header。 +PTO_INCLUDE_ROOT="${PTO_ISA_ROOT:-$ASCEND_HOME_PATH/x86_64-linux}" + +if [[ ! -x "$CCEC" || ! -x "$LD" || ! -x "$HCC" ]]; then + echo "CCEC, ld.lld, or AICPU cross compiler is missing under $ASCEND_HOME_PATH" >&2 + exit 1 +fi +if [[ ! -f "$PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" ]]; then + echo "Local PTO metadata header is missing: $PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" >&2 + exit 1 +fi +if ! command -v "$READELF_BIN" >/dev/null 2>&1; then + echo "readelf is required." >&2 + exit 1 +fi + +KERNEL="$BUILD_DIR/atomic_scalar_pmu_kernel.o" +HOST="$BUILD_DIR/atomic_scalar_pmu_host" + +build_probe() { + mkdir -p "$BUILD_DIR" + local common_flags=( + -c -O3 -g -x cce -Wall -std=c++17 + --cce-aicore-only + --cce-aicore-arch=dav-c310-vec + -mllvm -cce-aicore-stack-size=0x8000 + -mllvm -cce-aicore-function-stack-size=0x8000 + -mllvm -cce-aicore-record-overflow=false + -mllvm -cce-aicore-addr-transform + -mllvm -cce-aicore-dcci-insert-for-scalar=false + -mllvm -cce-aicore-dcci-before-kernel-end=false + -DCCEC_SYNC_AIV_ONLY + -I"$SCRIPT_DIR" + -I"$PTO_INCLUDE_ROOT/include" + ) + + echo "[BUILD] single-AIV CCEC kernel" + "$CCEC" "${common_flags[@]}" \ + -o "$BUILD_DIR/atomic_scalar_pmu_vec.o" \ + "$SCRIPT_DIR/atomic_scalar_pmu.cpp" + "$LD" -m aicorelinux -Ttext=0 -static \ + -o "$KERNEL" "$BUILD_DIR/atomic_scalar_pmu_vec.o" + + local symbols sections entry="atomic_scalar_pmu_0_mix_aiv" + symbols="$("$READELF_BIN" --symbols --wide "$KERNEL")" + sections="$("$READELF_BIN" --sections --wide "$KERNEL")" + if [[ "$symbols" != *" $entry"* || "$sections" != *".ascend.meta.$entry"* ]]; then + echo "Missing AIV entry or metadata for $entry" >&2 + exit 1 + fi + echo "[CHECK] AIV-only entry and metadata present" + + # 两个 scalar PMU 探针复用同一份 108-subcore 配置/恢复 helper。 + echo "[BUILD] AICPU PMU configure/restore helper" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra -Werror \ + -Wl,--build-id \ + -I"$SCRIPT_DIR" \ + -I"$REPO_ROOT/src/a5/platform/include" \ + "$SCRIPT_DIR/pmu_probe_aicpu.cpp" \ + "$REPO_ROOT/src/a5/platform/onboard/aicpu/inner_platform_regs.cpp" \ + -o "$BUILD_DIR/libatomic_scalar_pmu_aicpu.so" + + echo "[BUILD] AICPU bootstrap dispatcher" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra \ + -Wl,--build-id \ + -I"$REPO_ROOT/src/common" \ + "$REPO_ROOT/src/common/aicpu_loader/device/aicpu_dispatcher.cpp" \ + -ldl \ + -o "$BUILD_DIR/libsimpler_aicpu_dispatcher.so" + + echo "[BUILD] host runner" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror -Wno-deprecated-declarations \ + -I"$REPO_ROOT/src/common" \ + -I"$REPO_ROOT/src/common/log/include" \ + -I"$REPO_ROOT/src/a5/platform/include" \ + -I"$ASCEND_HOME_PATH/include" \ + -I"$ASCEND_HOME_PATH/pkg_inc" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime/runtime" \ + "$SCRIPT_DIR/atomic_scalar_pmu_host.cpp" \ + "$REPO_ROOT/src/common/aicpu_loader/host/load_aicpu_op.cpp" \ + "$REPO_ROOT/src/common/log/host_log.cpp" \ + "$REPO_ROOT/src/common/log/unified_log_host.cpp" \ + -L"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -Wl,-rpath,"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -lascendcl -lruntime -ldl -pthread \ + -o "$HOST" + echo "[BUILD] complete: $BUILD_DIR" +} + +run_probe() { + if [[ ! -x "$HOST" || ! -f "$KERNEL" || ! -f "$BUILD_DIR/libatomic_scalar_pmu_aicpu.so" || + ! -f "$BUILD_DIR/libsimpler_aicpu_dispatcher.so" ]]; then + echo "Build artifacts are incomplete; run '$0 build' first." >&2 + exit 1 + fi + echo "[RUN] device=${ATOMIC_PROBE_DEVICE:-${TASK_DEVICE:-0}}" + timeout "${ATOMIC_PROBE_TIMEOUT:-120}" "$HOST" "$KERNEL" +} + +if [[ "$ACTION" == "all" || "$ACTION" == "build" ]]; then + build_probe +fi +if [[ "$ACTION" == "all" || "$ACTION" == "run" ]]; then + run_probe +fi diff --git a/tests/atomic_probe/ccec/run_cache_preload.sh b/tests/atomic_probe/ccec/run_cache_preload.sh new file mode 100755 index 0000000000..9570bcaf6b --- /dev/null +++ b/tests/atomic_probe/ccec/run_cache_preload.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +# Build and run the pure-CCEC A5 cache-preload reference probe without +# changing ccec/run_all.sh. +# +# Usage: +# ./run_cache_preload.sh # build, linked-ELF checks, and A5 run +# ./run_cache_preload.sh build # build and linked-ELF checks only +# ./run_cache_preload.sh run # recheck existing ELF and run +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BUILD_DIR="$SCRIPT_DIR/build/cache_preload" +ACTION="${1:-all}" +if [[ "$ACTION" != "all" && "$ACTION" != "build" && "$ACTION" != "run" ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ $# -gt 1 ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then + echo "ASCEND_HOME_PATH is not set; source the local CANN environment first." >&2 + exit 1 +fi + +CCEC="$ASCEND_HOME_PATH/bin/ccec" +LD="$ASCEND_HOME_PATH/bin/ld.lld" +CXX_BIN="${CXX:-g++}" +READELF_BIN="${READELF:-readelf}" +PTO_INCLUDE_ROOT="${PTO_ISA_ROOT:-$ASCEND_HOME_PATH/x86_64-linux}" +KERNEL="$BUILD_DIR/cache_preload_kernel.o" +HOST="$BUILD_DIR/cache_preload_host" + +if [[ ! -x "$CCEC" || ! -x "$LD" ]]; then + echo "ccec or ld.lld is missing under ASCEND_HOME_PATH=$ASCEND_HOME_PATH" >&2 + exit 1 +fi +if [[ ! -f "$PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" ]]; then + echo "PTO metadata header is missing: $PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" >&2 + exit 1 +fi +if ! command -v "$CXX_BIN" >/dev/null 2>&1; then + echo "Host C++ compiler is missing: $CXX_BIN" >&2 + exit 1 +fi +if ! command -v "$READELF_BIN" >/dev/null 2>&1; then + echo "readelf is required: $READELF_BIN" >&2 + exit 1 +fi + +symbol_address_hex="" +symbol_size="" +read_unique_function_symbol() { + local elf="$1" + local name="$2" + local -a matches=() + mapfile -t matches < <( + "$READELF_BIN" --symbols --wide "$elf" | + awk -v symbol="$name" '$4 == "FUNC" && $8 == symbol {print $2 " " $3}' + ) + if [[ ${#matches[@]} -ne 1 ]]; then + echo "Expected exactly one FUNC symbol named $name; found ${#matches[@]}" >&2 + exit 1 + fi + read -r symbol_address_hex symbol_size <<<"${matches[0]}" + if [[ ! "$symbol_address_hex" =~ ^[0-9a-fA-F]+$ || + ! "$symbol_size" =~ ^[0-9]+$ ]]; then + echo "Cannot parse ELF symbol $name: ${matches[0]}" >&2 + exit 1 + fi +} + +check_kernel_elf() { + if [[ ! -f "$KERNEL" ]]; then + echo "Kernel ELF is missing: $KERNEL" >&2 + exit 1 + fi + local symbols sections entry="cache_preload_0_mix_aiv" + symbols="$("$READELF_BIN" --symbols --wide "$KERNEL")" + sections="$("$READELF_BIN" --sections --wide "$KERNEL")" + if [[ "$symbols" != *" $entry"* || + "$sections" != *".ascend.meta.$entry"* ]]; then + echo "Missing AIV entry or metadata for $entry" >&2 + exit 1 + fi + + read_unique_function_symbol "$KERNEL" "cache_preload_icache_path" + local path_address_hex="$symbol_address_hex" + local path_size="$symbol_size" + read_unique_function_symbol "$KERNEL" "cache_preload_icache_evictor" + local evictor_address_hex="$symbol_address_hex" + local evictor_size="$symbol_size" + read_unique_function_symbol "$KERNEL" "cache_preload_gap" + + local path_address=$((16#$path_address_hex)) + local evictor_address=$((16#$evictor_address_hex)) + local path_end=$((path_address + path_size)) + local evictor_end=$((evictor_address + evictor_size)) + if ((path_size < 4096)); then + echo "current-PC path is smaller than 4096B: $path_size" >&2 + exit 1 + fi + if ((evictor_size < 32768)); then + echo "ICache evictor is smaller than 32768B: $evictor_size" >&2 + exit 1 + fi + if ((path_address % 128 != 0 || evictor_address % 128 != 0)); then + printf 'ICache symbols are not 128B-aligned: path=0x%x evictor=0x%x\n' \ + "$path_address" "$evictor_address" >&2 + exit 1 + fi + if ! ((path_end <= evictor_address || evictor_end <= path_address)); then + printf 'path/evictor overlap: path=[0x%x,0x%x) evictor=[0x%x,0x%x)\n' \ + "$path_address" "$path_end" "$evictor_address" "$evictor_end" >&2 + exit 1 + fi + printf '[CHECK] AIV entry/meta PASS; current-PC path=0x%x/%uB; evictor=0x%x/%uB; ' \ + "$path_address" "$path_size" "$evictor_address" "$evictor_size" + echo "alignment=128B ranges=non-overlap" +} + +build_probe() { + mkdir -p "$BUILD_DIR" + local common_flags=( + -c -O3 -g -x cce -Wall -std=c++17 + --cce-aicore-only + --cce-aicore-arch=dav-c310-vec + -mllvm -cce-aicore-stack-size=0x8000 + -mllvm -cce-aicore-function-stack-size=0x8000 + -mllvm -cce-aicore-record-overflow=false + -mllvm -cce-aicore-addr-transform + -mllvm -cce-aicore-dcci-insert-for-scalar=false + -mllvm -cce-aicore-dcci-before-kernel-end=false + -DCCEC_SYNC_AIV_ONLY + -I"$SCRIPT_DIR" + -I"$PTO_INCLUDE_ROOT/include" + ) + + echo "[BUILD] pure CCEC cache preload kernel (dav-c310-vec)" + "$CCEC" "${common_flags[@]}" \ + -o "$BUILD_DIR/cache_preload_vec.o" \ + "$SCRIPT_DIR/cache_preload.cpp" + "$LD" -m aicorelinux -Ttext=0 -static \ + -o "$KERNEL" "$BUILD_DIR/cache_preload_vec.o" + check_kernel_elf + + echo "[BUILD] host runner" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror \ + -I"$ASCEND_HOME_PATH/include" \ + "$SCRIPT_DIR/cache_preload_host.cpp" \ + -L"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -Wl,-rpath,"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -lascendcl \ + -o "$HOST" + echo "[BUILD] complete: $BUILD_DIR" +} + +run_probe() { + if [[ ! -x "$HOST" || ! -f "$KERNEL" ]]; then + echo "Build artifacts are incomplete; run '$0 build' first." >&2 + exit 1 + fi + check_kernel_elf + echo "[RUN] device=${ATOMIC_PROBE_DEVICE:-${TASK_DEVICE:-0}}" + timeout "${CACHE_PRELOAD_TIMEOUT:-120}" "$HOST" "$KERNEL" +} + +echo "=== Pure CCEC Cache Preload Probe ===" +echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" +echo "git_head=$(git -C "$SCRIPT_DIR" rev-parse HEAD 2>/dev/null || echo unknown)" +echo "ascend_home=$ASCEND_HOME_PATH" +echo "ccec=$($CCEC --version 2>&1 | head -n 1)" +echo "action=$ACTION" + +if [[ "$ACTION" == "all" || "$ACTION" == "build" ]]; then + build_probe +fi +if [[ "$ACTION" == "all" || "$ACTION" == "run" ]]; then + run_probe +fi diff --git a/tests/atomic_probe/ccec/run_icache_scalar_pmu.sh b/tests/atomic_probe/ccec/run_icache_scalar_pmu.sh new file mode 100755 index 0000000000..14159538a7 --- /dev/null +++ b/tests/atomic_probe/ccec/run_icache_scalar_pmu.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +# 独立构建并运行单 AIV WARM/COLD I-cache PMU 配对；不修改 run_all.sh， +# 也不下载 PTO_ISA。最终 ELF 的符号尺寸、128B 对齐和区间不重叠是运行前硬条件。 +# 用法： +# ./run_icache_scalar_pmu.sh # build + run（默认 11 对） +# ./run_icache_scalar_pmu.sh build # 仅构建和检查 ELF +# ./run_icache_scalar_pmu.sh run # 仅检查已有 ELF 后运行 +# 可选环境：ATOMIC_PROBE_DEVICE、ICACHE_SCALAR_PMU_REPEATS（11..101）、 +# ICACHE_SCALAR_PMU_SEED、ICACHE_SCALAR_PMU_TIMEOUT。 +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)" +BUILD_DIR="$SCRIPT_DIR/build/icache_scalar_pmu" +ACTION="${1:-all}" +if [[ "$ACTION" != "all" && "$ACTION" != "build" && "$ACTION" != "run" ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ $# -gt 1 ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then + echo "ASCEND_HOME_PATH is not set; source the local CANN environment first." >&2 + exit 1 +fi + +CCEC="$ASCEND_HOME_PATH/bin/ccec" +LD="$ASCEND_HOME_PATH/bin/ld.lld" +HCC="$ASCEND_HOME_PATH/tools/hcc/bin/aarch64-target-linux-gnu-g++" +CXX_BIN="${CXX:-g++}" +READELF_BIN="${READELF:-readelf}" +# 优先复用调用者指定的本机 PTO_ISA；未指定时使用 CANN 安装包自带的 metadata header。 +PTO_INCLUDE_ROOT="${PTO_ISA_ROOT:-$ASCEND_HOME_PATH/x86_64-linux}" + +if [[ ! -x "$CCEC" || ! -x "$LD" || ! -x "$HCC" ]]; then + echo "CCEC, ld.lld, or AICPU cross compiler is missing under $ASCEND_HOME_PATH" >&2 + exit 1 +fi +if [[ ! -f "$PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" ]]; then + echo "Local PTO metadata header is missing: $PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" >&2 + exit 1 +fi +if ! command -v "$CXX_BIN" >/dev/null 2>&1; then + echo "Host C++ compiler is missing: $CXX_BIN" >&2 + exit 1 +fi +if ! command -v "$READELF_BIN" >/dev/null 2>&1; then + echo "readelf is required: $READELF_BIN" >&2 + exit 1 +fi + +KERNEL="$BUILD_DIR/icache_scalar_pmu_kernel.o" +HOST="$BUILD_DIR/icache_scalar_pmu_host" + +symbol_address_hex="" +symbol_size="" +read_unique_function_symbol() { + local elf="$1" + local name="$2" + local -a matches=() + mapfile -t matches < <( + "$READELF_BIN" --symbols --wide "$elf" | + awk -v symbol="$name" '$4 == "FUNC" && $8 == symbol {print $2 " " $3}' + ) + if [[ ${#matches[@]} -ne 1 ]]; then + echo "Expected exactly one FUNC symbol named $name; found ${#matches[@]}" >&2 + exit 1 + fi + read -r symbol_address_hex symbol_size <<<"${matches[0]}" + if [[ ! "$symbol_address_hex" =~ ^[0-9a-fA-F]+$ || ! "$symbol_size" =~ ^[0-9]+$ ]]; then + echo "Cannot parse ELF symbol $name: ${matches[0]}" >&2 + exit 1 + fi +} + +check_kernel_elf() { + if [[ ! -f "$KERNEL" ]]; then + echo "Kernel ELF is missing: $KERNEL" >&2 + exit 1 + fi + + local symbols sections entry="icache_scalar_pmu_0_mix_aiv" + symbols="$("$READELF_BIN" --symbols --wide "$KERNEL")" + sections="$("$READELF_BIN" --sections --wide "$KERNEL")" + if [[ "$symbols" != *" $entry"* || "$sections" != *".ascend.meta.$entry"* ]]; then + echo "Missing AIV entry or metadata for $entry" >&2 + exit 1 + fi + + read_unique_function_symbol "$KERNEL" "icache_scalar_pmu_target" + local target_address_hex="$symbol_address_hex" + local target_size="$symbol_size" + read_unique_function_symbol "$KERNEL" "icache_scalar_pmu_evictor" + local evictor_address_hex="$symbol_address_hex" + local evictor_size="$symbol_size" + + local target_address=$((16#$target_address_hex)) + local evictor_address=$((16#$evictor_address_hex)) + local target_end=$((target_address + target_size)) + local evictor_end=$((evictor_address + evictor_size)) + if ((target_size < 8192)); then + echo "target is smaller than 8192B: $target_size" >&2 + exit 1 + fi + if ((evictor_size < 32768)); then + echo "evictor is smaller than 32768B: $evictor_size" >&2 + exit 1 + fi + if ((target_address % 128 != 0)); then + echo "target address is not 128B-aligned: 0x$target_address_hex" >&2 + exit 1 + fi + if ((evictor_address % 128 != 0)); then + echo "evictor address is not 128B-aligned: 0x$evictor_address_hex" >&2 + exit 1 + fi + if ! ((target_end <= evictor_address || evictor_end <= target_address)); then + printf 'target/evictor ranges overlap: target=[0x%x,0x%x) evictor=[0x%x,0x%x)\n' \ + "$target_address" "$target_end" "$evictor_address" "$evictor_end" >&2 + exit 1 + fi + printf '[CHECK] AIV entry/meta PASS; target address=0x%x size=%u; evictor address=0x%x size=%u; ' \ + "$target_address" "$target_size" "$evictor_address" "$evictor_size" + echo "alignment=128B ranges=non-overlap" +} + +build_probe() { + mkdir -p "$BUILD_DIR" + local common_flags=( + -c -O3 -g -x cce -Wall -std=c++17 + --cce-aicore-only + --cce-aicore-arch=dav-c310-vec + -mllvm -cce-aicore-stack-size=0x8000 + -mllvm -cce-aicore-function-stack-size=0x8000 + -mllvm -cce-aicore-record-overflow=false + -mllvm -cce-aicore-addr-transform + -mllvm -cce-aicore-dcci-insert-for-scalar=false + -mllvm -cce-aicore-dcci-before-kernel-end=false + -DCCEC_SYNC_AIV_ONLY + -I"$SCRIPT_DIR" + -I"$PTO_INCLUDE_ROOT/include" + ) + + echo "[BUILD] single-AIV CCEC WARM/COLD I-cache kernel" + "$CCEC" "${common_flags[@]}" \ + -o "$BUILD_DIR/icache_scalar_pmu_vec.o" \ + "$SCRIPT_DIR/icache_scalar_pmu.cpp" + "$LD" -m aicorelinux -Ttext=0 -static \ + -o "$KERNEL" "$BUILD_DIR/icache_scalar_pmu_vec.o" + check_kernel_elf + + # 与 atomic_scalar_pmu 复用同一份 108-subcore PMU 配置/恢复源码； + # 仅产物名独立，避免两个 probe 的 build 目录相互依赖。 + echo "[BUILD] AICPU PMU configure/restore helper" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra -Werror \ + -Wl,--build-id \ + -I"$SCRIPT_DIR" \ + -I"$REPO_ROOT/src/a5/platform/include" \ + "$SCRIPT_DIR/pmu_probe_aicpu.cpp" \ + "$REPO_ROOT/src/a5/platform/onboard/aicpu/inner_platform_regs.cpp" \ + -o "$BUILD_DIR/libicache_scalar_pmu_aicpu.so" + + echo "[BUILD] AICPU bootstrap dispatcher" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra \ + -Wl,--build-id \ + -I"$REPO_ROOT/src/common" \ + "$REPO_ROOT/src/common/aicpu_loader/device/aicpu_dispatcher.cpp" \ + -ldl \ + -o "$BUILD_DIR/libsimpler_aicpu_dispatcher.so" + + echo "[BUILD] host runner" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror -Wno-deprecated-declarations \ + -I"$REPO_ROOT/src/common" \ + -I"$REPO_ROOT/src/common/log/include" \ + -I"$REPO_ROOT/src/a5/platform/include" \ + -I"$ASCEND_HOME_PATH/include" \ + -I"$ASCEND_HOME_PATH/pkg_inc" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime/runtime" \ + "$SCRIPT_DIR/icache_scalar_pmu_host.cpp" \ + "$REPO_ROOT/src/common/aicpu_loader/host/load_aicpu_op.cpp" \ + "$REPO_ROOT/src/common/log/host_log.cpp" \ + "$REPO_ROOT/src/common/log/unified_log_host.cpp" \ + -L"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -Wl,-rpath,"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -lascendcl -lruntime -ldl -pthread \ + -o "$HOST" + echo "[BUILD] complete: $BUILD_DIR" +} + +run_probe() { + if [[ ! -x "$HOST" || ! -f "$KERNEL" || ! -f "$BUILD_DIR/libicache_scalar_pmu_aicpu.so" || + ! -f "$BUILD_DIR/libsimpler_aicpu_dispatcher.so" ]]; then + echo "Build artifacts are incomplete; run '$0 build' first." >&2 + exit 1 + fi + # run-only 也重新检查 ELF，不能绕开尺寸/对齐/区间 oracle。 + check_kernel_elf + echo "[RUN] device=${ATOMIC_PROBE_DEVICE:-${TASK_DEVICE:-0}} pairs=${ICACHE_SCALAR_PMU_REPEATS:-11}" + timeout "${ICACHE_SCALAR_PMU_TIMEOUT:-120}" "$HOST" "$KERNEL" +} + +if [[ "$ACTION" == "all" || "$ACTION" == "build" ]]; then + build_probe +fi +if [[ "$ACTION" == "all" || "$ACTION" == "run" ]]; then + run_probe +fi diff --git a/tests/atomic_probe/ccec/run_shared_preload_model.sh b/tests/atomic_probe/ccec/run_shared_preload_model.sh new file mode 100755 index 0000000000..0389a82d4c --- /dev/null +++ b/tests/atomic_probe/ccec/run_shared_preload_model.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the LICENSE file for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the license. +# ----------------------------------------------------------------------------------------------------------- + +# Build and run the isolated PA-shared cache-preload model. +# +# Usage: +# ./run_shared_preload_model.sh +# ./run_shared_preload_model.sh build +# ./run_shared_preload_model.sh run +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BUILD_DIR="$SCRIPT_DIR/build/shared_preload_model" +ACTION="${1:-all}" +if [[ "$ACTION" != "all" && "$ACTION" != "build" && "$ACTION" != "run" ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ $# -gt 1 ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then + echo "ASCEND_HOME_PATH is not set; source the CANN environment first." >&2 + exit 1 +fi + +CCEC="$ASCEND_HOME_PATH/bin/ccec" +LD="$ASCEND_HOME_PATH/bin/ld.lld" +CXX_BIN="${CXX:-g++}" +READELF_BIN="${READELF:-readelf}" +PTO_INCLUDE_ROOT="${PTO_ISA_ROOT:-$ASCEND_HOME_PATH/x86_64-linux}" +KERNEL="$BUILD_DIR/shared_preload_model_kernel.o" +HOST="$BUILD_DIR/shared_preload_model_host" + +if [[ ! -x "$CCEC" || ! -x "$LD" ]]; then + echo "ccec or ld.lld is missing under ASCEND_HOME_PATH=$ASCEND_HOME_PATH" >&2 + exit 1 +fi +if [[ ! -f "$PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" ]]; then + echo "PTO metadata header is missing: $PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" >&2 + exit 1 +fi +if ! command -v "$CXX_BIN" >/dev/null 2>&1; then + echo "Host C++ compiler is missing: $CXX_BIN" >&2 + exit 1 +fi +if ! command -v "$READELF_BIN" >/dev/null 2>&1; then + echo "readelf is required: $READELF_BIN" >&2 + exit 1 +fi + +symbol_address_hex="" +symbol_size="" +read_unique_function_symbol() { + local elf="$1" + local name="$2" + local -a matches=() + mapfile -t matches < <( + "$READELF_BIN" --symbols --wide "$elf" | + awk -v symbol="$name" \ + '$4 == "FUNC" && $7 != "UND" && $8 == symbol {print $2 " " $3}' + ) + if [[ ${#matches[@]} -ne 1 ]]; then + echo "Expected exactly one defined FUNC named $name; found ${#matches[@]}" >&2 + exit 1 + fi + read -r symbol_address_hex symbol_size <<<"${matches[0]}" + if [[ ! "$symbol_address_hex" =~ ^[0-9a-fA-F]+$ || + ! "$symbol_size" =~ ^[0-9]+$ ]]; then + echo "Cannot parse ELF symbol $name: ${matches[0]}" >&2 + exit 1 + fi +} + +check_kernel_elf() { + if [[ ! -f "$KERNEL" ]]; then + echo "Kernel ELF is missing: $KERNEL" >&2 + exit 1 + fi + local symbols sections + symbols="$("$READELF_BIN" --symbols --wide "$KERNEL")" + sections="$("$READELF_BIN" --sections --wide "$KERNEL")" + for entry in shared_preload_model_0_mix_aic shared_preload_model_0_mix_aiv; do + if ! awk -v name="$entry" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {found = 1} END {exit !found}' \ + <<<"$symbols"; then + echo "Missing non-empty mixed entry: $entry" >&2 + exit 1 + fi + if [[ "$sections" != *".ascend.meta.$entry"* ]]; then + echo "Missing metadata section: .ascend.meta.$entry" >&2 + exit 1 + fi + done + + read_unique_function_symbol "$KERNEL" \ + "shared_preload_model_icache_target" + local target_address_hex="$symbol_address_hex" + local target_size="$symbol_size" + read_unique_function_symbol "$KERNEL" \ + "shared_preload_model_icache_evictor" + local evictor_address_hex="$symbol_address_hex" + local evictor_size="$symbol_size" + read_unique_function_symbol "$KERNEL" \ + "shared_preload_model_icache_caller" + local caller_address_hex="$symbol_address_hex" + local caller_size="$symbol_size" + + local target_address=$((16#$target_address_hex)) + local evictor_address=$((16#$evictor_address_hex)) + local caller_address=$((16#$caller_address_hex)) + local target_end=$((target_address + target_size)) + local evictor_end=$((evictor_address + evictor_size)) + local caller_end=$((caller_address + caller_size)) + local caller_forward_window_end=$((caller_address + 4096)) + + if ((target_size < 4096)); then + echo "ICache target is smaller than 4096B: $target_size" >&2 + exit 1 + fi + if ((evictor_size < 32768)); then + echo "ICache evictor is smaller than 32768B: $evictor_size" >&2 + exit 1 + fi + if ((target_address % 128 != 0 || + evictor_address % 128 != 0 || + caller_address % 128 != 0)); then + printf 'ICache symbols are not 128B aligned: target=0x%x evictor=0x%x caller=0x%x\n' \ + "$target_address" "$evictor_address" "$caller_address" >&2 + exit 1 + fi + if ! ((target_end <= evictor_address || evictor_end <= target_address)); then + echo "ICache target and evictor overlap" >&2 + exit 1 + fi + if ! ((target_end <= caller_address || caller_end <= target_address)); then + echo "ICache target and caller overlap" >&2 + exit 1 + fi + if ! ((target_end <= caller_address || + target_address >= caller_forward_window_end)); then + printf 'Target overlaps caller current-PC 4096B window: target=[0x%x,0x%x) caller=[0x%x,0x%x)\n' \ + "$target_address" "$target_end" "$caller_address" \ + "$caller_forward_window_end" >&2 + exit 1 + fi + + printf '[CHECK] mixed entries/meta PASS; target=0x%x/%uB; evictor=0x%x/%uB; caller=0x%x/%uB; caller-target-window=disjoint\n' \ + "$target_address" "$target_size" "$evictor_address" \ + "$evictor_size" "$caller_address" "$caller_size" +} + +build_probe() { + mkdir -p "$BUILD_DIR" + local common_flags=( + -c -O3 -g -x cce -Wall -std=c++17 + --cce-aicore-only + -mllvm -cce-aicore-stack-size=0x8000 + -mllvm -cce-aicore-function-stack-size=0x8000 + -mllvm -cce-aicore-record-overflow=false + -mllvm -cce-aicore-addr-transform + -mllvm -cce-aicore-dcci-insert-for-scalar=false + -mllvm -cce-aicore-dcci-before-kernel-end=false + -I"$SCRIPT_DIR" + -I"$PTO_INCLUDE_ROOT/include" + ) + + echo "[BUILD] CCEC AIC shared preload model" + "$CCEC" "${common_flags[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DSHARED_PRELOAD_MODEL_BUILD_AIC \ + -o "$BUILD_DIR/shared_preload_model_aic.o" \ + "$SCRIPT_DIR/shared_preload_model.cpp" + echo "[BUILD] CCEC AIV shared preload model" + "$CCEC" "${common_flags[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DSHARED_PRELOAD_MODEL_BUILD_AIV \ + -o "$BUILD_DIR/shared_preload_model_aiv.o" \ + "$SCRIPT_DIR/shared_preload_model.cpp" + echo "[BUILD] Static 1:2 mixed AICore ELF" + "$LD" -m aicorelinux -Ttext=0 -static \ + -o "$KERNEL" \ + "$BUILD_DIR/shared_preload_model_aic.o" \ + "$BUILD_DIR/shared_preload_model_aiv.o" + check_kernel_elf + + echo "[BUILD] host runner" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror \ + -I"$ASCEND_HOME_PATH/include" \ + "$SCRIPT_DIR/shared_preload_model_host.cpp" \ + -L"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -Wl,-rpath,"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -lascendcl \ + -o "$HOST" + echo "[BUILD] complete: $BUILD_DIR" +} + +run_probe() { + if [[ ! -x "$HOST" || ! -f "$KERNEL" ]]; then + echo "Build artifacts are incomplete; run '$0 build' first." >&2 + exit 1 + fi + check_kernel_elf + echo "[RUN] device=${ATOMIC_PROBE_DEVICE:-${TASK_DEVICE:-0}}" + timeout "${SHARED_PRELOAD_MODEL_TIMEOUT:-180}" \ + "$HOST" "$KERNEL" +} + +echo "=== CCEC PA-shared Cache Preload Model ===" +echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" +echo "git_head=$(git -C "$SCRIPT_DIR" rev-parse HEAD 2>/dev/null || echo unknown)" +echo "git_branch=$(git -C "$SCRIPT_DIR" branch --show-current 2>/dev/null || echo unknown)" +echo "ascend_home=$ASCEND_HOME_PATH" +echo "ccec=$($CCEC --version 2>&1 | head -n 1)" +echo "action=$ACTION" + +if [[ "$ACTION" == "all" || "$ACTION" == "build" ]]; then + build_probe +fi +if [[ "$ACTION" == "all" || "$ACTION" == "run" ]]; then + run_probe +fi diff --git a/tests/atomic_probe/ccec/run_trace_write_preload.sh b/tests/atomic_probe/ccec/run_trace_write_preload.sh new file mode 100755 index 0000000000..b3ddba3e5e --- /dev/null +++ b/tests/atomic_probe/ccec/run_trace_write_preload.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the LICENSE file for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the license. +# ----------------------------------------------------------------------------------------------------------- + +# Build and run the isolated 1:2 mixed A5 trace-write preload experiment. +# +# Usage: +# ./run_trace_write_preload.sh +# ./run_trace_write_preload.sh build +# ./run_trace_write_preload.sh run +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BUILD_DIR="$SCRIPT_DIR/build/trace_write_preload" +ACTION="${1:-all}" +if [[ "$ACTION" != "all" && "$ACTION" != "build" && "$ACTION" != "run" ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ $# -gt 1 ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then + echo "ASCEND_HOME_PATH is not set; source the CANN environment first." >&2 + exit 1 +fi + +CCEC="$ASCEND_HOME_PATH/bin/ccec" +LD="$ASCEND_HOME_PATH/bin/ld.lld" +CXX_BIN="${CXX:-g++}" +READELF_BIN="${READELF:-readelf}" +PTO_INCLUDE_ROOT="${PTO_ISA_ROOT:-$ASCEND_HOME_PATH/x86_64-linux}" +KERNEL="$BUILD_DIR/trace_write_preload_kernel.o" +HOST="$BUILD_DIR/trace_write_preload_host" + +if [[ ! -x "$CCEC" || ! -x "$LD" ]]; then + echo "ccec or ld.lld is missing under ASCEND_HOME_PATH=$ASCEND_HOME_PATH" >&2 + exit 1 +fi +if [[ ! -f "$PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" ]]; then + echo "PTO metadata header is missing: $PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" >&2 + exit 1 +fi +if ! command -v "$CXX_BIN" >/dev/null 2>&1; then + echo "Host C++ compiler is missing: $CXX_BIN" >&2 + exit 1 +fi +if ! command -v "$READELF_BIN" >/dev/null 2>&1; then + echo "readelf is required: $READELF_BIN" >&2 + exit 1 +fi + +check_kernel_elf() { + if [[ ! -f "$KERNEL" ]]; then + echo "Kernel ELF is missing: $KERNEL" >&2 + exit 1 + fi + local symbols sections + symbols="$("$READELF_BIN" --symbols --wide "$KERNEL")" + sections="$("$READELF_BIN" --sections --wide "$KERNEL")" + for entry in trace_write_preload_0_mix_aic trace_write_preload_0_mix_aiv; do + if ! awk -v name="$entry" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {found = 1} END {exit !found}' \ + <<<"$symbols"; then + echo "Missing non-empty mixed entry: $entry" >&2 + exit 1 + fi + if [[ "$sections" != *".ascend.meta.$entry"* ]]; then + echo "Missing metadata section: .ascend.meta.$entry" >&2 + exit 1 + fi + done + echo "[CHECK] 1:2 mixed AIC/AIV entries and metadata PASS" +} + +build_probe() { + mkdir -p "$BUILD_DIR" + local common_flags=( + -c -O3 -g -x cce -Wall -std=c++17 + --cce-aicore-only + -mllvm -cce-aicore-stack-size=0x8000 + -mllvm -cce-aicore-function-stack-size=0x8000 + -mllvm -cce-aicore-record-overflow=false + -mllvm -cce-aicore-addr-transform + -mllvm -cce-aicore-dcci-insert-for-scalar=false + -mllvm -cce-aicore-dcci-before-kernel-end=false + -I"$SCRIPT_DIR" + -I"$PTO_INCLUDE_ROOT/include" + ) + + echo "[BUILD] CCEC AIC trace-write preload entry" + "$CCEC" "${common_flags[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DTRACE_WRITE_PRELOAD_BUILD_AIC \ + -o "$BUILD_DIR/trace_write_preload_aic.o" \ + "$SCRIPT_DIR/trace_write_preload.cpp" + echo "[BUILD] CCEC AIV trace-write preload entry" + "$CCEC" "${common_flags[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DTRACE_WRITE_PRELOAD_BUILD_AIV \ + -o "$BUILD_DIR/trace_write_preload_aiv.o" \ + "$SCRIPT_DIR/trace_write_preload.cpp" + echo "[BUILD] Static 1:2 mixed AICore ELF" + "$LD" -m aicorelinux -Ttext=0 -static \ + -o "$KERNEL" \ + "$BUILD_DIR/trace_write_preload_aic.o" \ + "$BUILD_DIR/trace_write_preload_aiv.o" + check_kernel_elf + + echo "[BUILD] host runner" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror \ + -I"$ASCEND_HOME_PATH/include" \ + "$SCRIPT_DIR/trace_write_preload_host.cpp" \ + -L"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -Wl,-rpath,"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -lascendcl \ + -o "$HOST" + echo "[BUILD] complete: $BUILD_DIR" +} + +run_probe() { + if [[ ! -x "$HOST" || ! -f "$KERNEL" ]]; then + echo "Build artifacts are incomplete; run '$0 build' first." >&2 + exit 1 + fi + check_kernel_elf + echo "[RUN] device=${ATOMIC_PROBE_DEVICE:-${TASK_DEVICE:-0}}" + timeout "${TRACE_WRITE_PRELOAD_TIMEOUT:-240}" "$HOST" "$KERNEL" +} + +echo "=== CCEC Trace Write Preload Experiment ===" +echo "timestamp_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" +echo "git_head=$(git -C "$SCRIPT_DIR" rev-parse HEAD 2>/dev/null || echo unknown)" +echo "git_branch=$(git -C "$SCRIPT_DIR" branch --show-current 2>/dev/null || echo unknown)" +echo "ascend_home=$ASCEND_HOME_PATH" +echo "ccec=$($CCEC --version 2>&1 | head -n 1)" +echo "action=$ACTION" + +if [[ "$ACTION" == "all" || "$ACTION" == "build" ]]; then + build_probe +fi +if [[ "$ACTION" == "all" || "$ACTION" == "run" ]]; then + run_probe +fi diff --git a/tests/atomic_probe/ccec/run_vector_scalar_pmu.sh b/tests/atomic_probe/ccec/run_vector_scalar_pmu.sh new file mode 100755 index 0000000000..a27970c579 --- /dev/null +++ b/tests/atomic_probe/ccec/run_vector_scalar_pmu.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +# Build and run the single-AIV PA vector-loop scalar-busy PMU probe without +# touching the aggregate run_all.sh. The probe reuses the pa_scheduler ten-slot +# PMU owner, compiled with PA_BUILD_SUBMIT_PMU=0 so CNT5 remains MTE3 busy. +# +# Usage: +# ./run_vector_scalar_pmu.sh # build + run +# ./run_vector_scalar_pmu.sh build # build only +# ./run_vector_scalar_pmu.sh run # run existing artifacts +# +# The probe intentionally fixes rounds={16,128}, five measured pairs and a +# 180-second process timeout. ATOMIC_PROBE_DEVICE / TASK_DEVICE reuse the +# existing atomic-probe device-selection contract. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)" +PA_CCEC_DIR="$SCRIPT_DIR/../pa_scheduler/ccec" +BUILD_DIR="$SCRIPT_DIR/build/vector_scalar_pmu" +ACTION="${1:-all}" +if [[ "$ACTION" != "all" && "$ACTION" != "build" && "$ACTION" != "run" ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ $# -gt 1 ]]; then + echo "Usage: $0 [all|build|run]" >&2 + exit 1 +fi +if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then + echo "ASCEND_HOME_PATH is not set; source the local CANN environment first." >&2 + exit 1 +fi + +CCEC="$ASCEND_HOME_PATH/bin/ccec" +LD="$ASCEND_HOME_PATH/bin/ld.lld" +HCC="$ASCEND_HOME_PATH/tools/hcc/bin/aarch64-target-linux-gnu-g++" +CXX_BIN="${CXX:-g++}" +READELF_BIN="${READELF:-readelf}" +PTO_INCLUDE_ROOT="${PTO_ISA_ROOT:-$ASCEND_HOME_PATH/x86_64-linux}" + +if [[ ! -x "$CCEC" || ! -x "$LD" || ! -x "$HCC" ]]; then + echo "CCEC, ld.lld, or AICPU cross compiler is missing under $ASCEND_HOME_PATH" >&2 + exit 1 +fi +for header in pto/common/kernel_meta.hpp pto/common/constants.hpp pto/common/pto_tile.hpp pto/pto-inst.hpp; do + if [[ ! -f "$PTO_INCLUDE_ROOT/include/$header" ]]; then + echo "Local PTO header is missing: $PTO_INCLUDE_ROOT/include/$header" >&2 + exit 1 + fi +done +if ! command -v "$CXX_BIN" >/dev/null 2>&1 || ! command -v "$READELF_BIN" >/dev/null 2>&1; then + echo "Host C++ compiler and readelf are required." >&2 + exit 1 +fi + +KERNEL="$BUILD_DIR/vector_scalar_pmu_kernel.o" +HOST="$BUILD_DIR/vector_scalar_pmu_host" +OWNER="$BUILD_DIR/libvector_scalar_pmu_owner_aicpu.so" +DISPATCHER="$BUILD_DIR/libvector_scalar_pmu_owner_dispatcher.so" + +check_artifacts() { + for artifact in "$KERNEL" "$HOST" "$OWNER" "$DISPATCHER"; do + if [[ ! -s "$artifact" ]]; then + echo "Missing or empty artifact: $artifact" >&2 + exit 1 + fi + done + + local entry="vector_scalar_pmu_0_mix_aiv" + local symbols sections owner_header owner_symbols dispatcher_header dispatcher_symbols + symbols="$("$READELF_BIN" --symbols --wide "$KERNEL")" + sections="$("$READELF_BIN" --sections --wide "$KERNEL")" + if [[ "$symbols" != *" $entry"* || "$sections" != *".ascend.meta.$entry"* ]]; then + echo "Missing non-empty AIV entry or metadata for $entry" >&2 + exit 1 + fi + + owner_header="$("$READELF_BIN" --file-header "$OWNER")" + owner_symbols="$("$READELF_BIN" --dyn-syms --wide "$OWNER")" + dispatcher_header="$("$READELF_BIN" --file-header "$DISPATCHER")" + dispatcher_symbols="$("$READELF_BIN" --dyn-syms --wide "$DISPATCHER")" + if [[ "$owner_header" != *"Machine: AArch64"* || + "$dispatcher_header" != *"Machine: AArch64"* || + "$owner_symbols" != *" simpler_aicpu_exec"* ]]; then + echo "PMU owner/dispatcher architecture or owner entry check failed." >&2 + exit 1 + fi + for entry in StaticTileFwkBackendKernelServer DynTileFwkBackendKernelServerInit DynTileFwkBackendKernelServer; do + if [[ "$dispatcher_symbols" != *" $entry"* ]]; then + echo "Missing PMU bootstrap dispatcher entry: $entry" >&2 + exit 1 + fi + done + echo "[CHECK] AIV entry/meta and self-contained ten-slot PMU owner artifacts PASS" +} + +build_probe() { + mkdir -p "$BUILD_DIR" + local common_flags=( + -c -O3 -g -x cce -Wall -std=c++17 + --cce-aicore-only + --cce-aicore-arch=dav-c310-vec + -mllvm -cce-aicore-stack-size=0x8000 + -mllvm -cce-aicore-function-stack-size=0x8000 + -mllvm -cce-aicore-record-overflow=false + -mllvm -cce-aicore-addr-transform + -mllvm -cce-aicore-dcci-insert-for-scalar=false + -mllvm -cce-aicore-dcci-before-kernel-end=false + -DCCEC_SYNC_AIV_ONLY + -I"$SCRIPT_DIR" + -I"$PTO_INCLUDE_ROOT/include" + ) + + echo "[BUILD] single-AIV PA vector-loop PMU kernel" + "$CCEC" "${common_flags[@]}" \ + -o "$BUILD_DIR/vector_scalar_pmu_vec.o" \ + "$SCRIPT_DIR/vector_scalar_pmu.cpp" + "$LD" -m aicorelinux -Ttext=0 -static \ + -o "$KERNEL" "$BUILD_DIR/vector_scalar_pmu_vec.o" + + echo "[BUILD] self-contained AICPU ten-slot PMU dispatcher" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra -Werror \ + -Wl,--build-id \ + "$PA_CCEC_DIR/pmu_owner_dispatcher.cpp" \ + -o "$DISPATCHER" + + echo "[BUILD] self-contained AICPU ten-slot PMU owner (CNT5=MTE3)" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra -Werror \ + -Wl,--build-id \ + -DPA_BUILD_SUBMIT_PMU=0 \ + -I"$PA_CCEC_DIR" \ + "$PA_CCEC_DIR/pmu_owner_aicpu.cpp" \ + -o "$OWNER" + + echo "[BUILD] host runner" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror -Wno-deprecated-declarations \ + -I"$SCRIPT_DIR" \ + -I"$PA_CCEC_DIR" \ + -I"$REPO_ROOT/src/common" \ + -I"$REPO_ROOT/src/a5/platform/include" \ + -I"$ASCEND_HOME_PATH/include" \ + -I"$ASCEND_HOME_PATH/pkg_inc" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime/runtime" \ + "$SCRIPT_DIR/vector_scalar_pmu_host.cpp" \ + -L"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -Wl,-rpath,"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -lascendcl -lruntime -ldl -pthread \ + -o "$HOST" + check_artifacts + echo "[BUILD] complete: $BUILD_DIR" +} + +run_probe() { + check_artifacts + echo "[RUN] device=${ATOMIC_PROBE_DEVICE:-${TASK_DEVICE:-0}} rounds=16,128 repeats=5" + timeout 180 "$HOST" "$KERNEL" +} + +if [[ "$ACTION" == "all" || "$ACTION" == "build" ]]; then + build_probe +fi +if [[ "$ACTION" == "all" || "$ACTION" == "run" ]]; then + run_probe +fi diff --git a/tests/atomic_probe/ccec/shared_preload_model.cpp b/tests/atomic_probe/ccec/shared_preload_model.cpp new file mode 100644 index 0000000000..96d606fa16 --- /dev/null +++ b/tests/atomic_probe/ccec/shared_preload_model.cpp @@ -0,0 +1,568 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the license. + * ----------------------------------------------------------------------------------------------------------- + */ + +// Isolated physical models for the current PA shared path: +// * writer-history 40 B destination footprint (not instruction-exact); +// * one/three shared TensorDesc publications: 128 B / 384 B; +// * shared TensorDesc consumption: invalidate, then ordinary GM-to-GM copy; +// * current-PC ICache preload at a same-function target versus at a distant +// caller. The build script verifies the final linked layout. +// +// Preload is never used as a publication, coherency, or ordering primitive. +// Every DCCI/DSB required by the modeled path remains present. + +#include "../shared_preload_model_shared.h" +#include "ccec_utils.h" + +#if defined(SHARED_PRELOAD_MODEL_BUILD_AIC) +PTO_SYNCALL_MIX_AIC_KERNEL_META(shared_preload_model_0_mix_aic, 1, 2); +#elif defined(SHARED_PRELOAD_MODEL_BUILD_AIV) +PTO_SYNCALL_MIX_AIC_KERNEL_META(shared_preload_model_0_mix_aiv, 1, 2); +#else +#error "Compile with SHARED_PRELOAD_MODEL_BUILD_AIC or SHARED_PRELOAD_MODEL_BUILD_AIV" +#endif + +namespace { + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadOrderedSysCount() { + uint64_t value = 0; + asm volatile("MOV %0, SYS_CNT\n" : "=&l"(value) : : "memory"); + return value; +} + +// Tied operands make the value-producing work stay between the two SYS_CNT +// boundaries. A plain memory-clobber bracket is insufficient for a pure +// scalar helper because the compiler may legally hoist that helper. +__aicore__ __attribute__((always_inline)) inline uint64_t CycleBeforeValue( + uint64_t &value +) { + uint64_t cycle = 0; + asm volatile( + "MOV %1, SYS_CNT\n" + "MOV %0, %0\n" + : "+l"(value), "=&l"(cycle) + : + : "memory" + ); + return cycle; +} + +__aicore__ __attribute__((always_inline)) inline uint64_t CycleAfterValue( + uint64_t &value +) { + uint64_t cycle = 0; + asm volatile( + "MOV %0, %0\n" + "MOV %1, SYS_CNT\n" + : "+l"(value), "=&l"(cycle) + : + : "memory" + ); + return cycle; +} + +__aicore__ __attribute__((always_inline)) inline uint64_t OpaqueIdentity( + uint64_t value +) { + asm volatile("MOV %0, %0\n" : "+l"(value) : : "memory"); + return value; +} + +__aicore__ __attribute__((always_inline)) inline void InvalidateLines( + __gm__ uint8_t *base, uint32_t lines +) { + for (uint32_t line = 0; line < lines; ++line) { + dcci( + base + static_cast(line) * + shared_preload_model::kCacheLineBytes, + SINGLE_CACHE_LINE + ); + } + dsb(DSB_ALL); + asm volatile("" ::: "memory"); +} + +__aicore__ __attribute__((always_inline)) inline void FlushLines( + __gm__ uint8_t *base, uint32_t lines +) { + asm volatile("" ::: "memory"); + for (uint32_t line = 0; line < lines; ++line) { + dcci( + base + static_cast(line) * + shared_preload_model::kCacheLineBytes, + SINGLE_CACHE_LINE, CACHELINE_OUT + ); + } + dsb(DSB_ALL); + asm volatile("" ::: "memory"); +} + +__aicore__ __attribute__((always_inline)) inline void PreloadLines( + __gm__ uint8_t *base, uint32_t lines +) { + __gm__ uint64_t *preload_base = + reinterpret_cast<__gm__ uint64_t *>(base); + for (uint32_t line = 0; line < lines; ++line) { + dc_preload( + preload_base, + static_cast( + static_cast(line) * + shared_preload_model::kCacheLineBytes + ) + ); + } +} + +__aicore__ __attribute__((noinline)) void CopyBytes( + volatile __gm__ uint8_t *destination, + const volatile __gm__ uint8_t *source, uint32_t bytes +) { + for (uint32_t byte = 0; byte < bytes; ++byte) { + destination[byte] = source[byte]; + } +} + +__aicore__ __attribute__((noinline)) uint64_t ChecksumBytes( + const volatile __gm__ uint8_t *source, uint32_t bytes +) { + uint64_t checksum = 0xcbf29ce484222325ULL; + for (uint32_t byte = 0; byte < bytes; ++byte) { + checksum ^= static_cast(source[byte]); + checksum *= 0x100000001b3ULL; + } + return checksum; +} + +__aicore__ __attribute__((noinline)) void WarmDestination( + volatile __gm__ uint8_t *destination, uint32_t bytes +) { + for (uint32_t byte = 0; byte < bytes; ++byte) { + destination[byte] = + static_cast(0xa5U ^ byte ^ (byte >> 2U)); + } + asm volatile("" ::: "memory"); +} + +__aicore__ __attribute__((noinline, used)) uint64_t +SharedPreloadModelGap(uint64_t seed, uint32_t gap_rounds) { + return cache_preload::GapOracle(seed, gap_rounds); +} + +__aicore__ __attribute__((always_inline)) inline void PublishResult( + __gm__ shared_preload_model::ProbeResult *result, + uint64_t setup_ticks, uint64_t issue_ticks, uint64_t gap_ticks, + uint64_t access_ticks, uint64_t publish_ticks, uint64_t total_ticks, + uint64_t preparation_checksum, uint64_t result_checksum, + uint64_t icache_immediate_status, uint64_t icache_final_status, + uint32_t worker, uint32_t experiment, uint32_t mode, + uint32_t active_bytes, uint32_t gap_rounds, uint32_t sample_id +) { + st_dev_b64(&result->setup_ticks, setup_ticks); + st_dev_b64(&result->issue_ticks, issue_ticks); + st_dev_b64(&result->gap_ticks, gap_ticks); + st_dev_b64(&result->access_ticks, access_ticks); + st_dev_b64(&result->publish_ticks, publish_ticks); + st_dev_b64(&result->total_ticks, total_ticks); + st_dev_b64(&result->preparation_checksum, preparation_checksum); + st_dev_b64(&result->result_checksum, result_checksum); + st_dev_b64( + &result->icache_immediate_status, icache_immediate_status + ); + st_dev_b64(&result->icache_final_status, icache_final_status); + st_dev_b32(&result->worker_id, worker); + st_dev_b32(&result->experiment, experiment); + st_dev_b32(&result->mode, mode); + st_dev_b32(&result->active_bytes, active_bytes); + st_dev_b32(&result->gap_rounds, gap_rounds); + st_dev_b32(&result->sample_id, sample_id); + st_dev_b32(&result->status, shared_preload_model::kStatusComplete); +} + +__aicore__ inline void RunDCache( + __gm__ shared_preload_model::WorkerData *worker_data, + __gm__ shared_preload_model::ProbeResult *result, + const shared_preload_model::ProbeControl &control, uint32_t worker +) { + const uint32_t bytes = control.active_bytes; + const uint32_t lines = + shared_preload_model::CacheLinesForBytes(bytes); + volatile __gm__ uint8_t *source = worker_data->source; + volatile __gm__ uint8_t *destination = worker_data->destination; + const bool use_preload = + control.mode == + static_cast( + shared_preload_model::Mode::DCachePreload + ); + const bool publish = + control.experiment == + static_cast( + shared_preload_model::Experiment::Publish + ); + + // The source TensorDesc/history payload is already materialized in the PA + // path. Warm it before the measured writer model. The consume model then + // performs its mandatory source invalidation inside the measured window. + const uint64_t preparation_checksum = + ChecksumBytes(source, bytes); + if (publish) { + InvalidateLines( + reinterpret_cast<__gm__ uint8_t *>(worker_data->destination), + lines + ); + } else { + WarmDestination(destination, bytes); + } + + const uint64_t total_begin = ReadOrderedSysCount(); + uint64_t setup_ticks = 0; + if (!publish) { + const uint64_t setup_begin = ReadOrderedSysCount(); + InvalidateLines( + reinterpret_cast<__gm__ uint8_t *>(worker_data->source), + lines + ); + const uint64_t setup_end = ReadOrderedSysCount(); + setup_ticks = setup_end - setup_begin; + } + + const uint64_t issue_begin = ReadOrderedSysCount(); + if (use_preload) { + PreloadLines( + publish + ? reinterpret_cast<__gm__ uint8_t *>( + worker_data->destination + ) + : reinterpret_cast<__gm__ uint8_t *>( + worker_data->source + ), + lines + ); + } + const uint64_t issue_end = ReadOrderedSysCount(); + + uint64_t gap_input = + control.seed ^ static_cast(worker); + const uint64_t gap_begin = CycleBeforeValue(gap_input); + uint64_t gap_checksum = SharedPreloadModelGap( + gap_input, control.gap_rounds + ); + const uint64_t gap_end = CycleAfterValue(gap_checksum); + + // The zero address delta makes the copy addresses data-dependent on the + // noinline independent gap without changing the runtime address. + const uint64_t opaque_gap = OpaqueIdentity(gap_checksum); + const uint64_t address_delta = opaque_gap - gap_checksum; + volatile __gm__ uint8_t *dependent_destination = + reinterpret_cast( + reinterpret_cast(destination) + address_delta + ); + const volatile __gm__ uint8_t *dependent_source = + reinterpret_cast( + reinterpret_cast(source) + address_delta + ); + + const uint64_t access_begin = ReadOrderedSysCount(); + CopyBytes( + dependent_destination, dependent_source, bytes + ); + const uint64_t access_end = ReadOrderedSysCount(); + + uint64_t total_end = access_end; + uint64_t publish_ticks = 0; + if (publish) { + const uint64_t publish_begin = ReadOrderedSysCount(); + FlushLines( + reinterpret_cast<__gm__ uint8_t *>( + worker_data->destination + ), + lines + ); + total_end = ReadOrderedSysCount(); + publish_ticks = total_end - publish_begin; + } else { + // Result validation needs the copied payload in GM. This cleanup is + // deliberately after total_end and is not part of the consume model. + FlushLines( + reinterpret_cast<__gm__ uint8_t *>( + worker_data->destination + ), + lines + ); + } + + const uint64_t actual_checksum = + ChecksumBytes(destination, bytes); + const uint64_t rotated_actual_checksum = + (actual_checksum << 1U) | (actual_checksum >> 63U); + PublishResult( + result, setup_ticks, issue_end - issue_begin, + gap_end - gap_begin, access_end - access_begin, publish_ticks, + total_end - total_begin, preparation_checksum, + rotated_actual_checksum ^ gap_checksum, + 0, 0, worker, control.experiment, control.mode, bytes, + control.gap_rounds, control.sample_id + ); +} + +#if defined(SHARED_PRELOAD_MODEL_BUILD_AIV) + +#define SHARED_PRELOAD_NOPS_1() asm volatile("nop"); +#define SHARED_PRELOAD_NOPS_2() SHARED_PRELOAD_NOPS_1() SHARED_PRELOAD_NOPS_1() +#define SHARED_PRELOAD_NOPS_4() SHARED_PRELOAD_NOPS_2() SHARED_PRELOAD_NOPS_2() +#define SHARED_PRELOAD_NOPS_8() SHARED_PRELOAD_NOPS_4() SHARED_PRELOAD_NOPS_4() +#define SHARED_PRELOAD_NOPS_16() SHARED_PRELOAD_NOPS_8() SHARED_PRELOAD_NOPS_8() +#define SHARED_PRELOAD_NOPS_32() SHARED_PRELOAD_NOPS_16() SHARED_PRELOAD_NOPS_16() +#define SHARED_PRELOAD_NOPS_64() SHARED_PRELOAD_NOPS_32() SHARED_PRELOAD_NOPS_32() +#define SHARED_PRELOAD_NOPS_128() SHARED_PRELOAD_NOPS_64() SHARED_PRELOAD_NOPS_64() +#define SHARED_PRELOAD_NOPS_256() SHARED_PRELOAD_NOPS_128() SHARED_PRELOAD_NOPS_128() +#define SHARED_PRELOAD_NOPS_512() SHARED_PRELOAD_NOPS_256() SHARED_PRELOAD_NOPS_256() +#define SHARED_PRELOAD_NOPS_1024() SHARED_PRELOAD_NOPS_512() SHARED_PRELOAD_NOPS_512() +#define SHARED_PRELOAD_NOPS_2048() SHARED_PRELOAD_NOPS_1024() SHARED_PRELOAD_NOPS_1024() +#define SHARED_PRELOAD_NOPS_4096() SHARED_PRELOAD_NOPS_2048() SHARED_PRELOAD_NOPS_2048() +#define SHARED_PRELOAD_NOPS_8192() SHARED_PRELOAD_NOPS_4096() SHARED_PRELOAD_NOPS_4096() + +extern "C" __aicore__ __attribute__((noinline, used, aligned(128))) +uint64_t shared_preload_model_icache_target( + uint32_t mode, uint64_t seed, uint32_t gap_rounds, + uint64_t wrapper_begin, uint64_t caller_issue_ticks, + uint64_t caller_immediate_status, uint64_t preparation_checksum, + uint32_t worker, uint32_t sample_id, + __gm__ shared_preload_model::ProbeResult *result +) { + uint64_t issue_ticks = caller_issue_ticks; + uint64_t immediate_status = caller_immediate_status; + const bool target_preload = + mode == + static_cast( + shared_preload_model::Mode::ICacheTargetPreload + ); + if (target_preload) { + const uint64_t issue_begin = ReadOrderedSysCount(); + icache_preload( + static_cast( + shared_preload_model::kICachePreloadUnits + ) + ); + const uint64_t issue_end = ReadOrderedSysCount(); + issue_ticks = issue_end - issue_begin; + immediate_status = + static_cast(get_icache_prl_st()); + } + + uint64_t gap_input = seed ^ static_cast(worker); + const uint64_t gap_begin = CycleBeforeValue(gap_input); + uint64_t gap_checksum = + SharedPreloadModelGap(gap_input, gap_rounds); + const uint64_t gap_end = CycleAfterValue(gap_checksum); + + uint64_t final_status = 0; + if (mode != static_cast( + shared_preload_model::Mode::ICacheBaseline + )) { + final_status = static_cast(get_icache_prl_st()); + } + + uint64_t value = seed ^ gap_checksum; + const uint64_t access_begin = ReadOrderedSysCount(); + SHARED_PRELOAD_NOPS_1024() + value = cache_preload::ICacheTargetOracle(value); + const uint64_t access_end = ReadOrderedSysCount(); + + PublishResult( + result, 0, issue_ticks, gap_end - gap_begin, + access_end - access_begin, 0, access_end - wrapper_begin, + preparation_checksum, value, immediate_status, final_status, + worker, + static_cast( + shared_preload_model::Experiment::ICachePlacement + ), + mode, 0, gap_rounds, sample_id + ); + return value; +} + +// The evictor exceeds the documented 16 KiB AIV ICache; the final ELF check +// enforces the 32 KiB minimum chosen by this cold-state model. +extern "C" __aicore__ __attribute__((noinline, used, aligned(128))) +uint64_t shared_preload_model_icache_evictor(uint64_t seed) { + SHARED_PRELOAD_NOPS_8192() + return cache_preload::ICacheEvictorOracle(seed); +} + +// The linker check requires the target to lie outside the caller's forward +// current-PC preload window. Thus caller-preload measures a real call-boundary +// mismatch, while target-preload uses the same physical target NOP region as +// baseline. +extern "C" __aicore__ __attribute__((noinline, used, aligned(128))) +void shared_preload_model_icache_caller( + uint32_t mode, uint64_t seed, uint32_t gap_rounds, + uint64_t preparation_checksum, uint32_t worker, uint32_t sample_id, + __gm__ shared_preload_model::ProbeResult *result +) { + const uint64_t wrapper_begin = ReadOrderedSysCount(); + uint64_t caller_issue_ticks = 0; + uint64_t caller_immediate_status = 0; + if (mode == + static_cast( + shared_preload_model::Mode::ICacheCallerPreload + )) { + const uint64_t issue_begin = ReadOrderedSysCount(); + icache_preload( + static_cast( + shared_preload_model::kICachePreloadUnits + ) + ); + const uint64_t issue_end = ReadOrderedSysCount(); + caller_issue_ticks = issue_end - issue_begin; + caller_immediate_status = + static_cast(get_icache_prl_st()); + } + (void)shared_preload_model_icache_target( + mode, seed, gap_rounds, wrapper_begin, caller_issue_ticks, + caller_immediate_status, preparation_checksum, worker, + sample_id, result + ); +} + +__aicore__ inline void RunICache( + __gm__ shared_preload_model::ProbeResult *result, + const shared_preload_model::ProbeControl &control, uint32_t worker +) { + const uint64_t preparation_checksum = + shared_preload_model_icache_evictor(control.seed); + shared_preload_model_icache_caller( + control.mode, control.seed, control.gap_rounds, + preparation_checksum, worker, control.sample_id, result + ); +} + +#undef SHARED_PRELOAD_NOPS_8192 +#undef SHARED_PRELOAD_NOPS_4096 +#undef SHARED_PRELOAD_NOPS_2048 +#undef SHARED_PRELOAD_NOPS_1024 +#undef SHARED_PRELOAD_NOPS_512 +#undef SHARED_PRELOAD_NOPS_256 +#undef SHARED_PRELOAD_NOPS_128 +#undef SHARED_PRELOAD_NOPS_64 +#undef SHARED_PRELOAD_NOPS_32 +#undef SHARED_PRELOAD_NOPS_16 +#undef SHARED_PRELOAD_NOPS_8 +#undef SHARED_PRELOAD_NOPS_4 +#undef SHARED_PRELOAD_NOPS_2 +#undef SHARED_PRELOAD_NOPS_1 + +#endif // SHARED_PRELOAD_MODEL_BUILD_AIV + +__aicore__ inline void RunParticipant( + __gm__ shared_preload_model::ProbeControl *control_pointer, + __gm__ shared_preload_model::ProbeResult *results, + __gm__ shared_preload_model::WorkerData *data, uint32_t worker +) { + dcci(control_pointer, SINGLE_CACHE_LINE); + dsb(DSB_ALL); + asm volatile("" ::: "memory"); + + shared_preload_model::ProbeControl control{}; + control.magic = control_pointer->magic; + control.experiment = control_pointer->experiment; + control.mode = control_pointer->mode; + control.active_bytes = control_pointer->active_bytes; + control.gap_rounds = control_pointer->gap_rounds; + control.sample_id = control_pointer->sample_id; + control.first_worker = control_pointer->first_worker; + control.worker_count = control_pointer->worker_count; + control.seed = control_pointer->seed; + + if (control.magic != shared_preload_model::kControlMagic || + worker < control.first_worker || + worker >= control.first_worker + control.worker_count) { + return; + } + __gm__ shared_preload_model::ProbeResult *result = + &results[worker]; + + if (control.experiment == + static_cast( + shared_preload_model::Experiment::ICachePlacement + )) { +#if defined(SHARED_PRELOAD_MODEL_BUILD_AIV) + if (control.active_bytes == 0U && + control.mode >= static_cast( + shared_preload_model::Mode::ICacheBaseline + ) && + control.mode <= static_cast( + shared_preload_model::Mode::ICacheTargetPreload + )) { + RunICache(result, control, worker); + } +#endif + return; + } + + const bool valid_bytes = + control.active_bytes == + shared_preload_model::kWriterHistoryBytes || + control.active_bytes == + shared_preload_model::kOneDescriptorBytes || + control.active_bytes == + shared_preload_model::kThreeDescriptorBytes; + const bool valid_experiment = + control.experiment == + static_cast( + shared_preload_model::Experiment::Publish + ) || + control.experiment == + static_cast( + shared_preload_model::Experiment::Consume + ); + const bool valid_mode = + control.mode == + static_cast( + shared_preload_model::Mode::DCacheBaseline + ) || + control.mode == + static_cast( + shared_preload_model::Mode::DCachePreload + ); + if (!valid_bytes || !valid_experiment || !valid_mode) { + return; + } + RunDCache(&data[worker], result, control, worker); +} + +} // namespace + +#if defined(SHARED_PRELOAD_MODEL_BUILD_AIC) +extern "C" __global__ __aicore__ void shared_preload_model_0_mix_aic( + __gm__ shared_preload_model::ProbeControl *control, + __gm__ shared_preload_model::ProbeResult *results, + __gm__ shared_preload_model::WorkerData *data +) { + RunParticipant( + control, results, data, + static_cast(get_block_idx()) + ); +} +#elif defined(SHARED_PRELOAD_MODEL_BUILD_AIV) +extern "C" __global__ __aicore__ void shared_preload_model_0_mix_aiv( + __gm__ shared_preload_model::ProbeControl *control, + __gm__ shared_preload_model::ProbeResult *results, + __gm__ shared_preload_model::WorkerData *data +) { + const uint32_t vector_id = static_cast( + get_block_idx() * get_subblockdim() + get_subblockid() + ); + RunParticipant( + control, results, data, + shared_preload_model::kAicWorkers + vector_id + ); +} +#endif diff --git a/tests/atomic_probe/ccec/shared_preload_model_host.cpp b/tests/atomic_probe/ccec/shared_preload_model_host.cpp new file mode 100644 index 0000000000..b768149144 --- /dev/null +++ b/tests/atomic_probe/ccec/shared_preload_model_host.cpp @@ -0,0 +1,732 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the LICENSE file for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the license. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "../probe_host.h" +#include "../shared_preload_model_shared.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct KernelArgs { + uint64_t control; + uint64_t results; + uint64_t data; +}; + +struct DeviceResources { + aclrtStream stream = nullptr; + aclrtBinHandle binary = nullptr; + aclrtFuncHandle function = nullptr; + void *control = nullptr; + void *results = nullptr; + void *data = nullptr; +}; + +struct TimingAggregate { + std::vector setup; + std::vector issue; + std::vector gap; + std::vector access; + std::vector publish; + std::vector total; + std::vector critical_access; + std::vector critical_total; + uint64_t immediate_busy = 0; + uint64_t final_busy = 0; + uint64_t result_count = 0; +}; + +struct DCacheCase { + const char *name; + uint32_t bytes; +}; + +constexpr std::array kDCacheCases = {{ + {"writer-history-3-symbols", shared_preload_model::kWriterHistoryBytes}, + {"shared-output-1-desc", shared_preload_model::kOneDescriptorBytes}, + {"shared-output-3-desc", shared_preload_model::kThreeDescriptorBytes}, +}}; + +static_assert( + sizeof(KernelArgs) == 3U * sizeof(uint64_t), + "unexpected CCEC kernel argument ABI" +); + +bool Check(aclError error, const char *expression) { + return atomic_probe::CheckAcl( + error, expression, __FILE__, __LINE__ + ); +} + +std::vector ReadBinary(const std::string &path) { + std::ifstream stream(path, std::ios::binary | std::ios::ate); + if (!stream) return {}; + const std::streamsize size = stream.tellg(); + if (size <= 0) return {}; + stream.seekg(0); + std::vector data(static_cast(size)); + if (!stream.read(data.data(), size)) return {}; + return data; +} + +uint64_t Quantile(std::vector values, double quantile) { + if (values.empty()) return 0; + std::sort(values.begin(), values.end()); + const size_t index = static_cast( + std::llround( + quantile * static_cast(values.size() - 1U) + ) + ); + return values[index]; +} + +uint64_t Median(const std::vector &values) { + return Quantile(values, 0.5); +} + +bool IsZero(const shared_preload_model::ProbeResult &result) { + const auto *bytes = reinterpret_cast(&result); + for (size_t byte = 0; byte < sizeof(result); ++byte) { + if (bytes[byte] != 0U) return false; + } + return true; +} + +uint64_t RotateLeftOne(uint64_t value) { + return (value << 1U) | (value >> 63U); +} + +std::array +MakeInitialData() { + std::array + data{}; + for (uint32_t worker = 0; + worker < shared_preload_model::kWorkers; ++worker) { + for (uint32_t byte = 0; + byte < shared_preload_model::kMaxPayloadBytes; ++byte) { + data[worker].source[byte] = + shared_preload_model::PayloadByte(worker, byte); + } + } + return data; +} + +bool ValidateResult( + const shared_preload_model::ProbeResult &result, + const shared_preload_model::ProbeControl &control, uint32_t worker, + std::string *reason +) { + const bool active = + worker >= control.first_worker && + worker < control.first_worker + control.worker_count; + if (!active) { + if (!IsZero(result)) { + *reason = "inactive-worker-wrote-result-" + + std::to_string(worker); + return false; + } + return true; + } + if (result.status != shared_preload_model::kStatusComplete || + result.worker_id != worker || + result.experiment != control.experiment || + result.mode != control.mode || + result.active_bytes != control.active_bytes || + result.gap_rounds != control.gap_rounds || + result.sample_id != control.sample_id || + result.access_ticks == 0U || result.total_ticks == 0U) { + *reason = "result-header-or-timing-" + std::to_string(worker); + return false; + } + if (control.gap_rounds >= + shared_preload_model::kOverlapGapRounds && + result.gap_ticks < control.gap_rounds) { + *reason = "independent-gap-was-not-bracketed-" + + std::to_string(worker); + return false; + } + + const uint64_t gap_checksum = cache_preload::GapOracle( + control.seed ^ static_cast(worker), + control.gap_rounds + ); + if (control.experiment == + static_cast( + shared_preload_model::Experiment::ICachePlacement + )) { + const uint64_t expected_preparation = + cache_preload::ICacheEvictorOracle(control.seed); + const uint64_t expected_result = + cache_preload::ICacheTargetOracle( + control.seed ^ gap_checksum + ); + if (result.preparation_checksum != expected_preparation || + result.result_checksum != expected_result || + result.setup_ticks != 0U || + result.publish_ticks != 0U) { + *reason = "icache-oracle-" + std::to_string(worker); + return false; + } + if (control.mode == + static_cast( + shared_preload_model::Mode::ICacheBaseline + ) && + (result.icache_immediate_status != 0U || + result.icache_final_status != 0U)) { + *reason = "icache-baseline-status-" + + std::to_string(worker); + return false; + } + return true; + } + + const uint64_t expected_payload = + shared_preload_model::PayloadChecksum( + worker, control.active_bytes + ); + const uint64_t expected_result = + RotateLeftOne(expected_payload) ^ gap_checksum; + if (result.preparation_checksum != expected_payload || + result.result_checksum != expected_result || + result.icache_immediate_status != 0U || + result.icache_final_status != 0U) { + *reason = "dcache-oracle-" + std::to_string(worker); + return false; + } + if (control.experiment == + static_cast( + shared_preload_model::Experiment::Publish + ) && + (result.setup_ticks != 0U || result.publish_ticks == 0U)) { + *reason = "publish-phase-shape-" + std::to_string(worker); + return false; + } + if (control.experiment == + static_cast( + shared_preload_model::Experiment::Consume + ) && + (result.setup_ticks == 0U || result.publish_ticks != 0U)) { + *reason = "consume-phase-shape-" + std::to_string(worker); + return false; + } + return true; +} + +bool ValidatePayload( + const std::array &data, + const shared_preload_model::ProbeControl &control, + std::string *reason +) { + if (control.experiment == + static_cast( + shared_preload_model::Experiment::ICachePlacement + )) { + return true; + } + for (uint32_t worker = control.first_worker; + worker < control.first_worker + control.worker_count; ++worker) { + for (uint32_t byte = 0; byte < control.active_bytes; ++byte) { + const uint8_t expected = + shared_preload_model::PayloadByte(worker, byte); + if (data[worker].source[byte] != expected || + data[worker].destination[byte] != expected) { + *reason = "payload-" + std::to_string(worker) + "-" + + std::to_string(byte); + return false; + } + } + } + return true; +} + +void AppendTimings( + const std::array &results, + const shared_preload_model::ProbeControl &control, + TimingAggregate *aggregate +) { + uint64_t critical_access = 0; + uint64_t critical_total = 0; + for (uint32_t worker = control.first_worker; + worker < control.first_worker + control.worker_count; ++worker) { + const auto &result = results[worker]; + aggregate->setup.push_back(result.setup_ticks); + aggregate->issue.push_back(result.issue_ticks); + aggregate->gap.push_back(result.gap_ticks); + aggregate->access.push_back(result.access_ticks); + aggregate->publish.push_back(result.publish_ticks); + aggregate->total.push_back(result.total_ticks); + critical_access = + std::max(critical_access, result.access_ticks); + critical_total = + std::max(critical_total, result.total_ticks); + aggregate->immediate_busy += + result.icache_immediate_status != 0U ? 1U : 0U; + aggregate->final_busy += + result.icache_final_status != 0U ? 1U : 0U; + ++aggregate->result_count; + } + aggregate->critical_access.push_back(critical_access); + aggregate->critical_total.push_back(critical_total); +} + +bool RunOne( + const DeviceResources &device, + const std::array &initial_data, + const shared_preload_model::ProbeControl &control, + TimingAggregate *aggregate, std::string *reason +) { + std::array + zero_results{}; + if (!Check( + aclrtMemcpy( + device.control, sizeof(control), &control, + sizeof(control), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D shared preload control)" + ) || + !Check( + aclrtMemcpy( + device.results, sizeof(zero_results), + zero_results.data(), sizeof(zero_results), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D shared preload result reset)" + ) || + !Check( + aclrtMemcpy( + device.data, sizeof(initial_data), + initial_data.data(), sizeof(initial_data), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D shared preload data reset)" + )) { + *reason = "setup-copy"; + return false; + } + + KernelArgs args{ + reinterpret_cast(device.control), + reinterpret_cast(device.results), + reinterpret_cast(device.data), + }; + if (!Check( + aclrtLaunchKernelWithHostArgs( + device.function, shared_preload_model::kAicWorkers, + device.stream, nullptr, &args, sizeof(args), nullptr, 0U + ), + "aclrtLaunchKernelWithHostArgs(shared preload model)" + ) || + !Check( + aclrtSynchronizeStream(device.stream), + "aclrtSynchronizeStream(shared preload model)" + )) { + *reason = "launch-or-sync"; + return false; + } + + std::array + results{}; + std::array + data{}; + if (!Check( + aclrtMemcpy( + results.data(), sizeof(results), device.results, + sizeof(results), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H shared preload results)" + ) || + !Check( + aclrtMemcpy( + data.data(), sizeof(data), device.data, sizeof(data), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H shared preload data)" + )) { + *reason = "result-copy"; + return false; + } + for (uint32_t worker = 0; + worker < shared_preload_model::kWorkers; ++worker) { + if (!ValidateResult(results[worker], control, worker, reason)) { + return false; + } + } + if (!ValidatePayload(data, control, reason)) return false; + AppendTimings(results, control, aggregate); + return true; +} + +double DeltaPercent(uint64_t baseline, uint64_t candidate) { + if (baseline == 0U) return 0.0; + return 100.0 * + (static_cast(candidate) - + static_cast(baseline)) / + static_cast(baseline); +} + +void PrintMetric( + const char *name, const std::vector &baseline, + const std::vector &candidate +) { + const uint64_t base = Median(baseline); + const uint64_t preload = Median(candidate); + std::printf( + " %-22s %8llu -> %8llu %+8.3f%%\n", name, + static_cast(base), + static_cast(preload), + DeltaPercent(base, preload) + ); +} + +void PrintComparison( + const char *label, uint32_t bytes, uint32_t gap_rounds, + const TimingAggregate &baseline, + const TimingAggregate &candidate +) { + std::printf( + "[COMPARE] %-31s bytes=%3u lines=%u gap_rounds=%u " + "samples=%zu workers/sample=%llu\n", + label, bytes, + shared_preload_model::CacheLinesForBytes(bytes), gap_rounds, + baseline.critical_total.size(), + static_cast( + baseline.result_count / + std::max(baseline.critical_total.size(), 1U) + ) + ); + PrintMetric("core-median setup", baseline.setup, candidate.setup); + PrintMetric("core-median issue", baseline.issue, candidate.issue); + PrintMetric("core-median gap", baseline.gap, candidate.gap); + PrintMetric("core-median copy/work", baseline.access, candidate.access); + PrintMetric( + "core-median publish", baseline.publish, candidate.publish + ); + PrintMetric("core-median total", baseline.total, candidate.total); + PrintMetric( + "critical copy/work", baseline.critical_access, + candidate.critical_access + ); + PrintMetric( + "critical total", baseline.critical_total, + candidate.critical_total + ); +} + +bool RunDCachePair( + const DeviceResources &device, + const std::array &initial_data, + shared_preload_model::Experiment experiment, + const DCacheCase &test_case, uint32_t gap_rounds +) { + TimingAggregate aggregates[2]; + for (uint32_t sample = 0; + sample < shared_preload_model::kDCacheSamples; ++sample) { + const uint32_t first = sample & 1U; + for (uint32_t order = 0; order < 2U; ++order) { + const uint32_t policy = first ^ order; + shared_preload_model::ProbeControl control{}; + control.magic = shared_preload_model::kControlMagic; + control.experiment = static_cast(experiment); + control.mode = policy == 0U + ? static_cast( + shared_preload_model::Mode:: + DCacheBaseline + ) + : static_cast( + shared_preload_model::Mode:: + DCachePreload + ); + control.active_bytes = test_case.bytes; + control.gap_rounds = gap_rounds; + control.sample_id = sample; + control.first_worker = 0; + control.worker_count = shared_preload_model::kWorkers; + control.seed = + 0x1020304050607080ULL + + static_cast(sample) * 0x10001ULL + + static_cast(test_case.bytes) * 17ULL + + static_cast(control.experiment) * 0x100000ULL; + std::string reason; + if (!RunOne( + device, initial_data, control, + &aggregates[policy], &reason + )) { + std::fprintf( + stderr, + "[FAIL] dcache experiment=%u case=%s gap=%u " + "sample=%u mode=%u reason=%s\n", + control.experiment, test_case.name, gap_rounds, + sample, control.mode, reason.c_str() + ); + return false; + } + } + } + const char *experiment_name = + experiment == shared_preload_model::Experiment::Publish + ? "publish" + : "consume"; + const std::string label = + std::string(experiment_name) + "/" + test_case.name; + PrintComparison( + label.c_str(), test_case.bytes, gap_rounds, + aggregates[0], aggregates[1] + ); + return true; +} + +bool RunICacheMatrix( + const DeviceResources &device, + const std::array &initial_data +) { + constexpr std::array modes = {{ + shared_preload_model::Mode::ICacheBaseline, + shared_preload_model::Mode::ICacheCallerPreload, + shared_preload_model::Mode::ICacheTargetPreload, + }}; + std::array aggregates; + for (uint32_t sample = 0; + sample < shared_preload_model::kICacheSamples; ++sample) { + const uint32_t rotation = sample % modes.size(); + for (uint32_t order = 0; order < modes.size(); ++order) { + const uint32_t index = (rotation + order) % modes.size(); + shared_preload_model::ProbeControl control{}; + control.magic = shared_preload_model::kControlMagic; + control.experiment = static_cast( + shared_preload_model::Experiment::ICachePlacement + ); + control.mode = static_cast(modes[index]); + control.active_bytes = 0; + control.gap_rounds = + shared_preload_model::kOverlapGapRounds; + control.sample_id = sample; + control.first_worker = + shared_preload_model::kAicWorkers; + control.worker_count = + shared_preload_model::kAivWorkers; + control.seed = + 0xa5a5000011110000ULL + + static_cast(sample) * 0x10001ULL; + std::string reason; + if (!RunOne( + device, initial_data, control, + &aggregates[index], &reason + )) { + std::fprintf( + stderr, + "[FAIL] icache sample=%u mode=%u reason=%s\n", + sample, control.mode, reason.c_str() + ); + return false; + } + } + } + + std::printf( + "[ICACHE] AIV-only same target region; baseline/caller/target " + "samples=%u workers/sample=%u gap_rounds=%u\n", + shared_preload_model::kICacheSamples, + shared_preload_model::kAivWorkers, + shared_preload_model::kOverlapGapRounds + ); + PrintComparison( + "icache caller-current-PC", 0, + shared_preload_model::kOverlapGapRounds, + aggregates[0], aggregates[1] + ); + PrintComparison( + "icache target-current-PC", 0, + shared_preload_model::kOverlapGapRounds, + aggregates[0], aggregates[2] + ); + std::printf( + " status caller immediate/final busy=%llu/%llu of %llu; " + "target=%llu/%llu of %llu\n", + static_cast(aggregates[1].immediate_busy), + static_cast(aggregates[1].final_busy), + static_cast(aggregates[1].result_count), + static_cast(aggregates[2].immediate_busy), + static_cast(aggregates[2].final_busy), + static_cast(aggregates[2].result_count) + ); + return true; +} + +bool Cleanup(DeviceResources *device, int32_t device_id) { + bool ok = true; + if (device->data != nullptr) { + ok &= Check(aclrtFree(device->data), "aclrtFree(shared data)"); + device->data = nullptr; + } + if (device->results != nullptr) { + ok &= Check( + aclrtFree(device->results), "aclrtFree(shared results)" + ); + device->results = nullptr; + } + if (device->control != nullptr) { + ok &= Check( + aclrtFree(device->control), "aclrtFree(shared control)" + ); + device->control = nullptr; + } + if (device->binary != nullptr) { + ok &= Check( + aclrtBinaryUnLoad(device->binary), "aclrtBinaryUnLoad" + ); + device->binary = nullptr; + } + if (device->stream != nullptr) { + ok &= Check( + aclrtDestroyStream(device->stream), "aclrtDestroyStream" + ); + device->stream = nullptr; + } + ok &= Check(aclrtResetDevice(device_id), "aclrtResetDevice"); + ok &= Check(aclFinalize(), "aclFinalize"); + return ok; +} + +} // namespace + +int main(int argc, char **argv) { + const std::string kernel_path = + argc > 1 ? argv[1] : "./shared_preload_model_kernel.o"; + if (argc > 2) { + std::fprintf( + stderr, "Usage: %s [shared_preload_model_kernel.o]\n", + argv[0] + ); + return EXIT_FAILURE; + } + const std::vector kernel_data = ReadBinary(kernel_path); + if (kernel_data.empty()) { + std::fprintf( + stderr, "Cannot read kernel binary: %s\n", + kernel_path.c_str() + ); + return EXIT_FAILURE; + } + const int32_t device_id = atomic_probe::DeviceId(); + if (device_id < 0) return EXIT_FAILURE; + + DeviceResources device; + if (!Check(aclInit(nullptr), "aclInit") || + !Check(aclrtSetDevice(device_id), "aclrtSetDevice") || + !Check( + aclrtCreateStream(&device.stream), "aclrtCreateStream" + ) || + !Check( + atomic_probe::LoadAicoreBinaryFromData( + kernel_data.data(), kernel_data.size(), &device.binary + ), + "LoadAicoreBinaryFromData" + ) || + !Check( + aclrtBinaryGetFunctionByEntry( + device.binary, 0U, &device.function + ), + "aclrtBinaryGetFunctionByEntry" + ) || + !Check( + aclrtMalloc( + &device.control, + sizeof(shared_preload_model::ProbeControl), + ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(shared control)" + ) || + !Check( + aclrtMalloc( + &device.results, + shared_preload_model::kWorkers * + sizeof(shared_preload_model::ProbeResult), + ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(shared results)" + ) || + !Check( + aclrtMalloc( + &device.data, + shared_preload_model::kWorkers * + sizeof(shared_preload_model::WorkerData), + ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(shared data)" + )) { + Cleanup(&device, device_id); + return EXIT_FAILURE; + } + + std::printf( + "=== A5 CCEC PA-shared cache preload model ===\n" + "kernel=%s bytes=%zu topology=32_AIC+64_AIV " + "dcache_samples=%u icache_samples=%u\n" + "All timing values are raw SYS_CNT deltas. DCache publish keeps " + "DCCI(CACHELINE_OUT)+DSB; consume keeps invalidate+DSB.\n", + kernel_path.c_str(), kernel_data.size(), + shared_preload_model::kDCacheSamples, + shared_preload_model::kICacheSamples + ); + + const auto initial_data = MakeInitialData(); + bool semantic_ok = true; + constexpr std::array gaps = {{ + shared_preload_model::kShortGapRounds, + shared_preload_model::kOverlapGapRounds, + }}; + for (const auto experiment : { + shared_preload_model::Experiment::Publish, + shared_preload_model::Experiment::Consume, + }) { + for (const auto &test_case : kDCacheCases) { + for (const uint32_t gap : gaps) { + if (!semantic_ok) break; + semantic_ok = RunDCachePair( + device, initial_data, experiment, test_case, gap + ); + } + } + } + if (semantic_ok) { + semantic_ok = RunICacheMatrix(device, initial_data); + } + + atomic_probe::Result result; + result.Expect( + semantic_ok, + "shared publish/consume payload, topology, and ICache oracles" + ); + const bool cleanup_ok = Cleanup(&device, device_id); + result.Expect(cleanup_ok, "shared preload model cleanup"); + return result.ExitCode(); +} diff --git a/tests/atomic_probe/ccec/shared_tensor_map_visibility_probe.cpp b/tests/atomic_probe/ccec/shared_tensor_map_visibility_probe.cpp new file mode 100644 index 0000000000..91124f5cd4 --- /dev/null +++ b/tests/atomic_probe/ccec/shared_tensor_map_visibility_probe.cpp @@ -0,0 +1,500 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +// A5 production shared TensorMap 跨核可见性探针。 +// +// 两个 AIV 对每个 task 轮换 writer/reader。reader 先用普通 scalar load +// 预热旧 committed/reclaim/head/tail/seq/payload cache line，再通过独立诊断 +// cache line 告知 writer。writer 只调用 production publish helper；reader +// 只以 production committed_tasks 从 N 变为 N+1 作为交权信号，随后调用 +// production read/lookup helper 验证 tail、seq、payload 与 lookup。 +// +// 被测窗口不插入 FFTS SyncAll、额外 DSB、额外 DCCI 或自造发布 atomic。 +// ready/done 仅位于窗口外，负责阻止 writer 在 reader 完成本 task 取证前进入 +// 下一 task。所有轮询都有设备侧有限超时，首错只记录一次。 +#include "ccec_utils.h" +#include "inner_kernel.h" +#include "dist_engine/aicore/shared_tensor_map.h" +#include "shared_tensor_map_visibility_probe_shared.h" + +using namespace shared_tensor_map_visibility_probe; + +static_assert(PTO_FDWIC_SHARED_MAP == 1, "visibility probe must compile the production shared backend"); +static_assert(PTO_FDWIC_TENSORMAP_RING_CAP == kRingCapacity, "visibility probe requires CAP=128"); +static_assert(kMapBucketCapacity == kRingCapacity); +static_assert(kMapBuckets == 128); + +CCEC_PROBE_KERNEL_META(shared_tensor_map_visibility_probe); + +enum class WaitResult : uint32_t { + Reached = 0, + Aborted = 1, + Timeout = 2, + Overshoot = 3, +}; + +struct TaskPlan { + SharedTensorMapValue entries[2]; + uint32_t count; +}; + +__aicore__ inline SharedTensorMapValue MakeEntry(uint64_t buffer, int32_t task, uint32_t ordinal) { + const uint64_t lower = (static_cast(static_cast(task)) * 4ULL + ordinal) * 64ULL; + SharedTensorMapValue value{}; + value.buf_addr = buffer; + value.lo = lower; + value.hi = lower + 32; + value.producer = task; + value.reserved = 0; + return value; +} + +__aicore__ inline TaskPlan MakeTaskPlan(int32_t task) { + TaskPlan plan{}; + if (task == 0) { + plan.count = 0; + } else if (task == 1) { + plan.entries[0] = MakeEntry(kBufferA0, task, 0); + plan.entries[1] = MakeEntry(kBufferA1, task, 1); + plan.count = 2; + } else { + plan.entries[0] = MakeEntry(kBufferA0, task, 0); + plan.entries[1] = MakeEntry(kBufferB, task, 1); + plan.count = 2; + } + return plan; +} + +__aicore__ inline int64_t LoadControl(__gm__ volatile int64_t *address) { + return DistSharedTensorMapAicoreOps::Load(address); +} + +__aicore__ inline void StoreDiagnostic(__gm__ volatile int64_t *address, int64_t value) { + (void)atomic_exchange(*address, value); +} + +__aicore__ inline void RecordFirstError( + __gm__ ProbeControl &control, ErrorCode code, Phase phase, int32_t task, uint32_t block, int64_t actual, + int64_t expected, int64_t aux0, int64_t aux1, int64_t commit, int64_t head, int64_t tail, int64_t sequence, + const SharedTensorMapValue *snapshot +) { + if (DistSharedTensorMapAicoreOps::CompareExchange(&control.first_error_claim.value, 0, 1) != 0) { + return; + } + + StoreDiagnostic(&control.first_error.words[0], static_cast(code)); + StoreDiagnostic(&control.first_error.words[1], static_cast(task)); + StoreDiagnostic(&control.first_error.words[2], static_cast(block)); + StoreDiagnostic(&control.first_error.words[3], static_cast(phase)); + StoreDiagnostic(&control.first_error.words[4], actual); + StoreDiagnostic(&control.first_error.words[5], expected); + StoreDiagnostic(&control.first_error.words[6], aux0); + StoreDiagnostic(&control.first_error.words[7], aux1); + + StoreDiagnostic(&control.first_snapshot.words[0], commit); + StoreDiagnostic(&control.first_snapshot.words[1], head); + StoreDiagnostic(&control.first_snapshot.words[2], tail); + StoreDiagnostic(&control.first_snapshot.words[3], sequence); + StoreDiagnostic( + &control.first_snapshot.words[4], snapshot == nullptr ? 0 : static_cast(snapshot->buf_addr) + ); + StoreDiagnostic(&control.first_snapshot.words[5], snapshot == nullptr ? 0 : static_cast(snapshot->lo)); + StoreDiagnostic(&control.first_snapshot.words[6], snapshot == nullptr ? 0 : static_cast(snapshot->hi)); + const int64_t producer_reserved = snapshot == nullptr ? + 0 : + (static_cast(static_cast(snapshot->producer)) << 32) | + static_cast(snapshot->reserved); + StoreDiagnostic(&control.first_snapshot.words[7], producer_reserved); + + (void)DistSharedTensorMapAicoreOps::CompareExchange(&control.abort_code.value, 0, static_cast(code)); +} + +__aicore__ inline WaitResult +WaitForExact(__gm__ ProbeControl &control, __gm__ volatile int64_t *address, int64_t target, int64_t &actual) { + const uint64_t begin = static_cast(get_sys_cnt()); + do { + actual = LoadControl(address); + if (actual == target) { + return WaitResult::Reached; + } + if (actual > target) { + return WaitResult::Overshoot; + } + if (LoadControl(&control.abort_code.value) != 0) { + return WaitResult::Aborted; + } + } while (static_cast(get_sys_cnt()) - begin < kWaitTimeoutCycles); + + actual = LoadControl(address); + if (actual == target) { + return WaitResult::Reached; + } + return actual > target ? WaitResult::Overshoot : WaitResult::Timeout; +} + +__aicore__ inline bool ValuesEqual(const SharedTensorMapValue &left, const SharedTensorMapValue &right) { + return left.buf_addr == right.buf_addr && left.lo == right.lo && left.hi == right.hi && + left.producer == right.producer && left.reserved == right.reserved; +} + +__aicore__ inline uint64_t +PreheatTask(__gm__ SharedTensorMapState &map, const TaskPlan &plan, uint64_t tail_a, uint64_t tail_b, uint64_t sink) { + // 这些必须是普通 scalar load，不能改成 ld_dev/atomic；目的正是让 reader + // 在 writer 发布前持有旧 cache line。 + volatile __gm__ int64_t *commit = &map.committed_tasks.v; + volatile __gm__ int64_t *reclaim = &map.reclaim_upto.v; + volatile __gm__ int64_t *head_a = &map.buckets[kExpectedBucketA].head.v; + volatile __gm__ int64_t *tail_a_ptr = &map.buckets[kExpectedBucketA].tail.v; + volatile __gm__ int64_t *head_b = &map.buckets[kExpectedBucketB].head.v; + volatile __gm__ int64_t *tail_b_ptr = &map.buckets[kExpectedBucketB].tail.v; + sink += static_cast(*commit) + static_cast(*reclaim); + sink += static_cast(*head_a) + static_cast(*tail_a_ptr); + sink += static_cast(*head_b) + static_cast(*tail_b_ptr); + + for (uint32_t index = 0; index < plan.count; ++index) { + const uint32_t bucket = dist_tensor_map_hash(plan.entries[index].buf_addr); + uint64_t cursor = bucket == kExpectedBucketA ? tail_a : tail_b; + if (index != 0 && bucket == dist_tensor_map_hash(plan.entries[0].buf_addr)) { + ++cursor; + } + __gm__ SharedTensorMapSlot &slot = map.slots[dist_shared_tensor_map_slot_index(bucket, cursor)]; + volatile __gm__ int64_t *sequence = &slot.sequence.v; + volatile __gm__ uint64_t *payload = reinterpret_cast(&slot.payload); + sink += static_cast(*sequence); + for (uint32_t word = 0; word < sizeof(slot.payload) / sizeof(uint64_t); ++word) { + sink += payload[word] * (word + 1U); + } + } + return sink; +} + +__aicore__ inline bool CheckErrorFree( + __gm__ ProbeControl &control, ErrorCode code, Phase phase, int32_t task, uint32_t block, int64_t actual, + int64_t expected, int64_t aux0, int64_t aux1, int64_t commit, int64_t head, int64_t tail, int64_t sequence, + const SharedTensorMapValue *snapshot +) { + if (actual == expected) { + return true; + } + RecordFirstError( + control, code, phase, task, block, actual, expected, aux0, aux1, commit, head, tail, sequence, snapshot + ); + return false; +} + +__aicore__ inline bool CheckPublishedEntry( + __gm__ SharedTensorMapState &map, __gm__ ProbeControl &control, const SharedTensorMapValue &expected, + uint64_t cursor, int32_t task, uint32_t block, int64_t expected_head, int64_t expected_tail, uint64_t &checks +) { + const uint32_t bucket = dist_tensor_map_hash(expected.buf_addr); + const int64_t commit = DistSharedTensorMapAicoreOps::Load(&map.committed_tasks.v); + const int64_t head = DistSharedTensorMapAicoreOps::Load(&map.buckets[bucket].head.v); + const int64_t tail = DistSharedTensorMapAicoreOps::Load(&map.buckets[bucket].tail.v); + ++checks; + if (!CheckErrorFree( + control, ErrorCode::HeadMismatch, Phase::CheckControl, task, block, head, expected_head, bucket, + static_cast(cursor), commit, head, tail, 0, nullptr + )) { + return false; + } + ++checks; + if (!CheckErrorFree( + control, ErrorCode::TailMismatch, Phase::CheckControl, task, block, tail, expected_tail, bucket, + static_cast(cursor), commit, head, tail, 0, nullptr + )) { + return false; + } + + __gm__ SharedTensorMapSlot &slot = map.slots[dist_shared_tensor_map_slot_index(bucket, cursor)]; + const int64_t sequence = DistSharedTensorMapAicoreOps::Load(&slot.sequence.v); + ++checks; + if (!CheckErrorFree( + control, ErrorCode::SequenceMismatch, Phase::CheckSlot, task, block, sequence, static_cast(cursor), + bucket, static_cast(cursor), commit, head, tail, sequence, nullptr + )) { + return false; + } + + SharedTensorMapValue snapshot{}; + const bool read_ok = + dist_shared_tensor_map_read_slot_impl(map, bucket, cursor, snapshot); + ++checks; + if (!read_ok) { + RecordFirstError( + control, ErrorCode::ReadFailed, Phase::CheckSlot, task, block, 0, 1, bucket, static_cast(cursor), + commit, head, tail, sequence, &snapshot + ); + return false; + } + ++checks; + if (!ValuesEqual(snapshot, expected)) { + RecordFirstError( + control, ErrorCode::PayloadMismatch, Phase::CheckSlot, task, block, snapshot.producer, expected.producer, + bucket, static_cast(cursor), commit, head, tail, sequence, &snapshot + ); + return false; + } + + bool protocol_ok = false; + const int32_t producer = dist_shared_tensor_map_lookup_region(map, expected, task + 1, 1, protocol_ok); + ++checks; + if (!protocol_ok) { + RecordFirstError( + control, ErrorCode::LookupProtocol, Phase::CheckLookup, task, block, producer, expected.producer, bucket, + static_cast(cursor), commit, head, tail, sequence, &snapshot + ); + return false; + } + ++checks; + if (producer != expected.producer) { + RecordFirstError( + control, ErrorCode::LookupMismatch, Phase::CheckLookup, task, block, producer, expected.producer, bucket, + static_cast(cursor), commit, head, tail, sequence, &snapshot + ); + return false; + } + return true; +} + +__aicore__ inline void PublishParticipant( + __gm__ ProbeControl &control, uint32_t block, uint32_t completed_tasks, uint32_t writer_tasks, + uint32_t reader_tasks, uint64_t checks, uint64_t preheat_sink +) { + __gm__ ParticipantLine &line = control.participants[block]; + StoreDiagnostic(&line.words[0], kParticipantMagic); + StoreDiagnostic(&line.words[1], static_cast(block)); + const uint64_t topology = + (static_cast(static_cast(get_coreid())) << 32) | static_cast(get_subblockid()); + StoreDiagnostic(&line.words[2], static_cast(topology)); + StoreDiagnostic(&line.words[3], static_cast(completed_tasks)); + const uint64_t role_counts = (static_cast(writer_tasks) << 32) | reader_tasks; + StoreDiagnostic(&line.words[4], static_cast(role_counts)); + StoreDiagnostic(&line.words[5], static_cast(checks)); + StoreDiagnostic(&line.words[6], static_cast(preheat_sink)); + StoreDiagnostic(&line.words[7], kParticipantFinish); + (void)atomic_fetch_add(control.finish_count.value, int64_t{1}); +} + +extern "C" __global__ __aicore__ void KERNEL_ENTRY(shared_tensor_map_visibility_probe)( + __gm__ SharedTensorMapState *map_pointer, __gm__ ProbeControl *control_pointer, uint32_t num_blocks, + uint32_t launch_id +) { + (void)launch_id; + __gm__ SharedTensorMapState &map = *map_pointer; + __gm__ ProbeControl &control = *control_pointer; + const uint32_t block = static_cast(get_block_idx()); + uint32_t completed_tasks = 0; + uint32_t writer_tasks = 0; + uint32_t reader_tasks = 0; + uint64_t checks = 0; + uint64_t preheat_sink = 0; + + const uint32_t bucket_a0 = dist_tensor_map_hash(kBufferA0); + const uint32_t bucket_a1 = dist_tensor_map_hash(kBufferA1); + const uint32_t bucket_b = dist_tensor_map_hash(kBufferB); + if (num_blocks != kAivBlocks || get_block_num() != kAivBlocks || block >= kAivBlocks) { + RecordFirstError( + control, ErrorCode::InvalidTopology, Phase::Setup, -1, block, get_block_num(), kAivBlocks, num_blocks, 0, 0, + 0, 0, 0, nullptr + ); + PublishParticipant( + control, block < kAivBlocks ? block : 0, completed_tasks, writer_tasks, reader_tasks, checks, preheat_sink + ); + return; + } + if (bucket_a0 != kExpectedBucketA || bucket_a1 != kExpectedBucketA || bucket_b != kExpectedBucketB || + bucket_a0 == bucket_b) { + RecordFirstError( + control, ErrorCode::HashConfiguration, Phase::Setup, -1, block, bucket_a0, kExpectedBucketA, bucket_a1, + bucket_b, 0, 0, 0, 0, nullptr + ); + PublishParticipant(control, block, completed_tasks, writer_tasks, reader_tasks, checks, preheat_sink); + return; + } + + uint64_t tail_a = 0; + uint64_t tail_b = 0; + for (int32_t task = 0; task < static_cast(kTotalTasks); ++task) { + if (LoadControl(&control.abort_code.value) != 0) { + break; + } + const TaskPlan plan = MakeTaskPlan(task); + const uint32_t writer = static_cast(task) & 1U; + const bool is_writer = block == writer; + const int64_t epoch = static_cast(task) + 1; + + if (!is_writer) { + preheat_sink = PreheatTask(map, plan, tail_a, tail_b, preheat_sink); + const int64_t old_ready = + DistSharedTensorMapAicoreOps::CompareExchange(&control.reader_ready.value, task, epoch); + if (!CheckErrorFree( + control, ErrorCode::ReadyOvershoot, Phase::WaitReady, task, block, old_ready, task, 0, 0, + LoadControl(&map.committed_tasks.v), 0, 0, 0, nullptr + )) { + break; + } + } else { + int64_t actual = 0; + const WaitResult wait = WaitForExact(control, &control.reader_ready.value, epoch, actual); + if (wait != WaitResult::Reached) { + if (wait != WaitResult::Aborted) { + RecordFirstError( + control, wait == WaitResult::Timeout ? ErrorCode::ReadyTimeout : ErrorCode::ReadyOvershoot, + Phase::WaitReady, task, block, actual, epoch, 0, 0, LoadControl(&map.committed_tasks.v), 0, 0, + 0, nullptr + ); + } + break; + } + + const DistSharedTensorMapTaskPublishResult result = + dist_shared_tensor_map_publish_task(map, plan.count == 0 ? nullptr : plan.entries, plan.count, task, 0); + ++writer_tasks; + if (result != DistSharedTensorMapTaskPublishResult::Committed) { + RecordFirstError( + control, ErrorCode::PublishFailed, Phase::Publish, task, block, static_cast(result), + static_cast(DistSharedTensorMapTaskPublishResult::Committed), plan.count, 0, + LoadControl(&map.committed_tasks.v), 0, 0, 0, nullptr + ); + break; + } + } + + if (!is_writer) { + int64_t commit = 0; + const WaitResult wait = WaitForExact(control, &map.committed_tasks.v, epoch, commit); + if (wait != WaitResult::Reached) { + if (wait != WaitResult::Aborted) { + RecordFirstError( + control, wait == WaitResult::Timeout ? ErrorCode::CommitTimeout : ErrorCode::CommitOvershoot, + Phase::WaitCommit, task, block, commit, epoch, 0, 0, commit, 0, 0, 0, nullptr + ); + } + break; + } + ++reader_tasks; + ++checks; + if (!CheckErrorFree( + control, ErrorCode::CommitMismatch, Phase::WaitCommit, task, block, commit, epoch, 0, 0, commit, 0, + 0, 0, nullptr + )) { + break; + } + + const int64_t expected_reclaim = task == 0 ? -1 : task - 1; + const int64_t reclaim = DistSharedTensorMapAicoreOps::Load(&map.reclaim_upto.v); + ++checks; + if (!CheckErrorFree( + control, ErrorCode::ReclaimMismatch, Phase::CheckControl, task, block, reclaim, expected_reclaim, 0, + 0, commit, 0, 0, 0, nullptr + )) { + break; + } + + bool task_ok = true; + if (task == 0) { + const int64_t head_a = DistSharedTensorMapAicoreOps::Load(&map.buckets[kExpectedBucketA].head.v); + const int64_t observed_tail_a = + DistSharedTensorMapAicoreOps::Load(&map.buckets[kExpectedBucketA].tail.v); + const int64_t head_b = DistSharedTensorMapAicoreOps::Load(&map.buckets[kExpectedBucketB].head.v); + const int64_t observed_tail_b = + DistSharedTensorMapAicoreOps::Load(&map.buckets[kExpectedBucketB].tail.v); + checks += 4; + task_ok = CheckErrorFree( + control, ErrorCode::HeadMismatch, Phase::CheckControl, task, block, head_a, 0, + kExpectedBucketA, 0, commit, head_a, observed_tail_a, 0, nullptr + ) && + CheckErrorFree( + control, ErrorCode::TailMismatch, Phase::CheckControl, task, block, observed_tail_a, 0, + kExpectedBucketA, 0, commit, head_a, observed_tail_a, 0, nullptr + ) && + CheckErrorFree( + control, ErrorCode::HeadMismatch, Phase::CheckControl, task, block, head_b, 0, + kExpectedBucketB, 0, commit, head_b, observed_tail_b, 0, nullptr + ) && + CheckErrorFree( + control, ErrorCode::TailMismatch, Phase::CheckControl, task, block, observed_tail_b, 0, + kExpectedBucketB, 0, commit, head_b, observed_tail_b, 0, nullptr + ); + + bool lookup_ok = false; + const SharedTensorMapValue empty_query = MakeEntry(kBufferA0, 0, 0); + const int32_t producer = dist_shared_tensor_map_lookup_region(map, empty_query, 1, 1, lookup_ok); + checks += 2; + task_ok = task_ok && lookup_ok && producer == -1; + if (!lookup_ok || producer != -1) { + RecordFirstError( + control, lookup_ok ? ErrorCode::LookupMismatch : ErrorCode::LookupProtocol, Phase::CheckLookup, + task, block, producer, -1, kExpectedBucketA, 0, commit, head_a, observed_tail_a, 0, nullptr + ); + } + } else { + const uint64_t old_tail_a = tail_a; + const uint64_t old_tail_b = tail_b; + const uint32_t a_count = task == 1 ? 2U : 1U; + const uint32_t b_count = task >= 2 ? 1U : 0U; + const int64_t expected_head_a = task == 1 ? 0 : static_cast(old_tail_a); + const int64_t expected_head_b = task == 2 ? 0 : static_cast(old_tail_b); + const int64_t expected_tail_a = static_cast(old_tail_a + a_count); + const int64_t expected_tail_b = static_cast(old_tail_b + b_count); + for (uint32_t index = 0; index < plan.count; ++index) { + const uint32_t bucket = dist_tensor_map_hash(plan.entries[index].buf_addr); + uint64_t cursor = bucket == kExpectedBucketA ? old_tail_a : old_tail_b; + if (index != 0 && bucket == dist_tensor_map_hash(plan.entries[0].buf_addr)) { + ++cursor; + } + task_ok = task_ok && CheckPublishedEntry( + map, control, plan.entries[index], cursor, task, block, + bucket == kExpectedBucketA ? expected_head_a : expected_head_b, + bucket == kExpectedBucketA ? expected_tail_a : expected_tail_b, checks + ); + if (!task_ok) { + break; + } + } + } + if (!task_ok) { + break; + } + + const int64_t old_done = + DistSharedTensorMapAicoreOps::CompareExchange(&control.reader_done.value, task, epoch); + if (!CheckErrorFree( + control, ErrorCode::DoneOvershoot, Phase::WaitDone, task, block, old_done, task, 0, 0, commit, 0, 0, + 0, nullptr + )) { + break; + } + } else { + int64_t actual = 0; + const WaitResult wait = WaitForExact(control, &control.reader_done.value, epoch, actual); + if (wait != WaitResult::Reached) { + if (wait != WaitResult::Aborted) { + RecordFirstError( + control, wait == WaitResult::Timeout ? ErrorCode::DoneTimeout : ErrorCode::DoneOvershoot, + Phase::WaitDone, task, block, actual, epoch, 0, 0, LoadControl(&map.committed_tasks.v), 0, 0, 0, + nullptr + ); + } + break; + } + } + + if (task == 1) { + tail_a += 2; + } else if (task >= 2) { + ++tail_a; + ++tail_b; + } + ++completed_tasks; + } + + PublishParticipant(control, block, completed_tasks, writer_tasks, reader_tasks, checks, preheat_sink); +} diff --git a/tests/atomic_probe/ccec/shared_tensor_map_visibility_probe_host.cpp b/tests/atomic_probe/ccec/shared_tensor_map_visibility_probe_host.cpp new file mode 100644 index 0000000000..18243d32bf --- /dev/null +++ b/tests/atomic_probe/ccec/shared_tensor_map_visibility_probe_host.cpp @@ -0,0 +1,378 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#include "../probe_host.h" +#include "dist_engine/aicpu/shared_tensor_map_init.h" +#include "shared_tensor_map_visibility_probe_shared.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace shared_tensor_map_visibility_probe; + +static_assert(PTO_FDWIC_SHARED_MAP == 1, "visibility host must use shared production layout"); +static_assert(PTO_FDWIC_TENSORMAP_RING_CAP == kRingCapacity, "visibility host requires CAP=128"); +static_assert(sizeof(SharedTensorMapState) == 2113664); + +namespace { + +void Check(aclError error, const char *label) { + if (!atomic_probe::CheckAcl(error, label, __FILE__, __LINE__)) { + std::exit(EXIT_FAILURE); + } +} + +bool OptionalLaunches(uint32_t *launches) { + const char *raw = std::getenv("ATOMIC_PROBE_VISIBILITY_LAUNCHES"); + if (raw == nullptr || raw[0] == '\0') { + *launches = kDefaultLaunches; + return true; + } + errno = 0; + char *end = nullptr; + const unsigned long parsed = std::strtoul(raw, &end, 10); + if (errno != 0 || end == raw || *end != '\0' || parsed == 0 || parsed > kMaxLaunches) { + std::fprintf(stderr, "ATOMIC_PROBE_VISIBILITY_LAUNCHES must be in [1, %u], got %s\n", kMaxLaunches, raw); + return false; + } + *launches = static_cast(parsed); + return true; +} + +const char *ErrorLabel(ErrorCode code) { + switch (code) { + case ErrorCode::None: + return "none"; + case ErrorCode::InvalidTopology: + return "invalid-topology"; + case ErrorCode::HashConfiguration: + return "hash-configuration"; + case ErrorCode::ReadyTimeout: + return "ready-timeout"; + case ErrorCode::ReadyOvershoot: + return "ready-overshoot"; + case ErrorCode::PublishFailed: + return "publish-failed"; + case ErrorCode::CommitTimeout: + return "commit-timeout"; + case ErrorCode::CommitOvershoot: + return "commit-overshoot"; + case ErrorCode::CommitMismatch: + return "commit-mismatch"; + case ErrorCode::ReclaimMismatch: + return "reclaim-mismatch"; + case ErrorCode::HeadMismatch: + return "head-mismatch"; + case ErrorCode::TailMismatch: + return "tail-mismatch"; + case ErrorCode::SequenceMismatch: + return "sequence-mismatch"; + case ErrorCode::ReadFailed: + return "read-failed"; + case ErrorCode::PayloadMismatch: + return "payload-mismatch"; + case ErrorCode::LookupProtocol: + return "lookup-protocol"; + case ErrorCode::LookupMismatch: + return "lookup-mismatch"; + case ErrorCode::DoneTimeout: + return "done-timeout"; + case ErrorCode::DoneOvershoot: + return "done-overshoot"; + } + return "unknown"; +} + +bool LineIsZero(const DiagnosticLine &line) { + for (uint32_t index = 0; index < 8; ++index) { + if (line.words[index] != 0) { + return false; + } + } + return true; +} + +SharedTensorMapValue ExpectedValue(uint64_t buffer, int32_t task, uint32_t ordinal) { + const uint64_t lower = (static_cast(static_cast(task)) * 4ULL + ordinal) * 64ULL; + return {buffer, lower, lower + 32, task, 0}; +} + +bool ValuesEqual(const SharedTensorMapValue &left, const SharedTensorMapValue &right) { + return left.buf_addr == right.buf_addr && left.lo == right.lo && left.hi == right.hi && + left.producer == right.producer && left.reserved == right.reserved; +} + +} // namespace + +int main(int argc, char **argv) { + if (argc != 2) { + std::fprintf(stderr, "Usage: %s \n", argv[0]); + return EXIT_FAILURE; + } + + uint32_t launches = 0; + if (!OptionalLaunches(&launches)) { + return EXIT_FAILURE; + } + const int32_t device = atomic_probe::DeviceId(); + if (device < 0) { + return EXIT_FAILURE; + } + + Check(aclInit(nullptr), "initialize ACL"); + Check(aclrtSetDevice(device), "set visibility probe device"); + aclrtStream stream = nullptr; + Check(aclrtCreateStream(&stream), "create visibility probe stream"); + + std::ifstream file(argv[1], std::ios::binary | std::ios::ate); + if (!file) { + std::fprintf(stderr, "cannot open kernel file: %s\n", argv[1]); + return EXIT_FAILURE; + } + const size_t binary_size = static_cast(file.tellg()); + file.seekg(0); + std::vector binary(binary_size); + file.read(binary.data(), static_cast(binary_size)); + if (!file) { + return EXIT_FAILURE; + } + + aclrtBinHandle binary_handle; + Check( + atomic_probe::LoadAicoreBinaryFromData(binary.data(), binary.size(), &binary_handle), + "load visibility probe AICore binary" + ); + aclrtFuncHandle function_handle; + Check(aclrtBinaryGetFunctionByEntry(binary_handle, 0, &function_handle), "get visibility probe kernel entry"); + + void *device_map = nullptr; + void *device_control = nullptr; + Check( + aclrtMalloc(&device_map, sizeof(SharedTensorMapState), ACL_MEM_MALLOC_HUGE_FIRST), + "allocate production shared TensorMap state" + ); + Check( + aclrtMalloc(&device_control, sizeof(ProbeControl), ACL_MEM_MALLOC_HUGE_FIRST), + "allocate visibility probe control" + ); + + const uintptr_t map_address = reinterpret_cast(device_map); + const uintptr_t control_address = reinterpret_cast(device_control); + const bool aligned = (map_address & (kCacheLineBytes - 1U)) == 0 && (control_address & (kCacheLineBytes - 1U)) == 0; + std::printf( + "=== production shared TensorMap A5 visibility probe ===\n" + "backend=shared CAP=%u buckets=%u AIVs=%u tasks/launch=%u " + "reuse_tasks=%u launches=%u timeout=%llu cycles\n" + "scenario=zero-entry + same-bucket(A0/A1) + different-bucket(A/B) + " + "alternating-writer + >3 slot laps\n" + "addresses map=0x%zx(mod64=%zu) control=0x%zx(mod64=%zu)\n", + kRingCapacity, kMapBuckets, kAivBlocks, kTotalTasks, kReuseTasks, launches, + static_cast(kWaitTimeoutCycles), static_cast(map_address), + static_cast(map_address & 63U), static_cast(control_address), + static_cast(control_address & 63U) + ); + if (!aligned) { + std::fprintf(stderr, "probe allocations are not 64B aligned\n"); + Check(aclrtFree(device_control), "free unaligned visibility control"); + Check(aclrtFree(device_map), "free unaligned visibility map"); + Check(aclrtBinaryUnLoad(binary_handle), "unload visibility binary"); + Check(aclrtDestroyStream(stream), "destroy visibility stream"); + Check(aclrtResetDevice(device), "reset visibility probe device"); + Check(aclFinalize(), "finalize ACL"); + return EXIT_FAILURE; + } + + auto host_map = std::make_unique(); + auto host_control = std::make_unique(); + uint64_t protocol_failures = 0; + uint64_t semantic_failures = 0; + std::set> first_topology; + + for (uint32_t launch = 0; launch < launches; ++launch) { + // 使用 production AICPU reset 形成准确控制初态。payload 保留 A5 + // pattern，供 reader 预热旧 cache line；seq 决定其当前不可达。 + std::memset(host_map.get(), 0xa5, sizeof(*host_map)); + dist_shared_tensor_map_reset(*host_map); + std::memset(host_control.get(), 0, sizeof(*host_control)); + Check( + aclrtMemcpy(device_map, sizeof(*host_map), host_map.get(), sizeof(*host_map), ACL_MEMCPY_HOST_TO_DEVICE), + "initialize production shared TensorMap state" + ); + Check( + aclrtMemcpy( + device_control, sizeof(*host_control), host_control.get(), sizeof(*host_control), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "initialize visibility probe control" + ); + + KernelArgs args{ + static_cast(map_address), + static_cast(control_address), + kAivBlocks, + launch, + }; + Check( + aclrtLaunchKernelWithHostArgs( + function_handle, kAivBlocks, stream, nullptr, &args, sizeof(args), nullptr, 0 + ), + "launch production shared TensorMap visibility probe" + ); + Check(aclrtSynchronizeStream(stream), "wait for production shared TensorMap visibility probe"); + Check( + aclrtMemcpy(host_map.get(), sizeof(*host_map), device_map, sizeof(*host_map), ACL_MEMCPY_DEVICE_TO_HOST), + "read production shared TensorMap state" + ); + Check( + aclrtMemcpy( + host_control.get(), sizeof(*host_control), device_control, sizeof(*host_control), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "read visibility probe control" + ); + + const ErrorCode error = static_cast(host_control->abort_code.value); + bool protocol_ok = + host_control->reader_ready.value == kTotalTasks && host_control->reader_done.value == kTotalTasks && + host_control->finish_count.value == kAivBlocks && host_control->first_error_claim.value == 0 && + error == ErrorCode::None && LineIsZero(host_control->first_error) && + LineIsZero(host_control->first_snapshot) && LineIsZero(host_control->guard); + + std::set> topology; + for (uint32_t block = 0; block < kAivBlocks; ++block) { + const ParticipantLine &participant = host_control->participants[block]; + const uint64_t topology_word = static_cast(participant.words[2]); + const uint32_t core = static_cast(topology_word >> 32); + const uint32_t subblock = static_cast(topology_word); + topology.emplace(core, subblock); + const uint64_t role_counts = static_cast(participant.words[4]); + const uint32_t writer_tasks = static_cast(role_counts >> 32); + const uint32_t reader_tasks = static_cast(role_counts); + protocol_ok = protocol_ok && participant.words[0] == kParticipantMagic && participant.words[1] == block && + participant.words[3] == kTotalTasks && writer_tasks == kTotalTasks / 2 && + reader_tasks == kTotalTasks / 2 && participant.words[5] > 0 && participant.words[6] != 0 && + participant.words[7] == kParticipantFinish; + } + protocol_ok = protocol_ok && topology.size() == kAivBlocks; + if (launch == 0) { + first_topology = topology; + } + + const uint64_t expected_tail_a = 2 + kReuseTasks; + const uint64_t expected_tail_b = kReuseTasks; + const uint64_t expected_head_a = expected_tail_a - 1; + const uint64_t expected_head_b = expected_tail_b - 1; + bool state_ok = host_map->committed_tasks.v == kTotalTasks && + host_map->reclaim_upto.v == static_cast(kTotalTasks) - 2 && + host_map->buckets[kExpectedBucketA].head.v == static_cast(expected_head_a) && + host_map->buckets[kExpectedBucketA].tail.v == static_cast(expected_tail_a) && + host_map->buckets[kExpectedBucketB].head.v == static_cast(expected_head_b) && + host_map->buckets[kExpectedBucketB].tail.v == static_cast(expected_tail_b); + + const uint64_t cursor_a = expected_tail_a - 1; + const uint64_t cursor_b = expected_tail_b - 1; + const SharedTensorMapSlot &slot_a = + host_map->slots[kExpectedBucketA * kRingCapacity + (cursor_a & (kRingCapacity - 1U))]; + const SharedTensorMapSlot &slot_b = + host_map->slots[kExpectedBucketB * kRingCapacity + (cursor_b & (kRingCapacity - 1U))]; + const SharedTensorMapValue expected_a = ExpectedValue(kBufferA0, kTotalTasks - 1, 0); + const SharedTensorMapValue expected_b = ExpectedValue(kBufferB, kTotalTasks - 1, 1); + state_ok = state_ok && slot_a.sequence.v == static_cast(cursor_a) && + slot_b.sequence.v == static_cast(cursor_b) && + ValuesEqual(slot_a.payload.value, expected_a) && ValuesEqual(slot_b.payload.value, expected_b); + + if (!protocol_ok) { + ++protocol_failures; + } + if (!state_ok) { + ++semantic_failures; + } + std::printf( + "launch=%u error=%lld(%s) first_claim=%lld ready=%lld/%u " + "done=%lld/%u finish=%lld/%u commit=%lld/%u reclaim=%lld/%d " + "A(head=%lld tail=%lld seq=%lld) " + "B(head=%lld tail=%lld seq=%lld) protocol=%s state=%s\n", + launch, static_cast(host_control->abort_code.value), ErrorLabel(error), + static_cast(host_control->first_error_claim.value), + static_cast(host_control->reader_ready.value), kTotalTasks, + static_cast(host_control->reader_done.value), kTotalTasks, + static_cast(host_control->finish_count.value), kAivBlocks, + static_cast(host_map->committed_tasks.v), kTotalTasks, + static_cast(host_map->reclaim_upto.v), static_cast(kTotalTasks) - 2, + static_cast(host_map->buckets[kExpectedBucketA].head.v), + static_cast(host_map->buckets[kExpectedBucketA].tail.v), + static_cast(slot_a.sequence.v), + static_cast(host_map->buckets[kExpectedBucketB].head.v), + static_cast(host_map->buckets[kExpectedBucketB].tail.v), + static_cast(slot_b.sequence.v), protocol_ok ? "exact" : "BAD", state_ok ? "exact" : "BAD" + ); + + if (error != ErrorCode::None) { + const DiagnosticLine &first = host_control->first_error; + const DiagnosticLine &snapshot = host_control->first_snapshot; + std::printf( + "FIRST_ERROR code=%lld(%s) task=%lld block=%lld phase=%lld " + "actual=%lld expected=%lld aux0=%lld aux1=%lld " + "snapshot(commit=%lld head=%lld tail=%lld seq=%lld " + "buf=0x%016llx lo=%lld hi=%lld producer_reserved=0x%016llx)\n", + static_cast(first.words[0]), ErrorLabel(static_cast(first.words[0])), + static_cast(first.words[1]), static_cast(first.words[2]), + static_cast(first.words[3]), static_cast(first.words[4]), + static_cast(first.words[5]), static_cast(first.words[6]), + static_cast(first.words[7]), static_cast(snapshot.words[0]), + static_cast(snapshot.words[1]), static_cast(snapshot.words[2]), + static_cast(snapshot.words[3]), static_cast(snapshot.words[4]), + static_cast(snapshot.words[5]), static_cast(snapshot.words[6]), + static_cast(snapshot.words[7]) + ); + } + } + + std::printf("topology"); + for (const auto &[core, subblock] : first_topology) { + std::printf(" (core=%u sub=%u)", core, subblock); + } + std::printf( + "\naggregate launches=%u tasks=%llu zero=%u same_bucket=%u " + "different_bucket=%u writer_rotations=%llu " + "A_entries=%u A_laps=%.3f B_entries=%u B_laps=%.3f " + "protocol_failures=%llu semantic_failures=%llu\n", + launches, static_cast(launches) * kTotalTasks, launches, launches, launches * kReuseTasks, + static_cast(launches) * (kTotalTasks - 1U), 2 + kReuseTasks, + static_cast(2 + kReuseTasks) / kRingCapacity, kReuseTasks, + static_cast(kReuseTasks) / kRingCapacity, static_cast(protocol_failures), + static_cast(semantic_failures) + ); + + atomic_probe::Result result; + result.Expect( + protocol_failures == 0, "two AIVs complete every alternating writer/reader epoch without timeout or first error" + ); + result.Expect( + semantic_failures == 0, + "after observing commit, production tail/seq/payload/read/lookup remain exact across three laps" + ); + + Check(aclrtFree(device_control), "free visibility probe control"); + Check(aclrtFree(device_map), "free production shared TensorMap state"); + Check(aclrtBinaryUnLoad(binary_handle), "unload visibility binary"); + Check(aclrtDestroyStream(stream), "destroy visibility stream"); + Check(aclrtResetDevice(device), "reset visibility probe device"); + Check(aclFinalize(), "finalize ACL"); + return result.ExitCode(); +} diff --git a/tests/atomic_probe/ccec/shared_tensor_map_visibility_probe_shared.h b/tests/atomic_probe/ccec/shared_tensor_map_visibility_probe_shared.h new file mode 100644 index 0000000000..4916fe80ea --- /dev/null +++ b/tests/atomic_probe/ccec/shared_tensor_map_visibility_probe_shared.h @@ -0,0 +1,115 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#pragma once + +#include +#include + +namespace shared_tensor_map_visibility_probe { + +inline constexpr uint32_t kAivBlocks = 2; +inline constexpr uint32_t kCacheLineBytes = 64; +inline constexpr uint32_t kRingCapacity = 128; +inline constexpr uint32_t kReuseTasks = 3 * kRingCapacity + 8; +inline constexpr uint32_t kTotalTasks = 2 + kReuseTasks; +inline constexpr uint32_t kDefaultLaunches = 20; +inline constexpr uint32_t kMaxLaunches = 1000; +inline constexpr uint64_t kWaitTimeoutCycles = 100000000; + +inline constexpr uint64_t kBufferA0 = 0x0000000100000000ULL; +inline constexpr uint64_t kBufferA1 = 0x000000010007a000ULL; +inline constexpr uint64_t kBufferB = 0x0000000100001000ULL; +inline constexpr uint32_t kExpectedBucketA = 63; +inline constexpr uint32_t kExpectedBucketB = 123; + +inline constexpr int64_t kParticipantMagic = 0x53544d5052544349LL; // "STMPRTCI" +inline constexpr int64_t kParticipantFinish = 0x53544d50444f4e45LL; // "STMPDONE" + +enum class ErrorCode : int64_t { + None = 0, + InvalidTopology = 1, + HashConfiguration = 2, + ReadyTimeout = 3, + ReadyOvershoot = 4, + PublishFailed = 5, + CommitTimeout = 6, + CommitOvershoot = 7, + CommitMismatch = 8, + ReclaimMismatch = 9, + HeadMismatch = 10, + TailMismatch = 11, + SequenceMismatch = 12, + ReadFailed = 13, + PayloadMismatch = 14, + LookupProtocol = 15, + LookupMismatch = 16, + DoneTimeout = 17, + DoneOvershoot = 18, +}; + +enum class Phase : int64_t { + Setup = 0, + WaitReady = 1, + Publish = 2, + WaitCommit = 3, + CheckControl = 4, + CheckSlot = 5, + CheckLookup = 6, + WaitDone = 7, +}; + +struct alignas(kCacheLineBytes) AtomicLine { + volatile int64_t value; + uint8_t pad[kCacheLineBytes - sizeof(int64_t)]; +}; +static_assert(sizeof(AtomicLine) == kCacheLineBytes); + +struct alignas(kCacheLineBytes) DiagnosticLine { + volatile int64_t words[8]; +}; +static_assert(sizeof(DiagnosticLine) == kCacheLineBytes); + +struct alignas(kCacheLineBytes) ParticipantLine { + volatile int64_t words[8]; +}; +static_assert(sizeof(ParticipantLine) == kCacheLineBytes); + +struct alignas(kCacheLineBytes) ProbeControl { + AtomicLine reader_ready; + AtomicLine reader_done; + AtomicLine abort_code; + AtomicLine first_error_claim; + AtomicLine finish_count; + DiagnosticLine first_error; + DiagnosticLine first_snapshot; + ParticipantLine participants[kAivBlocks]; + DiagnosticLine guard; +}; +static_assert(offsetof(ProbeControl, reader_ready) == 0); +static_assert(offsetof(ProbeControl, reader_done) == 1 * kCacheLineBytes); +static_assert(offsetof(ProbeControl, abort_code) == 2 * kCacheLineBytes); +static_assert(offsetof(ProbeControl, first_error_claim) == 3 * kCacheLineBytes); +static_assert(offsetof(ProbeControl, finish_count) == 4 * kCacheLineBytes); +static_assert(offsetof(ProbeControl, first_error) == 5 * kCacheLineBytes); +static_assert(offsetof(ProbeControl, first_snapshot) == 6 * kCacheLineBytes); +static_assert(offsetof(ProbeControl, participants) == 7 * kCacheLineBytes); +static_assert(offsetof(ProbeControl, guard) == 9 * kCacheLineBytes); +static_assert(sizeof(ProbeControl) == 10 * kCacheLineBytes); + +struct KernelArgs { + uint64_t map_pointer; + uint64_t control_pointer; + uint32_t num_blocks; + uint32_t launch_id; +}; +static_assert(sizeof(KernelArgs) == 24); + +} // namespace shared_tensor_map_visibility_probe diff --git a/tests/atomic_probe/ccec/st_dev_ld_dev_sync.cpp b/tests/atomic_probe/ccec/st_dev_ld_dev_sync.cpp new file mode 100644 index 0000000000..a32b27ec6b --- /dev/null +++ b/tests/atomic_probe/ccec/st_dev_ld_dev_sync.cpp @@ -0,0 +1,106 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +// One-shot 2..20-AIV rendezvous (20 by default): +// initial SyncAll (outside the measured interval) +// -> block 0 writes one value with raw st_dev (no explicit DSB) +// -> blocks 1..19 poll the same value with ld_dev +// -> final SyncAll (inside the measured interval) +// +// Results are written only after the final timestamp, so result publication is +// not part of the reported synchronization span. No DCCI is used. +#include "ccec_utils.h" +#include "st_dev_ld_dev_sync_shared.h" + +CCEC_PROBE_KERNEL_META(st_dev_ld_dev_sync); + +namespace { + +using st_dev_ld_dev_sync::ProbeStorage; +using st_dev_ld_dev_sync::WorkerResult; + +__aicore__ inline uint64_t Timestamp() +{ + __asm__ __volatile__("" ::: "memory"); + return static_cast(get_sys_cnt()); +} + +__aicore__ inline void PublishResult( + __gm__ WorkerResult *result, uint32_t block_id, uint32_t block_count, + uint32_t observed_value, uint32_t poll_count, uint32_t flags, + uint64_t begin_tick, uint64_t observe_tick, + uint64_t final_barrier_arrive_tick, uint64_t end_tick) +{ + st_dev_b32(&result->magic, st_dev_ld_dev_sync::kResultMagic); + st_dev_b32(&result->block_id, block_id); + st_dev_b32(&result->core_id, static_cast(get_coreid())); + st_dev_b32(&result->subblock_id, static_cast(get_subblockid())); + st_dev_b32(&result->block_count, block_count); + st_dev_b32(&result->observed_value, observed_value); + st_dev_b32(&result->poll_count, poll_count); + st_dev_b32(&result->flags, flags); + st_dev_b64(&result->begin_tick, begin_tick); + st_dev_b64(&result->observe_tick, observe_tick); + st_dev_b64(&result->final_barrier_arrive_tick, final_barrier_arrive_tick); + st_dev_b64(&result->end_tick, end_tick); + // Host-result publication happens after end_tick and is not part of the + // tested signal-store protocol or any reported timing interval. + dsb(DSB_ALL); +} + +} // namespace + +extern "C" __global__ __aicore__ void +KERNEL_ENTRY(st_dev_ld_dev_sync)(__gm__ st_dev_ld_dev_sync::ProbeStorage *storage) +{ + const uint32_t block_id = static_cast(get_block_idx()); + const uint32_t block_count = static_cast(get_block_num()); + uint32_t observed_value = 0; + uint32_t poll_count = 0; + uint32_t flags = + block_count >= st_dev_ld_dev_sync::kMinAivWorkers && + block_count <= st_dev_ld_dev_sync::kMaxAivWorkers + ? 0U + : st_dev_ld_dev_sync::kFlagInvalidBlockCount; + + ccec_sync_all(); + const uint64_t begin_tick = Timestamp(); + + // signal.value is the only live object on this 64 B cache line. Both the + // writer's st_dev and every reader's ld_dev target that exclusive line. + if (block_id == 0U) { + st_dev_b32(&storage->signal.value, st_dev_ld_dev_sync::kExpectedValue); + observed_value = st_dev_ld_dev_sync::kExpectedValue; + } else { + const uint64_t wait_begin = Timestamp(); + do { + observed_value = ld_dev_b32(&storage->signal.value); + ++poll_count; + if (observed_value == st_dev_ld_dev_sync::kExpectedValue) { + break; + } + } while (Timestamp() - wait_begin < st_dev_ld_dev_sync::kWaitTimeoutTicks); + if (observed_value != st_dev_ld_dev_sync::kExpectedValue) { + flags |= st_dev_ld_dev_sync::kFlagReadTimeout; + } + } + + const uint64_t observe_tick = Timestamp(); + const uint64_t final_barrier_arrive_tick = Timestamp(); + ccec_sync_all(); + const uint64_t end_tick = Timestamp(); + + if (block_id < st_dev_ld_dev_sync::kMaxAivWorkers) { + PublishResult( + &storage->workers[block_id], block_id, block_count, observed_value, + poll_count, flags, begin_tick, observe_tick, + final_barrier_arrive_tick, end_tick); + } +} diff --git a/tests/atomic_probe/ccec/st_dev_ld_dev_sync_host.cpp b/tests/atomic_probe/ccec/st_dev_ld_dev_sync_host.cpp new file mode 100644 index 0000000000..6165d19736 --- /dev/null +++ b/tests/atomic_probe/ccec/st_dev_ld_dev_sync_host.cpp @@ -0,0 +1,411 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#include "../probe_host.h" +#include "st_dev_ld_dev_sync_shared.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using st_dev_ld_dev_sync::ProbeStorage; +using st_dev_ld_dev_sync::SignalLine; +using st_dev_ld_dev_sync::WorkerResult; + +constexpr uint32_t kWarmupLaunches = 10; +constexpr uint32_t kMeasuredLaunches = 200; +constexpr double kSysCntFrequencyHz = 1000000000.0; + +struct KernelArgs { + uint64_t storage_pointer; +}; + +struct Sample { + uint64_t overall_ticks; + uint64_t start_skew_ticks; + uint64_t writer_st_dev_ticks; + uint64_t last_reader_observe_ticks; + uint64_t final_arrival_skew_ticks; + uint64_t final_release_ticks; + uint64_t end_skew_ticks; + uint64_t maximum_reader_polls; +}; + +struct Distribution { + uint64_t minimum; + uint64_t p50; + uint64_t p95; + uint64_t maximum; + double mean; +}; + +static_assert(sizeof(KernelArgs) == 8, "unexpected CCEC kernel argument ABI"); + +void Check(aclError error, const char *label) +{ + if (!atomic_probe::CheckAcl(error, label, __FILE__, __LINE__)) { + std::exit(EXIT_FAILURE); + } +} + +std::vector ReadBinary(const char *path) +{ + std::ifstream stream(path, std::ios::binary | std::ios::ate); + if (!stream) { + return {}; + } + const std::streamsize size = stream.tellg(); + if (size <= 0) { + return {}; + } + stream.seekg(0); + std::vector data(static_cast(size)); + if (!stream.read(data.data(), size)) { + return {}; + } + return data; +} + +bool ReadWorkerCount(uint32_t *workers) +{ + const char *raw = std::getenv("ATOMIC_PROBE_AIVS"); + if (raw == nullptr || raw[0] == '\0') { + *workers = st_dev_ld_dev_sync::kDefaultAivWorkers; + return true; + } + errno = 0; + char *end = nullptr; + const unsigned long parsed = std::strtoul(raw, &end, 10); + if (errno != 0 || end == raw || *end != '\0' || + parsed < st_dev_ld_dev_sync::kMinAivWorkers || + parsed > st_dev_ld_dev_sync::kMaxAivWorkers) { + std::fprintf( + stderr, "ATOMIC_PROBE_AIVS must be in [%u, %u], got: %s\n", + st_dev_ld_dev_sync::kMinAivWorkers, + st_dev_ld_dev_sync::kMaxAivWorkers, raw); + return false; + } + *workers = static_cast(parsed); + return true; +} + +bool SignalPaddingIsZero(const SignalLine &line) +{ + return std::all_of(std::begin(line.padding), std::end(line.padding), + [](uint8_t value) { return value == 0U; }); +} + +bool WorkerResultIsZero(const WorkerResult &worker) +{ + const auto *bytes = reinterpret_cast(&worker); + return std::all_of(bytes, bytes + sizeof(worker), + [](uint8_t value) { return value == 0U; }); +} + +Distribution Summarize(std::vector values) +{ + std::sort(values.begin(), values.end()); + const uint64_t sum = std::accumulate(values.begin(), values.end(), uint64_t{0}); + return Distribution{ + values.front(), + values[(values.size() - 1U) * 50U / 100U], + values[(values.size() - 1U) * 95U / 100U], + values.back(), + static_cast(sum) / static_cast(values.size()), + }; +} + +void PrintDistribution(const char *label, const std::vector &values) +{ + const Distribution distribution = Summarize(values); + std::printf( + "[METRIC] %-24s n=%zu min=%" PRIu64 " p50=%" PRIu64 + " p95=%" PRIu64 " max=%" PRIu64 " mean=%.1f SYS_CNT ticks" + " (p50=%.3f us @1GHz SYS_CNT)\n", + label, values.size(), distribution.minimum, distribution.p50, + distribution.p95, distribution.maximum, distribution.mean, + static_cast(distribution.p50) / + (kSysCntFrequencyHz / 1000000.0)); +} + +void PrintCountDistribution(const char *label, + const std::vector &values) +{ + const Distribution distribution = Summarize(values); + std::printf( + "[METRIC] %-24s n=%zu min=%" PRIu64 " p50=%" PRIu64 + " p95=%" PRIu64 " max=%" PRIu64 " mean=%.1f polls\n", + label, values.size(), distribution.minimum, distribution.p50, + distribution.p95, distribution.maximum, distribution.mean); +} + +bool ValidateAndMeasure(const ProbeStorage &storage, uint32_t launch, + uint32_t workers, bool print_topology, Sample *sample) +{ + bool valid = true; + auto Mismatch = [&](uint32_t block, const char *field, uint64_t actual, + uint64_t expected) { + std::fprintf(stderr, + "[MISMATCH] launch=%u block=%u field=%s actual=%" PRIu64 + " expected=%" PRIu64 "\n", + launch, block, field, actual, expected); + valid = false; + }; + + if (storage.signal.value != st_dev_ld_dev_sync::kExpectedValue) { + Mismatch(0U, "signal.value", storage.signal.value, + st_dev_ld_dev_sync::kExpectedValue); + } + if (!SignalPaddingIsZero(storage.signal)) { + Mismatch(0U, "signal.padding", 1U, 0U); + } + if (storage.guard.value != 0U || !SignalPaddingIsZero(storage.guard)) { + Mismatch(0U, "guard", 1U, 0U); + } + + uint64_t minimum_begin = UINT64_MAX; + uint64_t maximum_begin = 0; + uint64_t maximum_reader_observe = 0; + uint64_t minimum_final_arrive = UINT64_MAX; + uint64_t maximum_final_arrive = 0; + uint64_t minimum_end = UINT64_MAX; + uint64_t maximum_end = 0; + uint64_t maximum_reader_polls = 0; + std::set> physical_aivs; + + if (print_topology) { + std::printf("[TOPOLOGY]"); + } + for (uint32_t block = 0; block < workers; ++block) { + const WorkerResult &worker = storage.workers[block]; + if (print_topology) { + std::printf(" block%u=(core%u,sub%u)", block, worker.core_id, + worker.subblock_id); + } + physical_aivs.emplace(worker.core_id, worker.subblock_id); + if (worker.magic != st_dev_ld_dev_sync::kResultMagic) { + Mismatch(block, "magic", worker.magic, + st_dev_ld_dev_sync::kResultMagic); + } + if (worker.block_id != block) { + Mismatch(block, "block_id", worker.block_id, block); + } + if (worker.block_count != workers) { + Mismatch(block, "block_count", worker.block_count, workers); + } + if (worker.observed_value != st_dev_ld_dev_sync::kExpectedValue) { + Mismatch(block, "observed_value", worker.observed_value, + st_dev_ld_dev_sync::kExpectedValue); + } + if (worker.flags != 0U) { + Mismatch(block, "flags", worker.flags, 0U); + } + if (block == 0U && worker.poll_count != 0U) { + Mismatch(block, "writer.poll_count", worker.poll_count, 0U); + } + if (block != 0U && worker.poll_count == 0U) { + Mismatch(block, "reader.poll_count", worker.poll_count, 1U); + } + if (worker.begin_tick == 0U || + worker.observe_tick < worker.begin_tick || + worker.final_barrier_arrive_tick < worker.observe_tick || + worker.end_tick < worker.final_barrier_arrive_tick) { + std::fprintf(stderr, + "[MISMATCH] launch=%u block=%u invalid timing order: begin=%" PRIu64 + " observe=%" PRIu64 " final_arrive=%" PRIu64 " end=%" PRIu64 "\n", + launch, block, worker.begin_tick, worker.observe_tick, + worker.final_barrier_arrive_tick, worker.end_tick); + valid = false; + } + minimum_begin = std::min(minimum_begin, worker.begin_tick); + maximum_begin = std::max(maximum_begin, worker.begin_tick); + minimum_final_arrive = std::min( + minimum_final_arrive, worker.final_barrier_arrive_tick); + maximum_final_arrive = std::max( + maximum_final_arrive, worker.final_barrier_arrive_tick); + minimum_end = std::min(minimum_end, worker.end_tick); + maximum_end = std::max(maximum_end, worker.end_tick); + if (block != 0U) { + maximum_reader_observe = std::max( + maximum_reader_observe, worker.observe_tick); + maximum_reader_polls = std::max( + maximum_reader_polls, worker.poll_count); + } + } + if (print_topology) { + std::printf("\n"); + } + for (uint32_t block = workers; + block < st_dev_ld_dev_sync::kMaxAivWorkers; ++block) { + if (!WorkerResultIsZero(storage.workers[block])) { + Mismatch(block, "unused_worker_result", 1U, 0U); + } + } + if (physical_aivs.size() != workers) { + Mismatch(0U, "unique_physical_aivs", physical_aivs.size(), + workers); + } + + const WorkerResult &writer = storage.workers[0]; + if (maximum_end < minimum_begin || maximum_begin < minimum_begin || + maximum_reader_observe < writer.begin_tick || + maximum_final_arrive < minimum_final_arrive || + maximum_end < maximum_final_arrive || minimum_end > maximum_end) { + std::fprintf(stderr, "[MISMATCH] launch=%u cross-core timing order invalid\n", + launch); + valid = false; + } + + sample->overall_ticks = maximum_end - minimum_begin; + sample->start_skew_ticks = maximum_begin - minimum_begin; + sample->writer_st_dev_ticks = writer.observe_tick - writer.begin_tick; + sample->last_reader_observe_ticks = + maximum_reader_observe - writer.begin_tick; + sample->final_arrival_skew_ticks = + maximum_final_arrive - minimum_final_arrive; + sample->final_release_ticks = maximum_end - maximum_final_arrive; + sample->end_skew_ticks = maximum_end - minimum_end; + sample->maximum_reader_polls = maximum_reader_polls; + return valid; +} + +} // namespace + +int main(int argc, char **argv) +{ + if (argc != 2) { + std::fprintf(stderr, "Usage: %s \n", argv[0]); + return EXIT_FAILURE; + } + + uint32_t workers = 0; + if (!ReadWorkerCount(&workers)) { + return EXIT_FAILURE; + } + + const std::vector binary = ReadBinary(argv[1]); + if (binary.empty()) { + std::fprintf(stderr, "Cannot read kernel binary: %s\n", argv[1]); + return EXIT_FAILURE; + } + + const int32_t device = atomic_probe::DeviceId(); + if (device < 0) { + return EXIT_FAILURE; + } + Check(aclInit(nullptr), "initialize ACL"); + Check(aclrtSetDevice(device), "set probe device"); + aclrtStream stream = nullptr; + Check(aclrtCreateStream(&stream), "create probe stream"); + + aclrtBinHandle binary_handle; + Check(atomic_probe::LoadAicoreBinaryFromData( + binary.data(), binary.size(), &binary_handle), + "load st_dev/ld_dev sync binary"); + aclrtFuncHandle function_handle; + Check(aclrtBinaryGetFunctionByEntry(binary_handle, 0, &function_handle), + "get st_dev/ld_dev sync entry"); + + void *device_storage = nullptr; + Check(aclrtMalloc(&device_storage, sizeof(ProbeStorage), + ACL_MEM_MALLOC_HUGE_FIRST), + "allocate st_dev/ld_dev sync storage"); + const uintptr_t storage_address = + reinterpret_cast(device_storage); + if ((storage_address & 63U) != 0U) { + std::fprintf(stderr, + "Probe storage is not 64-byte aligned: 0x%zx\n", + static_cast(storage_address)); + return EXIT_FAILURE; + } + + std::printf( + "=== %u-AIV st_dev/ld_dev one-shot synchronization ===\n" + "writer=block0 readers=blocks1..%u publish='raw st_dev, no explicit DSB' " + "poll='ld_dev until exact value'\n" + "layout=signal owns one exclusive 64B cache line; results and guard use separate lines\n" + "measurement=after initial SyncAll -> publish/poll -> after final " + "SyncAll; result stores excluded\n" + "warmup_launches=%u measured_launches=%u timeout=%" PRIu64 + " SYS_CNT ticks SYS_CNT_frequency=1GHz device=%d\n", + workers, workers - 1U, kWarmupLaunches, kMeasuredLaunches, + st_dev_ld_dev_sync::kWaitTimeoutTicks, device); + + std::array, 8> metrics; + bool all_valid = true; + const uint32_t total_launches = kWarmupLaunches + kMeasuredLaunches; + for (uint32_t launch = 0; launch < total_launches; ++launch) { + ProbeStorage host_storage{}; + Check(aclrtMemcpy(device_storage, sizeof(host_storage), &host_storage, + sizeof(host_storage), ACL_MEMCPY_HOST_TO_DEVICE), + "initialize st_dev/ld_dev sync storage"); + KernelArgs args{static_cast(storage_address)}; + Check(aclrtLaunchKernelWithHostArgs( + function_handle, workers, stream, + nullptr, &args, sizeof(args), nullptr, 0), + "launch st_dev/ld_dev sync kernel"); + Check(aclrtSynchronizeStream(stream), + "wait for st_dev/ld_dev sync kernel"); + Check(aclrtMemcpy(&host_storage, sizeof(host_storage), device_storage, + sizeof(host_storage), ACL_MEMCPY_DEVICE_TO_HOST), + "read st_dev/ld_dev sync result"); + + Sample sample{}; + const bool valid = ValidateAndMeasure( + host_storage, launch, workers, launch == 0U, &sample); + all_valid = all_valid && valid; + if (launch >= kWarmupLaunches && valid) { + metrics[0].push_back(sample.overall_ticks); + metrics[1].push_back(sample.start_skew_ticks); + metrics[2].push_back(sample.writer_st_dev_ticks); + metrics[3].push_back(sample.last_reader_observe_ticks); + metrics[4].push_back(sample.final_arrival_skew_ticks); + metrics[5].push_back(sample.final_release_ticks); + metrics[6].push_back(sample.end_skew_ticks); + metrics[7].push_back(sample.maximum_reader_polls); + } + } + + atomic_probe::Result result; + char semantics_label[96]; + std::snprintf( + semantics_label, sizeof(semantics_label), + "all launches preserve one-writer/%u-reader semantics", workers - 1U); + result.Expect(all_valid, semantics_label); + result.Expect(metrics[0].size() == kMeasuredLaunches, + "all measured launches produced valid samples"); + if (metrics[0].size() == kMeasuredLaunches) { + PrintDistribution("overall", metrics[0]); + PrintDistribution("initial_start_skew", metrics[1]); + PrintDistribution("writer_st_dev_span", metrics[2]); + PrintDistribution("last_reader_observe", metrics[3]); + PrintDistribution("final_arrival_skew", metrics[4]); + PrintDistribution("final_sync_release", metrics[5]); + PrintDistribution("final_end_skew", metrics[6]); + PrintCountDistribution("maximum_reader_polls", metrics[7]); + } + + Check(aclrtFree(device_storage), "free st_dev/ld_dev sync storage"); + Check(aclrtBinaryUnLoad(binary_handle), "unload st_dev/ld_dev sync binary"); + Check(aclrtDestroyStream(stream), "destroy st_dev/ld_dev sync stream"); + Check(aclrtResetDevice(device), "reset probe device"); + Check(aclFinalize(), "finalize ACL"); + return result.ExitCode(); +} diff --git a/tests/atomic_probe/ccec/st_dev_ld_dev_sync_shared.h b/tests/atomic_probe/ccec/st_dev_ld_dev_sync_shared.h new file mode 100644 index 0000000000..84e9367a9f --- /dev/null +++ b/tests/atomic_probe/ccec/st_dev_ld_dev_sync_shared.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#ifndef TESTS_ATOMIC_PROBE_CCEC_ST_DEV_LD_DEV_SYNC_SHARED_H +#define TESTS_ATOMIC_PROBE_CCEC_ST_DEV_LD_DEV_SYNC_SHARED_H + +#include +#include + +namespace st_dev_ld_dev_sync { + +constexpr uint32_t kMinAivWorkers = 2; +constexpr uint32_t kDefaultAivWorkers = 20; +constexpr uint32_t kMaxAivWorkers = 20; +constexpr uint32_t kExpectedValue = 0x53594e43U; // "SYNC" +constexpr uint32_t kResultMagic = 0x534c4453U; // "SLDS" +constexpr uint64_t kWaitTimeoutTicks = 20000000ULL; // 20 ms with the A5 1 GHz SYS_CNT. + +constexpr uint32_t kFlagInvalidBlockCount = 1U << 0; +constexpr uint32_t kFlagReadTimeout = 1U << 1; + +// Protocol requirement: the word polled by ld_dev must own an entire cache +// line. Do not reuse the padding for flags, counters, results, or any other +// live object; unrelated accesses on this line would change the fanout model. +struct alignas(64) SignalLine { + uint32_t value; + uint8_t padding[64 - sizeof(uint32_t)]; +}; + +struct alignas(64) WorkerResult { + uint32_t magic; + uint32_t block_id; + uint32_t core_id; + uint32_t subblock_id; + uint32_t block_count; + uint32_t observed_value; + uint32_t poll_count; + uint32_t flags; + uint64_t begin_tick; + uint64_t observe_tick; + uint64_t final_barrier_arrive_tick; + uint64_t end_tick; +}; + +struct alignas(64) ProbeStorage { + SignalLine signal; + WorkerResult workers[kMaxAivWorkers]; + SignalLine guard; +}; + +static_assert(sizeof(SignalLine) == 64, "signal line must occupy one cache line"); +static_assert(alignof(SignalLine) == 64, "signal line must start on a cache-line boundary"); +static_assert(sizeof(WorkerResult) == 64, "worker result must occupy one cache line"); +static_assert(offsetof(ProbeStorage, signal) % 64 == 0, "signal must be cache-line aligned"); +static_assert(offsetof(ProbeStorage, workers) % 64 == 0, "worker results must be cache-line aligned"); +static_assert(offsetof(ProbeStorage, workers) == sizeof(SignalLine), + "no object may share the signal cache line"); +static_assert(offsetof(ProbeStorage, guard) % 64 == 0, "guard must be cache-line aligned"); + +} // namespace st_dev_ld_dev_sync + +#endif // TESTS_ATOMIC_PROBE_CCEC_ST_DEV_LD_DEV_SYNC_SHARED_H diff --git a/tests/atomic_probe/ccec/taskcell_atomic_dcci.cpp b/tests/atomic_probe/ccec/taskcell_atomic_dcci.cpp new file mode 100644 index 0000000000..0d38171dc1 --- /dev/null +++ b/tests/atomic_probe/ccec/taskcell_atomic_dcci.cpp @@ -0,0 +1,444 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +// AIV-only TaskCell::deps_prepared 与 DCCI 精确场景探针。 +// +// 固定两个 AIV。两个参与者在每个独立 ProbeStorage 的 role_claim line +// 上领取 writer/reader 角色； +// reader_ready、phase、reader_ack 也各自独占 64B atomic-only line， +// 所有控制流量都不会触碰被测 TaskCell 或 dedicated deps line。 +// 五类场景和全部 trial 在同一次 kernel 内顺序推进：这既保证每个 trial +// 仍使用从未被 device 访问过的新地址，也规避重复 raw launch 的动态参数 +// 更新差异干扰被测 cache 行协议。 +// +// 五种场景： +// 0. TaskCell 共线：ordinary 构造旧 deps -> CAS 发布 -> reader 看见新值 +// -> 构造核 DCCI 同一 TaskCell line。 +// 1. TaskCell 共线：ordinary 写新 deps -> reader 在 DCCI 前有限轮询 +// -> 构造核 DCCI 同一 TaskCell line。 +// 2. deps 独占 line：CAS 发布 -> reader 看见新值 -> 对 atomic-only line DCCI。 +// 3. deps 独占 line：ordinary 写新值 -> reader 在 DCCI 前有限轮询 -> DCCI。 +// 4. deps 独占 line：ordinary 写新值 -> 全程不 DCCI，作为负对照。 +// +// runner 会关闭 scalar 自动 DCCI 和 kernel-end DCCI。任何普通写的可见性 +// 只能来自显式 DCCI、自然 writeback/eviction 或设备实现，不能被编译器补写污染。 +#include "ccec_utils.h" +#include "taskcell_atomic_dcci_shared.h" + +using namespace taskcell_atomic_dcci_probe; + +CCEC_PROBE_KERNEL_META(taskcell_atomic_dcci); + +namespace { + +struct PollResult { + int64_t first; + int64_t final; + uint32_t count; + bool saw_token; + bool timed_out; + bool saw_unexpected; + bool saw_regression; +}; + +__aicore__ inline int64_t AtomicLoad(__gm__ volatile int64_t *address) +{ + return atomicAdd( + const_cast<__gm__ int64_t *>(address), static_cast(0)); +} + +__aicore__ inline int64_t AtomicCompareExchange( + __gm__ volatile int64_t *address, int64_t expected, int64_t desired) +{ + return atomicCAS( + const_cast<__gm__ int64_t *>(address), expected, desired); +} + +__aicore__ inline void AtomicStoreResult( + __gm__ volatile int64_t *address, int64_t value) +{ + (void)atomicExch( + reinterpret_cast<__gm__ uint64_t *>( + const_cast<__gm__ int64_t *>(address)), + static_cast(value)); +} + +__aicore__ inline bool WaitForAtLeast( + __gm__ volatile int64_t *address, int64_t target, uint16_t &flags, + uint16_t timeout_flag) +{ + const uint64_t begin = static_cast(get_sys_cnt()); + while (true) { + const int64_t observed = AtomicLoad(address); + if (observed >= target) { + if (observed != target) { + flags |= kFlagControlTransition; + } + return observed == target; + } + if (static_cast(get_sys_cnt()) - begin >= + kWaitTimeoutTicks) { + flags |= timeout_flag; + return false; + } + } +} + +__aicore__ inline void AdvanceControl( + __gm__ volatile int64_t *address, int64_t expected, int64_t desired, + uint16_t &flags) +{ + if (AtomicCompareExchange(address, expected, desired) != expected) { + flags |= kFlagControlTransition; + } +} + +__aicore__ inline PollResult PollFixed( + __gm__ volatile int64_t *address, int64_t token) +{ + PollResult result{ + kNotApplicable, kNotApplicable, 0, false, false, false, false}; + for (uint32_t index = 0; index < kObservationPolls; ++index) { + const int64_t value = AtomicLoad(address); + if (index == 0) { + result.first = value; + } + result.final = value; + ++result.count; + if (value == token) { + result.saw_token = true; + } else if (value == kInitialDeps) { + if (result.saw_token) { + result.saw_regression = true; + } + } else { + result.saw_unexpected = true; + } + } + return result; +} + +__aicore__ inline PollResult PollOnce( + __gm__ volatile int64_t *address, int64_t token) +{ + const int64_t value = AtomicLoad(address); + return PollResult{ + value, + value, + 1, + value == token, + false, + value != kInitialDeps && value != token, + false}; +} + +__aicore__ inline PollResult PollUntilToken( + __gm__ volatile int64_t *address, int64_t token) +{ + PollResult result{ + kNotApplicable, kNotApplicable, 0, false, false, false, false}; + const uint64_t begin = static_cast(get_sys_cnt()); + while (true) { + const int64_t value = AtomicLoad(address); + if (result.count == 0) { + result.first = value; + } + result.final = value; + ++result.count; + if (value == token) { + result.saw_token = true; + return result; + } + if (value != kInitialDeps) { + result.saw_unexpected = true; + } + if (static_cast(get_sys_cnt()) - begin >= + kWaitTimeoutTicks) { + result.timed_out = true; + return result; + } + } +} + +__aicore__ inline void FlushSingleLine(__gm__ volatile int64_t *address) +{ + // CCEC 9.1 不能把 dcci/dsb 当作普通 GM 的 compiler barrier。 + // 前后空汇编只约束编译器；DCCI+DSB 才是被测设备动作。 + __asm__ volatile("" ::: "memory"); + dcci( + reinterpret_cast<__gm__ uint8_t *>( + const_cast<__gm__ int64_t *>(address)), + SINGLE_CACHE_LINE, CACHELINE_OUT); + dsb(DSB_ALL); + __asm__ volatile("" ::: "memory"); +} + +__aicore__ inline void CompleteBeforePublish() +{ + // DSB 负责设备侧完成，空汇编的 memory clobber 负责禁止 CCEC + // 把 ordinary GM 访问跨过这条完成边界移动。 + __asm__ volatile("" ::: "memory"); + dsb(DSB_ALL); + __asm__ volatile("" ::: "memory"); +} + +__aicore__ inline void BuildSharedTask( + __gm__ TaskCellLine *task, uint32_t trial, int64_t deps) +{ + task->flag = static_cast(BuiltFlag(trial)); + task->vend = BuiltVend(trial); + task->deps_prepared = deps; + for (uint32_t index = 0; index < 5; ++index) { + task->padding_words[index] = BuiltTaskPadding(index, trial); + } +} + +__aicore__ inline uint64_t PackTopology() +{ + return + (static_cast(static_cast(get_coreid())) << 32) | + static_cast(get_subblockid()); +} + +__aicore__ inline uint64_t PackPollMeta( + uint32_t pre_count, uint32_t post_count, uint16_t flags) +{ + constexpr uint64_t kCountMask = (1ULL << 24) - 1; + return (static_cast(flags) << 48) | + ((static_cast(post_count) & kCountMask) << 24) | + (static_cast(pre_count) & kCountMask); +} + +__aicore__ inline void PublishWriterResult( + __gm__ ResultLine &line, Scenario scenario, uint32_t trial, + int64_t cas_old, int64_t local_after, int64_t gm_before, + int64_t gm_after, uint16_t flags) +{ + AtomicStoreResult( + &line.words[1], + static_cast(TrialTag(scenario, trial))); + AtomicStoreResult( + &line.words[2], static_cast(PackTopology())); + AtomicStoreResult(&line.words[3], cas_old); + AtomicStoreResult(&line.words[4], local_after); + AtomicStoreResult(&line.words[5], gm_before); + AtomicStoreResult(&line.words[6], gm_after); + AtomicStoreResult(&line.words[7], static_cast(flags)); + dsb(DSB_ALL); + AtomicStoreResult(&line.words[0], kWriterMagic); +} + +__aicore__ inline void PublishReaderResult( + __gm__ ResultLine &line, Scenario scenario, uint32_t trial, + const PollResult &pre, const PollResult &post, uint16_t flags) +{ + if (pre.saw_token) { + flags |= kFlagPreSawToken; + } + if (post.saw_token) { + flags |= kFlagPostSawToken; + } + if (pre.timed_out) { + flags |= kFlagPrePollTimeout; + } + if (post.timed_out) { + flags |= kFlagPostPollTimeout; + } + if (pre.saw_unexpected || post.saw_unexpected) { + flags |= kFlagUnexpectedPollValue; + } + if (pre.saw_regression || post.saw_regression) { + flags |= kFlagPollRegression; + } + + AtomicStoreResult( + &line.words[1], + static_cast(TrialTag(scenario, trial))); + AtomicStoreResult( + &line.words[2], static_cast(PackTopology())); + AtomicStoreResult(&line.words[3], pre.first); + AtomicStoreResult(&line.words[4], pre.final); + AtomicStoreResult(&line.words[5], post.first); + AtomicStoreResult(&line.words[6], post.final); + AtomicStoreResult( + &line.words[7], + static_cast( + PackPollMeta(pre.count, post.count, flags))); + dsb(DSB_ALL); + AtomicStoreResult(&line.words[0], kReaderMagic); +} + +__aicore__ inline void CaptureTargetSnapshot( + __gm__ volatile int64_t *target, __gm__ ResultLine &snapshot) +{ + __gm__ uint64_t *line = + reinterpret_cast<__gm__ uint64_t *>( + const_cast<__gm__ int64_t *>(target)); + for (uint32_t index = 0; index < 8; ++index) { + AtomicStoreResult( + &snapshot.words[index], + static_cast(ld_dev_b64(&line[index]))); + } + dsb(DSB_ALL); +} + +__aicore__ inline void RunWriter( + __gm__ ProbeStorage &storage, Scenario scenario, uint32_t trial, + uint32_t num_blocks) +{ + uint16_t flags = kFlagNone; + if (num_blocks != kAivBlocks) { + flags |= kFlagInvalidTopology; + } + (void)WaitForAtLeast( + &storage.reader_ready.value, 1, flags, kFlagReadyTimeout); + + __gm__ volatile int64_t *target = IsSharedScenario(scenario) ? + &storage.shared_task.deps_prepared : + &storage.dedicated_deps.deps_prepared; + __gm__ volatile int64_t *snapshot_base = IsSharedScenario(scenario) ? + &storage.shared_task.flag : + &storage.dedicated_deps.deps_prepared; + const int64_t token = PublishedToken(trial); + int64_t cas_old = kNotApplicable; + int64_t local_after = kNotApplicable; + + if (scenario == Scenario::SharedAtomicThenDcci) { + // 先用 ordinary store 构造完整 TaskCell dirty 快照，其中 deps + // 仍为 -1；随后 CAS 只更新 GM/atomic 路径。 + BuildSharedTask(&storage.shared_task, trial, kInitialDeps); + CompleteBeforePublish(); + cas_old = AtomicCompareExchange(target, kInitialDeps, token); + if (cas_old != kInitialDeps) { + flags |= kFlagAtomicPublishFailed; + } + CompleteBeforePublish(); + } else if (scenario == Scenario::SharedOrdinaryThenDcci) { + BuildSharedTask(&storage.shared_task, trial, token); + CompleteBeforePublish(); + local_after = storage.shared_task.deps_prepared; + } else if (scenario == Scenario::DedicatedAtomicThenDcci) { + // atomic-only 对照禁止任何 ordinary load/store 接触整条 deps line。 + cas_old = AtomicCompareExchange(target, kInitialDeps, token); + if (cas_old != kInitialDeps) { + flags |= kFlagAtomicPublishFailed; + } + CompleteBeforePublish(); + } else { + storage.dedicated_deps.deps_prepared = token; + CompleteBeforePublish(); + local_after = storage.dedicated_deps.deps_prepared; + } + + __asm__ volatile("" ::: "memory"); + AdvanceControl(&storage.phase.value, 0, 1, flags); + (void)WaitForAtLeast( + &storage.reader_ack.value, 1, flags, kFlagAckTimeout); + + // pre 状态已经由远端 reader 的 atomic poll 取证。这里不额外插入 + // ld_dev，确保 DCCI 前没有第三条目标访问路径。 + const int64_t gm_before = kNotApplicable; + if (UsesDcci(scenario)) { + FlushSingleLine(target); + } + AdvanceControl(&storage.phase.value, 1, 2, flags); + (void)WaitForAtLeast( + &storage.reader_ack.value, 2, flags, kFlagAckTimeout); + + // 远端 reader 的 post atomic poll 必须是 DCCI 完成后的第一个目标 + // 访问；等它取证并 ack 后，writer 才做旁路整线快照。 + const int64_t gm_after = + static_cast( + ld_dev_b64( + reinterpret_cast<__gm__ uint64_t *>( + const_cast<__gm__ int64_t *>(target)))); + CaptureTargetSnapshot(snapshot_base, storage.target_snapshot); + + PublishWriterResult( + storage.writer_result, scenario, trial, cas_old, local_after, + gm_before, gm_after, flags); +} + +__aicore__ inline void RunReader( + __gm__ ProbeStorage &storage, Scenario scenario, uint32_t trial, + uint32_t num_blocks) +{ + uint16_t flags = kFlagNone; + if (num_blocks != kAivBlocks) { + flags |= kFlagInvalidTopology; + } + AdvanceControl(&storage.reader_ready.value, 0, 1, flags); + (void)WaitForAtLeast( + &storage.phase.value, 1, flags, kFlagPhaseTimeout); + + __gm__ volatile int64_t *target = + IsSharedScenario(scenario) ? + &storage.shared_task.deps_prepared : + &storage.dedicated_deps.deps_prepared; + const int64_t token = PublishedToken(trial); + const PollResult pre = IsAtomicPublishScenario(scenario) ? + PollUntilToken(target, token) : + PollFixed(target, token); + + AdvanceControl(&storage.reader_ack.value, 0, 1, flags); + (void)WaitForAtLeast( + &storage.phase.value, 2, flags, kFlagPhaseTimeout); + + PollResult post{}; + if (scenario == Scenario::SharedAtomicThenDcci || + scenario == Scenario::DedicatedAtomicThenDcci) { + // atomic 发布在 DCCI 前已由 pre 窗口确认；这里只取 DCCI 后状态， + // 不用长轮询掩盖“新值被覆盖”的瞬时事实。 + post = PollOnce(target, token); + } else if (scenario == Scenario::DedicatedOrdinaryNoDcci) { + post = PollFixed(target, token); + } else { + post = PollUntilToken(target, token); + } + + PublishReaderResult( + storage.reader_result, scenario, trial, pre, post, flags); + AdvanceControl(&storage.reader_ack.value, 1, 2, flags); +} + +} // namespace + +extern "C" __global__ __aicore__ void KERNEL_ENTRY(taskcell_atomic_dcci)( + __gm__ ProbeStorage *storage, uint32_t trials, uint32_t num_blocks) +{ + if (storage == nullptr || trials == 0 || trials > kMaxTrials) { + return; + } + const uint32_t actual_blocks = + static_cast(get_block_num()); + const uint32_t checked_blocks = + actual_blocks == num_blocks ? num_blocks : 0; + + for (uint32_t scenario_raw = 0; scenario_raw < kScenarioCount; + ++scenario_raw) { + const Scenario scenario = static_cast(scenario_raw); + for (uint32_t trial = 0; trial < trials; ++trial) { + __gm__ ProbeStorage ¤t = + storage[static_cast(scenario_raw) * trials + + trial]; + const int64_t role = atomicAdd( + const_cast<__gm__ int64_t *>( + ¤t.role_claim.value), + static_cast(1)); + if (role == 0) { + RunWriter( + current, scenario, trial, checked_blocks); + } else if (role == 1) { + RunReader( + current, scenario, trial, checked_blocks); + } + } + } +} diff --git a/tests/atomic_probe/ccec/taskcell_atomic_dcci_host.cpp b/tests/atomic_probe/ccec/taskcell_atomic_dcci_host.cpp new file mode 100644 index 0000000000..579df9a217 --- /dev/null +++ b/tests/atomic_probe/ccec/taskcell_atomic_dcci_host.cpp @@ -0,0 +1,647 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +// AIV-only TaskCell atomic/DCCI 探针的 host 端精确判定。 +// +// 一次分配全部场景和轮次的存储，并用一次 2-AIV launch 顺序执行全部 +// 场景。每个试次使用唯一的 10 条 cache line，不复用目标地址。 +#include "../probe_host.h" +#include "taskcell_atomic_dcci_shared.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace taskcell_atomic_dcci_probe; + +namespace { + +constexpr uint64_t kPollCountMask = (1ULL << 24) - 1; +constexpr uint16_t kInformationalFlags = + kFlagPreSawToken | kFlagPostSawToken; + +struct PollRecord { + int64_t first; + int64_t final; + uint32_t count; + bool saw_token; +}; + +struct ScenarioStats { + uint32_t exact = 0; + uint32_t failed = 0; + uint32_t pre_visible = 0; + uint32_t post_visible = 0; + uint32_t snapshot_visible = 0; + uint32_t host_visible = 0; + uint32_t clobbered = 0; + uint32_t survived = 0; +}; + +void Check(aclError error, const char *label) +{ + if (!atomic_probe::CheckAcl(error, label, __FILE__, __LINE__)) { + std::exit(EXIT_FAILURE); + } +} + +bool ParseTrials(const char *raw, uint32_t *trials) +{ + errno = 0; + char *end = nullptr; + const unsigned long value = std::strtoul(raw, &end, 10); + if (errno != 0 || end == raw || *end != '\0' || value == 0 || + value > kMaxTrials) { + std::fprintf( + stderr, "trials 必须在 [1, %u]，当前输入：%s\n", + kMaxTrials, raw); + return false; + } + *trials = static_cast(value); + return true; +} + +const char *ScenarioName(Scenario scenario) +{ + switch (scenario) { + case Scenario::SharedAtomicThenDcci: + return "TaskCell 共线：CAS 发布后 DCCI"; + case Scenario::SharedOrdinaryThenDcci: + return "TaskCell 共线：普通写后 DCCI"; + case Scenario::DedicatedAtomicThenDcci: + return "deps 独占行：CAS 发布后 DCCI"; + case Scenario::DedicatedOrdinaryThenDcci: + return "deps 独占行：普通写后 DCCI"; + case Scenario::DedicatedOrdinaryNoDcci: + return "deps 独占行：普通写且无 DCCI"; + } + return "未知场景"; +} + +void InitializeStorage(ProbeStorage *storage, uint32_t trial) +{ + std::memset(storage, 0, sizeof(*storage)); + storage->shared_task.flag = static_cast(InitialFlag(trial)); + storage->shared_task.vend = InitialVend(trial); + storage->shared_task.deps_prepared = kInitialDeps; + for (uint32_t index = 0; index < 5; ++index) { + storage->shared_task.padding_words[index] = + InitialTaskPadding(index, trial); + } + storage->dedicated_deps.deps_prepared = kInitialDeps; + for (uint32_t index = 0; index < 7; ++index) { + storage->dedicated_deps.guards[index] = + DedicatedGuard(index, trial); + } + for (uint32_t index = 0; index < 8; ++index) { + storage->guard.words[index] = + static_cast(StorageGuard(index, trial)); + } +} + +bool ExactAtomicLine(const AtomicLine &line, int64_t expected) +{ + if (line.value != expected) { + return false; + } + for (uint32_t index = 0; index < sizeof(line.padding); ++index) { + if (line.padding[index] != 0) { + return false; + } + } + return true; +} + +bool ExactSharedTask( + const TaskCellLine &line, uint32_t trial, int64_t deps, bool built) +{ + const uint64_t expected_flag = + built ? BuiltFlag(trial) : InitialFlag(trial); + const uint64_t expected_vend = + built ? BuiltVend(trial) : InitialVend(trial); + if (static_cast(line.flag) != expected_flag || + line.vend != expected_vend || line.deps_prepared != deps) { + return false; + } + for (uint32_t index = 0; index < 5; ++index) { + const uint64_t expected = built ? + BuiltTaskPadding(index, trial) : + InitialTaskPadding(index, trial); + if (line.padding_words[index] != expected) { + return false; + } + } + return true; +} + +bool ExactDedicatedLine( + const DedicatedDepsLine &line, uint32_t trial, int64_t deps) +{ + if (line.deps_prepared != deps) { + return false; + } + for (uint32_t index = 0; index < 7; ++index) { + if (line.guards[index] != DedicatedGuard(index, trial)) { + return false; + } + } + return true; +} + +bool ExactGuard(const ResultLine &line, uint32_t trial) +{ + for (uint32_t index = 0; index < 8; ++index) { + if (static_cast(line.words[index]) != + StorageGuard(index, trial)) { + return false; + } + } + return true; +} + +bool ExactSharedSnapshot( + const ResultLine &line, uint32_t trial, int64_t deps, bool built) +{ + const uint64_t expected_flag = + built ? BuiltFlag(trial) : InitialFlag(trial); + const uint64_t expected_vend = + built ? BuiltVend(trial) : InitialVend(trial); + if (static_cast(line.words[0]) != expected_flag || + static_cast(line.words[1]) != expected_vend || + line.words[2] != deps) { + return false; + } + for (uint32_t index = 0; index < 5; ++index) { + const uint64_t expected = built ? + BuiltTaskPadding(index, trial) : + InitialTaskPadding(index, trial); + if (static_cast(line.words[index + 3]) != expected) { + return false; + } + } + return true; +} + +bool ExactDedicatedSnapshot( + const ResultLine &line, uint32_t trial, int64_t deps) +{ + if (line.words[0] != deps) { + return false; + } + for (uint32_t index = 0; index < 7; ++index) { + if (static_cast(line.words[index + 1]) != + DedicatedGuard(index, trial)) { + return false; + } + } + return true; +} + +uint16_t ReaderFlags(const ResultLine &line) +{ + return static_cast( + static_cast(line.words[7]) >> 48); +} + +PollRecord ReaderPre(const ResultLine &line) +{ + const uint64_t meta = static_cast(line.words[7]); + const uint16_t flags = ReaderFlags(line); + return { + line.words[3], + line.words[4], + static_cast(meta & kPollCountMask), + (flags & kFlagPreSawToken) != 0, + }; +} + +PollRecord ReaderPost(const ResultLine &line) +{ + const uint64_t meta = static_cast(line.words[7]); + const uint16_t flags = ReaderFlags(line); + return { + line.words[5], + line.words[6], + static_cast((meta >> 24) & kPollCountMask), + (flags & kFlagPostSawToken) != 0, + }; +} + +bool IsKnownValue(int64_t value, int64_t token) +{ + return value == kInitialDeps || value == token; +} + +bool IsMonotonicPair(int64_t first, int64_t final, int64_t token) +{ + return IsKnownValue(first, token) && IsKnownValue(final, token) && + !(first == token && final == kInitialDeps); +} + +bool IsMonotonicSequence( + const PollRecord &pre, const PollRecord &post, int64_t gm_after, + int64_t snapshot, int64_t host, int64_t token) +{ + const int64_t values[] = { + pre.first, pre.final, post.first, post.final, gm_after, + snapshot, host, + }; + bool saw_token = false; + for (int64_t value : values) { + if (!IsKnownValue(value, token)) { + return false; + } + if (saw_token && value == kInitialDeps) { + return false; + } + saw_token = saw_token || value == token; + } + return true; +} + +bool ExactResultHeader( + const ResultLine &line, int64_t magic, Scenario scenario, + uint32_t trial) +{ + return line.words[0] == magic && + static_cast(line.words[1]) == + TrialTag(scenario, trial); +} + +bool ReaderInfoMatches( + const PollRecord &pre, const PollRecord &post) +{ + // Kernel 的轮询器一旦发现 new 后禁止回退，因此 final 足以与 saw + // 信息位交叉核对；如果出现 new->old，kernel 会另设硬失败 flag。 + return pre.saw_token == (pre.final != kInitialDeps) && + post.saw_token == (post.final != kInitialDeps); +} + +void PrintLine(const char *label, const volatile int64_t *words) +{ + std::printf(" %s:", label); + for (uint32_t index = 0; index < 8; ++index) { + std::printf( + " %016" PRIx64, static_cast(words[index])); + } + std::printf("\n"); +} + +void PrintFailure( + Scenario scenario, uint32_t trial, const ProbeStorage &storage, + const PollRecord &pre, const PollRecord &post) +{ + std::printf( + "[异常] scenario=%u(%s) trial=%u token=%016" PRIx64 + " controls=(ready=%lld phase=%lld ack=%lld)" + " writer_topology=%016" PRIx64 + " reader_topology=%016" PRIx64 + " writer=(cas_old=%lld local=%lld gm_before=%lld gm_after=%lld" + " flags=0x%04" PRIx64 ")" + " reader=(pre=%lld->%lld/%u post=%lld->%lld/%u" + " flags=0x%04x)\n", + static_cast(scenario), ScenarioName(scenario), trial, + static_cast(PublishedToken(trial)), + static_cast(storage.reader_ready.value), + static_cast(storage.phase.value), + static_cast(storage.reader_ack.value), + static_cast(storage.writer_result.words[2]), + static_cast(storage.reader_result.words[2]), + static_cast(storage.writer_result.words[3]), + static_cast(storage.writer_result.words[4]), + static_cast(storage.writer_result.words[5]), + static_cast(storage.writer_result.words[6]), + static_cast(storage.writer_result.words[7]), + static_cast(pre.first), + static_cast(pre.final), pre.count, + static_cast(post.first), + static_cast(post.final), post.count, + ReaderFlags(storage.reader_result)); + PrintLine( + "target_snapshot", + storage.target_snapshot.words); + if (IsSharedScenario(scenario)) { + PrintLine( + "host_target", + reinterpret_cast( + &storage.shared_task)); + } else { + PrintLine( + "host_target", + reinterpret_cast( + &storage.dedicated_deps)); + } + PrintLine( + "storage_guard", + storage.guard.words); +} + +bool ValidateTrial( + Scenario scenario, uint32_t trial, const ProbeStorage &storage, + ScenarioStats *stats) +{ + const int64_t token = PublishedToken(trial); + const ResultLine &writer = storage.writer_result; + const ResultLine &reader = storage.reader_result; + const PollRecord pre = ReaderPre(reader); + const PollRecord post = ReaderPost(reader); + const uint16_t reader_flags = ReaderFlags(reader); + + bool exact = + ExactAtomicLine(storage.role_claim, kAivBlocks) && + ExactAtomicLine(storage.reader_ready, 1) && + ExactAtomicLine(storage.phase, 2) && + ExactAtomicLine(storage.reader_ack, 2) && + ExactResultHeader( + writer, kWriterMagic, scenario, trial) && + ExactResultHeader( + reader, kReaderMagic, scenario, trial) && + writer.words[2] != reader.words[2] && + writer.words[7] == kFlagNone && + (reader_flags & ~kInformationalFlags) == 0 && + ReaderInfoMatches(pre, post) && + ExactGuard(storage.guard, trial); + + const bool atomic_publish = IsAtomicPublishScenario(scenario); + exact = exact && + writer.words[3] == + (atomic_publish ? kInitialDeps : kNotApplicable) && + writer.words[4] == + (atomic_publish ? kNotApplicable : token) && + writer.words[5] == kNotApplicable; + + if (scenario == Scenario::SharedAtomicThenDcci) { + const bool pre_exact = + pre.count > 0 && pre.final == token && pre.saw_token && + IsKnownValue(pre.first, token); + const bool clobbered = + post.count == 1 && post.first == kInitialDeps && + post.final == kInitialDeps && !post.saw_token && + writer.words[6] == kInitialDeps && + ExactSharedSnapshot( + storage.target_snapshot, trial, kInitialDeps, true) && + ExactSharedTask( + storage.shared_task, trial, kInitialDeps, true); + const bool survived = + post.count == 1 && post.first == token && + post.final == token && post.saw_token && + writer.words[6] == token && + ExactSharedSnapshot( + storage.target_snapshot, trial, token, true) && + ExactSharedTask(storage.shared_task, trial, token, true); + exact = exact && pre_exact && (clobbered != survived) && + ExactDedicatedLine( + storage.dedicated_deps, trial, kInitialDeps); + stats->clobbered += clobbered ? 1U : 0U; + stats->survived += survived ? 1U : 0U; + } else if (scenario == Scenario::SharedOrdinaryThenDcci) { + exact = exact && + pre.count == kObservationPolls && + IsMonotonicPair(pre.first, pre.final, token) && + post.count > 0 && + IsMonotonicPair(post.first, post.final, token) && + post.final == token && post.saw_token && + writer.words[6] == token && + ExactSharedSnapshot( + storage.target_snapshot, trial, token, true) && + ExactSharedTask(storage.shared_task, trial, token, true) && + ExactDedicatedLine( + storage.dedicated_deps, trial, kInitialDeps); + } else if (scenario == Scenario::DedicatedAtomicThenDcci) { + exact = exact && + pre.count > 0 && pre.final == token && pre.saw_token && + IsKnownValue(pre.first, token) && + post.count == 1 && post.first == token && + post.final == token && post.saw_token && + writer.words[6] == token && + ExactDedicatedSnapshot( + storage.target_snapshot, trial, token) && + ExactDedicatedLine(storage.dedicated_deps, trial, token) && + ExactSharedTask( + storage.shared_task, trial, kInitialDeps, false); + } else if (scenario == Scenario::DedicatedOrdinaryThenDcci) { + exact = exact && + pre.count == kObservationPolls && + IsMonotonicPair(pre.first, pre.final, token) && + post.count > 0 && + IsMonotonicPair(post.first, post.final, token) && + post.final == token && post.saw_token && + writer.words[6] == token && + ExactDedicatedSnapshot( + storage.target_snapshot, trial, token) && + ExactDedicatedLine(storage.dedicated_deps, trial, token) && + ExactSharedTask( + storage.shared_task, trial, kInitialDeps, false); + } else { + const int64_t snapshot = storage.target_snapshot.words[0]; + const int64_t host = storage.dedicated_deps.deps_prepared; + exact = exact && + pre.count == kObservationPolls && + post.count == kObservationPolls && + IsMonotonicSequence( + pre, post, writer.words[6], snapshot, host, token) && + ExactDedicatedSnapshot( + storage.target_snapshot, trial, snapshot) && + ExactDedicatedLine(storage.dedicated_deps, trial, host) && + ExactSharedTask( + storage.shared_task, trial, kInitialDeps, false); + } + + stats->pre_visible += pre.saw_token ? 1U : 0U; + stats->post_visible += post.saw_token ? 1U : 0U; + stats->snapshot_visible += + storage.target_snapshot.words[ + IsSharedScenario(scenario) ? 2 : 0] == token ? 1U : 0U; + stats->host_visible += + (IsSharedScenario(scenario) ? + storage.shared_task.deps_prepared : + storage.dedicated_deps.deps_prepared) == token ? 1U : 0U; + if (exact) { + ++stats->exact; + } else { + ++stats->failed; + PrintFailure(scenario, trial, storage, pre, post); + } + return exact; +} + +void PrintStats( + Scenario scenario, const ScenarioStats &stats, uint32_t trials) +{ + std::printf( + "[场景%u] %s：exact=%u/%u failed=%u" + " pre可见=%u post可见=%u snapshot新值=%u host最终新值=%u", + static_cast(scenario), ScenarioName(scenario), + stats.exact, trials, stats.failed, stats.pre_visible, + stats.post_visible, stats.snapshot_visible, stats.host_visible); + if (scenario == Scenario::SharedAtomicThenDcci) { + std::printf( + " exact_clobbered=%u exact_survived=%u", + stats.clobbered, stats.survived); + } + std::printf("\n"); +} + +} // namespace + +int main(int argc, char **argv) +{ + if (argc > 3) { + std::fprintf( + stderr, "Usage: %s [kernel.o] [trials:1..%u]\n", + argv[0], kMaxTrials); + return EXIT_FAILURE; + } + const char *kernel_path = + argc >= 2 ? argv[1] : "./taskcell_atomic_dcci_kernel.o"; + uint32_t trials = kDefaultTrials; + if (argc == 3 && !ParseTrials(argv[2], &trials)) { + return EXIT_FAILURE; + } + + const int32_t device = atomic_probe::DeviceId(); + if (device < 0) { + return EXIT_FAILURE; + } + Check(aclInit(nullptr), "初始化 ACL"); + Check(aclrtSetDevice(device), "设置 TaskCell 探针设备"); + aclrtStream stream = nullptr; + Check(aclrtCreateStream(&stream), "创建 TaskCell 探针 stream"); + + std::ifstream file(kernel_path, std::ios::binary | std::ios::ate); + if (!file) { + std::fprintf(stderr, "无法打开 kernel 文件：%s\n", kernel_path); + return EXIT_FAILURE; + } + const size_t binary_size = static_cast(file.tellg()); + file.seekg(0); + std::vector binary(binary_size); + file.read(binary.data(), static_cast(binary_size)); + if (!file) { + std::fprintf(stderr, "读取 kernel 文件失败：%s\n", kernel_path); + return EXIT_FAILURE; + } + + aclrtBinHandle binary_handle = nullptr; + Check( + atomic_probe::LoadAicoreBinaryFromData( + binary.data(), binary.size(), &binary_handle), + "加载 TaskCell atomic/DCCI AICore binary"); + aclrtFuncHandle function = nullptr; + Check( + aclrtBinaryGetFunctionByEntry(binary_handle, 0, &function), + "取得 TaskCell atomic/DCCI kernel 入口"); + + const size_t storage_count = + static_cast(kScenarioCount) * trials; + const size_t storage_bytes = storage_count * sizeof(ProbeStorage); + std::vector initial(storage_count); + for (uint32_t scenario = 0; scenario < kScenarioCount; ++scenario) { + for (uint32_t trial = 0; trial < trials; ++trial) { + InitializeStorage( + &initial[static_cast(scenario) * trials + trial], + trial); + } + } + + void *device_storage = nullptr; + Check( + aclrtMalloc( + &device_storage, storage_bytes, ACL_MEM_MALLOC_HUGE_FIRST), + "一次分配全部场景和轮次的 10-line ProbeStorage"); + if ((reinterpret_cast(device_storage) & + (kCacheLineBytes - 1U)) != 0) { + std::fprintf( + stderr, "ProbeStorage 首地址没有按 64B 对齐\n"); + return EXIT_FAILURE; + } + Check( + aclrtMemcpy( + device_storage, storage_bytes, initial.data(), storage_bytes, + ACL_MEMCPY_HOST_TO_DEVICE), + "一次初始化全部唯一 ProbeStorage"); + + std::printf( + "=== AIV-only TaskCell::deps_prepared atomic/DCCI 探针 ===\n" + "scenarios=%u trials/scenario=%u launches=1 blocks/launch=%u " + "storage=%zu bytes line=%u bytes\n" + "DCCI 场景固定 SINGLE_CACHE_LINE + CACHELINE_OUT + DSB；" + "无 DCCI 场景只观察，不把可见/不可见写成架构保证。\n", + kScenarioCount, trials, kAivBlocks, + storage_bytes, kCacheLineBytes); + + std::vector stats(kScenarioCount); + KernelArgs args{ + static_cast( + reinterpret_cast(device_storage)), + trials, + kAivBlocks, + }; + Check( + aclrtLaunchKernelWithHostArgs( + function, kAivBlocks, stream, nullptr, &args, + sizeof(args), nullptr, 0), + "启动单次 2-AIV TaskCell atomic/DCCI kernel"); + Check( + aclrtSynchronizeStream(stream), + "等待全部 TaskCell atomic/DCCI 场景"); + + std::vector observed(storage_count); + Check( + aclrtMemcpy( + observed.data(), storage_bytes, device_storage, + storage_bytes, ACL_MEMCPY_DEVICE_TO_HOST), + "一次回读全部唯一 ProbeStorage"); + + for (uint32_t scenario_raw = 0; scenario_raw < kScenarioCount; + ++scenario_raw) { + const Scenario scenario = static_cast(scenario_raw); + for (uint32_t trial = 0; trial < trials; ++trial) { + const size_t index = + static_cast(scenario_raw) * trials + trial; + (void)ValidateTrial( + scenario, trial, observed[index], + &stats[scenario_raw]); + } + PrintStats(scenario, stats[scenario_raw], trials); + } + + atomic_probe::Result result; + result.Expect( + stats[0].failed == 0 && + stats[0].clobbered + stats[0].survived == trials, + "场景0只允许完整 clobbered 或完整 survived，拒绝 torn/第四态"); + result.Expect( + stats[1].failed == 0 && stats[1].post_visible == trials, + "场景1普通写+DCCI 后 reader/完整 TaskCell/guards 全部精确"); + result.Expect( + stats[2].failed == 0 && stats[2].post_visible == trials && + stats[2].snapshot_visible == trials && + stats[2].host_visible == trials, + "场景2 atomic-only 行 DCCI 前后均为 token，guards 全部精确"); + result.Expect( + stats[3].failed == 0 && stats[3].post_visible == trials, + "场景3独占行普通写+DCCI 后 token 与 guards 全部精确"); + result.Expect( + stats[4].failed == 0, + "场景4无 DCCI 只接受 old/token 单调序列，不预设最终可见性"); + + Check(aclrtFree(device_storage), "释放全部 ProbeStorage"); + Check(aclrtBinaryUnLoad(binary_handle), "卸载 TaskCell 探针 binary"); + Check(aclrtDestroyStream(stream), "销毁 TaskCell 探针 stream"); + Check(aclrtResetDevice(device), "重置 TaskCell 探针设备"); + Check(aclFinalize(), "结束 ACL"); + return result.ExitCode(); +} diff --git a/tests/atomic_probe/ccec/taskcell_atomic_dcci_shared.h b/tests/atomic_probe/ccec/taskcell_atomic_dcci_shared.h new file mode 100644 index 0000000000..2d2e72f326 --- /dev/null +++ b/tests/atomic_probe/ccec/taskcell_atomic_dcci_shared.h @@ -0,0 +1,208 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#pragma once + +#include +#include + +namespace taskcell_atomic_dcci_probe { + +#if defined(__CCE_AICORE__) +#define TASKCELL_ATOMIC_DCCI_SHARED_FN __aicore__ +#else +#define TASKCELL_ATOMIC_DCCI_SHARED_FN +#endif + +inline constexpr uint32_t kAivBlocks = 2; +inline constexpr uint32_t kCacheLineBytes = 64; +inline constexpr uint32_t kScenarioCount = 5; +inline constexpr uint32_t kDefaultTrials = 100; +inline constexpr uint32_t kMaxTrials = 1000; +inline constexpr uint32_t kObservationPolls = 4096; +// 4096 次跨核 atomic 观察在 A5 上存在长尾；等待上限只负责识别真正 +// 的协议停滞，不能比一个合法观察窗口更短。 +inline constexpr uint64_t kWaitTimeoutTicks = 200000000ULL; + +inline constexpr int64_t kInitialDeps = -1; +inline constexpr int64_t kNotApplicable = INT64_MIN; +inline constexpr int64_t kWriterMagic = 0x544344574f4e4501LL; +inline constexpr int64_t kReaderMagic = 0x544344524f4e4501LL; +inline constexpr uint64_t kInitialFlagBase = 0x1100000000000000ULL; +inline constexpr uint64_t kInitialVendBase = 0x2200000000000000ULL; +inline constexpr uint64_t kInitialTaskPaddingBase = 0x3300000000000000ULL; +inline constexpr uint64_t kBuiltFlagBase = 0x4400000000000000ULL; +inline constexpr uint64_t kBuiltVendBase = 0x5500000000000000ULL; +inline constexpr uint64_t kBuiltTaskPaddingBase = 0x6600000000000000ULL; +inline constexpr uint64_t kDedicatedGuardBase = 0x7700000000000000ULL; +inline constexpr uint64_t kStorageGuardBase = 0x7f00000000000000ULL; +inline constexpr int64_t kTokenBase = 0x0010000000000000LL; +enum class Scenario : uint32_t { + SharedAtomicThenDcci = 0, + SharedOrdinaryThenDcci = 1, + DedicatedAtomicThenDcci = 2, + DedicatedOrdinaryThenDcci = 3, + DedicatedOrdinaryNoDcci = 4, +}; + +enum ResultFlag : uint16_t { + kFlagNone = 0, + kFlagInvalidTopology = 1U << 0, + kFlagReadyTimeout = 1U << 1, + kFlagPhaseTimeout = 1U << 2, + kFlagAckTimeout = 1U << 3, + kFlagControlTransition = 1U << 4, + kFlagAtomicPublishFailed = 1U << 5, + kFlagPreSawToken = 1U << 6, + kFlagPostSawToken = 1U << 7, + kFlagPrePollTimeout = 1U << 8, + kFlagPostPollTimeout = 1U << 9, + kFlagUnexpectedPollValue = 1U << 10, + kFlagPollRegression = 1U << 11, +}; + +// 复刻 shared PA 当前 64B TaskCell 的字段位置。探针不包含生产头文件， +// 以免把待测 cache 行协议与完整 scheduler 的模板和诊断逻辑耦合。 +struct alignas(kCacheLineBytes) TaskCellLine { + volatile int64_t flag; + volatile uint64_t vend; + volatile int64_t deps_prepared; + volatile uint64_t padding_words[5]; +}; +static_assert(sizeof(TaskCellLine) == kCacheLineBytes); +static_assert(alignof(TaskCellLine) == kCacheLineBytes); +static_assert(offsetof(TaskCellLine, flag) == 0); +static_assert(offsetof(TaskCellLine, vend) == 8); +static_assert(offsetof(TaskCellLine, deps_prepared) == 16); +static_assert(offsetof(TaskCellLine, padding_words) == 24); + +// probe-only 的拆分布局：deps_prepared 独占整条 line，其余 56B 只放 +// host 初始化的只读 guard，device 不得以 ordinary load/store 接触。 +struct alignas(kCacheLineBytes) DedicatedDepsLine { + volatile int64_t deps_prepared; + volatile uint64_t guards[7]; +}; +static_assert(sizeof(DedicatedDepsLine) == kCacheLineBytes); +static_assert(alignof(DedicatedDepsLine) == kCacheLineBytes); +static_assert(offsetof(DedicatedDepsLine, deps_prepared) == 0); + +struct alignas(kCacheLineBytes) AtomicLine { + volatile int64_t value; + uint8_t padding[kCacheLineBytes - sizeof(int64_t)]; +}; +static_assert(sizeof(AtomicLine) == kCacheLineBytes); +static_assert(alignof(AtomicLine) == kCacheLineBytes); + +struct alignas(kCacheLineBytes) ResultLine { + volatile int64_t words[8]; +}; +static_assert(sizeof(ResultLine) == kCacheLineBytes); +static_assert(alignof(ResultLine) == kCacheLineBytes); + +struct alignas(kCacheLineBytes) ProbeStorage { + TaskCellLine shared_task; + DedicatedDepsLine dedicated_deps; + AtomicLine role_claim; + AtomicLine reader_ready; + AtomicLine phase; + AtomicLine reader_ack; + ResultLine writer_result; + ResultLine reader_result; + ResultLine target_snapshot; + ResultLine guard; +}; +static_assert(sizeof(ProbeStorage) == 10 * kCacheLineBytes); +static_assert(offsetof(ProbeStorage, shared_task) == 0 * kCacheLineBytes); +static_assert(offsetof(ProbeStorage, dedicated_deps) == 1 * kCacheLineBytes); +static_assert(offsetof(ProbeStorage, role_claim) == 2 * kCacheLineBytes); +static_assert(offsetof(ProbeStorage, reader_ready) == 3 * kCacheLineBytes); +static_assert(offsetof(ProbeStorage, phase) == 4 * kCacheLineBytes); +static_assert(offsetof(ProbeStorage, reader_ack) == 5 * kCacheLineBytes); +static_assert(offsetof(ProbeStorage, writer_result) == 6 * kCacheLineBytes); +static_assert(offsetof(ProbeStorage, reader_result) == 7 * kCacheLineBytes); +static_assert(offsetof(ProbeStorage, target_snapshot) == 8 * kCacheLineBytes); +static_assert(offsetof(ProbeStorage, guard) == 9 * kCacheLineBytes); + +struct KernelArgs { + uint64_t storage_pointer; + uint32_t trials; + uint32_t num_blocks; +}; +static_assert(sizeof(KernelArgs) == 16); + +TASKCELL_ATOMIC_DCCI_SHARED_FN constexpr bool IsSharedScenario( + Scenario scenario) { + return scenario == Scenario::SharedAtomicThenDcci || + scenario == Scenario::SharedOrdinaryThenDcci; +} + +TASKCELL_ATOMIC_DCCI_SHARED_FN constexpr bool IsAtomicPublishScenario( + Scenario scenario) { + return scenario == Scenario::SharedAtomicThenDcci || + scenario == Scenario::DedicatedAtomicThenDcci; +} + +TASKCELL_ATOMIC_DCCI_SHARED_FN constexpr bool UsesDcci(Scenario scenario) { + return scenario != Scenario::DedicatedOrdinaryNoDcci; +} + +TASKCELL_ATOMIC_DCCI_SHARED_FN constexpr int64_t PublishedToken( + uint32_t trial) { + return kTokenBase + static_cast(trial); +} + +TASKCELL_ATOMIC_DCCI_SHARED_FN constexpr uint64_t TrialTag( + Scenario scenario, uint32_t trial) { + return (static_cast(scenario) << 32) | trial; +} + +TASKCELL_ATOMIC_DCCI_SHARED_FN constexpr uint64_t InitialFlag( + uint32_t trial) { + return kInitialFlagBase | trial; +} + +TASKCELL_ATOMIC_DCCI_SHARED_FN constexpr uint64_t InitialVend( + uint32_t trial) { + return kInitialVendBase | trial; +} + +TASKCELL_ATOMIC_DCCI_SHARED_FN constexpr uint64_t InitialTaskPadding( + uint32_t index, uint32_t trial) { + return kInitialTaskPaddingBase | (static_cast(index) << 32) | trial; +} + +TASKCELL_ATOMIC_DCCI_SHARED_FN constexpr uint64_t BuiltFlag( + uint32_t trial) { + return kBuiltFlagBase | trial; +} + +TASKCELL_ATOMIC_DCCI_SHARED_FN constexpr uint64_t BuiltVend( + uint32_t trial) { + return kBuiltVendBase | trial; +} + +TASKCELL_ATOMIC_DCCI_SHARED_FN constexpr uint64_t BuiltTaskPadding( + uint32_t index, uint32_t trial) { + return kBuiltTaskPaddingBase | (static_cast(index) << 32) | trial; +} + +TASKCELL_ATOMIC_DCCI_SHARED_FN constexpr uint64_t DedicatedGuard( + uint32_t index, uint32_t trial) { + return kDedicatedGuardBase | (static_cast(index) << 32) | trial; +} + +TASKCELL_ATOMIC_DCCI_SHARED_FN constexpr uint64_t StorageGuard( + uint32_t index, uint32_t trial) { + return kStorageGuardBase | (static_cast(index) << 32) | trial; +} + +#undef TASKCELL_ATOMIC_DCCI_SHARED_FN + +} // namespace taskcell_atomic_dcci_probe diff --git a/tests/atomic_probe/ccec/trace_write_preload.cpp b/tests/atomic_probe/ccec/trace_write_preload.cpp new file mode 100644 index 0000000000..74fea92be7 --- /dev/null +++ b/tests/atomic_probe/ccec/trace_write_preload.cpp @@ -0,0 +1,286 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the license. + * ----------------------------------------------------------------------------------------------------------- + */ + +// This probe deliberately models the physical PA swimlane write path rather +// than a single isolated store: +// * one 32 B TraceRecord contains the same seven scalar fields; +// * two consecutive records fill one 64 B DCache line; +// * each worker owns an aligned, disjoint sequential output range; +// * the publication window uses the same per-line CACHELINE_OUT loop and one +// trailing DSB as CcecOps::FlushRegion. + +#include "../trace_write_preload_shared.h" +#include "ccec_utils.h" + +#if defined(TRACE_WRITE_PRELOAD_BUILD_AIC) +PTO_SYNCALL_MIX_AIC_KERNEL_META(trace_write_preload_0_mix_aic, 1, 2); +#elif defined(TRACE_WRITE_PRELOAD_BUILD_AIV) +PTO_SYNCALL_MIX_AIC_KERNEL_META(trace_write_preload_0_mix_aiv, 1, 2); +#else +#error "Compile with TRACE_WRITE_PRELOAD_BUILD_AIC or TRACE_WRITE_PRELOAD_BUILD_AIV" +#endif + +namespace { + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadOrderedSysCount() { + uint64_t value = 0; + asm volatile("MOV %0, SYS_CNT\n" : "=&l"(value) : : "memory"); + return value; +} + +__aicore__ __attribute__((always_inline)) inline void InvalidateLines( + __gm__ uint8_t *base, uint32_t lines +) { + for (uint32_t line = 0; line < lines; ++line) { + dcci(base + static_cast(line) * trace_write_preload::kCacheLineBytes, SINGLE_CACHE_LINE); + } + dsb(DSB_ALL); + asm volatile("" ::: "memory"); +} + +__aicore__ __attribute__((always_inline)) inline void FlushLines( + __gm__ uint8_t *base, uint32_t lines +) { + asm volatile("" ::: "memory"); + for (uint32_t line = 0; line < lines; ++line) { + dcci( + base + static_cast(line) * trace_write_preload::kCacheLineBytes, + SINGLE_CACHE_LINE, CACHELINE_OUT + ); + } + dsb(DSB_ALL); + asm volatile("" ::: "memory"); +} + +__aicore__ __attribute__((always_inline)) inline void WriteRecord( + volatile __gm__ trace_write_preload::TraceRecord *record, + uint64_t seed, uint32_t worker, uint32_t record_id +) { + const uint64_t stamp = trace_write_preload::RecordStamp(seed, worker, record_id); + record->start_cycle = stamp; + record->end_cycle = stamp + 1U; + record->task_id = static_cast(record_id); + record->function_id = static_cast(worker); + record->flags = trace_write_preload::RecordFlags(worker, record_id); + record->phase = static_cast(record_id & 0xffffU); + record->auxiliary = static_cast(worker); +} + +template +__aicore__ __attribute__((noinline)) uint64_t WriteTraceRecords( + __gm__ uint8_t *worker_base, uint32_t records, uint32_t distance_lines, + uint32_t cadence_lines, uint64_t seed, uint32_t worker +) { + const uint32_t lines = records / trace_write_preload::kRecordsPerLine; + volatile __gm__ trace_write_preload::TraceRecord *output = + reinterpret_cast(worker_base); + __gm__ uint64_t *preload_base = reinterpret_cast<__gm__ uint64_t *>(worker_base); + + for (uint32_t line = 0; line < lines; ++line) { + if constexpr (Preload) { + const uint32_t future_line = line + distance_lines; + if ((line % cadence_lines) == 0U && future_line < lines) { + dc_preload( + preload_base, + static_cast( + static_cast(future_line) * trace_write_preload::kCacheLineBytes + ) + ); + } + } + const uint32_t first_record = line * trace_write_preload::kRecordsPerLine; + WriteRecord(&output[first_record], seed, worker, first_record); + WriteRecord(&output[first_record + 1U], seed, worker, first_record + 1U); + } + return trace_write_preload::RecordStamp(seed, worker, records - 1U); +} + +__aicore__ __attribute__((always_inline)) inline void PublishResult( + __gm__ trace_write_preload::ProbeResult *result, + uint64_t begin, uint64_t split, uint64_t end, uint64_t terminal, + uint32_t worker, uint32_t experiment, uint32_t records_or_lines, + uint32_t distance, uint32_t cadence, uint32_t sample +) { + st_dev_b64(&result->phase_begin, begin); + st_dev_b64(&result->phase_split, split); + st_dev_b64(&result->phase_end, end); + st_dev_b64(&result->terminal_value, terminal); + st_dev_b32(&result->worker_id, worker); + st_dev_b32(&result->experiment, experiment); + st_dev_b32(&result->records_or_lines, records_or_lines); + st_dev_b32(&result->preload_distance_lines, distance); + st_dev_b32(&result->preload_cadence_lines, cadence); + st_dev_b32(&result->sample_id, sample); + st_dev_b32(&result->status, trace_write_preload::kStatusComplete); +} + +__aicore__ inline void RunTraceWrite( + __gm__ uint8_t *worker_base, __gm__ trace_write_preload::ProbeResult *result, + const trace_write_preload::ProbeControl &control, uint32_t worker +) { + const uint32_t records = control.records_per_worker; + const uint32_t lines = records / trace_write_preload::kRecordsPerLine; + + // Cold setup is deliberately outside both measured windows. + InvalidateLines(worker_base, lines); + + const uint64_t begin = ReadOrderedSysCount(); + uint64_t terminal = 0; + if (control.preload_distance_lines == 0U) { + terminal = WriteTraceRecords( + worker_base, records, 0U, 1U, control.seed, worker + ); + } else { + terminal = WriteTraceRecords( + worker_base, records, control.preload_distance_lines, + control.preload_cadence_lines, control.seed, worker + ); + } + const uint64_t split = ReadOrderedSysCount(); + FlushLines(worker_base, lines); + const uint64_t end = ReadOrderedSysCount(); + + PublishResult( + result, begin, split, end, terminal, worker, + static_cast(trace_write_preload::Experiment::TraceWrite), + records, control.preload_distance_lines, control.preload_cadence_lines, + control.sample_id + ); +} + +__aicore__ __attribute__((noinline)) uint32_t TraverseCapacityCycle( + volatile __gm__ uint8_t *worker_base, uint32_t start_line, + uint32_t lines, uint32_t passes +) { + uint32_t current = start_line; + const uint32_t accesses = lines * passes; + for (uint32_t access = 0; access < accesses; ++access) { + const volatile __gm__ uint32_t *next = + reinterpret_cast( + worker_base + static_cast(current) * trace_write_preload::kCacheLineBytes + ); + current = *next; + } + return current; +} + +__aicore__ inline void RunCapacitySweep( + __gm__ uint8_t *worker_base, __gm__ trace_write_preload::ProbeResult *result, + const trace_write_preload::ProbeControl &control, uint32_t worker +) { + InvalidateLines(worker_base, control.capacity_lines); + + // One dependent cold traversal fills the working set. The immediately + // following repeated traversals expose the effective resident-set knee + // without allowing a sequential hardware prefetcher to hide misses. + const uint64_t begin = ReadOrderedSysCount(); + uint32_t terminal = TraverseCapacityCycle( + worker_base, control.capacity_start_line, control.capacity_lines, 1U + ); + const uint64_t split = ReadOrderedSysCount(); + terminal ^= TraverseCapacityCycle( + worker_base, control.capacity_start_line, control.capacity_lines, + control.capacity_passes + ); + const uint64_t end = ReadOrderedSysCount(); + + PublishResult( + result, begin, split, end, terminal, worker, + static_cast(trace_write_preload::Experiment::CapacitySweep), + control.capacity_lines, 0U, 0U, control.sample_id + ); +} + +__aicore__ inline void RunParticipant( + __gm__ trace_write_preload::ProbeControl *control_pointer, + __gm__ trace_write_preload::ProbeResult *results, + __gm__ uint8_t *records, uint32_t worker +) { + dcci(control_pointer, SINGLE_CACHE_LINE); + dsb(DSB_ALL); + asm volatile("" ::: "memory"); + trace_write_preload::ProbeControl control{}; + control.magic = control_pointer->magic; + control.experiment = control_pointer->experiment; + control.first_worker = control_pointer->first_worker; + control.worker_count = control_pointer->worker_count; + control.records_per_worker = control_pointer->records_per_worker; + control.preload_distance_lines = control_pointer->preload_distance_lines; + control.preload_cadence_lines = control_pointer->preload_cadence_lines; + control.sample_id = control_pointer->sample_id; + control.capacity_lines = control_pointer->capacity_lines; + control.capacity_passes = control_pointer->capacity_passes; + control.capacity_start_line = control_pointer->capacity_start_line; + control.seed = control_pointer->seed; + control.worker_stride_bytes = control_pointer->worker_stride_bytes; + + if (control.magic != trace_write_preload::kControlMagic || + control.worker_stride_bytes != trace_write_preload::kWorkerStrideBytes || + worker < control.first_worker || + worker >= control.first_worker + control.worker_count) { + return; + } + + __gm__ uint8_t *worker_base = + records + static_cast(worker) * control.worker_stride_bytes; + __gm__ trace_write_preload::ProbeResult *result = &results[worker]; + if (control.experiment == + static_cast(trace_write_preload::Experiment::TraceWrite)) { + if (control.records_per_worker == 0U || + control.records_per_worker > trace_write_preload::kMaxRecordsPerWorker || + (control.records_per_worker % trace_write_preload::kRecordsPerLine) != 0U || + (control.preload_distance_lines != 0U && + control.preload_cadence_lines == 0U)) { + return; + } + RunTraceWrite(worker_base, result, control, worker); + return; + } + if (control.experiment == + static_cast(trace_write_preload::Experiment::CapacitySweep)) { + if (control.capacity_lines == 0U || + control.capacity_lines > + trace_write_preload::kWorkerStrideBytes / + trace_write_preload::kCacheLineBytes || + control.capacity_passes == 0U || + control.capacity_start_line >= control.capacity_lines) { + return; + } + RunCapacitySweep(worker_base, result, control, worker); + } +} + +} // namespace + +#if defined(TRACE_WRITE_PRELOAD_BUILD_AIC) +extern "C" __global__ __aicore__ void trace_write_preload_0_mix_aic( + __gm__ trace_write_preload::ProbeControl *control, + __gm__ trace_write_preload::ProbeResult *results, + __gm__ uint8_t *records +) { + RunParticipant( + control, results, records, static_cast(get_block_idx()) + ); +} +#elif defined(TRACE_WRITE_PRELOAD_BUILD_AIV) +extern "C" __global__ __aicore__ void trace_write_preload_0_mix_aiv( + __gm__ trace_write_preload::ProbeControl *control, + __gm__ trace_write_preload::ProbeResult *results, + __gm__ uint8_t *records +) { + const uint32_t vector_id = static_cast( + get_block_idx() * get_subblockdim() + get_subblockid() + ); + RunParticipant( + control, results, records, trace_write_preload::kAicWorkers + vector_id + ); +} +#endif diff --git a/tests/atomic_probe/ccec/trace_write_preload_host.cpp b/tests/atomic_probe/ccec/trace_write_preload_host.cpp new file mode 100644 index 0000000000..0a4aa4a972 --- /dev/null +++ b/tests/atomic_probe/ccec/trace_write_preload_host.cpp @@ -0,0 +1,812 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the LICENSE file for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the license. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "../probe_host.h" +#include "../trace_write_preload_shared.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct KernelArgs { + uint64_t control; + uint64_t results; + uint64_t records; +}; + +struct Policy { + const char *name; + uint32_t distance; + uint32_t cadence; +}; + +struct RunObservation { + std::vector results; + double wall_us = 0.0; +}; + +struct PolicyAggregate { + Policy policy{}; + std::vector issue_ticks; + std::vector flush_ticks; + std::vector total_ticks; + std::vector aic_total_ticks; + std::vector aiv_total_ticks; + std::vector critical_issue_ticks; + std::vector critical_total_ticks; + std::vector phase_issue_span_ticks; + std::vector phase_total_span_ticks; + std::vector start_skew_ticks; + std::vector wall_us; +}; + +struct DeviceResources { + aclrtStream stream = nullptr; + aclrtBinHandle binary = nullptr; + aclrtFuncHandle function = nullptr; + void *control = nullptr; + void *results = nullptr; + void *records = nullptr; +}; + +constexpr std::array kTunePolicies = {{ + {"baseline", 0U, 1U}, + {"d1-c1", 1U, 1U}, + {"d2-c1", 2U, 1U}, + {"d4-c1", 4U, 1U}, + {"d8-c1", 8U, 1U}, + {"d16-c1", 16U, 1U}, + {"d4-c4", 4U, 4U}, + {"d8-c4", 8U, 4U}, + {"d16-c4", 16U, 4U}, +}}; + +constexpr uint32_t kTuneSamples = 9; +constexpr uint32_t kConfirmSamples = 21; +constexpr uint32_t kFollowupSamples = 13; +constexpr uint32_t kCapacitySamples = 9; +constexpr uint32_t kPrimaryRecords = (120U * 1024U) / trace_write_preload::kRecordBytes; +constexpr uint32_t kSecondaryRecords = (96U * 1024U) / trace_write_preload::kRecordBytes; + +static_assert(sizeof(KernelArgs) == 3U * sizeof(uint64_t), "unexpected CCEC kernel argument ABI"); +static_assert(kPrimaryRecords <= trace_write_preload::kMaxRecordsPerWorker, "primary write set exceeds stride"); + +bool Check(aclError error, const char *expression) { + return atomic_probe::CheckAcl(error, expression, __FILE__, __LINE__); +} + +std::vector ReadBinary(const std::string &path) { + std::ifstream stream(path, std::ios::binary | std::ios::ate); + if (!stream) return {}; + const std::streamsize size = stream.tellg(); + if (size <= 0) return {}; + stream.seekg(0); + std::vector data(static_cast(size)); + if (!stream.read(data.data(), size)) return {}; + return data; +} + +uint64_t Quantile(std::vector values, double quantile) { + if (values.empty()) return 0U; + std::sort(values.begin(), values.end()); + const size_t index = static_cast( + std::llround(quantile * static_cast(values.size() - 1U)) + ); + return values[index]; +} + +double QuantileDouble(std::vector values, double quantile) { + if (values.empty()) return 0.0; + std::sort(values.begin(), values.end()); + const size_t index = static_cast( + std::llround(quantile * static_cast(values.size() - 1U)) + ); + return values[index]; +} + +uint64_t Median(const std::vector &values) { + return Quantile(values, 0.5); +} + +double MedianDouble(const std::vector &values) { + return QuantileDouble(values, 0.5); +} + +bool ResultIsZero(const trace_write_preload::ProbeResult &result) { + const auto *bytes = reinterpret_cast(&result); + for (size_t index = 0; index < sizeof(result); ++index) { + if (bytes[index] != 0U) return false; + } + return true; +} + +bool IsActiveWorker( + uint32_t worker, const trace_write_preload::ProbeControl &control +) { + return worker >= control.first_worker && + worker < control.first_worker + control.worker_count; +} + +bool ValidateResultSet( + const std::vector &results, + const trace_write_preload::ProbeControl &control, std::string *reason +) { + if (results.size() != trace_write_preload::kWorkers) { + *reason = "result-count"; + return false; + } + for (uint32_t worker = 0; worker < trace_write_preload::kWorkers; ++worker) { + const auto &result = results[worker]; + if (!IsActiveWorker(worker, control)) { + if (!ResultIsZero(result)) { + *reason = "inactive-worker-wrote-result"; + return false; + } + continue; + } + if (result.status != trace_write_preload::kStatusComplete || + result.worker_id != worker || + result.experiment != control.experiment || + result.sample_id != control.sample_id || + result.phase_begin == 0U || + result.phase_split <= result.phase_begin || + result.phase_end <= result.phase_split) { + *reason = "active-result-header-or-timing"; + return false; + } + if (control.experiment == + static_cast(trace_write_preload::Experiment::TraceWrite)) { + if (result.records_or_lines != control.records_per_worker || + result.preload_distance_lines != control.preload_distance_lines || + result.preload_cadence_lines != control.preload_cadence_lines || + result.terminal_value != trace_write_preload::RecordStamp( + control.seed, worker, control.records_per_worker - 1U + )) { + *reason = "trace-write-result-echo"; + return false; + } + } else { + if (result.records_or_lines != control.capacity_lines || + result.preload_distance_lines != 0U || + result.preload_cadence_lines != 0U || + result.terminal_value != 0U) { + *reason = "capacity-result-echo"; + return false; + } + } + } + return true; +} + +bool ValidateTracePayload( + void *records_device, const trace_write_preload::ProbeControl &control, + std::string *reason +) { + const size_t span_bytes = + static_cast(control.worker_count) * + trace_write_preload::kWorkerStrideBytes; + std::vector host(span_bytes); + const auto *records_bytes = reinterpret_cast(records_device); + const void *source = + records_bytes + static_cast(control.first_worker) * + trace_write_preload::kWorkerStrideBytes; + if (!Check( + aclrtMemcpy( + host.data(), host.size(), source, host.size(), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H trace payload verification)" + )) { + *reason = "payload-copy"; + return false; + } + + for (uint32_t local = 0; local < control.worker_count; ++local) { + const uint32_t worker = control.first_worker + local; + const auto *records = reinterpret_cast( + host.data() + static_cast(local) * + trace_write_preload::kWorkerStrideBytes + ); + for (uint32_t record_id = 0; + record_id < control.records_per_worker; ++record_id) { + const auto &record = records[record_id]; + const uint64_t stamp = trace_write_preload::RecordStamp( + control.seed, worker, record_id + ); + if (record.start_cycle != stamp || + record.end_cycle != stamp + 1U || + record.task_id != static_cast(record_id) || + record.function_id != static_cast(worker) || + record.flags != + trace_write_preload::RecordFlags(worker, record_id) || + record.phase != static_cast(record_id & 0xffffU) || + record.auxiliary != static_cast(worker)) { + *reason = "payload-record-" + std::to_string(worker) + "-" + + std::to_string(record_id); + return false; + } + } + } + return true; +} + +bool RunOne( + const DeviceResources &device, + const trace_write_preload::ProbeControl &control, + bool validate_payload, RunObservation *observation, std::string *reason +) { + std::array + zero_results{}; + if (!Check( + aclrtMemcpy( + device.control, sizeof(control), &control, sizeof(control), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D trace-write control)" + ) || + !Check( + aclrtMemcpy( + device.results, sizeof(zero_results), zero_results.data(), + sizeof(zero_results), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D trace-write result reset)" + )) { + *reason = "setup-copy"; + return false; + } + + KernelArgs args{ + reinterpret_cast(device.control), + reinterpret_cast(device.results), + reinterpret_cast(device.records), + }; + const auto wall_begin = std::chrono::steady_clock::now(); + if (!Check( + aclrtLaunchKernelWithHostArgs( + device.function, trace_write_preload::kAicWorkers, + device.stream, nullptr, &args, sizeof(args), nullptr, 0U + ), + "aclrtLaunchKernelWithHostArgs(trace write preload)" + ) || + !Check( + aclrtSynchronizeStream(device.stream), + "aclrtSynchronizeStream(trace write preload)" + )) { + *reason = "launch-or-sync"; + return false; + } + const auto wall_end = std::chrono::steady_clock::now(); + observation->wall_us = + std::chrono::duration(wall_end - wall_begin).count(); + observation->results.resize(trace_write_preload::kWorkers); + if (!Check( + aclrtMemcpy( + observation->results.data(), + observation->results.size() * + sizeof(trace_write_preload::ProbeResult), + device.results, + observation->results.size() * + sizeof(trace_write_preload::ProbeResult), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H trace-write results)" + )) { + *reason = "result-copy"; + return false; + } + if (!ValidateResultSet(observation->results, control, reason)) return false; + if (validate_payload && + control.experiment == + static_cast( + trace_write_preload::Experiment::TraceWrite + ) && + !ValidateTracePayload(device.records, control, reason)) { + return false; + } + return true; +} + +void AccumulateWrite( + const RunObservation &observation, + const trace_write_preload::ProbeControl &control, + PolicyAggregate *aggregate +) { + uint64_t min_begin = std::numeric_limits::max(); + uint64_t max_begin = 0U; + uint64_t max_split = 0U; + uint64_t max_end = 0U; + uint64_t max_issue = 0U; + uint64_t max_total = 0U; + for (uint32_t worker = control.first_worker; + worker < control.first_worker + control.worker_count; ++worker) { + const auto &result = observation.results[worker]; + const uint64_t issue = result.phase_split - result.phase_begin; + const uint64_t flush = result.phase_end - result.phase_split; + const uint64_t total = result.phase_end - result.phase_begin; + aggregate->issue_ticks.push_back(issue); + aggregate->flush_ticks.push_back(flush); + aggregate->total_ticks.push_back(total); + if (worker < trace_write_preload::kAicWorkers) { + aggregate->aic_total_ticks.push_back(total); + } else { + aggregate->aiv_total_ticks.push_back(total); + } + min_begin = std::min(min_begin, result.phase_begin); + max_begin = std::max(max_begin, result.phase_begin); + max_split = std::max(max_split, result.phase_split); + max_end = std::max(max_end, result.phase_end); + max_issue = std::max(max_issue, issue); + max_total = std::max(max_total, total); + } + aggregate->critical_issue_ticks.push_back(max_issue); + aggregate->critical_total_ticks.push_back(max_total); + aggregate->phase_issue_span_ticks.push_back(max_split - min_begin); + aggregate->phase_total_span_ticks.push_back(max_end - min_begin); + aggregate->start_skew_ticks.push_back(max_begin - min_begin); + aggregate->wall_us.push_back(observation.wall_us); +} + +void PrintWriteAggregate( + const char *scope, uint32_t records, uint32_t samples, + const PolicyAggregate &aggregate +) { + const uint64_t issue_median = Median(aggregate.issue_ticks); + const uint64_t flush_median = Median(aggregate.flush_ticks); + const uint64_t total_median = Median(aggregate.total_ticks); + const double issue_per_record = + static_cast(issue_median) / static_cast(records); + const double total_per_record = + static_cast(total_median) / static_cast(records); + std::printf( + "[WRITE] scope=%-10s bytes_per_worker=%6u samples=%2u " + "policy=%-8s(d=%u,c=%u) " + "core_issue=%llu[%llu,%llu] core_flush=%llu core_total=%llu " + "critical_issue=%llu critical_total=%llu phase_total=%llu " + "start_skew=%llu wall_us=%.3f issue/record=%.3f total/record=%.3f " + "aic_total=%llu aiv_total=%llu\n", + scope, records * trace_write_preload::kRecordBytes, samples, + aggregate.policy.name, aggregate.policy.distance, + aggregate.policy.cadence, + static_cast(issue_median), + static_cast( + Quantile(aggregate.issue_ticks, 0.1) + ), + static_cast( + Quantile(aggregate.issue_ticks, 0.9) + ), + static_cast(flush_median), + static_cast(total_median), + static_cast( + Median(aggregate.critical_issue_ticks) + ), + static_cast( + Median(aggregate.critical_total_ticks) + ), + static_cast( + Median(aggregate.phase_total_span_ticks) + ), + static_cast(Median(aggregate.start_skew_ticks)), + MedianDouble(aggregate.wall_us), issue_per_record, total_per_record, + static_cast(Median(aggregate.aic_total_ticks)), + static_cast(Median(aggregate.aiv_total_ticks)) + ); +} + +bool RunWriteMatrix( + const DeviceResources &device, const char *scope, + uint32_t first_worker, uint32_t worker_count, uint32_t records, + const std::vector &policies, uint32_t samples, + std::vector *aggregates +) { + aggregates->clear(); + for (const Policy &policy : policies) { + PolicyAggregate aggregate; + aggregate.policy = policy; + aggregates->push_back(aggregate); + } + + for (uint32_t sample = 0; sample < samples; ++sample) { + for (uint32_t order = 0; order < policies.size(); ++order) { + const uint32_t policy_index = + (order + sample) % static_cast(policies.size()); + const Policy &policy = policies[policy_index]; + trace_write_preload::ProbeControl control{}; + control.magic = trace_write_preload::kControlMagic; + control.experiment = static_cast( + trace_write_preload::Experiment::TraceWrite + ); + control.first_worker = first_worker; + control.worker_count = worker_count; + control.records_per_worker = records; + control.preload_distance_lines = policy.distance; + control.preload_cadence_lines = policy.cadence; + control.sample_id = sample; + control.seed = + 0x123456789abcdef0ULL ^ + (static_cast(sample) * 0x9e3779b97f4a7c15ULL); + control.worker_stride_bytes = + trace_write_preload::kWorkerStrideBytes; + + RunObservation observation; + std::string reason; + if (!RunOne( + device, control, sample == 0U, &observation, &reason + )) { + std::fprintf( + stderr, + "[MISMATCH] scope=%s sample=%u policy=%s reason=%s\n", + scope, sample, policy.name, reason.c_str() + ); + return false; + } + AccumulateWrite( + observation, control, &(*aggregates)[policy_index] + ); + } + } + for (const auto &aggregate : *aggregates) { + PrintWriteAggregate(scope, records, samples, aggregate); + } + return true; +} + +double PercentDelta(uint64_t candidate, uint64_t baseline) { + if (baseline == 0U) return 0.0; + return 100.0 * + (static_cast(candidate) - static_cast(baseline)) / + static_cast(baseline); +} + +size_t BestNonBaseline(const std::vector &aggregates) { + size_t best = 1U; + uint64_t best_value = + Median(aggregates[best].critical_total_ticks); + for (size_t index = 2U; index < aggregates.size(); ++index) { + const uint64_t candidate = + Median(aggregates[index].critical_total_ticks); + if (candidate < best_value) { + best = index; + best_value = candidate; + } + } + return best; +} + +void PrintComparison( + const char *label, const PolicyAggregate &baseline, + const PolicyAggregate &candidate +) { + const uint64_t baseline_issue = + Median(baseline.critical_issue_ticks); + const uint64_t candidate_issue = + Median(candidate.critical_issue_ticks); + const uint64_t baseline_total = + Median(baseline.critical_total_ticks); + const uint64_t candidate_total = + Median(candidate.critical_total_ticks); + std::printf( + "[DELTA] %-18s candidate=%s critical_issue=%llu->%llu (%+.3f%%) " + "critical_total=%llu->%llu (%+.3f%%)\n", + label, candidate.policy.name, + static_cast(baseline_issue), + static_cast(candidate_issue), + PercentDelta(candidate_issue, baseline_issue), + static_cast(baseline_total), + static_cast(candidate_total), + PercentDelta(candidate_total, baseline_total) + ); +} + +std::vector BuildPointerCycle( + uint32_t lines, uint32_t seed, uint32_t *start_line +) { + std::vector order(lines); + for (uint32_t line = 0; line < lines; ++line) order[line] = line; + std::mt19937 generator(seed); + std::shuffle(order.begin(), order.end(), generator); + std::vector bytes(trace_write_preload::kWorkerStrideBytes, 0U); + for (uint32_t index = 0; index < lines; ++index) { + const uint32_t line = order[index]; + const uint32_t next = order[(index + 1U) % lines]; + std::memcpy( + bytes.data() + + static_cast(line) * + trace_write_preload::kCacheLineBytes, + &next, sizeof(next) + ); + } + *start_line = order[0]; + return bytes; +} + +bool RunCapacitySweep( + const DeviceResources &device, const char *scope, uint32_t worker +) { + constexpr std::array kWorkingSetKiB = { + 4U, 8U, 12U, 16U, 20U, 24U, 32U, 48U, 64U, + }; + for (const uint32_t kib : kWorkingSetKiB) { + const uint32_t lines = + kib * 1024U / trace_write_preload::kCacheLineBytes; + uint32_t start_line = 0U; + std::vector cycle = BuildPointerCycle( + lines, 0x95000000U ^ worker ^ kib, &start_line + ); + auto *device_bytes = reinterpret_cast(device.records); + void *worker_records = + device_bytes + static_cast(worker) * + trace_write_preload::kWorkerStrideBytes; + if (!Check( + aclrtMemcpy( + worker_records, cycle.size(), cycle.data(), cycle.size(), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D capacity pointer cycle)" + )) { + return false; + } + + std::vector cold_per_load; + std::vector reuse_per_load; + for (uint32_t sample = 0; sample < kCapacitySamples; ++sample) { + trace_write_preload::ProbeControl control{}; + control.magic = trace_write_preload::kControlMagic; + control.experiment = static_cast( + trace_write_preload::Experiment::CapacitySweep + ); + control.first_worker = worker; + control.worker_count = 1U; + control.sample_id = sample; + control.capacity_lines = lines; + control.capacity_passes = trace_write_preload::kCapacityPasses; + control.capacity_start_line = start_line; + control.seed = 0x9500000000000000ULL ^ sample; + control.worker_stride_bytes = + trace_write_preload::kWorkerStrideBytes; + + RunObservation observation; + std::string reason; + if (!RunOne(device, control, false, &observation, &reason)) { + std::fprintf( + stderr, + "[MISMATCH] capacity scope=%s kib=%u sample=%u " + "reason=%s\n", + scope, kib, sample, reason.c_str() + ); + return false; + } + const auto &result = observation.results[worker]; + cold_per_load.push_back( + static_cast(result.phase_split - result.phase_begin) / + static_cast(lines) + ); + reuse_per_load.push_back( + static_cast(result.phase_end - result.phase_split) / + static_cast( + lines * trace_write_preload::kCapacityPasses + ) + ); + } + std::printf( + "[CAPACITY] scope=%-10s working_set=%2uKiB lines=%4u " + "samples=%u cold_ticks/load=%.3f " + "reuse_ticks/load=%.3f[%.3f,%.3f]\n", + scope, kib, lines, kCapacitySamples, + MedianDouble(cold_per_load), MedianDouble(reuse_per_load), + QuantileDouble(reuse_per_load, 0.1), + QuantileDouble(reuse_per_load, 0.9) + ); + } + return true; +} + +bool Cleanup(DeviceResources *device, int32_t device_id) { + bool ok = true; + if (device->records != nullptr) { + ok &= Check(aclrtFree(device->records), "aclrtFree(trace records)"); + device->records = nullptr; + } + if (device->results != nullptr) { + ok &= Check(aclrtFree(device->results), "aclrtFree(trace results)"); + device->results = nullptr; + } + if (device->control != nullptr) { + ok &= Check(aclrtFree(device->control), "aclrtFree(trace control)"); + device->control = nullptr; + } + if (device->binary != nullptr) { + ok &= Check( + aclrtBinaryUnLoad(device->binary), "aclrtBinaryUnLoad" + ); + device->binary = nullptr; + } + if (device->stream != nullptr) { + ok &= Check( + aclrtDestroyStream(device->stream), "aclrtDestroyStream" + ); + device->stream = nullptr; + } + ok &= Check(aclrtResetDevice(device_id), "aclrtResetDevice"); + ok &= Check(aclFinalize(), "aclFinalize"); + return ok; +} + +} // namespace + +int main(int argc, char **argv) { + const std::string kernel_path = + argc > 1 ? argv[1] : "./trace_write_preload_kernel.o"; + if (argc > 2) { + std::fprintf( + stderr, "Usage: %s [trace_write_preload_kernel.o]\n", argv[0] + ); + return EXIT_FAILURE; + } + const std::vector kernel_data = ReadBinary(kernel_path); + if (kernel_data.empty()) { + std::fprintf( + stderr, "Cannot read kernel binary: %s\n", kernel_path.c_str() + ); + return EXIT_FAILURE; + } + const int32_t device_id = atomic_probe::DeviceId(); + if (device_id < 0) return EXIT_FAILURE; + + DeviceResources device; + if (!Check(aclInit(nullptr), "aclInit") || + !Check(aclrtSetDevice(device_id), "aclrtSetDevice") || + !Check(aclrtCreateStream(&device.stream), "aclrtCreateStream") || + !Check( + atomic_probe::LoadAicoreBinaryFromData( + kernel_data.data(), kernel_data.size(), &device.binary + ), + "LoadAicoreBinaryFromData" + ) || + !Check( + aclrtBinaryGetFunctionByEntry( + device.binary, 0U, &device.function + ), + "aclrtBinaryGetFunctionByEntry" + ) || + !Check( + aclrtMalloc( + &device.control, sizeof(trace_write_preload::ProbeControl), + ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(trace control)" + ) || + !Check( + aclrtMalloc( + &device.results, + trace_write_preload::kWorkers * + sizeof(trace_write_preload::ProbeResult), + ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(trace results)" + ) || + !Check( + aclrtMalloc( + &device.records, + static_cast(trace_write_preload::kWorkers) * + trace_write_preload::kWorkerStrideBytes, + ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(trace records)" + )) { + Cleanup(&device, device_id); + return EXIT_FAILURE; + } + + std::printf( + "=== A5 CCEC trace-write DCache preload probe ===\n" + "kernel=%s bytes=%zu mixed_topology=32_AIC+64_AIV " + "record=32B fields=7 records_per_cacheline=2 " + "worker_stride=%uKiB\n" + "Timing values are raw SYS_CNT deltas. issue ends after the last " + "ordinary scalar store; total ends after per-line " + "DCCI(CACHELINE_OUT)+DSB.\n", + kernel_path.c_str(), kernel_data.size(), + trace_write_preload::kWorkerStrideBytes / 1024U + ); + + std::vector tune_policies( + kTunePolicies.begin(), kTunePolicies.end() + ); + std::vector tune; + bool semantic_ok = RunWriteMatrix( + device, "all96-tune", 0U, trace_write_preload::kWorkers, + kPrimaryRecords, tune_policies, kTuneSamples, &tune + ); + size_t best_index = 1U; + if (semantic_ok) { + best_index = BestNonBaseline(tune); + PrintComparison("all96-tune", tune[0], tune[best_index]); + } + + const std::vector confirm_policies = { + kTunePolicies[0], kTunePolicies[best_index], + }; + std::vector confirm; + if (semantic_ok) { + semantic_ok = RunWriteMatrix( + device, "all96-confirm", 0U, trace_write_preload::kWorkers, + kPrimaryRecords, confirm_policies, kConfirmSamples, &confirm + ); + } + if (semantic_ok) { + PrintComparison("all96-confirm", confirm[0], confirm[1]); + } + + std::vector secondary; + if (semantic_ok) { + semantic_ok = RunWriteMatrix( + device, "all96-96K", 0U, trace_write_preload::kWorkers, + kSecondaryRecords, confirm_policies, kFollowupSamples, &secondary + ); + } + if (semantic_ok) { + PrintComparison("all96-96K", secondary[0], secondary[1]); + } + + std::vector single_aic; + if (semantic_ok) { + semantic_ok = RunWriteMatrix( + device, "single-AIC", 0U, 1U, kPrimaryRecords, + confirm_policies, kFollowupSamples, &single_aic + ); + } + if (semantic_ok) { + PrintComparison("single-AIC", single_aic[0], single_aic[1]); + } + + std::vector single_aiv; + if (semantic_ok) { + semantic_ok = RunWriteMatrix( + device, "single-AIV", trace_write_preload::kAicWorkers, 1U, + kPrimaryRecords, confirm_policies, kFollowupSamples, &single_aiv + ); + } + if (semantic_ok) { + PrintComparison("single-AIV", single_aiv[0], single_aiv[1]); + } + + if (semantic_ok) { + semantic_ok = RunCapacitySweep(device, "single-AIC", 0U); + } + if (semantic_ok) { + semantic_ok = RunCapacitySweep( + device, "single-AIV", trace_write_preload::kAicWorkers + ); + } + + atomic_probe::Result result; + result.Expect( + semantic_ok, + "mixed trace writes, GM publication, topology, and pointer cycles" + ); + const bool cleanup_ok = Cleanup(&device, device_id); + result.Expect(cleanup_ok, "trace-write preload probe cleanup"); + return result.ExitCode(); +} diff --git a/tests/atomic_probe/ccec/vector_scalar_pmu.cpp b/tests/atomic_probe/ccec/vector_scalar_pmu.cpp new file mode 100644 index 0000000000..69868aa672 --- /dev/null +++ b/tests/atomic_probe/ccec/vector_scalar_pmu.cpp @@ -0,0 +1,198 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +// Single-AIV probe for the scalar-busy classification of the PA vector loop. +// The PMU owner configures the normal A5 PIPE_UTILIZATION map before launch: +// CNT0=vector(0x501), CNT2=scalar(0x001), CNT4=MTE2(0x202), +// CNT5=MTE3(0x203), CNT6=I-cache request(0x034), CNT7=I-cache miss(0x035). +// All counters are read-to-clear. Each kernel therefore clears the ten custom +// counters and total, opens one gate around exactly one selected workload, and +// publishes results only after metrics_prof_stop() has closed the gate. + +#include +#include +#include +#include + +#include "vector_scalar_pmu_shared.h" +// ccec_utils defines legacy sync flag macros, so include it only after PTO's +// same-named constexpr declarations have been parsed. +#include "ccec_utils.h" + +CCEC_PROBE_KERNEL_META(vector_scalar_pmu); + +namespace { + +using namespace pto; + +constexpr uint32_t kPmuPhysicalSubcores = 108U; +constexpr uint64_t kPmuCounterBlockOffset = 0x4200ULL; +constexpr uint64_t kPmuSelectorBlockOffset = 0x2400ULL; + +__aicore__ __attribute__((always_inline)) inline int32_t *PmuCounterBase(uint64_t register_base) { + return reinterpret_cast(register_base + kPmuCounterBlockOffset); +} + +__aicore__ __attribute__((always_inline)) inline int32_t *PmuSelectorBase(uint64_t register_base) { + return reinterpret_cast(register_base + kPmuSelectorBlockOffset); +} + +__aicore__ __attribute__((always_inline)) inline void ClearPmuCounters(uint64_t register_base) { + int32_t *base = PmuCounterBase(register_base); + (void)ld_dev(base, 0x10); + (void)ld_dev(base, 0x18); + (void)ld_dev(base, 0x20); + (void)ld_dev(base, 0x28); + (void)ld_dev(base, 0x30); + (void)ld_dev(base, 0x38); + (void)ld_dev(base, 0x40); + (void)ld_dev(base, 0x48); + (void)ld_dev(base, 0x50); + (void)ld_dev(base, 0x54); + (void)ld_dev(base, 0x60); + (void)ld_dev(base, 0x64); +} + +template +__aicore__ __attribute__((always_inline)) inline uint64_t ReadCounter(uint64_t register_base) { + return static_cast(ld_dev(PmuCounterBase(register_base), Offset)); +} + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadPmuTotal(uint64_t register_base) { + int32_t *base = PmuCounterBase(register_base); + const uint64_t low = static_cast(ld_dev(base, 0x60)); + const uint64_t high = static_cast(ld_dev(base, 0x64)); + return low | (high << 32U); +} + +__aicore__ __attribute__((always_inline)) inline uint64_t ReadSelectorStatus(uint64_t register_base) { + int32_t *base = PmuSelectorBase(register_base); + uint64_t status = 0U; + status |= static_cast(ld_dev(base, 0x100)) == 0x501U ? vector_scalar_pmu::kSelectorVector : 0U; + status |= static_cast(ld_dev(base, 0x108)) == 0x001U ? vector_scalar_pmu::kSelectorScalar : 0U; + status |= static_cast(ld_dev(base, 0x110)) == 0x202U ? vector_scalar_pmu::kSelectorMte2 : 0U; + status |= static_cast(ld_dev(base, 0x114)) == 0x203U ? vector_scalar_pmu::kSelectorMte3 : 0U; + status |= static_cast(ld_dev(base, 0x118)) == 0x034U ? vector_scalar_pmu::kSelectorIcacheRequest : 0U; + status |= static_cast(ld_dev(base, 0x11c)) == 0x035U ? vector_scalar_pmu::kSelectorIcacheMiss : 0U; + return status; +} + +__aicore__ __attribute__((always_inline)) inline void Publish64(__gm__ uint64_t *address, uint64_t value) { + __builtin_cce_st_dev(value, address, 0); +} + +} // namespace + +extern "C" __global__ __aicore__ void KERNEL_ENTRY(vector_scalar_pmu)(__gm__ vector_scalar_pmu::ProbeState *state) { + using namespace pto; + using vector_scalar_pmu::Mode; + + // Runtime profiling may leave the gate enabled before entry. Close it + // first so control invalidation, object setup and counter clearing stay + // outside the measured window. + bisheng::cce::metrics_prof_stop(); + dcci(&state->control, SINGLE_CACHE_LINE); + dsb(DSB_ALL); + + const uint32_t mode = state->control.mode; + const uint32_t rounds = state->control.rounds; + const uint32_t physical_core_id = static_cast(get_coreid()) & 0x0fffU; + + uint64_t register_base = 0U; + if (state->control.pmu_register_bases != 0U && physical_core_id < kPmuPhysicalSubcores) { + __gm__ uint64_t *register_bases = reinterpret_cast<__gm__ uint64_t *>(state->control.pmu_register_bases); + register_base = register_bases[physical_core_id]; + } + + constexpr int kRows = static_cast(vector_scalar_pmu::kTileRows); + constexpr int kCols = static_cast(vector_scalar_pmu::kTileCols); + using GlobalData = GlobalTensor, pto::Stride<1, 1, 1, kCols, 1>>; + using TileData = Tile; + + GlobalData input_a_global(reinterpret_cast<__gm__ float *>(state->control.input_a)); + GlobalData input_b_global(reinterpret_cast<__gm__ float *>(state->control.input_b)); + GlobalData output_global(reinterpret_cast<__gm__ float *>(state->control.output)); + TileData input_a_tile(kRows, kCols); + TileData input_b_tile(kRows, kCols); + TileData output_tile(kRows, kCols); + TASSIGN(input_a_tile, 0x0); + TASSIGN(input_b_tile, 0x10000); + TASSIGN(output_tile, 0x20000); + + uint64_t selector_status = 0U; + if (register_base != 0U) { + selector_status = ReadSelectorStatus(register_base); + ClearPmuCounters(register_base); + } + + const uint64_t sys_begin = static_cast(get_sys_cnt()); + bisheng::cce::metrics_prof_start(); + + if (mode == static_cast(Mode::LoopControl)) { + for (uint32_t iteration = 0U; iteration < rounds; ++iteration) { + // Preserve one runtime loop iteration without issuing work to V, + // MTE2 or MTE3. The NOP prevents the loop from being deleted. + asm volatile("nop"); + } + } else if (mode == static_cast(Mode::VectorAdd)) { + // Keep this body mechanically identical to RunRealVectorWorkload + // in pa_scheduler/ccec/ccec_ops.h. + for (uint32_t iteration = 0U; iteration < rounds; ++iteration) { + TLOAD(input_a_tile, input_a_global); + TLOAD(input_b_tile, input_b_global); + set_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + wait_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + TADD(output_tile, input_a_tile, input_b_tile); + set_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + wait_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + TSTORE(output_global, output_tile); + set_flag(PIPE_MTE3, PIPE_S, EVENT_ID7); + wait_flag(PIPE_MTE3, PIPE_S, EVENT_ID7); + } + } + + bisheng::cce::metrics_prof_stop(); + const uint64_t sys_end = static_cast(get_sys_cnt()); + const uint64_t ctrl_after_stop = static_cast(get_ctrl()); + + uint64_t total = 0U; + uint64_t vector_busy = 0U; + uint64_t scalar_busy = 0U; + uint64_t mte2_busy = 0U; + uint64_t mte3_busy = 0U; + uint64_t icache_request = 0U; + uint64_t icache_miss = 0U; + if (register_base != 0U) { + vector_busy = ReadCounter<0x10>(register_base); + scalar_busy = ReadCounter<0x20>(register_base); + mte2_busy = ReadCounter<0x30>(register_base); + mte3_busy = ReadCounter<0x38>(register_base); + icache_request = ReadCounter<0x40>(register_base); + icache_miss = ReadCounter<0x48>(register_base); + total = ReadPmuTotal(register_base); + } + + __gm__ vector_scalar_pmu::ProbeResult *result = &state->result; + Publish64(&result->sys_ticks, sys_end - sys_begin); + Publish64(&result->pmu_total_cycles, total); + Publish64(&result->pmu_vector_busy, vector_busy); + Publish64(&result->pmu_scalar_busy, scalar_busy); + Publish64(&result->pmu_mte2_busy, mte2_busy); + Publish64(&result->pmu_mte3_busy, mte3_busy); + Publish64(&result->pmu_icache_request, icache_request); + Publish64(&result->pmu_icache_miss, icache_miss); + Publish64(&result->physical_core_id, physical_core_id); + Publish64(&result->pmu_ctrl_after_stop, ctrl_after_stop); + Publish64(&result->selector_status, selector_status); + Publish64(&result->observed_mode, mode); + Publish64(&result->observed_rounds, rounds); + dsb(DSB_ALL); +} diff --git a/tests/atomic_probe/ccec/vector_scalar_pmu_host.cpp b/tests/atomic_probe/ccec/vector_scalar_pmu_host.cpp new file mode 100644 index 0000000000..aec7f0c88c --- /dev/null +++ b/tests/atomic_probe/ccec/vector_scalar_pmu_host.cpp @@ -0,0 +1,439 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "vector_scalar_pmu_shared.h" +#include "pmu_probe_host_support.h" +#include "../probe_host.h" +#include "../pa_scheduler/ccec/pmu_owner_host.h" + +#include "acl/acl.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using atomic_probe::pmu::CheckAcl; + +constexpr double kAicoreCyclesPerNanosecond = 1.65; +constexpr uint32_t kMeasuredRepeats = 5U; +constexpr std::array kRoundValues = {16U, 128U}; + +const char *ModeName(vector_scalar_pmu::Mode mode) { + switch (mode) { + case vector_scalar_pmu::Mode::Empty: + return "EMPTY"; + case vector_scalar_pmu::Mode::LoopControl: + return "LOOP_CONTROL"; + case vector_scalar_pmu::Mode::VectorAdd: + return "VECTOR_ADD"; + default: + return "UNKNOWN"; + } +} + +struct Sample { + vector_scalar_pmu::ProbeResult result{}; + bool output_ok = false; +}; + +bool ValidateSample( + const Sample &sample, vector_scalar_pmu::Mode mode, uint32_t rounds, + const pa_scheduler::pmu_owner::PmuOwnerControl &owner, std::string *reason +) { + const auto &result = sample.result; + if (result.observed_mode != static_cast(mode) || result.observed_rounds != rounds) { + *reason = "stale-or-mismatched-control"; + return false; + } + if (result.physical_core_id >= pa_scheduler::pmu_owner::kPhysicalSubcoreCount || + !pa_scheduler::pmu_owner::IsConfigured(owner, static_cast(result.physical_core_id))) { + *reason = "physical-core-not-owned"; + return false; + } + if (result.selector_status != vector_scalar_pmu::kRequiredSelectorStatus) { + *reason = "selector-map"; + return false; + } + if ((result.pmu_ctrl_after_stop & 1ULL) != 0U) { + *reason = "pmu-gate-still-enabled"; + return false; + } + if (result.sys_ticks == 0U || result.pmu_total_cycles == 0U) { + *reason = "zero-cycle-window"; + return false; + } + if (result.pmu_scalar_busy > result.pmu_total_cycles || result.pmu_vector_busy > result.pmu_total_cycles || + result.pmu_mte2_busy > result.pmu_total_cycles || result.pmu_mte3_busy > result.pmu_total_cycles) { + *reason = "busy-counter-exceeds-total"; + return false; + } + if (result.pmu_icache_miss > result.pmu_icache_request) { + *reason = "icache-miss-exceeds-request"; + return false; + } + if (mode == vector_scalar_pmu::Mode::VectorAdd && (result.pmu_vector_busy == 0U || result.pmu_scalar_busy == 0U || + result.pmu_mte2_busy == 0U || result.pmu_mte3_busy == 0U)) { + *reason = "vector-pipeline-counter-zero"; + return false; + } + if (mode == vector_scalar_pmu::Mode::VectorAdd && !sample.output_ok) { + *reason = "vector-output"; + return false; + } + return true; +} + +bool RunOne( + aclrtFuncHandle function, aclrtStream stream, void *state_device, void *input_a_device, void *input_b_device, + void *output_device, uint64_t pmu_register_bases, vector_scalar_pmu::Mode mode, uint32_t rounds, uint32_t repeat, + const pa_scheduler::pmu_owner::PmuOwnerControl &owner, Sample *sample, bool print_raw +) { + vector_scalar_pmu::ProbeState state{}; + state.control.pmu_register_bases = pmu_register_bases; + state.control.input_a = reinterpret_cast(input_a_device); + state.control.input_b = reinterpret_cast(input_b_device); + state.control.output = reinterpret_cast(output_device); + state.control.mode = static_cast(mode); + state.control.rounds = rounds; + if (!CheckAcl( + aclrtMemcpy(state_device, sizeof(state), &state, sizeof(state), ACL_MEMCPY_HOST_TO_DEVICE), + "aclrtMemcpy(H2D vector PMU state)" + )) { + return false; + } + if (mode == vector_scalar_pmu::Mode::VectorAdd && + !CheckAcl( + aclrtMemset(output_device, vector_scalar_pmu::kTileBytes, 0xff, vector_scalar_pmu::kTileBytes), + "aclrtMemset(vector output sentinel)" + )) { + return false; + } + + struct KernelArgs { + uint64_t state_pointer; + } args{reinterpret_cast(state_device)}; + static_assert(sizeof(KernelArgs) == sizeof(uint64_t), "unexpected CCEC kernel argument ABI"); + if (!CheckAcl( + aclrtLaunchKernelWithHostArgs(function, 1U, stream, nullptr, &args, sizeof(args), nullptr, 0U), + "aclrtLaunchKernelWithHostArgs(vector scalar PMU)" + ) || + !CheckAcl(aclrtSynchronizeStream(stream), "aclrtSynchronizeStream(vector scalar PMU)") || + !CheckAcl( + aclrtMemcpy(&state, sizeof(state), state_device, sizeof(state), ACL_MEMCPY_DEVICE_TO_HOST), + "aclrtMemcpy(D2H vector PMU state)" + )) { + return false; + } + + sample->result = state.result; + sample->output_ok = mode != vector_scalar_pmu::Mode::VectorAdd; + if (mode == vector_scalar_pmu::Mode::VectorAdd) { + std::vector output(vector_scalar_pmu::kTileElements); + if (!CheckAcl( + aclrtMemcpy( + output.data(), vector_scalar_pmu::kTileBytes, output_device, vector_scalar_pmu::kTileBytes, + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H vector output)" + )) { + return false; + } + sample->output_ok = std::all_of(output.begin(), output.end(), [](float value) { + return value == 5.0F; + }); + } + + std::string reason; + const bool passed = ValidateSample(*sample, mode, rounds, owner, &reason); + if (print_raw) { + const uint64_t residual = sample->result.pmu_total_cycles - sample->result.pmu_scalar_busy; + const double scalar_ratio = + static_cast(sample->result.pmu_scalar_busy) / static_cast(sample->result.pmu_total_cycles); + std::printf( + "[RAW] repeat=%u rounds=%u mode=%s sys_ticks=%llu total=%llu scalar=%llu " + "non_scalar_residual=%llu scalar_ratio=%.9f vector=%llu mte2=%llu mte3=%llu " + "icache_req=%llu icache_miss=%llu physical=%llu selectors=0x%llx output=%s status=%s%s%s\n", + repeat, rounds, ModeName(mode), static_cast(sample->result.sys_ticks), + static_cast(sample->result.pmu_total_cycles), + static_cast(sample->result.pmu_scalar_busy), static_cast(residual), + scalar_ratio, static_cast(sample->result.pmu_vector_busy), + static_cast(sample->result.pmu_mte2_busy), + static_cast(sample->result.pmu_mte3_busy), + static_cast(sample->result.pmu_icache_request), + static_cast(sample->result.pmu_icache_miss), + static_cast(sample->result.physical_core_id), + static_cast(sample->result.selector_status), sample->output_ok ? "PASS" : "FAIL", + passed ? "PASS" : "FAIL", passed ? "" : " reason=", passed ? "" : reason.c_str() + ); + } + return passed; +} + +uint64_t Median(std::vector values) { + std::sort(values.begin(), values.end()); + const size_t middle = values.size() / 2U; + if ((values.size() & 1U) != 0U) return values[middle]; + return values[middle - 1U] + (values[middle] - values[middle - 1U]) / 2U; +} + +double Median(std::vector values) { + std::sort(values.begin(), values.end()); + const size_t middle = values.size() / 2U; + return (values.size() & 1U) != 0U ? values[middle] : (values[middle - 1U] + values[middle]) / 2.0; +} + +using CounterMember = uint64_t vector_scalar_pmu::ProbeResult::*; + +uint64_t MedianCounter(const std::vector &samples, CounterMember member) { + std::vector values; + values.reserve(samples.size()); + for (const Sample &sample : samples) + values.push_back(sample.result.*member); + return Median(std::move(values)); +} + +double PairedDeltaPerIteration( + const std::vector &minuend, const std::vector &subtrahend, CounterMember member, uint32_t rounds +) { + std::vector deltas; + deltas.reserve(minuend.size()); + for (size_t index = 0U; index < minuend.size(); ++index) { + deltas.push_back( + (static_cast(minuend[index].result.*member) - static_cast(subtrahend[index].result.*member) + ) / + static_cast(rounds) + ); + } + return Median(std::move(deltas)); +} + +void PrintRoundSummary(uint32_t rounds, const std::array, 3> &samples) { + struct Metric { + const char *name; + CounterMember member; + }; + constexpr Metric metrics[] = { + {"sys_ticks", &vector_scalar_pmu::ProbeResult::sys_ticks}, + {"total", &vector_scalar_pmu::ProbeResult::pmu_total_cycles}, + {"scalar", &vector_scalar_pmu::ProbeResult::pmu_scalar_busy}, + {"vector", &vector_scalar_pmu::ProbeResult::pmu_vector_busy}, + {"mte2", &vector_scalar_pmu::ProbeResult::pmu_mte2_busy}, + {"mte3", &vector_scalar_pmu::ProbeResult::pmu_mte3_busy}, + {"icache_req", &vector_scalar_pmu::ProbeResult::pmu_icache_request}, + {"icache_miss", &vector_scalar_pmu::ProbeResult::pmu_icache_miss}, + }; + + for (uint32_t mode_index = 0U; mode_index < static_cast(vector_scalar_pmu::Mode::Count); ++mode_index) { + const auto mode = static_cast(mode_index); + std::printf("[MEDIAN] rounds=%u mode=%s", rounds, ModeName(mode)); + for (const Metric &metric : metrics) { + std::printf( + " %s=%llu", metric.name, + static_cast(MedianCounter(samples[mode_index], metric.member)) + ); + } + std::printf("\n"); + } + + const auto &loop = samples[static_cast(vector_scalar_pmu::Mode::LoopControl)]; + const auto &vector = samples[static_cast(vector_scalar_pmu::Mode::VectorAdd)]; + for (const Metric &metric : metrics) { + std::printf( + "[VECTOR_MINUS_LOOP_PER_ITER] rounds=%u metric=%s value=%.6f\n", rounds, metric.name, + PairedDeltaPerIteration(vector, loop, metric.member, rounds) + ); + } + + std::vector scalar_ratios; + std::vector residual_ratios; + scalar_ratios.reserve(vector.size()); + residual_ratios.reserve(vector.size()); + for (const Sample &sample : vector) { + const double total = static_cast(sample.result.pmu_total_cycles); + const double scalar = static_cast(sample.result.pmu_scalar_busy); + scalar_ratios.push_back(scalar / total); + residual_ratios.push_back((total - scalar) / total); + } + const uint64_t median_total = MedianCounter(vector, &vector_scalar_pmu::ProbeResult::pmu_total_cycles); + const uint64_t median_scalar = MedianCounter(vector, &vector_scalar_pmu::ProbeResult::pmu_scalar_busy); + const double scalar_ratio = Median(std::move(scalar_ratios)); + const double residual_ratio = Median(std::move(residual_ratios)); + std::printf( + "[VECTOR_CLASSIFICATION] rounds=%u median_total=%llu median_total_ns_at_1p65ghz=%.3f " + "median_scalar=%llu median_non_scalar_residual=%llu scalar_share=%.9f residual_share=%.9f " + "dominant=%s\n", + rounds, static_cast(median_total), + static_cast(median_total) / kAicoreCyclesPerNanosecond, static_cast(median_scalar), + static_cast(median_total - median_scalar), scalar_ratio, residual_ratio, + scalar_ratio > residual_ratio ? "scalar_busy" : "non_scalar_residual" + ); +} + +} // namespace + +int main(int argc, char **argv) { + const std::string kernel_path = argc > 1 ? argv[1] : "./vector_scalar_pmu_kernel.o"; + if (argc > 2) { + std::fprintf(stderr, "Usage: %s [vector_scalar_pmu_kernel.o]\n", argv[0]); + return EXIT_FAILURE; + } + + const int32_t device = atomic_probe::DeviceId(); + if (device < 0) return EXIT_FAILURE; + const std::vector kernel_data = atomic_probe::pmu::ReadBinary(kernel_path); + if (kernel_data.empty()) { + std::fprintf(stderr, "Cannot read kernel binary: %s\n", kernel_path.c_str()); + return EXIT_FAILURE; + } + + if (!CheckAcl(aclInit(nullptr), "aclInit") || !CheckAcl(aclrtSetDevice(device), "aclrtSetDevice")) { + return EXIT_FAILURE; + } + aclrtStream stream = nullptr; + if (!CheckAcl(aclrtCreateStream(&stream), "aclrtCreateStream")) return EXIT_FAILURE; + + aclrtBinHandle binary_handle = nullptr; + if (!CheckAcl( + atomic_probe::LoadAicoreBinaryFromData(kernel_data.data(), kernel_data.size(), &binary_handle), + "LoadAicoreBinaryFromData" + )) { + return EXIT_FAILURE; + } + aclrtFuncHandle function = nullptr; + if (!CheckAcl(aclrtBinaryGetFunctionByEntry(binary_handle, 0U, &function), "aclrtBinaryGetFunctionByEntry")) { + return EXIT_FAILURE; + } + + void *state_device = nullptr; + void *input_a_device = nullptr; + void *input_b_device = nullptr; + void *output_device = nullptr; + if (!CheckAcl( + aclrtMalloc(&state_device, sizeof(vector_scalar_pmu::ProbeState), ACL_MEM_MALLOC_NORMAL_ONLY), + "aclrtMalloc(vector PMU state)" + ) || + !CheckAcl( + aclrtMalloc(&input_a_device, vector_scalar_pmu::kTileBytes, ACL_MEM_MALLOC_NORMAL_ONLY), + "aclrtMalloc(vector input A)" + ) || + !CheckAcl( + aclrtMalloc(&input_b_device, vector_scalar_pmu::kTileBytes, ACL_MEM_MALLOC_NORMAL_ONLY), + "aclrtMalloc(vector input B)" + ) || + !CheckAcl( + aclrtMalloc(&output_device, vector_scalar_pmu::kTileBytes, ACL_MEM_MALLOC_NORMAL_ONLY), + "aclrtMalloc(vector output)" + )) { + return EXIT_FAILURE; + } + + const std::vector input_a(vector_scalar_pmu::kTileElements, 2.0F); + const std::vector input_b(vector_scalar_pmu::kTileElements, 3.0F); + if (!CheckAcl( + aclrtMemcpy( + input_a_device, vector_scalar_pmu::kTileBytes, input_a.data(), vector_scalar_pmu::kTileBytes, + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D vector input A)" + ) || + !CheckAcl( + aclrtMemcpy( + input_b_device, vector_scalar_pmu::kTileBytes, input_b.data(), vector_scalar_pmu::kTileBytes, + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D vector input B)" + )) { + return EXIT_FAILURE; + } + + atomic_probe::pmu::RegisterMappings mappings; + if (!mappings.Initialize(static_cast(device))) return EXIT_FAILURE; + const auto &mapped_array = mappings.RegisterBases(); + const std::vector mapped_bases(mapped_array.begin(), mapped_array.end()); + + pa_scheduler::pmu_owner::PmuOwnerSession owner; + const std::string dispatcher_path = + atomic_probe::pmu::ArtifactBesideKernel(kernel_path, "libvector_scalar_pmu_owner_dispatcher.so"); + const std::string owner_path = + atomic_probe::pmu::ArtifactBesideKernel(kernel_path, "libvector_scalar_pmu_owner_aicpu.so"); + if (!owner.Initialize(static_cast(device), stream, dispatcher_path, owner_path, mapped_bases) || + !owner.Configure()) { + std::fprintf(stderr, "Cannot establish the ten-slot PMU owner session.\n"); + return EXIT_FAILURE; + } + + std::printf( + "=== Single-AIV PA vector-loop scalar-busy PMU probe ===\n" + "device=%d repeats=%u tile=128x128-f32 bytes_per_iteration=196608 " + "aicore_hz=1.65GHz sys_counter_hz=1GHz\n" + "events=CNT0:vector(0x501),CNT2:scalar(0x001),CNT4:MTE2(0x202)," + "CNT5:MTE3(0x203),CNT6:icache_req(0x034),CNT7:icache_miss(0x035)\n" + "note=pipe busy counters may overlap; only total-scalar is reported as the non-scalar residual\n", + device, kMeasuredRepeats + ); + + bool all_passed = true; + // One unreported warm-up per mode removes first-launch/runtime setup from + // the measured samples. It remains outside every reported paired repeat. + for (const uint32_t rounds : kRoundValues) { + for (uint32_t mode_index = 0U; mode_index < static_cast(vector_scalar_pmu::Mode::Count); + ++mode_index) { + Sample warmup; + all_passed &= RunOne( + function, stream, state_device, input_a_device, input_b_device, output_device, + owner.RegisterTableDeviceAddress(), static_cast(mode_index), rounds, 0U, + owner.Control(), &warmup, false + ); + } + if (!all_passed) break; + + std::array, 3> samples; + for (uint32_t repeat = 1U; repeat <= kMeasuredRepeats && all_passed; ++repeat) { + for (uint32_t mode_index = 0U; mode_index < static_cast(vector_scalar_pmu::Mode::Count); + ++mode_index) { + Sample sample; + const bool passed = RunOne( + function, stream, state_device, input_a_device, input_b_device, output_device, + owner.RegisterTableDeviceAddress(), static_cast(mode_index), rounds, + repeat, owner.Control(), &sample, true + ); + all_passed &= passed; + samples[mode_index].push_back(sample); + if (!passed) break; + } + } + if (all_passed) PrintRoundSummary(rounds, samples); + } + + const bool owner_cleanup_ok = owner.Finalize(); + mappings.Release(); + bool cleanup_ok = owner_cleanup_ok; + cleanup_ok &= CheckAcl(aclrtFree(output_device), "aclrtFree(vector output)"); + cleanup_ok &= CheckAcl(aclrtFree(input_b_device), "aclrtFree(vector input B)"); + cleanup_ok &= CheckAcl(aclrtFree(input_a_device), "aclrtFree(vector input A)"); + cleanup_ok &= CheckAcl(aclrtFree(state_device), "aclrtFree(vector PMU state)"); + cleanup_ok &= CheckAcl(aclrtBinaryUnLoad(binary_handle), "aclrtBinaryUnLoad"); + cleanup_ok &= CheckAcl(aclrtDestroyStream(stream), "aclrtDestroyStream"); + cleanup_ok &= CheckAcl(aclrtResetDevice(device), "aclrtResetDevice"); + cleanup_ok &= CheckAcl(aclFinalize(), "aclFinalize"); + std::printf( + "[SUMMARY] semantic_status=%s pmu_restore_and_cleanup=%s\n", all_passed ? "PASS" : "FAIL", + cleanup_ok ? "PASS" : "FAIL" + ); + return all_passed && cleanup_ok ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/atomic_probe/ccec/vector_scalar_pmu_shared.h b/tests/atomic_probe/ccec/vector_scalar_pmu_shared.h new file mode 100644 index 0000000000..5051add576 --- /dev/null +++ b/tests/atomic_probe/ccec/vector_scalar_pmu_shared.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef TESTS_ATOMIC_PROBE_CCEC_VECTOR_SCALAR_PMU_SHARED_H_ +#define TESTS_ATOMIC_PROBE_CCEC_VECTOR_SCALAR_PMU_SHARED_H_ + +#include +#include + +namespace vector_scalar_pmu { + +constexpr uint32_t kTileRows = 128U; +constexpr uint32_t kTileCols = 128U; +constexpr uint32_t kTileElements = kTileRows * kTileCols; +constexpr uint32_t kTileBytes = kTileElements * sizeof(float); + +// EMPTY measures only the PMU gate. LOOP_CONTROL keeps the same runtime loop +// shape but replaces the pipeline body with one scalar NOP. VECTOR_ADD is the +// exact TLOAD -> MTE2/V -> TADD -> V/MTE3 -> TSTORE -> MTE3/S sequence used by +// the pa_scheduler AIV real workload. +enum class Mode : uint32_t { + Empty = 0U, + LoopControl = 1U, + VectorAdd = 2U, + Count = 3U, +}; + +struct alignas(64) ProbeControl { + uint64_t pmu_register_bases; + uint64_t input_a; + uint64_t input_b; + uint64_t output; + uint32_t mode; + uint32_t rounds; + uint64_t reserved[3]; +}; + +// Every field is widened to 64 bits in the shared ABI even though the custom +// PMU counters are 32 bits. This keeps host formatting unambiguous and gives +// the result two complete cache lines that are published only after PMU stop. +struct alignas(64) ProbeResult { + uint64_t sys_ticks; + uint64_t pmu_total_cycles; + uint64_t pmu_vector_busy; + uint64_t pmu_scalar_busy; + uint64_t pmu_mte2_busy; + uint64_t pmu_mte3_busy; + uint64_t pmu_icache_request; + uint64_t pmu_icache_miss; + + uint64_t physical_core_id; + uint64_t pmu_ctrl_after_stop; + uint64_t selector_status; + uint64_t observed_mode; + uint64_t observed_rounds; + uint64_t reserved[3]; +}; + +struct alignas(64) ProbeState { + ProbeControl control; + ProbeResult result; +}; + +constexpr uint64_t kSelectorVector = 1ULL << 0; +constexpr uint64_t kSelectorScalar = 1ULL << 1; +constexpr uint64_t kSelectorMte2 = 1ULL << 2; +constexpr uint64_t kSelectorMte3 = 1ULL << 3; +constexpr uint64_t kSelectorIcacheRequest = 1ULL << 4; +constexpr uint64_t kSelectorIcacheMiss = 1ULL << 5; +constexpr uint64_t kRequiredSelectorStatus = + kSelectorVector | kSelectorScalar | kSelectorMte2 | kSelectorMte3 | kSelectorIcacheRequest | kSelectorIcacheMiss; + +static_assert(kTileBytes == 65536U, "the PA vector tile must remain 64 KiB"); +static_assert(sizeof(ProbeControl) == 64U, "probe control must occupy one cache line"); +static_assert(sizeof(ProbeResult) == 128U, "probe result must occupy two cache lines"); +static_assert(offsetof(ProbeState, result) == 64U, "probe result offset changed"); +static_assert(sizeof(ProbeState) == 192U, "probe state ABI changed"); + +} // namespace vector_scalar_pmu + +#endif // TESTS_ATOMIC_PROBE_CCEC_VECTOR_SCALAR_PMU_SHARED_H_ diff --git a/tests/atomic_probe/cpu/nested_lambda_args_runtime_read.cpp b/tests/atomic_probe/cpu/nested_lambda_args_runtime_read.cpp index 83213282f4..3547ed71b2 100644 --- a/tests/atomic_probe/cpu/nested_lambda_args_runtime_read.cpp +++ b/tests/atomic_probe/cpu/nested_lambda_args_runtime_read.cpp @@ -16,18 +16,16 @@ namespace { -void InitTensor(Tensor &tensor, uint32_t round, uint32_t tensor_index) -{ +void InitTensor(Tensor &tensor, uint32_t round, uint32_t tensor_index) { tensor.buffer.addr = nested_lambda_cross_tu_probe::TensorAddress(round, tensor_index); tensor.start_offset = nested_lambda_cross_tu_probe::TensorOffset(round, tensor_index); tensor.version = nested_lambda_cross_tu_probe::TensorVersion(round, tensor_index); tensor.shapes[0] = nested_lambda_cross_tu_probe::TensorShape(round, tensor_index); } -} // namespace +} // namespace -int main() -{ +int main() { L0TaskArgs args; uint32_t completed_rounds = 0; uint32_t mismatches = 0; @@ -68,16 +66,15 @@ int main() completed_rounds++; } - const bool exact = completed_rounds == nested_lambda_cross_tu_probe::kRounds && - mismatches == 0 && checksum == nested_lambda_cross_tu_probe::ExpectedTotalChecksum() && - sizeof(L0TaskArgs) > 0 && sizeof(L0TaskArgs) < 32U * 1024U && sizeof(L0TaskArgs) % 64U == 0; + const bool exact = completed_rounds == nested_lambda_cross_tu_probe::kRounds && mismatches == 0 && + checksum == nested_lambda_cross_tu_probe::ExpectedTotalChecksum() && + sizeof(L0TaskArgs) == nested_lambda_cross_tu_probe::kExpectedL0TaskArgsBytes; std::printf("=== CPU L0TaskArgs Runtime-Read Probe ===\n"); std::printf( - "[VALUES] rounds=%u mismatches=%u checksum=0x%016llx L0TaskArgs=%zuB\n", - completed_rounds, mismatches, static_cast(checksum), sizeof(L0TaskArgs)); - std::printf( - "[ASSERT] CPU L0TaskArgs args-runtime-read semantics %s\n", - exact ? "PASS" : "FAIL"); + "[VALUES] rounds=%u mismatches=%u checksum=0x%016llx L0TaskArgs=%zuB\n", completed_rounds, mismatches, + static_cast(checksum), sizeof(L0TaskArgs) + ); + std::printf("[ASSERT] CPU L0TaskArgs args-runtime-read semantics %s\n", exact ? "PASS" : "FAIL"); std::printf("[SUMMARY] semantic_failures=%u\n", exact ? 0U : 1U); return exact ? 0 : 1; } diff --git a/tests/atomic_probe/icache_miss_usage_guide.md b/tests/atomic_probe/icache_miss_usage_guide.md new file mode 100644 index 0000000000..1070d75469 --- /dev/null +++ b/tests/atomic_probe/icache_miss_usage_guide.md @@ -0,0 +1,2151 @@ +# A5 PA Scheduler I-cache Miss 采集与分析指南 + +## 1. 目标与最终构建口径 + +本指南同时记录真实 simpler FDWIC PA 与 `tests/atomic_probe/pa_scheduler` standalone CCEC 的 I-cache 观察方法。目标是回答：在 Submit-all 整个调度回放期，32 个 AIC 和 64 个 AIV 每核发生了多少 I-cache request/miss，其中哪些 miss 值得继续优化。真实 PA 结论以真实 PA 独立诊断 ELF 为准；standalone 只保留历史方法、接口校准和模型边界证据，不能替代真实 PA profile。 + +standalone 当前保留两类正式观察构建： + +| 构建 | 内容 | 是否包含 PMU | +| --- | --- | --- | +| `swimlane` | 普通阶段泳道 + 逐 atomic 泳道，在同一 AIC/AIV scalar lane 合并采集 | 否 | +| `submit-pmu` | 每物理子核的 Submit-all PMU 整窗，并可编译一个局部 phase | 是，仅 CCEC | + +`run` 、`smoke` 和 phase 名是运行或编译选择，不是额外的第三类构建。 + +### 1.1 真实 PA `submit-pmu-none` + +> **2026-07-23 当前 ABI/schema v3 口径**：当前设备 ABI 与 raw JSON schema 为 +> `fdwic-submit-pmu-v3`。阶段主时间不再使用 `phase_elapsed_ticks`，而是直接读取 +> running read-clear 的 PMU total cycle，并在软件中重建整窗 total。HTML 对 ALL/AIC/AIV +> 分别按 `1.649844/1.650062/1.649731 cycles/ns` 换算等效时间。`phase_elapsed_ticks` +> 只保留为未换算的 SYS 边界闭合诊断，不参与阶段时间、阶段占比或跨阶段比较；不能再把它按 +> 1 GHz 冒充阶段耗时。下文 2026-07-22 及更早的 v1/v2 `phase_elapsed_ticks` 时间表和比例均为 +> 历史采集口径，不能作为当前 v3 结果使用。 +> +> v3 同时用 CNT3 采集阶段 local scalar-busy，并保留 CNT2 作为整窗 primary。阶段报告直接给出 +> PMU total、scalar-busy 和逐核先算 `total-scalar` 后聚合的非 Scalar-busy 残余。该残余不是 +> “空闲时间”，还可能混有等待、I-cache、atomic、观察器以及其他非 scalar-busy 周期。linked +> vector/cube Kernel 仍从阶段 PMU 和 SYS 两类窗口中成对门控排除；result-used return-ready +> atomic 只从 SYS 边界诊断扣除，仍会进入 PMU total/scalar，因此不能把 v3 阶段 PMU total +> 解释为去除 atomic 后的纯 scalar 指令时间。 + +#### 2026-07-23 Case1 v3 全量产物 + +下列 13 个 profile 均通过 96 核拓扑、调用次数、begin/end、PMU selector、owner restore、 +linked-Kernel 排除、return-ready atomic SYS 扣时以及 v3 local/whole PMU 关系门禁。每个目录都包含 +raw、provenance 和 HTML： + +| profile | Case1 产物目录 | +| --- | --- | +| `submit-pmu-none` | `outputs/TestPagedAttentionUnroll_Case1_20260723_043811/` | +| `submit-pmu-arg-build` | `outputs/TestPagedAttentionUnroll_Case1_20260723_043857/` | +| `submit-pmu-empty-bracket` | `outputs/TestPagedAttentionUnroll_Case1_20260723_043533/` | +| `submit-pmu-materialize` | `outputs/TestPagedAttentionUnroll_Case1_20260723_043943/` | +| `submit-pmu-claim` | `outputs/TestPagedAttentionUnroll_Case1_20260723_043639/` | +| `submit-pmu-register` | `outputs/TestPagedAttentionUnroll_Case1_20260723_044029/` | +| `submit-pmu-submit-transition` | `outputs/TestPagedAttentionUnroll_Case1_20260723_044115/` | +| `submit-pmu-efdrain-control` | `outputs/TestPagedAttentionUnroll_Case1_20260723_044202/` | +| `submit-pmu-prepare-map` | `outputs/TestPagedAttentionUnroll_Case1_20260723_044248/` | +| `submit-pmu-fanin` | `outputs/TestPagedAttentionUnroll_Case1_20260723_044336/` | +| `submit-pmu-winner-build-control` | `outputs/TestPagedAttentionUnroll_Case1_20260723_044422/` | +| `submit-pmu-alloc-complete-control` | `outputs/TestPagedAttentionUnroll_Case1_20260723_044508/` | +| `submit-pmu-loser-replay` | `outputs/TestPagedAttentionUnroll_Case1_20260723_044555/` | + +最新全 span 汇总位于： + +```text +outputs/fdwic_submit_span_overview_20260723_v3/ + fdwic_submit_span_overview.json + fdwic_submit_span_overview.html +``` + +目录名 `_v3` 表示输入采用 Submit-PMU v3 cohort；overview JSON 自身的聚合 schema 为 +`fdwic-submit-span-overview-v5`。v5 为每个业务 phase 固化了 +`recording_cost_reference`，避免旧 payload 在新页面中把 raw 比例继续当主值。 + +总览的每张泳道分区表同时给出泳道同父区间占比，以及对应 phase ELF 的 PMU total、scalar-busy +两类参考占比。PMU 两列不再把带有高频 begin/end 记录代码的原始 observed 比例直接冒充业务占比。 +每个指标都保留两个数： + +```text +raw 比例 = phase raw observed / 同 phase ELF 的 raw whole + +扣除局部记录估算后的参考值 = + phase raw observed + - AIC empty 每组记录开销 × 本阶段 AIC 记录组数 + - AIV empty 每组记录开销 × 本阶段 AIV 记录组数 + +参考占比 = 扣除局部记录估算后的参考值 / 同 phase ELF 的 raw whole +``` + +业务 phase 的单份 HTML 必须显式带校准 raw 重新加工；上板采集结束时自动生成的无校准 HTML +只会保留 raw 并给出警告，不能直接复制进归档： + +```bash +python simpler_setup/tools/fdwic_submit_pmu_report.py \ + /fdwic_submit_pmu_raw.json \ + --calibration-input \ + /fdwic_submit_pmu_raw.json +``` + +`submit-pmu-none` 与 `submit-pmu-empty-bracket` 自身不传 `--calibration-input`。overview 读取完整 +13-profile cohort 后会自动完成同一校准，并把参考值写入 v5 JSON。 + +ALL 必须先按上述公式分别形成 AIC/AIV 的分子，再将两个角色的分子与 raw whole 分母各自相加后 +相除，不能平均 AIC/AIV 百分比。分母始终保留该 phase ELF 实测的 **raw whole**；empty-bracket +只测到了局部 begin/end 记录组的开销，没有测完整 Submit 整窗中的全部观察代码，因此无权从 +whole 分母再扣一个估算值。生成器还要求业务 phase 与 empty-bracket 的 provenance 属于同一 +场景和同一 Git revision；只有两份 raw 都没有 sidecar 时才退化为“仅校验配置与核拓扑、revision +未证明”的明确提示。 + +以 Materialize 为例，PMU total 的 raw 比例为 `42.825%`，扣除局部记录估算后的参考占比为 +`20.469%`；scalar-busy 分别为 `38.698%` 和 `22.393%`。同一总览中的泳道 Materialize 业务时间 +占比为 `25.323%`。三者来自不同计数口径和不同 ELF，只能并列观察量级，不能互相相减或用接近程度 +证明精确闭合。曾经使用 +`(phase raw - 记录估算) / (whole raw - 记录估算)` 得到的 `26.363%/26.756%` 已废弃：它把并未 +测得的整窗观察成本也从分母扣掉，会把估算后的分母伪装成“纯业务 whole”。 + +EfDrain、WinnerBuild、AllocComplete 显式标为 `control-only`，没有语义对等 phase 的聚合 residual +或 Kernel 行显示“—”，不会用 ArgBuild 冒充整个 SubmitInternalResidual。所有业务 phase 单报告 +和 overview 已统一刷新：单报告与 overview 使用同一个分角色估算公式，同时保留 raw 比例、 +参考占比和“记录估算/raw”敏感度，避免两个加工入口给出不同口径。 + +总览中的 11 个业务 phase 跨 ELF 合成诊断改为四层显示，不再把 raw observed 合计相对 +`submit-pmu-none` 的膨胀直接命名为“观测偏差”或“插桩开销”： + +1. **原始 observed 合计**：11 个业务 phase 独立 ELF 的原始计数之和，不含 empty-bracket； +2. **空区间估算的记录代码开销**： + `AIC/AIV 各自的 empty-bracket 每组 begin/end 开销 × 该角色实际 begin/end 记录组数`； +3. **扣除记录开销后的参考值**：`原始 observed 合计 - 记录开销估算`； +4. **`submit-pmu-none` 基线**：完整 Submit 的独立 PMU ELF，每核只开关一次 PMU。 + +ALL 的 PMU total 三层 phase 值分别为 `14.351794/9.492491/4.859303 ms/core`，Scalar busy +分别为 `10.271346/6.113844/4.157502 ms/core`，非 Scalar-busy 残余分别为 +`4.080448/3.378647/0.701800 ms/core`；对应的 `submit-pmu-none` 基线分别为 +`5.884932/5.733441/0.151491 ms/core`。页面对 I-cache request/miss 也使用同一四层结构，但保持 +events/core，不把事件数强行换算成时间。 + +记录开销估算只依据当前 empty-bracket ELF 在固定调用点测得的数据。11 个业务 phase、 +empty-bracket 与 none 都来自不同 ELF 和独立进程，业务边界还可能存在覆盖空洞、交叠和 +control-only 语义；因此第三层只能用于判断量级，不能冒充零记录代码下的业务真值，也不能与 none +直接相减成可兑现的性能收益。尤其 none 的非 Scalar-busy 残余很小，任何以它为分母的比值都会被 +显著放大。 + +SubmitUnion 分区表采用更严格的均值口径。普通分段的时间列显示 `Σ core-time / 96`，三个占比 +依次由泳道、phase PMU、phase scalar 的每核均值除以各自同口径父均值；不再展示跨核累加时间。 +表尾改为五行： + +1. “SubmitUnion 平均每核时间合计”把 11 个泳道分段的每核均值相加，真实值为 + `4033.203 us/core`，并与 SubmitUnion 父区间严格闭合为 100%； +2. “10-phase 原始 observed 合计”包含 9 个直接对应表格行的 phase，以及精确覆盖 + `Claim.end→Materialize.begin` residual 子段的 ArgBuild，共 10 个独立 ELF； +3. “空区间估算的记录代码开销”按 AIC/AIV 各自的 empty-bracket 单组开销和实际 begin/end + 记录组数计算，再按 32/64 核合并；linked Kernel pause/resume 产生的额外记录组也计入； +4. “扣除记录开销后的参考值”显示第 2 行减第 3 行，仅作诊断； +5. `submit-pmu-none` 展示完整 PMU gate 的每核均值，作为独立基线。 + +真实 ALL 每核等效时间如下： + +| 指标 | 原始 observed | 记录开销估算 | 扣除后的参考值 | `submit-pmu-none` | +| --- | ---: | ---: | ---: | ---: | +| PMU total | 12.366143 ms/core | 8.279383 ms/core | 4.086759 ms/core | 5.884932 ms/core | +| Scalar busy | 8.966986 ms/core | 5.332525 ms/core | 3.634461 ms/core | 5.733441 ms/core | +| 非 Scalar-busy 残余 | 3.399157 ms/core | 2.946858 ms/core | 0.452299 ms/core | 0.151491 ms/core | + +这里严格排除了位于 SubmitUnion 之外的 SubmitTransition；ArgBuild 不能冒充整个 +SubmitInternalResidual，其余 internal residual 和全部 tail residual 仍未覆盖。另一方面， +`submit-pmu-none` 从首个 Submit begin 覆盖到末个 Submit end，包含 BetweenSubmitResidual。 +这组跨 ELF 合计仍只表示“各 phase 原始 observed 合计减去局部记录估算”的诊断量。即使它与 +泳道 SubmitUnion 的数值接近，也不能据此声称恢复了完整业务阶段、纯业务 whole 或严格 partition +closure；各 phase 的覆盖空洞、交叠、control-only 语义、代码布局和独立进程波动都没有被该减法 +消除。能够严格提供的占比仅是上一段定义的“扣除局部记录估算后的 phase 参考值 / 本 phase ELF +raw whole”，且必须继续标注为参考值。 + +汇总中的每张 phase 卡都以该 phase ELF 自己的 whole PMU total/scalar 和 primary I-cache +作为分母；不同 phase 是不同 ELF、不同进程，比例不能求和。泳道输入与这些 PMU ELF 没有共同 +provenance，当前只对齐 96 核拓扑和每核 1,280 个 Submit，不能把泳道时间与 PMU cycle 逐值相减。 + +v3 真机校准还得到三个直接证据： + +- `submit-pmu-none` B1 的 96 个核均精确满足 CNT3 shadow scalar = CNT2 primary scalar，证明重复 + scalar selector 在不做运行中 read-clear 时一致； +- phase 构建中 CNT3 shadow 相对 CNT2 primary 固定少 `2 cycles/call`，Case1 empty 和 Claim + 均逐核严格符合该线性关系。raw 和 HTML 显式展示这项 shadow loss，不把它藏进阶段业务值； +- Case1 empty-bracket 的 ALL phase PMU total 约 `1564.846 cycles/call`，按当前 ALL 校准频率 + 换算约 `948.481 ns/call`。这是当前 ELF 在该固定调用点实测的每组记录代码开销，说明高频 phase 插桩不可当作 + 零开销观察，也不能从其他 ELF 精确相减。 + +构建阶段还确认过一个容易造成假失败的产物问题：AICore cache 已更新但 +`build/lib/.../libhost_runtime.so` 仍为旧 ABI 时，设备 v3 与 host v2 会表现为 96 核记录全零。 +必须通过正式 +`RuntimeBuilder("a5").get_binaries("fully_distributed_within_core", build=True)` +同步重建和落盘 host runtime， +并核对 cache 与 `build/lib` 的实际加载 SO SHA 一致；不能把 ABI 不匹配误判成 PMU counter 不工作。 + +> **2026-07-22 历史 ABI/schema v2 口径提示**：当时设备 ABI 与 raw JSON schema 同步升级为 +> `fdwic-submit-pmu-v2`。所有真实 PA submit-PMU profile +> 都在 linked vector/cube Kernel 调用前后统一门控，完整窗的 PMU counter 与 +> `scalar_submit_elapsed_ticks`、以及命中的 phase 时间都排除 Kernel 整段。仅消费返回值的 +> return-ready/result-used atomic 依赖区间从 Scalar SYS 时间分母及命中的 phase 时间中扣除， +> 不停 PMU；source-issue atomic 保留。因而 PMU total、scalar busy、I-cache request/miss +> 仍包含 atomic 指令及最小时间 hook 的指令事件。`wall - scalar_submit_elapsed_ticks` 同时混有 +> linked Kernel、被扣除的 return-ready atomic 时间和门控边界间隙，不能命名为纯 Kernel 时间。 +> 本章现存 v1 HTML 与数值只作历史记录；当时采用 v2 做新分析时必须重新上板采集，不能与旧结果 +> 合并、相减或拼接比例。 + +#### 2026-07-22 Case1 v2 全量 HTML 索引 + +下表是同一轮 v2 口径下的完整入口。每个目录均包含 +`fdwic_submit_pmu_raw.json`、`fdwic_submit_pmu_provenance.json` 和 +`fdwic_submit_pmu_report.html`： + +| profile | Case1 产物目录 | +| --- | --- | +| `submit-pmu-none` | `outputs/TestPagedAttentionUnroll_Case1_20260722_211333/` | +| `submit-pmu-arg-build` | `outputs/TestPagedAttentionUnroll_Case1_20260722_212608/` | +| `submit-pmu-empty-bracket` | `outputs/TestPagedAttentionUnroll_Case1_20260722_212653/` | +| `submit-pmu-materialize` | `outputs/TestPagedAttentionUnroll_Case1_20260722_212739/` | +| `submit-pmu-claim` | `outputs/TestPagedAttentionUnroll_Case1_20260722_212857/` | +| `submit-pmu-register` | `outputs/TestPagedAttentionUnroll_Case1_20260722_212943/` | +| `submit-pmu-submit-transition` | `outputs/TestPagedAttentionUnroll_Case1_20260722_213030/` | +| `submit-pmu-efdrain-control` | `outputs/TestPagedAttentionUnroll_Case1_20260722_213152/` | +| `submit-pmu-prepare-map` | `outputs/TestPagedAttentionUnroll_Case1_20260722_213240/` | +| `submit-pmu-fanin` | `outputs/TestPagedAttentionUnroll_Case1_20260722_213326/` | +| `submit-pmu-winner-build-control` | `outputs/TestPagedAttentionUnroll_Case1_20260722_211624/` | +| `submit-pmu-alloc-complete-control` | `outputs/TestPagedAttentionUnroll_Case1_20260722_220027/` | +| `submit-pmu-loser-replay` | `outputs/TestPagedAttentionUnroll_Case1_20260722_222533/` | + +严格 loader 已复验每个 profile 的 96/96 trusted、linked-Kernel gate 和 return-ready atomic +时间门禁；所有 HTML 的 I-cache 卡片只显示逐核 `min/max`。旧 v1 各章节中的数表继续作为历史记录， +新的分析结论应以本索引中的 v2 三件套为入口。不同 profile 来自不同诊断 ELF 和独立进程， +时间及计数不得跨 profile 相加或相减；`submit-pmu-empty-bracket` 只用于校准观察器本身，不能解释为 +业务阶段占比。 + +真实 PA 已建立第三条独立证据链 `--fdwic-profile submit-pmu-none`。它不是 standalone 的 `submit-pmu` 产物，也不与泳道 ELF 共用 I-cache 结论： + +- 编译期固定 `PTO_FDWIC_TRACE_ENABLED=0`，去除普通泳道和逐 atomic 泳道记录；ABI v2 仍保留 + return-ready atomic 的最小 SYS 时间扣除 hook，该 hook 不生成 atomic 泳道记录，也不停止 PMU； +- 去除通用逐 task PMU ring，只保留每核首个 Submit 到末个 Submit 的一次 gate； +- CNT2 采集 scalar busy，CNT6/CNT7 采集 primary request/miss，CNT8/CNT5 做逐核 shadow 复核； +- AICPU owner 在 96 个物理子核上保存、配置、读回并恢复 PMU 寄存器； +- host 只有在 32 AIC、64 AIV、96 个唯一物理 ID、32 个 1:2 mixed triplet、每核实际 Submit 数与 orchestration 声明值一致（Case1 为 1280、B1 为 5）、主影子一致和 Restore 96/96 全部闭合时才发布正式 raw； +- 固定输出 `fdwic_submit_pmu_raw.json` 和 `fdwic_submit_pmu_report.html`，先写临时文件再原子替换正式文件。 + +真实 Case1/B256 命令为： + +```bash +source /home/q00473782/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh +source /home/q00473782/.venv/bin/activate +export PTO_ISA_ROOT=/home/q00473782/atomic/private/gpt/pto-isa-ddafa +export PYTHONPATH="$PWD:$PWD/python${PYTHONPATH:+:$PYTHONPATH}" + +python -m pytest \ + examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py \ + --platform a5 --runtime fully_distributed_within_core --level 2 \ + --case Case1 --manual include --fdwic-profile submit-pmu-none \ + --rounds 1 -s -v +``` + +2026-07-21 的正式产物位于： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_003335/ + fdwic_submit_pmu_raw.json 46,274 B + fdwic_submit_pmu_report.html 76,933 B +``` + +该轮全局 Submit 为 **5,075.360 us**。AIC/AIV 的 request/core mean 分别为 646,963.94/594,542.61，miss/core mean 分别为 1,196.88/16,943.17，聚合 miss 率分别为 0.1850%/2.8498%。PMU total、scalar busy、request、miss 都是同一 ELF、同一物理核、同一首末 Submit 窗口的数据；仍不能把 `miss * 90 ns` 当作可直接消除的跨核墙钟损失。 + +真实 PA raw 顶部的“完整 Submit”表示**全部 worker 中最早的首个 Submit 起点到最晚的末个 Submit 终点**，是 96 核共同形成的全局 `SYS_CNT` 墙钟范围，不等于逐核 PMU gate 的平均值。对单个 worker，源码先读取 `first_submit_start_tick` 再调用 `metrics_prof_start()`，末次 Submit 则先调用 `metrics_prof_stop()` 再读取 `last_submit_end_tick`；因此该核的 `SYS_CNT` 首尾区间在两侧包住 PMU gate，并包含 start/stop 及其 `PIPE_ALL` 边界成本。PMU total 按本机约 1.65 cycles/ns 的长窗校准值换算，`SYS_CNT` 为 1 ns/tick；两者的边界、时钟口径和聚合对象均不同，只能在同一 ELF 内校验数量级和跨轮方向，不能直接相减来归因观察成本或业务耗时。 + +现行真实 PA raw summary 包含 `sum/min/mean/max`，生产 HTML 还会从通过 96 核门禁的 records 重新校验这些聚合。I-cache request/miss 在 HTML 中只展示逐核 `min/max`，不再展示 mean；raw 聚合口径保持不变。PMU total 与 scalar busy 各自独立取极值，不保证来自同一个物理核。 + +#### 1.1.1 HTML 的逐核时间与 PMU 加工口径 + +下述 2026-07-21 报告增强在**当时的 v1 producer/设备 ABI**上只改了 Python 加工层；“ABI 不变”仅描述那次历史提交，不适用于 2026-07-22 的 ABI v2。该版 ALL/AIC/AIV 卡片展示： + +- `Submit SYS_CNT/core`：每核 `last_submit_end_tick-first_submit_start_tick` 的 mean/min/max，按 1 ns/tick 显示为 us；它不是顶部“最早核起点到最晚核终点”的跨核全局时间范围； +- PMU total、Scalar busy 与非 Scalar-busy 残余的 cycles mean/min/max，以及按 ALL/AIC/AIV 各自 1.649844/1.650062/1.649731 cycles/ns 换算的等效时间范围； +- `PMU-total / SYS-window/core`：先逐核计算 `total_cycles/submit_elapsed_ticks`，再展示 mean/min/max。它只表示当前 ELF 两套相近长窗的有效比，不是瞬时 AICore 频率，也不是利用率； +- primary I-cache request/miss 的 min/max、`Σmiss/Σrequest` 和按 miss 极值换算的 90 ns 直觉量尺。 + +非 Scalar-busy 残余必须先对每条 record 计算 `total_cycles-scalar_busy`，再求 min/mean/max；不能用 `min(total)-min(scalar)` 或 `max(total)-max(scalar)` 拼接，因为两个极值可能来自不同物理核。有效 cycle 比同样采用逐核 ratio 的算术均值，不是 `Σtotal/Σelapsed`。这些派生字段只存在于受信 `SubmitPmuCapture` 和 HTML，raw summary 仍保持 producer 原有字段，因此没有给设备热路径、GM 容量或 raw 合同增加成本。 + +上述等效时间都只解释当前诊断 ELF。不得拿它们与 perf-clock、swimlane 或另一个 phase ELF 相减，也不得把 `total-scalar` 改名为 Scalar 空闲、I-cache stall 或 vector/cube wait。Register 的 12 轮正式样本还出现过 primary=shadow 完整闭合、但 AIV miss/call 中途跃迁而阶段时间保持稳定的状态，说明报告必须把时间与 I-cache 计数并列展示；单轮 miss 或 miss rate 不能独立决定优化结论。 + +#### 1.1.2 新采集的构建 provenance 三件套 + +新的正式 Submit-PMU case 成功后应同时出现： + +```text +fdwic_submit_pmu_raw.json +fdwic_submit_pmu_provenance.json +fdwic_submit_pmu_report.html +``` + +`raw` 仍由 C++ producer 原子发布，Python 只读，不允许为了加入构建信息而回写。`provenance` 使用固定 schema `fdwic-submit-pmu-provenance-v1`，通过同一次 raw 读取冻结的文件名、字节数、SHA256 和 capture mode 与该轮数据绑定。HTML 在同次闭合后展示 sidecar SHA、构建时 source-v2、profile 宏/cache key，以及下列四件实物的完整文件与 literal `.text` SHA/大小： + +- worker 实际加载的 `build/lib/.../aicore_kernel.o`； +- 对应 `build/cache/.../aicore/aicore_aic_combined.o`； +- 对应 `build/cache/.../aicore/aicore_aiv_combined.o`； +- worker 使用的 `libhost_runtime.so`。 + +final ELF 与 AIC/AIV combined 不在同一目录。前者必须从实际 output path 取证，后两者和 `.git_commit` source-state stamp 必须从对应 build cache 取证，不能拿 cache 中间件冒充实际加载 ELF。身份在 ELF profile 符号门禁通过后、case 开始前冻结，case 返回后发布报告前再次重算；任一文件、stamp、raw binding、profile 宏或 cache key 不一致都会拒绝三件套闭合。sidecar 与 HTML 先完整生成到临时文件，发布前后都复核 raw 快照；任一步失败会恢复调用前的整对产物，不能留下只有一件更新或绑定旧 raw 的“正式”文件。 + +这条机制只在编译完成或 case 返回后运行，不进入 Submit/AICore 热路径，也不扩设备 GM 或 raw ABI。历史无 sidecar 的 raw 仍可离线生成基础 HTML，但从该机制落地后的新正式采集若缺少 provenance，应视为产物不完整，不能再靠采集时的当前 HEAD 或文档手工猜测 ELF 身份。 + +真实 A5 B1 已验证完整窗和分段窗两种 profile： + +- `submit-pmu-none`：`outputs/TestPagedAttentionUnroll_CaseB1_20260721_111118/`，96 核均为 5 次 Submit，primary/shadow request 与 miss 逐核完全相等；raw/provenance/HTML 分别为 44,339/3,050/80,808 B； +- `submit-pmu-register`：`outputs/TestPagedAttentionUnroll_CaseB1_20260721_111427/`，96 核均为 5 次 Submit、5 对 Register begin/end，primary/shadow 同样逐核相等；三件套分别为 69,638/3,103/83,714 B。 + +两轮 provenance 都绑定构建提交 `15c54b33`，但 profiled cache key 和 AICore extra cache key 分别落到 `submit-pmu-none`/`aa43623282e2a7db` 与 `submit-pmu-register`/`32c26e06ad76d186`，据此确认完整窗与分段窗没有复用错误 ELF。报告可被 loader 重新严格解析，并与重新渲染的 HTML 逐字节一致。 + +### 1.2 真实 PA 首个单阶段 profile：`submit-pmu-arg-build` + +真实 PA 已完成首个跟随最新泳道业务边界的单阶段 profile： + +```bash +python -m pytest \ + examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py \ + --platform a5 --runtime fully_distributed_within_core --level 2 \ + --case Case1 --manual include --fdwic-profile submit-pmu-arg-build \ + --rounds 1 -s -v +``` + +它仍保留本 ELF 的完整 Submit primary，只在 Kernel/Alloc 的 compete-first Claim 完成后开启局部 bracket，在匹配 Finish 恢复并校验 ticket 后、Materialize 入口前结束。实际覆盖 Begin 返回、同步 eager callback 构参和 Finish 重入，不包含 Claim 与 Materialize 本体。编译宏为： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=1 +PTO_FDWIC_TRACE_ENABLED=0 +``` + +none 的 ABI 仍为 6,272 B；arg-build 在相同 96 份整窗 64 B 记录后追加 96 份 64 B phase sidecar，总计 12,416 B。raw schema 仍为 `fdwic-submit-pmu-v1`，通过 `capture.mode` 区分： + +- `submit-pmu-none` 不含 `configuration.phase`，每条 record 也不含 phase 字段； +- `submit-pmu-arg-build` 增加 `phase_elapsed_ticks`、`phase_icache_requests_observed`、`phase_icache_misses_observed`、begin/end 次数、最大 shadow 分段和 `phase_status`； +- phase 时间只与同一 ELF 的 `Σsubmit_elapsed_ticks` 比较；request/miss observed 只与同一 ELF、同一角色的 primary 比较； +- 不提供 phase-local PMU total、scalar busy 或 I-cache stall 时间。 + +CNT6/CNT7 在完整窗口中不读取，继续作为 primary；CNT8/CNT5 运行中 read-clear 并软件重建 shadow whole。`primary - shadow` 是分段重建的 capture gap。phase observed 会包含 counter 边界附近少量观测 bookkeeping 的取指，因此它不是原业务事件数的数学下界；`observed + capture gap` 也不是数学上界。HTML 对 ALL/AIC/AIV 各自展示 phase core-time 与时间占比；request/miss 展示 observed 总数、逐核 min/max、同 ELF primary 分母与占比，并明示 capture gap 敏感性量尺。I-cache per-call 加权均值仍保留在 Python 派生数据中，不再作为 HTML 主统计展示。不同 profile 的 ELF 绝对时间、request 和 miss 都不能相减。 + +首轮 Case1/B256 闭合件为： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_014355/ +``` + +该轮 96 核每核 1,280 次 bracket 全部闭合，phase status 为 `0x3f`，primary/shadow 96/96 精确相等；同一 ELF 内 ALL 的 phase core-time、request observed 和 miss observed 份额分别为 5.557%、20.716% 和 21.334%。该数据首先证明采集链闭合，并提示后续需要空 bracket 校准观察 bookkeeping；不能直接把约 21% 写成零插桩业务区间的真实 I-cache 比例。 + +后续 selector 必须继续跟随真实泳道的排他 span，一次 ELF 只测一个区域，并保留该 ELF 自己的完整 Submit primary 作为比例分母。 + +### 1.3 真实 PA 空区间校准：`submit-pmu-empty-bracket` + +局部 phase 的 begin/end 本身会执行 shadow counter read-clear、状态检查和累计 bookkeeping。为了先量出这套观察器在真实 simpler A5 PA 热路径上的记录开销量级，已增加独立诊断 profile： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=2 +PTO_FDWIC_TRACE_ENABLED=0 +``` + +它不包围任何业务代码。在 Kernel/Alloc compete-first 路径的 Claim 结束处，每次 Submit 紧邻执行一次 phase begin 和 phase end；Case1/B256 每核固定 1,280 次，B1 每核固定 5 次。raw 中必须同时匹配： + +```text +capture.mode = submit-pmu-empty-bracket +configuration.phase.id = 2 +configuration.phase.name = empty-bracket +configuration.phase.boundary = claim_end_adjacent_empty_bracket +configuration.phase.counter_semantics + = running_read_clear_empty_bracket_calibration +configuration.phase.time_semantics + = outer_sys_cnt_around_adjacent_begin_end_pair +``` + +真实 Case1 的构建和运行仍通过 pytest profile 入口完成；构建缓存会按 profile 宏生成独立 AIC/AIV ELF，不能拿其他 phase 的旧 ELF 拼接： + +```bash +source /home/q00473782/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh +source /home/q00473782/.venv/bin/activate +export PTO_ISA_ROOT=/home/q00473782/atomic/private/gpt/pto-isa-ddafa +export PYTHONPATH="$PWD:$PWD/python${PYTHONPATH:+:$PYTHONPATH}" + +python -m pytest \ + examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py \ + --platform a5 --runtime fully_distributed_within_core --level 2 \ + --case Case1 --manual include --fdwic-profile submit-pmu-empty-bracket \ + --rounds 1 -s -v +``` + +用例成功后，会在本轮 `outputs/TestPagedAttentionUnroll_Case1_/` 中自动生成并严格校验： + +```text +fdwic_submit_pmu_raw.json +fdwic_submit_pmu_report.html +``` + +HTML 可直接用浏览器打开。若只需对已存在的 raw 重新生成报告，可在仓库根目录执行： + +```bash +python -m simpler_setup.tools.fdwic_submit_pmu_report \ + outputs/TestPagedAttentionUnroll_Case1_/fdwic_submit_pmu_raw.json +``` + +输出文件名固定为同目录下的 `fdwic_submit_pmu_report.html`。分析器会重新核对 profile、phase 元数据、32 AIC + 64 AIV、每核调用次数、begin/end 闭合、状态位、primary/shadow 和 owner restore；不能把手工摘出的局部数字绕过这些门禁后当正式结果。 + +#### 两套边界必须分开解释 + +empty-bracket 的时间与 I-cache 事件故意使用两套不同边界： + +1. `phase_elapsed_ticks` 由一对外层 SYS_CNT 包围相邻的完整 begin/end 调用，包含 shadow read-clear、begin/end 内部 SYS_CNT、状态检查和累计等观察器路径；它还带有外层时间戳自身的测量粒度，是“完整观察器调用对”的经验耗时，不是某段业务时间。 +2. `phase_icache_requests_observed` 与 `phase_icache_misses_observed` 只统计 begin 的 shadow read-clear 到 end 的 shadow read-clear 之间被 CNT8/CNT5 读出的事件。它没有覆盖完整 begin/end 调用对，尤其不能把外层 SYS_CNT 时间边界等同为 I-cache request/miss 边界。 + +因此报告中的每 call 指标分别按同一角色聚合后计算： + +```text +time/call = Σphase_elapsed_ticks / Σphase_end_reads +request/call = Σphase_icache_requests_observed / Σphase_end_reads +miss/call = Σphase_icache_misses_observed / Σphase_end_reads +``` + +它们是 ALL/AIC/AIV 各自的加权每次调用均值，不是 raw 为每次调用保存了一条记录。empty-bracket 继续复用每核 64 B phase sidecar，phase ABI 总大小仍为 12,416 B，没有为了逐调用校准扩充 raw。 + +#### Case1 稳态校准结果 + +两轮 Case1/B256 正式闭合件位于： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_021158/ +outputs/TestPagedAttentionUnroll_Case1_20260721_021311/ +``` + +两轮均为 96 核每核 1,280 次、phase status `0x3f`、primary/shadow 96/96 精确相等，request/miss capture gap 均为 0。每 call 结果如下： + +| 轮次 | 角色 | 外层时间 ns/call | request observed/call | miss observed/call | +| --- | --- | ---: | ---: | ---: | +| `021158` | ALL | 640.465 | 49.340 | 1.342 | +| `021158` | AIC | 567.962 | 48.919 | 0.008 | +| `021158` | AIV | 676.717 | 49.550 | 2.009 | +| `021311` | ALL | 639.272 | 49.337 | 1.356 | +| `021311` | AIC | 567.619 | 48.870 | 0.008 | +| `021311` | AIV | 675.099 | 49.570 | 2.030 | + +两轮全局 Submit 时间范围分别为 4,972.718 us 和 4,866.126 us。Case1 的每 call 校准值高度接近， +可作为当前源码和工具版本下估算局部 phase 记录代码开销的稳定参考。AIV 稳定出现约 2 次 +miss/call，而 AIC 接近 0；这是记录代码在不同 scalar 角色上的实测差异，尚不能在没有进一步 +代码布局证据时归因为某一条具体指令。 + +当前 ELF 核验也不支持直接改 reader 来“压低校准值”：AIC/AIV reader 都是同一份 92 B noinline 实现，且在 128 B line 下都跨两行；本机 DAV3510 模型配置中的 AIV scalar I-cache 容量和 set 数只有 AIC 一半，而 AIV 角色代码更大。现阶段只能把约 2 miss/call 视为容量、角色代码与具体布局共同形成的稳定记录开销量级，聚合 PMU 不能定位到某一条 cache line。因而保留当前 reader 和布局；若该底噪妨碍后续 selector，应另做只改对齐的 empty A/B，不能把布局变化夹带进业务 phase。 + +B1 闭合件位于： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_020932/ +outputs/TestPagedAttentionUnroll_CaseB1_20260721_021100/ +``` + +这两轮只有每核 5 次调用，ALL 分别为 725.192/719.304 ns、84.844/69.194 request 和 1.956/2.444 miss 每 call；AIC 的 request/miss 也明显比 Case1 稳态更易波动。B1 适合快速验证接口、次数和闭合，不适合代替 Case1 判断稳态观察器成本；少量调用会把 ELF 首次进入、取指预热和启动时序放大到每 call。 + +#### 使用边界 + +- empty-bracket 是观察器的**经验校准 profile**，不是观察成本的数学下界，也不是可以从业务 phase 中直接扣除的固定常数。 +- `submit-pmu-empty-bracket` 与 `submit-pmu-arg-build` 使用不同诊断 ELF；phase 宏、代码布局、I-cache 状态和跨核到达时序都会改变。可以用 empty 的量级判断局部 observed 是否主要由观察器构成，不能逐项相减生成所谓“修正后的 arg-build”。 +- 即使同一轮 capture gap 为 0，也只证明 primary/shadow 整窗重建闭合，不会把上述两套边界变成同一个区间。 +- 现行单子核受控微基准给出的 AIC `77.376 ns/miss`、AIV `94.030 ns/miss` 只是一把 core-latency 直觉量尺；既有报告中的 90 ns 则是历史兼容口径。miss 可在单核内重叠、被流水隐藏，96 核之间也并行，其中还可能包含观察器自身带来的 miss。因此无论采用分角色标尺还是旧 `miss × 90 ns`，都不能解释为 Submit 墙钟损失，更不能当成候选优化的可兑现收益。 + +### 1.4 真实 PA `submit-pmu-materialize` + +`submit-pmu-materialize` 是首个经过 empty-bracket 校准后落到真实业务 span 的局部 profile。它继续保留该 ELF 自己的完整 Submit primary，只把 running read-clear bracket 放到泳道 `Materialize` 的同一业务边界。编译宏为： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=3 +PTO_FDWIC_TRACE_ENABLED=0 +``` + +raw 中的 profile 元数据必须精确匹配： + +```text +capture.mode = submit-pmu-materialize +configuration.phase.id = 3 +configuration.phase.name = materialize +configuration.phase.boundary = materialize_begin_to_materialize_end +configuration.phase.counter_semantics + = running_read_clear_observed_bracket +configuration.phase.time_semantics + = inner_sys_cnt_between_boundary_observers +``` + +当前真实 PA 使用 compete-first 路径：Kernel/Alloc 的 Finish 完成 ticket 恢复与校验、结束 arg-build 后，在泳道 `Materialize.begin` 对应位置打开 bracket；`dist_submit_materialize_and_prepare_map()` 内完成 task-cap 检查和 `dist_submit_materialize_args()` 后立即关闭，再进入 `PrepareMap`。因此该 profile 覆盖 Materialize 自身，不包含 Claim-to-Materialize 构参区间，也不包含后续 `dist_submit_prepare_map()`。仍受支持的 one-shot 入口使用相同的 Materialize 业务首尾边界。 + +每个成功 Submit 恰好执行一次 Materialize bracket，所以调用 shape 与每核 Submit 数完全一致：Case1/B256 为每核 1,280 次、全局 122,880 次，B1 为每核 5 次、全局 480 次。失败路径不会伪造 phase end；只要某核 begin/end 不平衡、次数不符或状态位不完整，host/分析器就拒绝发布受信结果。 + +真实 Case1 命令为： + +```bash +source /home/q00473782/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh +source /home/q00473782/.venv/bin/activate +export PTO_ISA_ROOT=/home/q00473782/atomic/private/gpt/pto-isa-ddafa +export PYTHONPATH="$PWD:$PWD/python${PYTHONPATH:+:$PYTHONPATH}" + +python -m pytest \ + examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py \ + --platform a5 --runtime fully_distributed_within_core --level 2 \ + --case Case1 --manual include --fdwic-profile submit-pmu-materialize \ + --rounds 1 -s -v +``` + +每轮测试会在对应输出目录自动生成并校验： + +```text +fdwic_submit_pmu_raw.json # 逐核权威原始数据 +fdwic_submit_pmu_report.html # 已加工的 ALL/AIC/AIV 报告 +``` + +HTML 可直接用浏览器打开。需要从已有 raw 重建时，在仓库根目录执行： + +```bash +python -m simpler_setup.tools.fdwic_submit_pmu_report \ + outputs/TestPagedAttentionUnroll_Case1_/fdwic_submit_pmu_raw.json +``` + +#### 两轮 Case1 稳态结果 + +两轮正式闭合件位于： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_024748/ +outputs/TestPagedAttentionUnroll_Case1_20260721_024909/ +``` + +两轮均为 96/96 受信记录、每核 1,280 次、begin/end 122,880/122,880、phase status `0x3f`，primary/shadow 的 request/miss capture gap 均为 0。下表的三个占比都只使用**本轮同一个 Materialize ELF** 的分母：时间以 `Σsubmit_elapsed_ticks` 为分母，request/miss 以各角色的完整 Submit primary 为分母。 + +| 轮次 | 角色 | 时间 ns/call | request observed/call | miss observed/call | 时间占比 | request 占比 | miss 占比 | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `024748` | ALL | 797.814 | 233.197 | 1.474 | 22.560% | 37.688% | 12.056% | +| `024748` | AIC | 776.265 | 228.942 | 0.022 | 22.540% | 36.532% | 10.487% | +| `024748` | AIV | 808.588 | 235.325 | 2.201 | 22.570% | 38.278% | 12.065% | +| `024909` | ALL | 797.061 | 233.238 | 1.418 | 22.105% | 37.741% | 11.681% | +| `024909` | AIC | 775.653 | 228.170 | 0.021 | 21.882% | 36.442% | 10.997% | +| `024909` | AIV | 807.765 | 235.772 | 2.116 | 22.214% | 38.404% | 11.685% | + +两轮全局 Submit 时间范围分别为 4,922.142 us 和 4,851.282 us。Materialize 的 ALL request observed 稳定在约 233.2 次/call，AIC/AIV 也都稳定在约 228～236 次/call；相对 empty-bracket 测得的约 49 次/call 记录开销量级，这个 request 信号在量级上更明显。但两者来自不同诊断 ELF，代码布局、缓存状态和到达时序不同，这里只能作方向性判断，不能执行 `materialize - empty`。 + +AIV 的 Materialize miss observed 为约 2.20/2.12 次/call，与 empty-bracket 两轮约 2.01/2.03 次/call 处于同一量级。这说明当前数据尚不能证明 Materialize 业务体本身带来明显的 AIV miss 增量；也绝不能跨 ELF 相减后把约 0.1 次/call 冒充业务净 miss。AIC 的 miss observed 同样很小。现阶段可受信的结论是：Materialize 时间和 request 信号两轮稳定，而 miss 归因仍受空区间记录开销量级限制。 + +B1 只用于结构与 cold-path 核验： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_024533/ +outputs/TestPagedAttentionUnroll_CaseB1_20260721_024641/ +``` + +两轮都满足 96/96 记录、每核 5 次、480/480 begin/end、`0x3f` 状态和零 capture gap，证明 profile 选择、边界次数、ABI 与报告链路闭合。由于每核只有 5 次调用，首次进入 ELF、取指预热和启动时序会被放大到每 call；B1 的 per-call 值只作 cold 证据，不用于替代上述 Case1 稳态归因。 + +### 1.5 真实 PA `submit-pmu-claim` + +#### 选型依据与 profile 身份 + +`submit-pmu-claim` 不是按历史 standalone phase 名单顺次补齐，而是从最新权威 Case1 泳道 `outputs/TestPagedAttentionUnroll_Case1_20260720_234305/` 的当前真实布局重新选型。该轮 Claim 为 **79,470,788 SYS_CNT ticks**，占 SubmitUnion 399,604,449 ticks 的 **19.887%**；在已经采集 arg-build 和 Materialize 后，它是剩余最大的明确 Submit 排他业务 span。该泳道中 Claim 同时为 96 核固定 1,280 次、全局 122,880 次，适合用严格固定 shape 验证局部 PMU 边界。 + +该 profile 的编译与设备身份固定为： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=4 +PTO_FDWIC_TRACE_ENABLED=0 + +capture.mode = submit-pmu-claim +configuration.phase.id = 4 +configuration.phase.name = claim +configuration.phase.boundary = claim_begin_to_claim_end +configuration.phase.counter_semantics + = running_read_clear_observed_bracket +configuration.phase.time_semantics + = inner_sys_cnt_between_boundary_observers +``` + +公共二进制协议中的 mode 为 `5`、phase enum 为 `4`。它继续复用既有 phase sidecar：128 B header、`96 × 64 B` whole record 和 `96 × 64 B` phase record，总计 **12,416 B**；没有增加逐 Claim 记录、逐核字段或新的设备 raw 容量。 + +#### 四个真实边界 + +设备端直接在四条现存 Submit 入口上复用泳道 Claim 的业务首尾边界，没有另造近似调用： + +1. 旧 Kernel `dist_submit_impl()`：在 PrepareMap 完成后的 `claim_begin` 打开，包围 `dist_submit_claim(Kernel, ...)`，在 `claim_end` 取时前关闭； +2. 旧 Alloc `dist_alloc_tensors()`：在 Register 完成后的 `claim_begin` 打开，包围 `dist_submit_claim(Alloc, ...)`，在 `claim_end` 取时前关闭； +3. compete-first Kernel begin：在 EfDrain 结束后的 `claim_begin` 打开，先执行 `dist_submit_check_task_cap()`，再执行条件 Claim，最后关闭； +4. compete-first Alloc begin：使用相同的 EfDrain.end 到 Claim.end 边界，同样包含 task-cap 检查与条件 Claim。 + +因此，当前真实 PA 使用的 compete-first 区间明确包含 **task-cap + Claim**；仍受支持的两个旧 API 区间只包含 Claim 主体，不应把两类入口的源码范围描述成完全相同。四条路径在正常 Case1 中都为每个 Submit 产生一次平衡的 begin/end；任一核次数、状态或边界不闭合都会被 host/分析器拒绝。 + +#### 运行与产物 + +真实 Case1 命令为： + +```bash +source /home/q00473782/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh +source /home/q00473782/.venv/bin/activate +export PTO_ISA_ROOT=/home/q00473782/atomic/private/gpt/pto-isa-ddafa +export PYTHONPATH="$PWD:$PWD/python${PYTHONPATH:+:$PYTHONPATH}" + +python -m pytest \ + examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py \ + --platform a5 --runtime fully_distributed_within_core --level 2 \ + --case Case1 --manual include --fdwic-profile submit-pmu-claim \ + --rounds 1 -s -v +``` + +每轮仍自动生成并校验： + +```text +fdwic_submit_pmu_raw.json # 逐核权威原始数据 +fdwic_submit_pmu_report.html # 同一 ELF 的阶段与整窗加工报告 +``` + +从已有 raw 手工重建 HTML 时执行： + +```bash +python -m simpler_setup.tools.fdwic_submit_pmu_report \ + outputs/TestPagedAttentionUnroll_Case1_/fdwic_submit_pmu_raw.json +``` + +#### B1 只作结构证据 + +两轮 B1 闭合件位于： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_031756/ +outputs/TestPagedAttentionUnroll_CaseB1_20260721_031954/ +``` + +两轮均为 96/96 受信记录、每核 5 次、480/480 begin/end、phase status `0x3f`，primary/shadow request/miss 精确相等、capture gap 为 0。全局 Submit 分别为 82.413 us 和 264.184 us，绝对时间明显波动；本阶段没有单独控制冷启动、取指预热和跨核到达，因此不对这段差值作原因归因。两轮只证明 mode/phase、四个挂点、固定 shape、ABI 与报告链路闭合，不作为 Claim 稳态性能结论。 + +#### 两轮 Case1 稳态结果 + +两轮完整 Case1 产物为： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_032101/ +outputs/TestPagedAttentionUnroll_Case1_20260721_032244/ +``` + +两轮均为 96 核每核 1,280 次、122,880/122,880 begin/end、phase status `0x3f`，owner 恢复、32 AIC + 64 AIV、固定 shape、数值顺序和风险阈值全部闭合；primary/shadow 96/96 精确相等，request/miss capture gap 均为 0。全局 Submit 时间范围分别为 4,994.863 us 和 4,704.936 us。 + +下表的时间、request、miss 都是原始 observed；三个占比只使用**同一轮、同一个 Claim ELF、同一角色**的分母。时间以 `Σsubmit_elapsed_ticks` 为分母，request/miss 分别以完整 Submit primary request/miss 为分母。 + +| 轮次 | 角色 | 时间 ns/call | request observed/call | miss observed/call | 时间占比 | request 占比 | miss 占比 | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `032101` | ALL | 641.816 | 80.219 | 1.957 | 17.942% | 14.188% | 16.870% | +| `032101` | AIC | 311.706 | 79.437 | 0.025 | 8.862% | 13.636% | 4.324% | +| `032101` | AIV | 806.871 | 80.609 | 2.922 | 22.368% | 14.476% | 17.086% | +| `032244` | ALL | 646.708 | 80.202 | 1.951 | 18.493% | 14.182% | 16.917% | +| `032244` | AIC | 313.109 | 79.465 | 0.026 | 9.213% | 13.627% | 4.436% | +| `032244` | AIV | 813.507 | 80.571 | 2.914 | 22.940% | 14.473% | 17.133% | + +两轮结果的角色差异稳定：AIV 为约 807～814 ns/call，AIC 为约 312～313 ns/call，而两者 request/call 都约为 79～81。结合独立泳道中 Claim 固定出现的 73,728 条返回型 ClaimMax Atomic，这一现象支持后续把 **AIV 角色和 atomic 等待**作为 Claim 时间重心继续核对，而不是先把差值归因于取指请求量。 + +这里仍有三条不可越过的解释边界： + +- phase elapsed 还包含 task-cap、角色路由、条件控制、结果整理和 running bracket 的观察影响，不能把约 807 ns 或 AIC/AIV 差值命名为“纯 atomic 延迟”； +- Claim、swimlane、empty-bracket 分属不同 ELF。empty 的外层 elapsed 与 Claim 的内层 elapsed 本来也不是同一时间边界；即使 request/miss 都用 running read-clear，也受不同代码布局和缓存状态影响，绝不能跨 ELF 扣减； +- I-cache 结果只能报告本 Claim ELF 内的原始 observed、同角色占比和两轮方向性。AIV 的约 2.92 miss/call 不能减去 empty 的约 2.01～2.03 后冒充业务净 miss，`miss × 90 ns` 也仍然不是可兑现的 Submit 墙钟收益。 + +### 1.6 真实 PA `submit-pmu-register` + +#### 选型依据与 profile 身份 + +`submit-pmu-register` 继续以最新权威 Case1 泳道 `outputs/TestPagedAttentionUnroll_Case1_20260720_234305/` 为选型依据。该轮普通泳道 Register 为 **47,991,560 SYS_CNT ticks**，占 SubmitUnion 399,604,449 ticks 的 **12.010%**，固定出现 122,880 次。在已经采集 arg-build、Materialize 和 Claim 后，它是下一个占比超过 10%、shape 固定且能映射到连续真实调用体的区域。 + +该 profile 的编译与设备身份固定为： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=5 +PTO_FDWIC_TRACE_ENABLED=0 + +capture.mode = submit-pmu-register +configuration.phase.id = 5 +configuration.phase.name = register +configuration.phase.boundary = register_outputs_call_entry_to_return +configuration.phase.counter_semantics + = running_read_clear_observed_bracket +configuration.phase.time_semantics + = inner_sys_cnt_between_boundary_observers +``` + +公共二进制协议中的 mode 为 `6`，phase enum 为 `5`，两者不能混用。该构建仍复用 12,416 B 设备容量：128 B header、`96 × 64 B` whole record 和 `96 × 64 B` phase record；没有新增逐调用记录或 task-kind 字段。当前缓存身份为 `aicore-extra/32c26e06ad76d186`，最终 `aicore_kernel.o` 的 SHA256 为 `8264a6afd39815c825e0630dcbb69d9a3492d2e26989a61136f06d5e371fb750`，`.text` 为 150,608 B。该历史 v1 ELF 含 Submit PMU 整窗与 phase reader，且不含 perf-clock、普通泳道、逐 atomic 泳道记录或通用逐 task PMU 符号；这不能外推为 ABI v2 也删除了 return-ready atomic 的最小 SYS 时间 hook。raw 当前不内嵌 ELF SHA，因此该 SHA 只证明最终缓存构建身份，不把历史四轮产物表述为逐字节留档。 + +#### 三个真实挂点与调用体边界 + +设备端只在现有 `dist_submit_register_outputs()` 三个调用点的入口和返回处复用通用 phase begin/end： + +1. `dist_submit_finish_kernel_tail()`：统一覆盖旧 Kernel 和 compete-first Kernel Finish，位于可选 Fanin 之后，传入 `include_existing=true`； +2. 旧 Alloc `dist_alloc_tensors()`：传入 `include_existing=false`； +3. compete-first Alloc `dist_alloc_compete_first_finish()`：同样传入 `include_existing=false`。 + +三处 end 都在 `TRACE_TIMESTAMP(register_end)` 之前。因此该边界只观察 `dist_submit_register_outputs()` 调用入口到返回，刻意排除前一阶段的记录发布、Register 结束时间戳和 caller 衔接；它对应普通泳道 Register 的核心调用体，**不是** 普通泳道 timestamp-to-timestamp span 的逐 tick 复制。 + +业务上，Kernel 的 `include_existing=true` 会按 `ctx.register_mask` 扫描 existing tensor 并插入 TensorMap；Alloc 的 `include_existing=false` 会在 helper 入口直接返回。当前 PA 每 batch 包含 1 个 Alloc 和 4 个 Kernel，所以固定 shape 中混合了 Alloc 空调用体、`register_mask=0` 的近空 Kernel 调用和真正的 TensorMap 工作。当前 raw 没有为此新增 task-kind 或逐 insert 字段，聚合结果不能命名为“单次 TensorMap insert 净成本”。 + +正常成功路径中，每个 Submit 恰好执行一次该 bracket：B1 为每核 5 次、全局 480 次；Case1 为每核 1,280 次、全局 122,880 次。若 ticket 或上游 Materialize 失败而提前返回，固定 calls、begin/end 和 phase status 门禁会拒绝整份 raw，不会静默发布缺边界的结果。 + +#### 运行与产物 + +```bash +source /home/q00473782/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh +source /home/q00473782/.venv/bin/activate +export PTO_ISA_ROOT=/home/q00473782/atomic/private/gpt/pto-isa-ddafa +export PYTHONPATH="$PWD:$PWD/python${PYTHONPATH:+:$PYTHONPATH}" + +python -m pytest \ + examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py \ + --platform a5 --runtime fully_distributed_within_core --level 2 \ + --case Case1 --manual include --fdwic-profile submit-pmu-register \ + --rounds 1 -s -v +``` + +每轮自动生成并校验 `fdwic_submit_pmu_raw.json` 和 `fdwic_submit_pmu_report.html`；重建 HTML 的命令与第 1.5 节相同。 + +#### B1 只作结构证据 + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_034517/ +outputs/TestPagedAttentionUnroll_CaseB1_20260721_034904/ +``` + +两轮均为 96/96 受信记录、480/480 begin/end、phase id `5`、phase status `0x3f`，primary/shadow request/miss 逐核精确相等，owner Restore 和风险阈值全部闭合。全局 Submit 分别为 80.904 us 和 267.167 us；本阶段没有分别控制冷启动、跨核到达或非 scalar-busy 等待，因而只记录 B1 绝对时间不稳定，不对差值作原因归因，也不用于 Register 稳态性能判断。 + +#### 两轮 Case1 稳态结果 + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_035025/ +outputs/TestPagedAttentionUnroll_Case1_20260721_035136/ +``` + +两轮均为 96 核每核 1,280 次、122,880/122,880 begin/end、phase status `0x3f`，32 AIC + 64 AIV、owner Restore、mixed triplet、primary/shadow、数值顺序和计数器风险阈值全部闭合。下表所有占比只在**同一轮 Register ELF 内**计算：时间以逐核完整 Submit elapsed 为分母，request/miss 以本轮整窗 primary 为分母。 + +| 轮次 | 角色 | 时间 ns/call | request observed/call | miss observed/call | 时间占比 | request 占比 | miss 占比 | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `035025` | ALL | 187.688 | 86.699 | 1.352 | 5.249% | 14.889% | 7.132% | +| `035025` | AIC | 187.833 | 88.301 | 0.045 | 5.390% | 14.902% | 9.492% | +| `035025` | AIV | 187.615 | 85.898 | 2.005 | 5.181% | 14.882% | 7.112% | +| `035136` | ALL | 187.879 | 86.517 | 1.392 | 5.055% | 14.884% | 7.335% | +| `035136` | AIC | 188.787 | 88.166 | 0.050 | 5.120% | 14.844% | 10.510% | +| `035136` | AIV | 187.425 | 85.692 | 2.063 | 5.023% | 14.905% | 7.308% | + +两轮完整 Submit 分别为 4,688.752 us 和 5,136.513 us，而 Register 调用体的 per-call、request 和角色差异保持稳定。当前证据支持“该调用体在本诊断构建中约占逐核 Submit 时间 5.1%～5.2%，AIV 约 2.0 miss/call、AIC 约 0.05 miss/call”；不支持把普通泳道较宽 Register 的 12.010% 全部归给 RegisterOutputs。 + +该 profile 仍受三条解释边界约束： + +- 普通泳道 Register 与本调用体边界不同，而且来自另一 ELF，不能逐 tick 对齐或相减； +- empty-bracket、Claim、Materialize 和 Register 均为独立诊断 ELF，不能用 empty 扣出“净 Register 时间/事件”； +- phase 没有局部 scalar busy，`miss × 90 ns` 只能作单核串行量级感知，不能当作可兑现的 Submit 墙钟收益。 + +### 1.7 真实 PA `submit-pmu-submit-transition` + +#### 选型依据与 profile 身份 + +`submit-pmu-submit-transition` 对应最新泳道中相邻 Submit 之间的真实业务衔接：从上一次 Submit 的统一 end hook 打开，到下一次 `dist_submit_begin()` 完成并到达统一 begin hook 后关闭。它继续保留本 ELF 自己的完整 Submit primary，同时只用既有 begin/end hook 控制 running read-clear bracket；没有在各条业务 Submit 路径重复增加挂点。 + +该 profile 的编译与设备身份固定为： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=6 +PTO_FDWIC_TRACE_ENABLED=0 + +capture.mode = submit-pmu-submit-transition +configuration.phase.id = 6 +configuration.phase.name = submit-transition +configuration.phase.boundary = previous_submit_end_to_next_submit_begin +configuration.phase.counter_semantics + = running_read_clear_observed_bracket +configuration.phase.time_semantics + = inner_sys_cnt_between_boundary_observers +``` + +公共二进制协议中的 mode 为 `7`、phase enum 为 `6`，两者不能混用。该构建仍复用 12,416 B 设备 ABI：128 B header、`96 × 64 B` whole record 和 `96 × 64 B` phase record；没有增加逐间隙记录、transition 类型或 task-kind 字段。 + +#### `N-1` 次数契约与聚合语义 + +首个 Submit 没有前驱，末个 Submit 没有后继，因此每核 `N` 次 Submit 只产生 `N-1` 个 transition bracket： + +- task 0 的 begin 只启动完整 Submit PMU 整窗，不关闭不存在的前驱区间； +- 每个非末次 Submit 的 end 打开一个 transition bracket； +- 下一次非首个 Submit 完成 `dist_submit_begin()` 后，在统一 begin hook 关闭 bracket； +- 末次 Submit 的 end 只停止完整 PMU 整窗，不制造没有后继 Submit 的悬空区间。 + +设备 shape 状态、host 导出和 Python 分析器共用相同的预期次数口径。B1 每核 `N=5`，所以要求 4 次、全局 384/384 begin/end；Case1 每核 `N=1280`，所以要求 1,279 次、全局 122,784/122,784 begin/end。任一核仍按 `N` 次、少一次、多一次、begin/end 不平衡或 phase id 不是 6，整份结果都会被拒绝；每核少于 2 次 Submit 也不会发布 transition 结果。 + +当前 PA 编排中的 Kernel→Kernel、Kernel→Alloc 和 Alloc→Kernel 间隙都进入同一累计值。raw 只保存每核总 elapsed/request/miss 与总调用次数，因此报告中的 ns/request/miss per gap 是**所有相邻 Submit 间隙的加权均值**，不能进一步解释为上述任一种 transition 的独立成本。 + +#### 运行与产物 + +```bash +source /home/q00473782/Ascend/cann-9.1.0-weekly-20260708/cann/set_env.sh +source /home/q00473782/.venv/bin/activate +export PTO_ISA_ROOT=/home/q00473782/atomic/private/gpt/pto-isa-ddafa +export PYTHONPATH="$PWD:$PWD/python${PYTHONPATH:+:$PYTHONPATH}" + +python -m pytest \ + examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py \ + --platform a5 --runtime fully_distributed_within_core --level 2 \ + --case Case1 --manual include --fdwic-profile submit-pmu-submit-transition \ + --rounds 1 -s -v +``` + +每轮仍在对应输出目录自动生成并校验： + +```text +fdwic_submit_pmu_raw.json # 96 核整窗 primary 与 transition 聚合原始数据 +fdwic_submit_pmu_report.html # 同一 ELF 的 ALL/AIC/AIV 加工报告 +``` + +已有 raw 的 HTML 重建命令与第 1.5 节相同。 + +#### B1 只作结构证据 + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_042627/ +outputs/TestPagedAttentionUnroll_CaseB1_20260721_042750/ +``` + +两轮均为 96/96 受信记录、每核 5 次 Submit/4 次 gap、384/384 begin/end、phase id `6`、phase status `0x3f`，primary/shadow request/miss 逐核精确相等，owner Restore 与风险阈值全部闭合。B1 调用次数太少，会放大首次进入、取指预热和跨核到达差异；这两轮只证明 profile、`N-1` shape、ABI 和 raw→HTML 链路闭合，不用于判断 transition 的稳态性能。 + +#### 两轮 Case1 稳态结果 + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_042914/ +outputs/TestPagedAttentionUnroll_Case1_20260721_043036/ +``` + +两轮均为 96 核每核 1,280 次 Submit/1,279 次 gap、122,784/122,784 begin/end、phase status `0x3f`，32 AIC + 64 AIV、owner Restore、mixed triplet、primary/shadow、数值顺序和计数器风险阈值全部闭合。完整 Submit 分别为 4,708.545 us 和 4,649.434 us。下表所有数据和占比都只来自**本轮同一个 SubmitTransition ELF**；每 gap 是按该角色累计 calls 加权的均值。 + +| 轮次 | 角色 | 时间 ns/gap | request observed/gap | miss observed/gap | 时间占比 | request 占比 | miss 占比 | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `042914` | ALL | 354.560 | 134.101 | 4.815 | 10.046% | 23.349% | 28.624% | +| `042914` | AIC | 303.374 | 133.426 | 0.503 | 8.881% | 23.098% | 17.115% | +| `042914` | AIV | 380.153 | 134.438 | 6.972 | 10.601% | 23.475% | 29.337% | +| `043036` | ALL | 350.516 | 134.145 | 4.459 | 9.933% | 23.378% | 27.560% | +| `043036` | AIC | 303.185 | 134.200 | 0.494 | 8.843% | 23.214% | 16.889% | +| `043036` | AIV | 374.181 | 134.118 | 6.441 | 10.456% | 23.462% | 28.244% | + +两轮的 per-gap 时间和 request 方向稳定；AIV 的时间与 miss 均高于 AIC，可作为后续核对跨 Submit scalar 衔接的观察信号。但该 phase 是三类 transition 的聚合，并且 bracket 本身会改变取指与多核到达，不能据此把 AIV/AIC 差值归给某一条业务调用或某一种间隙。 + +该 profile 还受以下解释边界约束： + +- PMU phase 与泳道取自同源源码边界，但使用不同 ELF 和不同内部观察口径。泳道在 trace-on 构建中记录阶段时间戳，PMU 在 trace-off 构建中读取、清零 shadow counter，并累计两侧 observer 之间的内层 SYS_CNT；二者不能逐 tick 对齐或相减； +- empty-bracket 的 elapsed 是外层 SYS_CNT 包围相邻 begin/end 调用对，且来自另一个诊断 ELF；不能从 SubmitTransition 的内层 elapsed/request/miss 中扣除 empty； +- 本 phase 不提供局部 scalar busy，也不区分三种 transition。约 350～355 ns/gap 和 AIV 约 6.4～7.0 miss/gap 都是当前诊断 ELF 的原始聚合 observed，不能改写成零插桩业务净成本或可兑现的墙钟收益。 + +### 1.8 真实 PA `submit-pmu-efdrain-control` + +#### 控制段定义与实现边界 + +最新排他泳道中，完整 EfDrain 同时包含 Scalar 调度控制与可能被回收执行的真实 linked Kernel。为了避免把 Kernel 执行期间混入 Scalar I-cache 归因，`submit-pmu-efdrain-control` 在四条真实 Submit 入口统一采用以下边界： + +```text +drain_block_won() 前开始 + -> drain_block_won() + drain_phase_b() +drain_phase_b() 返回后结束 +``` + +当 `drain_phase_b()` 进入 `execute_slot()` 时，观察器在 `dist_aicore_call_slot_kernel()` 紧邻前暂停、返回后恢复。因此局部结果是多个不连续控制片段之和： + +```text +EfDrain-control = EfDrain 外层区间 - 真实 linked-Kernel 调用区间 +``` + +它仍保留 fanin 检查、ring 扫描、atomic、完成发布、frontier 推进、slot 清理等 Scalar 控制逻辑。背压和 FinalDrain 也会调用 `execute_slot()`，但它们不在 EfDrain 外层 phase 内，pause helper 会在编译后的当前状态下直接返回 false，不会混入本 selector。 + +#### 固定容量与闭合公式 + +该 profile 的身份为： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=7 +PTO_FDWIC_TRACE_ENABLED=0 + +capture.mode = submit-pmu-efdrain-control +configuration.phase.id = 7 +configuration.phase.name = efdrain-control +configuration.phase.boundary = efdrain_begin_to_end_excluding_linked_kernel_calls +``` + +公共 C++ capture mode 为 `8`，phase id 为 `7`，两者不能混用。设某核有 `N` 次 Submit，并在这些 EfDrain 中排除 `K` 次 linked-Kernel 调用，则设备、host 与报告端共同要求： + +```text +phase_begin_reads = phase_end_reads = N + K +报告中的 phase per-call 分母 = N +``` + +历史 v1 中 `K` 保存在 phase record 的 `reserved[0]`，当时 `reserved[1..3]` 为零。ABI v2 继续用 `reserved[0]` 保存排除次数，同时用 `reserved[1]/[2]` 发布重建的 shadow request/miss，仅 `reserved[3]` 必须为零；每核 phase record 仍为 64 B，没有增加逐 Submit、逐 Kernel 或逐事件 raw。正式结果还必须同时闭合 96 核 phase boundary/shape/value/time/status 和 `phase_kernel_exclusion_closed_records=96`。 + +#### 观察效应与使用方式 + +每排除一次 Kernel，phase 外层会多一对 pause/resume 边界及相应 bookkeeping。ABI v2 的统一 gate 使 linked Kernel 不进入完整 Submit primary/shadow、`scalar_submit_elapsed_ticks`，也不进入命中的 phase elapsed/request/miss；return-ready atomic 则只扣 SYS 时间，不能把两种排除混为一谈。这个切分会改变诊断 ELF 布局和多核到达，只能在本 ELF 内分析同次采集的占比和方向；不能与 `none`、泳道或 perf-clock 绝对相减，也不能机械扣除 empty-bracket 后声称零观察开销下的业务净值。 + +运行命令沿用其他真实 phase，只替换 profile： + +```bash +python -m pytest \ + examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py \ + --platform a5 --runtime fully_distributed_within_core --level 2 \ + --case CaseB1 --manual include --fdwic-profile submit-pmu-efdrain-control \ + --rounds 1 -s -v +``` + +host 回归覆盖 mode/phase/provenance、`N+K` 读数闭合、旧 profile 拒绝专属字段、外层 `N` 分母和 HTML 语义。实现提交 `21e0414c` 后的真实 A5 B1 位于： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_115019/ + fdwic_submit_pmu_raw.json + fdwic_submit_pmu_provenance.json + fdwic_submit_pmu_report.html +``` + +该轮 96 核均为 5 次 Submit。实际只回收执行 1 次 linked Kernel：95 核 `K=0`，logical/physical core 9 的 AIC 为 `K=1`；所以全局 begin/end 均为 `96×5+1=481`，每核都满足 `N+K`。96/96 phase status 为 `0x3f`，primary/shadow request 与 miss 也逐核完全相等；owner Restore、数值顺序、风险阈值和专属 `phase_kernel_exclusion_closed_records=96` 全部通过。完整 B1 Submit 为 279.551 us，只作冷启动结构证据，不作为稳态性能。 + +本轮局部累计 elapsed/request/miss 分别为 64,751 ticks、56,187、3,414；相对同一 ELF 的逐核 Submit 累计值为 2.7441%/16.3993%/19.4021%。它证明不连续 control segments 已能采集且真实 Kernel 被单独排除，不证明这些 B1 比例能代表 Case1。 + +三件套大小为 72,951/3,124/84,377 B，SHA256 分别为 `79d6014e892b20823da039e3a2bea6c9761946b6c24444db71d7c61d16caca02`、`24be202086e9fda893012ca999073ceffa160aa9abba12861cee95414006e4d4`、`d8b2b4b77c243ac90e71fbb99084e7f5be7205f5d2a28ee224d4e56b9ed6c892`；重新加载 raw+sidecar 后渲染的 HTML 与正式文件逐字节一致。provenance 的 Git head 精确为 `21e0414c35ae7738a89f8994bfaf6870b733dea3`，extra cache key 为 `88075a1848686623`。 + +首次上板尝试 `..._114357/` 还暴露了一个必须保留的复现门禁：AICore 已按新 profile 重编，但预装 `libhost_runtime.so` 仍是只认识到 submit-transition 的旧缓存，host init 返回 0。最终实现因此在冻结 provenance 前先核验实际 host ELF 的三个 hook 和精确 profile marker；旧 host 现在会在设备执行前明确要求重建，而不是留下无 raw 的失败目录。重建后本轮 host SHA256 为 `441e54ac3d997e110de792d6597b8cf47d31a764ccf9bc63551387b9a597b919`。 + +#### Case1 首轮稳态原因数据 + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_115559/ +``` + +该轮 golden 通过，完整 Submit 为 4,840.463 us。96 核均为 1,280 次 Submit；EfDrain 内实际排除 936 次 linked Kernel，逐核 `K` 为 3～19，AIC/AIV 分别累计 429/507 次。全局 begin/end 都是 `96×1280+936=123816`，96/96 primary/shadow request 与 miss 逐核完全相等，其他 producer/consumer/provenance 门禁也全部闭合。 + +同一 ELF 内的局部归因如下；时间占比的分母是各角色逐核 Submit elapsed 之和，request/miss 占比的分母是各角色完整 Submit primary： + +| 角色 | control 时间/call | 时间占比 | request/call | request 占比 | miss/call | miss 占比 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| ALL | 162.654 ns | 4.5920% | 82.110 | 14.0406% | 2.258 | 17.0455% | +| AIC | 199.792 ns | 5.7719% | 83.945 | 14.0450% | 0.110 | 28.6828% | +| AIV | 144.085 ns | 4.0219% | 81.193 | 14.0383% | 3.332 | 16.9316% | + +完整 Submit 的 AIC/AIV 每核平均 request 为 765,037/740,310，平均 miss 为 493/25,187.344，miss rate 为 0.06444%/3.40227%。AIV 的 EfDrain-control 每核平均 miss 为 4,264.625，也就是它贡献本轮 AIV 完整 Submit miss 的 16.93%。这回答的是 “平均每个 AIV 在本诊断 ELF 的完整 Submit 期看到多少 miss”，不是零观察构建的墙钟损失。 + +按 90 ns/miss 只作串行直觉量尺，AIV 完整窗约为 2,266.861 us/core，局部 control 约为 383.816 us/core；但局部实际 elapsed 只有 184.428 us/core。量尺大于被观察阶段时间本身，直接反证了把 `miss×90ns` 当作可相减停顿的做法：miss 可重叠、被流水隐藏，标尺也来自另一个隔离微基准。另一个重要现象是 AIC control 时间/call 反而高于 AIV，而 AIC miss/call 低两个数量级；因此当前单轮不支持“I-cache miss 主导 EfDrain-control 时间”，后续应把 atomic/同步等待和纯控制指令一并纳入解释。 + +Case1 三件套大小为 76,269/3,124/86,600 B，SHA256 分别为 `6d6aa06fbf972f36fbb9260485bb5ee41f5cbb97e9246cb15a398ba2497201ce`、`b4a535ec671f7416945b5205e11268308068759e75ecc324a10ea2a57fb3fa03`、`f5751e86f503273d1b8f8e386301d1d84dfdc49ee6de45d7372199ce90a68841`；provenance Git head 为 `77df395941f86b3b546a6f50d6288fb88acb7078`，实际 ELF SHA 与 B1 相同，离线重渲染也逐字节一致。 + +### 1.9 真实 PA `submit-pmu-prepare-map` + +最新排他泳道把 PrepareMap 定义为 Materialize 结束后到 `dist_submit_prepare_map()` 返回并取得 `prepare_map_finish` 的区间。四条 Submit 入口都汇聚到 `dist_submit_materialize_and_prepare_map()`，因此 PMU profile 在该 helper 内只包围真实调用体： + +```text +fdwic_submit_pmu_phase_begin() +dist_submit_prepare_map(self, task_id) +fdwic_submit_pmu_phase_end() +``` + +profile 身份为： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=8 +PTO_FDWIC_TRACE_ENABLED=0 + +capture.mode = submit-pmu-prepare-map +configuration.phase.id = 8 +configuration.phase.name = prepare-map +configuration.phase.boundary = dist_submit_prepare_map_call_entry_to_return +``` + +PrepareMap 每个 Submit 固定调用一次，因此 CaseB1 每核严格 5 对、Case1 每核严格 1,280 对 begin/end。它继续复用 64 B phase sidecar 和 12,416 B 固定总容量，没有增加 GM 字段、逐 Submit record 或逐事件 raw。运行方式为： + +```bash +python -m pytest \ + examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py \ + --platform a5 --runtime fully_distributed_within_core --level 2 \ + --case Case1 --manual include --fdwic-profile submit-pmu-prepare-map \ + --rounds 1 -s -v +``` + +B1 结构回归位于 `outputs/TestPagedAttentionUnroll_CaseB1_20260722_143131/`：golden 通过，96 核共 480 对边界，所有 phase status 为 `0x3f`，raw/provenance/HTML 闭合。完整 Case1 位于 `outputs/TestPagedAttentionUnroll_Case1_20260722_143242/`：golden 通过，96 核共 122,880 对边界，所有 producer/consumer/provenance 门禁通过。 + +Case1 同一 ELF 内的直接观察结果如下。时间占比以各角色 `Σsubmit_elapsed_ticks` 为分母；request/miss 占比以同次采集、同角色的 primary 为分母： + +| 角色 | elapsed/call | 时间占比 | request observed 总数 | 逐核 request min–max | request 占比 | miss observed 总数 | 逐核 miss min–max | miss 占比 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| ALL | 80.603 ns | 2.3078% | 10,095,628 | 103,633–106,941 | 13.9869% | 1,636 | 7–39 | 0.1767% | +| AIC | 81.468 ns | 2.3930% | 3,365,879 | 103,752–106,338 | 13.6735% | 314 | 7–15 | 4.0391% | +| AIV | 80.171 ns | 2.2668% | 6,729,749 | 103,633–106,941 | 14.1491% | 1,322 | 10–39 | 0.1440% | + +这组数只说明独立 PMU ELF 中 `dist_submit_prepare_map()` 调用体的观察结果。泳道 PrepareMap 还包含相邻取时与 trace record 边界，而 submit-PMU 编译期去除了 trace；两类 ELF 的代码布局和多核时序也不同。因此不能拿本轮 2.3078% 与泳道占比直接相减来声称某段 record 成本，也不能把各 phase 的 request/miss 跨 ELF 相加成完整 Submit。 + +三件套 raw/provenance/HTML 的 SHA256 分别为 `62aa8f74aaa439f088963e4e3a768bbac4053ab292f392fc8c39a49b81995ae8`、`2fe946082ffbc9f4a91f9f5d1cdda38398be19e2c3a2f607df1a48f268b0c365`、`288cfc4238290ba11558f26f6e4798d43d13b2cca583e65002ca5a50a5dc2ebf`。 + +### 1.10 真实 PA `submit-pmu-fanin` 与动态调用闭合 + +Fanin 只在 Kernel winner 路径执行，winner 落在哪个 worker 由多核竞争决定，不能继续套用“每核固定 N 次”的 phase shape。该 profile 的源码边界与泳道继承边界一致：legacy Kernel winner 从 Claim.end 打开；compete-first Kernel winner 从 PrepareMap.end 打开；二者都在 `dist_submit_collect_fanin()` 返回、取得 fanin_end 前关闭。PMU ELF 编译掉中间 trace record，但仍保留同一业务起止位置。 + +身份为： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=9 +PTO_FDWIC_TRACE_ENABLED=0 + +capture.mode = submit-pmu-fanin +configuration.phase.id = 9 +configuration.phase.name = fanin +configuration.phase.boundary = fanin_begin_to_fanin_end +configuration.phase.call_shape = dynamic_balanced +``` + +设每核 Submit 数为 `N=5B`。当前 PA 每 batch 固定一个 Alloc 和四个 Kernel task，四个 Kernel task 的唯一 winner 分别有两个落在 AIC、两个落在 AIV。因此 producer、consumer 与报告端分别复算： + +```text +逐核：begin_reads == end_reads，允许为 0，且不超过 2B +全局：Σcalls = 4B +AIC： Σcalls = 2B +AIV： Σcalls = 2B +``` + +零调用核必须满足 elapsed/request/miss observed 都为 0，但完整 Submit 末尾的 shadow tail read 仍可能更新 max chunk，因此不能错误要求 max chunk 为 0。实际调用数继续使用已有 `phase_begin_reads/end_reads`，没有新增 GM/header/record/reserved 字段；phase 构建仍为 12,416 B。host 只有在逐核平衡、逐核上界和全局/角色公式全部闭合后，才发布 `phase_global_call_count_closed=true` 的 raw。 + +B1 结构回归位于 `outputs/TestPagedAttentionUnroll_CaseB1_20260722_145237/`：golden 通过，实际 `4=2(AIC)+2(AIV)`，92 个 worker 为零调用，逐核 0～1 次，所有 96 核 phase status 为 `0x3f`。完整 Case1 位于 `outputs/TestPagedAttentionUnroll_Case1_20260722_145338/`：golden 通过，实际 `1024=512+512`，AIC 逐核 11～23 次、AIV 逐核 3～15 次，所有门禁闭合。 + +Case1 同一 ELF 内的观察结果为： + +| 角色 | 调用数 | elapsed/call | 时间占比 | request observed | request 占比 | miss observed | miss 占比 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| ALL | 1,024 | 990.228 ns | 0.2438% | 354,845 | 0.5837% | 8,284 | 1.2911% | +| AIC | 512 | 552.570 ns | 0.2122% | 126,334 | 0.6004% | 437 | 5.4212% | +| AIV | 512 | 1,427.885 ns | 0.2588% | 228,511 | 0.5749% | 7,847 | 1.2386% | + +这里的百分比仍只使用 Fanin 这一独立 ELF 自己的 Submit elapsed/primary 作为分母。它不能与 PrepareMap、Claim 等其他 profile 横向求和；AIV per-call 明显高于 AIC 也只是当前诊断 ELF 的直接现象，尚不能仅凭单轮归因为 I-cache miss。 + +Case1 三件套 raw/provenance/HTML 的 SHA256 分别为 `0df28acdd7a429b92736447d25344e697cced0bb486de6b26518ce784c92a563`、`9d2d907d3225236e0c1645fe892a4030ed9929027e8a644983cf47a8068e75ab`、`48d5cb8f0781eedd43f55e437fbe340d68428531416f18da1d8dde8a19f1ef50`。 + +### 1.11 真实 PA `submit-pmu-winner-build-control` + +WinnerBuild 只在 Kernel winner 路径执行。该 profile 在 Register 结束后、进入 winner 尾段前打开， +在 `dist_submit_build_winner_task()` 返回后关闭，对应完整 WinnerBuild 业务边界： + +```text +capture.mode = submit-pmu-winner-build-control +configuration.phase.id = 10 +configuration.phase.name = winner-build-control +configuration.phase.boundary = winner_build_begin_to_end_excluding_linked_kernel_calls +configuration.phase.call_shape = dynamic_balanced +``` + +设每核 Submit 数为 `N=5B`。它与 Fanin 使用同一组 Kernel winner 动态公式，但逐核不能伪造固定次数： + +```text +逐核：outer_calls = phase_begin_reads - phase_excluded_kernel_calls + outer_calls = phase_end_reads - phase_excluded_kernel_calls + 两侧相等，允许为 0，且不超过 2B +全局：Σouter_calls = 4B +AIC： Σouter_calls = 2B +AIV： Σouter_calls = 2B +``` + +WinnerBuild 内回收执行的 linked vector/cube Kernel 通过统一 gate 从 phase 时间和 PMU counter +同时排除，`phase_excluded_kernel_calls` 记录相应的 `K`。消费返回值的 return-ready/result-used +atomic 依赖区间只从 `scalar_submit_elapsed_ticks` 及命中的 phase elapsed 扣除，不停止 PMU； +I-cache/PMU counter 仍包含 atomic 与最小 hook 的指令事件，source-issue atomic 也继续保留。 + +B1 结构回归位于： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260722_211158/ +``` + +该轮业务调用严格闭合为 `4=2(AIC)+2(AIV)`，排除 linked Kernel 为 `0`。96/96 记录的 +owner/selector/status、linked-Kernel gate、return-ready atomic 时间、phase boundary/shape/value/time、 +shadow-primary bounded 和动态全局调用数全部闭合；它只提供动态 shape 的上板结构证据。 + +完整 Case1 位于： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260722_211624/ +``` + +该轮业务调用为 `1024=512(AIC)+512(AIV)`；phase 内共排除 53 次 linked Kernel，全部落在 +AIC，AIV 为 0。严格 producer/consumer/provenance 门禁与 B1 相同，均已闭合。同一 ELF 内的 +直接观察如下；时间列是 96 核累计 core-time，时间占比的分母是同角色 +`Σscalar_submit_elapsed_ticks`，request/miss 占比的分母是同次采集、同角色的完整 primary： + +| 角色 | calls | 排除 Kernel | phase core-time sum | Scalar 时间占比 | request observed / 占比 | miss observed / 占比 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| ALL | 1,024 | 53 | 10,031.073 us | 3.4716228% | 3,561,057 / 5.0471511% | 33,047 / 1.8154869% | +| AIC | 512 | 53 | 7,213.506 us | 7.9254424% | 2,611,337 / 10.4338074% | 11,544 / 11.8110478% | +| AIV | 512 | 0 | 2,817.567 us | 1.4235334% | 949,720 / 2.0860070% | 21,503 / 1.2483281% | + +`10,031.073 us` 是跨核累计的 phase core-time，不是 Submit 墙钟。该 WinnerBuild ELF 的全局 +Submit 墙钟为 4.599385 ms；另一次 `submit-pmu-none` 为 4.982069 ms，但两者来自不同诊断 +ELF 和独立进程，不能相减成 0.382684 ms 的 WinnerBuild 收益，也不能据此杜撰任何优化收益。 +本节两轮均为 ABI/schema v2；旧 v1 HTML 和数值只能作为历史记录，不能与这里的计数或比例混用。 + +### 1.12 真实 PA `submit-pmu-alloc-complete-control` + +AllocComplete 只在 Alloc winner 路径执行。legacy API 从 Claim.end 打开，compete-first API 从 +Register.end 打开，均在 `dist_submit_complete_alloc()` 返回后关闭，对应完整 AllocComplete +Scalar 控制边界： + +```text +capture.mode = submit-pmu-alloc-complete-control +configuration.phase.id = 11 +configuration.phase.name = alloc-complete-control +configuration.phase.boundary = alloc_complete_begin_to_end_excluding_linked_kernel_calls +configuration.phase.call_shape = dynamic_global +``` + +设每核 Submit 数为 `N=5B`。每 batch 只有一个 Alloc winner，但其核角色由竞争结果决定，不能把 +AIC/AIV 次数伪造成固定比例： + +```text +逐核：outer_calls = phase_begin_reads - phase_excluded_kernel_calls + outer_calls = phase_end_reads - phase_excluded_kernel_calls + 两侧相等，允许为 0，且不超过 B +全局：Σouter_calls = B +角色：AIC outer_calls + AIV outer_calls = B；两者分别不锁定 +``` + +HeapGuard 慢路径若回收并执行 linked vector/cube Kernel，统一 gate 会同时从 phase elapsed 和 +PMU counter 排除 Kernel 整段，并用 `phase_excluded_kernel_calls` 记录额外边界对。消费返回值的 +return-ready/result-used atomic 依赖区间只从 `scalar_submit_elapsed_ticks` 和命中的 phase elapsed +扣除，不停止 PMU；因此 request/miss 仍包含 atomic 指令及最小 hook 的取指事件，source-issue +atomic 也继续保留。本次 B1 和 Case1 的 phase 内排除 Kernel 次数都为 0。 + +B1 与完整 Case1 的三件套分别位于： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260722_215826/ +outputs/TestPagedAttentionUnroll_CaseB1_20260722_215826/fdwic_submit_pmu_report.html +outputs/TestPagedAttentionUnroll_Case1_20260722_220027/ +outputs/TestPagedAttentionUnroll_Case1_20260722_220027/fdwic_submit_pmu_report.html +``` + +B1 严格闭合 `1=0(AIC)+1(AIV)`，95 个 worker 为零调用；这只是一次实际 winner 分布，不是角色 +公式。Case1 严格闭合 `256=255(AIC)+1(AIV)`，同样不能把 255/1 固化成后续运行的期望值。两轮 +96/96 trusted、linked-Kernel gate、return-ready atomic 时间、phase boundary/shape/value/time、 +shadow-primary bounded 和动态全局调用数门禁全部闭合。 + +Case1 同一 ELF 内的直接观察如下。phase ticks 是跨核累计 core-time，Scalar 时间占比的分母是 +同角色 `Σscalar_submit_elapsed_ticks`，不是全局 Submit 墙钟： + +| 角色 | calls | phase elapsed ticks | request observed | miss observed | Scalar 时间占比 | +| --- | ---: | ---: | ---: | ---: | ---: | +| ALL | 256 | 584,689 | 170,344 | 8,790 | 0.1968758%（约 0.197%） | +| AIC | 255 | 580,355 | 169,762 | 8,724 | 0.6376930% | +| AIV | 1 | 4,334 | 582 | 66 | 0.0021041% | + +`584,689 ticks` 即 96 核累计的 584.689 us，不是 AllocComplete 墙钟耗时；0.197% 也只描述该 +AllocComplete ELF 的 phase core-time 相对同一 ELF Scalar Submit 分母的比例。不得与 none、 +WinnerBuild 或其他独立诊断 ELF 的时间和计数相加、相减，也不能据此杜撰独立性能收益。 + +### 1.13 真实 PA `submit-pmu-loser-replay` + +LoserReplay 只在 Kernel loser 路径执行。legacy 与 compete-first Kernel Submit 最终共用同一个 +tail：在 Register 结束边界打开 bracket，调用 `drain_block_won()`，函数返回后立即关闭。因此该 +profile 的身份为： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=12 +PTO_FDWIC_TRACE_ENABLED=0 + +capture.mode = submit-pmu-loser-replay +configuration.phase.id = 12 +configuration.phase.name = loser-replay +configuration.phase.boundary = register_end_to_drain_block_won_return +configuration.phase.call_shape = dynamic_balanced +``` + +设每核 Submit 数为 `N=5B`。当前 PA 每 batch 有四个 Kernel task；96 个 worker 都回放这四次 +Kernel Submit，每个 task 只有一个 winner，且四个 winner 中两个落在 AIC、两个落在 AIV。 +LoserReplay 是对应 Kernel winner 集合的补集，因此 producer、consumer 和分析器共同复算： + +```text +逐核：outer_calls = phase_begin_reads - phase_excluded_kernel_calls + outer_calls = phase_end_reads - phase_excluded_kernel_calls + 两侧相等，允许动态分布，且不超过 4B +全局：Σouter_calls = (96 × 4 - 4)B = 380B +AIC： Σouter_calls = (32 × 4 - 2)B = 126B +AIV： Σouter_calls = (64 × 4 - 2)B = 254B +``` + +`drain_block_won()` 只把已经发布的 joint lane 搬入本核 RingSlot，不调用 +`drain_phase_b()`/`execute_slot()`，所以当前 LoserReplay 边界内不会执行 linked vector/cube +Kernel，`phase_excluded_kernel_calls` 必须为 0。其通用 joint 路径仍可能执行消费返回值的 +`WonAnyLoad`、`WonStateLoad` 和 `WonLaneClaimExchange`：这些 return-ready atomic 的依赖等待时间 +沿 ABI v2 口径从 `phase_elapsed_ticks` 和 `scalar_submit_elapsed_ticks` 扣除，但 PMU gate 不停止， +request/miss 仍包含 atomic 指令及最小时间 hook 的取指事件。返回值不消费的 source-issue atomic +继续保留在 Scalar 控制口径中。 + +B1 与完整 Case1 的 raw/provenance/HTML 三件套分别位于： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260722_222358/fdwic_submit_pmu_raw.json +outputs/TestPagedAttentionUnroll_CaseB1_20260722_222358/fdwic_submit_pmu_provenance.json +outputs/TestPagedAttentionUnroll_CaseB1_20260722_222358/fdwic_submit_pmu_report.html + +outputs/TestPagedAttentionUnroll_Case1_20260722_222533/fdwic_submit_pmu_raw.json +outputs/TestPagedAttentionUnroll_Case1_20260722_222533/fdwic_submit_pmu_provenance.json +outputs/TestPagedAttentionUnroll_Case1_20260722_222533/fdwic_submit_pmu_report.html +``` + +B1 严格闭合 `380=126(AIC)+254(AIV)`，逐核 3～4 次。Case1 严格闭合 +`97,280=32,256(AIC)+65,024(AIV)`，全体逐核 1,005～1,020 次。两轮均为 96/96 +trusted，owner/selector/status、linked-Kernel gate、return-ready atomic 时间、phase +boundary/shape/value/time、shadow-primary bounded 和动态全局/角色调用数门禁全部闭合。 + +Case1 同一 ELF 内的直接观察如下。phase ticks 是 96 核累计 core-time，Scalar 时间占比的分母是 +同角色 `Σscalar_submit_elapsed_ticks`： + +| 角色 | calls | phase elapsed ticks | request observed | miss observed | Scalar 时间占比 | +| --- | ---: | ---: | ---: | ---: | ---: | +| ALL | 97,280 | 5,021,979 | 8,218,072 | 453,234 | 1.2719749% | +| AIC | 32,256 | 1,696,385 | 2,539,793 | 4,148 | 1.5188385% | +| AIV | 65,024 | 3,325,594 | 5,678,279 | 449,086 | 1.1745909% | + +这是高频 observer 边界：Case1 执行 97,280 对 begin/end read-clear。`phase_elapsed_ticks` 使用 +`inner_sys_cnt_between_boundary_observers`，不把两次 counter reader 自身当成业务时间；但 +`phase_icache_requests/misses_observed` 是 `running_read_clear_observed_bracket`，会包含边界附近的 +状态检查、累计 bookkeeping 和 observer 取指。高频固定开销会随调用次数累积，因此表中 request/miss +不能解释成无观察器 LoserReplay 函数体的数学下界，也不能与其他 phase ELF 的 observed 相加。 + +本轮全局首个 Submit 到末个 Submit 为 5.256747 ms，只用于证明该诊断运行的全局窗口闭合；它不是 +上述 96 核累计 phase ticks 的分母，也不能与 none 或其他独立 ELF 的墙钟相减成 LoserReplay 收益。 + +### 1.14 真实 PA 全 span 证据汇总 + +单份 PMU HTML 只回答一个独立 ELF。为了同时查看泳道业务时间树、完整 `none` 以及 11 个业务 +phase，新增离线工具： + +```text +simpler_setup/tools/fdwic_submit_span_overview.py +``` + +它不读取 Perfetto 展示派生物 `merged_swimlane.json`，也不直接信任旁边已有的 +`swimlane_exclusive_analysis.json`。输入必须是 producer 的 `l2_swimlane_records.json`；工具对 raw +做前后 SHA-256 快照，并调用现有 schema-v4 analyzer 当场重算 96 核拓扑、父子包含、Kernel +containment、`dropped_records=0` 和全部整数闭合。每个 PMU 输入目录必须同时存在固定名 raw 与 +provenance，13 个 mode 唯一且齐全，并逐份通过 `load_capture()` 和 `load_provenance()`。 + +离线命令使用本用户 Python 环境： + +```bash +source /home/q00473782/.venv/bin/activate +export PYTHONPATH="$PWD:$PWD/python${PYTHONPATH:+:$PYTHONPATH}" +export PTO_ISA_ROOT=/home/q00473782/atomic/private/gpt/pto-isa-ddafa + +python -m simpler_setup.tools.fdwic_submit_span_overview \ + --swimlane-raw outputs/TestPagedAttentionUnroll_Case1_20260722_104657/l2_swimlane_records.json \ + --pmu-dir outputs/TestPagedAttentionUnroll_Case1_20260722_211333 \ + --pmu-dir outputs/TestPagedAttentionUnroll_Case1_20260722_212608 \ + --pmu-dir outputs/TestPagedAttentionUnroll_Case1_20260722_212653 \ + --pmu-dir outputs/TestPagedAttentionUnroll_Case1_20260722_212739 \ + --pmu-dir outputs/TestPagedAttentionUnroll_Case1_20260722_212857 \ + --pmu-dir outputs/TestPagedAttentionUnroll_Case1_20260722_212943 \ + --pmu-dir outputs/TestPagedAttentionUnroll_Case1_20260722_213030 \ + --pmu-dir outputs/TestPagedAttentionUnroll_Case1_20260722_213152 \ + --pmu-dir outputs/TestPagedAttentionUnroll_Case1_20260722_213240 \ + --pmu-dir outputs/TestPagedAttentionUnroll_Case1_20260722_213326 \ + --pmu-dir outputs/TestPagedAttentionUnroll_Case1_20260722_211624 \ + --pmu-dir outputs/TestPagedAttentionUnroll_Case1_20260722_220027 \ + --pmu-dir outputs/TestPagedAttentionUnroll_Case1_20260722_222533 \ + --output-dir outputs/down/fdwic_submit_span_overview_20260722_v2 +``` + +输出为： + +```text +outputs/down/fdwic_submit_span_overview_20260722_v2/ + fdwic_submit_span_overview.json + fdwic_submit_span_overview.html +``` + +JSON 是紧凑机器件，HTML 是可离线浏览的加工件。页面只展示逐核 `min/max`，phase extrema 表示该核 +累计整个 phase 的极值，不是单次调用极值；动态 phase 同时显示逐核 calls `min/max` 和零调用核数， +避免把 AllocComplete 的 AIV `min=0` 误读成一次调用耗时为零。每个 phase 卡都直接写出 +`分子 / 本 ELF 分母 = 占比`，不复用 `none` 分母，也不生成跨 profile 合计。 + +#### 两条证据链的边界 + +泳道与 PMU 必须分栏： + +- 泳道百分比只在同一个泳道 ELF 的排他树中闭合；SYS counter 为 1 GHz 换算，即 1 tick=1 ns， + 与约 1.65 GHz 的 PMU cycle 频率不是同一个量； +- PMU phase 时间只除以该 phase ELF 自己的 `Σscalar_submit_elapsed_ticks`，request/miss observed 只除以 + 同一 ELF 的 primary request/miss；不同 PMU ELF 的占比不能相加； +- 每核首个 Submit 先读 start tick、再 `metrics_prof_start()`；末个 Submit 先 stop、再读 end tick， + 因而 PMU gate 嵌在首末 SYS closure 内。遇 linked Kernel 时 gate 成对暂停/恢复； + `scalar_submit_elapsed_ticks` 是 gate-running SYS 段之和，再扣 result-used return-ready atomic 等待； +- PMU counter 遇 linked Kernel 同步停表，但不因 return-ready atomic 停表，所以仍含 atomic 指令及最小 + 时间 hook 的事件；source-issue atomic 同时保留在时间与 counter 口径； +- 最新泳道 raw 没有 build-provenance sidecar。工具只能证明 raw SHA 与 schema-v4 重算闭合，并将 + `swimlane_to_pmu_identity_bound=false` 写入 JSON；不能伪称泳道与 PMU 已绑定到同一 ELF。 + +当前 13 份 PMU 来自 `9d3acca8`、`908f9adf`、`94172c64`、`d67f3f5e` 四组 revision，13 个 +`aicore_kernel` SHA 均不同。mixed revision 允许汇总展示，但进一步禁止跨 ELF 相减；工具只要求 +PMU provenance 的测试/平台/场景键一致，并与泳道对齐 96 核拓扑和每核 1,280 个 Submit。 + +#### 当前同一泳道 ELF 的时间分布 + +`20260722_104657` 的全局 Submit 墙钟范围为 4.844066 ms；下面是 96 核累计 core-time 的同 ELF +排他比例，不是墙钟占比。SubmitEnvelope 先闭合为 SubmitUnion `85.3839%` 与相邻 Submit 间隙 +`14.6161%`。SubmitUnion 内部为： + +| 泳道区域 | SubmitUnion 占比 | +| --- | ---: | +| EfDrain | 16.1853% | +| Materialize | 25.3230% | +| PrepareMap | 5.3936% | +| Claim | 20.4422% | +| Fanin | 0.3989% | +| Register | 12.5173% | +| WinnerBuild | 1.6655% | +| AllocComplete | 0.2154% | +| LoserReplay | 3.1810% | +| SubmitInternalResidual | 10.8563% | +| SubmitTailResidual | 3.8217% | + +EfDrain 可继续严格替换为 KernelUnion `51.2020%` 与 Scalar control `48.7980%`。当前泳道在 +WinnerBuild 中记录到 2 个 Kernel event，但 analyzer 没有给出该子区间的 Kernel union 时长，所以泳道 +树只能称原始业务 elapsed，不能伪装成纯 Scalar 分布；WinnerBuild/AllocComplete 的纯 Scalar 归因以 +对应 `*-control` PMU ELF 为准。Atomic、ClockBaseline、Commit、RingBp、DrainWon 都是非加和 overlay， +不进入上述 100% 分区。 + +#### 当前各独立 PMU ELF 的局部比例 + +下表每一行有自己的 Scalar/primary 分母；列间可在本行内理解，行与行之间不能求和： + +| 独立 PMU profile | Scalar 时间占比 | request observed 占比 | miss observed 占比 | +| --- | ---: | ---: | ---: | +| ArgBuild | 5.927% | 19.119% | 27.021% | +| Materialize | 25.044% | 32.352% | 12.503% | +| Claim | 10.626% | 20.996% | 20.683% | +| Register | 6.028% | 14.119% | 14.486% | +| SubmitTransition | 14.231% | 22.198% | 27.843% | +| EfDrainControl | 7.092% | 16.735% | 12.784% | +| PrepareMap | 3.016% | 11.785% | 1.480% | +| Fanin | 0.358% | 0.486% | 1.027% | +| WinnerBuildControl | 3.472% | 5.047% | 1.815% | +| AllocCompleteControl | 0.197% | 0.244% | 0.403% | +| LoserReplay | 1.272% | 9.723% | 11.619% | + +`empty-bracket` 单列为 observer 经验校准，不是业务 phase。覆盖矩阵还如实标出当前没有独立 PMU 的 +Submit tail residual、Orchestration setup/tail 和 FinalDrain；缺失处显示未覆盖，不用 residual 名字或 +跨 ELF 算术伪造数据。 + +## 2. 为什么 I-cache miss 必须独立重编译 + +I-cache 数据对代码布局极其敏感。泳道和逐 atomic 观察会增加： + +- 阶段和 atomic 的 `SYS_CNT` 读取； +- atomic wrapper、ClockBaseline 与 record 发布分支； +- 更大的 scalar `.text` 和不同的函数/对齐布局； +- 由此引起的 worker 到达、轮询和跨核竞争时序变化。 + +因此，“代码仍在 ELF 中，只在运行时关闭 record”不足以得到干净的 I-cache 观察。`submit-pmu` 会独立重编译，编译掉泳道 record、逐 atomic 泳道记录慢体、ClockBaseline、runtime phase-profile 和旧 cold/warm 冲刷体。ABI v2 仍保留 return-ready/result-used atomic 的最小 SYS 时间扣除 hook；它不落泳道记录，也不门控 PMU counter。`swimlane` 则不构建 PMU owner。 + +两种数据不在同一进程采集： + +- 看事件时序和 atomic bracket，使用 `swimlane`； +- 看 Submit-all 整窗每核 I-cache request/miss，使用 `submit-pmu`。 + +两份证据可以按同一源码版本交叉理解，不能做逐 tick 对齐，也不能把 PMU 的平均 miss 回填为某一条 atomic span 的属性。 + +### 2.1 运行时关闭不等于编译期去除：真实 PA 回退案例 + +真实 PA 已出现过一次完整反例：level 1 的 raw 中没有任何 Atomic 或 ClockBaseline 记录，但同一 ELF 为了允许运行时切到 level 4，仍把 atomic wrapper、PollBatch 遍历和记录发布慢体大量内联进 Submit 热函数。结果不是 “没有写 record 就没有代价”，而是未执行的诊断代码仍改变 `.text`、基本块布局和取指工作集。 + +当时真实 A5 Case1 的证据链为： + +| 构建状态 | AIC/AIV `dist_engine .text` | AIC/AIV `dist_submit_impl` | 三轮首末 Submit 中位数 | +| --- | ---: | ---: | ---: | +| pre-atomic 历史构建 | 80,824 / 80,912 B | 42,136 / 42,152 B | 5.115620 ms | +| atomic 观测接入、level 1 不落 Atomic | 347,360 / 355,968 B | 100,724 / 102,588 B | 5.631038 ms | +| 两处低频 winner 调整为冷分支 | 347,536 / 357,112 B | 100,860 / 103,676 B | 5.192087 ms | +| atomic 冷代码共享外提 | 66,768 / 67,120 B | 18,812 / 18,872 B | 4.821897 ms | + +最后一行正式三轮为 4.821897/4.890447/4.752956 ms。这里能证明的是 “代码复制和布局回退已被消除”；没有同时采集 I-cache PMU，因此不能把恢复量进一步写成某个确定的 miss 降幅。 + +冷代码外提遵守两条边界： + +1. direct atomic 的 begin、真实 atomic、返回值地址依赖和 end 仍留在原 wrapper，只共享 end 之后的 record 发布。因此不把 source-issue 操作改成等待返回型，atomic span 口径也不变。 +2. PollBatch 保留内联的 `level >= 4 && active_mask != 0` 快速判断，只把命中后的十类遍历与落盘外提。level 1 不新增 call/ret，level 4 的 `end_cycle` 仍在冷函数调用前取得。 + +保留此类修改前应依次检查：AIC/AIV 对象的 `.text` 和独立符号、level-1 多轮 Submit、level-4 logical/physical Atomic 公式、ClockBaseline 和 `dropped_records=0`。上述真实 PA level-4 复核得到 115,309 次 atomic 调用、107,608 条 Atomic、8,056 次批处理轮询和 355 条 PollBatch，满足 `107608 = 115309 - 8056 + 355`。 + +这个案例同时说明为什么 `submit-pmu` 不能只传运行时 level=0：专门分析 I-cache 时，普通泳道、atomic 泳道记录 wrapper、ClockBaseline 和相关慢体必须在编译期从待测 AIC/AIV ELF 中剔除；ABI v2 的最小 return-ready 时间 hook 是当前测量合同的一部分，不属于应删除的泳道慢体。运行时 gate 只控制“执行没有”，不能控制“代码存在没有”。 + +### 2.2 Claim-first eager 重编后的观察结论 + +真实 PA 切换到 compete-first eager 后，Submit 的阶段顺序变为 `EfDrain -> Claim -> Materialize -> PrepareMap -> Fanin/Register -> 尾阶段`。这类热路径重排会同时改变基本块与跨 TU 的代码布局，因此它的性能结论应记录在 I-cache 观察指南中，不归因为某个 atomic 本身的收益。 + +迁移后的 atomic 观察仍包围原位置的真实指令：消费返回值的调用继续使用 `return_ready` 边界，不消费返回值的 Exchange/FetchAdd 继续使用 `source_issue` 边界，PollBatch 的逻辑调用与物理压缩记录口径也没有改变。Claim 前移只改变业务阶段顺序，没有把 atomic 记录提前、延后或改写成另一种完成语义。 + +最终真实 A5 level-4 复核位于： + +~~~text +outputs/TestPagedAttentionUnroll_Case1_20260720_104406/ +~~~ + +该轮包含 122,880 个 Submit、945,653 条事件，`dropped_records=0`。Atomic 物理记录、逻辑调用、轮询调用和 PollBatch 记录满足： + +~~~text +106355 = 109392 - 3361 + 324 +~~~ + +raw 转换、阶段顺序和整数闭合均通过，schema 能完整解析 site/op、`result_used` 与 `return_ready`。这些结果只用于证明热路径重排后的观察能力和计数口径仍然正确；该轮没有同时采集专用 I-cache PMU，不能据此推导 I-cache miss 降幅或某个 atomic 的独立收益。 + +## 3. Standalone 历史 `none` 与局部 phase 如何选择 + +从本节到第 11 节主要描述 `tests/atomic_probe/pa_scheduler` standalone 的历史 schema-v5、构建目录、命令和字段；不能套用到上面的真实 PA profile。真实 PA 当前 profile 为： + +- `submit-pmu-none`：完整 Submit 整窗； +- `submit-pmu-arg-build`：Claim.end 到 Materialize.begin 的同步构参区间； +- `submit-pmu-empty-bracket`：第 1.3 节定义的观察器经验校准，不是业务 phase； +- `submit-pmu-materialize`：第 1.4 节定义的真实 Materialize 业务 span； +- `submit-pmu-claim`：第 1.5 节定义的真实 Claim 业务 span； +- `submit-pmu-register`：第 1.6 节定义的 RegisterOutputs 调用体； +- `submit-pmu-submit-transition`：第 1.7 节定义的所有相邻 Submit 间隙聚合； +- `submit-pmu-efdrain-control`：第 1.8 节定义的排除 linked Kernel 的不连续 Scalar 控制段； +- `submit-pmu-prepare-map`：第 1.9 节定义的 `dist_submit_prepare_map()` 调用体； +- `submit-pmu-fanin`：第 1.10 节定义的 Kernel winner 动态 Fanin 区间； +- `submit-pmu-winner-build-control`：第 1.11 节定义的完整 WinnerBuild 业务边界内、排除 linked Kernel 的 Scalar 控制段； +- `submit-pmu-alloc-complete-control`：第 1.12 节定义的完整 AllocComplete 业务边界内、排除 linked Kernel 的 Scalar 控制段； +- `submit-pmu-loser-replay`：第 1.13 节定义的 Kernel loser `Register.end` 到 `drain_block_won()` 返回区间。 + +真实 phase id 依次为 ArgBuild `1`、EmptyBracket `2`、Materialize `3`、Claim `4`、Register `5`、SubmitTransition `6`、EfDrainControl `7`、PrepareMap `8`、Fanin `9`、WinnerBuild `10`、AllocComplete `11` 和 LoserReplay `12`；`none` 不含 phase。SubmitTransition 的 outer calls 为 `N-1`，所有 phase 的实际 begin/end 读数都要加本阶段内被统一排除的 linked Kernel 数 `K`；Fanin 与 WinnerBuild 使用 `4B/2B/2B` 的角色平衡动态公式，AllocComplete 只锁定全局 `B` 次而不锁定 AIC/AIV 分布，LoserReplay 使用每 batch `380/126/254` 的全局/AIC/AIV 补集公式且逐核不超过 `4B`，其余业务/校准 phase 都为每核 `N`。 + +当前十三种 profile 的新 raw schema 均为 `fdwic-submit-pmu-v2`，但分别来自独立诊断 ELF，不能跨 profile 相减或拼接。此前已经生成的 `fdwic-submit-pmu-v1` 文件只作历史记录，新 loader 不兼容读取。 + +standalone 历史 schema 中的 `lower/upper` 是已经固化的字段名，只表达 read-clear observed 与 `primary-shadow` capture gap；由于 bracket 两侧也有观察 bookkeeping，它们同样不能解释为零插桩业务事件数的数学上下界。 + +standalone 当前保留五个编译期 phase： + +| phase | 边界 | 优先用途 | +| --- | --- | --- | +| `none` | Submit-all 整窗中不读局部 shadow counter | 回答整个调度回放期的 AIC/AIV 每核 request/miss；这是默认选择 | +| `claim` | `Claim()`、结果写回 context 及 claim 本地统计前后读局部 shadow counter | 当 `none` 已证明 miss 值得追踪时，试验 Claim 的 running read-clear 下界/上界归因链路 | +| `efdrain` | 每次 Submit 开头唯一的 `DrainReady(...EfDrain...)` 前后 | 观察 opportunistic drain；不混入 RingBackpressure 或 FinalDrain | +| `materialize` | `MaterializeTask()` 及成功路径 `materialized_outputs` 本地统计前后 | 观察输出 descriptor/layout、本地 register mask、输出字节数和 heap 游标等 scalar 工作；不包含后续 slot payload 拷贝 | +| `register` | 每次 Submit 统一的 `RegisterOutputs()`，非 Alloc 还包含 `map_inserts` 本地统计 | 观察输出注册语义体 | + +普通泳道为了减少观察扰动，会让相邻阶段复用同一个 `SYS_CNT` 边界。PMU-only ELF 不生成这些泳道时间戳；局部 PMU bracket 只包围同一语义体，使用自己的 shadow read-clear 和 `SYS_CNT`，不做跨 ELF 的逐 tick 对齐。 + +running phase 的 begin/end 读取本身会执行 scalar 指令、占用取指并改变多核时序；schema-v5 还在每次 begin/end 各读取一次 1 ns/tick 的 SYS_CNT。因此： + +- running phase 的 phase request/miss 是带局部边界扰动的观察值； +- running phase 的累计时间也是带边界扰动的直接观察值；起点在 begin 的 shadow read-clear 之后，终点在 end 的 shadow read-clear 之前，所以不包含两侧 `ld_dev`，但包含每次调用两次 SYS_CNT 的观察扰动； +- request/miss raw 观察值是下界，上界为该核下界加 primary-shadow loss；阶段时间是单点观察值，没有伪造的上下界； +- `none` 和任一 running phase 是不同 ELF、不同进程，不能以两者相减声称得到了零扰动的局部净值； +- 未来不同 phase ELF 的局部 request/miss 不可相加成 whole gate；Submit-all 整窗始终以每个 ELF 自己的 primary whole 为准。 + +`none`/`claim`/`efdrain` 与正式 swimlane 使用 block-local runtime state 和跨 TU noinline finish；Claim/EfDrain 边界在 finish 之前已闭合。`materialize`/`register` 为了在 finish 内继续操作同一份真实 `PmuContext`，当前使用 inline-finish 诊断 ELF。后两者与 `none` 不具备字节级相同的指令布局，它们的局部结果只能在各自 ELF 内解释。 + +该区间只描述同一插桩 ELF、当前边界定义下的局部事件，不是无插桩局部阶段的真实区间。Submit-all `none` 没有运行中 read-clear，仍执行 96/96 逐核严格闭合。 + +报告中有三个相关但不相同的时间窗，不能都简称为“完整 Submit”： + +1. **PMU whole gate**：每核在 `InitPaOrchestration()` 之前 `metrics_prof_start()`，在末次 UP `SubmitCallbackTask()` 返回后立即 stop。它包含 orchestration 初始化、`EfDrain`、`Claim`、同步 eager 构参、finish、`AcceptTaskOutputs()` 和 Submit 间输出接收/调用衔接，排除 FinalDrain；与泳道 `OrchestrationReplay` 父区间接近但不做逐 tick 对齐。CNT6/CNT7 primary 和 PMU total/scalar-busy 都使用这个窗。 +2. **`submit_elapsed_ticks`**：每核从首个 `BeginCallbackSubmit()` 之后到最后一个 Submit 的 `submits++` 之后，包含两端之间的 Submit 间衔接，但不包含 whole gate 首尾的 orchestration 外围。局部 phase 时间占比以它为分母。 +3. **`configuration.submit_span_us`**：96 核中最早 `submit_begin` 到最晚 `submit_end` 的全局墙钟范围，不是逐核 PMU 时间的平均值。 + +## 4. 环境、构建与产物 + +命令在 `tests/atomic_probe/pa_scheduler` 目录下执行。非交互 shell 建议显式 source CANN 并选择本用户 GCC 15： + +```bash +source /home/q00473782/Ascend/cann-9.1.0-weekly-20260708/cann-9.1.0/set_env.sh +export GCC15_ROOT=/home/q00473782/.local/gcc-15/root +export PATH="$GCC15_ROOT/usr/bin:$PATH" +export LD_LIBRARY_PATH="$GCC15_ROOT/usr/lib/x86_64-linux-gnu:$GCC15_ROOT/usr/lib/gcc/x86_64-linux-gnu/15${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +export CXX="$GCC15_ROOT/usr/bin/g++-15" +export PYTHON=/home/q00473782/.venv/bin/python +``` + +构建 Submit-all 整窗基准： + +```bash +./run.sh build-submit-pmu ccec none +``` + +需要 Claim 局部归因时另行构建： + +```bash +./run.sh build-submit-pmu ccec claim +``` + +需要 EfDrain 局部归因时独立构建： + +```bash +./run.sh build-submit-pmu ccec efdrain +``` + +需要 Materialize 局部归因时独立构建： + +```bash +./run.sh build-submit-pmu ccec materialize +``` + +需要 Register 局部归因时独立构建： + +```bash +./run.sh build-submit-pmu ccec register +``` + +产物分别位于： + +```text +build/ccec/submit-pmu/none/ +build/ccec/submit-pmu/claim/ +build/ccec/submit-pmu/efdrain/ +build/ccec/submit-pmu/materialize/ +build/ccec/submit-pmu/register/ +``` + +每个 phase 目录中的 `pa_scheduler_host`、`pa_scheduler_kernel.o`、`libpa_scheduler_pmu_owner_aicpu.so` 和 `libpa_scheduler_pmu_owner_dispatcher.so` 是一个不可拆分的构建集。host 会按 kernel 所在目录加载两个 SO；不得从另一个 phase 目录复制或拼接产物。构建只在四件套全部完成后原子发布 `submit_pmu_artifacts.manifest`；`run.sh` 在启动 host 前核对 schema、phase、固定文件列表和四个 SHA256。 + +`swimlane` 的 CCEC 产物仍在 `build/ccec/`，不是 PMU 产物。 + +## 5. 采集 Submit-all 整窗与局部 phase + +### 5.1 Submit-all 整窗 `none` + +```bash +OUT="./outputs/submit_pmu_none_$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p "$OUT" + +./run.sh submit-pmu ccec none \ + --device 0 --batches 1 \ + --winner-workload real-compute --real-compute-counts 6,28,4,1 \ + --pmu-json "$OUT/submit_icache_raw.json" +``` + +多轮比较必须使用多个独立进程和独立子目录；每个采集目录内部都保持同一组描述性文件名： + +```bash +mkdir -p "$OUT/capture_02" "$OUT/capture_03" + +./run.sh submit-pmu ccec none --device 0 --batches 1 \ + --winner-workload real-compute --real-compute-counts 6,28,4,1 \ + --pmu-json "$OUT/capture_02/submit_icache_raw.json" + +./run.sh submit-pmu ccec none --device 0 --batches 1 \ + --winner-workload real-compute --real-compute-counts 6,28,4,1 \ + --pmu-json "$OUT/capture_03/submit_icache_raw.json" +``` + +### 5.2 Claim 局部归因 + +```bash +OUT_CLAIM="./outputs/submit_pmu_claim_$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p "$OUT_CLAIM" + +./run.sh submit-pmu ccec claim \ + --device 0 --batches 1 \ + --winner-workload real-compute --real-compute-counts 6,28,4,1 \ + --pmu-json "$OUT_CLAIM/submit_icache_raw.json" +``` + +### 5.3 EfDrain 局部归因 + +```bash +OUT_EFDRAIN="./outputs/submit_pmu_efdrain_$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p "$OUT_EFDRAIN" + +./run.sh submit-pmu ccec efdrain \ + --device 0 --batches 1 \ + --winner-workload real-compute --real-compute-counts 6,28,4,1 \ + --pmu-json "$OUT_EFDRAIN/submit_icache_raw.json" +``` + +`efdrain` 每核固定调用 `batches * 5` 次；b1 的 AIC/AIV/global calls 分别为 160/320/480。插点只位于 Submit 开头的 EfDrain 专属 call-site；复用的 `DrainReady()` 函数体不插桩。 + +### 5.4 Materialize 局部归因 + +后续边界和 96 核闭合验证固定使用 b1： + +```bash +OUT_MAT_B1="./outputs/submit_pmu_materialize_$(date -u +%Y%m%dT%H%M%SZ)_b1" +mkdir -p "$OUT_MAT_B1" + +./run.sh submit-pmu ccec materialize \ + --device 0 --batches 1 \ + --winner-workload real-compute --real-compute-counts 6,28,4,1 \ + --pmu-json "$OUT_MAT_B1/submit_icache_raw.json" +``` + +边界位于 `MaterializeTask()` 唯一调用点前后。实现必须先保存返回值、关闭 phase，再处理失败返回，避免失败路径留下 begin/end 不平衡。每核固定 `5 * batches` 次；b1 的 AIC/AIV/global calls 为 160/320/480。 + +2026-07-19 A5 历史实测闭环如下；四轮均满足 capture accepted、语义、PMU 和 phase measurement PASS。b256 数据只作归档证据，不是后续迭代的重跑要求： + +| phase | batches | calls/expected | begin/end 与 call shape | primary=shadow | shadow≤primary | request/miss loss | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `materialize` | 1 | 480/480 | 96/96 | 94/96 | 96/96 | 2/0 | +| `materialize` | 256 | 122,880/122,880 | 96/96 | 74/96 | 96/96 | 160/8 | +| `register` | 1 | 480/480 | 96/96 | 93/96 | 96/96 | 2/1 | +| `register` | 256 | 122,880/122,880 | 96/96 | 36/96 | 96/96 | 2,834/654 | + +对应 raw/HTML 位于 `outputs/submit_pmu_{materialize,register}_20260719_b{1,256}/`；raw 是权威取数件，HTML 是同目录的加工展示件。 + +running read-clear 的硬门禁是 96/96 `shadow≤primary`，不是要求 96/96 逐值相等；表中的 loss 已进入每核局部 lower/upper 区间，不能被静默忽略。 + +### 5.5 Register 局部归因 + +```bash +OUT_REG_B1="./outputs/submit_pmu_register_$(date -u +%Y%m%dT%H%M%SZ)_b1" +mkdir -p "$OUT_REG_B1" + +./run.sh submit-pmu ccec register \ + --device 0 --batches 1 \ + --winner-workload real-compute --real-compute-counts 6,28,4,1 \ + --pmu-json "$OUT_REG_B1/submit_icache_raw.json" +``` + +Register 已合并为每次 Submit 唯一的 `RegisterOutputs()` 调用点；Alloc 通过 `include_existing=false` 保留语义差异，winner/loser 都经过该边界。因此仍是每核固定 `5 * batches`，调用规模与 Materialize 相同。它与普通泳道包围同一 Register 语义体，但两个 ELF 各自取时，不做逐 tick 对齐；其中较短或未实际插入 map 的调用仍会放大 PMU begin/end 边界扰动，解释结果时必须使用 lower/upper，而不能把 observed lower 当成无扰动净开销。 + +`submit-pmu` action 已固定： + +```text +--runs 1 --no-swimlane --pmu-window submit-all +``` + +调用者不要重复传入这三项，也不能添加 `--profile-phases`、`--trace-atomics`、`--analyze-swimlane` 或 `--swimlane-json`。`--pmu-json` 可选；但要做 raw 复算、HTML 可视报告和多轮汇总时必须使用它。host 拒绝覆盖已有 JSON 或同名 `.tmp`。 + +raw 成功发布后，`run.sh` 会调用本目录的独立分析器并在同一目录生成： + +```text +submit_icache_raw.json # 96 核权威原始件及 host summary +submit_icache_report.html # 可离线浏览的加工件 +``` + +HTML 使用内联 CSS/SVG，不依赖外部前端库；浏览器直接打开即可。它包含 Submit-all PMU 整窗的 AIC/AIV 对比、逐物理核 request/miss/rate 分布、96 核明细和 90 ns core-equivalent 提示。报告同时展示 AIC/AIV/ALL 的 PMU raw `total_cycles`、CNT2 `scalar_busy`、`Σscalar/Σtotal` 以及 “非 Scalar-busy 残余”，并在保留 raw cycle 的同时给出校准后的每核等效时间。顶部的“完整 Submit”固定表示**最早一次 Submit 进入到最晚一次 Submit 返回**；它是 96 核共同形成的整体墙钟范围，不等于逐核 PMU whole gate 的平均值；对应到每个 worker，首末 Submit 边界也窄于该核的 PMU whole gate。ALL/AIC/AIV 响应式卡片中，I-cache request/miss 与 90 ns 直觉量尺只展示逐核 `min/max`，不展示 mean；其他 PMU 时间类字段仍保留现有聚合口径。HTML 从通过 96 核门禁的 `records` 对称推导 `min/max`；PMU total 与 scalar busy 各自独立取极值，不保证来自同一个物理核。较宽的 I-cache 对比表和 96 核明细只在表格内部横向滚动，不再撑宽整页。 + +本机 A5 受控 cold/warm 同窗校准得到： + +```text +PMU cycle_delta = 1,817,457 +SYS_CNT tick_delta = 1,101,593 ns +ALL = 1.649844 cycles/ns +AIC = 1.650062 cycles/ns +AIV = 1.649731 cycles/ns +time_us = PMU cycles / (cycles_per_ns * 1000) +``` + +ALL/AIC/AIV 汇总分别使用对应频率，逐核明细按该核角色使用 AIC 或 AIV 频率。其中 total 是每个物理子核在 PMU whole gate 内的累计周期，96 核求和是 core-work，不是 Submit 墙钟；“非 Scalar-busy 残余” 严格等于 `total−scalar_busy`，既不是 Scalar 空闲时间，也不是 I-cache stall，其中还包含同步等待、vector/cube engine 等待以及其他未归因周期。受控微基准中，依赖返回的 atomic 等待大部分进入 scalar busy，而 I-cache refill 的额外周期大部分只进入 total；这个现象不能把二者之差提升为 I-cache 专属计数器。 + +对于局部 phase，HTML 最前面先按 ALL/AIC/AIV 展示 calls、阶段时间占比、request/miss 占比；后面再列阶段时间/core、阶段时间/call、request/miss 下界—上界和 shadow loss。阶段时间占比是同一角色内 `Σphase_elapsed_ticks / Σsubmit_elapsed_ticks`，request/miss 则仍是占同一 ELF PMU whole-gate primary 的比例区间。两类占比的分母边界不同，只能分别用于时间和 I-cache 归因，不能把它们当成同一精确分区。`none` 明确显示“不适用”，历史 schema-v4 因没有阶段时间 raw 字段而显示“不可用”，不会伪造 0%。报告生成失败时 action 返回非零，但已成功发布的 raw 会保留用于排查。 + +## 6. Primary/shadow 计数和可信门禁 + +### 6.1 计数器分工 + +早期曾尝试用 external task-based `msprof` 汇总配合 kernel 内 start/stop 取得 Submit 子窗口，实测 raw counter 不受该门控缩窗，因此不能用于局部归因。现行链路由 standalone 自带的 Main AICPU owner 保存、配置、读回并恢复每个物理子核 PMU，kernel 只在同一 runtime TU 内控制 gate 和发布 worker 独占结果。任何 owner membership、selector readback 或 Restore 失败都会拒绝最终 JSON。 + +| 计数 | selector | 用途 | +| --- | --- | --- | +| PMU raw total | 固定 64-bit total low/high | PMU whole gate 内每个物理子核的累计周期；不是 96 核求和后的墙钟；HTML 另按实测频率显示等效时间 | +| CNT2 | `0x001` | scalar instruction busy cycle；不包含全部等待周期；HTML 另按实测频率显示等效时间 | +| CNT6 | `0x34` | PMU whole-gate primary I-cache request；局部边界从不读它 | +| CNT7 | `0x35` | PMU whole-gate primary I-cache miss；局部边界从不读它 | +| CNT8 | `0x34` | read-to-clear shadow request | +| CNT5 | `0x35` | read-to-clear shadow miss；诊断 ELF 因此不再提供 MTE3 busy | +| CNT9 | `0x0` | 未使用 | + +A5 b1 实测已证明将 `0x35` 配置到 CNT9 时计数始终为 0，所以 CNT9 不能作 shadow miss。这个变化只影响 `submit-pmu` 诊断构建，不影响 `swimlane` 构建。 + +#### A5 `scalar_wait_ib_time` 的支持边界 + +2026-07-19 使用本机 CANN 9.1 在同一 A5/DAV3510 上对 standalone b1 依次验证了 `PipeUtilization`、`PipeUtilization,MemoryDetail` 和 `Default` 三种正式 `msopprof` 采集入口。三次均能生成 `PipeUtilization.csv`，但表头都不包含 `aic/aiv_scalar_wait_ib_time`，也不包含 `aic/aiv_scalar_wait_time`。本机 CANN 9.1 `CHIP_V6_MAP` 与本仓 DAV3510 正式事件表同样只给出已使用的 `scalar_busy(0x001)`、I-cache request `0x34`、I-cache miss `0x35` 等事件，没有 wait-IB/wait 的 selector 或派生公式。 + +三次原始证据分别保存在： + +```text +outputs/wait_ib_official_msopprof_20260719_b1_probe2/ +outputs/wait_ib_official_msopprof_20260719_b1_probe3_memory_detail/ +outputs/wait_ib_official_msopprof_20260719_b1_probe4_default/ +``` + +因此当前 A5 正式可编程路径的结论是：**不能采集这两个指标**。CANN 共享 `msopprof` 二进制包含相应字段字符串、官方文档也在 A2/A3 产品章节解释其含义，但这些证据不能推出 DAV3510 selector。不得把旧架构或其他产品的事件号套到 A5。若后续 CANN/A5 正式事件表新增这两项，必须重新用 scalar NOP、I-cache warm/cold、真实 Vector/Cube `PIPE_* -> PIPE_S` wait 和依赖 atomic 四组对照校准后再纳入报告。 + +#### A5 真实 Vector 流水的 `scalar_busy` 归类验证 + +2026-07-22 在同一台 A5/DAV3510、CANN 9.1 weekly 20260708 上新增了单 AIV +受控探针，专门回答“执行真实 Vector 计算时，PMU total 中是否以 +`scalar_instr_busy(0x001)` 为主”。探针源码和独立 runner 为： + +```text +tests/atomic_probe/ccec/vector_scalar_pmu.cpp +tests/atomic_probe/ccec/vector_scalar_pmu_host.cpp +tests/atomic_probe/ccec/vector_scalar_pmu_shared.h +tests/atomic_probe/ccec/run_vector_scalar_pmu.sh +``` + +运行命令为： + +```bash +cd tests/atomic_probe/ccec +./run_vector_scalar_pmu.sh +``` + +测试基线是 `fdwic-swimlane-deps@7bd59a8f`。被测 `VECTOR_ADD` 循环与 +`pa_scheduler/ccec/ccec_ops.h` 的 `RunRealVectorWorkload` 保持相同 +流水顺序，tile 为 `128 x 128 x float`： + +```text +TLOAD(A) + TLOAD(B) +MTE2 -> V set/wait +TADD +V -> MTE3 set/wait +TSTORE +MTE3 -> S set/wait +``` + +每轮从 GM 读取 128 KiB、写回 64 KiB；最后的 `MTE3 -> S wait_flag` +保证本轮写回完成后才进入下一轮。PMU gate 只包住三种可替换工作负载： + +- `EMPTY`：空窗口，用于确认 gate 固有成本； +- `LOOP_CONTROL`：相同运行时循环次数，每轮只有一个 scalar NOP，不向 + Vector/MTE 发工作； +- `VECTOR_ADD`：上面的完整真实流水。 + +探针直接复用 standalone 已验证的 10-slot PMU owner，而不是新增一套寄存器 +协议。该构建固定 `PA_BUILD_SUBMIT_PMU=0`，所以同一窗口内的 selector 是 +`CNT0=vector(0x501)`、`CNT2=scalar(0x001)`、`CNT4=MTE2(0x202)`、 +`CNT5=MTE3(0x203)`、`CNT6=request(0x034)`、`CNT7=miss(0x035)`；kernel +逐项回读得到 `selector_status=0x3f` 后才接受样本。每个工作量先各模式预热 +一次，再采 5 个 `EMPTY -> LOOP_CONTROL -> VECTOR_ADD` 配对样本。五轮都落在 +同一物理 AIV 18，16,384 个输出元素都严格等于 `2.0 + 3.0 = 5.0`，I-cache +miss 都为 0，96 个 active owner slot、32 个完整 `1 AIC + 2 AIV` triplet +和最终 Restore 全部通过。 + +本机 `task-submit` 与 `npu-smi` 均不在 `PATH`，本轮按用户明确给出的 +“没有 `npu-smi` 直接运行”许可执行，因此属于 unlocked 采样。五轮同窗计数 +高度稳定，足以回答本节的 PMU 状态归类问题；这些绝对耗时不作为跨进程或跨设备 +性能基线。 + +5 轮中位数如下。`non-scalar residual` 只按定义计算 +`total - scalar_busy`： + +| rounds | mode | SYS_CNT | PMU total | scalar busy | non-scalar residual | scalar/total | vector busy | MTE2 busy | MTE3 busy | +| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 16 | `EMPTY` | 45 ns | 66 | 62 | 4 | 93.9394% | 0 | 0 | 0 | +| 16 | `LOOP_CONTROL` | 186 ns | 299 | 299 | 0 | 100.0000% | 0 | 0 | 0 | +| 16 | `VECTOR_ADD` | 31,312 ns | 51,659 | 262 | 51,397 | **0.5072%** | 14,976 | 23,233 | 12,869 | +| 128 | `EMPTY` | 45 ns | 66 | 62 | 4 | 93.9394% | 0 | 0 | 0 | +| 128 | `LOOP_CONTROL` | 1,212 ns | 1,993 | 1,993 | 0 | 100.0000% | 0 | 0 | 0 | +| 128 | `VECTOR_ADD` | 239,067 ns | 394,488 | 1,718 | 392,770 | **0.4355%** | 119,808 | 185,881 | 84,519 | + +两个工作量的 `VECTOR_ADD - LOOP_CONTROL` 配对中位数斜率也一致： + +| rounds | PMU total/轮 | scalar busy/轮 | vector busy/轮 | MTE2 busy/轮 | MTE3 busy/轮 | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 16 | 3,210.0000 | -2.3125 | 936.0000 | 1,452.0625 | 804.3125 | +| 128 | 3,066.3672 | -2.1484 | 936.0000 | 1,452.1953 | 660.3047 | + +这里的负 `scalar busy/轮` 不是“Vector 产生负周期”，而是 +`LOOP_CONTROL` 的 scalar 循环在几乎整个短窗口内都 busy；真实 Vector 循环发出 +异步指令后大部分时间不满足 `scalar_instr_busy` 条件，所以其 scalar 增量反而 +略低。频率也得到独立交叉检查：16 轮的 `51,659 / 1.65 = 31,308.5 ns`， +128 轮的 `394,488 / 1.65 = 239,083.6 ns`，分别与 1 GHz SYS_CNT 的 +31,312 ns 和 239,067 ns 对齐。 + +本次可以下的结论是：**对这条带每轮最终 `MTE3 -> S wait_flag` 的 TADD +真实流水，Scalar 并非大部分时间处于 `scalar_busy`；`scalar_busy` 只占 +0.44%～0.51%，`total - scalar_busy` 占 99.49% 以上。** 因而该路径上的 +Vector/MTE 执行和等待没有被 `0x001` 大量记入 scalar busy。完整 Submit 曾观察到 +的约 70% AIV scalar ratio 不能外推到这个 Vector 循环，更不能据此判断 I-cache +miss；它还包含 PA 调度、atomic、轮询和其他 scalar 控制路径。 + +边界同样必须保留：本轮只验证了 TADD 分支，没有验证 TMUL/Cube； +`vector_busy`、`MTE2 busy`、`MTE3 busy` 可能彼此或与 scalar 状态重叠，不能求和 +还原 total；`total - scalar_busy` 仍只能叫“非 Scalar-busy 残余”，其中同时包含 +Vector/MTE 执行、DMA/存储等待、同步等待和流水空隙，不能进一步改名为 +`wait_flag` 时间、Scalar idle 或 I-cache stall。 + +shadow PMU counter 是 read-to-clear。任一 running phase 在阶段 begin 读取 CNT8/CNT5，将之前的片段加入 shadow whole；在 end 再读一次，同时加入 shadow whole 和所选 phase；PMU whole gate stop 后读 tail。`none` 不做中途读取，只在 stop 后取 tail。 + +`none` 对每个物理子核必须精确满足： + +```text +shadow_whole_icache_requests == icache_requests +shadow_whole_icache_misses == icache_misses +``` + +即 stop 后读取的 CNT8/CNT5 分别等于同 selector、同 gate 的 CNT6/CNT7。这个 96/96 精确相等门禁验证 whole-gate 观察闭合；它不把 PMU 进程的 Submit span 变成无诊断墙钟基线，也不把 standalone 数据冒充真实 PA profile。 + +历史 A5 b1/b256 取证已证明运行中切片可能发生单向少计，接受规则为： + +```text +shadow_request <= primary_request +shadow_miss <= primary_miss + +request_loss = primary_request - shadow_request +miss_loss = primary_miss - shadow_miss + +phase_request_lower = phase_request_observed +phase_request_upper = phase_request_observed + request_loss +phase_miss_lower = phase_miss_observed +phase_miss_upper = phase_miss_observed + miss_loss +``` + +上下界必须先逐核计算，再分别聚合；不能拿聚合后的 median 相减拼区间。CNT8/CNT5 是顺序 `ld_dev`，不是同一时刻的原子配对快照，因此局部 `phase_miss <= phase_request` 不是硬门禁。二者分别不超过对应 shadow，各自上界不超过对应 primary。 + +结合 `none` exact、running phase bounded 和调用次数门禁，可分别验证： + +- 复制 selector 在本机 A5 上确实计数； +- 边界调用覆盖预定代码片段，且 begin/end/tail 次数闭合；边界竞态少计由 loss 和 lower/upper 区间显式保留； +- primary whole 没有被局部归因读取破坏。 + +### 6.2 正式 JSON 必须通过的门禁 + +只有下列条件全部成立，host 才发布最终 JSON： + +- 语义、winner 真计算输出和 Submit placement/engine 闭合通过； +- 96 条记录可信，32 AIC + 64 AIV，物理子核 id 唯一； +- owner bitmap membership、worker slot、物理 role 和 32 个 1:2 triplet 全部匹配； +- 96 个核都真实执行 Submit-all PMU whole-gate start/stop，owner Restore 成功； +- build variant 和编译 phase id 在 96 条记录中全部匹配； +- `none` 的 shadow whole 与 primary whole 逐核精确相等； +- running phase 的 shadow whole 逐核不大于 primary，loss 与 upper-bound 公式逐核闭合；exact 核数只作诊断； +- `none` 的 phase calls/begin/end/request/miss 全部为 0； +- `claim`、`efdrain`、`materialize`、`register` 的 begin/end/calls 逐核平衡，每核 calls 为 `batches * 5`，全局 calls 为 `batches * 5 * 96`； +- 四个 running phase 必须分别命中自己的边界：Claim 调用、Submit 开头 EfDrain 专属 call-site、Materialize 唯一调用点，以及每次 Submit 统一的 Register 调用点； +- phase request/miss 分别不超过对应 shadow/primary，且可编程 counter 低于当前 25% 保守风险阈值。 +- 96 个核的 `submit_elapsed_ticks` 都大于 0；running phase 的 `phase_elapsed_ticks` 大于 0 且不超过本核 `submit_elapsed_ticks`，`none` 则必须精确为 0；phase time 状态位和 host 复核都必须通过。 + +`metrics_prof_start/stop()` 在 PMU whole gate 前后各执行一次，其 `PIPE_ALL` 边界会改变流水和多核时序。PMU 结果只与相同构建、相同 phase、相同负载的独立进程比较，不把 PMU 进程的 Submit span 当作无诊断性能基线。 + +## 7. JSON 字段与 AIC/AIV 分析口径 + +当前 `submit-pmu` 输出 schema v5；分析器继续只读兼容历史 schema-v4。`records` 保留 96 个 worker 的 raw，`summary.all/aic/aiv` 分别对 96/32/64 个核统计： + +```text +sum / mean / median / p95 / max +``` + +Submit-all 整窗优先查看： + +- `configuration.submit_span_us`：本轮从第一个 Submit 进入到最后一个 Submit 返回的整体 span；HTML 顶部换算成毫秒展示； +- `total_cycles`：每核 PMU whole gate 内的 64-bit PMU raw total；按角色的 sum 是 core-work，raw 仍保留 mean/median/p95；HTML 只选 mean 表示典型单核，并辅以从逐核记录得到的 min/max，三者均不等于 host 看到的 Submit 墙钟。HTML 按 ALL/AIC/AIV 的实测频率将 raw cycle 换算为单核 cycle-equivalent； +- `scalar_busy`：CNT2 `scalar_instr_busy(0x001)`，表示 scalar instruction busy cycle；依赖返回的 atomic 等待可进入此项，但它不是“纯算术指令数”；换算后的时间也只是 scalar-busy cycle-equivalent； +- `icache_requests` / `icache_misses`：CNT6/CNT7 PMU whole-gate primary； +- `shadow_whole_icache_requests` / `shadow_whole_icache_misses`：闭合或分段 loss 用 shadow whole； +- `shadow_request_loss` / `shadow_miss_loss`：本核 primary-shadow residual； +- `phase_calls` / `phase_icache_requests` / `phase_icache_misses`：选定 phase 的 running read-clear lower； +- `submit_elapsed_ticks`：本 worker 从首个 `submit_begin` 计时点到末个 `submit_end` 计时点的 SYS_CNT 差值；起点位于首个 `BeginCallbackSubmit()` 上下文初始化之后，终点位于末个 Submit 返回之前；1 tick = 1 ns； +- `phase_elapsed_ticks`：所选 phase 所有调用的 SYS_CNT 差值累计；running phase 必须非零且不超过同核 `submit_elapsed_ticks`，`none` 必须为 0； +- `phase_icache_requests_upper_bound` / `phase_icache_misses_upper_bound`：lower 加本核对应 loss； +- `configuration.compiled_phase` 和 `validation.phase_measurement_valid`：确认文件口径。 +- `validation.shadow_primary_match_records` / `shadow_primary_bounded_records`：区分逐值 exact 与单向 bounded 核数； +- `configuration.phase_values_are_running_read_clear_lower_bounds`：确认局部字段是否采用下界语义。 + +HTML 中展示的“非 Scalar-busy 残余/core”严格等于 `(total−scalar_busy)/core`，只用于观察未被 scalar busy 覆盖周期的数量级。它可能同时包含 I-cache refill、同步等待、vector/cube engine 等待和其他流水空隙，不能命名为 Scalar 空闲或 I-cache stall，也不能用它反推单次 miss 代价。 + +离线分析器与 HTML 会从这些 raw 字段继续派生 `phase_icache_request_lower_bound_share_of_submit`、`phase_icache_request_upper_bound_share_of_submit`、`phase_icache_miss_lower_bound_share_of_submit` 和 `phase_icache_miss_upper_bound_share_of_submit`：局部 lower/upper 分别除以同一角色、同一次采集的 PMU whole-gate primary 总数。 + +`phase_observed_read_clear_ratio` 只是 lower miss/lower request 的观测比值；分子和分母各有独立区间，因此它不是实际 phase miss rate 的数学下界。 + +每核平均值按角色求： + +```text +AIC PMU total/core = summary.aic.total_cycles.sum / 32 +AIC scalar/core = summary.aic.scalar_busy.sum / 32 +AIV PMU total/core = summary.aiv.total_cycles.sum / 64 +AIV scalar/core = summary.aiv.scalar_busy.sum / 64 +AIC request/core = summary.aic.icache_requests.sum / 32 +AIC miss/core = summary.aic.icache_misses.sum / 32 +AIV request/core = summary.aiv.icache_requests.sum / 64 +AIV miss/core = summary.aiv.icache_misses.sum / 64 +``` + +PMU cycle 的时间换算通式和默认校准值为： + +```text +time_us = cycles / cycles_per_ns / 1000 +ALL cycles_per_ns = 1.649844 +AIC cycles_per_ns = 1.650062 +AIV cycles_per_ns = 1.649731 +``` + +这里的 `time_us` 是每核 cycle-equivalent。`total−scalar_busy` 即使换算为时间，仍不是 Scalar 空闲时间或 I-cache stall；96 核 sum 换算后也仍是 core-work，不能冒充 Submit 墙钟。 + +`median` 和 `p95` 直接来自同角色逐核 raw 分布，用于观察典型核和高尾核。I-cache miss rate 只按组内加权口径计算： + +```text +AIC miss rate = Σ(AIC miss) / Σ(AIC request) +AIV miss rate = Σ(AIV miss) / Σ(AIV request) +``` + +不平均 32 或 64 个逐核百分比。AIC/AIV 核数不同，比较每核强度时使用 mean/median/p95 或 miss rate，不直接比较两组 sum。 + +局部 phase 的时间和 I-cache 占比都使用组内总量，但分母边界不同： + +```text +time share = Σphase_elapsed_ticks / Σsubmit_elapsed_ticks +request share lower = Σphase_request_lower / Σprimary_request +request share upper = Σphase_request_upper / Σprimary_request +miss share lower = Σphase_miss_lower / Σprimary_miss +miss share upper = Σphase_miss_upper / Σprimary_miss +``` + +时间分子、分母来自同一个 1 ns SYS_CNT，是逐核累计 core-time 构成；不能用 `Σphase_elapsed_ticks` 除以 96 核共同形成的 `submit_span_us`。时间是直接观察单值，不仿造 request/miss 那样的 lower/upper。多个独立进程聚合时，分析器展示每轮上述 `Σ/Σ` 比值的分布，不把不同轮的 raw 重新拼成一次虚构运行。 + +分子与分母必须来自同一个 phase ELF、同一轮采集和同一角色。该比例回答“当前插桩 ELF 中局部窗口占自身对应分母的多少”，不能拿 `claim` 分子除以另一份 `none` 的分母。 + +## 8. HTML 报告与多轮分析命令 + +单份 raw 的 HTML 已由 `submit-pmu` action 自动生成。需要手工重建时： + +```bash +PYTHON=/home/q00473782/.venv/bin/python + +"$PYTHON" ./pmu_html_report.py \ + "$OUT/submit_icache_raw.json" +``` + +默认输出为同目录的 `submit_icache_report.html`；也可用 `-o` 指定路径。生成器先调用 `pmu_sidecar_analyzer.py` 的完整 raw 门禁，只有 96 核拓扑、raw→summary、primary/shadow、采集接受状态和 owner Restore 全部通过才发布 HTML。 + +使用本用户 Python 环境从 raw 重算 host summary，并聚合相同配置的多个独立进程： + +```bash +PYTHON=/home/q00473782/.venv/bin/python + +"$PYTHON" ./pmu_sidecar_analyzer.py \ + "$OUT/submit_icache_raw.json" "$OUT"/capture_*/submit_icache_raw.json +``` + +需要机器可读汇总时： + +```bash +"$PYTHON" ./pmu_sidecar_analyzer.py --json \ + "$OUT/submit_icache_raw.json" "$OUT"/capture_*/submit_icache_raw.json \ + > "$OUT/summary.json" +``` + +分析器会先逐份复算 96 条 raw 与 host summary，然后拒绝聚合下列混用： + +- 任意不同 phase，例如 `none` 与 `claim`，或 `materialize` 与 `register`； +- 不同 schema/build variant； +- 不同 batches、winner mode/count/pattern、selector 或观察开关。 + +建议将每次采集的 JSON 和分析器生成的 summary 放在同一唯一目录；`outputs/` 为本机证据目录，不作为源码提交的一部分。 + +## 9. 如何使用分角色的单次 I-cache miss 标尺 + +### 现行参考：2026-07-22 单子核严格 cold/warm 校准 + +此前统一使用的约 `90 ns/miss` 来自 96-worker mixed launch 的聚合试验。为消除各核同时执行 64 KiB capacity sweep 带来的取指争用，并区分 AIC/AIV 角色，2026-07-22 在提交 `b04f592b` 的干净 detached worktree 中复用 `tests/atomic_probe/ccec/icache_scalar_pmu` 的 PMU owner、host 校验和 cold/warm 配对框架，做了单物理子核校准。该校准没有修改真实 PA 热路，也没有把临时探针接入正式 `submit-pmu` 构建。 + +最终 AIC/AIV ELF 都通过以下运行前硬门禁： + +- target 固定为 `8 B @ 0x0`，按 128 B I-cache line 对齐，只占一个 16 B fetch block； +- 布局固定为 `target -> prepare -> warm harness -> evictor`。AIC/AIV 的 harness 均为 `172 B @ 0x180`，evictor 均为 `65,604 B @ 0x280`； +- 64 KiB 以上的 evictor 同时超过本机模型配置中的 AIC 32 KiB、AIV 16 KiB scalar I-cache 容量； +- warm/cold 交替采用 `WARM,COLD` 和 `COLD,WARM` 顺序，每一对必须落在同一 physical subcore；任何一对不满足 `warm CNT7=0 && cold CNT7=1` 都立即判失败，不进入统计。 + +AIC 与 AIV 各执行一次 101 对试验，合计 404/404 个 cold/warm 样本的 checksum、mode echo、PMU 关窗、physical subcore 和 owner Restore 全部通过；两组都是 101/101 对严格增加一次 CNT7 miss。结果如下： + +| 角色 | physical subcore | 有效配对 | SYS_CNT cold-warm 均值 | SYS_CNT 中位数 / 范围 | PMU total cold-warm 均值 | PMU total 中位数 / 范围 | PMU cycle 按 1.65 GHz 换算 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| AIC | 0 | 101/101 | **77.376 ns** | 77 ns / 76～79 ns | **127.733 cycles** | 128 / 127～130 cycles | 77.414 ns | +| AIV | 18 | 101/101 | **94.030 ns** | 94 ns / 93～96 ns | **155.287 cycles** | 155 / 154～157 cycles | 94.113 ns | + +1 ns/tick 的 SYS_CNT 原始分布为：AIC `76:5, 77:58, 78:33, 79:5`，AIV `93:15, 94:69, 95:16, 96:1`。PMU total cycle 分布为：AIC `127:34, 128:61, 129:5, 130:1`，AIV `154:7, 155:61, 156:30, 157:3`。两套时基换算后的差异均小于 0.1 ns，互相验证了约 1.65 cycles/ns 的当前频率口径。 + +每一对的额外 request 都严格为 4、额外 miss 严格为 1；额外 `scalar_instr_busy(0x1)` 则严格为 0。它证明本试验中的单次回填等待形成 PMU total/scalar-busy gap，而不是增加 scalar busy。request 的 `+4` 与本机模型配置的取指加三条顺序预取相符，但只有 target 所在行产生 CNT7 miss，后续 warm harness 行没有再 miss。 + +因此，当前受控口径不应再把 AIC/AIV 折成同一个 90 ns 常数。对真实 PA 已分别聚合出的角色 miss 数，应计算： + +```text +隔离串行 core-work 等效量(ns) = 77.376 * M_AIC + 94.030 * M_AIV +分角色加权标尺(ns/miss) = + (77.376 * M_AIC + 94.030 * M_AIV) / (M_AIC + M_AIV) +``` + +若仅为了与旧校准比较，假设 32 个 AIC 和 64 个 AIV 每核 miss 数相同，则 worker 加权结果为 `88.479 ns/miss`；对应 PMU 加权均值为 `146.102 cycles/miss`，按 1.65 GHz 为 `88.547 ns/miss`。旧 `90 ns` 相对此口径高 `1.521 ns`，约 `1.72%`。如果真实 PA 的 AIC/AIV miss 比例不是 1:2，就必须使用实际 `M_AIC/M_AIV`，不能套用 `88.5 ns`。 + +这里的“单次 miss 时间”严格限定为：**一个物理子核、容量驱逐后、CNT7 恰好增加 1 的 target 完成时间 cold-warm 增量**。本轮固定采样 physical AIC 0 和 AIV 18，没有轮转全部 96 个活跃子核，因此表中范围是同一子核的时间分布，不是核间 min/max。它也没有控制下一级 refill 命中位置，不等于连续顺序取指的 miss 吞吐，更不是 Submit 已经暴露的墙钟 stall。真实热路中的 compulsory/capacity/conflict miss、预取、同核流水重叠和跨核并行仍可能改变可见代价。 + +### 历史参考：96-worker 聚合得到的约 90 ns + +2026-07-18 的旧 `icache-single` 在同一个 mixed launch 中让 32 个 AIC 和 64 个 AIV 各自执行 64 KiB sweep，再聚合全部 worker。64 trials/core × 10 轮和 128 trials/core × 5 轮都满足： + +```text +cold CNT7 miss == trials +warm CNT7 miss == 0 +miss_delta == 96 * trials +calibrated_cores == 96/96 +``` + +前者 ALL 中位数为 86.596 ns/miss，范围 86.532～86.792；后者中位数为 89.629 ns/miss，范围 89.615～89.648。AIC/AIV 差值方向还随 trials 规模改变，所以这组数据只保留为“96 核并发驱逐条件下”的历史数量级证据，不再用来建立现行角色常数。历史原始日志为： + +```text +pa_scheduler/outputs/pmu_validation/icache_single_64x10_20260718_085929_3232836_console.log +pa_scheduler/outputs/pmu_validation/icache_single_128x5_20260718_090151_3235468_console.log +``` + +当前 `pmu_sidecar_analyzer.py` 和 `pmu_html_report.py` 的单一 `--icache-miss-ns` 默认值仍为 `90.0`，用于保持已生成历史报告和命令接口的口径。本节只记录更严格的分角色校准结果，不静默回填旧 HTML、旧表格或旧 JSON，也不把单一 CLI 参数伪装成已经支持 AIC/AIV 两套常数。 + +无论使用历史 90 ns 还是现行分角色标尺，所得结果都只能称为隔离串行 core-work 等效量。不得从 Submit 墙钟中直接扣除，原因包括： + +- 64 个 AIV、32 个 AIC 之间并行； +- 同一核的 miss、预取、其他流水和等待可能重叠； +- 隔离 cold miss 与真实热路 capacity/conflict/compulsory miss 不一定同价； +- 当前已核实的 A5 事件中没有可直接换算墙钟损失的 I-cache stall-cycle counter。 + +要测真正暴露的性能损失，必须对同语义代码做交错 A/B： + +1. 用相同 `submit-pmu none` 口径确认 `ΔAIC/AIV miss/core`； +2. 另用不开 PMU/泳道的性能构建测 `ΔSubmit span`； +3. 只有第 2 项是实际暴露的墙钟收益；第 1 项只用于证明收益与 I-cache 变化同时出现，分角色单 miss 标尺仅提供数量级解释。 + +### 9.1 历史 b256 `none` 参考数据 + +2026-07-19 用当时的 `submit-pmu none` ELF、`real-compute/6,28,4,1` 在 A5 上采集一轮，全局首末 Submit span 为 `4.750810 ms`。96 核 raw、owner Restore、selector、counter 阈值和离线复算全部通过。该 b256 只作归档数据，不是后续重跑要求。 + +| 指标 | AIC（32 核） | AIV（64 核） | +| --- | ---: | ---: | +| request/core | 408,317.344 | 422,480.609 | +| miss/core | 38,664.344 | 55,098.625 | +| `Σmiss/Σrequest` | 9.4692% | 13.0417% | +| total/core | 7,471,385.531 | 7,085,178.734 | +| total/core 校准等效时间 | 4,527.942 us | 4,294.748 us | +| scalar busy/core | 5,944,751.250 | 5,603,587.469 | +| scalar busy/core 校准等效时间 | 3,602.744 us | 3,396.667 us | +| `Σscalar/Σtotal` | 79.5669% | 79.0889% | +| 非 Scalar-busy 残余/core | 1,526,634.281 | 1,481,591.266 | +| 非 Scalar-busy 残余/core 校准等效时间 | 925.198 us | 898.081 us | +| `miss/core × 90 ns` | 3,479.791 us | 4,958.876 us | + +最后一行只是单核串行等效标尺，不能与 `4.750810 ms` 相减或解释成端到端损失；非 Scalar-busy 残余也不是空闲或 I-cache stall。原始件和自包含报告位于： + +```text +outputs/submit_pmu_none_20260719_b256_final/submit_icache_raw.json +outputs/submit_pmu_none_20260719_b256_final/submit_icache_report.html +``` + +### 9.2 历史 schema-v5 b256 分段时间参考数据 + +2026-07-19 在同一 A5、`real-compute/6,28,4,1`、b256 配置下重新独立采集 `none|claim|efdrain|materialize|register`。五轮均为 96/96 有效记录；四个 running phase 都是 1,280 calls/core，时间均满足 `0 < phase <= 同核 Submit`。这些是历史归档件；后续边界迭代和重采默认只使用 b1。 + +| phase | Submit span | ALL 时间占比 | AIC 时间占比 | AIV 时间占比 | +| --- | ---: | ---: | ---: | ---: | +| `none` | 3.711584 ms | 不适用 | 不适用 | 不适用 | +| `claim` | 4.401747 ms | 12.0845% | 7.7096% | 14.2972% | +| `efdrain` | 3.592376 ms | 15.5068% | 20.2974% | 13.1974% | +| `materialize` | 6.770266 ms | 16.7186% | 15.4860% | 17.3556% | +| `register` | 4.086936 ms | 4.3089% | 3.6256% | 4.6568% | + +每行都只解释自己的诊断 ELF。尤其 Materialize 的运行中边界读取显著改变了该轮 Submit 时序；这些时间占比不能跨行相加，Submit span 也不能与 `none` 相减成局部净开销。对应权威 raw 和加工 HTML 为： + +```text +pa_scheduler/outputs/submit_pmu_phase_time_v5_20260719/none_b256/ +pa_scheduler/outputs/submit_pmu_phase_time_v5_20260719/claim_b256/ +pa_scheduler/outputs/submit_pmu_phase_time_v5_20260719/efdrain_b256/ +pa_scheduler/outputs/submit_pmu_phase_time_v5_20260719/materialize_b256/ +pa_scheduler/outputs/submit_pmu_phase_time_v5_20260719/register_b256/ +``` + +### 9.3 当前边界联动版 b1 门禁 + +2026-07-19 在普通泳道相邻边界收敛后，重新构建五个 `submit-pmu` ELF，并只跑 A5 b1。五轮均使用 `real-compute/6,28,4,1`，都通过 96/96 物理核、真计算输出、mixed 引擎观察、PMU start/stop、owner Restore、phase call/time 和 primary/shadow 门禁： + +| phase | 全局首末 Submit | calls | exact/bounded 核 | request/miss loss | phase 时间占比 | +| --- | ---: | ---: | ---: | ---: | ---: | +| `none` | 113.821 us | 0/0 | 96/96 | 0/0 | 不适用 | +| `claim` | 61.689 us | 480/480 | 80/96 | 7/9 | 22.1744% | +| `efdrain` | 127.969 us | 480/480 | 92/96 | 4/0 | 9.7505% | +| `materialize` | 65.023 us | 480/480 | 96/96 | 0/0 | 41.7821% | +| `register` | 134.714 us | 480/480 | 90/96 | 6/0 | 8.5663% | + +`exact/bounded` 中 exact 表示 shadow 与 primary 逐值相等，bounded 表示满足 `shadow <= primary`。五轮 bounded 都是 96/96，小幅 loss 已进入局部 lower/upper，没有被忽略。 + +该表只证明当前业务语义体与局部 PMU bracket 同步修正后仍严格闭合。b1 的全局 Submit 易受冷启动、多核到达和局部边界扰动影响，不用五行之间的时间差声称 phase 净成本或性能改善。权威 raw 和各自 HTML 位于： + +```text +pa_scheduler/outputs/submit_pmu_boundary_sync_b1_20260719/{none,claim,efdrain,materialize,register}/ +``` + +### 9.4 历史 O1 owner/PMU bring-up 证据 + +现行 `submit-pmu` schema 和 selector 分工建立前，直接 owner 曾用 empty、100,000 scalar NOP、2×100,000 scalar NOP 验证 gate 可以闭合并继续累计；96 核 PMU raw total 中位数约为 214、56,568、112,994。它们只证明链路响应和近似倍增，不表示同数值的纳秒或硬件 cycle。 + +同一历史版本还采过三个独立 b256 `submit-all` PMU-only 进程：Submit span 为 3.688236/4.089057/4.673237 ms，I-cache request 总和约 69.45M～70.07M，miss 总和约 5.83M～5.85M，整体 miss rate 为 8.32%～8.42%。这些旧统一 ELF 仍含当时的诊断代码，且 PMU gate 会改变多核时序；它们只保留为 owner、96 核拓扑和 raw→summary 演进证据，不覆盖 9.1～9.3 的现行构建口径。历史文件为： + +```text +pa_scheduler/outputs/scalar_observation_final_20260718/pmu_submit_all_ccec_b256_v2/pmu_submit_all.json +pa_scheduler/outputs/scalar_observation_final_20260718/pmu_submit_all_ccec_b256_v2_run2/pmu_submit_all.json +pa_scheduler/outputs/scalar_observation_final_20260718/pmu_submit_all_ccec_b256_v2_run3/pmu_submit_all.json +``` + +## 10. 新增局部 phase 的修改清单 + +新 phase 不能只增加一个 CLI 字符串。最小完整修改包括： + +1. `pa_scheduler/common/pa_model.h`：在 `SubmitPmuPhase` 尾部追加稳定 id，不重排已有 `None=0/Claim=1/EfDrain=2/Materialize=4/Register=5`。 +2. `pa_scheduler/ccec/pmu_probe.h`：为 `SubmitPmuPhaseName()` 增加名称映射，并核对 phase status/边界闭合定义。 +3. `pa_scheduler/ccec/build.sh`：在白名单中将 phase 名映射到稳定 `PA_SUBMIT_PMU_PHASE_ID`；先校验名称，再用于输出目录。 +4. `pa_scheduler/run.sh`：同步 `build-submit-pmu/submit-pmu` 的 phase 白名单和 usage。 +5. `pa_scheduler/common/pa_scheduler_core.h`：在真实目标代码段前后放置 `BeginSubmitPmuPhase<...>()` / `EndSubmitPmuPhase<...>()`。必须检查所有早退、winner/loser 和 Alloc/非 Alloc 分支，不得留下只 begin 不 end 的路径。 +6. `pa_scheduler/ccec/host.cpp`：增加该 phase 的预期 calls/begin/end 形状。如果它不是每次 Submit 都调用，不能复用 `batches * 5 * 96`。 +7. `pa_scheduler/pmu_sidecar_analyzer.py`：同步 phase 名/id 和配置指纹，保证不同 phase 输入不会被聚合。 +8. 补充 host/analyzer 回归：`none` 验证 96/96 primary-shadow 精确相等；running phase 验证逐核 bounded、loss/upper 公式、begin/end/calls 和语义，任一 shadow 反向大于 primary 都必须拒绝。新增 phase 的构建、门禁和迭代默认只跑 A5 b1。普通泳道仍复用相邻既有 end，不为了对齐 PMU 而额外增加泳道 `SYS_CNT`。 + +每个 phase 必须是独立 ELF 和独立进程。不为了一次运行得到多个 phase，而在热路加运行时 phase switch 或多组 begin/end。 + +## 11. 常见问题与排错 + +### 提示缺少 submit-pmu 产物 + +确认 phase 名与构建命令一致： + +```bash +./run.sh build-submit-pmu ccec none +./run.sh submit-pmu ccec none --device 0 --batches 1 +``` + +不要用 `./run.sh build ccec` 代替；后者生成的是 `swimlane` 产物。 + +### host 提示 swimlane 构建不能采 PMU + +这表示运行了 `build/ccec/pa_scheduler_host`。应通过 `./run.sh submit-pmu ...` 启动 phase 目录内的 host/kernel/SO 整套产物，不要手工指向根目录 kernel。 + +### shadow miss 始终为 0 + +先检查 selector 是否错把 `0x35` 放到 CNT9。本机 A5 b1 已经反证 CNT9 路径；正式配置应为 CNT5 shadow miss、CNT8 shadow request、CNT9 unused。 + +### `none` 不相等，或任意 phase 出现 `shadow > primary` + +这两种情况都表示观察链路门禁失败，最终 JSON 不应发布。按下列顺序排查： + +1. host/kernel/owner/dispatcher 是否来自同一 phase 目录； +2. owner 读回的 CNT5/CNT6/CNT7/CNT8 selector 是否与期望一致； +3. begin/end 数是否精确相等，是否有早退路径留下 armed phase； +4. 先缩到 b1；`none` 确认 96/96 exact，running phase 确认 96/96 bounded； +5. 检查可编程 counter 是否超过风险门槛。 + +running phase 出现小幅 `shadow < primary` 时，代码显式发布 loss 和局部 lower/upper；这不是 standalone 调度正确性异常。只有协议、数值输出、placement/engine 也失败时，才应转向调度代码排查。 + +### owner 或 Restore 失败 + +不要在同一设备上并发运行另一个 standalone PMU owner、`msprof` PMU 会话或其他会改 selector 的进程。检查 CANN 环境、两个 AArch64 SO 是否在 kernel 同目录，以及 96 个可用 slot/32 个完整 triplet 是否闭合。 + +### JSON 拒绝覆盖 + +每个独立进程使用新文件名，并处理上次失败留下的同名 `.tmp`。host 不会静默覆盖旧证据。 + +### 分析器报配置不一致 + +不要强行合并。逐项比较 phase、batches、winner workload/count/pattern、selector、schema 和 build variant。重采相同配置的独立进程。 + +### miss rate 高，但 Submit 没有同比例变慢 + +这不构成矛盾。miss rate 是事件比例，不是 stall 时间比例；多核、预取、流水重叠和资源等待都会改变实际暴露量。优先看 AIC/AIV 每核 miss、median/p95 和优化前后的 `Δmiss`，实际性能收益仍以无 PMU/泳道的交错 A/B 为准。 diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/MANIFEST.sha256 b/tests/atomic_probe/lazy_lamda_sample/A_original/MANIFEST.sha256 new file mode 100644 index 0000000000..30dd347599 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/MANIFEST.sha256 @@ -0,0 +1,107 @@ +b138d97bfc224656f53c91a4503dddac709ef7d1af12b8d7c015fe80e13c40cc ./README.md +25718360e05d3c2d0963d1381e9dd4dae5fca789244ee4b9f861adcc0cc96218 ./VARIANT +909eff5ca6e3f88d5b077945077864a1557549a3e171bcf233aa236dbd96d660 ./annotate_disassembly.py +1e230f4bb24bf0bfedc154397586e6af5f4773d8c18e147a1b10be3a5e370db3 ./artifacts/measured/artifacts.manifest +fde5c575160293b1a1cf60b330a9605db507650b8c9185fa60f6122563a2da66 ./artifacts/measured/device_text_layout.manifest +97cf327377857bcab61cbca10357a90bc49cd5c389c919873c2f3209d5b01188 ./artifacts/measured/pa_scheduler_host +76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc ./artifacts/measured/pa_scheduler_kernel.o +76c7cac9181a7cddd17b3b38a982cb12aee970e76ef217c64ee3b2dcd70af865 ./artifacts/measured/published.sha256 +1e230f4bb24bf0bfedc154397586e6af5f4773d8c18e147a1b10be3a5e370db3 ./artifacts/rebuilt/artifacts.manifest +fde5c575160293b1a1cf60b330a9605db507650b8c9185fa60f6122563a2da66 ./artifacts/rebuilt/device_text_layout.manifest +97cf327377857bcab61cbca10357a90bc49cd5c389c919873c2f3209d5b01188 ./artifacts/rebuilt/pa_scheduler_host +76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc ./artifacts/rebuilt/pa_scheduler_kernel.o +76c7cac9181a7cddd17b3b38a982cb12aee970e76ef217c64ee3b2dcd70af865 ./artifacts/rebuilt/published.sha256 +4a6421024403f6bafc555b4f3f99d736ecffda2524f2ab874192f25dca85f6fa ./artifacts/runtime_identity.json +c89b2add93ff70181f1c2574835be2948ffa959734962122eafa8c2444f6438a ./build.sh +1e230f4bb24bf0bfedc154397586e6af5f4773d8c18e147a1b10be3a5e370db3 ./build/original/artifacts.manifest +fde5c575160293b1a1cf60b330a9605db507650b8c9185fa60f6122563a2da66 ./build/original/device_text_layout.manifest +f3e62e50400035e5535abd2c74830f1458b862de759a4eadaec0762e8463e5dc ./build/original/pa_scheduler_aic.o +995faf6a73d4f50cfa55b83d3fde4a1db2ad7b43c3d18a480492fa03bed5163a ./build/original/pa_scheduler_aiv.o +97cf327377857bcab61cbca10357a90bc49cd5c389c919873c2f3209d5b01188 ./build/original/pa_scheduler_host +76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc ./build/original/pa_scheduler_kernel.o +b0f6647a5034e448ebfe216cedef6eb0ce332b7894f5ef4d382d4179c067efb2 ./build/published.sha256 +02c7d7ad77a8fab471c9f1b2a215029b9b407886699534076a9f48d178db0140 ./ccec/build.sh +ea82967763894d2726b68b19e74c2993ffe938f5bb0c90b92d3eae0966e643b7 ./ccec/callback_finish.cpp +7090c8405e1602476d1edf4ae6103ec6ec145e22bf15db742eac06aa2a6b7f9f ./ccec/callback_finish_api.h +4cef27650943d3973c6d56db687850ea3a7cbe28d64bf666b55cf735ce5bd141 ./ccec/callback_runtime_entry.cpp +47ca7623791bdd408b50c5e5ece4d63c7073775a1e3b636e046419acf948ddda ./ccec/ccec_ops.h +ac8a9766aa71a2bfc2a21c9021fc9fc30bc869575daa0e0f01121789230173e2 ./ccec/host.cpp +766ec34d6d4a7d80d07774782c68d701653cadf1bd8427fd1a0e922933c10236 ./ccec/kernel.cpp +f88d5ed31277c2692ecec2e8f5528cb6c2de752aea6a45850ab72d9e9b600075 ./ccec/pa_scheduler_device_exports.map +3d65b4a89dcf88554321a5435a2abf701a26757e8c229df7ac9783abbb5b96c3 ./ccec/pmu_owner_control.h +cb81c2a24670d893fd485845f13b8b106bfffe28d8c471f92a93775eee1a69e6 ./ccec/pmu_owner_host.h +09263e6698598f7be5b16b92e9e34bc59719de54aee132c43df24cec0ce42b4a ./ccec/pmu_owner_main_abi.h +9115d77d1be306c9f3b225805ed3df63fbdbdadcb9b4ff6ad690b56cfcf58867 ./ccec/pmu_owner_main_loader.h +2a926b355f78d4758fdc61a54827baec54ef781df34234a2edc9aa6a1a824f9e ./ccec/pmu_probe.h +3e3c6c99df27ff9e329242c64680559bf69cb4dc96650acc0ba1fe7442d68c70 ./common/host_support.h +f4a22840aef81ec05fb9532559957712c76ebd2dd1199d6969fc99688fc2e63d ./common/pa_frontend.h +9d42688f933e7b20eaab9eadcec40c3f0c0babc8ebdd3dd7369a2afcca1edbec ./common/pa_model.h +6645ce2aefe39b6132ce649c6af4c597a7a6978aa8983375fe09c4e186a812e5 ./common/pa_scheduler_core.h +8fd7c429dbbf0aae4e37968e7f90d953793f30f9caeb18eba515f9201deca1fa ./common/pa_trace.h +9d099151d0303c37dbb88f5cd2b15ee483dc0623759c45cbcf9d80c37704a80d ./common/winner_workload.h +8be384a62ed669e5645139aab1528570ffab184b5856f685e8056eed68e1e5cc ./common/winner_workload_host.h +4091251537a0405f83514d99fc6891300650a01cb61ac3b6c2291e899f7f0747 ./disassemble.py +7b364d19466391c4048afc9492dbb8bea3665617b03c639fb26cbb5f90c45036 ./disassembly/annotated/00_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz +59c9042ea410ea8c4e1e62da7fd197a95520e3bd3e102b4d9fd6411c430c7b5e ./disassembly/annotated/01_pa_execute_real_winner_workload_aic_729699de.source.asm.gz +f86aa6d3ab5dc84aa897e8d92401f33f15bae8281ded890a4931b6a525637950 ./disassembly/annotated/02_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz +4ce297449c240be11b6aa00e125309eed2ad22d1ffdd8a8885d49794b1dec607 ./disassembly/annotated/03_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +aee6acc662edb138e7fa40e762d1c340038114c8b47643e7d5a7514b60deb149 ./disassembly/annotated/04_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz +60211d286d798c524b76eaedec97a407d04ab330241a119ac53c87a198f8c952 ./disassembly/annotated/05_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz +d71f1cebcabe69d49e515faa3b4f47bf3254233cc63e3c36561b92aa5e310362 ./disassembly/annotated/06_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz +e480aae49727c8463471bcb3e92c8db8c75f37792437fe587668b08d5531e9b5 ./disassembly/annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz +bf66fc8e3a91bf6efa3b751a087af4c0728810566e85b13c690ad3841d2411f2 ./disassembly/annotated/08_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +e9ec66d6f2729262f19deed5afd0eaf4c321b893987ad5a8e7b5de9050de0054 ./disassembly/annotated/09_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz +e88e6a49d249eb7f00de9353c70af43f023e3a721495b7d724935c38da462254 ./disassembly/annotated/10_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz +3468940354f71bc3619221a8298e72523f01d052c19210063c02db991b805bd4 ./disassembly/key_flow/aic_original_submit.source.asm +c26b91b5b02e1ec87bd7bbcd01bab16f26a1a4de27e6a31ccc3a3e2a7c1193fa ./disassembly/published.sha256 +934a467bb8aae45fbfb7f13ae26823117645ab82760552aefb706431bacf2f24 ./disassembly/raw/00_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz +d801ffe741f89b78386383caf8dce91940dbf62f3983621b043ec1c04bda7933 ./disassembly/raw/01_pa_execute_real_winner_workload_aic_729699de.asm.gz +2b536564a5f6cacdc634ecbc4ed1a85eeb93d41290a3033e0b5375d5e2e6b8dc ./disassembly/raw/02_pa_scheduler_0_mix_aic_4f68fde0.asm.gz +cc502ce9041184fe05d07c064d19c27204c30dcadb656a82bcda2e4b4d45f3b5 ./disassembly/raw/03_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +789b75b419e9d9608b56d07e73479edaa6c221320d235c5bf6aab51ba5035387 ./disassembly/raw/04_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz +0f98b856714ff292b423d48fd84e5cdfac96ecca180ba891d1eb7830267619b5 ./disassembly/raw/05_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz +1f365c1a8bf10b269bff2095f96d420783ce1d4bd99df542e60659ea18f4a27e ./disassembly/raw/06_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz +9ceb3d65e13277114aa9c0de5efc771e452577a46d13e6f74c7c241db58e0c54 ./disassembly/raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz +da169141e544c5220e87673d3e829a2c509be23db49de4934e1c96fd1e044018 ./disassembly/raw/08_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +f66e5b7f5071add1f390a4f52a10c7bd649a08565477f0f5d1c8bacf0e49fac4 ./disassembly/raw/09_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz +bce988226efb1bafb961f4a8ef56ca2c8d269733b9ba38d43ba45b1f4e79e437 ./disassembly/raw/10_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz +27b1273b5e461ef004bdfaa743cde304827256e40d265917dcd798b3123eaefd ./disassembly/raw/README.md +75fad81fef86cadf69767382cc10bf436529f6f61c0e320c883b63e1e885d047 ./disassembly/raw/gaps.tsv +a4a13b0d14a99744153bc9a4baeacd9c61fea73c690c044c2596aa89d0b14370 ./disassembly/raw/manifest.tsv +564c95bd7f07ae167ac9c3f17290402cf77b82ae1df973ef6e3c931c9ade4184 ./disassembly/raw/published.sha256 +912a11b6a1241027ee5da46b1c545dcedc47e223de498490e6589c35be0dd611 ./generate_disassembly.sh +d392ab37d0ee4684d818fbf42b7f04e3fc54ff03485883b0b8b963d48c52d269 ./output/functional.txt +f98ffa1c7ee34859f5ee83ffe5ef8a1430846d0809b5051672256647162ead7d ./output/performance/RUN_METADATA.md +7ec7682e381951f8472821951193fb90f99f86d39273a3e2103ed7aebcdc082d ./output/performance/SUMMARY.md +5e8b3174b43ae4084533e09e141047ca5d15225cb2e34f5d8b8ea7d795421bae ./output/performance/raw/block_01_ABC_pos1_original.txt +231422a75f8eaff466923c73666cf53432677e9db990fe20fc41ce6985eb0ddd ./output/performance/raw/block_02_BCA_pos3_original.txt +34c7de1c02da900e493bc36e83ef927c8f03ad1c2ae2b486c61e8656e6afa300 ./output/performance/raw/block_03_CAB_pos2_original.txt +1936d15a4541f54f496f83b595833ae0cf43dfa8477aa1fc18f7c637f741f301 ./output/performance/raw/block_04_ACB_pos1_original.txt +375af2bcc02f3ebc80319e215e0e5591c2f1d9ca57c274cee173cbf070ab2bf0 ./output/performance/raw/block_05_BAC_pos2_original.txt +7a055adba803a5b6acee7c203d700e4fa7544a1a4d8c5aa4665664fc7e590f8e ./output/performance/raw/block_06_CBA_pos3_original.txt +2d17bf2d85f2e03c78ef3205cf72d32056d2e10bb49ee88a7ca1ed857ede7e5c ./output/performance/raw/block_07_CBA_pos3_original.txt +3b024b97937fea83406ac856954e475078f21cc7344f5a3c9edcb9d38de4a2fb ./output/performance/raw/block_08_BAC_pos2_original.txt +f9a4e204ad985bda15447ab05e56c435245c66124a4e8f13389b3205f5531c63 ./output/performance/raw/block_09_ACB_pos1_original.txt +735190d3c5958b00527ef09b072df4a12a78e263d0d0343ececb36bc8e3cc353 ./output/performance/raw/block_10_CAB_pos2_original.txt +2bc738fea8e20f4b9b66b2a896689a14835f8d6dad1c345b61af8032ed506e7c ./output/performance/raw/block_11_BCA_pos3_original.txt +9d951914bac426b6f82ef80aa0cd2bcd969c2fecc6dd76e10e944611fb48b91f ./output/performance/raw/block_12_ABC_pos1_original.txt +8b69729bc7bedc161ed94a0685bab4c8f117eba2cb0aad463def139a94cec354 ./output/performance/raw/block_13_BCA_pos3_original.txt +1bf1626d008161b09fa0d0d1b7c55d9c39f6bc445bfd7e0044e19109d4921cf4 ./output/performance/raw/block_14_ACB_pos1_original.txt +db4d7d1e5cd8ebc6743998646999dcae97da6c7c7271d060b8fbfb8e02325fde ./output/performance/raw/block_15_CBA_pos3_original.txt +edc3f444992bb11619a4e42c5a3cd86b9082a58b6d2821dbf0ae37680fef3f83 ./output/performance/raw/block_16_ABC_pos1_original.txt +c4ce09472727f0e382df96143aa879dc5aa90991cfd416161273499723dc8609 ./output/performance/raw/block_17_CAB_pos2_original.txt +765716ec3e48da56fa67f2d1fbee8fb0a5ee1728bc2be0ddb9f6e46c8fd134f8 ./output/performance/raw/block_18_BAC_pos2_original.txt +62f70f90a38fde497fb201734b399317d9b54004838fa58db88b74ee43495f38 ./output/performance/raw/block_19_BAC_pos2_original.txt +3f21404390fa29fd2214893eaf27b38f0cd5abdbdef48a4d76757a977c9b6804 ./output/performance/raw/block_20_CAB_pos2_original.txt +f8ab313f7a045073ac7fcff3501695bbcaab2607455914ba4a1ca387f8523bff ./output/performance/raw/block_21_ABC_pos1_original.txt +0127939f3de25c2dce0e6867587d7a1a2add1878cd24113e050d10c8f83cf09a ./output/performance/raw/block_22_CBA_pos3_original.txt +9596448e024b7f28058714ea611d6f036b41e7489a970fc3dcba41d9c25683eb ./output/performance/raw/block_23_ACB_pos1_original.txt +59fe17e2f2ad07058f52476ab50b70273e0394e8c972e9531a1910743655c86b ./output/performance/raw/block_24_BCA_pos3_original.txt +9f3a85c11fa87a62b12b836a20eb450136f582d453c2ee663f8a07a7574929fe ./output/performance/samples.tsv +0f147aa40d92070aabcf8e618e0e6cf4dacc56f97e606a0330c6e64356055fcc ./output/performance/summary.json +1f5fe5f824469fe05f52f9f62926d3eb7ac00c1e1a980210913019ad3aa3e86f ./output/published.sha256 +7d3dc51260b3dfa0855b7d2ff6c343eac73e07b915c92b3362a9954c860facfc ./publish_artifacts.sh +33180b0902d7376fa9129f245115e61e75acc7ec4fd9d3231085a232f85054d7 ./run_functional.sh +4586375705279003c6aa610079f03fa6b54568975165eeff760cd5c4e85a4ee7 ./source.sha256 +72471b0594aee1069f9fc774faa6d5900e374d3a283fdccc12b2c26a1302472a ./summarize_performance.py +e50378f7afdb15e3a84207b029e2d0aced10e1ac5a363372a791a0a8d177c937 ./verify_runtime_identity.py diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/README.md b/tests/atomic_probe/lazy_lamda_sample/A_original/README.md new file mode 100644 index 0000000000..e6112d6d87 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/README.md @@ -0,0 +1,51 @@ +# A：original eager + +本目录是一份可单独复制、构建和查看的原始版样例，不依赖 B/C 目录或上级目录中的源码、脚本与二进制。 +完整 task 流为 `Alloc -> QK -> SF -> PV -> UP`，32 AIC + 64 AIV。 + +## 固定形态 + +- `VARIANT=original`；`ccec/build.sh` 会拒绝其他 shape； +- 不定义 `PA_LAZY_SAMPLE_SHAPE_ID`，不定义 `PA_LAZY_SAMPLE_SPLIT_FINISH`； +- `PA_BUILD_SWIMLANE=1`、`PA_BUILD_SUBMIT_PMU=0`，运行时显式 `--no-swimlane`； +- 每个 task 先在 Submit 外执行 eager `Build*Args`，再调用原始 `SubmitTask`； +- `SubmitTask` 内为 `EfDrain -> Materialize -> TensorMap retire -> Claim`，之后保持原始的 + winner fanin、全员 register、winner Build/Complete 流程。Alloc 的 register/Claim 次序见源码注释。 + +主路径源码在 `common/pa_scheduler_core.h` 的 `SubmitTask` 和默认 replay 分支；固定构建参数在 +`build.sh` 与 `ccec/build.sh` 顶部。未删除的 inactive `#if` 分支只用于保留来源上下文，固定构建入口无法选择它。 + +## 独立交付内容 + +- `common/`、`ccec/`：本版完整源码； +- `build/original/`：清空历史目录后生成的完整本地编译结果和中间对象； +- `artifacts/measured/`：正式 24 样本性能测试实际使用的 host/final ELF； +- `artifacts/rebuilt/`：当前同一 clean-build 的发布副本； +- `artifacts/runtime_identity.json`：两套 ELF 的运行时节、NOBITS 布局、运行时符号和 host 身份校验； +- `disassembly/raw/`：final-linked 机器码权威解码； +- `disassembly/annotated/`：每个函数的完整源码注释版反汇编； +- `disassembly/key_flow/aic_original_submit.source.asm`：便于直接阅读的未压缩关键窗口； +- `output/functional.txt`：最新 device0 b1 功能门禁； +- `output/performance/`：24 次独立 b256 原始日志、运行 provenance、样本表和统计； +- `source.sha256`、`build/published.sha256`、`output/published.sha256`、 + `MANIFEST.sha256`：源码、干净构建、输出与整包校验入口。 + +Measured final `.text` 为 `780344 B`，SHA256 +`018ae7dc29ce78249b2e3bff84b0faa2e671e2448c9316719fb48bc59139d2fc`。 +去异常 Submit span 中位数为 `3956.298 us`（原始 `24` 个样本，保留 `22` 个）。 + +## 复跑 + +先 source CANN 9.1 环境，使 `ASCEND_HOME_PATH` 有效： + +```bash +./build.sh # 先精确删除 build/original，再完整构建 +./publish_artifacts.sh # 只刷新 artifacts/rebuilt +./verify_runtime_identity.py # 对比 measured/rebuilt 的运行时身份 +./run_functional.sh # device0、b1、PMU off、--no-swimlane +./generate_disassembly.sh # 从 measured ELF 重建 raw + annotated 全量反汇编 +./summarize_performance.py # 重新校验本目录 24 个独立正式样本 +``` + +`*.source.asm.gz` 用 `zless` 阅读。注释来自 DWARF 行号回查本目录源码；注释不是 ELF 中保存的文本， +文件头已明确标注这一证据边界。 diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/VARIANT b/tests/atomic_probe/lazy_lamda_sample/A_original/VARIANT new file mode 100644 index 0000000000..4b48deed3a --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/VARIANT @@ -0,0 +1 @@ +original diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/annotate_disassembly.py b/tests/atomic_probe/lazy_lamda_sample/A_original/annotate_disassembly.py new file mode 100755 index 0000000000..c6eb096a1a --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/annotate_disassembly.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Add DWARF-mapped source and local source comments to one raw disassembly.""" + +from __future__ import annotations + +import argparse +import gzip +import os +import re +import subprocess +from pathlib import Path + + +INSTRUCTION_RE = re.compile(r"^(0x([0-9a-fA-F]+) \(\+0x[0-9a-fA-F]+\).*)$") +LOCATION_RE = re.compile(r"^(.*):(\d+)(?: \(discriminator \d+\))?$") +ANCHOR_RE = re.compile(r"^(.*):(\d+)$") + + +def read_text(path: Path) -> str: + if path.suffix == ".gz": + with gzip.open(path, "rt", encoding="utf-8") as source: + return source.read() + return path.read_text(encoding="utf-8") + + +def normalize_source_path(path_text: str, source_root: Path) -> tuple[Path | None, str]: + if path_text in {"??", ""}: + return None, "??" + # Some CCEC line-table rows are rendered as /path/file:?:0. The final + # :0 is parsed as the line number, leaving :? attached to the path. + if path_text.endswith(":?"): + path_text = path_text[:-2] + candidate = Path(path_text) + normalized = Path(candidate.as_posix().replace("/ccec/../common/", "/common/")) + parts = normalized.parts + if "lazy_lamda_sample" in parts: + marker = len(parts) - 1 - list(reversed(parts)).index("lazy_lamda_sample") + tail = parts[marker + 1 :] + source_index = next( + (index for index, part in enumerate(tail) if part in {"common", "ccec"}), + None, + ) + if source_index is not None: + suffix = Path(*tail[source_index:]) + local = source_root / suffix + return (local if local.is_file() else None), suffix.as_posix() + if candidate.is_file(): + ascend_home = os.environ.get("ASCEND_HOME_PATH") + if ascend_home: + try: + relative = candidate.resolve().relative_to(Path(ascend_home).resolve()) + except ValueError: + pass + else: + return candidate, f"$ASCEND_HOME_PATH/{relative.as_posix()}" + return candidate, candidate.as_posix() + return None, candidate.as_posix() + + +def source_lines(path: Path | None, cache: dict[Path, list[str]]) -> list[str] | None: + if path is None: + return None + if path not in cache: + cache[path] = path.read_text(encoding="utf-8").splitlines() + return cache[path] + + +def write_output(path: Path, text: str) -> None: + if path.suffix == ".gz": + with path.open("wb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed: + compressed.write(text.encode("utf-8")) + return + path.write_text(text, encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser() + script_dir = Path(__file__).resolve().parent + parser.add_argument("--elf", type=Path, required=True) + parser.add_argument("--raw", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--source-root", type=Path, default=script_dir) + parser.add_argument("--anchor", help="First DWARF location matching SUFFIX:LINE") + parser.add_argument("--before", type=int, default=40) + parser.add_argument("--after", type=int, default=160) + args = parser.parse_args() + + raw_lines = read_text(args.raw.resolve()).splitlines() + instruction_lines = [line for line in raw_lines if INSTRUCTION_RE.match(line)] + if not instruction_lines: + raise SystemExit(f"No instruction rows in {args.raw}") + pcs = [int(INSTRUCTION_RE.match(line).group(2), 16) for line in instruction_lines] + process = subprocess.run( + ["addr2line", "-e", str(args.elf.resolve()), "-C"], + input="".join(f"0x{pc:x}\n" for pc in pcs), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + locations = process.stdout.splitlines() + if len(locations) != len(instruction_lines): + raise SystemExit( + f"addr2line returned {len(locations)} locations for {len(instruction_lines)} instructions" + ) + + normalized_locations: list[tuple[Path | None, str, int]] = [] + source_root = args.source_root.resolve() + for location in locations: + if location.endswith(":?"): + local_path, display = normalize_source_path(location[:-2], source_root) + normalized_locations.append((local_path, display, 0)) + continue + match = LOCATION_RE.match(location) + if match is None: + normalized_locations.append((None, location, 0)) + continue + local_path, display = normalize_source_path(match.group(1), source_root) + normalized_locations.append((local_path, display, int(match.group(2)))) + + start = 0 + end = len(instruction_lines) + if args.anchor: + anchor_match = ANCHOR_RE.match(args.anchor) + if anchor_match is None: + raise SystemExit("--anchor must be SUFFIX:LINE") + suffix = anchor_match.group(1) + anchor_line = int(anchor_match.group(2)) + anchor_index = next( + ( + index + for index, (_, display, line) in enumerate(normalized_locations) + if display.endswith(suffix) and line == anchor_line + ), + None, + ) + if anchor_index is None: + raise SystemExit(f"DWARF anchor not found: {args.anchor}") + start = max(0, anchor_index - args.before) + end = min(len(instruction_lines), anchor_index + args.after + 1) + + headers = [line for line in raw_lines if line.startswith("#")] + output = headers + [ + "# annotation_schema=pa_source_annotated_disassembly/v1", + "# annotation_rule=DWARF supplies only file:line; SOURCE rows are copied from local source files", + "# annotation_warning=comments have source context only and do not own an exact machine address", + f"# annotation_instruction_slice={start}:{end}", + "#", + ] + cache: dict[Path, list[str]] = {} + previous_display = "" + previous_line = 0 + for index in range(start, end): + local_path, display, line_number = normalized_locations[index] + if display != previous_display or line_number != previous_line: + lines = source_lines(local_path, cache) + output.append(f"# [DWARF] {display}:{line_number}") + if lines is None or line_number <= 0 or line_number > len(lines): + output.append("# [SOURCE unavailable]") + else: + if display == previous_display and previous_line < line_number <= previous_line + 12: + context_start = previous_line + 1 + else: + context_start = max(1, line_number - 6) + for source_line in range(context_start, line_number + 1): + marker = ">" if source_line == line_number else " " + output.append(f"# {marker} {source_line:5d} | {lines[source_line - 1]}") + previous_display = display + previous_line = line_number + output.append(instruction_lines[index]) + + args.output.parent.mkdir(parents=True, exist_ok=True) + write_output(args.output, "\n".join(output) + "\n") + print( + f"wrote {args.output}: instructions={end - start} " + f"range=0x{pcs[start]:x}..0x{pcs[end - 1]:x}" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/artifacts.manifest b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/artifacts.manifest new file mode 100644 index 0000000000..4ea3c41ec7 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/artifacts.manifest @@ -0,0 +1,10 @@ +# schema=pa_scheduler_lazy_sample_callback_artifacts/v1 +# backend=ccec +# shape=original +# shape_id=0 +# observation=legacy-straight-line-eager +97cf327377857bcab61cbca10357a90bc49cd5c389c919873c2f3209d5b01188 pa_scheduler_host +76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc pa_scheduler_kernel.o +f3e62e50400035e5535abd2c74830f1458b862de759a4eadaec0762e8463e5dc pa_scheduler_aic.o +995faf6a73d4f50cfa55b83d3fde4a1db2ad7b43c3d18a480492fa03bed5163a pa_scheduler_aiv.o +fde5c575160293b1a1cf60b330a9605db507650b8c9185fa60f6122563a2da66 device_text_layout.manifest diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/device_text_layout.manifest b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/device_text_layout.manifest new file mode 100644 index 0000000000..6f6feeef97 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/device_text_layout.manifest @@ -0,0 +1,15 @@ +# schema=pa_scheduler_device_text_layout/v1 +# backend=ccec +# shape=original +# block_local_reserve_bytes=0 +# compiler=$ASCEND_HOME_PATH/bin/ccec +# compiler_sha256=34355dde8c995f3c7d769ddf0e1b853aba65cf6ef6fb3863ab86c026d40aebfd +# linker=$ASCEND_HOME_PATH/bin/ld.lld +# linker_sha256=727810dd7f72dedbe1453edeb81d3a5371d8dc0591bc58693edf1aa7f3fa7eff +text pa_scheduler_kernel.o 780344 018ae7dc29ce78249b2e3bff84b0faa2e671e2448c9316719fb48bc59139d2fc +text pa_scheduler_aic.o 387864 a0e871f4c8c3046dd77fba85977b38d81a453e53922e52654de8e871bf79989a +text pa_scheduler_aiv.o 392248 101cc6613e9028cec74d6533e7668d8beffa6f9ec0f1bdd2893eac18cc28d53b +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aic 387044 d7337df3d1786f2236031aaa4d252ca82696e3a563e11ce56d9b98495bbf1aea +symbol pa_scheduler_aic.o pa_scheduler_0_mix_aic 387044 9306a0d3ad16d74f736e6e5e62198311e4cf74f42b9a3b075c3b40624b5ef38d +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aiv 390516 67b4fd4c11780b2730e10a0f1b124481f96b2ea1c548d626ff45b5d67b807833 +symbol pa_scheduler_aiv.o pa_scheduler_0_mix_aiv 390516 b6d23dc6d25a3eff57deea579f08910b91cfab662c3c7348a7e9155df2a310d2 diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/pa_scheduler_host b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/pa_scheduler_host new file mode 100755 index 0000000000..f740e4afc2 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/pa_scheduler_host differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/pa_scheduler_kernel.o b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/pa_scheduler_kernel.o new file mode 100755 index 0000000000..53ab09e9e7 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/pa_scheduler_kernel.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/published.sha256 new file mode 100644 index 0000000000..7246113350 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/measured/published.sha256 @@ -0,0 +1,4 @@ +1e230f4bb24bf0bfedc154397586e6af5f4773d8c18e147a1b10be3a5e370db3 artifacts.manifest +fde5c575160293b1a1cf60b330a9605db507650b8c9185fa60f6122563a2da66 device_text_layout.manifest +97cf327377857bcab61cbca10357a90bc49cd5c389c919873c2f3209d5b01188 pa_scheduler_host +76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc pa_scheduler_kernel.o diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/artifacts.manifest b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/artifacts.manifest new file mode 100644 index 0000000000..4ea3c41ec7 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/artifacts.manifest @@ -0,0 +1,10 @@ +# schema=pa_scheduler_lazy_sample_callback_artifacts/v1 +# backend=ccec +# shape=original +# shape_id=0 +# observation=legacy-straight-line-eager +97cf327377857bcab61cbca10357a90bc49cd5c389c919873c2f3209d5b01188 pa_scheduler_host +76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc pa_scheduler_kernel.o +f3e62e50400035e5535abd2c74830f1458b862de759a4eadaec0762e8463e5dc pa_scheduler_aic.o +995faf6a73d4f50cfa55b83d3fde4a1db2ad7b43c3d18a480492fa03bed5163a pa_scheduler_aiv.o +fde5c575160293b1a1cf60b330a9605db507650b8c9185fa60f6122563a2da66 device_text_layout.manifest diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/device_text_layout.manifest b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/device_text_layout.manifest new file mode 100644 index 0000000000..6f6feeef97 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/device_text_layout.manifest @@ -0,0 +1,15 @@ +# schema=pa_scheduler_device_text_layout/v1 +# backend=ccec +# shape=original +# block_local_reserve_bytes=0 +# compiler=$ASCEND_HOME_PATH/bin/ccec +# compiler_sha256=34355dde8c995f3c7d769ddf0e1b853aba65cf6ef6fb3863ab86c026d40aebfd +# linker=$ASCEND_HOME_PATH/bin/ld.lld +# linker_sha256=727810dd7f72dedbe1453edeb81d3a5371d8dc0591bc58693edf1aa7f3fa7eff +text pa_scheduler_kernel.o 780344 018ae7dc29ce78249b2e3bff84b0faa2e671e2448c9316719fb48bc59139d2fc +text pa_scheduler_aic.o 387864 a0e871f4c8c3046dd77fba85977b38d81a453e53922e52654de8e871bf79989a +text pa_scheduler_aiv.o 392248 101cc6613e9028cec74d6533e7668d8beffa6f9ec0f1bdd2893eac18cc28d53b +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aic 387044 d7337df3d1786f2236031aaa4d252ca82696e3a563e11ce56d9b98495bbf1aea +symbol pa_scheduler_aic.o pa_scheduler_0_mix_aic 387044 9306a0d3ad16d74f736e6e5e62198311e4cf74f42b9a3b075c3b40624b5ef38d +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aiv 390516 67b4fd4c11780b2730e10a0f1b124481f96b2ea1c548d626ff45b5d67b807833 +symbol pa_scheduler_aiv.o pa_scheduler_0_mix_aiv 390516 b6d23dc6d25a3eff57deea579f08910b91cfab662c3c7348a7e9155df2a310d2 diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/pa_scheduler_host b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/pa_scheduler_host new file mode 100755 index 0000000000..f740e4afc2 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/pa_scheduler_host differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/pa_scheduler_kernel.o b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/pa_scheduler_kernel.o new file mode 100755 index 0000000000..53ab09e9e7 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/pa_scheduler_kernel.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/published.sha256 new file mode 100644 index 0000000000..7246113350 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/rebuilt/published.sha256 @@ -0,0 +1,4 @@ +1e230f4bb24bf0bfedc154397586e6af5f4773d8c18e147a1b10be3a5e370db3 artifacts.manifest +fde5c575160293b1a1cf60b330a9605db507650b8c9185fa60f6122563a2da66 device_text_layout.manifest +97cf327377857bcab61cbca10357a90bc49cd5c389c919873c2f3209d5b01188 pa_scheduler_host +76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc pa_scheduler_kernel.o diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/runtime_identity.json b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/runtime_identity.json new file mode 100644 index 0000000000..627ef58303 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/artifacts/runtime_identity.json @@ -0,0 +1,51 @@ +{ + "full_elf_equal": true, + "host_equal": true, + "host_sha256": "97cf327377857bcab61cbca10357a90bc49cd5c389c919873c2f3209d5b01188", + "measured_elf_sha256": "76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc", + "nobits_layout": [], + "note": "Measured is the exact performance binary; full ELF and runtime identities are checked explicitly.", + "rebuilt_elf_sha256": "76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc", + "runtime_content_sections": [ + { + "content_equal": true, + "metadata_equal": true, + "name": ".text", + "sha256": "018ae7dc29ce78249b2e3bff84b0faa2e671e2448c9316719fb48bc59139d2fc", + "size": 780344 + }, + { + "content_equal": true, + "metadata_equal": true, + "name": ".rodata", + "sha256": "a3aab0297c4ff5ee3592c078b30f69bdfd6ad5e637115b21063589bd2e310164", + "size": 512 + }, + { + "content_equal": true, + "metadata_equal": true, + "name": ".ascend.meta.pa_scheduler_0_mix_aic", + "sha256": "21d5aa86fefde2c6c9264a0ddde378b67951e3c0932a81c1eaa38e52659aa40f", + "size": 24 + }, + { + "content_equal": true, + "metadata_equal": true, + "name": "__CCE_KernelArgSize", + "sha256": "c0cdbb6e45249a718e7e605f801cf43f673a22fcf0347034da71f6be2e4f4d5e", + "size": 8 + }, + { + "content_equal": true, + "metadata_equal": true, + "name": ".ascend.meta.pa_scheduler_0_mix_aiv", + "sha256": "71dbf120b5acf5cb4b7ba8be62e1c558c80a1627f2d1f9d93380b7e87536a1c9", + "size": 56 + } + ], + "runtime_symbol_count": 14, + "runtime_symbols_equal": true, + "schema": "pa_lazy_lambda_runtime_identity/v1", + "status": "PASS", + "variant": "original" +} diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/build.sh b/tests/atomic_probe/lazy_lamda_sample/A_original/build.sh new file mode 100755 index 0000000000..10d3ed8f0c --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +VARIANT="original" +BUILD_DIR="$SCRIPT_DIR/build/$VARIANT" + +# This package deliberately owns exactly one build directory. Remove the +# complete old directory so stale objects can never survive a rebuild. +if [[ "$BUILD_DIR" != "$SCRIPT_DIR/build/original" ]]; then + echo "Refusing to clean an unexpected build path: $BUILD_DIR" >&2 + exit 1 +fi +if [[ -e "$BUILD_DIR" ]]; then + rm -rf -- "$BUILD_DIR" +fi + +"$SCRIPT_DIR/ccec/build.sh" "$VARIANT" diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/artifacts.manifest b/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/artifacts.manifest new file mode 100644 index 0000000000..4ea3c41ec7 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/artifacts.manifest @@ -0,0 +1,10 @@ +# schema=pa_scheduler_lazy_sample_callback_artifacts/v1 +# backend=ccec +# shape=original +# shape_id=0 +# observation=legacy-straight-line-eager +97cf327377857bcab61cbca10357a90bc49cd5c389c919873c2f3209d5b01188 pa_scheduler_host +76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc pa_scheduler_kernel.o +f3e62e50400035e5535abd2c74830f1458b862de759a4eadaec0762e8463e5dc pa_scheduler_aic.o +995faf6a73d4f50cfa55b83d3fde4a1db2ad7b43c3d18a480492fa03bed5163a pa_scheduler_aiv.o +fde5c575160293b1a1cf60b330a9605db507650b8c9185fa60f6122563a2da66 device_text_layout.manifest diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/device_text_layout.manifest b/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/device_text_layout.manifest new file mode 100644 index 0000000000..6f6feeef97 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/device_text_layout.manifest @@ -0,0 +1,15 @@ +# schema=pa_scheduler_device_text_layout/v1 +# backend=ccec +# shape=original +# block_local_reserve_bytes=0 +# compiler=$ASCEND_HOME_PATH/bin/ccec +# compiler_sha256=34355dde8c995f3c7d769ddf0e1b853aba65cf6ef6fb3863ab86c026d40aebfd +# linker=$ASCEND_HOME_PATH/bin/ld.lld +# linker_sha256=727810dd7f72dedbe1453edeb81d3a5371d8dc0591bc58693edf1aa7f3fa7eff +text pa_scheduler_kernel.o 780344 018ae7dc29ce78249b2e3bff84b0faa2e671e2448c9316719fb48bc59139d2fc +text pa_scheduler_aic.o 387864 a0e871f4c8c3046dd77fba85977b38d81a453e53922e52654de8e871bf79989a +text pa_scheduler_aiv.o 392248 101cc6613e9028cec74d6533e7668d8beffa6f9ec0f1bdd2893eac18cc28d53b +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aic 387044 d7337df3d1786f2236031aaa4d252ca82696e3a563e11ce56d9b98495bbf1aea +symbol pa_scheduler_aic.o pa_scheduler_0_mix_aic 387044 9306a0d3ad16d74f736e6e5e62198311e4cf74f42b9a3b075c3b40624b5ef38d +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aiv 390516 67b4fd4c11780b2730e10a0f1b124481f96b2ea1c548d626ff45b5d67b807833 +symbol pa_scheduler_aiv.o pa_scheduler_0_mix_aiv 390516 b6d23dc6d25a3eff57deea579f08910b91cfab662c3c7348a7e9155df2a310d2 diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/pa_scheduler_aic.o b/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/pa_scheduler_aic.o new file mode 100644 index 0000000000..8fb367f0e4 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/pa_scheduler_aic.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/pa_scheduler_aiv.o b/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/pa_scheduler_aiv.o new file mode 100644 index 0000000000..0b0218b693 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/pa_scheduler_aiv.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/pa_scheduler_host b/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/pa_scheduler_host new file mode 100755 index 0000000000..f740e4afc2 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/pa_scheduler_host differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/pa_scheduler_kernel.o b/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/pa_scheduler_kernel.o new file mode 100755 index 0000000000..53ab09e9e7 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/build/original/pa_scheduler_kernel.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/build/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/A_original/build/published.sha256 new file mode 100644 index 0000000000..94466927fa --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/build/published.sha256 @@ -0,0 +1,6 @@ +1e230f4bb24bf0bfedc154397586e6af5f4773d8c18e147a1b10be3a5e370db3 build/original/artifacts.manifest +fde5c575160293b1a1cf60b330a9605db507650b8c9185fa60f6122563a2da66 build/original/device_text_layout.manifest +f3e62e50400035e5535abd2c74830f1458b862de759a4eadaec0762e8463e5dc build/original/pa_scheduler_aic.o +995faf6a73d4f50cfa55b83d3fde4a1db2ad7b43c3d18a480492fa03bed5163a build/original/pa_scheduler_aiv.o +97cf327377857bcab61cbca10357a90bc49cd5c389c919873c2f3209d5b01188 build/original/pa_scheduler_host +76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc build/original/pa_scheduler_kernel.o diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/build.sh b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/build.sh new file mode 100755 index 0000000000..ff9c1a60a9 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/build.sh @@ -0,0 +1,934 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +# 任一工具失败、未定义变量或管道中间失败都立即终止，避免继续使用半成品 device ELF。 +set -euo pipefail + +# 所有输入和产物都从脚本自身位置解析，调用者无需位于仓库根目录。 +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +LAZY_SAMPLE_MANIFEST_NAME="artifacts.manifest" +LAZY_SAMPLE_TEXT_LAYOUT_NAME="device_text_layout.manifest" +LAZY_SAMPLE_SPLIT=0 +LAZY_SAMPLE_VARIANT=1 +PMU_VARIANT=0 + +# All three artifacts compile the same swimlane-capable source with submit PMU +# disabled. Performance runs pass --no-swimlane at runtime. +if [[ $# -ne 1 || "$1" != "original" ]]; then + echo "This frozen package accepts only: $0 original" >&2 + exit 1 +fi +BUILD_VARIANT="$1" +case "$BUILD_VARIANT" in + original) + LAZY_SAMPLE_SHAPE="original" + LAZY_SAMPLE_SHAPE_ID=0 + LAZY_SAMPLE_OBSERVATION="legacy-straight-line-eager" + LAZY_SAMPLE_FINISH_SHAPE="legacy-inline-submit" + BUILD_DIR="$ROOT_DIR/build/original" + VARIANT_DEFINES=(-DPA_BUILD_SWIMLANE=1 -DPA_BUILD_SUBMIT_PMU=0 -DPA_SUBMIT_PMU_PHASE_ID=0) + ;; + compete-first|compete-first-lazy) + LAZY_SAMPLE_SPLIT=1 + LAZY_SAMPLE_OBSERVATION="split-combination-semantic" + LAZY_SAMPLE_FINISH_SHAPE="noinline-cross-tu" + if [[ "$BUILD_VARIANT" == "compete-first" ]]; then + LAZY_SAMPLE_SHAPE="compete-first" + LAZY_SAMPLE_SHAPE_ID=1 + else + LAZY_SAMPLE_SHAPE="compete-first-lazy" + LAZY_SAMPLE_SHAPE_ID=2 + fi + BUILD_DIR="$ROOT_DIR/build/$LAZY_SAMPLE_SHAPE" + VARIANT_DEFINES=( + -DPA_BUILD_SWIMLANE=1 + -DPA_BUILD_SUBMIT_PMU=0 + -DPA_SUBMIT_PMU_PHASE_ID=0 + "-DPA_LAZY_SAMPLE_SHAPE_ID=$LAZY_SAMPLE_SHAPE_ID" + -DPA_LAZY_SAMPLE_SPLIT_FINISH=1 + ) + ;; + *) + echo "Unknown variant: $BUILD_VARIANT (expected original|compete-first|compete-first-lazy)" >&2 + exit 1 + ;; +esac + +# 编译只依赖本目录源码与用户安装的 CANN/PTO 头，不引用 pa_scheduler 目录外的 simpler 构建产物。 +if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then + echo "ASCEND_HOME_PATH is not set; source the CANN 9.1 set_env.sh first." >&2 + exit 1 +fi + +CCEC="$ASCEND_HOME_PATH/bin/ccec" +LD="$ASCEND_HOME_PATH/bin/ld.lld" +HCC="$ASCEND_HOME_PATH/tools/hcc/bin/aarch64-target-linux-gnu-g++" +CXX_BIN="${CXX:-g++}" +READELF_BIN="${READELF:-readelf}" +PTO_INCLUDE_ROOT="${PTO_ISA_ROOT:-$ASCEND_HOME_PATH/x86_64-linux}" + +# ccec/ld.lld 必须来自当前已 source 的 CANN；host 编译器和 readelf 允许用户通过环境变量替换。 +if [[ ! -x "$CCEC" || ! -x "$LD" ]]; then + echo "CCEC or ld.lld is missing under ASCEND_HOME_PATH=$ASCEND_HOME_PATH" >&2 + exit 1 +fi +if [[ "$PMU_VARIANT" -eq 1 && ! -x "$HCC" ]]; then + echo "The AICPU HCC compiler is missing under ASCEND_HOME_PATH=$ASCEND_HOME_PATH" >&2 + exit 1 +fi +if ! command -v "$READELF_BIN" >/dev/null 2>&1; then + echo "readelf is required to verify the mixed AICore ELF." >&2 + exit 1 +fi +if [[ "$BUILD_VARIANT" != "swimlane" ]] && ! command -v sha256sum >/dev/null 2>&1; then + echo "sha256sum is required to publish variant artifact manifests." >&2 + exit 1 +fi +if [[ ! -f "$PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" ]]; then + echo "PTO kernel metadata header is missing under $PTO_INCLUDE_ROOT/include" >&2 + exit 1 +fi +for header in pto/pto-inst.hpp pto/common/constants.hpp pto/common/pto_tile.hpp; do + if [[ ! -f "$PTO_INCLUDE_ROOT/include/$header" ]]; then + echo "PTO real-compute header is missing: $PTO_INCLUDE_ROOT/include/$header" >&2 + exit 1 + fi +done + +mkdir -p "$BUILD_DIR" +if [[ "$PMU_VARIANT" -eq 0 ]]; then + # 旧统一构建可能在根目录残留 PMU owner；swimlane 构建主动移除这两个 + # 不属于本变体的产物，避免 direct host 调用误加载上一版诊断 SO。 + rm -f \ + "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so" \ + "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so" +else + # manifest 是同一 phase 四件套唯一的“可运行”标记。重建一开始先使旧 + # manifest 失效；即使后续编译中断，run.sh 也不会消费目录里的半成品。 + rm -f -- "$BUILD_DIR/$SUBMIT_PMU_MANIFEST_NAME" +fi +if [[ "$LAZY_SAMPLE_VARIANT" -eq 1 ]]; then + rm -f -- \ + "$BUILD_DIR/$LAZY_SAMPLE_MANIFEST_NAME" \ + "$BUILD_DIR/$LAZY_SAMPLE_TEXT_LAYOUT_NAME" +fi + +# 关闭编译器自动插入的 scalar DCCI，由 kernel.cpp 中与 PA 对齐的显式失效/回写协议负责 cache 可见性。 +# 两种架构共用这些 ABI、栈和优化参数，避免 AIC/AIV 对共享 SchedulerState 产生不同解释。 +COMMON_FLAGS=( + -c -O3 -g -x cce -Wall -std=c++17 + --cce-aicore-only + -mllvm -cce-aicore-stack-size=0x8000 + -mllvm -cce-aicore-function-stack-size=0x8000 + -mllvm -cce-aicore-record-overflow=false + -mllvm -cce-aicore-addr-transform + -mllvm -cce-aicore-dcci-insert-for-scalar=false + -mllvm -cce-aicore-dcci-before-kernel-end=false + -I"$ROOT_DIR/common" + -I"$PTO_INCLUDE_ROOT/include" + "${VARIANT_DEFINES[@]}" +) + +# split finish 的完整 runtime state 为 1600B，超过 CCEC 默认保留的 +# block-local 栈空间。编译器 hidden help 明确该参数以 byte 为单位、上限 +# 4KiB。实测 1600B 与 2048B 虽生成相同大小的 .text，内容 SHA 却不同， +# 因此使用当前 ABI 的精确尺寸而不增加无依据余量，并严格限于 split 变体。 +LAZY_SAMPLE_SPLIT_STATE_BYTES=1600 +LAZY_SAMPLE_FINISH_CALL_SITES=5 +LAZY_SAMPLE_BLOCK_LOCAL_RESERVE_BYTES=0 +if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_BLOCK_LOCAL_RESERVE_BYTES=$LAZY_SAMPLE_SPLIT_STATE_BYTES + COMMON_FLAGS+=( + -mllvm -cce-block-local-relocate=true + -mllvm "-cce-block-local-reserve-size=$LAZY_SAMPLE_BLOCK_LOCAL_RESERVE_BYTES" + ) +fi + +# 同一入口源码分别面向 cube 与 vector ISA 编译，宏只选择各自的全局入口和 mixed metadata。 +echo "[BUILD] CCEC AIC entry (dav-c310-cube)" +"$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DPA_BUILD_AIC \ + -o "$BUILD_DIR/pa_scheduler_aic.o" \ + "$SCRIPT_DIR/kernel.cpp" + +echo "[BUILD] CCEC AIV entry (dav-c310-vec)" +"$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DPA_BUILD_AIV \ + -o "$BUILD_DIR/pa_scheduler_aiv.o" \ + "$SCRIPT_DIR/kernel.cpp" + +check_workload_dispatcher_object() { + local object_path="$1" + local expected_symbol="$2" + local wrong_role_symbol="$3" + local object_symbols + object_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$object_path")" + if ! awk -v name="$expected_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$object_symbols"; then + echo "Expected exactly one non-empty strong workload dispatcher in $object_path: $expected_symbol" >&2 + exit 1 + fi + if awk -v name="$wrong_role_symbol" \ + '$NF == name {found = 1} END {exit !found}' <<<"$object_symbols"; then + echo "Wrong-role workload dispatcher leaked into $object_path: $wrong_role_symbol" >&2 + exit 1 + fi +} +check_workload_dispatcher_object \ + "$BUILD_DIR/pa_scheduler_aic.o" \ + pa_execute_real_winner_workload_aic \ + pa_execute_real_winner_workload_aiv +check_workload_dispatcher_object \ + "$BUILD_DIR/pa_scheduler_aiv.o" \ + pa_execute_real_winner_workload_aiv \ + pa_execute_real_winner_workload_aic +echo "[CHECK] role-specific real-compute dispatchers are strong and do not cross roles" + +text_relocation_count_for_symbol() { + local object_path="$1" + local symbol_name="$2" + "$READELF_BIN" --relocs --wide "$object_path" | awk -v name="$symbol_name" ' + /^Relocation section '\''\.rela\.text'\''/ {in_text = 1; next} + /^Relocation section / {in_text = 0} + in_text { + for (column = 1; column <= NF; ++column) { + if ($column == name) { + count++ + next + } + } + } + END {print count + 0} + ' +} + +check_split_role_objects() { + local role="$1" + local wrong_role="$2" + local caller="$BUILD_DIR/pa_scheduler_${role}.o" + local runtime="$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_${role}.o" + local finish="$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_${role}.o" + local state_symbol="pa_scheduler_lazy_sample_callback_state_${role}" + local finish_symbol="pa_scheduler_lazy_sample_callback_finish_${role}" + local orchestration_symbol="pa_scheduler_lazy_sample_callback_orchestration_${role}" + local dispatcher_symbol="pa_execute_real_winner_workload_${role}" + local entry_symbol="pa_scheduler_0_mix_${role}" + local caller_symbols runtime_symbols finish_symbols + caller_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$caller")" + runtime_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$runtime")" + finish_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$finish")" + + if ! awk -v name="$orchestration_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$caller_symbols"; then + echo "Missing unique strong split orchestration in caller object: $orchestration_symbol" >&2 + exit 1 + fi + for imported in "$state_symbol" "$finish_symbol"; do + if ! awk -v name="$imported" \ + '$5 == "GLOBAL" && $7 == "UND" && $NF == name {count++} + END {exit count != 1}' <<<"$caller_symbols"; then + echo "Caller must import exactly one matching split symbol: $caller ($imported)" >&2 + exit 1 + fi + done + if [[ "$(text_relocation_count_for_symbol "$caller" "$finish_symbol")" -ne "$LAZY_SAMPLE_FINISH_CALL_SITES" ]]; then + echo "Caller must contain exactly $LAZY_SAMPLE_FINISH_CALL_SITES all-task split-finish .rela.text relocations: $caller" >&2 + exit 1 + fi + if [[ "$(text_relocation_count_for_symbol "$caller" "$state_symbol")" -eq 0 ]]; then + echo "Caller must access its matching external block-local state: $caller" >&2 + exit 1 + fi + if "$READELF_BIN" --sections --wide "$caller" | awk \ + 'index($0, ".ascend.meta.") != 0 {found = 1} END {exit !found}'; then + echo "Split caller object must not define launch metadata: $caller" >&2 + exit 1 + fi + + if ! awk -v name="$state_symbol" -v bytes="$LAZY_SAMPLE_SPLIT_STATE_BYTES" \ + '$4 == "OBJECT" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 == bytes {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime must own one exact-size block-local state: $runtime ($state_symbol)" >&2 + exit 1 + fi + if ! awk -v name="$entry_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime must own one non-empty mixed entry: $runtime ($entry_symbol)" >&2 + exit 1 + fi + if ! awk -v name="$orchestration_symbol" \ + '$5 == "GLOBAL" && $7 == "UND" && $NF == name {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime must strongly import one role-specific orchestration: $runtime ($orchestration_symbol)" >&2 + exit 1 + fi + if [[ "$(text_relocation_count_for_symbol "$runtime" "$orchestration_symbol")" -ne 1 ]]; then + echo "Runtime entry must contain exactly one orchestration call relocation: $runtime" >&2 + exit 1 + fi + local block_local_record block_local_section_index block_local_size_hex block_local_alignment + block_local_record="$( + "$READELF_BIN" --sections --wide "$runtime" | awk ' + {for (column = 1; column <= NF; ++column) { + if ($column == ".bl.uninit") { + section_index = $(column - 1) + gsub(/\[/, "", section_index) + gsub(/\]/, "", section_index) + print section_index, $(column + 4), $NF + exit + } + }} + ' + )" + read -r block_local_section_index block_local_size_hex block_local_alignment \ + <<<"$block_local_record" + if [[ -z "$block_local_section_index" || -z "$block_local_size_hex" || + $((16#$block_local_size_hex)) -ne "$LAZY_SAMPLE_SPLIT_STATE_BYTES" || + "$block_local_alignment" -ne 64 ]]; then + echo "Runtime block-local section must be exact-size and 64B aligned: $runtime" >&2 + exit 1 + fi + if ! awk -v name="$state_symbol" -v section="$block_local_section_index" \ + '$4 == "OBJECT" && $7 == section && $NF == name {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime state must be defined in its exact .bl.uninit section: $runtime" >&2 + exit 1 + fi + local runtime_sections + runtime_sections="$("$READELF_BIN" --sections --wide "$runtime")" + if ! awk -v name=".ascend.meta.$entry_symbol" ' + {for (column = 1; column <= NF; ++column) { + if ($column == name) found = 1 + }} + END {exit !found} + ' <<<"$runtime_sections"; then + echo "Runtime object is missing matching mixed-entry metadata: $runtime" >&2 + exit 1 + fi + if awk -v name=".ascend.meta.pa_scheduler_0_mix_${wrong_role}" ' + {for (column = 1; column <= NF; ++column) { + if ($column == name) found = 1 + }} + END {exit !found} + ' <<<"$runtime_sections"; then + echo "Wrong-role mixed-entry metadata leaked into runtime object: $runtime" >&2 + exit 1 + fi + + if ! awk -v name="$finish_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$finish_symbols"; then + echo "Finish object must define one non-empty strong finish: $finish ($finish_symbol)" >&2 + exit 1 + fi + for imported in "$state_symbol" "$dispatcher_symbol"; do + if ! awk -v name="$imported" \ + '$5 == "GLOBAL" && $7 == "UND" && $NF == name {count++} + END {exit count != 1}' <<<"$finish_symbols"; then + echo "Finish object must import exactly one matching symbol: $finish ($imported)" >&2 + exit 1 + fi + if [[ "$(text_relocation_count_for_symbol "$finish" "$imported")" -eq 0 ]]; then + echo "Finish object must reference its matching imported symbol: $finish ($imported)" >&2 + exit 1 + fi + done + if "$READELF_BIN" --sections --wide "$finish" | awk \ + 'index($0, ".ascend.meta.") != 0 {found = 1} END {exit !found}'; then + echo "Split finish object must not define launch metadata: $finish" >&2 + exit 1 + fi + + local forbidden symbol_table object_path + for object_path in "$caller" "$runtime" "$finish"; do + case "$object_path" in + "$caller") symbol_table="$caller_symbols" ;; + "$runtime") symbol_table="$runtime_symbols" ;; + *) symbol_table="$finish_symbols" ;; + esac + for forbidden in \ + "pa_scheduler_lazy_sample_callback_state_${wrong_role}" \ + "pa_scheduler_lazy_sample_callback_finish_${wrong_role}" \ + "pa_scheduler_lazy_sample_callback_orchestration_${wrong_role}" \ + "pa_execute_real_winner_workload_${wrong_role}" \ + "pa_scheduler_0_mix_${wrong_role}"; do + if awk -v name="$forbidden" \ + '$NF == name {found = 1} END {exit !found}' <<<"$symbol_table"; then + echo "Wrong-role split symbol leaked into $object_path: $forbidden" >&2 + exit 1 + fi + done + done + + for forbidden in "$entry_symbol"; do + if awk -v name="$forbidden" '$NF == name {found = 1} END {exit !found}' \ + <<<"$caller_symbols"; then + echo "Split caller must not own a launch entry: $caller ($forbidden)" >&2 + exit 1 + fi + if awk -v name="$forbidden" '$NF == name {found = 1} END {exit !found}' \ + <<<"$finish_symbols"; then + echo "Split finish must not own a launch entry: $finish ($forbidden)" >&2 + exit 1 + fi + done + for forbidden in "$finish_symbol" "$dispatcher_symbol"; do + if awk -v name="$forbidden" '$NF == name {found = 1} END {exit !found}' \ + <<<"$runtime_symbols"; then + echo "Runtime entry/state owner contains an unexpected split helper: $runtime ($forbidden)" >&2 + exit 1 + fi + done + if awk -v name="$orchestration_symbol" '$NF == name {found = 1} END {exit !found}' \ + <<<"$finish_symbols"; then + echo "Split finish must not contain orchestration: $finish ($orchestration_symbol)" >&2 + exit 1 + fi +} + +if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + echo "[BUILD] CCEC AIC split runtime entry/state owner (dav-c310-cube)" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DPA_BUILD_AIC \ + -o "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_aic.o" \ + "$SCRIPT_DIR/callback_runtime_entry.cpp" + echo "[BUILD] CCEC AIC all-task split finish (dav-c310-cube)" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DPA_BUILD_AIC \ + -o "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_aic.o" \ + "$SCRIPT_DIR/callback_finish.cpp" + echo "[BUILD] CCEC AIV split runtime entry/state owner (dav-c310-vec)" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DPA_BUILD_AIV \ + -o "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_aiv.o" \ + "$SCRIPT_DIR/callback_runtime_entry.cpp" + echo "[BUILD] CCEC AIV all-task split finish (dav-c310-vec)" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DPA_BUILD_AIV \ + -o "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_aiv.o" \ + "$SCRIPT_DIR/callback_finish.cpp" + check_split_role_objects aic aiv + check_split_role_objects aiv aic + echo "[CHECK] split caller/runtime/finish objects satisfy role, state, metadata, and call-boundary gates" + DEVICE_OBJECTS=( + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_aic.o" + "$BUILD_DIR/pa_scheduler_aic.o" + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_aic.o" + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_aiv.o" + "$BUILD_DIR/pa_scheduler_aiv.o" + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_aiv.o" + ) +else + DEVICE_OBJECTS=( + "$BUILD_DIR/pa_scheduler_aic.o" + "$BUILD_DIR/pa_scheduler_aiv.o" + ) +fi + +# 静态链接把两个 device object 合成一个可由 runtime 按 1:2 比例启动的 mixed AICore ELF。 +echo "[BUILD] Static 1:2 mixed AICore ELF" +"$LD" -m aicorelinux -Ttext=0 -static \ + --version-script="$SCRIPT_DIR/pa_scheduler_device_exports.map" \ + -o "$BUILD_DIR/pa_scheduler_kernel.o" \ + "${DEVICE_OBJECTS[@]}" + +SYMBOL_TABLE="$("$READELF_BIN" --symbols --wide --sym-base=10 "$BUILD_DIR/pa_scheduler_kernel.o")" +SECTION_TABLE="$("$READELF_BIN" --sections --wide "$BUILD_DIR/pa_scheduler_kernel.o")" +# 构建成功不等于 mixed launch 可用：同时检查两个入口符号及其 metadata section，缺一即拒绝产物。 +# `set -e` 同时保证 readelf 自身失败时不会拿空字符串继续做伪检查。 +for entry in pa_scheduler_0_mix_aic pa_scheduler_0_mix_aiv; do + if ! awk -v name="$entry" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 != "0" && $3 != "0x0" {found = 1} END {exit !found}' \ + <<<"$SYMBOL_TABLE"; then + echo "Missing non-empty defined GLOBAL mixed-kernel entry: $entry" >&2 + exit 1 + fi + if ! awk -v name=".ascend.meta.$entry" ' + {for (column = 1; column <= NF; ++column) { + if ($column == name) found = 1 + }} + END {exit !found} + ' <<<"$SECTION_TABLE"; then + echo "Missing mixed-kernel metadata section: .ascend.meta.$entry" >&2 + exit 1 + fi +done +echo "[CHECK] both 1:2 mixed entries and metadata sections are present" + +# A5 runtime 会把已定义的 GLOBAL FUNC 当作可启动候选；最终 device ELF 只允许 +# 两个带 metadata 的 mixed 入口暴露为全局函数。任何新增 helper 都必须保持 LOCAL。 +while IFS= read -r global_func; do + case "$global_func" in + pa_scheduler_0_mix_aic|pa_scheduler_0_mix_aiv) ;; + *) + echo "Unexpected GLOBAL device function (possible kernel-entry pollution): $global_func" >&2 + exit 1 + ;; + esac +done < <(awk '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" {print $NF}' <<<"$SYMBOL_TABLE") +echo "[CHECK] only the two mixed entries are exported as GLOBAL device functions" + +# runtime-finish TU 后续需要复用同一个真计算 dispatcher，因此 caller object +# 按核型提供 strong 定义；version script 必须把它们在最终 mixed ELF 中重新 +# 局部化。这里同时检查两个 role-specific dispatcher 与底层 Cube/Vector 实体， +# 禁止因抽取 adapter 漏掉任一真实负载路径。 +for workload_symbol in \ + pa_execute_real_winner_workload_aic \ + pa_execute_real_winner_workload_aiv \ + pa_real_cube_workload_aic \ + pa_real_vector_add_workload_aiv \ + pa_real_vector_mul_workload_aiv; do + workload_size="$( + awk -v name="$workload_symbol" \ + '$4 == "FUNC" && $5 == "LOCAL" && $7 != "UND" && index($NF, name) != 0 && $3 + 0 > 0 {print $3; exit}' \ + <<<"$SYMBOL_TABLE" + )" + if [[ -z "$workload_size" ]]; then + echo "Missing non-empty LOCAL CCEC real-compute workload function: $workload_symbol" >&2 + exit 1 + fi + if awk -v name="$workload_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && index($NF, name) != 0 {found = 1} END {exit !found}' \ + <<<"$SYMBOL_TABLE"; then + echo "CCEC real-compute helper must not be a GLOBAL kernel candidate: $workload_symbol" >&2 + exit 1 + fi +done +echo "[CHECK] CCEC cube/vector real-compute helpers are non-empty LOCAL functions" + +if [[ "$LAZY_SAMPLE_VARIANT" -eq 1 ]]; then + # callback/builder/front must remain inline in both families. Only split + # shapes may retain the two fixed, role-specific runtime finish functions. + if awk \ + '$4 == "FUNC" && $7 != "UND" && + (index($NF, "LazySampleCallback") != 0 || index($NF, "SubmitQk") != 0 || + index($NF, "LazySampleSplitState") != 0) {found = 1} + END {exit !found}' <<<"$SYMBOL_TABLE"; then + echo "lazy sample callback builder/front unexpectedly survived as an out-of-line device function." >&2 + exit 1 + fi + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + for role in aic aiv; do + finish_symbol="pa_scheduler_lazy_sample_callback_finish_${role}" + if ! awk -v name="$finish_symbol" \ + '$4 == "FUNC" && $5 == "LOCAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Missing unique non-empty LOCAL split finish in final ELF: $finish_symbol" >&2 + exit 1 + fi + orchestration_symbol="pa_scheduler_lazy_sample_callback_orchestration_${role}" + if ! awk -v name="$orchestration_symbol" \ + '$4 == "FUNC" && $5 == "LOCAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Missing unique non-empty LOCAL split orchestration in final ELF: $orchestration_symbol" >&2 + exit 1 + fi + state_symbol="pa_scheduler_lazy_sample_callback_state_${role}" + if ! awk -v name="$state_symbol" -v bytes="$LAZY_SAMPLE_SPLIT_STATE_BYTES" \ + '$4 == "OBJECT" && $5 == "LOCAL" && $7 != "UND" && $NF == name && $3 + 0 == bytes {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Missing unique exact-size LOCAL split state in final ELF: $state_symbol" >&2 + exit 1 + fi + done + aic_state_hex="$(awk '$NF == "pa_scheduler_lazy_sample_callback_state_aic" {print $2; exit}' <<<"$SYMBOL_TABLE")" + aiv_state_hex="$(awk '$NF == "pa_scheduler_lazy_sample_callback_state_aiv" {print $2; exit}' <<<"$SYMBOL_TABLE")" + final_block_local_record="$( + awk '{for (column = 1; column <= NF; ++column) { + if ($column == ".bl_uninit") { + section_index = $(column - 1) + gsub(/\[/, "", section_index) + gsub(/\]/, "", section_index) + print section_index, $(column + 4), $NF + exit + } + }}' <<<"$SECTION_TABLE" + )" + read -r final_block_local_section_index final_block_local_size_hex \ + final_block_local_alignment \ + <<<"$final_block_local_record" + if [[ -z "$aic_state_hex" || -z "$aiv_state_hex" || + $((16#$aic_state_hex)) -ne 0 || + $((16#$aiv_state_hex)) -ne "$LAZY_SAMPLE_SPLIT_STATE_BYTES" || + -z "$final_block_local_section_index" || -z "$final_block_local_size_hex" || + $((16#$final_block_local_size_hex)) -ne $((2 * LAZY_SAMPLE_SPLIT_STATE_BYTES)) || + "$final_block_local_alignment" -ne 64 ]]; then + echo "Final split block-local layout must be two exact, non-overlapping 64B-aligned states." >&2 + exit 1 + fi + for state_symbol in \ + pa_scheduler_lazy_sample_callback_state_aic \ + pa_scheduler_lazy_sample_callback_state_aiv; do + if ! awk -v name="$state_symbol" -v section="$final_block_local_section_index" \ + '$4 == "OBJECT" && $7 == section && $NF == name {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Final split state must be bound to the exact .bl_uninit section: $state_symbol" >&2 + exit 1 + fi + done + echo "[CHECK] split finishes/orchestrations/states are LOCAL and final block-local layout is exact" + fi + if [[ -n "$("$READELF_BIN" --relocs --wide "$BUILD_DIR/pa_scheduler_kernel.o" | sed -n '/Relocation section/p')" ]]; then + echo "Final lazy sample callback mixed ELF must not retain relocations." >&2 + exit 1 + fi + echo "[CHECK] lazy sample callback builder/front is inline and final ELF has no relocations" +fi + +check_icache_probe_layout() { + local role="$1" + local target="pa_icache_target_${role}" + local harness="pa_icache_measure_${role}" + local thrash="pa_icache_thrash_${role}" + local target_record + local harness_record + local thrash_record + target_record="$(awk -v name="$target" '$4 == "FUNC" && index($NF, name) != 0 {print $2, $3; exit}' <<<"$SYMBOL_TABLE")" + harness_record="$(awk -v name="$harness" '$4 == "FUNC" && index($NF, name) != 0 {print $2, $3; exit}' <<<"$SYMBOL_TABLE")" + thrash_record="$(awk -v name="$thrash" '$4 == "FUNC" && index($NF, name) != 0 {print $2, $3; exit}' <<<"$SYMBOL_TABLE")" + if [[ -z "$target_record" || -z "$harness_record" || -z "$thrash_record" ]]; then + echo "Missing I-cache probe symbols for $role" >&2 + exit 1 + fi + + local target_hex target_size harness_hex harness_size thrash_hex thrash_size + read -r target_hex target_size <<<"$target_record" + read -r harness_hex harness_size <<<"$harness_record" + read -r thrash_hex thrash_size <<<"$thrash_record" + local target_address=$((16#$target_hex)) + local harness_address=$((16#$harness_hex)) + local thrash_address=$((16#$thrash_hex)) + if (( target_address % 128 != 0 || target_size == 0 || target_size > 16 )); then + echo "Invalid single-fetch-block I-cache target for $role: address=0x$target_hex size=$target_size" >&2 + exit 1 + fi + if (( thrash_size < 65536 )); then + echo "I-cache thrash body is smaller than 64 KiB for $role: size=$thrash_size" >&2 + exit 1 + fi + if (( harness_address % 128 != 0 || target_address + 128 > harness_address || + harness_address + harness_size > thrash_address )); then + echo "I-cache layout must be target -> harness -> thrash for $role" >&2 + exit 1 + fi + echo "[CHECK] $role I-cache target=0x$target_hex/$target_size harness=0x$harness_hex/$harness_size "\ + "thrash=0x$thrash_hex/$thrash_size" +} + +emit_text_section_fingerprint() { + local object_path="$1" + local artifact_name + artifact_name="$(basename "$object_path")" + local text_record text_address_hex text_offset_hex text_size_hex + text_record="$( + "$READELF_BIN" --sections --wide "$object_path" | awk ' + {for (column = 1; column <= NF; ++column) { + if ($column == ".text") { + print $(column + 2), $(column + 3), $(column + 4) + exit + } + }} + ' + )" + read -r text_address_hex text_offset_hex text_size_hex <<<"$text_record" + if [[ -z "$text_address_hex" || -z "$text_offset_hex" || -z "$text_size_hex" ]]; then + echo "Cannot fingerprint missing .text section: $object_path" >&2 + return 1 + fi + local text_size=$((16#$text_size_hex)) + local text_sha + text_sha="$( + dd if="$object_path" bs=1 skip=$((16#$text_offset_hex)) count="$text_size" status=none | + sha256sum | awk '{print $1}' + )" + printf 'text %s %u %s\n' "$artifact_name" "$text_size" "$text_sha" +} + +emit_symbol_body_fingerprint() { + local object_path="$1" + local symbol_name="$2" + local artifact_name + artifact_name="$(basename "$object_path")" + local symbol_record symbol_address_hex symbol_size + symbol_record="$( + "$READELF_BIN" --symbols --wide --sym-base=10 "$object_path" | awk -v name="$symbol_name" ' + $4 == "FUNC" && $7 != "UND" && $NF == name && $3 + 0 > 0 { + count++ + address = $2 + size = $3 + } + END { + if (count != 1) exit 1 + print address, size + } + ' + )" || { + echo "Cannot fingerprint non-unique or empty function: $object_path ($symbol_name)" >&2 + return 1 + } + read -r symbol_address_hex symbol_size <<<"$symbol_record" + + local text_record text_address_hex text_offset_hex text_size_hex + text_record="$( + "$READELF_BIN" --sections --wide "$object_path" | awk ' + {for (column = 1; column <= NF; ++column) { + if ($column == ".text") { + print $(column + 2), $(column + 3), $(column + 4) + exit + } + }} + ' + )" + read -r text_address_hex text_offset_hex text_size_hex <<<"$text_record" + if [[ -z "$text_address_hex" || -z "$text_offset_hex" || -z "$text_size_hex" ]]; then + echo "Cannot locate .text for function fingerprint: $object_path ($symbol_name)" >&2 + return 1 + fi + local symbol_address=$((16#$symbol_address_hex)) + local text_address=$((16#$text_address_hex)) + local text_size=$((16#$text_size_hex)) + local relative_offset=$((symbol_address - text_address)) + if (( relative_offset < 0 || symbol_size <= 0 || relative_offset + symbol_size > text_size )); then + echo "Function body lies outside .text: $object_path ($symbol_name)" >&2 + return 1 + fi + local symbol_sha + symbol_sha="$( + dd if="$object_path" bs=1 \ + skip=$((16#$text_offset_hex + relative_offset)) \ + count="$symbol_size" status=none | + sha256sum | awk '{print $1}' + )" + printf 'symbol %s %s %u %s\n' \ + "$artifact_name" "$symbol_name" "$symbol_size" "$symbol_sha" +} + +# 两个正式 ELF 都不携带旧 cold/warm 校准冲刷体；submit-pmu 只观察真实 +# Submit。保留上面的检查函数供历史布局取证时复核，但正式构建不调用它。 + +# PMU selector/CTRL 的所有权必须由主 aicpu_scheduler 配置并在退出前恢复。 +# standalone 目录内自带 Path-A dispatcher 与 owner：前者负责把 owner SO +# 落到设备预安装目录，后者由 mode=0 JSON 注册并通过统一入口执行命令。 +# swimlane 构建不生成 PMU owner/dispatcher；submit-pmu 则把 kernel、host、 +# owner 与 dispatcher 全部放在同一个 phase 目录，禁止跨 phase 复用。 +if [[ "$PMU_VARIANT" -eq 1 ]]; then + echo "[BUILD] self-contained AICPU PMU dispatcher" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra -Werror \ + -Wl,--build-id \ + "$SCRIPT_DIR/pmu_owner_dispatcher.cpp" \ + -o "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so" + + echo "[BUILD] self-contained AICPU PMU owner" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra -Werror \ + -Wl,--build-id \ + "${VARIANT_DEFINES[@]}" \ + -I"$SCRIPT_DIR" \ + "$SCRIPT_DIR/pmu_owner_aicpu.cpp" \ + -o "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so" + + OWNER_HEADER="$("$READELF_BIN" --file-header "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so")" + OWNER_SYMBOLS="$("$READELF_BIN" --dyn-syms --wide "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so")" + DISPATCHER_HEADER="$("$READELF_BIN" --file-header "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so")" + DISPATCHER_SYMBOLS="$("$READELF_BIN" --dyn-syms --wide "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so")" + if [[ "$OWNER_HEADER" != *"Type: DYN"* || + "$OWNER_HEADER" != *"Machine: AArch64"* || + "$DISPATCHER_HEADER" != *"Type: DYN"* || + "$DISPATCHER_HEADER" != *"Machine: AArch64"* ]]; then + echo "PMU dispatcher and owner must both be AArch64 shared objects." >&2 + exit 1 + fi + if [[ "$OWNER_SYMBOLS" != *" simpler_aicpu_exec"* ]]; then + echo "Missing main AICPU PMU owner entry: simpler_aicpu_exec" >&2 + exit 1 + fi + for entry in StaticTileFwkBackendKernelServer DynTileFwkBackendKernelServerInit DynTileFwkBackendKernelServer; do + if [[ "$DISPATCHER_SYMBOLS" != *" $entry"* ]]; then + echo "Missing AICPU PMU dispatcher entry: $entry" >&2 + exit 1 + fi + done + echo "[CHECK] Path-A dispatcher and main AICPU PMU owner exports are present" +fi + +# host runner 只链接用户 CANN 9.1 的 ACL/runtime，并写入同一安装目录的 rpath，运行时不需要 simpler 动态库。 +# `-Werror` 让 host API 签名或尺寸类型变化在构建期暴露，避免到上板阶段才出现参数截断。 +echo "[BUILD] CCEC host runner" +"$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror -Wno-deprecated-declarations \ + "${VARIANT_DEFINES[@]}" \ + -I"$ROOT_DIR/common" \ + -I"$ASCEND_HOME_PATH/include" \ + -I"$ASCEND_HOME_PATH/pkg_inc" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime/runtime" \ + "$SCRIPT_DIR/host.cpp" \ + -L"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -Wl,-rpath,"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -lascendcl -lruntime \ + -ldl \ + -o "$BUILD_DIR/pa_scheduler_host" + +if [[ "$PMU_VARIANT" -eq 1 ]]; then + # host、kernel、owner、dispatcher 全部成功后才生成 manifest；校验和使用 + # 相对文件名，目录复制后仍可在 run 前原样复核。临时文件与最终文件位于 + # 同一目录，mv 只承担单文件原子发布，不会暴露半写 manifest。 + SUBMIT_PMU_ARTIFACTS=( + pa_scheduler_host + pa_scheduler_kernel.o + libpa_scheduler_pmu_owner_aicpu.so + libpa_scheduler_pmu_owner_dispatcher.so + ) + for artifact in "${SUBMIT_PMU_ARTIFACTS[@]}"; do + if [[ ! -s "$BUILD_DIR/$artifact" ]]; then + echo "Cannot publish submit-pmu manifest; artifact is missing or empty: $artifact" >&2 + exit 1 + fi + done + if [[ ! -x "$BUILD_DIR/pa_scheduler_host" ]]; then + echo "Cannot publish submit-pmu manifest; host runner is not executable." >&2 + exit 1 + fi + + MANIFEST_PATH="$BUILD_DIR/$SUBMIT_PMU_MANIFEST_NAME" + MANIFEST_TMP="$(mktemp "$BUILD_DIR/.${SUBMIT_PMU_MANIFEST_NAME}.tmp.XXXXXX")" + cleanup_manifest_tmp() { + if [[ -n "${MANIFEST_TMP:-}" ]]; then + rm -f -- "$MANIFEST_TMP" + fi + } + trap cleanup_manifest_tmp EXIT + { + printf '# schema=pa_scheduler_submit_pmu_artifacts/v1\n' + printf '# variant=submit-pmu\n' + printf '# phase=%s\n' "$PHASE_NAME" + printf '# phase_id=%u\n' "$PHASE_ID" + (cd "$BUILD_DIR" && sha256sum "${SUBMIT_PMU_ARTIFACTS[@]}") + } > "$MANIFEST_TMP" + mv -f -- "$MANIFEST_TMP" "$MANIFEST_PATH" + MANIFEST_TMP="" + trap - EXIT + echo "[CHECK] submit-pmu artifact manifest published: $MANIFEST_PATH" +fi +if [[ "$LAZY_SAMPLE_VARIANT" -eq 1 ]]; then + TEXT_LAYOUT_PATH="$BUILD_DIR/$LAZY_SAMPLE_TEXT_LAYOUT_NAME" + TEXT_LAYOUT_TMP="$(mktemp "$BUILD_DIR/.${LAZY_SAMPLE_TEXT_LAYOUT_NAME}.tmp.XXXXXX")" + cleanup_text_layout_tmp() { + if [[ -n "${TEXT_LAYOUT_TMP:-}" ]]; then + rm -f -- "$TEXT_LAYOUT_TMP" + fi + } + trap cleanup_text_layout_tmp EXIT + { + printf '# schema=pa_scheduler_device_text_layout/v1\n' + printf '# backend=ccec\n' + printf '# shape=%s\n' "$LAZY_SAMPLE_SHAPE" + printf '# block_local_reserve_bytes=%u\n' "$LAZY_SAMPLE_BLOCK_LOCAL_RESERVE_BYTES" + printf '# compiler=%s\n' '$ASCEND_HOME_PATH/bin/ccec' + printf '# compiler_sha256=%s\n' "$(sha256sum "$CCEC" | awk '{print $1}')" + printf '# linker=%s\n' '$ASCEND_HOME_PATH/bin/ld.lld' + printf '# linker_sha256=%s\n' "$(sha256sum "$LD" | awk '{print $1}')" + emit_text_section_fingerprint "$BUILD_DIR/pa_scheduler_kernel.o" + for object_path in "${DEVICE_OBJECTS[@]}"; do + emit_text_section_fingerprint "$object_path" + done + for role in aic aiv; do + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_kernel.o" "pa_scheduler_0_mix_${role}" + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_${role}.o" \ + "pa_scheduler_0_mix_${role}" + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_${role}.o" \ + "pa_scheduler_lazy_sample_callback_orchestration_${role}" + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_${role}.o" \ + "pa_scheduler_lazy_sample_callback_finish_${role}" + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_kernel.o" \ + "pa_scheduler_lazy_sample_callback_orchestration_${role}" + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_kernel.o" \ + "pa_scheduler_lazy_sample_callback_finish_${role}" + else + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_${role}.o" "pa_scheduler_0_mix_${role}" + fi + done + } > "$TEXT_LAYOUT_TMP" + mv -f -- "$TEXT_LAYOUT_TMP" "$TEXT_LAYOUT_PATH" + TEXT_LAYOUT_TMP="" + trap - EXIT + awk '$1 == "text" { + printf "[TEXT] %s size=%s sha256=%s\n", $2, $3, $4 + }' "$TEXT_LAYOUT_PATH" + echo "[CHECK] lazy sample callback device .text layout manifest published: $TEXT_LAYOUT_PATH" + + LAZY_SAMPLE_ARTIFACTS=( + pa_scheduler_host + pa_scheduler_kernel.o + ) + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_lazy_sample_callback_runtime_aic.o) + fi + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_aic.o) + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_lazy_sample_callback_finish_aic.o) + fi + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_lazy_sample_callback_runtime_aiv.o) + fi + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_aiv.o) + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_lazy_sample_callback_finish_aiv.o) + fi + LAZY_SAMPLE_ARTIFACTS+=("$LAZY_SAMPLE_TEXT_LAYOUT_NAME") + for artifact in "${LAZY_SAMPLE_ARTIFACTS[@]}"; do + if [[ ! -s "$BUILD_DIR/$artifact" ]]; then + echo "Cannot publish lazy sample callback manifest; artifact is missing or empty: $artifact" >&2 + exit 1 + fi + done + if [[ ! -x "$BUILD_DIR/pa_scheduler_host" ]]; then + echo "Cannot publish lazy sample callback manifest; host runner is not executable." >&2 + exit 1 + fi + MANIFEST_PATH="$BUILD_DIR/$LAZY_SAMPLE_MANIFEST_NAME" + MANIFEST_TMP="$(mktemp "$BUILD_DIR/.${LAZY_SAMPLE_MANIFEST_NAME}.tmp.XXXXXX")" + cleanup_callback_manifest_tmp() { + if [[ -n "${MANIFEST_TMP:-}" ]]; then + rm -f -- "$MANIFEST_TMP" + fi + } + trap cleanup_callback_manifest_tmp EXIT + { + printf '# schema=pa_scheduler_lazy_sample_callback_artifacts/v1\n' + printf '# backend=ccec\n' + printf '# shape=%s\n' "$LAZY_SAMPLE_SHAPE" + printf '# shape_id=%u\n' "$LAZY_SAMPLE_SHAPE_ID" + printf '# observation=%s\n' "$LAZY_SAMPLE_OBSERVATION" + (cd "$BUILD_DIR" && sha256sum "${LAZY_SAMPLE_ARTIFACTS[@]}") + } > "$MANIFEST_TMP" + mv -f -- "$MANIFEST_TMP" "$MANIFEST_PATH" + MANIFEST_TMP="" + trap - EXIT + echo "[CHECK] lazy sample callback artifact manifest published: $MANIFEST_PATH" +fi + +echo "[BUILD] complete: $BUILD_DIR" diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/callback_finish.cpp b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/callback_finish.cpp new file mode 100644 index 0000000000..9422fef200 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/callback_finish.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include +#include +#include + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" +#include "ccec_ops.h" + +using pa_scheduler_ccec::CcecOps; + +extern "C" { +#if defined(PA_BUILD_AIC) +__attribute__((noinline)) __aicore__ uint32_t +pa_scheduler_lazy_sample_callback_finish_aic(const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args) { + return pa_scheduler::FinishSplitLazySampleCallbackFromRuntime(ticket, args); +} +#elif defined(PA_BUILD_AIV) +__attribute__((noinline)) __aicore__ uint32_t +pa_scheduler_lazy_sample_callback_finish_aiv(const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args) { + return pa_scheduler::FinishSplitLazySampleCallbackFromRuntime(ticket, args); +} +#else +#error "Compile split finish with PA_BUILD_AIC or PA_BUILD_AIV" +#endif +} diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/callback_finish_api.h b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/callback_finish_api.h new file mode 100644 index 0000000000..9c87d77675 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/callback_finish_api.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#ifndef TESTS_ATOMIC_PROBE_LAZY_LAMDA_SAMPLE_CCEC_CALLBACK_FINISH_API_H +#define TESTS_ATOMIC_PROBE_LAZY_LAMDA_SAMPLE_CCEC_CALLBACK_FINISH_API_H + +#if !defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +#error "callback_finish_api.h is only valid for split-finish artifacts" +#endif + +// The runtime TU owns the role-specific block-local object. The caller only +// imports it; no weak/generic state symbol is allowed in the mixed AIC/AIV ELF. +extern "C" { +#if defined(PA_BUILD_AIC) +[[block_local]] extern pa_scheduler::LazySampleSplitRuntimeState pa_scheduler_lazy_sample_callback_state_aic; +__aicore__ uint32_t +pa_scheduler_lazy_sample_callback_finish_aic(const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args); +#elif defined(PA_BUILD_AIV) +[[block_local]] extern pa_scheduler::LazySampleSplitRuntimeState pa_scheduler_lazy_sample_callback_state_aiv; +__aicore__ uint32_t +pa_scheduler_lazy_sample_callback_finish_aiv(const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args); +#else +#error "Compile split finish with PA_BUILD_AIC or PA_BUILD_AIV" +#endif +} + +#endif // TESTS_ATOMIC_PROBE_LAZY_LAMDA_SAMPLE_CCEC_CALLBACK_FINISH_API_H diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/callback_runtime_entry.cpp b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/callback_runtime_entry.cpp new file mode 100644 index 0000000000..56adcd3a71 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/callback_runtime_entry.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" + +extern "C" { +#if defined(PA_BUILD_AIC) +[[block_local]] pa_scheduler::LazySampleSplitRuntimeState pa_scheduler_lazy_sample_callback_state_aic; +__aicore__ void +pa_scheduler_lazy_sample_callback_orchestration_aic(__gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id); +#elif defined(PA_BUILD_AIV) +[[block_local]] pa_scheduler::LazySampleSplitRuntimeState pa_scheduler_lazy_sample_callback_state_aiv; +__aicore__ void +pa_scheduler_lazy_sample_callback_orchestration_aiv(__gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id); +#else +#error "Compile split runtime entry with PA_BUILD_AIC or PA_BUILD_AIV" +#endif +} + +#if defined(PA_BUILD_AIC) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aic, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aic(__gm__ pa_scheduler::SchedulerState *state) { + const uint32_t worker_id = static_cast(get_block_idx()); + pa_scheduler_lazy_sample_callback_orchestration_aic(state, worker_id); +} +#elif defined(PA_BUILD_AIV) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aiv, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aiv(__gm__ pa_scheduler::SchedulerState *state) { + const uint32_t vector_id = static_cast(get_block_idx() * get_subblockdim() + get_subblockid()); + const uint32_t worker_id = pa_scheduler::kAicWorkers + vector_id; + pa_scheduler_lazy_sample_callback_orchestration_aiv(state, worker_id); +} +#endif diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/ccec_ops.h b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/ccec_ops.h new file mode 100644 index 0000000000..9a64d8bba1 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/ccec_ops.h @@ -0,0 +1,408 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#ifndef TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CCEC_OPS_H +#define TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CCEC_OPS_H + +// This private header is included after the CCEC/PTO headers and +// common/pa_scheduler_core.h. kernel.cpp defines the implementation switch +// exactly once per architecture; split runtime TUs consume only the external +// dispatcher declaration and the shared inline adapter. +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +#include "callback_finish_api.h" +#endif + +#if defined(PA_BUILD_AIC) +extern "C" __attribute__((noinline)) __aicore__ void pa_execute_real_winner_workload_aic( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +); +#elif defined(PA_BUILD_AIV) +extern "C" __attribute__((noinline)) __aicore__ void pa_execute_real_winner_workload_aiv( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +); +#else +#error "Compile CcecOps with PA_BUILD_AIC or PA_BUILD_AIV" +#endif + +namespace pa_scheduler_ccec { + +using namespace pto; + +template +__aicore__ inline void EmitNops() { +#pragma unroll + for (uint32_t index = 0; index < Count; ++index) { + asm volatile("nop"); + } +} + +__aicore__ inline void RuntimeNop(uint32_t count) { + // 两侧全流水屏障把可调 NOP 段限定为 kernel 模拟体，避免前后调度访存进入被测计算区间。 + __builtin_cce_pipe_barrier(PIPE_ALL); + while (count >= 256) { + EmitNops<256>(); + count -= 256; + } + if ((count & 128U) != 0) EmitNops<128>(); + if ((count & 64U) != 0) EmitNops<64>(); + if ((count & 32U) != 0) EmitNops<32>(); + if ((count & 16U) != 0) EmitNops<16>(); + if ((count & 8U) != 0) EmitNops<8>(); + if ((count & 4U) != 0) EmitNops<4>(); + if ((count & 2U) != 0) EmitNops<2>(); + if ((count & 1U) != 0) EmitNops<1>(); + __builtin_cce_pipe_barrier(PIPE_ALL); +} + +#if defined(PA_CCEC_OPS_DEFINE_REAL_WORKLOAD) && defined(PA_BUILD_AIC) +// QK/PV 的首版真实负载使用完整的 128x128 float cube 路径。输入来自独立 GM +// workspace，输出属于当前 worker；每次迭代都等 FIX 写回 GM 后再复用 L0C， +// 因而函数返回就是该模拟 task 的完成边界，而不是单纯的指令发射边界。 +static __aicore__ __attribute__((noinline, used)) void pa_real_cube_workload_aic( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + constexpr int kTile = static_cast(pa_scheduler::winner_workload::kTileRows); + constexpr int kBlockAlign = C0_SIZE_BYTE / sizeof(float); + static_assert(kTile % 16 == 0, "cube M must be 16-aligned"); + static_assert(kTile % kBlockAlign == 0, "cube K/N must satisfy C0 alignment"); + + using GlobalData = GlobalTensor< + float, Shape<1, 1, 1, kTile, kTile>, + pto::Stride>; + using TileMatA = Tile< + TileType::Mat, float, kTile, kTile, BLayout::ColMajor, + kTile, kTile, SLayout::RowMajor, 512>; + using TileMatB = Tile< + TileType::Mat, float, kTile, kTile, BLayout::ColMajor, + kTile, kTile, SLayout::RowMajor, 512>; + using LeftTile = TileLeft; + using RightTile = TileRight; + using AccTile = TileAcc; + + GlobalData input_a_global(input_a); + GlobalData input_b_global(input_b); + GlobalData output_global(output); + TileMatA input_a_mat; + TileMatB input_b_mat; + LeftTile input_a_l0; + RightTile input_b_l0; + AccTile output_l0; + TASSIGN(input_a_mat, 0x0); + TASSIGN(input_b_mat, 0x20000); + TASSIGN(input_a_l0, 0x0); + TASSIGN(input_b_l0, 0x0); + TASSIGN(output_l0, 0x0); + + for (uint32_t iteration = 0; iteration < repeats; ++iteration) { + TLOAD(input_a_mat, input_a_global); + TLOAD(input_b_mat, input_b_global); + set_flag(PIPE_MTE2, PIPE_MTE1, EVENT_ID0); + wait_flag(PIPE_MTE2, PIPE_MTE1, EVENT_ID0); + TMOV(input_a_l0, input_a_mat); + TMOV(input_b_l0, input_b_mat); + set_flag(PIPE_MTE1, PIPE_M, EVENT_ID0); + wait_flag(PIPE_MTE1, PIPE_M, EVENT_ID0); + TMATMUL(output_l0, input_a_l0, input_b_l0); + set_flag(PIPE_M, PIPE_FIX, EVENT_ID0); + wait_flag(PIPE_M, PIPE_FIX, EVENT_ID0); + TSTORE(output_global, output_l0); + set_flag(PIPE_FIX, PIPE_S, EVENT_ID7); + wait_flag(PIPE_FIX, PIPE_S, EVENT_ID7); + } +} +#elif defined(PA_CCEC_OPS_DEFINE_REAL_WORKLOAD) && defined(PA_BUILD_AIV) +template +__aicore__ inline void RunRealVectorWorkload( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + constexpr int kRows = static_cast(pa_scheduler::winner_workload::kTileRows); + constexpr int kCols = static_cast(pa_scheduler::winner_workload::kTileCols); + using GlobalData = GlobalTensor< + float, Shape<1, 1, 1, kRows, kCols>, pto::Stride<1, 1, 1, kCols, 1>>; + using TileData = Tile< + TileType::Vec, float, kRows, kCols, BLayout::RowMajor, -1, -1>; + + GlobalData input_a_global(input_a); + GlobalData input_b_global(input_b); + GlobalData output_global(output); + TileData input_a_tile(kRows, kCols); + TileData input_b_tile(kRows, kCols); + TileData output_tile(kRows, kCols); + TASSIGN(input_a_tile, 0x0); + TASSIGN(input_b_tile, 0x10000); + TASSIGN(output_tile, 0x20000); + + for (uint32_t iteration = 0; iteration < repeats; ++iteration) { + TLOAD(input_a_tile, input_a_global); + TLOAD(input_b_tile, input_b_global); + set_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + wait_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + if constexpr (Multiply) { + TMUL(output_tile, input_a_tile, input_b_tile); + } else { + TADD(output_tile, input_a_tile, input_b_tile); + } + set_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + wait_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + TSTORE(output_global, output_tile); + set_flag(PIPE_MTE3, PIPE_S, EVENT_ID7); + wait_flag(PIPE_MTE3, PIPE_S, EVENT_ID7); + } +} + +static __aicore__ __attribute__((noinline, used)) void pa_real_vector_add_workload_aiv( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + RunRealVectorWorkload(input_a, input_b, output, repeats); +} + +static __aicore__ __attribute__((noinline, used)) void pa_real_vector_mul_workload_aiv( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + RunRealVectorWorkload(input_a, input_b, output, repeats); +} +#endif + +#if PA_BUILD_SUBMIT_PMU +struct SubmitPmuContext; +#endif + +struct CcecOps { + static constexpr bool kAtomicReturnReadyObserved = true; + + // 该适配层把平台无关调度器需要的原子、计时、NOP 和 cache 操作逐一映射到 CCEC intrinsic。 + // A5 上 PA 的共享“读取”使用 atomicAdd(addr, 0)，不是普通 GM load；这里保留其 RMW 竞争语义。 + __aicore__ static inline int32_t Load(__gm__ volatile int32_t *address) { + // atomicAdd 返回加法发生前的值；加数为 0，因此它就是本次共享读取的结果。 + return atomicAdd(const_cast<__gm__ int32_t *>(address), static_cast(0)); + } + + __aicore__ static inline int64_t Load(__gm__ volatile int64_t *address) { + return atomicAdd(const_cast<__gm__ int64_t *>(address), static_cast(0)); + } + + __aicore__ static inline uint64_t Load(__gm__ volatile uint64_t *address) { + return atomicAdd(const_cast<__gm__ uint64_t *>(address), static_cast(0)); + } + + __aicore__ static inline int32_t Exchange(__gm__ volatile int32_t *address, int32_t value) { + // atomicExch 同样返回旧值；当前 completion/fatal 发布只需要其原子写入副作用。 + return atomicExch(const_cast<__gm__ int32_t *>(address), value); + } + + __aicore__ static inline int64_t Exchange(__gm__ volatile int64_t *address, int64_t value) { + return atomicExch(const_cast<__gm__ int64_t *>(address), value); + } + + __aicore__ static inline uint64_t Exchange(__gm__ volatile uint64_t *address, uint64_t value) { + return atomicExch(const_cast<__gm__ uint64_t *>(address), value); + } + + __aicore__ static inline int64_t FetchAdd(__gm__ volatile int64_t *address, int64_t value) { + // 返回递增前的计数；启动和 replay 屏障只关心全局累加结果，因此调用方不使用该返回值。 + return atomicAdd(const_cast<__gm__ int64_t *>(address), value); + } + + __aicore__ static inline int64_t FetchMax(__gm__ volatile int64_t *address, int64_t value, uint64_t &retries) { + // CCEC 直接生成单条硬件 atomicMax，不存在 CPU CAS 循环可观测的重试次数。 + retries = 0; + // 返回更新前的 cursor/frontier，Claim 用它判定 winner，frontier 扫描用它吸收其他核的进度。 + return atomicMax(const_cast<__gm__ int64_t *>(address), value); + } + + // PA's A5 OUT_OF_ORDER_STORE_BARRIER is intentionally a no-op; cache + // coherency is handled by the runtime's DCCI protocol. + // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， + // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache + // 可见性则由各自既有的 DCCI 路径处理。 + __aicore__ static inline void StoreBarrier() {} + + __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } + + template + __aicore__ static inline uint64_t NowAfterAtomicResult(T value) { + static_assert(sizeof(T) == 4 || sizeof(T) == 8, "atomic dependency expects a scalar result"); + uint64_t cycle = 0; + // 同一个 inline asm 块先真正消费 atomic 返回寄存器，再读取 + // SYS_CNT；编译器不能把 t1 拆到依赖 MOV 之前。AIC/AIV 对该序列 + // 生成相同指令字节，且不增加 DSB/ISB/GM 访存。该边界仍只表示 + // 返回值已可被本核 scalar 消费，不表示跨核全局可见。 + asm volatile( + "MOV %0, %0\n" + "MOV %1, SYS_CNT\n" + : "+l"(value), "=&l"(cycle) + ); + return cycle; + } + + __aicore__ static inline void ExecuteKernel( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState &worker, + pa_scheduler::TaskKind kind, uint32_t nop_count + ) { + const auto mode = static_cast(state->winner_workload.mode); + if (mode != pa_scheduler::WinnerWorkloadMode::RealCompute) { + RuntimeNop(nop_count); + return; + } +#if defined(PA_BUILD_AIC) + ::pa_execute_real_winner_workload_aic(state, &worker, kind); +#elif defined(PA_BUILD_AIV) + ::pa_execute_real_winner_workload_aiv(state, &worker, kind); +#endif + } + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + __aicore__ static inline pa_scheduler::LazySampleSplitRuntimeState &LazySampleSplitState() { +#if defined(PA_BUILD_AIC) + return ::pa_scheduler_lazy_sample_callback_state_aic; +#elif defined(PA_BUILD_AIV) + return ::pa_scheduler_lazy_sample_callback_state_aiv; +#endif + } + + __aicore__ static inline bool FinishLazySampleCallback( + const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args + ) { +#if defined(PA_BUILD_AIC) + return ::pa_scheduler_lazy_sample_callback_finish_aic(ticket, args) != 0; +#elif defined(PA_BUILD_AIV) + return ::pa_scheduler_lazy_sample_callback_finish_aiv(ticket, args) != 0; +#endif + } +#endif + +#if PA_BUILD_SUBMIT_PMU + using PmuContext = SubmitPmuContext; + + __aicore__ static inline PmuContext PmuWindowStart( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id + ); + + __aicore__ static inline void PmuPhaseBegin(PmuContext &context); + + __aicore__ static inline void PmuPhaseEnd(PmuContext &context); + + __aicore__ static inline void PmuWindowStop( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id, PmuContext &context + ); +#else + // swimlane 产物不携带 PMU 读寄存器或门控代码；公共调度器保留同一 hook + // 形状，编译器会把这两个空实现完整消去。 + __aicore__ static inline bool PmuWindowStart( + __gm__ pa_scheduler::SchedulerState *, uint32_t + ) { return false; } + + __aicore__ static inline void PmuWindowStop( + __gm__ pa_scheduler::SchedulerState *, uint32_t, bool + ) {} +#endif + + // SPIN_WAIT_HINT is also a no-op in the real A5 inner-kernel contract. + // 同理不额外插入 nop，让等待循环保留真实 PA 内核“不主动退避”的指令成本。 + __aicore__ static inline void SpinHint() {} + + __aicore__ static inline void InvalidateRegion(__gm__ const void *address, uint64_t bytes) { + // 逐 cache line 失效并以 dsb 收口，供 worker 在启动时读取 host 刚写入的 standalone 控制区。 + if (bytes == 0) return; + const uint64_t start = reinterpret_cast(address) & ~uint64_t{63}; + const uint64_t end = (reinterpret_cast(address) + bytes + 63) & ~uint64_t{63}; + for (uint64_t current = start; current < end; current += 64) { + dcci(reinterpret_cast<__gm__ uint8_t *>(current), SINGLE_CACHE_LINE); + } + dsb((mem_dsb_t)0); + } + + __aicore__ static inline void FlushRegion(__gm__ void *address, uint64_t bytes) { + // 泳道记录先写普通 GM cache，kernel 结束前显式 CACHELINE_OUT，确保 host D2H 能看到完整记录。 + if (bytes == 0) return; + __asm__ volatile("" ::: "memory"); + const uint64_t start = reinterpret_cast(address) & ~uint64_t{63}; + const uint64_t end = (reinterpret_cast(address) + bytes + 63) & ~uint64_t{63}; + for (uint64_t current = start; current < end; current += 64) { + dcci(reinterpret_cast<__gm__ uint8_t *>(current), SINGLE_CACHE_LINE, CACHELINE_OUT); + } + dsb((mem_dsb_t)0); + } + + __aicore__ static inline void Publish(__gm__ uint64_t *address, uint64_t value) { + // 每核独占的 WorkerResult 用 bypass-DCache store 发布，host 同步后可直接 D2H，无需共享原子竞争。 + __builtin_cce_st_dev(value, address, 0); + } + + __aicore__ static inline void Publish(__gm__ uint32_t *address, uint32_t value) { + __builtin_cce_st_dev(value, address, 0); + } +}; + +} // namespace pa_scheduler_ccec + +#if defined(PA_CCEC_OPS_DEFINE_REAL_WORKLOAD) +// 跨 TU 只暴露按架构区分的真实负载分派；Cube/Vector 实体仍保持 LOCAL。 +// 最终 mixed ELF 由 version script 把该 strong 定义重新局部化，避免成为 kernel entry。 +#if defined(PA_BUILD_AIC) +extern "C" __attribute__((noinline, used)) __aicore__ void pa_execute_real_winner_workload_aic( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +) { +#elif defined(PA_BUILD_AIV) +extern "C" __attribute__((noinline, used)) __aicore__ void pa_execute_real_winner_workload_aiv( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +) { +#endif + const uint64_t workspace = state->winner_workload.workspace_base; + const uint32_t repeats = pa_scheduler::WorkloadCountForKind( + state->winner_workload.repeats, kind + ); + // 错版 host、截断 workspace 或越界 worker 不能继续解引用 GM。这里不额外 + // 写共享 fatal，避免在正常热路增加 atomic；host 的逐 kind sentinel/数值 + // 闭环会把这种配置错误判为失败。 + if (state->winner_workload.version != pa_scheduler::kWinnerWorkloadConfigVersion || + workspace == 0 || + state->winner_workload.workspace_bytes < pa_scheduler::winner_workload::kWorkspaceBytes || + worker->core_idx < 0 || static_cast(worker->core_idx) >= pa_scheduler::kWorkers || + repeats == 0 || repeats > pa_scheduler::winner_workload::kMaxRealComputeCount) { + return; + } +#if defined(PA_BUILD_AIC) + if (kind != pa_scheduler::TaskKind::Qk && kind != pa_scheduler::TaskKind::Pv) return; +#elif defined(PA_BUILD_AIV) + if (kind != pa_scheduler::TaskKind::Sf && kind != pa_scheduler::TaskKind::Up) return; +#endif + __gm__ float *input_a = reinterpret_cast<__gm__ float *>(workspace); + __gm__ float *input_b = reinterpret_cast<__gm__ float *>( + workspace + pa_scheduler::winner_workload::kTileBytes + ); + const uint32_t kind_slot = + (kind == pa_scheduler::TaskKind::Pv || kind == pa_scheduler::TaskKind::Up) ? 1U : 0U; + const uint32_t output_tile = + pa_scheduler::winner_workload::kSharedInputTiles + + static_cast(worker->core_idx) * + pa_scheduler::winner_workload::kOutputTilesPerWorker + + kind_slot; + __gm__ float *output = reinterpret_cast<__gm__ float *>( + workspace + static_cast(output_tile) * + pa_scheduler::winner_workload::kTileBytes + ); +#if defined(PA_BUILD_AIC) + pa_scheduler_ccec::pa_real_cube_workload_aic(input_a, input_b, output, repeats); +#elif defined(PA_BUILD_AIV) + if (kind == pa_scheduler::TaskKind::Sf) { + pa_scheduler_ccec::pa_real_vector_add_workload_aiv(input_a, input_b, output, repeats); + } else { + pa_scheduler_ccec::pa_real_vector_mul_workload_aiv(input_a, input_b, output, repeats); + } +#endif +} +#endif // PA_CCEC_OPS_DEFINE_REAL_WORKLOAD + +#endif // TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CCEC_OPS_H diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/host.cpp b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/host.cpp new file mode 100644 index 0000000000..1ee342645f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/host.cpp @@ -0,0 +1,1989 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "../common/host_support.h" +#include "../common/winner_workload_host.h" +#include "pmu_owner_host.h" +#include "pmu_probe.h" + +#include "acl/acl.h" +#include "driver/ascend_hal.h" +#include "runtime/rt.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +bool CheckAcl(aclError error, const char *label) { + if (error == ACL_SUCCESS) return true; + std::fprintf(stderr, "ACL error %d: %s\n", static_cast(error), label); + return false; +} + +bool CheckRt(rtError_t error, const char *label) { + if (error == RT_ERROR_NONE) return true; + std::fprintf(stderr, "RT error %d: %s\n", static_cast(error), label); + return false; +} + +class ScopedAclDeviceAllocation { +public: + ScopedAclDeviceAllocation() = default; + ScopedAclDeviceAllocation(const ScopedAclDeviceAllocation &) = delete; + ScopedAclDeviceAllocation &operator=(const ScopedAclDeviceAllocation &) = delete; + + ~ScopedAclDeviceAllocation() { + // 早退路径没有机会汇入末尾 cleanup；这里只负责尽力释放本类新增的 + // real-compute workspace。正常路径会先 Release，再检查 aclrtFree 返回值。 + if (pointer_ != nullptr) (void)aclrtFree(pointer_); + } + + void **Address() { return &pointer_; } + void *Get() const { return pointer_; } + + void *Release() { + void *pointer = pointer_; + pointer_ = nullptr; + return pointer; + } + +private: + void *pointer_ = nullptr; +}; + +std::vector ReadBinary(const std::string &path) { + // ELF 整体保存在 vector 中直到 runtime 卸载完成，保证 rtDevBinary_t.data 在整个注册生命周期内有效。 + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) return {}; + const std::streamsize size = file.tellg(); + if (size <= 0) return {}; + std::vector data(static_cast(size)); + file.seekg(0, std::ios::beg); + if (!file.read(data.data(), size)) return {}; + return data; +} + +struct PmuOptions { + pa_scheduler::ccec_pmu::WindowMode mode = pa_scheduler::ccec_pmu::WindowMode::Off; + uint32_t scalar_nops = 100000; + uint32_t icache_trials = 64; + std::string json_path; +}; + +using pa_scheduler::host::ConfigureWinnerWorkload; +using pa_scheduler::host::ParseWinnerWorkloadOptions; +using pa_scheduler::host::ValidateRealComputeOutputs; +using pa_scheduler::host::ValidateWinnerWorkloadOptions; +using pa_scheduler::host::WinnerWorkloadModeName; +using pa_scheduler::host::WinnerWorkloadOptions; + +const char *PmuModeName(pa_scheduler::ccec_pmu::WindowMode mode) { + switch (mode) { + case pa_scheduler::ccec_pmu::WindowMode::Off: + return "off"; + case pa_scheduler::ccec_pmu::WindowMode::Empty: + return "empty"; + case pa_scheduler::ccec_pmu::WindowMode::Scalar: + return "scalar"; + case pa_scheduler::ccec_pmu::WindowMode::ScalarDouble: + return "scalar-double"; + case pa_scheduler::ccec_pmu::WindowMode::IcacheSingle: + return "icache-single"; + case pa_scheduler::ccec_pmu::WindowMode::SubmitAll: + return "submit-all"; + } + return "invalid"; +} + +bool ParsePmuOptions(int argc, char **argv, PmuOptions *pmu, std::vector *common_argv) { + // PMU 参数只属于 CCEC 验证分支；先摘出再交给三后端共享 parser，避免 CPU/AscendC 静默接受却不生效。 + bool mode_seen = false; + bool nops_seen = false; + bool icache_trials_seen = false; + bool json_seen = false; + common_argv->clear(); + common_argv->push_back(argv[0]); + for (int index = 1; index < argc; ++index) { + const std::string argument = argv[index]; + if (argument != "--pmu-window" && argument != "--pmu-scalar-nops" && + argument != "--pmu-icache-trials" && argument != "--pmu-json") { + common_argv->push_back(argv[index]); + continue; + } + if (index + 1 >= argc) { + std::fprintf(stderr, "Missing value after %s\n", argument.c_str()); + return false; + } + const char *value = argv[++index]; + if (argument == "--pmu-window") { + if (mode_seen) { + std::fprintf(stderr, "Specify --pmu-window only once.\n"); + return false; + } + const std::string name = value; + if (name == "off") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::Off; + } else if (name == "empty") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::Empty; + } else if (name == "scalar") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::Scalar; + } else if (name == "scalar-double") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::ScalarDouble; + } else if (name == "icache-single") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::IcacheSingle; + } else if (name == "submit-all") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::SubmitAll; + } else { + std::fprintf( + stderr, + "Invalid --pmu-window value: %s " + "(expected off|empty|scalar|scalar-double|icache-single|submit-all)\n", + value + ); + return false; + } + mode_seen = true; + } else if (argument == "--pmu-scalar-nops") { + if (nops_seen || !pa_scheduler::host::ParseUint(value, 0, 10000000, &pmu->scalar_nops)) { + std::fprintf(stderr, "Invalid or duplicate --pmu-scalar-nops value: %s\n", value); + return false; + } + nops_seen = true; + } else if (argument == "--pmu-icache-trials") { + if (icache_trials_seen || + !pa_scheduler::host::ParseUint(value, 1, 10000, &pmu->icache_trials)) { + std::fprintf(stderr, "Invalid or duplicate --pmu-icache-trials value: %s\n", value); + return false; + } + icache_trials_seen = true; + } else { + if (json_seen || *value == '\0') { + std::fprintf(stderr, "Invalid or duplicate --pmu-json path: %s\n", value); + return false; + } + pmu->json_path = value; + json_seen = true; + } + } + if (nops_seen && pmu->mode != pa_scheduler::ccec_pmu::WindowMode::Scalar && + pmu->mode != pa_scheduler::ccec_pmu::WindowMode::ScalarDouble) { + std::fprintf(stderr, "--pmu-scalar-nops requires a scalar PMU window.\n"); + return false; + } + if (icache_trials_seen && pmu->mode != pa_scheduler::ccec_pmu::WindowMode::IcacheSingle) { + std::fprintf(stderr, "--pmu-icache-trials requires the icache-single PMU window.\n"); + return false; + } + return true; +} + +using HalResMapFn = int (*)(uint32_t, struct res_map_info *, unsigned long *, uint32_t *); +using HalResUnmapFn = int (*)(uint32_t, struct res_map_info *); + +struct PmuRegisterMappings { + HalResUnmapFn unmap = nullptr; + std::vector mapped_resources; + std::vector register_bases; +}; + +bool UnmapPmuRegisters(uint32_t device, PmuRegisterMappings *mappings) { + bool ok = true; + if (mappings->unmap != nullptr) { + for (auto iterator = mappings->mapped_resources.rbegin(); iterator != mappings->mapped_resources.rend(); + ++iterator) { + const int error = mappings->unmap(device, &*iterator); + if (error != 0) { + std::fprintf(stderr, "halResUnmap failed for core %u (rc=%d)\n", iterator->res_id, error); + ok = false; + } + } + } + mappings->mapped_resources.clear(); + mappings->register_bases.clear(); + return ok; +} + +bool MapPmuRegisters(uint32_t device, PmuRegisterMappings *mappings) { + using namespace pa_scheduler::ccec_pmu; + const auto map = reinterpret_cast(dlsym(RTLD_DEFAULT, "halResMap")); + mappings->unmap = reinterpret_cast(dlsym(RTLD_DEFAULT, "halResUnmap")); + if (map == nullptr || mappings->unmap == nullptr) { + std::fprintf(stderr, "halResMap/halResUnmap is unavailable in the current CANN driver process.\n"); + return false; + } + + mappings->register_bases.assign(kPhysicalSubcoreCount, 0); + mappings->mapped_resources.reserve(kPhysicalAicoreCount); + for (uint32_t aicore = 0; aicore < kPhysicalAicoreCount; ++aicore) { + res_map_info info{}; + info.target_proc_type = PROCESS_CP1; + info.res_type = RES_AICORE; + info.res_id = aicore; + unsigned long map_address = 0; + uint32_t map_bytes = kAicoreMapBytes; + const int error = map(device, &info, &map_address, &map_bytes); + if (error != 0 || map_address == 0 || map_bytes < kAicoreMapBytes) { + std::fprintf( + stderr, "halResMap failed for core %u (rc=%d address=0x%lx bytes=%u)\n", aicore, error, + map_address, map_bytes + ); + (void)UnmapPmuRegisters(device, mappings); + return false; + } + mappings->mapped_resources.push_back(info); + + // 与正式 A5 host_regs.cpp 相同：每个 die 的布局为 18 AIC，随后是 36 AIV。 + const uint32_t die = aicore / kAicorePerDie; + const uint32_t local = aicore % kAicorePerDie; + const uint32_t die_base = die * kSubcoresPerDie; + mappings->register_bases[die_base + local] = static_cast(map_address); + mappings->register_bases[die_base + kAicorePerDie + local * 2] = + static_cast(map_address) + kAivFirstOffset; + mappings->register_bases[die_base + kAicorePerDie + local * 2 + 1] = + static_cast(map_address) + kAivSecondOffset; + } + return true; +} + +void ConfigurePmu(pa_scheduler::SchedulerState *state, const PmuOptions &pmu, const void *register_table) { + using namespace pa_scheduler::ccec_pmu; + state->config.reserved[kConfigMode] = static_cast(pmu.mode); + state->config.reserved[kConfigWorkAmount] = + pmu.mode == WindowMode::IcacheSingle ? pmu.icache_trials : pmu.scalar_nops; + StorePointer(state->config.reserved, register_table); + state->config.reserved[kConfigMagic] = pmu.mode == WindowMode::Off ? 0 : kConfigMagicValue; +} + +struct PmuAggregate { + std::vector total_cycles; + std::vector window_ticks; + std::vector submit_elapsed_ticks; + std::vector phase_elapsed_ticks; + std::vector warm_total_cycles; + std::vector warm_window_ticks; + std::vector vector_busy; + std::vector cube_busy; + std::vector scalar_busy; + std::vector mte1_busy; + std::vector mte2_busy; + std::vector icache_requests; + std::vector icache_misses; + std::vector warm_icache_requests; + std::vector warm_icache_misses; + std::vector fix_busy; + std::vector phase_calls; + std::vector phase_icache_requests; + std::vector phase_icache_misses; + std::vector shadow_icache_requests; + std::vector shadow_icache_misses; + uint32_t trusted = 0; +}; + +void AddPmuSample(const pa_scheduler::WorkerResult &result, PmuAggregate *aggregate) { + aggregate->total_cycles.push_back(result.pmu_total_cycles); + aggregate->window_ticks.push_back(result.pmu_window_ticks); + aggregate->submit_elapsed_ticks.push_back( + result.submit_end >= result.submit_begin ? result.submit_end - result.submit_begin : 0U + ); + aggregate->phase_elapsed_ticks.push_back(result.pmu_phase_elapsed_ticks); + aggregate->warm_total_cycles.push_back(result.pmu_warm_total_cycles); + aggregate->warm_window_ticks.push_back(result.pmu_warm_window_ticks); + aggregate->vector_busy.push_back(result.pmu_vector_busy); + aggregate->cube_busy.push_back(result.pmu_cube_busy); + aggregate->scalar_busy.push_back(result.pmu_scalar_busy); + aggregate->mte1_busy.push_back(result.pmu_mte1_busy); + aggregate->mte2_busy.push_back(result.pmu_mte2_busy); + aggregate->icache_requests.push_back(result.pmu_icache_requests); + aggregate->icache_misses.push_back(result.pmu_icache_misses); + aggregate->warm_icache_requests.push_back(result.pmu_warm_icache_requests); + aggregate->warm_icache_misses.push_back(result.pmu_warm_icache_misses); + aggregate->fix_busy.push_back(result.pmu_fix_busy); + aggregate->phase_calls.push_back(result.pmu_phase_calls); + aggregate->phase_icache_requests.push_back(result.pmu_phase_icache_requests); + aggregate->phase_icache_misses.push_back(result.pmu_phase_icache_misses); + aggregate->shadow_icache_requests.push_back(result.pmu_shadow_icache_requests); + aggregate->shadow_icache_misses.push_back(result.pmu_shadow_icache_misses); + aggregate->trusted += + (result.pmu_status & pa_scheduler::ccec_pmu::kStatusRequired) == + pa_scheduler::ccec_pmu::kStatusRequired && + (result.pmu_phase_status & pa_scheduler::ccec_pmu::kPhaseStatusRequired) == + pa_scheduler::ccec_pmu::kPhaseStatusRequired; +} + +bool PrintSingleIcacheAggregate( + const char *name, const PmuAggregate &aggregate, uint32_t trials_per_core +) { + const pa_scheduler::host::Uint64Distribution cold_cycles = + pa_scheduler::host::SummarizeUint64(aggregate.total_cycles); + const pa_scheduler::host::Uint64Distribution warm_cycles = + pa_scheduler::host::SummarizeUint64(aggregate.warm_total_cycles); + const pa_scheduler::host::Uint64Distribution cold_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.window_ticks); + const pa_scheduler::host::Uint64Distribution warm_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.warm_window_ticks); + const pa_scheduler::host::Uint64Distribution cold_requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution warm_requests = + pa_scheduler::host::SummarizeUint64(aggregate.warm_icache_requests); + const pa_scheduler::host::Uint64Distribution cold_misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + const pa_scheduler::host::Uint64Distribution warm_misses = + pa_scheduler::host::SummarizeUint64(aggregate.warm_icache_misses); + const int64_t cycle_delta = static_cast(cold_cycles.total) - + static_cast(warm_cycles.total); + const int64_t tick_delta = static_cast(cold_ticks.total) - + static_cast(warm_ticks.total); + const int64_t miss_delta = static_cast(cold_misses.total) - + static_cast(warm_misses.total); + const uint64_t attempted = static_cast(trials_per_core) * aggregate.total_cycles.size(); + const double misses_per_trial = attempted == 0U ? 0.0 : static_cast(miss_delta) / attempted; + const double cycles_per_miss = miss_delta <= 0 ? 0.0 : static_cast(cycle_delta) / miss_delta; + // 本用例的 get_sys_cnt 已按 1 GHz 时间基准校准，因此一个 tick 对应 1 ns。 + const double ns_per_miss = miss_delta <= 0 ? 0.0 : static_cast(tick_delta) / miss_delta; + std::printf( + "[ICACHE-SINGLE-%s] cores=%zu trials_per_core=%u attempted=%llu " + "cold_cycles=%llu warm_cycles=%llu cycle_delta=%lld cold_ticks=%llu warm_ticks=%llu " + "tick_delta=%lld cold_req=%llu warm_req=%llu cold_miss=%llu warm_miss=%llu " + "miss_delta=%lld misses_per_trial=%.6f cycles_per_miss=%.3f ns_per_miss=%.3f\n", + name, aggregate.total_cycles.size(), trials_per_core, static_cast(attempted), + static_cast(cold_cycles.total), + static_cast(warm_cycles.total), static_cast(cycle_delta), + static_cast(cold_ticks.total), + static_cast(warm_ticks.total), static_cast(tick_delta), + static_cast(cold_requests.total), + static_cast(warm_requests.total), + static_cast(cold_misses.total), + static_cast(warm_misses.total), static_cast(miss_delta), + misses_per_trial, cycles_per_miss, ns_per_miss + ); + std::printf( + "[ICACHE-FORMULA-%s] estimated_scalar_icache_time_ns = cnt7_icache_miss * %.3f\n", + name, ns_per_miss + ); + return cycle_delta > 0 && tick_delta > 0 && miss_delta == static_cast(attempted); +} + +void PrintPmuAggregate(const char *name, const PmuAggregate &aggregate) { + const pa_scheduler::host::Uint64Distribution total = + pa_scheduler::host::SummarizeUint64(aggregate.total_cycles); + const pa_scheduler::host::Uint64Distribution scalar = + pa_scheduler::host::SummarizeUint64(aggregate.scalar_busy); + const pa_scheduler::host::Uint64Distribution vector = + pa_scheduler::host::SummarizeUint64(aggregate.vector_busy); + const pa_scheduler::host::Uint64Distribution cube = + pa_scheduler::host::SummarizeUint64(aggregate.cube_busy); + const pa_scheduler::host::Uint64Distribution mte1 = + pa_scheduler::host::SummarizeUint64(aggregate.mte1_busy); + const pa_scheduler::host::Uint64Distribution mte2 = + pa_scheduler::host::SummarizeUint64(aggregate.mte2_busy); + const pa_scheduler::host::Uint64Distribution requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + const double miss_rate = requests.total == 0 ? 0.0 : 100.0 * misses.total / requests.total; + std::printf( + "[PMU-%s] cores=%zu total_sum=%llu total_median=%.1f total_p95=%llu " + "scalar_busy=%llu vector_busy=%llu cube_busy=%llu mte1_busy=%llu mte2_busy=%llu " + "icache_req=%llu icache_miss=%llu miss_rate=%.4f%%\n", + name, aggregate.total_cycles.size(), static_cast(total.total), total.median, + static_cast(total.p95), static_cast(scalar.total), + static_cast(vector.total), static_cast(cube.total), + static_cast(mte1.total), static_cast(mte2.total), + static_cast(requests.total), static_cast(misses.total), miss_rate + ); +} + +void PrintSubmitPmuPhaseAggregate(const char *name, const PmuAggregate &aggregate) { + const pa_scheduler::host::Uint64Distribution calls = + pa_scheduler::host::SummarizeUint64(aggregate.phase_calls); + const pa_scheduler::host::Uint64Distribution requests = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_requests); + const pa_scheduler::host::Uint64Distribution misses = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_misses); + const pa_scheduler::host::Uint64Distribution primary_requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution primary_misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + const pa_scheduler::host::Uint64Distribution shadow_requests = + pa_scheduler::host::SummarizeUint64(aggregate.shadow_icache_requests); + const pa_scheduler::host::Uint64Distribution shadow_misses = + pa_scheduler::host::SummarizeUint64(aggregate.shadow_icache_misses); + const pa_scheduler::host::Uint64Distribution submit_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.submit_elapsed_ticks); + const pa_scheduler::host::Uint64Distribution phase_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.phase_elapsed_ticks); + const uint64_t request_loss = primary_requests.total >= shadow_requests.total + ? primary_requests.total - shadow_requests.total + : 0U; + const uint64_t miss_loss = primary_misses.total >= shadow_misses.total + ? primary_misses.total - shadow_misses.total + : 0U; + const double miss_rate = requests.total == 0U ? 0.0 : 100.0 * misses.total / requests.total; + const double phase_time_share = submit_ticks.total == 0U + ? 0.0 + : 100.0 * phase_ticks.total / submit_ticks.total; + std::printf( + "[PMU-PHASE-%s] phase=%s semantics=%s cores=%zu calls=%llu " + "icache_req=[%llu,%llu] icache_miss=[%llu,%llu] " + "observed_read_clear_ratio=%.4f%% phase_ticks=%llu submit_ticks=%llu " + "phase_time_share=%.4f%% shadow_loss=%llu/%llu\n", + name, pa_scheduler::ccec_pmu::SubmitPmuPhaseName(pa_scheduler::kCompiledSubmitPmuPhase), + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "disabled" + : "running_read_clear_lower_bound", + aggregate.phase_calls.size(), static_cast(calls.total), + static_cast(requests.total), + static_cast(requests.total + request_loss), + static_cast(misses.total), + static_cast(misses.total + miss_loss), miss_rate, + static_cast(phase_ticks.total), + static_cast(submit_ticks.total), phase_time_share, + static_cast(request_loss), + static_cast(miss_loss) + ); +} + +struct PmuValidation { + uint32_t trusted = 0; + uint32_t unique_physical_core_ids = 0; + uint32_t owner_bitmap_members = 0; + uint32_t exact_worker_slots = 0; + uint32_t physical_role_matches = 0; + uint32_t mixed_triplet_matches = 0; + uint32_t window_started = 0; + uint32_t window_stopped = 0; + uint32_t icache_pairs = 0; + uint32_t icache_calibrated_cores = 0; + uint32_t prior_snapshot_larger = 0; + uint32_t submit_engine_workers_expected = 0; + uint32_t submit_engine_workers_matched = 0; + uint32_t maximum_programmable_counter = 0; + uint32_t build_variant_matches = 0; + uint32_t phase_id_matches = 0; + uint32_t phase_status_trusted = 0; + uint32_t shadow_primary_matches = 0; + uint32_t shadow_primary_bounded = 0; + uint32_t phase_shadow_acceptable = 0; + uint32_t phase_boundary_matches = 0; + uint32_t phase_call_shape_matches = 0; + uint32_t phase_time_valid_records = 0; + uint64_t phase_calls = 0; + uint64_t expected_phase_calls = 0; + uint64_t shadow_request_abs_delta_sum = 0; + uint64_t shadow_miss_abs_delta_sum = 0; + int64_t shadow_request_signed_delta_sum = 0; + int64_t shadow_miss_signed_delta_sum = 0; + uint32_t shadow_request_abs_delta_max = 0; + uint32_t shadow_miss_abs_delta_max = 0; + bool icache_order_valid = true; + bool icache_measurement_valid = true; + bool submit_engine_observation_valid = true; + bool counter_below_risk_threshold = true; + bool phase_measurement_valid = false; + bool passed = true; +}; + +// 32-bit programmable counter 无法仅凭终值证明从未回卷。正式文件采用 25% +// 高水位作为保守拒绝阈值；它只降低风险，不把“未越线”表述成回卷证明。 +constexpr uint32_t kProgrammableCounterRiskThreshold = UINT32_MAX / 4U; + +uint32_t ExpectedSubmitPmuPhaseCallsPerWorker(uint32_t batches) { + // 当前所有 running phase 都覆盖每个 worker 的每次 Submit，固定为 5B。 + switch (pa_scheduler::kCompiledSubmitPmuPhase) { + case pa_scheduler::SubmitPmuPhase::None: + return 0U; + case pa_scheduler::SubmitPmuPhase::Claim: + case pa_scheduler::SubmitPmuPhase::EfDrain: + case pa_scheduler::SubmitPmuPhase::Materialize: + case pa_scheduler::SubmitPmuPhase::Register: + return batches * pa_scheduler::kTasksPerBatch; + case pa_scheduler::SubmitPmuPhase::Count: + break; + } + return UINT32_MAX; +} + +bool ValidatePmu( + const pa_scheduler::SchedulerState &state, uint32_t run, const PmuOptions &pmu, + const WinnerWorkloadOptions &workload, + const pa_scheduler::pmu_owner::PmuOwnerControl *owner, PmuValidation *validation +) { + using namespace pa_scheduler::ccec_pmu; + if (pmu.mode == WindowMode::Off) { + *validation = PmuValidation{}; + return true; + } + + bool seen[kPhysicalSubcoreCount] = {}; + uint32_t trusted = 0; + uint32_t unique = 0; + uint32_t owner_members = 0; + uint32_t exact_worker_slots = 0; + uint32_t physical_role_matches = 0; + uint32_t window_started = 0; + uint32_t window_stopped = 0; + uint32_t icache_pairs = 0; + uint32_t icache_calibrated_cores = 0; + uint32_t prior_larger = 0; + uint32_t submit_engine_workers_expected = 0; + uint32_t submit_engine_workers_matched = 0; + uint32_t maximum_programmable_counter = 0; + uint32_t build_variant_matches = 0; + uint32_t phase_id_matches = 0; + uint32_t phase_status_trusted = 0; + uint32_t shadow_primary_matches = 0; + uint32_t shadow_primary_bounded = 0; + uint32_t phase_shadow_acceptable = 0; + uint32_t phase_boundary_matches = 0; + uint32_t phase_call_shape_matches = 0; + uint32_t phase_time_valid_records = 0; + uint64_t phase_calls = 0; + uint64_t expected_phase_calls = 0; + uint64_t shadow_request_abs_delta_sum = 0; + uint64_t shadow_miss_abs_delta_sum = 0; + int64_t shadow_request_signed_delta_sum = 0; + int64_t shadow_miss_signed_delta_sum = 0; + uint32_t shadow_request_abs_delta_max = 0; + uint32_t shadow_miss_abs_delta_max = 0; + bool icache_order_valid = true; + uint32_t bad_printed = 0; + PmuAggregate all; + PmuAggregate aic; + PmuAggregate aiv; + for (uint32_t worker = 0; worker < pa_scheduler::kWorkers; ++worker) { + const pa_scheduler::WorkerResult &result = state.results[worker]; + const uint32_t expected_phase_calls_per_worker = + ExpectedSubmitPmuPhaseCallsPerWorker(state.config.batches); + const uint32_t status = result.pmu_status; + const uint32_t core_id = StatusCoreId(status); + const bool record_trusted = (status & kStatusRequired) == kStatusRequired; + const bool variant_matches = result.pmu_build_variant == pa_scheduler::kBuildVariantSubmitPmu; + const bool phase_id_matches_record = + result.pmu_phase_id == static_cast(pa_scheduler::kCompiledSubmitPmuPhase); + const bool phase_status_ok = + (result.pmu_phase_status & kPhaseStatusRequired) == kPhaseStatusRequired; + const bool shadow_matches = + result.pmu_shadow_icache_requests == result.pmu_icache_requests && + result.pmu_shadow_icache_misses == result.pmu_icache_misses; + const bool shadow_bounded = + result.pmu_shadow_icache_requests <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_icache_misses; + const bool phase_requires_exact_shadow = + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None; + const bool shadow_acceptable = + phase_requires_exact_shadow + ? shadow_matches + : shadow_bounded; + const uint32_t request_abs_delta = + result.pmu_shadow_icache_requests >= result.pmu_icache_requests + ? result.pmu_shadow_icache_requests - result.pmu_icache_requests + : result.pmu_icache_requests - result.pmu_shadow_icache_requests; + const uint32_t miss_abs_delta = + result.pmu_shadow_icache_misses >= result.pmu_icache_misses + ? result.pmu_shadow_icache_misses - result.pmu_icache_misses + : result.pmu_icache_misses - result.pmu_shadow_icache_misses; + const bool boundaries_match = + result.pmu_phase_begin_reads == result.pmu_phase_calls && + result.pmu_phase_end_reads == result.pmu_phase_calls; + const bool phase_call_shape_matches_record = + result.pmu_phase_calls == expected_phase_calls_per_worker; + const uint64_t submit_elapsed_ticks = result.submit_end >= result.submit_begin + ? result.submit_end - result.submit_begin + : 0U; + const bool phase_time_valid_record = + submit_elapsed_ticks != 0U && + result.pmu_phase_elapsed_ticks <= submit_elapsed_ticks && + (pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? result.pmu_phase_elapsed_ticks == 0U + : result.pmu_phase_elapsed_ticks != 0U); + const bool logical_aic = worker < pa_scheduler::kAicWorkers; + const bool physical_aic = pa_scheduler::pmu_owner::IsAicPhysicalSlot(core_id); + trusted += record_trusted; + build_variant_matches += variant_matches; + phase_id_matches += phase_id_matches_record; + phase_status_trusted += phase_status_ok; + shadow_primary_matches += shadow_matches; + shadow_primary_bounded += shadow_bounded; + phase_shadow_acceptable += shadow_acceptable; + phase_boundary_matches += boundaries_match; + phase_call_shape_matches += phase_call_shape_matches_record; + phase_time_valid_records += phase_time_valid_record; + phase_calls += result.pmu_phase_calls; + expected_phase_calls += expected_phase_calls_per_worker; + shadow_request_abs_delta_sum += request_abs_delta; + shadow_miss_abs_delta_sum += miss_abs_delta; + shadow_request_signed_delta_sum += + static_cast(result.pmu_shadow_icache_requests) - result.pmu_icache_requests; + shadow_miss_signed_delta_sum += + static_cast(result.pmu_shadow_icache_misses) - result.pmu_icache_misses; + shadow_request_abs_delta_max = std::max(shadow_request_abs_delta_max, request_abs_delta); + shadow_miss_abs_delta_max = std::max(shadow_miss_abs_delta_max, miss_abs_delta); + owner_members += owner != nullptr && pa_scheduler::pmu_owner::IsConfigured(*owner, core_id); + exact_worker_slots += result.worker_id == worker; + physical_role_matches += logical_aic == physical_aic; + window_started += (status & kStatusWindowStarted) != 0U; + window_stopped += (status & kStatusWindowStopped) != 0U; + icache_pairs += (status & kStatusIcachePairObserved) != 0U; + prior_larger += (status & kStatusPriorSnapshotLarger) != 0; + if (pmu.mode == WindowMode::SubmitAll && + workload.mode == pa_scheduler::WinnerWorkloadMode::RealCompute) { + const uint64_t submit_engine_tasks = + result.placement[static_cast(pa_scheduler::DrainPlace::EfDrain)] + + result.placement[static_cast(pa_scheduler::DrainPlace::RingBackpressure)]; + const uint32_t relevant_busy = logical_aic ? result.pmu_cube_busy : result.pmu_vector_busy; + const bool engine_observation_matches = + (submit_engine_tasks == 0U) == (relevant_busy == 0U); + submit_engine_workers_expected += submit_engine_tasks != 0U; + submit_engine_workers_matched += engine_observation_matches; + } + // phase 的 request/miss 由两条顺序 ld_dev 划界，局部窗口边界并不 + // 完全重合；只要求它们各自不超过完整窗口，不把 phase miss<=request + // 误设成硬门槛。完整 Submit 的 miss<=request 仍必须成立。 + icache_order_valid &= result.pmu_icache_misses <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_shadow_icache_requests && + result.pmu_shadow_icache_requests <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_icache_misses && + result.pmu_phase_icache_requests <= result.pmu_shadow_icache_requests && + result.pmu_phase_icache_misses <= result.pmu_shadow_icache_misses; + if (pmu.mode == WindowMode::IcacheSingle) { + const int64_t worker_cycle_delta = static_cast(result.pmu_total_cycles) - + static_cast(result.pmu_warm_total_cycles); + const int64_t worker_tick_delta = static_cast(result.pmu_window_ticks) - + static_cast(result.pmu_warm_window_ticks); + const int64_t worker_miss_delta = static_cast(result.pmu_icache_misses) - + static_cast(result.pmu_warm_icache_misses); + icache_calibrated_cores += worker_cycle_delta > 0 && worker_tick_delta > 0 && + worker_miss_delta == static_cast(pmu.icache_trials) && + result.pmu_warm_icache_misses == 0U && + result.pmu_icache_misses == pmu.icache_trials; + } + const uint32_t programmable[] = { + result.pmu_vector_busy, result.pmu_cube_busy, result.pmu_scalar_busy, + result.pmu_mte1_busy, result.pmu_mte2_busy, + result.pmu_icache_requests, result.pmu_icache_misses, + result.pmu_phase_icache_requests, result.pmu_phase_icache_misses, + result.pmu_shadow_icache_requests, result.pmu_shadow_icache_misses, + }; + for (uint32_t value : programmable) { + maximum_programmable_counter = std::max(maximum_programmable_counter, value); + } + if (core_id < kPhysicalSubcoreCount && !seen[core_id]) { + seen[core_id] = true; + ++unique; + } + if ((!record_trusted || !variant_matches || !phase_id_matches_record || + !phase_status_ok || !shadow_acceptable || !boundaries_match || + !phase_call_shape_matches_record || !phase_time_valid_record) && bad_printed < 8) { + std::printf( + "[PMU-BAD] worker=%u role=%llu coreid=%u status=0x%08x total=%llu scalar=%u " + "req=%u miss=%u phase_status=0x%08x phase=%u/%u calls=%u/%u boundaries=%u/%u " + "phase_ticks=%llu submit_ticks=%llu shadow=%u/%u\n", + worker, static_cast(result.role), core_id, status, + static_cast(result.pmu_total_cycles), result.pmu_scalar_busy, + result.pmu_icache_requests, result.pmu_icache_misses, + result.pmu_phase_status, result.pmu_phase_id, + static_cast(pa_scheduler::kCompiledSubmitPmuPhase), result.pmu_phase_calls, + expected_phase_calls_per_worker, + result.pmu_phase_begin_reads, result.pmu_phase_end_reads, + static_cast(result.pmu_phase_elapsed_ticks), + static_cast(submit_elapsed_ticks), + result.pmu_shadow_icache_requests, result.pmu_shadow_icache_misses + ); + ++bad_printed; + } + AddPmuSample(result, &all); + AddPmuSample(result, result.role == static_cast(pa_scheduler::CoreRole::Aic) ? &aic : &aiv); + } + + uint32_t mixed_triplet_matches = 0U; + for (uint32_t block = 0U; block < pa_scheduler::kAicWorkers; ++block) { + const uint32_t aic_id = StatusCoreId(state.results[block].pmu_status); + if (!pa_scheduler::pmu_owner::IsAicPhysicalSlot(aic_id)) continue; + const uint32_t die_base = (aic_id / pa_scheduler::pmu_owner::kSubcoresPerDie) * + pa_scheduler::pmu_owner::kSubcoresPerDie; + const uint32_t local = aic_id % pa_scheduler::pmu_owner::kSubcoresPerDie; + const uint32_t expected_aiv0 = die_base + pa_scheduler::pmu_owner::kAicPerDie + local * 2U; + const uint32_t expected_aiv1 = expected_aiv0 + 1U; + const uint32_t aiv0_id = StatusCoreId( + state.results[pa_scheduler::kAicWorkers + block * 2U].pmu_status + ); + const uint32_t aiv1_id = StatusCoreId( + state.results[pa_scheduler::kAicWorkers + block * 2U + 1U].pmu_status + ); + mixed_triplet_matches += aiv0_id == expected_aiv0 && aiv1_id == expected_aiv1; + } + + PrintPmuAggregate("ALL", all); + PrintPmuAggregate("AIC", aic); + PrintPmuAggregate("AIV", aiv); + PrintSubmitPmuPhaseAggregate("ALL", all); + PrintSubmitPmuPhaseAggregate("AIC", aic); + PrintSubmitPmuPhaseAggregate("AIV", aiv); + bool icache_measurement_ok = true; + if (pmu.mode == WindowMode::IcacheSingle) { + const bool all_ok = PrintSingleIcacheAggregate("ALL", all, pmu.icache_trials); + const bool aic_ok = PrintSingleIcacheAggregate("AIC", aic, pmu.icache_trials); + const bool aiv_ok = PrintSingleIcacheAggregate("AIV", aiv, pmu.icache_trials); + icache_measurement_ok = all_ok && aic_ok && aiv_ok && + icache_pairs == pa_scheduler::kWorkers && + icache_calibrated_cores == pa_scheduler::kWorkers; + } + const bool records_ok = trusted == pa_scheduler::kWorkers; + const bool core_ids_ok = unique == pa_scheduler::kWorkers; + const bool owner_members_ok = owner_members == pa_scheduler::kWorkers; + const bool worker_slots_ok = exact_worker_slots == pa_scheduler::kWorkers; + const bool physical_roles_ok = physical_role_matches == pa_scheduler::kWorkers; + const bool mixed_triplets_ok = mixed_triplet_matches == pa_scheduler::kAicWorkers; + const bool windows_started_ok = window_started == pa_scheduler::kWorkers; + const bool windows_stopped_ok = window_stopped == pa_scheduler::kWorkers; + const bool build_variant_ok = build_variant_matches == pa_scheduler::kWorkers; + const bool phase_id_ok = phase_id_matches == pa_scheduler::kWorkers; + const bool phase_status_ok = phase_status_trusted == pa_scheduler::kWorkers; + const bool shadow_partition_ok = phase_shadow_acceptable == pa_scheduler::kWorkers; + const bool phase_boundaries_ok = phase_boundary_matches == pa_scheduler::kWorkers; + const bool phase_call_shape_ok = phase_call_shape_matches == pa_scheduler::kWorkers; + const bool phase_time_ok = phase_time_valid_records == pa_scheduler::kWorkers; + const bool phase_calls_ok = phase_calls == expected_phase_calls; + const bool submit_engine_observation_ok = + pmu.mode != WindowMode::SubmitAll || + workload.mode != pa_scheduler::WinnerWorkloadMode::RealCompute || + submit_engine_workers_matched == pa_scheduler::kWorkers; + const bool counter_below_risk_threshold = + maximum_programmable_counter < kProgrammableCounterRiskThreshold; + std::printf( + "[PMU] run=%u window=%s calibration_scalar_nops=%u icache_trials=%u trusted=%u/%u " + "unique_coreids=%u/%u prior_larger=%u/%u icache_pairs=%u/%u calibrated_cores=%u/%u " + "programmable_max=%u headroom=%u\n", run, + PmuModeName(pmu.mode), pmu.scalar_nops, pmu.icache_trials, trusted, pa_scheduler::kWorkers, + unique, pa_scheduler::kWorkers, prior_larger, pa_scheduler::kWorkers, + icache_pairs, pa_scheduler::kWorkers, icache_calibrated_cores, pa_scheduler::kWorkers, + maximum_programmable_counter, + UINT32_MAX - maximum_programmable_counter + ); + std::printf( + "[PMU-SHADOW-DELTA] exact=%u/%u bounded=%u/%u request_abs_sum=%llu request_abs_max=%u " + "request_signed_sum=%lld miss_abs_sum=%llu miss_abs_max=%u miss_signed_sum=%lld\n", + shadow_primary_matches, pa_scheduler::kWorkers, + shadow_primary_bounded, pa_scheduler::kWorkers, + static_cast(shadow_request_abs_delta_sum), + shadow_request_abs_delta_max, static_cast(shadow_request_signed_delta_sum), + static_cast(shadow_miss_abs_delta_sum), + shadow_miss_abs_delta_max, static_cast(shadow_miss_signed_delta_sum) + ); + std::printf("[ASSERT] %-48s %s\n", "all PMU records have configured selectors and data", + records_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 96 PMU physical subcore ids are unique", + core_ids_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all PMU physical ids belong to the owner bitmap", + owner_members_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "worker result slots and ids match exactly", + worker_slots_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "logical AIC/AIV roles match physical subcores", + physical_roles_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 32 mixed blocks map to physical 1:2 triplets", + mixed_triplets_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 96 PMU windows executed start", + windows_started_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 96 started PMU windows executed stop", + windows_stopped_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all records match submit-pmu build and phase ids", + build_variant_ok && phase_id_ok ? "PASS" : "FAIL"); + std::printf( + "[ASSERT] %-48s %s\n", + "phase shadow partitions satisfy exact-or-bounded contract", + shadow_partition_ok ? "PASS" : "FAIL" + ); + std::printf("[ASSERT] %-48s %s\n", "all phase boundaries and per-worker calls are exact", + phase_status_ok && phase_boundaries_ok && phase_call_shape_ok && phase_calls_ok + ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all phase times fit their per-worker Submit windows", + phase_time_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "I-cache misses do not exceed requests", + icache_order_valid ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "programmable counters stay below 25% risk threshold", + counter_below_risk_threshold ? "PASS" : "FAIL"); + if (pmu.mode == WindowMode::SubmitAll && + workload.mode == pa_scheduler::WinnerWorkloadMode::RealCompute) { + std::printf( + "[ASSERT] %-48s %s (active_workers=%u matched_workers=%u/%u)\n", + "Submit placement has matching AIC/AIV engine PMU", + submit_engine_observation_ok ? "PASS" : "FAIL", submit_engine_workers_expected, + submit_engine_workers_matched, pa_scheduler::kWorkers + ); + } + if (pmu.mode == WindowMode::IcacheSingle) { + std::printf("[ASSERT] %-48s %s\n", "each cold trial adds exactly one CNT7 I-cache miss", + icache_measurement_ok ? "PASS" : "FAIL"); + } + validation->trusted = trusted; + validation->unique_physical_core_ids = unique; + validation->owner_bitmap_members = owner_members; + validation->exact_worker_slots = exact_worker_slots; + validation->physical_role_matches = physical_role_matches; + validation->mixed_triplet_matches = mixed_triplet_matches; + validation->window_started = window_started; + validation->window_stopped = window_stopped; + validation->icache_pairs = icache_pairs; + validation->icache_calibrated_cores = icache_calibrated_cores; + validation->prior_snapshot_larger = prior_larger; + validation->submit_engine_workers_expected = submit_engine_workers_expected; + validation->submit_engine_workers_matched = submit_engine_workers_matched; + validation->maximum_programmable_counter = maximum_programmable_counter; + validation->build_variant_matches = build_variant_matches; + validation->phase_id_matches = phase_id_matches; + validation->phase_status_trusted = phase_status_trusted; + validation->shadow_primary_matches = shadow_primary_matches; + validation->shadow_primary_bounded = shadow_primary_bounded; + validation->phase_shadow_acceptable = phase_shadow_acceptable; + validation->phase_boundary_matches = phase_boundary_matches; + validation->phase_call_shape_matches = phase_call_shape_matches; + validation->phase_time_valid_records = phase_time_valid_records; + validation->phase_calls = phase_calls; + validation->expected_phase_calls = expected_phase_calls; + validation->shadow_request_abs_delta_sum = shadow_request_abs_delta_sum; + validation->shadow_miss_abs_delta_sum = shadow_miss_abs_delta_sum; + validation->shadow_request_signed_delta_sum = shadow_request_signed_delta_sum; + validation->shadow_miss_signed_delta_sum = shadow_miss_signed_delta_sum; + validation->shadow_request_abs_delta_max = shadow_request_abs_delta_max; + validation->shadow_miss_abs_delta_max = shadow_miss_abs_delta_max; + validation->icache_order_valid = icache_order_valid; + validation->icache_measurement_valid = icache_measurement_ok; + validation->submit_engine_observation_valid = submit_engine_observation_ok; + validation->counter_below_risk_threshold = counter_below_risk_threshold; + validation->phase_measurement_valid = + build_variant_ok && phase_id_ok && phase_status_ok && shadow_partition_ok && + phase_boundaries_ok && phase_call_shape_ok && phase_calls_ok && phase_time_ok; + validation->passed = records_ok && core_ids_ok && owner_members_ok && worker_slots_ok && + physical_roles_ok && mixed_triplets_ok && windows_started_ok && windows_stopped_ok && + icache_order_valid && icache_measurement_ok && submit_engine_observation_ok && + validation->phase_measurement_valid && + counter_below_risk_threshold; + return validation->passed; +} + +void WriteJsonString(std::FILE *output, const std::string &value) { + std::fputc('"', output); + for (unsigned char character : value) { + switch (character) { + case '"': + std::fputs("\\\"", output); + break; + case '\\': + std::fputs("\\\\", output); + break; + case '\b': + std::fputs("\\b", output); + break; + case '\f': + std::fputs("\\f", output); + break; + case '\n': + std::fputs("\\n", output); + break; + case '\r': + std::fputs("\\r", output); + break; + case '\t': + std::fputs("\\t", output); + break; + default: + if (character < 0x20U) { + std::fprintf(output, "\\u%04x", static_cast(character)); + } else { + std::fputc(character, output); + } + } + } + std::fputc('"', output); +} + +void WriteMetricDistribution(std::FILE *output, const std::vector &values) { + const pa_scheduler::host::Uint64Distribution summary = pa_scheduler::host::SummarizeUint64(values); + const double mean = values.empty() ? 0.0 : static_cast(summary.total) / values.size(); + std::fprintf( + output, "{\"sum\":%llu,\"mean\":%.17g,\"median\":%.17g,\"p95\":%llu,\"max\":%llu}", + static_cast(summary.total), mean, summary.median, + static_cast(summary.p95), static_cast(summary.maximum) + ); +} + +void WritePmuAggregateJson( + std::FILE *output, const PmuAggregate &aggregate, bool icache_single +) { + (void)icache_single; + const pa_scheduler::host::Uint64Distribution requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + uint32_t active_cores = 0; + for (uint64_t cycles : aggregate.total_cycles) active_cores += cycles != 0; + std::fprintf( + output, "{\"cores\":%zu,\"active_cores\":%u,\"trusted_cores\":%u,\"total_cycles\":", + aggregate.total_cycles.size(), active_cores, aggregate.trusted + ); + WriteMetricDistribution(output, aggregate.total_cycles); + std::fputs(",\"vector_busy\":", output); + WriteMetricDistribution(output, aggregate.vector_busy); + std::fputs(",\"cube_busy\":", output); + WriteMetricDistribution(output, aggregate.cube_busy); + std::fputs(",\"scalar_busy\":", output); + WriteMetricDistribution(output, aggregate.scalar_busy); + std::fputs(",\"mte1_busy\":", output); + WriteMetricDistribution(output, aggregate.mte1_busy); + std::fputs(",\"mte2_busy\":", output); + WriteMetricDistribution(output, aggregate.mte2_busy); + std::fputs(",\"icache_requests\":", output); + WriteMetricDistribution(output, aggregate.icache_requests); + std::fputs(",\"icache_misses\":", output); + WriteMetricDistribution(output, aggregate.icache_misses); + std::fputs(",\"shadow_whole_icache_requests\":", output); + WriteMetricDistribution(output, aggregate.shadow_icache_requests); + std::fputs(",\"shadow_whole_icache_misses\":", output); + WriteMetricDistribution(output, aggregate.shadow_icache_misses); + std::fputs(",\"phase_calls\":", output); + WriteMetricDistribution(output, aggregate.phase_calls); + std::fputs(",\"submit_elapsed_ticks\":", output); + WriteMetricDistribution(output, aggregate.submit_elapsed_ticks); + std::fputs(",\"phase_elapsed_ticks\":", output); + WriteMetricDistribution(output, aggregate.phase_elapsed_ticks); + std::fputs(",\"phase_icache_requests\":", output); + WriteMetricDistribution(output, aggregate.phase_icache_requests); + std::fputs(",\"phase_icache_misses\":", output); + WriteMetricDistribution(output, aggregate.phase_icache_misses); + std::fputs(",\"icache_miss_rate\":", output); + if (requests.total == 0) { + std::fputs("null", output); + } else { + // 全局 miss rate 必须以总 miss/总 request 计算，不能平均逐核百分比。 + std::fprintf(output, "%.17g", static_cast(misses.total) / requests.total); + } + const pa_scheduler::host::Uint64Distribution phase_requests = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_requests); + const pa_scheduler::host::Uint64Distribution phase_misses = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_misses); + // 两个 phase counter 是顺序 read-to-clear，下界之比不是实际 miss rate + // 的数学下界；字段名只陈述它是本次 read-clear 观察值之比。 + std::fputs(",\"phase_observed_read_clear_ratio\":", output); + if (phase_requests.total == 0U) { + std::fputs("null", output); + } else { + std::fprintf(output, "%.17g", static_cast(phase_misses.total) / phase_requests.total); + } + std::fputc('}', output); +} + +uint32_t CountConfiguredMixedTriplets(const pa_scheduler::pmu_owner::PmuOwnerControl &owner) { + uint32_t complete = 0U; + for (uint32_t die_base = 0U; + die_base < pa_scheduler::pmu_owner::kPhysicalSubcoreCount; + die_base += pa_scheduler::pmu_owner::kSubcoresPerDie) { + for (uint32_t local = 0U; local < pa_scheduler::pmu_owner::kAicPerDie; ++local) { + const uint32_t aic = die_base + local; + const uint32_t aiv0 = die_base + pa_scheduler::pmu_owner::kAicPerDie + local * 2U; + const uint32_t aiv1 = aiv0 + 1U; + complete += pa_scheduler::pmu_owner::IsConfigured(owner, aic) && + pa_scheduler::pmu_owner::IsConfigured(owner, aiv0) && + pa_scheduler::pmu_owner::IsConfigured(owner, aiv1); + } + } + return complete; +} + +bool ExportPmuJson( + const pa_scheduler::SchedulerState &state, const pa_scheduler::host::Options &options, + const PmuOptions &pmu, const WinnerWorkloadOptions &workload, + uint32_t run, double host_us, double submit_span_us, + const PmuValidation &validation, bool semantic_passed, bool workload_output_passed, + const pa_scheduler::pmu_owner::PmuOwnerControl &owner, bool restore_passed, + const std::string &output_path +) { + using namespace pa_scheduler::ccec_pmu; + PmuAggregate all; + PmuAggregate aic; + PmuAggregate aiv; + uint32_t active_output_tiles = 0; + uint64_t ef_drain_kernels = 0; + uint64_t ring_backpressure_kernels = 0; + uint64_t final_drain_kernels = 0; + for (uint32_t worker = 0; worker < pa_scheduler::kWorkers; ++worker) { + const pa_scheduler::WorkerResult &result = state.results[worker]; + AddPmuSample(result, &all); + AddPmuSample(result, result.role == static_cast(pa_scheduler::CoreRole::Aic) ? &aic : &aiv); + if (result.role == static_cast(pa_scheduler::CoreRole::Aic)) { + active_output_tiles += result.kernel_counts[0] != 0; + active_output_tiles += result.kernel_counts[2] != 0; + } else if (result.role == static_cast(pa_scheduler::CoreRole::Aiv)) { + active_output_tiles += result.kernel_counts[1] != 0; + active_output_tiles += result.kernel_counts[3] != 0; + } + ef_drain_kernels += result.placement[static_cast(pa_scheduler::DrainPlace::EfDrain)]; + ring_backpressure_kernels += + result.placement[static_cast(pa_scheduler::DrainPlace::RingBackpressure)]; + final_drain_kernels += result.placement[static_cast(pa_scheduler::DrainPlace::FinalDrain)]; + } + + const auto generated = std::chrono::system_clock::now().time_since_epoch(); + const uint64_t generated_ns = static_cast( + std::chrono::duration_cast(generated).count() + ); + const std::string capture_id = "pa-pmu-" + std::to_string(generated_ns) + "-run" + std::to_string(run); + const std::string temporary_path = output_path + ".tmp"; + // 临时文件与最终文件都采用 no-replace 语义：并发采集不能截断同名 tmp， + // 也不能在最终发布时覆盖另一份已经完成的证据文件。 + const int output_fd = open(temporary_path.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0644); + std::FILE *output = output_fd < 0 ? nullptr : fdopen(output_fd, "wb"); + if (output == nullptr) { + const int open_error = errno; + if (output_fd >= 0) { + (void)close(output_fd); + (void)std::remove(temporary_path.c_str()); + } + std::fprintf( + stderr, "Cannot exclusively create PMU JSON output %s: %s\n", temporary_path.c_str(), + std::strerror(open_error) + ); + return false; + } + std::vector output_buffer(1U << 20); + std::setvbuf(output, output_buffer.data(), _IOFBF, output_buffer.size()); + + const bool submit_window = IsSubmitWindow(pmu.mode); + const bool icache_single = pmu.mode == WindowMode::IcacheSingle; + const bool real_compute = workload.mode == pa_scheduler::WinnerWorkloadMode::RealCompute; + const bool simulated_task_nops_nonzero = !real_compute && + (options.nops.qk != 0U || options.nops.sf != 0U || + options.nops.pv != 0U || options.nops.up != 0U); + const pa_scheduler::WorkloadCounts active_counts = real_compute + ? workload.repeats + : pa_scheduler::WorkloadCounts{ + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + }; + const uint32_t owner_bitmap_count = pa_scheduler::pmu_owner::CountConfigured(owner); + const uint32_t owner_complete_triplets = CountConfiguredMixedTriplets(owner); + std::fputs("{\n\"schema\":{\"name\":\"pa_scheduler_pmu_phase_windows\",\"version\":5},\n", output); + std::fputs("\"capture\":{\"capture_id\":", output); + WriteJsonString(output, capture_id); + std::fprintf( + output, + ",\"generated_unix_time_ns\":%llu,\"run_index\":%u,\"accepted\":true," + "\"usable_for_same_configuration_submit_comparison\":%s," + "\"usable_as_absolute_real_pa_profile\":false,\"window_scope\":\"%s\"," + "\"pmu_probe_position\":\"%s\",\"scheduler_hot_path_included\":%s," + "\"total_sum_is_core_work_not_wall_time\":true," + "\"submit_window_excludes_final_drain\":%s," + "\"published_after_runtime_cleanup\":true,\"runtime_cleanup_passed\":true," + "\"owner_restore_passed\":%s},\n", + static_cast(generated_ns), run, + submit_window ? "true" : "false", + submit_window ? "per_worker_orchestration_to_last_submit_return" : "post_scheduler_calibration_probe", + submit_window ? "inside_RunScheduler" : "after_RunScheduler", + submit_window ? "true" : "false", submit_window ? "true" : "false", + restore_passed ? "true" : "false" + ); + std::fputs("\"configuration\":{\"kernel_path\":", output); + WriteJsonString(output, options.kernel_path); + std::fputs(",\"build_variant\":\"submit-pmu\",\"build_variant_id\":2,\"compiled_phase\":", output); + WriteJsonString(output, SubmitPmuPhaseName(pa_scheduler::kCompiledSubmitPmuPhase)); + std::fprintf( + output, ",\"compiled_phase_id\":%u", + static_cast(pa_scheduler::kCompiledSubmitPmuPhase) + ); + std::fprintf( + output, + ",\"device\":%u,\"batches\":%u,\"workers\":%u,\"aic_workers\":%u,\"aiv_workers\":%u," + "\"trace_enabled\":%s,\"trace_atomics\":%s,\"profile_phases\":%s," + "\"winner_workload\":{\"mode\":", + options.device, options.batches, pa_scheduler::kWorkers, pa_scheduler::kAicWorkers, + pa_scheduler::kAivWorkers, options.trace_enabled ? "true" : "false", + options.trace_atomics ? "true" : "false", + options.profile_phases ? "true" : "false" + ); + WriteJsonString(output, WinnerWorkloadModeName(workload.mode)); + std::fputs(",\"input_pattern\":", output); + WriteJsonString( + output, + real_compute ? pa_scheduler::host::RealComputePatternName(workload.pattern) : "none" + ); + std::fprintf( + output, + ",\"config_version\":%u,\"counts\":{\"qk\":%u,\"sf\":%u,\"pv\":%u,\"up\":%u}," + "\"unit\":", + pa_scheduler::kWinnerWorkloadConfigVersion, active_counts.qk, active_counts.sf, + active_counts.pv, active_counts.up + ); + WriteJsonString( + output, + real_compute ? "complete_128x128_engine_pipeline_iteration" : "scalar_nop_instruction" + ); + std::fprintf( + output, + ",\"tile_rows\":%u,\"tile_cols\":%u,\"shared_input_tiles\":%u," + "\"output_tiles_per_worker\":%u,\"workspace_bytes\":%zu,\"role_mapping\":", + real_compute ? pa_scheduler::winner_workload::kTileRows : 0, + real_compute ? pa_scheduler::winner_workload::kTileCols : 0, + real_compute ? pa_scheduler::winner_workload::kSharedInputTiles : 0, + real_compute ? pa_scheduler::winner_workload::kOutputTilesPerWorker : 0, + real_compute ? pa_scheduler::winner_workload::kWorkspaceBytes : 0 + ); + if (real_compute) { + std::fputs( + "{\"qk\":\"cube_matmul\",\"pv\":\"cube_matmul\"," + "\"sf\":\"vector_add\",\"up\":\"vector_mul\"}", + output + ); + } else { + std::fputs("null", output); + } + std::fprintf( + output, ",\"engine_completion_waited_before_task_publish\":%s},\"nop_counts\":", + real_compute ? "true" : "false" + ); + if (real_compute) { + std::fputs("null", output); + } else { + std::fprintf( + output, "{\"qk\":%u,\"sf\":%u,\"pv\":%u,\"up\":%u}", + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + ); + } + std::fputs(",\"pmu_window\":", output); + WriteJsonString(output, PmuModeName(pmu.mode)); + std::fputs(",\"calibration_scalar_nops_per_segment\":", output); + if (submit_window || icache_single) { + std::fputs("null", output); + } else { + std::fprintf(output, "%u", pmu.scalar_nops); + } + std::fputs(",\"icache_single_trials_per_core\":", output); + if (icache_single) { + std::fprintf(output, "%u", pmu.icache_trials); + } else { + std::fputs("null", output); + } + std::fprintf( + output, + ",\"primary_window_segments_per_record\":1," + "\"icache_single_discarded_training_samples_per_core\":%u," + "\"icache_single_sys_counter_tick_ns\":%s," + "\"host_launch_to_sync_us\":%.17g,\"submit_span_us\":%.17g," + "\"selectors\":{\"cnt0_vector_busy\":%u,\"cnt1_cube_busy\":%u," + "\"cnt2_scalar_busy\":%u,\"cnt3_mte1_busy\":%u,\"cnt4_mte2_busy\":%u," + "\"cnt5_shadow_icache_miss\":%u,\"cnt6_primary_icache_request\":%u," + "\"cnt7_primary_icache_miss\":%u,\"cnt8_shadow_icache_request\":%u," + "\"cnt9_unused\":0},\"unavailable_metrics\":[\"mte3_busy\"]," + "\"counter_width_bits\":{\"total\":64,\"programmable\":32}," + "\"counter_wrap_not_directly_detectable\":true,\"counter_wrap_absence_proven\":false," + "\"programmable_counter_risk_threshold\":%u," + "\"gate_start_stop_have_pipe_all_barriers\":true," + "\"phase_timestamp_calls_present\":%s,\"phase_record_writes\":false," + "\"atomic_trace\":false,\"profile_accumulation\":false," + "\"phase_boundary_observation_included\":%s," + "\"phase_time_observation_included\":%s," + "\"phase_time_sys_counter_tick_ns\":1," + "\"phase_time_boundary\":\"after_begin_read_clear_to_before_end_read_clear\"," + "\"phase_time_excludes_shadow_read_overhead\":true," + "\"phase_time_includes_timestamp_overhead\":true," + "\"phase_time_share_definition\":" + "\"sum(phase_elapsed_ticks)/sum(submit_elapsed_ticks)\"," + "\"phase_time_denominator_scope\":" + "\"per_worker_first_submit_begin_to_last_submit_end\"," + "\"phase_counter_pair_snapshot_atomic\":false," + "\"primary_counters_read_at_phase_boundaries\":false," + "\"phase_shadow_partition_exact_required\":%s," + "\"phase_values_are_running_read_clear_lower_bounds\":%s," + "\"cross_phase_elf_sums_valid\":false," + "\"simulated_task_nop_mechanism_executes_on_scalar\":%s," + "\"simulated_task_nops_nonzero\":%s," + "\"icache_miss_rate_definition\":\"sum(icache_misses)/sum(icache_requests)\"},\n", + icache_single ? 2U : 0U, icache_single ? "1" : "null", + host_us, submit_span_us, kVectorBusyEvent, kCubeBusyEvent, kScalarBusyEvent, + kMte1BusyEvent, kMte2BusyEvent, kIcacheMissEvent, kIcacheRequestEvent, kIcacheMissEvent, + kIcacheRequestEvent, kProgrammableCounterRiskThreshold, + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "true" + : "false", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + real_compute ? "false" : "true", + simulated_task_nops_nonzero ? "true" : "false" + ); + std::fprintf( + output, + "\"validation\":{\"semantic_passed\":%s,\"pmu_passed\":%s," + "\"real_compute_output_validation_required\":%s," + "\"real_compute_output_validation_passed\":%s," + "\"real_compute_active_output_tiles\":%u," + "\"real_compute_inactive_sentinel_tiles\":%u," + "\"real_compute_output_mismatches\":%u," + "\"submit_engine_observation_valid\":%s," + "\"submit_engine_workers_expected\":%u," + "\"submit_engine_workers_matched\":%u," + "\"kernel_placement_counts\":{\"ef_drain\":%llu,\"ring_backpressure\":%llu," + "\"final_drain\":%llu},\"trusted_records\":%u," + "\"expected_records\":%u,\"unique_physical_core_ids\":%u,\"expected_unique_core_ids\":%u," + "\"owner_bitmap_member_records\":%u,\"expected_owner_bitmap_member_records\":%u," + "\"exact_worker_slot_records\":%u,\"expected_exact_worker_slot_records\":%u," + "\"physical_role_match_records\":%u,\"expected_physical_role_match_records\":%u," + "\"mixed_triplet_matches\":%u,\"expected_mixed_triplet_matches\":%u," + "\"window_started_records\":%u,\"window_stopped_records\":%u," + "\"expected_window_records\":%u,\"prior_snapshot_larger_records\":%u," + "\"icache_pair_records\":%u,\"icache_calibrated_cores\":%u," + "\"icache_measurement_valid\":%s," + "\"icache_miss_le_request\":%s,\"counter_below_risk_threshold\":%s," + "\"maximum_programmable_counter\":%u,\"programmable_counter_risk_threshold\":%u," + "\"programmable_counter_headroom\":%u," + "\"build_variant_match_records\":%u,\"phase_id_match_records\":%u," + "\"phase_status_trusted_records\":%u,\"shadow_primary_match_records\":%u," + "\"shadow_primary_bounded_records\":%u," + "\"phase_shadow_acceptable_records\":%u," + "\"shadow_request_abs_delta_sum\":%llu,\"shadow_request_abs_delta_max\":%u," + "\"shadow_request_signed_delta_sum\":%lld," + "\"shadow_miss_abs_delta_sum\":%llu,\"shadow_miss_abs_delta_max\":%u," + "\"shadow_miss_signed_delta_sum\":%lld," + "\"phase_boundary_match_records\":%u,\"phase_call_shape_match_records\":%u," + "\"phase_time_valid_records\":%u,\"phase_time_measurement_valid\":%s," + "\"phase_calls\":%llu,\"phase_expected_calls\":%llu," + "\"phase_measurement_valid\":%s},\n", + semantic_passed ? "true" : "false", validation.passed ? "true" : "false", + real_compute ? "true" : "false", + real_compute ? (workload_output_passed ? "true" : "false") : "null", + real_compute ? active_output_tiles : 0, + real_compute ? pa_scheduler::winner_workload::kOutputTiles - active_output_tiles : 0, + real_compute && !workload_output_passed ? 1U : 0U, + validation.submit_engine_observation_valid ? "true" : "false", + validation.submit_engine_workers_expected, + validation.submit_engine_workers_matched, + static_cast(ef_drain_kernels), + static_cast(ring_backpressure_kernels), + static_cast(final_drain_kernels), + validation.trusted, + pa_scheduler::kWorkers, validation.unique_physical_core_ids, pa_scheduler::kWorkers, + validation.owner_bitmap_members, pa_scheduler::kWorkers, + validation.exact_worker_slots, pa_scheduler::kWorkers, + validation.physical_role_matches, pa_scheduler::kWorkers, + validation.mixed_triplet_matches, pa_scheduler::kAicWorkers, + validation.window_started, validation.window_stopped, pa_scheduler::kWorkers, + validation.prior_snapshot_larger, validation.icache_pairs, + validation.icache_calibrated_cores, + validation.icache_measurement_valid ? "true" : "false", + validation.icache_order_valid ? "true" : "false", + validation.counter_below_risk_threshold ? "true" : "false", + validation.maximum_programmable_counter, kProgrammableCounterRiskThreshold, + UINT32_MAX - validation.maximum_programmable_counter, + validation.build_variant_matches, validation.phase_id_matches, + validation.phase_status_trusted, validation.shadow_primary_matches, + validation.shadow_primary_bounded, validation.phase_shadow_acceptable, + static_cast(validation.shadow_request_abs_delta_sum), + validation.shadow_request_abs_delta_max, + static_cast(validation.shadow_request_signed_delta_sum), + static_cast(validation.shadow_miss_abs_delta_sum), + validation.shadow_miss_abs_delta_max, + static_cast(validation.shadow_miss_signed_delta_sum), + validation.phase_boundary_matches, + validation.phase_call_shape_matches, + validation.phase_time_valid_records, + validation.phase_time_valid_records == pa_scheduler::kWorkers ? "true" : "false", + static_cast(validation.phase_calls), + static_cast(validation.expected_phase_calls), + validation.phase_measurement_valid ? "true" : "false" + ); + std::fprintf( + output, + "\"owner\":{\"mode\":\"main_aicpu_path_a\"," + "\"snapshot_phase\":\"after_configure_before_restore\"," + "\"control_magic\":%u,\"control_version\":%u,\"configure_status\":%d," + "\"configured_flag\":%u,\"configured_bitmap_count\":%u," + "\"expected\":{\"total\":%u,\"aic\":%u,\"aiv\":%u}," + "\"active\":{\"total\":%u,\"aic\":%u,\"aiv\":%u}," + "\"discovered\":{\"total\":%u,\"aic\":%u,\"aiv\":%u}," + "\"physical_slots_scanned\":%u,\"skipped_physical_slots\":%u," + "\"configured_bitmap_word_order\":\"least_significant_physical_ids_first\"," + "\"configured_bitmap_words\":[%u,%u,%u,%u]," + "\"configured_complete_mixed_triplets\":%u,\"expected_complete_mixed_triplets\":%u," + "\"configured_broken_mixed_triplets\":%u,\"restore_passed\":%s},\n", + owner.magic, owner.version, static_cast(owner.status), owner.configured, owner_bitmap_count, + owner.expected_total, owner.expected_aic, owner.expected_aiv, + owner.active_total, owner.active_aic, owner.active_aiv, + owner.discovered_total, owner.discovered_aic, owner.discovered_aiv, + pa_scheduler::pmu_owner::kPhysicalSubcoreCount, owner.skipped_total, + owner.configured_bitmap[0], owner.configured_bitmap[1], + owner.configured_bitmap[2], owner.configured_bitmap[3], + owner_complete_triplets, pa_scheduler::kAicWorkers, + owner_bitmap_count / 3U - owner_complete_triplets, restore_passed ? "true" : "false" + ); + std::fputs("\"records\":[\n", output); + for (uint32_t worker = 0; worker < pa_scheduler::kWorkers; ++worker) { + const pa_scheduler::WorkerResult &result = state.results[worker]; + const uint32_t status = result.pmu_status; + const uint32_t physical_core_id = StatusCoreId(status); + const bool primary_trusted = (status & kStatusRequired) == kStatusRequired; + const bool phase_trusted = + (result.pmu_phase_status & kPhaseStatusRequired) == kPhaseStatusRequired; + const uint64_t submit_elapsed_ticks = result.submit_end >= result.submit_begin + ? result.submit_end - result.submit_begin + : 0U; + const bool phase_time_valid = + submit_elapsed_ticks != 0U && + result.pmu_phase_elapsed_ticks <= submit_elapsed_ticks && + (pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? result.pmu_phase_elapsed_ticks == 0U + : result.pmu_phase_elapsed_ticks != 0U); + const bool trusted = primary_trusted && phase_trusted && phase_time_valid; + const bool is_aic = result.role == static_cast(pa_scheduler::CoreRole::Aic); + const uint32_t vector_id = is_aic ? 0U : worker - pa_scheduler::kAicWorkers; + const uint32_t block_id = is_aic ? worker : vector_id / 2U; + const uint32_t lane = is_aic ? 0U : 1U + vector_id % 2U; + const uint32_t shadow_read_segments = result.pmu_phase_calls * 2U + 1U; + const bool owner_bitmap_member = pa_scheduler::pmu_owner::IsConfigured(owner, physical_core_id); + const bool worker_slot_exact = result.worker_id == worker; + const bool physical_role_matches = + is_aic == pa_scheduler::pmu_owner::IsAicPhysicalSlot(physical_core_id); + const bool window_started = (status & kStatusWindowStarted) != 0U; + const bool window_stopped = (status & kStatusWindowStopped) != 0U; + const bool shadow_matches = + result.pmu_shadow_icache_requests == result.pmu_icache_requests && + result.pmu_shadow_icache_misses == result.pmu_icache_misses; + const bool shadow_bounded = + result.pmu_shadow_icache_requests <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_icache_misses; + const uint32_t shadow_request_loss = + result.pmu_icache_requests - result.pmu_shadow_icache_requests; + const uint32_t shadow_miss_loss = + result.pmu_icache_misses - result.pmu_shadow_icache_misses; + const uint32_t phase_request_upper = + result.pmu_phase_icache_requests + shadow_request_loss; + const uint32_t phase_miss_upper = + result.pmu_phase_icache_misses + shadow_miss_loss; + const bool boundaries_balanced = + result.pmu_phase_begin_reads == result.pmu_phase_calls && + result.pmu_phase_end_reads == result.pmu_phase_calls; + const uint32_t expected_phase_calls = + ExpectedSubmitPmuPhaseCallsPerWorker(state.config.batches); + std::fprintf( + output, + "%s{\"worker_id\":%u,\"physical_core_id\":%u,\"role\":\"%s\",\"block_id\":%u," + "\"lane\":%u,\"primary_window_segments\":1,\"shadow_read_segments\":%u," + "\"window_started\":%s,\"window_stopped\":%s,\"total_cycles\":%llu,\"vector_busy\":%u," + "\"cube_busy\":%u,\"scalar_busy\":%u,\"mte1_busy\":%u,\"mte2_busy\":%u," + "\"icache_requests\":%u,\"icache_misses\":%u," + "\"build_variant_id\":%u,\"compiled_phase_id\":%u,\"phase_calls\":%u," + "\"phase_expected_calls\":%u," + "\"phase_begin_reads\":%u,\"phase_end_reads\":%u," + "\"submit_elapsed_ticks\":%llu,\"phase_elapsed_ticks\":%llu," + "\"phase_time_valid\":%s," + "\"phase_icache_requests\":%u,\"phase_icache_misses\":%u," + "\"phase_icache_requests_upper_bound\":%u," + "\"phase_icache_misses_upper_bound\":%u," + "\"shadow_whole_icache_requests\":%u,\"shadow_whole_icache_misses\":%u," + "\"shadow_matches_primary\":%s,\"shadow_not_greater_than_primary\":%s," + "\"shadow_request_loss\":%u,\"shadow_miss_loss\":%u," + "\"phase_boundaries_balanced\":%s," + "\"phase_status\":%u,\"phase_status_hex\":\"0x%08x\"," + "\"status\":%u,\"status_hex\":" + "\"0x%08x\",\"trusted\":%s,\"physical_core_id_valid\":%s,\"selectors_match\":%s," + "\"owner_bitmap_member\":%s,\"worker_slot_exact\":%s," + "\"physical_role_matches\":%s}", + worker == 0 ? "" : ",\n", worker, physical_core_id, is_aic ? "aic" : "aiv", block_id, + lane, shadow_read_segments, window_started ? "true" : "false", window_stopped ? "true" : "false", + static_cast(result.pmu_total_cycles), result.pmu_vector_busy, + result.pmu_cube_busy, result.pmu_scalar_busy, result.pmu_mte1_busy, result.pmu_mte2_busy, + result.pmu_icache_requests, result.pmu_icache_misses, + result.pmu_build_variant, result.pmu_phase_id, result.pmu_phase_calls, + expected_phase_calls, + result.pmu_phase_begin_reads, result.pmu_phase_end_reads, + static_cast(submit_elapsed_ticks), + static_cast(result.pmu_phase_elapsed_ticks), + phase_time_valid ? "true" : "false", + result.pmu_phase_icache_requests, result.pmu_phase_icache_misses, + phase_request_upper, phase_miss_upper, + result.pmu_shadow_icache_requests, result.pmu_shadow_icache_misses, + shadow_matches ? "true" : "false", shadow_bounded ? "true" : "false", + shadow_request_loss, shadow_miss_loss, boundaries_balanced ? "true" : "false", + result.pmu_phase_status, result.pmu_phase_status, + status, status, trusted ? "true" : "false", + (status & kStatusCoreIdValid) != 0 ? "true" : "false", + (status & (kStatusCnt0Selector | kStatusCnt1Selector | kStatusCnt2Selector | + kStatusCnt3Selector | kStatusCnt4Selector | kStatusCnt5Selector | + kStatusCnt6Selector | kStatusCnt7Selector | kStatusCnt8Selector)) == + (kStatusCnt0Selector | kStatusCnt1Selector | kStatusCnt2Selector | + kStatusCnt3Selector | kStatusCnt4Selector | kStatusCnt5Selector | + kStatusCnt6Selector | kStatusCnt7Selector | kStatusCnt8Selector) + ? "true" + : "false", + owner_bitmap_member ? "true" : "false", worker_slot_exact ? "true" : "false", + physical_role_matches ? "true" : "false" + ); + } + std::fputs("\n],\n\"summary\":{\"all\":", output); + WritePmuAggregateJson(output, all, icache_single); + std::fputs(",\"aic\":", output); + WritePmuAggregateJson(output, aic, icache_single); + std::fputs(",\"aiv\":", output); + WritePmuAggregateJson(output, aiv, icache_single); + std::fputs("}\n}\n", output); + + bool success = std::ferror(output) == 0; + int write_error = success ? 0 : EIO; + if (std::fflush(output) != 0) { + success = false; + write_error = errno; + } + if (success && fsync(fileno(output)) != 0) { + success = false; + write_error = errno; + } + if (std::fclose(output) != 0) { + success = false; + write_error = errno; + } + if (!success) { + std::fprintf(stderr, "Failed while writing PMU JSON output %s: %s\n", temporary_path.c_str(), + std::strerror(write_error)); + (void)std::remove(temporary_path.c_str()); + return false; + } + // 同目录 hard-link 在最终名称不存在时原子发布；EEXIST 时保留既有证据， + // 不采用会替换目标的 POSIX rename。 + if (link(temporary_path.c_str(), output_path.c_str()) != 0) { + std::fprintf( + stderr, "Cannot publish PMU JSON without replacement %s -> %s: %s\n", + temporary_path.c_str(), output_path.c_str(), + std::strerror(errno) + ); + (void)std::remove(temporary_path.c_str()); + return false; + } + if (unlink(temporary_path.c_str()) != 0) { + const int unlink_error = errno; + // 最终文件已链接但事务尚未完成；尽力撤回最终名称，避免失败返回时留下 + // 一份被调用方误认为成功发布的文件。 + (void)unlink(output_path.c_str()); + std::fprintf( + stderr, "Cannot remove PMU JSON temporary link %s: %s\n", temporary_path.c_str(), + std::strerror(unlink_error) + ); + return false; + } + std::printf("[PMU-JSON] capture_id=%s records=%u output=%s\n", capture_id.c_str(), pa_scheduler::kWorkers, + output_path.c_str()); + return true; +} + +} // namespace + +int main(int argc, char **argv) { + // 参数和 ELF 在创建 ACL 资源前完成校验，早期错误不会留下 device、stream 或 kernel handle。 + pa_scheduler::host::Options options; + PmuOptions pmu_options; + WinnerWorkloadOptions workload_options; + std::vector pmu_argv; + std::vector common_argv; + if (!ParseWinnerWorkloadOptions(argc, argv, &workload_options, &pmu_argv) || + !ParsePmuOptions( + static_cast(pmu_argv.size()), pmu_argv.data(), &pmu_options, &common_argv + )) { + return EXIT_FAILURE; + } + const pa_scheduler::host::ParseStatus parse_status = pa_scheduler::host::ParseOptions( + static_cast(common_argv.size()), common_argv.data(), true, &options + ); + if (parse_status != pa_scheduler::host::ParseStatus::Ok) { + if (parse_status == pa_scheduler::host::ParseStatus::Help) { + std::fprintf( + stderr, + "CCEC PMU options: [--pmu-window " + "off|empty|scalar|scalar-double|icache-single|submit-all] " + "[--pmu-scalar-nops N] [--pmu-icache-trials N] [--pmu-json FILE]\n" + ); + std::fprintf( + stderr, + "CCEC winner workload options: [--winner-workload scalar-nop|real-compute] " + "[--real-compute-count N | --real-compute-counts QK,SF,PV,UP] " + "[--real-compute-pattern constant|layout-diagnostic]\n" + ); + std::fprintf( + stderr, + "Default: real-compute, constant, counts=6,28,4,1; " + "scalar-nop is the calibration compatibility mode.\n" + ); + } + return parse_status == pa_scheduler::host::ParseStatus::Help ? EXIT_SUCCESS : EXIT_FAILURE; + } + if (!ValidateWinnerWorkloadOptions(workload_options)) return EXIT_FAILURE; +#if PA_BUILD_SWIMLANE + // swimlane host 与同目录 kernel 是成套产物；它不允许借旧参数重新开启 + // 已从 device ELF 编译掉的 PMU/phase-profile 路径。 + if (pmu_options.mode != pa_scheduler::ccec_pmu::WindowMode::Off || + !pmu_options.json_path.empty()) { + std::fprintf( + stderr, + "This is a swimlane build; PMU collection requires the separate submit-pmu build.\n" + ); + return EXIT_FAILURE; + } + if (options.profile_phases) { + std::fprintf( + stderr, + "--profile-phases is not part of the swimlane build; use submit-pmu phase attribution.\n" + ); + return EXIT_FAILURE; + } +#elif PA_BUILD_SUBMIT_PMU + // submit-pmu 是编译期固定 phase 的单轮诊断产物；host、kernel 与 owner + // 必须共同拒绝旧校准窗口和任何泳道/phase-profile 观察代码。 + if (pmu_options.mode != pa_scheduler::ccec_pmu::WindowMode::SubmitAll) { + std::fprintf(stderr, "The submit-pmu build requires --pmu-window submit-all.\n"); + return EXIT_FAILURE; + } + if (options.runs != 1 || options.trace_enabled || options.trace_atomics || + options.profile_phases || options.analyze_swimlane || !options.swimlane_json.empty()) { + std::fprintf( + stderr, + "submit-pmu requires one PMU-only run: --runs 1 --no-swimlane and no trace/profile options.\n" + ); + return EXIT_FAILURE; + } +#endif + if (!pmu_options.json_path.empty() && + pmu_options.mode == pa_scheduler::ccec_pmu::WindowMode::Off) { + std::fprintf(stderr, "--pmu-json requires a non-off --pmu-window.\n"); + return EXIT_FAILURE; + } + if (!pmu_options.json_path.empty() && options.runs != 1) { + // 一个 sidecar 对应一次采集，禁止多轮覆写后丢失逐轮边界。 + std::fprintf(stderr, "--pmu-json requires --runs 1 to avoid overwriting captures.\n"); + return EXIT_FAILURE; + } + if (!pmu_options.json_path.empty() && + (options.trace_enabled || options.trace_atomics || options.profile_phases || + options.analyze_swimlane || !options.swimlane_json.empty())) { + std::fprintf( + stderr, + "--pmu-json requires PMU-only collection: add --no-swimlane and do not enable " + "phase profiling, atomic tracing, swimlane analysis, or swimlane JSON.\n" + ); + return EXIT_FAILURE; + } + if (!pmu_options.json_path.empty() && + (access(pmu_options.json_path.c_str(), F_OK) == 0 || + access((pmu_options.json_path + ".tmp").c_str(), F_OK) == 0)) { + std::fprintf( + stderr, "Refusing to overwrite an existing PMU JSON or temporary file: %s\n", + pmu_options.json_path.c_str() + ); + return EXIT_FAILURE; + } + const std::vector binary_data = ReadBinary(options.kernel_path); + if (binary_data.empty()) { + std::fprintf(stderr, "Cannot read kernel binary: %s\n", options.kernel_path.c_str()); + return EXIT_FAILURE; + } + const bool real_compute = + workload_options.mode == pa_scheduler::WinnerWorkloadMode::RealCompute; + std::vector workload_image; + std::vector workload_outputs; + if (real_compute) { + pa_scheduler::host::InitializeWinnerWorkloadBuffers( + workload_options, &workload_image, &workload_outputs + ); + } + pa_scheduler::host::PrintBanner("CCEC", options); + pa_scheduler::host::PrintWinnerWorkloadConfig(workload_options, options.nops); + std::printf( + "[PMU-CONFIG] window=%s calibration_scalar_nops=%u icache_trials=%u source=direct-per-core " + "owner=main-aicpu-path-a\n", + PmuModeName(pmu_options.mode), pmu_options.scalar_nops, pmu_options.icache_trials + ); + + // 正常及后处理路径依次完成 ACL 初始化、选卡、stream/ELF/设备区创建、launch/D2H + // 和尾部清理；初始化、传输或 launch 的早期错误仍按当前实现就地返回。 + if (!CheckAcl(aclInit(nullptr), "aclInit") || !CheckAcl(aclrtSetDevice(options.device), "aclrtSetDevice")) { + return EXIT_FAILURE; + } + aclrtStream stream = nullptr; + if (!CheckAcl(aclrtCreateStream(&stream), "aclrtCreateStream")) return EXIT_FAILURE; + + rtDevBinary_t binary{RT_DEV_BINARY_MAGIC_ELF, 0, binary_data.data(), binary_data.size()}; + void *kernel_handle = nullptr; + bool registered_all = true; + // 先尝试注册带 mixed metadata 的 ELF；若 rtRegisterAllKernel 报错或未返回 handle， + // 再尝试无 tiling-key 装载。这里仅描述实际回退条件，不假设具体运行时原因。 + rtError_t register_error = rtRegisterAllKernel(&binary, &kernel_handle); + if (register_error != RT_ERROR_NONE || kernel_handle == nullptr) { + registered_all = false; + register_error = rtBinaryLoadWithoutTilingKey(binary_data.data(), binary_data.size(), &kernel_handle); + } + if (!CheckRt(register_error, "register mixed AICore ELF") || kernel_handle == nullptr) return EXIT_FAILURE; + + // SchedulerState 保留被测关键 offset、DistCore ABI 和约 1 GiB 生产总跨度； + // 使用 HUGE_FIRST 降低大块设备内存碎片风险。 + void *state_device = nullptr; + if (!CheckAcl( + aclrtMalloc(&state_device, sizeof(pa_scheduler::SchedulerState), ACL_MEM_MALLOC_HUGE_FIRST), + "aclrtMalloc(state)" + )) { + return EXIT_FAILURE; + } + if ((reinterpret_cast(state_device) & 63U) != 0) { + std::fprintf(stderr, "Device state is not 64-byte aligned: %p\n", state_device); + return EXIT_FAILURE; + } + + // 真实 PTO 负载使用独立 GM，不解引用调度器中只用于依赖建模的 synthetic tensor 地址。 + // 这里先于 PMU owner 分配；每轮 H2D 初始化虽在 owner 配置之后，但仍位于 + // launch/wall 计时之前，因此两者都不进入 Submit 性能窗口。 + ScopedAclDeviceAllocation workload_allocation; + if (real_compute && + !CheckAcl( + aclrtMalloc( + workload_allocation.Address(), pa_scheduler::winner_workload::kWorkspaceBytes, + ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(real-compute workspace)" + )) { + return EXIT_FAILURE; + } + void *workload_device = workload_allocation.Get(); + if (real_compute && (reinterpret_cast(workload_device) & 63U) != 0) { + std::fprintf(stderr, "Real-compute workspace is not 64-byte aligned: %p\n", workload_device); + return EXIT_FAILURE; + } + + // 泳道区按 96 worker 各 65536 条记录预留，约 384 MiB；关闭泳道时不申请，也不会传递有效 base。 + // 该分配先于 PMU owner 配置，失败时不会留下需要恢复的 selector/MMIO 会话。 + void *trace_device = nullptr; + if (options.trace_enabled && + !CheckAcl( + aclrtMalloc(&trace_device, pa_scheduler::kTraceBytes, ACL_MEM_MALLOC_HUGE_FIRST), + "aclrtMalloc(swimlane trace)" + )) { + return EXIT_FAILURE; + } + if (options.trace_enabled && (reinterpret_cast(trace_device) & 63U) != 0) { + std::fprintf(stderr, "Device swimlane trace is not 64-byte aligned: %p\n", trace_device); + return EXIT_FAILURE; + } + + PmuRegisterMappings pmu_mappings; + pa_scheduler::pmu_owner::PmuOwnerSession pmu_owner; + pa_scheduler::pmu_owner::PmuOwnerControl pmu_owner_evidence{}; + bool pmu_owner_evidence_valid = false; + const void *pmu_registers_device = nullptr; + if (pmu_options.mode != pa_scheduler::ccec_pmu::WindowMode::Off) { + if (!MapPmuRegisters(options.device, &pmu_mappings)) return EXIT_FAILURE; + const std::string dispatcher_path = pa_scheduler::pmu_owner::ArtifactBesideKernel( + options.kernel_path, "libpa_scheduler_pmu_owner_dispatcher.so" + ); + const std::string owner_path = pa_scheduler::pmu_owner::ArtifactBesideKernel( + options.kernel_path, "libpa_scheduler_pmu_owner_aicpu.so" + ); + if (!pmu_owner.Initialize( + options.device, stream, dispatcher_path, owner_path, pmu_mappings.register_bases + ) || + !pmu_owner.Configure()) { + (void)pmu_owner.Finalize(); + (void)UnmapPmuRegisters(options.device, &pmu_mappings); + return EXIT_FAILURE; + } + pmu_owner_evidence = pmu_owner.Control(); + pmu_owner_evidence_valid = true; + pmu_registers_device = reinterpret_cast(pmu_owner.RegisterTableDeviceAddress()); + } + + // host shadow 保留约 1 GiB 总跨度以便按关键 offset 寻址，但每轮传输只选择 + // 共享前缀、控制区和结果区。 + std::unique_ptr state(new pa_scheduler::SchedulerState); + pa_scheduler::TraceHeader trace_header{}; + std::vector spans; + bool execution_ok = true; + bool all_passed = true; + bool postprocess_ok = true; + bool pmu_json_ready = false; + bool pmu_json_semantic_passed = false; + bool pmu_json_workload_output_passed = false; + uint32_t pmu_json_run = 0U; + double pmu_json_host_us = 0.0; + double pmu_json_submit_span_us = 0.0; + PmuValidation pmu_json_validation; + for (uint32_t run = 1; run <= options.runs; ++run) { + pa_scheduler::host::InitializeState(state.get(), options); + pa_scheduler::host::ConfigureTrace(state.get(), options, trace_device); + ConfigurePmu(state.get(), pmu_options, pmu_registers_device); + ConfigureWinnerWorkload(state.get(), workload_options, workload_device); + if (real_compute && + !CheckAcl( + aclrtMemcpy( + workload_device, pa_scheduler::winner_workload::kWorkspaceBytes, + workload_image.data(), pa_scheduler::winner_workload::kWorkspaceBytes, + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D real-compute workspace)" + )) { + execution_ok = false; + break; + } + if (options.trace_enabled) { + // 每轮只需重置约 7 KiB header；各 worker 会从 count=0 覆盖自己的记录区，无需清零整块 384 MiB。 + pa_scheduler::host::InitializeTraceHeader(&trace_header); + if (!CheckAcl( + aclrtMemcpy( + trace_device, sizeof(trace_header), &trace_header, sizeof(trace_header), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D swimlane header)" + )) { + execution_ok = false; + break; + } + } + // 为避免每轮搬运约 1 GiB，只 H2D 被测共享前缀和位于生产总跨度之后的 + // standalone 控制区； + // 每个 worker 的大块私有状态由 device kernel 自行初始化。 + if (!CheckAcl( + aclrtMemcpy( + state_device, pa_scheduler::host::StatePrefixBytes(), state.get(), + pa_scheduler::host::StatePrefixBytes(), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D state prefix)" + ) || + !CheckAcl( + aclrtMemcpy( + &static_cast(state_device)->config, + pa_scheduler::host::ControlBytes(), &state->config, pa_scheduler::host::ControlBytes(), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D standalone controls)" + )) { + execution_ok = false; + break; + } + void *kernel_args[] = {state_device}; + rtArgsEx_t args_info{}; + args_info.args = kernel_args; + args_info.argsSize = sizeof(kernel_args); + rtTaskCfgInfo_t task_config{}; + // launch 维度是 32 个物理 mixed block；ELF metadata 让每个 block 同时产生 1 AIC + 2 AIV，共 96 worker。 + // wall time 在同步完成处截止，包含 launch、完整调度、最终 drain 和 stream 同步，但不包含后续 D2H/JSON。 + const auto wall_begin = std::chrono::steady_clock::now(); + if (!CheckRt( + rtKernelLaunchWithHandleV2( + kernel_handle, 0, pa_scheduler::kAicWorkers, &args_info, nullptr, stream, &task_config + ), + "rtKernelLaunchWithHandleV2" + ) || + !CheckAcl(aclrtSynchronizeStream(stream), "aclrtSynchronizeStream")) { + execution_ok = false; + break; + } + const auto wall_end = std::chrono::steady_clock::now(); + const double host_us = std::chrono::duration(wall_end - wall_begin).count(); + // D2H 同样避开约 1 GiB 的 worker arena：共享前缀用于 flag/vend/frontier 校验，末尾 results 单独回传。 + if (!CheckAcl( + aclrtMemcpy( + state.get(), pa_scheduler::host::StatePrefixBytes(), state_device, + pa_scheduler::host::StatePrefixBytes(), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H state prefix)" + ) || + !CheckAcl( + aclrtMemcpy( + state->results, pa_scheduler::host::ResultBytes(), + &static_cast(state_device)->results[0], + pa_scheduler::host::ResultBytes(), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H worker results)" + )) { + execution_ok = false; + break; + } + if (real_compute && + !CheckAcl( + aclrtMemcpy( + workload_outputs.data(), + static_cast(pa_scheduler::winner_workload::kOutputTiles) * + pa_scheduler::winner_workload::kTileBytes, + static_cast(workload_device) + + pa_scheduler::winner_workload::kSharedInputTiles * + pa_scheduler::winner_workload::kTileBytes, + static_cast(pa_scheduler::winner_workload::kOutputTiles) * + pa_scheduler::winner_workload::kTileBytes, + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H real-compute outputs)" + )) { + execution_ok = false; + break; + } + if (options.trace_enabled && + !CheckAcl( + aclrtMemcpy( + &trace_header, sizeof(trace_header), trace_device, sizeof(trace_header), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H swimlane header)" + )) { + execution_ok = false; + break; + } + // 常规校验只需 header 中的 per-worker count；真实 records 在分析或导出时才按核、按实际 count 懒加载。 + const auto read_trace_records = + [trace_device](uint32_t worker, uint32_t count, pa_scheduler::TraceRecord *records) { + // 每核记录区采用固定容量 stride；只复制 header 声明的实际 count，避免 D2H 未使用的尾部空间。 + const uint64_t offset = sizeof(pa_scheduler::TraceHeader) + + static_cast(worker) * pa_scheduler::kTraceRecordsPerCore * + sizeof(pa_scheduler::TraceRecord); + return CheckAcl( + aclrtMemcpy( + records, static_cast(count) * sizeof(pa_scheduler::TraceRecord), + static_cast(trace_device) + offset, + static_cast(count) * sizeof(pa_scheduler::TraceRecord), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H swimlane records)" + ); + }; + // 先完成共享状态、拓扑、计数和 trace header 的语义校验，再允许 raw JSON 成为性能证据。 + const pa_scheduler::host::Metrics metrics = pa_scheduler::host::Validate( + *state, run, host_us, options.trace_enabled ? &trace_header : nullptr + ); + all_passed &= metrics.passed; + const bool workload_passed = + !real_compute || ValidateRealComputeOutputs( + *state, workload_options, workload_outputs, run + ); + all_passed &= workload_passed; + PmuValidation pmu_validation; + const bool pmu_passed = ValidatePmu( + *state, run, pmu_options, workload_options, + pmu_options.mode == pa_scheduler::ccec_pmu::WindowMode::Off ? nullptr : &pmu_owner.Control(), + &pmu_validation + ); + all_passed &= pmu_passed; + spans.push_back(metrics.submit_span_us); + if (!pmu_options.json_path.empty()) { + if (!metrics.passed || !workload_passed || !pmu_passed || !pmu_owner_evidence_valid) { + std::fprintf(stderr, "PMU JSON rejected because semantic, PMU, or owner validation failed.\n"); + postprocess_ok = false; + break; + } + pmu_json_ready = true; + pmu_json_semantic_passed = metrics.passed && workload_passed; + pmu_json_workload_output_passed = workload_passed; + pmu_json_run = run; + pmu_json_host_us = host_us; + pmu_json_submit_span_us = metrics.submit_span_us; + pmu_json_validation = pmu_validation; + } + if (options.analyze_swimlane && + !pa_scheduler::host::AnalyzeSwimlaneRecords(trace_header, *state, read_trace_records)) { + // 后处理错误使用 break 汇入统一 cleanup；与初始化/launch 失败的进程级立即返回语义区分开。 + postprocess_ok = false; + break; + } + if (!options.swimlane_json.empty()) { + // 只有语义校验通过才把 raw JSON 经“临时文件写完后 rename”发布， + // 避免把截断或错误调度结果误当成可用性能证据。 + if (!metrics.passed || !workload_passed) { + std::fprintf(stderr, "Skipping swimlane export because semantic validation failed.\n"); + postprocess_ok = false; + break; + } + if (!pa_scheduler::host::ExportSwimlaneRecords( + trace_header, options.swimlane_json, workload_options.mode, + real_compute + ? workload_options.repeats + : pa_scheduler::WorkloadCounts{ + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + }, + real_compute + ? pa_scheduler::host::RealComputePatternName(workload_options.pattern) + : "none", + options.trace_atomics, + read_trace_records + )) { + postprocess_ok = false; + break; + } + } + } + + const double median_submit_span_us = spans.empty() ? 0.0 : pa_scheduler::host::Median(spans); + std::printf( + "[SUMMARY] runs=%u completed_runs=%zu median_submit_span_us=%.3f " + "execution_status=%s semantic_status=%s postprocess_status=%s\n", + options.runs, spans.size(), median_submit_span_us, execution_ok ? "PASS" : "FAIL", + all_passed ? "PASS" : "FAIL", postprocess_ok ? "PASS" : "FAIL" + ); + + // 后处理失败也统一走设备资源释放、ELF 卸载和 ACL 收尾，避免文件系统错误遗留运行时上下文。 + bool cleanup_ok = true; + bool pmu_owner_restore_ok = true; + // 先释放依赖当前 device/context 的大块内存，再卸载 ELF、销毁 stream，最后 reset device 与 finalize ACL。 + if (trace_device != nullptr) { + cleanup_ok &= CheckAcl(aclrtFree(trace_device), "aclrtFree(swimlane trace)"); + } + if (pmu_registers_device != nullptr) { + // owner 必须在 MMIO 映射、device context 和 ACL runtime 仍有效时恢复。 + pmu_owner_restore_ok = pmu_owner.Finalize(); + cleanup_ok &= pmu_owner_restore_ok; + cleanup_ok &= UnmapPmuRegisters(options.device, &pmu_mappings); + } + if (workload_device != nullptr) { + cleanup_ok &= CheckAcl( + aclrtFree(workload_allocation.Release()), "aclrtFree(real-compute workspace)" + ); + } + cleanup_ok &= CheckAcl(aclrtFree(state_device), "aclrtFree(state)"); + const rtError_t unload_error = + registered_all ? rtDevBinaryUnRegister(kernel_handle) : rtBinaryUnLoad(kernel_handle); + cleanup_ok &= CheckRt(unload_error, "unload mixed AICore ELF"); + cleanup_ok &= CheckAcl(aclrtDestroyStream(stream), "aclrtDestroyStream"); + cleanup_ok &= CheckAcl(aclrtResetDevice(options.device), "aclrtResetDevice"); + cleanup_ok &= CheckAcl(aclFinalize(), "aclFinalize"); + if (!pmu_options.json_path.empty()) { + if (!pmu_json_ready || !all_passed || !postprocess_ok || !cleanup_ok || !pmu_owner_restore_ok) { + std::fprintf(stderr, "PMU JSON was not published because the capture or restore transaction failed.\n"); + postprocess_ok = false; + } else if (!ExportPmuJson( + *state, options, pmu_options, workload_options, pmu_json_run, + pmu_json_host_us, pmu_json_submit_span_us, pmu_json_validation, + pmu_json_semantic_passed, pmu_json_workload_output_passed, + pmu_owner_evidence, + pmu_owner_restore_ok, pmu_options.json_path + )) { + postprocess_ok = false; + } + } + // 运行语义、后处理和资源清理三者全部成功，进程才返回成功，脚本据此决定是否继续生成 merged 泳道。 + return execution_ok && all_passed && postprocess_ok && cleanup_ok ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/kernel.cpp b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/kernel.cpp new file mode 100644 index 0000000000..74c5aa3f0f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/kernel.cpp @@ -0,0 +1,401 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include +#include +#include +#include + +#include "pmu_probe.h" +#include "../common/winner_workload.h" + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" + +#define PA_CCEC_OPS_DEFINE_REAL_WORKLOAD 1 +#include "ccec_ops.h" +#undef PA_CCEC_OPS_DEFINE_REAL_WORKLOAD + +using pa_scheduler_ccec::CcecOps; + +namespace { +#if PA_BUILD_SUBMIT_PMU +struct PmuSnapshot { + uint64_t total_cycles = 0; + uint32_t vector_busy = 0; + uint32_t cube_busy = 0; + uint32_t scalar_busy = 0; + uint32_t mte1_busy = 0; + uint32_t mte2_busy = 0; + uint32_t mte3_busy = 0; + uint32_t icache_requests = 0; + uint32_t icache_misses = 0; + uint32_t fix_busy = 0; + uint32_t status = 0; +}; + +struct IcacheShadowSnapshot { + uint32_t requests = 0; + uint32_t misses = 0; +}; + +} // namespace + +namespace pa_scheduler_ccec { + +struct SubmitPmuContext { + uint64_t reg_base = 0; + uint64_t shadow_requests = 0; + uint64_t shadow_misses = 0; + uint64_t phase_requests = 0; + uint64_t phase_misses = 0; + // begin 的 shadow read-clear 完成后取起点，end 的 shadow read-clear 之前 + // 取终点；累计值因此不包含两次 PMU 寄存器读取本身。 + uint64_t phase_elapsed_ticks = 0; + uint64_t phase_begin_tick = 0; + uint32_t selector_status = 0; + uint32_t phase_status = pa_scheduler::ccec_pmu::kPhaseStatusRequested; + uint32_t phase_calls = 0; + uint32_t begin_reads = 0; + uint32_t end_reads = 0; + bool started = false; + bool phase_armed = false; + bool boundary_error = false; +}; + +} // namespace pa_scheduler_ccec + +namespace { + +using pa_scheduler_ccec::SubmitPmuContext; + +template +__aicore__ inline uint32_t ReadPmuRegister(uint64_t reg_base) { + // 传给 ld_dev 的是重基址后的 __gm__ 指针；相对 offset 均落在编译器允许的 [-2048, 2047]。 + int32_t *block = reinterpret_cast(reg_base + BlockOffset); + return static_cast(ld_dev(block, static_cast(RegisterOffset - BlockOffset))); +} + +__aicore__ inline PmuSnapshot ReadObservedCounters(uint64_t reg_base) { + PmuSnapshot sample; + sample.vector_busy = ReadPmuRegister(reg_base); + sample.cube_busy = ReadPmuRegister(reg_base); + sample.scalar_busy = ReadPmuRegister(reg_base); + sample.mte1_busy = ReadPmuRegister(reg_base); + sample.mte2_busy = ReadPmuRegister(reg_base); + // submit-pmu 将 CNT5 留给 shadow I-cache miss。这里不能提前读取，否则 + // read-to-clear 会让随后的 shadow tail 漏计；MTE3 busy 在该诊断构建不可用。 + sample.mte3_busy = 0; + sample.icache_requests = ReadPmuRegister(reg_base); + sample.icache_misses = ReadPmuRegister(reg_base); + const uint64_t low = ReadPmuRegister(reg_base); + const uint64_t high = ReadPmuRegister(reg_base); + sample.total_cycles = low | (high << 32); + return sample; +} + +__aicore__ inline IcacheShadowSnapshot ReadShadowCounters(uint64_t reg_base) { + IcacheShadowSnapshot sample; + sample.requests = ReadPmuRegister(reg_base); + sample.misses = ReadPmuRegister(reg_base); + return sample; +} + +struct PmuRegisterContext { + uint64_t reg_base = 0; + uint32_t status = 0; + bool shadow_selectors = false; +}; + +__aicore__ inline PmuRegisterContext ResolvePmuRegisters(__gm__ pa_scheduler::SchedulerState *state) { + using namespace pa_scheduler::ccec_pmu; + PmuRegisterContext context; + const uint32_t physical_core_id = static_cast(get_coreid()) & kStatusCoreIdMask; + context.status = kStatusRequested | (physical_core_id << kStatusCoreIdShift); + const uint64_t table_address = + static_cast(state->config.reserved[kConfigRegTableLow]) | + (static_cast(state->config.reserved[kConfigRegTableHigh]) << 32); + if (state->config.reserved[kConfigMagic] != kConfigMagicValue || table_address == 0 || + physical_core_id >= kPhysicalSubcoreCount) { + return context; + } + context.status |= kStatusCoreIdValid; + __gm__ const uint64_t *register_bases = reinterpret_cast<__gm__ const uint64_t *>(table_address); + context.reg_base = register_bases[physical_core_id]; + if (context.reg_base == 0) return context; + context.status |= kStatusRegMapped; + + // selector 与同 phase 目录内的 owner 逐项核对。CNT5/CNT8 分别重复 + // CNT7/CNT6；CNT9 保持正式 PIPE_UTIL 的 unused(0) 口径。 + if (ReadPmuRegister(context.reg_base) == kVectorBusyEvent) + context.status |= kStatusCnt0Selector; + if (ReadPmuRegister(context.reg_base) == kCubeBusyEvent) + context.status |= kStatusCnt1Selector; + if (ReadPmuRegister(context.reg_base) == kScalarBusyEvent) + context.status |= kStatusCnt2Selector; + if (ReadPmuRegister(context.reg_base) == kMte1BusyEvent) + context.status |= kStatusCnt3Selector; + if (ReadPmuRegister(context.reg_base) == kMte2BusyEvent) + context.status |= kStatusCnt4Selector; + const bool cnt5_ok = + ReadPmuRegister(context.reg_base) == kIcacheMissEvent; + if (cnt5_ok) + context.status |= kStatusCnt5Selector; + if (ReadPmuRegister(context.reg_base) == kIcacheRequestEvent) + context.status |= kStatusCnt6Selector; + if (ReadPmuRegister(context.reg_base) == kIcacheMissEvent) + context.status |= kStatusCnt7Selector; + const bool cnt8_ok = + ReadPmuRegister(context.reg_base) == kIcacheRequestEvent; + const bool cnt9_unused = + ReadPmuRegister(context.reg_base) == 0U; + if (cnt8_ok) + context.status |= kStatusCnt8Selector; + context.shadow_selectors = cnt5_ok && cnt8_ok && cnt9_unused; + return context; +} + +__aicore__ inline void PublishPmuSnapshot( + __gm__ pa_scheduler::WorkerResult &result, const PmuSnapshot &sample +) { + // 每核独占 sidecar 通过 bypass store 一次性发布；这些写发生在 PMU stop/read 之后， + // 不进入被导出的 Submit 窗口。 + CcecOps::Publish(&result.pmu_total_cycles, sample.total_cycles); + CcecOps::Publish(&result.pmu_scalar_busy, sample.scalar_busy); + CcecOps::Publish(&result.pmu_icache_requests, sample.icache_requests); + CcecOps::Publish(&result.pmu_icache_misses, sample.icache_misses); + CcecOps::Publish(&result.pmu_status, sample.status); + CcecOps::Publish(&result.pmu_vector_busy, sample.vector_busy); + CcecOps::Publish(&result.pmu_cube_busy, sample.cube_busy); + CcecOps::Publish(&result.pmu_mte1_busy, sample.mte1_busy); + CcecOps::Publish(&result.pmu_mte2_busy, sample.mte2_busy); + CcecOps::Publish(&result.pmu_mte3_busy, sample.mte3_busy); + // CNT8 已改作 shadow request，submit-pmu 不再发布 fix-busy。 + CcecOps::Publish(&result.pmu_fix_busy, static_cast(0)); + CcecOps::Publish(&result.pmu_warm_total_cycles, static_cast(0)); + CcecOps::Publish(&result.pmu_warm_window_ticks, static_cast(0)); +} + +__aicore__ inline bool FitsUint32(uint64_t value) { + return value <= 0xffffffffULL; +} + +__aicore__ inline void PublishSubmitPmuContext( + __gm__ pa_scheduler::WorkerResult &result, const SubmitPmuContext &context +) { + CcecOps::Publish(&result.pmu_build_variant, pa_scheduler::kBuildVariantSubmitPmu); + CcecOps::Publish( + &result.pmu_phase_id, + static_cast(pa_scheduler::kCompiledSubmitPmuPhase) + ); + CcecOps::Publish(&result.pmu_phase_calls, context.phase_calls); + CcecOps::Publish(&result.pmu_phase_status, context.phase_status); + CcecOps::Publish(&result.pmu_phase_begin_reads, context.begin_reads); + CcecOps::Publish(&result.pmu_phase_end_reads, context.end_reads); + CcecOps::Publish(&result.pmu_phase_elapsed_ticks, context.phase_elapsed_ticks); + CcecOps::Publish(&result.pmu_phase_icache_requests, static_cast(context.phase_requests)); + CcecOps::Publish(&result.pmu_phase_icache_misses, static_cast(context.phase_misses)); + CcecOps::Publish(&result.pmu_shadow_icache_requests, static_cast(context.shadow_requests)); + CcecOps::Publish(&result.pmu_shadow_icache_misses, static_cast(context.shadow_misses)); +} + +} // namespace + +namespace pa_scheduler_ccec { + +__aicore__ inline CcecOps::PmuContext CcecOps::PmuWindowStart( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +) { + using namespace pa_scheduler::ccec_pmu; + (void)worker_id; + SubmitPmuContext context; + const WindowMode mode = static_cast(state->config.reserved[kConfigMode]); + if (mode != WindowMode::SubmitAll) return context; + // Main AICPU owner 已在 launch 前配置并开启计数；先 stop + snapshot/read-clear， + // 再解析 selector，避免这些 ld_dev 污染完整 Submit 窗口。 + bisheng::cce::metrics_prof_stop(); + const PmuRegisterContext registers = ResolvePmuRegisters(state); + context.reg_base = registers.reg_base; + context.selector_status = registers.status; + if (registers.shadow_selectors) { + context.phase_status |= kPhaseStatusShadowSelectors; + } + if (context.reg_base == 0) return context; + (void)ReadObservedCounters(context.reg_base); + (void)ReadShadowCounters(context.reg_base); + bisheng::cce::metrics_prof_start(); + context.started = true; + context.phase_status |= kPhaseStatusWindowStarted; + return context; +} + +__aicore__ inline void CcecOps::PmuPhaseBegin(PmuContext &context) { + if (!context.started || context.reg_base == 0 || context.phase_armed) { + context.boundary_error = true; + return; + } + // counter 在运行中读取即清零；begin 之前的片段只进入 shadow whole。 + const IcacheShadowSnapshot sample = ReadShadowCounters(context.reg_base); + context.shadow_requests += sample.requests; + context.shadow_misses += sample.misses; + ++context.begin_reads; + context.phase_armed = true; + // get_sys_cnt() 是本机已校准为 1 ns/tick 的 A5 系统计数器。该读取位于 + // begin 的两条 ld_dev 之后，因此不会把 read-clear 成本算进阶段时间。 + context.phase_begin_tick = CcecOps::Now(); +} + +__aicore__ inline void CcecOps::PmuPhaseEnd(PmuContext &context) { + // 先取终点再读取 shadow counter，使 end 的两条 ld_dev 同样位于阶段之外。 + const uint64_t phase_end_tick = CcecOps::Now(); + if (!context.started || context.reg_base == 0 || !context.phase_armed) { + context.boundary_error = true; + return; + } + if (phase_end_tick < context.phase_begin_tick) { + context.boundary_error = true; + } else { + context.phase_elapsed_ticks += phase_end_tick - context.phase_begin_tick; + } + // end 读出的片段同时属于完整 shadow 重建与被选中的局部阶段。 + const IcacheShadowSnapshot sample = ReadShadowCounters(context.reg_base); + context.shadow_requests += sample.requests; + context.shadow_misses += sample.misses; + context.phase_requests += sample.requests; + context.phase_misses += sample.misses; + ++context.end_reads; + ++context.phase_calls; + context.phase_armed = false; + context.phase_begin_tick = 0; +} + +__aicore__ inline void CcecOps::PmuWindowStop( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id, PmuContext &context +) { + using namespace pa_scheduler::ccec_pmu; + __gm__ pa_scheduler::WorkerResult &result = state->results[worker_id]; + PmuSnapshot sample; + sample.status = context.selector_status; + if (context.started && context.reg_base != 0) { + // gate 只在整个 Submit 前后各操作一次。停止后先读从未中途清零的 + // primary counter（不含 shadow CNT5）之后，再读取 CNT8/CNT5 tail + // 完成软件重建。 + bisheng::cce::metrics_prof_stop(); + sample = ReadObservedCounters(context.reg_base); + const IcacheShadowSnapshot tail = ReadShadowCounters(context.reg_base); + context.shadow_requests += tail.requests; + context.shadow_misses += tail.misses; + sample.status = context.selector_status | kStatusWindowStarted | kStatusWindowStopped; + context.phase_status |= kPhaseStatusWindowStopped; + if (sample.total_cycles != 0) sample.status |= kStatusTotalNonzero; + } + + if (context.shadow_requests == sample.icache_requests) + context.phase_status |= kPhaseStatusShadowRequestsMatch; + if (context.shadow_misses == sample.icache_misses) + context.phase_status |= kPhaseStatusShadowMissesMatch; + if (!context.boundary_error && !context.phase_armed && + context.begin_reads == context.end_reads && context.end_reads == context.phase_calls) + context.phase_status |= kPhaseStatusBoundariesBalanced; + // 两个 shadow counter 是顺序 ld_dev，并非同一时刻的原子快照；局部 + // phase 的 miss/request 边界会错开数条指令，故不能硬性要求局部 + // miss<=request。A5 上运行中 read-to-clear 还会与同周期事件递增竞争， + // shadow 允许小于未中途读取的 primary，但绝不能反向超过它。primary- + // shadow 是该次采集可直接给出的局部分段误差包络，而不是要静默吞掉的差值。 + if (context.phase_requests <= context.shadow_requests && + context.phase_misses <= context.shadow_misses && + context.shadow_misses <= context.shadow_requests && + context.shadow_requests <= sample.icache_requests && + context.shadow_misses <= sample.icache_misses) + context.phase_status |= kPhaseStatusValuesOrdered; + if (FitsUint32(context.shadow_requests) && FitsUint32(context.shadow_misses) && + FitsUint32(context.phase_requests) && FitsUint32(context.phase_misses)) + context.phase_status |= kPhaseStatusUint32Fit; + + const bool none_shape = + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None && + context.phase_calls == 0 && context.begin_reads == 0 && context.end_reads == 0 && + context.phase_requests == 0 && context.phase_misses == 0 && + context.phase_elapsed_ticks == 0; + const bool running_shape = + pa_scheduler::kCompiledSubmitPmuPhase != pa_scheduler::SubmitPmuPhase::None && + context.phase_calls == state->config.batches * pa_scheduler::kTasksPerBatch && + context.begin_reads == context.phase_calls && + context.end_reads == context.phase_calls; + if (none_shape || running_shape) + context.phase_status |= kPhaseStatusPhaseShape; + const bool phase_time_valid = + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? context.phase_elapsed_ticks == 0 + : context.phase_calls != 0 && context.phase_elapsed_ticks != 0; + if (phase_time_valid) + context.phase_status |= kPhaseStatusTimeValid; + + PublishPmuSnapshot(result, sample); + PublishSubmitPmuContext(result, context); +} +#endif // PA_BUILD_SUBMIT_PMU + +} // namespace + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) && defined(PA_BUILD_AIC) +// split artifact: the runtime entry/state-owner TU calls this orchestration +// function once per kernel launch. It is not a launchable kernel and the +// version script localizes it in the final mixed ELF. +extern "C" __attribute__((noinline, used)) __aicore__ void +pa_scheduler_lazy_sample_callback_orchestration_aic( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +) { + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aic); +} +#elif defined(PA_LAZY_SAMPLE_SPLIT_FINISH) && defined(PA_BUILD_AIV) +extern "C" __attribute__((noinline, used)) __aicore__ void +pa_scheduler_lazy_sample_callback_orchestration_aiv( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +) { + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aiv); +} +#elif defined(PA_BUILD_AIC) +// 同一源码分别按 cube/vec 架构编译；metadata 声明每个物理 block 静态组合 1 个 AIC 与 2 个 AIV。 +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aic, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aic(__gm__ pa_scheduler::SchedulerState *state) { + // 32 个物理 block 的 AIC 直接使用 block_idx，形成连续 worker 0..31。 + const uint32_t worker_id = static_cast(get_block_idx()); + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aic); +} +#elif defined(PA_BUILD_AIV) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aiv, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aiv(__gm__ pa_scheduler::SchedulerState *state) { + // 每个 block 的两个 vector sub-block 展平为 vector_id=2*b+subblock，偏移 32 后形成 worker 32..95。 + const uint32_t vector_id = static_cast(get_block_idx() * get_subblockdim() + get_subblockid()); + const uint32_t worker_id = pa_scheduler::kAicWorkers + vector_id; + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aiv); +} +#else +#error "Compile with PA_BUILD_AIC or PA_BUILD_AIV" +#endif diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pa_scheduler_device_exports.map b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pa_scheduler_device_exports.map new file mode 100644 index 0000000000..b4068c90f1 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pa_scheduler_device_exports.map @@ -0,0 +1,7 @@ +{ + global: + pa_scheduler_0_mix_aic; + pa_scheduler_0_mix_aiv; + local: + *; +}; diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_owner_control.h b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_owner_control.h new file mode 100644 index 0000000000..eb17c4c1b0 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_owner_control.h @@ -0,0 +1,225 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_CONTROL_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_CONTROL_H_ + +#include +#include + +namespace pa_scheduler::pmu_owner { + +// 这份头文件同时供 x86 host 与 AArch64 AICPU helper 使用。所有跨端字段都采用 +// 固定宽度整数；禁止在 ABI 中放 host 指针、bool、STL 容器或编译器相关位域。 +constexpr uint32_t kPmuOwnerControlMagic = 0x504d554fU; // "PMUO" +constexpr uint32_t kPmuOwnerControlVersion = 1U; + +// DAV_3510 一共有 2 die；每个 die 依次放 18 个 AIC 和 36 个 AIV 物理槽。 +// 当前 A5 stream 实际开放 32 个 AIC 与 64 个 AIV，其余 12 个槽的 MMIO +// 读回不会匹配配置值，因此 owner 必须扫描 108 槽，最终取得 96 个可用槽。 +constexpr uint32_t kPhysicalSubcoreCount = 108U; +constexpr uint32_t kExpectedSubcoreCount = 96U; +constexpr uint32_t kExpectedAicCount = 32U; +constexpr uint32_t kExpectedAivCount = 64U; +constexpr uint32_t kAicPerDie = 18U; +constexpr uint32_t kSubcoresPerDie = 54U; +constexpr uint32_t kConfiguredBitmapWords = 4U; +constexpr uint32_t kDiagnosticIndexUnset = 0xffffffffU; + +static_assert(kExpectedAicCount + kExpectedAivCount == kExpectedSubcoreCount, "active topology count mismatch"); +static_assert(kAicPerDie * 2U == 36U, "physical AIC topology changed"); +static_assert(kSubcoresPerDie * 2U == kPhysicalSubcoreCount, "physical subcore topology changed"); + +// A5 PIPE_UTILIZATION 的正式 counter 槽位布局。submit-pmu 用 CNT8/CNT5 +// 重复配置 I-cache request/miss，作为允许中途 read-to-clear 的 shadow； +// CNT6/7 始终不在阶段边界读取，保留为完整 Submit 的权威对照。 +// +// 不能把 miss 放进 CNT9：A5 b1 实测表明 CNT9 selector 虽能回读 0x35， +// 但计数恒为 0；正式 PIPE_UTIL 表也把 CNT9 标成 unused。0x35 在独立 +// I-cache 微基准的低位 counter 已验证可计数，因此诊断构建让 CNT5 承担 +// shadow miss，并明确放弃该构建中的 MTE3 busy。 +constexpr uint32_t kPmuCounterCount = 10U; +constexpr uint32_t kConfiguredSelectors[kPmuCounterCount] = { + 0x501U, // CNT0: vector busy + 0x301U, // CNT1: cube busy + 0x001U, // CNT2: scalar busy + 0x701U, // CNT3: MTE1 busy + 0x202U, // CNT4: MTE2 busy +#if PA_BUILD_SUBMIT_PMU + 0x035U, // CNT5: shadow I-cache miss + 0x034U, // CNT6: I-cache request（完整 Submit） + 0x035U, // CNT7: I-cache miss（完整 Submit） + 0x034U, // CNT8: shadow I-cache request + 0x000U, // CNT9: A5 PIPE_UTIL 正式未使用 +#else + 0x203U, // CNT5: MTE3 busy + 0x034U, // CNT6: I-cache request + 0x035U, // CNT7: I-cache miss + 0x714U, // CNT8: fix-pipe busy + 0x000U, // CNT9: 未使用 +#endif +}; + +constexpr int32_t kStatusPending = 0x7fffffff; + +// AICPU entry 始终向 runtime 返回 0；协议结果只通过 control.status 回传， +// 从而避免一次可诊断的配置失败被 runtime 升格成整条 stream 异常。 +enum class PmuOwnerStatus : int32_t { + Success = 0, + InvalidArguments = -1, + InvalidControl = -2, + UnexpectedTopology = -3, + AlreadyConfigured = -4, + ConfigureCountMismatch = -5, + ConfigureRollbackFailed = -6, + ConfigureSlotRestoreFailed = -7, + RestoreFailed = -8, +}; + +// 首个异常寄存器使用稳定的枚举编号，host 不需要解析 AICPU 日志即可定位 +// 是基址、selector、计数范围还是 enable 控制读回不一致。 +enum class PmuOwnerField : uint32_t { + None = 0, + Arguments, + ControlMagic, + ControlVersion, + ControlSize, + State, + RegisterBase, + Ctrl0, + Ctrl1, + Selector0, + Selector1, + Selector2, + Selector3, + Selector4, + Selector5, + Selector6, + Selector7, + Selector8, + Selector9, + StartCycleLow, + StartCycleHigh, + StopCycleLow, + StopCycleHigh, + BitmapCount, + TotalCount, + AicCount, + AivCount, +}; + +// 单个物理子核被 owner 改动的完整可恢复状态恰好占一条 cache line。 +// PMU counter 是 read-to-clear，旧 counter 值无法恢复；owner 会话必须独占。 +struct alignas(64) PmuSavedRegisters { + uint32_t ctrl0; + uint32_t ctrl1; + uint32_t selectors[kPmuCounterCount]; + uint32_t start_cycle_low; + uint32_t start_cycle_high; + uint32_t stop_cycle_low; + uint32_t stop_cycle_high; +}; + +// Host 与 AICPU 共享的 owner 状态。前 128B 是命令结果和诊断，随后内嵌 +// 108 个 MMIO 基址、4-word 所有权 bitmap，以及每槽 64B 的 Configure 快照。 +// bitmap 的严格语义是“原值已保存、且 owner 可能已经改写 MMIO、但尚未 +// 完整恢复”的槽；它在 Configure 写第一项 MMIO 前置位，仅在恢复读回完整 +// 一致后清位。Restore 期间不得清零或重建 saved[]，只能按 bitmap 逆序消费。 +struct alignas(64) PmuOwnerControl { + uint32_t magic; + uint32_t version; + uint32_t struct_bytes; + volatile int32_t status; + + uint32_t configured; + uint32_t expected_total; + uint32_t expected_aic; + uint32_t expected_aiv; + + // active_* 与 bitmap 表示仍由本 owner 持有、尚未恢复的物理槽。 + uint32_t active_total; + uint32_t active_aic; + uint32_t active_aiv; + // discovered_* 保留本次 Configure 扫描结果；即使计数不匹配后回滚， + // host 仍能看到回滚前究竟探测到了多少 AIC/AIV。 + uint32_t discovered_total; + uint32_t discovered_aic; + uint32_t discovered_aiv; + uint32_t skipped_total; + + uint32_t first_failed_index; + uint32_t first_failed_field; + uint32_t first_failed_observed; + uint32_t first_failed_expected; + + uint32_t restore_failures; + uint32_t first_restore_failed_index; + uint32_t first_restore_failed_field; + uint32_t first_restore_failed_observed; + uint32_t first_restore_failed_expected; + uint32_t reserved_header[8]; + + uint64_t register_bases[kPhysicalSubcoreCount]; + // 字段名保留 configured_bitmap 以稳定 host/device ABI；失败路径中它还会 + // 临时包含“配置未通过但恢复仍待重试”的 owned 槽。 + uint32_t configured_bitmap[kConfiguredBitmapWords]; + // 让 saved[] 从新的 64B cache line 开始；该 padding 不承载协议含义。 + uint32_t reserved_bitmap[4]; + PmuSavedRegisters saved[kPhysicalSubcoreCount]; +}; + +static_assert(sizeof(PmuSavedRegisters) == 64U, "one saved PMU slot must occupy one cache line"); +static_assert(alignof(PmuSavedRegisters) == 64U, "saved PMU slot alignment changed"); +static_assert(offsetof(PmuOwnerControl, status) == 12U, "PMU owner status offset changed"); +static_assert(offsetof(PmuOwnerControl, register_bases) == 128U, "PMU owner header must occupy two cache lines"); +static_assert(offsetof(PmuOwnerControl, configured_bitmap) == 992U, "PMU owner bitmap offset changed"); +static_assert(offsetof(PmuOwnerControl, saved) == 1024U, "PMU owner saved area must be cache-line aligned"); +static_assert(sizeof(PmuOwnerControl) == 7936U, "PMU owner control ABI changed"); +static_assert(sizeof(PmuOwnerControl) % 64U == 0U, "PMU owner control must use complete cache lines"); +static_assert(alignof(PmuOwnerControl) == 64U, "PMU owner control alignment changed"); + +inline bool IsAicPhysicalSlot(uint32_t index) +{ + return index < kPhysicalSubcoreCount && (index % kSubcoresPerDie) < kAicPerDie; +} + +inline bool IsConfigured(const PmuOwnerControl &control, uint32_t index) +{ + return index < kPhysicalSubcoreCount && + (control.configured_bitmap[index / 32U] & (1U << (index % 32U))) != 0U; +} + +inline void SetConfigured(PmuOwnerControl *control, uint32_t index) +{ + if (index < kPhysicalSubcoreCount) { + control->configured_bitmap[index / 32U] |= 1U << (index % 32U); + } +} + +inline void ClearConfigured(PmuOwnerControl *control, uint32_t index) +{ + if (index < kPhysicalSubcoreCount) { + control->configured_bitmap[index / 32U] &= ~(1U << (index % 32U)); + } +} + +inline uint32_t CountConfigured(const PmuOwnerControl &control) +{ + uint32_t count = 0U; + for (uint32_t index = 0U; index < kPhysicalSubcoreCount; ++index) { + count += IsConfigured(control, index) ? 1U : 0U; + } + return count; +} + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_CONTROL_H_ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_owner_host.h b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_owner_host.h new file mode 100644 index 0000000000..68dfac042e --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_owner_host.h @@ -0,0 +1,345 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_HOST_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_HOST_H_ + +#include "pmu_owner_control.h" +#include "pmu_owner_main_abi.h" +#include "pmu_owner_main_loader.h" + +#include "acl/acl.h" + +#include +#include +#include +#include +#include +#include + +namespace pa_scheduler::pmu_owner { + +inline bool OwnerCheckAcl(aclError error, const char *label) +{ + if (error == ACL_SUCCESS) return true; + std::fprintf(stderr, "ACL error %d: %s\n", static_cast(error), label); + return false; +} + +inline std::string ArtifactBesideKernel(const std::string &kernel_path, const char *name) +{ + const size_t slash = kernel_path.find_last_of('/'); + return slash == std::string::npos ? std::string(name) : kernel_path.substr(0, slash + 1U) + name; +} + +inline const char *OwnerFieldName(PmuOwnerField field) +{ + switch (field) { + case PmuOwnerField::None: return "none"; + case PmuOwnerField::Arguments: return "arguments"; + case PmuOwnerField::ControlMagic: return "control-magic"; + case PmuOwnerField::ControlVersion: return "control-version"; + case PmuOwnerField::ControlSize: return "control-size"; + case PmuOwnerField::State: return "state"; + case PmuOwnerField::RegisterBase: return "register-base"; + case PmuOwnerField::Ctrl0: return "ctrl0"; + case PmuOwnerField::Ctrl1: return "ctrl1"; + case PmuOwnerField::Selector0: return "selector0"; + case PmuOwnerField::Selector1: return "selector1"; + case PmuOwnerField::Selector2: return "selector2"; + case PmuOwnerField::Selector3: return "selector3"; + case PmuOwnerField::Selector4: return "selector4"; + case PmuOwnerField::Selector5: return "selector5"; + case PmuOwnerField::Selector6: return "selector6"; + case PmuOwnerField::Selector7: return "selector7"; + case PmuOwnerField::Selector8: return "selector8"; + case PmuOwnerField::Selector9: return "selector9"; + case PmuOwnerField::StartCycleLow: return "start-cycle-low"; + case PmuOwnerField::StartCycleHigh: return "start-cycle-high"; + case PmuOwnerField::StopCycleLow: return "stop-cycle-low"; + case PmuOwnerField::StopCycleHigh: return "stop-cycle-high"; + case PmuOwnerField::BitmapCount: return "bitmap-count"; + case PmuOwnerField::TotalCount: return "total-count"; + case PmuOwnerField::AicCount: return "aic-count"; + case PmuOwnerField::AivCount: return "aiv-count"; + } + return "unknown"; +} + +struct ActiveSubcoreLimits { + uint32_t aic = 0U; + uint32_t aiv = 0U; + uint32_t total = 0U; +}; + +inline bool QueryActiveSubcoreLimits(aclrtStream scheduling_stream, ActiveSubcoreLimits *limits) +{ + if (scheduling_stream == nullptr || limits == nullptr) { + std::fprintf(stderr, "Cannot query active PMU subcores with a null stream/result.\n"); + return false; + } + uint32_t aic = 0U; + uint32_t aiv = 0U; + const aclError aic_error = + aclrtGetStreamResLimit(scheduling_stream, ACL_RT_DEV_RES_CUBE_CORE, &aic); + const aclError aiv_error = + aclrtGetStreamResLimit(scheduling_stream, ACL_RT_DEV_RES_VECTOR_CORE, &aiv); + const uint64_t total = static_cast(aic) + aiv; + if (aic_error != ACL_SUCCESS || aiv_error != ACL_SUCCESS || + aic != kExpectedAicCount || aiv != kExpectedAivCount || total != kExpectedSubcoreCount) { + std::fprintf( + stderr, + "Unexpected stream PMU topology: aic_error=%d aiv_error=%d " + "aic=%u/%u aiv=%u/%u total=%llu/%u\n", + static_cast(aic_error), static_cast(aiv_error), aic, kExpectedAicCount, + aiv, kExpectedAivCount, static_cast(total), kExpectedSubcoreCount + ); + return false; + } + limits->aic = aic; + limits->aiv = aiv; + limits->total = static_cast(total); + std::printf( + "[PMU_OWNER] stream_active aic=%u aiv=%u total=%u physical_slots=%u\n", + limits->aic, limits->aiv, limits->total, kPhysicalSubcoreCount + ); + return true; +} + +// mixed launch 的一个物理 block 必须同时拥有 1 个 AIC 和相邻的 2 个 AIV。 +// 只检查 32/64 总数仍可能放过孤立 AIV；这里直接按两 die 的真实编号布局验闭包。 +inline bool ValidateConfiguredTripletTopology(const PmuOwnerControl &control) +{ + uint32_t complete_triplets = 0U; + uint32_t broken_triplets = 0U; + const uint32_t dies = kPhysicalSubcoreCount / kSubcoresPerDie; + for (uint32_t die = 0U; die < dies; ++die) { + const uint32_t die_base = die * kSubcoresPerDie; + for (uint32_t local = 0U; local < kAicPerDie; ++local) { + const bool aic = IsConfigured(control, die_base + local); + const bool aiv0 = IsConfigured(control, die_base + kAicPerDie + local * 2U); + const bool aiv1 = IsConfigured(control, die_base + kAicPerDie + local * 2U + 1U); + if (aic == aiv0 && aic == aiv1) { + complete_triplets += aic ? 1U : 0U; + } else { + ++broken_triplets; + } + } + } + const bool passed = complete_triplets == kExpectedAicCount && broken_triplets == 0U; + std::printf( + "[ASSERT] %-48s %s (complete=%u broken=%u)\n", + "PMU owner bitmap is complete 1-AIC + 2-AIV triplets", + passed ? "PASS" : "FAIL", complete_triplets, broken_triplets + ); + return passed; +} + +// owner 命令使用独立 stream，但通过 mode=0 JSON 在主 aicpu_scheduler 中执行。 +// Configure 同步完成后才允许启动 AICore；AICore 正常或异常退出后，Restore +// 都不会依赖业务 stream。MMIO 映射必须保持到 Finalize 完成之后。 +class PmuOwnerSession { +public: + PmuOwnerSession() = default; + PmuOwnerSession(const PmuOwnerSession &) = delete; + PmuOwnerSession &operator=(const PmuOwnerSession &) = delete; + + ~PmuOwnerSession() + { + if (HasResources()) (void)Finalize(); + } + + bool Initialize( + uint32_t device, aclrtStream scheduling_stream, const std::string &dispatcher_path, + const std::string &owner_path, const std::vector ®ister_bases + ) + { + if (HasResources() || register_bases.size() != kPhysicalSubcoreCount) { + std::fprintf( + stderr, "Invalid PMU owner initialization state or register table size: %zu\n", + register_bases.size() + ); + return false; + } + device_ = device; + if (!QueryActiveSubcoreLimits(scheduling_stream, &limits_)) return false; + if (!OwnerCheckAcl(aclrtCreateStream(&owner_stream_), "aclrtCreateStream(PMU owner)")) return false; + if (loader_.Initialize( + dispatcher_path, owner_path, owner_stream_, static_cast(device_) + ) != 0) { + return false; + } + if (!OwnerCheckAcl( + aclrtMalloc(&control_device_, sizeof(PmuOwnerControl), ACL_MEM_MALLOC_NORMAL_ONLY), + "aclrtMalloc(PMU owner control)" + )) { + return false; + } + if ((reinterpret_cast(control_device_) & (alignof(PmuOwnerControl) - 1U)) != 0U) { + std::fprintf(stderr, "PMU owner control is not 64-byte aligned: %p\n", control_device_); + return false; + } + + control_ = PmuOwnerControl{}; + control_.magic = kPmuOwnerControlMagic; + control_.version = kPmuOwnerControlVersion; + control_.struct_bytes = sizeof(PmuOwnerControl); + control_.status = kStatusPending; + control_.expected_total = limits_.total; + control_.expected_aic = limits_.aic; + control_.expected_aiv = limits_.aiv; + std::memcpy(control_.register_bases, register_bases.data(), sizeof(control_.register_bases)); + if (!OwnerCheckAcl( + aclrtMemcpy( + control_device_, sizeof(control_), &control_, sizeof(control_), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D initial PMU owner control)" + )) { + return false; + } + ready_ = true; + return true; + } + + bool Configure() + { + if (!ready_) return false; + const bool command_ok = RunCommand(PmuOwnerMainCommand::Configure, "Configure"); + configured_ = CountConfigured(control_) != 0U; + const uint32_t bitmap_count = CountConfigured(control_); + const bool triplets_ok = ValidateConfiguredTripletTopology(control_); + const bool state_ok = control_.status == static_cast(PmuOwnerStatus::Success) && + control_.configured == 1U && control_.active_total == limits_.total && + control_.active_aic == limits_.aic && control_.active_aiv == limits_.aiv && + control_.discovered_total == limits_.total && control_.discovered_aic == limits_.aic && + control_.discovered_aiv == limits_.aiv && bitmap_count == limits_.total && + control_.skipped_total + bitmap_count == kPhysicalSubcoreCount && triplets_ok; + PrintControl("Configure", bitmap_count); + return command_ok && state_ok; + } + + bool Restore() + { + if (control_device_ == nullptr || owner_stream_ == nullptr || !loader_.IsInitialized()) { + return !configured_; + } + const bool command_ok = RunCommand(PmuOwnerMainCommand::Restore, "Restore"); + const uint32_t bitmap_count = CountConfigured(control_); + const bool state_ok = control_.status == static_cast(PmuOwnerStatus::Success) && + control_.configured == 0U && control_.active_total == 0U && + control_.active_aic == 0U && control_.active_aiv == 0U && bitmap_count == 0U; + configured_ = bitmap_count != 0U; + PrintControl("Restore", bitmap_count); + return command_ok && state_ok; + } + + bool Finalize() + { + bool ok = true; + if (control_device_ != nullptr && owner_stream_ != nullptr && loader_.IsInitialized()) { + bool restored = Restore(); + if (!restored) restored = Restore(); + ok &= restored; + } else { + ok &= !configured_; + } + ok &= loader_.Finalize() == 0; + if (control_device_ != nullptr) { + ok &= OwnerCheckAcl(aclrtFree(control_device_), "aclrtFree(PMU owner control)"); + control_device_ = nullptr; + } + if (owner_stream_ != nullptr) { + ok &= OwnerCheckAcl(aclrtDestroyStream(owner_stream_), "aclrtDestroyStream(PMU owner)"); + owner_stream_ = nullptr; + } + ready_ = false; + configured_ = false; + std::printf("[PMU_OWNER] restore_and_cleanup=%s\n", ok ? "PASS" : "FAIL"); + return ok; + } + + uint64_t RegisterTableDeviceAddress() const + { + if (control_device_ == nullptr) return 0U; + return reinterpret_cast(control_device_) + offsetof(PmuOwnerControl, register_bases); + } + + const PmuOwnerControl &Control() const { return control_; } + + bool IsConfiguredSubcore(uint32_t index) const + { + return ready_ && IsConfigured(control_, index); + } + +private: + bool HasResources() const + { + return owner_stream_ != nullptr || control_device_ != nullptr || loader_.IsInitialized(); + } + + bool RunCommand(PmuOwnerMainCommand command, const char *label) + { + const PmuOwnerMainKernelArgs arguments = MakePmuOwnerMainKernelArgs( + reinterpret_cast(control_device_), command, device_ + ); + const std::string sync_label = std::string("aclrtSynchronizeStream(PMU ") + label + ")"; + const std::string copy_label = std::string("aclrtMemcpy(D2H PMU ") + label + ")"; + if (loader_.Launch(owner_stream_, const_cast(&arguments), sizeof(arguments)) != 0 || + !OwnerCheckAcl(aclrtSynchronizeStream(owner_stream_), sync_label.c_str()) || + !OwnerCheckAcl( + aclrtMemcpy( + &control_, sizeof(control_), control_device_, sizeof(control_), ACL_MEMCPY_DEVICE_TO_HOST + ), + copy_label.c_str() + )) { + return false; + } + return true; + } + + void PrintControl(const char *command, uint32_t bitmap_count) const + { + const auto failed_field = static_cast(control_.first_failed_field); + const auto restore_field = static_cast(control_.first_restore_failed_field); + std::printf( + "[PMU_OWNER] command=%s status=%d configured=%u active=%u/%u/%u " + "discovered=%u/%u/%u bitmap=%u skipped=%u first_failed=%u:%s(%u):0x%x/0x%x " + "restore_failures=%u first_restore=%u:%s(%u):0x%x/0x%x\n", + command, static_cast(control_.status), control_.configured, + control_.active_total, control_.active_aic, control_.active_aiv, + control_.discovered_total, control_.discovered_aic, control_.discovered_aiv, + bitmap_count, control_.skipped_total, control_.first_failed_index, + OwnerFieldName(failed_field), control_.first_failed_field, + control_.first_failed_observed, control_.first_failed_expected, + control_.restore_failures, control_.first_restore_failed_index, + OwnerFieldName(restore_field), control_.first_restore_failed_field, + control_.first_restore_failed_observed, control_.first_restore_failed_expected + ); + std::printf( + "[PMU_OWNER] configured_bitmap=%08x:%08x:%08x:%08x\n", + control_.configured_bitmap[3], control_.configured_bitmap[2], + control_.configured_bitmap[1], control_.configured_bitmap[0] + ); + } + + uint32_t device_ = 0U; + aclrtStream owner_stream_ = nullptr; + MainAicpuLoader loader_; + void *control_device_ = nullptr; + PmuOwnerControl control_{}; + ActiveSubcoreLimits limits_{}; + bool ready_ = false; + bool configured_ = false; +}; + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_HOST_H_ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_owner_main_abi.h b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_owner_main_abi.h new file mode 100644 index 0000000000..d6ba9e2239 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_owner_main_abi.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_ABI_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_ABI_H_ + +#include +#include + +namespace pa_scheduler::pmu_owner { + +// 主 aicpu_scheduler 的统一入口 simpler_aicpu_exec 根据该命令选择配置或 +// 恢复。0 特意保留为 Invalid，避免零初始化参数意外改写 PMU 寄存器。 +enum class PmuOwnerMainCommand : uint32_t { + Invalid = 0U, + Configure = 1U, + Restore = 2U, +}; + +// 该结构逐字段复刻 A5 KernelArgs 的 152B ABI，但只使用固定宽度整数，因而 +// 不依赖 Simpler 的 DeviceArgs/Runtime C++ 类型。runtime_args_device 指向 +// PmuOwnerControl；command 位于原 enable_profiling_flag 的 offset 128。 +// 其余字段保持为零，既满足主 aicpu_scheduler 固定布局，也不引入外部依赖。 +struct PmuOwnerMainKernelArgs { + uint64_t unused[5]; // 0..39 + uint64_t device_args_device; // 40，当前 owner 不使用 + uint64_t runtime_args_device; // 48，PmuOwnerControl 的 GM 地址 + uint64_t register_bases_device; // 56，当前 control 已内嵌基址，保持为零 + uint64_t dump_data_base; // 64 + uint64_t l2_swimlane_data_base; // 72 + uint64_t pmu_data_base; // 80 + uint64_t dep_gen_data_base; // 88 + uint64_t l2_swimlane_rotation_table; // 96 + uint64_t aicore_pmu_ring_addrs; // 104 + uint64_t scope_stats_data_base; // 112 + uint32_t log_level; // 120 + uint32_t log_info_v; // 124 + uint32_t command; // 128，PmuOwnerMainCommand + uint32_t reserved_alignment; // 132 + uint64_t device_wall_data_base; // 136 + uint32_t device_id; // 144 + uint32_t force_simt_anchor; // 148 +}; + +static_assert(sizeof(PmuOwnerMainCommand) == sizeof(uint32_t), "PMU owner command ABI changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, device_args_device) == 40U, "device args offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, runtime_args_device) == 48U, "control pointer offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, register_bases_device) == 56U, "register pointer offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, command) == 128U, "PMU owner command offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, device_id) == 144U, "device id offset changed"); +static_assert(sizeof(PmuOwnerMainKernelArgs) == 152U, "main aicpu_scheduler KernelArgs ABI changed"); +static_assert(alignof(PmuOwnerMainKernelArgs) == 8U, "KernelArgs alignment changed"); + +inline PmuOwnerMainKernelArgs MakePmuOwnerMainKernelArgs( + uint64_t control_device, PmuOwnerMainCommand command, uint32_t device_id +) +{ + PmuOwnerMainKernelArgs arguments{}; + arguments.runtime_args_device = control_device; + arguments.command = static_cast(command); + arguments.device_id = device_id; + return arguments; +} + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_ABI_H_ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_owner_main_loader.h b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_owner_main_loader.h new file mode 100644 index 0000000000..ea34f5fe59 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_owner_main_loader.h @@ -0,0 +1,389 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_LOADER_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_LOADER_H_ + +// PMU owner 的自包含 host 装载器： +// 1. 通过 libaicpu_extend_kernels bootstrap 临时 dispatcher； +// 2. dispatcher 将 owner SO 落到主 aicpu_scheduler 的预安装目录； +// 3. 用 cpuKernelMode=0 JSON 注册 owner 的 simpler_aicpu_exec； +// 4. 后续 Configure/Restore 都用缓存的 rtFuncHandle 直接下发。 +// +// 本头文件故意不定义 owner 命令字段。Launch 接受调用方构造的完整参数块， +// 从而让装载 ABI 与 PMU 状态机 ABI 解耦，也便于先独立验证 Path-A。 + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "runtime/rt.h" +#include "runtime/runtime/rts/rts_kernel.h" + +namespace pa_scheduler::pmu_owner { + +class MainAicpuLoader { +public: + MainAicpuLoader() = default; + MainAicpuLoader(const MainAicpuLoader &) = delete; + MainAicpuLoader &operator=(const MainAicpuLoader &) = delete; + MainAicpuLoader(MainAicpuLoader &&) = delete; + MainAicpuLoader &operator=(MainAicpuLoader &&) = delete; + + ~MainAicpuLoader() { (void)Finalize(); } + + // stream 必须属于当前 device，并且调用期间当前 ACL device 不能切换。 + // 成功后 owner SO 已注册到主 aicpu_scheduler，但尚未执行任何 PMU 命令。 + int Initialize( + const std::string &dispatcher_so_path, const std::string &owner_so_path, + aclrtStream stream, int32_t device_id + ) + { + if (IsInitialized() || stream == nullptr || device_id < 0) { + return Fail("Initialize received invalid state, stream, or device id", kInvalidArgument); + } + + const std::vector dispatcher = ReadBinary(dispatcher_so_path); + const std::vector owner = ReadBinary(owner_so_path); + if (dispatcher.empty() || owner.empty()) { + std::fprintf( + stderr, "[PMU_OWNER_LOADER] cannot read dispatcher/owner: %s (%zu B), %s (%zu B)\n", + dispatcher_so_path.c_str(), dispatcher.size(), owner_so_path.c_str(), owner.size() + ); + return kFileError; + } + + device_id_ = device_id; + owner_fingerprint_ = FingerprintBytes(owner.data(), owner.size()); + owner_so_basename_ = MakeOwnerSoBasename(owner_fingerprint_, device_id_); + op_type_ = MakeOpType(owner_fingerprint_, device_id_); + + int result = Bootstrap(dispatcher, owner, stream); + if (result == 0) result = RegisterOwner(); + if (result != 0) { + (void)Finalize(); + return result; + } + return 0; + } + + // 参数块由 runtime 在 launch 时复制；调用方只需保证本函数返回前 host + // buffer 有效。参数中的 GM 指针仍必须在设备命令同步结束前保持有效。 + int Launch( + aclrtStream stream, void *kernel_arguments, size_t argument_bytes, + uint32_t aicpu_blocks = 1U + ) const + { + if (!IsInitialized() || stream == nullptr || kernel_arguments == nullptr || + argument_bytes == 0U || argument_bytes > std::numeric_limits::max() || + aicpu_blocks == 0U) { + return Fail("Launch received invalid state or arguments", kInvalidArgument); + } + + rtCpuKernelArgs_t cpu_arguments = {}; + cpu_arguments.baseArgs.args = kernel_arguments; + cpu_arguments.baseArgs.argsSize = static_cast(argument_bytes); + rtKernelLaunchCfg_t launch_config = {}; + rtLaunchKernelAttr_t launch_attribute = {}; + launch_config.attrs = &launch_attribute; + launch_config.numAttrs = 0U; + + const rtError_t result = rtsLaunchCpuKernel( + function_handle_, aicpu_blocks, static_cast(stream), + &launch_config, &cpu_arguments + ); + if (result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsLaunchCpuKernel failed: %d\n", result); + } + return static_cast(result); + } + + // Finalize 只释放 host/runtime 注册资源，不删除设备侧预安装 SO；后者按内容 + // 指纹命名，可由同一设备上的后续进程原子覆盖。 + int Finalize() + { + int result = 0; + function_handle_ = nullptr; + if (binary_handle_ != nullptr) { + const rtError_t unload_result = rtsBinaryUnload(binary_handle_); + if (unload_result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsBinaryUnload failed: %d\n", unload_result); + result = static_cast(unload_result); + } + binary_handle_ = nullptr; + } + if (!json_path_.empty()) { + if (std::remove(json_path_.c_str()) != 0 && result == 0) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] remove JSON failed: %s\n", json_path_.c_str()); + result = kFileError; + } + json_path_.clear(); + } + device_id_ = -1; + owner_fingerprint_ = 0U; + owner_so_basename_.clear(); + op_type_.clear(); + return result; + } + + bool IsInitialized() const { return binary_handle_ != nullptr && function_handle_ != nullptr; } + uint64_t OwnerFingerprint() const { return owner_fingerprint_; } + const std::string &OwnerSoBasename() const { return owner_so_basename_; } + const std::string &OpType() const { return op_type_; } + +private: + static constexpr int kInvalidArgument = -1; + static constexpr int kFileError = -2; + static constexpr int kBootstrapError = -3; + static constexpr uint64_t kFnvOffsetBasis = UINT64_C(14695981039346656037); + static constexpr uint64_t kFnvPrime = UINT64_C(1099511628211); + static constexpr const char *kOwnerFunction = "simpler_aicpu_exec"; + + struct DeviceBuffer { + void *address = nullptr; + DeviceBuffer() = default; + DeviceBuffer(const DeviceBuffer &) = delete; + DeviceBuffer &operator=(const DeviceBuffer &) = delete; + ~DeviceBuffer() + { + if (address != nullptr) (void)aclrtFree(address); + } + aclError Allocate(size_t bytes) + { + return aclrtMalloc(&address, bytes, ACL_MEM_MALLOC_HUGE_FIRST); + } + }; + + static int Fail(const char *message, int code) + { + std::fprintf(stderr, "[PMU_OWNER_LOADER] %s\n", message); + return code; + } + + static std::vector ReadBinary(const std::string &path) + { + std::ifstream input(path, std::ios::binary | std::ios::ate); + if (!input.is_open()) return {}; + const std::streampos end = input.tellg(); + if (end <= std::streampos(0) || + static_cast(end) > static_cast(std::numeric_limits::max())) { + return {}; + } + std::vector bytes(static_cast(end)); + input.seekg(0, std::ios::beg); + if (!input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size()))) { + return {}; + } + return bytes; + } + + static uint64_t FingerprintBytes(const void *data, size_t bytes) + { + const auto *input = static_cast(data); + uint64_t hash = kFnvOffsetBasis; + for (size_t index = 0U; index < bytes; ++index) { + hash ^= input[index]; + hash *= kFnvPrime; + } + return hash; + } + + static std::string MakeOwnerSoBasename(uint64_t fingerprint, int32_t device_id) + { + char name[128] = {}; + (void)snprintf( + name, sizeof(name), "pa_scheduler_pmu_owner_%016llx_d%d.so", + static_cast(fingerprint), device_id + ); + return name; + } + + static std::string MakeOpType(uint64_t fingerprint, int32_t device_id) + { + char name[160] = {}; + (void)snprintf( + name, sizeof(name), "pa_scheduler_pmu_owner_%016llx_d%d", + static_cast(fingerprint), device_id + ); + return name; + } + + int Bootstrap( + const std::vector &dispatcher, const std::vector &owner, + aclrtStream stream + ) const + { + DeviceBuffer dispatcher_device; + DeviceBuffer owner_device; + DeviceBuffer device_args; + aclError acl_result = dispatcher_device.Allocate(dispatcher.size()); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMalloc(dispatcher)", acl_result); + acl_result = aclrtMemcpy( + dispatcher_device.address, dispatcher.size(), dispatcher.data(), dispatcher.size(), + ACL_MEMCPY_HOST_TO_DEVICE + ); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMemcpy(dispatcher H2D)", acl_result); + + acl_result = owner_device.Allocate(owner.size()); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMalloc(owner)", acl_result); + acl_result = aclrtMemcpy( + owner_device.address, owner.size(), owner.data(), owner.size(), ACL_MEMCPY_HOST_TO_DEVICE + ); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMemcpy(owner H2D)", acl_result); + + constexpr size_t kDeviceArgsBytes = 160U; + uint8_t host_device_args[kDeviceArgsBytes] = {}; + const auto write_qword = [&](size_t offset, uint64_t value) { + std::memcpy(host_device_args + offset, &value, sizeof(value)); + }; + write_qword(96U, reinterpret_cast(dispatcher_device.address)); + write_qword(104U, static_cast(dispatcher.size())); + write_qword(112U, static_cast(device_id_)); + write_qword(120U, reinterpret_cast(owner_device.address)); + write_qword(128U, static_cast(owner.size())); + + acl_result = device_args.Allocate(kDeviceArgsBytes); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMalloc(bootstrap args)", acl_result); + acl_result = aclrtMemcpy( + device_args.address, kDeviceArgsBytes, host_device_args, kDeviceArgsBytes, + ACL_MEMCPY_HOST_TO_DEVICE + ); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMemcpy(bootstrap args H2D)", acl_result); + + // k_args 总长和三个字符串 offset 与仓内已上板的 Path-A 完全一致。 + struct BootstrapArguments { + struct { + uint64_t unused[5]; + uint64_t device_args_address; + uint64_t padding[20]; + } kernel_args; + char kernel_name[32]; + char so_name[32]; + char op_name[32]; + } arguments = {}; + static_assert(offsetof(BootstrapArguments, kernel_args.device_args_address) == 40U, "bootstrap ABI changed"); + arguments.kernel_args.device_args_address = reinterpret_cast(device_args.address); + constexpr char kBootstrapKernel[] = "DynTileFwkKernelServerInit"; + constexpr char kBootstrapSo[] = "libaicpu_extend_kernels.so"; + static_assert(sizeof(kBootstrapKernel) <= sizeof(arguments.kernel_name), "bootstrap kernel name too long"); + static_assert(sizeof(kBootstrapSo) <= sizeof(arguments.so_name), "bootstrap SO name too long"); + std::memcpy(arguments.kernel_name, kBootstrapKernel, sizeof(kBootstrapKernel)); + std::memcpy(arguments.so_name, kBootstrapSo, sizeof(kBootstrapSo)); + + rtAicpuArgsEx_t runtime_arguments = {}; + runtime_arguments.args = &arguments; + runtime_arguments.argsSize = sizeof(arguments); + runtime_arguments.kernelNameAddrOffset = offsetof(BootstrapArguments, kernel_name); + runtime_arguments.soNameAddrOffset = offsetof(BootstrapArguments, so_name); + + const rtError_t launch_result = rtAicpuKernelLaunchExWithArgs( + rtKernelType_t::KERNEL_TYPE_AICPU_KFC, "AST_DYN_AICPU", 1U, + &runtime_arguments, nullptr, static_cast(stream), 0U + ); + if (launch_result != RT_ERROR_NONE) { + std::fprintf( + stderr, "[PMU_OWNER_LOADER] rtAicpuKernelLaunchExWithArgs failed: %d\n", + launch_result + ); + return static_cast(launch_result); + } + acl_result = aclrtSynchronizeStream(stream); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtSynchronizeStream(bootstrap)", acl_result); + return 0; + } + + int RegisterOwner() + { + char path[256] = {}; + (void)snprintf( + path, sizeof(path), "/tmp/pa_scheduler_pmu_owner_%016llx_d%d_p%d_i%016llx.json", + static_cast(owner_fingerprint_), device_id_, static_cast(getpid()), + static_cast(reinterpret_cast(this)) + ); + json_path_ = path; + if (!WriteJson()) return kFileError; + + rtLoadBinaryOption_t option = {}; + option.optionId = RT_LOAD_BINARY_OPT_CPU_KERNEL_MODE; + option.value.cpuKernelMode = 0; + rtLoadBinaryConfig_t configuration = {}; + configuration.options = &option; + configuration.numOpt = 1U; + + rtError_t result = rtsBinaryLoadFromFile(json_path_.c_str(), &configuration, &binary_handle_); + if (result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsBinaryLoadFromFile failed: %d\n", result); + return static_cast(result); + } + result = rtsFuncGetByName(binary_handle_, op_type_.c_str(), &function_handle_); + if (result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsFuncGetByName(%s) failed: %d\n", op_type_.c_str(), result); + return static_cast(result); + } + return 0; + } + + bool WriteJson() const + { + std::ofstream json(json_path_, std::ios::out | std::ios::trunc); + if (!json.is_open()) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] cannot create JSON: %s\n", json_path_.c_str()); + return false; + } + // 所有动态字段仅含固定前缀、十六进制、十进制和下划线，不需要 JSON 转义。 + json << "{\n" + << " \"" << op_type_ << "\": {\n" + << " \"opInfo\": {\n" + << " \"functionName\": \"" << kOwnerFunction << "\",\n" + << " \"kernelSo\": \"" << owner_so_basename_ << "\",\n" + << " \"opKernelLib\": \"AICPUKernel\",\n" + << " \"computeCost\": \"100\",\n" + << " \"engine\": \"DNN_VM_AICPU\",\n" + << " \"flagAsync\": \"False\",\n" + << " \"flagPartial\": \"False\",\n" + << " \"userDefined\": \"False\"\n" + << " }\n" + << " }\n" + << "}\n"; + json.close(); + if (!json) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] writing JSON failed: %s\n", json_path_.c_str()); + return false; + } + return true; + } + + static int ReportAcl(const char *operation, aclError result) + { + std::fprintf(stderr, "[PMU_OWNER_LOADER] %s failed: %d\n", operation, static_cast(result)); + return static_cast(result == ACL_SUCCESS ? kBootstrapError : result); + } + + int32_t device_id_ = -1; + uint64_t owner_fingerprint_ = 0U; + std::string owner_so_basename_; + std::string op_type_; + std::string json_path_; + rtBinHandle binary_handle_ = nullptr; + rtFuncHandle function_handle_ = nullptr; +}; + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_LOADER_H_ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_probe.h b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_probe.h new file mode 100644 index 0000000000..a2e8cab6a0 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/ccec/pmu_probe.h @@ -0,0 +1,185 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_PROBE_H +#define PA_SCHEDULER_CCEC_PMU_PROBE_H + +#include + +#include "../common/pa_model.h" + +namespace pa_scheduler::ccec_pmu { + +// Empty/Scalar/ScalarDouble 在调度结束后校准门控底噪和 scalar 正向响应； +// IcacheSingle 在每核上成对累计隔离的 cold/warm 目标调用；SubmitAll 则在 +// 公共调度器 hook 内覆盖本 worker 的完整 Submit 回放窗口。 +enum class WindowMode : uint32_t { + Off = 0, + Empty = 1, + Scalar = 2, + ScalarDouble = 3, + // 保留已落远端的 I-cache 校准模式值，避免 standalone host/kernel 混用旧产物时 + // 把校准请求误解释成 Submit 窗口；新增模式只在枚举尾部扩展。 + IcacheSingle = 4, + // SubmitAll 从本 worker 的 orchestration/Submit 回放前开始，到最后一次 + // Submit 返回后停止。 + SubmitAll = 5, +}; + +inline bool IsSubmitWindow(WindowMode mode) { + return mode == WindowMode::SubmitAll; +} + +// RunConfig::reserved 保持既有 64B ABI；CCEC 独占解释以下五个槽位，其他后端仍看到全零。 +constexpr uint32_t kConfigMode = 0; +constexpr uint32_t kConfigWorkAmount = 1; +constexpr uint32_t kConfigScalarNops = kConfigWorkAmount; +constexpr uint32_t kConfigIcacheTrials = kConfigWorkAmount; +constexpr uint32_t kConfigRegTableLow = 2; +constexpr uint32_t kConfigRegTableHigh = 3; +constexpr uint32_t kConfigMagic = 4; +constexpr uint32_t kConfigMagicValue = 0x504d5531U; // "PMU1" + +// DAV_3510 有 36 个物理 AICore，每个 AICore 展开为 1 AIC + 2 AIV，共 108 个物理子核编号。 +constexpr uint32_t kPhysicalAicoreCount = 36; +constexpr uint32_t kPhysicalSubcoreCount = 108; +constexpr uint32_t kAicorePerDie = 18; +constexpr uint32_t kSubcoresPerDie = 54; +constexpr uint64_t kAivFirstOffset = 0x100000ULL; +constexpr uint64_t kAivSecondOffset = 0x200000ULL; +constexpr uint32_t kAicoreMapBytes = 0x300000U; + +// PIPE_UTILIZATION 事件由 standalone Main AICPU owner 配置，kernel 逐核读回并核对 selector。 +constexpr uint32_t kScalarBusyEvent = 0x1U; +constexpr uint32_t kIcacheRequestEvent = 0x34U; +constexpr uint32_t kIcacheMissEvent = 0x35U; +constexpr uint32_t kVectorBusyEvent = 0x501U; +constexpr uint32_t kCubeBusyEvent = 0x301U; +constexpr uint32_t kMte1BusyEvent = 0x701U; +constexpr uint32_t kMte2BusyEvent = 0x202U; +constexpr uint32_t kMte3BusyEvent = 0x203U; +constexpr uint32_t kFixBusyEvent = 0x714U; + +// DAV_3510 PMU MMIO offset。ld_dev 的立即数只有 12 bit，因此 kernel 会分别重基址到 0x2400/0x4200。 +constexpr uint32_t kSelectorBlockOffset = 0x2400U; +constexpr uint32_t kCounterBlockOffset = 0x4200U; +constexpr uint32_t kCnt2Offset = 0x4220U; +constexpr uint32_t kCnt0Offset = 0x4210U; +constexpr uint32_t kCnt1Offset = 0x4218U; +constexpr uint32_t kCnt3Offset = 0x4228U; +constexpr uint32_t kCnt4Offset = 0x4230U; +constexpr uint32_t kCnt5Offset = 0x4238U; +constexpr uint32_t kCnt6Offset = 0x4240U; +constexpr uint32_t kCnt7Offset = 0x4248U; +constexpr uint32_t kCnt8Offset = 0x4250U; +constexpr uint32_t kCnt9Offset = 0x4254U; +constexpr uint32_t kTotalLowOffset = 0x4260U; +constexpr uint32_t kTotalHighOffset = 0x4264U; +constexpr uint32_t kCnt2SelectorOffset = 0x2508U; +constexpr uint32_t kCnt0SelectorOffset = 0x2500U; +constexpr uint32_t kCnt1SelectorOffset = 0x2504U; +constexpr uint32_t kCnt3SelectorOffset = 0x250cU; +constexpr uint32_t kCnt4SelectorOffset = 0x2510U; +constexpr uint32_t kCnt5SelectorOffset = 0x2514U; +constexpr uint32_t kCnt6SelectorOffset = 0x2518U; +constexpr uint32_t kCnt7SelectorOffset = 0x251cU; +constexpr uint32_t kCnt8SelectorOffset = 0x2520U; +constexpr uint32_t kCnt9SelectorOffset = 0x2524U; + +// pmu_status 的 bits16..27 保存 get_coreid()；bits28..31 留给不参与 core id +// 解码的模式诊断。其余低位描述本条记录是否可信。 +constexpr uint32_t kStatusRequested = 1U << 0; +constexpr uint32_t kStatusRegMapped = 1U << 1; +constexpr uint32_t kStatusCoreIdValid = 1U << 2; +constexpr uint32_t kStatusCnt2Selector = 1U << 3; +constexpr uint32_t kStatusCnt6Selector = 1U << 4; +constexpr uint32_t kStatusCnt7Selector = 1U << 5; +constexpr uint32_t kStatusWindowStarted = 1U << 6; +constexpr uint32_t kStatusTotalNonzero = 1U << 7; +constexpr uint32_t kStatusPriorSnapshotLarger = 1U << 8; +constexpr uint32_t kStatusCnt0Selector = 1U << 9; +constexpr uint32_t kStatusCnt1Selector = 1U << 10; +constexpr uint32_t kStatusCnt3Selector = 1U << 11; +constexpr uint32_t kStatusCnt4Selector = 1U << 12; +constexpr uint32_t kStatusCnt5Selector = 1U << 13; +constexpr uint32_t kStatusCnt8Selector = 1U << 14; +constexpr uint32_t kStatusWindowStopped = 1U << 15; +// I-cache 配对标志不能复用 Submit start bit；StatusCoreId 只取 12 bit，故将 +// 它放在 core-id 区间之上的独立诊断位。 +constexpr uint32_t kStatusIcachePairObserved = 1U << 28; +constexpr uint32_t kStatusRequired = kStatusRequested | kStatusRegMapped | kStatusCoreIdValid | + kStatusCnt2Selector | kStatusCnt6Selector | kStatusCnt7Selector | + kStatusCnt0Selector | kStatusCnt1Selector | kStatusCnt3Selector | + kStatusCnt4Selector | kStatusCnt5Selector | kStatusCnt8Selector | + kStatusWindowStarted | kStatusWindowStopped | kStatusTotalNonzero; +constexpr uint32_t kStatusCoreIdShift = 16; +constexpr uint32_t kStatusCoreIdMask = 0x0fffU; + +// pmu_phase_status 独立于旧 pmu_status，避免与其中的物理 core-id 位域 +// 冲突。bits4/5 只记录 shadow 是否恰好等于 primary：phase=none 没有 +// 运行中 read-to-clear，host 会要求两位都成立；局部 phase 会在计数仍开启时 +// 读取 shadow，A5 实测存在同周期递增与读清竞争，因此不能把“逐次严格相等” +// 作为可信记录的共同必选位。局部 phase 的方向和误差包络由 host/raw 独立校验。 +constexpr uint32_t kPhaseStatusRequested = 1U << 0; +constexpr uint32_t kPhaseStatusShadowSelectors = 1U << 1; +constexpr uint32_t kPhaseStatusWindowStarted = 1U << 2; +constexpr uint32_t kPhaseStatusWindowStopped = 1U << 3; +constexpr uint32_t kPhaseStatusShadowRequestsMatch = 1U << 4; +constexpr uint32_t kPhaseStatusShadowMissesMatch = 1U << 5; +constexpr uint32_t kPhaseStatusBoundariesBalanced = 1U << 6; +constexpr uint32_t kPhaseStatusValuesOrdered = 1U << 7; +constexpr uint32_t kPhaseStatusUint32Fit = 1U << 8; +constexpr uint32_t kPhaseStatusPhaseShape = 1U << 9; +// none 必须保持 0 tick；运行阶段则必须确实累计到非零 SYS_CNT。阶段时间是否 +// 不超过同核首 Submit 到末 Submit 的完整区间，由拿到两端结果的 host 再校验。 +constexpr uint32_t kPhaseStatusTimeValid = 1U << 10; +constexpr uint32_t kPhaseStatusRequired = + kPhaseStatusRequested | kPhaseStatusShadowSelectors | + kPhaseStatusWindowStarted | kPhaseStatusWindowStopped | + kPhaseStatusBoundariesBalanced | kPhaseStatusValuesOrdered | + kPhaseStatusUint32Fit | kPhaseStatusPhaseShape | kPhaseStatusTimeValid; + +inline const char *SubmitPmuPhaseName(SubmitPmuPhase phase) { + switch (phase) { + case SubmitPmuPhase::None: + return "none"; + case SubmitPmuPhase::Claim: + return "claim"; + case SubmitPmuPhase::EfDrain: + return "efdrain"; + case SubmitPmuPhase::Materialize: + return "materialize"; + case SubmitPmuPhase::Register: + return "register"; + case SubmitPmuPhase::Count: + break; + } + return "invalid"; +} + +inline uint64_t PackPointer(const uint32_t *words) { + return static_cast(words[kConfigRegTableLow]) | + (static_cast(words[kConfigRegTableHigh]) << 32); +} + +inline void StorePointer(uint32_t *words, const void *pointer) { + const uint64_t raw = reinterpret_cast(pointer); + words[kConfigRegTableLow] = static_cast(raw); + words[kConfigRegTableHigh] = static_cast(raw >> 32); +} + +inline uint32_t StatusCoreId(uint32_t status) { + return (status >> kStatusCoreIdShift) & kStatusCoreIdMask; +} + +} // namespace pa_scheduler::ccec_pmu + +#endif // PA_SCHEDULER_CCEC_PMU_PROBE_H diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/common/host_support.h b/tests/atomic_probe/lazy_lamda_sample/A_original/common/host_support.h new file mode 100644 index 0000000000..1185536866 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/common/host_support.h @@ -0,0 +1,1609 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_HOST_SUPPORT_H +#define PA_SCHEDULER_COMMON_HOST_SUPPORT_H + +#include "pa_model.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pa_scheduler::host { + +// 三种后端共用同一套命令行配置，保证 CPU 语义回归与 A5 上板使用完全相同的工作量。 +struct Options { + std::string kernel_path; + std::string swimlane_json; + uint32_t device = 0; + uint32_t batches = kDefaultBatches; + uint32_t runs = 5; + NopCounts nops{kDefaultQkNops, kDefaultSfNops, kDefaultPvNops, kDefaultUpNops}; + bool profile_phases = false; + bool trace_enabled = true; + bool trace_atomics = false; + bool analyze_swimlane = false; +}; + +enum class ParseStatus { + Ok, + Help, + Error, +}; + +inline bool ParseUint(const char *raw, uint32_t minimum, uint32_t maximum, uint32_t *value) { + // 要求整串都能被 strtoul 解析且结果落在给定范围内，拒绝尾随字符和溢出值， + // 避免参数被部分解析后悄悄改变工作量。 + errno = 0; + char *end = nullptr; + const unsigned long parsed = std::strtoul(raw, &end, 10); + if (errno != 0 || end == raw || *end != '\0' || parsed < minimum || parsed > maximum) { + return false; + } + *value = static_cast(parsed); + return true; +} + +inline bool ParseNopCounts(const char *raw, NopCounts *counts) { + // 四类 kernel 的 NOP 数必须一次性完整给出，顺序固定为 QK、SF、PV、UP。 + unsigned int qk = 0; + unsigned int sf = 0; + unsigned int pv = 0; + unsigned int up = 0; + char tail = '\0'; + if (std::sscanf(raw, "%u,%u,%u,%u%c", &qk, &sf, &pv, &up, &tail) != 4) { + return false; + } + constexpr uint32_t kMaxNopCount = 10000000; + if (qk > kMaxNopCount || sf > kMaxNopCount || pv > kMaxNopCount || up > kMaxNopCount) { + return false; + } + *counts = NopCounts{qk, sf, pv, up}; + return true; +} + +inline void PrintUsage(const char *program, bool require_kernel) { + // require_kernel 只影响 CCEC host 的用法文本，其余 benchmark 参数在三后端完全一致。 + std::fprintf( + stderr, "Usage: %s%s [--device N] [--batches 1..256] [--runs N] ", program, + require_kernel ? " --kernel FILE" : "" + ); + std::fprintf( + stderr, + "[--nop-count N | --nop-counts QK,SF,PV,UP] [--profile-phases] [--analyze-swimlane] " + "[--trace-atomics] [--swimlane-json FILE] [--no-swimlane]\n" + ); +} + +inline ParseStatus ParseOptions(int argc, char **argv, bool require_kernel, Options *options) { + // CCEC host 需要外部 kernel ELF；AscendC 和 CPU 的可执行文件已包含 kernel，因此不需要该参数。 + bool nop_override_seen = false; + bool swimlane_json_seen = false; + for (int index = 1; index < argc; ++index) { + // 无值开关先处理；其余参数统一在消费下一个 argv 前检查缺值，保证错误位置明确。 + const std::string argument = argv[index]; + if (argument == "--help" || argument == "-h") { + PrintUsage(argv[0], require_kernel); + return ParseStatus::Help; + } + if (argument == "--profile-phases") { + options->profile_phases = true; + continue; + } + if (argument == "--no-swimlane") { + options->trace_enabled = false; + continue; + } + if (argument == "--trace-atomics") { + options->trace_atomics = true; + continue; + } + if (argument == "--analyze-swimlane") { + options->analyze_swimlane = true; + continue; + } + if (index + 1 >= argc) { + std::fprintf(stderr, "Missing value after %s\n", argument.c_str()); + return ParseStatus::Error; + } + const char *value = argv[++index]; + if (argument == "--kernel" && require_kernel) { + options->kernel_path = value; + } else if (argument == "--device") { + if (!ParseUint(value, 0, INT32_MAX, &options->device)) return ParseStatus::Error; + } else if (argument == "--batches") { + if (!ParseUint(value, 1, kMaxBatches, &options->batches)) return ParseStatus::Error; + } else if (argument == "--runs") { + if (!ParseUint(value, 1, 1000, &options->runs)) return ParseStatus::Error; + } else if (argument == "--swimlane-json") { + if (swimlane_json_seen) { + std::fprintf(stderr, "Specify --swimlane-json only once.\n"); + return ParseStatus::Error; + } + if (*value == '\0') { + std::fprintf(stderr, "--swimlane-json requires a non-empty path.\n"); + return ParseStatus::Error; + } + options->swimlane_json = value; + swimlane_json_seen = true; + } else if (argument == "--nop-count") { + if (nop_override_seen) { + std::fprintf(stderr, "Specify only one NOP override.\n"); + return ParseStatus::Error; + } + uint32_t count = 0; + if (!ParseUint(value, 0, 10000000, &count)) return ParseStatus::Error; + options->nops = NopCounts{count, count, count, count}; + nop_override_seen = true; + } else if (argument == "--nop-counts") { + if (nop_override_seen) { + std::fprintf(stderr, "Specify only one NOP override.\n"); + return ParseStatus::Error; + } + if (!ParseNopCounts(value, &options->nops)) return ParseStatus::Error; + nop_override_seen = true; + } else { + std::fprintf(stderr, "Unknown argument: %s\n", argument.c_str()); + return ParseStatus::Error; + } + } + if (require_kernel && options->kernel_path.empty()) { + std::fprintf(stderr, "--kernel is required\n"); + return ParseStatus::Error; + } + if (options->analyze_swimlane && !options->trace_enabled) { + // 分析和导出都依赖完整 record 缓冲，不能与节省内存的 --no-swimlane 同时使用。 + std::fprintf(stderr, "--analyze-swimlane requires swimlane tracing.\n"); + return ParseStatus::Error; + } + if (options->trace_atomics && !options->trace_enabled) { + std::fprintf(stderr, "--trace-atomics cannot be combined with --no-swimlane.\n"); + return ParseStatus::Error; + } + if (!options->swimlane_json.empty() && !options->trace_enabled) { + std::fprintf(stderr, "--swimlane-json requires swimlane tracing.\n"); + return ParseStatus::Error; + } + if (!options->swimlane_json.empty() && options->runs != 1) { + // 一个文件只对应一次完整采集，禁止多轮运行反复覆盖而丢失轮次边界。 + std::fprintf(stderr, "--swimlane-json requires --runs 1 to avoid overwriting captures.\n"); + return ParseStatus::Error; + } + return ParseStatus::Ok; +} + +inline void InitializeState(SchedulerState *state, const Options &options) { + // WorkerState 有意保持真实 PA 每核约 9 MiB 的布局。若 host 每轮清空全部 worker， + // 会额外触碰并拷贝近 1 GiB 内存；因此只初始化全局前缀和结果区，worker 的活跃字段 + // 由各自 kernel 在启动后复位，这也与真实 PA 的生命周期一致。 + std::memset(state, 0, offsetof(SchedulerState, workers)); + std::memset(&state->config, 0, offsetof(SchedulerState, results) - offsetof(SchedulerState, config)); + std::memset(state->results, 0, sizeof(state->results)); + state->heap_window = kHeapWindow; + state->heap_base = kSyntheticHeapBase; + state->heap_size = kHeapBytes; + state->num_workers = kWorkers; + state->num_blocks = kAicWorkers; + state->config.batches = options.batches; + state->config.workers = kWorkers; + state->config.nops = options.nops; + state->config.profile_phases = options.profile_phases ? 1U : 0U; + for (uint32_t batch = 0; batch < options.batches; ++batch) { + state->context_lens[batch] = 8192; + } + for (uint32_t shard = 0; shard < kCursorShards; ++shard) { + // -1 表示尚无 task 被 claim；task 0 的 atomicMax 因而也能正常判定唯一 winner。 + state->cube_cursor[shard].value = -1; + state->vector_cursor[shard].value = -1; + state->alloc_cursor[shard].value = -1; + } + state->frontier.value = -1; +} + +inline void ConfigureTrace(SchedulerState *state, const Options &options, const void *trace_base) { + // device 只持有裸地址和每核容量；TraceHeader/record 缓冲区由 host 单独分配并初始化。 + state->config.trace_enabled = options.trace_enabled + ? kTracePhasesEnabled | (options.trace_atomics ? kTraceAtomicsEnabled : 0U) + : 0U; + state->config.trace_base = options.trace_enabled ? reinterpret_cast(trace_base) : 0; + state->config.trace_records_per_core = options.trace_enabled ? kTraceRecordsPerCore : 0; +} + +inline void InitializeTraceHeader(TraceHeader *header) { + // version=4 表示 phase ABI 已追加父区间和真实 Submit 尾动作；core state + // 继续携带 weighted atomic/PollBatch 计数和权威拓扑。 + std::memset(header, 0, sizeof(*header)); + header->magic = 0x4653574cU; + header->version = 4; + header->num_cores = kWorkers; + header->records_per_core = kTraceRecordsPerCore; + header->frequency_hz = kSystemCounterHz; +} + +// 巨大的 WorkerState 不参与每轮 H2D/D2H；以下三个范围只搬运运行所需的前缀、控制量和结果。 +inline constexpr size_t StatePrefixBytes() { return offsetof(SchedulerState, workers); } + +inline constexpr size_t ControlBytes() { + // control sidecar 位于为生产 DistGlobal 保留的总跨度之后，到 results 之前为止。 + return offsetof(SchedulerState, results) - offsetof(SchedulerState, config); +} + +inline constexpr size_t ResultBytes() { return sizeof(WorkerResult) * kWorkers; } + +struct Metrics { + // passed 是全部语义断言的合取；submit_span_us 是本用例唯一用于对比 PA 的性能口径。 + bool passed = true; + double submit_span_us = 0; +}; + +inline void Expect(bool condition, const char *label, Metrics *metrics) { + // 所有断言都继续执行，以便一次失败运行尽可能暴露完整状态，而不是遇到首错立即退出。 + std::printf("[ASSERT] %-48s %s\n", label, condition ? "PASS" : "FAIL"); + if (!condition) metrics->passed = false; +} + +struct Uint64Distribution { + uint64_t total = 0; + double median = 0.0; + uint64_t p95 = 0; + uint64_t maximum = 0; +}; + +inline Uint64Distribution SummarizeUint64(std::vector values) { + // 这里按 worker 维度统计累计周期，p95 使用 nearest-rank，避免插值掩盖慢核。 + Uint64Distribution summary; + if (values.empty()) return summary; + + std::sort(values.begin(), values.end()); + for (uint64_t value : values) summary.total += value; + const size_t middle = values.size() / 2; + summary.median = (values.size() & 1U) != 0 + ? static_cast(values[middle]) + : (static_cast(values[middle - 1]) + static_cast(values[middle])) / 2.0; + const size_t p95_rank = (95U * values.size() + 99U) / 100U; + summary.p95 = values[p95_rank - 1]; + summary.maximum = values.back(); + return summary; +} + +inline void PrintPhaseDiagnostics(const SchedulerState &state) { + if (state.config.profile_phases == 0) return; + + // WaitForSlot/HeapGuard 没有各自独立命名的 TracePhase；实际发生等待时会写 + // RingBp 记录，汇总诊断则使用 WorkerResult 中的累计周期和等待次数。 + struct PhaseSpec { + ProfilePhase phase; + const char *name; + int32_t wait_event_index; + }; + const PhaseSpec phases[] = { + {ProfilePhase::Claim, "Claim", -1}, + {ProfilePhase::EfDrain, "EfDrain", -1}, + {ProfilePhase::WaitForSlot, "WaitForSlot", 0}, + {ProfilePhase::HeapGuard, "HeapGuard", 1}, + }; + const CoreRole roles[] = {CoreRole::Aic, CoreRole::Aiv}; + const char *role_names[] = {"AIC", "AIV"}; + + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + // AIC/AIV 分开统计，避免 32:64 的参与者数量差异掩盖某一类核上的长尾。 + for (const PhaseSpec &phase : phases) { + std::vector cycles; + std::vector calls; + std::vector wait_events; + const uint32_t phase_index = static_cast(phase.phase); + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const WorkerResult &result = state.results[worker]; + if (result.role != static_cast(roles[role_index])) continue; + cycles.push_back(result.phase_cycles[phase_index]); + calls.push_back(result.phase_calls[phase_index]); + wait_events.push_back( + phase.wait_event_index < 0 ? 0 : result.wait_events[static_cast(phase.wait_event_index)] + ); + } + const Uint64Distribution cycle_summary = SummarizeUint64(cycles); + const Uint64Distribution call_summary = SummarizeUint64(calls); + const Uint64Distribution wait_summary = SummarizeUint64(wait_events); + std::printf( + "[PHASE] role=%s phase=%s workers=%zu accumulated_us_median=%.3f " + "accumulated_us_p95=%.3f accumulated_us_max=%.3f calls_total=%llu " + "calls_per_worker_median=%.1f calls_per_worker_p95=%llu calls_per_worker_max=%llu " + "wait_events_total=%llu wait_events_per_worker_median=%.1f " + "wait_events_per_worker_p95=%llu wait_events_per_worker_max=%llu\n", + role_names[role_index], phase.name, cycles.size(), cycle_summary.median / 1000.0, + static_cast(cycle_summary.p95) / 1000.0, + static_cast(cycle_summary.maximum) / 1000.0, + static_cast(call_summary.total), call_summary.median, + static_cast(call_summary.p95), + static_cast(call_summary.maximum), + static_cast(wait_summary.total), wait_summary.median, + static_cast(wait_summary.p95), + static_cast(wait_summary.maximum) + ); + } + } +} + +inline const char *TracePhaseName(uint32_t phase) { + // 名称必须与 l2_swimlane_records.json 的 fdwic_events schema 保持一致。 + const char *names[] = { + "Kernel", "Alloc", "Build", "DrainWon", "Replay", "RingBp", "EfDrain", "Commit", + "Submit", "Materialize", "PrepareMap", "Claim", "Fanin", "Register", "Atomic", + "ClockBaseline", "OrchestrationReplay", "FinalDrain", "WinnerBuild", + "AllocComplete", + }; + static_assert( + sizeof(names) / sizeof(names[0]) == static_cast(TracePhase::Count), + "TracePhaseName must cover every trace phase" + ); + return phase < sizeof(names) / sizeof(names[0]) ? names[phase] : "Unknown"; +} + +inline const char *AtomicSiteName(uint32_t site) { + // 顺序与 pa_model.h::AtomicSite 的稳定 raw ABI 完全一致。 + const char *names[] = { + "StartupIncrement", "StartupPoll", "FatalPoll", "FatalSet", "ClaimMax", + "FaninFlagLoad", "CompletionVendExchange", "CompletionFlagExchange", + "FrontierInitialLoad", "FrontierFlagLoad", "FrontierMax", "HeapFrontierLoad", + "HeapVendLoad", "ReplayDoneIncrement", "ReplayDonePoll", + }; + return site < sizeof(names) / sizeof(names[0]) ? names[site] : "Unknown"; +} + +inline const char *AtomicOpName(uint32_t op) { + const char *names[] = {"Load", "Exchange", "FetchAdd", "FetchMax"}; + return op < sizeof(names) / sizeof(names[0]) ? names[op] : "Unknown"; +} + +inline AtomicOp AtomicSiteOp(AtomicSite site) { + return AtomicSiteExpectedOp(site); +} + +inline bool ValidateTraceHeader(const TraceHeader &header, const char *operation) { + // 在任何 D2H record 搬运前先验证容量和 dropped，防止损坏 header 导致 scratch 越界或导出残缺泳道。 + // 频率也要求精确为 1 GHz，否则后续 ns/us 换算即使 JSON 合法也没有性能意义。 + const bool valid = header.magic == 0x4653574cU && header.version == 4 && + header.num_cores == kWorkers && header.records_per_core == kTraceRecordsPerCore && + header.frequency_hz == kSystemCounterHz; + bool core_states_valid = true; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const TraceCoreState &core = header.cores[worker]; + core_states_valid &= core.count <= kTraceRecordsPerCore; + core_states_valid &= core.dropped == 0; + core_states_valid &= core.poll_calls <= core.atomic_calls; + core_states_valid &= (core.poll_calls == 0) == (core.poll_batch_records == 0); + const uint64_t physical_atomic = + static_cast(core.atomic_calls) - core.poll_calls + core.poll_batch_records; + core_states_valid &= physical_atomic <= core.count; + } + if (!valid || !core_states_valid) { + std::fprintf( + stderr, + "%s rejected an invalid trace header: magic=0x%08x version=%u cores=%u " + "records_per_core=%u frequency_hz=%llu core_states_valid=%s\n", + operation, header.magic, header.version, header.num_cores, header.records_per_core, + static_cast(header.frequency_hz), core_states_valid ? "yes" : "no" + ); + } + return valid && core_states_valid; +} + +struct TraceExportSummary { + uint64_t records = 0; + uint64_t atomic_records = 0; + uint64_t clock_baseline_records = 0; + uint64_t atomic_calls = 0; + uint64_t poll_calls = 0; + uint64_t poll_batch_records = 0; + uint64_t dropped_records = 0; +}; + +inline bool SameTraceSummary(const TraceExportSummary &left, const TraceExportSummary &right) { + return left.records == right.records && left.atomic_records == right.atomic_records && + left.clock_baseline_records == right.clock_baseline_records && + left.atomic_calls == right.atomic_calls && left.poll_calls == right.poll_calls && + left.poll_batch_records == right.poll_batch_records && + left.dropped_records == right.dropped_records; +} + +inline uint32_t AtomicRecordCallCount(const TraceRecord &record) { + return (record.flags & kAtomicPollBatch) != 0 + ? record.flags >> kAtomicPollCountShift + : 1U; +} + +inline bool AtomicRecordSchemaValid(const TraceRecord &record, bool atomic_trace_enabled) { + if (!atomic_trace_enabled || record.auxiliary >= static_cast(AtomicSite::Count)) { + return false; + } + const AtomicSite site = static_cast(record.auxiliary); + const uint32_t op = record.flags & kAtomicOpMask; + if (op != static_cast(AtomicSiteExpectedOp(site))) return false; + + const bool result_used = (record.flags & kAtomicResultUsed) != 0; + const bool value_zero = (record.flags & kAtomicValueZero) != 0; + const bool return_ready = (record.flags & kAtomicReturnReady) != 0; + const bool poll_batch = (record.flags & kAtomicPollBatch) != 0; + const uint32_t payload = record.flags >> kAtomicRetriesShift; + if (poll_batch) { + return AtomicSiteIsPollBatchable(site) && result_used && !value_zero && !return_ready && + payload > 0 && record.task_id == -1 && record.function_id == -1; + } + if (result_used != AtomicSiteResultUsed(site) || (return_ready && !result_used)) return false; + if (value_zero && op != static_cast(AtomicOp::Load)) return false; + if (payload != 0 && op != static_cast(AtomicOp::FetchMax)) return false; + return record.function_id == -1; +} + +inline bool ClockRecordSchemaValid(const TraceRecord &record) { + const bool dependency = (record.flags & kClockAtomicDependency) != 0; + const bool dependency_applied = (record.flags & kClockAtomicDependencyApplied) != 0; + return (record.flags & ~(kClockAtomicDependency | kClockAtomicDependencyApplied)) == 0 && + (!dependency_applied || dependency) && record.task_id == -1 && + record.function_id == -1 && record.auxiliary == 0; +} + +inline void ExpectedTraceTopology(uint32_t worker, int32_t *block_id, int32_t *lane) { + if (worker < kAicWorkers) { + *block_id = static_cast(worker); + *lane = 0; + return; + } + const uint32_t vector_id = worker - kAicWorkers; + *block_id = static_cast(vector_id / 2); + *lane = static_cast(1 + vector_id % 2); +} + +template +inline bool ExportSwimlaneRecords( + const TraceHeader &header, const std::string &output_path, + WinnerWorkloadMode workload_mode, const WorkloadCounts &workload_counts, + const char *workload_pattern, bool atomic_trace_enabled, ReadRecords read_records +) { + if (!ValidateTraceHeader(header, "swimlane export")) return false; + if (workload_mode != WinnerWorkloadMode::ScalarNop && + workload_mode != WinnerWorkloadMode::RealCompute) { + std::fprintf(stderr, "swimlane export rejected invalid winner workload mode.\n"); + return false; + } + const bool real_compute = workload_mode == WinnerWorkloadMode::RealCompute; + const bool pattern_valid = workload_pattern != nullptr && + ((real_compute && + (std::strcmp(workload_pattern, "constant") == 0 || + std::strcmp(workload_pattern, "layout-diagnostic") == 0)) || + (!real_compute && std::strcmp(workload_pattern, "none") == 0)); + if (!pattern_valid) { + std::fprintf(stderr, "swimlane export rejected invalid winner workload input pattern.\n"); + return false; + } + + TraceExportSummary producer_summary; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const TraceCoreState &core = header.cores[worker]; + int32_t expected_block = -1; + int32_t expected_lane = -1; + ExpectedTraceTopology(worker, &expected_block, &expected_lane); + if (core.core_idx != static_cast(worker) || core.block_id != expected_block || + core.lane != expected_lane) { + std::fprintf( + stderr, + "swimlane export rejected worker topology: worker=%u core=%d block=%d/%d lane=%d/%d\n", + worker, core.core_idx, core.block_id, expected_block, core.lane, expected_lane + ); + return false; + } + producer_summary.records += core.count; + producer_summary.atomic_calls += core.atomic_calls; + producer_summary.poll_calls += core.poll_calls; + producer_summary.poll_batch_records += core.poll_batch_records; + producer_summary.dropped_records += core.dropped; + if (!atomic_trace_enabled) { + if (core.atomic_calls != 0 || core.poll_calls != 0 || core.poll_batch_records != 0) { + std::fprintf( + stderr, + "phase-only swimlane worker %u unexpectedly reports atomic counters: calls=%u polls=%u batches=%u\n", + worker, core.atomic_calls, core.poll_calls, core.poll_batch_records + ); + return false; + } + continue; + } + if (core.poll_calls > core.atomic_calls || + (core.poll_calls == 0) != (core.poll_batch_records == 0)) { + std::fprintf( + stderr, + "atomic swimlane worker %u has invalid counters: calls=%u polls=%u batches=%u\n", + worker, core.atomic_calls, core.poll_calls, core.poll_batch_records + ); + return false; + } + producer_summary.atomic_records += + static_cast(core.atomic_calls) - core.poll_calls + core.poll_batch_records; + } + producer_summary.clock_baseline_records = atomic_trace_enabled ? 2ULL * kWorkers : 0; + + // 先写同目录临时文件，全部记录写完并关闭后再 rename 替换，避免把半截 JSON + // 当成有效采集；这里没有 fsync 文件和目录，不承诺掉电后的持久化原子性。 + const std::string temporary_path = output_path + ".tmp"; + std::FILE *output = std::fopen(temporary_path.c_str(), "wb"); + if (output == nullptr) { + std::fprintf( + stderr, "Cannot open swimlane output %s: %s\n", temporary_path.c_str(), std::strerror(errno) + ); + return false; + } + + // 采用固定 1 MiB stdio 缓冲并逐核流式写出；默认 256 batch 时约 86 万条， + // 无论实际 batch 数是多少都不在 host 侧一次性聚合全部 JSON 记录。 + std::vector output_buffer(1U << 20); + std::setvbuf(output, output_buffer.data(), _IOFBF, output_buffer.size()); + std::fprintf( + output, + "{\n\"l2_swimlane_level\":%u,\n" + "\"metadata\":{\"clock_freq_hz\":%llu,\"num_cores\":%u," + "\"trace_schema_version\":%u," + "\"winner_workload\":{\"mode\":\"%s\"," + "\"counts\":{\"qk\":%u,\"sf\":%u,\"pv\":%u,\"up\":%u}," + "\"unit\":\"%s\",\"input_pattern\":\"%s\"," + "\"engine_mapping\":%s},\"core_types\":[", + atomic_trace_enabled ? 4U : 1U, + static_cast(header.frequency_hz), kWorkers, + 4U, + workload_mode == WinnerWorkloadMode::RealCompute ? "real-compute" : "scalar-nop", + workload_counts.qk, workload_counts.sf, workload_counts.pv, workload_counts.up, + workload_mode == WinnerWorkloadMode::RealCompute + ? "complete_128x128_engine_pipeline_iteration" + : "scalar_nop_instruction", + workload_pattern, + workload_mode == WinnerWorkloadMode::RealCompute + ? "{\"qk\":\"cube_matmul\",\"sf\":\"vector_add\"," + "\"pv\":\"cube_matmul\",\"up\":\"vector_mul\"}" + : "null" + ); + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + std::fprintf(output, "%s\"%s\"", worker == 0 ? "" : ",", worker < kAicWorkers ? "aic" : "aiv"); + } + // schema-v4 无论是否开启 atomic 都导出 producer summary；phase-only 的 + // atomic/clock 字段为零，离线分析仍可独立证明 records 与 dropped 闭合。 +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + std::fprintf( + output, + "],\"lazy_sample_shape\":{\"name\":\"%s\",\"id\":%u," + "\"observation\":\"%s\",\"finish\":\"%s\",\"control_family\":\"%s\"}," + "\"fdwic_summary\":{\"records\":%llu,\"atomic_records\":%llu," + "\"clock_baseline_records\":%llu,\"atomic_calls\":%llu," + "\"batched_poll_calls\":%llu,\"poll_batch_records\":%llu," + "\"dropped_records\":%llu}", + kLazySampleShapeName, kLazySampleShapeId, kLazySampleObservation, + kLazySampleFinishShape, kLazySampleControlFamily, + static_cast(producer_summary.records), + static_cast(producer_summary.atomic_records), + static_cast(producer_summary.clock_baseline_records), + static_cast(producer_summary.atomic_calls), + static_cast(producer_summary.poll_calls), + static_cast(producer_summary.poll_batch_records), + static_cast(producer_summary.dropped_records) + ); +#else + std::fprintf( + output, + "],\"fdwic_summary\":{\"records\":%llu,\"atomic_records\":%llu," + "\"clock_baseline_records\":%llu,\"atomic_calls\":%llu," + "\"batched_poll_calls\":%llu,\"poll_batch_records\":%llu," + "\"dropped_records\":%llu}", + static_cast(producer_summary.records), + static_cast(producer_summary.atomic_records), + static_cast(producer_summary.clock_baseline_records), + static_cast(producer_summary.atomic_calls), + static_cast(producer_summary.poll_calls), + static_cast(producer_summary.poll_batch_records), + static_cast(producer_summary.dropped_records) + ); +#endif + std::fprintf( + output, + "},\n\"aicore_tasks\":[],\n\"aicpu_tasks\":[],\n" + "\"aicpu_scheduler_phases\":[],\n\"aicpu_orchestrator_phases\":[],\n\"fdwic_events\":[\n" + ); + // fdwic_events 每行固定十列：core、block、lane、task、function、phase、起止周期、flags、aux。 + + bool success = true; + bool first_record = true; + uint64_t exported_records = 0; + TraceExportSummary observed_summary; + std::vector scratch(kTraceRecordsPerCore); + constexpr int32_t kTracePhaseCount = static_cast(TracePhase::Count); + for (uint32_t worker = 0; worker < kWorkers && success; ++worker) { + // 每次只读取一个 worker 的有效区间；完整 384 MiB trace 缓冲无需整体回拷。 + const uint32_t available = header.cores[worker].count; + if (available > header.records_per_core) { + std::fprintf( + stderr, "Trace core %u count %u exceeds capacity %u.\n", worker, available, + header.records_per_core + ); + success = false; + break; + } + if (available != 0 && !read_records(worker, available, scratch.data())) { + success = false; + break; + } + const TraceCoreState &core = header.cores[worker]; + uint64_t core_atomic_calls = 0; + uint64_t core_poll_calls = 0; + uint32_t core_atomic_records = 0; + uint32_t core_poll_batch_records = 0; + uint32_t core_clock_records = 0; + uint32_t core_plain_clock_records = 0; + uint32_t core_dependency_clock_records = 0; + bool dependency_applied = false; + bool direct_result_used_return_ready = false; + bool direct_result_used_source_issue = false; + for (uint32_t index = 0; index < available; ++index) { + const TraceRecord &record = scratch[index]; + const bool atomic_record = record.phase == static_cast(TracePhase::Atomic); + const bool claim_record = record.phase == static_cast(TracePhase::Claim); + const bool clock_record = record.phase == static_cast(TracePhase::ClockBaseline); + const bool atomic_schema_valid = !atomic_record || + AtomicRecordSchemaValid(record, atomic_trace_enabled); + const bool claim_schema_valid = !claim_record || + ((record.flags & ~(kClaimWon | kClaimAttempted)) == 0 && + ((record.flags & kClaimWon) == 0 || (record.flags & kClaimAttempted) != 0) && + record.auxiliary <= 1); + const bool clock_schema_valid = !clock_record || + (atomic_trace_enabled && ClockRecordSchemaValid(record)); + const bool record_valid = record.end_cycle >= record.start_cycle && record.phase >= 0 && + record.phase < kTracePhaseCount && record.task_id >= -1 && + record.function_id >= -1 && record.lane == core.lane && + record.block_id == core.block_id && + record.core_idx == core.core_idx && atomic_schema_valid && + claim_schema_valid && clock_schema_valid; + if (!record_valid) { + std::fprintf( + stderr, + "Invalid trace record at worker=%u index=%u: phase=%d lane=%d block=%d core=%d " + "start=%llu end=%llu flags=0x%08x aux=%u\n", + worker, index, record.phase, record.lane, record.block_id, record.core_idx, + static_cast(record.start_cycle), + static_cast(record.end_cycle), record.flags, record.auxiliary + ); + success = false; + break; + } + if (atomic_record) { + ++core_atomic_records; + const uint32_t call_count = AtomicRecordCallCount(record); + core_atomic_calls += call_count; + if ((record.flags & kAtomicPollBatch) != 0) { + core_poll_calls += call_count; + ++core_poll_batch_records; + } else if ((record.flags & kAtomicResultUsed) != 0) { + if ((record.flags & kAtomicReturnReady) != 0) { + direct_result_used_return_ready = true; + } else { + direct_result_used_source_issue = true; + } + } + } else if (clock_record) { + ++core_clock_records; + if ((record.flags & kClockAtomicDependency) != 0) { + ++core_dependency_clock_records; + dependency_applied = (record.flags & kClockAtomicDependencyApplied) != 0; + } else { + ++core_plain_clock_records; + } + } + std::fprintf( + output, + "%s[%d,%d,%d,%d,%d,\"%s\",%llu,%llu,%u,%u]", + first_record ? "" : ",\n", record.core_idx, record.block_id, record.lane, record.task_id, + record.function_id, TracePhaseName(static_cast(record.phase)), + static_cast(record.start_cycle), + static_cast(record.end_cycle), record.flags, record.auxiliary + ); + first_record = false; + ++exported_records; + } + if (!success) break; + bool core_closed = true; + if (atomic_trace_enabled) { + const uint64_t expected_atomic_records = + static_cast(core.atomic_calls) - core.poll_calls + core.poll_batch_records; + core_closed = core_atomic_records == expected_atomic_records && + core_atomic_calls == core.atomic_calls && core_poll_calls == core.poll_calls && + core_poll_batch_records == core.poll_batch_records && core_clock_records == 2 && + core_plain_clock_records == 1 && core_dependency_clock_records == 1 && + (!dependency_applied || !direct_result_used_source_issue) && + (dependency_applied || !direct_result_used_return_ready); + } else { + core_closed = core_atomic_records == 0 && core_atomic_calls == 0 && core_poll_calls == 0 && + core_poll_batch_records == 0 && core_clock_records == 0; + } + if (!core_closed) { + std::fprintf( + stderr, + "swimlane closure failed on worker=%u: physical_atomic=%u logical_atomic=%llu/%u " + "poll_calls=%llu/%u poll_batches=%u/%u clock=%u plain=%u dependency=%u " + "dependency_applied=%s direct_ready=%s direct_issue=%s\n", + worker, core_atomic_records, static_cast(core_atomic_calls), + core.atomic_calls, static_cast(core_poll_calls), core.poll_calls, + core_poll_batch_records, core.poll_batch_records, core_clock_records, + core_plain_clock_records, core_dependency_clock_records, + dependency_applied ? "yes" : "no", direct_result_used_return_ready ? "yes" : "no", + direct_result_used_source_issue ? "yes" : "no" + ); + success = false; + break; + } + observed_summary.records += available; + observed_summary.atomic_records += core_atomic_records; + observed_summary.clock_baseline_records += core_clock_records; + observed_summary.atomic_calls += core_atomic_calls; + observed_summary.poll_calls += core_poll_calls; + observed_summary.poll_batch_records += core_poll_batch_records; + observed_summary.dropped_records += core.dropped; + } + if (success && !SameTraceSummary(producer_summary, observed_summary)) { + std::fprintf( + stderr, + "swimlane producer/raw summary mismatch: records=%llu/%llu atomic_records=%llu/%llu " + "atomic_calls=%llu/%llu poll_calls=%llu/%llu poll_batches=%llu/%llu clock=%llu/%llu\n", + static_cast(observed_summary.records), + static_cast(producer_summary.records), + static_cast(observed_summary.atomic_records), + static_cast(producer_summary.atomic_records), + static_cast(observed_summary.atomic_calls), + static_cast(producer_summary.atomic_calls), + static_cast(observed_summary.poll_calls), + static_cast(producer_summary.poll_calls), + static_cast(observed_summary.poll_batch_records), + static_cast(producer_summary.poll_batch_records), + static_cast(observed_summary.clock_baseline_records), + static_cast(producer_summary.clock_baseline_records) + ); + success = false; + } + if (success) std::fprintf(output, "\n]}\n"); + if (std::ferror(output) != 0) { + std::fprintf(stderr, "Failed while writing swimlane output %s.\n", temporary_path.c_str()); + success = false; + } + if (std::fclose(output) != 0) { + std::fprintf(stderr, "Failed to close swimlane output %s: %s\n", temporary_path.c_str(), std::strerror(errno)); + success = false; + } + if (success && std::rename(temporary_path.c_str(), output_path.c_str()) != 0) { + std::fprintf( + stderr, "Cannot finalize swimlane output %s: %s\n", output_path.c_str(), std::strerror(errno) + ); + success = false; + } + if (!success) { + std::remove(temporary_path.c_str()); + return false; + } + std::printf( + "[SWIMLANE] raw_json=%s events=%llu\n", output_path.c_str(), + static_cast(exported_records) + ); + return true; +} + +template +inline bool AnalyzeSwimlaneRecords( + const TraceHeader &header, const SchedulerState &state, ReadRecords read_records +) { + if (!ValidateTraceHeader(header, "swimlane analysis")) return false; + + // 第一组数组统计“每个 worker 在某阶段的累计时间”；task_durations 则保留重点阶段的单事件分布。 + constexpr uint32_t kTracePhaseCount = static_cast(TracePhase::Count); + constexpr TracePhase kDetailedPhases[] = { + TracePhase::EfDrain, TracePhase::Materialize, TracePhase::Claim, TracePhase::Register, + }; + uint64_t cycles[kWorkers][kTracePhaseCount] = {}; + uint64_t counts[kWorkers][kTracePhaseCount] = {}; + std::vector task_durations[2][kTasksPerBatch][sizeof(kDetailedPhases) / sizeof(kDetailedPhases[0])]; + std::vector atomic_durations[2][static_cast(AtomicSite::Count)]; + uint64_t atomic_return_ready_counts[2][static_cast(AtomicSite::Count)] = {}; + std::vector atomic_poll_windows[2][static_cast(AtomicSite::Count)]; + uint64_t atomic_poll_calls[2][static_cast(AtomicSite::Count)] = {}; + std::vector clock_baselines[2]; + std::vector clock_dependency_baselines[2]; + uint64_t clock_dependency_applied[2] = {}; + std::vector scratch(kTraceRecordsPerCore); + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const uint32_t available = header.cores[worker].count; + const uint32_t count = std::min(available, header.records_per_core); + if (count != 0 && !read_records(worker, count, scratch.data())) { + return false; + } + for (uint32_t index = 0; index < count; ++index) { + const TraceRecord &record = scratch[index]; + if (record.phase < 0 || record.phase >= static_cast(kTracePhaseCount) || + record.end_cycle < record.start_cycle) { + // 分析器面对单条坏记录选择跳过；严格导出路径会直接拒绝，二者服务于不同诊断目的。 + continue; + } + const uint32_t phase = static_cast(record.phase); + const uint64_t duration = record.end_cycle - record.start_cycle; + const bool atomic_poll_batch = + record.phase == static_cast(TracePhase::Atomic) && + (record.flags & kAtomicPollBatch) != 0; + // PollBatch 的 duration 是一次等待 episode 的包络，允许夹着其他直接 + // atomic/调度代码；不能混入“Atomic 单次括号”的累计时间或分位数。 + if (!atomic_poll_batch) { + cycles[worker][phase] += duration; + ++counts[worker][phase]; + } + if (record.phase == static_cast(TracePhase::Atomic) && + record.auxiliary < static_cast(AtomicSite::Count)) { + const uint32_t role_index = + state.results[worker].role == static_cast(CoreRole::Aic) ? 0U : 1U; + if (atomic_poll_batch) { + atomic_poll_windows[role_index][record.auxiliary].push_back(duration); + atomic_poll_calls[role_index][record.auxiliary] += AtomicRecordCallCount(record); + } else { + atomic_durations[role_index][record.auxiliary].push_back(duration); + atomic_return_ready_counts[role_index][record.auxiliary] += + (record.flags & kAtomicReturnReady) != 0; + } + } + if (record.phase == static_cast(TracePhase::ClockBaseline)) { + const uint32_t role_index = + state.results[worker].role == static_cast(CoreRole::Aic) ? 0U : 1U; + if ((record.flags & kClockAtomicDependency) != 0) { + clock_dependency_baselines[role_index].push_back(duration); + clock_dependency_applied[role_index] += + (record.flags & kClockAtomicDependencyApplied) != 0; + } else { + clock_baselines[role_index].push_back(duration); + } + } + if (record.task_id >= 0) { + // task_id % 5 恰好对应 Alloc/QK/SF/PV/UP，这是固定 PA Case1 图的拓扑约束。 + const uint32_t role_index = + state.results[worker].role == static_cast(CoreRole::Aic) ? 0U : 1U; + const uint32_t kind = static_cast(record.task_id) % kTasksPerBatch; + for (uint32_t detail = 0; detail < sizeof(kDetailedPhases) / sizeof(kDetailedPhases[0]); ++detail) { + if (phase == static_cast(kDetailedPhases[detail])) { + task_durations[role_index][kind][detail].push_back(duration); + } + } + } + } + } + + const CoreRole roles[] = {CoreRole::Aic, CoreRole::Aiv}; + const char *role_names[] = {"AIC", "AIV"}; + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t phase = 0; phase < kTracePhaseCount; ++phase) { + std::vector role_cycles; + uint64_t record_count = 0; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + if (state.results[worker].role != static_cast(roles[role_index])) continue; + role_cycles.push_back(cycles[worker][phase]); + record_count += counts[worker][phase]; + } + const Uint64Distribution summary = SummarizeUint64(role_cycles); + std::printf( + "[TRACE_PHASE] role=%s phase=%s records=%llu accumulated_us_median=%.3f " + "accumulated_us_p95=%.3f accumulated_us_max=%.3f\n", + role_names[role_index], TracePhaseName(phase), + static_cast(record_count), summary.median / 1000.0, + static_cast(summary.p95) / 1000.0, + static_cast(summary.maximum) / 1000.0 + ); + } + } + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + const Uint64Distribution summary = SummarizeUint64(clock_baselines[role_index]); + if (!clock_baselines[role_index].empty()) { + std::printf( + "[TRACE_CLOCK] role=%s samples=%zu definition=consecutive-sys-cnt-reads " + "median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], clock_baselines[role_index].size(), summary.median, + static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + const Uint64Distribution dependency_summary = + SummarizeUint64(clock_dependency_baselines[role_index]); + if (!clock_dependency_baselines[role_index].empty()) { + std::printf( + "[TRACE_CLOCK] role=%s samples=%zu definition=atomic-return-dependency-hook " + "dependency_applied=%llu/%zu median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], clock_dependency_baselines[role_index].size(), + static_cast(clock_dependency_applied[role_index]), + clock_dependency_baselines[role_index].size(), dependency_summary.median, + static_cast(dependency_summary.p95), + static_cast(dependency_summary.maximum) + ); + } + } + // Atomic 只报告原始括号分布，不扣除计时底噪，也不把 total_cycles + // 解释成可与 Submit 墙钟直接相加的“atomic 占比”。return-ready 只表示 + // 本核可消费返回值，不表示其他核已经观察到更新。 + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t site = 0; site < static_cast(AtomicSite::Count); ++site) { + const std::vector &durations = atomic_durations[role_index][site]; + if (durations.empty()) continue; + const Uint64Distribution summary = SummarizeUint64(durations); + const AtomicOp op = AtomicSiteOp(static_cast(site)); + const uint64_t return_ready_count = atomic_return_ready_counts[role_index][site]; + const char *boundary = return_ready_count == durations.size() + ? "return-ready" + : (return_ready_count == 0 ? "source-issue" : "mixed"); + std::printf( + "[TRACE_ATOMIC] role=%s site=%s op=%s events=%zu boundary=%s " + "return_ready=%llu/%zu bracket_cycles_total=%llu median_ns=%.1f " + "p95_ns=%llu max_ns=%llu\n", + role_names[role_index], AtomicSiteName(site), AtomicOpName(static_cast(op)), + durations.size(), boundary, static_cast(return_ready_count), + durations.size(), static_cast(summary.total), summary.median, + static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + } + // 等待聚合只报告 episode 数、精确逻辑调用数与包络分布。window 不能除以 + // calls 当作单次 atomic latency，也不能与 Submit 墙钟直接相加。 + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t site = 0; site < static_cast(AtomicSite::Count); ++site) { + const std::vector &windows = atomic_poll_windows[role_index][site]; + if (windows.empty()) continue; + const Uint64Distribution summary = SummarizeUint64(windows); + std::printf( + "[TRACE_ATOMIC_POLL] role=%s site=%s op=%s episodes=%zu logical_calls=%llu " + "window_definition=wait-episode-envelope median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], AtomicSiteName(site), + AtomicOpName(static_cast(AtomicSiteOp(static_cast(site)))), + windows.size(), static_cast(atomic_poll_calls[role_index][site]), + summary.median, static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + } + const char *kind_names[] = {"Alloc", "QK", "SF", "PV", "UP"}; + // 单事件统计按 role 与 task kind 展开，可区分“该 role 真实参与”与“只回放前端”的成本。 + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t kind = 0; kind < kTasksPerBatch; ++kind) { + for (uint32_t detail = 0; detail < sizeof(kDetailedPhases) / sizeof(kDetailedPhases[0]); ++detail) { + const std::vector &durations = task_durations[role_index][kind][detail]; + const Uint64Distribution summary = SummarizeUint64(durations); + std::printf( + "[TRACE_TASK] role=%s kind=%s phase=%s events=%zu median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], kind_names[kind], + TracePhaseName(static_cast(kDetailedPhases[detail])), durations.size(), summary.median, + static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + } + } + return true; +} + +inline Metrics Validate( + const SchedulerState &state, uint32_t run, double host_us, const TraceHeader *trace_header = nullptr +) { + Metrics metrics; + // 每个 worker 都回放全部 task。Alloc 由 96 个 worker 全部执行 atomicMax Claim； + // 其余 kernel task 只有与 active role 匹配的 AIC 或 AIV 参与 Claim。 + const uint32_t batches = state.config.batches; + const uint32_t task_count = batches * kTasksPerBatch; + const uint64_t expected_submits = static_cast(kWorkers) * task_count; + const uint64_t expected_claims = + static_cast(batches) * (kWorkers + kAicWorkers + kAivWorkers + kAicWorkers + kAivWorkers); + // 上式依次对应 Alloc、QK、SF、PV、UP 的 active worker 数，默认 256 batch 时为 73728。 + + // 聚合量分为调度核心计数、kernel 分布、前端操作数和最终状态四组，便于定位语义偏差。 + uint64_t first_submit = UINT64_MAX; + uint64_t last_submit = 0; + uint64_t submits = 0; + uint64_t claims = 0; + uint64_t wins = 0; + uint64_t heap_guards = 0; + uint64_t fanin_ready_loads = 0; + uint64_t fanin_not_ready_loads = 0; + uint64_t frontier_initial_loads = 0; + uint64_t frontier_updates = 0; + uint64_t frontier_terminal_loads = 0; + uint64_t atomic_trace_calls = 0; + uint64_t duplicates = 0; + uint64_t cas_retries = 0; + uint64_t joint_polls = 0; + uint64_t trace_wait_records = 0; + uint64_t wins_by_kind[5] = {}; + uint64_t kernel_counts[4] = {}; + uint64_t kernel_cycles[4] = {}; + uint64_t kernel_min[4] = {}; + uint64_t kernel_max[4] = {}; + uint64_t placements[3] = {}; + uint64_t phase_calls[static_cast(ProfilePhase::Count)] = {}; + uint64_t context_reads = 0; + uint64_t views_created = 0; + uint64_t dynamic_create_infos = 0; + uint64_t arg_resets = 0; + uint64_t tensor_args_added = 0; + uint64_t scalar_args_added = 0; + uint64_t materialized_outputs = 0; + uint64_t map_inserts = 0; + uint64_t map_lookups = 0; + uint64_t slot_tensor_copies = 0; + uint64_t slot_scalar_copies = 0; + uint64_t fanin_edges = 0; + bool worker_ids[kWorkers] = {}; + uint32_t aic_count = 0; + uint32_t aiv_count = 0; + uint32_t winning_workers = 0; + uint64_t max_worker_wins = 0; + bool worker_shape_ok = true; + bool submit_timestamps_ok = true; + bool vend_values_ok = true; + bool frontend_worker_counts_ok = true; + bool final_worker_state_ok = true; + bool worker_checksums_ok = true; + bool fanin_worker_counts_ok = true; + bool frontier_worker_counts_ok = true; + bool role_kernel_routing_ok = true; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + bool lazy_sample_split_runtime_oracle_ok = true; + const uint64_t expected_lazy_sample_split_task_id_sum = + static_cast(task_count) * (task_count - 1U) / 2U; +#endif + + // 按真实输出大小、1 KiB 对齐和 256 MiB 环回规则重算每个 task 可接受的最小 vend。 + uint64_t expected_heap_next = 0; + bool vend_progress_bounds_ok = true; + uint32_t first_bad_vend = task_count; + uint64_t first_bad_vend_minimum = 0; + uint64_t first_bad_vend_actual = 0; + std::vector minimum_vends(task_count); + const uint64_t output_bytes_by_kind[] = {10240, 524288, 264192, 8192, 0}; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + const uint64_t output_bytes = output_bytes_by_kind[task_id % kTasksPerBatch]; + uint64_t task_base = (expected_heap_next + kOutputAlignment - 1) / kOutputAlignment * kOutputAlignment; + if (output_bytes != 0 && (task_base % state.heap_size) + output_bytes > state.heap_size) { + task_base = (task_base / state.heap_size + 1) * state.heap_size; + } + expected_heap_next = task_base + output_bytes; + minimum_vends[task_id] = expected_heap_next; + } + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + // vend 可以大于本 task 的最小末端，因为 winner 发布的是其本地 heap_cursor 快照； + // 但不能超过该 worker 完整回放所有 task 后的最终 heap_next。 + if (state.tasks[task_id].vend < minimum_vends[task_id] || + state.tasks[task_id].vend > expected_heap_next) { + vend_progress_bounds_ok = false; + if (first_bad_vend == task_count) { + first_bad_vend = task_id; + first_bad_vend_minimum = minimum_vends[task_id]; + first_bad_vend_actual = state.tasks[task_id].vend; + } + } + } + // TensorMap 只保留 heap window 内仍可能被依赖的四类输出；floor 对应已安全退休的 task 边界。 + const uint64_t expected_map_live = 4ULL * std::min(batches, 13); + const uint64_t expected_map_floor = task_count > kHeapWindow + 1 ? task_count - kHeapWindow - 1 : 0; + + for (uint32_t index = 0; index < kWorkers; ++index) { + // 每核只写自己独占且按 cache line 隔离的 WorkerResult；host 在 kernel 完成后统一汇总，不引入额外 atomic。 + const WorkerResult &result = state.results[index]; + if (result.worker_id < kWorkers) { + worker_ids[result.worker_id] = true; + } else { + worker_shape_ok = false; + } + aic_count += result.role == static_cast(CoreRole::Aic); + aiv_count += result.role == static_cast(CoreRole::Aiv); + worker_shape_ok &= result.submits == task_count; + worker_shape_ok &= result.max_occupied <= kUsableSlots; + worker_shape_ok &= result.final_occupied == 0; + submit_timestamps_ok &= result.submit_begin != 0; + submit_timestamps_ok &= result.submit_end >= result.submit_begin; + submit_timestamps_ok &= result.finish_cycle >= result.submit_end; + first_submit = std::min(first_submit, result.submit_begin); + last_submit = std::max(last_submit, result.submit_end); + submits += result.submits; + claims += result.claim_attempts; + wins += result.claim_wins; + if (result.claim_wins != 0) ++winning_workers; + max_worker_wins = std::max(max_worker_wins, result.claim_wins); + heap_guards += result.heap_guards; + fanin_ready_loads += result.fanin_ready_loads; + fanin_not_ready_loads += result.fanin_not_ready_loads; + frontier_initial_loads += result.frontier_initial_loads; + frontier_updates += result.frontier_updates; + frontier_terminal_loads += result.frontier_terminal_loads; + atomic_trace_calls += result.atomic_trace_calls; + duplicates += result.completion_duplicates; + cas_retries += result.cas_retries; + joint_polls += result.joint_polls; + trace_wait_records += result.wait_events[0] + result.wait_events[1]; + context_reads += result.context_reads; + views_created += result.views_created; + dynamic_create_infos += result.dynamic_create_infos; + arg_resets += result.arg_resets; + tensor_args_added += result.tensor_args_added; + scalar_args_added += result.scalar_args_added; + materialized_outputs += result.materialized_outputs; + map_inserts += result.map_inserts; + map_lookups += result.map_lookups; + slot_tensor_copies += result.slot_tensor_copies; + slot_scalar_copies += result.slot_scalar_copies; + fanin_edges += result.fanin_edges; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + const CoreRole expected_role = index < kAicWorkers ? CoreRole::Aic : CoreRole::Aiv; + lazy_sample_split_runtime_oracle_ok &= result.worker_id == index; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_caller_state_address != 0; + lazy_sample_split_runtime_oracle_ok &= + result.lazy_sample_split_finish_state_address == result.lazy_sample_split_caller_state_address; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_finish_calls == task_count; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_protocol_errors == 0; + lazy_sample_split_runtime_oracle_ok &= + result.lazy_sample_split_state_cookie == + (kLazySampleSplitStateCookieBase ^ static_cast(index) ^ + (static_cast(static_cast(expected_role)) << 32U)); + lazy_sample_split_runtime_oracle_ok &= + result.lazy_sample_split_task_id_sum == expected_lazy_sample_split_task_id_sum; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_owner_worker_id == index; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_reserved == 0; +#endif +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + const uint64_t qk_wins = result.wins[static_cast(TaskKind::Qk)]; + const uint64_t sf_wins = result.wins[static_cast(TaskKind::Sf)]; + const uint64_t pv_wins = result.wins[static_cast(TaskKind::Pv)]; + const uint64_t up_wins = result.wins[static_cast(TaskKind::Up)]; + const uint64_t expected_worker_views = kLazySampleLazy + ? static_cast(batches) + qk_wins + : static_cast(batches) * 2; + const uint64_t expected_worker_tensors = kLazySampleLazy + ? static_cast(batches) * 12 + + qk_wins * 3 + sf_wins + pv_wins * 3 + up_wins * 3 + : static_cast(batches) * 22; + const uint64_t expected_worker_scalars = kLazySampleLazy + ? qk_wins * 2 + sf_wins * 3 + pv_wins * 2 + up_wins * 2 + : static_cast(batches) * 9; + frontend_worker_counts_ok &= result.context_reads == batches; + frontend_worker_counts_ok &= result.views_created == expected_worker_views; + frontend_worker_counts_ok &= result.dynamic_create_infos == static_cast(batches) * 2; + // Alloc constructs once; QK/SF/PV/UP each reset once inside their callback. + frontend_worker_counts_ok &= result.arg_resets == static_cast(batches) * 4; + frontend_worker_counts_ok &= result.tensor_args_added == expected_worker_tensors; + frontend_worker_counts_ok &= result.scalar_args_added == expected_worker_scalars; + frontend_worker_counts_ok &= result.materialized_outputs == static_cast(batches) * 8; + frontend_worker_counts_ok &= result.map_inserts == static_cast(batches) * 4; +#else + frontend_worker_counts_ok &= result.context_reads == batches; + frontend_worker_counts_ok &= result.views_created == static_cast(batches) * 2; + frontend_worker_counts_ok &= result.dynamic_create_infos == static_cast(batches) * 2; + frontend_worker_counts_ok &= result.arg_resets == static_cast(batches) * 4; + frontend_worker_counts_ok &= result.tensor_args_added == static_cast(batches) * 22; + frontend_worker_counts_ok &= result.scalar_args_added == static_cast(batches) * 9; + frontend_worker_counts_ok &= result.materialized_outputs == static_cast(batches) * 8; + frontend_worker_counts_ok &= result.map_inserts == static_cast(batches) * 4; +#endif + final_worker_state_ok &= result.final_heap_next == expected_heap_next; + final_worker_state_ok &= result.map_high_water == expected_map_live; + final_worker_state_ok &= result.map_live_entries == expected_map_live; + final_worker_state_ok &= result.map_alive_floor == expected_map_floor; + final_worker_state_ok &= result.map_cleaned_upto == expected_map_floor; + worker_checksums_ok &= result.checksum == (0xcbf29ce484222325ULL ^ result.worker_id); + const uint64_t worker_kernel_completions = result.kernel_counts[0] + result.kernel_counts[1] + + result.kernel_counts[2] + result.kernel_counts[3]; + if (result.role == static_cast(CoreRole::Aic)) { + role_kernel_routing_ok &= result.kernel_counts[1] == 0 && result.kernel_counts[3] == 0; + } else if (result.role == static_cast(CoreRole::Aiv)) { + role_kernel_routing_ok &= result.kernel_counts[0] == 0 && result.kernel_counts[2] == 0; + } else { + role_kernel_routing_ok = false; + } + const uint64_t worker_completions = result.wins[0] + worker_kernel_completions; + frontier_worker_counts_ok &= result.frontier_initial_loads == worker_completions; + frontier_worker_counts_ok &= result.frontier_terminal_loads == result.frontier_initial_loads; + fanin_worker_counts_ok &= result.fanin_ready_loads >= result.fanin_edges; + if (result.fanin_ready_loads >= result.fanin_edges) { + // PA 最大 fanin 为 3；每次失败检查最多先重读两个 ready 前缀，再遇到一个 not-ready。 + fanin_worker_counts_ok &= + result.fanin_ready_loads - result.fanin_edges <= 2 * result.fanin_not_ready_loads; + } + for (uint32_t kind = 0; kind < 5; ++kind) + wins_by_kind[kind] += result.wins[kind]; + for (uint32_t kind = 0; kind < 4; ++kind) { + kernel_counts[kind] += result.kernel_counts[kind]; + kernel_cycles[kind] += result.kernel_cycles[kind]; + if (result.kernel_min_cycles[kind] != 0 && + (kernel_min[kind] == 0 || result.kernel_min_cycles[kind] < kernel_min[kind])) { + kernel_min[kind] = result.kernel_min_cycles[kind]; + } + kernel_max[kind] = std::max(kernel_max[kind], result.kernel_max_cycles[kind]); + } + for (uint32_t place = 0; place < 3; ++place) + placements[place] += result.placement[place]; + for (uint32_t phase = 0; phase < static_cast(ProfilePhase::Count); ++phase) + phase_calls[phase] += result.phase_calls[phase]; + } + for (bool seen : worker_ids) + worker_shape_ok &= seen; + + uint32_t ready_flags = 0; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + // ready flag 和 vend 是跨核 completion 的最终外部可见状态，不能只依赖 worker 私有计数判断完成。 + ready_flags += state.tasks[task_id].flag == 1; + vend_values_ok &= state.tasks[task_id].vend != 0; + vend_values_ok &= state.tasks[task_id].vend % kOutputAlignment == 0; + } + const uint64_t kernel_total = kernel_counts[0] + kernel_counts[1] + kernel_counts[2] + kernel_counts[3]; + const uint64_t placement_total = placements[0] + placements[1] + placements[2]; + const uint64_t fanin_loads = fanin_ready_loads + fanin_not_ready_loads; + const uint64_t frontier_flag_loads = frontier_updates + frontier_terminal_loads; + + // 第一组断言覆盖参与者拓扑、Claim/winner、completion 和最终 drain 等调度主协议。 + Expect(aic_count == kAicWorkers && aiv_count == kAivWorkers, "participant topology is 32 AIC + 64 AIV", &metrics); + Expect(worker_shape_ok, "all 96 worker markers and private rings are valid", &metrics); +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + Expect( + lazy_sample_split_runtime_oracle_ok, + "split caller/finish share one role-specific block-local state and finish every task once", + &metrics + ); +#endif + Expect(submit_timestamps_ok, "all Submit timing markers are valid", &metrics); + Expect(state.started_count.value == kWorkers, "started_count is 96", &metrics); + Expect(submits == expected_submits, "replay count is workers * tasks", &metrics); + Expect(claims == expected_claims, "Claim attempt count matches PA topology", &metrics); + Expect(wins == task_count, "exactly one winner per task", &metrics); + Expect( + wins_by_kind[0] == batches && wins_by_kind[1] == batches && wins_by_kind[2] == batches && + wins_by_kind[3] == batches && wins_by_kind[4] == batches, + "Alloc/QK/SF/PV/UP winners are one per batch", &metrics + ); + Expect(kernel_total == static_cast(batches) * 4, "kernel count is four per batch", &metrics); + Expect( + kernel_counts[0] == batches && kernel_counts[1] == batches && kernel_counts[2] == batches && + kernel_counts[3] == batches, + "each kernel kind executes once per batch", &metrics + ); + Expect( + role_kernel_routing_ok, + "AIC executes only QK/PV and AIV executes only SF/UP", &metrics + ); + Expect(heap_guards == static_cast(batches) * 4, "heap guard count matches output winners", &metrics); + Expect( + fanin_worker_counts_ok && fanin_ready_loads >= fanin_edges && + fanin_ready_loads - fanin_edges <= 2 * fanin_not_ready_loads, + "fanin ready/failure load classification is complete", &metrics + ); + Expect( + frontier_worker_counts_ok && frontier_initial_loads == task_count, + "frontier initial loads match completed tasks", &metrics + ); + Expect( + frontier_terminal_loads == task_count && frontier_updates >= task_count, + "frontier ready/update/terminal load identity is exact", &metrics + ); + Expect(duplicates == 0, "completion flags are published once", &metrics); + Expect(ready_flags == task_count, "all task flags are ready", &metrics); + Expect(vend_values_ok, "all published vend values are nonzero and aligned", &metrics); + Expect(vend_progress_bounds_ok, "every task vend is within PA worker heap progress bounds", &metrics); + Expect(state.frontier.value == static_cast(task_count) - 1, "frontier reaches the final task", &metrics); + Expect(state.replay_done.value == kWorkers, "replay_done is 96", &metrics); + Expect(state.fatal.value == 0, "fatal remains clear", &metrics); + Expect(placement_total == kernel_total, "EfDrain + RingBp + final placement covers every kernel", &metrics); + // joint_polls 是为未来 BlockWon 模拟预留的结果字段，当前调度路径没有递增点； + // 此断言只确认现有输出保持零，不能单独证明 active_count>=2 分支不可达。 + Expect(joint_polls == 0, "single-lane PA performs no BlockWon polling", &metrics); + // 第二组断言锁定 scalar 前端工作量，防止编译器优化或后续改动悄悄删掉 PA 模拟步骤。 + Expect(frontend_worker_counts_ok, "every worker replays the exact PA frontend operation counts", &metrics); +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + const uint64_t expected_global_views = static_cast(batches) * + (kLazySampleLazy ? kWorkers + 1 : kWorkers * 2); + const uint64_t expected_global_tensors = static_cast(batches) * + (kLazySampleLazy ? kWorkers * 12 + 10 : kWorkers * 22); + const uint64_t expected_global_scalars = static_cast(batches) * + (kLazySampleLazy ? 9 : kWorkers * 9); + Expect( + context_reads == static_cast(kWorkers) * batches && + views_created == expected_global_views && + dynamic_create_infos == static_cast(kWorkers) * batches * 2 && + arg_resets == static_cast(kWorkers) * batches * 4 && + tensor_args_added == expected_global_tensors && + scalar_args_added == expected_global_scalars && + materialized_outputs == static_cast(kWorkers) * batches * 8 && + map_inserts == static_cast(kWorkers) * batches * 4, + "global PA frontend operation totals are exact", &metrics + ); + std::printf( + "[LAZY_SAMPLE_FRONTEND] shape=%s views=%llu/%llu tensor_args=%llu/%llu " + "scalar_args=%llu/%llu resets=%llu/%llu\n", + kLazySampleShapeName, + static_cast(views_created), + static_cast(expected_global_views), + static_cast(tensor_args_added), + static_cast(expected_global_tensors), + static_cast(scalar_args_added), + static_cast(expected_global_scalars), + static_cast(arg_resets), + static_cast(static_cast(kWorkers) * batches * 4) + ); +#else + Expect( + context_reads == static_cast(kWorkers) * batches && + views_created == static_cast(kWorkers) * batches * 2 && + dynamic_create_infos == static_cast(kWorkers) * batches * 2 && + arg_resets == static_cast(kWorkers) * batches * 4 && + tensor_args_added == static_cast(kWorkers) * batches * 22 && + scalar_args_added == static_cast(kWorkers) * batches * 9 && + materialized_outputs == static_cast(kWorkers) * batches * 8 && + map_inserts == static_cast(kWorkers) * batches * 4, + "global PA frontend operation totals are exact", &metrics + ); +#endif + Expect( + map_lookups == static_cast(batches) * 14 && + slot_tensor_copies == static_cast(batches) * 19 && + slot_scalar_copies == static_cast(batches) * 9 && + fanin_edges == static_cast(batches) * 5, + "winner-only map, slot-copy, and fanin totals are exact", &metrics + ); + Expect(final_worker_state_ok, "every worker final heap and TensorMap state is exact", &metrics); + Expect(worker_checksums_ok, "all frontend registration checksums remain clean", &metrics); + + // 三类 Claim cursor 各有四个 shard；按 task_id 重新推导每个 shard 应停留的最后任务。 + int64_t expected_cube[kCursorShards] = {-1, -1, -1, -1}; + int64_t expected_vector[kCursorShards] = {-1, -1, -1, -1}; + int64_t expected_alloc[kCursorShards] = {-1, -1, -1, -1}; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + const TaskKind kind = static_cast(task_id % kTasksPerBatch); + int64_t *cursors = kind == TaskKind::Alloc + ? expected_alloc + : (kind == TaskKind::Qk || kind == TaskKind::Pv ? expected_cube : expected_vector); + cursors[task_id % kCursorShards] = task_id; + } + bool cursors_ok = true; + for (uint32_t shard = 0; shard < kCursorShards; ++shard) { + cursors_ok &= state.cube_cursor[shard].value == expected_cube[shard]; + cursors_ok &= state.vector_cursor[shard].value == expected_vector[shard]; + cursors_ok &= state.alloc_cursor[shard].value == expected_alloc[shard]; + } + Expect(cursors_ok, "all sharded Claim cursors reach their exact final task", &metrics); + + if (state.config.profile_phases != 0) { + // profile 开关关闭时这些字段允许保持零，避免把可选诊断本身变成语义门禁。 + Expect( + phase_calls[static_cast(ProfilePhase::Claim)] == expected_submits && + phase_calls[static_cast(ProfilePhase::EfDrain)] == expected_submits && + phase_calls[static_cast(ProfilePhase::WaitForSlot)] == + static_cast(batches) * 4 && + phase_calls[static_cast(ProfilePhase::HeapGuard)] == + static_cast(batches) * 4, + "profile call counts match Claim/EfDrain/WaitForSlot/HeapGuard flow", &metrics + ); + } + + if (state.config.trace_enabled != 0) { + // 固定阶段记录数加上动态等待记录数，应与所有 worker 的 header count 精确相等。 + bool trace_shape_ok = trace_header != nullptr; + uint64_t trace_records = 0; + uint64_t trace_dropped = 0; + uint64_t physical_atomic_records = 0; + uint64_t batched_poll_calls = 0; + uint64_t poll_batch_records = 0; + bool per_worker_trace_counts_ok = true; + if (trace_header != nullptr) { + trace_shape_ok &= trace_header->magic == 0x4653574cU; + trace_shape_ok &= trace_header->version == 4; + trace_shape_ok &= trace_header->num_cores == kWorkers; + trace_shape_ok &= trace_header->records_per_core == kTraceRecordsPerCore; + trace_shape_ok &= trace_header->frequency_hz == kSystemCounterHz; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const TraceCoreState &core = trace_header->cores[worker]; + trace_records += core.count; + trace_dropped += core.dropped; + trace_shape_ok &= core.count <= kTraceRecordsPerCore; + int32_t expected_block = -1; + int32_t expected_lane = -1; + ExpectedTraceTopology(worker, &expected_block, &expected_lane); + trace_shape_ok &= core.core_idx == static_cast(worker); + trace_shape_ok &= core.block_id == expected_block; + trace_shape_ok &= core.lane == expected_lane; + const WorkerResult &result = state.results[worker]; + const uint64_t worker_kernels = result.kernel_counts[0] + result.kernel_counts[1] + + result.kernel_counts[2] + result.kernel_counts[3]; + uint64_t worker_physical_atomic = 0; + if ((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) { + trace_shape_ok &= core.atomic_calls == result.atomic_trace_calls; + trace_shape_ok &= core.poll_calls <= core.atomic_calls; + trace_shape_ok &= (core.poll_calls == 0) == (core.poll_batch_records == 0); + worker_physical_atomic = + static_cast(core.atomic_calls) - core.poll_calls + core.poll_batch_records; + physical_atomic_records += worker_physical_atomic; + batched_poll_calls += core.poll_calls; + poll_batch_records += core.poll_batch_records; + } else { + trace_shape_ok &= core.atomic_calls == 0 && core.poll_calls == 0 && + core.poll_batch_records == 0; + } + const uint64_t worker_expected = + 6 * result.submits + 2 * result.claim_wins - result.wins[0] + + 2 * worker_kernels + result.wait_events[0] + result.wait_events[1] + 2 + + (((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) + ? worker_physical_atomic + 2 + : 0); + per_worker_trace_counts_ok &= core.count == worker_expected; + } + } + const uint64_t expected_trace_records = + static_cast(batches) * (static_cast(kWorkers) * 30 + 17) + + trace_wait_records + 2 * kWorkers + + (((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) + ? physical_atomic_records + 2 * kWorkers + : 0); + // 每 batch 的 96*30 是六条每 Submit 固定记录；17 条是 + // 5 条 winner tail、4 条 Fanin 和 8 条 Kernel/Commit。loser 不再写 + // 零时长 marker。两个父 span 再各核固定增加 2 条； + // RingBp 等真实等待按运行时次数额外加入。 + Expect(trace_shape_ok, "swimlane header and per-worker capacities are valid", &metrics); + Expect(trace_dropped == 0, "swimlane records fit without drops", &metrics); + Expect(trace_records == expected_trace_records, "swimlane record count matches PA phase flow", &metrics); + Expect(per_worker_trace_counts_ok, "every worker swimlane record count is exact", &metrics); + if ((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) { + Expect(atomic_trace_calls != 0, "atomic trace captured source-level calls", &metrics); + } else { + Expect(atomic_trace_calls == 0, "atomic trace counters stay zero when disabled", &metrics); + } + std::printf( + "[TRACE] records=%llu expected=%llu dropped=%llu bytes=%zu\n", + static_cast(trace_records), + static_cast(expected_trace_records), + static_cast(trace_dropped), kTraceBytes + ); + std::printf( + "[ATOMIC_TRACE] enabled=%s logical_calls=%llu physical_records=%llu " + "batched_poll_calls=%llu poll_batch_records=%llu " + "closure=physical=logical-batched+batch_records\n", + (state.config.trace_enabled & kTraceAtomicsEnabled) != 0 ? "yes" : "no", + static_cast(atomic_trace_calls), + static_cast(physical_atomic_records), + static_cast(batched_poll_calls), + static_cast(poll_batch_records) + ); + } + + if (first_submit != UINT64_MAX && last_submit >= first_submit) { + // 性能口径只覆盖最早 Submit.begin 到最晚 Submit.end，不含启动屏障、最终 drain 和 host 同步。 + metrics.submit_span_us = static_cast(last_submit - first_submit) / 1000.0; + } + std::printf( + "[METRIC] run=%u submit_span_us=%.3f host_launch_us=%.3f claims=%llu fanin_loads=%llu cas_retries=%llu\n", run, + metrics.submit_span_us, host_us, static_cast(claims), + static_cast(fanin_loads), static_cast(cas_retries) + ); + const uint64_t submit_completion_ops = + claims + heap_guards + fanin_loads + 2ULL * task_count + frontier_initial_loads + + frontier_flag_loads + frontier_updates; + std::printf( + "[ATOMIC] submit_completion_ops=%llu fanin_ready=%llu fanin_not_ready=%llu frontier_initial=%llu " + "frontier_flag=%llu frontier_ready_fetch_max=%llu frontier_terminal=%llu\n", + static_cast(submit_completion_ops), + static_cast(fanin_ready_loads), + static_cast(fanin_not_ready_loads), + static_cast(frontier_initial_loads), + static_cast(frontier_flag_loads), + static_cast(frontier_updates), + static_cast(frontier_terminal_loads) + ); + std::printf( + "[WINNERS] active_workers=%u max_wins_per_worker=%llu\n", winning_workers, + static_cast(max_worker_wins) + ); + std::printf( + "[PLACEMENT] EfDrain=%llu RingBp=%llu FinalDrain=%llu\n", + static_cast(placements[static_cast(DrainPlace::EfDrain)]), + static_cast(placements[static_cast(DrainPlace::RingBackpressure)]), + static_cast(placements[static_cast(DrainPlace::FinalDrain)]) + ); + // placement 统计回答 kernel 最终在哪个 drain 点执行，与 TracePhase 的累计 span 互补。 + const char *kernel_names[] = {"QK", "SF", "PV", "UP"}; + const uint32_t targets[] = {kTargetQkTicks, kTargetSfTicks, kTargetPvTicks, kTargetUpTicks}; + for (uint32_t kind = 0; kind < 4; ++kind) { + const double mean = + kernel_counts[kind] == 0 ? 0.0 : static_cast(kernel_cycles[kind]) / kernel_counts[kind]; + std::printf( + "[KERNEL] %-2s count=%llu mean_us=%.3f min_us=%.3f max_us=%.3f target_us=%.3f\n", kernel_names[kind], + static_cast(kernel_counts[kind]), mean / 1000.0, kernel_min[kind] / 1000.0, + kernel_max[kind] / 1000.0, targets[kind] / 1000.0 + ); + } + PrintPhaseDiagnostics(state); + if (!metrics.passed) { + // 失败时补充第一处未完成 task、vend 边界和 worker 进度，避免只有笼统的 ASSERT FAIL。 + uint32_t first_not_ready = task_count; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + if (state.tasks[task_id].flag != 1) { + first_not_ready = task_id; + break; + } + } + uint64_t min_worker_submits = UINT64_MAX; + uint64_t max_worker_submits = 0; + uint32_t incomplete_workers = 0; + uint32_t occupied_workers = 0; + uint64_t max_final_occupied = 0; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const WorkerResult &result = state.results[worker]; + min_worker_submits = std::min(min_worker_submits, result.submits); + max_worker_submits = std::max(max_worker_submits, result.submits); + incomplete_workers += result.submits != task_count; + occupied_workers += result.final_occupied != 0; + max_final_occupied = std::max(max_final_occupied, result.final_occupied); + } + const int64_t retire = state.frontier.value - static_cast(kHeapWindow); + const uint64_t retire_vend = + retire >= 0 && retire < static_cast(task_count) ? state.tasks[retire].vend : 0; + std::printf( + "[FAILURE_STATE] fatal=%d frontier=%lld first_not_ready=%u first_bad_vend=%u " + "vend_minimum=%llu vend_actual=%llu retire=%lld retire_vend=%llu " + "worker_submits_min=%llu worker_submits_max=%llu incomplete_workers=%u " + "final_occupied_workers=%u max_final_occupied=%llu\n", + state.fatal.value, static_cast(state.frontier.value), first_not_ready, first_bad_vend, + static_cast(first_bad_vend_minimum), + static_cast(first_bad_vend_actual), + static_cast(retire), static_cast(retire_vend), + static_cast(min_worker_submits), + static_cast(max_worker_submits), incomplete_workers, occupied_workers, + static_cast(max_final_occupied) + ); + } + return metrics; +} + +inline double Median(std::vector values) { + // 多轮 benchmark 只报告中位数；上板基线比较仍应优先采用独立进程首轮。 + std::sort(values.begin(), values.end()); + const size_t middle = values.size() / 2; + if ((values.size() & 1U) != 0) return values[middle]; + return (values[middle - 1] + values[middle]) / 2.0; +} + +inline void PrintBanner(const char *backend, const Options &options) { + // 开始运行前完整打印工作量和大内存占用，便于确认比较口径没有混用。 + std::printf("=== Standalone PA Scheduler Benchmark: %s ===\n", backend); + std::printf( + "device=%u batches=%u tasks=%u workers=%u runs=%u nops=%u,%u,%u,%u state_bytes=%zu " + "swimlane=%s trace_atomics=%s trace_bytes=%zu\n", options.device, + options.batches, options.batches * kTasksPerBatch, kWorkers, options.runs, options.nops.qk, options.nops.sf, + options.nops.pv, options.nops.up, sizeof(SchedulerState), options.trace_enabled ? "on" : "off", + options.trace_atomics ? "on" : "off", + options.trace_enabled ? kTraceBytes : 0 + ); +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + std::printf( + "lazy_sample_shape=%s lazy_sample_shape_id=%u observation=%s finish=%s " + "control_family=%s\n", + kLazySampleShapeName, kLazySampleShapeId, kLazySampleObservation, + kLazySampleFinishShape, kLazySampleControlFamily + ); +#endif + if (!options.swimlane_json.empty()) { + std::printf("swimlane_json=%s\n", options.swimlane_json.c_str()); + } +} + +} // namespace pa_scheduler::host + +#endif // PA_SCHEDULER_COMMON_HOST_SUPPORT_H diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/common/pa_frontend.h b/tests/atomic_probe/lazy_lamda_sample/A_original/common/pa_frontend.h new file mode 100644 index 0000000000..f2cc8fcabb --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/common/pa_frontend.h @@ -0,0 +1,1308 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_FRONTEND_H +#define PA_SCHEDULER_COMMON_PA_FRONTEND_H + +#include "pa_model.h" + +namespace pa_scheduler { + +// 这些基址只充当稳定的 tensor identity，供 descriptor、区间重叠和 heap 地址 +// 计算使用；winner workload 不解引用这些 synthetic 地址，real-compute 使用独立 +// workspace。context_lens 是唯一按真实 GM 指针读取的 PA 前端输入。 +constexpr uint64_t kInvalidTaskId = UINT64_MAX; +constexpr uint64_t kSyntheticQueryBase = 0x200000000ULL; +constexpr uint64_t kSyntheticKeyBase = 0x300000000ULL; +constexpr uint64_t kSyntheticValueBase = 0x400000000ULL; +constexpr uint64_t kSyntheticBlockTableBase = 0x500000000ULL; +constexpr uint64_t kSyntheticOutputBase = 0x600000000ULL; +constexpr uint64_t kSyntheticContextLensBase = 0x700000000ULL; +constexpr uint32_t kPaHeads = 16; +constexpr uint32_t kPaHeadDim = 128; +constexpr uint32_t kPaBlockSize = 128; +constexpr uint32_t kPaBlocksPerRequest = 64; +constexpr uint32_t kPaMaxBlocksPerRequest = 256; +constexpr uint64_t kPaScaleBits = 0x3F800000ULL; +constexpr uint32_t kSpmdLocalContextIndex = kMaxTaskTensors + kMaxTaskScalars; +constexpr uint32_t kSpmdGlobalContextIndex = kSpmdLocalContextIndex + 1; +static_assert(kMaxTaskTensors == 32, "PA frontend requires the real 32 tensor slots"); +static_assert(kMaxTaskScalars == 16, "PA frontend requires the real 16 scalar slots"); +static_assert(kSpmdLocalContextIndex == 48, "PA local-context dispatch index mismatch"); +static_assert(kSpmdGlobalContextIndex == 49, "PA global-context dispatch index mismatch"); +static_assert(kMaxFanin == 16, "PA frontend requires the real 16 fanin slots"); + +enum class TensorRefKind : uint8_t { + LocalTensor = 0, + GmTensor = 1, + CreateInfo = 2, +}; + +// TaskArgs 同时容纳 orchestration 栈上的 descriptor、GM 中已物化的 descriptor， +// 以及尚待 Materialize 的 CreateInfo。显式 kind 保留生产 TensorRef 的地址空间分支。 +union TensorPointer { + const TensorDesc *local_tensor; + PA_GM const TensorDesc *gm_tensor; + const TensorCreateInfo *create_info; +}; + +struct TaskTensorRef { + TensorPointer pointer; + TensorRefKind kind; +}; +static_assert(sizeof(TaskTensorRef) == 16, "TaskTensorRef must match the PA TensorRef ABI"); +static_assert(offsetof(TaskTensorRef, pointer) == 0, "TaskTensorRef pointer offset mismatch"); +static_assert(offsetof(TaskTensorRef, kind) == 8, "TaskTensorRef kind offset mismatch"); + +struct PaLaunchSpec { + int16_t core_num; + bool require_sync_start; +}; +static_assert(sizeof(PaLaunchSpec) == 4, "PA launch spec ABI mismatch"); + +struct PaAsyncContext { + uint64_t completion_count; + uint64_t completion_error_code; + uint64_t completion_entries; + uint32_t completion_capacity; + uint32_t alignment_padding; + uint64_t task_token; +}; +static_assert(sizeof(PaAsyncContext) == 40, "PA async context ABI mismatch"); + +struct PaLocalContext { + int32_t block_index; + int32_t block_count; + PaAsyncContext async; +}; +// Local/GlobalContext 最终放进 RingSlot 的固定 dispatch 参数位 48/49；它们不是 +// standalone 自定义参数，offset 必须与真实 SPMD kernel 调用约定一致。 +static_assert(sizeof(PaLocalContext) == 48, "PA local context ABI mismatch"); + +struct PaGlobalContext { + int32_t sub_block_id; +}; +static_assert(sizeof(PaGlobalContext) == 4, "PA global context ABI mismatch"); + +// PTO2 profiling is enabled in the PA baseline. reset() clears all 160 bytes +// below on every QK/SF/PV/UP argument rebuild, even though Case1 does not ask +// to dump an argument. Keeping this storage and write stream matters to the +// spacing between consecutive Claim operations. +// 这段看似未使用的清零属于真实前端成本，删除会改变各 worker 到达 +// Claim 的波形与竞争强度，因此仍按生产构造/reset 顺序执行。 +struct PaDumpArgSelection { + uint64_t dump_arg_mask; + uint64_t dump_arg_index_ambiguous_mask; + uint64_t scalar_source_ptrs[kMaxTaskScalars]; + uint8_t scalar_dtypes[kMaxTaskScalars]; +}; +static_assert(sizeof(PaDumpArgSelection) == 160, "PA dump-selection ABI mismatch"); + +struct TaskArgs { + // The real TaskArgsTpl inherits its tag mixin first. TensorArgType is an + // int32 enum in the PA ABI; keeping tags first also reproduces its offsets. + // tag 数组位于对象首部不是任意排布；Materialize、fanin 与 register + // 都会重复扫描/复用这些 tag，错误 offset 会同时改变语义和前端访存成本。 + int32_t tags[kMaxTaskTensors]; + TaskTensorRef tensors[kMaxTaskTensors]; + uint64_t scalars[kMaxTaskScalars]; + int32_t tensor_count; + int32_t scalar_count; + + bool has_error; + uint64_t error_msg; + PaLaunchSpec launch_spec; + PaDumpArgSelection dump_arg_selection; + uint64_t explicit_deps; + uint32_t explicit_dep_count; + uint8_t cacheline_pad[48]; +}; +static_assert(sizeof(TaskArgs) == 1024, "TaskArgs must match the PA L0TaskArgs ABI size"); +static_assert(offsetof(TaskArgs, tags) == 0, "TaskArgs tag offset mismatch"); +static_assert(offsetof(TaskArgs, tensors) == 128, "TaskArgs tensor-ref offset mismatch"); +static_assert(offsetof(TaskArgs, scalars) == 640, "TaskArgs scalar offset mismatch"); +static_assert(offsetof(TaskArgs, tensor_count) == 768, "TaskArgs tensor-count offset mismatch"); +static_assert(offsetof(TaskArgs, scalar_count) == 772, "TaskArgs scalar-count offset mismatch"); +static_assert(offsetof(TaskArgs, has_error) == 776, "TaskArgs error flag offset mismatch"); +static_assert(offsetof(TaskArgs, error_msg) == 784, "TaskArgs error pointer offset mismatch"); +static_assert(offsetof(TaskArgs, launch_spec) == 792, "TaskArgs launch-spec offset mismatch"); +static_assert(offsetof(TaskArgs, dump_arg_selection) == 800, "TaskArgs dump-selection offset mismatch"); +static_assert(offsetof(TaskArgs, explicit_deps) == 960, "TaskArgs dependency pointer offset mismatch"); +static_assert(offsetof(TaskArgs, explicit_dep_count) == 968, "TaskArgs dependency count offset mismatch"); + +struct TaskOutputs { + uint64_t task_id; + uint32_t count; + PA_GM TensorDesc *tensors[kMaxTaskTensors]; +}; +static_assert(sizeof(TaskOutputs) == 272, "TaskOutputs must match the PA TaskOutputTensors ABI size"); +static_assert(offsetof(TaskOutputs, tensors) == 16, "TaskOutputs tensor pointer offset mismatch"); + +struct SubmitContext { + PA_GM WorkerState *self; + PA_GM TaskPayload *payload; + int32_t task_id; + int32_t tensor_count; + int32_t scalar_count; + uint32_t register_mask; + uint64_t output_bytes; + TaskOutputs result; + int32_t fanin[kMaxFanin]; + int32_t fanin_count; + int32_t kernel_id; + bool won; + bool joint; + bool joint_init; + int32_t joint_block; + int32_t joint_slot; + int32_t joint_count; +}; +// SubmitContext 贯穿一次 Submit：Begin 绑定 task/payload，Materialize 填充输出与 +// register_mask，winner 收集 fanin 并构建 slot。它复刻 DistSubmitCtx 而非诊断结构。 +static_assert(sizeof(SubmitContext) == 400, "SubmitContext must match DistSubmitCtx"); +static_assert(offsetof(SubmitContext, output_bytes) == 32, "SubmitContext output-byte offset mismatch"); +static_assert(offsetof(SubmitContext, result) == 40, "SubmitContext result offset mismatch"); +static_assert(offsetof(SubmitContext, fanin) == 312, "SubmitContext fanin offset mismatch"); + +struct OutputLayout { + uint64_t buffer_sizes[kMaxTaskTensors]; + uint64_t total_output_size; +}; +// 只有 tag=Output 的槽位拥有有效 buffer_sizes；总大小按 1 KiB 对齐累计，随后 +// 作为 HeapGuard 的 output_bytes 和本 worker heap_next 的推进量。 +static_assert(sizeof(OutputLayout) == 264, "OutputLayout must match DistOutputLayout"); + +// 该状态保存真实 PA orchestration 在五个 Submit 之间传递的 descriptor。输出指针 +// 指向每个 worker 自己 materialize 的 payload，不能跨 worker 共享或简化为全局对象。 +struct PaOrchestrationState { + TensorDesc query; + TensorDesc key_cache; + TensorDesc value_cache; + TensorDesc block_table; + TensorDesc context_lens; + TensorDesc output; + TensorDesc query_view; + TensorDesc output_view; + + TensorCreateInfo tile_create_info; + TensorCreateInfo scalar_create_info; + TensorCreateInfo qk_create_info; + TensorCreateInfo sf_create_info; + + // The pointer is supplied by the standalone backend. On A5 it must point + // at GM so every batch performs the same descriptor-based load as PA. + PA_GM const volatile int32_t *context_lens_data; + uint64_t scale_bits; + uint64_t current_sequence; + uint64_t current_blocks; + uint64_t current_block_offset; + uint64_t current_nblocks; + uint64_t current_valid_len; + uint32_t current_batch; + + PA_GM TensorDesc *accumulated_output; + PA_GM TensorDesc *accumulated_sum; + PA_GM TensorDesc *accumulated_max; + PA_GM TensorDesc *qk_scores; + PA_GM TensorDesc *sf_probs; + PA_GM TensorDesc *sf_max; + PA_GM TensorDesc *sf_sum; + PA_GM TensorDesc *pv_output; +}; + +PA_DEVICE uint64_t ElementSize(DataType dtype) { + // 输入 dtype 来自已通过 PA ABI 构造的 descriptor/create-info，必须落在 Count 前； + // 输出字节数同时用于外部 tensor range 与新 Output 的 heap 大小计算。 + constexpr static uint64_t sizes[static_cast(DataType::Count)] = { + 4, 2, 4, 2, 1, 1, 2, 8, 8, 2, 4, 1, + }; + return sizes[static_cast(dtype)]; +} + +PA_DEVICE int32_t TagValue(TensorArgType tag) { return static_cast(tag); } + +PA_DEVICE TensorArgType TaskTag(const TaskArgs &args, uint32_t index) { + // index 的有效范围由 tensor_count 保证；集中转换避免各阶段对 int32 ABI tag + // 做不同解释，Materialize/CollectFanin/Register 因而共享同一分类结果。 + return static_cast(args.tags[index]); +} + +PA_DEVICE void ClearDumpArgSelection(PaDumpArgSelection &selection) { + // Volatile stores intentionally preserve the profiling-enabled PA reset + // traffic even though the standalone winner workload never consumes dump data. + // volatile 的目的不是同步，而是阻止编译器删掉这段生产基线中存在的写流量。 + volatile uint64_t *masks = &selection.dump_arg_mask; + masks[0] = 0; + masks[1] = 0; + volatile uint64_t *sources = &selection.scalar_source_ptrs[0]; + for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { + sources[index] = 0; + } + volatile uint8_t *dtypes = &selection.scalar_dtypes[0]; + for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { + dtypes[index] = 0; + } +} + +PA_DEVICE void ConstructTaskArgs(TaskArgs &args) { + // TensorTagMixin::tags_{} is value-initialized by the + // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. + // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar + // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 + volatile int32_t *tags = &args.tags[0]; + for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { + tags[index] = 0; + } + args.tensor_count = 0; + args.scalar_count = 0; + args.has_error = false; + args.error_msg = 0; + args.launch_spec.core_num = 1; + args.launch_spec.require_sync_start = false; + ClearDumpArgSelection(args.dump_arg_selection); + args.explicit_deps = 0; + args.explicit_dep_count = 0; +} + +PA_DEVICE void ResetTaskArgs(TaskArgs &args) { + // reset 的输出是不含 tensor/scalar/显式依赖的新逻辑参数表，但保留已分配对象及 + // launch_spec；QK/SF/PV/UP 在同一个 1 KiB TaskArgs 上依次复用这一状态。 + args.tensor_count = 0; + args.scalar_count = 0; + ClearDumpArgSelection(args.dump_arg_selection); + args.explicit_deps = 0; + args.explicit_dep_count = 0; + args.has_error = false; + args.error_msg = 0; +} + +PA_DEVICE bool ReserveTensorArgs(TaskArgs &args, int32_t count) { + // tensor 必须先于 scalar 追加，以保持 dispatch args 的 [tensor..., scalar...] + // 排列；失败只置 has_error，不发生部分追加。 + if (args.scalar_count != 0 || count < 0 || + args.tensor_count + count > static_cast(kMaxTaskTensors)) { + args.has_error = true; + return false; + } + return true; +} + +PA_DEVICE void AppendLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { + const uint32_t index = static_cast(args.tensor_count++); + args.tensors[index].pointer.local_tensor = &tensor; + args.tensors[index].kind = TensorRefKind::LocalTensor; + args.tags[index] = TagValue(tag); +} + +PA_DEVICE void AppendGmTensor(TaskArgs &args, PA_GM const TensorDesc &tensor, TensorArgType tag) { + const uint32_t index = static_cast(args.tensor_count++); + args.tensors[index].pointer.gm_tensor = &tensor; + args.tensors[index].kind = TensorRefKind::GmTensor; + args.tags[index] = TagValue(tag); +} + +PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { + const uint32_t index = static_cast(args.tensor_count++); + args.tensors[index].pointer.create_info = &create_info; + args.tensors[index].kind = TensorRefKind::CreateInfo; + args.tags[index] = TagValue(TensorArgType::Output); +} + +PA_DEVICE bool ReserveScalarArgs(TaskArgs &args, int32_t count) { + // 先整体校验容量再由 AddTwo/AddThree 连续写入，保证多 scalar 操作全有或全无。 + if (count < 0 || args.scalar_count + count > static_cast(kMaxTaskScalars)) { + args.has_error = true; + return false; + } + return true; +} + +PA_DEVICE void AppendScalar(TaskArgs &args, uint64_t value) { + args.scalars[static_cast(args.scalar_count++)] = value; +} + +PA_DEVICE void AddLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { + if (ReserveTensorArgs(args, 1)) AppendLocalTensor(args, tensor, tag); +} + +PA_DEVICE void AddGmTensor(TaskArgs &args, PA_GM const TensorDesc &tensor, TensorArgType tag) { + if (ReserveTensorArgs(args, 1)) AppendGmTensor(args, tensor, tag); +} + +PA_DEVICE void AddOutput(TaskArgs &args, const TensorCreateInfo &create_info) { + if (ReserveTensorArgs(args, 1)) AppendOutput(args, create_info); +} + +PA_DEVICE void AddScalar(TaskArgs &args, uint64_t value) { + if (ReserveScalarArgs(args, 1)) AppendScalar(args, value); +} + +PA_DEVICE void AddTwoScalars(TaskArgs &args, uint64_t value0, uint64_t value1) { + if (!ReserveScalarArgs(args, 2)) return; + AppendScalar(args, value0); + AppendScalar(args, value1); +} + +PA_DEVICE void AddThreeScalars(TaskArgs &args, uint64_t value0, uint64_t value1, uint64_t value2) { + if (!ReserveScalarArgs(args, 3)) return; + AppendScalar(args, value0); + AppendScalar(args, value1); + AppendScalar(args, value2); +} + +PA_DEVICE void InitCreateInfo( + TensorCreateInfo &info, const uint32_t shapes[kMaxTensorDims], uint32_t ndims, DataType dtype +) { + info.initial_value = 0; + info.has_initial_value = false; + info.reserved0 = 0; + info.start_offset = 0; + info.version = 0; + info.ndims = ndims; + info.dtype = dtype; + info.manual_dep = false; + info.is_contiguous = true; + info.child_memory = 0; + // TensorCreateInfo's real constructor only writes active dimensions. + // 只写 ndims 个 shape，保留生产构造器的写入范围，不能为方便把五维全清零。 + for (uint32_t index = 0; index < ndims; ++index) { + info.shapes[index] = shapes[index]; + } +} + +PA_DEVICE void ClearCreateInfo(TensorCreateInfo &info) { + volatile uint8_t *bytes = reinterpret_cast(&info); + for (uint32_t index = 0; index < sizeof(TensorCreateInfo); ++index) { + bytes[index] = 0; + } +} + +PA_DEVICE void InitExternalTensor( + TensorDesc &tensor, uint64_t address, const uint32_t shapes[kMaxTensorDims], uint32_t ndims, DataType dtype, + bool manual_dep +) { + // 输入为稳定 backing address、逻辑 shape 和依赖属性；输出是完整连续 descriptor， + // owner 无效表示它不是本轮 task 产生，row-major stride 从末维向前计算。 + uint64_t elements = 1; + for (uint32_t index = 0; index < ndims; ++index) { + elements *= shapes[index]; + } + tensor.buffer_addr = address; + tensor.buffer_size = elements * ElementSize(dtype); + tensor.owner_task_id = kInvalidTaskId; + tensor.start_offset = 0; + tensor.version = 0; + tensor.ndims = ndims; + tensor.dtype = dtype; + tensor.manual_dep = manual_dep; + tensor.is_contiguous = true; + tensor.child_memory = 0; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + tensor.shapes[index] = shapes[index]; + tensor.strides[index] = 0; + } + uint32_t stride = 1; + for (int32_t index = static_cast(ndims) - 1; index >= 0; --index) { + tensor.strides[index] = stride; + stride *= tensor.shapes[index]; + } + tensor.extent_elem_cache = stride; +} + +PA_DEVICE bool InitTensorFromCreateInfo( + PA_GM TensorDesc &tensor, const TensorCreateInfo &info, uint64_t address, uint64_t buffer_size +) { + tensor.buffer_addr = address; + tensor.buffer_size = buffer_size; + tensor.owner_task_id = kInvalidTaskId; + tensor.start_offset = info.start_offset; + tensor.version = info.version; + tensor.ndims = info.ndims; + tensor.dtype = info.dtype; + tensor.manual_dep = info.manual_dep; + tensor.is_contiguous = info.is_contiguous; + tensor.child_memory = info.child_memory; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + tensor.shapes[index] = info.shapes[index]; + } + uint32_t stride = 1; + for (int32_t index = static_cast(tensor.ndims) - 1; index >= 0; --index) { + tensor.strides[index] = stride; + stride *= tensor.shapes[index]; + } + tensor.extent_elem_cache = stride; + // PA can initialize the backing allocation here. Case1 never requests it; + // the standalone uses synthetic heap addresses and therefore rejects that + // unsupported path instead of writing to a fabricated GM pointer. + // Case1 的 has_initial_value 恒为 false；返回 false 是对未模拟分支的 + // 明确保护，不会在合成地址上伪造初始化写入。 + return !info.has_initial_value; +} + +PA_DEVICE uint64_t CreateInfoBytes(const TensorCreateInfo &info) { + uint64_t elements = 1; + for (uint32_t index = 0; index < info.ndims; ++index) { + elements *= info.shapes[index]; + } + return elements * ElementSize(info.dtype); +} + +template +PA_DEVICE void CopyTensorLine1(TensorDesc &destination, const Source &source) { + // view 只需复制 descriptor 第一条 cache line 的身份/shape 字段，随后由调用方 + // 覆盖 offset、shape、stride 与 extent；不做整 128-byte 拷贝以匹配 PA 写流。 + destination.buffer_addr = source.buffer_addr; + destination.buffer_size = source.buffer_size; + destination.owner_task_id = source.owner_task_id; + destination.start_offset = source.start_offset; + destination.version = source.version; + destination.ndims = source.ndims; + destination.dtype = source.dtype; + destination.manual_dep = source.manual_dep; + destination.is_contiguous = source.is_contiguous; + destination.child_memory = source.child_memory; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + destination.shapes[index] = source.shapes[index]; + } +} + +PA_DEVICE void MakeBatchViews(PaOrchestrationState &orch, uint32_t batch) { + // query/output view 共享原 backing buffer，仅通过 start_offset 切出当前 batch。 + // output_view 保留真实 manual_dep 标记；UP 的生产者依赖由 Alloc、SF、PV 返回 descriptor 的 owner 闭合。 + CopyTensorLine1(orch.query_view, orch.query); + orch.query_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.query_view.ndims = 2; + orch.query_view.shapes[0] = kPaHeads; + orch.query_view.shapes[1] = kPaHeadDim; + orch.query_view.strides[0] = kPaHeadDim; + orch.query_view.strides[1] = 1; + orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; + + CopyTensorLine1(orch.output_view, orch.output); + orch.output_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.output_view.ndims = 2; + orch.output_view.manual_dep = true; + orch.output_view.shapes[0] = kPaHeads; + orch.output_view.shapes[1] = kPaHeadDim; + orch.output_view.strides[0] = kPaHeadDim; + orch.output_view.strides[1] = 1; + orch.output_view.extent_elem_cache = kPaHeads * kPaHeadDim; +} + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +// The two views remain final parameter-packing operations. query_view is a +// winner-only QK input; output_view is the produce side of a private INOUT and +// is therefore evaluated on every worker by the callback builder. +PA_DEVICE void MakeLazySampleCallbackOutputView(PaOrchestrationState &orch, uint32_t batch) { + CopyTensorLine1(orch.output_view, orch.output); + orch.output_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.output_view.ndims = 2; + orch.output_view.manual_dep = true; + orch.output_view.shapes[0] = kPaHeads; + orch.output_view.shapes[1] = kPaHeadDim; + orch.output_view.strides[0] = kPaHeadDim; + orch.output_view.strides[1] = 1; + orch.output_view.extent_elem_cache = kPaHeads * kPaHeadDim; +} + +PA_DEVICE void MakeLazySampleCallbackQueryView(PaOrchestrationState &orch, uint32_t batch) { + CopyTensorLine1(orch.query_view, orch.query); + orch.query_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.query_view.ndims = 2; + orch.query_view.shapes[0] = kPaHeads; + orch.query_view.shapes[1] = kPaHeadDim; + orch.query_view.strides[0] = kPaHeadDim; + orch.query_view.strides[1] = 1; + orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; +} +#endif + +PA_DEVICE uint64_t MinU64(uint64_t lhs, uint64_t rhs) { return lhs < rhs ? lhs : rhs; } + +PA_DEVICE uint64_t ReadPaContextLength(const PaOrchestrationState &orch, uint32_t batch) { + if (orch.context_lens_data == nullptr) { + // Compatibility fallback for a backend that has not yet supplied the + // 256-int GM buffer. Exact PA runs must pass a non-null pointer. + // 正式对等运行必须走下方 descriptor+stride 的 GM load；fallback + // 只用于不具备该缓冲区的兼容后端。 + return kPaBlocksPerRequest * kPaBlockSize; + } + const uint64_t flat_index = orch.context_lens.start_offset + + static_cast(batch) * orch.context_lens.strides[0]; + PA_GM const volatile int32_t *value = reinterpret_cast( + orch.context_lens.buffer_addr + flat_index * ElementSize(DataType::Int32) + ); + return static_cast(*value); +} + +PA_DEVICE void PreparePaBlockGroup(PaOrchestrationState &orch, uint64_t block_offset) { + // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 + // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 + orch.current_block_offset = block_offset; + orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); + const uint64_t last_block_sequence_start = + (block_offset + orch.current_nblocks - 1) * kPaBlockSize; + orch.current_valid_len = MinU64(kPaBlockSize, orch.current_sequence - last_block_sequence_start); +} + +PA_DEVICE void BeginPaBatch(PaOrchestrationState &orch, uint32_t batch) { + // Match paged_attention_orch.cpp: context GM load and block arithmetic + // happen before entering the q scope and before constructing either view. + // 该顺序会影响 Submit 前的指令与访存间隔，故不把长度读取挪进 QK 构参。 + orch.current_batch = batch; + orch.current_sequence = ReadPaContextLength(orch, batch); + orch.current_blocks = (orch.current_sequence + kPaBlockSize - 1) / kPaBlockSize; + MakeBatchViews(orch, batch); +} + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +PA_DEVICE void BeginPaBatchForLazySampleCallback(PaOrchestrationState &orch, uint32_t batch) { + // Tensor reads and arithmetic that feed more than one task stay on the + // common path. Descriptor construction is deferred to the task callback. + orch.current_batch = batch; + orch.current_sequence = ReadPaContextLength(orch, batch); + orch.current_blocks = (orch.current_sequence + kPaBlockSize - 1) / kPaBlockSize; +} +#endif + +PA_DEVICE void InitPaOrchestration( + PaOrchestrationState &orch, uint32_t batches, PA_GM const volatile int32_t *context_lens_data +) { + // 初始化只建立整轮回放共享的外部 descriptor/create-info 模板；每 batch 的 view、 + // context length、动态 QK/SF shape 和返回 descriptor 留给五阶段流按原顺序更新。 + const uint32_t query_shape[kMaxTensorDims] = {batches * kPaHeads, kPaHeadDim, 0, 0, 0}; + const uint32_t cache_shape[kMaxTensorDims] = { + batches * kPaBlocksPerRequest * kPaBlockSize, kPaHeadDim, 0, 0, 0 + }; + const uint32_t table_shape[kMaxTensorDims] = {batches, kPaMaxBlocksPerRequest, 0, 0, 0}; + const uint32_t context_shape[kMaxTensorDims] = {batches, 0, 0, 0, 0}; + InitExternalTensor(orch.query, kSyntheticQueryBase, query_shape, 2, DataType::Bfloat16, false); + InitExternalTensor(orch.key_cache, kSyntheticKeyBase, cache_shape, 2, DataType::Bfloat16, false); + InitExternalTensor(orch.value_cache, kSyntheticValueBase, cache_shape, 2, DataType::Bfloat16, false); + InitExternalTensor(orch.block_table, kSyntheticBlockTableBase, table_shape, 2, DataType::Int32, false); + const uint64_t context_address = context_lens_data == nullptr + ? kSyntheticContextLensBase + : reinterpret_cast(context_lens_data); + InitExternalTensor(orch.context_lens, context_address, context_shape, 1, DataType::Int32, false); + InitExternalTensor(orch.output, kSyntheticOutputBase, query_shape, 2, DataType::Float32, false); + + const uint32_t tile_shape[kMaxTensorDims] = {kPaHeads, kPaHeadDim, 0, 0, 0}; + const uint32_t scalar_shape[kMaxTensorDims] = {kPaHeads, 0, 0, 0, 0}; + ClearCreateInfo(orch.tile_create_info); + ClearCreateInfo(orch.scalar_create_info); + ClearCreateInfo(orch.qk_create_info); + ClearCreateInfo(orch.sf_create_info); + InitCreateInfo(orch.tile_create_info, tile_shape, 2, DataType::Float32); + InitCreateInfo(orch.scalar_create_info, scalar_shape, 1, DataType::Float32); + + // QK/SF create infos are deliberately not constructed here: in PA they are + // constructed inside the group after Alloc and QK respectively. + // 动态 shape 依赖当前 block group，提前构造既不符合业务数据流，也会 + // 把真实发生在两个 Submit 之间的前端工作错误搬到初始化阶段。 + orch.context_lens_data = context_lens_data; + orch.scale_bits = kPaScaleBits; + orch.current_sequence = 0; + orch.current_blocks = 0; + orch.current_block_offset = 0; + orch.current_nblocks = 0; + orch.current_valid_len = 0; + orch.current_batch = 0; + + orch.accumulated_output = nullptr; + orch.accumulated_sum = nullptr; + orch.accumulated_max = nullptr; + orch.qk_scores = nullptr; + orch.sf_probs = nullptr; + orch.sf_max = nullptr; + orch.sf_sum = nullptr; + orch.pv_output = nullptr; +} + +PA_DEVICE void InitPaOrchestration(PaOrchestrationState &orch, uint32_t batches) { + InitPaOrchestration(orch, batches, nullptr); +} + +PA_DEVICE void BuildAllocArgs(PaOrchestrationState &orch, TaskArgs &args, uint32_t batch) { + BeginPaBatch(orch, batch); + // PA constructs a fresh L0TaskArgs after its two views; Alloc is populated + // without calling reset(). + // 三个 Output 分别成为累计 output/sum/max；Alloc 无 kernel slot，winner 在 + // HeapGuard 后直接发布 task completion。 + ConstructTaskArgs(args); + if (!ReserveTensorArgs(args, 3)) return; + AppendOutput(args, orch.tile_create_info); + AppendOutput(args, orch.scalar_create_info); + AppendOutput(args, orch.scalar_create_info); +} + +PA_DEVICE void BuildQkArgs(PaOrchestrationState &orch, TaskArgs &args, uint32_t batch) { + (void)batch; + // PA computes the block group after Alloc returns, immediately before it + // constructs the dynamic QK output create-info. + // QK 消费 query/key/block-table，产出 score；其 active role 为 AIC。 + PreparePaBlockGroup(orch, 0); + const uint32_t score_shape[kMaxTensorDims] = { + kPaHeads, static_cast(orch.current_nblocks * kPaBlockSize), 0, 0, 0 + }; + // Constructed after Alloc submit and immediately before QK reset/adds. + InitCreateInfo(orch.qk_create_info, score_shape, 2, DataType::Float32); + ResetTaskArgs(args); + if (!ReserveTensorArgs(args, 3)) return; + AppendLocalTensor(args, orch.query_view, TensorArgType::Input); + AppendLocalTensor(args, orch.key_cache, TensorArgType::Input); + AppendLocalTensor(args, orch.block_table, TensorArgType::Input); + AddOutput(args, orch.qk_create_info); + AddTwoScalars( + args, orch.current_nblocks, + static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + orch.current_block_offset + ); +} + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +struct LazySampleCallbackBuildCounts { + uint32_t reset_calls; + uint32_t views_created; + uint32_t dynamic_create_infos; + uint32_t tensor_args_added; + uint32_t scalar_args_added; +}; + +template +class LazySampleCallbackArgsBuilder { +public: + PA_DEVICE LazySampleCallbackArgsBuilder(TaskArgs &args, TaskKind kind, bool won) + : args_(args), kind_(kind), won_(won), begin_calls_(0), counts_{} {} + + PA_DEVICE void Begin() { + if (++begin_calls_ != 1) { + args_.has_error = true; + return; + } + if (kind_ == TaskKind::Alloc) { + ConstructTaskArgs(args_); + } else { + ResetTaskArgs(args_); + ++counts_.reset_calls; + } + } + + PA_DEVICE void RecordView() { ++counts_.views_created; } + PA_DEVICE void RecordDynamicCreateInfo() { ++counts_.dynamic_create_infos; } + + template + PA_DEVICE void AddLocalInput(Thunk thunk) { + if constexpr (Lazy) { + if (!won_) return; + } + if (!Ready()) return; + const TensorDesc &tensor = thunk(); + AddLocalTensor(args_, tensor, TensorArgType::Input); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddGmInput(Thunk thunk) { + if constexpr (Lazy) { + if (!won_) return; + } + if (!Ready()) return; + PA_GM const TensorDesc &tensor = thunk(); + AddGmTensor(args_, tensor, TensorArgType::Input); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddOutput(Thunk thunk) { + if (!Ready()) return; + const TensorCreateInfo &create_info = thunk(); + pa_scheduler::AddOutput(args_, create_info); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddLocalInout(Thunk thunk) { + if (!Ready()) return; + const TensorDesc &tensor = thunk(); + AddLocalTensor(args_, tensor, TensorArgType::Inout); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddGmInout(Thunk thunk) { + if (!Ready()) return; + PA_GM const TensorDesc &tensor = thunk(); + AddGmTensor(args_, tensor, TensorArgType::Inout); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddScalar(Thunk thunk) { + if constexpr (Lazy) { + if (!won_) return; + } + if (!Ready()) return; + pa_scheduler::AddScalar(args_, thunk()); + if (!args_.has_error) ++counts_.scalar_args_added; + } + + PA_DEVICE bool Valid() const { return begin_calls_ == 1 && !args_.has_error; } + PA_DEVICE const LazySampleCallbackBuildCounts &Counts() const { return counts_; } + +private: + PA_DEVICE bool Ready() { + if (begin_calls_ == 1 && !args_.has_error) return true; + args_.has_error = true; + return false; + } + + TaskArgs &args_; + TaskKind kind_; + bool won_; + uint32_t begin_calls_; + LazySampleCallbackBuildCounts counts_; +}; +#endif + +PA_DEVICE void BuildSfArgs(PaOrchestrationState &orch, TaskArgs &args) { + const uint32_t probability_shape[kMaxTensorDims] = { + kPaHeads, static_cast(orch.current_nblocks * kPaBlockSize), 0, 0, 0 + }; + // Constructed only after QK submit returns its sij descriptor. + // SF 通过 qk_scores.owner 得到 QK fanin，产出 probability/max/sum；active role 为 AIV。 + InitCreateInfo(orch.sf_create_info, probability_shape, 2, DataType::Bfloat16); + ResetTaskArgs(args); + AddGmTensor(args, *orch.qk_scores, TensorArgType::Input); + if (!ReserveTensorArgs(args, 3)) return; + AppendOutput(args, orch.sf_create_info); + AppendOutput(args, orch.scalar_create_info); + AppendOutput(args, orch.scalar_create_info); + AddThreeScalars(args, orch.scale_bits, orch.current_nblocks, orch.current_valid_len); +} + +PA_DEVICE void BuildPvArgs(PaOrchestrationState &orch, TaskArgs &args, uint32_t batch) { + (void)batch; + // PV 消费 SF probability 与 value/block-table，owner 形成一条 SF->PV 依赖； + // 结果 pv_output 供最后的 UP 使用，active role 回到 AIC。 + ResetTaskArgs(args); + if (!ReserveTensorArgs(args, 3)) return; + AppendGmTensor(args, *orch.sf_probs, TensorArgType::Input); + AppendLocalTensor(args, orch.value_cache, TensorArgType::Input); + AppendLocalTensor(args, orch.block_table, TensorArgType::Input); + AddOutput(args, orch.tile_create_info); + AddTwoScalars( + args, orch.current_nblocks, + static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + orch.current_block_offset + ); +} + +PA_DEVICE void BuildUpdateArgs(PaOrchestrationState &orch, TaskArgs &args) { + // UP 的 SF max/sum 共享一个 SF owner，PV output 提供一个 PV owner，三个累计 + // Inout 共享 Alloc owner，去重后共 3 条 fanin；output_view 把更新写回当前 batch。 + ResetTaskArgs(args); + if (!ReserveTensorArgs(args, 3)) return; + AppendGmTensor(args, *orch.sf_max, TensorArgType::Input); + AppendGmTensor(args, *orch.sf_sum, TensorArgType::Input); + AppendGmTensor(args, *orch.pv_output, TensorArgType::Input); + if (!ReserveTensorArgs(args, 4)) return; + AppendGmTensor(args, *orch.accumulated_max, TensorArgType::Inout); + AppendGmTensor(args, *orch.accumulated_sum, TensorArgType::Inout); + AppendGmTensor(args, *orch.accumulated_output, TensorArgType::Inout); + AppendLocalTensor(args, orch.output_view, TensorArgType::Inout); + AddTwoScalars( + args, orch.current_block_offset == 0 ? 1 : 0, + orch.current_block_offset + orch.current_nblocks >= orch.current_blocks ? 1 : 0 + ); +} + +PA_DEVICE void AcceptTaskOutputs(PaOrchestrationState &orch, TaskKind kind, const TaskOutputs &outputs) { + // 每个 worker 都完整回放并接收自己 materialize 的 descriptor；只有 Claim winner + // 会执行 kernel，但 loser 的后续 orchestration 仍使用相同 task_id/owner 拓扑。 + switch (kind) { + case TaskKind::Alloc: + orch.accumulated_output = outputs.tensors[0]; + orch.accumulated_sum = outputs.tensors[1]; + orch.accumulated_max = outputs.tensors[2]; + break; + case TaskKind::Qk: + orch.qk_scores = outputs.tensors[0]; + break; + case TaskKind::Sf: + orch.sf_probs = outputs.tensors[0]; + orch.sf_max = outputs.tensors[1]; + orch.sf_sum = outputs.tensors[2]; + break; + case TaskKind::Pv: + orch.pv_output = outputs.tensors[0]; + break; + default: + // UP 只更新既有 Inout，没有新 Output descriptor 需要传给下一阶段。 + break; + } +} + +PA_DEVICE void ResetTensorMap(PA_GM TensorMap &map) { + // TensorMap 完全属于当前 worker，初始化和后续增删都不需要 atomic。bucket 与 + // task_heads 置空后，entry 存储按 high_water 首次分配、再经 free_head 复用。 + map.free_head = -1; + map.high_water = 0; + map.alive_floor = 0; + map.cleaned_upto = 0; + for (uint32_t index = 0; index < kMapBuckets; ++index) { + map.buckets[index] = -1; + } + for (uint32_t index = 0; index < kTaskWindow; ++index) { + map.task_heads[index] = -1; + } +} + +PA_DEVICE uint32_t TensorMapHash(uint64_t address) { + address *= 0x9E3779B97F4A7C15ULL; + return static_cast(address >> (64 - kMapBucketShift)); +} + +template +PA_DEVICE void TensorByteRange(const TensorReference &tensor, uint64_t &address, uint64_t &lo, uint64_t &hi) { + // identity 先按 backing buffer 地址分桶，再用半开字节区间 [lo, hi) 判断 view + // 是否重叠。连续 tensor 由 shape 现算 extent，非连续 tensor 使用缓存 extent。 + const uint64_t element_size = ElementSize(tensor.dtype); + address = tensor.buffer_addr; + lo = tensor.start_offset * element_size; + uint64_t extent; + if (tensor.is_contiguous) { + extent = 1; + for (uint32_t index = 0; index < tensor.ndims; ++index) { + extent *= tensor.shapes[index]; + } + } else { + extent = tensor.extent_elem_cache; + } + hi = (tensor.start_offset + extent) * element_size; +} + +PA_DEVICE int32_t AllocateMapEntry(PA_GM TensorMap &map) { + // 输出为可写 entry 下标：优先复用退休链，之后增长 high_water；返回 -1 表示 + // 固定容量耗尽。分配动作尚未把 entry 接入任何 bucket/task 链。 + if (map.free_head >= 0) { + const int32_t slot = map.free_head; + map.free_head = map.entries[slot].next_in_bucket; + return slot; + } + if (map.high_water < static_cast(kMapCapacity)) { + return map.high_water++; + } + return -1; +} + +PA_DEVICE void FreeMapEntry(PA_GM TensorMap &map, int32_t index) { + // 输入 index 必须仍位于其 bucket 链。输出状态是从双向 bucket 链完整摘除， + // 再把 next_in_bucket 改作 free-list next；task 链由 AdvanceTensorMap 顺序遍历。 + PA_GM MapEntry &entry = map.entries[index]; + if (entry.prev_in_bucket < 0) { + map.buckets[entry.bucket] = entry.next_in_bucket; + } else { + map.entries[entry.prev_in_bucket].next_in_bucket = entry.next_in_bucket; + } + if (entry.next_in_bucket >= 0) { + map.entries[entry.next_in_bucket].prev_in_bucket = entry.prev_in_bucket; + } + entry.bucket = -1; + entry.next_in_bucket = map.free_head; + map.free_head = index; +} + +PA_DEVICE void AdvanceTensorMap(PA_GM TensorMap &map, uint32_t task_id, int32_t heap_window) { + // PrepareMap 在 Claim 前把存活下界推进到 task_id-H。离开窗口的 producer 先按 + // task_heads 找到其全部 entry，再从 bucket 链摘除并进入 free list。TensorMap 与 + // heap 共享窗口宽度 H，但前者按本 worker 的 task_id 推进，后者按跨核连续 + // frontier 推进，二者并不要求同步到达同一位置。 + const int32_t new_floor = static_cast(task_id) - heap_window; + if (new_floor <= map.cleaned_upto) { + if (new_floor > map.alive_floor) { + map.alive_floor = new_floor; + } + return; + } + for (int32_t id = map.cleaned_upto; id < new_floor; ++id) { + int32_t current = map.task_heads[static_cast(id) & kTaskWindowMask]; + while (current >= 0) { + const int32_t next = map.entries[current].next_in_task; + FreeMapEntry(map, current); + current = next; + } + map.task_heads[static_cast(id) & kTaskWindowMask] = -1; + } + map.cleaned_upto = new_floor; + map.alive_floor = new_floor; +} + +template +PA_DEVICE void InsertTensor(PA_GM TensorMap &map, const TensorReference &tensor, int32_t producer) { + // 新 producer 同时插入地址 bucket 的表头与 producer 对应 task 链的表头。 + // map 满时生产语义是静默放弃登记；standalone 保持该行为，不新增异常分支。 + uint64_t address = 0; + uint64_t lo = 0; + uint64_t hi = 0; + TensorByteRange(tensor, address, lo, hi); + const int32_t slot = AllocateMapEntry(map); + if (slot < 0) { + return; + } + const uint32_t bucket = TensorMapHash(address); + PA_GM MapEntry &entry = map.entries[slot]; + entry.buffer_addr = address; + entry.lo = lo; + entry.hi = hi; + entry.producer = producer; + entry.bucket = static_cast(bucket); + entry.prev_in_bucket = -1; + entry.next_in_bucket = map.buckets[bucket]; + if (map.buckets[bucket] >= 0) { + map.entries[map.buckets[bucket]].prev_in_bucket = slot; + } + map.buckets[bucket] = slot; + const uint32_t task_slot = static_cast(producer) & kTaskWindowMask; + entry.next_in_task = map.task_heads[task_slot]; + map.task_heads[task_slot] = slot; +} + +template +PA_DEVICE int32_t LookupTensor(PA_GM const TensorMap &map, const TensorReference &tensor) { + uint64_t address = 0; + uint64_t lo = 0; + uint64_t hi = 0; + TensorByteRange(tensor, address, lo, hi); + int32_t best = -1; + // 同一 buffer 可能存在多个历史写者；只考虑仍存活且区间重叠的 entry，并选择 + // task_id 最大的最新 producer，防止依赖回退到更老版本。 + for (int32_t current = map.buckets[TensorMapHash(address)]; current >= 0; + current = map.entries[current].next_in_bucket) { + PA_GM const MapEntry &entry = map.entries[current]; + if (entry.producer < map.alive_floor) { + continue; + } + if (entry.buffer_addr == address && lo < entry.hi && entry.lo < hi && entry.producer > best) { + best = entry.producer; + } + } + return best; +} + +PA_DEVICE uint64_t TensorOwner(const TaskTensorRef &reference) { + // CreateInfo 只会出现在 tag=Output 且在 fanin 前已被跳过；这里的输入不变量是 + // LocalTensor/GmTensor，输出为显式 owner 或 kInvalidTaskId。 + if (reference.kind == TensorRefKind::GmTensor) { + return reference.pointer.gm_tensor->owner_task_id; + } + return reference.pointer.local_tensor->owner_task_id; +} + +PA_DEVICE int32_t LookupTensorRef(PA_GM const TensorMap &map, const TaskTensorRef &reference) { + // 与 TensorOwner 相同，此辅助入口只接收已存在 descriptor；返回最新重叠 producer， + // 未登记或已退休则返回 -1。 + if (reference.kind == TensorRefKind::GmTensor) { + return LookupTensor(map, *reference.pointer.gm_tensor); + } + return LookupTensor(map, *reference.pointer.local_tensor); +} + +PA_DEVICE void AddFanin(int32_t fanin[kMaxFanin], uint32_t &count, int32_t producer) { + // owner 与 TensorMap lookup 可能指向同一 producer，先去重再写固定 16 槽数组； + // Case1 的最大 fanin 为 UP 的 3，正常路径不会截断。 + if (producer < 0) { + return; + } + for (uint32_t index = 0; index < count; ++index) { + if (fanin[index] == producer) { + return; + } + } + if (count < kMaxFanin) { + fanin[count++] = producer; + } +} + +PA_DEVICE uint32_t CollectFanin( + PA_GM const TensorMap &map, const TaskArgs &args, int32_t fanin[kMaxFanin] +) { + // fanin 只由 winner 收集：先吸收 descriptor 的显式 owner，再对 Input/Inout + // 查询最新重叠写者；纯 Output 尚未存在，不应成为本次 task 的输入依赖。 + uint32_t count = 0; + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = TaskTag(args, static_cast(index)); + if (tag == TensorArgType::Output) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + // Keep the two address spaces in separate control-flow arms. CCEC's + // O2/O3 backend rejects a merged pointer phi even when both arms only + // feed scalar field loads; this is also how PA's production helper is + // written. + // 分支重复是后端约束与生产写法的一部分，不应抽成一个混合地址空间指针。 + if (reference.kind == TensorRefKind::GmTensor) { + PA_GM const TensorDesc &tensor = *reference.pointer.gm_tensor; + const uint64_t owner = tensor.owner_task_id; + if (owner != kInvalidTaskId) { + AddFanin(fanin, count, static_cast(owner & 0xFFFFFFFFU)); + } + if (tag == TensorArgType::Input || tag == TensorArgType::Inout) { + AddFanin(fanin, count, LookupTensor(map, tensor)); + } + } else { + const TensorDesc &tensor = *reference.pointer.local_tensor; + const uint64_t owner = tensor.owner_task_id; + if (owner != kInvalidTaskId) { + AddFanin(fanin, count, static_cast(owner & 0xFFFFFFFFU)); + } + if (tag == TensorArgType::Input || tag == TensorArgType::Inout) { + AddFanin(fanin, count, LookupTensor(map, tensor)); + } + } + } + return count; +} + +PA_DEVICE void InsertExistingTensor(SubmitContext &context, const TaskArgs &args, int32_t index) { + // 输入 index 来自 register_mask，故必为已有 descriptor 而非 CreateInfo；写入结果 + // 只影响 context.self 对应 worker 的 map，并把当前 task_id 登记为新的 hazard 版本。 + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind == TensorRefKind::GmTensor) { + InsertTensor(context.self->map, *reference.pointer.gm_tensor, context.task_id); + } else { + InsertTensor(context.self->map, *reference.pointer.local_tensor, context.task_id); + } +} + +PA_DEVICE void RegisterOutputs(SubmitContext &context, const TaskArgs &args, bool include_existing) { + // register_mask 只覆盖 Inout/OutputExisting。新 Output 已带本次 owner；现有 + // backing buffer 的新写者则必须登记到本 worker TensorMap，供后继 task 查 hazard。 + if (!include_existing) { + return; + } + uint32_t register_mask = context.register_mask; + for (uint32_t index = 0; register_mask != 0; ++index, register_mask >>= 1) { + if ((register_mask & 1U) != 0) { + InsertExistingTensor(context, args, static_cast(index)); + } + } +} + +PA_DEVICE uint64_t FrontendAlignUp(uint64_t value, uint64_t alignment) { + // alignment 在本模型中固定为2的幂1 KiB；返回逻辑 heap 地址，不做 ring 取模。 + return (value + alignment - 1) & ~(alignment - 1); +} + +PA_DEVICE void BeginSubmit( + PA_GM WorkerState &worker, const TaskArgs &args, SubmitContext &context +) { + // Mirrors dist_submit_begin(). The production Submit and Materialize spans + // both start after this per-call context initialization. + // local_index 在所有 worker 上按同一 orchestration 顺序递增，因此 + // task_id 一致；该初始化位于 Submit 计时起点之前，不能误计进阶段耗时。 + const uint32_t task_id = static_cast(worker.local_index++); + context.self = &worker; + context.payload = &worker.payloads[task_id & kPayloadMask]; + context.task_id = static_cast(task_id); + context.tensor_count = args.tensor_count; + context.scalar_count = args.scalar_count; + context.result.task_id = task_id; + context.result.count = 0; + context.register_mask = 0; + context.output_bytes = 0; + context.fanin_count = 0; + context.kernel_id = -1; + context.won = false; + context.joint = false; + context.joint_init = false; + context.joint_block = -1; + context.joint_slot = -1; + context.joint_count = 0; +} + +PA_DEVICE bool MaterializeTask( + PA_GM WorkerState &worker, uint32_t task_id, const TaskArgs &args, SubmitContext &context, + uint64_t heap_base, uint64_t heap_size +) { + // 输入是 BeginSubmit 已绑定的 payload/context 与当前 worker.heap_next；成功输出 + // 包括本 task 的 GM TensorDesc 指针、output_bytes 和推进后的单调 heap_next。 + // 失败不得进入 slot/build 流程，由上层设置 fatal 并终止该 worker 回放。 + // legacy 路径在 Claim 前物化；claim-first callback 实验则可能已经完成 Claim， + // 因而不能把“尚未 Claim”写成这个共用 helper 的普遍前置条件。 + if (context.payload == nullptr) { + return false; + } + context.tensor_count = args.tensor_count; + context.scalar_count = args.scalar_count; + context.register_mask = 0; + + // DistOutputLayout leaves non-output slots lazy and writes only the sizes + // selected by output_mask. + // 第一次 tag 扫描同时产生 output_mask/register_mask；第二次只遍历 + // Output 位，避免读取未初始化的非输出 buffer_sizes。 + OutputLayout layout; + layout.total_output_size = 0; + uint32_t output_mask = 0; + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = TaskTag(args, static_cast(index)); + if (tag == TensorArgType::Inout || tag == TensorArgType::OutputExisting) { + context.register_mask |= 1U << index; + } + if (tag != TensorArgType::Output) { + continue; + } + output_mask |= 1U << index; + layout.buffer_sizes[index] = CreateInfoBytes(*args.tensors[index].pointer.create_info); + layout.total_output_size += FrontendAlignUp(layout.buffer_sizes[index], kOutputAlignment); + } + + uint64_t task_base = FrontendAlignUp(worker.heap_next, kOutputAlignment); + const uint64_t total = layout.total_output_size; + if (total > heap_size || (total != 0 && heap_base == 0)) { + return false; + } + if (total != 0 && (task_base % heap_size) + total > heap_size) { + // 单个 task 的输出必须物理连续；若跨 ring 尾部则把逻辑 task_base 推到 + // 下一圈起点。heap_next 仍保持单调，不在这里取模。 + task_base = (task_base / heap_size + 1) * heap_size; + } + + uint64_t output_offset = 0; + // 各 Output 在同一 task_base 内按参数顺序排布；result 只收集 Output，索引与 + // TaskArgs 中非输出槽无关，而 payload 仍按原参数 index 保存 descriptor。 + for (int32_t index = 0; output_mask != 0; ++index, output_mask >>= 1) { + if ((output_mask & 1U) == 0) { + continue; + } + const uint64_t physical = (task_base + output_offset) % heap_size; + PA_GM TensorDesc &tensor = context.payload->tensors[index]; + if (!InitTensorFromCreateInfo( + tensor, *args.tensors[index].pointer.create_info, heap_base + physical, layout.buffer_sizes[index] + )) { + return false; + } + tensor.owner_task_id = task_id; + context.result.tensors[context.result.count++] = &tensor; + output_offset += FrontendAlignUp(layout.buffer_sizes[index], kOutputAlignment); + } + worker.heap_next = task_base + total; + context.output_bytes = total; + return true; +} + +PA_DEVICE void CopyTensorFromRef(PA_GM TensorDesc &destination, const TaskTensorRef &reference) { + // slot 必须拥有 descriptor 快照，不能保存指向 orchestration 栈对象的引用； + // 按 byte volatile copy 同时兼容 local/GM 源并保留真实 128-byte 搬运量。 + PA_GM volatile uint8_t *destination_bytes = reinterpret_cast(&destination); + if (reference.kind == TensorRefKind::GmTensor) { + PA_GM const volatile uint8_t *source_bytes = + reinterpret_cast(reference.pointer.gm_tensor); + for (uint32_t byte = 0; byte < sizeof(TensorDesc); ++byte) { + destination_bytes[byte] = source_bytes[byte]; + } + return; + } + const volatile uint8_t *source_bytes = + reinterpret_cast(reference.pointer.local_tensor); + for (uint32_t byte = 0; byte < sizeof(TensorDesc); ++byte) { + destination_bytes[byte] = source_bytes[byte]; + } +} + +PA_DEVICE void CopyGmTensor(PA_GM TensorDesc &destination, PA_GM const TensorDesc &source) { + // 新 Output 的源 descriptor 已位于 GM payload；单独入口避免把 GM 指针误走 + // local 地址空间分支，输出仍是 slot 内独立副本。 + PA_GM volatile uint8_t *destination_bytes = reinterpret_cast(&destination); + PA_GM const volatile uint8_t *source_bytes = reinterpret_cast(&source); + for (uint32_t byte = 0; byte < sizeof(TensorDesc); ++byte) { + destination_bytes[byte] = source_bytes[byte]; + } +} + +PA_DEVICE void PopulateSlotPayload( + PA_GM LocalSlot &slot, const TaskArgs &args, const SubmitContext &context, const int32_t fanin[kMaxFanin], + uint32_t fanin_count, int32_t sub_block_id, bool is_multicore, int32_t won_block, int32_t won_slot +) { + // winner 将活动 descriptor/scalar 复制进私有 slot，dispatch args 指向 slot 内 + // 副本而非 orchestration 临时对象；fanin 随 slot 保存，kernel 执行前逐 flag 检查。 + slot.tensor_count = context.tensor_count; + slot.scalar_count = context.scalar_count; + for (int32_t index = 0; index < context.tensor_count; ++index) { + if (TaskTag(args, static_cast(index)) == TensorArgType::Output) { + CopyGmTensor(slot.tensors[index], context.payload->tensors[index]); + } else { + CopyTensorFromRef(slot.tensors[index], args.tensors[index]); + } + slot.args[index] = static_cast(reinterpret_cast(&slot.tensors[index])); + } + for (int32_t index = 0; index < context.scalar_count; ++index) { + slot.scalars[index] = args.scalars[index]; + slot.args[context.tensor_count + index] = args.scalars[index]; + } + + PA_GM PaLocalContext &local = + *reinterpret_cast(&slot.local_context[0]); + // standalone 每个 task 只由一个 lane kernel 执行，故 block_index/count 固定0/1； + // async completion 未启用，task_token 保持 invalid，与 PA 普通同步 slot 一致。 + local.block_index = 0; + local.block_count = 1; + local.async.completion_count = 0; + local.async.completion_error_code = 0; + local.async.completion_entries = 0; + local.async.completion_capacity = 0; + local.async.task_token = kInvalidTaskId; + slot.global_context = static_cast(sub_block_id); + slot.args[kSpmdLocalContextIndex] = + static_cast(reinterpret_cast(&slot.local_context[0])); + slot.args[kSpmdGlobalContextIndex] = + static_cast(reinterpret_cast(&slot.global_context)); + slot.fanin_count = fanin_count; + // fanin 数组只复制有效前缀；执行端以 fanin_count 为边界，未使用尾部保持惰性。 + for (uint32_t index = 0; index < fanin_count; ++index) { + slot.fanin[index] = fanin[index]; + } + slot.is_multicore = is_multicore; + slot.won_block = won_block; + slot.won_slot = won_slot; +} + +PA_DEVICE void BuildSlotPayload( + PA_GM LocalSlot &slot, uint32_t task_id, uint32_t function_id, uint64_t function_address, const TaskArgs &args, + const SubmitContext &context, const int32_t fanin[kMaxFanin], uint32_t fanin_count, int32_t sub_block_id = 0, + bool is_multicore = false, int32_t won_block = -1, int32_t won_slot = -1 +) { + // Match build_ring_slot_from_submit ordering: publish the header first, + // then copy the active descriptors/scalars and construct dispatch payload. + // slot 仅由所属 worker 消费，这里的写入次序用于复刻真实构建成本与 + // 状态机；跨核可见性由 task completion 的 flag/vend 协议承担。 + slot.occupied = true; + slot.task_id = task_id; + slot.kind = function_id; + slot.function_address = function_address; + slot.built = 1; + PopulateSlotPayload( + slot, args, context, fanin, fanin_count, sub_block_id, is_multicore, won_block, won_slot + ); +} + +// Compatibility overload for a core that has already populated the slot +// header before calling the PA frontend. +// 该入口只补 payload，不改变既有 task/function 头；输出不变量与完整 +// BuildSlotPayload 相同，均得到 built 且可由 DrainReady 检查 fanin 的私有 slot。 +PA_DEVICE void BuildSlotPayload( + PA_GM LocalSlot &slot, const TaskArgs &args, const SubmitContext &context, const int32_t fanin[kMaxFanin], + uint32_t fanin_count +) { + slot.built = 1; + PopulateSlotPayload(slot, args, context, fanin, fanin_count, 0, false, -1, -1); +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_FRONTEND_H diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/common/pa_model.h b/tests/atomic_probe/lazy_lamda_sample/A_original/common/pa_model.h new file mode 100644 index 0000000000..0c74aa2607 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/common/pa_model.h @@ -0,0 +1,941 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_MODEL_H +#define PA_SCHEDULER_COMMON_PA_MODEL_H + +#include +#include + +// CCEC 的正式产物只允许二选一：swimlane 保存普通阶段与 atomic 记录， +// submit-pmu 则编译掉泳道观察代码。CPU/AscendC 未传这些宏时继续使用原有 +// 通用实现，避免公共模型反向依赖某个后端的构建脚本。 +#ifndef PA_BUILD_SWIMLANE +#define PA_BUILD_SWIMLANE 0 +#endif + +#ifndef PA_BUILD_SUBMIT_PMU +#define PA_BUILD_SUBMIT_PMU 0 +#endif + +#if PA_BUILD_SWIMLANE && PA_BUILD_SUBMIT_PMU +#error "PA_BUILD_SWIMLANE and PA_BUILD_SUBMIT_PMU are mutually exclusive" +#endif + +namespace pa_scheduler { + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +// The original artifact is compiled without this selector. The two selected +// artifacts deliberately share one compete-first/split-finish skeleton; only +// the builder's loser replay policy differs. +static_assert( + PA_LAZY_SAMPLE_SHAPE_ID == 1 || PA_LAZY_SAMPLE_SHAPE_ID == 2, + "PA_LAZY_SAMPLE_SHAPE_ID must select compete-first(1) or compete-first-lazy(2)" +); +constexpr bool kLazySampleLazy = PA_LAZY_SAMPLE_SHAPE_ID == 2; +constexpr bool kLazySampleSplitFinish = true; +constexpr uint32_t kLazySampleShapeId = PA_LAZY_SAMPLE_SHAPE_ID; +constexpr const char *kLazySampleShapeName = + PA_LAZY_SAMPLE_SHAPE_ID == 1 ? "compete-first" : "compete-first-lazy"; +constexpr const char *kLazySampleObservation = "split-combination-semantic"; +constexpr const char *kLazySampleFinishShape = "noinline-cross-tu"; +constexpr const char *kLazySampleControlFamily = "all-task-compete-first-callback"; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +static_assert(PA_LAZY_SAMPLE_SPLIT_FINISH == 1, "split-finish feature macro must equal one"); +static_assert(kLazySampleSplitFinish, "both lazy-sample callback shapes require split finish"); +#else +#error "lazy-sample callback artifacts require PA_LAZY_SAMPLE_SPLIT_FINISH" +#endif +#endif + +// 这里固定的是 PA Case1 的调度拓扑，而不是为了缩小 standalone 人为选择的规模： +// 每个 batch 依次回放 Alloc/QK/SF/PV/UP 五个 task，32 个 AIC 与 64 个 AIV +// 都执行同一条 orchestration 流，只在 Claim 时按 task 的 active role 分流。 +constexpr uint32_t kDefaultBatches = 256; +constexpr uint32_t kMaxBatches = 256; +constexpr uint32_t kTasksPerBatch = 5; +constexpr uint32_t kMaxTasks = kMaxBatches * kTasksPerBatch; +constexpr uint32_t kTaskCellCapacity = 1U << 16; + +constexpr uint32_t kAicWorkers = 32; +constexpr uint32_t kAivWorkers = 64; +constexpr uint32_t kWorkers = kAicWorkers + kAivWorkers; +constexpr uint32_t kRuntimeMaxWorkers = 108; +constexpr uint32_t kCursorShards = 4; +// 每个 worker 私有 ring 有 4 个物理 slot，其中 2 个为 BlockWon 协议预留； +// 单 lane Case1 虽不进入 BlockWon，普通 kernel 仍只能占用剩余 2 个 slot。 +constexpr uint32_t kPrivateSlots = 4; +constexpr uint32_t kWonReserve = 2; +constexpr uint32_t kUsableSlots = kPrivateSlots - kWonReserve; +constexpr uint32_t kMaxFanin = 16; +// H=64 同时约束 heap 可回收 frontier 和 TensorMap producer 的存活下界。 +// heap_next 使用单调逻辑地址；真正落到 256 MiB 环形 heap 时才取模，因而可判断覆盖风险。 +constexpr uint32_t kHeapWindow = 64; +constexpr uint64_t kHeapBytes = 256ULL << 20; +constexpr uint64_t kSyntheticHeapBase = 0x100000000ULL; +constexpr uint64_t kOutputAlignment = 1024; +constexpr uint32_t kMaxTensorDims = 5; +constexpr uint32_t kMaxTaskTensors = 32; +constexpr uint32_t kMaxTaskScalars = 16; +constexpr uint32_t kPayloadSlots = 2048; +constexpr uint32_t kPayloadMask = kPayloadSlots - 1; +constexpr uint32_t kPayloadStride = 4096; +constexpr uint32_t kMapCapacity = 16384; +constexpr uint32_t kMapBuckets = 1 << 13; +constexpr uint32_t kMapBucketShift = 13; +constexpr uint32_t kTaskWindow = 1 << 10; +constexpr uint32_t kTaskWindowMask = kTaskWindow - 1; +constexpr uint64_t kSystemCounterHz = 1000000000ULL; +constexpr uint64_t kWatchdogTicks = 2 * kSystemCounterHz; +// trace_enabled 是位图而不是 bool：bit0 保持既有阶段泳道，bit1 额外开启 +// 逐条 atomic 源码括号记录。atomic 记录依赖同一份 trace buffer，因此 bit1 +// 只能与 bit0 一起配置。 +constexpr uint32_t kTracePhasesEnabled = 1U << 0; +constexpr uint32_t kTraceAtomicsEnabled = 1U << 1; +// Claim trace flags 是独立 raw ABI：bit0 表示获胜，bit1 表示已经通过 +// AIC/AIV role 路由并真正执行 atomicMax。未 attempted 的 Claim 仍保留 +// role-selection 开销，但转换器会明确标成 claim.not_attempted。 +constexpr uint32_t kClaimWon = 1U << 0; +constexpr uint32_t kClaimAttempted = 1U << 1; +// 下列 offset/size 来自真实 DistGlobal/DistCore ABI。standalone 保留被测关键字段的 +// offset、DistCore ABI 和 kRealDistGlobalBytes 总跨度；其余区域可用 opaque padding， +// 并不是对生产结构全部字段的逐一镜像。 +constexpr size_t kRealDistCoreOffset = 10043904; +constexpr size_t kRealDistGlobalBytes = 1007023872; +constexpr size_t kRealTasksOffset = 896; +constexpr size_t kRealFatalOffset = 4195264; +constexpr size_t kRealReplayDoneOffset = 10043776; +constexpr size_t kRealStartedCountOffset = 10043840; +constexpr uint32_t kTraceRecordsPerCore = 1U << 16; +static_assert((kPayloadSlots & kPayloadMask) == 0, "payload slots must be a power of two"); +static_assert(kMaxTasks < kTaskCellCapacity, "every frontier scan must terminate on an in-range not-ready flag"); + +// These are the measured means from the best PA A5 trace, in 1 GHz ticks. +// The scalar-NOP compatibility baseline calibrates its counts against these targets. +// 无参数默认使用 real-compute：CCEC/AscendC 执行完整 Cube/Vector 流水， +// CPU 执行对等算术；下列 NOP 常量只供显式 scalar-nop 校准。两种模式的 +// Submit、依赖、heap 与 completion 路径都不靠补时修改。target 是真实泳道 +// 均值，不是调度阶段预算。 +constexpr uint32_t kTargetQkTicks = 44170; +constexpr uint32_t kTargetSfTicks = 53729; +constexpr uint32_t kTargetPvTicks = 27626; +constexpr uint32_t kTargetUpTicks = 1565; + +// Calibrated on the local A5 with the CCEC RuntimeNop implementation. These +// counts resolve to the measured targets above; they are not cycle guesses. +constexpr uint32_t kDefaultQkNops = 129600; +constexpr uint32_t kDefaultSfNops = 157900; +constexpr uint32_t kDefaultPvNops = 79950; +constexpr uint32_t kDefaultUpNops = 2400; + +enum class CoreRole : uint32_t { + Aic = 0, + Aiv = 1, +}; + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +constexpr uint64_t kLazySampleSplitStateCookieBase = 0x514b53504c495400ULL; +#endif + +// task_id % 5 即 kind；该周期性是不变量，既决定 Claim cursor/active role， +// 也决定输出大小、fanin 拓扑和 winner workload 的选择。 +enum class TaskKind : uint32_t { + Alloc = 0, + Qk = 1, + Sf = 2, + Pv = 3, + Up = 4, + Count = 5, +}; + +// 记录 kernel 最终在哪次 drain 中落地：Submit 开头、slot/heap 背压期间，或 +// 所有 worker 回放结束后的最终清空。三者之和必须等于实际 kernel 数。 +enum class DrainPlace : uint32_t { + EfDrain = 0, + RingBackpressure = 1, + FinalDrain = 2, + Count = 3, +}; + +enum class TensorArgType : int32_t { + Input = 0, + Output = 1, + Inout = 2, + OutputExisting = 3, + NoDependency = 4, +}; +// Input 作为 kernel 输入并参与依赖、但不登记为写者；Output 由本次 Submit 在 heap 中物化； +// Inout 与 OutputExisting 还需登记进每 worker 私有 TensorMap，供后续重叠区间查询 producer。 +static_assert(sizeof(TensorArgType) == sizeof(int32_t), "TensorArgType must match the PA tag ABI"); + +enum class DataType : uint8_t { + Float32 = 0, + Float16 = 1, + Int32 = 2, + Int16 = 3, + Int8 = 4, + Uint8 = 5, + Bfloat16 = 6, + Int64 = 7, + Uint64 = 8, + Uint16 = 9, + Uint32 = 10, + Bool = 11, + Count = 12, +}; + +// ProfilePhase 是聚合计数下标，TracePhase 是原始泳道事件 ABI；二者故意分离， +// 不能假设枚举值相同。一次 trace 写入可同时归入一个不同命名的 profile 阶段。 +enum class ProfilePhase : uint32_t { + Orchestration = 0, + Submit = 1, + EfDrain = 2, + Materialize = 3, + PrepareMap = 4, + Claim = 5, + Fanin = 6, + Register = 7, + WaitForSlot = 8, + HeapGuard = 9, + Build = 10, + ReplayTail = 11, + Count = 12, +}; + +// submit-pmu 每个 ELF 只编译一个局部归因阶段。none 不做中途 counter +// 读取，是完整 Submit 的正式基线；其余阶段都在每个 worker 的五次 Submit +// 上各执行一次，因此统一按固定 5*batches 次数闭合。历史 ID=3 曾用于 +// winner-only WaitForSlot，现已退役且不复用，避免旧 raw 被误认成新阶段。 +enum class SubmitPmuPhase : uint32_t { + None = 0, + Claim = 1, + EfDrain = 2, + Materialize = 4, + Register = 5, + Count = 6, +}; + +#ifndef PA_SUBMIT_PMU_PHASE_ID +#define PA_SUBMIT_PMU_PHASE_ID 0 +#endif + +constexpr SubmitPmuPhase kCompiledSubmitPmuPhase = + static_cast(PA_SUBMIT_PMU_PHASE_ID); +constexpr uint32_t kBuildVariantSwimlane = 1U; +constexpr uint32_t kBuildVariantSubmitPmu = 2U; +static_assert( + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::None) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::Claim) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::EfDrain) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::Materialize) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::Register), + "invalid compiled submit-pmu phase" +); + +struct NopCounts { + uint32_t qk; + uint32_t sf; + uint32_t pv; + uint32_t up; +}; + +// winner 的计算负载与 NOP 校准量使用两套独立计数，禁止把同一个数字同时解释成 +// scalar 指令条数和 vector/cube 工作迭代数。首阶段只有 CCEC 实现 RealCompute； +// 该 ABI 放在公共模型中，便于后续按相同配置逐步迁移 AscendC 与 CPU。 +struct WorkloadCounts { + uint32_t qk; + uint32_t sf; + uint32_t pv; + uint32_t up; +}; +static_assert(sizeof(WorkloadCounts) == 16, "workload counts ABI changed"); + +enum class WinnerWorkloadMode : uint32_t { + ScalarNop = 0, + RealCompute = 1, +}; + +constexpr uint32_t kWinnerWorkloadConfigVersion = 1; + +// 真实计算工作区是 standalone sidecar，不属于生产 DistGlobal/DistCore ABI。 +// workspace_base 指向 host 单独申请并初始化的 GM；每个 worker 只写自己的输出片段。 +struct alignas(64) WinnerWorkloadConfig { + uint32_t mode; + uint32_t version; + WorkloadCounts repeats; + uint64_t workspace_base; + uint64_t workspace_bytes; + uint32_t reserved[6]; +}; +static_assert(sizeof(WinnerWorkloadConfig) == 64, "winner workload config must occupy one cache line"); +static_assert(offsetof(WinnerWorkloadConfig, mode) == 0, "winner workload mode offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, version) == 4, "winner workload version offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, repeats) == 8, "winner workload counts offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, workspace_base) == 24, "winner workload base offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, workspace_bytes) == 32, "winner workload bytes offset changed"); + +// RunConfig 是 host 在 launch 前写、worker 启动时只读的控制 cache line。 +// 输入为 batch/NOP/诊断开关；输出不回写这里，而发布到独立 WorkerResult。 +struct alignas(64) RunConfig { + uint32_t batches; + uint32_t workers; + NopCounts nops; + uint32_t profile_phases; + uint32_t trace_enabled; + uint64_t trace_base; + uint32_t trace_records_per_core; + uint32_t reserved[5]; +}; +static_assert(sizeof(RunConfig) == 64, "RunConfig must occupy one cache line"); + +enum class TracePhase : int32_t { + Kernel = 0, + Alloc = 1, + Build = 2, + DrainWon = 3, + Replay = 4, + RingBp = 5, + EfDrain = 6, + Commit = 7, + Submit = 8, + Materialize = 9, + PrepareMap = 10, + Claim = 11, + Fanin = 12, + Register = 13, + Atomic = 14, + // 逐 atomic 诊断构建中，每个 worker 只记录一次连续两次 SYS_CNT 的 + // 空括号，用来给出同一二进制、同一物理核上的计时分辨率下限。 + ClockBaseline = 15, + // schema-v4 追加的父区间与真实动作区间。loser 没有可单列的真实动作， + // 其时间直接归入离线计算的 Submit residual，不占用 raw 记录。 + OrchestrationReplay = 16, + FinalDrain = 17, + WinnerBuild = 18, + AllocComplete = 19, + Count = 20, +}; + +// AtomicSite 按 standalone PA 中真实出现的源码调用点分类。编号写入 TraceRecord::auxiliary， +// 是离线泳道 schema 的一部分；追加新位置只能在 Count 前扩展，不能重排既有值。 +enum class AtomicSite : uint32_t { + StartupIncrement = 0, + StartupPoll = 1, + FatalPoll = 2, + FatalSet = 3, + ClaimMax = 4, + FaninFlagLoad = 5, + CompletionVendExchange = 6, + CompletionFlagExchange = 7, + FrontierInitialLoad = 8, + FrontierFlagLoad = 9, + FrontierMax = 10, + HeapFrontierLoad = 11, + HeapVendLoad = 12, + ReplayDoneIncrement = 13, + ReplayDonePoll = 14, + Count = 15, +}; + +// Atomic 记录 flags 的低四位保存操作种类；bit4 表示返回值参与后续判断， +// bit5 表示 Load 观察到零，bit6 表示结束时间已由返回值依赖推进到 +// return-ready 边界。schema-v3 中 bit7 区分等待区 PollBatch：此时 +// bits[31:8] 是精确调用次数；直接 FetchMax 中同一区域仍表示软件重试数。 +enum class AtomicOp : uint32_t { + Load = 0, + Exchange = 1, + FetchAdd = 2, + FetchMax = 3, +}; +constexpr uint32_t kAtomicOpMask = 0x0fU; +constexpr uint32_t kAtomicResultUsed = 1U << 4; +constexpr uint32_t kAtomicValueZero = 1U << 5; +constexpr uint32_t kAtomicReturnReady = 1U << 6; +constexpr uint32_t kAtomicPollBatch = 1U << 7; +constexpr uint32_t kAtomicRetriesShift = 8; +constexpr uint32_t kAtomicPollCountShift = 8; +constexpr uint32_t kAtomicPollCountMax = 0x00ffffffU; +constexpr uint32_t kAtomicPollBatchSiteCount = 6; + +// 这些映射是 raw ABI 的一部分，同时被 device 聚合器与 host 闭环校验使用。 +// 0..14 与真实 PA 保持稳定；BlockWon 尚未在 standalone 中实现，不能只为 +// 编号齐全而追加没有真实调用路径的 site。 +#ifdef PA_DEVICE +#define PA_MODEL_INLINE PA_DEVICE +#else +#define PA_MODEL_INLINE inline +#endif + +PA_MODEL_INLINE constexpr AtomicOp AtomicSiteExpectedOp(AtomicSite site) { + switch (site) { + case AtomicSite::StartupIncrement: + case AtomicSite::ReplayDoneIncrement: + return AtomicOp::FetchAdd; + case AtomicSite::FatalSet: + case AtomicSite::CompletionVendExchange: + case AtomicSite::CompletionFlagExchange: + return AtomicOp::Exchange; + case AtomicSite::ClaimMax: + case AtomicSite::FrontierMax: + return AtomicOp::FetchMax; + default: + return AtomicOp::Load; + } +} + +PA_MODEL_INLINE constexpr bool AtomicSiteResultUsed(AtomicSite site) { + switch (site) { + case AtomicSite::StartupIncrement: + case AtomicSite::FatalSet: + case AtomicSite::CompletionVendExchange: + case AtomicSite::CompletionFlagExchange: + case AtomicSite::ReplayDoneIncrement: + return false; + default: + return true; + } +} + +PA_MODEL_INLINE constexpr int32_t AtomicPollBatchIndex(AtomicSite site) { + switch (site) { + case AtomicSite::StartupPoll: + return 0; + case AtomicSite::FatalPoll: + return 1; + case AtomicSite::FaninFlagLoad: + return 2; + case AtomicSite::HeapFrontierLoad: + return 3; + case AtomicSite::HeapVendLoad: + return 4; + case AtomicSite::ReplayDonePoll: + return 5; + default: + return -1; + } +} + +PA_MODEL_INLINE constexpr AtomicSite AtomicPollBatchSite(uint32_t index) { + switch (index) { + case 0: + return AtomicSite::StartupPoll; + case 1: + return AtomicSite::FatalPoll; + case 2: + return AtomicSite::FaninFlagLoad; + case 3: + return AtomicSite::HeapFrontierLoad; + case 4: + return AtomicSite::HeapVendLoad; + case 5: + return AtomicSite::ReplayDonePoll; + default: + return AtomicSite::Count; + } +} + +PA_MODEL_INLINE constexpr bool AtomicSiteIsPollBatchable(AtomicSite site) { + return AtomicPollBatchIndex(site) >= 0; +} + +PA_MODEL_INLINE constexpr uint32_t AtomicSiteMask(AtomicSite site) { + return 1U << static_cast(site); +} + +#undef PA_MODEL_INLINE + +// ClockBaseline 的 bit0 区分普通连续 SYS_CNT 与后端的 atomic 返回依赖 +// 计时钩子；后者用于量化那一条依赖 MOV 自身带来的固定底噪。 +constexpr uint32_t kClockAtomicDependency = 1U << 0; +constexpr uint32_t kClockAtomicDependencyApplied = 1U << 1; + +struct alignas(64) TraceCoreState { + volatile uint32_t count; + volatile uint32_t dropped; + // logical atomic 调用数与物理记录数分开闭合：PollBatch 的一条记录可以 + // 表示多次只读轮询，physical = atomic_calls - poll_calls + batch_records。 + volatile uint32_t atomic_calls; + volatile uint32_t poll_calls; + volatile uint32_t poll_batch_records; + // 拓扑在一个 worker 分区内恒定；当前仍保留 64B TraceRecord 兼容布局， + // 但在 core state 再保存一份权威身份，host 会验证每条记录与之相符。 + volatile int32_t core_idx; + volatile int32_t block_id; + volatile int32_t lane; + uint32_t padding[8]; +}; +// 每个 worker 独占一个计数 cache line 和一段定长 records，不需要为了写 trace +// 再引入跨核 atomic；满容量后只增加本 worker 的 dropped。 +static_assert(sizeof(TraceCoreState) == 64, "trace core state must occupy one cache line"); + +struct alignas(64) TraceHeader { + uint32_t magic; + uint32_t version; + uint32_t num_cores; + uint32_t records_per_core; + uint64_t frequency_hz; + TraceCoreState cores[kRuntimeMaxWorkers]; +}; +// 本 benchmark 固定物理分配 kWorkers=96 个定长 record 分区，合法 header 也要求 +// num_cores==96；其 header/record 布局和 phase 编号可转换为真实泳道使用的 JSON。 +static_assert(sizeof(TraceHeader) == 6976, "trace header must match PA swimlane layout"); + +struct alignas(64) TraceRecord { + uint64_t start_cycle; + uint64_t end_cycle; + int32_t task_id; + int32_t function_id; + int32_t phase; + int32_t lane; + int32_t block_id; + int32_t core_idx; + uint32_t flags; + uint32_t auxiliary; +}; +// start/end 保留原始 1 GHz counter；task/function/物理 lane 用于离线还原轨道。 +// flags/aux 的含义由 phase 决定，例如 winner、Alloc 或 RingBp 类型，不参与调度决策。 +static_assert(sizeof(TraceRecord) == 64, "trace record must occupy one cache line"); + +constexpr size_t kTraceBytes = + sizeof(TraceHeader) + static_cast(kWorkers) * kTraceRecordsPerCore * sizeof(TraceRecord); + +struct alignas(64) AtomicLine { + volatile int64_t value; + uint8_t padding[64 - sizeof(int64_t)]; +}; +// 热点共享量各占一个 cache line，保持生产代码的地址隔离，避免 standalone +// 因伪共享额外放大 Claim/frontier/start barrier 的竞争。 +static_assert(sizeof(AtomicLine) == 64, "AtomicLine must occupy one cache line"); + +struct alignas(64) AtomicFlagLine { + volatile int32_t value; + uint8_t padding[64 - sizeof(int32_t)]; +}; +// 32-bit fatal 与 64-bit cursor 使用不同封装，但都独占 cache line；成功路径中 +// fatal 始终为零，任何写一都表示协议已终止，不能作为普通等待条件清除。 +static_assert(sizeof(AtomicFlagLine) == 64, "AtomicFlagLine must occupy one cache line"); + +struct alignas(64) TaskCell { + volatile int64_t flag; + volatile uint64_t vend; + uint8_t padding[64 - 2 * sizeof(int64_t)]; +}; +// flag 是依赖就绪与 frontier 连续前推的发布位；vend 是该 task 完成时 worker 的 +// 单调 heap_next 快照。HeapGuard 读取 frontier-H 对应 vend，判断环形 heap 是否可覆盖。 +static_assert(sizeof(TaskCell) == 64, "TaskCell must occupy one cache line"); + +// TensorDesc 保留真实 Tensor 的两条 64-byte 数据线。owner_task_id 表达显式生产者， +// buffer_addr + 字节区间用于 TensorMap 发现同一 backing buffer 上的读写依赖。 +struct TensorDesc { + uint64_t buffer_addr; + uint64_t buffer_size; + uint64_t owner_task_id; + uint64_t start_offset; + int32_t version; + uint32_t ndims; + DataType dtype; + bool manual_dep; + bool is_contiguous; + uint8_t child_memory; + uint32_t shapes[kMaxTensorDims]; + + uint64_t extent_elem_cache; + uint32_t strides[kMaxTensorDims]; + uint8_t padding[36]; +}; +static_assert(sizeof(TensorDesc) == 128, "TensorDesc must match the PA Tensor ABI size"); +static_assert(offsetof(TensorDesc, buffer_addr) == 0, "TensorDesc buffer offset mismatch"); +static_assert(offsetof(TensorDesc, owner_task_id) == 16, "TensorDesc owner offset mismatch"); +static_assert(offsetof(TensorDesc, start_offset) == 24, "TensorDesc view offset mismatch"); +static_assert(offsetof(TensorDesc, version) == 32, "TensorDesc version offset mismatch"); +static_assert(offsetof(TensorDesc, shapes) == 44, "TensorDesc shape offset mismatch"); +static_assert(offsetof(TensorDesc, extent_elem_cache) == 64, "TensorDesc extent offset mismatch"); +static_assert(offsetof(TensorDesc, strides) == 72, "TensorDesc stride offset mismatch"); + +struct TensorCreateInfo { + uint64_t initial_value; + bool has_initial_value; + uint8_t padding0[7]; + uint64_t reserved0; + uint64_t start_offset; + int32_t version; + uint32_t ndims; + DataType dtype; + bool manual_dep; + bool is_contiguous; + uint8_t child_memory; + uint32_t shapes[kMaxTensorDims]; +}; +// CreateInfo 只描述尚未分配的 Output；Materialize 根据形状和 dtype 计算大小， +// 再把它变成位于 worker 逻辑 heap 上的 TensorDesc。 +static_assert(sizeof(TensorCreateInfo) == 64, "TensorCreateInfo must match the PA create-info ABI size"); +static_assert(offsetof(TensorCreateInfo, start_offset) == 24, "TensorCreateInfo start offset mismatch"); +static_assert(offsetof(TensorCreateInfo, version) == 32, "TensorCreateInfo version offset mismatch"); +static_assert(offsetof(TensorCreateInfo, shapes) == 44, "TensorCreateInfo shape offset mismatch"); + +struct MapEntry { + uint64_t buffer_addr; + uint64_t lo; + uint64_t hi; + int32_t producer; + int32_t bucket; + int32_t next_in_bucket; + int32_t prev_in_bucket; + int32_t next_in_task; +}; +// 同一 entry 同时挂在两条链上：bucket 链按 buffer 地址查询重叠区间，task 链按 +// producer 批量退休。next_in_bucket 在空闲状态下复用为 free-list 链接。 +static_assert(sizeof(MapEntry) == 48, "MapEntry must match the PA tensor-map entry ABI"); +static_assert(offsetof(MapEntry, producer) == 24, "MapEntry producer offset mismatch"); +static_assert(offsetof(MapEntry, next_in_task) == 40, "MapEntry task-link offset mismatch"); + +struct TensorMap { + MapEntry entries[kMapCapacity]; + int32_t buckets[kMapBuckets]; + int32_t task_heads[kTaskWindow]; + int32_t free_head; + int32_t high_water; + int32_t alive_floor; + int32_t cleaned_upto; +}; +// TensorMap 是 worker 私有状态，不在多核间共享。alive_floor 表达查询存活下界， +// cleaned_upto 表达已物理摘链的进度；即使 Case1 中通常同步推进，也不能合并其 ABI 字段。 +static_assert(sizeof(TensorMap) == 823312, "TensorMap must match the PA fixed-capacity layout"); +static_assert(offsetof(TensorMap, buckets) == 786432, "TensorMap bucket offset mismatch"); +static_assert(offsetof(TensorMap, task_heads) == 819200, "TensorMap task-head offset mismatch"); +static_assert(offsetof(TensorMap, free_head) == 823296, "TensorMap control offset mismatch"); + +struct TaskPayload { + TensorDesc tensors[kMaxTaskTensors]; +}; +// task_id 通过 kPayloadMask 映射到 2048 个 4 KiB payload；Case1 只有 1280 个 task， +// 本轮不会回绕，但仍保留生产容量、寻址方式和 4 KiB stride。 +static_assert(sizeof(TaskPayload) == kPayloadStride, "TaskPayload must preserve the real 4 KiB task stride"); +static_assert(alignof(TaskPayload) == 8, "TaskPayload alignment must match DistTaskPayload"); +static_assert(offsetof(TaskPayload, tensors) == 0, "TaskPayload tensor offset mismatch"); + +struct LocalSlot { + // occupied 先保留容量，built 表示 payload 已按生产顺序构建；task/function + // 标识决定执行哪个 NOP 体，后续大数组则是 kernel 真正看到的参数快照。 + bool occupied; + bool built; + uint8_t header_padding[2]; + uint32_t task_id; + uint32_t kind; + uint32_t function_padding; + uint64_t function_address; + uint32_t tensor_count; + uint32_t scalar_count; + uint8_t tensor_padding[32]; + + TensorDesc tensors[kMaxTaskTensors]; + uint64_t scalars[kMaxTaskScalars]; + uint64_t args[kMaxTaskTensors + kMaxTaskScalars + 2]; + union { + struct { + uint8_t local_context[48]; + uint32_t global_context; + int32_t fanin[kMaxFanin]; + uint32_t fanin_count; + }; + // Compatibility view used by the standalone NOP payload builder. The + // first six words are the real 48-byte LocalContext; the remaining + // words overlap GlobalContext and the beginning of fanin, exactly as + // dictated by the real RingSlot offsets. + // 该视图只用于按真实 offset 填充 dispatch context，不增加另一份 + // 存储；修改其中后两字会同步覆盖 GlobalContext/fanin 的对应 ABI 字节。 + uint64_t context_words[8]; + }; + bool is_multicore; + int32_t won_block; + int32_t won_slot; +}; +// LocalSlot 是每个 winner 写入自己私有 ring 的完整 dispatch 包。fanin 在执行前 +// 逐项检查 task.flag；occupied/built 与计数共同约束最多两个普通 kernel 在途。 +static_assert(sizeof(LocalSlot) == 4824, "LocalSlot must match the PA RingSlot ABI size"); +static_assert(alignof(LocalSlot) == 8, "LocalSlot alignment must match RingSlot"); +static_assert(offsetof(LocalSlot, occupied) == 0, "LocalSlot occupied offset mismatch"); +static_assert(offsetof(LocalSlot, built) == 1, "LocalSlot built offset mismatch"); +static_assert(offsetof(LocalSlot, task_id) == 4, "LocalSlot task offset mismatch"); +static_assert(offsetof(LocalSlot, kind) == 8, "LocalSlot function-id offset mismatch"); +static_assert(offsetof(LocalSlot, function_address) == 16, "LocalSlot function address offset mismatch"); +static_assert(offsetof(LocalSlot, tensor_count) == 24, "LocalSlot tensor-count offset mismatch"); +static_assert(offsetof(LocalSlot, tensors) == 64, "LocalSlot tensor payload offset mismatch"); +static_assert(offsetof(LocalSlot, scalars) == 4160, "LocalSlot scalar payload offset mismatch"); +static_assert(offsetof(LocalSlot, args) == 4288, "LocalSlot dispatch-args offset mismatch"); +static_assert(offsetof(LocalSlot, local_context) == 4688, "LocalSlot local-context offset mismatch"); +static_assert(offsetof(LocalSlot, global_context) == 4736, "LocalSlot global-context offset mismatch"); +static_assert(offsetof(LocalSlot, fanin) == 4740, "LocalSlot fanin offset mismatch"); +static_assert(offsetof(LocalSlot, fanin_count) == 4804, "LocalSlot fanin-count offset mismatch"); +static_assert(offsetof(LocalSlot, is_multicore) == 4808, "LocalSlot multicore offset mismatch"); +static_assert(offsetof(LocalSlot, won_block) == 4812, "LocalSlot won-block offset mismatch"); +static_assert(offsetof(LocalSlot, won_slot) == 4816, "LocalSlot won-slot offset mismatch"); + +struct WorkerState { + CoreRole role; + int32_t core_idx; + int32_t block_id; + int32_t lane; + int32_t sub_block_id; + int32_t local_index; + uint64_t heap_next; + TensorMap map; + uint8_t slot_padding[16]; + LocalSlot slots[kPrivateSlots]; + uint32_t occupied_count; + uint32_t owned_total; + uint64_t swimlane_last_cycle; + uint8_t payload_padding[16]; + TaskPayload payloads[kPayloadSlots]; +}; +// 每个物理 worker 都持有独立 heap cursor、TensorMap、ring 与 task payload arena； +// 多核共享的只有 SchedulerState 前缀中的 cursor/task/frontier 等协议状态。 +static_assert(sizeof(WorkerState) == 9231296, "WorkerState must match the PA DistCore ABI size"); +static_assert(alignof(WorkerState) == 8, "WorkerState alignment must match DistCore"); +static_assert(offsetof(WorkerState, role) == 0, "WorkerState role offset mismatch"); +static_assert(offsetof(WorkerState, local_index) == 20, "WorkerState replay-index offset mismatch"); +static_assert(offsetof(WorkerState, heap_next) == 24, "WorkerState heap cursor offset mismatch"); +static_assert(offsetof(WorkerState, map) == 32, "WorkerState tensor-map offset mismatch"); +static_assert(offsetof(WorkerState, slots) == 823360, "WorkerState ring-slot offset mismatch"); +static_assert(offsetof(WorkerState, occupied_count) == 842656, "WorkerState occupancy offset mismatch"); +static_assert(offsetof(WorkerState, owned_total) == 842660, "WorkerState owned-count offset mismatch"); +static_assert(offsetof(WorkerState, swimlane_last_cycle) == 842664, "WorkerState trace clock offset mismatch"); +static_assert(offsetof(WorkerState, payloads) == 842688, "WorkerState task-payload offset mismatch"); + +struct alignas(64) WorkerResult { + // 时间边界：Submit 口径不含启动屏障和最终 drain，finish_cycle 则覆盖完整 worker 生命周期。 + uint64_t submit_begin; + uint64_t submit_end; + uint64_t finish_cycle; + uint64_t checksum; + + // 协议计数用于验证固定 Claim 拓扑及等待/依赖动态次数；joint_polls 是为未来 + // BlockWon 模拟保留的兼容字段，当前实现没有递增点，不能据其检测 joint 分支。 + uint64_t submits; + uint64_t claim_attempts; + uint64_t claim_wins; + uint64_t heap_guards; + uint64_t fanin_ready_loads; + uint64_t completion_duplicates; + uint64_t cas_retries; + uint64_t joint_polls; + + // 默认 256 batch 时 winner、kernel 分别闭合到 1280 task 和 1024 kernel； + // 非默认配置按 5*batches、4*batches 计算，placement 仍闭合到全部 kernel。 + uint64_t wins[static_cast(TaskKind::Count)]; + uint64_t kernel_counts[4]; + uint64_t kernel_cycles[4]; + uint64_t kernel_min_cycles[4]; + uint64_t kernel_max_cycles[4]; + uint64_t placement[static_cast(DrainPlace::Count)]; + uint64_t phase_cycles[static_cast(ProfilePhase::Count)]; + uint64_t phase_calls[static_cast(ProfilePhase::Count)]; + uint64_t wait_events[2]; + uint64_t wait_iterations[2]; + + // 前端工作量计数不是性能填充：构参、materialize 和 map insert 用于核对全部 + // 96 个 worker 的回放；map lookup、slot copy 与 fanin 则是 winner-only 全局计数。 + uint64_t context_reads; + uint64_t views_created; + uint64_t dynamic_create_infos; + uint64_t arg_resets; + uint64_t tensor_args_added; + uint64_t scalar_args_added; + uint64_t materialized_outputs; + uint64_t map_inserts; + uint64_t map_lookups; + uint64_t slot_tensor_copies; + uint64_t slot_scalar_copies; + uint64_t fanin_edges; + + // 最终快照用于跨 worker 比较逻辑 heap 与 TensorMap 回收状态是否完全一致。 + uint64_t final_heap_next; + uint64_t map_high_water; + uint64_t map_alive_floor; + uint64_t map_cleaned_upto; + uint64_t map_live_entries; + + uint64_t worker_id; + uint64_t role; + uint64_t max_occupied; + uint64_t final_occupied; + + // CCEC 标量 PMU 取证使用 WorkerResult 的诊断 sidecar，不改变生产 DistCore ABI。 + // 该诊断只在显式开启时有效；CNT2/CNT6/CNT7 分别对应 scalar busy、I-cache req/miss。 + uint64_t pmu_total_cycles; + uint32_t pmu_scalar_busy; + uint32_t pmu_icache_requests; + uint32_t pmu_icache_misses; + uint32_t pmu_status; + + // 这些计数只在 worker 私有 LocalStats 中递增，结束时一次性发布；它们把动态 + // fanin 重试和 frontier helping 展开为准确次数，不为取数再增加共享 atomic。 + uint64_t fanin_not_ready_loads; + uint64_t frontier_initial_loads; + uint64_t frontier_updates; + uint64_t frontier_terminal_loads; + + // 仅在 trace_enabled bit1 开启时递增；每次源码 atomic 调用恰好增加一， + // host 用它与 Atomic span 数逐 worker 闭合，禁止把丢记录的泳道当成完整结果。 + uint64_t atomic_trace_calls; + + // I-cache 单 miss 探针用该槽保存 cold 窗口的 1 GHz SYS_CNT；submit-pmu + // 复用同一 64-bit 槽保存所选局部阶段的逐调用累计时间。两种构建互斥， + // 因而无需扩大 832B WorkerResult，也不会改变相邻 worker 的 cache-line 布局。 + union { + uint64_t pmu_window_ticks; + uint64_t pmu_phase_elapsed_ticks; + }; + uint64_t pmu_warm_total_cycles; + uint64_t pmu_warm_window_ticks; + union { + uint32_t pmu_warm_icache_requests; + uint32_t pmu_phase_begin_reads; + }; + union { + uint32_t pmu_warm_icache_misses; + uint32_t pmu_phase_end_reads; + }; + + // PIPE_UTILIZATION 已同时配置 CNT0/1/3/4/5/8；与上面的 scalar/I-cache + // 一样只保存每核原始累计值，AIC/AIV 汇总与比率统一在 host sidecar 中计算。 + // 六个 32-bit 值复用本结构扩展到 832B 后的尾部空间，不再增加 cache line。 + uint32_t pmu_vector_busy; + uint32_t pmu_cube_busy; + uint32_t pmu_mte1_busy; + uint32_t pmu_mte2_busy; + // swimlane ABI 保留该槽；submit-pmu 将物理 CNT5 改作 shadow miss， + // 因而显式发布 0，并在当前 submit-pmu schema-v5 标记 mte3_busy 不可用。 + uint32_t pmu_mte3_busy; + uint32_t pmu_fix_busy; + + // 复用 WorkerResult 原有的 32B cache-line 尾洞，不扩大 832B stride。 + // CNT6/7 是从不中途读取的权威整窗，CNT8/CNT5 是 read-to-clear shadow； + // none 在 stop 后要求逐核精确相等；运行中切片的 phase 只允许 shadow + // 单向小于 primary，并显式导出差值形成局部观测区间。 + uint32_t pmu_build_variant; + uint32_t pmu_phase_id; + uint32_t pmu_phase_calls; + uint32_t pmu_phase_status; + uint32_t pmu_phase_icache_requests; + uint32_t pmu_phase_icache_misses; + uint32_t pmu_shadow_icache_requests; + uint32_t pmu_shadow_icache_misses; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + // split 组合 oracle 独占一条诊断 cache line；legacy 与 inline shape 完全 + // 不带这些字段，避免改变既有 WorkerResult/SchedulerState 的布局。 + uint64_t lazy_sample_split_caller_state_address; + uint64_t lazy_sample_split_finish_state_address; + uint64_t lazy_sample_split_finish_calls; + uint64_t lazy_sample_split_protocol_errors; + uint64_t lazy_sample_split_state_cookie; + uint64_t lazy_sample_split_task_id_sum; + uint64_t lazy_sample_split_owner_worker_id; + uint64_t lazy_sample_split_reserved; +#endif +}; +// WorkerResult 是 standalone 尾部的诊断 sidecar，不属于真实 DistCore ABI；按 +// cache line 隔离后，各 worker 发布统计不会相互覆盖或污染被测共享状态。 +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +static_assert(sizeof(WorkerResult) == 896, "split WorkerResult diagnostics must occupy whole cache lines"); +static_assert(offsetof(WorkerResult, lazy_sample_split_caller_state_address) == 832, + "split WorkerResult oracle offset mismatch"); +static_assert(offsetof(WorkerResult, lazy_sample_split_reserved) == 888, + "split WorkerResult oracle tail mismatch"); +#else +static_assert(sizeof(WorkerResult) == 832, "WorkerResult diagnostics must occupy whole cache lines"); +#endif +static_assert(offsetof(WorkerResult, pmu_total_cycles) == 680, "WorkerResult PMU offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_status) == 700, "WorkerResult PMU status offset mismatch"); +static_assert(offsetof(WorkerResult, fanin_not_ready_loads) == 704, "WorkerResult atomic diagnostic offset mismatch"); +static_assert(offsetof(WorkerResult, atomic_trace_calls) == 736, "WorkerResult atomic trace offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_window_ticks) == 744, "WorkerResult PMU timing offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_vector_busy) == 776, "WorkerResult extended PMU offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_build_variant) == 800, "WorkerResult submit-PMU offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_shadow_icache_misses) == 828, "WorkerResult submit-PMU tail mismatch"); + +// 从 cube_cursor 到 workers 结束保留关键字段 offset、DistCore ABI 和生产总字节跨度， +// 并非字段级完整镜像。RunConfig、输入 context_lens 与校验结果追加在该跨度之后， +// 因此测试控制信息不会改变被测字段 offset。 +struct alignas(64) SchedulerState { + // 三组四分片 cursor 分别服务 AIC kernel、AIV kernel 和 Alloc；同 task 的 + // eligible workers 竞争同一 shard，只有旧值小于 task_id 的调用成为 winner。 + AtomicLine cube_cursor[kCursorShards]; + AtomicLine vector_cursor[kCursorShards]; + AtomicLine alloc_cursor[kCursorShards]; + AtomicLine frontier; + int32_t heap_window; + uint8_t tasks_padding[60]; + TaskCell tasks[kTaskCellCapacity]; + // heap_base/size 描述共享物理环，worker.heap_next 则是各 worker 一致推进的逻辑游标。 + uint64_t heap_base; + uint64_t heap_size; + uint64_t orchestration_args; + uint64_t runtime_state; + uint64_t runtime; + uint8_t fatal_padding[24]; + AtomicFlagLine fatal; + int32_t num_workers; + int32_t num_blocks; + // Case1 never enters BlockWon, but the inactive layout and BlockWon arena + // remains byte-for-byte reserved so every subsequent PA atomic line keeps + // its production offset. + // 此处不能因 Case1 动态次数为零而删减，否则 replay_done、started_count + // 和 DistCore 数组整体前移，便不再是对真实 PA 地址布局的等价测试。 + uint8_t layout_and_block_won[5848440]; + AtomicLine replay_done; + AtomicLine started_count; + // started_count 形成 launch 屏障；replay_done 只用于最终 drain 判定所有 worker + // 已不再产生新 slot。两者都位于 Submit 性能口径之外，但属于完整协议。 + WorkerState workers[kRuntimeMaxWorkers]; + // Standalone-only controls live after the complete DistGlobal image. They + // therefore do not shift any cursor/task/fatal/worker address under test. + RunConfig config; + WinnerWorkloadConfig winner_workload; + // Context lengths are the only PA input elements read by orchestration; + // keeping them in GM preserves the per-batch descriptor-based load. + // 除这 256 个长度值外，其余 tensor 仅需稳定的合成地址来复现 + // descriptor、依赖和 heap 行为，不会解引用成真实计算数据。 + volatile int32_t context_lens[kMaxBatches]; + WorkerResult results[kWorkers]; +}; +static_assert(offsetof(SchedulerState, cube_cursor) == 0, "cube cursor offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, vector_cursor) == 256, "vector cursor offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, alloc_cursor) == 512, "alloc cursor offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, frontier) == 768, "frontier offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, heap_window) == 832, "H offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, tasks) == kRealTasksOffset, "task table offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, heap_base) == 4195200, "heap base offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, heap_size) == 4195208, "heap size offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, fatal) == kRealFatalOffset, "fatal offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, replay_done) == kRealReplayDoneOffset, "replay offset must match PA DistGlobal"); +static_assert( + offsetof(SchedulerState, started_count) == kRealStartedCountOffset, + "started-count offset must match PA DistGlobal" +); +static_assert(offsetof(SchedulerState, tasks) % 64 == 0, "task table must be cache-line aligned"); +static_assert(offsetof(SchedulerState, workers) % 64 == 0, "worker table must be cache-line aligned"); +static_assert(offsetof(SchedulerState, results) % 64 == 0, "result table must be cache-line aligned"); +static_assert(offsetof(SchedulerState, workers) == kRealDistCoreOffset, "DistCore table offset must match PA"); +static_assert(offsetof(SchedulerState, config) == kRealDistGlobalBytes, "DistGlobal byte size must match PA"); +static_assert( + offsetof(SchedulerState, winner_workload) == kRealDistGlobalBytes + sizeof(RunConfig), + "winner workload sidecar must follow RunConfig" +); +static_assert( + offsetof(SchedulerState, context_lens) == + kRealDistGlobalBytes + sizeof(RunConfig) + sizeof(WinnerWorkloadConfig), + "context lengths must follow standalone controls" +); + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_MODEL_H diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/common/pa_scheduler_core.h b/tests/atomic_probe/lazy_lamda_sample/A_original/common/pa_scheduler_core.h new file mode 100644 index 0000000000..afdd0896ce --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/common/pa_scheduler_core.h @@ -0,0 +1,1618 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_SCHEDULER_CORE_H +#define PA_SCHEDULER_COMMON_PA_SCHEDULER_CORE_H + +#ifndef PA_DEVICE +#define PA_DEVICE inline +#endif + +#ifndef PA_GM +#define PA_GM +#endif + +#include "pa_frontend.h" +#include "pa_trace.h" + +namespace pa_scheduler { + +struct LocalStats { + WorkerResult result; + uint32_t max_occupied; + TraceContext trace; +}; + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +// runtime TU owns one external [[block_local]] instance per architecture. +// The caller imports that same object and keeps all Submit-internal context in +// it, so the only cross-TU function arguments are a POD ticket and built args. +struct alignas(64) LazySampleSplitRuntimeState { + PA_GM SchedulerState *scheduler; + PA_GM WorkerState *worker; + uint32_t task_count; + uint32_t worker_id; + SubmitContext context; + LocalStats stats; + uint64_t caller_state_address; + uint64_t finish_state_address; + uint64_t finish_calls; + uint64_t protocol_errors; + uint64_t state_cookie; + uint64_t task_id_sum; + uint64_t owner_worker_id; + uint64_t reserved; +}; +static_assert(sizeof(LazySampleSplitRuntimeState) % 64 == 0, + "split runtime state must occupy whole cache lines"); + +PA_DEVICE uint64_t LazySampleSplitStateCookie(uint32_t worker_id, CoreRole role) { + return kLazySampleSplitStateCookieBase ^ static_cast(worker_id) ^ + (static_cast(static_cast(role)) << 32U); +} +#endif + +// submit-pmu 的 phase 在编译期固定；非诊断构建完全不引用 Ops 的 phase +// 接口。这样公共调度代码保持一份，swimlane/CPU/AscendC 也不会多出运行时分支。 +template +PA_DEVICE void BeginSubmitPmuPhase(PmuContext &context) { +#if PA_BUILD_SUBMIT_PMU + if constexpr (kCompiledSubmitPmuPhase == Phase) { + Ops::PmuPhaseBegin(context); + } +#else + (void)context; +#endif +} + +template +PA_DEVICE void EndSubmitPmuPhase(PmuContext &context) { +#if PA_BUILD_SUBMIT_PMU + if constexpr (kCompiledSubmitPmuPhase == Phase) { + Ops::PmuPhaseEnd(context); + } +#else + (void)context; +#endif +} + +template +PA_DEVICE uint64_t TraceTimestamp(TraceContext &trace, WorkerResult &result) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + return 0; +#else + (void)result; + // 对齐真实 FDWIC 的 TRACE_SPAN_BEGIN/END：取一次时间后立即以同一 + // cycle 关闭活跃 PollBatch。不能在 WriteTrace 中统一关闭，否则直接 + // Atomic 记录也会错误切断等待 episode。 + const uint64_t cycle = Ops::Now(); + AtomicPollBoundaryAt(trace, cycle); + return cycle; +#endif +} + +PA_DEVICE uint32_t KindIndex(TaskKind kind) { return static_cast(kind); } + +PA_DEVICE TaskKind GetTaskKind(uint32_t task_id) { return static_cast(task_id % kTasksPerBatch); } + +PA_DEVICE int32_t FunctionId(TaskKind kind) { + return kind == TaskKind::Alloc ? -1 : static_cast(KindIndex(kind) - 1); +} + +PA_DEVICE uint32_t NopCountForKind(PA_GM const NopCounts &nops, TaskKind kind) { + switch (kind) { + case TaskKind::Qk: + return nops.qk; + case TaskKind::Sf: + return nops.sf; + case TaskKind::Pv: + return nops.pv; + case TaskKind::Up: + return nops.up; + default: + return 0; + } +} + +PA_DEVICE uint32_t WorkloadCountForKind(PA_GM const WorkloadCounts &counts, TaskKind kind) { + switch (kind) { + case TaskKind::Qk: + return counts.qk; + case TaskKind::Sf: + return counts.sf; + case TaskKind::Pv: + return counts.pv; + case TaskKind::Up: + return counts.up; + default: + return 0; + } +} + +PA_DEVICE uint32_t CountBits(uint32_t value) { + uint32_t count = 0; + while (value != 0) { + count += value & 1U; + value >>= 1; + } + return count; +} + +template +PA_DEVICE int64_t LoadLine( + PA_GM AtomicLine &line, LocalStats &stats, AtomicSite site, int32_t task_id = -1 +) { + // Ops::Load 在 A5 后端是 atomicAdd(0)；返回值是该 RMW 线性化时观察到的共享值。 + return TraceAtomicLoad(stats.trace, stats.result, task_id, site, &line.value); +} + +template +PA_DEVICE int32_t LoadLine( + PA_GM AtomicFlagLine &line, LocalStats &stats, AtomicSite site, int32_t task_id = -1 +) { + return TraceAtomicLoad(stats.trace, stats.result, task_id, site, &line.value); +} + +template +PA_DEVICE void SetFatal(PA_GM SchedulerState *state, LocalStats &stats, int32_t task_id = -1) { + // fatal 只从 0 单调置 1，重复 Exchange 不会把其他 worker 已观察到的失败状态清除。 + TraceAtomicExchange( + stats.trace, stats.result, task_id, AtomicSite::FatalSet, &state->fatal.value, + static_cast(1) + ); +} + +template +PA_DEVICE bool IsFatal(PA_GM SchedulerState *state, LocalStats &stats, int32_t task_id = -1) { + return LoadLine(state->fatal, stats, AtomicSite::FatalPoll, task_id) != 0; +} + +template +PA_DEVICE bool WatchdogExpired( + PA_GM SchedulerState *state, LocalStats &stats, uint64_t begin, uint32_t &polls +) { + // 每 1024 次自旋才读取系统计数器，降低正常启动屏障上的计时开销；超时后向所有 worker 广播 fatal。 + ++polls; + if ((polls & 1023U) != 0 || Ops::Now() - begin <= kWatchdogTicks) { + return false; + } + SetFatal(state, stats); + return true; +} + +template +PA_DEVICE void AdvanceFrontier(PA_GM SchedulerState *state, LocalStats &stats) { + // frontier 只表示“从 task 0 开始已经连续完成”的最高 task id，不能越过尚未发布 flag 的空洞。 + // 多个完成者可以同时扫描同一段连续区间，FetchMax 保证共享 frontier 只前进、不回退。 + ++stats.result.frontier_initial_loads; + int64_t frontier = LoadLine( + state->frontier, stats, AtomicSite::FrontierInitialLoad + ); + while (true) { + const int64_t next = frontier + 1; + if (next < 0 || next >= static_cast(kTaskCellCapacity)) { + break; + } + if (TraceAtomicLoad( + stats.trace, stats.result, static_cast(next), AtomicSite::FrontierFlagLoad, + &state->tasks[next].flag + ) == 0) { + ++stats.result.frontier_terminal_loads; + break; + } + uint64_t retries = 0; + // FetchMax 返回更新前的值；若其他核已经走得更远，就从其 old 值继续扫描，避免重复从 next 起步。 + ++stats.result.frontier_updates; + const int64_t old = TraceAtomicFetchMax( + stats.trace, stats.result, static_cast(next), AtomicSite::FrontierMax, + &state->frontier.value, next, retries + ); + stats.result.cas_retries += retries; + frontier = old > next ? old : next; + } +} + +template +PA_DEVICE void CompleteTask( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, LocalStats &stats +) { + // 完成发布顺序与 PA 一致：先公布该 worker 的 heap 游标，再发布 ready flag，最后推进连续 frontier。 + // fanin 和 heap 回收方以 flag/frontier 为可见性条件，因此不能交换 vend 与 flag 的先后关系。 + TraceAtomicExchange( + stats.trace, stats.result, static_cast(task_id), AtomicSite::CompletionVendExchange, + &state->tasks[task_id].vend, worker.heap_next + ); + Ops::StoreBarrier(); + TraceAtomicExchange( + stats.trace, stats.result, static_cast(task_id), AtomicSite::CompletionFlagExchange, + &state->tasks[task_id].flag, static_cast(1) + ); + AdvanceFrontier(state, stats); +} + +template +PA_DEVICE bool SlotReady(PA_GM SchedulerState *state, PA_GM LocalSlot &slot, LocalStats &stats) { + // 每个 fanin flag 都是跨核共享的完成条件；遇到第一个未就绪依赖即返回，后续 drain 会再次轮询。 + for (uint32_t index = 0; index < slot.fanin_count; ++index) { + const int32_t dependency = slot.fanin[index]; + if (TraceAtomicLoad( + stats.trace, stats.result, dependency, AtomicSite::FaninFlagLoad, + &state->tasks[dependency].flag + ) == 0) { + ++stats.result.fanin_not_ready_loads; + return false; + } + ++stats.result.fanin_ready_loads; + } + return true; +} + +PA_DEVICE void RecordKernelCycles(LocalStats &stats, TaskKind kind, uint64_t cycles) { + const uint32_t index = KindIndex(kind) - 1; + ++stats.result.kernel_counts[index]; + stats.result.kernel_cycles[index] += cycles; + if (stats.result.kernel_min_cycles[index] == 0 || cycles < stats.result.kernel_min_cycles[index]) { + stats.result.kernel_min_cycles[index] = cycles; + } + if (cycles > stats.result.kernel_max_cycles[index]) { + stats.result.kernel_max_cycles[index] = cycles; + } +} + +template +PA_DEVICE uint32_t DrainReady( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, DrainPlace place, LocalStats &stats +) { + // 同一套 drain 被三个位置复用：每次 Submit 开头的 EfDrain、ring 背压等待和所有 Submit 后的最终 drain。 + // slot 属于当前 worker；只有其全部跨核 fanin 已 ready 时才执行所选 winner 负载、发布完成并释放 slot。 + if (worker.occupied_count == 0) { + return 0; + } + uint32_t freed = 0; + // 一次调用遍历完本核全部私有 slot；未就绪项保留 occupied/built，已完成项立即清槽并继续扫描。 + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + PA_GM LocalSlot &slot = worker.slots[index]; + if (!slot.occupied || !slot.built || !SlotReady(state, slot, stats)) { + continue; + } + const TaskKind kind = static_cast(slot.kind + 1); + const uint64_t kernel_begin = TraceTimestamp(stats.trace, stats.result); + Ops::ExecuteKernel(state, worker, kind, NopCountForKind(state->config.nops, kind)); + const uint64_t kernel_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(slot.task_id), static_cast(slot.kind), + TracePhase::Kernel, ProfilePhase::ReplayTail, kernel_begin, kernel_end + ); + RecordKernelCycles(stats, kind, kernel_end - kernel_begin); + CompleteTask(state, worker, slot.task_id, stats); + const uint64_t commit_cycle = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(slot.task_id), static_cast(slot.kind), + TracePhase::Commit, ProfilePhase::ReplayTail, commit_cycle, commit_cycle + ); + slot.built = false; + slot.occupied = false; + --worker.occupied_count; + ++stats.result.placement[static_cast(place)]; + ++freed; + } + return freed; +} + +PA_DEVICE int32_t FindFreeSlot(PA_GM WorkerState &worker) { + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + if (!worker.slots[index].occupied) { + return static_cast(index); + } + } + return -1; +} + +template +PA_DEVICE void WaitForSlot( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, LocalStats &stats +) { + // 四个物理 slot 中预留两个 won slot 语义位，仅有 kUsableSlots 个可供本图使用；满时靠 drain 取得进展。 + const uint64_t wait_begin = TraceTimestamp(stats.trace, stats.result); + bool waited = false; + // 只聚合这个显式背压等待区中的 fanin 观察；每次 Submit 开头的 + // opportunistic EfDrain 仍保留逐条 Atomic，不能仅凭 site 名称全局聚合。 + const uint32_t poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, TraceAtomicSiteMask(AtomicSite::FaninFlagLoad) + ); + // 退出条件只有 occupied_count 重新低于可用容量；依赖尚未 ready 时 SpinHint 后继续重试。 + while (worker.occupied_count >= kUsableSlots) { + waited = true; + ++stats.result.wait_iterations[0]; + if (DrainReady( + state, worker, DrainPlace::RingBackpressure, stats + ) == 0) { + Ops::SpinHint(); + } + } + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + if (waited) { + ++stats.result.wait_events[0]; + const uint64_t wait_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::RingBp, + ProfilePhase::WaitForSlot, wait_begin, wait_end, 0, 0 + ); + } else if constexpr (Profile) { + AccumulatePhase( + stats.result, ProfilePhase::WaitForSlot, wait_begin, + TraceTimestamp(stats.trace, stats.result) + ); + } +} + +template +PA_DEVICE bool HeapGuard( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, uint64_t output_bytes, + LocalStats &stats +) { + // 只有产生新输出的 winner 需要保护环形 heap；retire=frontier-H 对应已经允许复用的最老任务 vend。 + // 等待期间也主动 drain 本核已就绪 slot，避免只自旋而阻塞能够推动 frontier 的 kernel。 + if (output_bytes == 0 || state->heap_base == 0) { + return true; + } + const uint64_t ring = state->heap_size; + ++stats.result.heap_guards; + const uint64_t wait_begin = TraceTimestamp(stats.trace, stats.result); + bool waited = false; + bool poll_region_active = false; + uint32_t poll_region = 0; + // 正常出口是 heap_next-vend 落入一个 ring；检测到不可能释放的覆盖或其他核 fatal 时返回失败。 + while (!IsFatal(state, stats, static_cast(task_id))) { + // 逻辑 heap 尚未走完第一圈时，所有物理输出区间都位于 [0, heap_next)， + // 不可能覆盖此前分配；保留上面的 fatal 原子检查后，可直接跳过 frontier/vend 读取。 + if (worker.heap_next <= ring) { + if (poll_region_active) { + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + } + if constexpr (Profile) { + AccumulatePhase( + stats.result, ProfilePhase::HeapGuard, wait_begin, + TraceTimestamp(stats.trace, stats.result) + ); + } + return true; + } + // 与真实 PA 一样，首圈 fast path 上方的 FatalPoll 仍是直接记录；只有 + // 确认进入 heap wrap 慢路径后，才开启本等待 episode 的四类观察聚合。 + if (!poll_region_active) { + poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, + TraceAtomicSiteMask(AtomicSite::FatalPoll) | + TraceAtomicSiteMask(AtomicSite::HeapFrontierLoad) | + TraceAtomicSiteMask(AtomicSite::HeapVendLoad) | + TraceAtomicSiteMask(AtomicSite::FaninFlagLoad) + ); + poll_region_active = true; + } + const int64_t frontier = LoadLine( + state->frontier, stats, AtomicSite::HeapFrontierLoad, static_cast(task_id) + ); + const int64_t retire = frontier - static_cast(state->heap_window); + const uint64_t vend = retire < 0 + ? 0 + : TraceAtomicLoad( + stats.trace, stats.result, static_cast(task_id), AtomicSite::HeapVendLoad, + &state->tasks[retire].vend + ); + if (worker.heap_next - vend <= ring) { + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + if (waited) { + ++stats.result.wait_events[1]; + const uint64_t wait_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::RingBp, + ProfilePhase::HeapGuard, wait_begin, wait_end, 0, 1 + ); + } else if constexpr (Profile) { + AccumulatePhase( + stats.result, ProfilePhase::HeapGuard, wait_begin, + TraceTimestamp(stats.trace, stats.result) + ); + } + return true; + } + if (frontier >= static_cast(task_id) - 1) { + SetFatal(state, stats, static_cast(task_id)); + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + return false; + } + waited = true; + ++stats.result.wait_iterations[1]; + if (DrainReady( + state, worker, DrainPlace::RingBackpressure, stats + ) == 0) { + Ops::SpinHint(); + } + } + if (poll_region_active) { + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + } + if (waited) { + ++stats.result.wait_events[1]; + const uint64_t wait_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::RingBp, + ProfilePhase::HeapGuard, wait_begin, wait_end, 0, 1 + ); + } + return false; +} + +struct ClaimOutcome { + bool attempted; + bool won; + uint64_t retries; + int32_t function_id; +}; + +template +PA_DEVICE ClaimOutcome Claim( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, TaskKind kind, + LocalStats &stats +) { + // Claim 在四个 shard 的单调 cursor 上执行 atomicMax：同一 task 只有观察到旧值更小的竞争者获胜。 + // Alloc 由全部 96 个 worker 竞争；QK/PV 仅 32 个 AIC，SF/UP 仅 64 个 AIV 进入真正的 atomicMax。 + ClaimOutcome outcome{false, false, 0, -1}; + if (task_id >= kTaskCellCapacity) { + return outcome; + } + PA_GM AtomicLine *cursor = nullptr; + if (kind == TaskKind::Alloc) { + cursor = &state->alloc_cursor[task_id % kCursorShards]; + } else { + // Mirror MixedKernels::to_active_mask(), core_mask(), popcount(), + // lane_active(), and self->role routing inside the real Claim span. + const int32_t aic_kernel = kind == TaskKind::Qk || kind == TaskKind::Pv ? FunctionId(kind) : -1; + const int32_t aiv0_kernel = kind == TaskKind::Sf || kind == TaskKind::Up ? FunctionId(kind) : -1; + const int32_t aiv1_kernel = -1; + uint8_t active_mask = 0; + if (aic_kernel >= 0) active_mask |= 1U; + if (aiv0_kernel >= 0) active_mask |= 2U; + if (aiv1_kernel >= 0) active_mask |= 4U; + const uint8_t core_mask = active_mask & 0x07U; + const int32_t active_count = __builtin_popcount(static_cast(core_mask)); + // 这里保留生产 Claim 的 lane-mask 路由边界。当前固定 PA 图按构造只生成单 lane + // 的 QK/PV 或 SF/UP；需要两个及以上 lane 协作的 joint task 本应进入 BlockWon + // 协议，本独立用例没有实现该动态路径，因此显式拒绝而不把它误当成单 lane task。 + if (active_count >= 2) { + return outcome; + } + if ((core_mask & 1U) != 0) { + if (worker.role != CoreRole::Aic) return outcome; + cursor = &state->cube_cursor[task_id % kCursorShards]; + outcome.function_id = aic_kernel; + } else if ((core_mask & 6U) != 0) { + if (worker.role != CoreRole::Aiv) return outcome; + cursor = &state->vector_cursor[task_id % kCursorShards]; + outcome.function_id = (core_mask & 2U) != 0 ? aiv0_kernel : aiv1_kernel; + } else { + return outcome; + } + } + outcome.attempted = true; + // atomicMax 返回写入前的 cursor：old=task_id 则必须 Replay。 + const int64_t old = TraceAtomicFetchMax( + stats.trace, stats.result, static_cast(task_id), AtomicSite::ClaimMax, + &cursor->value, static_cast(task_id), outcome.retries + ); + outcome.won = old < static_cast(task_id); + if (!outcome.won) outcome.function_id = -1; + return outcome; +} + +PA_DEVICE void RecordClaimOutcome(LocalStats &stats, TaskKind kind, const ClaimOutcome &outcome) { + if (outcome.attempted) ++stats.result.claim_attempts; + stats.result.cas_retries += outcome.retries; + if (outcome.won) { + ++stats.result.claim_wins; + ++stats.result.wins[KindIndex(kind)]; + } +} + +template +PA_DEVICE bool BuildWinner( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, TaskKind kind, + const TaskArgs &args, const SubmitContext &context, + const int32_t fanin[kMaxFanin], uint32_t fanin_count, LocalStats &stats +) { + // kernel winner 不在 Submit 内立即执行计算，而是把完整 payload 和 fanin 存入自己的私有 ring slot。 + // 后续 EfDrain/背压 drain/最终 drain 在依赖满足后执行它，这正是 PA 的 Submit 与执行解耦点。 + WaitForSlot(state, worker, task_id, stats); + if (!HeapGuard( + state, worker, task_id, context.output_bytes, stats + )) { + return false; + } + const int32_t slot_index = FindFreeSlot(worker); + if (slot_index < 0) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + PA_GM LocalSlot &slot = worker.slots[slot_index]; + // Match dist_submit_alloc_slot(): reserve and account the private slot + // before build_ring_slot_from_submit publishes its completed payload. + // 状态按“occupied 占位 -> built 清零 -> 计入占用 -> BuildSlotPayload”推进；后者会先 + // 置 built，再填充 payload。slot 为 worker 私有、没有跨核发布竞争，所以此处的 built + // 只是复刻生产状态机与构建成本，不承担对其他核发布完整 payload 的同步语义。 + slot.occupied = true; + slot.built = 0; + ++worker.occupied_count; + if (worker.occupied_count > stats.max_occupied) { + stats.max_occupied = worker.occupied_count; + } + const int32_t sub_block_id = worker.lane == 2 ? 1 : 0; + BuildSlotPayload( + slot, task_id, static_cast(FunctionId(kind)), 0, args, context, fanin, fanin_count, + sub_block_id + ); + stats.result.slot_tensor_copies += static_cast(context.tensor_count); + stats.result.slot_scalar_copies += static_cast(context.scalar_count); + stats.result.fanin_edges += fanin_count; + return true; +} + +template +PA_DEVICE bool SubmitTask( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_count, TaskKind kind, + const TaskArgs &args, SubmitContext &context, LocalStats &stats, PmuContext &pmu_context +) { + // 每个 worker 都完整回放相同 task stream。主流程为：EfDrain -> materialize -> TensorMap retire + // -> Claim -> winner 收集 fanin -> 全员 register -> winner Build / loser Replay。Alloc 在 Claim 前 register， + // 且 winner 不入 kernel slot，而是在 heap guard 后直接发布完成。 + BeginSubmit(worker, args, context); + const uint32_t task_id = static_cast(context.task_id); +#if PA_BUILD_SUBMIT_PMU + // PMU-only ELF 只保留首/末 Submit 的全局时间边界，不再为 1280 次调用 + // 各执行两条 trace-only SYS_CNT。 + const uint64_t submit_begin = task_id == 0 ? Ops::Now() : 0; +#else + const uint64_t submit_begin = TraceTimestamp(stats.trace, stats.result); +#endif + if (task_id == 0) { + stats.result.submit_begin = submit_begin; + } + + // EfDrain 在当前 Submit 的参数物化前执行上一批已就绪 slot，是绝大多数 kernel 的正常落点。 + // 只在这个唯一 call-site 划 PMU 边界；DrainReady 还被 ring 背压和最终 drain + // 复用，不能把 phase 插入函数体后按 place 混合累计。 + // EfDrain 是 Submit 的第一个真实阶段，直接复用父区间起点；这样每次 + // Submit 少一次 trace-only SYS_CNT，也不会留下人为的 prefix residual。 + const uint64_t efdrain_begin = submit_begin; + BeginSubmitPmuPhase(pmu_context); + DrainReady(state, worker, DrainPlace::EfDrain, stats); + EndSubmitPmuPhase(pmu_context); + const uint64_t efdrain_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::EfDrain, + ProfilePhase::EfDrain, efdrain_begin, efdrain_end + ); + + // schema-v4 的所有 Submit 子阶段都使用显式 start/end；不再通过共同 lap + // 起点生成相互覆盖的 Build/Replay/Alloc 区间。 + // 后继 segment 复用前一阶段 end：既少一次 SYS_CNT，也把前一条 trace + // 发布和阶段间胶水明确归入 Materialize，而不是留成无名 residual。 + const uint64_t materialize_begin = efdrain_end; + BeginSubmitPmuPhase(pmu_context); + const bool materialized = + MaterializeTask(worker, task_id, args, context, state->heap_base, state->heap_size); + if (!materialized) { + EndSubmitPmuPhase(pmu_context); + SetFatal(state, stats, static_cast(task_id)); + return false; + } + stats.result.materialized_outputs += context.result.count; + EndSubmitPmuPhase(pmu_context); + const uint64_t materialize_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::Materialize, + ProfilePhase::Materialize, materialize_begin, materialize_end, 0, + kind == TaskKind::Alloc ? 1U : 0U + ); + const uint64_t prepare_begin = materialize_end; + AdvanceTensorMap(worker.map, task_id, static_cast(state->heap_window)); + const uint64_t prepare_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::PrepareMap, + ProfilePhase::PrepareMap, prepare_begin, prepare_end, 0, kind == TaskKind::Alloc ? 1U : 0U + ); + + bool winner = false; + int32_t function_id = -1; + + if (kind == TaskKind::Alloc) { + // Alloc 没有 kernel lane，96 个 worker 都维护本地物化/heap 状态，但只有 Claim winner 发布全局完成。 + const uint64_t register_begin = prepare_end; + BeginSubmitPmuPhase(pmu_context); + RegisterOutputs(context, args, false); + EndSubmitPmuPhase(pmu_context); + const uint64_t register_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::Register, + ProfilePhase::Register, register_begin, register_end, 0, 0 + ); + + const uint64_t claim_begin = register_end; + BeginSubmitPmuPhase(pmu_context); + const ClaimOutcome claim = Claim(state, worker, task_id, kind, stats); + winner = claim.won; + context.won = winner; + context.kernel_id = claim.function_id; + // Claim 的本地结果归档属于同一阶段；放在共同 end 边界内，避免把 + // winner/context/stat bookkeeping 留成无法归因的 Submit residual。 + RecordClaimOutcome(stats, kind, claim); + EndSubmitPmuPhase(pmu_context); + const uint64_t claim_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::Claim, + ProfilePhase::Claim, claim_begin, claim_end, + (winner ? kClaimWon : 0U) | (claim.attempted ? kClaimAttempted : 0U), 1 + ); + // 每个 task 只有 1/96 worker 进入 winner 重型路径。把该分支标成冷路 + // 只影响基本块布局，使占绝大多数的 loser 尽量顺序进入 Submit 公共 + // 尾部；不改变 Claim 结果、完成发布或泳道边界。 + if (__builtin_expect(winner, 0)) { + const uint64_t alloc_complete_begin = claim_end; + if (!HeapGuard( + state, worker, task_id, context.output_bytes, stats + )) { + return false; + } + CompleteTask(state, worker, task_id, stats); + const uint64_t alloc_complete_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, + TracePhase::AllocComplete, ProfilePhase::ReplayTail, + alloc_complete_begin, alloc_complete_end + ); + } else { + // standalone 的 Alloc loser 没有真实 GM/Replay 动作，不再为业务 + // 路径名字写一条零时长记录。Claim 后到 Submit.end 的真实 + // scalar 时间由离线 submit_tail_gap 补集展示；排他报告将其汇总为 + // submit_tail_residual，避免伪装成 Alloc loser 业务阶段。 + } + } else { + const uint64_t claim_begin = prepare_end; + BeginSubmitPmuPhase(pmu_context); + const ClaimOutcome claim = Claim(state, worker, task_id, kind, stats); + winner = claim.won; + function_id = claim.function_id; + context.won = winner; + context.kernel_id = function_id; + RecordClaimOutcome(stats, kind, claim); + EndSubmitPmuPhase(pmu_context); + const uint64_t claim_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, TracePhase::Claim, + ProfilePhase::Claim, claim_begin, claim_end, + (winner ? kClaimWon : 0U) | (claim.attempted ? kClaimAttempted : 0U), 0 + ); + uint64_t register_begin = claim_end; + if (winner) { + const uint64_t fanin_begin = claim_end; + context.fanin_count = static_cast(CollectFanin(worker.map, args, context.fanin)); + stats.result.map_lookups += static_cast(args.tensor_count) - context.result.count; + const uint64_t fanin_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, TracePhase::Fanin, + ProfilePhase::Fanin, fanin_begin, fanin_end, 0, static_cast(context.fanin_count) + ); + register_begin = fanin_end; + } + + // loser 直接承接 Claim.end；winner 则承接 Fanin.end。两条路径都 + // 复用已有边界，不再为 Register 单独读取 SYS_CNT。 + BeginSubmitPmuPhase(pmu_context); + RegisterOutputs(context, args, true); + stats.result.map_inserts += CountBits(context.register_mask); + EndSubmitPmuPhase(pmu_context); + const uint64_t register_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, TracePhase::Register, + ProfilePhase::Register, register_begin, register_end, 0, 1 + ); + // BuildWinner 会内联 ring、heap 和真计算提交逻辑；提示其为 1/96 + // 冷路，避免 loser 为跳过大块代码付出额外取指代价。 + if (__builtin_expect(winner, 0)) { + const uint64_t winner_build_begin = register_end; + if (!BuildWinner( + state, worker, task_id, kind, args, context, context.fanin, + static_cast(context.fanin_count), stats + )) { + return false; + } + const uint64_t winner_build_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::WinnerBuild, ProfilePhase::ReplayTail, + winner_build_begin, winner_build_end + ); + } else { + // 非 winner 不占用私有 ring slot，也没有可单列的 Replay 计算。 + // Register 后到 Submit.end 由离线 submit_tail_gap 补集展示；排他 + // 报告汇总为 submit_tail_residual，避免零时长 marker 增加 raw + // 体积和 trace-buffer 写开销。 + } + } + + ++stats.result.submits; +#if PA_BUILD_SUBMIT_PMU + const uint64_t submit_end = task_id + 1 == task_count ? Ops::Now() : 0; +#else + const uint64_t submit_end = TraceTimestamp(stats.trace, stats.result); +#endif + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, TracePhase::Submit, + ProfilePhase::Submit, submit_begin, submit_end, winner ? 1U : 0U, kind == TaskKind::Alloc ? 1U : 0U + ); + if (task_id + 1 == task_count) { + stats.result.submit_end = submit_end; + } + return true; +} + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +// The all-task callback path crosses the split-TU finish boundary through this +// fixed 16-byte POD. No callback closure or nested thunk is stored in it. +struct LazySampleCallbackTicket { + uint64_t submit_begin; + uint32_t task_id; + int16_t function_id; + uint8_t won; + uint8_t reserved; +}; +static_assert(sizeof(LazySampleCallbackTicket) == 16, "lazy sample callback ticket must remain a 16-byte POD"); +static_assert(offsetof(LazySampleCallbackTicket, submit_begin) == 0, "lazy sample callback ticket timestamp offset mismatch"); +static_assert(offsetof(LazySampleCallbackTicket, task_id) == 8, "lazy sample callback ticket task offset mismatch"); +static_assert(offsetof(LazySampleCallbackTicket, function_id) == 12, "lazy sample callback ticket function offset mismatch"); +static_assert(offsetof(LazySampleCallbackTicket, won) == 14, "lazy sample callback ticket winner offset mismatch"); + +PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { + // This is BeginSubmit without an already-materialized TaskArgs. The same + // fields are completed synchronously after the single callback builds args. + const uint32_t task_id = static_cast(worker.local_index++); + context.self = &worker; + context.payload = &worker.payloads[task_id & kPayloadMask]; + context.task_id = static_cast(task_id); + context.tensor_count = 0; + context.scalar_count = 0; + context.result.task_id = task_id; + context.result.count = 0; + context.register_mask = 0; + context.output_bytes = 0; + context.fanin_count = 0; + context.kernel_id = -1; + context.won = false; + context.joint = false; + context.joint_init = false; + context.joint_block = -1; + context.joint_slot = -1; + context.joint_count = 0; +} + +#if defined(__CCE_AICORE__) || defined(__NPU_ARCH__) +#define PA_LAZY_LAMBDA_DEVICE __aicore__ +#else +#define PA_LAZY_LAMBDA_DEVICE +#endif + +template +PA_DEVICE bool BuildLazySampleCallbackArgs( + PaOrchestrationState &orch, TaskArgs &args, uint32_t batch, bool won, + LocalStats &stats +) { + LazySampleCallbackArgsBuilder callback_builder(args, Kind, won); + auto callback = [&](LazySampleCallbackArgsBuilder &builder) PA_LAZY_LAMBDA_DEVICE { + builder.Begin(); + if constexpr (Kind == TaskKind::Alloc) { + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.tile_create_info; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + } else if constexpr (Kind == TaskKind::Qk) { + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + MakeLazySampleCallbackQueryView(orch, batch); + builder.RecordView(); + return orch.query_view; + }); + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + return orch.key_cache; + }); + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + return orch.block_table; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + const uint32_t score_shape[kMaxTensorDims] = { + kPaHeads, + static_cast(orch.current_nblocks * kPaBlockSize), + 0, 0, 0 + }; + InitCreateInfo(orch.qk_create_info, score_shape, 2, DataType::Float32); + builder.RecordDynamicCreateInfo(); + return orch.qk_create_info; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_nblocks; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + + orch.current_block_offset; + }); + } else if constexpr (Kind == TaskKind::Sf) { + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.qk_scores; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + const uint32_t probability_shape[kMaxTensorDims] = { + kPaHeads, + static_cast(orch.current_nblocks * kPaBlockSize), + 0, 0, 0 + }; + InitCreateInfo(orch.sf_create_info, probability_shape, 2, DataType::Bfloat16); + builder.RecordDynamicCreateInfo(); + return orch.sf_create_info; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { return orch.scale_bits; }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_nblocks; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_valid_len; + }); + } else if constexpr (Kind == TaskKind::Pv) { + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.sf_probs; + }); + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + return orch.value_cache; + }); + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + return orch.block_table; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.tile_create_info; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_nblocks; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + + orch.current_block_offset; + }); + } else { + static_assert(Kind == TaskKind::Up, "unsupported PA task kind"); + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.sf_max; + }); + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.sf_sum; + }); + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.pv_output; + }); + builder.AddGmInout([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.accumulated_max; + }); + builder.AddGmInout([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.accumulated_sum; + }); + builder.AddGmInout([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.accumulated_output; + }); + builder.AddLocalInout([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + MakeLazySampleCallbackOutputView(orch, batch); + builder.RecordView(); + return orch.output_view; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_block_offset == 0 ? 1 : 0; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_block_offset + orch.current_nblocks >= orch.current_blocks ? 1 : 0; + }); + } + }; + + callback(callback_builder); + if (!callback_builder.Valid()) return false; + const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); + stats.result.arg_resets += counts.reset_calls; + stats.result.views_created += counts.views_created; + stats.result.dynamic_create_infos += counts.dynamic_create_infos; + stats.result.tensor_args_added += counts.tensor_args_added; + stats.result.scalar_args_added += counts.scalar_args_added; + return true; +} + +#undef PA_LAZY_LAMBDA_DEVICE + +template +PA_DEVICE bool FinishLazySampleCallbackSubmitBody( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_count, + const TaskArgs &args, SubmitContext &context, LocalStats &stats, + PmuContext &pmu_context, const LazySampleCallbackTicket &ticket +) { + const uint32_t task_id = ticket.task_id; + const TaskKind kind = GetTaskKind(task_id); + const int32_t function_id = static_cast(ticket.function_id); + const bool winner = ticket.won != 0; + + // The callback has ended before this body consumes TaskArgs. No closure + // or nested thunk escapes its lifetime. Inline shapes instantiate this in + // the caller; split shapes instantiate it inside the runtime finish TU. + const uint64_t materialize_begin = TraceTimestamp(stats.trace, stats.result); + BeginSubmitPmuPhase(pmu_context); + const bool materialized = + MaterializeTask(worker, task_id, args, context, state->heap_base, state->heap_size); + if (!materialized) { + EndSubmitPmuPhase(pmu_context); + SetFatal(state, stats, static_cast(task_id)); + return false; + } + stats.result.materialized_outputs += context.result.count; + EndSubmitPmuPhase(pmu_context); + const uint64_t materialize_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Materialize, ProfilePhase::Materialize, + materialize_begin, materialize_end, 0, kind == TaskKind::Alloc ? 1U : 0U + ); + + const uint64_t prepare_begin = materialize_end; + AdvanceTensorMap(worker.map, task_id, static_cast(state->heap_window)); + const uint64_t prepare_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::PrepareMap, ProfilePhase::PrepareMap, + prepare_begin, prepare_end, 0, kind == TaskKind::Alloc ? 1U : 0U + ); + + uint64_t register_begin = prepare_end; + if (kind != TaskKind::Alloc && __builtin_expect(winner, 0)) { + const uint64_t fanin_begin = prepare_end; + context.fanin_count = static_cast(CollectFanin(worker.map, args, context.fanin)); + stats.result.map_lookups += static_cast(args.tensor_count) - context.result.count; + const uint64_t fanin_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Fanin, ProfilePhase::Fanin, + fanin_begin, fanin_end, 0, static_cast(context.fanin_count) + ); + register_begin = fanin_end; + } + + BeginSubmitPmuPhase(pmu_context); + RegisterOutputs(context, args, kind != TaskKind::Alloc); + if (kind != TaskKind::Alloc) stats.result.map_inserts += CountBits(context.register_mask); + EndSubmitPmuPhase(pmu_context); + const uint64_t register_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Register, ProfilePhase::Register, + register_begin, register_end, 0, kind == TaskKind::Alloc ? 0U : 1U + ); + + if (__builtin_expect(winner, 0)) { + const uint64_t winner_build_begin = register_end; + if (kind == TaskKind::Alloc) { + if (!HeapGuard(state, worker, task_id, context.output_bytes, stats)) { + return false; + } + CompleteTask(state, worker, task_id, stats); + } else { + if (!BuildWinner( + state, worker, task_id, kind, args, context, context.fanin, + static_cast(context.fanin_count), stats + )) { + return false; + } + } + const uint64_t winner_build_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + kind == TaskKind::Alloc ? TracePhase::AllocComplete : TracePhase::WinnerBuild, + ProfilePhase::ReplayTail, + winner_build_begin, winner_build_end + ); + } + + ++stats.result.submits; +#if PA_BUILD_SUBMIT_PMU + const uint64_t submit_end = task_id + 1 == task_count ? Ops::Now() : 0; +#else + const uint64_t submit_end = TraceTimestamp(stats.trace, stats.result); +#endif + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Submit, ProfilePhase::Submit, + ticket.submit_begin, submit_end, winner ? 1U : 0U, 0 + ); + if (task_id + 1 == task_count) { + stats.result.submit_end = submit_end; + } + return true; +} + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +template +PA_DEVICE uint32_t FinishSplitLazySampleCallbackFromRuntime( + const LazySampleCallbackTicket *ticket, const TaskArgs *args +) { + LazySampleSplitRuntimeState &runtime = Ops::LazySampleSplitState(); + const uint64_t state_address = reinterpret_cast(&runtime); + runtime.finish_state_address = state_address; + + bool valid = ticket != nullptr && args != nullptr && runtime.scheduler != nullptr && + runtime.worker != nullptr && runtime.task_count != 0 && + runtime.worker_id < kWorkers && runtime.owner_worker_id == runtime.worker_id && + runtime.worker->core_idx == static_cast(runtime.worker_id) && + runtime.caller_state_address == state_address && + runtime.state_cookie == LazySampleSplitStateCookie( + runtime.worker_id, runtime.worker->role + ) && runtime.reserved == 0; + if (valid) { + valid = ticket->reserved == 0 && ticket->task_id < runtime.task_count && + runtime.context.task_id == static_cast(ticket->task_id) && + runtime.context.kernel_id == static_cast(ticket->function_id) && + runtime.context.won == (ticket->won != 0); + } + ++runtime.finish_calls; + if (ticket != nullptr) runtime.task_id_sum += ticket->task_id; + if (!valid) { + ++runtime.protocol_errors; + if (runtime.scheduler != nullptr) { + SetFatal( + runtime.scheduler, runtime.stats, + ticket == nullptr ? -1 : static_cast(ticket->task_id) + ); + } + return 0; + } + + // split finish 不让 caller 的 SubmitContext/LocalStats/PMU 对象跨过 + // noinline 边界。诊断构建只允许 phase=none：权威 CNT6/7 仍由 caller + // 外层完整窗口读取，finish 内部不做 read-clear 局部快照。 +#if PA_BUILD_SUBMIT_PMU + static_assert( + kCompiledSubmitPmuPhase == SubmitPmuPhase::None, + "split callback submit-PMU supports only the whole-window none phase" + ); +#endif + bool pmu_context = false; + return FinishLazySampleCallbackSubmitBody( + runtime.scheduler, *runtime.worker, runtime.task_count, *args, + runtime.context, runtime.stats, pmu_context, *ticket + ) ? 1U : 0U; +} +#endif + +template +PA_DEVICE bool SubmitLazySampleCallback( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_count, + PaOrchestrationState &orch, TaskArgs &args, uint32_t batch, + SubmitContext &context, LocalStats &stats, PmuContext &pmu_context +) { + BeginLazySampleCallbackSubmit(worker, context); + const uint32_t task_id = static_cast(context.task_id); +#if PA_BUILD_SUBMIT_PMU + const uint64_t submit_begin = task_id == 0 ? Ops::Now() : 0; +#else + const uint64_t submit_begin = TraceTimestamp(stats.trace, stats.result); +#endif + if (task_id == 0) stats.result.submit_begin = submit_begin; + + const uint64_t efdrain_begin = submit_begin; + BeginSubmitPmuPhase(pmu_context); + DrainReady(state, worker, DrainPlace::EfDrain, stats); + EndSubmitPmuPhase(pmu_context); + const uint64_t efdrain_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, + TracePhase::EfDrain, ProfilePhase::EfDrain, + efdrain_begin, efdrain_end + ); + + const uint64_t claim_begin = efdrain_end; + BeginSubmitPmuPhase(pmu_context); + const ClaimOutcome claim = Claim(state, worker, task_id, Kind, stats); + context.won = claim.won; + context.kernel_id = claim.function_id; + RecordClaimOutcome(stats, Kind, claim); + EndSubmitPmuPhase(pmu_context); + const uint64_t claim_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), claim.function_id, + TracePhase::Claim, ProfilePhase::Claim, + claim_begin, claim_end, + (claim.won ? kClaimWon : 0U) | (claim.attempted ? kClaimAttempted : 0U), + Kind == TaskKind::Alloc ? 1U : 0U + ); + + if (!BuildLazySampleCallbackArgs(orch, args, batch, claim.won, stats)) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + const LazySampleCallbackTicket ticket{ + submit_begin, + task_id, + // All standalone function IDs are -1..3 and exactly fit this ABI field. + static_cast(claim.function_id), + static_cast(claim.won ? 1 : 0), + 0, + }; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + (void)state; + (void)worker; + (void)task_count; + (void)context; + (void)stats; + (void)pmu_context; + return Ops::FinishLazySampleCallback(&ticket, &args); +#else + return FinishLazySampleCallbackSubmitBody( + state, worker, task_count, args, context, stats, pmu_context, ticket + ); +#endif +} +#endif + +PA_DEVICE uint32_t CountLiveMapEntries(PA_GM const TensorMap &map) { + uint32_t free_entries = 0; + for (int32_t current = map.free_head; current >= 0; current = map.entries[current].next_in_bucket) { + ++free_entries; + } + return static_cast(map.high_water) - free_entries; +} + +template +PA_DEVICE void PublishResult(PA_GM WorkerResult &destination, const WorkerResult &source) { + // 每个 worker 只写自己独占、覆盖多条 cache line 的 WorkerResult 分区；逐字段 + // bypass 保证结果对 host 可见，而独立 sidecar 允许 D2H 只搬结果、不搬约 9 MiB WorkerState。 +#define PA_PUBLISH_FIELD(field) Ops::Publish(&destination.field, source.field) + PA_PUBLISH_FIELD(submit_begin); + PA_PUBLISH_FIELD(submit_end); + PA_PUBLISH_FIELD(finish_cycle); + PA_PUBLISH_FIELD(checksum); + PA_PUBLISH_FIELD(submits); + PA_PUBLISH_FIELD(claim_attempts); + PA_PUBLISH_FIELD(claim_wins); + PA_PUBLISH_FIELD(heap_guards); + PA_PUBLISH_FIELD(fanin_ready_loads); + PA_PUBLISH_FIELD(completion_duplicates); + PA_PUBLISH_FIELD(cas_retries); + PA_PUBLISH_FIELD(joint_polls); + for (uint32_t index = 0; index < static_cast(TaskKind::Count); ++index) { + Ops::Publish(&destination.wins[index], source.wins[index]); + } + for (uint32_t index = 0; index < 4; ++index) { + Ops::Publish(&destination.kernel_counts[index], source.kernel_counts[index]); + Ops::Publish(&destination.kernel_cycles[index], source.kernel_cycles[index]); + Ops::Publish(&destination.kernel_min_cycles[index], source.kernel_min_cycles[index]); + Ops::Publish(&destination.kernel_max_cycles[index], source.kernel_max_cycles[index]); + } + for (uint32_t index = 0; index < static_cast(DrainPlace::Count); ++index) { + Ops::Publish(&destination.placement[index], source.placement[index]); + } + for (uint32_t index = 0; index < static_cast(ProfilePhase::Count); ++index) { + Ops::Publish(&destination.phase_cycles[index], source.phase_cycles[index]); + Ops::Publish(&destination.phase_calls[index], source.phase_calls[index]); + } + for (uint32_t index = 0; index < 2; ++index) { + Ops::Publish(&destination.wait_events[index], source.wait_events[index]); + Ops::Publish(&destination.wait_iterations[index], source.wait_iterations[index]); + } + PA_PUBLISH_FIELD(context_reads); + PA_PUBLISH_FIELD(views_created); + PA_PUBLISH_FIELD(dynamic_create_infos); + PA_PUBLISH_FIELD(arg_resets); + PA_PUBLISH_FIELD(tensor_args_added); + PA_PUBLISH_FIELD(scalar_args_added); + PA_PUBLISH_FIELD(materialized_outputs); + PA_PUBLISH_FIELD(map_inserts); + PA_PUBLISH_FIELD(map_lookups); + PA_PUBLISH_FIELD(slot_tensor_copies); + PA_PUBLISH_FIELD(slot_scalar_copies); + PA_PUBLISH_FIELD(fanin_edges); + PA_PUBLISH_FIELD(final_heap_next); + PA_PUBLISH_FIELD(map_high_water); + PA_PUBLISH_FIELD(map_alive_floor); + PA_PUBLISH_FIELD(map_cleaned_upto); + PA_PUBLISH_FIELD(map_live_entries); + PA_PUBLISH_FIELD(worker_id); + PA_PUBLISH_FIELD(role); + PA_PUBLISH_FIELD(max_occupied); + PA_PUBLISH_FIELD(final_occupied); + PA_PUBLISH_FIELD(fanin_not_ready_loads); + PA_PUBLISH_FIELD(frontier_initial_loads); + PA_PUBLISH_FIELD(frontier_updates); + PA_PUBLISH_FIELD(frontier_terminal_loads); + PA_PUBLISH_FIELD(atomic_trace_calls); +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + PA_PUBLISH_FIELD(lazy_sample_split_caller_state_address); + PA_PUBLISH_FIELD(lazy_sample_split_finish_state_address); + PA_PUBLISH_FIELD(lazy_sample_split_finish_calls); + PA_PUBLISH_FIELD(lazy_sample_split_protocol_errors); + PA_PUBLISH_FIELD(lazy_sample_split_state_cookie); + PA_PUBLISH_FIELD(lazy_sample_split_task_id_sum); + PA_PUBLISH_FIELD(lazy_sample_split_owner_worker_id); + PA_PUBLISH_FIELD(lazy_sample_split_reserved); +#endif +#undef PA_PUBLISH_FIELD + Ops::StoreBarrier(); +} + +template +PA_DEVICE void RunSchedulerImpl(PA_GM SchedulerState *state, uint32_t worker_id, CoreRole role) { + // 一个入口实例只拥有 state->workers[worker_id] 的私有 map/ring/payload；cursor、task cell 和屏障为跨核共享区。 + if (worker_id >= kWorkers) { + return; + } + PA_GM WorkerState &worker = state->workers[worker_id]; + worker.role = role; + worker.core_idx = static_cast(worker_id); + // standalone 使用连续 worker 编号：AIC 为 0..31；AIV 为 32..95。 + // 每个物理 block b 对应 AIC(b, lane0)、AIV(32+2b, lane1)、AIV(33+2b, lane2)。 + if (role == CoreRole::Aic) { + worker.block_id = static_cast(worker_id); + worker.lane = 0; + } else { + const uint32_t vector_id = worker_id - kAicWorkers; + worker.block_id = static_cast(vector_id / 2); + worker.lane = static_cast(1 + vector_id % 2); + } + worker.sub_block_id = worker.lane == 2 ? 1 : 0; + worker.local_index = 0; + worker.heap_next = 0; + ResetTensorMap(worker.map); + worker.occupied_count = 0; + worker.owned_total = 0; + worker.swimlane_last_cycle = 0; + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + worker.slots[index].occupied = false; + worker.slots[index].built = false; + } + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + LazySampleSplitRuntimeState &lazy_sample_split_runtime = Ops::LazySampleSplitState(); + lazy_sample_split_runtime.context = SubmitContext{}; + lazy_sample_split_runtime.stats = LocalStats{}; + lazy_sample_split_runtime.scheduler = state; + lazy_sample_split_runtime.worker = &worker; + lazy_sample_split_runtime.task_count = 0; + lazy_sample_split_runtime.worker_id = worker_id; + lazy_sample_split_runtime.caller_state_address = reinterpret_cast(&lazy_sample_split_runtime); + lazy_sample_split_runtime.finish_state_address = 0; + lazy_sample_split_runtime.finish_calls = 0; + lazy_sample_split_runtime.protocol_errors = 0; + lazy_sample_split_runtime.state_cookie = LazySampleSplitStateCookie(worker_id, role); + lazy_sample_split_runtime.task_id_sum = 0; + lazy_sample_split_runtime.owner_worker_id = worker_id; + lazy_sample_split_runtime.reserved = 0; + LocalStats &stats = lazy_sample_split_runtime.stats; +#else + LocalStats stats{}; +#endif + stats.result.worker_id = worker_id; + stats.result.role = static_cast(role); + stats.result.checksum = 0xcbf29ce484222325ULL ^ worker_id; + // 该 invalidate 原先藏在 AttachTrace 中；它同时保护 PMU mode/register + // table 与 winner workload，必须在两个构建中都执行。 + Ops::InvalidateRegion( + &state->config, sizeof(state->config) + sizeof(state->winner_workload) + ); + stats.trace = AttachTrace(state, worker, worker_id); + + // 96 个参与者全部完成本地状态初始化后再进入 task 0，主要用于压低启动偏斜对 + // winner 分布和 Submit 时序的干扰；atomicMax 的唯一 winner 正确性本身不依赖该屏障。 + TraceAtomicFetchAdd( + stats.trace, stats.result, -1, AtomicSite::StartupIncrement, + &state->started_count.value, 1 + ); + const uint64_t start_wait = Ops::Now(); + uint32_t start_polls = 0; + const uint32_t startup_poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, + TraceAtomicSiteMask(AtomicSite::StartupPoll) | TraceAtomicSiteMask(AtomicSite::FatalPoll) + ); + // 全员到齐或任一核发布 fatal 即退出启动等待；watchdog 防止缺失参与者造成永久挂死。 + while (LoadLine(state->started_count, stats, AtomicSite::StartupPoll) < + static_cast(state->config.workers) && + !IsFatal(state, stats)) { + Ops::SpinHint(); + if (WatchdogExpired(state, stats, start_wait, start_polls)) { + break; + } + } + AtomicPollRegionEnd(stats.trace, stats.result, startup_poll_region); + + const uint32_t batches = state->config.batches; + const uint32_t task_count = batches * kTasksPerBatch; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + lazy_sample_split_runtime.task_count = task_count; +#endif + PaOrchestrationState orchestration; + TaskArgs args; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + SubmitContext &context = lazy_sample_split_runtime.context; +#else + SubmitContext context; +#endif + uint64_t orchestration_begin = 0; + uint64_t orchestration_end = 0; + if (!IsFatal(state, stats)) { + // Case1 每个 batch 固定回放 Alloc/QK/SF/PV/UP 五个 task；所有 worker 顺序相同，执行 lane 由 Claim 筛选。 + // CCEC 可在这里开启本 worker 私有 PMU 窗口；CPU/AscendC 适配层是空实现。 + // 窗口覆盖从首个参数构造到末次 Submit 返回，与全局“首 Submit.begin～末 Submit.end” + // 口径接近但不相同，host sidecar 必须按 per-worker 累计解释。 + // 泳道父边界在 PMU-only 构建中会被编译为空，不应污染 Submit 取数；窗口从 + // orchestration 初始化（即首批参数构造）前一条边界开始。 + auto pmu_context = Ops::PmuWindowStart(state, worker_id); + orchestration_begin = TraceTimestamp(stats.trace, stats.result); + InitPaOrchestration(orchestration, batches, &state->context_lens[0]); + for (uint32_t batch = 0; batch < batches; ++batch) { +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + BeginPaBatchForLazySampleCallback(orchestration, batch); + ++stats.result.context_reads; + if (!SubmitLazySampleCallback< + TaskKind::Alloc, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Alloc, context.result); + + PreparePaBlockGroup(orchestration, 0); + if (!SubmitLazySampleCallback< + TaskKind::Qk, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Qk, context.result); + + if (!SubmitLazySampleCallback< + TaskKind::Sf, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Sf, context.result); + + if (!SubmitLazySampleCallback< + TaskKind::Pv, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Pv, context.result); + + if (!SubmitLazySampleCallback< + TaskKind::Up, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } +#else + BuildAllocArgs(orchestration, args, batch); + ++stats.result.context_reads; + stats.result.views_created += 2; + stats.result.tensor_args_added += 3; + if (!SubmitTask( + state, worker, task_count, TaskKind::Alloc, args, context, stats, pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Alloc, context.result); + + BuildQkArgs(orchestration, args, batch); + ++stats.result.dynamic_create_infos; + ++stats.result.arg_resets; + stats.result.tensor_args_added += 4; + stats.result.scalar_args_added += 2; + if (!SubmitTask( + state, worker, task_count, TaskKind::Qk, args, context, stats, pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Qk, context.result); + + BuildSfArgs(orchestration, args); + ++stats.result.dynamic_create_infos; + ++stats.result.arg_resets; + stats.result.tensor_args_added += 4; + stats.result.scalar_args_added += 3; + if (!SubmitTask( + state, worker, task_count, TaskKind::Sf, args, context, stats, pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Sf, context.result); + + BuildPvArgs(orchestration, args, batch); + ++stats.result.arg_resets; + stats.result.tensor_args_added += 4; + stats.result.scalar_args_added += 2; + if (!SubmitTask( + state, worker, task_count, TaskKind::Pv, args, context, stats, pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Pv, context.result); + + BuildUpdateArgs(orchestration, args); + ++stats.result.arg_resets; + stats.result.tensor_args_added += 7; + stats.result.scalar_args_added += 2; + if (!SubmitTask( + state, worker, task_count, TaskKind::Up, args, context, stats, pmu_context + )) { + break; + } +#endif + } + Ops::PmuWindowStop(state, worker_id, pmu_context); + orchestration_end = TraceTimestamp(stats.trace, stats.result); + } + + // replay_done 表示所有 worker 已退出回放循环（成功路径即完整提交）；之后仍需 drain 到本核 slot 为空。 + // 成功路径复用 orchestration end 作为 final drain start，使两个业务父区间 + // 首尾相接；父记录延后到 final drain 结束再写，避免记录自身落进任一业务 span。 + const uint64_t final_drain_begin = orchestration_end != 0 + ? orchestration_end + : TraceTimestamp(stats.trace, stats.result); + TraceAtomicFetchAdd( + stats.trace, stats.result, -1, AtomicSite::ReplayDoneIncrement, + &state->replay_done.value, 1 + ); + const uint32_t final_poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, + TraceAtomicSiteMask(AtomicSite::ReplayDonePoll) | TraceAtomicSiteMask(AtomicSite::FaninFlagLoad) + ); + while (true) { + const uint32_t freed = + DrainReady(state, worker, DrainPlace::FinalDrain, stats); + const bool all_replayed = + LoadLine(state->replay_done, stats, AtomicSite::ReplayDonePoll) >= + static_cast(state->config.workers); + // 必须同时满足“无人再生产新 slot”和“本核旧 slot 全部完成”，否则继续帮助系统推进 completion。 + if (all_replayed && worker.occupied_count == 0) { + break; + } + if (freed == 0) { + Ops::SpinHint(); + } + } + AtomicPollRegionEnd(stats.trace, stats.result, final_poll_region); + const uint64_t final_drain_end = TraceTimestamp(stats.trace, stats.result); + if (orchestration_begin != 0 && orchestration_end >= orchestration_begin) { + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::OrchestrationReplay, + ProfilePhase::Orchestration, orchestration_begin, orchestration_end + ); + } + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::FinalDrain, + ProfilePhase::ReplayTail, final_drain_begin, final_drain_end + ); + +#if !PA_BUILD_SUBMIT_PMU + if (stats.trace.atomics_enabled) { + // 两条基线都放在最终 drain 之后。第一条量连续 + // SYS_CNT，第二条量返回依赖钩子的固定成本；它们只描述计时底噪，不能 + // 从每条 atomic 中机械相减后宣称得到跨核全局可见性延迟。 + const uint64_t clock_begin = Ops::Now(); + const uint64_t clock_end = Ops::Now(); + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::ClockBaseline, + ProfilePhase::ReplayTail, clock_begin, clock_end + ); + const uint64_t dependency_begin = Ops::Now(); + const uint64_t dependency_end = Ops::NowAfterAtomicResult( + static_cast(worker_id) + ); + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::ClockBaseline, + ProfilePhase::ReplayTail, dependency_begin, dependency_end, + kClockAtomicDependency | + (Ops::kAtomicReturnReadyObserved ? kClockAtomicDependencyApplied : 0U) + ); + } +#endif + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + const uint64_t expected_callback_task_id_sum = + static_cast(task_count) * (task_count - 1U) / 2U; + const bool split_protocol_ok = + lazy_sample_split_runtime.scheduler == state && lazy_sample_split_runtime.worker == &worker && + lazy_sample_split_runtime.task_count == task_count && lazy_sample_split_runtime.worker_id == worker_id && + lazy_sample_split_runtime.owner_worker_id == worker_id && + lazy_sample_split_runtime.caller_state_address != 0 && + lazy_sample_split_runtime.finish_state_address == lazy_sample_split_runtime.caller_state_address && + lazy_sample_split_runtime.finish_calls == task_count && + lazy_sample_split_runtime.task_id_sum == expected_callback_task_id_sum && + lazy_sample_split_runtime.state_cookie == LazySampleSplitStateCookie(worker_id, role) && + lazy_sample_split_runtime.reserved == 0; + if (!split_protocol_ok) { + ++lazy_sample_split_runtime.protocol_errors; + SetFatal(state, stats); + } + stats.result.lazy_sample_split_caller_state_address = lazy_sample_split_runtime.caller_state_address; + stats.result.lazy_sample_split_finish_state_address = lazy_sample_split_runtime.finish_state_address; + stats.result.lazy_sample_split_finish_calls = lazy_sample_split_runtime.finish_calls; + stats.result.lazy_sample_split_protocol_errors = lazy_sample_split_runtime.protocol_errors; + stats.result.lazy_sample_split_state_cookie = lazy_sample_split_runtime.state_cookie; + stats.result.lazy_sample_split_task_id_sum = lazy_sample_split_runtime.task_id_sum; + stats.result.lazy_sample_split_owner_worker_id = lazy_sample_split_runtime.owner_worker_id; + stats.result.lazy_sample_split_reserved = lazy_sample_split_runtime.reserved; +#endif + + // PA writes swimlane records through the ordinary GM cache and explicitly + // cleans each worker's record range before the kernel finishes. + FlushTraceCore(stats.trace, stats.result); + stats.result.finish_cycle = Ops::Now(); + stats.result.max_occupied = stats.max_occupied; + stats.result.final_occupied = worker.occupied_count; + stats.result.final_heap_next = worker.heap_next; + stats.result.map_high_water = static_cast(worker.map.high_water); + stats.result.map_alive_floor = static_cast(worker.map.alive_floor); + stats.result.map_cleaned_upto = static_cast(worker.map.cleaned_upto); + stats.result.map_live_entries = CountLiveMapEntries(worker.map); + PublishResult(state->results[worker_id], stats.result); +} + +template +PA_DEVICE void RunScheduler(PA_GM SchedulerState *state, uint32_t worker_id, CoreRole role) { + // 两个正式 CCEC 构建都不再携带旧 phase-profile 模板副本：swimlane 用 + // records 表达阶段，submit-pmu 使用独立 PMU 边界。其他后端暂时保留原 + // 运行时入口，保证公共 standalone 的 CPU/AscendC 回归不被 CCEC 构建切分影响。 +#if PA_BUILD_SWIMLANE || PA_BUILD_SUBMIT_PMU + RunSchedulerImpl(state, worker_id, role); +#else + // Profile 作为编译期模板参数，只在显式开启时保留阶段累计代码，关闭时不在热路径增加运行时分支。 + if (state->config.profile_phases != 0) { + RunSchedulerImpl(state, worker_id, role); + } else { + RunSchedulerImpl(state, worker_id, role); + } +#endif +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_SCHEDULER_CORE_H diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/common/pa_trace.h b/tests/atomic_probe/lazy_lamda_sample/A_original/common/pa_trace.h new file mode 100644 index 0000000000..c70a4e681b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/common/pa_trace.h @@ -0,0 +1,655 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_TRACE_H +#define PA_SCHEDULER_COMMON_PA_TRACE_H + +#include "pa_model.h" + +#ifndef PA_DEVICE_NOINLINE +#define PA_DEVICE_NOINLINE PA_DEVICE +#endif + +#ifndef PA_LOOP_NOUNROLL +#define PA_LOOP_NOUNROLL +#endif + +namespace pa_scheduler { + +// 记录区与真实 PA 一样直接拼在定长 Header 后面；worker_id 只选择自己的 +// records 分区，避免记录动作本身制造跨核共享写热点。 +PA_DEVICE PA_GM TraceRecord *GetTraceRecords(PA_GM TraceHeader *header) { + // 输入必须指向完整且按 64 byte 对齐的 trace buffer；返回值只是首条记录， + // 调用方还需按 worker_id * capacity 选择自己的分区。 + return reinterpret_cast(reinterpret_cast(header) + sizeof(TraceHeader)); +} + +struct AtomicPollBurst { + uint64_t start_cycle[kAtomicPollBatchSiteCount]; + uint32_t call_count[kAtomicPollBatchSiteCount]; + uint32_t active_mask; + uint32_t enabled_mask; +}; + +struct TraceContext { + PA_GM TraceCoreState *core; + PA_GM TraceRecord *records; + uint32_t capacity; + bool atomics_enabled; + int32_t lane; + int32_t block_id; + int32_t core_idx; + // 轮询调用数留在 worker 私有上下文，最终一次性发布到 core state； + // 等待热路不为计数再写共享/GM 状态。 + uint64_t poll_calls; + uint64_t poll_batch_records; + bool atomic_counter_overflow; + AtomicPollBurst poll_burst; +}; + +// pa_model.h 也向 host 暴露同一 raw ABI 映射，但 CCEC/AscendC 的单个 TU +// 会先以 host 语境包含该头，再实例化 device 调度器。这里保留明确的 device +// 版本，避免设备函数误调用先前已实例化的 __host__ helper。 +PA_DEVICE AtomicOp TraceAtomicSiteExpectedOp(AtomicSite site) { + switch (site) { + case AtomicSite::StartupIncrement: + case AtomicSite::ReplayDoneIncrement: + return AtomicOp::FetchAdd; + case AtomicSite::FatalSet: + case AtomicSite::CompletionVendExchange: + case AtomicSite::CompletionFlagExchange: + return AtomicOp::Exchange; + case AtomicSite::ClaimMax: + case AtomicSite::FrontierMax: + return AtomicOp::FetchMax; + default: + return AtomicOp::Load; + } +} + +PA_DEVICE int32_t TraceAtomicPollBatchIndex(AtomicSite site) { + switch (site) { + case AtomicSite::StartupPoll: + return 0; + case AtomicSite::FatalPoll: + return 1; + case AtomicSite::FaninFlagLoad: + return 2; + case AtomicSite::HeapFrontierLoad: + return 3; + case AtomicSite::HeapVendLoad: + return 4; + case AtomicSite::ReplayDonePoll: + return 5; + default: + return -1; + } +} + +PA_DEVICE AtomicSite TraceAtomicPollBatchSite(uint32_t index) { + switch (index) { + case 0: + return AtomicSite::StartupPoll; + case 1: + return AtomicSite::FatalPoll; + case 2: + return AtomicSite::FaninFlagLoad; + case 3: + return AtomicSite::HeapFrontierLoad; + case 4: + return AtomicSite::HeapVendLoad; + case 5: + return AtomicSite::ReplayDonePoll; + default: + return AtomicSite::Count; + } +} + +PA_DEVICE bool TraceAtomicSiteIsPollBatchable(AtomicSite site) { + return TraceAtomicPollBatchIndex(site) >= 0; +} + +PA_DEVICE uint32_t TraceAtomicSiteMask(AtomicSite site) { + return 1U << static_cast(site); +} + +// Attach 只缓存本 worker 的 header 状态、分区首址和物理 lane 信息。控制区 +// cache invalidate 在公共调度入口完成，不能随 submit-pmu 编译掉泳道而消失。 +template +PA_DEVICE TraceContext AttachTrace( + PA_GM SchedulerState *state, PA_GM const WorkerState &worker, uint32_t worker_id +) { + TraceContext trace{}; + trace.lane = worker.lane; + trace.block_id = worker.block_id; + trace.core_idx = static_cast(worker_id); +#if PA_BUILD_SUBMIT_PMU + // 诊断 ELF 不含 records 写入、atomic span 或 trace-only SYS_CNT；保留同一 + // TraceContext 形状只是为了复用调度协议源码。 + (void)state; + return trace; +#else + const uint64_t base = state->config.trace_base; + const uint32_t capacity = state->config.trace_records_per_core; + if ((state->config.trace_enabled & kTracePhasesEnabled) == 0 || base == 0 || capacity == 0 || + worker_id >= kWorkers) { + return trace; + } + PA_GM TraceHeader *header = reinterpret_cast(base); + if (worker_id >= header->num_cores) { + return trace; + } + trace.core = &header->cores[worker_id]; + trace.records = &GetTraceRecords(header)[static_cast(worker_id) * capacity]; + // 成功返回的不变量是 core/records/capacity 同时有效；任一前置条件失败则三者 + // 保持空值，后续 WriteTrace/FlushTraceCore 可无分支地安全退化为 no-op。 + trace.capacity = capacity; + trace.atomics_enabled = (state->config.trace_enabled & kTraceAtomicsEnabled) != 0; + trace.core->count = 0; + trace.core->dropped = 0; + trace.core->atomic_calls = 0; + trace.core->poll_calls = 0; + trace.core->poll_batch_records = 0; + trace.core->core_idx = trace.core_idx; + trace.core->block_id = trace.block_id; + trace.core->lane = trace.lane; + return trace; +#endif +} + +template +PA_DEVICE void WriteTrace( + TraceContext &trace, WorkerResult &result, int32_t task_id, int32_t function_id, + TracePhase trace_phase, ProfilePhase profile_phase, uint64_t start_cycle, uint64_t end_cycle, + uint32_t flags = 0, uint32_t auxiliary = 0 +); + +// CCEC 不让栈上的 TraceContext/WorkerResult 引用跨非内联调用。这里仅把 +// PollBatch 固定形状的 64-byte GM 写入抽成共享函数，以抑制各 phase 边界 +// 内联后的代码膨胀；参数只有 GM 指针与标量，局部 batch 状态仍由调用者维护。 +PA_DEVICE_NOINLINE bool WritePollBatchRecordRaw( + PA_GM TraceCoreState *core, PA_GM TraceRecord *records, uint32_t capacity, + uint64_t start_cycle, uint64_t end_cycle, uint32_t call_count, uint32_t site_id +) { +#if PA_BUILD_SUBMIT_PMU + (void)core; + (void)records; + (void)capacity; + (void)start_cycle; + (void)end_cycle; + (void)call_count; + (void)site_id; + return false; +#else + if (core == nullptr || records == nullptr || capacity == 0) { + return false; + } + const uint32_t slot = core->count; + if (slot >= capacity) { + core->dropped = core->dropped + 1; + return false; + } + PA_GM TraceRecord &record = records[slot]; + record.start_cycle = start_cycle; + record.end_cycle = end_cycle; + record.task_id = -1; + record.function_id = -1; + record.phase = static_cast(TracePhase::Atomic); + record.lane = core->lane; + record.block_id = core->block_id; + record.core_idx = core->core_idx; + const AtomicSite site = static_cast(site_id); + record.flags = static_cast(TraceAtomicSiteExpectedOp(site)) | + kAtomicResultUsed | kAtomicPollBatch | + (call_count << kAtomicPollCountShift); + record.auxiliary = site_id; + // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 + core->count = slot + 1; + return true; +#endif +} + +PA_DEVICE uint32_t AtomicTraceFlags( + AtomicOp op, bool result_used, bool return_ready, bool value_zero = false, + uint64_t retries = 0 +) { + // 高 24 bit 只能容纳有限重试次数；A5 硬件 atomicMax 当前报告 0，CPU CAS + // 回归若超过范围则饱和，避免溢出覆盖低位的 op/语义标志。 + constexpr uint64_t kMaxRetries = (1ULL << (32 - kAtomicRetriesShift)) - 1; + const uint32_t encoded_retries = static_cast(retries > kMaxRetries ? kMaxRetries : retries); + return static_cast(op) | (result_used ? kAtomicResultUsed : 0U) | + (value_zero ? kAtomicValueZero : 0U) | (return_ready ? kAtomicReturnReady : 0U) | + (encoded_retries << kAtomicRetriesShift); +} + +PA_DEVICE void CountAtomicCall( + TraceContext &trace, WorkerResult &result, bool poll_batch +) { + if (result.atomic_trace_calls == UINT64_MAX) { + trace.atomic_counter_overflow = true; + return; + } + ++result.atomic_trace_calls; + if (!poll_batch) return; + if (trace.poll_calls == UINT64_MAX) { + trace.atomic_counter_overflow = true; + return; + } + ++trace.poll_calls; +} + +template +PA_DEVICE void AtomicPollBoundaryAt( + TraceContext &trace, uint64_t end_cycle +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)end_cycle; +#else + if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; + const uint32_t active_mask = trace.poll_burst.active_mask; + // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 + // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 + PA_LOOP_NOUNROLL + for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { + const uint32_t bit = 1U << index; + if ((active_mask & bit) == 0) continue; + const uint32_t call_count = trace.poll_burst.call_count[index]; + if (call_count == 0 || call_count > kAtomicPollCountMax) { + trace.atomic_counter_overflow = true; + continue; + } + const AtomicSite site = TraceAtomicPollBatchSite(index); + const bool written = WritePollBatchRecordRaw( + trace.core, trace.records, trace.capacity, + trace.poll_burst.start_cycle[index], end_cycle, + call_count, static_cast(site) + ); + if (written) { + if (trace.poll_batch_records == UINT64_MAX) { + trace.atomic_counter_overflow = true; + } else { + ++trace.poll_batch_records; + } + } + trace.poll_burst.call_count[index] = 0; + } + trace.poll_burst.active_mask = 0; +#endif +} + +template +PA_DEVICE void AtomicPollBoundary(TraceContext &trace, WorkerResult &result) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; +#else + (void)result; + if (trace.poll_burst.active_mask == 0) return; + AtomicPollBoundaryAt(trace, Ops::Now()); +#endif +} + +template +PA_DEVICE uint32_t AtomicPollRegionBegin( + TraceContext &trace, WorkerResult &result, uint32_t site_mask +) { + const uint32_t previous_mask = trace.poll_burst.enabled_mask; +#if PA_BUILD_SUBMIT_PMU + (void)result; + (void)site_mask; +#else + if (!trace.atomics_enabled) return previous_mask; + AtomicPollBoundary(trace, result); + trace.poll_burst.enabled_mask = previous_mask | site_mask; +#endif + return previous_mask; +} + +template +PA_DEVICE void AtomicPollRegionEnd( + TraceContext &trace, WorkerResult &result, uint32_t previous_mask +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)previous_mask; +#else + if (!trace.atomics_enabled) return; + AtomicPollBoundary(trace, result); + trace.poll_burst.enabled_mask = previous_mask; +#endif +} + +PA_DEVICE bool AtomicPollBatchEnabled( + TraceContext &trace, AtomicSite site, AtomicOp actual_op +) { + return trace.atomics_enabled && TraceAtomicSiteIsPollBatchable(site) && + TraceAtomicSiteExpectedOp(site) == actual_op && + (trace.poll_burst.enabled_mask & TraceAtomicSiteMask(site)) != 0; +} + +template +PA_DEVICE void AccumulateAtomicPollCall( + TraceContext &trace, WorkerResult &result, AtomicSite site, uint64_t start_cycle +) { + const int32_t signed_index = TraceAtomicPollBatchIndex(site); + if (signed_index < 0) { + trace.atomic_counter_overflow = true; + return; + } + const uint32_t index = static_cast(signed_index); + const uint32_t bit = 1U << index; + if ((trace.poll_burst.active_mask & bit) == 0) { + trace.poll_burst.start_cycle[index] = start_cycle; + trace.poll_burst.call_count[index] = 0; + trace.poll_burst.active_mask |= bit; + } + uint32_t &call_count = trace.poll_burst.call_count[index]; + ++call_count; + if (call_count == kAtomicPollCountMax) { + AtomicPollBoundary(trace, result); + } +} + +template +PA_DEVICE void WriteAtomicTrace( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, AtomicOp op, + uint64_t start_cycle, uint64_t end_cycle, bool result_used, bool return_ready, + bool value_zero = false, uint64_t retries = 0 +) { + // 一次源码 atomic 只写一条同时含 start/end 的 span；结束时间先于 64B record + // 写入，因此本条区间不直接包含自己的记录写开销，但下一次竞争到达会受它影响。 + CountAtomicCall(trace, result, false); + WriteTrace( + trace, result, task_id, -1, TracePhase::Atomic, ProfilePhase::ReplayTail, + start_cycle, end_cycle, + AtomicTraceFlags(op, result_used, return_ready, value_zero, retries), + static_cast(site) + ); +} + +template +PA_DEVICE T TraceAtomicLoad( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile T *address, bool result_used = true +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::Load(address); +#else + if (!trace.atomics_enabled) return Ops::Load(address); + const bool poll_batch = result_used && AtomicPollBatchEnabled(trace, site, AtomicOp::Load); + const int32_t poll_index = poll_batch ? TraceAtomicPollBatchIndex(site) : -1; + const bool first_in_batch = poll_batch && + (trace.poll_burst.active_mask & (1U << static_cast(poll_index))) == 0; + const uint64_t begin = !poll_batch || first_in_batch ? Ops::Now() : 0; + const T old = Ops::Load(address); + if (poll_batch) { + CountAtomicCall(trace, result, true); + AccumulateAtomicPollCall(trace, result, site, begin); + return old; + } + // CCEC 只在返回值本来就参与协议判断时插入一条依赖 MOV，再读 SYS_CNT。 + // 这样不会把未消费返回值的 RED/no-return 路径强制改成返回型 ATOM。 + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::Load, begin, end, result_used, return_ready, + old == static_cast(0) + ); + return old; +#endif +} + +template +PA_DEVICE T TraceAtomicExchange( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile T *address, T value, bool result_used = false +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::Exchange(address, value); +#else + if (!trace.atomics_enabled) return Ops::Exchange(address, value); + const uint64_t begin = Ops::Now(); + const T old = Ops::Exchange(address, value); + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::Exchange, begin, end, result_used, return_ready + ); + return old; +#endif +} + +template +PA_DEVICE int64_t TraceAtomicFetchAdd( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile int64_t *address, int64_t value, bool result_used = false +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::FetchAdd(address, value); +#else + if (!trace.atomics_enabled) return Ops::FetchAdd(address, value); + const uint64_t begin = Ops::Now(); + const int64_t old = Ops::FetchAdd(address, value); + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::FetchAdd, begin, end, result_used, return_ready + ); + return old; +#endif +} + +template +PA_DEVICE int64_t TraceAtomicFetchMax( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile int64_t *address, int64_t value, uint64_t &retries, bool result_used = true +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::FetchMax(address, value, retries); +#else + if (!trace.atomics_enabled) return Ops::FetchMax(address, value, retries); + const uint64_t begin = Ops::Now(); + const int64_t old = Ops::FetchMax(address, value, retries); + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::FetchMax, begin, end, result_used, + return_ready, false, retries + ); + return old; +#endif +} + +template +PA_DEVICE void AccumulatePhase( + WorkerResult &result, ProfilePhase phase, uint64_t start_cycle, uint64_t end_cycle +) { +#if PA_BUILD_SUBMIT_PMU + (void)result; + (void)phase; + (void)start_cycle; + (void)end_cycle; + return; +#else + // phase profile 与完整泳道是两套正交机制：即使关闭 records，Profile=true + // 仍会累计用户当前关注的 Claim/EfDrain/WaitForSlot/HeapGuard 四段。 + if constexpr (Profile) { + if (phase != ProfilePhase::Claim && phase != ProfilePhase::EfDrain && + phase != ProfilePhase::WaitForSlot && phase != ProfilePhase::HeapGuard) { + return; + } + const uint32_t index = static_cast(phase); + // 调用方保证 end_cycle>=start_cycle；各后端把 Now() 归一到每 tick 1 ns 的 + // 数值标度，聚合持续时间可直接相加并在 host 侧按 1000 换算为微秒。 + const uint64_t duration = end_cycle - start_cycle; + result.phase_cycles[index] += duration; + ++result.phase_calls[index]; + } +#endif +} + +template +PA_DEVICE void WriteTrace( + TraceContext &trace, WorkerResult &result, int32_t task_id, int32_t function_id, TracePhase trace_phase, + ProfilePhase profile_phase, uint64_t start_cycle, uint64_t end_cycle, uint32_t flags, + uint32_t auxiliary +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)function_id; + (void)trace_phase; + (void)profile_phase; + (void)start_cycle; + (void)end_cycle; + (void)flags; + (void)auxiliary; + return; +#else + // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 + // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 + AccumulatePhase(result, profile_phase, start_cycle, end_cycle); + if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { + return; + } + PA_GM TraceCoreState &core = *trace.core; + const uint32_t slot = core.count; + if (slot >= trace.capacity) { + core.dropped = core.dropped + 1; + return; + } + PA_GM TraceRecord &record = trace.records[slot]; + record.start_cycle = start_cycle; + record.end_cycle = end_cycle; + record.task_id = task_id; + record.function_id = function_id; + record.phase = static_cast(trace_phase); + record.lane = trace.lane; + record.block_id = trace.block_id; + record.core_idx = trace.core_idx; + record.flags = flags; + record.auxiliary = auxiliary; + // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 + // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 + core.count = slot + 1; +#endif +} + +template +PA_DEVICE void ResetTraceLap( + TraceContext &trace, WorkerResult &result, PA_GM WorkerState &worker +) { + // lap 是后续 Build/Replay/Alloc 等覆盖式阶段的共同起点，不代表新增嵌套 span。 + // 因此分析时不能把 lap 时长再与其中的 Materialize/Claim/Register 直接相加。 +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)worker; +#else + (void)result; + const uint64_t cycle = Ops::Now(); + // 与真实 FDWIC 的 TRACE_LAP_RESET 保持同一边界：等待区 PollBatch + // 只能覆盖本次逻辑轮询 episode，不能跨进下一段 lap 或计算单元执行。 + AtomicPollBoundaryAt(trace, cycle); + worker.swimlane_last_cycle = cycle; +#endif +} + +template +PA_DEVICE void FlushTraceCore(TraceContext &trace, WorkerResult &result) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + return; +#else + if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { + return; + } + // 防御性关闭任何尚未由显式 region end 关闭的等待包；正常路径上 active_mask + // 应为 0，这里仍保证异常早退不会留下“有逻辑调用、无物理 batch”的半截采集。 + AtomicPollBoundary(trace, result); + PA_GM TraceCoreState &core = *trace.core; + if (trace.atomic_counter_overflow || result.atomic_trace_calls > UINT32_MAX || + trace.poll_calls > UINT32_MAX || trace.poll_batch_records > UINT32_MAX) { + if (core.dropped != UINT32_MAX) core.dropped = core.dropped + 1; + } + core.atomic_calls = static_cast(result.atomic_trace_calls); + core.poll_calls = static_cast(trace.poll_calls); + core.poll_batch_records = static_cast(trace.poll_batch_records); + const uint32_t count = core.count < trace.capacity ? core.count : trace.capacity; + // A5 侧记录经普通 GM cache 写入，kernel 结束前必须把有效 records 与最后的 + // count/dropped cache line 显式 clean，host 的 D2H 才能得到完整且自洽的快照。 + if (count != 0) { + Ops::FlushRegion(trace.records, static_cast(count) * sizeof(TraceRecord)); + } + Ops::FlushRegion(&core, sizeof(core)); +#endif +} + +template +PA_DEVICE uint64_t WriteTraceLap( + TraceContext &trace, PA_GM WorkerState &worker, WorkerResult &result, int32_t task_id, + int32_t function_id, TracePhase trace_phase, ProfilePhase profile_phase, + uint32_t flags = 0, uint32_t auxiliary = 0 +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)worker; + (void)result; + (void)task_id; + (void)function_id; + (void)trace_phase; + (void)profile_phase; + (void)flags; + (void)auxiliary; + return 0; +#else + // lap 记录区间 [上一次 Reset/WriteTraceLap, 当前时刻]，写完立即推进起点。 + // 显式 WriteTrace span 不会修改该起点，这正是生产泳道中阶段可重叠的原因。 + const uint64_t end_cycle = Ops::Now(); + // 真实 FDWIC 在 TRACE_LAP 取到结束时间后先关闭 PollBatch，再写 lap。 + // 复用同一个 end_cycle，避免额外 SYS_CNT 造成可见缝隙。 + AtomicPollBoundaryAt(trace, end_cycle); + WriteTrace( + trace, result, task_id, function_id, trace_phase, profile_phase, worker.swimlane_last_cycle, end_cycle, + flags, auxiliary + ); + worker.swimlane_last_cycle = end_cycle; + return end_cycle; +#endif +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_TRACE_H diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/common/winner_workload.h b/tests/atomic_probe/lazy_lamda_sample/A_original/common/winner_workload.h new file mode 100644 index 0000000000..a3995fb50f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/common/winner_workload.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_WINNER_WORKLOAD_H +#define PA_SCHEDULER_COMMON_WINNER_WORKLOAD_H + +#include "pa_model.h" + +#include +#include + +namespace pa_scheduler::winner_workload { + +// 三种 standalone 后端共享完全相同的 GM/host 布局。128x128 float 是 CCEC +// 已在 A5 上验证过的基本形状；后端可以采用不同指令接口，但不能改变输入、 +// 输出 tile 编址或 host 数值校验口径。 +constexpr uint32_t kTileRows = 128; +constexpr uint32_t kTileCols = 128; +constexpr size_t kTileElements = static_cast(kTileRows) * kTileCols; +constexpr size_t kTileBytes = kTileElements * sizeof(float); +constexpr uint32_t kSharedInputTiles = 2; +constexpr uint32_t kOutputTilesPerWorker = 2; +constexpr uint32_t kOutputTiles = kWorkers * kOutputTilesPerWorker; +constexpr size_t kWorkspaceTiles = kSharedInputTiles + kOutputTiles; +constexpr size_t kWorkspaceBytes = kWorkspaceTiles * kTileBytes; +constexpr float kInputAValue = 2.0F; +constexpr float kInputBValue = 3.0F; +constexpr float kExpectedAicValue = 768.0F; +constexpr float kExpectedSfValue = 5.0F; +constexpr float kExpectedUpValue = 6.0F; +constexpr float kOutputSentinel = -12345.0F; + +// 256 batch 下，即使同一 AIC 极端地拿到全部 QK/PV，128 次完整 Cube +// 迭代的 CCEC 实测 busy 上界仍低于 32-bit PMU 的 25% 门槛。其他后端也沿用 +// 此参数边界，避免相同命令在不同实现上产生不同含义。 +constexpr uint32_t kMaxRealComputeCount = 128; + +// 1 次用于最小正确性取证。默认次数来自 CCEC 的三个独立 b256 A5 进程； +// AscendC 必须重新标定后才能宣称达到同样时长，不能仅因共享默认参数便沿用 +// CCEC 的性能结论。UP 的一次完整 128x128 流水是当前正整数下限。 +constexpr WorkloadCounts kRealComputeSmokeCounts{1, 1, 1, 1}; +constexpr WorkloadCounts kDefaultRealComputeCounts{6, 28, 4, 1}; + +static_assert(kTileBytes == 65536, "real-compute tile must occupy 64 KiB"); +static_assert(kWorkspaceBytes == 12713984, "real-compute workspace size changed unexpectedly"); + +} // namespace pa_scheduler::winner_workload + +#endif // PA_SCHEDULER_COMMON_WINNER_WORKLOAD_H diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/common/winner_workload_host.h b/tests/atomic_probe/lazy_lamda_sample/A_original/common/winner_workload_host.h new file mode 100644 index 0000000000..07972cc96b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/common/winner_workload_host.h @@ -0,0 +1,361 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_WINNER_WORKLOAD_HOST_H +#define PA_SCHEDULER_COMMON_WINNER_WORKLOAD_HOST_H + +#include "host_support.h" +#include "winner_workload.h" + +#include +#include +#include + +namespace pa_scheduler::host { + +// 常量模式用于稳定的性能负载；布局诊断模式使用带权对角 A 和非对称稠密 B， +// 专门暴露 B 转置、ND/NZ stride 与输出重排错误。该选择只影响计时窗外的 +// host 输入生成和结果校验，不进入 SchedulerState，也不增加 device 热路径分支。 +enum class RealComputePattern : uint32_t { + Constant = 0, + LayoutDiagnostic = 1, +}; + +// winner 负载参数在通用 benchmark parser 前单独剥离，CCEC 可在其后继续剥离 +// PMU 参数；AscendC/CPU 则直接把剩余 argv 交给 ParseOptions。这样不把后端 +// 私有功能塞入公共 PA 参数结构，也不会复制三套互斥规则。 +struct WinnerWorkloadOptions { + WinnerWorkloadMode mode = WinnerWorkloadMode::RealCompute; + WorkloadCounts repeats = winner_workload::kDefaultRealComputeCounts; + RealComputePattern pattern = RealComputePattern::Constant; + bool counts_explicit = false; + bool pattern_explicit = false; + bool nop_override_explicit = false; +}; + +inline const char *WinnerWorkloadModeName(WinnerWorkloadMode mode) { + switch (mode) { + case WinnerWorkloadMode::ScalarNop: + return "scalar-nop"; + case WinnerWorkloadMode::RealCompute: + return "real-compute"; + } + return "invalid"; +} + +inline const char *RealComputePatternName(RealComputePattern pattern) { + switch (pattern) { + case RealComputePattern::Constant: + return "constant"; + case RealComputePattern::LayoutDiagnostic: + return "layout-diagnostic"; + } + return "invalid"; +} + +inline bool ParseWorkloadCounts(const char *raw, WorkloadCounts *counts) { + unsigned int qk = 0; + unsigned int sf = 0; + unsigned int pv = 0; + unsigned int up = 0; + char tail = '\0'; + if (std::sscanf(raw, "%u,%u,%u,%u%c", &qk, &sf, &pv, &up, &tail) != 4) return false; + const uint32_t maximum = winner_workload::kMaxRealComputeCount; + if (qk == 0 || sf == 0 || pv == 0 || up == 0 || + qk > maximum || sf > maximum || pv > maximum || up > maximum) { + return false; + } + *counts = WorkloadCounts{qk, sf, pv, up}; + return true; +} + +inline bool ParseWinnerWorkloadOptions( + int argc, char **argv, WinnerWorkloadOptions *workload, std::vector *remaining_argv +) { + bool mode_seen = false; + bool count_seen = false; + bool pattern_seen = false; + remaining_argv->clear(); + remaining_argv->push_back(argv[0]); + for (int index = 1; index < argc; ++index) { + const std::string argument = argv[index]; + if (argument == "--nop-count" || argument == "--nop-counts") { + workload->nop_override_explicit = true; + } + if (argument != "--winner-workload" && argument != "--real-compute-count" && + argument != "--real-compute-counts" && argument != "--real-compute-pattern") { + remaining_argv->push_back(argv[index]); + continue; + } + if (index + 1 >= argc) { + std::fprintf(stderr, "Missing value after %s\n", argument.c_str()); + return false; + } + const char *value = argv[++index]; + if (argument == "--winner-workload") { + if (mode_seen) { + std::fprintf(stderr, "Specify --winner-workload only once.\n"); + return false; + } + const std::string name = value; + if (name == "scalar-nop") { + workload->mode = WinnerWorkloadMode::ScalarNop; + } else if (name == "real-compute") { + workload->mode = WinnerWorkloadMode::RealCompute; + } else { + std::fprintf( + stderr, + "Invalid --winner-workload value: %s (expected scalar-nop|real-compute)\n", + value + ); + return false; + } + mode_seen = true; + continue; + } + if (argument == "--real-compute-pattern") { + if (pattern_seen) { + std::fprintf(stderr, "Specify --real-compute-pattern only once.\n"); + return false; + } + const std::string name = value; + if (name == "constant") { + workload->pattern = RealComputePattern::Constant; + } else if (name == "layout-diagnostic") { + workload->pattern = RealComputePattern::LayoutDiagnostic; + } else { + std::fprintf( + stderr, + "Invalid --real-compute-pattern value: %s " + "(expected constant|layout-diagnostic)\n", + value + ); + return false; + } + pattern_seen = true; + workload->pattern_explicit = true; + continue; + } + if (count_seen) { + std::fprintf(stderr, "Specify only one real-compute count override.\n"); + return false; + } + if (argument == "--real-compute-count") { + uint32_t count = 0; + if (!ParseUint(value, 1, winner_workload::kMaxRealComputeCount, &count)) { + std::fprintf(stderr, "Invalid --real-compute-count value: %s\n", value); + return false; + } + workload->repeats = WorkloadCounts{count, count, count, count}; + } else if (!ParseWorkloadCounts(value, &workload->repeats)) { + std::fprintf(stderr, "Invalid --real-compute-counts value: %s\n", value); + return false; + } + count_seen = true; + workload->counts_explicit = true; + } + // 无参数运行以真实 Cube/Vector 为默认。旧命令若显式给出 NOP 次数但没有 + // 指定 workload mode，则把 NOP override 本身视为选择 scalar-nop；这样 + // 既不让 --nop-count 悄悄失效，也不破坏既有标定脚本。显式指定 + // real-compute 再叠加 NOP 仍由下方互斥校验拒绝。 + if (!mode_seen && workload->nop_override_explicit) { + workload->mode = WinnerWorkloadMode::ScalarNop; + } + return true; +} + +inline bool ValidateWinnerWorkloadOptions(const WinnerWorkloadOptions &workload) { + if (workload.mode == WinnerWorkloadMode::RealCompute) { + if (workload.pattern != RealComputePattern::Constant && + workload.pattern != RealComputePattern::LayoutDiagnostic) { + std::fprintf(stderr, "Invalid real-compute input pattern.\n"); + return false; + } + if (workload.nop_override_explicit) { + std::fprintf( + stderr, + "--winner-workload real-compute cannot be combined with --nop-count or --nop-counts.\n" + ); + return false; + } + return true; + } + if (workload.counts_explicit || workload.pattern_explicit) { + std::fprintf( + stderr, + "--real-compute-count(s)/pattern requires real-compute workload mode.\n" + ); + return false; + } + return true; +} + +inline void ConfigureWinnerWorkload( + SchedulerState *state, const WinnerWorkloadOptions &workload, const void *workspace_device +) { + state->winner_workload.mode = static_cast(workload.mode); + state->winner_workload.version = kWinnerWorkloadConfigVersion; + state->winner_workload.repeats = workload.repeats; + state->winner_workload.workspace_base = reinterpret_cast(workspace_device); + state->winner_workload.workspace_bytes = + workload.mode == WinnerWorkloadMode::RealCompute ? winner_workload::kWorkspaceBytes : 0; +} + +inline float LayoutDiagnosticInputA(uint32_t row, uint32_t column) { + return row == column ? static_cast(row + 1U) : 0.0F; +} + +inline float LayoutDiagnosticInputB(uint32_t row, uint32_t column) { + const uint32_t value = + (131U * row + 17U * column + 7U * row * column) % 251U; + return static_cast(value + 1U); +} + +inline void InitializeWinnerWorkloadBuffers( + const WinnerWorkloadOptions &workload, std::vector *workspace_image, + std::vector *workspace_outputs +) { + using namespace winner_workload; + workspace_image->assign(kWorkspaceTiles * kTileElements, kOutputSentinel); + if (workload.pattern == RealComputePattern::Constant) { + std::fill_n(workspace_image->begin(), kTileElements, kInputAValue); + std::fill_n(workspace_image->begin() + kTileElements, kTileElements, kInputBValue); + } else { + for (uint32_t row = 0; row < kTileRows; ++row) { + for (uint32_t column = 0; column < kTileCols; ++column) { + const size_t element = static_cast(row) * kTileCols + column; + (*workspace_image)[element] = LayoutDiagnosticInputA(row, column); + (*workspace_image)[kTileElements + element] = + LayoutDiagnosticInputB(row, column); + } + } + } + workspace_outputs->resize(static_cast(kOutputTiles) * kTileElements); +} + +inline const char *TaskKindName(TaskKind kind) { + switch (kind) { + case TaskKind::Qk: + return "QK"; + case TaskKind::Sf: + return "SF"; + case TaskKind::Pv: + return "PV"; + case TaskKind::Up: + return "UP"; + default: + return "invalid"; + } +} + +inline float ExpectedRealComputeValue( + RealComputePattern pattern, TaskKind kind, uint32_t row, uint32_t column +) { + using namespace winner_workload; + if (pattern == RealComputePattern::Constant) { + return kind == TaskKind::Qk || kind == TaskKind::Pv + ? kExpectedAicValue + : (kind == TaskKind::Sf ? kExpectedSfValue : kExpectedUpValue); + } + const float input_a = LayoutDiagnosticInputA(row, column); + const float input_b = LayoutDiagnosticInputB(row, column); + if (kind == TaskKind::Qk || kind == TaskKind::Pv) { + // A 是带权对角矩阵，因此 A*B 的 (row,column) 只有一个非零乘积。 + return static_cast(row + 1U) * input_b; + } + return kind == TaskKind::Sf ? input_a + input_b : input_a * input_b; +} + +inline bool ValidateRealComputeOutputs( + const SchedulerState &state, const WinnerWorkloadOptions &workload, + const std::vector &outputs, uint32_t run +) { + using namespace winner_workload; + const size_t expected_elements = static_cast(kOutputTiles) * kTileElements; + if (outputs.size() != expected_elements) { + std::fprintf( + stderr, "[ASSERT] real-compute output buffer size matches workspace layout FAIL\n" + ); + return false; + } + + uint32_t active_tiles = 0; + uint32_t inactive_tiles = 0; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const WorkerResult &result = state.results[worker]; + const bool aic = result.role == static_cast(CoreRole::Aic); + const TaskKind kinds[2] = { + aic ? TaskKind::Qk : TaskKind::Sf, + aic ? TaskKind::Pv : TaskKind::Up, + }; + for (uint32_t kind_slot = 0; kind_slot < 2; ++kind_slot) { + const TaskKind kind = kinds[kind_slot]; + const uint32_t kernel_index = static_cast(kind) - 1; + const bool active = result.kernel_counts[kernel_index] != 0; + const size_t tile_index = + static_cast(worker) * kOutputTilesPerWorker + kind_slot; + const size_t begin = tile_index * kTileElements; + for (size_t element = 0; element < kTileElements; ++element) { + const uint32_t row = static_cast(element / kTileCols); + const uint32_t column = static_cast(element % kTileCols); + const float expected = active + ? ExpectedRealComputeValue(workload.pattern, kind, row, column) + : kOutputSentinel; + if (outputs[begin + element] == expected) continue; + std::fprintf( + stderr, + "[REAL-COMPUTE-FAIL] run=%u worker=%u kind=%s element=%zu " + "expected=%.1f actual=%.9g\n", + run, worker, TaskKindName(kind), element, expected, + static_cast(outputs[begin + element]) + ); + std::fprintf( + stderr, + "[ASSERT] real-compute output tiles match role-specific engine results FAIL\n" + ); + return false; + } + active_tiles += active ? 1U : 0U; + inactive_tiles += active ? 0U : 1U; + } + } + const bool passed = active_tiles != 0 && active_tiles + inactive_tiles == kOutputTiles; + std::printf( + "[ASSERT] %-48s %s (active_tiles=%u inactive_sentinel_tiles=%u)\n", + "real-compute output tiles match role-specific engine results", + passed ? "PASS" : "FAIL", active_tiles, inactive_tiles + ); + return passed; +} + +inline void PrintWinnerWorkloadConfig( + const WinnerWorkloadOptions &workload, const NopCounts &nops +) { + if (workload.mode == WinnerWorkloadMode::RealCompute) { + std::printf( + "[WINNER-WORKLOAD] mode=real-compute pattern=%s counts=%u,%u,%u,%u " + "unit=complete_128x128_engine_pipeline_iteration workspace_bytes=%zu\n", + RealComputePatternName(workload.pattern), + workload.repeats.qk, workload.repeats.sf, workload.repeats.pv, + workload.repeats.up, winner_workload::kWorkspaceBytes + ); + return; + } + std::printf( + "[WINNER-WORKLOAD] mode=scalar-nop counts=%u,%u,%u,%u " + "unit=scalar_nop_instruction workspace_bytes=0\n", + nops.qk, nops.sf, nops.pv, nops.up + ); +} + +} // namespace pa_scheduler::host + +#endif // PA_SCHEDULER_COMMON_WINNER_WORKLOAD_HOST_H diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassemble.py b/tests/atomic_probe/lazy_lamda_sample/A_original/disassemble.py new file mode 100755 index 0000000000..16eac59111 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/disassemble.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python3 +"""Disassemble every final-linked STT_FUNC with the verified A5 PEM decoder.""" + +from __future__ import annotations + +import argparse +import ctypes +import gzip +import hashlib +import io +import re +import struct +from dataclasses import dataclass +from pathlib import Path + + +EXPECTED_DECODER_SHA256 = "29835d2439d6dd464d34a212ad4bbd5c29af6a38465da09a6c273401d9a96dcb" +INSTRUCTION_RE = re.compile(r"^\s*([0-9a-fA-F]+):\s+((?:[0-9a-fA-F]{8})+)\s+(.*?)\s*$") +BAD_MNEMONICS = {"UNDEF", "UNKNOWN", "INVALID", "ILLEGAL", "ERROR", " str: + return hashlib.sha256(data).hexdigest() + + +def c_string(table: bytes, offset: int) -> str: + end = table.find(b"\0", offset) + if end < 0: + raise ValueError("unterminated ELF string") + return table[offset:end].decode("utf-8") + + +@dataclass(frozen=True) +class Section: + index: int + name: str + section_type: int + address: int + offset: int + size: int + link: int + entry_size: int + + +@dataclass(frozen=True) +class Function: + name: str + value: int + size: int + binding: str + + +class Elf64: + def __init__(self, path: Path): + self.path = path + self.data = path.read_bytes() + header_format = "<16sHHIQQQIHHHHHH" + if len(self.data) < struct.calcsize(header_format): + raise ValueError(f"truncated ELF: {path}") + header = struct.unpack_from(header_format, self.data, 0) + ident = header[0] + if ident[:4] != b"\x7fELF" or ident[4] != 2 or ident[5] != 1: + raise ValueError(f"expected ELF64 little-endian image: {path}") + section_header_offset = header[6] + section_header_size = header[11] + section_count = header[12] + section_name_index = header[13] + if section_header_size != 64 or section_count == 0: + raise ValueError(f"unsupported section table: {path}") + + raw_sections = [] + section_format = " bytes: + return self.data[section.offset : section.offset + section.size] + + def functions(self) -> list[Function]: + symtab = next(section for section in self.sections if section.section_type == 2) + if symtab.entry_size != 24 or symtab.size % symtab.entry_size != 0: + raise ValueError(f"unsupported symbol table: {self.path}") + strings = self.section_bytes(self.sections[symtab.link]) + functions = [] + for offset in range(symtab.offset, symtab.offset + symtab.size, symtab.entry_size): + name_offset, info, _, section_index, value, size = struct.unpack_from( + "> 4, str(info >> 4)) + functions.append(Function(c_string(strings, name_offset), value, size, binding)) + functions.sort(key=lambda function: (function.value, function.name)) + if not functions: + raise ValueError(f"no final-linked .text functions: {self.path}") + return functions + + +class PemDecoder: + def __init__(self, path: Path): + self.path = path + decoder_bytes = path.read_bytes() + self.digest = sha256(decoder_bytes) + if self.digest != EXPECTED_DECODER_SHA256: + raise ValueError( + f"unsupported PEM decoder SHA256 {self.digest}; expected {EXPECTED_DECODER_SHA256}" + ) + library = ctypes.CDLL(str(path)) + self.dump = library.pem_turbo_objdump + self.dump.argtypes = ( + ctypes.POINTER(ctypes.c_uint32), + ctypes.c_size_t, + ctypes.c_void_p, + ctypes.c_size_t, + ) + self.dump.restype = ctypes.c_void_p + self.libc = ctypes.CDLL(None) + self.libc.free.argtypes = (ctypes.c_void_p,) + + def decode(self, body: bytes, rvec: bool) -> list[tuple[int, str, str]]: + if len(body) % 4 != 0: + raise ValueError("function body is not an integral number of instruction words") + word_count = len(body) // 4 + words = (ctypes.c_uint32 * word_count).from_buffer_copy(body) + range_pointer = None + range_count = 0 + if rvec: + ranges_type = (ctypes.c_uint64 * 2) * 1 + ranges = ranges_type() + ranges[0][0] = 0 + ranges[0][1] = len(body) + range_pointer = ctypes.cast(ranges, ctypes.c_void_p) + range_count = 1 + result = self.dump(words, word_count, range_pointer, range_count) + if not result: + raise RuntimeError("pem_turbo_objdump returned null") + try: + decoded = ctypes.string_at(result).decode("utf-8") + finally: + self.libc.free(result) + + instructions = [] + for line in decoded.splitlines(): + match = INSTRUCTION_RE.fullmatch(line) + if match is None: + if line.strip(): + raise ValueError(f"unexpected decoder output: {line!r}") + continue + relative = int(match.group(1), 16) + machine_word = match.group(2).lower() + mnemonic = match.group(3).strip() + instructions.append((relative, machine_word, mnemonic)) + expected_relative = 0 + for index, (relative, machine_word, mnemonic) in enumerate(instructions): + encoded_bytes = len(machine_word) // 2 + if encoded_bytes % 4 != 0: + raise ValueError( + f"decoder returned a non-word-sized encoding at row {index}: {machine_word}" + ) + expected_word = ( + f"{int.from_bytes(body[expected_relative:expected_relative + encoded_bytes], 'little'):0{len(machine_word)}x}" + ) + if relative != expected_relative or machine_word != expected_word: + raise ValueError( + f"decoder coverage mismatch at word {index}: {relative:x}/{machine_word} " + f"!= {expected_relative:x}/{expected_word}" + ) + first_token = mnemonic.upper().split(maxsplit=1)[0] if mnemonic else "" + if first_token in BAD_MNEMONICS or "NOT AVAILABLE" in mnemonic.upper(): + raise ValueError(f"invalid decoded instruction at +0x{relative:x}: {mnemonic}") + expected_relative += encoded_bytes + if expected_relative != len(body): + raise ValueError( + f"decoder covered {expected_relative} bytes of a {len(body)}-byte function" + ) + return instructions + + +def safe_filename(index: int, name: str) -> str: + safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", name).strip("._") or "function" + suffix = hashlib.sha256(name.encode("utf-8")).hexdigest()[:8] + return f"{index:02d}_{safe[:72]}_{suffix}.asm.gz" + + +def write_gzip(path: Path, text: str) -> None: + with path.open("wb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed: + compressed.write(text.encode("utf-8")) + + +def disassemble_variant( + variant: str, elf_path: Path, output_dir: Path, decoder: PemDecoder +) -> tuple[list[str], list[str], dict[str, object]]: + elf = Elf64(elf_path) + text = elf.section_bytes(elf.text) + functions = elf.functions() + variant_dir = output_dir + variant_dir.mkdir(parents=True) + elf_digest = sha256(elf.data) + text_digest = sha256(text) + manifest_rows = [] + gap_rows = [] + cursor = elf.text.address + total_function_bytes = 0 + + for index, function in enumerate(functions): + if function.value < cursor: + raise ValueError(f"overlapping final function symbols in {elf_path}: {function.name}") + if function.value > cursor: + gap_start = cursor + gap_end = function.value + start = gap_start - elf.text.address + gap = text[start : start + gap_end - gap_start] + gap_rows.append( + f"0x{gap_start:x}\t0x{gap_end:x}\t{len(gap)}\t{sha256(gap)}" + ) + body_offset = function.value - elf.text.address + body = text[body_offset : body_offset + function.size] + if len(body) != function.size: + raise ValueError(f"function outside .text: {function.name}") + rvec = ".vector.thread" in function.name + try: + instructions = decoder.decode(body, rvec) + except Exception as error: + raise ValueError(f"{variant}:{function.name}: {error}") from error + body_digest = sha256(body) + filename = safe_filename(index, function.name) + header = [ + "# schema=pa_final_linked_disassembly/v1", + f"# variant={variant}", + f"# final_elf={elf_path.name}", + f"# final_elf_sha256={elf_digest}", + f"# final_text_address=0x{elf.text.address:x}", + f"# final_text_size={elf.text.size}", + f"# final_text_sha256={text_digest}", + f"# symbol={function.name}", + f"# binding={function.binding}", + f"# final_pc=0x{function.value:x}", + f"# size={function.size}", + f"# instruction_count={len(instructions)}", + f"# encoded_word_count={function.size // 4}", + f"# body_sha256={body_digest}", + "# decoder=$ASCEND_HOME_PATH/x86_64-linux/simulator/dav_3510/lib/libpem_davinci.so", + f"# decoder_sha256={decoder.digest}", + f"# decoder_mode={'rvec' if rvec else 'scalar'}", + "# columns=final_pc function_relative_offset machine_word instruction", + "", + ] + body_lines = [ + f"0x{function.value + relative:016x} (+0x{relative:08x}) {machine_word} {mnemonic}" + for relative, machine_word, mnemonic in instructions + ] + write_gzip(variant_dir / filename, "\n".join(header + body_lines) + "\n") + last_mnemonic = instructions[-1][2].split(maxsplit=1)[0] + manifest_rows.append( + "\t".join( + ( + variant, + filename, + function.binding, + function.name, + f"0x{function.value:x}", + str(function.size), + str(len(instructions)), + "rvec" if rvec else "scalar", + body_digest, + last_mnemonic, + ) + ) + ) + cursor = function.value + function.size + total_function_bytes += function.size + + text_end = elf.text.address + elf.text.size + if cursor < text_end: + start = cursor - elf.text.address + gap = text[start:] + gap_rows.append(f"0x{cursor:x}\t0x{text_end:x}\t{len(gap)}\t{sha256(gap)}") + elif cursor > text_end: + raise ValueError(f"function extends past .text: {elf_path}") + + metadata = { + "variant": variant, + "elf_sha256": elf_digest, + "text_address": elf.text.address, + "text_size": elf.text.size, + "text_sha256": text_digest, + "function_count": len(functions), + "function_bytes": total_function_bytes, + "function_coverage_percent": total_function_bytes / elf.text.size * 100.0, + "gap_count": len(gap_rows), + } + return manifest_rows, gap_rows, metadata + + +def main() -> None: + parser = argparse.ArgumentParser() + script_dir = Path(__file__).resolve().parent + fixed_variant = (script_dir / "VARIANT").read_text(encoding="utf-8").strip() + parser.add_argument("--variant", default=fixed_variant) + parser.add_argument( + "--elf", type=Path, + default=script_dir / "artifacts" / "measured" / "pa_scheduler_kernel.o" + ) + parser.add_argument("--output", type=Path, default=script_dir / "disassembly" / "raw") + default_decoder = None + import os + + if os.environ.get("ASCEND_HOME_PATH"): + default_decoder = ( + Path(os.environ["ASCEND_HOME_PATH"]) + / "x86_64-linux/simulator/dav_3510/lib/libpem_davinci.so" + ) + parser.add_argument("--decoder", type=Path, default=default_decoder, required=default_decoder is None) + args = parser.parse_args() + elf_path = args.elf.resolve() + output_dir = args.output.resolve() + decoder_path = args.decoder.resolve() + if output_dir.exists(): + raise SystemExit(f"Output directory already exists; choose a new path: {output_dir}") + decoder = PemDecoder(decoder_path) + + if not elf_path.is_file(): + raise SystemExit(f"Missing published final ELF: {elf_path}") + manifest_rows, local_gaps, item = disassemble_variant( + args.variant, elf_path, output_dir, decoder + ) + gap_rows = [f"{args.variant}\t{row}" for row in local_gaps] + + manifest_header = ( + "variant\tfile\tbinding\tsymbol\tfinal_pc\tsize\tinstructions\tdecoder_mode\t" + "body_sha256\tlast_mnemonic" + ) + (output_dir / "manifest.tsv").write_text( + manifest_header + "\n" + "\n".join(manifest_rows) + "\n", encoding="utf-8" + ) + (output_dir / "gaps.tsv").write_text( + "variant\tstart_pc\tend_pc\tsize\tsha256\n" + "\n".join(gap_rows) + "\n", + encoding="utf-8", + ) + readme = [ + "# Final-linked A5 disassembly", + "", + "Every non-empty final `.text` `STT_FUNC` symbol was extracted from the published final ELF and decoded with the verified `dav_3510` PEM decoder.", + "RVec `.vector.thread` symbols are passed through an explicit full-body RVec range; all other symbols use scalar decoding.", + "Each compressed file records final PC, function-relative offset, machine word, mnemonic, ELF/body hashes and decoder identity.", + "Function gaps are alignment/padding outside symbol bodies and are hashed in `gaps.tsv`; they are not presented as instructions.", + "", + f"Decoder SHA256: `{decoder.digest}`", + "", + "| Variant | Final .text | Function symbols | Function bytes | Coverage | Final ELF SHA256 |", + "| --- | ---: | ---: | ---: | ---: | --- |", + ] + readme.append( + f"| {item['variant']} | {item['text_size']} B | {item['function_count']} | " + f"{item['function_bytes']} B | {item['function_coverage_percent']:.3f}% | " + f"`{item['elf_sha256']}` |" + ) + readme.extend( + [ + "", + "Inspect with `zless FILE.asm.gz`; the sibling `annotated/` directory adds DWARF-mapped local source and comments.", + ] + ) + (output_dir / "README.md").write_text("\n".join(readme) + "\n", encoding="utf-8") + + published = [] + for path in sorted(item for item in output_dir.rglob("*") if item.is_file()): + if path.name == "published.sha256": + continue + published.append(f"{sha256(path.read_bytes())} {path.relative_to(output_dir)}") + (output_dir / "published.sha256").write_text("\n".join(published) + "\n", encoding="utf-8") + print("\n".join(readme)) + + +if __name__ == "__main__": + main() diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/00_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/00_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz new file mode 100644 index 0000000000..f6b95233d6 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/00_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/01_pa_execute_real_winner_workload_aic_729699de.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/01_pa_execute_real_winner_workload_aic_729699de.source.asm.gz new file mode 100644 index 0000000000..b2749b2ba2 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/01_pa_execute_real_winner_workload_aic_729699de.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/02_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/02_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz new file mode 100644 index 0000000000..4b137ff8cb Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/02_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/03_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/03_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz new file mode 100644 index 0000000000..c911f51458 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/03_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/04_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/04_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz new file mode 100644 index 0000000000..5ce12e7189 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/04_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/05_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/05_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz new file mode 100644 index 0000000000..369a57ced4 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/05_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/06_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/06_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz new file mode 100644 index 0000000000..a2d09c6b95 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/06_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz new file mode 100644 index 0000000000..076b92b0b7 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/08_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/08_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz new file mode 100644 index 0000000000..018c9e6a5f Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/08_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/09_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/09_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz new file mode 100644 index 0000000000..3476e85235 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/09_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/10_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/10_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz new file mode 100644 index 0000000000..0e1029aec4 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/annotated/10_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/key_flow/aic_original_submit.source.asm b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/key_flow/aic_original_submit.source.asm new file mode 100644 index 0000000000..2fdbc03758 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/key_flow/aic_original_submit.source.asm @@ -0,0 +1,2038 @@ +# schema=pa_final_linked_disassembly/v1 +# variant=original +# final_elf=pa_scheduler_kernel.o +# final_elf_sha256=76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc +# final_text_address=0x0 +# final_text_size=780344 +# final_text_sha256=018ae7dc29ce78249b2e3bff84b0faa2e671e2448c9316719fb48bc59139d2fc +# symbol=pa_scheduler_0_mix_aic +# binding=GLOBAL +# final_pc=0x250 +# size=387044 +# instruction_count=96761 +# encoded_word_count=96761 +# body_sha256=d7337df3d1786f2236031aaa4d252ca82696e3a563e11ce56d9b98495bbf1aea +# decoder=$ASCEND_HOME_PATH/x86_64-linux/simulator/dav_3510/lib/libpem_davinci.so +# decoder_sha256=29835d2439d6dd464d34a212ad4bbd5c29af6a38465da09a6c273401d9a96dcb +# decoder_mode=scalar +# columns=final_pc function_relative_offset machine_word instruction +# annotation_schema=pa_source_annotated_disassembly/v1 +# annotation_rule=DWARF supplies only file:line; SOURCE rows are copied from local source files +# annotation_warning=comments have source context only and do not own an exact machine address +# annotation_instruction_slice=87055:87776 +# +# [DWARF] common/pa_trace.h:549 +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# > 549 | core.dropped = core.dropped + 1; +0x000000000005528c (+0x0005503c) 1c801004 LD_XD_XN_IMM.B32 X0, X1, #4 +0x0000000000055290 (+0x00055040) 08000001 ADD_IMM.S64 X0, X0, #1 +0x0000000000055294 (+0x00055044) 03801004 ST_XD_XN_IMM.B32 X0, X1, #4 +0x0000000000055298 (+0x00055048) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000005529c (+0x0005504c) 0700ddbf MOV_XD_IMM X0, #56767 +0x00000000000552a0 (+0x00055050) 0741ffff MOVK X0, #65535, #1 +0x00000000000552a4 (+0x00055054) 0781ffff MOVK X0, #65535, #2 +0x00000000000552a8 (+0x00055058) 40020000 JUMP X0, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000552ac (+0x0005505c) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x00000000000552b0 (+0x00055060) 07000000 MOV_XD_IMM X0, #0 +0x00000000000552b4 (+0x00055064) 03d3ea40 ST_XD_XN_IMM.B64 X9, X30, #2624 +0x00000000000552b8 (+0x00055068) 1ce5eb48 LD_XD_XN_IMM.B64 X18, X30, #2888 +0x00000000000552bc (+0x0005506c) 03c1e9e8 ST_XD_XN_IMM.B64 X0, X30, #2536 +0x00000000000552c0 (+0x00055070) 07000000 MOV_XD_IMM X0, #0 +0x00000000000552c4 (+0x00055074) 1cebeb40 LD_XD_XN_IMM.B64 X21, X30, #2880 +0x00000000000552c8 (+0x00055078) 03c1ea90 ST_XD_XN_IMM.B64 X0, X30, #2704 +0x00000000000552cc (+0x0005507c) 07000000 MOV_XD_IMM X0, #0 +0x00000000000552d0 (+0x00055080) 1ce7eb00 LD_XD_XN_IMM.B64 X19, X30, #2816 +0x00000000000552d4 (+0x00055084) 03c1e9c8 ST_XD_XN_IMM.B64 X0, X30, #2504 +0x00000000000552d8 (+0x00055088) 07000000 MOV_XD_IMM X0, #0 +0x00000000000552dc (+0x0005508c) 1cf9eb70 LD_XD_XN_IMM.B64 X28, X30, #2928 +0x00000000000552e0 (+0x00055090) 03c1e988 ST_XD_XN_IMM.B64 X0, X30, #2440 +0x00000000000552e4 (+0x00055094) 07000000 MOV_XD_IMM X0, #0 +0x00000000000552e8 (+0x00055098) 03c1e980 ST_XD_XN_IMM.B64 X0, X30, #2432 +0x00000000000552ec (+0x0005509c) 07000000 MOV_XD_IMM X0, #0 +0x00000000000552f0 (+0x000550a0) 03c1e9c0 ST_XD_XN_IMM.B64 X0, X30, #2496 +0x00000000000552f4 (+0x000550a4) 07000000 MOV_XD_IMM X0, #0 +0x00000000000552f8 (+0x000550a8) 03c1e9b8 ST_XD_XN_IMM.B64 X0, X30, #2488 +0x00000000000552fc (+0x000550ac) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055300 (+0x000550b0) 03c1e9d0 ST_XD_XN_IMM.B64 X0, X30, #2512 +0x0000000000055304 (+0x000550b4) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055308 (+0x000550b8) 03c1e9d8 ST_XD_XN_IMM.B64 X0, X30, #2520 +0x000000000005530c (+0x000550bc) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055310 (+0x000550c0) 03c1e9e0 ST_XD_XN_IMM.B64 X0, X30, #2528 +0x0000000000055314 (+0x000550c4) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055318 (+0x000550c8) 03c1e9f0 ST_XD_XN_IMM.B64 X0, X30, #2544 +0x000000000005531c (+0x000550cc) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055320 (+0x000550d0) 03c1ea68 ST_XD_XN_IMM.B64 X0, X30, #2664 +0x0000000000055324 (+0x000550d4) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055328 (+0x000550d8) 03c1ea58 ST_XD_XN_IMM.B64 X0, X30, #2648 +0x000000000005532c (+0x000550dc) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055330 (+0x000550e0) 03c1ea20 ST_XD_XN_IMM.B64 X0, X30, #2592 +0x0000000000055334 (+0x000550e4) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055338 (+0x000550e8) 03c1e9f8 ST_XD_XN_IMM.B64 X0, X30, #2552 +0x000000000005533c (+0x000550ec) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055340 (+0x000550f0) 03c1e990 ST_XD_XN_IMM.B64 X0, X30, #2448 +0x0000000000055344 (+0x000550f4) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055348 (+0x000550f8) 03c1ea48 ST_XD_XN_IMM.B64 X0, X30, #2632 +0x000000000005534c (+0x000550fc) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055350 (+0x00055100) 03c1ea60 ST_XD_XN_IMM.B64 X0, X30, #2656 +0x0000000000055354 (+0x00055104) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055358 (+0x00055108) 03c1ea50 ST_XD_XN_IMM.B64 X0, X30, #2640 +0x000000000005535c (+0x0005510c) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055360 (+0x00055110) 03c1ea28 ST_XD_XN_IMM.B64 X0, X30, #2600 +0x0000000000055364 (+0x00055114) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055368 (+0x00055118) 03c1e998 ST_XD_XN_IMM.B64 X0, X30, #2456 +0x000000000005536c (+0x0005511c) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055370 (+0x00055120) 03c1e9a0 ST_XD_XN_IMM.B64 X0, X30, #2464 +0x0000000000055374 (+0x00055124) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055378 (+0x00055128) 03c1e9a8 ST_XD_XN_IMM.B64 X0, X30, #2472 +0x000000000005537c (+0x0005512c) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055380 (+0x00055130) 03c1eae8 ST_XD_XN_IMM.B64 X0, X30, #2792 +0x0000000000055384 (+0x00055134) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055388 (+0x00055138) 03c1eb28 ST_XD_XN_IMM.B64 X0, X30, #2856 +0x000000000005538c (+0x0005513c) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055390 (+0x00055140) 03c1ead0 ST_XD_XN_IMM.B64 X0, X30, #2768 +0x0000000000055394 (+0x00055144) 07000000 MOV_XD_IMM X0, #0 +0x0000000000055398 (+0x00055148) 07320000 MOV_XD_IMM X25, #0 +0x000000000005539c (+0x0005514c) 03c1e9b0 ST_XD_XN_IMM.B64 X0, X30, #2480 +0x00000000000553a0 (+0x00055150) 07340000 MOV_XD_IMM X26, #0 +0x00000000000553a4 (+0x00055154) 070c0000 MOV_XD_IMM X6, #0 +0x00000000000553a8 (+0x00055158) 0216f800 MOV_XD_XN.S64 X11, X15 +0x00000000000553ac (+0x0005515c) 021d7800 MOV_XD_XN.S64 X14, X23 +0x00000000000553b0 (+0x00055160) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x00000000000553b4 (+0x00055164) 070002ea MOV_XD_IMM X0, #746 +0x00000000000553b8 (+0x00055168) 07410000 MOVK X0, #0, #1 +0x00000000000553bc (+0x0005516c) 07810000 MOVK X0, #0, #2 +0x00000000000553c0 (+0x00055170) 40020000 JUMP X0, #0 +0x00000000000553c4 (+0x00055174) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x00000000000553c8 (+0x00055178) 02808880 MOV_XD_SPR.F32 X0, SYS_CNT +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x00000000000553cc (+0x0005517c) 07040001 MOV_XD_IMM X2, #1 +0x00000000000553d0 (+0x00055180) 07060001 MOV_XD_IMM X3, #1 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:45 +# 39 | CCE_INTRINSIC[aicore] FTYPE atomicSub(__gm__ FTYPE *base, FTYPE inc) { \ +# 40 | return __builtin_cce_atom_add_G_##ADD_SUFFIX(base, -inc, \ +# 41 | (uint32_t)L2Cache); \ +# 42 | } +# 43 | +# 44 | __CCE__ATOM_G_CAS_EXCH_INT32(uint32_t, u32, u32) +# > 45 | __CCE__ATOM_G_CAS_EXCH_INT32(int32_t, s32, s32) +0x00000000000553d4 (+0x00055184) 50109c40 ATOM XN, XM, XD, EXCH +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:0 +# [SOURCE unavailable] +0x00000000000553d8 (+0x00055188) 02063080 NEG.S64 X3, X3 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x00000000000553dc (+0x0005518c) 02828880 MOV_XD_SPR.F32 X1, SYS_CNT +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x00000000000553e0 (+0x00055190) 0001918e CMP.S64.EQ X25, X3 +0x00000000000553e4 (+0x00055194) 00da2689 SEL.B64 X13, X2, X13 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x00000000000553e8 (+0x00055198) 1cc5eb40 LD_XD_XN_IMM.B64 X2, X30, #2880 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000553ec (+0x0005519c) 08099001 ADD_IMM.S64 X4, X25, #1 +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x00000000000553f0 (+0x000551a0) 00e63209 SEL.B64 X19, X3, X4 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x00000000000553f4 (+0x000551a4) 02162900 MOV_SPR_XN.S64 CONDITION_FLAG, X2 +0x00000000000553f8 (+0x000551a8) 40200151 JUMPC #337 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000553fc (+0x000551ac) 1cc9eb80 LD_XD_XN_IMM.B64 X4, X30, #2944 +# [DWARF] common/pa_trace.h:548 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# > 548 | if (slot >= trace.capacity) { +0x0000000000055400 (+0x000551b0) 1cc7eb98 LD_XD_XN_IMM.B64 X3, X30, #2968 +# [DWARF] common/pa_trace.h:547 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# > 547 | const uint32_t slot = core.count; +0x0000000000055404 (+0x000551b4) 1c844000 LD_XD_XN_IMM.B32 X2, X4, #0 +# [DWARF] common/pa_trace.h:548 +# > 548 | if (slot >= trace.capacity) { +0x0000000000055408 (+0x000551b8) 004021ae CMP.U64.LT X2, X3 +0x000000000005540c (+0x000551bc) 40200002 JUMPC #2 +0x0000000000055410 (+0x000551c0) 40000146 JUMP #326 +# [DWARF] common/pa_trace.h:552 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x0000000000055414 (+0x000551c4) 1ccbeb90 LD_XD_XN_IMM.B64 X5, X30, #2960 +0x0000000000055418 (+0x000551c8) 02062800 MOV_XD_XN.S64 X3, X2 +0x000000000005541c (+0x000551cc) 02c60206 SHL.B64 X3, #6 +0x0000000000055420 (+0x000551d0) 00065181 ADD.S64 X3, X5, X3 +# [DWARF] common/pa_trace.h:553 +# > 553 | record.start_cycle = start_cycle; +0x0000000000055424 (+0x000551d4) 09c03081 STP_XI_XJ_XN.B64 X0, X1, X3, #0 +0x0000000000055428 (+0x000551d8) 07000001 MOV_XD_IMM X0, #1 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000005542c (+0x000551dc) 02000080 NEG.S64 X0, X0 +0x0000000000055430 (+0x000551e0) 0702000e MOV_XD_IMM X1, #14 +# [DWARF] common/pa_trace.h:555 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x0000000000055434 (+0x000551e4) 09c030a1 STP_XI_XJ_XN.B64 X0, X1, X3, #16 +# [DWARF] common/pa_trace.h:559 +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x0000000000055438 (+0x000551e8) 08003020 ADD_IMM.S64 X0, X3, #32 +0x000000000005543c (+0x000551ec) 099e0781 STP_XI_XJ_XN.B32 X15, X15, X0, #0 +0x0000000000055440 (+0x000551f0) 07000001 MOV_XD_IMM X0, #1 +0x0000000000055444 (+0x000551f4) 07810003 MOVK X0, #3, #2 +# [DWARF] common/pa_trace.h:561 +# 560 | record.core_idx = trace.core_idx; +# > 561 | record.flags = flags; +0x0000000000055448 (+0x000551f8) 03c03028 ST_XD_XN_IMM.B64 X0, X3, #40 +# [DWARF] common/pa_trace.h:565 +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x000000000005544c (+0x000551fc) 08002001 ADD_IMM.S64 X0, X2, #1 +0x0000000000055450 (+0x00055200) 03804000 ST_XD_XN_IMM.B32 X0, X4, #0 +0x0000000000055454 (+0x00055204) 4000013a JUMP #314 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055458 (+0x00055208) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000005545c (+0x0005520c) 1ce5eb48 LD_XD_XN_IMM.B64 X18, X30, #2888 +0x0000000000055460 (+0x00055210) 02179800 MOV_XD_XN.S64 X11, X25 +0x0000000000055464 (+0x00055214) 40000002 JUMP #2 +0x0000000000055468 (+0x00055218) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_scheduler_core.h:1435 +# 1429 | break; +# 1430 | } +# 1431 | #else +# 1432 | BuildAllocArgs(orchestration, args, batch); +# 1433 | ++stats.result.context_reads; +# 1434 | stats.result.views_created += 2; +# > 1435 | stats.result.tensor_args_added += 3; +0x000000000005546c (+0x0005521c) 1cc1e930 LD_XD_XN_IMM.B64 X0, X30, #2352 +# [DWARF] common/pa_trace.h:430 +# 424 | (void)result; +# 425 | (void)task_id; +# 426 | (void)site; +# 427 | (void)result_used; +# 428 | return Ops::Exchange(address, value); +# 429 | #else +# > 430 | if (!trace.atomics_enabled) return Ops::Exchange(address, value); +0x0000000000055470 (+0x00055220) 02172900 MOV_SPR_XN.S64 CONDITION_FLAG, X18 +# [DWARF] common/pa_scheduler_core.h:1435 +# 1429 | break; +# 1430 | } +# 1431 | #else +# 1432 | BuildAllocArgs(orchestration, args, batch); +# 1433 | ++stats.result.context_reads; +# 1434 | stats.result.views_created += 2; +# > 1435 | stats.result.tensor_args_added += 3; +0x0000000000055474 (+0x00055224) 08000003 ADD_IMM.S64 X0, X0, #3 +# [DWARF] common/pa_trace.h:430 +# 424 | (void)result; +# 425 | (void)task_id; +# 426 | (void)site; +# 427 | (void)result_used; +# 428 | return Ops::Exchange(address, value); +# 429 | #else +# > 430 | if (!trace.atomics_enabled) return Ops::Exchange(address, value); +0x0000000000055478 (+0x00055228) 03c1ea08 ST_XD_XN_IMM.B64 X0, X30, #2568 +0x000000000005547c (+0x0005522c) 40200002 JUMPC #2 +0x0000000000055480 (+0x00055230) 4000001d JUMP #29 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055484 (+0x00055234) 03d7eb68 ST_XD_XN_IMM.B64 X11, X30, #2920 +0x0000000000055488 (+0x00055238) 1cc5e928 LD_XD_XN_IMM.B64 X2, X30, #2344 +0x000000000005548c (+0x0005523c) 40000057 JUMP #87 +0x0000000000055490 (+0x00055240) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055494 (+0x00055244) 1ce5eb48 LD_XD_XN_IMM.B64 X18, X30, #2888 +0x0000000000055498 (+0x00055248) 1cd7eb68 LD_XD_XN_IMM.B64 X11, X30, #2920 +0x000000000005549c (+0x0005524c) 40000002 JUMP #2 +0x00000000000554a0 (+0x00055250) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_scheduler_core.h:753 +# 747 | // Register 后到 Submit.end 由离线 submit_tail_gap 补集展示；排他 +# 748 | // 报告汇总为 submit_tail_residual，避免零时长 marker 增加 raw +# 749 | // 体积和 trace-buffer 写开销。 +# 750 | } +# 751 | } +# 752 | +# > 753 | ++stats.result.submits; +0x00000000000554a4 (+0x00055254) 1cc1e940 LD_XD_XN_IMM.B64 X0, X30, #2368 +# [DWARF] common/pa_trace.h:430 +# 424 | (void)result; +# 425 | (void)task_id; +# 426 | (void)site; +# 427 | (void)result_used; +# 428 | return Ops::Exchange(address, value); +# 429 | #else +# > 430 | if (!trace.atomics_enabled) return Ops::Exchange(address, value); +0x00000000000554a8 (+0x00055258) 02172900 MOV_SPR_XN.S64 CONDITION_FLAG, X18 +0x00000000000554ac (+0x0005525c) 1ccbe948 LD_XD_XN_IMM.B64 X5, X30, #2376 +# [DWARF] common/pa_scheduler_core.h:1444 +# 1438 | )) { +# 1439 | break; +# 1440 | } +# 1441 | AcceptTaskOutputs(orchestration, TaskKind::Alloc, context.result); +# 1442 | +# 1443 | BuildQkArgs(orchestration, args, batch); +# > 1444 | ++stats.result.dynamic_create_infos; +0x00000000000554b0 (+0x00055260) 1cc7ea40 LD_XD_XN_IMM.B64 X3, X30, #2624 +# [DWARF] common/pa_scheduler_core.h:753 +# 747 | // Register 后到 Submit.end 由离线 submit_tail_gap 补集展示；排他 +# 748 | // 报告汇总为 submit_tail_residual，避免零时长 marker 增加 raw +# 749 | // 体积和 trace-buffer 写开销。 +# 750 | } +# 751 | } +# 752 | +# > 753 | ++stats.result.submits; +0x00000000000554b4 (+0x00055264) 08020001 ADD_IMM.S64 X1, X0, #1 +# [DWARF] common/pa_scheduler_core.h:1447 +# 1441 | AcceptTaskOutputs(orchestration, TaskKind::Alloc, context.result); +# 1442 | +# 1443 | BuildQkArgs(orchestration, args, batch); +# 1444 | ++stats.result.dynamic_create_infos; +# 1445 | ++stats.result.arg_resets; +# 1446 | stats.result.tensor_args_added += 4; +# > 1447 | stats.result.scalar_args_added += 2; +0x00000000000554b8 (+0x00055268) 1cc1e928 LD_XD_XN_IMM.B64 X0, X30, #2344 +0x00000000000554bc (+0x0005526c) 08040002 ADD_IMM.S64 X2, X0, #2 +# [DWARF] common/pa_scheduler_core.h:1446 +# 1440 | } +# 1441 | AcceptTaskOutputs(orchestration, TaskKind::Alloc, context.result); +# 1442 | +# 1443 | BuildQkArgs(orchestration, args, batch); +# 1444 | ++stats.result.dynamic_create_infos; +# 1445 | ++stats.result.arg_resets; +# > 1446 | stats.result.tensor_args_added += 4; +0x00000000000554c0 (+0x00055270) 1cc1e930 LD_XD_XN_IMM.B64 X0, X30, #2352 +0x00000000000554c4 (+0x00055274) 08000007 ADD_IMM.S64 X0, X0, #7 +0x00000000000554c8 (+0x00055278) 03c1ea08 ST_XD_XN_IMM.B64 X0, X30, #2568 +0x00000000000554cc (+0x0005527c) 07000000 MOV_XD_IMM X0, #0 +# [DWARF] common/pa_scheduler_core.h:1445 +# 1439 | break; +# 1440 | } +# 1441 | AcceptTaskOutputs(orchestration, TaskKind::Alloc, context.result); +# 1442 | +# 1443 | BuildQkArgs(orchestration, args, batch); +# 1444 | ++stats.result.dynamic_create_infos; +# > 1445 | ++stats.result.arg_resets; +0x00000000000554d0 (+0x00055280) 02ca0440 SBITSET.B64 X5, X0 +# [DWARF] common/pa_scheduler_core.h:1444 +# 1438 | )) { +# 1439 | break; +# 1440 | } +# 1441 | AcceptTaskOutputs(orchestration, TaskKind::Alloc, context.result); +# 1442 | +# 1443 | BuildQkArgs(orchestration, args, batch); +# > 1444 | ++stats.result.dynamic_create_infos; +0x00000000000554d4 (+0x00055284) 02c60440 SBITSET.B64 X3, X0 +# [DWARF] common/pa_trace.h:430 +# 424 | (void)result; +# 425 | (void)task_id; +# 426 | (void)site; +# 427 | (void)result_used; +# 428 | return Ops::Exchange(address, value); +# 429 | #else +# > 430 | if (!trace.atomics_enabled) return Ops::Exchange(address, value); +0x00000000000554d8 (+0x00055288) 03c7ea40 ST_XD_XN_IMM.B64 X3, X30, #2624 +0x00000000000554dc (+0x0005528c) 40200002 JUMPC #2 +0x00000000000554e0 (+0x00055290) 40000051 JUMP #81 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000554e4 (+0x00055294) 03cbe948 ST_XD_XN_IMM.B64 X5, X30, #2376 +0x00000000000554e8 (+0x00055298) 03f3eb70 ST_XD_XN_IMM.B64 X25, X30, #2928 +0x00000000000554ec (+0x0005529c) 03c3e940 ST_XD_XN_IMM.B64 X1, X30, #2368 +0x00000000000554f0 (+0x000552a0) 40000086 JUMP #134 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x00000000000554f4 (+0x000552a4) 02808880 MOV_XD_SPR.F32 X0, SYS_CNT +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:45 +# 39 | CCE_INTRINSIC[aicore] FTYPE atomicSub(__gm__ FTYPE *base, FTYPE inc) { \ +# 40 | return __builtin_cce_atom_add_G_##ADD_SUFFIX(base, -inc, \ +# 41 | (uint32_t)L2Cache); \ +# 42 | } +# 43 | +# 44 | __CCE__ATOM_G_CAS_EXCH_INT32(uint32_t, u32, u32) +# > 45 | __CCE__ATOM_G_CAS_EXCH_INT32(int32_t, s32, s32) +0x00000000000554f8 (+0x000552a8) 1cc3eac8 LD_XD_XN_IMM.B64 X1, X30, #2760 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:0 +# [SOURCE unavailable] +0x00000000000554fc (+0x000552ac) 07040001 MOV_XD_IMM X2, #1 +0x0000000000055500 (+0x000552b0) 07060001 MOV_XD_IMM X3, #1 +0x0000000000055504 (+0x000552b4) 02063080 NEG.S64 X3, X3 +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x0000000000055508 (+0x000552b8) 0001318e CMP.S64.EQ X19, X3 +0x000000000005550c (+0x000552bc) 08093001 ADD_IMM.S64 X4, X19, #1 +0x0000000000055510 (+0x000552c0) 00e63209 SEL.B64 X19, X3, X4 +0x0000000000055514 (+0x000552c4) 00da2689 SEL.B64 X13, X2, X13 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000055518 (+0x000552c8) 02175900 MOV_SPR_XN.S64 CONDITION_FLAG, X21 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:45 +# 39 | CCE_INTRINSIC[aicore] FTYPE atomicSub(__gm__ FTYPE *base, FTYPE inc) { \ +# 40 | return __builtin_cce_atom_add_G_##ADD_SUFFIX(base, -inc, \ +# 41 | (uint32_t)L2Cache); \ +# 42 | } +# 43 | +# 44 | __CCE__ATOM_G_CAS_EXCH_INT32(uint32_t, u32, u32) +# > 45 | __CCE__ATOM_G_CAS_EXCH_INT32(int32_t, s32, s32) +0x000000000005551c (+0x000552cc) 50108440 ATOM XN, XM, XD, EXCH +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000055520 (+0x000552d0) 02828880 MOV_XD_SPR.F32 X1, SYS_CNT +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x0000000000055524 (+0x000552d4) 1ce3eb10 LD_XD_XN_IMM.B64 X17, X30, #2832 +0x0000000000055528 (+0x000552d8) 1ce9eb80 LD_XD_XN_IMM.B64 X20, X30, #2944 +0x000000000005552c (+0x000552dc) 1cddeb98 LD_XD_XN_IMM.B64 X14, X30, #2968 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000055530 (+0x000552e0) 40200002 JUMPC #2 +0x0000000000055534 (+0x000552e4) 400000b9 JUMP #185 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055538 (+0x000552e8) 1cc1e940 LD_XD_XN_IMM.B64 X0, X30, #2368 +0x000000000005553c (+0x000552ec) 02398800 MOV_XD_XN.S64 X28, X24 +0x0000000000055540 (+0x000552f0) 1cd3ea30 LD_XD_XN_IMM.B64 X9, X30, #2608 +0x0000000000055544 (+0x000552f4) 1cf3eb70 LD_XD_XN_IMM.B64 X25, X30, #2928 +0x0000000000055548 (+0x000552f8) 03c1ea18 ST_XD_XN_IMM.B64 X0, X30, #2584 +0x000000000005554c (+0x000552fc) 1cc1e948 LD_XD_XN_IMM.B64 X0, X30, #2376 +0x0000000000055550 (+0x00055300) 03c1ea00 ST_XD_XN_IMM.B64 X0, X30, #2560 +0x0000000000055554 (+0x00055304) 1cc1e928 LD_XD_XN_IMM.B64 X0, X30, #2344 +0x0000000000055558 (+0x00055308) 03c1ea10 ST_XD_XN_IMM.B64 X0, X30, #2576 +0x000000000005555c (+0x0005530c) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055560 (+0x00055310) 070001bb MOV_XD_IMM X0, #443 +0x0000000000055564 (+0x00055314) 07410000 MOVK X0, #0, #1 +0x0000000000055568 (+0x00055318) 07810000 MOVK X0, #0, #2 +0x000000000005556c (+0x0005531c) 40020000 JUMP X0, #0 +0x0000000000055570 (+0x00055320) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055574 (+0x00055324) 1ce5eb48 LD_XD_XN_IMM.B64 X18, X30, #2888 +0x0000000000055578 (+0x00055328) 1cd7eb68 LD_XD_XN_IMM.B64 X11, X30, #2920 +0x000000000005557c (+0x0005532c) 40000002 JUMP #2 +0x0000000000055580 (+0x00055330) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_scheduler_core.h:753 +# 747 | // Register 后到 Submit.end 由离线 submit_tail_gap 补集展示；排他 +# 748 | // 报告汇总为 submit_tail_residual，避免零时长 marker 增加 raw +# 749 | // 体积和 trace-buffer 写开销。 +# 750 | } +# 751 | } +# 752 | +# > 753 | ++stats.result.submits; +0x0000000000055584 (+0x00055334) 1cc1e940 LD_XD_XN_IMM.B64 X0, X30, #2368 +0x0000000000055588 (+0x00055338) 07040001 MOV_XD_IMM X2, #1 +0x000000000005558c (+0x0005533c) 1ccbe948 LD_XD_XN_IMM.B64 X5, X30, #2376 +# [DWARF] common/pa_trace.h:430 +# 424 | (void)result; +# 425 | (void)task_id; +# 426 | (void)site; +# 427 | (void)result_used; +# 428 | return Ops::Exchange(address, value); +# 429 | #else +# > 430 | if (!trace.atomics_enabled) return Ops::Exchange(address, value); +0x0000000000055590 (+0x00055340) 02172900 MOV_SPR_XN.S64 CONDITION_FLAG, X18 +# [DWARF] common/pa_scheduler_core.h:753 +# 747 | // Register 后到 Submit.end 由离线 submit_tail_gap 补集展示；排他 +# 748 | // 报告汇总为 submit_tail_residual，避免零时长 marker 增加 raw +# 749 | // 体积和 trace-buffer 写开销。 +# 750 | } +# 751 | } +# 752 | +# > 753 | ++stats.result.submits; +0x0000000000055594 (+0x00055344) 08020002 ADD_IMM.S64 X1, X0, #2 +# [DWARF] common/pa_scheduler_core.h:1459 +# 1453 | AcceptTaskOutputs(orchestration, TaskKind::Qk, context.result); +# 1454 | +# 1455 | BuildSfArgs(orchestration, args); +# 1456 | ++stats.result.dynamic_create_infos; +# 1457 | ++stats.result.arg_resets; +# 1458 | stats.result.tensor_args_added += 4; +# > 1459 | stats.result.scalar_args_added += 3; +0x0000000000055598 (+0x00055348) 1cc1e928 LD_XD_XN_IMM.B64 X0, X30, #2344 +# [DWARF] common/pa_scheduler_core.h:1457 +# 1451 | break; +# 1452 | } +# 1453 | AcceptTaskOutputs(orchestration, TaskKind::Qk, context.result); +# 1454 | +# 1455 | BuildSfArgs(orchestration, args); +# 1456 | ++stats.result.dynamic_create_infos; +# > 1457 | ++stats.result.arg_resets; +0x000000000005559c (+0x0005534c) 02ca2440 SBITSET.B64 X5, X2 +# [DWARF] common/pa_scheduler_core.h:1459 +# 1458 | stats.result.tensor_args_added += 4; +# > 1459 | stats.result.scalar_args_added += 3; +0x00000000000555a0 (+0x00055350) 08060005 ADD_IMM.S64 X3, X0, #5 +# [DWARF] common/pa_scheduler_core.h:1458 +# 1452 | } +# 1453 | AcceptTaskOutputs(orchestration, TaskKind::Qk, context.result); +# 1454 | +# 1455 | BuildSfArgs(orchestration, args); +# 1456 | ++stats.result.dynamic_create_infos; +# 1457 | ++stats.result.arg_resets; +# > 1458 | stats.result.tensor_args_added += 4; +0x00000000000555a4 (+0x00055354) 1cc1e930 LD_XD_XN_IMM.B64 X0, X30, #2352 +0x00000000000555a8 (+0x00055358) 0800000b ADD_IMM.S64 X0, X0, #11 +0x00000000000555ac (+0x0005535c) 03c1ea08 ST_XD_XN_IMM.B64 X0, X30, #2568 +# [DWARF] common/pa_trace.h:430 +# 424 | (void)result; +# 425 | (void)task_id; +# 426 | (void)site; +# 427 | (void)result_used; +# 428 | return Ops::Exchange(address, value); +# 429 | #else +# > 430 | if (!trace.atomics_enabled) return Ops::Exchange(address, value); +0x00000000000555b0 (+0x00055360) 40200002 JUMPC #2 +0x00000000000555b4 (+0x00055364) 40000079 JUMP #121 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000555b8 (+0x00055368) 1cc1ea30 LD_XD_XN_IMM.B64 X0, X30, #2608 +0x00000000000555bc (+0x0005536c) 03cbe948 ST_XD_XN_IMM.B64 X5, X30, #2376 +0x00000000000555c0 (+0x00055370) 02043800 MOV_XD_XN.S64 X2, X3 +0x00000000000555c4 (+0x00055374) 03f3eb70 ST_XD_XN_IMM.B64 X25, X30, #2928 +0x00000000000555c8 (+0x00055378) 02198800 MOV_XD_XN.S64 X12, X24 +0x00000000000555cc (+0x0005537c) 03c3e940 ST_XD_XN_IMM.B64 X1, X30, #2368 +0x00000000000555d0 (+0x00055380) 03c1ea40 ST_XD_XN_IMM.B64 X0, X30, #2624 +0x00000000000555d4 (+0x00055384) 40000002 JUMP #2 +0x00000000000555d8 (+0x00055388) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x00000000000555dc (+0x0005538c) 1cf5eb58 LD_XD_XN_IMM.B64 X26, X30, #2904 +0x00000000000555e0 (+0x00055390) 40000002 JUMP #2 +0x00000000000555e4 (+0x00055394) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x00000000000555e8 (+0x00055398) 1cc3eac8 LD_XD_XN_IMM.B64 X1, X30, #2760 +0x00000000000555ec (+0x0005539c) 1cc7eab0 LD_XD_XN_IMM.B64 X3, X30, #2736 +0x00000000000555f0 (+0x000553a0) 4000004e JUMP #78 +0x00000000000555f4 (+0x000553a4) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_trace.h:549 +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# > 549 | core.dropped = core.dropped + 1; +0x00000000000555f8 (+0x000553a8) 1c826004 LD_XD_XN_IMM.B32 X1, X6, #4 +0x00000000000555fc (+0x000553ac) 08021001 ADD_IMM.S64 X1, X1, #1 +0x0000000000055600 (+0x000553b0) 03826004 ST_XD_XN_IMM.B32 X1, X6, #4 +0x0000000000055604 (+0x000553b4) 40000002 JUMP #2 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055608 (+0x000553b8) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000005560c (+0x000553bc) 07140000 MOV_XD_IMM X10, #0 +0x0000000000055610 (+0x000553c0) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055614 (+0x000553c4) 0700ac2f MOV_XD_IMM X0, #44079 +0x0000000000055618 (+0x000553c8) 0741fffe MOVK X0, #65534, #1 +0x000000000005561c (+0x000553cc) 0781ffff MOVK X0, #65535, #2 +0x0000000000055620 (+0x000553d0) 40020000 JUMP X0, #0 +0x0000000000055624 (+0x000553d4) 03c5ea10 ST_XD_XN_IMM.B64 X2, X30, #2576 +0x0000000000055628 (+0x000553d8) 07040001 MOV_XD_IMM X2, #1 +0x000000000005562c (+0x000553dc) 03c3ea18 ST_XD_XN_IMM.B64 X1, X30, #2584 +0x0000000000055630 (+0x000553e0) 07060001 MOV_XD_IMM X3, #1 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000055634 (+0x000553e4) 02808880 MOV_XD_SPR.F32 X0, SYS_CNT +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:45 +# 39 | CCE_INTRINSIC[aicore] FTYPE atomicSub(__gm__ FTYPE *base, FTYPE inc) { \ +# 40 | return __builtin_cce_atom_add_G_##ADD_SUFFIX(base, -inc, \ +# 41 | (uint32_t)L2Cache); \ +# 42 | } +# 43 | +# 44 | __CCE__ATOM_G_CAS_EXCH_INT32(uint32_t, u32, u32) +# > 45 | __CCE__ATOM_G_CAS_EXCH_INT32(int32_t, s32, s32) +0x0000000000055638 (+0x000553e8) 1cc3eac8 LD_XD_XN_IMM.B64 X1, X30, #2760 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:0 +# [SOURCE unavailable] +0x000000000005563c (+0x000553ec) 02063080 NEG.S64 X3, X3 +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x0000000000055640 (+0x000553f0) 0001318e CMP.S64.EQ X19, X3 +0x0000000000055644 (+0x000553f4) 08093001 ADD_IMM.S64 X4, X19, #1 +0x0000000000055648 (+0x000553f8) 00e63209 SEL.B64 X19, X3, X4 +0x000000000005564c (+0x000553fc) 00da2689 SEL.B64 X13, X2, X13 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000055650 (+0x00055400) 02175900 MOV_SPR_XN.S64 CONDITION_FLAG, X21 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:45 +# 39 | CCE_INTRINSIC[aicore] FTYPE atomicSub(__gm__ FTYPE *base, FTYPE inc) { \ +# 40 | return __builtin_cce_atom_add_G_##ADD_SUFFIX(base, -inc, \ +# 41 | (uint32_t)L2Cache); \ +# 42 | } +# 43 | +# 44 | __CCE__ATOM_G_CAS_EXCH_INT32(uint32_t, u32, u32) +# > 45 | __CCE__ATOM_G_CAS_EXCH_INT32(int32_t, s32, s32) +0x0000000000055654 (+0x00055404) 50108440 ATOM XN, XM, XD, EXCH +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000055658 (+0x00055408) 02828880 MOV_XD_SPR.F32 X1, SYS_CNT +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x000000000005565c (+0x0005540c) 1ce9eb80 LD_XD_XN_IMM.B64 X20, X30, #2944 +0x0000000000055660 (+0x00055410) 1cddeb98 LD_XD_XN_IMM.B64 X14, X30, #2968 +0x0000000000055664 (+0x00055414) 1cf5eb58 LD_XD_XN_IMM.B64 X26, X30, #2904 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000055668 (+0x00055418) 40200006 JUMPC #6 +0x000000000005566c (+0x0005541c) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055670 (+0x00055420) 0700017a MOV_XD_IMM X0, #378 +0x0000000000055674 (+0x00055424) 07410000 MOVK X0, #0, #1 +0x0000000000055678 (+0x00055428) 07810000 MOVK X0, #0, #2 +0x000000000005567c (+0x0005542c) 40020000 JUMP X0, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055680 (+0x00055430) 1cd3ea30 LD_XD_XN_IMM.B64 X9, X30, #2608 +0x0000000000055684 (+0x00055434) 03cbea00 ST_XD_XN_IMM.B64 X5, X30, #2560 +0x0000000000055688 (+0x00055438) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000005568c (+0x0005543c) 0700016b MOV_XD_IMM X0, #363 +0x0000000000055690 (+0x00055440) 07410000 MOVK X0, #0, #1 +0x0000000000055694 (+0x00055444) 07810000 MOVK X0, #0, #2 +0x0000000000055698 (+0x00055448) 40020000 JUMP X0, #0 +0x000000000005569c (+0x0005544c) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x00000000000556a0 (+0x00055450) 1ce5eb48 LD_XD_XN_IMM.B64 X18, X30, #2888 +0x00000000000556a4 (+0x00055454) 1cd7eb68 LD_XD_XN_IMM.B64 X11, X30, #2920 +0x00000000000556a8 (+0x00055458) 40000002 JUMP #2 +0x00000000000556ac (+0x0005545c) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_scheduler_core.h:753 +# 747 | // Register 后到 Submit.end 由离线 submit_tail_gap 补集展示；排他 +# 748 | // 报告汇总为 submit_tail_residual，避免零时长 marker 增加 raw +# 749 | // 体积和 trace-buffer 写开销。 +# 750 | } +# 751 | } +# 752 | +# > 753 | ++stats.result.submits; +0x00000000000556b0 (+0x00055460) 1cc1e940 LD_XD_XN_IMM.B64 X0, X30, #2368 +# [DWARF] common/pa_trace.h:430 +# 424 | (void)result; +# 425 | (void)task_id; +# 426 | (void)site; +# 427 | (void)result_used; +# 428 | return Ops::Exchange(address, value); +# 429 | #else +# > 430 | if (!trace.atomics_enabled) return Ops::Exchange(address, value); +0x00000000000556b4 (+0x00055464) 02172900 MOV_SPR_XN.S64 CONDITION_FLAG, X18 +# [DWARF] common/pa_scheduler_core.h:1468 +# 1462 | )) { +# 1463 | break; +# 1464 | } +# 1465 | AcceptTaskOutputs(orchestration, TaskKind::Sf, context.result); +# 1466 | +# 1467 | BuildPvArgs(orchestration, args, batch); +# > 1468 | ++stats.result.arg_resets; +0x00000000000556b8 (+0x00055468) 1cc7e948 LD_XD_XN_IMM.B64 X3, X30, #2376 +# [DWARF] common/pa_scheduler_core.h:753 +# 747 | // Register 后到 Submit.end 由离线 submit_tail_gap 补集展示；排他 +# 748 | // 报告汇总为 submit_tail_residual，避免零时长 marker 增加 raw +# 749 | // 体积和 trace-buffer 写开销。 +# 750 | } +# 751 | } +# 752 | +# > 753 | ++stats.result.submits; +0x00000000000556bc (+0x0005546c) 08020003 ADD_IMM.S64 X1, X0, #3 +# [DWARF] common/pa_scheduler_core.h:1470 +# 1464 | } +# 1465 | AcceptTaskOutputs(orchestration, TaskKind::Sf, context.result); +# 1466 | +# 1467 | BuildPvArgs(orchestration, args, batch); +# 1468 | ++stats.result.arg_resets; +# 1469 | stats.result.tensor_args_added += 4; +# > 1470 | stats.result.scalar_args_added += 2; +0x00000000000556c0 (+0x00055470) 1cc1e928 LD_XD_XN_IMM.B64 X0, X30, #2344 +0x00000000000556c4 (+0x00055474) 08040007 ADD_IMM.S64 X2, X0, #7 +# [DWARF] common/pa_scheduler_core.h:1469 +# 1463 | break; +# 1464 | } +# 1465 | AcceptTaskOutputs(orchestration, TaskKind::Sf, context.result); +# 1466 | +# 1467 | BuildPvArgs(orchestration, args, batch); +# 1468 | ++stats.result.arg_resets; +# > 1469 | stats.result.tensor_args_added += 4; +0x00000000000556c8 (+0x00055478) 1cc1e930 LD_XD_XN_IMM.B64 X0, X30, #2352 +0x00000000000556cc (+0x0005547c) 0800000f ADD_IMM.S64 X0, X0, #15 +0x00000000000556d0 (+0x00055480) 03c1ea08 ST_XD_XN_IMM.B64 X0, X30, #2568 +0x00000000000556d4 (+0x00055484) 07000003 MOV_XD_IMM X0, #3 +# [DWARF] common/pa_scheduler_core.h:1468 +# 1462 | )) { +# 1463 | break; +# 1464 | } +# 1465 | AcceptTaskOutputs(orchestration, TaskKind::Sf, context.result); +# 1466 | +# 1467 | BuildPvArgs(orchestration, args, batch); +# > 1468 | ++stats.result.arg_resets; +0x00000000000556d8 (+0x00055488) 00c0300b OR.B64 X0, X3, X0 +# [DWARF] common/pa_trace.h:430 +# 424 | (void)result; +# 425 | (void)task_id; +# 426 | (void)site; +# 427 | (void)result_used; +# 428 | return Ops::Exchange(address, value); +# 429 | #else +# > 430 | if (!trace.atomics_enabled) return Ops::Exchange(address, value); +0x00000000000556dc (+0x0005548c) 40200006 JUMPC #6 +0x00000000000556e0 (+0x00055490) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x00000000000556e4 (+0x00055494) 0700013c MOV_XD_IMM X0, #316 +0x00000000000556e8 (+0x00055498) 07410000 MOVK X0, #0, #1 +0x00000000000556ec (+0x0005549c) 07810000 MOVK X0, #0, #2 +0x00000000000556f0 (+0x000554a0) 40020000 JUMP X0, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000556f4 (+0x000554a4) 03c3e940 ST_XD_XN_IMM.B64 X1, X30, #2368 +0x00000000000556f8 (+0x000554a8) 1cc3ea30 LD_XD_XN_IMM.B64 X1, X30, #2608 +0x00000000000556fc (+0x000554ac) 03f3eb70 ST_XD_XN_IMM.B64 X25, X30, #2928 +0x0000000000055700 (+0x000554b0) 03c1e948 ST_XD_XN_IMM.B64 X0, X30, #2376 +0x0000000000055704 (+0x000554b4) 03c3ea40 ST_XD_XN_IMM.B64 X1, X30, #2624 +0x0000000000055708 (+0x000554b8) 02198800 MOV_XD_XN.S64 X12, X24 +0x000000000005570c (+0x000554bc) 1cc7e950 LD_XD_XN_IMM.B64 X3, X30, #2384 +0x0000000000055710 (+0x000554c0) 40000002 JUMP #2 +0x0000000000055714 (+0x000554c4) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055718 (+0x000554c8) 1cf5eb58 LD_XD_XN_IMM.B64 X26, X30, #2904 +0x000000000005571c (+0x000554cc) 40000002 JUMP #2 +0x0000000000055720 (+0x000554d0) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055724 (+0x000554d4) 1cc3eac8 LD_XD_XN_IMM.B64 X1, X30, #2760 +0x0000000000055728 (+0x000554d8) 07000001 MOV_XD_IMM X0, #1 +0x000000000005572c (+0x000554dc) 50100400 ATOM XN, XM, XD, EXCH +0x0000000000055730 (+0x000554e0) 02023800 MOV_XD_XN.S64 X1, X3 +0x0000000000055734 (+0x000554e4) 40000002 JUMP #2 +0x0000000000055738 (+0x000554e8) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x000000000005573c (+0x000554ec) 02808880 MOV_XD_SPR.F32 X0, SYS_CNT +0x0000000000055740 (+0x000554f0) 03c5ea10 ST_XD_XN_IMM.B64 X2, X30, #2576 +0x0000000000055744 (+0x000554f4) 03c3eab0 ST_XD_XN_IMM.B64 X1, X30, #2736 +0x0000000000055748 (+0x000554f8) 1ce3eb10 LD_XD_XN_IMM.B64 X17, X30, #2832 +0x000000000005574c (+0x000554fc) 07020000 MOV_XD_IMM X1, #0 +0x0000000000055750 (+0x00055500) 1ce9eb80 LD_XD_XN_IMM.B64 X20, X30, #2944 +0x0000000000055754 (+0x00055504) 0238c800 MOV_XD_XN.S64 X28, X12 +0x0000000000055758 (+0x00055508) 1cddeb98 LD_XD_XN_IMM.B64 X14, X30, #2968 +0x000000000005575c (+0x0005550c) 1cd7eb68 LD_XD_XN_IMM.B64 X11, X30, #2920 +0x0000000000055760 (+0x00055510) 1ce5eb48 LD_XD_XN_IMM.B64 X18, X30, #2888 +0x0000000000055764 (+0x00055514) 1cebeb40 LD_XD_XN_IMM.B64 X21, X30, #2880 +0x0000000000055768 (+0x00055518) 1cf3eb70 LD_XD_XN_IMM.B64 X25, X30, #2928 +0x000000000005576c (+0x0005551c) 1ce7eb00 LD_XD_XN_IMM.B64 X19, X30, #2816 +0x0000000000055770 (+0x00055520) 03c1ea70 ST_XD_XN_IMM.B64 X0, X30, #2672 +0x0000000000055774 (+0x00055524) 1cc1e940 LD_XD_XN_IMM.B64 X0, X30, #2368 +0x0000000000055778 (+0x00055528) 03c1ea18 ST_XD_XN_IMM.B64 X0, X30, #2584 +0x000000000005577c (+0x0005552c) 1cc1e948 LD_XD_XN_IMM.B64 X0, X30, #2376 +0x0000000000055780 (+0x00055530) 03c1ea00 ST_XD_XN_IMM.B64 X0, X30, #2560 +0x0000000000055784 (+0x00055534) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055788 (+0x00055538) 0700025a MOV_XD_IMM X0, #602 +0x000000000005578c (+0x0005553c) 07410000 MOVK X0, #0, #1 +0x0000000000055790 (+0x00055540) 07810000 MOVK X0, #0, #2 +0x0000000000055794 (+0x00055544) 40020000 JUMP X0, #0 +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x0000000000055798 (+0x00055548) 03c7ea10 ST_XD_XN_IMM.B64 X3, X30, #2576 +0x000000000005579c (+0x0005554c) 07060001 MOV_XD_IMM X3, #1 +0x00000000000557a0 (+0x00055550) 03c3ea18 ST_XD_XN_IMM.B64 X1, X30, #2584 +0x00000000000557a4 (+0x00055554) 02063080 NEG.S64 X3, X3 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x00000000000557a8 (+0x00055558) 02808880 MOV_XD_SPR.F32 X0, SYS_CNT +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:45 +# 39 | CCE_INTRINSIC[aicore] FTYPE atomicSub(__gm__ FTYPE *base, FTYPE inc) { \ +# 40 | return __builtin_cce_atom_add_G_##ADD_SUFFIX(base, -inc, \ +# 41 | (uint32_t)L2Cache); \ +# 42 | } +# 43 | +# 44 | __CCE__ATOM_G_CAS_EXCH_INT32(uint32_t, u32, u32) +# > 45 | __CCE__ATOM_G_CAS_EXCH_INT32(int32_t, s32, s32) +0x00000000000557ac (+0x0005555c) 1cc3eac8 LD_XD_XN_IMM.B64 X1, X30, #2760 +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x00000000000557b0 (+0x00055560) 0001318e CMP.S64.EQ X19, X3 +0x00000000000557b4 (+0x00055564) 08093001 ADD_IMM.S64 X4, X19, #1 +0x00000000000557b8 (+0x00055568) 00e63209 SEL.B64 X19, X3, X4 +0x00000000000557bc (+0x0005556c) 00da2689 SEL.B64 X13, X2, X13 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x00000000000557c0 (+0x00055570) 02175900 MOV_SPR_XN.S64 CONDITION_FLAG, X21 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:45 +# 39 | CCE_INTRINSIC[aicore] FTYPE atomicSub(__gm__ FTYPE *base, FTYPE inc) { \ +# 40 | return __builtin_cce_atom_add_G_##ADD_SUFFIX(base, -inc, \ +# 41 | (uint32_t)L2Cache); \ +# 42 | } +# 43 | +# 44 | __CCE__ATOM_G_CAS_EXCH_INT32(uint32_t, u32, u32) +# > 45 | __CCE__ATOM_G_CAS_EXCH_INT32(int32_t, s32, s32) +0x00000000000557c4 (+0x00055574) 50108440 ATOM XN, XM, XD, EXCH +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x00000000000557c8 (+0x00055578) 02828880 MOV_XD_SPR.F32 X1, SYS_CNT +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x00000000000557cc (+0x0005557c) 1ce9eb80 LD_XD_XN_IMM.B64 X20, X30, #2944 +0x00000000000557d0 (+0x00055580) 1cddeb98 LD_XD_XN_IMM.B64 X14, X30, #2968 +0x00000000000557d4 (+0x00055584) 1cf5eb58 LD_XD_XN_IMM.B64 X26, X30, #2904 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x00000000000557d8 (+0x00055588) 40200006 JUMPC #6 +0x00000000000557dc (+0x0005558c) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x00000000000557e0 (+0x00055590) 07000157 MOV_XD_IMM X0, #343 +0x00000000000557e4 (+0x00055594) 07410000 MOVK X0, #0, #1 +0x00000000000557e8 (+0x00055598) 07810000 MOVK X0, #0, #2 +0x00000000000557ec (+0x0005559c) 40020000 JUMP X0, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000557f0 (+0x000555a0) 1cc1ea30 LD_XD_XN_IMM.B64 X0, X30, #2608 +0x00000000000557f4 (+0x000555a4) 03cbea00 ST_XD_XN_IMM.B64 X5, X30, #2560 +0x00000000000557f8 (+0x000555a8) 02398800 MOV_XD_XN.S64 X28, X24 +0x00000000000557fc (+0x000555ac) 02120800 MOV_XD_XN.S64 X9, X0 +0x0000000000055800 (+0x000555b0) 03c1ea40 ST_XD_XN_IMM.B64 X0, X30, #2624 +0x0000000000055804 (+0x000555b4) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055808 (+0x000555b8) 07000111 MOV_XD_IMM X0, #273 +0x000000000005580c (+0x000555bc) 07410000 MOVK X0, #0, #1 +0x0000000000055810 (+0x000555c0) 07810000 MOVK X0, #0, #2 +0x0000000000055814 (+0x000555c4) 40020000 JUMP X0, #0 +# [DWARF] common/pa_trace.h:547 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# > 547 | const uint32_t slot = core.count; +0x0000000000055818 (+0x000555c8) 1c854000 LD_XD_XN_IMM.B32 X2, X20, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000005581c (+0x000555cc) 070c0000 MOV_XD_IMM X6, #0 +0x0000000000055820 (+0x000555d0) 1cf3eb70 LD_XD_XN_IMM.B64 X25, X30, #2928 +0x0000000000055824 (+0x000555d4) 1ccbe948 LD_XD_XN_IMM.B64 X5, X30, #2376 +0x0000000000055828 (+0x000555d8) 1ccfe940 LD_XD_XN_IMM.B64 X7, X30, #2368 +# [DWARF] common/pa_trace.h:548 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# > 548 | if (slot >= trace.capacity) { +0x000000000005582c (+0x000555dc) 0040272e CMP.U64.LT X2, X14 +0x0000000000055830 (+0x000555e0) 40200006 JUMPC #6 +0x0000000000055834 (+0x000555e4) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055838 (+0x000555e8) 07000160 MOV_XD_IMM X0, #352 +0x000000000005583c (+0x000555ec) 07410000 MOVK X0, #0, #1 +0x0000000000055840 (+0x000555f0) 07810000 MOVK X0, #0, #2 +0x0000000000055844 (+0x000555f4) 40020000 JUMP X0, #0 +# [DWARF] common/pa_trace.h:552 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x0000000000055848 (+0x000555f8) 1cc9eb90 LD_XD_XN_IMM.B64 X4, X30, #2960 +0x000000000005584c (+0x000555fc) 02062800 MOV_XD_XN.S64 X3, X2 +0x0000000000055850 (+0x00055600) 03cfea18 ST_XD_XN_IMM.B64 X7, X30, #2584 +# [DWARF] common/pa_trace.h:555 +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x0000000000055854 (+0x00055604) 1cd1eaa8 LD_XD_XN_IMM.B64 X8, X30, #2728 +# [DWARF] common/pa_trace.h:552 +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x0000000000055858 (+0x00055608) 02c60206 SHL.B64 X3, #6 +0x000000000005585c (+0x0005560c) 03cbea00 ST_XD_XN_IMM.B64 X5, X30, #2560 +0x0000000000055860 (+0x00055610) 1cd3ea30 LD_XD_XN_IMM.B64 X9, X30, #2608 +0x0000000000055864 (+0x00055614) 02398800 MOV_XD_XN.S64 X28, X24 +0x0000000000055868 (+0x00055618) 00064181 ADD.S64 X3, X4, X3 +# [DWARF] common/pa_trace.h:553 +# > 553 | record.start_cycle = start_cycle; +0x000000000005586c (+0x0005561c) 09c03081 STP_XI_XJ_XN.B64 X0, X1, X3, #0 +0x0000000000055870 (+0x00055620) 0700ffff MOV_XD_IMM X0, #65535 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055874 (+0x00055624) 0741ffff MOVK X0, #65535, #1 +0x0000000000055878 (+0x00055628) 0702000e MOV_XD_IMM X1, #14 +# [DWARF] common/pa_trace.h:555 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x000000000005587c (+0x0005562c) 09903021 STP_XI_XJ_XN.B32 X8, X0, X3, #16 +# [DWARF] common/pa_trace.h:557 +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x0000000000055880 (+0x00055630) 03c23018 ST_XD_XN_IMM.B64 X1, X3, #24 +0x0000000000055884 (+0x00055634) 1cc3eb30 LD_XD_XN_IMM.B64 X1, X30, #2864 +# [DWARF] common/pa_trace.h:565 +# 558 | record.lane = trace.lane; +# 559 | record.block_id = trace.block_id; +# 560 | record.core_idx = trace.core_idx; +# 561 | record.flags = flags; +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x0000000000055888 (+0x00055638) 08002001 ADD_IMM.S64 X0, X2, #1 +# [DWARF] common/pa_trace.h:559 +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x000000000005588c (+0x0005563c) 08043020 ADD_IMM.S64 X2, X3, #32 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055890 (+0x00055640) 07080001 MOV_XD_IMM X4, #1 +0x0000000000055894 (+0x00055644) 07890003 MOVK X4, #3, #2 +# [DWARF] common/pa_trace.h:559 +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x0000000000055898 (+0x00055648) 09822081 STP_XI_XJ_XN.B32 X1, X1, X2, #0 +0x000000000005589c (+0x0005564c) 1cc3e928 LD_XD_XN_IMM.B64 X1, X30, #2344 +# [DWARF] common/pa_trace.h:561 +# 560 | record.core_idx = trace.core_idx; +# > 561 | record.flags = flags; +0x00000000000558a0 (+0x00055650) 03c83028 ST_XD_XN_IMM.B64 X4, X3, #40 +0x00000000000558a4 (+0x00055654) 03c3ea10 ST_XD_XN_IMM.B64 X1, X30, #2576 +# [DWARF] common/pa_trace.h:565 +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x00000000000558a8 (+0x00055658) 03814000 ST_XD_XN_IMM.B32 X0, X20, #0 +0x00000000000558ac (+0x0005565c) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x00000000000558b0 (+0x00055660) 070001ab MOV_XD_IMM X0, #427 +0x00000000000558b4 (+0x00055664) 07410000 MOVK X0, #0, #1 +0x00000000000558b8 (+0x00055668) 07810000 MOVK X0, #0, #2 +0x00000000000558bc (+0x0005566c) 40020000 JUMP X0, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000558c0 (+0x00055670) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x00000000000558c4 (+0x00055674) 1ce5eb48 LD_XD_XN_IMM.B64 X18, X30, #2888 +0x00000000000558c8 (+0x00055678) 02177800 MOV_XD_XN.S64 X11, X23 +0x00000000000558cc (+0x0005567c) 1cc5e928 LD_XD_XN_IMM.B64 X2, X30, #2344 +0x00000000000558d0 (+0x00055680) 40000002 JUMP #2 +0x00000000000558d4 (+0x00055684) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_scheduler_core.h:753 +# 747 | // Register 后到 Submit.end 由离线 submit_tail_gap 补集展示；排他 +# 748 | // 报告汇总为 submit_tail_residual，避免零时长 marker 增加 raw +# 749 | // 体积和 trace-buffer 写开销。 +# 750 | } +# 751 | } +# 752 | +# > 753 | ++stats.result.submits; +0x00000000000558d8 (+0x00055688) 1cc1e940 LD_XD_XN_IMM.B64 X0, X30, #2368 +# [DWARF] common/pa_trace.h:430 +# 424 | (void)result; +# 425 | (void)task_id; +# 426 | (void)site; +# 427 | (void)result_used; +# 428 | return Ops::Exchange(address, value); +# 429 | #else +# > 430 | if (!trace.atomics_enabled) return Ops::Exchange(address, value); +0x00000000000558dc (+0x0005568c) 02172900 MOV_SPR_XN.S64 CONDITION_FLAG, X18 +# [DWARF] common/pa_scheduler_core.h:753 +# 747 | // Register 后到 Submit.end 由离线 submit_tail_gap 补集展示；排他 +# 748 | // 报告汇总为 submit_tail_residual，避免零时长 marker 增加 raw +# 749 | // 体积和 trace-buffer 写开销。 +# 750 | } +# 751 | } +# 752 | +# > 753 | ++stats.result.submits; +0x00000000000558e0 (+0x00055690) 08000004 ADD_IMM.S64 X0, X0, #4 +# [DWARF] common/pa_trace.h:430 +# 424 | (void)result; +# 425 | (void)task_id; +# 426 | (void)site; +# 427 | (void)result_used; +# 428 | return Ops::Exchange(address, value); +# 429 | #else +# > 430 | if (!trace.atomics_enabled) return Ops::Exchange(address, value); +0x00000000000558e4 (+0x00055694) 40200006 JUMPC #6 +0x00000000000558e8 (+0x00055698) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x00000000000558ec (+0x0005569c) 070000f5 MOV_XD_IMM X0, #245 +0x00000000000558f0 (+0x000556a0) 07410000 MOVK X0, #0, #1 +0x00000000000558f4 (+0x000556a4) 07810000 MOVK X0, #0, #2 +0x00000000000558f8 (+0x000556a8) 40020000 JUMP X0, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000558fc (+0x000556ac) 03c1e940 ST_XD_XN_IMM.B64 X0, X30, #2368 +0x0000000000055900 (+0x000556b0) 1cc1ea30 LD_XD_XN_IMM.B64 X0, X30, #2608 +0x0000000000055904 (+0x000556b4) 03f3eb70 ST_XD_XN_IMM.B64 X25, X30, #2928 +0x0000000000055908 (+0x000556b8) 1cd9eb38 LD_XD_XN_IMM.B64 X12, X30, #2872 +0x000000000005590c (+0x000556bc) 03d7eb68 ST_XD_XN_IMM.B64 X11, X30, #2920 +0x0000000000055910 (+0x000556c0) 1cc7e918 LD_XD_XN_IMM.B64 X3, X30, #2328 +0x0000000000055914 (+0x000556c4) 1cf5eb58 LD_XD_XN_IMM.B64 X26, X30, #2904 +0x0000000000055918 (+0x000556c8) 03c1ea40 ST_XD_XN_IMM.B64 X0, X30, #2624 +0x000000000005591c (+0x000556cc) 1cc1e930 LD_XD_XN_IMM.B64 X0, X30, #2352 +0x0000000000055920 (+0x000556d0) 03c1ea08 ST_XD_XN_IMM.B64 X0, X30, #2568 +0x0000000000055924 (+0x000556d4) 4000ff80 JUMP #65408 +# [DWARF] common/pa_trace.h:549 +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# > 549 | core.dropped = core.dropped + 1; +0x0000000000055928 (+0x000556d8) 1c804004 LD_XD_XN_IMM.B32 X0, X4, #4 +0x000000000005592c (+0x000556dc) 08000001 ADD_IMM.S64 X0, X0, #1 +0x0000000000055930 (+0x000556e0) 03804004 ST_XD_XN_IMM.B32 X0, X4, #4 +0x0000000000055934 (+0x000556e4) 40000002 JUMP #2 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055938 (+0x000556e8) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000005593c (+0x000556ec) 02018800 MOV_XD_XN.S64 X0, X24 +0x0000000000055940 (+0x000556f0) 03cdeb60 ST_XD_XN_IMM.B64 X6, X30, #2912 +# [DWARF] common/pa_trace.h:296 +# 290 | PA_DEVICE void AtomicPollBoundary(TraceContext &trace, WorkerResult &result) { +# 291 | #if PA_BUILD_SUBMIT_PMU +# 292 | (void)trace; +# 293 | (void)result; +# 294 | #else +# 295 | (void)result; +# > 296 | if (trace.poll_burst.active_mask == 0) return; +0x0000000000055944 (+0x000556f4) 02800a00 ZEROEXT.U32 X0, X0 +0x0000000000055948 (+0x000556f8) 07240000 MOV_XD_IMM X18, #0 +0x000000000005594c (+0x000556fc) 0000091e CMP.S64.NE X0, X18 +0x0000000000055950 (+0x00055700) 40200002 JUMPC #2 +0x0000000000055954 (+0x00055704) 4000005f JUMP #95 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055958 (+0x00055708) 03e7eb00 ST_XD_XN_IMM.B64 X19, X30, #2816 +0x000000000005595c (+0x0005570c) 07040000 MOV_XD_IMM X2, #0 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000055960 (+0x00055710) 02ae8880 MOV_XD_SPR.F32 X23, SYS_CNT +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x0000000000055964 (+0x00055714) 07060001 MOV_XD_IMM X3, #1 +0x0000000000055968 (+0x00055718) 07360001 MOV_XD_IMM X27, #1 +0x000000000005596c (+0x0005571c) 07280001 MOV_XD_IMM X20, #1 +0x0000000000055970 (+0x00055720) 072a0006 MOV_XD_IMM X21, #6 +0x0000000000055974 (+0x00055724) 07450100 MOVK X2, #256, #1 +0x0000000000055978 (+0x00055728) 0747ff00 MOVK X3, #65280, #1 +0x000000000005597c (+0x0005572c) 0237b080 NEG.S64 X27, X27 +0x0000000000055980 (+0x00055730) 07320000 MOV_XD_IMM X25, #0 +0x0000000000055984 (+0x00055734) 07380000 MOV_XD_IMM X28, #0 +0x0000000000055988 (+0x00055738) 07268fa0 MOV_XD_IMM X19, #36768 +0x000000000005598c (+0x0005573c) 07670006 MOVK X19, #6, #1 +0x0000000000055990 (+0x00055740) 07a70000 MOVK X19, #0, #2 +0x0000000000055994 (+0x00055744) 07e70000 MOVK X19, #0, #3 +0x0000000000055998 (+0x00055748) 02020880 MOV_XD_SPR.S64 X1, PC +0x000000000005599c (+0x0005574c) 00273081 ADD.S64 X19, X19, X1 +0x00000000000559a0 (+0x00055750) 40000020 JUMP #32 +# [DWARF] common/pa_trace.h:273 +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# > 273 | trace.poll_burst.start_cycle[index], end_cycle, +0x00000000000559a4 (+0x00055754) 070006a8 MOV_XD_IMM X0, #1704 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x00000000000559a8 (+0x00055758) 1cc3eb90 LD_XD_XN_IMM.B64 X1, X30, #2960 +# [DWARF] common/pa_trace.h:273 +# 272 | trace.core, trace.records, trace.capacity, +# > 273 | trace.poll_burst.start_cycle[index], end_cycle, +0x00000000000559ac (+0x0005575c) 0001e001 ADD.S64 X0, X30, X0 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x00000000000559b0 (+0x00055760) 1cc5eb98 LD_XD_XN_IMM.B64 X2, X30, #2968 +# [DWARF] common/pa_trace.h:273 +# 272 | trace.core, trace.records, trace.capacity, +# > 273 | trace.poll_burst.start_cycle[index], end_cycle, +0x00000000000559b4 (+0x00055764) 01c60e00 LD_XD_XN.B64 X3, X0, X28 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x00000000000559b8 (+0x00055768) 02097800 MOV_XD_XN.S64 X4, X23 +0x00000000000559bc (+0x0005576c) 1cc1eb80 LD_XD_XN_IMM.B64 X0, X30, #2944 +0x00000000000559c0 (+0x00055770) 070e241a MOV_XD_IMM X7, #9242 +0x00000000000559c4 (+0x00055774) 074f0000 MOVK X7, #0, #1 +0x00000000000559c8 (+0x00055778) 078f0000 MOVK X7, #0, #2 +0x00000000000559cc (+0x0005577c) 40427000 CALL X7, #0 +# [DWARF] common/pa_trace.h:276 +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x00000000000559d0 (+0x00055780) 00016d8e CMP.S64.EQ X22, X27 +0x00000000000559d4 (+0x00055784) 08036001 ADD_IMM.S64 X1, X22, #1 +0x00000000000559d8 (+0x00055788) 021ba800 MOV_XD_XN.S64 X13, X26 +0x00000000000559dc (+0x0005578c) 00c3b089 SEL.B64 X1, X27, X1 +0x00000000000559e0 (+0x00055790) 00c54689 SEL.B64 X2, X20, X13 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x00000000000559e4 (+0x00055794) 02160900 MOV_SPR_XN.S64 CONDITION_FLAG, X0 +# [DWARF] common/pa_trace.h:276 +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x00000000000559e8 (+0x00055798) 00da2689 SEL.B64 X13, X2, X13 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000559ec (+0x0005579c) 07200690 MOV_XD_IMM X16, #1680 +0x00000000000559f0 (+0x000557a0) 07040000 MOV_XD_IMM X2, #0 +0x00000000000559f4 (+0x000557a4) 07060001 MOV_XD_IMM X3, #1 +# [DWARF] common/pa_trace.h:276 +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x00000000000559f8 (+0x000557a8) 00ec1b09 SEL.B64 X22, X1, X22 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000559fc (+0x000557ac) 0021e801 ADD.S64 X16, X30, X16 +0x0000000000055a00 (+0x000557b0) 07450100 MOVK X2, #256, #1 +# [DWARF] common/pa_trace.h:283 +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# 278 | trace.atomic_counter_overflow = true; +# 279 | } else { +# 280 | ++trace.poll_batch_records; +# 281 | } +# 282 | } +# > 283 | trace.poll_burst.call_count[index] = 0; +0x0000000000055a04 (+0x000557b4) 0e810e00 STI_XN_XM.B32 X16, X28 +0x0000000000055a08 (+0x000557b8) 0747ff00 MOVK X3, #65280, #1 +# [DWARF] common/pa_trace.h:262 +# 256 | #else +# 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +# 258 | const uint32_t active_mask = trace.poll_burst.active_mask; +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# > 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +0x0000000000055a0c (+0x000557bc) 0839c001 ADD_IMM.S64 X28, X28, #1 +0x0000000000055a10 (+0x000557c0) 08273004 ADD_IMM.S64 X19, X19, #4 +0x0000000000055a14 (+0x000557c4) 0001ca8e CMP.S64.EQ X28, X21 +0x0000000000055a18 (+0x000557c8) 08339001 ADD_IMM.S64 X25, X25, #1 +0x0000000000055a1c (+0x000557cc) 40200017 JUMPC #23 +# [DWARF] common/pa_trace.h:263 +# > 263 | const uint32_t bit = 1U << index; +0x0000000000055a20 (+0x000557d0) 02819a00 ZEROEXT.U32 X0, X25 +# [DWARF] common/pa_trace.h:264 +# > 264 | if ((active_mask & bit) == 0) continue; +0x0000000000055a24 (+0x000557d4) 02838a00 ZEROEXT.U32 X1, X24 +0x0000000000055a28 (+0x000557d8) 024202c0 SHR.U64 X1, X0, #0 +0x0000000000055a2c (+0x000557dc) 00c01a0a AND.B64 X0, X1, X20 +0x0000000000055a30 (+0x000557e0) 0000090e CMP.S64.EQ X0, X18 +0x0000000000055a34 (+0x000557e4) 4020fff6 JUMPC #65526 +# [DWARF] common/pa_trace.h:265 +# > 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +0x0000000000055a38 (+0x000557e8) 018b0e00 LD_XD_XN.B32 X5, X16, X28 +# [DWARF] common/pa_trace.h:266 +# > 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +0x0000000000055a3c (+0x000557ec) 00005102 SUB.S64 X0, X5, X2 +0x0000000000055a40 (+0x000557f0) 02800a00 ZEROEXT.U32 X0, X0 +0x0000000000055a44 (+0x000557f4) 004001ae CMP.U64.LT X0, X3 +0x0000000000055a48 (+0x000557f8) 40200002 JUMPC #2 +0x0000000000055a4c (+0x000557fc) 40000003 JUMP #3 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055a50 (+0x00055800) 071a0001 MOV_XD_IMM X13, #1 +0x0000000000055a54 (+0x00055804) 4000ffee JUMP #65518 +# [DWARF] common/pa_trace.h:98 +# 92 | default: +# 93 | return -1; +# 94 | } +# 95 | } +# 96 | +# 97 | PA_DEVICE AtomicSite TraceAtomicPollBatchSite(uint32_t index) { +# > 98 | switch (index) { +0x0000000000055a58 (+0x00055808) 02819a00 ZEROEXT.U32 X0, X25 +0x0000000000055a5c (+0x0005580c) 07020005 MOV_XD_IMM X1, #5 +0x0000000000055a60 (+0x00055810) 0234d800 MOV_XD_XN.S64 X26, X13 +0x0000000000055a64 (+0x00055814) 004000be CMP.U64.GT X0, X1 +0x0000000000055a68 (+0x00055818) 070c000f MOV_XD_IMM X6, #15 +0x0000000000055a6c (+0x0005581c) 4020ffce JUMPC #65486 +0x0000000000055a70 (+0x00055820) 1c8d3000 LD_XD_XN_IMM.B32 X6, X19, #0 +0x0000000000055a74 (+0x00055824) 4000ffcc JUMP #65484 +# [DWARF] common/pa_scheduler_core.h:1354 +# 1348 | if (WatchdogExpired(state, stats, start_wait, start_polls)) { +# 1349 | break; +# 1350 | } +# 1351 | } +# 1352 | AtomicPollRegionEnd(stats.trace, stats.result, startup_poll_region); +# 1353 | +# > 1354 | const uint32_t batches = state->config.batches; +0x0000000000055a78 (+0x00055828) 1cc1eaf0 LD_XD_XN_IMM.B64 X0, X30, #2800 +0x0000000000055a7c (+0x0005582c) 07040000 MOV_XD_IMM X2, #0 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x0000000000055a80 (+0x00055830) 1ce3eb10 LD_XD_XN_IMM.B64 X17, X30, #2832 +0x0000000000055a84 (+0x00055834) 07020000 MOV_XD_IMM X1, #0 +0x0000000000055a88 (+0x00055838) 1cdfeb30 LD_XD_XN_IMM.B64 X15, X30, #2864 +# [DWARF] common/pa_trace.h:393 +# 387 | (void)result; +# 388 | (void)task_id; +# 389 | (void)site; +# 390 | (void)result_used; +# 391 | return Ops::Load(address); +# 392 | #else +# > 393 | if (!trace.atomics_enabled) return Ops::Load(address); +0x0000000000055a8c (+0x0005583c) 03c3eab0 ST_XD_XN_IMM.B64 X1, X30, #2736 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055a90 (+0x00055840) 1ce7eb00 LD_XD_XN_IMM.B64 X19, X30, #2816 +0x0000000000055a94 (+0x00055844) 1ccfeac8 LD_XD_XN_IMM.B64 X7, X30, #2760 +# [DWARF] common/pa_scheduler_core.h:1354 +# 1348 | if (WatchdogExpired(state, stats, start_wait, start_polls)) { +# 1349 | break; +# 1350 | } +# 1351 | } +# 1352 | AtomicPollRegionEnd(stats.trace, stats.result, startup_poll_region); +# 1353 | +# > 1354 | const uint32_t batches = state->config.batches; +0x0000000000055a98 (+0x00055848) 1ca40000 LD_XD_XN_IMM.B32 X18, X0, #0 +# [DWARF] common/pa_trace.h:393 +# 387 | (void)result; +# 388 | (void)task_id; +# 389 | (void)site; +# 390 | (void)result_used; +# 391 | return Ops::Load(address); +# 392 | #else +# > 393 | if (!trace.atomics_enabled) return Ops::Load(address); +0x0000000000055a9c (+0x0005584c) 1cc1eb48 LD_XD_XN_IMM.B64 X0, X30, #2888 +0x0000000000055aa0 (+0x00055850) 0000010e CMP.S64.EQ X0, X2 +# [DWARF] common/pa_scheduler_core.h:1355 +# 1349 | break; +# 1350 | } +# 1351 | } +# 1352 | AtomicPollRegionEnd(stats.trace, stats.result, startup_poll_region); +# 1353 | +# 1354 | const uint32_t batches = state->config.batches; +# > 1355 | const uint32_t task_count = batches * kTasksPerBatch; +0x0000000000055aa4 (+0x00055854) 08412005 MUL_IMM.S64 X0, X18, #5 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x0000000000055aa8 (+0x00055858) 03c1ea70 ST_XD_XN_IMM.B64 X0, X30, #2672 +# [DWARF] common/pa_trace.h:393 +# 387 | (void)result; +# 388 | (void)task_id; +# 389 | (void)site; +# 390 | (void)result_used; +# 391 | return Ops::Load(address); +# 392 | #else +# > 393 | if (!trace.atomics_enabled) return Ops::Load(address); +0x0000000000055aac (+0x0005585c) 4020000d JUMPC #13 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055ab0 (+0x00055860) 07380000 MOV_XD_IMM X28, #0 +0x0000000000055ab4 (+0x00055864) 1ce9eb80 LD_XD_XN_IMM.B64 X20, X30, #2944 +0x0000000000055ab8 (+0x00055868) 02176800 MOV_XD_XN.S64 X11, X22 +0x0000000000055abc (+0x0005586c) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055ac0 (+0x00055870) 0700afdb MOV_XD_IMM X0, #45019 +0x0000000000055ac4 (+0x00055874) 0741fffe MOVK X0, #65534, #1 +0x0000000000055ac8 (+0x00055878) 0781ffff MOVK X0, #65535, #2 +0x0000000000055acc (+0x0005587c) 40020000 JUMP X0, #0 +# [DWARF] common/pa_scheduler_core.h:1354 +# 1348 | if (WatchdogExpired(state, stats, start_wait, start_polls)) { +# 1349 | break; +# 1350 | } +# 1351 | } +# 1352 | AtomicPollRegionEnd(stats.trace, stats.result, startup_poll_region); +# 1353 | +# > 1354 | const uint32_t batches = state->config.batches; +0x0000000000055ad0 (+0x00055880) 1cc1eaf0 LD_XD_XN_IMM.B64 X0, X30, #2800 +0x0000000000055ad4 (+0x00055884) 1ca40000 LD_XD_XN_IMM.B32 X18, X0, #0 +# [DWARF] common/pa_scheduler_core.h:1355 +# > 1355 | const uint32_t task_count = batches * kTasksPerBatch; +0x0000000000055ad8 (+0x00055888) 08412005 MUL_IMM.S64 X0, X18, #5 +# [DWARF] common/pa_trace.h:393 +# 387 | (void)result; +# 388 | (void)task_id; +# 389 | (void)site; +# 390 | (void)result_used; +# 391 | return Ops::Load(address); +# 392 | #else +# > 393 | if (!trace.atomics_enabled) return Ops::Load(address); +0x0000000000055adc (+0x0005588c) 03c1ea70 ST_XD_XN_IMM.B64 X0, X30, #2672 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055ae0 (+0x00055890) 07000000 MOV_XD_IMM X0, #0 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000055ae4 (+0x00055894) 02828880 MOV_XD_SPR.F32 X1, SYS_CNT +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:84 +# 78 | template \ +# 79 | CCE_INTRINSIC[aicore] FTYPE atomicMax(__gm__ FTYPE *base, FTYPE inc) { \ +# 80 | return __builtin_cce_atom_max_G_##SUFFIX(base, inc, (uint32_t)L2Cache); \ +# 81 | } +# 82 | +# 83 | __CCE__ATOM_G_BUILTIN(uint32_t, u32) +# > 84 | __CCE__ATOM_G_BUILTIN(int32_t, s32) +0x0000000000055ae8 (+0x00055898) 03c1eab0 ST_XD_XN_IMM.B64 X0, X30, #2736 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:0 +# [SOURCE unavailable] +0x0000000000055aec (+0x0005589c) 07040000 MOV_XD_IMM X2, #0 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:84 +# 78 | template \ +# 79 | CCE_INTRINSIC[aicore] FTYPE atomicMax(__gm__ FTYPE *base, FTYPE inc) { \ +# 80 | return __builtin_cce_atom_max_G_##SUFFIX(base, inc, (uint32_t)L2Cache); \ +# 81 | } +# 82 | +# 83 | __CCE__ATOM_G_BUILTIN(uint32_t, u32) +# > 84 | __CCE__ATOM_G_BUILTIN(int32_t, s32) +0x0000000000055af0 (+0x000558a0) 50a01c40 ATOM XN, XM, XD, ADD +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:0 +# [SOURCE unavailable] +0x0000000000055af4 (+0x000558a4) 07060001 MOV_XD_IMM X3, #1 +0x0000000000055af8 (+0x000558a8) 02063080 NEG.S64 X3, X3 +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x0000000000055afc (+0x000558ac) 0001318e CMP.S64.EQ X19, X3 +0x0000000000055b00 (+0x000558b0) 08093001 ADD_IMM.S64 X4, X19, #1 +0x0000000000055b04 (+0x000558b4) 00e63209 SEL.B64 X19, X3, X4 +0x0000000000055b08 (+0x000558b8) 07060001 MOV_XD_IMM X3, #1 +0x0000000000055b0c (+0x000558bc) 00da3689 SEL.B64 X13, X3, X13 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055b10 (+0x000558c0) 02176800 MOV_XD_XN.S64 X11, X22 +# [DWARF] ccec/ccec_ops.h:239 +# 233 | static_assert(sizeof(T) == 4 || sizeof(T) == 8, "atomic dependency expects a scalar result"); +# 234 | uint64_t cycle = 0; +# 235 | // 同一个 inline asm 块先真正消费 atomic 返回寄存器，再读取 +# 236 | // SYS_CNT；编译器不能把 t1 拆到依赖 MOV 之前。AIC/AIV 对该序列 +# 237 | // 生成相同指令字节，且不增加 DSB/ISB/GM 访存。该边界仍只表示 +# 238 | // 返回值已可被本核 scalar 消费，不表示跨核全局可见。 +# > 239 | asm volatile( +0x0000000000055b14 (+0x000558c4) 02040800 MOV_XD_XN.S64 X2, X0 +0x0000000000055b18 (+0x000558c8) 02042800 MOV_XD_XN.S64 X2, X2 +0x0000000000055b1c (+0x000558cc) 02868880 MOV_XD_SPR.F32 X3, SYS_CNT +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x0000000000055b20 (+0x000558d0) 1ce9eb80 LD_XD_XN_IMM.B64 X20, X30, #2944 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000055b24 (+0x000558d4) 1cc5eb40 LD_XD_XN_IMM.B64 X2, X30, #2880 +0x0000000000055b28 (+0x000558d8) 02162900 MOV_SPR_XN.S64 CONDITION_FLAG, X2 +0x0000000000055b2c (+0x000558dc) 40200002 JUMPC #2 +0x0000000000055b30 (+0x000558e0) 40000007 JUMP #7 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055b34 (+0x000558e4) 07380000 MOV_XD_IMM X28, #0 +0x0000000000055b38 (+0x000558e8) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055b3c (+0x000558ec) 0700afc0 MOV_XD_IMM X0, #44992 +0x0000000000055b40 (+0x000558f0) 0741fffe MOVK X0, #65534, #1 +0x0000000000055b44 (+0x000558f4) 0781ffff MOVK X0, #65535, #2 +0x0000000000055b48 (+0x000558f8) 40020000 JUMP X0, #0 +# [DWARF] common/pa_trace.h:547 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# > 547 | const uint32_t slot = core.count; +0x0000000000055b4c (+0x000558fc) 1c854000 LD_XD_XN_IMM.B32 X2, X20, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055b50 (+0x00055900) 1cddeb98 LD_XD_XN_IMM.B64 X14, X30, #2968 +# [DWARF] common/pa_trace.h:548 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# > 548 | if (slot >= trace.capacity) { +0x0000000000055b54 (+0x00055904) 0040272e CMP.U64.LT X2, X14 +0x0000000000055b58 (+0x00055908) 40200002 JUMPC #2 +0x0000000000055b5c (+0x0005590c) 40000018 JUMP #24 +# [DWARF] common/pa_trace.h:552 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x0000000000055b60 (+0x00055910) 1ccdeb90 LD_XD_XN_IMM.B64 X6, X30, #2960 +0x0000000000055b64 (+0x00055914) 020a2800 MOV_XD_XN.S64 X5, X2 +0x0000000000055b68 (+0x00055918) 02ca0206 SHL.B64 X5, #6 +0x0000000000055b6c (+0x0005591c) 02880a00 ZEROEXT.U32 X4, X0 +0x0000000000055b70 (+0x00055920) 000a6281 ADD.S64 X5, X6, X5 +# [DWARF] common/pa_trace.h:553 +# > 553 | record.start_cycle = start_cycle; +0x0000000000055b74 (+0x00055924) 09c25181 STP_XI_XJ_XN.B64 X1, X3, X5, #0 +0x0000000000055b78 (+0x00055928) 07020001 MOV_XD_IMM X1, #1 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055b7c (+0x0005592c) 070c0000 MOV_XD_IMM X6, #0 +0x0000000000055b80 (+0x00055930) 02021080 NEG.S64 X1, X1 +0x0000000000055b84 (+0x00055934) 0706000e MOV_XD_IMM X3, #14 +0x0000000000055b88 (+0x00055938) 0000430e CMP.S64.EQ X4, X6 +# [DWARF] common/pa_trace.h:555 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x0000000000055b8c (+0x0005593c) 09c251a1 STP_XI_XJ_XN.B64 X1, X3, X5, #16 +0x0000000000055b90 (+0x00055940) 07020050 MOV_XD_IMM X1, #80 +0x0000000000055b94 (+0x00055944) 07060070 MOV_XD_IMM X3, #112 +# [DWARF] common/pa_trace.h:559 +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x0000000000055b98 (+0x00055948) 08085020 ADD_IMM.S64 X4, X5, #32 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055b9c (+0x0005594c) 00c23089 SEL.B64 X1, X3, X1 +# [DWARF] common/pa_trace.h:559 +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x0000000000055ba0 (+0x00055950) 099e4781 STP_XI_XJ_XN.B32 X15, X15, X4, #0 +0x0000000000055ba4 (+0x00055954) 07060002 MOV_XD_IMM X3, #2 +# [DWARF] common/pa_trace.h:561 +# 560 | record.core_idx = trace.core_idx; +# > 561 | record.flags = flags; +0x0000000000055ba8 (+0x00055958) 080a5028 ADD_IMM.S64 X5, X5, #40 +0x0000000000055bac (+0x0005595c) 09825181 STP_XI_XJ_XN.B32 X1, X3, X5, #0 +# [DWARF] common/pa_trace.h:565 +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x0000000000055bb0 (+0x00055960) 08022001 ADD_IMM.S64 X1, X2, #1 +0x0000000000055bb4 (+0x00055964) 03834000 ST_XD_XN_IMM.B32 X1, X20, #0 +0x0000000000055bb8 (+0x00055968) 40000004 JUMP #4 +# [DWARF] common/pa_trace.h:549 +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# > 549 | core.dropped = core.dropped + 1; +0x0000000000055bbc (+0x0005596c) 1c834004 LD_XD_XN_IMM.B32 X1, X20, #4 +0x0000000000055bc0 (+0x00055970) 08021001 ADD_IMM.S64 X1, X1, #1 +0x0000000000055bc4 (+0x00055974) 03834004 ST_XD_XN_IMM.B32 X1, X20, #4 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055bc8 (+0x00055978) 07380000 MOV_XD_IMM X28, #0 +0x0000000000055bcc (+0x0005597c) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055bd0 (+0x00055980) 0700af9e MOV_XD_IMM X0, #44958 +0x0000000000055bd4 (+0x00055984) 0741fffe MOVK X0, #65534, #1 +0x0000000000055bd8 (+0x00055988) 0781ffff MOVK X0, #65535, #2 +0x0000000000055bdc (+0x0005598c) 40020000 JUMP X0, #0 +0x0000000000055be0 (+0x00055990) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055be4 (+0x00055994) 03c1ea00 ST_XD_XN_IMM.B64 X0, X30, #2560 +0x0000000000055be8 (+0x00055998) 07060001 MOV_XD_IMM X3, #1 +0x0000000000055bec (+0x0005599c) 03c5ea10 ST_XD_XN_IMM.B64 X2, X30, #2576 +0x0000000000055bf0 (+0x000559a0) 07040001 MOV_XD_IMM X2, #1 +0x0000000000055bf4 (+0x000559a4) 03c3ea18 ST_XD_XN_IMM.B64 X1, X30, #2584 +0x0000000000055bf8 (+0x000559a8) 02063080 NEG.S64 X3, X3 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000055bfc (+0x000559ac) 02808880 MOV_XD_SPR.F32 X0, SYS_CNT +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:45 +# 39 | CCE_INTRINSIC[aicore] FTYPE atomicSub(__gm__ FTYPE *base, FTYPE inc) { \ +# 40 | return __builtin_cce_atom_add_G_##ADD_SUFFIX(base, -inc, \ +# 41 | (uint32_t)L2Cache); \ +# 42 | } +# 43 | +# 44 | __CCE__ATOM_G_CAS_EXCH_INT32(uint32_t, u32, u32) +# > 45 | __CCE__ATOM_G_CAS_EXCH_INT32(int32_t, s32, s32) +0x0000000000055c00 (+0x000559b0) 1cc3eac8 LD_XD_XN_IMM.B64 X1, X30, #2760 +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x0000000000055c04 (+0x000559b4) 0001318e CMP.S64.EQ X19, X3 +0x0000000000055c08 (+0x000559b8) 08093001 ADD_IMM.S64 X4, X19, #1 +0x0000000000055c0c (+0x000559bc) 00e63209 SEL.B64 X19, X3, X4 +0x0000000000055c10 (+0x000559c0) 00da2689 SEL.B64 X13, X2, X13 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000055c14 (+0x000559c4) 02175900 MOV_SPR_XN.S64 CONDITION_FLAG, X21 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:45 +# 39 | CCE_INTRINSIC[aicore] FTYPE atomicSub(__gm__ FTYPE *base, FTYPE inc) { \ +# 40 | return __builtin_cce_atom_add_G_##ADD_SUFFIX(base, -inc, \ +# 41 | (uint32_t)L2Cache); \ +# 42 | } +# 43 | +# 44 | __CCE__ATOM_G_CAS_EXCH_INT32(uint32_t, u32, u32) +# > 45 | __CCE__ATOM_G_CAS_EXCH_INT32(int32_t, s32, s32) +0x0000000000055c18 (+0x000559c8) 50108440 ATOM XN, XM, XD, EXCH +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000055c1c (+0x000559cc) 02828880 MOV_XD_SPR.F32 X1, SYS_CNT +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x0000000000055c20 (+0x000559d0) 1ce9eb80 LD_XD_XN_IMM.B64 X20, X30, #2944 +0x0000000000055c24 (+0x000559d4) 1cddeb98 LD_XD_XN_IMM.B64 X14, X30, #2968 +0x0000000000055c28 (+0x000559d8) 1cf5eb58 LD_XD_XN_IMM.B64 X26, X30, #2904 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000055c2c (+0x000559dc) 40200002 JUMPC #2 +0x0000000000055c30 (+0x000559e0) 4000006e JUMP #110 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055c34 (+0x000559e4) 1cc1ea30 LD_XD_XN_IMM.B64 X0, X30, #2608 +0x0000000000055c38 (+0x000559e8) 02120800 MOV_XD_XN.S64 X9, X0 +0x0000000000055c3c (+0x000559ec) 03c1ea40 ST_XD_XN_IMM.B64 X0, X30, #2624 +0x0000000000055c40 (+0x000559f0) 40000002 JUMP #2 +0x0000000000055c44 (+0x000559f4) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055c48 (+0x000559f8) 1cc1e950 LD_XD_XN_IMM.B64 X0, X30, #2384 +0x0000000000055c4c (+0x000559fc) 02398800 MOV_XD_XN.S64 X28, X24 +0x0000000000055c50 (+0x00055a00) 03c1eab0 ST_XD_XN_IMM.B64 X0, X30, #2736 +0x0000000000055c54 (+0x00055a04) 40000002 JUMP #2 +0x0000000000055c58 (+0x00055a08) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055c5c (+0x00055a0c) 070c0000 MOV_XD_IMM X6, #0 +0x0000000000055c60 (+0x00055a10) 400000c3 JUMP #195 +0x0000000000055c64 (+0x00055a14) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_trace.h:547 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# > 547 | const uint32_t slot = core.count; +0x0000000000055c68 (+0x00055a18) 1c854000 LD_XD_XN_IMM.B32 X2, X20, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055c6c (+0x00055a1c) 070c0000 MOV_XD_IMM X6, #0 +# [DWARF] common/pa_trace.h:548 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# > 548 | if (slot >= trace.capacity) { +0x0000000000055c70 (+0x00055a20) 0040272e CMP.U64.LT X2, X14 +0x0000000000055c74 (+0x00055a24) 40200002 JUMPC #2 +0x0000000000055c78 (+0x00055a28) 40000079 JUMP #121 +# [DWARF] common/pa_trace.h:552 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x0000000000055c7c (+0x00055a2c) 1cc9eb90 LD_XD_XN_IMM.B64 X4, X30, #2960 +0x0000000000055c80 (+0x00055a30) 02062800 MOV_XD_XN.S64 X3, X2 +0x0000000000055c84 (+0x00055a34) 03cbea00 ST_XD_XN_IMM.B64 X5, X30, #2560 +0x0000000000055c88 (+0x00055a38) 02c60206 SHL.B64 X3, #6 +0x0000000000055c8c (+0x00055a3c) 1cd3ea30 LD_XD_XN_IMM.B64 X9, X30, #2608 +0x0000000000055c90 (+0x00055a40) 00064181 ADD.S64 X3, X4, X3 +# [DWARF] common/pa_trace.h:555 +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x0000000000055c94 (+0x00055a44) 1cc9eaa8 LD_XD_XN_IMM.B64 X4, X30, #2728 +# [DWARF] common/pa_trace.h:553 +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# > 553 | record.start_cycle = start_cycle; +0x0000000000055c98 (+0x00055a48) 09c03081 STP_XI_XJ_XN.B64 X0, X1, X3, #0 +0x0000000000055c9c (+0x00055a4c) 0700ffff MOV_XD_IMM X0, #65535 +0x0000000000055ca0 (+0x00055a50) 0741ffff MOVK X0, #65535, #1 +0x0000000000055ca4 (+0x00055a54) 0702000e MOV_XD_IMM X1, #14 +# [DWARF] common/pa_trace.h:555 +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x0000000000055ca8 (+0x00055a58) 09883021 STP_XI_XJ_XN.B32 X4, X0, X3, #16 +0x0000000000055cac (+0x00055a5c) 07000001 MOV_XD_IMM X0, #1 +# [DWARF] common/pa_trace.h:557 +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x0000000000055cb0 (+0x00055a60) 03c23018 ST_XD_XN_IMM.B64 X1, X3, #24 +# [DWARF] common/pa_trace.h:565 +# 558 | record.lane = trace.lane; +# 559 | record.block_id = trace.block_id; +# 560 | record.core_idx = trace.core_idx; +# 561 | record.flags = flags; +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x0000000000055cb4 (+0x00055a64) 08022001 ADD_IMM.S64 X1, X2, #1 +0x0000000000055cb8 (+0x00055a68) 1cc5eb30 LD_XD_XN_IMM.B64 X2, X30, #2864 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055cbc (+0x00055a6c) 07810003 MOVK X0, #3, #2 +# [DWARF] common/pa_trace.h:559 +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x0000000000055cc0 (+0x00055a70) 08083020 ADD_IMM.S64 X4, X3, #32 +0x0000000000055cc4 (+0x00055a74) 09844101 STP_XI_XJ_XN.B32 X2, X2, X4, #0 +0x0000000000055cc8 (+0x00055a78) 40000061 JUMP #97 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055ccc (+0x00055a7c) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000055cd0 (+0x00055a80) 03c1ea18 ST_XD_XN_IMM.B64 X0, X30, #2584 +0x0000000000055cd4 (+0x00055a84) 02102800 MOV_XD_XN.S64 X8, X2 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000055cd8 (+0x00055a88) 02808880 MOV_XD_SPR.F32 X0, SYS_CNT +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:45 +# 39 | CCE_INTRINSIC[aicore] FTYPE atomicSub(__gm__ FTYPE *base, FTYPE inc) { \ +# 40 | return __builtin_cce_atom_add_G_##ADD_SUFFIX(base, -inc, \ +# 41 | (uint32_t)L2Cache); \ +# 42 | } +# 43 | +# 44 | __CCE__ATOM_G_CAS_EXCH_INT32(uint32_t, u32, u32) +# > 45 | __CCE__ATOM_G_CAS_EXCH_INT32(int32_t, s32, s32) +0x0000000000055cdc (+0x00055a8c) 1cc3eac8 LD_XD_XN_IMM.B64 X1, X30, #2760 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:0 +# [SOURCE unavailable] +0x0000000000055ce0 (+0x00055a90) 07040001 MOV_XD_IMM X2, #1 +0x0000000000055ce4 (+0x00055a94) 07060001 MOV_XD_IMM X3, #1 +0x0000000000055ce8 (+0x00055a98) 02063080 NEG.S64 X3, X3 +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x0000000000055cec (+0x00055a9c) 0001318e CMP.S64.EQ X19, X3 +0x0000000000055cf0 (+0x00055aa0) 08093001 ADD_IMM.S64 X4, X19, #1 +0x0000000000055cf4 (+0x00055aa4) 00e63209 SEL.B64 X19, X3, X4 +0x0000000000055cf8 (+0x00055aa8) 00da2689 SEL.B64 X13, X2, X13 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000055cfc (+0x00055aac) 02175900 MOV_SPR_XN.S64 CONDITION_FLAG, X21 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:45 +# 39 | CCE_INTRINSIC[aicore] FTYPE atomicSub(__gm__ FTYPE *base, FTYPE inc) { \ +# 40 | return __builtin_cce_atom_add_G_##ADD_SUFFIX(base, -inc, \ +# 41 | (uint32_t)L2Cache); \ +# 42 | } +# 43 | +# 44 | __CCE__ATOM_G_CAS_EXCH_INT32(uint32_t, u32, u32) +# > 45 | __CCE__ATOM_G_CAS_EXCH_INT32(int32_t, s32, s32) +0x0000000000055d00 (+0x00055ab0) 50108440 ATOM XN, XM, XD, EXCH +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000055d04 (+0x00055ab4) 02828880 MOV_XD_SPR.F32 X1, SYS_CNT +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x0000000000055d08 (+0x00055ab8) 1ce9eb80 LD_XD_XN_IMM.B64 X20, X30, #2944 +0x0000000000055d0c (+0x00055abc) 1cddeb98 LD_XD_XN_IMM.B64 X14, X30, #2968 +0x0000000000055d10 (+0x00055ac0) 1cf5eb58 LD_XD_XN_IMM.B64 X26, X30, #2904 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000055d14 (+0x00055ac4) 40200002 JUMPC #2 +0x0000000000055d18 (+0x00055ac8) 40000056 JUMP #86 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055d1c (+0x00055acc) 1cc1ea30 LD_XD_XN_IMM.B64 X0, X30, #2608 +0x0000000000055d20 (+0x00055ad0) 03d1ea10 ST_XD_XN_IMM.B64 X8, X30, #2576 +0x0000000000055d24 (+0x00055ad4) 1cf9eb38 LD_XD_XN_IMM.B64 X28, X30, #2872 +0x0000000000055d28 (+0x00055ad8) 02120800 MOV_XD_XN.S64 X9, X0 +0x0000000000055d2c (+0x00055adc) 03c1ea40 ST_XD_XN_IMM.B64 X0, X30, #2624 +0x0000000000055d30 (+0x00055ae0) 1cc1e948 LD_XD_XN_IMM.B64 X0, X30, #2376 +0x0000000000055d34 (+0x00055ae4) 03c1ea00 ST_XD_XN_IMM.B64 X0, X30, #2560 +0x0000000000055d38 (+0x00055ae8) 1cc1e930 LD_XD_XN_IMM.B64 X0, X30, #2352 +0x0000000000055d3c (+0x00055aec) 03c1ea08 ST_XD_XN_IMM.B64 X0, X30, #2568 +0x0000000000055d40 (+0x00055af0) 1cc1e918 LD_XD_XN_IMM.B64 X0, X30, #2328 +0x0000000000055d44 (+0x00055af4) 4000ffc3 JUMP #65475 +0x0000000000055d48 (+0x00055af8) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_trace.h:547 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# > 547 | const uint32_t slot = core.count; +0x0000000000055d4c (+0x00055afc) 1c854000 LD_XD_XN_IMM.B32 X2, X20, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055d50 (+0x00055b00) 070c0000 MOV_XD_IMM X6, #0 +# [DWARF] common/pa_trace.h:548 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# > 548 | if (slot >= trace.capacity) { +0x0000000000055d54 (+0x00055b04) 0040272e CMP.U64.LT X2, X14 +0x0000000000055d58 (+0x00055b08) 40200002 JUMPC #2 +0x0000000000055d5c (+0x00055b0c) 4000006e JUMP #110 +# [DWARF] common/pa_trace.h:552 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x0000000000055d60 (+0x00055b10) 1cc9eb90 LD_XD_XN_IMM.B64 X4, X30, #2960 +0x0000000000055d64 (+0x00055b14) 02062800 MOV_XD_XN.S64 X3, X2 +0x0000000000055d68 (+0x00055b18) 03cbea00 ST_XD_XN_IMM.B64 X5, X30, #2560 +0x0000000000055d6c (+0x00055b1c) 02c60206 SHL.B64 X3, #6 +0x0000000000055d70 (+0x00055b20) 02398800 MOV_XD_XN.S64 X28, X24 +0x0000000000055d74 (+0x00055b24) 00064181 ADD.S64 X3, X4, X3 +# [DWARF] common/pa_trace.h:555 +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x0000000000055d78 (+0x00055b28) 1cc9eaa8 LD_XD_XN_IMM.B64 X4, X30, #2728 +# [DWARF] common/pa_trace.h:553 +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# > 553 | record.start_cycle = start_cycle; +0x0000000000055d7c (+0x00055b2c) 09c03081 STP_XI_XJ_XN.B64 X0, X1, X3, #0 +0x0000000000055d80 (+0x00055b30) 0700ffff MOV_XD_IMM X0, #65535 +0x0000000000055d84 (+0x00055b34) 0741ffff MOVK X0, #65535, #1 +0x0000000000055d88 (+0x00055b38) 0702000e MOV_XD_IMM X1, #14 +# [DWARF] common/pa_trace.h:559 +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x0000000000055d8c (+0x00055b3c) 080e3020 ADD_IMM.S64 X7, X3, #32 +# [DWARF] common/pa_trace.h:555 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x0000000000055d90 (+0x00055b40) 09883021 STP_XI_XJ_XN.B32 X4, X0, X3, #16 +0x0000000000055d94 (+0x00055b44) 1cc9eb30 LD_XD_XN_IMM.B64 X4, X30, #2864 +0x0000000000055d98 (+0x00055b48) 07000001 MOV_XD_IMM X0, #1 +# [DWARF] common/pa_trace.h:557 +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x0000000000055d9c (+0x00055b4c) 03c23018 ST_XD_XN_IMM.B64 X1, X3, #24 +# [DWARF] common/pa_trace.h:565 +# 558 | record.lane = trace.lane; +# 559 | record.block_id = trace.block_id; +# 560 | record.core_idx = trace.core_idx; +# 561 | record.flags = flags; +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x0000000000055da0 (+0x00055b50) 08022001 ADD_IMM.S64 X1, X2, #1 +0x0000000000055da4 (+0x00055b54) 1cc5ea30 LD_XD_XN_IMM.B64 X2, X30, #2608 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055da8 (+0x00055b58) 07810003 MOVK X0, #3, #2 +# [DWARF] common/pa_trace.h:559 +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x0000000000055dac (+0x00055b5c) 09887201 STP_XI_XJ_XN.B32 X4, X4, X7, #0 +0x0000000000055db0 (+0x00055b60) 02122800 MOV_XD_XN.S64 X9, X2 +0x0000000000055db4 (+0x00055b64) 03c5ea40 ST_XD_XN_IMM.B64 X2, X30, #2624 +# [DWARF] common/pa_trace.h:561 +# 560 | record.core_idx = trace.core_idx; +# > 561 | record.flags = flags; +0x0000000000055db8 (+0x00055b68) 03c03028 ST_XD_XN_IMM.B64 X0, X3, #40 +# [DWARF] common/pa_trace.h:565 +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x0000000000055dbc (+0x00055b6c) 03834000 ST_XD_XN_IMM.B32 X1, X20, #0 +0x0000000000055dc0 (+0x00055b70) 4000006b JUMP #107 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000055dc4 (+0x00055b74) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_trace.h:549 +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# > 549 | core.dropped = core.dropped + 1; +0x0000000000055dc8 (+0x00055b78) 1c814004 LD_XD_XN_IMM.B32 X0, X20, #4 +0x0000000000055dcc (+0x00055b7c) 03cfea18 ST_XD_XN_IMM.B64 X7, X30, #2584 diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/published.sha256 new file mode 100644 index 0000000000..71ba890025 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/published.sha256 @@ -0,0 +1,27 @@ +7b364d19466391c4048afc9492dbb8bea3665617b03c639fb26cbb5f90c45036 annotated/00_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz +59c9042ea410ea8c4e1e62da7fd197a95520e3bd3e102b4d9fd6411c430c7b5e annotated/01_pa_execute_real_winner_workload_aic_729699de.source.asm.gz +f86aa6d3ab5dc84aa897e8d92401f33f15bae8281ded890a4931b6a525637950 annotated/02_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz +4ce297449c240be11b6aa00e125309eed2ad22d1ffdd8a8885d49794b1dec607 annotated/03_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +aee6acc662edb138e7fa40e762d1c340038114c8b47643e7d5a7514b60deb149 annotated/04_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz +60211d286d798c524b76eaedec97a407d04ab330241a119ac53c87a198f8c952 annotated/05_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz +d71f1cebcabe69d49e515faa3b4f47bf3254233cc63e3c36561b92aa5e310362 annotated/06_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz +e480aae49727c8463471bcb3e92c8db8c75f37792437fe587668b08d5531e9b5 annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz +bf66fc8e3a91bf6efa3b751a087af4c0728810566e85b13c690ad3841d2411f2 annotated/08_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +e9ec66d6f2729262f19deed5afd0eaf4c321b893987ad5a8e7b5de9050de0054 annotated/09_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz +e88e6a49d249eb7f00de9353c70af43f023e3a721495b7d724935c38da462254 annotated/10_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz +3468940354f71bc3619221a8298e72523f01d052c19210063c02db991b805bd4 key_flow/aic_original_submit.source.asm +934a467bb8aae45fbfb7f13ae26823117645ab82760552aefb706431bacf2f24 raw/00_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz +d801ffe741f89b78386383caf8dce91940dbf62f3983621b043ec1c04bda7933 raw/01_pa_execute_real_winner_workload_aic_729699de.asm.gz +2b536564a5f6cacdc634ecbc4ed1a85eeb93d41290a3033e0b5375d5e2e6b8dc raw/02_pa_scheduler_0_mix_aic_4f68fde0.asm.gz +cc502ce9041184fe05d07c064d19c27204c30dcadb656a82bcda2e4b4d45f3b5 raw/03_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +789b75b419e9d9608b56d07e73479edaa6c221320d235c5bf6aab51ba5035387 raw/04_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz +0f98b856714ff292b423d48fd84e5cdfac96ecca180ba891d1eb7830267619b5 raw/05_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz +1f365c1a8bf10b269bff2095f96d420783ce1d4bd99df542e60659ea18f4a27e raw/06_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz +9ceb3d65e13277114aa9c0de5efc771e452577a46d13e6f74c7c241db58e0c54 raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz +da169141e544c5220e87673d3e829a2c509be23db49de4934e1c96fd1e044018 raw/08_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +f66e5b7f5071add1f390a4f52a10c7bd649a08565477f0f5d1c8bacf0e49fac4 raw/09_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz +bce988226efb1bafb961f4a8ef56ca2c8d269733b9ba38d43ba45b1f4e79e437 raw/10_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz +27b1273b5e461ef004bdfaa743cde304827256e40d265917dcd798b3123eaefd raw/README.md +75fad81fef86cadf69767382cc10bf436529f6f61c0e320c883b63e1e885d047 raw/gaps.tsv +a4a13b0d14a99744153bc9a4baeacd9c61fea73c690c044c2596aa89d0b14370 raw/manifest.tsv +564c95bd7f07ae167ac9c3f17290402cf77b82ae1df973ef6e3c931c9ade4184 raw/published.sha256 diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/00_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/00_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz new file mode 100644 index 0000000000..a78e402c30 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/00_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/01_pa_execute_real_winner_workload_aic_729699de.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/01_pa_execute_real_winner_workload_aic_729699de.asm.gz new file mode 100644 index 0000000000..ad092c27ed Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/01_pa_execute_real_winner_workload_aic_729699de.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/02_pa_scheduler_0_mix_aic_4f68fde0.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/02_pa_scheduler_0_mix_aic_4f68fde0.asm.gz new file mode 100644 index 0000000000..bec52741ee Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/02_pa_scheduler_0_mix_aic_4f68fde0.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/03_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/03_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz new file mode 100644 index 0000000000..5ac2af6b39 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/03_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/04_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/04_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz new file mode 100644 index 0000000000..79df94b89e Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/04_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/05_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/05_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz new file mode 100644 index 0000000000..646e0aa7b0 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/05_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/06_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/06_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz new file mode 100644 index 0000000000..237fafabdc Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/06_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz new file mode 100644 index 0000000000..2014ca169a Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/08_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/08_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz new file mode 100644 index 0000000000..70c1f5dad5 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/08_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/09_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/09_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz new file mode 100644 index 0000000000..60d9a71e41 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/09_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/10_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/10_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz new file mode 100644 index 0000000000..d86361d837 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/10_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/README.md b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/README.md new file mode 100644 index 0000000000..10965df034 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/README.md @@ -0,0 +1,14 @@ +# Final-linked A5 disassembly + +Every non-empty final `.text` `STT_FUNC` symbol was extracted from the published final ELF and decoded with the verified `dav_3510` PEM decoder. +RVec `.vector.thread` symbols are passed through an explicit full-body RVec range; all other symbols use scalar decoding. +Each compressed file records final PC, function-relative offset, machine word, mnemonic, ELF/body hashes and decoder identity. +Function gaps are alignment/padding outside symbol bodies and are hashed in `gaps.tsv`; they are not presented as instructions. + +Decoder SHA256: `29835d2439d6dd464d34a212ad4bbd5c29af6a38465da09a6c273401d9a96dcb` + +| Variant | Final .text | Function symbols | Function bytes | Coverage | Final ELF SHA256 | +| --- | ---: | ---: | ---: | ---: | --- | +| original | 780344 B | 11 | 779700 B | 99.917% | `76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc` | + +Inspect with `zless FILE.asm.gz`; the sibling `annotated/` directory adds DWARF-mapped local source and comments. diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/gaps.tsv b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/gaps.tsv new file mode 100644 index 0000000000..b3ca888d07 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/gaps.tsv @@ -0,0 +1,5 @@ +variant start_pc end_pc size sha256 +original 0x5eb10 0x5ec00 240 e36b7dc9483ace26162b44036c330d044f622b04030d1a9c9a1e3afda891d33c +original 0xbe644 0xbe700 188 e3714d86fb370e0c7492093721f710b7beaa2d279069955ffa604ae303a98b4c +original 0xbe730 0xbe800 208 c9364bd8435644c4dcf160545b42c2cfb80001a10890c7a1cdd1f7a0a28dc8ac +original 0xbe830 0xbe838 8 c573025535b1cea90f421ad7615d65296651dd5bc7287d7dfce7cc79c878ef4b diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/manifest.tsv b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/manifest.tsv new file mode 100644 index 0000000000..17fb041977 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/manifest.tsv @@ -0,0 +1,12 @@ +variant file binding symbol final_pc size instructions decoder_mode body_sha256 last_mnemonic +original 00_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz LOCAL _ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j 0x0 192 48 scalar b8afa7640101b76f5e91a2e172ddd1cb39b6e11ccecd0c4395057eabc1e2bc00 RET +original 01_pa_execute_real_winner_workload_aic_729699de.asm.gz LOCAL pa_execute_real_winner_workload_aic 0xc0 400 100 scalar fe9fe832423d57f5a670d87da542d8e5ffbfbaccfea89dc42aeb067f0326f100 RET +original 02_pa_scheduler_0_mix_aic_4f68fde0.asm.gz GLOBAL pa_scheduler_0_mix_aic 0x250 387044 96761 scalar d7337df3d1786f2236031aaa4d252ca82696e3a563e11ce56d9b98495bbf1aea JUMP +original 03_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz LOCAL _ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3AS1NS_11TraceRecordEjmmjj 0x5ea34 220 55 scalar ffa662e6c38ba766793106b6df796f73dd7e1b7a3117d34b2db1a09fcc689b2e RET +original 04_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz LOCAL _ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j 0x5ec00 268 66 scalar 06ddec8fb7448f56d476215e83f550ada3abfb57f1af95f8701fde1bf6e0982e RET +original 05_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz LOCAL _ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j 0x5ed0c 268 66 scalar 4f646e608218acf17c756b004287a61e254a37809096498026c6deab8f4e5e01 RET +original 06_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz LOCAL pa_execute_real_winner_workload_aiv 0x5ee18 476 119 scalar 158c11e2f546c21643c2addb244ffefe3e46e5aeee5b04fbd4873dcd33a07363 RET +original 07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz GLOBAL pa_scheduler_0_mix_aiv 0x5eff4 390516 97629 scalar 67b4fd4c11780b2730e10a0f1b124481f96b2ea1c548d626ff45b5d67b807833 JUMP +original 08_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz LOCAL _ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3AS1NS_11TraceRecordEjmmjj 0xbe568 220 55 scalar b55f41f8bc8a2afc519eb15c23d4f8fefed134862a7d50c31643803d15c66d8d RET +original 09_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz LOCAL _ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128EEEvPU3AS6T0_S5_S5_jj.vector.thread 0xbe700 48 12 rvec 40cfc6402a5ba38eb3edbe949d63afe52206a592975835aa76b9f028b1d1a67a RV_SEND.U16 +original 10_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz LOCAL _ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128EEEvPU3AS6T0_S5_S5_jj.vector.thread 0xbe800 48 12 rvec 9afa7fa1b6031acff6f90ef3822167d31eeec4d90ee419db0b85dd7c40e5404c RV_SEND.U16 diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/published.sha256 new file mode 100644 index 0000000000..8df95b5f56 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/disassembly/raw/published.sha256 @@ -0,0 +1,14 @@ +934a467bb8aae45fbfb7f13ae26823117645ab82760552aefb706431bacf2f24 00_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz +d801ffe741f89b78386383caf8dce91940dbf62f3983621b043ec1c04bda7933 01_pa_execute_real_winner_workload_aic_729699de.asm.gz +2b536564a5f6cacdc634ecbc4ed1a85eeb93d41290a3033e0b5375d5e2e6b8dc 02_pa_scheduler_0_mix_aic_4f68fde0.asm.gz +cc502ce9041184fe05d07c064d19c27204c30dcadb656a82bcda2e4b4d45f3b5 03_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +789b75b419e9d9608b56d07e73479edaa6c221320d235c5bf6aab51ba5035387 04_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz +0f98b856714ff292b423d48fd84e5cdfac96ecca180ba891d1eb7830267619b5 05_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz +1f365c1a8bf10b269bff2095f96d420783ce1d4bd99df542e60659ea18f4a27e 06_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz +9ceb3d65e13277114aa9c0de5efc771e452577a46d13e6f74c7c241db58e0c54 07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz +da169141e544c5220e87673d3e829a2c509be23db49de4934e1c96fd1e044018 08_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +f66e5b7f5071add1f390a4f52a10c7bd649a08565477f0f5d1c8bacf0e49fac4 09_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz +bce988226efb1bafb961f4a8ef56ca2c8d269733b9ba38d43ba45b1f4e79e437 10_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz +27b1273b5e461ef004bdfaa743cde304827256e40d265917dcd798b3123eaefd README.md +75fad81fef86cadf69767382cc10bf436529f6f61c0e320c883b63e1e885d047 gaps.tsv +a4a13b0d14a99744153bc9a4baeacd9c61fea73c690c044c2596aa89d0b14370 manifest.tsv diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/generate_disassembly.sh b/tests/atomic_probe/lazy_lamda_sample/A_original/generate_disassembly.sh new file mode 100755 index 0000000000..4a98b08e77 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/generate_disassembly.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +RAW_DIR="$SCRIPT_DIR/disassembly/raw" +ANNOTATED_DIR="$SCRIPT_DIR/disassembly/annotated" + +case "$RAW_DIR:$ANNOTATED_DIR" in + "$SCRIPT_DIR/disassembly/raw:$SCRIPT_DIR/disassembly/annotated") ;; + *) echo "Refusing to replace unexpected disassembly paths" >&2; exit 1 ;; +esac +rm -rf -- "$RAW_DIR" "$ANNOTATED_DIR" + +python3 "$SCRIPT_DIR/disassemble.py" --output "$RAW_DIR" +mkdir -p "$ANNOTATED_DIR" +for raw in "$RAW_DIR"/*.asm.gz; do + name="$(basename "$raw" .asm.gz)" + python3 "$SCRIPT_DIR/annotate_disassembly.py" \ + --elf "$SCRIPT_DIR/artifacts/measured/pa_scheduler_kernel.o" \ + --raw "$raw" \ + --source-root "$SCRIPT_DIR" \ + --output "$ANNOTATED_DIR/${name}.source.asm.gz" +done +mkdir -p "$SCRIPT_DIR/disassembly/key_flow" +python3 "$SCRIPT_DIR/annotate_disassembly.py" \ + --elf "$SCRIPT_DIR/artifacts/measured/pa_scheduler_kernel.o" \ + --raw "$RAW_DIR/02_pa_scheduler_0_mix_aic_4f68fde0.asm.gz" \ + --source-root "$SCRIPT_DIR" \ + --output "$SCRIPT_DIR/disassembly/key_flow/aic_original_submit.source.asm" \ + --anchor common/pa_scheduler_core.h:1435 --before 120 --after 600 +( + cd "$SCRIPT_DIR/disassembly" + find raw annotated key_flow -type f -print0 | sort -z | xargs -0 sha256sum +) > "$SCRIPT_DIR/disassembly/published.sha256" diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/functional.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/functional.txt new file mode 100644 index 0000000000..bf5b8805cc --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/functional.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=1 tasks=5 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=73.191 host_launch_us=1079.893 claims=288 fanin_loads=235 cas_retries=0 +[ATOMIC] submit_completion_ops=557 fanin_ready=43 fanin_not_ready=192 frontier_initial=5 frontier_flag=10 frontier_ready_fetch_max=5 frontier_terminal=5 +[WINNERS] active_workers=5 max_wins_per_worker=1 +[PLACEMENT] EfDrain=1 RingBp=0 FinalDrain=3 +[KERNEL] QK count=1 mean_us=42.738 min_us=42.738 max_us=42.738 target_us=44.170 +[KERNEL] SF count=1 mean_us=77.629 min_us=77.629 max_us=77.629 target_us=53.729 +[KERNEL] PV count=1 mean_us=28.522 min_us=28.522 max_us=28.522 target_us=27.626 +[KERNEL] UP count=1 mean_us=2.614 min_us=2.614 max_us=2.614 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=4 inactive_sentinel_tiles=188) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=73.191 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/RUN_METADATA.md b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/RUN_METADATA.md new file mode 100644 index 0000000000..ffa9e14c07 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/RUN_METADATA.md @@ -0,0 +1,17 @@ +# Final b256 run provenance + +- Variant: `original`; +- collection interval: 2026-07-20 07:02:23–07:12:45 UTC; +- direct device0 execution, 96 workers, b256; +- 24 independent host launches and one device run per launch; +- all six A/B/C launch permutations repeated four times; launch positions 1/2/3 are 8/8/8; +- PMU explicitly off, runtime swimlane explicitly off, real-compute; +- executable source: this package's clean `build/original`; +- exact final ELF copy: `artifacts/measured/pa_scheduler_kernel.o`; +- final ELF SHA256: `76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc`; +- host SHA256: `97cf327377857bcab61cbca10357a90bc49cd5c389c919873c2f3209d5b01188`. + +`task-submit` and `npu-smi` were unavailable in the collection shell. The run used the user's explicit +authorization to access device0 directly and was not queue-isolated. All raw logs passed execution, +semantic and postprocess checks. Raw evidence is retained; the primary statistic applies the common +per-variant Hampel rule documented in the root performance comparison. diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/SUMMARY.md b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/SUMMARY.md new file mode 100644 index 0000000000..f4f6a8575f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/SUMMARY.md @@ -0,0 +1,11 @@ +# original device0 b256 result + +24 independent host launches, one device run per launch; PMU off, runtime swimlane off, real-compute. +Raw logs are retained. The primary row excludes samples outside median ± 3×1.4826×MAD. + +| View | N | Median (us) | Mean (us) | Min (us) | Max (us) | Stddev (us) | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Raw | 24 | 3954.845 | 3930.327 | 3788.296 | 4000.058 | 51.730 | +| Outlier-filtered | 22 | 3956.298 | 3941.249 | 3856.072 | 4000.058 | 38.004 | + +Excluded samples: 2. diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_01_ABC_pos1_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_01_ABC_pos1_original.txt new file mode 100644 index 0000000000..2ebf91d085 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_01_ABC_pos1_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3963.375 host_launch_us=4840.717 claims=73728 fanin_loads=31894 cas_retries=0 +[ATOMIC] submit_completion_ops=117260 fanin_ready=3720 fanin_not_ready=28174 frontier_initial=1280 frontier_flag=4027 frontier_ready_fetch_max=2747 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=34 +[PLACEMENT] EfDrain=974 RingBp=25 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.593 min_us=40.738 max_us=45.976 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.318 min_us=52.288 max_us=231.412 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.223 min_us=27.003 max_us=75.355 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.668 min_us=2.324 max_us=16.267 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=190 inactive_sentinel_tiles=2) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3963.375 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_02_BCA_pos3_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_02_BCA_pos3_original.txt new file mode 100644 index 0000000000..bdf924dc5b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_02_BCA_pos3_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3856.072 host_launch_us=4748.418 claims=73728 fanin_loads=38428 cas_retries=0 +[ATOMIC] submit_completion_ops=122614 fanin_ready=3721 fanin_not_ready=34707 frontier_initial=1280 frontier_flag=3437 frontier_ready_fetch_max=2157 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=29 +[PLACEMENT] EfDrain=970 RingBp=36 FinalDrain=18 +[KERNEL] QK count=256 mean_us=41.538 min_us=40.723 max_us=45.640 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.923 min_us=52.382 max_us=130.376 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.834 min_us=27.024 max_us=34.347 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.600 min_us=2.347 max_us=7.522 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=190 inactive_sentinel_tiles=2) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3856.072 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_03_CAB_pos2_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_03_CAB_pos2_original.txt new file mode 100644 index 0000000000..56e5ea032e --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_03_CAB_pos2_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3870.407 host_launch_us=4828.119 claims=73728 fanin_loads=36534 cas_retries=0 +[ATOMIC] submit_completion_ops=120394 fanin_ready=3651 fanin_not_ready=32883 frontier_initial=1280 frontier_flag=3274 frontier_ready_fetch_max=1994 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=979 RingBp=30 FinalDrain=15 +[KERNEL] QK count=256 mean_us=41.406 min_us=40.717 max_us=47.901 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.987 min_us=52.117 max_us=134.530 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.050 min_us=26.979 max_us=90.674 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.570 min_us=2.341 max_us=3.589 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=190 inactive_sentinel_tiles=2) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3870.407 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_04_ACB_pos1_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_04_ACB_pos1_original.txt new file mode 100644 index 0000000000..c8d9e9d428 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_04_ACB_pos1_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3956.964 host_launch_us=4960.990 claims=73728 fanin_loads=38253 cas_retries=0 +[ATOMIC] submit_completion_ops=122301 fanin_ready=3710 fanin_not_ready=34543 frontier_initial=1280 frontier_flag=3368 frontier_ready_fetch_max=2088 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=970 RingBp=38 FinalDrain=16 +[KERNEL] QK count=256 mean_us=41.544 min_us=40.610 max_us=46.055 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.449 min_us=52.351 max_us=243.774 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.415 min_us=27.050 max_us=78.248 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.706 min_us=2.357 max_us=15.334 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=190 inactive_sentinel_tiles=2) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3956.964 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_05_BAC_pos2_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_05_BAC_pos2_original.txt new file mode 100644 index 0000000000..4f3d5bfadf --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_05_BAC_pos2_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3989.566 host_launch_us=5514.869 claims=73728 fanin_loads=39529 cas_retries=0 +[ATOMIC] submit_completion_ops=123619 fanin_ready=3671 fanin_not_ready=35858 frontier_initial=1280 frontier_flag=3389 frontier_ready_fetch_max=2109 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=969 RingBp=37 FinalDrain=18 +[KERNEL] QK count=256 mean_us=41.460 min_us=40.642 max_us=46.729 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.625 min_us=52.514 max_us=240.514 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.799 min_us=26.998 max_us=31.437 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.608 min_us=2.330 max_us=9.925 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=188 inactive_sentinel_tiles=4) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3989.566 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_06_CBA_pos3_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_06_CBA_pos3_original.txt new file mode 100644 index 0000000000..cbd3b86a64 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_06_CBA_pos3_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=4000.058 host_launch_us=4909.120 claims=73728 fanin_loads=34543 cas_retries=0 +[ATOMIC] submit_completion_ops=118481 fanin_ready=3641 fanin_not_ready=30902 frontier_initial=1280 frontier_flag=3313 frontier_ready_fetch_max=2033 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=977 RingBp=23 FinalDrain=24 +[KERNEL] QK count=256 mean_us=41.508 min_us=40.589 max_us=46.151 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.575 min_us=52.114 max_us=259.746 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.119 min_us=26.953 max_us=69.870 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.590 min_us=2.389 max_us=3.434 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=4000.058 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_07_CBA_pos3_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_07_CBA_pos3_original.txt new file mode 100644 index 0000000000..eeeacd3740 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_07_CBA_pos3_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3930.496 host_launch_us=4874.213 claims=73728 fanin_loads=31233 cas_retries=0 +[ATOMIC] submit_completion_ops=116577 fanin_ready=3753 fanin_not_ready=27480 frontier_initial=1280 frontier_flag=4016 frontier_ready_fetch_max=2736 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=33 +[PLACEMENT] EfDrain=974 RingBp=26 FinalDrain=24 +[KERNEL] QK count=256 mean_us=41.577 min_us=40.675 max_us=49.448 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.223 min_us=52.392 max_us=174.776 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.483 min_us=26.977 max_us=80.250 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.704 min_us=2.347 max_us=21.410 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3930.496 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_08_BAC_pos2_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_08_BAC_pos2_original.txt new file mode 100644 index 0000000000..7cf2d339b1 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_08_BAC_pos2_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3903.443 host_launch_us=4825.675 claims=73728 fanin_loads=34306 cas_retries=0 +[ATOMIC] submit_completion_ops=118088 fanin_ready=3645 fanin_not_ready=30661 frontier_initial=1280 frontier_flag=3235 frontier_ready_fetch_max=1955 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=982 RingBp=24 FinalDrain=18 +[KERNEL] QK count=256 mean_us=41.532 min_us=40.553 max_us=51.742 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.089 min_us=52.561 max_us=152.503 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.186 min_us=26.982 max_us=82.756 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.671 min_us=2.329 max_us=17.336 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3903.443 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_09_ACB_pos1_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_09_ACB_pos1_original.txt new file mode 100644 index 0000000000..2b111b65ae --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_09_ACB_pos1_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3901.414 host_launch_us=5107.014 claims=73728 fanin_loads=33235 cas_retries=0 +[ATOMIC] submit_completion_ops=118787 fanin_ready=3735 fanin_not_ready=29500 frontier_initial=1280 frontier_flag=4120 frontier_ready_fetch_max=2840 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=969 RingBp=35 FinalDrain=20 +[KERNEL] QK count=256 mean_us=41.563 min_us=40.688 max_us=51.231 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.165 min_us=52.488 max_us=161.111 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.348 min_us=26.994 max_us=77.245 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.647 min_us=2.326 max_us=21.097 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3901.414 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_10_CAB_pos2_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_10_CAB_pos2_original.txt new file mode 100644 index 0000000000..9072a6fd94 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_10_CAB_pos2_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3948.008 host_launch_us=4905.562 claims=73728 fanin_loads=35157 cas_retries=0 +[ATOMIC] submit_completion_ops=118931 fanin_ready=3698 fanin_not_ready=31459 frontier_initial=1280 frontier_flag=3231 frontier_ready_fetch_max=1951 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=29 +[PLACEMENT] EfDrain=984 RingBp=22 FinalDrain=18 +[KERNEL] QK count=256 mean_us=41.534 min_us=40.670 max_us=51.620 target_us=44.170 +[KERNEL] SF count=256 mean_us=55.476 min_us=52.490 max_us=248.949 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.994 min_us=26.921 max_us=79.908 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.606 min_us=2.334 max_us=10.409 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3948.008 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_11_BCA_pos3_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_11_BCA_pos3_original.txt new file mode 100644 index 0000000000..7a414445be --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_11_BCA_pos3_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3832.087 host_launch_us=4926.323 claims=73728 fanin_loads=38351 cas_retries=0 +[ATOMIC] submit_completion_ops=124819 fanin_ready=3819 fanin_not_ready=34532 frontier_initial=1280 frontier_flag=4578 frontier_ready_fetch_max=3298 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=29 +[PLACEMENT] EfDrain=973 RingBp=36 FinalDrain=15 +[KERNEL] QK count=256 mean_us=41.518 min_us=40.588 max_us=46.823 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.051 min_us=52.216 max_us=131.364 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.826 min_us=26.954 max_us=31.024 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.581 min_us=2.384 max_us=3.718 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=189 inactive_sentinel_tiles=3) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3832.087 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_12_ABC_pos1_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_12_ABC_pos1_original.txt new file mode 100644 index 0000000000..429cb8cfa6 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_12_ABC_pos1_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3971.406 host_launch_us=4864.768 claims=73728 fanin_loads=37579 cas_retries=0 +[ATOMIC] submit_completion_ops=121147 fanin_ready=3664 fanin_not_ready=33915 frontier_initial=1280 frontier_flag=3128 frontier_ready_fetch_max=1848 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=973 RingBp=32 FinalDrain=19 +[KERNEL] QK count=256 mean_us=41.329 min_us=40.632 max_us=46.034 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.326 min_us=52.543 max_us=255.715 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.083 min_us=27.031 max_us=75.392 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.580 min_us=2.343 max_us=3.395 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3971.406 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_13_BCA_pos3_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_13_BCA_pos3_original.txt new file mode 100644 index 0000000000..a2d52072f4 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_13_BCA_pos3_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3953.491 host_launch_us=4852.575 claims=73728 fanin_loads=33478 cas_retries=0 +[ATOMIC] submit_completion_ops=116984 fanin_ready=3611 fanin_not_ready=29867 frontier_initial=1280 frontier_flag=3097 frontier_ready_fetch_max=1817 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=980 RingBp=20 FinalDrain=24 +[KERNEL] QK count=256 mean_us=41.442 min_us=40.668 max_us=46.212 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.723 min_us=52.446 max_us=246.794 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.097 min_us=26.974 max_us=77.523 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.615 min_us=2.352 max_us=13.599 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3953.491 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_14_ACB_pos1_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_14_ACB_pos1_original.txt new file mode 100644 index 0000000000..a2733418a3 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_14_ACB_pos1_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3961.384 host_launch_us=4900.896 claims=73728 fanin_loads=41499 cas_retries=0 +[ATOMIC] submit_completion_ops=125595 fanin_ready=3691 fanin_not_ready=37808 frontier_initial=1280 frontier_flag=3392 frontier_ready_fetch_max=2112 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=32 +[PLACEMENT] EfDrain=968 RingBp=39 FinalDrain=17 +[KERNEL] QK count=256 mean_us=41.559 min_us=40.703 max_us=49.607 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.218 min_us=52.496 max_us=218.183 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.178 min_us=26.971 max_us=74.315 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.765 min_us=2.314 max_us=14.860 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3961.384 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_15_CBA_pos3_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_15_CBA_pos3_original.txt new file mode 100644 index 0000000000..c29f39c351 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_15_CBA_pos3_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3926.037 host_launch_us=4903.135 claims=73728 fanin_loads=35594 cas_retries=0 +[ATOMIC] submit_completion_ops=119820 fanin_ready=3766 fanin_not_ready=31828 frontier_initial=1280 frontier_flag=3457 frontier_ready_fetch_max=2177 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=968 RingBp=33 FinalDrain=23 +[KERNEL] QK count=256 mean_us=41.800 min_us=40.554 max_us=123.036 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.414 min_us=52.258 max_us=175.942 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.089 min_us=26.996 max_us=73.299 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.685 min_us=2.373 max_us=19.710 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3926.037 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_16_ABC_pos1_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_16_ABC_pos1_original.txt new file mode 100644 index 0000000000..6c52a37397 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_16_ABC_pos1_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3972.611 host_launch_us=4845.415 claims=73728 fanin_loads=41697 cas_retries=0 +[ATOMIC] submit_completion_ops=125893 fanin_ready=3667 fanin_not_ready=38030 frontier_initial=1280 frontier_flag=3442 frontier_ready_fetch_max=2162 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=32 +[PLACEMENT] EfDrain=961 RingBp=45 FinalDrain=18 +[KERNEL] QK count=256 mean_us=41.557 min_us=40.522 max_us=53.214 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.814 min_us=52.490 max_us=227.113 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.585 min_us=26.976 max_us=79.098 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.751 min_us=2.336 max_us=17.865 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3972.611 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_17_CAB_pos2_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_17_CAB_pos2_original.txt new file mode 100644 index 0000000000..96e959b14e --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_17_CAB_pos2_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3956.200 host_launch_us=4960.107 claims=73728 fanin_loads=35544 cas_retries=0 +[ATOMIC] submit_completion_ops=119346 fanin_ready=3661 fanin_not_ready=31883 frontier_initial=1280 frontier_flag=3245 frontier_ready_fetch_max=1965 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=34 +[PLACEMENT] EfDrain=982 RingBp=23 FinalDrain=19 +[KERNEL] QK count=256 mean_us=41.549 min_us=40.717 max_us=52.512 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.493 min_us=52.480 max_us=234.465 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.135 min_us=27.035 max_us=80.965 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.714 min_us=2.379 max_us=13.775 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3956.200 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_18_BAC_pos2_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_18_BAC_pos2_original.txt new file mode 100644 index 0000000000..f7dcf75f07 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_18_BAC_pos2_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3918.430 host_launch_us=5198.564 claims=73728 fanin_loads=39851 cas_retries=0 +[ATOMIC] submit_completion_ops=124469 fanin_ready=3769 fanin_not_ready=36082 frontier_initial=1280 frontier_flag=3653 frontier_ready_fetch_max=2373 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=973 RingBp=32 FinalDrain=19 +[KERNEL] QK count=256 mean_us=41.470 min_us=40.678 max_us=49.438 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.816 min_us=52.545 max_us=225.335 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.320 min_us=27.005 max_us=68.760 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.633 min_us=2.322 max_us=15.556 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=190 inactive_sentinel_tiles=2) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3918.430 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_19_BAC_pos2_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_19_BAC_pos2_original.txt new file mode 100644 index 0000000000..7f3350db93 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_19_BAC_pos2_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3960.714 host_launch_us=4860.018 claims=73728 fanin_loads=36373 cas_retries=0 +[ATOMIC] submit_completion_ops=120455 fanin_ready=3633 fanin_not_ready=32740 frontier_initial=1280 frontier_flag=3385 frontier_ready_fetch_max=2105 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=975 RingBp=28 FinalDrain=21 +[KERNEL] QK count=256 mean_us=41.504 min_us=40.693 max_us=46.343 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.542 min_us=52.337 max_us=213.047 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.925 min_us=26.980 max_us=55.890 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.654 min_us=2.389 max_us=13.374 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=190 inactive_sentinel_tiles=2) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3960.714 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_20_CAB_pos2_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_20_CAB_pos2_original.txt new file mode 100644 index 0000000000..e4bba00a59 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_20_CAB_pos2_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3876.067 host_launch_us=4794.053 claims=73728 fanin_loads=36242 cas_retries=0 +[ATOMIC] submit_completion_ops=120630 fanin_ready=3681 fanin_not_ready=32561 frontier_initial=1280 frontier_flag=3538 frontier_ready_fetch_max=2258 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=979 RingBp=29 FinalDrain=16 +[KERNEL] QK count=256 mean_us=41.445 min_us=40.641 max_us=46.000 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.894 min_us=52.327 max_us=134.741 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.098 min_us=26.969 max_us=93.333 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.740 min_us=2.312 max_us=20.455 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3876.067 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_21_ABC_pos1_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_21_ABC_pos1_original.txt new file mode 100644 index 0000000000..bf8ef26a23 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_21_ABC_pos1_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3959.145 host_launch_us=5190.038 claims=73728 fanin_loads=38049 cas_retries=0 +[ATOMIC] submit_completion_ops=121953 fanin_ready=3633 fanin_not_ready=34416 frontier_initial=1280 frontier_flag=3296 frontier_ready_fetch_max=2016 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=971 RingBp=35 FinalDrain=18 +[KERNEL] QK count=256 mean_us=41.544 min_us=40.599 max_us=48.198 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.346 min_us=52.410 max_us=239.966 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.095 min_us=26.973 max_us=76.882 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.736 min_us=2.358 max_us=21.202 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3959.145 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_22_CBA_pos3_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_22_CBA_pos3_original.txt new file mode 100644 index 0000000000..61b473c02c --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_22_CBA_pos3_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3975.785 host_launch_us=4974.285 claims=73728 fanin_loads=36256 cas_retries=0 +[ATOMIC] submit_completion_ops=119776 fanin_ready=3636 fanin_not_ready=32620 frontier_initial=1280 frontier_flag=3104 frontier_ready_fetch_max=1824 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=975 RingBp=25 FinalDrain=24 +[KERNEL] QK count=256 mean_us=41.470 min_us=40.667 max_us=45.918 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.347 min_us=52.385 max_us=258.685 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.403 min_us=27.079 max_us=75.568 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.681 min_us=2.366 max_us=15.094 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3975.785 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_23_ACB_pos1_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_23_ACB_pos1_original.txt new file mode 100644 index 0000000000..005ee91ed4 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_23_ACB_pos1_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3788.296 host_launch_us=4783.019 claims=73728 fanin_loads=39637 cas_retries=0 +[ATOMIC] submit_completion_ops=123541 fanin_ready=3667 fanin_not_ready=35970 frontier_initial=1280 frontier_flag=3296 frontier_ready_fetch_max=2016 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=36 +[PLACEMENT] EfDrain=966 RingBp=38 FinalDrain=20 +[KERNEL] QK count=256 mean_us=41.556 min_us=40.556 max_us=49.436 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.580 min_us=52.359 max_us=62.361 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.132 min_us=27.041 max_us=66.505 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.663 min_us=2.329 max_us=20.456 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3788.296 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_24_BCA_pos3_original.txt b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_24_BCA_pos3_original.txt new file mode 100644 index 0000000000..7d5e34d156 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/raw/block_24_BCA_pos3_original.txt @@ -0,0 +1,44 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007104896 swimlane=off trace_atomics=off trace_bytes=0 +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3956.396 host_launch_us=4951.339 claims=73728 fanin_loads=38551 cas_retries=0 +[ATOMIC] submit_completion_ops=122499 fanin_ready=3659 fanin_not_ready=34892 frontier_initial=1280 frontier_flag=3318 frontier_ready_fetch_max=2038 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=967 RingBp=36 FinalDrain=21 +[KERNEL] QK count=256 mean_us=41.440 min_us=40.533 max_us=46.624 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.363 min_us=52.375 max_us=216.761 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.947 min_us=26.987 max_us=53.119 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.691 min_us=2.384 max_us=12.905 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=189 inactive_sentinel_tiles=3) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3956.396 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/samples.tsv b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/samples.tsv new file mode 100644 index 0000000000..75b8b64a7b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/samples.tsv @@ -0,0 +1,25 @@ +variant block order position submit_span_us included exclusion_reason source +original 1 ABC 1 3963.375 True block_01_ABC_pos1_original.txt +original 2 BCA 3 3856.072 True block_02_BCA_pos3_original.txt +original 3 CAB 2 3870.407 True block_03_CAB_pos2_original.txt +original 4 ACB 1 3956.964 True block_04_ACB_pos1_original.txt +original 5 BAC 2 3989.566 True block_05_BAC_pos2_original.txt +original 6 CBA 3 4000.058 True block_06_CBA_pos3_original.txt +original 7 CBA 3 3930.496 True block_07_CBA_pos3_original.txt +original 8 BAC 2 3903.443 True block_08_BAC_pos2_original.txt +original 9 ACB 1 3901.414 True block_09_ACB_pos1_original.txt +original 10 CAB 2 3948.008 True block_10_CAB_pos2_original.txt +original 11 BCA 3 3832.087 False hampel_3_scaled_mad block_11_BCA_pos3_original.txt +original 12 ABC 1 3971.406 True block_12_ABC_pos1_original.txt +original 13 BCA 3 3953.491 True block_13_BCA_pos3_original.txt +original 14 ACB 1 3961.384 True block_14_ACB_pos1_original.txt +original 15 CBA 3 3926.037 True block_15_CBA_pos3_original.txt +original 16 ABC 1 3972.611 True block_16_ABC_pos1_original.txt +original 17 CAB 2 3956.200 True block_17_CAB_pos2_original.txt +original 18 BAC 2 3918.430 True block_18_BAC_pos2_original.txt +original 19 BAC 2 3960.714 True block_19_BAC_pos2_original.txt +original 20 CAB 2 3876.067 True block_20_CAB_pos2_original.txt +original 21 ABC 1 3959.145 True block_21_ABC_pos1_original.txt +original 22 CBA 3 3975.785 True block_22_CBA_pos3_original.txt +original 23 ACB 1 3788.296 False hampel_3_scaled_mad block_23_ACB_pos1_original.txt +original 24 BCA 3 3956.396 True block_24_BCA_pos3_original.txt diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/summary.json b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/summary.json new file mode 100644 index 0000000000..2c22093fd1 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/performance/summary.json @@ -0,0 +1,64 @@ +{ + "configuration": { + "batches": 256, + "device": 0, + "launch_orders": "all six permutations, each repeated four times", + "launch_positions": { + "1": 8, + "2": 8, + "3": 8 + }, + "pmu_window": "off", + "runs_per_host_launch": 1, + "samples": 24, + "swimlane": "off-runtime", + "winner_workload": "real-compute", + "workers": 96 + }, + "excluded_samples": [ + { + "block": 11, + "exclusion_reason": "hampel_3_scaled_mad", + "included": false, + "order": "BCA", + "position": 3, + "source": "block_11_BCA_pos3_original.txt", + "submit_span_us": 3832.087 + }, + { + "block": 23, + "exclusion_reason": "hampel_3_scaled_mad", + "included": false, + "order": "ACB", + "position": 1, + "source": "block_23_ACB_pos1_original.txt", + "submit_span_us": 3788.296 + } + ], + "outlier_filtered": { + "count": 22, + "max_submit_span_us": 4000.058, + "mean_submit_span_us": 3941.2485909090906, + "median_submit_span_us": 3956.298, + "min_submit_span_us": 3856.072, + "population_stdev_us": 38.00398063440082 + }, + "outlier_rule": { + "lower_us": 3854.1272929000006, + "mad_us": 22.64449999999988, + "median_us": 3954.8455, + "name": "Hampel", + "raw_logs_retained": true, + "upper_us": 4055.563707099999 + }, + "raw": { + "count": 24, + "max_submit_span_us": 4000.058, + "mean_submit_span_us": 3930.3271666666665, + "median_submit_span_us": 3954.8455, + "min_submit_span_us": 3788.296, + "population_stdev_us": 51.729562705692956 + }, + "schema": "pa_lazy_lambda_variant_independent_performance/v1", + "variant": "original" +} diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/output/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/A_original/output/published.sha256 new file mode 100644 index 0000000000..578b3e2cfa --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/output/published.sha256 @@ -0,0 +1,29 @@ +d392ab37d0ee4684d818fbf42b7f04e3fc54ff03485883b0b8b963d48c52d269 output/functional.txt +f98ffa1c7ee34859f5ee83ffe5ef8a1430846d0809b5051672256647162ead7d output/performance/RUN_METADATA.md +7ec7682e381951f8472821951193fb90f99f86d39273a3e2103ed7aebcdc082d output/performance/SUMMARY.md +5e8b3174b43ae4084533e09e141047ca5d15225cb2e34f5d8b8ea7d795421bae output/performance/raw/block_01_ABC_pos1_original.txt +231422a75f8eaff466923c73666cf53432677e9db990fe20fc41ce6985eb0ddd output/performance/raw/block_02_BCA_pos3_original.txt +34c7de1c02da900e493bc36e83ef927c8f03ad1c2ae2b486c61e8656e6afa300 output/performance/raw/block_03_CAB_pos2_original.txt +1936d15a4541f54f496f83b595833ae0cf43dfa8477aa1fc18f7c637f741f301 output/performance/raw/block_04_ACB_pos1_original.txt +375af2bcc02f3ebc80319e215e0e5591c2f1d9ca57c274cee173cbf070ab2bf0 output/performance/raw/block_05_BAC_pos2_original.txt +7a055adba803a5b6acee7c203d700e4fa7544a1a4d8c5aa4665664fc7e590f8e output/performance/raw/block_06_CBA_pos3_original.txt +2d17bf2d85f2e03c78ef3205cf72d32056d2e10bb49ee88a7ca1ed857ede7e5c output/performance/raw/block_07_CBA_pos3_original.txt +3b024b97937fea83406ac856954e475078f21cc7344f5a3c9edcb9d38de4a2fb output/performance/raw/block_08_BAC_pos2_original.txt +f9a4e204ad985bda15447ab05e56c435245c66124a4e8f13389b3205f5531c63 output/performance/raw/block_09_ACB_pos1_original.txt +735190d3c5958b00527ef09b072df4a12a78e263d0d0343ececb36bc8e3cc353 output/performance/raw/block_10_CAB_pos2_original.txt +2bc738fea8e20f4b9b66b2a896689a14835f8d6dad1c345b61af8032ed506e7c output/performance/raw/block_11_BCA_pos3_original.txt +9d951914bac426b6f82ef80aa0cd2bcd969c2fecc6dd76e10e944611fb48b91f output/performance/raw/block_12_ABC_pos1_original.txt +8b69729bc7bedc161ed94a0685bab4c8f117eba2cb0aad463def139a94cec354 output/performance/raw/block_13_BCA_pos3_original.txt +1bf1626d008161b09fa0d0d1b7c55d9c39f6bc445bfd7e0044e19109d4921cf4 output/performance/raw/block_14_ACB_pos1_original.txt +db4d7d1e5cd8ebc6743998646999dcae97da6c7c7271d060b8fbfb8e02325fde output/performance/raw/block_15_CBA_pos3_original.txt +edc3f444992bb11619a4e42c5a3cd86b9082a58b6d2821dbf0ae37680fef3f83 output/performance/raw/block_16_ABC_pos1_original.txt +c4ce09472727f0e382df96143aa879dc5aa90991cfd416161273499723dc8609 output/performance/raw/block_17_CAB_pos2_original.txt +765716ec3e48da56fa67f2d1fbee8fb0a5ee1728bc2be0ddb9f6e46c8fd134f8 output/performance/raw/block_18_BAC_pos2_original.txt +62f70f90a38fde497fb201734b399317d9b54004838fa58db88b74ee43495f38 output/performance/raw/block_19_BAC_pos2_original.txt +3f21404390fa29fd2214893eaf27b38f0cd5abdbdef48a4d76757a977c9b6804 output/performance/raw/block_20_CAB_pos2_original.txt +f8ab313f7a045073ac7fcff3501695bbcaab2607455914ba4a1ca387f8523bff output/performance/raw/block_21_ABC_pos1_original.txt +0127939f3de25c2dce0e6867587d7a1a2add1878cd24113e050d10c8f83cf09a output/performance/raw/block_22_CBA_pos3_original.txt +9596448e024b7f28058714ea611d6f036b41e7489a970fc3dcba41d9c25683eb output/performance/raw/block_23_ACB_pos1_original.txt +59fe17e2f2ad07058f52476ab50b70273e0394e8c972e9531a1910743655c86b output/performance/raw/block_24_BCA_pos3_original.txt +9f3a85c11fa87a62b12b836a20eb450136f582d453c2ee663f8a07a7574929fe output/performance/samples.tsv +0f147aa40d92070aabcf8e618e0e6cf4dacc56f97e606a0330c6e64356055fcc output/performance/summary.json diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/publish_artifacts.sh b/tests/atomic_probe/lazy_lamda_sample/A_original/publish_artifacts.sh new file mode 100755 index 0000000000..a3b94dbb06 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/publish_artifacts.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SOURCE_DIR="$SCRIPT_DIR/build/original" +TARGET_DIR="$SCRIPT_DIR/artifacts/rebuilt" +mkdir -p "$TARGET_DIR" + +for name in pa_scheduler_host pa_scheduler_kernel.o device_text_layout.manifest artifacts.manifest; do + if [[ ! -f "$SOURCE_DIR/$name" ]]; then + echo "Missing clean-build result: $SOURCE_DIR/$name" >&2 + exit 1 + fi + cp -f -- "$SOURCE_DIR/$name" "$TARGET_DIR/$name" +done +(cd "$TARGET_DIR" && sha256sum artifacts.manifest device_text_layout.manifest pa_scheduler_host pa_scheduler_kernel.o) \ + > "$TARGET_DIR/published.sha256" diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/run_functional.sh b/tests/atomic_probe/lazy_lamda_sample/A_original/run_functional.sh new file mode 100755 index 0000000000..ef7acc7c08 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/run_functional.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BUILD_DIR="$SCRIPT_DIR/build/original" +OUTPUT="$SCRIPT_DIR/output/functional.txt" + +if [[ ! -x "$BUILD_DIR/pa_scheduler_host" || ! -f "$BUILD_DIR/pa_scheduler_kernel.o" ]]; then + echo "Missing clean build; run ./build.sh first." >&2 + exit 1 +fi +"$BUILD_DIR/pa_scheduler_host" \ + --kernel "$BUILD_DIR/pa_scheduler_kernel.o" \ + --device 0 --batches 1 --runs 1 \ + --winner-workload real-compute --pmu-window off --no-swimlane | tee "$OUTPUT" +if ! grep -Eq '^\[SUMMARY\].*completed_runs=1.*execution_status=PASS semantic_status=PASS postprocess_status=PASS$' "$OUTPUT"; then + echo "Functional oracle failed for original" >&2 + exit 1 +fi diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/source.sha256 b/tests/atomic_probe/lazy_lamda_sample/A_original/source.sha256 new file mode 100644 index 0000000000..d37f001f2c --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/source.sha256 @@ -0,0 +1,30 @@ +909eff5ca6e3f88d5b077945077864a1557549a3e171bcf233aa236dbd96d660 ./annotate_disassembly.py +c89b2add93ff70181f1c2574835be2948ffa959734962122eafa8c2444f6438a ./build.sh +4091251537a0405f83514d99fc6891300650a01cb61ac3b6c2291e899f7f0747 ./disassemble.py +912a11b6a1241027ee5da46b1c545dcedc47e223de498490e6589c35be0dd611 ./generate_disassembly.sh +7d3dc51260b3dfa0855b7d2ff6c343eac73e07b915c92b3362a9954c860facfc ./publish_artifacts.sh +33180b0902d7376fa9129f245115e61e75acc7ec4fd9d3231085a232f85054d7 ./run_functional.sh +72471b0594aee1069f9fc774faa6d5900e374d3a283fdccc12b2c26a1302472a ./summarize_performance.py +e50378f7afdb15e3a84207b029e2d0aced10e1ac5a363372a791a0a8d177c937 ./verify_runtime_identity.py +b138d97bfc224656f53c91a4503dddac709ef7d1af12b8d7c015fe80e13c40cc README.md +25718360e05d3c2d0963d1381e9dd4dae5fca789244ee4b9f861adcc0cc96218 VARIANT +02c7d7ad77a8fab471c9f1b2a215029b9b407886699534076a9f48d178db0140 ccec/build.sh +ea82967763894d2726b68b19e74c2993ffe938f5bb0c90b92d3eae0966e643b7 ccec/callback_finish.cpp +7090c8405e1602476d1edf4ae6103ec6ec145e22bf15db742eac06aa2a6b7f9f ccec/callback_finish_api.h +4cef27650943d3973c6d56db687850ea3a7cbe28d64bf666b55cf735ce5bd141 ccec/callback_runtime_entry.cpp +47ca7623791bdd408b50c5e5ece4d63c7073775a1e3b636e046419acf948ddda ccec/ccec_ops.h +ac8a9766aa71a2bfc2a21c9021fc9fc30bc869575daa0e0f01121789230173e2 ccec/host.cpp +766ec34d6d4a7d80d07774782c68d701653cadf1bd8427fd1a0e922933c10236 ccec/kernel.cpp +f88d5ed31277c2692ecec2e8f5528cb6c2de752aea6a45850ab72d9e9b600075 ccec/pa_scheduler_device_exports.map +3d65b4a89dcf88554321a5435a2abf701a26757e8c229df7ac9783abbb5b96c3 ccec/pmu_owner_control.h +cb81c2a24670d893fd485845f13b8b106bfffe28d8c471f92a93775eee1a69e6 ccec/pmu_owner_host.h +09263e6698598f7be5b16b92e9e34bc59719de54aee132c43df24cec0ce42b4a ccec/pmu_owner_main_abi.h +9115d77d1be306c9f3b225805ed3df63fbdbdadcb9b4ff6ad690b56cfcf58867 ccec/pmu_owner_main_loader.h +2a926b355f78d4758fdc61a54827baec54ef781df34234a2edc9aa6a1a824f9e ccec/pmu_probe.h +3e3c6c99df27ff9e329242c64680559bf69cb4dc96650acc0ba1fe7442d68c70 common/host_support.h +f4a22840aef81ec05fb9532559957712c76ebd2dd1199d6969fc99688fc2e63d common/pa_frontend.h +9d42688f933e7b20eaab9eadcec40c3f0c0babc8ebdd3dd7369a2afcca1edbec common/pa_model.h +6645ce2aefe39b6132ce649c6af4c597a7a6978aa8983375fe09c4e186a812e5 common/pa_scheduler_core.h +8fd7c429dbbf0aae4e37968e7f90d953793f30f9caeb18eba515f9201deca1fa common/pa_trace.h +9d099151d0303c37dbb88f5cd2b15ee483dc0623759c45cbcf9d80c37704a80d common/winner_workload.h +8be384a62ed669e5645139aab1528570ffab184b5856f685e8056eed68e1e5cc common/winner_workload_host.h diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/summarize_performance.py b/tests/atomic_probe/lazy_lamda_sample/A_original/summarize_performance.py new file mode 100755 index 0000000000..08a98ae100 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/summarize_performance.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Validate this standalone package's 24 independent b256 measurements.""" + +from __future__ import annotations + +import json +import re +import statistics +from pathlib import Path + + +FILE_RE = re.compile( + r"block_(?P\d{2})_(?P[ABC]{3})_pos(?P[123])_" + r"(?Poriginal|compete-first|compete-first-lazy)\.txt$" +) +CONFIG_RE = re.compile( + r"^device=0 batches=256 tasks=1280 workers=96 runs=1 .*\bswimlane=off\b", + re.MULTILINE, +) +METRIC_RE = re.compile(r"^\[METRIC\] run=1 submit_span_us=([0-9.]+)\b", re.MULTILINE) +SUMMARY_RE = re.compile( + r"^\[SUMMARY\] runs=1 completed_runs=1 median_submit_span_us=([0-9.]+) " + r"execution_status=PASS semantic_status=PASS postprocess_status=PASS$", + re.MULTILINE, +) +HAMPEL_SCALE = 1.4826 +HAMPEL_SIGMAS = 3.0 + + +def fail(message: str) -> None: + raise SystemExit(message) + + +def describe(values: list[float]) -> dict[str, float | int]: + return { + "count": len(values), + "median_submit_span_us": statistics.median(values), + "mean_submit_span_us": statistics.fmean(values), + "min_submit_span_us": min(values), + "max_submit_span_us": max(values), + "population_stdev_us": statistics.pstdev(values), + } + + +def main() -> None: + script_dir = Path(__file__).resolve().parent + variant = (script_dir / "VARIANT").read_text(encoding="utf-8").strip() + result_dir = script_dir / "output" / "performance" + paths = sorted((result_dir / "raw").glob("*.txt")) + if len(paths) != 24: + fail(f"{variant}: expected 24 independent raw files, found {len(paths)}") + + rows: list[dict[str, object]] = [] + for path in paths: + name_match = FILE_RE.fullmatch(path.name) + if name_match is None or name_match.group("variant") != variant: + fail(f"{variant}: unexpected raw filename {path.name}") + text = path.read_text(encoding="utf-8") + if CONFIG_RE.search(text) is None: + fail(f"{path}: fixed device0/b256/runs=1/no-swimlane config missing") + if "[WINNER-WORKLOAD] mode=real-compute " not in text: + fail(f"{path}: real-compute workload missing") + if "[PMU-CONFIG] window=off " not in text: + fail(f"{path}: PMU is not explicitly off") + metrics = METRIC_RE.findall(text) + summaries = SUMMARY_RE.findall(text) + if len(metrics) != 1 or len(summaries) != 1 or metrics[0] != summaries[0]: + fail(f"{path}: expected one matching METRIC and PASS SUMMARY") + rows.append( + { + "block": int(name_match.group("block")), + "order": name_match.group("order"), + "position": int(name_match.group("position")), + "submit_span_us": float(metrics[0]), + "source": path.name, + } + ) + + if {int(row["block"]) for row in rows} != set(range(1, 25)): + fail(f"{variant}: blocks must be exactly 01..24") + order_counts = { + order: sum(row["order"] == order for row in rows) + for order in ("ABC", "ACB", "BAC", "BCA", "CAB", "CBA") + } + if set(order_counts.values()) != {4}: + fail(f"{variant}: six orders are not each repeated four times: {order_counts}") + position_counts = { + position: sum(row["position"] == position for row in rows) + for position in (1, 2, 3) + } + if set(position_counts.values()) != {8}: + fail(f"{variant}: launch positions are not 8/8/8: {position_counts}") + + raw_values = [float(row["submit_span_us"]) for row in rows] + center = statistics.median(raw_values) + mad = statistics.median(abs(value - center) for value in raw_values) + if mad == 0.0: + fail(f"{variant}: zero MAD cannot define an outlier threshold") + lower = center - HAMPEL_SIGMAS * HAMPEL_SCALE * mad + upper = center + HAMPEL_SIGMAS * HAMPEL_SCALE * mad + for row in rows: + value = float(row["submit_span_us"]) + row["included"] = lower <= value <= upper + row["exclusion_reason"] = "" if row["included"] else "hampel_3_scaled_mad" + clean_values = [ + float(row["submit_span_us"]) for row in rows if bool(row["included"]) + ] + + document = { + "schema": "pa_lazy_lambda_variant_independent_performance/v1", + "variant": variant, + "configuration": { + "device": 0, + "workers": 96, + "batches": 256, + "samples": 24, + "runs_per_host_launch": 1, + "launch_orders": "all six permutations, each repeated four times", + "launch_positions": {"1": 8, "2": 8, "3": 8}, + "pmu_window": "off", + "swimlane": "off-runtime", + "winner_workload": "real-compute", + }, + "outlier_rule": { + "name": "Hampel", + "median_us": center, + "mad_us": mad, + "lower_us": lower, + "upper_us": upper, + "raw_logs_retained": True, + }, + "raw": describe(raw_values), + "outlier_filtered": describe(clean_values), + "excluded_samples": [row for row in rows if not bool(row["included"])], + } + (result_dir / "summary.json").write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + with (result_dir / "samples.tsv").open("w", encoding="utf-8") as output: + output.write( + "variant\tblock\torder\tposition\tsubmit_span_us\tincluded\t" + "exclusion_reason\tsource\n" + ) + for row in sorted(rows, key=lambda item: int(item["block"])): + output.write( + f"{variant}\t{row['block']}\t{row['order']}\t{row['position']}\t" + f"{float(row['submit_span_us']):.3f}\t{row['included']}\t" + f"{row['exclusion_reason']}\t{row['source']}\n" + ) + raw = document["raw"] + clean = document["outlier_filtered"] + lines = [ + f"# {variant} device0 b256 result", + "", + "24 independent host launches, one device run per launch; PMU off, runtime swimlane off, real-compute.", + "Raw logs are retained. The primary row excludes samples outside median ± 3×1.4826×MAD.", + "", + "| View | N | Median (us) | Mean (us) | Min (us) | Max (us) | Stddev (us) |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + f"| Raw | {raw['count']} | {raw['median_submit_span_us']:.3f} | " + f"{raw['mean_submit_span_us']:.3f} | {raw['min_submit_span_us']:.3f} | " + f"{raw['max_submit_span_us']:.3f} | {raw['population_stdev_us']:.3f} |", + f"| Outlier-filtered | {clean['count']} | {clean['median_submit_span_us']:.3f} | " + f"{clean['mean_submit_span_us']:.3f} | {clean['min_submit_span_us']:.3f} | " + f"{clean['max_submit_span_us']:.3f} | {clean['population_stdev_us']:.3f} |", + "", + f"Excluded samples: {raw['count'] - clean['count']}.", + ] + (result_dir / "SUMMARY.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + print("\n".join(lines)) + + +if __name__ == "__main__": + main() diff --git a/tests/atomic_probe/lazy_lamda_sample/A_original/verify_runtime_identity.py b/tests/atomic_probe/lazy_lamda_sample/A_original/verify_runtime_identity.py new file mode 100755 index 0000000000..786fcc487f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/A_original/verify_runtime_identity.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Prove that measured and local-rebuild AICore ELFs have identical runtime content.""" + +from __future__ import annotations + +import hashlib +import json +import struct +from dataclasses import dataclass +from pathlib import Path + + +SHT_NOBITS = 8 +SHT_SYMTAB = 2 +STT_FUNC = 2 +STT_OBJECT = 1 + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def c_string(table: bytes, offset: int) -> str: + end = table.find(b"\0", offset) + if end < 0: + raise ValueError("unterminated ELF string") + return table[offset:end].decode("utf-8") + + +@dataclass(frozen=True) +class Section: + index: int + name: str + section_type: int + flags: int + address: int + offset: int + size: int + link: int + alignment: int + entry_size: int + + +class Elf64: + def __init__(self, path: Path): + self.path = path + self.data = path.read_bytes() + header = struct.unpack_from("<16sHHIQQQIHHHHHH", self.data, 0) + if header[0][:6] != b"\x7fELF\x02\x01": + raise ValueError(f"not ELF64 little-endian: {path}") + section_offset, section_size, count, names_index = header[6], header[11], header[12], header[13] + if section_size != 64: + raise ValueError(f"unexpected section header size: {section_size}") + raw = [ + struct.unpack_from(" bytes: + if section.section_type == SHT_NOBITS: + return b"" + return self.data[section.offset : section.offset + section.size] + + def runtime_symbols(self) -> dict[str, tuple[int, int, int, int]]: + symtab = next(section for section in self.sections if section.section_type == SHT_SYMTAB) + strings = self.content(self.sections[symtab.link]) + symbols: dict[str, tuple[int, int, int, int]] = {} + for offset in range(symtab.offset, symtab.offset + symtab.size, symtab.entry_size): + name_offset, info, _, section_index, value, size = struct.unpack_from( + "= len(self.sections): + continue + section = self.sections[section_index] + if section.name not in {".text", ".rodata", ".bl_uninit"}: + continue + name = c_string(strings, name_offset) + if name: + symbols[name] = (symbol_type, info >> 4, value, size) + return symbols + + +def main() -> None: + script_dir = Path(__file__).resolve().parent + measured_dir = script_dir / "artifacts" / "measured" + rebuilt_dir = script_dir / "artifacts" / "rebuilt" + measured = Elf64(measured_dir / "pa_scheduler_kernel.o") + rebuilt = Elf64(rebuilt_dir / "pa_scheduler_kernel.o") + + content_sections = [ + ".text", + ".rodata", + ".ascend.meta.pa_scheduler_0_mix_aic", + "__CCE_KernelArgSize", + ".ascend.meta.pa_scheduler_0_mix_aiv", + ] + section_rows = [] + all_ok = True + for name in content_sections: + left = measured.by_name[name] + right = rebuilt.by_name[name] + metadata_equal = ( + left.section_type, left.flags, left.address, left.size, left.alignment + ) == ( + right.section_type, right.flags, right.address, right.size, right.alignment + ) + content_equal = measured.content(left) == rebuilt.content(right) + all_ok &= metadata_equal and content_equal + section_rows.append( + { + "name": name, + "size": left.size, + "sha256": digest(measured.content(left)), + "metadata_equal": metadata_equal, + "content_equal": content_equal, + } + ) + + nobits_rows = [] + for name in [".bl_uninit"]: + if name not in measured.by_name and name not in rebuilt.by_name: + continue + if name not in measured.by_name or name not in rebuilt.by_name: + all_ok = False + nobits_rows.append({"name": name, "layout_equal": False}) + continue + left = measured.by_name[name] + right = rebuilt.by_name[name] + layout_equal = ( + left.section_type, left.flags, left.address, left.size, left.alignment + ) == ( + right.section_type, right.flags, right.address, right.size, right.alignment + ) + all_ok &= layout_equal and left.section_type == SHT_NOBITS + nobits_rows.append( + { + "name": name, + "type": "SHT_NOBITS", + "size": left.size, + "alignment": left.alignment, + "layout_equal": layout_equal, + } + ) + + measured_symbols = measured.runtime_symbols() + rebuilt_symbols = rebuilt.runtime_symbols() + symbols_equal = measured_symbols == rebuilt_symbols + all_ok &= symbols_equal + measured_host = (measured_dir / "pa_scheduler_host").read_bytes() + rebuilt_host = (rebuilt_dir / "pa_scheduler_host").read_bytes() + host_equal = measured_host == rebuilt_host + all_ok &= host_equal + + document = { + "schema": "pa_lazy_lambda_runtime_identity/v1", + "variant": (script_dir / "VARIANT").read_text(encoding="utf-8").strip(), + "status": "PASS" if all_ok else "FAIL", + "measured_elf_sha256": digest(measured.data), + "rebuilt_elf_sha256": digest(rebuilt.data), + "full_elf_equal": measured.data == rebuilt.data, + "runtime_content_sections": section_rows, + "nobits_layout": nobits_rows, + "runtime_symbols_equal": symbols_equal, + "runtime_symbol_count": len(measured_symbols), + "host_equal": host_equal, + "host_sha256": digest(measured_host), + "note": "Measured is the exact performance binary; full ELF and runtime identities are checked explicitly.", + } + output = script_dir / "artifacts" / "runtime_identity.json" + output.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8") + if not all_ok: + raise SystemExit(f"runtime identity check failed; see {output}") + print( + f"PASS variant={document['variant']} runtime_sections={len(section_rows)} " + f"runtime_symbols={len(measured_symbols)} host_sha256={document['host_sha256']}" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/MANIFEST.sha256 b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/MANIFEST.sha256 new file mode 100644 index 0000000000..54416d135d --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/MANIFEST.sha256 @@ -0,0 +1,123 @@ +a1aced021c2f2cc7bd40a736d602f8ebc17045a30969366d869e76d0462782af ./README.md +a8685790bf5928a1e7e719760aa87383e52e269d85d6005804647e899798dd2e ./VARIANT +909eff5ca6e3f88d5b077945077864a1557549a3e171bcf233aa236dbd96d660 ./annotate_disassembly.py +b1de9a82dfd3bc1889a946ce43db63e1ec4d642f0e49e2079c7eeb1aa3edb61a ./artifacts/measured/artifacts.manifest +c44bab7dd1c36001a5ebaead1c857c54c6ceade63d85f3a2f0a138c16f217d88 ./artifacts/measured/device_text_layout.manifest +ee86b1e9457a190ba4594372853492997831f954953e1d037e34655fff212e70 ./artifacts/measured/pa_scheduler_host +82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b ./artifacts/measured/pa_scheduler_kernel.o +1cf43b4fbdb4745b099437c68074b7c2fb3ce0a8755926393ae2c3e572268a73 ./artifacts/measured/published.sha256 +b1de9a82dfd3bc1889a946ce43db63e1ec4d642f0e49e2079c7eeb1aa3edb61a ./artifacts/rebuilt/artifacts.manifest +c44bab7dd1c36001a5ebaead1c857c54c6ceade63d85f3a2f0a138c16f217d88 ./artifacts/rebuilt/device_text_layout.manifest +ee86b1e9457a190ba4594372853492997831f954953e1d037e34655fff212e70 ./artifacts/rebuilt/pa_scheduler_host +82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b ./artifacts/rebuilt/pa_scheduler_kernel.o +1cf43b4fbdb4745b099437c68074b7c2fb3ce0a8755926393ae2c3e572268a73 ./artifacts/rebuilt/published.sha256 +4eaf69233f6a3f1f700c8089467d74d941fdbc164338323c55349fbc543011bc ./artifacts/runtime_identity.json +cc9948d9ec75d8402345238b4ec986b82493765bda6114289b997789f2cf3df6 ./build.sh +b1de9a82dfd3bc1889a946ce43db63e1ec4d642f0e49e2079c7eeb1aa3edb61a ./build/compete-first/artifacts.manifest +c44bab7dd1c36001a5ebaead1c857c54c6ceade63d85f3a2f0a138c16f217d88 ./build/compete-first/device_text_layout.manifest +eb69d0502d445f3af70f247138f883075e55194644e898b7cc10ba3cbd5c2a6d ./build/compete-first/pa_scheduler_aic.o +825bf448c4f3446c26ea18e73f0968d8a7792a14a2e61f68b9abc9946ba1351e ./build/compete-first/pa_scheduler_aiv.o +ee86b1e9457a190ba4594372853492997831f954953e1d037e34655fff212e70 ./build/compete-first/pa_scheduler_host +82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b ./build/compete-first/pa_scheduler_kernel.o +0a0bdda39da95e9abd56981f68f393be1b7e9afc5f2e798b1ce67dfea8671f13 ./build/compete-first/pa_scheduler_lazy_sample_callback_finish_aic.o +68ac75a1035f5d4bd98f65cd108fc1a9075b872deeb27e9fb5b70f6453790371 ./build/compete-first/pa_scheduler_lazy_sample_callback_finish_aiv.o +1f2232795b7031a325539e73a8eda708c4addc5072855d98a6ba5aa5aac1c2f3 ./build/compete-first/pa_scheduler_lazy_sample_callback_runtime_aic.o +c6d4ecfad8d8930fe39bbaa3792059f755df1110e1b2ebbfbdce852bc0e650d7 ./build/compete-first/pa_scheduler_lazy_sample_callback_runtime_aiv.o +3ec0b1cff001bb202bb4fc076af93960129f1c8f47b3a7786c3e6ae5b19bca1e ./build/published.sha256 +061dfa1a8baa9b08d6360d3cfb899a17d58b0cb001421f8a0de387cfea53a5e0 ./ccec/build.sh +ea82967763894d2726b68b19e74c2993ffe938f5bb0c90b92d3eae0966e643b7 ./ccec/callback_finish.cpp +7090c8405e1602476d1edf4ae6103ec6ec145e22bf15db742eac06aa2a6b7f9f ./ccec/callback_finish_api.h +4cef27650943d3973c6d56db687850ea3a7cbe28d64bf666b55cf735ce5bd141 ./ccec/callback_runtime_entry.cpp +47ca7623791bdd408b50c5e5ece4d63c7073775a1e3b636e046419acf948ddda ./ccec/ccec_ops.h +ac8a9766aa71a2bfc2a21c9021fc9fc30bc869575daa0e0f01121789230173e2 ./ccec/host.cpp +766ec34d6d4a7d80d07774782c68d701653cadf1bd8427fd1a0e922933c10236 ./ccec/kernel.cpp +f88d5ed31277c2692ecec2e8f5528cb6c2de752aea6a45850ab72d9e9b600075 ./ccec/pa_scheduler_device_exports.map +3d65b4a89dcf88554321a5435a2abf701a26757e8c229df7ac9783abbb5b96c3 ./ccec/pmu_owner_control.h +cb81c2a24670d893fd485845f13b8b106bfffe28d8c471f92a93775eee1a69e6 ./ccec/pmu_owner_host.h +09263e6698598f7be5b16b92e9e34bc59719de54aee132c43df24cec0ce42b4a ./ccec/pmu_owner_main_abi.h +9115d77d1be306c9f3b225805ed3df63fbdbdadcb9b4ff6ad690b56cfcf58867 ./ccec/pmu_owner_main_loader.h +2a926b355f78d4758fdc61a54827baec54ef781df34234a2edc9aa6a1a824f9e ./ccec/pmu_probe.h +3e3c6c99df27ff9e329242c64680559bf69cb4dc96650acc0ba1fe7442d68c70 ./common/host_support.h +f4a22840aef81ec05fb9532559957712c76ebd2dd1199d6969fc99688fc2e63d ./common/pa_frontend.h +9d42688f933e7b20eaab9eadcec40c3f0c0babc8ebdd3dd7369a2afcca1edbec ./common/pa_model.h +6645ce2aefe39b6132ce649c6af4c597a7a6978aa8983375fe09c4e186a812e5 ./common/pa_scheduler_core.h +8fd7c429dbbf0aae4e37968e7f90d953793f30f9caeb18eba515f9201deca1fa ./common/pa_trace.h +9d099151d0303c37dbb88f5cd2b15ee483dc0623759c45cbcf9d80c37704a80d ./common/winner_workload.h +8be384a62ed669e5645139aab1528570ffab184b5856f685e8056eed68e1e5cc ./common/winner_workload_host.h +4091251537a0405f83514d99fc6891300650a01cb61ac3b6c2291e899f7f0747 ./disassemble.py +3292a86b93eab8348954313002fc1dd65f738064eba46a55046b82db2f00b05c ./disassembly/annotated/00_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz +c4e1068852d4913326c9966b980778be629c6342f2a572f935b9195ac3ce5acf ./disassembly/annotated/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz +2c8f08e69e6d275d14c21752b013cdca005f76fe85c82d927f77dd5d009251eb ./disassembly/annotated/02_pa_execute_real_winner_workload_aic_729699de.source.asm.gz +25a4cea4738ceda57bf84c563351c6c58ab3bd4cbe43fbef4a537a87dbec31b8 ./disassembly/annotated/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.source.asm.gz +e028d624bc208e64ede4f6c56984f7d4327097f57ba7368d47d08c56be8b595e ./disassembly/annotated/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +4a5974e6be2ebb736fe03677ae942dea94220c137c8f15dcb6921132d8ded67e ./disassembly/annotated/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.source.asm.gz +0aaf06d33ab00c2a7e7dd9db07cc8dbcb7c8ace605f802b35e22ee32eebdf053 ./disassembly/annotated/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +df8dc382afac580888cd730a9c100f6bb4db3d78bcaaf650a71e10ae2874d9b8 ./disassembly/annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz +26f8f4222cf44beecb8162c737e5dc5634f04d60a3112bf2b7cbb8f5bdfdb007 ./disassembly/annotated/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.source.asm.gz +aa8c58ba245738d297d1bfd417a461bba620f1f19449ab7f558ce7775b7baebe ./disassembly/annotated/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +58115295576f99ee50e7d30c79d010c1f344f32267dca760dc1e54d0f65584f8 ./disassembly/annotated/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz +8427b688bb1cb27b9ce5fa18b1e32e43aa00db1d77c56046c1ecc6de2536e1f0 ./disassembly/annotated/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz +9647b3f76dcfb9191b379c104d2f55df284cadf87c66b7dca706a05512456f04 ./disassembly/annotated/12_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz +ad138409ad8cc12e50a8da6d53508b2a5b2ada8de885cf36fffe9c24b72b35d3 ./disassembly/annotated/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.source.asm.gz +ede15075567acbd2fb310a8605c03e516770c1d5b8d2aeb0c84e56c6a79b1603 ./disassembly/annotated/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +efb4b5fa5adbb4eab3024373eac7732eb0822a1a47053c4b7e48b40eecf9f27d ./disassembly/annotated/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz +2fb817ccfef3b0e1a84858b3fb372abff3a46ad8e4c80cf31e2edde1ee174082 ./disassembly/annotated/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz +10b4fd32aa317ea69a229d45c2bc81089204cf05bb0491e2e21b25e178e79020 ./disassembly/key_flow/aic_compete_first.source.asm +87e4803df4a79065ad2767cda040d1de578527156c3d4813cc9ab9eaeae8045d ./disassembly/published.sha256 +818c59ec85db9ebffdebec406ebbf434534cfb1920d8517b7ee1ebd88d6fba06 ./disassembly/raw/00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz +713680d66b9a4010516e94c8b1eaf4699de3d3d168a0071ffcc683d485e6fcfd ./disassembly/raw/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz +000992bf2e056f3f0e25e2e951d7ce17f574ce42fe85808eab4c81d64e82fd96 ./disassembly/raw/02_pa_execute_real_winner_workload_aic_729699de.asm.gz +af912f03ca0d8d5a25ab0eb0e18bc2f1fea568c2170926e4956b978a56ad4e86 ./disassembly/raw/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz +f588ab8d1c7c292c69d4bb25a20a74cc2072a7a5ae70cb021b167601ec7254bc ./disassembly/raw/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +dee953904a64d9bbc05899f8c6af53a61e67a61da29e6b94047d565b91139861 ./disassembly/raw/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz +505a11beffc1964f5f60cdd5a535207c7e9e81f84cc9f352dc7cea77f01ff612 ./disassembly/raw/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +ea465c73c294f044872df4d536cee4f14299a3937acc9bf86549c5e1026b1bf9 ./disassembly/raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz +8df4f8d7705b55eef87d28c821425083f4eb3e6e00a7b42cc6233f1c2e04c2d8 ./disassembly/raw/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz +359c91af50cb07eab96b6e31e08e9dbf202b16068835a11c6ce97884df2f0e8e ./disassembly/raw/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +feed9d8a6d512e0710afca1661e453175c953fea9d07503f1732b4c0b605f02c ./disassembly/raw/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz +7470b8f6211167da842268a5454438b45db0abc7222e143ced6bc10012875a22 ./disassembly/raw/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz +e2d30bf63296ad647bdfd996f7b1119273753e5820b90e72e0082c0a80725a58 ./disassembly/raw/12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz +ccac55531be6275b35fe9e8af72f1307050df9817c93b4857a7406f8713f764d ./disassembly/raw/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz +3bedb6520d68993867c5f264efbf93d720f64c55387adc6c90c29638ff7a559a ./disassembly/raw/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +c27e21715651d040bd90609566937e55b73a2995ba83f931cec8d43eec90dafc ./disassembly/raw/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz +785be4d5d33fd2d6f4e914b0ef3980d11565ae336587289f208c18572763da39 ./disassembly/raw/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz +3ea3b8526cce898bfd3dcc89428fd6224aaff736e093b3d548563b1da65e0111 ./disassembly/raw/README.md +f3b2a710693b2203667061eba93f9f25125447688f6b119fc538331ec7262cdf ./disassembly/raw/gaps.tsv +37e5eb8364d0bc99702167e350de753564eb203007cf46994d3e63154ca182c4 ./disassembly/raw/manifest.tsv +388d35f89270629c2d65334cf652d9f0c36cf96f39fdbfa79e0414e73a749fd7 ./disassembly/raw/published.sha256 +ac61de825e41468de26a785ef55b3f9228386dfdd729e1b375d043e00f0b8fc5 ./generate_disassembly.sh +dff00fa736fa9bddfa8f311935a9b678ab148616b62bd6f6774f0e43b257a0ab ./output/functional.txt +aacbdf1712d3cc9470fdcddc96702a940346a88e3d0f0e3bc748c2715bcfa811 ./output/performance/RUN_METADATA.md +4afe4aafa2d88ecff92ce9894edfbbdb12fd38ed48e0ef789d06e02b7e031e3c ./output/performance/SUMMARY.md +bdada9610d2585615d1fb42e75436110b1df3f5ad6a19b6a9e32a4a953e597b3 ./output/performance/raw/block_01_ABC_pos2_compete-first.txt +3774d404a7377b2780b4d9c43ef42dd3dda83e4b88f0cf0c05a5384786672c79 ./output/performance/raw/block_02_BCA_pos1_compete-first.txt +36aa6a4bf0489792150148e980138f99cd9e6e5af767fa58b50b3a3424d153c8 ./output/performance/raw/block_03_CAB_pos3_compete-first.txt +135da9673048751a5ebfda521179c461b63eeda92c5db800a7e047f1eba215f7 ./output/performance/raw/block_04_ACB_pos3_compete-first.txt +4e48747c67199fbac808da2e7884e6b198393b50b16e99cfd4035918f6924320 ./output/performance/raw/block_05_BAC_pos1_compete-first.txt +f8de8efc390cbe221e5a0614cbdc3c4cf4c8bfa828c7e72a043baa5052e81e96 ./output/performance/raw/block_06_CBA_pos2_compete-first.txt +3b0d321edffbb8c0bfd2d871b054eef12a26d67ef43a003785fa7dadad66c579 ./output/performance/raw/block_07_CBA_pos2_compete-first.txt +71a5006347c1e8552ec70122b9e13db6a7c40ca2ca864824e48abe2a84393809 ./output/performance/raw/block_08_BAC_pos1_compete-first.txt +4ca7114686e131a44933f7f797b1d400a56f1d38c98e06931b7250481bbc5753 ./output/performance/raw/block_09_ACB_pos3_compete-first.txt +fb57fc79164745b772db6250071e8e182a26cb45fbca4885dde95123457ec4a7 ./output/performance/raw/block_10_CAB_pos3_compete-first.txt +bf4fc5a32d595fa3b1c5162697a83899d3af15086614b7edb253f48bd4c62644 ./output/performance/raw/block_11_BCA_pos1_compete-first.txt +5e3bf10747c8f311308b95247b580806555ea63836e470a5006962a54a25739b ./output/performance/raw/block_12_ABC_pos2_compete-first.txt +75103bfe53e8fa80418183586b32c20d7999b9b22a088a918fbb529ceff71bd6 ./output/performance/raw/block_13_BCA_pos1_compete-first.txt +ca04f2ed8a4779f32502ee36152a37a56339b199f71e728de9127ded34cab602 ./output/performance/raw/block_14_ACB_pos3_compete-first.txt +31f7d1cda1a8cff22c9dc169582fcf988ae4f51a505d874c117a8c789d24d0b0 ./output/performance/raw/block_15_CBA_pos2_compete-first.txt +fd560782ace04f756d1f6d7fb31d5e770e1a91d136e3bb385bfa2fe2021b56bc ./output/performance/raw/block_16_ABC_pos2_compete-first.txt +5f946a71e527199c82e751ec658b6224641908084fe5be63fc237479caf939ea ./output/performance/raw/block_17_CAB_pos3_compete-first.txt +d2a0ead197fd686eb45c40ae975b4ba1fc4b90983bdfde9f51b9884a1bf6a586 ./output/performance/raw/block_18_BAC_pos1_compete-first.txt +8df6439b1216a7d0a548cb1167c23ab269cd2465d0c90083c3440c2075f4c27f ./output/performance/raw/block_19_BAC_pos1_compete-first.txt +d37d9827a7c7a2cc3083ee0621233d069b96b2c248cf0cbf613114fe43872456 ./output/performance/raw/block_20_CAB_pos3_compete-first.txt +b46fe72351e897f746d197f6d63dab7d50271f720df6dced125841424f519678 ./output/performance/raw/block_21_ABC_pos2_compete-first.txt +9308547bf6c947fead642cbaefb47afb3a00f328a57af8b26e7ff8192fa1d1a7 ./output/performance/raw/block_22_CBA_pos2_compete-first.txt +0847887bdbc0212fa6531ee1d9fd7dac293722fe92174ed25aa041db12a98b51 ./output/performance/raw/block_23_ACB_pos3_compete-first.txt +fe9642e9f007641d1b68ea1f1bf4f120e785c2dd423455679f6d3e73c192f1f4 ./output/performance/raw/block_24_BCA_pos1_compete-first.txt +4b561fa972c22d8ae2f6663e805132440451a613785943b856e88406a6b86613 ./output/performance/samples.tsv +4cea5fc4dfcb815e6841d050d2024a61b224f3dc9f76d816da25464d99e56e29 ./output/performance/summary.json +928dd514330a6604af1f4878aee829e15a8a84f52a8beaf41acc464ff80a39cf ./output/published.sha256 +d38bd2f74659b7b7cdaa392cde321f112ac31ae699942bd2d4a901f5ecf5cbb0 ./publish_artifacts.sh +e3a218ad05fdbf7bcda1cab32fbfe616efc31281cef588a0b91f465dd96fddf3 ./run_functional.sh +960b85ad9242cc9ef4f04efb44b27d6cbc39b188b09ea443e409c90b9d0198f5 ./source.sha256 +72471b0594aee1069f9fc774faa6d5900e374d3a283fdccc12b2c26a1302472a ./summarize_performance.py +e50378f7afdb15e3a84207b029e2d0aced10e1ac5a363372a791a0a8d177c937 ./verify_runtime_identity.py diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/README.md b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/README.md new file mode 100644 index 0000000000..50ff55698f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/README.md @@ -0,0 +1,52 @@ +# B：compete-first eager + +本目录是一份可单独复制、构建和查看的 compete-first eager 样例，不依赖 A/C 目录或上级目录中的源码、 +脚本与二进制。完整 task 流为 `Alloc -> QK -> SF -> PV -> UP`，32 AIC + 64 AIV。 + +## 固定形态 + +- `VARIANT=compete-first`，`PA_LAZY_SAMPLE_SHAPE_ID=1`；`ccec/build.sh` 会拒绝其他 shape； +- `PA_LAZY_SAMPLE_SPLIT_FINISH=1`，finish 为 noinline cross-TU； +- `PA_BUILD_SWIMLANE=1`、`PA_BUILD_SUBMIT_PMU=0`，运行时显式 `--no-swimlane`； +- 控制流是 `Begin -> EfDrain -> Claim -> outer callback -> Materialize -> TensorMap retire` + `-> winner Fanin -> Register -> winner Build/Complete`； +- callback 内的 nested lambda 由 `Add*` 同步求值，不保存 thunk/closure；`Lazy=false`，所以每个 worker + 都求值全部 input/output/inout/scalar thunk。 + +主路径源码在 `common/pa_scheduler_core.h` 的 `SubmitLazySampleCallback`、 +`BuildLazySampleCallbackArgs`，builder 在 `common/pa_frontend.h`。固定构建入口无法选择 A/C； +未删除的 inactive `#if` 分支只保留来源上下文。 + +## 独立交付内容 + +- `common/`、`ccec/`：本版完整源码； +- `build/compete-first/`：清空历史目录后生成的完整本地编译结果和中间对象； +- `artifacts/measured/`：正式 24 样本性能测试实际使用的 host/final ELF； +- `artifacts/rebuilt/`：当前同一 clean-build 的发布副本； +- `artifacts/runtime_identity.json`：measured/rebuilt 运行时身份校验； +- `disassembly/raw/`：final-linked 机器码权威解码； +- `disassembly/annotated/`：每个函数的完整源码注释版反汇编； +- `disassembly/key_flow/aic_compete_first.source.asm`：未压缩的 Claim/callback 关键窗口； +- `output/functional.txt`：最新 device0 b1 功能门禁； +- `output/performance/`：24 次独立 b256 原始日志、运行 provenance、样本表和统计； +- `source.sha256`、`build/published.sha256`、`output/published.sha256`、 + `MANIFEST.sha256`：源码、干净构建、输出与整包校验入口。 + +Measured final `.text` 为 `547640 B`，SHA256 +`8fd6209b2f0f9d1fb48d4d8a16cf27649e26071153b908e1e0e39ca107d63589`。 +去异常 Submit span 中位数为 `3744.417 us`（24 个样本均保留）。b1 精确前端计数为 +`views/tensor/scalar/resets = 192/2112/864/384`。 + +## 复跑 + +```bash +./build.sh # 先精确删除 build/compete-first +./publish_artifacts.sh # 只刷新 artifacts/rebuilt +./verify_runtime_identity.py +./run_functional.sh # device0、b1、PMU off、--no-swimlane +./generate_disassembly.sh # measured ELF 的 raw + annotated 全量反汇编 +./summarize_performance.py +``` + +`*.source.asm.gz` 用 `zless` 阅读。DWARF 只提供地址到文件/行映射；显示的源码与原注释由脚本从本目录 +源码复制，文件头不会把它们冒充成 ELF 自带注释。 diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/VARIANT b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/VARIANT new file mode 100644 index 0000000000..1f4735fc2a --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/VARIANT @@ -0,0 +1 @@ +compete-first diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/annotate_disassembly.py b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/annotate_disassembly.py new file mode 100755 index 0000000000..c6eb096a1a --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/annotate_disassembly.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Add DWARF-mapped source and local source comments to one raw disassembly.""" + +from __future__ import annotations + +import argparse +import gzip +import os +import re +import subprocess +from pathlib import Path + + +INSTRUCTION_RE = re.compile(r"^(0x([0-9a-fA-F]+) \(\+0x[0-9a-fA-F]+\).*)$") +LOCATION_RE = re.compile(r"^(.*):(\d+)(?: \(discriminator \d+\))?$") +ANCHOR_RE = re.compile(r"^(.*):(\d+)$") + + +def read_text(path: Path) -> str: + if path.suffix == ".gz": + with gzip.open(path, "rt", encoding="utf-8") as source: + return source.read() + return path.read_text(encoding="utf-8") + + +def normalize_source_path(path_text: str, source_root: Path) -> tuple[Path | None, str]: + if path_text in {"??", ""}: + return None, "??" + # Some CCEC line-table rows are rendered as /path/file:?:0. The final + # :0 is parsed as the line number, leaving :? attached to the path. + if path_text.endswith(":?"): + path_text = path_text[:-2] + candidate = Path(path_text) + normalized = Path(candidate.as_posix().replace("/ccec/../common/", "/common/")) + parts = normalized.parts + if "lazy_lamda_sample" in parts: + marker = len(parts) - 1 - list(reversed(parts)).index("lazy_lamda_sample") + tail = parts[marker + 1 :] + source_index = next( + (index for index, part in enumerate(tail) if part in {"common", "ccec"}), + None, + ) + if source_index is not None: + suffix = Path(*tail[source_index:]) + local = source_root / suffix + return (local if local.is_file() else None), suffix.as_posix() + if candidate.is_file(): + ascend_home = os.environ.get("ASCEND_HOME_PATH") + if ascend_home: + try: + relative = candidate.resolve().relative_to(Path(ascend_home).resolve()) + except ValueError: + pass + else: + return candidate, f"$ASCEND_HOME_PATH/{relative.as_posix()}" + return candidate, candidate.as_posix() + return None, candidate.as_posix() + + +def source_lines(path: Path | None, cache: dict[Path, list[str]]) -> list[str] | None: + if path is None: + return None + if path not in cache: + cache[path] = path.read_text(encoding="utf-8").splitlines() + return cache[path] + + +def write_output(path: Path, text: str) -> None: + if path.suffix == ".gz": + with path.open("wb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed: + compressed.write(text.encode("utf-8")) + return + path.write_text(text, encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser() + script_dir = Path(__file__).resolve().parent + parser.add_argument("--elf", type=Path, required=True) + parser.add_argument("--raw", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--source-root", type=Path, default=script_dir) + parser.add_argument("--anchor", help="First DWARF location matching SUFFIX:LINE") + parser.add_argument("--before", type=int, default=40) + parser.add_argument("--after", type=int, default=160) + args = parser.parse_args() + + raw_lines = read_text(args.raw.resolve()).splitlines() + instruction_lines = [line for line in raw_lines if INSTRUCTION_RE.match(line)] + if not instruction_lines: + raise SystemExit(f"No instruction rows in {args.raw}") + pcs = [int(INSTRUCTION_RE.match(line).group(2), 16) for line in instruction_lines] + process = subprocess.run( + ["addr2line", "-e", str(args.elf.resolve()), "-C"], + input="".join(f"0x{pc:x}\n" for pc in pcs), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + locations = process.stdout.splitlines() + if len(locations) != len(instruction_lines): + raise SystemExit( + f"addr2line returned {len(locations)} locations for {len(instruction_lines)} instructions" + ) + + normalized_locations: list[tuple[Path | None, str, int]] = [] + source_root = args.source_root.resolve() + for location in locations: + if location.endswith(":?"): + local_path, display = normalize_source_path(location[:-2], source_root) + normalized_locations.append((local_path, display, 0)) + continue + match = LOCATION_RE.match(location) + if match is None: + normalized_locations.append((None, location, 0)) + continue + local_path, display = normalize_source_path(match.group(1), source_root) + normalized_locations.append((local_path, display, int(match.group(2)))) + + start = 0 + end = len(instruction_lines) + if args.anchor: + anchor_match = ANCHOR_RE.match(args.anchor) + if anchor_match is None: + raise SystemExit("--anchor must be SUFFIX:LINE") + suffix = anchor_match.group(1) + anchor_line = int(anchor_match.group(2)) + anchor_index = next( + ( + index + for index, (_, display, line) in enumerate(normalized_locations) + if display.endswith(suffix) and line == anchor_line + ), + None, + ) + if anchor_index is None: + raise SystemExit(f"DWARF anchor not found: {args.anchor}") + start = max(0, anchor_index - args.before) + end = min(len(instruction_lines), anchor_index + args.after + 1) + + headers = [line for line in raw_lines if line.startswith("#")] + output = headers + [ + "# annotation_schema=pa_source_annotated_disassembly/v1", + "# annotation_rule=DWARF supplies only file:line; SOURCE rows are copied from local source files", + "# annotation_warning=comments have source context only and do not own an exact machine address", + f"# annotation_instruction_slice={start}:{end}", + "#", + ] + cache: dict[Path, list[str]] = {} + previous_display = "" + previous_line = 0 + for index in range(start, end): + local_path, display, line_number = normalized_locations[index] + if display != previous_display or line_number != previous_line: + lines = source_lines(local_path, cache) + output.append(f"# [DWARF] {display}:{line_number}") + if lines is None or line_number <= 0 or line_number > len(lines): + output.append("# [SOURCE unavailable]") + else: + if display == previous_display and previous_line < line_number <= previous_line + 12: + context_start = previous_line + 1 + else: + context_start = max(1, line_number - 6) + for source_line in range(context_start, line_number + 1): + marker = ">" if source_line == line_number else " " + output.append(f"# {marker} {source_line:5d} | {lines[source_line - 1]}") + previous_display = display + previous_line = line_number + output.append(instruction_lines[index]) + + args.output.parent.mkdir(parents=True, exist_ok=True) + write_output(args.output, "\n".join(output) + "\n") + print( + f"wrote {args.output}: instructions={end - start} " + f"range=0x{pcs[start]:x}..0x{pcs[end - 1]:x}" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/artifacts.manifest b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/artifacts.manifest new file mode 100644 index 0000000000..d3aef03393 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/artifacts.manifest @@ -0,0 +1,14 @@ +# schema=pa_scheduler_lazy_sample_callback_artifacts/v1 +# backend=ccec +# shape=compete-first +# shape_id=1 +# observation=split-combination-semantic +ee86b1e9457a190ba4594372853492997831f954953e1d037e34655fff212e70 pa_scheduler_host +82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b pa_scheduler_kernel.o +1f2232795b7031a325539e73a8eda708c4addc5072855d98a6ba5aa5aac1c2f3 pa_scheduler_lazy_sample_callback_runtime_aic.o +eb69d0502d445f3af70f247138f883075e55194644e898b7cc10ba3cbd5c2a6d pa_scheduler_aic.o +0a0bdda39da95e9abd56981f68f393be1b7e9afc5f2e798b1ce67dfea8671f13 pa_scheduler_lazy_sample_callback_finish_aic.o +c6d4ecfad8d8930fe39bbaa3792059f755df1110e1b2ebbfbdce852bc0e650d7 pa_scheduler_lazy_sample_callback_runtime_aiv.o +825bf448c4f3446c26ea18e73f0968d8a7792a14a2e61f68b9abc9946ba1351e pa_scheduler_aiv.o +68ac75a1035f5d4bd98f65cd108fc1a9075b872deeb27e9fb5b70f6453790371 pa_scheduler_lazy_sample_callback_finish_aiv.o +c44bab7dd1c36001a5ebaead1c857c54c6ceade63d85f3a2f0a138c16f217d88 device_text_layout.manifest diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/device_text_layout.manifest b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/device_text_layout.manifest new file mode 100644 index 0000000000..d98e8aa19d --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/device_text_layout.manifest @@ -0,0 +1,27 @@ +# schema=pa_scheduler_device_text_layout/v1 +# backend=ccec +# shape=compete-first +# block_local_reserve_bytes=1600 +# compiler=$ASCEND_HOME_PATH/bin/ccec +# compiler_sha256=34355dde8c995f3c7d769ddf0e1b853aba65cf6ef6fb3863ab86c026d40aebfd +# linker=$ASCEND_HOME_PATH/bin/ld.lld +# linker_sha256=727810dd7f72dedbe1453edeb81d3a5371d8dc0591bc58693edf1aa7f3fa7eff +text pa_scheduler_kernel.o 547640 8fd6209b2f0f9d1fb48d4d8a16cf27649e26071153b908e1e0e39ca107d63589 +text pa_scheduler_lazy_sample_callback_runtime_aic.o 112 9391340d3a45b712c7b883e97e1f6889eae31e54e4ba1ce52864012bc4141582 +text pa_scheduler_aic.o 162608 6f6c83824de9294f6716a9c2e7a4209a30eb6ac857cd6ea6bc1387ee13f388fd +text pa_scheduler_lazy_sample_callback_finish_aic.o 110352 f1e086fe71d1d83ac7654eb0e38475f3e79996d841b18e21830df2fd40e873bc +text pa_scheduler_lazy_sample_callback_runtime_aiv.o 128 7bf82e7aa5167025d62c32b482dd5222f913fce1d08d1770917f4f757f165c1d +text pa_scheduler_aiv.o 163640 09c2d46c59baff8796cce19bb4c1459a9cc272ce08708a571f30ec580e893128 +text pa_scheduler_lazy_sample_callback_finish_aiv.o 110800 260afa427b5935b15f8368b2e4c054f6d57d4b0e5bf861b9ef769758a7c3b0d7 +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aic 104 3a8ecf84e154c1afc9dab2c992e3357d338c1d996a677931069f16310bdb28e5 +symbol pa_scheduler_lazy_sample_callback_runtime_aic.o pa_scheduler_0_mix_aic 104 42b5cc25a392a99eb3366f12177f028db8a27f55c00166753083fa75cf404b79 +symbol pa_scheduler_aic.o pa_scheduler_lazy_sample_callback_orchestration_aic 161788 8d90f3eece36ba60dca57f9f6f80278eb81d57a93fd836b8b119ac08066c6633 +symbol pa_scheduler_lazy_sample_callback_finish_aic.o pa_scheduler_lazy_sample_callback_finish_aic 110128 2adeb947c5bd08fa13d6ea3c403174e067e333d62dda31e59ee1380fece174b5 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_orchestration_aic 161788 cab368a779b3870411a46bd169f6af9657c025967797ba7743b93fcfe27f45db +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_finish_aic 110128 cdb102a84acb7a681565d97805b405d526a94c604812faed3850773575acee08 +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aiv 124 63f9349fa440318678bcf3e98ad719502bd907f7537fb0f083eac32895e3ad9b +symbol pa_scheduler_lazy_sample_callback_runtime_aiv.o pa_scheduler_0_mix_aiv 124 f0f4b9a2d2c1acd43efba6a24731f90b8424b1681b0091797f0bd4b3da6500a5 +symbol pa_scheduler_aiv.o pa_scheduler_lazy_sample_callback_orchestration_aiv 161876 d8bf52012ec8ca744dd05c8444edc1095ebd9f8ae37751ff43bc9b21a00feb9d +symbol pa_scheduler_lazy_sample_callback_finish_aiv.o pa_scheduler_lazy_sample_callback_finish_aiv 110576 c9656e23b23851999d185a6b440b2ed70cb3639db51fd7e3feab7f35acb8e4c7 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_orchestration_aiv 161876 cc2b188dca238188ac76fd0ee6ff232c4be99f8e3274f4cee173316d67c3e503 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_finish_aiv 110576 3f5ec8fee73b82e72f40566908c21f4fcfd3fd85983892c323a526c66557cade diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/pa_scheduler_host b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/pa_scheduler_host new file mode 100755 index 0000000000..e61e0f6901 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/pa_scheduler_host differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/pa_scheduler_kernel.o b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/pa_scheduler_kernel.o new file mode 100755 index 0000000000..376af0a630 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/pa_scheduler_kernel.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/published.sha256 new file mode 100644 index 0000000000..0dc8075915 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/measured/published.sha256 @@ -0,0 +1,4 @@ +b1de9a82dfd3bc1889a946ce43db63e1ec4d642f0e49e2079c7eeb1aa3edb61a artifacts.manifest +c44bab7dd1c36001a5ebaead1c857c54c6ceade63d85f3a2f0a138c16f217d88 device_text_layout.manifest +ee86b1e9457a190ba4594372853492997831f954953e1d037e34655fff212e70 pa_scheduler_host +82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b pa_scheduler_kernel.o diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/artifacts.manifest b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/artifacts.manifest new file mode 100644 index 0000000000..d3aef03393 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/artifacts.manifest @@ -0,0 +1,14 @@ +# schema=pa_scheduler_lazy_sample_callback_artifacts/v1 +# backend=ccec +# shape=compete-first +# shape_id=1 +# observation=split-combination-semantic +ee86b1e9457a190ba4594372853492997831f954953e1d037e34655fff212e70 pa_scheduler_host +82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b pa_scheduler_kernel.o +1f2232795b7031a325539e73a8eda708c4addc5072855d98a6ba5aa5aac1c2f3 pa_scheduler_lazy_sample_callback_runtime_aic.o +eb69d0502d445f3af70f247138f883075e55194644e898b7cc10ba3cbd5c2a6d pa_scheduler_aic.o +0a0bdda39da95e9abd56981f68f393be1b7e9afc5f2e798b1ce67dfea8671f13 pa_scheduler_lazy_sample_callback_finish_aic.o +c6d4ecfad8d8930fe39bbaa3792059f755df1110e1b2ebbfbdce852bc0e650d7 pa_scheduler_lazy_sample_callback_runtime_aiv.o +825bf448c4f3446c26ea18e73f0968d8a7792a14a2e61f68b9abc9946ba1351e pa_scheduler_aiv.o +68ac75a1035f5d4bd98f65cd108fc1a9075b872deeb27e9fb5b70f6453790371 pa_scheduler_lazy_sample_callback_finish_aiv.o +c44bab7dd1c36001a5ebaead1c857c54c6ceade63d85f3a2f0a138c16f217d88 device_text_layout.manifest diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/device_text_layout.manifest b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/device_text_layout.manifest new file mode 100644 index 0000000000..d98e8aa19d --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/device_text_layout.manifest @@ -0,0 +1,27 @@ +# schema=pa_scheduler_device_text_layout/v1 +# backend=ccec +# shape=compete-first +# block_local_reserve_bytes=1600 +# compiler=$ASCEND_HOME_PATH/bin/ccec +# compiler_sha256=34355dde8c995f3c7d769ddf0e1b853aba65cf6ef6fb3863ab86c026d40aebfd +# linker=$ASCEND_HOME_PATH/bin/ld.lld +# linker_sha256=727810dd7f72dedbe1453edeb81d3a5371d8dc0591bc58693edf1aa7f3fa7eff +text pa_scheduler_kernel.o 547640 8fd6209b2f0f9d1fb48d4d8a16cf27649e26071153b908e1e0e39ca107d63589 +text pa_scheduler_lazy_sample_callback_runtime_aic.o 112 9391340d3a45b712c7b883e97e1f6889eae31e54e4ba1ce52864012bc4141582 +text pa_scheduler_aic.o 162608 6f6c83824de9294f6716a9c2e7a4209a30eb6ac857cd6ea6bc1387ee13f388fd +text pa_scheduler_lazy_sample_callback_finish_aic.o 110352 f1e086fe71d1d83ac7654eb0e38475f3e79996d841b18e21830df2fd40e873bc +text pa_scheduler_lazy_sample_callback_runtime_aiv.o 128 7bf82e7aa5167025d62c32b482dd5222f913fce1d08d1770917f4f757f165c1d +text pa_scheduler_aiv.o 163640 09c2d46c59baff8796cce19bb4c1459a9cc272ce08708a571f30ec580e893128 +text pa_scheduler_lazy_sample_callback_finish_aiv.o 110800 260afa427b5935b15f8368b2e4c054f6d57d4b0e5bf861b9ef769758a7c3b0d7 +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aic 104 3a8ecf84e154c1afc9dab2c992e3357d338c1d996a677931069f16310bdb28e5 +symbol pa_scheduler_lazy_sample_callback_runtime_aic.o pa_scheduler_0_mix_aic 104 42b5cc25a392a99eb3366f12177f028db8a27f55c00166753083fa75cf404b79 +symbol pa_scheduler_aic.o pa_scheduler_lazy_sample_callback_orchestration_aic 161788 8d90f3eece36ba60dca57f9f6f80278eb81d57a93fd836b8b119ac08066c6633 +symbol pa_scheduler_lazy_sample_callback_finish_aic.o pa_scheduler_lazy_sample_callback_finish_aic 110128 2adeb947c5bd08fa13d6ea3c403174e067e333d62dda31e59ee1380fece174b5 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_orchestration_aic 161788 cab368a779b3870411a46bd169f6af9657c025967797ba7743b93fcfe27f45db +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_finish_aic 110128 cdb102a84acb7a681565d97805b405d526a94c604812faed3850773575acee08 +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aiv 124 63f9349fa440318678bcf3e98ad719502bd907f7537fb0f083eac32895e3ad9b +symbol pa_scheduler_lazy_sample_callback_runtime_aiv.o pa_scheduler_0_mix_aiv 124 f0f4b9a2d2c1acd43efba6a24731f90b8424b1681b0091797f0bd4b3da6500a5 +symbol pa_scheduler_aiv.o pa_scheduler_lazy_sample_callback_orchestration_aiv 161876 d8bf52012ec8ca744dd05c8444edc1095ebd9f8ae37751ff43bc9b21a00feb9d +symbol pa_scheduler_lazy_sample_callback_finish_aiv.o pa_scheduler_lazy_sample_callback_finish_aiv 110576 c9656e23b23851999d185a6b440b2ed70cb3639db51fd7e3feab7f35acb8e4c7 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_orchestration_aiv 161876 cc2b188dca238188ac76fd0ee6ff232c4be99f8e3274f4cee173316d67c3e503 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_finish_aiv 110576 3f5ec8fee73b82e72f40566908c21f4fcfd3fd85983892c323a526c66557cade diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/pa_scheduler_host b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/pa_scheduler_host new file mode 100755 index 0000000000..e61e0f6901 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/pa_scheduler_host differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/pa_scheduler_kernel.o b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/pa_scheduler_kernel.o new file mode 100755 index 0000000000..376af0a630 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/pa_scheduler_kernel.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/published.sha256 new file mode 100644 index 0000000000..0dc8075915 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/rebuilt/published.sha256 @@ -0,0 +1,4 @@ +b1de9a82dfd3bc1889a946ce43db63e1ec4d642f0e49e2079c7eeb1aa3edb61a artifacts.manifest +c44bab7dd1c36001a5ebaead1c857c54c6ceade63d85f3a2f0a138c16f217d88 device_text_layout.manifest +ee86b1e9457a190ba4594372853492997831f954953e1d037e34655fff212e70 pa_scheduler_host +82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b pa_scheduler_kernel.o diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/runtime_identity.json b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/runtime_identity.json new file mode 100644 index 0000000000..49c1186421 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/artifacts/runtime_identity.json @@ -0,0 +1,59 @@ +{ + "full_elf_equal": true, + "host_equal": true, + "host_sha256": "ee86b1e9457a190ba4594372853492997831f954953e1d037e34655fff212e70", + "measured_elf_sha256": "82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b", + "nobits_layout": [ + { + "alignment": 64, + "layout_equal": true, + "name": ".bl_uninit", + "size": 3200, + "type": "SHT_NOBITS" + } + ], + "note": "Measured is the exact performance binary; full ELF and runtime identities are checked explicitly.", + "rebuilt_elf_sha256": "82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b", + "runtime_content_sections": [ + { + "content_equal": true, + "metadata_equal": true, + "name": ".text", + "sha256": "8fd6209b2f0f9d1fb48d4d8a16cf27649e26071153b908e1e0e39ca107d63589", + "size": 547640 + }, + { + "content_equal": true, + "metadata_equal": true, + "name": ".rodata", + "sha256": "182a132211bc2b56e7d47defb5ef87aa4a343d26a404261233ba62b12454fdc6", + "size": 576 + }, + { + "content_equal": true, + "metadata_equal": true, + "name": ".ascend.meta.pa_scheduler_0_mix_aic", + "sha256": "ff019d3ac6a3ece0ac9868ad2c6c86743efd7015d2ffa24b0fa85f585442cafb", + "size": 24 + }, + { + "content_equal": true, + "metadata_equal": true, + "name": "__CCE_KernelArgSize", + "sha256": "c0cdbb6e45249a718e7e605f801cf43f673a22fcf0347034da71f6be2e4f4d5e", + "size": 8 + }, + { + "content_equal": true, + "metadata_equal": true, + "name": ".ascend.meta.pa_scheduler_0_mix_aiv", + "sha256": "d498029b3bd8fd059bfaca013163a729768f11edf540f96b77bf40ea1e604921", + "size": 56 + } + ], + "runtime_symbol_count": 22, + "runtime_symbols_equal": true, + "schema": "pa_lazy_lambda_runtime_identity/v1", + "status": "PASS", + "variant": "compete-first" +} diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build.sh b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build.sh new file mode 100755 index 0000000000..d77dfb7a97 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +VARIANT="compete-first" +BUILD_DIR="$SCRIPT_DIR/build/$VARIANT" + +# This package deliberately owns exactly one build directory. Remove the +# complete old directory so stale objects can never survive a rebuild. +if [[ "$BUILD_DIR" != "$SCRIPT_DIR/build/compete-first" ]]; then + echo "Refusing to clean an unexpected build path: $BUILD_DIR" >&2 + exit 1 +fi +if [[ -e "$BUILD_DIR" ]]; then + rm -rf -- "$BUILD_DIR" +fi + +"$SCRIPT_DIR/ccec/build.sh" "$VARIANT" diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/artifacts.manifest b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/artifacts.manifest new file mode 100644 index 0000000000..d3aef03393 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/artifacts.manifest @@ -0,0 +1,14 @@ +# schema=pa_scheduler_lazy_sample_callback_artifacts/v1 +# backend=ccec +# shape=compete-first +# shape_id=1 +# observation=split-combination-semantic +ee86b1e9457a190ba4594372853492997831f954953e1d037e34655fff212e70 pa_scheduler_host +82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b pa_scheduler_kernel.o +1f2232795b7031a325539e73a8eda708c4addc5072855d98a6ba5aa5aac1c2f3 pa_scheduler_lazy_sample_callback_runtime_aic.o +eb69d0502d445f3af70f247138f883075e55194644e898b7cc10ba3cbd5c2a6d pa_scheduler_aic.o +0a0bdda39da95e9abd56981f68f393be1b7e9afc5f2e798b1ce67dfea8671f13 pa_scheduler_lazy_sample_callback_finish_aic.o +c6d4ecfad8d8930fe39bbaa3792059f755df1110e1b2ebbfbdce852bc0e650d7 pa_scheduler_lazy_sample_callback_runtime_aiv.o +825bf448c4f3446c26ea18e73f0968d8a7792a14a2e61f68b9abc9946ba1351e pa_scheduler_aiv.o +68ac75a1035f5d4bd98f65cd108fc1a9075b872deeb27e9fb5b70f6453790371 pa_scheduler_lazy_sample_callback_finish_aiv.o +c44bab7dd1c36001a5ebaead1c857c54c6ceade63d85f3a2f0a138c16f217d88 device_text_layout.manifest diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/device_text_layout.manifest b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/device_text_layout.manifest new file mode 100644 index 0000000000..d98e8aa19d --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/device_text_layout.manifest @@ -0,0 +1,27 @@ +# schema=pa_scheduler_device_text_layout/v1 +# backend=ccec +# shape=compete-first +# block_local_reserve_bytes=1600 +# compiler=$ASCEND_HOME_PATH/bin/ccec +# compiler_sha256=34355dde8c995f3c7d769ddf0e1b853aba65cf6ef6fb3863ab86c026d40aebfd +# linker=$ASCEND_HOME_PATH/bin/ld.lld +# linker_sha256=727810dd7f72dedbe1453edeb81d3a5371d8dc0591bc58693edf1aa7f3fa7eff +text pa_scheduler_kernel.o 547640 8fd6209b2f0f9d1fb48d4d8a16cf27649e26071153b908e1e0e39ca107d63589 +text pa_scheduler_lazy_sample_callback_runtime_aic.o 112 9391340d3a45b712c7b883e97e1f6889eae31e54e4ba1ce52864012bc4141582 +text pa_scheduler_aic.o 162608 6f6c83824de9294f6716a9c2e7a4209a30eb6ac857cd6ea6bc1387ee13f388fd +text pa_scheduler_lazy_sample_callback_finish_aic.o 110352 f1e086fe71d1d83ac7654eb0e38475f3e79996d841b18e21830df2fd40e873bc +text pa_scheduler_lazy_sample_callback_runtime_aiv.o 128 7bf82e7aa5167025d62c32b482dd5222f913fce1d08d1770917f4f757f165c1d +text pa_scheduler_aiv.o 163640 09c2d46c59baff8796cce19bb4c1459a9cc272ce08708a571f30ec580e893128 +text pa_scheduler_lazy_sample_callback_finish_aiv.o 110800 260afa427b5935b15f8368b2e4c054f6d57d4b0e5bf861b9ef769758a7c3b0d7 +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aic 104 3a8ecf84e154c1afc9dab2c992e3357d338c1d996a677931069f16310bdb28e5 +symbol pa_scheduler_lazy_sample_callback_runtime_aic.o pa_scheduler_0_mix_aic 104 42b5cc25a392a99eb3366f12177f028db8a27f55c00166753083fa75cf404b79 +symbol pa_scheduler_aic.o pa_scheduler_lazy_sample_callback_orchestration_aic 161788 8d90f3eece36ba60dca57f9f6f80278eb81d57a93fd836b8b119ac08066c6633 +symbol pa_scheduler_lazy_sample_callback_finish_aic.o pa_scheduler_lazy_sample_callback_finish_aic 110128 2adeb947c5bd08fa13d6ea3c403174e067e333d62dda31e59ee1380fece174b5 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_orchestration_aic 161788 cab368a779b3870411a46bd169f6af9657c025967797ba7743b93fcfe27f45db +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_finish_aic 110128 cdb102a84acb7a681565d97805b405d526a94c604812faed3850773575acee08 +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aiv 124 63f9349fa440318678bcf3e98ad719502bd907f7537fb0f083eac32895e3ad9b +symbol pa_scheduler_lazy_sample_callback_runtime_aiv.o pa_scheduler_0_mix_aiv 124 f0f4b9a2d2c1acd43efba6a24731f90b8424b1681b0091797f0bd4b3da6500a5 +symbol pa_scheduler_aiv.o pa_scheduler_lazy_sample_callback_orchestration_aiv 161876 d8bf52012ec8ca744dd05c8444edc1095ebd9f8ae37751ff43bc9b21a00feb9d +symbol pa_scheduler_lazy_sample_callback_finish_aiv.o pa_scheduler_lazy_sample_callback_finish_aiv 110576 c9656e23b23851999d185a6b440b2ed70cb3639db51fd7e3feab7f35acb8e4c7 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_orchestration_aiv 161876 cc2b188dca238188ac76fd0ee6ff232c4be99f8e3274f4cee173316d67c3e503 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_finish_aiv 110576 3f5ec8fee73b82e72f40566908c21f4fcfd3fd85983892c323a526c66557cade diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_aic.o b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_aic.o new file mode 100644 index 0000000000..60bc649864 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_aic.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_aiv.o b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_aiv.o new file mode 100644 index 0000000000..b1261b25e9 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_aiv.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_host b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_host new file mode 100755 index 0000000000..e61e0f6901 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_host differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_kernel.o b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_kernel.o new file mode 100755 index 0000000000..376af0a630 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_kernel.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_lazy_sample_callback_finish_aic.o b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_lazy_sample_callback_finish_aic.o new file mode 100644 index 0000000000..ec5cd134a2 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_lazy_sample_callback_finish_aic.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_lazy_sample_callback_finish_aiv.o b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_lazy_sample_callback_finish_aiv.o new file mode 100644 index 0000000000..c3ce9b9d28 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_lazy_sample_callback_finish_aiv.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_lazy_sample_callback_runtime_aic.o b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_lazy_sample_callback_runtime_aic.o new file mode 100644 index 0000000000..382325dbf8 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_lazy_sample_callback_runtime_aic.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_lazy_sample_callback_runtime_aiv.o b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_lazy_sample_callback_runtime_aiv.o new file mode 100644 index 0000000000..ae3cd7f8a0 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/compete-first/pa_scheduler_lazy_sample_callback_runtime_aiv.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/published.sha256 new file mode 100644 index 0000000000..15f029d325 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/build/published.sha256 @@ -0,0 +1,10 @@ +b1de9a82dfd3bc1889a946ce43db63e1ec4d642f0e49e2079c7eeb1aa3edb61a build/compete-first/artifacts.manifest +c44bab7dd1c36001a5ebaead1c857c54c6ceade63d85f3a2f0a138c16f217d88 build/compete-first/device_text_layout.manifest +eb69d0502d445f3af70f247138f883075e55194644e898b7cc10ba3cbd5c2a6d build/compete-first/pa_scheduler_aic.o +825bf448c4f3446c26ea18e73f0968d8a7792a14a2e61f68b9abc9946ba1351e build/compete-first/pa_scheduler_aiv.o +ee86b1e9457a190ba4594372853492997831f954953e1d037e34655fff212e70 build/compete-first/pa_scheduler_host +82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b build/compete-first/pa_scheduler_kernel.o +0a0bdda39da95e9abd56981f68f393be1b7e9afc5f2e798b1ce67dfea8671f13 build/compete-first/pa_scheduler_lazy_sample_callback_finish_aic.o +68ac75a1035f5d4bd98f65cd108fc1a9075b872deeb27e9fb5b70f6453790371 build/compete-first/pa_scheduler_lazy_sample_callback_finish_aiv.o +1f2232795b7031a325539e73a8eda708c4addc5072855d98a6ba5aa5aac1c2f3 build/compete-first/pa_scheduler_lazy_sample_callback_runtime_aic.o +c6d4ecfad8d8930fe39bbaa3792059f755df1110e1b2ebbfbdce852bc0e650d7 build/compete-first/pa_scheduler_lazy_sample_callback_runtime_aiv.o diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/build.sh b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/build.sh new file mode 100755 index 0000000000..a1f6a49c0b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/build.sh @@ -0,0 +1,934 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +# 任一工具失败、未定义变量或管道中间失败都立即终止，避免继续使用半成品 device ELF。 +set -euo pipefail + +# 所有输入和产物都从脚本自身位置解析，调用者无需位于仓库根目录。 +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +LAZY_SAMPLE_MANIFEST_NAME="artifacts.manifest" +LAZY_SAMPLE_TEXT_LAYOUT_NAME="device_text_layout.manifest" +LAZY_SAMPLE_SPLIT=0 +LAZY_SAMPLE_VARIANT=1 +PMU_VARIANT=0 + +# All three artifacts compile the same swimlane-capable source with submit PMU +# disabled. Performance runs pass --no-swimlane at runtime. +if [[ $# -ne 1 || "$1" != "compete-first" ]]; then + echo "This frozen package accepts only: $0 compete-first" >&2 + exit 1 +fi +BUILD_VARIANT="$1" +case "$BUILD_VARIANT" in + original) + LAZY_SAMPLE_SHAPE="original" + LAZY_SAMPLE_SHAPE_ID=0 + LAZY_SAMPLE_OBSERVATION="legacy-straight-line-eager" + LAZY_SAMPLE_FINISH_SHAPE="legacy-inline-submit" + BUILD_DIR="$ROOT_DIR/build/original" + VARIANT_DEFINES=(-DPA_BUILD_SWIMLANE=1 -DPA_BUILD_SUBMIT_PMU=0 -DPA_SUBMIT_PMU_PHASE_ID=0) + ;; + compete-first|compete-first-lazy) + LAZY_SAMPLE_SPLIT=1 + LAZY_SAMPLE_OBSERVATION="split-combination-semantic" + LAZY_SAMPLE_FINISH_SHAPE="noinline-cross-tu" + if [[ "$BUILD_VARIANT" == "compete-first" ]]; then + LAZY_SAMPLE_SHAPE="compete-first" + LAZY_SAMPLE_SHAPE_ID=1 + else + LAZY_SAMPLE_SHAPE="compete-first-lazy" + LAZY_SAMPLE_SHAPE_ID=2 + fi + BUILD_DIR="$ROOT_DIR/build/$LAZY_SAMPLE_SHAPE" + VARIANT_DEFINES=( + -DPA_BUILD_SWIMLANE=1 + -DPA_BUILD_SUBMIT_PMU=0 + -DPA_SUBMIT_PMU_PHASE_ID=0 + "-DPA_LAZY_SAMPLE_SHAPE_ID=$LAZY_SAMPLE_SHAPE_ID" + -DPA_LAZY_SAMPLE_SPLIT_FINISH=1 + ) + ;; + *) + echo "Unknown variant: $BUILD_VARIANT (expected original|compete-first|compete-first-lazy)" >&2 + exit 1 + ;; +esac + +# 编译只依赖本目录源码与用户安装的 CANN/PTO 头，不引用 pa_scheduler 目录外的 simpler 构建产物。 +if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then + echo "ASCEND_HOME_PATH is not set; source the CANN 9.1 set_env.sh first." >&2 + exit 1 +fi + +CCEC="$ASCEND_HOME_PATH/bin/ccec" +LD="$ASCEND_HOME_PATH/bin/ld.lld" +HCC="$ASCEND_HOME_PATH/tools/hcc/bin/aarch64-target-linux-gnu-g++" +CXX_BIN="${CXX:-g++}" +READELF_BIN="${READELF:-readelf}" +PTO_INCLUDE_ROOT="${PTO_ISA_ROOT:-$ASCEND_HOME_PATH/x86_64-linux}" + +# ccec/ld.lld 必须来自当前已 source 的 CANN；host 编译器和 readelf 允许用户通过环境变量替换。 +if [[ ! -x "$CCEC" || ! -x "$LD" ]]; then + echo "CCEC or ld.lld is missing under ASCEND_HOME_PATH=$ASCEND_HOME_PATH" >&2 + exit 1 +fi +if [[ "$PMU_VARIANT" -eq 1 && ! -x "$HCC" ]]; then + echo "The AICPU HCC compiler is missing under ASCEND_HOME_PATH=$ASCEND_HOME_PATH" >&2 + exit 1 +fi +if ! command -v "$READELF_BIN" >/dev/null 2>&1; then + echo "readelf is required to verify the mixed AICore ELF." >&2 + exit 1 +fi +if [[ "$BUILD_VARIANT" != "swimlane" ]] && ! command -v sha256sum >/dev/null 2>&1; then + echo "sha256sum is required to publish variant artifact manifests." >&2 + exit 1 +fi +if [[ ! -f "$PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" ]]; then + echo "PTO kernel metadata header is missing under $PTO_INCLUDE_ROOT/include" >&2 + exit 1 +fi +for header in pto/pto-inst.hpp pto/common/constants.hpp pto/common/pto_tile.hpp; do + if [[ ! -f "$PTO_INCLUDE_ROOT/include/$header" ]]; then + echo "PTO real-compute header is missing: $PTO_INCLUDE_ROOT/include/$header" >&2 + exit 1 + fi +done + +mkdir -p "$BUILD_DIR" +if [[ "$PMU_VARIANT" -eq 0 ]]; then + # 旧统一构建可能在根目录残留 PMU owner；swimlane 构建主动移除这两个 + # 不属于本变体的产物，避免 direct host 调用误加载上一版诊断 SO。 + rm -f \ + "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so" \ + "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so" +else + # manifest 是同一 phase 四件套唯一的“可运行”标记。重建一开始先使旧 + # manifest 失效；即使后续编译中断，run.sh 也不会消费目录里的半成品。 + rm -f -- "$BUILD_DIR/$SUBMIT_PMU_MANIFEST_NAME" +fi +if [[ "$LAZY_SAMPLE_VARIANT" -eq 1 ]]; then + rm -f -- \ + "$BUILD_DIR/$LAZY_SAMPLE_MANIFEST_NAME" \ + "$BUILD_DIR/$LAZY_SAMPLE_TEXT_LAYOUT_NAME" +fi + +# 关闭编译器自动插入的 scalar DCCI，由 kernel.cpp 中与 PA 对齐的显式失效/回写协议负责 cache 可见性。 +# 两种架构共用这些 ABI、栈和优化参数，避免 AIC/AIV 对共享 SchedulerState 产生不同解释。 +COMMON_FLAGS=( + -c -O3 -g -x cce -Wall -std=c++17 + --cce-aicore-only + -mllvm -cce-aicore-stack-size=0x8000 + -mllvm -cce-aicore-function-stack-size=0x8000 + -mllvm -cce-aicore-record-overflow=false + -mllvm -cce-aicore-addr-transform + -mllvm -cce-aicore-dcci-insert-for-scalar=false + -mllvm -cce-aicore-dcci-before-kernel-end=false + -I"$ROOT_DIR/common" + -I"$PTO_INCLUDE_ROOT/include" + "${VARIANT_DEFINES[@]}" +) + +# split finish 的完整 runtime state 为 1600B，超过 CCEC 默认保留的 +# block-local 栈空间。编译器 hidden help 明确该参数以 byte 为单位、上限 +# 4KiB。实测 1600B 与 2048B 虽生成相同大小的 .text，内容 SHA 却不同， +# 因此使用当前 ABI 的精确尺寸而不增加无依据余量，并严格限于 split 变体。 +LAZY_SAMPLE_SPLIT_STATE_BYTES=1600 +LAZY_SAMPLE_FINISH_CALL_SITES=5 +LAZY_SAMPLE_BLOCK_LOCAL_RESERVE_BYTES=0 +if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_BLOCK_LOCAL_RESERVE_BYTES=$LAZY_SAMPLE_SPLIT_STATE_BYTES + COMMON_FLAGS+=( + -mllvm -cce-block-local-relocate=true + -mllvm "-cce-block-local-reserve-size=$LAZY_SAMPLE_BLOCK_LOCAL_RESERVE_BYTES" + ) +fi + +# 同一入口源码分别面向 cube 与 vector ISA 编译，宏只选择各自的全局入口和 mixed metadata。 +echo "[BUILD] CCEC AIC entry (dav-c310-cube)" +"$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DPA_BUILD_AIC \ + -o "$BUILD_DIR/pa_scheduler_aic.o" \ + "$SCRIPT_DIR/kernel.cpp" + +echo "[BUILD] CCEC AIV entry (dav-c310-vec)" +"$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DPA_BUILD_AIV \ + -o "$BUILD_DIR/pa_scheduler_aiv.o" \ + "$SCRIPT_DIR/kernel.cpp" + +check_workload_dispatcher_object() { + local object_path="$1" + local expected_symbol="$2" + local wrong_role_symbol="$3" + local object_symbols + object_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$object_path")" + if ! awk -v name="$expected_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$object_symbols"; then + echo "Expected exactly one non-empty strong workload dispatcher in $object_path: $expected_symbol" >&2 + exit 1 + fi + if awk -v name="$wrong_role_symbol" \ + '$NF == name {found = 1} END {exit !found}' <<<"$object_symbols"; then + echo "Wrong-role workload dispatcher leaked into $object_path: $wrong_role_symbol" >&2 + exit 1 + fi +} +check_workload_dispatcher_object \ + "$BUILD_DIR/pa_scheduler_aic.o" \ + pa_execute_real_winner_workload_aic \ + pa_execute_real_winner_workload_aiv +check_workload_dispatcher_object \ + "$BUILD_DIR/pa_scheduler_aiv.o" \ + pa_execute_real_winner_workload_aiv \ + pa_execute_real_winner_workload_aic +echo "[CHECK] role-specific real-compute dispatchers are strong and do not cross roles" + +text_relocation_count_for_symbol() { + local object_path="$1" + local symbol_name="$2" + "$READELF_BIN" --relocs --wide "$object_path" | awk -v name="$symbol_name" ' + /^Relocation section '\''\.rela\.text'\''/ {in_text = 1; next} + /^Relocation section / {in_text = 0} + in_text { + for (column = 1; column <= NF; ++column) { + if ($column == name) { + count++ + next + } + } + } + END {print count + 0} + ' +} + +check_split_role_objects() { + local role="$1" + local wrong_role="$2" + local caller="$BUILD_DIR/pa_scheduler_${role}.o" + local runtime="$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_${role}.o" + local finish="$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_${role}.o" + local state_symbol="pa_scheduler_lazy_sample_callback_state_${role}" + local finish_symbol="pa_scheduler_lazy_sample_callback_finish_${role}" + local orchestration_symbol="pa_scheduler_lazy_sample_callback_orchestration_${role}" + local dispatcher_symbol="pa_execute_real_winner_workload_${role}" + local entry_symbol="pa_scheduler_0_mix_${role}" + local caller_symbols runtime_symbols finish_symbols + caller_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$caller")" + runtime_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$runtime")" + finish_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$finish")" + + if ! awk -v name="$orchestration_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$caller_symbols"; then + echo "Missing unique strong split orchestration in caller object: $orchestration_symbol" >&2 + exit 1 + fi + for imported in "$state_symbol" "$finish_symbol"; do + if ! awk -v name="$imported" \ + '$5 == "GLOBAL" && $7 == "UND" && $NF == name {count++} + END {exit count != 1}' <<<"$caller_symbols"; then + echo "Caller must import exactly one matching split symbol: $caller ($imported)" >&2 + exit 1 + fi + done + if [[ "$(text_relocation_count_for_symbol "$caller" "$finish_symbol")" -ne "$LAZY_SAMPLE_FINISH_CALL_SITES" ]]; then + echo "Caller must contain exactly $LAZY_SAMPLE_FINISH_CALL_SITES all-task split-finish .rela.text relocations: $caller" >&2 + exit 1 + fi + if [[ "$(text_relocation_count_for_symbol "$caller" "$state_symbol")" -eq 0 ]]; then + echo "Caller must access its matching external block-local state: $caller" >&2 + exit 1 + fi + if "$READELF_BIN" --sections --wide "$caller" | awk \ + 'index($0, ".ascend.meta.") != 0 {found = 1} END {exit !found}'; then + echo "Split caller object must not define launch metadata: $caller" >&2 + exit 1 + fi + + if ! awk -v name="$state_symbol" -v bytes="$LAZY_SAMPLE_SPLIT_STATE_BYTES" \ + '$4 == "OBJECT" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 == bytes {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime must own one exact-size block-local state: $runtime ($state_symbol)" >&2 + exit 1 + fi + if ! awk -v name="$entry_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime must own one non-empty mixed entry: $runtime ($entry_symbol)" >&2 + exit 1 + fi + if ! awk -v name="$orchestration_symbol" \ + '$5 == "GLOBAL" && $7 == "UND" && $NF == name {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime must strongly import one role-specific orchestration: $runtime ($orchestration_symbol)" >&2 + exit 1 + fi + if [[ "$(text_relocation_count_for_symbol "$runtime" "$orchestration_symbol")" -ne 1 ]]; then + echo "Runtime entry must contain exactly one orchestration call relocation: $runtime" >&2 + exit 1 + fi + local block_local_record block_local_section_index block_local_size_hex block_local_alignment + block_local_record="$( + "$READELF_BIN" --sections --wide "$runtime" | awk ' + {for (column = 1; column <= NF; ++column) { + if ($column == ".bl.uninit") { + section_index = $(column - 1) + gsub(/\[/, "", section_index) + gsub(/\]/, "", section_index) + print section_index, $(column + 4), $NF + exit + } + }} + ' + )" + read -r block_local_section_index block_local_size_hex block_local_alignment \ + <<<"$block_local_record" + if [[ -z "$block_local_section_index" || -z "$block_local_size_hex" || + $((16#$block_local_size_hex)) -ne "$LAZY_SAMPLE_SPLIT_STATE_BYTES" || + "$block_local_alignment" -ne 64 ]]; then + echo "Runtime block-local section must be exact-size and 64B aligned: $runtime" >&2 + exit 1 + fi + if ! awk -v name="$state_symbol" -v section="$block_local_section_index" \ + '$4 == "OBJECT" && $7 == section && $NF == name {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime state must be defined in its exact .bl.uninit section: $runtime" >&2 + exit 1 + fi + local runtime_sections + runtime_sections="$("$READELF_BIN" --sections --wide "$runtime")" + if ! awk -v name=".ascend.meta.$entry_symbol" ' + {for (column = 1; column <= NF; ++column) { + if ($column == name) found = 1 + }} + END {exit !found} + ' <<<"$runtime_sections"; then + echo "Runtime object is missing matching mixed-entry metadata: $runtime" >&2 + exit 1 + fi + if awk -v name=".ascend.meta.pa_scheduler_0_mix_${wrong_role}" ' + {for (column = 1; column <= NF; ++column) { + if ($column == name) found = 1 + }} + END {exit !found} + ' <<<"$runtime_sections"; then + echo "Wrong-role mixed-entry metadata leaked into runtime object: $runtime" >&2 + exit 1 + fi + + if ! awk -v name="$finish_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$finish_symbols"; then + echo "Finish object must define one non-empty strong finish: $finish ($finish_symbol)" >&2 + exit 1 + fi + for imported in "$state_symbol" "$dispatcher_symbol"; do + if ! awk -v name="$imported" \ + '$5 == "GLOBAL" && $7 == "UND" && $NF == name {count++} + END {exit count != 1}' <<<"$finish_symbols"; then + echo "Finish object must import exactly one matching symbol: $finish ($imported)" >&2 + exit 1 + fi + if [[ "$(text_relocation_count_for_symbol "$finish" "$imported")" -eq 0 ]]; then + echo "Finish object must reference its matching imported symbol: $finish ($imported)" >&2 + exit 1 + fi + done + if "$READELF_BIN" --sections --wide "$finish" | awk \ + 'index($0, ".ascend.meta.") != 0 {found = 1} END {exit !found}'; then + echo "Split finish object must not define launch metadata: $finish" >&2 + exit 1 + fi + + local forbidden symbol_table object_path + for object_path in "$caller" "$runtime" "$finish"; do + case "$object_path" in + "$caller") symbol_table="$caller_symbols" ;; + "$runtime") symbol_table="$runtime_symbols" ;; + *) symbol_table="$finish_symbols" ;; + esac + for forbidden in \ + "pa_scheduler_lazy_sample_callback_state_${wrong_role}" \ + "pa_scheduler_lazy_sample_callback_finish_${wrong_role}" \ + "pa_scheduler_lazy_sample_callback_orchestration_${wrong_role}" \ + "pa_execute_real_winner_workload_${wrong_role}" \ + "pa_scheduler_0_mix_${wrong_role}"; do + if awk -v name="$forbidden" \ + '$NF == name {found = 1} END {exit !found}' <<<"$symbol_table"; then + echo "Wrong-role split symbol leaked into $object_path: $forbidden" >&2 + exit 1 + fi + done + done + + for forbidden in "$entry_symbol"; do + if awk -v name="$forbidden" '$NF == name {found = 1} END {exit !found}' \ + <<<"$caller_symbols"; then + echo "Split caller must not own a launch entry: $caller ($forbidden)" >&2 + exit 1 + fi + if awk -v name="$forbidden" '$NF == name {found = 1} END {exit !found}' \ + <<<"$finish_symbols"; then + echo "Split finish must not own a launch entry: $finish ($forbidden)" >&2 + exit 1 + fi + done + for forbidden in "$finish_symbol" "$dispatcher_symbol"; do + if awk -v name="$forbidden" '$NF == name {found = 1} END {exit !found}' \ + <<<"$runtime_symbols"; then + echo "Runtime entry/state owner contains an unexpected split helper: $runtime ($forbidden)" >&2 + exit 1 + fi + done + if awk -v name="$orchestration_symbol" '$NF == name {found = 1} END {exit !found}' \ + <<<"$finish_symbols"; then + echo "Split finish must not contain orchestration: $finish ($orchestration_symbol)" >&2 + exit 1 + fi +} + +if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + echo "[BUILD] CCEC AIC split runtime entry/state owner (dav-c310-cube)" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DPA_BUILD_AIC \ + -o "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_aic.o" \ + "$SCRIPT_DIR/callback_runtime_entry.cpp" + echo "[BUILD] CCEC AIC all-task split finish (dav-c310-cube)" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DPA_BUILD_AIC \ + -o "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_aic.o" \ + "$SCRIPT_DIR/callback_finish.cpp" + echo "[BUILD] CCEC AIV split runtime entry/state owner (dav-c310-vec)" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DPA_BUILD_AIV \ + -o "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_aiv.o" \ + "$SCRIPT_DIR/callback_runtime_entry.cpp" + echo "[BUILD] CCEC AIV all-task split finish (dav-c310-vec)" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DPA_BUILD_AIV \ + -o "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_aiv.o" \ + "$SCRIPT_DIR/callback_finish.cpp" + check_split_role_objects aic aiv + check_split_role_objects aiv aic + echo "[CHECK] split caller/runtime/finish objects satisfy role, state, metadata, and call-boundary gates" + DEVICE_OBJECTS=( + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_aic.o" + "$BUILD_DIR/pa_scheduler_aic.o" + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_aic.o" + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_aiv.o" + "$BUILD_DIR/pa_scheduler_aiv.o" + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_aiv.o" + ) +else + DEVICE_OBJECTS=( + "$BUILD_DIR/pa_scheduler_aic.o" + "$BUILD_DIR/pa_scheduler_aiv.o" + ) +fi + +# 静态链接把两个 device object 合成一个可由 runtime 按 1:2 比例启动的 mixed AICore ELF。 +echo "[BUILD] Static 1:2 mixed AICore ELF" +"$LD" -m aicorelinux -Ttext=0 -static \ + --version-script="$SCRIPT_DIR/pa_scheduler_device_exports.map" \ + -o "$BUILD_DIR/pa_scheduler_kernel.o" \ + "${DEVICE_OBJECTS[@]}" + +SYMBOL_TABLE="$("$READELF_BIN" --symbols --wide --sym-base=10 "$BUILD_DIR/pa_scheduler_kernel.o")" +SECTION_TABLE="$("$READELF_BIN" --sections --wide "$BUILD_DIR/pa_scheduler_kernel.o")" +# 构建成功不等于 mixed launch 可用：同时检查两个入口符号及其 metadata section，缺一即拒绝产物。 +# `set -e` 同时保证 readelf 自身失败时不会拿空字符串继续做伪检查。 +for entry in pa_scheduler_0_mix_aic pa_scheduler_0_mix_aiv; do + if ! awk -v name="$entry" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 != "0" && $3 != "0x0" {found = 1} END {exit !found}' \ + <<<"$SYMBOL_TABLE"; then + echo "Missing non-empty defined GLOBAL mixed-kernel entry: $entry" >&2 + exit 1 + fi + if ! awk -v name=".ascend.meta.$entry" ' + {for (column = 1; column <= NF; ++column) { + if ($column == name) found = 1 + }} + END {exit !found} + ' <<<"$SECTION_TABLE"; then + echo "Missing mixed-kernel metadata section: .ascend.meta.$entry" >&2 + exit 1 + fi +done +echo "[CHECK] both 1:2 mixed entries and metadata sections are present" + +# A5 runtime 会把已定义的 GLOBAL FUNC 当作可启动候选；最终 device ELF 只允许 +# 两个带 metadata 的 mixed 入口暴露为全局函数。任何新增 helper 都必须保持 LOCAL。 +while IFS= read -r global_func; do + case "$global_func" in + pa_scheduler_0_mix_aic|pa_scheduler_0_mix_aiv) ;; + *) + echo "Unexpected GLOBAL device function (possible kernel-entry pollution): $global_func" >&2 + exit 1 + ;; + esac +done < <(awk '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" {print $NF}' <<<"$SYMBOL_TABLE") +echo "[CHECK] only the two mixed entries are exported as GLOBAL device functions" + +# runtime-finish TU 后续需要复用同一个真计算 dispatcher，因此 caller object +# 按核型提供 strong 定义；version script 必须把它们在最终 mixed ELF 中重新 +# 局部化。这里同时检查两个 role-specific dispatcher 与底层 Cube/Vector 实体， +# 禁止因抽取 adapter 漏掉任一真实负载路径。 +for workload_symbol in \ + pa_execute_real_winner_workload_aic \ + pa_execute_real_winner_workload_aiv \ + pa_real_cube_workload_aic \ + pa_real_vector_add_workload_aiv \ + pa_real_vector_mul_workload_aiv; do + workload_size="$( + awk -v name="$workload_symbol" \ + '$4 == "FUNC" && $5 == "LOCAL" && $7 != "UND" && index($NF, name) != 0 && $3 + 0 > 0 {print $3; exit}' \ + <<<"$SYMBOL_TABLE" + )" + if [[ -z "$workload_size" ]]; then + echo "Missing non-empty LOCAL CCEC real-compute workload function: $workload_symbol" >&2 + exit 1 + fi + if awk -v name="$workload_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && index($NF, name) != 0 {found = 1} END {exit !found}' \ + <<<"$SYMBOL_TABLE"; then + echo "CCEC real-compute helper must not be a GLOBAL kernel candidate: $workload_symbol" >&2 + exit 1 + fi +done +echo "[CHECK] CCEC cube/vector real-compute helpers are non-empty LOCAL functions" + +if [[ "$LAZY_SAMPLE_VARIANT" -eq 1 ]]; then + # callback/builder/front must remain inline in both families. Only split + # shapes may retain the two fixed, role-specific runtime finish functions. + if awk \ + '$4 == "FUNC" && $7 != "UND" && + (index($NF, "LazySampleCallback") != 0 || index($NF, "SubmitQk") != 0 || + index($NF, "LazySampleSplitState") != 0) {found = 1} + END {exit !found}' <<<"$SYMBOL_TABLE"; then + echo "lazy sample callback builder/front unexpectedly survived as an out-of-line device function." >&2 + exit 1 + fi + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + for role in aic aiv; do + finish_symbol="pa_scheduler_lazy_sample_callback_finish_${role}" + if ! awk -v name="$finish_symbol" \ + '$4 == "FUNC" && $5 == "LOCAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Missing unique non-empty LOCAL split finish in final ELF: $finish_symbol" >&2 + exit 1 + fi + orchestration_symbol="pa_scheduler_lazy_sample_callback_orchestration_${role}" + if ! awk -v name="$orchestration_symbol" \ + '$4 == "FUNC" && $5 == "LOCAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Missing unique non-empty LOCAL split orchestration in final ELF: $orchestration_symbol" >&2 + exit 1 + fi + state_symbol="pa_scheduler_lazy_sample_callback_state_${role}" + if ! awk -v name="$state_symbol" -v bytes="$LAZY_SAMPLE_SPLIT_STATE_BYTES" \ + '$4 == "OBJECT" && $5 == "LOCAL" && $7 != "UND" && $NF == name && $3 + 0 == bytes {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Missing unique exact-size LOCAL split state in final ELF: $state_symbol" >&2 + exit 1 + fi + done + aic_state_hex="$(awk '$NF == "pa_scheduler_lazy_sample_callback_state_aic" {print $2; exit}' <<<"$SYMBOL_TABLE")" + aiv_state_hex="$(awk '$NF == "pa_scheduler_lazy_sample_callback_state_aiv" {print $2; exit}' <<<"$SYMBOL_TABLE")" + final_block_local_record="$( + awk '{for (column = 1; column <= NF; ++column) { + if ($column == ".bl_uninit") { + section_index = $(column - 1) + gsub(/\[/, "", section_index) + gsub(/\]/, "", section_index) + print section_index, $(column + 4), $NF + exit + } + }}' <<<"$SECTION_TABLE" + )" + read -r final_block_local_section_index final_block_local_size_hex \ + final_block_local_alignment \ + <<<"$final_block_local_record" + if [[ -z "$aic_state_hex" || -z "$aiv_state_hex" || + $((16#$aic_state_hex)) -ne 0 || + $((16#$aiv_state_hex)) -ne "$LAZY_SAMPLE_SPLIT_STATE_BYTES" || + -z "$final_block_local_section_index" || -z "$final_block_local_size_hex" || + $((16#$final_block_local_size_hex)) -ne $((2 * LAZY_SAMPLE_SPLIT_STATE_BYTES)) || + "$final_block_local_alignment" -ne 64 ]]; then + echo "Final split block-local layout must be two exact, non-overlapping 64B-aligned states." >&2 + exit 1 + fi + for state_symbol in \ + pa_scheduler_lazy_sample_callback_state_aic \ + pa_scheduler_lazy_sample_callback_state_aiv; do + if ! awk -v name="$state_symbol" -v section="$final_block_local_section_index" \ + '$4 == "OBJECT" && $7 == section && $NF == name {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Final split state must be bound to the exact .bl_uninit section: $state_symbol" >&2 + exit 1 + fi + done + echo "[CHECK] split finishes/orchestrations/states are LOCAL and final block-local layout is exact" + fi + if [[ -n "$("$READELF_BIN" --relocs --wide "$BUILD_DIR/pa_scheduler_kernel.o" | sed -n '/Relocation section/p')" ]]; then + echo "Final lazy sample callback mixed ELF must not retain relocations." >&2 + exit 1 + fi + echo "[CHECK] lazy sample callback builder/front is inline and final ELF has no relocations" +fi + +check_icache_probe_layout() { + local role="$1" + local target="pa_icache_target_${role}" + local harness="pa_icache_measure_${role}" + local thrash="pa_icache_thrash_${role}" + local target_record + local harness_record + local thrash_record + target_record="$(awk -v name="$target" '$4 == "FUNC" && index($NF, name) != 0 {print $2, $3; exit}' <<<"$SYMBOL_TABLE")" + harness_record="$(awk -v name="$harness" '$4 == "FUNC" && index($NF, name) != 0 {print $2, $3; exit}' <<<"$SYMBOL_TABLE")" + thrash_record="$(awk -v name="$thrash" '$4 == "FUNC" && index($NF, name) != 0 {print $2, $3; exit}' <<<"$SYMBOL_TABLE")" + if [[ -z "$target_record" || -z "$harness_record" || -z "$thrash_record" ]]; then + echo "Missing I-cache probe symbols for $role" >&2 + exit 1 + fi + + local target_hex target_size harness_hex harness_size thrash_hex thrash_size + read -r target_hex target_size <<<"$target_record" + read -r harness_hex harness_size <<<"$harness_record" + read -r thrash_hex thrash_size <<<"$thrash_record" + local target_address=$((16#$target_hex)) + local harness_address=$((16#$harness_hex)) + local thrash_address=$((16#$thrash_hex)) + if (( target_address % 128 != 0 || target_size == 0 || target_size > 16 )); then + echo "Invalid single-fetch-block I-cache target for $role: address=0x$target_hex size=$target_size" >&2 + exit 1 + fi + if (( thrash_size < 65536 )); then + echo "I-cache thrash body is smaller than 64 KiB for $role: size=$thrash_size" >&2 + exit 1 + fi + if (( harness_address % 128 != 0 || target_address + 128 > harness_address || + harness_address + harness_size > thrash_address )); then + echo "I-cache layout must be target -> harness -> thrash for $role" >&2 + exit 1 + fi + echo "[CHECK] $role I-cache target=0x$target_hex/$target_size harness=0x$harness_hex/$harness_size "\ + "thrash=0x$thrash_hex/$thrash_size" +} + +emit_text_section_fingerprint() { + local object_path="$1" + local artifact_name + artifact_name="$(basename "$object_path")" + local text_record text_address_hex text_offset_hex text_size_hex + text_record="$( + "$READELF_BIN" --sections --wide "$object_path" | awk ' + {for (column = 1; column <= NF; ++column) { + if ($column == ".text") { + print $(column + 2), $(column + 3), $(column + 4) + exit + } + }} + ' + )" + read -r text_address_hex text_offset_hex text_size_hex <<<"$text_record" + if [[ -z "$text_address_hex" || -z "$text_offset_hex" || -z "$text_size_hex" ]]; then + echo "Cannot fingerprint missing .text section: $object_path" >&2 + return 1 + fi + local text_size=$((16#$text_size_hex)) + local text_sha + text_sha="$( + dd if="$object_path" bs=1 skip=$((16#$text_offset_hex)) count="$text_size" status=none | + sha256sum | awk '{print $1}' + )" + printf 'text %s %u %s\n' "$artifact_name" "$text_size" "$text_sha" +} + +emit_symbol_body_fingerprint() { + local object_path="$1" + local symbol_name="$2" + local artifact_name + artifact_name="$(basename "$object_path")" + local symbol_record symbol_address_hex symbol_size + symbol_record="$( + "$READELF_BIN" --symbols --wide --sym-base=10 "$object_path" | awk -v name="$symbol_name" ' + $4 == "FUNC" && $7 != "UND" && $NF == name && $3 + 0 > 0 { + count++ + address = $2 + size = $3 + } + END { + if (count != 1) exit 1 + print address, size + } + ' + )" || { + echo "Cannot fingerprint non-unique or empty function: $object_path ($symbol_name)" >&2 + return 1 + } + read -r symbol_address_hex symbol_size <<<"$symbol_record" + + local text_record text_address_hex text_offset_hex text_size_hex + text_record="$( + "$READELF_BIN" --sections --wide "$object_path" | awk ' + {for (column = 1; column <= NF; ++column) { + if ($column == ".text") { + print $(column + 2), $(column + 3), $(column + 4) + exit + } + }} + ' + )" + read -r text_address_hex text_offset_hex text_size_hex <<<"$text_record" + if [[ -z "$text_address_hex" || -z "$text_offset_hex" || -z "$text_size_hex" ]]; then + echo "Cannot locate .text for function fingerprint: $object_path ($symbol_name)" >&2 + return 1 + fi + local symbol_address=$((16#$symbol_address_hex)) + local text_address=$((16#$text_address_hex)) + local text_size=$((16#$text_size_hex)) + local relative_offset=$((symbol_address - text_address)) + if (( relative_offset < 0 || symbol_size <= 0 || relative_offset + symbol_size > text_size )); then + echo "Function body lies outside .text: $object_path ($symbol_name)" >&2 + return 1 + fi + local symbol_sha + symbol_sha="$( + dd if="$object_path" bs=1 \ + skip=$((16#$text_offset_hex + relative_offset)) \ + count="$symbol_size" status=none | + sha256sum | awk '{print $1}' + )" + printf 'symbol %s %s %u %s\n' \ + "$artifact_name" "$symbol_name" "$symbol_size" "$symbol_sha" +} + +# 两个正式 ELF 都不携带旧 cold/warm 校准冲刷体；submit-pmu 只观察真实 +# Submit。保留上面的检查函数供历史布局取证时复核，但正式构建不调用它。 + +# PMU selector/CTRL 的所有权必须由主 aicpu_scheduler 配置并在退出前恢复。 +# standalone 目录内自带 Path-A dispatcher 与 owner：前者负责把 owner SO +# 落到设备预安装目录，后者由 mode=0 JSON 注册并通过统一入口执行命令。 +# swimlane 构建不生成 PMU owner/dispatcher；submit-pmu 则把 kernel、host、 +# owner 与 dispatcher 全部放在同一个 phase 目录，禁止跨 phase 复用。 +if [[ "$PMU_VARIANT" -eq 1 ]]; then + echo "[BUILD] self-contained AICPU PMU dispatcher" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra -Werror \ + -Wl,--build-id \ + "$SCRIPT_DIR/pmu_owner_dispatcher.cpp" \ + -o "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so" + + echo "[BUILD] self-contained AICPU PMU owner" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra -Werror \ + -Wl,--build-id \ + "${VARIANT_DEFINES[@]}" \ + -I"$SCRIPT_DIR" \ + "$SCRIPT_DIR/pmu_owner_aicpu.cpp" \ + -o "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so" + + OWNER_HEADER="$("$READELF_BIN" --file-header "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so")" + OWNER_SYMBOLS="$("$READELF_BIN" --dyn-syms --wide "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so")" + DISPATCHER_HEADER="$("$READELF_BIN" --file-header "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so")" + DISPATCHER_SYMBOLS="$("$READELF_BIN" --dyn-syms --wide "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so")" + if [[ "$OWNER_HEADER" != *"Type: DYN"* || + "$OWNER_HEADER" != *"Machine: AArch64"* || + "$DISPATCHER_HEADER" != *"Type: DYN"* || + "$DISPATCHER_HEADER" != *"Machine: AArch64"* ]]; then + echo "PMU dispatcher and owner must both be AArch64 shared objects." >&2 + exit 1 + fi + if [[ "$OWNER_SYMBOLS" != *" simpler_aicpu_exec"* ]]; then + echo "Missing main AICPU PMU owner entry: simpler_aicpu_exec" >&2 + exit 1 + fi + for entry in StaticTileFwkBackendKernelServer DynTileFwkBackendKernelServerInit DynTileFwkBackendKernelServer; do + if [[ "$DISPATCHER_SYMBOLS" != *" $entry"* ]]; then + echo "Missing AICPU PMU dispatcher entry: $entry" >&2 + exit 1 + fi + done + echo "[CHECK] Path-A dispatcher and main AICPU PMU owner exports are present" +fi + +# host runner 只链接用户 CANN 9.1 的 ACL/runtime，并写入同一安装目录的 rpath，运行时不需要 simpler 动态库。 +# `-Werror` 让 host API 签名或尺寸类型变化在构建期暴露，避免到上板阶段才出现参数截断。 +echo "[BUILD] CCEC host runner" +"$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror -Wno-deprecated-declarations \ + "${VARIANT_DEFINES[@]}" \ + -I"$ROOT_DIR/common" \ + -I"$ASCEND_HOME_PATH/include" \ + -I"$ASCEND_HOME_PATH/pkg_inc" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime/runtime" \ + "$SCRIPT_DIR/host.cpp" \ + -L"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -Wl,-rpath,"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -lascendcl -lruntime \ + -ldl \ + -o "$BUILD_DIR/pa_scheduler_host" + +if [[ "$PMU_VARIANT" -eq 1 ]]; then + # host、kernel、owner、dispatcher 全部成功后才生成 manifest；校验和使用 + # 相对文件名，目录复制后仍可在 run 前原样复核。临时文件与最终文件位于 + # 同一目录，mv 只承担单文件原子发布，不会暴露半写 manifest。 + SUBMIT_PMU_ARTIFACTS=( + pa_scheduler_host + pa_scheduler_kernel.o + libpa_scheduler_pmu_owner_aicpu.so + libpa_scheduler_pmu_owner_dispatcher.so + ) + for artifact in "${SUBMIT_PMU_ARTIFACTS[@]}"; do + if [[ ! -s "$BUILD_DIR/$artifact" ]]; then + echo "Cannot publish submit-pmu manifest; artifact is missing or empty: $artifact" >&2 + exit 1 + fi + done + if [[ ! -x "$BUILD_DIR/pa_scheduler_host" ]]; then + echo "Cannot publish submit-pmu manifest; host runner is not executable." >&2 + exit 1 + fi + + MANIFEST_PATH="$BUILD_DIR/$SUBMIT_PMU_MANIFEST_NAME" + MANIFEST_TMP="$(mktemp "$BUILD_DIR/.${SUBMIT_PMU_MANIFEST_NAME}.tmp.XXXXXX")" + cleanup_manifest_tmp() { + if [[ -n "${MANIFEST_TMP:-}" ]]; then + rm -f -- "$MANIFEST_TMP" + fi + } + trap cleanup_manifest_tmp EXIT + { + printf '# schema=pa_scheduler_submit_pmu_artifacts/v1\n' + printf '# variant=submit-pmu\n' + printf '# phase=%s\n' "$PHASE_NAME" + printf '# phase_id=%u\n' "$PHASE_ID" + (cd "$BUILD_DIR" && sha256sum "${SUBMIT_PMU_ARTIFACTS[@]}") + } > "$MANIFEST_TMP" + mv -f -- "$MANIFEST_TMP" "$MANIFEST_PATH" + MANIFEST_TMP="" + trap - EXIT + echo "[CHECK] submit-pmu artifact manifest published: $MANIFEST_PATH" +fi +if [[ "$LAZY_SAMPLE_VARIANT" -eq 1 ]]; then + TEXT_LAYOUT_PATH="$BUILD_DIR/$LAZY_SAMPLE_TEXT_LAYOUT_NAME" + TEXT_LAYOUT_TMP="$(mktemp "$BUILD_DIR/.${LAZY_SAMPLE_TEXT_LAYOUT_NAME}.tmp.XXXXXX")" + cleanup_text_layout_tmp() { + if [[ -n "${TEXT_LAYOUT_TMP:-}" ]]; then + rm -f -- "$TEXT_LAYOUT_TMP" + fi + } + trap cleanup_text_layout_tmp EXIT + { + printf '# schema=pa_scheduler_device_text_layout/v1\n' + printf '# backend=ccec\n' + printf '# shape=%s\n' "$LAZY_SAMPLE_SHAPE" + printf '# block_local_reserve_bytes=%u\n' "$LAZY_SAMPLE_BLOCK_LOCAL_RESERVE_BYTES" + printf '# compiler=%s\n' '$ASCEND_HOME_PATH/bin/ccec' + printf '# compiler_sha256=%s\n' "$(sha256sum "$CCEC" | awk '{print $1}')" + printf '# linker=%s\n' '$ASCEND_HOME_PATH/bin/ld.lld' + printf '# linker_sha256=%s\n' "$(sha256sum "$LD" | awk '{print $1}')" + emit_text_section_fingerprint "$BUILD_DIR/pa_scheduler_kernel.o" + for object_path in "${DEVICE_OBJECTS[@]}"; do + emit_text_section_fingerprint "$object_path" + done + for role in aic aiv; do + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_kernel.o" "pa_scheduler_0_mix_${role}" + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_${role}.o" \ + "pa_scheduler_0_mix_${role}" + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_${role}.o" \ + "pa_scheduler_lazy_sample_callback_orchestration_${role}" + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_${role}.o" \ + "pa_scheduler_lazy_sample_callback_finish_${role}" + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_kernel.o" \ + "pa_scheduler_lazy_sample_callback_orchestration_${role}" + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_kernel.o" \ + "pa_scheduler_lazy_sample_callback_finish_${role}" + else + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_${role}.o" "pa_scheduler_0_mix_${role}" + fi + done + } > "$TEXT_LAYOUT_TMP" + mv -f -- "$TEXT_LAYOUT_TMP" "$TEXT_LAYOUT_PATH" + TEXT_LAYOUT_TMP="" + trap - EXIT + awk '$1 == "text" { + printf "[TEXT] %s size=%s sha256=%s\n", $2, $3, $4 + }' "$TEXT_LAYOUT_PATH" + echo "[CHECK] lazy sample callback device .text layout manifest published: $TEXT_LAYOUT_PATH" + + LAZY_SAMPLE_ARTIFACTS=( + pa_scheduler_host + pa_scheduler_kernel.o + ) + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_lazy_sample_callback_runtime_aic.o) + fi + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_aic.o) + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_lazy_sample_callback_finish_aic.o) + fi + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_lazy_sample_callback_runtime_aiv.o) + fi + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_aiv.o) + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_lazy_sample_callback_finish_aiv.o) + fi + LAZY_SAMPLE_ARTIFACTS+=("$LAZY_SAMPLE_TEXT_LAYOUT_NAME") + for artifact in "${LAZY_SAMPLE_ARTIFACTS[@]}"; do + if [[ ! -s "$BUILD_DIR/$artifact" ]]; then + echo "Cannot publish lazy sample callback manifest; artifact is missing or empty: $artifact" >&2 + exit 1 + fi + done + if [[ ! -x "$BUILD_DIR/pa_scheduler_host" ]]; then + echo "Cannot publish lazy sample callback manifest; host runner is not executable." >&2 + exit 1 + fi + MANIFEST_PATH="$BUILD_DIR/$LAZY_SAMPLE_MANIFEST_NAME" + MANIFEST_TMP="$(mktemp "$BUILD_DIR/.${LAZY_SAMPLE_MANIFEST_NAME}.tmp.XXXXXX")" + cleanup_callback_manifest_tmp() { + if [[ -n "${MANIFEST_TMP:-}" ]]; then + rm -f -- "$MANIFEST_TMP" + fi + } + trap cleanup_callback_manifest_tmp EXIT + { + printf '# schema=pa_scheduler_lazy_sample_callback_artifacts/v1\n' + printf '# backend=ccec\n' + printf '# shape=%s\n' "$LAZY_SAMPLE_SHAPE" + printf '# shape_id=%u\n' "$LAZY_SAMPLE_SHAPE_ID" + printf '# observation=%s\n' "$LAZY_SAMPLE_OBSERVATION" + (cd "$BUILD_DIR" && sha256sum "${LAZY_SAMPLE_ARTIFACTS[@]}") + } > "$MANIFEST_TMP" + mv -f -- "$MANIFEST_TMP" "$MANIFEST_PATH" + MANIFEST_TMP="" + trap - EXIT + echo "[CHECK] lazy sample callback artifact manifest published: $MANIFEST_PATH" +fi + +echo "[BUILD] complete: $BUILD_DIR" diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/callback_finish.cpp b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/callback_finish.cpp new file mode 100644 index 0000000000..9422fef200 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/callback_finish.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include +#include +#include + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" +#include "ccec_ops.h" + +using pa_scheduler_ccec::CcecOps; + +extern "C" { +#if defined(PA_BUILD_AIC) +__attribute__((noinline)) __aicore__ uint32_t +pa_scheduler_lazy_sample_callback_finish_aic(const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args) { + return pa_scheduler::FinishSplitLazySampleCallbackFromRuntime(ticket, args); +} +#elif defined(PA_BUILD_AIV) +__attribute__((noinline)) __aicore__ uint32_t +pa_scheduler_lazy_sample_callback_finish_aiv(const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args) { + return pa_scheduler::FinishSplitLazySampleCallbackFromRuntime(ticket, args); +} +#else +#error "Compile split finish with PA_BUILD_AIC or PA_BUILD_AIV" +#endif +} diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/callback_finish_api.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/callback_finish_api.h new file mode 100644 index 0000000000..9c87d77675 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/callback_finish_api.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#ifndef TESTS_ATOMIC_PROBE_LAZY_LAMDA_SAMPLE_CCEC_CALLBACK_FINISH_API_H +#define TESTS_ATOMIC_PROBE_LAZY_LAMDA_SAMPLE_CCEC_CALLBACK_FINISH_API_H + +#if !defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +#error "callback_finish_api.h is only valid for split-finish artifacts" +#endif + +// The runtime TU owns the role-specific block-local object. The caller only +// imports it; no weak/generic state symbol is allowed in the mixed AIC/AIV ELF. +extern "C" { +#if defined(PA_BUILD_AIC) +[[block_local]] extern pa_scheduler::LazySampleSplitRuntimeState pa_scheduler_lazy_sample_callback_state_aic; +__aicore__ uint32_t +pa_scheduler_lazy_sample_callback_finish_aic(const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args); +#elif defined(PA_BUILD_AIV) +[[block_local]] extern pa_scheduler::LazySampleSplitRuntimeState pa_scheduler_lazy_sample_callback_state_aiv; +__aicore__ uint32_t +pa_scheduler_lazy_sample_callback_finish_aiv(const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args); +#else +#error "Compile split finish with PA_BUILD_AIC or PA_BUILD_AIV" +#endif +} + +#endif // TESTS_ATOMIC_PROBE_LAZY_LAMDA_SAMPLE_CCEC_CALLBACK_FINISH_API_H diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/callback_runtime_entry.cpp b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/callback_runtime_entry.cpp new file mode 100644 index 0000000000..56adcd3a71 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/callback_runtime_entry.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" + +extern "C" { +#if defined(PA_BUILD_AIC) +[[block_local]] pa_scheduler::LazySampleSplitRuntimeState pa_scheduler_lazy_sample_callback_state_aic; +__aicore__ void +pa_scheduler_lazy_sample_callback_orchestration_aic(__gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id); +#elif defined(PA_BUILD_AIV) +[[block_local]] pa_scheduler::LazySampleSplitRuntimeState pa_scheduler_lazy_sample_callback_state_aiv; +__aicore__ void +pa_scheduler_lazy_sample_callback_orchestration_aiv(__gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id); +#else +#error "Compile split runtime entry with PA_BUILD_AIC or PA_BUILD_AIV" +#endif +} + +#if defined(PA_BUILD_AIC) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aic, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aic(__gm__ pa_scheduler::SchedulerState *state) { + const uint32_t worker_id = static_cast(get_block_idx()); + pa_scheduler_lazy_sample_callback_orchestration_aic(state, worker_id); +} +#elif defined(PA_BUILD_AIV) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aiv, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aiv(__gm__ pa_scheduler::SchedulerState *state) { + const uint32_t vector_id = static_cast(get_block_idx() * get_subblockdim() + get_subblockid()); + const uint32_t worker_id = pa_scheduler::kAicWorkers + vector_id; + pa_scheduler_lazy_sample_callback_orchestration_aiv(state, worker_id); +} +#endif diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/ccec_ops.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/ccec_ops.h new file mode 100644 index 0000000000..9a64d8bba1 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/ccec_ops.h @@ -0,0 +1,408 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#ifndef TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CCEC_OPS_H +#define TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CCEC_OPS_H + +// This private header is included after the CCEC/PTO headers and +// common/pa_scheduler_core.h. kernel.cpp defines the implementation switch +// exactly once per architecture; split runtime TUs consume only the external +// dispatcher declaration and the shared inline adapter. +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +#include "callback_finish_api.h" +#endif + +#if defined(PA_BUILD_AIC) +extern "C" __attribute__((noinline)) __aicore__ void pa_execute_real_winner_workload_aic( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +); +#elif defined(PA_BUILD_AIV) +extern "C" __attribute__((noinline)) __aicore__ void pa_execute_real_winner_workload_aiv( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +); +#else +#error "Compile CcecOps with PA_BUILD_AIC or PA_BUILD_AIV" +#endif + +namespace pa_scheduler_ccec { + +using namespace pto; + +template +__aicore__ inline void EmitNops() { +#pragma unroll + for (uint32_t index = 0; index < Count; ++index) { + asm volatile("nop"); + } +} + +__aicore__ inline void RuntimeNop(uint32_t count) { + // 两侧全流水屏障把可调 NOP 段限定为 kernel 模拟体，避免前后调度访存进入被测计算区间。 + __builtin_cce_pipe_barrier(PIPE_ALL); + while (count >= 256) { + EmitNops<256>(); + count -= 256; + } + if ((count & 128U) != 0) EmitNops<128>(); + if ((count & 64U) != 0) EmitNops<64>(); + if ((count & 32U) != 0) EmitNops<32>(); + if ((count & 16U) != 0) EmitNops<16>(); + if ((count & 8U) != 0) EmitNops<8>(); + if ((count & 4U) != 0) EmitNops<4>(); + if ((count & 2U) != 0) EmitNops<2>(); + if ((count & 1U) != 0) EmitNops<1>(); + __builtin_cce_pipe_barrier(PIPE_ALL); +} + +#if defined(PA_CCEC_OPS_DEFINE_REAL_WORKLOAD) && defined(PA_BUILD_AIC) +// QK/PV 的首版真实负载使用完整的 128x128 float cube 路径。输入来自独立 GM +// workspace，输出属于当前 worker；每次迭代都等 FIX 写回 GM 后再复用 L0C， +// 因而函数返回就是该模拟 task 的完成边界，而不是单纯的指令发射边界。 +static __aicore__ __attribute__((noinline, used)) void pa_real_cube_workload_aic( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + constexpr int kTile = static_cast(pa_scheduler::winner_workload::kTileRows); + constexpr int kBlockAlign = C0_SIZE_BYTE / sizeof(float); + static_assert(kTile % 16 == 0, "cube M must be 16-aligned"); + static_assert(kTile % kBlockAlign == 0, "cube K/N must satisfy C0 alignment"); + + using GlobalData = GlobalTensor< + float, Shape<1, 1, 1, kTile, kTile>, + pto::Stride>; + using TileMatA = Tile< + TileType::Mat, float, kTile, kTile, BLayout::ColMajor, + kTile, kTile, SLayout::RowMajor, 512>; + using TileMatB = Tile< + TileType::Mat, float, kTile, kTile, BLayout::ColMajor, + kTile, kTile, SLayout::RowMajor, 512>; + using LeftTile = TileLeft; + using RightTile = TileRight; + using AccTile = TileAcc; + + GlobalData input_a_global(input_a); + GlobalData input_b_global(input_b); + GlobalData output_global(output); + TileMatA input_a_mat; + TileMatB input_b_mat; + LeftTile input_a_l0; + RightTile input_b_l0; + AccTile output_l0; + TASSIGN(input_a_mat, 0x0); + TASSIGN(input_b_mat, 0x20000); + TASSIGN(input_a_l0, 0x0); + TASSIGN(input_b_l0, 0x0); + TASSIGN(output_l0, 0x0); + + for (uint32_t iteration = 0; iteration < repeats; ++iteration) { + TLOAD(input_a_mat, input_a_global); + TLOAD(input_b_mat, input_b_global); + set_flag(PIPE_MTE2, PIPE_MTE1, EVENT_ID0); + wait_flag(PIPE_MTE2, PIPE_MTE1, EVENT_ID0); + TMOV(input_a_l0, input_a_mat); + TMOV(input_b_l0, input_b_mat); + set_flag(PIPE_MTE1, PIPE_M, EVENT_ID0); + wait_flag(PIPE_MTE1, PIPE_M, EVENT_ID0); + TMATMUL(output_l0, input_a_l0, input_b_l0); + set_flag(PIPE_M, PIPE_FIX, EVENT_ID0); + wait_flag(PIPE_M, PIPE_FIX, EVENT_ID0); + TSTORE(output_global, output_l0); + set_flag(PIPE_FIX, PIPE_S, EVENT_ID7); + wait_flag(PIPE_FIX, PIPE_S, EVENT_ID7); + } +} +#elif defined(PA_CCEC_OPS_DEFINE_REAL_WORKLOAD) && defined(PA_BUILD_AIV) +template +__aicore__ inline void RunRealVectorWorkload( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + constexpr int kRows = static_cast(pa_scheduler::winner_workload::kTileRows); + constexpr int kCols = static_cast(pa_scheduler::winner_workload::kTileCols); + using GlobalData = GlobalTensor< + float, Shape<1, 1, 1, kRows, kCols>, pto::Stride<1, 1, 1, kCols, 1>>; + using TileData = Tile< + TileType::Vec, float, kRows, kCols, BLayout::RowMajor, -1, -1>; + + GlobalData input_a_global(input_a); + GlobalData input_b_global(input_b); + GlobalData output_global(output); + TileData input_a_tile(kRows, kCols); + TileData input_b_tile(kRows, kCols); + TileData output_tile(kRows, kCols); + TASSIGN(input_a_tile, 0x0); + TASSIGN(input_b_tile, 0x10000); + TASSIGN(output_tile, 0x20000); + + for (uint32_t iteration = 0; iteration < repeats; ++iteration) { + TLOAD(input_a_tile, input_a_global); + TLOAD(input_b_tile, input_b_global); + set_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + wait_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + if constexpr (Multiply) { + TMUL(output_tile, input_a_tile, input_b_tile); + } else { + TADD(output_tile, input_a_tile, input_b_tile); + } + set_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + wait_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + TSTORE(output_global, output_tile); + set_flag(PIPE_MTE3, PIPE_S, EVENT_ID7); + wait_flag(PIPE_MTE3, PIPE_S, EVENT_ID7); + } +} + +static __aicore__ __attribute__((noinline, used)) void pa_real_vector_add_workload_aiv( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + RunRealVectorWorkload(input_a, input_b, output, repeats); +} + +static __aicore__ __attribute__((noinline, used)) void pa_real_vector_mul_workload_aiv( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + RunRealVectorWorkload(input_a, input_b, output, repeats); +} +#endif + +#if PA_BUILD_SUBMIT_PMU +struct SubmitPmuContext; +#endif + +struct CcecOps { + static constexpr bool kAtomicReturnReadyObserved = true; + + // 该适配层把平台无关调度器需要的原子、计时、NOP 和 cache 操作逐一映射到 CCEC intrinsic。 + // A5 上 PA 的共享“读取”使用 atomicAdd(addr, 0)，不是普通 GM load；这里保留其 RMW 竞争语义。 + __aicore__ static inline int32_t Load(__gm__ volatile int32_t *address) { + // atomicAdd 返回加法发生前的值；加数为 0，因此它就是本次共享读取的结果。 + return atomicAdd(const_cast<__gm__ int32_t *>(address), static_cast(0)); + } + + __aicore__ static inline int64_t Load(__gm__ volatile int64_t *address) { + return atomicAdd(const_cast<__gm__ int64_t *>(address), static_cast(0)); + } + + __aicore__ static inline uint64_t Load(__gm__ volatile uint64_t *address) { + return atomicAdd(const_cast<__gm__ uint64_t *>(address), static_cast(0)); + } + + __aicore__ static inline int32_t Exchange(__gm__ volatile int32_t *address, int32_t value) { + // atomicExch 同样返回旧值；当前 completion/fatal 发布只需要其原子写入副作用。 + return atomicExch(const_cast<__gm__ int32_t *>(address), value); + } + + __aicore__ static inline int64_t Exchange(__gm__ volatile int64_t *address, int64_t value) { + return atomicExch(const_cast<__gm__ int64_t *>(address), value); + } + + __aicore__ static inline uint64_t Exchange(__gm__ volatile uint64_t *address, uint64_t value) { + return atomicExch(const_cast<__gm__ uint64_t *>(address), value); + } + + __aicore__ static inline int64_t FetchAdd(__gm__ volatile int64_t *address, int64_t value) { + // 返回递增前的计数；启动和 replay 屏障只关心全局累加结果，因此调用方不使用该返回值。 + return atomicAdd(const_cast<__gm__ int64_t *>(address), value); + } + + __aicore__ static inline int64_t FetchMax(__gm__ volatile int64_t *address, int64_t value, uint64_t &retries) { + // CCEC 直接生成单条硬件 atomicMax，不存在 CPU CAS 循环可观测的重试次数。 + retries = 0; + // 返回更新前的 cursor/frontier，Claim 用它判定 winner，frontier 扫描用它吸收其他核的进度。 + return atomicMax(const_cast<__gm__ int64_t *>(address), value); + } + + // PA's A5 OUT_OF_ORDER_STORE_BARRIER is intentionally a no-op; cache + // coherency is handled by the runtime's DCCI protocol. + // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， + // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache + // 可见性则由各自既有的 DCCI 路径处理。 + __aicore__ static inline void StoreBarrier() {} + + __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } + + template + __aicore__ static inline uint64_t NowAfterAtomicResult(T value) { + static_assert(sizeof(T) == 4 || sizeof(T) == 8, "atomic dependency expects a scalar result"); + uint64_t cycle = 0; + // 同一个 inline asm 块先真正消费 atomic 返回寄存器，再读取 + // SYS_CNT；编译器不能把 t1 拆到依赖 MOV 之前。AIC/AIV 对该序列 + // 生成相同指令字节，且不增加 DSB/ISB/GM 访存。该边界仍只表示 + // 返回值已可被本核 scalar 消费，不表示跨核全局可见。 + asm volatile( + "MOV %0, %0\n" + "MOV %1, SYS_CNT\n" + : "+l"(value), "=&l"(cycle) + ); + return cycle; + } + + __aicore__ static inline void ExecuteKernel( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState &worker, + pa_scheduler::TaskKind kind, uint32_t nop_count + ) { + const auto mode = static_cast(state->winner_workload.mode); + if (mode != pa_scheduler::WinnerWorkloadMode::RealCompute) { + RuntimeNop(nop_count); + return; + } +#if defined(PA_BUILD_AIC) + ::pa_execute_real_winner_workload_aic(state, &worker, kind); +#elif defined(PA_BUILD_AIV) + ::pa_execute_real_winner_workload_aiv(state, &worker, kind); +#endif + } + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + __aicore__ static inline pa_scheduler::LazySampleSplitRuntimeState &LazySampleSplitState() { +#if defined(PA_BUILD_AIC) + return ::pa_scheduler_lazy_sample_callback_state_aic; +#elif defined(PA_BUILD_AIV) + return ::pa_scheduler_lazy_sample_callback_state_aiv; +#endif + } + + __aicore__ static inline bool FinishLazySampleCallback( + const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args + ) { +#if defined(PA_BUILD_AIC) + return ::pa_scheduler_lazy_sample_callback_finish_aic(ticket, args) != 0; +#elif defined(PA_BUILD_AIV) + return ::pa_scheduler_lazy_sample_callback_finish_aiv(ticket, args) != 0; +#endif + } +#endif + +#if PA_BUILD_SUBMIT_PMU + using PmuContext = SubmitPmuContext; + + __aicore__ static inline PmuContext PmuWindowStart( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id + ); + + __aicore__ static inline void PmuPhaseBegin(PmuContext &context); + + __aicore__ static inline void PmuPhaseEnd(PmuContext &context); + + __aicore__ static inline void PmuWindowStop( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id, PmuContext &context + ); +#else + // swimlane 产物不携带 PMU 读寄存器或门控代码；公共调度器保留同一 hook + // 形状，编译器会把这两个空实现完整消去。 + __aicore__ static inline bool PmuWindowStart( + __gm__ pa_scheduler::SchedulerState *, uint32_t + ) { return false; } + + __aicore__ static inline void PmuWindowStop( + __gm__ pa_scheduler::SchedulerState *, uint32_t, bool + ) {} +#endif + + // SPIN_WAIT_HINT is also a no-op in the real A5 inner-kernel contract. + // 同理不额外插入 nop，让等待循环保留真实 PA 内核“不主动退避”的指令成本。 + __aicore__ static inline void SpinHint() {} + + __aicore__ static inline void InvalidateRegion(__gm__ const void *address, uint64_t bytes) { + // 逐 cache line 失效并以 dsb 收口，供 worker 在启动时读取 host 刚写入的 standalone 控制区。 + if (bytes == 0) return; + const uint64_t start = reinterpret_cast(address) & ~uint64_t{63}; + const uint64_t end = (reinterpret_cast(address) + bytes + 63) & ~uint64_t{63}; + for (uint64_t current = start; current < end; current += 64) { + dcci(reinterpret_cast<__gm__ uint8_t *>(current), SINGLE_CACHE_LINE); + } + dsb((mem_dsb_t)0); + } + + __aicore__ static inline void FlushRegion(__gm__ void *address, uint64_t bytes) { + // 泳道记录先写普通 GM cache，kernel 结束前显式 CACHELINE_OUT，确保 host D2H 能看到完整记录。 + if (bytes == 0) return; + __asm__ volatile("" ::: "memory"); + const uint64_t start = reinterpret_cast(address) & ~uint64_t{63}; + const uint64_t end = (reinterpret_cast(address) + bytes + 63) & ~uint64_t{63}; + for (uint64_t current = start; current < end; current += 64) { + dcci(reinterpret_cast<__gm__ uint8_t *>(current), SINGLE_CACHE_LINE, CACHELINE_OUT); + } + dsb((mem_dsb_t)0); + } + + __aicore__ static inline void Publish(__gm__ uint64_t *address, uint64_t value) { + // 每核独占的 WorkerResult 用 bypass-DCache store 发布，host 同步后可直接 D2H，无需共享原子竞争。 + __builtin_cce_st_dev(value, address, 0); + } + + __aicore__ static inline void Publish(__gm__ uint32_t *address, uint32_t value) { + __builtin_cce_st_dev(value, address, 0); + } +}; + +} // namespace pa_scheduler_ccec + +#if defined(PA_CCEC_OPS_DEFINE_REAL_WORKLOAD) +// 跨 TU 只暴露按架构区分的真实负载分派；Cube/Vector 实体仍保持 LOCAL。 +// 最终 mixed ELF 由 version script 把该 strong 定义重新局部化，避免成为 kernel entry。 +#if defined(PA_BUILD_AIC) +extern "C" __attribute__((noinline, used)) __aicore__ void pa_execute_real_winner_workload_aic( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +) { +#elif defined(PA_BUILD_AIV) +extern "C" __attribute__((noinline, used)) __aicore__ void pa_execute_real_winner_workload_aiv( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +) { +#endif + const uint64_t workspace = state->winner_workload.workspace_base; + const uint32_t repeats = pa_scheduler::WorkloadCountForKind( + state->winner_workload.repeats, kind + ); + // 错版 host、截断 workspace 或越界 worker 不能继续解引用 GM。这里不额外 + // 写共享 fatal，避免在正常热路增加 atomic；host 的逐 kind sentinel/数值 + // 闭环会把这种配置错误判为失败。 + if (state->winner_workload.version != pa_scheduler::kWinnerWorkloadConfigVersion || + workspace == 0 || + state->winner_workload.workspace_bytes < pa_scheduler::winner_workload::kWorkspaceBytes || + worker->core_idx < 0 || static_cast(worker->core_idx) >= pa_scheduler::kWorkers || + repeats == 0 || repeats > pa_scheduler::winner_workload::kMaxRealComputeCount) { + return; + } +#if defined(PA_BUILD_AIC) + if (kind != pa_scheduler::TaskKind::Qk && kind != pa_scheduler::TaskKind::Pv) return; +#elif defined(PA_BUILD_AIV) + if (kind != pa_scheduler::TaskKind::Sf && kind != pa_scheduler::TaskKind::Up) return; +#endif + __gm__ float *input_a = reinterpret_cast<__gm__ float *>(workspace); + __gm__ float *input_b = reinterpret_cast<__gm__ float *>( + workspace + pa_scheduler::winner_workload::kTileBytes + ); + const uint32_t kind_slot = + (kind == pa_scheduler::TaskKind::Pv || kind == pa_scheduler::TaskKind::Up) ? 1U : 0U; + const uint32_t output_tile = + pa_scheduler::winner_workload::kSharedInputTiles + + static_cast(worker->core_idx) * + pa_scheduler::winner_workload::kOutputTilesPerWorker + + kind_slot; + __gm__ float *output = reinterpret_cast<__gm__ float *>( + workspace + static_cast(output_tile) * + pa_scheduler::winner_workload::kTileBytes + ); +#if defined(PA_BUILD_AIC) + pa_scheduler_ccec::pa_real_cube_workload_aic(input_a, input_b, output, repeats); +#elif defined(PA_BUILD_AIV) + if (kind == pa_scheduler::TaskKind::Sf) { + pa_scheduler_ccec::pa_real_vector_add_workload_aiv(input_a, input_b, output, repeats); + } else { + pa_scheduler_ccec::pa_real_vector_mul_workload_aiv(input_a, input_b, output, repeats); + } +#endif +} +#endif // PA_CCEC_OPS_DEFINE_REAL_WORKLOAD + +#endif // TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CCEC_OPS_H diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/host.cpp b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/host.cpp new file mode 100644 index 0000000000..1ee342645f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/host.cpp @@ -0,0 +1,1989 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "../common/host_support.h" +#include "../common/winner_workload_host.h" +#include "pmu_owner_host.h" +#include "pmu_probe.h" + +#include "acl/acl.h" +#include "driver/ascend_hal.h" +#include "runtime/rt.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +bool CheckAcl(aclError error, const char *label) { + if (error == ACL_SUCCESS) return true; + std::fprintf(stderr, "ACL error %d: %s\n", static_cast(error), label); + return false; +} + +bool CheckRt(rtError_t error, const char *label) { + if (error == RT_ERROR_NONE) return true; + std::fprintf(stderr, "RT error %d: %s\n", static_cast(error), label); + return false; +} + +class ScopedAclDeviceAllocation { +public: + ScopedAclDeviceAllocation() = default; + ScopedAclDeviceAllocation(const ScopedAclDeviceAllocation &) = delete; + ScopedAclDeviceAllocation &operator=(const ScopedAclDeviceAllocation &) = delete; + + ~ScopedAclDeviceAllocation() { + // 早退路径没有机会汇入末尾 cleanup；这里只负责尽力释放本类新增的 + // real-compute workspace。正常路径会先 Release，再检查 aclrtFree 返回值。 + if (pointer_ != nullptr) (void)aclrtFree(pointer_); + } + + void **Address() { return &pointer_; } + void *Get() const { return pointer_; } + + void *Release() { + void *pointer = pointer_; + pointer_ = nullptr; + return pointer; + } + +private: + void *pointer_ = nullptr; +}; + +std::vector ReadBinary(const std::string &path) { + // ELF 整体保存在 vector 中直到 runtime 卸载完成，保证 rtDevBinary_t.data 在整个注册生命周期内有效。 + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) return {}; + const std::streamsize size = file.tellg(); + if (size <= 0) return {}; + std::vector data(static_cast(size)); + file.seekg(0, std::ios::beg); + if (!file.read(data.data(), size)) return {}; + return data; +} + +struct PmuOptions { + pa_scheduler::ccec_pmu::WindowMode mode = pa_scheduler::ccec_pmu::WindowMode::Off; + uint32_t scalar_nops = 100000; + uint32_t icache_trials = 64; + std::string json_path; +}; + +using pa_scheduler::host::ConfigureWinnerWorkload; +using pa_scheduler::host::ParseWinnerWorkloadOptions; +using pa_scheduler::host::ValidateRealComputeOutputs; +using pa_scheduler::host::ValidateWinnerWorkloadOptions; +using pa_scheduler::host::WinnerWorkloadModeName; +using pa_scheduler::host::WinnerWorkloadOptions; + +const char *PmuModeName(pa_scheduler::ccec_pmu::WindowMode mode) { + switch (mode) { + case pa_scheduler::ccec_pmu::WindowMode::Off: + return "off"; + case pa_scheduler::ccec_pmu::WindowMode::Empty: + return "empty"; + case pa_scheduler::ccec_pmu::WindowMode::Scalar: + return "scalar"; + case pa_scheduler::ccec_pmu::WindowMode::ScalarDouble: + return "scalar-double"; + case pa_scheduler::ccec_pmu::WindowMode::IcacheSingle: + return "icache-single"; + case pa_scheduler::ccec_pmu::WindowMode::SubmitAll: + return "submit-all"; + } + return "invalid"; +} + +bool ParsePmuOptions(int argc, char **argv, PmuOptions *pmu, std::vector *common_argv) { + // PMU 参数只属于 CCEC 验证分支；先摘出再交给三后端共享 parser，避免 CPU/AscendC 静默接受却不生效。 + bool mode_seen = false; + bool nops_seen = false; + bool icache_trials_seen = false; + bool json_seen = false; + common_argv->clear(); + common_argv->push_back(argv[0]); + for (int index = 1; index < argc; ++index) { + const std::string argument = argv[index]; + if (argument != "--pmu-window" && argument != "--pmu-scalar-nops" && + argument != "--pmu-icache-trials" && argument != "--pmu-json") { + common_argv->push_back(argv[index]); + continue; + } + if (index + 1 >= argc) { + std::fprintf(stderr, "Missing value after %s\n", argument.c_str()); + return false; + } + const char *value = argv[++index]; + if (argument == "--pmu-window") { + if (mode_seen) { + std::fprintf(stderr, "Specify --pmu-window only once.\n"); + return false; + } + const std::string name = value; + if (name == "off") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::Off; + } else if (name == "empty") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::Empty; + } else if (name == "scalar") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::Scalar; + } else if (name == "scalar-double") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::ScalarDouble; + } else if (name == "icache-single") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::IcacheSingle; + } else if (name == "submit-all") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::SubmitAll; + } else { + std::fprintf( + stderr, + "Invalid --pmu-window value: %s " + "(expected off|empty|scalar|scalar-double|icache-single|submit-all)\n", + value + ); + return false; + } + mode_seen = true; + } else if (argument == "--pmu-scalar-nops") { + if (nops_seen || !pa_scheduler::host::ParseUint(value, 0, 10000000, &pmu->scalar_nops)) { + std::fprintf(stderr, "Invalid or duplicate --pmu-scalar-nops value: %s\n", value); + return false; + } + nops_seen = true; + } else if (argument == "--pmu-icache-trials") { + if (icache_trials_seen || + !pa_scheduler::host::ParseUint(value, 1, 10000, &pmu->icache_trials)) { + std::fprintf(stderr, "Invalid or duplicate --pmu-icache-trials value: %s\n", value); + return false; + } + icache_trials_seen = true; + } else { + if (json_seen || *value == '\0') { + std::fprintf(stderr, "Invalid or duplicate --pmu-json path: %s\n", value); + return false; + } + pmu->json_path = value; + json_seen = true; + } + } + if (nops_seen && pmu->mode != pa_scheduler::ccec_pmu::WindowMode::Scalar && + pmu->mode != pa_scheduler::ccec_pmu::WindowMode::ScalarDouble) { + std::fprintf(stderr, "--pmu-scalar-nops requires a scalar PMU window.\n"); + return false; + } + if (icache_trials_seen && pmu->mode != pa_scheduler::ccec_pmu::WindowMode::IcacheSingle) { + std::fprintf(stderr, "--pmu-icache-trials requires the icache-single PMU window.\n"); + return false; + } + return true; +} + +using HalResMapFn = int (*)(uint32_t, struct res_map_info *, unsigned long *, uint32_t *); +using HalResUnmapFn = int (*)(uint32_t, struct res_map_info *); + +struct PmuRegisterMappings { + HalResUnmapFn unmap = nullptr; + std::vector mapped_resources; + std::vector register_bases; +}; + +bool UnmapPmuRegisters(uint32_t device, PmuRegisterMappings *mappings) { + bool ok = true; + if (mappings->unmap != nullptr) { + for (auto iterator = mappings->mapped_resources.rbegin(); iterator != mappings->mapped_resources.rend(); + ++iterator) { + const int error = mappings->unmap(device, &*iterator); + if (error != 0) { + std::fprintf(stderr, "halResUnmap failed for core %u (rc=%d)\n", iterator->res_id, error); + ok = false; + } + } + } + mappings->mapped_resources.clear(); + mappings->register_bases.clear(); + return ok; +} + +bool MapPmuRegisters(uint32_t device, PmuRegisterMappings *mappings) { + using namespace pa_scheduler::ccec_pmu; + const auto map = reinterpret_cast(dlsym(RTLD_DEFAULT, "halResMap")); + mappings->unmap = reinterpret_cast(dlsym(RTLD_DEFAULT, "halResUnmap")); + if (map == nullptr || mappings->unmap == nullptr) { + std::fprintf(stderr, "halResMap/halResUnmap is unavailable in the current CANN driver process.\n"); + return false; + } + + mappings->register_bases.assign(kPhysicalSubcoreCount, 0); + mappings->mapped_resources.reserve(kPhysicalAicoreCount); + for (uint32_t aicore = 0; aicore < kPhysicalAicoreCount; ++aicore) { + res_map_info info{}; + info.target_proc_type = PROCESS_CP1; + info.res_type = RES_AICORE; + info.res_id = aicore; + unsigned long map_address = 0; + uint32_t map_bytes = kAicoreMapBytes; + const int error = map(device, &info, &map_address, &map_bytes); + if (error != 0 || map_address == 0 || map_bytes < kAicoreMapBytes) { + std::fprintf( + stderr, "halResMap failed for core %u (rc=%d address=0x%lx bytes=%u)\n", aicore, error, + map_address, map_bytes + ); + (void)UnmapPmuRegisters(device, mappings); + return false; + } + mappings->mapped_resources.push_back(info); + + // 与正式 A5 host_regs.cpp 相同：每个 die 的布局为 18 AIC，随后是 36 AIV。 + const uint32_t die = aicore / kAicorePerDie; + const uint32_t local = aicore % kAicorePerDie; + const uint32_t die_base = die * kSubcoresPerDie; + mappings->register_bases[die_base + local] = static_cast(map_address); + mappings->register_bases[die_base + kAicorePerDie + local * 2] = + static_cast(map_address) + kAivFirstOffset; + mappings->register_bases[die_base + kAicorePerDie + local * 2 + 1] = + static_cast(map_address) + kAivSecondOffset; + } + return true; +} + +void ConfigurePmu(pa_scheduler::SchedulerState *state, const PmuOptions &pmu, const void *register_table) { + using namespace pa_scheduler::ccec_pmu; + state->config.reserved[kConfigMode] = static_cast(pmu.mode); + state->config.reserved[kConfigWorkAmount] = + pmu.mode == WindowMode::IcacheSingle ? pmu.icache_trials : pmu.scalar_nops; + StorePointer(state->config.reserved, register_table); + state->config.reserved[kConfigMagic] = pmu.mode == WindowMode::Off ? 0 : kConfigMagicValue; +} + +struct PmuAggregate { + std::vector total_cycles; + std::vector window_ticks; + std::vector submit_elapsed_ticks; + std::vector phase_elapsed_ticks; + std::vector warm_total_cycles; + std::vector warm_window_ticks; + std::vector vector_busy; + std::vector cube_busy; + std::vector scalar_busy; + std::vector mte1_busy; + std::vector mte2_busy; + std::vector icache_requests; + std::vector icache_misses; + std::vector warm_icache_requests; + std::vector warm_icache_misses; + std::vector fix_busy; + std::vector phase_calls; + std::vector phase_icache_requests; + std::vector phase_icache_misses; + std::vector shadow_icache_requests; + std::vector shadow_icache_misses; + uint32_t trusted = 0; +}; + +void AddPmuSample(const pa_scheduler::WorkerResult &result, PmuAggregate *aggregate) { + aggregate->total_cycles.push_back(result.pmu_total_cycles); + aggregate->window_ticks.push_back(result.pmu_window_ticks); + aggregate->submit_elapsed_ticks.push_back( + result.submit_end >= result.submit_begin ? result.submit_end - result.submit_begin : 0U + ); + aggregate->phase_elapsed_ticks.push_back(result.pmu_phase_elapsed_ticks); + aggregate->warm_total_cycles.push_back(result.pmu_warm_total_cycles); + aggregate->warm_window_ticks.push_back(result.pmu_warm_window_ticks); + aggregate->vector_busy.push_back(result.pmu_vector_busy); + aggregate->cube_busy.push_back(result.pmu_cube_busy); + aggregate->scalar_busy.push_back(result.pmu_scalar_busy); + aggregate->mte1_busy.push_back(result.pmu_mte1_busy); + aggregate->mte2_busy.push_back(result.pmu_mte2_busy); + aggregate->icache_requests.push_back(result.pmu_icache_requests); + aggregate->icache_misses.push_back(result.pmu_icache_misses); + aggregate->warm_icache_requests.push_back(result.pmu_warm_icache_requests); + aggregate->warm_icache_misses.push_back(result.pmu_warm_icache_misses); + aggregate->fix_busy.push_back(result.pmu_fix_busy); + aggregate->phase_calls.push_back(result.pmu_phase_calls); + aggregate->phase_icache_requests.push_back(result.pmu_phase_icache_requests); + aggregate->phase_icache_misses.push_back(result.pmu_phase_icache_misses); + aggregate->shadow_icache_requests.push_back(result.pmu_shadow_icache_requests); + aggregate->shadow_icache_misses.push_back(result.pmu_shadow_icache_misses); + aggregate->trusted += + (result.pmu_status & pa_scheduler::ccec_pmu::kStatusRequired) == + pa_scheduler::ccec_pmu::kStatusRequired && + (result.pmu_phase_status & pa_scheduler::ccec_pmu::kPhaseStatusRequired) == + pa_scheduler::ccec_pmu::kPhaseStatusRequired; +} + +bool PrintSingleIcacheAggregate( + const char *name, const PmuAggregate &aggregate, uint32_t trials_per_core +) { + const pa_scheduler::host::Uint64Distribution cold_cycles = + pa_scheduler::host::SummarizeUint64(aggregate.total_cycles); + const pa_scheduler::host::Uint64Distribution warm_cycles = + pa_scheduler::host::SummarizeUint64(aggregate.warm_total_cycles); + const pa_scheduler::host::Uint64Distribution cold_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.window_ticks); + const pa_scheduler::host::Uint64Distribution warm_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.warm_window_ticks); + const pa_scheduler::host::Uint64Distribution cold_requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution warm_requests = + pa_scheduler::host::SummarizeUint64(aggregate.warm_icache_requests); + const pa_scheduler::host::Uint64Distribution cold_misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + const pa_scheduler::host::Uint64Distribution warm_misses = + pa_scheduler::host::SummarizeUint64(aggregate.warm_icache_misses); + const int64_t cycle_delta = static_cast(cold_cycles.total) - + static_cast(warm_cycles.total); + const int64_t tick_delta = static_cast(cold_ticks.total) - + static_cast(warm_ticks.total); + const int64_t miss_delta = static_cast(cold_misses.total) - + static_cast(warm_misses.total); + const uint64_t attempted = static_cast(trials_per_core) * aggregate.total_cycles.size(); + const double misses_per_trial = attempted == 0U ? 0.0 : static_cast(miss_delta) / attempted; + const double cycles_per_miss = miss_delta <= 0 ? 0.0 : static_cast(cycle_delta) / miss_delta; + // 本用例的 get_sys_cnt 已按 1 GHz 时间基准校准，因此一个 tick 对应 1 ns。 + const double ns_per_miss = miss_delta <= 0 ? 0.0 : static_cast(tick_delta) / miss_delta; + std::printf( + "[ICACHE-SINGLE-%s] cores=%zu trials_per_core=%u attempted=%llu " + "cold_cycles=%llu warm_cycles=%llu cycle_delta=%lld cold_ticks=%llu warm_ticks=%llu " + "tick_delta=%lld cold_req=%llu warm_req=%llu cold_miss=%llu warm_miss=%llu " + "miss_delta=%lld misses_per_trial=%.6f cycles_per_miss=%.3f ns_per_miss=%.3f\n", + name, aggregate.total_cycles.size(), trials_per_core, static_cast(attempted), + static_cast(cold_cycles.total), + static_cast(warm_cycles.total), static_cast(cycle_delta), + static_cast(cold_ticks.total), + static_cast(warm_ticks.total), static_cast(tick_delta), + static_cast(cold_requests.total), + static_cast(warm_requests.total), + static_cast(cold_misses.total), + static_cast(warm_misses.total), static_cast(miss_delta), + misses_per_trial, cycles_per_miss, ns_per_miss + ); + std::printf( + "[ICACHE-FORMULA-%s] estimated_scalar_icache_time_ns = cnt7_icache_miss * %.3f\n", + name, ns_per_miss + ); + return cycle_delta > 0 && tick_delta > 0 && miss_delta == static_cast(attempted); +} + +void PrintPmuAggregate(const char *name, const PmuAggregate &aggregate) { + const pa_scheduler::host::Uint64Distribution total = + pa_scheduler::host::SummarizeUint64(aggregate.total_cycles); + const pa_scheduler::host::Uint64Distribution scalar = + pa_scheduler::host::SummarizeUint64(aggregate.scalar_busy); + const pa_scheduler::host::Uint64Distribution vector = + pa_scheduler::host::SummarizeUint64(aggregate.vector_busy); + const pa_scheduler::host::Uint64Distribution cube = + pa_scheduler::host::SummarizeUint64(aggregate.cube_busy); + const pa_scheduler::host::Uint64Distribution mte1 = + pa_scheduler::host::SummarizeUint64(aggregate.mte1_busy); + const pa_scheduler::host::Uint64Distribution mte2 = + pa_scheduler::host::SummarizeUint64(aggregate.mte2_busy); + const pa_scheduler::host::Uint64Distribution requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + const double miss_rate = requests.total == 0 ? 0.0 : 100.0 * misses.total / requests.total; + std::printf( + "[PMU-%s] cores=%zu total_sum=%llu total_median=%.1f total_p95=%llu " + "scalar_busy=%llu vector_busy=%llu cube_busy=%llu mte1_busy=%llu mte2_busy=%llu " + "icache_req=%llu icache_miss=%llu miss_rate=%.4f%%\n", + name, aggregate.total_cycles.size(), static_cast(total.total), total.median, + static_cast(total.p95), static_cast(scalar.total), + static_cast(vector.total), static_cast(cube.total), + static_cast(mte1.total), static_cast(mte2.total), + static_cast(requests.total), static_cast(misses.total), miss_rate + ); +} + +void PrintSubmitPmuPhaseAggregate(const char *name, const PmuAggregate &aggregate) { + const pa_scheduler::host::Uint64Distribution calls = + pa_scheduler::host::SummarizeUint64(aggregate.phase_calls); + const pa_scheduler::host::Uint64Distribution requests = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_requests); + const pa_scheduler::host::Uint64Distribution misses = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_misses); + const pa_scheduler::host::Uint64Distribution primary_requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution primary_misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + const pa_scheduler::host::Uint64Distribution shadow_requests = + pa_scheduler::host::SummarizeUint64(aggregate.shadow_icache_requests); + const pa_scheduler::host::Uint64Distribution shadow_misses = + pa_scheduler::host::SummarizeUint64(aggregate.shadow_icache_misses); + const pa_scheduler::host::Uint64Distribution submit_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.submit_elapsed_ticks); + const pa_scheduler::host::Uint64Distribution phase_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.phase_elapsed_ticks); + const uint64_t request_loss = primary_requests.total >= shadow_requests.total + ? primary_requests.total - shadow_requests.total + : 0U; + const uint64_t miss_loss = primary_misses.total >= shadow_misses.total + ? primary_misses.total - shadow_misses.total + : 0U; + const double miss_rate = requests.total == 0U ? 0.0 : 100.0 * misses.total / requests.total; + const double phase_time_share = submit_ticks.total == 0U + ? 0.0 + : 100.0 * phase_ticks.total / submit_ticks.total; + std::printf( + "[PMU-PHASE-%s] phase=%s semantics=%s cores=%zu calls=%llu " + "icache_req=[%llu,%llu] icache_miss=[%llu,%llu] " + "observed_read_clear_ratio=%.4f%% phase_ticks=%llu submit_ticks=%llu " + "phase_time_share=%.4f%% shadow_loss=%llu/%llu\n", + name, pa_scheduler::ccec_pmu::SubmitPmuPhaseName(pa_scheduler::kCompiledSubmitPmuPhase), + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "disabled" + : "running_read_clear_lower_bound", + aggregate.phase_calls.size(), static_cast(calls.total), + static_cast(requests.total), + static_cast(requests.total + request_loss), + static_cast(misses.total), + static_cast(misses.total + miss_loss), miss_rate, + static_cast(phase_ticks.total), + static_cast(submit_ticks.total), phase_time_share, + static_cast(request_loss), + static_cast(miss_loss) + ); +} + +struct PmuValidation { + uint32_t trusted = 0; + uint32_t unique_physical_core_ids = 0; + uint32_t owner_bitmap_members = 0; + uint32_t exact_worker_slots = 0; + uint32_t physical_role_matches = 0; + uint32_t mixed_triplet_matches = 0; + uint32_t window_started = 0; + uint32_t window_stopped = 0; + uint32_t icache_pairs = 0; + uint32_t icache_calibrated_cores = 0; + uint32_t prior_snapshot_larger = 0; + uint32_t submit_engine_workers_expected = 0; + uint32_t submit_engine_workers_matched = 0; + uint32_t maximum_programmable_counter = 0; + uint32_t build_variant_matches = 0; + uint32_t phase_id_matches = 0; + uint32_t phase_status_trusted = 0; + uint32_t shadow_primary_matches = 0; + uint32_t shadow_primary_bounded = 0; + uint32_t phase_shadow_acceptable = 0; + uint32_t phase_boundary_matches = 0; + uint32_t phase_call_shape_matches = 0; + uint32_t phase_time_valid_records = 0; + uint64_t phase_calls = 0; + uint64_t expected_phase_calls = 0; + uint64_t shadow_request_abs_delta_sum = 0; + uint64_t shadow_miss_abs_delta_sum = 0; + int64_t shadow_request_signed_delta_sum = 0; + int64_t shadow_miss_signed_delta_sum = 0; + uint32_t shadow_request_abs_delta_max = 0; + uint32_t shadow_miss_abs_delta_max = 0; + bool icache_order_valid = true; + bool icache_measurement_valid = true; + bool submit_engine_observation_valid = true; + bool counter_below_risk_threshold = true; + bool phase_measurement_valid = false; + bool passed = true; +}; + +// 32-bit programmable counter 无法仅凭终值证明从未回卷。正式文件采用 25% +// 高水位作为保守拒绝阈值；它只降低风险，不把“未越线”表述成回卷证明。 +constexpr uint32_t kProgrammableCounterRiskThreshold = UINT32_MAX / 4U; + +uint32_t ExpectedSubmitPmuPhaseCallsPerWorker(uint32_t batches) { + // 当前所有 running phase 都覆盖每个 worker 的每次 Submit，固定为 5B。 + switch (pa_scheduler::kCompiledSubmitPmuPhase) { + case pa_scheduler::SubmitPmuPhase::None: + return 0U; + case pa_scheduler::SubmitPmuPhase::Claim: + case pa_scheduler::SubmitPmuPhase::EfDrain: + case pa_scheduler::SubmitPmuPhase::Materialize: + case pa_scheduler::SubmitPmuPhase::Register: + return batches * pa_scheduler::kTasksPerBatch; + case pa_scheduler::SubmitPmuPhase::Count: + break; + } + return UINT32_MAX; +} + +bool ValidatePmu( + const pa_scheduler::SchedulerState &state, uint32_t run, const PmuOptions &pmu, + const WinnerWorkloadOptions &workload, + const pa_scheduler::pmu_owner::PmuOwnerControl *owner, PmuValidation *validation +) { + using namespace pa_scheduler::ccec_pmu; + if (pmu.mode == WindowMode::Off) { + *validation = PmuValidation{}; + return true; + } + + bool seen[kPhysicalSubcoreCount] = {}; + uint32_t trusted = 0; + uint32_t unique = 0; + uint32_t owner_members = 0; + uint32_t exact_worker_slots = 0; + uint32_t physical_role_matches = 0; + uint32_t window_started = 0; + uint32_t window_stopped = 0; + uint32_t icache_pairs = 0; + uint32_t icache_calibrated_cores = 0; + uint32_t prior_larger = 0; + uint32_t submit_engine_workers_expected = 0; + uint32_t submit_engine_workers_matched = 0; + uint32_t maximum_programmable_counter = 0; + uint32_t build_variant_matches = 0; + uint32_t phase_id_matches = 0; + uint32_t phase_status_trusted = 0; + uint32_t shadow_primary_matches = 0; + uint32_t shadow_primary_bounded = 0; + uint32_t phase_shadow_acceptable = 0; + uint32_t phase_boundary_matches = 0; + uint32_t phase_call_shape_matches = 0; + uint32_t phase_time_valid_records = 0; + uint64_t phase_calls = 0; + uint64_t expected_phase_calls = 0; + uint64_t shadow_request_abs_delta_sum = 0; + uint64_t shadow_miss_abs_delta_sum = 0; + int64_t shadow_request_signed_delta_sum = 0; + int64_t shadow_miss_signed_delta_sum = 0; + uint32_t shadow_request_abs_delta_max = 0; + uint32_t shadow_miss_abs_delta_max = 0; + bool icache_order_valid = true; + uint32_t bad_printed = 0; + PmuAggregate all; + PmuAggregate aic; + PmuAggregate aiv; + for (uint32_t worker = 0; worker < pa_scheduler::kWorkers; ++worker) { + const pa_scheduler::WorkerResult &result = state.results[worker]; + const uint32_t expected_phase_calls_per_worker = + ExpectedSubmitPmuPhaseCallsPerWorker(state.config.batches); + const uint32_t status = result.pmu_status; + const uint32_t core_id = StatusCoreId(status); + const bool record_trusted = (status & kStatusRequired) == kStatusRequired; + const bool variant_matches = result.pmu_build_variant == pa_scheduler::kBuildVariantSubmitPmu; + const bool phase_id_matches_record = + result.pmu_phase_id == static_cast(pa_scheduler::kCompiledSubmitPmuPhase); + const bool phase_status_ok = + (result.pmu_phase_status & kPhaseStatusRequired) == kPhaseStatusRequired; + const bool shadow_matches = + result.pmu_shadow_icache_requests == result.pmu_icache_requests && + result.pmu_shadow_icache_misses == result.pmu_icache_misses; + const bool shadow_bounded = + result.pmu_shadow_icache_requests <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_icache_misses; + const bool phase_requires_exact_shadow = + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None; + const bool shadow_acceptable = + phase_requires_exact_shadow + ? shadow_matches + : shadow_bounded; + const uint32_t request_abs_delta = + result.pmu_shadow_icache_requests >= result.pmu_icache_requests + ? result.pmu_shadow_icache_requests - result.pmu_icache_requests + : result.pmu_icache_requests - result.pmu_shadow_icache_requests; + const uint32_t miss_abs_delta = + result.pmu_shadow_icache_misses >= result.pmu_icache_misses + ? result.pmu_shadow_icache_misses - result.pmu_icache_misses + : result.pmu_icache_misses - result.pmu_shadow_icache_misses; + const bool boundaries_match = + result.pmu_phase_begin_reads == result.pmu_phase_calls && + result.pmu_phase_end_reads == result.pmu_phase_calls; + const bool phase_call_shape_matches_record = + result.pmu_phase_calls == expected_phase_calls_per_worker; + const uint64_t submit_elapsed_ticks = result.submit_end >= result.submit_begin + ? result.submit_end - result.submit_begin + : 0U; + const bool phase_time_valid_record = + submit_elapsed_ticks != 0U && + result.pmu_phase_elapsed_ticks <= submit_elapsed_ticks && + (pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? result.pmu_phase_elapsed_ticks == 0U + : result.pmu_phase_elapsed_ticks != 0U); + const bool logical_aic = worker < pa_scheduler::kAicWorkers; + const bool physical_aic = pa_scheduler::pmu_owner::IsAicPhysicalSlot(core_id); + trusted += record_trusted; + build_variant_matches += variant_matches; + phase_id_matches += phase_id_matches_record; + phase_status_trusted += phase_status_ok; + shadow_primary_matches += shadow_matches; + shadow_primary_bounded += shadow_bounded; + phase_shadow_acceptable += shadow_acceptable; + phase_boundary_matches += boundaries_match; + phase_call_shape_matches += phase_call_shape_matches_record; + phase_time_valid_records += phase_time_valid_record; + phase_calls += result.pmu_phase_calls; + expected_phase_calls += expected_phase_calls_per_worker; + shadow_request_abs_delta_sum += request_abs_delta; + shadow_miss_abs_delta_sum += miss_abs_delta; + shadow_request_signed_delta_sum += + static_cast(result.pmu_shadow_icache_requests) - result.pmu_icache_requests; + shadow_miss_signed_delta_sum += + static_cast(result.pmu_shadow_icache_misses) - result.pmu_icache_misses; + shadow_request_abs_delta_max = std::max(shadow_request_abs_delta_max, request_abs_delta); + shadow_miss_abs_delta_max = std::max(shadow_miss_abs_delta_max, miss_abs_delta); + owner_members += owner != nullptr && pa_scheduler::pmu_owner::IsConfigured(*owner, core_id); + exact_worker_slots += result.worker_id == worker; + physical_role_matches += logical_aic == physical_aic; + window_started += (status & kStatusWindowStarted) != 0U; + window_stopped += (status & kStatusWindowStopped) != 0U; + icache_pairs += (status & kStatusIcachePairObserved) != 0U; + prior_larger += (status & kStatusPriorSnapshotLarger) != 0; + if (pmu.mode == WindowMode::SubmitAll && + workload.mode == pa_scheduler::WinnerWorkloadMode::RealCompute) { + const uint64_t submit_engine_tasks = + result.placement[static_cast(pa_scheduler::DrainPlace::EfDrain)] + + result.placement[static_cast(pa_scheduler::DrainPlace::RingBackpressure)]; + const uint32_t relevant_busy = logical_aic ? result.pmu_cube_busy : result.pmu_vector_busy; + const bool engine_observation_matches = + (submit_engine_tasks == 0U) == (relevant_busy == 0U); + submit_engine_workers_expected += submit_engine_tasks != 0U; + submit_engine_workers_matched += engine_observation_matches; + } + // phase 的 request/miss 由两条顺序 ld_dev 划界，局部窗口边界并不 + // 完全重合；只要求它们各自不超过完整窗口，不把 phase miss<=request + // 误设成硬门槛。完整 Submit 的 miss<=request 仍必须成立。 + icache_order_valid &= result.pmu_icache_misses <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_shadow_icache_requests && + result.pmu_shadow_icache_requests <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_icache_misses && + result.pmu_phase_icache_requests <= result.pmu_shadow_icache_requests && + result.pmu_phase_icache_misses <= result.pmu_shadow_icache_misses; + if (pmu.mode == WindowMode::IcacheSingle) { + const int64_t worker_cycle_delta = static_cast(result.pmu_total_cycles) - + static_cast(result.pmu_warm_total_cycles); + const int64_t worker_tick_delta = static_cast(result.pmu_window_ticks) - + static_cast(result.pmu_warm_window_ticks); + const int64_t worker_miss_delta = static_cast(result.pmu_icache_misses) - + static_cast(result.pmu_warm_icache_misses); + icache_calibrated_cores += worker_cycle_delta > 0 && worker_tick_delta > 0 && + worker_miss_delta == static_cast(pmu.icache_trials) && + result.pmu_warm_icache_misses == 0U && + result.pmu_icache_misses == pmu.icache_trials; + } + const uint32_t programmable[] = { + result.pmu_vector_busy, result.pmu_cube_busy, result.pmu_scalar_busy, + result.pmu_mte1_busy, result.pmu_mte2_busy, + result.pmu_icache_requests, result.pmu_icache_misses, + result.pmu_phase_icache_requests, result.pmu_phase_icache_misses, + result.pmu_shadow_icache_requests, result.pmu_shadow_icache_misses, + }; + for (uint32_t value : programmable) { + maximum_programmable_counter = std::max(maximum_programmable_counter, value); + } + if (core_id < kPhysicalSubcoreCount && !seen[core_id]) { + seen[core_id] = true; + ++unique; + } + if ((!record_trusted || !variant_matches || !phase_id_matches_record || + !phase_status_ok || !shadow_acceptable || !boundaries_match || + !phase_call_shape_matches_record || !phase_time_valid_record) && bad_printed < 8) { + std::printf( + "[PMU-BAD] worker=%u role=%llu coreid=%u status=0x%08x total=%llu scalar=%u " + "req=%u miss=%u phase_status=0x%08x phase=%u/%u calls=%u/%u boundaries=%u/%u " + "phase_ticks=%llu submit_ticks=%llu shadow=%u/%u\n", + worker, static_cast(result.role), core_id, status, + static_cast(result.pmu_total_cycles), result.pmu_scalar_busy, + result.pmu_icache_requests, result.pmu_icache_misses, + result.pmu_phase_status, result.pmu_phase_id, + static_cast(pa_scheduler::kCompiledSubmitPmuPhase), result.pmu_phase_calls, + expected_phase_calls_per_worker, + result.pmu_phase_begin_reads, result.pmu_phase_end_reads, + static_cast(result.pmu_phase_elapsed_ticks), + static_cast(submit_elapsed_ticks), + result.pmu_shadow_icache_requests, result.pmu_shadow_icache_misses + ); + ++bad_printed; + } + AddPmuSample(result, &all); + AddPmuSample(result, result.role == static_cast(pa_scheduler::CoreRole::Aic) ? &aic : &aiv); + } + + uint32_t mixed_triplet_matches = 0U; + for (uint32_t block = 0U; block < pa_scheduler::kAicWorkers; ++block) { + const uint32_t aic_id = StatusCoreId(state.results[block].pmu_status); + if (!pa_scheduler::pmu_owner::IsAicPhysicalSlot(aic_id)) continue; + const uint32_t die_base = (aic_id / pa_scheduler::pmu_owner::kSubcoresPerDie) * + pa_scheduler::pmu_owner::kSubcoresPerDie; + const uint32_t local = aic_id % pa_scheduler::pmu_owner::kSubcoresPerDie; + const uint32_t expected_aiv0 = die_base + pa_scheduler::pmu_owner::kAicPerDie + local * 2U; + const uint32_t expected_aiv1 = expected_aiv0 + 1U; + const uint32_t aiv0_id = StatusCoreId( + state.results[pa_scheduler::kAicWorkers + block * 2U].pmu_status + ); + const uint32_t aiv1_id = StatusCoreId( + state.results[pa_scheduler::kAicWorkers + block * 2U + 1U].pmu_status + ); + mixed_triplet_matches += aiv0_id == expected_aiv0 && aiv1_id == expected_aiv1; + } + + PrintPmuAggregate("ALL", all); + PrintPmuAggregate("AIC", aic); + PrintPmuAggregate("AIV", aiv); + PrintSubmitPmuPhaseAggregate("ALL", all); + PrintSubmitPmuPhaseAggregate("AIC", aic); + PrintSubmitPmuPhaseAggregate("AIV", aiv); + bool icache_measurement_ok = true; + if (pmu.mode == WindowMode::IcacheSingle) { + const bool all_ok = PrintSingleIcacheAggregate("ALL", all, pmu.icache_trials); + const bool aic_ok = PrintSingleIcacheAggregate("AIC", aic, pmu.icache_trials); + const bool aiv_ok = PrintSingleIcacheAggregate("AIV", aiv, pmu.icache_trials); + icache_measurement_ok = all_ok && aic_ok && aiv_ok && + icache_pairs == pa_scheduler::kWorkers && + icache_calibrated_cores == pa_scheduler::kWorkers; + } + const bool records_ok = trusted == pa_scheduler::kWorkers; + const bool core_ids_ok = unique == pa_scheduler::kWorkers; + const bool owner_members_ok = owner_members == pa_scheduler::kWorkers; + const bool worker_slots_ok = exact_worker_slots == pa_scheduler::kWorkers; + const bool physical_roles_ok = physical_role_matches == pa_scheduler::kWorkers; + const bool mixed_triplets_ok = mixed_triplet_matches == pa_scheduler::kAicWorkers; + const bool windows_started_ok = window_started == pa_scheduler::kWorkers; + const bool windows_stopped_ok = window_stopped == pa_scheduler::kWorkers; + const bool build_variant_ok = build_variant_matches == pa_scheduler::kWorkers; + const bool phase_id_ok = phase_id_matches == pa_scheduler::kWorkers; + const bool phase_status_ok = phase_status_trusted == pa_scheduler::kWorkers; + const bool shadow_partition_ok = phase_shadow_acceptable == pa_scheduler::kWorkers; + const bool phase_boundaries_ok = phase_boundary_matches == pa_scheduler::kWorkers; + const bool phase_call_shape_ok = phase_call_shape_matches == pa_scheduler::kWorkers; + const bool phase_time_ok = phase_time_valid_records == pa_scheduler::kWorkers; + const bool phase_calls_ok = phase_calls == expected_phase_calls; + const bool submit_engine_observation_ok = + pmu.mode != WindowMode::SubmitAll || + workload.mode != pa_scheduler::WinnerWorkloadMode::RealCompute || + submit_engine_workers_matched == pa_scheduler::kWorkers; + const bool counter_below_risk_threshold = + maximum_programmable_counter < kProgrammableCounterRiskThreshold; + std::printf( + "[PMU] run=%u window=%s calibration_scalar_nops=%u icache_trials=%u trusted=%u/%u " + "unique_coreids=%u/%u prior_larger=%u/%u icache_pairs=%u/%u calibrated_cores=%u/%u " + "programmable_max=%u headroom=%u\n", run, + PmuModeName(pmu.mode), pmu.scalar_nops, pmu.icache_trials, trusted, pa_scheduler::kWorkers, + unique, pa_scheduler::kWorkers, prior_larger, pa_scheduler::kWorkers, + icache_pairs, pa_scheduler::kWorkers, icache_calibrated_cores, pa_scheduler::kWorkers, + maximum_programmable_counter, + UINT32_MAX - maximum_programmable_counter + ); + std::printf( + "[PMU-SHADOW-DELTA] exact=%u/%u bounded=%u/%u request_abs_sum=%llu request_abs_max=%u " + "request_signed_sum=%lld miss_abs_sum=%llu miss_abs_max=%u miss_signed_sum=%lld\n", + shadow_primary_matches, pa_scheduler::kWorkers, + shadow_primary_bounded, pa_scheduler::kWorkers, + static_cast(shadow_request_abs_delta_sum), + shadow_request_abs_delta_max, static_cast(shadow_request_signed_delta_sum), + static_cast(shadow_miss_abs_delta_sum), + shadow_miss_abs_delta_max, static_cast(shadow_miss_signed_delta_sum) + ); + std::printf("[ASSERT] %-48s %s\n", "all PMU records have configured selectors and data", + records_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 96 PMU physical subcore ids are unique", + core_ids_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all PMU physical ids belong to the owner bitmap", + owner_members_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "worker result slots and ids match exactly", + worker_slots_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "logical AIC/AIV roles match physical subcores", + physical_roles_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 32 mixed blocks map to physical 1:2 triplets", + mixed_triplets_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 96 PMU windows executed start", + windows_started_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 96 started PMU windows executed stop", + windows_stopped_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all records match submit-pmu build and phase ids", + build_variant_ok && phase_id_ok ? "PASS" : "FAIL"); + std::printf( + "[ASSERT] %-48s %s\n", + "phase shadow partitions satisfy exact-or-bounded contract", + shadow_partition_ok ? "PASS" : "FAIL" + ); + std::printf("[ASSERT] %-48s %s\n", "all phase boundaries and per-worker calls are exact", + phase_status_ok && phase_boundaries_ok && phase_call_shape_ok && phase_calls_ok + ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all phase times fit their per-worker Submit windows", + phase_time_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "I-cache misses do not exceed requests", + icache_order_valid ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "programmable counters stay below 25% risk threshold", + counter_below_risk_threshold ? "PASS" : "FAIL"); + if (pmu.mode == WindowMode::SubmitAll && + workload.mode == pa_scheduler::WinnerWorkloadMode::RealCompute) { + std::printf( + "[ASSERT] %-48s %s (active_workers=%u matched_workers=%u/%u)\n", + "Submit placement has matching AIC/AIV engine PMU", + submit_engine_observation_ok ? "PASS" : "FAIL", submit_engine_workers_expected, + submit_engine_workers_matched, pa_scheduler::kWorkers + ); + } + if (pmu.mode == WindowMode::IcacheSingle) { + std::printf("[ASSERT] %-48s %s\n", "each cold trial adds exactly one CNT7 I-cache miss", + icache_measurement_ok ? "PASS" : "FAIL"); + } + validation->trusted = trusted; + validation->unique_physical_core_ids = unique; + validation->owner_bitmap_members = owner_members; + validation->exact_worker_slots = exact_worker_slots; + validation->physical_role_matches = physical_role_matches; + validation->mixed_triplet_matches = mixed_triplet_matches; + validation->window_started = window_started; + validation->window_stopped = window_stopped; + validation->icache_pairs = icache_pairs; + validation->icache_calibrated_cores = icache_calibrated_cores; + validation->prior_snapshot_larger = prior_larger; + validation->submit_engine_workers_expected = submit_engine_workers_expected; + validation->submit_engine_workers_matched = submit_engine_workers_matched; + validation->maximum_programmable_counter = maximum_programmable_counter; + validation->build_variant_matches = build_variant_matches; + validation->phase_id_matches = phase_id_matches; + validation->phase_status_trusted = phase_status_trusted; + validation->shadow_primary_matches = shadow_primary_matches; + validation->shadow_primary_bounded = shadow_primary_bounded; + validation->phase_shadow_acceptable = phase_shadow_acceptable; + validation->phase_boundary_matches = phase_boundary_matches; + validation->phase_call_shape_matches = phase_call_shape_matches; + validation->phase_time_valid_records = phase_time_valid_records; + validation->phase_calls = phase_calls; + validation->expected_phase_calls = expected_phase_calls; + validation->shadow_request_abs_delta_sum = shadow_request_abs_delta_sum; + validation->shadow_miss_abs_delta_sum = shadow_miss_abs_delta_sum; + validation->shadow_request_signed_delta_sum = shadow_request_signed_delta_sum; + validation->shadow_miss_signed_delta_sum = shadow_miss_signed_delta_sum; + validation->shadow_request_abs_delta_max = shadow_request_abs_delta_max; + validation->shadow_miss_abs_delta_max = shadow_miss_abs_delta_max; + validation->icache_order_valid = icache_order_valid; + validation->icache_measurement_valid = icache_measurement_ok; + validation->submit_engine_observation_valid = submit_engine_observation_ok; + validation->counter_below_risk_threshold = counter_below_risk_threshold; + validation->phase_measurement_valid = + build_variant_ok && phase_id_ok && phase_status_ok && shadow_partition_ok && + phase_boundaries_ok && phase_call_shape_ok && phase_calls_ok && phase_time_ok; + validation->passed = records_ok && core_ids_ok && owner_members_ok && worker_slots_ok && + physical_roles_ok && mixed_triplets_ok && windows_started_ok && windows_stopped_ok && + icache_order_valid && icache_measurement_ok && submit_engine_observation_ok && + validation->phase_measurement_valid && + counter_below_risk_threshold; + return validation->passed; +} + +void WriteJsonString(std::FILE *output, const std::string &value) { + std::fputc('"', output); + for (unsigned char character : value) { + switch (character) { + case '"': + std::fputs("\\\"", output); + break; + case '\\': + std::fputs("\\\\", output); + break; + case '\b': + std::fputs("\\b", output); + break; + case '\f': + std::fputs("\\f", output); + break; + case '\n': + std::fputs("\\n", output); + break; + case '\r': + std::fputs("\\r", output); + break; + case '\t': + std::fputs("\\t", output); + break; + default: + if (character < 0x20U) { + std::fprintf(output, "\\u%04x", static_cast(character)); + } else { + std::fputc(character, output); + } + } + } + std::fputc('"', output); +} + +void WriteMetricDistribution(std::FILE *output, const std::vector &values) { + const pa_scheduler::host::Uint64Distribution summary = pa_scheduler::host::SummarizeUint64(values); + const double mean = values.empty() ? 0.0 : static_cast(summary.total) / values.size(); + std::fprintf( + output, "{\"sum\":%llu,\"mean\":%.17g,\"median\":%.17g,\"p95\":%llu,\"max\":%llu}", + static_cast(summary.total), mean, summary.median, + static_cast(summary.p95), static_cast(summary.maximum) + ); +} + +void WritePmuAggregateJson( + std::FILE *output, const PmuAggregate &aggregate, bool icache_single +) { + (void)icache_single; + const pa_scheduler::host::Uint64Distribution requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + uint32_t active_cores = 0; + for (uint64_t cycles : aggregate.total_cycles) active_cores += cycles != 0; + std::fprintf( + output, "{\"cores\":%zu,\"active_cores\":%u,\"trusted_cores\":%u,\"total_cycles\":", + aggregate.total_cycles.size(), active_cores, aggregate.trusted + ); + WriteMetricDistribution(output, aggregate.total_cycles); + std::fputs(",\"vector_busy\":", output); + WriteMetricDistribution(output, aggregate.vector_busy); + std::fputs(",\"cube_busy\":", output); + WriteMetricDistribution(output, aggregate.cube_busy); + std::fputs(",\"scalar_busy\":", output); + WriteMetricDistribution(output, aggregate.scalar_busy); + std::fputs(",\"mte1_busy\":", output); + WriteMetricDistribution(output, aggregate.mte1_busy); + std::fputs(",\"mte2_busy\":", output); + WriteMetricDistribution(output, aggregate.mte2_busy); + std::fputs(",\"icache_requests\":", output); + WriteMetricDistribution(output, aggregate.icache_requests); + std::fputs(",\"icache_misses\":", output); + WriteMetricDistribution(output, aggregate.icache_misses); + std::fputs(",\"shadow_whole_icache_requests\":", output); + WriteMetricDistribution(output, aggregate.shadow_icache_requests); + std::fputs(",\"shadow_whole_icache_misses\":", output); + WriteMetricDistribution(output, aggregate.shadow_icache_misses); + std::fputs(",\"phase_calls\":", output); + WriteMetricDistribution(output, aggregate.phase_calls); + std::fputs(",\"submit_elapsed_ticks\":", output); + WriteMetricDistribution(output, aggregate.submit_elapsed_ticks); + std::fputs(",\"phase_elapsed_ticks\":", output); + WriteMetricDistribution(output, aggregate.phase_elapsed_ticks); + std::fputs(",\"phase_icache_requests\":", output); + WriteMetricDistribution(output, aggregate.phase_icache_requests); + std::fputs(",\"phase_icache_misses\":", output); + WriteMetricDistribution(output, aggregate.phase_icache_misses); + std::fputs(",\"icache_miss_rate\":", output); + if (requests.total == 0) { + std::fputs("null", output); + } else { + // 全局 miss rate 必须以总 miss/总 request 计算，不能平均逐核百分比。 + std::fprintf(output, "%.17g", static_cast(misses.total) / requests.total); + } + const pa_scheduler::host::Uint64Distribution phase_requests = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_requests); + const pa_scheduler::host::Uint64Distribution phase_misses = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_misses); + // 两个 phase counter 是顺序 read-to-clear，下界之比不是实际 miss rate + // 的数学下界；字段名只陈述它是本次 read-clear 观察值之比。 + std::fputs(",\"phase_observed_read_clear_ratio\":", output); + if (phase_requests.total == 0U) { + std::fputs("null", output); + } else { + std::fprintf(output, "%.17g", static_cast(phase_misses.total) / phase_requests.total); + } + std::fputc('}', output); +} + +uint32_t CountConfiguredMixedTriplets(const pa_scheduler::pmu_owner::PmuOwnerControl &owner) { + uint32_t complete = 0U; + for (uint32_t die_base = 0U; + die_base < pa_scheduler::pmu_owner::kPhysicalSubcoreCount; + die_base += pa_scheduler::pmu_owner::kSubcoresPerDie) { + for (uint32_t local = 0U; local < pa_scheduler::pmu_owner::kAicPerDie; ++local) { + const uint32_t aic = die_base + local; + const uint32_t aiv0 = die_base + pa_scheduler::pmu_owner::kAicPerDie + local * 2U; + const uint32_t aiv1 = aiv0 + 1U; + complete += pa_scheduler::pmu_owner::IsConfigured(owner, aic) && + pa_scheduler::pmu_owner::IsConfigured(owner, aiv0) && + pa_scheduler::pmu_owner::IsConfigured(owner, aiv1); + } + } + return complete; +} + +bool ExportPmuJson( + const pa_scheduler::SchedulerState &state, const pa_scheduler::host::Options &options, + const PmuOptions &pmu, const WinnerWorkloadOptions &workload, + uint32_t run, double host_us, double submit_span_us, + const PmuValidation &validation, bool semantic_passed, bool workload_output_passed, + const pa_scheduler::pmu_owner::PmuOwnerControl &owner, bool restore_passed, + const std::string &output_path +) { + using namespace pa_scheduler::ccec_pmu; + PmuAggregate all; + PmuAggregate aic; + PmuAggregate aiv; + uint32_t active_output_tiles = 0; + uint64_t ef_drain_kernels = 0; + uint64_t ring_backpressure_kernels = 0; + uint64_t final_drain_kernels = 0; + for (uint32_t worker = 0; worker < pa_scheduler::kWorkers; ++worker) { + const pa_scheduler::WorkerResult &result = state.results[worker]; + AddPmuSample(result, &all); + AddPmuSample(result, result.role == static_cast(pa_scheduler::CoreRole::Aic) ? &aic : &aiv); + if (result.role == static_cast(pa_scheduler::CoreRole::Aic)) { + active_output_tiles += result.kernel_counts[0] != 0; + active_output_tiles += result.kernel_counts[2] != 0; + } else if (result.role == static_cast(pa_scheduler::CoreRole::Aiv)) { + active_output_tiles += result.kernel_counts[1] != 0; + active_output_tiles += result.kernel_counts[3] != 0; + } + ef_drain_kernels += result.placement[static_cast(pa_scheduler::DrainPlace::EfDrain)]; + ring_backpressure_kernels += + result.placement[static_cast(pa_scheduler::DrainPlace::RingBackpressure)]; + final_drain_kernels += result.placement[static_cast(pa_scheduler::DrainPlace::FinalDrain)]; + } + + const auto generated = std::chrono::system_clock::now().time_since_epoch(); + const uint64_t generated_ns = static_cast( + std::chrono::duration_cast(generated).count() + ); + const std::string capture_id = "pa-pmu-" + std::to_string(generated_ns) + "-run" + std::to_string(run); + const std::string temporary_path = output_path + ".tmp"; + // 临时文件与最终文件都采用 no-replace 语义：并发采集不能截断同名 tmp， + // 也不能在最终发布时覆盖另一份已经完成的证据文件。 + const int output_fd = open(temporary_path.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0644); + std::FILE *output = output_fd < 0 ? nullptr : fdopen(output_fd, "wb"); + if (output == nullptr) { + const int open_error = errno; + if (output_fd >= 0) { + (void)close(output_fd); + (void)std::remove(temporary_path.c_str()); + } + std::fprintf( + stderr, "Cannot exclusively create PMU JSON output %s: %s\n", temporary_path.c_str(), + std::strerror(open_error) + ); + return false; + } + std::vector output_buffer(1U << 20); + std::setvbuf(output, output_buffer.data(), _IOFBF, output_buffer.size()); + + const bool submit_window = IsSubmitWindow(pmu.mode); + const bool icache_single = pmu.mode == WindowMode::IcacheSingle; + const bool real_compute = workload.mode == pa_scheduler::WinnerWorkloadMode::RealCompute; + const bool simulated_task_nops_nonzero = !real_compute && + (options.nops.qk != 0U || options.nops.sf != 0U || + options.nops.pv != 0U || options.nops.up != 0U); + const pa_scheduler::WorkloadCounts active_counts = real_compute + ? workload.repeats + : pa_scheduler::WorkloadCounts{ + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + }; + const uint32_t owner_bitmap_count = pa_scheduler::pmu_owner::CountConfigured(owner); + const uint32_t owner_complete_triplets = CountConfiguredMixedTriplets(owner); + std::fputs("{\n\"schema\":{\"name\":\"pa_scheduler_pmu_phase_windows\",\"version\":5},\n", output); + std::fputs("\"capture\":{\"capture_id\":", output); + WriteJsonString(output, capture_id); + std::fprintf( + output, + ",\"generated_unix_time_ns\":%llu,\"run_index\":%u,\"accepted\":true," + "\"usable_for_same_configuration_submit_comparison\":%s," + "\"usable_as_absolute_real_pa_profile\":false,\"window_scope\":\"%s\"," + "\"pmu_probe_position\":\"%s\",\"scheduler_hot_path_included\":%s," + "\"total_sum_is_core_work_not_wall_time\":true," + "\"submit_window_excludes_final_drain\":%s," + "\"published_after_runtime_cleanup\":true,\"runtime_cleanup_passed\":true," + "\"owner_restore_passed\":%s},\n", + static_cast(generated_ns), run, + submit_window ? "true" : "false", + submit_window ? "per_worker_orchestration_to_last_submit_return" : "post_scheduler_calibration_probe", + submit_window ? "inside_RunScheduler" : "after_RunScheduler", + submit_window ? "true" : "false", submit_window ? "true" : "false", + restore_passed ? "true" : "false" + ); + std::fputs("\"configuration\":{\"kernel_path\":", output); + WriteJsonString(output, options.kernel_path); + std::fputs(",\"build_variant\":\"submit-pmu\",\"build_variant_id\":2,\"compiled_phase\":", output); + WriteJsonString(output, SubmitPmuPhaseName(pa_scheduler::kCompiledSubmitPmuPhase)); + std::fprintf( + output, ",\"compiled_phase_id\":%u", + static_cast(pa_scheduler::kCompiledSubmitPmuPhase) + ); + std::fprintf( + output, + ",\"device\":%u,\"batches\":%u,\"workers\":%u,\"aic_workers\":%u,\"aiv_workers\":%u," + "\"trace_enabled\":%s,\"trace_atomics\":%s,\"profile_phases\":%s," + "\"winner_workload\":{\"mode\":", + options.device, options.batches, pa_scheduler::kWorkers, pa_scheduler::kAicWorkers, + pa_scheduler::kAivWorkers, options.trace_enabled ? "true" : "false", + options.trace_atomics ? "true" : "false", + options.profile_phases ? "true" : "false" + ); + WriteJsonString(output, WinnerWorkloadModeName(workload.mode)); + std::fputs(",\"input_pattern\":", output); + WriteJsonString( + output, + real_compute ? pa_scheduler::host::RealComputePatternName(workload.pattern) : "none" + ); + std::fprintf( + output, + ",\"config_version\":%u,\"counts\":{\"qk\":%u,\"sf\":%u,\"pv\":%u,\"up\":%u}," + "\"unit\":", + pa_scheduler::kWinnerWorkloadConfigVersion, active_counts.qk, active_counts.sf, + active_counts.pv, active_counts.up + ); + WriteJsonString( + output, + real_compute ? "complete_128x128_engine_pipeline_iteration" : "scalar_nop_instruction" + ); + std::fprintf( + output, + ",\"tile_rows\":%u,\"tile_cols\":%u,\"shared_input_tiles\":%u," + "\"output_tiles_per_worker\":%u,\"workspace_bytes\":%zu,\"role_mapping\":", + real_compute ? pa_scheduler::winner_workload::kTileRows : 0, + real_compute ? pa_scheduler::winner_workload::kTileCols : 0, + real_compute ? pa_scheduler::winner_workload::kSharedInputTiles : 0, + real_compute ? pa_scheduler::winner_workload::kOutputTilesPerWorker : 0, + real_compute ? pa_scheduler::winner_workload::kWorkspaceBytes : 0 + ); + if (real_compute) { + std::fputs( + "{\"qk\":\"cube_matmul\",\"pv\":\"cube_matmul\"," + "\"sf\":\"vector_add\",\"up\":\"vector_mul\"}", + output + ); + } else { + std::fputs("null", output); + } + std::fprintf( + output, ",\"engine_completion_waited_before_task_publish\":%s},\"nop_counts\":", + real_compute ? "true" : "false" + ); + if (real_compute) { + std::fputs("null", output); + } else { + std::fprintf( + output, "{\"qk\":%u,\"sf\":%u,\"pv\":%u,\"up\":%u}", + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + ); + } + std::fputs(",\"pmu_window\":", output); + WriteJsonString(output, PmuModeName(pmu.mode)); + std::fputs(",\"calibration_scalar_nops_per_segment\":", output); + if (submit_window || icache_single) { + std::fputs("null", output); + } else { + std::fprintf(output, "%u", pmu.scalar_nops); + } + std::fputs(",\"icache_single_trials_per_core\":", output); + if (icache_single) { + std::fprintf(output, "%u", pmu.icache_trials); + } else { + std::fputs("null", output); + } + std::fprintf( + output, + ",\"primary_window_segments_per_record\":1," + "\"icache_single_discarded_training_samples_per_core\":%u," + "\"icache_single_sys_counter_tick_ns\":%s," + "\"host_launch_to_sync_us\":%.17g,\"submit_span_us\":%.17g," + "\"selectors\":{\"cnt0_vector_busy\":%u,\"cnt1_cube_busy\":%u," + "\"cnt2_scalar_busy\":%u,\"cnt3_mte1_busy\":%u,\"cnt4_mte2_busy\":%u," + "\"cnt5_shadow_icache_miss\":%u,\"cnt6_primary_icache_request\":%u," + "\"cnt7_primary_icache_miss\":%u,\"cnt8_shadow_icache_request\":%u," + "\"cnt9_unused\":0},\"unavailable_metrics\":[\"mte3_busy\"]," + "\"counter_width_bits\":{\"total\":64,\"programmable\":32}," + "\"counter_wrap_not_directly_detectable\":true,\"counter_wrap_absence_proven\":false," + "\"programmable_counter_risk_threshold\":%u," + "\"gate_start_stop_have_pipe_all_barriers\":true," + "\"phase_timestamp_calls_present\":%s,\"phase_record_writes\":false," + "\"atomic_trace\":false,\"profile_accumulation\":false," + "\"phase_boundary_observation_included\":%s," + "\"phase_time_observation_included\":%s," + "\"phase_time_sys_counter_tick_ns\":1," + "\"phase_time_boundary\":\"after_begin_read_clear_to_before_end_read_clear\"," + "\"phase_time_excludes_shadow_read_overhead\":true," + "\"phase_time_includes_timestamp_overhead\":true," + "\"phase_time_share_definition\":" + "\"sum(phase_elapsed_ticks)/sum(submit_elapsed_ticks)\"," + "\"phase_time_denominator_scope\":" + "\"per_worker_first_submit_begin_to_last_submit_end\"," + "\"phase_counter_pair_snapshot_atomic\":false," + "\"primary_counters_read_at_phase_boundaries\":false," + "\"phase_shadow_partition_exact_required\":%s," + "\"phase_values_are_running_read_clear_lower_bounds\":%s," + "\"cross_phase_elf_sums_valid\":false," + "\"simulated_task_nop_mechanism_executes_on_scalar\":%s," + "\"simulated_task_nops_nonzero\":%s," + "\"icache_miss_rate_definition\":\"sum(icache_misses)/sum(icache_requests)\"},\n", + icache_single ? 2U : 0U, icache_single ? "1" : "null", + host_us, submit_span_us, kVectorBusyEvent, kCubeBusyEvent, kScalarBusyEvent, + kMte1BusyEvent, kMte2BusyEvent, kIcacheMissEvent, kIcacheRequestEvent, kIcacheMissEvent, + kIcacheRequestEvent, kProgrammableCounterRiskThreshold, + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "true" + : "false", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + real_compute ? "false" : "true", + simulated_task_nops_nonzero ? "true" : "false" + ); + std::fprintf( + output, + "\"validation\":{\"semantic_passed\":%s,\"pmu_passed\":%s," + "\"real_compute_output_validation_required\":%s," + "\"real_compute_output_validation_passed\":%s," + "\"real_compute_active_output_tiles\":%u," + "\"real_compute_inactive_sentinel_tiles\":%u," + "\"real_compute_output_mismatches\":%u," + "\"submit_engine_observation_valid\":%s," + "\"submit_engine_workers_expected\":%u," + "\"submit_engine_workers_matched\":%u," + "\"kernel_placement_counts\":{\"ef_drain\":%llu,\"ring_backpressure\":%llu," + "\"final_drain\":%llu},\"trusted_records\":%u," + "\"expected_records\":%u,\"unique_physical_core_ids\":%u,\"expected_unique_core_ids\":%u," + "\"owner_bitmap_member_records\":%u,\"expected_owner_bitmap_member_records\":%u," + "\"exact_worker_slot_records\":%u,\"expected_exact_worker_slot_records\":%u," + "\"physical_role_match_records\":%u,\"expected_physical_role_match_records\":%u," + "\"mixed_triplet_matches\":%u,\"expected_mixed_triplet_matches\":%u," + "\"window_started_records\":%u,\"window_stopped_records\":%u," + "\"expected_window_records\":%u,\"prior_snapshot_larger_records\":%u," + "\"icache_pair_records\":%u,\"icache_calibrated_cores\":%u," + "\"icache_measurement_valid\":%s," + "\"icache_miss_le_request\":%s,\"counter_below_risk_threshold\":%s," + "\"maximum_programmable_counter\":%u,\"programmable_counter_risk_threshold\":%u," + "\"programmable_counter_headroom\":%u," + "\"build_variant_match_records\":%u,\"phase_id_match_records\":%u," + "\"phase_status_trusted_records\":%u,\"shadow_primary_match_records\":%u," + "\"shadow_primary_bounded_records\":%u," + "\"phase_shadow_acceptable_records\":%u," + "\"shadow_request_abs_delta_sum\":%llu,\"shadow_request_abs_delta_max\":%u," + "\"shadow_request_signed_delta_sum\":%lld," + "\"shadow_miss_abs_delta_sum\":%llu,\"shadow_miss_abs_delta_max\":%u," + "\"shadow_miss_signed_delta_sum\":%lld," + "\"phase_boundary_match_records\":%u,\"phase_call_shape_match_records\":%u," + "\"phase_time_valid_records\":%u,\"phase_time_measurement_valid\":%s," + "\"phase_calls\":%llu,\"phase_expected_calls\":%llu," + "\"phase_measurement_valid\":%s},\n", + semantic_passed ? "true" : "false", validation.passed ? "true" : "false", + real_compute ? "true" : "false", + real_compute ? (workload_output_passed ? "true" : "false") : "null", + real_compute ? active_output_tiles : 0, + real_compute ? pa_scheduler::winner_workload::kOutputTiles - active_output_tiles : 0, + real_compute && !workload_output_passed ? 1U : 0U, + validation.submit_engine_observation_valid ? "true" : "false", + validation.submit_engine_workers_expected, + validation.submit_engine_workers_matched, + static_cast(ef_drain_kernels), + static_cast(ring_backpressure_kernels), + static_cast(final_drain_kernels), + validation.trusted, + pa_scheduler::kWorkers, validation.unique_physical_core_ids, pa_scheduler::kWorkers, + validation.owner_bitmap_members, pa_scheduler::kWorkers, + validation.exact_worker_slots, pa_scheduler::kWorkers, + validation.physical_role_matches, pa_scheduler::kWorkers, + validation.mixed_triplet_matches, pa_scheduler::kAicWorkers, + validation.window_started, validation.window_stopped, pa_scheduler::kWorkers, + validation.prior_snapshot_larger, validation.icache_pairs, + validation.icache_calibrated_cores, + validation.icache_measurement_valid ? "true" : "false", + validation.icache_order_valid ? "true" : "false", + validation.counter_below_risk_threshold ? "true" : "false", + validation.maximum_programmable_counter, kProgrammableCounterRiskThreshold, + UINT32_MAX - validation.maximum_programmable_counter, + validation.build_variant_matches, validation.phase_id_matches, + validation.phase_status_trusted, validation.shadow_primary_matches, + validation.shadow_primary_bounded, validation.phase_shadow_acceptable, + static_cast(validation.shadow_request_abs_delta_sum), + validation.shadow_request_abs_delta_max, + static_cast(validation.shadow_request_signed_delta_sum), + static_cast(validation.shadow_miss_abs_delta_sum), + validation.shadow_miss_abs_delta_max, + static_cast(validation.shadow_miss_signed_delta_sum), + validation.phase_boundary_matches, + validation.phase_call_shape_matches, + validation.phase_time_valid_records, + validation.phase_time_valid_records == pa_scheduler::kWorkers ? "true" : "false", + static_cast(validation.phase_calls), + static_cast(validation.expected_phase_calls), + validation.phase_measurement_valid ? "true" : "false" + ); + std::fprintf( + output, + "\"owner\":{\"mode\":\"main_aicpu_path_a\"," + "\"snapshot_phase\":\"after_configure_before_restore\"," + "\"control_magic\":%u,\"control_version\":%u,\"configure_status\":%d," + "\"configured_flag\":%u,\"configured_bitmap_count\":%u," + "\"expected\":{\"total\":%u,\"aic\":%u,\"aiv\":%u}," + "\"active\":{\"total\":%u,\"aic\":%u,\"aiv\":%u}," + "\"discovered\":{\"total\":%u,\"aic\":%u,\"aiv\":%u}," + "\"physical_slots_scanned\":%u,\"skipped_physical_slots\":%u," + "\"configured_bitmap_word_order\":\"least_significant_physical_ids_first\"," + "\"configured_bitmap_words\":[%u,%u,%u,%u]," + "\"configured_complete_mixed_triplets\":%u,\"expected_complete_mixed_triplets\":%u," + "\"configured_broken_mixed_triplets\":%u,\"restore_passed\":%s},\n", + owner.magic, owner.version, static_cast(owner.status), owner.configured, owner_bitmap_count, + owner.expected_total, owner.expected_aic, owner.expected_aiv, + owner.active_total, owner.active_aic, owner.active_aiv, + owner.discovered_total, owner.discovered_aic, owner.discovered_aiv, + pa_scheduler::pmu_owner::kPhysicalSubcoreCount, owner.skipped_total, + owner.configured_bitmap[0], owner.configured_bitmap[1], + owner.configured_bitmap[2], owner.configured_bitmap[3], + owner_complete_triplets, pa_scheduler::kAicWorkers, + owner_bitmap_count / 3U - owner_complete_triplets, restore_passed ? "true" : "false" + ); + std::fputs("\"records\":[\n", output); + for (uint32_t worker = 0; worker < pa_scheduler::kWorkers; ++worker) { + const pa_scheduler::WorkerResult &result = state.results[worker]; + const uint32_t status = result.pmu_status; + const uint32_t physical_core_id = StatusCoreId(status); + const bool primary_trusted = (status & kStatusRequired) == kStatusRequired; + const bool phase_trusted = + (result.pmu_phase_status & kPhaseStatusRequired) == kPhaseStatusRequired; + const uint64_t submit_elapsed_ticks = result.submit_end >= result.submit_begin + ? result.submit_end - result.submit_begin + : 0U; + const bool phase_time_valid = + submit_elapsed_ticks != 0U && + result.pmu_phase_elapsed_ticks <= submit_elapsed_ticks && + (pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? result.pmu_phase_elapsed_ticks == 0U + : result.pmu_phase_elapsed_ticks != 0U); + const bool trusted = primary_trusted && phase_trusted && phase_time_valid; + const bool is_aic = result.role == static_cast(pa_scheduler::CoreRole::Aic); + const uint32_t vector_id = is_aic ? 0U : worker - pa_scheduler::kAicWorkers; + const uint32_t block_id = is_aic ? worker : vector_id / 2U; + const uint32_t lane = is_aic ? 0U : 1U + vector_id % 2U; + const uint32_t shadow_read_segments = result.pmu_phase_calls * 2U + 1U; + const bool owner_bitmap_member = pa_scheduler::pmu_owner::IsConfigured(owner, physical_core_id); + const bool worker_slot_exact = result.worker_id == worker; + const bool physical_role_matches = + is_aic == pa_scheduler::pmu_owner::IsAicPhysicalSlot(physical_core_id); + const bool window_started = (status & kStatusWindowStarted) != 0U; + const bool window_stopped = (status & kStatusWindowStopped) != 0U; + const bool shadow_matches = + result.pmu_shadow_icache_requests == result.pmu_icache_requests && + result.pmu_shadow_icache_misses == result.pmu_icache_misses; + const bool shadow_bounded = + result.pmu_shadow_icache_requests <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_icache_misses; + const uint32_t shadow_request_loss = + result.pmu_icache_requests - result.pmu_shadow_icache_requests; + const uint32_t shadow_miss_loss = + result.pmu_icache_misses - result.pmu_shadow_icache_misses; + const uint32_t phase_request_upper = + result.pmu_phase_icache_requests + shadow_request_loss; + const uint32_t phase_miss_upper = + result.pmu_phase_icache_misses + shadow_miss_loss; + const bool boundaries_balanced = + result.pmu_phase_begin_reads == result.pmu_phase_calls && + result.pmu_phase_end_reads == result.pmu_phase_calls; + const uint32_t expected_phase_calls = + ExpectedSubmitPmuPhaseCallsPerWorker(state.config.batches); + std::fprintf( + output, + "%s{\"worker_id\":%u,\"physical_core_id\":%u,\"role\":\"%s\",\"block_id\":%u," + "\"lane\":%u,\"primary_window_segments\":1,\"shadow_read_segments\":%u," + "\"window_started\":%s,\"window_stopped\":%s,\"total_cycles\":%llu,\"vector_busy\":%u," + "\"cube_busy\":%u,\"scalar_busy\":%u,\"mte1_busy\":%u,\"mte2_busy\":%u," + "\"icache_requests\":%u,\"icache_misses\":%u," + "\"build_variant_id\":%u,\"compiled_phase_id\":%u,\"phase_calls\":%u," + "\"phase_expected_calls\":%u," + "\"phase_begin_reads\":%u,\"phase_end_reads\":%u," + "\"submit_elapsed_ticks\":%llu,\"phase_elapsed_ticks\":%llu," + "\"phase_time_valid\":%s," + "\"phase_icache_requests\":%u,\"phase_icache_misses\":%u," + "\"phase_icache_requests_upper_bound\":%u," + "\"phase_icache_misses_upper_bound\":%u," + "\"shadow_whole_icache_requests\":%u,\"shadow_whole_icache_misses\":%u," + "\"shadow_matches_primary\":%s,\"shadow_not_greater_than_primary\":%s," + "\"shadow_request_loss\":%u,\"shadow_miss_loss\":%u," + "\"phase_boundaries_balanced\":%s," + "\"phase_status\":%u,\"phase_status_hex\":\"0x%08x\"," + "\"status\":%u,\"status_hex\":" + "\"0x%08x\",\"trusted\":%s,\"physical_core_id_valid\":%s,\"selectors_match\":%s," + "\"owner_bitmap_member\":%s,\"worker_slot_exact\":%s," + "\"physical_role_matches\":%s}", + worker == 0 ? "" : ",\n", worker, physical_core_id, is_aic ? "aic" : "aiv", block_id, + lane, shadow_read_segments, window_started ? "true" : "false", window_stopped ? "true" : "false", + static_cast(result.pmu_total_cycles), result.pmu_vector_busy, + result.pmu_cube_busy, result.pmu_scalar_busy, result.pmu_mte1_busy, result.pmu_mte2_busy, + result.pmu_icache_requests, result.pmu_icache_misses, + result.pmu_build_variant, result.pmu_phase_id, result.pmu_phase_calls, + expected_phase_calls, + result.pmu_phase_begin_reads, result.pmu_phase_end_reads, + static_cast(submit_elapsed_ticks), + static_cast(result.pmu_phase_elapsed_ticks), + phase_time_valid ? "true" : "false", + result.pmu_phase_icache_requests, result.pmu_phase_icache_misses, + phase_request_upper, phase_miss_upper, + result.pmu_shadow_icache_requests, result.pmu_shadow_icache_misses, + shadow_matches ? "true" : "false", shadow_bounded ? "true" : "false", + shadow_request_loss, shadow_miss_loss, boundaries_balanced ? "true" : "false", + result.pmu_phase_status, result.pmu_phase_status, + status, status, trusted ? "true" : "false", + (status & kStatusCoreIdValid) != 0 ? "true" : "false", + (status & (kStatusCnt0Selector | kStatusCnt1Selector | kStatusCnt2Selector | + kStatusCnt3Selector | kStatusCnt4Selector | kStatusCnt5Selector | + kStatusCnt6Selector | kStatusCnt7Selector | kStatusCnt8Selector)) == + (kStatusCnt0Selector | kStatusCnt1Selector | kStatusCnt2Selector | + kStatusCnt3Selector | kStatusCnt4Selector | kStatusCnt5Selector | + kStatusCnt6Selector | kStatusCnt7Selector | kStatusCnt8Selector) + ? "true" + : "false", + owner_bitmap_member ? "true" : "false", worker_slot_exact ? "true" : "false", + physical_role_matches ? "true" : "false" + ); + } + std::fputs("\n],\n\"summary\":{\"all\":", output); + WritePmuAggregateJson(output, all, icache_single); + std::fputs(",\"aic\":", output); + WritePmuAggregateJson(output, aic, icache_single); + std::fputs(",\"aiv\":", output); + WritePmuAggregateJson(output, aiv, icache_single); + std::fputs("}\n}\n", output); + + bool success = std::ferror(output) == 0; + int write_error = success ? 0 : EIO; + if (std::fflush(output) != 0) { + success = false; + write_error = errno; + } + if (success && fsync(fileno(output)) != 0) { + success = false; + write_error = errno; + } + if (std::fclose(output) != 0) { + success = false; + write_error = errno; + } + if (!success) { + std::fprintf(stderr, "Failed while writing PMU JSON output %s: %s\n", temporary_path.c_str(), + std::strerror(write_error)); + (void)std::remove(temporary_path.c_str()); + return false; + } + // 同目录 hard-link 在最终名称不存在时原子发布；EEXIST 时保留既有证据， + // 不采用会替换目标的 POSIX rename。 + if (link(temporary_path.c_str(), output_path.c_str()) != 0) { + std::fprintf( + stderr, "Cannot publish PMU JSON without replacement %s -> %s: %s\n", + temporary_path.c_str(), output_path.c_str(), + std::strerror(errno) + ); + (void)std::remove(temporary_path.c_str()); + return false; + } + if (unlink(temporary_path.c_str()) != 0) { + const int unlink_error = errno; + // 最终文件已链接但事务尚未完成；尽力撤回最终名称，避免失败返回时留下 + // 一份被调用方误认为成功发布的文件。 + (void)unlink(output_path.c_str()); + std::fprintf( + stderr, "Cannot remove PMU JSON temporary link %s: %s\n", temporary_path.c_str(), + std::strerror(unlink_error) + ); + return false; + } + std::printf("[PMU-JSON] capture_id=%s records=%u output=%s\n", capture_id.c_str(), pa_scheduler::kWorkers, + output_path.c_str()); + return true; +} + +} // namespace + +int main(int argc, char **argv) { + // 参数和 ELF 在创建 ACL 资源前完成校验，早期错误不会留下 device、stream 或 kernel handle。 + pa_scheduler::host::Options options; + PmuOptions pmu_options; + WinnerWorkloadOptions workload_options; + std::vector pmu_argv; + std::vector common_argv; + if (!ParseWinnerWorkloadOptions(argc, argv, &workload_options, &pmu_argv) || + !ParsePmuOptions( + static_cast(pmu_argv.size()), pmu_argv.data(), &pmu_options, &common_argv + )) { + return EXIT_FAILURE; + } + const pa_scheduler::host::ParseStatus parse_status = pa_scheduler::host::ParseOptions( + static_cast(common_argv.size()), common_argv.data(), true, &options + ); + if (parse_status != pa_scheduler::host::ParseStatus::Ok) { + if (parse_status == pa_scheduler::host::ParseStatus::Help) { + std::fprintf( + stderr, + "CCEC PMU options: [--pmu-window " + "off|empty|scalar|scalar-double|icache-single|submit-all] " + "[--pmu-scalar-nops N] [--pmu-icache-trials N] [--pmu-json FILE]\n" + ); + std::fprintf( + stderr, + "CCEC winner workload options: [--winner-workload scalar-nop|real-compute] " + "[--real-compute-count N | --real-compute-counts QK,SF,PV,UP] " + "[--real-compute-pattern constant|layout-diagnostic]\n" + ); + std::fprintf( + stderr, + "Default: real-compute, constant, counts=6,28,4,1; " + "scalar-nop is the calibration compatibility mode.\n" + ); + } + return parse_status == pa_scheduler::host::ParseStatus::Help ? EXIT_SUCCESS : EXIT_FAILURE; + } + if (!ValidateWinnerWorkloadOptions(workload_options)) return EXIT_FAILURE; +#if PA_BUILD_SWIMLANE + // swimlane host 与同目录 kernel 是成套产物；它不允许借旧参数重新开启 + // 已从 device ELF 编译掉的 PMU/phase-profile 路径。 + if (pmu_options.mode != pa_scheduler::ccec_pmu::WindowMode::Off || + !pmu_options.json_path.empty()) { + std::fprintf( + stderr, + "This is a swimlane build; PMU collection requires the separate submit-pmu build.\n" + ); + return EXIT_FAILURE; + } + if (options.profile_phases) { + std::fprintf( + stderr, + "--profile-phases is not part of the swimlane build; use submit-pmu phase attribution.\n" + ); + return EXIT_FAILURE; + } +#elif PA_BUILD_SUBMIT_PMU + // submit-pmu 是编译期固定 phase 的单轮诊断产物；host、kernel 与 owner + // 必须共同拒绝旧校准窗口和任何泳道/phase-profile 观察代码。 + if (pmu_options.mode != pa_scheduler::ccec_pmu::WindowMode::SubmitAll) { + std::fprintf(stderr, "The submit-pmu build requires --pmu-window submit-all.\n"); + return EXIT_FAILURE; + } + if (options.runs != 1 || options.trace_enabled || options.trace_atomics || + options.profile_phases || options.analyze_swimlane || !options.swimlane_json.empty()) { + std::fprintf( + stderr, + "submit-pmu requires one PMU-only run: --runs 1 --no-swimlane and no trace/profile options.\n" + ); + return EXIT_FAILURE; + } +#endif + if (!pmu_options.json_path.empty() && + pmu_options.mode == pa_scheduler::ccec_pmu::WindowMode::Off) { + std::fprintf(stderr, "--pmu-json requires a non-off --pmu-window.\n"); + return EXIT_FAILURE; + } + if (!pmu_options.json_path.empty() && options.runs != 1) { + // 一个 sidecar 对应一次采集，禁止多轮覆写后丢失逐轮边界。 + std::fprintf(stderr, "--pmu-json requires --runs 1 to avoid overwriting captures.\n"); + return EXIT_FAILURE; + } + if (!pmu_options.json_path.empty() && + (options.trace_enabled || options.trace_atomics || options.profile_phases || + options.analyze_swimlane || !options.swimlane_json.empty())) { + std::fprintf( + stderr, + "--pmu-json requires PMU-only collection: add --no-swimlane and do not enable " + "phase profiling, atomic tracing, swimlane analysis, or swimlane JSON.\n" + ); + return EXIT_FAILURE; + } + if (!pmu_options.json_path.empty() && + (access(pmu_options.json_path.c_str(), F_OK) == 0 || + access((pmu_options.json_path + ".tmp").c_str(), F_OK) == 0)) { + std::fprintf( + stderr, "Refusing to overwrite an existing PMU JSON or temporary file: %s\n", + pmu_options.json_path.c_str() + ); + return EXIT_FAILURE; + } + const std::vector binary_data = ReadBinary(options.kernel_path); + if (binary_data.empty()) { + std::fprintf(stderr, "Cannot read kernel binary: %s\n", options.kernel_path.c_str()); + return EXIT_FAILURE; + } + const bool real_compute = + workload_options.mode == pa_scheduler::WinnerWorkloadMode::RealCompute; + std::vector workload_image; + std::vector workload_outputs; + if (real_compute) { + pa_scheduler::host::InitializeWinnerWorkloadBuffers( + workload_options, &workload_image, &workload_outputs + ); + } + pa_scheduler::host::PrintBanner("CCEC", options); + pa_scheduler::host::PrintWinnerWorkloadConfig(workload_options, options.nops); + std::printf( + "[PMU-CONFIG] window=%s calibration_scalar_nops=%u icache_trials=%u source=direct-per-core " + "owner=main-aicpu-path-a\n", + PmuModeName(pmu_options.mode), pmu_options.scalar_nops, pmu_options.icache_trials + ); + + // 正常及后处理路径依次完成 ACL 初始化、选卡、stream/ELF/设备区创建、launch/D2H + // 和尾部清理；初始化、传输或 launch 的早期错误仍按当前实现就地返回。 + if (!CheckAcl(aclInit(nullptr), "aclInit") || !CheckAcl(aclrtSetDevice(options.device), "aclrtSetDevice")) { + return EXIT_FAILURE; + } + aclrtStream stream = nullptr; + if (!CheckAcl(aclrtCreateStream(&stream), "aclrtCreateStream")) return EXIT_FAILURE; + + rtDevBinary_t binary{RT_DEV_BINARY_MAGIC_ELF, 0, binary_data.data(), binary_data.size()}; + void *kernel_handle = nullptr; + bool registered_all = true; + // 先尝试注册带 mixed metadata 的 ELF；若 rtRegisterAllKernel 报错或未返回 handle， + // 再尝试无 tiling-key 装载。这里仅描述实际回退条件，不假设具体运行时原因。 + rtError_t register_error = rtRegisterAllKernel(&binary, &kernel_handle); + if (register_error != RT_ERROR_NONE || kernel_handle == nullptr) { + registered_all = false; + register_error = rtBinaryLoadWithoutTilingKey(binary_data.data(), binary_data.size(), &kernel_handle); + } + if (!CheckRt(register_error, "register mixed AICore ELF") || kernel_handle == nullptr) return EXIT_FAILURE; + + // SchedulerState 保留被测关键 offset、DistCore ABI 和约 1 GiB 生产总跨度； + // 使用 HUGE_FIRST 降低大块设备内存碎片风险。 + void *state_device = nullptr; + if (!CheckAcl( + aclrtMalloc(&state_device, sizeof(pa_scheduler::SchedulerState), ACL_MEM_MALLOC_HUGE_FIRST), + "aclrtMalloc(state)" + )) { + return EXIT_FAILURE; + } + if ((reinterpret_cast(state_device) & 63U) != 0) { + std::fprintf(stderr, "Device state is not 64-byte aligned: %p\n", state_device); + return EXIT_FAILURE; + } + + // 真实 PTO 负载使用独立 GM，不解引用调度器中只用于依赖建模的 synthetic tensor 地址。 + // 这里先于 PMU owner 分配；每轮 H2D 初始化虽在 owner 配置之后，但仍位于 + // launch/wall 计时之前，因此两者都不进入 Submit 性能窗口。 + ScopedAclDeviceAllocation workload_allocation; + if (real_compute && + !CheckAcl( + aclrtMalloc( + workload_allocation.Address(), pa_scheduler::winner_workload::kWorkspaceBytes, + ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(real-compute workspace)" + )) { + return EXIT_FAILURE; + } + void *workload_device = workload_allocation.Get(); + if (real_compute && (reinterpret_cast(workload_device) & 63U) != 0) { + std::fprintf(stderr, "Real-compute workspace is not 64-byte aligned: %p\n", workload_device); + return EXIT_FAILURE; + } + + // 泳道区按 96 worker 各 65536 条记录预留，约 384 MiB；关闭泳道时不申请，也不会传递有效 base。 + // 该分配先于 PMU owner 配置，失败时不会留下需要恢复的 selector/MMIO 会话。 + void *trace_device = nullptr; + if (options.trace_enabled && + !CheckAcl( + aclrtMalloc(&trace_device, pa_scheduler::kTraceBytes, ACL_MEM_MALLOC_HUGE_FIRST), + "aclrtMalloc(swimlane trace)" + )) { + return EXIT_FAILURE; + } + if (options.trace_enabled && (reinterpret_cast(trace_device) & 63U) != 0) { + std::fprintf(stderr, "Device swimlane trace is not 64-byte aligned: %p\n", trace_device); + return EXIT_FAILURE; + } + + PmuRegisterMappings pmu_mappings; + pa_scheduler::pmu_owner::PmuOwnerSession pmu_owner; + pa_scheduler::pmu_owner::PmuOwnerControl pmu_owner_evidence{}; + bool pmu_owner_evidence_valid = false; + const void *pmu_registers_device = nullptr; + if (pmu_options.mode != pa_scheduler::ccec_pmu::WindowMode::Off) { + if (!MapPmuRegisters(options.device, &pmu_mappings)) return EXIT_FAILURE; + const std::string dispatcher_path = pa_scheduler::pmu_owner::ArtifactBesideKernel( + options.kernel_path, "libpa_scheduler_pmu_owner_dispatcher.so" + ); + const std::string owner_path = pa_scheduler::pmu_owner::ArtifactBesideKernel( + options.kernel_path, "libpa_scheduler_pmu_owner_aicpu.so" + ); + if (!pmu_owner.Initialize( + options.device, stream, dispatcher_path, owner_path, pmu_mappings.register_bases + ) || + !pmu_owner.Configure()) { + (void)pmu_owner.Finalize(); + (void)UnmapPmuRegisters(options.device, &pmu_mappings); + return EXIT_FAILURE; + } + pmu_owner_evidence = pmu_owner.Control(); + pmu_owner_evidence_valid = true; + pmu_registers_device = reinterpret_cast(pmu_owner.RegisterTableDeviceAddress()); + } + + // host shadow 保留约 1 GiB 总跨度以便按关键 offset 寻址，但每轮传输只选择 + // 共享前缀、控制区和结果区。 + std::unique_ptr state(new pa_scheduler::SchedulerState); + pa_scheduler::TraceHeader trace_header{}; + std::vector spans; + bool execution_ok = true; + bool all_passed = true; + bool postprocess_ok = true; + bool pmu_json_ready = false; + bool pmu_json_semantic_passed = false; + bool pmu_json_workload_output_passed = false; + uint32_t pmu_json_run = 0U; + double pmu_json_host_us = 0.0; + double pmu_json_submit_span_us = 0.0; + PmuValidation pmu_json_validation; + for (uint32_t run = 1; run <= options.runs; ++run) { + pa_scheduler::host::InitializeState(state.get(), options); + pa_scheduler::host::ConfigureTrace(state.get(), options, trace_device); + ConfigurePmu(state.get(), pmu_options, pmu_registers_device); + ConfigureWinnerWorkload(state.get(), workload_options, workload_device); + if (real_compute && + !CheckAcl( + aclrtMemcpy( + workload_device, pa_scheduler::winner_workload::kWorkspaceBytes, + workload_image.data(), pa_scheduler::winner_workload::kWorkspaceBytes, + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D real-compute workspace)" + )) { + execution_ok = false; + break; + } + if (options.trace_enabled) { + // 每轮只需重置约 7 KiB header；各 worker 会从 count=0 覆盖自己的记录区，无需清零整块 384 MiB。 + pa_scheduler::host::InitializeTraceHeader(&trace_header); + if (!CheckAcl( + aclrtMemcpy( + trace_device, sizeof(trace_header), &trace_header, sizeof(trace_header), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D swimlane header)" + )) { + execution_ok = false; + break; + } + } + // 为避免每轮搬运约 1 GiB，只 H2D 被测共享前缀和位于生产总跨度之后的 + // standalone 控制区； + // 每个 worker 的大块私有状态由 device kernel 自行初始化。 + if (!CheckAcl( + aclrtMemcpy( + state_device, pa_scheduler::host::StatePrefixBytes(), state.get(), + pa_scheduler::host::StatePrefixBytes(), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D state prefix)" + ) || + !CheckAcl( + aclrtMemcpy( + &static_cast(state_device)->config, + pa_scheduler::host::ControlBytes(), &state->config, pa_scheduler::host::ControlBytes(), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D standalone controls)" + )) { + execution_ok = false; + break; + } + void *kernel_args[] = {state_device}; + rtArgsEx_t args_info{}; + args_info.args = kernel_args; + args_info.argsSize = sizeof(kernel_args); + rtTaskCfgInfo_t task_config{}; + // launch 维度是 32 个物理 mixed block；ELF metadata 让每个 block 同时产生 1 AIC + 2 AIV，共 96 worker。 + // wall time 在同步完成处截止，包含 launch、完整调度、最终 drain 和 stream 同步，但不包含后续 D2H/JSON。 + const auto wall_begin = std::chrono::steady_clock::now(); + if (!CheckRt( + rtKernelLaunchWithHandleV2( + kernel_handle, 0, pa_scheduler::kAicWorkers, &args_info, nullptr, stream, &task_config + ), + "rtKernelLaunchWithHandleV2" + ) || + !CheckAcl(aclrtSynchronizeStream(stream), "aclrtSynchronizeStream")) { + execution_ok = false; + break; + } + const auto wall_end = std::chrono::steady_clock::now(); + const double host_us = std::chrono::duration(wall_end - wall_begin).count(); + // D2H 同样避开约 1 GiB 的 worker arena：共享前缀用于 flag/vend/frontier 校验，末尾 results 单独回传。 + if (!CheckAcl( + aclrtMemcpy( + state.get(), pa_scheduler::host::StatePrefixBytes(), state_device, + pa_scheduler::host::StatePrefixBytes(), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H state prefix)" + ) || + !CheckAcl( + aclrtMemcpy( + state->results, pa_scheduler::host::ResultBytes(), + &static_cast(state_device)->results[0], + pa_scheduler::host::ResultBytes(), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H worker results)" + )) { + execution_ok = false; + break; + } + if (real_compute && + !CheckAcl( + aclrtMemcpy( + workload_outputs.data(), + static_cast(pa_scheduler::winner_workload::kOutputTiles) * + pa_scheduler::winner_workload::kTileBytes, + static_cast(workload_device) + + pa_scheduler::winner_workload::kSharedInputTiles * + pa_scheduler::winner_workload::kTileBytes, + static_cast(pa_scheduler::winner_workload::kOutputTiles) * + pa_scheduler::winner_workload::kTileBytes, + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H real-compute outputs)" + )) { + execution_ok = false; + break; + } + if (options.trace_enabled && + !CheckAcl( + aclrtMemcpy( + &trace_header, sizeof(trace_header), trace_device, sizeof(trace_header), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H swimlane header)" + )) { + execution_ok = false; + break; + } + // 常规校验只需 header 中的 per-worker count；真实 records 在分析或导出时才按核、按实际 count 懒加载。 + const auto read_trace_records = + [trace_device](uint32_t worker, uint32_t count, pa_scheduler::TraceRecord *records) { + // 每核记录区采用固定容量 stride；只复制 header 声明的实际 count，避免 D2H 未使用的尾部空间。 + const uint64_t offset = sizeof(pa_scheduler::TraceHeader) + + static_cast(worker) * pa_scheduler::kTraceRecordsPerCore * + sizeof(pa_scheduler::TraceRecord); + return CheckAcl( + aclrtMemcpy( + records, static_cast(count) * sizeof(pa_scheduler::TraceRecord), + static_cast(trace_device) + offset, + static_cast(count) * sizeof(pa_scheduler::TraceRecord), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H swimlane records)" + ); + }; + // 先完成共享状态、拓扑、计数和 trace header 的语义校验，再允许 raw JSON 成为性能证据。 + const pa_scheduler::host::Metrics metrics = pa_scheduler::host::Validate( + *state, run, host_us, options.trace_enabled ? &trace_header : nullptr + ); + all_passed &= metrics.passed; + const bool workload_passed = + !real_compute || ValidateRealComputeOutputs( + *state, workload_options, workload_outputs, run + ); + all_passed &= workload_passed; + PmuValidation pmu_validation; + const bool pmu_passed = ValidatePmu( + *state, run, pmu_options, workload_options, + pmu_options.mode == pa_scheduler::ccec_pmu::WindowMode::Off ? nullptr : &pmu_owner.Control(), + &pmu_validation + ); + all_passed &= pmu_passed; + spans.push_back(metrics.submit_span_us); + if (!pmu_options.json_path.empty()) { + if (!metrics.passed || !workload_passed || !pmu_passed || !pmu_owner_evidence_valid) { + std::fprintf(stderr, "PMU JSON rejected because semantic, PMU, or owner validation failed.\n"); + postprocess_ok = false; + break; + } + pmu_json_ready = true; + pmu_json_semantic_passed = metrics.passed && workload_passed; + pmu_json_workload_output_passed = workload_passed; + pmu_json_run = run; + pmu_json_host_us = host_us; + pmu_json_submit_span_us = metrics.submit_span_us; + pmu_json_validation = pmu_validation; + } + if (options.analyze_swimlane && + !pa_scheduler::host::AnalyzeSwimlaneRecords(trace_header, *state, read_trace_records)) { + // 后处理错误使用 break 汇入统一 cleanup；与初始化/launch 失败的进程级立即返回语义区分开。 + postprocess_ok = false; + break; + } + if (!options.swimlane_json.empty()) { + // 只有语义校验通过才把 raw JSON 经“临时文件写完后 rename”发布， + // 避免把截断或错误调度结果误当成可用性能证据。 + if (!metrics.passed || !workload_passed) { + std::fprintf(stderr, "Skipping swimlane export because semantic validation failed.\n"); + postprocess_ok = false; + break; + } + if (!pa_scheduler::host::ExportSwimlaneRecords( + trace_header, options.swimlane_json, workload_options.mode, + real_compute + ? workload_options.repeats + : pa_scheduler::WorkloadCounts{ + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + }, + real_compute + ? pa_scheduler::host::RealComputePatternName(workload_options.pattern) + : "none", + options.trace_atomics, + read_trace_records + )) { + postprocess_ok = false; + break; + } + } + } + + const double median_submit_span_us = spans.empty() ? 0.0 : pa_scheduler::host::Median(spans); + std::printf( + "[SUMMARY] runs=%u completed_runs=%zu median_submit_span_us=%.3f " + "execution_status=%s semantic_status=%s postprocess_status=%s\n", + options.runs, spans.size(), median_submit_span_us, execution_ok ? "PASS" : "FAIL", + all_passed ? "PASS" : "FAIL", postprocess_ok ? "PASS" : "FAIL" + ); + + // 后处理失败也统一走设备资源释放、ELF 卸载和 ACL 收尾，避免文件系统错误遗留运行时上下文。 + bool cleanup_ok = true; + bool pmu_owner_restore_ok = true; + // 先释放依赖当前 device/context 的大块内存，再卸载 ELF、销毁 stream，最后 reset device 与 finalize ACL。 + if (trace_device != nullptr) { + cleanup_ok &= CheckAcl(aclrtFree(trace_device), "aclrtFree(swimlane trace)"); + } + if (pmu_registers_device != nullptr) { + // owner 必须在 MMIO 映射、device context 和 ACL runtime 仍有效时恢复。 + pmu_owner_restore_ok = pmu_owner.Finalize(); + cleanup_ok &= pmu_owner_restore_ok; + cleanup_ok &= UnmapPmuRegisters(options.device, &pmu_mappings); + } + if (workload_device != nullptr) { + cleanup_ok &= CheckAcl( + aclrtFree(workload_allocation.Release()), "aclrtFree(real-compute workspace)" + ); + } + cleanup_ok &= CheckAcl(aclrtFree(state_device), "aclrtFree(state)"); + const rtError_t unload_error = + registered_all ? rtDevBinaryUnRegister(kernel_handle) : rtBinaryUnLoad(kernel_handle); + cleanup_ok &= CheckRt(unload_error, "unload mixed AICore ELF"); + cleanup_ok &= CheckAcl(aclrtDestroyStream(stream), "aclrtDestroyStream"); + cleanup_ok &= CheckAcl(aclrtResetDevice(options.device), "aclrtResetDevice"); + cleanup_ok &= CheckAcl(aclFinalize(), "aclFinalize"); + if (!pmu_options.json_path.empty()) { + if (!pmu_json_ready || !all_passed || !postprocess_ok || !cleanup_ok || !pmu_owner_restore_ok) { + std::fprintf(stderr, "PMU JSON was not published because the capture or restore transaction failed.\n"); + postprocess_ok = false; + } else if (!ExportPmuJson( + *state, options, pmu_options, workload_options, pmu_json_run, + pmu_json_host_us, pmu_json_submit_span_us, pmu_json_validation, + pmu_json_semantic_passed, pmu_json_workload_output_passed, + pmu_owner_evidence, + pmu_owner_restore_ok, pmu_options.json_path + )) { + postprocess_ok = false; + } + } + // 运行语义、后处理和资源清理三者全部成功，进程才返回成功，脚本据此决定是否继续生成 merged 泳道。 + return execution_ok && all_passed && postprocess_ok && cleanup_ok ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/kernel.cpp b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/kernel.cpp new file mode 100644 index 0000000000..74c5aa3f0f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/kernel.cpp @@ -0,0 +1,401 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include +#include +#include +#include + +#include "pmu_probe.h" +#include "../common/winner_workload.h" + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" + +#define PA_CCEC_OPS_DEFINE_REAL_WORKLOAD 1 +#include "ccec_ops.h" +#undef PA_CCEC_OPS_DEFINE_REAL_WORKLOAD + +using pa_scheduler_ccec::CcecOps; + +namespace { +#if PA_BUILD_SUBMIT_PMU +struct PmuSnapshot { + uint64_t total_cycles = 0; + uint32_t vector_busy = 0; + uint32_t cube_busy = 0; + uint32_t scalar_busy = 0; + uint32_t mte1_busy = 0; + uint32_t mte2_busy = 0; + uint32_t mte3_busy = 0; + uint32_t icache_requests = 0; + uint32_t icache_misses = 0; + uint32_t fix_busy = 0; + uint32_t status = 0; +}; + +struct IcacheShadowSnapshot { + uint32_t requests = 0; + uint32_t misses = 0; +}; + +} // namespace + +namespace pa_scheduler_ccec { + +struct SubmitPmuContext { + uint64_t reg_base = 0; + uint64_t shadow_requests = 0; + uint64_t shadow_misses = 0; + uint64_t phase_requests = 0; + uint64_t phase_misses = 0; + // begin 的 shadow read-clear 完成后取起点，end 的 shadow read-clear 之前 + // 取终点；累计值因此不包含两次 PMU 寄存器读取本身。 + uint64_t phase_elapsed_ticks = 0; + uint64_t phase_begin_tick = 0; + uint32_t selector_status = 0; + uint32_t phase_status = pa_scheduler::ccec_pmu::kPhaseStatusRequested; + uint32_t phase_calls = 0; + uint32_t begin_reads = 0; + uint32_t end_reads = 0; + bool started = false; + bool phase_armed = false; + bool boundary_error = false; +}; + +} // namespace pa_scheduler_ccec + +namespace { + +using pa_scheduler_ccec::SubmitPmuContext; + +template +__aicore__ inline uint32_t ReadPmuRegister(uint64_t reg_base) { + // 传给 ld_dev 的是重基址后的 __gm__ 指针；相对 offset 均落在编译器允许的 [-2048, 2047]。 + int32_t *block = reinterpret_cast(reg_base + BlockOffset); + return static_cast(ld_dev(block, static_cast(RegisterOffset - BlockOffset))); +} + +__aicore__ inline PmuSnapshot ReadObservedCounters(uint64_t reg_base) { + PmuSnapshot sample; + sample.vector_busy = ReadPmuRegister(reg_base); + sample.cube_busy = ReadPmuRegister(reg_base); + sample.scalar_busy = ReadPmuRegister(reg_base); + sample.mte1_busy = ReadPmuRegister(reg_base); + sample.mte2_busy = ReadPmuRegister(reg_base); + // submit-pmu 将 CNT5 留给 shadow I-cache miss。这里不能提前读取，否则 + // read-to-clear 会让随后的 shadow tail 漏计；MTE3 busy 在该诊断构建不可用。 + sample.mte3_busy = 0; + sample.icache_requests = ReadPmuRegister(reg_base); + sample.icache_misses = ReadPmuRegister(reg_base); + const uint64_t low = ReadPmuRegister(reg_base); + const uint64_t high = ReadPmuRegister(reg_base); + sample.total_cycles = low | (high << 32); + return sample; +} + +__aicore__ inline IcacheShadowSnapshot ReadShadowCounters(uint64_t reg_base) { + IcacheShadowSnapshot sample; + sample.requests = ReadPmuRegister(reg_base); + sample.misses = ReadPmuRegister(reg_base); + return sample; +} + +struct PmuRegisterContext { + uint64_t reg_base = 0; + uint32_t status = 0; + bool shadow_selectors = false; +}; + +__aicore__ inline PmuRegisterContext ResolvePmuRegisters(__gm__ pa_scheduler::SchedulerState *state) { + using namespace pa_scheduler::ccec_pmu; + PmuRegisterContext context; + const uint32_t physical_core_id = static_cast(get_coreid()) & kStatusCoreIdMask; + context.status = kStatusRequested | (physical_core_id << kStatusCoreIdShift); + const uint64_t table_address = + static_cast(state->config.reserved[kConfigRegTableLow]) | + (static_cast(state->config.reserved[kConfigRegTableHigh]) << 32); + if (state->config.reserved[kConfigMagic] != kConfigMagicValue || table_address == 0 || + physical_core_id >= kPhysicalSubcoreCount) { + return context; + } + context.status |= kStatusCoreIdValid; + __gm__ const uint64_t *register_bases = reinterpret_cast<__gm__ const uint64_t *>(table_address); + context.reg_base = register_bases[physical_core_id]; + if (context.reg_base == 0) return context; + context.status |= kStatusRegMapped; + + // selector 与同 phase 目录内的 owner 逐项核对。CNT5/CNT8 分别重复 + // CNT7/CNT6；CNT9 保持正式 PIPE_UTIL 的 unused(0) 口径。 + if (ReadPmuRegister(context.reg_base) == kVectorBusyEvent) + context.status |= kStatusCnt0Selector; + if (ReadPmuRegister(context.reg_base) == kCubeBusyEvent) + context.status |= kStatusCnt1Selector; + if (ReadPmuRegister(context.reg_base) == kScalarBusyEvent) + context.status |= kStatusCnt2Selector; + if (ReadPmuRegister(context.reg_base) == kMte1BusyEvent) + context.status |= kStatusCnt3Selector; + if (ReadPmuRegister(context.reg_base) == kMte2BusyEvent) + context.status |= kStatusCnt4Selector; + const bool cnt5_ok = + ReadPmuRegister(context.reg_base) == kIcacheMissEvent; + if (cnt5_ok) + context.status |= kStatusCnt5Selector; + if (ReadPmuRegister(context.reg_base) == kIcacheRequestEvent) + context.status |= kStatusCnt6Selector; + if (ReadPmuRegister(context.reg_base) == kIcacheMissEvent) + context.status |= kStatusCnt7Selector; + const bool cnt8_ok = + ReadPmuRegister(context.reg_base) == kIcacheRequestEvent; + const bool cnt9_unused = + ReadPmuRegister(context.reg_base) == 0U; + if (cnt8_ok) + context.status |= kStatusCnt8Selector; + context.shadow_selectors = cnt5_ok && cnt8_ok && cnt9_unused; + return context; +} + +__aicore__ inline void PublishPmuSnapshot( + __gm__ pa_scheduler::WorkerResult &result, const PmuSnapshot &sample +) { + // 每核独占 sidecar 通过 bypass store 一次性发布；这些写发生在 PMU stop/read 之后， + // 不进入被导出的 Submit 窗口。 + CcecOps::Publish(&result.pmu_total_cycles, sample.total_cycles); + CcecOps::Publish(&result.pmu_scalar_busy, sample.scalar_busy); + CcecOps::Publish(&result.pmu_icache_requests, sample.icache_requests); + CcecOps::Publish(&result.pmu_icache_misses, sample.icache_misses); + CcecOps::Publish(&result.pmu_status, sample.status); + CcecOps::Publish(&result.pmu_vector_busy, sample.vector_busy); + CcecOps::Publish(&result.pmu_cube_busy, sample.cube_busy); + CcecOps::Publish(&result.pmu_mte1_busy, sample.mte1_busy); + CcecOps::Publish(&result.pmu_mte2_busy, sample.mte2_busy); + CcecOps::Publish(&result.pmu_mte3_busy, sample.mte3_busy); + // CNT8 已改作 shadow request，submit-pmu 不再发布 fix-busy。 + CcecOps::Publish(&result.pmu_fix_busy, static_cast(0)); + CcecOps::Publish(&result.pmu_warm_total_cycles, static_cast(0)); + CcecOps::Publish(&result.pmu_warm_window_ticks, static_cast(0)); +} + +__aicore__ inline bool FitsUint32(uint64_t value) { + return value <= 0xffffffffULL; +} + +__aicore__ inline void PublishSubmitPmuContext( + __gm__ pa_scheduler::WorkerResult &result, const SubmitPmuContext &context +) { + CcecOps::Publish(&result.pmu_build_variant, pa_scheduler::kBuildVariantSubmitPmu); + CcecOps::Publish( + &result.pmu_phase_id, + static_cast(pa_scheduler::kCompiledSubmitPmuPhase) + ); + CcecOps::Publish(&result.pmu_phase_calls, context.phase_calls); + CcecOps::Publish(&result.pmu_phase_status, context.phase_status); + CcecOps::Publish(&result.pmu_phase_begin_reads, context.begin_reads); + CcecOps::Publish(&result.pmu_phase_end_reads, context.end_reads); + CcecOps::Publish(&result.pmu_phase_elapsed_ticks, context.phase_elapsed_ticks); + CcecOps::Publish(&result.pmu_phase_icache_requests, static_cast(context.phase_requests)); + CcecOps::Publish(&result.pmu_phase_icache_misses, static_cast(context.phase_misses)); + CcecOps::Publish(&result.pmu_shadow_icache_requests, static_cast(context.shadow_requests)); + CcecOps::Publish(&result.pmu_shadow_icache_misses, static_cast(context.shadow_misses)); +} + +} // namespace + +namespace pa_scheduler_ccec { + +__aicore__ inline CcecOps::PmuContext CcecOps::PmuWindowStart( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +) { + using namespace pa_scheduler::ccec_pmu; + (void)worker_id; + SubmitPmuContext context; + const WindowMode mode = static_cast(state->config.reserved[kConfigMode]); + if (mode != WindowMode::SubmitAll) return context; + // Main AICPU owner 已在 launch 前配置并开启计数；先 stop + snapshot/read-clear， + // 再解析 selector，避免这些 ld_dev 污染完整 Submit 窗口。 + bisheng::cce::metrics_prof_stop(); + const PmuRegisterContext registers = ResolvePmuRegisters(state); + context.reg_base = registers.reg_base; + context.selector_status = registers.status; + if (registers.shadow_selectors) { + context.phase_status |= kPhaseStatusShadowSelectors; + } + if (context.reg_base == 0) return context; + (void)ReadObservedCounters(context.reg_base); + (void)ReadShadowCounters(context.reg_base); + bisheng::cce::metrics_prof_start(); + context.started = true; + context.phase_status |= kPhaseStatusWindowStarted; + return context; +} + +__aicore__ inline void CcecOps::PmuPhaseBegin(PmuContext &context) { + if (!context.started || context.reg_base == 0 || context.phase_armed) { + context.boundary_error = true; + return; + } + // counter 在运行中读取即清零；begin 之前的片段只进入 shadow whole。 + const IcacheShadowSnapshot sample = ReadShadowCounters(context.reg_base); + context.shadow_requests += sample.requests; + context.shadow_misses += sample.misses; + ++context.begin_reads; + context.phase_armed = true; + // get_sys_cnt() 是本机已校准为 1 ns/tick 的 A5 系统计数器。该读取位于 + // begin 的两条 ld_dev 之后，因此不会把 read-clear 成本算进阶段时间。 + context.phase_begin_tick = CcecOps::Now(); +} + +__aicore__ inline void CcecOps::PmuPhaseEnd(PmuContext &context) { + // 先取终点再读取 shadow counter，使 end 的两条 ld_dev 同样位于阶段之外。 + const uint64_t phase_end_tick = CcecOps::Now(); + if (!context.started || context.reg_base == 0 || !context.phase_armed) { + context.boundary_error = true; + return; + } + if (phase_end_tick < context.phase_begin_tick) { + context.boundary_error = true; + } else { + context.phase_elapsed_ticks += phase_end_tick - context.phase_begin_tick; + } + // end 读出的片段同时属于完整 shadow 重建与被选中的局部阶段。 + const IcacheShadowSnapshot sample = ReadShadowCounters(context.reg_base); + context.shadow_requests += sample.requests; + context.shadow_misses += sample.misses; + context.phase_requests += sample.requests; + context.phase_misses += sample.misses; + ++context.end_reads; + ++context.phase_calls; + context.phase_armed = false; + context.phase_begin_tick = 0; +} + +__aicore__ inline void CcecOps::PmuWindowStop( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id, PmuContext &context +) { + using namespace pa_scheduler::ccec_pmu; + __gm__ pa_scheduler::WorkerResult &result = state->results[worker_id]; + PmuSnapshot sample; + sample.status = context.selector_status; + if (context.started && context.reg_base != 0) { + // gate 只在整个 Submit 前后各操作一次。停止后先读从未中途清零的 + // primary counter（不含 shadow CNT5）之后，再读取 CNT8/CNT5 tail + // 完成软件重建。 + bisheng::cce::metrics_prof_stop(); + sample = ReadObservedCounters(context.reg_base); + const IcacheShadowSnapshot tail = ReadShadowCounters(context.reg_base); + context.shadow_requests += tail.requests; + context.shadow_misses += tail.misses; + sample.status = context.selector_status | kStatusWindowStarted | kStatusWindowStopped; + context.phase_status |= kPhaseStatusWindowStopped; + if (sample.total_cycles != 0) sample.status |= kStatusTotalNonzero; + } + + if (context.shadow_requests == sample.icache_requests) + context.phase_status |= kPhaseStatusShadowRequestsMatch; + if (context.shadow_misses == sample.icache_misses) + context.phase_status |= kPhaseStatusShadowMissesMatch; + if (!context.boundary_error && !context.phase_armed && + context.begin_reads == context.end_reads && context.end_reads == context.phase_calls) + context.phase_status |= kPhaseStatusBoundariesBalanced; + // 两个 shadow counter 是顺序 ld_dev，并非同一时刻的原子快照；局部 + // phase 的 miss/request 边界会错开数条指令，故不能硬性要求局部 + // miss<=request。A5 上运行中 read-to-clear 还会与同周期事件递增竞争， + // shadow 允许小于未中途读取的 primary，但绝不能反向超过它。primary- + // shadow 是该次采集可直接给出的局部分段误差包络，而不是要静默吞掉的差值。 + if (context.phase_requests <= context.shadow_requests && + context.phase_misses <= context.shadow_misses && + context.shadow_misses <= context.shadow_requests && + context.shadow_requests <= sample.icache_requests && + context.shadow_misses <= sample.icache_misses) + context.phase_status |= kPhaseStatusValuesOrdered; + if (FitsUint32(context.shadow_requests) && FitsUint32(context.shadow_misses) && + FitsUint32(context.phase_requests) && FitsUint32(context.phase_misses)) + context.phase_status |= kPhaseStatusUint32Fit; + + const bool none_shape = + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None && + context.phase_calls == 0 && context.begin_reads == 0 && context.end_reads == 0 && + context.phase_requests == 0 && context.phase_misses == 0 && + context.phase_elapsed_ticks == 0; + const bool running_shape = + pa_scheduler::kCompiledSubmitPmuPhase != pa_scheduler::SubmitPmuPhase::None && + context.phase_calls == state->config.batches * pa_scheduler::kTasksPerBatch && + context.begin_reads == context.phase_calls && + context.end_reads == context.phase_calls; + if (none_shape || running_shape) + context.phase_status |= kPhaseStatusPhaseShape; + const bool phase_time_valid = + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? context.phase_elapsed_ticks == 0 + : context.phase_calls != 0 && context.phase_elapsed_ticks != 0; + if (phase_time_valid) + context.phase_status |= kPhaseStatusTimeValid; + + PublishPmuSnapshot(result, sample); + PublishSubmitPmuContext(result, context); +} +#endif // PA_BUILD_SUBMIT_PMU + +} // namespace + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) && defined(PA_BUILD_AIC) +// split artifact: the runtime entry/state-owner TU calls this orchestration +// function once per kernel launch. It is not a launchable kernel and the +// version script localizes it in the final mixed ELF. +extern "C" __attribute__((noinline, used)) __aicore__ void +pa_scheduler_lazy_sample_callback_orchestration_aic( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +) { + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aic); +} +#elif defined(PA_LAZY_SAMPLE_SPLIT_FINISH) && defined(PA_BUILD_AIV) +extern "C" __attribute__((noinline, used)) __aicore__ void +pa_scheduler_lazy_sample_callback_orchestration_aiv( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +) { + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aiv); +} +#elif defined(PA_BUILD_AIC) +// 同一源码分别按 cube/vec 架构编译；metadata 声明每个物理 block 静态组合 1 个 AIC 与 2 个 AIV。 +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aic, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aic(__gm__ pa_scheduler::SchedulerState *state) { + // 32 个物理 block 的 AIC 直接使用 block_idx，形成连续 worker 0..31。 + const uint32_t worker_id = static_cast(get_block_idx()); + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aic); +} +#elif defined(PA_BUILD_AIV) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aiv, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aiv(__gm__ pa_scheduler::SchedulerState *state) { + // 每个 block 的两个 vector sub-block 展平为 vector_id=2*b+subblock，偏移 32 后形成 worker 32..95。 + const uint32_t vector_id = static_cast(get_block_idx() * get_subblockdim() + get_subblockid()); + const uint32_t worker_id = pa_scheduler::kAicWorkers + vector_id; + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aiv); +} +#else +#error "Compile with PA_BUILD_AIC or PA_BUILD_AIV" +#endif diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pa_scheduler_device_exports.map b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pa_scheduler_device_exports.map new file mode 100644 index 0000000000..b4068c90f1 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pa_scheduler_device_exports.map @@ -0,0 +1,7 @@ +{ + global: + pa_scheduler_0_mix_aic; + pa_scheduler_0_mix_aiv; + local: + *; +}; diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_owner_control.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_owner_control.h new file mode 100644 index 0000000000..eb17c4c1b0 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_owner_control.h @@ -0,0 +1,225 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_CONTROL_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_CONTROL_H_ + +#include +#include + +namespace pa_scheduler::pmu_owner { + +// 这份头文件同时供 x86 host 与 AArch64 AICPU helper 使用。所有跨端字段都采用 +// 固定宽度整数；禁止在 ABI 中放 host 指针、bool、STL 容器或编译器相关位域。 +constexpr uint32_t kPmuOwnerControlMagic = 0x504d554fU; // "PMUO" +constexpr uint32_t kPmuOwnerControlVersion = 1U; + +// DAV_3510 一共有 2 die；每个 die 依次放 18 个 AIC 和 36 个 AIV 物理槽。 +// 当前 A5 stream 实际开放 32 个 AIC 与 64 个 AIV，其余 12 个槽的 MMIO +// 读回不会匹配配置值，因此 owner 必须扫描 108 槽，最终取得 96 个可用槽。 +constexpr uint32_t kPhysicalSubcoreCount = 108U; +constexpr uint32_t kExpectedSubcoreCount = 96U; +constexpr uint32_t kExpectedAicCount = 32U; +constexpr uint32_t kExpectedAivCount = 64U; +constexpr uint32_t kAicPerDie = 18U; +constexpr uint32_t kSubcoresPerDie = 54U; +constexpr uint32_t kConfiguredBitmapWords = 4U; +constexpr uint32_t kDiagnosticIndexUnset = 0xffffffffU; + +static_assert(kExpectedAicCount + kExpectedAivCount == kExpectedSubcoreCount, "active topology count mismatch"); +static_assert(kAicPerDie * 2U == 36U, "physical AIC topology changed"); +static_assert(kSubcoresPerDie * 2U == kPhysicalSubcoreCount, "physical subcore topology changed"); + +// A5 PIPE_UTILIZATION 的正式 counter 槽位布局。submit-pmu 用 CNT8/CNT5 +// 重复配置 I-cache request/miss，作为允许中途 read-to-clear 的 shadow； +// CNT6/7 始终不在阶段边界读取，保留为完整 Submit 的权威对照。 +// +// 不能把 miss 放进 CNT9：A5 b1 实测表明 CNT9 selector 虽能回读 0x35， +// 但计数恒为 0；正式 PIPE_UTIL 表也把 CNT9 标成 unused。0x35 在独立 +// I-cache 微基准的低位 counter 已验证可计数，因此诊断构建让 CNT5 承担 +// shadow miss，并明确放弃该构建中的 MTE3 busy。 +constexpr uint32_t kPmuCounterCount = 10U; +constexpr uint32_t kConfiguredSelectors[kPmuCounterCount] = { + 0x501U, // CNT0: vector busy + 0x301U, // CNT1: cube busy + 0x001U, // CNT2: scalar busy + 0x701U, // CNT3: MTE1 busy + 0x202U, // CNT4: MTE2 busy +#if PA_BUILD_SUBMIT_PMU + 0x035U, // CNT5: shadow I-cache miss + 0x034U, // CNT6: I-cache request（完整 Submit） + 0x035U, // CNT7: I-cache miss（完整 Submit） + 0x034U, // CNT8: shadow I-cache request + 0x000U, // CNT9: A5 PIPE_UTIL 正式未使用 +#else + 0x203U, // CNT5: MTE3 busy + 0x034U, // CNT6: I-cache request + 0x035U, // CNT7: I-cache miss + 0x714U, // CNT8: fix-pipe busy + 0x000U, // CNT9: 未使用 +#endif +}; + +constexpr int32_t kStatusPending = 0x7fffffff; + +// AICPU entry 始终向 runtime 返回 0；协议结果只通过 control.status 回传， +// 从而避免一次可诊断的配置失败被 runtime 升格成整条 stream 异常。 +enum class PmuOwnerStatus : int32_t { + Success = 0, + InvalidArguments = -1, + InvalidControl = -2, + UnexpectedTopology = -3, + AlreadyConfigured = -4, + ConfigureCountMismatch = -5, + ConfigureRollbackFailed = -6, + ConfigureSlotRestoreFailed = -7, + RestoreFailed = -8, +}; + +// 首个异常寄存器使用稳定的枚举编号，host 不需要解析 AICPU 日志即可定位 +// 是基址、selector、计数范围还是 enable 控制读回不一致。 +enum class PmuOwnerField : uint32_t { + None = 0, + Arguments, + ControlMagic, + ControlVersion, + ControlSize, + State, + RegisterBase, + Ctrl0, + Ctrl1, + Selector0, + Selector1, + Selector2, + Selector3, + Selector4, + Selector5, + Selector6, + Selector7, + Selector8, + Selector9, + StartCycleLow, + StartCycleHigh, + StopCycleLow, + StopCycleHigh, + BitmapCount, + TotalCount, + AicCount, + AivCount, +}; + +// 单个物理子核被 owner 改动的完整可恢复状态恰好占一条 cache line。 +// PMU counter 是 read-to-clear，旧 counter 值无法恢复；owner 会话必须独占。 +struct alignas(64) PmuSavedRegisters { + uint32_t ctrl0; + uint32_t ctrl1; + uint32_t selectors[kPmuCounterCount]; + uint32_t start_cycle_low; + uint32_t start_cycle_high; + uint32_t stop_cycle_low; + uint32_t stop_cycle_high; +}; + +// Host 与 AICPU 共享的 owner 状态。前 128B 是命令结果和诊断，随后内嵌 +// 108 个 MMIO 基址、4-word 所有权 bitmap，以及每槽 64B 的 Configure 快照。 +// bitmap 的严格语义是“原值已保存、且 owner 可能已经改写 MMIO、但尚未 +// 完整恢复”的槽；它在 Configure 写第一项 MMIO 前置位，仅在恢复读回完整 +// 一致后清位。Restore 期间不得清零或重建 saved[]，只能按 bitmap 逆序消费。 +struct alignas(64) PmuOwnerControl { + uint32_t magic; + uint32_t version; + uint32_t struct_bytes; + volatile int32_t status; + + uint32_t configured; + uint32_t expected_total; + uint32_t expected_aic; + uint32_t expected_aiv; + + // active_* 与 bitmap 表示仍由本 owner 持有、尚未恢复的物理槽。 + uint32_t active_total; + uint32_t active_aic; + uint32_t active_aiv; + // discovered_* 保留本次 Configure 扫描结果；即使计数不匹配后回滚， + // host 仍能看到回滚前究竟探测到了多少 AIC/AIV。 + uint32_t discovered_total; + uint32_t discovered_aic; + uint32_t discovered_aiv; + uint32_t skipped_total; + + uint32_t first_failed_index; + uint32_t first_failed_field; + uint32_t first_failed_observed; + uint32_t first_failed_expected; + + uint32_t restore_failures; + uint32_t first_restore_failed_index; + uint32_t first_restore_failed_field; + uint32_t first_restore_failed_observed; + uint32_t first_restore_failed_expected; + uint32_t reserved_header[8]; + + uint64_t register_bases[kPhysicalSubcoreCount]; + // 字段名保留 configured_bitmap 以稳定 host/device ABI；失败路径中它还会 + // 临时包含“配置未通过但恢复仍待重试”的 owned 槽。 + uint32_t configured_bitmap[kConfiguredBitmapWords]; + // 让 saved[] 从新的 64B cache line 开始；该 padding 不承载协议含义。 + uint32_t reserved_bitmap[4]; + PmuSavedRegisters saved[kPhysicalSubcoreCount]; +}; + +static_assert(sizeof(PmuSavedRegisters) == 64U, "one saved PMU slot must occupy one cache line"); +static_assert(alignof(PmuSavedRegisters) == 64U, "saved PMU slot alignment changed"); +static_assert(offsetof(PmuOwnerControl, status) == 12U, "PMU owner status offset changed"); +static_assert(offsetof(PmuOwnerControl, register_bases) == 128U, "PMU owner header must occupy two cache lines"); +static_assert(offsetof(PmuOwnerControl, configured_bitmap) == 992U, "PMU owner bitmap offset changed"); +static_assert(offsetof(PmuOwnerControl, saved) == 1024U, "PMU owner saved area must be cache-line aligned"); +static_assert(sizeof(PmuOwnerControl) == 7936U, "PMU owner control ABI changed"); +static_assert(sizeof(PmuOwnerControl) % 64U == 0U, "PMU owner control must use complete cache lines"); +static_assert(alignof(PmuOwnerControl) == 64U, "PMU owner control alignment changed"); + +inline bool IsAicPhysicalSlot(uint32_t index) +{ + return index < kPhysicalSubcoreCount && (index % kSubcoresPerDie) < kAicPerDie; +} + +inline bool IsConfigured(const PmuOwnerControl &control, uint32_t index) +{ + return index < kPhysicalSubcoreCount && + (control.configured_bitmap[index / 32U] & (1U << (index % 32U))) != 0U; +} + +inline void SetConfigured(PmuOwnerControl *control, uint32_t index) +{ + if (index < kPhysicalSubcoreCount) { + control->configured_bitmap[index / 32U] |= 1U << (index % 32U); + } +} + +inline void ClearConfigured(PmuOwnerControl *control, uint32_t index) +{ + if (index < kPhysicalSubcoreCount) { + control->configured_bitmap[index / 32U] &= ~(1U << (index % 32U)); + } +} + +inline uint32_t CountConfigured(const PmuOwnerControl &control) +{ + uint32_t count = 0U; + for (uint32_t index = 0U; index < kPhysicalSubcoreCount; ++index) { + count += IsConfigured(control, index) ? 1U : 0U; + } + return count; +} + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_CONTROL_H_ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_owner_host.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_owner_host.h new file mode 100644 index 0000000000..68dfac042e --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_owner_host.h @@ -0,0 +1,345 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_HOST_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_HOST_H_ + +#include "pmu_owner_control.h" +#include "pmu_owner_main_abi.h" +#include "pmu_owner_main_loader.h" + +#include "acl/acl.h" + +#include +#include +#include +#include +#include +#include + +namespace pa_scheduler::pmu_owner { + +inline bool OwnerCheckAcl(aclError error, const char *label) +{ + if (error == ACL_SUCCESS) return true; + std::fprintf(stderr, "ACL error %d: %s\n", static_cast(error), label); + return false; +} + +inline std::string ArtifactBesideKernel(const std::string &kernel_path, const char *name) +{ + const size_t slash = kernel_path.find_last_of('/'); + return slash == std::string::npos ? std::string(name) : kernel_path.substr(0, slash + 1U) + name; +} + +inline const char *OwnerFieldName(PmuOwnerField field) +{ + switch (field) { + case PmuOwnerField::None: return "none"; + case PmuOwnerField::Arguments: return "arguments"; + case PmuOwnerField::ControlMagic: return "control-magic"; + case PmuOwnerField::ControlVersion: return "control-version"; + case PmuOwnerField::ControlSize: return "control-size"; + case PmuOwnerField::State: return "state"; + case PmuOwnerField::RegisterBase: return "register-base"; + case PmuOwnerField::Ctrl0: return "ctrl0"; + case PmuOwnerField::Ctrl1: return "ctrl1"; + case PmuOwnerField::Selector0: return "selector0"; + case PmuOwnerField::Selector1: return "selector1"; + case PmuOwnerField::Selector2: return "selector2"; + case PmuOwnerField::Selector3: return "selector3"; + case PmuOwnerField::Selector4: return "selector4"; + case PmuOwnerField::Selector5: return "selector5"; + case PmuOwnerField::Selector6: return "selector6"; + case PmuOwnerField::Selector7: return "selector7"; + case PmuOwnerField::Selector8: return "selector8"; + case PmuOwnerField::Selector9: return "selector9"; + case PmuOwnerField::StartCycleLow: return "start-cycle-low"; + case PmuOwnerField::StartCycleHigh: return "start-cycle-high"; + case PmuOwnerField::StopCycleLow: return "stop-cycle-low"; + case PmuOwnerField::StopCycleHigh: return "stop-cycle-high"; + case PmuOwnerField::BitmapCount: return "bitmap-count"; + case PmuOwnerField::TotalCount: return "total-count"; + case PmuOwnerField::AicCount: return "aic-count"; + case PmuOwnerField::AivCount: return "aiv-count"; + } + return "unknown"; +} + +struct ActiveSubcoreLimits { + uint32_t aic = 0U; + uint32_t aiv = 0U; + uint32_t total = 0U; +}; + +inline bool QueryActiveSubcoreLimits(aclrtStream scheduling_stream, ActiveSubcoreLimits *limits) +{ + if (scheduling_stream == nullptr || limits == nullptr) { + std::fprintf(stderr, "Cannot query active PMU subcores with a null stream/result.\n"); + return false; + } + uint32_t aic = 0U; + uint32_t aiv = 0U; + const aclError aic_error = + aclrtGetStreamResLimit(scheduling_stream, ACL_RT_DEV_RES_CUBE_CORE, &aic); + const aclError aiv_error = + aclrtGetStreamResLimit(scheduling_stream, ACL_RT_DEV_RES_VECTOR_CORE, &aiv); + const uint64_t total = static_cast(aic) + aiv; + if (aic_error != ACL_SUCCESS || aiv_error != ACL_SUCCESS || + aic != kExpectedAicCount || aiv != kExpectedAivCount || total != kExpectedSubcoreCount) { + std::fprintf( + stderr, + "Unexpected stream PMU topology: aic_error=%d aiv_error=%d " + "aic=%u/%u aiv=%u/%u total=%llu/%u\n", + static_cast(aic_error), static_cast(aiv_error), aic, kExpectedAicCount, + aiv, kExpectedAivCount, static_cast(total), kExpectedSubcoreCount + ); + return false; + } + limits->aic = aic; + limits->aiv = aiv; + limits->total = static_cast(total); + std::printf( + "[PMU_OWNER] stream_active aic=%u aiv=%u total=%u physical_slots=%u\n", + limits->aic, limits->aiv, limits->total, kPhysicalSubcoreCount + ); + return true; +} + +// mixed launch 的一个物理 block 必须同时拥有 1 个 AIC 和相邻的 2 个 AIV。 +// 只检查 32/64 总数仍可能放过孤立 AIV；这里直接按两 die 的真实编号布局验闭包。 +inline bool ValidateConfiguredTripletTopology(const PmuOwnerControl &control) +{ + uint32_t complete_triplets = 0U; + uint32_t broken_triplets = 0U; + const uint32_t dies = kPhysicalSubcoreCount / kSubcoresPerDie; + for (uint32_t die = 0U; die < dies; ++die) { + const uint32_t die_base = die * kSubcoresPerDie; + for (uint32_t local = 0U; local < kAicPerDie; ++local) { + const bool aic = IsConfigured(control, die_base + local); + const bool aiv0 = IsConfigured(control, die_base + kAicPerDie + local * 2U); + const bool aiv1 = IsConfigured(control, die_base + kAicPerDie + local * 2U + 1U); + if (aic == aiv0 && aic == aiv1) { + complete_triplets += aic ? 1U : 0U; + } else { + ++broken_triplets; + } + } + } + const bool passed = complete_triplets == kExpectedAicCount && broken_triplets == 0U; + std::printf( + "[ASSERT] %-48s %s (complete=%u broken=%u)\n", + "PMU owner bitmap is complete 1-AIC + 2-AIV triplets", + passed ? "PASS" : "FAIL", complete_triplets, broken_triplets + ); + return passed; +} + +// owner 命令使用独立 stream，但通过 mode=0 JSON 在主 aicpu_scheduler 中执行。 +// Configure 同步完成后才允许启动 AICore；AICore 正常或异常退出后，Restore +// 都不会依赖业务 stream。MMIO 映射必须保持到 Finalize 完成之后。 +class PmuOwnerSession { +public: + PmuOwnerSession() = default; + PmuOwnerSession(const PmuOwnerSession &) = delete; + PmuOwnerSession &operator=(const PmuOwnerSession &) = delete; + + ~PmuOwnerSession() + { + if (HasResources()) (void)Finalize(); + } + + bool Initialize( + uint32_t device, aclrtStream scheduling_stream, const std::string &dispatcher_path, + const std::string &owner_path, const std::vector ®ister_bases + ) + { + if (HasResources() || register_bases.size() != kPhysicalSubcoreCount) { + std::fprintf( + stderr, "Invalid PMU owner initialization state or register table size: %zu\n", + register_bases.size() + ); + return false; + } + device_ = device; + if (!QueryActiveSubcoreLimits(scheduling_stream, &limits_)) return false; + if (!OwnerCheckAcl(aclrtCreateStream(&owner_stream_), "aclrtCreateStream(PMU owner)")) return false; + if (loader_.Initialize( + dispatcher_path, owner_path, owner_stream_, static_cast(device_) + ) != 0) { + return false; + } + if (!OwnerCheckAcl( + aclrtMalloc(&control_device_, sizeof(PmuOwnerControl), ACL_MEM_MALLOC_NORMAL_ONLY), + "aclrtMalloc(PMU owner control)" + )) { + return false; + } + if ((reinterpret_cast(control_device_) & (alignof(PmuOwnerControl) - 1U)) != 0U) { + std::fprintf(stderr, "PMU owner control is not 64-byte aligned: %p\n", control_device_); + return false; + } + + control_ = PmuOwnerControl{}; + control_.magic = kPmuOwnerControlMagic; + control_.version = kPmuOwnerControlVersion; + control_.struct_bytes = sizeof(PmuOwnerControl); + control_.status = kStatusPending; + control_.expected_total = limits_.total; + control_.expected_aic = limits_.aic; + control_.expected_aiv = limits_.aiv; + std::memcpy(control_.register_bases, register_bases.data(), sizeof(control_.register_bases)); + if (!OwnerCheckAcl( + aclrtMemcpy( + control_device_, sizeof(control_), &control_, sizeof(control_), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D initial PMU owner control)" + )) { + return false; + } + ready_ = true; + return true; + } + + bool Configure() + { + if (!ready_) return false; + const bool command_ok = RunCommand(PmuOwnerMainCommand::Configure, "Configure"); + configured_ = CountConfigured(control_) != 0U; + const uint32_t bitmap_count = CountConfigured(control_); + const bool triplets_ok = ValidateConfiguredTripletTopology(control_); + const bool state_ok = control_.status == static_cast(PmuOwnerStatus::Success) && + control_.configured == 1U && control_.active_total == limits_.total && + control_.active_aic == limits_.aic && control_.active_aiv == limits_.aiv && + control_.discovered_total == limits_.total && control_.discovered_aic == limits_.aic && + control_.discovered_aiv == limits_.aiv && bitmap_count == limits_.total && + control_.skipped_total + bitmap_count == kPhysicalSubcoreCount && triplets_ok; + PrintControl("Configure", bitmap_count); + return command_ok && state_ok; + } + + bool Restore() + { + if (control_device_ == nullptr || owner_stream_ == nullptr || !loader_.IsInitialized()) { + return !configured_; + } + const bool command_ok = RunCommand(PmuOwnerMainCommand::Restore, "Restore"); + const uint32_t bitmap_count = CountConfigured(control_); + const bool state_ok = control_.status == static_cast(PmuOwnerStatus::Success) && + control_.configured == 0U && control_.active_total == 0U && + control_.active_aic == 0U && control_.active_aiv == 0U && bitmap_count == 0U; + configured_ = bitmap_count != 0U; + PrintControl("Restore", bitmap_count); + return command_ok && state_ok; + } + + bool Finalize() + { + bool ok = true; + if (control_device_ != nullptr && owner_stream_ != nullptr && loader_.IsInitialized()) { + bool restored = Restore(); + if (!restored) restored = Restore(); + ok &= restored; + } else { + ok &= !configured_; + } + ok &= loader_.Finalize() == 0; + if (control_device_ != nullptr) { + ok &= OwnerCheckAcl(aclrtFree(control_device_), "aclrtFree(PMU owner control)"); + control_device_ = nullptr; + } + if (owner_stream_ != nullptr) { + ok &= OwnerCheckAcl(aclrtDestroyStream(owner_stream_), "aclrtDestroyStream(PMU owner)"); + owner_stream_ = nullptr; + } + ready_ = false; + configured_ = false; + std::printf("[PMU_OWNER] restore_and_cleanup=%s\n", ok ? "PASS" : "FAIL"); + return ok; + } + + uint64_t RegisterTableDeviceAddress() const + { + if (control_device_ == nullptr) return 0U; + return reinterpret_cast(control_device_) + offsetof(PmuOwnerControl, register_bases); + } + + const PmuOwnerControl &Control() const { return control_; } + + bool IsConfiguredSubcore(uint32_t index) const + { + return ready_ && IsConfigured(control_, index); + } + +private: + bool HasResources() const + { + return owner_stream_ != nullptr || control_device_ != nullptr || loader_.IsInitialized(); + } + + bool RunCommand(PmuOwnerMainCommand command, const char *label) + { + const PmuOwnerMainKernelArgs arguments = MakePmuOwnerMainKernelArgs( + reinterpret_cast(control_device_), command, device_ + ); + const std::string sync_label = std::string("aclrtSynchronizeStream(PMU ") + label + ")"; + const std::string copy_label = std::string("aclrtMemcpy(D2H PMU ") + label + ")"; + if (loader_.Launch(owner_stream_, const_cast(&arguments), sizeof(arguments)) != 0 || + !OwnerCheckAcl(aclrtSynchronizeStream(owner_stream_), sync_label.c_str()) || + !OwnerCheckAcl( + aclrtMemcpy( + &control_, sizeof(control_), control_device_, sizeof(control_), ACL_MEMCPY_DEVICE_TO_HOST + ), + copy_label.c_str() + )) { + return false; + } + return true; + } + + void PrintControl(const char *command, uint32_t bitmap_count) const + { + const auto failed_field = static_cast(control_.first_failed_field); + const auto restore_field = static_cast(control_.first_restore_failed_field); + std::printf( + "[PMU_OWNER] command=%s status=%d configured=%u active=%u/%u/%u " + "discovered=%u/%u/%u bitmap=%u skipped=%u first_failed=%u:%s(%u):0x%x/0x%x " + "restore_failures=%u first_restore=%u:%s(%u):0x%x/0x%x\n", + command, static_cast(control_.status), control_.configured, + control_.active_total, control_.active_aic, control_.active_aiv, + control_.discovered_total, control_.discovered_aic, control_.discovered_aiv, + bitmap_count, control_.skipped_total, control_.first_failed_index, + OwnerFieldName(failed_field), control_.first_failed_field, + control_.first_failed_observed, control_.first_failed_expected, + control_.restore_failures, control_.first_restore_failed_index, + OwnerFieldName(restore_field), control_.first_restore_failed_field, + control_.first_restore_failed_observed, control_.first_restore_failed_expected + ); + std::printf( + "[PMU_OWNER] configured_bitmap=%08x:%08x:%08x:%08x\n", + control_.configured_bitmap[3], control_.configured_bitmap[2], + control_.configured_bitmap[1], control_.configured_bitmap[0] + ); + } + + uint32_t device_ = 0U; + aclrtStream owner_stream_ = nullptr; + MainAicpuLoader loader_; + void *control_device_ = nullptr; + PmuOwnerControl control_{}; + ActiveSubcoreLimits limits_{}; + bool ready_ = false; + bool configured_ = false; +}; + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_HOST_H_ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_owner_main_abi.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_owner_main_abi.h new file mode 100644 index 0000000000..d6ba9e2239 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_owner_main_abi.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_ABI_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_ABI_H_ + +#include +#include + +namespace pa_scheduler::pmu_owner { + +// 主 aicpu_scheduler 的统一入口 simpler_aicpu_exec 根据该命令选择配置或 +// 恢复。0 特意保留为 Invalid，避免零初始化参数意外改写 PMU 寄存器。 +enum class PmuOwnerMainCommand : uint32_t { + Invalid = 0U, + Configure = 1U, + Restore = 2U, +}; + +// 该结构逐字段复刻 A5 KernelArgs 的 152B ABI，但只使用固定宽度整数，因而 +// 不依赖 Simpler 的 DeviceArgs/Runtime C++ 类型。runtime_args_device 指向 +// PmuOwnerControl；command 位于原 enable_profiling_flag 的 offset 128。 +// 其余字段保持为零，既满足主 aicpu_scheduler 固定布局，也不引入外部依赖。 +struct PmuOwnerMainKernelArgs { + uint64_t unused[5]; // 0..39 + uint64_t device_args_device; // 40，当前 owner 不使用 + uint64_t runtime_args_device; // 48，PmuOwnerControl 的 GM 地址 + uint64_t register_bases_device; // 56，当前 control 已内嵌基址，保持为零 + uint64_t dump_data_base; // 64 + uint64_t l2_swimlane_data_base; // 72 + uint64_t pmu_data_base; // 80 + uint64_t dep_gen_data_base; // 88 + uint64_t l2_swimlane_rotation_table; // 96 + uint64_t aicore_pmu_ring_addrs; // 104 + uint64_t scope_stats_data_base; // 112 + uint32_t log_level; // 120 + uint32_t log_info_v; // 124 + uint32_t command; // 128，PmuOwnerMainCommand + uint32_t reserved_alignment; // 132 + uint64_t device_wall_data_base; // 136 + uint32_t device_id; // 144 + uint32_t force_simt_anchor; // 148 +}; + +static_assert(sizeof(PmuOwnerMainCommand) == sizeof(uint32_t), "PMU owner command ABI changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, device_args_device) == 40U, "device args offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, runtime_args_device) == 48U, "control pointer offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, register_bases_device) == 56U, "register pointer offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, command) == 128U, "PMU owner command offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, device_id) == 144U, "device id offset changed"); +static_assert(sizeof(PmuOwnerMainKernelArgs) == 152U, "main aicpu_scheduler KernelArgs ABI changed"); +static_assert(alignof(PmuOwnerMainKernelArgs) == 8U, "KernelArgs alignment changed"); + +inline PmuOwnerMainKernelArgs MakePmuOwnerMainKernelArgs( + uint64_t control_device, PmuOwnerMainCommand command, uint32_t device_id +) +{ + PmuOwnerMainKernelArgs arguments{}; + arguments.runtime_args_device = control_device; + arguments.command = static_cast(command); + arguments.device_id = device_id; + return arguments; +} + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_ABI_H_ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_owner_main_loader.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_owner_main_loader.h new file mode 100644 index 0000000000..ea34f5fe59 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_owner_main_loader.h @@ -0,0 +1,389 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_LOADER_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_LOADER_H_ + +// PMU owner 的自包含 host 装载器： +// 1. 通过 libaicpu_extend_kernels bootstrap 临时 dispatcher； +// 2. dispatcher 将 owner SO 落到主 aicpu_scheduler 的预安装目录； +// 3. 用 cpuKernelMode=0 JSON 注册 owner 的 simpler_aicpu_exec； +// 4. 后续 Configure/Restore 都用缓存的 rtFuncHandle 直接下发。 +// +// 本头文件故意不定义 owner 命令字段。Launch 接受调用方构造的完整参数块， +// 从而让装载 ABI 与 PMU 状态机 ABI 解耦，也便于先独立验证 Path-A。 + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "runtime/rt.h" +#include "runtime/runtime/rts/rts_kernel.h" + +namespace pa_scheduler::pmu_owner { + +class MainAicpuLoader { +public: + MainAicpuLoader() = default; + MainAicpuLoader(const MainAicpuLoader &) = delete; + MainAicpuLoader &operator=(const MainAicpuLoader &) = delete; + MainAicpuLoader(MainAicpuLoader &&) = delete; + MainAicpuLoader &operator=(MainAicpuLoader &&) = delete; + + ~MainAicpuLoader() { (void)Finalize(); } + + // stream 必须属于当前 device，并且调用期间当前 ACL device 不能切换。 + // 成功后 owner SO 已注册到主 aicpu_scheduler，但尚未执行任何 PMU 命令。 + int Initialize( + const std::string &dispatcher_so_path, const std::string &owner_so_path, + aclrtStream stream, int32_t device_id + ) + { + if (IsInitialized() || stream == nullptr || device_id < 0) { + return Fail("Initialize received invalid state, stream, or device id", kInvalidArgument); + } + + const std::vector dispatcher = ReadBinary(dispatcher_so_path); + const std::vector owner = ReadBinary(owner_so_path); + if (dispatcher.empty() || owner.empty()) { + std::fprintf( + stderr, "[PMU_OWNER_LOADER] cannot read dispatcher/owner: %s (%zu B), %s (%zu B)\n", + dispatcher_so_path.c_str(), dispatcher.size(), owner_so_path.c_str(), owner.size() + ); + return kFileError; + } + + device_id_ = device_id; + owner_fingerprint_ = FingerprintBytes(owner.data(), owner.size()); + owner_so_basename_ = MakeOwnerSoBasename(owner_fingerprint_, device_id_); + op_type_ = MakeOpType(owner_fingerprint_, device_id_); + + int result = Bootstrap(dispatcher, owner, stream); + if (result == 0) result = RegisterOwner(); + if (result != 0) { + (void)Finalize(); + return result; + } + return 0; + } + + // 参数块由 runtime 在 launch 时复制；调用方只需保证本函数返回前 host + // buffer 有效。参数中的 GM 指针仍必须在设备命令同步结束前保持有效。 + int Launch( + aclrtStream stream, void *kernel_arguments, size_t argument_bytes, + uint32_t aicpu_blocks = 1U + ) const + { + if (!IsInitialized() || stream == nullptr || kernel_arguments == nullptr || + argument_bytes == 0U || argument_bytes > std::numeric_limits::max() || + aicpu_blocks == 0U) { + return Fail("Launch received invalid state or arguments", kInvalidArgument); + } + + rtCpuKernelArgs_t cpu_arguments = {}; + cpu_arguments.baseArgs.args = kernel_arguments; + cpu_arguments.baseArgs.argsSize = static_cast(argument_bytes); + rtKernelLaunchCfg_t launch_config = {}; + rtLaunchKernelAttr_t launch_attribute = {}; + launch_config.attrs = &launch_attribute; + launch_config.numAttrs = 0U; + + const rtError_t result = rtsLaunchCpuKernel( + function_handle_, aicpu_blocks, static_cast(stream), + &launch_config, &cpu_arguments + ); + if (result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsLaunchCpuKernel failed: %d\n", result); + } + return static_cast(result); + } + + // Finalize 只释放 host/runtime 注册资源，不删除设备侧预安装 SO；后者按内容 + // 指纹命名，可由同一设备上的后续进程原子覆盖。 + int Finalize() + { + int result = 0; + function_handle_ = nullptr; + if (binary_handle_ != nullptr) { + const rtError_t unload_result = rtsBinaryUnload(binary_handle_); + if (unload_result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsBinaryUnload failed: %d\n", unload_result); + result = static_cast(unload_result); + } + binary_handle_ = nullptr; + } + if (!json_path_.empty()) { + if (std::remove(json_path_.c_str()) != 0 && result == 0) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] remove JSON failed: %s\n", json_path_.c_str()); + result = kFileError; + } + json_path_.clear(); + } + device_id_ = -1; + owner_fingerprint_ = 0U; + owner_so_basename_.clear(); + op_type_.clear(); + return result; + } + + bool IsInitialized() const { return binary_handle_ != nullptr && function_handle_ != nullptr; } + uint64_t OwnerFingerprint() const { return owner_fingerprint_; } + const std::string &OwnerSoBasename() const { return owner_so_basename_; } + const std::string &OpType() const { return op_type_; } + +private: + static constexpr int kInvalidArgument = -1; + static constexpr int kFileError = -2; + static constexpr int kBootstrapError = -3; + static constexpr uint64_t kFnvOffsetBasis = UINT64_C(14695981039346656037); + static constexpr uint64_t kFnvPrime = UINT64_C(1099511628211); + static constexpr const char *kOwnerFunction = "simpler_aicpu_exec"; + + struct DeviceBuffer { + void *address = nullptr; + DeviceBuffer() = default; + DeviceBuffer(const DeviceBuffer &) = delete; + DeviceBuffer &operator=(const DeviceBuffer &) = delete; + ~DeviceBuffer() + { + if (address != nullptr) (void)aclrtFree(address); + } + aclError Allocate(size_t bytes) + { + return aclrtMalloc(&address, bytes, ACL_MEM_MALLOC_HUGE_FIRST); + } + }; + + static int Fail(const char *message, int code) + { + std::fprintf(stderr, "[PMU_OWNER_LOADER] %s\n", message); + return code; + } + + static std::vector ReadBinary(const std::string &path) + { + std::ifstream input(path, std::ios::binary | std::ios::ate); + if (!input.is_open()) return {}; + const std::streampos end = input.tellg(); + if (end <= std::streampos(0) || + static_cast(end) > static_cast(std::numeric_limits::max())) { + return {}; + } + std::vector bytes(static_cast(end)); + input.seekg(0, std::ios::beg); + if (!input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size()))) { + return {}; + } + return bytes; + } + + static uint64_t FingerprintBytes(const void *data, size_t bytes) + { + const auto *input = static_cast(data); + uint64_t hash = kFnvOffsetBasis; + for (size_t index = 0U; index < bytes; ++index) { + hash ^= input[index]; + hash *= kFnvPrime; + } + return hash; + } + + static std::string MakeOwnerSoBasename(uint64_t fingerprint, int32_t device_id) + { + char name[128] = {}; + (void)snprintf( + name, sizeof(name), "pa_scheduler_pmu_owner_%016llx_d%d.so", + static_cast(fingerprint), device_id + ); + return name; + } + + static std::string MakeOpType(uint64_t fingerprint, int32_t device_id) + { + char name[160] = {}; + (void)snprintf( + name, sizeof(name), "pa_scheduler_pmu_owner_%016llx_d%d", + static_cast(fingerprint), device_id + ); + return name; + } + + int Bootstrap( + const std::vector &dispatcher, const std::vector &owner, + aclrtStream stream + ) const + { + DeviceBuffer dispatcher_device; + DeviceBuffer owner_device; + DeviceBuffer device_args; + aclError acl_result = dispatcher_device.Allocate(dispatcher.size()); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMalloc(dispatcher)", acl_result); + acl_result = aclrtMemcpy( + dispatcher_device.address, dispatcher.size(), dispatcher.data(), dispatcher.size(), + ACL_MEMCPY_HOST_TO_DEVICE + ); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMemcpy(dispatcher H2D)", acl_result); + + acl_result = owner_device.Allocate(owner.size()); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMalloc(owner)", acl_result); + acl_result = aclrtMemcpy( + owner_device.address, owner.size(), owner.data(), owner.size(), ACL_MEMCPY_HOST_TO_DEVICE + ); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMemcpy(owner H2D)", acl_result); + + constexpr size_t kDeviceArgsBytes = 160U; + uint8_t host_device_args[kDeviceArgsBytes] = {}; + const auto write_qword = [&](size_t offset, uint64_t value) { + std::memcpy(host_device_args + offset, &value, sizeof(value)); + }; + write_qword(96U, reinterpret_cast(dispatcher_device.address)); + write_qword(104U, static_cast(dispatcher.size())); + write_qword(112U, static_cast(device_id_)); + write_qword(120U, reinterpret_cast(owner_device.address)); + write_qword(128U, static_cast(owner.size())); + + acl_result = device_args.Allocate(kDeviceArgsBytes); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMalloc(bootstrap args)", acl_result); + acl_result = aclrtMemcpy( + device_args.address, kDeviceArgsBytes, host_device_args, kDeviceArgsBytes, + ACL_MEMCPY_HOST_TO_DEVICE + ); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMemcpy(bootstrap args H2D)", acl_result); + + // k_args 总长和三个字符串 offset 与仓内已上板的 Path-A 完全一致。 + struct BootstrapArguments { + struct { + uint64_t unused[5]; + uint64_t device_args_address; + uint64_t padding[20]; + } kernel_args; + char kernel_name[32]; + char so_name[32]; + char op_name[32]; + } arguments = {}; + static_assert(offsetof(BootstrapArguments, kernel_args.device_args_address) == 40U, "bootstrap ABI changed"); + arguments.kernel_args.device_args_address = reinterpret_cast(device_args.address); + constexpr char kBootstrapKernel[] = "DynTileFwkKernelServerInit"; + constexpr char kBootstrapSo[] = "libaicpu_extend_kernels.so"; + static_assert(sizeof(kBootstrapKernel) <= sizeof(arguments.kernel_name), "bootstrap kernel name too long"); + static_assert(sizeof(kBootstrapSo) <= sizeof(arguments.so_name), "bootstrap SO name too long"); + std::memcpy(arguments.kernel_name, kBootstrapKernel, sizeof(kBootstrapKernel)); + std::memcpy(arguments.so_name, kBootstrapSo, sizeof(kBootstrapSo)); + + rtAicpuArgsEx_t runtime_arguments = {}; + runtime_arguments.args = &arguments; + runtime_arguments.argsSize = sizeof(arguments); + runtime_arguments.kernelNameAddrOffset = offsetof(BootstrapArguments, kernel_name); + runtime_arguments.soNameAddrOffset = offsetof(BootstrapArguments, so_name); + + const rtError_t launch_result = rtAicpuKernelLaunchExWithArgs( + rtKernelType_t::KERNEL_TYPE_AICPU_KFC, "AST_DYN_AICPU", 1U, + &runtime_arguments, nullptr, static_cast(stream), 0U + ); + if (launch_result != RT_ERROR_NONE) { + std::fprintf( + stderr, "[PMU_OWNER_LOADER] rtAicpuKernelLaunchExWithArgs failed: %d\n", + launch_result + ); + return static_cast(launch_result); + } + acl_result = aclrtSynchronizeStream(stream); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtSynchronizeStream(bootstrap)", acl_result); + return 0; + } + + int RegisterOwner() + { + char path[256] = {}; + (void)snprintf( + path, sizeof(path), "/tmp/pa_scheduler_pmu_owner_%016llx_d%d_p%d_i%016llx.json", + static_cast(owner_fingerprint_), device_id_, static_cast(getpid()), + static_cast(reinterpret_cast(this)) + ); + json_path_ = path; + if (!WriteJson()) return kFileError; + + rtLoadBinaryOption_t option = {}; + option.optionId = RT_LOAD_BINARY_OPT_CPU_KERNEL_MODE; + option.value.cpuKernelMode = 0; + rtLoadBinaryConfig_t configuration = {}; + configuration.options = &option; + configuration.numOpt = 1U; + + rtError_t result = rtsBinaryLoadFromFile(json_path_.c_str(), &configuration, &binary_handle_); + if (result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsBinaryLoadFromFile failed: %d\n", result); + return static_cast(result); + } + result = rtsFuncGetByName(binary_handle_, op_type_.c_str(), &function_handle_); + if (result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsFuncGetByName(%s) failed: %d\n", op_type_.c_str(), result); + return static_cast(result); + } + return 0; + } + + bool WriteJson() const + { + std::ofstream json(json_path_, std::ios::out | std::ios::trunc); + if (!json.is_open()) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] cannot create JSON: %s\n", json_path_.c_str()); + return false; + } + // 所有动态字段仅含固定前缀、十六进制、十进制和下划线，不需要 JSON 转义。 + json << "{\n" + << " \"" << op_type_ << "\": {\n" + << " \"opInfo\": {\n" + << " \"functionName\": \"" << kOwnerFunction << "\",\n" + << " \"kernelSo\": \"" << owner_so_basename_ << "\",\n" + << " \"opKernelLib\": \"AICPUKernel\",\n" + << " \"computeCost\": \"100\",\n" + << " \"engine\": \"DNN_VM_AICPU\",\n" + << " \"flagAsync\": \"False\",\n" + << " \"flagPartial\": \"False\",\n" + << " \"userDefined\": \"False\"\n" + << " }\n" + << " }\n" + << "}\n"; + json.close(); + if (!json) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] writing JSON failed: %s\n", json_path_.c_str()); + return false; + } + return true; + } + + static int ReportAcl(const char *operation, aclError result) + { + std::fprintf(stderr, "[PMU_OWNER_LOADER] %s failed: %d\n", operation, static_cast(result)); + return static_cast(result == ACL_SUCCESS ? kBootstrapError : result); + } + + int32_t device_id_ = -1; + uint64_t owner_fingerprint_ = 0U; + std::string owner_so_basename_; + std::string op_type_; + std::string json_path_; + rtBinHandle binary_handle_ = nullptr; + rtFuncHandle function_handle_ = nullptr; +}; + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_LOADER_H_ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_probe.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_probe.h new file mode 100644 index 0000000000..a2e8cab6a0 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/ccec/pmu_probe.h @@ -0,0 +1,185 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_PROBE_H +#define PA_SCHEDULER_CCEC_PMU_PROBE_H + +#include + +#include "../common/pa_model.h" + +namespace pa_scheduler::ccec_pmu { + +// Empty/Scalar/ScalarDouble 在调度结束后校准门控底噪和 scalar 正向响应； +// IcacheSingle 在每核上成对累计隔离的 cold/warm 目标调用；SubmitAll 则在 +// 公共调度器 hook 内覆盖本 worker 的完整 Submit 回放窗口。 +enum class WindowMode : uint32_t { + Off = 0, + Empty = 1, + Scalar = 2, + ScalarDouble = 3, + // 保留已落远端的 I-cache 校准模式值，避免 standalone host/kernel 混用旧产物时 + // 把校准请求误解释成 Submit 窗口；新增模式只在枚举尾部扩展。 + IcacheSingle = 4, + // SubmitAll 从本 worker 的 orchestration/Submit 回放前开始，到最后一次 + // Submit 返回后停止。 + SubmitAll = 5, +}; + +inline bool IsSubmitWindow(WindowMode mode) { + return mode == WindowMode::SubmitAll; +} + +// RunConfig::reserved 保持既有 64B ABI；CCEC 独占解释以下五个槽位，其他后端仍看到全零。 +constexpr uint32_t kConfigMode = 0; +constexpr uint32_t kConfigWorkAmount = 1; +constexpr uint32_t kConfigScalarNops = kConfigWorkAmount; +constexpr uint32_t kConfigIcacheTrials = kConfigWorkAmount; +constexpr uint32_t kConfigRegTableLow = 2; +constexpr uint32_t kConfigRegTableHigh = 3; +constexpr uint32_t kConfigMagic = 4; +constexpr uint32_t kConfigMagicValue = 0x504d5531U; // "PMU1" + +// DAV_3510 有 36 个物理 AICore，每个 AICore 展开为 1 AIC + 2 AIV，共 108 个物理子核编号。 +constexpr uint32_t kPhysicalAicoreCount = 36; +constexpr uint32_t kPhysicalSubcoreCount = 108; +constexpr uint32_t kAicorePerDie = 18; +constexpr uint32_t kSubcoresPerDie = 54; +constexpr uint64_t kAivFirstOffset = 0x100000ULL; +constexpr uint64_t kAivSecondOffset = 0x200000ULL; +constexpr uint32_t kAicoreMapBytes = 0x300000U; + +// PIPE_UTILIZATION 事件由 standalone Main AICPU owner 配置，kernel 逐核读回并核对 selector。 +constexpr uint32_t kScalarBusyEvent = 0x1U; +constexpr uint32_t kIcacheRequestEvent = 0x34U; +constexpr uint32_t kIcacheMissEvent = 0x35U; +constexpr uint32_t kVectorBusyEvent = 0x501U; +constexpr uint32_t kCubeBusyEvent = 0x301U; +constexpr uint32_t kMte1BusyEvent = 0x701U; +constexpr uint32_t kMte2BusyEvent = 0x202U; +constexpr uint32_t kMte3BusyEvent = 0x203U; +constexpr uint32_t kFixBusyEvent = 0x714U; + +// DAV_3510 PMU MMIO offset。ld_dev 的立即数只有 12 bit，因此 kernel 会分别重基址到 0x2400/0x4200。 +constexpr uint32_t kSelectorBlockOffset = 0x2400U; +constexpr uint32_t kCounterBlockOffset = 0x4200U; +constexpr uint32_t kCnt2Offset = 0x4220U; +constexpr uint32_t kCnt0Offset = 0x4210U; +constexpr uint32_t kCnt1Offset = 0x4218U; +constexpr uint32_t kCnt3Offset = 0x4228U; +constexpr uint32_t kCnt4Offset = 0x4230U; +constexpr uint32_t kCnt5Offset = 0x4238U; +constexpr uint32_t kCnt6Offset = 0x4240U; +constexpr uint32_t kCnt7Offset = 0x4248U; +constexpr uint32_t kCnt8Offset = 0x4250U; +constexpr uint32_t kCnt9Offset = 0x4254U; +constexpr uint32_t kTotalLowOffset = 0x4260U; +constexpr uint32_t kTotalHighOffset = 0x4264U; +constexpr uint32_t kCnt2SelectorOffset = 0x2508U; +constexpr uint32_t kCnt0SelectorOffset = 0x2500U; +constexpr uint32_t kCnt1SelectorOffset = 0x2504U; +constexpr uint32_t kCnt3SelectorOffset = 0x250cU; +constexpr uint32_t kCnt4SelectorOffset = 0x2510U; +constexpr uint32_t kCnt5SelectorOffset = 0x2514U; +constexpr uint32_t kCnt6SelectorOffset = 0x2518U; +constexpr uint32_t kCnt7SelectorOffset = 0x251cU; +constexpr uint32_t kCnt8SelectorOffset = 0x2520U; +constexpr uint32_t kCnt9SelectorOffset = 0x2524U; + +// pmu_status 的 bits16..27 保存 get_coreid()；bits28..31 留给不参与 core id +// 解码的模式诊断。其余低位描述本条记录是否可信。 +constexpr uint32_t kStatusRequested = 1U << 0; +constexpr uint32_t kStatusRegMapped = 1U << 1; +constexpr uint32_t kStatusCoreIdValid = 1U << 2; +constexpr uint32_t kStatusCnt2Selector = 1U << 3; +constexpr uint32_t kStatusCnt6Selector = 1U << 4; +constexpr uint32_t kStatusCnt7Selector = 1U << 5; +constexpr uint32_t kStatusWindowStarted = 1U << 6; +constexpr uint32_t kStatusTotalNonzero = 1U << 7; +constexpr uint32_t kStatusPriorSnapshotLarger = 1U << 8; +constexpr uint32_t kStatusCnt0Selector = 1U << 9; +constexpr uint32_t kStatusCnt1Selector = 1U << 10; +constexpr uint32_t kStatusCnt3Selector = 1U << 11; +constexpr uint32_t kStatusCnt4Selector = 1U << 12; +constexpr uint32_t kStatusCnt5Selector = 1U << 13; +constexpr uint32_t kStatusCnt8Selector = 1U << 14; +constexpr uint32_t kStatusWindowStopped = 1U << 15; +// I-cache 配对标志不能复用 Submit start bit；StatusCoreId 只取 12 bit，故将 +// 它放在 core-id 区间之上的独立诊断位。 +constexpr uint32_t kStatusIcachePairObserved = 1U << 28; +constexpr uint32_t kStatusRequired = kStatusRequested | kStatusRegMapped | kStatusCoreIdValid | + kStatusCnt2Selector | kStatusCnt6Selector | kStatusCnt7Selector | + kStatusCnt0Selector | kStatusCnt1Selector | kStatusCnt3Selector | + kStatusCnt4Selector | kStatusCnt5Selector | kStatusCnt8Selector | + kStatusWindowStarted | kStatusWindowStopped | kStatusTotalNonzero; +constexpr uint32_t kStatusCoreIdShift = 16; +constexpr uint32_t kStatusCoreIdMask = 0x0fffU; + +// pmu_phase_status 独立于旧 pmu_status，避免与其中的物理 core-id 位域 +// 冲突。bits4/5 只记录 shadow 是否恰好等于 primary：phase=none 没有 +// 运行中 read-to-clear，host 会要求两位都成立；局部 phase 会在计数仍开启时 +// 读取 shadow，A5 实测存在同周期递增与读清竞争，因此不能把“逐次严格相等” +// 作为可信记录的共同必选位。局部 phase 的方向和误差包络由 host/raw 独立校验。 +constexpr uint32_t kPhaseStatusRequested = 1U << 0; +constexpr uint32_t kPhaseStatusShadowSelectors = 1U << 1; +constexpr uint32_t kPhaseStatusWindowStarted = 1U << 2; +constexpr uint32_t kPhaseStatusWindowStopped = 1U << 3; +constexpr uint32_t kPhaseStatusShadowRequestsMatch = 1U << 4; +constexpr uint32_t kPhaseStatusShadowMissesMatch = 1U << 5; +constexpr uint32_t kPhaseStatusBoundariesBalanced = 1U << 6; +constexpr uint32_t kPhaseStatusValuesOrdered = 1U << 7; +constexpr uint32_t kPhaseStatusUint32Fit = 1U << 8; +constexpr uint32_t kPhaseStatusPhaseShape = 1U << 9; +// none 必须保持 0 tick；运行阶段则必须确实累计到非零 SYS_CNT。阶段时间是否 +// 不超过同核首 Submit 到末 Submit 的完整区间，由拿到两端结果的 host 再校验。 +constexpr uint32_t kPhaseStatusTimeValid = 1U << 10; +constexpr uint32_t kPhaseStatusRequired = + kPhaseStatusRequested | kPhaseStatusShadowSelectors | + kPhaseStatusWindowStarted | kPhaseStatusWindowStopped | + kPhaseStatusBoundariesBalanced | kPhaseStatusValuesOrdered | + kPhaseStatusUint32Fit | kPhaseStatusPhaseShape | kPhaseStatusTimeValid; + +inline const char *SubmitPmuPhaseName(SubmitPmuPhase phase) { + switch (phase) { + case SubmitPmuPhase::None: + return "none"; + case SubmitPmuPhase::Claim: + return "claim"; + case SubmitPmuPhase::EfDrain: + return "efdrain"; + case SubmitPmuPhase::Materialize: + return "materialize"; + case SubmitPmuPhase::Register: + return "register"; + case SubmitPmuPhase::Count: + break; + } + return "invalid"; +} + +inline uint64_t PackPointer(const uint32_t *words) { + return static_cast(words[kConfigRegTableLow]) | + (static_cast(words[kConfigRegTableHigh]) << 32); +} + +inline void StorePointer(uint32_t *words, const void *pointer) { + const uint64_t raw = reinterpret_cast(pointer); + words[kConfigRegTableLow] = static_cast(raw); + words[kConfigRegTableHigh] = static_cast(raw >> 32); +} + +inline uint32_t StatusCoreId(uint32_t status) { + return (status >> kStatusCoreIdShift) & kStatusCoreIdMask; +} + +} // namespace pa_scheduler::ccec_pmu + +#endif // PA_SCHEDULER_CCEC_PMU_PROBE_H diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/host_support.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/host_support.h new file mode 100644 index 0000000000..1185536866 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/host_support.h @@ -0,0 +1,1609 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_HOST_SUPPORT_H +#define PA_SCHEDULER_COMMON_HOST_SUPPORT_H + +#include "pa_model.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pa_scheduler::host { + +// 三种后端共用同一套命令行配置，保证 CPU 语义回归与 A5 上板使用完全相同的工作量。 +struct Options { + std::string kernel_path; + std::string swimlane_json; + uint32_t device = 0; + uint32_t batches = kDefaultBatches; + uint32_t runs = 5; + NopCounts nops{kDefaultQkNops, kDefaultSfNops, kDefaultPvNops, kDefaultUpNops}; + bool profile_phases = false; + bool trace_enabled = true; + bool trace_atomics = false; + bool analyze_swimlane = false; +}; + +enum class ParseStatus { + Ok, + Help, + Error, +}; + +inline bool ParseUint(const char *raw, uint32_t minimum, uint32_t maximum, uint32_t *value) { + // 要求整串都能被 strtoul 解析且结果落在给定范围内，拒绝尾随字符和溢出值， + // 避免参数被部分解析后悄悄改变工作量。 + errno = 0; + char *end = nullptr; + const unsigned long parsed = std::strtoul(raw, &end, 10); + if (errno != 0 || end == raw || *end != '\0' || parsed < minimum || parsed > maximum) { + return false; + } + *value = static_cast(parsed); + return true; +} + +inline bool ParseNopCounts(const char *raw, NopCounts *counts) { + // 四类 kernel 的 NOP 数必须一次性完整给出，顺序固定为 QK、SF、PV、UP。 + unsigned int qk = 0; + unsigned int sf = 0; + unsigned int pv = 0; + unsigned int up = 0; + char tail = '\0'; + if (std::sscanf(raw, "%u,%u,%u,%u%c", &qk, &sf, &pv, &up, &tail) != 4) { + return false; + } + constexpr uint32_t kMaxNopCount = 10000000; + if (qk > kMaxNopCount || sf > kMaxNopCount || pv > kMaxNopCount || up > kMaxNopCount) { + return false; + } + *counts = NopCounts{qk, sf, pv, up}; + return true; +} + +inline void PrintUsage(const char *program, bool require_kernel) { + // require_kernel 只影响 CCEC host 的用法文本，其余 benchmark 参数在三后端完全一致。 + std::fprintf( + stderr, "Usage: %s%s [--device N] [--batches 1..256] [--runs N] ", program, + require_kernel ? " --kernel FILE" : "" + ); + std::fprintf( + stderr, + "[--nop-count N | --nop-counts QK,SF,PV,UP] [--profile-phases] [--analyze-swimlane] " + "[--trace-atomics] [--swimlane-json FILE] [--no-swimlane]\n" + ); +} + +inline ParseStatus ParseOptions(int argc, char **argv, bool require_kernel, Options *options) { + // CCEC host 需要外部 kernel ELF；AscendC 和 CPU 的可执行文件已包含 kernel，因此不需要该参数。 + bool nop_override_seen = false; + bool swimlane_json_seen = false; + for (int index = 1; index < argc; ++index) { + // 无值开关先处理；其余参数统一在消费下一个 argv 前检查缺值，保证错误位置明确。 + const std::string argument = argv[index]; + if (argument == "--help" || argument == "-h") { + PrintUsage(argv[0], require_kernel); + return ParseStatus::Help; + } + if (argument == "--profile-phases") { + options->profile_phases = true; + continue; + } + if (argument == "--no-swimlane") { + options->trace_enabled = false; + continue; + } + if (argument == "--trace-atomics") { + options->trace_atomics = true; + continue; + } + if (argument == "--analyze-swimlane") { + options->analyze_swimlane = true; + continue; + } + if (index + 1 >= argc) { + std::fprintf(stderr, "Missing value after %s\n", argument.c_str()); + return ParseStatus::Error; + } + const char *value = argv[++index]; + if (argument == "--kernel" && require_kernel) { + options->kernel_path = value; + } else if (argument == "--device") { + if (!ParseUint(value, 0, INT32_MAX, &options->device)) return ParseStatus::Error; + } else if (argument == "--batches") { + if (!ParseUint(value, 1, kMaxBatches, &options->batches)) return ParseStatus::Error; + } else if (argument == "--runs") { + if (!ParseUint(value, 1, 1000, &options->runs)) return ParseStatus::Error; + } else if (argument == "--swimlane-json") { + if (swimlane_json_seen) { + std::fprintf(stderr, "Specify --swimlane-json only once.\n"); + return ParseStatus::Error; + } + if (*value == '\0') { + std::fprintf(stderr, "--swimlane-json requires a non-empty path.\n"); + return ParseStatus::Error; + } + options->swimlane_json = value; + swimlane_json_seen = true; + } else if (argument == "--nop-count") { + if (nop_override_seen) { + std::fprintf(stderr, "Specify only one NOP override.\n"); + return ParseStatus::Error; + } + uint32_t count = 0; + if (!ParseUint(value, 0, 10000000, &count)) return ParseStatus::Error; + options->nops = NopCounts{count, count, count, count}; + nop_override_seen = true; + } else if (argument == "--nop-counts") { + if (nop_override_seen) { + std::fprintf(stderr, "Specify only one NOP override.\n"); + return ParseStatus::Error; + } + if (!ParseNopCounts(value, &options->nops)) return ParseStatus::Error; + nop_override_seen = true; + } else { + std::fprintf(stderr, "Unknown argument: %s\n", argument.c_str()); + return ParseStatus::Error; + } + } + if (require_kernel && options->kernel_path.empty()) { + std::fprintf(stderr, "--kernel is required\n"); + return ParseStatus::Error; + } + if (options->analyze_swimlane && !options->trace_enabled) { + // 分析和导出都依赖完整 record 缓冲，不能与节省内存的 --no-swimlane 同时使用。 + std::fprintf(stderr, "--analyze-swimlane requires swimlane tracing.\n"); + return ParseStatus::Error; + } + if (options->trace_atomics && !options->trace_enabled) { + std::fprintf(stderr, "--trace-atomics cannot be combined with --no-swimlane.\n"); + return ParseStatus::Error; + } + if (!options->swimlane_json.empty() && !options->trace_enabled) { + std::fprintf(stderr, "--swimlane-json requires swimlane tracing.\n"); + return ParseStatus::Error; + } + if (!options->swimlane_json.empty() && options->runs != 1) { + // 一个文件只对应一次完整采集，禁止多轮运行反复覆盖而丢失轮次边界。 + std::fprintf(stderr, "--swimlane-json requires --runs 1 to avoid overwriting captures.\n"); + return ParseStatus::Error; + } + return ParseStatus::Ok; +} + +inline void InitializeState(SchedulerState *state, const Options &options) { + // WorkerState 有意保持真实 PA 每核约 9 MiB 的布局。若 host 每轮清空全部 worker， + // 会额外触碰并拷贝近 1 GiB 内存；因此只初始化全局前缀和结果区，worker 的活跃字段 + // 由各自 kernel 在启动后复位，这也与真实 PA 的生命周期一致。 + std::memset(state, 0, offsetof(SchedulerState, workers)); + std::memset(&state->config, 0, offsetof(SchedulerState, results) - offsetof(SchedulerState, config)); + std::memset(state->results, 0, sizeof(state->results)); + state->heap_window = kHeapWindow; + state->heap_base = kSyntheticHeapBase; + state->heap_size = kHeapBytes; + state->num_workers = kWorkers; + state->num_blocks = kAicWorkers; + state->config.batches = options.batches; + state->config.workers = kWorkers; + state->config.nops = options.nops; + state->config.profile_phases = options.profile_phases ? 1U : 0U; + for (uint32_t batch = 0; batch < options.batches; ++batch) { + state->context_lens[batch] = 8192; + } + for (uint32_t shard = 0; shard < kCursorShards; ++shard) { + // -1 表示尚无 task 被 claim；task 0 的 atomicMax 因而也能正常判定唯一 winner。 + state->cube_cursor[shard].value = -1; + state->vector_cursor[shard].value = -1; + state->alloc_cursor[shard].value = -1; + } + state->frontier.value = -1; +} + +inline void ConfigureTrace(SchedulerState *state, const Options &options, const void *trace_base) { + // device 只持有裸地址和每核容量；TraceHeader/record 缓冲区由 host 单独分配并初始化。 + state->config.trace_enabled = options.trace_enabled + ? kTracePhasesEnabled | (options.trace_atomics ? kTraceAtomicsEnabled : 0U) + : 0U; + state->config.trace_base = options.trace_enabled ? reinterpret_cast(trace_base) : 0; + state->config.trace_records_per_core = options.trace_enabled ? kTraceRecordsPerCore : 0; +} + +inline void InitializeTraceHeader(TraceHeader *header) { + // version=4 表示 phase ABI 已追加父区间和真实 Submit 尾动作；core state + // 继续携带 weighted atomic/PollBatch 计数和权威拓扑。 + std::memset(header, 0, sizeof(*header)); + header->magic = 0x4653574cU; + header->version = 4; + header->num_cores = kWorkers; + header->records_per_core = kTraceRecordsPerCore; + header->frequency_hz = kSystemCounterHz; +} + +// 巨大的 WorkerState 不参与每轮 H2D/D2H；以下三个范围只搬运运行所需的前缀、控制量和结果。 +inline constexpr size_t StatePrefixBytes() { return offsetof(SchedulerState, workers); } + +inline constexpr size_t ControlBytes() { + // control sidecar 位于为生产 DistGlobal 保留的总跨度之后，到 results 之前为止。 + return offsetof(SchedulerState, results) - offsetof(SchedulerState, config); +} + +inline constexpr size_t ResultBytes() { return sizeof(WorkerResult) * kWorkers; } + +struct Metrics { + // passed 是全部语义断言的合取；submit_span_us 是本用例唯一用于对比 PA 的性能口径。 + bool passed = true; + double submit_span_us = 0; +}; + +inline void Expect(bool condition, const char *label, Metrics *metrics) { + // 所有断言都继续执行，以便一次失败运行尽可能暴露完整状态，而不是遇到首错立即退出。 + std::printf("[ASSERT] %-48s %s\n", label, condition ? "PASS" : "FAIL"); + if (!condition) metrics->passed = false; +} + +struct Uint64Distribution { + uint64_t total = 0; + double median = 0.0; + uint64_t p95 = 0; + uint64_t maximum = 0; +}; + +inline Uint64Distribution SummarizeUint64(std::vector values) { + // 这里按 worker 维度统计累计周期，p95 使用 nearest-rank，避免插值掩盖慢核。 + Uint64Distribution summary; + if (values.empty()) return summary; + + std::sort(values.begin(), values.end()); + for (uint64_t value : values) summary.total += value; + const size_t middle = values.size() / 2; + summary.median = (values.size() & 1U) != 0 + ? static_cast(values[middle]) + : (static_cast(values[middle - 1]) + static_cast(values[middle])) / 2.0; + const size_t p95_rank = (95U * values.size() + 99U) / 100U; + summary.p95 = values[p95_rank - 1]; + summary.maximum = values.back(); + return summary; +} + +inline void PrintPhaseDiagnostics(const SchedulerState &state) { + if (state.config.profile_phases == 0) return; + + // WaitForSlot/HeapGuard 没有各自独立命名的 TracePhase；实际发生等待时会写 + // RingBp 记录，汇总诊断则使用 WorkerResult 中的累计周期和等待次数。 + struct PhaseSpec { + ProfilePhase phase; + const char *name; + int32_t wait_event_index; + }; + const PhaseSpec phases[] = { + {ProfilePhase::Claim, "Claim", -1}, + {ProfilePhase::EfDrain, "EfDrain", -1}, + {ProfilePhase::WaitForSlot, "WaitForSlot", 0}, + {ProfilePhase::HeapGuard, "HeapGuard", 1}, + }; + const CoreRole roles[] = {CoreRole::Aic, CoreRole::Aiv}; + const char *role_names[] = {"AIC", "AIV"}; + + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + // AIC/AIV 分开统计，避免 32:64 的参与者数量差异掩盖某一类核上的长尾。 + for (const PhaseSpec &phase : phases) { + std::vector cycles; + std::vector calls; + std::vector wait_events; + const uint32_t phase_index = static_cast(phase.phase); + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const WorkerResult &result = state.results[worker]; + if (result.role != static_cast(roles[role_index])) continue; + cycles.push_back(result.phase_cycles[phase_index]); + calls.push_back(result.phase_calls[phase_index]); + wait_events.push_back( + phase.wait_event_index < 0 ? 0 : result.wait_events[static_cast(phase.wait_event_index)] + ); + } + const Uint64Distribution cycle_summary = SummarizeUint64(cycles); + const Uint64Distribution call_summary = SummarizeUint64(calls); + const Uint64Distribution wait_summary = SummarizeUint64(wait_events); + std::printf( + "[PHASE] role=%s phase=%s workers=%zu accumulated_us_median=%.3f " + "accumulated_us_p95=%.3f accumulated_us_max=%.3f calls_total=%llu " + "calls_per_worker_median=%.1f calls_per_worker_p95=%llu calls_per_worker_max=%llu " + "wait_events_total=%llu wait_events_per_worker_median=%.1f " + "wait_events_per_worker_p95=%llu wait_events_per_worker_max=%llu\n", + role_names[role_index], phase.name, cycles.size(), cycle_summary.median / 1000.0, + static_cast(cycle_summary.p95) / 1000.0, + static_cast(cycle_summary.maximum) / 1000.0, + static_cast(call_summary.total), call_summary.median, + static_cast(call_summary.p95), + static_cast(call_summary.maximum), + static_cast(wait_summary.total), wait_summary.median, + static_cast(wait_summary.p95), + static_cast(wait_summary.maximum) + ); + } + } +} + +inline const char *TracePhaseName(uint32_t phase) { + // 名称必须与 l2_swimlane_records.json 的 fdwic_events schema 保持一致。 + const char *names[] = { + "Kernel", "Alloc", "Build", "DrainWon", "Replay", "RingBp", "EfDrain", "Commit", + "Submit", "Materialize", "PrepareMap", "Claim", "Fanin", "Register", "Atomic", + "ClockBaseline", "OrchestrationReplay", "FinalDrain", "WinnerBuild", + "AllocComplete", + }; + static_assert( + sizeof(names) / sizeof(names[0]) == static_cast(TracePhase::Count), + "TracePhaseName must cover every trace phase" + ); + return phase < sizeof(names) / sizeof(names[0]) ? names[phase] : "Unknown"; +} + +inline const char *AtomicSiteName(uint32_t site) { + // 顺序与 pa_model.h::AtomicSite 的稳定 raw ABI 完全一致。 + const char *names[] = { + "StartupIncrement", "StartupPoll", "FatalPoll", "FatalSet", "ClaimMax", + "FaninFlagLoad", "CompletionVendExchange", "CompletionFlagExchange", + "FrontierInitialLoad", "FrontierFlagLoad", "FrontierMax", "HeapFrontierLoad", + "HeapVendLoad", "ReplayDoneIncrement", "ReplayDonePoll", + }; + return site < sizeof(names) / sizeof(names[0]) ? names[site] : "Unknown"; +} + +inline const char *AtomicOpName(uint32_t op) { + const char *names[] = {"Load", "Exchange", "FetchAdd", "FetchMax"}; + return op < sizeof(names) / sizeof(names[0]) ? names[op] : "Unknown"; +} + +inline AtomicOp AtomicSiteOp(AtomicSite site) { + return AtomicSiteExpectedOp(site); +} + +inline bool ValidateTraceHeader(const TraceHeader &header, const char *operation) { + // 在任何 D2H record 搬运前先验证容量和 dropped，防止损坏 header 导致 scratch 越界或导出残缺泳道。 + // 频率也要求精确为 1 GHz，否则后续 ns/us 换算即使 JSON 合法也没有性能意义。 + const bool valid = header.magic == 0x4653574cU && header.version == 4 && + header.num_cores == kWorkers && header.records_per_core == kTraceRecordsPerCore && + header.frequency_hz == kSystemCounterHz; + bool core_states_valid = true; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const TraceCoreState &core = header.cores[worker]; + core_states_valid &= core.count <= kTraceRecordsPerCore; + core_states_valid &= core.dropped == 0; + core_states_valid &= core.poll_calls <= core.atomic_calls; + core_states_valid &= (core.poll_calls == 0) == (core.poll_batch_records == 0); + const uint64_t physical_atomic = + static_cast(core.atomic_calls) - core.poll_calls + core.poll_batch_records; + core_states_valid &= physical_atomic <= core.count; + } + if (!valid || !core_states_valid) { + std::fprintf( + stderr, + "%s rejected an invalid trace header: magic=0x%08x version=%u cores=%u " + "records_per_core=%u frequency_hz=%llu core_states_valid=%s\n", + operation, header.magic, header.version, header.num_cores, header.records_per_core, + static_cast(header.frequency_hz), core_states_valid ? "yes" : "no" + ); + } + return valid && core_states_valid; +} + +struct TraceExportSummary { + uint64_t records = 0; + uint64_t atomic_records = 0; + uint64_t clock_baseline_records = 0; + uint64_t atomic_calls = 0; + uint64_t poll_calls = 0; + uint64_t poll_batch_records = 0; + uint64_t dropped_records = 0; +}; + +inline bool SameTraceSummary(const TraceExportSummary &left, const TraceExportSummary &right) { + return left.records == right.records && left.atomic_records == right.atomic_records && + left.clock_baseline_records == right.clock_baseline_records && + left.atomic_calls == right.atomic_calls && left.poll_calls == right.poll_calls && + left.poll_batch_records == right.poll_batch_records && + left.dropped_records == right.dropped_records; +} + +inline uint32_t AtomicRecordCallCount(const TraceRecord &record) { + return (record.flags & kAtomicPollBatch) != 0 + ? record.flags >> kAtomicPollCountShift + : 1U; +} + +inline bool AtomicRecordSchemaValid(const TraceRecord &record, bool atomic_trace_enabled) { + if (!atomic_trace_enabled || record.auxiliary >= static_cast(AtomicSite::Count)) { + return false; + } + const AtomicSite site = static_cast(record.auxiliary); + const uint32_t op = record.flags & kAtomicOpMask; + if (op != static_cast(AtomicSiteExpectedOp(site))) return false; + + const bool result_used = (record.flags & kAtomicResultUsed) != 0; + const bool value_zero = (record.flags & kAtomicValueZero) != 0; + const bool return_ready = (record.flags & kAtomicReturnReady) != 0; + const bool poll_batch = (record.flags & kAtomicPollBatch) != 0; + const uint32_t payload = record.flags >> kAtomicRetriesShift; + if (poll_batch) { + return AtomicSiteIsPollBatchable(site) && result_used && !value_zero && !return_ready && + payload > 0 && record.task_id == -1 && record.function_id == -1; + } + if (result_used != AtomicSiteResultUsed(site) || (return_ready && !result_used)) return false; + if (value_zero && op != static_cast(AtomicOp::Load)) return false; + if (payload != 0 && op != static_cast(AtomicOp::FetchMax)) return false; + return record.function_id == -1; +} + +inline bool ClockRecordSchemaValid(const TraceRecord &record) { + const bool dependency = (record.flags & kClockAtomicDependency) != 0; + const bool dependency_applied = (record.flags & kClockAtomicDependencyApplied) != 0; + return (record.flags & ~(kClockAtomicDependency | kClockAtomicDependencyApplied)) == 0 && + (!dependency_applied || dependency) && record.task_id == -1 && + record.function_id == -1 && record.auxiliary == 0; +} + +inline void ExpectedTraceTopology(uint32_t worker, int32_t *block_id, int32_t *lane) { + if (worker < kAicWorkers) { + *block_id = static_cast(worker); + *lane = 0; + return; + } + const uint32_t vector_id = worker - kAicWorkers; + *block_id = static_cast(vector_id / 2); + *lane = static_cast(1 + vector_id % 2); +} + +template +inline bool ExportSwimlaneRecords( + const TraceHeader &header, const std::string &output_path, + WinnerWorkloadMode workload_mode, const WorkloadCounts &workload_counts, + const char *workload_pattern, bool atomic_trace_enabled, ReadRecords read_records +) { + if (!ValidateTraceHeader(header, "swimlane export")) return false; + if (workload_mode != WinnerWorkloadMode::ScalarNop && + workload_mode != WinnerWorkloadMode::RealCompute) { + std::fprintf(stderr, "swimlane export rejected invalid winner workload mode.\n"); + return false; + } + const bool real_compute = workload_mode == WinnerWorkloadMode::RealCompute; + const bool pattern_valid = workload_pattern != nullptr && + ((real_compute && + (std::strcmp(workload_pattern, "constant") == 0 || + std::strcmp(workload_pattern, "layout-diagnostic") == 0)) || + (!real_compute && std::strcmp(workload_pattern, "none") == 0)); + if (!pattern_valid) { + std::fprintf(stderr, "swimlane export rejected invalid winner workload input pattern.\n"); + return false; + } + + TraceExportSummary producer_summary; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const TraceCoreState &core = header.cores[worker]; + int32_t expected_block = -1; + int32_t expected_lane = -1; + ExpectedTraceTopology(worker, &expected_block, &expected_lane); + if (core.core_idx != static_cast(worker) || core.block_id != expected_block || + core.lane != expected_lane) { + std::fprintf( + stderr, + "swimlane export rejected worker topology: worker=%u core=%d block=%d/%d lane=%d/%d\n", + worker, core.core_idx, core.block_id, expected_block, core.lane, expected_lane + ); + return false; + } + producer_summary.records += core.count; + producer_summary.atomic_calls += core.atomic_calls; + producer_summary.poll_calls += core.poll_calls; + producer_summary.poll_batch_records += core.poll_batch_records; + producer_summary.dropped_records += core.dropped; + if (!atomic_trace_enabled) { + if (core.atomic_calls != 0 || core.poll_calls != 0 || core.poll_batch_records != 0) { + std::fprintf( + stderr, + "phase-only swimlane worker %u unexpectedly reports atomic counters: calls=%u polls=%u batches=%u\n", + worker, core.atomic_calls, core.poll_calls, core.poll_batch_records + ); + return false; + } + continue; + } + if (core.poll_calls > core.atomic_calls || + (core.poll_calls == 0) != (core.poll_batch_records == 0)) { + std::fprintf( + stderr, + "atomic swimlane worker %u has invalid counters: calls=%u polls=%u batches=%u\n", + worker, core.atomic_calls, core.poll_calls, core.poll_batch_records + ); + return false; + } + producer_summary.atomic_records += + static_cast(core.atomic_calls) - core.poll_calls + core.poll_batch_records; + } + producer_summary.clock_baseline_records = atomic_trace_enabled ? 2ULL * kWorkers : 0; + + // 先写同目录临时文件，全部记录写完并关闭后再 rename 替换，避免把半截 JSON + // 当成有效采集；这里没有 fsync 文件和目录，不承诺掉电后的持久化原子性。 + const std::string temporary_path = output_path + ".tmp"; + std::FILE *output = std::fopen(temporary_path.c_str(), "wb"); + if (output == nullptr) { + std::fprintf( + stderr, "Cannot open swimlane output %s: %s\n", temporary_path.c_str(), std::strerror(errno) + ); + return false; + } + + // 采用固定 1 MiB stdio 缓冲并逐核流式写出；默认 256 batch 时约 86 万条， + // 无论实际 batch 数是多少都不在 host 侧一次性聚合全部 JSON 记录。 + std::vector output_buffer(1U << 20); + std::setvbuf(output, output_buffer.data(), _IOFBF, output_buffer.size()); + std::fprintf( + output, + "{\n\"l2_swimlane_level\":%u,\n" + "\"metadata\":{\"clock_freq_hz\":%llu,\"num_cores\":%u," + "\"trace_schema_version\":%u," + "\"winner_workload\":{\"mode\":\"%s\"," + "\"counts\":{\"qk\":%u,\"sf\":%u,\"pv\":%u,\"up\":%u}," + "\"unit\":\"%s\",\"input_pattern\":\"%s\"," + "\"engine_mapping\":%s},\"core_types\":[", + atomic_trace_enabled ? 4U : 1U, + static_cast(header.frequency_hz), kWorkers, + 4U, + workload_mode == WinnerWorkloadMode::RealCompute ? "real-compute" : "scalar-nop", + workload_counts.qk, workload_counts.sf, workload_counts.pv, workload_counts.up, + workload_mode == WinnerWorkloadMode::RealCompute + ? "complete_128x128_engine_pipeline_iteration" + : "scalar_nop_instruction", + workload_pattern, + workload_mode == WinnerWorkloadMode::RealCompute + ? "{\"qk\":\"cube_matmul\",\"sf\":\"vector_add\"," + "\"pv\":\"cube_matmul\",\"up\":\"vector_mul\"}" + : "null" + ); + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + std::fprintf(output, "%s\"%s\"", worker == 0 ? "" : ",", worker < kAicWorkers ? "aic" : "aiv"); + } + // schema-v4 无论是否开启 atomic 都导出 producer summary；phase-only 的 + // atomic/clock 字段为零，离线分析仍可独立证明 records 与 dropped 闭合。 +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + std::fprintf( + output, + "],\"lazy_sample_shape\":{\"name\":\"%s\",\"id\":%u," + "\"observation\":\"%s\",\"finish\":\"%s\",\"control_family\":\"%s\"}," + "\"fdwic_summary\":{\"records\":%llu,\"atomic_records\":%llu," + "\"clock_baseline_records\":%llu,\"atomic_calls\":%llu," + "\"batched_poll_calls\":%llu,\"poll_batch_records\":%llu," + "\"dropped_records\":%llu}", + kLazySampleShapeName, kLazySampleShapeId, kLazySampleObservation, + kLazySampleFinishShape, kLazySampleControlFamily, + static_cast(producer_summary.records), + static_cast(producer_summary.atomic_records), + static_cast(producer_summary.clock_baseline_records), + static_cast(producer_summary.atomic_calls), + static_cast(producer_summary.poll_calls), + static_cast(producer_summary.poll_batch_records), + static_cast(producer_summary.dropped_records) + ); +#else + std::fprintf( + output, + "],\"fdwic_summary\":{\"records\":%llu,\"atomic_records\":%llu," + "\"clock_baseline_records\":%llu,\"atomic_calls\":%llu," + "\"batched_poll_calls\":%llu,\"poll_batch_records\":%llu," + "\"dropped_records\":%llu}", + static_cast(producer_summary.records), + static_cast(producer_summary.atomic_records), + static_cast(producer_summary.clock_baseline_records), + static_cast(producer_summary.atomic_calls), + static_cast(producer_summary.poll_calls), + static_cast(producer_summary.poll_batch_records), + static_cast(producer_summary.dropped_records) + ); +#endif + std::fprintf( + output, + "},\n\"aicore_tasks\":[],\n\"aicpu_tasks\":[],\n" + "\"aicpu_scheduler_phases\":[],\n\"aicpu_orchestrator_phases\":[],\n\"fdwic_events\":[\n" + ); + // fdwic_events 每行固定十列：core、block、lane、task、function、phase、起止周期、flags、aux。 + + bool success = true; + bool first_record = true; + uint64_t exported_records = 0; + TraceExportSummary observed_summary; + std::vector scratch(kTraceRecordsPerCore); + constexpr int32_t kTracePhaseCount = static_cast(TracePhase::Count); + for (uint32_t worker = 0; worker < kWorkers && success; ++worker) { + // 每次只读取一个 worker 的有效区间；完整 384 MiB trace 缓冲无需整体回拷。 + const uint32_t available = header.cores[worker].count; + if (available > header.records_per_core) { + std::fprintf( + stderr, "Trace core %u count %u exceeds capacity %u.\n", worker, available, + header.records_per_core + ); + success = false; + break; + } + if (available != 0 && !read_records(worker, available, scratch.data())) { + success = false; + break; + } + const TraceCoreState &core = header.cores[worker]; + uint64_t core_atomic_calls = 0; + uint64_t core_poll_calls = 0; + uint32_t core_atomic_records = 0; + uint32_t core_poll_batch_records = 0; + uint32_t core_clock_records = 0; + uint32_t core_plain_clock_records = 0; + uint32_t core_dependency_clock_records = 0; + bool dependency_applied = false; + bool direct_result_used_return_ready = false; + bool direct_result_used_source_issue = false; + for (uint32_t index = 0; index < available; ++index) { + const TraceRecord &record = scratch[index]; + const bool atomic_record = record.phase == static_cast(TracePhase::Atomic); + const bool claim_record = record.phase == static_cast(TracePhase::Claim); + const bool clock_record = record.phase == static_cast(TracePhase::ClockBaseline); + const bool atomic_schema_valid = !atomic_record || + AtomicRecordSchemaValid(record, atomic_trace_enabled); + const bool claim_schema_valid = !claim_record || + ((record.flags & ~(kClaimWon | kClaimAttempted)) == 0 && + ((record.flags & kClaimWon) == 0 || (record.flags & kClaimAttempted) != 0) && + record.auxiliary <= 1); + const bool clock_schema_valid = !clock_record || + (atomic_trace_enabled && ClockRecordSchemaValid(record)); + const bool record_valid = record.end_cycle >= record.start_cycle && record.phase >= 0 && + record.phase < kTracePhaseCount && record.task_id >= -1 && + record.function_id >= -1 && record.lane == core.lane && + record.block_id == core.block_id && + record.core_idx == core.core_idx && atomic_schema_valid && + claim_schema_valid && clock_schema_valid; + if (!record_valid) { + std::fprintf( + stderr, + "Invalid trace record at worker=%u index=%u: phase=%d lane=%d block=%d core=%d " + "start=%llu end=%llu flags=0x%08x aux=%u\n", + worker, index, record.phase, record.lane, record.block_id, record.core_idx, + static_cast(record.start_cycle), + static_cast(record.end_cycle), record.flags, record.auxiliary + ); + success = false; + break; + } + if (atomic_record) { + ++core_atomic_records; + const uint32_t call_count = AtomicRecordCallCount(record); + core_atomic_calls += call_count; + if ((record.flags & kAtomicPollBatch) != 0) { + core_poll_calls += call_count; + ++core_poll_batch_records; + } else if ((record.flags & kAtomicResultUsed) != 0) { + if ((record.flags & kAtomicReturnReady) != 0) { + direct_result_used_return_ready = true; + } else { + direct_result_used_source_issue = true; + } + } + } else if (clock_record) { + ++core_clock_records; + if ((record.flags & kClockAtomicDependency) != 0) { + ++core_dependency_clock_records; + dependency_applied = (record.flags & kClockAtomicDependencyApplied) != 0; + } else { + ++core_plain_clock_records; + } + } + std::fprintf( + output, + "%s[%d,%d,%d,%d,%d,\"%s\",%llu,%llu,%u,%u]", + first_record ? "" : ",\n", record.core_idx, record.block_id, record.lane, record.task_id, + record.function_id, TracePhaseName(static_cast(record.phase)), + static_cast(record.start_cycle), + static_cast(record.end_cycle), record.flags, record.auxiliary + ); + first_record = false; + ++exported_records; + } + if (!success) break; + bool core_closed = true; + if (atomic_trace_enabled) { + const uint64_t expected_atomic_records = + static_cast(core.atomic_calls) - core.poll_calls + core.poll_batch_records; + core_closed = core_atomic_records == expected_atomic_records && + core_atomic_calls == core.atomic_calls && core_poll_calls == core.poll_calls && + core_poll_batch_records == core.poll_batch_records && core_clock_records == 2 && + core_plain_clock_records == 1 && core_dependency_clock_records == 1 && + (!dependency_applied || !direct_result_used_source_issue) && + (dependency_applied || !direct_result_used_return_ready); + } else { + core_closed = core_atomic_records == 0 && core_atomic_calls == 0 && core_poll_calls == 0 && + core_poll_batch_records == 0 && core_clock_records == 0; + } + if (!core_closed) { + std::fprintf( + stderr, + "swimlane closure failed on worker=%u: physical_atomic=%u logical_atomic=%llu/%u " + "poll_calls=%llu/%u poll_batches=%u/%u clock=%u plain=%u dependency=%u " + "dependency_applied=%s direct_ready=%s direct_issue=%s\n", + worker, core_atomic_records, static_cast(core_atomic_calls), + core.atomic_calls, static_cast(core_poll_calls), core.poll_calls, + core_poll_batch_records, core.poll_batch_records, core_clock_records, + core_plain_clock_records, core_dependency_clock_records, + dependency_applied ? "yes" : "no", direct_result_used_return_ready ? "yes" : "no", + direct_result_used_source_issue ? "yes" : "no" + ); + success = false; + break; + } + observed_summary.records += available; + observed_summary.atomic_records += core_atomic_records; + observed_summary.clock_baseline_records += core_clock_records; + observed_summary.atomic_calls += core_atomic_calls; + observed_summary.poll_calls += core_poll_calls; + observed_summary.poll_batch_records += core_poll_batch_records; + observed_summary.dropped_records += core.dropped; + } + if (success && !SameTraceSummary(producer_summary, observed_summary)) { + std::fprintf( + stderr, + "swimlane producer/raw summary mismatch: records=%llu/%llu atomic_records=%llu/%llu " + "atomic_calls=%llu/%llu poll_calls=%llu/%llu poll_batches=%llu/%llu clock=%llu/%llu\n", + static_cast(observed_summary.records), + static_cast(producer_summary.records), + static_cast(observed_summary.atomic_records), + static_cast(producer_summary.atomic_records), + static_cast(observed_summary.atomic_calls), + static_cast(producer_summary.atomic_calls), + static_cast(observed_summary.poll_calls), + static_cast(producer_summary.poll_calls), + static_cast(observed_summary.poll_batch_records), + static_cast(producer_summary.poll_batch_records), + static_cast(observed_summary.clock_baseline_records), + static_cast(producer_summary.clock_baseline_records) + ); + success = false; + } + if (success) std::fprintf(output, "\n]}\n"); + if (std::ferror(output) != 0) { + std::fprintf(stderr, "Failed while writing swimlane output %s.\n", temporary_path.c_str()); + success = false; + } + if (std::fclose(output) != 0) { + std::fprintf(stderr, "Failed to close swimlane output %s: %s\n", temporary_path.c_str(), std::strerror(errno)); + success = false; + } + if (success && std::rename(temporary_path.c_str(), output_path.c_str()) != 0) { + std::fprintf( + stderr, "Cannot finalize swimlane output %s: %s\n", output_path.c_str(), std::strerror(errno) + ); + success = false; + } + if (!success) { + std::remove(temporary_path.c_str()); + return false; + } + std::printf( + "[SWIMLANE] raw_json=%s events=%llu\n", output_path.c_str(), + static_cast(exported_records) + ); + return true; +} + +template +inline bool AnalyzeSwimlaneRecords( + const TraceHeader &header, const SchedulerState &state, ReadRecords read_records +) { + if (!ValidateTraceHeader(header, "swimlane analysis")) return false; + + // 第一组数组统计“每个 worker 在某阶段的累计时间”；task_durations 则保留重点阶段的单事件分布。 + constexpr uint32_t kTracePhaseCount = static_cast(TracePhase::Count); + constexpr TracePhase kDetailedPhases[] = { + TracePhase::EfDrain, TracePhase::Materialize, TracePhase::Claim, TracePhase::Register, + }; + uint64_t cycles[kWorkers][kTracePhaseCount] = {}; + uint64_t counts[kWorkers][kTracePhaseCount] = {}; + std::vector task_durations[2][kTasksPerBatch][sizeof(kDetailedPhases) / sizeof(kDetailedPhases[0])]; + std::vector atomic_durations[2][static_cast(AtomicSite::Count)]; + uint64_t atomic_return_ready_counts[2][static_cast(AtomicSite::Count)] = {}; + std::vector atomic_poll_windows[2][static_cast(AtomicSite::Count)]; + uint64_t atomic_poll_calls[2][static_cast(AtomicSite::Count)] = {}; + std::vector clock_baselines[2]; + std::vector clock_dependency_baselines[2]; + uint64_t clock_dependency_applied[2] = {}; + std::vector scratch(kTraceRecordsPerCore); + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const uint32_t available = header.cores[worker].count; + const uint32_t count = std::min(available, header.records_per_core); + if (count != 0 && !read_records(worker, count, scratch.data())) { + return false; + } + for (uint32_t index = 0; index < count; ++index) { + const TraceRecord &record = scratch[index]; + if (record.phase < 0 || record.phase >= static_cast(kTracePhaseCount) || + record.end_cycle < record.start_cycle) { + // 分析器面对单条坏记录选择跳过；严格导出路径会直接拒绝，二者服务于不同诊断目的。 + continue; + } + const uint32_t phase = static_cast(record.phase); + const uint64_t duration = record.end_cycle - record.start_cycle; + const bool atomic_poll_batch = + record.phase == static_cast(TracePhase::Atomic) && + (record.flags & kAtomicPollBatch) != 0; + // PollBatch 的 duration 是一次等待 episode 的包络，允许夹着其他直接 + // atomic/调度代码；不能混入“Atomic 单次括号”的累计时间或分位数。 + if (!atomic_poll_batch) { + cycles[worker][phase] += duration; + ++counts[worker][phase]; + } + if (record.phase == static_cast(TracePhase::Atomic) && + record.auxiliary < static_cast(AtomicSite::Count)) { + const uint32_t role_index = + state.results[worker].role == static_cast(CoreRole::Aic) ? 0U : 1U; + if (atomic_poll_batch) { + atomic_poll_windows[role_index][record.auxiliary].push_back(duration); + atomic_poll_calls[role_index][record.auxiliary] += AtomicRecordCallCount(record); + } else { + atomic_durations[role_index][record.auxiliary].push_back(duration); + atomic_return_ready_counts[role_index][record.auxiliary] += + (record.flags & kAtomicReturnReady) != 0; + } + } + if (record.phase == static_cast(TracePhase::ClockBaseline)) { + const uint32_t role_index = + state.results[worker].role == static_cast(CoreRole::Aic) ? 0U : 1U; + if ((record.flags & kClockAtomicDependency) != 0) { + clock_dependency_baselines[role_index].push_back(duration); + clock_dependency_applied[role_index] += + (record.flags & kClockAtomicDependencyApplied) != 0; + } else { + clock_baselines[role_index].push_back(duration); + } + } + if (record.task_id >= 0) { + // task_id % 5 恰好对应 Alloc/QK/SF/PV/UP，这是固定 PA Case1 图的拓扑约束。 + const uint32_t role_index = + state.results[worker].role == static_cast(CoreRole::Aic) ? 0U : 1U; + const uint32_t kind = static_cast(record.task_id) % kTasksPerBatch; + for (uint32_t detail = 0; detail < sizeof(kDetailedPhases) / sizeof(kDetailedPhases[0]); ++detail) { + if (phase == static_cast(kDetailedPhases[detail])) { + task_durations[role_index][kind][detail].push_back(duration); + } + } + } + } + } + + const CoreRole roles[] = {CoreRole::Aic, CoreRole::Aiv}; + const char *role_names[] = {"AIC", "AIV"}; + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t phase = 0; phase < kTracePhaseCount; ++phase) { + std::vector role_cycles; + uint64_t record_count = 0; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + if (state.results[worker].role != static_cast(roles[role_index])) continue; + role_cycles.push_back(cycles[worker][phase]); + record_count += counts[worker][phase]; + } + const Uint64Distribution summary = SummarizeUint64(role_cycles); + std::printf( + "[TRACE_PHASE] role=%s phase=%s records=%llu accumulated_us_median=%.3f " + "accumulated_us_p95=%.3f accumulated_us_max=%.3f\n", + role_names[role_index], TracePhaseName(phase), + static_cast(record_count), summary.median / 1000.0, + static_cast(summary.p95) / 1000.0, + static_cast(summary.maximum) / 1000.0 + ); + } + } + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + const Uint64Distribution summary = SummarizeUint64(clock_baselines[role_index]); + if (!clock_baselines[role_index].empty()) { + std::printf( + "[TRACE_CLOCK] role=%s samples=%zu definition=consecutive-sys-cnt-reads " + "median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], clock_baselines[role_index].size(), summary.median, + static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + const Uint64Distribution dependency_summary = + SummarizeUint64(clock_dependency_baselines[role_index]); + if (!clock_dependency_baselines[role_index].empty()) { + std::printf( + "[TRACE_CLOCK] role=%s samples=%zu definition=atomic-return-dependency-hook " + "dependency_applied=%llu/%zu median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], clock_dependency_baselines[role_index].size(), + static_cast(clock_dependency_applied[role_index]), + clock_dependency_baselines[role_index].size(), dependency_summary.median, + static_cast(dependency_summary.p95), + static_cast(dependency_summary.maximum) + ); + } + } + // Atomic 只报告原始括号分布，不扣除计时底噪，也不把 total_cycles + // 解释成可与 Submit 墙钟直接相加的“atomic 占比”。return-ready 只表示 + // 本核可消费返回值，不表示其他核已经观察到更新。 + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t site = 0; site < static_cast(AtomicSite::Count); ++site) { + const std::vector &durations = atomic_durations[role_index][site]; + if (durations.empty()) continue; + const Uint64Distribution summary = SummarizeUint64(durations); + const AtomicOp op = AtomicSiteOp(static_cast(site)); + const uint64_t return_ready_count = atomic_return_ready_counts[role_index][site]; + const char *boundary = return_ready_count == durations.size() + ? "return-ready" + : (return_ready_count == 0 ? "source-issue" : "mixed"); + std::printf( + "[TRACE_ATOMIC] role=%s site=%s op=%s events=%zu boundary=%s " + "return_ready=%llu/%zu bracket_cycles_total=%llu median_ns=%.1f " + "p95_ns=%llu max_ns=%llu\n", + role_names[role_index], AtomicSiteName(site), AtomicOpName(static_cast(op)), + durations.size(), boundary, static_cast(return_ready_count), + durations.size(), static_cast(summary.total), summary.median, + static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + } + // 等待聚合只报告 episode 数、精确逻辑调用数与包络分布。window 不能除以 + // calls 当作单次 atomic latency，也不能与 Submit 墙钟直接相加。 + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t site = 0; site < static_cast(AtomicSite::Count); ++site) { + const std::vector &windows = atomic_poll_windows[role_index][site]; + if (windows.empty()) continue; + const Uint64Distribution summary = SummarizeUint64(windows); + std::printf( + "[TRACE_ATOMIC_POLL] role=%s site=%s op=%s episodes=%zu logical_calls=%llu " + "window_definition=wait-episode-envelope median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], AtomicSiteName(site), + AtomicOpName(static_cast(AtomicSiteOp(static_cast(site)))), + windows.size(), static_cast(atomic_poll_calls[role_index][site]), + summary.median, static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + } + const char *kind_names[] = {"Alloc", "QK", "SF", "PV", "UP"}; + // 单事件统计按 role 与 task kind 展开，可区分“该 role 真实参与”与“只回放前端”的成本。 + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t kind = 0; kind < kTasksPerBatch; ++kind) { + for (uint32_t detail = 0; detail < sizeof(kDetailedPhases) / sizeof(kDetailedPhases[0]); ++detail) { + const std::vector &durations = task_durations[role_index][kind][detail]; + const Uint64Distribution summary = SummarizeUint64(durations); + std::printf( + "[TRACE_TASK] role=%s kind=%s phase=%s events=%zu median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], kind_names[kind], + TracePhaseName(static_cast(kDetailedPhases[detail])), durations.size(), summary.median, + static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + } + } + return true; +} + +inline Metrics Validate( + const SchedulerState &state, uint32_t run, double host_us, const TraceHeader *trace_header = nullptr +) { + Metrics metrics; + // 每个 worker 都回放全部 task。Alloc 由 96 个 worker 全部执行 atomicMax Claim； + // 其余 kernel task 只有与 active role 匹配的 AIC 或 AIV 参与 Claim。 + const uint32_t batches = state.config.batches; + const uint32_t task_count = batches * kTasksPerBatch; + const uint64_t expected_submits = static_cast(kWorkers) * task_count; + const uint64_t expected_claims = + static_cast(batches) * (kWorkers + kAicWorkers + kAivWorkers + kAicWorkers + kAivWorkers); + // 上式依次对应 Alloc、QK、SF、PV、UP 的 active worker 数，默认 256 batch 时为 73728。 + + // 聚合量分为调度核心计数、kernel 分布、前端操作数和最终状态四组，便于定位语义偏差。 + uint64_t first_submit = UINT64_MAX; + uint64_t last_submit = 0; + uint64_t submits = 0; + uint64_t claims = 0; + uint64_t wins = 0; + uint64_t heap_guards = 0; + uint64_t fanin_ready_loads = 0; + uint64_t fanin_not_ready_loads = 0; + uint64_t frontier_initial_loads = 0; + uint64_t frontier_updates = 0; + uint64_t frontier_terminal_loads = 0; + uint64_t atomic_trace_calls = 0; + uint64_t duplicates = 0; + uint64_t cas_retries = 0; + uint64_t joint_polls = 0; + uint64_t trace_wait_records = 0; + uint64_t wins_by_kind[5] = {}; + uint64_t kernel_counts[4] = {}; + uint64_t kernel_cycles[4] = {}; + uint64_t kernel_min[4] = {}; + uint64_t kernel_max[4] = {}; + uint64_t placements[3] = {}; + uint64_t phase_calls[static_cast(ProfilePhase::Count)] = {}; + uint64_t context_reads = 0; + uint64_t views_created = 0; + uint64_t dynamic_create_infos = 0; + uint64_t arg_resets = 0; + uint64_t tensor_args_added = 0; + uint64_t scalar_args_added = 0; + uint64_t materialized_outputs = 0; + uint64_t map_inserts = 0; + uint64_t map_lookups = 0; + uint64_t slot_tensor_copies = 0; + uint64_t slot_scalar_copies = 0; + uint64_t fanin_edges = 0; + bool worker_ids[kWorkers] = {}; + uint32_t aic_count = 0; + uint32_t aiv_count = 0; + uint32_t winning_workers = 0; + uint64_t max_worker_wins = 0; + bool worker_shape_ok = true; + bool submit_timestamps_ok = true; + bool vend_values_ok = true; + bool frontend_worker_counts_ok = true; + bool final_worker_state_ok = true; + bool worker_checksums_ok = true; + bool fanin_worker_counts_ok = true; + bool frontier_worker_counts_ok = true; + bool role_kernel_routing_ok = true; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + bool lazy_sample_split_runtime_oracle_ok = true; + const uint64_t expected_lazy_sample_split_task_id_sum = + static_cast(task_count) * (task_count - 1U) / 2U; +#endif + + // 按真实输出大小、1 KiB 对齐和 256 MiB 环回规则重算每个 task 可接受的最小 vend。 + uint64_t expected_heap_next = 0; + bool vend_progress_bounds_ok = true; + uint32_t first_bad_vend = task_count; + uint64_t first_bad_vend_minimum = 0; + uint64_t first_bad_vend_actual = 0; + std::vector minimum_vends(task_count); + const uint64_t output_bytes_by_kind[] = {10240, 524288, 264192, 8192, 0}; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + const uint64_t output_bytes = output_bytes_by_kind[task_id % kTasksPerBatch]; + uint64_t task_base = (expected_heap_next + kOutputAlignment - 1) / kOutputAlignment * kOutputAlignment; + if (output_bytes != 0 && (task_base % state.heap_size) + output_bytes > state.heap_size) { + task_base = (task_base / state.heap_size + 1) * state.heap_size; + } + expected_heap_next = task_base + output_bytes; + minimum_vends[task_id] = expected_heap_next; + } + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + // vend 可以大于本 task 的最小末端，因为 winner 发布的是其本地 heap_cursor 快照； + // 但不能超过该 worker 完整回放所有 task 后的最终 heap_next。 + if (state.tasks[task_id].vend < minimum_vends[task_id] || + state.tasks[task_id].vend > expected_heap_next) { + vend_progress_bounds_ok = false; + if (first_bad_vend == task_count) { + first_bad_vend = task_id; + first_bad_vend_minimum = minimum_vends[task_id]; + first_bad_vend_actual = state.tasks[task_id].vend; + } + } + } + // TensorMap 只保留 heap window 内仍可能被依赖的四类输出；floor 对应已安全退休的 task 边界。 + const uint64_t expected_map_live = 4ULL * std::min(batches, 13); + const uint64_t expected_map_floor = task_count > kHeapWindow + 1 ? task_count - kHeapWindow - 1 : 0; + + for (uint32_t index = 0; index < kWorkers; ++index) { + // 每核只写自己独占且按 cache line 隔离的 WorkerResult；host 在 kernel 完成后统一汇总，不引入额外 atomic。 + const WorkerResult &result = state.results[index]; + if (result.worker_id < kWorkers) { + worker_ids[result.worker_id] = true; + } else { + worker_shape_ok = false; + } + aic_count += result.role == static_cast(CoreRole::Aic); + aiv_count += result.role == static_cast(CoreRole::Aiv); + worker_shape_ok &= result.submits == task_count; + worker_shape_ok &= result.max_occupied <= kUsableSlots; + worker_shape_ok &= result.final_occupied == 0; + submit_timestamps_ok &= result.submit_begin != 0; + submit_timestamps_ok &= result.submit_end >= result.submit_begin; + submit_timestamps_ok &= result.finish_cycle >= result.submit_end; + first_submit = std::min(first_submit, result.submit_begin); + last_submit = std::max(last_submit, result.submit_end); + submits += result.submits; + claims += result.claim_attempts; + wins += result.claim_wins; + if (result.claim_wins != 0) ++winning_workers; + max_worker_wins = std::max(max_worker_wins, result.claim_wins); + heap_guards += result.heap_guards; + fanin_ready_loads += result.fanin_ready_loads; + fanin_not_ready_loads += result.fanin_not_ready_loads; + frontier_initial_loads += result.frontier_initial_loads; + frontier_updates += result.frontier_updates; + frontier_terminal_loads += result.frontier_terminal_loads; + atomic_trace_calls += result.atomic_trace_calls; + duplicates += result.completion_duplicates; + cas_retries += result.cas_retries; + joint_polls += result.joint_polls; + trace_wait_records += result.wait_events[0] + result.wait_events[1]; + context_reads += result.context_reads; + views_created += result.views_created; + dynamic_create_infos += result.dynamic_create_infos; + arg_resets += result.arg_resets; + tensor_args_added += result.tensor_args_added; + scalar_args_added += result.scalar_args_added; + materialized_outputs += result.materialized_outputs; + map_inserts += result.map_inserts; + map_lookups += result.map_lookups; + slot_tensor_copies += result.slot_tensor_copies; + slot_scalar_copies += result.slot_scalar_copies; + fanin_edges += result.fanin_edges; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + const CoreRole expected_role = index < kAicWorkers ? CoreRole::Aic : CoreRole::Aiv; + lazy_sample_split_runtime_oracle_ok &= result.worker_id == index; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_caller_state_address != 0; + lazy_sample_split_runtime_oracle_ok &= + result.lazy_sample_split_finish_state_address == result.lazy_sample_split_caller_state_address; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_finish_calls == task_count; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_protocol_errors == 0; + lazy_sample_split_runtime_oracle_ok &= + result.lazy_sample_split_state_cookie == + (kLazySampleSplitStateCookieBase ^ static_cast(index) ^ + (static_cast(static_cast(expected_role)) << 32U)); + lazy_sample_split_runtime_oracle_ok &= + result.lazy_sample_split_task_id_sum == expected_lazy_sample_split_task_id_sum; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_owner_worker_id == index; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_reserved == 0; +#endif +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + const uint64_t qk_wins = result.wins[static_cast(TaskKind::Qk)]; + const uint64_t sf_wins = result.wins[static_cast(TaskKind::Sf)]; + const uint64_t pv_wins = result.wins[static_cast(TaskKind::Pv)]; + const uint64_t up_wins = result.wins[static_cast(TaskKind::Up)]; + const uint64_t expected_worker_views = kLazySampleLazy + ? static_cast(batches) + qk_wins + : static_cast(batches) * 2; + const uint64_t expected_worker_tensors = kLazySampleLazy + ? static_cast(batches) * 12 + + qk_wins * 3 + sf_wins + pv_wins * 3 + up_wins * 3 + : static_cast(batches) * 22; + const uint64_t expected_worker_scalars = kLazySampleLazy + ? qk_wins * 2 + sf_wins * 3 + pv_wins * 2 + up_wins * 2 + : static_cast(batches) * 9; + frontend_worker_counts_ok &= result.context_reads == batches; + frontend_worker_counts_ok &= result.views_created == expected_worker_views; + frontend_worker_counts_ok &= result.dynamic_create_infos == static_cast(batches) * 2; + // Alloc constructs once; QK/SF/PV/UP each reset once inside their callback. + frontend_worker_counts_ok &= result.arg_resets == static_cast(batches) * 4; + frontend_worker_counts_ok &= result.tensor_args_added == expected_worker_tensors; + frontend_worker_counts_ok &= result.scalar_args_added == expected_worker_scalars; + frontend_worker_counts_ok &= result.materialized_outputs == static_cast(batches) * 8; + frontend_worker_counts_ok &= result.map_inserts == static_cast(batches) * 4; +#else + frontend_worker_counts_ok &= result.context_reads == batches; + frontend_worker_counts_ok &= result.views_created == static_cast(batches) * 2; + frontend_worker_counts_ok &= result.dynamic_create_infos == static_cast(batches) * 2; + frontend_worker_counts_ok &= result.arg_resets == static_cast(batches) * 4; + frontend_worker_counts_ok &= result.tensor_args_added == static_cast(batches) * 22; + frontend_worker_counts_ok &= result.scalar_args_added == static_cast(batches) * 9; + frontend_worker_counts_ok &= result.materialized_outputs == static_cast(batches) * 8; + frontend_worker_counts_ok &= result.map_inserts == static_cast(batches) * 4; +#endif + final_worker_state_ok &= result.final_heap_next == expected_heap_next; + final_worker_state_ok &= result.map_high_water == expected_map_live; + final_worker_state_ok &= result.map_live_entries == expected_map_live; + final_worker_state_ok &= result.map_alive_floor == expected_map_floor; + final_worker_state_ok &= result.map_cleaned_upto == expected_map_floor; + worker_checksums_ok &= result.checksum == (0xcbf29ce484222325ULL ^ result.worker_id); + const uint64_t worker_kernel_completions = result.kernel_counts[0] + result.kernel_counts[1] + + result.kernel_counts[2] + result.kernel_counts[3]; + if (result.role == static_cast(CoreRole::Aic)) { + role_kernel_routing_ok &= result.kernel_counts[1] == 0 && result.kernel_counts[3] == 0; + } else if (result.role == static_cast(CoreRole::Aiv)) { + role_kernel_routing_ok &= result.kernel_counts[0] == 0 && result.kernel_counts[2] == 0; + } else { + role_kernel_routing_ok = false; + } + const uint64_t worker_completions = result.wins[0] + worker_kernel_completions; + frontier_worker_counts_ok &= result.frontier_initial_loads == worker_completions; + frontier_worker_counts_ok &= result.frontier_terminal_loads == result.frontier_initial_loads; + fanin_worker_counts_ok &= result.fanin_ready_loads >= result.fanin_edges; + if (result.fanin_ready_loads >= result.fanin_edges) { + // PA 最大 fanin 为 3；每次失败检查最多先重读两个 ready 前缀，再遇到一个 not-ready。 + fanin_worker_counts_ok &= + result.fanin_ready_loads - result.fanin_edges <= 2 * result.fanin_not_ready_loads; + } + for (uint32_t kind = 0; kind < 5; ++kind) + wins_by_kind[kind] += result.wins[kind]; + for (uint32_t kind = 0; kind < 4; ++kind) { + kernel_counts[kind] += result.kernel_counts[kind]; + kernel_cycles[kind] += result.kernel_cycles[kind]; + if (result.kernel_min_cycles[kind] != 0 && + (kernel_min[kind] == 0 || result.kernel_min_cycles[kind] < kernel_min[kind])) { + kernel_min[kind] = result.kernel_min_cycles[kind]; + } + kernel_max[kind] = std::max(kernel_max[kind], result.kernel_max_cycles[kind]); + } + for (uint32_t place = 0; place < 3; ++place) + placements[place] += result.placement[place]; + for (uint32_t phase = 0; phase < static_cast(ProfilePhase::Count); ++phase) + phase_calls[phase] += result.phase_calls[phase]; + } + for (bool seen : worker_ids) + worker_shape_ok &= seen; + + uint32_t ready_flags = 0; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + // ready flag 和 vend 是跨核 completion 的最终外部可见状态，不能只依赖 worker 私有计数判断完成。 + ready_flags += state.tasks[task_id].flag == 1; + vend_values_ok &= state.tasks[task_id].vend != 0; + vend_values_ok &= state.tasks[task_id].vend % kOutputAlignment == 0; + } + const uint64_t kernel_total = kernel_counts[0] + kernel_counts[1] + kernel_counts[2] + kernel_counts[3]; + const uint64_t placement_total = placements[0] + placements[1] + placements[2]; + const uint64_t fanin_loads = fanin_ready_loads + fanin_not_ready_loads; + const uint64_t frontier_flag_loads = frontier_updates + frontier_terminal_loads; + + // 第一组断言覆盖参与者拓扑、Claim/winner、completion 和最终 drain 等调度主协议。 + Expect(aic_count == kAicWorkers && aiv_count == kAivWorkers, "participant topology is 32 AIC + 64 AIV", &metrics); + Expect(worker_shape_ok, "all 96 worker markers and private rings are valid", &metrics); +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + Expect( + lazy_sample_split_runtime_oracle_ok, + "split caller/finish share one role-specific block-local state and finish every task once", + &metrics + ); +#endif + Expect(submit_timestamps_ok, "all Submit timing markers are valid", &metrics); + Expect(state.started_count.value == kWorkers, "started_count is 96", &metrics); + Expect(submits == expected_submits, "replay count is workers * tasks", &metrics); + Expect(claims == expected_claims, "Claim attempt count matches PA topology", &metrics); + Expect(wins == task_count, "exactly one winner per task", &metrics); + Expect( + wins_by_kind[0] == batches && wins_by_kind[1] == batches && wins_by_kind[2] == batches && + wins_by_kind[3] == batches && wins_by_kind[4] == batches, + "Alloc/QK/SF/PV/UP winners are one per batch", &metrics + ); + Expect(kernel_total == static_cast(batches) * 4, "kernel count is four per batch", &metrics); + Expect( + kernel_counts[0] == batches && kernel_counts[1] == batches && kernel_counts[2] == batches && + kernel_counts[3] == batches, + "each kernel kind executes once per batch", &metrics + ); + Expect( + role_kernel_routing_ok, + "AIC executes only QK/PV and AIV executes only SF/UP", &metrics + ); + Expect(heap_guards == static_cast(batches) * 4, "heap guard count matches output winners", &metrics); + Expect( + fanin_worker_counts_ok && fanin_ready_loads >= fanin_edges && + fanin_ready_loads - fanin_edges <= 2 * fanin_not_ready_loads, + "fanin ready/failure load classification is complete", &metrics + ); + Expect( + frontier_worker_counts_ok && frontier_initial_loads == task_count, + "frontier initial loads match completed tasks", &metrics + ); + Expect( + frontier_terminal_loads == task_count && frontier_updates >= task_count, + "frontier ready/update/terminal load identity is exact", &metrics + ); + Expect(duplicates == 0, "completion flags are published once", &metrics); + Expect(ready_flags == task_count, "all task flags are ready", &metrics); + Expect(vend_values_ok, "all published vend values are nonzero and aligned", &metrics); + Expect(vend_progress_bounds_ok, "every task vend is within PA worker heap progress bounds", &metrics); + Expect(state.frontier.value == static_cast(task_count) - 1, "frontier reaches the final task", &metrics); + Expect(state.replay_done.value == kWorkers, "replay_done is 96", &metrics); + Expect(state.fatal.value == 0, "fatal remains clear", &metrics); + Expect(placement_total == kernel_total, "EfDrain + RingBp + final placement covers every kernel", &metrics); + // joint_polls 是为未来 BlockWon 模拟预留的结果字段，当前调度路径没有递增点； + // 此断言只确认现有输出保持零，不能单独证明 active_count>=2 分支不可达。 + Expect(joint_polls == 0, "single-lane PA performs no BlockWon polling", &metrics); + // 第二组断言锁定 scalar 前端工作量，防止编译器优化或后续改动悄悄删掉 PA 模拟步骤。 + Expect(frontend_worker_counts_ok, "every worker replays the exact PA frontend operation counts", &metrics); +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + const uint64_t expected_global_views = static_cast(batches) * + (kLazySampleLazy ? kWorkers + 1 : kWorkers * 2); + const uint64_t expected_global_tensors = static_cast(batches) * + (kLazySampleLazy ? kWorkers * 12 + 10 : kWorkers * 22); + const uint64_t expected_global_scalars = static_cast(batches) * + (kLazySampleLazy ? 9 : kWorkers * 9); + Expect( + context_reads == static_cast(kWorkers) * batches && + views_created == expected_global_views && + dynamic_create_infos == static_cast(kWorkers) * batches * 2 && + arg_resets == static_cast(kWorkers) * batches * 4 && + tensor_args_added == expected_global_tensors && + scalar_args_added == expected_global_scalars && + materialized_outputs == static_cast(kWorkers) * batches * 8 && + map_inserts == static_cast(kWorkers) * batches * 4, + "global PA frontend operation totals are exact", &metrics + ); + std::printf( + "[LAZY_SAMPLE_FRONTEND] shape=%s views=%llu/%llu tensor_args=%llu/%llu " + "scalar_args=%llu/%llu resets=%llu/%llu\n", + kLazySampleShapeName, + static_cast(views_created), + static_cast(expected_global_views), + static_cast(tensor_args_added), + static_cast(expected_global_tensors), + static_cast(scalar_args_added), + static_cast(expected_global_scalars), + static_cast(arg_resets), + static_cast(static_cast(kWorkers) * batches * 4) + ); +#else + Expect( + context_reads == static_cast(kWorkers) * batches && + views_created == static_cast(kWorkers) * batches * 2 && + dynamic_create_infos == static_cast(kWorkers) * batches * 2 && + arg_resets == static_cast(kWorkers) * batches * 4 && + tensor_args_added == static_cast(kWorkers) * batches * 22 && + scalar_args_added == static_cast(kWorkers) * batches * 9 && + materialized_outputs == static_cast(kWorkers) * batches * 8 && + map_inserts == static_cast(kWorkers) * batches * 4, + "global PA frontend operation totals are exact", &metrics + ); +#endif + Expect( + map_lookups == static_cast(batches) * 14 && + slot_tensor_copies == static_cast(batches) * 19 && + slot_scalar_copies == static_cast(batches) * 9 && + fanin_edges == static_cast(batches) * 5, + "winner-only map, slot-copy, and fanin totals are exact", &metrics + ); + Expect(final_worker_state_ok, "every worker final heap and TensorMap state is exact", &metrics); + Expect(worker_checksums_ok, "all frontend registration checksums remain clean", &metrics); + + // 三类 Claim cursor 各有四个 shard；按 task_id 重新推导每个 shard 应停留的最后任务。 + int64_t expected_cube[kCursorShards] = {-1, -1, -1, -1}; + int64_t expected_vector[kCursorShards] = {-1, -1, -1, -1}; + int64_t expected_alloc[kCursorShards] = {-1, -1, -1, -1}; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + const TaskKind kind = static_cast(task_id % kTasksPerBatch); + int64_t *cursors = kind == TaskKind::Alloc + ? expected_alloc + : (kind == TaskKind::Qk || kind == TaskKind::Pv ? expected_cube : expected_vector); + cursors[task_id % kCursorShards] = task_id; + } + bool cursors_ok = true; + for (uint32_t shard = 0; shard < kCursorShards; ++shard) { + cursors_ok &= state.cube_cursor[shard].value == expected_cube[shard]; + cursors_ok &= state.vector_cursor[shard].value == expected_vector[shard]; + cursors_ok &= state.alloc_cursor[shard].value == expected_alloc[shard]; + } + Expect(cursors_ok, "all sharded Claim cursors reach their exact final task", &metrics); + + if (state.config.profile_phases != 0) { + // profile 开关关闭时这些字段允许保持零，避免把可选诊断本身变成语义门禁。 + Expect( + phase_calls[static_cast(ProfilePhase::Claim)] == expected_submits && + phase_calls[static_cast(ProfilePhase::EfDrain)] == expected_submits && + phase_calls[static_cast(ProfilePhase::WaitForSlot)] == + static_cast(batches) * 4 && + phase_calls[static_cast(ProfilePhase::HeapGuard)] == + static_cast(batches) * 4, + "profile call counts match Claim/EfDrain/WaitForSlot/HeapGuard flow", &metrics + ); + } + + if (state.config.trace_enabled != 0) { + // 固定阶段记录数加上动态等待记录数，应与所有 worker 的 header count 精确相等。 + bool trace_shape_ok = trace_header != nullptr; + uint64_t trace_records = 0; + uint64_t trace_dropped = 0; + uint64_t physical_atomic_records = 0; + uint64_t batched_poll_calls = 0; + uint64_t poll_batch_records = 0; + bool per_worker_trace_counts_ok = true; + if (trace_header != nullptr) { + trace_shape_ok &= trace_header->magic == 0x4653574cU; + trace_shape_ok &= trace_header->version == 4; + trace_shape_ok &= trace_header->num_cores == kWorkers; + trace_shape_ok &= trace_header->records_per_core == kTraceRecordsPerCore; + trace_shape_ok &= trace_header->frequency_hz == kSystemCounterHz; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const TraceCoreState &core = trace_header->cores[worker]; + trace_records += core.count; + trace_dropped += core.dropped; + trace_shape_ok &= core.count <= kTraceRecordsPerCore; + int32_t expected_block = -1; + int32_t expected_lane = -1; + ExpectedTraceTopology(worker, &expected_block, &expected_lane); + trace_shape_ok &= core.core_idx == static_cast(worker); + trace_shape_ok &= core.block_id == expected_block; + trace_shape_ok &= core.lane == expected_lane; + const WorkerResult &result = state.results[worker]; + const uint64_t worker_kernels = result.kernel_counts[0] + result.kernel_counts[1] + + result.kernel_counts[2] + result.kernel_counts[3]; + uint64_t worker_physical_atomic = 0; + if ((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) { + trace_shape_ok &= core.atomic_calls == result.atomic_trace_calls; + trace_shape_ok &= core.poll_calls <= core.atomic_calls; + trace_shape_ok &= (core.poll_calls == 0) == (core.poll_batch_records == 0); + worker_physical_atomic = + static_cast(core.atomic_calls) - core.poll_calls + core.poll_batch_records; + physical_atomic_records += worker_physical_atomic; + batched_poll_calls += core.poll_calls; + poll_batch_records += core.poll_batch_records; + } else { + trace_shape_ok &= core.atomic_calls == 0 && core.poll_calls == 0 && + core.poll_batch_records == 0; + } + const uint64_t worker_expected = + 6 * result.submits + 2 * result.claim_wins - result.wins[0] + + 2 * worker_kernels + result.wait_events[0] + result.wait_events[1] + 2 + + (((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) + ? worker_physical_atomic + 2 + : 0); + per_worker_trace_counts_ok &= core.count == worker_expected; + } + } + const uint64_t expected_trace_records = + static_cast(batches) * (static_cast(kWorkers) * 30 + 17) + + trace_wait_records + 2 * kWorkers + + (((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) + ? physical_atomic_records + 2 * kWorkers + : 0); + // 每 batch 的 96*30 是六条每 Submit 固定记录；17 条是 + // 5 条 winner tail、4 条 Fanin 和 8 条 Kernel/Commit。loser 不再写 + // 零时长 marker。两个父 span 再各核固定增加 2 条； + // RingBp 等真实等待按运行时次数额外加入。 + Expect(trace_shape_ok, "swimlane header and per-worker capacities are valid", &metrics); + Expect(trace_dropped == 0, "swimlane records fit without drops", &metrics); + Expect(trace_records == expected_trace_records, "swimlane record count matches PA phase flow", &metrics); + Expect(per_worker_trace_counts_ok, "every worker swimlane record count is exact", &metrics); + if ((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) { + Expect(atomic_trace_calls != 0, "atomic trace captured source-level calls", &metrics); + } else { + Expect(atomic_trace_calls == 0, "atomic trace counters stay zero when disabled", &metrics); + } + std::printf( + "[TRACE] records=%llu expected=%llu dropped=%llu bytes=%zu\n", + static_cast(trace_records), + static_cast(expected_trace_records), + static_cast(trace_dropped), kTraceBytes + ); + std::printf( + "[ATOMIC_TRACE] enabled=%s logical_calls=%llu physical_records=%llu " + "batched_poll_calls=%llu poll_batch_records=%llu " + "closure=physical=logical-batched+batch_records\n", + (state.config.trace_enabled & kTraceAtomicsEnabled) != 0 ? "yes" : "no", + static_cast(atomic_trace_calls), + static_cast(physical_atomic_records), + static_cast(batched_poll_calls), + static_cast(poll_batch_records) + ); + } + + if (first_submit != UINT64_MAX && last_submit >= first_submit) { + // 性能口径只覆盖最早 Submit.begin 到最晚 Submit.end，不含启动屏障、最终 drain 和 host 同步。 + metrics.submit_span_us = static_cast(last_submit - first_submit) / 1000.0; + } + std::printf( + "[METRIC] run=%u submit_span_us=%.3f host_launch_us=%.3f claims=%llu fanin_loads=%llu cas_retries=%llu\n", run, + metrics.submit_span_us, host_us, static_cast(claims), + static_cast(fanin_loads), static_cast(cas_retries) + ); + const uint64_t submit_completion_ops = + claims + heap_guards + fanin_loads + 2ULL * task_count + frontier_initial_loads + + frontier_flag_loads + frontier_updates; + std::printf( + "[ATOMIC] submit_completion_ops=%llu fanin_ready=%llu fanin_not_ready=%llu frontier_initial=%llu " + "frontier_flag=%llu frontier_ready_fetch_max=%llu frontier_terminal=%llu\n", + static_cast(submit_completion_ops), + static_cast(fanin_ready_loads), + static_cast(fanin_not_ready_loads), + static_cast(frontier_initial_loads), + static_cast(frontier_flag_loads), + static_cast(frontier_updates), + static_cast(frontier_terminal_loads) + ); + std::printf( + "[WINNERS] active_workers=%u max_wins_per_worker=%llu\n", winning_workers, + static_cast(max_worker_wins) + ); + std::printf( + "[PLACEMENT] EfDrain=%llu RingBp=%llu FinalDrain=%llu\n", + static_cast(placements[static_cast(DrainPlace::EfDrain)]), + static_cast(placements[static_cast(DrainPlace::RingBackpressure)]), + static_cast(placements[static_cast(DrainPlace::FinalDrain)]) + ); + // placement 统计回答 kernel 最终在哪个 drain 点执行，与 TracePhase 的累计 span 互补。 + const char *kernel_names[] = {"QK", "SF", "PV", "UP"}; + const uint32_t targets[] = {kTargetQkTicks, kTargetSfTicks, kTargetPvTicks, kTargetUpTicks}; + for (uint32_t kind = 0; kind < 4; ++kind) { + const double mean = + kernel_counts[kind] == 0 ? 0.0 : static_cast(kernel_cycles[kind]) / kernel_counts[kind]; + std::printf( + "[KERNEL] %-2s count=%llu mean_us=%.3f min_us=%.3f max_us=%.3f target_us=%.3f\n", kernel_names[kind], + static_cast(kernel_counts[kind]), mean / 1000.0, kernel_min[kind] / 1000.0, + kernel_max[kind] / 1000.0, targets[kind] / 1000.0 + ); + } + PrintPhaseDiagnostics(state); + if (!metrics.passed) { + // 失败时补充第一处未完成 task、vend 边界和 worker 进度，避免只有笼统的 ASSERT FAIL。 + uint32_t first_not_ready = task_count; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + if (state.tasks[task_id].flag != 1) { + first_not_ready = task_id; + break; + } + } + uint64_t min_worker_submits = UINT64_MAX; + uint64_t max_worker_submits = 0; + uint32_t incomplete_workers = 0; + uint32_t occupied_workers = 0; + uint64_t max_final_occupied = 0; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const WorkerResult &result = state.results[worker]; + min_worker_submits = std::min(min_worker_submits, result.submits); + max_worker_submits = std::max(max_worker_submits, result.submits); + incomplete_workers += result.submits != task_count; + occupied_workers += result.final_occupied != 0; + max_final_occupied = std::max(max_final_occupied, result.final_occupied); + } + const int64_t retire = state.frontier.value - static_cast(kHeapWindow); + const uint64_t retire_vend = + retire >= 0 && retire < static_cast(task_count) ? state.tasks[retire].vend : 0; + std::printf( + "[FAILURE_STATE] fatal=%d frontier=%lld first_not_ready=%u first_bad_vend=%u " + "vend_minimum=%llu vend_actual=%llu retire=%lld retire_vend=%llu " + "worker_submits_min=%llu worker_submits_max=%llu incomplete_workers=%u " + "final_occupied_workers=%u max_final_occupied=%llu\n", + state.fatal.value, static_cast(state.frontier.value), first_not_ready, first_bad_vend, + static_cast(first_bad_vend_minimum), + static_cast(first_bad_vend_actual), + static_cast(retire), static_cast(retire_vend), + static_cast(min_worker_submits), + static_cast(max_worker_submits), incomplete_workers, occupied_workers, + static_cast(max_final_occupied) + ); + } + return metrics; +} + +inline double Median(std::vector values) { + // 多轮 benchmark 只报告中位数；上板基线比较仍应优先采用独立进程首轮。 + std::sort(values.begin(), values.end()); + const size_t middle = values.size() / 2; + if ((values.size() & 1U) != 0) return values[middle]; + return (values[middle - 1] + values[middle]) / 2.0; +} + +inline void PrintBanner(const char *backend, const Options &options) { + // 开始运行前完整打印工作量和大内存占用，便于确认比较口径没有混用。 + std::printf("=== Standalone PA Scheduler Benchmark: %s ===\n", backend); + std::printf( + "device=%u batches=%u tasks=%u workers=%u runs=%u nops=%u,%u,%u,%u state_bytes=%zu " + "swimlane=%s trace_atomics=%s trace_bytes=%zu\n", options.device, + options.batches, options.batches * kTasksPerBatch, kWorkers, options.runs, options.nops.qk, options.nops.sf, + options.nops.pv, options.nops.up, sizeof(SchedulerState), options.trace_enabled ? "on" : "off", + options.trace_atomics ? "on" : "off", + options.trace_enabled ? kTraceBytes : 0 + ); +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + std::printf( + "lazy_sample_shape=%s lazy_sample_shape_id=%u observation=%s finish=%s " + "control_family=%s\n", + kLazySampleShapeName, kLazySampleShapeId, kLazySampleObservation, + kLazySampleFinishShape, kLazySampleControlFamily + ); +#endif + if (!options.swimlane_json.empty()) { + std::printf("swimlane_json=%s\n", options.swimlane_json.c_str()); + } +} + +} // namespace pa_scheduler::host + +#endif // PA_SCHEDULER_COMMON_HOST_SUPPORT_H diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/pa_frontend.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/pa_frontend.h new file mode 100644 index 0000000000..f2cc8fcabb --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/pa_frontend.h @@ -0,0 +1,1308 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_FRONTEND_H +#define PA_SCHEDULER_COMMON_PA_FRONTEND_H + +#include "pa_model.h" + +namespace pa_scheduler { + +// 这些基址只充当稳定的 tensor identity，供 descriptor、区间重叠和 heap 地址 +// 计算使用；winner workload 不解引用这些 synthetic 地址，real-compute 使用独立 +// workspace。context_lens 是唯一按真实 GM 指针读取的 PA 前端输入。 +constexpr uint64_t kInvalidTaskId = UINT64_MAX; +constexpr uint64_t kSyntheticQueryBase = 0x200000000ULL; +constexpr uint64_t kSyntheticKeyBase = 0x300000000ULL; +constexpr uint64_t kSyntheticValueBase = 0x400000000ULL; +constexpr uint64_t kSyntheticBlockTableBase = 0x500000000ULL; +constexpr uint64_t kSyntheticOutputBase = 0x600000000ULL; +constexpr uint64_t kSyntheticContextLensBase = 0x700000000ULL; +constexpr uint32_t kPaHeads = 16; +constexpr uint32_t kPaHeadDim = 128; +constexpr uint32_t kPaBlockSize = 128; +constexpr uint32_t kPaBlocksPerRequest = 64; +constexpr uint32_t kPaMaxBlocksPerRequest = 256; +constexpr uint64_t kPaScaleBits = 0x3F800000ULL; +constexpr uint32_t kSpmdLocalContextIndex = kMaxTaskTensors + kMaxTaskScalars; +constexpr uint32_t kSpmdGlobalContextIndex = kSpmdLocalContextIndex + 1; +static_assert(kMaxTaskTensors == 32, "PA frontend requires the real 32 tensor slots"); +static_assert(kMaxTaskScalars == 16, "PA frontend requires the real 16 scalar slots"); +static_assert(kSpmdLocalContextIndex == 48, "PA local-context dispatch index mismatch"); +static_assert(kSpmdGlobalContextIndex == 49, "PA global-context dispatch index mismatch"); +static_assert(kMaxFanin == 16, "PA frontend requires the real 16 fanin slots"); + +enum class TensorRefKind : uint8_t { + LocalTensor = 0, + GmTensor = 1, + CreateInfo = 2, +}; + +// TaskArgs 同时容纳 orchestration 栈上的 descriptor、GM 中已物化的 descriptor， +// 以及尚待 Materialize 的 CreateInfo。显式 kind 保留生产 TensorRef 的地址空间分支。 +union TensorPointer { + const TensorDesc *local_tensor; + PA_GM const TensorDesc *gm_tensor; + const TensorCreateInfo *create_info; +}; + +struct TaskTensorRef { + TensorPointer pointer; + TensorRefKind kind; +}; +static_assert(sizeof(TaskTensorRef) == 16, "TaskTensorRef must match the PA TensorRef ABI"); +static_assert(offsetof(TaskTensorRef, pointer) == 0, "TaskTensorRef pointer offset mismatch"); +static_assert(offsetof(TaskTensorRef, kind) == 8, "TaskTensorRef kind offset mismatch"); + +struct PaLaunchSpec { + int16_t core_num; + bool require_sync_start; +}; +static_assert(sizeof(PaLaunchSpec) == 4, "PA launch spec ABI mismatch"); + +struct PaAsyncContext { + uint64_t completion_count; + uint64_t completion_error_code; + uint64_t completion_entries; + uint32_t completion_capacity; + uint32_t alignment_padding; + uint64_t task_token; +}; +static_assert(sizeof(PaAsyncContext) == 40, "PA async context ABI mismatch"); + +struct PaLocalContext { + int32_t block_index; + int32_t block_count; + PaAsyncContext async; +}; +// Local/GlobalContext 最终放进 RingSlot 的固定 dispatch 参数位 48/49；它们不是 +// standalone 自定义参数，offset 必须与真实 SPMD kernel 调用约定一致。 +static_assert(sizeof(PaLocalContext) == 48, "PA local context ABI mismatch"); + +struct PaGlobalContext { + int32_t sub_block_id; +}; +static_assert(sizeof(PaGlobalContext) == 4, "PA global context ABI mismatch"); + +// PTO2 profiling is enabled in the PA baseline. reset() clears all 160 bytes +// below on every QK/SF/PV/UP argument rebuild, even though Case1 does not ask +// to dump an argument. Keeping this storage and write stream matters to the +// spacing between consecutive Claim operations. +// 这段看似未使用的清零属于真实前端成本，删除会改变各 worker 到达 +// Claim 的波形与竞争强度，因此仍按生产构造/reset 顺序执行。 +struct PaDumpArgSelection { + uint64_t dump_arg_mask; + uint64_t dump_arg_index_ambiguous_mask; + uint64_t scalar_source_ptrs[kMaxTaskScalars]; + uint8_t scalar_dtypes[kMaxTaskScalars]; +}; +static_assert(sizeof(PaDumpArgSelection) == 160, "PA dump-selection ABI mismatch"); + +struct TaskArgs { + // The real TaskArgsTpl inherits its tag mixin first. TensorArgType is an + // int32 enum in the PA ABI; keeping tags first also reproduces its offsets. + // tag 数组位于对象首部不是任意排布；Materialize、fanin 与 register + // 都会重复扫描/复用这些 tag，错误 offset 会同时改变语义和前端访存成本。 + int32_t tags[kMaxTaskTensors]; + TaskTensorRef tensors[kMaxTaskTensors]; + uint64_t scalars[kMaxTaskScalars]; + int32_t tensor_count; + int32_t scalar_count; + + bool has_error; + uint64_t error_msg; + PaLaunchSpec launch_spec; + PaDumpArgSelection dump_arg_selection; + uint64_t explicit_deps; + uint32_t explicit_dep_count; + uint8_t cacheline_pad[48]; +}; +static_assert(sizeof(TaskArgs) == 1024, "TaskArgs must match the PA L0TaskArgs ABI size"); +static_assert(offsetof(TaskArgs, tags) == 0, "TaskArgs tag offset mismatch"); +static_assert(offsetof(TaskArgs, tensors) == 128, "TaskArgs tensor-ref offset mismatch"); +static_assert(offsetof(TaskArgs, scalars) == 640, "TaskArgs scalar offset mismatch"); +static_assert(offsetof(TaskArgs, tensor_count) == 768, "TaskArgs tensor-count offset mismatch"); +static_assert(offsetof(TaskArgs, scalar_count) == 772, "TaskArgs scalar-count offset mismatch"); +static_assert(offsetof(TaskArgs, has_error) == 776, "TaskArgs error flag offset mismatch"); +static_assert(offsetof(TaskArgs, error_msg) == 784, "TaskArgs error pointer offset mismatch"); +static_assert(offsetof(TaskArgs, launch_spec) == 792, "TaskArgs launch-spec offset mismatch"); +static_assert(offsetof(TaskArgs, dump_arg_selection) == 800, "TaskArgs dump-selection offset mismatch"); +static_assert(offsetof(TaskArgs, explicit_deps) == 960, "TaskArgs dependency pointer offset mismatch"); +static_assert(offsetof(TaskArgs, explicit_dep_count) == 968, "TaskArgs dependency count offset mismatch"); + +struct TaskOutputs { + uint64_t task_id; + uint32_t count; + PA_GM TensorDesc *tensors[kMaxTaskTensors]; +}; +static_assert(sizeof(TaskOutputs) == 272, "TaskOutputs must match the PA TaskOutputTensors ABI size"); +static_assert(offsetof(TaskOutputs, tensors) == 16, "TaskOutputs tensor pointer offset mismatch"); + +struct SubmitContext { + PA_GM WorkerState *self; + PA_GM TaskPayload *payload; + int32_t task_id; + int32_t tensor_count; + int32_t scalar_count; + uint32_t register_mask; + uint64_t output_bytes; + TaskOutputs result; + int32_t fanin[kMaxFanin]; + int32_t fanin_count; + int32_t kernel_id; + bool won; + bool joint; + bool joint_init; + int32_t joint_block; + int32_t joint_slot; + int32_t joint_count; +}; +// SubmitContext 贯穿一次 Submit：Begin 绑定 task/payload，Materialize 填充输出与 +// register_mask，winner 收集 fanin 并构建 slot。它复刻 DistSubmitCtx 而非诊断结构。 +static_assert(sizeof(SubmitContext) == 400, "SubmitContext must match DistSubmitCtx"); +static_assert(offsetof(SubmitContext, output_bytes) == 32, "SubmitContext output-byte offset mismatch"); +static_assert(offsetof(SubmitContext, result) == 40, "SubmitContext result offset mismatch"); +static_assert(offsetof(SubmitContext, fanin) == 312, "SubmitContext fanin offset mismatch"); + +struct OutputLayout { + uint64_t buffer_sizes[kMaxTaskTensors]; + uint64_t total_output_size; +}; +// 只有 tag=Output 的槽位拥有有效 buffer_sizes；总大小按 1 KiB 对齐累计，随后 +// 作为 HeapGuard 的 output_bytes 和本 worker heap_next 的推进量。 +static_assert(sizeof(OutputLayout) == 264, "OutputLayout must match DistOutputLayout"); + +// 该状态保存真实 PA orchestration 在五个 Submit 之间传递的 descriptor。输出指针 +// 指向每个 worker 自己 materialize 的 payload，不能跨 worker 共享或简化为全局对象。 +struct PaOrchestrationState { + TensorDesc query; + TensorDesc key_cache; + TensorDesc value_cache; + TensorDesc block_table; + TensorDesc context_lens; + TensorDesc output; + TensorDesc query_view; + TensorDesc output_view; + + TensorCreateInfo tile_create_info; + TensorCreateInfo scalar_create_info; + TensorCreateInfo qk_create_info; + TensorCreateInfo sf_create_info; + + // The pointer is supplied by the standalone backend. On A5 it must point + // at GM so every batch performs the same descriptor-based load as PA. + PA_GM const volatile int32_t *context_lens_data; + uint64_t scale_bits; + uint64_t current_sequence; + uint64_t current_blocks; + uint64_t current_block_offset; + uint64_t current_nblocks; + uint64_t current_valid_len; + uint32_t current_batch; + + PA_GM TensorDesc *accumulated_output; + PA_GM TensorDesc *accumulated_sum; + PA_GM TensorDesc *accumulated_max; + PA_GM TensorDesc *qk_scores; + PA_GM TensorDesc *sf_probs; + PA_GM TensorDesc *sf_max; + PA_GM TensorDesc *sf_sum; + PA_GM TensorDesc *pv_output; +}; + +PA_DEVICE uint64_t ElementSize(DataType dtype) { + // 输入 dtype 来自已通过 PA ABI 构造的 descriptor/create-info，必须落在 Count 前； + // 输出字节数同时用于外部 tensor range 与新 Output 的 heap 大小计算。 + constexpr static uint64_t sizes[static_cast(DataType::Count)] = { + 4, 2, 4, 2, 1, 1, 2, 8, 8, 2, 4, 1, + }; + return sizes[static_cast(dtype)]; +} + +PA_DEVICE int32_t TagValue(TensorArgType tag) { return static_cast(tag); } + +PA_DEVICE TensorArgType TaskTag(const TaskArgs &args, uint32_t index) { + // index 的有效范围由 tensor_count 保证；集中转换避免各阶段对 int32 ABI tag + // 做不同解释，Materialize/CollectFanin/Register 因而共享同一分类结果。 + return static_cast(args.tags[index]); +} + +PA_DEVICE void ClearDumpArgSelection(PaDumpArgSelection &selection) { + // Volatile stores intentionally preserve the profiling-enabled PA reset + // traffic even though the standalone winner workload never consumes dump data. + // volatile 的目的不是同步，而是阻止编译器删掉这段生产基线中存在的写流量。 + volatile uint64_t *masks = &selection.dump_arg_mask; + masks[0] = 0; + masks[1] = 0; + volatile uint64_t *sources = &selection.scalar_source_ptrs[0]; + for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { + sources[index] = 0; + } + volatile uint8_t *dtypes = &selection.scalar_dtypes[0]; + for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { + dtypes[index] = 0; + } +} + +PA_DEVICE void ConstructTaskArgs(TaskArgs &args) { + // TensorTagMixin::tags_{} is value-initialized by the + // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. + // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar + // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 + volatile int32_t *tags = &args.tags[0]; + for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { + tags[index] = 0; + } + args.tensor_count = 0; + args.scalar_count = 0; + args.has_error = false; + args.error_msg = 0; + args.launch_spec.core_num = 1; + args.launch_spec.require_sync_start = false; + ClearDumpArgSelection(args.dump_arg_selection); + args.explicit_deps = 0; + args.explicit_dep_count = 0; +} + +PA_DEVICE void ResetTaskArgs(TaskArgs &args) { + // reset 的输出是不含 tensor/scalar/显式依赖的新逻辑参数表，但保留已分配对象及 + // launch_spec；QK/SF/PV/UP 在同一个 1 KiB TaskArgs 上依次复用这一状态。 + args.tensor_count = 0; + args.scalar_count = 0; + ClearDumpArgSelection(args.dump_arg_selection); + args.explicit_deps = 0; + args.explicit_dep_count = 0; + args.has_error = false; + args.error_msg = 0; +} + +PA_DEVICE bool ReserveTensorArgs(TaskArgs &args, int32_t count) { + // tensor 必须先于 scalar 追加，以保持 dispatch args 的 [tensor..., scalar...] + // 排列；失败只置 has_error，不发生部分追加。 + if (args.scalar_count != 0 || count < 0 || + args.tensor_count + count > static_cast(kMaxTaskTensors)) { + args.has_error = true; + return false; + } + return true; +} + +PA_DEVICE void AppendLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { + const uint32_t index = static_cast(args.tensor_count++); + args.tensors[index].pointer.local_tensor = &tensor; + args.tensors[index].kind = TensorRefKind::LocalTensor; + args.tags[index] = TagValue(tag); +} + +PA_DEVICE void AppendGmTensor(TaskArgs &args, PA_GM const TensorDesc &tensor, TensorArgType tag) { + const uint32_t index = static_cast(args.tensor_count++); + args.tensors[index].pointer.gm_tensor = &tensor; + args.tensors[index].kind = TensorRefKind::GmTensor; + args.tags[index] = TagValue(tag); +} + +PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { + const uint32_t index = static_cast(args.tensor_count++); + args.tensors[index].pointer.create_info = &create_info; + args.tensors[index].kind = TensorRefKind::CreateInfo; + args.tags[index] = TagValue(TensorArgType::Output); +} + +PA_DEVICE bool ReserveScalarArgs(TaskArgs &args, int32_t count) { + // 先整体校验容量再由 AddTwo/AddThree 连续写入，保证多 scalar 操作全有或全无。 + if (count < 0 || args.scalar_count + count > static_cast(kMaxTaskScalars)) { + args.has_error = true; + return false; + } + return true; +} + +PA_DEVICE void AppendScalar(TaskArgs &args, uint64_t value) { + args.scalars[static_cast(args.scalar_count++)] = value; +} + +PA_DEVICE void AddLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { + if (ReserveTensorArgs(args, 1)) AppendLocalTensor(args, tensor, tag); +} + +PA_DEVICE void AddGmTensor(TaskArgs &args, PA_GM const TensorDesc &tensor, TensorArgType tag) { + if (ReserveTensorArgs(args, 1)) AppendGmTensor(args, tensor, tag); +} + +PA_DEVICE void AddOutput(TaskArgs &args, const TensorCreateInfo &create_info) { + if (ReserveTensorArgs(args, 1)) AppendOutput(args, create_info); +} + +PA_DEVICE void AddScalar(TaskArgs &args, uint64_t value) { + if (ReserveScalarArgs(args, 1)) AppendScalar(args, value); +} + +PA_DEVICE void AddTwoScalars(TaskArgs &args, uint64_t value0, uint64_t value1) { + if (!ReserveScalarArgs(args, 2)) return; + AppendScalar(args, value0); + AppendScalar(args, value1); +} + +PA_DEVICE void AddThreeScalars(TaskArgs &args, uint64_t value0, uint64_t value1, uint64_t value2) { + if (!ReserveScalarArgs(args, 3)) return; + AppendScalar(args, value0); + AppendScalar(args, value1); + AppendScalar(args, value2); +} + +PA_DEVICE void InitCreateInfo( + TensorCreateInfo &info, const uint32_t shapes[kMaxTensorDims], uint32_t ndims, DataType dtype +) { + info.initial_value = 0; + info.has_initial_value = false; + info.reserved0 = 0; + info.start_offset = 0; + info.version = 0; + info.ndims = ndims; + info.dtype = dtype; + info.manual_dep = false; + info.is_contiguous = true; + info.child_memory = 0; + // TensorCreateInfo's real constructor only writes active dimensions. + // 只写 ndims 个 shape，保留生产构造器的写入范围，不能为方便把五维全清零。 + for (uint32_t index = 0; index < ndims; ++index) { + info.shapes[index] = shapes[index]; + } +} + +PA_DEVICE void ClearCreateInfo(TensorCreateInfo &info) { + volatile uint8_t *bytes = reinterpret_cast(&info); + for (uint32_t index = 0; index < sizeof(TensorCreateInfo); ++index) { + bytes[index] = 0; + } +} + +PA_DEVICE void InitExternalTensor( + TensorDesc &tensor, uint64_t address, const uint32_t shapes[kMaxTensorDims], uint32_t ndims, DataType dtype, + bool manual_dep +) { + // 输入为稳定 backing address、逻辑 shape 和依赖属性；输出是完整连续 descriptor， + // owner 无效表示它不是本轮 task 产生，row-major stride 从末维向前计算。 + uint64_t elements = 1; + for (uint32_t index = 0; index < ndims; ++index) { + elements *= shapes[index]; + } + tensor.buffer_addr = address; + tensor.buffer_size = elements * ElementSize(dtype); + tensor.owner_task_id = kInvalidTaskId; + tensor.start_offset = 0; + tensor.version = 0; + tensor.ndims = ndims; + tensor.dtype = dtype; + tensor.manual_dep = manual_dep; + tensor.is_contiguous = true; + tensor.child_memory = 0; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + tensor.shapes[index] = shapes[index]; + tensor.strides[index] = 0; + } + uint32_t stride = 1; + for (int32_t index = static_cast(ndims) - 1; index >= 0; --index) { + tensor.strides[index] = stride; + stride *= tensor.shapes[index]; + } + tensor.extent_elem_cache = stride; +} + +PA_DEVICE bool InitTensorFromCreateInfo( + PA_GM TensorDesc &tensor, const TensorCreateInfo &info, uint64_t address, uint64_t buffer_size +) { + tensor.buffer_addr = address; + tensor.buffer_size = buffer_size; + tensor.owner_task_id = kInvalidTaskId; + tensor.start_offset = info.start_offset; + tensor.version = info.version; + tensor.ndims = info.ndims; + tensor.dtype = info.dtype; + tensor.manual_dep = info.manual_dep; + tensor.is_contiguous = info.is_contiguous; + tensor.child_memory = info.child_memory; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + tensor.shapes[index] = info.shapes[index]; + } + uint32_t stride = 1; + for (int32_t index = static_cast(tensor.ndims) - 1; index >= 0; --index) { + tensor.strides[index] = stride; + stride *= tensor.shapes[index]; + } + tensor.extent_elem_cache = stride; + // PA can initialize the backing allocation here. Case1 never requests it; + // the standalone uses synthetic heap addresses and therefore rejects that + // unsupported path instead of writing to a fabricated GM pointer. + // Case1 的 has_initial_value 恒为 false；返回 false 是对未模拟分支的 + // 明确保护，不会在合成地址上伪造初始化写入。 + return !info.has_initial_value; +} + +PA_DEVICE uint64_t CreateInfoBytes(const TensorCreateInfo &info) { + uint64_t elements = 1; + for (uint32_t index = 0; index < info.ndims; ++index) { + elements *= info.shapes[index]; + } + return elements * ElementSize(info.dtype); +} + +template +PA_DEVICE void CopyTensorLine1(TensorDesc &destination, const Source &source) { + // view 只需复制 descriptor 第一条 cache line 的身份/shape 字段，随后由调用方 + // 覆盖 offset、shape、stride 与 extent；不做整 128-byte 拷贝以匹配 PA 写流。 + destination.buffer_addr = source.buffer_addr; + destination.buffer_size = source.buffer_size; + destination.owner_task_id = source.owner_task_id; + destination.start_offset = source.start_offset; + destination.version = source.version; + destination.ndims = source.ndims; + destination.dtype = source.dtype; + destination.manual_dep = source.manual_dep; + destination.is_contiguous = source.is_contiguous; + destination.child_memory = source.child_memory; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + destination.shapes[index] = source.shapes[index]; + } +} + +PA_DEVICE void MakeBatchViews(PaOrchestrationState &orch, uint32_t batch) { + // query/output view 共享原 backing buffer，仅通过 start_offset 切出当前 batch。 + // output_view 保留真实 manual_dep 标记；UP 的生产者依赖由 Alloc、SF、PV 返回 descriptor 的 owner 闭合。 + CopyTensorLine1(orch.query_view, orch.query); + orch.query_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.query_view.ndims = 2; + orch.query_view.shapes[0] = kPaHeads; + orch.query_view.shapes[1] = kPaHeadDim; + orch.query_view.strides[0] = kPaHeadDim; + orch.query_view.strides[1] = 1; + orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; + + CopyTensorLine1(orch.output_view, orch.output); + orch.output_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.output_view.ndims = 2; + orch.output_view.manual_dep = true; + orch.output_view.shapes[0] = kPaHeads; + orch.output_view.shapes[1] = kPaHeadDim; + orch.output_view.strides[0] = kPaHeadDim; + orch.output_view.strides[1] = 1; + orch.output_view.extent_elem_cache = kPaHeads * kPaHeadDim; +} + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +// The two views remain final parameter-packing operations. query_view is a +// winner-only QK input; output_view is the produce side of a private INOUT and +// is therefore evaluated on every worker by the callback builder. +PA_DEVICE void MakeLazySampleCallbackOutputView(PaOrchestrationState &orch, uint32_t batch) { + CopyTensorLine1(orch.output_view, orch.output); + orch.output_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.output_view.ndims = 2; + orch.output_view.manual_dep = true; + orch.output_view.shapes[0] = kPaHeads; + orch.output_view.shapes[1] = kPaHeadDim; + orch.output_view.strides[0] = kPaHeadDim; + orch.output_view.strides[1] = 1; + orch.output_view.extent_elem_cache = kPaHeads * kPaHeadDim; +} + +PA_DEVICE void MakeLazySampleCallbackQueryView(PaOrchestrationState &orch, uint32_t batch) { + CopyTensorLine1(orch.query_view, orch.query); + orch.query_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.query_view.ndims = 2; + orch.query_view.shapes[0] = kPaHeads; + orch.query_view.shapes[1] = kPaHeadDim; + orch.query_view.strides[0] = kPaHeadDim; + orch.query_view.strides[1] = 1; + orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; +} +#endif + +PA_DEVICE uint64_t MinU64(uint64_t lhs, uint64_t rhs) { return lhs < rhs ? lhs : rhs; } + +PA_DEVICE uint64_t ReadPaContextLength(const PaOrchestrationState &orch, uint32_t batch) { + if (orch.context_lens_data == nullptr) { + // Compatibility fallback for a backend that has not yet supplied the + // 256-int GM buffer. Exact PA runs must pass a non-null pointer. + // 正式对等运行必须走下方 descriptor+stride 的 GM load；fallback + // 只用于不具备该缓冲区的兼容后端。 + return kPaBlocksPerRequest * kPaBlockSize; + } + const uint64_t flat_index = orch.context_lens.start_offset + + static_cast(batch) * orch.context_lens.strides[0]; + PA_GM const volatile int32_t *value = reinterpret_cast( + orch.context_lens.buffer_addr + flat_index * ElementSize(DataType::Int32) + ); + return static_cast(*value); +} + +PA_DEVICE void PreparePaBlockGroup(PaOrchestrationState &orch, uint64_t block_offset) { + // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 + // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 + orch.current_block_offset = block_offset; + orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); + const uint64_t last_block_sequence_start = + (block_offset + orch.current_nblocks - 1) * kPaBlockSize; + orch.current_valid_len = MinU64(kPaBlockSize, orch.current_sequence - last_block_sequence_start); +} + +PA_DEVICE void BeginPaBatch(PaOrchestrationState &orch, uint32_t batch) { + // Match paged_attention_orch.cpp: context GM load and block arithmetic + // happen before entering the q scope and before constructing either view. + // 该顺序会影响 Submit 前的指令与访存间隔，故不把长度读取挪进 QK 构参。 + orch.current_batch = batch; + orch.current_sequence = ReadPaContextLength(orch, batch); + orch.current_blocks = (orch.current_sequence + kPaBlockSize - 1) / kPaBlockSize; + MakeBatchViews(orch, batch); +} + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +PA_DEVICE void BeginPaBatchForLazySampleCallback(PaOrchestrationState &orch, uint32_t batch) { + // Tensor reads and arithmetic that feed more than one task stay on the + // common path. Descriptor construction is deferred to the task callback. + orch.current_batch = batch; + orch.current_sequence = ReadPaContextLength(orch, batch); + orch.current_blocks = (orch.current_sequence + kPaBlockSize - 1) / kPaBlockSize; +} +#endif + +PA_DEVICE void InitPaOrchestration( + PaOrchestrationState &orch, uint32_t batches, PA_GM const volatile int32_t *context_lens_data +) { + // 初始化只建立整轮回放共享的外部 descriptor/create-info 模板；每 batch 的 view、 + // context length、动态 QK/SF shape 和返回 descriptor 留给五阶段流按原顺序更新。 + const uint32_t query_shape[kMaxTensorDims] = {batches * kPaHeads, kPaHeadDim, 0, 0, 0}; + const uint32_t cache_shape[kMaxTensorDims] = { + batches * kPaBlocksPerRequest * kPaBlockSize, kPaHeadDim, 0, 0, 0 + }; + const uint32_t table_shape[kMaxTensorDims] = {batches, kPaMaxBlocksPerRequest, 0, 0, 0}; + const uint32_t context_shape[kMaxTensorDims] = {batches, 0, 0, 0, 0}; + InitExternalTensor(orch.query, kSyntheticQueryBase, query_shape, 2, DataType::Bfloat16, false); + InitExternalTensor(orch.key_cache, kSyntheticKeyBase, cache_shape, 2, DataType::Bfloat16, false); + InitExternalTensor(orch.value_cache, kSyntheticValueBase, cache_shape, 2, DataType::Bfloat16, false); + InitExternalTensor(orch.block_table, kSyntheticBlockTableBase, table_shape, 2, DataType::Int32, false); + const uint64_t context_address = context_lens_data == nullptr + ? kSyntheticContextLensBase + : reinterpret_cast(context_lens_data); + InitExternalTensor(orch.context_lens, context_address, context_shape, 1, DataType::Int32, false); + InitExternalTensor(orch.output, kSyntheticOutputBase, query_shape, 2, DataType::Float32, false); + + const uint32_t tile_shape[kMaxTensorDims] = {kPaHeads, kPaHeadDim, 0, 0, 0}; + const uint32_t scalar_shape[kMaxTensorDims] = {kPaHeads, 0, 0, 0, 0}; + ClearCreateInfo(orch.tile_create_info); + ClearCreateInfo(orch.scalar_create_info); + ClearCreateInfo(orch.qk_create_info); + ClearCreateInfo(orch.sf_create_info); + InitCreateInfo(orch.tile_create_info, tile_shape, 2, DataType::Float32); + InitCreateInfo(orch.scalar_create_info, scalar_shape, 1, DataType::Float32); + + // QK/SF create infos are deliberately not constructed here: in PA they are + // constructed inside the group after Alloc and QK respectively. + // 动态 shape 依赖当前 block group，提前构造既不符合业务数据流，也会 + // 把真实发生在两个 Submit 之间的前端工作错误搬到初始化阶段。 + orch.context_lens_data = context_lens_data; + orch.scale_bits = kPaScaleBits; + orch.current_sequence = 0; + orch.current_blocks = 0; + orch.current_block_offset = 0; + orch.current_nblocks = 0; + orch.current_valid_len = 0; + orch.current_batch = 0; + + orch.accumulated_output = nullptr; + orch.accumulated_sum = nullptr; + orch.accumulated_max = nullptr; + orch.qk_scores = nullptr; + orch.sf_probs = nullptr; + orch.sf_max = nullptr; + orch.sf_sum = nullptr; + orch.pv_output = nullptr; +} + +PA_DEVICE void InitPaOrchestration(PaOrchestrationState &orch, uint32_t batches) { + InitPaOrchestration(orch, batches, nullptr); +} + +PA_DEVICE void BuildAllocArgs(PaOrchestrationState &orch, TaskArgs &args, uint32_t batch) { + BeginPaBatch(orch, batch); + // PA constructs a fresh L0TaskArgs after its two views; Alloc is populated + // without calling reset(). + // 三个 Output 分别成为累计 output/sum/max；Alloc 无 kernel slot，winner 在 + // HeapGuard 后直接发布 task completion。 + ConstructTaskArgs(args); + if (!ReserveTensorArgs(args, 3)) return; + AppendOutput(args, orch.tile_create_info); + AppendOutput(args, orch.scalar_create_info); + AppendOutput(args, orch.scalar_create_info); +} + +PA_DEVICE void BuildQkArgs(PaOrchestrationState &orch, TaskArgs &args, uint32_t batch) { + (void)batch; + // PA computes the block group after Alloc returns, immediately before it + // constructs the dynamic QK output create-info. + // QK 消费 query/key/block-table，产出 score；其 active role 为 AIC。 + PreparePaBlockGroup(orch, 0); + const uint32_t score_shape[kMaxTensorDims] = { + kPaHeads, static_cast(orch.current_nblocks * kPaBlockSize), 0, 0, 0 + }; + // Constructed after Alloc submit and immediately before QK reset/adds. + InitCreateInfo(orch.qk_create_info, score_shape, 2, DataType::Float32); + ResetTaskArgs(args); + if (!ReserveTensorArgs(args, 3)) return; + AppendLocalTensor(args, orch.query_view, TensorArgType::Input); + AppendLocalTensor(args, orch.key_cache, TensorArgType::Input); + AppendLocalTensor(args, orch.block_table, TensorArgType::Input); + AddOutput(args, orch.qk_create_info); + AddTwoScalars( + args, orch.current_nblocks, + static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + orch.current_block_offset + ); +} + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +struct LazySampleCallbackBuildCounts { + uint32_t reset_calls; + uint32_t views_created; + uint32_t dynamic_create_infos; + uint32_t tensor_args_added; + uint32_t scalar_args_added; +}; + +template +class LazySampleCallbackArgsBuilder { +public: + PA_DEVICE LazySampleCallbackArgsBuilder(TaskArgs &args, TaskKind kind, bool won) + : args_(args), kind_(kind), won_(won), begin_calls_(0), counts_{} {} + + PA_DEVICE void Begin() { + if (++begin_calls_ != 1) { + args_.has_error = true; + return; + } + if (kind_ == TaskKind::Alloc) { + ConstructTaskArgs(args_); + } else { + ResetTaskArgs(args_); + ++counts_.reset_calls; + } + } + + PA_DEVICE void RecordView() { ++counts_.views_created; } + PA_DEVICE void RecordDynamicCreateInfo() { ++counts_.dynamic_create_infos; } + + template + PA_DEVICE void AddLocalInput(Thunk thunk) { + if constexpr (Lazy) { + if (!won_) return; + } + if (!Ready()) return; + const TensorDesc &tensor = thunk(); + AddLocalTensor(args_, tensor, TensorArgType::Input); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddGmInput(Thunk thunk) { + if constexpr (Lazy) { + if (!won_) return; + } + if (!Ready()) return; + PA_GM const TensorDesc &tensor = thunk(); + AddGmTensor(args_, tensor, TensorArgType::Input); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddOutput(Thunk thunk) { + if (!Ready()) return; + const TensorCreateInfo &create_info = thunk(); + pa_scheduler::AddOutput(args_, create_info); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddLocalInout(Thunk thunk) { + if (!Ready()) return; + const TensorDesc &tensor = thunk(); + AddLocalTensor(args_, tensor, TensorArgType::Inout); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddGmInout(Thunk thunk) { + if (!Ready()) return; + PA_GM const TensorDesc &tensor = thunk(); + AddGmTensor(args_, tensor, TensorArgType::Inout); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddScalar(Thunk thunk) { + if constexpr (Lazy) { + if (!won_) return; + } + if (!Ready()) return; + pa_scheduler::AddScalar(args_, thunk()); + if (!args_.has_error) ++counts_.scalar_args_added; + } + + PA_DEVICE bool Valid() const { return begin_calls_ == 1 && !args_.has_error; } + PA_DEVICE const LazySampleCallbackBuildCounts &Counts() const { return counts_; } + +private: + PA_DEVICE bool Ready() { + if (begin_calls_ == 1 && !args_.has_error) return true; + args_.has_error = true; + return false; + } + + TaskArgs &args_; + TaskKind kind_; + bool won_; + uint32_t begin_calls_; + LazySampleCallbackBuildCounts counts_; +}; +#endif + +PA_DEVICE void BuildSfArgs(PaOrchestrationState &orch, TaskArgs &args) { + const uint32_t probability_shape[kMaxTensorDims] = { + kPaHeads, static_cast(orch.current_nblocks * kPaBlockSize), 0, 0, 0 + }; + // Constructed only after QK submit returns its sij descriptor. + // SF 通过 qk_scores.owner 得到 QK fanin，产出 probability/max/sum；active role 为 AIV。 + InitCreateInfo(orch.sf_create_info, probability_shape, 2, DataType::Bfloat16); + ResetTaskArgs(args); + AddGmTensor(args, *orch.qk_scores, TensorArgType::Input); + if (!ReserveTensorArgs(args, 3)) return; + AppendOutput(args, orch.sf_create_info); + AppendOutput(args, orch.scalar_create_info); + AppendOutput(args, orch.scalar_create_info); + AddThreeScalars(args, orch.scale_bits, orch.current_nblocks, orch.current_valid_len); +} + +PA_DEVICE void BuildPvArgs(PaOrchestrationState &orch, TaskArgs &args, uint32_t batch) { + (void)batch; + // PV 消费 SF probability 与 value/block-table，owner 形成一条 SF->PV 依赖； + // 结果 pv_output 供最后的 UP 使用，active role 回到 AIC。 + ResetTaskArgs(args); + if (!ReserveTensorArgs(args, 3)) return; + AppendGmTensor(args, *orch.sf_probs, TensorArgType::Input); + AppendLocalTensor(args, orch.value_cache, TensorArgType::Input); + AppendLocalTensor(args, orch.block_table, TensorArgType::Input); + AddOutput(args, orch.tile_create_info); + AddTwoScalars( + args, orch.current_nblocks, + static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + orch.current_block_offset + ); +} + +PA_DEVICE void BuildUpdateArgs(PaOrchestrationState &orch, TaskArgs &args) { + // UP 的 SF max/sum 共享一个 SF owner，PV output 提供一个 PV owner，三个累计 + // Inout 共享 Alloc owner，去重后共 3 条 fanin；output_view 把更新写回当前 batch。 + ResetTaskArgs(args); + if (!ReserveTensorArgs(args, 3)) return; + AppendGmTensor(args, *orch.sf_max, TensorArgType::Input); + AppendGmTensor(args, *orch.sf_sum, TensorArgType::Input); + AppendGmTensor(args, *orch.pv_output, TensorArgType::Input); + if (!ReserveTensorArgs(args, 4)) return; + AppendGmTensor(args, *orch.accumulated_max, TensorArgType::Inout); + AppendGmTensor(args, *orch.accumulated_sum, TensorArgType::Inout); + AppendGmTensor(args, *orch.accumulated_output, TensorArgType::Inout); + AppendLocalTensor(args, orch.output_view, TensorArgType::Inout); + AddTwoScalars( + args, orch.current_block_offset == 0 ? 1 : 0, + orch.current_block_offset + orch.current_nblocks >= orch.current_blocks ? 1 : 0 + ); +} + +PA_DEVICE void AcceptTaskOutputs(PaOrchestrationState &orch, TaskKind kind, const TaskOutputs &outputs) { + // 每个 worker 都完整回放并接收自己 materialize 的 descriptor；只有 Claim winner + // 会执行 kernel，但 loser 的后续 orchestration 仍使用相同 task_id/owner 拓扑。 + switch (kind) { + case TaskKind::Alloc: + orch.accumulated_output = outputs.tensors[0]; + orch.accumulated_sum = outputs.tensors[1]; + orch.accumulated_max = outputs.tensors[2]; + break; + case TaskKind::Qk: + orch.qk_scores = outputs.tensors[0]; + break; + case TaskKind::Sf: + orch.sf_probs = outputs.tensors[0]; + orch.sf_max = outputs.tensors[1]; + orch.sf_sum = outputs.tensors[2]; + break; + case TaskKind::Pv: + orch.pv_output = outputs.tensors[0]; + break; + default: + // UP 只更新既有 Inout，没有新 Output descriptor 需要传给下一阶段。 + break; + } +} + +PA_DEVICE void ResetTensorMap(PA_GM TensorMap &map) { + // TensorMap 完全属于当前 worker，初始化和后续增删都不需要 atomic。bucket 与 + // task_heads 置空后，entry 存储按 high_water 首次分配、再经 free_head 复用。 + map.free_head = -1; + map.high_water = 0; + map.alive_floor = 0; + map.cleaned_upto = 0; + for (uint32_t index = 0; index < kMapBuckets; ++index) { + map.buckets[index] = -1; + } + for (uint32_t index = 0; index < kTaskWindow; ++index) { + map.task_heads[index] = -1; + } +} + +PA_DEVICE uint32_t TensorMapHash(uint64_t address) { + address *= 0x9E3779B97F4A7C15ULL; + return static_cast(address >> (64 - kMapBucketShift)); +} + +template +PA_DEVICE void TensorByteRange(const TensorReference &tensor, uint64_t &address, uint64_t &lo, uint64_t &hi) { + // identity 先按 backing buffer 地址分桶，再用半开字节区间 [lo, hi) 判断 view + // 是否重叠。连续 tensor 由 shape 现算 extent，非连续 tensor 使用缓存 extent。 + const uint64_t element_size = ElementSize(tensor.dtype); + address = tensor.buffer_addr; + lo = tensor.start_offset * element_size; + uint64_t extent; + if (tensor.is_contiguous) { + extent = 1; + for (uint32_t index = 0; index < tensor.ndims; ++index) { + extent *= tensor.shapes[index]; + } + } else { + extent = tensor.extent_elem_cache; + } + hi = (tensor.start_offset + extent) * element_size; +} + +PA_DEVICE int32_t AllocateMapEntry(PA_GM TensorMap &map) { + // 输出为可写 entry 下标：优先复用退休链，之后增长 high_water；返回 -1 表示 + // 固定容量耗尽。分配动作尚未把 entry 接入任何 bucket/task 链。 + if (map.free_head >= 0) { + const int32_t slot = map.free_head; + map.free_head = map.entries[slot].next_in_bucket; + return slot; + } + if (map.high_water < static_cast(kMapCapacity)) { + return map.high_water++; + } + return -1; +} + +PA_DEVICE void FreeMapEntry(PA_GM TensorMap &map, int32_t index) { + // 输入 index 必须仍位于其 bucket 链。输出状态是从双向 bucket 链完整摘除， + // 再把 next_in_bucket 改作 free-list next；task 链由 AdvanceTensorMap 顺序遍历。 + PA_GM MapEntry &entry = map.entries[index]; + if (entry.prev_in_bucket < 0) { + map.buckets[entry.bucket] = entry.next_in_bucket; + } else { + map.entries[entry.prev_in_bucket].next_in_bucket = entry.next_in_bucket; + } + if (entry.next_in_bucket >= 0) { + map.entries[entry.next_in_bucket].prev_in_bucket = entry.prev_in_bucket; + } + entry.bucket = -1; + entry.next_in_bucket = map.free_head; + map.free_head = index; +} + +PA_DEVICE void AdvanceTensorMap(PA_GM TensorMap &map, uint32_t task_id, int32_t heap_window) { + // PrepareMap 在 Claim 前把存活下界推进到 task_id-H。离开窗口的 producer 先按 + // task_heads 找到其全部 entry，再从 bucket 链摘除并进入 free list。TensorMap 与 + // heap 共享窗口宽度 H，但前者按本 worker 的 task_id 推进，后者按跨核连续 + // frontier 推进，二者并不要求同步到达同一位置。 + const int32_t new_floor = static_cast(task_id) - heap_window; + if (new_floor <= map.cleaned_upto) { + if (new_floor > map.alive_floor) { + map.alive_floor = new_floor; + } + return; + } + for (int32_t id = map.cleaned_upto; id < new_floor; ++id) { + int32_t current = map.task_heads[static_cast(id) & kTaskWindowMask]; + while (current >= 0) { + const int32_t next = map.entries[current].next_in_task; + FreeMapEntry(map, current); + current = next; + } + map.task_heads[static_cast(id) & kTaskWindowMask] = -1; + } + map.cleaned_upto = new_floor; + map.alive_floor = new_floor; +} + +template +PA_DEVICE void InsertTensor(PA_GM TensorMap &map, const TensorReference &tensor, int32_t producer) { + // 新 producer 同时插入地址 bucket 的表头与 producer 对应 task 链的表头。 + // map 满时生产语义是静默放弃登记；standalone 保持该行为，不新增异常分支。 + uint64_t address = 0; + uint64_t lo = 0; + uint64_t hi = 0; + TensorByteRange(tensor, address, lo, hi); + const int32_t slot = AllocateMapEntry(map); + if (slot < 0) { + return; + } + const uint32_t bucket = TensorMapHash(address); + PA_GM MapEntry &entry = map.entries[slot]; + entry.buffer_addr = address; + entry.lo = lo; + entry.hi = hi; + entry.producer = producer; + entry.bucket = static_cast(bucket); + entry.prev_in_bucket = -1; + entry.next_in_bucket = map.buckets[bucket]; + if (map.buckets[bucket] >= 0) { + map.entries[map.buckets[bucket]].prev_in_bucket = slot; + } + map.buckets[bucket] = slot; + const uint32_t task_slot = static_cast(producer) & kTaskWindowMask; + entry.next_in_task = map.task_heads[task_slot]; + map.task_heads[task_slot] = slot; +} + +template +PA_DEVICE int32_t LookupTensor(PA_GM const TensorMap &map, const TensorReference &tensor) { + uint64_t address = 0; + uint64_t lo = 0; + uint64_t hi = 0; + TensorByteRange(tensor, address, lo, hi); + int32_t best = -1; + // 同一 buffer 可能存在多个历史写者；只考虑仍存活且区间重叠的 entry，并选择 + // task_id 最大的最新 producer，防止依赖回退到更老版本。 + for (int32_t current = map.buckets[TensorMapHash(address)]; current >= 0; + current = map.entries[current].next_in_bucket) { + PA_GM const MapEntry &entry = map.entries[current]; + if (entry.producer < map.alive_floor) { + continue; + } + if (entry.buffer_addr == address && lo < entry.hi && entry.lo < hi && entry.producer > best) { + best = entry.producer; + } + } + return best; +} + +PA_DEVICE uint64_t TensorOwner(const TaskTensorRef &reference) { + // CreateInfo 只会出现在 tag=Output 且在 fanin 前已被跳过；这里的输入不变量是 + // LocalTensor/GmTensor，输出为显式 owner 或 kInvalidTaskId。 + if (reference.kind == TensorRefKind::GmTensor) { + return reference.pointer.gm_tensor->owner_task_id; + } + return reference.pointer.local_tensor->owner_task_id; +} + +PA_DEVICE int32_t LookupTensorRef(PA_GM const TensorMap &map, const TaskTensorRef &reference) { + // 与 TensorOwner 相同，此辅助入口只接收已存在 descriptor；返回最新重叠 producer， + // 未登记或已退休则返回 -1。 + if (reference.kind == TensorRefKind::GmTensor) { + return LookupTensor(map, *reference.pointer.gm_tensor); + } + return LookupTensor(map, *reference.pointer.local_tensor); +} + +PA_DEVICE void AddFanin(int32_t fanin[kMaxFanin], uint32_t &count, int32_t producer) { + // owner 与 TensorMap lookup 可能指向同一 producer，先去重再写固定 16 槽数组； + // Case1 的最大 fanin 为 UP 的 3，正常路径不会截断。 + if (producer < 0) { + return; + } + for (uint32_t index = 0; index < count; ++index) { + if (fanin[index] == producer) { + return; + } + } + if (count < kMaxFanin) { + fanin[count++] = producer; + } +} + +PA_DEVICE uint32_t CollectFanin( + PA_GM const TensorMap &map, const TaskArgs &args, int32_t fanin[kMaxFanin] +) { + // fanin 只由 winner 收集：先吸收 descriptor 的显式 owner，再对 Input/Inout + // 查询最新重叠写者；纯 Output 尚未存在，不应成为本次 task 的输入依赖。 + uint32_t count = 0; + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = TaskTag(args, static_cast(index)); + if (tag == TensorArgType::Output) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + // Keep the two address spaces in separate control-flow arms. CCEC's + // O2/O3 backend rejects a merged pointer phi even when both arms only + // feed scalar field loads; this is also how PA's production helper is + // written. + // 分支重复是后端约束与生产写法的一部分，不应抽成一个混合地址空间指针。 + if (reference.kind == TensorRefKind::GmTensor) { + PA_GM const TensorDesc &tensor = *reference.pointer.gm_tensor; + const uint64_t owner = tensor.owner_task_id; + if (owner != kInvalidTaskId) { + AddFanin(fanin, count, static_cast(owner & 0xFFFFFFFFU)); + } + if (tag == TensorArgType::Input || tag == TensorArgType::Inout) { + AddFanin(fanin, count, LookupTensor(map, tensor)); + } + } else { + const TensorDesc &tensor = *reference.pointer.local_tensor; + const uint64_t owner = tensor.owner_task_id; + if (owner != kInvalidTaskId) { + AddFanin(fanin, count, static_cast(owner & 0xFFFFFFFFU)); + } + if (tag == TensorArgType::Input || tag == TensorArgType::Inout) { + AddFanin(fanin, count, LookupTensor(map, tensor)); + } + } + } + return count; +} + +PA_DEVICE void InsertExistingTensor(SubmitContext &context, const TaskArgs &args, int32_t index) { + // 输入 index 来自 register_mask，故必为已有 descriptor 而非 CreateInfo；写入结果 + // 只影响 context.self 对应 worker 的 map，并把当前 task_id 登记为新的 hazard 版本。 + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind == TensorRefKind::GmTensor) { + InsertTensor(context.self->map, *reference.pointer.gm_tensor, context.task_id); + } else { + InsertTensor(context.self->map, *reference.pointer.local_tensor, context.task_id); + } +} + +PA_DEVICE void RegisterOutputs(SubmitContext &context, const TaskArgs &args, bool include_existing) { + // register_mask 只覆盖 Inout/OutputExisting。新 Output 已带本次 owner；现有 + // backing buffer 的新写者则必须登记到本 worker TensorMap，供后继 task 查 hazard。 + if (!include_existing) { + return; + } + uint32_t register_mask = context.register_mask; + for (uint32_t index = 0; register_mask != 0; ++index, register_mask >>= 1) { + if ((register_mask & 1U) != 0) { + InsertExistingTensor(context, args, static_cast(index)); + } + } +} + +PA_DEVICE uint64_t FrontendAlignUp(uint64_t value, uint64_t alignment) { + // alignment 在本模型中固定为2的幂1 KiB；返回逻辑 heap 地址，不做 ring 取模。 + return (value + alignment - 1) & ~(alignment - 1); +} + +PA_DEVICE void BeginSubmit( + PA_GM WorkerState &worker, const TaskArgs &args, SubmitContext &context +) { + // Mirrors dist_submit_begin(). The production Submit and Materialize spans + // both start after this per-call context initialization. + // local_index 在所有 worker 上按同一 orchestration 顺序递增，因此 + // task_id 一致；该初始化位于 Submit 计时起点之前，不能误计进阶段耗时。 + const uint32_t task_id = static_cast(worker.local_index++); + context.self = &worker; + context.payload = &worker.payloads[task_id & kPayloadMask]; + context.task_id = static_cast(task_id); + context.tensor_count = args.tensor_count; + context.scalar_count = args.scalar_count; + context.result.task_id = task_id; + context.result.count = 0; + context.register_mask = 0; + context.output_bytes = 0; + context.fanin_count = 0; + context.kernel_id = -1; + context.won = false; + context.joint = false; + context.joint_init = false; + context.joint_block = -1; + context.joint_slot = -1; + context.joint_count = 0; +} + +PA_DEVICE bool MaterializeTask( + PA_GM WorkerState &worker, uint32_t task_id, const TaskArgs &args, SubmitContext &context, + uint64_t heap_base, uint64_t heap_size +) { + // 输入是 BeginSubmit 已绑定的 payload/context 与当前 worker.heap_next；成功输出 + // 包括本 task 的 GM TensorDesc 指针、output_bytes 和推进后的单调 heap_next。 + // 失败不得进入 slot/build 流程，由上层设置 fatal 并终止该 worker 回放。 + // legacy 路径在 Claim 前物化；claim-first callback 实验则可能已经完成 Claim， + // 因而不能把“尚未 Claim”写成这个共用 helper 的普遍前置条件。 + if (context.payload == nullptr) { + return false; + } + context.tensor_count = args.tensor_count; + context.scalar_count = args.scalar_count; + context.register_mask = 0; + + // DistOutputLayout leaves non-output slots lazy and writes only the sizes + // selected by output_mask. + // 第一次 tag 扫描同时产生 output_mask/register_mask；第二次只遍历 + // Output 位，避免读取未初始化的非输出 buffer_sizes。 + OutputLayout layout; + layout.total_output_size = 0; + uint32_t output_mask = 0; + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = TaskTag(args, static_cast(index)); + if (tag == TensorArgType::Inout || tag == TensorArgType::OutputExisting) { + context.register_mask |= 1U << index; + } + if (tag != TensorArgType::Output) { + continue; + } + output_mask |= 1U << index; + layout.buffer_sizes[index] = CreateInfoBytes(*args.tensors[index].pointer.create_info); + layout.total_output_size += FrontendAlignUp(layout.buffer_sizes[index], kOutputAlignment); + } + + uint64_t task_base = FrontendAlignUp(worker.heap_next, kOutputAlignment); + const uint64_t total = layout.total_output_size; + if (total > heap_size || (total != 0 && heap_base == 0)) { + return false; + } + if (total != 0 && (task_base % heap_size) + total > heap_size) { + // 单个 task 的输出必须物理连续；若跨 ring 尾部则把逻辑 task_base 推到 + // 下一圈起点。heap_next 仍保持单调，不在这里取模。 + task_base = (task_base / heap_size + 1) * heap_size; + } + + uint64_t output_offset = 0; + // 各 Output 在同一 task_base 内按参数顺序排布；result 只收集 Output，索引与 + // TaskArgs 中非输出槽无关，而 payload 仍按原参数 index 保存 descriptor。 + for (int32_t index = 0; output_mask != 0; ++index, output_mask >>= 1) { + if ((output_mask & 1U) == 0) { + continue; + } + const uint64_t physical = (task_base + output_offset) % heap_size; + PA_GM TensorDesc &tensor = context.payload->tensors[index]; + if (!InitTensorFromCreateInfo( + tensor, *args.tensors[index].pointer.create_info, heap_base + physical, layout.buffer_sizes[index] + )) { + return false; + } + tensor.owner_task_id = task_id; + context.result.tensors[context.result.count++] = &tensor; + output_offset += FrontendAlignUp(layout.buffer_sizes[index], kOutputAlignment); + } + worker.heap_next = task_base + total; + context.output_bytes = total; + return true; +} + +PA_DEVICE void CopyTensorFromRef(PA_GM TensorDesc &destination, const TaskTensorRef &reference) { + // slot 必须拥有 descriptor 快照，不能保存指向 orchestration 栈对象的引用； + // 按 byte volatile copy 同时兼容 local/GM 源并保留真实 128-byte 搬运量。 + PA_GM volatile uint8_t *destination_bytes = reinterpret_cast(&destination); + if (reference.kind == TensorRefKind::GmTensor) { + PA_GM const volatile uint8_t *source_bytes = + reinterpret_cast(reference.pointer.gm_tensor); + for (uint32_t byte = 0; byte < sizeof(TensorDesc); ++byte) { + destination_bytes[byte] = source_bytes[byte]; + } + return; + } + const volatile uint8_t *source_bytes = + reinterpret_cast(reference.pointer.local_tensor); + for (uint32_t byte = 0; byte < sizeof(TensorDesc); ++byte) { + destination_bytes[byte] = source_bytes[byte]; + } +} + +PA_DEVICE void CopyGmTensor(PA_GM TensorDesc &destination, PA_GM const TensorDesc &source) { + // 新 Output 的源 descriptor 已位于 GM payload；单独入口避免把 GM 指针误走 + // local 地址空间分支，输出仍是 slot 内独立副本。 + PA_GM volatile uint8_t *destination_bytes = reinterpret_cast(&destination); + PA_GM const volatile uint8_t *source_bytes = reinterpret_cast(&source); + for (uint32_t byte = 0; byte < sizeof(TensorDesc); ++byte) { + destination_bytes[byte] = source_bytes[byte]; + } +} + +PA_DEVICE void PopulateSlotPayload( + PA_GM LocalSlot &slot, const TaskArgs &args, const SubmitContext &context, const int32_t fanin[kMaxFanin], + uint32_t fanin_count, int32_t sub_block_id, bool is_multicore, int32_t won_block, int32_t won_slot +) { + // winner 将活动 descriptor/scalar 复制进私有 slot，dispatch args 指向 slot 内 + // 副本而非 orchestration 临时对象；fanin 随 slot 保存，kernel 执行前逐 flag 检查。 + slot.tensor_count = context.tensor_count; + slot.scalar_count = context.scalar_count; + for (int32_t index = 0; index < context.tensor_count; ++index) { + if (TaskTag(args, static_cast(index)) == TensorArgType::Output) { + CopyGmTensor(slot.tensors[index], context.payload->tensors[index]); + } else { + CopyTensorFromRef(slot.tensors[index], args.tensors[index]); + } + slot.args[index] = static_cast(reinterpret_cast(&slot.tensors[index])); + } + for (int32_t index = 0; index < context.scalar_count; ++index) { + slot.scalars[index] = args.scalars[index]; + slot.args[context.tensor_count + index] = args.scalars[index]; + } + + PA_GM PaLocalContext &local = + *reinterpret_cast(&slot.local_context[0]); + // standalone 每个 task 只由一个 lane kernel 执行，故 block_index/count 固定0/1； + // async completion 未启用，task_token 保持 invalid，与 PA 普通同步 slot 一致。 + local.block_index = 0; + local.block_count = 1; + local.async.completion_count = 0; + local.async.completion_error_code = 0; + local.async.completion_entries = 0; + local.async.completion_capacity = 0; + local.async.task_token = kInvalidTaskId; + slot.global_context = static_cast(sub_block_id); + slot.args[kSpmdLocalContextIndex] = + static_cast(reinterpret_cast(&slot.local_context[0])); + slot.args[kSpmdGlobalContextIndex] = + static_cast(reinterpret_cast(&slot.global_context)); + slot.fanin_count = fanin_count; + // fanin 数组只复制有效前缀；执行端以 fanin_count 为边界，未使用尾部保持惰性。 + for (uint32_t index = 0; index < fanin_count; ++index) { + slot.fanin[index] = fanin[index]; + } + slot.is_multicore = is_multicore; + slot.won_block = won_block; + slot.won_slot = won_slot; +} + +PA_DEVICE void BuildSlotPayload( + PA_GM LocalSlot &slot, uint32_t task_id, uint32_t function_id, uint64_t function_address, const TaskArgs &args, + const SubmitContext &context, const int32_t fanin[kMaxFanin], uint32_t fanin_count, int32_t sub_block_id = 0, + bool is_multicore = false, int32_t won_block = -1, int32_t won_slot = -1 +) { + // Match build_ring_slot_from_submit ordering: publish the header first, + // then copy the active descriptors/scalars and construct dispatch payload. + // slot 仅由所属 worker 消费，这里的写入次序用于复刻真实构建成本与 + // 状态机；跨核可见性由 task completion 的 flag/vend 协议承担。 + slot.occupied = true; + slot.task_id = task_id; + slot.kind = function_id; + slot.function_address = function_address; + slot.built = 1; + PopulateSlotPayload( + slot, args, context, fanin, fanin_count, sub_block_id, is_multicore, won_block, won_slot + ); +} + +// Compatibility overload for a core that has already populated the slot +// header before calling the PA frontend. +// 该入口只补 payload，不改变既有 task/function 头；输出不变量与完整 +// BuildSlotPayload 相同，均得到 built 且可由 DrainReady 检查 fanin 的私有 slot。 +PA_DEVICE void BuildSlotPayload( + PA_GM LocalSlot &slot, const TaskArgs &args, const SubmitContext &context, const int32_t fanin[kMaxFanin], + uint32_t fanin_count +) { + slot.built = 1; + PopulateSlotPayload(slot, args, context, fanin, fanin_count, 0, false, -1, -1); +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_FRONTEND_H diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/pa_model.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/pa_model.h new file mode 100644 index 0000000000..0c74aa2607 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/pa_model.h @@ -0,0 +1,941 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_MODEL_H +#define PA_SCHEDULER_COMMON_PA_MODEL_H + +#include +#include + +// CCEC 的正式产物只允许二选一：swimlane 保存普通阶段与 atomic 记录， +// submit-pmu 则编译掉泳道观察代码。CPU/AscendC 未传这些宏时继续使用原有 +// 通用实现，避免公共模型反向依赖某个后端的构建脚本。 +#ifndef PA_BUILD_SWIMLANE +#define PA_BUILD_SWIMLANE 0 +#endif + +#ifndef PA_BUILD_SUBMIT_PMU +#define PA_BUILD_SUBMIT_PMU 0 +#endif + +#if PA_BUILD_SWIMLANE && PA_BUILD_SUBMIT_PMU +#error "PA_BUILD_SWIMLANE and PA_BUILD_SUBMIT_PMU are mutually exclusive" +#endif + +namespace pa_scheduler { + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +// The original artifact is compiled without this selector. The two selected +// artifacts deliberately share one compete-first/split-finish skeleton; only +// the builder's loser replay policy differs. +static_assert( + PA_LAZY_SAMPLE_SHAPE_ID == 1 || PA_LAZY_SAMPLE_SHAPE_ID == 2, + "PA_LAZY_SAMPLE_SHAPE_ID must select compete-first(1) or compete-first-lazy(2)" +); +constexpr bool kLazySampleLazy = PA_LAZY_SAMPLE_SHAPE_ID == 2; +constexpr bool kLazySampleSplitFinish = true; +constexpr uint32_t kLazySampleShapeId = PA_LAZY_SAMPLE_SHAPE_ID; +constexpr const char *kLazySampleShapeName = + PA_LAZY_SAMPLE_SHAPE_ID == 1 ? "compete-first" : "compete-first-lazy"; +constexpr const char *kLazySampleObservation = "split-combination-semantic"; +constexpr const char *kLazySampleFinishShape = "noinline-cross-tu"; +constexpr const char *kLazySampleControlFamily = "all-task-compete-first-callback"; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +static_assert(PA_LAZY_SAMPLE_SPLIT_FINISH == 1, "split-finish feature macro must equal one"); +static_assert(kLazySampleSplitFinish, "both lazy-sample callback shapes require split finish"); +#else +#error "lazy-sample callback artifacts require PA_LAZY_SAMPLE_SPLIT_FINISH" +#endif +#endif + +// 这里固定的是 PA Case1 的调度拓扑，而不是为了缩小 standalone 人为选择的规模： +// 每个 batch 依次回放 Alloc/QK/SF/PV/UP 五个 task，32 个 AIC 与 64 个 AIV +// 都执行同一条 orchestration 流，只在 Claim 时按 task 的 active role 分流。 +constexpr uint32_t kDefaultBatches = 256; +constexpr uint32_t kMaxBatches = 256; +constexpr uint32_t kTasksPerBatch = 5; +constexpr uint32_t kMaxTasks = kMaxBatches * kTasksPerBatch; +constexpr uint32_t kTaskCellCapacity = 1U << 16; + +constexpr uint32_t kAicWorkers = 32; +constexpr uint32_t kAivWorkers = 64; +constexpr uint32_t kWorkers = kAicWorkers + kAivWorkers; +constexpr uint32_t kRuntimeMaxWorkers = 108; +constexpr uint32_t kCursorShards = 4; +// 每个 worker 私有 ring 有 4 个物理 slot，其中 2 个为 BlockWon 协议预留； +// 单 lane Case1 虽不进入 BlockWon，普通 kernel 仍只能占用剩余 2 个 slot。 +constexpr uint32_t kPrivateSlots = 4; +constexpr uint32_t kWonReserve = 2; +constexpr uint32_t kUsableSlots = kPrivateSlots - kWonReserve; +constexpr uint32_t kMaxFanin = 16; +// H=64 同时约束 heap 可回收 frontier 和 TensorMap producer 的存活下界。 +// heap_next 使用单调逻辑地址；真正落到 256 MiB 环形 heap 时才取模，因而可判断覆盖风险。 +constexpr uint32_t kHeapWindow = 64; +constexpr uint64_t kHeapBytes = 256ULL << 20; +constexpr uint64_t kSyntheticHeapBase = 0x100000000ULL; +constexpr uint64_t kOutputAlignment = 1024; +constexpr uint32_t kMaxTensorDims = 5; +constexpr uint32_t kMaxTaskTensors = 32; +constexpr uint32_t kMaxTaskScalars = 16; +constexpr uint32_t kPayloadSlots = 2048; +constexpr uint32_t kPayloadMask = kPayloadSlots - 1; +constexpr uint32_t kPayloadStride = 4096; +constexpr uint32_t kMapCapacity = 16384; +constexpr uint32_t kMapBuckets = 1 << 13; +constexpr uint32_t kMapBucketShift = 13; +constexpr uint32_t kTaskWindow = 1 << 10; +constexpr uint32_t kTaskWindowMask = kTaskWindow - 1; +constexpr uint64_t kSystemCounterHz = 1000000000ULL; +constexpr uint64_t kWatchdogTicks = 2 * kSystemCounterHz; +// trace_enabled 是位图而不是 bool：bit0 保持既有阶段泳道，bit1 额外开启 +// 逐条 atomic 源码括号记录。atomic 记录依赖同一份 trace buffer，因此 bit1 +// 只能与 bit0 一起配置。 +constexpr uint32_t kTracePhasesEnabled = 1U << 0; +constexpr uint32_t kTraceAtomicsEnabled = 1U << 1; +// Claim trace flags 是独立 raw ABI：bit0 表示获胜，bit1 表示已经通过 +// AIC/AIV role 路由并真正执行 atomicMax。未 attempted 的 Claim 仍保留 +// role-selection 开销，但转换器会明确标成 claim.not_attempted。 +constexpr uint32_t kClaimWon = 1U << 0; +constexpr uint32_t kClaimAttempted = 1U << 1; +// 下列 offset/size 来自真实 DistGlobal/DistCore ABI。standalone 保留被测关键字段的 +// offset、DistCore ABI 和 kRealDistGlobalBytes 总跨度；其余区域可用 opaque padding， +// 并不是对生产结构全部字段的逐一镜像。 +constexpr size_t kRealDistCoreOffset = 10043904; +constexpr size_t kRealDistGlobalBytes = 1007023872; +constexpr size_t kRealTasksOffset = 896; +constexpr size_t kRealFatalOffset = 4195264; +constexpr size_t kRealReplayDoneOffset = 10043776; +constexpr size_t kRealStartedCountOffset = 10043840; +constexpr uint32_t kTraceRecordsPerCore = 1U << 16; +static_assert((kPayloadSlots & kPayloadMask) == 0, "payload slots must be a power of two"); +static_assert(kMaxTasks < kTaskCellCapacity, "every frontier scan must terminate on an in-range not-ready flag"); + +// These are the measured means from the best PA A5 trace, in 1 GHz ticks. +// The scalar-NOP compatibility baseline calibrates its counts against these targets. +// 无参数默认使用 real-compute：CCEC/AscendC 执行完整 Cube/Vector 流水， +// CPU 执行对等算术；下列 NOP 常量只供显式 scalar-nop 校准。两种模式的 +// Submit、依赖、heap 与 completion 路径都不靠补时修改。target 是真实泳道 +// 均值，不是调度阶段预算。 +constexpr uint32_t kTargetQkTicks = 44170; +constexpr uint32_t kTargetSfTicks = 53729; +constexpr uint32_t kTargetPvTicks = 27626; +constexpr uint32_t kTargetUpTicks = 1565; + +// Calibrated on the local A5 with the CCEC RuntimeNop implementation. These +// counts resolve to the measured targets above; they are not cycle guesses. +constexpr uint32_t kDefaultQkNops = 129600; +constexpr uint32_t kDefaultSfNops = 157900; +constexpr uint32_t kDefaultPvNops = 79950; +constexpr uint32_t kDefaultUpNops = 2400; + +enum class CoreRole : uint32_t { + Aic = 0, + Aiv = 1, +}; + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +constexpr uint64_t kLazySampleSplitStateCookieBase = 0x514b53504c495400ULL; +#endif + +// task_id % 5 即 kind；该周期性是不变量，既决定 Claim cursor/active role， +// 也决定输出大小、fanin 拓扑和 winner workload 的选择。 +enum class TaskKind : uint32_t { + Alloc = 0, + Qk = 1, + Sf = 2, + Pv = 3, + Up = 4, + Count = 5, +}; + +// 记录 kernel 最终在哪次 drain 中落地：Submit 开头、slot/heap 背压期间，或 +// 所有 worker 回放结束后的最终清空。三者之和必须等于实际 kernel 数。 +enum class DrainPlace : uint32_t { + EfDrain = 0, + RingBackpressure = 1, + FinalDrain = 2, + Count = 3, +}; + +enum class TensorArgType : int32_t { + Input = 0, + Output = 1, + Inout = 2, + OutputExisting = 3, + NoDependency = 4, +}; +// Input 作为 kernel 输入并参与依赖、但不登记为写者；Output 由本次 Submit 在 heap 中物化； +// Inout 与 OutputExisting 还需登记进每 worker 私有 TensorMap，供后续重叠区间查询 producer。 +static_assert(sizeof(TensorArgType) == sizeof(int32_t), "TensorArgType must match the PA tag ABI"); + +enum class DataType : uint8_t { + Float32 = 0, + Float16 = 1, + Int32 = 2, + Int16 = 3, + Int8 = 4, + Uint8 = 5, + Bfloat16 = 6, + Int64 = 7, + Uint64 = 8, + Uint16 = 9, + Uint32 = 10, + Bool = 11, + Count = 12, +}; + +// ProfilePhase 是聚合计数下标，TracePhase 是原始泳道事件 ABI；二者故意分离， +// 不能假设枚举值相同。一次 trace 写入可同时归入一个不同命名的 profile 阶段。 +enum class ProfilePhase : uint32_t { + Orchestration = 0, + Submit = 1, + EfDrain = 2, + Materialize = 3, + PrepareMap = 4, + Claim = 5, + Fanin = 6, + Register = 7, + WaitForSlot = 8, + HeapGuard = 9, + Build = 10, + ReplayTail = 11, + Count = 12, +}; + +// submit-pmu 每个 ELF 只编译一个局部归因阶段。none 不做中途 counter +// 读取，是完整 Submit 的正式基线；其余阶段都在每个 worker 的五次 Submit +// 上各执行一次，因此统一按固定 5*batches 次数闭合。历史 ID=3 曾用于 +// winner-only WaitForSlot，现已退役且不复用，避免旧 raw 被误认成新阶段。 +enum class SubmitPmuPhase : uint32_t { + None = 0, + Claim = 1, + EfDrain = 2, + Materialize = 4, + Register = 5, + Count = 6, +}; + +#ifndef PA_SUBMIT_PMU_PHASE_ID +#define PA_SUBMIT_PMU_PHASE_ID 0 +#endif + +constexpr SubmitPmuPhase kCompiledSubmitPmuPhase = + static_cast(PA_SUBMIT_PMU_PHASE_ID); +constexpr uint32_t kBuildVariantSwimlane = 1U; +constexpr uint32_t kBuildVariantSubmitPmu = 2U; +static_assert( + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::None) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::Claim) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::EfDrain) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::Materialize) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::Register), + "invalid compiled submit-pmu phase" +); + +struct NopCounts { + uint32_t qk; + uint32_t sf; + uint32_t pv; + uint32_t up; +}; + +// winner 的计算负载与 NOP 校准量使用两套独立计数，禁止把同一个数字同时解释成 +// scalar 指令条数和 vector/cube 工作迭代数。首阶段只有 CCEC 实现 RealCompute； +// 该 ABI 放在公共模型中，便于后续按相同配置逐步迁移 AscendC 与 CPU。 +struct WorkloadCounts { + uint32_t qk; + uint32_t sf; + uint32_t pv; + uint32_t up; +}; +static_assert(sizeof(WorkloadCounts) == 16, "workload counts ABI changed"); + +enum class WinnerWorkloadMode : uint32_t { + ScalarNop = 0, + RealCompute = 1, +}; + +constexpr uint32_t kWinnerWorkloadConfigVersion = 1; + +// 真实计算工作区是 standalone sidecar，不属于生产 DistGlobal/DistCore ABI。 +// workspace_base 指向 host 单独申请并初始化的 GM；每个 worker 只写自己的输出片段。 +struct alignas(64) WinnerWorkloadConfig { + uint32_t mode; + uint32_t version; + WorkloadCounts repeats; + uint64_t workspace_base; + uint64_t workspace_bytes; + uint32_t reserved[6]; +}; +static_assert(sizeof(WinnerWorkloadConfig) == 64, "winner workload config must occupy one cache line"); +static_assert(offsetof(WinnerWorkloadConfig, mode) == 0, "winner workload mode offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, version) == 4, "winner workload version offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, repeats) == 8, "winner workload counts offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, workspace_base) == 24, "winner workload base offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, workspace_bytes) == 32, "winner workload bytes offset changed"); + +// RunConfig 是 host 在 launch 前写、worker 启动时只读的控制 cache line。 +// 输入为 batch/NOP/诊断开关；输出不回写这里，而发布到独立 WorkerResult。 +struct alignas(64) RunConfig { + uint32_t batches; + uint32_t workers; + NopCounts nops; + uint32_t profile_phases; + uint32_t trace_enabled; + uint64_t trace_base; + uint32_t trace_records_per_core; + uint32_t reserved[5]; +}; +static_assert(sizeof(RunConfig) == 64, "RunConfig must occupy one cache line"); + +enum class TracePhase : int32_t { + Kernel = 0, + Alloc = 1, + Build = 2, + DrainWon = 3, + Replay = 4, + RingBp = 5, + EfDrain = 6, + Commit = 7, + Submit = 8, + Materialize = 9, + PrepareMap = 10, + Claim = 11, + Fanin = 12, + Register = 13, + Atomic = 14, + // 逐 atomic 诊断构建中，每个 worker 只记录一次连续两次 SYS_CNT 的 + // 空括号，用来给出同一二进制、同一物理核上的计时分辨率下限。 + ClockBaseline = 15, + // schema-v4 追加的父区间与真实动作区间。loser 没有可单列的真实动作， + // 其时间直接归入离线计算的 Submit residual，不占用 raw 记录。 + OrchestrationReplay = 16, + FinalDrain = 17, + WinnerBuild = 18, + AllocComplete = 19, + Count = 20, +}; + +// AtomicSite 按 standalone PA 中真实出现的源码调用点分类。编号写入 TraceRecord::auxiliary， +// 是离线泳道 schema 的一部分；追加新位置只能在 Count 前扩展，不能重排既有值。 +enum class AtomicSite : uint32_t { + StartupIncrement = 0, + StartupPoll = 1, + FatalPoll = 2, + FatalSet = 3, + ClaimMax = 4, + FaninFlagLoad = 5, + CompletionVendExchange = 6, + CompletionFlagExchange = 7, + FrontierInitialLoad = 8, + FrontierFlagLoad = 9, + FrontierMax = 10, + HeapFrontierLoad = 11, + HeapVendLoad = 12, + ReplayDoneIncrement = 13, + ReplayDonePoll = 14, + Count = 15, +}; + +// Atomic 记录 flags 的低四位保存操作种类；bit4 表示返回值参与后续判断， +// bit5 表示 Load 观察到零，bit6 表示结束时间已由返回值依赖推进到 +// return-ready 边界。schema-v3 中 bit7 区分等待区 PollBatch：此时 +// bits[31:8] 是精确调用次数；直接 FetchMax 中同一区域仍表示软件重试数。 +enum class AtomicOp : uint32_t { + Load = 0, + Exchange = 1, + FetchAdd = 2, + FetchMax = 3, +}; +constexpr uint32_t kAtomicOpMask = 0x0fU; +constexpr uint32_t kAtomicResultUsed = 1U << 4; +constexpr uint32_t kAtomicValueZero = 1U << 5; +constexpr uint32_t kAtomicReturnReady = 1U << 6; +constexpr uint32_t kAtomicPollBatch = 1U << 7; +constexpr uint32_t kAtomicRetriesShift = 8; +constexpr uint32_t kAtomicPollCountShift = 8; +constexpr uint32_t kAtomicPollCountMax = 0x00ffffffU; +constexpr uint32_t kAtomicPollBatchSiteCount = 6; + +// 这些映射是 raw ABI 的一部分，同时被 device 聚合器与 host 闭环校验使用。 +// 0..14 与真实 PA 保持稳定；BlockWon 尚未在 standalone 中实现，不能只为 +// 编号齐全而追加没有真实调用路径的 site。 +#ifdef PA_DEVICE +#define PA_MODEL_INLINE PA_DEVICE +#else +#define PA_MODEL_INLINE inline +#endif + +PA_MODEL_INLINE constexpr AtomicOp AtomicSiteExpectedOp(AtomicSite site) { + switch (site) { + case AtomicSite::StartupIncrement: + case AtomicSite::ReplayDoneIncrement: + return AtomicOp::FetchAdd; + case AtomicSite::FatalSet: + case AtomicSite::CompletionVendExchange: + case AtomicSite::CompletionFlagExchange: + return AtomicOp::Exchange; + case AtomicSite::ClaimMax: + case AtomicSite::FrontierMax: + return AtomicOp::FetchMax; + default: + return AtomicOp::Load; + } +} + +PA_MODEL_INLINE constexpr bool AtomicSiteResultUsed(AtomicSite site) { + switch (site) { + case AtomicSite::StartupIncrement: + case AtomicSite::FatalSet: + case AtomicSite::CompletionVendExchange: + case AtomicSite::CompletionFlagExchange: + case AtomicSite::ReplayDoneIncrement: + return false; + default: + return true; + } +} + +PA_MODEL_INLINE constexpr int32_t AtomicPollBatchIndex(AtomicSite site) { + switch (site) { + case AtomicSite::StartupPoll: + return 0; + case AtomicSite::FatalPoll: + return 1; + case AtomicSite::FaninFlagLoad: + return 2; + case AtomicSite::HeapFrontierLoad: + return 3; + case AtomicSite::HeapVendLoad: + return 4; + case AtomicSite::ReplayDonePoll: + return 5; + default: + return -1; + } +} + +PA_MODEL_INLINE constexpr AtomicSite AtomicPollBatchSite(uint32_t index) { + switch (index) { + case 0: + return AtomicSite::StartupPoll; + case 1: + return AtomicSite::FatalPoll; + case 2: + return AtomicSite::FaninFlagLoad; + case 3: + return AtomicSite::HeapFrontierLoad; + case 4: + return AtomicSite::HeapVendLoad; + case 5: + return AtomicSite::ReplayDonePoll; + default: + return AtomicSite::Count; + } +} + +PA_MODEL_INLINE constexpr bool AtomicSiteIsPollBatchable(AtomicSite site) { + return AtomicPollBatchIndex(site) >= 0; +} + +PA_MODEL_INLINE constexpr uint32_t AtomicSiteMask(AtomicSite site) { + return 1U << static_cast(site); +} + +#undef PA_MODEL_INLINE + +// ClockBaseline 的 bit0 区分普通连续 SYS_CNT 与后端的 atomic 返回依赖 +// 计时钩子；后者用于量化那一条依赖 MOV 自身带来的固定底噪。 +constexpr uint32_t kClockAtomicDependency = 1U << 0; +constexpr uint32_t kClockAtomicDependencyApplied = 1U << 1; + +struct alignas(64) TraceCoreState { + volatile uint32_t count; + volatile uint32_t dropped; + // logical atomic 调用数与物理记录数分开闭合：PollBatch 的一条记录可以 + // 表示多次只读轮询，physical = atomic_calls - poll_calls + batch_records。 + volatile uint32_t atomic_calls; + volatile uint32_t poll_calls; + volatile uint32_t poll_batch_records; + // 拓扑在一个 worker 分区内恒定；当前仍保留 64B TraceRecord 兼容布局， + // 但在 core state 再保存一份权威身份，host 会验证每条记录与之相符。 + volatile int32_t core_idx; + volatile int32_t block_id; + volatile int32_t lane; + uint32_t padding[8]; +}; +// 每个 worker 独占一个计数 cache line 和一段定长 records，不需要为了写 trace +// 再引入跨核 atomic；满容量后只增加本 worker 的 dropped。 +static_assert(sizeof(TraceCoreState) == 64, "trace core state must occupy one cache line"); + +struct alignas(64) TraceHeader { + uint32_t magic; + uint32_t version; + uint32_t num_cores; + uint32_t records_per_core; + uint64_t frequency_hz; + TraceCoreState cores[kRuntimeMaxWorkers]; +}; +// 本 benchmark 固定物理分配 kWorkers=96 个定长 record 分区，合法 header 也要求 +// num_cores==96；其 header/record 布局和 phase 编号可转换为真实泳道使用的 JSON。 +static_assert(sizeof(TraceHeader) == 6976, "trace header must match PA swimlane layout"); + +struct alignas(64) TraceRecord { + uint64_t start_cycle; + uint64_t end_cycle; + int32_t task_id; + int32_t function_id; + int32_t phase; + int32_t lane; + int32_t block_id; + int32_t core_idx; + uint32_t flags; + uint32_t auxiliary; +}; +// start/end 保留原始 1 GHz counter；task/function/物理 lane 用于离线还原轨道。 +// flags/aux 的含义由 phase 决定，例如 winner、Alloc 或 RingBp 类型，不参与调度决策。 +static_assert(sizeof(TraceRecord) == 64, "trace record must occupy one cache line"); + +constexpr size_t kTraceBytes = + sizeof(TraceHeader) + static_cast(kWorkers) * kTraceRecordsPerCore * sizeof(TraceRecord); + +struct alignas(64) AtomicLine { + volatile int64_t value; + uint8_t padding[64 - sizeof(int64_t)]; +}; +// 热点共享量各占一个 cache line，保持生产代码的地址隔离，避免 standalone +// 因伪共享额外放大 Claim/frontier/start barrier 的竞争。 +static_assert(sizeof(AtomicLine) == 64, "AtomicLine must occupy one cache line"); + +struct alignas(64) AtomicFlagLine { + volatile int32_t value; + uint8_t padding[64 - sizeof(int32_t)]; +}; +// 32-bit fatal 与 64-bit cursor 使用不同封装，但都独占 cache line；成功路径中 +// fatal 始终为零，任何写一都表示协议已终止，不能作为普通等待条件清除。 +static_assert(sizeof(AtomicFlagLine) == 64, "AtomicFlagLine must occupy one cache line"); + +struct alignas(64) TaskCell { + volatile int64_t flag; + volatile uint64_t vend; + uint8_t padding[64 - 2 * sizeof(int64_t)]; +}; +// flag 是依赖就绪与 frontier 连续前推的发布位；vend 是该 task 完成时 worker 的 +// 单调 heap_next 快照。HeapGuard 读取 frontier-H 对应 vend，判断环形 heap 是否可覆盖。 +static_assert(sizeof(TaskCell) == 64, "TaskCell must occupy one cache line"); + +// TensorDesc 保留真实 Tensor 的两条 64-byte 数据线。owner_task_id 表达显式生产者， +// buffer_addr + 字节区间用于 TensorMap 发现同一 backing buffer 上的读写依赖。 +struct TensorDesc { + uint64_t buffer_addr; + uint64_t buffer_size; + uint64_t owner_task_id; + uint64_t start_offset; + int32_t version; + uint32_t ndims; + DataType dtype; + bool manual_dep; + bool is_contiguous; + uint8_t child_memory; + uint32_t shapes[kMaxTensorDims]; + + uint64_t extent_elem_cache; + uint32_t strides[kMaxTensorDims]; + uint8_t padding[36]; +}; +static_assert(sizeof(TensorDesc) == 128, "TensorDesc must match the PA Tensor ABI size"); +static_assert(offsetof(TensorDesc, buffer_addr) == 0, "TensorDesc buffer offset mismatch"); +static_assert(offsetof(TensorDesc, owner_task_id) == 16, "TensorDesc owner offset mismatch"); +static_assert(offsetof(TensorDesc, start_offset) == 24, "TensorDesc view offset mismatch"); +static_assert(offsetof(TensorDesc, version) == 32, "TensorDesc version offset mismatch"); +static_assert(offsetof(TensorDesc, shapes) == 44, "TensorDesc shape offset mismatch"); +static_assert(offsetof(TensorDesc, extent_elem_cache) == 64, "TensorDesc extent offset mismatch"); +static_assert(offsetof(TensorDesc, strides) == 72, "TensorDesc stride offset mismatch"); + +struct TensorCreateInfo { + uint64_t initial_value; + bool has_initial_value; + uint8_t padding0[7]; + uint64_t reserved0; + uint64_t start_offset; + int32_t version; + uint32_t ndims; + DataType dtype; + bool manual_dep; + bool is_contiguous; + uint8_t child_memory; + uint32_t shapes[kMaxTensorDims]; +}; +// CreateInfo 只描述尚未分配的 Output；Materialize 根据形状和 dtype 计算大小， +// 再把它变成位于 worker 逻辑 heap 上的 TensorDesc。 +static_assert(sizeof(TensorCreateInfo) == 64, "TensorCreateInfo must match the PA create-info ABI size"); +static_assert(offsetof(TensorCreateInfo, start_offset) == 24, "TensorCreateInfo start offset mismatch"); +static_assert(offsetof(TensorCreateInfo, version) == 32, "TensorCreateInfo version offset mismatch"); +static_assert(offsetof(TensorCreateInfo, shapes) == 44, "TensorCreateInfo shape offset mismatch"); + +struct MapEntry { + uint64_t buffer_addr; + uint64_t lo; + uint64_t hi; + int32_t producer; + int32_t bucket; + int32_t next_in_bucket; + int32_t prev_in_bucket; + int32_t next_in_task; +}; +// 同一 entry 同时挂在两条链上：bucket 链按 buffer 地址查询重叠区间，task 链按 +// producer 批量退休。next_in_bucket 在空闲状态下复用为 free-list 链接。 +static_assert(sizeof(MapEntry) == 48, "MapEntry must match the PA tensor-map entry ABI"); +static_assert(offsetof(MapEntry, producer) == 24, "MapEntry producer offset mismatch"); +static_assert(offsetof(MapEntry, next_in_task) == 40, "MapEntry task-link offset mismatch"); + +struct TensorMap { + MapEntry entries[kMapCapacity]; + int32_t buckets[kMapBuckets]; + int32_t task_heads[kTaskWindow]; + int32_t free_head; + int32_t high_water; + int32_t alive_floor; + int32_t cleaned_upto; +}; +// TensorMap 是 worker 私有状态，不在多核间共享。alive_floor 表达查询存活下界， +// cleaned_upto 表达已物理摘链的进度；即使 Case1 中通常同步推进，也不能合并其 ABI 字段。 +static_assert(sizeof(TensorMap) == 823312, "TensorMap must match the PA fixed-capacity layout"); +static_assert(offsetof(TensorMap, buckets) == 786432, "TensorMap bucket offset mismatch"); +static_assert(offsetof(TensorMap, task_heads) == 819200, "TensorMap task-head offset mismatch"); +static_assert(offsetof(TensorMap, free_head) == 823296, "TensorMap control offset mismatch"); + +struct TaskPayload { + TensorDesc tensors[kMaxTaskTensors]; +}; +// task_id 通过 kPayloadMask 映射到 2048 个 4 KiB payload；Case1 只有 1280 个 task， +// 本轮不会回绕，但仍保留生产容量、寻址方式和 4 KiB stride。 +static_assert(sizeof(TaskPayload) == kPayloadStride, "TaskPayload must preserve the real 4 KiB task stride"); +static_assert(alignof(TaskPayload) == 8, "TaskPayload alignment must match DistTaskPayload"); +static_assert(offsetof(TaskPayload, tensors) == 0, "TaskPayload tensor offset mismatch"); + +struct LocalSlot { + // occupied 先保留容量，built 表示 payload 已按生产顺序构建；task/function + // 标识决定执行哪个 NOP 体，后续大数组则是 kernel 真正看到的参数快照。 + bool occupied; + bool built; + uint8_t header_padding[2]; + uint32_t task_id; + uint32_t kind; + uint32_t function_padding; + uint64_t function_address; + uint32_t tensor_count; + uint32_t scalar_count; + uint8_t tensor_padding[32]; + + TensorDesc tensors[kMaxTaskTensors]; + uint64_t scalars[kMaxTaskScalars]; + uint64_t args[kMaxTaskTensors + kMaxTaskScalars + 2]; + union { + struct { + uint8_t local_context[48]; + uint32_t global_context; + int32_t fanin[kMaxFanin]; + uint32_t fanin_count; + }; + // Compatibility view used by the standalone NOP payload builder. The + // first six words are the real 48-byte LocalContext; the remaining + // words overlap GlobalContext and the beginning of fanin, exactly as + // dictated by the real RingSlot offsets. + // 该视图只用于按真实 offset 填充 dispatch context，不增加另一份 + // 存储；修改其中后两字会同步覆盖 GlobalContext/fanin 的对应 ABI 字节。 + uint64_t context_words[8]; + }; + bool is_multicore; + int32_t won_block; + int32_t won_slot; +}; +// LocalSlot 是每个 winner 写入自己私有 ring 的完整 dispatch 包。fanin 在执行前 +// 逐项检查 task.flag；occupied/built 与计数共同约束最多两个普通 kernel 在途。 +static_assert(sizeof(LocalSlot) == 4824, "LocalSlot must match the PA RingSlot ABI size"); +static_assert(alignof(LocalSlot) == 8, "LocalSlot alignment must match RingSlot"); +static_assert(offsetof(LocalSlot, occupied) == 0, "LocalSlot occupied offset mismatch"); +static_assert(offsetof(LocalSlot, built) == 1, "LocalSlot built offset mismatch"); +static_assert(offsetof(LocalSlot, task_id) == 4, "LocalSlot task offset mismatch"); +static_assert(offsetof(LocalSlot, kind) == 8, "LocalSlot function-id offset mismatch"); +static_assert(offsetof(LocalSlot, function_address) == 16, "LocalSlot function address offset mismatch"); +static_assert(offsetof(LocalSlot, tensor_count) == 24, "LocalSlot tensor-count offset mismatch"); +static_assert(offsetof(LocalSlot, tensors) == 64, "LocalSlot tensor payload offset mismatch"); +static_assert(offsetof(LocalSlot, scalars) == 4160, "LocalSlot scalar payload offset mismatch"); +static_assert(offsetof(LocalSlot, args) == 4288, "LocalSlot dispatch-args offset mismatch"); +static_assert(offsetof(LocalSlot, local_context) == 4688, "LocalSlot local-context offset mismatch"); +static_assert(offsetof(LocalSlot, global_context) == 4736, "LocalSlot global-context offset mismatch"); +static_assert(offsetof(LocalSlot, fanin) == 4740, "LocalSlot fanin offset mismatch"); +static_assert(offsetof(LocalSlot, fanin_count) == 4804, "LocalSlot fanin-count offset mismatch"); +static_assert(offsetof(LocalSlot, is_multicore) == 4808, "LocalSlot multicore offset mismatch"); +static_assert(offsetof(LocalSlot, won_block) == 4812, "LocalSlot won-block offset mismatch"); +static_assert(offsetof(LocalSlot, won_slot) == 4816, "LocalSlot won-slot offset mismatch"); + +struct WorkerState { + CoreRole role; + int32_t core_idx; + int32_t block_id; + int32_t lane; + int32_t sub_block_id; + int32_t local_index; + uint64_t heap_next; + TensorMap map; + uint8_t slot_padding[16]; + LocalSlot slots[kPrivateSlots]; + uint32_t occupied_count; + uint32_t owned_total; + uint64_t swimlane_last_cycle; + uint8_t payload_padding[16]; + TaskPayload payloads[kPayloadSlots]; +}; +// 每个物理 worker 都持有独立 heap cursor、TensorMap、ring 与 task payload arena； +// 多核共享的只有 SchedulerState 前缀中的 cursor/task/frontier 等协议状态。 +static_assert(sizeof(WorkerState) == 9231296, "WorkerState must match the PA DistCore ABI size"); +static_assert(alignof(WorkerState) == 8, "WorkerState alignment must match DistCore"); +static_assert(offsetof(WorkerState, role) == 0, "WorkerState role offset mismatch"); +static_assert(offsetof(WorkerState, local_index) == 20, "WorkerState replay-index offset mismatch"); +static_assert(offsetof(WorkerState, heap_next) == 24, "WorkerState heap cursor offset mismatch"); +static_assert(offsetof(WorkerState, map) == 32, "WorkerState tensor-map offset mismatch"); +static_assert(offsetof(WorkerState, slots) == 823360, "WorkerState ring-slot offset mismatch"); +static_assert(offsetof(WorkerState, occupied_count) == 842656, "WorkerState occupancy offset mismatch"); +static_assert(offsetof(WorkerState, owned_total) == 842660, "WorkerState owned-count offset mismatch"); +static_assert(offsetof(WorkerState, swimlane_last_cycle) == 842664, "WorkerState trace clock offset mismatch"); +static_assert(offsetof(WorkerState, payloads) == 842688, "WorkerState task-payload offset mismatch"); + +struct alignas(64) WorkerResult { + // 时间边界：Submit 口径不含启动屏障和最终 drain，finish_cycle 则覆盖完整 worker 生命周期。 + uint64_t submit_begin; + uint64_t submit_end; + uint64_t finish_cycle; + uint64_t checksum; + + // 协议计数用于验证固定 Claim 拓扑及等待/依赖动态次数；joint_polls 是为未来 + // BlockWon 模拟保留的兼容字段，当前实现没有递增点，不能据其检测 joint 分支。 + uint64_t submits; + uint64_t claim_attempts; + uint64_t claim_wins; + uint64_t heap_guards; + uint64_t fanin_ready_loads; + uint64_t completion_duplicates; + uint64_t cas_retries; + uint64_t joint_polls; + + // 默认 256 batch 时 winner、kernel 分别闭合到 1280 task 和 1024 kernel； + // 非默认配置按 5*batches、4*batches 计算，placement 仍闭合到全部 kernel。 + uint64_t wins[static_cast(TaskKind::Count)]; + uint64_t kernel_counts[4]; + uint64_t kernel_cycles[4]; + uint64_t kernel_min_cycles[4]; + uint64_t kernel_max_cycles[4]; + uint64_t placement[static_cast(DrainPlace::Count)]; + uint64_t phase_cycles[static_cast(ProfilePhase::Count)]; + uint64_t phase_calls[static_cast(ProfilePhase::Count)]; + uint64_t wait_events[2]; + uint64_t wait_iterations[2]; + + // 前端工作量计数不是性能填充：构参、materialize 和 map insert 用于核对全部 + // 96 个 worker 的回放；map lookup、slot copy 与 fanin 则是 winner-only 全局计数。 + uint64_t context_reads; + uint64_t views_created; + uint64_t dynamic_create_infos; + uint64_t arg_resets; + uint64_t tensor_args_added; + uint64_t scalar_args_added; + uint64_t materialized_outputs; + uint64_t map_inserts; + uint64_t map_lookups; + uint64_t slot_tensor_copies; + uint64_t slot_scalar_copies; + uint64_t fanin_edges; + + // 最终快照用于跨 worker 比较逻辑 heap 与 TensorMap 回收状态是否完全一致。 + uint64_t final_heap_next; + uint64_t map_high_water; + uint64_t map_alive_floor; + uint64_t map_cleaned_upto; + uint64_t map_live_entries; + + uint64_t worker_id; + uint64_t role; + uint64_t max_occupied; + uint64_t final_occupied; + + // CCEC 标量 PMU 取证使用 WorkerResult 的诊断 sidecar，不改变生产 DistCore ABI。 + // 该诊断只在显式开启时有效；CNT2/CNT6/CNT7 分别对应 scalar busy、I-cache req/miss。 + uint64_t pmu_total_cycles; + uint32_t pmu_scalar_busy; + uint32_t pmu_icache_requests; + uint32_t pmu_icache_misses; + uint32_t pmu_status; + + // 这些计数只在 worker 私有 LocalStats 中递增，结束时一次性发布；它们把动态 + // fanin 重试和 frontier helping 展开为准确次数，不为取数再增加共享 atomic。 + uint64_t fanin_not_ready_loads; + uint64_t frontier_initial_loads; + uint64_t frontier_updates; + uint64_t frontier_terminal_loads; + + // 仅在 trace_enabled bit1 开启时递增；每次源码 atomic 调用恰好增加一， + // host 用它与 Atomic span 数逐 worker 闭合，禁止把丢记录的泳道当成完整结果。 + uint64_t atomic_trace_calls; + + // I-cache 单 miss 探针用该槽保存 cold 窗口的 1 GHz SYS_CNT；submit-pmu + // 复用同一 64-bit 槽保存所选局部阶段的逐调用累计时间。两种构建互斥， + // 因而无需扩大 832B WorkerResult，也不会改变相邻 worker 的 cache-line 布局。 + union { + uint64_t pmu_window_ticks; + uint64_t pmu_phase_elapsed_ticks; + }; + uint64_t pmu_warm_total_cycles; + uint64_t pmu_warm_window_ticks; + union { + uint32_t pmu_warm_icache_requests; + uint32_t pmu_phase_begin_reads; + }; + union { + uint32_t pmu_warm_icache_misses; + uint32_t pmu_phase_end_reads; + }; + + // PIPE_UTILIZATION 已同时配置 CNT0/1/3/4/5/8；与上面的 scalar/I-cache + // 一样只保存每核原始累计值，AIC/AIV 汇总与比率统一在 host sidecar 中计算。 + // 六个 32-bit 值复用本结构扩展到 832B 后的尾部空间，不再增加 cache line。 + uint32_t pmu_vector_busy; + uint32_t pmu_cube_busy; + uint32_t pmu_mte1_busy; + uint32_t pmu_mte2_busy; + // swimlane ABI 保留该槽；submit-pmu 将物理 CNT5 改作 shadow miss， + // 因而显式发布 0，并在当前 submit-pmu schema-v5 标记 mte3_busy 不可用。 + uint32_t pmu_mte3_busy; + uint32_t pmu_fix_busy; + + // 复用 WorkerResult 原有的 32B cache-line 尾洞，不扩大 832B stride。 + // CNT6/7 是从不中途读取的权威整窗，CNT8/CNT5 是 read-to-clear shadow； + // none 在 stop 后要求逐核精确相等；运行中切片的 phase 只允许 shadow + // 单向小于 primary，并显式导出差值形成局部观测区间。 + uint32_t pmu_build_variant; + uint32_t pmu_phase_id; + uint32_t pmu_phase_calls; + uint32_t pmu_phase_status; + uint32_t pmu_phase_icache_requests; + uint32_t pmu_phase_icache_misses; + uint32_t pmu_shadow_icache_requests; + uint32_t pmu_shadow_icache_misses; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + // split 组合 oracle 独占一条诊断 cache line；legacy 与 inline shape 完全 + // 不带这些字段，避免改变既有 WorkerResult/SchedulerState 的布局。 + uint64_t lazy_sample_split_caller_state_address; + uint64_t lazy_sample_split_finish_state_address; + uint64_t lazy_sample_split_finish_calls; + uint64_t lazy_sample_split_protocol_errors; + uint64_t lazy_sample_split_state_cookie; + uint64_t lazy_sample_split_task_id_sum; + uint64_t lazy_sample_split_owner_worker_id; + uint64_t lazy_sample_split_reserved; +#endif +}; +// WorkerResult 是 standalone 尾部的诊断 sidecar，不属于真实 DistCore ABI；按 +// cache line 隔离后，各 worker 发布统计不会相互覆盖或污染被测共享状态。 +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +static_assert(sizeof(WorkerResult) == 896, "split WorkerResult diagnostics must occupy whole cache lines"); +static_assert(offsetof(WorkerResult, lazy_sample_split_caller_state_address) == 832, + "split WorkerResult oracle offset mismatch"); +static_assert(offsetof(WorkerResult, lazy_sample_split_reserved) == 888, + "split WorkerResult oracle tail mismatch"); +#else +static_assert(sizeof(WorkerResult) == 832, "WorkerResult diagnostics must occupy whole cache lines"); +#endif +static_assert(offsetof(WorkerResult, pmu_total_cycles) == 680, "WorkerResult PMU offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_status) == 700, "WorkerResult PMU status offset mismatch"); +static_assert(offsetof(WorkerResult, fanin_not_ready_loads) == 704, "WorkerResult atomic diagnostic offset mismatch"); +static_assert(offsetof(WorkerResult, atomic_trace_calls) == 736, "WorkerResult atomic trace offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_window_ticks) == 744, "WorkerResult PMU timing offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_vector_busy) == 776, "WorkerResult extended PMU offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_build_variant) == 800, "WorkerResult submit-PMU offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_shadow_icache_misses) == 828, "WorkerResult submit-PMU tail mismatch"); + +// 从 cube_cursor 到 workers 结束保留关键字段 offset、DistCore ABI 和生产总字节跨度， +// 并非字段级完整镜像。RunConfig、输入 context_lens 与校验结果追加在该跨度之后， +// 因此测试控制信息不会改变被测字段 offset。 +struct alignas(64) SchedulerState { + // 三组四分片 cursor 分别服务 AIC kernel、AIV kernel 和 Alloc；同 task 的 + // eligible workers 竞争同一 shard，只有旧值小于 task_id 的调用成为 winner。 + AtomicLine cube_cursor[kCursorShards]; + AtomicLine vector_cursor[kCursorShards]; + AtomicLine alloc_cursor[kCursorShards]; + AtomicLine frontier; + int32_t heap_window; + uint8_t tasks_padding[60]; + TaskCell tasks[kTaskCellCapacity]; + // heap_base/size 描述共享物理环，worker.heap_next 则是各 worker 一致推进的逻辑游标。 + uint64_t heap_base; + uint64_t heap_size; + uint64_t orchestration_args; + uint64_t runtime_state; + uint64_t runtime; + uint8_t fatal_padding[24]; + AtomicFlagLine fatal; + int32_t num_workers; + int32_t num_blocks; + // Case1 never enters BlockWon, but the inactive layout and BlockWon arena + // remains byte-for-byte reserved so every subsequent PA atomic line keeps + // its production offset. + // 此处不能因 Case1 动态次数为零而删减，否则 replay_done、started_count + // 和 DistCore 数组整体前移，便不再是对真实 PA 地址布局的等价测试。 + uint8_t layout_and_block_won[5848440]; + AtomicLine replay_done; + AtomicLine started_count; + // started_count 形成 launch 屏障；replay_done 只用于最终 drain 判定所有 worker + // 已不再产生新 slot。两者都位于 Submit 性能口径之外，但属于完整协议。 + WorkerState workers[kRuntimeMaxWorkers]; + // Standalone-only controls live after the complete DistGlobal image. They + // therefore do not shift any cursor/task/fatal/worker address under test. + RunConfig config; + WinnerWorkloadConfig winner_workload; + // Context lengths are the only PA input elements read by orchestration; + // keeping them in GM preserves the per-batch descriptor-based load. + // 除这 256 个长度值外，其余 tensor 仅需稳定的合成地址来复现 + // descriptor、依赖和 heap 行为，不会解引用成真实计算数据。 + volatile int32_t context_lens[kMaxBatches]; + WorkerResult results[kWorkers]; +}; +static_assert(offsetof(SchedulerState, cube_cursor) == 0, "cube cursor offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, vector_cursor) == 256, "vector cursor offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, alloc_cursor) == 512, "alloc cursor offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, frontier) == 768, "frontier offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, heap_window) == 832, "H offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, tasks) == kRealTasksOffset, "task table offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, heap_base) == 4195200, "heap base offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, heap_size) == 4195208, "heap size offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, fatal) == kRealFatalOffset, "fatal offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, replay_done) == kRealReplayDoneOffset, "replay offset must match PA DistGlobal"); +static_assert( + offsetof(SchedulerState, started_count) == kRealStartedCountOffset, + "started-count offset must match PA DistGlobal" +); +static_assert(offsetof(SchedulerState, tasks) % 64 == 0, "task table must be cache-line aligned"); +static_assert(offsetof(SchedulerState, workers) % 64 == 0, "worker table must be cache-line aligned"); +static_assert(offsetof(SchedulerState, results) % 64 == 0, "result table must be cache-line aligned"); +static_assert(offsetof(SchedulerState, workers) == kRealDistCoreOffset, "DistCore table offset must match PA"); +static_assert(offsetof(SchedulerState, config) == kRealDistGlobalBytes, "DistGlobal byte size must match PA"); +static_assert( + offsetof(SchedulerState, winner_workload) == kRealDistGlobalBytes + sizeof(RunConfig), + "winner workload sidecar must follow RunConfig" +); +static_assert( + offsetof(SchedulerState, context_lens) == + kRealDistGlobalBytes + sizeof(RunConfig) + sizeof(WinnerWorkloadConfig), + "context lengths must follow standalone controls" +); + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_MODEL_H diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/pa_scheduler_core.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/pa_scheduler_core.h new file mode 100644 index 0000000000..afdd0896ce --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/pa_scheduler_core.h @@ -0,0 +1,1618 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_SCHEDULER_CORE_H +#define PA_SCHEDULER_COMMON_PA_SCHEDULER_CORE_H + +#ifndef PA_DEVICE +#define PA_DEVICE inline +#endif + +#ifndef PA_GM +#define PA_GM +#endif + +#include "pa_frontend.h" +#include "pa_trace.h" + +namespace pa_scheduler { + +struct LocalStats { + WorkerResult result; + uint32_t max_occupied; + TraceContext trace; +}; + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +// runtime TU owns one external [[block_local]] instance per architecture. +// The caller imports that same object and keeps all Submit-internal context in +// it, so the only cross-TU function arguments are a POD ticket and built args. +struct alignas(64) LazySampleSplitRuntimeState { + PA_GM SchedulerState *scheduler; + PA_GM WorkerState *worker; + uint32_t task_count; + uint32_t worker_id; + SubmitContext context; + LocalStats stats; + uint64_t caller_state_address; + uint64_t finish_state_address; + uint64_t finish_calls; + uint64_t protocol_errors; + uint64_t state_cookie; + uint64_t task_id_sum; + uint64_t owner_worker_id; + uint64_t reserved; +}; +static_assert(sizeof(LazySampleSplitRuntimeState) % 64 == 0, + "split runtime state must occupy whole cache lines"); + +PA_DEVICE uint64_t LazySampleSplitStateCookie(uint32_t worker_id, CoreRole role) { + return kLazySampleSplitStateCookieBase ^ static_cast(worker_id) ^ + (static_cast(static_cast(role)) << 32U); +} +#endif + +// submit-pmu 的 phase 在编译期固定；非诊断构建完全不引用 Ops 的 phase +// 接口。这样公共调度代码保持一份，swimlane/CPU/AscendC 也不会多出运行时分支。 +template +PA_DEVICE void BeginSubmitPmuPhase(PmuContext &context) { +#if PA_BUILD_SUBMIT_PMU + if constexpr (kCompiledSubmitPmuPhase == Phase) { + Ops::PmuPhaseBegin(context); + } +#else + (void)context; +#endif +} + +template +PA_DEVICE void EndSubmitPmuPhase(PmuContext &context) { +#if PA_BUILD_SUBMIT_PMU + if constexpr (kCompiledSubmitPmuPhase == Phase) { + Ops::PmuPhaseEnd(context); + } +#else + (void)context; +#endif +} + +template +PA_DEVICE uint64_t TraceTimestamp(TraceContext &trace, WorkerResult &result) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + return 0; +#else + (void)result; + // 对齐真实 FDWIC 的 TRACE_SPAN_BEGIN/END：取一次时间后立即以同一 + // cycle 关闭活跃 PollBatch。不能在 WriteTrace 中统一关闭，否则直接 + // Atomic 记录也会错误切断等待 episode。 + const uint64_t cycle = Ops::Now(); + AtomicPollBoundaryAt(trace, cycle); + return cycle; +#endif +} + +PA_DEVICE uint32_t KindIndex(TaskKind kind) { return static_cast(kind); } + +PA_DEVICE TaskKind GetTaskKind(uint32_t task_id) { return static_cast(task_id % kTasksPerBatch); } + +PA_DEVICE int32_t FunctionId(TaskKind kind) { + return kind == TaskKind::Alloc ? -1 : static_cast(KindIndex(kind) - 1); +} + +PA_DEVICE uint32_t NopCountForKind(PA_GM const NopCounts &nops, TaskKind kind) { + switch (kind) { + case TaskKind::Qk: + return nops.qk; + case TaskKind::Sf: + return nops.sf; + case TaskKind::Pv: + return nops.pv; + case TaskKind::Up: + return nops.up; + default: + return 0; + } +} + +PA_DEVICE uint32_t WorkloadCountForKind(PA_GM const WorkloadCounts &counts, TaskKind kind) { + switch (kind) { + case TaskKind::Qk: + return counts.qk; + case TaskKind::Sf: + return counts.sf; + case TaskKind::Pv: + return counts.pv; + case TaskKind::Up: + return counts.up; + default: + return 0; + } +} + +PA_DEVICE uint32_t CountBits(uint32_t value) { + uint32_t count = 0; + while (value != 0) { + count += value & 1U; + value >>= 1; + } + return count; +} + +template +PA_DEVICE int64_t LoadLine( + PA_GM AtomicLine &line, LocalStats &stats, AtomicSite site, int32_t task_id = -1 +) { + // Ops::Load 在 A5 后端是 atomicAdd(0)；返回值是该 RMW 线性化时观察到的共享值。 + return TraceAtomicLoad(stats.trace, stats.result, task_id, site, &line.value); +} + +template +PA_DEVICE int32_t LoadLine( + PA_GM AtomicFlagLine &line, LocalStats &stats, AtomicSite site, int32_t task_id = -1 +) { + return TraceAtomicLoad(stats.trace, stats.result, task_id, site, &line.value); +} + +template +PA_DEVICE void SetFatal(PA_GM SchedulerState *state, LocalStats &stats, int32_t task_id = -1) { + // fatal 只从 0 单调置 1，重复 Exchange 不会把其他 worker 已观察到的失败状态清除。 + TraceAtomicExchange( + stats.trace, stats.result, task_id, AtomicSite::FatalSet, &state->fatal.value, + static_cast(1) + ); +} + +template +PA_DEVICE bool IsFatal(PA_GM SchedulerState *state, LocalStats &stats, int32_t task_id = -1) { + return LoadLine(state->fatal, stats, AtomicSite::FatalPoll, task_id) != 0; +} + +template +PA_DEVICE bool WatchdogExpired( + PA_GM SchedulerState *state, LocalStats &stats, uint64_t begin, uint32_t &polls +) { + // 每 1024 次自旋才读取系统计数器，降低正常启动屏障上的计时开销；超时后向所有 worker 广播 fatal。 + ++polls; + if ((polls & 1023U) != 0 || Ops::Now() - begin <= kWatchdogTicks) { + return false; + } + SetFatal(state, stats); + return true; +} + +template +PA_DEVICE void AdvanceFrontier(PA_GM SchedulerState *state, LocalStats &stats) { + // frontier 只表示“从 task 0 开始已经连续完成”的最高 task id，不能越过尚未发布 flag 的空洞。 + // 多个完成者可以同时扫描同一段连续区间，FetchMax 保证共享 frontier 只前进、不回退。 + ++stats.result.frontier_initial_loads; + int64_t frontier = LoadLine( + state->frontier, stats, AtomicSite::FrontierInitialLoad + ); + while (true) { + const int64_t next = frontier + 1; + if (next < 0 || next >= static_cast(kTaskCellCapacity)) { + break; + } + if (TraceAtomicLoad( + stats.trace, stats.result, static_cast(next), AtomicSite::FrontierFlagLoad, + &state->tasks[next].flag + ) == 0) { + ++stats.result.frontier_terminal_loads; + break; + } + uint64_t retries = 0; + // FetchMax 返回更新前的值；若其他核已经走得更远，就从其 old 值继续扫描，避免重复从 next 起步。 + ++stats.result.frontier_updates; + const int64_t old = TraceAtomicFetchMax( + stats.trace, stats.result, static_cast(next), AtomicSite::FrontierMax, + &state->frontier.value, next, retries + ); + stats.result.cas_retries += retries; + frontier = old > next ? old : next; + } +} + +template +PA_DEVICE void CompleteTask( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, LocalStats &stats +) { + // 完成发布顺序与 PA 一致：先公布该 worker 的 heap 游标，再发布 ready flag，最后推进连续 frontier。 + // fanin 和 heap 回收方以 flag/frontier 为可见性条件，因此不能交换 vend 与 flag 的先后关系。 + TraceAtomicExchange( + stats.trace, stats.result, static_cast(task_id), AtomicSite::CompletionVendExchange, + &state->tasks[task_id].vend, worker.heap_next + ); + Ops::StoreBarrier(); + TraceAtomicExchange( + stats.trace, stats.result, static_cast(task_id), AtomicSite::CompletionFlagExchange, + &state->tasks[task_id].flag, static_cast(1) + ); + AdvanceFrontier(state, stats); +} + +template +PA_DEVICE bool SlotReady(PA_GM SchedulerState *state, PA_GM LocalSlot &slot, LocalStats &stats) { + // 每个 fanin flag 都是跨核共享的完成条件；遇到第一个未就绪依赖即返回，后续 drain 会再次轮询。 + for (uint32_t index = 0; index < slot.fanin_count; ++index) { + const int32_t dependency = slot.fanin[index]; + if (TraceAtomicLoad( + stats.trace, stats.result, dependency, AtomicSite::FaninFlagLoad, + &state->tasks[dependency].flag + ) == 0) { + ++stats.result.fanin_not_ready_loads; + return false; + } + ++stats.result.fanin_ready_loads; + } + return true; +} + +PA_DEVICE void RecordKernelCycles(LocalStats &stats, TaskKind kind, uint64_t cycles) { + const uint32_t index = KindIndex(kind) - 1; + ++stats.result.kernel_counts[index]; + stats.result.kernel_cycles[index] += cycles; + if (stats.result.kernel_min_cycles[index] == 0 || cycles < stats.result.kernel_min_cycles[index]) { + stats.result.kernel_min_cycles[index] = cycles; + } + if (cycles > stats.result.kernel_max_cycles[index]) { + stats.result.kernel_max_cycles[index] = cycles; + } +} + +template +PA_DEVICE uint32_t DrainReady( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, DrainPlace place, LocalStats &stats +) { + // 同一套 drain 被三个位置复用：每次 Submit 开头的 EfDrain、ring 背压等待和所有 Submit 后的最终 drain。 + // slot 属于当前 worker；只有其全部跨核 fanin 已 ready 时才执行所选 winner 负载、发布完成并释放 slot。 + if (worker.occupied_count == 0) { + return 0; + } + uint32_t freed = 0; + // 一次调用遍历完本核全部私有 slot；未就绪项保留 occupied/built，已完成项立即清槽并继续扫描。 + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + PA_GM LocalSlot &slot = worker.slots[index]; + if (!slot.occupied || !slot.built || !SlotReady(state, slot, stats)) { + continue; + } + const TaskKind kind = static_cast(slot.kind + 1); + const uint64_t kernel_begin = TraceTimestamp(stats.trace, stats.result); + Ops::ExecuteKernel(state, worker, kind, NopCountForKind(state->config.nops, kind)); + const uint64_t kernel_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(slot.task_id), static_cast(slot.kind), + TracePhase::Kernel, ProfilePhase::ReplayTail, kernel_begin, kernel_end + ); + RecordKernelCycles(stats, kind, kernel_end - kernel_begin); + CompleteTask(state, worker, slot.task_id, stats); + const uint64_t commit_cycle = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(slot.task_id), static_cast(slot.kind), + TracePhase::Commit, ProfilePhase::ReplayTail, commit_cycle, commit_cycle + ); + slot.built = false; + slot.occupied = false; + --worker.occupied_count; + ++stats.result.placement[static_cast(place)]; + ++freed; + } + return freed; +} + +PA_DEVICE int32_t FindFreeSlot(PA_GM WorkerState &worker) { + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + if (!worker.slots[index].occupied) { + return static_cast(index); + } + } + return -1; +} + +template +PA_DEVICE void WaitForSlot( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, LocalStats &stats +) { + // 四个物理 slot 中预留两个 won slot 语义位，仅有 kUsableSlots 个可供本图使用；满时靠 drain 取得进展。 + const uint64_t wait_begin = TraceTimestamp(stats.trace, stats.result); + bool waited = false; + // 只聚合这个显式背压等待区中的 fanin 观察；每次 Submit 开头的 + // opportunistic EfDrain 仍保留逐条 Atomic，不能仅凭 site 名称全局聚合。 + const uint32_t poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, TraceAtomicSiteMask(AtomicSite::FaninFlagLoad) + ); + // 退出条件只有 occupied_count 重新低于可用容量；依赖尚未 ready 时 SpinHint 后继续重试。 + while (worker.occupied_count >= kUsableSlots) { + waited = true; + ++stats.result.wait_iterations[0]; + if (DrainReady( + state, worker, DrainPlace::RingBackpressure, stats + ) == 0) { + Ops::SpinHint(); + } + } + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + if (waited) { + ++stats.result.wait_events[0]; + const uint64_t wait_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::RingBp, + ProfilePhase::WaitForSlot, wait_begin, wait_end, 0, 0 + ); + } else if constexpr (Profile) { + AccumulatePhase( + stats.result, ProfilePhase::WaitForSlot, wait_begin, + TraceTimestamp(stats.trace, stats.result) + ); + } +} + +template +PA_DEVICE bool HeapGuard( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, uint64_t output_bytes, + LocalStats &stats +) { + // 只有产生新输出的 winner 需要保护环形 heap；retire=frontier-H 对应已经允许复用的最老任务 vend。 + // 等待期间也主动 drain 本核已就绪 slot，避免只自旋而阻塞能够推动 frontier 的 kernel。 + if (output_bytes == 0 || state->heap_base == 0) { + return true; + } + const uint64_t ring = state->heap_size; + ++stats.result.heap_guards; + const uint64_t wait_begin = TraceTimestamp(stats.trace, stats.result); + bool waited = false; + bool poll_region_active = false; + uint32_t poll_region = 0; + // 正常出口是 heap_next-vend 落入一个 ring；检测到不可能释放的覆盖或其他核 fatal 时返回失败。 + while (!IsFatal(state, stats, static_cast(task_id))) { + // 逻辑 heap 尚未走完第一圈时，所有物理输出区间都位于 [0, heap_next)， + // 不可能覆盖此前分配；保留上面的 fatal 原子检查后，可直接跳过 frontier/vend 读取。 + if (worker.heap_next <= ring) { + if (poll_region_active) { + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + } + if constexpr (Profile) { + AccumulatePhase( + stats.result, ProfilePhase::HeapGuard, wait_begin, + TraceTimestamp(stats.trace, stats.result) + ); + } + return true; + } + // 与真实 PA 一样，首圈 fast path 上方的 FatalPoll 仍是直接记录；只有 + // 确认进入 heap wrap 慢路径后，才开启本等待 episode 的四类观察聚合。 + if (!poll_region_active) { + poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, + TraceAtomicSiteMask(AtomicSite::FatalPoll) | + TraceAtomicSiteMask(AtomicSite::HeapFrontierLoad) | + TraceAtomicSiteMask(AtomicSite::HeapVendLoad) | + TraceAtomicSiteMask(AtomicSite::FaninFlagLoad) + ); + poll_region_active = true; + } + const int64_t frontier = LoadLine( + state->frontier, stats, AtomicSite::HeapFrontierLoad, static_cast(task_id) + ); + const int64_t retire = frontier - static_cast(state->heap_window); + const uint64_t vend = retire < 0 + ? 0 + : TraceAtomicLoad( + stats.trace, stats.result, static_cast(task_id), AtomicSite::HeapVendLoad, + &state->tasks[retire].vend + ); + if (worker.heap_next - vend <= ring) { + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + if (waited) { + ++stats.result.wait_events[1]; + const uint64_t wait_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::RingBp, + ProfilePhase::HeapGuard, wait_begin, wait_end, 0, 1 + ); + } else if constexpr (Profile) { + AccumulatePhase( + stats.result, ProfilePhase::HeapGuard, wait_begin, + TraceTimestamp(stats.trace, stats.result) + ); + } + return true; + } + if (frontier >= static_cast(task_id) - 1) { + SetFatal(state, stats, static_cast(task_id)); + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + return false; + } + waited = true; + ++stats.result.wait_iterations[1]; + if (DrainReady( + state, worker, DrainPlace::RingBackpressure, stats + ) == 0) { + Ops::SpinHint(); + } + } + if (poll_region_active) { + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + } + if (waited) { + ++stats.result.wait_events[1]; + const uint64_t wait_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::RingBp, + ProfilePhase::HeapGuard, wait_begin, wait_end, 0, 1 + ); + } + return false; +} + +struct ClaimOutcome { + bool attempted; + bool won; + uint64_t retries; + int32_t function_id; +}; + +template +PA_DEVICE ClaimOutcome Claim( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, TaskKind kind, + LocalStats &stats +) { + // Claim 在四个 shard 的单调 cursor 上执行 atomicMax：同一 task 只有观察到旧值更小的竞争者获胜。 + // Alloc 由全部 96 个 worker 竞争；QK/PV 仅 32 个 AIC，SF/UP 仅 64 个 AIV 进入真正的 atomicMax。 + ClaimOutcome outcome{false, false, 0, -1}; + if (task_id >= kTaskCellCapacity) { + return outcome; + } + PA_GM AtomicLine *cursor = nullptr; + if (kind == TaskKind::Alloc) { + cursor = &state->alloc_cursor[task_id % kCursorShards]; + } else { + // Mirror MixedKernels::to_active_mask(), core_mask(), popcount(), + // lane_active(), and self->role routing inside the real Claim span. + const int32_t aic_kernel = kind == TaskKind::Qk || kind == TaskKind::Pv ? FunctionId(kind) : -1; + const int32_t aiv0_kernel = kind == TaskKind::Sf || kind == TaskKind::Up ? FunctionId(kind) : -1; + const int32_t aiv1_kernel = -1; + uint8_t active_mask = 0; + if (aic_kernel >= 0) active_mask |= 1U; + if (aiv0_kernel >= 0) active_mask |= 2U; + if (aiv1_kernel >= 0) active_mask |= 4U; + const uint8_t core_mask = active_mask & 0x07U; + const int32_t active_count = __builtin_popcount(static_cast(core_mask)); + // 这里保留生产 Claim 的 lane-mask 路由边界。当前固定 PA 图按构造只生成单 lane + // 的 QK/PV 或 SF/UP；需要两个及以上 lane 协作的 joint task 本应进入 BlockWon + // 协议，本独立用例没有实现该动态路径，因此显式拒绝而不把它误当成单 lane task。 + if (active_count >= 2) { + return outcome; + } + if ((core_mask & 1U) != 0) { + if (worker.role != CoreRole::Aic) return outcome; + cursor = &state->cube_cursor[task_id % kCursorShards]; + outcome.function_id = aic_kernel; + } else if ((core_mask & 6U) != 0) { + if (worker.role != CoreRole::Aiv) return outcome; + cursor = &state->vector_cursor[task_id % kCursorShards]; + outcome.function_id = (core_mask & 2U) != 0 ? aiv0_kernel : aiv1_kernel; + } else { + return outcome; + } + } + outcome.attempted = true; + // atomicMax 返回写入前的 cursor：old=task_id 则必须 Replay。 + const int64_t old = TraceAtomicFetchMax( + stats.trace, stats.result, static_cast(task_id), AtomicSite::ClaimMax, + &cursor->value, static_cast(task_id), outcome.retries + ); + outcome.won = old < static_cast(task_id); + if (!outcome.won) outcome.function_id = -1; + return outcome; +} + +PA_DEVICE void RecordClaimOutcome(LocalStats &stats, TaskKind kind, const ClaimOutcome &outcome) { + if (outcome.attempted) ++stats.result.claim_attempts; + stats.result.cas_retries += outcome.retries; + if (outcome.won) { + ++stats.result.claim_wins; + ++stats.result.wins[KindIndex(kind)]; + } +} + +template +PA_DEVICE bool BuildWinner( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, TaskKind kind, + const TaskArgs &args, const SubmitContext &context, + const int32_t fanin[kMaxFanin], uint32_t fanin_count, LocalStats &stats +) { + // kernel winner 不在 Submit 内立即执行计算，而是把完整 payload 和 fanin 存入自己的私有 ring slot。 + // 后续 EfDrain/背压 drain/最终 drain 在依赖满足后执行它，这正是 PA 的 Submit 与执行解耦点。 + WaitForSlot(state, worker, task_id, stats); + if (!HeapGuard( + state, worker, task_id, context.output_bytes, stats + )) { + return false; + } + const int32_t slot_index = FindFreeSlot(worker); + if (slot_index < 0) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + PA_GM LocalSlot &slot = worker.slots[slot_index]; + // Match dist_submit_alloc_slot(): reserve and account the private slot + // before build_ring_slot_from_submit publishes its completed payload. + // 状态按“occupied 占位 -> built 清零 -> 计入占用 -> BuildSlotPayload”推进；后者会先 + // 置 built，再填充 payload。slot 为 worker 私有、没有跨核发布竞争，所以此处的 built + // 只是复刻生产状态机与构建成本，不承担对其他核发布完整 payload 的同步语义。 + slot.occupied = true; + slot.built = 0; + ++worker.occupied_count; + if (worker.occupied_count > stats.max_occupied) { + stats.max_occupied = worker.occupied_count; + } + const int32_t sub_block_id = worker.lane == 2 ? 1 : 0; + BuildSlotPayload( + slot, task_id, static_cast(FunctionId(kind)), 0, args, context, fanin, fanin_count, + sub_block_id + ); + stats.result.slot_tensor_copies += static_cast(context.tensor_count); + stats.result.slot_scalar_copies += static_cast(context.scalar_count); + stats.result.fanin_edges += fanin_count; + return true; +} + +template +PA_DEVICE bool SubmitTask( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_count, TaskKind kind, + const TaskArgs &args, SubmitContext &context, LocalStats &stats, PmuContext &pmu_context +) { + // 每个 worker 都完整回放相同 task stream。主流程为：EfDrain -> materialize -> TensorMap retire + // -> Claim -> winner 收集 fanin -> 全员 register -> winner Build / loser Replay。Alloc 在 Claim 前 register， + // 且 winner 不入 kernel slot，而是在 heap guard 后直接发布完成。 + BeginSubmit(worker, args, context); + const uint32_t task_id = static_cast(context.task_id); +#if PA_BUILD_SUBMIT_PMU + // PMU-only ELF 只保留首/末 Submit 的全局时间边界，不再为 1280 次调用 + // 各执行两条 trace-only SYS_CNT。 + const uint64_t submit_begin = task_id == 0 ? Ops::Now() : 0; +#else + const uint64_t submit_begin = TraceTimestamp(stats.trace, stats.result); +#endif + if (task_id == 0) { + stats.result.submit_begin = submit_begin; + } + + // EfDrain 在当前 Submit 的参数物化前执行上一批已就绪 slot，是绝大多数 kernel 的正常落点。 + // 只在这个唯一 call-site 划 PMU 边界；DrainReady 还被 ring 背压和最终 drain + // 复用，不能把 phase 插入函数体后按 place 混合累计。 + // EfDrain 是 Submit 的第一个真实阶段，直接复用父区间起点；这样每次 + // Submit 少一次 trace-only SYS_CNT，也不会留下人为的 prefix residual。 + const uint64_t efdrain_begin = submit_begin; + BeginSubmitPmuPhase(pmu_context); + DrainReady(state, worker, DrainPlace::EfDrain, stats); + EndSubmitPmuPhase(pmu_context); + const uint64_t efdrain_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::EfDrain, + ProfilePhase::EfDrain, efdrain_begin, efdrain_end + ); + + // schema-v4 的所有 Submit 子阶段都使用显式 start/end；不再通过共同 lap + // 起点生成相互覆盖的 Build/Replay/Alloc 区间。 + // 后继 segment 复用前一阶段 end：既少一次 SYS_CNT，也把前一条 trace + // 发布和阶段间胶水明确归入 Materialize，而不是留成无名 residual。 + const uint64_t materialize_begin = efdrain_end; + BeginSubmitPmuPhase(pmu_context); + const bool materialized = + MaterializeTask(worker, task_id, args, context, state->heap_base, state->heap_size); + if (!materialized) { + EndSubmitPmuPhase(pmu_context); + SetFatal(state, stats, static_cast(task_id)); + return false; + } + stats.result.materialized_outputs += context.result.count; + EndSubmitPmuPhase(pmu_context); + const uint64_t materialize_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::Materialize, + ProfilePhase::Materialize, materialize_begin, materialize_end, 0, + kind == TaskKind::Alloc ? 1U : 0U + ); + const uint64_t prepare_begin = materialize_end; + AdvanceTensorMap(worker.map, task_id, static_cast(state->heap_window)); + const uint64_t prepare_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::PrepareMap, + ProfilePhase::PrepareMap, prepare_begin, prepare_end, 0, kind == TaskKind::Alloc ? 1U : 0U + ); + + bool winner = false; + int32_t function_id = -1; + + if (kind == TaskKind::Alloc) { + // Alloc 没有 kernel lane，96 个 worker 都维护本地物化/heap 状态，但只有 Claim winner 发布全局完成。 + const uint64_t register_begin = prepare_end; + BeginSubmitPmuPhase(pmu_context); + RegisterOutputs(context, args, false); + EndSubmitPmuPhase(pmu_context); + const uint64_t register_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::Register, + ProfilePhase::Register, register_begin, register_end, 0, 0 + ); + + const uint64_t claim_begin = register_end; + BeginSubmitPmuPhase(pmu_context); + const ClaimOutcome claim = Claim(state, worker, task_id, kind, stats); + winner = claim.won; + context.won = winner; + context.kernel_id = claim.function_id; + // Claim 的本地结果归档属于同一阶段；放在共同 end 边界内，避免把 + // winner/context/stat bookkeeping 留成无法归因的 Submit residual。 + RecordClaimOutcome(stats, kind, claim); + EndSubmitPmuPhase(pmu_context); + const uint64_t claim_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::Claim, + ProfilePhase::Claim, claim_begin, claim_end, + (winner ? kClaimWon : 0U) | (claim.attempted ? kClaimAttempted : 0U), 1 + ); + // 每个 task 只有 1/96 worker 进入 winner 重型路径。把该分支标成冷路 + // 只影响基本块布局，使占绝大多数的 loser 尽量顺序进入 Submit 公共 + // 尾部；不改变 Claim 结果、完成发布或泳道边界。 + if (__builtin_expect(winner, 0)) { + const uint64_t alloc_complete_begin = claim_end; + if (!HeapGuard( + state, worker, task_id, context.output_bytes, stats + )) { + return false; + } + CompleteTask(state, worker, task_id, stats); + const uint64_t alloc_complete_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, + TracePhase::AllocComplete, ProfilePhase::ReplayTail, + alloc_complete_begin, alloc_complete_end + ); + } else { + // standalone 的 Alloc loser 没有真实 GM/Replay 动作，不再为业务 + // 路径名字写一条零时长记录。Claim 后到 Submit.end 的真实 + // scalar 时间由离线 submit_tail_gap 补集展示；排他报告将其汇总为 + // submit_tail_residual，避免伪装成 Alloc loser 业务阶段。 + } + } else { + const uint64_t claim_begin = prepare_end; + BeginSubmitPmuPhase(pmu_context); + const ClaimOutcome claim = Claim(state, worker, task_id, kind, stats); + winner = claim.won; + function_id = claim.function_id; + context.won = winner; + context.kernel_id = function_id; + RecordClaimOutcome(stats, kind, claim); + EndSubmitPmuPhase(pmu_context); + const uint64_t claim_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, TracePhase::Claim, + ProfilePhase::Claim, claim_begin, claim_end, + (winner ? kClaimWon : 0U) | (claim.attempted ? kClaimAttempted : 0U), 0 + ); + uint64_t register_begin = claim_end; + if (winner) { + const uint64_t fanin_begin = claim_end; + context.fanin_count = static_cast(CollectFanin(worker.map, args, context.fanin)); + stats.result.map_lookups += static_cast(args.tensor_count) - context.result.count; + const uint64_t fanin_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, TracePhase::Fanin, + ProfilePhase::Fanin, fanin_begin, fanin_end, 0, static_cast(context.fanin_count) + ); + register_begin = fanin_end; + } + + // loser 直接承接 Claim.end；winner 则承接 Fanin.end。两条路径都 + // 复用已有边界，不再为 Register 单独读取 SYS_CNT。 + BeginSubmitPmuPhase(pmu_context); + RegisterOutputs(context, args, true); + stats.result.map_inserts += CountBits(context.register_mask); + EndSubmitPmuPhase(pmu_context); + const uint64_t register_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, TracePhase::Register, + ProfilePhase::Register, register_begin, register_end, 0, 1 + ); + // BuildWinner 会内联 ring、heap 和真计算提交逻辑；提示其为 1/96 + // 冷路，避免 loser 为跳过大块代码付出额外取指代价。 + if (__builtin_expect(winner, 0)) { + const uint64_t winner_build_begin = register_end; + if (!BuildWinner( + state, worker, task_id, kind, args, context, context.fanin, + static_cast(context.fanin_count), stats + )) { + return false; + } + const uint64_t winner_build_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::WinnerBuild, ProfilePhase::ReplayTail, + winner_build_begin, winner_build_end + ); + } else { + // 非 winner 不占用私有 ring slot，也没有可单列的 Replay 计算。 + // Register 后到 Submit.end 由离线 submit_tail_gap 补集展示；排他 + // 报告汇总为 submit_tail_residual，避免零时长 marker 增加 raw + // 体积和 trace-buffer 写开销。 + } + } + + ++stats.result.submits; +#if PA_BUILD_SUBMIT_PMU + const uint64_t submit_end = task_id + 1 == task_count ? Ops::Now() : 0; +#else + const uint64_t submit_end = TraceTimestamp(stats.trace, stats.result); +#endif + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, TracePhase::Submit, + ProfilePhase::Submit, submit_begin, submit_end, winner ? 1U : 0U, kind == TaskKind::Alloc ? 1U : 0U + ); + if (task_id + 1 == task_count) { + stats.result.submit_end = submit_end; + } + return true; +} + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +// The all-task callback path crosses the split-TU finish boundary through this +// fixed 16-byte POD. No callback closure or nested thunk is stored in it. +struct LazySampleCallbackTicket { + uint64_t submit_begin; + uint32_t task_id; + int16_t function_id; + uint8_t won; + uint8_t reserved; +}; +static_assert(sizeof(LazySampleCallbackTicket) == 16, "lazy sample callback ticket must remain a 16-byte POD"); +static_assert(offsetof(LazySampleCallbackTicket, submit_begin) == 0, "lazy sample callback ticket timestamp offset mismatch"); +static_assert(offsetof(LazySampleCallbackTicket, task_id) == 8, "lazy sample callback ticket task offset mismatch"); +static_assert(offsetof(LazySampleCallbackTicket, function_id) == 12, "lazy sample callback ticket function offset mismatch"); +static_assert(offsetof(LazySampleCallbackTicket, won) == 14, "lazy sample callback ticket winner offset mismatch"); + +PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { + // This is BeginSubmit without an already-materialized TaskArgs. The same + // fields are completed synchronously after the single callback builds args. + const uint32_t task_id = static_cast(worker.local_index++); + context.self = &worker; + context.payload = &worker.payloads[task_id & kPayloadMask]; + context.task_id = static_cast(task_id); + context.tensor_count = 0; + context.scalar_count = 0; + context.result.task_id = task_id; + context.result.count = 0; + context.register_mask = 0; + context.output_bytes = 0; + context.fanin_count = 0; + context.kernel_id = -1; + context.won = false; + context.joint = false; + context.joint_init = false; + context.joint_block = -1; + context.joint_slot = -1; + context.joint_count = 0; +} + +#if defined(__CCE_AICORE__) || defined(__NPU_ARCH__) +#define PA_LAZY_LAMBDA_DEVICE __aicore__ +#else +#define PA_LAZY_LAMBDA_DEVICE +#endif + +template +PA_DEVICE bool BuildLazySampleCallbackArgs( + PaOrchestrationState &orch, TaskArgs &args, uint32_t batch, bool won, + LocalStats &stats +) { + LazySampleCallbackArgsBuilder callback_builder(args, Kind, won); + auto callback = [&](LazySampleCallbackArgsBuilder &builder) PA_LAZY_LAMBDA_DEVICE { + builder.Begin(); + if constexpr (Kind == TaskKind::Alloc) { + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.tile_create_info; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + } else if constexpr (Kind == TaskKind::Qk) { + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + MakeLazySampleCallbackQueryView(orch, batch); + builder.RecordView(); + return orch.query_view; + }); + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + return orch.key_cache; + }); + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + return orch.block_table; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + const uint32_t score_shape[kMaxTensorDims] = { + kPaHeads, + static_cast(orch.current_nblocks * kPaBlockSize), + 0, 0, 0 + }; + InitCreateInfo(orch.qk_create_info, score_shape, 2, DataType::Float32); + builder.RecordDynamicCreateInfo(); + return orch.qk_create_info; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_nblocks; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + + orch.current_block_offset; + }); + } else if constexpr (Kind == TaskKind::Sf) { + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.qk_scores; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + const uint32_t probability_shape[kMaxTensorDims] = { + kPaHeads, + static_cast(orch.current_nblocks * kPaBlockSize), + 0, 0, 0 + }; + InitCreateInfo(orch.sf_create_info, probability_shape, 2, DataType::Bfloat16); + builder.RecordDynamicCreateInfo(); + return orch.sf_create_info; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { return orch.scale_bits; }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_nblocks; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_valid_len; + }); + } else if constexpr (Kind == TaskKind::Pv) { + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.sf_probs; + }); + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + return orch.value_cache; + }); + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + return orch.block_table; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.tile_create_info; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_nblocks; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + + orch.current_block_offset; + }); + } else { + static_assert(Kind == TaskKind::Up, "unsupported PA task kind"); + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.sf_max; + }); + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.sf_sum; + }); + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.pv_output; + }); + builder.AddGmInout([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.accumulated_max; + }); + builder.AddGmInout([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.accumulated_sum; + }); + builder.AddGmInout([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.accumulated_output; + }); + builder.AddLocalInout([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + MakeLazySampleCallbackOutputView(orch, batch); + builder.RecordView(); + return orch.output_view; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_block_offset == 0 ? 1 : 0; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_block_offset + orch.current_nblocks >= orch.current_blocks ? 1 : 0; + }); + } + }; + + callback(callback_builder); + if (!callback_builder.Valid()) return false; + const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); + stats.result.arg_resets += counts.reset_calls; + stats.result.views_created += counts.views_created; + stats.result.dynamic_create_infos += counts.dynamic_create_infos; + stats.result.tensor_args_added += counts.tensor_args_added; + stats.result.scalar_args_added += counts.scalar_args_added; + return true; +} + +#undef PA_LAZY_LAMBDA_DEVICE + +template +PA_DEVICE bool FinishLazySampleCallbackSubmitBody( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_count, + const TaskArgs &args, SubmitContext &context, LocalStats &stats, + PmuContext &pmu_context, const LazySampleCallbackTicket &ticket +) { + const uint32_t task_id = ticket.task_id; + const TaskKind kind = GetTaskKind(task_id); + const int32_t function_id = static_cast(ticket.function_id); + const bool winner = ticket.won != 0; + + // The callback has ended before this body consumes TaskArgs. No closure + // or nested thunk escapes its lifetime. Inline shapes instantiate this in + // the caller; split shapes instantiate it inside the runtime finish TU. + const uint64_t materialize_begin = TraceTimestamp(stats.trace, stats.result); + BeginSubmitPmuPhase(pmu_context); + const bool materialized = + MaterializeTask(worker, task_id, args, context, state->heap_base, state->heap_size); + if (!materialized) { + EndSubmitPmuPhase(pmu_context); + SetFatal(state, stats, static_cast(task_id)); + return false; + } + stats.result.materialized_outputs += context.result.count; + EndSubmitPmuPhase(pmu_context); + const uint64_t materialize_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Materialize, ProfilePhase::Materialize, + materialize_begin, materialize_end, 0, kind == TaskKind::Alloc ? 1U : 0U + ); + + const uint64_t prepare_begin = materialize_end; + AdvanceTensorMap(worker.map, task_id, static_cast(state->heap_window)); + const uint64_t prepare_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::PrepareMap, ProfilePhase::PrepareMap, + prepare_begin, prepare_end, 0, kind == TaskKind::Alloc ? 1U : 0U + ); + + uint64_t register_begin = prepare_end; + if (kind != TaskKind::Alloc && __builtin_expect(winner, 0)) { + const uint64_t fanin_begin = prepare_end; + context.fanin_count = static_cast(CollectFanin(worker.map, args, context.fanin)); + stats.result.map_lookups += static_cast(args.tensor_count) - context.result.count; + const uint64_t fanin_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Fanin, ProfilePhase::Fanin, + fanin_begin, fanin_end, 0, static_cast(context.fanin_count) + ); + register_begin = fanin_end; + } + + BeginSubmitPmuPhase(pmu_context); + RegisterOutputs(context, args, kind != TaskKind::Alloc); + if (kind != TaskKind::Alloc) stats.result.map_inserts += CountBits(context.register_mask); + EndSubmitPmuPhase(pmu_context); + const uint64_t register_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Register, ProfilePhase::Register, + register_begin, register_end, 0, kind == TaskKind::Alloc ? 0U : 1U + ); + + if (__builtin_expect(winner, 0)) { + const uint64_t winner_build_begin = register_end; + if (kind == TaskKind::Alloc) { + if (!HeapGuard(state, worker, task_id, context.output_bytes, stats)) { + return false; + } + CompleteTask(state, worker, task_id, stats); + } else { + if (!BuildWinner( + state, worker, task_id, kind, args, context, context.fanin, + static_cast(context.fanin_count), stats + )) { + return false; + } + } + const uint64_t winner_build_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + kind == TaskKind::Alloc ? TracePhase::AllocComplete : TracePhase::WinnerBuild, + ProfilePhase::ReplayTail, + winner_build_begin, winner_build_end + ); + } + + ++stats.result.submits; +#if PA_BUILD_SUBMIT_PMU + const uint64_t submit_end = task_id + 1 == task_count ? Ops::Now() : 0; +#else + const uint64_t submit_end = TraceTimestamp(stats.trace, stats.result); +#endif + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Submit, ProfilePhase::Submit, + ticket.submit_begin, submit_end, winner ? 1U : 0U, 0 + ); + if (task_id + 1 == task_count) { + stats.result.submit_end = submit_end; + } + return true; +} + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +template +PA_DEVICE uint32_t FinishSplitLazySampleCallbackFromRuntime( + const LazySampleCallbackTicket *ticket, const TaskArgs *args +) { + LazySampleSplitRuntimeState &runtime = Ops::LazySampleSplitState(); + const uint64_t state_address = reinterpret_cast(&runtime); + runtime.finish_state_address = state_address; + + bool valid = ticket != nullptr && args != nullptr && runtime.scheduler != nullptr && + runtime.worker != nullptr && runtime.task_count != 0 && + runtime.worker_id < kWorkers && runtime.owner_worker_id == runtime.worker_id && + runtime.worker->core_idx == static_cast(runtime.worker_id) && + runtime.caller_state_address == state_address && + runtime.state_cookie == LazySampleSplitStateCookie( + runtime.worker_id, runtime.worker->role + ) && runtime.reserved == 0; + if (valid) { + valid = ticket->reserved == 0 && ticket->task_id < runtime.task_count && + runtime.context.task_id == static_cast(ticket->task_id) && + runtime.context.kernel_id == static_cast(ticket->function_id) && + runtime.context.won == (ticket->won != 0); + } + ++runtime.finish_calls; + if (ticket != nullptr) runtime.task_id_sum += ticket->task_id; + if (!valid) { + ++runtime.protocol_errors; + if (runtime.scheduler != nullptr) { + SetFatal( + runtime.scheduler, runtime.stats, + ticket == nullptr ? -1 : static_cast(ticket->task_id) + ); + } + return 0; + } + + // split finish 不让 caller 的 SubmitContext/LocalStats/PMU 对象跨过 + // noinline 边界。诊断构建只允许 phase=none：权威 CNT6/7 仍由 caller + // 外层完整窗口读取，finish 内部不做 read-clear 局部快照。 +#if PA_BUILD_SUBMIT_PMU + static_assert( + kCompiledSubmitPmuPhase == SubmitPmuPhase::None, + "split callback submit-PMU supports only the whole-window none phase" + ); +#endif + bool pmu_context = false; + return FinishLazySampleCallbackSubmitBody( + runtime.scheduler, *runtime.worker, runtime.task_count, *args, + runtime.context, runtime.stats, pmu_context, *ticket + ) ? 1U : 0U; +} +#endif + +template +PA_DEVICE bool SubmitLazySampleCallback( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_count, + PaOrchestrationState &orch, TaskArgs &args, uint32_t batch, + SubmitContext &context, LocalStats &stats, PmuContext &pmu_context +) { + BeginLazySampleCallbackSubmit(worker, context); + const uint32_t task_id = static_cast(context.task_id); +#if PA_BUILD_SUBMIT_PMU + const uint64_t submit_begin = task_id == 0 ? Ops::Now() : 0; +#else + const uint64_t submit_begin = TraceTimestamp(stats.trace, stats.result); +#endif + if (task_id == 0) stats.result.submit_begin = submit_begin; + + const uint64_t efdrain_begin = submit_begin; + BeginSubmitPmuPhase(pmu_context); + DrainReady(state, worker, DrainPlace::EfDrain, stats); + EndSubmitPmuPhase(pmu_context); + const uint64_t efdrain_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, + TracePhase::EfDrain, ProfilePhase::EfDrain, + efdrain_begin, efdrain_end + ); + + const uint64_t claim_begin = efdrain_end; + BeginSubmitPmuPhase(pmu_context); + const ClaimOutcome claim = Claim(state, worker, task_id, Kind, stats); + context.won = claim.won; + context.kernel_id = claim.function_id; + RecordClaimOutcome(stats, Kind, claim); + EndSubmitPmuPhase(pmu_context); + const uint64_t claim_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), claim.function_id, + TracePhase::Claim, ProfilePhase::Claim, + claim_begin, claim_end, + (claim.won ? kClaimWon : 0U) | (claim.attempted ? kClaimAttempted : 0U), + Kind == TaskKind::Alloc ? 1U : 0U + ); + + if (!BuildLazySampleCallbackArgs(orch, args, batch, claim.won, stats)) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + const LazySampleCallbackTicket ticket{ + submit_begin, + task_id, + // All standalone function IDs are -1..3 and exactly fit this ABI field. + static_cast(claim.function_id), + static_cast(claim.won ? 1 : 0), + 0, + }; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + (void)state; + (void)worker; + (void)task_count; + (void)context; + (void)stats; + (void)pmu_context; + return Ops::FinishLazySampleCallback(&ticket, &args); +#else + return FinishLazySampleCallbackSubmitBody( + state, worker, task_count, args, context, stats, pmu_context, ticket + ); +#endif +} +#endif + +PA_DEVICE uint32_t CountLiveMapEntries(PA_GM const TensorMap &map) { + uint32_t free_entries = 0; + for (int32_t current = map.free_head; current >= 0; current = map.entries[current].next_in_bucket) { + ++free_entries; + } + return static_cast(map.high_water) - free_entries; +} + +template +PA_DEVICE void PublishResult(PA_GM WorkerResult &destination, const WorkerResult &source) { + // 每个 worker 只写自己独占、覆盖多条 cache line 的 WorkerResult 分区；逐字段 + // bypass 保证结果对 host 可见，而独立 sidecar 允许 D2H 只搬结果、不搬约 9 MiB WorkerState。 +#define PA_PUBLISH_FIELD(field) Ops::Publish(&destination.field, source.field) + PA_PUBLISH_FIELD(submit_begin); + PA_PUBLISH_FIELD(submit_end); + PA_PUBLISH_FIELD(finish_cycle); + PA_PUBLISH_FIELD(checksum); + PA_PUBLISH_FIELD(submits); + PA_PUBLISH_FIELD(claim_attempts); + PA_PUBLISH_FIELD(claim_wins); + PA_PUBLISH_FIELD(heap_guards); + PA_PUBLISH_FIELD(fanin_ready_loads); + PA_PUBLISH_FIELD(completion_duplicates); + PA_PUBLISH_FIELD(cas_retries); + PA_PUBLISH_FIELD(joint_polls); + for (uint32_t index = 0; index < static_cast(TaskKind::Count); ++index) { + Ops::Publish(&destination.wins[index], source.wins[index]); + } + for (uint32_t index = 0; index < 4; ++index) { + Ops::Publish(&destination.kernel_counts[index], source.kernel_counts[index]); + Ops::Publish(&destination.kernel_cycles[index], source.kernel_cycles[index]); + Ops::Publish(&destination.kernel_min_cycles[index], source.kernel_min_cycles[index]); + Ops::Publish(&destination.kernel_max_cycles[index], source.kernel_max_cycles[index]); + } + for (uint32_t index = 0; index < static_cast(DrainPlace::Count); ++index) { + Ops::Publish(&destination.placement[index], source.placement[index]); + } + for (uint32_t index = 0; index < static_cast(ProfilePhase::Count); ++index) { + Ops::Publish(&destination.phase_cycles[index], source.phase_cycles[index]); + Ops::Publish(&destination.phase_calls[index], source.phase_calls[index]); + } + for (uint32_t index = 0; index < 2; ++index) { + Ops::Publish(&destination.wait_events[index], source.wait_events[index]); + Ops::Publish(&destination.wait_iterations[index], source.wait_iterations[index]); + } + PA_PUBLISH_FIELD(context_reads); + PA_PUBLISH_FIELD(views_created); + PA_PUBLISH_FIELD(dynamic_create_infos); + PA_PUBLISH_FIELD(arg_resets); + PA_PUBLISH_FIELD(tensor_args_added); + PA_PUBLISH_FIELD(scalar_args_added); + PA_PUBLISH_FIELD(materialized_outputs); + PA_PUBLISH_FIELD(map_inserts); + PA_PUBLISH_FIELD(map_lookups); + PA_PUBLISH_FIELD(slot_tensor_copies); + PA_PUBLISH_FIELD(slot_scalar_copies); + PA_PUBLISH_FIELD(fanin_edges); + PA_PUBLISH_FIELD(final_heap_next); + PA_PUBLISH_FIELD(map_high_water); + PA_PUBLISH_FIELD(map_alive_floor); + PA_PUBLISH_FIELD(map_cleaned_upto); + PA_PUBLISH_FIELD(map_live_entries); + PA_PUBLISH_FIELD(worker_id); + PA_PUBLISH_FIELD(role); + PA_PUBLISH_FIELD(max_occupied); + PA_PUBLISH_FIELD(final_occupied); + PA_PUBLISH_FIELD(fanin_not_ready_loads); + PA_PUBLISH_FIELD(frontier_initial_loads); + PA_PUBLISH_FIELD(frontier_updates); + PA_PUBLISH_FIELD(frontier_terminal_loads); + PA_PUBLISH_FIELD(atomic_trace_calls); +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + PA_PUBLISH_FIELD(lazy_sample_split_caller_state_address); + PA_PUBLISH_FIELD(lazy_sample_split_finish_state_address); + PA_PUBLISH_FIELD(lazy_sample_split_finish_calls); + PA_PUBLISH_FIELD(lazy_sample_split_protocol_errors); + PA_PUBLISH_FIELD(lazy_sample_split_state_cookie); + PA_PUBLISH_FIELD(lazy_sample_split_task_id_sum); + PA_PUBLISH_FIELD(lazy_sample_split_owner_worker_id); + PA_PUBLISH_FIELD(lazy_sample_split_reserved); +#endif +#undef PA_PUBLISH_FIELD + Ops::StoreBarrier(); +} + +template +PA_DEVICE void RunSchedulerImpl(PA_GM SchedulerState *state, uint32_t worker_id, CoreRole role) { + // 一个入口实例只拥有 state->workers[worker_id] 的私有 map/ring/payload；cursor、task cell 和屏障为跨核共享区。 + if (worker_id >= kWorkers) { + return; + } + PA_GM WorkerState &worker = state->workers[worker_id]; + worker.role = role; + worker.core_idx = static_cast(worker_id); + // standalone 使用连续 worker 编号：AIC 为 0..31；AIV 为 32..95。 + // 每个物理 block b 对应 AIC(b, lane0)、AIV(32+2b, lane1)、AIV(33+2b, lane2)。 + if (role == CoreRole::Aic) { + worker.block_id = static_cast(worker_id); + worker.lane = 0; + } else { + const uint32_t vector_id = worker_id - kAicWorkers; + worker.block_id = static_cast(vector_id / 2); + worker.lane = static_cast(1 + vector_id % 2); + } + worker.sub_block_id = worker.lane == 2 ? 1 : 0; + worker.local_index = 0; + worker.heap_next = 0; + ResetTensorMap(worker.map); + worker.occupied_count = 0; + worker.owned_total = 0; + worker.swimlane_last_cycle = 0; + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + worker.slots[index].occupied = false; + worker.slots[index].built = false; + } + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + LazySampleSplitRuntimeState &lazy_sample_split_runtime = Ops::LazySampleSplitState(); + lazy_sample_split_runtime.context = SubmitContext{}; + lazy_sample_split_runtime.stats = LocalStats{}; + lazy_sample_split_runtime.scheduler = state; + lazy_sample_split_runtime.worker = &worker; + lazy_sample_split_runtime.task_count = 0; + lazy_sample_split_runtime.worker_id = worker_id; + lazy_sample_split_runtime.caller_state_address = reinterpret_cast(&lazy_sample_split_runtime); + lazy_sample_split_runtime.finish_state_address = 0; + lazy_sample_split_runtime.finish_calls = 0; + lazy_sample_split_runtime.protocol_errors = 0; + lazy_sample_split_runtime.state_cookie = LazySampleSplitStateCookie(worker_id, role); + lazy_sample_split_runtime.task_id_sum = 0; + lazy_sample_split_runtime.owner_worker_id = worker_id; + lazy_sample_split_runtime.reserved = 0; + LocalStats &stats = lazy_sample_split_runtime.stats; +#else + LocalStats stats{}; +#endif + stats.result.worker_id = worker_id; + stats.result.role = static_cast(role); + stats.result.checksum = 0xcbf29ce484222325ULL ^ worker_id; + // 该 invalidate 原先藏在 AttachTrace 中；它同时保护 PMU mode/register + // table 与 winner workload，必须在两个构建中都执行。 + Ops::InvalidateRegion( + &state->config, sizeof(state->config) + sizeof(state->winner_workload) + ); + stats.trace = AttachTrace(state, worker, worker_id); + + // 96 个参与者全部完成本地状态初始化后再进入 task 0，主要用于压低启动偏斜对 + // winner 分布和 Submit 时序的干扰；atomicMax 的唯一 winner 正确性本身不依赖该屏障。 + TraceAtomicFetchAdd( + stats.trace, stats.result, -1, AtomicSite::StartupIncrement, + &state->started_count.value, 1 + ); + const uint64_t start_wait = Ops::Now(); + uint32_t start_polls = 0; + const uint32_t startup_poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, + TraceAtomicSiteMask(AtomicSite::StartupPoll) | TraceAtomicSiteMask(AtomicSite::FatalPoll) + ); + // 全员到齐或任一核发布 fatal 即退出启动等待；watchdog 防止缺失参与者造成永久挂死。 + while (LoadLine(state->started_count, stats, AtomicSite::StartupPoll) < + static_cast(state->config.workers) && + !IsFatal(state, stats)) { + Ops::SpinHint(); + if (WatchdogExpired(state, stats, start_wait, start_polls)) { + break; + } + } + AtomicPollRegionEnd(stats.trace, stats.result, startup_poll_region); + + const uint32_t batches = state->config.batches; + const uint32_t task_count = batches * kTasksPerBatch; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + lazy_sample_split_runtime.task_count = task_count; +#endif + PaOrchestrationState orchestration; + TaskArgs args; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + SubmitContext &context = lazy_sample_split_runtime.context; +#else + SubmitContext context; +#endif + uint64_t orchestration_begin = 0; + uint64_t orchestration_end = 0; + if (!IsFatal(state, stats)) { + // Case1 每个 batch 固定回放 Alloc/QK/SF/PV/UP 五个 task；所有 worker 顺序相同，执行 lane 由 Claim 筛选。 + // CCEC 可在这里开启本 worker 私有 PMU 窗口；CPU/AscendC 适配层是空实现。 + // 窗口覆盖从首个参数构造到末次 Submit 返回，与全局“首 Submit.begin～末 Submit.end” + // 口径接近但不相同，host sidecar 必须按 per-worker 累计解释。 + // 泳道父边界在 PMU-only 构建中会被编译为空，不应污染 Submit 取数；窗口从 + // orchestration 初始化（即首批参数构造）前一条边界开始。 + auto pmu_context = Ops::PmuWindowStart(state, worker_id); + orchestration_begin = TraceTimestamp(stats.trace, stats.result); + InitPaOrchestration(orchestration, batches, &state->context_lens[0]); + for (uint32_t batch = 0; batch < batches; ++batch) { +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + BeginPaBatchForLazySampleCallback(orchestration, batch); + ++stats.result.context_reads; + if (!SubmitLazySampleCallback< + TaskKind::Alloc, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Alloc, context.result); + + PreparePaBlockGroup(orchestration, 0); + if (!SubmitLazySampleCallback< + TaskKind::Qk, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Qk, context.result); + + if (!SubmitLazySampleCallback< + TaskKind::Sf, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Sf, context.result); + + if (!SubmitLazySampleCallback< + TaskKind::Pv, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Pv, context.result); + + if (!SubmitLazySampleCallback< + TaskKind::Up, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } +#else + BuildAllocArgs(orchestration, args, batch); + ++stats.result.context_reads; + stats.result.views_created += 2; + stats.result.tensor_args_added += 3; + if (!SubmitTask( + state, worker, task_count, TaskKind::Alloc, args, context, stats, pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Alloc, context.result); + + BuildQkArgs(orchestration, args, batch); + ++stats.result.dynamic_create_infos; + ++stats.result.arg_resets; + stats.result.tensor_args_added += 4; + stats.result.scalar_args_added += 2; + if (!SubmitTask( + state, worker, task_count, TaskKind::Qk, args, context, stats, pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Qk, context.result); + + BuildSfArgs(orchestration, args); + ++stats.result.dynamic_create_infos; + ++stats.result.arg_resets; + stats.result.tensor_args_added += 4; + stats.result.scalar_args_added += 3; + if (!SubmitTask( + state, worker, task_count, TaskKind::Sf, args, context, stats, pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Sf, context.result); + + BuildPvArgs(orchestration, args, batch); + ++stats.result.arg_resets; + stats.result.tensor_args_added += 4; + stats.result.scalar_args_added += 2; + if (!SubmitTask( + state, worker, task_count, TaskKind::Pv, args, context, stats, pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Pv, context.result); + + BuildUpdateArgs(orchestration, args); + ++stats.result.arg_resets; + stats.result.tensor_args_added += 7; + stats.result.scalar_args_added += 2; + if (!SubmitTask( + state, worker, task_count, TaskKind::Up, args, context, stats, pmu_context + )) { + break; + } +#endif + } + Ops::PmuWindowStop(state, worker_id, pmu_context); + orchestration_end = TraceTimestamp(stats.trace, stats.result); + } + + // replay_done 表示所有 worker 已退出回放循环（成功路径即完整提交）；之后仍需 drain 到本核 slot 为空。 + // 成功路径复用 orchestration end 作为 final drain start，使两个业务父区间 + // 首尾相接；父记录延后到 final drain 结束再写，避免记录自身落进任一业务 span。 + const uint64_t final_drain_begin = orchestration_end != 0 + ? orchestration_end + : TraceTimestamp(stats.trace, stats.result); + TraceAtomicFetchAdd( + stats.trace, stats.result, -1, AtomicSite::ReplayDoneIncrement, + &state->replay_done.value, 1 + ); + const uint32_t final_poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, + TraceAtomicSiteMask(AtomicSite::ReplayDonePoll) | TraceAtomicSiteMask(AtomicSite::FaninFlagLoad) + ); + while (true) { + const uint32_t freed = + DrainReady(state, worker, DrainPlace::FinalDrain, stats); + const bool all_replayed = + LoadLine(state->replay_done, stats, AtomicSite::ReplayDonePoll) >= + static_cast(state->config.workers); + // 必须同时满足“无人再生产新 slot”和“本核旧 slot 全部完成”，否则继续帮助系统推进 completion。 + if (all_replayed && worker.occupied_count == 0) { + break; + } + if (freed == 0) { + Ops::SpinHint(); + } + } + AtomicPollRegionEnd(stats.trace, stats.result, final_poll_region); + const uint64_t final_drain_end = TraceTimestamp(stats.trace, stats.result); + if (orchestration_begin != 0 && orchestration_end >= orchestration_begin) { + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::OrchestrationReplay, + ProfilePhase::Orchestration, orchestration_begin, orchestration_end + ); + } + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::FinalDrain, + ProfilePhase::ReplayTail, final_drain_begin, final_drain_end + ); + +#if !PA_BUILD_SUBMIT_PMU + if (stats.trace.atomics_enabled) { + // 两条基线都放在最终 drain 之后。第一条量连续 + // SYS_CNT，第二条量返回依赖钩子的固定成本；它们只描述计时底噪，不能 + // 从每条 atomic 中机械相减后宣称得到跨核全局可见性延迟。 + const uint64_t clock_begin = Ops::Now(); + const uint64_t clock_end = Ops::Now(); + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::ClockBaseline, + ProfilePhase::ReplayTail, clock_begin, clock_end + ); + const uint64_t dependency_begin = Ops::Now(); + const uint64_t dependency_end = Ops::NowAfterAtomicResult( + static_cast(worker_id) + ); + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::ClockBaseline, + ProfilePhase::ReplayTail, dependency_begin, dependency_end, + kClockAtomicDependency | + (Ops::kAtomicReturnReadyObserved ? kClockAtomicDependencyApplied : 0U) + ); + } +#endif + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + const uint64_t expected_callback_task_id_sum = + static_cast(task_count) * (task_count - 1U) / 2U; + const bool split_protocol_ok = + lazy_sample_split_runtime.scheduler == state && lazy_sample_split_runtime.worker == &worker && + lazy_sample_split_runtime.task_count == task_count && lazy_sample_split_runtime.worker_id == worker_id && + lazy_sample_split_runtime.owner_worker_id == worker_id && + lazy_sample_split_runtime.caller_state_address != 0 && + lazy_sample_split_runtime.finish_state_address == lazy_sample_split_runtime.caller_state_address && + lazy_sample_split_runtime.finish_calls == task_count && + lazy_sample_split_runtime.task_id_sum == expected_callback_task_id_sum && + lazy_sample_split_runtime.state_cookie == LazySampleSplitStateCookie(worker_id, role) && + lazy_sample_split_runtime.reserved == 0; + if (!split_protocol_ok) { + ++lazy_sample_split_runtime.protocol_errors; + SetFatal(state, stats); + } + stats.result.lazy_sample_split_caller_state_address = lazy_sample_split_runtime.caller_state_address; + stats.result.lazy_sample_split_finish_state_address = lazy_sample_split_runtime.finish_state_address; + stats.result.lazy_sample_split_finish_calls = lazy_sample_split_runtime.finish_calls; + stats.result.lazy_sample_split_protocol_errors = lazy_sample_split_runtime.protocol_errors; + stats.result.lazy_sample_split_state_cookie = lazy_sample_split_runtime.state_cookie; + stats.result.lazy_sample_split_task_id_sum = lazy_sample_split_runtime.task_id_sum; + stats.result.lazy_sample_split_owner_worker_id = lazy_sample_split_runtime.owner_worker_id; + stats.result.lazy_sample_split_reserved = lazy_sample_split_runtime.reserved; +#endif + + // PA writes swimlane records through the ordinary GM cache and explicitly + // cleans each worker's record range before the kernel finishes. + FlushTraceCore(stats.trace, stats.result); + stats.result.finish_cycle = Ops::Now(); + stats.result.max_occupied = stats.max_occupied; + stats.result.final_occupied = worker.occupied_count; + stats.result.final_heap_next = worker.heap_next; + stats.result.map_high_water = static_cast(worker.map.high_water); + stats.result.map_alive_floor = static_cast(worker.map.alive_floor); + stats.result.map_cleaned_upto = static_cast(worker.map.cleaned_upto); + stats.result.map_live_entries = CountLiveMapEntries(worker.map); + PublishResult(state->results[worker_id], stats.result); +} + +template +PA_DEVICE void RunScheduler(PA_GM SchedulerState *state, uint32_t worker_id, CoreRole role) { + // 两个正式 CCEC 构建都不再携带旧 phase-profile 模板副本：swimlane 用 + // records 表达阶段，submit-pmu 使用独立 PMU 边界。其他后端暂时保留原 + // 运行时入口，保证公共 standalone 的 CPU/AscendC 回归不被 CCEC 构建切分影响。 +#if PA_BUILD_SWIMLANE || PA_BUILD_SUBMIT_PMU + RunSchedulerImpl(state, worker_id, role); +#else + // Profile 作为编译期模板参数，只在显式开启时保留阶段累计代码，关闭时不在热路径增加运行时分支。 + if (state->config.profile_phases != 0) { + RunSchedulerImpl(state, worker_id, role); + } else { + RunSchedulerImpl(state, worker_id, role); + } +#endif +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_SCHEDULER_CORE_H diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/pa_trace.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/pa_trace.h new file mode 100644 index 0000000000..c70a4e681b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/pa_trace.h @@ -0,0 +1,655 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_TRACE_H +#define PA_SCHEDULER_COMMON_PA_TRACE_H + +#include "pa_model.h" + +#ifndef PA_DEVICE_NOINLINE +#define PA_DEVICE_NOINLINE PA_DEVICE +#endif + +#ifndef PA_LOOP_NOUNROLL +#define PA_LOOP_NOUNROLL +#endif + +namespace pa_scheduler { + +// 记录区与真实 PA 一样直接拼在定长 Header 后面；worker_id 只选择自己的 +// records 分区，避免记录动作本身制造跨核共享写热点。 +PA_DEVICE PA_GM TraceRecord *GetTraceRecords(PA_GM TraceHeader *header) { + // 输入必须指向完整且按 64 byte 对齐的 trace buffer；返回值只是首条记录， + // 调用方还需按 worker_id * capacity 选择自己的分区。 + return reinterpret_cast(reinterpret_cast(header) + sizeof(TraceHeader)); +} + +struct AtomicPollBurst { + uint64_t start_cycle[kAtomicPollBatchSiteCount]; + uint32_t call_count[kAtomicPollBatchSiteCount]; + uint32_t active_mask; + uint32_t enabled_mask; +}; + +struct TraceContext { + PA_GM TraceCoreState *core; + PA_GM TraceRecord *records; + uint32_t capacity; + bool atomics_enabled; + int32_t lane; + int32_t block_id; + int32_t core_idx; + // 轮询调用数留在 worker 私有上下文，最终一次性发布到 core state； + // 等待热路不为计数再写共享/GM 状态。 + uint64_t poll_calls; + uint64_t poll_batch_records; + bool atomic_counter_overflow; + AtomicPollBurst poll_burst; +}; + +// pa_model.h 也向 host 暴露同一 raw ABI 映射，但 CCEC/AscendC 的单个 TU +// 会先以 host 语境包含该头，再实例化 device 调度器。这里保留明确的 device +// 版本，避免设备函数误调用先前已实例化的 __host__ helper。 +PA_DEVICE AtomicOp TraceAtomicSiteExpectedOp(AtomicSite site) { + switch (site) { + case AtomicSite::StartupIncrement: + case AtomicSite::ReplayDoneIncrement: + return AtomicOp::FetchAdd; + case AtomicSite::FatalSet: + case AtomicSite::CompletionVendExchange: + case AtomicSite::CompletionFlagExchange: + return AtomicOp::Exchange; + case AtomicSite::ClaimMax: + case AtomicSite::FrontierMax: + return AtomicOp::FetchMax; + default: + return AtomicOp::Load; + } +} + +PA_DEVICE int32_t TraceAtomicPollBatchIndex(AtomicSite site) { + switch (site) { + case AtomicSite::StartupPoll: + return 0; + case AtomicSite::FatalPoll: + return 1; + case AtomicSite::FaninFlagLoad: + return 2; + case AtomicSite::HeapFrontierLoad: + return 3; + case AtomicSite::HeapVendLoad: + return 4; + case AtomicSite::ReplayDonePoll: + return 5; + default: + return -1; + } +} + +PA_DEVICE AtomicSite TraceAtomicPollBatchSite(uint32_t index) { + switch (index) { + case 0: + return AtomicSite::StartupPoll; + case 1: + return AtomicSite::FatalPoll; + case 2: + return AtomicSite::FaninFlagLoad; + case 3: + return AtomicSite::HeapFrontierLoad; + case 4: + return AtomicSite::HeapVendLoad; + case 5: + return AtomicSite::ReplayDonePoll; + default: + return AtomicSite::Count; + } +} + +PA_DEVICE bool TraceAtomicSiteIsPollBatchable(AtomicSite site) { + return TraceAtomicPollBatchIndex(site) >= 0; +} + +PA_DEVICE uint32_t TraceAtomicSiteMask(AtomicSite site) { + return 1U << static_cast(site); +} + +// Attach 只缓存本 worker 的 header 状态、分区首址和物理 lane 信息。控制区 +// cache invalidate 在公共调度入口完成，不能随 submit-pmu 编译掉泳道而消失。 +template +PA_DEVICE TraceContext AttachTrace( + PA_GM SchedulerState *state, PA_GM const WorkerState &worker, uint32_t worker_id +) { + TraceContext trace{}; + trace.lane = worker.lane; + trace.block_id = worker.block_id; + trace.core_idx = static_cast(worker_id); +#if PA_BUILD_SUBMIT_PMU + // 诊断 ELF 不含 records 写入、atomic span 或 trace-only SYS_CNT；保留同一 + // TraceContext 形状只是为了复用调度协议源码。 + (void)state; + return trace; +#else + const uint64_t base = state->config.trace_base; + const uint32_t capacity = state->config.trace_records_per_core; + if ((state->config.trace_enabled & kTracePhasesEnabled) == 0 || base == 0 || capacity == 0 || + worker_id >= kWorkers) { + return trace; + } + PA_GM TraceHeader *header = reinterpret_cast(base); + if (worker_id >= header->num_cores) { + return trace; + } + trace.core = &header->cores[worker_id]; + trace.records = &GetTraceRecords(header)[static_cast(worker_id) * capacity]; + // 成功返回的不变量是 core/records/capacity 同时有效；任一前置条件失败则三者 + // 保持空值，后续 WriteTrace/FlushTraceCore 可无分支地安全退化为 no-op。 + trace.capacity = capacity; + trace.atomics_enabled = (state->config.trace_enabled & kTraceAtomicsEnabled) != 0; + trace.core->count = 0; + trace.core->dropped = 0; + trace.core->atomic_calls = 0; + trace.core->poll_calls = 0; + trace.core->poll_batch_records = 0; + trace.core->core_idx = trace.core_idx; + trace.core->block_id = trace.block_id; + trace.core->lane = trace.lane; + return trace; +#endif +} + +template +PA_DEVICE void WriteTrace( + TraceContext &trace, WorkerResult &result, int32_t task_id, int32_t function_id, + TracePhase trace_phase, ProfilePhase profile_phase, uint64_t start_cycle, uint64_t end_cycle, + uint32_t flags = 0, uint32_t auxiliary = 0 +); + +// CCEC 不让栈上的 TraceContext/WorkerResult 引用跨非内联调用。这里仅把 +// PollBatch 固定形状的 64-byte GM 写入抽成共享函数，以抑制各 phase 边界 +// 内联后的代码膨胀；参数只有 GM 指针与标量，局部 batch 状态仍由调用者维护。 +PA_DEVICE_NOINLINE bool WritePollBatchRecordRaw( + PA_GM TraceCoreState *core, PA_GM TraceRecord *records, uint32_t capacity, + uint64_t start_cycle, uint64_t end_cycle, uint32_t call_count, uint32_t site_id +) { +#if PA_BUILD_SUBMIT_PMU + (void)core; + (void)records; + (void)capacity; + (void)start_cycle; + (void)end_cycle; + (void)call_count; + (void)site_id; + return false; +#else + if (core == nullptr || records == nullptr || capacity == 0) { + return false; + } + const uint32_t slot = core->count; + if (slot >= capacity) { + core->dropped = core->dropped + 1; + return false; + } + PA_GM TraceRecord &record = records[slot]; + record.start_cycle = start_cycle; + record.end_cycle = end_cycle; + record.task_id = -1; + record.function_id = -1; + record.phase = static_cast(TracePhase::Atomic); + record.lane = core->lane; + record.block_id = core->block_id; + record.core_idx = core->core_idx; + const AtomicSite site = static_cast(site_id); + record.flags = static_cast(TraceAtomicSiteExpectedOp(site)) | + kAtomicResultUsed | kAtomicPollBatch | + (call_count << kAtomicPollCountShift); + record.auxiliary = site_id; + // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 + core->count = slot + 1; + return true; +#endif +} + +PA_DEVICE uint32_t AtomicTraceFlags( + AtomicOp op, bool result_used, bool return_ready, bool value_zero = false, + uint64_t retries = 0 +) { + // 高 24 bit 只能容纳有限重试次数；A5 硬件 atomicMax 当前报告 0，CPU CAS + // 回归若超过范围则饱和，避免溢出覆盖低位的 op/语义标志。 + constexpr uint64_t kMaxRetries = (1ULL << (32 - kAtomicRetriesShift)) - 1; + const uint32_t encoded_retries = static_cast(retries > kMaxRetries ? kMaxRetries : retries); + return static_cast(op) | (result_used ? kAtomicResultUsed : 0U) | + (value_zero ? kAtomicValueZero : 0U) | (return_ready ? kAtomicReturnReady : 0U) | + (encoded_retries << kAtomicRetriesShift); +} + +PA_DEVICE void CountAtomicCall( + TraceContext &trace, WorkerResult &result, bool poll_batch +) { + if (result.atomic_trace_calls == UINT64_MAX) { + trace.atomic_counter_overflow = true; + return; + } + ++result.atomic_trace_calls; + if (!poll_batch) return; + if (trace.poll_calls == UINT64_MAX) { + trace.atomic_counter_overflow = true; + return; + } + ++trace.poll_calls; +} + +template +PA_DEVICE void AtomicPollBoundaryAt( + TraceContext &trace, uint64_t end_cycle +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)end_cycle; +#else + if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; + const uint32_t active_mask = trace.poll_burst.active_mask; + // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 + // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 + PA_LOOP_NOUNROLL + for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { + const uint32_t bit = 1U << index; + if ((active_mask & bit) == 0) continue; + const uint32_t call_count = trace.poll_burst.call_count[index]; + if (call_count == 0 || call_count > kAtomicPollCountMax) { + trace.atomic_counter_overflow = true; + continue; + } + const AtomicSite site = TraceAtomicPollBatchSite(index); + const bool written = WritePollBatchRecordRaw( + trace.core, trace.records, trace.capacity, + trace.poll_burst.start_cycle[index], end_cycle, + call_count, static_cast(site) + ); + if (written) { + if (trace.poll_batch_records == UINT64_MAX) { + trace.atomic_counter_overflow = true; + } else { + ++trace.poll_batch_records; + } + } + trace.poll_burst.call_count[index] = 0; + } + trace.poll_burst.active_mask = 0; +#endif +} + +template +PA_DEVICE void AtomicPollBoundary(TraceContext &trace, WorkerResult &result) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; +#else + (void)result; + if (trace.poll_burst.active_mask == 0) return; + AtomicPollBoundaryAt(trace, Ops::Now()); +#endif +} + +template +PA_DEVICE uint32_t AtomicPollRegionBegin( + TraceContext &trace, WorkerResult &result, uint32_t site_mask +) { + const uint32_t previous_mask = trace.poll_burst.enabled_mask; +#if PA_BUILD_SUBMIT_PMU + (void)result; + (void)site_mask; +#else + if (!trace.atomics_enabled) return previous_mask; + AtomicPollBoundary(trace, result); + trace.poll_burst.enabled_mask = previous_mask | site_mask; +#endif + return previous_mask; +} + +template +PA_DEVICE void AtomicPollRegionEnd( + TraceContext &trace, WorkerResult &result, uint32_t previous_mask +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)previous_mask; +#else + if (!trace.atomics_enabled) return; + AtomicPollBoundary(trace, result); + trace.poll_burst.enabled_mask = previous_mask; +#endif +} + +PA_DEVICE bool AtomicPollBatchEnabled( + TraceContext &trace, AtomicSite site, AtomicOp actual_op +) { + return trace.atomics_enabled && TraceAtomicSiteIsPollBatchable(site) && + TraceAtomicSiteExpectedOp(site) == actual_op && + (trace.poll_burst.enabled_mask & TraceAtomicSiteMask(site)) != 0; +} + +template +PA_DEVICE void AccumulateAtomicPollCall( + TraceContext &trace, WorkerResult &result, AtomicSite site, uint64_t start_cycle +) { + const int32_t signed_index = TraceAtomicPollBatchIndex(site); + if (signed_index < 0) { + trace.atomic_counter_overflow = true; + return; + } + const uint32_t index = static_cast(signed_index); + const uint32_t bit = 1U << index; + if ((trace.poll_burst.active_mask & bit) == 0) { + trace.poll_burst.start_cycle[index] = start_cycle; + trace.poll_burst.call_count[index] = 0; + trace.poll_burst.active_mask |= bit; + } + uint32_t &call_count = trace.poll_burst.call_count[index]; + ++call_count; + if (call_count == kAtomicPollCountMax) { + AtomicPollBoundary(trace, result); + } +} + +template +PA_DEVICE void WriteAtomicTrace( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, AtomicOp op, + uint64_t start_cycle, uint64_t end_cycle, bool result_used, bool return_ready, + bool value_zero = false, uint64_t retries = 0 +) { + // 一次源码 atomic 只写一条同时含 start/end 的 span；结束时间先于 64B record + // 写入，因此本条区间不直接包含自己的记录写开销，但下一次竞争到达会受它影响。 + CountAtomicCall(trace, result, false); + WriteTrace( + trace, result, task_id, -1, TracePhase::Atomic, ProfilePhase::ReplayTail, + start_cycle, end_cycle, + AtomicTraceFlags(op, result_used, return_ready, value_zero, retries), + static_cast(site) + ); +} + +template +PA_DEVICE T TraceAtomicLoad( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile T *address, bool result_used = true +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::Load(address); +#else + if (!trace.atomics_enabled) return Ops::Load(address); + const bool poll_batch = result_used && AtomicPollBatchEnabled(trace, site, AtomicOp::Load); + const int32_t poll_index = poll_batch ? TraceAtomicPollBatchIndex(site) : -1; + const bool first_in_batch = poll_batch && + (trace.poll_burst.active_mask & (1U << static_cast(poll_index))) == 0; + const uint64_t begin = !poll_batch || first_in_batch ? Ops::Now() : 0; + const T old = Ops::Load(address); + if (poll_batch) { + CountAtomicCall(trace, result, true); + AccumulateAtomicPollCall(trace, result, site, begin); + return old; + } + // CCEC 只在返回值本来就参与协议判断时插入一条依赖 MOV，再读 SYS_CNT。 + // 这样不会把未消费返回值的 RED/no-return 路径强制改成返回型 ATOM。 + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::Load, begin, end, result_used, return_ready, + old == static_cast(0) + ); + return old; +#endif +} + +template +PA_DEVICE T TraceAtomicExchange( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile T *address, T value, bool result_used = false +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::Exchange(address, value); +#else + if (!trace.atomics_enabled) return Ops::Exchange(address, value); + const uint64_t begin = Ops::Now(); + const T old = Ops::Exchange(address, value); + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::Exchange, begin, end, result_used, return_ready + ); + return old; +#endif +} + +template +PA_DEVICE int64_t TraceAtomicFetchAdd( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile int64_t *address, int64_t value, bool result_used = false +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::FetchAdd(address, value); +#else + if (!trace.atomics_enabled) return Ops::FetchAdd(address, value); + const uint64_t begin = Ops::Now(); + const int64_t old = Ops::FetchAdd(address, value); + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::FetchAdd, begin, end, result_used, return_ready + ); + return old; +#endif +} + +template +PA_DEVICE int64_t TraceAtomicFetchMax( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile int64_t *address, int64_t value, uint64_t &retries, bool result_used = true +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::FetchMax(address, value, retries); +#else + if (!trace.atomics_enabled) return Ops::FetchMax(address, value, retries); + const uint64_t begin = Ops::Now(); + const int64_t old = Ops::FetchMax(address, value, retries); + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::FetchMax, begin, end, result_used, + return_ready, false, retries + ); + return old; +#endif +} + +template +PA_DEVICE void AccumulatePhase( + WorkerResult &result, ProfilePhase phase, uint64_t start_cycle, uint64_t end_cycle +) { +#if PA_BUILD_SUBMIT_PMU + (void)result; + (void)phase; + (void)start_cycle; + (void)end_cycle; + return; +#else + // phase profile 与完整泳道是两套正交机制：即使关闭 records，Profile=true + // 仍会累计用户当前关注的 Claim/EfDrain/WaitForSlot/HeapGuard 四段。 + if constexpr (Profile) { + if (phase != ProfilePhase::Claim && phase != ProfilePhase::EfDrain && + phase != ProfilePhase::WaitForSlot && phase != ProfilePhase::HeapGuard) { + return; + } + const uint32_t index = static_cast(phase); + // 调用方保证 end_cycle>=start_cycle；各后端把 Now() 归一到每 tick 1 ns 的 + // 数值标度，聚合持续时间可直接相加并在 host 侧按 1000 换算为微秒。 + const uint64_t duration = end_cycle - start_cycle; + result.phase_cycles[index] += duration; + ++result.phase_calls[index]; + } +#endif +} + +template +PA_DEVICE void WriteTrace( + TraceContext &trace, WorkerResult &result, int32_t task_id, int32_t function_id, TracePhase trace_phase, + ProfilePhase profile_phase, uint64_t start_cycle, uint64_t end_cycle, uint32_t flags, + uint32_t auxiliary +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)function_id; + (void)trace_phase; + (void)profile_phase; + (void)start_cycle; + (void)end_cycle; + (void)flags; + (void)auxiliary; + return; +#else + // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 + // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 + AccumulatePhase(result, profile_phase, start_cycle, end_cycle); + if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { + return; + } + PA_GM TraceCoreState &core = *trace.core; + const uint32_t slot = core.count; + if (slot >= trace.capacity) { + core.dropped = core.dropped + 1; + return; + } + PA_GM TraceRecord &record = trace.records[slot]; + record.start_cycle = start_cycle; + record.end_cycle = end_cycle; + record.task_id = task_id; + record.function_id = function_id; + record.phase = static_cast(trace_phase); + record.lane = trace.lane; + record.block_id = trace.block_id; + record.core_idx = trace.core_idx; + record.flags = flags; + record.auxiliary = auxiliary; + // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 + // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 + core.count = slot + 1; +#endif +} + +template +PA_DEVICE void ResetTraceLap( + TraceContext &trace, WorkerResult &result, PA_GM WorkerState &worker +) { + // lap 是后续 Build/Replay/Alloc 等覆盖式阶段的共同起点，不代表新增嵌套 span。 + // 因此分析时不能把 lap 时长再与其中的 Materialize/Claim/Register 直接相加。 +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)worker; +#else + (void)result; + const uint64_t cycle = Ops::Now(); + // 与真实 FDWIC 的 TRACE_LAP_RESET 保持同一边界：等待区 PollBatch + // 只能覆盖本次逻辑轮询 episode，不能跨进下一段 lap 或计算单元执行。 + AtomicPollBoundaryAt(trace, cycle); + worker.swimlane_last_cycle = cycle; +#endif +} + +template +PA_DEVICE void FlushTraceCore(TraceContext &trace, WorkerResult &result) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + return; +#else + if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { + return; + } + // 防御性关闭任何尚未由显式 region end 关闭的等待包；正常路径上 active_mask + // 应为 0，这里仍保证异常早退不会留下“有逻辑调用、无物理 batch”的半截采集。 + AtomicPollBoundary(trace, result); + PA_GM TraceCoreState &core = *trace.core; + if (trace.atomic_counter_overflow || result.atomic_trace_calls > UINT32_MAX || + trace.poll_calls > UINT32_MAX || trace.poll_batch_records > UINT32_MAX) { + if (core.dropped != UINT32_MAX) core.dropped = core.dropped + 1; + } + core.atomic_calls = static_cast(result.atomic_trace_calls); + core.poll_calls = static_cast(trace.poll_calls); + core.poll_batch_records = static_cast(trace.poll_batch_records); + const uint32_t count = core.count < trace.capacity ? core.count : trace.capacity; + // A5 侧记录经普通 GM cache 写入，kernel 结束前必须把有效 records 与最后的 + // count/dropped cache line 显式 clean，host 的 D2H 才能得到完整且自洽的快照。 + if (count != 0) { + Ops::FlushRegion(trace.records, static_cast(count) * sizeof(TraceRecord)); + } + Ops::FlushRegion(&core, sizeof(core)); +#endif +} + +template +PA_DEVICE uint64_t WriteTraceLap( + TraceContext &trace, PA_GM WorkerState &worker, WorkerResult &result, int32_t task_id, + int32_t function_id, TracePhase trace_phase, ProfilePhase profile_phase, + uint32_t flags = 0, uint32_t auxiliary = 0 +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)worker; + (void)result; + (void)task_id; + (void)function_id; + (void)trace_phase; + (void)profile_phase; + (void)flags; + (void)auxiliary; + return 0; +#else + // lap 记录区间 [上一次 Reset/WriteTraceLap, 当前时刻]，写完立即推进起点。 + // 显式 WriteTrace span 不会修改该起点，这正是生产泳道中阶段可重叠的原因。 + const uint64_t end_cycle = Ops::Now(); + // 真实 FDWIC 在 TRACE_LAP 取到结束时间后先关闭 PollBatch，再写 lap。 + // 复用同一个 end_cycle，避免额外 SYS_CNT 造成可见缝隙。 + AtomicPollBoundaryAt(trace, end_cycle); + WriteTrace( + trace, result, task_id, function_id, trace_phase, profile_phase, worker.swimlane_last_cycle, end_cycle, + flags, auxiliary + ); + worker.swimlane_last_cycle = end_cycle; + return end_cycle; +#endif +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_TRACE_H diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/winner_workload.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/winner_workload.h new file mode 100644 index 0000000000..a3995fb50f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/winner_workload.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_WINNER_WORKLOAD_H +#define PA_SCHEDULER_COMMON_WINNER_WORKLOAD_H + +#include "pa_model.h" + +#include +#include + +namespace pa_scheduler::winner_workload { + +// 三种 standalone 后端共享完全相同的 GM/host 布局。128x128 float 是 CCEC +// 已在 A5 上验证过的基本形状；后端可以采用不同指令接口，但不能改变输入、 +// 输出 tile 编址或 host 数值校验口径。 +constexpr uint32_t kTileRows = 128; +constexpr uint32_t kTileCols = 128; +constexpr size_t kTileElements = static_cast(kTileRows) * kTileCols; +constexpr size_t kTileBytes = kTileElements * sizeof(float); +constexpr uint32_t kSharedInputTiles = 2; +constexpr uint32_t kOutputTilesPerWorker = 2; +constexpr uint32_t kOutputTiles = kWorkers * kOutputTilesPerWorker; +constexpr size_t kWorkspaceTiles = kSharedInputTiles + kOutputTiles; +constexpr size_t kWorkspaceBytes = kWorkspaceTiles * kTileBytes; +constexpr float kInputAValue = 2.0F; +constexpr float kInputBValue = 3.0F; +constexpr float kExpectedAicValue = 768.0F; +constexpr float kExpectedSfValue = 5.0F; +constexpr float kExpectedUpValue = 6.0F; +constexpr float kOutputSentinel = -12345.0F; + +// 256 batch 下，即使同一 AIC 极端地拿到全部 QK/PV，128 次完整 Cube +// 迭代的 CCEC 实测 busy 上界仍低于 32-bit PMU 的 25% 门槛。其他后端也沿用 +// 此参数边界，避免相同命令在不同实现上产生不同含义。 +constexpr uint32_t kMaxRealComputeCount = 128; + +// 1 次用于最小正确性取证。默认次数来自 CCEC 的三个独立 b256 A5 进程； +// AscendC 必须重新标定后才能宣称达到同样时长，不能仅因共享默认参数便沿用 +// CCEC 的性能结论。UP 的一次完整 128x128 流水是当前正整数下限。 +constexpr WorkloadCounts kRealComputeSmokeCounts{1, 1, 1, 1}; +constexpr WorkloadCounts kDefaultRealComputeCounts{6, 28, 4, 1}; + +static_assert(kTileBytes == 65536, "real-compute tile must occupy 64 KiB"); +static_assert(kWorkspaceBytes == 12713984, "real-compute workspace size changed unexpectedly"); + +} // namespace pa_scheduler::winner_workload + +#endif // PA_SCHEDULER_COMMON_WINNER_WORKLOAD_H diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/winner_workload_host.h b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/winner_workload_host.h new file mode 100644 index 0000000000..07972cc96b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/common/winner_workload_host.h @@ -0,0 +1,361 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_WINNER_WORKLOAD_HOST_H +#define PA_SCHEDULER_COMMON_WINNER_WORKLOAD_HOST_H + +#include "host_support.h" +#include "winner_workload.h" + +#include +#include +#include + +namespace pa_scheduler::host { + +// 常量模式用于稳定的性能负载；布局诊断模式使用带权对角 A 和非对称稠密 B， +// 专门暴露 B 转置、ND/NZ stride 与输出重排错误。该选择只影响计时窗外的 +// host 输入生成和结果校验，不进入 SchedulerState，也不增加 device 热路径分支。 +enum class RealComputePattern : uint32_t { + Constant = 0, + LayoutDiagnostic = 1, +}; + +// winner 负载参数在通用 benchmark parser 前单独剥离，CCEC 可在其后继续剥离 +// PMU 参数；AscendC/CPU 则直接把剩余 argv 交给 ParseOptions。这样不把后端 +// 私有功能塞入公共 PA 参数结构，也不会复制三套互斥规则。 +struct WinnerWorkloadOptions { + WinnerWorkloadMode mode = WinnerWorkloadMode::RealCompute; + WorkloadCounts repeats = winner_workload::kDefaultRealComputeCounts; + RealComputePattern pattern = RealComputePattern::Constant; + bool counts_explicit = false; + bool pattern_explicit = false; + bool nop_override_explicit = false; +}; + +inline const char *WinnerWorkloadModeName(WinnerWorkloadMode mode) { + switch (mode) { + case WinnerWorkloadMode::ScalarNop: + return "scalar-nop"; + case WinnerWorkloadMode::RealCompute: + return "real-compute"; + } + return "invalid"; +} + +inline const char *RealComputePatternName(RealComputePattern pattern) { + switch (pattern) { + case RealComputePattern::Constant: + return "constant"; + case RealComputePattern::LayoutDiagnostic: + return "layout-diagnostic"; + } + return "invalid"; +} + +inline bool ParseWorkloadCounts(const char *raw, WorkloadCounts *counts) { + unsigned int qk = 0; + unsigned int sf = 0; + unsigned int pv = 0; + unsigned int up = 0; + char tail = '\0'; + if (std::sscanf(raw, "%u,%u,%u,%u%c", &qk, &sf, &pv, &up, &tail) != 4) return false; + const uint32_t maximum = winner_workload::kMaxRealComputeCount; + if (qk == 0 || sf == 0 || pv == 0 || up == 0 || + qk > maximum || sf > maximum || pv > maximum || up > maximum) { + return false; + } + *counts = WorkloadCounts{qk, sf, pv, up}; + return true; +} + +inline bool ParseWinnerWorkloadOptions( + int argc, char **argv, WinnerWorkloadOptions *workload, std::vector *remaining_argv +) { + bool mode_seen = false; + bool count_seen = false; + bool pattern_seen = false; + remaining_argv->clear(); + remaining_argv->push_back(argv[0]); + for (int index = 1; index < argc; ++index) { + const std::string argument = argv[index]; + if (argument == "--nop-count" || argument == "--nop-counts") { + workload->nop_override_explicit = true; + } + if (argument != "--winner-workload" && argument != "--real-compute-count" && + argument != "--real-compute-counts" && argument != "--real-compute-pattern") { + remaining_argv->push_back(argv[index]); + continue; + } + if (index + 1 >= argc) { + std::fprintf(stderr, "Missing value after %s\n", argument.c_str()); + return false; + } + const char *value = argv[++index]; + if (argument == "--winner-workload") { + if (mode_seen) { + std::fprintf(stderr, "Specify --winner-workload only once.\n"); + return false; + } + const std::string name = value; + if (name == "scalar-nop") { + workload->mode = WinnerWorkloadMode::ScalarNop; + } else if (name == "real-compute") { + workload->mode = WinnerWorkloadMode::RealCompute; + } else { + std::fprintf( + stderr, + "Invalid --winner-workload value: %s (expected scalar-nop|real-compute)\n", + value + ); + return false; + } + mode_seen = true; + continue; + } + if (argument == "--real-compute-pattern") { + if (pattern_seen) { + std::fprintf(stderr, "Specify --real-compute-pattern only once.\n"); + return false; + } + const std::string name = value; + if (name == "constant") { + workload->pattern = RealComputePattern::Constant; + } else if (name == "layout-diagnostic") { + workload->pattern = RealComputePattern::LayoutDiagnostic; + } else { + std::fprintf( + stderr, + "Invalid --real-compute-pattern value: %s " + "(expected constant|layout-diagnostic)\n", + value + ); + return false; + } + pattern_seen = true; + workload->pattern_explicit = true; + continue; + } + if (count_seen) { + std::fprintf(stderr, "Specify only one real-compute count override.\n"); + return false; + } + if (argument == "--real-compute-count") { + uint32_t count = 0; + if (!ParseUint(value, 1, winner_workload::kMaxRealComputeCount, &count)) { + std::fprintf(stderr, "Invalid --real-compute-count value: %s\n", value); + return false; + } + workload->repeats = WorkloadCounts{count, count, count, count}; + } else if (!ParseWorkloadCounts(value, &workload->repeats)) { + std::fprintf(stderr, "Invalid --real-compute-counts value: %s\n", value); + return false; + } + count_seen = true; + workload->counts_explicit = true; + } + // 无参数运行以真实 Cube/Vector 为默认。旧命令若显式给出 NOP 次数但没有 + // 指定 workload mode，则把 NOP override 本身视为选择 scalar-nop；这样 + // 既不让 --nop-count 悄悄失效，也不破坏既有标定脚本。显式指定 + // real-compute 再叠加 NOP 仍由下方互斥校验拒绝。 + if (!mode_seen && workload->nop_override_explicit) { + workload->mode = WinnerWorkloadMode::ScalarNop; + } + return true; +} + +inline bool ValidateWinnerWorkloadOptions(const WinnerWorkloadOptions &workload) { + if (workload.mode == WinnerWorkloadMode::RealCompute) { + if (workload.pattern != RealComputePattern::Constant && + workload.pattern != RealComputePattern::LayoutDiagnostic) { + std::fprintf(stderr, "Invalid real-compute input pattern.\n"); + return false; + } + if (workload.nop_override_explicit) { + std::fprintf( + stderr, + "--winner-workload real-compute cannot be combined with --nop-count or --nop-counts.\n" + ); + return false; + } + return true; + } + if (workload.counts_explicit || workload.pattern_explicit) { + std::fprintf( + stderr, + "--real-compute-count(s)/pattern requires real-compute workload mode.\n" + ); + return false; + } + return true; +} + +inline void ConfigureWinnerWorkload( + SchedulerState *state, const WinnerWorkloadOptions &workload, const void *workspace_device +) { + state->winner_workload.mode = static_cast(workload.mode); + state->winner_workload.version = kWinnerWorkloadConfigVersion; + state->winner_workload.repeats = workload.repeats; + state->winner_workload.workspace_base = reinterpret_cast(workspace_device); + state->winner_workload.workspace_bytes = + workload.mode == WinnerWorkloadMode::RealCompute ? winner_workload::kWorkspaceBytes : 0; +} + +inline float LayoutDiagnosticInputA(uint32_t row, uint32_t column) { + return row == column ? static_cast(row + 1U) : 0.0F; +} + +inline float LayoutDiagnosticInputB(uint32_t row, uint32_t column) { + const uint32_t value = + (131U * row + 17U * column + 7U * row * column) % 251U; + return static_cast(value + 1U); +} + +inline void InitializeWinnerWorkloadBuffers( + const WinnerWorkloadOptions &workload, std::vector *workspace_image, + std::vector *workspace_outputs +) { + using namespace winner_workload; + workspace_image->assign(kWorkspaceTiles * kTileElements, kOutputSentinel); + if (workload.pattern == RealComputePattern::Constant) { + std::fill_n(workspace_image->begin(), kTileElements, kInputAValue); + std::fill_n(workspace_image->begin() + kTileElements, kTileElements, kInputBValue); + } else { + for (uint32_t row = 0; row < kTileRows; ++row) { + for (uint32_t column = 0; column < kTileCols; ++column) { + const size_t element = static_cast(row) * kTileCols + column; + (*workspace_image)[element] = LayoutDiagnosticInputA(row, column); + (*workspace_image)[kTileElements + element] = + LayoutDiagnosticInputB(row, column); + } + } + } + workspace_outputs->resize(static_cast(kOutputTiles) * kTileElements); +} + +inline const char *TaskKindName(TaskKind kind) { + switch (kind) { + case TaskKind::Qk: + return "QK"; + case TaskKind::Sf: + return "SF"; + case TaskKind::Pv: + return "PV"; + case TaskKind::Up: + return "UP"; + default: + return "invalid"; + } +} + +inline float ExpectedRealComputeValue( + RealComputePattern pattern, TaskKind kind, uint32_t row, uint32_t column +) { + using namespace winner_workload; + if (pattern == RealComputePattern::Constant) { + return kind == TaskKind::Qk || kind == TaskKind::Pv + ? kExpectedAicValue + : (kind == TaskKind::Sf ? kExpectedSfValue : kExpectedUpValue); + } + const float input_a = LayoutDiagnosticInputA(row, column); + const float input_b = LayoutDiagnosticInputB(row, column); + if (kind == TaskKind::Qk || kind == TaskKind::Pv) { + // A 是带权对角矩阵，因此 A*B 的 (row,column) 只有一个非零乘积。 + return static_cast(row + 1U) * input_b; + } + return kind == TaskKind::Sf ? input_a + input_b : input_a * input_b; +} + +inline bool ValidateRealComputeOutputs( + const SchedulerState &state, const WinnerWorkloadOptions &workload, + const std::vector &outputs, uint32_t run +) { + using namespace winner_workload; + const size_t expected_elements = static_cast(kOutputTiles) * kTileElements; + if (outputs.size() != expected_elements) { + std::fprintf( + stderr, "[ASSERT] real-compute output buffer size matches workspace layout FAIL\n" + ); + return false; + } + + uint32_t active_tiles = 0; + uint32_t inactive_tiles = 0; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const WorkerResult &result = state.results[worker]; + const bool aic = result.role == static_cast(CoreRole::Aic); + const TaskKind kinds[2] = { + aic ? TaskKind::Qk : TaskKind::Sf, + aic ? TaskKind::Pv : TaskKind::Up, + }; + for (uint32_t kind_slot = 0; kind_slot < 2; ++kind_slot) { + const TaskKind kind = kinds[kind_slot]; + const uint32_t kernel_index = static_cast(kind) - 1; + const bool active = result.kernel_counts[kernel_index] != 0; + const size_t tile_index = + static_cast(worker) * kOutputTilesPerWorker + kind_slot; + const size_t begin = tile_index * kTileElements; + for (size_t element = 0; element < kTileElements; ++element) { + const uint32_t row = static_cast(element / kTileCols); + const uint32_t column = static_cast(element % kTileCols); + const float expected = active + ? ExpectedRealComputeValue(workload.pattern, kind, row, column) + : kOutputSentinel; + if (outputs[begin + element] == expected) continue; + std::fprintf( + stderr, + "[REAL-COMPUTE-FAIL] run=%u worker=%u kind=%s element=%zu " + "expected=%.1f actual=%.9g\n", + run, worker, TaskKindName(kind), element, expected, + static_cast(outputs[begin + element]) + ); + std::fprintf( + stderr, + "[ASSERT] real-compute output tiles match role-specific engine results FAIL\n" + ); + return false; + } + active_tiles += active ? 1U : 0U; + inactive_tiles += active ? 0U : 1U; + } + } + const bool passed = active_tiles != 0 && active_tiles + inactive_tiles == kOutputTiles; + std::printf( + "[ASSERT] %-48s %s (active_tiles=%u inactive_sentinel_tiles=%u)\n", + "real-compute output tiles match role-specific engine results", + passed ? "PASS" : "FAIL", active_tiles, inactive_tiles + ); + return passed; +} + +inline void PrintWinnerWorkloadConfig( + const WinnerWorkloadOptions &workload, const NopCounts &nops +) { + if (workload.mode == WinnerWorkloadMode::RealCompute) { + std::printf( + "[WINNER-WORKLOAD] mode=real-compute pattern=%s counts=%u,%u,%u,%u " + "unit=complete_128x128_engine_pipeline_iteration workspace_bytes=%zu\n", + RealComputePatternName(workload.pattern), + workload.repeats.qk, workload.repeats.sf, workload.repeats.pv, + workload.repeats.up, winner_workload::kWorkspaceBytes + ); + return; + } + std::printf( + "[WINNER-WORKLOAD] mode=scalar-nop counts=%u,%u,%u,%u " + "unit=scalar_nop_instruction workspace_bytes=0\n", + nops.qk, nops.sf, nops.pv, nops.up + ); +} + +} // namespace pa_scheduler::host + +#endif // PA_SCHEDULER_COMMON_WINNER_WORKLOAD_HOST_H diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassemble.py b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassemble.py new file mode 100755 index 0000000000..16eac59111 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassemble.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python3 +"""Disassemble every final-linked STT_FUNC with the verified A5 PEM decoder.""" + +from __future__ import annotations + +import argparse +import ctypes +import gzip +import hashlib +import io +import re +import struct +from dataclasses import dataclass +from pathlib import Path + + +EXPECTED_DECODER_SHA256 = "29835d2439d6dd464d34a212ad4bbd5c29af6a38465da09a6c273401d9a96dcb" +INSTRUCTION_RE = re.compile(r"^\s*([0-9a-fA-F]+):\s+((?:[0-9a-fA-F]{8})+)\s+(.*?)\s*$") +BAD_MNEMONICS = {"UNDEF", "UNKNOWN", "INVALID", "ILLEGAL", "ERROR", " str: + return hashlib.sha256(data).hexdigest() + + +def c_string(table: bytes, offset: int) -> str: + end = table.find(b"\0", offset) + if end < 0: + raise ValueError("unterminated ELF string") + return table[offset:end].decode("utf-8") + + +@dataclass(frozen=True) +class Section: + index: int + name: str + section_type: int + address: int + offset: int + size: int + link: int + entry_size: int + + +@dataclass(frozen=True) +class Function: + name: str + value: int + size: int + binding: str + + +class Elf64: + def __init__(self, path: Path): + self.path = path + self.data = path.read_bytes() + header_format = "<16sHHIQQQIHHHHHH" + if len(self.data) < struct.calcsize(header_format): + raise ValueError(f"truncated ELF: {path}") + header = struct.unpack_from(header_format, self.data, 0) + ident = header[0] + if ident[:4] != b"\x7fELF" or ident[4] != 2 or ident[5] != 1: + raise ValueError(f"expected ELF64 little-endian image: {path}") + section_header_offset = header[6] + section_header_size = header[11] + section_count = header[12] + section_name_index = header[13] + if section_header_size != 64 or section_count == 0: + raise ValueError(f"unsupported section table: {path}") + + raw_sections = [] + section_format = " bytes: + return self.data[section.offset : section.offset + section.size] + + def functions(self) -> list[Function]: + symtab = next(section for section in self.sections if section.section_type == 2) + if symtab.entry_size != 24 or symtab.size % symtab.entry_size != 0: + raise ValueError(f"unsupported symbol table: {self.path}") + strings = self.section_bytes(self.sections[symtab.link]) + functions = [] + for offset in range(symtab.offset, symtab.offset + symtab.size, symtab.entry_size): + name_offset, info, _, section_index, value, size = struct.unpack_from( + "> 4, str(info >> 4)) + functions.append(Function(c_string(strings, name_offset), value, size, binding)) + functions.sort(key=lambda function: (function.value, function.name)) + if not functions: + raise ValueError(f"no final-linked .text functions: {self.path}") + return functions + + +class PemDecoder: + def __init__(self, path: Path): + self.path = path + decoder_bytes = path.read_bytes() + self.digest = sha256(decoder_bytes) + if self.digest != EXPECTED_DECODER_SHA256: + raise ValueError( + f"unsupported PEM decoder SHA256 {self.digest}; expected {EXPECTED_DECODER_SHA256}" + ) + library = ctypes.CDLL(str(path)) + self.dump = library.pem_turbo_objdump + self.dump.argtypes = ( + ctypes.POINTER(ctypes.c_uint32), + ctypes.c_size_t, + ctypes.c_void_p, + ctypes.c_size_t, + ) + self.dump.restype = ctypes.c_void_p + self.libc = ctypes.CDLL(None) + self.libc.free.argtypes = (ctypes.c_void_p,) + + def decode(self, body: bytes, rvec: bool) -> list[tuple[int, str, str]]: + if len(body) % 4 != 0: + raise ValueError("function body is not an integral number of instruction words") + word_count = len(body) // 4 + words = (ctypes.c_uint32 * word_count).from_buffer_copy(body) + range_pointer = None + range_count = 0 + if rvec: + ranges_type = (ctypes.c_uint64 * 2) * 1 + ranges = ranges_type() + ranges[0][0] = 0 + ranges[0][1] = len(body) + range_pointer = ctypes.cast(ranges, ctypes.c_void_p) + range_count = 1 + result = self.dump(words, word_count, range_pointer, range_count) + if not result: + raise RuntimeError("pem_turbo_objdump returned null") + try: + decoded = ctypes.string_at(result).decode("utf-8") + finally: + self.libc.free(result) + + instructions = [] + for line in decoded.splitlines(): + match = INSTRUCTION_RE.fullmatch(line) + if match is None: + if line.strip(): + raise ValueError(f"unexpected decoder output: {line!r}") + continue + relative = int(match.group(1), 16) + machine_word = match.group(2).lower() + mnemonic = match.group(3).strip() + instructions.append((relative, machine_word, mnemonic)) + expected_relative = 0 + for index, (relative, machine_word, mnemonic) in enumerate(instructions): + encoded_bytes = len(machine_word) // 2 + if encoded_bytes % 4 != 0: + raise ValueError( + f"decoder returned a non-word-sized encoding at row {index}: {machine_word}" + ) + expected_word = ( + f"{int.from_bytes(body[expected_relative:expected_relative + encoded_bytes], 'little'):0{len(machine_word)}x}" + ) + if relative != expected_relative or machine_word != expected_word: + raise ValueError( + f"decoder coverage mismatch at word {index}: {relative:x}/{machine_word} " + f"!= {expected_relative:x}/{expected_word}" + ) + first_token = mnemonic.upper().split(maxsplit=1)[0] if mnemonic else "" + if first_token in BAD_MNEMONICS or "NOT AVAILABLE" in mnemonic.upper(): + raise ValueError(f"invalid decoded instruction at +0x{relative:x}: {mnemonic}") + expected_relative += encoded_bytes + if expected_relative != len(body): + raise ValueError( + f"decoder covered {expected_relative} bytes of a {len(body)}-byte function" + ) + return instructions + + +def safe_filename(index: int, name: str) -> str: + safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", name).strip("._") or "function" + suffix = hashlib.sha256(name.encode("utf-8")).hexdigest()[:8] + return f"{index:02d}_{safe[:72]}_{suffix}.asm.gz" + + +def write_gzip(path: Path, text: str) -> None: + with path.open("wb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed: + compressed.write(text.encode("utf-8")) + + +def disassemble_variant( + variant: str, elf_path: Path, output_dir: Path, decoder: PemDecoder +) -> tuple[list[str], list[str], dict[str, object]]: + elf = Elf64(elf_path) + text = elf.section_bytes(elf.text) + functions = elf.functions() + variant_dir = output_dir + variant_dir.mkdir(parents=True) + elf_digest = sha256(elf.data) + text_digest = sha256(text) + manifest_rows = [] + gap_rows = [] + cursor = elf.text.address + total_function_bytes = 0 + + for index, function in enumerate(functions): + if function.value < cursor: + raise ValueError(f"overlapping final function symbols in {elf_path}: {function.name}") + if function.value > cursor: + gap_start = cursor + gap_end = function.value + start = gap_start - elf.text.address + gap = text[start : start + gap_end - gap_start] + gap_rows.append( + f"0x{gap_start:x}\t0x{gap_end:x}\t{len(gap)}\t{sha256(gap)}" + ) + body_offset = function.value - elf.text.address + body = text[body_offset : body_offset + function.size] + if len(body) != function.size: + raise ValueError(f"function outside .text: {function.name}") + rvec = ".vector.thread" in function.name + try: + instructions = decoder.decode(body, rvec) + except Exception as error: + raise ValueError(f"{variant}:{function.name}: {error}") from error + body_digest = sha256(body) + filename = safe_filename(index, function.name) + header = [ + "# schema=pa_final_linked_disassembly/v1", + f"# variant={variant}", + f"# final_elf={elf_path.name}", + f"# final_elf_sha256={elf_digest}", + f"# final_text_address=0x{elf.text.address:x}", + f"# final_text_size={elf.text.size}", + f"# final_text_sha256={text_digest}", + f"# symbol={function.name}", + f"# binding={function.binding}", + f"# final_pc=0x{function.value:x}", + f"# size={function.size}", + f"# instruction_count={len(instructions)}", + f"# encoded_word_count={function.size // 4}", + f"# body_sha256={body_digest}", + "# decoder=$ASCEND_HOME_PATH/x86_64-linux/simulator/dav_3510/lib/libpem_davinci.so", + f"# decoder_sha256={decoder.digest}", + f"# decoder_mode={'rvec' if rvec else 'scalar'}", + "# columns=final_pc function_relative_offset machine_word instruction", + "", + ] + body_lines = [ + f"0x{function.value + relative:016x} (+0x{relative:08x}) {machine_word} {mnemonic}" + for relative, machine_word, mnemonic in instructions + ] + write_gzip(variant_dir / filename, "\n".join(header + body_lines) + "\n") + last_mnemonic = instructions[-1][2].split(maxsplit=1)[0] + manifest_rows.append( + "\t".join( + ( + variant, + filename, + function.binding, + function.name, + f"0x{function.value:x}", + str(function.size), + str(len(instructions)), + "rvec" if rvec else "scalar", + body_digest, + last_mnemonic, + ) + ) + ) + cursor = function.value + function.size + total_function_bytes += function.size + + text_end = elf.text.address + elf.text.size + if cursor < text_end: + start = cursor - elf.text.address + gap = text[start:] + gap_rows.append(f"0x{cursor:x}\t0x{text_end:x}\t{len(gap)}\t{sha256(gap)}") + elif cursor > text_end: + raise ValueError(f"function extends past .text: {elf_path}") + + metadata = { + "variant": variant, + "elf_sha256": elf_digest, + "text_address": elf.text.address, + "text_size": elf.text.size, + "text_sha256": text_digest, + "function_count": len(functions), + "function_bytes": total_function_bytes, + "function_coverage_percent": total_function_bytes / elf.text.size * 100.0, + "gap_count": len(gap_rows), + } + return manifest_rows, gap_rows, metadata + + +def main() -> None: + parser = argparse.ArgumentParser() + script_dir = Path(__file__).resolve().parent + fixed_variant = (script_dir / "VARIANT").read_text(encoding="utf-8").strip() + parser.add_argument("--variant", default=fixed_variant) + parser.add_argument( + "--elf", type=Path, + default=script_dir / "artifacts" / "measured" / "pa_scheduler_kernel.o" + ) + parser.add_argument("--output", type=Path, default=script_dir / "disassembly" / "raw") + default_decoder = None + import os + + if os.environ.get("ASCEND_HOME_PATH"): + default_decoder = ( + Path(os.environ["ASCEND_HOME_PATH"]) + / "x86_64-linux/simulator/dav_3510/lib/libpem_davinci.so" + ) + parser.add_argument("--decoder", type=Path, default=default_decoder, required=default_decoder is None) + args = parser.parse_args() + elf_path = args.elf.resolve() + output_dir = args.output.resolve() + decoder_path = args.decoder.resolve() + if output_dir.exists(): + raise SystemExit(f"Output directory already exists; choose a new path: {output_dir}") + decoder = PemDecoder(decoder_path) + + if not elf_path.is_file(): + raise SystemExit(f"Missing published final ELF: {elf_path}") + manifest_rows, local_gaps, item = disassemble_variant( + args.variant, elf_path, output_dir, decoder + ) + gap_rows = [f"{args.variant}\t{row}" for row in local_gaps] + + manifest_header = ( + "variant\tfile\tbinding\tsymbol\tfinal_pc\tsize\tinstructions\tdecoder_mode\t" + "body_sha256\tlast_mnemonic" + ) + (output_dir / "manifest.tsv").write_text( + manifest_header + "\n" + "\n".join(manifest_rows) + "\n", encoding="utf-8" + ) + (output_dir / "gaps.tsv").write_text( + "variant\tstart_pc\tend_pc\tsize\tsha256\n" + "\n".join(gap_rows) + "\n", + encoding="utf-8", + ) + readme = [ + "# Final-linked A5 disassembly", + "", + "Every non-empty final `.text` `STT_FUNC` symbol was extracted from the published final ELF and decoded with the verified `dav_3510` PEM decoder.", + "RVec `.vector.thread` symbols are passed through an explicit full-body RVec range; all other symbols use scalar decoding.", + "Each compressed file records final PC, function-relative offset, machine word, mnemonic, ELF/body hashes and decoder identity.", + "Function gaps are alignment/padding outside symbol bodies and are hashed in `gaps.tsv`; they are not presented as instructions.", + "", + f"Decoder SHA256: `{decoder.digest}`", + "", + "| Variant | Final .text | Function symbols | Function bytes | Coverage | Final ELF SHA256 |", + "| --- | ---: | ---: | ---: | ---: | --- |", + ] + readme.append( + f"| {item['variant']} | {item['text_size']} B | {item['function_count']} | " + f"{item['function_bytes']} B | {item['function_coverage_percent']:.3f}% | " + f"`{item['elf_sha256']}` |" + ) + readme.extend( + [ + "", + "Inspect with `zless FILE.asm.gz`; the sibling `annotated/` directory adds DWARF-mapped local source and comments.", + ] + ) + (output_dir / "README.md").write_text("\n".join(readme) + "\n", encoding="utf-8") + + published = [] + for path in sorted(item for item in output_dir.rglob("*") if item.is_file()): + if path.name == "published.sha256": + continue + published.append(f"{sha256(path.read_bytes())} {path.relative_to(output_dir)}") + (output_dir / "published.sha256").write_text("\n".join(published) + "\n", encoding="utf-8") + print("\n".join(readme)) + + +if __name__ == "__main__": + main() diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/00_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/00_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz new file mode 100644 index 0000000000..1d09f45c7a Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/00_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz new file mode 100644 index 0000000000..ef29b27c53 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/02_pa_execute_real_winner_workload_aic_729699de.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/02_pa_execute_real_winner_workload_aic_729699de.source.asm.gz new file mode 100644 index 0000000000..26229b6971 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/02_pa_execute_real_winner_workload_aic_729699de.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.source.asm.gz new file mode 100644 index 0000000000..11d39da6ed Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz new file mode 100644 index 0000000000..8d52246564 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.source.asm.gz new file mode 100644 index 0000000000..ab79c3627a Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz new file mode 100644 index 0000000000..4d8c820cfb Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz new file mode 100644 index 0000000000..e991387d19 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.source.asm.gz new file mode 100644 index 0000000000..4baed5ad6a Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz new file mode 100644 index 0000000000..9023dcd9ab Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz new file mode 100644 index 0000000000..2d262b3bd5 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz new file mode 100644 index 0000000000..740cd20c0c Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/12_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/12_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz new file mode 100644 index 0000000000..82d2571896 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/12_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.source.asm.gz new file mode 100644 index 0000000000..8867e3e14a Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz new file mode 100644 index 0000000000..68ee0ae98c Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz new file mode 100644 index 0000000000..f9de9b1a6c Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz new file mode 100644 index 0000000000..f89cca27a5 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/annotated/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/key_flow/aic_compete_first.source.asm b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/key_flow/aic_compete_first.source.asm new file mode 100644 index 0000000000..1c2195260c --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/key_flow/aic_compete_first.source.asm @@ -0,0 +1,2049 @@ +# schema=pa_final_linked_disassembly/v1 +# variant=compete-first +# final_elf=pa_scheduler_kernel.o +# final_elf_sha256=82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b +# final_text_address=0x0 +# final_text_size=547640 +# final_text_sha256=8fd6209b2f0f9d1fb48d4d8a16cf27649e26071153b908e1e0e39ca107d63589 +# symbol=pa_scheduler_lazy_sample_callback_orchestration_aic +# binding=LOCAL +# final_pc=0x2c0 +# size=161788 +# instruction_count=40447 +# encoded_word_count=40447 +# body_sha256=cab368a779b3870411a46bd169f6af9657c025967797ba7743b93fcfe27f45db +# decoder=$ASCEND_HOME_PATH/x86_64-linux/simulator/dav_3510/lib/libpem_davinci.so +# decoder_sha256=29835d2439d6dd464d34a212ad4bbd5c29af6a38465da09a6c273401d9a96dcb +# decoder_mode=scalar +# columns=final_pc function_relative_offset machine_word instruction +# annotation_schema=pa_source_annotated_disassembly/v1 +# annotation_rule=DWARF supplies only file:line; SOURCE rows are copied from local source files +# annotation_warning=comments have source context only and do not own an exact machine address +# annotation_instruction_slice=9241:9962 +# +# [DWARF] common/pa_trace.h:266 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +# 263 | const uint32_t bit = 1U << index; +# 264 | if ((active_mask & bit) == 0) continue; +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# > 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +0x0000000000009324 (+0x00009064) 0040113e CMP.U64.GT X1, X2 +0x0000000000009328 (+0x00009068) 40200003 JUMPC #3 +# [DWARF] common/pa_trace.h:267 +# > 267 | trace.atomic_counter_overflow = true; +0x000000000000932c (+0x0000906c) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x0000000000009330 (+0x00009070) 4000ffdf JUMP #65503 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009334 (+0x00009074) 07320001 MOV_XD_IMM X25, #1 +# [DWARF] common/pa_trace.h:98 +# 92 | default: +# 93 | return -1; +# 94 | } +# 95 | } +# 96 | +# 97 | PA_DEVICE AtomicSite TraceAtomicPollBatchSite(uint32_t index) { +# > 98 | switch (index) { +0x0000000000009338 (+0x00009078) 0281ca00 ZEROEXT.U32 X0, X28 +0x000000000000933c (+0x0000907c) 07020005 MOV_XD_IMM X1, #5 +0x0000000000009340 (+0x00009080) 02339080 NEG.S64 X25, X25 +0x0000000000009344 (+0x00009084) 004000be CMP.U64.GT X0, X1 +0x0000000000009348 (+0x00009088) 070c000f MOV_XD_IMM X6, #15 +0x000000000000934c (+0x0000908c) 40200002 JUMPC #2 +0x0000000000009350 (+0x00009090) 1c8d7000 LD_XD_XN_IMM.B32 X6, X23, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009354 (+0x00009094) 0804c002 ADD_IMM.S64 X2, X12, #2 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x0000000000009358 (+0x00009098) 1cc9e988 LD_XD_XN_IMM.B64 X4, X30, #2440 +0x000000000000935c (+0x0000909c) 00c2b78a AND.B64 X1, X11, X15 +0x0000000000009360 (+0x000090a0) 02c4020e SHL.B64 X2, #14 +0x0000000000009364 (+0x000090a4) 0006a381 ADD.S64 X3, X10, X7 +0x0000000000009368 (+0x000090a8) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:273 +# 272 | trace.core, trace.records, trace.capacity, +# > 273 | trace.poll_burst.start_cycle[index], end_cycle, +0x000000000000936c (+0x000090ac) 02018800 MOV_XD_XN.S64 X0, X24 +0x0000000000009370 (+0x000090b0) 00023101 ADD.S64 X1, X3, X2 +0x0000000000009374 (+0x000090b4) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x0000000000009378 (+0x000090b8) 02c00203 SHL.B64 X0, #3 +0x000000000000937c (+0x000090bc) 08241000 ADD_IMM.S64 X18, X1, #0 +0x0000000000009380 (+0x000090c0) 00012001 ADD.S64 X0, X18, X0 +0x0000000000009384 (+0x000090c4) 1cc60588 LD_XD_XN_IMM.B64 X3, X0, #1416 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x0000000000009388 (+0x000090c8) 02015800 MOV_XD_XN.S64 X0, X21 +0x000000000000938c (+0x000090cc) 0203a800 MOV_XD_XN.S64 X1, X26 +0x0000000000009390 (+0x000090d0) 0205b800 MOV_XD_XN.S64 X2, X27 +0x0000000000009394 (+0x000090d4) 070e79c7 MOV_XD_IMM X7, #31175 +0x0000000000009398 (+0x000090d8) 074f0000 MOVK X7, #0, #1 +0x000000000000939c (+0x000090dc) 078f0000 MOVK X7, #0, #2 +0x00000000000093a0 (+0x000090e0) 40427000 CALL X7, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000093a4 (+0x000090e4) 1ccfe990 LD_XD_XN_IMM.B64 X7, X30, #2448 +0x00000000000093a8 (+0x000090e8) 071c0000 MOV_XD_IMM X14, #0 +0x00000000000093ac (+0x000090ec) 1cd9e9a0 LD_XD_XN_IMM.B64 X12, X30, #2464 +0x00000000000093b0 (+0x000090f0) 071e7fff MOV_XD_IMM X15, #32767 +0x00000000000093b4 (+0x000090f4) 1cd7e9a8 LD_XD_XN_IMM.B64 X11, X30, #2472 +# [DWARF] common/pa_trace.h:276 +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x00000000000093b8 (+0x000090f8) 0000070e CMP.S64.EQ X0, X14 +0x00000000000093bc (+0x000090fc) 1cd5e9b0 LD_XD_XN_IMM.B64 X10, X30, #2480 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000093c0 (+0x00009100) 07220001 MOV_XD_IMM X17, #1 +0x00000000000093c4 (+0x00009104) 07120006 MOV_XD_IMM X9, #6 +# [DWARF] common/pa_trace.h:276 +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x00000000000093c8 (+0x00009108) 4020ffb6 JUMPC #65462 +# [DWARF] common/pa_trace.h:277 +# > 277 | if (trace.poll_batch_records == UINT64_MAX) { +0x00000000000093cc (+0x0000910c) 1cc12578 LD_XD_XN_IMM.B64 X0, X18, #1400 +0x00000000000093d0 (+0x00009110) 00000c9e CMP.S64.NE X0, X25 +0x00000000000093d4 (+0x00009114) 4020ffa9 JUMPC #65449 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000093d8 (+0x00009118) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x00000000000093dc (+0x0000911c) 00c0b78a AND.B64 X0, X11, X15 +0x00000000000093e0 (+0x00009120) 02c2020e SHL.B64 X1, #14 +0x00000000000093e4 (+0x00009124) 0004a381 ADD.S64 X2, X10, X7 +0x00000000000093e8 (+0x00009128) 00040084 MADD.S64 X2, X0, X1 +0x00000000000093ec (+0x0000912c) 00002081 ADD.S64 X0, X2, X1 +0x00000000000093f0 (+0x00009130) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:278 +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# 276 | if (written) { +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# > 278 | trace.atomic_counter_overflow = true; +0x00000000000093f4 (+0x00009134) 08000000 ADD_IMM.S64 X0, X0, #0 +0x00000000000093f8 (+0x00009138) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x00000000000093fc (+0x0000913c) 4000ffa9 JUMP #65449 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009400 (+0x00009140) 1cd1e978 LD_XD_XN_IMM.B64 X8, X30, #2424 +0x0000000000009404 (+0x00009144) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x0000000000009408 (+0x00009148) 00c0b78a AND.B64 X0, X11, X15 +0x000000000000940c (+0x0000914c) 02c2020e SHL.B64 X1, #14 +0x0000000000009410 (+0x00009150) 0004a381 ADD.S64 X2, X10, X7 +0x0000000000009414 (+0x00009154) 00040084 MADD.S64 X2, X0, X1 +0x0000000000009418 (+0x00009158) 00002081 ADD.S64 X0, X2, X1 +0x000000000000941c (+0x0000915c) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:285 +# 279 | } else { +# 280 | ++trace.poll_batch_records; +# 281 | } +# 282 | } +# 283 | trace.poll_burst.call_count[index] = 0; +# 284 | } +# > 285 | trace.poll_burst.active_mask = 0; +0x0000000000009420 (+0x00009160) 08000000 ADD_IMM.S64 X0, X0, #0 +0x0000000000009424 (+0x00009164) 0f960a00 STI_XN_IMM.B32 X0, #1488 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000009428 (+0x00009168) 08353548 ADD_IMM.S64 X26, X19, #1352 +0x000000000000942c (+0x0000916c) 1cb93558 LD_XD_XN_IMM.B32 X28, X19, #1368 +0x0000000000009430 (+0x00009170) 0cf5ad80 LDP_XI_XJ_XN.B64 X26, X27, X26, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009434 (+0x00009174) 070affff MOV_XD_IMM X5, #65535 +0x0000000000009438 (+0x00009178) 1cc5e8f8 LD_XD_XN_IMM.B64 X2, X30, #2296 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x000000000000943c (+0x0000917c) 0001a70e CMP.S64.EQ X26, X14 +0x0000000000009440 (+0x00009180) 40200026 JUMPC #38 +0x0000000000009444 (+0x00009184) 0001b70e CMP.S64.EQ X27, X14 +0x0000000000009448 (+0x00009188) 40200024 JUMPC #36 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000944c (+0x0000918c) 0283ca00 ZEROEXT.U32 X1, X28 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000009450 (+0x00009190) 0000170e CMP.S64.EQ X1, X14 +0x0000000000009454 (+0x00009194) 40200021 JUMPC #33 +# [DWARF] common/pa_trace.h:547 +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# > 547 | const uint32_t slot = core.count; +0x0000000000009458 (+0x00009198) 1c81a000 LD_XD_XN_IMM.B32 X0, X26, #0 +# [DWARF] common/pa_trace.h:548 +# > 548 | if (slot >= trace.capacity) { +0x000000000000945c (+0x0000919c) 004000ae CMP.U64.LT X0, X1 +0x0000000000009460 (+0x000091a0) 40200002 JUMPC #2 +0x0000000000009464 (+0x000091a4) 40000030 JUMP #48 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009468 (+0x000091a8) 0806c002 ADD_IMM.S64 X3, X12, #2 +# [DWARF] common/pa_trace.h:553 +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# > 553 | record.start_cycle = start_cycle; +0x000000000000946c (+0x000091ac) 1cdbe928 LD_XD_XN_IMM.B64 X13, X30, #2344 +0x0000000000009470 (+0x000091b0) 00c4b78a AND.B64 X2, X11, X15 +# [DWARF] common/pa_trace.h:554 +# > 554 | record.end_cycle = end_cycle; +0x0000000000009474 (+0x000091b4) 1ccde988 LD_XD_XN_IMM.B64 X6, X30, #2440 +0x0000000000009478 (+0x000091b8) 02c6020e SHL.B64 X3, #14 +0x000000000000947c (+0x000091bc) 0008a381 ADD.S64 X4, X10, X7 +0x0000000000009480 (+0x000091c0) 00082184 MADD.S64 X4, X2, X3 +# [DWARF] common/pa_trace.h:552 +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x0000000000009484 (+0x000091c4) 02020800 MOV_XD_XN.S64 X1, X0 +0x0000000000009488 (+0x000091c8) 00044181 ADD.S64 X2, X4, X3 +# [DWARF] common/pa_trace.h:555 +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x000000000000948c (+0x000091cc) 1cc7e8f8 LD_XD_XN_IMM.B64 X3, X30, #2296 +# [DWARF] common/pa_trace.h:552 +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x0000000000009490 (+0x000091d0) 02c20206 SHL.B64 X1, #6 +0x0000000000009494 (+0x000091d4) 0003b081 ADD.S64 X1, X27, X1 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009498 (+0x000091d8) 08842680 SUB_IMM.S64 X2, X2, #1664 +# [DWARF] common/pa_trace.h:558 +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# > 558 | record.lane = trace.lane; +0x000000000000949c (+0x000091dc) 08042000 ADD_IMM.S64 X2, X2, #0 +# [DWARF] common/pa_trace.h:565 +# 559 | record.block_id = trace.block_id; +# 560 | record.core_idx = trace.core_idx; +# 561 | record.flags = flags; +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x00000000000094a0 (+0x000091e0) 08000001 ADD_IMM.S64 X0, X0, #1 +# [DWARF] common/pa_trace.h:553 +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# > 553 | record.start_cycle = start_cycle; +0x00000000000094a4 (+0x000091e4) 09da1301 STP_XI_XJ_XN.B64 X13, X6, X1, #0 +# [DWARF] common/pa_trace.h:555 +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x00000000000094a8 (+0x000091e8) 03861010 ST_XD_XN_IMM.B32 X3, X1, #16 +# [DWARF] common/pa_trace.h:556 +# > 556 | record.function_id = function_id; +0x00000000000094ac (+0x000091ec) 0706ffff MOV_XD_IMM X3, #65535 +0x00000000000094b0 (+0x000091f0) 0747ffff MOVK X3, #65535, #1 +0x00000000000094b4 (+0x000091f4) 098614a9 STP_XI_XJ_XN.B32 X3, X9, X1, #20 +# [DWARF] common/pa_trace.h:559 +# 557 | record.phase = static_cast(trace_phase); +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x00000000000094b8 (+0x000091f8) 08062564 ADD_IMM.S64 X3, X2, #1380 +# [DWARF] common/pa_trace.h:558 +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# > 558 | record.lane = trace.lane; +0x00000000000094bc (+0x000091fc) 1c882560 LD_XD_XN_IMM.B32 X4, X2, #1376 +# [DWARF] common/pa_trace.h:559 +# > 559 | record.block_id = trace.block_id; +0x00000000000094c0 (+0x00009200) 0c863100 LDP_XI_XJ_XN.B32 X3, X2, X3, #0 +# [DWARF] common/pa_trace.h:558 +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# > 558 | record.lane = trace.lane; +0x00000000000094c4 (+0x00009204) 098811b9 STP_XI_XJ_XN.B32 X4, X3, X1, #28 +# [DWARF] common/pa_trace.h:560 +# 559 | record.block_id = trace.block_id; +# > 560 | record.core_idx = trace.core_idx; +0x00000000000094c8 (+0x00009208) 03841024 ST_XD_XN_IMM.B32 X2, X1, #36 +0x00000000000094cc (+0x0000920c) 1cc5e8f8 LD_XD_XN_IMM.B64 X2, X30, #2296 +# [DWARF] common/pa_trace.h:561 +# > 561 | record.flags = flags; +0x00000000000094d0 (+0x00009210) 03dc1028 ST_XD_XN_IMM.B64 X14, X1, #40 +# [DWARF] common/pa_trace.h:565 +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x00000000000094d4 (+0x00009214) 0381a000 ST_XD_XN_IMM.B32 X0, X26, #0 +# [DWARF] common/pa_scheduler_core.h:472 +# 466 | PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, TaskKind kind, +# 467 | LocalStats &stats +# 468 | ) { +# 469 | // Claim 在四个 shard 的单调 cursor 上执行 atomicMax：同一 task 只有观察到旧值更小的竞争者获胜。 +# 470 | // Alloc 由全部 96 个 worker 竞争；QK/PV 仅 32 个 AIC，SF/UP 仅 64 个 AIV 进入真正的 atomicMax。 +# 471 | ClaimOutcome outcome{false, false, 0, -1}; +# > 472 | if (task_id >= kTaskCellCapacity) { +0x00000000000094d8 (+0x00009218) 004022be CMP.U64.GT X2, X5 +0x00000000000094dc (+0x0000921c) 40200002 JUMPC #2 +0x00000000000094e0 (+0x00009220) 40000016 JUMP #22 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x00000000000094e4 (+0x00009224) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x00000000000094e8 (+0x00009228) 00c0b78a AND.B64 X0, X11, X15 +0x00000000000094ec (+0x0000922c) 02c2020e SHL.B64 X1, #14 +0x00000000000094f0 (+0x00009230) 0004a381 ADD.S64 X2, X10, X7 +0x00000000000094f4 (+0x00009234) 00040084 MADD.S64 X2, X0, X1 +0x00000000000094f8 (+0x00009238) 07320000 MOV_XD_IMM X25, #0 +0x00000000000094fc (+0x0000923c) 00002081 ADD.S64 X0, X2, X1 +0x0000000000009500 (+0x00009240) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_scheduler_core.h:1144 +# 1138 | efdrain_begin, efdrain_end +# 1139 | ); +# 1140 | +# 1141 | const uint64_t claim_begin = efdrain_end; +# 1142 | BeginSubmitPmuPhase(pmu_context); +# 1143 | const ClaimOutcome claim = Claim(state, worker, task_id, Kind, stats); +# > 1144 | context.won = claim.won; +0x0000000000009504 (+0x00009244) 08000000 ADD_IMM.S64 X0, X0, #0 +0x0000000000009508 (+0x00009248) 0f060300 STI_XN_IMM.B8 X0, #408 +# [DWARF] common/pa_scheduler_core.h:1145 +# > 1145 | context.kernel_id = claim.function_id; +0x000000000000950c (+0x0000924c) 0f860282 STI_XN_IMM.B32 X0, #404 +0x0000000000009510 (+0x00009250) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000009514 (+0x00009254) 0700007c MOV_XD_IMM X0, #124 +0x0000000000009518 (+0x00009258) 07410000 MOVK X0, #0, #1 +0x000000000000951c (+0x0000925c) 07810000 MOVK X0, #0, #2 +0x0000000000009520 (+0x00009260) 40020000 JUMP X0, #0 +# [DWARF] common/pa_trace.h:549 +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# > 549 | core.dropped = core.dropped + 1; +0x0000000000009524 (+0x00009264) 1c81a004 LD_XD_XN_IMM.B32 X0, X26, #4 +0x0000000000009528 (+0x00009268) 08000001 ADD_IMM.S64 X0, X0, #1 +0x000000000000952c (+0x0000926c) 0381a004 ST_XD_XN_IMM.B32 X0, X26, #4 +# [DWARF] common/pa_scheduler_core.h:472 +# 466 | PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, TaskKind kind, +# 467 | LocalStats &stats +# 468 | ) { +# 469 | // Claim 在四个 shard 的单调 cursor 上执行 atomicMax：同一 task 只有观察到旧值更小的竞争者获胜。 +# 470 | // Alloc 由全部 96 个 worker 竞争；QK/PV 仅 32 个 AIC，SF/UP 仅 64 个 AIV 进入真正的 atomicMax。 +# 471 | ClaimOutcome outcome{false, false, 0, -1}; +# > 472 | if (task_id >= kTaskCellCapacity) { +0x0000000000009530 (+0x00009270) 004022be CMP.U64.GT X2, X5 +0x0000000000009534 (+0x00009274) 4020ffec JUMPC #65516 +# [DWARF] common/pa_scheduler_core.h:477 +# 473 | return outcome; +# 474 | } +# 475 | PA_GM AtomicLine *cursor = nullptr; +# 476 | if (kind == TaskKind::Alloc) { +# > 477 | cursor = &state->alloc_cursor[task_id % kCursorShards]; +0x0000000000009538 (+0x00009278) 1cc3e880 LD_XD_XN_IMM.B64 X1, X30, #2176 +0x000000000000953c (+0x0000927c) 07000003 MOV_XD_IMM X0, #3 +0x0000000000009540 (+0x00009280) 00c0200a AND.B64 X0, X2, X0 +0x0000000000009544 (+0x00009284) 02c00206 SHL.B64 X0, #6 +# [DWARF] common/pa_trace.h:257 +# 251 | TraceContext &trace, uint64_t end_cycle +# 252 | ) { +# 253 | #if PA_BUILD_SUBMIT_PMU +# 254 | (void)trace; +# 255 | (void)end_cycle; +# 256 | #else +# > 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +0x0000000000009548 (+0x00009288) 0000871e CMP.S64.NE X8, X14 +# [DWARF] common/pa_scheduler_core.h:477 +# 471 | ClaimOutcome outcome{false, false, 0, -1}; +# 472 | if (task_id >= kTaskCellCapacity) { +# 473 | return outcome; +# 474 | } +# 475 | PA_GM AtomicLine *cursor = nullptr; +# 476 | if (kind == TaskKind::Alloc) { +# > 477 | cursor = &state->alloc_cursor[task_id % kCursorShards]; +0x000000000000954c (+0x0000928c) 00001001 ADD.S64 X0, X1, X0 +# [DWARF] common/pa_trace.h:480 +# 474 | (void)result; +# 475 | (void)task_id; +# 476 | (void)site; +# 477 | (void)result_used; +# 478 | return Ops::FetchMax(address, value, retries); +# 479 | #else +# > 480 | if (!trace.atomics_enabled) return Ops::FetchMax(address, value, retries); +0x0000000000009550 (+0x00009290) 40200006 JUMPC #6 +0x0000000000009554 (+0x00009294) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000009558 (+0x00009298) 07000015 MOV_XD_IMM X0, #21 +0x000000000000955c (+0x0000929c) 07410000 MOVK X0, #0, #1 +0x0000000000009560 (+0x000092a0) 07810000 MOVK X0, #0, #2 +0x0000000000009564 (+0x000092a4) 40020000 JUMP X0, #0 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000009568 (+0x000092a8) 02828880 MOV_XD_SPR.F32 X1, SYS_CNT +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:86 +# 80 | return __builtin_cce_atom_max_G_##SUFFIX(base, inc, (uint32_t)L2Cache); \ +# 81 | } +# 82 | +# 83 | __CCE__ATOM_G_BUILTIN(uint32_t, u32) +# 84 | __CCE__ATOM_G_BUILTIN(int32_t, s32) +# 85 | __CCE__ATOM_G_BUILTIN(uint64_t, u64) +# > 86 | __CCE__ATOM_G_BUILTIN(int64_t, s64) +0x000000000000956c (+0x000092ac) 51c00040 ATOM XN, XM, XD, MAX +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:0 +# [SOURCE unavailable] +0x0000000000009570 (+0x000092b0) 0806c002 ADD_IMM.S64 X3, X12, #2 +0x0000000000009574 (+0x000092b4) 00c4b78a AND.B64 X2, X11, X15 +0x0000000000009578 (+0x000092b8) 02c6020e SHL.B64 X3, #14 +0x000000000000957c (+0x000092bc) 0008a381 ADD.S64 X4, X10, X7 +0x0000000000009580 (+0x000092c0) 00082184 MADD.S64 X4, X2, X3 +0x0000000000009584 (+0x000092c4) 00044181 ADD.S64 X2, X4, X3 +0x0000000000009588 (+0x000092c8) 08842680 SUB_IMM.S64 X2, X2, #1664 +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x000000000000958c (+0x000092cc) 08062000 ADD_IMM.S64 X3, X2, #0 +# [DWARF] ccec/ccec_ops.h:239 +# 233 | static_assert(sizeof(T) == 4 || sizeof(T) == 8, "atomic dependency expects a scalar result"); +# 234 | uint64_t cycle = 0; +# 235 | // 同一个 inline asm 块先真正消费 atomic 返回寄存器，再读取 +# 236 | // SYS_CNT；编译器不能把 t1 拆到依赖 MOV 之前。AIC/AIV 对该序列 +# 237 | // 生成相同指令字节，且不增加 DSB/ISB/GM 访存。该边界仍只表示 +# 238 | // 返回值已可被本核 scalar 消费，不表示跨核全局可见。 +# > 239 | asm volatile( +0x0000000000009590 (+0x000092d0) 020a0800 MOV_XD_XN.S64 X5, X0 +0x0000000000009594 (+0x000092d4) 020a5800 MOV_XD_XN.S64 X5, X5 +0x0000000000009598 (+0x000092d8) 02848880 MOV_XD_SPR.F32 X2, SYS_CNT +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x000000000000959c (+0x000092dc) 1cc834a0 LD_XD_XN_IMM.B64 X4, X3, #1184 +0x00000000000095a0 (+0x000092e0) 0000491e CMP.S64.NE X4, X18 +0x00000000000095a4 (+0x000092e4) 40200002 JUMPC #2 +0x00000000000095a8 (+0x000092e8) 40000007 JUMP #7 +# [DWARF] common/pa_trace.h:240 +# 237 | trace.atomic_counter_overflow = true; +# 238 | return; +# 239 | } +# > 240 | ++result.atomic_trace_calls; +0x00000000000095ac (+0x000092ec) 08084001 ADD_IMM.S64 X4, X4, #1 +0x00000000000095b0 (+0x000092f0) 03c834a0 ST_XD_XN_IMM.B64 X4, X3, #1184 +0x00000000000095b4 (+0x000092f4) 40000005 JUMP #5 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000095b8 (+0x000092f8) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:86 +# 80 | return __builtin_cce_atom_max_G_##SUFFIX(base, inc, (uint32_t)L2Cache); \ +# 81 | } +# 82 | +# 83 | __CCE__ATOM_G_BUILTIN(uint32_t, u32) +# 84 | __CCE__ATOM_G_BUILTIN(int32_t, s32) +# 85 | __CCE__ATOM_G_BUILTIN(uint64_t, u64) +# > 86 | __CCE__ATOM_G_BUILTIN(int64_t, s64) +0x00000000000095bc (+0x000092fc) 51c00040 ATOM XN, XM, XD, MAX +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:0 +# [SOURCE unavailable] +0x00000000000095c0 (+0x00009300) 40000039 JUMP #57 +# [DWARF] common/pa_trace.h:237 +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# 236 | if (result.atomic_trace_calls == UINT64_MAX) { +# > 237 | trace.atomic_counter_overflow = true; +0x00000000000095c4 (+0x00009304) 0f163001 STI_XN_IMM.B8 X3, #1408 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000095c8 (+0x00009308) 0808c002 ADD_IMM.S64 X4, X12, #2 +0x00000000000095cc (+0x0000930c) 00c6b78a AND.B64 X3, X11, X15 +0x00000000000095d0 (+0x00009310) 02c8020e SHL.B64 X4, #14 +0x00000000000095d4 (+0x00009314) 000aa381 ADD.S64 X5, X10, X7 +0x00000000000095d8 (+0x00009318) 000a3204 MADD.S64 X5, X3, X4 +0x00000000000095dc (+0x0000931c) 00065201 ADD.S64 X3, X5, X4 +0x00000000000095e0 (+0x00009320) 08863680 SUB_IMM.S64 X3, X3, #1664 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x00000000000095e4 (+0x00009324) 08063000 ADD_IMM.S64 X3, X3, #0 +0x00000000000095e8 (+0x00009328) 08343548 ADD_IMM.S64 X26, X3, #1352 +0x00000000000095ec (+0x0000932c) 1cb83558 LD_XD_XN_IMM.B32 X28, X3, #1368 +0x00000000000095f0 (+0x00009330) 0cf5ad80 LDP_XI_XJ_XN.B64 X26, X27, X26, #0 +0x00000000000095f4 (+0x00009334) 0001a70e CMP.S64.EQ X26, X14 +0x00000000000095f8 (+0x00009338) 4020002b JUMPC #43 +0x00000000000095fc (+0x0000933c) 0001b70e CMP.S64.EQ X27, X14 +0x0000000000009600 (+0x00009340) 40200029 JUMPC #41 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009604 (+0x00009344) 0289ca00 ZEROEXT.U32 X4, X28 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000009608 (+0x00009348) 0000470e CMP.S64.EQ X4, X14 +0x000000000000960c (+0x0000934c) 40200026 JUMPC #38 +# [DWARF] common/pa_trace.h:547 +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# > 547 | const uint32_t slot = core.count; +0x0000000000009610 (+0x00009350) 1c87a000 LD_XD_XN_IMM.B32 X3, X26, #0 +# [DWARF] common/pa_trace.h:548 +# > 548 | if (slot >= trace.capacity) { +0x0000000000009614 (+0x00009354) 0040322e CMP.U64.LT X3, X4 +0x0000000000009618 (+0x00009358) 40200002 JUMPC #2 +0x000000000000961c (+0x0000935c) 4000001f JUMP #31 +# [DWARF] common/pa_trace.h:552 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x0000000000009620 (+0x00009360) 02083800 MOV_XD_XN.S64 X4, X3 +0x0000000000009624 (+0x00009364) 02c80206 SHL.B64 X4, #6 +0x0000000000009628 (+0x00009368) 0009b201 ADD.S64 X4, X27, X4 +# [DWARF] common/pa_trace.h:553 +# > 553 | record.start_cycle = start_cycle; +0x000000000000962c (+0x0000936c) 09c24101 STP_XI_XJ_XN.B64 X1, X2, X4, #0 +# [DWARF] common/pa_trace.h:555 +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x0000000000009630 (+0x00009370) 1cc5e8f8 LD_XD_XN_IMM.B64 X2, X30, #2296 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009634 (+0x00009374) 080cc002 ADD_IMM.S64 X6, X12, #2 +0x0000000000009638 (+0x00009378) 00cab78a AND.B64 X5, X11, X15 +0x000000000000963c (+0x0000937c) 02cc020e SHL.B64 X6, #14 +0x0000000000009640 (+0x00009380) 0002a381 ADD.S64 X1, X10, X7 +0x0000000000009644 (+0x00009384) 00025304 MADD.S64 X1, X5, X6 +# [DWARF] common/pa_trace.h:557 +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x0000000000009648 (+0x00009388) 070a000e MOV_XD_IMM X5, #14 +0x000000000000964c (+0x0000938c) 00021301 ADD.S64 X1, X1, X6 +0x0000000000009650 (+0x00009390) 08821680 SUB_IMM.S64 X1, X1, #1664 +# [DWARF] common/pa_trace.h:558 +# > 558 | record.lane = trace.lane; +0x0000000000009654 (+0x00009394) 08021000 ADD_IMM.S64 X1, X1, #0 +# [DWARF] common/pa_trace.h:555 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x0000000000009658 (+0x00009398) 03844010 ST_XD_XN_IMM.B32 X2, X4, #16 +# [DWARF] common/pa_trace.h:556 +# > 556 | record.function_id = function_id; +0x000000000000965c (+0x0000939c) 0704ffff MOV_XD_IMM X2, #65535 +0x0000000000009660 (+0x000093a0) 0745ffff MOVK X2, #65535, #1 +0x0000000000009664 (+0x000093a4) 03844014 ST_XD_XN_IMM.B32 X2, X4, #20 +# [DWARF] common/pa_trace.h:558 +# 557 | record.phase = static_cast(trace_phase); +# > 558 | record.lane = trace.lane; +0x0000000000009668 (+0x000093a8) 1c841560 LD_XD_XN_IMM.B32 X2, X1, #1376 +# [DWARF] common/pa_trace.h:557 +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x000000000000966c (+0x000093ac) 098a4131 STP_XI_XJ_XN.B32 X5, X2, X4, #24 +# [DWARF] common/pa_trace.h:559 +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x0000000000009670 (+0x000093b0) 08041564 ADD_IMM.S64 X2, X1, #1380 +0x0000000000009674 (+0x000093b4) 0c842080 LDP_XI_XJ_XN.B32 X2, X1, X2, #0 +0x0000000000009678 (+0x000093b8) 080a4020 ADD_IMM.S64 X5, X4, #32 +0x000000000000967c (+0x000093bc) 09845081 STP_XI_XJ_XN.B32 X2, X1, X5, #0 +# [DWARF] common/pa_trace.h:561 +# 560 | record.core_idx = trace.core_idx; +# > 561 | record.flags = flags; +0x0000000000009680 (+0x000093c0) 07020053 MOV_XD_IMM X1, #83 +0x0000000000009684 (+0x000093c4) 07830004 MOVK X1, #4, #2 +0x0000000000009688 (+0x000093c8) 03c24028 ST_XD_XN_IMM.B64 X1, X4, #40 +# [DWARF] common/pa_trace.h:565 +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x000000000000968c (+0x000093cc) 08023001 ADD_IMM.S64 X1, X3, #1 +0x0000000000009690 (+0x000093d0) 0383a000 ST_XD_XN_IMM.B32 X1, X26, #0 +0x0000000000009694 (+0x000093d4) 40000004 JUMP #4 +# [DWARF] common/pa_trace.h:549 +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# > 549 | core.dropped = core.dropped + 1; +0x0000000000009698 (+0x000093d8) 1c83a004 LD_XD_XN_IMM.B32 X1, X26, #4 +0x000000000000969c (+0x000093dc) 08021001 ADD_IMM.S64 X1, X1, #1 +0x00000000000096a0 (+0x000093e0) 0383a004 ST_XD_XN_IMM.B32 X1, X26, #4 +# [DWARF] common/pa_scheduler_core.h:514 +# 508 | outcome.attempted = true; +# 509 | // atomicMax 返回写入前的 cursor：old=task_id 则必须 Replay。 +# 510 | const int64_t old = TraceAtomicFetchMax( +# 511 | stats.trace, stats.result, static_cast(task_id), AtomicSite::ClaimMax, +# 512 | &cursor->value, static_cast(task_id), outcome.retries +# 513 | ); +# > 514 | outcome.won = old < static_cast(task_id); +0x00000000000096a4 (+0x000093e4) 1cc9e8f8 LD_XD_XN_IMM.B64 X4, X30, #2296 +0x00000000000096a8 (+0x000093e8) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x00000000000096ac (+0x000093ec) 00c2b78a AND.B64 X1, X11, X15 +0x00000000000096b0 (+0x000093f0) 02c4020e SHL.B64 X2, #14 +0x00000000000096b4 (+0x000093f4) 0006a381 ADD.S64 X3, X10, X7 +0x00000000000096b8 (+0x000093f8) 00061104 MADD.S64 X3, X1, X2 +0x00000000000096bc (+0x000093fc) 07320000 MOV_XD_IMM X25, #0 +0x00000000000096c0 (+0x00009400) 0000022e CMP.S64.LT X0, X4 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x00000000000096c4 (+0x00009404) 00003101 ADD.S64 X0, X3, X2 +0x00000000000096c8 (+0x00009408) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_scheduler_core.h:1144 +# 1138 | efdrain_begin, efdrain_end +# 1139 | ); +# 1140 | +# 1141 | const uint64_t claim_begin = efdrain_end; +# 1142 | BeginSubmitPmuPhase(pmu_context); +# 1143 | const ClaimOutcome claim = Claim(state, worker, task_id, Kind, stats); +# > 1144 | context.won = claim.won; +0x00000000000096cc (+0x0000940c) 08000000 ADD_IMM.S64 X0, X0, #0 +# [DWARF] common/pa_scheduler_core.h:514 +# 508 | outcome.attempted = true; +# 509 | // atomicMax 返回写入前的 cursor：old=task_id 则必须 Replay。 +# 510 | const int64_t old = TraceAtomicFetchMax( +# 511 | stats.trace, stats.result, static_cast(task_id), AtomicSite::ClaimMax, +# 512 | &cursor->value, static_cast(task_id), outcome.retries +# 513 | ); +# > 514 | outcome.won = old < static_cast(task_id); +0x00000000000096d0 (+0x00009410) 0202b880 MOV_XD_SPR.S64 X1, CONDITION_FLAG +# [DWARF] common/pa_scheduler_core.h:1144 +# 1138 | efdrain_begin, efdrain_end +# 1139 | ); +# 1140 | +# 1141 | const uint64_t claim_begin = efdrain_end; +# 1142 | BeginSubmitPmuPhase(pmu_context); +# 1143 | const ClaimOutcome claim = Claim(state, worker, task_id, Kind, stats); +# > 1144 | context.won = claim.won; +0x00000000000096d4 (+0x00009414) 03020198 ST_XD_XN_IMM.B8 X1, X0, #408 +# [DWARF] common/pa_scheduler_core.h:1145 +# > 1145 | context.kernel_id = claim.function_id; +0x00000000000096d8 (+0x00009418) 0f860282 STI_XN_IMM.B32 X0, #404 +# [DWARF] common/pa_scheduler_core.h:520 +# 514 | outcome.won = old < static_cast(task_id); +# 515 | if (!outcome.won) outcome.function_id = -1; +# 516 | return outcome; +# 517 | } +# 518 | +# 519 | PA_DEVICE void RecordClaimOutcome(LocalStats &stats, TaskKind kind, const ClaimOutcome &outcome) { +# > 520 | if (outcome.attempted) ++stats.result.claim_attempts; +0x00000000000096dc (+0x0000941c) 1cc201e8 LD_XD_XN_IMM.B64 X1, X0, #488 +0x00000000000096e0 (+0x00009420) 08021001 ADD_IMM.S64 X1, X1, #1 +0x00000000000096e4 (+0x00009424) 03c201e8 ST_XD_XN_IMM.B64 X1, X0, #488 +# [DWARF] common/pa_scheduler_core.h:522 +# 521 | stats.result.cas_retries += outcome.retries; +# > 522 | if (outcome.won) { +0x00000000000096e8 (+0x00009428) 40200002 JUMPC #2 +0x00000000000096ec (+0x0000942c) 4000000a JUMP #10 +# [DWARF] common/pa_scheduler_core.h:523 +# > 523 | ++stats.result.claim_wins; +0x00000000000096f0 (+0x00009430) 1cc201f0 LD_XD_XN_IMM.B64 X1, X0, #496 +0x00000000000096f4 (+0x00009434) 07320001 MOV_XD_IMM X25, #1 +# [DWARF] common/pa_scheduler_core.h:524 +# > 524 | ++stats.result.wins[KindIndex(kind)]; +0x00000000000096f8 (+0x00009438) 1cc40220 LD_XD_XN_IMM.B64 X2, X0, #544 +# [DWARF] common/pa_scheduler_core.h:523 +# 517 | } +# 518 | +# 519 | PA_DEVICE void RecordClaimOutcome(LocalStats &stats, TaskKind kind, const ClaimOutcome &outcome) { +# 520 | if (outcome.attempted) ++stats.result.claim_attempts; +# 521 | stats.result.cas_retries += outcome.retries; +# 522 | if (outcome.won) { +# > 523 | ++stats.result.claim_wins; +0x00000000000096fc (+0x0000943c) 08021001 ADD_IMM.S64 X1, X1, #1 +0x0000000000009700 (+0x00009440) 03c201f0 ST_XD_XN_IMM.B64 X1, X0, #496 +# [DWARF] common/pa_scheduler_core.h:524 +# > 524 | ++stats.result.wins[KindIndex(kind)]; +0x0000000000009704 (+0x00009444) 08022001 ADD_IMM.S64 X1, X2, #1 +0x0000000000009708 (+0x00009448) 03c20220 ST_XD_XN_IMM.B64 X1, X0, #544 +0x000000000000970c (+0x0000944c) 40000002 JUMP #2 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x0000000000009710 (+0x00009450) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000009714 (+0x00009454) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x0000000000009718 (+0x00009458) 00c0b78a AND.B64 X0, X11, X15 +0x000000000000971c (+0x0000945c) 02c2020e SHL.B64 X1, #14 +0x0000000000009720 (+0x00009460) 0004a381 ADD.S64 X2, X10, X7 +0x0000000000009724 (+0x00009464) 00040084 MADD.S64 X2, X0, X1 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000009728 (+0x00009468) 02aa8880 MOV_XD_SPR.F32 X21, SYS_CNT +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x000000000000972c (+0x0000946c) 00002081 ADD.S64 X0, X2, X1 +0x0000000000009730 (+0x00009470) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:257 +# 251 | TraceContext &trace, uint64_t end_cycle +# 252 | ) { +# 253 | #if PA_BUILD_SUBMIT_PMU +# 254 | (void)trace; +# 255 | (void)end_cycle; +# 256 | #else +# > 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +0x0000000000009734 (+0x00009474) 08000000 ADD_IMM.S64 X0, X0, #0 +0x0000000000009738 (+0x00009478) 1c02055c LD_XD_XN_IMM.B8 X1, X0, #1372 +0x000000000000973c (+0x0000947c) 0000170e CMP.S64.EQ X1, X14 +0x0000000000009740 (+0x00009480) 40200077 JUMPC #119 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009744 (+0x00009484) 1ca605d0 LD_XD_XN_IMM.B32 X19, X0, #1488 +0x0000000000009748 (+0x00009488) 02813a00 ZEROEXT.U32 X0, X19 +# [DWARF] common/pa_trace.h:257 +# 251 | TraceContext &trace, uint64_t end_cycle +# 252 | ) { +# 253 | #if PA_BUILD_SUBMIT_PMU +# 254 | (void)trace; +# 255 | (void)end_cycle; +# 256 | #else +# > 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +0x000000000000974c (+0x0000948c) 0000070e CMP.S64.EQ X0, X14 +0x0000000000009750 (+0x00009490) 40200073 JUMPC #115 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009754 (+0x00009494) 072e0000 MOV_XD_IMM X23, #0 +0x0000000000009758 (+0x00009498) 07300000 MOV_XD_IMM X24, #0 +0x000000000000975c (+0x0000949c) 0728c43c MOV_XD_IMM X20, #50236 +0x0000000000009760 (+0x000094a0) 07690007 MOVK X20, #7, #1 +0x0000000000009764 (+0x000094a4) 07a90000 MOVK X20, #0, #2 +0x0000000000009768 (+0x000094a8) 07e90000 MOVK X20, #0, #3 +0x000000000000976c (+0x000094ac) 02020880 MOV_XD_SPR.S64 X1, PC +0x0000000000009770 (+0x000094b0) 00294081 ADD.S64 X20, X20, X1 +0x0000000000009774 (+0x000094b4) 40000012 JUMP #18 +0x0000000000009778 (+0x000094b8) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x000000000000977c (+0x000094bc) 00c2b78a AND.B64 X1, X11, X15 +0x0000000000009780 (+0x000094c0) 02c4020e SHL.B64 X2, #14 +0x0000000000009784 (+0x000094c4) 0006a381 ADD.S64 X3, X10, X7 +0x0000000000009788 (+0x000094c8) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:280 +# 274 | call_count, static_cast(site) +# 275 | ); +# 276 | if (written) { +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# 278 | trace.atomic_counter_overflow = true; +# 279 | } else { +# > 280 | ++trace.poll_batch_records; +0x000000000000978c (+0x000094cc) 08000001 ADD_IMM.S64 X0, X0, #1 +0x0000000000009790 (+0x000094d0) 00023101 ADD.S64 X1, X3, X2 +0x0000000000009794 (+0x000094d4) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x0000000000009798 (+0x000094d8) 08021000 ADD_IMM.S64 X1, X1, #0 +0x000000000000979c (+0x000094dc) 03c01578 ST_XD_XN_IMM.B64 X0, X1, #1400 +# [DWARF] common/pa_trace.h:283 +# 281 | } +# 282 | } +# > 283 | trace.poll_burst.call_count[index] = 0; +0x00000000000097a0 (+0x000094e0) 0f976700 STI_XN_IMM.B32 X22, #1464 +# [DWARF] common/pa_trace.h:262 +# 256 | #else +# 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +# 258 | const uint32_t active_mask = trace.poll_burst.active_mask; +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# > 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +0x00000000000097a4 (+0x000094e4) 08318001 ADD_IMM.S64 X24, X24, #1 +0x00000000000097a8 (+0x000094e8) 08294004 ADD_IMM.S64 X20, X20, #4 +0x00000000000097ac (+0x000094ec) 0001849e CMP.S64.NE X24, X9 +0x00000000000097b0 (+0x000094f0) 082f7001 ADD_IMM.S64 X23, X23, #1 +0x00000000000097b4 (+0x000094f4) 40200002 JUMPC #2 +0x00000000000097b8 (+0x000094f8) 40000050 JUMP #80 +# [DWARF] common/pa_trace.h:263 +# > 263 | const uint32_t bit = 1U << index; +0x00000000000097bc (+0x000094fc) 02817a00 ZEROEXT.U32 X0, X23 +# [DWARF] common/pa_trace.h:264 +# > 264 | if ((active_mask & bit) == 0) continue; +0x00000000000097c0 (+0x00009500) 02833a00 ZEROEXT.U32 X1, X19 +0x00000000000097c4 (+0x00009504) 024202c0 SHR.U64 X1, X0, #0 +0x00000000000097c8 (+0x00009508) 00c0188a AND.B64 X0, X1, X17 +0x00000000000097cc (+0x0000950c) 0000070e CMP.S64.EQ X0, X14 +0x00000000000097d0 (+0x00009510) 4020fff5 JUMPC #65525 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000097d4 (+0x00009514) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x00000000000097d8 (+0x00009518) 00c0b78a AND.B64 X0, X11, X15 +0x00000000000097dc (+0x0000951c) 02c4020e SHL.B64 X2, #14 +0x00000000000097e0 (+0x00009520) 0006a381 ADD.S64 X3, X10, X7 +0x00000000000097e4 (+0x00009524) 00060104 MADD.S64 X3, X0, X2 +# [DWARF] common/pa_trace.h:265 +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +# 263 | const uint32_t bit = 1U << index; +# 264 | if ((active_mask & bit) == 0) continue; +# > 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +0x00000000000097e8 (+0x00009528) 02038800 MOV_XD_XN.S64 X1, X24 +0x00000000000097ec (+0x0000952c) 00003101 ADD.S64 X0, X3, X2 +0x00000000000097f0 (+0x00009530) 08800680 SUB_IMM.S64 X0, X0, #1664 +0x00000000000097f4 (+0x00009534) 02c20202 SHL.B64 X1, #2 +0x00000000000097f8 (+0x00009538) 08000000 ADD_IMM.S64 X0, X0, #0 +0x00000000000097fc (+0x0000953c) 002c0081 ADD.S64 X22, X0, X1 +0x0000000000009800 (+0x00009540) 1c8b65b8 LD_XD_XN_IMM.B32 X5, X22, #1464 +# [DWARF] common/pa_trace.h:266 +# > 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +0x0000000000009804 (+0x00009544) 07020000 MOV_XD_IMM X1, #0 +0x0000000000009808 (+0x00009548) 07430100 MOVK X1, #256, #1 +0x000000000000980c (+0x0000954c) 07040000 MOV_XD_IMM X2, #0 +0x0000000000009810 (+0x00009550) 0745ff00 MOVK X2, #65280, #1 +0x0000000000009814 (+0x00009554) 00025082 SUB.S64 X1, X5, X1 +0x0000000000009818 (+0x00009558) 02821a00 ZEROEXT.U32 X1, X1 +0x000000000000981c (+0x0000955c) 0040113e CMP.U64.GT X1, X2 +0x0000000000009820 (+0x00009560) 40200003 JUMPC #3 +# [DWARF] common/pa_trace.h:267 +# > 267 | trace.atomic_counter_overflow = true; +0x0000000000009824 (+0x00009564) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x0000000000009828 (+0x00009568) 4000ffdf JUMP #65503 +# [DWARF] common/pa_trace.h:98 +# 92 | default: +# 93 | return -1; +# 94 | } +# 95 | } +# 96 | +# 97 | PA_DEVICE AtomicSite TraceAtomicPollBatchSite(uint32_t index) { +# > 98 | switch (index) { +0x000000000000982c (+0x0000956c) 02817a00 ZEROEXT.U32 X0, X23 +0x0000000000009830 (+0x00009570) 07020005 MOV_XD_IMM X1, #5 +0x0000000000009834 (+0x00009574) 004000be CMP.U64.GT X0, X1 +0x0000000000009838 (+0x00009578) 070c000f MOV_XD_IMM X6, #15 +0x000000000000983c (+0x0000957c) 40200002 JUMPC #2 +0x0000000000009840 (+0x00009580) 1c8d4000 LD_XD_XN_IMM.B32 X6, X20, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009844 (+0x00009584) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x0000000000009848 (+0x00009588) 00c2b78a AND.B64 X1, X11, X15 +0x000000000000984c (+0x0000958c) 02c4020e SHL.B64 X2, #14 +0x0000000000009850 (+0x00009590) 0006a381 ADD.S64 X3, X10, X7 +0x0000000000009854 (+0x00009594) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:273 +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# > 273 | trace.poll_burst.start_cycle[index], end_cycle, +0x0000000000009858 (+0x00009598) 02018800 MOV_XD_XN.S64 X0, X24 +0x000000000000985c (+0x0000959c) 00023101 ADD.S64 X1, X3, X2 +0x0000000000009860 (+0x000095a0) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x0000000000009864 (+0x000095a4) 02c00203 SHL.B64 X0, #3 +0x0000000000009868 (+0x000095a8) 08241000 ADD_IMM.S64 X18, X1, #0 +0x000000000000986c (+0x000095ac) 00012001 ADD.S64 X0, X18, X0 +0x0000000000009870 (+0x000095b0) 1cc60588 LD_XD_XN_IMM.B64 X3, X0, #1416 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x0000000000009874 (+0x000095b4) 0201a800 MOV_XD_XN.S64 X0, X26 +0x0000000000009878 (+0x000095b8) 0203b800 MOV_XD_XN.S64 X1, X27 +0x000000000000987c (+0x000095bc) 0205c800 MOV_XD_XN.S64 X2, X28 +0x0000000000009880 (+0x000095c0) 02095800 MOV_XD_XN.S64 X4, X21 +0x0000000000009884 (+0x000095c4) 070e788b MOV_XD_IMM X7, #30859 +0x0000000000009888 (+0x000095c8) 074f0000 MOVK X7, #0, #1 +0x000000000000988c (+0x000095cc) 078f0000 MOVK X7, #0, #2 +0x0000000000009890 (+0x000095d0) 40427000 CALL X7, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009894 (+0x000095d4) 1ccfe990 LD_XD_XN_IMM.B64 X7, X30, #2448 +0x0000000000009898 (+0x000095d8) 071c0000 MOV_XD_IMM X14, #0 +0x000000000000989c (+0x000095dc) 1cd9e9a0 LD_XD_XN_IMM.B64 X12, X30, #2464 +0x00000000000098a0 (+0x000095e0) 071e7fff MOV_XD_IMM X15, #32767 +0x00000000000098a4 (+0x000095e4) 1cd7e9a8 LD_XD_XN_IMM.B64 X11, X30, #2472 +# [DWARF] common/pa_trace.h:276 +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x00000000000098a8 (+0x000095e8) 0000070e CMP.S64.EQ X0, X14 +0x00000000000098ac (+0x000095ec) 1cd5e9b0 LD_XD_XN_IMM.B64 X10, X30, #2480 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000098b0 (+0x000095f0) 07220001 MOV_XD_IMM X17, #1 +0x00000000000098b4 (+0x000095f4) 07120006 MOV_XD_IMM X9, #6 +# [DWARF] common/pa_trace.h:276 +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x00000000000098b8 (+0x000095f8) 4020ffba JUMPC #65466 +# [DWARF] common/pa_trace.h:277 +# > 277 | if (trace.poll_batch_records == UINT64_MAX) { +0x00000000000098bc (+0x000095fc) 1cc12578 LD_XD_XN_IMM.B64 X0, X18, #1400 +0x00000000000098c0 (+0x00009600) 07020001 MOV_XD_IMM X1, #1 +0x00000000000098c4 (+0x00009604) 02021080 NEG.S64 X1, X1 +0x00000000000098c8 (+0x00009608) 0000009e CMP.S64.NE X0, X1 +0x00000000000098cc (+0x0000960c) 4020ffab JUMPC #65451 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000098d0 (+0x00009610) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x00000000000098d4 (+0x00009614) 00c0b78a AND.B64 X0, X11, X15 +0x00000000000098d8 (+0x00009618) 02c2020e SHL.B64 X1, #14 +0x00000000000098dc (+0x0000961c) 0004a381 ADD.S64 X2, X10, X7 +0x00000000000098e0 (+0x00009620) 00040084 MADD.S64 X2, X0, X1 +0x00000000000098e4 (+0x00009624) 00002081 ADD.S64 X0, X2, X1 +0x00000000000098e8 (+0x00009628) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:278 +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# 276 | if (written) { +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# > 278 | trace.atomic_counter_overflow = true; +0x00000000000098ec (+0x0000962c) 08000000 ADD_IMM.S64 X0, X0, #0 +0x00000000000098f0 (+0x00009630) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x00000000000098f4 (+0x00009634) 4000ffab JUMP #65451 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000098f8 (+0x00009638) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x00000000000098fc (+0x0000963c) 00c0b78a AND.B64 X0, X11, X15 +0x0000000000009900 (+0x00009640) 02c2020e SHL.B64 X1, #14 +0x0000000000009904 (+0x00009644) 0004a381 ADD.S64 X2, X10, X7 +0x0000000000009908 (+0x00009648) 00040084 MADD.S64 X2, X0, X1 +0x000000000000990c (+0x0000964c) 00002081 ADD.S64 X0, X2, X1 +0x0000000000009910 (+0x00009650) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:285 +# 279 | } else { +# 280 | ++trace.poll_batch_records; +# 281 | } +# 282 | } +# 283 | trace.poll_burst.call_count[index] = 0; +# 284 | } +# > 285 | trace.poll_burst.active_mask = 0; +0x0000000000009914 (+0x00009654) 08000000 ADD_IMM.S64 X0, X0, #0 +0x0000000000009918 (+0x00009658) 0f960a00 STI_XN_IMM.B32 X0, #1488 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x000000000000991c (+0x0000965c) 0001a70e CMP.S64.EQ X26, X14 +0x0000000000009920 (+0x00009660) 4020002f JUMPC #47 +0x0000000000009924 (+0x00009664) 0001b70e CMP.S64.EQ X27, X14 +0x0000000000009928 (+0x00009668) 4020002d JUMPC #45 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000992c (+0x0000966c) 0283ca00 ZEROEXT.U32 X1, X28 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000009930 (+0x00009670) 0000170e CMP.S64.EQ X1, X14 +0x0000000000009934 (+0x00009674) 4020002a JUMPC #42 +# [DWARF] common/pa_trace.h:547 +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# > 547 | const uint32_t slot = core.count; +0x0000000000009938 (+0x00009678) 1c81a000 LD_XD_XN_IMM.B32 X0, X26, #0 +# [DWARF] common/pa_trace.h:548 +# > 548 | if (slot >= trace.capacity) { +0x000000000000993c (+0x0000967c) 004000ae CMP.U64.LT X0, X1 +0x0000000000009940 (+0x00009680) 40200002 JUMPC #2 +0x0000000000009944 (+0x00009684) 40000023 JUMP #35 +# [DWARF] common/pa_trace.h:553 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# > 553 | record.start_cycle = start_cycle; +0x0000000000009948 (+0x00009688) 1cc9e988 LD_XD_XN_IMM.B64 X4, X30, #2440 +# [DWARF] common/pa_trace.h:552 +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x000000000000994c (+0x0000968c) 02020800 MOV_XD_XN.S64 X1, X0 +0x0000000000009950 (+0x00009690) 02c20206 SHL.B64 X1, #6 +0x0000000000009954 (+0x00009694) 0003b081 ADD.S64 X1, X27, X1 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009958 (+0x00009698) 0806c002 ADD_IMM.S64 X3, X12, #2 +0x000000000000995c (+0x0000969c) 00c4b78a AND.B64 X2, X11, X15 +0x0000000000009960 (+0x000096a0) 02c6020e SHL.B64 X3, #14 +# [DWARF] common/pa_trace.h:557 +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x0000000000009964 (+0x000096a4) 070a000b MOV_XD_IMM X5, #11 +# [DWARF] common/pa_trace.h:565 +# 558 | record.lane = trace.lane; +# 559 | record.block_id = trace.block_id; +# 560 | record.core_idx = trace.core_idx; +# 561 | record.flags = flags; +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x0000000000009968 (+0x000096a8) 08000001 ADD_IMM.S64 X0, X0, #1 +# [DWARF] common/pa_trace.h:553 +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# > 553 | record.start_cycle = start_cycle; +0x000000000000996c (+0x000096ac) 09c81a81 STP_XI_XJ_XN.B64 X4, X21, X1, #0 +0x0000000000009970 (+0x000096b0) 0008a381 ADD.S64 X4, X10, X7 +0x0000000000009974 (+0x000096b4) 00082184 MADD.S64 X4, X2, X3 +0x0000000000009978 (+0x000096b8) 00044181 ADD.S64 X2, X4, X3 +0x000000000000997c (+0x000096bc) 1cc9e8f8 LD_XD_XN_IMM.B64 X4, X30, #2296 +# [DWARF] common/pa_trace.h:556 +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# > 556 | record.function_id = function_id; +0x0000000000009980 (+0x000096c0) 0706ffff MOV_XD_IMM X3, #65535 +0x0000000000009984 (+0x000096c4) 08842680 SUB_IMM.S64 X2, X2, #1664 +0x0000000000009988 (+0x000096c8) 0747ffff MOVK X3, #65535, #1 +0x000000000000998c (+0x000096cc) 098811a1 STP_XI_XJ_XN.B32 X4, X3, X1, #16 +# [DWARF] common/pa_trace.h:558 +# 557 | record.phase = static_cast(trace_phase); +# > 558 | record.lane = trace.lane; +0x0000000000009990 (+0x000096d0) 08042000 ADD_IMM.S64 X2, X2, #0 +0x0000000000009994 (+0x000096d4) 1c862560 LD_XD_XN_IMM.B32 X3, X2, #1376 +# [DWARF] common/pa_trace.h:557 +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x0000000000009998 (+0x000096d8) 098a11b1 STP_XI_XJ_XN.B32 X5, X3, X1, #24 +# [DWARF] common/pa_trace.h:559 +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x000000000000999c (+0x000096dc) 08062564 ADD_IMM.S64 X3, X2, #1380 +0x00000000000099a0 (+0x000096e0) 0c863100 LDP_XI_XJ_XN.B32 X3, X2, X3, #0 +0x00000000000099a4 (+0x000096e4) 03861020 ST_XD_XN_IMM.B32 X3, X1, #32 +0x00000000000099a8 (+0x000096e8) 07060000 MOV_XD_IMM X3, #0 +0x00000000000099ac (+0x000096ec) 07470001 MOVK X3, #1, #1 +0x00000000000099b0 (+0x000096f0) 004641af CMPN.U64.LT X3, X4, X3 +0x00000000000099b4 (+0x000096f4) 02c60201 SHL.B64 X3, #1 +0x00000000000099b8 (+0x000096f8) 00c63c8b OR.B64 X3, X3, X25 +# [DWARF] common/pa_trace.h:560 +# > 560 | record.core_idx = trace.core_idx; +0x00000000000099bc (+0x000096fc) 08081024 ADD_IMM.S64 X4, X1, #36 +0x00000000000099c0 (+0x00009700) 09844181 STP_XI_XJ_XN.B32 X2, X3, X4, #0 +# [DWARF] common/pa_trace.h:562 +# 561 | record.flags = flags; +# > 562 | record.auxiliary = auxiliary; +0x00000000000099c4 (+0x00009704) 03a2102c ST_XD_XN_IMM.B32 X17, X1, #44 +# [DWARF] common/pa_trace.h:565 +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x00000000000099c8 (+0x00009708) 0381a000 ST_XD_XN_IMM.B32 X0, X26, #0 +0x00000000000099cc (+0x0000970c) 40000004 JUMP #4 +# [DWARF] common/pa_trace.h:549 +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# > 549 | core.dropped = core.dropped + 1; +0x00000000000099d0 (+0x00009710) 1c81a004 LD_XD_XN_IMM.B32 X0, X26, #4 +0x00000000000099d4 (+0x00009714) 08000001 ADD_IMM.S64 X0, X0, #1 +0x00000000000099d8 (+0x00009718) 0381a004 ST_XD_XN_IMM.B32 X0, X26, #4 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000099dc (+0x0000971c) 0802c002 ADD_IMM.S64 X1, X12, #2 +# [DWARF] common/pa_frontend.h:262 +# 256 | // TensorTagMixin::tags_{} is value-initialized by the +# 257 | // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. +# 258 | // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar +# 259 | // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 +# 260 | volatile int32_t *tags = &args.tags[0]; +# 261 | for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { +# > 262 | tags[index] = 0; +0x00000000000099e0 (+0x00009720) 0fa7e700 STI_XN_IMM.B32 X30, #2488 +# [DWARF] common/pa_frontend.h:0 +# [SOURCE unavailable] +0x00000000000099e4 (+0x00009724) 07080648 MOV_XD_IMM X4, #1608 +0x00000000000099e8 (+0x00009728) 00c0b78a AND.B64 X0, X11, X15 +0x00000000000099ec (+0x0000972c) 02c2020e SHL.B64 X1, #14 +0x00000000000099f0 (+0x00009730) 0004a381 ADD.S64 X2, X10, X7 +0x00000000000099f4 (+0x00009734) 0009e202 SUB.S64 X4, X30, X4 +0x00000000000099f8 (+0x00009738) 00040084 MADD.S64 X2, X0, X1 +# [DWARF] common/pa_frontend.h:262 +# 256 | // TensorTagMixin::tags_{} is value-initialized by the +# 257 | // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. +# 258 | // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar +# 259 | // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 +# 260 | volatile int32_t *tags = &args.tags[0]; +# 261 | for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { +# > 262 | tags[index] = 0; +0x00000000000099fc (+0x0000973c) 020a4800 MOV_XD_XN.S64 X5, X4 +0x0000000000009a00 (+0x00009740) 07060002 MOV_XD_IMM X3, #2 +0x0000000000009a04 (+0x00009744) 00002081 ADD.S64 X0, X2, X1 +0x0000000000009a08 (+0x00009748) 02ca3440 SBITSET.B64 X5, X3 +0x0000000000009a0c (+0x0000974c) 08800680 SUB_IMM.S64 X0, X0, #1664 +0x0000000000009a10 (+0x00009750) 0f805000 STI_XN_IMM.B32 X5, #0 +0x0000000000009a14 (+0x00009754) 0fa7e800 STI_XN_IMM.B32 X30, #2496 +# [DWARF] common/pa_scheduler_core.h:948 +# 942 | callback(callback_builder); +# 943 | if (!callback_builder.Valid()) return false; +# 944 | const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); +# 945 | stats.result.arg_resets += counts.reset_calls; +# 946 | stats.result.views_created += counts.views_created; +# 947 | stats.result.dynamic_create_infos += counts.dynamic_create_infos; +# > 948 | stats.result.tensor_args_added += counts.tensor_args_added; +0x0000000000009a18 (+0x00009758) 08260000 ADD_IMM.S64 X19, X0, #0 +# [DWARF] common/pa_frontend.h:262 +# 256 | // TensorTagMixin::tags_{} is value-initialized by the +# 257 | // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. +# 258 | // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar +# 259 | // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 +# 260 | volatile int32_t *tags = &args.tags[0]; +# 261 | for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { +# > 262 | tags[index] = 0; +0x0000000000009a1c (+0x0000975c) 0fa7e880 STI_XN_IMM.B32 X30, #2500 +# [DWARF] common/pa_frontend.h:314 +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# > 314 | args.tensors[index].pointer.create_info = &create_info; +0x0000000000009a20 (+0x00009760) 070201b8 MOV_XD_IMM X1, #440 +# [DWARF] common/pa_frontend.h:262 +# 256 | // TensorTagMixin::tags_{} is value-initialized by the +# 257 | // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. +# 258 | // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar +# 259 | // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 +# 260 | volatile int32_t *tags = &args.tags[0]; +# 261 | for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { +# > 262 | tags[index] = 0; +0x0000000000009a24 (+0x00009764) 0fa7e900 STI_XN_IMM.B32 X30, #2504 +# [DWARF] common/pa_frontend.h:314 +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# > 314 | args.tensors[index].pointer.create_info = &create_info; +0x0000000000009a28 (+0x00009768) 0003e081 ADD.S64 X1, X30, X1 +# [DWARF] common/pa_frontend.h:262 +# 256 | // TensorTagMixin::tags_{} is value-initialized by the +# 257 | // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. +# 258 | // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar +# 259 | // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 +# 260 | volatile int32_t *tags = &args.tags[0]; +# 261 | for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { +# > 262 | tags[index] = 0; +0x0000000000009a2c (+0x0000976c) 0fa7e980 STI_XN_IMM.B32 X30, #2508 +# [DWARF] common/pa_frontend.h:313 +# 307 | args.tensors[index].pointer.gm_tensor = &tensor; +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# > 313 | const uint32_t index = static_cast(args.tensor_count++); +0x0000000000009a30 (+0x00009770) 07040003 MOV_XD_IMM X2, #3 +# [DWARF] common/pa_frontend.h:262 +# 256 | // TensorTagMixin::tags_{} is value-initialized by the +# 257 | // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. +# 258 | // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar +# 259 | // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 +# 260 | volatile int32_t *tags = &args.tags[0]; +# 261 | for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { +# > 262 | tags[index] = 0; +0x0000000000009a34 (+0x00009774) 0fa7ea00 STI_XN_IMM.B32 X30, #2512 +0x0000000000009a38 (+0x00009778) 0fa7ea80 STI_XN_IMM.B32 X30, #2516 +0x0000000000009a3c (+0x0000977c) 0fa7eb00 STI_XN_IMM.B32 X30, #2520 +0x0000000000009a40 (+0x00009780) 0fa7eb80 STI_XN_IMM.B32 X30, #2524 +0x0000000000009a44 (+0x00009784) 0fa7ec00 STI_XN_IMM.B32 X30, #2528 +0x0000000000009a48 (+0x00009788) 0fa7ec80 STI_XN_IMM.B32 X30, #2532 +0x0000000000009a4c (+0x0000978c) 0fa7ed00 STI_XN_IMM.B32 X30, #2536 +0x0000000000009a50 (+0x00009790) 0fa7ed80 STI_XN_IMM.B32 X30, #2540 +0x0000000000009a54 (+0x00009794) 0fa7ee00 STI_XN_IMM.B32 X30, #2544 +0x0000000000009a58 (+0x00009798) 0fa7ee80 STI_XN_IMM.B32 X30, #2548 +0x0000000000009a5c (+0x0000979c) 0fa7ef00 STI_XN_IMM.B32 X30, #2552 +0x0000000000009a60 (+0x000097a0) 0fa7ef80 STI_XN_IMM.B32 X30, #2556 +0x0000000000009a64 (+0x000097a4) 0fa9e000 STI_XN_IMM.B32 X30, #2560 +0x0000000000009a68 (+0x000097a8) 0fa9e080 STI_XN_IMM.B32 X30, #2564 +0x0000000000009a6c (+0x000097ac) 0fa9e100 STI_XN_IMM.B32 X30, #2568 +0x0000000000009a70 (+0x000097b0) 0fa9e180 STI_XN_IMM.B32 X30, #2572 +0x0000000000009a74 (+0x000097b4) 0fa9e200 STI_XN_IMM.B32 X30, #2576 +0x0000000000009a78 (+0x000097b8) 0fa9e280 STI_XN_IMM.B32 X30, #2580 +0x0000000000009a7c (+0x000097bc) 0fa9e300 STI_XN_IMM.B32 X30, #2584 +0x0000000000009a80 (+0x000097c0) 0fa9e380 STI_XN_IMM.B32 X30, #2588 +0x0000000000009a84 (+0x000097c4) 0fa9e400 STI_XN_IMM.B32 X30, #2592 +0x0000000000009a88 (+0x000097c8) 0fa9e480 STI_XN_IMM.B32 X30, #2596 +0x0000000000009a8c (+0x000097cc) 0fa9e500 STI_XN_IMM.B32 X30, #2600 +0x0000000000009a90 (+0x000097d0) 0fa9e580 STI_XN_IMM.B32 X30, #2604 +0x0000000000009a94 (+0x000097d4) 0fa9e600 STI_XN_IMM.B32 X30, #2608 +0x0000000000009a98 (+0x000097d8) 0fa9e680 STI_XN_IMM.B32 X30, #2612 +# [DWARF] common/pa_frontend.h:265 +# 263 | } +# 264 | args.tensor_count = 0; +# > 265 | args.scalar_count = 0; +0x0000000000009a9c (+0x000097dc) 0fb3e780 STI_XN_IMM.B32 X30, #3260 +# [DWARF] common/pa_frontend.h:266 +# > 266 | args.has_error = false; +0x0000000000009aa0 (+0x000097e0) 0f33e800 STI_XN_IMM.B8 X30, #3264 +# [DWARF] common/pa_frontend.h:267 +# > 267 | args.error_msg = 0; +0x0000000000009aa4 (+0x000097e4) 0ff3e900 STI_XN_IMM.B64 X30, #3272 +# [DWARF] common/pa_frontend.h:268 +# > 268 | args.launch_spec.core_num = 1; +0x0000000000009aa8 (+0x000097e8) 0f73ea01 STI_XN_IMM.B16 X30, #3280 +# [DWARF] common/pa_frontend.h:269 +# > 269 | args.launch_spec.require_sync_start = false; +0x0000000000009aac (+0x000097ec) 0f33ea40 STI_XN_IMM.B8 X30, #3282 +# [DWARF] common/pa_frontend.h:243 +# 237 | +# 238 | PA_DEVICE void ClearDumpArgSelection(PaDumpArgSelection &selection) { +# 239 | // Volatile stores intentionally preserve the profiling-enabled PA reset +# 240 | // traffic even though the standalone winner workload never consumes dump data. +# 241 | // volatile 的目的不是同步，而是阻止编译器删掉这段生产基线中存在的写流量。 +# 242 | volatile uint64_t *masks = &selection.dump_arg_mask; +# > 243 | masks[0] = 0; +0x0000000000009ab0 (+0x000097f0) 0ff3eb00 STI_XN_IMM.B64 X30, #3288 +# [DWARF] common/pa_frontend.h:244 +# > 244 | masks[1] = 0; +0x0000000000009ab4 (+0x000097f4) 0ff3ec00 STI_XN_IMM.B64 X30, #3296 +# [DWARF] common/pa_frontend.h:247 +# 245 | volatile uint64_t *sources = &selection.scalar_source_ptrs[0]; +# 246 | for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { +# > 247 | sources[index] = 0; +0x0000000000009ab8 (+0x000097f8) 0ff3ed00 STI_XN_IMM.B64 X30, #3304 +0x0000000000009abc (+0x000097fc) 0ff3ee00 STI_XN_IMM.B64 X30, #3312 +0x0000000000009ac0 (+0x00009800) 0ff3ef00 STI_XN_IMM.B64 X30, #3320 +0x0000000000009ac4 (+0x00009804) 0ff5e000 STI_XN_IMM.B64 X30, #3328 +0x0000000000009ac8 (+0x00009808) 0ff5e100 STI_XN_IMM.B64 X30, #3336 +0x0000000000009acc (+0x0000980c) 0ff5e200 STI_XN_IMM.B64 X30, #3344 +0x0000000000009ad0 (+0x00009810) 0ff5e300 STI_XN_IMM.B64 X30, #3352 +0x0000000000009ad4 (+0x00009814) 0ff5e400 STI_XN_IMM.B64 X30, #3360 +0x0000000000009ad8 (+0x00009818) 0ff5e500 STI_XN_IMM.B64 X30, #3368 +0x0000000000009adc (+0x0000981c) 0ff5e600 STI_XN_IMM.B64 X30, #3376 +0x0000000000009ae0 (+0x00009820) 0ff5e700 STI_XN_IMM.B64 X30, #3384 +0x0000000000009ae4 (+0x00009824) 0ff5e800 STI_XN_IMM.B64 X30, #3392 +0x0000000000009ae8 (+0x00009828) 0ff5e900 STI_XN_IMM.B64 X30, #3400 +0x0000000000009aec (+0x0000982c) 0ff5ea00 STI_XN_IMM.B64 X30, #3408 +0x0000000000009af0 (+0x00009830) 0ff5eb00 STI_XN_IMM.B64 X30, #3416 +0x0000000000009af4 (+0x00009834) 0ff5ec00 STI_XN_IMM.B64 X30, #3424 +# [DWARF] common/pa_frontend.h:251 +# 248 | } +# 249 | volatile uint8_t *dtypes = &selection.scalar_dtypes[0]; +# 250 | for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { +# > 251 | dtypes[index] = 0; +0x0000000000009af8 (+0x00009838) 0f35ed00 STI_XN_IMM.B8 X30, #3432 +0x0000000000009afc (+0x0000983c) 0f35ed20 STI_XN_IMM.B8 X30, #3433 +0x0000000000009b00 (+0x00009840) 0f35ed40 STI_XN_IMM.B8 X30, #3434 +0x0000000000009b04 (+0x00009844) 0f35ed60 STI_XN_IMM.B8 X30, #3435 +0x0000000000009b08 (+0x00009848) 0f35ed80 STI_XN_IMM.B8 X30, #3436 +0x0000000000009b0c (+0x0000984c) 0f35eda0 STI_XN_IMM.B8 X30, #3437 +0x0000000000009b10 (+0x00009850) 0f35edc0 STI_XN_IMM.B8 X30, #3438 +0x0000000000009b14 (+0x00009854) 0f35ede0 STI_XN_IMM.B8 X30, #3439 +0x0000000000009b18 (+0x00009858) 0f35ee00 STI_XN_IMM.B8 X30, #3440 +0x0000000000009b1c (+0x0000985c) 0f35ee20 STI_XN_IMM.B8 X30, #3441 +0x0000000000009b20 (+0x00009860) 0f35ee40 STI_XN_IMM.B8 X30, #3442 +0x0000000000009b24 (+0x00009864) 0f35ee60 STI_XN_IMM.B8 X30, #3443 +0x0000000000009b28 (+0x00009868) 0f35ee80 STI_XN_IMM.B8 X30, #3444 +0x0000000000009b2c (+0x0000986c) 0f35eea0 STI_XN_IMM.B8 X30, #3445 +0x0000000000009b30 (+0x00009870) 0f35eec0 STI_XN_IMM.B8 X30, #3446 +0x0000000000009b34 (+0x00009874) 0f35eee0 STI_XN_IMM.B8 X30, #3447 +# [DWARF] common/pa_scheduler_core.h:948 +# 942 | callback(callback_builder); +# 943 | if (!callback_builder.Valid()) return false; +# 944 | const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); +# 945 | stats.result.arg_resets += counts.reset_calls; +# 946 | stats.result.views_created += counts.views_created; +# 947 | stats.result.dynamic_create_infos += counts.dynamic_create_infos; +# > 948 | stats.result.tensor_args_added += counts.tensor_args_added; +0x0000000000009b38 (+0x00009878) 1cc133e0 LD_XD_XN_IMM.B64 X0, X19, #992 +# [DWARF] common/pa_frontend.h:314 +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# > 314 | args.tensors[index].pointer.create_info = &create_info; +0x0000000000009b3c (+0x0000987c) 03c3ea38 ST_XD_XN_IMM.B64 X1, X30, #2616 +# [DWARF] common/pa_frontend.h:0 +# [SOURCE unavailable] +0x0000000000009b40 (+0x00009880) 070201f8 MOV_XD_IMM X1, #504 +0x0000000000009b44 (+0x00009884) 0003e081 ADD.S64 X1, X30, X1 +# [DWARF] common/pa_frontend.h:271 +# 265 | args.scalar_count = 0; +# 266 | args.has_error = false; +# 267 | args.error_msg = 0; +# 268 | args.launch_spec.core_num = 1; +# 269 | args.launch_spec.require_sync_start = false; +# 270 | ClearDumpArgSelection(args.dump_arg_selection); +# > 271 | args.explicit_deps = 0; +0x0000000000009b48 (+0x00009888) 0ff5ef00 STI_XN_IMM.B64 X30, #3448 +# [DWARF] common/pa_frontend.h:272 +# > 272 | args.explicit_dep_count = 0; +0x0000000000009b4c (+0x0000988c) 0fb7e000 STI_XN_IMM.B32 X30, #3456 +# [DWARF] common/pa_frontend.h:315 +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# 314 | args.tensors[index].pointer.create_info = &create_info; +# > 315 | args.tensors[index].kind = TensorRefKind::CreateInfo; +0x0000000000009b50 (+0x00009890) 0307ea40 ST_XD_XN_IMM.B8 X3, X30, #2624 +# [DWARF] common/pa_frontend.h:316 +# > 316 | args.tags[index] = TagValue(TensorArgType::Output); +0x0000000000009b54 (+0x00009894) 0fa7e701 STI_XN_IMM.B32 X30, #2488 +# [DWARF] common/pa_frontend.h:314 +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# > 314 | args.tensors[index].pointer.create_info = &create_info; +0x0000000000009b58 (+0x00009898) 03c3ea48 ST_XD_XN_IMM.B64 X1, X30, #2632 +# [DWARF] common/pa_frontend.h:315 +# > 315 | args.tensors[index].kind = TensorRefKind::CreateInfo; +0x0000000000009b5c (+0x0000989c) 0307ea50 ST_XD_XN_IMM.B8 X3, X30, #2640 +# [DWARF] common/pa_frontend.h:316 +# > 316 | args.tags[index] = TagValue(TensorArgType::Output); +0x0000000000009b60 (+0x000098a0) 0f805001 STI_XN_IMM.B32 X5, #0 +# [DWARF] common/pa_frontend.h:313 +# 307 | args.tensors[index].pointer.gm_tensor = &tensor; +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# > 313 | const uint32_t index = static_cast(args.tensor_count++); +0x0000000000009b64 (+0x000098a4) 0385ecb8 ST_XD_XN_IMM.B32 X2, X30, #3256 +# [DWARF] common/pa_frontend.h:314 +# > 314 | args.tensors[index].pointer.create_info = &create_info; +0x0000000000009b68 (+0x000098a8) 03c3ea58 ST_XD_XN_IMM.B64 X1, X30, #2648 +# [DWARF] ccec/ccec_ops.h:276 +# 270 | } +# 271 | +# 272 | __aicore__ static inline bool FinishLazySampleCallback( +# 273 | const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args +# 274 | ) { +# 275 | #if defined(PA_BUILD_AIC) +# > 276 | return ::pa_scheduler_lazy_sample_callback_finish_aic(ticket, args) != 0; +0x0000000000009b6c (+0x000098ac) 02024800 MOV_XD_XN.S64 X1, X4 +0x0000000000009b70 (+0x000098b0) 03cbe8f0 ST_XD_XN_IMM.B64 X5, X30, #2288 +# [DWARF] common/pa_frontend.h:315 +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# 314 | args.tensors[index].pointer.create_info = &create_info; +# > 315 | args.tensors[index].kind = TensorRefKind::CreateInfo; +0x0000000000009b74 (+0x000098b4) 0307ea60 ST_XD_XN_IMM.B8 X3, X30, #2656 +# [DWARF] common/pa_frontend.h:316 +# > 316 | args.tags[index] = TagValue(TensorArgType::Output); +0x0000000000009b78 (+0x000098b8) 0fa7e801 STI_XN_IMM.B32 X30, #2496 +# [DWARF] common/pa_scheduler_core.h:948 +# 942 | callback(callback_builder); +# 943 | if (!callback_builder.Valid()) return false; +# 944 | const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); +# 945 | stats.result.arg_resets += counts.reset_calls; +# 946 | stats.result.views_created += counts.views_created; +# 947 | stats.result.dynamic_create_infos += counts.dynamic_create_infos; +# > 948 | stats.result.tensor_args_added += counts.tensor_args_added; +0x0000000000009b7c (+0x000098bc) 08000003 ADD_IMM.S64 X0, X0, #3 +0x0000000000009b80 (+0x000098c0) 03c133e0 ST_XD_XN_IMM.B64 X0, X19, #992 +# [DWARF] common/pa_scheduler_core.h:1161 +# 1155 | ); +# 1156 | +# 1157 | if (!BuildLazySampleCallbackArgs(orch, args, batch, claim.won, stats)) { +# 1158 | SetFatal(state, stats, static_cast(task_id)); +# 1159 | return false; +# 1160 | } +# > 1161 | const LazySampleCallbackTicket ticket{ +0x0000000000009b84 (+0x000098c4) 1cc1e928 LD_XD_XN_IMM.B64 X0, X30, #2344 +0x0000000000009b88 (+0x000098c8) 03c1e338 ST_XD_XN_IMM.B64 X0, X30, #824 +0x0000000000009b8c (+0x000098cc) 1cc1e8f8 LD_XD_XN_IMM.B64 X0, X30, #2296 +0x0000000000009b90 (+0x000098d0) 0381e340 ST_XD_XN_IMM.B32 X0, X30, #832 +# [DWARF] ccec/ccec_ops.h:276 +# 270 | } +# 271 | +# 272 | __aicore__ static inline bool FinishLazySampleCallback( +# 273 | const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args +# 274 | ) { +# 275 | #if defined(PA_BUILD_AIC) +# > 276 | return ::pa_scheduler_lazy_sample_callback_finish_aic(ticket, args) != 0; +0x0000000000009b94 (+0x000098d4) 07000338 MOV_XD_IMM X0, #824 +0x0000000000009b98 (+0x000098d8) 0001e001 ADD.S64 X0, X30, X0 +# [DWARF] common/pa_scheduler_core.h:1161 +# 1155 | ); +# 1156 | +# 1157 | if (!BuildLazySampleCallbackArgs(orch, args, batch, claim.won, stats)) { +# 1158 | SetFatal(state, stats, static_cast(task_id)); +# 1159 | return false; +# 1160 | } +# > 1161 | const LazySampleCallbackTicket ticket{ +0x0000000000009b9c (+0x000098dc) 0f4de882 STI_XN_IMM.B16 X30, #836 +0x0000000000009ba0 (+0x000098e0) 0333e346 ST_XD_XN_IMM.B8 X25, X30, #838 +0x0000000000009ba4 (+0x000098e4) 0f0de8e0 STI_XN_IMM.B8 X30, #839 +# [DWARF] ccec/ccec_ops.h:276 +# 270 | } +# 271 | +# 272 | __aicore__ static inline bool FinishLazySampleCallback( +# 273 | const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args +# 274 | ) { +# 275 | #if defined(PA_BUILD_AIC) +# > 276 | return ::pa_scheduler_lazy_sample_callback_finish_aic(ticket, args) != 0; +0x0000000000009ba8 (+0x000098e8) 070477fb MOV_XD_IMM X2, #30715 +0x0000000000009bac (+0x000098ec) 07450000 MOVK X2, #0, #1 +0x0000000000009bb0 (+0x000098f0) 07850000 MOVK X2, #0, #2 +0x0000000000009bb4 (+0x000098f4) 40422000 CALL X2, #0 +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x0000000000009bb8 (+0x000098f8) 1cf3e980 LD_XD_XN_IMM.B64 X25, X30, #2432 +0x0000000000009bbc (+0x000098fc) 071c0000 MOV_XD_IMM X14, #0 +# [DWARF] ccec/ccec_ops.h:276 +# 270 | } +# 271 | +# 272 | __aicore__ static inline bool FinishLazySampleCallback( +# 273 | const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args +# 274 | ) { +# 275 | #if defined(PA_BUILD_AIC) +# > 276 | return ::pa_scheduler_lazy_sample_callback_finish_aic(ticket, args) != 0; +0x0000000000009bc0 (+0x00009900) 02800a00 ZEROEXT.U32 X0, X0 +0x0000000000009bc4 (+0x00009904) 07220001 MOV_XD_IMM X17, #1 +0x0000000000009bc8 (+0x00009908) 0000070e CMP.S64.EQ X0, X14 +0x0000000000009bcc (+0x0000990c) 07100004 MOV_XD_IMM X8, #4 +0x0000000000009bd0 (+0x00009910) 07120006 MOV_XD_IMM X9, #6 +0x0000000000009bd4 (+0x00009914) 07200020 MOV_XD_IMM X16, #32 +# [DWARF] common/pa_scheduler_core.h:1382 +# 1376 | orchestration_begin = TraceTimestamp(stats.trace, stats.result); +# 1377 | InitPaOrchestration(orchestration, batches, &state->context_lens[0]); +# 1378 | for (uint32_t batch = 0; batch < batches; ++batch) { +# 1379 | #if defined(PA_LAZY_SAMPLE_SHAPE_ID) +# 1380 | BeginPaBatchForLazySampleCallback(orchestration, batch); +# 1381 | ++stats.result.context_reads; +# > 1382 | if (!SubmitLazySampleCallback< +0x0000000000009bd8 (+0x00009918) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000009bdc (+0x0000991c) 070058cf MOV_XD_IMM X0, #22735 +0x0000000000009be0 (+0x00009920) 07410000 MOVK X0, #0, #1 +0x0000000000009be4 (+0x00009924) 07810000 MOVK X0, #0, #2 +0x0000000000009be8 (+0x00009928) 40220000 JUMPC X0, #0 +0x0000000000009bec (+0x0000992c) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_frontend.h:550 +# 544 | } +# 545 | +# 546 | PA_DEVICE void PreparePaBlockGroup(PaOrchestrationState &orch, uint64_t block_offset) { +# 547 | // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 +# 548 | // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 +# 549 | orch.current_block_offset = block_offset; +# > 550 | orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); +0x0000000000009bf0 (+0x00009930) 0801e2c8 ADD_IMM.S64 X0, X30, #712 +# [DWARF] common/pa_frontend.h:832 +# 826 | // 每个 worker 都完整回放并接收自己 materialize 的 descriptor；只有 Claim winner +# 827 | // 会执行 kernel，但 loser 的后续 orchestration 仍使用相同 task_id/owner 拓扑。 +# 828 | switch (kind) { +# 829 | case TaskKind::Alloc: +# 830 | orch.accumulated_output = outputs.tensors[0]; +# 831 | orch.accumulated_sum = outputs.tensors[1]; +# > 832 | orch.accumulated_max = outputs.tensors[2]; +0x0000000000009bf4 (+0x00009934) 1cc93060 LD_XD_XN_IMM.B64 X4, X19, #96 +# [DWARF] common/pa_frontend.h:550 +# 544 | } +# 545 | +# 546 | PA_DEVICE void PreparePaBlockGroup(PaOrchestrationState &orch, uint64_t block_offset) { +# 547 | // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 +# 548 | // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 +# 549 | orch.current_block_offset = block_offset; +# > 550 | orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); +0x0000000000009bf8 (+0x00009938) 0cc20000 LDP_XI_XJ_XN.B64 X1, X0, X0, #0 +# [DWARF] common/pa_frontend.h:830 +# 824 | +# 825 | PA_DEVICE void AcceptTaskOutputs(PaOrchestrationState &orch, TaskKind kind, const TaskOutputs &outputs) { +# 826 | // 每个 worker 都完整回放并接收自己 materialize 的 descriptor；只有 Claim winner +# 827 | // 会执行 kernel，但 loser 的后续 orchestration 仍使用相同 task_id/owner 拓扑。 +# 828 | switch (kind) { +# 829 | case TaskKind::Alloc: +# > 830 | orch.accumulated_output = outputs.tensors[0]; +0x0000000000009bfc (+0x0000993c) 08053050 ADD_IMM.S64 X2, X19, #80 +0x0000000000009c00 (+0x00009940) 070a0040 MOV_XD_IMM X5, #64 +0x0000000000009c04 (+0x00009944) 0cc42180 LDP_XI_XJ_XN.B64 X2, X3, X2, #0 +# [DWARF] common/pa_frontend.h:0 +# [SOURCE unavailable] +0x0000000000009c08 (+0x00009948) 071e7fff MOV_XD_IMM X15, #32767 +# [DWARF] common/pa_frontend.h:528 +# 522 | orch.query_view.strides[0] = kPaHeadDim; +# 523 | orch.query_view.strides[1] = 1; +# 524 | orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; +# 525 | } +# 526 | #endif +# 527 | +# > 528 | PA_DEVICE uint64_t MinU64(uint64_t lhs, uint64_t rhs) { return lhs < rhs ? lhs : rhs; } +0x0000000000009c0c (+0x0000994c) 004002ae CMP.U64.LT X0, X5 +0x0000000000009c10 (+0x00009950) 00c00289 SEL.B64 X0, X0, X5 +# [DWARF] common/pa_frontend.h:550 +# 544 | } +# 545 | +# 546 | PA_DEVICE void PreparePaBlockGroup(PaOrchestrationState &orch, uint64_t block_offset) { +# 547 | // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 +# 548 | // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 +# 549 | orch.current_block_offset = block_offset; +# > 550 | orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); +0x0000000000009c14 (+0x00009954) 03c1e2e0 ST_XD_XN_IMM.B64 X0, X30, #736 +# [DWARF] common/pa_frontend.h:552 +# 551 | const uint64_t last_block_sequence_start = +# > 552 | (block_offset + orch.current_nblocks - 1) * kPaBlockSize; +0x0000000000009c18 (+0x00009958) 02c00207 SHL.B64 X0, #7 +0x0000000000009c1c (+0x0000995c) 00001002 SUB.S64 X0, X1, X0 +# [DWARF] common/pa_frontend.h:830 +# 824 | +# 825 | PA_DEVICE void AcceptTaskOutputs(PaOrchestrationState &orch, TaskKind kind, const TaskOutputs &outputs) { +# 826 | // 每个 worker 都完整回放并接收自己 materialize 的 descriptor；只有 Claim winner +# 827 | // 会执行 kernel，但 loser 的后续 orchestration 仍使用相同 task_id/owner 拓扑。 +# 828 | switch (kind) { +# 829 | case TaskKind::Alloc: +# > 830 | orch.accumulated_output = outputs.tensors[0]; +0x0000000000009c20 (+0x00009960) 0803e2f8 ADD_IMM.S64 X1, X30, #760 +0x0000000000009c24 (+0x00009964) 09c41181 STP_XI_XJ_XN.B64 X2, X3, X1, #0 +0x0000000000009c28 (+0x00009968) 1cc5e960 LD_XD_XN_IMM.B64 X2, X30, #2400 +# [DWARF] common/pa_frontend.h:553 +# 547 | // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 +# 548 | // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 +# 549 | orch.current_block_offset = block_offset; +# 550 | orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); +# 551 | const uint64_t last_block_sequence_start = +# 552 | (block_offset + orch.current_nblocks - 1) * kPaBlockSize; +# > 553 | orch.current_valid_len = MinU64(kPaBlockSize, orch.current_sequence - last_block_sequence_start); +0x0000000000009c2c (+0x0000996c) 08000080 ADD_IMM.S64 X0, X0, #128 +0x0000000000009c30 (+0x00009970) 07020080 MOV_XD_IMM X1, #128 +# [DWARF] common/pa_frontend.h:832 +# 826 | // 每个 worker 都完整回放并接收自己 materialize 的 descriptor；只有 Claim winner +# 827 | // 会执行 kernel，但 loser 的后续 orchestration 仍使用相同 task_id/owner 拓扑。 +# 828 | switch (kind) { +# 829 | case TaskKind::Alloc: +# 830 | orch.accumulated_output = outputs.tensors[0]; +# 831 | orch.accumulated_sum = outputs.tensors[1]; +# > 832 | orch.accumulated_max = outputs.tensors[2]; +0x0000000000009c34 (+0x00009974) 03c9e308 ST_XD_XN_IMM.B64 X4, X30, #776 +# [DWARF] common/pa_frontend.h:528 +# 522 | orch.query_view.strides[0] = kPaHeadDim; +# 523 | orch.query_view.strides[1] = 1; +# 524 | orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; +# 525 | } +# 526 | #endif +# 527 | +# > 528 | PA_DEVICE uint64_t MinU64(uint64_t lhs, uint64_t rhs) { return lhs < rhs ? lhs : rhs; } +0x0000000000009c38 (+0x00009978) 004000ae CMP.U64.LT X0, X1 +# [DWARF] common/pa_frontend.h:549 +# 543 | return static_cast(*value); +# 544 | } +# 545 | +# 546 | PA_DEVICE void PreparePaBlockGroup(PaOrchestrationState &orch, uint64_t block_offset) { +# 547 | // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 +# 548 | // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 +# > 549 | orch.current_block_offset = block_offset; +0x0000000000009c3c (+0x0000997c) 0fcbeb00 STI_XN_IMM.B64 X30, #728 +# [DWARF] common/pa_frontend.h:528 +# 522 | orch.query_view.strides[0] = kPaHeadDim; +# 523 | orch.query_view.strides[1] = 1; +# 524 | orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; +# 525 | } +# 526 | #endif +# 527 | +# > 528 | PA_DEVICE uint64_t MinU64(uint64_t lhs, uint64_t rhs) { return lhs < rhs ? lhs : rhs; } +0x0000000000009c40 (+0x00009980) 00c00089 SEL.B64 X0, X0, X1 +# [DWARF] common/pa_frontend.h:553 +# 547 | // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 +# 548 | // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 +# 549 | orch.current_block_offset = block_offset; +# 550 | orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); +# 551 | const uint64_t last_block_sequence_start = +# 552 | (block_offset + orch.current_nblocks - 1) * kPaBlockSize; +# > 553 | orch.current_valid_len = MinU64(kPaBlockSize, orch.current_sequence - last_block_sequence_start); +0x0000000000009c44 (+0x00009984) 03c1e2e8 ST_XD_XN_IMM.B64 X0, X30, #744 +# [DWARF] common/pa_scheduler_core.h:790 +# 784 | +# 785 | PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { +# 786 | // This is BeginSubmit without an already-materialized TaskArgs. The same +# 787 | // fields are completed synchronously after the single callback builds args. +# 788 | const uint32_t task_id = static_cast(worker.local_index++); +# 789 | context.self = &worker; +# > 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +0x0000000000009c48 (+0x00009988) 070207ff MOV_XD_IMM X1, #2047 +# [DWARF] common/pa_scheduler_core.h:788 +# 782 | static_assert(offsetof(LazySampleCallbackTicket, function_id) == 12, "lazy sample callback ticket function offset mismatch"); +# 783 | static_assert(offsetof(LazySampleCallbackTicket, won) == 14, "lazy sample callback ticket winner offset mismatch"); +# 784 | +# 785 | PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { +# 786 | // This is BeginSubmit without an already-materialized TaskArgs. The same +# 787 | // fields are completed synchronously after the single callback builds args. +# > 788 | const uint32_t task_id = static_cast(worker.local_index++); +0x0000000000009c4c (+0x0000998c) 1c862014 LD_XD_XN_IMM.B32 X3, X2, #20 +0x0000000000009c50 (+0x00009990) 08003001 ADD_IMM.S64 X0, X3, #1 +0x0000000000009c54 (+0x00009994) 03802014 ST_XD_XN_IMM.B32 X0, X2, #20 +# [DWARF] common/pa_scheduler_core.h:790 +# 789 | context.self = &worker; +# > 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +0x0000000000009c58 (+0x00009998) 1cc1e900 LD_XD_XN_IMM.B64 X0, X30, #2304 +0x0000000000009c5c (+0x0000999c) 00c2308a AND.B64 X1, X3, X1 +0x0000000000009c60 (+0x000099a0) 02c2020c SHL.B64 X1, #12 +0x0000000000009c64 (+0x000099a4) 00000081 ADD.S64 X0, X0, X1 +0x0000000000009c68 (+0x000099a8) 02023800 MOV_XD_XN.S64 X1, X3 +# [DWARF] common/pa_scheduler_core.h:789 +# 783 | static_assert(offsetof(LazySampleCallbackTicket, won) == 14, "lazy sample callback ticket winner offset mismatch"); +# 784 | +# 785 | PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { +# 786 | // This is BeginSubmit without an already-materialized TaskArgs. The same +# 787 | // fields are completed synchronously after the single callback builds args. +# 788 | const uint32_t task_id = static_cast(worker.local_index++); +# > 789 | context.self = &worker; +0x0000000000009c6c (+0x000099ac) 09c53031 STP_XI_XJ_XN.B64 X2, X0, X19, #24 +# [DWARF] common/pa_scheduler_core.h:793 +# 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +# 791 | context.task_id = static_cast(task_id); +# 792 | context.tensor_count = 0; +# > 793 | context.scalar_count = 0; +0x0000000000009c70 (+0x000099b0) 07000000 MOV_XD_IMM X0, #0 +# [DWARF] common/pa_scheduler_core.h:791 +# 785 | PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { +# 786 | // This is BeginSubmit without an already-materialized TaskArgs. The same +# 787 | // fields are completed synchronously after the single callback builds args. +# 788 | const uint32_t task_id = static_cast(worker.local_index++); +# 789 | context.self = &worker; +# 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +# > 791 | context.task_id = static_cast(task_id); +0x0000000000009c74 (+0x000099b4) 03833028 ST_XD_XN_IMM.B32 X1, X19, #40 +# [DWARF] common/pa_scheduler_core.h:793 +# 792 | context.tensor_count = 0; +# > 793 | context.scalar_count = 0; +0x0000000000009c78 (+0x000099b8) 08053030 ADD_IMM.S64 X2, X19, #48 +# [DWARF] common/pa_scheduler_core.h:792 +# 786 | // This is BeginSubmit without an already-materialized TaskArgs. The same +# 787 | // fields are completed synchronously after the single callback builds args. +# 788 | const uint32_t task_id = static_cast(worker.local_index++); +# 789 | context.self = &worker; +# 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +# 791 | context.task_id = static_cast(task_id); +# > 792 | context.tensor_count = 0; +0x0000000000009c7c (+0x000099bc) 0f813580 STI_XN_IMM.B32 X19, #44 +# [DWARF] common/pa_scheduler_core.h:794 +# 793 | context.scalar_count = 0; +# > 794 | context.result.task_id = task_id; +0x0000000000009c80 (+0x000099c0) 03c33040 ST_XD_XN_IMM.B64 X1, X19, #64 +# [DWARF] common/pa_scheduler_core.h:795 +# > 795 | context.result.count = 0; +0x0000000000009c84 (+0x000099c4) 0f813900 STI_XN_IMM.B32 X19, #72 +# [DWARF] common/pa_scheduler_core.h:793 +# 787 | // fields are completed synchronously after the single callback builds args. +# 788 | const uint32_t task_id = static_cast(worker.local_index++); +# 789 | context.self = &worker; +# 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +# 791 | context.task_id = static_cast(task_id); +# 792 | context.tensor_count = 0; +# > 793 | context.scalar_count = 0; +0x0000000000009c88 (+0x000099c8) 09c02001 STP_XI_XJ_XN.B64 X0, X0, X2, #0 +# [DWARF] common/pa_scheduler_core.h:798 +# 794 | context.result.task_id = task_id; +# 795 | context.result.count = 0; +# 796 | context.register_mask = 0; +# 797 | context.output_bytes = 0; +# > 798 | context.fanin_count = 0; +0x0000000000009c8c (+0x000099cc) 07000000 MOV_XD_IMM X0, #0 +0x0000000000009c90 (+0x000099d0) 0781ffff MOVK X0, #65535, #2 +# [DWARF] common/pa_scheduler_core.h:800 +# 799 | context.kernel_id = -1; +# > 800 | context.won = false; +0x0000000000009c94 (+0x000099d4) 0f473300 STI_XN_IMM.B16 X19, #408 +# [DWARF] common/pa_scheduler_core.h:798 +# 792 | context.tensor_count = 0; +# 793 | context.scalar_count = 0; +# 794 | context.result.task_id = task_id; +# 795 | context.result.count = 0; +# 796 | context.register_mask = 0; +# 797 | context.output_bytes = 0; +# > 798 | context.fanin_count = 0; +0x0000000000009c98 (+0x000099d8) 07c1ffff MOVK X0, #65535, #3 +# [DWARF] common/pa_scheduler_core.h:802 +# 799 | context.kernel_id = -1; +# 800 | context.won = false; +# 801 | context.joint = false; +# > 802 | context.joint_init = false; +0x0000000000009c9c (+0x000099dc) 0f073340 STI_XN_IMM.B8 X19, #410 +# [DWARF] common/pa_scheduler_core.h:798 +# 792 | context.tensor_count = 0; +# 793 | context.scalar_count = 0; +# 794 | context.result.task_id = task_id; +# 795 | context.result.count = 0; +# 796 | context.register_mask = 0; +# 797 | context.output_bytes = 0; +# > 798 | context.fanin_count = 0; +0x0000000000009ca0 (+0x000099e0) 03c13190 ST_XD_XN_IMM.B64 X0, X19, #400 +# [DWARF] common/pa_scheduler_core.h:804 +# 799 | context.kernel_id = -1; +# 800 | context.won = false; +# 801 | context.joint = false; +# 802 | context.joint_init = false; +# 803 | context.joint_block = -1; +# > 804 | context.joint_slot = -1; +0x0000000000009ca4 (+0x000099e4) 0700ffff MOV_XD_IMM X0, #65535 +0x0000000000009ca8 (+0x000099e8) 0741ffff MOVK X0, #65535, #1 +# [DWARF] common/pa_scheduler_core.h:803 +# 797 | context.output_bytes = 0; +# 798 | context.fanin_count = 0; +# 799 | context.kernel_id = -1; +# 800 | context.won = false; +# 801 | context.joint = false; +# 802 | context.joint_init = false; +# > 803 | context.joint_block = -1; +0x0000000000009cac (+0x000099ec) 0f873382 STI_XN_IMM.B32 X19, #412 +# [DWARF] common/pa_scheduler_core.h:804 +# > 804 | context.joint_slot = -1; +0x0000000000009cb0 (+0x000099f0) 03c131a0 ST_XD_XN_IMM.B64 X0, X19, #416 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000009cb4 (+0x000099f4) 02808880 MOV_XD_SPR.F32 X0, SYS_CNT +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x0000000000009cb8 (+0x000099f8) 03c3e920 ST_XD_XN_IMM.B64 X1, X30, #2336 +0x0000000000009cbc (+0x000099fc) 1cd7e9b0 LD_XD_XN_IMM.B64 X11, X30, #2480 +0x0000000000009cc0 (+0x00009a00) 1cd5e9a8 LD_XD_XN_IMM.B64 X10, X30, #2472 +0x0000000000009cc4 (+0x00009a04) 1cd9e9a0 LD_XD_XN_IMM.B64 X12, X30, #2464 +0x0000000000009cc8 (+0x00009a08) 1ccfe990 LD_XD_XN_IMM.B64 X7, X30, #2448 +# [DWARF] common/pa_trace.h:257 +# 251 | TraceContext &trace, uint64_t end_cycle +# 252 | ) { +# 253 | #if PA_BUILD_SUBMIT_PMU +# 254 | (void)trace; +# 255 | (void)end_cycle; +# 256 | #else +# > 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +0x0000000000009ccc (+0x00009a0c) 03c1e928 ST_XD_XN_IMM.B64 X0, X30, #2344 +0x0000000000009cd0 (+0x00009a10) 1c01355c LD_XD_XN_IMM.B8 X0, X19, #1372 +0x0000000000009cd4 (+0x00009a14) 0000070e CMP.S64.EQ X0, X14 +0x0000000000009cd8 (+0x00009a18) 40200086 JUMPC #134 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009cdc (+0x00009a1c) 1ca535d0 LD_XD_XN_IMM.B32 X18, X19, #1488 +0x0000000000009ce0 (+0x00009a20) 02812a00 ZEROEXT.U32 X0, X18 +# [DWARF] common/pa_trace.h:257 +# 251 | TraceContext &trace, uint64_t end_cycle +# 252 | ) { +# 253 | #if PA_BUILD_SUBMIT_PMU +# 254 | (void)trace; +# 255 | (void)end_cycle; +# 256 | #else +# > 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +0x0000000000009ce4 (+0x00009a24) 0000070e CMP.S64.EQ X0, X14 +0x0000000000009ce8 (+0x00009a28) 40200082 JUMPC #130 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009cec (+0x00009a2c) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x0000000000009cf0 (+0x00009a30) 00c0a78a AND.B64 X0, X10, X15 +0x0000000000009cf4 (+0x00009a34) 02c2020e SHL.B64 X1, #14 +0x0000000000009cf8 (+0x00009a38) 0004b381 ADD.S64 X2, X11, X7 +0x0000000000009cfc (+0x00009a3c) 00040084 MADD.S64 X2, X0, X1 +0x0000000000009d00 (+0x00009a40) 07280000 MOV_XD_IMM X20, #0 +0x0000000000009d04 (+0x00009a44) 00002081 ADD.S64 X0, X2, X1 +0x0000000000009d08 (+0x00009a48) 08800680 SUB_IMM.S64 X0, X0, #1664 +0x0000000000009d0c (+0x00009a4c) 08000000 ADD_IMM.S64 X0, X0, #0 +0x0000000000009d10 (+0x00009a50) 082a0548 ADD_IMM.S64 X21, X0, #1352 +0x0000000000009d14 (+0x00009a54) 1cb40558 LD_XD_XN_IMM.B32 X26, X0, #1368 +0x0000000000009d18 (+0x00009a58) 0ceb5c80 LDP_XI_XJ_XN.B64 X21, X25, X21, #0 +0x0000000000009d1c (+0x00009a5c) 072e0000 MOV_XD_IMM X23, #0 +0x0000000000009d20 (+0x00009a60) 0726be78 MOV_XD_IMM X19, #48760 +0x0000000000009d24 (+0x00009a64) 07670007 MOVK X19, #7, #1 +0x0000000000009d28 (+0x00009a68) 07a70000 MOVK X19, #0, #2 +0x0000000000009d2c (+0x00009a6c) 07e70000 MOVK X19, #0, #3 +0x0000000000009d30 (+0x00009a70) 02040880 MOV_XD_SPR.S64 X2, PC +0x0000000000009d34 (+0x00009a74) 00273101 ADD.S64 X19, X19, X2 +0x0000000000009d38 (+0x00009a78) 40000012 JUMP #18 +0x0000000000009d3c (+0x00009a7c) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x0000000000009d40 (+0x00009a80) 00c2a78a AND.B64 X1, X10, X15 +0x0000000000009d44 (+0x00009a84) 02c4020e SHL.B64 X2, #14 +0x0000000000009d48 (+0x00009a88) 0006b381 ADD.S64 X3, X11, X7 +0x0000000000009d4c (+0x00009a8c) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:280 +# 274 | call_count, static_cast(site) +# 275 | ); +# 276 | if (written) { +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# 278 | trace.atomic_counter_overflow = true; +# 279 | } else { +# > 280 | ++trace.poll_batch_records; +0x0000000000009d50 (+0x00009a90) 08000001 ADD_IMM.S64 X0, X0, #1 +0x0000000000009d54 (+0x00009a94) 00023101 ADD.S64 X1, X3, X2 +0x0000000000009d58 (+0x00009a98) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x0000000000009d5c (+0x00009a9c) 08021000 ADD_IMM.S64 X1, X1, #0 +0x0000000000009d60 (+0x00009aa0) 03c01578 ST_XD_XN_IMM.B64 X0, X1, #1400 +# [DWARF] common/pa_trace.h:283 +# 281 | } +# 282 | } +# > 283 | trace.poll_burst.call_count[index] = 0; +0x0000000000009d64 (+0x00009aa4) 0f976700 STI_XN_IMM.B32 X22, #1464 +# [DWARF] common/pa_trace.h:262 +# 256 | #else +# 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +# 258 | const uint32_t active_mask = trace.poll_burst.active_mask; +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# > 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +0x0000000000009d68 (+0x00009aa8) 082f7001 ADD_IMM.S64 X23, X23, #1 +0x0000000000009d6c (+0x00009aac) 08273004 ADD_IMM.S64 X19, X19, #4 +0x0000000000009d70 (+0x00009ab0) 0001749e CMP.S64.NE X23, X9 +0x0000000000009d74 (+0x00009ab4) 08294001 ADD_IMM.S64 X20, X20, #1 +0x0000000000009d78 (+0x00009ab8) 40200002 JUMPC #2 +0x0000000000009d7c (+0x00009abc) 40000050 JUMP #80 +# [DWARF] common/pa_trace.h:263 +# > 263 | const uint32_t bit = 1U << index; +0x0000000000009d80 (+0x00009ac0) 02814a00 ZEROEXT.U32 X0, X20 +# [DWARF] common/pa_trace.h:264 +# > 264 | if ((active_mask & bit) == 0) continue; +0x0000000000009d84 (+0x00009ac4) 02832a00 ZEROEXT.U32 X1, X18 +0x0000000000009d88 (+0x00009ac8) 024202c0 SHR.U64 X1, X0, #0 +0x0000000000009d8c (+0x00009acc) 00c0188a AND.B64 X0, X1, X17 +0x0000000000009d90 (+0x00009ad0) 0000070e CMP.S64.EQ X0, X14 +0x0000000000009d94 (+0x00009ad4) 4020fff5 JUMPC #65525 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009d98 (+0x00009ad8) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x0000000000009d9c (+0x00009adc) 00c0a78a AND.B64 X0, X10, X15 +0x0000000000009da0 (+0x00009ae0) 02c4020e SHL.B64 X2, #14 +0x0000000000009da4 (+0x00009ae4) 0006b381 ADD.S64 X3, X11, X7 +0x0000000000009da8 (+0x00009ae8) 00060104 MADD.S64 X3, X0, X2 +# [DWARF] common/pa_trace.h:265 +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +# 263 | const uint32_t bit = 1U << index; +# 264 | if ((active_mask & bit) == 0) continue; +# > 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +0x0000000000009dac (+0x00009aec) 02037800 MOV_XD_XN.S64 X1, X23 +0x0000000000009db0 (+0x00009af0) 00003101 ADD.S64 X0, X3, X2 +0x0000000000009db4 (+0x00009af4) 08800680 SUB_IMM.S64 X0, X0, #1664 +0x0000000000009db8 (+0x00009af8) 02c20202 SHL.B64 X1, #2 +0x0000000000009dbc (+0x00009afc) 08000000 ADD_IMM.S64 X0, X0, #0 +0x0000000000009dc0 (+0x00009b00) 002c0081 ADD.S64 X22, X0, X1 +0x0000000000009dc4 (+0x00009b04) 1c8b65b8 LD_XD_XN_IMM.B32 X5, X22, #1464 +# [DWARF] common/pa_trace.h:266 +# > 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +0x0000000000009dc8 (+0x00009b08) 07020000 MOV_XD_IMM X1, #0 +0x0000000000009dcc (+0x00009b0c) 07430100 MOVK X1, #256, #1 +0x0000000000009dd0 (+0x00009b10) 07040000 MOV_XD_IMM X2, #0 +0x0000000000009dd4 (+0x00009b14) 0745ff00 MOVK X2, #65280, #1 +0x0000000000009dd8 (+0x00009b18) 00025082 SUB.S64 X1, X5, X1 +0x0000000000009ddc (+0x00009b1c) 02821a00 ZEROEXT.U32 X1, X1 +0x0000000000009de0 (+0x00009b20) 0040113e CMP.U64.GT X1, X2 +0x0000000000009de4 (+0x00009b24) 40200003 JUMPC #3 +# [DWARF] common/pa_trace.h:267 +# > 267 | trace.atomic_counter_overflow = true; +0x0000000000009de8 (+0x00009b28) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x0000000000009dec (+0x00009b2c) 4000ffdf JUMP #65503 +# [DWARF] common/pa_trace.h:98 +# 92 | default: +# 93 | return -1; +# 94 | } +# 95 | } +# 96 | +# 97 | PA_DEVICE AtomicSite TraceAtomicPollBatchSite(uint32_t index) { +# > 98 | switch (index) { +0x0000000000009df0 (+0x00009b30) 02814a00 ZEROEXT.U32 X0, X20 +0x0000000000009df4 (+0x00009b34) 07020005 MOV_XD_IMM X1, #5 +0x0000000000009df8 (+0x00009b38) 004000be CMP.U64.GT X0, X1 +0x0000000000009dfc (+0x00009b3c) 070c000f MOV_XD_IMM X6, #15 +0x0000000000009e00 (+0x00009b40) 40200002 JUMPC #2 +0x0000000000009e04 (+0x00009b44) 1c8d3000 LD_XD_XN_IMM.B32 X6, X19, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009e08 (+0x00009b48) 0804c002 ADD_IMM.S64 X2, X12, #2 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x0000000000009e0c (+0x00009b4c) 1cc9e928 LD_XD_XN_IMM.B64 X4, X30, #2344 +0x0000000000009e10 (+0x00009b50) 00c2a78a AND.B64 X1, X10, X15 +0x0000000000009e14 (+0x00009b54) 02c4020e SHL.B64 X2, #14 +0x0000000000009e18 (+0x00009b58) 0006b381 ADD.S64 X3, X11, X7 +0x0000000000009e1c (+0x00009b5c) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:273 +# 272 | trace.core, trace.records, trace.capacity, +# > 273 | trace.poll_burst.start_cycle[index], end_cycle, +0x0000000000009e20 (+0x00009b60) 02017800 MOV_XD_XN.S64 X0, X23 +0x0000000000009e24 (+0x00009b64) 00023101 ADD.S64 X1, X3, X2 +0x0000000000009e28 (+0x00009b68) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x0000000000009e2c (+0x00009b6c) 02c00203 SHL.B64 X0, #3 +0x0000000000009e30 (+0x00009b70) 08301000 ADD_IMM.S64 X24, X1, #0 +0x0000000000009e34 (+0x00009b74) 00018001 ADD.S64 X0, X24, X0 +0x0000000000009e38 (+0x00009b78) 1cc60588 LD_XD_XN_IMM.B64 X3, X0, #1416 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x0000000000009e3c (+0x00009b7c) 02015800 MOV_XD_XN.S64 X0, X21 +0x0000000000009e40 (+0x00009b80) 02039800 MOV_XD_XN.S64 X1, X25 +0x0000000000009e44 (+0x00009b84) 0205a800 MOV_XD_XN.S64 X2, X26 +0x0000000000009e48 (+0x00009b88) 070e771a MOV_XD_IMM X7, #30490 +0x0000000000009e4c (+0x00009b8c) 074f0000 MOVK X7, #0, #1 +0x0000000000009e50 (+0x00009b90) 078f0000 MOVK X7, #0, #2 +0x0000000000009e54 (+0x00009b94) 40427000 CALL X7, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009e58 (+0x00009b98) 1ccfe990 LD_XD_XN_IMM.B64 X7, X30, #2448 +0x0000000000009e5c (+0x00009b9c) 071c0000 MOV_XD_IMM X14, #0 +0x0000000000009e60 (+0x00009ba0) 1cd9e9a0 LD_XD_XN_IMM.B64 X12, X30, #2464 +0x0000000000009e64 (+0x00009ba4) 071e7fff MOV_XD_IMM X15, #32767 diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/published.sha256 new file mode 100644 index 0000000000..69b80159f1 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/published.sha256 @@ -0,0 +1,39 @@ +3292a86b93eab8348954313002fc1dd65f738064eba46a55046b82db2f00b05c annotated/00_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz +c4e1068852d4913326c9966b980778be629c6342f2a572f935b9195ac3ce5acf annotated/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz +2c8f08e69e6d275d14c21752b013cdca005f76fe85c82d927f77dd5d009251eb annotated/02_pa_execute_real_winner_workload_aic_729699de.source.asm.gz +25a4cea4738ceda57bf84c563351c6c58ab3bd4cbe43fbef4a537a87dbec31b8 annotated/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.source.asm.gz +e028d624bc208e64ede4f6c56984f7d4327097f57ba7368d47d08c56be8b595e annotated/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +4a5974e6be2ebb736fe03677ae942dea94220c137c8f15dcb6921132d8ded67e annotated/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.source.asm.gz +0aaf06d33ab00c2a7e7dd9db07cc8dbcb7c8ace605f802b35e22ee32eebdf053 annotated/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +df8dc382afac580888cd730a9c100f6bb4db3d78bcaaf650a71e10ae2874d9b8 annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz +26f8f4222cf44beecb8162c737e5dc5634f04d60a3112bf2b7cbb8f5bdfdb007 annotated/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.source.asm.gz +aa8c58ba245738d297d1bfd417a461bba620f1f19449ab7f558ce7775b7baebe annotated/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +58115295576f99ee50e7d30c79d010c1f344f32267dca760dc1e54d0f65584f8 annotated/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz +8427b688bb1cb27b9ce5fa18b1e32e43aa00db1d77c56046c1ecc6de2536e1f0 annotated/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz +9647b3f76dcfb9191b379c104d2f55df284cadf87c66b7dca706a05512456f04 annotated/12_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz +ad138409ad8cc12e50a8da6d53508b2a5b2ada8de885cf36fffe9c24b72b35d3 annotated/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.source.asm.gz +ede15075567acbd2fb310a8605c03e516770c1d5b8d2aeb0c84e56c6a79b1603 annotated/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +efb4b5fa5adbb4eab3024373eac7732eb0822a1a47053c4b7e48b40eecf9f27d annotated/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz +2fb817ccfef3b0e1a84858b3fb372abff3a46ad8e4c80cf31e2edde1ee174082 annotated/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz +10b4fd32aa317ea69a229d45c2bc81089204cf05bb0491e2e21b25e178e79020 key_flow/aic_compete_first.source.asm +818c59ec85db9ebffdebec406ebbf434534cfb1920d8517b7ee1ebd88d6fba06 raw/00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz +713680d66b9a4010516e94c8b1eaf4699de3d3d168a0071ffcc683d485e6fcfd raw/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz +000992bf2e056f3f0e25e2e951d7ce17f574ce42fe85808eab4c81d64e82fd96 raw/02_pa_execute_real_winner_workload_aic_729699de.asm.gz +af912f03ca0d8d5a25ab0eb0e18bc2f1fea568c2170926e4956b978a56ad4e86 raw/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz +f588ab8d1c7c292c69d4bb25a20a74cc2072a7a5ae70cb021b167601ec7254bc raw/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +dee953904a64d9bbc05899f8c6af53a61e67a61da29e6b94047d565b91139861 raw/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz +505a11beffc1964f5f60cdd5a535207c7e9e81f84cc9f352dc7cea77f01ff612 raw/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +ea465c73c294f044872df4d536cee4f14299a3937acc9bf86549c5e1026b1bf9 raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz +8df4f8d7705b55eef87d28c821425083f4eb3e6e00a7b42cc6233f1c2e04c2d8 raw/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz +359c91af50cb07eab96b6e31e08e9dbf202b16068835a11c6ce97884df2f0e8e raw/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +feed9d8a6d512e0710afca1661e453175c953fea9d07503f1732b4c0b605f02c raw/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz +7470b8f6211167da842268a5454438b45db0abc7222e143ced6bc10012875a22 raw/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz +e2d30bf63296ad647bdfd996f7b1119273753e5820b90e72e0082c0a80725a58 raw/12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz +ccac55531be6275b35fe9e8af72f1307050df9817c93b4857a7406f8713f764d raw/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz +3bedb6520d68993867c5f264efbf93d720f64c55387adc6c90c29638ff7a559a raw/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +c27e21715651d040bd90609566937e55b73a2995ba83f931cec8d43eec90dafc raw/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz +785be4d5d33fd2d6f4e914b0ef3980d11565ae336587289f208c18572763da39 raw/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz +3ea3b8526cce898bfd3dcc89428fd6224aaff736e093b3d548563b1da65e0111 raw/README.md +f3b2a710693b2203667061eba93f9f25125447688f6b119fc538331ec7262cdf raw/gaps.tsv +37e5eb8364d0bc99702167e350de753564eb203007cf46994d3e63154ca182c4 raw/manifest.tsv +388d35f89270629c2d65334cf652d9f0c36cf96f39fdbfa79e0414e73a749fd7 raw/published.sha256 diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz new file mode 100644 index 0000000000..472990ea84 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz new file mode 100644 index 0000000000..6ed56bb5e1 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/02_pa_execute_real_winner_workload_aic_729699de.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/02_pa_execute_real_winner_workload_aic_729699de.asm.gz new file mode 100644 index 0000000000..abb60066cc Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/02_pa_execute_real_winner_workload_aic_729699de.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz new file mode 100644 index 0000000000..77ab6cbe45 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz new file mode 100644 index 0000000000..c474b0b0f0 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz new file mode 100644 index 0000000000..5cb4dca53a Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz new file mode 100644 index 0000000000..e59c662104 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz new file mode 100644 index 0000000000..ed220b1231 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz new file mode 100644 index 0000000000..92a44cf355 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz new file mode 100644 index 0000000000..198f1cc53d Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz new file mode 100644 index 0000000000..635b30d623 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz new file mode 100644 index 0000000000..127c7c130d Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz new file mode 100644 index 0000000000..18c10bfd19 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz new file mode 100644 index 0000000000..25ee7dcf53 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz new file mode 100644 index 0000000000..eea3fef03c Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz new file mode 100644 index 0000000000..9d359aa5b7 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz new file mode 100644 index 0000000000..08b6ea4062 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/README.md b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/README.md new file mode 100644 index 0000000000..3d11a008af --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/README.md @@ -0,0 +1,14 @@ +# Final-linked A5 disassembly + +Every non-empty final `.text` `STT_FUNC` symbol was extracted from the published final ELF and decoded with the verified `dav_3510` PEM decoder. +RVec `.vector.thread` symbols are passed through an explicit full-body RVec range; all other symbols use scalar decoding. +Each compressed file records final PC, function-relative offset, machine word, mnemonic, ELF/body hashes and decoder identity. +Function gaps are alignment/padding outside symbol bodies and are hashed in `gaps.tsv`; they are not presented as instructions. + +Decoder SHA256: `29835d2439d6dd464d34a212ad4bbd5c29af6a38465da09a6c273401d9a96dcb` + +| Variant | Final .text | Function symbols | Function bytes | Coverage | Final ELF SHA256 | +| --- | ---: | ---: | ---: | ---: | --- | +| compete-first | 547640 B | 17 | 547176 B | 99.915% | `82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b` | + +Inspect with `zless FILE.asm.gz`; the sibling `annotated/` directory adds DWARF-mapped local source and comments. diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/gaps.tsv b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/gaps.tsv new file mode 100644 index 0000000000..c6ce267e1e --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/gaps.tsv @@ -0,0 +1,9 @@ +variant start_pc end_pc size sha256 +compete-first 0x68 0x70 8 c573025535b1cea90f421ad7615d65296651dd5bc7287d7dfce7cc79c878ef4b +compete-first 0x27b98 0x27ba0 8 c573025535b1cea90f421ad7615d65296651dd5bc7287d7dfce7cc79c878ef4b +compete-first 0x42aac 0x42ab0 4 d9fc8a51763953481a1808af3156bcb8144c2f925e96dec623c886f6d9d975b2 +compete-first 0x42b2c 0x42b30 4 d9fc8a51763953481a1808af3156bcb8144c2f925e96dec623c886f6d9d975b2 +compete-first 0x5dbfc 0x5dc00 4 d9fc8a51763953481a1808af3156bcb8144c2f925e96dec623c886f6d9d975b2 +compete-first 0x85924 0x85a00 220 4e2f4af2b7331e77e1bea78cbdae527b13fd7d38fb35aa9db95a3dda9ed1c5b8 +compete-first 0x85a30 0x85b00 208 c9364bd8435644c4dcf160545b42c2cfb80001a10890c7a1cdd1f7a0a28dc8ac +compete-first 0x85b30 0x85b38 8 c573025535b1cea90f421ad7615d65296651dd5bc7287d7dfce7cc79c878ef4b diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/manifest.tsv b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/manifest.tsv new file mode 100644 index 0000000000..beecf8301d --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/manifest.tsv @@ -0,0 +1,18 @@ +variant file binding symbol final_pc size instructions decoder_mode body_sha256 last_mnemonic +compete-first 00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz GLOBAL pa_scheduler_0_mix_aic 0x0 104 26 scalar 3a8ecf84e154c1afc9dab2c992e3357d338c1d996a677931069f16310bdb28e5 END +compete-first 01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz LOCAL _ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j 0x70 192 48 scalar b8afa7640101b76f5e91a2e172ddd1cb39b6e11ccecd0c4395057eabc1e2bc00 RET +compete-first 02_pa_execute_real_winner_workload_aic_729699de.asm.gz LOCAL pa_execute_real_winner_workload_aic 0x130 400 100 scalar 88b6e6169f66470b7d9746ae3b1dc48a5e2bf4bd46994a85cc37926c394adeb6 RET +compete-first 03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz LOCAL pa_scheduler_lazy_sample_callback_orchestration_aic 0x2c0 161788 40447 scalar cab368a779b3870411a46bd169f6af9657c025967797ba7743b93fcfe27f45db RET +compete-first 04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz LOCAL _ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3AS1NS_11TraceRecordEjmmjj 0x27abc 220 55 scalar 38a5fe6dd99194bb6e77f41b4e40c23dfd587016d92a20f4505d0a816ce02e21 RET +compete-first 05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz LOCAL pa_scheduler_lazy_sample_callback_finish_aic 0x27ba0 110128 27532 scalar cdb102a84acb7a681565d97805b405d526a94c604812faed3850773575acee08 JUMP +compete-first 06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz LOCAL _ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3AS1NS_11TraceRecordEjmmjj 0x429d0 220 55 scalar 422635da4a38494a486262b1261ddcd2127aa8aac6f695aee3bea17bebe7f36c RET +compete-first 07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz GLOBAL pa_scheduler_0_mix_aiv 0x42ab0 124 31 scalar 63f9349fa440318678bcf3e98ad719502bd907f7537fb0f083eac32895e3ad9b END +compete-first 08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz LOCAL pa_scheduler_lazy_sample_callback_finish_aiv 0x42b30 110576 27644 scalar 3f5ec8fee73b82e72f40566908c21f4fcfd3fd85983892c323a526c66557cade JUMP +compete-first 09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz LOCAL _ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3AS1NS_11TraceRecordEjmmjj 0x5db20 220 55 scalar 1fd702642d07bfb00a40e2f710068a59fc171927a070d63778975d757fe07add RET +compete-first 10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz LOCAL _ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j 0x5dc00 268 66 scalar d55e1e765b39016fa5139e91d947f6816b6bea5ff18a176c45b48d5ada5a5d8f RET +compete-first 11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz LOCAL _ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j 0x5dd0c 268 66 scalar 8bb7a51e530f1c2f2c336a661b18e890bdce87740c931135a953e81e046ef3eb RET +compete-first 12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz LOCAL pa_execute_real_winner_workload_aiv 0x5de18 476 119 scalar 158c11e2f546c21643c2addb244ffefe3e46e5aeee5b04fbd4873dcd33a07363 RET +compete-first 13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz LOCAL pa_scheduler_lazy_sample_callback_orchestration_aiv 0x5dff4 161876 40469 scalar cc2b188dca238188ac76fd0ee6ff232c4be99f8e3274f4cee173316d67c3e503 RET +compete-first 14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz LOCAL _ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3AS1NS_11TraceRecordEjmmjj 0x85848 220 55 scalar 50460b9bd0d33e55466ebf252d284b0660113b8396c12783583aa92aae7d860f RET +compete-first 15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz LOCAL _ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128EEEvPU3AS6T0_S5_S5_jj.vector.thread 0x85a00 48 12 rvec 40cfc6402a5ba38eb3edbe949d63afe52206a592975835aa76b9f028b1d1a67a RV_SEND.U16 +compete-first 16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz LOCAL _ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128EEEvPU3AS6T0_S5_S5_jj.vector.thread 0x85b00 48 12 rvec 9afa7fa1b6031acff6f90ef3822167d31eeec4d90ee419db0b85dd7c40e5404c RV_SEND.U16 diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/published.sha256 new file mode 100644 index 0000000000..be38439e29 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/disassembly/raw/published.sha256 @@ -0,0 +1,20 @@ +818c59ec85db9ebffdebec406ebbf434534cfb1920d8517b7ee1ebd88d6fba06 00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz +713680d66b9a4010516e94c8b1eaf4699de3d3d168a0071ffcc683d485e6fcfd 01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz +000992bf2e056f3f0e25e2e951d7ce17f574ce42fe85808eab4c81d64e82fd96 02_pa_execute_real_winner_workload_aic_729699de.asm.gz +af912f03ca0d8d5a25ab0eb0e18bc2f1fea568c2170926e4956b978a56ad4e86 03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz +f588ab8d1c7c292c69d4bb25a20a74cc2072a7a5ae70cb021b167601ec7254bc 04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +dee953904a64d9bbc05899f8c6af53a61e67a61da29e6b94047d565b91139861 05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz +505a11beffc1964f5f60cdd5a535207c7e9e81f84cc9f352dc7cea77f01ff612 06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +ea465c73c294f044872df4d536cee4f14299a3937acc9bf86549c5e1026b1bf9 07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz +8df4f8d7705b55eef87d28c821425083f4eb3e6e00a7b42cc6233f1c2e04c2d8 08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz +359c91af50cb07eab96b6e31e08e9dbf202b16068835a11c6ce97884df2f0e8e 09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +feed9d8a6d512e0710afca1661e453175c953fea9d07503f1732b4c0b605f02c 10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz +7470b8f6211167da842268a5454438b45db0abc7222e143ced6bc10012875a22 11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz +e2d30bf63296ad647bdfd996f7b1119273753e5820b90e72e0082c0a80725a58 12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz +ccac55531be6275b35fe9e8af72f1307050df9817c93b4857a7406f8713f764d 13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz +3bedb6520d68993867c5f264efbf93d720f64c55387adc6c90c29638ff7a559a 14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +c27e21715651d040bd90609566937e55b73a2995ba83f931cec8d43eec90dafc 15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz +785be4d5d33fd2d6f4e914b0ef3980d11565ae336587289f208c18572763da39 16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz +3ea3b8526cce898bfd3dcc89428fd6224aaff736e093b3d548563b1da65e0111 README.md +f3b2a710693b2203667061eba93f9f25125447688f6b119fc538331ec7262cdf gaps.tsv +37e5eb8364d0bc99702167e350de753564eb203007cf46994d3e63154ca182c4 manifest.tsv diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/generate_disassembly.sh b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/generate_disassembly.sh new file mode 100755 index 0000000000..d48cef2b39 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/generate_disassembly.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +RAW_DIR="$SCRIPT_DIR/disassembly/raw" +ANNOTATED_DIR="$SCRIPT_DIR/disassembly/annotated" + +case "$RAW_DIR:$ANNOTATED_DIR" in + "$SCRIPT_DIR/disassembly/raw:$SCRIPT_DIR/disassembly/annotated") ;; + *) echo "Refusing to replace unexpected disassembly paths" >&2; exit 1 ;; +esac +rm -rf -- "$RAW_DIR" "$ANNOTATED_DIR" + +python3 "$SCRIPT_DIR/disassemble.py" --output "$RAW_DIR" +mkdir -p "$ANNOTATED_DIR" +for raw in "$RAW_DIR"/*.asm.gz; do + name="$(basename "$raw" .asm.gz)" + python3 "$SCRIPT_DIR/annotate_disassembly.py" \ + --elf "$SCRIPT_DIR/artifacts/measured/pa_scheduler_kernel.o" \ + --raw "$raw" \ + --source-root "$SCRIPT_DIR" \ + --output "$ANNOTATED_DIR/${name}.source.asm.gz" +done +mkdir -p "$SCRIPT_DIR/disassembly/key_flow" +python3 "$SCRIPT_DIR/annotate_disassembly.py" \ + --elf "$SCRIPT_DIR/artifacts/measured/pa_scheduler_kernel.o" \ + --raw "$RAW_DIR/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz" \ + --source-root "$SCRIPT_DIR" \ + --output "$SCRIPT_DIR/disassembly/key_flow/aic_compete_first.source.asm" \ + --anchor common/pa_scheduler_core.h:1382 --before 557 --after 163 +( + cd "$SCRIPT_DIR/disassembly" + find raw annotated key_flow -type f -print0 | sort -z | xargs -0 sha256sum +) > "$SCRIPT_DIR/disassembly/published.sha256" diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/functional.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/functional.txt new file mode 100644 index 0000000000..c5b5ee6b78 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/functional.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=1 tasks=5 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=192/192 tensor_args=2112/2112 scalar_args=864/864 resets=384/384 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=76.918 host_launch_us=1026.349 claims=288 fanin_loads=155 cas_retries=0 +[ATOMIC] submit_completion_ops=477 fanin_ready=36 fanin_not_ready=119 frontier_initial=5 frontier_flag=10 frontier_ready_fetch_max=5 frontier_terminal=5 +[WINNERS] active_workers=5 max_wins_per_worker=1 +[PLACEMENT] EfDrain=1 RingBp=0 FinalDrain=3 +[KERNEL] QK count=1 mean_us=42.349 min_us=42.349 max_us=42.349 target_us=44.170 +[KERNEL] SF count=1 mean_us=55.129 min_us=55.129 max_us=55.129 target_us=53.729 +[KERNEL] PV count=1 mean_us=27.824 min_us=27.824 max_us=27.824 target_us=27.626 +[KERNEL] UP count=1 mean_us=2.588 min_us=2.588 max_us=2.588 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=4 inactive_sentinel_tiles=188) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=76.918 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/RUN_METADATA.md b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/RUN_METADATA.md new file mode 100644 index 0000000000..7158473cd2 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/RUN_METADATA.md @@ -0,0 +1,17 @@ +# Final b256 run provenance + +- Variant: `compete-first`; +- collection interval: 2026-07-20 07:02:23–07:12:45 UTC; +- direct device0 execution, 96 workers, b256; +- 24 independent host launches and one device run per launch; +- all six A/B/C launch permutations repeated four times; launch positions 1/2/3 are 8/8/8; +- PMU explicitly off, runtime swimlane explicitly off, real-compute; +- executable source: this package's clean `build/compete-first`; +- exact final ELF copy: `artifacts/measured/pa_scheduler_kernel.o`; +- final ELF SHA256: `82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b`; +- host SHA256: `ee86b1e9457a190ba4594372853492997831f954953e1d037e34655fff212e70`. + +`task-submit` and `npu-smi` were unavailable in the collection shell. The run used the user's explicit +authorization to access device0 directly and was not queue-isolated. All raw logs passed execution, +semantic and postprocess checks. Raw evidence is retained; the primary statistic applies the common +per-variant Hampel rule documented in the root performance comparison. diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/SUMMARY.md b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/SUMMARY.md new file mode 100644 index 0000000000..1d541cb14e --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/SUMMARY.md @@ -0,0 +1,11 @@ +# compete-first device0 b256 result + +24 independent host launches, one device run per launch; PMU off, runtime swimlane off, real-compute. +Raw logs are retained. The primary row excludes samples outside median ± 3×1.4826×MAD. + +| View | N | Median (us) | Mean (us) | Min (us) | Max (us) | Stddev (us) | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Raw | 24 | 3744.417 | 3747.214 | 3625.727 | 3866.154 | 81.935 | +| Outlier-filtered | 24 | 3744.417 | 3747.214 | 3625.727 | 3866.154 | 81.935 | + +Excluded samples: 0. diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_01_ABC_pos2_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_01_ABC_pos2_compete-first.txt new file mode 100644 index 0000000000..0052c3c386 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_01_ABC_pos2_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3635.160 host_launch_us=4576.513 claims=73728 fanin_loads=27021 cas_retries=0 +[ATOMIC] submit_completion_ops=110825 fanin_ready=3474 fanin_not_ready=23547 frontier_initial=1280 frontier_flag=3246 frontier_ready_fetch_max=1966 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=995 RingBp=3 FinalDrain=26 +[KERNEL] QK count=256 mean_us=41.587 min_us=40.629 max_us=46.187 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.601 min_us=52.359 max_us=63.162 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.761 min_us=26.899 max_us=55.289 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.647 min_us=2.344 max_us=12.581 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3635.160 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_02_BCA_pos1_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_02_BCA_pos1_compete-first.txt new file mode 100644 index 0000000000..e930300e92 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_02_BCA_pos1_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3650.095 host_launch_us=4532.621 claims=73728 fanin_loads=27321 cas_retries=0 +[ATOMIC] submit_completion_ops=111119 fanin_ready=3452 fanin_not_ready=23869 frontier_initial=1280 frontier_flag=3243 frontier_ready_fetch_max=1963 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=32 +[PLACEMENT] EfDrain=997 RingBp=4 FinalDrain=23 +[KERNEL] QK count=256 mean_us=41.471 min_us=40.675 max_us=50.595 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.502 min_us=52.281 max_us=61.468 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.967 min_us=26.980 max_us=49.365 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.662 min_us=2.348 max_us=15.314 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=190 inactive_sentinel_tiles=2) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3650.095 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_03_CAB_pos3_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_03_CAB_pos3_compete-first.txt new file mode 100644 index 0000000000..0d660e15b3 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_03_CAB_pos3_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3652.166 host_launch_us=4566.144 claims=73728 fanin_loads=26983 cas_retries=0 +[ATOMIC] submit_completion_ops=110959 fanin_ready=3477 fanin_not_ready=23506 frontier_initial=1280 frontier_flag=3332 frontier_ready_fetch_max=2052 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=32 +[PLACEMENT] EfDrain=999 RingBp=3 FinalDrain=22 +[KERNEL] QK count=256 mean_us=41.465 min_us=40.511 max_us=49.301 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.462 min_us=52.250 max_us=66.399 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.006 min_us=27.008 max_us=49.683 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.699 min_us=2.405 max_us=18.406 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3652.166 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_04_ACB_pos3_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_04_ACB_pos3_compete-first.txt new file mode 100644 index 0000000000..6d6725207d --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_04_ACB_pos3_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3656.991 host_launch_us=4596.597 claims=73728 fanin_loads=26856 cas_retries=0 +[ATOMIC] submit_completion_ops=110546 fanin_ready=3441 fanin_not_ready=23415 frontier_initial=1280 frontier_flag=3189 frontier_ready_fetch_max=1909 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=997 RingBp=1 FinalDrain=26 +[KERNEL] QK count=256 mean_us=41.606 min_us=40.704 max_us=47.647 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.464 min_us=52.105 max_us=69.523 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.696 min_us=27.002 max_us=31.312 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.717 min_us=2.395 max_us=14.636 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3656.991 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_05_BAC_pos1_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_05_BAC_pos1_compete-first.txt new file mode 100644 index 0000000000..021a3b452b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_05_BAC_pos1_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3730.272 host_launch_us=4607.015 claims=73728 fanin_loads=26251 cas_retries=0 +[ATOMIC] submit_completion_ops=110041 fanin_ready=3422 fanin_not_ready=22829 frontier_initial=1280 frontier_flag=3239 frontier_ready_fetch_max=1959 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=33 +[PLACEMENT] EfDrain=1000 RingBp=0 FinalDrain=24 +[KERNEL] QK count=256 mean_us=41.481 min_us=40.626 max_us=46.201 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.796 min_us=52.181 max_us=147.170 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.970 min_us=27.047 max_us=35.999 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.678 min_us=2.403 max_us=15.484 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3730.272 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_06_CBA_pos2_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_06_CBA_pos2_compete-first.txt new file mode 100644 index 0000000000..809a265f3e --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_06_CBA_pos2_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3847.468 host_launch_us=4690.890 claims=73728 fanin_loads=26949 cas_retries=0 +[ATOMIC] submit_completion_ops=110385 fanin_ready=3427 fanin_not_ready=23522 frontier_initial=1280 frontier_flag=3062 frontier_ready_fetch_max=1782 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=1000 RingBp=0 FinalDrain=24 +[KERNEL] QK count=256 mean_us=41.496 min_us=40.652 max_us=46.597 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.604 min_us=52.242 max_us=255.816 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.107 min_us=27.041 max_us=83.029 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.659 min_us=2.380 max_us=20.594 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3847.468 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_07_CBA_pos2_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_07_CBA_pos2_compete-first.txt new file mode 100644 index 0000000000..f7c52df790 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_07_CBA_pos2_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3706.997 host_launch_us=4624.574 claims=73728 fanin_loads=26669 cas_retries=0 +[ATOMIC] submit_completion_ops=110667 fanin_ready=3472 fanin_not_ready=23197 frontier_initial=1280 frontier_flag=3343 frontier_ready_fetch_max=2063 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=34 +[PLACEMENT] EfDrain=998 RingBp=1 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.510 min_us=40.621 max_us=47.479 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.841 min_us=52.389 max_us=112.030 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.245 min_us=27.007 max_us=80.117 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.681 min_us=2.393 max_us=14.214 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3706.997 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_08_BAC_pos1_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_08_BAC_pos1_compete-first.txt new file mode 100644 index 0000000000..5bc2952769 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_08_BAC_pos1_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3837.104 host_launch_us=4777.883 claims=73728 fanin_loads=26623 cas_retries=0 +[ATOMIC] submit_completion_ops=110321 fanin_ready=3439 fanin_not_ready=23184 frontier_initial=1280 frontier_flag=3193 frontier_ready_fetch_max=1913 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=35 +[PLACEMENT] EfDrain=999 RingBp=0 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.514 min_us=40.573 max_us=47.023 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.176 min_us=52.419 max_us=254.556 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.264 min_us=26.964 max_us=79.784 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.621 min_us=2.401 max_us=14.458 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3837.104 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_09_ACB_pos3_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_09_ACB_pos3_compete-first.txt new file mode 100644 index 0000000000..70acc5a21b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_09_ACB_pos3_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3625.727 host_launch_us=4961.603 claims=73728 fanin_loads=26464 cas_retries=0 +[ATOMIC] submit_completion_ops=110128 fanin_ready=3476 fanin_not_ready=22988 frontier_initial=1280 frontier_flag=3176 frontier_ready_fetch_max=1896 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=34 +[PLACEMENT] EfDrain=997 RingBp=2 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.609 min_us=40.490 max_us=46.301 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.491 min_us=52.318 max_us=59.553 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.074 min_us=26.985 max_us=58.807 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.625 min_us=2.397 max_us=15.334 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3625.727 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_10_CAB_pos3_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_10_CAB_pos3_compete-first.txt new file mode 100644 index 0000000000..18602b60df --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_10_CAB_pos3_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3823.817 host_launch_us=5051.993 claims=73728 fanin_loads=27153 cas_retries=0 +[ATOMIC] submit_completion_ops=110959 fanin_ready=3475 fanin_not_ready=23678 frontier_initial=1280 frontier_flag=3247 frontier_ready_fetch_max=1967 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=999 RingBp=1 FinalDrain=24 +[KERNEL] QK count=256 mean_us=41.438 min_us=40.483 max_us=46.127 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.739 min_us=52.352 max_us=238.224 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.178 min_us=27.037 max_us=74.863 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.676 min_us=2.374 max_us=15.836 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3823.817 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_11_BCA_pos1_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_11_BCA_pos1_compete-first.txt new file mode 100644 index 0000000000..4f9ee106ae --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_11_BCA_pos1_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3679.901 host_launch_us=4577.907 claims=73728 fanin_loads=27047 cas_retries=0 +[ATOMIC] submit_completion_ops=110547 fanin_ready=3445 fanin_not_ready=23602 frontier_initial=1280 frontier_flag=3094 frontier_ready_fetch_max=1814 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=28 +[PLACEMENT] EfDrain=997 RingBp=1 FinalDrain=26 +[KERNEL] QK count=256 mean_us=41.584 min_us=40.598 max_us=47.036 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.567 min_us=52.208 max_us=95.801 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.035 min_us=27.035 max_us=77.215 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.600 min_us=2.393 max_us=12.444 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3679.901 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_12_ABC_pos2_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_12_ABC_pos2_compete-first.txt new file mode 100644 index 0000000000..10a08836dd --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_12_ABC_pos2_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3637.016 host_launch_us=4620.597 claims=73728 fanin_loads=26654 cas_retries=0 +[ATOMIC] submit_completion_ops=110720 fanin_ready=3518 fanin_not_ready=23136 frontier_initial=1280 frontier_flag=3377 frontier_ready_fetch_max=2097 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=28 +[PLACEMENT] EfDrain=998 RingBp=1 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.583 min_us=40.636 max_us=49.219 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.303 min_us=52.254 max_us=56.887 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.132 min_us=26.945 max_us=59.801 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.630 min_us=2.353 max_us=15.279 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3637.016 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_13_BCA_pos1_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_13_BCA_pos1_compete-first.txt new file mode 100644 index 0000000000..014276e9b1 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_13_BCA_pos1_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3753.752 host_launch_us=4761.040 claims=73728 fanin_loads=27031 cas_retries=0 +[ATOMIC] submit_completion_ops=111257 fanin_ready=3469 fanin_not_ready=23562 frontier_initial=1280 frontier_flag=3457 frontier_ready_fetch_max=2177 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=33 +[PLACEMENT] EfDrain=996 RingBp=3 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.508 min_us=40.523 max_us=46.209 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.066 min_us=52.309 max_us=160.883 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.085 min_us=26.961 max_us=59.068 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.710 min_us=2.400 max_us=22.206 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3753.752 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_14_ACB_pos3_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_14_ACB_pos3_compete-first.txt new file mode 100644 index 0000000000..8e18ff9bb7 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_14_ACB_pos3_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3866.154 host_launch_us=4786.660 claims=73728 fanin_loads=27060 cas_retries=0 +[ATOMIC] submit_completion_ops=110748 fanin_ready=3447 fanin_not_ready=23613 frontier_initial=1280 frontier_flag=3188 frontier_ready_fetch_max=1908 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=998 RingBp=1 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.404 min_us=40.520 max_us=45.796 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.271 min_us=52.215 max_us=266.041 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.010 min_us=26.972 max_us=58.930 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.678 min_us=2.408 max_us=14.576 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3866.154 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_15_CBA_pos2_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_15_CBA_pos2_compete-first.txt new file mode 100644 index 0000000000..c7d0f7fe6c --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_15_CBA_pos2_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3766.289 host_launch_us=4678.773 claims=73728 fanin_loads=26932 cas_retries=0 +[ATOMIC] submit_completion_ops=110970 fanin_ready=3489 fanin_not_ready=23443 frontier_initial=1280 frontier_flag=3363 frontier_ready_fetch_max=2083 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=998 RingBp=3 FinalDrain=23 +[KERNEL] QK count=256 mean_us=41.503 min_us=40.677 max_us=45.946 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.882 min_us=52.371 max_us=160.846 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.128 min_us=27.025 max_us=60.510 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.626 min_us=2.345 max_us=14.670 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=189 inactive_sentinel_tiles=3) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3766.289 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_16_ABC_pos2_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_16_ABC_pos2_compete-first.txt new file mode 100644 index 0000000000..c0008eed17 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_16_ABC_pos2_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3834.848 host_launch_us=4668.863 claims=73728 fanin_loads=27303 cas_retries=0 +[ATOMIC] submit_completion_ops=110993 fanin_ready=3436 fanin_not_ready=23867 frontier_initial=1280 frontier_flag=3189 frontier_ready_fetch_max=1909 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=32 +[PLACEMENT] EfDrain=997 RingBp=2 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.508 min_us=40.615 max_us=49.788 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.145 min_us=52.201 max_us=236.561 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.979 min_us=27.020 max_us=56.929 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.738 min_us=2.409 max_us=21.487 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3834.848 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_17_CAB_pos3_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_17_CAB_pos3_compete-first.txt new file mode 100644 index 0000000000..0373d964b2 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_17_CAB_pos3_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3649.524 host_launch_us=4638.995 claims=73728 fanin_loads=26777 cas_retries=0 +[ATOMIC] submit_completion_ops=110583 fanin_ready=3464 fanin_not_ready=23313 frontier_initial=1280 frontier_flag=3247 frontier_ready_fetch_max=1967 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=32 +[PLACEMENT] EfDrain=1000 RingBp=1 FinalDrain=23 +[KERNEL] QK count=256 mean_us=41.447 min_us=40.492 max_us=46.646 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.414 min_us=52.351 max_us=69.083 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.013 min_us=27.008 max_us=63.387 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.676 min_us=2.415 max_us=12.878 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3649.524 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_18_BAC_pos1_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_18_BAC_pos1_compete-first.txt new file mode 100644 index 0000000000..1e8513ac86 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_18_BAC_pos1_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3848.497 host_launch_us=4732.380 claims=73728 fanin_loads=27019 cas_retries=0 +[ATOMIC] submit_completion_ops=110701 fanin_ready=3444 fanin_not_ready=23575 frontier_initial=1280 frontier_flag=3185 frontier_ready_fetch_max=1905 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=29 +[PLACEMENT] EfDrain=997 RingBp=2 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.380 min_us=40.655 max_us=48.749 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.587 min_us=52.299 max_us=261.138 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.346 min_us=26.994 max_us=71.872 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.685 min_us=2.397 max_us=25.651 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3848.497 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_19_BAC_pos1_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_19_BAC_pos1_compete-first.txt new file mode 100644 index 0000000000..b568326090 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_19_BAC_pos1_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3807.280 host_launch_us=4697.900 claims=73728 fanin_loads=26759 cas_retries=0 +[ATOMIC] submit_completion_ops=110679 fanin_ready=3487 fanin_not_ready=23272 frontier_initial=1280 frontier_flag=3304 frontier_ready_fetch_max=2024 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=998 RingBp=1 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.684 min_us=40.608 max_us=51.548 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.634 min_us=52.245 max_us=226.020 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.248 min_us=27.023 max_us=82.153 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.605 min_us=2.343 max_us=15.913 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3807.280 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_20_CAB_pos3_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_20_CAB_pos3_compete-first.txt new file mode 100644 index 0000000000..60fad99a86 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_20_CAB_pos3_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3836.932 host_launch_us=4753.030 claims=73728 fanin_loads=26659 cas_retries=0 +[ATOMIC] submit_completion_ops=110513 fanin_ready=3498 fanin_not_ready=23161 frontier_initial=1280 frontier_flag=3271 frontier_ready_fetch_max=1991 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=32 +[PLACEMENT] EfDrain=999 RingBp=2 FinalDrain=23 +[KERNEL] QK count=256 mean_us=41.449 min_us=40.447 max_us=46.037 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.568 min_us=52.214 max_us=246.828 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.479 min_us=26.989 max_us=81.008 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.593 min_us=2.358 max_us=5.601 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3836.932 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_21_ABC_pos2_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_21_ABC_pos2_compete-first.txt new file mode 100644 index 0000000000..369500ed59 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_21_ABC_pos2_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3734.900 host_launch_us=5034.938 claims=73728 fanin_loads=26321 cas_retries=0 +[ATOMIC] submit_completion_ops=112057 fanin_ready=3580 fanin_not_ready=22741 frontier_initial=1280 frontier_flag=4212 frontier_ready_fetch_max=2932 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=34 +[PLACEMENT] EfDrain=992 RingBp=8 FinalDrain=24 +[KERNEL] QK count=256 mean_us=41.453 min_us=40.608 max_us=47.788 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.669 min_us=52.375 max_us=129.333 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.193 min_us=26.989 max_us=79.240 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.622 min_us=2.396 max_us=15.503 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=190 inactive_sentinel_tiles=2) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3734.900 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_22_CBA_pos2_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_22_CBA_pos2_compete-first.txt new file mode 100644 index 0000000000..8354367f64 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_22_CBA_pos2_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3769.223 host_launch_us=5905.797 claims=73728 fanin_loads=26462 cas_retries=0 +[ATOMIC] submit_completion_ops=110598 fanin_ready=3429 fanin_not_ready=23033 frontier_initial=1280 frontier_flag=3412 frontier_ready_fetch_max=2132 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=1000 RingBp=0 FinalDrain=24 +[KERNEL] QK count=256 mean_us=41.519 min_us=40.615 max_us=49.219 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.164 min_us=52.277 max_us=182.966 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.351 min_us=27.001 max_us=79.878 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.584 min_us=2.402 max_us=3.434 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3769.223 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_23_ACB_pos3_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_23_ACB_pos3_compete-first.txt new file mode 100644 index 0000000000..0881d3ca3e --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_23_ACB_pos3_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3847.947 host_launch_us=4711.295 claims=73728 fanin_loads=27434 cas_retries=0 +[ATOMIC] submit_completion_ops=111382 fanin_ready=3481 fanin_not_ready=23953 frontier_initial=1280 frontier_flag=3318 frontier_ready_fetch_max=2038 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=996 RingBp=3 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.461 min_us=40.643 max_us=48.205 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.217 min_us=52.306 max_us=249.729 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.092 min_us=26.934 max_us=85.597 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.582 min_us=2.378 max_us=3.720 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3847.947 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_24_BCA_pos1_compete-first.txt b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_24_BCA_pos1_compete-first.txt new file mode 100644 index 0000000000..d101624abf --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/raw/block_24_BCA_pos1_compete-first.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first lazy_sample_shape_id=1 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first views=49152/49152 tensor_args=540672/540672 scalar_args=221184/221184 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3735.082 host_launch_us=4666.874 claims=73728 fanin_loads=27255 cas_retries=0 +[ATOMIC] submit_completion_ops=111145 fanin_ready=3452 fanin_not_ready=23803 frontier_initial=1280 frontier_flag=3289 frontier_ready_fetch_max=2009 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=34 +[PLACEMENT] EfDrain=994 RingBp=5 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.349 min_us=40.643 max_us=46.969 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.903 min_us=52.355 max_us=151.231 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.035 min_us=26.988 max_us=77.168 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.708 min_us=2.408 max_us=14.876 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3735.082 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/samples.tsv b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/samples.tsv new file mode 100644 index 0000000000..7f7f95500a --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/samples.tsv @@ -0,0 +1,25 @@ +variant block order position submit_span_us included exclusion_reason source +compete-first 1 ABC 2 3635.160 True block_01_ABC_pos2_compete-first.txt +compete-first 2 BCA 1 3650.095 True block_02_BCA_pos1_compete-first.txt +compete-first 3 CAB 3 3652.166 True block_03_CAB_pos3_compete-first.txt +compete-first 4 ACB 3 3656.991 True block_04_ACB_pos3_compete-first.txt +compete-first 5 BAC 1 3730.272 True block_05_BAC_pos1_compete-first.txt +compete-first 6 CBA 2 3847.468 True block_06_CBA_pos2_compete-first.txt +compete-first 7 CBA 2 3706.997 True block_07_CBA_pos2_compete-first.txt +compete-first 8 BAC 1 3837.104 True block_08_BAC_pos1_compete-first.txt +compete-first 9 ACB 3 3625.727 True block_09_ACB_pos3_compete-first.txt +compete-first 10 CAB 3 3823.817 True block_10_CAB_pos3_compete-first.txt +compete-first 11 BCA 1 3679.901 True block_11_BCA_pos1_compete-first.txt +compete-first 12 ABC 2 3637.016 True block_12_ABC_pos2_compete-first.txt +compete-first 13 BCA 1 3753.752 True block_13_BCA_pos1_compete-first.txt +compete-first 14 ACB 3 3866.154 True block_14_ACB_pos3_compete-first.txt +compete-first 15 CBA 2 3766.289 True block_15_CBA_pos2_compete-first.txt +compete-first 16 ABC 2 3834.848 True block_16_ABC_pos2_compete-first.txt +compete-first 17 CAB 3 3649.524 True block_17_CAB_pos3_compete-first.txt +compete-first 18 BAC 1 3848.497 True block_18_BAC_pos1_compete-first.txt +compete-first 19 BAC 1 3807.280 True block_19_BAC_pos1_compete-first.txt +compete-first 20 CAB 3 3836.932 True block_20_CAB_pos3_compete-first.txt +compete-first 21 ABC 2 3734.900 True block_21_ABC_pos2_compete-first.txt +compete-first 22 CBA 2 3769.223 True block_22_CBA_pos2_compete-first.txt +compete-first 23 ACB 3 3847.947 True block_23_ACB_pos3_compete-first.txt +compete-first 24 BCA 1 3735.082 True block_24_BCA_pos1_compete-first.txt diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/summary.json b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/summary.json new file mode 100644 index 0000000000..9a2d433030 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/performance/summary.json @@ -0,0 +1,45 @@ +{ + "configuration": { + "batches": 256, + "device": 0, + "launch_orders": "all six permutations, each repeated four times", + "launch_positions": { + "1": 8, + "2": 8, + "3": 8 + }, + "pmu_window": "off", + "runs_per_host_launch": 1, + "samples": 24, + "swimlane": "off-runtime", + "winner_workload": "real-compute", + "workers": 96 + }, + "excluded_samples": [], + "outlier_filtered": { + "count": 24, + "max_submit_span_us": 3866.154, + "mean_submit_span_us": 3747.2142499999995, + "median_submit_span_us": 3744.417, + "min_submit_span_us": 3625.727, + "population_stdev_us": 81.93544161220773 + }, + "outlier_rule": { + "lower_us": 3338.1505002000004, + "mad_us": 91.3409999999999, + "median_us": 3744.417, + "name": "Hampel", + "raw_logs_retained": true, + "upper_us": 4150.6834997999995 + }, + "raw": { + "count": 24, + "max_submit_span_us": 3866.154, + "mean_submit_span_us": 3747.2142499999995, + "median_submit_span_us": 3744.417, + "min_submit_span_us": 3625.727, + "population_stdev_us": 81.93544161220773 + }, + "schema": "pa_lazy_lambda_variant_independent_performance/v1", + "variant": "compete-first" +} diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/published.sha256 new file mode 100644 index 0000000000..a5c7368432 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/output/published.sha256 @@ -0,0 +1,29 @@ +dff00fa736fa9bddfa8f311935a9b678ab148616b62bd6f6774f0e43b257a0ab output/functional.txt +aacbdf1712d3cc9470fdcddc96702a940346a88e3d0f0e3bc748c2715bcfa811 output/performance/RUN_METADATA.md +4afe4aafa2d88ecff92ce9894edfbbdb12fd38ed48e0ef789d06e02b7e031e3c output/performance/SUMMARY.md +bdada9610d2585615d1fb42e75436110b1df3f5ad6a19b6a9e32a4a953e597b3 output/performance/raw/block_01_ABC_pos2_compete-first.txt +3774d404a7377b2780b4d9c43ef42dd3dda83e4b88f0cf0c05a5384786672c79 output/performance/raw/block_02_BCA_pos1_compete-first.txt +36aa6a4bf0489792150148e980138f99cd9e6e5af767fa58b50b3a3424d153c8 output/performance/raw/block_03_CAB_pos3_compete-first.txt +135da9673048751a5ebfda521179c461b63eeda92c5db800a7e047f1eba215f7 output/performance/raw/block_04_ACB_pos3_compete-first.txt +4e48747c67199fbac808da2e7884e6b198393b50b16e99cfd4035918f6924320 output/performance/raw/block_05_BAC_pos1_compete-first.txt +f8de8efc390cbe221e5a0614cbdc3c4cf4c8bfa828c7e72a043baa5052e81e96 output/performance/raw/block_06_CBA_pos2_compete-first.txt +3b0d321edffbb8c0bfd2d871b054eef12a26d67ef43a003785fa7dadad66c579 output/performance/raw/block_07_CBA_pos2_compete-first.txt +71a5006347c1e8552ec70122b9e13db6a7c40ca2ca864824e48abe2a84393809 output/performance/raw/block_08_BAC_pos1_compete-first.txt +4ca7114686e131a44933f7f797b1d400a56f1d38c98e06931b7250481bbc5753 output/performance/raw/block_09_ACB_pos3_compete-first.txt +fb57fc79164745b772db6250071e8e182a26cb45fbca4885dde95123457ec4a7 output/performance/raw/block_10_CAB_pos3_compete-first.txt +bf4fc5a32d595fa3b1c5162697a83899d3af15086614b7edb253f48bd4c62644 output/performance/raw/block_11_BCA_pos1_compete-first.txt +5e3bf10747c8f311308b95247b580806555ea63836e470a5006962a54a25739b output/performance/raw/block_12_ABC_pos2_compete-first.txt +75103bfe53e8fa80418183586b32c20d7999b9b22a088a918fbb529ceff71bd6 output/performance/raw/block_13_BCA_pos1_compete-first.txt +ca04f2ed8a4779f32502ee36152a37a56339b199f71e728de9127ded34cab602 output/performance/raw/block_14_ACB_pos3_compete-first.txt +31f7d1cda1a8cff22c9dc169582fcf988ae4f51a505d874c117a8c789d24d0b0 output/performance/raw/block_15_CBA_pos2_compete-first.txt +fd560782ace04f756d1f6d7fb31d5e770e1a91d136e3bb385bfa2fe2021b56bc output/performance/raw/block_16_ABC_pos2_compete-first.txt +5f946a71e527199c82e751ec658b6224641908084fe5be63fc237479caf939ea output/performance/raw/block_17_CAB_pos3_compete-first.txt +d2a0ead197fd686eb45c40ae975b4ba1fc4b90983bdfde9f51b9884a1bf6a586 output/performance/raw/block_18_BAC_pos1_compete-first.txt +8df6439b1216a7d0a548cb1167c23ab269cd2465d0c90083c3440c2075f4c27f output/performance/raw/block_19_BAC_pos1_compete-first.txt +d37d9827a7c7a2cc3083ee0621233d069b96b2c248cf0cbf613114fe43872456 output/performance/raw/block_20_CAB_pos3_compete-first.txt +b46fe72351e897f746d197f6d63dab7d50271f720df6dced125841424f519678 output/performance/raw/block_21_ABC_pos2_compete-first.txt +9308547bf6c947fead642cbaefb47afb3a00f328a57af8b26e7ff8192fa1d1a7 output/performance/raw/block_22_CBA_pos2_compete-first.txt +0847887bdbc0212fa6531ee1d9fd7dac293722fe92174ed25aa041db12a98b51 output/performance/raw/block_23_ACB_pos3_compete-first.txt +fe9642e9f007641d1b68ea1f1bf4f120e785c2dd423455679f6d3e73c192f1f4 output/performance/raw/block_24_BCA_pos1_compete-first.txt +4b561fa972c22d8ae2f6663e805132440451a613785943b856e88406a6b86613 output/performance/samples.tsv +4cea5fc4dfcb815e6841d050d2024a61b224f3dc9f76d816da25464d99e56e29 output/performance/summary.json diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/publish_artifacts.sh b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/publish_artifacts.sh new file mode 100755 index 0000000000..853d3b0ed1 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/publish_artifacts.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SOURCE_DIR="$SCRIPT_DIR/build/compete-first" +TARGET_DIR="$SCRIPT_DIR/artifacts/rebuilt" +mkdir -p "$TARGET_DIR" + +for name in pa_scheduler_host pa_scheduler_kernel.o device_text_layout.manifest artifacts.manifest; do + if [[ ! -f "$SOURCE_DIR/$name" ]]; then + echo "Missing clean-build result: $SOURCE_DIR/$name" >&2 + exit 1 + fi + cp -f -- "$SOURCE_DIR/$name" "$TARGET_DIR/$name" +done +(cd "$TARGET_DIR" && sha256sum artifacts.manifest device_text_layout.manifest pa_scheduler_host pa_scheduler_kernel.o) \ + > "$TARGET_DIR/published.sha256" diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/run_functional.sh b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/run_functional.sh new file mode 100755 index 0000000000..40b1b7b983 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/run_functional.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BUILD_DIR="$SCRIPT_DIR/build/compete-first" +OUTPUT="$SCRIPT_DIR/output/functional.txt" + +if [[ ! -x "$BUILD_DIR/pa_scheduler_host" || ! -f "$BUILD_DIR/pa_scheduler_kernel.o" ]]; then + echo "Missing clean build; run ./build.sh first." >&2 + exit 1 +fi +"$BUILD_DIR/pa_scheduler_host" \ + --kernel "$BUILD_DIR/pa_scheduler_kernel.o" \ + --device 0 --batches 1 --runs 1 \ + --winner-workload real-compute --pmu-window off --no-swimlane | tee "$OUTPUT" +if ! grep -Eq '^\[SUMMARY\].*completed_runs=1.*execution_status=PASS semantic_status=PASS postprocess_status=PASS$' "$OUTPUT"; then + echo "Functional oracle failed for compete-first" >&2 + exit 1 +fi +if ! grep -Fq '[LAZY_SAMPLE_FRONTEND] shape=compete-first views=192/192 tensor_args=2112/2112 scalar_args=864/864 resets=384/384' "$OUTPUT"; then + echo "Eager frontend-count oracle failed for compete-first" >&2 + exit 1 +fi diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/source.sha256 b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/source.sha256 new file mode 100644 index 0000000000..964577c110 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/source.sha256 @@ -0,0 +1,30 @@ +909eff5ca6e3f88d5b077945077864a1557549a3e171bcf233aa236dbd96d660 ./annotate_disassembly.py +cc9948d9ec75d8402345238b4ec986b82493765bda6114289b997789f2cf3df6 ./build.sh +4091251537a0405f83514d99fc6891300650a01cb61ac3b6c2291e899f7f0747 ./disassemble.py +ac61de825e41468de26a785ef55b3f9228386dfdd729e1b375d043e00f0b8fc5 ./generate_disassembly.sh +d38bd2f74659b7b7cdaa392cde321f112ac31ae699942bd2d4a901f5ecf5cbb0 ./publish_artifacts.sh +e3a218ad05fdbf7bcda1cab32fbfe616efc31281cef588a0b91f465dd96fddf3 ./run_functional.sh +72471b0594aee1069f9fc774faa6d5900e374d3a283fdccc12b2c26a1302472a ./summarize_performance.py +e50378f7afdb15e3a84207b029e2d0aced10e1ac5a363372a791a0a8d177c937 ./verify_runtime_identity.py +a1aced021c2f2cc7bd40a736d602f8ebc17045a30969366d869e76d0462782af README.md +a8685790bf5928a1e7e719760aa87383e52e269d85d6005804647e899798dd2e VARIANT +061dfa1a8baa9b08d6360d3cfb899a17d58b0cb001421f8a0de387cfea53a5e0 ccec/build.sh +ea82967763894d2726b68b19e74c2993ffe938f5bb0c90b92d3eae0966e643b7 ccec/callback_finish.cpp +7090c8405e1602476d1edf4ae6103ec6ec145e22bf15db742eac06aa2a6b7f9f ccec/callback_finish_api.h +4cef27650943d3973c6d56db687850ea3a7cbe28d64bf666b55cf735ce5bd141 ccec/callback_runtime_entry.cpp +47ca7623791bdd408b50c5e5ece4d63c7073775a1e3b636e046419acf948ddda ccec/ccec_ops.h +ac8a9766aa71a2bfc2a21c9021fc9fc30bc869575daa0e0f01121789230173e2 ccec/host.cpp +766ec34d6d4a7d80d07774782c68d701653cadf1bd8427fd1a0e922933c10236 ccec/kernel.cpp +f88d5ed31277c2692ecec2e8f5528cb6c2de752aea6a45850ab72d9e9b600075 ccec/pa_scheduler_device_exports.map +3d65b4a89dcf88554321a5435a2abf701a26757e8c229df7ac9783abbb5b96c3 ccec/pmu_owner_control.h +cb81c2a24670d893fd485845f13b8b106bfffe28d8c471f92a93775eee1a69e6 ccec/pmu_owner_host.h +09263e6698598f7be5b16b92e9e34bc59719de54aee132c43df24cec0ce42b4a ccec/pmu_owner_main_abi.h +9115d77d1be306c9f3b225805ed3df63fbdbdadcb9b4ff6ad690b56cfcf58867 ccec/pmu_owner_main_loader.h +2a926b355f78d4758fdc61a54827baec54ef781df34234a2edc9aa6a1a824f9e ccec/pmu_probe.h +3e3c6c99df27ff9e329242c64680559bf69cb4dc96650acc0ba1fe7442d68c70 common/host_support.h +f4a22840aef81ec05fb9532559957712c76ebd2dd1199d6969fc99688fc2e63d common/pa_frontend.h +9d42688f933e7b20eaab9eadcec40c3f0c0babc8ebdd3dd7369a2afcca1edbec common/pa_model.h +6645ce2aefe39b6132ce649c6af4c597a7a6978aa8983375fe09c4e186a812e5 common/pa_scheduler_core.h +8fd7c429dbbf0aae4e37968e7f90d953793f30f9caeb18eba515f9201deca1fa common/pa_trace.h +9d099151d0303c37dbb88f5cd2b15ee483dc0623759c45cbcf9d80c37704a80d common/winner_workload.h +8be384a62ed669e5645139aab1528570ffab184b5856f685e8056eed68e1e5cc common/winner_workload_host.h diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/summarize_performance.py b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/summarize_performance.py new file mode 100755 index 0000000000..08a98ae100 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/summarize_performance.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Validate this standalone package's 24 independent b256 measurements.""" + +from __future__ import annotations + +import json +import re +import statistics +from pathlib import Path + + +FILE_RE = re.compile( + r"block_(?P\d{2})_(?P[ABC]{3})_pos(?P[123])_" + r"(?Poriginal|compete-first|compete-first-lazy)\.txt$" +) +CONFIG_RE = re.compile( + r"^device=0 batches=256 tasks=1280 workers=96 runs=1 .*\bswimlane=off\b", + re.MULTILINE, +) +METRIC_RE = re.compile(r"^\[METRIC\] run=1 submit_span_us=([0-9.]+)\b", re.MULTILINE) +SUMMARY_RE = re.compile( + r"^\[SUMMARY\] runs=1 completed_runs=1 median_submit_span_us=([0-9.]+) " + r"execution_status=PASS semantic_status=PASS postprocess_status=PASS$", + re.MULTILINE, +) +HAMPEL_SCALE = 1.4826 +HAMPEL_SIGMAS = 3.0 + + +def fail(message: str) -> None: + raise SystemExit(message) + + +def describe(values: list[float]) -> dict[str, float | int]: + return { + "count": len(values), + "median_submit_span_us": statistics.median(values), + "mean_submit_span_us": statistics.fmean(values), + "min_submit_span_us": min(values), + "max_submit_span_us": max(values), + "population_stdev_us": statistics.pstdev(values), + } + + +def main() -> None: + script_dir = Path(__file__).resolve().parent + variant = (script_dir / "VARIANT").read_text(encoding="utf-8").strip() + result_dir = script_dir / "output" / "performance" + paths = sorted((result_dir / "raw").glob("*.txt")) + if len(paths) != 24: + fail(f"{variant}: expected 24 independent raw files, found {len(paths)}") + + rows: list[dict[str, object]] = [] + for path in paths: + name_match = FILE_RE.fullmatch(path.name) + if name_match is None or name_match.group("variant") != variant: + fail(f"{variant}: unexpected raw filename {path.name}") + text = path.read_text(encoding="utf-8") + if CONFIG_RE.search(text) is None: + fail(f"{path}: fixed device0/b256/runs=1/no-swimlane config missing") + if "[WINNER-WORKLOAD] mode=real-compute " not in text: + fail(f"{path}: real-compute workload missing") + if "[PMU-CONFIG] window=off " not in text: + fail(f"{path}: PMU is not explicitly off") + metrics = METRIC_RE.findall(text) + summaries = SUMMARY_RE.findall(text) + if len(metrics) != 1 or len(summaries) != 1 or metrics[0] != summaries[0]: + fail(f"{path}: expected one matching METRIC and PASS SUMMARY") + rows.append( + { + "block": int(name_match.group("block")), + "order": name_match.group("order"), + "position": int(name_match.group("position")), + "submit_span_us": float(metrics[0]), + "source": path.name, + } + ) + + if {int(row["block"]) for row in rows} != set(range(1, 25)): + fail(f"{variant}: blocks must be exactly 01..24") + order_counts = { + order: sum(row["order"] == order for row in rows) + for order in ("ABC", "ACB", "BAC", "BCA", "CAB", "CBA") + } + if set(order_counts.values()) != {4}: + fail(f"{variant}: six orders are not each repeated four times: {order_counts}") + position_counts = { + position: sum(row["position"] == position for row in rows) + for position in (1, 2, 3) + } + if set(position_counts.values()) != {8}: + fail(f"{variant}: launch positions are not 8/8/8: {position_counts}") + + raw_values = [float(row["submit_span_us"]) for row in rows] + center = statistics.median(raw_values) + mad = statistics.median(abs(value - center) for value in raw_values) + if mad == 0.0: + fail(f"{variant}: zero MAD cannot define an outlier threshold") + lower = center - HAMPEL_SIGMAS * HAMPEL_SCALE * mad + upper = center + HAMPEL_SIGMAS * HAMPEL_SCALE * mad + for row in rows: + value = float(row["submit_span_us"]) + row["included"] = lower <= value <= upper + row["exclusion_reason"] = "" if row["included"] else "hampel_3_scaled_mad" + clean_values = [ + float(row["submit_span_us"]) for row in rows if bool(row["included"]) + ] + + document = { + "schema": "pa_lazy_lambda_variant_independent_performance/v1", + "variant": variant, + "configuration": { + "device": 0, + "workers": 96, + "batches": 256, + "samples": 24, + "runs_per_host_launch": 1, + "launch_orders": "all six permutations, each repeated four times", + "launch_positions": {"1": 8, "2": 8, "3": 8}, + "pmu_window": "off", + "swimlane": "off-runtime", + "winner_workload": "real-compute", + }, + "outlier_rule": { + "name": "Hampel", + "median_us": center, + "mad_us": mad, + "lower_us": lower, + "upper_us": upper, + "raw_logs_retained": True, + }, + "raw": describe(raw_values), + "outlier_filtered": describe(clean_values), + "excluded_samples": [row for row in rows if not bool(row["included"])], + } + (result_dir / "summary.json").write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + with (result_dir / "samples.tsv").open("w", encoding="utf-8") as output: + output.write( + "variant\tblock\torder\tposition\tsubmit_span_us\tincluded\t" + "exclusion_reason\tsource\n" + ) + for row in sorted(rows, key=lambda item: int(item["block"])): + output.write( + f"{variant}\t{row['block']}\t{row['order']}\t{row['position']}\t" + f"{float(row['submit_span_us']):.3f}\t{row['included']}\t" + f"{row['exclusion_reason']}\t{row['source']}\n" + ) + raw = document["raw"] + clean = document["outlier_filtered"] + lines = [ + f"# {variant} device0 b256 result", + "", + "24 independent host launches, one device run per launch; PMU off, runtime swimlane off, real-compute.", + "Raw logs are retained. The primary row excludes samples outside median ± 3×1.4826×MAD.", + "", + "| View | N | Median (us) | Mean (us) | Min (us) | Max (us) | Stddev (us) |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + f"| Raw | {raw['count']} | {raw['median_submit_span_us']:.3f} | " + f"{raw['mean_submit_span_us']:.3f} | {raw['min_submit_span_us']:.3f} | " + f"{raw['max_submit_span_us']:.3f} | {raw['population_stdev_us']:.3f} |", + f"| Outlier-filtered | {clean['count']} | {clean['median_submit_span_us']:.3f} | " + f"{clean['mean_submit_span_us']:.3f} | {clean['min_submit_span_us']:.3f} | " + f"{clean['max_submit_span_us']:.3f} | {clean['population_stdev_us']:.3f} |", + "", + f"Excluded samples: {raw['count'] - clean['count']}.", + ] + (result_dir / "SUMMARY.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + print("\n".join(lines)) + + +if __name__ == "__main__": + main() diff --git a/tests/atomic_probe/lazy_lamda_sample/B_compete_first/verify_runtime_identity.py b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/verify_runtime_identity.py new file mode 100755 index 0000000000..786fcc487f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/B_compete_first/verify_runtime_identity.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Prove that measured and local-rebuild AICore ELFs have identical runtime content.""" + +from __future__ import annotations + +import hashlib +import json +import struct +from dataclasses import dataclass +from pathlib import Path + + +SHT_NOBITS = 8 +SHT_SYMTAB = 2 +STT_FUNC = 2 +STT_OBJECT = 1 + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def c_string(table: bytes, offset: int) -> str: + end = table.find(b"\0", offset) + if end < 0: + raise ValueError("unterminated ELF string") + return table[offset:end].decode("utf-8") + + +@dataclass(frozen=True) +class Section: + index: int + name: str + section_type: int + flags: int + address: int + offset: int + size: int + link: int + alignment: int + entry_size: int + + +class Elf64: + def __init__(self, path: Path): + self.path = path + self.data = path.read_bytes() + header = struct.unpack_from("<16sHHIQQQIHHHHHH", self.data, 0) + if header[0][:6] != b"\x7fELF\x02\x01": + raise ValueError(f"not ELF64 little-endian: {path}") + section_offset, section_size, count, names_index = header[6], header[11], header[12], header[13] + if section_size != 64: + raise ValueError(f"unexpected section header size: {section_size}") + raw = [ + struct.unpack_from(" bytes: + if section.section_type == SHT_NOBITS: + return b"" + return self.data[section.offset : section.offset + section.size] + + def runtime_symbols(self) -> dict[str, tuple[int, int, int, int]]: + symtab = next(section for section in self.sections if section.section_type == SHT_SYMTAB) + strings = self.content(self.sections[symtab.link]) + symbols: dict[str, tuple[int, int, int, int]] = {} + for offset in range(symtab.offset, symtab.offset + symtab.size, symtab.entry_size): + name_offset, info, _, section_index, value, size = struct.unpack_from( + "= len(self.sections): + continue + section = self.sections[section_index] + if section.name not in {".text", ".rodata", ".bl_uninit"}: + continue + name = c_string(strings, name_offset) + if name: + symbols[name] = (symbol_type, info >> 4, value, size) + return symbols + + +def main() -> None: + script_dir = Path(__file__).resolve().parent + measured_dir = script_dir / "artifacts" / "measured" + rebuilt_dir = script_dir / "artifacts" / "rebuilt" + measured = Elf64(measured_dir / "pa_scheduler_kernel.o") + rebuilt = Elf64(rebuilt_dir / "pa_scheduler_kernel.o") + + content_sections = [ + ".text", + ".rodata", + ".ascend.meta.pa_scheduler_0_mix_aic", + "__CCE_KernelArgSize", + ".ascend.meta.pa_scheduler_0_mix_aiv", + ] + section_rows = [] + all_ok = True + for name in content_sections: + left = measured.by_name[name] + right = rebuilt.by_name[name] + metadata_equal = ( + left.section_type, left.flags, left.address, left.size, left.alignment + ) == ( + right.section_type, right.flags, right.address, right.size, right.alignment + ) + content_equal = measured.content(left) == rebuilt.content(right) + all_ok &= metadata_equal and content_equal + section_rows.append( + { + "name": name, + "size": left.size, + "sha256": digest(measured.content(left)), + "metadata_equal": metadata_equal, + "content_equal": content_equal, + } + ) + + nobits_rows = [] + for name in [".bl_uninit"]: + if name not in measured.by_name and name not in rebuilt.by_name: + continue + if name not in measured.by_name or name not in rebuilt.by_name: + all_ok = False + nobits_rows.append({"name": name, "layout_equal": False}) + continue + left = measured.by_name[name] + right = rebuilt.by_name[name] + layout_equal = ( + left.section_type, left.flags, left.address, left.size, left.alignment + ) == ( + right.section_type, right.flags, right.address, right.size, right.alignment + ) + all_ok &= layout_equal and left.section_type == SHT_NOBITS + nobits_rows.append( + { + "name": name, + "type": "SHT_NOBITS", + "size": left.size, + "alignment": left.alignment, + "layout_equal": layout_equal, + } + ) + + measured_symbols = measured.runtime_symbols() + rebuilt_symbols = rebuilt.runtime_symbols() + symbols_equal = measured_symbols == rebuilt_symbols + all_ok &= symbols_equal + measured_host = (measured_dir / "pa_scheduler_host").read_bytes() + rebuilt_host = (rebuilt_dir / "pa_scheduler_host").read_bytes() + host_equal = measured_host == rebuilt_host + all_ok &= host_equal + + document = { + "schema": "pa_lazy_lambda_runtime_identity/v1", + "variant": (script_dir / "VARIANT").read_text(encoding="utf-8").strip(), + "status": "PASS" if all_ok else "FAIL", + "measured_elf_sha256": digest(measured.data), + "rebuilt_elf_sha256": digest(rebuilt.data), + "full_elf_equal": measured.data == rebuilt.data, + "runtime_content_sections": section_rows, + "nobits_layout": nobits_rows, + "runtime_symbols_equal": symbols_equal, + "runtime_symbol_count": len(measured_symbols), + "host_equal": host_equal, + "host_sha256": digest(measured_host), + "note": "Measured is the exact performance binary; full ELF and runtime identities are checked explicitly.", + } + output = script_dir / "artifacts" / "runtime_identity.json" + output.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8") + if not all_ok: + raise SystemExit(f"runtime identity check failed; see {output}") + print( + f"PASS variant={document['variant']} runtime_sections={len(section_rows)} " + f"runtime_symbols={len(measured_symbols)} host_sha256={document['host_sha256']}" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/MANIFEST.sha256 b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/MANIFEST.sha256 new file mode 100644 index 0000000000..d220a614b8 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/MANIFEST.sha256 @@ -0,0 +1,124 @@ +34b058acc11dde4072bea0a43243b7ea2d63625370011491697090dda256eeb9 ./README.md +c554b42c0f521da8e4a14249b078d5b816fa1bfafd9c7d461490bd787e1a0abc ./VARIANT +909eff5ca6e3f88d5b077945077864a1557549a3e171bcf233aa236dbd96d660 ./annotate_disassembly.py +66ea555034fc2c4986b28039456cb53f39060434ca233f521c07397671ce6034 ./artifacts/measured/artifacts.manifest +09e64f195785513bad69b6a1517d8fae385e85130333a7e77abb26fe618935b8 ./artifacts/measured/device_text_layout.manifest +d6d9e9bb04e28c7682e47230e20722f1e7843937b50017ddfe8a515acbe0ccd9 ./artifacts/measured/pa_scheduler_host +8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893 ./artifacts/measured/pa_scheduler_kernel.o +93e35fe96ee7f4802d8e0f8d88dbd336c0d319d4c72292f0dce25052275a94c3 ./artifacts/measured/published.sha256 +66ea555034fc2c4986b28039456cb53f39060434ca233f521c07397671ce6034 ./artifacts/rebuilt/artifacts.manifest +09e64f195785513bad69b6a1517d8fae385e85130333a7e77abb26fe618935b8 ./artifacts/rebuilt/device_text_layout.manifest +d6d9e9bb04e28c7682e47230e20722f1e7843937b50017ddfe8a515acbe0ccd9 ./artifacts/rebuilt/pa_scheduler_host +8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893 ./artifacts/rebuilt/pa_scheduler_kernel.o +93e35fe96ee7f4802d8e0f8d88dbd336c0d319d4c72292f0dce25052275a94c3 ./artifacts/rebuilt/published.sha256 +9cfafccdfcdc0d203c961c1bdb2bcf89261b4ecbe77af7378fb03390ecbefae8 ./artifacts/runtime_identity.json +aaf4d34a6551fa7c6a95c0942ab273e4b94576e846fc8265848e24d524b6b2cb ./build.sh +66ea555034fc2c4986b28039456cb53f39060434ca233f521c07397671ce6034 ./build/compete-first-lazy/artifacts.manifest +09e64f195785513bad69b6a1517d8fae385e85130333a7e77abb26fe618935b8 ./build/compete-first-lazy/device_text_layout.manifest +972e7f28e6192e420f107da3a7afc46d76a3fcc3116e3a0c752255f82ada4704 ./build/compete-first-lazy/pa_scheduler_aic.o +8b6f2ae2770760498d52e5bc5c0a99d62def79a04e9f70f918345cf698241ecc ./build/compete-first-lazy/pa_scheduler_aiv.o +d6d9e9bb04e28c7682e47230e20722f1e7843937b50017ddfe8a515acbe0ccd9 ./build/compete-first-lazy/pa_scheduler_host +8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893 ./build/compete-first-lazy/pa_scheduler_kernel.o +cf9511b54c9764456fac384e76c22f13e17beee49cee4464e8abdd2fb8cce900 ./build/compete-first-lazy/pa_scheduler_lazy_sample_callback_finish_aic.o +9ce4faa8493dc4cf2aa6c6f4395778a1cd2c18627c8196767d238f227f573c18 ./build/compete-first-lazy/pa_scheduler_lazy_sample_callback_finish_aiv.o +d2c5e0850dbc459d2f942efe0d894d55b4345843c6d957a12c0de3e08b450208 ./build/compete-first-lazy/pa_scheduler_lazy_sample_callback_runtime_aic.o +82622516dbd9f3374f72552f5e5d1ffebc5f29128bca3b0dd35937731e924ea7 ./build/compete-first-lazy/pa_scheduler_lazy_sample_callback_runtime_aiv.o +f0f401c67df72b8e0437bacbd9bbc4d8838f2c05e579ad6029867dbb02fd1a3d ./build/published.sha256 +e5732a50ddf852b98e23a89b3b48276cc1874ae70242a3a016440421782c6cb4 ./ccec/build.sh +ea82967763894d2726b68b19e74c2993ffe938f5bb0c90b92d3eae0966e643b7 ./ccec/callback_finish.cpp +7090c8405e1602476d1edf4ae6103ec6ec145e22bf15db742eac06aa2a6b7f9f ./ccec/callback_finish_api.h +4cef27650943d3973c6d56db687850ea3a7cbe28d64bf666b55cf735ce5bd141 ./ccec/callback_runtime_entry.cpp +47ca7623791bdd408b50c5e5ece4d63c7073775a1e3b636e046419acf948ddda ./ccec/ccec_ops.h +ac8a9766aa71a2bfc2a21c9021fc9fc30bc869575daa0e0f01121789230173e2 ./ccec/host.cpp +766ec34d6d4a7d80d07774782c68d701653cadf1bd8427fd1a0e922933c10236 ./ccec/kernel.cpp +f88d5ed31277c2692ecec2e8f5528cb6c2de752aea6a45850ab72d9e9b600075 ./ccec/pa_scheduler_device_exports.map +3d65b4a89dcf88554321a5435a2abf701a26757e8c229df7ac9783abbb5b96c3 ./ccec/pmu_owner_control.h +cb81c2a24670d893fd485845f13b8b106bfffe28d8c471f92a93775eee1a69e6 ./ccec/pmu_owner_host.h +09263e6698598f7be5b16b92e9e34bc59719de54aee132c43df24cec0ce42b4a ./ccec/pmu_owner_main_abi.h +9115d77d1be306c9f3b225805ed3df63fbdbdadcb9b4ff6ad690b56cfcf58867 ./ccec/pmu_owner_main_loader.h +2a926b355f78d4758fdc61a54827baec54ef781df34234a2edc9aa6a1a824f9e ./ccec/pmu_probe.h +3e3c6c99df27ff9e329242c64680559bf69cb4dc96650acc0ba1fe7442d68c70 ./common/host_support.h +f4a22840aef81ec05fb9532559957712c76ebd2dd1199d6969fc99688fc2e63d ./common/pa_frontend.h +9d42688f933e7b20eaab9eadcec40c3f0c0babc8ebdd3dd7369a2afcca1edbec ./common/pa_model.h +6645ce2aefe39b6132ce649c6af4c597a7a6978aa8983375fe09c4e186a812e5 ./common/pa_scheduler_core.h +8fd7c429dbbf0aae4e37968e7f90d953793f30f9caeb18eba515f9201deca1fa ./common/pa_trace.h +9d099151d0303c37dbb88f5cd2b15ee483dc0623759c45cbcf9d80c37704a80d ./common/winner_workload.h +8be384a62ed669e5645139aab1528570ffab184b5856f685e8056eed68e1e5cc ./common/winner_workload_host.h +4091251537a0405f83514d99fc6891300650a01cb61ac3b6c2291e899f7f0747 ./disassemble.py +e2b85f242dd06690b878504895c9a03e24489f75dc3f514a50c196ed436de1d3 ./disassembly/annotated/00_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz +3af0b26f2ed6b8e45121c71e6f875b19744d1f41c5148b2ecc66acef7b493f56 ./disassembly/annotated/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz +3a1f6528b8a5176152825047fcd72c7ba2194976379316565db070f5b333784c ./disassembly/annotated/02_pa_execute_real_winner_workload_aic_729699de.source.asm.gz +3fdb270573fdba87c76e236cbc3f2d64636170f8b0e5a481f0c82f099e19fded ./disassembly/annotated/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.source.asm.gz +af87679df78972fb4f775bc12589a41a14b0077f5d474b58c41891f5577e2d04 ./disassembly/annotated/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +8554244b60e19d60eb718727677a970b2299ef2e2102561ac674e711f8174373 ./disassembly/annotated/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.source.asm.gz +bd6e27107a25cf442fdae4164cb291807ec6fc5569d518635e63ebf18d43d15a ./disassembly/annotated/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +a5b644bb9cc8014991e7674924685713881cbe40f1aa407e3d26fc8eefe6e51c ./disassembly/annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz +897f10ddca4124c4a54159d131148ea06df4bada8e2dce28e50a225ba3772dce ./disassembly/annotated/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.source.asm.gz +15c05cda9dca6f3786e0d44884a6a7d06229c25e74349feed28314bd8bf91673 ./disassembly/annotated/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +a94815bdfaf87c5fdc63204b05e7b05237ed3051d16340b9147e3f9797bca99e ./disassembly/annotated/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz +a880e686b7a3e328114d681a3354b6f0e620e7b462844896567ffa55ae92de02 ./disassembly/annotated/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz +29397ff5303bb880ab957a906232c83c3163a860683ab4c51fb97b016d2a90f6 ./disassembly/annotated/12_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz +83ff83e20ad342151e2e6a41101d848da59d8b637b415ea468f16271689965ce ./disassembly/annotated/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.source.asm.gz +bf616bf378a19637c4711121631956493754ece990c39d584ca3d08e80881fe9 ./disassembly/annotated/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +681a3b99edba16bf5b58bc545c7ed82ccdaec900dafb25be8d10f0f3f75ab7a4 ./disassembly/annotated/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz +331f429dfde1898c71a649d4f4988b4c6fe654171bea29d049fe7c1dec992f09 ./disassembly/annotated/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz +51cdcbbfeaf38c1b574d6ff3ad744c60f31b2c11953e30f158bd634f1a1fc408 ./disassembly/key_flow/aic_compete_first_lazy.source.asm +8c169dca78cff0614a74bf9bd4e61e94171391eda04b25bdce2a7f665897c4af ./disassembly/key_flow/aic_lazy_input_policy.source.asm +dfd6b1e8c93a0f6147ed086f75f4937ffbdbc67ff787b17a36e13b141329dae5 ./disassembly/published.sha256 +d89cdc40aace173c873e1c82837a0ac1060662e4230c084cb9feb2d19881837b ./disassembly/raw/00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz +53d0444c1330462e69fd44d33f99b3f41772f084b66b61bbcffd979c9073cbeb ./disassembly/raw/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz +50719af7205a8c687bef91f9790d25d6a2d558d9d968a47b283fc4226cb69763 ./disassembly/raw/02_pa_execute_real_winner_workload_aic_729699de.asm.gz +cff035bd198985edc0b291bd5a946027d22db960378236170c29e96c1e42ac03 ./disassembly/raw/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz +c59d2bd2bd9f8d1ef3b1f1e5a12a6e964cef9db83ba3ea14c62d26aa1abbea03 ./disassembly/raw/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +6c4fe9a6d200d469e3cebf7dbdc6cd295679c3457e44ae56574b68e187f3b696 ./disassembly/raw/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz +d6a03ec582bd3e24d9fad8d489c5ce93c5d21343929a9b3be202df7b4e7f9a2d ./disassembly/raw/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +5b75f4842c3c023ad033240263e35457ad363f65f85bb7a41c737a242dd7b70f ./disassembly/raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz +a6e37748e0854ddc7b73c797b59e9b3dcb468365ddbfe1b3bea0e956f0a22f05 ./disassembly/raw/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz +81f766a4e5acc4e504339614c246451d06e7eb35149fa51a40a15d7f7ddba076 ./disassembly/raw/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +1da561a3de041604b48571185ef15db7e6677074566d1f578a6b3279248dd29d ./disassembly/raw/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz +8fd6f62d3f9a826100d055ebf29b801e969513548a9d4c68bbdc25c1f3a6758f ./disassembly/raw/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz +f98d6795cd66b34a5acdda1370c0c749d98b0a254b09e4097e81f13c2b0e5ff4 ./disassembly/raw/12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz +edd0df92b1cae8411d355e4a9f9058c9306b08e366755567a7e674e16c406c75 ./disassembly/raw/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz +85f987e18c6af8ebc39466316281a58d9ccf51249e6fd7817617185bdaf9c0bd ./disassembly/raw/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +ad259b20157f3ff9c8de366f47efefe6ef4db225c7611c97682650aeafe77634 ./disassembly/raw/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz +d57d1870e347095aa393da8232fc29b8657e900d3af8a924748337226ca6f6e3 ./disassembly/raw/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz +7b609f0665fcb3b9649e21073700db468cec223a20953958337da9bd7feb73e8 ./disassembly/raw/README.md +3957c577e5f1ac803ce0bf1ad7668f8bc08f5752e59dfbf4d69591c3c0e323a1 ./disassembly/raw/gaps.tsv +84aac4f16730d8afb63074bc9d61cd426451ea5823f2dce96850d86770500aa0 ./disassembly/raw/manifest.tsv +f236e5eaf5b0852b54da299d3ac2d46a8fb55cdbbfdae8d9e6545ca0084fc967 ./disassembly/raw/published.sha256 +b478bf22959d68f3ea62772cd1e39284eb1abf280ccb47b7ae594441d5badf12 ./generate_disassembly.sh +198c1dae69443432bda7acf9654b8cd0aa13f7400d4cb4eacc7661d8314c930f ./output/functional.txt +181e940f6f7ef426807815f067a04644f16b47b9d2ea34efa84b56ee6348e3bc ./output/performance/RUN_METADATA.md +819224c10dfc33a19987a0336c1f65aca896519ad4dece2f800985b1fe78672d ./output/performance/SUMMARY.md +b7e34eeb01420d8cbf7c62a759d872b714427fa66f89152e5d2ea4786cdf3f73 ./output/performance/raw/block_01_ABC_pos3_compete-first-lazy.txt +f8ca39b89944c54446b1ffe75e7a7ba3d396d8bd69414c0b7e757e5430a81cc8 ./output/performance/raw/block_02_BCA_pos2_compete-first-lazy.txt +67289429873f280a368a584a09dff24acaf10a48a7cefe0739878cd8b190e0a4 ./output/performance/raw/block_03_CAB_pos1_compete-first-lazy.txt +bb3d5dd33b41ce1a0414071e8c68959f6ab286cdb285b2280645fcff81c1058b ./output/performance/raw/block_04_ACB_pos2_compete-first-lazy.txt +981d32b6faa618eb1f36ffa304dd032887422c520276e6b87636ce45f4e4b6ed ./output/performance/raw/block_05_BAC_pos3_compete-first-lazy.txt +a2618a2b8a9a07ef1c662f0f2d5bc5fb12df08bf3c094bc98f01559a59dcd650 ./output/performance/raw/block_06_CBA_pos1_compete-first-lazy.txt +705d06ec8ff01376ab5ece572bf45a13102a1aacee10f8955559bc44b7fec4f5 ./output/performance/raw/block_07_CBA_pos1_compete-first-lazy.txt +8525a1261cfd255b20251f73ef591c9aa7c82b06a44af97405746cda7dbb814b ./output/performance/raw/block_08_BAC_pos3_compete-first-lazy.txt +4071b68c44ec5204db83600fadaada9b20a64816ad6fbc949d3a23ad27955461 ./output/performance/raw/block_09_ACB_pos2_compete-first-lazy.txt +82d5a5292c1289c4b7a81fa426b6d8a308335b5dcaa86c0f5bd4dd4aea010229 ./output/performance/raw/block_10_CAB_pos1_compete-first-lazy.txt +c9deb5b5086ad4e86332c327540e76b149a9114db07029888900d9e2cb98525e ./output/performance/raw/block_11_BCA_pos2_compete-first-lazy.txt +f99773915f09333be845a74511801d207c7e3b17a6dc44eff28ed7a4a192772d ./output/performance/raw/block_12_ABC_pos3_compete-first-lazy.txt +d8d9e78d6d1fed705ecd7e95be12d5806ce0469670a47aae404833e4aede820e ./output/performance/raw/block_13_BCA_pos2_compete-first-lazy.txt +3a9adce969e72517d1f271a42d42dc8f9e25401e7a98d14e7f3253532823e172 ./output/performance/raw/block_14_ACB_pos2_compete-first-lazy.txt +1926e06aef95c4f29f3156638540a4ca1294c08bf7bad8eac2e9fd9525da0a77 ./output/performance/raw/block_15_CBA_pos1_compete-first-lazy.txt +b4b0010d30ef67c0c37042780e29a9287427fdfc3694b6473b41bed8ac4339af ./output/performance/raw/block_16_ABC_pos3_compete-first-lazy.txt +e3d494ed59449ffdfa2430f63e1f6ec33dc35d98484bd0d88c3b0a6647b30981 ./output/performance/raw/block_17_CAB_pos1_compete-first-lazy.txt +9ed141ad0e390c168158363d65603f5ca61ebd562db6485cbb1c549a68eab51b ./output/performance/raw/block_18_BAC_pos3_compete-first-lazy.txt +e974874695446193691e474262374cd5116c472ddaa98198924a04056ed0ad42 ./output/performance/raw/block_19_BAC_pos3_compete-first-lazy.txt +24ea883845f30ceaad5ffab804a213f61dfb3b798c413066f313f3dbdebf91b4 ./output/performance/raw/block_20_CAB_pos1_compete-first-lazy.txt +2a780b3e4259c387ba708d717e4ea6208725912c15850a4644eee025ab7aaeff ./output/performance/raw/block_21_ABC_pos3_compete-first-lazy.txt +3b04c805efe3ab3e53172dc9a110c5584edec42af838be701271fd30ec9d7fd3 ./output/performance/raw/block_22_CBA_pos1_compete-first-lazy.txt +c908b48133b89b6b73928a66d1957bdc2268f82fa30a59bc7296885d4b9a49e4 ./output/performance/raw/block_23_ACB_pos2_compete-first-lazy.txt +02069577e750eb9113d3ad6853cff5448cb4d19c978eb48e9b245cd8a4e9e0bb ./output/performance/raw/block_24_BCA_pos2_compete-first-lazy.txt +5becdee926c6c1922bdbf7d583d56578a9b045fd5e5aed8800bf400af0704e1a ./output/performance/samples.tsv +0bad46524c2e2c16184e880bb17e986d7ff84095933c11df5b7e0216e74c54e8 ./output/performance/summary.json +087e410b2f2b19c50f9d40a96306a9ec185f4fb602d5d3fb1ef7fcd205157eb8 ./output/published.sha256 +067ffc84e28fe7385ae163ea2db7e55efd2e9b6d8943f239e12db2ead07a042b ./publish_artifacts.sh +29bfe64ba516d0df208f0738682b72d43d093c335ecc74cb5f674272c9365236 ./run_functional.sh +76618169e41352d1a9ebb883a9d4649ff76b409ac9129db27693dbbdcfd9ce95 ./source.sha256 +72471b0594aee1069f9fc774faa6d5900e374d3a283fdccc12b2c26a1302472a ./summarize_performance.py +e50378f7afdb15e3a84207b029e2d0aced10e1ac5a363372a791a0a8d177c937 ./verify_runtime_identity.py diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/README.md b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/README.md new file mode 100644 index 0000000000..ed6b5dda99 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/README.md @@ -0,0 +1,55 @@ +# C：compete-first + lazy lambda + +本目录是一份可单独复制、构建和查看的 lazy lambda 样例，不依赖 A/B 目录或上级目录中的源码、脚本与 +二进制。完整 task 流为 `Alloc -> QK -> SF -> PV -> UP`，32 AIC + 64 AIV。 + +## 固定形态 + +- `VARIANT=compete-first-lazy`，`PA_LAZY_SAMPLE_SHAPE_ID=2`；`ccec/build.sh` 会拒绝其他 shape； +- `PA_LAZY_SAMPLE_SPLIT_FINISH=1`，finish 为 noinline cross-TU； +- `PA_BUILD_SWIMLANE=1`、`PA_BUILD_SUBMIT_PMU=0`，运行时显式 `--no-swimlane`； +- 控制流与 B 完全同族：`Begin -> EfDrain -> Claim -> outer callback -> Materialize` + `-> TensorMap retire -> winner Fanin -> Register -> winner Build/Complete`； +- `AddLocalInput`、`AddGmInput`、`AddScalar` 在 `Lazy=true && !won` 时不调用 thunk； +- output/inout 仍由所有 worker 求值，以保持 Tier-1 样例的私有 heap/TensorMap 状态一致； +- nested lambda 只在 `Add*` 内同步求值，没有 thunk/closure 越过 outer callback 或 split-TU 边界。 + +lazy 判定的正式源码在 `common/pa_frontend.h` 的 `LazySampleCallbackArgsBuilder`，五类 task 的 +outer callback 在 `common/pa_scheduler_core.h::BuildLazySampleCallbackArgs`。固定构建入口无法选择 A/B； +未删除的 inactive `#if` 分支只保留来源上下文。 + +## 独立交付内容 + +- `common/`、`ccec/`：本版完整源码； +- `build/compete-first-lazy/`：清空历史目录后生成的完整本地编译结果和中间对象； +- `artifacts/measured/`：正式 24 样本性能测试实际使用的 host/final ELF； +- `artifacts/rebuilt/`：当前同一 clean-build 的发布副本； +- `artifacts/runtime_identity.json`：measured/rebuilt 运行时身份校验； +- `disassembly/raw/`：final-linked 机器码权威解码； +- `disassembly/annotated/`：每个函数的完整源码注释版反汇编； +- `disassembly/key_flow/aic_compete_first_lazy.source.asm`：compete-first 主路径窗口； +- `disassembly/key_flow/aic_lazy_input_policy.source.asm`：直接包含 + `if constexpr (Lazy) / if (!won_) return` 源码注释及对应机器码的窗口； +- `output/functional.txt`：最新 device0 b1 功能门禁； +- `output/performance/`：24 次独立 b256 原始日志、运行 provenance、样本表和统计； +- `source.sha256`、`build/published.sha256`、`output/published.sha256`、 + `MANIFEST.sha256`：源码、干净构建、输出与整包校验入口。 + +Measured final `.text` 为 `547640 B`，SHA256 +`866ed1081ebb94038a8feca87ccedf95e67aebfdaef11651c109f86672be2bdd`。 +去异常 Submit span 中位数为 `3818.781 us`（原始 `24` 个样本，保留 `22` 个）。b1 精确前端计数为 +`views/tensor/scalar/resets = 97/1162/9/384`。 + +## 复跑 + +```bash +./build.sh # 先精确删除 build/compete-first-lazy +./publish_artifacts.sh # 只刷新 artifacts/rebuilt +./verify_runtime_identity.py +./run_functional.sh # device0、b1、PMU off、--no-swimlane +./generate_disassembly.sh # measured ELF 的 raw + annotated 全量反汇编 +./summarize_performance.py +``` + +`*.source.asm.gz` 用 `zless` 阅读。DWARF 只提供地址到文件/行映射；显示的源码与原注释由脚本从本目录 +源码复制，文件头不会把它们冒充成 ELF 自带注释。 diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/VARIANT b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/VARIANT new file mode 100644 index 0000000000..c06f8e2b02 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/VARIANT @@ -0,0 +1 @@ +compete-first-lazy diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/annotate_disassembly.py b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/annotate_disassembly.py new file mode 100755 index 0000000000..c6eb096a1a --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/annotate_disassembly.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Add DWARF-mapped source and local source comments to one raw disassembly.""" + +from __future__ import annotations + +import argparse +import gzip +import os +import re +import subprocess +from pathlib import Path + + +INSTRUCTION_RE = re.compile(r"^(0x([0-9a-fA-F]+) \(\+0x[0-9a-fA-F]+\).*)$") +LOCATION_RE = re.compile(r"^(.*):(\d+)(?: \(discriminator \d+\))?$") +ANCHOR_RE = re.compile(r"^(.*):(\d+)$") + + +def read_text(path: Path) -> str: + if path.suffix == ".gz": + with gzip.open(path, "rt", encoding="utf-8") as source: + return source.read() + return path.read_text(encoding="utf-8") + + +def normalize_source_path(path_text: str, source_root: Path) -> tuple[Path | None, str]: + if path_text in {"??", ""}: + return None, "??" + # Some CCEC line-table rows are rendered as /path/file:?:0. The final + # :0 is parsed as the line number, leaving :? attached to the path. + if path_text.endswith(":?"): + path_text = path_text[:-2] + candidate = Path(path_text) + normalized = Path(candidate.as_posix().replace("/ccec/../common/", "/common/")) + parts = normalized.parts + if "lazy_lamda_sample" in parts: + marker = len(parts) - 1 - list(reversed(parts)).index("lazy_lamda_sample") + tail = parts[marker + 1 :] + source_index = next( + (index for index, part in enumerate(tail) if part in {"common", "ccec"}), + None, + ) + if source_index is not None: + suffix = Path(*tail[source_index:]) + local = source_root / suffix + return (local if local.is_file() else None), suffix.as_posix() + if candidate.is_file(): + ascend_home = os.environ.get("ASCEND_HOME_PATH") + if ascend_home: + try: + relative = candidate.resolve().relative_to(Path(ascend_home).resolve()) + except ValueError: + pass + else: + return candidate, f"$ASCEND_HOME_PATH/{relative.as_posix()}" + return candidate, candidate.as_posix() + return None, candidate.as_posix() + + +def source_lines(path: Path | None, cache: dict[Path, list[str]]) -> list[str] | None: + if path is None: + return None + if path not in cache: + cache[path] = path.read_text(encoding="utf-8").splitlines() + return cache[path] + + +def write_output(path: Path, text: str) -> None: + if path.suffix == ".gz": + with path.open("wb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed: + compressed.write(text.encode("utf-8")) + return + path.write_text(text, encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser() + script_dir = Path(__file__).resolve().parent + parser.add_argument("--elf", type=Path, required=True) + parser.add_argument("--raw", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--source-root", type=Path, default=script_dir) + parser.add_argument("--anchor", help="First DWARF location matching SUFFIX:LINE") + parser.add_argument("--before", type=int, default=40) + parser.add_argument("--after", type=int, default=160) + args = parser.parse_args() + + raw_lines = read_text(args.raw.resolve()).splitlines() + instruction_lines = [line for line in raw_lines if INSTRUCTION_RE.match(line)] + if not instruction_lines: + raise SystemExit(f"No instruction rows in {args.raw}") + pcs = [int(INSTRUCTION_RE.match(line).group(2), 16) for line in instruction_lines] + process = subprocess.run( + ["addr2line", "-e", str(args.elf.resolve()), "-C"], + input="".join(f"0x{pc:x}\n" for pc in pcs), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + locations = process.stdout.splitlines() + if len(locations) != len(instruction_lines): + raise SystemExit( + f"addr2line returned {len(locations)} locations for {len(instruction_lines)} instructions" + ) + + normalized_locations: list[tuple[Path | None, str, int]] = [] + source_root = args.source_root.resolve() + for location in locations: + if location.endswith(":?"): + local_path, display = normalize_source_path(location[:-2], source_root) + normalized_locations.append((local_path, display, 0)) + continue + match = LOCATION_RE.match(location) + if match is None: + normalized_locations.append((None, location, 0)) + continue + local_path, display = normalize_source_path(match.group(1), source_root) + normalized_locations.append((local_path, display, int(match.group(2)))) + + start = 0 + end = len(instruction_lines) + if args.anchor: + anchor_match = ANCHOR_RE.match(args.anchor) + if anchor_match is None: + raise SystemExit("--anchor must be SUFFIX:LINE") + suffix = anchor_match.group(1) + anchor_line = int(anchor_match.group(2)) + anchor_index = next( + ( + index + for index, (_, display, line) in enumerate(normalized_locations) + if display.endswith(suffix) and line == anchor_line + ), + None, + ) + if anchor_index is None: + raise SystemExit(f"DWARF anchor not found: {args.anchor}") + start = max(0, anchor_index - args.before) + end = min(len(instruction_lines), anchor_index + args.after + 1) + + headers = [line for line in raw_lines if line.startswith("#")] + output = headers + [ + "# annotation_schema=pa_source_annotated_disassembly/v1", + "# annotation_rule=DWARF supplies only file:line; SOURCE rows are copied from local source files", + "# annotation_warning=comments have source context only and do not own an exact machine address", + f"# annotation_instruction_slice={start}:{end}", + "#", + ] + cache: dict[Path, list[str]] = {} + previous_display = "" + previous_line = 0 + for index in range(start, end): + local_path, display, line_number = normalized_locations[index] + if display != previous_display or line_number != previous_line: + lines = source_lines(local_path, cache) + output.append(f"# [DWARF] {display}:{line_number}") + if lines is None or line_number <= 0 or line_number > len(lines): + output.append("# [SOURCE unavailable]") + else: + if display == previous_display and previous_line < line_number <= previous_line + 12: + context_start = previous_line + 1 + else: + context_start = max(1, line_number - 6) + for source_line in range(context_start, line_number + 1): + marker = ">" if source_line == line_number else " " + output.append(f"# {marker} {source_line:5d} | {lines[source_line - 1]}") + previous_display = display + previous_line = line_number + output.append(instruction_lines[index]) + + args.output.parent.mkdir(parents=True, exist_ok=True) + write_output(args.output, "\n".join(output) + "\n") + print( + f"wrote {args.output}: instructions={end - start} " + f"range=0x{pcs[start]:x}..0x{pcs[end - 1]:x}" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/artifacts.manifest b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/artifacts.manifest new file mode 100644 index 0000000000..e4c4850b05 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/artifacts.manifest @@ -0,0 +1,14 @@ +# schema=pa_scheduler_lazy_sample_callback_artifacts/v1 +# backend=ccec +# shape=compete-first-lazy +# shape_id=2 +# observation=split-combination-semantic +d6d9e9bb04e28c7682e47230e20722f1e7843937b50017ddfe8a515acbe0ccd9 pa_scheduler_host +8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893 pa_scheduler_kernel.o +d2c5e0850dbc459d2f942efe0d894d55b4345843c6d957a12c0de3e08b450208 pa_scheduler_lazy_sample_callback_runtime_aic.o +972e7f28e6192e420f107da3a7afc46d76a3fcc3116e3a0c752255f82ada4704 pa_scheduler_aic.o +cf9511b54c9764456fac384e76c22f13e17beee49cee4464e8abdd2fb8cce900 pa_scheduler_lazy_sample_callback_finish_aic.o +82622516dbd9f3374f72552f5e5d1ffebc5f29128bca3b0dd35937731e924ea7 pa_scheduler_lazy_sample_callback_runtime_aiv.o +8b6f2ae2770760498d52e5bc5c0a99d62def79a04e9f70f918345cf698241ecc pa_scheduler_aiv.o +9ce4faa8493dc4cf2aa6c6f4395778a1cd2c18627c8196767d238f227f573c18 pa_scheduler_lazy_sample_callback_finish_aiv.o +09e64f195785513bad69b6a1517d8fae385e85130333a7e77abb26fe618935b8 device_text_layout.manifest diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/device_text_layout.manifest b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/device_text_layout.manifest new file mode 100644 index 0000000000..f1d2a3c741 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/device_text_layout.manifest @@ -0,0 +1,27 @@ +# schema=pa_scheduler_device_text_layout/v1 +# backend=ccec +# shape=compete-first-lazy +# block_local_reserve_bytes=1600 +# compiler=$ASCEND_HOME_PATH/bin/ccec +# compiler_sha256=34355dde8c995f3c7d769ddf0e1b853aba65cf6ef6fb3863ab86c026d40aebfd +# linker=$ASCEND_HOME_PATH/bin/ld.lld +# linker_sha256=727810dd7f72dedbe1453edeb81d3a5371d8dc0591bc58693edf1aa7f3fa7eff +text pa_scheduler_kernel.o 547640 866ed1081ebb94038a8feca87ccedf95e67aebfdaef11651c109f86672be2bdd +text pa_scheduler_lazy_sample_callback_runtime_aic.o 112 9391340d3a45b712c7b883e97e1f6889eae31e54e4ba1ce52864012bc4141582 +text pa_scheduler_aic.o 162608 e84700f3996932c9930bcf1561b242ce3e25c2093bcff23af82f46515ff1f940 +text pa_scheduler_lazy_sample_callback_finish_aic.o 110352 f1e086fe71d1d83ac7654eb0e38475f3e79996d841b18e21830df2fd40e873bc +text pa_scheduler_lazy_sample_callback_runtime_aiv.o 128 7bf82e7aa5167025d62c32b482dd5222f913fce1d08d1770917f4f757f165c1d +text pa_scheduler_aiv.o 163640 3ae08b4dab63e6760e03dd0fa206e8e4184c7b061e0c9cd0294d673740dc3139 +text pa_scheduler_lazy_sample_callback_finish_aiv.o 110800 260afa427b5935b15f8368b2e4c054f6d57d4b0e5bf861b9ef769758a7c3b0d7 +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aic 104 3a8ecf84e154c1afc9dab2c992e3357d338c1d996a677931069f16310bdb28e5 +symbol pa_scheduler_lazy_sample_callback_runtime_aic.o pa_scheduler_0_mix_aic 104 42b5cc25a392a99eb3366f12177f028db8a27f55c00166753083fa75cf404b79 +symbol pa_scheduler_aic.o pa_scheduler_lazy_sample_callback_orchestration_aic 161792 aabd53e0cc59a5411536a03d8d5be0cb725cad1bcc4645bebbc7d8b6b155302d +symbol pa_scheduler_lazy_sample_callback_finish_aic.o pa_scheduler_lazy_sample_callback_finish_aic 110128 2adeb947c5bd08fa13d6ea3c403174e067e333d62dda31e59ee1380fece174b5 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_orchestration_aic 161792 35bcc11cdad126fa5d2f92ff1df421d8556843a412ea3497addfe820d84bd852 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_finish_aic 110128 cdb102a84acb7a681565d97805b405d526a94c604812faed3850773575acee08 +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aiv 124 63f9349fa440318678bcf3e98ad719502bd907f7537fb0f083eac32895e3ad9b +symbol pa_scheduler_lazy_sample_callback_runtime_aiv.o pa_scheduler_0_mix_aiv 124 f0f4b9a2d2c1acd43efba6a24731f90b8424b1681b0091797f0bd4b3da6500a5 +symbol pa_scheduler_aiv.o pa_scheduler_lazy_sample_callback_orchestration_aiv 161944 309fdcf64f16e4f8c97d8323df18f40f7764d08f7376ec70ef608a5fda5ad204 +symbol pa_scheduler_lazy_sample_callback_finish_aiv.o pa_scheduler_lazy_sample_callback_finish_aiv 110576 c9656e23b23851999d185a6b440b2ed70cb3639db51fd7e3feab7f35acb8e4c7 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_orchestration_aiv 161944 62d3784c0d6bcf33939e500cdcdf7464889437785157f68492a6f13c8ec5e60c +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_finish_aiv 110576 3f5ec8fee73b82e72f40566908c21f4fcfd3fd85983892c323a526c66557cade diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/pa_scheduler_host b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/pa_scheduler_host new file mode 100755 index 0000000000..d8d80a70a0 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/pa_scheduler_host differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/pa_scheduler_kernel.o b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/pa_scheduler_kernel.o new file mode 100755 index 0000000000..ceb7eb0f76 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/pa_scheduler_kernel.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/published.sha256 new file mode 100644 index 0000000000..ade7ae1609 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/measured/published.sha256 @@ -0,0 +1,4 @@ +66ea555034fc2c4986b28039456cb53f39060434ca233f521c07397671ce6034 artifacts.manifest +09e64f195785513bad69b6a1517d8fae385e85130333a7e77abb26fe618935b8 device_text_layout.manifest +d6d9e9bb04e28c7682e47230e20722f1e7843937b50017ddfe8a515acbe0ccd9 pa_scheduler_host +8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893 pa_scheduler_kernel.o diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/artifacts.manifest b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/artifacts.manifest new file mode 100644 index 0000000000..e4c4850b05 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/artifacts.manifest @@ -0,0 +1,14 @@ +# schema=pa_scheduler_lazy_sample_callback_artifacts/v1 +# backend=ccec +# shape=compete-first-lazy +# shape_id=2 +# observation=split-combination-semantic +d6d9e9bb04e28c7682e47230e20722f1e7843937b50017ddfe8a515acbe0ccd9 pa_scheduler_host +8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893 pa_scheduler_kernel.o +d2c5e0850dbc459d2f942efe0d894d55b4345843c6d957a12c0de3e08b450208 pa_scheduler_lazy_sample_callback_runtime_aic.o +972e7f28e6192e420f107da3a7afc46d76a3fcc3116e3a0c752255f82ada4704 pa_scheduler_aic.o +cf9511b54c9764456fac384e76c22f13e17beee49cee4464e8abdd2fb8cce900 pa_scheduler_lazy_sample_callback_finish_aic.o +82622516dbd9f3374f72552f5e5d1ffebc5f29128bca3b0dd35937731e924ea7 pa_scheduler_lazy_sample_callback_runtime_aiv.o +8b6f2ae2770760498d52e5bc5c0a99d62def79a04e9f70f918345cf698241ecc pa_scheduler_aiv.o +9ce4faa8493dc4cf2aa6c6f4395778a1cd2c18627c8196767d238f227f573c18 pa_scheduler_lazy_sample_callback_finish_aiv.o +09e64f195785513bad69b6a1517d8fae385e85130333a7e77abb26fe618935b8 device_text_layout.manifest diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/device_text_layout.manifest b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/device_text_layout.manifest new file mode 100644 index 0000000000..f1d2a3c741 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/device_text_layout.manifest @@ -0,0 +1,27 @@ +# schema=pa_scheduler_device_text_layout/v1 +# backend=ccec +# shape=compete-first-lazy +# block_local_reserve_bytes=1600 +# compiler=$ASCEND_HOME_PATH/bin/ccec +# compiler_sha256=34355dde8c995f3c7d769ddf0e1b853aba65cf6ef6fb3863ab86c026d40aebfd +# linker=$ASCEND_HOME_PATH/bin/ld.lld +# linker_sha256=727810dd7f72dedbe1453edeb81d3a5371d8dc0591bc58693edf1aa7f3fa7eff +text pa_scheduler_kernel.o 547640 866ed1081ebb94038a8feca87ccedf95e67aebfdaef11651c109f86672be2bdd +text pa_scheduler_lazy_sample_callback_runtime_aic.o 112 9391340d3a45b712c7b883e97e1f6889eae31e54e4ba1ce52864012bc4141582 +text pa_scheduler_aic.o 162608 e84700f3996932c9930bcf1561b242ce3e25c2093bcff23af82f46515ff1f940 +text pa_scheduler_lazy_sample_callback_finish_aic.o 110352 f1e086fe71d1d83ac7654eb0e38475f3e79996d841b18e21830df2fd40e873bc +text pa_scheduler_lazy_sample_callback_runtime_aiv.o 128 7bf82e7aa5167025d62c32b482dd5222f913fce1d08d1770917f4f757f165c1d +text pa_scheduler_aiv.o 163640 3ae08b4dab63e6760e03dd0fa206e8e4184c7b061e0c9cd0294d673740dc3139 +text pa_scheduler_lazy_sample_callback_finish_aiv.o 110800 260afa427b5935b15f8368b2e4c054f6d57d4b0e5bf861b9ef769758a7c3b0d7 +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aic 104 3a8ecf84e154c1afc9dab2c992e3357d338c1d996a677931069f16310bdb28e5 +symbol pa_scheduler_lazy_sample_callback_runtime_aic.o pa_scheduler_0_mix_aic 104 42b5cc25a392a99eb3366f12177f028db8a27f55c00166753083fa75cf404b79 +symbol pa_scheduler_aic.o pa_scheduler_lazy_sample_callback_orchestration_aic 161792 aabd53e0cc59a5411536a03d8d5be0cb725cad1bcc4645bebbc7d8b6b155302d +symbol pa_scheduler_lazy_sample_callback_finish_aic.o pa_scheduler_lazy_sample_callback_finish_aic 110128 2adeb947c5bd08fa13d6ea3c403174e067e333d62dda31e59ee1380fece174b5 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_orchestration_aic 161792 35bcc11cdad126fa5d2f92ff1df421d8556843a412ea3497addfe820d84bd852 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_finish_aic 110128 cdb102a84acb7a681565d97805b405d526a94c604812faed3850773575acee08 +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aiv 124 63f9349fa440318678bcf3e98ad719502bd907f7537fb0f083eac32895e3ad9b +symbol pa_scheduler_lazy_sample_callback_runtime_aiv.o pa_scheduler_0_mix_aiv 124 f0f4b9a2d2c1acd43efba6a24731f90b8424b1681b0091797f0bd4b3da6500a5 +symbol pa_scheduler_aiv.o pa_scheduler_lazy_sample_callback_orchestration_aiv 161944 309fdcf64f16e4f8c97d8323df18f40f7764d08f7376ec70ef608a5fda5ad204 +symbol pa_scheduler_lazy_sample_callback_finish_aiv.o pa_scheduler_lazy_sample_callback_finish_aiv 110576 c9656e23b23851999d185a6b440b2ed70cb3639db51fd7e3feab7f35acb8e4c7 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_orchestration_aiv 161944 62d3784c0d6bcf33939e500cdcdf7464889437785157f68492a6f13c8ec5e60c +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_finish_aiv 110576 3f5ec8fee73b82e72f40566908c21f4fcfd3fd85983892c323a526c66557cade diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/pa_scheduler_host b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/pa_scheduler_host new file mode 100755 index 0000000000..d8d80a70a0 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/pa_scheduler_host differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/pa_scheduler_kernel.o b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/pa_scheduler_kernel.o new file mode 100755 index 0000000000..ceb7eb0f76 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/pa_scheduler_kernel.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/published.sha256 new file mode 100644 index 0000000000..ade7ae1609 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/rebuilt/published.sha256 @@ -0,0 +1,4 @@ +66ea555034fc2c4986b28039456cb53f39060434ca233f521c07397671ce6034 artifacts.manifest +09e64f195785513bad69b6a1517d8fae385e85130333a7e77abb26fe618935b8 device_text_layout.manifest +d6d9e9bb04e28c7682e47230e20722f1e7843937b50017ddfe8a515acbe0ccd9 pa_scheduler_host +8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893 pa_scheduler_kernel.o diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/runtime_identity.json b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/runtime_identity.json new file mode 100644 index 0000000000..f5f03d1536 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/artifacts/runtime_identity.json @@ -0,0 +1,59 @@ +{ + "full_elf_equal": true, + "host_equal": true, + "host_sha256": "d6d9e9bb04e28c7682e47230e20722f1e7843937b50017ddfe8a515acbe0ccd9", + "measured_elf_sha256": "8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893", + "nobits_layout": [ + { + "alignment": 64, + "layout_equal": true, + "name": ".bl_uninit", + "size": 3200, + "type": "SHT_NOBITS" + } + ], + "note": "Measured is the exact performance binary; full ELF and runtime identities are checked explicitly.", + "rebuilt_elf_sha256": "8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893", + "runtime_content_sections": [ + { + "content_equal": true, + "metadata_equal": true, + "name": ".text", + "sha256": "866ed1081ebb94038a8feca87ccedf95e67aebfdaef11651c109f86672be2bdd", + "size": 547640 + }, + { + "content_equal": true, + "metadata_equal": true, + "name": ".rodata", + "sha256": "4a9aa85eb4fb09ef61bf0e0a752e6a6cbefbbc3cf879aa6695a8b85365fc30e5", + "size": 576 + }, + { + "content_equal": true, + "metadata_equal": true, + "name": ".ascend.meta.pa_scheduler_0_mix_aic", + "sha256": "ff019d3ac6a3ece0ac9868ad2c6c86743efd7015d2ffa24b0fa85f585442cafb", + "size": 24 + }, + { + "content_equal": true, + "metadata_equal": true, + "name": "__CCE_KernelArgSize", + "sha256": "c0cdbb6e45249a718e7e605f801cf43f673a22fcf0347034da71f6be2e4f4d5e", + "size": 8 + }, + { + "content_equal": true, + "metadata_equal": true, + "name": ".ascend.meta.pa_scheduler_0_mix_aiv", + "sha256": "d498029b3bd8fd059bfaca013163a729768f11edf540f96b77bf40ea1e604921", + "size": 56 + } + ], + "runtime_symbol_count": 22, + "runtime_symbols_equal": true, + "schema": "pa_lazy_lambda_runtime_identity/v1", + "status": "PASS", + "variant": "compete-first-lazy" +} diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build.sh b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build.sh new file mode 100755 index 0000000000..73b935f13e --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +VARIANT="compete-first-lazy" +BUILD_DIR="$SCRIPT_DIR/build/$VARIANT" + +# This package deliberately owns exactly one build directory. Remove the +# complete old directory so stale objects can never survive a rebuild. +if [[ "$BUILD_DIR" != "$SCRIPT_DIR/build/compete-first-lazy" ]]; then + echo "Refusing to clean an unexpected build path: $BUILD_DIR" >&2 + exit 1 +fi +if [[ -e "$BUILD_DIR" ]]; then + rm -rf -- "$BUILD_DIR" +fi + +"$SCRIPT_DIR/ccec/build.sh" "$VARIANT" diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/artifacts.manifest b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/artifacts.manifest new file mode 100644 index 0000000000..e4c4850b05 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/artifacts.manifest @@ -0,0 +1,14 @@ +# schema=pa_scheduler_lazy_sample_callback_artifacts/v1 +# backend=ccec +# shape=compete-first-lazy +# shape_id=2 +# observation=split-combination-semantic +d6d9e9bb04e28c7682e47230e20722f1e7843937b50017ddfe8a515acbe0ccd9 pa_scheduler_host +8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893 pa_scheduler_kernel.o +d2c5e0850dbc459d2f942efe0d894d55b4345843c6d957a12c0de3e08b450208 pa_scheduler_lazy_sample_callback_runtime_aic.o +972e7f28e6192e420f107da3a7afc46d76a3fcc3116e3a0c752255f82ada4704 pa_scheduler_aic.o +cf9511b54c9764456fac384e76c22f13e17beee49cee4464e8abdd2fb8cce900 pa_scheduler_lazy_sample_callback_finish_aic.o +82622516dbd9f3374f72552f5e5d1ffebc5f29128bca3b0dd35937731e924ea7 pa_scheduler_lazy_sample_callback_runtime_aiv.o +8b6f2ae2770760498d52e5bc5c0a99d62def79a04e9f70f918345cf698241ecc pa_scheduler_aiv.o +9ce4faa8493dc4cf2aa6c6f4395778a1cd2c18627c8196767d238f227f573c18 pa_scheduler_lazy_sample_callback_finish_aiv.o +09e64f195785513bad69b6a1517d8fae385e85130333a7e77abb26fe618935b8 device_text_layout.manifest diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/device_text_layout.manifest b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/device_text_layout.manifest new file mode 100644 index 0000000000..f1d2a3c741 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/device_text_layout.manifest @@ -0,0 +1,27 @@ +# schema=pa_scheduler_device_text_layout/v1 +# backend=ccec +# shape=compete-first-lazy +# block_local_reserve_bytes=1600 +# compiler=$ASCEND_HOME_PATH/bin/ccec +# compiler_sha256=34355dde8c995f3c7d769ddf0e1b853aba65cf6ef6fb3863ab86c026d40aebfd +# linker=$ASCEND_HOME_PATH/bin/ld.lld +# linker_sha256=727810dd7f72dedbe1453edeb81d3a5371d8dc0591bc58693edf1aa7f3fa7eff +text pa_scheduler_kernel.o 547640 866ed1081ebb94038a8feca87ccedf95e67aebfdaef11651c109f86672be2bdd +text pa_scheduler_lazy_sample_callback_runtime_aic.o 112 9391340d3a45b712c7b883e97e1f6889eae31e54e4ba1ce52864012bc4141582 +text pa_scheduler_aic.o 162608 e84700f3996932c9930bcf1561b242ce3e25c2093bcff23af82f46515ff1f940 +text pa_scheduler_lazy_sample_callback_finish_aic.o 110352 f1e086fe71d1d83ac7654eb0e38475f3e79996d841b18e21830df2fd40e873bc +text pa_scheduler_lazy_sample_callback_runtime_aiv.o 128 7bf82e7aa5167025d62c32b482dd5222f913fce1d08d1770917f4f757f165c1d +text pa_scheduler_aiv.o 163640 3ae08b4dab63e6760e03dd0fa206e8e4184c7b061e0c9cd0294d673740dc3139 +text pa_scheduler_lazy_sample_callback_finish_aiv.o 110800 260afa427b5935b15f8368b2e4c054f6d57d4b0e5bf861b9ef769758a7c3b0d7 +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aic 104 3a8ecf84e154c1afc9dab2c992e3357d338c1d996a677931069f16310bdb28e5 +symbol pa_scheduler_lazy_sample_callback_runtime_aic.o pa_scheduler_0_mix_aic 104 42b5cc25a392a99eb3366f12177f028db8a27f55c00166753083fa75cf404b79 +symbol pa_scheduler_aic.o pa_scheduler_lazy_sample_callback_orchestration_aic 161792 aabd53e0cc59a5411536a03d8d5be0cb725cad1bcc4645bebbc7d8b6b155302d +symbol pa_scheduler_lazy_sample_callback_finish_aic.o pa_scheduler_lazy_sample_callback_finish_aic 110128 2adeb947c5bd08fa13d6ea3c403174e067e333d62dda31e59ee1380fece174b5 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_orchestration_aic 161792 35bcc11cdad126fa5d2f92ff1df421d8556843a412ea3497addfe820d84bd852 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_finish_aic 110128 cdb102a84acb7a681565d97805b405d526a94c604812faed3850773575acee08 +symbol pa_scheduler_kernel.o pa_scheduler_0_mix_aiv 124 63f9349fa440318678bcf3e98ad719502bd907f7537fb0f083eac32895e3ad9b +symbol pa_scheduler_lazy_sample_callback_runtime_aiv.o pa_scheduler_0_mix_aiv 124 f0f4b9a2d2c1acd43efba6a24731f90b8424b1681b0091797f0bd4b3da6500a5 +symbol pa_scheduler_aiv.o pa_scheduler_lazy_sample_callback_orchestration_aiv 161944 309fdcf64f16e4f8c97d8323df18f40f7764d08f7376ec70ef608a5fda5ad204 +symbol pa_scheduler_lazy_sample_callback_finish_aiv.o pa_scheduler_lazy_sample_callback_finish_aiv 110576 c9656e23b23851999d185a6b440b2ed70cb3639db51fd7e3feab7f35acb8e4c7 +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_orchestration_aiv 161944 62d3784c0d6bcf33939e500cdcdf7464889437785157f68492a6f13c8ec5e60c +symbol pa_scheduler_kernel.o pa_scheduler_lazy_sample_callback_finish_aiv 110576 3f5ec8fee73b82e72f40566908c21f4fcfd3fd85983892c323a526c66557cade diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_aic.o b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_aic.o new file mode 100644 index 0000000000..2487e85854 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_aic.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_aiv.o b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_aiv.o new file mode 100644 index 0000000000..db90664364 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_aiv.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_host b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_host new file mode 100755 index 0000000000..d8d80a70a0 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_host differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_kernel.o b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_kernel.o new file mode 100755 index 0000000000..ceb7eb0f76 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_kernel.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_lazy_sample_callback_finish_aic.o b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_lazy_sample_callback_finish_aic.o new file mode 100644 index 0000000000..4e9f5ca9cd Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_lazy_sample_callback_finish_aic.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_lazy_sample_callback_finish_aiv.o b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_lazy_sample_callback_finish_aiv.o new file mode 100644 index 0000000000..779584c485 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_lazy_sample_callback_finish_aiv.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_lazy_sample_callback_runtime_aic.o b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_lazy_sample_callback_runtime_aic.o new file mode 100644 index 0000000000..19bd35633d Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_lazy_sample_callback_runtime_aic.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_lazy_sample_callback_runtime_aiv.o b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_lazy_sample_callback_runtime_aiv.o new file mode 100644 index 0000000000..9ce2570a09 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/compete-first-lazy/pa_scheduler_lazy_sample_callback_runtime_aiv.o differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/published.sha256 new file mode 100644 index 0000000000..88b25f7632 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/build/published.sha256 @@ -0,0 +1,10 @@ +66ea555034fc2c4986b28039456cb53f39060434ca233f521c07397671ce6034 build/compete-first-lazy/artifacts.manifest +09e64f195785513bad69b6a1517d8fae385e85130333a7e77abb26fe618935b8 build/compete-first-lazy/device_text_layout.manifest +972e7f28e6192e420f107da3a7afc46d76a3fcc3116e3a0c752255f82ada4704 build/compete-first-lazy/pa_scheduler_aic.o +8b6f2ae2770760498d52e5bc5c0a99d62def79a04e9f70f918345cf698241ecc build/compete-first-lazy/pa_scheduler_aiv.o +d6d9e9bb04e28c7682e47230e20722f1e7843937b50017ddfe8a515acbe0ccd9 build/compete-first-lazy/pa_scheduler_host +8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893 build/compete-first-lazy/pa_scheduler_kernel.o +cf9511b54c9764456fac384e76c22f13e17beee49cee4464e8abdd2fb8cce900 build/compete-first-lazy/pa_scheduler_lazy_sample_callback_finish_aic.o +9ce4faa8493dc4cf2aa6c6f4395778a1cd2c18627c8196767d238f227f573c18 build/compete-first-lazy/pa_scheduler_lazy_sample_callback_finish_aiv.o +d2c5e0850dbc459d2f942efe0d894d55b4345843c6d957a12c0de3e08b450208 build/compete-first-lazy/pa_scheduler_lazy_sample_callback_runtime_aic.o +82622516dbd9f3374f72552f5e5d1ffebc5f29128bca3b0dd35937731e924ea7 build/compete-first-lazy/pa_scheduler_lazy_sample_callback_runtime_aiv.o diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/build.sh b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/build.sh new file mode 100755 index 0000000000..dd5510e6d6 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/build.sh @@ -0,0 +1,934 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +# 任一工具失败、未定义变量或管道中间失败都立即终止，避免继续使用半成品 device ELF。 +set -euo pipefail + +# 所有输入和产物都从脚本自身位置解析，调用者无需位于仓库根目录。 +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +LAZY_SAMPLE_MANIFEST_NAME="artifacts.manifest" +LAZY_SAMPLE_TEXT_LAYOUT_NAME="device_text_layout.manifest" +LAZY_SAMPLE_SPLIT=0 +LAZY_SAMPLE_VARIANT=1 +PMU_VARIANT=0 + +# All three artifacts compile the same swimlane-capable source with submit PMU +# disabled. Performance runs pass --no-swimlane at runtime. +if [[ $# -ne 1 || "$1" != "compete-first-lazy" ]]; then + echo "This frozen package accepts only: $0 compete-first-lazy" >&2 + exit 1 +fi +BUILD_VARIANT="$1" +case "$BUILD_VARIANT" in + original) + LAZY_SAMPLE_SHAPE="original" + LAZY_SAMPLE_SHAPE_ID=0 + LAZY_SAMPLE_OBSERVATION="legacy-straight-line-eager" + LAZY_SAMPLE_FINISH_SHAPE="legacy-inline-submit" + BUILD_DIR="$ROOT_DIR/build/original" + VARIANT_DEFINES=(-DPA_BUILD_SWIMLANE=1 -DPA_BUILD_SUBMIT_PMU=0 -DPA_SUBMIT_PMU_PHASE_ID=0) + ;; + compete-first|compete-first-lazy) + LAZY_SAMPLE_SPLIT=1 + LAZY_SAMPLE_OBSERVATION="split-combination-semantic" + LAZY_SAMPLE_FINISH_SHAPE="noinline-cross-tu" + if [[ "$BUILD_VARIANT" == "compete-first" ]]; then + LAZY_SAMPLE_SHAPE="compete-first" + LAZY_SAMPLE_SHAPE_ID=1 + else + LAZY_SAMPLE_SHAPE="compete-first-lazy" + LAZY_SAMPLE_SHAPE_ID=2 + fi + BUILD_DIR="$ROOT_DIR/build/$LAZY_SAMPLE_SHAPE" + VARIANT_DEFINES=( + -DPA_BUILD_SWIMLANE=1 + -DPA_BUILD_SUBMIT_PMU=0 + -DPA_SUBMIT_PMU_PHASE_ID=0 + "-DPA_LAZY_SAMPLE_SHAPE_ID=$LAZY_SAMPLE_SHAPE_ID" + -DPA_LAZY_SAMPLE_SPLIT_FINISH=1 + ) + ;; + *) + echo "Unknown variant: $BUILD_VARIANT (expected original|compete-first|compete-first-lazy)" >&2 + exit 1 + ;; +esac + +# 编译只依赖本目录源码与用户安装的 CANN/PTO 头，不引用 pa_scheduler 目录外的 simpler 构建产物。 +if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then + echo "ASCEND_HOME_PATH is not set; source the CANN 9.1 set_env.sh first." >&2 + exit 1 +fi + +CCEC="$ASCEND_HOME_PATH/bin/ccec" +LD="$ASCEND_HOME_PATH/bin/ld.lld" +HCC="$ASCEND_HOME_PATH/tools/hcc/bin/aarch64-target-linux-gnu-g++" +CXX_BIN="${CXX:-g++}" +READELF_BIN="${READELF:-readelf}" +PTO_INCLUDE_ROOT="${PTO_ISA_ROOT:-$ASCEND_HOME_PATH/x86_64-linux}" + +# ccec/ld.lld 必须来自当前已 source 的 CANN；host 编译器和 readelf 允许用户通过环境变量替换。 +if [[ ! -x "$CCEC" || ! -x "$LD" ]]; then + echo "CCEC or ld.lld is missing under ASCEND_HOME_PATH=$ASCEND_HOME_PATH" >&2 + exit 1 +fi +if [[ "$PMU_VARIANT" -eq 1 && ! -x "$HCC" ]]; then + echo "The AICPU HCC compiler is missing under ASCEND_HOME_PATH=$ASCEND_HOME_PATH" >&2 + exit 1 +fi +if ! command -v "$READELF_BIN" >/dev/null 2>&1; then + echo "readelf is required to verify the mixed AICore ELF." >&2 + exit 1 +fi +if [[ "$BUILD_VARIANT" != "swimlane" ]] && ! command -v sha256sum >/dev/null 2>&1; then + echo "sha256sum is required to publish variant artifact manifests." >&2 + exit 1 +fi +if [[ ! -f "$PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" ]]; then + echo "PTO kernel metadata header is missing under $PTO_INCLUDE_ROOT/include" >&2 + exit 1 +fi +for header in pto/pto-inst.hpp pto/common/constants.hpp pto/common/pto_tile.hpp; do + if [[ ! -f "$PTO_INCLUDE_ROOT/include/$header" ]]; then + echo "PTO real-compute header is missing: $PTO_INCLUDE_ROOT/include/$header" >&2 + exit 1 + fi +done + +mkdir -p "$BUILD_DIR" +if [[ "$PMU_VARIANT" -eq 0 ]]; then + # 旧统一构建可能在根目录残留 PMU owner；swimlane 构建主动移除这两个 + # 不属于本变体的产物，避免 direct host 调用误加载上一版诊断 SO。 + rm -f \ + "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so" \ + "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so" +else + # manifest 是同一 phase 四件套唯一的“可运行”标记。重建一开始先使旧 + # manifest 失效；即使后续编译中断，run.sh 也不会消费目录里的半成品。 + rm -f -- "$BUILD_DIR/$SUBMIT_PMU_MANIFEST_NAME" +fi +if [[ "$LAZY_SAMPLE_VARIANT" -eq 1 ]]; then + rm -f -- \ + "$BUILD_DIR/$LAZY_SAMPLE_MANIFEST_NAME" \ + "$BUILD_DIR/$LAZY_SAMPLE_TEXT_LAYOUT_NAME" +fi + +# 关闭编译器自动插入的 scalar DCCI，由 kernel.cpp 中与 PA 对齐的显式失效/回写协议负责 cache 可见性。 +# 两种架构共用这些 ABI、栈和优化参数，避免 AIC/AIV 对共享 SchedulerState 产生不同解释。 +COMMON_FLAGS=( + -c -O3 -g -x cce -Wall -std=c++17 + --cce-aicore-only + -mllvm -cce-aicore-stack-size=0x8000 + -mllvm -cce-aicore-function-stack-size=0x8000 + -mllvm -cce-aicore-record-overflow=false + -mllvm -cce-aicore-addr-transform + -mllvm -cce-aicore-dcci-insert-for-scalar=false + -mllvm -cce-aicore-dcci-before-kernel-end=false + -I"$ROOT_DIR/common" + -I"$PTO_INCLUDE_ROOT/include" + "${VARIANT_DEFINES[@]}" +) + +# split finish 的完整 runtime state 为 1600B，超过 CCEC 默认保留的 +# block-local 栈空间。编译器 hidden help 明确该参数以 byte 为单位、上限 +# 4KiB。实测 1600B 与 2048B 虽生成相同大小的 .text，内容 SHA 却不同， +# 因此使用当前 ABI 的精确尺寸而不增加无依据余量，并严格限于 split 变体。 +LAZY_SAMPLE_SPLIT_STATE_BYTES=1600 +LAZY_SAMPLE_FINISH_CALL_SITES=5 +LAZY_SAMPLE_BLOCK_LOCAL_RESERVE_BYTES=0 +if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_BLOCK_LOCAL_RESERVE_BYTES=$LAZY_SAMPLE_SPLIT_STATE_BYTES + COMMON_FLAGS+=( + -mllvm -cce-block-local-relocate=true + -mllvm "-cce-block-local-reserve-size=$LAZY_SAMPLE_BLOCK_LOCAL_RESERVE_BYTES" + ) +fi + +# 同一入口源码分别面向 cube 与 vector ISA 编译，宏只选择各自的全局入口和 mixed metadata。 +echo "[BUILD] CCEC AIC entry (dav-c310-cube)" +"$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DPA_BUILD_AIC \ + -o "$BUILD_DIR/pa_scheduler_aic.o" \ + "$SCRIPT_DIR/kernel.cpp" + +echo "[BUILD] CCEC AIV entry (dav-c310-vec)" +"$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DPA_BUILD_AIV \ + -o "$BUILD_DIR/pa_scheduler_aiv.o" \ + "$SCRIPT_DIR/kernel.cpp" + +check_workload_dispatcher_object() { + local object_path="$1" + local expected_symbol="$2" + local wrong_role_symbol="$3" + local object_symbols + object_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$object_path")" + if ! awk -v name="$expected_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$object_symbols"; then + echo "Expected exactly one non-empty strong workload dispatcher in $object_path: $expected_symbol" >&2 + exit 1 + fi + if awk -v name="$wrong_role_symbol" \ + '$NF == name {found = 1} END {exit !found}' <<<"$object_symbols"; then + echo "Wrong-role workload dispatcher leaked into $object_path: $wrong_role_symbol" >&2 + exit 1 + fi +} +check_workload_dispatcher_object \ + "$BUILD_DIR/pa_scheduler_aic.o" \ + pa_execute_real_winner_workload_aic \ + pa_execute_real_winner_workload_aiv +check_workload_dispatcher_object \ + "$BUILD_DIR/pa_scheduler_aiv.o" \ + pa_execute_real_winner_workload_aiv \ + pa_execute_real_winner_workload_aic +echo "[CHECK] role-specific real-compute dispatchers are strong and do not cross roles" + +text_relocation_count_for_symbol() { + local object_path="$1" + local symbol_name="$2" + "$READELF_BIN" --relocs --wide "$object_path" | awk -v name="$symbol_name" ' + /^Relocation section '\''\.rela\.text'\''/ {in_text = 1; next} + /^Relocation section / {in_text = 0} + in_text { + for (column = 1; column <= NF; ++column) { + if ($column == name) { + count++ + next + } + } + } + END {print count + 0} + ' +} + +check_split_role_objects() { + local role="$1" + local wrong_role="$2" + local caller="$BUILD_DIR/pa_scheduler_${role}.o" + local runtime="$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_${role}.o" + local finish="$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_${role}.o" + local state_symbol="pa_scheduler_lazy_sample_callback_state_${role}" + local finish_symbol="pa_scheduler_lazy_sample_callback_finish_${role}" + local orchestration_symbol="pa_scheduler_lazy_sample_callback_orchestration_${role}" + local dispatcher_symbol="pa_execute_real_winner_workload_${role}" + local entry_symbol="pa_scheduler_0_mix_${role}" + local caller_symbols runtime_symbols finish_symbols + caller_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$caller")" + runtime_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$runtime")" + finish_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$finish")" + + if ! awk -v name="$orchestration_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$caller_symbols"; then + echo "Missing unique strong split orchestration in caller object: $orchestration_symbol" >&2 + exit 1 + fi + for imported in "$state_symbol" "$finish_symbol"; do + if ! awk -v name="$imported" \ + '$5 == "GLOBAL" && $7 == "UND" && $NF == name {count++} + END {exit count != 1}' <<<"$caller_symbols"; then + echo "Caller must import exactly one matching split symbol: $caller ($imported)" >&2 + exit 1 + fi + done + if [[ "$(text_relocation_count_for_symbol "$caller" "$finish_symbol")" -ne "$LAZY_SAMPLE_FINISH_CALL_SITES" ]]; then + echo "Caller must contain exactly $LAZY_SAMPLE_FINISH_CALL_SITES all-task split-finish .rela.text relocations: $caller" >&2 + exit 1 + fi + if [[ "$(text_relocation_count_for_symbol "$caller" "$state_symbol")" -eq 0 ]]; then + echo "Caller must access its matching external block-local state: $caller" >&2 + exit 1 + fi + if "$READELF_BIN" --sections --wide "$caller" | awk \ + 'index($0, ".ascend.meta.") != 0 {found = 1} END {exit !found}'; then + echo "Split caller object must not define launch metadata: $caller" >&2 + exit 1 + fi + + if ! awk -v name="$state_symbol" -v bytes="$LAZY_SAMPLE_SPLIT_STATE_BYTES" \ + '$4 == "OBJECT" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 == bytes {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime must own one exact-size block-local state: $runtime ($state_symbol)" >&2 + exit 1 + fi + if ! awk -v name="$entry_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime must own one non-empty mixed entry: $runtime ($entry_symbol)" >&2 + exit 1 + fi + if ! awk -v name="$orchestration_symbol" \ + '$5 == "GLOBAL" && $7 == "UND" && $NF == name {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime must strongly import one role-specific orchestration: $runtime ($orchestration_symbol)" >&2 + exit 1 + fi + if [[ "$(text_relocation_count_for_symbol "$runtime" "$orchestration_symbol")" -ne 1 ]]; then + echo "Runtime entry must contain exactly one orchestration call relocation: $runtime" >&2 + exit 1 + fi + local block_local_record block_local_section_index block_local_size_hex block_local_alignment + block_local_record="$( + "$READELF_BIN" --sections --wide "$runtime" | awk ' + {for (column = 1; column <= NF; ++column) { + if ($column == ".bl.uninit") { + section_index = $(column - 1) + gsub(/\[/, "", section_index) + gsub(/\]/, "", section_index) + print section_index, $(column + 4), $NF + exit + } + }} + ' + )" + read -r block_local_section_index block_local_size_hex block_local_alignment \ + <<<"$block_local_record" + if [[ -z "$block_local_section_index" || -z "$block_local_size_hex" || + $((16#$block_local_size_hex)) -ne "$LAZY_SAMPLE_SPLIT_STATE_BYTES" || + "$block_local_alignment" -ne 64 ]]; then + echo "Runtime block-local section must be exact-size and 64B aligned: $runtime" >&2 + exit 1 + fi + if ! awk -v name="$state_symbol" -v section="$block_local_section_index" \ + '$4 == "OBJECT" && $7 == section && $NF == name {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime state must be defined in its exact .bl.uninit section: $runtime" >&2 + exit 1 + fi + local runtime_sections + runtime_sections="$("$READELF_BIN" --sections --wide "$runtime")" + if ! awk -v name=".ascend.meta.$entry_symbol" ' + {for (column = 1; column <= NF; ++column) { + if ($column == name) found = 1 + }} + END {exit !found} + ' <<<"$runtime_sections"; then + echo "Runtime object is missing matching mixed-entry metadata: $runtime" >&2 + exit 1 + fi + if awk -v name=".ascend.meta.pa_scheduler_0_mix_${wrong_role}" ' + {for (column = 1; column <= NF; ++column) { + if ($column == name) found = 1 + }} + END {exit !found} + ' <<<"$runtime_sections"; then + echo "Wrong-role mixed-entry metadata leaked into runtime object: $runtime" >&2 + exit 1 + fi + + if ! awk -v name="$finish_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$finish_symbols"; then + echo "Finish object must define one non-empty strong finish: $finish ($finish_symbol)" >&2 + exit 1 + fi + for imported in "$state_symbol" "$dispatcher_symbol"; do + if ! awk -v name="$imported" \ + '$5 == "GLOBAL" && $7 == "UND" && $NF == name {count++} + END {exit count != 1}' <<<"$finish_symbols"; then + echo "Finish object must import exactly one matching symbol: $finish ($imported)" >&2 + exit 1 + fi + if [[ "$(text_relocation_count_for_symbol "$finish" "$imported")" -eq 0 ]]; then + echo "Finish object must reference its matching imported symbol: $finish ($imported)" >&2 + exit 1 + fi + done + if "$READELF_BIN" --sections --wide "$finish" | awk \ + 'index($0, ".ascend.meta.") != 0 {found = 1} END {exit !found}'; then + echo "Split finish object must not define launch metadata: $finish" >&2 + exit 1 + fi + + local forbidden symbol_table object_path + for object_path in "$caller" "$runtime" "$finish"; do + case "$object_path" in + "$caller") symbol_table="$caller_symbols" ;; + "$runtime") symbol_table="$runtime_symbols" ;; + *) symbol_table="$finish_symbols" ;; + esac + for forbidden in \ + "pa_scheduler_lazy_sample_callback_state_${wrong_role}" \ + "pa_scheduler_lazy_sample_callback_finish_${wrong_role}" \ + "pa_scheduler_lazy_sample_callback_orchestration_${wrong_role}" \ + "pa_execute_real_winner_workload_${wrong_role}" \ + "pa_scheduler_0_mix_${wrong_role}"; do + if awk -v name="$forbidden" \ + '$NF == name {found = 1} END {exit !found}' <<<"$symbol_table"; then + echo "Wrong-role split symbol leaked into $object_path: $forbidden" >&2 + exit 1 + fi + done + done + + for forbidden in "$entry_symbol"; do + if awk -v name="$forbidden" '$NF == name {found = 1} END {exit !found}' \ + <<<"$caller_symbols"; then + echo "Split caller must not own a launch entry: $caller ($forbidden)" >&2 + exit 1 + fi + if awk -v name="$forbidden" '$NF == name {found = 1} END {exit !found}' \ + <<<"$finish_symbols"; then + echo "Split finish must not own a launch entry: $finish ($forbidden)" >&2 + exit 1 + fi + done + for forbidden in "$finish_symbol" "$dispatcher_symbol"; do + if awk -v name="$forbidden" '$NF == name {found = 1} END {exit !found}' \ + <<<"$runtime_symbols"; then + echo "Runtime entry/state owner contains an unexpected split helper: $runtime ($forbidden)" >&2 + exit 1 + fi + done + if awk -v name="$orchestration_symbol" '$NF == name {found = 1} END {exit !found}' \ + <<<"$finish_symbols"; then + echo "Split finish must not contain orchestration: $finish ($orchestration_symbol)" >&2 + exit 1 + fi +} + +if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + echo "[BUILD] CCEC AIC split runtime entry/state owner (dav-c310-cube)" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DPA_BUILD_AIC \ + -o "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_aic.o" \ + "$SCRIPT_DIR/callback_runtime_entry.cpp" + echo "[BUILD] CCEC AIC all-task split finish (dav-c310-cube)" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DPA_BUILD_AIC \ + -o "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_aic.o" \ + "$SCRIPT_DIR/callback_finish.cpp" + echo "[BUILD] CCEC AIV split runtime entry/state owner (dav-c310-vec)" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DPA_BUILD_AIV \ + -o "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_aiv.o" \ + "$SCRIPT_DIR/callback_runtime_entry.cpp" + echo "[BUILD] CCEC AIV all-task split finish (dav-c310-vec)" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DPA_BUILD_AIV \ + -o "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_aiv.o" \ + "$SCRIPT_DIR/callback_finish.cpp" + check_split_role_objects aic aiv + check_split_role_objects aiv aic + echo "[CHECK] split caller/runtime/finish objects satisfy role, state, metadata, and call-boundary gates" + DEVICE_OBJECTS=( + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_aic.o" + "$BUILD_DIR/pa_scheduler_aic.o" + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_aic.o" + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_aiv.o" + "$BUILD_DIR/pa_scheduler_aiv.o" + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_aiv.o" + ) +else + DEVICE_OBJECTS=( + "$BUILD_DIR/pa_scheduler_aic.o" + "$BUILD_DIR/pa_scheduler_aiv.o" + ) +fi + +# 静态链接把两个 device object 合成一个可由 runtime 按 1:2 比例启动的 mixed AICore ELF。 +echo "[BUILD] Static 1:2 mixed AICore ELF" +"$LD" -m aicorelinux -Ttext=0 -static \ + --version-script="$SCRIPT_DIR/pa_scheduler_device_exports.map" \ + -o "$BUILD_DIR/pa_scheduler_kernel.o" \ + "${DEVICE_OBJECTS[@]}" + +SYMBOL_TABLE="$("$READELF_BIN" --symbols --wide --sym-base=10 "$BUILD_DIR/pa_scheduler_kernel.o")" +SECTION_TABLE="$("$READELF_BIN" --sections --wide "$BUILD_DIR/pa_scheduler_kernel.o")" +# 构建成功不等于 mixed launch 可用：同时检查两个入口符号及其 metadata section，缺一即拒绝产物。 +# `set -e` 同时保证 readelf 自身失败时不会拿空字符串继续做伪检查。 +for entry in pa_scheduler_0_mix_aic pa_scheduler_0_mix_aiv; do + if ! awk -v name="$entry" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 != "0" && $3 != "0x0" {found = 1} END {exit !found}' \ + <<<"$SYMBOL_TABLE"; then + echo "Missing non-empty defined GLOBAL mixed-kernel entry: $entry" >&2 + exit 1 + fi + if ! awk -v name=".ascend.meta.$entry" ' + {for (column = 1; column <= NF; ++column) { + if ($column == name) found = 1 + }} + END {exit !found} + ' <<<"$SECTION_TABLE"; then + echo "Missing mixed-kernel metadata section: .ascend.meta.$entry" >&2 + exit 1 + fi +done +echo "[CHECK] both 1:2 mixed entries and metadata sections are present" + +# A5 runtime 会把已定义的 GLOBAL FUNC 当作可启动候选；最终 device ELF 只允许 +# 两个带 metadata 的 mixed 入口暴露为全局函数。任何新增 helper 都必须保持 LOCAL。 +while IFS= read -r global_func; do + case "$global_func" in + pa_scheduler_0_mix_aic|pa_scheduler_0_mix_aiv) ;; + *) + echo "Unexpected GLOBAL device function (possible kernel-entry pollution): $global_func" >&2 + exit 1 + ;; + esac +done < <(awk '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" {print $NF}' <<<"$SYMBOL_TABLE") +echo "[CHECK] only the two mixed entries are exported as GLOBAL device functions" + +# runtime-finish TU 后续需要复用同一个真计算 dispatcher，因此 caller object +# 按核型提供 strong 定义；version script 必须把它们在最终 mixed ELF 中重新 +# 局部化。这里同时检查两个 role-specific dispatcher 与底层 Cube/Vector 实体， +# 禁止因抽取 adapter 漏掉任一真实负载路径。 +for workload_symbol in \ + pa_execute_real_winner_workload_aic \ + pa_execute_real_winner_workload_aiv \ + pa_real_cube_workload_aic \ + pa_real_vector_add_workload_aiv \ + pa_real_vector_mul_workload_aiv; do + workload_size="$( + awk -v name="$workload_symbol" \ + '$4 == "FUNC" && $5 == "LOCAL" && $7 != "UND" && index($NF, name) != 0 && $3 + 0 > 0 {print $3; exit}' \ + <<<"$SYMBOL_TABLE" + )" + if [[ -z "$workload_size" ]]; then + echo "Missing non-empty LOCAL CCEC real-compute workload function: $workload_symbol" >&2 + exit 1 + fi + if awk -v name="$workload_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && index($NF, name) != 0 {found = 1} END {exit !found}' \ + <<<"$SYMBOL_TABLE"; then + echo "CCEC real-compute helper must not be a GLOBAL kernel candidate: $workload_symbol" >&2 + exit 1 + fi +done +echo "[CHECK] CCEC cube/vector real-compute helpers are non-empty LOCAL functions" + +if [[ "$LAZY_SAMPLE_VARIANT" -eq 1 ]]; then + # callback/builder/front must remain inline in both families. Only split + # shapes may retain the two fixed, role-specific runtime finish functions. + if awk \ + '$4 == "FUNC" && $7 != "UND" && + (index($NF, "LazySampleCallback") != 0 || index($NF, "SubmitQk") != 0 || + index($NF, "LazySampleSplitState") != 0) {found = 1} + END {exit !found}' <<<"$SYMBOL_TABLE"; then + echo "lazy sample callback builder/front unexpectedly survived as an out-of-line device function." >&2 + exit 1 + fi + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + for role in aic aiv; do + finish_symbol="pa_scheduler_lazy_sample_callback_finish_${role}" + if ! awk -v name="$finish_symbol" \ + '$4 == "FUNC" && $5 == "LOCAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Missing unique non-empty LOCAL split finish in final ELF: $finish_symbol" >&2 + exit 1 + fi + orchestration_symbol="pa_scheduler_lazy_sample_callback_orchestration_${role}" + if ! awk -v name="$orchestration_symbol" \ + '$4 == "FUNC" && $5 == "LOCAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Missing unique non-empty LOCAL split orchestration in final ELF: $orchestration_symbol" >&2 + exit 1 + fi + state_symbol="pa_scheduler_lazy_sample_callback_state_${role}" + if ! awk -v name="$state_symbol" -v bytes="$LAZY_SAMPLE_SPLIT_STATE_BYTES" \ + '$4 == "OBJECT" && $5 == "LOCAL" && $7 != "UND" && $NF == name && $3 + 0 == bytes {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Missing unique exact-size LOCAL split state in final ELF: $state_symbol" >&2 + exit 1 + fi + done + aic_state_hex="$(awk '$NF == "pa_scheduler_lazy_sample_callback_state_aic" {print $2; exit}' <<<"$SYMBOL_TABLE")" + aiv_state_hex="$(awk '$NF == "pa_scheduler_lazy_sample_callback_state_aiv" {print $2; exit}' <<<"$SYMBOL_TABLE")" + final_block_local_record="$( + awk '{for (column = 1; column <= NF; ++column) { + if ($column == ".bl_uninit") { + section_index = $(column - 1) + gsub(/\[/, "", section_index) + gsub(/\]/, "", section_index) + print section_index, $(column + 4), $NF + exit + } + }}' <<<"$SECTION_TABLE" + )" + read -r final_block_local_section_index final_block_local_size_hex \ + final_block_local_alignment \ + <<<"$final_block_local_record" + if [[ -z "$aic_state_hex" || -z "$aiv_state_hex" || + $((16#$aic_state_hex)) -ne 0 || + $((16#$aiv_state_hex)) -ne "$LAZY_SAMPLE_SPLIT_STATE_BYTES" || + -z "$final_block_local_section_index" || -z "$final_block_local_size_hex" || + $((16#$final_block_local_size_hex)) -ne $((2 * LAZY_SAMPLE_SPLIT_STATE_BYTES)) || + "$final_block_local_alignment" -ne 64 ]]; then + echo "Final split block-local layout must be two exact, non-overlapping 64B-aligned states." >&2 + exit 1 + fi + for state_symbol in \ + pa_scheduler_lazy_sample_callback_state_aic \ + pa_scheduler_lazy_sample_callback_state_aiv; do + if ! awk -v name="$state_symbol" -v section="$final_block_local_section_index" \ + '$4 == "OBJECT" && $7 == section && $NF == name {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Final split state must be bound to the exact .bl_uninit section: $state_symbol" >&2 + exit 1 + fi + done + echo "[CHECK] split finishes/orchestrations/states are LOCAL and final block-local layout is exact" + fi + if [[ -n "$("$READELF_BIN" --relocs --wide "$BUILD_DIR/pa_scheduler_kernel.o" | sed -n '/Relocation section/p')" ]]; then + echo "Final lazy sample callback mixed ELF must not retain relocations." >&2 + exit 1 + fi + echo "[CHECK] lazy sample callback builder/front is inline and final ELF has no relocations" +fi + +check_icache_probe_layout() { + local role="$1" + local target="pa_icache_target_${role}" + local harness="pa_icache_measure_${role}" + local thrash="pa_icache_thrash_${role}" + local target_record + local harness_record + local thrash_record + target_record="$(awk -v name="$target" '$4 == "FUNC" && index($NF, name) != 0 {print $2, $3; exit}' <<<"$SYMBOL_TABLE")" + harness_record="$(awk -v name="$harness" '$4 == "FUNC" && index($NF, name) != 0 {print $2, $3; exit}' <<<"$SYMBOL_TABLE")" + thrash_record="$(awk -v name="$thrash" '$4 == "FUNC" && index($NF, name) != 0 {print $2, $3; exit}' <<<"$SYMBOL_TABLE")" + if [[ -z "$target_record" || -z "$harness_record" || -z "$thrash_record" ]]; then + echo "Missing I-cache probe symbols for $role" >&2 + exit 1 + fi + + local target_hex target_size harness_hex harness_size thrash_hex thrash_size + read -r target_hex target_size <<<"$target_record" + read -r harness_hex harness_size <<<"$harness_record" + read -r thrash_hex thrash_size <<<"$thrash_record" + local target_address=$((16#$target_hex)) + local harness_address=$((16#$harness_hex)) + local thrash_address=$((16#$thrash_hex)) + if (( target_address % 128 != 0 || target_size == 0 || target_size > 16 )); then + echo "Invalid single-fetch-block I-cache target for $role: address=0x$target_hex size=$target_size" >&2 + exit 1 + fi + if (( thrash_size < 65536 )); then + echo "I-cache thrash body is smaller than 64 KiB for $role: size=$thrash_size" >&2 + exit 1 + fi + if (( harness_address % 128 != 0 || target_address + 128 > harness_address || + harness_address + harness_size > thrash_address )); then + echo "I-cache layout must be target -> harness -> thrash for $role" >&2 + exit 1 + fi + echo "[CHECK] $role I-cache target=0x$target_hex/$target_size harness=0x$harness_hex/$harness_size "\ + "thrash=0x$thrash_hex/$thrash_size" +} + +emit_text_section_fingerprint() { + local object_path="$1" + local artifact_name + artifact_name="$(basename "$object_path")" + local text_record text_address_hex text_offset_hex text_size_hex + text_record="$( + "$READELF_BIN" --sections --wide "$object_path" | awk ' + {for (column = 1; column <= NF; ++column) { + if ($column == ".text") { + print $(column + 2), $(column + 3), $(column + 4) + exit + } + }} + ' + )" + read -r text_address_hex text_offset_hex text_size_hex <<<"$text_record" + if [[ -z "$text_address_hex" || -z "$text_offset_hex" || -z "$text_size_hex" ]]; then + echo "Cannot fingerprint missing .text section: $object_path" >&2 + return 1 + fi + local text_size=$((16#$text_size_hex)) + local text_sha + text_sha="$( + dd if="$object_path" bs=1 skip=$((16#$text_offset_hex)) count="$text_size" status=none | + sha256sum | awk '{print $1}' + )" + printf 'text %s %u %s\n' "$artifact_name" "$text_size" "$text_sha" +} + +emit_symbol_body_fingerprint() { + local object_path="$1" + local symbol_name="$2" + local artifact_name + artifact_name="$(basename "$object_path")" + local symbol_record symbol_address_hex symbol_size + symbol_record="$( + "$READELF_BIN" --symbols --wide --sym-base=10 "$object_path" | awk -v name="$symbol_name" ' + $4 == "FUNC" && $7 != "UND" && $NF == name && $3 + 0 > 0 { + count++ + address = $2 + size = $3 + } + END { + if (count != 1) exit 1 + print address, size + } + ' + )" || { + echo "Cannot fingerprint non-unique or empty function: $object_path ($symbol_name)" >&2 + return 1 + } + read -r symbol_address_hex symbol_size <<<"$symbol_record" + + local text_record text_address_hex text_offset_hex text_size_hex + text_record="$( + "$READELF_BIN" --sections --wide "$object_path" | awk ' + {for (column = 1; column <= NF; ++column) { + if ($column == ".text") { + print $(column + 2), $(column + 3), $(column + 4) + exit + } + }} + ' + )" + read -r text_address_hex text_offset_hex text_size_hex <<<"$text_record" + if [[ -z "$text_address_hex" || -z "$text_offset_hex" || -z "$text_size_hex" ]]; then + echo "Cannot locate .text for function fingerprint: $object_path ($symbol_name)" >&2 + return 1 + fi + local symbol_address=$((16#$symbol_address_hex)) + local text_address=$((16#$text_address_hex)) + local text_size=$((16#$text_size_hex)) + local relative_offset=$((symbol_address - text_address)) + if (( relative_offset < 0 || symbol_size <= 0 || relative_offset + symbol_size > text_size )); then + echo "Function body lies outside .text: $object_path ($symbol_name)" >&2 + return 1 + fi + local symbol_sha + symbol_sha="$( + dd if="$object_path" bs=1 \ + skip=$((16#$text_offset_hex + relative_offset)) \ + count="$symbol_size" status=none | + sha256sum | awk '{print $1}' + )" + printf 'symbol %s %s %u %s\n' \ + "$artifact_name" "$symbol_name" "$symbol_size" "$symbol_sha" +} + +# 两个正式 ELF 都不携带旧 cold/warm 校准冲刷体；submit-pmu 只观察真实 +# Submit。保留上面的检查函数供历史布局取证时复核，但正式构建不调用它。 + +# PMU selector/CTRL 的所有权必须由主 aicpu_scheduler 配置并在退出前恢复。 +# standalone 目录内自带 Path-A dispatcher 与 owner：前者负责把 owner SO +# 落到设备预安装目录，后者由 mode=0 JSON 注册并通过统一入口执行命令。 +# swimlane 构建不生成 PMU owner/dispatcher；submit-pmu 则把 kernel、host、 +# owner 与 dispatcher 全部放在同一个 phase 目录，禁止跨 phase 复用。 +if [[ "$PMU_VARIANT" -eq 1 ]]; then + echo "[BUILD] self-contained AICPU PMU dispatcher" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra -Werror \ + -Wl,--build-id \ + "$SCRIPT_DIR/pmu_owner_dispatcher.cpp" \ + -o "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so" + + echo "[BUILD] self-contained AICPU PMU owner" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra -Werror \ + -Wl,--build-id \ + "${VARIANT_DEFINES[@]}" \ + -I"$SCRIPT_DIR" \ + "$SCRIPT_DIR/pmu_owner_aicpu.cpp" \ + -o "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so" + + OWNER_HEADER="$("$READELF_BIN" --file-header "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so")" + OWNER_SYMBOLS="$("$READELF_BIN" --dyn-syms --wide "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so")" + DISPATCHER_HEADER="$("$READELF_BIN" --file-header "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so")" + DISPATCHER_SYMBOLS="$("$READELF_BIN" --dyn-syms --wide "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so")" + if [[ "$OWNER_HEADER" != *"Type: DYN"* || + "$OWNER_HEADER" != *"Machine: AArch64"* || + "$DISPATCHER_HEADER" != *"Type: DYN"* || + "$DISPATCHER_HEADER" != *"Machine: AArch64"* ]]; then + echo "PMU dispatcher and owner must both be AArch64 shared objects." >&2 + exit 1 + fi + if [[ "$OWNER_SYMBOLS" != *" simpler_aicpu_exec"* ]]; then + echo "Missing main AICPU PMU owner entry: simpler_aicpu_exec" >&2 + exit 1 + fi + for entry in StaticTileFwkBackendKernelServer DynTileFwkBackendKernelServerInit DynTileFwkBackendKernelServer; do + if [[ "$DISPATCHER_SYMBOLS" != *" $entry"* ]]; then + echo "Missing AICPU PMU dispatcher entry: $entry" >&2 + exit 1 + fi + done + echo "[CHECK] Path-A dispatcher and main AICPU PMU owner exports are present" +fi + +# host runner 只链接用户 CANN 9.1 的 ACL/runtime，并写入同一安装目录的 rpath，运行时不需要 simpler 动态库。 +# `-Werror` 让 host API 签名或尺寸类型变化在构建期暴露，避免到上板阶段才出现参数截断。 +echo "[BUILD] CCEC host runner" +"$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror -Wno-deprecated-declarations \ + "${VARIANT_DEFINES[@]}" \ + -I"$ROOT_DIR/common" \ + -I"$ASCEND_HOME_PATH/include" \ + -I"$ASCEND_HOME_PATH/pkg_inc" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime/runtime" \ + "$SCRIPT_DIR/host.cpp" \ + -L"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -Wl,-rpath,"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -lascendcl -lruntime \ + -ldl \ + -o "$BUILD_DIR/pa_scheduler_host" + +if [[ "$PMU_VARIANT" -eq 1 ]]; then + # host、kernel、owner、dispatcher 全部成功后才生成 manifest；校验和使用 + # 相对文件名，目录复制后仍可在 run 前原样复核。临时文件与最终文件位于 + # 同一目录，mv 只承担单文件原子发布，不会暴露半写 manifest。 + SUBMIT_PMU_ARTIFACTS=( + pa_scheduler_host + pa_scheduler_kernel.o + libpa_scheduler_pmu_owner_aicpu.so + libpa_scheduler_pmu_owner_dispatcher.so + ) + for artifact in "${SUBMIT_PMU_ARTIFACTS[@]}"; do + if [[ ! -s "$BUILD_DIR/$artifact" ]]; then + echo "Cannot publish submit-pmu manifest; artifact is missing or empty: $artifact" >&2 + exit 1 + fi + done + if [[ ! -x "$BUILD_DIR/pa_scheduler_host" ]]; then + echo "Cannot publish submit-pmu manifest; host runner is not executable." >&2 + exit 1 + fi + + MANIFEST_PATH="$BUILD_DIR/$SUBMIT_PMU_MANIFEST_NAME" + MANIFEST_TMP="$(mktemp "$BUILD_DIR/.${SUBMIT_PMU_MANIFEST_NAME}.tmp.XXXXXX")" + cleanup_manifest_tmp() { + if [[ -n "${MANIFEST_TMP:-}" ]]; then + rm -f -- "$MANIFEST_TMP" + fi + } + trap cleanup_manifest_tmp EXIT + { + printf '# schema=pa_scheduler_submit_pmu_artifacts/v1\n' + printf '# variant=submit-pmu\n' + printf '# phase=%s\n' "$PHASE_NAME" + printf '# phase_id=%u\n' "$PHASE_ID" + (cd "$BUILD_DIR" && sha256sum "${SUBMIT_PMU_ARTIFACTS[@]}") + } > "$MANIFEST_TMP" + mv -f -- "$MANIFEST_TMP" "$MANIFEST_PATH" + MANIFEST_TMP="" + trap - EXIT + echo "[CHECK] submit-pmu artifact manifest published: $MANIFEST_PATH" +fi +if [[ "$LAZY_SAMPLE_VARIANT" -eq 1 ]]; then + TEXT_LAYOUT_PATH="$BUILD_DIR/$LAZY_SAMPLE_TEXT_LAYOUT_NAME" + TEXT_LAYOUT_TMP="$(mktemp "$BUILD_DIR/.${LAZY_SAMPLE_TEXT_LAYOUT_NAME}.tmp.XXXXXX")" + cleanup_text_layout_tmp() { + if [[ -n "${TEXT_LAYOUT_TMP:-}" ]]; then + rm -f -- "$TEXT_LAYOUT_TMP" + fi + } + trap cleanup_text_layout_tmp EXIT + { + printf '# schema=pa_scheduler_device_text_layout/v1\n' + printf '# backend=ccec\n' + printf '# shape=%s\n' "$LAZY_SAMPLE_SHAPE" + printf '# block_local_reserve_bytes=%u\n' "$LAZY_SAMPLE_BLOCK_LOCAL_RESERVE_BYTES" + printf '# compiler=%s\n' '$ASCEND_HOME_PATH/bin/ccec' + printf '# compiler_sha256=%s\n' "$(sha256sum "$CCEC" | awk '{print $1}')" + printf '# linker=%s\n' '$ASCEND_HOME_PATH/bin/ld.lld' + printf '# linker_sha256=%s\n' "$(sha256sum "$LD" | awk '{print $1}')" + emit_text_section_fingerprint "$BUILD_DIR/pa_scheduler_kernel.o" + for object_path in "${DEVICE_OBJECTS[@]}"; do + emit_text_section_fingerprint "$object_path" + done + for role in aic aiv; do + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_kernel.o" "pa_scheduler_0_mix_${role}" + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_runtime_${role}.o" \ + "pa_scheduler_0_mix_${role}" + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_${role}.o" \ + "pa_scheduler_lazy_sample_callback_orchestration_${role}" + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_lazy_sample_callback_finish_${role}.o" \ + "pa_scheduler_lazy_sample_callback_finish_${role}" + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_kernel.o" \ + "pa_scheduler_lazy_sample_callback_orchestration_${role}" + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_kernel.o" \ + "pa_scheduler_lazy_sample_callback_finish_${role}" + else + emit_symbol_body_fingerprint \ + "$BUILD_DIR/pa_scheduler_${role}.o" "pa_scheduler_0_mix_${role}" + fi + done + } > "$TEXT_LAYOUT_TMP" + mv -f -- "$TEXT_LAYOUT_TMP" "$TEXT_LAYOUT_PATH" + TEXT_LAYOUT_TMP="" + trap - EXIT + awk '$1 == "text" { + printf "[TEXT] %s size=%s sha256=%s\n", $2, $3, $4 + }' "$TEXT_LAYOUT_PATH" + echo "[CHECK] lazy sample callback device .text layout manifest published: $TEXT_LAYOUT_PATH" + + LAZY_SAMPLE_ARTIFACTS=( + pa_scheduler_host + pa_scheduler_kernel.o + ) + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_lazy_sample_callback_runtime_aic.o) + fi + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_aic.o) + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_lazy_sample_callback_finish_aic.o) + fi + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_lazy_sample_callback_runtime_aiv.o) + fi + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_aiv.o) + if [[ "$LAZY_SAMPLE_SPLIT" -eq 1 ]]; then + LAZY_SAMPLE_ARTIFACTS+=(pa_scheduler_lazy_sample_callback_finish_aiv.o) + fi + LAZY_SAMPLE_ARTIFACTS+=("$LAZY_SAMPLE_TEXT_LAYOUT_NAME") + for artifact in "${LAZY_SAMPLE_ARTIFACTS[@]}"; do + if [[ ! -s "$BUILD_DIR/$artifact" ]]; then + echo "Cannot publish lazy sample callback manifest; artifact is missing or empty: $artifact" >&2 + exit 1 + fi + done + if [[ ! -x "$BUILD_DIR/pa_scheduler_host" ]]; then + echo "Cannot publish lazy sample callback manifest; host runner is not executable." >&2 + exit 1 + fi + MANIFEST_PATH="$BUILD_DIR/$LAZY_SAMPLE_MANIFEST_NAME" + MANIFEST_TMP="$(mktemp "$BUILD_DIR/.${LAZY_SAMPLE_MANIFEST_NAME}.tmp.XXXXXX")" + cleanup_callback_manifest_tmp() { + if [[ -n "${MANIFEST_TMP:-}" ]]; then + rm -f -- "$MANIFEST_TMP" + fi + } + trap cleanup_callback_manifest_tmp EXIT + { + printf '# schema=pa_scheduler_lazy_sample_callback_artifacts/v1\n' + printf '# backend=ccec\n' + printf '# shape=%s\n' "$LAZY_SAMPLE_SHAPE" + printf '# shape_id=%u\n' "$LAZY_SAMPLE_SHAPE_ID" + printf '# observation=%s\n' "$LAZY_SAMPLE_OBSERVATION" + (cd "$BUILD_DIR" && sha256sum "${LAZY_SAMPLE_ARTIFACTS[@]}") + } > "$MANIFEST_TMP" + mv -f -- "$MANIFEST_TMP" "$MANIFEST_PATH" + MANIFEST_TMP="" + trap - EXIT + echo "[CHECK] lazy sample callback artifact manifest published: $MANIFEST_PATH" +fi + +echo "[BUILD] complete: $BUILD_DIR" diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/callback_finish.cpp b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/callback_finish.cpp new file mode 100644 index 0000000000..9422fef200 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/callback_finish.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include +#include +#include + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" +#include "ccec_ops.h" + +using pa_scheduler_ccec::CcecOps; + +extern "C" { +#if defined(PA_BUILD_AIC) +__attribute__((noinline)) __aicore__ uint32_t +pa_scheduler_lazy_sample_callback_finish_aic(const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args) { + return pa_scheduler::FinishSplitLazySampleCallbackFromRuntime(ticket, args); +} +#elif defined(PA_BUILD_AIV) +__attribute__((noinline)) __aicore__ uint32_t +pa_scheduler_lazy_sample_callback_finish_aiv(const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args) { + return pa_scheduler::FinishSplitLazySampleCallbackFromRuntime(ticket, args); +} +#else +#error "Compile split finish with PA_BUILD_AIC or PA_BUILD_AIV" +#endif +} diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/callback_finish_api.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/callback_finish_api.h new file mode 100644 index 0000000000..9c87d77675 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/callback_finish_api.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#ifndef TESTS_ATOMIC_PROBE_LAZY_LAMDA_SAMPLE_CCEC_CALLBACK_FINISH_API_H +#define TESTS_ATOMIC_PROBE_LAZY_LAMDA_SAMPLE_CCEC_CALLBACK_FINISH_API_H + +#if !defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +#error "callback_finish_api.h is only valid for split-finish artifacts" +#endif + +// The runtime TU owns the role-specific block-local object. The caller only +// imports it; no weak/generic state symbol is allowed in the mixed AIC/AIV ELF. +extern "C" { +#if defined(PA_BUILD_AIC) +[[block_local]] extern pa_scheduler::LazySampleSplitRuntimeState pa_scheduler_lazy_sample_callback_state_aic; +__aicore__ uint32_t +pa_scheduler_lazy_sample_callback_finish_aic(const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args); +#elif defined(PA_BUILD_AIV) +[[block_local]] extern pa_scheduler::LazySampleSplitRuntimeState pa_scheduler_lazy_sample_callback_state_aiv; +__aicore__ uint32_t +pa_scheduler_lazy_sample_callback_finish_aiv(const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args); +#else +#error "Compile split finish with PA_BUILD_AIC or PA_BUILD_AIV" +#endif +} + +#endif // TESTS_ATOMIC_PROBE_LAZY_LAMDA_SAMPLE_CCEC_CALLBACK_FINISH_API_H diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/callback_runtime_entry.cpp b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/callback_runtime_entry.cpp new file mode 100644 index 0000000000..56adcd3a71 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/callback_runtime_entry.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" + +extern "C" { +#if defined(PA_BUILD_AIC) +[[block_local]] pa_scheduler::LazySampleSplitRuntimeState pa_scheduler_lazy_sample_callback_state_aic; +__aicore__ void +pa_scheduler_lazy_sample_callback_orchestration_aic(__gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id); +#elif defined(PA_BUILD_AIV) +[[block_local]] pa_scheduler::LazySampleSplitRuntimeState pa_scheduler_lazy_sample_callback_state_aiv; +__aicore__ void +pa_scheduler_lazy_sample_callback_orchestration_aiv(__gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id); +#else +#error "Compile split runtime entry with PA_BUILD_AIC or PA_BUILD_AIV" +#endif +} + +#if defined(PA_BUILD_AIC) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aic, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aic(__gm__ pa_scheduler::SchedulerState *state) { + const uint32_t worker_id = static_cast(get_block_idx()); + pa_scheduler_lazy_sample_callback_orchestration_aic(state, worker_id); +} +#elif defined(PA_BUILD_AIV) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aiv, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aiv(__gm__ pa_scheduler::SchedulerState *state) { + const uint32_t vector_id = static_cast(get_block_idx() * get_subblockdim() + get_subblockid()); + const uint32_t worker_id = pa_scheduler::kAicWorkers + vector_id; + pa_scheduler_lazy_sample_callback_orchestration_aiv(state, worker_id); +} +#endif diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/ccec_ops.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/ccec_ops.h new file mode 100644 index 0000000000..9a64d8bba1 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/ccec_ops.h @@ -0,0 +1,408 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#ifndef TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CCEC_OPS_H +#define TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CCEC_OPS_H + +// This private header is included after the CCEC/PTO headers and +// common/pa_scheduler_core.h. kernel.cpp defines the implementation switch +// exactly once per architecture; split runtime TUs consume only the external +// dispatcher declaration and the shared inline adapter. +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +#include "callback_finish_api.h" +#endif + +#if defined(PA_BUILD_AIC) +extern "C" __attribute__((noinline)) __aicore__ void pa_execute_real_winner_workload_aic( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +); +#elif defined(PA_BUILD_AIV) +extern "C" __attribute__((noinline)) __aicore__ void pa_execute_real_winner_workload_aiv( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +); +#else +#error "Compile CcecOps with PA_BUILD_AIC or PA_BUILD_AIV" +#endif + +namespace pa_scheduler_ccec { + +using namespace pto; + +template +__aicore__ inline void EmitNops() { +#pragma unroll + for (uint32_t index = 0; index < Count; ++index) { + asm volatile("nop"); + } +} + +__aicore__ inline void RuntimeNop(uint32_t count) { + // 两侧全流水屏障把可调 NOP 段限定为 kernel 模拟体，避免前后调度访存进入被测计算区间。 + __builtin_cce_pipe_barrier(PIPE_ALL); + while (count >= 256) { + EmitNops<256>(); + count -= 256; + } + if ((count & 128U) != 0) EmitNops<128>(); + if ((count & 64U) != 0) EmitNops<64>(); + if ((count & 32U) != 0) EmitNops<32>(); + if ((count & 16U) != 0) EmitNops<16>(); + if ((count & 8U) != 0) EmitNops<8>(); + if ((count & 4U) != 0) EmitNops<4>(); + if ((count & 2U) != 0) EmitNops<2>(); + if ((count & 1U) != 0) EmitNops<1>(); + __builtin_cce_pipe_barrier(PIPE_ALL); +} + +#if defined(PA_CCEC_OPS_DEFINE_REAL_WORKLOAD) && defined(PA_BUILD_AIC) +// QK/PV 的首版真实负载使用完整的 128x128 float cube 路径。输入来自独立 GM +// workspace，输出属于当前 worker；每次迭代都等 FIX 写回 GM 后再复用 L0C， +// 因而函数返回就是该模拟 task 的完成边界，而不是单纯的指令发射边界。 +static __aicore__ __attribute__((noinline, used)) void pa_real_cube_workload_aic( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + constexpr int kTile = static_cast(pa_scheduler::winner_workload::kTileRows); + constexpr int kBlockAlign = C0_SIZE_BYTE / sizeof(float); + static_assert(kTile % 16 == 0, "cube M must be 16-aligned"); + static_assert(kTile % kBlockAlign == 0, "cube K/N must satisfy C0 alignment"); + + using GlobalData = GlobalTensor< + float, Shape<1, 1, 1, kTile, kTile>, + pto::Stride>; + using TileMatA = Tile< + TileType::Mat, float, kTile, kTile, BLayout::ColMajor, + kTile, kTile, SLayout::RowMajor, 512>; + using TileMatB = Tile< + TileType::Mat, float, kTile, kTile, BLayout::ColMajor, + kTile, kTile, SLayout::RowMajor, 512>; + using LeftTile = TileLeft; + using RightTile = TileRight; + using AccTile = TileAcc; + + GlobalData input_a_global(input_a); + GlobalData input_b_global(input_b); + GlobalData output_global(output); + TileMatA input_a_mat; + TileMatB input_b_mat; + LeftTile input_a_l0; + RightTile input_b_l0; + AccTile output_l0; + TASSIGN(input_a_mat, 0x0); + TASSIGN(input_b_mat, 0x20000); + TASSIGN(input_a_l0, 0x0); + TASSIGN(input_b_l0, 0x0); + TASSIGN(output_l0, 0x0); + + for (uint32_t iteration = 0; iteration < repeats; ++iteration) { + TLOAD(input_a_mat, input_a_global); + TLOAD(input_b_mat, input_b_global); + set_flag(PIPE_MTE2, PIPE_MTE1, EVENT_ID0); + wait_flag(PIPE_MTE2, PIPE_MTE1, EVENT_ID0); + TMOV(input_a_l0, input_a_mat); + TMOV(input_b_l0, input_b_mat); + set_flag(PIPE_MTE1, PIPE_M, EVENT_ID0); + wait_flag(PIPE_MTE1, PIPE_M, EVENT_ID0); + TMATMUL(output_l0, input_a_l0, input_b_l0); + set_flag(PIPE_M, PIPE_FIX, EVENT_ID0); + wait_flag(PIPE_M, PIPE_FIX, EVENT_ID0); + TSTORE(output_global, output_l0); + set_flag(PIPE_FIX, PIPE_S, EVENT_ID7); + wait_flag(PIPE_FIX, PIPE_S, EVENT_ID7); + } +} +#elif defined(PA_CCEC_OPS_DEFINE_REAL_WORKLOAD) && defined(PA_BUILD_AIV) +template +__aicore__ inline void RunRealVectorWorkload( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + constexpr int kRows = static_cast(pa_scheduler::winner_workload::kTileRows); + constexpr int kCols = static_cast(pa_scheduler::winner_workload::kTileCols); + using GlobalData = GlobalTensor< + float, Shape<1, 1, 1, kRows, kCols>, pto::Stride<1, 1, 1, kCols, 1>>; + using TileData = Tile< + TileType::Vec, float, kRows, kCols, BLayout::RowMajor, -1, -1>; + + GlobalData input_a_global(input_a); + GlobalData input_b_global(input_b); + GlobalData output_global(output); + TileData input_a_tile(kRows, kCols); + TileData input_b_tile(kRows, kCols); + TileData output_tile(kRows, kCols); + TASSIGN(input_a_tile, 0x0); + TASSIGN(input_b_tile, 0x10000); + TASSIGN(output_tile, 0x20000); + + for (uint32_t iteration = 0; iteration < repeats; ++iteration) { + TLOAD(input_a_tile, input_a_global); + TLOAD(input_b_tile, input_b_global); + set_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + wait_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + if constexpr (Multiply) { + TMUL(output_tile, input_a_tile, input_b_tile); + } else { + TADD(output_tile, input_a_tile, input_b_tile); + } + set_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + wait_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + TSTORE(output_global, output_tile); + set_flag(PIPE_MTE3, PIPE_S, EVENT_ID7); + wait_flag(PIPE_MTE3, PIPE_S, EVENT_ID7); + } +} + +static __aicore__ __attribute__((noinline, used)) void pa_real_vector_add_workload_aiv( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + RunRealVectorWorkload(input_a, input_b, output, repeats); +} + +static __aicore__ __attribute__((noinline, used)) void pa_real_vector_mul_workload_aiv( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + RunRealVectorWorkload(input_a, input_b, output, repeats); +} +#endif + +#if PA_BUILD_SUBMIT_PMU +struct SubmitPmuContext; +#endif + +struct CcecOps { + static constexpr bool kAtomicReturnReadyObserved = true; + + // 该适配层把平台无关调度器需要的原子、计时、NOP 和 cache 操作逐一映射到 CCEC intrinsic。 + // A5 上 PA 的共享“读取”使用 atomicAdd(addr, 0)，不是普通 GM load；这里保留其 RMW 竞争语义。 + __aicore__ static inline int32_t Load(__gm__ volatile int32_t *address) { + // atomicAdd 返回加法发生前的值；加数为 0，因此它就是本次共享读取的结果。 + return atomicAdd(const_cast<__gm__ int32_t *>(address), static_cast(0)); + } + + __aicore__ static inline int64_t Load(__gm__ volatile int64_t *address) { + return atomicAdd(const_cast<__gm__ int64_t *>(address), static_cast(0)); + } + + __aicore__ static inline uint64_t Load(__gm__ volatile uint64_t *address) { + return atomicAdd(const_cast<__gm__ uint64_t *>(address), static_cast(0)); + } + + __aicore__ static inline int32_t Exchange(__gm__ volatile int32_t *address, int32_t value) { + // atomicExch 同样返回旧值；当前 completion/fatal 发布只需要其原子写入副作用。 + return atomicExch(const_cast<__gm__ int32_t *>(address), value); + } + + __aicore__ static inline int64_t Exchange(__gm__ volatile int64_t *address, int64_t value) { + return atomicExch(const_cast<__gm__ int64_t *>(address), value); + } + + __aicore__ static inline uint64_t Exchange(__gm__ volatile uint64_t *address, uint64_t value) { + return atomicExch(const_cast<__gm__ uint64_t *>(address), value); + } + + __aicore__ static inline int64_t FetchAdd(__gm__ volatile int64_t *address, int64_t value) { + // 返回递增前的计数；启动和 replay 屏障只关心全局累加结果，因此调用方不使用该返回值。 + return atomicAdd(const_cast<__gm__ int64_t *>(address), value); + } + + __aicore__ static inline int64_t FetchMax(__gm__ volatile int64_t *address, int64_t value, uint64_t &retries) { + // CCEC 直接生成单条硬件 atomicMax，不存在 CPU CAS 循环可观测的重试次数。 + retries = 0; + // 返回更新前的 cursor/frontier，Claim 用它判定 winner，frontier 扫描用它吸收其他核的进度。 + return atomicMax(const_cast<__gm__ int64_t *>(address), value); + } + + // PA's A5 OUT_OF_ORDER_STORE_BARRIER is intentionally a no-op; cache + // coherency is handled by the runtime's DCCI protocol. + // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， + // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache + // 可见性则由各自既有的 DCCI 路径处理。 + __aicore__ static inline void StoreBarrier() {} + + __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } + + template + __aicore__ static inline uint64_t NowAfterAtomicResult(T value) { + static_assert(sizeof(T) == 4 || sizeof(T) == 8, "atomic dependency expects a scalar result"); + uint64_t cycle = 0; + // 同一个 inline asm 块先真正消费 atomic 返回寄存器，再读取 + // SYS_CNT；编译器不能把 t1 拆到依赖 MOV 之前。AIC/AIV 对该序列 + // 生成相同指令字节，且不增加 DSB/ISB/GM 访存。该边界仍只表示 + // 返回值已可被本核 scalar 消费，不表示跨核全局可见。 + asm volatile( + "MOV %0, %0\n" + "MOV %1, SYS_CNT\n" + : "+l"(value), "=&l"(cycle) + ); + return cycle; + } + + __aicore__ static inline void ExecuteKernel( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState &worker, + pa_scheduler::TaskKind kind, uint32_t nop_count + ) { + const auto mode = static_cast(state->winner_workload.mode); + if (mode != pa_scheduler::WinnerWorkloadMode::RealCompute) { + RuntimeNop(nop_count); + return; + } +#if defined(PA_BUILD_AIC) + ::pa_execute_real_winner_workload_aic(state, &worker, kind); +#elif defined(PA_BUILD_AIV) + ::pa_execute_real_winner_workload_aiv(state, &worker, kind); +#endif + } + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + __aicore__ static inline pa_scheduler::LazySampleSplitRuntimeState &LazySampleSplitState() { +#if defined(PA_BUILD_AIC) + return ::pa_scheduler_lazy_sample_callback_state_aic; +#elif defined(PA_BUILD_AIV) + return ::pa_scheduler_lazy_sample_callback_state_aiv; +#endif + } + + __aicore__ static inline bool FinishLazySampleCallback( + const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args + ) { +#if defined(PA_BUILD_AIC) + return ::pa_scheduler_lazy_sample_callback_finish_aic(ticket, args) != 0; +#elif defined(PA_BUILD_AIV) + return ::pa_scheduler_lazy_sample_callback_finish_aiv(ticket, args) != 0; +#endif + } +#endif + +#if PA_BUILD_SUBMIT_PMU + using PmuContext = SubmitPmuContext; + + __aicore__ static inline PmuContext PmuWindowStart( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id + ); + + __aicore__ static inline void PmuPhaseBegin(PmuContext &context); + + __aicore__ static inline void PmuPhaseEnd(PmuContext &context); + + __aicore__ static inline void PmuWindowStop( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id, PmuContext &context + ); +#else + // swimlane 产物不携带 PMU 读寄存器或门控代码；公共调度器保留同一 hook + // 形状，编译器会把这两个空实现完整消去。 + __aicore__ static inline bool PmuWindowStart( + __gm__ pa_scheduler::SchedulerState *, uint32_t + ) { return false; } + + __aicore__ static inline void PmuWindowStop( + __gm__ pa_scheduler::SchedulerState *, uint32_t, bool + ) {} +#endif + + // SPIN_WAIT_HINT is also a no-op in the real A5 inner-kernel contract. + // 同理不额外插入 nop，让等待循环保留真实 PA 内核“不主动退避”的指令成本。 + __aicore__ static inline void SpinHint() {} + + __aicore__ static inline void InvalidateRegion(__gm__ const void *address, uint64_t bytes) { + // 逐 cache line 失效并以 dsb 收口，供 worker 在启动时读取 host 刚写入的 standalone 控制区。 + if (bytes == 0) return; + const uint64_t start = reinterpret_cast(address) & ~uint64_t{63}; + const uint64_t end = (reinterpret_cast(address) + bytes + 63) & ~uint64_t{63}; + for (uint64_t current = start; current < end; current += 64) { + dcci(reinterpret_cast<__gm__ uint8_t *>(current), SINGLE_CACHE_LINE); + } + dsb((mem_dsb_t)0); + } + + __aicore__ static inline void FlushRegion(__gm__ void *address, uint64_t bytes) { + // 泳道记录先写普通 GM cache，kernel 结束前显式 CACHELINE_OUT，确保 host D2H 能看到完整记录。 + if (bytes == 0) return; + __asm__ volatile("" ::: "memory"); + const uint64_t start = reinterpret_cast(address) & ~uint64_t{63}; + const uint64_t end = (reinterpret_cast(address) + bytes + 63) & ~uint64_t{63}; + for (uint64_t current = start; current < end; current += 64) { + dcci(reinterpret_cast<__gm__ uint8_t *>(current), SINGLE_CACHE_LINE, CACHELINE_OUT); + } + dsb((mem_dsb_t)0); + } + + __aicore__ static inline void Publish(__gm__ uint64_t *address, uint64_t value) { + // 每核独占的 WorkerResult 用 bypass-DCache store 发布，host 同步后可直接 D2H，无需共享原子竞争。 + __builtin_cce_st_dev(value, address, 0); + } + + __aicore__ static inline void Publish(__gm__ uint32_t *address, uint32_t value) { + __builtin_cce_st_dev(value, address, 0); + } +}; + +} // namespace pa_scheduler_ccec + +#if defined(PA_CCEC_OPS_DEFINE_REAL_WORKLOAD) +// 跨 TU 只暴露按架构区分的真实负载分派；Cube/Vector 实体仍保持 LOCAL。 +// 最终 mixed ELF 由 version script 把该 strong 定义重新局部化，避免成为 kernel entry。 +#if defined(PA_BUILD_AIC) +extern "C" __attribute__((noinline, used)) __aicore__ void pa_execute_real_winner_workload_aic( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +) { +#elif defined(PA_BUILD_AIV) +extern "C" __attribute__((noinline, used)) __aicore__ void pa_execute_real_winner_workload_aiv( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +) { +#endif + const uint64_t workspace = state->winner_workload.workspace_base; + const uint32_t repeats = pa_scheduler::WorkloadCountForKind( + state->winner_workload.repeats, kind + ); + // 错版 host、截断 workspace 或越界 worker 不能继续解引用 GM。这里不额外 + // 写共享 fatal，避免在正常热路增加 atomic；host 的逐 kind sentinel/数值 + // 闭环会把这种配置错误判为失败。 + if (state->winner_workload.version != pa_scheduler::kWinnerWorkloadConfigVersion || + workspace == 0 || + state->winner_workload.workspace_bytes < pa_scheduler::winner_workload::kWorkspaceBytes || + worker->core_idx < 0 || static_cast(worker->core_idx) >= pa_scheduler::kWorkers || + repeats == 0 || repeats > pa_scheduler::winner_workload::kMaxRealComputeCount) { + return; + } +#if defined(PA_BUILD_AIC) + if (kind != pa_scheduler::TaskKind::Qk && kind != pa_scheduler::TaskKind::Pv) return; +#elif defined(PA_BUILD_AIV) + if (kind != pa_scheduler::TaskKind::Sf && kind != pa_scheduler::TaskKind::Up) return; +#endif + __gm__ float *input_a = reinterpret_cast<__gm__ float *>(workspace); + __gm__ float *input_b = reinterpret_cast<__gm__ float *>( + workspace + pa_scheduler::winner_workload::kTileBytes + ); + const uint32_t kind_slot = + (kind == pa_scheduler::TaskKind::Pv || kind == pa_scheduler::TaskKind::Up) ? 1U : 0U; + const uint32_t output_tile = + pa_scheduler::winner_workload::kSharedInputTiles + + static_cast(worker->core_idx) * + pa_scheduler::winner_workload::kOutputTilesPerWorker + + kind_slot; + __gm__ float *output = reinterpret_cast<__gm__ float *>( + workspace + static_cast(output_tile) * + pa_scheduler::winner_workload::kTileBytes + ); +#if defined(PA_BUILD_AIC) + pa_scheduler_ccec::pa_real_cube_workload_aic(input_a, input_b, output, repeats); +#elif defined(PA_BUILD_AIV) + if (kind == pa_scheduler::TaskKind::Sf) { + pa_scheduler_ccec::pa_real_vector_add_workload_aiv(input_a, input_b, output, repeats); + } else { + pa_scheduler_ccec::pa_real_vector_mul_workload_aiv(input_a, input_b, output, repeats); + } +#endif +} +#endif // PA_CCEC_OPS_DEFINE_REAL_WORKLOAD + +#endif // TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CCEC_OPS_H diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/host.cpp b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/host.cpp new file mode 100644 index 0000000000..1ee342645f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/host.cpp @@ -0,0 +1,1989 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "../common/host_support.h" +#include "../common/winner_workload_host.h" +#include "pmu_owner_host.h" +#include "pmu_probe.h" + +#include "acl/acl.h" +#include "driver/ascend_hal.h" +#include "runtime/rt.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +bool CheckAcl(aclError error, const char *label) { + if (error == ACL_SUCCESS) return true; + std::fprintf(stderr, "ACL error %d: %s\n", static_cast(error), label); + return false; +} + +bool CheckRt(rtError_t error, const char *label) { + if (error == RT_ERROR_NONE) return true; + std::fprintf(stderr, "RT error %d: %s\n", static_cast(error), label); + return false; +} + +class ScopedAclDeviceAllocation { +public: + ScopedAclDeviceAllocation() = default; + ScopedAclDeviceAllocation(const ScopedAclDeviceAllocation &) = delete; + ScopedAclDeviceAllocation &operator=(const ScopedAclDeviceAllocation &) = delete; + + ~ScopedAclDeviceAllocation() { + // 早退路径没有机会汇入末尾 cleanup；这里只负责尽力释放本类新增的 + // real-compute workspace。正常路径会先 Release，再检查 aclrtFree 返回值。 + if (pointer_ != nullptr) (void)aclrtFree(pointer_); + } + + void **Address() { return &pointer_; } + void *Get() const { return pointer_; } + + void *Release() { + void *pointer = pointer_; + pointer_ = nullptr; + return pointer; + } + +private: + void *pointer_ = nullptr; +}; + +std::vector ReadBinary(const std::string &path) { + // ELF 整体保存在 vector 中直到 runtime 卸载完成，保证 rtDevBinary_t.data 在整个注册生命周期内有效。 + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) return {}; + const std::streamsize size = file.tellg(); + if (size <= 0) return {}; + std::vector data(static_cast(size)); + file.seekg(0, std::ios::beg); + if (!file.read(data.data(), size)) return {}; + return data; +} + +struct PmuOptions { + pa_scheduler::ccec_pmu::WindowMode mode = pa_scheduler::ccec_pmu::WindowMode::Off; + uint32_t scalar_nops = 100000; + uint32_t icache_trials = 64; + std::string json_path; +}; + +using pa_scheduler::host::ConfigureWinnerWorkload; +using pa_scheduler::host::ParseWinnerWorkloadOptions; +using pa_scheduler::host::ValidateRealComputeOutputs; +using pa_scheduler::host::ValidateWinnerWorkloadOptions; +using pa_scheduler::host::WinnerWorkloadModeName; +using pa_scheduler::host::WinnerWorkloadOptions; + +const char *PmuModeName(pa_scheduler::ccec_pmu::WindowMode mode) { + switch (mode) { + case pa_scheduler::ccec_pmu::WindowMode::Off: + return "off"; + case pa_scheduler::ccec_pmu::WindowMode::Empty: + return "empty"; + case pa_scheduler::ccec_pmu::WindowMode::Scalar: + return "scalar"; + case pa_scheduler::ccec_pmu::WindowMode::ScalarDouble: + return "scalar-double"; + case pa_scheduler::ccec_pmu::WindowMode::IcacheSingle: + return "icache-single"; + case pa_scheduler::ccec_pmu::WindowMode::SubmitAll: + return "submit-all"; + } + return "invalid"; +} + +bool ParsePmuOptions(int argc, char **argv, PmuOptions *pmu, std::vector *common_argv) { + // PMU 参数只属于 CCEC 验证分支；先摘出再交给三后端共享 parser，避免 CPU/AscendC 静默接受却不生效。 + bool mode_seen = false; + bool nops_seen = false; + bool icache_trials_seen = false; + bool json_seen = false; + common_argv->clear(); + common_argv->push_back(argv[0]); + for (int index = 1; index < argc; ++index) { + const std::string argument = argv[index]; + if (argument != "--pmu-window" && argument != "--pmu-scalar-nops" && + argument != "--pmu-icache-trials" && argument != "--pmu-json") { + common_argv->push_back(argv[index]); + continue; + } + if (index + 1 >= argc) { + std::fprintf(stderr, "Missing value after %s\n", argument.c_str()); + return false; + } + const char *value = argv[++index]; + if (argument == "--pmu-window") { + if (mode_seen) { + std::fprintf(stderr, "Specify --pmu-window only once.\n"); + return false; + } + const std::string name = value; + if (name == "off") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::Off; + } else if (name == "empty") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::Empty; + } else if (name == "scalar") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::Scalar; + } else if (name == "scalar-double") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::ScalarDouble; + } else if (name == "icache-single") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::IcacheSingle; + } else if (name == "submit-all") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::SubmitAll; + } else { + std::fprintf( + stderr, + "Invalid --pmu-window value: %s " + "(expected off|empty|scalar|scalar-double|icache-single|submit-all)\n", + value + ); + return false; + } + mode_seen = true; + } else if (argument == "--pmu-scalar-nops") { + if (nops_seen || !pa_scheduler::host::ParseUint(value, 0, 10000000, &pmu->scalar_nops)) { + std::fprintf(stderr, "Invalid or duplicate --pmu-scalar-nops value: %s\n", value); + return false; + } + nops_seen = true; + } else if (argument == "--pmu-icache-trials") { + if (icache_trials_seen || + !pa_scheduler::host::ParseUint(value, 1, 10000, &pmu->icache_trials)) { + std::fprintf(stderr, "Invalid or duplicate --pmu-icache-trials value: %s\n", value); + return false; + } + icache_trials_seen = true; + } else { + if (json_seen || *value == '\0') { + std::fprintf(stderr, "Invalid or duplicate --pmu-json path: %s\n", value); + return false; + } + pmu->json_path = value; + json_seen = true; + } + } + if (nops_seen && pmu->mode != pa_scheduler::ccec_pmu::WindowMode::Scalar && + pmu->mode != pa_scheduler::ccec_pmu::WindowMode::ScalarDouble) { + std::fprintf(stderr, "--pmu-scalar-nops requires a scalar PMU window.\n"); + return false; + } + if (icache_trials_seen && pmu->mode != pa_scheduler::ccec_pmu::WindowMode::IcacheSingle) { + std::fprintf(stderr, "--pmu-icache-trials requires the icache-single PMU window.\n"); + return false; + } + return true; +} + +using HalResMapFn = int (*)(uint32_t, struct res_map_info *, unsigned long *, uint32_t *); +using HalResUnmapFn = int (*)(uint32_t, struct res_map_info *); + +struct PmuRegisterMappings { + HalResUnmapFn unmap = nullptr; + std::vector mapped_resources; + std::vector register_bases; +}; + +bool UnmapPmuRegisters(uint32_t device, PmuRegisterMappings *mappings) { + bool ok = true; + if (mappings->unmap != nullptr) { + for (auto iterator = mappings->mapped_resources.rbegin(); iterator != mappings->mapped_resources.rend(); + ++iterator) { + const int error = mappings->unmap(device, &*iterator); + if (error != 0) { + std::fprintf(stderr, "halResUnmap failed for core %u (rc=%d)\n", iterator->res_id, error); + ok = false; + } + } + } + mappings->mapped_resources.clear(); + mappings->register_bases.clear(); + return ok; +} + +bool MapPmuRegisters(uint32_t device, PmuRegisterMappings *mappings) { + using namespace pa_scheduler::ccec_pmu; + const auto map = reinterpret_cast(dlsym(RTLD_DEFAULT, "halResMap")); + mappings->unmap = reinterpret_cast(dlsym(RTLD_DEFAULT, "halResUnmap")); + if (map == nullptr || mappings->unmap == nullptr) { + std::fprintf(stderr, "halResMap/halResUnmap is unavailable in the current CANN driver process.\n"); + return false; + } + + mappings->register_bases.assign(kPhysicalSubcoreCount, 0); + mappings->mapped_resources.reserve(kPhysicalAicoreCount); + for (uint32_t aicore = 0; aicore < kPhysicalAicoreCount; ++aicore) { + res_map_info info{}; + info.target_proc_type = PROCESS_CP1; + info.res_type = RES_AICORE; + info.res_id = aicore; + unsigned long map_address = 0; + uint32_t map_bytes = kAicoreMapBytes; + const int error = map(device, &info, &map_address, &map_bytes); + if (error != 0 || map_address == 0 || map_bytes < kAicoreMapBytes) { + std::fprintf( + stderr, "halResMap failed for core %u (rc=%d address=0x%lx bytes=%u)\n", aicore, error, + map_address, map_bytes + ); + (void)UnmapPmuRegisters(device, mappings); + return false; + } + mappings->mapped_resources.push_back(info); + + // 与正式 A5 host_regs.cpp 相同：每个 die 的布局为 18 AIC，随后是 36 AIV。 + const uint32_t die = aicore / kAicorePerDie; + const uint32_t local = aicore % kAicorePerDie; + const uint32_t die_base = die * kSubcoresPerDie; + mappings->register_bases[die_base + local] = static_cast(map_address); + mappings->register_bases[die_base + kAicorePerDie + local * 2] = + static_cast(map_address) + kAivFirstOffset; + mappings->register_bases[die_base + kAicorePerDie + local * 2 + 1] = + static_cast(map_address) + kAivSecondOffset; + } + return true; +} + +void ConfigurePmu(pa_scheduler::SchedulerState *state, const PmuOptions &pmu, const void *register_table) { + using namespace pa_scheduler::ccec_pmu; + state->config.reserved[kConfigMode] = static_cast(pmu.mode); + state->config.reserved[kConfigWorkAmount] = + pmu.mode == WindowMode::IcacheSingle ? pmu.icache_trials : pmu.scalar_nops; + StorePointer(state->config.reserved, register_table); + state->config.reserved[kConfigMagic] = pmu.mode == WindowMode::Off ? 0 : kConfigMagicValue; +} + +struct PmuAggregate { + std::vector total_cycles; + std::vector window_ticks; + std::vector submit_elapsed_ticks; + std::vector phase_elapsed_ticks; + std::vector warm_total_cycles; + std::vector warm_window_ticks; + std::vector vector_busy; + std::vector cube_busy; + std::vector scalar_busy; + std::vector mte1_busy; + std::vector mte2_busy; + std::vector icache_requests; + std::vector icache_misses; + std::vector warm_icache_requests; + std::vector warm_icache_misses; + std::vector fix_busy; + std::vector phase_calls; + std::vector phase_icache_requests; + std::vector phase_icache_misses; + std::vector shadow_icache_requests; + std::vector shadow_icache_misses; + uint32_t trusted = 0; +}; + +void AddPmuSample(const pa_scheduler::WorkerResult &result, PmuAggregate *aggregate) { + aggregate->total_cycles.push_back(result.pmu_total_cycles); + aggregate->window_ticks.push_back(result.pmu_window_ticks); + aggregate->submit_elapsed_ticks.push_back( + result.submit_end >= result.submit_begin ? result.submit_end - result.submit_begin : 0U + ); + aggregate->phase_elapsed_ticks.push_back(result.pmu_phase_elapsed_ticks); + aggregate->warm_total_cycles.push_back(result.pmu_warm_total_cycles); + aggregate->warm_window_ticks.push_back(result.pmu_warm_window_ticks); + aggregate->vector_busy.push_back(result.pmu_vector_busy); + aggregate->cube_busy.push_back(result.pmu_cube_busy); + aggregate->scalar_busy.push_back(result.pmu_scalar_busy); + aggregate->mte1_busy.push_back(result.pmu_mte1_busy); + aggregate->mte2_busy.push_back(result.pmu_mte2_busy); + aggregate->icache_requests.push_back(result.pmu_icache_requests); + aggregate->icache_misses.push_back(result.pmu_icache_misses); + aggregate->warm_icache_requests.push_back(result.pmu_warm_icache_requests); + aggregate->warm_icache_misses.push_back(result.pmu_warm_icache_misses); + aggregate->fix_busy.push_back(result.pmu_fix_busy); + aggregate->phase_calls.push_back(result.pmu_phase_calls); + aggregate->phase_icache_requests.push_back(result.pmu_phase_icache_requests); + aggregate->phase_icache_misses.push_back(result.pmu_phase_icache_misses); + aggregate->shadow_icache_requests.push_back(result.pmu_shadow_icache_requests); + aggregate->shadow_icache_misses.push_back(result.pmu_shadow_icache_misses); + aggregate->trusted += + (result.pmu_status & pa_scheduler::ccec_pmu::kStatusRequired) == + pa_scheduler::ccec_pmu::kStatusRequired && + (result.pmu_phase_status & pa_scheduler::ccec_pmu::kPhaseStatusRequired) == + pa_scheduler::ccec_pmu::kPhaseStatusRequired; +} + +bool PrintSingleIcacheAggregate( + const char *name, const PmuAggregate &aggregate, uint32_t trials_per_core +) { + const pa_scheduler::host::Uint64Distribution cold_cycles = + pa_scheduler::host::SummarizeUint64(aggregate.total_cycles); + const pa_scheduler::host::Uint64Distribution warm_cycles = + pa_scheduler::host::SummarizeUint64(aggregate.warm_total_cycles); + const pa_scheduler::host::Uint64Distribution cold_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.window_ticks); + const pa_scheduler::host::Uint64Distribution warm_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.warm_window_ticks); + const pa_scheduler::host::Uint64Distribution cold_requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution warm_requests = + pa_scheduler::host::SummarizeUint64(aggregate.warm_icache_requests); + const pa_scheduler::host::Uint64Distribution cold_misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + const pa_scheduler::host::Uint64Distribution warm_misses = + pa_scheduler::host::SummarizeUint64(aggregate.warm_icache_misses); + const int64_t cycle_delta = static_cast(cold_cycles.total) - + static_cast(warm_cycles.total); + const int64_t tick_delta = static_cast(cold_ticks.total) - + static_cast(warm_ticks.total); + const int64_t miss_delta = static_cast(cold_misses.total) - + static_cast(warm_misses.total); + const uint64_t attempted = static_cast(trials_per_core) * aggregate.total_cycles.size(); + const double misses_per_trial = attempted == 0U ? 0.0 : static_cast(miss_delta) / attempted; + const double cycles_per_miss = miss_delta <= 0 ? 0.0 : static_cast(cycle_delta) / miss_delta; + // 本用例的 get_sys_cnt 已按 1 GHz 时间基准校准，因此一个 tick 对应 1 ns。 + const double ns_per_miss = miss_delta <= 0 ? 0.0 : static_cast(tick_delta) / miss_delta; + std::printf( + "[ICACHE-SINGLE-%s] cores=%zu trials_per_core=%u attempted=%llu " + "cold_cycles=%llu warm_cycles=%llu cycle_delta=%lld cold_ticks=%llu warm_ticks=%llu " + "tick_delta=%lld cold_req=%llu warm_req=%llu cold_miss=%llu warm_miss=%llu " + "miss_delta=%lld misses_per_trial=%.6f cycles_per_miss=%.3f ns_per_miss=%.3f\n", + name, aggregate.total_cycles.size(), trials_per_core, static_cast(attempted), + static_cast(cold_cycles.total), + static_cast(warm_cycles.total), static_cast(cycle_delta), + static_cast(cold_ticks.total), + static_cast(warm_ticks.total), static_cast(tick_delta), + static_cast(cold_requests.total), + static_cast(warm_requests.total), + static_cast(cold_misses.total), + static_cast(warm_misses.total), static_cast(miss_delta), + misses_per_trial, cycles_per_miss, ns_per_miss + ); + std::printf( + "[ICACHE-FORMULA-%s] estimated_scalar_icache_time_ns = cnt7_icache_miss * %.3f\n", + name, ns_per_miss + ); + return cycle_delta > 0 && tick_delta > 0 && miss_delta == static_cast(attempted); +} + +void PrintPmuAggregate(const char *name, const PmuAggregate &aggregate) { + const pa_scheduler::host::Uint64Distribution total = + pa_scheduler::host::SummarizeUint64(aggregate.total_cycles); + const pa_scheduler::host::Uint64Distribution scalar = + pa_scheduler::host::SummarizeUint64(aggregate.scalar_busy); + const pa_scheduler::host::Uint64Distribution vector = + pa_scheduler::host::SummarizeUint64(aggregate.vector_busy); + const pa_scheduler::host::Uint64Distribution cube = + pa_scheduler::host::SummarizeUint64(aggregate.cube_busy); + const pa_scheduler::host::Uint64Distribution mte1 = + pa_scheduler::host::SummarizeUint64(aggregate.mte1_busy); + const pa_scheduler::host::Uint64Distribution mte2 = + pa_scheduler::host::SummarizeUint64(aggregate.mte2_busy); + const pa_scheduler::host::Uint64Distribution requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + const double miss_rate = requests.total == 0 ? 0.0 : 100.0 * misses.total / requests.total; + std::printf( + "[PMU-%s] cores=%zu total_sum=%llu total_median=%.1f total_p95=%llu " + "scalar_busy=%llu vector_busy=%llu cube_busy=%llu mte1_busy=%llu mte2_busy=%llu " + "icache_req=%llu icache_miss=%llu miss_rate=%.4f%%\n", + name, aggregate.total_cycles.size(), static_cast(total.total), total.median, + static_cast(total.p95), static_cast(scalar.total), + static_cast(vector.total), static_cast(cube.total), + static_cast(mte1.total), static_cast(mte2.total), + static_cast(requests.total), static_cast(misses.total), miss_rate + ); +} + +void PrintSubmitPmuPhaseAggregate(const char *name, const PmuAggregate &aggregate) { + const pa_scheduler::host::Uint64Distribution calls = + pa_scheduler::host::SummarizeUint64(aggregate.phase_calls); + const pa_scheduler::host::Uint64Distribution requests = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_requests); + const pa_scheduler::host::Uint64Distribution misses = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_misses); + const pa_scheduler::host::Uint64Distribution primary_requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution primary_misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + const pa_scheduler::host::Uint64Distribution shadow_requests = + pa_scheduler::host::SummarizeUint64(aggregate.shadow_icache_requests); + const pa_scheduler::host::Uint64Distribution shadow_misses = + pa_scheduler::host::SummarizeUint64(aggregate.shadow_icache_misses); + const pa_scheduler::host::Uint64Distribution submit_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.submit_elapsed_ticks); + const pa_scheduler::host::Uint64Distribution phase_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.phase_elapsed_ticks); + const uint64_t request_loss = primary_requests.total >= shadow_requests.total + ? primary_requests.total - shadow_requests.total + : 0U; + const uint64_t miss_loss = primary_misses.total >= shadow_misses.total + ? primary_misses.total - shadow_misses.total + : 0U; + const double miss_rate = requests.total == 0U ? 0.0 : 100.0 * misses.total / requests.total; + const double phase_time_share = submit_ticks.total == 0U + ? 0.0 + : 100.0 * phase_ticks.total / submit_ticks.total; + std::printf( + "[PMU-PHASE-%s] phase=%s semantics=%s cores=%zu calls=%llu " + "icache_req=[%llu,%llu] icache_miss=[%llu,%llu] " + "observed_read_clear_ratio=%.4f%% phase_ticks=%llu submit_ticks=%llu " + "phase_time_share=%.4f%% shadow_loss=%llu/%llu\n", + name, pa_scheduler::ccec_pmu::SubmitPmuPhaseName(pa_scheduler::kCompiledSubmitPmuPhase), + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "disabled" + : "running_read_clear_lower_bound", + aggregate.phase_calls.size(), static_cast(calls.total), + static_cast(requests.total), + static_cast(requests.total + request_loss), + static_cast(misses.total), + static_cast(misses.total + miss_loss), miss_rate, + static_cast(phase_ticks.total), + static_cast(submit_ticks.total), phase_time_share, + static_cast(request_loss), + static_cast(miss_loss) + ); +} + +struct PmuValidation { + uint32_t trusted = 0; + uint32_t unique_physical_core_ids = 0; + uint32_t owner_bitmap_members = 0; + uint32_t exact_worker_slots = 0; + uint32_t physical_role_matches = 0; + uint32_t mixed_triplet_matches = 0; + uint32_t window_started = 0; + uint32_t window_stopped = 0; + uint32_t icache_pairs = 0; + uint32_t icache_calibrated_cores = 0; + uint32_t prior_snapshot_larger = 0; + uint32_t submit_engine_workers_expected = 0; + uint32_t submit_engine_workers_matched = 0; + uint32_t maximum_programmable_counter = 0; + uint32_t build_variant_matches = 0; + uint32_t phase_id_matches = 0; + uint32_t phase_status_trusted = 0; + uint32_t shadow_primary_matches = 0; + uint32_t shadow_primary_bounded = 0; + uint32_t phase_shadow_acceptable = 0; + uint32_t phase_boundary_matches = 0; + uint32_t phase_call_shape_matches = 0; + uint32_t phase_time_valid_records = 0; + uint64_t phase_calls = 0; + uint64_t expected_phase_calls = 0; + uint64_t shadow_request_abs_delta_sum = 0; + uint64_t shadow_miss_abs_delta_sum = 0; + int64_t shadow_request_signed_delta_sum = 0; + int64_t shadow_miss_signed_delta_sum = 0; + uint32_t shadow_request_abs_delta_max = 0; + uint32_t shadow_miss_abs_delta_max = 0; + bool icache_order_valid = true; + bool icache_measurement_valid = true; + bool submit_engine_observation_valid = true; + bool counter_below_risk_threshold = true; + bool phase_measurement_valid = false; + bool passed = true; +}; + +// 32-bit programmable counter 无法仅凭终值证明从未回卷。正式文件采用 25% +// 高水位作为保守拒绝阈值；它只降低风险，不把“未越线”表述成回卷证明。 +constexpr uint32_t kProgrammableCounterRiskThreshold = UINT32_MAX / 4U; + +uint32_t ExpectedSubmitPmuPhaseCallsPerWorker(uint32_t batches) { + // 当前所有 running phase 都覆盖每个 worker 的每次 Submit，固定为 5B。 + switch (pa_scheduler::kCompiledSubmitPmuPhase) { + case pa_scheduler::SubmitPmuPhase::None: + return 0U; + case pa_scheduler::SubmitPmuPhase::Claim: + case pa_scheduler::SubmitPmuPhase::EfDrain: + case pa_scheduler::SubmitPmuPhase::Materialize: + case pa_scheduler::SubmitPmuPhase::Register: + return batches * pa_scheduler::kTasksPerBatch; + case pa_scheduler::SubmitPmuPhase::Count: + break; + } + return UINT32_MAX; +} + +bool ValidatePmu( + const pa_scheduler::SchedulerState &state, uint32_t run, const PmuOptions &pmu, + const WinnerWorkloadOptions &workload, + const pa_scheduler::pmu_owner::PmuOwnerControl *owner, PmuValidation *validation +) { + using namespace pa_scheduler::ccec_pmu; + if (pmu.mode == WindowMode::Off) { + *validation = PmuValidation{}; + return true; + } + + bool seen[kPhysicalSubcoreCount] = {}; + uint32_t trusted = 0; + uint32_t unique = 0; + uint32_t owner_members = 0; + uint32_t exact_worker_slots = 0; + uint32_t physical_role_matches = 0; + uint32_t window_started = 0; + uint32_t window_stopped = 0; + uint32_t icache_pairs = 0; + uint32_t icache_calibrated_cores = 0; + uint32_t prior_larger = 0; + uint32_t submit_engine_workers_expected = 0; + uint32_t submit_engine_workers_matched = 0; + uint32_t maximum_programmable_counter = 0; + uint32_t build_variant_matches = 0; + uint32_t phase_id_matches = 0; + uint32_t phase_status_trusted = 0; + uint32_t shadow_primary_matches = 0; + uint32_t shadow_primary_bounded = 0; + uint32_t phase_shadow_acceptable = 0; + uint32_t phase_boundary_matches = 0; + uint32_t phase_call_shape_matches = 0; + uint32_t phase_time_valid_records = 0; + uint64_t phase_calls = 0; + uint64_t expected_phase_calls = 0; + uint64_t shadow_request_abs_delta_sum = 0; + uint64_t shadow_miss_abs_delta_sum = 0; + int64_t shadow_request_signed_delta_sum = 0; + int64_t shadow_miss_signed_delta_sum = 0; + uint32_t shadow_request_abs_delta_max = 0; + uint32_t shadow_miss_abs_delta_max = 0; + bool icache_order_valid = true; + uint32_t bad_printed = 0; + PmuAggregate all; + PmuAggregate aic; + PmuAggregate aiv; + for (uint32_t worker = 0; worker < pa_scheduler::kWorkers; ++worker) { + const pa_scheduler::WorkerResult &result = state.results[worker]; + const uint32_t expected_phase_calls_per_worker = + ExpectedSubmitPmuPhaseCallsPerWorker(state.config.batches); + const uint32_t status = result.pmu_status; + const uint32_t core_id = StatusCoreId(status); + const bool record_trusted = (status & kStatusRequired) == kStatusRequired; + const bool variant_matches = result.pmu_build_variant == pa_scheduler::kBuildVariantSubmitPmu; + const bool phase_id_matches_record = + result.pmu_phase_id == static_cast(pa_scheduler::kCompiledSubmitPmuPhase); + const bool phase_status_ok = + (result.pmu_phase_status & kPhaseStatusRequired) == kPhaseStatusRequired; + const bool shadow_matches = + result.pmu_shadow_icache_requests == result.pmu_icache_requests && + result.pmu_shadow_icache_misses == result.pmu_icache_misses; + const bool shadow_bounded = + result.pmu_shadow_icache_requests <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_icache_misses; + const bool phase_requires_exact_shadow = + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None; + const bool shadow_acceptable = + phase_requires_exact_shadow + ? shadow_matches + : shadow_bounded; + const uint32_t request_abs_delta = + result.pmu_shadow_icache_requests >= result.pmu_icache_requests + ? result.pmu_shadow_icache_requests - result.pmu_icache_requests + : result.pmu_icache_requests - result.pmu_shadow_icache_requests; + const uint32_t miss_abs_delta = + result.pmu_shadow_icache_misses >= result.pmu_icache_misses + ? result.pmu_shadow_icache_misses - result.pmu_icache_misses + : result.pmu_icache_misses - result.pmu_shadow_icache_misses; + const bool boundaries_match = + result.pmu_phase_begin_reads == result.pmu_phase_calls && + result.pmu_phase_end_reads == result.pmu_phase_calls; + const bool phase_call_shape_matches_record = + result.pmu_phase_calls == expected_phase_calls_per_worker; + const uint64_t submit_elapsed_ticks = result.submit_end >= result.submit_begin + ? result.submit_end - result.submit_begin + : 0U; + const bool phase_time_valid_record = + submit_elapsed_ticks != 0U && + result.pmu_phase_elapsed_ticks <= submit_elapsed_ticks && + (pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? result.pmu_phase_elapsed_ticks == 0U + : result.pmu_phase_elapsed_ticks != 0U); + const bool logical_aic = worker < pa_scheduler::kAicWorkers; + const bool physical_aic = pa_scheduler::pmu_owner::IsAicPhysicalSlot(core_id); + trusted += record_trusted; + build_variant_matches += variant_matches; + phase_id_matches += phase_id_matches_record; + phase_status_trusted += phase_status_ok; + shadow_primary_matches += shadow_matches; + shadow_primary_bounded += shadow_bounded; + phase_shadow_acceptable += shadow_acceptable; + phase_boundary_matches += boundaries_match; + phase_call_shape_matches += phase_call_shape_matches_record; + phase_time_valid_records += phase_time_valid_record; + phase_calls += result.pmu_phase_calls; + expected_phase_calls += expected_phase_calls_per_worker; + shadow_request_abs_delta_sum += request_abs_delta; + shadow_miss_abs_delta_sum += miss_abs_delta; + shadow_request_signed_delta_sum += + static_cast(result.pmu_shadow_icache_requests) - result.pmu_icache_requests; + shadow_miss_signed_delta_sum += + static_cast(result.pmu_shadow_icache_misses) - result.pmu_icache_misses; + shadow_request_abs_delta_max = std::max(shadow_request_abs_delta_max, request_abs_delta); + shadow_miss_abs_delta_max = std::max(shadow_miss_abs_delta_max, miss_abs_delta); + owner_members += owner != nullptr && pa_scheduler::pmu_owner::IsConfigured(*owner, core_id); + exact_worker_slots += result.worker_id == worker; + physical_role_matches += logical_aic == physical_aic; + window_started += (status & kStatusWindowStarted) != 0U; + window_stopped += (status & kStatusWindowStopped) != 0U; + icache_pairs += (status & kStatusIcachePairObserved) != 0U; + prior_larger += (status & kStatusPriorSnapshotLarger) != 0; + if (pmu.mode == WindowMode::SubmitAll && + workload.mode == pa_scheduler::WinnerWorkloadMode::RealCompute) { + const uint64_t submit_engine_tasks = + result.placement[static_cast(pa_scheduler::DrainPlace::EfDrain)] + + result.placement[static_cast(pa_scheduler::DrainPlace::RingBackpressure)]; + const uint32_t relevant_busy = logical_aic ? result.pmu_cube_busy : result.pmu_vector_busy; + const bool engine_observation_matches = + (submit_engine_tasks == 0U) == (relevant_busy == 0U); + submit_engine_workers_expected += submit_engine_tasks != 0U; + submit_engine_workers_matched += engine_observation_matches; + } + // phase 的 request/miss 由两条顺序 ld_dev 划界，局部窗口边界并不 + // 完全重合；只要求它们各自不超过完整窗口，不把 phase miss<=request + // 误设成硬门槛。完整 Submit 的 miss<=request 仍必须成立。 + icache_order_valid &= result.pmu_icache_misses <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_shadow_icache_requests && + result.pmu_shadow_icache_requests <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_icache_misses && + result.pmu_phase_icache_requests <= result.pmu_shadow_icache_requests && + result.pmu_phase_icache_misses <= result.pmu_shadow_icache_misses; + if (pmu.mode == WindowMode::IcacheSingle) { + const int64_t worker_cycle_delta = static_cast(result.pmu_total_cycles) - + static_cast(result.pmu_warm_total_cycles); + const int64_t worker_tick_delta = static_cast(result.pmu_window_ticks) - + static_cast(result.pmu_warm_window_ticks); + const int64_t worker_miss_delta = static_cast(result.pmu_icache_misses) - + static_cast(result.pmu_warm_icache_misses); + icache_calibrated_cores += worker_cycle_delta > 0 && worker_tick_delta > 0 && + worker_miss_delta == static_cast(pmu.icache_trials) && + result.pmu_warm_icache_misses == 0U && + result.pmu_icache_misses == pmu.icache_trials; + } + const uint32_t programmable[] = { + result.pmu_vector_busy, result.pmu_cube_busy, result.pmu_scalar_busy, + result.pmu_mte1_busy, result.pmu_mte2_busy, + result.pmu_icache_requests, result.pmu_icache_misses, + result.pmu_phase_icache_requests, result.pmu_phase_icache_misses, + result.pmu_shadow_icache_requests, result.pmu_shadow_icache_misses, + }; + for (uint32_t value : programmable) { + maximum_programmable_counter = std::max(maximum_programmable_counter, value); + } + if (core_id < kPhysicalSubcoreCount && !seen[core_id]) { + seen[core_id] = true; + ++unique; + } + if ((!record_trusted || !variant_matches || !phase_id_matches_record || + !phase_status_ok || !shadow_acceptable || !boundaries_match || + !phase_call_shape_matches_record || !phase_time_valid_record) && bad_printed < 8) { + std::printf( + "[PMU-BAD] worker=%u role=%llu coreid=%u status=0x%08x total=%llu scalar=%u " + "req=%u miss=%u phase_status=0x%08x phase=%u/%u calls=%u/%u boundaries=%u/%u " + "phase_ticks=%llu submit_ticks=%llu shadow=%u/%u\n", + worker, static_cast(result.role), core_id, status, + static_cast(result.pmu_total_cycles), result.pmu_scalar_busy, + result.pmu_icache_requests, result.pmu_icache_misses, + result.pmu_phase_status, result.pmu_phase_id, + static_cast(pa_scheduler::kCompiledSubmitPmuPhase), result.pmu_phase_calls, + expected_phase_calls_per_worker, + result.pmu_phase_begin_reads, result.pmu_phase_end_reads, + static_cast(result.pmu_phase_elapsed_ticks), + static_cast(submit_elapsed_ticks), + result.pmu_shadow_icache_requests, result.pmu_shadow_icache_misses + ); + ++bad_printed; + } + AddPmuSample(result, &all); + AddPmuSample(result, result.role == static_cast(pa_scheduler::CoreRole::Aic) ? &aic : &aiv); + } + + uint32_t mixed_triplet_matches = 0U; + for (uint32_t block = 0U; block < pa_scheduler::kAicWorkers; ++block) { + const uint32_t aic_id = StatusCoreId(state.results[block].pmu_status); + if (!pa_scheduler::pmu_owner::IsAicPhysicalSlot(aic_id)) continue; + const uint32_t die_base = (aic_id / pa_scheduler::pmu_owner::kSubcoresPerDie) * + pa_scheduler::pmu_owner::kSubcoresPerDie; + const uint32_t local = aic_id % pa_scheduler::pmu_owner::kSubcoresPerDie; + const uint32_t expected_aiv0 = die_base + pa_scheduler::pmu_owner::kAicPerDie + local * 2U; + const uint32_t expected_aiv1 = expected_aiv0 + 1U; + const uint32_t aiv0_id = StatusCoreId( + state.results[pa_scheduler::kAicWorkers + block * 2U].pmu_status + ); + const uint32_t aiv1_id = StatusCoreId( + state.results[pa_scheduler::kAicWorkers + block * 2U + 1U].pmu_status + ); + mixed_triplet_matches += aiv0_id == expected_aiv0 && aiv1_id == expected_aiv1; + } + + PrintPmuAggregate("ALL", all); + PrintPmuAggregate("AIC", aic); + PrintPmuAggregate("AIV", aiv); + PrintSubmitPmuPhaseAggregate("ALL", all); + PrintSubmitPmuPhaseAggregate("AIC", aic); + PrintSubmitPmuPhaseAggregate("AIV", aiv); + bool icache_measurement_ok = true; + if (pmu.mode == WindowMode::IcacheSingle) { + const bool all_ok = PrintSingleIcacheAggregate("ALL", all, pmu.icache_trials); + const bool aic_ok = PrintSingleIcacheAggregate("AIC", aic, pmu.icache_trials); + const bool aiv_ok = PrintSingleIcacheAggregate("AIV", aiv, pmu.icache_trials); + icache_measurement_ok = all_ok && aic_ok && aiv_ok && + icache_pairs == pa_scheduler::kWorkers && + icache_calibrated_cores == pa_scheduler::kWorkers; + } + const bool records_ok = trusted == pa_scheduler::kWorkers; + const bool core_ids_ok = unique == pa_scheduler::kWorkers; + const bool owner_members_ok = owner_members == pa_scheduler::kWorkers; + const bool worker_slots_ok = exact_worker_slots == pa_scheduler::kWorkers; + const bool physical_roles_ok = physical_role_matches == pa_scheduler::kWorkers; + const bool mixed_triplets_ok = mixed_triplet_matches == pa_scheduler::kAicWorkers; + const bool windows_started_ok = window_started == pa_scheduler::kWorkers; + const bool windows_stopped_ok = window_stopped == pa_scheduler::kWorkers; + const bool build_variant_ok = build_variant_matches == pa_scheduler::kWorkers; + const bool phase_id_ok = phase_id_matches == pa_scheduler::kWorkers; + const bool phase_status_ok = phase_status_trusted == pa_scheduler::kWorkers; + const bool shadow_partition_ok = phase_shadow_acceptable == pa_scheduler::kWorkers; + const bool phase_boundaries_ok = phase_boundary_matches == pa_scheduler::kWorkers; + const bool phase_call_shape_ok = phase_call_shape_matches == pa_scheduler::kWorkers; + const bool phase_time_ok = phase_time_valid_records == pa_scheduler::kWorkers; + const bool phase_calls_ok = phase_calls == expected_phase_calls; + const bool submit_engine_observation_ok = + pmu.mode != WindowMode::SubmitAll || + workload.mode != pa_scheduler::WinnerWorkloadMode::RealCompute || + submit_engine_workers_matched == pa_scheduler::kWorkers; + const bool counter_below_risk_threshold = + maximum_programmable_counter < kProgrammableCounterRiskThreshold; + std::printf( + "[PMU] run=%u window=%s calibration_scalar_nops=%u icache_trials=%u trusted=%u/%u " + "unique_coreids=%u/%u prior_larger=%u/%u icache_pairs=%u/%u calibrated_cores=%u/%u " + "programmable_max=%u headroom=%u\n", run, + PmuModeName(pmu.mode), pmu.scalar_nops, pmu.icache_trials, trusted, pa_scheduler::kWorkers, + unique, pa_scheduler::kWorkers, prior_larger, pa_scheduler::kWorkers, + icache_pairs, pa_scheduler::kWorkers, icache_calibrated_cores, pa_scheduler::kWorkers, + maximum_programmable_counter, + UINT32_MAX - maximum_programmable_counter + ); + std::printf( + "[PMU-SHADOW-DELTA] exact=%u/%u bounded=%u/%u request_abs_sum=%llu request_abs_max=%u " + "request_signed_sum=%lld miss_abs_sum=%llu miss_abs_max=%u miss_signed_sum=%lld\n", + shadow_primary_matches, pa_scheduler::kWorkers, + shadow_primary_bounded, pa_scheduler::kWorkers, + static_cast(shadow_request_abs_delta_sum), + shadow_request_abs_delta_max, static_cast(shadow_request_signed_delta_sum), + static_cast(shadow_miss_abs_delta_sum), + shadow_miss_abs_delta_max, static_cast(shadow_miss_signed_delta_sum) + ); + std::printf("[ASSERT] %-48s %s\n", "all PMU records have configured selectors and data", + records_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 96 PMU physical subcore ids are unique", + core_ids_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all PMU physical ids belong to the owner bitmap", + owner_members_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "worker result slots and ids match exactly", + worker_slots_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "logical AIC/AIV roles match physical subcores", + physical_roles_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 32 mixed blocks map to physical 1:2 triplets", + mixed_triplets_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 96 PMU windows executed start", + windows_started_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 96 started PMU windows executed stop", + windows_stopped_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all records match submit-pmu build and phase ids", + build_variant_ok && phase_id_ok ? "PASS" : "FAIL"); + std::printf( + "[ASSERT] %-48s %s\n", + "phase shadow partitions satisfy exact-or-bounded contract", + shadow_partition_ok ? "PASS" : "FAIL" + ); + std::printf("[ASSERT] %-48s %s\n", "all phase boundaries and per-worker calls are exact", + phase_status_ok && phase_boundaries_ok && phase_call_shape_ok && phase_calls_ok + ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all phase times fit their per-worker Submit windows", + phase_time_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "I-cache misses do not exceed requests", + icache_order_valid ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "programmable counters stay below 25% risk threshold", + counter_below_risk_threshold ? "PASS" : "FAIL"); + if (pmu.mode == WindowMode::SubmitAll && + workload.mode == pa_scheduler::WinnerWorkloadMode::RealCompute) { + std::printf( + "[ASSERT] %-48s %s (active_workers=%u matched_workers=%u/%u)\n", + "Submit placement has matching AIC/AIV engine PMU", + submit_engine_observation_ok ? "PASS" : "FAIL", submit_engine_workers_expected, + submit_engine_workers_matched, pa_scheduler::kWorkers + ); + } + if (pmu.mode == WindowMode::IcacheSingle) { + std::printf("[ASSERT] %-48s %s\n", "each cold trial adds exactly one CNT7 I-cache miss", + icache_measurement_ok ? "PASS" : "FAIL"); + } + validation->trusted = trusted; + validation->unique_physical_core_ids = unique; + validation->owner_bitmap_members = owner_members; + validation->exact_worker_slots = exact_worker_slots; + validation->physical_role_matches = physical_role_matches; + validation->mixed_triplet_matches = mixed_triplet_matches; + validation->window_started = window_started; + validation->window_stopped = window_stopped; + validation->icache_pairs = icache_pairs; + validation->icache_calibrated_cores = icache_calibrated_cores; + validation->prior_snapshot_larger = prior_larger; + validation->submit_engine_workers_expected = submit_engine_workers_expected; + validation->submit_engine_workers_matched = submit_engine_workers_matched; + validation->maximum_programmable_counter = maximum_programmable_counter; + validation->build_variant_matches = build_variant_matches; + validation->phase_id_matches = phase_id_matches; + validation->phase_status_trusted = phase_status_trusted; + validation->shadow_primary_matches = shadow_primary_matches; + validation->shadow_primary_bounded = shadow_primary_bounded; + validation->phase_shadow_acceptable = phase_shadow_acceptable; + validation->phase_boundary_matches = phase_boundary_matches; + validation->phase_call_shape_matches = phase_call_shape_matches; + validation->phase_time_valid_records = phase_time_valid_records; + validation->phase_calls = phase_calls; + validation->expected_phase_calls = expected_phase_calls; + validation->shadow_request_abs_delta_sum = shadow_request_abs_delta_sum; + validation->shadow_miss_abs_delta_sum = shadow_miss_abs_delta_sum; + validation->shadow_request_signed_delta_sum = shadow_request_signed_delta_sum; + validation->shadow_miss_signed_delta_sum = shadow_miss_signed_delta_sum; + validation->shadow_request_abs_delta_max = shadow_request_abs_delta_max; + validation->shadow_miss_abs_delta_max = shadow_miss_abs_delta_max; + validation->icache_order_valid = icache_order_valid; + validation->icache_measurement_valid = icache_measurement_ok; + validation->submit_engine_observation_valid = submit_engine_observation_ok; + validation->counter_below_risk_threshold = counter_below_risk_threshold; + validation->phase_measurement_valid = + build_variant_ok && phase_id_ok && phase_status_ok && shadow_partition_ok && + phase_boundaries_ok && phase_call_shape_ok && phase_calls_ok && phase_time_ok; + validation->passed = records_ok && core_ids_ok && owner_members_ok && worker_slots_ok && + physical_roles_ok && mixed_triplets_ok && windows_started_ok && windows_stopped_ok && + icache_order_valid && icache_measurement_ok && submit_engine_observation_ok && + validation->phase_measurement_valid && + counter_below_risk_threshold; + return validation->passed; +} + +void WriteJsonString(std::FILE *output, const std::string &value) { + std::fputc('"', output); + for (unsigned char character : value) { + switch (character) { + case '"': + std::fputs("\\\"", output); + break; + case '\\': + std::fputs("\\\\", output); + break; + case '\b': + std::fputs("\\b", output); + break; + case '\f': + std::fputs("\\f", output); + break; + case '\n': + std::fputs("\\n", output); + break; + case '\r': + std::fputs("\\r", output); + break; + case '\t': + std::fputs("\\t", output); + break; + default: + if (character < 0x20U) { + std::fprintf(output, "\\u%04x", static_cast(character)); + } else { + std::fputc(character, output); + } + } + } + std::fputc('"', output); +} + +void WriteMetricDistribution(std::FILE *output, const std::vector &values) { + const pa_scheduler::host::Uint64Distribution summary = pa_scheduler::host::SummarizeUint64(values); + const double mean = values.empty() ? 0.0 : static_cast(summary.total) / values.size(); + std::fprintf( + output, "{\"sum\":%llu,\"mean\":%.17g,\"median\":%.17g,\"p95\":%llu,\"max\":%llu}", + static_cast(summary.total), mean, summary.median, + static_cast(summary.p95), static_cast(summary.maximum) + ); +} + +void WritePmuAggregateJson( + std::FILE *output, const PmuAggregate &aggregate, bool icache_single +) { + (void)icache_single; + const pa_scheduler::host::Uint64Distribution requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + uint32_t active_cores = 0; + for (uint64_t cycles : aggregate.total_cycles) active_cores += cycles != 0; + std::fprintf( + output, "{\"cores\":%zu,\"active_cores\":%u,\"trusted_cores\":%u,\"total_cycles\":", + aggregate.total_cycles.size(), active_cores, aggregate.trusted + ); + WriteMetricDistribution(output, aggregate.total_cycles); + std::fputs(",\"vector_busy\":", output); + WriteMetricDistribution(output, aggregate.vector_busy); + std::fputs(",\"cube_busy\":", output); + WriteMetricDistribution(output, aggregate.cube_busy); + std::fputs(",\"scalar_busy\":", output); + WriteMetricDistribution(output, aggregate.scalar_busy); + std::fputs(",\"mte1_busy\":", output); + WriteMetricDistribution(output, aggregate.mte1_busy); + std::fputs(",\"mte2_busy\":", output); + WriteMetricDistribution(output, aggregate.mte2_busy); + std::fputs(",\"icache_requests\":", output); + WriteMetricDistribution(output, aggregate.icache_requests); + std::fputs(",\"icache_misses\":", output); + WriteMetricDistribution(output, aggregate.icache_misses); + std::fputs(",\"shadow_whole_icache_requests\":", output); + WriteMetricDistribution(output, aggregate.shadow_icache_requests); + std::fputs(",\"shadow_whole_icache_misses\":", output); + WriteMetricDistribution(output, aggregate.shadow_icache_misses); + std::fputs(",\"phase_calls\":", output); + WriteMetricDistribution(output, aggregate.phase_calls); + std::fputs(",\"submit_elapsed_ticks\":", output); + WriteMetricDistribution(output, aggregate.submit_elapsed_ticks); + std::fputs(",\"phase_elapsed_ticks\":", output); + WriteMetricDistribution(output, aggregate.phase_elapsed_ticks); + std::fputs(",\"phase_icache_requests\":", output); + WriteMetricDistribution(output, aggregate.phase_icache_requests); + std::fputs(",\"phase_icache_misses\":", output); + WriteMetricDistribution(output, aggregate.phase_icache_misses); + std::fputs(",\"icache_miss_rate\":", output); + if (requests.total == 0) { + std::fputs("null", output); + } else { + // 全局 miss rate 必须以总 miss/总 request 计算，不能平均逐核百分比。 + std::fprintf(output, "%.17g", static_cast(misses.total) / requests.total); + } + const pa_scheduler::host::Uint64Distribution phase_requests = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_requests); + const pa_scheduler::host::Uint64Distribution phase_misses = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_misses); + // 两个 phase counter 是顺序 read-to-clear，下界之比不是实际 miss rate + // 的数学下界；字段名只陈述它是本次 read-clear 观察值之比。 + std::fputs(",\"phase_observed_read_clear_ratio\":", output); + if (phase_requests.total == 0U) { + std::fputs("null", output); + } else { + std::fprintf(output, "%.17g", static_cast(phase_misses.total) / phase_requests.total); + } + std::fputc('}', output); +} + +uint32_t CountConfiguredMixedTriplets(const pa_scheduler::pmu_owner::PmuOwnerControl &owner) { + uint32_t complete = 0U; + for (uint32_t die_base = 0U; + die_base < pa_scheduler::pmu_owner::kPhysicalSubcoreCount; + die_base += pa_scheduler::pmu_owner::kSubcoresPerDie) { + for (uint32_t local = 0U; local < pa_scheduler::pmu_owner::kAicPerDie; ++local) { + const uint32_t aic = die_base + local; + const uint32_t aiv0 = die_base + pa_scheduler::pmu_owner::kAicPerDie + local * 2U; + const uint32_t aiv1 = aiv0 + 1U; + complete += pa_scheduler::pmu_owner::IsConfigured(owner, aic) && + pa_scheduler::pmu_owner::IsConfigured(owner, aiv0) && + pa_scheduler::pmu_owner::IsConfigured(owner, aiv1); + } + } + return complete; +} + +bool ExportPmuJson( + const pa_scheduler::SchedulerState &state, const pa_scheduler::host::Options &options, + const PmuOptions &pmu, const WinnerWorkloadOptions &workload, + uint32_t run, double host_us, double submit_span_us, + const PmuValidation &validation, bool semantic_passed, bool workload_output_passed, + const pa_scheduler::pmu_owner::PmuOwnerControl &owner, bool restore_passed, + const std::string &output_path +) { + using namespace pa_scheduler::ccec_pmu; + PmuAggregate all; + PmuAggregate aic; + PmuAggregate aiv; + uint32_t active_output_tiles = 0; + uint64_t ef_drain_kernels = 0; + uint64_t ring_backpressure_kernels = 0; + uint64_t final_drain_kernels = 0; + for (uint32_t worker = 0; worker < pa_scheduler::kWorkers; ++worker) { + const pa_scheduler::WorkerResult &result = state.results[worker]; + AddPmuSample(result, &all); + AddPmuSample(result, result.role == static_cast(pa_scheduler::CoreRole::Aic) ? &aic : &aiv); + if (result.role == static_cast(pa_scheduler::CoreRole::Aic)) { + active_output_tiles += result.kernel_counts[0] != 0; + active_output_tiles += result.kernel_counts[2] != 0; + } else if (result.role == static_cast(pa_scheduler::CoreRole::Aiv)) { + active_output_tiles += result.kernel_counts[1] != 0; + active_output_tiles += result.kernel_counts[3] != 0; + } + ef_drain_kernels += result.placement[static_cast(pa_scheduler::DrainPlace::EfDrain)]; + ring_backpressure_kernels += + result.placement[static_cast(pa_scheduler::DrainPlace::RingBackpressure)]; + final_drain_kernels += result.placement[static_cast(pa_scheduler::DrainPlace::FinalDrain)]; + } + + const auto generated = std::chrono::system_clock::now().time_since_epoch(); + const uint64_t generated_ns = static_cast( + std::chrono::duration_cast(generated).count() + ); + const std::string capture_id = "pa-pmu-" + std::to_string(generated_ns) + "-run" + std::to_string(run); + const std::string temporary_path = output_path + ".tmp"; + // 临时文件与最终文件都采用 no-replace 语义：并发采集不能截断同名 tmp， + // 也不能在最终发布时覆盖另一份已经完成的证据文件。 + const int output_fd = open(temporary_path.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0644); + std::FILE *output = output_fd < 0 ? nullptr : fdopen(output_fd, "wb"); + if (output == nullptr) { + const int open_error = errno; + if (output_fd >= 0) { + (void)close(output_fd); + (void)std::remove(temporary_path.c_str()); + } + std::fprintf( + stderr, "Cannot exclusively create PMU JSON output %s: %s\n", temporary_path.c_str(), + std::strerror(open_error) + ); + return false; + } + std::vector output_buffer(1U << 20); + std::setvbuf(output, output_buffer.data(), _IOFBF, output_buffer.size()); + + const bool submit_window = IsSubmitWindow(pmu.mode); + const bool icache_single = pmu.mode == WindowMode::IcacheSingle; + const bool real_compute = workload.mode == pa_scheduler::WinnerWorkloadMode::RealCompute; + const bool simulated_task_nops_nonzero = !real_compute && + (options.nops.qk != 0U || options.nops.sf != 0U || + options.nops.pv != 0U || options.nops.up != 0U); + const pa_scheduler::WorkloadCounts active_counts = real_compute + ? workload.repeats + : pa_scheduler::WorkloadCounts{ + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + }; + const uint32_t owner_bitmap_count = pa_scheduler::pmu_owner::CountConfigured(owner); + const uint32_t owner_complete_triplets = CountConfiguredMixedTriplets(owner); + std::fputs("{\n\"schema\":{\"name\":\"pa_scheduler_pmu_phase_windows\",\"version\":5},\n", output); + std::fputs("\"capture\":{\"capture_id\":", output); + WriteJsonString(output, capture_id); + std::fprintf( + output, + ",\"generated_unix_time_ns\":%llu,\"run_index\":%u,\"accepted\":true," + "\"usable_for_same_configuration_submit_comparison\":%s," + "\"usable_as_absolute_real_pa_profile\":false,\"window_scope\":\"%s\"," + "\"pmu_probe_position\":\"%s\",\"scheduler_hot_path_included\":%s," + "\"total_sum_is_core_work_not_wall_time\":true," + "\"submit_window_excludes_final_drain\":%s," + "\"published_after_runtime_cleanup\":true,\"runtime_cleanup_passed\":true," + "\"owner_restore_passed\":%s},\n", + static_cast(generated_ns), run, + submit_window ? "true" : "false", + submit_window ? "per_worker_orchestration_to_last_submit_return" : "post_scheduler_calibration_probe", + submit_window ? "inside_RunScheduler" : "after_RunScheduler", + submit_window ? "true" : "false", submit_window ? "true" : "false", + restore_passed ? "true" : "false" + ); + std::fputs("\"configuration\":{\"kernel_path\":", output); + WriteJsonString(output, options.kernel_path); + std::fputs(",\"build_variant\":\"submit-pmu\",\"build_variant_id\":2,\"compiled_phase\":", output); + WriteJsonString(output, SubmitPmuPhaseName(pa_scheduler::kCompiledSubmitPmuPhase)); + std::fprintf( + output, ",\"compiled_phase_id\":%u", + static_cast(pa_scheduler::kCompiledSubmitPmuPhase) + ); + std::fprintf( + output, + ",\"device\":%u,\"batches\":%u,\"workers\":%u,\"aic_workers\":%u,\"aiv_workers\":%u," + "\"trace_enabled\":%s,\"trace_atomics\":%s,\"profile_phases\":%s," + "\"winner_workload\":{\"mode\":", + options.device, options.batches, pa_scheduler::kWorkers, pa_scheduler::kAicWorkers, + pa_scheduler::kAivWorkers, options.trace_enabled ? "true" : "false", + options.trace_atomics ? "true" : "false", + options.profile_phases ? "true" : "false" + ); + WriteJsonString(output, WinnerWorkloadModeName(workload.mode)); + std::fputs(",\"input_pattern\":", output); + WriteJsonString( + output, + real_compute ? pa_scheduler::host::RealComputePatternName(workload.pattern) : "none" + ); + std::fprintf( + output, + ",\"config_version\":%u,\"counts\":{\"qk\":%u,\"sf\":%u,\"pv\":%u,\"up\":%u}," + "\"unit\":", + pa_scheduler::kWinnerWorkloadConfigVersion, active_counts.qk, active_counts.sf, + active_counts.pv, active_counts.up + ); + WriteJsonString( + output, + real_compute ? "complete_128x128_engine_pipeline_iteration" : "scalar_nop_instruction" + ); + std::fprintf( + output, + ",\"tile_rows\":%u,\"tile_cols\":%u,\"shared_input_tiles\":%u," + "\"output_tiles_per_worker\":%u,\"workspace_bytes\":%zu,\"role_mapping\":", + real_compute ? pa_scheduler::winner_workload::kTileRows : 0, + real_compute ? pa_scheduler::winner_workload::kTileCols : 0, + real_compute ? pa_scheduler::winner_workload::kSharedInputTiles : 0, + real_compute ? pa_scheduler::winner_workload::kOutputTilesPerWorker : 0, + real_compute ? pa_scheduler::winner_workload::kWorkspaceBytes : 0 + ); + if (real_compute) { + std::fputs( + "{\"qk\":\"cube_matmul\",\"pv\":\"cube_matmul\"," + "\"sf\":\"vector_add\",\"up\":\"vector_mul\"}", + output + ); + } else { + std::fputs("null", output); + } + std::fprintf( + output, ",\"engine_completion_waited_before_task_publish\":%s},\"nop_counts\":", + real_compute ? "true" : "false" + ); + if (real_compute) { + std::fputs("null", output); + } else { + std::fprintf( + output, "{\"qk\":%u,\"sf\":%u,\"pv\":%u,\"up\":%u}", + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + ); + } + std::fputs(",\"pmu_window\":", output); + WriteJsonString(output, PmuModeName(pmu.mode)); + std::fputs(",\"calibration_scalar_nops_per_segment\":", output); + if (submit_window || icache_single) { + std::fputs("null", output); + } else { + std::fprintf(output, "%u", pmu.scalar_nops); + } + std::fputs(",\"icache_single_trials_per_core\":", output); + if (icache_single) { + std::fprintf(output, "%u", pmu.icache_trials); + } else { + std::fputs("null", output); + } + std::fprintf( + output, + ",\"primary_window_segments_per_record\":1," + "\"icache_single_discarded_training_samples_per_core\":%u," + "\"icache_single_sys_counter_tick_ns\":%s," + "\"host_launch_to_sync_us\":%.17g,\"submit_span_us\":%.17g," + "\"selectors\":{\"cnt0_vector_busy\":%u,\"cnt1_cube_busy\":%u," + "\"cnt2_scalar_busy\":%u,\"cnt3_mte1_busy\":%u,\"cnt4_mte2_busy\":%u," + "\"cnt5_shadow_icache_miss\":%u,\"cnt6_primary_icache_request\":%u," + "\"cnt7_primary_icache_miss\":%u,\"cnt8_shadow_icache_request\":%u," + "\"cnt9_unused\":0},\"unavailable_metrics\":[\"mte3_busy\"]," + "\"counter_width_bits\":{\"total\":64,\"programmable\":32}," + "\"counter_wrap_not_directly_detectable\":true,\"counter_wrap_absence_proven\":false," + "\"programmable_counter_risk_threshold\":%u," + "\"gate_start_stop_have_pipe_all_barriers\":true," + "\"phase_timestamp_calls_present\":%s,\"phase_record_writes\":false," + "\"atomic_trace\":false,\"profile_accumulation\":false," + "\"phase_boundary_observation_included\":%s," + "\"phase_time_observation_included\":%s," + "\"phase_time_sys_counter_tick_ns\":1," + "\"phase_time_boundary\":\"after_begin_read_clear_to_before_end_read_clear\"," + "\"phase_time_excludes_shadow_read_overhead\":true," + "\"phase_time_includes_timestamp_overhead\":true," + "\"phase_time_share_definition\":" + "\"sum(phase_elapsed_ticks)/sum(submit_elapsed_ticks)\"," + "\"phase_time_denominator_scope\":" + "\"per_worker_first_submit_begin_to_last_submit_end\"," + "\"phase_counter_pair_snapshot_atomic\":false," + "\"primary_counters_read_at_phase_boundaries\":false," + "\"phase_shadow_partition_exact_required\":%s," + "\"phase_values_are_running_read_clear_lower_bounds\":%s," + "\"cross_phase_elf_sums_valid\":false," + "\"simulated_task_nop_mechanism_executes_on_scalar\":%s," + "\"simulated_task_nops_nonzero\":%s," + "\"icache_miss_rate_definition\":\"sum(icache_misses)/sum(icache_requests)\"},\n", + icache_single ? 2U : 0U, icache_single ? "1" : "null", + host_us, submit_span_us, kVectorBusyEvent, kCubeBusyEvent, kScalarBusyEvent, + kMte1BusyEvent, kMte2BusyEvent, kIcacheMissEvent, kIcacheRequestEvent, kIcacheMissEvent, + kIcacheRequestEvent, kProgrammableCounterRiskThreshold, + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "true" + : "false", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + real_compute ? "false" : "true", + simulated_task_nops_nonzero ? "true" : "false" + ); + std::fprintf( + output, + "\"validation\":{\"semantic_passed\":%s,\"pmu_passed\":%s," + "\"real_compute_output_validation_required\":%s," + "\"real_compute_output_validation_passed\":%s," + "\"real_compute_active_output_tiles\":%u," + "\"real_compute_inactive_sentinel_tiles\":%u," + "\"real_compute_output_mismatches\":%u," + "\"submit_engine_observation_valid\":%s," + "\"submit_engine_workers_expected\":%u," + "\"submit_engine_workers_matched\":%u," + "\"kernel_placement_counts\":{\"ef_drain\":%llu,\"ring_backpressure\":%llu," + "\"final_drain\":%llu},\"trusted_records\":%u," + "\"expected_records\":%u,\"unique_physical_core_ids\":%u,\"expected_unique_core_ids\":%u," + "\"owner_bitmap_member_records\":%u,\"expected_owner_bitmap_member_records\":%u," + "\"exact_worker_slot_records\":%u,\"expected_exact_worker_slot_records\":%u," + "\"physical_role_match_records\":%u,\"expected_physical_role_match_records\":%u," + "\"mixed_triplet_matches\":%u,\"expected_mixed_triplet_matches\":%u," + "\"window_started_records\":%u,\"window_stopped_records\":%u," + "\"expected_window_records\":%u,\"prior_snapshot_larger_records\":%u," + "\"icache_pair_records\":%u,\"icache_calibrated_cores\":%u," + "\"icache_measurement_valid\":%s," + "\"icache_miss_le_request\":%s,\"counter_below_risk_threshold\":%s," + "\"maximum_programmable_counter\":%u,\"programmable_counter_risk_threshold\":%u," + "\"programmable_counter_headroom\":%u," + "\"build_variant_match_records\":%u,\"phase_id_match_records\":%u," + "\"phase_status_trusted_records\":%u,\"shadow_primary_match_records\":%u," + "\"shadow_primary_bounded_records\":%u," + "\"phase_shadow_acceptable_records\":%u," + "\"shadow_request_abs_delta_sum\":%llu,\"shadow_request_abs_delta_max\":%u," + "\"shadow_request_signed_delta_sum\":%lld," + "\"shadow_miss_abs_delta_sum\":%llu,\"shadow_miss_abs_delta_max\":%u," + "\"shadow_miss_signed_delta_sum\":%lld," + "\"phase_boundary_match_records\":%u,\"phase_call_shape_match_records\":%u," + "\"phase_time_valid_records\":%u,\"phase_time_measurement_valid\":%s," + "\"phase_calls\":%llu,\"phase_expected_calls\":%llu," + "\"phase_measurement_valid\":%s},\n", + semantic_passed ? "true" : "false", validation.passed ? "true" : "false", + real_compute ? "true" : "false", + real_compute ? (workload_output_passed ? "true" : "false") : "null", + real_compute ? active_output_tiles : 0, + real_compute ? pa_scheduler::winner_workload::kOutputTiles - active_output_tiles : 0, + real_compute && !workload_output_passed ? 1U : 0U, + validation.submit_engine_observation_valid ? "true" : "false", + validation.submit_engine_workers_expected, + validation.submit_engine_workers_matched, + static_cast(ef_drain_kernels), + static_cast(ring_backpressure_kernels), + static_cast(final_drain_kernels), + validation.trusted, + pa_scheduler::kWorkers, validation.unique_physical_core_ids, pa_scheduler::kWorkers, + validation.owner_bitmap_members, pa_scheduler::kWorkers, + validation.exact_worker_slots, pa_scheduler::kWorkers, + validation.physical_role_matches, pa_scheduler::kWorkers, + validation.mixed_triplet_matches, pa_scheduler::kAicWorkers, + validation.window_started, validation.window_stopped, pa_scheduler::kWorkers, + validation.prior_snapshot_larger, validation.icache_pairs, + validation.icache_calibrated_cores, + validation.icache_measurement_valid ? "true" : "false", + validation.icache_order_valid ? "true" : "false", + validation.counter_below_risk_threshold ? "true" : "false", + validation.maximum_programmable_counter, kProgrammableCounterRiskThreshold, + UINT32_MAX - validation.maximum_programmable_counter, + validation.build_variant_matches, validation.phase_id_matches, + validation.phase_status_trusted, validation.shadow_primary_matches, + validation.shadow_primary_bounded, validation.phase_shadow_acceptable, + static_cast(validation.shadow_request_abs_delta_sum), + validation.shadow_request_abs_delta_max, + static_cast(validation.shadow_request_signed_delta_sum), + static_cast(validation.shadow_miss_abs_delta_sum), + validation.shadow_miss_abs_delta_max, + static_cast(validation.shadow_miss_signed_delta_sum), + validation.phase_boundary_matches, + validation.phase_call_shape_matches, + validation.phase_time_valid_records, + validation.phase_time_valid_records == pa_scheduler::kWorkers ? "true" : "false", + static_cast(validation.phase_calls), + static_cast(validation.expected_phase_calls), + validation.phase_measurement_valid ? "true" : "false" + ); + std::fprintf( + output, + "\"owner\":{\"mode\":\"main_aicpu_path_a\"," + "\"snapshot_phase\":\"after_configure_before_restore\"," + "\"control_magic\":%u,\"control_version\":%u,\"configure_status\":%d," + "\"configured_flag\":%u,\"configured_bitmap_count\":%u," + "\"expected\":{\"total\":%u,\"aic\":%u,\"aiv\":%u}," + "\"active\":{\"total\":%u,\"aic\":%u,\"aiv\":%u}," + "\"discovered\":{\"total\":%u,\"aic\":%u,\"aiv\":%u}," + "\"physical_slots_scanned\":%u,\"skipped_physical_slots\":%u," + "\"configured_bitmap_word_order\":\"least_significant_physical_ids_first\"," + "\"configured_bitmap_words\":[%u,%u,%u,%u]," + "\"configured_complete_mixed_triplets\":%u,\"expected_complete_mixed_triplets\":%u," + "\"configured_broken_mixed_triplets\":%u,\"restore_passed\":%s},\n", + owner.magic, owner.version, static_cast(owner.status), owner.configured, owner_bitmap_count, + owner.expected_total, owner.expected_aic, owner.expected_aiv, + owner.active_total, owner.active_aic, owner.active_aiv, + owner.discovered_total, owner.discovered_aic, owner.discovered_aiv, + pa_scheduler::pmu_owner::kPhysicalSubcoreCount, owner.skipped_total, + owner.configured_bitmap[0], owner.configured_bitmap[1], + owner.configured_bitmap[2], owner.configured_bitmap[3], + owner_complete_triplets, pa_scheduler::kAicWorkers, + owner_bitmap_count / 3U - owner_complete_triplets, restore_passed ? "true" : "false" + ); + std::fputs("\"records\":[\n", output); + for (uint32_t worker = 0; worker < pa_scheduler::kWorkers; ++worker) { + const pa_scheduler::WorkerResult &result = state.results[worker]; + const uint32_t status = result.pmu_status; + const uint32_t physical_core_id = StatusCoreId(status); + const bool primary_trusted = (status & kStatusRequired) == kStatusRequired; + const bool phase_trusted = + (result.pmu_phase_status & kPhaseStatusRequired) == kPhaseStatusRequired; + const uint64_t submit_elapsed_ticks = result.submit_end >= result.submit_begin + ? result.submit_end - result.submit_begin + : 0U; + const bool phase_time_valid = + submit_elapsed_ticks != 0U && + result.pmu_phase_elapsed_ticks <= submit_elapsed_ticks && + (pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? result.pmu_phase_elapsed_ticks == 0U + : result.pmu_phase_elapsed_ticks != 0U); + const bool trusted = primary_trusted && phase_trusted && phase_time_valid; + const bool is_aic = result.role == static_cast(pa_scheduler::CoreRole::Aic); + const uint32_t vector_id = is_aic ? 0U : worker - pa_scheduler::kAicWorkers; + const uint32_t block_id = is_aic ? worker : vector_id / 2U; + const uint32_t lane = is_aic ? 0U : 1U + vector_id % 2U; + const uint32_t shadow_read_segments = result.pmu_phase_calls * 2U + 1U; + const bool owner_bitmap_member = pa_scheduler::pmu_owner::IsConfigured(owner, physical_core_id); + const bool worker_slot_exact = result.worker_id == worker; + const bool physical_role_matches = + is_aic == pa_scheduler::pmu_owner::IsAicPhysicalSlot(physical_core_id); + const bool window_started = (status & kStatusWindowStarted) != 0U; + const bool window_stopped = (status & kStatusWindowStopped) != 0U; + const bool shadow_matches = + result.pmu_shadow_icache_requests == result.pmu_icache_requests && + result.pmu_shadow_icache_misses == result.pmu_icache_misses; + const bool shadow_bounded = + result.pmu_shadow_icache_requests <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_icache_misses; + const uint32_t shadow_request_loss = + result.pmu_icache_requests - result.pmu_shadow_icache_requests; + const uint32_t shadow_miss_loss = + result.pmu_icache_misses - result.pmu_shadow_icache_misses; + const uint32_t phase_request_upper = + result.pmu_phase_icache_requests + shadow_request_loss; + const uint32_t phase_miss_upper = + result.pmu_phase_icache_misses + shadow_miss_loss; + const bool boundaries_balanced = + result.pmu_phase_begin_reads == result.pmu_phase_calls && + result.pmu_phase_end_reads == result.pmu_phase_calls; + const uint32_t expected_phase_calls = + ExpectedSubmitPmuPhaseCallsPerWorker(state.config.batches); + std::fprintf( + output, + "%s{\"worker_id\":%u,\"physical_core_id\":%u,\"role\":\"%s\",\"block_id\":%u," + "\"lane\":%u,\"primary_window_segments\":1,\"shadow_read_segments\":%u," + "\"window_started\":%s,\"window_stopped\":%s,\"total_cycles\":%llu,\"vector_busy\":%u," + "\"cube_busy\":%u,\"scalar_busy\":%u,\"mte1_busy\":%u,\"mte2_busy\":%u," + "\"icache_requests\":%u,\"icache_misses\":%u," + "\"build_variant_id\":%u,\"compiled_phase_id\":%u,\"phase_calls\":%u," + "\"phase_expected_calls\":%u," + "\"phase_begin_reads\":%u,\"phase_end_reads\":%u," + "\"submit_elapsed_ticks\":%llu,\"phase_elapsed_ticks\":%llu," + "\"phase_time_valid\":%s," + "\"phase_icache_requests\":%u,\"phase_icache_misses\":%u," + "\"phase_icache_requests_upper_bound\":%u," + "\"phase_icache_misses_upper_bound\":%u," + "\"shadow_whole_icache_requests\":%u,\"shadow_whole_icache_misses\":%u," + "\"shadow_matches_primary\":%s,\"shadow_not_greater_than_primary\":%s," + "\"shadow_request_loss\":%u,\"shadow_miss_loss\":%u," + "\"phase_boundaries_balanced\":%s," + "\"phase_status\":%u,\"phase_status_hex\":\"0x%08x\"," + "\"status\":%u,\"status_hex\":" + "\"0x%08x\",\"trusted\":%s,\"physical_core_id_valid\":%s,\"selectors_match\":%s," + "\"owner_bitmap_member\":%s,\"worker_slot_exact\":%s," + "\"physical_role_matches\":%s}", + worker == 0 ? "" : ",\n", worker, physical_core_id, is_aic ? "aic" : "aiv", block_id, + lane, shadow_read_segments, window_started ? "true" : "false", window_stopped ? "true" : "false", + static_cast(result.pmu_total_cycles), result.pmu_vector_busy, + result.pmu_cube_busy, result.pmu_scalar_busy, result.pmu_mte1_busy, result.pmu_mte2_busy, + result.pmu_icache_requests, result.pmu_icache_misses, + result.pmu_build_variant, result.pmu_phase_id, result.pmu_phase_calls, + expected_phase_calls, + result.pmu_phase_begin_reads, result.pmu_phase_end_reads, + static_cast(submit_elapsed_ticks), + static_cast(result.pmu_phase_elapsed_ticks), + phase_time_valid ? "true" : "false", + result.pmu_phase_icache_requests, result.pmu_phase_icache_misses, + phase_request_upper, phase_miss_upper, + result.pmu_shadow_icache_requests, result.pmu_shadow_icache_misses, + shadow_matches ? "true" : "false", shadow_bounded ? "true" : "false", + shadow_request_loss, shadow_miss_loss, boundaries_balanced ? "true" : "false", + result.pmu_phase_status, result.pmu_phase_status, + status, status, trusted ? "true" : "false", + (status & kStatusCoreIdValid) != 0 ? "true" : "false", + (status & (kStatusCnt0Selector | kStatusCnt1Selector | kStatusCnt2Selector | + kStatusCnt3Selector | kStatusCnt4Selector | kStatusCnt5Selector | + kStatusCnt6Selector | kStatusCnt7Selector | kStatusCnt8Selector)) == + (kStatusCnt0Selector | kStatusCnt1Selector | kStatusCnt2Selector | + kStatusCnt3Selector | kStatusCnt4Selector | kStatusCnt5Selector | + kStatusCnt6Selector | kStatusCnt7Selector | kStatusCnt8Selector) + ? "true" + : "false", + owner_bitmap_member ? "true" : "false", worker_slot_exact ? "true" : "false", + physical_role_matches ? "true" : "false" + ); + } + std::fputs("\n],\n\"summary\":{\"all\":", output); + WritePmuAggregateJson(output, all, icache_single); + std::fputs(",\"aic\":", output); + WritePmuAggregateJson(output, aic, icache_single); + std::fputs(",\"aiv\":", output); + WritePmuAggregateJson(output, aiv, icache_single); + std::fputs("}\n}\n", output); + + bool success = std::ferror(output) == 0; + int write_error = success ? 0 : EIO; + if (std::fflush(output) != 0) { + success = false; + write_error = errno; + } + if (success && fsync(fileno(output)) != 0) { + success = false; + write_error = errno; + } + if (std::fclose(output) != 0) { + success = false; + write_error = errno; + } + if (!success) { + std::fprintf(stderr, "Failed while writing PMU JSON output %s: %s\n", temporary_path.c_str(), + std::strerror(write_error)); + (void)std::remove(temporary_path.c_str()); + return false; + } + // 同目录 hard-link 在最终名称不存在时原子发布；EEXIST 时保留既有证据， + // 不采用会替换目标的 POSIX rename。 + if (link(temporary_path.c_str(), output_path.c_str()) != 0) { + std::fprintf( + stderr, "Cannot publish PMU JSON without replacement %s -> %s: %s\n", + temporary_path.c_str(), output_path.c_str(), + std::strerror(errno) + ); + (void)std::remove(temporary_path.c_str()); + return false; + } + if (unlink(temporary_path.c_str()) != 0) { + const int unlink_error = errno; + // 最终文件已链接但事务尚未完成；尽力撤回最终名称，避免失败返回时留下 + // 一份被调用方误认为成功发布的文件。 + (void)unlink(output_path.c_str()); + std::fprintf( + stderr, "Cannot remove PMU JSON temporary link %s: %s\n", temporary_path.c_str(), + std::strerror(unlink_error) + ); + return false; + } + std::printf("[PMU-JSON] capture_id=%s records=%u output=%s\n", capture_id.c_str(), pa_scheduler::kWorkers, + output_path.c_str()); + return true; +} + +} // namespace + +int main(int argc, char **argv) { + // 参数和 ELF 在创建 ACL 资源前完成校验，早期错误不会留下 device、stream 或 kernel handle。 + pa_scheduler::host::Options options; + PmuOptions pmu_options; + WinnerWorkloadOptions workload_options; + std::vector pmu_argv; + std::vector common_argv; + if (!ParseWinnerWorkloadOptions(argc, argv, &workload_options, &pmu_argv) || + !ParsePmuOptions( + static_cast(pmu_argv.size()), pmu_argv.data(), &pmu_options, &common_argv + )) { + return EXIT_FAILURE; + } + const pa_scheduler::host::ParseStatus parse_status = pa_scheduler::host::ParseOptions( + static_cast(common_argv.size()), common_argv.data(), true, &options + ); + if (parse_status != pa_scheduler::host::ParseStatus::Ok) { + if (parse_status == pa_scheduler::host::ParseStatus::Help) { + std::fprintf( + stderr, + "CCEC PMU options: [--pmu-window " + "off|empty|scalar|scalar-double|icache-single|submit-all] " + "[--pmu-scalar-nops N] [--pmu-icache-trials N] [--pmu-json FILE]\n" + ); + std::fprintf( + stderr, + "CCEC winner workload options: [--winner-workload scalar-nop|real-compute] " + "[--real-compute-count N | --real-compute-counts QK,SF,PV,UP] " + "[--real-compute-pattern constant|layout-diagnostic]\n" + ); + std::fprintf( + stderr, + "Default: real-compute, constant, counts=6,28,4,1; " + "scalar-nop is the calibration compatibility mode.\n" + ); + } + return parse_status == pa_scheduler::host::ParseStatus::Help ? EXIT_SUCCESS : EXIT_FAILURE; + } + if (!ValidateWinnerWorkloadOptions(workload_options)) return EXIT_FAILURE; +#if PA_BUILD_SWIMLANE + // swimlane host 与同目录 kernel 是成套产物；它不允许借旧参数重新开启 + // 已从 device ELF 编译掉的 PMU/phase-profile 路径。 + if (pmu_options.mode != pa_scheduler::ccec_pmu::WindowMode::Off || + !pmu_options.json_path.empty()) { + std::fprintf( + stderr, + "This is a swimlane build; PMU collection requires the separate submit-pmu build.\n" + ); + return EXIT_FAILURE; + } + if (options.profile_phases) { + std::fprintf( + stderr, + "--profile-phases is not part of the swimlane build; use submit-pmu phase attribution.\n" + ); + return EXIT_FAILURE; + } +#elif PA_BUILD_SUBMIT_PMU + // submit-pmu 是编译期固定 phase 的单轮诊断产物；host、kernel 与 owner + // 必须共同拒绝旧校准窗口和任何泳道/phase-profile 观察代码。 + if (pmu_options.mode != pa_scheduler::ccec_pmu::WindowMode::SubmitAll) { + std::fprintf(stderr, "The submit-pmu build requires --pmu-window submit-all.\n"); + return EXIT_FAILURE; + } + if (options.runs != 1 || options.trace_enabled || options.trace_atomics || + options.profile_phases || options.analyze_swimlane || !options.swimlane_json.empty()) { + std::fprintf( + stderr, + "submit-pmu requires one PMU-only run: --runs 1 --no-swimlane and no trace/profile options.\n" + ); + return EXIT_FAILURE; + } +#endif + if (!pmu_options.json_path.empty() && + pmu_options.mode == pa_scheduler::ccec_pmu::WindowMode::Off) { + std::fprintf(stderr, "--pmu-json requires a non-off --pmu-window.\n"); + return EXIT_FAILURE; + } + if (!pmu_options.json_path.empty() && options.runs != 1) { + // 一个 sidecar 对应一次采集，禁止多轮覆写后丢失逐轮边界。 + std::fprintf(stderr, "--pmu-json requires --runs 1 to avoid overwriting captures.\n"); + return EXIT_FAILURE; + } + if (!pmu_options.json_path.empty() && + (options.trace_enabled || options.trace_atomics || options.profile_phases || + options.analyze_swimlane || !options.swimlane_json.empty())) { + std::fprintf( + stderr, + "--pmu-json requires PMU-only collection: add --no-swimlane and do not enable " + "phase profiling, atomic tracing, swimlane analysis, or swimlane JSON.\n" + ); + return EXIT_FAILURE; + } + if (!pmu_options.json_path.empty() && + (access(pmu_options.json_path.c_str(), F_OK) == 0 || + access((pmu_options.json_path + ".tmp").c_str(), F_OK) == 0)) { + std::fprintf( + stderr, "Refusing to overwrite an existing PMU JSON or temporary file: %s\n", + pmu_options.json_path.c_str() + ); + return EXIT_FAILURE; + } + const std::vector binary_data = ReadBinary(options.kernel_path); + if (binary_data.empty()) { + std::fprintf(stderr, "Cannot read kernel binary: %s\n", options.kernel_path.c_str()); + return EXIT_FAILURE; + } + const bool real_compute = + workload_options.mode == pa_scheduler::WinnerWorkloadMode::RealCompute; + std::vector workload_image; + std::vector workload_outputs; + if (real_compute) { + pa_scheduler::host::InitializeWinnerWorkloadBuffers( + workload_options, &workload_image, &workload_outputs + ); + } + pa_scheduler::host::PrintBanner("CCEC", options); + pa_scheduler::host::PrintWinnerWorkloadConfig(workload_options, options.nops); + std::printf( + "[PMU-CONFIG] window=%s calibration_scalar_nops=%u icache_trials=%u source=direct-per-core " + "owner=main-aicpu-path-a\n", + PmuModeName(pmu_options.mode), pmu_options.scalar_nops, pmu_options.icache_trials + ); + + // 正常及后处理路径依次完成 ACL 初始化、选卡、stream/ELF/设备区创建、launch/D2H + // 和尾部清理；初始化、传输或 launch 的早期错误仍按当前实现就地返回。 + if (!CheckAcl(aclInit(nullptr), "aclInit") || !CheckAcl(aclrtSetDevice(options.device), "aclrtSetDevice")) { + return EXIT_FAILURE; + } + aclrtStream stream = nullptr; + if (!CheckAcl(aclrtCreateStream(&stream), "aclrtCreateStream")) return EXIT_FAILURE; + + rtDevBinary_t binary{RT_DEV_BINARY_MAGIC_ELF, 0, binary_data.data(), binary_data.size()}; + void *kernel_handle = nullptr; + bool registered_all = true; + // 先尝试注册带 mixed metadata 的 ELF；若 rtRegisterAllKernel 报错或未返回 handle， + // 再尝试无 tiling-key 装载。这里仅描述实际回退条件，不假设具体运行时原因。 + rtError_t register_error = rtRegisterAllKernel(&binary, &kernel_handle); + if (register_error != RT_ERROR_NONE || kernel_handle == nullptr) { + registered_all = false; + register_error = rtBinaryLoadWithoutTilingKey(binary_data.data(), binary_data.size(), &kernel_handle); + } + if (!CheckRt(register_error, "register mixed AICore ELF") || kernel_handle == nullptr) return EXIT_FAILURE; + + // SchedulerState 保留被测关键 offset、DistCore ABI 和约 1 GiB 生产总跨度； + // 使用 HUGE_FIRST 降低大块设备内存碎片风险。 + void *state_device = nullptr; + if (!CheckAcl( + aclrtMalloc(&state_device, sizeof(pa_scheduler::SchedulerState), ACL_MEM_MALLOC_HUGE_FIRST), + "aclrtMalloc(state)" + )) { + return EXIT_FAILURE; + } + if ((reinterpret_cast(state_device) & 63U) != 0) { + std::fprintf(stderr, "Device state is not 64-byte aligned: %p\n", state_device); + return EXIT_FAILURE; + } + + // 真实 PTO 负载使用独立 GM，不解引用调度器中只用于依赖建模的 synthetic tensor 地址。 + // 这里先于 PMU owner 分配；每轮 H2D 初始化虽在 owner 配置之后，但仍位于 + // launch/wall 计时之前，因此两者都不进入 Submit 性能窗口。 + ScopedAclDeviceAllocation workload_allocation; + if (real_compute && + !CheckAcl( + aclrtMalloc( + workload_allocation.Address(), pa_scheduler::winner_workload::kWorkspaceBytes, + ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(real-compute workspace)" + )) { + return EXIT_FAILURE; + } + void *workload_device = workload_allocation.Get(); + if (real_compute && (reinterpret_cast(workload_device) & 63U) != 0) { + std::fprintf(stderr, "Real-compute workspace is not 64-byte aligned: %p\n", workload_device); + return EXIT_FAILURE; + } + + // 泳道区按 96 worker 各 65536 条记录预留，约 384 MiB；关闭泳道时不申请，也不会传递有效 base。 + // 该分配先于 PMU owner 配置，失败时不会留下需要恢复的 selector/MMIO 会话。 + void *trace_device = nullptr; + if (options.trace_enabled && + !CheckAcl( + aclrtMalloc(&trace_device, pa_scheduler::kTraceBytes, ACL_MEM_MALLOC_HUGE_FIRST), + "aclrtMalloc(swimlane trace)" + )) { + return EXIT_FAILURE; + } + if (options.trace_enabled && (reinterpret_cast(trace_device) & 63U) != 0) { + std::fprintf(stderr, "Device swimlane trace is not 64-byte aligned: %p\n", trace_device); + return EXIT_FAILURE; + } + + PmuRegisterMappings pmu_mappings; + pa_scheduler::pmu_owner::PmuOwnerSession pmu_owner; + pa_scheduler::pmu_owner::PmuOwnerControl pmu_owner_evidence{}; + bool pmu_owner_evidence_valid = false; + const void *pmu_registers_device = nullptr; + if (pmu_options.mode != pa_scheduler::ccec_pmu::WindowMode::Off) { + if (!MapPmuRegisters(options.device, &pmu_mappings)) return EXIT_FAILURE; + const std::string dispatcher_path = pa_scheduler::pmu_owner::ArtifactBesideKernel( + options.kernel_path, "libpa_scheduler_pmu_owner_dispatcher.so" + ); + const std::string owner_path = pa_scheduler::pmu_owner::ArtifactBesideKernel( + options.kernel_path, "libpa_scheduler_pmu_owner_aicpu.so" + ); + if (!pmu_owner.Initialize( + options.device, stream, dispatcher_path, owner_path, pmu_mappings.register_bases + ) || + !pmu_owner.Configure()) { + (void)pmu_owner.Finalize(); + (void)UnmapPmuRegisters(options.device, &pmu_mappings); + return EXIT_FAILURE; + } + pmu_owner_evidence = pmu_owner.Control(); + pmu_owner_evidence_valid = true; + pmu_registers_device = reinterpret_cast(pmu_owner.RegisterTableDeviceAddress()); + } + + // host shadow 保留约 1 GiB 总跨度以便按关键 offset 寻址，但每轮传输只选择 + // 共享前缀、控制区和结果区。 + std::unique_ptr state(new pa_scheduler::SchedulerState); + pa_scheduler::TraceHeader trace_header{}; + std::vector spans; + bool execution_ok = true; + bool all_passed = true; + bool postprocess_ok = true; + bool pmu_json_ready = false; + bool pmu_json_semantic_passed = false; + bool pmu_json_workload_output_passed = false; + uint32_t pmu_json_run = 0U; + double pmu_json_host_us = 0.0; + double pmu_json_submit_span_us = 0.0; + PmuValidation pmu_json_validation; + for (uint32_t run = 1; run <= options.runs; ++run) { + pa_scheduler::host::InitializeState(state.get(), options); + pa_scheduler::host::ConfigureTrace(state.get(), options, trace_device); + ConfigurePmu(state.get(), pmu_options, pmu_registers_device); + ConfigureWinnerWorkload(state.get(), workload_options, workload_device); + if (real_compute && + !CheckAcl( + aclrtMemcpy( + workload_device, pa_scheduler::winner_workload::kWorkspaceBytes, + workload_image.data(), pa_scheduler::winner_workload::kWorkspaceBytes, + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D real-compute workspace)" + )) { + execution_ok = false; + break; + } + if (options.trace_enabled) { + // 每轮只需重置约 7 KiB header；各 worker 会从 count=0 覆盖自己的记录区，无需清零整块 384 MiB。 + pa_scheduler::host::InitializeTraceHeader(&trace_header); + if (!CheckAcl( + aclrtMemcpy( + trace_device, sizeof(trace_header), &trace_header, sizeof(trace_header), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D swimlane header)" + )) { + execution_ok = false; + break; + } + } + // 为避免每轮搬运约 1 GiB，只 H2D 被测共享前缀和位于生产总跨度之后的 + // standalone 控制区； + // 每个 worker 的大块私有状态由 device kernel 自行初始化。 + if (!CheckAcl( + aclrtMemcpy( + state_device, pa_scheduler::host::StatePrefixBytes(), state.get(), + pa_scheduler::host::StatePrefixBytes(), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D state prefix)" + ) || + !CheckAcl( + aclrtMemcpy( + &static_cast(state_device)->config, + pa_scheduler::host::ControlBytes(), &state->config, pa_scheduler::host::ControlBytes(), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D standalone controls)" + )) { + execution_ok = false; + break; + } + void *kernel_args[] = {state_device}; + rtArgsEx_t args_info{}; + args_info.args = kernel_args; + args_info.argsSize = sizeof(kernel_args); + rtTaskCfgInfo_t task_config{}; + // launch 维度是 32 个物理 mixed block；ELF metadata 让每个 block 同时产生 1 AIC + 2 AIV，共 96 worker。 + // wall time 在同步完成处截止，包含 launch、完整调度、最终 drain 和 stream 同步，但不包含后续 D2H/JSON。 + const auto wall_begin = std::chrono::steady_clock::now(); + if (!CheckRt( + rtKernelLaunchWithHandleV2( + kernel_handle, 0, pa_scheduler::kAicWorkers, &args_info, nullptr, stream, &task_config + ), + "rtKernelLaunchWithHandleV2" + ) || + !CheckAcl(aclrtSynchronizeStream(stream), "aclrtSynchronizeStream")) { + execution_ok = false; + break; + } + const auto wall_end = std::chrono::steady_clock::now(); + const double host_us = std::chrono::duration(wall_end - wall_begin).count(); + // D2H 同样避开约 1 GiB 的 worker arena：共享前缀用于 flag/vend/frontier 校验，末尾 results 单独回传。 + if (!CheckAcl( + aclrtMemcpy( + state.get(), pa_scheduler::host::StatePrefixBytes(), state_device, + pa_scheduler::host::StatePrefixBytes(), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H state prefix)" + ) || + !CheckAcl( + aclrtMemcpy( + state->results, pa_scheduler::host::ResultBytes(), + &static_cast(state_device)->results[0], + pa_scheduler::host::ResultBytes(), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H worker results)" + )) { + execution_ok = false; + break; + } + if (real_compute && + !CheckAcl( + aclrtMemcpy( + workload_outputs.data(), + static_cast(pa_scheduler::winner_workload::kOutputTiles) * + pa_scheduler::winner_workload::kTileBytes, + static_cast(workload_device) + + pa_scheduler::winner_workload::kSharedInputTiles * + pa_scheduler::winner_workload::kTileBytes, + static_cast(pa_scheduler::winner_workload::kOutputTiles) * + pa_scheduler::winner_workload::kTileBytes, + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H real-compute outputs)" + )) { + execution_ok = false; + break; + } + if (options.trace_enabled && + !CheckAcl( + aclrtMemcpy( + &trace_header, sizeof(trace_header), trace_device, sizeof(trace_header), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H swimlane header)" + )) { + execution_ok = false; + break; + } + // 常规校验只需 header 中的 per-worker count；真实 records 在分析或导出时才按核、按实际 count 懒加载。 + const auto read_trace_records = + [trace_device](uint32_t worker, uint32_t count, pa_scheduler::TraceRecord *records) { + // 每核记录区采用固定容量 stride；只复制 header 声明的实际 count，避免 D2H 未使用的尾部空间。 + const uint64_t offset = sizeof(pa_scheduler::TraceHeader) + + static_cast(worker) * pa_scheduler::kTraceRecordsPerCore * + sizeof(pa_scheduler::TraceRecord); + return CheckAcl( + aclrtMemcpy( + records, static_cast(count) * sizeof(pa_scheduler::TraceRecord), + static_cast(trace_device) + offset, + static_cast(count) * sizeof(pa_scheduler::TraceRecord), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H swimlane records)" + ); + }; + // 先完成共享状态、拓扑、计数和 trace header 的语义校验，再允许 raw JSON 成为性能证据。 + const pa_scheduler::host::Metrics metrics = pa_scheduler::host::Validate( + *state, run, host_us, options.trace_enabled ? &trace_header : nullptr + ); + all_passed &= metrics.passed; + const bool workload_passed = + !real_compute || ValidateRealComputeOutputs( + *state, workload_options, workload_outputs, run + ); + all_passed &= workload_passed; + PmuValidation pmu_validation; + const bool pmu_passed = ValidatePmu( + *state, run, pmu_options, workload_options, + pmu_options.mode == pa_scheduler::ccec_pmu::WindowMode::Off ? nullptr : &pmu_owner.Control(), + &pmu_validation + ); + all_passed &= pmu_passed; + spans.push_back(metrics.submit_span_us); + if (!pmu_options.json_path.empty()) { + if (!metrics.passed || !workload_passed || !pmu_passed || !pmu_owner_evidence_valid) { + std::fprintf(stderr, "PMU JSON rejected because semantic, PMU, or owner validation failed.\n"); + postprocess_ok = false; + break; + } + pmu_json_ready = true; + pmu_json_semantic_passed = metrics.passed && workload_passed; + pmu_json_workload_output_passed = workload_passed; + pmu_json_run = run; + pmu_json_host_us = host_us; + pmu_json_submit_span_us = metrics.submit_span_us; + pmu_json_validation = pmu_validation; + } + if (options.analyze_swimlane && + !pa_scheduler::host::AnalyzeSwimlaneRecords(trace_header, *state, read_trace_records)) { + // 后处理错误使用 break 汇入统一 cleanup；与初始化/launch 失败的进程级立即返回语义区分开。 + postprocess_ok = false; + break; + } + if (!options.swimlane_json.empty()) { + // 只有语义校验通过才把 raw JSON 经“临时文件写完后 rename”发布， + // 避免把截断或错误调度结果误当成可用性能证据。 + if (!metrics.passed || !workload_passed) { + std::fprintf(stderr, "Skipping swimlane export because semantic validation failed.\n"); + postprocess_ok = false; + break; + } + if (!pa_scheduler::host::ExportSwimlaneRecords( + trace_header, options.swimlane_json, workload_options.mode, + real_compute + ? workload_options.repeats + : pa_scheduler::WorkloadCounts{ + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + }, + real_compute + ? pa_scheduler::host::RealComputePatternName(workload_options.pattern) + : "none", + options.trace_atomics, + read_trace_records + )) { + postprocess_ok = false; + break; + } + } + } + + const double median_submit_span_us = spans.empty() ? 0.0 : pa_scheduler::host::Median(spans); + std::printf( + "[SUMMARY] runs=%u completed_runs=%zu median_submit_span_us=%.3f " + "execution_status=%s semantic_status=%s postprocess_status=%s\n", + options.runs, spans.size(), median_submit_span_us, execution_ok ? "PASS" : "FAIL", + all_passed ? "PASS" : "FAIL", postprocess_ok ? "PASS" : "FAIL" + ); + + // 后处理失败也统一走设备资源释放、ELF 卸载和 ACL 收尾，避免文件系统错误遗留运行时上下文。 + bool cleanup_ok = true; + bool pmu_owner_restore_ok = true; + // 先释放依赖当前 device/context 的大块内存，再卸载 ELF、销毁 stream，最后 reset device 与 finalize ACL。 + if (trace_device != nullptr) { + cleanup_ok &= CheckAcl(aclrtFree(trace_device), "aclrtFree(swimlane trace)"); + } + if (pmu_registers_device != nullptr) { + // owner 必须在 MMIO 映射、device context 和 ACL runtime 仍有效时恢复。 + pmu_owner_restore_ok = pmu_owner.Finalize(); + cleanup_ok &= pmu_owner_restore_ok; + cleanup_ok &= UnmapPmuRegisters(options.device, &pmu_mappings); + } + if (workload_device != nullptr) { + cleanup_ok &= CheckAcl( + aclrtFree(workload_allocation.Release()), "aclrtFree(real-compute workspace)" + ); + } + cleanup_ok &= CheckAcl(aclrtFree(state_device), "aclrtFree(state)"); + const rtError_t unload_error = + registered_all ? rtDevBinaryUnRegister(kernel_handle) : rtBinaryUnLoad(kernel_handle); + cleanup_ok &= CheckRt(unload_error, "unload mixed AICore ELF"); + cleanup_ok &= CheckAcl(aclrtDestroyStream(stream), "aclrtDestroyStream"); + cleanup_ok &= CheckAcl(aclrtResetDevice(options.device), "aclrtResetDevice"); + cleanup_ok &= CheckAcl(aclFinalize(), "aclFinalize"); + if (!pmu_options.json_path.empty()) { + if (!pmu_json_ready || !all_passed || !postprocess_ok || !cleanup_ok || !pmu_owner_restore_ok) { + std::fprintf(stderr, "PMU JSON was not published because the capture or restore transaction failed.\n"); + postprocess_ok = false; + } else if (!ExportPmuJson( + *state, options, pmu_options, workload_options, pmu_json_run, + pmu_json_host_us, pmu_json_submit_span_us, pmu_json_validation, + pmu_json_semantic_passed, pmu_json_workload_output_passed, + pmu_owner_evidence, + pmu_owner_restore_ok, pmu_options.json_path + )) { + postprocess_ok = false; + } + } + // 运行语义、后处理和资源清理三者全部成功，进程才返回成功，脚本据此决定是否继续生成 merged 泳道。 + return execution_ok && all_passed && postprocess_ok && cleanup_ok ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/kernel.cpp b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/kernel.cpp new file mode 100644 index 0000000000..74c5aa3f0f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/kernel.cpp @@ -0,0 +1,401 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include +#include +#include +#include + +#include "pmu_probe.h" +#include "../common/winner_workload.h" + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" + +#define PA_CCEC_OPS_DEFINE_REAL_WORKLOAD 1 +#include "ccec_ops.h" +#undef PA_CCEC_OPS_DEFINE_REAL_WORKLOAD + +using pa_scheduler_ccec::CcecOps; + +namespace { +#if PA_BUILD_SUBMIT_PMU +struct PmuSnapshot { + uint64_t total_cycles = 0; + uint32_t vector_busy = 0; + uint32_t cube_busy = 0; + uint32_t scalar_busy = 0; + uint32_t mte1_busy = 0; + uint32_t mte2_busy = 0; + uint32_t mte3_busy = 0; + uint32_t icache_requests = 0; + uint32_t icache_misses = 0; + uint32_t fix_busy = 0; + uint32_t status = 0; +}; + +struct IcacheShadowSnapshot { + uint32_t requests = 0; + uint32_t misses = 0; +}; + +} // namespace + +namespace pa_scheduler_ccec { + +struct SubmitPmuContext { + uint64_t reg_base = 0; + uint64_t shadow_requests = 0; + uint64_t shadow_misses = 0; + uint64_t phase_requests = 0; + uint64_t phase_misses = 0; + // begin 的 shadow read-clear 完成后取起点，end 的 shadow read-clear 之前 + // 取终点；累计值因此不包含两次 PMU 寄存器读取本身。 + uint64_t phase_elapsed_ticks = 0; + uint64_t phase_begin_tick = 0; + uint32_t selector_status = 0; + uint32_t phase_status = pa_scheduler::ccec_pmu::kPhaseStatusRequested; + uint32_t phase_calls = 0; + uint32_t begin_reads = 0; + uint32_t end_reads = 0; + bool started = false; + bool phase_armed = false; + bool boundary_error = false; +}; + +} // namespace pa_scheduler_ccec + +namespace { + +using pa_scheduler_ccec::SubmitPmuContext; + +template +__aicore__ inline uint32_t ReadPmuRegister(uint64_t reg_base) { + // 传给 ld_dev 的是重基址后的 __gm__ 指针；相对 offset 均落在编译器允许的 [-2048, 2047]。 + int32_t *block = reinterpret_cast(reg_base + BlockOffset); + return static_cast(ld_dev(block, static_cast(RegisterOffset - BlockOffset))); +} + +__aicore__ inline PmuSnapshot ReadObservedCounters(uint64_t reg_base) { + PmuSnapshot sample; + sample.vector_busy = ReadPmuRegister(reg_base); + sample.cube_busy = ReadPmuRegister(reg_base); + sample.scalar_busy = ReadPmuRegister(reg_base); + sample.mte1_busy = ReadPmuRegister(reg_base); + sample.mte2_busy = ReadPmuRegister(reg_base); + // submit-pmu 将 CNT5 留给 shadow I-cache miss。这里不能提前读取，否则 + // read-to-clear 会让随后的 shadow tail 漏计；MTE3 busy 在该诊断构建不可用。 + sample.mte3_busy = 0; + sample.icache_requests = ReadPmuRegister(reg_base); + sample.icache_misses = ReadPmuRegister(reg_base); + const uint64_t low = ReadPmuRegister(reg_base); + const uint64_t high = ReadPmuRegister(reg_base); + sample.total_cycles = low | (high << 32); + return sample; +} + +__aicore__ inline IcacheShadowSnapshot ReadShadowCounters(uint64_t reg_base) { + IcacheShadowSnapshot sample; + sample.requests = ReadPmuRegister(reg_base); + sample.misses = ReadPmuRegister(reg_base); + return sample; +} + +struct PmuRegisterContext { + uint64_t reg_base = 0; + uint32_t status = 0; + bool shadow_selectors = false; +}; + +__aicore__ inline PmuRegisterContext ResolvePmuRegisters(__gm__ pa_scheduler::SchedulerState *state) { + using namespace pa_scheduler::ccec_pmu; + PmuRegisterContext context; + const uint32_t physical_core_id = static_cast(get_coreid()) & kStatusCoreIdMask; + context.status = kStatusRequested | (physical_core_id << kStatusCoreIdShift); + const uint64_t table_address = + static_cast(state->config.reserved[kConfigRegTableLow]) | + (static_cast(state->config.reserved[kConfigRegTableHigh]) << 32); + if (state->config.reserved[kConfigMagic] != kConfigMagicValue || table_address == 0 || + physical_core_id >= kPhysicalSubcoreCount) { + return context; + } + context.status |= kStatusCoreIdValid; + __gm__ const uint64_t *register_bases = reinterpret_cast<__gm__ const uint64_t *>(table_address); + context.reg_base = register_bases[physical_core_id]; + if (context.reg_base == 0) return context; + context.status |= kStatusRegMapped; + + // selector 与同 phase 目录内的 owner 逐项核对。CNT5/CNT8 分别重复 + // CNT7/CNT6；CNT9 保持正式 PIPE_UTIL 的 unused(0) 口径。 + if (ReadPmuRegister(context.reg_base) == kVectorBusyEvent) + context.status |= kStatusCnt0Selector; + if (ReadPmuRegister(context.reg_base) == kCubeBusyEvent) + context.status |= kStatusCnt1Selector; + if (ReadPmuRegister(context.reg_base) == kScalarBusyEvent) + context.status |= kStatusCnt2Selector; + if (ReadPmuRegister(context.reg_base) == kMte1BusyEvent) + context.status |= kStatusCnt3Selector; + if (ReadPmuRegister(context.reg_base) == kMte2BusyEvent) + context.status |= kStatusCnt4Selector; + const bool cnt5_ok = + ReadPmuRegister(context.reg_base) == kIcacheMissEvent; + if (cnt5_ok) + context.status |= kStatusCnt5Selector; + if (ReadPmuRegister(context.reg_base) == kIcacheRequestEvent) + context.status |= kStatusCnt6Selector; + if (ReadPmuRegister(context.reg_base) == kIcacheMissEvent) + context.status |= kStatusCnt7Selector; + const bool cnt8_ok = + ReadPmuRegister(context.reg_base) == kIcacheRequestEvent; + const bool cnt9_unused = + ReadPmuRegister(context.reg_base) == 0U; + if (cnt8_ok) + context.status |= kStatusCnt8Selector; + context.shadow_selectors = cnt5_ok && cnt8_ok && cnt9_unused; + return context; +} + +__aicore__ inline void PublishPmuSnapshot( + __gm__ pa_scheduler::WorkerResult &result, const PmuSnapshot &sample +) { + // 每核独占 sidecar 通过 bypass store 一次性发布；这些写发生在 PMU stop/read 之后， + // 不进入被导出的 Submit 窗口。 + CcecOps::Publish(&result.pmu_total_cycles, sample.total_cycles); + CcecOps::Publish(&result.pmu_scalar_busy, sample.scalar_busy); + CcecOps::Publish(&result.pmu_icache_requests, sample.icache_requests); + CcecOps::Publish(&result.pmu_icache_misses, sample.icache_misses); + CcecOps::Publish(&result.pmu_status, sample.status); + CcecOps::Publish(&result.pmu_vector_busy, sample.vector_busy); + CcecOps::Publish(&result.pmu_cube_busy, sample.cube_busy); + CcecOps::Publish(&result.pmu_mte1_busy, sample.mte1_busy); + CcecOps::Publish(&result.pmu_mte2_busy, sample.mte2_busy); + CcecOps::Publish(&result.pmu_mte3_busy, sample.mte3_busy); + // CNT8 已改作 shadow request，submit-pmu 不再发布 fix-busy。 + CcecOps::Publish(&result.pmu_fix_busy, static_cast(0)); + CcecOps::Publish(&result.pmu_warm_total_cycles, static_cast(0)); + CcecOps::Publish(&result.pmu_warm_window_ticks, static_cast(0)); +} + +__aicore__ inline bool FitsUint32(uint64_t value) { + return value <= 0xffffffffULL; +} + +__aicore__ inline void PublishSubmitPmuContext( + __gm__ pa_scheduler::WorkerResult &result, const SubmitPmuContext &context +) { + CcecOps::Publish(&result.pmu_build_variant, pa_scheduler::kBuildVariantSubmitPmu); + CcecOps::Publish( + &result.pmu_phase_id, + static_cast(pa_scheduler::kCompiledSubmitPmuPhase) + ); + CcecOps::Publish(&result.pmu_phase_calls, context.phase_calls); + CcecOps::Publish(&result.pmu_phase_status, context.phase_status); + CcecOps::Publish(&result.pmu_phase_begin_reads, context.begin_reads); + CcecOps::Publish(&result.pmu_phase_end_reads, context.end_reads); + CcecOps::Publish(&result.pmu_phase_elapsed_ticks, context.phase_elapsed_ticks); + CcecOps::Publish(&result.pmu_phase_icache_requests, static_cast(context.phase_requests)); + CcecOps::Publish(&result.pmu_phase_icache_misses, static_cast(context.phase_misses)); + CcecOps::Publish(&result.pmu_shadow_icache_requests, static_cast(context.shadow_requests)); + CcecOps::Publish(&result.pmu_shadow_icache_misses, static_cast(context.shadow_misses)); +} + +} // namespace + +namespace pa_scheduler_ccec { + +__aicore__ inline CcecOps::PmuContext CcecOps::PmuWindowStart( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +) { + using namespace pa_scheduler::ccec_pmu; + (void)worker_id; + SubmitPmuContext context; + const WindowMode mode = static_cast(state->config.reserved[kConfigMode]); + if (mode != WindowMode::SubmitAll) return context; + // Main AICPU owner 已在 launch 前配置并开启计数；先 stop + snapshot/read-clear， + // 再解析 selector，避免这些 ld_dev 污染完整 Submit 窗口。 + bisheng::cce::metrics_prof_stop(); + const PmuRegisterContext registers = ResolvePmuRegisters(state); + context.reg_base = registers.reg_base; + context.selector_status = registers.status; + if (registers.shadow_selectors) { + context.phase_status |= kPhaseStatusShadowSelectors; + } + if (context.reg_base == 0) return context; + (void)ReadObservedCounters(context.reg_base); + (void)ReadShadowCounters(context.reg_base); + bisheng::cce::metrics_prof_start(); + context.started = true; + context.phase_status |= kPhaseStatusWindowStarted; + return context; +} + +__aicore__ inline void CcecOps::PmuPhaseBegin(PmuContext &context) { + if (!context.started || context.reg_base == 0 || context.phase_armed) { + context.boundary_error = true; + return; + } + // counter 在运行中读取即清零；begin 之前的片段只进入 shadow whole。 + const IcacheShadowSnapshot sample = ReadShadowCounters(context.reg_base); + context.shadow_requests += sample.requests; + context.shadow_misses += sample.misses; + ++context.begin_reads; + context.phase_armed = true; + // get_sys_cnt() 是本机已校准为 1 ns/tick 的 A5 系统计数器。该读取位于 + // begin 的两条 ld_dev 之后，因此不会把 read-clear 成本算进阶段时间。 + context.phase_begin_tick = CcecOps::Now(); +} + +__aicore__ inline void CcecOps::PmuPhaseEnd(PmuContext &context) { + // 先取终点再读取 shadow counter，使 end 的两条 ld_dev 同样位于阶段之外。 + const uint64_t phase_end_tick = CcecOps::Now(); + if (!context.started || context.reg_base == 0 || !context.phase_armed) { + context.boundary_error = true; + return; + } + if (phase_end_tick < context.phase_begin_tick) { + context.boundary_error = true; + } else { + context.phase_elapsed_ticks += phase_end_tick - context.phase_begin_tick; + } + // end 读出的片段同时属于完整 shadow 重建与被选中的局部阶段。 + const IcacheShadowSnapshot sample = ReadShadowCounters(context.reg_base); + context.shadow_requests += sample.requests; + context.shadow_misses += sample.misses; + context.phase_requests += sample.requests; + context.phase_misses += sample.misses; + ++context.end_reads; + ++context.phase_calls; + context.phase_armed = false; + context.phase_begin_tick = 0; +} + +__aicore__ inline void CcecOps::PmuWindowStop( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id, PmuContext &context +) { + using namespace pa_scheduler::ccec_pmu; + __gm__ pa_scheduler::WorkerResult &result = state->results[worker_id]; + PmuSnapshot sample; + sample.status = context.selector_status; + if (context.started && context.reg_base != 0) { + // gate 只在整个 Submit 前后各操作一次。停止后先读从未中途清零的 + // primary counter（不含 shadow CNT5）之后，再读取 CNT8/CNT5 tail + // 完成软件重建。 + bisheng::cce::metrics_prof_stop(); + sample = ReadObservedCounters(context.reg_base); + const IcacheShadowSnapshot tail = ReadShadowCounters(context.reg_base); + context.shadow_requests += tail.requests; + context.shadow_misses += tail.misses; + sample.status = context.selector_status | kStatusWindowStarted | kStatusWindowStopped; + context.phase_status |= kPhaseStatusWindowStopped; + if (sample.total_cycles != 0) sample.status |= kStatusTotalNonzero; + } + + if (context.shadow_requests == sample.icache_requests) + context.phase_status |= kPhaseStatusShadowRequestsMatch; + if (context.shadow_misses == sample.icache_misses) + context.phase_status |= kPhaseStatusShadowMissesMatch; + if (!context.boundary_error && !context.phase_armed && + context.begin_reads == context.end_reads && context.end_reads == context.phase_calls) + context.phase_status |= kPhaseStatusBoundariesBalanced; + // 两个 shadow counter 是顺序 ld_dev，并非同一时刻的原子快照；局部 + // phase 的 miss/request 边界会错开数条指令，故不能硬性要求局部 + // miss<=request。A5 上运行中 read-to-clear 还会与同周期事件递增竞争， + // shadow 允许小于未中途读取的 primary，但绝不能反向超过它。primary- + // shadow 是该次采集可直接给出的局部分段误差包络，而不是要静默吞掉的差值。 + if (context.phase_requests <= context.shadow_requests && + context.phase_misses <= context.shadow_misses && + context.shadow_misses <= context.shadow_requests && + context.shadow_requests <= sample.icache_requests && + context.shadow_misses <= sample.icache_misses) + context.phase_status |= kPhaseStatusValuesOrdered; + if (FitsUint32(context.shadow_requests) && FitsUint32(context.shadow_misses) && + FitsUint32(context.phase_requests) && FitsUint32(context.phase_misses)) + context.phase_status |= kPhaseStatusUint32Fit; + + const bool none_shape = + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None && + context.phase_calls == 0 && context.begin_reads == 0 && context.end_reads == 0 && + context.phase_requests == 0 && context.phase_misses == 0 && + context.phase_elapsed_ticks == 0; + const bool running_shape = + pa_scheduler::kCompiledSubmitPmuPhase != pa_scheduler::SubmitPmuPhase::None && + context.phase_calls == state->config.batches * pa_scheduler::kTasksPerBatch && + context.begin_reads == context.phase_calls && + context.end_reads == context.phase_calls; + if (none_shape || running_shape) + context.phase_status |= kPhaseStatusPhaseShape; + const bool phase_time_valid = + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? context.phase_elapsed_ticks == 0 + : context.phase_calls != 0 && context.phase_elapsed_ticks != 0; + if (phase_time_valid) + context.phase_status |= kPhaseStatusTimeValid; + + PublishPmuSnapshot(result, sample); + PublishSubmitPmuContext(result, context); +} +#endif // PA_BUILD_SUBMIT_PMU + +} // namespace + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) && defined(PA_BUILD_AIC) +// split artifact: the runtime entry/state-owner TU calls this orchestration +// function once per kernel launch. It is not a launchable kernel and the +// version script localizes it in the final mixed ELF. +extern "C" __attribute__((noinline, used)) __aicore__ void +pa_scheduler_lazy_sample_callback_orchestration_aic( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +) { + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aic); +} +#elif defined(PA_LAZY_SAMPLE_SPLIT_FINISH) && defined(PA_BUILD_AIV) +extern "C" __attribute__((noinline, used)) __aicore__ void +pa_scheduler_lazy_sample_callback_orchestration_aiv( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +) { + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aiv); +} +#elif defined(PA_BUILD_AIC) +// 同一源码分别按 cube/vec 架构编译；metadata 声明每个物理 block 静态组合 1 个 AIC 与 2 个 AIV。 +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aic, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aic(__gm__ pa_scheduler::SchedulerState *state) { + // 32 个物理 block 的 AIC 直接使用 block_idx，形成连续 worker 0..31。 + const uint32_t worker_id = static_cast(get_block_idx()); + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aic); +} +#elif defined(PA_BUILD_AIV) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aiv, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aiv(__gm__ pa_scheduler::SchedulerState *state) { + // 每个 block 的两个 vector sub-block 展平为 vector_id=2*b+subblock，偏移 32 后形成 worker 32..95。 + const uint32_t vector_id = static_cast(get_block_idx() * get_subblockdim() + get_subblockid()); + const uint32_t worker_id = pa_scheduler::kAicWorkers + vector_id; + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aiv); +} +#else +#error "Compile with PA_BUILD_AIC or PA_BUILD_AIV" +#endif diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pa_scheduler_device_exports.map b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pa_scheduler_device_exports.map new file mode 100644 index 0000000000..b4068c90f1 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pa_scheduler_device_exports.map @@ -0,0 +1,7 @@ +{ + global: + pa_scheduler_0_mix_aic; + pa_scheduler_0_mix_aiv; + local: + *; +}; diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_owner_control.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_owner_control.h new file mode 100644 index 0000000000..eb17c4c1b0 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_owner_control.h @@ -0,0 +1,225 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_CONTROL_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_CONTROL_H_ + +#include +#include + +namespace pa_scheduler::pmu_owner { + +// 这份头文件同时供 x86 host 与 AArch64 AICPU helper 使用。所有跨端字段都采用 +// 固定宽度整数；禁止在 ABI 中放 host 指针、bool、STL 容器或编译器相关位域。 +constexpr uint32_t kPmuOwnerControlMagic = 0x504d554fU; // "PMUO" +constexpr uint32_t kPmuOwnerControlVersion = 1U; + +// DAV_3510 一共有 2 die；每个 die 依次放 18 个 AIC 和 36 个 AIV 物理槽。 +// 当前 A5 stream 实际开放 32 个 AIC 与 64 个 AIV，其余 12 个槽的 MMIO +// 读回不会匹配配置值，因此 owner 必须扫描 108 槽，最终取得 96 个可用槽。 +constexpr uint32_t kPhysicalSubcoreCount = 108U; +constexpr uint32_t kExpectedSubcoreCount = 96U; +constexpr uint32_t kExpectedAicCount = 32U; +constexpr uint32_t kExpectedAivCount = 64U; +constexpr uint32_t kAicPerDie = 18U; +constexpr uint32_t kSubcoresPerDie = 54U; +constexpr uint32_t kConfiguredBitmapWords = 4U; +constexpr uint32_t kDiagnosticIndexUnset = 0xffffffffU; + +static_assert(kExpectedAicCount + kExpectedAivCount == kExpectedSubcoreCount, "active topology count mismatch"); +static_assert(kAicPerDie * 2U == 36U, "physical AIC topology changed"); +static_assert(kSubcoresPerDie * 2U == kPhysicalSubcoreCount, "physical subcore topology changed"); + +// A5 PIPE_UTILIZATION 的正式 counter 槽位布局。submit-pmu 用 CNT8/CNT5 +// 重复配置 I-cache request/miss，作为允许中途 read-to-clear 的 shadow； +// CNT6/7 始终不在阶段边界读取，保留为完整 Submit 的权威对照。 +// +// 不能把 miss 放进 CNT9：A5 b1 实测表明 CNT9 selector 虽能回读 0x35， +// 但计数恒为 0；正式 PIPE_UTIL 表也把 CNT9 标成 unused。0x35 在独立 +// I-cache 微基准的低位 counter 已验证可计数，因此诊断构建让 CNT5 承担 +// shadow miss，并明确放弃该构建中的 MTE3 busy。 +constexpr uint32_t kPmuCounterCount = 10U; +constexpr uint32_t kConfiguredSelectors[kPmuCounterCount] = { + 0x501U, // CNT0: vector busy + 0x301U, // CNT1: cube busy + 0x001U, // CNT2: scalar busy + 0x701U, // CNT3: MTE1 busy + 0x202U, // CNT4: MTE2 busy +#if PA_BUILD_SUBMIT_PMU + 0x035U, // CNT5: shadow I-cache miss + 0x034U, // CNT6: I-cache request（完整 Submit） + 0x035U, // CNT7: I-cache miss（完整 Submit） + 0x034U, // CNT8: shadow I-cache request + 0x000U, // CNT9: A5 PIPE_UTIL 正式未使用 +#else + 0x203U, // CNT5: MTE3 busy + 0x034U, // CNT6: I-cache request + 0x035U, // CNT7: I-cache miss + 0x714U, // CNT8: fix-pipe busy + 0x000U, // CNT9: 未使用 +#endif +}; + +constexpr int32_t kStatusPending = 0x7fffffff; + +// AICPU entry 始终向 runtime 返回 0；协议结果只通过 control.status 回传， +// 从而避免一次可诊断的配置失败被 runtime 升格成整条 stream 异常。 +enum class PmuOwnerStatus : int32_t { + Success = 0, + InvalidArguments = -1, + InvalidControl = -2, + UnexpectedTopology = -3, + AlreadyConfigured = -4, + ConfigureCountMismatch = -5, + ConfigureRollbackFailed = -6, + ConfigureSlotRestoreFailed = -7, + RestoreFailed = -8, +}; + +// 首个异常寄存器使用稳定的枚举编号，host 不需要解析 AICPU 日志即可定位 +// 是基址、selector、计数范围还是 enable 控制读回不一致。 +enum class PmuOwnerField : uint32_t { + None = 0, + Arguments, + ControlMagic, + ControlVersion, + ControlSize, + State, + RegisterBase, + Ctrl0, + Ctrl1, + Selector0, + Selector1, + Selector2, + Selector3, + Selector4, + Selector5, + Selector6, + Selector7, + Selector8, + Selector9, + StartCycleLow, + StartCycleHigh, + StopCycleLow, + StopCycleHigh, + BitmapCount, + TotalCount, + AicCount, + AivCount, +}; + +// 单个物理子核被 owner 改动的完整可恢复状态恰好占一条 cache line。 +// PMU counter 是 read-to-clear，旧 counter 值无法恢复；owner 会话必须独占。 +struct alignas(64) PmuSavedRegisters { + uint32_t ctrl0; + uint32_t ctrl1; + uint32_t selectors[kPmuCounterCount]; + uint32_t start_cycle_low; + uint32_t start_cycle_high; + uint32_t stop_cycle_low; + uint32_t stop_cycle_high; +}; + +// Host 与 AICPU 共享的 owner 状态。前 128B 是命令结果和诊断，随后内嵌 +// 108 个 MMIO 基址、4-word 所有权 bitmap，以及每槽 64B 的 Configure 快照。 +// bitmap 的严格语义是“原值已保存、且 owner 可能已经改写 MMIO、但尚未 +// 完整恢复”的槽；它在 Configure 写第一项 MMIO 前置位，仅在恢复读回完整 +// 一致后清位。Restore 期间不得清零或重建 saved[]，只能按 bitmap 逆序消费。 +struct alignas(64) PmuOwnerControl { + uint32_t magic; + uint32_t version; + uint32_t struct_bytes; + volatile int32_t status; + + uint32_t configured; + uint32_t expected_total; + uint32_t expected_aic; + uint32_t expected_aiv; + + // active_* 与 bitmap 表示仍由本 owner 持有、尚未恢复的物理槽。 + uint32_t active_total; + uint32_t active_aic; + uint32_t active_aiv; + // discovered_* 保留本次 Configure 扫描结果；即使计数不匹配后回滚， + // host 仍能看到回滚前究竟探测到了多少 AIC/AIV。 + uint32_t discovered_total; + uint32_t discovered_aic; + uint32_t discovered_aiv; + uint32_t skipped_total; + + uint32_t first_failed_index; + uint32_t first_failed_field; + uint32_t first_failed_observed; + uint32_t first_failed_expected; + + uint32_t restore_failures; + uint32_t first_restore_failed_index; + uint32_t first_restore_failed_field; + uint32_t first_restore_failed_observed; + uint32_t first_restore_failed_expected; + uint32_t reserved_header[8]; + + uint64_t register_bases[kPhysicalSubcoreCount]; + // 字段名保留 configured_bitmap 以稳定 host/device ABI；失败路径中它还会 + // 临时包含“配置未通过但恢复仍待重试”的 owned 槽。 + uint32_t configured_bitmap[kConfiguredBitmapWords]; + // 让 saved[] 从新的 64B cache line 开始；该 padding 不承载协议含义。 + uint32_t reserved_bitmap[4]; + PmuSavedRegisters saved[kPhysicalSubcoreCount]; +}; + +static_assert(sizeof(PmuSavedRegisters) == 64U, "one saved PMU slot must occupy one cache line"); +static_assert(alignof(PmuSavedRegisters) == 64U, "saved PMU slot alignment changed"); +static_assert(offsetof(PmuOwnerControl, status) == 12U, "PMU owner status offset changed"); +static_assert(offsetof(PmuOwnerControl, register_bases) == 128U, "PMU owner header must occupy two cache lines"); +static_assert(offsetof(PmuOwnerControl, configured_bitmap) == 992U, "PMU owner bitmap offset changed"); +static_assert(offsetof(PmuOwnerControl, saved) == 1024U, "PMU owner saved area must be cache-line aligned"); +static_assert(sizeof(PmuOwnerControl) == 7936U, "PMU owner control ABI changed"); +static_assert(sizeof(PmuOwnerControl) % 64U == 0U, "PMU owner control must use complete cache lines"); +static_assert(alignof(PmuOwnerControl) == 64U, "PMU owner control alignment changed"); + +inline bool IsAicPhysicalSlot(uint32_t index) +{ + return index < kPhysicalSubcoreCount && (index % kSubcoresPerDie) < kAicPerDie; +} + +inline bool IsConfigured(const PmuOwnerControl &control, uint32_t index) +{ + return index < kPhysicalSubcoreCount && + (control.configured_bitmap[index / 32U] & (1U << (index % 32U))) != 0U; +} + +inline void SetConfigured(PmuOwnerControl *control, uint32_t index) +{ + if (index < kPhysicalSubcoreCount) { + control->configured_bitmap[index / 32U] |= 1U << (index % 32U); + } +} + +inline void ClearConfigured(PmuOwnerControl *control, uint32_t index) +{ + if (index < kPhysicalSubcoreCount) { + control->configured_bitmap[index / 32U] &= ~(1U << (index % 32U)); + } +} + +inline uint32_t CountConfigured(const PmuOwnerControl &control) +{ + uint32_t count = 0U; + for (uint32_t index = 0U; index < kPhysicalSubcoreCount; ++index) { + count += IsConfigured(control, index) ? 1U : 0U; + } + return count; +} + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_CONTROL_H_ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_owner_host.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_owner_host.h new file mode 100644 index 0000000000..68dfac042e --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_owner_host.h @@ -0,0 +1,345 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_HOST_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_HOST_H_ + +#include "pmu_owner_control.h" +#include "pmu_owner_main_abi.h" +#include "pmu_owner_main_loader.h" + +#include "acl/acl.h" + +#include +#include +#include +#include +#include +#include + +namespace pa_scheduler::pmu_owner { + +inline bool OwnerCheckAcl(aclError error, const char *label) +{ + if (error == ACL_SUCCESS) return true; + std::fprintf(stderr, "ACL error %d: %s\n", static_cast(error), label); + return false; +} + +inline std::string ArtifactBesideKernel(const std::string &kernel_path, const char *name) +{ + const size_t slash = kernel_path.find_last_of('/'); + return slash == std::string::npos ? std::string(name) : kernel_path.substr(0, slash + 1U) + name; +} + +inline const char *OwnerFieldName(PmuOwnerField field) +{ + switch (field) { + case PmuOwnerField::None: return "none"; + case PmuOwnerField::Arguments: return "arguments"; + case PmuOwnerField::ControlMagic: return "control-magic"; + case PmuOwnerField::ControlVersion: return "control-version"; + case PmuOwnerField::ControlSize: return "control-size"; + case PmuOwnerField::State: return "state"; + case PmuOwnerField::RegisterBase: return "register-base"; + case PmuOwnerField::Ctrl0: return "ctrl0"; + case PmuOwnerField::Ctrl1: return "ctrl1"; + case PmuOwnerField::Selector0: return "selector0"; + case PmuOwnerField::Selector1: return "selector1"; + case PmuOwnerField::Selector2: return "selector2"; + case PmuOwnerField::Selector3: return "selector3"; + case PmuOwnerField::Selector4: return "selector4"; + case PmuOwnerField::Selector5: return "selector5"; + case PmuOwnerField::Selector6: return "selector6"; + case PmuOwnerField::Selector7: return "selector7"; + case PmuOwnerField::Selector8: return "selector8"; + case PmuOwnerField::Selector9: return "selector9"; + case PmuOwnerField::StartCycleLow: return "start-cycle-low"; + case PmuOwnerField::StartCycleHigh: return "start-cycle-high"; + case PmuOwnerField::StopCycleLow: return "stop-cycle-low"; + case PmuOwnerField::StopCycleHigh: return "stop-cycle-high"; + case PmuOwnerField::BitmapCount: return "bitmap-count"; + case PmuOwnerField::TotalCount: return "total-count"; + case PmuOwnerField::AicCount: return "aic-count"; + case PmuOwnerField::AivCount: return "aiv-count"; + } + return "unknown"; +} + +struct ActiveSubcoreLimits { + uint32_t aic = 0U; + uint32_t aiv = 0U; + uint32_t total = 0U; +}; + +inline bool QueryActiveSubcoreLimits(aclrtStream scheduling_stream, ActiveSubcoreLimits *limits) +{ + if (scheduling_stream == nullptr || limits == nullptr) { + std::fprintf(stderr, "Cannot query active PMU subcores with a null stream/result.\n"); + return false; + } + uint32_t aic = 0U; + uint32_t aiv = 0U; + const aclError aic_error = + aclrtGetStreamResLimit(scheduling_stream, ACL_RT_DEV_RES_CUBE_CORE, &aic); + const aclError aiv_error = + aclrtGetStreamResLimit(scheduling_stream, ACL_RT_DEV_RES_VECTOR_CORE, &aiv); + const uint64_t total = static_cast(aic) + aiv; + if (aic_error != ACL_SUCCESS || aiv_error != ACL_SUCCESS || + aic != kExpectedAicCount || aiv != kExpectedAivCount || total != kExpectedSubcoreCount) { + std::fprintf( + stderr, + "Unexpected stream PMU topology: aic_error=%d aiv_error=%d " + "aic=%u/%u aiv=%u/%u total=%llu/%u\n", + static_cast(aic_error), static_cast(aiv_error), aic, kExpectedAicCount, + aiv, kExpectedAivCount, static_cast(total), kExpectedSubcoreCount + ); + return false; + } + limits->aic = aic; + limits->aiv = aiv; + limits->total = static_cast(total); + std::printf( + "[PMU_OWNER] stream_active aic=%u aiv=%u total=%u physical_slots=%u\n", + limits->aic, limits->aiv, limits->total, kPhysicalSubcoreCount + ); + return true; +} + +// mixed launch 的一个物理 block 必须同时拥有 1 个 AIC 和相邻的 2 个 AIV。 +// 只检查 32/64 总数仍可能放过孤立 AIV；这里直接按两 die 的真实编号布局验闭包。 +inline bool ValidateConfiguredTripletTopology(const PmuOwnerControl &control) +{ + uint32_t complete_triplets = 0U; + uint32_t broken_triplets = 0U; + const uint32_t dies = kPhysicalSubcoreCount / kSubcoresPerDie; + for (uint32_t die = 0U; die < dies; ++die) { + const uint32_t die_base = die * kSubcoresPerDie; + for (uint32_t local = 0U; local < kAicPerDie; ++local) { + const bool aic = IsConfigured(control, die_base + local); + const bool aiv0 = IsConfigured(control, die_base + kAicPerDie + local * 2U); + const bool aiv1 = IsConfigured(control, die_base + kAicPerDie + local * 2U + 1U); + if (aic == aiv0 && aic == aiv1) { + complete_triplets += aic ? 1U : 0U; + } else { + ++broken_triplets; + } + } + } + const bool passed = complete_triplets == kExpectedAicCount && broken_triplets == 0U; + std::printf( + "[ASSERT] %-48s %s (complete=%u broken=%u)\n", + "PMU owner bitmap is complete 1-AIC + 2-AIV triplets", + passed ? "PASS" : "FAIL", complete_triplets, broken_triplets + ); + return passed; +} + +// owner 命令使用独立 stream，但通过 mode=0 JSON 在主 aicpu_scheduler 中执行。 +// Configure 同步完成后才允许启动 AICore；AICore 正常或异常退出后，Restore +// 都不会依赖业务 stream。MMIO 映射必须保持到 Finalize 完成之后。 +class PmuOwnerSession { +public: + PmuOwnerSession() = default; + PmuOwnerSession(const PmuOwnerSession &) = delete; + PmuOwnerSession &operator=(const PmuOwnerSession &) = delete; + + ~PmuOwnerSession() + { + if (HasResources()) (void)Finalize(); + } + + bool Initialize( + uint32_t device, aclrtStream scheduling_stream, const std::string &dispatcher_path, + const std::string &owner_path, const std::vector ®ister_bases + ) + { + if (HasResources() || register_bases.size() != kPhysicalSubcoreCount) { + std::fprintf( + stderr, "Invalid PMU owner initialization state or register table size: %zu\n", + register_bases.size() + ); + return false; + } + device_ = device; + if (!QueryActiveSubcoreLimits(scheduling_stream, &limits_)) return false; + if (!OwnerCheckAcl(aclrtCreateStream(&owner_stream_), "aclrtCreateStream(PMU owner)")) return false; + if (loader_.Initialize( + dispatcher_path, owner_path, owner_stream_, static_cast(device_) + ) != 0) { + return false; + } + if (!OwnerCheckAcl( + aclrtMalloc(&control_device_, sizeof(PmuOwnerControl), ACL_MEM_MALLOC_NORMAL_ONLY), + "aclrtMalloc(PMU owner control)" + )) { + return false; + } + if ((reinterpret_cast(control_device_) & (alignof(PmuOwnerControl) - 1U)) != 0U) { + std::fprintf(stderr, "PMU owner control is not 64-byte aligned: %p\n", control_device_); + return false; + } + + control_ = PmuOwnerControl{}; + control_.magic = kPmuOwnerControlMagic; + control_.version = kPmuOwnerControlVersion; + control_.struct_bytes = sizeof(PmuOwnerControl); + control_.status = kStatusPending; + control_.expected_total = limits_.total; + control_.expected_aic = limits_.aic; + control_.expected_aiv = limits_.aiv; + std::memcpy(control_.register_bases, register_bases.data(), sizeof(control_.register_bases)); + if (!OwnerCheckAcl( + aclrtMemcpy( + control_device_, sizeof(control_), &control_, sizeof(control_), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D initial PMU owner control)" + )) { + return false; + } + ready_ = true; + return true; + } + + bool Configure() + { + if (!ready_) return false; + const bool command_ok = RunCommand(PmuOwnerMainCommand::Configure, "Configure"); + configured_ = CountConfigured(control_) != 0U; + const uint32_t bitmap_count = CountConfigured(control_); + const bool triplets_ok = ValidateConfiguredTripletTopology(control_); + const bool state_ok = control_.status == static_cast(PmuOwnerStatus::Success) && + control_.configured == 1U && control_.active_total == limits_.total && + control_.active_aic == limits_.aic && control_.active_aiv == limits_.aiv && + control_.discovered_total == limits_.total && control_.discovered_aic == limits_.aic && + control_.discovered_aiv == limits_.aiv && bitmap_count == limits_.total && + control_.skipped_total + bitmap_count == kPhysicalSubcoreCount && triplets_ok; + PrintControl("Configure", bitmap_count); + return command_ok && state_ok; + } + + bool Restore() + { + if (control_device_ == nullptr || owner_stream_ == nullptr || !loader_.IsInitialized()) { + return !configured_; + } + const bool command_ok = RunCommand(PmuOwnerMainCommand::Restore, "Restore"); + const uint32_t bitmap_count = CountConfigured(control_); + const bool state_ok = control_.status == static_cast(PmuOwnerStatus::Success) && + control_.configured == 0U && control_.active_total == 0U && + control_.active_aic == 0U && control_.active_aiv == 0U && bitmap_count == 0U; + configured_ = bitmap_count != 0U; + PrintControl("Restore", bitmap_count); + return command_ok && state_ok; + } + + bool Finalize() + { + bool ok = true; + if (control_device_ != nullptr && owner_stream_ != nullptr && loader_.IsInitialized()) { + bool restored = Restore(); + if (!restored) restored = Restore(); + ok &= restored; + } else { + ok &= !configured_; + } + ok &= loader_.Finalize() == 0; + if (control_device_ != nullptr) { + ok &= OwnerCheckAcl(aclrtFree(control_device_), "aclrtFree(PMU owner control)"); + control_device_ = nullptr; + } + if (owner_stream_ != nullptr) { + ok &= OwnerCheckAcl(aclrtDestroyStream(owner_stream_), "aclrtDestroyStream(PMU owner)"); + owner_stream_ = nullptr; + } + ready_ = false; + configured_ = false; + std::printf("[PMU_OWNER] restore_and_cleanup=%s\n", ok ? "PASS" : "FAIL"); + return ok; + } + + uint64_t RegisterTableDeviceAddress() const + { + if (control_device_ == nullptr) return 0U; + return reinterpret_cast(control_device_) + offsetof(PmuOwnerControl, register_bases); + } + + const PmuOwnerControl &Control() const { return control_; } + + bool IsConfiguredSubcore(uint32_t index) const + { + return ready_ && IsConfigured(control_, index); + } + +private: + bool HasResources() const + { + return owner_stream_ != nullptr || control_device_ != nullptr || loader_.IsInitialized(); + } + + bool RunCommand(PmuOwnerMainCommand command, const char *label) + { + const PmuOwnerMainKernelArgs arguments = MakePmuOwnerMainKernelArgs( + reinterpret_cast(control_device_), command, device_ + ); + const std::string sync_label = std::string("aclrtSynchronizeStream(PMU ") + label + ")"; + const std::string copy_label = std::string("aclrtMemcpy(D2H PMU ") + label + ")"; + if (loader_.Launch(owner_stream_, const_cast(&arguments), sizeof(arguments)) != 0 || + !OwnerCheckAcl(aclrtSynchronizeStream(owner_stream_), sync_label.c_str()) || + !OwnerCheckAcl( + aclrtMemcpy( + &control_, sizeof(control_), control_device_, sizeof(control_), ACL_MEMCPY_DEVICE_TO_HOST + ), + copy_label.c_str() + )) { + return false; + } + return true; + } + + void PrintControl(const char *command, uint32_t bitmap_count) const + { + const auto failed_field = static_cast(control_.first_failed_field); + const auto restore_field = static_cast(control_.first_restore_failed_field); + std::printf( + "[PMU_OWNER] command=%s status=%d configured=%u active=%u/%u/%u " + "discovered=%u/%u/%u bitmap=%u skipped=%u first_failed=%u:%s(%u):0x%x/0x%x " + "restore_failures=%u first_restore=%u:%s(%u):0x%x/0x%x\n", + command, static_cast(control_.status), control_.configured, + control_.active_total, control_.active_aic, control_.active_aiv, + control_.discovered_total, control_.discovered_aic, control_.discovered_aiv, + bitmap_count, control_.skipped_total, control_.first_failed_index, + OwnerFieldName(failed_field), control_.first_failed_field, + control_.first_failed_observed, control_.first_failed_expected, + control_.restore_failures, control_.first_restore_failed_index, + OwnerFieldName(restore_field), control_.first_restore_failed_field, + control_.first_restore_failed_observed, control_.first_restore_failed_expected + ); + std::printf( + "[PMU_OWNER] configured_bitmap=%08x:%08x:%08x:%08x\n", + control_.configured_bitmap[3], control_.configured_bitmap[2], + control_.configured_bitmap[1], control_.configured_bitmap[0] + ); + } + + uint32_t device_ = 0U; + aclrtStream owner_stream_ = nullptr; + MainAicpuLoader loader_; + void *control_device_ = nullptr; + PmuOwnerControl control_{}; + ActiveSubcoreLimits limits_{}; + bool ready_ = false; + bool configured_ = false; +}; + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_HOST_H_ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_owner_main_abi.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_owner_main_abi.h new file mode 100644 index 0000000000..d6ba9e2239 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_owner_main_abi.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_ABI_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_ABI_H_ + +#include +#include + +namespace pa_scheduler::pmu_owner { + +// 主 aicpu_scheduler 的统一入口 simpler_aicpu_exec 根据该命令选择配置或 +// 恢复。0 特意保留为 Invalid，避免零初始化参数意外改写 PMU 寄存器。 +enum class PmuOwnerMainCommand : uint32_t { + Invalid = 0U, + Configure = 1U, + Restore = 2U, +}; + +// 该结构逐字段复刻 A5 KernelArgs 的 152B ABI，但只使用固定宽度整数，因而 +// 不依赖 Simpler 的 DeviceArgs/Runtime C++ 类型。runtime_args_device 指向 +// PmuOwnerControl；command 位于原 enable_profiling_flag 的 offset 128。 +// 其余字段保持为零，既满足主 aicpu_scheduler 固定布局，也不引入外部依赖。 +struct PmuOwnerMainKernelArgs { + uint64_t unused[5]; // 0..39 + uint64_t device_args_device; // 40，当前 owner 不使用 + uint64_t runtime_args_device; // 48，PmuOwnerControl 的 GM 地址 + uint64_t register_bases_device; // 56，当前 control 已内嵌基址，保持为零 + uint64_t dump_data_base; // 64 + uint64_t l2_swimlane_data_base; // 72 + uint64_t pmu_data_base; // 80 + uint64_t dep_gen_data_base; // 88 + uint64_t l2_swimlane_rotation_table; // 96 + uint64_t aicore_pmu_ring_addrs; // 104 + uint64_t scope_stats_data_base; // 112 + uint32_t log_level; // 120 + uint32_t log_info_v; // 124 + uint32_t command; // 128，PmuOwnerMainCommand + uint32_t reserved_alignment; // 132 + uint64_t device_wall_data_base; // 136 + uint32_t device_id; // 144 + uint32_t force_simt_anchor; // 148 +}; + +static_assert(sizeof(PmuOwnerMainCommand) == sizeof(uint32_t), "PMU owner command ABI changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, device_args_device) == 40U, "device args offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, runtime_args_device) == 48U, "control pointer offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, register_bases_device) == 56U, "register pointer offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, command) == 128U, "PMU owner command offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, device_id) == 144U, "device id offset changed"); +static_assert(sizeof(PmuOwnerMainKernelArgs) == 152U, "main aicpu_scheduler KernelArgs ABI changed"); +static_assert(alignof(PmuOwnerMainKernelArgs) == 8U, "KernelArgs alignment changed"); + +inline PmuOwnerMainKernelArgs MakePmuOwnerMainKernelArgs( + uint64_t control_device, PmuOwnerMainCommand command, uint32_t device_id +) +{ + PmuOwnerMainKernelArgs arguments{}; + arguments.runtime_args_device = control_device; + arguments.command = static_cast(command); + arguments.device_id = device_id; + return arguments; +} + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_ABI_H_ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_owner_main_loader.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_owner_main_loader.h new file mode 100644 index 0000000000..ea34f5fe59 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_owner_main_loader.h @@ -0,0 +1,389 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_LOADER_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_LOADER_H_ + +// PMU owner 的自包含 host 装载器： +// 1. 通过 libaicpu_extend_kernels bootstrap 临时 dispatcher； +// 2. dispatcher 将 owner SO 落到主 aicpu_scheduler 的预安装目录； +// 3. 用 cpuKernelMode=0 JSON 注册 owner 的 simpler_aicpu_exec； +// 4. 后续 Configure/Restore 都用缓存的 rtFuncHandle 直接下发。 +// +// 本头文件故意不定义 owner 命令字段。Launch 接受调用方构造的完整参数块， +// 从而让装载 ABI 与 PMU 状态机 ABI 解耦，也便于先独立验证 Path-A。 + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "runtime/rt.h" +#include "runtime/runtime/rts/rts_kernel.h" + +namespace pa_scheduler::pmu_owner { + +class MainAicpuLoader { +public: + MainAicpuLoader() = default; + MainAicpuLoader(const MainAicpuLoader &) = delete; + MainAicpuLoader &operator=(const MainAicpuLoader &) = delete; + MainAicpuLoader(MainAicpuLoader &&) = delete; + MainAicpuLoader &operator=(MainAicpuLoader &&) = delete; + + ~MainAicpuLoader() { (void)Finalize(); } + + // stream 必须属于当前 device，并且调用期间当前 ACL device 不能切换。 + // 成功后 owner SO 已注册到主 aicpu_scheduler，但尚未执行任何 PMU 命令。 + int Initialize( + const std::string &dispatcher_so_path, const std::string &owner_so_path, + aclrtStream stream, int32_t device_id + ) + { + if (IsInitialized() || stream == nullptr || device_id < 0) { + return Fail("Initialize received invalid state, stream, or device id", kInvalidArgument); + } + + const std::vector dispatcher = ReadBinary(dispatcher_so_path); + const std::vector owner = ReadBinary(owner_so_path); + if (dispatcher.empty() || owner.empty()) { + std::fprintf( + stderr, "[PMU_OWNER_LOADER] cannot read dispatcher/owner: %s (%zu B), %s (%zu B)\n", + dispatcher_so_path.c_str(), dispatcher.size(), owner_so_path.c_str(), owner.size() + ); + return kFileError; + } + + device_id_ = device_id; + owner_fingerprint_ = FingerprintBytes(owner.data(), owner.size()); + owner_so_basename_ = MakeOwnerSoBasename(owner_fingerprint_, device_id_); + op_type_ = MakeOpType(owner_fingerprint_, device_id_); + + int result = Bootstrap(dispatcher, owner, stream); + if (result == 0) result = RegisterOwner(); + if (result != 0) { + (void)Finalize(); + return result; + } + return 0; + } + + // 参数块由 runtime 在 launch 时复制；调用方只需保证本函数返回前 host + // buffer 有效。参数中的 GM 指针仍必须在设备命令同步结束前保持有效。 + int Launch( + aclrtStream stream, void *kernel_arguments, size_t argument_bytes, + uint32_t aicpu_blocks = 1U + ) const + { + if (!IsInitialized() || stream == nullptr || kernel_arguments == nullptr || + argument_bytes == 0U || argument_bytes > std::numeric_limits::max() || + aicpu_blocks == 0U) { + return Fail("Launch received invalid state or arguments", kInvalidArgument); + } + + rtCpuKernelArgs_t cpu_arguments = {}; + cpu_arguments.baseArgs.args = kernel_arguments; + cpu_arguments.baseArgs.argsSize = static_cast(argument_bytes); + rtKernelLaunchCfg_t launch_config = {}; + rtLaunchKernelAttr_t launch_attribute = {}; + launch_config.attrs = &launch_attribute; + launch_config.numAttrs = 0U; + + const rtError_t result = rtsLaunchCpuKernel( + function_handle_, aicpu_blocks, static_cast(stream), + &launch_config, &cpu_arguments + ); + if (result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsLaunchCpuKernel failed: %d\n", result); + } + return static_cast(result); + } + + // Finalize 只释放 host/runtime 注册资源，不删除设备侧预安装 SO；后者按内容 + // 指纹命名，可由同一设备上的后续进程原子覆盖。 + int Finalize() + { + int result = 0; + function_handle_ = nullptr; + if (binary_handle_ != nullptr) { + const rtError_t unload_result = rtsBinaryUnload(binary_handle_); + if (unload_result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsBinaryUnload failed: %d\n", unload_result); + result = static_cast(unload_result); + } + binary_handle_ = nullptr; + } + if (!json_path_.empty()) { + if (std::remove(json_path_.c_str()) != 0 && result == 0) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] remove JSON failed: %s\n", json_path_.c_str()); + result = kFileError; + } + json_path_.clear(); + } + device_id_ = -1; + owner_fingerprint_ = 0U; + owner_so_basename_.clear(); + op_type_.clear(); + return result; + } + + bool IsInitialized() const { return binary_handle_ != nullptr && function_handle_ != nullptr; } + uint64_t OwnerFingerprint() const { return owner_fingerprint_; } + const std::string &OwnerSoBasename() const { return owner_so_basename_; } + const std::string &OpType() const { return op_type_; } + +private: + static constexpr int kInvalidArgument = -1; + static constexpr int kFileError = -2; + static constexpr int kBootstrapError = -3; + static constexpr uint64_t kFnvOffsetBasis = UINT64_C(14695981039346656037); + static constexpr uint64_t kFnvPrime = UINT64_C(1099511628211); + static constexpr const char *kOwnerFunction = "simpler_aicpu_exec"; + + struct DeviceBuffer { + void *address = nullptr; + DeviceBuffer() = default; + DeviceBuffer(const DeviceBuffer &) = delete; + DeviceBuffer &operator=(const DeviceBuffer &) = delete; + ~DeviceBuffer() + { + if (address != nullptr) (void)aclrtFree(address); + } + aclError Allocate(size_t bytes) + { + return aclrtMalloc(&address, bytes, ACL_MEM_MALLOC_HUGE_FIRST); + } + }; + + static int Fail(const char *message, int code) + { + std::fprintf(stderr, "[PMU_OWNER_LOADER] %s\n", message); + return code; + } + + static std::vector ReadBinary(const std::string &path) + { + std::ifstream input(path, std::ios::binary | std::ios::ate); + if (!input.is_open()) return {}; + const std::streampos end = input.tellg(); + if (end <= std::streampos(0) || + static_cast(end) > static_cast(std::numeric_limits::max())) { + return {}; + } + std::vector bytes(static_cast(end)); + input.seekg(0, std::ios::beg); + if (!input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size()))) { + return {}; + } + return bytes; + } + + static uint64_t FingerprintBytes(const void *data, size_t bytes) + { + const auto *input = static_cast(data); + uint64_t hash = kFnvOffsetBasis; + for (size_t index = 0U; index < bytes; ++index) { + hash ^= input[index]; + hash *= kFnvPrime; + } + return hash; + } + + static std::string MakeOwnerSoBasename(uint64_t fingerprint, int32_t device_id) + { + char name[128] = {}; + (void)snprintf( + name, sizeof(name), "pa_scheduler_pmu_owner_%016llx_d%d.so", + static_cast(fingerprint), device_id + ); + return name; + } + + static std::string MakeOpType(uint64_t fingerprint, int32_t device_id) + { + char name[160] = {}; + (void)snprintf( + name, sizeof(name), "pa_scheduler_pmu_owner_%016llx_d%d", + static_cast(fingerprint), device_id + ); + return name; + } + + int Bootstrap( + const std::vector &dispatcher, const std::vector &owner, + aclrtStream stream + ) const + { + DeviceBuffer dispatcher_device; + DeviceBuffer owner_device; + DeviceBuffer device_args; + aclError acl_result = dispatcher_device.Allocate(dispatcher.size()); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMalloc(dispatcher)", acl_result); + acl_result = aclrtMemcpy( + dispatcher_device.address, dispatcher.size(), dispatcher.data(), dispatcher.size(), + ACL_MEMCPY_HOST_TO_DEVICE + ); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMemcpy(dispatcher H2D)", acl_result); + + acl_result = owner_device.Allocate(owner.size()); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMalloc(owner)", acl_result); + acl_result = aclrtMemcpy( + owner_device.address, owner.size(), owner.data(), owner.size(), ACL_MEMCPY_HOST_TO_DEVICE + ); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMemcpy(owner H2D)", acl_result); + + constexpr size_t kDeviceArgsBytes = 160U; + uint8_t host_device_args[kDeviceArgsBytes] = {}; + const auto write_qword = [&](size_t offset, uint64_t value) { + std::memcpy(host_device_args + offset, &value, sizeof(value)); + }; + write_qword(96U, reinterpret_cast(dispatcher_device.address)); + write_qword(104U, static_cast(dispatcher.size())); + write_qword(112U, static_cast(device_id_)); + write_qword(120U, reinterpret_cast(owner_device.address)); + write_qword(128U, static_cast(owner.size())); + + acl_result = device_args.Allocate(kDeviceArgsBytes); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMalloc(bootstrap args)", acl_result); + acl_result = aclrtMemcpy( + device_args.address, kDeviceArgsBytes, host_device_args, kDeviceArgsBytes, + ACL_MEMCPY_HOST_TO_DEVICE + ); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMemcpy(bootstrap args H2D)", acl_result); + + // k_args 总长和三个字符串 offset 与仓内已上板的 Path-A 完全一致。 + struct BootstrapArguments { + struct { + uint64_t unused[5]; + uint64_t device_args_address; + uint64_t padding[20]; + } kernel_args; + char kernel_name[32]; + char so_name[32]; + char op_name[32]; + } arguments = {}; + static_assert(offsetof(BootstrapArguments, kernel_args.device_args_address) == 40U, "bootstrap ABI changed"); + arguments.kernel_args.device_args_address = reinterpret_cast(device_args.address); + constexpr char kBootstrapKernel[] = "DynTileFwkKernelServerInit"; + constexpr char kBootstrapSo[] = "libaicpu_extend_kernels.so"; + static_assert(sizeof(kBootstrapKernel) <= sizeof(arguments.kernel_name), "bootstrap kernel name too long"); + static_assert(sizeof(kBootstrapSo) <= sizeof(arguments.so_name), "bootstrap SO name too long"); + std::memcpy(arguments.kernel_name, kBootstrapKernel, sizeof(kBootstrapKernel)); + std::memcpy(arguments.so_name, kBootstrapSo, sizeof(kBootstrapSo)); + + rtAicpuArgsEx_t runtime_arguments = {}; + runtime_arguments.args = &arguments; + runtime_arguments.argsSize = sizeof(arguments); + runtime_arguments.kernelNameAddrOffset = offsetof(BootstrapArguments, kernel_name); + runtime_arguments.soNameAddrOffset = offsetof(BootstrapArguments, so_name); + + const rtError_t launch_result = rtAicpuKernelLaunchExWithArgs( + rtKernelType_t::KERNEL_TYPE_AICPU_KFC, "AST_DYN_AICPU", 1U, + &runtime_arguments, nullptr, static_cast(stream), 0U + ); + if (launch_result != RT_ERROR_NONE) { + std::fprintf( + stderr, "[PMU_OWNER_LOADER] rtAicpuKernelLaunchExWithArgs failed: %d\n", + launch_result + ); + return static_cast(launch_result); + } + acl_result = aclrtSynchronizeStream(stream); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtSynchronizeStream(bootstrap)", acl_result); + return 0; + } + + int RegisterOwner() + { + char path[256] = {}; + (void)snprintf( + path, sizeof(path), "/tmp/pa_scheduler_pmu_owner_%016llx_d%d_p%d_i%016llx.json", + static_cast(owner_fingerprint_), device_id_, static_cast(getpid()), + static_cast(reinterpret_cast(this)) + ); + json_path_ = path; + if (!WriteJson()) return kFileError; + + rtLoadBinaryOption_t option = {}; + option.optionId = RT_LOAD_BINARY_OPT_CPU_KERNEL_MODE; + option.value.cpuKernelMode = 0; + rtLoadBinaryConfig_t configuration = {}; + configuration.options = &option; + configuration.numOpt = 1U; + + rtError_t result = rtsBinaryLoadFromFile(json_path_.c_str(), &configuration, &binary_handle_); + if (result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsBinaryLoadFromFile failed: %d\n", result); + return static_cast(result); + } + result = rtsFuncGetByName(binary_handle_, op_type_.c_str(), &function_handle_); + if (result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsFuncGetByName(%s) failed: %d\n", op_type_.c_str(), result); + return static_cast(result); + } + return 0; + } + + bool WriteJson() const + { + std::ofstream json(json_path_, std::ios::out | std::ios::trunc); + if (!json.is_open()) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] cannot create JSON: %s\n", json_path_.c_str()); + return false; + } + // 所有动态字段仅含固定前缀、十六进制、十进制和下划线，不需要 JSON 转义。 + json << "{\n" + << " \"" << op_type_ << "\": {\n" + << " \"opInfo\": {\n" + << " \"functionName\": \"" << kOwnerFunction << "\",\n" + << " \"kernelSo\": \"" << owner_so_basename_ << "\",\n" + << " \"opKernelLib\": \"AICPUKernel\",\n" + << " \"computeCost\": \"100\",\n" + << " \"engine\": \"DNN_VM_AICPU\",\n" + << " \"flagAsync\": \"False\",\n" + << " \"flagPartial\": \"False\",\n" + << " \"userDefined\": \"False\"\n" + << " }\n" + << " }\n" + << "}\n"; + json.close(); + if (!json) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] writing JSON failed: %s\n", json_path_.c_str()); + return false; + } + return true; + } + + static int ReportAcl(const char *operation, aclError result) + { + std::fprintf(stderr, "[PMU_OWNER_LOADER] %s failed: %d\n", operation, static_cast(result)); + return static_cast(result == ACL_SUCCESS ? kBootstrapError : result); + } + + int32_t device_id_ = -1; + uint64_t owner_fingerprint_ = 0U; + std::string owner_so_basename_; + std::string op_type_; + std::string json_path_; + rtBinHandle binary_handle_ = nullptr; + rtFuncHandle function_handle_ = nullptr; +}; + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_LOADER_H_ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_probe.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_probe.h new file mode 100644 index 0000000000..a2e8cab6a0 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/ccec/pmu_probe.h @@ -0,0 +1,185 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_PROBE_H +#define PA_SCHEDULER_CCEC_PMU_PROBE_H + +#include + +#include "../common/pa_model.h" + +namespace pa_scheduler::ccec_pmu { + +// Empty/Scalar/ScalarDouble 在调度结束后校准门控底噪和 scalar 正向响应； +// IcacheSingle 在每核上成对累计隔离的 cold/warm 目标调用；SubmitAll 则在 +// 公共调度器 hook 内覆盖本 worker 的完整 Submit 回放窗口。 +enum class WindowMode : uint32_t { + Off = 0, + Empty = 1, + Scalar = 2, + ScalarDouble = 3, + // 保留已落远端的 I-cache 校准模式值，避免 standalone host/kernel 混用旧产物时 + // 把校准请求误解释成 Submit 窗口；新增模式只在枚举尾部扩展。 + IcacheSingle = 4, + // SubmitAll 从本 worker 的 orchestration/Submit 回放前开始，到最后一次 + // Submit 返回后停止。 + SubmitAll = 5, +}; + +inline bool IsSubmitWindow(WindowMode mode) { + return mode == WindowMode::SubmitAll; +} + +// RunConfig::reserved 保持既有 64B ABI；CCEC 独占解释以下五个槽位，其他后端仍看到全零。 +constexpr uint32_t kConfigMode = 0; +constexpr uint32_t kConfigWorkAmount = 1; +constexpr uint32_t kConfigScalarNops = kConfigWorkAmount; +constexpr uint32_t kConfigIcacheTrials = kConfigWorkAmount; +constexpr uint32_t kConfigRegTableLow = 2; +constexpr uint32_t kConfigRegTableHigh = 3; +constexpr uint32_t kConfigMagic = 4; +constexpr uint32_t kConfigMagicValue = 0x504d5531U; // "PMU1" + +// DAV_3510 有 36 个物理 AICore，每个 AICore 展开为 1 AIC + 2 AIV，共 108 个物理子核编号。 +constexpr uint32_t kPhysicalAicoreCount = 36; +constexpr uint32_t kPhysicalSubcoreCount = 108; +constexpr uint32_t kAicorePerDie = 18; +constexpr uint32_t kSubcoresPerDie = 54; +constexpr uint64_t kAivFirstOffset = 0x100000ULL; +constexpr uint64_t kAivSecondOffset = 0x200000ULL; +constexpr uint32_t kAicoreMapBytes = 0x300000U; + +// PIPE_UTILIZATION 事件由 standalone Main AICPU owner 配置，kernel 逐核读回并核对 selector。 +constexpr uint32_t kScalarBusyEvent = 0x1U; +constexpr uint32_t kIcacheRequestEvent = 0x34U; +constexpr uint32_t kIcacheMissEvent = 0x35U; +constexpr uint32_t kVectorBusyEvent = 0x501U; +constexpr uint32_t kCubeBusyEvent = 0x301U; +constexpr uint32_t kMte1BusyEvent = 0x701U; +constexpr uint32_t kMte2BusyEvent = 0x202U; +constexpr uint32_t kMte3BusyEvent = 0x203U; +constexpr uint32_t kFixBusyEvent = 0x714U; + +// DAV_3510 PMU MMIO offset。ld_dev 的立即数只有 12 bit，因此 kernel 会分别重基址到 0x2400/0x4200。 +constexpr uint32_t kSelectorBlockOffset = 0x2400U; +constexpr uint32_t kCounterBlockOffset = 0x4200U; +constexpr uint32_t kCnt2Offset = 0x4220U; +constexpr uint32_t kCnt0Offset = 0x4210U; +constexpr uint32_t kCnt1Offset = 0x4218U; +constexpr uint32_t kCnt3Offset = 0x4228U; +constexpr uint32_t kCnt4Offset = 0x4230U; +constexpr uint32_t kCnt5Offset = 0x4238U; +constexpr uint32_t kCnt6Offset = 0x4240U; +constexpr uint32_t kCnt7Offset = 0x4248U; +constexpr uint32_t kCnt8Offset = 0x4250U; +constexpr uint32_t kCnt9Offset = 0x4254U; +constexpr uint32_t kTotalLowOffset = 0x4260U; +constexpr uint32_t kTotalHighOffset = 0x4264U; +constexpr uint32_t kCnt2SelectorOffset = 0x2508U; +constexpr uint32_t kCnt0SelectorOffset = 0x2500U; +constexpr uint32_t kCnt1SelectorOffset = 0x2504U; +constexpr uint32_t kCnt3SelectorOffset = 0x250cU; +constexpr uint32_t kCnt4SelectorOffset = 0x2510U; +constexpr uint32_t kCnt5SelectorOffset = 0x2514U; +constexpr uint32_t kCnt6SelectorOffset = 0x2518U; +constexpr uint32_t kCnt7SelectorOffset = 0x251cU; +constexpr uint32_t kCnt8SelectorOffset = 0x2520U; +constexpr uint32_t kCnt9SelectorOffset = 0x2524U; + +// pmu_status 的 bits16..27 保存 get_coreid()；bits28..31 留给不参与 core id +// 解码的模式诊断。其余低位描述本条记录是否可信。 +constexpr uint32_t kStatusRequested = 1U << 0; +constexpr uint32_t kStatusRegMapped = 1U << 1; +constexpr uint32_t kStatusCoreIdValid = 1U << 2; +constexpr uint32_t kStatusCnt2Selector = 1U << 3; +constexpr uint32_t kStatusCnt6Selector = 1U << 4; +constexpr uint32_t kStatusCnt7Selector = 1U << 5; +constexpr uint32_t kStatusWindowStarted = 1U << 6; +constexpr uint32_t kStatusTotalNonzero = 1U << 7; +constexpr uint32_t kStatusPriorSnapshotLarger = 1U << 8; +constexpr uint32_t kStatusCnt0Selector = 1U << 9; +constexpr uint32_t kStatusCnt1Selector = 1U << 10; +constexpr uint32_t kStatusCnt3Selector = 1U << 11; +constexpr uint32_t kStatusCnt4Selector = 1U << 12; +constexpr uint32_t kStatusCnt5Selector = 1U << 13; +constexpr uint32_t kStatusCnt8Selector = 1U << 14; +constexpr uint32_t kStatusWindowStopped = 1U << 15; +// I-cache 配对标志不能复用 Submit start bit；StatusCoreId 只取 12 bit，故将 +// 它放在 core-id 区间之上的独立诊断位。 +constexpr uint32_t kStatusIcachePairObserved = 1U << 28; +constexpr uint32_t kStatusRequired = kStatusRequested | kStatusRegMapped | kStatusCoreIdValid | + kStatusCnt2Selector | kStatusCnt6Selector | kStatusCnt7Selector | + kStatusCnt0Selector | kStatusCnt1Selector | kStatusCnt3Selector | + kStatusCnt4Selector | kStatusCnt5Selector | kStatusCnt8Selector | + kStatusWindowStarted | kStatusWindowStopped | kStatusTotalNonzero; +constexpr uint32_t kStatusCoreIdShift = 16; +constexpr uint32_t kStatusCoreIdMask = 0x0fffU; + +// pmu_phase_status 独立于旧 pmu_status，避免与其中的物理 core-id 位域 +// 冲突。bits4/5 只记录 shadow 是否恰好等于 primary：phase=none 没有 +// 运行中 read-to-clear，host 会要求两位都成立；局部 phase 会在计数仍开启时 +// 读取 shadow，A5 实测存在同周期递增与读清竞争，因此不能把“逐次严格相等” +// 作为可信记录的共同必选位。局部 phase 的方向和误差包络由 host/raw 独立校验。 +constexpr uint32_t kPhaseStatusRequested = 1U << 0; +constexpr uint32_t kPhaseStatusShadowSelectors = 1U << 1; +constexpr uint32_t kPhaseStatusWindowStarted = 1U << 2; +constexpr uint32_t kPhaseStatusWindowStopped = 1U << 3; +constexpr uint32_t kPhaseStatusShadowRequestsMatch = 1U << 4; +constexpr uint32_t kPhaseStatusShadowMissesMatch = 1U << 5; +constexpr uint32_t kPhaseStatusBoundariesBalanced = 1U << 6; +constexpr uint32_t kPhaseStatusValuesOrdered = 1U << 7; +constexpr uint32_t kPhaseStatusUint32Fit = 1U << 8; +constexpr uint32_t kPhaseStatusPhaseShape = 1U << 9; +// none 必须保持 0 tick；运行阶段则必须确实累计到非零 SYS_CNT。阶段时间是否 +// 不超过同核首 Submit 到末 Submit 的完整区间，由拿到两端结果的 host 再校验。 +constexpr uint32_t kPhaseStatusTimeValid = 1U << 10; +constexpr uint32_t kPhaseStatusRequired = + kPhaseStatusRequested | kPhaseStatusShadowSelectors | + kPhaseStatusWindowStarted | kPhaseStatusWindowStopped | + kPhaseStatusBoundariesBalanced | kPhaseStatusValuesOrdered | + kPhaseStatusUint32Fit | kPhaseStatusPhaseShape | kPhaseStatusTimeValid; + +inline const char *SubmitPmuPhaseName(SubmitPmuPhase phase) { + switch (phase) { + case SubmitPmuPhase::None: + return "none"; + case SubmitPmuPhase::Claim: + return "claim"; + case SubmitPmuPhase::EfDrain: + return "efdrain"; + case SubmitPmuPhase::Materialize: + return "materialize"; + case SubmitPmuPhase::Register: + return "register"; + case SubmitPmuPhase::Count: + break; + } + return "invalid"; +} + +inline uint64_t PackPointer(const uint32_t *words) { + return static_cast(words[kConfigRegTableLow]) | + (static_cast(words[kConfigRegTableHigh]) << 32); +} + +inline void StorePointer(uint32_t *words, const void *pointer) { + const uint64_t raw = reinterpret_cast(pointer); + words[kConfigRegTableLow] = static_cast(raw); + words[kConfigRegTableHigh] = static_cast(raw >> 32); +} + +inline uint32_t StatusCoreId(uint32_t status) { + return (status >> kStatusCoreIdShift) & kStatusCoreIdMask; +} + +} // namespace pa_scheduler::ccec_pmu + +#endif // PA_SCHEDULER_CCEC_PMU_PROBE_H diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/host_support.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/host_support.h new file mode 100644 index 0000000000..1185536866 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/host_support.h @@ -0,0 +1,1609 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_HOST_SUPPORT_H +#define PA_SCHEDULER_COMMON_HOST_SUPPORT_H + +#include "pa_model.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pa_scheduler::host { + +// 三种后端共用同一套命令行配置，保证 CPU 语义回归与 A5 上板使用完全相同的工作量。 +struct Options { + std::string kernel_path; + std::string swimlane_json; + uint32_t device = 0; + uint32_t batches = kDefaultBatches; + uint32_t runs = 5; + NopCounts nops{kDefaultQkNops, kDefaultSfNops, kDefaultPvNops, kDefaultUpNops}; + bool profile_phases = false; + bool trace_enabled = true; + bool trace_atomics = false; + bool analyze_swimlane = false; +}; + +enum class ParseStatus { + Ok, + Help, + Error, +}; + +inline bool ParseUint(const char *raw, uint32_t minimum, uint32_t maximum, uint32_t *value) { + // 要求整串都能被 strtoul 解析且结果落在给定范围内，拒绝尾随字符和溢出值， + // 避免参数被部分解析后悄悄改变工作量。 + errno = 0; + char *end = nullptr; + const unsigned long parsed = std::strtoul(raw, &end, 10); + if (errno != 0 || end == raw || *end != '\0' || parsed < minimum || parsed > maximum) { + return false; + } + *value = static_cast(parsed); + return true; +} + +inline bool ParseNopCounts(const char *raw, NopCounts *counts) { + // 四类 kernel 的 NOP 数必须一次性完整给出，顺序固定为 QK、SF、PV、UP。 + unsigned int qk = 0; + unsigned int sf = 0; + unsigned int pv = 0; + unsigned int up = 0; + char tail = '\0'; + if (std::sscanf(raw, "%u,%u,%u,%u%c", &qk, &sf, &pv, &up, &tail) != 4) { + return false; + } + constexpr uint32_t kMaxNopCount = 10000000; + if (qk > kMaxNopCount || sf > kMaxNopCount || pv > kMaxNopCount || up > kMaxNopCount) { + return false; + } + *counts = NopCounts{qk, sf, pv, up}; + return true; +} + +inline void PrintUsage(const char *program, bool require_kernel) { + // require_kernel 只影响 CCEC host 的用法文本，其余 benchmark 参数在三后端完全一致。 + std::fprintf( + stderr, "Usage: %s%s [--device N] [--batches 1..256] [--runs N] ", program, + require_kernel ? " --kernel FILE" : "" + ); + std::fprintf( + stderr, + "[--nop-count N | --nop-counts QK,SF,PV,UP] [--profile-phases] [--analyze-swimlane] " + "[--trace-atomics] [--swimlane-json FILE] [--no-swimlane]\n" + ); +} + +inline ParseStatus ParseOptions(int argc, char **argv, bool require_kernel, Options *options) { + // CCEC host 需要外部 kernel ELF；AscendC 和 CPU 的可执行文件已包含 kernel，因此不需要该参数。 + bool nop_override_seen = false; + bool swimlane_json_seen = false; + for (int index = 1; index < argc; ++index) { + // 无值开关先处理；其余参数统一在消费下一个 argv 前检查缺值，保证错误位置明确。 + const std::string argument = argv[index]; + if (argument == "--help" || argument == "-h") { + PrintUsage(argv[0], require_kernel); + return ParseStatus::Help; + } + if (argument == "--profile-phases") { + options->profile_phases = true; + continue; + } + if (argument == "--no-swimlane") { + options->trace_enabled = false; + continue; + } + if (argument == "--trace-atomics") { + options->trace_atomics = true; + continue; + } + if (argument == "--analyze-swimlane") { + options->analyze_swimlane = true; + continue; + } + if (index + 1 >= argc) { + std::fprintf(stderr, "Missing value after %s\n", argument.c_str()); + return ParseStatus::Error; + } + const char *value = argv[++index]; + if (argument == "--kernel" && require_kernel) { + options->kernel_path = value; + } else if (argument == "--device") { + if (!ParseUint(value, 0, INT32_MAX, &options->device)) return ParseStatus::Error; + } else if (argument == "--batches") { + if (!ParseUint(value, 1, kMaxBatches, &options->batches)) return ParseStatus::Error; + } else if (argument == "--runs") { + if (!ParseUint(value, 1, 1000, &options->runs)) return ParseStatus::Error; + } else if (argument == "--swimlane-json") { + if (swimlane_json_seen) { + std::fprintf(stderr, "Specify --swimlane-json only once.\n"); + return ParseStatus::Error; + } + if (*value == '\0') { + std::fprintf(stderr, "--swimlane-json requires a non-empty path.\n"); + return ParseStatus::Error; + } + options->swimlane_json = value; + swimlane_json_seen = true; + } else if (argument == "--nop-count") { + if (nop_override_seen) { + std::fprintf(stderr, "Specify only one NOP override.\n"); + return ParseStatus::Error; + } + uint32_t count = 0; + if (!ParseUint(value, 0, 10000000, &count)) return ParseStatus::Error; + options->nops = NopCounts{count, count, count, count}; + nop_override_seen = true; + } else if (argument == "--nop-counts") { + if (nop_override_seen) { + std::fprintf(stderr, "Specify only one NOP override.\n"); + return ParseStatus::Error; + } + if (!ParseNopCounts(value, &options->nops)) return ParseStatus::Error; + nop_override_seen = true; + } else { + std::fprintf(stderr, "Unknown argument: %s\n", argument.c_str()); + return ParseStatus::Error; + } + } + if (require_kernel && options->kernel_path.empty()) { + std::fprintf(stderr, "--kernel is required\n"); + return ParseStatus::Error; + } + if (options->analyze_swimlane && !options->trace_enabled) { + // 分析和导出都依赖完整 record 缓冲，不能与节省内存的 --no-swimlane 同时使用。 + std::fprintf(stderr, "--analyze-swimlane requires swimlane tracing.\n"); + return ParseStatus::Error; + } + if (options->trace_atomics && !options->trace_enabled) { + std::fprintf(stderr, "--trace-atomics cannot be combined with --no-swimlane.\n"); + return ParseStatus::Error; + } + if (!options->swimlane_json.empty() && !options->trace_enabled) { + std::fprintf(stderr, "--swimlane-json requires swimlane tracing.\n"); + return ParseStatus::Error; + } + if (!options->swimlane_json.empty() && options->runs != 1) { + // 一个文件只对应一次完整采集，禁止多轮运行反复覆盖而丢失轮次边界。 + std::fprintf(stderr, "--swimlane-json requires --runs 1 to avoid overwriting captures.\n"); + return ParseStatus::Error; + } + return ParseStatus::Ok; +} + +inline void InitializeState(SchedulerState *state, const Options &options) { + // WorkerState 有意保持真实 PA 每核约 9 MiB 的布局。若 host 每轮清空全部 worker， + // 会额外触碰并拷贝近 1 GiB 内存；因此只初始化全局前缀和结果区，worker 的活跃字段 + // 由各自 kernel 在启动后复位，这也与真实 PA 的生命周期一致。 + std::memset(state, 0, offsetof(SchedulerState, workers)); + std::memset(&state->config, 0, offsetof(SchedulerState, results) - offsetof(SchedulerState, config)); + std::memset(state->results, 0, sizeof(state->results)); + state->heap_window = kHeapWindow; + state->heap_base = kSyntheticHeapBase; + state->heap_size = kHeapBytes; + state->num_workers = kWorkers; + state->num_blocks = kAicWorkers; + state->config.batches = options.batches; + state->config.workers = kWorkers; + state->config.nops = options.nops; + state->config.profile_phases = options.profile_phases ? 1U : 0U; + for (uint32_t batch = 0; batch < options.batches; ++batch) { + state->context_lens[batch] = 8192; + } + for (uint32_t shard = 0; shard < kCursorShards; ++shard) { + // -1 表示尚无 task 被 claim；task 0 的 atomicMax 因而也能正常判定唯一 winner。 + state->cube_cursor[shard].value = -1; + state->vector_cursor[shard].value = -1; + state->alloc_cursor[shard].value = -1; + } + state->frontier.value = -1; +} + +inline void ConfigureTrace(SchedulerState *state, const Options &options, const void *trace_base) { + // device 只持有裸地址和每核容量；TraceHeader/record 缓冲区由 host 单独分配并初始化。 + state->config.trace_enabled = options.trace_enabled + ? kTracePhasesEnabled | (options.trace_atomics ? kTraceAtomicsEnabled : 0U) + : 0U; + state->config.trace_base = options.trace_enabled ? reinterpret_cast(trace_base) : 0; + state->config.trace_records_per_core = options.trace_enabled ? kTraceRecordsPerCore : 0; +} + +inline void InitializeTraceHeader(TraceHeader *header) { + // version=4 表示 phase ABI 已追加父区间和真实 Submit 尾动作；core state + // 继续携带 weighted atomic/PollBatch 计数和权威拓扑。 + std::memset(header, 0, sizeof(*header)); + header->magic = 0x4653574cU; + header->version = 4; + header->num_cores = kWorkers; + header->records_per_core = kTraceRecordsPerCore; + header->frequency_hz = kSystemCounterHz; +} + +// 巨大的 WorkerState 不参与每轮 H2D/D2H；以下三个范围只搬运运行所需的前缀、控制量和结果。 +inline constexpr size_t StatePrefixBytes() { return offsetof(SchedulerState, workers); } + +inline constexpr size_t ControlBytes() { + // control sidecar 位于为生产 DistGlobal 保留的总跨度之后，到 results 之前为止。 + return offsetof(SchedulerState, results) - offsetof(SchedulerState, config); +} + +inline constexpr size_t ResultBytes() { return sizeof(WorkerResult) * kWorkers; } + +struct Metrics { + // passed 是全部语义断言的合取；submit_span_us 是本用例唯一用于对比 PA 的性能口径。 + bool passed = true; + double submit_span_us = 0; +}; + +inline void Expect(bool condition, const char *label, Metrics *metrics) { + // 所有断言都继续执行，以便一次失败运行尽可能暴露完整状态，而不是遇到首错立即退出。 + std::printf("[ASSERT] %-48s %s\n", label, condition ? "PASS" : "FAIL"); + if (!condition) metrics->passed = false; +} + +struct Uint64Distribution { + uint64_t total = 0; + double median = 0.0; + uint64_t p95 = 0; + uint64_t maximum = 0; +}; + +inline Uint64Distribution SummarizeUint64(std::vector values) { + // 这里按 worker 维度统计累计周期，p95 使用 nearest-rank，避免插值掩盖慢核。 + Uint64Distribution summary; + if (values.empty()) return summary; + + std::sort(values.begin(), values.end()); + for (uint64_t value : values) summary.total += value; + const size_t middle = values.size() / 2; + summary.median = (values.size() & 1U) != 0 + ? static_cast(values[middle]) + : (static_cast(values[middle - 1]) + static_cast(values[middle])) / 2.0; + const size_t p95_rank = (95U * values.size() + 99U) / 100U; + summary.p95 = values[p95_rank - 1]; + summary.maximum = values.back(); + return summary; +} + +inline void PrintPhaseDiagnostics(const SchedulerState &state) { + if (state.config.profile_phases == 0) return; + + // WaitForSlot/HeapGuard 没有各自独立命名的 TracePhase；实际发生等待时会写 + // RingBp 记录，汇总诊断则使用 WorkerResult 中的累计周期和等待次数。 + struct PhaseSpec { + ProfilePhase phase; + const char *name; + int32_t wait_event_index; + }; + const PhaseSpec phases[] = { + {ProfilePhase::Claim, "Claim", -1}, + {ProfilePhase::EfDrain, "EfDrain", -1}, + {ProfilePhase::WaitForSlot, "WaitForSlot", 0}, + {ProfilePhase::HeapGuard, "HeapGuard", 1}, + }; + const CoreRole roles[] = {CoreRole::Aic, CoreRole::Aiv}; + const char *role_names[] = {"AIC", "AIV"}; + + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + // AIC/AIV 分开统计，避免 32:64 的参与者数量差异掩盖某一类核上的长尾。 + for (const PhaseSpec &phase : phases) { + std::vector cycles; + std::vector calls; + std::vector wait_events; + const uint32_t phase_index = static_cast(phase.phase); + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const WorkerResult &result = state.results[worker]; + if (result.role != static_cast(roles[role_index])) continue; + cycles.push_back(result.phase_cycles[phase_index]); + calls.push_back(result.phase_calls[phase_index]); + wait_events.push_back( + phase.wait_event_index < 0 ? 0 : result.wait_events[static_cast(phase.wait_event_index)] + ); + } + const Uint64Distribution cycle_summary = SummarizeUint64(cycles); + const Uint64Distribution call_summary = SummarizeUint64(calls); + const Uint64Distribution wait_summary = SummarizeUint64(wait_events); + std::printf( + "[PHASE] role=%s phase=%s workers=%zu accumulated_us_median=%.3f " + "accumulated_us_p95=%.3f accumulated_us_max=%.3f calls_total=%llu " + "calls_per_worker_median=%.1f calls_per_worker_p95=%llu calls_per_worker_max=%llu " + "wait_events_total=%llu wait_events_per_worker_median=%.1f " + "wait_events_per_worker_p95=%llu wait_events_per_worker_max=%llu\n", + role_names[role_index], phase.name, cycles.size(), cycle_summary.median / 1000.0, + static_cast(cycle_summary.p95) / 1000.0, + static_cast(cycle_summary.maximum) / 1000.0, + static_cast(call_summary.total), call_summary.median, + static_cast(call_summary.p95), + static_cast(call_summary.maximum), + static_cast(wait_summary.total), wait_summary.median, + static_cast(wait_summary.p95), + static_cast(wait_summary.maximum) + ); + } + } +} + +inline const char *TracePhaseName(uint32_t phase) { + // 名称必须与 l2_swimlane_records.json 的 fdwic_events schema 保持一致。 + const char *names[] = { + "Kernel", "Alloc", "Build", "DrainWon", "Replay", "RingBp", "EfDrain", "Commit", + "Submit", "Materialize", "PrepareMap", "Claim", "Fanin", "Register", "Atomic", + "ClockBaseline", "OrchestrationReplay", "FinalDrain", "WinnerBuild", + "AllocComplete", + }; + static_assert( + sizeof(names) / sizeof(names[0]) == static_cast(TracePhase::Count), + "TracePhaseName must cover every trace phase" + ); + return phase < sizeof(names) / sizeof(names[0]) ? names[phase] : "Unknown"; +} + +inline const char *AtomicSiteName(uint32_t site) { + // 顺序与 pa_model.h::AtomicSite 的稳定 raw ABI 完全一致。 + const char *names[] = { + "StartupIncrement", "StartupPoll", "FatalPoll", "FatalSet", "ClaimMax", + "FaninFlagLoad", "CompletionVendExchange", "CompletionFlagExchange", + "FrontierInitialLoad", "FrontierFlagLoad", "FrontierMax", "HeapFrontierLoad", + "HeapVendLoad", "ReplayDoneIncrement", "ReplayDonePoll", + }; + return site < sizeof(names) / sizeof(names[0]) ? names[site] : "Unknown"; +} + +inline const char *AtomicOpName(uint32_t op) { + const char *names[] = {"Load", "Exchange", "FetchAdd", "FetchMax"}; + return op < sizeof(names) / sizeof(names[0]) ? names[op] : "Unknown"; +} + +inline AtomicOp AtomicSiteOp(AtomicSite site) { + return AtomicSiteExpectedOp(site); +} + +inline bool ValidateTraceHeader(const TraceHeader &header, const char *operation) { + // 在任何 D2H record 搬运前先验证容量和 dropped，防止损坏 header 导致 scratch 越界或导出残缺泳道。 + // 频率也要求精确为 1 GHz，否则后续 ns/us 换算即使 JSON 合法也没有性能意义。 + const bool valid = header.magic == 0x4653574cU && header.version == 4 && + header.num_cores == kWorkers && header.records_per_core == kTraceRecordsPerCore && + header.frequency_hz == kSystemCounterHz; + bool core_states_valid = true; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const TraceCoreState &core = header.cores[worker]; + core_states_valid &= core.count <= kTraceRecordsPerCore; + core_states_valid &= core.dropped == 0; + core_states_valid &= core.poll_calls <= core.atomic_calls; + core_states_valid &= (core.poll_calls == 0) == (core.poll_batch_records == 0); + const uint64_t physical_atomic = + static_cast(core.atomic_calls) - core.poll_calls + core.poll_batch_records; + core_states_valid &= physical_atomic <= core.count; + } + if (!valid || !core_states_valid) { + std::fprintf( + stderr, + "%s rejected an invalid trace header: magic=0x%08x version=%u cores=%u " + "records_per_core=%u frequency_hz=%llu core_states_valid=%s\n", + operation, header.magic, header.version, header.num_cores, header.records_per_core, + static_cast(header.frequency_hz), core_states_valid ? "yes" : "no" + ); + } + return valid && core_states_valid; +} + +struct TraceExportSummary { + uint64_t records = 0; + uint64_t atomic_records = 0; + uint64_t clock_baseline_records = 0; + uint64_t atomic_calls = 0; + uint64_t poll_calls = 0; + uint64_t poll_batch_records = 0; + uint64_t dropped_records = 0; +}; + +inline bool SameTraceSummary(const TraceExportSummary &left, const TraceExportSummary &right) { + return left.records == right.records && left.atomic_records == right.atomic_records && + left.clock_baseline_records == right.clock_baseline_records && + left.atomic_calls == right.atomic_calls && left.poll_calls == right.poll_calls && + left.poll_batch_records == right.poll_batch_records && + left.dropped_records == right.dropped_records; +} + +inline uint32_t AtomicRecordCallCount(const TraceRecord &record) { + return (record.flags & kAtomicPollBatch) != 0 + ? record.flags >> kAtomicPollCountShift + : 1U; +} + +inline bool AtomicRecordSchemaValid(const TraceRecord &record, bool atomic_trace_enabled) { + if (!atomic_trace_enabled || record.auxiliary >= static_cast(AtomicSite::Count)) { + return false; + } + const AtomicSite site = static_cast(record.auxiliary); + const uint32_t op = record.flags & kAtomicOpMask; + if (op != static_cast(AtomicSiteExpectedOp(site))) return false; + + const bool result_used = (record.flags & kAtomicResultUsed) != 0; + const bool value_zero = (record.flags & kAtomicValueZero) != 0; + const bool return_ready = (record.flags & kAtomicReturnReady) != 0; + const bool poll_batch = (record.flags & kAtomicPollBatch) != 0; + const uint32_t payload = record.flags >> kAtomicRetriesShift; + if (poll_batch) { + return AtomicSiteIsPollBatchable(site) && result_used && !value_zero && !return_ready && + payload > 0 && record.task_id == -1 && record.function_id == -1; + } + if (result_used != AtomicSiteResultUsed(site) || (return_ready && !result_used)) return false; + if (value_zero && op != static_cast(AtomicOp::Load)) return false; + if (payload != 0 && op != static_cast(AtomicOp::FetchMax)) return false; + return record.function_id == -1; +} + +inline bool ClockRecordSchemaValid(const TraceRecord &record) { + const bool dependency = (record.flags & kClockAtomicDependency) != 0; + const bool dependency_applied = (record.flags & kClockAtomicDependencyApplied) != 0; + return (record.flags & ~(kClockAtomicDependency | kClockAtomicDependencyApplied)) == 0 && + (!dependency_applied || dependency) && record.task_id == -1 && + record.function_id == -1 && record.auxiliary == 0; +} + +inline void ExpectedTraceTopology(uint32_t worker, int32_t *block_id, int32_t *lane) { + if (worker < kAicWorkers) { + *block_id = static_cast(worker); + *lane = 0; + return; + } + const uint32_t vector_id = worker - kAicWorkers; + *block_id = static_cast(vector_id / 2); + *lane = static_cast(1 + vector_id % 2); +} + +template +inline bool ExportSwimlaneRecords( + const TraceHeader &header, const std::string &output_path, + WinnerWorkloadMode workload_mode, const WorkloadCounts &workload_counts, + const char *workload_pattern, bool atomic_trace_enabled, ReadRecords read_records +) { + if (!ValidateTraceHeader(header, "swimlane export")) return false; + if (workload_mode != WinnerWorkloadMode::ScalarNop && + workload_mode != WinnerWorkloadMode::RealCompute) { + std::fprintf(stderr, "swimlane export rejected invalid winner workload mode.\n"); + return false; + } + const bool real_compute = workload_mode == WinnerWorkloadMode::RealCompute; + const bool pattern_valid = workload_pattern != nullptr && + ((real_compute && + (std::strcmp(workload_pattern, "constant") == 0 || + std::strcmp(workload_pattern, "layout-diagnostic") == 0)) || + (!real_compute && std::strcmp(workload_pattern, "none") == 0)); + if (!pattern_valid) { + std::fprintf(stderr, "swimlane export rejected invalid winner workload input pattern.\n"); + return false; + } + + TraceExportSummary producer_summary; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const TraceCoreState &core = header.cores[worker]; + int32_t expected_block = -1; + int32_t expected_lane = -1; + ExpectedTraceTopology(worker, &expected_block, &expected_lane); + if (core.core_idx != static_cast(worker) || core.block_id != expected_block || + core.lane != expected_lane) { + std::fprintf( + stderr, + "swimlane export rejected worker topology: worker=%u core=%d block=%d/%d lane=%d/%d\n", + worker, core.core_idx, core.block_id, expected_block, core.lane, expected_lane + ); + return false; + } + producer_summary.records += core.count; + producer_summary.atomic_calls += core.atomic_calls; + producer_summary.poll_calls += core.poll_calls; + producer_summary.poll_batch_records += core.poll_batch_records; + producer_summary.dropped_records += core.dropped; + if (!atomic_trace_enabled) { + if (core.atomic_calls != 0 || core.poll_calls != 0 || core.poll_batch_records != 0) { + std::fprintf( + stderr, + "phase-only swimlane worker %u unexpectedly reports atomic counters: calls=%u polls=%u batches=%u\n", + worker, core.atomic_calls, core.poll_calls, core.poll_batch_records + ); + return false; + } + continue; + } + if (core.poll_calls > core.atomic_calls || + (core.poll_calls == 0) != (core.poll_batch_records == 0)) { + std::fprintf( + stderr, + "atomic swimlane worker %u has invalid counters: calls=%u polls=%u batches=%u\n", + worker, core.atomic_calls, core.poll_calls, core.poll_batch_records + ); + return false; + } + producer_summary.atomic_records += + static_cast(core.atomic_calls) - core.poll_calls + core.poll_batch_records; + } + producer_summary.clock_baseline_records = atomic_trace_enabled ? 2ULL * kWorkers : 0; + + // 先写同目录临时文件，全部记录写完并关闭后再 rename 替换，避免把半截 JSON + // 当成有效采集；这里没有 fsync 文件和目录，不承诺掉电后的持久化原子性。 + const std::string temporary_path = output_path + ".tmp"; + std::FILE *output = std::fopen(temporary_path.c_str(), "wb"); + if (output == nullptr) { + std::fprintf( + stderr, "Cannot open swimlane output %s: %s\n", temporary_path.c_str(), std::strerror(errno) + ); + return false; + } + + // 采用固定 1 MiB stdio 缓冲并逐核流式写出；默认 256 batch 时约 86 万条， + // 无论实际 batch 数是多少都不在 host 侧一次性聚合全部 JSON 记录。 + std::vector output_buffer(1U << 20); + std::setvbuf(output, output_buffer.data(), _IOFBF, output_buffer.size()); + std::fprintf( + output, + "{\n\"l2_swimlane_level\":%u,\n" + "\"metadata\":{\"clock_freq_hz\":%llu,\"num_cores\":%u," + "\"trace_schema_version\":%u," + "\"winner_workload\":{\"mode\":\"%s\"," + "\"counts\":{\"qk\":%u,\"sf\":%u,\"pv\":%u,\"up\":%u}," + "\"unit\":\"%s\",\"input_pattern\":\"%s\"," + "\"engine_mapping\":%s},\"core_types\":[", + atomic_trace_enabled ? 4U : 1U, + static_cast(header.frequency_hz), kWorkers, + 4U, + workload_mode == WinnerWorkloadMode::RealCompute ? "real-compute" : "scalar-nop", + workload_counts.qk, workload_counts.sf, workload_counts.pv, workload_counts.up, + workload_mode == WinnerWorkloadMode::RealCompute + ? "complete_128x128_engine_pipeline_iteration" + : "scalar_nop_instruction", + workload_pattern, + workload_mode == WinnerWorkloadMode::RealCompute + ? "{\"qk\":\"cube_matmul\",\"sf\":\"vector_add\"," + "\"pv\":\"cube_matmul\",\"up\":\"vector_mul\"}" + : "null" + ); + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + std::fprintf(output, "%s\"%s\"", worker == 0 ? "" : ",", worker < kAicWorkers ? "aic" : "aiv"); + } + // schema-v4 无论是否开启 atomic 都导出 producer summary；phase-only 的 + // atomic/clock 字段为零，离线分析仍可独立证明 records 与 dropped 闭合。 +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + std::fprintf( + output, + "],\"lazy_sample_shape\":{\"name\":\"%s\",\"id\":%u," + "\"observation\":\"%s\",\"finish\":\"%s\",\"control_family\":\"%s\"}," + "\"fdwic_summary\":{\"records\":%llu,\"atomic_records\":%llu," + "\"clock_baseline_records\":%llu,\"atomic_calls\":%llu," + "\"batched_poll_calls\":%llu,\"poll_batch_records\":%llu," + "\"dropped_records\":%llu}", + kLazySampleShapeName, kLazySampleShapeId, kLazySampleObservation, + kLazySampleFinishShape, kLazySampleControlFamily, + static_cast(producer_summary.records), + static_cast(producer_summary.atomic_records), + static_cast(producer_summary.clock_baseline_records), + static_cast(producer_summary.atomic_calls), + static_cast(producer_summary.poll_calls), + static_cast(producer_summary.poll_batch_records), + static_cast(producer_summary.dropped_records) + ); +#else + std::fprintf( + output, + "],\"fdwic_summary\":{\"records\":%llu,\"atomic_records\":%llu," + "\"clock_baseline_records\":%llu,\"atomic_calls\":%llu," + "\"batched_poll_calls\":%llu,\"poll_batch_records\":%llu," + "\"dropped_records\":%llu}", + static_cast(producer_summary.records), + static_cast(producer_summary.atomic_records), + static_cast(producer_summary.clock_baseline_records), + static_cast(producer_summary.atomic_calls), + static_cast(producer_summary.poll_calls), + static_cast(producer_summary.poll_batch_records), + static_cast(producer_summary.dropped_records) + ); +#endif + std::fprintf( + output, + "},\n\"aicore_tasks\":[],\n\"aicpu_tasks\":[],\n" + "\"aicpu_scheduler_phases\":[],\n\"aicpu_orchestrator_phases\":[],\n\"fdwic_events\":[\n" + ); + // fdwic_events 每行固定十列：core、block、lane、task、function、phase、起止周期、flags、aux。 + + bool success = true; + bool first_record = true; + uint64_t exported_records = 0; + TraceExportSummary observed_summary; + std::vector scratch(kTraceRecordsPerCore); + constexpr int32_t kTracePhaseCount = static_cast(TracePhase::Count); + for (uint32_t worker = 0; worker < kWorkers && success; ++worker) { + // 每次只读取一个 worker 的有效区间；完整 384 MiB trace 缓冲无需整体回拷。 + const uint32_t available = header.cores[worker].count; + if (available > header.records_per_core) { + std::fprintf( + stderr, "Trace core %u count %u exceeds capacity %u.\n", worker, available, + header.records_per_core + ); + success = false; + break; + } + if (available != 0 && !read_records(worker, available, scratch.data())) { + success = false; + break; + } + const TraceCoreState &core = header.cores[worker]; + uint64_t core_atomic_calls = 0; + uint64_t core_poll_calls = 0; + uint32_t core_atomic_records = 0; + uint32_t core_poll_batch_records = 0; + uint32_t core_clock_records = 0; + uint32_t core_plain_clock_records = 0; + uint32_t core_dependency_clock_records = 0; + bool dependency_applied = false; + bool direct_result_used_return_ready = false; + bool direct_result_used_source_issue = false; + for (uint32_t index = 0; index < available; ++index) { + const TraceRecord &record = scratch[index]; + const bool atomic_record = record.phase == static_cast(TracePhase::Atomic); + const bool claim_record = record.phase == static_cast(TracePhase::Claim); + const bool clock_record = record.phase == static_cast(TracePhase::ClockBaseline); + const bool atomic_schema_valid = !atomic_record || + AtomicRecordSchemaValid(record, atomic_trace_enabled); + const bool claim_schema_valid = !claim_record || + ((record.flags & ~(kClaimWon | kClaimAttempted)) == 0 && + ((record.flags & kClaimWon) == 0 || (record.flags & kClaimAttempted) != 0) && + record.auxiliary <= 1); + const bool clock_schema_valid = !clock_record || + (atomic_trace_enabled && ClockRecordSchemaValid(record)); + const bool record_valid = record.end_cycle >= record.start_cycle && record.phase >= 0 && + record.phase < kTracePhaseCount && record.task_id >= -1 && + record.function_id >= -1 && record.lane == core.lane && + record.block_id == core.block_id && + record.core_idx == core.core_idx && atomic_schema_valid && + claim_schema_valid && clock_schema_valid; + if (!record_valid) { + std::fprintf( + stderr, + "Invalid trace record at worker=%u index=%u: phase=%d lane=%d block=%d core=%d " + "start=%llu end=%llu flags=0x%08x aux=%u\n", + worker, index, record.phase, record.lane, record.block_id, record.core_idx, + static_cast(record.start_cycle), + static_cast(record.end_cycle), record.flags, record.auxiliary + ); + success = false; + break; + } + if (atomic_record) { + ++core_atomic_records; + const uint32_t call_count = AtomicRecordCallCount(record); + core_atomic_calls += call_count; + if ((record.flags & kAtomicPollBatch) != 0) { + core_poll_calls += call_count; + ++core_poll_batch_records; + } else if ((record.flags & kAtomicResultUsed) != 0) { + if ((record.flags & kAtomicReturnReady) != 0) { + direct_result_used_return_ready = true; + } else { + direct_result_used_source_issue = true; + } + } + } else if (clock_record) { + ++core_clock_records; + if ((record.flags & kClockAtomicDependency) != 0) { + ++core_dependency_clock_records; + dependency_applied = (record.flags & kClockAtomicDependencyApplied) != 0; + } else { + ++core_plain_clock_records; + } + } + std::fprintf( + output, + "%s[%d,%d,%d,%d,%d,\"%s\",%llu,%llu,%u,%u]", + first_record ? "" : ",\n", record.core_idx, record.block_id, record.lane, record.task_id, + record.function_id, TracePhaseName(static_cast(record.phase)), + static_cast(record.start_cycle), + static_cast(record.end_cycle), record.flags, record.auxiliary + ); + first_record = false; + ++exported_records; + } + if (!success) break; + bool core_closed = true; + if (atomic_trace_enabled) { + const uint64_t expected_atomic_records = + static_cast(core.atomic_calls) - core.poll_calls + core.poll_batch_records; + core_closed = core_atomic_records == expected_atomic_records && + core_atomic_calls == core.atomic_calls && core_poll_calls == core.poll_calls && + core_poll_batch_records == core.poll_batch_records && core_clock_records == 2 && + core_plain_clock_records == 1 && core_dependency_clock_records == 1 && + (!dependency_applied || !direct_result_used_source_issue) && + (dependency_applied || !direct_result_used_return_ready); + } else { + core_closed = core_atomic_records == 0 && core_atomic_calls == 0 && core_poll_calls == 0 && + core_poll_batch_records == 0 && core_clock_records == 0; + } + if (!core_closed) { + std::fprintf( + stderr, + "swimlane closure failed on worker=%u: physical_atomic=%u logical_atomic=%llu/%u " + "poll_calls=%llu/%u poll_batches=%u/%u clock=%u plain=%u dependency=%u " + "dependency_applied=%s direct_ready=%s direct_issue=%s\n", + worker, core_atomic_records, static_cast(core_atomic_calls), + core.atomic_calls, static_cast(core_poll_calls), core.poll_calls, + core_poll_batch_records, core.poll_batch_records, core_clock_records, + core_plain_clock_records, core_dependency_clock_records, + dependency_applied ? "yes" : "no", direct_result_used_return_ready ? "yes" : "no", + direct_result_used_source_issue ? "yes" : "no" + ); + success = false; + break; + } + observed_summary.records += available; + observed_summary.atomic_records += core_atomic_records; + observed_summary.clock_baseline_records += core_clock_records; + observed_summary.atomic_calls += core_atomic_calls; + observed_summary.poll_calls += core_poll_calls; + observed_summary.poll_batch_records += core_poll_batch_records; + observed_summary.dropped_records += core.dropped; + } + if (success && !SameTraceSummary(producer_summary, observed_summary)) { + std::fprintf( + stderr, + "swimlane producer/raw summary mismatch: records=%llu/%llu atomic_records=%llu/%llu " + "atomic_calls=%llu/%llu poll_calls=%llu/%llu poll_batches=%llu/%llu clock=%llu/%llu\n", + static_cast(observed_summary.records), + static_cast(producer_summary.records), + static_cast(observed_summary.atomic_records), + static_cast(producer_summary.atomic_records), + static_cast(observed_summary.atomic_calls), + static_cast(producer_summary.atomic_calls), + static_cast(observed_summary.poll_calls), + static_cast(producer_summary.poll_calls), + static_cast(observed_summary.poll_batch_records), + static_cast(producer_summary.poll_batch_records), + static_cast(observed_summary.clock_baseline_records), + static_cast(producer_summary.clock_baseline_records) + ); + success = false; + } + if (success) std::fprintf(output, "\n]}\n"); + if (std::ferror(output) != 0) { + std::fprintf(stderr, "Failed while writing swimlane output %s.\n", temporary_path.c_str()); + success = false; + } + if (std::fclose(output) != 0) { + std::fprintf(stderr, "Failed to close swimlane output %s: %s\n", temporary_path.c_str(), std::strerror(errno)); + success = false; + } + if (success && std::rename(temporary_path.c_str(), output_path.c_str()) != 0) { + std::fprintf( + stderr, "Cannot finalize swimlane output %s: %s\n", output_path.c_str(), std::strerror(errno) + ); + success = false; + } + if (!success) { + std::remove(temporary_path.c_str()); + return false; + } + std::printf( + "[SWIMLANE] raw_json=%s events=%llu\n", output_path.c_str(), + static_cast(exported_records) + ); + return true; +} + +template +inline bool AnalyzeSwimlaneRecords( + const TraceHeader &header, const SchedulerState &state, ReadRecords read_records +) { + if (!ValidateTraceHeader(header, "swimlane analysis")) return false; + + // 第一组数组统计“每个 worker 在某阶段的累计时间”；task_durations 则保留重点阶段的单事件分布。 + constexpr uint32_t kTracePhaseCount = static_cast(TracePhase::Count); + constexpr TracePhase kDetailedPhases[] = { + TracePhase::EfDrain, TracePhase::Materialize, TracePhase::Claim, TracePhase::Register, + }; + uint64_t cycles[kWorkers][kTracePhaseCount] = {}; + uint64_t counts[kWorkers][kTracePhaseCount] = {}; + std::vector task_durations[2][kTasksPerBatch][sizeof(kDetailedPhases) / sizeof(kDetailedPhases[0])]; + std::vector atomic_durations[2][static_cast(AtomicSite::Count)]; + uint64_t atomic_return_ready_counts[2][static_cast(AtomicSite::Count)] = {}; + std::vector atomic_poll_windows[2][static_cast(AtomicSite::Count)]; + uint64_t atomic_poll_calls[2][static_cast(AtomicSite::Count)] = {}; + std::vector clock_baselines[2]; + std::vector clock_dependency_baselines[2]; + uint64_t clock_dependency_applied[2] = {}; + std::vector scratch(kTraceRecordsPerCore); + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const uint32_t available = header.cores[worker].count; + const uint32_t count = std::min(available, header.records_per_core); + if (count != 0 && !read_records(worker, count, scratch.data())) { + return false; + } + for (uint32_t index = 0; index < count; ++index) { + const TraceRecord &record = scratch[index]; + if (record.phase < 0 || record.phase >= static_cast(kTracePhaseCount) || + record.end_cycle < record.start_cycle) { + // 分析器面对单条坏记录选择跳过；严格导出路径会直接拒绝，二者服务于不同诊断目的。 + continue; + } + const uint32_t phase = static_cast(record.phase); + const uint64_t duration = record.end_cycle - record.start_cycle; + const bool atomic_poll_batch = + record.phase == static_cast(TracePhase::Atomic) && + (record.flags & kAtomicPollBatch) != 0; + // PollBatch 的 duration 是一次等待 episode 的包络，允许夹着其他直接 + // atomic/调度代码；不能混入“Atomic 单次括号”的累计时间或分位数。 + if (!atomic_poll_batch) { + cycles[worker][phase] += duration; + ++counts[worker][phase]; + } + if (record.phase == static_cast(TracePhase::Atomic) && + record.auxiliary < static_cast(AtomicSite::Count)) { + const uint32_t role_index = + state.results[worker].role == static_cast(CoreRole::Aic) ? 0U : 1U; + if (atomic_poll_batch) { + atomic_poll_windows[role_index][record.auxiliary].push_back(duration); + atomic_poll_calls[role_index][record.auxiliary] += AtomicRecordCallCount(record); + } else { + atomic_durations[role_index][record.auxiliary].push_back(duration); + atomic_return_ready_counts[role_index][record.auxiliary] += + (record.flags & kAtomicReturnReady) != 0; + } + } + if (record.phase == static_cast(TracePhase::ClockBaseline)) { + const uint32_t role_index = + state.results[worker].role == static_cast(CoreRole::Aic) ? 0U : 1U; + if ((record.flags & kClockAtomicDependency) != 0) { + clock_dependency_baselines[role_index].push_back(duration); + clock_dependency_applied[role_index] += + (record.flags & kClockAtomicDependencyApplied) != 0; + } else { + clock_baselines[role_index].push_back(duration); + } + } + if (record.task_id >= 0) { + // task_id % 5 恰好对应 Alloc/QK/SF/PV/UP，这是固定 PA Case1 图的拓扑约束。 + const uint32_t role_index = + state.results[worker].role == static_cast(CoreRole::Aic) ? 0U : 1U; + const uint32_t kind = static_cast(record.task_id) % kTasksPerBatch; + for (uint32_t detail = 0; detail < sizeof(kDetailedPhases) / sizeof(kDetailedPhases[0]); ++detail) { + if (phase == static_cast(kDetailedPhases[detail])) { + task_durations[role_index][kind][detail].push_back(duration); + } + } + } + } + } + + const CoreRole roles[] = {CoreRole::Aic, CoreRole::Aiv}; + const char *role_names[] = {"AIC", "AIV"}; + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t phase = 0; phase < kTracePhaseCount; ++phase) { + std::vector role_cycles; + uint64_t record_count = 0; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + if (state.results[worker].role != static_cast(roles[role_index])) continue; + role_cycles.push_back(cycles[worker][phase]); + record_count += counts[worker][phase]; + } + const Uint64Distribution summary = SummarizeUint64(role_cycles); + std::printf( + "[TRACE_PHASE] role=%s phase=%s records=%llu accumulated_us_median=%.3f " + "accumulated_us_p95=%.3f accumulated_us_max=%.3f\n", + role_names[role_index], TracePhaseName(phase), + static_cast(record_count), summary.median / 1000.0, + static_cast(summary.p95) / 1000.0, + static_cast(summary.maximum) / 1000.0 + ); + } + } + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + const Uint64Distribution summary = SummarizeUint64(clock_baselines[role_index]); + if (!clock_baselines[role_index].empty()) { + std::printf( + "[TRACE_CLOCK] role=%s samples=%zu definition=consecutive-sys-cnt-reads " + "median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], clock_baselines[role_index].size(), summary.median, + static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + const Uint64Distribution dependency_summary = + SummarizeUint64(clock_dependency_baselines[role_index]); + if (!clock_dependency_baselines[role_index].empty()) { + std::printf( + "[TRACE_CLOCK] role=%s samples=%zu definition=atomic-return-dependency-hook " + "dependency_applied=%llu/%zu median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], clock_dependency_baselines[role_index].size(), + static_cast(clock_dependency_applied[role_index]), + clock_dependency_baselines[role_index].size(), dependency_summary.median, + static_cast(dependency_summary.p95), + static_cast(dependency_summary.maximum) + ); + } + } + // Atomic 只报告原始括号分布，不扣除计时底噪，也不把 total_cycles + // 解释成可与 Submit 墙钟直接相加的“atomic 占比”。return-ready 只表示 + // 本核可消费返回值，不表示其他核已经观察到更新。 + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t site = 0; site < static_cast(AtomicSite::Count); ++site) { + const std::vector &durations = atomic_durations[role_index][site]; + if (durations.empty()) continue; + const Uint64Distribution summary = SummarizeUint64(durations); + const AtomicOp op = AtomicSiteOp(static_cast(site)); + const uint64_t return_ready_count = atomic_return_ready_counts[role_index][site]; + const char *boundary = return_ready_count == durations.size() + ? "return-ready" + : (return_ready_count == 0 ? "source-issue" : "mixed"); + std::printf( + "[TRACE_ATOMIC] role=%s site=%s op=%s events=%zu boundary=%s " + "return_ready=%llu/%zu bracket_cycles_total=%llu median_ns=%.1f " + "p95_ns=%llu max_ns=%llu\n", + role_names[role_index], AtomicSiteName(site), AtomicOpName(static_cast(op)), + durations.size(), boundary, static_cast(return_ready_count), + durations.size(), static_cast(summary.total), summary.median, + static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + } + // 等待聚合只报告 episode 数、精确逻辑调用数与包络分布。window 不能除以 + // calls 当作单次 atomic latency，也不能与 Submit 墙钟直接相加。 + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t site = 0; site < static_cast(AtomicSite::Count); ++site) { + const std::vector &windows = atomic_poll_windows[role_index][site]; + if (windows.empty()) continue; + const Uint64Distribution summary = SummarizeUint64(windows); + std::printf( + "[TRACE_ATOMIC_POLL] role=%s site=%s op=%s episodes=%zu logical_calls=%llu " + "window_definition=wait-episode-envelope median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], AtomicSiteName(site), + AtomicOpName(static_cast(AtomicSiteOp(static_cast(site)))), + windows.size(), static_cast(atomic_poll_calls[role_index][site]), + summary.median, static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + } + const char *kind_names[] = {"Alloc", "QK", "SF", "PV", "UP"}; + // 单事件统计按 role 与 task kind 展开，可区分“该 role 真实参与”与“只回放前端”的成本。 + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t kind = 0; kind < kTasksPerBatch; ++kind) { + for (uint32_t detail = 0; detail < sizeof(kDetailedPhases) / sizeof(kDetailedPhases[0]); ++detail) { + const std::vector &durations = task_durations[role_index][kind][detail]; + const Uint64Distribution summary = SummarizeUint64(durations); + std::printf( + "[TRACE_TASK] role=%s kind=%s phase=%s events=%zu median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], kind_names[kind], + TracePhaseName(static_cast(kDetailedPhases[detail])), durations.size(), summary.median, + static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + } + } + return true; +} + +inline Metrics Validate( + const SchedulerState &state, uint32_t run, double host_us, const TraceHeader *trace_header = nullptr +) { + Metrics metrics; + // 每个 worker 都回放全部 task。Alloc 由 96 个 worker 全部执行 atomicMax Claim； + // 其余 kernel task 只有与 active role 匹配的 AIC 或 AIV 参与 Claim。 + const uint32_t batches = state.config.batches; + const uint32_t task_count = batches * kTasksPerBatch; + const uint64_t expected_submits = static_cast(kWorkers) * task_count; + const uint64_t expected_claims = + static_cast(batches) * (kWorkers + kAicWorkers + kAivWorkers + kAicWorkers + kAivWorkers); + // 上式依次对应 Alloc、QK、SF、PV、UP 的 active worker 数，默认 256 batch 时为 73728。 + + // 聚合量分为调度核心计数、kernel 分布、前端操作数和最终状态四组，便于定位语义偏差。 + uint64_t first_submit = UINT64_MAX; + uint64_t last_submit = 0; + uint64_t submits = 0; + uint64_t claims = 0; + uint64_t wins = 0; + uint64_t heap_guards = 0; + uint64_t fanin_ready_loads = 0; + uint64_t fanin_not_ready_loads = 0; + uint64_t frontier_initial_loads = 0; + uint64_t frontier_updates = 0; + uint64_t frontier_terminal_loads = 0; + uint64_t atomic_trace_calls = 0; + uint64_t duplicates = 0; + uint64_t cas_retries = 0; + uint64_t joint_polls = 0; + uint64_t trace_wait_records = 0; + uint64_t wins_by_kind[5] = {}; + uint64_t kernel_counts[4] = {}; + uint64_t kernel_cycles[4] = {}; + uint64_t kernel_min[4] = {}; + uint64_t kernel_max[4] = {}; + uint64_t placements[3] = {}; + uint64_t phase_calls[static_cast(ProfilePhase::Count)] = {}; + uint64_t context_reads = 0; + uint64_t views_created = 0; + uint64_t dynamic_create_infos = 0; + uint64_t arg_resets = 0; + uint64_t tensor_args_added = 0; + uint64_t scalar_args_added = 0; + uint64_t materialized_outputs = 0; + uint64_t map_inserts = 0; + uint64_t map_lookups = 0; + uint64_t slot_tensor_copies = 0; + uint64_t slot_scalar_copies = 0; + uint64_t fanin_edges = 0; + bool worker_ids[kWorkers] = {}; + uint32_t aic_count = 0; + uint32_t aiv_count = 0; + uint32_t winning_workers = 0; + uint64_t max_worker_wins = 0; + bool worker_shape_ok = true; + bool submit_timestamps_ok = true; + bool vend_values_ok = true; + bool frontend_worker_counts_ok = true; + bool final_worker_state_ok = true; + bool worker_checksums_ok = true; + bool fanin_worker_counts_ok = true; + bool frontier_worker_counts_ok = true; + bool role_kernel_routing_ok = true; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + bool lazy_sample_split_runtime_oracle_ok = true; + const uint64_t expected_lazy_sample_split_task_id_sum = + static_cast(task_count) * (task_count - 1U) / 2U; +#endif + + // 按真实输出大小、1 KiB 对齐和 256 MiB 环回规则重算每个 task 可接受的最小 vend。 + uint64_t expected_heap_next = 0; + bool vend_progress_bounds_ok = true; + uint32_t first_bad_vend = task_count; + uint64_t first_bad_vend_minimum = 0; + uint64_t first_bad_vend_actual = 0; + std::vector minimum_vends(task_count); + const uint64_t output_bytes_by_kind[] = {10240, 524288, 264192, 8192, 0}; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + const uint64_t output_bytes = output_bytes_by_kind[task_id % kTasksPerBatch]; + uint64_t task_base = (expected_heap_next + kOutputAlignment - 1) / kOutputAlignment * kOutputAlignment; + if (output_bytes != 0 && (task_base % state.heap_size) + output_bytes > state.heap_size) { + task_base = (task_base / state.heap_size + 1) * state.heap_size; + } + expected_heap_next = task_base + output_bytes; + minimum_vends[task_id] = expected_heap_next; + } + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + // vend 可以大于本 task 的最小末端，因为 winner 发布的是其本地 heap_cursor 快照； + // 但不能超过该 worker 完整回放所有 task 后的最终 heap_next。 + if (state.tasks[task_id].vend < minimum_vends[task_id] || + state.tasks[task_id].vend > expected_heap_next) { + vend_progress_bounds_ok = false; + if (first_bad_vend == task_count) { + first_bad_vend = task_id; + first_bad_vend_minimum = minimum_vends[task_id]; + first_bad_vend_actual = state.tasks[task_id].vend; + } + } + } + // TensorMap 只保留 heap window 内仍可能被依赖的四类输出；floor 对应已安全退休的 task 边界。 + const uint64_t expected_map_live = 4ULL * std::min(batches, 13); + const uint64_t expected_map_floor = task_count > kHeapWindow + 1 ? task_count - kHeapWindow - 1 : 0; + + for (uint32_t index = 0; index < kWorkers; ++index) { + // 每核只写自己独占且按 cache line 隔离的 WorkerResult；host 在 kernel 完成后统一汇总，不引入额外 atomic。 + const WorkerResult &result = state.results[index]; + if (result.worker_id < kWorkers) { + worker_ids[result.worker_id] = true; + } else { + worker_shape_ok = false; + } + aic_count += result.role == static_cast(CoreRole::Aic); + aiv_count += result.role == static_cast(CoreRole::Aiv); + worker_shape_ok &= result.submits == task_count; + worker_shape_ok &= result.max_occupied <= kUsableSlots; + worker_shape_ok &= result.final_occupied == 0; + submit_timestamps_ok &= result.submit_begin != 0; + submit_timestamps_ok &= result.submit_end >= result.submit_begin; + submit_timestamps_ok &= result.finish_cycle >= result.submit_end; + first_submit = std::min(first_submit, result.submit_begin); + last_submit = std::max(last_submit, result.submit_end); + submits += result.submits; + claims += result.claim_attempts; + wins += result.claim_wins; + if (result.claim_wins != 0) ++winning_workers; + max_worker_wins = std::max(max_worker_wins, result.claim_wins); + heap_guards += result.heap_guards; + fanin_ready_loads += result.fanin_ready_loads; + fanin_not_ready_loads += result.fanin_not_ready_loads; + frontier_initial_loads += result.frontier_initial_loads; + frontier_updates += result.frontier_updates; + frontier_terminal_loads += result.frontier_terminal_loads; + atomic_trace_calls += result.atomic_trace_calls; + duplicates += result.completion_duplicates; + cas_retries += result.cas_retries; + joint_polls += result.joint_polls; + trace_wait_records += result.wait_events[0] + result.wait_events[1]; + context_reads += result.context_reads; + views_created += result.views_created; + dynamic_create_infos += result.dynamic_create_infos; + arg_resets += result.arg_resets; + tensor_args_added += result.tensor_args_added; + scalar_args_added += result.scalar_args_added; + materialized_outputs += result.materialized_outputs; + map_inserts += result.map_inserts; + map_lookups += result.map_lookups; + slot_tensor_copies += result.slot_tensor_copies; + slot_scalar_copies += result.slot_scalar_copies; + fanin_edges += result.fanin_edges; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + const CoreRole expected_role = index < kAicWorkers ? CoreRole::Aic : CoreRole::Aiv; + lazy_sample_split_runtime_oracle_ok &= result.worker_id == index; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_caller_state_address != 0; + lazy_sample_split_runtime_oracle_ok &= + result.lazy_sample_split_finish_state_address == result.lazy_sample_split_caller_state_address; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_finish_calls == task_count; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_protocol_errors == 0; + lazy_sample_split_runtime_oracle_ok &= + result.lazy_sample_split_state_cookie == + (kLazySampleSplitStateCookieBase ^ static_cast(index) ^ + (static_cast(static_cast(expected_role)) << 32U)); + lazy_sample_split_runtime_oracle_ok &= + result.lazy_sample_split_task_id_sum == expected_lazy_sample_split_task_id_sum; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_owner_worker_id == index; + lazy_sample_split_runtime_oracle_ok &= result.lazy_sample_split_reserved == 0; +#endif +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + const uint64_t qk_wins = result.wins[static_cast(TaskKind::Qk)]; + const uint64_t sf_wins = result.wins[static_cast(TaskKind::Sf)]; + const uint64_t pv_wins = result.wins[static_cast(TaskKind::Pv)]; + const uint64_t up_wins = result.wins[static_cast(TaskKind::Up)]; + const uint64_t expected_worker_views = kLazySampleLazy + ? static_cast(batches) + qk_wins + : static_cast(batches) * 2; + const uint64_t expected_worker_tensors = kLazySampleLazy + ? static_cast(batches) * 12 + + qk_wins * 3 + sf_wins + pv_wins * 3 + up_wins * 3 + : static_cast(batches) * 22; + const uint64_t expected_worker_scalars = kLazySampleLazy + ? qk_wins * 2 + sf_wins * 3 + pv_wins * 2 + up_wins * 2 + : static_cast(batches) * 9; + frontend_worker_counts_ok &= result.context_reads == batches; + frontend_worker_counts_ok &= result.views_created == expected_worker_views; + frontend_worker_counts_ok &= result.dynamic_create_infos == static_cast(batches) * 2; + // Alloc constructs once; QK/SF/PV/UP each reset once inside their callback. + frontend_worker_counts_ok &= result.arg_resets == static_cast(batches) * 4; + frontend_worker_counts_ok &= result.tensor_args_added == expected_worker_tensors; + frontend_worker_counts_ok &= result.scalar_args_added == expected_worker_scalars; + frontend_worker_counts_ok &= result.materialized_outputs == static_cast(batches) * 8; + frontend_worker_counts_ok &= result.map_inserts == static_cast(batches) * 4; +#else + frontend_worker_counts_ok &= result.context_reads == batches; + frontend_worker_counts_ok &= result.views_created == static_cast(batches) * 2; + frontend_worker_counts_ok &= result.dynamic_create_infos == static_cast(batches) * 2; + frontend_worker_counts_ok &= result.arg_resets == static_cast(batches) * 4; + frontend_worker_counts_ok &= result.tensor_args_added == static_cast(batches) * 22; + frontend_worker_counts_ok &= result.scalar_args_added == static_cast(batches) * 9; + frontend_worker_counts_ok &= result.materialized_outputs == static_cast(batches) * 8; + frontend_worker_counts_ok &= result.map_inserts == static_cast(batches) * 4; +#endif + final_worker_state_ok &= result.final_heap_next == expected_heap_next; + final_worker_state_ok &= result.map_high_water == expected_map_live; + final_worker_state_ok &= result.map_live_entries == expected_map_live; + final_worker_state_ok &= result.map_alive_floor == expected_map_floor; + final_worker_state_ok &= result.map_cleaned_upto == expected_map_floor; + worker_checksums_ok &= result.checksum == (0xcbf29ce484222325ULL ^ result.worker_id); + const uint64_t worker_kernel_completions = result.kernel_counts[0] + result.kernel_counts[1] + + result.kernel_counts[2] + result.kernel_counts[3]; + if (result.role == static_cast(CoreRole::Aic)) { + role_kernel_routing_ok &= result.kernel_counts[1] == 0 && result.kernel_counts[3] == 0; + } else if (result.role == static_cast(CoreRole::Aiv)) { + role_kernel_routing_ok &= result.kernel_counts[0] == 0 && result.kernel_counts[2] == 0; + } else { + role_kernel_routing_ok = false; + } + const uint64_t worker_completions = result.wins[0] + worker_kernel_completions; + frontier_worker_counts_ok &= result.frontier_initial_loads == worker_completions; + frontier_worker_counts_ok &= result.frontier_terminal_loads == result.frontier_initial_loads; + fanin_worker_counts_ok &= result.fanin_ready_loads >= result.fanin_edges; + if (result.fanin_ready_loads >= result.fanin_edges) { + // PA 最大 fanin 为 3；每次失败检查最多先重读两个 ready 前缀，再遇到一个 not-ready。 + fanin_worker_counts_ok &= + result.fanin_ready_loads - result.fanin_edges <= 2 * result.fanin_not_ready_loads; + } + for (uint32_t kind = 0; kind < 5; ++kind) + wins_by_kind[kind] += result.wins[kind]; + for (uint32_t kind = 0; kind < 4; ++kind) { + kernel_counts[kind] += result.kernel_counts[kind]; + kernel_cycles[kind] += result.kernel_cycles[kind]; + if (result.kernel_min_cycles[kind] != 0 && + (kernel_min[kind] == 0 || result.kernel_min_cycles[kind] < kernel_min[kind])) { + kernel_min[kind] = result.kernel_min_cycles[kind]; + } + kernel_max[kind] = std::max(kernel_max[kind], result.kernel_max_cycles[kind]); + } + for (uint32_t place = 0; place < 3; ++place) + placements[place] += result.placement[place]; + for (uint32_t phase = 0; phase < static_cast(ProfilePhase::Count); ++phase) + phase_calls[phase] += result.phase_calls[phase]; + } + for (bool seen : worker_ids) + worker_shape_ok &= seen; + + uint32_t ready_flags = 0; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + // ready flag 和 vend 是跨核 completion 的最终外部可见状态，不能只依赖 worker 私有计数判断完成。 + ready_flags += state.tasks[task_id].flag == 1; + vend_values_ok &= state.tasks[task_id].vend != 0; + vend_values_ok &= state.tasks[task_id].vend % kOutputAlignment == 0; + } + const uint64_t kernel_total = kernel_counts[0] + kernel_counts[1] + kernel_counts[2] + kernel_counts[3]; + const uint64_t placement_total = placements[0] + placements[1] + placements[2]; + const uint64_t fanin_loads = fanin_ready_loads + fanin_not_ready_loads; + const uint64_t frontier_flag_loads = frontier_updates + frontier_terminal_loads; + + // 第一组断言覆盖参与者拓扑、Claim/winner、completion 和最终 drain 等调度主协议。 + Expect(aic_count == kAicWorkers && aiv_count == kAivWorkers, "participant topology is 32 AIC + 64 AIV", &metrics); + Expect(worker_shape_ok, "all 96 worker markers and private rings are valid", &metrics); +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + Expect( + lazy_sample_split_runtime_oracle_ok, + "split caller/finish share one role-specific block-local state and finish every task once", + &metrics + ); +#endif + Expect(submit_timestamps_ok, "all Submit timing markers are valid", &metrics); + Expect(state.started_count.value == kWorkers, "started_count is 96", &metrics); + Expect(submits == expected_submits, "replay count is workers * tasks", &metrics); + Expect(claims == expected_claims, "Claim attempt count matches PA topology", &metrics); + Expect(wins == task_count, "exactly one winner per task", &metrics); + Expect( + wins_by_kind[0] == batches && wins_by_kind[1] == batches && wins_by_kind[2] == batches && + wins_by_kind[3] == batches && wins_by_kind[4] == batches, + "Alloc/QK/SF/PV/UP winners are one per batch", &metrics + ); + Expect(kernel_total == static_cast(batches) * 4, "kernel count is four per batch", &metrics); + Expect( + kernel_counts[0] == batches && kernel_counts[1] == batches && kernel_counts[2] == batches && + kernel_counts[3] == batches, + "each kernel kind executes once per batch", &metrics + ); + Expect( + role_kernel_routing_ok, + "AIC executes only QK/PV and AIV executes only SF/UP", &metrics + ); + Expect(heap_guards == static_cast(batches) * 4, "heap guard count matches output winners", &metrics); + Expect( + fanin_worker_counts_ok && fanin_ready_loads >= fanin_edges && + fanin_ready_loads - fanin_edges <= 2 * fanin_not_ready_loads, + "fanin ready/failure load classification is complete", &metrics + ); + Expect( + frontier_worker_counts_ok && frontier_initial_loads == task_count, + "frontier initial loads match completed tasks", &metrics + ); + Expect( + frontier_terminal_loads == task_count && frontier_updates >= task_count, + "frontier ready/update/terminal load identity is exact", &metrics + ); + Expect(duplicates == 0, "completion flags are published once", &metrics); + Expect(ready_flags == task_count, "all task flags are ready", &metrics); + Expect(vend_values_ok, "all published vend values are nonzero and aligned", &metrics); + Expect(vend_progress_bounds_ok, "every task vend is within PA worker heap progress bounds", &metrics); + Expect(state.frontier.value == static_cast(task_count) - 1, "frontier reaches the final task", &metrics); + Expect(state.replay_done.value == kWorkers, "replay_done is 96", &metrics); + Expect(state.fatal.value == 0, "fatal remains clear", &metrics); + Expect(placement_total == kernel_total, "EfDrain + RingBp + final placement covers every kernel", &metrics); + // joint_polls 是为未来 BlockWon 模拟预留的结果字段，当前调度路径没有递增点； + // 此断言只确认现有输出保持零，不能单独证明 active_count>=2 分支不可达。 + Expect(joint_polls == 0, "single-lane PA performs no BlockWon polling", &metrics); + // 第二组断言锁定 scalar 前端工作量，防止编译器优化或后续改动悄悄删掉 PA 模拟步骤。 + Expect(frontend_worker_counts_ok, "every worker replays the exact PA frontend operation counts", &metrics); +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + const uint64_t expected_global_views = static_cast(batches) * + (kLazySampleLazy ? kWorkers + 1 : kWorkers * 2); + const uint64_t expected_global_tensors = static_cast(batches) * + (kLazySampleLazy ? kWorkers * 12 + 10 : kWorkers * 22); + const uint64_t expected_global_scalars = static_cast(batches) * + (kLazySampleLazy ? 9 : kWorkers * 9); + Expect( + context_reads == static_cast(kWorkers) * batches && + views_created == expected_global_views && + dynamic_create_infos == static_cast(kWorkers) * batches * 2 && + arg_resets == static_cast(kWorkers) * batches * 4 && + tensor_args_added == expected_global_tensors && + scalar_args_added == expected_global_scalars && + materialized_outputs == static_cast(kWorkers) * batches * 8 && + map_inserts == static_cast(kWorkers) * batches * 4, + "global PA frontend operation totals are exact", &metrics + ); + std::printf( + "[LAZY_SAMPLE_FRONTEND] shape=%s views=%llu/%llu tensor_args=%llu/%llu " + "scalar_args=%llu/%llu resets=%llu/%llu\n", + kLazySampleShapeName, + static_cast(views_created), + static_cast(expected_global_views), + static_cast(tensor_args_added), + static_cast(expected_global_tensors), + static_cast(scalar_args_added), + static_cast(expected_global_scalars), + static_cast(arg_resets), + static_cast(static_cast(kWorkers) * batches * 4) + ); +#else + Expect( + context_reads == static_cast(kWorkers) * batches && + views_created == static_cast(kWorkers) * batches * 2 && + dynamic_create_infos == static_cast(kWorkers) * batches * 2 && + arg_resets == static_cast(kWorkers) * batches * 4 && + tensor_args_added == static_cast(kWorkers) * batches * 22 && + scalar_args_added == static_cast(kWorkers) * batches * 9 && + materialized_outputs == static_cast(kWorkers) * batches * 8 && + map_inserts == static_cast(kWorkers) * batches * 4, + "global PA frontend operation totals are exact", &metrics + ); +#endif + Expect( + map_lookups == static_cast(batches) * 14 && + slot_tensor_copies == static_cast(batches) * 19 && + slot_scalar_copies == static_cast(batches) * 9 && + fanin_edges == static_cast(batches) * 5, + "winner-only map, slot-copy, and fanin totals are exact", &metrics + ); + Expect(final_worker_state_ok, "every worker final heap and TensorMap state is exact", &metrics); + Expect(worker_checksums_ok, "all frontend registration checksums remain clean", &metrics); + + // 三类 Claim cursor 各有四个 shard；按 task_id 重新推导每个 shard 应停留的最后任务。 + int64_t expected_cube[kCursorShards] = {-1, -1, -1, -1}; + int64_t expected_vector[kCursorShards] = {-1, -1, -1, -1}; + int64_t expected_alloc[kCursorShards] = {-1, -1, -1, -1}; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + const TaskKind kind = static_cast(task_id % kTasksPerBatch); + int64_t *cursors = kind == TaskKind::Alloc + ? expected_alloc + : (kind == TaskKind::Qk || kind == TaskKind::Pv ? expected_cube : expected_vector); + cursors[task_id % kCursorShards] = task_id; + } + bool cursors_ok = true; + for (uint32_t shard = 0; shard < kCursorShards; ++shard) { + cursors_ok &= state.cube_cursor[shard].value == expected_cube[shard]; + cursors_ok &= state.vector_cursor[shard].value == expected_vector[shard]; + cursors_ok &= state.alloc_cursor[shard].value == expected_alloc[shard]; + } + Expect(cursors_ok, "all sharded Claim cursors reach their exact final task", &metrics); + + if (state.config.profile_phases != 0) { + // profile 开关关闭时这些字段允许保持零，避免把可选诊断本身变成语义门禁。 + Expect( + phase_calls[static_cast(ProfilePhase::Claim)] == expected_submits && + phase_calls[static_cast(ProfilePhase::EfDrain)] == expected_submits && + phase_calls[static_cast(ProfilePhase::WaitForSlot)] == + static_cast(batches) * 4 && + phase_calls[static_cast(ProfilePhase::HeapGuard)] == + static_cast(batches) * 4, + "profile call counts match Claim/EfDrain/WaitForSlot/HeapGuard flow", &metrics + ); + } + + if (state.config.trace_enabled != 0) { + // 固定阶段记录数加上动态等待记录数，应与所有 worker 的 header count 精确相等。 + bool trace_shape_ok = trace_header != nullptr; + uint64_t trace_records = 0; + uint64_t trace_dropped = 0; + uint64_t physical_atomic_records = 0; + uint64_t batched_poll_calls = 0; + uint64_t poll_batch_records = 0; + bool per_worker_trace_counts_ok = true; + if (trace_header != nullptr) { + trace_shape_ok &= trace_header->magic == 0x4653574cU; + trace_shape_ok &= trace_header->version == 4; + trace_shape_ok &= trace_header->num_cores == kWorkers; + trace_shape_ok &= trace_header->records_per_core == kTraceRecordsPerCore; + trace_shape_ok &= trace_header->frequency_hz == kSystemCounterHz; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const TraceCoreState &core = trace_header->cores[worker]; + trace_records += core.count; + trace_dropped += core.dropped; + trace_shape_ok &= core.count <= kTraceRecordsPerCore; + int32_t expected_block = -1; + int32_t expected_lane = -1; + ExpectedTraceTopology(worker, &expected_block, &expected_lane); + trace_shape_ok &= core.core_idx == static_cast(worker); + trace_shape_ok &= core.block_id == expected_block; + trace_shape_ok &= core.lane == expected_lane; + const WorkerResult &result = state.results[worker]; + const uint64_t worker_kernels = result.kernel_counts[0] + result.kernel_counts[1] + + result.kernel_counts[2] + result.kernel_counts[3]; + uint64_t worker_physical_atomic = 0; + if ((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) { + trace_shape_ok &= core.atomic_calls == result.atomic_trace_calls; + trace_shape_ok &= core.poll_calls <= core.atomic_calls; + trace_shape_ok &= (core.poll_calls == 0) == (core.poll_batch_records == 0); + worker_physical_atomic = + static_cast(core.atomic_calls) - core.poll_calls + core.poll_batch_records; + physical_atomic_records += worker_physical_atomic; + batched_poll_calls += core.poll_calls; + poll_batch_records += core.poll_batch_records; + } else { + trace_shape_ok &= core.atomic_calls == 0 && core.poll_calls == 0 && + core.poll_batch_records == 0; + } + const uint64_t worker_expected = + 6 * result.submits + 2 * result.claim_wins - result.wins[0] + + 2 * worker_kernels + result.wait_events[0] + result.wait_events[1] + 2 + + (((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) + ? worker_physical_atomic + 2 + : 0); + per_worker_trace_counts_ok &= core.count == worker_expected; + } + } + const uint64_t expected_trace_records = + static_cast(batches) * (static_cast(kWorkers) * 30 + 17) + + trace_wait_records + 2 * kWorkers + + (((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) + ? physical_atomic_records + 2 * kWorkers + : 0); + // 每 batch 的 96*30 是六条每 Submit 固定记录；17 条是 + // 5 条 winner tail、4 条 Fanin 和 8 条 Kernel/Commit。loser 不再写 + // 零时长 marker。两个父 span 再各核固定增加 2 条； + // RingBp 等真实等待按运行时次数额外加入。 + Expect(trace_shape_ok, "swimlane header and per-worker capacities are valid", &metrics); + Expect(trace_dropped == 0, "swimlane records fit without drops", &metrics); + Expect(trace_records == expected_trace_records, "swimlane record count matches PA phase flow", &metrics); + Expect(per_worker_trace_counts_ok, "every worker swimlane record count is exact", &metrics); + if ((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) { + Expect(atomic_trace_calls != 0, "atomic trace captured source-level calls", &metrics); + } else { + Expect(atomic_trace_calls == 0, "atomic trace counters stay zero when disabled", &metrics); + } + std::printf( + "[TRACE] records=%llu expected=%llu dropped=%llu bytes=%zu\n", + static_cast(trace_records), + static_cast(expected_trace_records), + static_cast(trace_dropped), kTraceBytes + ); + std::printf( + "[ATOMIC_TRACE] enabled=%s logical_calls=%llu physical_records=%llu " + "batched_poll_calls=%llu poll_batch_records=%llu " + "closure=physical=logical-batched+batch_records\n", + (state.config.trace_enabled & kTraceAtomicsEnabled) != 0 ? "yes" : "no", + static_cast(atomic_trace_calls), + static_cast(physical_atomic_records), + static_cast(batched_poll_calls), + static_cast(poll_batch_records) + ); + } + + if (first_submit != UINT64_MAX && last_submit >= first_submit) { + // 性能口径只覆盖最早 Submit.begin 到最晚 Submit.end，不含启动屏障、最终 drain 和 host 同步。 + metrics.submit_span_us = static_cast(last_submit - first_submit) / 1000.0; + } + std::printf( + "[METRIC] run=%u submit_span_us=%.3f host_launch_us=%.3f claims=%llu fanin_loads=%llu cas_retries=%llu\n", run, + metrics.submit_span_us, host_us, static_cast(claims), + static_cast(fanin_loads), static_cast(cas_retries) + ); + const uint64_t submit_completion_ops = + claims + heap_guards + fanin_loads + 2ULL * task_count + frontier_initial_loads + + frontier_flag_loads + frontier_updates; + std::printf( + "[ATOMIC] submit_completion_ops=%llu fanin_ready=%llu fanin_not_ready=%llu frontier_initial=%llu " + "frontier_flag=%llu frontier_ready_fetch_max=%llu frontier_terminal=%llu\n", + static_cast(submit_completion_ops), + static_cast(fanin_ready_loads), + static_cast(fanin_not_ready_loads), + static_cast(frontier_initial_loads), + static_cast(frontier_flag_loads), + static_cast(frontier_updates), + static_cast(frontier_terminal_loads) + ); + std::printf( + "[WINNERS] active_workers=%u max_wins_per_worker=%llu\n", winning_workers, + static_cast(max_worker_wins) + ); + std::printf( + "[PLACEMENT] EfDrain=%llu RingBp=%llu FinalDrain=%llu\n", + static_cast(placements[static_cast(DrainPlace::EfDrain)]), + static_cast(placements[static_cast(DrainPlace::RingBackpressure)]), + static_cast(placements[static_cast(DrainPlace::FinalDrain)]) + ); + // placement 统计回答 kernel 最终在哪个 drain 点执行，与 TracePhase 的累计 span 互补。 + const char *kernel_names[] = {"QK", "SF", "PV", "UP"}; + const uint32_t targets[] = {kTargetQkTicks, kTargetSfTicks, kTargetPvTicks, kTargetUpTicks}; + for (uint32_t kind = 0; kind < 4; ++kind) { + const double mean = + kernel_counts[kind] == 0 ? 0.0 : static_cast(kernel_cycles[kind]) / kernel_counts[kind]; + std::printf( + "[KERNEL] %-2s count=%llu mean_us=%.3f min_us=%.3f max_us=%.3f target_us=%.3f\n", kernel_names[kind], + static_cast(kernel_counts[kind]), mean / 1000.0, kernel_min[kind] / 1000.0, + kernel_max[kind] / 1000.0, targets[kind] / 1000.0 + ); + } + PrintPhaseDiagnostics(state); + if (!metrics.passed) { + // 失败时补充第一处未完成 task、vend 边界和 worker 进度，避免只有笼统的 ASSERT FAIL。 + uint32_t first_not_ready = task_count; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + if (state.tasks[task_id].flag != 1) { + first_not_ready = task_id; + break; + } + } + uint64_t min_worker_submits = UINT64_MAX; + uint64_t max_worker_submits = 0; + uint32_t incomplete_workers = 0; + uint32_t occupied_workers = 0; + uint64_t max_final_occupied = 0; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const WorkerResult &result = state.results[worker]; + min_worker_submits = std::min(min_worker_submits, result.submits); + max_worker_submits = std::max(max_worker_submits, result.submits); + incomplete_workers += result.submits != task_count; + occupied_workers += result.final_occupied != 0; + max_final_occupied = std::max(max_final_occupied, result.final_occupied); + } + const int64_t retire = state.frontier.value - static_cast(kHeapWindow); + const uint64_t retire_vend = + retire >= 0 && retire < static_cast(task_count) ? state.tasks[retire].vend : 0; + std::printf( + "[FAILURE_STATE] fatal=%d frontier=%lld first_not_ready=%u first_bad_vend=%u " + "vend_minimum=%llu vend_actual=%llu retire=%lld retire_vend=%llu " + "worker_submits_min=%llu worker_submits_max=%llu incomplete_workers=%u " + "final_occupied_workers=%u max_final_occupied=%llu\n", + state.fatal.value, static_cast(state.frontier.value), first_not_ready, first_bad_vend, + static_cast(first_bad_vend_minimum), + static_cast(first_bad_vend_actual), + static_cast(retire), static_cast(retire_vend), + static_cast(min_worker_submits), + static_cast(max_worker_submits), incomplete_workers, occupied_workers, + static_cast(max_final_occupied) + ); + } + return metrics; +} + +inline double Median(std::vector values) { + // 多轮 benchmark 只报告中位数；上板基线比较仍应优先采用独立进程首轮。 + std::sort(values.begin(), values.end()); + const size_t middle = values.size() / 2; + if ((values.size() & 1U) != 0) return values[middle]; + return (values[middle - 1] + values[middle]) / 2.0; +} + +inline void PrintBanner(const char *backend, const Options &options) { + // 开始运行前完整打印工作量和大内存占用，便于确认比较口径没有混用。 + std::printf("=== Standalone PA Scheduler Benchmark: %s ===\n", backend); + std::printf( + "device=%u batches=%u tasks=%u workers=%u runs=%u nops=%u,%u,%u,%u state_bytes=%zu " + "swimlane=%s trace_atomics=%s trace_bytes=%zu\n", options.device, + options.batches, options.batches * kTasksPerBatch, kWorkers, options.runs, options.nops.qk, options.nops.sf, + options.nops.pv, options.nops.up, sizeof(SchedulerState), options.trace_enabled ? "on" : "off", + options.trace_atomics ? "on" : "off", + options.trace_enabled ? kTraceBytes : 0 + ); +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + std::printf( + "lazy_sample_shape=%s lazy_sample_shape_id=%u observation=%s finish=%s " + "control_family=%s\n", + kLazySampleShapeName, kLazySampleShapeId, kLazySampleObservation, + kLazySampleFinishShape, kLazySampleControlFamily + ); +#endif + if (!options.swimlane_json.empty()) { + std::printf("swimlane_json=%s\n", options.swimlane_json.c_str()); + } +} + +} // namespace pa_scheduler::host + +#endif // PA_SCHEDULER_COMMON_HOST_SUPPORT_H diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/pa_frontend.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/pa_frontend.h new file mode 100644 index 0000000000..f2cc8fcabb --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/pa_frontend.h @@ -0,0 +1,1308 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_FRONTEND_H +#define PA_SCHEDULER_COMMON_PA_FRONTEND_H + +#include "pa_model.h" + +namespace pa_scheduler { + +// 这些基址只充当稳定的 tensor identity，供 descriptor、区间重叠和 heap 地址 +// 计算使用；winner workload 不解引用这些 synthetic 地址，real-compute 使用独立 +// workspace。context_lens 是唯一按真实 GM 指针读取的 PA 前端输入。 +constexpr uint64_t kInvalidTaskId = UINT64_MAX; +constexpr uint64_t kSyntheticQueryBase = 0x200000000ULL; +constexpr uint64_t kSyntheticKeyBase = 0x300000000ULL; +constexpr uint64_t kSyntheticValueBase = 0x400000000ULL; +constexpr uint64_t kSyntheticBlockTableBase = 0x500000000ULL; +constexpr uint64_t kSyntheticOutputBase = 0x600000000ULL; +constexpr uint64_t kSyntheticContextLensBase = 0x700000000ULL; +constexpr uint32_t kPaHeads = 16; +constexpr uint32_t kPaHeadDim = 128; +constexpr uint32_t kPaBlockSize = 128; +constexpr uint32_t kPaBlocksPerRequest = 64; +constexpr uint32_t kPaMaxBlocksPerRequest = 256; +constexpr uint64_t kPaScaleBits = 0x3F800000ULL; +constexpr uint32_t kSpmdLocalContextIndex = kMaxTaskTensors + kMaxTaskScalars; +constexpr uint32_t kSpmdGlobalContextIndex = kSpmdLocalContextIndex + 1; +static_assert(kMaxTaskTensors == 32, "PA frontend requires the real 32 tensor slots"); +static_assert(kMaxTaskScalars == 16, "PA frontend requires the real 16 scalar slots"); +static_assert(kSpmdLocalContextIndex == 48, "PA local-context dispatch index mismatch"); +static_assert(kSpmdGlobalContextIndex == 49, "PA global-context dispatch index mismatch"); +static_assert(kMaxFanin == 16, "PA frontend requires the real 16 fanin slots"); + +enum class TensorRefKind : uint8_t { + LocalTensor = 0, + GmTensor = 1, + CreateInfo = 2, +}; + +// TaskArgs 同时容纳 orchestration 栈上的 descriptor、GM 中已物化的 descriptor， +// 以及尚待 Materialize 的 CreateInfo。显式 kind 保留生产 TensorRef 的地址空间分支。 +union TensorPointer { + const TensorDesc *local_tensor; + PA_GM const TensorDesc *gm_tensor; + const TensorCreateInfo *create_info; +}; + +struct TaskTensorRef { + TensorPointer pointer; + TensorRefKind kind; +}; +static_assert(sizeof(TaskTensorRef) == 16, "TaskTensorRef must match the PA TensorRef ABI"); +static_assert(offsetof(TaskTensorRef, pointer) == 0, "TaskTensorRef pointer offset mismatch"); +static_assert(offsetof(TaskTensorRef, kind) == 8, "TaskTensorRef kind offset mismatch"); + +struct PaLaunchSpec { + int16_t core_num; + bool require_sync_start; +}; +static_assert(sizeof(PaLaunchSpec) == 4, "PA launch spec ABI mismatch"); + +struct PaAsyncContext { + uint64_t completion_count; + uint64_t completion_error_code; + uint64_t completion_entries; + uint32_t completion_capacity; + uint32_t alignment_padding; + uint64_t task_token; +}; +static_assert(sizeof(PaAsyncContext) == 40, "PA async context ABI mismatch"); + +struct PaLocalContext { + int32_t block_index; + int32_t block_count; + PaAsyncContext async; +}; +// Local/GlobalContext 最终放进 RingSlot 的固定 dispatch 参数位 48/49；它们不是 +// standalone 自定义参数，offset 必须与真实 SPMD kernel 调用约定一致。 +static_assert(sizeof(PaLocalContext) == 48, "PA local context ABI mismatch"); + +struct PaGlobalContext { + int32_t sub_block_id; +}; +static_assert(sizeof(PaGlobalContext) == 4, "PA global context ABI mismatch"); + +// PTO2 profiling is enabled in the PA baseline. reset() clears all 160 bytes +// below on every QK/SF/PV/UP argument rebuild, even though Case1 does not ask +// to dump an argument. Keeping this storage and write stream matters to the +// spacing between consecutive Claim operations. +// 这段看似未使用的清零属于真实前端成本，删除会改变各 worker 到达 +// Claim 的波形与竞争强度，因此仍按生产构造/reset 顺序执行。 +struct PaDumpArgSelection { + uint64_t dump_arg_mask; + uint64_t dump_arg_index_ambiguous_mask; + uint64_t scalar_source_ptrs[kMaxTaskScalars]; + uint8_t scalar_dtypes[kMaxTaskScalars]; +}; +static_assert(sizeof(PaDumpArgSelection) == 160, "PA dump-selection ABI mismatch"); + +struct TaskArgs { + // The real TaskArgsTpl inherits its tag mixin first. TensorArgType is an + // int32 enum in the PA ABI; keeping tags first also reproduces its offsets. + // tag 数组位于对象首部不是任意排布；Materialize、fanin 与 register + // 都会重复扫描/复用这些 tag，错误 offset 会同时改变语义和前端访存成本。 + int32_t tags[kMaxTaskTensors]; + TaskTensorRef tensors[kMaxTaskTensors]; + uint64_t scalars[kMaxTaskScalars]; + int32_t tensor_count; + int32_t scalar_count; + + bool has_error; + uint64_t error_msg; + PaLaunchSpec launch_spec; + PaDumpArgSelection dump_arg_selection; + uint64_t explicit_deps; + uint32_t explicit_dep_count; + uint8_t cacheline_pad[48]; +}; +static_assert(sizeof(TaskArgs) == 1024, "TaskArgs must match the PA L0TaskArgs ABI size"); +static_assert(offsetof(TaskArgs, tags) == 0, "TaskArgs tag offset mismatch"); +static_assert(offsetof(TaskArgs, tensors) == 128, "TaskArgs tensor-ref offset mismatch"); +static_assert(offsetof(TaskArgs, scalars) == 640, "TaskArgs scalar offset mismatch"); +static_assert(offsetof(TaskArgs, tensor_count) == 768, "TaskArgs tensor-count offset mismatch"); +static_assert(offsetof(TaskArgs, scalar_count) == 772, "TaskArgs scalar-count offset mismatch"); +static_assert(offsetof(TaskArgs, has_error) == 776, "TaskArgs error flag offset mismatch"); +static_assert(offsetof(TaskArgs, error_msg) == 784, "TaskArgs error pointer offset mismatch"); +static_assert(offsetof(TaskArgs, launch_spec) == 792, "TaskArgs launch-spec offset mismatch"); +static_assert(offsetof(TaskArgs, dump_arg_selection) == 800, "TaskArgs dump-selection offset mismatch"); +static_assert(offsetof(TaskArgs, explicit_deps) == 960, "TaskArgs dependency pointer offset mismatch"); +static_assert(offsetof(TaskArgs, explicit_dep_count) == 968, "TaskArgs dependency count offset mismatch"); + +struct TaskOutputs { + uint64_t task_id; + uint32_t count; + PA_GM TensorDesc *tensors[kMaxTaskTensors]; +}; +static_assert(sizeof(TaskOutputs) == 272, "TaskOutputs must match the PA TaskOutputTensors ABI size"); +static_assert(offsetof(TaskOutputs, tensors) == 16, "TaskOutputs tensor pointer offset mismatch"); + +struct SubmitContext { + PA_GM WorkerState *self; + PA_GM TaskPayload *payload; + int32_t task_id; + int32_t tensor_count; + int32_t scalar_count; + uint32_t register_mask; + uint64_t output_bytes; + TaskOutputs result; + int32_t fanin[kMaxFanin]; + int32_t fanin_count; + int32_t kernel_id; + bool won; + bool joint; + bool joint_init; + int32_t joint_block; + int32_t joint_slot; + int32_t joint_count; +}; +// SubmitContext 贯穿一次 Submit：Begin 绑定 task/payload，Materialize 填充输出与 +// register_mask，winner 收集 fanin 并构建 slot。它复刻 DistSubmitCtx 而非诊断结构。 +static_assert(sizeof(SubmitContext) == 400, "SubmitContext must match DistSubmitCtx"); +static_assert(offsetof(SubmitContext, output_bytes) == 32, "SubmitContext output-byte offset mismatch"); +static_assert(offsetof(SubmitContext, result) == 40, "SubmitContext result offset mismatch"); +static_assert(offsetof(SubmitContext, fanin) == 312, "SubmitContext fanin offset mismatch"); + +struct OutputLayout { + uint64_t buffer_sizes[kMaxTaskTensors]; + uint64_t total_output_size; +}; +// 只有 tag=Output 的槽位拥有有效 buffer_sizes；总大小按 1 KiB 对齐累计，随后 +// 作为 HeapGuard 的 output_bytes 和本 worker heap_next 的推进量。 +static_assert(sizeof(OutputLayout) == 264, "OutputLayout must match DistOutputLayout"); + +// 该状态保存真实 PA orchestration 在五个 Submit 之间传递的 descriptor。输出指针 +// 指向每个 worker 自己 materialize 的 payload，不能跨 worker 共享或简化为全局对象。 +struct PaOrchestrationState { + TensorDesc query; + TensorDesc key_cache; + TensorDesc value_cache; + TensorDesc block_table; + TensorDesc context_lens; + TensorDesc output; + TensorDesc query_view; + TensorDesc output_view; + + TensorCreateInfo tile_create_info; + TensorCreateInfo scalar_create_info; + TensorCreateInfo qk_create_info; + TensorCreateInfo sf_create_info; + + // The pointer is supplied by the standalone backend. On A5 it must point + // at GM so every batch performs the same descriptor-based load as PA. + PA_GM const volatile int32_t *context_lens_data; + uint64_t scale_bits; + uint64_t current_sequence; + uint64_t current_blocks; + uint64_t current_block_offset; + uint64_t current_nblocks; + uint64_t current_valid_len; + uint32_t current_batch; + + PA_GM TensorDesc *accumulated_output; + PA_GM TensorDesc *accumulated_sum; + PA_GM TensorDesc *accumulated_max; + PA_GM TensorDesc *qk_scores; + PA_GM TensorDesc *sf_probs; + PA_GM TensorDesc *sf_max; + PA_GM TensorDesc *sf_sum; + PA_GM TensorDesc *pv_output; +}; + +PA_DEVICE uint64_t ElementSize(DataType dtype) { + // 输入 dtype 来自已通过 PA ABI 构造的 descriptor/create-info，必须落在 Count 前； + // 输出字节数同时用于外部 tensor range 与新 Output 的 heap 大小计算。 + constexpr static uint64_t sizes[static_cast(DataType::Count)] = { + 4, 2, 4, 2, 1, 1, 2, 8, 8, 2, 4, 1, + }; + return sizes[static_cast(dtype)]; +} + +PA_DEVICE int32_t TagValue(TensorArgType tag) { return static_cast(tag); } + +PA_DEVICE TensorArgType TaskTag(const TaskArgs &args, uint32_t index) { + // index 的有效范围由 tensor_count 保证；集中转换避免各阶段对 int32 ABI tag + // 做不同解释，Materialize/CollectFanin/Register 因而共享同一分类结果。 + return static_cast(args.tags[index]); +} + +PA_DEVICE void ClearDumpArgSelection(PaDumpArgSelection &selection) { + // Volatile stores intentionally preserve the profiling-enabled PA reset + // traffic even though the standalone winner workload never consumes dump data. + // volatile 的目的不是同步，而是阻止编译器删掉这段生产基线中存在的写流量。 + volatile uint64_t *masks = &selection.dump_arg_mask; + masks[0] = 0; + masks[1] = 0; + volatile uint64_t *sources = &selection.scalar_source_ptrs[0]; + for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { + sources[index] = 0; + } + volatile uint8_t *dtypes = &selection.scalar_dtypes[0]; + for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { + dtypes[index] = 0; + } +} + +PA_DEVICE void ConstructTaskArgs(TaskArgs &args) { + // TensorTagMixin::tags_{} is value-initialized by the + // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. + // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar + // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 + volatile int32_t *tags = &args.tags[0]; + for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { + tags[index] = 0; + } + args.tensor_count = 0; + args.scalar_count = 0; + args.has_error = false; + args.error_msg = 0; + args.launch_spec.core_num = 1; + args.launch_spec.require_sync_start = false; + ClearDumpArgSelection(args.dump_arg_selection); + args.explicit_deps = 0; + args.explicit_dep_count = 0; +} + +PA_DEVICE void ResetTaskArgs(TaskArgs &args) { + // reset 的输出是不含 tensor/scalar/显式依赖的新逻辑参数表，但保留已分配对象及 + // launch_spec；QK/SF/PV/UP 在同一个 1 KiB TaskArgs 上依次复用这一状态。 + args.tensor_count = 0; + args.scalar_count = 0; + ClearDumpArgSelection(args.dump_arg_selection); + args.explicit_deps = 0; + args.explicit_dep_count = 0; + args.has_error = false; + args.error_msg = 0; +} + +PA_DEVICE bool ReserveTensorArgs(TaskArgs &args, int32_t count) { + // tensor 必须先于 scalar 追加，以保持 dispatch args 的 [tensor..., scalar...] + // 排列；失败只置 has_error，不发生部分追加。 + if (args.scalar_count != 0 || count < 0 || + args.tensor_count + count > static_cast(kMaxTaskTensors)) { + args.has_error = true; + return false; + } + return true; +} + +PA_DEVICE void AppendLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { + const uint32_t index = static_cast(args.tensor_count++); + args.tensors[index].pointer.local_tensor = &tensor; + args.tensors[index].kind = TensorRefKind::LocalTensor; + args.tags[index] = TagValue(tag); +} + +PA_DEVICE void AppendGmTensor(TaskArgs &args, PA_GM const TensorDesc &tensor, TensorArgType tag) { + const uint32_t index = static_cast(args.tensor_count++); + args.tensors[index].pointer.gm_tensor = &tensor; + args.tensors[index].kind = TensorRefKind::GmTensor; + args.tags[index] = TagValue(tag); +} + +PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { + const uint32_t index = static_cast(args.tensor_count++); + args.tensors[index].pointer.create_info = &create_info; + args.tensors[index].kind = TensorRefKind::CreateInfo; + args.tags[index] = TagValue(TensorArgType::Output); +} + +PA_DEVICE bool ReserveScalarArgs(TaskArgs &args, int32_t count) { + // 先整体校验容量再由 AddTwo/AddThree 连续写入，保证多 scalar 操作全有或全无。 + if (count < 0 || args.scalar_count + count > static_cast(kMaxTaskScalars)) { + args.has_error = true; + return false; + } + return true; +} + +PA_DEVICE void AppendScalar(TaskArgs &args, uint64_t value) { + args.scalars[static_cast(args.scalar_count++)] = value; +} + +PA_DEVICE void AddLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { + if (ReserveTensorArgs(args, 1)) AppendLocalTensor(args, tensor, tag); +} + +PA_DEVICE void AddGmTensor(TaskArgs &args, PA_GM const TensorDesc &tensor, TensorArgType tag) { + if (ReserveTensorArgs(args, 1)) AppendGmTensor(args, tensor, tag); +} + +PA_DEVICE void AddOutput(TaskArgs &args, const TensorCreateInfo &create_info) { + if (ReserveTensorArgs(args, 1)) AppendOutput(args, create_info); +} + +PA_DEVICE void AddScalar(TaskArgs &args, uint64_t value) { + if (ReserveScalarArgs(args, 1)) AppendScalar(args, value); +} + +PA_DEVICE void AddTwoScalars(TaskArgs &args, uint64_t value0, uint64_t value1) { + if (!ReserveScalarArgs(args, 2)) return; + AppendScalar(args, value0); + AppendScalar(args, value1); +} + +PA_DEVICE void AddThreeScalars(TaskArgs &args, uint64_t value0, uint64_t value1, uint64_t value2) { + if (!ReserveScalarArgs(args, 3)) return; + AppendScalar(args, value0); + AppendScalar(args, value1); + AppendScalar(args, value2); +} + +PA_DEVICE void InitCreateInfo( + TensorCreateInfo &info, const uint32_t shapes[kMaxTensorDims], uint32_t ndims, DataType dtype +) { + info.initial_value = 0; + info.has_initial_value = false; + info.reserved0 = 0; + info.start_offset = 0; + info.version = 0; + info.ndims = ndims; + info.dtype = dtype; + info.manual_dep = false; + info.is_contiguous = true; + info.child_memory = 0; + // TensorCreateInfo's real constructor only writes active dimensions. + // 只写 ndims 个 shape，保留生产构造器的写入范围，不能为方便把五维全清零。 + for (uint32_t index = 0; index < ndims; ++index) { + info.shapes[index] = shapes[index]; + } +} + +PA_DEVICE void ClearCreateInfo(TensorCreateInfo &info) { + volatile uint8_t *bytes = reinterpret_cast(&info); + for (uint32_t index = 0; index < sizeof(TensorCreateInfo); ++index) { + bytes[index] = 0; + } +} + +PA_DEVICE void InitExternalTensor( + TensorDesc &tensor, uint64_t address, const uint32_t shapes[kMaxTensorDims], uint32_t ndims, DataType dtype, + bool manual_dep +) { + // 输入为稳定 backing address、逻辑 shape 和依赖属性；输出是完整连续 descriptor， + // owner 无效表示它不是本轮 task 产生，row-major stride 从末维向前计算。 + uint64_t elements = 1; + for (uint32_t index = 0; index < ndims; ++index) { + elements *= shapes[index]; + } + tensor.buffer_addr = address; + tensor.buffer_size = elements * ElementSize(dtype); + tensor.owner_task_id = kInvalidTaskId; + tensor.start_offset = 0; + tensor.version = 0; + tensor.ndims = ndims; + tensor.dtype = dtype; + tensor.manual_dep = manual_dep; + tensor.is_contiguous = true; + tensor.child_memory = 0; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + tensor.shapes[index] = shapes[index]; + tensor.strides[index] = 0; + } + uint32_t stride = 1; + for (int32_t index = static_cast(ndims) - 1; index >= 0; --index) { + tensor.strides[index] = stride; + stride *= tensor.shapes[index]; + } + tensor.extent_elem_cache = stride; +} + +PA_DEVICE bool InitTensorFromCreateInfo( + PA_GM TensorDesc &tensor, const TensorCreateInfo &info, uint64_t address, uint64_t buffer_size +) { + tensor.buffer_addr = address; + tensor.buffer_size = buffer_size; + tensor.owner_task_id = kInvalidTaskId; + tensor.start_offset = info.start_offset; + tensor.version = info.version; + tensor.ndims = info.ndims; + tensor.dtype = info.dtype; + tensor.manual_dep = info.manual_dep; + tensor.is_contiguous = info.is_contiguous; + tensor.child_memory = info.child_memory; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + tensor.shapes[index] = info.shapes[index]; + } + uint32_t stride = 1; + for (int32_t index = static_cast(tensor.ndims) - 1; index >= 0; --index) { + tensor.strides[index] = stride; + stride *= tensor.shapes[index]; + } + tensor.extent_elem_cache = stride; + // PA can initialize the backing allocation here. Case1 never requests it; + // the standalone uses synthetic heap addresses and therefore rejects that + // unsupported path instead of writing to a fabricated GM pointer. + // Case1 的 has_initial_value 恒为 false；返回 false 是对未模拟分支的 + // 明确保护，不会在合成地址上伪造初始化写入。 + return !info.has_initial_value; +} + +PA_DEVICE uint64_t CreateInfoBytes(const TensorCreateInfo &info) { + uint64_t elements = 1; + for (uint32_t index = 0; index < info.ndims; ++index) { + elements *= info.shapes[index]; + } + return elements * ElementSize(info.dtype); +} + +template +PA_DEVICE void CopyTensorLine1(TensorDesc &destination, const Source &source) { + // view 只需复制 descriptor 第一条 cache line 的身份/shape 字段，随后由调用方 + // 覆盖 offset、shape、stride 与 extent；不做整 128-byte 拷贝以匹配 PA 写流。 + destination.buffer_addr = source.buffer_addr; + destination.buffer_size = source.buffer_size; + destination.owner_task_id = source.owner_task_id; + destination.start_offset = source.start_offset; + destination.version = source.version; + destination.ndims = source.ndims; + destination.dtype = source.dtype; + destination.manual_dep = source.manual_dep; + destination.is_contiguous = source.is_contiguous; + destination.child_memory = source.child_memory; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + destination.shapes[index] = source.shapes[index]; + } +} + +PA_DEVICE void MakeBatchViews(PaOrchestrationState &orch, uint32_t batch) { + // query/output view 共享原 backing buffer，仅通过 start_offset 切出当前 batch。 + // output_view 保留真实 manual_dep 标记；UP 的生产者依赖由 Alloc、SF、PV 返回 descriptor 的 owner 闭合。 + CopyTensorLine1(orch.query_view, orch.query); + orch.query_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.query_view.ndims = 2; + orch.query_view.shapes[0] = kPaHeads; + orch.query_view.shapes[1] = kPaHeadDim; + orch.query_view.strides[0] = kPaHeadDim; + orch.query_view.strides[1] = 1; + orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; + + CopyTensorLine1(orch.output_view, orch.output); + orch.output_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.output_view.ndims = 2; + orch.output_view.manual_dep = true; + orch.output_view.shapes[0] = kPaHeads; + orch.output_view.shapes[1] = kPaHeadDim; + orch.output_view.strides[0] = kPaHeadDim; + orch.output_view.strides[1] = 1; + orch.output_view.extent_elem_cache = kPaHeads * kPaHeadDim; +} + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +// The two views remain final parameter-packing operations. query_view is a +// winner-only QK input; output_view is the produce side of a private INOUT and +// is therefore evaluated on every worker by the callback builder. +PA_DEVICE void MakeLazySampleCallbackOutputView(PaOrchestrationState &orch, uint32_t batch) { + CopyTensorLine1(orch.output_view, orch.output); + orch.output_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.output_view.ndims = 2; + orch.output_view.manual_dep = true; + orch.output_view.shapes[0] = kPaHeads; + orch.output_view.shapes[1] = kPaHeadDim; + orch.output_view.strides[0] = kPaHeadDim; + orch.output_view.strides[1] = 1; + orch.output_view.extent_elem_cache = kPaHeads * kPaHeadDim; +} + +PA_DEVICE void MakeLazySampleCallbackQueryView(PaOrchestrationState &orch, uint32_t batch) { + CopyTensorLine1(orch.query_view, orch.query); + orch.query_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.query_view.ndims = 2; + orch.query_view.shapes[0] = kPaHeads; + orch.query_view.shapes[1] = kPaHeadDim; + orch.query_view.strides[0] = kPaHeadDim; + orch.query_view.strides[1] = 1; + orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; +} +#endif + +PA_DEVICE uint64_t MinU64(uint64_t lhs, uint64_t rhs) { return lhs < rhs ? lhs : rhs; } + +PA_DEVICE uint64_t ReadPaContextLength(const PaOrchestrationState &orch, uint32_t batch) { + if (orch.context_lens_data == nullptr) { + // Compatibility fallback for a backend that has not yet supplied the + // 256-int GM buffer. Exact PA runs must pass a non-null pointer. + // 正式对等运行必须走下方 descriptor+stride 的 GM load；fallback + // 只用于不具备该缓冲区的兼容后端。 + return kPaBlocksPerRequest * kPaBlockSize; + } + const uint64_t flat_index = orch.context_lens.start_offset + + static_cast(batch) * orch.context_lens.strides[0]; + PA_GM const volatile int32_t *value = reinterpret_cast( + orch.context_lens.buffer_addr + flat_index * ElementSize(DataType::Int32) + ); + return static_cast(*value); +} + +PA_DEVICE void PreparePaBlockGroup(PaOrchestrationState &orch, uint64_t block_offset) { + // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 + // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 + orch.current_block_offset = block_offset; + orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); + const uint64_t last_block_sequence_start = + (block_offset + orch.current_nblocks - 1) * kPaBlockSize; + orch.current_valid_len = MinU64(kPaBlockSize, orch.current_sequence - last_block_sequence_start); +} + +PA_DEVICE void BeginPaBatch(PaOrchestrationState &orch, uint32_t batch) { + // Match paged_attention_orch.cpp: context GM load and block arithmetic + // happen before entering the q scope and before constructing either view. + // 该顺序会影响 Submit 前的指令与访存间隔，故不把长度读取挪进 QK 构参。 + orch.current_batch = batch; + orch.current_sequence = ReadPaContextLength(orch, batch); + orch.current_blocks = (orch.current_sequence + kPaBlockSize - 1) / kPaBlockSize; + MakeBatchViews(orch, batch); +} + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +PA_DEVICE void BeginPaBatchForLazySampleCallback(PaOrchestrationState &orch, uint32_t batch) { + // Tensor reads and arithmetic that feed more than one task stay on the + // common path. Descriptor construction is deferred to the task callback. + orch.current_batch = batch; + orch.current_sequence = ReadPaContextLength(orch, batch); + orch.current_blocks = (orch.current_sequence + kPaBlockSize - 1) / kPaBlockSize; +} +#endif + +PA_DEVICE void InitPaOrchestration( + PaOrchestrationState &orch, uint32_t batches, PA_GM const volatile int32_t *context_lens_data +) { + // 初始化只建立整轮回放共享的外部 descriptor/create-info 模板；每 batch 的 view、 + // context length、动态 QK/SF shape 和返回 descriptor 留给五阶段流按原顺序更新。 + const uint32_t query_shape[kMaxTensorDims] = {batches * kPaHeads, kPaHeadDim, 0, 0, 0}; + const uint32_t cache_shape[kMaxTensorDims] = { + batches * kPaBlocksPerRequest * kPaBlockSize, kPaHeadDim, 0, 0, 0 + }; + const uint32_t table_shape[kMaxTensorDims] = {batches, kPaMaxBlocksPerRequest, 0, 0, 0}; + const uint32_t context_shape[kMaxTensorDims] = {batches, 0, 0, 0, 0}; + InitExternalTensor(orch.query, kSyntheticQueryBase, query_shape, 2, DataType::Bfloat16, false); + InitExternalTensor(orch.key_cache, kSyntheticKeyBase, cache_shape, 2, DataType::Bfloat16, false); + InitExternalTensor(orch.value_cache, kSyntheticValueBase, cache_shape, 2, DataType::Bfloat16, false); + InitExternalTensor(orch.block_table, kSyntheticBlockTableBase, table_shape, 2, DataType::Int32, false); + const uint64_t context_address = context_lens_data == nullptr + ? kSyntheticContextLensBase + : reinterpret_cast(context_lens_data); + InitExternalTensor(orch.context_lens, context_address, context_shape, 1, DataType::Int32, false); + InitExternalTensor(orch.output, kSyntheticOutputBase, query_shape, 2, DataType::Float32, false); + + const uint32_t tile_shape[kMaxTensorDims] = {kPaHeads, kPaHeadDim, 0, 0, 0}; + const uint32_t scalar_shape[kMaxTensorDims] = {kPaHeads, 0, 0, 0, 0}; + ClearCreateInfo(orch.tile_create_info); + ClearCreateInfo(orch.scalar_create_info); + ClearCreateInfo(orch.qk_create_info); + ClearCreateInfo(orch.sf_create_info); + InitCreateInfo(orch.tile_create_info, tile_shape, 2, DataType::Float32); + InitCreateInfo(orch.scalar_create_info, scalar_shape, 1, DataType::Float32); + + // QK/SF create infos are deliberately not constructed here: in PA they are + // constructed inside the group after Alloc and QK respectively. + // 动态 shape 依赖当前 block group，提前构造既不符合业务数据流，也会 + // 把真实发生在两个 Submit 之间的前端工作错误搬到初始化阶段。 + orch.context_lens_data = context_lens_data; + orch.scale_bits = kPaScaleBits; + orch.current_sequence = 0; + orch.current_blocks = 0; + orch.current_block_offset = 0; + orch.current_nblocks = 0; + orch.current_valid_len = 0; + orch.current_batch = 0; + + orch.accumulated_output = nullptr; + orch.accumulated_sum = nullptr; + orch.accumulated_max = nullptr; + orch.qk_scores = nullptr; + orch.sf_probs = nullptr; + orch.sf_max = nullptr; + orch.sf_sum = nullptr; + orch.pv_output = nullptr; +} + +PA_DEVICE void InitPaOrchestration(PaOrchestrationState &orch, uint32_t batches) { + InitPaOrchestration(orch, batches, nullptr); +} + +PA_DEVICE void BuildAllocArgs(PaOrchestrationState &orch, TaskArgs &args, uint32_t batch) { + BeginPaBatch(orch, batch); + // PA constructs a fresh L0TaskArgs after its two views; Alloc is populated + // without calling reset(). + // 三个 Output 分别成为累计 output/sum/max；Alloc 无 kernel slot，winner 在 + // HeapGuard 后直接发布 task completion。 + ConstructTaskArgs(args); + if (!ReserveTensorArgs(args, 3)) return; + AppendOutput(args, orch.tile_create_info); + AppendOutput(args, orch.scalar_create_info); + AppendOutput(args, orch.scalar_create_info); +} + +PA_DEVICE void BuildQkArgs(PaOrchestrationState &orch, TaskArgs &args, uint32_t batch) { + (void)batch; + // PA computes the block group after Alloc returns, immediately before it + // constructs the dynamic QK output create-info. + // QK 消费 query/key/block-table，产出 score；其 active role 为 AIC。 + PreparePaBlockGroup(orch, 0); + const uint32_t score_shape[kMaxTensorDims] = { + kPaHeads, static_cast(orch.current_nblocks * kPaBlockSize), 0, 0, 0 + }; + // Constructed after Alloc submit and immediately before QK reset/adds. + InitCreateInfo(orch.qk_create_info, score_shape, 2, DataType::Float32); + ResetTaskArgs(args); + if (!ReserveTensorArgs(args, 3)) return; + AppendLocalTensor(args, orch.query_view, TensorArgType::Input); + AppendLocalTensor(args, orch.key_cache, TensorArgType::Input); + AppendLocalTensor(args, orch.block_table, TensorArgType::Input); + AddOutput(args, orch.qk_create_info); + AddTwoScalars( + args, orch.current_nblocks, + static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + orch.current_block_offset + ); +} + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +struct LazySampleCallbackBuildCounts { + uint32_t reset_calls; + uint32_t views_created; + uint32_t dynamic_create_infos; + uint32_t tensor_args_added; + uint32_t scalar_args_added; +}; + +template +class LazySampleCallbackArgsBuilder { +public: + PA_DEVICE LazySampleCallbackArgsBuilder(TaskArgs &args, TaskKind kind, bool won) + : args_(args), kind_(kind), won_(won), begin_calls_(0), counts_{} {} + + PA_DEVICE void Begin() { + if (++begin_calls_ != 1) { + args_.has_error = true; + return; + } + if (kind_ == TaskKind::Alloc) { + ConstructTaskArgs(args_); + } else { + ResetTaskArgs(args_); + ++counts_.reset_calls; + } + } + + PA_DEVICE void RecordView() { ++counts_.views_created; } + PA_DEVICE void RecordDynamicCreateInfo() { ++counts_.dynamic_create_infos; } + + template + PA_DEVICE void AddLocalInput(Thunk thunk) { + if constexpr (Lazy) { + if (!won_) return; + } + if (!Ready()) return; + const TensorDesc &tensor = thunk(); + AddLocalTensor(args_, tensor, TensorArgType::Input); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddGmInput(Thunk thunk) { + if constexpr (Lazy) { + if (!won_) return; + } + if (!Ready()) return; + PA_GM const TensorDesc &tensor = thunk(); + AddGmTensor(args_, tensor, TensorArgType::Input); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddOutput(Thunk thunk) { + if (!Ready()) return; + const TensorCreateInfo &create_info = thunk(); + pa_scheduler::AddOutput(args_, create_info); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddLocalInout(Thunk thunk) { + if (!Ready()) return; + const TensorDesc &tensor = thunk(); + AddLocalTensor(args_, tensor, TensorArgType::Inout); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddGmInout(Thunk thunk) { + if (!Ready()) return; + PA_GM const TensorDesc &tensor = thunk(); + AddGmTensor(args_, tensor, TensorArgType::Inout); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddScalar(Thunk thunk) { + if constexpr (Lazy) { + if (!won_) return; + } + if (!Ready()) return; + pa_scheduler::AddScalar(args_, thunk()); + if (!args_.has_error) ++counts_.scalar_args_added; + } + + PA_DEVICE bool Valid() const { return begin_calls_ == 1 && !args_.has_error; } + PA_DEVICE const LazySampleCallbackBuildCounts &Counts() const { return counts_; } + +private: + PA_DEVICE bool Ready() { + if (begin_calls_ == 1 && !args_.has_error) return true; + args_.has_error = true; + return false; + } + + TaskArgs &args_; + TaskKind kind_; + bool won_; + uint32_t begin_calls_; + LazySampleCallbackBuildCounts counts_; +}; +#endif + +PA_DEVICE void BuildSfArgs(PaOrchestrationState &orch, TaskArgs &args) { + const uint32_t probability_shape[kMaxTensorDims] = { + kPaHeads, static_cast(orch.current_nblocks * kPaBlockSize), 0, 0, 0 + }; + // Constructed only after QK submit returns its sij descriptor. + // SF 通过 qk_scores.owner 得到 QK fanin，产出 probability/max/sum；active role 为 AIV。 + InitCreateInfo(orch.sf_create_info, probability_shape, 2, DataType::Bfloat16); + ResetTaskArgs(args); + AddGmTensor(args, *orch.qk_scores, TensorArgType::Input); + if (!ReserveTensorArgs(args, 3)) return; + AppendOutput(args, orch.sf_create_info); + AppendOutput(args, orch.scalar_create_info); + AppendOutput(args, orch.scalar_create_info); + AddThreeScalars(args, orch.scale_bits, orch.current_nblocks, orch.current_valid_len); +} + +PA_DEVICE void BuildPvArgs(PaOrchestrationState &orch, TaskArgs &args, uint32_t batch) { + (void)batch; + // PV 消费 SF probability 与 value/block-table，owner 形成一条 SF->PV 依赖； + // 结果 pv_output 供最后的 UP 使用，active role 回到 AIC。 + ResetTaskArgs(args); + if (!ReserveTensorArgs(args, 3)) return; + AppendGmTensor(args, *orch.sf_probs, TensorArgType::Input); + AppendLocalTensor(args, orch.value_cache, TensorArgType::Input); + AppendLocalTensor(args, orch.block_table, TensorArgType::Input); + AddOutput(args, orch.tile_create_info); + AddTwoScalars( + args, orch.current_nblocks, + static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + orch.current_block_offset + ); +} + +PA_DEVICE void BuildUpdateArgs(PaOrchestrationState &orch, TaskArgs &args) { + // UP 的 SF max/sum 共享一个 SF owner，PV output 提供一个 PV owner，三个累计 + // Inout 共享 Alloc owner，去重后共 3 条 fanin；output_view 把更新写回当前 batch。 + ResetTaskArgs(args); + if (!ReserveTensorArgs(args, 3)) return; + AppendGmTensor(args, *orch.sf_max, TensorArgType::Input); + AppendGmTensor(args, *orch.sf_sum, TensorArgType::Input); + AppendGmTensor(args, *orch.pv_output, TensorArgType::Input); + if (!ReserveTensorArgs(args, 4)) return; + AppendGmTensor(args, *orch.accumulated_max, TensorArgType::Inout); + AppendGmTensor(args, *orch.accumulated_sum, TensorArgType::Inout); + AppendGmTensor(args, *orch.accumulated_output, TensorArgType::Inout); + AppendLocalTensor(args, orch.output_view, TensorArgType::Inout); + AddTwoScalars( + args, orch.current_block_offset == 0 ? 1 : 0, + orch.current_block_offset + orch.current_nblocks >= orch.current_blocks ? 1 : 0 + ); +} + +PA_DEVICE void AcceptTaskOutputs(PaOrchestrationState &orch, TaskKind kind, const TaskOutputs &outputs) { + // 每个 worker 都完整回放并接收自己 materialize 的 descriptor；只有 Claim winner + // 会执行 kernel，但 loser 的后续 orchestration 仍使用相同 task_id/owner 拓扑。 + switch (kind) { + case TaskKind::Alloc: + orch.accumulated_output = outputs.tensors[0]; + orch.accumulated_sum = outputs.tensors[1]; + orch.accumulated_max = outputs.tensors[2]; + break; + case TaskKind::Qk: + orch.qk_scores = outputs.tensors[0]; + break; + case TaskKind::Sf: + orch.sf_probs = outputs.tensors[0]; + orch.sf_max = outputs.tensors[1]; + orch.sf_sum = outputs.tensors[2]; + break; + case TaskKind::Pv: + orch.pv_output = outputs.tensors[0]; + break; + default: + // UP 只更新既有 Inout，没有新 Output descriptor 需要传给下一阶段。 + break; + } +} + +PA_DEVICE void ResetTensorMap(PA_GM TensorMap &map) { + // TensorMap 完全属于当前 worker，初始化和后续增删都不需要 atomic。bucket 与 + // task_heads 置空后，entry 存储按 high_water 首次分配、再经 free_head 复用。 + map.free_head = -1; + map.high_water = 0; + map.alive_floor = 0; + map.cleaned_upto = 0; + for (uint32_t index = 0; index < kMapBuckets; ++index) { + map.buckets[index] = -1; + } + for (uint32_t index = 0; index < kTaskWindow; ++index) { + map.task_heads[index] = -1; + } +} + +PA_DEVICE uint32_t TensorMapHash(uint64_t address) { + address *= 0x9E3779B97F4A7C15ULL; + return static_cast(address >> (64 - kMapBucketShift)); +} + +template +PA_DEVICE void TensorByteRange(const TensorReference &tensor, uint64_t &address, uint64_t &lo, uint64_t &hi) { + // identity 先按 backing buffer 地址分桶，再用半开字节区间 [lo, hi) 判断 view + // 是否重叠。连续 tensor 由 shape 现算 extent，非连续 tensor 使用缓存 extent。 + const uint64_t element_size = ElementSize(tensor.dtype); + address = tensor.buffer_addr; + lo = tensor.start_offset * element_size; + uint64_t extent; + if (tensor.is_contiguous) { + extent = 1; + for (uint32_t index = 0; index < tensor.ndims; ++index) { + extent *= tensor.shapes[index]; + } + } else { + extent = tensor.extent_elem_cache; + } + hi = (tensor.start_offset + extent) * element_size; +} + +PA_DEVICE int32_t AllocateMapEntry(PA_GM TensorMap &map) { + // 输出为可写 entry 下标：优先复用退休链，之后增长 high_water；返回 -1 表示 + // 固定容量耗尽。分配动作尚未把 entry 接入任何 bucket/task 链。 + if (map.free_head >= 0) { + const int32_t slot = map.free_head; + map.free_head = map.entries[slot].next_in_bucket; + return slot; + } + if (map.high_water < static_cast(kMapCapacity)) { + return map.high_water++; + } + return -1; +} + +PA_DEVICE void FreeMapEntry(PA_GM TensorMap &map, int32_t index) { + // 输入 index 必须仍位于其 bucket 链。输出状态是从双向 bucket 链完整摘除， + // 再把 next_in_bucket 改作 free-list next；task 链由 AdvanceTensorMap 顺序遍历。 + PA_GM MapEntry &entry = map.entries[index]; + if (entry.prev_in_bucket < 0) { + map.buckets[entry.bucket] = entry.next_in_bucket; + } else { + map.entries[entry.prev_in_bucket].next_in_bucket = entry.next_in_bucket; + } + if (entry.next_in_bucket >= 0) { + map.entries[entry.next_in_bucket].prev_in_bucket = entry.prev_in_bucket; + } + entry.bucket = -1; + entry.next_in_bucket = map.free_head; + map.free_head = index; +} + +PA_DEVICE void AdvanceTensorMap(PA_GM TensorMap &map, uint32_t task_id, int32_t heap_window) { + // PrepareMap 在 Claim 前把存活下界推进到 task_id-H。离开窗口的 producer 先按 + // task_heads 找到其全部 entry，再从 bucket 链摘除并进入 free list。TensorMap 与 + // heap 共享窗口宽度 H，但前者按本 worker 的 task_id 推进，后者按跨核连续 + // frontier 推进，二者并不要求同步到达同一位置。 + const int32_t new_floor = static_cast(task_id) - heap_window; + if (new_floor <= map.cleaned_upto) { + if (new_floor > map.alive_floor) { + map.alive_floor = new_floor; + } + return; + } + for (int32_t id = map.cleaned_upto; id < new_floor; ++id) { + int32_t current = map.task_heads[static_cast(id) & kTaskWindowMask]; + while (current >= 0) { + const int32_t next = map.entries[current].next_in_task; + FreeMapEntry(map, current); + current = next; + } + map.task_heads[static_cast(id) & kTaskWindowMask] = -1; + } + map.cleaned_upto = new_floor; + map.alive_floor = new_floor; +} + +template +PA_DEVICE void InsertTensor(PA_GM TensorMap &map, const TensorReference &tensor, int32_t producer) { + // 新 producer 同时插入地址 bucket 的表头与 producer 对应 task 链的表头。 + // map 满时生产语义是静默放弃登记；standalone 保持该行为，不新增异常分支。 + uint64_t address = 0; + uint64_t lo = 0; + uint64_t hi = 0; + TensorByteRange(tensor, address, lo, hi); + const int32_t slot = AllocateMapEntry(map); + if (slot < 0) { + return; + } + const uint32_t bucket = TensorMapHash(address); + PA_GM MapEntry &entry = map.entries[slot]; + entry.buffer_addr = address; + entry.lo = lo; + entry.hi = hi; + entry.producer = producer; + entry.bucket = static_cast(bucket); + entry.prev_in_bucket = -1; + entry.next_in_bucket = map.buckets[bucket]; + if (map.buckets[bucket] >= 0) { + map.entries[map.buckets[bucket]].prev_in_bucket = slot; + } + map.buckets[bucket] = slot; + const uint32_t task_slot = static_cast(producer) & kTaskWindowMask; + entry.next_in_task = map.task_heads[task_slot]; + map.task_heads[task_slot] = slot; +} + +template +PA_DEVICE int32_t LookupTensor(PA_GM const TensorMap &map, const TensorReference &tensor) { + uint64_t address = 0; + uint64_t lo = 0; + uint64_t hi = 0; + TensorByteRange(tensor, address, lo, hi); + int32_t best = -1; + // 同一 buffer 可能存在多个历史写者；只考虑仍存活且区间重叠的 entry，并选择 + // task_id 最大的最新 producer，防止依赖回退到更老版本。 + for (int32_t current = map.buckets[TensorMapHash(address)]; current >= 0; + current = map.entries[current].next_in_bucket) { + PA_GM const MapEntry &entry = map.entries[current]; + if (entry.producer < map.alive_floor) { + continue; + } + if (entry.buffer_addr == address && lo < entry.hi && entry.lo < hi && entry.producer > best) { + best = entry.producer; + } + } + return best; +} + +PA_DEVICE uint64_t TensorOwner(const TaskTensorRef &reference) { + // CreateInfo 只会出现在 tag=Output 且在 fanin 前已被跳过；这里的输入不变量是 + // LocalTensor/GmTensor，输出为显式 owner 或 kInvalidTaskId。 + if (reference.kind == TensorRefKind::GmTensor) { + return reference.pointer.gm_tensor->owner_task_id; + } + return reference.pointer.local_tensor->owner_task_id; +} + +PA_DEVICE int32_t LookupTensorRef(PA_GM const TensorMap &map, const TaskTensorRef &reference) { + // 与 TensorOwner 相同，此辅助入口只接收已存在 descriptor；返回最新重叠 producer， + // 未登记或已退休则返回 -1。 + if (reference.kind == TensorRefKind::GmTensor) { + return LookupTensor(map, *reference.pointer.gm_tensor); + } + return LookupTensor(map, *reference.pointer.local_tensor); +} + +PA_DEVICE void AddFanin(int32_t fanin[kMaxFanin], uint32_t &count, int32_t producer) { + // owner 与 TensorMap lookup 可能指向同一 producer，先去重再写固定 16 槽数组； + // Case1 的最大 fanin 为 UP 的 3，正常路径不会截断。 + if (producer < 0) { + return; + } + for (uint32_t index = 0; index < count; ++index) { + if (fanin[index] == producer) { + return; + } + } + if (count < kMaxFanin) { + fanin[count++] = producer; + } +} + +PA_DEVICE uint32_t CollectFanin( + PA_GM const TensorMap &map, const TaskArgs &args, int32_t fanin[kMaxFanin] +) { + // fanin 只由 winner 收集：先吸收 descriptor 的显式 owner，再对 Input/Inout + // 查询最新重叠写者；纯 Output 尚未存在，不应成为本次 task 的输入依赖。 + uint32_t count = 0; + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = TaskTag(args, static_cast(index)); + if (tag == TensorArgType::Output) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + // Keep the two address spaces in separate control-flow arms. CCEC's + // O2/O3 backend rejects a merged pointer phi even when both arms only + // feed scalar field loads; this is also how PA's production helper is + // written. + // 分支重复是后端约束与生产写法的一部分，不应抽成一个混合地址空间指针。 + if (reference.kind == TensorRefKind::GmTensor) { + PA_GM const TensorDesc &tensor = *reference.pointer.gm_tensor; + const uint64_t owner = tensor.owner_task_id; + if (owner != kInvalidTaskId) { + AddFanin(fanin, count, static_cast(owner & 0xFFFFFFFFU)); + } + if (tag == TensorArgType::Input || tag == TensorArgType::Inout) { + AddFanin(fanin, count, LookupTensor(map, tensor)); + } + } else { + const TensorDesc &tensor = *reference.pointer.local_tensor; + const uint64_t owner = tensor.owner_task_id; + if (owner != kInvalidTaskId) { + AddFanin(fanin, count, static_cast(owner & 0xFFFFFFFFU)); + } + if (tag == TensorArgType::Input || tag == TensorArgType::Inout) { + AddFanin(fanin, count, LookupTensor(map, tensor)); + } + } + } + return count; +} + +PA_DEVICE void InsertExistingTensor(SubmitContext &context, const TaskArgs &args, int32_t index) { + // 输入 index 来自 register_mask，故必为已有 descriptor 而非 CreateInfo；写入结果 + // 只影响 context.self 对应 worker 的 map，并把当前 task_id 登记为新的 hazard 版本。 + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind == TensorRefKind::GmTensor) { + InsertTensor(context.self->map, *reference.pointer.gm_tensor, context.task_id); + } else { + InsertTensor(context.self->map, *reference.pointer.local_tensor, context.task_id); + } +} + +PA_DEVICE void RegisterOutputs(SubmitContext &context, const TaskArgs &args, bool include_existing) { + // register_mask 只覆盖 Inout/OutputExisting。新 Output 已带本次 owner；现有 + // backing buffer 的新写者则必须登记到本 worker TensorMap，供后继 task 查 hazard。 + if (!include_existing) { + return; + } + uint32_t register_mask = context.register_mask; + for (uint32_t index = 0; register_mask != 0; ++index, register_mask >>= 1) { + if ((register_mask & 1U) != 0) { + InsertExistingTensor(context, args, static_cast(index)); + } + } +} + +PA_DEVICE uint64_t FrontendAlignUp(uint64_t value, uint64_t alignment) { + // alignment 在本模型中固定为2的幂1 KiB；返回逻辑 heap 地址，不做 ring 取模。 + return (value + alignment - 1) & ~(alignment - 1); +} + +PA_DEVICE void BeginSubmit( + PA_GM WorkerState &worker, const TaskArgs &args, SubmitContext &context +) { + // Mirrors dist_submit_begin(). The production Submit and Materialize spans + // both start after this per-call context initialization. + // local_index 在所有 worker 上按同一 orchestration 顺序递增，因此 + // task_id 一致；该初始化位于 Submit 计时起点之前，不能误计进阶段耗时。 + const uint32_t task_id = static_cast(worker.local_index++); + context.self = &worker; + context.payload = &worker.payloads[task_id & kPayloadMask]; + context.task_id = static_cast(task_id); + context.tensor_count = args.tensor_count; + context.scalar_count = args.scalar_count; + context.result.task_id = task_id; + context.result.count = 0; + context.register_mask = 0; + context.output_bytes = 0; + context.fanin_count = 0; + context.kernel_id = -1; + context.won = false; + context.joint = false; + context.joint_init = false; + context.joint_block = -1; + context.joint_slot = -1; + context.joint_count = 0; +} + +PA_DEVICE bool MaterializeTask( + PA_GM WorkerState &worker, uint32_t task_id, const TaskArgs &args, SubmitContext &context, + uint64_t heap_base, uint64_t heap_size +) { + // 输入是 BeginSubmit 已绑定的 payload/context 与当前 worker.heap_next；成功输出 + // 包括本 task 的 GM TensorDesc 指针、output_bytes 和推进后的单调 heap_next。 + // 失败不得进入 slot/build 流程，由上层设置 fatal 并终止该 worker 回放。 + // legacy 路径在 Claim 前物化；claim-first callback 实验则可能已经完成 Claim， + // 因而不能把“尚未 Claim”写成这个共用 helper 的普遍前置条件。 + if (context.payload == nullptr) { + return false; + } + context.tensor_count = args.tensor_count; + context.scalar_count = args.scalar_count; + context.register_mask = 0; + + // DistOutputLayout leaves non-output slots lazy and writes only the sizes + // selected by output_mask. + // 第一次 tag 扫描同时产生 output_mask/register_mask；第二次只遍历 + // Output 位，避免读取未初始化的非输出 buffer_sizes。 + OutputLayout layout; + layout.total_output_size = 0; + uint32_t output_mask = 0; + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = TaskTag(args, static_cast(index)); + if (tag == TensorArgType::Inout || tag == TensorArgType::OutputExisting) { + context.register_mask |= 1U << index; + } + if (tag != TensorArgType::Output) { + continue; + } + output_mask |= 1U << index; + layout.buffer_sizes[index] = CreateInfoBytes(*args.tensors[index].pointer.create_info); + layout.total_output_size += FrontendAlignUp(layout.buffer_sizes[index], kOutputAlignment); + } + + uint64_t task_base = FrontendAlignUp(worker.heap_next, kOutputAlignment); + const uint64_t total = layout.total_output_size; + if (total > heap_size || (total != 0 && heap_base == 0)) { + return false; + } + if (total != 0 && (task_base % heap_size) + total > heap_size) { + // 单个 task 的输出必须物理连续；若跨 ring 尾部则把逻辑 task_base 推到 + // 下一圈起点。heap_next 仍保持单调，不在这里取模。 + task_base = (task_base / heap_size + 1) * heap_size; + } + + uint64_t output_offset = 0; + // 各 Output 在同一 task_base 内按参数顺序排布；result 只收集 Output，索引与 + // TaskArgs 中非输出槽无关，而 payload 仍按原参数 index 保存 descriptor。 + for (int32_t index = 0; output_mask != 0; ++index, output_mask >>= 1) { + if ((output_mask & 1U) == 0) { + continue; + } + const uint64_t physical = (task_base + output_offset) % heap_size; + PA_GM TensorDesc &tensor = context.payload->tensors[index]; + if (!InitTensorFromCreateInfo( + tensor, *args.tensors[index].pointer.create_info, heap_base + physical, layout.buffer_sizes[index] + )) { + return false; + } + tensor.owner_task_id = task_id; + context.result.tensors[context.result.count++] = &tensor; + output_offset += FrontendAlignUp(layout.buffer_sizes[index], kOutputAlignment); + } + worker.heap_next = task_base + total; + context.output_bytes = total; + return true; +} + +PA_DEVICE void CopyTensorFromRef(PA_GM TensorDesc &destination, const TaskTensorRef &reference) { + // slot 必须拥有 descriptor 快照，不能保存指向 orchestration 栈对象的引用； + // 按 byte volatile copy 同时兼容 local/GM 源并保留真实 128-byte 搬运量。 + PA_GM volatile uint8_t *destination_bytes = reinterpret_cast(&destination); + if (reference.kind == TensorRefKind::GmTensor) { + PA_GM const volatile uint8_t *source_bytes = + reinterpret_cast(reference.pointer.gm_tensor); + for (uint32_t byte = 0; byte < sizeof(TensorDesc); ++byte) { + destination_bytes[byte] = source_bytes[byte]; + } + return; + } + const volatile uint8_t *source_bytes = + reinterpret_cast(reference.pointer.local_tensor); + for (uint32_t byte = 0; byte < sizeof(TensorDesc); ++byte) { + destination_bytes[byte] = source_bytes[byte]; + } +} + +PA_DEVICE void CopyGmTensor(PA_GM TensorDesc &destination, PA_GM const TensorDesc &source) { + // 新 Output 的源 descriptor 已位于 GM payload；单独入口避免把 GM 指针误走 + // local 地址空间分支，输出仍是 slot 内独立副本。 + PA_GM volatile uint8_t *destination_bytes = reinterpret_cast(&destination); + PA_GM const volatile uint8_t *source_bytes = reinterpret_cast(&source); + for (uint32_t byte = 0; byte < sizeof(TensorDesc); ++byte) { + destination_bytes[byte] = source_bytes[byte]; + } +} + +PA_DEVICE void PopulateSlotPayload( + PA_GM LocalSlot &slot, const TaskArgs &args, const SubmitContext &context, const int32_t fanin[kMaxFanin], + uint32_t fanin_count, int32_t sub_block_id, bool is_multicore, int32_t won_block, int32_t won_slot +) { + // winner 将活动 descriptor/scalar 复制进私有 slot，dispatch args 指向 slot 内 + // 副本而非 orchestration 临时对象；fanin 随 slot 保存，kernel 执行前逐 flag 检查。 + slot.tensor_count = context.tensor_count; + slot.scalar_count = context.scalar_count; + for (int32_t index = 0; index < context.tensor_count; ++index) { + if (TaskTag(args, static_cast(index)) == TensorArgType::Output) { + CopyGmTensor(slot.tensors[index], context.payload->tensors[index]); + } else { + CopyTensorFromRef(slot.tensors[index], args.tensors[index]); + } + slot.args[index] = static_cast(reinterpret_cast(&slot.tensors[index])); + } + for (int32_t index = 0; index < context.scalar_count; ++index) { + slot.scalars[index] = args.scalars[index]; + slot.args[context.tensor_count + index] = args.scalars[index]; + } + + PA_GM PaLocalContext &local = + *reinterpret_cast(&slot.local_context[0]); + // standalone 每个 task 只由一个 lane kernel 执行，故 block_index/count 固定0/1； + // async completion 未启用，task_token 保持 invalid，与 PA 普通同步 slot 一致。 + local.block_index = 0; + local.block_count = 1; + local.async.completion_count = 0; + local.async.completion_error_code = 0; + local.async.completion_entries = 0; + local.async.completion_capacity = 0; + local.async.task_token = kInvalidTaskId; + slot.global_context = static_cast(sub_block_id); + slot.args[kSpmdLocalContextIndex] = + static_cast(reinterpret_cast(&slot.local_context[0])); + slot.args[kSpmdGlobalContextIndex] = + static_cast(reinterpret_cast(&slot.global_context)); + slot.fanin_count = fanin_count; + // fanin 数组只复制有效前缀；执行端以 fanin_count 为边界，未使用尾部保持惰性。 + for (uint32_t index = 0; index < fanin_count; ++index) { + slot.fanin[index] = fanin[index]; + } + slot.is_multicore = is_multicore; + slot.won_block = won_block; + slot.won_slot = won_slot; +} + +PA_DEVICE void BuildSlotPayload( + PA_GM LocalSlot &slot, uint32_t task_id, uint32_t function_id, uint64_t function_address, const TaskArgs &args, + const SubmitContext &context, const int32_t fanin[kMaxFanin], uint32_t fanin_count, int32_t sub_block_id = 0, + bool is_multicore = false, int32_t won_block = -1, int32_t won_slot = -1 +) { + // Match build_ring_slot_from_submit ordering: publish the header first, + // then copy the active descriptors/scalars and construct dispatch payload. + // slot 仅由所属 worker 消费，这里的写入次序用于复刻真实构建成本与 + // 状态机；跨核可见性由 task completion 的 flag/vend 协议承担。 + slot.occupied = true; + slot.task_id = task_id; + slot.kind = function_id; + slot.function_address = function_address; + slot.built = 1; + PopulateSlotPayload( + slot, args, context, fanin, fanin_count, sub_block_id, is_multicore, won_block, won_slot + ); +} + +// Compatibility overload for a core that has already populated the slot +// header before calling the PA frontend. +// 该入口只补 payload，不改变既有 task/function 头；输出不变量与完整 +// BuildSlotPayload 相同，均得到 built 且可由 DrainReady 检查 fanin 的私有 slot。 +PA_DEVICE void BuildSlotPayload( + PA_GM LocalSlot &slot, const TaskArgs &args, const SubmitContext &context, const int32_t fanin[kMaxFanin], + uint32_t fanin_count +) { + slot.built = 1; + PopulateSlotPayload(slot, args, context, fanin, fanin_count, 0, false, -1, -1); +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_FRONTEND_H diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/pa_model.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/pa_model.h new file mode 100644 index 0000000000..0c74aa2607 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/pa_model.h @@ -0,0 +1,941 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_MODEL_H +#define PA_SCHEDULER_COMMON_PA_MODEL_H + +#include +#include + +// CCEC 的正式产物只允许二选一：swimlane 保存普通阶段与 atomic 记录， +// submit-pmu 则编译掉泳道观察代码。CPU/AscendC 未传这些宏时继续使用原有 +// 通用实现，避免公共模型反向依赖某个后端的构建脚本。 +#ifndef PA_BUILD_SWIMLANE +#define PA_BUILD_SWIMLANE 0 +#endif + +#ifndef PA_BUILD_SUBMIT_PMU +#define PA_BUILD_SUBMIT_PMU 0 +#endif + +#if PA_BUILD_SWIMLANE && PA_BUILD_SUBMIT_PMU +#error "PA_BUILD_SWIMLANE and PA_BUILD_SUBMIT_PMU are mutually exclusive" +#endif + +namespace pa_scheduler { + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +// The original artifact is compiled without this selector. The two selected +// artifacts deliberately share one compete-first/split-finish skeleton; only +// the builder's loser replay policy differs. +static_assert( + PA_LAZY_SAMPLE_SHAPE_ID == 1 || PA_LAZY_SAMPLE_SHAPE_ID == 2, + "PA_LAZY_SAMPLE_SHAPE_ID must select compete-first(1) or compete-first-lazy(2)" +); +constexpr bool kLazySampleLazy = PA_LAZY_SAMPLE_SHAPE_ID == 2; +constexpr bool kLazySampleSplitFinish = true; +constexpr uint32_t kLazySampleShapeId = PA_LAZY_SAMPLE_SHAPE_ID; +constexpr const char *kLazySampleShapeName = + PA_LAZY_SAMPLE_SHAPE_ID == 1 ? "compete-first" : "compete-first-lazy"; +constexpr const char *kLazySampleObservation = "split-combination-semantic"; +constexpr const char *kLazySampleFinishShape = "noinline-cross-tu"; +constexpr const char *kLazySampleControlFamily = "all-task-compete-first-callback"; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +static_assert(PA_LAZY_SAMPLE_SPLIT_FINISH == 1, "split-finish feature macro must equal one"); +static_assert(kLazySampleSplitFinish, "both lazy-sample callback shapes require split finish"); +#else +#error "lazy-sample callback artifacts require PA_LAZY_SAMPLE_SPLIT_FINISH" +#endif +#endif + +// 这里固定的是 PA Case1 的调度拓扑，而不是为了缩小 standalone 人为选择的规模： +// 每个 batch 依次回放 Alloc/QK/SF/PV/UP 五个 task，32 个 AIC 与 64 个 AIV +// 都执行同一条 orchestration 流，只在 Claim 时按 task 的 active role 分流。 +constexpr uint32_t kDefaultBatches = 256; +constexpr uint32_t kMaxBatches = 256; +constexpr uint32_t kTasksPerBatch = 5; +constexpr uint32_t kMaxTasks = kMaxBatches * kTasksPerBatch; +constexpr uint32_t kTaskCellCapacity = 1U << 16; + +constexpr uint32_t kAicWorkers = 32; +constexpr uint32_t kAivWorkers = 64; +constexpr uint32_t kWorkers = kAicWorkers + kAivWorkers; +constexpr uint32_t kRuntimeMaxWorkers = 108; +constexpr uint32_t kCursorShards = 4; +// 每个 worker 私有 ring 有 4 个物理 slot，其中 2 个为 BlockWon 协议预留； +// 单 lane Case1 虽不进入 BlockWon，普通 kernel 仍只能占用剩余 2 个 slot。 +constexpr uint32_t kPrivateSlots = 4; +constexpr uint32_t kWonReserve = 2; +constexpr uint32_t kUsableSlots = kPrivateSlots - kWonReserve; +constexpr uint32_t kMaxFanin = 16; +// H=64 同时约束 heap 可回收 frontier 和 TensorMap producer 的存活下界。 +// heap_next 使用单调逻辑地址；真正落到 256 MiB 环形 heap 时才取模，因而可判断覆盖风险。 +constexpr uint32_t kHeapWindow = 64; +constexpr uint64_t kHeapBytes = 256ULL << 20; +constexpr uint64_t kSyntheticHeapBase = 0x100000000ULL; +constexpr uint64_t kOutputAlignment = 1024; +constexpr uint32_t kMaxTensorDims = 5; +constexpr uint32_t kMaxTaskTensors = 32; +constexpr uint32_t kMaxTaskScalars = 16; +constexpr uint32_t kPayloadSlots = 2048; +constexpr uint32_t kPayloadMask = kPayloadSlots - 1; +constexpr uint32_t kPayloadStride = 4096; +constexpr uint32_t kMapCapacity = 16384; +constexpr uint32_t kMapBuckets = 1 << 13; +constexpr uint32_t kMapBucketShift = 13; +constexpr uint32_t kTaskWindow = 1 << 10; +constexpr uint32_t kTaskWindowMask = kTaskWindow - 1; +constexpr uint64_t kSystemCounterHz = 1000000000ULL; +constexpr uint64_t kWatchdogTicks = 2 * kSystemCounterHz; +// trace_enabled 是位图而不是 bool：bit0 保持既有阶段泳道，bit1 额外开启 +// 逐条 atomic 源码括号记录。atomic 记录依赖同一份 trace buffer，因此 bit1 +// 只能与 bit0 一起配置。 +constexpr uint32_t kTracePhasesEnabled = 1U << 0; +constexpr uint32_t kTraceAtomicsEnabled = 1U << 1; +// Claim trace flags 是独立 raw ABI：bit0 表示获胜，bit1 表示已经通过 +// AIC/AIV role 路由并真正执行 atomicMax。未 attempted 的 Claim 仍保留 +// role-selection 开销，但转换器会明确标成 claim.not_attempted。 +constexpr uint32_t kClaimWon = 1U << 0; +constexpr uint32_t kClaimAttempted = 1U << 1; +// 下列 offset/size 来自真实 DistGlobal/DistCore ABI。standalone 保留被测关键字段的 +// offset、DistCore ABI 和 kRealDistGlobalBytes 总跨度；其余区域可用 opaque padding， +// 并不是对生产结构全部字段的逐一镜像。 +constexpr size_t kRealDistCoreOffset = 10043904; +constexpr size_t kRealDistGlobalBytes = 1007023872; +constexpr size_t kRealTasksOffset = 896; +constexpr size_t kRealFatalOffset = 4195264; +constexpr size_t kRealReplayDoneOffset = 10043776; +constexpr size_t kRealStartedCountOffset = 10043840; +constexpr uint32_t kTraceRecordsPerCore = 1U << 16; +static_assert((kPayloadSlots & kPayloadMask) == 0, "payload slots must be a power of two"); +static_assert(kMaxTasks < kTaskCellCapacity, "every frontier scan must terminate on an in-range not-ready flag"); + +// These are the measured means from the best PA A5 trace, in 1 GHz ticks. +// The scalar-NOP compatibility baseline calibrates its counts against these targets. +// 无参数默认使用 real-compute：CCEC/AscendC 执行完整 Cube/Vector 流水， +// CPU 执行对等算术；下列 NOP 常量只供显式 scalar-nop 校准。两种模式的 +// Submit、依赖、heap 与 completion 路径都不靠补时修改。target 是真实泳道 +// 均值，不是调度阶段预算。 +constexpr uint32_t kTargetQkTicks = 44170; +constexpr uint32_t kTargetSfTicks = 53729; +constexpr uint32_t kTargetPvTicks = 27626; +constexpr uint32_t kTargetUpTicks = 1565; + +// Calibrated on the local A5 with the CCEC RuntimeNop implementation. These +// counts resolve to the measured targets above; they are not cycle guesses. +constexpr uint32_t kDefaultQkNops = 129600; +constexpr uint32_t kDefaultSfNops = 157900; +constexpr uint32_t kDefaultPvNops = 79950; +constexpr uint32_t kDefaultUpNops = 2400; + +enum class CoreRole : uint32_t { + Aic = 0, + Aiv = 1, +}; + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +constexpr uint64_t kLazySampleSplitStateCookieBase = 0x514b53504c495400ULL; +#endif + +// task_id % 5 即 kind；该周期性是不变量，既决定 Claim cursor/active role， +// 也决定输出大小、fanin 拓扑和 winner workload 的选择。 +enum class TaskKind : uint32_t { + Alloc = 0, + Qk = 1, + Sf = 2, + Pv = 3, + Up = 4, + Count = 5, +}; + +// 记录 kernel 最终在哪次 drain 中落地：Submit 开头、slot/heap 背压期间，或 +// 所有 worker 回放结束后的最终清空。三者之和必须等于实际 kernel 数。 +enum class DrainPlace : uint32_t { + EfDrain = 0, + RingBackpressure = 1, + FinalDrain = 2, + Count = 3, +}; + +enum class TensorArgType : int32_t { + Input = 0, + Output = 1, + Inout = 2, + OutputExisting = 3, + NoDependency = 4, +}; +// Input 作为 kernel 输入并参与依赖、但不登记为写者；Output 由本次 Submit 在 heap 中物化； +// Inout 与 OutputExisting 还需登记进每 worker 私有 TensorMap，供后续重叠区间查询 producer。 +static_assert(sizeof(TensorArgType) == sizeof(int32_t), "TensorArgType must match the PA tag ABI"); + +enum class DataType : uint8_t { + Float32 = 0, + Float16 = 1, + Int32 = 2, + Int16 = 3, + Int8 = 4, + Uint8 = 5, + Bfloat16 = 6, + Int64 = 7, + Uint64 = 8, + Uint16 = 9, + Uint32 = 10, + Bool = 11, + Count = 12, +}; + +// ProfilePhase 是聚合计数下标，TracePhase 是原始泳道事件 ABI；二者故意分离， +// 不能假设枚举值相同。一次 trace 写入可同时归入一个不同命名的 profile 阶段。 +enum class ProfilePhase : uint32_t { + Orchestration = 0, + Submit = 1, + EfDrain = 2, + Materialize = 3, + PrepareMap = 4, + Claim = 5, + Fanin = 6, + Register = 7, + WaitForSlot = 8, + HeapGuard = 9, + Build = 10, + ReplayTail = 11, + Count = 12, +}; + +// submit-pmu 每个 ELF 只编译一个局部归因阶段。none 不做中途 counter +// 读取，是完整 Submit 的正式基线；其余阶段都在每个 worker 的五次 Submit +// 上各执行一次，因此统一按固定 5*batches 次数闭合。历史 ID=3 曾用于 +// winner-only WaitForSlot，现已退役且不复用，避免旧 raw 被误认成新阶段。 +enum class SubmitPmuPhase : uint32_t { + None = 0, + Claim = 1, + EfDrain = 2, + Materialize = 4, + Register = 5, + Count = 6, +}; + +#ifndef PA_SUBMIT_PMU_PHASE_ID +#define PA_SUBMIT_PMU_PHASE_ID 0 +#endif + +constexpr SubmitPmuPhase kCompiledSubmitPmuPhase = + static_cast(PA_SUBMIT_PMU_PHASE_ID); +constexpr uint32_t kBuildVariantSwimlane = 1U; +constexpr uint32_t kBuildVariantSubmitPmu = 2U; +static_assert( + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::None) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::Claim) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::EfDrain) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::Materialize) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::Register), + "invalid compiled submit-pmu phase" +); + +struct NopCounts { + uint32_t qk; + uint32_t sf; + uint32_t pv; + uint32_t up; +}; + +// winner 的计算负载与 NOP 校准量使用两套独立计数，禁止把同一个数字同时解释成 +// scalar 指令条数和 vector/cube 工作迭代数。首阶段只有 CCEC 实现 RealCompute； +// 该 ABI 放在公共模型中，便于后续按相同配置逐步迁移 AscendC 与 CPU。 +struct WorkloadCounts { + uint32_t qk; + uint32_t sf; + uint32_t pv; + uint32_t up; +}; +static_assert(sizeof(WorkloadCounts) == 16, "workload counts ABI changed"); + +enum class WinnerWorkloadMode : uint32_t { + ScalarNop = 0, + RealCompute = 1, +}; + +constexpr uint32_t kWinnerWorkloadConfigVersion = 1; + +// 真实计算工作区是 standalone sidecar，不属于生产 DistGlobal/DistCore ABI。 +// workspace_base 指向 host 单独申请并初始化的 GM；每个 worker 只写自己的输出片段。 +struct alignas(64) WinnerWorkloadConfig { + uint32_t mode; + uint32_t version; + WorkloadCounts repeats; + uint64_t workspace_base; + uint64_t workspace_bytes; + uint32_t reserved[6]; +}; +static_assert(sizeof(WinnerWorkloadConfig) == 64, "winner workload config must occupy one cache line"); +static_assert(offsetof(WinnerWorkloadConfig, mode) == 0, "winner workload mode offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, version) == 4, "winner workload version offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, repeats) == 8, "winner workload counts offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, workspace_base) == 24, "winner workload base offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, workspace_bytes) == 32, "winner workload bytes offset changed"); + +// RunConfig 是 host 在 launch 前写、worker 启动时只读的控制 cache line。 +// 输入为 batch/NOP/诊断开关；输出不回写这里，而发布到独立 WorkerResult。 +struct alignas(64) RunConfig { + uint32_t batches; + uint32_t workers; + NopCounts nops; + uint32_t profile_phases; + uint32_t trace_enabled; + uint64_t trace_base; + uint32_t trace_records_per_core; + uint32_t reserved[5]; +}; +static_assert(sizeof(RunConfig) == 64, "RunConfig must occupy one cache line"); + +enum class TracePhase : int32_t { + Kernel = 0, + Alloc = 1, + Build = 2, + DrainWon = 3, + Replay = 4, + RingBp = 5, + EfDrain = 6, + Commit = 7, + Submit = 8, + Materialize = 9, + PrepareMap = 10, + Claim = 11, + Fanin = 12, + Register = 13, + Atomic = 14, + // 逐 atomic 诊断构建中，每个 worker 只记录一次连续两次 SYS_CNT 的 + // 空括号，用来给出同一二进制、同一物理核上的计时分辨率下限。 + ClockBaseline = 15, + // schema-v4 追加的父区间与真实动作区间。loser 没有可单列的真实动作， + // 其时间直接归入离线计算的 Submit residual，不占用 raw 记录。 + OrchestrationReplay = 16, + FinalDrain = 17, + WinnerBuild = 18, + AllocComplete = 19, + Count = 20, +}; + +// AtomicSite 按 standalone PA 中真实出现的源码调用点分类。编号写入 TraceRecord::auxiliary， +// 是离线泳道 schema 的一部分；追加新位置只能在 Count 前扩展，不能重排既有值。 +enum class AtomicSite : uint32_t { + StartupIncrement = 0, + StartupPoll = 1, + FatalPoll = 2, + FatalSet = 3, + ClaimMax = 4, + FaninFlagLoad = 5, + CompletionVendExchange = 6, + CompletionFlagExchange = 7, + FrontierInitialLoad = 8, + FrontierFlagLoad = 9, + FrontierMax = 10, + HeapFrontierLoad = 11, + HeapVendLoad = 12, + ReplayDoneIncrement = 13, + ReplayDonePoll = 14, + Count = 15, +}; + +// Atomic 记录 flags 的低四位保存操作种类；bit4 表示返回值参与后续判断， +// bit5 表示 Load 观察到零，bit6 表示结束时间已由返回值依赖推进到 +// return-ready 边界。schema-v3 中 bit7 区分等待区 PollBatch：此时 +// bits[31:8] 是精确调用次数；直接 FetchMax 中同一区域仍表示软件重试数。 +enum class AtomicOp : uint32_t { + Load = 0, + Exchange = 1, + FetchAdd = 2, + FetchMax = 3, +}; +constexpr uint32_t kAtomicOpMask = 0x0fU; +constexpr uint32_t kAtomicResultUsed = 1U << 4; +constexpr uint32_t kAtomicValueZero = 1U << 5; +constexpr uint32_t kAtomicReturnReady = 1U << 6; +constexpr uint32_t kAtomicPollBatch = 1U << 7; +constexpr uint32_t kAtomicRetriesShift = 8; +constexpr uint32_t kAtomicPollCountShift = 8; +constexpr uint32_t kAtomicPollCountMax = 0x00ffffffU; +constexpr uint32_t kAtomicPollBatchSiteCount = 6; + +// 这些映射是 raw ABI 的一部分，同时被 device 聚合器与 host 闭环校验使用。 +// 0..14 与真实 PA 保持稳定；BlockWon 尚未在 standalone 中实现，不能只为 +// 编号齐全而追加没有真实调用路径的 site。 +#ifdef PA_DEVICE +#define PA_MODEL_INLINE PA_DEVICE +#else +#define PA_MODEL_INLINE inline +#endif + +PA_MODEL_INLINE constexpr AtomicOp AtomicSiteExpectedOp(AtomicSite site) { + switch (site) { + case AtomicSite::StartupIncrement: + case AtomicSite::ReplayDoneIncrement: + return AtomicOp::FetchAdd; + case AtomicSite::FatalSet: + case AtomicSite::CompletionVendExchange: + case AtomicSite::CompletionFlagExchange: + return AtomicOp::Exchange; + case AtomicSite::ClaimMax: + case AtomicSite::FrontierMax: + return AtomicOp::FetchMax; + default: + return AtomicOp::Load; + } +} + +PA_MODEL_INLINE constexpr bool AtomicSiteResultUsed(AtomicSite site) { + switch (site) { + case AtomicSite::StartupIncrement: + case AtomicSite::FatalSet: + case AtomicSite::CompletionVendExchange: + case AtomicSite::CompletionFlagExchange: + case AtomicSite::ReplayDoneIncrement: + return false; + default: + return true; + } +} + +PA_MODEL_INLINE constexpr int32_t AtomicPollBatchIndex(AtomicSite site) { + switch (site) { + case AtomicSite::StartupPoll: + return 0; + case AtomicSite::FatalPoll: + return 1; + case AtomicSite::FaninFlagLoad: + return 2; + case AtomicSite::HeapFrontierLoad: + return 3; + case AtomicSite::HeapVendLoad: + return 4; + case AtomicSite::ReplayDonePoll: + return 5; + default: + return -1; + } +} + +PA_MODEL_INLINE constexpr AtomicSite AtomicPollBatchSite(uint32_t index) { + switch (index) { + case 0: + return AtomicSite::StartupPoll; + case 1: + return AtomicSite::FatalPoll; + case 2: + return AtomicSite::FaninFlagLoad; + case 3: + return AtomicSite::HeapFrontierLoad; + case 4: + return AtomicSite::HeapVendLoad; + case 5: + return AtomicSite::ReplayDonePoll; + default: + return AtomicSite::Count; + } +} + +PA_MODEL_INLINE constexpr bool AtomicSiteIsPollBatchable(AtomicSite site) { + return AtomicPollBatchIndex(site) >= 0; +} + +PA_MODEL_INLINE constexpr uint32_t AtomicSiteMask(AtomicSite site) { + return 1U << static_cast(site); +} + +#undef PA_MODEL_INLINE + +// ClockBaseline 的 bit0 区分普通连续 SYS_CNT 与后端的 atomic 返回依赖 +// 计时钩子；后者用于量化那一条依赖 MOV 自身带来的固定底噪。 +constexpr uint32_t kClockAtomicDependency = 1U << 0; +constexpr uint32_t kClockAtomicDependencyApplied = 1U << 1; + +struct alignas(64) TraceCoreState { + volatile uint32_t count; + volatile uint32_t dropped; + // logical atomic 调用数与物理记录数分开闭合：PollBatch 的一条记录可以 + // 表示多次只读轮询，physical = atomic_calls - poll_calls + batch_records。 + volatile uint32_t atomic_calls; + volatile uint32_t poll_calls; + volatile uint32_t poll_batch_records; + // 拓扑在一个 worker 分区内恒定；当前仍保留 64B TraceRecord 兼容布局， + // 但在 core state 再保存一份权威身份，host 会验证每条记录与之相符。 + volatile int32_t core_idx; + volatile int32_t block_id; + volatile int32_t lane; + uint32_t padding[8]; +}; +// 每个 worker 独占一个计数 cache line 和一段定长 records，不需要为了写 trace +// 再引入跨核 atomic；满容量后只增加本 worker 的 dropped。 +static_assert(sizeof(TraceCoreState) == 64, "trace core state must occupy one cache line"); + +struct alignas(64) TraceHeader { + uint32_t magic; + uint32_t version; + uint32_t num_cores; + uint32_t records_per_core; + uint64_t frequency_hz; + TraceCoreState cores[kRuntimeMaxWorkers]; +}; +// 本 benchmark 固定物理分配 kWorkers=96 个定长 record 分区，合法 header 也要求 +// num_cores==96；其 header/record 布局和 phase 编号可转换为真实泳道使用的 JSON。 +static_assert(sizeof(TraceHeader) == 6976, "trace header must match PA swimlane layout"); + +struct alignas(64) TraceRecord { + uint64_t start_cycle; + uint64_t end_cycle; + int32_t task_id; + int32_t function_id; + int32_t phase; + int32_t lane; + int32_t block_id; + int32_t core_idx; + uint32_t flags; + uint32_t auxiliary; +}; +// start/end 保留原始 1 GHz counter；task/function/物理 lane 用于离线还原轨道。 +// flags/aux 的含义由 phase 决定，例如 winner、Alloc 或 RingBp 类型，不参与调度决策。 +static_assert(sizeof(TraceRecord) == 64, "trace record must occupy one cache line"); + +constexpr size_t kTraceBytes = + sizeof(TraceHeader) + static_cast(kWorkers) * kTraceRecordsPerCore * sizeof(TraceRecord); + +struct alignas(64) AtomicLine { + volatile int64_t value; + uint8_t padding[64 - sizeof(int64_t)]; +}; +// 热点共享量各占一个 cache line，保持生产代码的地址隔离，避免 standalone +// 因伪共享额外放大 Claim/frontier/start barrier 的竞争。 +static_assert(sizeof(AtomicLine) == 64, "AtomicLine must occupy one cache line"); + +struct alignas(64) AtomicFlagLine { + volatile int32_t value; + uint8_t padding[64 - sizeof(int32_t)]; +}; +// 32-bit fatal 与 64-bit cursor 使用不同封装，但都独占 cache line；成功路径中 +// fatal 始终为零，任何写一都表示协议已终止，不能作为普通等待条件清除。 +static_assert(sizeof(AtomicFlagLine) == 64, "AtomicFlagLine must occupy one cache line"); + +struct alignas(64) TaskCell { + volatile int64_t flag; + volatile uint64_t vend; + uint8_t padding[64 - 2 * sizeof(int64_t)]; +}; +// flag 是依赖就绪与 frontier 连续前推的发布位；vend 是该 task 完成时 worker 的 +// 单调 heap_next 快照。HeapGuard 读取 frontier-H 对应 vend，判断环形 heap 是否可覆盖。 +static_assert(sizeof(TaskCell) == 64, "TaskCell must occupy one cache line"); + +// TensorDesc 保留真实 Tensor 的两条 64-byte 数据线。owner_task_id 表达显式生产者， +// buffer_addr + 字节区间用于 TensorMap 发现同一 backing buffer 上的读写依赖。 +struct TensorDesc { + uint64_t buffer_addr; + uint64_t buffer_size; + uint64_t owner_task_id; + uint64_t start_offset; + int32_t version; + uint32_t ndims; + DataType dtype; + bool manual_dep; + bool is_contiguous; + uint8_t child_memory; + uint32_t shapes[kMaxTensorDims]; + + uint64_t extent_elem_cache; + uint32_t strides[kMaxTensorDims]; + uint8_t padding[36]; +}; +static_assert(sizeof(TensorDesc) == 128, "TensorDesc must match the PA Tensor ABI size"); +static_assert(offsetof(TensorDesc, buffer_addr) == 0, "TensorDesc buffer offset mismatch"); +static_assert(offsetof(TensorDesc, owner_task_id) == 16, "TensorDesc owner offset mismatch"); +static_assert(offsetof(TensorDesc, start_offset) == 24, "TensorDesc view offset mismatch"); +static_assert(offsetof(TensorDesc, version) == 32, "TensorDesc version offset mismatch"); +static_assert(offsetof(TensorDesc, shapes) == 44, "TensorDesc shape offset mismatch"); +static_assert(offsetof(TensorDesc, extent_elem_cache) == 64, "TensorDesc extent offset mismatch"); +static_assert(offsetof(TensorDesc, strides) == 72, "TensorDesc stride offset mismatch"); + +struct TensorCreateInfo { + uint64_t initial_value; + bool has_initial_value; + uint8_t padding0[7]; + uint64_t reserved0; + uint64_t start_offset; + int32_t version; + uint32_t ndims; + DataType dtype; + bool manual_dep; + bool is_contiguous; + uint8_t child_memory; + uint32_t shapes[kMaxTensorDims]; +}; +// CreateInfo 只描述尚未分配的 Output；Materialize 根据形状和 dtype 计算大小， +// 再把它变成位于 worker 逻辑 heap 上的 TensorDesc。 +static_assert(sizeof(TensorCreateInfo) == 64, "TensorCreateInfo must match the PA create-info ABI size"); +static_assert(offsetof(TensorCreateInfo, start_offset) == 24, "TensorCreateInfo start offset mismatch"); +static_assert(offsetof(TensorCreateInfo, version) == 32, "TensorCreateInfo version offset mismatch"); +static_assert(offsetof(TensorCreateInfo, shapes) == 44, "TensorCreateInfo shape offset mismatch"); + +struct MapEntry { + uint64_t buffer_addr; + uint64_t lo; + uint64_t hi; + int32_t producer; + int32_t bucket; + int32_t next_in_bucket; + int32_t prev_in_bucket; + int32_t next_in_task; +}; +// 同一 entry 同时挂在两条链上：bucket 链按 buffer 地址查询重叠区间，task 链按 +// producer 批量退休。next_in_bucket 在空闲状态下复用为 free-list 链接。 +static_assert(sizeof(MapEntry) == 48, "MapEntry must match the PA tensor-map entry ABI"); +static_assert(offsetof(MapEntry, producer) == 24, "MapEntry producer offset mismatch"); +static_assert(offsetof(MapEntry, next_in_task) == 40, "MapEntry task-link offset mismatch"); + +struct TensorMap { + MapEntry entries[kMapCapacity]; + int32_t buckets[kMapBuckets]; + int32_t task_heads[kTaskWindow]; + int32_t free_head; + int32_t high_water; + int32_t alive_floor; + int32_t cleaned_upto; +}; +// TensorMap 是 worker 私有状态，不在多核间共享。alive_floor 表达查询存活下界， +// cleaned_upto 表达已物理摘链的进度；即使 Case1 中通常同步推进，也不能合并其 ABI 字段。 +static_assert(sizeof(TensorMap) == 823312, "TensorMap must match the PA fixed-capacity layout"); +static_assert(offsetof(TensorMap, buckets) == 786432, "TensorMap bucket offset mismatch"); +static_assert(offsetof(TensorMap, task_heads) == 819200, "TensorMap task-head offset mismatch"); +static_assert(offsetof(TensorMap, free_head) == 823296, "TensorMap control offset mismatch"); + +struct TaskPayload { + TensorDesc tensors[kMaxTaskTensors]; +}; +// task_id 通过 kPayloadMask 映射到 2048 个 4 KiB payload；Case1 只有 1280 个 task， +// 本轮不会回绕，但仍保留生产容量、寻址方式和 4 KiB stride。 +static_assert(sizeof(TaskPayload) == kPayloadStride, "TaskPayload must preserve the real 4 KiB task stride"); +static_assert(alignof(TaskPayload) == 8, "TaskPayload alignment must match DistTaskPayload"); +static_assert(offsetof(TaskPayload, tensors) == 0, "TaskPayload tensor offset mismatch"); + +struct LocalSlot { + // occupied 先保留容量，built 表示 payload 已按生产顺序构建；task/function + // 标识决定执行哪个 NOP 体，后续大数组则是 kernel 真正看到的参数快照。 + bool occupied; + bool built; + uint8_t header_padding[2]; + uint32_t task_id; + uint32_t kind; + uint32_t function_padding; + uint64_t function_address; + uint32_t tensor_count; + uint32_t scalar_count; + uint8_t tensor_padding[32]; + + TensorDesc tensors[kMaxTaskTensors]; + uint64_t scalars[kMaxTaskScalars]; + uint64_t args[kMaxTaskTensors + kMaxTaskScalars + 2]; + union { + struct { + uint8_t local_context[48]; + uint32_t global_context; + int32_t fanin[kMaxFanin]; + uint32_t fanin_count; + }; + // Compatibility view used by the standalone NOP payload builder. The + // first six words are the real 48-byte LocalContext; the remaining + // words overlap GlobalContext and the beginning of fanin, exactly as + // dictated by the real RingSlot offsets. + // 该视图只用于按真实 offset 填充 dispatch context，不增加另一份 + // 存储；修改其中后两字会同步覆盖 GlobalContext/fanin 的对应 ABI 字节。 + uint64_t context_words[8]; + }; + bool is_multicore; + int32_t won_block; + int32_t won_slot; +}; +// LocalSlot 是每个 winner 写入自己私有 ring 的完整 dispatch 包。fanin 在执行前 +// 逐项检查 task.flag；occupied/built 与计数共同约束最多两个普通 kernel 在途。 +static_assert(sizeof(LocalSlot) == 4824, "LocalSlot must match the PA RingSlot ABI size"); +static_assert(alignof(LocalSlot) == 8, "LocalSlot alignment must match RingSlot"); +static_assert(offsetof(LocalSlot, occupied) == 0, "LocalSlot occupied offset mismatch"); +static_assert(offsetof(LocalSlot, built) == 1, "LocalSlot built offset mismatch"); +static_assert(offsetof(LocalSlot, task_id) == 4, "LocalSlot task offset mismatch"); +static_assert(offsetof(LocalSlot, kind) == 8, "LocalSlot function-id offset mismatch"); +static_assert(offsetof(LocalSlot, function_address) == 16, "LocalSlot function address offset mismatch"); +static_assert(offsetof(LocalSlot, tensor_count) == 24, "LocalSlot tensor-count offset mismatch"); +static_assert(offsetof(LocalSlot, tensors) == 64, "LocalSlot tensor payload offset mismatch"); +static_assert(offsetof(LocalSlot, scalars) == 4160, "LocalSlot scalar payload offset mismatch"); +static_assert(offsetof(LocalSlot, args) == 4288, "LocalSlot dispatch-args offset mismatch"); +static_assert(offsetof(LocalSlot, local_context) == 4688, "LocalSlot local-context offset mismatch"); +static_assert(offsetof(LocalSlot, global_context) == 4736, "LocalSlot global-context offset mismatch"); +static_assert(offsetof(LocalSlot, fanin) == 4740, "LocalSlot fanin offset mismatch"); +static_assert(offsetof(LocalSlot, fanin_count) == 4804, "LocalSlot fanin-count offset mismatch"); +static_assert(offsetof(LocalSlot, is_multicore) == 4808, "LocalSlot multicore offset mismatch"); +static_assert(offsetof(LocalSlot, won_block) == 4812, "LocalSlot won-block offset mismatch"); +static_assert(offsetof(LocalSlot, won_slot) == 4816, "LocalSlot won-slot offset mismatch"); + +struct WorkerState { + CoreRole role; + int32_t core_idx; + int32_t block_id; + int32_t lane; + int32_t sub_block_id; + int32_t local_index; + uint64_t heap_next; + TensorMap map; + uint8_t slot_padding[16]; + LocalSlot slots[kPrivateSlots]; + uint32_t occupied_count; + uint32_t owned_total; + uint64_t swimlane_last_cycle; + uint8_t payload_padding[16]; + TaskPayload payloads[kPayloadSlots]; +}; +// 每个物理 worker 都持有独立 heap cursor、TensorMap、ring 与 task payload arena； +// 多核共享的只有 SchedulerState 前缀中的 cursor/task/frontier 等协议状态。 +static_assert(sizeof(WorkerState) == 9231296, "WorkerState must match the PA DistCore ABI size"); +static_assert(alignof(WorkerState) == 8, "WorkerState alignment must match DistCore"); +static_assert(offsetof(WorkerState, role) == 0, "WorkerState role offset mismatch"); +static_assert(offsetof(WorkerState, local_index) == 20, "WorkerState replay-index offset mismatch"); +static_assert(offsetof(WorkerState, heap_next) == 24, "WorkerState heap cursor offset mismatch"); +static_assert(offsetof(WorkerState, map) == 32, "WorkerState tensor-map offset mismatch"); +static_assert(offsetof(WorkerState, slots) == 823360, "WorkerState ring-slot offset mismatch"); +static_assert(offsetof(WorkerState, occupied_count) == 842656, "WorkerState occupancy offset mismatch"); +static_assert(offsetof(WorkerState, owned_total) == 842660, "WorkerState owned-count offset mismatch"); +static_assert(offsetof(WorkerState, swimlane_last_cycle) == 842664, "WorkerState trace clock offset mismatch"); +static_assert(offsetof(WorkerState, payloads) == 842688, "WorkerState task-payload offset mismatch"); + +struct alignas(64) WorkerResult { + // 时间边界：Submit 口径不含启动屏障和最终 drain，finish_cycle 则覆盖完整 worker 生命周期。 + uint64_t submit_begin; + uint64_t submit_end; + uint64_t finish_cycle; + uint64_t checksum; + + // 协议计数用于验证固定 Claim 拓扑及等待/依赖动态次数；joint_polls 是为未来 + // BlockWon 模拟保留的兼容字段，当前实现没有递增点，不能据其检测 joint 分支。 + uint64_t submits; + uint64_t claim_attempts; + uint64_t claim_wins; + uint64_t heap_guards; + uint64_t fanin_ready_loads; + uint64_t completion_duplicates; + uint64_t cas_retries; + uint64_t joint_polls; + + // 默认 256 batch 时 winner、kernel 分别闭合到 1280 task 和 1024 kernel； + // 非默认配置按 5*batches、4*batches 计算，placement 仍闭合到全部 kernel。 + uint64_t wins[static_cast(TaskKind::Count)]; + uint64_t kernel_counts[4]; + uint64_t kernel_cycles[4]; + uint64_t kernel_min_cycles[4]; + uint64_t kernel_max_cycles[4]; + uint64_t placement[static_cast(DrainPlace::Count)]; + uint64_t phase_cycles[static_cast(ProfilePhase::Count)]; + uint64_t phase_calls[static_cast(ProfilePhase::Count)]; + uint64_t wait_events[2]; + uint64_t wait_iterations[2]; + + // 前端工作量计数不是性能填充：构参、materialize 和 map insert 用于核对全部 + // 96 个 worker 的回放；map lookup、slot copy 与 fanin 则是 winner-only 全局计数。 + uint64_t context_reads; + uint64_t views_created; + uint64_t dynamic_create_infos; + uint64_t arg_resets; + uint64_t tensor_args_added; + uint64_t scalar_args_added; + uint64_t materialized_outputs; + uint64_t map_inserts; + uint64_t map_lookups; + uint64_t slot_tensor_copies; + uint64_t slot_scalar_copies; + uint64_t fanin_edges; + + // 最终快照用于跨 worker 比较逻辑 heap 与 TensorMap 回收状态是否完全一致。 + uint64_t final_heap_next; + uint64_t map_high_water; + uint64_t map_alive_floor; + uint64_t map_cleaned_upto; + uint64_t map_live_entries; + + uint64_t worker_id; + uint64_t role; + uint64_t max_occupied; + uint64_t final_occupied; + + // CCEC 标量 PMU 取证使用 WorkerResult 的诊断 sidecar，不改变生产 DistCore ABI。 + // 该诊断只在显式开启时有效；CNT2/CNT6/CNT7 分别对应 scalar busy、I-cache req/miss。 + uint64_t pmu_total_cycles; + uint32_t pmu_scalar_busy; + uint32_t pmu_icache_requests; + uint32_t pmu_icache_misses; + uint32_t pmu_status; + + // 这些计数只在 worker 私有 LocalStats 中递增，结束时一次性发布；它们把动态 + // fanin 重试和 frontier helping 展开为准确次数，不为取数再增加共享 atomic。 + uint64_t fanin_not_ready_loads; + uint64_t frontier_initial_loads; + uint64_t frontier_updates; + uint64_t frontier_terminal_loads; + + // 仅在 trace_enabled bit1 开启时递增；每次源码 atomic 调用恰好增加一， + // host 用它与 Atomic span 数逐 worker 闭合，禁止把丢记录的泳道当成完整结果。 + uint64_t atomic_trace_calls; + + // I-cache 单 miss 探针用该槽保存 cold 窗口的 1 GHz SYS_CNT；submit-pmu + // 复用同一 64-bit 槽保存所选局部阶段的逐调用累计时间。两种构建互斥， + // 因而无需扩大 832B WorkerResult，也不会改变相邻 worker 的 cache-line 布局。 + union { + uint64_t pmu_window_ticks; + uint64_t pmu_phase_elapsed_ticks; + }; + uint64_t pmu_warm_total_cycles; + uint64_t pmu_warm_window_ticks; + union { + uint32_t pmu_warm_icache_requests; + uint32_t pmu_phase_begin_reads; + }; + union { + uint32_t pmu_warm_icache_misses; + uint32_t pmu_phase_end_reads; + }; + + // PIPE_UTILIZATION 已同时配置 CNT0/1/3/4/5/8；与上面的 scalar/I-cache + // 一样只保存每核原始累计值，AIC/AIV 汇总与比率统一在 host sidecar 中计算。 + // 六个 32-bit 值复用本结构扩展到 832B 后的尾部空间，不再增加 cache line。 + uint32_t pmu_vector_busy; + uint32_t pmu_cube_busy; + uint32_t pmu_mte1_busy; + uint32_t pmu_mte2_busy; + // swimlane ABI 保留该槽；submit-pmu 将物理 CNT5 改作 shadow miss， + // 因而显式发布 0，并在当前 submit-pmu schema-v5 标记 mte3_busy 不可用。 + uint32_t pmu_mte3_busy; + uint32_t pmu_fix_busy; + + // 复用 WorkerResult 原有的 32B cache-line 尾洞，不扩大 832B stride。 + // CNT6/7 是从不中途读取的权威整窗，CNT8/CNT5 是 read-to-clear shadow； + // none 在 stop 后要求逐核精确相等；运行中切片的 phase 只允许 shadow + // 单向小于 primary，并显式导出差值形成局部观测区间。 + uint32_t pmu_build_variant; + uint32_t pmu_phase_id; + uint32_t pmu_phase_calls; + uint32_t pmu_phase_status; + uint32_t pmu_phase_icache_requests; + uint32_t pmu_phase_icache_misses; + uint32_t pmu_shadow_icache_requests; + uint32_t pmu_shadow_icache_misses; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + // split 组合 oracle 独占一条诊断 cache line；legacy 与 inline shape 完全 + // 不带这些字段，避免改变既有 WorkerResult/SchedulerState 的布局。 + uint64_t lazy_sample_split_caller_state_address; + uint64_t lazy_sample_split_finish_state_address; + uint64_t lazy_sample_split_finish_calls; + uint64_t lazy_sample_split_protocol_errors; + uint64_t lazy_sample_split_state_cookie; + uint64_t lazy_sample_split_task_id_sum; + uint64_t lazy_sample_split_owner_worker_id; + uint64_t lazy_sample_split_reserved; +#endif +}; +// WorkerResult 是 standalone 尾部的诊断 sidecar，不属于真实 DistCore ABI；按 +// cache line 隔离后，各 worker 发布统计不会相互覆盖或污染被测共享状态。 +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +static_assert(sizeof(WorkerResult) == 896, "split WorkerResult diagnostics must occupy whole cache lines"); +static_assert(offsetof(WorkerResult, lazy_sample_split_caller_state_address) == 832, + "split WorkerResult oracle offset mismatch"); +static_assert(offsetof(WorkerResult, lazy_sample_split_reserved) == 888, + "split WorkerResult oracle tail mismatch"); +#else +static_assert(sizeof(WorkerResult) == 832, "WorkerResult diagnostics must occupy whole cache lines"); +#endif +static_assert(offsetof(WorkerResult, pmu_total_cycles) == 680, "WorkerResult PMU offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_status) == 700, "WorkerResult PMU status offset mismatch"); +static_assert(offsetof(WorkerResult, fanin_not_ready_loads) == 704, "WorkerResult atomic diagnostic offset mismatch"); +static_assert(offsetof(WorkerResult, atomic_trace_calls) == 736, "WorkerResult atomic trace offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_window_ticks) == 744, "WorkerResult PMU timing offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_vector_busy) == 776, "WorkerResult extended PMU offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_build_variant) == 800, "WorkerResult submit-PMU offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_shadow_icache_misses) == 828, "WorkerResult submit-PMU tail mismatch"); + +// 从 cube_cursor 到 workers 结束保留关键字段 offset、DistCore ABI 和生产总字节跨度， +// 并非字段级完整镜像。RunConfig、输入 context_lens 与校验结果追加在该跨度之后， +// 因此测试控制信息不会改变被测字段 offset。 +struct alignas(64) SchedulerState { + // 三组四分片 cursor 分别服务 AIC kernel、AIV kernel 和 Alloc；同 task 的 + // eligible workers 竞争同一 shard，只有旧值小于 task_id 的调用成为 winner。 + AtomicLine cube_cursor[kCursorShards]; + AtomicLine vector_cursor[kCursorShards]; + AtomicLine alloc_cursor[kCursorShards]; + AtomicLine frontier; + int32_t heap_window; + uint8_t tasks_padding[60]; + TaskCell tasks[kTaskCellCapacity]; + // heap_base/size 描述共享物理环，worker.heap_next 则是各 worker 一致推进的逻辑游标。 + uint64_t heap_base; + uint64_t heap_size; + uint64_t orchestration_args; + uint64_t runtime_state; + uint64_t runtime; + uint8_t fatal_padding[24]; + AtomicFlagLine fatal; + int32_t num_workers; + int32_t num_blocks; + // Case1 never enters BlockWon, but the inactive layout and BlockWon arena + // remains byte-for-byte reserved so every subsequent PA atomic line keeps + // its production offset. + // 此处不能因 Case1 动态次数为零而删减，否则 replay_done、started_count + // 和 DistCore 数组整体前移，便不再是对真实 PA 地址布局的等价测试。 + uint8_t layout_and_block_won[5848440]; + AtomicLine replay_done; + AtomicLine started_count; + // started_count 形成 launch 屏障；replay_done 只用于最终 drain 判定所有 worker + // 已不再产生新 slot。两者都位于 Submit 性能口径之外，但属于完整协议。 + WorkerState workers[kRuntimeMaxWorkers]; + // Standalone-only controls live after the complete DistGlobal image. They + // therefore do not shift any cursor/task/fatal/worker address under test. + RunConfig config; + WinnerWorkloadConfig winner_workload; + // Context lengths are the only PA input elements read by orchestration; + // keeping them in GM preserves the per-batch descriptor-based load. + // 除这 256 个长度值外，其余 tensor 仅需稳定的合成地址来复现 + // descriptor、依赖和 heap 行为，不会解引用成真实计算数据。 + volatile int32_t context_lens[kMaxBatches]; + WorkerResult results[kWorkers]; +}; +static_assert(offsetof(SchedulerState, cube_cursor) == 0, "cube cursor offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, vector_cursor) == 256, "vector cursor offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, alloc_cursor) == 512, "alloc cursor offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, frontier) == 768, "frontier offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, heap_window) == 832, "H offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, tasks) == kRealTasksOffset, "task table offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, heap_base) == 4195200, "heap base offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, heap_size) == 4195208, "heap size offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, fatal) == kRealFatalOffset, "fatal offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, replay_done) == kRealReplayDoneOffset, "replay offset must match PA DistGlobal"); +static_assert( + offsetof(SchedulerState, started_count) == kRealStartedCountOffset, + "started-count offset must match PA DistGlobal" +); +static_assert(offsetof(SchedulerState, tasks) % 64 == 0, "task table must be cache-line aligned"); +static_assert(offsetof(SchedulerState, workers) % 64 == 0, "worker table must be cache-line aligned"); +static_assert(offsetof(SchedulerState, results) % 64 == 0, "result table must be cache-line aligned"); +static_assert(offsetof(SchedulerState, workers) == kRealDistCoreOffset, "DistCore table offset must match PA"); +static_assert(offsetof(SchedulerState, config) == kRealDistGlobalBytes, "DistGlobal byte size must match PA"); +static_assert( + offsetof(SchedulerState, winner_workload) == kRealDistGlobalBytes + sizeof(RunConfig), + "winner workload sidecar must follow RunConfig" +); +static_assert( + offsetof(SchedulerState, context_lens) == + kRealDistGlobalBytes + sizeof(RunConfig) + sizeof(WinnerWorkloadConfig), + "context lengths must follow standalone controls" +); + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_MODEL_H diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/pa_scheduler_core.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/pa_scheduler_core.h new file mode 100644 index 0000000000..afdd0896ce --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/pa_scheduler_core.h @@ -0,0 +1,1618 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_SCHEDULER_CORE_H +#define PA_SCHEDULER_COMMON_PA_SCHEDULER_CORE_H + +#ifndef PA_DEVICE +#define PA_DEVICE inline +#endif + +#ifndef PA_GM +#define PA_GM +#endif + +#include "pa_frontend.h" +#include "pa_trace.h" + +namespace pa_scheduler { + +struct LocalStats { + WorkerResult result; + uint32_t max_occupied; + TraceContext trace; +}; + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +// runtime TU owns one external [[block_local]] instance per architecture. +// The caller imports that same object and keeps all Submit-internal context in +// it, so the only cross-TU function arguments are a POD ticket and built args. +struct alignas(64) LazySampleSplitRuntimeState { + PA_GM SchedulerState *scheduler; + PA_GM WorkerState *worker; + uint32_t task_count; + uint32_t worker_id; + SubmitContext context; + LocalStats stats; + uint64_t caller_state_address; + uint64_t finish_state_address; + uint64_t finish_calls; + uint64_t protocol_errors; + uint64_t state_cookie; + uint64_t task_id_sum; + uint64_t owner_worker_id; + uint64_t reserved; +}; +static_assert(sizeof(LazySampleSplitRuntimeState) % 64 == 0, + "split runtime state must occupy whole cache lines"); + +PA_DEVICE uint64_t LazySampleSplitStateCookie(uint32_t worker_id, CoreRole role) { + return kLazySampleSplitStateCookieBase ^ static_cast(worker_id) ^ + (static_cast(static_cast(role)) << 32U); +} +#endif + +// submit-pmu 的 phase 在编译期固定；非诊断构建完全不引用 Ops 的 phase +// 接口。这样公共调度代码保持一份，swimlane/CPU/AscendC 也不会多出运行时分支。 +template +PA_DEVICE void BeginSubmitPmuPhase(PmuContext &context) { +#if PA_BUILD_SUBMIT_PMU + if constexpr (kCompiledSubmitPmuPhase == Phase) { + Ops::PmuPhaseBegin(context); + } +#else + (void)context; +#endif +} + +template +PA_DEVICE void EndSubmitPmuPhase(PmuContext &context) { +#if PA_BUILD_SUBMIT_PMU + if constexpr (kCompiledSubmitPmuPhase == Phase) { + Ops::PmuPhaseEnd(context); + } +#else + (void)context; +#endif +} + +template +PA_DEVICE uint64_t TraceTimestamp(TraceContext &trace, WorkerResult &result) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + return 0; +#else + (void)result; + // 对齐真实 FDWIC 的 TRACE_SPAN_BEGIN/END：取一次时间后立即以同一 + // cycle 关闭活跃 PollBatch。不能在 WriteTrace 中统一关闭，否则直接 + // Atomic 记录也会错误切断等待 episode。 + const uint64_t cycle = Ops::Now(); + AtomicPollBoundaryAt(trace, cycle); + return cycle; +#endif +} + +PA_DEVICE uint32_t KindIndex(TaskKind kind) { return static_cast(kind); } + +PA_DEVICE TaskKind GetTaskKind(uint32_t task_id) { return static_cast(task_id % kTasksPerBatch); } + +PA_DEVICE int32_t FunctionId(TaskKind kind) { + return kind == TaskKind::Alloc ? -1 : static_cast(KindIndex(kind) - 1); +} + +PA_DEVICE uint32_t NopCountForKind(PA_GM const NopCounts &nops, TaskKind kind) { + switch (kind) { + case TaskKind::Qk: + return nops.qk; + case TaskKind::Sf: + return nops.sf; + case TaskKind::Pv: + return nops.pv; + case TaskKind::Up: + return nops.up; + default: + return 0; + } +} + +PA_DEVICE uint32_t WorkloadCountForKind(PA_GM const WorkloadCounts &counts, TaskKind kind) { + switch (kind) { + case TaskKind::Qk: + return counts.qk; + case TaskKind::Sf: + return counts.sf; + case TaskKind::Pv: + return counts.pv; + case TaskKind::Up: + return counts.up; + default: + return 0; + } +} + +PA_DEVICE uint32_t CountBits(uint32_t value) { + uint32_t count = 0; + while (value != 0) { + count += value & 1U; + value >>= 1; + } + return count; +} + +template +PA_DEVICE int64_t LoadLine( + PA_GM AtomicLine &line, LocalStats &stats, AtomicSite site, int32_t task_id = -1 +) { + // Ops::Load 在 A5 后端是 atomicAdd(0)；返回值是该 RMW 线性化时观察到的共享值。 + return TraceAtomicLoad(stats.trace, stats.result, task_id, site, &line.value); +} + +template +PA_DEVICE int32_t LoadLine( + PA_GM AtomicFlagLine &line, LocalStats &stats, AtomicSite site, int32_t task_id = -1 +) { + return TraceAtomicLoad(stats.trace, stats.result, task_id, site, &line.value); +} + +template +PA_DEVICE void SetFatal(PA_GM SchedulerState *state, LocalStats &stats, int32_t task_id = -1) { + // fatal 只从 0 单调置 1，重复 Exchange 不会把其他 worker 已观察到的失败状态清除。 + TraceAtomicExchange( + stats.trace, stats.result, task_id, AtomicSite::FatalSet, &state->fatal.value, + static_cast(1) + ); +} + +template +PA_DEVICE bool IsFatal(PA_GM SchedulerState *state, LocalStats &stats, int32_t task_id = -1) { + return LoadLine(state->fatal, stats, AtomicSite::FatalPoll, task_id) != 0; +} + +template +PA_DEVICE bool WatchdogExpired( + PA_GM SchedulerState *state, LocalStats &stats, uint64_t begin, uint32_t &polls +) { + // 每 1024 次自旋才读取系统计数器，降低正常启动屏障上的计时开销；超时后向所有 worker 广播 fatal。 + ++polls; + if ((polls & 1023U) != 0 || Ops::Now() - begin <= kWatchdogTicks) { + return false; + } + SetFatal(state, stats); + return true; +} + +template +PA_DEVICE void AdvanceFrontier(PA_GM SchedulerState *state, LocalStats &stats) { + // frontier 只表示“从 task 0 开始已经连续完成”的最高 task id，不能越过尚未发布 flag 的空洞。 + // 多个完成者可以同时扫描同一段连续区间，FetchMax 保证共享 frontier 只前进、不回退。 + ++stats.result.frontier_initial_loads; + int64_t frontier = LoadLine( + state->frontier, stats, AtomicSite::FrontierInitialLoad + ); + while (true) { + const int64_t next = frontier + 1; + if (next < 0 || next >= static_cast(kTaskCellCapacity)) { + break; + } + if (TraceAtomicLoad( + stats.trace, stats.result, static_cast(next), AtomicSite::FrontierFlagLoad, + &state->tasks[next].flag + ) == 0) { + ++stats.result.frontier_terminal_loads; + break; + } + uint64_t retries = 0; + // FetchMax 返回更新前的值；若其他核已经走得更远，就从其 old 值继续扫描，避免重复从 next 起步。 + ++stats.result.frontier_updates; + const int64_t old = TraceAtomicFetchMax( + stats.trace, stats.result, static_cast(next), AtomicSite::FrontierMax, + &state->frontier.value, next, retries + ); + stats.result.cas_retries += retries; + frontier = old > next ? old : next; + } +} + +template +PA_DEVICE void CompleteTask( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, LocalStats &stats +) { + // 完成发布顺序与 PA 一致：先公布该 worker 的 heap 游标，再发布 ready flag，最后推进连续 frontier。 + // fanin 和 heap 回收方以 flag/frontier 为可见性条件，因此不能交换 vend 与 flag 的先后关系。 + TraceAtomicExchange( + stats.trace, stats.result, static_cast(task_id), AtomicSite::CompletionVendExchange, + &state->tasks[task_id].vend, worker.heap_next + ); + Ops::StoreBarrier(); + TraceAtomicExchange( + stats.trace, stats.result, static_cast(task_id), AtomicSite::CompletionFlagExchange, + &state->tasks[task_id].flag, static_cast(1) + ); + AdvanceFrontier(state, stats); +} + +template +PA_DEVICE bool SlotReady(PA_GM SchedulerState *state, PA_GM LocalSlot &slot, LocalStats &stats) { + // 每个 fanin flag 都是跨核共享的完成条件；遇到第一个未就绪依赖即返回，后续 drain 会再次轮询。 + for (uint32_t index = 0; index < slot.fanin_count; ++index) { + const int32_t dependency = slot.fanin[index]; + if (TraceAtomicLoad( + stats.trace, stats.result, dependency, AtomicSite::FaninFlagLoad, + &state->tasks[dependency].flag + ) == 0) { + ++stats.result.fanin_not_ready_loads; + return false; + } + ++stats.result.fanin_ready_loads; + } + return true; +} + +PA_DEVICE void RecordKernelCycles(LocalStats &stats, TaskKind kind, uint64_t cycles) { + const uint32_t index = KindIndex(kind) - 1; + ++stats.result.kernel_counts[index]; + stats.result.kernel_cycles[index] += cycles; + if (stats.result.kernel_min_cycles[index] == 0 || cycles < stats.result.kernel_min_cycles[index]) { + stats.result.kernel_min_cycles[index] = cycles; + } + if (cycles > stats.result.kernel_max_cycles[index]) { + stats.result.kernel_max_cycles[index] = cycles; + } +} + +template +PA_DEVICE uint32_t DrainReady( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, DrainPlace place, LocalStats &stats +) { + // 同一套 drain 被三个位置复用：每次 Submit 开头的 EfDrain、ring 背压等待和所有 Submit 后的最终 drain。 + // slot 属于当前 worker；只有其全部跨核 fanin 已 ready 时才执行所选 winner 负载、发布完成并释放 slot。 + if (worker.occupied_count == 0) { + return 0; + } + uint32_t freed = 0; + // 一次调用遍历完本核全部私有 slot；未就绪项保留 occupied/built，已完成项立即清槽并继续扫描。 + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + PA_GM LocalSlot &slot = worker.slots[index]; + if (!slot.occupied || !slot.built || !SlotReady(state, slot, stats)) { + continue; + } + const TaskKind kind = static_cast(slot.kind + 1); + const uint64_t kernel_begin = TraceTimestamp(stats.trace, stats.result); + Ops::ExecuteKernel(state, worker, kind, NopCountForKind(state->config.nops, kind)); + const uint64_t kernel_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(slot.task_id), static_cast(slot.kind), + TracePhase::Kernel, ProfilePhase::ReplayTail, kernel_begin, kernel_end + ); + RecordKernelCycles(stats, kind, kernel_end - kernel_begin); + CompleteTask(state, worker, slot.task_id, stats); + const uint64_t commit_cycle = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(slot.task_id), static_cast(slot.kind), + TracePhase::Commit, ProfilePhase::ReplayTail, commit_cycle, commit_cycle + ); + slot.built = false; + slot.occupied = false; + --worker.occupied_count; + ++stats.result.placement[static_cast(place)]; + ++freed; + } + return freed; +} + +PA_DEVICE int32_t FindFreeSlot(PA_GM WorkerState &worker) { + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + if (!worker.slots[index].occupied) { + return static_cast(index); + } + } + return -1; +} + +template +PA_DEVICE void WaitForSlot( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, LocalStats &stats +) { + // 四个物理 slot 中预留两个 won slot 语义位，仅有 kUsableSlots 个可供本图使用；满时靠 drain 取得进展。 + const uint64_t wait_begin = TraceTimestamp(stats.trace, stats.result); + bool waited = false; + // 只聚合这个显式背压等待区中的 fanin 观察；每次 Submit 开头的 + // opportunistic EfDrain 仍保留逐条 Atomic，不能仅凭 site 名称全局聚合。 + const uint32_t poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, TraceAtomicSiteMask(AtomicSite::FaninFlagLoad) + ); + // 退出条件只有 occupied_count 重新低于可用容量；依赖尚未 ready 时 SpinHint 后继续重试。 + while (worker.occupied_count >= kUsableSlots) { + waited = true; + ++stats.result.wait_iterations[0]; + if (DrainReady( + state, worker, DrainPlace::RingBackpressure, stats + ) == 0) { + Ops::SpinHint(); + } + } + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + if (waited) { + ++stats.result.wait_events[0]; + const uint64_t wait_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::RingBp, + ProfilePhase::WaitForSlot, wait_begin, wait_end, 0, 0 + ); + } else if constexpr (Profile) { + AccumulatePhase( + stats.result, ProfilePhase::WaitForSlot, wait_begin, + TraceTimestamp(stats.trace, stats.result) + ); + } +} + +template +PA_DEVICE bool HeapGuard( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, uint64_t output_bytes, + LocalStats &stats +) { + // 只有产生新输出的 winner 需要保护环形 heap；retire=frontier-H 对应已经允许复用的最老任务 vend。 + // 等待期间也主动 drain 本核已就绪 slot，避免只自旋而阻塞能够推动 frontier 的 kernel。 + if (output_bytes == 0 || state->heap_base == 0) { + return true; + } + const uint64_t ring = state->heap_size; + ++stats.result.heap_guards; + const uint64_t wait_begin = TraceTimestamp(stats.trace, stats.result); + bool waited = false; + bool poll_region_active = false; + uint32_t poll_region = 0; + // 正常出口是 heap_next-vend 落入一个 ring；检测到不可能释放的覆盖或其他核 fatal 时返回失败。 + while (!IsFatal(state, stats, static_cast(task_id))) { + // 逻辑 heap 尚未走完第一圈时，所有物理输出区间都位于 [0, heap_next)， + // 不可能覆盖此前分配；保留上面的 fatal 原子检查后，可直接跳过 frontier/vend 读取。 + if (worker.heap_next <= ring) { + if (poll_region_active) { + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + } + if constexpr (Profile) { + AccumulatePhase( + stats.result, ProfilePhase::HeapGuard, wait_begin, + TraceTimestamp(stats.trace, stats.result) + ); + } + return true; + } + // 与真实 PA 一样，首圈 fast path 上方的 FatalPoll 仍是直接记录；只有 + // 确认进入 heap wrap 慢路径后，才开启本等待 episode 的四类观察聚合。 + if (!poll_region_active) { + poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, + TraceAtomicSiteMask(AtomicSite::FatalPoll) | + TraceAtomicSiteMask(AtomicSite::HeapFrontierLoad) | + TraceAtomicSiteMask(AtomicSite::HeapVendLoad) | + TraceAtomicSiteMask(AtomicSite::FaninFlagLoad) + ); + poll_region_active = true; + } + const int64_t frontier = LoadLine( + state->frontier, stats, AtomicSite::HeapFrontierLoad, static_cast(task_id) + ); + const int64_t retire = frontier - static_cast(state->heap_window); + const uint64_t vend = retire < 0 + ? 0 + : TraceAtomicLoad( + stats.trace, stats.result, static_cast(task_id), AtomicSite::HeapVendLoad, + &state->tasks[retire].vend + ); + if (worker.heap_next - vend <= ring) { + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + if (waited) { + ++stats.result.wait_events[1]; + const uint64_t wait_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::RingBp, + ProfilePhase::HeapGuard, wait_begin, wait_end, 0, 1 + ); + } else if constexpr (Profile) { + AccumulatePhase( + stats.result, ProfilePhase::HeapGuard, wait_begin, + TraceTimestamp(stats.trace, stats.result) + ); + } + return true; + } + if (frontier >= static_cast(task_id) - 1) { + SetFatal(state, stats, static_cast(task_id)); + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + return false; + } + waited = true; + ++stats.result.wait_iterations[1]; + if (DrainReady( + state, worker, DrainPlace::RingBackpressure, stats + ) == 0) { + Ops::SpinHint(); + } + } + if (poll_region_active) { + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + } + if (waited) { + ++stats.result.wait_events[1]; + const uint64_t wait_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::RingBp, + ProfilePhase::HeapGuard, wait_begin, wait_end, 0, 1 + ); + } + return false; +} + +struct ClaimOutcome { + bool attempted; + bool won; + uint64_t retries; + int32_t function_id; +}; + +template +PA_DEVICE ClaimOutcome Claim( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, TaskKind kind, + LocalStats &stats +) { + // Claim 在四个 shard 的单调 cursor 上执行 atomicMax：同一 task 只有观察到旧值更小的竞争者获胜。 + // Alloc 由全部 96 个 worker 竞争；QK/PV 仅 32 个 AIC，SF/UP 仅 64 个 AIV 进入真正的 atomicMax。 + ClaimOutcome outcome{false, false, 0, -1}; + if (task_id >= kTaskCellCapacity) { + return outcome; + } + PA_GM AtomicLine *cursor = nullptr; + if (kind == TaskKind::Alloc) { + cursor = &state->alloc_cursor[task_id % kCursorShards]; + } else { + // Mirror MixedKernels::to_active_mask(), core_mask(), popcount(), + // lane_active(), and self->role routing inside the real Claim span. + const int32_t aic_kernel = kind == TaskKind::Qk || kind == TaskKind::Pv ? FunctionId(kind) : -1; + const int32_t aiv0_kernel = kind == TaskKind::Sf || kind == TaskKind::Up ? FunctionId(kind) : -1; + const int32_t aiv1_kernel = -1; + uint8_t active_mask = 0; + if (aic_kernel >= 0) active_mask |= 1U; + if (aiv0_kernel >= 0) active_mask |= 2U; + if (aiv1_kernel >= 0) active_mask |= 4U; + const uint8_t core_mask = active_mask & 0x07U; + const int32_t active_count = __builtin_popcount(static_cast(core_mask)); + // 这里保留生产 Claim 的 lane-mask 路由边界。当前固定 PA 图按构造只生成单 lane + // 的 QK/PV 或 SF/UP；需要两个及以上 lane 协作的 joint task 本应进入 BlockWon + // 协议，本独立用例没有实现该动态路径，因此显式拒绝而不把它误当成单 lane task。 + if (active_count >= 2) { + return outcome; + } + if ((core_mask & 1U) != 0) { + if (worker.role != CoreRole::Aic) return outcome; + cursor = &state->cube_cursor[task_id % kCursorShards]; + outcome.function_id = aic_kernel; + } else if ((core_mask & 6U) != 0) { + if (worker.role != CoreRole::Aiv) return outcome; + cursor = &state->vector_cursor[task_id % kCursorShards]; + outcome.function_id = (core_mask & 2U) != 0 ? aiv0_kernel : aiv1_kernel; + } else { + return outcome; + } + } + outcome.attempted = true; + // atomicMax 返回写入前的 cursor：old=task_id 则必须 Replay。 + const int64_t old = TraceAtomicFetchMax( + stats.trace, stats.result, static_cast(task_id), AtomicSite::ClaimMax, + &cursor->value, static_cast(task_id), outcome.retries + ); + outcome.won = old < static_cast(task_id); + if (!outcome.won) outcome.function_id = -1; + return outcome; +} + +PA_DEVICE void RecordClaimOutcome(LocalStats &stats, TaskKind kind, const ClaimOutcome &outcome) { + if (outcome.attempted) ++stats.result.claim_attempts; + stats.result.cas_retries += outcome.retries; + if (outcome.won) { + ++stats.result.claim_wins; + ++stats.result.wins[KindIndex(kind)]; + } +} + +template +PA_DEVICE bool BuildWinner( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, TaskKind kind, + const TaskArgs &args, const SubmitContext &context, + const int32_t fanin[kMaxFanin], uint32_t fanin_count, LocalStats &stats +) { + // kernel winner 不在 Submit 内立即执行计算，而是把完整 payload 和 fanin 存入自己的私有 ring slot。 + // 后续 EfDrain/背压 drain/最终 drain 在依赖满足后执行它，这正是 PA 的 Submit 与执行解耦点。 + WaitForSlot(state, worker, task_id, stats); + if (!HeapGuard( + state, worker, task_id, context.output_bytes, stats + )) { + return false; + } + const int32_t slot_index = FindFreeSlot(worker); + if (slot_index < 0) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + PA_GM LocalSlot &slot = worker.slots[slot_index]; + // Match dist_submit_alloc_slot(): reserve and account the private slot + // before build_ring_slot_from_submit publishes its completed payload. + // 状态按“occupied 占位 -> built 清零 -> 计入占用 -> BuildSlotPayload”推进；后者会先 + // 置 built，再填充 payload。slot 为 worker 私有、没有跨核发布竞争，所以此处的 built + // 只是复刻生产状态机与构建成本，不承担对其他核发布完整 payload 的同步语义。 + slot.occupied = true; + slot.built = 0; + ++worker.occupied_count; + if (worker.occupied_count > stats.max_occupied) { + stats.max_occupied = worker.occupied_count; + } + const int32_t sub_block_id = worker.lane == 2 ? 1 : 0; + BuildSlotPayload( + slot, task_id, static_cast(FunctionId(kind)), 0, args, context, fanin, fanin_count, + sub_block_id + ); + stats.result.slot_tensor_copies += static_cast(context.tensor_count); + stats.result.slot_scalar_copies += static_cast(context.scalar_count); + stats.result.fanin_edges += fanin_count; + return true; +} + +template +PA_DEVICE bool SubmitTask( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_count, TaskKind kind, + const TaskArgs &args, SubmitContext &context, LocalStats &stats, PmuContext &pmu_context +) { + // 每个 worker 都完整回放相同 task stream。主流程为：EfDrain -> materialize -> TensorMap retire + // -> Claim -> winner 收集 fanin -> 全员 register -> winner Build / loser Replay。Alloc 在 Claim 前 register， + // 且 winner 不入 kernel slot，而是在 heap guard 后直接发布完成。 + BeginSubmit(worker, args, context); + const uint32_t task_id = static_cast(context.task_id); +#if PA_BUILD_SUBMIT_PMU + // PMU-only ELF 只保留首/末 Submit 的全局时间边界，不再为 1280 次调用 + // 各执行两条 trace-only SYS_CNT。 + const uint64_t submit_begin = task_id == 0 ? Ops::Now() : 0; +#else + const uint64_t submit_begin = TraceTimestamp(stats.trace, stats.result); +#endif + if (task_id == 0) { + stats.result.submit_begin = submit_begin; + } + + // EfDrain 在当前 Submit 的参数物化前执行上一批已就绪 slot，是绝大多数 kernel 的正常落点。 + // 只在这个唯一 call-site 划 PMU 边界；DrainReady 还被 ring 背压和最终 drain + // 复用，不能把 phase 插入函数体后按 place 混合累计。 + // EfDrain 是 Submit 的第一个真实阶段，直接复用父区间起点；这样每次 + // Submit 少一次 trace-only SYS_CNT，也不会留下人为的 prefix residual。 + const uint64_t efdrain_begin = submit_begin; + BeginSubmitPmuPhase(pmu_context); + DrainReady(state, worker, DrainPlace::EfDrain, stats); + EndSubmitPmuPhase(pmu_context); + const uint64_t efdrain_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::EfDrain, + ProfilePhase::EfDrain, efdrain_begin, efdrain_end + ); + + // schema-v4 的所有 Submit 子阶段都使用显式 start/end；不再通过共同 lap + // 起点生成相互覆盖的 Build/Replay/Alloc 区间。 + // 后继 segment 复用前一阶段 end：既少一次 SYS_CNT，也把前一条 trace + // 发布和阶段间胶水明确归入 Materialize，而不是留成无名 residual。 + const uint64_t materialize_begin = efdrain_end; + BeginSubmitPmuPhase(pmu_context); + const bool materialized = + MaterializeTask(worker, task_id, args, context, state->heap_base, state->heap_size); + if (!materialized) { + EndSubmitPmuPhase(pmu_context); + SetFatal(state, stats, static_cast(task_id)); + return false; + } + stats.result.materialized_outputs += context.result.count; + EndSubmitPmuPhase(pmu_context); + const uint64_t materialize_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::Materialize, + ProfilePhase::Materialize, materialize_begin, materialize_end, 0, + kind == TaskKind::Alloc ? 1U : 0U + ); + const uint64_t prepare_begin = materialize_end; + AdvanceTensorMap(worker.map, task_id, static_cast(state->heap_window)); + const uint64_t prepare_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::PrepareMap, + ProfilePhase::PrepareMap, prepare_begin, prepare_end, 0, kind == TaskKind::Alloc ? 1U : 0U + ); + + bool winner = false; + int32_t function_id = -1; + + if (kind == TaskKind::Alloc) { + // Alloc 没有 kernel lane，96 个 worker 都维护本地物化/heap 状态，但只有 Claim winner 发布全局完成。 + const uint64_t register_begin = prepare_end; + BeginSubmitPmuPhase(pmu_context); + RegisterOutputs(context, args, false); + EndSubmitPmuPhase(pmu_context); + const uint64_t register_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::Register, + ProfilePhase::Register, register_begin, register_end, 0, 0 + ); + + const uint64_t claim_begin = register_end; + BeginSubmitPmuPhase(pmu_context); + const ClaimOutcome claim = Claim(state, worker, task_id, kind, stats); + winner = claim.won; + context.won = winner; + context.kernel_id = claim.function_id; + // Claim 的本地结果归档属于同一阶段；放在共同 end 边界内，避免把 + // winner/context/stat bookkeeping 留成无法归因的 Submit residual。 + RecordClaimOutcome(stats, kind, claim); + EndSubmitPmuPhase(pmu_context); + const uint64_t claim_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::Claim, + ProfilePhase::Claim, claim_begin, claim_end, + (winner ? kClaimWon : 0U) | (claim.attempted ? kClaimAttempted : 0U), 1 + ); + // 每个 task 只有 1/96 worker 进入 winner 重型路径。把该分支标成冷路 + // 只影响基本块布局，使占绝大多数的 loser 尽量顺序进入 Submit 公共 + // 尾部；不改变 Claim 结果、完成发布或泳道边界。 + if (__builtin_expect(winner, 0)) { + const uint64_t alloc_complete_begin = claim_end; + if (!HeapGuard( + state, worker, task_id, context.output_bytes, stats + )) { + return false; + } + CompleteTask(state, worker, task_id, stats); + const uint64_t alloc_complete_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, + TracePhase::AllocComplete, ProfilePhase::ReplayTail, + alloc_complete_begin, alloc_complete_end + ); + } else { + // standalone 的 Alloc loser 没有真实 GM/Replay 动作，不再为业务 + // 路径名字写一条零时长记录。Claim 后到 Submit.end 的真实 + // scalar 时间由离线 submit_tail_gap 补集展示；排他报告将其汇总为 + // submit_tail_residual，避免伪装成 Alloc loser 业务阶段。 + } + } else { + const uint64_t claim_begin = prepare_end; + BeginSubmitPmuPhase(pmu_context); + const ClaimOutcome claim = Claim(state, worker, task_id, kind, stats); + winner = claim.won; + function_id = claim.function_id; + context.won = winner; + context.kernel_id = function_id; + RecordClaimOutcome(stats, kind, claim); + EndSubmitPmuPhase(pmu_context); + const uint64_t claim_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, TracePhase::Claim, + ProfilePhase::Claim, claim_begin, claim_end, + (winner ? kClaimWon : 0U) | (claim.attempted ? kClaimAttempted : 0U), 0 + ); + uint64_t register_begin = claim_end; + if (winner) { + const uint64_t fanin_begin = claim_end; + context.fanin_count = static_cast(CollectFanin(worker.map, args, context.fanin)); + stats.result.map_lookups += static_cast(args.tensor_count) - context.result.count; + const uint64_t fanin_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, TracePhase::Fanin, + ProfilePhase::Fanin, fanin_begin, fanin_end, 0, static_cast(context.fanin_count) + ); + register_begin = fanin_end; + } + + // loser 直接承接 Claim.end；winner 则承接 Fanin.end。两条路径都 + // 复用已有边界，不再为 Register 单独读取 SYS_CNT。 + BeginSubmitPmuPhase(pmu_context); + RegisterOutputs(context, args, true); + stats.result.map_inserts += CountBits(context.register_mask); + EndSubmitPmuPhase(pmu_context); + const uint64_t register_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, TracePhase::Register, + ProfilePhase::Register, register_begin, register_end, 0, 1 + ); + // BuildWinner 会内联 ring、heap 和真计算提交逻辑；提示其为 1/96 + // 冷路，避免 loser 为跳过大块代码付出额外取指代价。 + if (__builtin_expect(winner, 0)) { + const uint64_t winner_build_begin = register_end; + if (!BuildWinner( + state, worker, task_id, kind, args, context, context.fanin, + static_cast(context.fanin_count), stats + )) { + return false; + } + const uint64_t winner_build_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::WinnerBuild, ProfilePhase::ReplayTail, + winner_build_begin, winner_build_end + ); + } else { + // 非 winner 不占用私有 ring slot，也没有可单列的 Replay 计算。 + // Register 后到 Submit.end 由离线 submit_tail_gap 补集展示；排他 + // 报告汇总为 submit_tail_residual，避免零时长 marker 增加 raw + // 体积和 trace-buffer 写开销。 + } + } + + ++stats.result.submits; +#if PA_BUILD_SUBMIT_PMU + const uint64_t submit_end = task_id + 1 == task_count ? Ops::Now() : 0; +#else + const uint64_t submit_end = TraceTimestamp(stats.trace, stats.result); +#endif + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, TracePhase::Submit, + ProfilePhase::Submit, submit_begin, submit_end, winner ? 1U : 0U, kind == TaskKind::Alloc ? 1U : 0U + ); + if (task_id + 1 == task_count) { + stats.result.submit_end = submit_end; + } + return true; +} + +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) +// The all-task callback path crosses the split-TU finish boundary through this +// fixed 16-byte POD. No callback closure or nested thunk is stored in it. +struct LazySampleCallbackTicket { + uint64_t submit_begin; + uint32_t task_id; + int16_t function_id; + uint8_t won; + uint8_t reserved; +}; +static_assert(sizeof(LazySampleCallbackTicket) == 16, "lazy sample callback ticket must remain a 16-byte POD"); +static_assert(offsetof(LazySampleCallbackTicket, submit_begin) == 0, "lazy sample callback ticket timestamp offset mismatch"); +static_assert(offsetof(LazySampleCallbackTicket, task_id) == 8, "lazy sample callback ticket task offset mismatch"); +static_assert(offsetof(LazySampleCallbackTicket, function_id) == 12, "lazy sample callback ticket function offset mismatch"); +static_assert(offsetof(LazySampleCallbackTicket, won) == 14, "lazy sample callback ticket winner offset mismatch"); + +PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { + // This is BeginSubmit without an already-materialized TaskArgs. The same + // fields are completed synchronously after the single callback builds args. + const uint32_t task_id = static_cast(worker.local_index++); + context.self = &worker; + context.payload = &worker.payloads[task_id & kPayloadMask]; + context.task_id = static_cast(task_id); + context.tensor_count = 0; + context.scalar_count = 0; + context.result.task_id = task_id; + context.result.count = 0; + context.register_mask = 0; + context.output_bytes = 0; + context.fanin_count = 0; + context.kernel_id = -1; + context.won = false; + context.joint = false; + context.joint_init = false; + context.joint_block = -1; + context.joint_slot = -1; + context.joint_count = 0; +} + +#if defined(__CCE_AICORE__) || defined(__NPU_ARCH__) +#define PA_LAZY_LAMBDA_DEVICE __aicore__ +#else +#define PA_LAZY_LAMBDA_DEVICE +#endif + +template +PA_DEVICE bool BuildLazySampleCallbackArgs( + PaOrchestrationState &orch, TaskArgs &args, uint32_t batch, bool won, + LocalStats &stats +) { + LazySampleCallbackArgsBuilder callback_builder(args, Kind, won); + auto callback = [&](LazySampleCallbackArgsBuilder &builder) PA_LAZY_LAMBDA_DEVICE { + builder.Begin(); + if constexpr (Kind == TaskKind::Alloc) { + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.tile_create_info; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + } else if constexpr (Kind == TaskKind::Qk) { + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + MakeLazySampleCallbackQueryView(orch, batch); + builder.RecordView(); + return orch.query_view; + }); + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + return orch.key_cache; + }); + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + return orch.block_table; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + const uint32_t score_shape[kMaxTensorDims] = { + kPaHeads, + static_cast(orch.current_nblocks * kPaBlockSize), + 0, 0, 0 + }; + InitCreateInfo(orch.qk_create_info, score_shape, 2, DataType::Float32); + builder.RecordDynamicCreateInfo(); + return orch.qk_create_info; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_nblocks; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + + orch.current_block_offset; + }); + } else if constexpr (Kind == TaskKind::Sf) { + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.qk_scores; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + const uint32_t probability_shape[kMaxTensorDims] = { + kPaHeads, + static_cast(orch.current_nblocks * kPaBlockSize), + 0, 0, 0 + }; + InitCreateInfo(orch.sf_create_info, probability_shape, 2, DataType::Bfloat16); + builder.RecordDynamicCreateInfo(); + return orch.sf_create_info; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { return orch.scale_bits; }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_nblocks; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_valid_len; + }); + } else if constexpr (Kind == TaskKind::Pv) { + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.sf_probs; + }); + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + return orch.value_cache; + }); + builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + return orch.block_table; + }); + builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.tile_create_info; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_nblocks; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + + orch.current_block_offset; + }); + } else { + static_assert(Kind == TaskKind::Up, "unsupported PA task kind"); + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.sf_max; + }); + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.sf_sum; + }); + builder.AddGmInput([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.pv_output; + }); + builder.AddGmInout([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.accumulated_max; + }); + builder.AddGmInout([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.accumulated_sum; + }); + builder.AddGmInout([&]() PA_LAZY_LAMBDA_DEVICE -> PA_GM const TensorDesc & { + return *orch.accumulated_output; + }); + builder.AddLocalInout([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { + MakeLazySampleCallbackOutputView(orch, batch); + builder.RecordView(); + return orch.output_view; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_block_offset == 0 ? 1 : 0; + }); + builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { + return orch.current_block_offset + orch.current_nblocks >= orch.current_blocks ? 1 : 0; + }); + } + }; + + callback(callback_builder); + if (!callback_builder.Valid()) return false; + const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); + stats.result.arg_resets += counts.reset_calls; + stats.result.views_created += counts.views_created; + stats.result.dynamic_create_infos += counts.dynamic_create_infos; + stats.result.tensor_args_added += counts.tensor_args_added; + stats.result.scalar_args_added += counts.scalar_args_added; + return true; +} + +#undef PA_LAZY_LAMBDA_DEVICE + +template +PA_DEVICE bool FinishLazySampleCallbackSubmitBody( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_count, + const TaskArgs &args, SubmitContext &context, LocalStats &stats, + PmuContext &pmu_context, const LazySampleCallbackTicket &ticket +) { + const uint32_t task_id = ticket.task_id; + const TaskKind kind = GetTaskKind(task_id); + const int32_t function_id = static_cast(ticket.function_id); + const bool winner = ticket.won != 0; + + // The callback has ended before this body consumes TaskArgs. No closure + // or nested thunk escapes its lifetime. Inline shapes instantiate this in + // the caller; split shapes instantiate it inside the runtime finish TU. + const uint64_t materialize_begin = TraceTimestamp(stats.trace, stats.result); + BeginSubmitPmuPhase(pmu_context); + const bool materialized = + MaterializeTask(worker, task_id, args, context, state->heap_base, state->heap_size); + if (!materialized) { + EndSubmitPmuPhase(pmu_context); + SetFatal(state, stats, static_cast(task_id)); + return false; + } + stats.result.materialized_outputs += context.result.count; + EndSubmitPmuPhase(pmu_context); + const uint64_t materialize_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Materialize, ProfilePhase::Materialize, + materialize_begin, materialize_end, 0, kind == TaskKind::Alloc ? 1U : 0U + ); + + const uint64_t prepare_begin = materialize_end; + AdvanceTensorMap(worker.map, task_id, static_cast(state->heap_window)); + const uint64_t prepare_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::PrepareMap, ProfilePhase::PrepareMap, + prepare_begin, prepare_end, 0, kind == TaskKind::Alloc ? 1U : 0U + ); + + uint64_t register_begin = prepare_end; + if (kind != TaskKind::Alloc && __builtin_expect(winner, 0)) { + const uint64_t fanin_begin = prepare_end; + context.fanin_count = static_cast(CollectFanin(worker.map, args, context.fanin)); + stats.result.map_lookups += static_cast(args.tensor_count) - context.result.count; + const uint64_t fanin_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Fanin, ProfilePhase::Fanin, + fanin_begin, fanin_end, 0, static_cast(context.fanin_count) + ); + register_begin = fanin_end; + } + + BeginSubmitPmuPhase(pmu_context); + RegisterOutputs(context, args, kind != TaskKind::Alloc); + if (kind != TaskKind::Alloc) stats.result.map_inserts += CountBits(context.register_mask); + EndSubmitPmuPhase(pmu_context); + const uint64_t register_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Register, ProfilePhase::Register, + register_begin, register_end, 0, kind == TaskKind::Alloc ? 0U : 1U + ); + + if (__builtin_expect(winner, 0)) { + const uint64_t winner_build_begin = register_end; + if (kind == TaskKind::Alloc) { + if (!HeapGuard(state, worker, task_id, context.output_bytes, stats)) { + return false; + } + CompleteTask(state, worker, task_id, stats); + } else { + if (!BuildWinner( + state, worker, task_id, kind, args, context, context.fanin, + static_cast(context.fanin_count), stats + )) { + return false; + } + } + const uint64_t winner_build_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + kind == TaskKind::Alloc ? TracePhase::AllocComplete : TracePhase::WinnerBuild, + ProfilePhase::ReplayTail, + winner_build_begin, winner_build_end + ); + } + + ++stats.result.submits; +#if PA_BUILD_SUBMIT_PMU + const uint64_t submit_end = task_id + 1 == task_count ? Ops::Now() : 0; +#else + const uint64_t submit_end = TraceTimestamp(stats.trace, stats.result); +#endif + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Submit, ProfilePhase::Submit, + ticket.submit_begin, submit_end, winner ? 1U : 0U, 0 + ); + if (task_id + 1 == task_count) { + stats.result.submit_end = submit_end; + } + return true; +} + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) +template +PA_DEVICE uint32_t FinishSplitLazySampleCallbackFromRuntime( + const LazySampleCallbackTicket *ticket, const TaskArgs *args +) { + LazySampleSplitRuntimeState &runtime = Ops::LazySampleSplitState(); + const uint64_t state_address = reinterpret_cast(&runtime); + runtime.finish_state_address = state_address; + + bool valid = ticket != nullptr && args != nullptr && runtime.scheduler != nullptr && + runtime.worker != nullptr && runtime.task_count != 0 && + runtime.worker_id < kWorkers && runtime.owner_worker_id == runtime.worker_id && + runtime.worker->core_idx == static_cast(runtime.worker_id) && + runtime.caller_state_address == state_address && + runtime.state_cookie == LazySampleSplitStateCookie( + runtime.worker_id, runtime.worker->role + ) && runtime.reserved == 0; + if (valid) { + valid = ticket->reserved == 0 && ticket->task_id < runtime.task_count && + runtime.context.task_id == static_cast(ticket->task_id) && + runtime.context.kernel_id == static_cast(ticket->function_id) && + runtime.context.won == (ticket->won != 0); + } + ++runtime.finish_calls; + if (ticket != nullptr) runtime.task_id_sum += ticket->task_id; + if (!valid) { + ++runtime.protocol_errors; + if (runtime.scheduler != nullptr) { + SetFatal( + runtime.scheduler, runtime.stats, + ticket == nullptr ? -1 : static_cast(ticket->task_id) + ); + } + return 0; + } + + // split finish 不让 caller 的 SubmitContext/LocalStats/PMU 对象跨过 + // noinline 边界。诊断构建只允许 phase=none：权威 CNT6/7 仍由 caller + // 外层完整窗口读取，finish 内部不做 read-clear 局部快照。 +#if PA_BUILD_SUBMIT_PMU + static_assert( + kCompiledSubmitPmuPhase == SubmitPmuPhase::None, + "split callback submit-PMU supports only the whole-window none phase" + ); +#endif + bool pmu_context = false; + return FinishLazySampleCallbackSubmitBody( + runtime.scheduler, *runtime.worker, runtime.task_count, *args, + runtime.context, runtime.stats, pmu_context, *ticket + ) ? 1U : 0U; +} +#endif + +template +PA_DEVICE bool SubmitLazySampleCallback( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_count, + PaOrchestrationState &orch, TaskArgs &args, uint32_t batch, + SubmitContext &context, LocalStats &stats, PmuContext &pmu_context +) { + BeginLazySampleCallbackSubmit(worker, context); + const uint32_t task_id = static_cast(context.task_id); +#if PA_BUILD_SUBMIT_PMU + const uint64_t submit_begin = task_id == 0 ? Ops::Now() : 0; +#else + const uint64_t submit_begin = TraceTimestamp(stats.trace, stats.result); +#endif + if (task_id == 0) stats.result.submit_begin = submit_begin; + + const uint64_t efdrain_begin = submit_begin; + BeginSubmitPmuPhase(pmu_context); + DrainReady(state, worker, DrainPlace::EfDrain, stats); + EndSubmitPmuPhase(pmu_context); + const uint64_t efdrain_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, + TracePhase::EfDrain, ProfilePhase::EfDrain, + efdrain_begin, efdrain_end + ); + + const uint64_t claim_begin = efdrain_end; + BeginSubmitPmuPhase(pmu_context); + const ClaimOutcome claim = Claim(state, worker, task_id, Kind, stats); + context.won = claim.won; + context.kernel_id = claim.function_id; + RecordClaimOutcome(stats, Kind, claim); + EndSubmitPmuPhase(pmu_context); + const uint64_t claim_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), claim.function_id, + TracePhase::Claim, ProfilePhase::Claim, + claim_begin, claim_end, + (claim.won ? kClaimWon : 0U) | (claim.attempted ? kClaimAttempted : 0U), + Kind == TaskKind::Alloc ? 1U : 0U + ); + + if (!BuildLazySampleCallbackArgs(orch, args, batch, claim.won, stats)) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + const LazySampleCallbackTicket ticket{ + submit_begin, + task_id, + // All standalone function IDs are -1..3 and exactly fit this ABI field. + static_cast(claim.function_id), + static_cast(claim.won ? 1 : 0), + 0, + }; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + (void)state; + (void)worker; + (void)task_count; + (void)context; + (void)stats; + (void)pmu_context; + return Ops::FinishLazySampleCallback(&ticket, &args); +#else + return FinishLazySampleCallbackSubmitBody( + state, worker, task_count, args, context, stats, pmu_context, ticket + ); +#endif +} +#endif + +PA_DEVICE uint32_t CountLiveMapEntries(PA_GM const TensorMap &map) { + uint32_t free_entries = 0; + for (int32_t current = map.free_head; current >= 0; current = map.entries[current].next_in_bucket) { + ++free_entries; + } + return static_cast(map.high_water) - free_entries; +} + +template +PA_DEVICE void PublishResult(PA_GM WorkerResult &destination, const WorkerResult &source) { + // 每个 worker 只写自己独占、覆盖多条 cache line 的 WorkerResult 分区；逐字段 + // bypass 保证结果对 host 可见，而独立 sidecar 允许 D2H 只搬结果、不搬约 9 MiB WorkerState。 +#define PA_PUBLISH_FIELD(field) Ops::Publish(&destination.field, source.field) + PA_PUBLISH_FIELD(submit_begin); + PA_PUBLISH_FIELD(submit_end); + PA_PUBLISH_FIELD(finish_cycle); + PA_PUBLISH_FIELD(checksum); + PA_PUBLISH_FIELD(submits); + PA_PUBLISH_FIELD(claim_attempts); + PA_PUBLISH_FIELD(claim_wins); + PA_PUBLISH_FIELD(heap_guards); + PA_PUBLISH_FIELD(fanin_ready_loads); + PA_PUBLISH_FIELD(completion_duplicates); + PA_PUBLISH_FIELD(cas_retries); + PA_PUBLISH_FIELD(joint_polls); + for (uint32_t index = 0; index < static_cast(TaskKind::Count); ++index) { + Ops::Publish(&destination.wins[index], source.wins[index]); + } + for (uint32_t index = 0; index < 4; ++index) { + Ops::Publish(&destination.kernel_counts[index], source.kernel_counts[index]); + Ops::Publish(&destination.kernel_cycles[index], source.kernel_cycles[index]); + Ops::Publish(&destination.kernel_min_cycles[index], source.kernel_min_cycles[index]); + Ops::Publish(&destination.kernel_max_cycles[index], source.kernel_max_cycles[index]); + } + for (uint32_t index = 0; index < static_cast(DrainPlace::Count); ++index) { + Ops::Publish(&destination.placement[index], source.placement[index]); + } + for (uint32_t index = 0; index < static_cast(ProfilePhase::Count); ++index) { + Ops::Publish(&destination.phase_cycles[index], source.phase_cycles[index]); + Ops::Publish(&destination.phase_calls[index], source.phase_calls[index]); + } + for (uint32_t index = 0; index < 2; ++index) { + Ops::Publish(&destination.wait_events[index], source.wait_events[index]); + Ops::Publish(&destination.wait_iterations[index], source.wait_iterations[index]); + } + PA_PUBLISH_FIELD(context_reads); + PA_PUBLISH_FIELD(views_created); + PA_PUBLISH_FIELD(dynamic_create_infos); + PA_PUBLISH_FIELD(arg_resets); + PA_PUBLISH_FIELD(tensor_args_added); + PA_PUBLISH_FIELD(scalar_args_added); + PA_PUBLISH_FIELD(materialized_outputs); + PA_PUBLISH_FIELD(map_inserts); + PA_PUBLISH_FIELD(map_lookups); + PA_PUBLISH_FIELD(slot_tensor_copies); + PA_PUBLISH_FIELD(slot_scalar_copies); + PA_PUBLISH_FIELD(fanin_edges); + PA_PUBLISH_FIELD(final_heap_next); + PA_PUBLISH_FIELD(map_high_water); + PA_PUBLISH_FIELD(map_alive_floor); + PA_PUBLISH_FIELD(map_cleaned_upto); + PA_PUBLISH_FIELD(map_live_entries); + PA_PUBLISH_FIELD(worker_id); + PA_PUBLISH_FIELD(role); + PA_PUBLISH_FIELD(max_occupied); + PA_PUBLISH_FIELD(final_occupied); + PA_PUBLISH_FIELD(fanin_not_ready_loads); + PA_PUBLISH_FIELD(frontier_initial_loads); + PA_PUBLISH_FIELD(frontier_updates); + PA_PUBLISH_FIELD(frontier_terminal_loads); + PA_PUBLISH_FIELD(atomic_trace_calls); +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + PA_PUBLISH_FIELD(lazy_sample_split_caller_state_address); + PA_PUBLISH_FIELD(lazy_sample_split_finish_state_address); + PA_PUBLISH_FIELD(lazy_sample_split_finish_calls); + PA_PUBLISH_FIELD(lazy_sample_split_protocol_errors); + PA_PUBLISH_FIELD(lazy_sample_split_state_cookie); + PA_PUBLISH_FIELD(lazy_sample_split_task_id_sum); + PA_PUBLISH_FIELD(lazy_sample_split_owner_worker_id); + PA_PUBLISH_FIELD(lazy_sample_split_reserved); +#endif +#undef PA_PUBLISH_FIELD + Ops::StoreBarrier(); +} + +template +PA_DEVICE void RunSchedulerImpl(PA_GM SchedulerState *state, uint32_t worker_id, CoreRole role) { + // 一个入口实例只拥有 state->workers[worker_id] 的私有 map/ring/payload；cursor、task cell 和屏障为跨核共享区。 + if (worker_id >= kWorkers) { + return; + } + PA_GM WorkerState &worker = state->workers[worker_id]; + worker.role = role; + worker.core_idx = static_cast(worker_id); + // standalone 使用连续 worker 编号：AIC 为 0..31；AIV 为 32..95。 + // 每个物理 block b 对应 AIC(b, lane0)、AIV(32+2b, lane1)、AIV(33+2b, lane2)。 + if (role == CoreRole::Aic) { + worker.block_id = static_cast(worker_id); + worker.lane = 0; + } else { + const uint32_t vector_id = worker_id - kAicWorkers; + worker.block_id = static_cast(vector_id / 2); + worker.lane = static_cast(1 + vector_id % 2); + } + worker.sub_block_id = worker.lane == 2 ? 1 : 0; + worker.local_index = 0; + worker.heap_next = 0; + ResetTensorMap(worker.map); + worker.occupied_count = 0; + worker.owned_total = 0; + worker.swimlane_last_cycle = 0; + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + worker.slots[index].occupied = false; + worker.slots[index].built = false; + } + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + LazySampleSplitRuntimeState &lazy_sample_split_runtime = Ops::LazySampleSplitState(); + lazy_sample_split_runtime.context = SubmitContext{}; + lazy_sample_split_runtime.stats = LocalStats{}; + lazy_sample_split_runtime.scheduler = state; + lazy_sample_split_runtime.worker = &worker; + lazy_sample_split_runtime.task_count = 0; + lazy_sample_split_runtime.worker_id = worker_id; + lazy_sample_split_runtime.caller_state_address = reinterpret_cast(&lazy_sample_split_runtime); + lazy_sample_split_runtime.finish_state_address = 0; + lazy_sample_split_runtime.finish_calls = 0; + lazy_sample_split_runtime.protocol_errors = 0; + lazy_sample_split_runtime.state_cookie = LazySampleSplitStateCookie(worker_id, role); + lazy_sample_split_runtime.task_id_sum = 0; + lazy_sample_split_runtime.owner_worker_id = worker_id; + lazy_sample_split_runtime.reserved = 0; + LocalStats &stats = lazy_sample_split_runtime.stats; +#else + LocalStats stats{}; +#endif + stats.result.worker_id = worker_id; + stats.result.role = static_cast(role); + stats.result.checksum = 0xcbf29ce484222325ULL ^ worker_id; + // 该 invalidate 原先藏在 AttachTrace 中；它同时保护 PMU mode/register + // table 与 winner workload，必须在两个构建中都执行。 + Ops::InvalidateRegion( + &state->config, sizeof(state->config) + sizeof(state->winner_workload) + ); + stats.trace = AttachTrace(state, worker, worker_id); + + // 96 个参与者全部完成本地状态初始化后再进入 task 0，主要用于压低启动偏斜对 + // winner 分布和 Submit 时序的干扰；atomicMax 的唯一 winner 正确性本身不依赖该屏障。 + TraceAtomicFetchAdd( + stats.trace, stats.result, -1, AtomicSite::StartupIncrement, + &state->started_count.value, 1 + ); + const uint64_t start_wait = Ops::Now(); + uint32_t start_polls = 0; + const uint32_t startup_poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, + TraceAtomicSiteMask(AtomicSite::StartupPoll) | TraceAtomicSiteMask(AtomicSite::FatalPoll) + ); + // 全员到齐或任一核发布 fatal 即退出启动等待；watchdog 防止缺失参与者造成永久挂死。 + while (LoadLine(state->started_count, stats, AtomicSite::StartupPoll) < + static_cast(state->config.workers) && + !IsFatal(state, stats)) { + Ops::SpinHint(); + if (WatchdogExpired(state, stats, start_wait, start_polls)) { + break; + } + } + AtomicPollRegionEnd(stats.trace, stats.result, startup_poll_region); + + const uint32_t batches = state->config.batches; + const uint32_t task_count = batches * kTasksPerBatch; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + lazy_sample_split_runtime.task_count = task_count; +#endif + PaOrchestrationState orchestration; + TaskArgs args; +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + SubmitContext &context = lazy_sample_split_runtime.context; +#else + SubmitContext context; +#endif + uint64_t orchestration_begin = 0; + uint64_t orchestration_end = 0; + if (!IsFatal(state, stats)) { + // Case1 每个 batch 固定回放 Alloc/QK/SF/PV/UP 五个 task；所有 worker 顺序相同，执行 lane 由 Claim 筛选。 + // CCEC 可在这里开启本 worker 私有 PMU 窗口；CPU/AscendC 适配层是空实现。 + // 窗口覆盖从首个参数构造到末次 Submit 返回，与全局“首 Submit.begin～末 Submit.end” + // 口径接近但不相同，host sidecar 必须按 per-worker 累计解释。 + // 泳道父边界在 PMU-only 构建中会被编译为空，不应污染 Submit 取数；窗口从 + // orchestration 初始化（即首批参数构造）前一条边界开始。 + auto pmu_context = Ops::PmuWindowStart(state, worker_id); + orchestration_begin = TraceTimestamp(stats.trace, stats.result); + InitPaOrchestration(orchestration, batches, &state->context_lens[0]); + for (uint32_t batch = 0; batch < batches; ++batch) { +#if defined(PA_LAZY_SAMPLE_SHAPE_ID) + BeginPaBatchForLazySampleCallback(orchestration, batch); + ++stats.result.context_reads; + if (!SubmitLazySampleCallback< + TaskKind::Alloc, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Alloc, context.result); + + PreparePaBlockGroup(orchestration, 0); + if (!SubmitLazySampleCallback< + TaskKind::Qk, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Qk, context.result); + + if (!SubmitLazySampleCallback< + TaskKind::Sf, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Sf, context.result); + + if (!SubmitLazySampleCallback< + TaskKind::Pv, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Pv, context.result); + + if (!SubmitLazySampleCallback< + TaskKind::Up, Ops, Profile, decltype(pmu_context), kLazySampleLazy + >( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } +#else + BuildAllocArgs(orchestration, args, batch); + ++stats.result.context_reads; + stats.result.views_created += 2; + stats.result.tensor_args_added += 3; + if (!SubmitTask( + state, worker, task_count, TaskKind::Alloc, args, context, stats, pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Alloc, context.result); + + BuildQkArgs(orchestration, args, batch); + ++stats.result.dynamic_create_infos; + ++stats.result.arg_resets; + stats.result.tensor_args_added += 4; + stats.result.scalar_args_added += 2; + if (!SubmitTask( + state, worker, task_count, TaskKind::Qk, args, context, stats, pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Qk, context.result); + + BuildSfArgs(orchestration, args); + ++stats.result.dynamic_create_infos; + ++stats.result.arg_resets; + stats.result.tensor_args_added += 4; + stats.result.scalar_args_added += 3; + if (!SubmitTask( + state, worker, task_count, TaskKind::Sf, args, context, stats, pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Sf, context.result); + + BuildPvArgs(orchestration, args, batch); + ++stats.result.arg_resets; + stats.result.tensor_args_added += 4; + stats.result.scalar_args_added += 2; + if (!SubmitTask( + state, worker, task_count, TaskKind::Pv, args, context, stats, pmu_context + )) { + break; + } + AcceptTaskOutputs(orchestration, TaskKind::Pv, context.result); + + BuildUpdateArgs(orchestration, args); + ++stats.result.arg_resets; + stats.result.tensor_args_added += 7; + stats.result.scalar_args_added += 2; + if (!SubmitTask( + state, worker, task_count, TaskKind::Up, args, context, stats, pmu_context + )) { + break; + } +#endif + } + Ops::PmuWindowStop(state, worker_id, pmu_context); + orchestration_end = TraceTimestamp(stats.trace, stats.result); + } + + // replay_done 表示所有 worker 已退出回放循环（成功路径即完整提交）；之后仍需 drain 到本核 slot 为空。 + // 成功路径复用 orchestration end 作为 final drain start，使两个业务父区间 + // 首尾相接；父记录延后到 final drain 结束再写，避免记录自身落进任一业务 span。 + const uint64_t final_drain_begin = orchestration_end != 0 + ? orchestration_end + : TraceTimestamp(stats.trace, stats.result); + TraceAtomicFetchAdd( + stats.trace, stats.result, -1, AtomicSite::ReplayDoneIncrement, + &state->replay_done.value, 1 + ); + const uint32_t final_poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, + TraceAtomicSiteMask(AtomicSite::ReplayDonePoll) | TraceAtomicSiteMask(AtomicSite::FaninFlagLoad) + ); + while (true) { + const uint32_t freed = + DrainReady(state, worker, DrainPlace::FinalDrain, stats); + const bool all_replayed = + LoadLine(state->replay_done, stats, AtomicSite::ReplayDonePoll) >= + static_cast(state->config.workers); + // 必须同时满足“无人再生产新 slot”和“本核旧 slot 全部完成”，否则继续帮助系统推进 completion。 + if (all_replayed && worker.occupied_count == 0) { + break; + } + if (freed == 0) { + Ops::SpinHint(); + } + } + AtomicPollRegionEnd(stats.trace, stats.result, final_poll_region); + const uint64_t final_drain_end = TraceTimestamp(stats.trace, stats.result); + if (orchestration_begin != 0 && orchestration_end >= orchestration_begin) { + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::OrchestrationReplay, + ProfilePhase::Orchestration, orchestration_begin, orchestration_end + ); + } + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::FinalDrain, + ProfilePhase::ReplayTail, final_drain_begin, final_drain_end + ); + +#if !PA_BUILD_SUBMIT_PMU + if (stats.trace.atomics_enabled) { + // 两条基线都放在最终 drain 之后。第一条量连续 + // SYS_CNT，第二条量返回依赖钩子的固定成本；它们只描述计时底噪，不能 + // 从每条 atomic 中机械相减后宣称得到跨核全局可见性延迟。 + const uint64_t clock_begin = Ops::Now(); + const uint64_t clock_end = Ops::Now(); + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::ClockBaseline, + ProfilePhase::ReplayTail, clock_begin, clock_end + ); + const uint64_t dependency_begin = Ops::Now(); + const uint64_t dependency_end = Ops::NowAfterAtomicResult( + static_cast(worker_id) + ); + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::ClockBaseline, + ProfilePhase::ReplayTail, dependency_begin, dependency_end, + kClockAtomicDependency | + (Ops::kAtomicReturnReadyObserved ? kClockAtomicDependencyApplied : 0U) + ); + } +#endif + +#if defined(PA_LAZY_SAMPLE_SPLIT_FINISH) + const uint64_t expected_callback_task_id_sum = + static_cast(task_count) * (task_count - 1U) / 2U; + const bool split_protocol_ok = + lazy_sample_split_runtime.scheduler == state && lazy_sample_split_runtime.worker == &worker && + lazy_sample_split_runtime.task_count == task_count && lazy_sample_split_runtime.worker_id == worker_id && + lazy_sample_split_runtime.owner_worker_id == worker_id && + lazy_sample_split_runtime.caller_state_address != 0 && + lazy_sample_split_runtime.finish_state_address == lazy_sample_split_runtime.caller_state_address && + lazy_sample_split_runtime.finish_calls == task_count && + lazy_sample_split_runtime.task_id_sum == expected_callback_task_id_sum && + lazy_sample_split_runtime.state_cookie == LazySampleSplitStateCookie(worker_id, role) && + lazy_sample_split_runtime.reserved == 0; + if (!split_protocol_ok) { + ++lazy_sample_split_runtime.protocol_errors; + SetFatal(state, stats); + } + stats.result.lazy_sample_split_caller_state_address = lazy_sample_split_runtime.caller_state_address; + stats.result.lazy_sample_split_finish_state_address = lazy_sample_split_runtime.finish_state_address; + stats.result.lazy_sample_split_finish_calls = lazy_sample_split_runtime.finish_calls; + stats.result.lazy_sample_split_protocol_errors = lazy_sample_split_runtime.protocol_errors; + stats.result.lazy_sample_split_state_cookie = lazy_sample_split_runtime.state_cookie; + stats.result.lazy_sample_split_task_id_sum = lazy_sample_split_runtime.task_id_sum; + stats.result.lazy_sample_split_owner_worker_id = lazy_sample_split_runtime.owner_worker_id; + stats.result.lazy_sample_split_reserved = lazy_sample_split_runtime.reserved; +#endif + + // PA writes swimlane records through the ordinary GM cache and explicitly + // cleans each worker's record range before the kernel finishes. + FlushTraceCore(stats.trace, stats.result); + stats.result.finish_cycle = Ops::Now(); + stats.result.max_occupied = stats.max_occupied; + stats.result.final_occupied = worker.occupied_count; + stats.result.final_heap_next = worker.heap_next; + stats.result.map_high_water = static_cast(worker.map.high_water); + stats.result.map_alive_floor = static_cast(worker.map.alive_floor); + stats.result.map_cleaned_upto = static_cast(worker.map.cleaned_upto); + stats.result.map_live_entries = CountLiveMapEntries(worker.map); + PublishResult(state->results[worker_id], stats.result); +} + +template +PA_DEVICE void RunScheduler(PA_GM SchedulerState *state, uint32_t worker_id, CoreRole role) { + // 两个正式 CCEC 构建都不再携带旧 phase-profile 模板副本：swimlane 用 + // records 表达阶段，submit-pmu 使用独立 PMU 边界。其他后端暂时保留原 + // 运行时入口，保证公共 standalone 的 CPU/AscendC 回归不被 CCEC 构建切分影响。 +#if PA_BUILD_SWIMLANE || PA_BUILD_SUBMIT_PMU + RunSchedulerImpl(state, worker_id, role); +#else + // Profile 作为编译期模板参数，只在显式开启时保留阶段累计代码，关闭时不在热路径增加运行时分支。 + if (state->config.profile_phases != 0) { + RunSchedulerImpl(state, worker_id, role); + } else { + RunSchedulerImpl(state, worker_id, role); + } +#endif +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_SCHEDULER_CORE_H diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/pa_trace.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/pa_trace.h new file mode 100644 index 0000000000..c70a4e681b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/pa_trace.h @@ -0,0 +1,655 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_TRACE_H +#define PA_SCHEDULER_COMMON_PA_TRACE_H + +#include "pa_model.h" + +#ifndef PA_DEVICE_NOINLINE +#define PA_DEVICE_NOINLINE PA_DEVICE +#endif + +#ifndef PA_LOOP_NOUNROLL +#define PA_LOOP_NOUNROLL +#endif + +namespace pa_scheduler { + +// 记录区与真实 PA 一样直接拼在定长 Header 后面；worker_id 只选择自己的 +// records 分区，避免记录动作本身制造跨核共享写热点。 +PA_DEVICE PA_GM TraceRecord *GetTraceRecords(PA_GM TraceHeader *header) { + // 输入必须指向完整且按 64 byte 对齐的 trace buffer；返回值只是首条记录， + // 调用方还需按 worker_id * capacity 选择自己的分区。 + return reinterpret_cast(reinterpret_cast(header) + sizeof(TraceHeader)); +} + +struct AtomicPollBurst { + uint64_t start_cycle[kAtomicPollBatchSiteCount]; + uint32_t call_count[kAtomicPollBatchSiteCount]; + uint32_t active_mask; + uint32_t enabled_mask; +}; + +struct TraceContext { + PA_GM TraceCoreState *core; + PA_GM TraceRecord *records; + uint32_t capacity; + bool atomics_enabled; + int32_t lane; + int32_t block_id; + int32_t core_idx; + // 轮询调用数留在 worker 私有上下文，最终一次性发布到 core state； + // 等待热路不为计数再写共享/GM 状态。 + uint64_t poll_calls; + uint64_t poll_batch_records; + bool atomic_counter_overflow; + AtomicPollBurst poll_burst; +}; + +// pa_model.h 也向 host 暴露同一 raw ABI 映射，但 CCEC/AscendC 的单个 TU +// 会先以 host 语境包含该头，再实例化 device 调度器。这里保留明确的 device +// 版本，避免设备函数误调用先前已实例化的 __host__ helper。 +PA_DEVICE AtomicOp TraceAtomicSiteExpectedOp(AtomicSite site) { + switch (site) { + case AtomicSite::StartupIncrement: + case AtomicSite::ReplayDoneIncrement: + return AtomicOp::FetchAdd; + case AtomicSite::FatalSet: + case AtomicSite::CompletionVendExchange: + case AtomicSite::CompletionFlagExchange: + return AtomicOp::Exchange; + case AtomicSite::ClaimMax: + case AtomicSite::FrontierMax: + return AtomicOp::FetchMax; + default: + return AtomicOp::Load; + } +} + +PA_DEVICE int32_t TraceAtomicPollBatchIndex(AtomicSite site) { + switch (site) { + case AtomicSite::StartupPoll: + return 0; + case AtomicSite::FatalPoll: + return 1; + case AtomicSite::FaninFlagLoad: + return 2; + case AtomicSite::HeapFrontierLoad: + return 3; + case AtomicSite::HeapVendLoad: + return 4; + case AtomicSite::ReplayDonePoll: + return 5; + default: + return -1; + } +} + +PA_DEVICE AtomicSite TraceAtomicPollBatchSite(uint32_t index) { + switch (index) { + case 0: + return AtomicSite::StartupPoll; + case 1: + return AtomicSite::FatalPoll; + case 2: + return AtomicSite::FaninFlagLoad; + case 3: + return AtomicSite::HeapFrontierLoad; + case 4: + return AtomicSite::HeapVendLoad; + case 5: + return AtomicSite::ReplayDonePoll; + default: + return AtomicSite::Count; + } +} + +PA_DEVICE bool TraceAtomicSiteIsPollBatchable(AtomicSite site) { + return TraceAtomicPollBatchIndex(site) >= 0; +} + +PA_DEVICE uint32_t TraceAtomicSiteMask(AtomicSite site) { + return 1U << static_cast(site); +} + +// Attach 只缓存本 worker 的 header 状态、分区首址和物理 lane 信息。控制区 +// cache invalidate 在公共调度入口完成，不能随 submit-pmu 编译掉泳道而消失。 +template +PA_DEVICE TraceContext AttachTrace( + PA_GM SchedulerState *state, PA_GM const WorkerState &worker, uint32_t worker_id +) { + TraceContext trace{}; + trace.lane = worker.lane; + trace.block_id = worker.block_id; + trace.core_idx = static_cast(worker_id); +#if PA_BUILD_SUBMIT_PMU + // 诊断 ELF 不含 records 写入、atomic span 或 trace-only SYS_CNT；保留同一 + // TraceContext 形状只是为了复用调度协议源码。 + (void)state; + return trace; +#else + const uint64_t base = state->config.trace_base; + const uint32_t capacity = state->config.trace_records_per_core; + if ((state->config.trace_enabled & kTracePhasesEnabled) == 0 || base == 0 || capacity == 0 || + worker_id >= kWorkers) { + return trace; + } + PA_GM TraceHeader *header = reinterpret_cast(base); + if (worker_id >= header->num_cores) { + return trace; + } + trace.core = &header->cores[worker_id]; + trace.records = &GetTraceRecords(header)[static_cast(worker_id) * capacity]; + // 成功返回的不变量是 core/records/capacity 同时有效；任一前置条件失败则三者 + // 保持空值，后续 WriteTrace/FlushTraceCore 可无分支地安全退化为 no-op。 + trace.capacity = capacity; + trace.atomics_enabled = (state->config.trace_enabled & kTraceAtomicsEnabled) != 0; + trace.core->count = 0; + trace.core->dropped = 0; + trace.core->atomic_calls = 0; + trace.core->poll_calls = 0; + trace.core->poll_batch_records = 0; + trace.core->core_idx = trace.core_idx; + trace.core->block_id = trace.block_id; + trace.core->lane = trace.lane; + return trace; +#endif +} + +template +PA_DEVICE void WriteTrace( + TraceContext &trace, WorkerResult &result, int32_t task_id, int32_t function_id, + TracePhase trace_phase, ProfilePhase profile_phase, uint64_t start_cycle, uint64_t end_cycle, + uint32_t flags = 0, uint32_t auxiliary = 0 +); + +// CCEC 不让栈上的 TraceContext/WorkerResult 引用跨非内联调用。这里仅把 +// PollBatch 固定形状的 64-byte GM 写入抽成共享函数，以抑制各 phase 边界 +// 内联后的代码膨胀；参数只有 GM 指针与标量，局部 batch 状态仍由调用者维护。 +PA_DEVICE_NOINLINE bool WritePollBatchRecordRaw( + PA_GM TraceCoreState *core, PA_GM TraceRecord *records, uint32_t capacity, + uint64_t start_cycle, uint64_t end_cycle, uint32_t call_count, uint32_t site_id +) { +#if PA_BUILD_SUBMIT_PMU + (void)core; + (void)records; + (void)capacity; + (void)start_cycle; + (void)end_cycle; + (void)call_count; + (void)site_id; + return false; +#else + if (core == nullptr || records == nullptr || capacity == 0) { + return false; + } + const uint32_t slot = core->count; + if (slot >= capacity) { + core->dropped = core->dropped + 1; + return false; + } + PA_GM TraceRecord &record = records[slot]; + record.start_cycle = start_cycle; + record.end_cycle = end_cycle; + record.task_id = -1; + record.function_id = -1; + record.phase = static_cast(TracePhase::Atomic); + record.lane = core->lane; + record.block_id = core->block_id; + record.core_idx = core->core_idx; + const AtomicSite site = static_cast(site_id); + record.flags = static_cast(TraceAtomicSiteExpectedOp(site)) | + kAtomicResultUsed | kAtomicPollBatch | + (call_count << kAtomicPollCountShift); + record.auxiliary = site_id; + // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 + core->count = slot + 1; + return true; +#endif +} + +PA_DEVICE uint32_t AtomicTraceFlags( + AtomicOp op, bool result_used, bool return_ready, bool value_zero = false, + uint64_t retries = 0 +) { + // 高 24 bit 只能容纳有限重试次数；A5 硬件 atomicMax 当前报告 0，CPU CAS + // 回归若超过范围则饱和，避免溢出覆盖低位的 op/语义标志。 + constexpr uint64_t kMaxRetries = (1ULL << (32 - kAtomicRetriesShift)) - 1; + const uint32_t encoded_retries = static_cast(retries > kMaxRetries ? kMaxRetries : retries); + return static_cast(op) | (result_used ? kAtomicResultUsed : 0U) | + (value_zero ? kAtomicValueZero : 0U) | (return_ready ? kAtomicReturnReady : 0U) | + (encoded_retries << kAtomicRetriesShift); +} + +PA_DEVICE void CountAtomicCall( + TraceContext &trace, WorkerResult &result, bool poll_batch +) { + if (result.atomic_trace_calls == UINT64_MAX) { + trace.atomic_counter_overflow = true; + return; + } + ++result.atomic_trace_calls; + if (!poll_batch) return; + if (trace.poll_calls == UINT64_MAX) { + trace.atomic_counter_overflow = true; + return; + } + ++trace.poll_calls; +} + +template +PA_DEVICE void AtomicPollBoundaryAt( + TraceContext &trace, uint64_t end_cycle +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)end_cycle; +#else + if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; + const uint32_t active_mask = trace.poll_burst.active_mask; + // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 + // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 + PA_LOOP_NOUNROLL + for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { + const uint32_t bit = 1U << index; + if ((active_mask & bit) == 0) continue; + const uint32_t call_count = trace.poll_burst.call_count[index]; + if (call_count == 0 || call_count > kAtomicPollCountMax) { + trace.atomic_counter_overflow = true; + continue; + } + const AtomicSite site = TraceAtomicPollBatchSite(index); + const bool written = WritePollBatchRecordRaw( + trace.core, trace.records, trace.capacity, + trace.poll_burst.start_cycle[index], end_cycle, + call_count, static_cast(site) + ); + if (written) { + if (trace.poll_batch_records == UINT64_MAX) { + trace.atomic_counter_overflow = true; + } else { + ++trace.poll_batch_records; + } + } + trace.poll_burst.call_count[index] = 0; + } + trace.poll_burst.active_mask = 0; +#endif +} + +template +PA_DEVICE void AtomicPollBoundary(TraceContext &trace, WorkerResult &result) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; +#else + (void)result; + if (trace.poll_burst.active_mask == 0) return; + AtomicPollBoundaryAt(trace, Ops::Now()); +#endif +} + +template +PA_DEVICE uint32_t AtomicPollRegionBegin( + TraceContext &trace, WorkerResult &result, uint32_t site_mask +) { + const uint32_t previous_mask = trace.poll_burst.enabled_mask; +#if PA_BUILD_SUBMIT_PMU + (void)result; + (void)site_mask; +#else + if (!trace.atomics_enabled) return previous_mask; + AtomicPollBoundary(trace, result); + trace.poll_burst.enabled_mask = previous_mask | site_mask; +#endif + return previous_mask; +} + +template +PA_DEVICE void AtomicPollRegionEnd( + TraceContext &trace, WorkerResult &result, uint32_t previous_mask +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)previous_mask; +#else + if (!trace.atomics_enabled) return; + AtomicPollBoundary(trace, result); + trace.poll_burst.enabled_mask = previous_mask; +#endif +} + +PA_DEVICE bool AtomicPollBatchEnabled( + TraceContext &trace, AtomicSite site, AtomicOp actual_op +) { + return trace.atomics_enabled && TraceAtomicSiteIsPollBatchable(site) && + TraceAtomicSiteExpectedOp(site) == actual_op && + (trace.poll_burst.enabled_mask & TraceAtomicSiteMask(site)) != 0; +} + +template +PA_DEVICE void AccumulateAtomicPollCall( + TraceContext &trace, WorkerResult &result, AtomicSite site, uint64_t start_cycle +) { + const int32_t signed_index = TraceAtomicPollBatchIndex(site); + if (signed_index < 0) { + trace.atomic_counter_overflow = true; + return; + } + const uint32_t index = static_cast(signed_index); + const uint32_t bit = 1U << index; + if ((trace.poll_burst.active_mask & bit) == 0) { + trace.poll_burst.start_cycle[index] = start_cycle; + trace.poll_burst.call_count[index] = 0; + trace.poll_burst.active_mask |= bit; + } + uint32_t &call_count = trace.poll_burst.call_count[index]; + ++call_count; + if (call_count == kAtomicPollCountMax) { + AtomicPollBoundary(trace, result); + } +} + +template +PA_DEVICE void WriteAtomicTrace( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, AtomicOp op, + uint64_t start_cycle, uint64_t end_cycle, bool result_used, bool return_ready, + bool value_zero = false, uint64_t retries = 0 +) { + // 一次源码 atomic 只写一条同时含 start/end 的 span；结束时间先于 64B record + // 写入，因此本条区间不直接包含自己的记录写开销，但下一次竞争到达会受它影响。 + CountAtomicCall(trace, result, false); + WriteTrace( + trace, result, task_id, -1, TracePhase::Atomic, ProfilePhase::ReplayTail, + start_cycle, end_cycle, + AtomicTraceFlags(op, result_used, return_ready, value_zero, retries), + static_cast(site) + ); +} + +template +PA_DEVICE T TraceAtomicLoad( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile T *address, bool result_used = true +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::Load(address); +#else + if (!trace.atomics_enabled) return Ops::Load(address); + const bool poll_batch = result_used && AtomicPollBatchEnabled(trace, site, AtomicOp::Load); + const int32_t poll_index = poll_batch ? TraceAtomicPollBatchIndex(site) : -1; + const bool first_in_batch = poll_batch && + (trace.poll_burst.active_mask & (1U << static_cast(poll_index))) == 0; + const uint64_t begin = !poll_batch || first_in_batch ? Ops::Now() : 0; + const T old = Ops::Load(address); + if (poll_batch) { + CountAtomicCall(trace, result, true); + AccumulateAtomicPollCall(trace, result, site, begin); + return old; + } + // CCEC 只在返回值本来就参与协议判断时插入一条依赖 MOV，再读 SYS_CNT。 + // 这样不会把未消费返回值的 RED/no-return 路径强制改成返回型 ATOM。 + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::Load, begin, end, result_used, return_ready, + old == static_cast(0) + ); + return old; +#endif +} + +template +PA_DEVICE T TraceAtomicExchange( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile T *address, T value, bool result_used = false +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::Exchange(address, value); +#else + if (!trace.atomics_enabled) return Ops::Exchange(address, value); + const uint64_t begin = Ops::Now(); + const T old = Ops::Exchange(address, value); + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::Exchange, begin, end, result_used, return_ready + ); + return old; +#endif +} + +template +PA_DEVICE int64_t TraceAtomicFetchAdd( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile int64_t *address, int64_t value, bool result_used = false +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::FetchAdd(address, value); +#else + if (!trace.atomics_enabled) return Ops::FetchAdd(address, value); + const uint64_t begin = Ops::Now(); + const int64_t old = Ops::FetchAdd(address, value); + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::FetchAdd, begin, end, result_used, return_ready + ); + return old; +#endif +} + +template +PA_DEVICE int64_t TraceAtomicFetchMax( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile int64_t *address, int64_t value, uint64_t &retries, bool result_used = true +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::FetchMax(address, value, retries); +#else + if (!trace.atomics_enabled) return Ops::FetchMax(address, value, retries); + const uint64_t begin = Ops::Now(); + const int64_t old = Ops::FetchMax(address, value, retries); + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::FetchMax, begin, end, result_used, + return_ready, false, retries + ); + return old; +#endif +} + +template +PA_DEVICE void AccumulatePhase( + WorkerResult &result, ProfilePhase phase, uint64_t start_cycle, uint64_t end_cycle +) { +#if PA_BUILD_SUBMIT_PMU + (void)result; + (void)phase; + (void)start_cycle; + (void)end_cycle; + return; +#else + // phase profile 与完整泳道是两套正交机制：即使关闭 records，Profile=true + // 仍会累计用户当前关注的 Claim/EfDrain/WaitForSlot/HeapGuard 四段。 + if constexpr (Profile) { + if (phase != ProfilePhase::Claim && phase != ProfilePhase::EfDrain && + phase != ProfilePhase::WaitForSlot && phase != ProfilePhase::HeapGuard) { + return; + } + const uint32_t index = static_cast(phase); + // 调用方保证 end_cycle>=start_cycle；各后端把 Now() 归一到每 tick 1 ns 的 + // 数值标度，聚合持续时间可直接相加并在 host 侧按 1000 换算为微秒。 + const uint64_t duration = end_cycle - start_cycle; + result.phase_cycles[index] += duration; + ++result.phase_calls[index]; + } +#endif +} + +template +PA_DEVICE void WriteTrace( + TraceContext &trace, WorkerResult &result, int32_t task_id, int32_t function_id, TracePhase trace_phase, + ProfilePhase profile_phase, uint64_t start_cycle, uint64_t end_cycle, uint32_t flags, + uint32_t auxiliary +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)task_id; + (void)function_id; + (void)trace_phase; + (void)profile_phase; + (void)start_cycle; + (void)end_cycle; + (void)flags; + (void)auxiliary; + return; +#else + // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 + // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 + AccumulatePhase(result, profile_phase, start_cycle, end_cycle); + if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { + return; + } + PA_GM TraceCoreState &core = *trace.core; + const uint32_t slot = core.count; + if (slot >= trace.capacity) { + core.dropped = core.dropped + 1; + return; + } + PA_GM TraceRecord &record = trace.records[slot]; + record.start_cycle = start_cycle; + record.end_cycle = end_cycle; + record.task_id = task_id; + record.function_id = function_id; + record.phase = static_cast(trace_phase); + record.lane = trace.lane; + record.block_id = trace.block_id; + record.core_idx = trace.core_idx; + record.flags = flags; + record.auxiliary = auxiliary; + // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 + // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 + core.count = slot + 1; +#endif +} + +template +PA_DEVICE void ResetTraceLap( + TraceContext &trace, WorkerResult &result, PA_GM WorkerState &worker +) { + // lap 是后续 Build/Replay/Alloc 等覆盖式阶段的共同起点，不代表新增嵌套 span。 + // 因此分析时不能把 lap 时长再与其中的 Materialize/Claim/Register 直接相加。 +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + (void)worker; +#else + (void)result; + const uint64_t cycle = Ops::Now(); + // 与真实 FDWIC 的 TRACE_LAP_RESET 保持同一边界：等待区 PollBatch + // 只能覆盖本次逻辑轮询 episode，不能跨进下一段 lap 或计算单元执行。 + AtomicPollBoundaryAt(trace, cycle); + worker.swimlane_last_cycle = cycle; +#endif +} + +template +PA_DEVICE void FlushTraceCore(TraceContext &trace, WorkerResult &result) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)result; + return; +#else + if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { + return; + } + // 防御性关闭任何尚未由显式 region end 关闭的等待包；正常路径上 active_mask + // 应为 0，这里仍保证异常早退不会留下“有逻辑调用、无物理 batch”的半截采集。 + AtomicPollBoundary(trace, result); + PA_GM TraceCoreState &core = *trace.core; + if (trace.atomic_counter_overflow || result.atomic_trace_calls > UINT32_MAX || + trace.poll_calls > UINT32_MAX || trace.poll_batch_records > UINT32_MAX) { + if (core.dropped != UINT32_MAX) core.dropped = core.dropped + 1; + } + core.atomic_calls = static_cast(result.atomic_trace_calls); + core.poll_calls = static_cast(trace.poll_calls); + core.poll_batch_records = static_cast(trace.poll_batch_records); + const uint32_t count = core.count < trace.capacity ? core.count : trace.capacity; + // A5 侧记录经普通 GM cache 写入，kernel 结束前必须把有效 records 与最后的 + // count/dropped cache line 显式 clean，host 的 D2H 才能得到完整且自洽的快照。 + if (count != 0) { + Ops::FlushRegion(trace.records, static_cast(count) * sizeof(TraceRecord)); + } + Ops::FlushRegion(&core, sizeof(core)); +#endif +} + +template +PA_DEVICE uint64_t WriteTraceLap( + TraceContext &trace, PA_GM WorkerState &worker, WorkerResult &result, int32_t task_id, + int32_t function_id, TracePhase trace_phase, ProfilePhase profile_phase, + uint32_t flags = 0, uint32_t auxiliary = 0 +) { +#if PA_BUILD_SUBMIT_PMU + (void)trace; + (void)worker; + (void)result; + (void)task_id; + (void)function_id; + (void)trace_phase; + (void)profile_phase; + (void)flags; + (void)auxiliary; + return 0; +#else + // lap 记录区间 [上一次 Reset/WriteTraceLap, 当前时刻]，写完立即推进起点。 + // 显式 WriteTrace span 不会修改该起点，这正是生产泳道中阶段可重叠的原因。 + const uint64_t end_cycle = Ops::Now(); + // 真实 FDWIC 在 TRACE_LAP 取到结束时间后先关闭 PollBatch，再写 lap。 + // 复用同一个 end_cycle，避免额外 SYS_CNT 造成可见缝隙。 + AtomicPollBoundaryAt(trace, end_cycle); + WriteTrace( + trace, result, task_id, function_id, trace_phase, profile_phase, worker.swimlane_last_cycle, end_cycle, + flags, auxiliary + ); + worker.swimlane_last_cycle = end_cycle; + return end_cycle; +#endif +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_TRACE_H diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/winner_workload.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/winner_workload.h new file mode 100644 index 0000000000..a3995fb50f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/winner_workload.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_WINNER_WORKLOAD_H +#define PA_SCHEDULER_COMMON_WINNER_WORKLOAD_H + +#include "pa_model.h" + +#include +#include + +namespace pa_scheduler::winner_workload { + +// 三种 standalone 后端共享完全相同的 GM/host 布局。128x128 float 是 CCEC +// 已在 A5 上验证过的基本形状；后端可以采用不同指令接口，但不能改变输入、 +// 输出 tile 编址或 host 数值校验口径。 +constexpr uint32_t kTileRows = 128; +constexpr uint32_t kTileCols = 128; +constexpr size_t kTileElements = static_cast(kTileRows) * kTileCols; +constexpr size_t kTileBytes = kTileElements * sizeof(float); +constexpr uint32_t kSharedInputTiles = 2; +constexpr uint32_t kOutputTilesPerWorker = 2; +constexpr uint32_t kOutputTiles = kWorkers * kOutputTilesPerWorker; +constexpr size_t kWorkspaceTiles = kSharedInputTiles + kOutputTiles; +constexpr size_t kWorkspaceBytes = kWorkspaceTiles * kTileBytes; +constexpr float kInputAValue = 2.0F; +constexpr float kInputBValue = 3.0F; +constexpr float kExpectedAicValue = 768.0F; +constexpr float kExpectedSfValue = 5.0F; +constexpr float kExpectedUpValue = 6.0F; +constexpr float kOutputSentinel = -12345.0F; + +// 256 batch 下，即使同一 AIC 极端地拿到全部 QK/PV，128 次完整 Cube +// 迭代的 CCEC 实测 busy 上界仍低于 32-bit PMU 的 25% 门槛。其他后端也沿用 +// 此参数边界，避免相同命令在不同实现上产生不同含义。 +constexpr uint32_t kMaxRealComputeCount = 128; + +// 1 次用于最小正确性取证。默认次数来自 CCEC 的三个独立 b256 A5 进程； +// AscendC 必须重新标定后才能宣称达到同样时长，不能仅因共享默认参数便沿用 +// CCEC 的性能结论。UP 的一次完整 128x128 流水是当前正整数下限。 +constexpr WorkloadCounts kRealComputeSmokeCounts{1, 1, 1, 1}; +constexpr WorkloadCounts kDefaultRealComputeCounts{6, 28, 4, 1}; + +static_assert(kTileBytes == 65536, "real-compute tile must occupy 64 KiB"); +static_assert(kWorkspaceBytes == 12713984, "real-compute workspace size changed unexpectedly"); + +} // namespace pa_scheduler::winner_workload + +#endif // PA_SCHEDULER_COMMON_WINNER_WORKLOAD_H diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/winner_workload_host.h b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/winner_workload_host.h new file mode 100644 index 0000000000..07972cc96b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/common/winner_workload_host.h @@ -0,0 +1,361 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_WINNER_WORKLOAD_HOST_H +#define PA_SCHEDULER_COMMON_WINNER_WORKLOAD_HOST_H + +#include "host_support.h" +#include "winner_workload.h" + +#include +#include +#include + +namespace pa_scheduler::host { + +// 常量模式用于稳定的性能负载；布局诊断模式使用带权对角 A 和非对称稠密 B， +// 专门暴露 B 转置、ND/NZ stride 与输出重排错误。该选择只影响计时窗外的 +// host 输入生成和结果校验，不进入 SchedulerState，也不增加 device 热路径分支。 +enum class RealComputePattern : uint32_t { + Constant = 0, + LayoutDiagnostic = 1, +}; + +// winner 负载参数在通用 benchmark parser 前单独剥离，CCEC 可在其后继续剥离 +// PMU 参数；AscendC/CPU 则直接把剩余 argv 交给 ParseOptions。这样不把后端 +// 私有功能塞入公共 PA 参数结构，也不会复制三套互斥规则。 +struct WinnerWorkloadOptions { + WinnerWorkloadMode mode = WinnerWorkloadMode::RealCompute; + WorkloadCounts repeats = winner_workload::kDefaultRealComputeCounts; + RealComputePattern pattern = RealComputePattern::Constant; + bool counts_explicit = false; + bool pattern_explicit = false; + bool nop_override_explicit = false; +}; + +inline const char *WinnerWorkloadModeName(WinnerWorkloadMode mode) { + switch (mode) { + case WinnerWorkloadMode::ScalarNop: + return "scalar-nop"; + case WinnerWorkloadMode::RealCompute: + return "real-compute"; + } + return "invalid"; +} + +inline const char *RealComputePatternName(RealComputePattern pattern) { + switch (pattern) { + case RealComputePattern::Constant: + return "constant"; + case RealComputePattern::LayoutDiagnostic: + return "layout-diagnostic"; + } + return "invalid"; +} + +inline bool ParseWorkloadCounts(const char *raw, WorkloadCounts *counts) { + unsigned int qk = 0; + unsigned int sf = 0; + unsigned int pv = 0; + unsigned int up = 0; + char tail = '\0'; + if (std::sscanf(raw, "%u,%u,%u,%u%c", &qk, &sf, &pv, &up, &tail) != 4) return false; + const uint32_t maximum = winner_workload::kMaxRealComputeCount; + if (qk == 0 || sf == 0 || pv == 0 || up == 0 || + qk > maximum || sf > maximum || pv > maximum || up > maximum) { + return false; + } + *counts = WorkloadCounts{qk, sf, pv, up}; + return true; +} + +inline bool ParseWinnerWorkloadOptions( + int argc, char **argv, WinnerWorkloadOptions *workload, std::vector *remaining_argv +) { + bool mode_seen = false; + bool count_seen = false; + bool pattern_seen = false; + remaining_argv->clear(); + remaining_argv->push_back(argv[0]); + for (int index = 1; index < argc; ++index) { + const std::string argument = argv[index]; + if (argument == "--nop-count" || argument == "--nop-counts") { + workload->nop_override_explicit = true; + } + if (argument != "--winner-workload" && argument != "--real-compute-count" && + argument != "--real-compute-counts" && argument != "--real-compute-pattern") { + remaining_argv->push_back(argv[index]); + continue; + } + if (index + 1 >= argc) { + std::fprintf(stderr, "Missing value after %s\n", argument.c_str()); + return false; + } + const char *value = argv[++index]; + if (argument == "--winner-workload") { + if (mode_seen) { + std::fprintf(stderr, "Specify --winner-workload only once.\n"); + return false; + } + const std::string name = value; + if (name == "scalar-nop") { + workload->mode = WinnerWorkloadMode::ScalarNop; + } else if (name == "real-compute") { + workload->mode = WinnerWorkloadMode::RealCompute; + } else { + std::fprintf( + stderr, + "Invalid --winner-workload value: %s (expected scalar-nop|real-compute)\n", + value + ); + return false; + } + mode_seen = true; + continue; + } + if (argument == "--real-compute-pattern") { + if (pattern_seen) { + std::fprintf(stderr, "Specify --real-compute-pattern only once.\n"); + return false; + } + const std::string name = value; + if (name == "constant") { + workload->pattern = RealComputePattern::Constant; + } else if (name == "layout-diagnostic") { + workload->pattern = RealComputePattern::LayoutDiagnostic; + } else { + std::fprintf( + stderr, + "Invalid --real-compute-pattern value: %s " + "(expected constant|layout-diagnostic)\n", + value + ); + return false; + } + pattern_seen = true; + workload->pattern_explicit = true; + continue; + } + if (count_seen) { + std::fprintf(stderr, "Specify only one real-compute count override.\n"); + return false; + } + if (argument == "--real-compute-count") { + uint32_t count = 0; + if (!ParseUint(value, 1, winner_workload::kMaxRealComputeCount, &count)) { + std::fprintf(stderr, "Invalid --real-compute-count value: %s\n", value); + return false; + } + workload->repeats = WorkloadCounts{count, count, count, count}; + } else if (!ParseWorkloadCounts(value, &workload->repeats)) { + std::fprintf(stderr, "Invalid --real-compute-counts value: %s\n", value); + return false; + } + count_seen = true; + workload->counts_explicit = true; + } + // 无参数运行以真实 Cube/Vector 为默认。旧命令若显式给出 NOP 次数但没有 + // 指定 workload mode，则把 NOP override 本身视为选择 scalar-nop；这样 + // 既不让 --nop-count 悄悄失效，也不破坏既有标定脚本。显式指定 + // real-compute 再叠加 NOP 仍由下方互斥校验拒绝。 + if (!mode_seen && workload->nop_override_explicit) { + workload->mode = WinnerWorkloadMode::ScalarNop; + } + return true; +} + +inline bool ValidateWinnerWorkloadOptions(const WinnerWorkloadOptions &workload) { + if (workload.mode == WinnerWorkloadMode::RealCompute) { + if (workload.pattern != RealComputePattern::Constant && + workload.pattern != RealComputePattern::LayoutDiagnostic) { + std::fprintf(stderr, "Invalid real-compute input pattern.\n"); + return false; + } + if (workload.nop_override_explicit) { + std::fprintf( + stderr, + "--winner-workload real-compute cannot be combined with --nop-count or --nop-counts.\n" + ); + return false; + } + return true; + } + if (workload.counts_explicit || workload.pattern_explicit) { + std::fprintf( + stderr, + "--real-compute-count(s)/pattern requires real-compute workload mode.\n" + ); + return false; + } + return true; +} + +inline void ConfigureWinnerWorkload( + SchedulerState *state, const WinnerWorkloadOptions &workload, const void *workspace_device +) { + state->winner_workload.mode = static_cast(workload.mode); + state->winner_workload.version = kWinnerWorkloadConfigVersion; + state->winner_workload.repeats = workload.repeats; + state->winner_workload.workspace_base = reinterpret_cast(workspace_device); + state->winner_workload.workspace_bytes = + workload.mode == WinnerWorkloadMode::RealCompute ? winner_workload::kWorkspaceBytes : 0; +} + +inline float LayoutDiagnosticInputA(uint32_t row, uint32_t column) { + return row == column ? static_cast(row + 1U) : 0.0F; +} + +inline float LayoutDiagnosticInputB(uint32_t row, uint32_t column) { + const uint32_t value = + (131U * row + 17U * column + 7U * row * column) % 251U; + return static_cast(value + 1U); +} + +inline void InitializeWinnerWorkloadBuffers( + const WinnerWorkloadOptions &workload, std::vector *workspace_image, + std::vector *workspace_outputs +) { + using namespace winner_workload; + workspace_image->assign(kWorkspaceTiles * kTileElements, kOutputSentinel); + if (workload.pattern == RealComputePattern::Constant) { + std::fill_n(workspace_image->begin(), kTileElements, kInputAValue); + std::fill_n(workspace_image->begin() + kTileElements, kTileElements, kInputBValue); + } else { + for (uint32_t row = 0; row < kTileRows; ++row) { + for (uint32_t column = 0; column < kTileCols; ++column) { + const size_t element = static_cast(row) * kTileCols + column; + (*workspace_image)[element] = LayoutDiagnosticInputA(row, column); + (*workspace_image)[kTileElements + element] = + LayoutDiagnosticInputB(row, column); + } + } + } + workspace_outputs->resize(static_cast(kOutputTiles) * kTileElements); +} + +inline const char *TaskKindName(TaskKind kind) { + switch (kind) { + case TaskKind::Qk: + return "QK"; + case TaskKind::Sf: + return "SF"; + case TaskKind::Pv: + return "PV"; + case TaskKind::Up: + return "UP"; + default: + return "invalid"; + } +} + +inline float ExpectedRealComputeValue( + RealComputePattern pattern, TaskKind kind, uint32_t row, uint32_t column +) { + using namespace winner_workload; + if (pattern == RealComputePattern::Constant) { + return kind == TaskKind::Qk || kind == TaskKind::Pv + ? kExpectedAicValue + : (kind == TaskKind::Sf ? kExpectedSfValue : kExpectedUpValue); + } + const float input_a = LayoutDiagnosticInputA(row, column); + const float input_b = LayoutDiagnosticInputB(row, column); + if (kind == TaskKind::Qk || kind == TaskKind::Pv) { + // A 是带权对角矩阵，因此 A*B 的 (row,column) 只有一个非零乘积。 + return static_cast(row + 1U) * input_b; + } + return kind == TaskKind::Sf ? input_a + input_b : input_a * input_b; +} + +inline bool ValidateRealComputeOutputs( + const SchedulerState &state, const WinnerWorkloadOptions &workload, + const std::vector &outputs, uint32_t run +) { + using namespace winner_workload; + const size_t expected_elements = static_cast(kOutputTiles) * kTileElements; + if (outputs.size() != expected_elements) { + std::fprintf( + stderr, "[ASSERT] real-compute output buffer size matches workspace layout FAIL\n" + ); + return false; + } + + uint32_t active_tiles = 0; + uint32_t inactive_tiles = 0; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const WorkerResult &result = state.results[worker]; + const bool aic = result.role == static_cast(CoreRole::Aic); + const TaskKind kinds[2] = { + aic ? TaskKind::Qk : TaskKind::Sf, + aic ? TaskKind::Pv : TaskKind::Up, + }; + for (uint32_t kind_slot = 0; kind_slot < 2; ++kind_slot) { + const TaskKind kind = kinds[kind_slot]; + const uint32_t kernel_index = static_cast(kind) - 1; + const bool active = result.kernel_counts[kernel_index] != 0; + const size_t tile_index = + static_cast(worker) * kOutputTilesPerWorker + kind_slot; + const size_t begin = tile_index * kTileElements; + for (size_t element = 0; element < kTileElements; ++element) { + const uint32_t row = static_cast(element / kTileCols); + const uint32_t column = static_cast(element % kTileCols); + const float expected = active + ? ExpectedRealComputeValue(workload.pattern, kind, row, column) + : kOutputSentinel; + if (outputs[begin + element] == expected) continue; + std::fprintf( + stderr, + "[REAL-COMPUTE-FAIL] run=%u worker=%u kind=%s element=%zu " + "expected=%.1f actual=%.9g\n", + run, worker, TaskKindName(kind), element, expected, + static_cast(outputs[begin + element]) + ); + std::fprintf( + stderr, + "[ASSERT] real-compute output tiles match role-specific engine results FAIL\n" + ); + return false; + } + active_tiles += active ? 1U : 0U; + inactive_tiles += active ? 0U : 1U; + } + } + const bool passed = active_tiles != 0 && active_tiles + inactive_tiles == kOutputTiles; + std::printf( + "[ASSERT] %-48s %s (active_tiles=%u inactive_sentinel_tiles=%u)\n", + "real-compute output tiles match role-specific engine results", + passed ? "PASS" : "FAIL", active_tiles, inactive_tiles + ); + return passed; +} + +inline void PrintWinnerWorkloadConfig( + const WinnerWorkloadOptions &workload, const NopCounts &nops +) { + if (workload.mode == WinnerWorkloadMode::RealCompute) { + std::printf( + "[WINNER-WORKLOAD] mode=real-compute pattern=%s counts=%u,%u,%u,%u " + "unit=complete_128x128_engine_pipeline_iteration workspace_bytes=%zu\n", + RealComputePatternName(workload.pattern), + workload.repeats.qk, workload.repeats.sf, workload.repeats.pv, + workload.repeats.up, winner_workload::kWorkspaceBytes + ); + return; + } + std::printf( + "[WINNER-WORKLOAD] mode=scalar-nop counts=%u,%u,%u,%u " + "unit=scalar_nop_instruction workspace_bytes=0\n", + nops.qk, nops.sf, nops.pv, nops.up + ); +} + +} // namespace pa_scheduler::host + +#endif // PA_SCHEDULER_COMMON_WINNER_WORKLOAD_HOST_H diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassemble.py b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassemble.py new file mode 100755 index 0000000000..16eac59111 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassemble.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python3 +"""Disassemble every final-linked STT_FUNC with the verified A5 PEM decoder.""" + +from __future__ import annotations + +import argparse +import ctypes +import gzip +import hashlib +import io +import re +import struct +from dataclasses import dataclass +from pathlib import Path + + +EXPECTED_DECODER_SHA256 = "29835d2439d6dd464d34a212ad4bbd5c29af6a38465da09a6c273401d9a96dcb" +INSTRUCTION_RE = re.compile(r"^\s*([0-9a-fA-F]+):\s+((?:[0-9a-fA-F]{8})+)\s+(.*?)\s*$") +BAD_MNEMONICS = {"UNDEF", "UNKNOWN", "INVALID", "ILLEGAL", "ERROR", " str: + return hashlib.sha256(data).hexdigest() + + +def c_string(table: bytes, offset: int) -> str: + end = table.find(b"\0", offset) + if end < 0: + raise ValueError("unterminated ELF string") + return table[offset:end].decode("utf-8") + + +@dataclass(frozen=True) +class Section: + index: int + name: str + section_type: int + address: int + offset: int + size: int + link: int + entry_size: int + + +@dataclass(frozen=True) +class Function: + name: str + value: int + size: int + binding: str + + +class Elf64: + def __init__(self, path: Path): + self.path = path + self.data = path.read_bytes() + header_format = "<16sHHIQQQIHHHHHH" + if len(self.data) < struct.calcsize(header_format): + raise ValueError(f"truncated ELF: {path}") + header = struct.unpack_from(header_format, self.data, 0) + ident = header[0] + if ident[:4] != b"\x7fELF" or ident[4] != 2 or ident[5] != 1: + raise ValueError(f"expected ELF64 little-endian image: {path}") + section_header_offset = header[6] + section_header_size = header[11] + section_count = header[12] + section_name_index = header[13] + if section_header_size != 64 or section_count == 0: + raise ValueError(f"unsupported section table: {path}") + + raw_sections = [] + section_format = " bytes: + return self.data[section.offset : section.offset + section.size] + + def functions(self) -> list[Function]: + symtab = next(section for section in self.sections if section.section_type == 2) + if symtab.entry_size != 24 or symtab.size % symtab.entry_size != 0: + raise ValueError(f"unsupported symbol table: {self.path}") + strings = self.section_bytes(self.sections[symtab.link]) + functions = [] + for offset in range(symtab.offset, symtab.offset + symtab.size, symtab.entry_size): + name_offset, info, _, section_index, value, size = struct.unpack_from( + "> 4, str(info >> 4)) + functions.append(Function(c_string(strings, name_offset), value, size, binding)) + functions.sort(key=lambda function: (function.value, function.name)) + if not functions: + raise ValueError(f"no final-linked .text functions: {self.path}") + return functions + + +class PemDecoder: + def __init__(self, path: Path): + self.path = path + decoder_bytes = path.read_bytes() + self.digest = sha256(decoder_bytes) + if self.digest != EXPECTED_DECODER_SHA256: + raise ValueError( + f"unsupported PEM decoder SHA256 {self.digest}; expected {EXPECTED_DECODER_SHA256}" + ) + library = ctypes.CDLL(str(path)) + self.dump = library.pem_turbo_objdump + self.dump.argtypes = ( + ctypes.POINTER(ctypes.c_uint32), + ctypes.c_size_t, + ctypes.c_void_p, + ctypes.c_size_t, + ) + self.dump.restype = ctypes.c_void_p + self.libc = ctypes.CDLL(None) + self.libc.free.argtypes = (ctypes.c_void_p,) + + def decode(self, body: bytes, rvec: bool) -> list[tuple[int, str, str]]: + if len(body) % 4 != 0: + raise ValueError("function body is not an integral number of instruction words") + word_count = len(body) // 4 + words = (ctypes.c_uint32 * word_count).from_buffer_copy(body) + range_pointer = None + range_count = 0 + if rvec: + ranges_type = (ctypes.c_uint64 * 2) * 1 + ranges = ranges_type() + ranges[0][0] = 0 + ranges[0][1] = len(body) + range_pointer = ctypes.cast(ranges, ctypes.c_void_p) + range_count = 1 + result = self.dump(words, word_count, range_pointer, range_count) + if not result: + raise RuntimeError("pem_turbo_objdump returned null") + try: + decoded = ctypes.string_at(result).decode("utf-8") + finally: + self.libc.free(result) + + instructions = [] + for line in decoded.splitlines(): + match = INSTRUCTION_RE.fullmatch(line) + if match is None: + if line.strip(): + raise ValueError(f"unexpected decoder output: {line!r}") + continue + relative = int(match.group(1), 16) + machine_word = match.group(2).lower() + mnemonic = match.group(3).strip() + instructions.append((relative, machine_word, mnemonic)) + expected_relative = 0 + for index, (relative, machine_word, mnemonic) in enumerate(instructions): + encoded_bytes = len(machine_word) // 2 + if encoded_bytes % 4 != 0: + raise ValueError( + f"decoder returned a non-word-sized encoding at row {index}: {machine_word}" + ) + expected_word = ( + f"{int.from_bytes(body[expected_relative:expected_relative + encoded_bytes], 'little'):0{len(machine_word)}x}" + ) + if relative != expected_relative or machine_word != expected_word: + raise ValueError( + f"decoder coverage mismatch at word {index}: {relative:x}/{machine_word} " + f"!= {expected_relative:x}/{expected_word}" + ) + first_token = mnemonic.upper().split(maxsplit=1)[0] if mnemonic else "" + if first_token in BAD_MNEMONICS or "NOT AVAILABLE" in mnemonic.upper(): + raise ValueError(f"invalid decoded instruction at +0x{relative:x}: {mnemonic}") + expected_relative += encoded_bytes + if expected_relative != len(body): + raise ValueError( + f"decoder covered {expected_relative} bytes of a {len(body)}-byte function" + ) + return instructions + + +def safe_filename(index: int, name: str) -> str: + safe = re.sub(r"[^A-Za-z0-9_.-]+", "_", name).strip("._") or "function" + suffix = hashlib.sha256(name.encode("utf-8")).hexdigest()[:8] + return f"{index:02d}_{safe[:72]}_{suffix}.asm.gz" + + +def write_gzip(path: Path, text: str) -> None: + with path.open("wb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed: + compressed.write(text.encode("utf-8")) + + +def disassemble_variant( + variant: str, elf_path: Path, output_dir: Path, decoder: PemDecoder +) -> tuple[list[str], list[str], dict[str, object]]: + elf = Elf64(elf_path) + text = elf.section_bytes(elf.text) + functions = elf.functions() + variant_dir = output_dir + variant_dir.mkdir(parents=True) + elf_digest = sha256(elf.data) + text_digest = sha256(text) + manifest_rows = [] + gap_rows = [] + cursor = elf.text.address + total_function_bytes = 0 + + for index, function in enumerate(functions): + if function.value < cursor: + raise ValueError(f"overlapping final function symbols in {elf_path}: {function.name}") + if function.value > cursor: + gap_start = cursor + gap_end = function.value + start = gap_start - elf.text.address + gap = text[start : start + gap_end - gap_start] + gap_rows.append( + f"0x{gap_start:x}\t0x{gap_end:x}\t{len(gap)}\t{sha256(gap)}" + ) + body_offset = function.value - elf.text.address + body = text[body_offset : body_offset + function.size] + if len(body) != function.size: + raise ValueError(f"function outside .text: {function.name}") + rvec = ".vector.thread" in function.name + try: + instructions = decoder.decode(body, rvec) + except Exception as error: + raise ValueError(f"{variant}:{function.name}: {error}") from error + body_digest = sha256(body) + filename = safe_filename(index, function.name) + header = [ + "# schema=pa_final_linked_disassembly/v1", + f"# variant={variant}", + f"# final_elf={elf_path.name}", + f"# final_elf_sha256={elf_digest}", + f"# final_text_address=0x{elf.text.address:x}", + f"# final_text_size={elf.text.size}", + f"# final_text_sha256={text_digest}", + f"# symbol={function.name}", + f"# binding={function.binding}", + f"# final_pc=0x{function.value:x}", + f"# size={function.size}", + f"# instruction_count={len(instructions)}", + f"# encoded_word_count={function.size // 4}", + f"# body_sha256={body_digest}", + "# decoder=$ASCEND_HOME_PATH/x86_64-linux/simulator/dav_3510/lib/libpem_davinci.so", + f"# decoder_sha256={decoder.digest}", + f"# decoder_mode={'rvec' if rvec else 'scalar'}", + "# columns=final_pc function_relative_offset machine_word instruction", + "", + ] + body_lines = [ + f"0x{function.value + relative:016x} (+0x{relative:08x}) {machine_word} {mnemonic}" + for relative, machine_word, mnemonic in instructions + ] + write_gzip(variant_dir / filename, "\n".join(header + body_lines) + "\n") + last_mnemonic = instructions[-1][2].split(maxsplit=1)[0] + manifest_rows.append( + "\t".join( + ( + variant, + filename, + function.binding, + function.name, + f"0x{function.value:x}", + str(function.size), + str(len(instructions)), + "rvec" if rvec else "scalar", + body_digest, + last_mnemonic, + ) + ) + ) + cursor = function.value + function.size + total_function_bytes += function.size + + text_end = elf.text.address + elf.text.size + if cursor < text_end: + start = cursor - elf.text.address + gap = text[start:] + gap_rows.append(f"0x{cursor:x}\t0x{text_end:x}\t{len(gap)}\t{sha256(gap)}") + elif cursor > text_end: + raise ValueError(f"function extends past .text: {elf_path}") + + metadata = { + "variant": variant, + "elf_sha256": elf_digest, + "text_address": elf.text.address, + "text_size": elf.text.size, + "text_sha256": text_digest, + "function_count": len(functions), + "function_bytes": total_function_bytes, + "function_coverage_percent": total_function_bytes / elf.text.size * 100.0, + "gap_count": len(gap_rows), + } + return manifest_rows, gap_rows, metadata + + +def main() -> None: + parser = argparse.ArgumentParser() + script_dir = Path(__file__).resolve().parent + fixed_variant = (script_dir / "VARIANT").read_text(encoding="utf-8").strip() + parser.add_argument("--variant", default=fixed_variant) + parser.add_argument( + "--elf", type=Path, + default=script_dir / "artifacts" / "measured" / "pa_scheduler_kernel.o" + ) + parser.add_argument("--output", type=Path, default=script_dir / "disassembly" / "raw") + default_decoder = None + import os + + if os.environ.get("ASCEND_HOME_PATH"): + default_decoder = ( + Path(os.environ["ASCEND_HOME_PATH"]) + / "x86_64-linux/simulator/dav_3510/lib/libpem_davinci.so" + ) + parser.add_argument("--decoder", type=Path, default=default_decoder, required=default_decoder is None) + args = parser.parse_args() + elf_path = args.elf.resolve() + output_dir = args.output.resolve() + decoder_path = args.decoder.resolve() + if output_dir.exists(): + raise SystemExit(f"Output directory already exists; choose a new path: {output_dir}") + decoder = PemDecoder(decoder_path) + + if not elf_path.is_file(): + raise SystemExit(f"Missing published final ELF: {elf_path}") + manifest_rows, local_gaps, item = disassemble_variant( + args.variant, elf_path, output_dir, decoder + ) + gap_rows = [f"{args.variant}\t{row}" for row in local_gaps] + + manifest_header = ( + "variant\tfile\tbinding\tsymbol\tfinal_pc\tsize\tinstructions\tdecoder_mode\t" + "body_sha256\tlast_mnemonic" + ) + (output_dir / "manifest.tsv").write_text( + manifest_header + "\n" + "\n".join(manifest_rows) + "\n", encoding="utf-8" + ) + (output_dir / "gaps.tsv").write_text( + "variant\tstart_pc\tend_pc\tsize\tsha256\n" + "\n".join(gap_rows) + "\n", + encoding="utf-8", + ) + readme = [ + "# Final-linked A5 disassembly", + "", + "Every non-empty final `.text` `STT_FUNC` symbol was extracted from the published final ELF and decoded with the verified `dav_3510` PEM decoder.", + "RVec `.vector.thread` symbols are passed through an explicit full-body RVec range; all other symbols use scalar decoding.", + "Each compressed file records final PC, function-relative offset, machine word, mnemonic, ELF/body hashes and decoder identity.", + "Function gaps are alignment/padding outside symbol bodies and are hashed in `gaps.tsv`; they are not presented as instructions.", + "", + f"Decoder SHA256: `{decoder.digest}`", + "", + "| Variant | Final .text | Function symbols | Function bytes | Coverage | Final ELF SHA256 |", + "| --- | ---: | ---: | ---: | ---: | --- |", + ] + readme.append( + f"| {item['variant']} | {item['text_size']} B | {item['function_count']} | " + f"{item['function_bytes']} B | {item['function_coverage_percent']:.3f}% | " + f"`{item['elf_sha256']}` |" + ) + readme.extend( + [ + "", + "Inspect with `zless FILE.asm.gz`; the sibling `annotated/` directory adds DWARF-mapped local source and comments.", + ] + ) + (output_dir / "README.md").write_text("\n".join(readme) + "\n", encoding="utf-8") + + published = [] + for path in sorted(item for item in output_dir.rglob("*") if item.is_file()): + if path.name == "published.sha256": + continue + published.append(f"{sha256(path.read_bytes())} {path.relative_to(output_dir)}") + (output_dir / "published.sha256").write_text("\n".join(published) + "\n", encoding="utf-8") + print("\n".join(readme)) + + +if __name__ == "__main__": + main() diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/00_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/00_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz new file mode 100644 index 0000000000..6413607776 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/00_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz new file mode 100644 index 0000000000..b5d4cb2ca0 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/02_pa_execute_real_winner_workload_aic_729699de.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/02_pa_execute_real_winner_workload_aic_729699de.source.asm.gz new file mode 100644 index 0000000000..24e0d7834a Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/02_pa_execute_real_winner_workload_aic_729699de.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.source.asm.gz new file mode 100644 index 0000000000..d4a1a1b347 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz new file mode 100644 index 0000000000..4b436dc256 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.source.asm.gz new file mode 100644 index 0000000000..1243ac3d7d Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz new file mode 100644 index 0000000000..1e0e8f3a3d Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz new file mode 100644 index 0000000000..8d6fff1085 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.source.asm.gz new file mode 100644 index 0000000000..d51fe068bb Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz new file mode 100644 index 0000000000..b1ec0a385f Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz new file mode 100644 index 0000000000..75001fee28 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz new file mode 100644 index 0000000000..059da6bfc8 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/12_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/12_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz new file mode 100644 index 0000000000..ddbcc580c8 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/12_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.source.asm.gz new file mode 100644 index 0000000000..bbce4e7d3a Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz new file mode 100644 index 0000000000..a301081b95 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz new file mode 100644 index 0000000000..a6df055abc Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz new file mode 100644 index 0000000000..27645475a7 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/annotated/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/key_flow/aic_compete_first_lazy.source.asm b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/key_flow/aic_compete_first_lazy.source.asm new file mode 100644 index 0000000000..2bc86c3d59 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/key_flow/aic_compete_first_lazy.source.asm @@ -0,0 +1,2064 @@ +# schema=pa_final_linked_disassembly/v1 +# variant=compete-first-lazy +# final_elf=pa_scheduler_kernel.o +# final_elf_sha256=8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893 +# final_text_address=0x0 +# final_text_size=547640 +# final_text_sha256=866ed1081ebb94038a8feca87ccedf95e67aebfdaef11651c109f86672be2bdd +# symbol=pa_scheduler_lazy_sample_callback_orchestration_aic +# binding=LOCAL +# final_pc=0x2c0 +# size=161792 +# instruction_count=40448 +# encoded_word_count=40448 +# body_sha256=35bcc11cdad126fa5d2f92ff1df421d8556843a412ea3497addfe820d84bd852 +# decoder=$ASCEND_HOME_PATH/x86_64-linux/simulator/dav_3510/lib/libpem_davinci.so +# decoder_sha256=29835d2439d6dd464d34a212ad4bbd5c29af6a38465da09a6c273401d9a96dcb +# decoder_mode=scalar +# columns=final_pc function_relative_offset machine_word instruction +# annotation_schema=pa_source_annotated_disassembly/v1 +# annotation_rule=DWARF supplies only file:line; SOURCE rows are copied from local source files +# annotation_warning=comments have source context only and do not own an exact machine address +# annotation_instruction_slice=9223:9944 +# +# [DWARF] common/pa_trace.h:266 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +# 263 | const uint32_t bit = 1U << index; +# 264 | if ((active_mask & bit) == 0) continue; +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# > 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +0x00000000000092dc (+0x0000901c) 02821a00 ZEROEXT.U32 X1, X1 +0x00000000000092e0 (+0x00009020) 0040113e CMP.U64.GT X1, X2 +0x00000000000092e4 (+0x00009024) 40200003 JUMPC #3 +# [DWARF] common/pa_trace.h:267 +# > 267 | trace.atomic_counter_overflow = true; +0x00000000000092e8 (+0x00009028) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x00000000000092ec (+0x0000902c) 4000ffdf JUMP #65503 +# [DWARF] common/pa_trace.h:98 +# 92 | default: +# 93 | return -1; +# 94 | } +# 95 | } +# 96 | +# 97 | PA_DEVICE AtomicSite TraceAtomicPollBatchSite(uint32_t index) { +# > 98 | switch (index) { +0x00000000000092f0 (+0x00009030) 0281ca00 ZEROEXT.U32 X0, X28 +0x00000000000092f4 (+0x00009034) 07020005 MOV_XD_IMM X1, #5 +0x00000000000092f8 (+0x00009038) 004000be CMP.U64.GT X0, X1 +0x00000000000092fc (+0x0000903c) 070c000f MOV_XD_IMM X6, #15 +0x0000000000009300 (+0x00009040) 40200002 JUMPC #2 +0x0000000000009304 (+0x00009044) 1c8d7000 LD_XD_XN_IMM.B32 X6, X23, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009308 (+0x00009048) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x000000000000930c (+0x0000904c) 00c2b78a AND.B64 X1, X11, X15 +0x0000000000009310 (+0x00009050) 02c4020e SHL.B64 X2, #14 +0x0000000000009314 (+0x00009054) 0006a481 ADD.S64 X3, X10, X9 +0x0000000000009318 (+0x00009058) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:273 +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# > 273 | trace.poll_burst.start_cycle[index], end_cycle, +0x000000000000931c (+0x0000905c) 02018800 MOV_XD_XN.S64 X0, X24 +0x0000000000009320 (+0x00009060) 00023101 ADD.S64 X1, X3, X2 +0x0000000000009324 (+0x00009064) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x0000000000009328 (+0x00009068) 02c00203 SHL.B64 X0, #3 +0x000000000000932c (+0x0000906c) 08241000 ADD_IMM.S64 X18, X1, #0 +0x0000000000009330 (+0x00009070) 00012001 ADD.S64 X0, X18, X0 +0x0000000000009334 (+0x00009074) 1cc60588 LD_XD_XN_IMM.B64 X3, X0, #1416 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x0000000000009338 (+0x00009078) 02015800 MOV_XD_XN.S64 X0, X21 +0x000000000000933c (+0x0000907c) 0203a800 MOV_XD_XN.S64 X1, X26 +0x0000000000009340 (+0x00009080) 0205b800 MOV_XD_XN.S64 X2, X27 +0x0000000000009344 (+0x00009084) 02099800 MOV_XD_XN.S64 X4, X25 +0x0000000000009348 (+0x00009088) 070e79db MOV_XD_IMM X7, #31195 +0x000000000000934c (+0x0000908c) 074f0000 MOVK X7, #0, #1 +0x0000000000009350 (+0x00009090) 078f0000 MOVK X7, #0, #2 +0x0000000000009354 (+0x00009094) 40427000 CALL X7, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009358 (+0x00009098) 1cd3e990 LD_XD_XN_IMM.B64 X9, X30, #2448 +0x000000000000935c (+0x0000909c) 071c0000 MOV_XD_IMM X14, #0 +0x0000000000009360 (+0x000090a0) 1cd9e9a0 LD_XD_XN_IMM.B64 X12, X30, #2464 +0x0000000000009364 (+0x000090a4) 071e7fff MOV_XD_IMM X15, #32767 +0x0000000000009368 (+0x000090a8) 1cd7e9a8 LD_XD_XN_IMM.B64 X11, X30, #2472 +# [DWARF] common/pa_trace.h:276 +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x000000000000936c (+0x000090ac) 0000070e CMP.S64.EQ X0, X14 +0x0000000000009370 (+0x000090b0) 1cd5e9b0 LD_XD_XN_IMM.B64 X10, X30, #2480 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009374 (+0x000090b4) 07220001 MOV_XD_IMM X17, #1 +0x0000000000009378 (+0x000090b8) 07100006 MOV_XD_IMM X8, #6 +# [DWARF] common/pa_trace.h:276 +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x000000000000937c (+0x000090bc) 4020ffba JUMPC #65466 +# [DWARF] common/pa_trace.h:277 +# > 277 | if (trace.poll_batch_records == UINT64_MAX) { +0x0000000000009380 (+0x000090c0) 1cc12578 LD_XD_XN_IMM.B64 X0, X18, #1400 +0x0000000000009384 (+0x000090c4) 07020001 MOV_XD_IMM X1, #1 +0x0000000000009388 (+0x000090c8) 02021080 NEG.S64 X1, X1 +0x000000000000938c (+0x000090cc) 0000009e CMP.S64.NE X0, X1 +0x0000000000009390 (+0x000090d0) 4020ffab JUMPC #65451 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009394 (+0x000090d4) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x0000000000009398 (+0x000090d8) 00c0b78a AND.B64 X0, X11, X15 +0x000000000000939c (+0x000090dc) 02c2020e SHL.B64 X1, #14 +0x00000000000093a0 (+0x000090e0) 0004a481 ADD.S64 X2, X10, X9 +0x00000000000093a4 (+0x000090e4) 00040084 MADD.S64 X2, X0, X1 +0x00000000000093a8 (+0x000090e8) 00002081 ADD.S64 X0, X2, X1 +0x00000000000093ac (+0x000090ec) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:278 +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# 276 | if (written) { +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# > 278 | trace.atomic_counter_overflow = true; +0x00000000000093b0 (+0x000090f0) 08000000 ADD_IMM.S64 X0, X0, #0 +0x00000000000093b4 (+0x000090f4) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x00000000000093b8 (+0x000090f8) 4000ffab JUMP #65451 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000093bc (+0x000090fc) 1ccde988 LD_XD_XN_IMM.B64 X6, X30, #2440 +0x00000000000093c0 (+0x00009100) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x00000000000093c4 (+0x00009104) 00c0b78a AND.B64 X0, X11, X15 +0x00000000000093c8 (+0x00009108) 02c2020e SHL.B64 X1, #14 +0x00000000000093cc (+0x0000910c) 0004a481 ADD.S64 X2, X10, X9 +0x00000000000093d0 (+0x00009110) 00040084 MADD.S64 X2, X0, X1 +0x00000000000093d4 (+0x00009114) 00002081 ADD.S64 X0, X2, X1 +0x00000000000093d8 (+0x00009118) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:285 +# 279 | } else { +# 280 | ++trace.poll_batch_records; +# 281 | } +# 282 | } +# 283 | trace.poll_burst.call_count[index] = 0; +# 284 | } +# > 285 | trace.poll_burst.active_mask = 0; +0x00000000000093dc (+0x0000911c) 08000000 ADD_IMM.S64 X0, X0, #0 +0x00000000000093e0 (+0x00009120) 0f960a00 STI_XN_IMM.B32 X0, #1488 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x00000000000093e4 (+0x00009124) 08353548 ADD_IMM.S64 X26, X19, #1352 +0x00000000000093e8 (+0x00009128) 1cb93558 LD_XD_XN_IMM.B32 X28, X19, #1368 +0x00000000000093ec (+0x0000912c) 0cf5ad80 LDP_XI_XJ_XN.B64 X26, X27, X26, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000093f0 (+0x00009130) 07280001 MOV_XD_IMM X20, #1 +0x00000000000093f4 (+0x00009134) 070affff MOV_XD_IMM X5, #65535 +0x00000000000093f8 (+0x00009138) 1ccfe8f0 LD_XD_XN_IMM.B64 X7, X30, #2288 +0x00000000000093fc (+0x0000913c) 02294080 NEG.S64 X20, X20 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000009400 (+0x00009140) 0001a70e CMP.S64.EQ X26, X14 +0x0000000000009404 (+0x00009144) 40200023 JUMPC #35 +0x0000000000009408 (+0x00009148) 0001b70e CMP.S64.EQ X27, X14 +0x000000000000940c (+0x0000914c) 40200021 JUMPC #33 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009410 (+0x00009150) 0283ca00 ZEROEXT.U32 X1, X28 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x0000000000009414 (+0x00009154) 0000170e CMP.S64.EQ X1, X14 +0x0000000000009418 (+0x00009158) 4020001e JUMPC #30 +# [DWARF] common/pa_trace.h:547 +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# > 547 | const uint32_t slot = core.count; +0x000000000000941c (+0x0000915c) 1c81a000 LD_XD_XN_IMM.B32 X0, X26, #0 +# [DWARF] common/pa_trace.h:548 +# > 548 | if (slot >= trace.capacity) { +0x0000000000009420 (+0x00009160) 004000ae CMP.U64.LT X0, X1 +0x0000000000009424 (+0x00009164) 40200002 JUMPC #2 +0x0000000000009428 (+0x00009168) 4000002d JUMP #45 +# [DWARF] common/pa_trace.h:553 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# > 553 | record.start_cycle = start_cycle; +0x000000000000942c (+0x0000916c) 1cc9e928 LD_XD_XN_IMM.B64 X4, X30, #2344 +# [DWARF] common/pa_trace.h:552 +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x0000000000009430 (+0x00009170) 02020800 MOV_XD_XN.S64 X1, X0 +0x0000000000009434 (+0x00009174) 02c20206 SHL.B64 X1, #6 +0x0000000000009438 (+0x00009178) 0003b081 ADD.S64 X1, X27, X1 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000943c (+0x0000917c) 0806c002 ADD_IMM.S64 X3, X12, #2 +0x0000000000009440 (+0x00009180) 00c4b78a AND.B64 X2, X11, X15 +0x0000000000009444 (+0x00009184) 02c6020e SHL.B64 X3, #14 +# [DWARF] common/pa_trace.h:565 +# 559 | record.block_id = trace.block_id; +# 560 | record.core_idx = trace.core_idx; +# 561 | record.flags = flags; +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x0000000000009448 (+0x00009188) 08000001 ADD_IMM.S64 X0, X0, #1 +# [DWARF] common/pa_trace.h:553 +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# > 553 | record.start_cycle = start_cycle; +0x000000000000944c (+0x0000918c) 09c81c81 STP_XI_XJ_XN.B64 X4, X25, X1, #0 +0x0000000000009450 (+0x00009190) 0008a481 ADD.S64 X4, X10, X9 +0x0000000000009454 (+0x00009194) 00082184 MADD.S64 X4, X2, X3 +# [DWARF] common/pa_trace.h:555 +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x0000000000009458 (+0x00009198) 038e1010 ST_XD_XN_IMM.B32 X7, X1, #16 +0x000000000000945c (+0x0000919c) 00044181 ADD.S64 X2, X4, X3 +0x0000000000009460 (+0x000091a0) 08842680 SUB_IMM.S64 X2, X2, #1664 +# [DWARF] common/pa_trace.h:556 +# > 556 | record.function_id = function_id; +0x0000000000009464 (+0x000091a4) 0706ffff MOV_XD_IMM X3, #65535 +0x0000000000009468 (+0x000091a8) 0747ffff MOVK X3, #65535, #1 +# [DWARF] common/pa_trace.h:558 +# 557 | record.phase = static_cast(trace_phase); +# > 558 | record.lane = trace.lane; +0x000000000000946c (+0x000091ac) 08042000 ADD_IMM.S64 X2, X2, #0 +# [DWARF] common/pa_trace.h:556 +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# > 556 | record.function_id = function_id; +0x0000000000009470 (+0x000091b0) 09861429 STP_XI_XJ_XN.B32 X3, X8, X1, #20 +# [DWARF] common/pa_trace.h:559 +# 557 | record.phase = static_cast(trace_phase); +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x0000000000009474 (+0x000091b4) 08062564 ADD_IMM.S64 X3, X2, #1380 +# [DWARF] common/pa_trace.h:558 +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# > 558 | record.lane = trace.lane; +0x0000000000009478 (+0x000091b8) 1c882560 LD_XD_XN_IMM.B32 X4, X2, #1376 +# [DWARF] common/pa_trace.h:559 +# > 559 | record.block_id = trace.block_id; +0x000000000000947c (+0x000091bc) 0c863100 LDP_XI_XJ_XN.B32 X3, X2, X3, #0 +# [DWARF] common/pa_trace.h:558 +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# > 558 | record.lane = trace.lane; +0x0000000000009480 (+0x000091c0) 098811b9 STP_XI_XJ_XN.B32 X4, X3, X1, #28 +# [DWARF] common/pa_trace.h:560 +# 559 | record.block_id = trace.block_id; +# > 560 | record.core_idx = trace.core_idx; +0x0000000000009484 (+0x000091c4) 03841024 ST_XD_XN_IMM.B32 X2, X1, #36 +# [DWARF] common/pa_trace.h:561 +# > 561 | record.flags = flags; +0x0000000000009488 (+0x000091c8) 03dc1028 ST_XD_XN_IMM.B64 X14, X1, #40 +# [DWARF] common/pa_trace.h:565 +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x000000000000948c (+0x000091cc) 0381a000 ST_XD_XN_IMM.B32 X0, X26, #0 +# [DWARF] common/pa_scheduler_core.h:472 +# 466 | PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, TaskKind kind, +# 467 | LocalStats &stats +# 468 | ) { +# 469 | // Claim 在四个 shard 的单调 cursor 上执行 atomicMax：同一 task 只有观察到旧值更小的竞争者获胜。 +# 470 | // Alloc 由全部 96 个 worker 竞争；QK/PV 仅 32 个 AIC，SF/UP 仅 64 个 AIV 进入真正的 atomicMax。 +# 471 | ClaimOutcome outcome{false, false, 0, -1}; +# > 472 | if (task_id >= kTaskCellCapacity) { +0x0000000000009490 (+0x000091d0) 004072be CMP.U64.GT X7, X5 +0x0000000000009494 (+0x000091d4) 40200002 JUMPC #2 +0x0000000000009498 (+0x000091d8) 40000016 JUMP #22 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x000000000000949c (+0x000091dc) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x00000000000094a0 (+0x000091e0) 00c0b78a AND.B64 X0, X11, X15 +0x00000000000094a4 (+0x000091e4) 02c2020e SHL.B64 X1, #14 +0x00000000000094a8 (+0x000091e8) 0004a481 ADD.S64 X2, X10, X9 +0x00000000000094ac (+0x000091ec) 00040084 MADD.S64 X2, X0, X1 +0x00000000000094b0 (+0x000091f0) 070c0000 MOV_XD_IMM X6, #0 +0x00000000000094b4 (+0x000091f4) 00002081 ADD.S64 X0, X2, X1 +0x00000000000094b8 (+0x000091f8) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_scheduler_core.h:1144 +# 1138 | efdrain_begin, efdrain_end +# 1139 | ); +# 1140 | +# 1141 | const uint64_t claim_begin = efdrain_end; +# 1142 | BeginSubmitPmuPhase(pmu_context); +# 1143 | const ClaimOutcome claim = Claim(state, worker, task_id, Kind, stats); +# > 1144 | context.won = claim.won; +0x00000000000094bc (+0x000091fc) 08000000 ADD_IMM.S64 X0, X0, #0 +0x00000000000094c0 (+0x00009200) 0f060300 STI_XN_IMM.B8 X0, #408 +# [DWARF] common/pa_scheduler_core.h:1145 +# > 1145 | context.kernel_id = claim.function_id; +0x00000000000094c4 (+0x00009204) 0f860282 STI_XN_IMM.B32 X0, #404 +0x00000000000094c8 (+0x00009208) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x00000000000094cc (+0x0000920c) 0700007a MOV_XD_IMM X0, #122 +0x00000000000094d0 (+0x00009210) 07410000 MOVK X0, #0, #1 +0x00000000000094d4 (+0x00009214) 07810000 MOVK X0, #0, #2 +0x00000000000094d8 (+0x00009218) 40020000 JUMP X0, #0 +# [DWARF] common/pa_trace.h:549 +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# > 549 | core.dropped = core.dropped + 1; +0x00000000000094dc (+0x0000921c) 1c81a004 LD_XD_XN_IMM.B32 X0, X26, #4 +0x00000000000094e0 (+0x00009220) 08000001 ADD_IMM.S64 X0, X0, #1 +0x00000000000094e4 (+0x00009224) 0381a004 ST_XD_XN_IMM.B32 X0, X26, #4 +# [DWARF] common/pa_scheduler_core.h:472 +# 466 | PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, TaskKind kind, +# 467 | LocalStats &stats +# 468 | ) { +# 469 | // Claim 在四个 shard 的单调 cursor 上执行 atomicMax：同一 task 只有观察到旧值更小的竞争者获胜。 +# 470 | // Alloc 由全部 96 个 worker 竞争；QK/PV 仅 32 个 AIC，SF/UP 仅 64 个 AIV 进入真正的 atomicMax。 +# 471 | ClaimOutcome outcome{false, false, 0, -1}; +# > 472 | if (task_id >= kTaskCellCapacity) { +0x00000000000094e8 (+0x00009228) 004072be CMP.U64.GT X7, X5 +0x00000000000094ec (+0x0000922c) 4020ffec JUMPC #65516 +# [DWARF] common/pa_scheduler_core.h:477 +# 473 | return outcome; +# 474 | } +# 475 | PA_GM AtomicLine *cursor = nullptr; +# 476 | if (kind == TaskKind::Alloc) { +# > 477 | cursor = &state->alloc_cursor[task_id % kCursorShards]; +0x00000000000094f0 (+0x00009230) 1cc3e880 LD_XD_XN_IMM.B64 X1, X30, #2176 +0x00000000000094f4 (+0x00009234) 07000003 MOV_XD_IMM X0, #3 +0x00000000000094f8 (+0x00009238) 00c0700a AND.B64 X0, X7, X0 +0x00000000000094fc (+0x0000923c) 02c00206 SHL.B64 X0, #6 +# [DWARF] common/pa_trace.h:257 +# 251 | TraceContext &trace, uint64_t end_cycle +# 252 | ) { +# 253 | #if PA_BUILD_SUBMIT_PMU +# 254 | (void)trace; +# 255 | (void)end_cycle; +# 256 | #else +# > 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +0x0000000000009500 (+0x00009240) 0000671e CMP.S64.NE X6, X14 +# [DWARF] common/pa_scheduler_core.h:477 +# 471 | ClaimOutcome outcome{false, false, 0, -1}; +# 472 | if (task_id >= kTaskCellCapacity) { +# 473 | return outcome; +# 474 | } +# 475 | PA_GM AtomicLine *cursor = nullptr; +# 476 | if (kind == TaskKind::Alloc) { +# > 477 | cursor = &state->alloc_cursor[task_id % kCursorShards]; +0x0000000000009504 (+0x00009244) 00001001 ADD.S64 X0, X1, X0 +# [DWARF] common/pa_trace.h:480 +# 474 | (void)result; +# 475 | (void)task_id; +# 476 | (void)site; +# 477 | (void)result_used; +# 478 | return Ops::FetchMax(address, value, retries); +# 479 | #else +# > 480 | if (!trace.atomics_enabled) return Ops::FetchMax(address, value, retries); +0x0000000000009508 (+0x00009248) 40200006 JUMPC #6 +0x000000000000950c (+0x0000924c) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000009510 (+0x00009250) 07000015 MOV_XD_IMM X0, #21 +0x0000000000009514 (+0x00009254) 07410000 MOVK X0, #0, #1 +0x0000000000009518 (+0x00009258) 07810000 MOVK X0, #0, #2 +0x000000000000951c (+0x0000925c) 40020000 JUMP X0, #0 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000009520 (+0x00009260) 02828880 MOV_XD_SPR.F32 X1, SYS_CNT +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:86 +# 80 | return __builtin_cce_atom_max_G_##SUFFIX(base, inc, (uint32_t)L2Cache); \ +# 81 | } +# 82 | +# 83 | __CCE__ATOM_G_BUILTIN(uint32_t, u32) +# 84 | __CCE__ATOM_G_BUILTIN(int32_t, s32) +# 85 | __CCE__ATOM_G_BUILTIN(uint64_t, u64) +# > 86 | __CCE__ATOM_G_BUILTIN(int64_t, s64) +0x0000000000009524 (+0x00009264) 51c000e0 ATOM XN, XM, XD, MAX +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:0 +# [SOURCE unavailable] +0x0000000000009528 (+0x00009268) 0806c002 ADD_IMM.S64 X3, X12, #2 +0x000000000000952c (+0x0000926c) 00c4b78a AND.B64 X2, X11, X15 +0x0000000000009530 (+0x00009270) 02c6020e SHL.B64 X3, #14 +0x0000000000009534 (+0x00009274) 0008a481 ADD.S64 X4, X10, X9 +0x0000000000009538 (+0x00009278) 00082184 MADD.S64 X4, X2, X3 +0x000000000000953c (+0x0000927c) 00044181 ADD.S64 X2, X4, X3 +0x0000000000009540 (+0x00009280) 08842680 SUB_IMM.S64 X2, X2, #1664 +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x0000000000009544 (+0x00009284) 08062000 ADD_IMM.S64 X3, X2, #0 +# [DWARF] ccec/ccec_ops.h:239 +# 233 | static_assert(sizeof(T) == 4 || sizeof(T) == 8, "atomic dependency expects a scalar result"); +# 234 | uint64_t cycle = 0; +# 235 | // 同一个 inline asm 块先真正消费 atomic 返回寄存器，再读取 +# 236 | // SYS_CNT；编译器不能把 t1 拆到依赖 MOV 之前。AIC/AIV 对该序列 +# 237 | // 生成相同指令字节，且不增加 DSB/ISB/GM 访存。该边界仍只表示 +# 238 | // 返回值已可被本核 scalar 消费，不表示跨核全局可见。 +# > 239 | asm volatile( +0x0000000000009548 (+0x00009288) 020a0800 MOV_XD_XN.S64 X5, X0 +0x000000000000954c (+0x0000928c) 020a5800 MOV_XD_XN.S64 X5, X5 +0x0000000000009550 (+0x00009290) 02848880 MOV_XD_SPR.F32 X2, SYS_CNT +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x0000000000009554 (+0x00009294) 1cc834a0 LD_XD_XN_IMM.B64 X4, X3, #1184 +0x0000000000009558 (+0x00009298) 00004a1e CMP.S64.NE X4, X20 +0x000000000000955c (+0x0000929c) 40200002 JUMPC #2 +0x0000000000009560 (+0x000092a0) 40000007 JUMP #7 +# [DWARF] common/pa_trace.h:240 +# 237 | trace.atomic_counter_overflow = true; +# 238 | return; +# 239 | } +# > 240 | ++result.atomic_trace_calls; +0x0000000000009564 (+0x000092a4) 08084001 ADD_IMM.S64 X4, X4, #1 +0x0000000000009568 (+0x000092a8) 03c834a0 ST_XD_XN_IMM.B64 X4, X3, #1184 +0x000000000000956c (+0x000092ac) 40000005 JUMP #5 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009570 (+0x000092b0) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:86 +# 80 | return __builtin_cce_atom_max_G_##SUFFIX(base, inc, (uint32_t)L2Cache); \ +# 81 | } +# 82 | +# 83 | __CCE__ATOM_G_BUILTIN(uint32_t, u32) +# 84 | __CCE__ATOM_G_BUILTIN(int32_t, s32) +# 85 | __CCE__ATOM_G_BUILTIN(uint64_t, u64) +# > 86 | __CCE__ATOM_G_BUILTIN(int64_t, s64) +0x0000000000009574 (+0x000092b4) 51c000e0 ATOM XN, XM, XD, MAX +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:0 +# [SOURCE unavailable] +0x0000000000009578 (+0x000092b8) 40000038 JUMP #56 +# [DWARF] common/pa_trace.h:237 +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# 236 | if (result.atomic_trace_calls == UINT64_MAX) { +# > 237 | trace.atomic_counter_overflow = true; +0x000000000000957c (+0x000092bc) 0f163001 STI_XN_IMM.B8 X3, #1408 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009580 (+0x000092c0) 0808c002 ADD_IMM.S64 X4, X12, #2 +0x0000000000009584 (+0x000092c4) 00c6b78a AND.B64 X3, X11, X15 +0x0000000000009588 (+0x000092c8) 02c8020e SHL.B64 X4, #14 +0x000000000000958c (+0x000092cc) 000aa481 ADD.S64 X5, X10, X9 +0x0000000000009590 (+0x000092d0) 000a3204 MADD.S64 X5, X3, X4 +0x0000000000009594 (+0x000092d4) 00065201 ADD.S64 X3, X5, X4 +0x0000000000009598 (+0x000092d8) 08863680 SUB_IMM.S64 X3, X3, #1664 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x000000000000959c (+0x000092dc) 08063000 ADD_IMM.S64 X3, X3, #0 +0x00000000000095a0 (+0x000092e0) 08343548 ADD_IMM.S64 X26, X3, #1352 +0x00000000000095a4 (+0x000092e4) 1cb83558 LD_XD_XN_IMM.B32 X28, X3, #1368 +0x00000000000095a8 (+0x000092e8) 0cf5ad80 LDP_XI_XJ_XN.B64 X26, X27, X26, #0 +0x00000000000095ac (+0x000092ec) 0001a70e CMP.S64.EQ X26, X14 +0x00000000000095b0 (+0x000092f0) 4020002a JUMPC #42 +0x00000000000095b4 (+0x000092f4) 0001b70e CMP.S64.EQ X27, X14 +0x00000000000095b8 (+0x000092f8) 40200028 JUMPC #40 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000095bc (+0x000092fc) 0289ca00 ZEROEXT.U32 X4, X28 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x00000000000095c0 (+0x00009300) 0000470e CMP.S64.EQ X4, X14 +0x00000000000095c4 (+0x00009304) 40200025 JUMPC #37 +# [DWARF] common/pa_trace.h:547 +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# > 547 | const uint32_t slot = core.count; +0x00000000000095c8 (+0x00009308) 1c87a000 LD_XD_XN_IMM.B32 X3, X26, #0 +# [DWARF] common/pa_trace.h:548 +# > 548 | if (slot >= trace.capacity) { +0x00000000000095cc (+0x0000930c) 0040322e CMP.U64.LT X3, X4 +0x00000000000095d0 (+0x00009310) 40200002 JUMPC #2 +0x00000000000095d4 (+0x00009314) 4000001e JUMP #30 +# [DWARF] common/pa_trace.h:552 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x00000000000095d8 (+0x00009318) 02083800 MOV_XD_XN.S64 X4, X3 +0x00000000000095dc (+0x0000931c) 02c80206 SHL.B64 X4, #6 +0x00000000000095e0 (+0x00009320) 0009b201 ADD.S64 X4, X27, X4 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000095e4 (+0x00009324) 080cc002 ADD_IMM.S64 X6, X12, #2 +# [DWARF] common/pa_trace.h:553 +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# > 553 | record.start_cycle = start_cycle; +0x00000000000095e8 (+0x00009328) 09c24101 STP_XI_XJ_XN.B64 X1, X2, X4, #0 +0x00000000000095ec (+0x0000932c) 00cab78a AND.B64 X5, X11, X15 +# [DWARF] common/pa_trace.h:555 +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x00000000000095f0 (+0x00009330) 038e4010 ST_XD_XN_IMM.B32 X7, X4, #16 +0x00000000000095f4 (+0x00009334) 02cc020e SHL.B64 X6, #14 +0x00000000000095f8 (+0x00009338) 0002a481 ADD.S64 X1, X10, X9 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000095fc (+0x0000933c) 00025304 MADD.S64 X1, X5, X6 +# [DWARF] common/pa_trace.h:556 +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# > 556 | record.function_id = function_id; +0x0000000000009600 (+0x00009340) 0704ffff MOV_XD_IMM X2, #65535 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009604 (+0x00009344) 00021301 ADD.S64 X1, X1, X6 +0x0000000000009608 (+0x00009348) 08821680 SUB_IMM.S64 X1, X1, #1664 +# [DWARF] common/pa_trace.h:556 +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# > 556 | record.function_id = function_id; +0x000000000000960c (+0x0000934c) 0745ffff MOVK X2, #65535, #1 +0x0000000000009610 (+0x00009350) 03844014 ST_XD_XN_IMM.B32 X2, X4, #20 +# [DWARF] common/pa_trace.h:558 +# 557 | record.phase = static_cast(trace_phase); +# > 558 | record.lane = trace.lane; +0x0000000000009614 (+0x00009354) 08021000 ADD_IMM.S64 X1, X1, #0 +0x0000000000009618 (+0x00009358) 1c841560 LD_XD_XN_IMM.B32 X2, X1, #1376 +# [DWARF] common/pa_trace.h:557 +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x000000000000961c (+0x0000935c) 070a000e MOV_XD_IMM X5, #14 +0x0000000000009620 (+0x00009360) 098a4131 STP_XI_XJ_XN.B32 X5, X2, X4, #24 +# [DWARF] common/pa_trace.h:559 +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x0000000000009624 (+0x00009364) 08041564 ADD_IMM.S64 X2, X1, #1380 +0x0000000000009628 (+0x00009368) 0c842080 LDP_XI_XJ_XN.B32 X2, X1, X2, #0 +0x000000000000962c (+0x0000936c) 080a4020 ADD_IMM.S64 X5, X4, #32 +0x0000000000009630 (+0x00009370) 09845081 STP_XI_XJ_XN.B32 X2, X1, X5, #0 +# [DWARF] common/pa_trace.h:561 +# 560 | record.core_idx = trace.core_idx; +# > 561 | record.flags = flags; +0x0000000000009634 (+0x00009374) 07020053 MOV_XD_IMM X1, #83 +0x0000000000009638 (+0x00009378) 07830004 MOVK X1, #4, #2 +0x000000000000963c (+0x0000937c) 03c24028 ST_XD_XN_IMM.B64 X1, X4, #40 +# [DWARF] common/pa_trace.h:565 +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x0000000000009640 (+0x00009380) 08023001 ADD_IMM.S64 X1, X3, #1 +0x0000000000009644 (+0x00009384) 0383a000 ST_XD_XN_IMM.B32 X1, X26, #0 +0x0000000000009648 (+0x00009388) 40000004 JUMP #4 +# [DWARF] common/pa_trace.h:549 +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# > 549 | core.dropped = core.dropped + 1; +0x000000000000964c (+0x0000938c) 1c83a004 LD_XD_XN_IMM.B32 X1, X26, #4 +0x0000000000009650 (+0x00009390) 08021001 ADD_IMM.S64 X1, X1, #1 +0x0000000000009654 (+0x00009394) 0383a004 ST_XD_XN_IMM.B32 X1, X26, #4 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009658 (+0x00009398) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x000000000000965c (+0x0000939c) 00c2b78a AND.B64 X1, X11, X15 +0x0000000000009660 (+0x000093a0) 02c4020e SHL.B64 X2, #14 +0x0000000000009664 (+0x000093a4) 0006a481 ADD.S64 X3, X10, X9 +0x0000000000009668 (+0x000093a8) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_scheduler_core.h:514 +# 508 | outcome.attempted = true; +# 509 | // atomicMax 返回写入前的 cursor：old=task_id 则必须 Replay。 +# 510 | const int64_t old = TraceAtomicFetchMax( +# 511 | stats.trace, stats.result, static_cast(task_id), AtomicSite::ClaimMax, +# 512 | &cursor->value, static_cast(task_id), outcome.retries +# 513 | ); +# > 514 | outcome.won = old < static_cast(task_id); +0x000000000000966c (+0x000093ac) 000003ae CMP.S64.LT X0, X7 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x0000000000009670 (+0x000093b0) 00003101 ADD.S64 X0, X3, X2 +0x0000000000009674 (+0x000093b4) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_scheduler_core.h:1144 +# 1138 | efdrain_begin, efdrain_end +# 1139 | ); +# 1140 | +# 1141 | const uint64_t claim_begin = efdrain_end; +# 1142 | BeginSubmitPmuPhase(pmu_context); +# 1143 | const ClaimOutcome claim = Claim(state, worker, task_id, Kind, stats); +# > 1144 | context.won = claim.won; +0x0000000000009678 (+0x000093b8) 08000000 ADD_IMM.S64 X0, X0, #0 +# [DWARF] common/pa_scheduler_core.h:514 +# 508 | outcome.attempted = true; +# 509 | // atomicMax 返回写入前的 cursor：old=task_id 则必须 Replay。 +# 510 | const int64_t old = TraceAtomicFetchMax( +# 511 | stats.trace, stats.result, static_cast(task_id), AtomicSite::ClaimMax, +# 512 | &cursor->value, static_cast(task_id), outcome.retries +# 513 | ); +# > 514 | outcome.won = old < static_cast(task_id); +0x000000000000967c (+0x000093bc) 0202b880 MOV_XD_SPR.S64 X1, CONDITION_FLAG +# [DWARF] common/pa_scheduler_core.h:1144 +# 1138 | efdrain_begin, efdrain_end +# 1139 | ); +# 1140 | +# 1141 | const uint64_t claim_begin = efdrain_end; +# 1142 | BeginSubmitPmuPhase(pmu_context); +# 1143 | const ClaimOutcome claim = Claim(state, worker, task_id, Kind, stats); +# > 1144 | context.won = claim.won; +0x0000000000009680 (+0x000093c0) 03020198 ST_XD_XN_IMM.B8 X1, X0, #408 +0x0000000000009684 (+0x000093c4) 070c0000 MOV_XD_IMM X6, #0 +# [DWARF] common/pa_scheduler_core.h:1145 +# > 1145 | context.kernel_id = claim.function_id; +0x0000000000009688 (+0x000093c8) 0f860282 STI_XN_IMM.B32 X0, #404 +# [DWARF] common/pa_scheduler_core.h:520 +# 514 | outcome.won = old < static_cast(task_id); +# 515 | if (!outcome.won) outcome.function_id = -1; +# 516 | return outcome; +# 517 | } +# 518 | +# 519 | PA_DEVICE void RecordClaimOutcome(LocalStats &stats, TaskKind kind, const ClaimOutcome &outcome) { +# > 520 | if (outcome.attempted) ++stats.result.claim_attempts; +0x000000000000968c (+0x000093cc) 1cc201e8 LD_XD_XN_IMM.B64 X1, X0, #488 +0x0000000000009690 (+0x000093d0) 08021001 ADD_IMM.S64 X1, X1, #1 +0x0000000000009694 (+0x000093d4) 03c201e8 ST_XD_XN_IMM.B64 X1, X0, #488 +# [DWARF] common/pa_scheduler_core.h:522 +# 521 | stats.result.cas_retries += outcome.retries; +# > 522 | if (outcome.won) { +0x0000000000009698 (+0x000093d8) 40200002 JUMPC #2 +0x000000000000969c (+0x000093dc) 4000000a JUMP #10 +# [DWARF] common/pa_scheduler_core.h:523 +# > 523 | ++stats.result.claim_wins; +0x00000000000096a0 (+0x000093e0) 1cc201f0 LD_XD_XN_IMM.B64 X1, X0, #496 +0x00000000000096a4 (+0x000093e4) 070c0001 MOV_XD_IMM X6, #1 +# [DWARF] common/pa_scheduler_core.h:524 +# > 524 | ++stats.result.wins[KindIndex(kind)]; +0x00000000000096a8 (+0x000093e8) 1cc40220 LD_XD_XN_IMM.B64 X2, X0, #544 +# [DWARF] common/pa_scheduler_core.h:523 +# 517 | } +# 518 | +# 519 | PA_DEVICE void RecordClaimOutcome(LocalStats &stats, TaskKind kind, const ClaimOutcome &outcome) { +# 520 | if (outcome.attempted) ++stats.result.claim_attempts; +# 521 | stats.result.cas_retries += outcome.retries; +# 522 | if (outcome.won) { +# > 523 | ++stats.result.claim_wins; +0x00000000000096ac (+0x000093ec) 08021001 ADD_IMM.S64 X1, X1, #1 +0x00000000000096b0 (+0x000093f0) 03c201f0 ST_XD_XN_IMM.B64 X1, X0, #496 +# [DWARF] common/pa_scheduler_core.h:524 +# > 524 | ++stats.result.wins[KindIndex(kind)]; +0x00000000000096b4 (+0x000093f4) 08022001 ADD_IMM.S64 X1, X2, #1 +0x00000000000096b8 (+0x000093f8) 03c20220 ST_XD_XN_IMM.B64 X1, X0, #544 +0x00000000000096bc (+0x000093fc) 40000002 JUMP #2 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x00000000000096c0 (+0x00009400) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x00000000000096c4 (+0x00009404) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x00000000000096c8 (+0x00009408) 00c0b78a AND.B64 X0, X11, X15 +0x00000000000096cc (+0x0000940c) 02c2020e SHL.B64 X1, #14 +0x00000000000096d0 (+0x00009410) 0004a481 ADD.S64 X2, X10, X9 +0x00000000000096d4 (+0x00009414) 00040084 MADD.S64 X2, X0, X1 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x00000000000096d8 (+0x00009418) 02aa8880 MOV_XD_SPR.F32 X21, SYS_CNT +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x00000000000096dc (+0x0000941c) 00002081 ADD.S64 X0, X2, X1 +0x00000000000096e0 (+0x00009420) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:257 +# 251 | TraceContext &trace, uint64_t end_cycle +# 252 | ) { +# 253 | #if PA_BUILD_SUBMIT_PMU +# 254 | (void)trace; +# 255 | (void)end_cycle; +# 256 | #else +# > 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +0x00000000000096e4 (+0x00009424) 08000000 ADD_IMM.S64 X0, X0, #0 +0x00000000000096e8 (+0x00009428) 1c02055c LD_XD_XN_IMM.B8 X1, X0, #1372 +0x00000000000096ec (+0x0000942c) 0000170e CMP.S64.EQ X1, X14 +0x00000000000096f0 (+0x00009430) 4020007c JUMPC #124 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000096f4 (+0x00009434) 1ca605d0 LD_XD_XN_IMM.B32 X19, X0, #1488 +0x00000000000096f8 (+0x00009438) 02813a00 ZEROEXT.U32 X0, X19 +# [DWARF] common/pa_trace.h:257 +# 251 | TraceContext &trace, uint64_t end_cycle +# 252 | ) { +# 253 | #if PA_BUILD_SUBMIT_PMU +# 254 | (void)trace; +# 255 | (void)end_cycle; +# 256 | #else +# > 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +0x00000000000096fc (+0x0000943c) 0000070e CMP.S64.EQ X0, X14 +0x0000000000009700 (+0x00009440) 40200078 JUMPC #120 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009704 (+0x00009444) 072e0000 MOV_XD_IMM X23, #0 +0x0000000000009708 (+0x00009448) 03cde988 ST_XD_XN_IMM.B64 X6, X30, #2440 +0x000000000000970c (+0x0000944c) 07300000 MOV_XD_IMM X24, #0 +0x0000000000009710 (+0x00009450) 0728c488 MOV_XD_IMM X20, #50312 +0x0000000000009714 (+0x00009454) 07690007 MOVK X20, #7, #1 +0x0000000000009718 (+0x00009458) 07a90000 MOVK X20, #0, #2 +0x000000000000971c (+0x0000945c) 07e90000 MOVK X20, #0, #3 +0x0000000000009720 (+0x00009460) 02020880 MOV_XD_SPR.S64 X1, PC +0x0000000000009724 (+0x00009464) 00294081 ADD.S64 X20, X20, X1 +0x0000000000009728 (+0x00009468) 40000012 JUMP #18 +0x000000000000972c (+0x0000946c) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x0000000000009730 (+0x00009470) 00c2b78a AND.B64 X1, X11, X15 +0x0000000000009734 (+0x00009474) 02c4020e SHL.B64 X2, #14 +0x0000000000009738 (+0x00009478) 0006a481 ADD.S64 X3, X10, X9 +0x000000000000973c (+0x0000947c) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:280 +# 274 | call_count, static_cast(site) +# 275 | ); +# 276 | if (written) { +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# 278 | trace.atomic_counter_overflow = true; +# 279 | } else { +# > 280 | ++trace.poll_batch_records; +0x0000000000009740 (+0x00009480) 08000001 ADD_IMM.S64 X0, X0, #1 +0x0000000000009744 (+0x00009484) 00023101 ADD.S64 X1, X3, X2 +0x0000000000009748 (+0x00009488) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x000000000000974c (+0x0000948c) 08021000 ADD_IMM.S64 X1, X1, #0 +0x0000000000009750 (+0x00009490) 03c01578 ST_XD_XN_IMM.B64 X0, X1, #1400 +# [DWARF] common/pa_trace.h:283 +# 281 | } +# 282 | } +# > 283 | trace.poll_burst.call_count[index] = 0; +0x0000000000009754 (+0x00009494) 0f976700 STI_XN_IMM.B32 X22, #1464 +# [DWARF] common/pa_trace.h:262 +# 256 | #else +# 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +# 258 | const uint32_t active_mask = trace.poll_burst.active_mask; +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# > 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +0x0000000000009758 (+0x00009498) 08318001 ADD_IMM.S64 X24, X24, #1 +0x000000000000975c (+0x0000949c) 08294004 ADD_IMM.S64 X20, X20, #4 +0x0000000000009760 (+0x000094a0) 0001841e CMP.S64.NE X24, X8 +0x0000000000009764 (+0x000094a4) 082f7001 ADD_IMM.S64 X23, X23, #1 +0x0000000000009768 (+0x000094a8) 40200002 JUMPC #2 +0x000000000000976c (+0x000094ac) 40000050 JUMP #80 +# [DWARF] common/pa_trace.h:263 +# > 263 | const uint32_t bit = 1U << index; +0x0000000000009770 (+0x000094b0) 02817a00 ZEROEXT.U32 X0, X23 +# [DWARF] common/pa_trace.h:264 +# > 264 | if ((active_mask & bit) == 0) continue; +0x0000000000009774 (+0x000094b4) 02833a00 ZEROEXT.U32 X1, X19 +0x0000000000009778 (+0x000094b8) 024202c0 SHR.U64 X1, X0, #0 +0x000000000000977c (+0x000094bc) 00c0188a AND.B64 X0, X1, X17 +0x0000000000009780 (+0x000094c0) 0000070e CMP.S64.EQ X0, X14 +0x0000000000009784 (+0x000094c4) 4020fff5 JUMPC #65525 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009788 (+0x000094c8) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x000000000000978c (+0x000094cc) 00c0b78a AND.B64 X0, X11, X15 +0x0000000000009790 (+0x000094d0) 02c4020e SHL.B64 X2, #14 +0x0000000000009794 (+0x000094d4) 0006a481 ADD.S64 X3, X10, X9 +0x0000000000009798 (+0x000094d8) 00060104 MADD.S64 X3, X0, X2 +# [DWARF] common/pa_trace.h:265 +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +# 263 | const uint32_t bit = 1U << index; +# 264 | if ((active_mask & bit) == 0) continue; +# > 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +0x000000000000979c (+0x000094dc) 02038800 MOV_XD_XN.S64 X1, X24 +0x00000000000097a0 (+0x000094e0) 00003101 ADD.S64 X0, X3, X2 +0x00000000000097a4 (+0x000094e4) 08800680 SUB_IMM.S64 X0, X0, #1664 +0x00000000000097a8 (+0x000094e8) 02c20202 SHL.B64 X1, #2 +0x00000000000097ac (+0x000094ec) 08000000 ADD_IMM.S64 X0, X0, #0 +0x00000000000097b0 (+0x000094f0) 002c0081 ADD.S64 X22, X0, X1 +0x00000000000097b4 (+0x000094f4) 1c8b65b8 LD_XD_XN_IMM.B32 X5, X22, #1464 +# [DWARF] common/pa_trace.h:266 +# > 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +0x00000000000097b8 (+0x000094f8) 07020000 MOV_XD_IMM X1, #0 +0x00000000000097bc (+0x000094fc) 07430100 MOVK X1, #256, #1 +0x00000000000097c0 (+0x00009500) 07040000 MOV_XD_IMM X2, #0 +0x00000000000097c4 (+0x00009504) 0745ff00 MOVK X2, #65280, #1 +0x00000000000097c8 (+0x00009508) 00025082 SUB.S64 X1, X5, X1 +0x00000000000097cc (+0x0000950c) 02821a00 ZEROEXT.U32 X1, X1 +0x00000000000097d0 (+0x00009510) 0040113e CMP.U64.GT X1, X2 +0x00000000000097d4 (+0x00009514) 40200003 JUMPC #3 +# [DWARF] common/pa_trace.h:267 +# > 267 | trace.atomic_counter_overflow = true; +0x00000000000097d8 (+0x00009518) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x00000000000097dc (+0x0000951c) 4000ffdf JUMP #65503 +# [DWARF] common/pa_trace.h:98 +# 92 | default: +# 93 | return -1; +# 94 | } +# 95 | } +# 96 | +# 97 | PA_DEVICE AtomicSite TraceAtomicPollBatchSite(uint32_t index) { +# > 98 | switch (index) { +0x00000000000097e0 (+0x00009520) 02817a00 ZEROEXT.U32 X0, X23 +0x00000000000097e4 (+0x00009524) 07020005 MOV_XD_IMM X1, #5 +0x00000000000097e8 (+0x00009528) 004000be CMP.U64.GT X0, X1 +0x00000000000097ec (+0x0000952c) 070c000f MOV_XD_IMM X6, #15 +0x00000000000097f0 (+0x00009530) 40200002 JUMPC #2 +0x00000000000097f4 (+0x00009534) 1c8d4000 LD_XD_XN_IMM.B32 X6, X20, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000097f8 (+0x00009538) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x00000000000097fc (+0x0000953c) 00c2b78a AND.B64 X1, X11, X15 +0x0000000000009800 (+0x00009540) 02c4020e SHL.B64 X2, #14 +0x0000000000009804 (+0x00009544) 0006a481 ADD.S64 X3, X10, X9 +0x0000000000009808 (+0x00009548) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:273 +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# > 273 | trace.poll_burst.start_cycle[index], end_cycle, +0x000000000000980c (+0x0000954c) 02018800 MOV_XD_XN.S64 X0, X24 +0x0000000000009810 (+0x00009550) 00023101 ADD.S64 X1, X3, X2 +0x0000000000009814 (+0x00009554) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x0000000000009818 (+0x00009558) 02c00203 SHL.B64 X0, #3 +0x000000000000981c (+0x0000955c) 08241000 ADD_IMM.S64 X18, X1, #0 +0x0000000000009820 (+0x00009560) 00012001 ADD.S64 X0, X18, X0 +0x0000000000009824 (+0x00009564) 1cc60588 LD_XD_XN_IMM.B64 X3, X0, #1416 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x0000000000009828 (+0x00009568) 0201a800 MOV_XD_XN.S64 X0, X26 +0x000000000000982c (+0x0000956c) 0203b800 MOV_XD_XN.S64 X1, X27 +0x0000000000009830 (+0x00009570) 0205c800 MOV_XD_XN.S64 X2, X28 +0x0000000000009834 (+0x00009574) 02095800 MOV_XD_XN.S64 X4, X21 +0x0000000000009838 (+0x00009578) 070e789f MOV_XD_IMM X7, #30879 +0x000000000000983c (+0x0000957c) 074f0000 MOVK X7, #0, #1 +0x0000000000009840 (+0x00009580) 078f0000 MOVK X7, #0, #2 +0x0000000000009844 (+0x00009584) 40427000 CALL X7, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009848 (+0x00009588) 1cd3e990 LD_XD_XN_IMM.B64 X9, X30, #2448 +0x000000000000984c (+0x0000958c) 071c0000 MOV_XD_IMM X14, #0 +0x0000000000009850 (+0x00009590) 1cd9e9a0 LD_XD_XN_IMM.B64 X12, X30, #2464 +0x0000000000009854 (+0x00009594) 071e7fff MOV_XD_IMM X15, #32767 +0x0000000000009858 (+0x00009598) 1cd7e9a8 LD_XD_XN_IMM.B64 X11, X30, #2472 +# [DWARF] common/pa_trace.h:276 +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x000000000000985c (+0x0000959c) 0000070e CMP.S64.EQ X0, X14 +0x0000000000009860 (+0x000095a0) 1cd5e9b0 LD_XD_XN_IMM.B64 X10, X30, #2480 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009864 (+0x000095a4) 07220001 MOV_XD_IMM X17, #1 +0x0000000000009868 (+0x000095a8) 07100006 MOV_XD_IMM X8, #6 +# [DWARF] common/pa_trace.h:276 +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x000000000000986c (+0x000095ac) 4020ffba JUMPC #65466 +# [DWARF] common/pa_trace.h:277 +# > 277 | if (trace.poll_batch_records == UINT64_MAX) { +0x0000000000009870 (+0x000095b0) 1cc12578 LD_XD_XN_IMM.B64 X0, X18, #1400 +0x0000000000009874 (+0x000095b4) 07020001 MOV_XD_IMM X1, #1 +0x0000000000009878 (+0x000095b8) 02021080 NEG.S64 X1, X1 +0x000000000000987c (+0x000095bc) 0000009e CMP.S64.NE X0, X1 +0x0000000000009880 (+0x000095c0) 4020ffab JUMPC #65451 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009884 (+0x000095c4) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x0000000000009888 (+0x000095c8) 00c0b78a AND.B64 X0, X11, X15 +0x000000000000988c (+0x000095cc) 02c2020e SHL.B64 X1, #14 +0x0000000000009890 (+0x000095d0) 0004a481 ADD.S64 X2, X10, X9 +0x0000000000009894 (+0x000095d4) 00040084 MADD.S64 X2, X0, X1 +0x0000000000009898 (+0x000095d8) 00002081 ADD.S64 X0, X2, X1 +0x000000000000989c (+0x000095dc) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:278 +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# 276 | if (written) { +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# > 278 | trace.atomic_counter_overflow = true; +0x00000000000098a0 (+0x000095e0) 08000000 ADD_IMM.S64 X0, X0, #0 +0x00000000000098a4 (+0x000095e4) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x00000000000098a8 (+0x000095e8) 4000ffab JUMP #65451 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000098ac (+0x000095ec) 1ccfe8f0 LD_XD_XN_IMM.B64 X7, X30, #2288 +0x00000000000098b0 (+0x000095f0) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x00000000000098b4 (+0x000095f4) 1ccde988 LD_XD_XN_IMM.B64 X6, X30, #2440 +0x00000000000098b8 (+0x000095f8) 00c0b78a AND.B64 X0, X11, X15 +0x00000000000098bc (+0x000095fc) 02c2020e SHL.B64 X1, #14 +0x00000000000098c0 (+0x00009600) 0004a481 ADD.S64 X2, X10, X9 +0x00000000000098c4 (+0x00009604) 00040084 MADD.S64 X2, X0, X1 +0x00000000000098c8 (+0x00009608) 07280001 MOV_XD_IMM X20, #1 +0x00000000000098cc (+0x0000960c) 00002081 ADD.S64 X0, X2, X1 +0x00000000000098d0 (+0x00009610) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:285 +# 279 | } else { +# 280 | ++trace.poll_batch_records; +# 281 | } +# 282 | } +# 283 | trace.poll_burst.call_count[index] = 0; +# 284 | } +# > 285 | trace.poll_burst.active_mask = 0; +0x00000000000098d4 (+0x00009614) 08000000 ADD_IMM.S64 X0, X0, #0 +0x00000000000098d8 (+0x00009618) 02294080 NEG.S64 X20, X20 +0x00000000000098dc (+0x0000961c) 0f960a00 STI_XN_IMM.B32 X0, #1488 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x00000000000098e0 (+0x00009620) 0001a70e CMP.S64.EQ X26, X14 +0x00000000000098e4 (+0x00009624) 4020002d JUMPC #45 +0x00000000000098e8 (+0x00009628) 0001b70e CMP.S64.EQ X27, X14 +0x00000000000098ec (+0x0000962c) 4020002b JUMPC #43 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x00000000000098f0 (+0x00009630) 0283ca00 ZEROEXT.U32 X1, X28 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x00000000000098f4 (+0x00009634) 0000170e CMP.S64.EQ X1, X14 +0x00000000000098f8 (+0x00009638) 40200028 JUMPC #40 +# [DWARF] common/pa_trace.h:547 +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# > 547 | const uint32_t slot = core.count; +0x00000000000098fc (+0x0000963c) 1c81a000 LD_XD_XN_IMM.B32 X0, X26, #0 +# [DWARF] common/pa_trace.h:548 +# > 548 | if (slot >= trace.capacity) { +0x0000000000009900 (+0x00009640) 004000ae CMP.U64.LT X0, X1 +0x0000000000009904 (+0x00009644) 40200002 JUMPC #2 +0x0000000000009908 (+0x00009648) 40000021 JUMP #33 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000990c (+0x0000964c) 0806c002 ADD_IMM.S64 X3, X12, #2 +0x0000000000009910 (+0x00009650) 00c4b78a AND.B64 X2, X11, X15 +0x0000000000009914 (+0x00009654) 02c6020e SHL.B64 X3, #14 +0x0000000000009918 (+0x00009658) 0008a481 ADD.S64 X4, X10, X9 +# [DWARF] common/pa_trace.h:552 +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x000000000000991c (+0x0000965c) 02020800 MOV_XD_XN.S64 X1, X0 +0x0000000000009920 (+0x00009660) 00082184 MADD.S64 X4, X2, X3 +0x0000000000009924 (+0x00009664) 02c20206 SHL.B64 X1, #6 +0x0000000000009928 (+0x00009668) 00044181 ADD.S64 X2, X4, X3 +# [DWARF] common/pa_trace.h:556 +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# > 556 | record.function_id = function_id; +0x000000000000992c (+0x0000966c) 0706ffff MOV_XD_IMM X3, #65535 +# [DWARF] common/pa_trace.h:552 +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x0000000000009930 (+0x00009670) 0003b081 ADD.S64 X1, X27, X1 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009934 (+0x00009674) 08842680 SUB_IMM.S64 X2, X2, #1664 +# [DWARF] common/pa_trace.h:553 +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# > 553 | record.start_cycle = start_cycle; +0x0000000000009938 (+0x00009678) 09f21a81 STP_XI_XJ_XN.B64 X25, X21, X1, #0 +# [DWARF] common/pa_trace.h:556 +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# > 556 | record.function_id = function_id; +0x000000000000993c (+0x0000967c) 0747ffff MOVK X3, #65535, #1 +# [DWARF] common/pa_trace.h:555 +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x0000000000009940 (+0x00009680) 098e11a1 STP_XI_XJ_XN.B32 X7, X3, X1, #16 +# [DWARF] common/pa_trace.h:558 +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# > 558 | record.lane = trace.lane; +0x0000000000009944 (+0x00009684) 08042000 ADD_IMM.S64 X2, X2, #0 +0x0000000000009948 (+0x00009688) 1c862560 LD_XD_XN_IMM.B32 X3, X2, #1376 +# [DWARF] common/pa_trace.h:557 +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x000000000000994c (+0x0000968c) 0708000b MOV_XD_IMM X4, #11 +# [DWARF] common/pa_trace.h:565 +# 558 | record.lane = trace.lane; +# 559 | record.block_id = trace.block_id; +# 560 | record.core_idx = trace.core_idx; +# 561 | record.flags = flags; +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x0000000000009950 (+0x00009690) 08000001 ADD_IMM.S64 X0, X0, #1 +# [DWARF] common/pa_trace.h:557 +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x0000000000009954 (+0x00009694) 098811b1 STP_XI_XJ_XN.B32 X4, X3, X1, #24 +# [DWARF] common/pa_trace.h:559 +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x0000000000009958 (+0x00009698) 08062564 ADD_IMM.S64 X3, X2, #1380 +0x000000000000995c (+0x0000969c) 0c863100 LDP_XI_XJ_XN.B32 X3, X2, X3, #0 +# [DWARF] common/pa_trace.h:560 +# > 560 | record.core_idx = trace.core_idx; +0x0000000000009960 (+0x000096a0) 08081024 ADD_IMM.S64 X4, X1, #36 +# [DWARF] common/pa_trace.h:559 +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x0000000000009964 (+0x000096a4) 03861020 ST_XD_XN_IMM.B32 X3, X1, #32 +0x0000000000009968 (+0x000096a8) 07060000 MOV_XD_IMM X3, #0 +0x000000000000996c (+0x000096ac) 07470001 MOVK X3, #1, #1 +0x0000000000009970 (+0x000096b0) 004671af CMPN.U64.LT X3, X7, X3 +0x0000000000009974 (+0x000096b4) 02c60201 SHL.B64 X3, #1 +0x0000000000009978 (+0x000096b8) 00c6330b OR.B64 X3, X3, X6 +# [DWARF] common/pa_trace.h:560 +# > 560 | record.core_idx = trace.core_idx; +0x000000000000997c (+0x000096bc) 09844181 STP_XI_XJ_XN.B32 X2, X3, X4, #0 +# [DWARF] common/pa_trace.h:562 +# 561 | record.flags = flags; +# > 562 | record.auxiliary = auxiliary; +0x0000000000009980 (+0x000096c0) 03a2102c ST_XD_XN_IMM.B32 X17, X1, #44 +# [DWARF] common/pa_trace.h:565 +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x0000000000009984 (+0x000096c4) 0381a000 ST_XD_XN_IMM.B32 X0, X26, #0 +0x0000000000009988 (+0x000096c8) 40000004 JUMP #4 +# [DWARF] common/pa_trace.h:549 +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# > 549 | core.dropped = core.dropped + 1; +0x000000000000998c (+0x000096cc) 1c81a004 LD_XD_XN_IMM.B32 X0, X26, #4 +0x0000000000009990 (+0x000096d0) 08000001 ADD_IMM.S64 X0, X0, #1 +0x0000000000009994 (+0x000096d4) 0381a004 ST_XD_XN_IMM.B32 X0, X26, #4 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009998 (+0x000096d8) 0802c002 ADD_IMM.S64 X1, X12, #2 +# [DWARF] common/pa_frontend.h:262 +# 256 | // TensorTagMixin::tags_{} is value-initialized by the +# 257 | // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. +# 258 | // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar +# 259 | // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 +# 260 | volatile int32_t *tags = &args.tags[0]; +# 261 | for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { +# > 262 | tags[index] = 0; +0x000000000000999c (+0x000096dc) 0fa7e700 STI_XN_IMM.B32 X30, #2488 +# [DWARF] common/pa_frontend.h:0 +# [SOURCE unavailable] +0x00000000000099a0 (+0x000096e0) 07080648 MOV_XD_IMM X4, #1608 +0x00000000000099a4 (+0x000096e4) 00c0b78a AND.B64 X0, X11, X15 +0x00000000000099a8 (+0x000096e8) 02c2020e SHL.B64 X1, #14 +0x00000000000099ac (+0x000096ec) 0004a481 ADD.S64 X2, X10, X9 +0x00000000000099b0 (+0x000096f0) 0009e202 SUB.S64 X4, X30, X4 +0x00000000000099b4 (+0x000096f4) 00040084 MADD.S64 X2, X0, X1 +# [DWARF] common/pa_frontend.h:262 +# 256 | // TensorTagMixin::tags_{} is value-initialized by the +# 257 | // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. +# 258 | // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar +# 259 | // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 +# 260 | volatile int32_t *tags = &args.tags[0]; +# 261 | for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { +# > 262 | tags[index] = 0; +0x00000000000099b8 (+0x000096f8) 020a4800 MOV_XD_XN.S64 X5, X4 +0x00000000000099bc (+0x000096fc) 07060002 MOV_XD_IMM X3, #2 +0x00000000000099c0 (+0x00009700) 00002081 ADD.S64 X0, X2, X1 +0x00000000000099c4 (+0x00009704) 02ca3440 SBITSET.B64 X5, X3 +0x00000000000099c8 (+0x00009708) 08800680 SUB_IMM.S64 X0, X0, #1664 +0x00000000000099cc (+0x0000970c) 0f805000 STI_XN_IMM.B32 X5, #0 +0x00000000000099d0 (+0x00009710) 0fa7e800 STI_XN_IMM.B32 X30, #2496 +# [DWARF] common/pa_scheduler_core.h:948 +# 942 | callback(callback_builder); +# 943 | if (!callback_builder.Valid()) return false; +# 944 | const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); +# 945 | stats.result.arg_resets += counts.reset_calls; +# 946 | stats.result.views_created += counts.views_created; +# 947 | stats.result.dynamic_create_infos += counts.dynamic_create_infos; +# > 948 | stats.result.tensor_args_added += counts.tensor_args_added; +0x00000000000099d4 (+0x00009714) 08260000 ADD_IMM.S64 X19, X0, #0 +# [DWARF] common/pa_frontend.h:262 +# 256 | // TensorTagMixin::tags_{} is value-initialized by the +# 257 | // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. +# 258 | // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar +# 259 | // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 +# 260 | volatile int32_t *tags = &args.tags[0]; +# 261 | for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { +# > 262 | tags[index] = 0; +0x00000000000099d8 (+0x00009718) 0fa7e880 STI_XN_IMM.B32 X30, #2500 +# [DWARF] common/pa_frontend.h:314 +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# > 314 | args.tensors[index].pointer.create_info = &create_info; +0x00000000000099dc (+0x0000971c) 070201b8 MOV_XD_IMM X1, #440 +# [DWARF] common/pa_frontend.h:262 +# 256 | // TensorTagMixin::tags_{} is value-initialized by the +# 257 | // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. +# 258 | // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar +# 259 | // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 +# 260 | volatile int32_t *tags = &args.tags[0]; +# 261 | for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { +# > 262 | tags[index] = 0; +0x00000000000099e0 (+0x00009720) 0fa7e900 STI_XN_IMM.B32 X30, #2504 +# [DWARF] common/pa_frontend.h:314 +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# > 314 | args.tensors[index].pointer.create_info = &create_info; +0x00000000000099e4 (+0x00009724) 0003e081 ADD.S64 X1, X30, X1 +# [DWARF] common/pa_frontend.h:262 +# 256 | // TensorTagMixin::tags_{} is value-initialized by the +# 257 | // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. +# 258 | // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar +# 259 | // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 +# 260 | volatile int32_t *tags = &args.tags[0]; +# 261 | for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { +# > 262 | tags[index] = 0; +0x00000000000099e8 (+0x00009728) 0fa7e980 STI_XN_IMM.B32 X30, #2508 +# [DWARF] common/pa_frontend.h:313 +# 307 | args.tensors[index].pointer.gm_tensor = &tensor; +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# > 313 | const uint32_t index = static_cast(args.tensor_count++); +0x00000000000099ec (+0x0000972c) 07040003 MOV_XD_IMM X2, #3 +# [DWARF] common/pa_frontend.h:262 +# 256 | // TensorTagMixin::tags_{} is value-initialized by the +# 257 | // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. +# 258 | // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar +# 259 | // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 +# 260 | volatile int32_t *tags = &args.tags[0]; +# 261 | for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { +# > 262 | tags[index] = 0; +0x00000000000099f0 (+0x00009730) 0fa7ea00 STI_XN_IMM.B32 X30, #2512 +0x00000000000099f4 (+0x00009734) 0fa7ea80 STI_XN_IMM.B32 X30, #2516 +0x00000000000099f8 (+0x00009738) 0fa7eb00 STI_XN_IMM.B32 X30, #2520 +0x00000000000099fc (+0x0000973c) 0fa7eb80 STI_XN_IMM.B32 X30, #2524 +0x0000000000009a00 (+0x00009740) 0fa7ec00 STI_XN_IMM.B32 X30, #2528 +0x0000000000009a04 (+0x00009744) 0fa7ec80 STI_XN_IMM.B32 X30, #2532 +0x0000000000009a08 (+0x00009748) 0fa7ed00 STI_XN_IMM.B32 X30, #2536 +0x0000000000009a0c (+0x0000974c) 0fa7ed80 STI_XN_IMM.B32 X30, #2540 +0x0000000000009a10 (+0x00009750) 0fa7ee00 STI_XN_IMM.B32 X30, #2544 +0x0000000000009a14 (+0x00009754) 0fa7ee80 STI_XN_IMM.B32 X30, #2548 +0x0000000000009a18 (+0x00009758) 0fa7ef00 STI_XN_IMM.B32 X30, #2552 +0x0000000000009a1c (+0x0000975c) 0fa7ef80 STI_XN_IMM.B32 X30, #2556 +0x0000000000009a20 (+0x00009760) 0fa9e000 STI_XN_IMM.B32 X30, #2560 +0x0000000000009a24 (+0x00009764) 0fa9e080 STI_XN_IMM.B32 X30, #2564 +0x0000000000009a28 (+0x00009768) 0fa9e100 STI_XN_IMM.B32 X30, #2568 +0x0000000000009a2c (+0x0000976c) 0fa9e180 STI_XN_IMM.B32 X30, #2572 +0x0000000000009a30 (+0x00009770) 0fa9e200 STI_XN_IMM.B32 X30, #2576 +0x0000000000009a34 (+0x00009774) 0fa9e280 STI_XN_IMM.B32 X30, #2580 +0x0000000000009a38 (+0x00009778) 0fa9e300 STI_XN_IMM.B32 X30, #2584 +0x0000000000009a3c (+0x0000977c) 0fa9e380 STI_XN_IMM.B32 X30, #2588 +0x0000000000009a40 (+0x00009780) 0fa9e400 STI_XN_IMM.B32 X30, #2592 +0x0000000000009a44 (+0x00009784) 0fa9e480 STI_XN_IMM.B32 X30, #2596 +0x0000000000009a48 (+0x00009788) 0fa9e500 STI_XN_IMM.B32 X30, #2600 +0x0000000000009a4c (+0x0000978c) 0fa9e580 STI_XN_IMM.B32 X30, #2604 +0x0000000000009a50 (+0x00009790) 0fa9e600 STI_XN_IMM.B32 X30, #2608 +0x0000000000009a54 (+0x00009794) 0fa9e680 STI_XN_IMM.B32 X30, #2612 +# [DWARF] common/pa_frontend.h:265 +# 263 | } +# 264 | args.tensor_count = 0; +# > 265 | args.scalar_count = 0; +0x0000000000009a58 (+0x00009798) 0fb3e780 STI_XN_IMM.B32 X30, #3260 +# [DWARF] common/pa_frontend.h:266 +# > 266 | args.has_error = false; +0x0000000000009a5c (+0x0000979c) 0f33e800 STI_XN_IMM.B8 X30, #3264 +# [DWARF] common/pa_frontend.h:267 +# > 267 | args.error_msg = 0; +0x0000000000009a60 (+0x000097a0) 0ff3e900 STI_XN_IMM.B64 X30, #3272 +# [DWARF] common/pa_frontend.h:268 +# > 268 | args.launch_spec.core_num = 1; +0x0000000000009a64 (+0x000097a4) 0f73ea01 STI_XN_IMM.B16 X30, #3280 +# [DWARF] common/pa_frontend.h:269 +# > 269 | args.launch_spec.require_sync_start = false; +0x0000000000009a68 (+0x000097a8) 0f33ea40 STI_XN_IMM.B8 X30, #3282 +# [DWARF] common/pa_frontend.h:243 +# 237 | +# 238 | PA_DEVICE void ClearDumpArgSelection(PaDumpArgSelection &selection) { +# 239 | // Volatile stores intentionally preserve the profiling-enabled PA reset +# 240 | // traffic even though the standalone winner workload never consumes dump data. +# 241 | // volatile 的目的不是同步，而是阻止编译器删掉这段生产基线中存在的写流量。 +# 242 | volatile uint64_t *masks = &selection.dump_arg_mask; +# > 243 | masks[0] = 0; +0x0000000000009a6c (+0x000097ac) 0ff3eb00 STI_XN_IMM.B64 X30, #3288 +# [DWARF] common/pa_frontend.h:244 +# > 244 | masks[1] = 0; +0x0000000000009a70 (+0x000097b0) 0ff3ec00 STI_XN_IMM.B64 X30, #3296 +# [DWARF] common/pa_frontend.h:247 +# 245 | volatile uint64_t *sources = &selection.scalar_source_ptrs[0]; +# 246 | for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { +# > 247 | sources[index] = 0; +0x0000000000009a74 (+0x000097b4) 0ff3ed00 STI_XN_IMM.B64 X30, #3304 +0x0000000000009a78 (+0x000097b8) 0ff3ee00 STI_XN_IMM.B64 X30, #3312 +0x0000000000009a7c (+0x000097bc) 0ff3ef00 STI_XN_IMM.B64 X30, #3320 +0x0000000000009a80 (+0x000097c0) 0ff5e000 STI_XN_IMM.B64 X30, #3328 +0x0000000000009a84 (+0x000097c4) 0ff5e100 STI_XN_IMM.B64 X30, #3336 +0x0000000000009a88 (+0x000097c8) 0ff5e200 STI_XN_IMM.B64 X30, #3344 +0x0000000000009a8c (+0x000097cc) 0ff5e300 STI_XN_IMM.B64 X30, #3352 +0x0000000000009a90 (+0x000097d0) 0ff5e400 STI_XN_IMM.B64 X30, #3360 +0x0000000000009a94 (+0x000097d4) 0ff5e500 STI_XN_IMM.B64 X30, #3368 +0x0000000000009a98 (+0x000097d8) 0ff5e600 STI_XN_IMM.B64 X30, #3376 +0x0000000000009a9c (+0x000097dc) 0ff5e700 STI_XN_IMM.B64 X30, #3384 +0x0000000000009aa0 (+0x000097e0) 0ff5e800 STI_XN_IMM.B64 X30, #3392 +0x0000000000009aa4 (+0x000097e4) 0ff5e900 STI_XN_IMM.B64 X30, #3400 +0x0000000000009aa8 (+0x000097e8) 0ff5ea00 STI_XN_IMM.B64 X30, #3408 +0x0000000000009aac (+0x000097ec) 0ff5eb00 STI_XN_IMM.B64 X30, #3416 +0x0000000000009ab0 (+0x000097f0) 0ff5ec00 STI_XN_IMM.B64 X30, #3424 +# [DWARF] common/pa_frontend.h:251 +# 248 | } +# 249 | volatile uint8_t *dtypes = &selection.scalar_dtypes[0]; +# 250 | for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { +# > 251 | dtypes[index] = 0; +0x0000000000009ab4 (+0x000097f4) 0f35ed00 STI_XN_IMM.B8 X30, #3432 +0x0000000000009ab8 (+0x000097f8) 0f35ed20 STI_XN_IMM.B8 X30, #3433 +0x0000000000009abc (+0x000097fc) 0f35ed40 STI_XN_IMM.B8 X30, #3434 +0x0000000000009ac0 (+0x00009800) 0f35ed60 STI_XN_IMM.B8 X30, #3435 +0x0000000000009ac4 (+0x00009804) 0f35ed80 STI_XN_IMM.B8 X30, #3436 +0x0000000000009ac8 (+0x00009808) 0f35eda0 STI_XN_IMM.B8 X30, #3437 +0x0000000000009acc (+0x0000980c) 0f35edc0 STI_XN_IMM.B8 X30, #3438 +0x0000000000009ad0 (+0x00009810) 0f35ede0 STI_XN_IMM.B8 X30, #3439 +0x0000000000009ad4 (+0x00009814) 0f35ee00 STI_XN_IMM.B8 X30, #3440 +0x0000000000009ad8 (+0x00009818) 0f35ee20 STI_XN_IMM.B8 X30, #3441 +0x0000000000009adc (+0x0000981c) 0f35ee40 STI_XN_IMM.B8 X30, #3442 +0x0000000000009ae0 (+0x00009820) 0f35ee60 STI_XN_IMM.B8 X30, #3443 +0x0000000000009ae4 (+0x00009824) 0f35ee80 STI_XN_IMM.B8 X30, #3444 +0x0000000000009ae8 (+0x00009828) 0f35eea0 STI_XN_IMM.B8 X30, #3445 +0x0000000000009aec (+0x0000982c) 0f35eec0 STI_XN_IMM.B8 X30, #3446 +0x0000000000009af0 (+0x00009830) 0f35eee0 STI_XN_IMM.B8 X30, #3447 +# [DWARF] common/pa_scheduler_core.h:948 +# 942 | callback(callback_builder); +# 943 | if (!callback_builder.Valid()) return false; +# 944 | const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); +# 945 | stats.result.arg_resets += counts.reset_calls; +# 946 | stats.result.views_created += counts.views_created; +# 947 | stats.result.dynamic_create_infos += counts.dynamic_create_infos; +# > 948 | stats.result.tensor_args_added += counts.tensor_args_added; +0x0000000000009af4 (+0x00009834) 1cc133e0 LD_XD_XN_IMM.B64 X0, X19, #992 +# [DWARF] common/pa_frontend.h:314 +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# > 314 | args.tensors[index].pointer.create_info = &create_info; +0x0000000000009af8 (+0x00009838) 03c3ea38 ST_XD_XN_IMM.B64 X1, X30, #2616 +# [DWARF] common/pa_frontend.h:0 +# [SOURCE unavailable] +0x0000000000009afc (+0x0000983c) 070201f8 MOV_XD_IMM X1, #504 +0x0000000000009b00 (+0x00009840) 0003e081 ADD.S64 X1, X30, X1 +# [DWARF] common/pa_frontend.h:271 +# 265 | args.scalar_count = 0; +# 266 | args.has_error = false; +# 267 | args.error_msg = 0; +# 268 | args.launch_spec.core_num = 1; +# 269 | args.launch_spec.require_sync_start = false; +# 270 | ClearDumpArgSelection(args.dump_arg_selection); +# > 271 | args.explicit_deps = 0; +0x0000000000009b04 (+0x00009844) 0ff5ef00 STI_XN_IMM.B64 X30, #3448 +# [DWARF] common/pa_frontend.h:272 +# > 272 | args.explicit_dep_count = 0; +0x0000000000009b08 (+0x00009848) 0fb7e000 STI_XN_IMM.B32 X30, #3456 +# [DWARF] common/pa_frontend.h:315 +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# 314 | args.tensors[index].pointer.create_info = &create_info; +# > 315 | args.tensors[index].kind = TensorRefKind::CreateInfo; +0x0000000000009b0c (+0x0000984c) 0307ea40 ST_XD_XN_IMM.B8 X3, X30, #2624 +# [DWARF] common/pa_frontend.h:316 +# > 316 | args.tags[index] = TagValue(TensorArgType::Output); +0x0000000000009b10 (+0x00009850) 0fa7e701 STI_XN_IMM.B32 X30, #2488 +# [DWARF] common/pa_frontend.h:314 +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# > 314 | args.tensors[index].pointer.create_info = &create_info; +0x0000000000009b14 (+0x00009854) 03c3ea48 ST_XD_XN_IMM.B64 X1, X30, #2632 +# [DWARF] common/pa_frontend.h:315 +# > 315 | args.tensors[index].kind = TensorRefKind::CreateInfo; +0x0000000000009b18 (+0x00009858) 0307ea50 ST_XD_XN_IMM.B8 X3, X30, #2640 +# [DWARF] common/pa_frontend.h:316 +# > 316 | args.tags[index] = TagValue(TensorArgType::Output); +0x0000000000009b1c (+0x0000985c) 0f805001 STI_XN_IMM.B32 X5, #0 +# [DWARF] common/pa_frontend.h:313 +# 307 | args.tensors[index].pointer.gm_tensor = &tensor; +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# > 313 | const uint32_t index = static_cast(args.tensor_count++); +0x0000000000009b20 (+0x00009860) 0385ecb8 ST_XD_XN_IMM.B32 X2, X30, #3256 +# [DWARF] common/pa_frontend.h:314 +# > 314 | args.tensors[index].pointer.create_info = &create_info; +0x0000000000009b24 (+0x00009864) 03c3ea58 ST_XD_XN_IMM.B64 X1, X30, #2648 +# [DWARF] ccec/ccec_ops.h:276 +# 270 | } +# 271 | +# 272 | __aicore__ static inline bool FinishLazySampleCallback( +# 273 | const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args +# 274 | ) { +# 275 | #if defined(PA_BUILD_AIC) +# > 276 | return ::pa_scheduler_lazy_sample_callback_finish_aic(ticket, args) != 0; +0x0000000000009b28 (+0x00009868) 02024800 MOV_XD_XN.S64 X1, X4 +0x0000000000009b2c (+0x0000986c) 03cbe8e8 ST_XD_XN_IMM.B64 X5, X30, #2280 +# [DWARF] common/pa_frontend.h:315 +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# 314 | args.tensors[index].pointer.create_info = &create_info; +# > 315 | args.tensors[index].kind = TensorRefKind::CreateInfo; +0x0000000000009b30 (+0x00009870) 0307ea60 ST_XD_XN_IMM.B8 X3, X30, #2656 +# [DWARF] common/pa_frontend.h:316 +# > 316 | args.tags[index] = TagValue(TensorArgType::Output); +0x0000000000009b34 (+0x00009874) 0fa7e801 STI_XN_IMM.B32 X30, #2496 +# [DWARF] common/pa_scheduler_core.h:1161 +# 1155 | ); +# 1156 | +# 1157 | if (!BuildLazySampleCallbackArgs(orch, args, batch, claim.won, stats)) { +# 1158 | SetFatal(state, stats, static_cast(task_id)); +# 1159 | return false; +# 1160 | } +# > 1161 | const LazySampleCallbackTicket ticket{ +0x0000000000009b38 (+0x00009878) 038fe340 ST_XD_XN_IMM.B32 X7, X30, #832 +# [DWARF] common/pa_scheduler_core.h:948 +# 942 | callback(callback_builder); +# 943 | if (!callback_builder.Valid()) return false; +# 944 | const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); +# 945 | stats.result.arg_resets += counts.reset_calls; +# 946 | stats.result.views_created += counts.views_created; +# 947 | stats.result.dynamic_create_infos += counts.dynamic_create_infos; +# > 948 | stats.result.tensor_args_added += counts.tensor_args_added; +0x0000000000009b3c (+0x0000987c) 08000003 ADD_IMM.S64 X0, X0, #3 +0x0000000000009b40 (+0x00009880) 03c133e0 ST_XD_XN_IMM.B64 X0, X19, #992 +# [DWARF] common/pa_scheduler_core.h:1161 +# 1155 | ); +# 1156 | +# 1157 | if (!BuildLazySampleCallbackArgs(orch, args, batch, claim.won, stats)) { +# 1158 | SetFatal(state, stats, static_cast(task_id)); +# 1159 | return false; +# 1160 | } +# > 1161 | const LazySampleCallbackTicket ticket{ +0x0000000000009b44 (+0x00009884) 1cc1e928 LD_XD_XN_IMM.B64 X0, X30, #2344 +0x0000000000009b48 (+0x00009888) 03c1e338 ST_XD_XN_IMM.B64 X0, X30, #824 +# [DWARF] ccec/ccec_ops.h:276 +# 270 | } +# 271 | +# 272 | __aicore__ static inline bool FinishLazySampleCallback( +# 273 | const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args +# 274 | ) { +# 275 | #if defined(PA_BUILD_AIC) +# > 276 | return ::pa_scheduler_lazy_sample_callback_finish_aic(ticket, args) != 0; +0x0000000000009b4c (+0x0000988c) 07000338 MOV_XD_IMM X0, #824 +0x0000000000009b50 (+0x00009890) 0001e001 ADD.S64 X0, X30, X0 +# [DWARF] common/pa_scheduler_core.h:1161 +# 1155 | ); +# 1156 | +# 1157 | if (!BuildLazySampleCallbackArgs(orch, args, batch, claim.won, stats)) { +# 1158 | SetFatal(state, stats, static_cast(task_id)); +# 1159 | return false; +# 1160 | } +# > 1161 | const LazySampleCallbackTicket ticket{ +0x0000000000009b54 (+0x00009894) 0f4de882 STI_XN_IMM.B16 X30, #836 +0x0000000000009b58 (+0x00009898) 030de346 ST_XD_XN_IMM.B8 X6, X30, #838 +0x0000000000009b5c (+0x0000989c) 0f0de8e0 STI_XN_IMM.B8 X30, #839 +# [DWARF] ccec/ccec_ops.h:276 +# 270 | } +# 271 | +# 272 | __aicore__ static inline bool FinishLazySampleCallback( +# 273 | const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args +# 274 | ) { +# 275 | #if defined(PA_BUILD_AIC) +# > 276 | return ::pa_scheduler_lazy_sample_callback_finish_aic(ticket, args) != 0; +0x0000000000009b60 (+0x000098a0) 0704780d MOV_XD_IMM X2, #30733 +0x0000000000009b64 (+0x000098a4) 07450000 MOVK X2, #0, #1 +0x0000000000009b68 (+0x000098a8) 07850000 MOVK X2, #0, #2 +0x0000000000009b6c (+0x000098ac) 40422000 CALL X2, #0 +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x0000000000009b70 (+0x000098b0) 1cf3e980 LD_XD_XN_IMM.B64 X25, X30, #2432 +0x0000000000009b74 (+0x000098b4) 07160000 MOV_XD_IMM X11, #0 +# [DWARF] ccec/ccec_ops.h:276 +# 270 | } +# 271 | +# 272 | __aicore__ static inline bool FinishLazySampleCallback( +# 273 | const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args +# 274 | ) { +# 275 | #if defined(PA_BUILD_AIC) +# > 276 | return ::pa_scheduler_lazy_sample_callback_finish_aic(ticket, args) != 0; +0x0000000000009b78 (+0x000098b8) 02800a00 ZEROEXT.U32 X0, X0 +0x0000000000009b7c (+0x000098bc) 07220001 MOV_XD_IMM X17, #1 +0x0000000000009b80 (+0x000098c0) 0000058e CMP.S64.EQ X0, X11 +0x0000000000009b84 (+0x000098c4) 07100006 MOV_XD_IMM X8, #6 +0x0000000000009b88 (+0x000098c8) 07120004 MOV_XD_IMM X9, #4 +0x0000000000009b8c (+0x000098cc) 071412c4 MOV_XD_IMM X10, #4804 +0x0000000000009b90 (+0x000098d0) 07200020 MOV_XD_IMM X16, #32 +# [DWARF] common/pa_scheduler_core.h:1382 +# 1376 | orchestration_begin = TraceTimestamp(stats.trace, stats.result); +# 1377 | InitPaOrchestration(orchestration, batches, &state->context_lens[0]); +# 1378 | for (uint32_t batch = 0; batch < batches; ++batch) { +# 1379 | #if defined(PA_LAZY_SAMPLE_SHAPE_ID) +# 1380 | BeginPaBatchForLazySampleCallback(orchestration, batch); +# 1381 | ++stats.result.context_reads; +# > 1382 | if (!SubmitLazySampleCallback< +0x0000000000009b94 (+0x000098d4) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x0000000000009b98 (+0x000098d8) 070058e1 MOV_XD_IMM X0, #22753 +0x0000000000009b9c (+0x000098dc) 07410000 MOVK X0, #0, #1 +0x0000000000009ba0 (+0x000098e0) 07810000 MOVK X0, #0, #2 +0x0000000000009ba4 (+0x000098e4) 40220000 JUMPC X0, #0 +0x0000000000009ba8 (+0x000098e8) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_frontend.h:550 +# 544 | } +# 545 | +# 546 | PA_DEVICE void PreparePaBlockGroup(PaOrchestrationState &orch, uint64_t block_offset) { +# 547 | // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 +# 548 | // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 +# 549 | orch.current_block_offset = block_offset; +# > 550 | orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); +0x0000000000009bac (+0x000098ec) 0801e2c8 ADD_IMM.S64 X0, X30, #712 +# [DWARF] common/pa_frontend.h:832 +# 826 | // 每个 worker 都完整回放并接收自己 materialize 的 descriptor；只有 Claim winner +# 827 | // 会执行 kernel，但 loser 的后续 orchestration 仍使用相同 task_id/owner 拓扑。 +# 828 | switch (kind) { +# 829 | case TaskKind::Alloc: +# 830 | orch.accumulated_output = outputs.tensors[0]; +# 831 | orch.accumulated_sum = outputs.tensors[1]; +# > 832 | orch.accumulated_max = outputs.tensors[2]; +0x0000000000009bb0 (+0x000098f0) 1cc93060 LD_XD_XN_IMM.B64 X4, X19, #96 +# [DWARF] common/pa_frontend.h:550 +# 544 | } +# 545 | +# 546 | PA_DEVICE void PreparePaBlockGroup(PaOrchestrationState &orch, uint64_t block_offset) { +# 547 | // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 +# 548 | // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 +# 549 | orch.current_block_offset = block_offset; +# > 550 | orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); +0x0000000000009bb4 (+0x000098f4) 0cc20000 LDP_XI_XJ_XN.B64 X1, X0, X0, #0 +# [DWARF] common/pa_frontend.h:830 +# 824 | +# 825 | PA_DEVICE void AcceptTaskOutputs(PaOrchestrationState &orch, TaskKind kind, const TaskOutputs &outputs) { +# 826 | // 每个 worker 都完整回放并接收自己 materialize 的 descriptor；只有 Claim winner +# 827 | // 会执行 kernel，但 loser 的后续 orchestration 仍使用相同 task_id/owner 拓扑。 +# 828 | switch (kind) { +# 829 | case TaskKind::Alloc: +# > 830 | orch.accumulated_output = outputs.tensors[0]; +0x0000000000009bb8 (+0x000098f8) 08053050 ADD_IMM.S64 X2, X19, #80 +0x0000000000009bbc (+0x000098fc) 070a0040 MOV_XD_IMM X5, #64 +0x0000000000009bc0 (+0x00009900) 0cc42180 LDP_XI_XJ_XN.B64 X2, X3, X2, #0 +# [DWARF] common/pa_frontend.h:0 +# [SOURCE unavailable] +0x0000000000009bc4 (+0x00009904) 071e7fff MOV_XD_IMM X15, #32767 +# [DWARF] common/pa_frontend.h:528 +# 522 | orch.query_view.strides[0] = kPaHeadDim; +# 523 | orch.query_view.strides[1] = 1; +# 524 | orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; +# 525 | } +# 526 | #endif +# 527 | +# > 528 | PA_DEVICE uint64_t MinU64(uint64_t lhs, uint64_t rhs) { return lhs < rhs ? lhs : rhs; } +0x0000000000009bc8 (+0x00009908) 004002ae CMP.U64.LT X0, X5 +0x0000000000009bcc (+0x0000990c) 00c00289 SEL.B64 X0, X0, X5 +# [DWARF] common/pa_frontend.h:550 +# 544 | } +# 545 | +# 546 | PA_DEVICE void PreparePaBlockGroup(PaOrchestrationState &orch, uint64_t block_offset) { +# 547 | // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 +# 548 | // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 +# 549 | orch.current_block_offset = block_offset; +# > 550 | orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); +0x0000000000009bd0 (+0x00009910) 03c1e2e0 ST_XD_XN_IMM.B64 X0, X30, #736 +# [DWARF] common/pa_frontend.h:552 +# 551 | const uint64_t last_block_sequence_start = +# > 552 | (block_offset + orch.current_nblocks - 1) * kPaBlockSize; +0x0000000000009bd4 (+0x00009914) 02c00207 SHL.B64 X0, #7 +0x0000000000009bd8 (+0x00009918) 00001002 SUB.S64 X0, X1, X0 +# [DWARF] common/pa_frontend.h:830 +# 824 | +# 825 | PA_DEVICE void AcceptTaskOutputs(PaOrchestrationState &orch, TaskKind kind, const TaskOutputs &outputs) { +# 826 | // 每个 worker 都完整回放并接收自己 materialize 的 descriptor；只有 Claim winner +# 827 | // 会执行 kernel，但 loser 的后续 orchestration 仍使用相同 task_id/owner 拓扑。 +# 828 | switch (kind) { +# 829 | case TaskKind::Alloc: +# > 830 | orch.accumulated_output = outputs.tensors[0]; +0x0000000000009bdc (+0x0000991c) 0803e2f8 ADD_IMM.S64 X1, X30, #760 +0x0000000000009be0 (+0x00009920) 09c41181 STP_XI_XJ_XN.B64 X2, X3, X1, #0 +0x0000000000009be4 (+0x00009924) 1cc5e960 LD_XD_XN_IMM.B64 X2, X30, #2400 +# [DWARF] common/pa_frontend.h:553 +# 547 | // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 +# 548 | // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 +# 549 | orch.current_block_offset = block_offset; +# 550 | orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); +# 551 | const uint64_t last_block_sequence_start = +# 552 | (block_offset + orch.current_nblocks - 1) * kPaBlockSize; +# > 553 | orch.current_valid_len = MinU64(kPaBlockSize, orch.current_sequence - last_block_sequence_start); +0x0000000000009be8 (+0x00009928) 08000080 ADD_IMM.S64 X0, X0, #128 +0x0000000000009bec (+0x0000992c) 07020080 MOV_XD_IMM X1, #128 +# [DWARF] common/pa_frontend.h:832 +# 826 | // 每个 worker 都完整回放并接收自己 materialize 的 descriptor；只有 Claim winner +# 827 | // 会执行 kernel，但 loser 的后续 orchestration 仍使用相同 task_id/owner 拓扑。 +# 828 | switch (kind) { +# 829 | case TaskKind::Alloc: +# 830 | orch.accumulated_output = outputs.tensors[0]; +# 831 | orch.accumulated_sum = outputs.tensors[1]; +# > 832 | orch.accumulated_max = outputs.tensors[2]; +0x0000000000009bf0 (+0x00009930) 03c9e308 ST_XD_XN_IMM.B64 X4, X30, #776 +# [DWARF] common/pa_frontend.h:528 +# 522 | orch.query_view.strides[0] = kPaHeadDim; +# 523 | orch.query_view.strides[1] = 1; +# 524 | orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; +# 525 | } +# 526 | #endif +# 527 | +# > 528 | PA_DEVICE uint64_t MinU64(uint64_t lhs, uint64_t rhs) { return lhs < rhs ? lhs : rhs; } +0x0000000000009bf4 (+0x00009934) 004000ae CMP.U64.LT X0, X1 +# [DWARF] common/pa_frontend.h:549 +# 543 | return static_cast(*value); +# 544 | } +# 545 | +# 546 | PA_DEVICE void PreparePaBlockGroup(PaOrchestrationState &orch, uint64_t block_offset) { +# 547 | // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 +# 548 | // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 +# > 549 | orch.current_block_offset = block_offset; +0x0000000000009bf8 (+0x00009938) 0fcbeb00 STI_XN_IMM.B64 X30, #728 +# [DWARF] common/pa_frontend.h:528 +# 522 | orch.query_view.strides[0] = kPaHeadDim; +# 523 | orch.query_view.strides[1] = 1; +# 524 | orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; +# 525 | } +# 526 | #endif +# 527 | +# > 528 | PA_DEVICE uint64_t MinU64(uint64_t lhs, uint64_t rhs) { return lhs < rhs ? lhs : rhs; } +0x0000000000009bfc (+0x0000993c) 00c00089 SEL.B64 X0, X0, X1 +# [DWARF] common/pa_frontend.h:553 +# 547 | // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 +# 548 | // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 +# 549 | orch.current_block_offset = block_offset; +# 550 | orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); +# 551 | const uint64_t last_block_sequence_start = +# 552 | (block_offset + orch.current_nblocks - 1) * kPaBlockSize; +# > 553 | orch.current_valid_len = MinU64(kPaBlockSize, orch.current_sequence - last_block_sequence_start); +0x0000000000009c00 (+0x00009940) 03c1e2e8 ST_XD_XN_IMM.B64 X0, X30, #744 +# [DWARF] common/pa_scheduler_core.h:790 +# 784 | +# 785 | PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { +# 786 | // This is BeginSubmit without an already-materialized TaskArgs. The same +# 787 | // fields are completed synchronously after the single callback builds args. +# 788 | const uint32_t task_id = static_cast(worker.local_index++); +# 789 | context.self = &worker; +# > 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +0x0000000000009c04 (+0x00009944) 070207ff MOV_XD_IMM X1, #2047 +# [DWARF] common/pa_scheduler_core.h:788 +# 782 | static_assert(offsetof(LazySampleCallbackTicket, function_id) == 12, "lazy sample callback ticket function offset mismatch"); +# 783 | static_assert(offsetof(LazySampleCallbackTicket, won) == 14, "lazy sample callback ticket winner offset mismatch"); +# 784 | +# 785 | PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { +# 786 | // This is BeginSubmit without an already-materialized TaskArgs. The same +# 787 | // fields are completed synchronously after the single callback builds args. +# > 788 | const uint32_t task_id = static_cast(worker.local_index++); +0x0000000000009c08 (+0x00009948) 1c862014 LD_XD_XN_IMM.B32 X3, X2, #20 +0x0000000000009c0c (+0x0000994c) 08003001 ADD_IMM.S64 X0, X3, #1 +0x0000000000009c10 (+0x00009950) 03c7e920 ST_XD_XN_IMM.B64 X3, X30, #2336 +0x0000000000009c14 (+0x00009954) 03802014 ST_XD_XN_IMM.B32 X0, X2, #20 +# [DWARF] common/pa_scheduler_core.h:790 +# 789 | context.self = &worker; +# > 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +0x0000000000009c18 (+0x00009958) 1cc1e900 LD_XD_XN_IMM.B64 X0, X30, #2304 +0x0000000000009c1c (+0x0000995c) 00c2308a AND.B64 X1, X3, X1 +0x0000000000009c20 (+0x00009960) 02c2020c SHL.B64 X1, #12 +0x0000000000009c24 (+0x00009964) 00000081 ADD.S64 X0, X0, X1 +# [DWARF] common/pa_scheduler_core.h:789 +# 783 | static_assert(offsetof(LazySampleCallbackTicket, won) == 14, "lazy sample callback ticket winner offset mismatch"); +# 784 | +# 785 | PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { +# 786 | // This is BeginSubmit without an already-materialized TaskArgs. The same +# 787 | // fields are completed synchronously after the single callback builds args. +# 788 | const uint32_t task_id = static_cast(worker.local_index++); +# > 789 | context.self = &worker; +0x0000000000009c28 (+0x00009968) 09c53031 STP_XI_XJ_XN.B64 X2, X0, X19, #24 +# [DWARF] common/pa_scheduler_core.h:793 +# 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +# 791 | context.task_id = static_cast(task_id); +# 792 | context.tensor_count = 0; +# > 793 | context.scalar_count = 0; +0x0000000000009c2c (+0x0000996c) 07000000 MOV_XD_IMM X0, #0 +0x0000000000009c30 (+0x00009970) 08033030 ADD_IMM.S64 X1, X19, #48 +# [DWARF] common/pa_scheduler_core.h:791 +# 785 | PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { +# 786 | // This is BeginSubmit without an already-materialized TaskArgs. The same +# 787 | // fields are completed synchronously after the single callback builds args. +# 788 | const uint32_t task_id = static_cast(worker.local_index++); +# 789 | context.self = &worker; +# 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +# > 791 | context.task_id = static_cast(task_id); +0x0000000000009c34 (+0x00009974) 03873028 ST_XD_XN_IMM.B32 X3, X19, #40 +# [DWARF] common/pa_scheduler_core.h:792 +# > 792 | context.tensor_count = 0; +0x0000000000009c38 (+0x00009978) 0f813580 STI_XN_IMM.B32 X19, #44 +# [DWARF] common/pa_scheduler_core.h:794 +# 793 | context.scalar_count = 0; +# > 794 | context.result.task_id = task_id; +0x0000000000009c3c (+0x0000997c) 03c73040 ST_XD_XN_IMM.B64 X3, X19, #64 +# [DWARF] common/pa_scheduler_core.h:795 +# > 795 | context.result.count = 0; +0x0000000000009c40 (+0x00009980) 0f813900 STI_XN_IMM.B32 X19, #72 +# [DWARF] common/pa_scheduler_core.h:793 +# 787 | // fields are completed synchronously after the single callback builds args. +# 788 | const uint32_t task_id = static_cast(worker.local_index++); +# 789 | context.self = &worker; +# 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +# 791 | context.task_id = static_cast(task_id); +# 792 | context.tensor_count = 0; +# > 793 | context.scalar_count = 0; +0x0000000000009c44 (+0x00009984) 09c01001 STP_XI_XJ_XN.B64 X0, X0, X1, #0 +# [DWARF] common/pa_scheduler_core.h:798 +# 794 | context.result.task_id = task_id; +# 795 | context.result.count = 0; +# 796 | context.register_mask = 0; +# 797 | context.output_bytes = 0; +# > 798 | context.fanin_count = 0; +0x0000000000009c48 (+0x00009988) 07000000 MOV_XD_IMM X0, #0 +0x0000000000009c4c (+0x0000998c) 0781ffff MOVK X0, #65535, #2 +# [DWARF] common/pa_scheduler_core.h:800 +# 799 | context.kernel_id = -1; +# > 800 | context.won = false; +0x0000000000009c50 (+0x00009990) 0f473300 STI_XN_IMM.B16 X19, #408 +# [DWARF] common/pa_scheduler_core.h:798 +# 792 | context.tensor_count = 0; +# 793 | context.scalar_count = 0; +# 794 | context.result.task_id = task_id; +# 795 | context.result.count = 0; +# 796 | context.register_mask = 0; +# 797 | context.output_bytes = 0; +# > 798 | context.fanin_count = 0; +0x0000000000009c54 (+0x00009994) 07c1ffff MOVK X0, #65535, #3 +# [DWARF] common/pa_scheduler_core.h:802 +# 799 | context.kernel_id = -1; +# 800 | context.won = false; +# 801 | context.joint = false; +# > 802 | context.joint_init = false; +0x0000000000009c58 (+0x00009998) 0f073340 STI_XN_IMM.B8 X19, #410 +# [DWARF] common/pa_scheduler_core.h:798 +# 792 | context.tensor_count = 0; +# 793 | context.scalar_count = 0; +# 794 | context.result.task_id = task_id; +# 795 | context.result.count = 0; +# 796 | context.register_mask = 0; +# 797 | context.output_bytes = 0; +# > 798 | context.fanin_count = 0; +0x0000000000009c5c (+0x0000999c) 03c13190 ST_XD_XN_IMM.B64 X0, X19, #400 +# [DWARF] common/pa_scheduler_core.h:804 +# 799 | context.kernel_id = -1; +# 800 | context.won = false; +# 801 | context.joint = false; +# 802 | context.joint_init = false; +# 803 | context.joint_block = -1; +# > 804 | context.joint_slot = -1; +0x0000000000009c60 (+0x000099a0) 0700ffff MOV_XD_IMM X0, #65535 +0x0000000000009c64 (+0x000099a4) 0741ffff MOVK X0, #65535, #1 +# [DWARF] common/pa_scheduler_core.h:803 +# 797 | context.output_bytes = 0; +# 798 | context.fanin_count = 0; +# 799 | context.kernel_id = -1; +# 800 | context.won = false; +# 801 | context.joint = false; +# 802 | context.joint_init = false; +# > 803 | context.joint_block = -1; +0x0000000000009c68 (+0x000099a8) 0f873382 STI_XN_IMM.B32 X19, #412 +# [DWARF] common/pa_scheduler_core.h:804 +# > 804 | context.joint_slot = -1; +0x0000000000009c6c (+0x000099ac) 03c131a0 ST_XD_XN_IMM.B64 X0, X19, #416 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x0000000000009c70 (+0x000099b0) 02808880 MOV_XD_SPR.F32 X0, SYS_CNT +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x0000000000009c74 (+0x000099b4) 1cd9e9b0 LD_XD_XN_IMM.B64 X12, X30, #2480 +0x0000000000009c78 (+0x000099b8) 1cdbe9a8 LD_XD_XN_IMM.B64 X13, X30, #2472 +0x0000000000009c7c (+0x000099bc) 1cdde9a0 LD_XD_XN_IMM.B64 X14, X30, #2464 +# [DWARF] common/pa_trace.h:257 +# 251 | TraceContext &trace, uint64_t end_cycle +# 252 | ) { +# 253 | #if PA_BUILD_SUBMIT_PMU +# 254 | (void)trace; +# 255 | (void)end_cycle; +# 256 | #else +# > 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +0x0000000000009c80 (+0x000099c0) 03c1e928 ST_XD_XN_IMM.B64 X0, X30, #2344 +0x0000000000009c84 (+0x000099c4) 1c01355c LD_XD_XN_IMM.B8 X0, X19, #1372 +0x0000000000009c88 (+0x000099c8) 0000058e CMP.S64.EQ X0, X11 +0x0000000000009c8c (+0x000099cc) 40200089 JUMPC #137 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009c90 (+0x000099d0) 1ca535d0 LD_XD_XN_IMM.B32 X18, X19, #1488 +0x0000000000009c94 (+0x000099d4) 02812a00 ZEROEXT.U32 X0, X18 +# [DWARF] common/pa_trace.h:257 +# 251 | TraceContext &trace, uint64_t end_cycle +# 252 | ) { +# 253 | #if PA_BUILD_SUBMIT_PMU +# 254 | (void)trace; +# 255 | (void)end_cycle; +# 256 | #else +# > 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +0x0000000000009c98 (+0x000099d8) 0000058e CMP.S64.EQ X0, X11 +0x0000000000009c9c (+0x000099dc) 40200085 JUMPC #133 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009ca0 (+0x000099e0) 1cc9e990 LD_XD_XN_IMM.B64 X4, X30, #2448 +0x0000000000009ca4 (+0x000099e4) 0802e002 ADD_IMM.S64 X1, X14, #2 +0x0000000000009ca8 (+0x000099e8) 00c0d78a AND.B64 X0, X13, X15 +0x0000000000009cac (+0x000099ec) 02c2020e SHL.B64 X1, #14 +0x0000000000009cb0 (+0x000099f0) 07360001 MOV_XD_IMM X27, #1 +0x0000000000009cb4 (+0x000099f4) 0237b080 NEG.S64 X27, X27 +0x0000000000009cb8 (+0x000099f8) 07280000 MOV_XD_IMM X20, #0 +0x0000000000009cbc (+0x000099fc) 072e0000 MOV_XD_IMM X23, #0 +0x0000000000009cc0 (+0x00009a00) 0004c201 ADD.S64 X2, X12, X4 +0x0000000000009cc4 (+0x00009a04) 00040084 MADD.S64 X2, X0, X1 +0x0000000000009cc8 (+0x00009a08) 00002081 ADD.S64 X0, X2, X1 +0x0000000000009ccc (+0x00009a0c) 08800680 SUB_IMM.S64 X0, X0, #1664 +0x0000000000009cd0 (+0x00009a10) 08000000 ADD_IMM.S64 X0, X0, #0 +0x0000000000009cd4 (+0x00009a14) 082a0548 ADD_IMM.S64 X21, X0, #1352 +0x0000000000009cd8 (+0x00009a18) 1cb40558 LD_XD_XN_IMM.B32 X26, X0, #1368 +0x0000000000009cdc (+0x00009a1c) 0ceb5c80 LDP_XI_XJ_XN.B64 X21, X25, X21, #0 +0x0000000000009ce0 (+0x00009a20) 0726beb8 MOV_XD_IMM X19, #48824 +0x0000000000009ce4 (+0x00009a24) 07670007 MOVK X19, #7, #1 +0x0000000000009ce8 (+0x00009a28) 07a70000 MOVK X19, #0, #2 +0x0000000000009cec (+0x00009a2c) 07e70000 MOVK X19, #0, #3 +0x0000000000009cf0 (+0x00009a30) 02040880 MOV_XD_SPR.S64 X2, PC +0x0000000000009cf4 (+0x00009a34) 00273101 ADD.S64 X19, X19, X2 +0x0000000000009cf8 (+0x00009a38) 40000012 JUMP #18 +0x0000000000009cfc (+0x00009a3c) 0804e002 ADD_IMM.S64 X2, X14, #2 +0x0000000000009d00 (+0x00009a40) 00c2d78a AND.B64 X1, X13, X15 +0x0000000000009d04 (+0x00009a44) 02c4020e SHL.B64 X2, #14 +0x0000000000009d08 (+0x00009a48) 0006c201 ADD.S64 X3, X12, X4 +0x0000000000009d0c (+0x00009a4c) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:280 +# 274 | call_count, static_cast(site) +# 275 | ); +# 276 | if (written) { +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# 278 | trace.atomic_counter_overflow = true; +# 279 | } else { +# > 280 | ++trace.poll_batch_records; +0x0000000000009d10 (+0x00009a50) 08000001 ADD_IMM.S64 X0, X0, #1 +0x0000000000009d14 (+0x00009a54) 00023101 ADD.S64 X1, X3, X2 +0x0000000000009d18 (+0x00009a58) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x0000000000009d1c (+0x00009a5c) 08021000 ADD_IMM.S64 X1, X1, #0 +0x0000000000009d20 (+0x00009a60) 03c01578 ST_XD_XN_IMM.B64 X0, X1, #1400 +# [DWARF] common/pa_trace.h:283 +# 281 | } +# 282 | } +# > 283 | trace.poll_burst.call_count[index] = 0; +0x0000000000009d24 (+0x00009a64) 0f976700 STI_XN_IMM.B32 X22, #1464 +# [DWARF] common/pa_trace.h:262 +# 256 | #else +# 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +# 258 | const uint32_t active_mask = trace.poll_burst.active_mask; +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# > 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +0x0000000000009d28 (+0x00009a68) 082f7001 ADD_IMM.S64 X23, X23, #1 +0x0000000000009d2c (+0x00009a6c) 08273004 ADD_IMM.S64 X19, X19, #4 +0x0000000000009d30 (+0x00009a70) 0001741e CMP.S64.NE X23, X8 +0x0000000000009d34 (+0x00009a74) 08294001 ADD_IMM.S64 X20, X20, #1 +0x0000000000009d38 (+0x00009a78) 40200002 JUMPC #2 +0x0000000000009d3c (+0x00009a7c) 4000004e JUMP #78 +# [DWARF] common/pa_trace.h:263 +# > 263 | const uint32_t bit = 1U << index; +0x0000000000009d40 (+0x00009a80) 02814a00 ZEROEXT.U32 X0, X20 +# [DWARF] common/pa_trace.h:264 +# > 264 | if ((active_mask & bit) == 0) continue; +0x0000000000009d44 (+0x00009a84) 02832a00 ZEROEXT.U32 X1, X18 +0x0000000000009d48 (+0x00009a88) 024202c0 SHR.U64 X1, X0, #0 +0x0000000000009d4c (+0x00009a8c) 00c0188a AND.B64 X0, X1, X17 +0x0000000000009d50 (+0x00009a90) 0000058e CMP.S64.EQ X0, X11 +0x0000000000009d54 (+0x00009a94) 4020fff5 JUMPC #65525 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009d58 (+0x00009a98) 0804e002 ADD_IMM.S64 X2, X14, #2 +0x0000000000009d5c (+0x00009a9c) 00c0d78a AND.B64 X0, X13, X15 +0x0000000000009d60 (+0x00009aa0) 02c4020e SHL.B64 X2, #14 +0x0000000000009d64 (+0x00009aa4) 0006c201 ADD.S64 X3, X12, X4 +0x0000000000009d68 (+0x00009aa8) 00060104 MADD.S64 X3, X0, X2 +# [DWARF] common/pa_trace.h:265 +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +# 263 | const uint32_t bit = 1U << index; +# 264 | if ((active_mask & bit) == 0) continue; +# > 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +0x0000000000009d6c (+0x00009aac) 02037800 MOV_XD_XN.S64 X1, X23 +0x0000000000009d70 (+0x00009ab0) 00003101 ADD.S64 X0, X3, X2 +0x0000000000009d74 (+0x00009ab4) 08800680 SUB_IMM.S64 X0, X0, #1664 +0x0000000000009d78 (+0x00009ab8) 02c20202 SHL.B64 X1, #2 +0x0000000000009d7c (+0x00009abc) 08000000 ADD_IMM.S64 X0, X0, #0 +0x0000000000009d80 (+0x00009ac0) 002c0081 ADD.S64 X22, X0, X1 +0x0000000000009d84 (+0x00009ac4) 1c8b65b8 LD_XD_XN_IMM.B32 X5, X22, #1464 +# [DWARF] common/pa_trace.h:266 +# > 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +0x0000000000009d88 (+0x00009ac8) 07020000 MOV_XD_IMM X1, #0 +0x0000000000009d8c (+0x00009acc) 07430100 MOVK X1, #256, #1 +0x0000000000009d90 (+0x00009ad0) 07040000 MOV_XD_IMM X2, #0 +0x0000000000009d94 (+0x00009ad4) 0745ff00 MOVK X2, #65280, #1 +0x0000000000009d98 (+0x00009ad8) 00025082 SUB.S64 X1, X5, X1 +0x0000000000009d9c (+0x00009adc) 02821a00 ZEROEXT.U32 X1, X1 +0x0000000000009da0 (+0x00009ae0) 0040113e CMP.U64.GT X1, X2 +0x0000000000009da4 (+0x00009ae4) 40200003 JUMPC #3 +# [DWARF] common/pa_trace.h:267 +# > 267 | trace.atomic_counter_overflow = true; +0x0000000000009da8 (+0x00009ae8) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x0000000000009dac (+0x00009aec) 4000ffdf JUMP #65503 +# [DWARF] common/pa_trace.h:98 +# 92 | default: +# 93 | return -1; +# 94 | } +# 95 | } +# 96 | +# 97 | PA_DEVICE AtomicSite TraceAtomicPollBatchSite(uint32_t index) { +# > 98 | switch (index) { +0x0000000000009db0 (+0x00009af0) 02814a00 ZEROEXT.U32 X0, X20 +0x0000000000009db4 (+0x00009af4) 07020005 MOV_XD_IMM X1, #5 +0x0000000000009db8 (+0x00009af8) 004000be CMP.U64.GT X0, X1 +0x0000000000009dbc (+0x00009afc) 070c000f MOV_XD_IMM X6, #15 +0x0000000000009dc0 (+0x00009b00) 40200002 JUMPC #2 +0x0000000000009dc4 (+0x00009b04) 1c8d3000 LD_XD_XN_IMM.B32 X6, X19, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009dc8 (+0x00009b08) 0804e002 ADD_IMM.S64 X2, X14, #2 +0x0000000000009dcc (+0x00009b0c) 00c2d78a AND.B64 X1, X13, X15 +0x0000000000009dd0 (+0x00009b10) 02c4020e SHL.B64 X2, #14 +0x0000000000009dd4 (+0x00009b14) 0006c201 ADD.S64 X3, X12, X4 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x0000000000009dd8 (+0x00009b18) 1cc9e928 LD_XD_XN_IMM.B64 X4, X30, #2344 +0x0000000000009ddc (+0x00009b1c) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:273 +# 272 | trace.core, trace.records, trace.capacity, +# > 273 | trace.poll_burst.start_cycle[index], end_cycle, +0x0000000000009de0 (+0x00009b20) 02017800 MOV_XD_XN.S64 X0, X23 +0x0000000000009de4 (+0x00009b24) 00023101 ADD.S64 X1, X3, X2 +0x0000000000009de8 (+0x00009b28) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x0000000000009dec (+0x00009b2c) 02c00203 SHL.B64 X0, #3 +0x0000000000009df0 (+0x00009b30) 08301000 ADD_IMM.S64 X24, X1, #0 +0x0000000000009df4 (+0x00009b34) 00018001 ADD.S64 X0, X24, X0 +0x0000000000009df8 (+0x00009b38) 1cc60588 LD_XD_XN_IMM.B64 X3, X0, #1416 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x0000000000009dfc (+0x00009b3c) 02015800 MOV_XD_XN.S64 X0, X21 +0x0000000000009e00 (+0x00009b40) 02039800 MOV_XD_XN.S64 X1, X25 +0x0000000000009e04 (+0x00009b44) 0205a800 MOV_XD_XN.S64 X2, X26 +0x0000000000009e08 (+0x00009b48) 070e772b MOV_XD_IMM X7, #30507 +0x0000000000009e0c (+0x00009b4c) 074f0000 MOVK X7, #0, #1 +0x0000000000009e10 (+0x00009b50) 078f0000 MOVK X7, #0, #2 +0x0000000000009e14 (+0x00009b54) 40427000 CALL X7, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x0000000000009e18 (+0x00009b58) 1cc9e990 LD_XD_XN_IMM.B64 X4, X30, #2448 +0x0000000000009e1c (+0x00009b5c) 07160000 MOV_XD_IMM X11, #0 diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/key_flow/aic_lazy_input_policy.source.asm b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/key_flow/aic_lazy_input_policy.source.asm new file mode 100644 index 0000000000..a5b50aff22 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/key_flow/aic_lazy_input_policy.source.asm @@ -0,0 +1,2377 @@ +# schema=pa_final_linked_disassembly/v1 +# variant=compete-first-lazy +# final_elf=pa_scheduler_kernel.o +# final_elf_sha256=8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893 +# final_text_address=0x0 +# final_text_size=547640 +# final_text_sha256=866ed1081ebb94038a8feca87ccedf95e67aebfdaef11651c109f86672be2bdd +# symbol=pa_scheduler_lazy_sample_callback_orchestration_aic +# binding=LOCAL +# final_pc=0x2c0 +# size=161792 +# instruction_count=40448 +# encoded_word_count=40448 +# body_sha256=35bcc11cdad126fa5d2f92ff1df421d8556843a412ea3497addfe820d84bd852 +# decoder=$ASCEND_HOME_PATH/x86_64-linux/simulator/dav_3510/lib/libpem_davinci.so +# decoder_sha256=29835d2439d6dd464d34a212ad4bbd5c29af6a38465da09a6c273401d9a96dcb +# decoder_mode=scalar +# columns=final_pc function_relative_offset machine_word instruction +# annotation_schema=pa_source_annotated_disassembly/v1 +# annotation_rule=DWARF supplies only file:line; SOURCE rows are copied from local source files +# annotation_warning=comments have source context only and do not own an exact machine address +# annotation_instruction_slice=15032:15833 +# +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000eda0 (+0x0000eae0) 07690007 MOVK X20, #7, #1 +0x000000000000eda4 (+0x0000eae4) 07a90000 MOVK X20, #0, #2 +0x000000000000eda8 (+0x0000eae8) 07e90000 MOVK X20, #0, #3 +0x000000000000edac (+0x0000eaec) 02020880 MOV_XD_SPR.S64 X1, PC +0x000000000000edb0 (+0x0000eaf0) 00294081 ADD.S64 X20, X20, X1 +0x000000000000edb4 (+0x0000eaf4) 40000012 JUMP #18 +0x000000000000edb8 (+0x0000eaf8) 0804e002 ADD_IMM.S64 X2, X14, #2 +0x000000000000edbc (+0x0000eafc) 00c2d78a AND.B64 X1, X13, X15 +0x000000000000edc0 (+0x0000eb00) 02c4020e SHL.B64 X2, #14 +0x000000000000edc4 (+0x0000eb04) 0006c381 ADD.S64 X3, X12, X7 +0x000000000000edc8 (+0x0000eb08) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:280 +# 274 | call_count, static_cast(site) +# 275 | ); +# 276 | if (written) { +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# 278 | trace.atomic_counter_overflow = true; +# 279 | } else { +# > 280 | ++trace.poll_batch_records; +0x000000000000edcc (+0x0000eb0c) 08000001 ADD_IMM.S64 X0, X0, #1 +0x000000000000edd0 (+0x0000eb10) 00023101 ADD.S64 X1, X3, X2 +0x000000000000edd4 (+0x0000eb14) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x000000000000edd8 (+0x0000eb18) 08021000 ADD_IMM.S64 X1, X1, #0 +0x000000000000eddc (+0x0000eb1c) 03c01578 ST_XD_XN_IMM.B64 X0, X1, #1400 +# [DWARF] common/pa_trace.h:283 +# 281 | } +# 282 | } +# > 283 | trace.poll_burst.call_count[index] = 0; +0x000000000000ede0 (+0x0000eb20) 0f976700 STI_XN_IMM.B32 X22, #1464 +# [DWARF] common/pa_trace.h:262 +# 256 | #else +# 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +# 258 | const uint32_t active_mask = trace.poll_burst.active_mask; +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# > 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +0x000000000000ede4 (+0x0000eb24) 08318001 ADD_IMM.S64 X24, X24, #1 +0x000000000000ede8 (+0x0000eb28) 08294004 ADD_IMM.S64 X20, X20, #4 +0x000000000000edec (+0x0000eb2c) 0001841e CMP.S64.NE X24, X8 +0x000000000000edf0 (+0x0000eb30) 08273001 ADD_IMM.S64 X19, X19, #1 +0x000000000000edf4 (+0x0000eb34) 40200002 JUMPC #2 +0x000000000000edf8 (+0x0000eb38) 40000050 JUMP #80 +# [DWARF] common/pa_trace.h:263 +# > 263 | const uint32_t bit = 1U << index; +0x000000000000edfc (+0x0000eb3c) 02813a00 ZEROEXT.U32 X0, X19 +# [DWARF] common/pa_trace.h:264 +# > 264 | if ((active_mask & bit) == 0) continue; +0x000000000000ee00 (+0x0000eb40) 02837a00 ZEROEXT.U32 X1, X23 +0x000000000000ee04 (+0x0000eb44) 024202c0 SHR.U64 X1, X0, #0 +0x000000000000ee08 (+0x0000eb48) 00c0188a AND.B64 X0, X1, X17 +0x000000000000ee0c (+0x0000eb4c) 0000058e CMP.S64.EQ X0, X11 +0x000000000000ee10 (+0x0000eb50) 4020fff5 JUMPC #65525 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000ee14 (+0x0000eb54) 0804e002 ADD_IMM.S64 X2, X14, #2 +0x000000000000ee18 (+0x0000eb58) 00c0d78a AND.B64 X0, X13, X15 +0x000000000000ee1c (+0x0000eb5c) 02c4020e SHL.B64 X2, #14 +0x000000000000ee20 (+0x0000eb60) 0006c381 ADD.S64 X3, X12, X7 +0x000000000000ee24 (+0x0000eb64) 00060104 MADD.S64 X3, X0, X2 +# [DWARF] common/pa_trace.h:265 +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +# 263 | const uint32_t bit = 1U << index; +# 264 | if ((active_mask & bit) == 0) continue; +# > 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +0x000000000000ee28 (+0x0000eb68) 02038800 MOV_XD_XN.S64 X1, X24 +0x000000000000ee2c (+0x0000eb6c) 00003101 ADD.S64 X0, X3, X2 +0x000000000000ee30 (+0x0000eb70) 08800680 SUB_IMM.S64 X0, X0, #1664 +0x000000000000ee34 (+0x0000eb74) 02c20202 SHL.B64 X1, #2 +0x000000000000ee38 (+0x0000eb78) 08000000 ADD_IMM.S64 X0, X0, #0 +0x000000000000ee3c (+0x0000eb7c) 002c0081 ADD.S64 X22, X0, X1 +0x000000000000ee40 (+0x0000eb80) 1c8b65b8 LD_XD_XN_IMM.B32 X5, X22, #1464 +# [DWARF] common/pa_trace.h:266 +# > 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +0x000000000000ee44 (+0x0000eb84) 07020000 MOV_XD_IMM X1, #0 +0x000000000000ee48 (+0x0000eb88) 07430100 MOVK X1, #256, #1 +0x000000000000ee4c (+0x0000eb8c) 07040000 MOV_XD_IMM X2, #0 +0x000000000000ee50 (+0x0000eb90) 0745ff00 MOVK X2, #65280, #1 +0x000000000000ee54 (+0x0000eb94) 00025082 SUB.S64 X1, X5, X1 +0x000000000000ee58 (+0x0000eb98) 02821a00 ZEROEXT.U32 X1, X1 +0x000000000000ee5c (+0x0000eb9c) 0040113e CMP.U64.GT X1, X2 +0x000000000000ee60 (+0x0000eba0) 40200003 JUMPC #3 +# [DWARF] common/pa_trace.h:267 +# > 267 | trace.atomic_counter_overflow = true; +0x000000000000ee64 (+0x0000eba4) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x000000000000ee68 (+0x0000eba8) 4000ffdf JUMP #65503 +# [DWARF] common/pa_trace.h:98 +# 92 | default: +# 93 | return -1; +# 94 | } +# 95 | } +# 96 | +# 97 | PA_DEVICE AtomicSite TraceAtomicPollBatchSite(uint32_t index) { +# > 98 | switch (index) { +0x000000000000ee6c (+0x0000ebac) 02813a00 ZEROEXT.U32 X0, X19 +0x000000000000ee70 (+0x0000ebb0) 07020005 MOV_XD_IMM X1, #5 +0x000000000000ee74 (+0x0000ebb4) 004000be CMP.U64.GT X0, X1 +0x000000000000ee78 (+0x0000ebb8) 070c000f MOV_XD_IMM X6, #15 +0x000000000000ee7c (+0x0000ebbc) 40200002 JUMPC #2 +0x000000000000ee80 (+0x0000ebc0) 1c8d4000 LD_XD_XN_IMM.B32 X6, X20, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000ee84 (+0x0000ebc4) 0804e002 ADD_IMM.S64 X2, X14, #2 +0x000000000000ee88 (+0x0000ebc8) 00c2d78a AND.B64 X1, X13, X15 +0x000000000000ee8c (+0x0000ebcc) 02c4020e SHL.B64 X2, #14 +0x000000000000ee90 (+0x0000ebd0) 0006c381 ADD.S64 X3, X12, X7 +0x000000000000ee94 (+0x0000ebd4) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:273 +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# > 273 | trace.poll_burst.start_cycle[index], end_cycle, +0x000000000000ee98 (+0x0000ebd8) 02018800 MOV_XD_XN.S64 X0, X24 +0x000000000000ee9c (+0x0000ebdc) 00023101 ADD.S64 X1, X3, X2 +0x000000000000eea0 (+0x0000ebe0) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x000000000000eea4 (+0x0000ebe4) 02c00203 SHL.B64 X0, #3 +0x000000000000eea8 (+0x0000ebe8) 08241000 ADD_IMM.S64 X18, X1, #0 +0x000000000000eeac (+0x0000ebec) 00012001 ADD.S64 X0, X18, X0 +0x000000000000eeb0 (+0x0000ebf0) 1cc60588 LD_XD_XN_IMM.B64 X3, X0, #1416 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x000000000000eeb4 (+0x0000ebf4) 0201a800 MOV_XD_XN.S64 X0, X26 +0x000000000000eeb8 (+0x0000ebf8) 0203b800 MOV_XD_XN.S64 X1, X27 +0x000000000000eebc (+0x0000ebfc) 0205c800 MOV_XD_XN.S64 X2, X28 +0x000000000000eec0 (+0x0000ec00) 02095800 MOV_XD_XN.S64 X4, X21 +0x000000000000eec4 (+0x0000ec04) 070e62fc MOV_XD_IMM X7, #25340 +0x000000000000eec8 (+0x0000ec08) 074f0000 MOVK X7, #0, #1 +0x000000000000eecc (+0x0000ec0c) 078f0000 MOVK X7, #0, #2 +0x000000000000eed0 (+0x0000ec10) 40427000 CALL X7, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000eed4 (+0x0000ec14) 1ccfe990 LD_XD_XN_IMM.B64 X7, X30, #2448 +0x000000000000eed8 (+0x0000ec18) 07160000 MOV_XD_IMM X11, #0 +0x000000000000eedc (+0x0000ec1c) 1cdde9a0 LD_XD_XN_IMM.B64 X14, X30, #2464 +0x000000000000eee0 (+0x0000ec20) 071e7fff MOV_XD_IMM X15, #32767 +0x000000000000eee4 (+0x0000ec24) 1cdbe9a8 LD_XD_XN_IMM.B64 X13, X30, #2472 +# [DWARF] common/pa_trace.h:276 +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x000000000000eee8 (+0x0000ec28) 0000058e CMP.S64.EQ X0, X11 +0x000000000000eeec (+0x0000ec2c) 1cd9e9b0 LD_XD_XN_IMM.B64 X12, X30, #2480 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000eef0 (+0x0000ec30) 07220001 MOV_XD_IMM X17, #1 +0x000000000000eef4 (+0x0000ec34) 07100006 MOV_XD_IMM X8, #6 +# [DWARF] common/pa_trace.h:276 +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x000000000000eef8 (+0x0000ec38) 4020ffba JUMPC #65466 +# [DWARF] common/pa_trace.h:277 +# > 277 | if (trace.poll_batch_records == UINT64_MAX) { +0x000000000000eefc (+0x0000ec3c) 1cc12578 LD_XD_XN_IMM.B64 X0, X18, #1400 +0x000000000000ef00 (+0x0000ec40) 07020001 MOV_XD_IMM X1, #1 +0x000000000000ef04 (+0x0000ec44) 02021080 NEG.S64 X1, X1 +0x000000000000ef08 (+0x0000ec48) 0000009e CMP.S64.NE X0, X1 +0x000000000000ef0c (+0x0000ec4c) 4020ffab JUMPC #65451 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000ef10 (+0x0000ec50) 0802e002 ADD_IMM.S64 X1, X14, #2 +0x000000000000ef14 (+0x0000ec54) 00c0d78a AND.B64 X0, X13, X15 +0x000000000000ef18 (+0x0000ec58) 02c2020e SHL.B64 X1, #14 +0x000000000000ef1c (+0x0000ec5c) 0004c381 ADD.S64 X2, X12, X7 +0x000000000000ef20 (+0x0000ec60) 00040084 MADD.S64 X2, X0, X1 +0x000000000000ef24 (+0x0000ec64) 00002081 ADD.S64 X0, X2, X1 +0x000000000000ef28 (+0x0000ec68) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:278 +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# 276 | if (written) { +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# > 278 | trace.atomic_counter_overflow = true; +0x000000000000ef2c (+0x0000ec6c) 08000000 ADD_IMM.S64 X0, X0, #0 +0x000000000000ef30 (+0x0000ec70) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x000000000000ef34 (+0x0000ec74) 4000ffab JUMP #65451 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000ef38 (+0x0000ec78) 1ccbe968 LD_XD_XN_IMM.B64 X5, X30, #2408 +0x000000000000ef3c (+0x0000ec7c) 0802e002 ADD_IMM.S64 X1, X14, #2 +0x000000000000ef40 (+0x0000ec80) 00c0d78a AND.B64 X0, X13, X15 +0x000000000000ef44 (+0x0000ec84) 02c2020e SHL.B64 X1, #14 +0x000000000000ef48 (+0x0000ec88) 0004c381 ADD.S64 X2, X12, X7 +0x000000000000ef4c (+0x0000ec8c) 00040084 MADD.S64 X2, X0, X1 +0x000000000000ef50 (+0x0000ec90) 07280001 MOV_XD_IMM X20, #1 +0x000000000000ef54 (+0x0000ec94) 00002081 ADD.S64 X0, X2, X1 +0x000000000000ef58 (+0x0000ec98) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:285 +# 279 | } else { +# 280 | ++trace.poll_batch_records; +# 281 | } +# 282 | } +# 283 | trace.poll_burst.call_count[index] = 0; +# 284 | } +# > 285 | trace.poll_burst.active_mask = 0; +0x000000000000ef5c (+0x0000ec9c) 08000000 ADD_IMM.S64 X0, X0, #0 +0x000000000000ef60 (+0x0000eca0) 02294080 NEG.S64 X20, X20 +0x000000000000ef64 (+0x0000eca4) 0f960a00 STI_XN_IMM.B32 X0, #1488 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000ef68 (+0x0000eca8) 1ce3e920 LD_XD_XN_IMM.B64 X17, X30, #2336 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x000000000000ef6c (+0x0000ecac) 0001a58e CMP.S64.EQ X26, X11 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000ef70 (+0x0000ecb0) 1ce5e988 LD_XD_XN_IMM.B64 X18, X30, #2440 +0x000000000000ef74 (+0x0000ecb4) 1cefe978 LD_XD_XN_IMM.B64 X23, X30, #2424 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x000000000000ef78 (+0x0000ecb8) 40200027 JUMPC #39 +0x000000000000ef7c (+0x0000ecbc) 0001b58e CMP.S64.EQ X27, X11 +0x000000000000ef80 (+0x0000ecc0) 40200025 JUMPC #37 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000ef84 (+0x0000ecc4) 0283ca00 ZEROEXT.U32 X1, X28 +# [DWARF] common/pa_trace.h:543 +# 537 | (void)auxiliary; +# 538 | return; +# 539 | #else +# 540 | // 每段先更新轻量 phase 统计，再按需写 64-byte 原始记录。一个分区只有对应 +# 541 | // worker 写入，因此 count/dropped 保持普通单写者更新，不额外引入 atomic。 +# 542 | AccumulatePhase(result, profile_phase, start_cycle, end_cycle); +# > 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +0x000000000000ef88 (+0x0000ecc8) 0000158e CMP.S64.EQ X1, X11 +0x000000000000ef8c (+0x0000eccc) 40200022 JUMPC #34 +# [DWARF] common/pa_trace.h:547 +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# > 547 | const uint32_t slot = core.count; +0x000000000000ef90 (+0x0000ecd0) 1c81a000 LD_XD_XN_IMM.B32 X0, X26, #0 +# [DWARF] common/pa_trace.h:548 +# > 548 | if (slot >= trace.capacity) { +0x000000000000ef94 (+0x0000ecd4) 004000ae CMP.U64.LT X0, X1 +0x000000000000ef98 (+0x0000ecd8) 40200002 JUMPC #2 +0x000000000000ef9c (+0x0000ecdc) 4000001b JUMP #27 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000efa0 (+0x0000ece0) 0806e002 ADD_IMM.S64 X3, X14, #2 +0x000000000000efa4 (+0x0000ece4) 00c4d78a AND.B64 X2, X13, X15 +0x000000000000efa8 (+0x0000ece8) 02c6020e SHL.B64 X3, #14 +0x000000000000efac (+0x0000ecec) 0008c381 ADD.S64 X4, X12, X7 +# [DWARF] common/pa_trace.h:552 +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x000000000000efb0 (+0x0000ecf0) 02020800 MOV_XD_XN.S64 X1, X0 +0x000000000000efb4 (+0x0000ecf4) 00082184 MADD.S64 X4, X2, X3 +0x000000000000efb8 (+0x0000ecf8) 02c20206 SHL.B64 X1, #6 +0x000000000000efbc (+0x0000ecfc) 00044181 ADD.S64 X2, X4, X3 +0x000000000000efc0 (+0x0000ed00) 0003b081 ADD.S64 X1, X27, X1 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000efc4 (+0x0000ed04) 08842680 SUB_IMM.S64 X2, X2, #1664 +# [DWARF] common/pa_trace.h:553 +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# > 553 | record.start_cycle = start_cycle; +0x000000000000efc8 (+0x0000ed08) 09f21a81 STP_XI_XJ_XN.B64 X25, X21, X1, #0 +# [DWARF] common/pa_trace.h:555 +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x000000000000efcc (+0x0000ed0c) 09a21ba1 STP_XI_XJ_XN.B32 X17, X23, X1, #16 +# [DWARF] common/pa_trace.h:558 +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# > 558 | record.lane = trace.lane; +0x000000000000efd0 (+0x0000ed10) 08042000 ADD_IMM.S64 X2, X2, #0 +0x000000000000efd4 (+0x0000ed14) 1c862560 LD_XD_XN_IMM.B32 X3, X2, #1376 +# [DWARF] common/pa_trace.h:557 +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x000000000000efd8 (+0x0000ed18) 0708000b MOV_XD_IMM X4, #11 +# [DWARF] common/pa_trace.h:565 +# 558 | record.lane = trace.lane; +# 559 | record.block_id = trace.block_id; +# 560 | record.core_idx = trace.core_idx; +# 561 | record.flags = flags; +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x000000000000efdc (+0x0000ed1c) 08000001 ADD_IMM.S64 X0, X0, #1 +# [DWARF] common/pa_trace.h:557 +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x000000000000efe0 (+0x0000ed20) 098811b1 STP_XI_XJ_XN.B32 X4, X3, X1, #24 +# [DWARF] common/pa_trace.h:559 +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x000000000000efe4 (+0x0000ed24) 08062564 ADD_IMM.S64 X3, X2, #1380 +0x000000000000efe8 (+0x0000ed28) 0c863100 LDP_XI_XJ_XN.B32 X3, X2, X3, #0 +0x000000000000efec (+0x0000ed2c) 08081020 ADD_IMM.S64 X4, X1, #32 +# [DWARF] common/pa_trace.h:561 +# 560 | record.core_idx = trace.core_idx; +# > 561 | record.flags = flags; +0x000000000000eff0 (+0x0000ed30) 08021028 ADD_IMM.S64 X1, X1, #40 +# [DWARF] common/pa_trace.h:559 +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x000000000000eff4 (+0x0000ed34) 09864101 STP_XI_XJ_XN.B32 X3, X2, X4, #0 +0x000000000000eff8 (+0x0000ed38) 00c6590b OR.B64 X3, X5, X18 +# [DWARF] common/pa_trace.h:561 +# 560 | record.core_idx = trace.core_idx; +# > 561 | record.flags = flags; +0x000000000000effc (+0x0000ed3c) 09861581 STP_XI_XJ_XN.B32 X3, X11, X1, #0 +# [DWARF] common/pa_trace.h:565 +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x000000000000f000 (+0x0000ed40) 0381a000 ST_XD_XN_IMM.B32 X0, X26, #0 +0x000000000000f004 (+0x0000ed44) 40000004 JUMP #4 +# [DWARF] common/pa_trace.h:549 +# 543 | if (trace.core == nullptr || trace.records == nullptr || trace.capacity == 0) { +# 544 | return; +# 545 | } +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# > 549 | core.dropped = core.dropped + 1; +0x000000000000f008 (+0x0000ed48) 1c81a004 LD_XD_XN_IMM.B32 X0, X26, #4 +0x000000000000f00c (+0x0000ed4c) 08000001 ADD_IMM.S64 X0, X0, #1 +0x000000000000f010 (+0x0000ed50) 0381a004 ST_XD_XN_IMM.B32 X0, X26, #4 +# [DWARF] common/pa_frontend.h:278 +# 272 | args.explicit_dep_count = 0; +# 273 | } +# 274 | +# 275 | PA_DEVICE void ResetTaskArgs(TaskArgs &args) { +# 276 | // reset 的输出是不含 tensor/scalar/显式依赖的新逻辑参数表，但保留已分配对象及 +# 277 | // launch_spec；QK/SF/PV/UP 在同一个 1 KiB TaskArgs 上依次复用这一状态。 +# > 278 | args.tensor_count = 0; +0x000000000000f014 (+0x0000ed54) 0ff3e700 STI_XN_IMM.B64 X30, #3256 +# [DWARF] common/pa_frontend.h:0 +# [SOURCE unavailable] +0x000000000000f018 (+0x0000ed58) 07000000 MOV_XD_IMM X0, #0 +# [DWARF] common/pa_frontend.h:243 +# 237 | +# 238 | PA_DEVICE void ClearDumpArgSelection(PaDumpArgSelection &selection) { +# 239 | // Volatile stores intentionally preserve the profiling-enabled PA reset +# 240 | // traffic even though the standalone winner workload never consumes dump data. +# 241 | // volatile 的目的不是同步，而是阻止编译器删掉这段生产基线中存在的写流量。 +# 242 | volatile uint64_t *masks = &selection.dump_arg_mask; +# > 243 | masks[0] = 0; +0x000000000000f01c (+0x0000ed5c) 0ff3eb00 STI_XN_IMM.B64 X30, #3288 +# [DWARF] common/pa_frontend.h:703 +# 697 | PA_DEVICE void RecordView() { ++counts_.views_created; } +# 698 | PA_DEVICE void RecordDynamicCreateInfo() { ++counts_.dynamic_create_infos; } +# 699 | +# 700 | template +# 701 | PA_DEVICE void AddLocalInput(Thunk thunk) { +# 702 | if constexpr (Lazy) { +# > 703 | if (!won_) return; +0x000000000000f020 (+0x0000ed60) 0001200e CMP.S64.EQ X18, X0 +# [DWARF] common/pa_frontend.h:244 +# 238 | PA_DEVICE void ClearDumpArgSelection(PaDumpArgSelection &selection) { +# 239 | // Volatile stores intentionally preserve the profiling-enabled PA reset +# 240 | // traffic even though the standalone winner workload never consumes dump data. +# 241 | // volatile 的目的不是同步，而是阻止编译器删掉这段生产基线中存在的写流量。 +# 242 | volatile uint64_t *masks = &selection.dump_arg_mask; +# 243 | masks[0] = 0; +# > 244 | masks[1] = 0; +0x000000000000f024 (+0x0000ed64) 0ff3ec00 STI_XN_IMM.B64 X30, #3296 +# [DWARF] common/pa_frontend.h:0 +# [SOURCE unavailable] +0x000000000000f028 (+0x0000ed68) 07060000 MOV_XD_IMM X3, #0 +# [DWARF] common/pa_frontend.h:247 +# 241 | // volatile 的目的不是同步，而是阻止编译器删掉这段生产基线中存在的写流量。 +# 242 | volatile uint64_t *masks = &selection.dump_arg_mask; +# 243 | masks[0] = 0; +# 244 | masks[1] = 0; +# 245 | volatile uint64_t *sources = &selection.scalar_source_ptrs[0]; +# 246 | for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { +# > 247 | sources[index] = 0; +0x000000000000f02c (+0x0000ed6c) 0ff3ed00 STI_XN_IMM.B64 X30, #3304 +# [DWARF] common/pa_frontend.h:0 +# [SOURCE unavailable] +0x000000000000f030 (+0x0000ed70) 07020000 MOV_XD_IMM X1, #0 +# [DWARF] common/pa_frontend.h:247 +# 241 | // volatile 的目的不是同步，而是阻止编译器删掉这段生产基线中存在的写流量。 +# 242 | volatile uint64_t *masks = &selection.dump_arg_mask; +# 243 | masks[0] = 0; +# 244 | masks[1] = 0; +# 245 | volatile uint64_t *sources = &selection.scalar_source_ptrs[0]; +# 246 | for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { +# > 247 | sources[index] = 0; +0x000000000000f034 (+0x0000ed74) 0ff3ee00 STI_XN_IMM.B64 X30, #3312 +0x000000000000f038 (+0x0000ed78) 0ff3ef00 STI_XN_IMM.B64 X30, #3320 +0x000000000000f03c (+0x0000ed7c) 0ff5e000 STI_XN_IMM.B64 X30, #3328 +0x000000000000f040 (+0x0000ed80) 0ff5e100 STI_XN_IMM.B64 X30, #3336 +0x000000000000f044 (+0x0000ed84) 0ff5e200 STI_XN_IMM.B64 X30, #3344 +0x000000000000f048 (+0x0000ed88) 0ff5e300 STI_XN_IMM.B64 X30, #3352 +0x000000000000f04c (+0x0000ed8c) 0ff5e400 STI_XN_IMM.B64 X30, #3360 +0x000000000000f050 (+0x0000ed90) 0ff5e500 STI_XN_IMM.B64 X30, #3368 +0x000000000000f054 (+0x0000ed94) 0ff5e600 STI_XN_IMM.B64 X30, #3376 +0x000000000000f058 (+0x0000ed98) 0ff5e700 STI_XN_IMM.B64 X30, #3384 +0x000000000000f05c (+0x0000ed9c) 0ff5e800 STI_XN_IMM.B64 X30, #3392 +0x000000000000f060 (+0x0000eda0) 0ff5e900 STI_XN_IMM.B64 X30, #3400 +0x000000000000f064 (+0x0000eda4) 0ff5ea00 STI_XN_IMM.B64 X30, #3408 +0x000000000000f068 (+0x0000eda8) 0ff5eb00 STI_XN_IMM.B64 X30, #3416 +0x000000000000f06c (+0x0000edac) 0ff5ec00 STI_XN_IMM.B64 X30, #3424 +# [DWARF] common/pa_frontend.h:251 +# 248 | } +# 249 | volatile uint8_t *dtypes = &selection.scalar_dtypes[0]; +# 250 | for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { +# > 251 | dtypes[index] = 0; +0x000000000000f070 (+0x0000edb0) 0f35ed00 STI_XN_IMM.B8 X30, #3432 +0x000000000000f074 (+0x0000edb4) 0f35ed20 STI_XN_IMM.B8 X30, #3433 +0x000000000000f078 (+0x0000edb8) 0f35ed40 STI_XN_IMM.B8 X30, #3434 +0x000000000000f07c (+0x0000edbc) 0f35ed60 STI_XN_IMM.B8 X30, #3435 +0x000000000000f080 (+0x0000edc0) 0f35ed80 STI_XN_IMM.B8 X30, #3436 +0x000000000000f084 (+0x0000edc4) 0f35eda0 STI_XN_IMM.B8 X30, #3437 +0x000000000000f088 (+0x0000edc8) 0f35edc0 STI_XN_IMM.B8 X30, #3438 +0x000000000000f08c (+0x0000edcc) 0f35ede0 STI_XN_IMM.B8 X30, #3439 +0x000000000000f090 (+0x0000edd0) 0f35ee00 STI_XN_IMM.B8 X30, #3440 +0x000000000000f094 (+0x0000edd4) 0f35ee20 STI_XN_IMM.B8 X30, #3441 +0x000000000000f098 (+0x0000edd8) 0f35ee40 STI_XN_IMM.B8 X30, #3442 +0x000000000000f09c (+0x0000eddc) 0f35ee60 STI_XN_IMM.B8 X30, #3443 +0x000000000000f0a0 (+0x0000ede0) 0f35ee80 STI_XN_IMM.B8 X30, #3444 +0x000000000000f0a4 (+0x0000ede4) 0f35eea0 STI_XN_IMM.B8 X30, #3445 +0x000000000000f0a8 (+0x0000ede8) 0f35eec0 STI_XN_IMM.B8 X30, #3446 +0x000000000000f0ac (+0x0000edec) 0f35eee0 STI_XN_IMM.B8 X30, #3447 +# [DWARF] common/pa_frontend.h:0 +# [SOURCE unavailable] +0x000000000000f0b0 (+0x0000edf0) 1ce1e8f8 LD_XD_XN_IMM.B64 X16, X30, #2296 +# [DWARF] common/pa_frontend.h:281 +# 275 | PA_DEVICE void ResetTaskArgs(TaskArgs &args) { +# 276 | // reset 的输出是不含 tensor/scalar/显式依赖的新逻辑参数表，但保留已分配对象及 +# 277 | // launch_spec；QK/SF/PV/UP 在同一个 1 KiB TaskArgs 上依次复用这一状态。 +# 278 | args.tensor_count = 0; +# 279 | args.scalar_count = 0; +# 280 | ClearDumpArgSelection(args.dump_arg_selection); +# > 281 | args.explicit_deps = 0; +0x000000000000f0b4 (+0x0000edf4) 0ff5ef00 STI_XN_IMM.B64 X30, #3448 +# [DWARF] common/pa_frontend.h:282 +# > 282 | args.explicit_dep_count = 0; +0x000000000000f0b8 (+0x0000edf8) 0fb7e000 STI_XN_IMM.B32 X30, #3456 +# [DWARF] common/pa_frontend.h:283 +# > 283 | args.has_error = false; +0x000000000000f0bc (+0x0000edfc) 0f33e800 STI_XN_IMM.B8 X30, #3264 +# [DWARF] common/pa_frontend.h:284 +# > 284 | args.error_msg = 0; +0x000000000000f0c0 (+0x0000ee00) 0ff3e900 STI_XN_IMM.B64 X30, #3272 +0x000000000000f0c4 (+0x0000ee04) 02e0020b SHL.B64 X16, #11 +# [DWARF] common/pa_frontend.h:703 +# 697 | PA_DEVICE void RecordView() { ++counts_.views_created; } +# 698 | PA_DEVICE void RecordDynamicCreateInfo() { ++counts_.dynamic_create_infos; } +# 699 | +# 700 | template +# 701 | PA_DEVICE void AddLocalInput(Thunk thunk) { +# 702 | if constexpr (Lazy) { +# > 703 | if (!won_) return; +0x000000000000f0c8 (+0x0000ee08) 4020003b JUMPC #59 +# [DWARF] common/pa_frontend.h:468 +# 462 | destination.buffer_addr = source.buffer_addr; +# 463 | destination.buffer_size = source.buffer_size; +# 464 | destination.owner_task_id = source.owner_task_id; +# 465 | destination.start_offset = source.start_offset; +# 466 | destination.version = source.version; +# 467 | destination.ndims = source.ndims; +# > 468 | destination.dtype = source.dtype; +0x000000000000f0cc (+0x0000ee0c) 0883e220 SUB_IMM.S64 X1, X30, #544 +# [DWARF] common/pa_frontend.h:464 +# 458 | template +# 459 | PA_DEVICE void CopyTensorLine1(TensorDesc &destination, const Source &source) { +# 460 | // view 只需复制 descriptor 第一条 cache line 的身份/shape 字段，随后由调用方 +# 461 | // 覆盖 offset、shape、stride 与 extent；不做整 128-byte 拷贝以匹配 PA 写流。 +# 462 | destination.buffer_addr = source.buffer_addr; +# 463 | destination.buffer_size = source.buffer_size; +# > 464 | destination.owner_task_id = source.owner_task_id; +0x000000000000f0d0 (+0x0000ee10) 1cd3edc8 LD_XD_XN_IMM.B64 X9, X30, #3528 +# [DWARF] common/pa_frontend.h:301 +# 295 | return true; +# 296 | } +# 297 | +# 298 | PA_DEVICE void AppendLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { +# 299 | const uint32_t index = static_cast(args.tensor_count++); +# 300 | args.tensors[index].pointer.local_tensor = &tensor; +# > 301 | args.tensors[index].kind = TensorRefKind::LocalTensor; +0x000000000000f0d4 (+0x0000ee14) 0f29e800 STI_XN_IMM.B8 X30, #2624 +# [DWARF] common/pa_frontend.h:468 +# 462 | destination.buffer_addr = source.buffer_addr; +# 463 | destination.buffer_size = source.buffer_size; +# 464 | destination.owner_task_id = source.owner_task_id; +# 465 | destination.start_offset = source.start_offset; +# 466 | destination.version = source.version; +# 467 | destination.ndims = source.ndims; +# > 468 | destination.dtype = source.dtype; +0x000000000000f0d8 (+0x0000ee18) 0c021100 LDP_XI_XJ_XN.B8 X1, X2, X1, #0 +# [DWARF] common/pa_frontend.h:470 +# 469 | destination.manual_dep = source.manual_dep; +# > 470 | destination.is_contiguous = source.is_contiguous; +0x000000000000f0dc (+0x0000ee1c) 0887e21e SUB_IMM.S64 X3, X30, #542 +# [DWARF] common/pa_frontend.h:473 +# 471 | destination.child_memory = source.child_memory; +# 472 | for (uint32_t index = 0; index < kMaxTensorDims; ++index) { +# > 473 | destination.shapes[index] = source.shapes[index]; +0x000000000000f0e0 (+0x0000ee20) 088be210 SUB_IMM.S64 X5, X30, #528 +# [DWARF] common/pa_frontend.h:470 +# 464 | destination.owner_task_id = source.owner_task_id; +# 465 | destination.start_offset = source.start_offset; +# 466 | destination.version = source.version; +# 467 | destination.ndims = source.ndims; +# 468 | destination.dtype = source.dtype; +# 469 | destination.manual_dep = source.manual_dep; +# > 470 | destination.is_contiguous = source.is_contiguous; +0x000000000000f0e4 (+0x0000ee24) 0c063200 LDP_XI_XJ_XN.B8 X3, X4, X3, #0 +# [DWARF] common/pa_frontend.h:462 +# 456 | } +# 457 | +# 458 | template +# 459 | PA_DEVICE void CopyTensorLine1(TensorDesc &destination, const Source &source) { +# 460 | // view 只需复制 descriptor 第一条 cache line 的身份/shape 字段，随后由调用方 +# 461 | // 覆盖 offset、shape、stride 与 extent；不做整 128-byte 拷贝以匹配 PA 写流。 +# > 462 | destination.buffer_addr = source.buffer_addr; +0x000000000000f0e8 (+0x0000ee28) 088fe248 SUB_IMM.S64 X7, X30, #584 +# [DWARF] common/pa_frontend.h:473 +# 463 | destination.buffer_size = source.buffer_size; +# 464 | destination.owner_task_id = source.owner_task_id; +# 465 | destination.start_offset = source.start_offset; +# 466 | destination.version = source.version; +# 467 | destination.ndims = source.ndims; +# 468 | destination.dtype = source.dtype; +# 469 | destination.manual_dep = source.manual_dep; +# 470 | destination.is_contiguous = source.is_contiguous; +# 471 | destination.child_memory = source.child_memory; +# 472 | for (uint32_t index = 0; index < kMaxTensorDims; ++index) { +# > 473 | destination.shapes[index] = source.shapes[index]; +0x000000000000f0ec (+0x0000ee2c) 0c8a5300 LDP_XI_XJ_XN.B32 X5, X6, X5, #0 +# [DWARF] common/pa_frontend.h:462 +# 456 | } +# 457 | +# 458 | template +# 459 | PA_DEVICE void CopyTensorLine1(TensorDesc &destination, const Source &source) { +# 460 | // view 只需复制 descriptor 第一条 cache line 的身份/shape 字段，随后由调用方 +# 461 | // 覆盖 offset、shape、stride 与 extent；不做整 128-byte 拷贝以匹配 PA 写流。 +# > 462 | destination.buffer_addr = source.buffer_addr; +0x000000000000f0f0 (+0x0000ee30) 0cce7400 LDP_XI_XJ_XN.B64 X7, X8, X7, #0 +# [DWARF] common/pa_frontend.h:466 +# 463 | destination.buffer_size = source.buffer_size; +# 464 | destination.owner_task_id = source.owner_task_id; +# 465 | destination.start_offset = source.start_offset; +# > 466 | destination.version = source.version; +0x000000000000f0f4 (+0x0000ee34) 1c95edd8 LD_XD_XN_IMM.B32 X10, X30, #3544 +# [DWARF] common/pa_frontend.h:473 +# 467 | destination.ndims = source.ndims; +# 468 | destination.dtype = source.dtype; +# 469 | destination.manual_dep = source.manual_dep; +# 470 | destination.is_contiguous = source.is_contiguous; +# 471 | destination.child_memory = source.child_memory; +# 472 | for (uint32_t index = 0; index < kMaxTensorDims; ++index) { +# > 473 | destination.shapes[index] = source.shapes[index]; +0x000000000000f0f8 (+0x0000ee38) 1c97edec LD_XD_XN_IMM.B32 X11, X30, #3564 +# [DWARF] common/pa_frontend.h:468 +# 462 | destination.buffer_addr = source.buffer_addr; +# 463 | destination.buffer_size = source.buffer_size; +# 464 | destination.owner_task_id = source.owner_task_id; +# 465 | destination.start_offset = source.start_offset; +# 466 | destination.version = source.version; +# 467 | destination.ndims = source.ndims; +# > 468 | destination.dtype = source.dtype; +0x000000000000f0fc (+0x0000ee3c) 0303e0e0 ST_XD_XN_IMM.B8 X1, X30, #224 +# [DWARF] common/pa_frontend.h:469 +# > 469 | destination.manual_dep = source.manual_dep; +0x000000000000f100 (+0x0000ee40) 0803e0e1 ADD_IMM.S64 X1, X30, #225 +0x000000000000f104 (+0x0000ee44) 09041181 STP_XI_XJ_XN.B8 X2, X3, X1, #0 +# [DWARF] common/pa_frontend.h:473 +# 470 | destination.is_contiguous = source.is_contiguous; +# 471 | destination.child_memory = source.child_memory; +# 472 | for (uint32_t index = 0; index < kMaxTensorDims; ++index) { +# > 473 | destination.shapes[index] = source.shapes[index]; +0x000000000000f108 (+0x0000ee48) 0803e0f0 ADD_IMM.S64 X1, X30, #240 +# [DWARF] common/pa_frontend.h:471 +# 465 | destination.start_offset = source.start_offset; +# 466 | destination.version = source.version; +# 467 | destination.ndims = source.ndims; +# 468 | destination.dtype = source.dtype; +# 469 | destination.manual_dep = source.manual_dep; +# 470 | destination.is_contiguous = source.is_contiguous; +# > 471 | destination.child_memory = source.child_memory; +0x000000000000f10c (+0x0000ee4c) 0309e0e3 ST_XD_XN_IMM.B8 X4, X30, #227 +# [DWARF] common/pa_frontend.h:466 +# 460 | // view 只需复制 descriptor 第一条 cache line 的身份/shape 字段，随后由调用方 +# 461 | // 覆盖 offset、shape、stride 与 extent；不做整 128-byte 拷贝以匹配 PA 写流。 +# 462 | destination.buffer_addr = source.buffer_addr; +# 463 | destination.buffer_size = source.buffer_size; +# 464 | destination.owner_task_id = source.owner_task_id; +# 465 | destination.start_offset = source.start_offset; +# > 466 | destination.version = source.version; +0x000000000000f110 (+0x0000ee50) 0805e0d8 ADD_IMM.S64 X2, X30, #216 +# [DWARF] common/pa_frontend.h:473 +# 467 | destination.ndims = source.ndims; +# 468 | destination.dtype = source.dtype; +# 469 | destination.manual_dep = source.manual_dep; +# 470 | destination.is_contiguous = source.is_contiguous; +# 471 | destination.child_memory = source.child_memory; +# 472 | for (uint32_t index = 0; index < kMaxTensorDims; ++index) { +# > 473 | destination.shapes[index] = source.shapes[index]; +0x000000000000f114 (+0x0000ee54) 098a1301 STP_XI_XJ_XN.B32 X5, X6, X1, #0 +# [DWARF] common/pa_frontend.h:462 +# 456 | } +# 457 | +# 458 | template +# 459 | PA_DEVICE void CopyTensorLine1(TensorDesc &destination, const Source &source) { +# 460 | // view 只需复制 descriptor 第一条 cache line 的身份/shape 字段，随后由调用方 +# 461 | // 覆盖 offset、shape、stride 与 extent；不做整 128-byte 拷贝以匹配 PA 写流。 +# > 462 | destination.buffer_addr = source.buffer_addr; +0x000000000000f118 (+0x0000ee58) 0803e0b8 ADD_IMM.S64 X1, X30, #184 +# [DWARF] common/pa_frontend.h:520 +# 514 | } +# 515 | +# 516 | PA_DEVICE void MakeLazySampleCallbackQueryView(PaOrchestrationState &orch, uint32_t batch) { +# 517 | CopyTensorLine1(orch.query_view, orch.query); +# 518 | orch.query_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; +# 519 | orch.query_view.ndims = 2; +# > 520 | orch.query_view.shapes[0] = kPaHeads; +0x000000000000f11c (+0x0000ee5c) 0807e0e4 ADD_IMM.S64 X3, X30, #228 +# [DWARF] common/pa_frontend.h:462 +# 456 | } +# 457 | +# 458 | template +# 459 | PA_DEVICE void CopyTensorLine1(TensorDesc &destination, const Source &source) { +# 460 | // view 只需复制 descriptor 第一条 cache line 的身份/shape 字段，随后由调用方 +# 461 | // 覆盖 offset、shape、stride 与 extent；不做整 128-byte 拷贝以匹配 PA 写流。 +# > 462 | destination.buffer_addr = source.buffer_addr; +0x000000000000f120 (+0x0000ee60) 09ce1401 STP_XI_XJ_XN.B64 X7, X8, X1, #0 +# [DWARF] common/pa_frontend.h:464 +# 463 | destination.buffer_size = source.buffer_size; +# > 464 | destination.owner_task_id = source.owner_task_id; +0x000000000000f124 (+0x0000ee64) 0803e0c8 ADD_IMM.S64 X1, X30, #200 +0x000000000000f128 (+0x0000ee68) 1ccfe990 LD_XD_XN_IMM.B64 X7, X30, #2448 +0x000000000000f12c (+0x0000ee6c) 09d21801 STP_XI_XJ_XN.B64 X9, X16, X1, #0 +# [DWARF] common/pa_frontend.h:519 +# 513 | orch.output_view.extent_elem_cache = kPaHeads * kPaHeadDim; +# 514 | } +# 515 | +# 516 | PA_DEVICE void MakeLazySampleCallbackQueryView(PaOrchestrationState &orch, uint32_t batch) { +# 517 | CopyTensorLine1(orch.query_view, orch.query); +# 518 | orch.query_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; +# > 519 | orch.query_view.ndims = 2; +0x000000000000f130 (+0x0000ee70) 07020002 MOV_XD_IMM X1, #2 +# [DWARF] common/pa_frontend.h:466 +# 460 | // view 只需复制 descriptor 第一条 cache line 的身份/shape 字段，随后由调用方 +# 461 | // 覆盖 offset、shape、stride 与 extent；不做整 128-byte 拷贝以匹配 PA 写流。 +# 462 | destination.buffer_addr = source.buffer_addr; +# 463 | destination.buffer_size = source.buffer_size; +# 464 | destination.owner_task_id = source.owner_task_id; +# 465 | destination.start_offset = source.start_offset; +# > 466 | destination.version = source.version; +0x000000000000f134 (+0x0000ee74) 09942081 STP_XI_XJ_XN.B32 X10, X1, X2, #0 +# [DWARF] common/pa_frontend.h:520 +# 514 | } +# 515 | +# 516 | PA_DEVICE void MakeLazySampleCallbackQueryView(PaOrchestrationState &orch, uint32_t batch) { +# 517 | CopyTensorLine1(orch.query_view, orch.query); +# 518 | orch.query_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; +# 519 | orch.query_view.ndims = 2; +# > 520 | orch.query_view.shapes[0] = kPaHeads; +0x000000000000f138 (+0x0000ee78) 07040010 MOV_XD_IMM X2, #16 +# [DWARF] common/pa_frontend.h:521 +# > 521 | orch.query_view.shapes[1] = kPaHeadDim; +0x000000000000f13c (+0x0000ee7c) 07020080 MOV_XD_IMM X1, #128 +# [DWARF] common/pa_frontend.h:473 +# 467 | destination.ndims = source.ndims; +# 468 | destination.dtype = source.dtype; +# 469 | destination.manual_dep = source.manual_dep; +# 470 | destination.is_contiguous = source.is_contiguous; +# 471 | destination.child_memory = source.child_memory; +# 472 | for (uint32_t index = 0; index < kMaxTensorDims; ++index) { +# > 473 | destination.shapes[index] = source.shapes[index]; +0x000000000000f140 (+0x0000ee80) 0397e0ec ST_XD_XN_IMM.B32 X11, X30, #236 +# [DWARF] common/pa_frontend.h:520 +# 514 | } +# 515 | +# 516 | PA_DEVICE void MakeLazySampleCallbackQueryView(PaOrchestrationState &orch, uint32_t batch) { +# 517 | CopyTensorLine1(orch.query_view, orch.query); +# 518 | orch.query_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; +# 519 | orch.query_view.ndims = 2; +# > 520 | orch.query_view.shapes[0] = kPaHeads; +0x000000000000f144 (+0x0000ee84) 09843081 STP_XI_XJ_XN.B32 X2, X1, X3, #0 +# [DWARF] common/pa_frontend.h:522 +# 521 | orch.query_view.shapes[1] = kPaHeadDim; +# > 522 | orch.query_view.strides[0] = kPaHeadDim; +0x000000000000f148 (+0x0000ee88) 07020080 MOV_XD_IMM X1, #128 +0x000000000000f14c (+0x0000ee8c) 07830001 MOVK X1, #1, #2 +0x000000000000f150 (+0x0000ee90) 03c3e100 ST_XD_XN_IMM.B64 X1, X30, #256 +# [DWARF] common/pa_frontend.h:524 +# 523 | orch.query_view.strides[1] = 1; +# > 524 | orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; +0x000000000000f154 (+0x0000ee94) 07020800 MOV_XD_IMM X1, #2048 +0x000000000000f158 (+0x0000ee98) 03c3e0f8 ST_XD_XN_IMM.B64 X1, X30, #248 +# [DWARF] common/pa_frontend.h:300 +# 294 | } +# 295 | return true; +# 296 | } +# 297 | +# 298 | PA_DEVICE void AppendLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { +# 299 | const uint32_t index = static_cast(args.tensor_count++); +# > 300 | args.tensors[index].pointer.local_tensor = &tensor; +0x000000000000f15c (+0x0000ee9c) 070200b8 MOV_XD_IMM X1, #184 +0x000000000000f160 (+0x0000eea0) 0003e081 ADD.S64 X1, X30, X1 +0x000000000000f164 (+0x0000eea4) 03c3ea38 ST_XD_XN_IMM.B64 X1, X30, #2616 +# [DWARF] common/pa_frontend.h:302 +# 301 | args.tensors[index].kind = TensorRefKind::LocalTensor; +# > 302 | args.tags[index] = TagValue(tag); +0x000000000000f168 (+0x0000eea8) 07020648 MOV_XD_IMM X1, #1608 +0x000000000000f16c (+0x0000eeac) 0003e082 SUB.S64 X1, X30, X1 +0x000000000000f170 (+0x0000eeb0) 0f801000 STI_XN_IMM.B32 X1, #0 +# [DWARF] common/pa_frontend.h:300 +# 294 | } +# 295 | return true; +# 296 | } +# 297 | +# 298 | PA_DEVICE void AppendLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { +# 299 | const uint32_t index = static_cast(args.tensor_count++); +# > 300 | args.tensors[index].pointer.local_tensor = &tensor; +0x000000000000f174 (+0x0000eeb4) 070201c8 MOV_XD_IMM X1, #456 +0x000000000000f178 (+0x0000eeb8) 0003e082 SUB.S64 X1, X30, X1 +# [DWARF] common/pa_frontend.h:301 +# > 301 | args.tensors[index].kind = TensorRefKind::LocalTensor; +0x000000000000f17c (+0x0000eebc) 0f29ea00 STI_XN_IMM.B8 X30, #2640 +# [DWARF] common/pa_frontend.h:300 +# 294 | } +# 295 | return true; +# 296 | } +# 297 | +# 298 | PA_DEVICE void AppendLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { +# 299 | const uint32_t index = static_cast(args.tensor_count++); +# > 300 | args.tensors[index].pointer.local_tensor = &tensor; +0x000000000000f180 (+0x0000eec0) 03c3ea48 ST_XD_XN_IMM.B64 X1, X30, #2632 +# [DWARF] common/pa_frontend.h:302 +# 301 | args.tensors[index].kind = TensorRefKind::LocalTensor; +# > 302 | args.tags[index] = TagValue(tag); +0x000000000000f184 (+0x0000eec4) 1cc3e8e8 LD_XD_XN_IMM.B64 X1, X30, #2280 +0x000000000000f188 (+0x0000eec8) 07060003 MOV_XD_IMM X3, #3 +0x000000000000f18c (+0x0000eecc) 0f801000 STI_XN_IMM.B32 X1, #0 +# [DWARF] common/pa_frontend.h:300 +# 294 | } +# 295 | return true; +# 296 | } +# 297 | +# 298 | PA_DEVICE void AppendLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { +# 299 | const uint32_t index = static_cast(args.tensor_count++); +# > 300 | args.tensors[index].pointer.local_tensor = &tensor; +0x000000000000f190 (+0x0000eed0) 070200c8 MOV_XD_IMM X1, #200 +0x000000000000f194 (+0x0000eed4) 0003e082 SUB.S64 X1, X30, X1 +# [DWARF] common/pa_frontend.h:299 +# 293 | return false; +# 294 | } +# 295 | return true; +# 296 | } +# 297 | +# 298 | PA_DEVICE void AppendLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { +# > 299 | const uint32_t index = static_cast(args.tensor_count++); +0x000000000000f198 (+0x0000eed8) 0387ecb8 ST_XD_XN_IMM.B32 X3, X30, #3256 +# [DWARF] common/pa_frontend.h:300 +# > 300 | args.tensors[index].pointer.local_tensor = &tensor; +0x000000000000f19c (+0x0000eedc) 03c3ea58 ST_XD_XN_IMM.B64 X1, X30, #2648 +# [DWARF] common/pa_frontend.h:302 +# 301 | args.tensors[index].kind = TensorRefKind::LocalTensor; +# > 302 | args.tags[index] = TagValue(tag); +0x000000000000f1a0 (+0x0000eee0) 07020640 MOV_XD_IMM X1, #1600 +0x000000000000f1a4 (+0x0000eee4) 0003e082 SUB.S64 X1, X30, X1 +# [DWARF] common/pa_frontend.h:301 +# 295 | return true; +# 296 | } +# 297 | +# 298 | PA_DEVICE void AppendLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { +# 299 | const uint32_t index = static_cast(args.tensor_count++); +# 300 | args.tensors[index].pointer.local_tensor = &tensor; +# > 301 | args.tensors[index].kind = TensorRefKind::LocalTensor; +0x000000000000f1a8 (+0x0000eee8) 0f29ec00 STI_XN_IMM.B8 X30, #2656 +# [DWARF] common/pa_frontend.h:302 +# > 302 | args.tags[index] = TagValue(tag); +0x000000000000f1ac (+0x0000eeec) 0f801000 STI_XN_IMM.B32 X1, #0 +0x000000000000f1b0 (+0x0000eef0) 07020001 MOV_XD_IMM X1, #1 +# [DWARF] common/pa_scheduler_core.h:847 +# 841 | builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { +# 842 | return orch.block_table; +# 843 | }); +# 844 | builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { +# 845 | const uint32_t score_shape[kMaxTensorDims] = { +# 846 | kPaHeads, +# > 847 | static_cast(orch.current_nblocks * kPaBlockSize), +0x000000000000f1b4 (+0x0000eef4) 1cc9e2e0 LD_XD_XN_IMM.B64 X4, X30, #736 +# [DWARF] common/pa_frontend.h:366 +# 360 | +# 361 | PA_DEVICE void InitCreateInfo( +# 362 | TensorCreateInfo &info, const uint32_t shapes[kMaxTensorDims], uint32_t ndims, DataType dtype +# 363 | ) { +# 364 | info.initial_value = 0; +# 365 | info.has_initial_value = false; +# > 366 | info.reserved0 = 0; +0x000000000000f1b8 (+0x0000eef8) 070a0000 MOV_XD_IMM X5, #0 +# [DWARF] common/pa_frontend.h:364 +# 358 | AppendScalar(args, value2); +# 359 | } +# 360 | +# 361 | PA_DEVICE void InitCreateInfo( +# 362 | TensorCreateInfo &info, const uint32_t shapes[kMaxTensorDims], uint32_t ndims, DataType dtype +# 363 | ) { +# > 364 | info.initial_value = 0; +0x000000000000f1bc (+0x0000eefc) 0fc9e700 STI_XN_IMM.B64 X30, #568 +0x000000000000f1c0 (+0x0000ef00) 1cf3e980 LD_XD_XN_IMM.B64 X25, X30, #2432 +# [DWARF] common/pa_frontend.h:366 +# 365 | info.has_initial_value = false; +# > 366 | info.reserved0 = 0; +0x000000000000f1c4 (+0x0000ef04) 080de248 ADD_IMM.S64 X6, X30, #584 +# [DWARF] common/pa_frontend.h:365 +# 359 | } +# 360 | +# 361 | PA_DEVICE void InitCreateInfo( +# 362 | TensorCreateInfo &info, const uint32_t shapes[kMaxTensorDims], uint32_t ndims, DataType dtype +# 363 | ) { +# 364 | info.initial_value = 0; +# > 365 | info.has_initial_value = false; +0x000000000000f1c8 (+0x0000ef08) 0f09e800 STI_XN_IMM.B8 X30, #576 +# [DWARF] common/pa_frontend.h:366 +# > 366 | info.reserved0 = 0; +0x000000000000f1cc (+0x0000ef0c) 09ca6281 STP_XI_XJ_XN.B64 X5, X5, X6, #0 +# [DWARF] common/pa_frontend.h:368 +# 367 | info.start_offset = 0; +# > 368 | info.version = 0; +0x000000000000f1d0 (+0x0000ef10) 070c0000 MOV_XD_IMM X6, #0 +0x000000000000f1d4 (+0x0000ef14) 078d0002 MOVK X6, #2, #2 +0x000000000000f1d8 (+0x0000ef18) 03e1e8d8 ST_XD_XN_IMM.B64 X16, X30, #2264 +0x000000000000f1dc (+0x0000ef1c) 03cde258 ST_XD_XN_IMM.B64 X6, X30, #600 +# [DWARF] common/pa_frontend.h:370 +# 369 | info.ndims = ndims; +# > 370 | info.dtype = dtype; +0x000000000000f1e0 (+0x0000ef20) 070c0000 MOV_XD_IMM X6, #0 +0x000000000000f1e4 (+0x0000ef24) 074d0001 MOVK X6, #1, #1 +0x000000000000f1e8 (+0x0000ef28) 078d0010 MOVK X6, #16, #2 +# [DWARF] common/pa_frontend.h:314 +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# > 314 | args.tensors[index].pointer.create_info = &create_info; +0x000000000000f1ec (+0x0000ef2c) 020a3800 MOV_XD_XN.S64 X5, X3 +# [DWARF] common/pa_frontend.h:370 +# 364 | info.initial_value = 0; +# 365 | info.has_initial_value = false; +# 366 | info.reserved0 = 0; +# 367 | info.start_offset = 0; +# 368 | info.version = 0; +# 369 | info.ndims = ndims; +# > 370 | info.dtype = dtype; +0x000000000000f1f0 (+0x0000ef30) 03cde260 ST_XD_XN_IMM.B64 X6, X30, #608 +# [DWARF] common/pa_frontend.h:314 +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# 313 | const uint32_t index = static_cast(args.tensor_count++); +# > 314 | args.tensors[index].pointer.create_info = &create_info; +0x000000000000f1f4 (+0x0000ef34) 070c05c8 MOV_XD_IMM X6, #1480 +0x000000000000f1f8 (+0x0000ef38) 02ca0204 SHL.B64 X5, #4 +0x000000000000f1fc (+0x0000ef3c) 000de302 SUB.S64 X6, X30, X6 +0x000000000000f200 (+0x0000ef40) 000a6281 ADD.S64 X5, X6, X5 +0x000000000000f204 (+0x0000ef44) 070c0238 MOV_XD_IMM X6, #568 +0x000000000000f208 (+0x0000ef48) 000de301 ADD.S64 X6, X30, X6 +0x000000000000f20c (+0x0000ef4c) 072a0004 MOV_XD_IMM X21, #4 +# [DWARF] common/pa_frontend.h:0 +# [SOURCE unavailable] +0x000000000000f210 (+0x0000ef50) 072c12c4 MOV_XD_IMM X22, #4804 +0x000000000000f214 (+0x0000ef54) 07300020 MOV_XD_IMM X24, #32 +# [DWARF] common/pa_scheduler_core.h:847 +# 841 | builder.AddLocalInput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorDesc & { +# 842 | return orch.block_table; +# 843 | }); +# 844 | builder.AddOutput([&]() PA_LAZY_LAMBDA_DEVICE -> const TensorCreateInfo & { +# 845 | const uint32_t score_shape[kMaxTensorDims] = { +# 846 | kPaHeads, +# > 847 | static_cast(orch.current_nblocks * kPaBlockSize), +0x000000000000f218 (+0x0000ef58) 02044800 MOV_XD_XN.S64 X2, X4 +0x000000000000f21c (+0x0000ef5c) 02c40207 SHL.B64 X2, #7 +# [DWARF] common/pa_frontend.h:377 +# 371 | info.manual_dep = false; +# 372 | info.is_contiguous = true; +# 373 | info.child_memory = 0; +# 374 | // TensorCreateInfo's real constructor only writes active dimensions. +# 375 | // 只写 ndims 个 shape，保留生产构造器的写入范围，不能为方便把五维全清零。 +# 376 | for (uint32_t index = 0; index < ndims; ++index) { +# > 377 | info.shapes[index] = shapes[index]; +0x000000000000f220 (+0x0000ef60) 0385e268 ST_XD_XN_IMM.B32 X2, X30, #616 +# [DWARF] common/pa_frontend.h:313 +# 307 | args.tensors[index].pointer.gm_tensor = &tensor; +# 308 | args.tensors[index].kind = TensorRefKind::GmTensor; +# 309 | args.tags[index] = TagValue(tag); +# 310 | } +# 311 | +# 312 | PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { +# > 313 | const uint32_t index = static_cast(args.tensor_count++); +0x000000000000f224 (+0x0000ef64) 08043001 ADD_IMM.S64 X2, X3, #1 +0x000000000000f228 (+0x0000ef68) 0385ecb8 ST_XD_XN_IMM.B32 X2, X30, #3256 +# [DWARF] common/pa_frontend.h:314 +# > 314 | args.tensors[index].pointer.create_info = &create_info; +0x000000000000f22c (+0x0000ef6c) 03cc5000 ST_XD_XN_IMM.B64 X6, X5, #0 +# [DWARF] common/pa_frontend.h:315 +# > 315 | args.tensors[index].kind = TensorRefKind::CreateInfo; +0x000000000000f230 (+0x0000ef70) 070c0002 MOV_XD_IMM X6, #2 +0x000000000000f234 (+0x0000ef74) 030c5008 ST_XD_XN_IMM.B8 X6, X5, #8 +# [DWARF] common/pa_frontend.h:316 +# > 316 | args.tags[index] = TagValue(TensorArgType::Output); +0x000000000000f238 (+0x0000ef78) 070a0648 MOV_XD_IMM X5, #1608 +0x000000000000f23c (+0x0000ef7c) 000be282 SUB.S64 X5, X30, X5 +0x000000000000f240 (+0x0000ef80) 0e805181 STI_XN_XM.B32 X5, X3 +# [DWARF] common/pa_frontend.h:749 +# 743 | if (!args_.has_error) ++counts_.tensor_args_added; +# 744 | } +# 745 | +# 746 | template +# 747 | PA_DEVICE void AddScalar(Thunk thunk) { +# 748 | if constexpr (Lazy) { +# > 749 | if (!won_) return; +0x000000000000f244 (+0x0000ef84) 40200009 JUMPC #9 +# [DWARF] common/pa_scheduler_core.h:858 +# 852 | return orch.qk_create_info; +# 853 | }); +# 854 | builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { +# 855 | return orch.current_nblocks; +# 856 | }); +# 857 | builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { +# > 858 | return static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + +0x000000000000f248 (+0x0000ef88) 1c87e2f0 LD_XD_XN_IMM.B32 X3, X30, #752 +0x000000000000f24c (+0x0000ef8c) 07000002 MOV_XD_IMM X0, #2 +# [DWARF] common/pa_scheduler_core.h:859 +# > 859 | orch.current_block_offset; +0x000000000000f250 (+0x0000ef90) 1ccbe2d8 LD_XD_XN_IMM.B64 X5, X30, #728 +# [DWARF] common/pa_frontend.h:329 +# 323 | return false; +# 324 | } +# 325 | return true; +# 326 | } +# 327 | +# 328 | PA_DEVICE void AppendScalar(TaskArgs &args, uint64_t value) { +# > 329 | args.scalars[static_cast(args.scalar_count++)] = value; +0x000000000000f254 (+0x0000ef94) 0381ecbc ST_XD_XN_IMM.B32 X0, X30, #3260 +# [DWARF] common/pa_scheduler_core.h:858 +# 852 | return orch.qk_create_info; +# 853 | }); +# 854 | builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { +# 855 | return orch.current_nblocks; +# 856 | }); +# 857 | builder.AddScalar([&]() PA_LAZY_LAMBDA_DEVICE -> uint64_t { +# > 858 | return static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + +0x000000000000f258 (+0x0000ef98) 02c60208 SHL.B64 X3, #8 +0x000000000000f25c (+0x0000ef9c) 00063281 ADD.S64 X3, X3, X5 +# [DWARF] common/pa_frontend.h:329 +# 323 | return false; +# 324 | } +# 325 | return true; +# 326 | } +# 327 | +# 328 | PA_DEVICE void AppendScalar(TaskArgs &args, uint64_t value) { +# > 329 | args.scalars[static_cast(args.scalar_count++)] = value; +0x000000000000f260 (+0x0000efa0) 088be3c8 SUB_IMM.S64 X5, X30, #968 +0x000000000000f264 (+0x0000efa4) 09c85181 STP_XI_XJ_XN.B64 X4, X3, X5, #0 +# [DWARF] common/pa_frontend.h:0 +# [SOURCE unavailable] +0x000000000000f268 (+0x0000efa8) 0808e002 ADD_IMM.S64 X4, X14, #2 +# [DWARF] common/pa_scheduler_core.h:1161 +# 1155 | ); +# 1156 | +# 1157 | if (!BuildLazySampleCallbackArgs(orch, args, batch, claim.won, stats)) { +# 1158 | SetFatal(state, stats, static_cast(task_id)); +# 1159 | return false; +# 1160 | } +# > 1161 | const LazySampleCallbackTicket ticket{ +0x000000000000f26c (+0x0000efac) 03a3e340 ST_XD_XN_IMM.B32 X17, X30, #832 +0x000000000000f270 (+0x0000efb0) 00c6d78a AND.B64 X3, X13, X15 +0x000000000000f274 (+0x0000efb4) 036fe344 ST_XD_XN_IMM.B16 X23, X30, #836 +0x000000000000f278 (+0x0000efb8) 02c8020e SHL.B64 X4, #14 +0x000000000000f27c (+0x0000efbc) 0325e346 ST_XD_XN_IMM.B8 X18, X30, #838 +0x000000000000f280 (+0x0000efc0) 000ac381 ADD.S64 X5, X12, X7 +0x000000000000f284 (+0x0000efc4) 0f0de8e0 STI_XN_IMM.B8 X30, #839 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x000000000000f288 (+0x0000efc8) 000a3204 MADD.S64 X5, X3, X4 +0x000000000000f28c (+0x0000efcc) 00065201 ADD.S64 X3, X5, X4 +0x000000000000f290 (+0x0000efd0) 08863680 SUB_IMM.S64 X3, X3, #1664 +# [DWARF] common/pa_scheduler_core.h:945 +# 939 | } +# 940 | }; +# 941 | +# 942 | callback(callback_builder); +# 943 | if (!callback_builder.Valid()) return false; +# 944 | const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); +# > 945 | stats.result.arg_resets += counts.reset_calls; +0x000000000000f294 (+0x0000efd4) 08263000 ADD_IMM.S64 X19, X3, #0 +# [DWARF] common/pa_scheduler_core.h:946 +# > 946 | stats.result.views_created += counts.views_created; +0x000000000000f298 (+0x0000efd8) 080733c8 ADD_IMM.S64 X3, X19, #968 +# [DWARF] common/pa_scheduler_core.h:945 +# 939 | } +# 940 | }; +# 941 | +# 942 | callback(callback_builder); +# 943 | if (!callback_builder.Valid()) return false; +# 944 | const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); +# > 945 | stats.result.arg_resets += counts.reset_calls; +0x000000000000f29c (+0x0000efdc) 1ccf33d8 LD_XD_XN_IMM.B64 X7, X19, #984 +# [DWARF] common/pa_scheduler_core.h:946 +# > 946 | stats.result.views_created += counts.views_created; +0x000000000000f2a0 (+0x0000efe0) 0cc63300 LDP_XI_XJ_XN.B64 X3, X6, X3, #0 +# [DWARF] common/pa_scheduler_core.h:948 +# 947 | stats.result.dynamic_create_infos += counts.dynamic_create_infos; +# > 948 | stats.result.tensor_args_added += counts.tensor_args_added; +0x000000000000f2a4 (+0x0000efe4) 080933e0 ADD_IMM.S64 X4, X19, #992 +0x000000000000f2a8 (+0x0000efe8) 0cc84280 LDP_XI_XJ_XN.B64 X4, X5, X4, #0 +# [DWARF] common/pa_scheduler_core.h:946 +# 940 | }; +# 941 | +# 942 | callback(callback_builder); +# 943 | if (!callback_builder.Valid()) return false; +# 944 | const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); +# 945 | stats.result.arg_resets += counts.reset_calls; +# > 946 | stats.result.views_created += counts.views_created; +0x000000000000f2ac (+0x0000efec) 00023081 ADD.S64 X1, X3, X1 +# [DWARF] common/pa_scheduler_core.h:947 +# > 947 | stats.result.dynamic_create_infos += counts.dynamic_create_infos; +0x000000000000f2b0 (+0x0000eff0) 08066001 ADD_IMM.S64 X3, X6, #1 +# [DWARF] common/pa_scheduler_core.h:946 +# 940 | }; +# 941 | +# 942 | callback(callback_builder); +# 943 | if (!callback_builder.Valid()) return false; +# 944 | const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); +# 945 | stats.result.arg_resets += counts.reset_calls; +# > 946 | stats.result.views_created += counts.views_created; +0x000000000000f2b4 (+0x0000eff4) 080d33c8 ADD_IMM.S64 X6, X19, #968 +0x000000000000f2b8 (+0x0000eff8) 09c26181 STP_XI_XJ_XN.B64 X1, X3, X6, #0 +# [DWARF] common/pa_scheduler_core.h:945 +# 939 | } +# 940 | }; +# 941 | +# 942 | callback(callback_builder); +# 943 | if (!callback_builder.Valid()) return false; +# 944 | const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); +# > 945 | stats.result.arg_resets += counts.reset_calls; +0x000000000000f2bc (+0x0000effc) 08027001 ADD_IMM.S64 X1, X7, #1 +# [DWARF] common/pa_scheduler_core.h:948 +# 946 | stats.result.views_created += counts.views_created; +# 947 | stats.result.dynamic_create_infos += counts.dynamic_create_infos; +# > 948 | stats.result.tensor_args_added += counts.tensor_args_added; +0x000000000000f2c0 (+0x0000f000) 00044101 ADD.S64 X2, X4, X2 +# [DWARF] common/pa_scheduler_core.h:945 +# 939 | } +# 940 | }; +# 941 | +# 942 | callback(callback_builder); +# 943 | if (!callback_builder.Valid()) return false; +# 944 | const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); +# > 945 | stats.result.arg_resets += counts.reset_calls; +0x000000000000f2c4 (+0x0000f004) 080733d8 ADD_IMM.S64 X3, X19, #984 +# [DWARF] common/pa_scheduler_core.h:949 +# 946 | stats.result.views_created += counts.views_created; +# 947 | stats.result.dynamic_create_infos += counts.dynamic_create_infos; +# 948 | stats.result.tensor_args_added += counts.tensor_args_added; +# > 949 | stats.result.scalar_args_added += counts.scalar_args_added; +0x000000000000f2c8 (+0x0000f008) 00005001 ADD.S64 X0, X5, X0 +# [DWARF] common/pa_scheduler_core.h:945 +# 939 | } +# 940 | }; +# 941 | +# 942 | callback(callback_builder); +# 943 | if (!callback_builder.Valid()) return false; +# 944 | const LazySampleCallbackBuildCounts &counts = callback_builder.Counts(); +# > 945 | stats.result.arg_resets += counts.reset_calls; +0x000000000000f2cc (+0x0000f00c) 09c23101 STP_XI_XJ_XN.B64 X1, X2, X3, #0 +# [DWARF] common/pa_scheduler_core.h:949 +# 946 | stats.result.views_created += counts.views_created; +# 947 | stats.result.dynamic_create_infos += counts.dynamic_create_infos; +# 948 | stats.result.tensor_args_added += counts.tensor_args_added; +# > 949 | stats.result.scalar_args_added += counts.scalar_args_added; +0x000000000000f2d0 (+0x0000f010) 03c133e8 ST_XD_XN_IMM.B64 X0, X19, #1000 +# [DWARF] common/pa_scheduler_core.h:1161 +# 1155 | ); +# 1156 | +# 1157 | if (!BuildLazySampleCallbackArgs(orch, args, batch, claim.won, stats)) { +# 1158 | SetFatal(state, stats, static_cast(task_id)); +# 1159 | return false; +# 1160 | } +# > 1161 | const LazySampleCallbackTicket ticket{ +0x000000000000f2d4 (+0x0000f014) 1cc1e928 LD_XD_XN_IMM.B64 X0, X30, #2344 +# [DWARF] ccec/ccec_ops.h:276 +# 270 | } +# 271 | +# 272 | __aicore__ static inline bool FinishLazySampleCallback( +# 273 | const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args +# 274 | ) { +# 275 | #if defined(PA_BUILD_AIC) +# > 276 | return ::pa_scheduler_lazy_sample_callback_finish_aic(ticket, args) != 0; +0x000000000000f2d8 (+0x0000f018) 07020648 MOV_XD_IMM X1, #1608 +0x000000000000f2dc (+0x0000f01c) 0003e082 SUB.S64 X1, X30, X1 +# [DWARF] common/pa_scheduler_core.h:1161 +# 1155 | ); +# 1156 | +# 1157 | if (!BuildLazySampleCallbackArgs(orch, args, batch, claim.won, stats)) { +# 1158 | SetFatal(state, stats, static_cast(task_id)); +# 1159 | return false; +# 1160 | } +# > 1161 | const LazySampleCallbackTicket ticket{ +0x000000000000f2e0 (+0x0000f020) 03c1e338 ST_XD_XN_IMM.B64 X0, X30, #824 +# [DWARF] ccec/ccec_ops.h:276 +# 270 | } +# 271 | +# 272 | __aicore__ static inline bool FinishLazySampleCallback( +# 273 | const pa_scheduler::LazySampleCallbackTicket *ticket, const pa_scheduler::TaskArgs *args +# 274 | ) { +# 275 | #if defined(PA_BUILD_AIC) +# > 276 | return ::pa_scheduler_lazy_sample_callback_finish_aic(ticket, args) != 0; +0x000000000000f2e4 (+0x0000f024) 07000338 MOV_XD_IMM X0, #824 +0x000000000000f2e8 (+0x0000f028) 0001e001 ADD.S64 X0, X30, X0 +0x000000000000f2ec (+0x0000f02c) 0704622a MOV_XD_IMM X2, #25130 +0x000000000000f2f0 (+0x0000f030) 07450000 MOVK X2, #0, #1 +0x000000000000f2f4 (+0x0000f034) 07850000 MOVK X2, #0, #2 +0x000000000000f2f8 (+0x0000f038) 40422000 CALL X2, #0 +0x000000000000f2fc (+0x0000f03c) 02800a00 ZEROEXT.U32 X0, X0 +0x000000000000f300 (+0x0000f040) 071c0000 MOV_XD_IMM X14, #0 +0x000000000000f304 (+0x0000f044) 0000070e CMP.S64.EQ X0, X14 +0x000000000000f308 (+0x0000f048) 07220001 MOV_XD_IMM X17, #1 +0x000000000000f30c (+0x0000f04c) 07100006 MOV_XD_IMM X8, #6 +# [DWARF] common/pa_scheduler_core.h:1393 +# 1387 | )) { +# 1388 | break; +# 1389 | } +# 1390 | AcceptTaskOutputs(orchestration, TaskKind::Alloc, context.result); +# 1391 | +# 1392 | PreparePaBlockGroup(orchestration, 0); +# > 1393 | if (!SubmitLazySampleCallback< +0x000000000000f310 (+0x0000f050) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000000f314 (+0x0000f054) 07004302 MOV_XD_IMM X0, #17154 +0x000000000000f318 (+0x0000f058) 07410000 MOVK X0, #0, #1 +0x000000000000f31c (+0x0000f05c) 07810000 MOVK X0, #0, #2 +0x000000000000f320 (+0x0000f060) 40220000 JUMPC X0, #0 +0x000000000000f324 (+0x0000f064) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_frontend.h:835 +# 829 | case TaskKind::Alloc: +# 830 | orch.accumulated_output = outputs.tensors[0]; +# 831 | orch.accumulated_sum = outputs.tensors[1]; +# 832 | orch.accumulated_max = outputs.tensors[2]; +# 833 | break; +# 834 | case TaskKind::Qk: +# > 835 | orch.qk_scores = outputs.tensors[0]; +0x000000000000f328 (+0x0000f068) 1cc13050 LD_XD_XN_IMM.B64 X0, X19, #80 +# [DWARF] common/pa_scheduler_core.h:790 +# 784 | +# 785 | PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { +# 786 | // This is BeginSubmit without an already-materialized TaskArgs. The same +# 787 | // fields are completed synchronously after the single callback builds args. +# 788 | const uint32_t task_id = static_cast(worker.local_index++); +# 789 | context.self = &worker; +# > 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +0x000000000000f32c (+0x0000f06c) 070207ff MOV_XD_IMM X1, #2047 +0x000000000000f330 (+0x0000f070) 1cc5e960 LD_XD_XN_IMM.B64 X2, X30, #2400 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x000000000000f334 (+0x0000f074) 071e7fff MOV_XD_IMM X15, #32767 +# [DWARF] common/pa_frontend.h:835 +# 829 | case TaskKind::Alloc: +# 830 | orch.accumulated_output = outputs.tensors[0]; +# 831 | orch.accumulated_sum = outputs.tensors[1]; +# 832 | orch.accumulated_max = outputs.tensors[2]; +# 833 | break; +# 834 | case TaskKind::Qk: +# > 835 | orch.qk_scores = outputs.tensors[0]; +0x000000000000f338 (+0x0000f078) 03c1e310 ST_XD_XN_IMM.B64 X0, X30, #784 +# [DWARF] common/pa_scheduler_core.h:788 +# 782 | static_assert(offsetof(LazySampleCallbackTicket, function_id) == 12, "lazy sample callback ticket function offset mismatch"); +# 783 | static_assert(offsetof(LazySampleCallbackTicket, won) == 14, "lazy sample callback ticket winner offset mismatch"); +# 784 | +# 785 | PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { +# 786 | // This is BeginSubmit without an already-materialized TaskArgs. The same +# 787 | // fields are completed synchronously after the single callback builds args. +# > 788 | const uint32_t task_id = static_cast(worker.local_index++); +0x000000000000f33c (+0x0000f07c) 1c862014 LD_XD_XN_IMM.B32 X3, X2, #20 +0x000000000000f340 (+0x0000f080) 08003001 ADD_IMM.S64 X0, X3, #1 +0x000000000000f344 (+0x0000f084) 03c7e920 ST_XD_XN_IMM.B64 X3, X30, #2336 +0x000000000000f348 (+0x0000f088) 03802014 ST_XD_XN_IMM.B32 X0, X2, #20 +# [DWARF] common/pa_scheduler_core.h:790 +# 789 | context.self = &worker; +# > 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +0x000000000000f34c (+0x0000f08c) 1cc1e900 LD_XD_XN_IMM.B64 X0, X30, #2304 +0x000000000000f350 (+0x0000f090) 00c2308a AND.B64 X1, X3, X1 +0x000000000000f354 (+0x0000f094) 02c2020c SHL.B64 X1, #12 +0x000000000000f358 (+0x0000f098) 00000081 ADD.S64 X0, X0, X1 +# [DWARF] common/pa_scheduler_core.h:789 +# 783 | static_assert(offsetof(LazySampleCallbackTicket, won) == 14, "lazy sample callback ticket winner offset mismatch"); +# 784 | +# 785 | PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { +# 786 | // This is BeginSubmit without an already-materialized TaskArgs. The same +# 787 | // fields are completed synchronously after the single callback builds args. +# 788 | const uint32_t task_id = static_cast(worker.local_index++); +# > 789 | context.self = &worker; +0x000000000000f35c (+0x0000f09c) 09c53031 STP_XI_XJ_XN.B64 X2, X0, X19, #24 +# [DWARF] common/pa_scheduler_core.h:793 +# 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +# 791 | context.task_id = static_cast(task_id); +# 792 | context.tensor_count = 0; +# > 793 | context.scalar_count = 0; +0x000000000000f360 (+0x0000f0a0) 07000000 MOV_XD_IMM X0, #0 +0x000000000000f364 (+0x0000f0a4) 08033030 ADD_IMM.S64 X1, X19, #48 +# [DWARF] common/pa_scheduler_core.h:791 +# 785 | PA_DEVICE void BeginLazySampleCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { +# 786 | // This is BeginSubmit without an already-materialized TaskArgs. The same +# 787 | // fields are completed synchronously after the single callback builds args. +# 788 | const uint32_t task_id = static_cast(worker.local_index++); +# 789 | context.self = &worker; +# 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +# > 791 | context.task_id = static_cast(task_id); +0x000000000000f368 (+0x0000f0a8) 03873028 ST_XD_XN_IMM.B32 X3, X19, #40 +# [DWARF] common/pa_scheduler_core.h:792 +# > 792 | context.tensor_count = 0; +0x000000000000f36c (+0x0000f0ac) 0f813580 STI_XN_IMM.B32 X19, #44 +# [DWARF] common/pa_scheduler_core.h:794 +# 793 | context.scalar_count = 0; +# > 794 | context.result.task_id = task_id; +0x000000000000f370 (+0x0000f0b0) 03c73040 ST_XD_XN_IMM.B64 X3, X19, #64 +# [DWARF] common/pa_scheduler_core.h:795 +# > 795 | context.result.count = 0; +0x000000000000f374 (+0x0000f0b4) 0f813900 STI_XN_IMM.B32 X19, #72 +# [DWARF] common/pa_scheduler_core.h:793 +# 787 | // fields are completed synchronously after the single callback builds args. +# 788 | const uint32_t task_id = static_cast(worker.local_index++); +# 789 | context.self = &worker; +# 790 | context.payload = &worker.payloads[task_id & kPayloadMask]; +# 791 | context.task_id = static_cast(task_id); +# 792 | context.tensor_count = 0; +# > 793 | context.scalar_count = 0; +0x000000000000f378 (+0x0000f0b8) 09c01001 STP_XI_XJ_XN.B64 X0, X0, X1, #0 +# [DWARF] common/pa_scheduler_core.h:798 +# 794 | context.result.task_id = task_id; +# 795 | context.result.count = 0; +# 796 | context.register_mask = 0; +# 797 | context.output_bytes = 0; +# > 798 | context.fanin_count = 0; +0x000000000000f37c (+0x0000f0bc) 07000000 MOV_XD_IMM X0, #0 +0x000000000000f380 (+0x0000f0c0) 0781ffff MOVK X0, #65535, #2 +# [DWARF] common/pa_scheduler_core.h:800 +# 799 | context.kernel_id = -1; +# > 800 | context.won = false; +0x000000000000f384 (+0x0000f0c4) 0f473300 STI_XN_IMM.B16 X19, #408 +# [DWARF] common/pa_scheduler_core.h:798 +# 792 | context.tensor_count = 0; +# 793 | context.scalar_count = 0; +# 794 | context.result.task_id = task_id; +# 795 | context.result.count = 0; +# 796 | context.register_mask = 0; +# 797 | context.output_bytes = 0; +# > 798 | context.fanin_count = 0; +0x000000000000f388 (+0x0000f0c8) 07c1ffff MOVK X0, #65535, #3 +# [DWARF] common/pa_scheduler_core.h:802 +# 799 | context.kernel_id = -1; +# 800 | context.won = false; +# 801 | context.joint = false; +# > 802 | context.joint_init = false; +0x000000000000f38c (+0x0000f0cc) 0f073340 STI_XN_IMM.B8 X19, #410 +# [DWARF] common/pa_scheduler_core.h:798 +# 792 | context.tensor_count = 0; +# 793 | context.scalar_count = 0; +# 794 | context.result.task_id = task_id; +# 795 | context.result.count = 0; +# 796 | context.register_mask = 0; +# 797 | context.output_bytes = 0; +# > 798 | context.fanin_count = 0; +0x000000000000f390 (+0x0000f0d0) 03c13190 ST_XD_XN_IMM.B64 X0, X19, #400 +# [DWARF] common/pa_scheduler_core.h:804 +# 799 | context.kernel_id = -1; +# 800 | context.won = false; +# 801 | context.joint = false; +# 802 | context.joint_init = false; +# 803 | context.joint_block = -1; +# > 804 | context.joint_slot = -1; +0x000000000000f394 (+0x0000f0d4) 0700ffff MOV_XD_IMM X0, #65535 +0x000000000000f398 (+0x0000f0d8) 0741ffff MOVK X0, #65535, #1 +# [DWARF] common/pa_scheduler_core.h:803 +# 797 | context.output_bytes = 0; +# 798 | context.fanin_count = 0; +# 799 | context.kernel_id = -1; +# 800 | context.won = false; +# 801 | context.joint = false; +# 802 | context.joint_init = false; +# > 803 | context.joint_block = -1; +0x000000000000f39c (+0x0000f0dc) 0f873382 STI_XN_IMM.B32 X19, #412 +# [DWARF] common/pa_scheduler_core.h:804 +# > 804 | context.joint_slot = -1; +0x000000000000f3a0 (+0x0000f0e0) 03c131a0 ST_XD_XN_IMM.B64 X0, X19, #416 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x000000000000f3a4 (+0x0000f0e4) 02808880 MOV_XD_SPR.F32 X0, SYS_CNT +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x000000000000f3a8 (+0x0000f0e8) 1cd5e9b0 LD_XD_XN_IMM.B64 X10, X30, #2480 +0x000000000000f3ac (+0x0000f0ec) 1cd7e9a8 LD_XD_XN_IMM.B64 X11, X30, #2472 +0x000000000000f3b0 (+0x0000f0f0) 1cd9e9a0 LD_XD_XN_IMM.B64 X12, X30, #2464 +0x000000000000f3b4 (+0x0000f0f4) 1cd3e990 LD_XD_XN_IMM.B64 X9, X30, #2448 +# [DWARF] common/pa_trace.h:257 +# 251 | TraceContext &trace, uint64_t end_cycle +# 252 | ) { +# 253 | #if PA_BUILD_SUBMIT_PMU +# 254 | (void)trace; +# 255 | (void)end_cycle; +# 256 | #else +# > 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +0x000000000000f3b8 (+0x0000f0f8) 03c1e928 ST_XD_XN_IMM.B64 X0, X30, #2344 +0x000000000000f3bc (+0x0000f0fc) 1c01355c LD_XD_XN_IMM.B8 X0, X19, #1372 +0x000000000000f3c0 (+0x0000f100) 0000070e CMP.S64.EQ X0, X14 +0x000000000000f3c4 (+0x0000f104) 40200088 JUMPC #136 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f3c8 (+0x0000f108) 1ca535d0 LD_XD_XN_IMM.B32 X18, X19, #1488 +0x000000000000f3cc (+0x0000f10c) 02812a00 ZEROEXT.U32 X0, X18 +# [DWARF] common/pa_trace.h:257 +# 251 | TraceContext &trace, uint64_t end_cycle +# 252 | ) { +# 253 | #if PA_BUILD_SUBMIT_PMU +# 254 | (void)trace; +# 255 | (void)end_cycle; +# 256 | #else +# > 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +0x000000000000f3d0 (+0x0000f110) 0000070e CMP.S64.EQ X0, X14 +0x000000000000f3d4 (+0x0000f114) 40200084 JUMPC #132 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f3d8 (+0x0000f118) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x000000000000f3dc (+0x0000f11c) 00c0b78a AND.B64 X0, X11, X15 +0x000000000000f3e0 (+0x0000f120) 02c2020e SHL.B64 X1, #14 +0x000000000000f3e4 (+0x0000f124) 0004a481 ADD.S64 X2, X10, X9 +0x000000000000f3e8 (+0x0000f128) 00040084 MADD.S64 X2, X0, X1 +0x000000000000f3ec (+0x0000f12c) 07360001 MOV_XD_IMM X27, #1 +0x000000000000f3f0 (+0x0000f130) 00002081 ADD.S64 X0, X2, X1 +0x000000000000f3f4 (+0x0000f134) 08800680 SUB_IMM.S64 X0, X0, #1664 +0x000000000000f3f8 (+0x0000f138) 08000000 ADD_IMM.S64 X0, X0, #0 +0x000000000000f3fc (+0x0000f13c) 082a0548 ADD_IMM.S64 X21, X0, #1352 +0x000000000000f400 (+0x0000f140) 1cb40558 LD_XD_XN_IMM.B32 X26, X0, #1368 +0x000000000000f404 (+0x0000f144) 0ceb5c80 LDP_XI_XJ_XN.B64 X21, X25, X21, #0 +0x000000000000f408 (+0x0000f148) 0237b080 NEG.S64 X27, X27 +0x000000000000f40c (+0x0000f14c) 07280000 MOV_XD_IMM X20, #0 +0x000000000000f410 (+0x0000f150) 072e0000 MOV_XD_IMM X23, #0 +0x000000000000f414 (+0x0000f154) 07266784 MOV_XD_IMM X19, #26500 +0x000000000000f418 (+0x0000f158) 07670007 MOVK X19, #7, #1 +0x000000000000f41c (+0x0000f15c) 07a70000 MOVK X19, #0, #2 +0x000000000000f420 (+0x0000f160) 07e70000 MOVK X19, #0, #3 +0x000000000000f424 (+0x0000f164) 02040880 MOV_XD_SPR.S64 X2, PC +0x000000000000f428 (+0x0000f168) 00273101 ADD.S64 X19, X19, X2 +0x000000000000f42c (+0x0000f16c) 40000012 JUMP #18 +0x000000000000f430 (+0x0000f170) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x000000000000f434 (+0x0000f174) 00c2b78a AND.B64 X1, X11, X15 +0x000000000000f438 (+0x0000f178) 02c4020e SHL.B64 X2, #14 +0x000000000000f43c (+0x0000f17c) 0006a481 ADD.S64 X3, X10, X9 +0x000000000000f440 (+0x0000f180) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:280 +# 274 | call_count, static_cast(site) +# 275 | ); +# 276 | if (written) { +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# 278 | trace.atomic_counter_overflow = true; +# 279 | } else { +# > 280 | ++trace.poll_batch_records; +0x000000000000f444 (+0x0000f184) 08000001 ADD_IMM.S64 X0, X0, #1 +0x000000000000f448 (+0x0000f188) 00023101 ADD.S64 X1, X3, X2 +0x000000000000f44c (+0x0000f18c) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x000000000000f450 (+0x0000f190) 08021000 ADD_IMM.S64 X1, X1, #0 +0x000000000000f454 (+0x0000f194) 03c01578 ST_XD_XN_IMM.B64 X0, X1, #1400 +# [DWARF] common/pa_trace.h:283 +# 281 | } +# 282 | } +# > 283 | trace.poll_burst.call_count[index] = 0; +0x000000000000f458 (+0x0000f198) 0f976700 STI_XN_IMM.B32 X22, #1464 +# [DWARF] common/pa_trace.h:262 +# 256 | #else +# 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +# 258 | const uint32_t active_mask = trace.poll_burst.active_mask; +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# > 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +0x000000000000f45c (+0x0000f19c) 082f7001 ADD_IMM.S64 X23, X23, #1 +0x000000000000f460 (+0x0000f1a0) 08273004 ADD_IMM.S64 X19, X19, #4 +0x000000000000f464 (+0x0000f1a4) 0001741e CMP.S64.NE X23, X8 +0x000000000000f468 (+0x0000f1a8) 08294001 ADD_IMM.S64 X20, X20, #1 +0x000000000000f46c (+0x0000f1ac) 40200002 JUMPC #2 +0x000000000000f470 (+0x0000f1b0) 4000004e JUMP #78 +# [DWARF] common/pa_trace.h:263 +# > 263 | const uint32_t bit = 1U << index; +0x000000000000f474 (+0x0000f1b4) 02814a00 ZEROEXT.U32 X0, X20 +# [DWARF] common/pa_trace.h:264 +# > 264 | if ((active_mask & bit) == 0) continue; +0x000000000000f478 (+0x0000f1b8) 02832a00 ZEROEXT.U32 X1, X18 +0x000000000000f47c (+0x0000f1bc) 024202c0 SHR.U64 X1, X0, #0 +0x000000000000f480 (+0x0000f1c0) 00c0188a AND.B64 X0, X1, X17 +0x000000000000f484 (+0x0000f1c4) 0000070e CMP.S64.EQ X0, X14 +0x000000000000f488 (+0x0000f1c8) 4020fff5 JUMPC #65525 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f48c (+0x0000f1cc) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x000000000000f490 (+0x0000f1d0) 00c0b78a AND.B64 X0, X11, X15 +0x000000000000f494 (+0x0000f1d4) 02c4020e SHL.B64 X2, #14 +0x000000000000f498 (+0x0000f1d8) 0006a481 ADD.S64 X3, X10, X9 +0x000000000000f49c (+0x0000f1dc) 00060104 MADD.S64 X3, X0, X2 +# [DWARF] common/pa_trace.h:265 +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +# 263 | const uint32_t bit = 1U << index; +# 264 | if ((active_mask & bit) == 0) continue; +# > 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +0x000000000000f4a0 (+0x0000f1e0) 02037800 MOV_XD_XN.S64 X1, X23 +0x000000000000f4a4 (+0x0000f1e4) 00003101 ADD.S64 X0, X3, X2 +0x000000000000f4a8 (+0x0000f1e8) 08800680 SUB_IMM.S64 X0, X0, #1664 +0x000000000000f4ac (+0x0000f1ec) 02c20202 SHL.B64 X1, #2 +0x000000000000f4b0 (+0x0000f1f0) 08000000 ADD_IMM.S64 X0, X0, #0 +0x000000000000f4b4 (+0x0000f1f4) 002c0081 ADD.S64 X22, X0, X1 +0x000000000000f4b8 (+0x0000f1f8) 1c8b65b8 LD_XD_XN_IMM.B32 X5, X22, #1464 +# [DWARF] common/pa_trace.h:266 +# > 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +0x000000000000f4bc (+0x0000f1fc) 07020000 MOV_XD_IMM X1, #0 +0x000000000000f4c0 (+0x0000f200) 07430100 MOVK X1, #256, #1 +0x000000000000f4c4 (+0x0000f204) 07040000 MOV_XD_IMM X2, #0 +0x000000000000f4c8 (+0x0000f208) 0745ff00 MOVK X2, #65280, #1 +0x000000000000f4cc (+0x0000f20c) 00025082 SUB.S64 X1, X5, X1 +0x000000000000f4d0 (+0x0000f210) 02821a00 ZEROEXT.U32 X1, X1 +0x000000000000f4d4 (+0x0000f214) 0040113e CMP.U64.GT X1, X2 +0x000000000000f4d8 (+0x0000f218) 40200003 JUMPC #3 +# [DWARF] common/pa_trace.h:267 +# > 267 | trace.atomic_counter_overflow = true; +0x000000000000f4dc (+0x0000f21c) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x000000000000f4e0 (+0x0000f220) 4000ffdf JUMP #65503 +# [DWARF] common/pa_trace.h:98 +# 92 | default: +# 93 | return -1; +# 94 | } +# 95 | } +# 96 | +# 97 | PA_DEVICE AtomicSite TraceAtomicPollBatchSite(uint32_t index) { +# > 98 | switch (index) { +0x000000000000f4e4 (+0x0000f224) 02814a00 ZEROEXT.U32 X0, X20 +0x000000000000f4e8 (+0x0000f228) 07020005 MOV_XD_IMM X1, #5 +0x000000000000f4ec (+0x0000f22c) 004000be CMP.U64.GT X0, X1 +0x000000000000f4f0 (+0x0000f230) 070c000f MOV_XD_IMM X6, #15 +0x000000000000f4f4 (+0x0000f234) 40200002 JUMPC #2 +0x000000000000f4f8 (+0x0000f238) 1c8d3000 LD_XD_XN_IMM.B32 X6, X19, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f4fc (+0x0000f23c) 0804c002 ADD_IMM.S64 X2, X12, #2 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x000000000000f500 (+0x0000f240) 1cc9e928 LD_XD_XN_IMM.B64 X4, X30, #2344 +0x000000000000f504 (+0x0000f244) 00c2b78a AND.B64 X1, X11, X15 +0x000000000000f508 (+0x0000f248) 02c4020e SHL.B64 X2, #14 +0x000000000000f50c (+0x0000f24c) 0006a481 ADD.S64 X3, X10, X9 +0x000000000000f510 (+0x0000f250) 00061104 MADD.S64 X3, X1, X2 +# [DWARF] common/pa_trace.h:273 +# 272 | trace.core, trace.records, trace.capacity, +# > 273 | trace.poll_burst.start_cycle[index], end_cycle, +0x000000000000f514 (+0x0000f254) 02017800 MOV_XD_XN.S64 X0, X23 +0x000000000000f518 (+0x0000f258) 00023101 ADD.S64 X1, X3, X2 +0x000000000000f51c (+0x0000f25c) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x000000000000f520 (+0x0000f260) 02c00203 SHL.B64 X0, #3 +0x000000000000f524 (+0x0000f264) 08301000 ADD_IMM.S64 X24, X1, #0 +0x000000000000f528 (+0x0000f268) 00018001 ADD.S64 X0, X24, X0 +0x000000000000f52c (+0x0000f26c) 1cc60588 LD_XD_XN_IMM.B64 X3, X0, #1416 +# [DWARF] common/pa_trace.h:271 +# 265 | const uint32_t call_count = trace.poll_burst.call_count[index]; +# 266 | if (call_count == 0 || call_count > kAtomicPollCountMax) { +# 267 | trace.atomic_counter_overflow = true; +# 268 | continue; +# 269 | } +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# > 271 | const bool written = WritePollBatchRecordRaw( +0x000000000000f530 (+0x0000f270) 02015800 MOV_XD_XN.S64 X0, X21 +0x000000000000f534 (+0x0000f274) 02039800 MOV_XD_XN.S64 X1, X25 +0x000000000000f538 (+0x0000f278) 0205a800 MOV_XD_XN.S64 X2, X26 +0x000000000000f53c (+0x0000f27c) 070e615e MOV_XD_IMM X7, #24926 +0x000000000000f540 (+0x0000f280) 074f0000 MOVK X7, #0, #1 +0x000000000000f544 (+0x0000f284) 078f0000 MOVK X7, #0, #2 +0x000000000000f548 (+0x0000f288) 40427000 CALL X7, #0 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f54c (+0x0000f28c) 1cd3e990 LD_XD_XN_IMM.B64 X9, X30, #2448 +0x000000000000f550 (+0x0000f290) 071c0000 MOV_XD_IMM X14, #0 +0x000000000000f554 (+0x0000f294) 1cd9e9a0 LD_XD_XN_IMM.B64 X12, X30, #2464 +0x000000000000f558 (+0x0000f298) 071e7fff MOV_XD_IMM X15, #32767 +0x000000000000f55c (+0x0000f29c) 1cd7e9a8 LD_XD_XN_IMM.B64 X11, X30, #2472 +# [DWARF] common/pa_trace.h:276 +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x000000000000f560 (+0x0000f2a0) 0000070e CMP.S64.EQ X0, X14 +0x000000000000f564 (+0x0000f2a4) 1cd5e9b0 LD_XD_XN_IMM.B64 X10, X30, #2480 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f568 (+0x0000f2a8) 07220001 MOV_XD_IMM X17, #1 +0x000000000000f56c (+0x0000f2ac) 07100006 MOV_XD_IMM X8, #6 +# [DWARF] common/pa_trace.h:276 +# 270 | const AtomicSite site = TraceAtomicPollBatchSite(index); +# 271 | const bool written = WritePollBatchRecordRaw( +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# > 276 | if (written) { +0x000000000000f570 (+0x0000f2b0) 4020ffba JUMPC #65466 +# [DWARF] common/pa_trace.h:277 +# > 277 | if (trace.poll_batch_records == UINT64_MAX) { +0x000000000000f574 (+0x0000f2b4) 1cc18578 LD_XD_XN_IMM.B64 X0, X24, #1400 +0x000000000000f578 (+0x0000f2b8) 00000d9e CMP.S64.NE X0, X27 +0x000000000000f57c (+0x0000f2bc) 4020ffad JUMPC #65453 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f580 (+0x0000f2c0) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x000000000000f584 (+0x0000f2c4) 00c0b78a AND.B64 X0, X11, X15 +0x000000000000f588 (+0x0000f2c8) 02c2020e SHL.B64 X1, #14 +0x000000000000f58c (+0x0000f2cc) 0004a481 ADD.S64 X2, X10, X9 +0x000000000000f590 (+0x0000f2d0) 00040084 MADD.S64 X2, X0, X1 +0x000000000000f594 (+0x0000f2d4) 00002081 ADD.S64 X0, X2, X1 +0x000000000000f598 (+0x0000f2d8) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:278 +# 272 | trace.core, trace.records, trace.capacity, +# 273 | trace.poll_burst.start_cycle[index], end_cycle, +# 274 | call_count, static_cast(site) +# 275 | ); +# 276 | if (written) { +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# > 278 | trace.atomic_counter_overflow = true; +0x000000000000f59c (+0x0000f2dc) 08000000 ADD_IMM.S64 X0, X0, #0 +0x000000000000f5a0 (+0x0000f2e0) 0f160001 STI_XN_IMM.B8 X0, #1408 +0x000000000000f5a4 (+0x0000f2e4) 4000ffad JUMP #65453 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f5a8 (+0x0000f2e8) 1cf3e980 LD_XD_XN_IMM.B64 X25, X30, #2432 +0x000000000000f5ac (+0x0000f2ec) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x000000000000f5b0 (+0x0000f2f0) 00c0b78a AND.B64 X0, X11, X15 +0x000000000000f5b4 (+0x0000f2f4) 02c2020e SHL.B64 X1, #14 +0x000000000000f5b8 (+0x0000f2f8) 0004a481 ADD.S64 X2, X10, X9 +0x000000000000f5bc (+0x0000f2fc) 00040084 MADD.S64 X2, X0, X1 +0x000000000000f5c0 (+0x0000f300) 07280001 MOV_XD_IMM X20, #1 +0x000000000000f5c4 (+0x0000f304) 00002081 ADD.S64 X0, X2, X1 +0x000000000000f5c8 (+0x0000f308) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:285 +# 279 | } else { +# 280 | ++trace.poll_batch_records; +# 281 | } +# 282 | } +# 283 | trace.poll_burst.call_count[index] = 0; +# 284 | } +# > 285 | trace.poll_burst.active_mask = 0; +0x000000000000f5cc (+0x0000f30c) 08000000 ADD_IMM.S64 X0, X0, #0 +0x000000000000f5d0 (+0x0000f310) 02294080 NEG.S64 X20, X20 +0x000000000000f5d4 (+0x0000f314) 0f960a00 STI_XN_IMM.B32 X0, #1488 +0x000000000000f5d8 (+0x0000f318) 072a0004 MOV_XD_IMM X21, #4 +0x000000000000f5dc (+0x0000f31c) 072c12c4 MOV_XD_IMM X22, #4804 +0x000000000000f5e0 (+0x0000f320) 07300020 MOV_XD_IMM X24, #32 +# [DWARF] common/pa_scheduler_core.h:1128 +# 1122 | const uint32_t task_id = static_cast(context.task_id); +# 1123 | #if PA_BUILD_SUBMIT_PMU +# 1124 | const uint64_t submit_begin = task_id == 0 ? Ops::Now() : 0; +# 1125 | #else +# 1126 | const uint64_t submit_begin = TraceTimestamp(stats.trace, stats.result); +# 1127 | #endif +# > 1128 | if (task_id == 0) stats.result.submit_begin = submit_begin; +0x000000000000f5e4 (+0x0000f324) 1cc1e920 LD_XD_XN_IMM.B64 X0, X30, #2336 +0x000000000000f5e8 (+0x0000f328) 0000071e CMP.S64.NE X0, X14 +0x000000000000f5ec (+0x0000f32c) 4020000b JUMPC #11 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x000000000000f5f0 (+0x0000f330) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x000000000000f5f4 (+0x0000f334) 00c0b78a AND.B64 X0, X11, X15 +0x000000000000f5f8 (+0x0000f338) 02c2020e SHL.B64 X1, #14 +0x000000000000f5fc (+0x0000f33c) 0004a481 ADD.S64 X2, X10, X9 +0x000000000000f600 (+0x0000f340) 00040084 MADD.S64 X2, X0, X1 +0x000000000000f604 (+0x0000f344) 00002081 ADD.S64 X0, X2, X1 +# [DWARF] common/pa_scheduler_core.h:1128 +# 1122 | const uint32_t task_id = static_cast(context.task_id); +# 1123 | #if PA_BUILD_SUBMIT_PMU +# 1124 | const uint64_t submit_begin = task_id == 0 ? Ops::Now() : 0; +# 1125 | #else +# 1126 | const uint64_t submit_begin = TraceTimestamp(stats.trace, stats.result); +# 1127 | #endif +# > 1128 | if (task_id == 0) stats.result.submit_begin = submit_begin; +0x000000000000f608 (+0x0000f348) 1cc3e928 LD_XD_XN_IMM.B64 X1, X30, #2344 +0x000000000000f60c (+0x0000f34c) 08800680 SUB_IMM.S64 X0, X0, #1664 +0x000000000000f610 (+0x0000f350) 08000000 ADD_IMM.S64 X0, X0, #0 +0x000000000000f614 (+0x0000f354) 03c201c0 ST_XD_XN_IMM.B64 X1, X0, #448 +# [DWARF] common/pa_scheduler_core.h:278 +# 272 | template +# 273 | PA_DEVICE uint32_t DrainReady( +# 274 | PA_GM SchedulerState *state, PA_GM WorkerState &worker, DrainPlace place, LocalStats &stats +# 275 | ) { +# 276 | // 同一套 drain 被三个位置复用：每次 Submit 开头的 EfDrain、ring 背压等待和所有 Submit 后的最终 drain。 +# 277 | // slot 属于当前 worker；只有其全部跨核 fanin 已 ready 时才执行所选 winner 负载、发布完成并释放 slot。 +# > 278 | if (worker.occupied_count == 0) { +0x000000000000f618 (+0x0000f358) 1cc1e960 LD_XD_XN_IMM.B64 X0, X30, #2400 +0x000000000000f61c (+0x0000f35c) 0702dba0 MOV_XD_IMM X1, #56224 +0x000000000000f620 (+0x0000f360) 0743000c MOVK X1, #12, #1 +0x000000000000f624 (+0x0000f364) 1cc5e940 LD_XD_XN_IMM.B64 X2, X30, #2368 +0x000000000000f628 (+0x0000f368) 00000081 ADD.S64 X0, X0, X1 +0x000000000000f62c (+0x0000f36c) 1c800000 LD_XD_XN_IMM.B32 X0, X0, #0 +0x000000000000f630 (+0x0000f370) 0000070e CMP.S64.EQ X0, X14 +0x000000000000f634 (+0x0000f374) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000000f638 (+0x0000f378) 07001277 MOV_XD_IMM X0, #4727 +0x000000000000f63c (+0x0000f37c) 07410000 MOVK X0, #0, #1 +0x000000000000f640 (+0x0000f380) 07810000 MOVK X0, #0, #2 +0x000000000000f644 (+0x0000f384) 40220000 JUMPC X0, #0 +0x000000000000f648 (+0x0000f388) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x000000000000f64c (+0x0000f38c) 07020000 MOV_XD_IMM X1, #0 +0x000000000000f650 (+0x0000f390) 4000003d JUMP #61 +0x000000000000f654 (+0x0000f394) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000000f658 (+0x0000f398) 080ac002 ADD_IMM.S64 X5, X12, #2 +# [DWARF] common/pa_trace.h:552 +# 546 | PA_GM TraceCoreState &core = *trace.core; +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# > 552 | PA_GM TraceRecord &record = trace.records[slot]; +0x000000000000f65c (+0x0000f39c) 02060800 MOV_XD_XN.S64 X3, X0 +0x000000000000f660 (+0x0000f3a0) 00c8b78a AND.B64 X4, X11, X15 +0x000000000000f664 (+0x0000f3a4) 02ca020e SHL.B64 X5, #14 +0x000000000000f668 (+0x0000f3a8) 000ca481 ADD.S64 X6, X10, X9 +0x000000000000f66c (+0x0000f3ac) 02c60206 SHL.B64 X3, #6 +0x000000000000f670 (+0x0000f3b0) 000c4284 MADD.S64 X6, X4, X5 +0x000000000000f674 (+0x0000f3b4) 0007b181 ADD.S64 X3, X27, X3 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f678 (+0x0000f3b8) 00086281 ADD.S64 X4, X6, X5 +# [DWARF] common/pa_trace.h:553 +# 547 | const uint32_t slot = core.count; +# 548 | if (slot >= trace.capacity) { +# 549 | core.dropped = core.dropped + 1; +# 550 | return; +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# > 553 | record.start_cycle = start_cycle; +0x000000000000f67c (+0x0000f3bc) 09ea3a81 STP_XI_XJ_XN.B64 X21, X21, X3, #0 +# [DWARF] common/pa_trace.h:555 +# 554 | record.end_cycle = end_cycle; +# > 555 | record.task_id = task_id; +0x000000000000f680 (+0x0000f3c0) 098430a1 STP_XI_XJ_XN.B32 X2, X1, X3, #16 +0x000000000000f684 (+0x0000f3c4) 08844680 SUB_IMM.S64 X2, X4, #1664 +# [DWARF] common/pa_trace.h:558 +# 556 | record.function_id = function_id; +# 557 | record.phase = static_cast(trace_phase); +# > 558 | record.lane = trace.lane; +0x000000000000f688 (+0x0000f3c8) 08022000 ADD_IMM.S64 X1, X2, #0 +0x000000000000f68c (+0x0000f3cc) 1c841560 LD_XD_XN_IMM.B32 X2, X1, #1376 +# [DWARF] common/pa_trace.h:557 +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x000000000000f690 (+0x0000f3d0) 07080007 MOV_XD_IMM X4, #7 +# [DWARF] common/pa_trace.h:565 +# 558 | record.lane = trace.lane; +# 559 | record.block_id = trace.block_id; +# 560 | record.core_idx = trace.core_idx; +# 561 | record.flags = flags; +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x000000000000f694 (+0x0000f3d4) 08000001 ADD_IMM.S64 X0, X0, #1 +# [DWARF] common/pa_trace.h:557 +# 551 | } +# 552 | PA_GM TraceRecord &record = trace.records[slot]; +# 553 | record.start_cycle = start_cycle; +# 554 | record.end_cycle = end_cycle; +# 555 | record.task_id = task_id; +# 556 | record.function_id = function_id; +# > 557 | record.phase = static_cast(trace_phase); +0x000000000000f698 (+0x0000f3d8) 09883131 STP_XI_XJ_XN.B32 X4, X2, X3, #24 +# [DWARF] common/pa_trace.h:559 +# 558 | record.lane = trace.lane; +# > 559 | record.block_id = trace.block_id; +0x000000000000f69c (+0x0000f3dc) 08041564 ADD_IMM.S64 X2, X1, #1380 +0x000000000000f6a0 (+0x0000f3e0) 0c842080 LDP_XI_XJ_XN.B32 X2, X1, X2, #0 +0x000000000000f6a4 (+0x0000f3e4) 08083020 ADD_IMM.S64 X4, X3, #32 +0x000000000000f6a8 (+0x0000f3e8) 09844081 STP_XI_XJ_XN.B32 X2, X1, X4, #0 +# [DWARF] common/pa_trace.h:561 +# 560 | record.core_idx = trace.core_idx; +# > 561 | record.flags = flags; +0x000000000000f6ac (+0x0000f3ec) 03dc3028 ST_XD_XN_IMM.B64 X14, X3, #40 +# [DWARF] common/pa_trace.h:565 +# 562 | record.auxiliary = auxiliary; +# 563 | // count 最后更新，使其始终指向下一空槽；单写者条件下无需 reserve/commit 两阶段。 +# 564 | // 最终 FlushTraceCore 会把记录体先于该计数一并导出。 +# > 565 | core.count = slot + 1; +0x000000000000f6b0 (+0x0000f3f0) 0381a000 ST_XD_XN_IMM.B32 X0, X26, #0 +0x000000000000f6b4 (+0x0000f3f4) 40000002 JUMP #2 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f6b8 (+0x0000f3f8) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_scheduler_core.h:305 +# 299 | WriteTrace( +# 300 | stats.trace, stats.result, static_cast(slot.task_id), static_cast(slot.kind), +# 301 | TracePhase::Commit, ProfilePhase::ReplayTail, commit_cycle, commit_cycle +# 302 | ); +# 303 | slot.built = false; +# 304 | slot.occupied = false; +# > 305 | --worker.occupied_count; +0x000000000000f6bc (+0x0000f3fc) 1cc1e960 LD_XD_XN_IMM.B64 X0, X30, #2400 +0x000000000000f6c0 (+0x0000f400) 0702dba0 MOV_XD_IMM X1, #56224 +# [DWARF] common/pa_scheduler_core.h:304 +# 298 | const uint64_t commit_cycle = TraceTimestamp(stats.trace, stats.result); +# 299 | WriteTrace( +# 300 | stats.trace, stats.result, static_cast(slot.task_id), static_cast(slot.kind), +# 301 | TracePhase::Commit, ProfilePhase::ReplayTail, commit_cycle, commit_cycle +# 302 | ); +# 303 | slot.built = false; +# > 304 | slot.occupied = false; +0x000000000000f6c4 (+0x0000f404) 035c7000 ST_XD_XN_IMM.B16 X14, X7, #0 +# [DWARF] common/pa_scheduler_core.h:305 +# > 305 | --worker.occupied_count; +0x000000000000f6c8 (+0x0000f408) 0743000c MOVK X1, #12, #1 +0x000000000000f6cc (+0x0000f40c) 0806c002 ADD_IMM.S64 X3, X12, #2 +0x000000000000f6d0 (+0x0000f410) 00c4b78a AND.B64 X2, X11, X15 +0x000000000000f6d4 (+0x0000f414) 02c6020e SHL.B64 X3, #14 +0x000000000000f6d8 (+0x0000f418) 0008a481 ADD.S64 X4, X10, X9 +0x000000000000f6dc (+0x0000f41c) 00082184 MADD.S64 X4, X2, X3 +0x000000000000f6e0 (+0x0000f420) 072a0004 MOV_XD_IMM X21, #4 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x000000000000f6e4 (+0x0000f424) 00044181 ADD.S64 X2, X4, X3 +0x000000000000f6e8 (+0x0000f428) 08842680 SUB_IMM.S64 X2, X2, #1664 +0x000000000000f6ec (+0x0000f42c) 072c12c4 MOV_XD_IMM X22, #4804 +0x000000000000f6f0 (+0x0000f430) 07300020 MOV_XD_IMM X24, #32 +# [DWARF] common/pa_scheduler_core.h:305 +# 299 | WriteTrace( +# 300 | stats.trace, stats.result, static_cast(slot.task_id), static_cast(slot.kind), +# 301 | TracePhase::Commit, ProfilePhase::ReplayTail, commit_cycle, commit_cycle +# 302 | ); +# 303 | slot.built = false; +# 304 | slot.occupied = false; +# > 305 | --worker.occupied_count; +0x000000000000f6f4 (+0x0000f434) 00000081 ADD.S64 X0, X0, X1 +0x000000000000f6f8 (+0x0000f438) 1c820000 LD_XD_XN_IMM.B32 X1, X0, #0 +0x000000000000f6fc (+0x0000f43c) 08821001 SUB_IMM.S64 X1, X1, #1 +0x000000000000f700 (+0x0000f440) 03820000 ST_XD_XN_IMM.B32 X1, X0, #0 +# [DWARF] common/pa_scheduler_core.h:306 +# > 306 | ++stats.result.placement[static_cast(place)]; +0x000000000000f704 (+0x0000f444) 08002000 ADD_IMM.S64 X0, X2, #0 +0x000000000000f708 (+0x0000f448) 1cc5e940 LD_XD_XN_IMM.B64 X2, X30, #2368 +0x000000000000f70c (+0x0000f44c) 1cc202c8 LD_XD_XN_IMM.B64 X1, X0, #712 +0x000000000000f710 (+0x0000f450) 08021001 ADD_IMM.S64 X1, X1, #1 +0x000000000000f714 (+0x0000f454) 03c202c8 ST_XD_XN_IMM.B64 X1, X0, #712 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x000000000000f718 (+0x0000f458) 40000002 JUMP #2 +0x000000000000f71c (+0x0000f45c) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000000f720 (+0x0000f460) 1cc3e978 LD_XD_XN_IMM.B64 X1, X30, #2424 +# [DWARF] common/pa_scheduler_core.h:283 +# 277 | // slot 属于当前 worker；只有其全部跨核 fanin 已 ready 时才执行所选 winner 负载、发布完成并释放 slot。 +# 278 | if (worker.occupied_count == 0) { +# 279 | return 0; +# 280 | } +# 281 | uint32_t freed = 0; +# 282 | // 一次调用遍历完本核全部私有 slot；未就绪项保留 occupied/built，已完成项立即清槽并继续扫描。 +# > 283 | for (uint32_t index = 0; index < kPrivateSlots; ++index) { +0x000000000000f724 (+0x0000f464) 08021001 ADD_IMM.S64 X1, X1, #1 +0x000000000000f728 (+0x0000f468) 00001a9e CMP.S64.NE X1, X21 +0x000000000000f72c (+0x0000f46c) 40200006 JUMPC #6 +0x000000000000f730 (+0x0000f470) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000000f734 (+0x0000f474) 07001238 MOV_XD_IMM X0, #4664 +0x000000000000f738 (+0x0000f478) 07410000 MOVK X0, #0, #1 +0x000000000000f73c (+0x0000f47c) 07810000 MOVK X0, #0, #2 +0x000000000000f740 (+0x0000f480) 40020000 JUMP X0, #0 +# [DWARF] common/pa_scheduler_core.h:284 +# > 284 | PA_GM LocalSlot &slot = worker.slots[index]; +0x000000000000f744 (+0x0000f484) 070012d8 MOV_XD_IMM X0, #4824 +0x000000000000f748 (+0x0000f488) 03c3e978 ST_XD_XN_IMM.B64 X1, X30, #2424 +0x000000000000f74c (+0x0000f48c) 00001003 MUL.S64 X0, X1, X0 +# [DWARF] common/pa_scheduler_core.h:285 +# > 285 | if (!slot.occupied || !slot.built || !SlotReady(state, slot, stats)) { +0x000000000000f750 (+0x0000f490) 01022000 LD_XD_XN.B8 X1, X2, X0 +0x000000000000f754 (+0x0000f494) 0000170e CMP.S64.EQ X1, X14 +0x000000000000f758 (+0x0000f498) 4020fff2 JUMPC #65522 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x000000000000f75c (+0x0000f49c) 00242001 ADD.S64 X18, X2, X0 +# [DWARF] common/pa_scheduler_core.h:285 +# 279 | return 0; +# 280 | } +# 281 | uint32_t freed = 0; +# 282 | // 一次调用遍历完本核全部私有 slot；未就绪项保留 occupied/built，已完成项立即清槽并继续扫描。 +# 283 | for (uint32_t index = 0; index < kPrivateSlots; ++index) { +# 284 | PA_GM LocalSlot &slot = worker.slots[index]; +# > 285 | if (!slot.occupied || !slot.built || !SlotReady(state, slot, stats)) { +0x000000000000f760 (+0x0000f4a0) 1c012001 LD_XD_XN_IMM.B8 X0, X18, #1 +0x000000000000f764 (+0x0000f4a4) 0000070e CMP.S64.EQ X0, X14 +0x000000000000f768 (+0x0000f4a8) 4020ffee JUMPC #65518 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x000000000000f76c (+0x0000f4ac) 00012b01 ADD.S64 X0, X18, X22 +0x000000000000f770 (+0x0000f4b0) 03e5e968 ST_XD_XN_IMM.B64 X18, X30, #2408 +# [DWARF] common/pa_scheduler_core.h:246 +# 240 | AdvanceFrontier(state, stats); +# 241 | } +# 242 | +# 243 | template +# 244 | PA_DEVICE bool SlotReady(PA_GM SchedulerState *state, PA_GM LocalSlot &slot, LocalStats &stats) { +# 245 | // 每个 fanin flag 都是跨核共享的完成条件；遇到第一个未就绪依赖即返回，后续 drain 会再次轮询。 +# > 246 | for (uint32_t index = 0; index < slot.fanin_count; ++index) { +0x000000000000f774 (+0x0000f4b4) 1c800000 LD_XD_XN_IMM.B32 X0, X0, #0 +0x000000000000f778 (+0x0000f4b8) 0000070e CMP.S64.EQ X0, X14 +0x000000000000f77c (+0x0000f4bc) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000000f780 (+0x0000f4c0) 0700048d MOV_XD_IMM X0, #1165 +0x000000000000f784 (+0x0000f4c4) 07410000 MOVK X0, #0, #1 +0x000000000000f788 (+0x0000f4c8) 07810000 MOVK X0, #0, #2 +0x000000000000f78c (+0x0000f4cc) 40220000 JUMPC X0, #0 +0x000000000000f790 (+0x0000f4d0) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x000000000000f794 (+0x0000f4d4) 07001284 MOV_XD_IMM X0, #4740 +0x000000000000f798 (+0x0000f4d8) 001b2001 ADD.S64 X13, X18, X0 +0x000000000000f79c (+0x0000f4dc) 072e0000 MOV_XD_IMM X23, #0 +0x000000000000f7a0 (+0x0000f4e0) 03dbe930 ST_XD_XN_IMM.B64 X13, X30, #2352 +0x000000000000f7a4 (+0x0000f4e4) 40000002 JUMP #2 +0x000000000000f7a8 (+0x0000f4e8) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000000f7ac (+0x0000f4ec) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x000000000000f7b0 (+0x0000f4f0) 00c0b78a AND.B64 X0, X11, X15 +0x000000000000f7b4 (+0x0000f4f4) 02c2020e SHL.B64 X1, #14 +0x000000000000f7b8 (+0x0000f4f8) 0004a481 ADD.S64 X2, X10, X9 +0x000000000000f7bc (+0x0000f4fc) 00040084 MADD.S64 X2, X0, X1 +# [DWARF] common/pa_scheduler_core.h:247 +# 241 | } +# 242 | +# 243 | template +# 244 | PA_DEVICE bool SlotReady(PA_GM SchedulerState *state, PA_GM LocalSlot &slot, LocalStats &stats) { +# 245 | // 每个 fanin flag 都是跨核共享的完成条件；遇到第一个未就绪依赖即返回，后续 drain 会再次轮询。 +# 246 | for (uint32_t index = 0; index < slot.fanin_count; ++index) { +# > 247 | const int32_t dependency = slot.fanin[index]; +0x000000000000f7c0 (+0x0000f500) 0180db80 LD_XD_XN.B32 X0, X13, X23 +# [DWARF] common/pa_scheduler_core.h:0 +# [SOURCE unavailable] +0x000000000000f7c4 (+0x0000f504) 00022081 ADD.S64 X1, X2, X1 +0x000000000000f7c8 (+0x0000f508) 08821680 SUB_IMM.S64 X1, X1, #1664 +# [DWARF] common/pa_trace.h:393 +# 387 | (void)result; +# 388 | (void)task_id; +# 389 | (void)site; +# 390 | (void)result_used; +# 391 | return Ops::Load(address); +# 392 | #else +# > 393 | if (!trace.atomics_enabled) return Ops::Load(address); +0x000000000000f7cc (+0x0000f50c) 08021000 ADD_IMM.S64 X1, X1, #0 +0x000000000000f7d0 (+0x0000f510) 1c04155c LD_XD_XN_IMM.B8 X2, X1, #1372 +# [DWARF] common/pa_scheduler_core.h:250 +# 244 | PA_DEVICE bool SlotReady(PA_GM SchedulerState *state, PA_GM LocalSlot &slot, LocalStats &stats) { +# 245 | // 每个 fanin flag 都是跨核共享的完成条件；遇到第一个未就绪依赖即返回，后续 drain 会再次轮询。 +# 246 | for (uint32_t index = 0; index < slot.fanin_count; ++index) { +# 247 | const int32_t dependency = slot.fanin[index]; +# 248 | if (TraceAtomicLoad( +# 249 | stats.trace, stats.result, dependency, AtomicSite::FaninFlagLoad, +# > 250 | &state->tasks[dependency].flag +0x000000000000f7d4 (+0x0000f514) 02860980 SIGNEXT.S32 X3, X0 +0x000000000000f7d8 (+0x0000f518) 02c60206 SHL.B64 X3, #6 +# [DWARF] common/pa_trace.h:393 +# 387 | (void)result; +# 388 | (void)task_id; +# 389 | (void)site; +# 390 | (void)result_used; +# 391 | return Ops::Load(address); +# 392 | #else +# > 393 | if (!trace.atomics_enabled) return Ops::Load(address); +0x000000000000f7dc (+0x0000f51c) 0000271e CMP.S64.NE X2, X14 +# [DWARF] common/pa_scheduler_core.h:250 +# 244 | PA_DEVICE bool SlotReady(PA_GM SchedulerState *state, PA_GM LocalSlot &slot, LocalStats &stats) { +# 245 | // 每个 fanin flag 都是跨核共享的完成条件；遇到第一个未就绪依赖即返回，后续 drain 会再次轮询。 +# 246 | for (uint32_t index = 0; index < slot.fanin_count; ++index) { +# 247 | const int32_t dependency = slot.fanin[index]; +# 248 | if (TraceAtomicLoad( +# 249 | stats.trace, stats.result, dependency, AtomicSite::FaninFlagLoad, +# > 250 | &state->tasks[dependency].flag +0x000000000000f7e0 (+0x0000f520) 00059181 ADD.S64 X2, X25, X3 +# [DWARF] common/pa_trace.h:393 +# 387 | (void)result; +# 388 | (void)task_id; +# 389 | (void)site; +# 390 | (void)result_used; +# 391 | return Ops::Load(address); +# 392 | #else +# > 393 | if (!trace.atomics_enabled) return Ops::Load(address); +0x000000000000f7e4 (+0x0000f524) 40200006 JUMPC #6 +0x000000000000f7e8 (+0x0000f528) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000000f7ec (+0x0000f52c) 0700001e MOV_XD_IMM X0, #30 +0x000000000000f7f0 (+0x0000f530) 07410000 MOVK X0, #0, #1 +0x000000000000f7f4 (+0x0000f534) 07810000 MOVK X0, #0, #2 +0x000000000000f7f8 (+0x0000f538) 40020000 JUMP X0, #0 +# [DWARF] common/pa_trace.h:337 +# 331 | +# 332 | PA_DEVICE bool AtomicPollBatchEnabled( +# 333 | TraceContext &trace, AtomicSite site, AtomicOp actual_op +# 334 | ) { +# 335 | return trace.atomics_enabled && TraceAtomicSiteIsPollBatchable(site) && +# 336 | TraceAtomicSiteExpectedOp(site) == actual_op && +# > 337 | (trace.poll_burst.enabled_mask & TraceAtomicSiteMask(site)) != 0; +0x000000000000f7fc (+0x0000f53c) 1c0215d4 LD_XD_XN_IMM.B8 X1, X1, #1492 +0x000000000000f800 (+0x0000f540) 00c21c0a AND.B64 X1, X1, X24 +0x000000000000f804 (+0x0000f544) 0000170e CMP.S64.EQ X1, X14 +# [DWARF] common/pa_trace.h:395 +# 389 | (void)site; +# 390 | (void)result_used; +# 391 | return Ops::Load(address); +# 392 | #else +# 393 | if (!trace.atomics_enabled) return Ops::Load(address); +# 394 | const bool poll_batch = result_used && AtomicPollBatchEnabled(trace, site, AtomicOp::Load); +# > 395 | const int32_t poll_index = poll_batch ? TraceAtomicPollBatchIndex(site) : -1; +0x000000000000f808 (+0x0000f548) 40200006 JUMPC #6 +0x000000000000f80c (+0x0000f54c) 03c1e870 ST_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000000f810 (+0x0000f550) 07000018 MOV_XD_IMM X0, #24 +0x000000000000f814 (+0x0000f554) 07410000 MOVK X0, #0, #1 +0x000000000000f818 (+0x0000f558) 07810000 MOVK X0, #0, #2 +0x000000000000f81c (+0x0000f55c) 40020000 JUMP X0, #0 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x000000000000f820 (+0x0000f560) 02828880 MOV_XD_SPR.F32 X1, SYS_CNT +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:86 +# 80 | return __builtin_cce_atom_max_G_##SUFFIX(base, inc, (uint32_t)L2Cache); \ +# 81 | } +# 82 | +# 83 | __CCE__ATOM_G_BUILTIN(uint32_t, u32) +# 84 | __CCE__ATOM_G_BUILTIN(int32_t, s32) +# 85 | __CCE__ATOM_G_BUILTIN(uint64_t, u64) +# > 86 | __CCE__ATOM_G_BUILTIN(int64_t, s64) +0x000000000000f824 (+0x0000f564) 51a809c0 ATOM XN, XM, XD, ADD +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:0 +# [SOURCE unavailable] +0x000000000000f828 (+0x0000f568) 0806c002 ADD_IMM.S64 X3, X12, #2 +0x000000000000f82c (+0x0000f56c) 00c4b78a AND.B64 X2, X11, X15 +0x000000000000f830 (+0x0000f570) 02c6020e SHL.B64 X3, #14 +0x000000000000f834 (+0x0000f574) 0008a481 ADD.S64 X4, X10, X9 +0x000000000000f838 (+0x0000f578) 00082184 MADD.S64 X4, X2, X3 +0x000000000000f83c (+0x0000f57c) 00044181 ADD.S64 X2, X4, X3 +0x000000000000f840 (+0x0000f580) 08842680 SUB_IMM.S64 X2, X2, #1664 +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x000000000000f844 (+0x0000f584) 08042000 ADD_IMM.S64 X2, X2, #0 +# [DWARF] ccec/ccec_ops.h:239 +# 233 | static_assert(sizeof(T) == 4 || sizeof(T) == 8, "atomic dependency expects a scalar result"); +# 234 | uint64_t cycle = 0; +# 235 | // 同一个 inline asm 块先真正消费 atomic 返回寄存器，再读取 +# 236 | // SYS_CNT；编译器不能把 t1 拆到依赖 MOV 之前。AIC/AIV 对该序列 +# 237 | // 生成相同指令字节，且不增加 DSB/ISB/GM 访存。该边界仍只表示 +# 238 | // 返回值已可被本核 scalar 消费，不表示跨核全局可见。 +# > 239 | asm volatile( +0x000000000000f848 (+0x0000f588) 020b0800 MOV_XD_XN.S64 X5, X16 +0x000000000000f84c (+0x0000f58c) 020a5800 MOV_XD_XN.S64 X5, X5 +0x000000000000f850 (+0x0000f590) 02868880 MOV_XD_SPR.F32 X3, SYS_CNT +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x000000000000f854 (+0x0000f594) 1cc824a0 LD_XD_XN_IMM.B64 X4, X2, #1184 +0x000000000000f858 (+0x0000f598) 00004a1e CMP.S64.NE X4, X20 +0x000000000000f85c (+0x0000f59c) 40200002 JUMPC #2 +0x000000000000f860 (+0x0000f5a0) 400000d5 JUMP #213 +# [DWARF] common/pa_trace.h:240 +# 237 | trace.atomic_counter_overflow = true; +# 238 | return; +# 239 | } +# > 240 | ++result.atomic_trace_calls; +0x000000000000f864 (+0x0000f5a4) 08084001 ADD_IMM.S64 X4, X4, #1 +0x000000000000f868 (+0x0000f5a8) 03c824a0 ST_XD_XN_IMM.B64 X4, X2, #1184 +0x000000000000f86c (+0x0000f5ac) 400000d3 JUMP #211 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f870 (+0x0000f5b0) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:86 +# 80 | return __builtin_cce_atom_max_G_##SUFFIX(base, inc, (uint32_t)L2Cache); \ +# 81 | } +# 82 | +# 83 | __CCE__ATOM_G_BUILTIN(uint32_t, u32) +# 84 | __CCE__ATOM_G_BUILTIN(int32_t, s32) +# 85 | __CCE__ATOM_G_BUILTIN(uint64_t, u64) +# > 86 | __CCE__ATOM_G_BUILTIN(int64_t, s64) +0x000000000000f874 (+0x0000f5b4) 51a809c0 ATOM XN, XM, XD, ADD +0x000000000000f878 (+0x0000f5b8) 40000436 JUMP #1078 +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:0 +# [SOURCE unavailable] +0x000000000000f87c (+0x0000f5bc) 1cc1e870 LD_XD_XN_IMM.B64 X0, X30, #2160 +0x000000000000f880 (+0x0000f5c0) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x000000000000f884 (+0x0000f5c4) 00c0b78a AND.B64 X0, X11, X15 +0x000000000000f888 (+0x0000f5c8) 02c2020e SHL.B64 X1, #14 +0x000000000000f88c (+0x0000f5cc) 0006a481 ADD.S64 X3, X10, X9 +0x000000000000f890 (+0x0000f5d0) 00060084 MADD.S64 X3, X0, X1 +0x000000000000f894 (+0x0000f5d4) 00003081 ADD.S64 X0, X3, X1 +0x000000000000f898 (+0x0000f5d8) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:396 +# 390 | (void)result_used; +# 391 | return Ops::Load(address); +# 392 | #else +# 393 | if (!trace.atomics_enabled) return Ops::Load(address); +# 394 | const bool poll_batch = result_used && AtomicPollBatchEnabled(trace, site, AtomicOp::Load); +# 395 | const int32_t poll_index = poll_batch ? TraceAtomicPollBatchIndex(site) : -1; +# > 396 | const bool first_in_batch = poll_batch && +0x000000000000f89c (+0x0000f5dc) 08060000 ADD_IMM.S64 X3, X0, #0 +0x000000000000f8a0 (+0x0000f5e0) 1ca635d0 LD_XD_XN_IMM.B32 X19, X3, #1488 +0x000000000000f8a4 (+0x0000f5e4) 07000000 MOV_XD_IMM X0, #0 +0x000000000000f8a8 (+0x0000f5e8) 00c33a8a AND.B64 X1, X19, X21 +0x000000000000f8ac (+0x0000f5ec) 0000101e CMP.S64.NE X1, X0 +# [DWARF] common/pa_trace.h:398 +# 397 | (trace.poll_burst.active_mask & (1U << static_cast(poll_index))) == 0; +# > 398 | const uint64_t begin = !poll_batch || first_in_batch ? Ops::Now() : 0; +0x000000000000f8b0 (+0x0000f5f0) 40200002 JUMPC #2 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x000000000000f8b4 (+0x0000f5f4) 02808880 MOV_XD_SPR.F32 X0, SYS_CNT +# [DWARF] $ASCEND_HOME_PATH/tools/bisheng_compiler/lib/clang/15.0.5/include/__clang_cce_simt_atomic.h:86 +# 80 | return __builtin_cce_atom_max_G_##SUFFIX(base, inc, (uint32_t)L2Cache); \ +# 81 | } +# 82 | +# 83 | __CCE__ATOM_G_BUILTIN(uint32_t, u32) +# 84 | __CCE__ATOM_G_BUILTIN(int32_t, s32) +# 85 | __CCE__ATOM_G_BUILTIN(uint64_t, u64) +# > 86 | __CCE__ATOM_G_BUILTIN(int64_t, s64) +0x000000000000f8b8 (+0x0000f5f8) 51a809c0 ATOM XN, XM, XD, ADD +# [DWARF] common/pa_trace.h:236 +# 230 | (encoded_retries << kAtomicRetriesShift); +# 231 | } +# 232 | +# 233 | PA_DEVICE void CountAtomicCall( +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# > 236 | if (result.atomic_trace_calls == UINT64_MAX) { +0x000000000000f8bc (+0x0000f5fc) 1cc434a0 LD_XD_XN_IMM.B64 X2, X3, #1184 +0x000000000000f8c0 (+0x0000f600) 00002a1e CMP.S64.NE X2, X20 +0x000000000000f8c4 (+0x0000f604) 40200002 JUMPC #2 +0x000000000000f8c8 (+0x0000f608) 40000413 JUMP #1043 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f8cc (+0x0000f60c) 0808c002 ADD_IMM.S64 X4, X12, #2 +0x000000000000f8d0 (+0x0000f610) 00c6b78a AND.B64 X3, X11, X15 +0x000000000000f8d4 (+0x0000f614) 02c8020e SHL.B64 X4, #14 +0x000000000000f8d8 (+0x0000f618) 000aa481 ADD.S64 X5, X10, X9 +0x000000000000f8dc (+0x0000f61c) 000a3204 MADD.S64 X5, X3, X4 +# [DWARF] common/pa_trace.h:240 +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# 236 | if (result.atomic_trace_calls == UINT64_MAX) { +# 237 | trace.atomic_counter_overflow = true; +# 238 | return; +# 239 | } +# > 240 | ++result.atomic_trace_calls; +0x000000000000f8e0 (+0x0000f620) 08042001 ADD_IMM.S64 X2, X2, #1 +0x000000000000f8e4 (+0x0000f624) 00065201 ADD.S64 X3, X5, X4 +0x000000000000f8e8 (+0x0000f628) 08863680 SUB_IMM.S64 X3, X3, #1664 +0x000000000000f8ec (+0x0000f62c) 08063000 ADD_IMM.S64 X3, X3, #0 +# [DWARF] common/pa_trace.h:242 +# 241 | if (!poll_batch) return; +# > 242 | if (trace.poll_calls == UINT64_MAX) { +0x000000000000f8f0 (+0x0000f630) 1cc83570 LD_XD_XN_IMM.B64 X4, X3, #1392 +# [DWARF] common/pa_trace.h:240 +# 234 | TraceContext &trace, WorkerResult &result, bool poll_batch +# 235 | ) { +# 236 | if (result.atomic_trace_calls == UINT64_MAX) { +# 237 | trace.atomic_counter_overflow = true; +# 238 | return; +# 239 | } +# > 240 | ++result.atomic_trace_calls; +0x000000000000f8f4 (+0x0000f634) 03c434a0 ST_XD_XN_IMM.B64 X2, X3, #1184 +# [DWARF] common/pa_trace.h:242 +# 241 | if (!poll_batch) return; +# > 242 | if (trace.poll_calls == UINT64_MAX) { +0x000000000000f8f8 (+0x0000f638) 00004a1e CMP.S64.NE X4, X20 +0x000000000000f8fc (+0x0000f63c) 40200002 JUMPC #2 +0x000000000000f900 (+0x0000f640) 4000042c JUMP #1068 +# [DWARF] common/pa_trace.h:246 +# 243 | trace.atomic_counter_overflow = true; +# 244 | return; +# 245 | } +# > 246 | ++trace.poll_calls; +0x000000000000f904 (+0x0000f644) 08044001 ADD_IMM.S64 X2, X4, #1 +0x000000000000f908 (+0x0000f648) 03c43570 ST_XD_XN_IMM.B64 X2, X3, #1392 +# [DWARF] common/pa_trace.h:396 +# 390 | (void)result_used; +# 391 | return Ops::Load(address); +# 392 | #else +# 393 | if (!trace.atomics_enabled) return Ops::Load(address); +# 394 | const bool poll_batch = result_used && AtomicPollBatchEnabled(trace, site, AtomicOp::Load); +# 395 | const int32_t poll_index = poll_batch ? TraceAtomicPollBatchIndex(site) : -1; +# > 396 | const bool first_in_batch = poll_batch && +0x000000000000f90c (+0x0000f64c) 0000171e CMP.S64.NE X1, X14 +# [DWARF] common/pa_trace.h:351 +# 345 | if (signed_index < 0) { +# 346 | trace.atomic_counter_overflow = true; +# 347 | return; +# 348 | } +# 349 | const uint32_t index = static_cast(signed_index); +# 350 | const uint32_t bit = 1U << index; +# > 351 | if ((trace.poll_burst.active_mask & bit) == 0) { +0x000000000000f910 (+0x0000f650) 4020000e JUMPC #14 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f914 (+0x0000f654) 0804c002 ADD_IMM.S64 X2, X12, #2 +0x000000000000f918 (+0x0000f658) 00c2b78a AND.B64 X1, X11, X15 +0x000000000000f91c (+0x0000f65c) 02c4020e SHL.B64 X2, #14 +0x000000000000f920 (+0x0000f660) 0006a481 ADD.S64 X3, X10, X9 +0x000000000000f924 (+0x0000f664) 00061104 MADD.S64 X3, X1, X2 +0x000000000000f928 (+0x0000f668) 00023101 ADD.S64 X1, X3, X2 +0x000000000000f92c (+0x0000f66c) 08821680 SUB_IMM.S64 X1, X1, #1664 +# [DWARF] common/pa_trace.h:352 +# 346 | trace.atomic_counter_overflow = true; +# 347 | return; +# 348 | } +# 349 | const uint32_t index = static_cast(signed_index); +# 350 | const uint32_t bit = 1U << index; +# 351 | if ((trace.poll_burst.active_mask & bit) == 0) { +# > 352 | trace.poll_burst.start_cycle[index] = start_cycle; +0x000000000000f930 (+0x0000f670) 08021000 ADD_IMM.S64 X1, X1, #0 +0x000000000000f934 (+0x0000f674) 03c01598 ST_XD_XN_IMM.B64 X0, X1, #1432 +# [DWARF] common/pa_trace.h:354 +# 353 | trace.poll_burst.call_count[index] = 0; +# > 354 | trace.poll_burst.active_mask |= bit; +0x000000000000f938 (+0x0000f678) 07000002 MOV_XD_IMM X0, #2 +0x000000000000f93c (+0x0000f67c) 02e60440 SBITSET.B64 X19, X0 +# [DWARF] common/pa_trace.h:353 +# 347 | return; +# 348 | } +# 349 | const uint32_t index = static_cast(signed_index); +# 350 | const uint32_t bit = 1U << index; +# 351 | if ((trace.poll_burst.active_mask & bit) == 0) { +# 352 | trace.poll_burst.start_cycle[index] = start_cycle; +# > 353 | trace.poll_burst.call_count[index] = 0; +0x000000000000f940 (+0x0000f680) 0f961800 STI_XN_IMM.B32 X1, #1472 +# [DWARF] common/pa_trace.h:354 +# > 354 | trace.poll_burst.active_mask |= bit; +0x000000000000f944 (+0x0000f684) 03a615d0 ST_XD_XN_IMM.B32 X19, X1, #1488 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f948 (+0x0000f688) 0802c002 ADD_IMM.S64 X1, X12, #2 +0x000000000000f94c (+0x0000f68c) 00c0b78a AND.B64 X0, X11, X15 +0x000000000000f950 (+0x0000f690) 02c2020e SHL.B64 X1, #14 +0x000000000000f954 (+0x0000f694) 0004a481 ADD.S64 X2, X10, X9 +0x000000000000f958 (+0x0000f698) 00040084 MADD.S64 X2, X0, X1 +# [DWARF] common/pa_trace.h:358 +# 352 | trace.poll_burst.start_cycle[index] = start_cycle; +# 353 | trace.poll_burst.call_count[index] = 0; +# 354 | trace.poll_burst.active_mask |= bit; +# 355 | } +# 356 | uint32_t &call_count = trace.poll_burst.call_count[index]; +# 357 | ++call_count; +# > 358 | if (call_count == kAtomicPollCountMax) { +0x000000000000f95c (+0x0000f69c) 0706ffff MOV_XD_IMM X3, #65535 +0x000000000000f960 (+0x0000f6a0) 00002081 ADD.S64 X0, X2, X1 +0x000000000000f964 (+0x0000f6a4) 08800680 SUB_IMM.S64 X0, X0, #1664 +# [DWARF] common/pa_trace.h:357 +# 351 | if ((trace.poll_burst.active_mask & bit) == 0) { +# 352 | trace.poll_burst.start_cycle[index] = start_cycle; +# 353 | trace.poll_burst.call_count[index] = 0; +# 354 | trace.poll_burst.active_mask |= bit; +# 355 | } +# 356 | uint32_t &call_count = trace.poll_burst.call_count[index]; +# > 357 | ++call_count; +0x000000000000f968 (+0x0000f6a8) 08000000 ADD_IMM.S64 X0, X0, #0 +0x000000000000f96c (+0x0000f6ac) 1c8205c0 LD_XD_XN_IMM.B32 X1, X0, #1472 +# [DWARF] common/pa_trace.h:358 +# > 358 | if (call_count == kAtomicPollCountMax) { +0x000000000000f970 (+0x0000f6b0) 074700ff MOVK X3, #255, #1 +# [DWARF] common/pa_trace.h:357 +# 351 | if ((trace.poll_burst.active_mask & bit) == 0) { +# 352 | trace.poll_burst.start_cycle[index] = start_cycle; +# 353 | trace.poll_burst.call_count[index] = 0; +# 354 | trace.poll_burst.active_mask |= bit; +# 355 | } +# 356 | uint32_t &call_count = trace.poll_burst.call_count[index]; +# > 357 | ++call_count; +0x000000000000f974 (+0x0000f6b4) 08021001 ADD_IMM.S64 X1, X1, #1 +0x000000000000f978 (+0x0000f6b8) 02841a00 ZEROEXT.U32 X2, X1 +0x000000000000f97c (+0x0000f6bc) 038205c0 ST_XD_XN_IMM.B32 X1, X0, #1472 +# [DWARF] common/pa_trace.h:358 +# > 358 | if (call_count == kAtomicPollCountMax) { +0x000000000000f980 (+0x0000f6c0) 0000219e CMP.S64.NE X2, X3 +0x000000000000f984 (+0x0000f6c4) 402003f3 JUMPC #1011 +# [DWARF] common/pa_trace.h:0 +# [SOURCE unavailable] +0x000000000000f988 (+0x0000f6c8) 03e1e958 ST_XD_XN_IMM.B64 X16, X30, #2392 +0x000000000000f98c (+0x0000f6cc) 08320548 ADD_IMM.S64 X25, X0, #1352 +# [DWARF] ccec/ccec_ops.h:229 +# 223 | // coherency is handled by the runtime's DCCI protocol. +# 224 | // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， +# 225 | // 会改变待测 Submit 热路径。completion 使用 atomic，config/trace 的 cache +# 226 | // 可见性则由各自既有的 DCCI 路径处理。 +# 227 | __aicore__ static inline void StoreBarrier() {} +# 228 | +# > 229 | __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } +0x000000000000f990 (+0x0000f6d0) 02828880 MOV_XD_SPR.F32 X1, SYS_CNT +0x000000000000f994 (+0x0000f6d4) 07280000 MOV_XD_IMM X20, #0 +0x000000000000f998 (+0x0000f6d8) 0cf39d00 LDP_XI_XJ_XN.B64 X25, X26, X25, #0 +0x000000000000f99c (+0x0000f6dc) 07300000 MOV_XD_IMM X24, #0 +0x000000000000f9a0 (+0x0000f6e0) 1cb60558 LD_XD_XN_IMM.B32 X27, X0, #1368 +0x000000000000f9a4 (+0x0000f6e4) 03c3e988 ST_XD_XN_IMM.B64 X1, X30, #2440 +0x000000000000f9a8 (+0x0000f6e8) 072461f0 MOV_XD_IMM X18, #25072 +0x000000000000f9ac (+0x0000f6ec) 07650007 MOVK X18, #7, #1 +0x000000000000f9b0 (+0x0000f6f0) 07a50000 MOVK X18, #0, #2 +0x000000000000f9b4 (+0x0000f6f4) 07e50000 MOVK X18, #0, #3 +0x000000000000f9b8 (+0x0000f6f8) 02040880 MOV_XD_SPR.S64 X2, PC +0x000000000000f9bc (+0x0000f6fc) 00252101 ADD.S64 X18, X18, X2 +0x000000000000f9c0 (+0x0000f700) 4000001b JUMP #27 +# [DWARF] ccec/ccec_ops.h:0 +# [SOURCE unavailable] +0x000000000000f9c4 (+0x0000f704) 1cc5e9a0 LD_XD_XN_IMM.B64 X2, X30, #2464 +# [DWARF] common/pa_trace.h:280 +# 274 | call_count, static_cast(site) +# 275 | ); +# 276 | if (written) { +# 277 | if (trace.poll_batch_records == UINT64_MAX) { +# 278 | trace.atomic_counter_overflow = true; +# 279 | } else { +# > 280 | ++trace.poll_batch_records; +0x000000000000f9c8 (+0x0000f708) 08000001 ADD_IMM.S64 X0, X0, #1 +0x000000000000f9cc (+0x0000f70c) 1cc3e9a8 LD_XD_XN_IMM.B64 X1, X30, #2472 +0x000000000000f9d0 (+0x0000f710) 1cc7e9b0 LD_XD_XN_IMM.B64 X3, X30, #2480 +0x000000000000f9d4 (+0x0000f714) 1cc9e990 LD_XD_XN_IMM.B64 X4, X30, #2448 +0x000000000000f9d8 (+0x0000f718) 08042002 ADD_IMM.S64 X2, X2, #2 +0x000000000000f9dc (+0x0000f71c) 00c21a8a AND.B64 X1, X1, X21 +0x000000000000f9e0 (+0x0000f720) 02c4020e SHL.B64 X2, #14 +0x000000000000f9e4 (+0x0000f724) 00063201 ADD.S64 X3, X3, X4 +0x000000000000f9e8 (+0x0000f728) 00061104 MADD.S64 X3, X1, X2 +0x000000000000f9ec (+0x0000f72c) 00023101 ADD.S64 X1, X3, X2 +0x000000000000f9f0 (+0x0000f730) 08821680 SUB_IMM.S64 X1, X1, #1664 +0x000000000000f9f4 (+0x0000f734) 08021000 ADD_IMM.S64 X1, X1, #0 +0x000000000000f9f8 (+0x0000f738) 03c01578 ST_XD_XN_IMM.B64 X0, X1, #1400 +# [DWARF] common/pa_trace.h:283 +# 281 | } +# 282 | } +# > 283 | trace.poll_burst.call_count[index] = 0; +0x000000000000f9fc (+0x0000f73c) 0f976700 STI_XN_IMM.B32 X22, #1464 +0x000000000000fa00 (+0x0000f740) 071e7fff MOV_XD_IMM X15, #32767 +0x000000000000fa04 (+0x0000f744) 1cd5e9b0 LD_XD_XN_IMM.B64 X10, X30, #2480 +0x000000000000fa08 (+0x0000f748) 1cd7e9a8 LD_XD_XN_IMM.B64 X11, X30, #2472 +0x000000000000fa0c (+0x0000f74c) 1cd9e9a0 LD_XD_XN_IMM.B64 X12, X30, #2464 +0x000000000000fa10 (+0x0000f750) 1cd3e990 LD_XD_XN_IMM.B64 X9, X30, #2448 +# [DWARF] common/pa_trace.h:262 +# 256 | #else +# 257 | if (!trace.atomics_enabled || trace.poll_burst.active_mask == 0) return; +# 258 | const uint32_t active_mask = trace.poll_burst.active_mask; +# 259 | // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 +# 260 | // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 +# 261 | PA_LOOP_NOUNROLL +# > 262 | for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { +0x000000000000fa14 (+0x0000f754) 08318001 ADD_IMM.S64 X24, X24, #1 +0x000000000000fa18 (+0x0000f758) 08252004 ADD_IMM.S64 X18, X18, #4 +0x000000000000fa1c (+0x0000f75c) 0001841e CMP.S64.NE X24, X8 +0x000000000000fa20 (+0x0000f760) 08294001 ADD_IMM.S64 X20, X20, #1 diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/published.sha256 new file mode 100644 index 0000000000..a7412f540b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/published.sha256 @@ -0,0 +1,40 @@ +e2b85f242dd06690b878504895c9a03e24489f75dc3f514a50c196ed436de1d3 annotated/00_pa_scheduler_0_mix_aic_4f68fde0.source.asm.gz +3af0b26f2ed6b8e45121c71e6f875b19744d1f41c5148b2ecc66acef7b493f56 annotated/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.source.asm.gz +3a1f6528b8a5176152825047fcd72c7ba2194976379316565db070f5b333784c annotated/02_pa_execute_real_winner_workload_aic_729699de.source.asm.gz +3fdb270573fdba87c76e236cbc3f2d64636170f8b0e5a481f0c82f099e19fded annotated/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.source.asm.gz +af87679df78972fb4f775bc12589a41a14b0077f5d474b58c41891f5577e2d04 annotated/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +8554244b60e19d60eb718727677a970b2299ef2e2102561ac674e711f8174373 annotated/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.source.asm.gz +bd6e27107a25cf442fdae4164cb291807ec6fc5569d518635e63ebf18d43d15a annotated/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +a5b644bb9cc8014991e7674924685713881cbe40f1aa407e3d26fc8eefe6e51c annotated/07_pa_scheduler_0_mix_aiv_9fb2e4b4.source.asm.gz +897f10ddca4124c4a54159d131148ea06df4bada8e2dce28e50a225ba3772dce annotated/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.source.asm.gz +15c05cda9dca6f3786e0d44884a6a7d06229c25e74349feed28314bd8bf91673 annotated/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +a94815bdfaf87c5fdc63204b05e7b05237ed3051d16340b9147e3f9797bca99e annotated/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.source.asm.gz +a880e686b7a3e328114d681a3354b6f0e620e7b462844896567ffa55ae92de02 annotated/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.source.asm.gz +29397ff5303bb880ab957a906232c83c3163a860683ab4c51fb97b016d2a90f6 annotated/12_pa_execute_real_winner_workload_aiv_0396bb2e.source.asm.gz +83ff83e20ad342151e2e6a41101d848da59d8b637b415ea468f16271689965ce annotated/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.source.asm.gz +bf616bf378a19637c4711121631956493754ece990c39d584ca3d08e80881fe9 annotated/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.source.asm.gz +681a3b99edba16bf5b58bc545c7ed82ccdaec900dafb25be8d10f0f3f75ab7a4 annotated/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.source.asm.gz +331f429dfde1898c71a649d4f4988b4c6fe654171bea29d049fe7c1dec992f09 annotated/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.source.asm.gz +51cdcbbfeaf38c1b574d6ff3ad744c60f31b2c11953e30f158bd634f1a1fc408 key_flow/aic_compete_first_lazy.source.asm +8c169dca78cff0614a74bf9bd4e61e94171391eda04b25bdce2a7f665897c4af key_flow/aic_lazy_input_policy.source.asm +d89cdc40aace173c873e1c82837a0ac1060662e4230c084cb9feb2d19881837b raw/00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz +53d0444c1330462e69fd44d33f99b3f41772f084b66b61bbcffd979c9073cbeb raw/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz +50719af7205a8c687bef91f9790d25d6a2d558d9d968a47b283fc4226cb69763 raw/02_pa_execute_real_winner_workload_aic_729699de.asm.gz +cff035bd198985edc0b291bd5a946027d22db960378236170c29e96c1e42ac03 raw/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz +c59d2bd2bd9f8d1ef3b1f1e5a12a6e964cef9db83ba3ea14c62d26aa1abbea03 raw/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +6c4fe9a6d200d469e3cebf7dbdc6cd295679c3457e44ae56574b68e187f3b696 raw/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz +d6a03ec582bd3e24d9fad8d489c5ce93c5d21343929a9b3be202df7b4e7f9a2d raw/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +5b75f4842c3c023ad033240263e35457ad363f65f85bb7a41c737a242dd7b70f raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz +a6e37748e0854ddc7b73c797b59e9b3dcb468365ddbfe1b3bea0e956f0a22f05 raw/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz +81f766a4e5acc4e504339614c246451d06e7eb35149fa51a40a15d7f7ddba076 raw/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +1da561a3de041604b48571185ef15db7e6677074566d1f578a6b3279248dd29d raw/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz +8fd6f62d3f9a826100d055ebf29b801e969513548a9d4c68bbdc25c1f3a6758f raw/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz +f98d6795cd66b34a5acdda1370c0c749d98b0a254b09e4097e81f13c2b0e5ff4 raw/12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz +edd0df92b1cae8411d355e4a9f9058c9306b08e366755567a7e674e16c406c75 raw/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz +85f987e18c6af8ebc39466316281a58d9ccf51249e6fd7817617185bdaf9c0bd raw/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +ad259b20157f3ff9c8de366f47efefe6ef4db225c7611c97682650aeafe77634 raw/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz +d57d1870e347095aa393da8232fc29b8657e900d3af8a924748337226ca6f6e3 raw/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz +7b609f0665fcb3b9649e21073700db468cec223a20953958337da9bd7feb73e8 raw/README.md +3957c577e5f1ac803ce0bf1ad7668f8bc08f5752e59dfbf4d69591c3c0e323a1 raw/gaps.tsv +84aac4f16730d8afb63074bc9d61cd426451ea5823f2dce96850d86770500aa0 raw/manifest.tsv +f236e5eaf5b0852b54da299d3ac2d46a8fb55cdbbfdae8d9e6545ca0084fc967 raw/published.sha256 diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz new file mode 100644 index 0000000000..6865629145 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz new file mode 100644 index 0000000000..f2e00f76f1 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/02_pa_execute_real_winner_workload_aic_729699de.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/02_pa_execute_real_winner_workload_aic_729699de.asm.gz new file mode 100644 index 0000000000..4aef833395 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/02_pa_execute_real_winner_workload_aic_729699de.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz new file mode 100644 index 0000000000..a5c96d6435 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz new file mode 100644 index 0000000000..0448f27221 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz new file mode 100644 index 0000000000..00e341c6be Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz new file mode 100644 index 0000000000..8149f18eb3 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz new file mode 100644 index 0000000000..13eadb1042 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz new file mode 100644 index 0000000000..577bc57422 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz new file mode 100644 index 0000000000..f5d5cacb82 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz new file mode 100644 index 0000000000..1576b3c3fe Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz new file mode 100644 index 0000000000..2597def790 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz new file mode 100644 index 0000000000..40e3cc0aa1 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz new file mode 100644 index 0000000000..8ab20ab7b6 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz new file mode 100644 index 0000000000..67b395e1e5 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz new file mode 100644 index 0000000000..18bad6572d Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz new file mode 100644 index 0000000000..d769738b72 Binary files /dev/null and b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz differ diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/README.md b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/README.md new file mode 100644 index 0000000000..23075297a2 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/README.md @@ -0,0 +1,14 @@ +# Final-linked A5 disassembly + +Every non-empty final `.text` `STT_FUNC` symbol was extracted from the published final ELF and decoded with the verified `dav_3510` PEM decoder. +RVec `.vector.thread` symbols are passed through an explicit full-body RVec range; all other symbols use scalar decoding. +Each compressed file records final PC, function-relative offset, machine word, mnemonic, ELF/body hashes and decoder identity. +Function gaps are alignment/padding outside symbol bodies and are hashed in `gaps.tsv`; they are not presented as instructions. + +Decoder SHA256: `29835d2439d6dd464d34a212ad4bbd5c29af6a38465da09a6c273401d9a96dcb` + +| Variant | Final .text | Function symbols | Function bytes | Coverage | Final ELF SHA256 | +| --- | ---: | ---: | ---: | ---: | --- | +| compete-first-lazy | 547640 B | 17 | 547248 B | 99.928% | `8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893` | + +Inspect with `zless FILE.asm.gz`; the sibling `annotated/` directory adds DWARF-mapped local source and comments. diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/gaps.tsv b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/gaps.tsv new file mode 100644 index 0000000000..7ca5bb26a3 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/gaps.tsv @@ -0,0 +1,9 @@ +variant start_pc end_pc size sha256 +compete-first-lazy 0x68 0x70 8 c573025535b1cea90f421ad7615d65296651dd5bc7287d7dfce7cc79c878ef4b +compete-first-lazy 0x27b9c 0x27ba0 4 d9fc8a51763953481a1808af3156bcb8144c2f925e96dec623c886f6d9d975b2 +compete-first-lazy 0x42aac 0x42ab0 4 d9fc8a51763953481a1808af3156bcb8144c2f925e96dec623c886f6d9d975b2 +compete-first-lazy 0x42b2c 0x42b30 4 d9fc8a51763953481a1808af3156bcb8144c2f925e96dec623c886f6d9d975b2 +compete-first-lazy 0x5dbfc 0x5dc00 4 d9fc8a51763953481a1808af3156bcb8144c2f925e96dec623c886f6d9d975b2 +compete-first-lazy 0x85968 0x85a00 152 ae7f0812b0e030af6ee3ce80c2a0d3146b819b16f8214c513e9022ddc52a81e6 +compete-first-lazy 0x85a30 0x85b00 208 c9364bd8435644c4dcf160545b42c2cfb80001a10890c7a1cdd1f7a0a28dc8ac +compete-first-lazy 0x85b30 0x85b38 8 c573025535b1cea90f421ad7615d65296651dd5bc7287d7dfce7cc79c878ef4b diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/manifest.tsv b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/manifest.tsv new file mode 100644 index 0000000000..4e9e9a1a00 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/manifest.tsv @@ -0,0 +1,18 @@ +variant file binding symbol final_pc size instructions decoder_mode body_sha256 last_mnemonic +compete-first-lazy 00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz GLOBAL pa_scheduler_0_mix_aic 0x0 104 26 scalar 3a8ecf84e154c1afc9dab2c992e3357d338c1d996a677931069f16310bdb28e5 END +compete-first-lazy 01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz LOCAL _ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j 0x70 192 48 scalar b8afa7640101b76f5e91a2e172ddd1cb39b6e11ccecd0c4395057eabc1e2bc00 RET +compete-first-lazy 02_pa_execute_real_winner_workload_aic_729699de.asm.gz LOCAL pa_execute_real_winner_workload_aic 0x130 400 100 scalar 88b6e6169f66470b7d9746ae3b1dc48a5e2bf4bd46994a85cc37926c394adeb6 RET +compete-first-lazy 03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz LOCAL pa_scheduler_lazy_sample_callback_orchestration_aic 0x2c0 161792 40448 scalar 35bcc11cdad126fa5d2f92ff1df421d8556843a412ea3497addfe820d84bd852 RET +compete-first-lazy 04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz LOCAL _ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3AS1NS_11TraceRecordEjmmjj 0x27ac0 220 55 scalar 257907e7fa33e1cc56a582855355e17e46973e11d7fa9d5c02f39f6424748c29 RET +compete-first-lazy 05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz LOCAL pa_scheduler_lazy_sample_callback_finish_aic 0x27ba0 110128 27532 scalar cdb102a84acb7a681565d97805b405d526a94c604812faed3850773575acee08 JUMP +compete-first-lazy 06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz LOCAL _ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3AS1NS_11TraceRecordEjmmjj 0x429d0 220 55 scalar 422635da4a38494a486262b1261ddcd2127aa8aac6f695aee3bea17bebe7f36c RET +compete-first-lazy 07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz GLOBAL pa_scheduler_0_mix_aiv 0x42ab0 124 31 scalar 63f9349fa440318678bcf3e98ad719502bd907f7537fb0f083eac32895e3ad9b END +compete-first-lazy 08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz LOCAL pa_scheduler_lazy_sample_callback_finish_aiv 0x42b30 110576 27644 scalar 3f5ec8fee73b82e72f40566908c21f4fcfd3fd85983892c323a526c66557cade JUMP +compete-first-lazy 09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz LOCAL _ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3AS1NS_11TraceRecordEjmmjj 0x5db20 220 55 scalar 1fd702642d07bfb00a40e2f710068a59fc171927a070d63778975d757fe07add RET +compete-first-lazy 10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz LOCAL _ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j 0x5dc00 268 66 scalar d55e1e765b39016fa5139e91d947f6816b6bea5ff18a176c45b48d5ada5a5d8f RET +compete-first-lazy 11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz LOCAL _ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j 0x5dd0c 268 66 scalar 8bb7a51e530f1c2f2c336a661b18e890bdce87740c931135a953e81e046ef3eb RET +compete-first-lazy 12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz LOCAL pa_execute_real_winner_workload_aiv 0x5de18 476 119 scalar 158c11e2f546c21643c2addb244ffefe3e46e5aeee5b04fbd4873dcd33a07363 RET +compete-first-lazy 13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz LOCAL pa_scheduler_lazy_sample_callback_orchestration_aiv 0x5dff4 161944 40486 scalar 62d3784c0d6bcf33939e500cdcdf7464889437785157f68492a6f13c8ec5e60c RET +compete-first-lazy 14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz LOCAL _ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3AS1NS_11TraceRecordEjmmjj 0x8588c 220 55 scalar f9d41908b2bc3f9b24be58c361de95327a6ef15de8061987608a9a5658120332 RET +compete-first-lazy 15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz LOCAL _ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128EEEvPU3AS6T0_S5_S5_jj.vector.thread 0x85a00 48 12 rvec 40cfc6402a5ba38eb3edbe949d63afe52206a592975835aa76b9f028b1d1a67a RV_SEND.U16 +compete-first-lazy 16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz LOCAL _ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128EEEvPU3AS6T0_S5_S5_jj.vector.thread 0x85b00 48 12 rvec 9afa7fa1b6031acff6f90ef3822167d31eeec4d90ee419db0b85dd7c40e5404c RV_SEND.U16 diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/published.sha256 new file mode 100644 index 0000000000..5ac45b926e --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/disassembly/raw/published.sha256 @@ -0,0 +1,20 @@ +d89cdc40aace173c873e1c82837a0ac1060662e4230c084cb9feb2d19881837b 00_pa_scheduler_0_mix_aic_4f68fde0.asm.gz +53d0444c1330462e69fd44d33f99b3f41772f084b66b61bbcffd979c9073cbeb 01_ZN17pa_scheduler_ccecL25pa_real_cube_workload_aicEPU3AS1fS1_S1_j_c02670e3.asm.gz +50719af7205a8c687bef91f9790d25d6a2d558d9d968a47b283fc4226cb69763 02_pa_execute_real_winner_workload_aic_729699de.asm.gz +cff035bd198985edc0b291bd5a946027d22db960378236170c29e96c1e42ac03 03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz +c59d2bd2bd9f8d1ef3b1f1e5a12a6e964cef9db83ba3ea14c62d26aa1abbea03 04_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +6c4fe9a6d200d469e3cebf7dbdc6cd295679c3457e44ae56574b68e187f3b696 05_pa_scheduler_lazy_sample_callback_finish_aic_a5ad7642.asm.gz +d6a03ec582bd3e24d9fad8d489c5ce93c5d21343929a9b3be202df7b4e7f9a2d 06_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +5b75f4842c3c023ad033240263e35457ad363f65f85bb7a41c737a242dd7b70f 07_pa_scheduler_0_mix_aiv_9fb2e4b4.asm.gz +a6e37748e0854ddc7b73c797b59e9b3dcb468365ddbfe1b3bea0e956f0a22f05 08_pa_scheduler_lazy_sample_callback_finish_aiv_cfc84242.asm.gz +81f766a4e5acc4e504339614c246451d06e7eb35149fa51a40a15d7f7ddba076 09_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +1da561a3de041604b48571185ef15db7e6677074566d1f578a6b3279248dd29d 10_ZN17pa_scheduler_ccecL31pa_real_vector_add_workload_aivEPU3AS1fS1_S1_j_b0244082.asm.gz +8fd6f62d3f9a826100d055ebf29b801e969513548a9d4c68bbdc25c1f3a6758f 11_ZN17pa_scheduler_ccecL31pa_real_vector_mul_workload_aivEPU3AS1fS1_S1_j_a23bc575.asm.gz +f98d6795cd66b34a5acdda1370c0c749d98b0a254b09e4097e81f13c2b0e5ff4 12_pa_execute_real_winner_workload_aiv_0396bb2e.asm.gz +edd0df92b1cae8411d355e4a9f9058c9306b08e366755567a7e674e16c406c75 13_pa_scheduler_lazy_sample_callback_orchestration_aiv_848852b1.asm.gz +85f987e18c6af8ebc39466316281a58d9ccf51249e6fd7817617185bdaf9c0bd 14_ZN12pa_schedulerL23WritePollBatchRecordRawEPU3AS1NS_14TraceCoreStateEPU3_7dc65765.asm.gz +ad259b20157f3ff9c8de366f47efefe6ef4db225c7611c97682650aeafe77634 15_ZN3pto23TBinOps_2D_NoPostUpdateINS_5AddOpIfEEfLj64ELj8ELj128ELj128ELj128_45ff7293.asm.gz +d57d1870e347095aa393da8232fc29b8657e900d3af8a924748337226ca6f6e3 16_ZN3pto23TBinOps_2D_NoPostUpdateINS_5MulOpIfEEfLj64ELj8ELj128ELj128ELj128_e7b99782.asm.gz +7b609f0665fcb3b9649e21073700db468cec223a20953958337da9bd7feb73e8 README.md +3957c577e5f1ac803ce0bf1ad7668f8bc08f5752e59dfbf4d69591c3c0e323a1 gaps.tsv +84aac4f16730d8afb63074bc9d61cd426451ea5823f2dce96850d86770500aa0 manifest.tsv diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/generate_disassembly.sh b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/generate_disassembly.sh new file mode 100755 index 0000000000..5554f7c29b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/generate_disassembly.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +RAW_DIR="$SCRIPT_DIR/disassembly/raw" +ANNOTATED_DIR="$SCRIPT_DIR/disassembly/annotated" + +case "$RAW_DIR:$ANNOTATED_DIR" in + "$SCRIPT_DIR/disassembly/raw:$SCRIPT_DIR/disassembly/annotated") ;; + *) echo "Refusing to replace unexpected disassembly paths" >&2; exit 1 ;; +esac +rm -rf -- "$RAW_DIR" "$ANNOTATED_DIR" + +python3 "$SCRIPT_DIR/disassemble.py" --output "$RAW_DIR" +mkdir -p "$ANNOTATED_DIR" +for raw in "$RAW_DIR"/*.asm.gz; do + name="$(basename "$raw" .asm.gz)" + python3 "$SCRIPT_DIR/annotate_disassembly.py" \ + --elf "$SCRIPT_DIR/artifacts/measured/pa_scheduler_kernel.o" \ + --raw "$raw" \ + --source-root "$SCRIPT_DIR" \ + --output "$ANNOTATED_DIR/${name}.source.asm.gz" +done +mkdir -p "$SCRIPT_DIR/disassembly/key_flow" +python3 "$SCRIPT_DIR/annotate_disassembly.py" \ + --elf "$SCRIPT_DIR/artifacts/measured/pa_scheduler_kernel.o" \ + --raw "$RAW_DIR/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz" \ + --source-root "$SCRIPT_DIR" \ + --output "$SCRIPT_DIR/disassembly/key_flow/aic_compete_first_lazy.source.asm" \ + --anchor common/pa_scheduler_core.h:1382 --before 558 --after 162 +python3 "$SCRIPT_DIR/annotate_disassembly.py" \ + --elf "$SCRIPT_DIR/artifacts/measured/pa_scheduler_kernel.o" \ + --raw "$RAW_DIR/03_pa_scheduler_lazy_sample_callback_orchestration_aic_9bd281ff.asm.gz" \ + --source-root "$SCRIPT_DIR" \ + --output "$SCRIPT_DIR/disassembly/key_flow/aic_lazy_input_policy.source.asm" \ + --anchor common/pa_frontend.h:703 --before 160 --after 640 +( + cd "$SCRIPT_DIR/disassembly" + find raw annotated key_flow -type f -print0 | sort -z | xargs -0 sha256sum +) > "$SCRIPT_DIR/disassembly/published.sha256" diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/functional.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/functional.txt new file mode 100644 index 0000000000..f920a3acba --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/functional.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=1 tasks=5 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=97/97 tensor_args=1162/1162 scalar_args=9/9 resets=384/384 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=75.689 host_launch_us=1006.556 claims=288 fanin_loads=162 cas_retries=0 +[ATOMIC] submit_completion_ops=484 fanin_ready=35 fanin_not_ready=127 frontier_initial=5 frontier_flag=10 frontier_ready_fetch_max=5 frontier_terminal=5 +[WINNERS] active_workers=5 max_wins_per_worker=1 +[PLACEMENT] EfDrain=1 RingBp=0 FinalDrain=3 +[KERNEL] QK count=1 mean_us=42.740 min_us=42.740 max_us=42.740 target_us=44.170 +[KERNEL] SF count=1 mean_us=53.606 min_us=53.606 max_us=53.606 target_us=53.729 +[KERNEL] PV count=1 mean_us=27.730 min_us=27.730 max_us=27.730 target_us=27.626 +[KERNEL] UP count=1 mean_us=2.540 min_us=2.540 max_us=2.540 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=4 inactive_sentinel_tiles=188) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=75.689 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/RUN_METADATA.md b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/RUN_METADATA.md new file mode 100644 index 0000000000..ad64b4dbd4 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/RUN_METADATA.md @@ -0,0 +1,17 @@ +# Final b256 run provenance + +- Variant: `compete-first-lazy`; +- collection interval: 2026-07-20 07:02:23–07:12:45 UTC; +- direct device0 execution, 96 workers, b256; +- 24 independent host launches and one device run per launch; +- all six A/B/C launch permutations repeated four times; launch positions 1/2/3 are 8/8/8; +- PMU explicitly off, runtime swimlane explicitly off, real-compute; +- executable source: this package's clean `build/compete-first-lazy`; +- exact final ELF copy: `artifacts/measured/pa_scheduler_kernel.o`; +- final ELF SHA256: `8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893`; +- host SHA256: `d6d9e9bb04e28c7682e47230e20722f1e7843937b50017ddfe8a515acbe0ccd9`. + +`task-submit` and `npu-smi` were unavailable in the collection shell. The run used the user's explicit +authorization to access device0 directly and was not queue-isolated. All raw logs passed execution, +semantic and postprocess checks. Raw evidence is retained; the primary statistic applies the common +per-variant Hampel rule documented in the root performance comparison. diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/SUMMARY.md b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/SUMMARY.md new file mode 100644 index 0000000000..5b4f600cb7 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/SUMMARY.md @@ -0,0 +1,11 @@ +# compete-first-lazy device0 b256 result + +24 independent host launches, one device run per launch; PMU off, runtime swimlane off, real-compute. +Raw logs are retained. The primary row excludes samples outside median ± 3×1.4826×MAD. + +| View | N | Median (us) | Mean (us) | Min (us) | Max (us) | Stddev (us) | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Raw | 24 | 3828.042 | 3825.094 | 3643.199 | 4370.394 | 151.243 | +| Outlier-filtered | 22 | 3818.781 | 3788.503 | 3643.199 | 3950.280 | 83.869 | + +Excluded samples: 2. diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_01_ABC_pos3_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_01_ABC_pos3_compete-first-lazy.txt new file mode 100644 index 0000000000..2840b0cfac --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_01_ABC_pos3_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=4084.775 host_launch_us=4990.563 claims=73728 fanin_loads=29650 cas_retries=0 +[ATOMIC] submit_completion_ops=130226 fanin_ready=4731 fanin_not_ready=24919 frontier_initial=1280 frontier_flag=11632 frontier_ready_fetch_max=10352 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=27 +[PLACEMENT] EfDrain=968 RingBp=47 FinalDrain=9 +[KERNEL] QK count=256 mean_us=41.467 min_us=40.580 max_us=47.418 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.564 min_us=51.959 max_us=113.194 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.952 min_us=26.977 max_us=74.210 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.713 min_us=2.021 max_us=16.484 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=189 inactive_sentinel_tiles=3) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=4084.775 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_02_BCA_pos2_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_02_BCA_pos2_compete-first-lazy.txt new file mode 100644 index 0000000000..e49579de48 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_02_BCA_pos2_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3845.228 host_launch_us=4773.478 claims=73728 fanin_loads=32549 cas_retries=0 +[ATOMIC] submit_completion_ops=117559 fanin_ready=3494 fanin_not_ready=29055 frontier_initial=1280 frontier_flag=3849 frontier_ready_fetch_max=2569 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=983 RingBp=16 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.473 min_us=40.621 max_us=48.475 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.125 min_us=52.251 max_us=249.945 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.172 min_us=26.990 max_us=75.647 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.745 min_us=2.337 max_us=21.829 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=188 inactive_sentinel_tiles=4) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3845.228 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_03_CAB_pos1_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_03_CAB_pos1_compete-first-lazy.txt new file mode 100644 index 0000000000..5b2891cd30 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_03_CAB_pos1_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3655.535 host_launch_us=6125.462 claims=73728 fanin_loads=32564 cas_retries=0 +[ATOMIC] submit_completion_ops=116540 fanin_ready=3443 fanin_not_ready=29121 frontier_initial=1280 frontier_flag=3332 frontier_ready_fetch_max=2052 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=33 +[PLACEMENT] EfDrain=991 RingBp=10 FinalDrain=23 +[KERNEL] QK count=256 mean_us=41.545 min_us=40.641 max_us=48.778 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.612 min_us=52.310 max_us=66.610 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.274 min_us=27.006 max_us=68.454 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.610 min_us=2.408 max_us=9.233 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3655.535 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_04_ACB_pos2_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_04_ACB_pos2_compete-first-lazy.txt new file mode 100644 index 0000000000..a9341b0b58 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_04_ACB_pos2_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3815.964 host_launch_us=4725.958 claims=73728 fanin_loads=33506 cas_retries=0 +[ATOMIC] submit_completion_ops=117206 fanin_ready=3466 fanin_not_ready=30040 frontier_initial=1280 frontier_flag=3194 frontier_ready_fetch_max=1914 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=36 +[PLACEMENT] EfDrain=984 RingBp=12 FinalDrain=28 +[KERNEL] QK count=256 mean_us=41.498 min_us=40.555 max_us=51.035 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.435 min_us=52.207 max_us=225.194 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.058 min_us=27.006 max_us=85.758 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.827 min_us=2.364 max_us=33.455 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=189 inactive_sentinel_tiles=3) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3815.964 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_05_BAC_pos3_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_05_BAC_pos3_compete-first-lazy.txt new file mode 100644 index 0000000000..483b23ab2c --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_05_BAC_pos3_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3645.710 host_launch_us=5229.678 claims=73728 fanin_loads=32107 cas_retries=0 +[ATOMIC] submit_completion_ops=115875 fanin_ready=3473 fanin_not_ready=28634 frontier_initial=1280 frontier_flag=3228 frontier_ready_fetch_max=1948 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=991 RingBp=9 FinalDrain=24 +[KERNEL] QK count=256 mean_us=41.471 min_us=40.572 max_us=46.166 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.449 min_us=52.323 max_us=61.252 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.834 min_us=26.983 max_us=43.978 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.763 min_us=2.406 max_us=20.156 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=190 inactive_sentinel_tiles=2) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3645.710 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_06_CBA_pos1_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_06_CBA_pos1_compete-first-lazy.txt new file mode 100644 index 0000000000..9ebdbedee7 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_06_CBA_pos1_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3836.215 host_launch_us=4701.966 claims=73728 fanin_loads=32630 cas_retries=0 +[ATOMIC] submit_completion_ops=116560 fanin_ready=3473 fanin_not_ready=29157 frontier_initial=1280 frontier_flag=3309 frontier_ready_fetch_max=2029 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=33 +[PLACEMENT] EfDrain=984 RingBp=14 FinalDrain=26 +[KERNEL] QK count=256 mean_us=41.541 min_us=40.632 max_us=48.021 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.181 min_us=52.315 max_us=249.587 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.191 min_us=26.976 max_us=76.248 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.675 min_us=2.424 max_us=14.889 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3836.215 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_07_CBA_pos1_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_07_CBA_pos1_compete-first-lazy.txt new file mode 100644 index 0000000000..75d98a90b7 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_07_CBA_pos1_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3751.926 host_launch_us=4561.026 claims=73728 fanin_loads=34745 cas_retries=0 +[ATOMIC] submit_completion_ops=118835 fanin_ready=3461 fanin_not_ready=31284 frontier_initial=1280 frontier_flag=3389 frontier_ready_fetch_max=2109 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=32 +[PLACEMENT] EfDrain=983 RingBp=18 FinalDrain=23 +[KERNEL] QK count=256 mean_us=41.426 min_us=40.655 max_us=46.143 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.857 min_us=52.283 max_us=158.647 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.420 min_us=26.932 max_us=83.822 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.799 min_us=2.418 max_us=22.086 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3751.926 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_08_BAC_pos3_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_08_BAC_pos3_compete-first-lazy.txt new file mode 100644 index 0000000000..fddb40956e --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_08_BAC_pos3_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3849.866 host_launch_us=4705.073 claims=73728 fanin_loads=31809 cas_retries=0 +[ATOMIC] submit_completion_ops=115865 fanin_ready=3432 fanin_not_ready=28377 frontier_initial=1280 frontier_flag=3372 frontier_ready_fetch_max=2092 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=988 RingBp=8 FinalDrain=28 +[KERNEL] QK count=256 mean_us=41.396 min_us=40.675 max_us=46.364 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.152 min_us=52.336 max_us=255.461 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.861 min_us=27.027 max_us=63.839 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.877 min_us=2.410 max_us=27.507 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3849.866 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_09_ACB_pos2_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_09_ACB_pos2_compete-first-lazy.txt new file mode 100644 index 0000000000..9b8fb05cac --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_09_ACB_pos2_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=4370.394 host_launch_us=6211.327 claims=73728 fanin_loads=31121 cas_retries=0 +[ATOMIC] submit_completion_ops=137045 fanin_ready=5583 fanin_not_ready=25538 frontier_initial=1280 frontier_flag=14306 frontier_ready_fetch_max=13026 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=949 RingBp=59 FinalDrain=16 +[KERNEL] QK count=256 mean_us=41.752 min_us=40.580 max_us=106.953 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.672 min_us=52.029 max_us=105.339 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.060 min_us=26.973 max_us=72.875 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.683 min_us=2.010 max_us=14.212 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=4370.394 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_10_CAB_pos1_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_10_CAB_pos1_compete-first-lazy.txt new file mode 100644 index 0000000000..cd7f7fa505 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_10_CAB_pos1_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3821.598 host_launch_us=4754.349 claims=73728 fanin_loads=33447 cas_retries=0 +[ATOMIC] submit_completion_ops=117139 fanin_ready=3476 fanin_not_ready=29971 frontier_initial=1280 frontier_flag=3190 frontier_ready_fetch_max=1910 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=986 RingBp=12 FinalDrain=26 +[KERNEL] QK count=256 mean_us=41.321 min_us=40.500 max_us=48.626 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.543 min_us=52.194 max_us=231.863 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.453 min_us=26.987 max_us=79.451 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.686 min_us=2.399 max_us=14.111 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3821.598 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_11_BCA_pos2_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_11_BCA_pos2_compete-first-lazy.txt new file mode 100644 index 0000000000..438b8e3367 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_11_BCA_pos2_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3950.280 host_launch_us=5531.069 claims=73728 fanin_loads=36360 cas_retries=0 +[ATOMIC] submit_completion_ops=127192 fanin_ready=3995 fanin_not_ready=32365 frontier_initial=1280 frontier_flag=6760 frontier_ready_fetch_max=5480 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=977 RingBp=34 FinalDrain=13 +[KERNEL] QK count=256 mean_us=41.586 min_us=40.533 max_us=118.569 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.377 min_us=51.991 max_us=76.439 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.887 min_us=26.977 max_us=49.634 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.679 min_us=2.001 max_us=15.283 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3950.280 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_12_ABC_pos3_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_12_ABC_pos3_compete-first-lazy.txt new file mode 100644 index 0000000000..344199e0d7 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_12_ABC_pos3_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3869.958 host_launch_us=5215.617 claims=73728 fanin_loads=33083 cas_retries=0 +[ATOMIC] submit_completion_ops=116979 fanin_ready=3443 fanin_not_ready=29640 frontier_initial=1280 frontier_flag=3292 frontier_ready_fetch_max=2012 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=32 +[PLACEMENT] EfDrain=990 RingBp=13 FinalDrain=21 +[KERNEL] QK count=256 mean_us=41.443 min_us=40.678 max_us=50.490 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.800 min_us=52.277 max_us=276.419 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.822 min_us=27.008 max_us=37.668 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.739 min_us=2.403 max_us=15.355 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=189 inactive_sentinel_tiles=3) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3869.958 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_13_BCA_pos2_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_13_BCA_pos2_compete-first-lazy.txt new file mode 100644 index 0000000000..55c338d491 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_13_BCA_pos2_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3751.215 host_launch_us=5057.378 claims=73728 fanin_loads=35269 cas_retries=0 +[ATOMIC] submit_completion_ops=119247 fanin_ready=3495 fanin_not_ready=31774 frontier_initial=1280 frontier_flag=3333 frontier_ready_fetch_max=2053 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=33 +[PLACEMENT] EfDrain=994 RingBp=13 FinalDrain=17 +[KERNEL] QK count=256 mean_us=41.418 min_us=40.515 max_us=45.855 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.724 min_us=52.196 max_us=159.881 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.222 min_us=26.975 max_us=79.608 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.865 min_us=2.400 max_us=26.986 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3751.215 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_14_ACB_pos2_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_14_ACB_pos2_compete-first-lazy.txt new file mode 100644 index 0000000000..863ea5516d --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_14_ACB_pos2_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3848.951 host_launch_us=8279.656 claims=73728 fanin_loads=33315 cas_retries=0 +[ATOMIC] submit_completion_ops=117353 fanin_ready=3479 fanin_not_ready=29836 frontier_initial=1280 frontier_flag=3363 frontier_ready_fetch_max=2083 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=988 RingBp=12 FinalDrain=24 +[KERNEL] QK count=256 mean_us=41.558 min_us=40.512 max_us=49.152 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.152 min_us=52.205 max_us=259.218 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.363 min_us=26.979 max_us=75.985 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.750 min_us=2.419 max_us=19.549 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3848.951 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_15_CBA_pos1_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_15_CBA_pos1_compete-first-lazy.txt new file mode 100644 index 0000000000..1246875957 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_15_CBA_pos1_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3858.263 host_launch_us=4723.966 claims=73728 fanin_loads=34190 cas_retries=0 +[ATOMIC] submit_completion_ops=117926 fanin_ready=3458 fanin_not_ready=30732 frontier_initial=1280 frontier_flag=3212 frontier_ready_fetch_max=1932 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=31 +[PLACEMENT] EfDrain=986 RingBp=16 FinalDrain=22 +[KERNEL] QK count=256 mean_us=41.403 min_us=40.522 max_us=48.987 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.674 min_us=52.178 max_us=273.350 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.223 min_us=27.030 max_us=81.823 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.695 min_us=2.339 max_us=19.194 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3858.263 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_16_ABC_pos3_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_16_ABC_pos3_compete-first-lazy.txt new file mode 100644 index 0000000000..d66ee4deae --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_16_ABC_pos3_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3834.485 host_launch_us=4679.956 claims=73728 fanin_loads=34651 cas_retries=0 +[ATOMIC] submit_completion_ops=118513 fanin_ready=3431 fanin_not_ready=31220 frontier_initial=1280 frontier_flag=3275 frontier_ready_fetch_max=1995 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=35 +[PLACEMENT] EfDrain=991 RingBp=14 FinalDrain=19 +[KERNEL] QK count=256 mean_us=41.499 min_us=40.452 max_us=50.867 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.090 min_us=52.272 max_us=238.105 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.073 min_us=27.022 max_us=79.346 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.672 min_us=2.400 max_us=14.229 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=191 inactive_sentinel_tiles=1) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3834.485 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_17_CAB_pos1_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_17_CAB_pos1_compete-first-lazy.txt new file mode 100644 index 0000000000..3c413e0f30 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_17_CAB_pos1_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3742.848 host_launch_us=5150.658 claims=73728 fanin_loads=33883 cas_retries=0 +[ATOMIC] submit_completion_ops=117963 fanin_ready=3498 fanin_not_ready=30385 frontier_initial=1280 frontier_flag=3384 frontier_ready_fetch_max=2104 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=33 +[PLACEMENT] EfDrain=985 RingBp=14 FinalDrain=25 +[KERNEL] QK count=256 mean_us=41.542 min_us=40.586 max_us=46.405 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.701 min_us=52.234 max_us=161.224 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.283 min_us=26.984 max_us=80.275 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.745 min_us=2.347 max_us=20.009 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=190 inactive_sentinel_tiles=2) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3742.848 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_18_BAC_pos3_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_18_BAC_pos3_compete-first-lazy.txt new file mode 100644 index 0000000000..87d098a053 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_18_BAC_pos3_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3838.856 host_launch_us=4749.526 claims=73728 fanin_loads=36185 cas_retries=0 +[ATOMIC] submit_completion_ops=120065 fanin_ready=3451 fanin_not_ready=32734 frontier_initial=1280 frontier_flag=3284 frontier_ready_fetch_max=2004 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=991 RingBp=16 FinalDrain=17 +[KERNEL] QK count=256 mean_us=41.494 min_us=40.650 max_us=45.787 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.567 min_us=52.318 max_us=246.848 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.914 min_us=26.953 max_us=49.010 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.731 min_us=2.397 max_us=14.768 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3838.856 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_19_BAC_pos3_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_19_BAC_pos3_compete-first-lazy.txt new file mode 100644 index 0000000000..ccb720a876 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_19_BAC_pos3_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3705.379 host_launch_us=4637.708 claims=73728 fanin_loads=37895 cas_retries=0 +[ATOMIC] submit_completion_ops=123655 fanin_ready=3559 fanin_not_ready=34336 frontier_initial=1280 frontier_flag=4224 frontier_ready_fetch_max=2944 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=33 +[PLACEMENT] EfDrain=979 RingBp=26 FinalDrain=19 +[KERNEL] QK count=256 mean_us=41.487 min_us=40.531 max_us=52.958 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.722 min_us=52.346 max_us=112.765 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.151 min_us=26.997 max_us=60.507 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.760 min_us=2.419 max_us=27.149 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=189 inactive_sentinel_tiles=3) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3705.379 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_20_CAB_pos1_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_20_CAB_pos1_compete-first-lazy.txt new file mode 100644 index 0000000000..a8f024d8b0 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_20_CAB_pos1_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3762.927 host_launch_us=5109.348 claims=73728 fanin_loads=34560 cas_retries=0 +[ATOMIC] submit_completion_ops=118498 fanin_ready=3461 fanin_not_ready=31099 frontier_initial=1280 frontier_flag=3313 frontier_ready_fetch_max=2033 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=29 +[PLACEMENT] EfDrain=979 RingBp=17 FinalDrain=28 +[KERNEL] QK count=256 mean_us=41.575 min_us=40.419 max_us=84.631 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.860 min_us=52.151 max_us=169.608 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.878 min_us=26.996 max_us=49.070 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.601 min_us=2.395 max_us=3.672 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3762.927 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_21_ABC_pos3_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_21_ABC_pos3_compete-first-lazy.txt new file mode 100644 index 0000000000..9cb22b93d3 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_21_ABC_pos3_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3810.508 host_launch_us=4808.424 claims=73728 fanin_loads=31257 cas_retries=0 +[ATOMIC] submit_completion_ops=114989 fanin_ready=3449 fanin_not_ready=27808 frontier_initial=1280 frontier_flag=3210 frontier_ready_fetch_max=1930 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=34 +[PLACEMENT] EfDrain=995 RingBp=2 FinalDrain=27 +[KERNEL] QK count=256 mean_us=41.422 min_us=40.492 max_us=46.864 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.078 min_us=52.179 max_us=228.054 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.177 min_us=27.003 max_us=74.712 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.708 min_us=2.406 max_us=14.074 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=192 inactive_sentinel_tiles=0) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3810.508 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_22_CBA_pos1_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_22_CBA_pos1_compete-first-lazy.txt new file mode 100644 index 0000000000..9d4cfb5c42 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_22_CBA_pos1_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3643.199 host_launch_us=4565.537 claims=73728 fanin_loads=34212 cas_retries=0 +[ATOMIC] submit_completion_ops=117910 fanin_ready=3427 fanin_not_ready=30785 frontier_initial=1280 frontier_flag=3193 frontier_ready_fetch_max=1913 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=37 +[PLACEMENT] EfDrain=983 RingBp=17 FinalDrain=24 +[KERNEL] QK count=256 mean_us=41.404 min_us=40.632 max_us=45.977 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.219 min_us=52.227 max_us=55.603 target_us=53.729 +[KERNEL] PV count=256 mean_us=28.081 min_us=26.934 max_us=64.577 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.623 min_us=2.409 max_us=11.426 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=189 inactive_sentinel_tiles=3) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3643.199 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_23_ACB_pos2_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_23_ACB_pos2_compete-first-lazy.txt new file mode 100644 index 0000000000..f38b119b93 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_23_ACB_pos2_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3861.894 host_launch_us=4877.291 claims=73728 fanin_loads=37096 cas_retries=0 +[ATOMIC] submit_completion_ops=121970 fanin_ready=3530 fanin_not_ready=33566 frontier_initial=1280 frontier_flag=3781 frontier_ready_fetch_max=2501 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=30 +[PLACEMENT] EfDrain=979 RingBp=25 FinalDrain=20 +[KERNEL] QK count=256 mean_us=41.694 min_us=40.650 max_us=106.544 target_us=44.170 +[KERNEL] SF count=256 mean_us=54.904 min_us=52.156 max_us=255.204 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.942 min_us=26.996 max_us=42.525 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.632 min_us=2.327 max_us=14.120 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=190 inactive_sentinel_tiles=2) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3861.894 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_24_BCA_pos2_compete-first-lazy.txt b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_24_BCA_pos2_compete-first-lazy.txt new file mode 100644 index 0000000000..5702f28865 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/raw/block_24_BCA_pos2_compete-first-lazy.txt @@ -0,0 +1,47 @@ +=== Standalone PA Scheduler Benchmark: CCEC === +device=0 batches=256 tasks=1280 workers=96 runs=1 nops=129600,157900,79950,2400 state_bytes=1007111040 swimlane=off trace_atomics=off trace_bytes=0 +lazy_sample_shape=compete-first-lazy lazy_sample_shape_id=2 observation=split-combination-semantic finish=noinline-cross-tu control_family=all-task-compete-first-callback +[WINNER-WORKLOAD] mode=real-compute pattern=constant counts=6,28,4,1 unit=complete_128x128_engine_pipeline_iteration workspace_bytes=12713984 +[PMU-CONFIG] window=off calibration_scalar_nops=100000 icache_trials=64 source=direct-per-core owner=main-aicpu-path-a +[ASSERT] participant topology is 32 AIC + 64 AIV PASS +[ASSERT] all 96 worker markers and private rings are valid PASS +[ASSERT] split caller/finish share one role-specific block-local state and finish every task once PASS +[ASSERT] all Submit timing markers are valid PASS +[ASSERT] started_count is 96 PASS +[ASSERT] replay count is workers * tasks PASS +[ASSERT] Claim attempt count matches PA topology PASS +[ASSERT] exactly one winner per task PASS +[ASSERT] Alloc/QK/SF/PV/UP winners are one per batch PASS +[ASSERT] kernel count is four per batch PASS +[ASSERT] each kernel kind executes once per batch PASS +[ASSERT] AIC executes only QK/PV and AIV executes only SF/UP PASS +[ASSERT] heap guard count matches output winners PASS +[ASSERT] fanin ready/failure load classification is complete PASS +[ASSERT] frontier initial loads match completed tasks PASS +[ASSERT] frontier ready/update/terminal load identity is exact PASS +[ASSERT] completion flags are published once PASS +[ASSERT] all task flags are ready PASS +[ASSERT] all published vend values are nonzero and aligned PASS +[ASSERT] every task vend is within PA worker heap progress bounds PASS +[ASSERT] frontier reaches the final task PASS +[ASSERT] replay_done is 96 PASS +[ASSERT] fatal remains clear PASS +[ASSERT] EfDrain + RingBp + final placement covers every kernel PASS +[ASSERT] single-lane PA performs no BlockWon polling PASS +[ASSERT] every worker replays the exact PA frontend operation counts PASS +[ASSERT] global PA frontend operation totals are exact PASS +[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=24832/24832 tensor_args=297472/297472 scalar_args=2304/2304 resets=98304/98304 +[ASSERT] winner-only map, slot-copy, and fanin totals are exact PASS +[ASSERT] every worker final heap and TensorMap state is exact PASS +[ASSERT] all frontend registration checksums remain clean PASS +[ASSERT] all sharded Claim cursors reach their exact final task PASS +[METRIC] run=1 submit_span_us=3646.270 host_launch_us=4503.558 claims=73728 fanin_loads=32960 cas_retries=0 +[ATOMIC] submit_completion_ops=116958 fanin_ready=3447 fanin_not_ready=29513 frontier_initial=1280 frontier_flag=3343 frontier_ready_fetch_max=2063 frontier_terminal=1280 +[WINNERS] active_workers=96 max_wins_per_worker=34 +[PLACEMENT] EfDrain=983 RingBp=12 FinalDrain=29 +[KERNEL] QK count=256 mean_us=41.427 min_us=40.682 max_us=47.004 target_us=44.170 +[KERNEL] SF count=256 mean_us=53.688 min_us=52.389 max_us=63.279 target_us=53.729 +[KERNEL] PV count=256 mean_us=27.951 min_us=26.988 max_us=55.588 target_us=27.626 +[KERNEL] UP count=256 mean_us=2.726 min_us=2.360 max_us=18.666 target_us=1.565 +[ASSERT] real-compute output tiles match role-specific engine results PASS (active_tiles=190 inactive_sentinel_tiles=2) +[SUMMARY] runs=1 completed_runs=1 median_submit_span_us=3646.270 execution_status=PASS semantic_status=PASS postprocess_status=PASS diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/samples.tsv b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/samples.tsv new file mode 100644 index 0000000000..458492badf --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/samples.tsv @@ -0,0 +1,25 @@ +variant block order position submit_span_us included exclusion_reason source +compete-first-lazy 1 ABC 3 4084.775 False hampel_3_scaled_mad block_01_ABC_pos3_compete-first-lazy.txt +compete-first-lazy 2 BCA 2 3845.228 True block_02_BCA_pos2_compete-first-lazy.txt +compete-first-lazy 3 CAB 1 3655.535 True block_03_CAB_pos1_compete-first-lazy.txt +compete-first-lazy 4 ACB 2 3815.964 True block_04_ACB_pos2_compete-first-lazy.txt +compete-first-lazy 5 BAC 3 3645.710 True block_05_BAC_pos3_compete-first-lazy.txt +compete-first-lazy 6 CBA 1 3836.215 True block_06_CBA_pos1_compete-first-lazy.txt +compete-first-lazy 7 CBA 1 3751.926 True block_07_CBA_pos1_compete-first-lazy.txt +compete-first-lazy 8 BAC 3 3849.866 True block_08_BAC_pos3_compete-first-lazy.txt +compete-first-lazy 9 ACB 2 4370.394 False hampel_3_scaled_mad block_09_ACB_pos2_compete-first-lazy.txt +compete-first-lazy 10 CAB 1 3821.598 True block_10_CAB_pos1_compete-first-lazy.txt +compete-first-lazy 11 BCA 2 3950.280 True block_11_BCA_pos2_compete-first-lazy.txt +compete-first-lazy 12 ABC 3 3869.958 True block_12_ABC_pos3_compete-first-lazy.txt +compete-first-lazy 13 BCA 2 3751.215 True block_13_BCA_pos2_compete-first-lazy.txt +compete-first-lazy 14 ACB 2 3848.951 True block_14_ACB_pos2_compete-first-lazy.txt +compete-first-lazy 15 CBA 1 3858.263 True block_15_CBA_pos1_compete-first-lazy.txt +compete-first-lazy 16 ABC 3 3834.485 True block_16_ABC_pos3_compete-first-lazy.txt +compete-first-lazy 17 CAB 1 3742.848 True block_17_CAB_pos1_compete-first-lazy.txt +compete-first-lazy 18 BAC 3 3838.856 True block_18_BAC_pos3_compete-first-lazy.txt +compete-first-lazy 19 BAC 3 3705.379 True block_19_BAC_pos3_compete-first-lazy.txt +compete-first-lazy 20 CAB 1 3762.927 True block_20_CAB_pos1_compete-first-lazy.txt +compete-first-lazy 21 ABC 3 3810.508 True block_21_ABC_pos3_compete-first-lazy.txt +compete-first-lazy 22 CBA 1 3643.199 True block_22_CBA_pos1_compete-first-lazy.txt +compete-first-lazy 23 ACB 2 3861.894 True block_23_ACB_pos2_compete-first-lazy.txt +compete-first-lazy 24 BCA 2 3646.270 True block_24_BCA_pos2_compete-first-lazy.txt diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/summary.json b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/summary.json new file mode 100644 index 0000000000..a93663d3dc --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/performance/summary.json @@ -0,0 +1,64 @@ +{ + "configuration": { + "batches": 256, + "device": 0, + "launch_orders": "all six permutations, each repeated four times", + "launch_positions": { + "1": 8, + "2": 8, + "3": 8 + }, + "pmu_window": "off", + "runs_per_host_launch": 1, + "samples": 24, + "swimlane": "off-runtime", + "winner_workload": "real-compute", + "workers": 96 + }, + "excluded_samples": [ + { + "block": 1, + "exclusion_reason": "hampel_3_scaled_mad", + "included": false, + "order": "ABC", + "position": 3, + "source": "block_01_ABC_pos3_compete-first-lazy.txt", + "submit_span_us": 4084.775 + }, + { + "block": 9, + "exclusion_reason": "hampel_3_scaled_mad", + "included": false, + "order": "ACB", + "position": 2, + "source": "block_09_ACB_pos2_compete-first-lazy.txt", + "submit_span_us": 4370.394 + } + ], + "outlier_filtered": { + "count": 22, + "max_submit_span_us": 3950.28, + "mean_submit_span_us": 3788.503409090909, + "median_submit_span_us": 3818.781, + "min_submit_span_us": 3643.199, + "population_stdev_us": 83.86854308036808 + }, + "outlier_rule": { + "lower_us": 3590.0152591000005, + "mad_us": 53.515499999999975, + "median_us": 3828.0415000000003, + "name": "Hampel", + "raw_logs_retained": true, + "upper_us": 4066.0677409 + }, + "raw": { + "count": 24, + "max_submit_span_us": 4370.394, + "mean_submit_span_us": 3825.0935000000004, + "median_submit_span_us": 3828.0415000000003, + "min_submit_span_us": 3643.199, + "population_stdev_us": 151.24318121103295 + }, + "schema": "pa_lazy_lambda_variant_independent_performance/v1", + "variant": "compete-first-lazy" +} diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/published.sha256 b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/published.sha256 new file mode 100644 index 0000000000..09726aaee2 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/output/published.sha256 @@ -0,0 +1,29 @@ +198c1dae69443432bda7acf9654b8cd0aa13f7400d4cb4eacc7661d8314c930f output/functional.txt +181e940f6f7ef426807815f067a04644f16b47b9d2ea34efa84b56ee6348e3bc output/performance/RUN_METADATA.md +819224c10dfc33a19987a0336c1f65aca896519ad4dece2f800985b1fe78672d output/performance/SUMMARY.md +b7e34eeb01420d8cbf7c62a759d872b714427fa66f89152e5d2ea4786cdf3f73 output/performance/raw/block_01_ABC_pos3_compete-first-lazy.txt +f8ca39b89944c54446b1ffe75e7a7ba3d396d8bd69414c0b7e757e5430a81cc8 output/performance/raw/block_02_BCA_pos2_compete-first-lazy.txt +67289429873f280a368a584a09dff24acaf10a48a7cefe0739878cd8b190e0a4 output/performance/raw/block_03_CAB_pos1_compete-first-lazy.txt +bb3d5dd33b41ce1a0414071e8c68959f6ab286cdb285b2280645fcff81c1058b output/performance/raw/block_04_ACB_pos2_compete-first-lazy.txt +981d32b6faa618eb1f36ffa304dd032887422c520276e6b87636ce45f4e4b6ed output/performance/raw/block_05_BAC_pos3_compete-first-lazy.txt +a2618a2b8a9a07ef1c662f0f2d5bc5fb12df08bf3c094bc98f01559a59dcd650 output/performance/raw/block_06_CBA_pos1_compete-first-lazy.txt +705d06ec8ff01376ab5ece572bf45a13102a1aacee10f8955559bc44b7fec4f5 output/performance/raw/block_07_CBA_pos1_compete-first-lazy.txt +8525a1261cfd255b20251f73ef591c9aa7c82b06a44af97405746cda7dbb814b output/performance/raw/block_08_BAC_pos3_compete-first-lazy.txt +4071b68c44ec5204db83600fadaada9b20a64816ad6fbc949d3a23ad27955461 output/performance/raw/block_09_ACB_pos2_compete-first-lazy.txt +82d5a5292c1289c4b7a81fa426b6d8a308335b5dcaa86c0f5bd4dd4aea010229 output/performance/raw/block_10_CAB_pos1_compete-first-lazy.txt +c9deb5b5086ad4e86332c327540e76b149a9114db07029888900d9e2cb98525e output/performance/raw/block_11_BCA_pos2_compete-first-lazy.txt +f99773915f09333be845a74511801d207c7e3b17a6dc44eff28ed7a4a192772d output/performance/raw/block_12_ABC_pos3_compete-first-lazy.txt +d8d9e78d6d1fed705ecd7e95be12d5806ce0469670a47aae404833e4aede820e output/performance/raw/block_13_BCA_pos2_compete-first-lazy.txt +3a9adce969e72517d1f271a42d42dc8f9e25401e7a98d14e7f3253532823e172 output/performance/raw/block_14_ACB_pos2_compete-first-lazy.txt +1926e06aef95c4f29f3156638540a4ca1294c08bf7bad8eac2e9fd9525da0a77 output/performance/raw/block_15_CBA_pos1_compete-first-lazy.txt +b4b0010d30ef67c0c37042780e29a9287427fdfc3694b6473b41bed8ac4339af output/performance/raw/block_16_ABC_pos3_compete-first-lazy.txt +e3d494ed59449ffdfa2430f63e1f6ec33dc35d98484bd0d88c3b0a6647b30981 output/performance/raw/block_17_CAB_pos1_compete-first-lazy.txt +9ed141ad0e390c168158363d65603f5ca61ebd562db6485cbb1c549a68eab51b output/performance/raw/block_18_BAC_pos3_compete-first-lazy.txt +e974874695446193691e474262374cd5116c472ddaa98198924a04056ed0ad42 output/performance/raw/block_19_BAC_pos3_compete-first-lazy.txt +24ea883845f30ceaad5ffab804a213f61dfb3b798c413066f313f3dbdebf91b4 output/performance/raw/block_20_CAB_pos1_compete-first-lazy.txt +2a780b3e4259c387ba708d717e4ea6208725912c15850a4644eee025ab7aaeff output/performance/raw/block_21_ABC_pos3_compete-first-lazy.txt +3b04c805efe3ab3e53172dc9a110c5584edec42af838be701271fd30ec9d7fd3 output/performance/raw/block_22_CBA_pos1_compete-first-lazy.txt +c908b48133b89b6b73928a66d1957bdc2268f82fa30a59bc7296885d4b9a49e4 output/performance/raw/block_23_ACB_pos2_compete-first-lazy.txt +02069577e750eb9113d3ad6853cff5448cb4d19c978eb48e9b245cd8a4e9e0bb output/performance/raw/block_24_BCA_pos2_compete-first-lazy.txt +5becdee926c6c1922bdbf7d583d56578a9b045fd5e5aed8800bf400af0704e1a output/performance/samples.tsv +0bad46524c2e2c16184e880bb17e986d7ff84095933c11df5b7e0216e74c54e8 output/performance/summary.json diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/publish_artifacts.sh b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/publish_artifacts.sh new file mode 100755 index 0000000000..7bcc4bb9c0 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/publish_artifacts.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SOURCE_DIR="$SCRIPT_DIR/build/compete-first-lazy" +TARGET_DIR="$SCRIPT_DIR/artifacts/rebuilt" +mkdir -p "$TARGET_DIR" + +for name in pa_scheduler_host pa_scheduler_kernel.o device_text_layout.manifest artifacts.manifest; do + if [[ ! -f "$SOURCE_DIR/$name" ]]; then + echo "Missing clean-build result: $SOURCE_DIR/$name" >&2 + exit 1 + fi + cp -f -- "$SOURCE_DIR/$name" "$TARGET_DIR/$name" +done +(cd "$TARGET_DIR" && sha256sum artifacts.manifest device_text_layout.manifest pa_scheduler_host pa_scheduler_kernel.o) \ + > "$TARGET_DIR/published.sha256" diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/run_functional.sh b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/run_functional.sh new file mode 100755 index 0000000000..9b12861532 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/run_functional.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +BUILD_DIR="$SCRIPT_DIR/build/compete-first-lazy" +OUTPUT="$SCRIPT_DIR/output/functional.txt" + +if [[ ! -x "$BUILD_DIR/pa_scheduler_host" || ! -f "$BUILD_DIR/pa_scheduler_kernel.o" ]]; then + echo "Missing clean build; run ./build.sh first." >&2 + exit 1 +fi +"$BUILD_DIR/pa_scheduler_host" \ + --kernel "$BUILD_DIR/pa_scheduler_kernel.o" \ + --device 0 --batches 1 --runs 1 \ + --winner-workload real-compute --pmu-window off --no-swimlane | tee "$OUTPUT" +if ! grep -Eq '^\[SUMMARY\].*completed_runs=1.*execution_status=PASS semantic_status=PASS postprocess_status=PASS$' "$OUTPUT"; then + echo "Functional oracle failed for compete-first-lazy" >&2 + exit 1 +fi +if ! grep -Fq '[LAZY_SAMPLE_FRONTEND] shape=compete-first-lazy views=97/97 tensor_args=1162/1162 scalar_args=9/9 resets=384/384' "$OUTPUT"; then + echo "Lazy frontend-count oracle failed for compete-first-lazy" >&2 + exit 1 +fi diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/source.sha256 b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/source.sha256 new file mode 100644 index 0000000000..a86cc0f8bf --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/source.sha256 @@ -0,0 +1,30 @@ +909eff5ca6e3f88d5b077945077864a1557549a3e171bcf233aa236dbd96d660 ./annotate_disassembly.py +aaf4d34a6551fa7c6a95c0942ab273e4b94576e846fc8265848e24d524b6b2cb ./build.sh +4091251537a0405f83514d99fc6891300650a01cb61ac3b6c2291e899f7f0747 ./disassemble.py +b478bf22959d68f3ea62772cd1e39284eb1abf280ccb47b7ae594441d5badf12 ./generate_disassembly.sh +067ffc84e28fe7385ae163ea2db7e55efd2e9b6d8943f239e12db2ead07a042b ./publish_artifacts.sh +29bfe64ba516d0df208f0738682b72d43d093c335ecc74cb5f674272c9365236 ./run_functional.sh +72471b0594aee1069f9fc774faa6d5900e374d3a283fdccc12b2c26a1302472a ./summarize_performance.py +e50378f7afdb15e3a84207b029e2d0aced10e1ac5a363372a791a0a8d177c937 ./verify_runtime_identity.py +34b058acc11dde4072bea0a43243b7ea2d63625370011491697090dda256eeb9 README.md +c554b42c0f521da8e4a14249b078d5b816fa1bfafd9c7d461490bd787e1a0abc VARIANT +e5732a50ddf852b98e23a89b3b48276cc1874ae70242a3a016440421782c6cb4 ccec/build.sh +ea82967763894d2726b68b19e74c2993ffe938f5bb0c90b92d3eae0966e643b7 ccec/callback_finish.cpp +7090c8405e1602476d1edf4ae6103ec6ec145e22bf15db742eac06aa2a6b7f9f ccec/callback_finish_api.h +4cef27650943d3973c6d56db687850ea3a7cbe28d64bf666b55cf735ce5bd141 ccec/callback_runtime_entry.cpp +47ca7623791bdd408b50c5e5ece4d63c7073775a1e3b636e046419acf948ddda ccec/ccec_ops.h +ac8a9766aa71a2bfc2a21c9021fc9fc30bc869575daa0e0f01121789230173e2 ccec/host.cpp +766ec34d6d4a7d80d07774782c68d701653cadf1bd8427fd1a0e922933c10236 ccec/kernel.cpp +f88d5ed31277c2692ecec2e8f5528cb6c2de752aea6a45850ab72d9e9b600075 ccec/pa_scheduler_device_exports.map +3d65b4a89dcf88554321a5435a2abf701a26757e8c229df7ac9783abbb5b96c3 ccec/pmu_owner_control.h +cb81c2a24670d893fd485845f13b8b106bfffe28d8c471f92a93775eee1a69e6 ccec/pmu_owner_host.h +09263e6698598f7be5b16b92e9e34bc59719de54aee132c43df24cec0ce42b4a ccec/pmu_owner_main_abi.h +9115d77d1be306c9f3b225805ed3df63fbdbdadcb9b4ff6ad690b56cfcf58867 ccec/pmu_owner_main_loader.h +2a926b355f78d4758fdc61a54827baec54ef781df34234a2edc9aa6a1a824f9e ccec/pmu_probe.h +3e3c6c99df27ff9e329242c64680559bf69cb4dc96650acc0ba1fe7442d68c70 common/host_support.h +f4a22840aef81ec05fb9532559957712c76ebd2dd1199d6969fc99688fc2e63d common/pa_frontend.h +9d42688f933e7b20eaab9eadcec40c3f0c0babc8ebdd3dd7369a2afcca1edbec common/pa_model.h +6645ce2aefe39b6132ce649c6af4c597a7a6978aa8983375fe09c4e186a812e5 common/pa_scheduler_core.h +8fd7c429dbbf0aae4e37968e7f90d953793f30f9caeb18eba515f9201deca1fa common/pa_trace.h +9d099151d0303c37dbb88f5cd2b15ee483dc0623759c45cbcf9d80c37704a80d common/winner_workload.h +8be384a62ed669e5645139aab1528570ffab184b5856f685e8056eed68e1e5cc common/winner_workload_host.h diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/summarize_performance.py b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/summarize_performance.py new file mode 100755 index 0000000000..08a98ae100 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/summarize_performance.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Validate this standalone package's 24 independent b256 measurements.""" + +from __future__ import annotations + +import json +import re +import statistics +from pathlib import Path + + +FILE_RE = re.compile( + r"block_(?P\d{2})_(?P[ABC]{3})_pos(?P[123])_" + r"(?Poriginal|compete-first|compete-first-lazy)\.txt$" +) +CONFIG_RE = re.compile( + r"^device=0 batches=256 tasks=1280 workers=96 runs=1 .*\bswimlane=off\b", + re.MULTILINE, +) +METRIC_RE = re.compile(r"^\[METRIC\] run=1 submit_span_us=([0-9.]+)\b", re.MULTILINE) +SUMMARY_RE = re.compile( + r"^\[SUMMARY\] runs=1 completed_runs=1 median_submit_span_us=([0-9.]+) " + r"execution_status=PASS semantic_status=PASS postprocess_status=PASS$", + re.MULTILINE, +) +HAMPEL_SCALE = 1.4826 +HAMPEL_SIGMAS = 3.0 + + +def fail(message: str) -> None: + raise SystemExit(message) + + +def describe(values: list[float]) -> dict[str, float | int]: + return { + "count": len(values), + "median_submit_span_us": statistics.median(values), + "mean_submit_span_us": statistics.fmean(values), + "min_submit_span_us": min(values), + "max_submit_span_us": max(values), + "population_stdev_us": statistics.pstdev(values), + } + + +def main() -> None: + script_dir = Path(__file__).resolve().parent + variant = (script_dir / "VARIANT").read_text(encoding="utf-8").strip() + result_dir = script_dir / "output" / "performance" + paths = sorted((result_dir / "raw").glob("*.txt")) + if len(paths) != 24: + fail(f"{variant}: expected 24 independent raw files, found {len(paths)}") + + rows: list[dict[str, object]] = [] + for path in paths: + name_match = FILE_RE.fullmatch(path.name) + if name_match is None or name_match.group("variant") != variant: + fail(f"{variant}: unexpected raw filename {path.name}") + text = path.read_text(encoding="utf-8") + if CONFIG_RE.search(text) is None: + fail(f"{path}: fixed device0/b256/runs=1/no-swimlane config missing") + if "[WINNER-WORKLOAD] mode=real-compute " not in text: + fail(f"{path}: real-compute workload missing") + if "[PMU-CONFIG] window=off " not in text: + fail(f"{path}: PMU is not explicitly off") + metrics = METRIC_RE.findall(text) + summaries = SUMMARY_RE.findall(text) + if len(metrics) != 1 or len(summaries) != 1 or metrics[0] != summaries[0]: + fail(f"{path}: expected one matching METRIC and PASS SUMMARY") + rows.append( + { + "block": int(name_match.group("block")), + "order": name_match.group("order"), + "position": int(name_match.group("position")), + "submit_span_us": float(metrics[0]), + "source": path.name, + } + ) + + if {int(row["block"]) for row in rows} != set(range(1, 25)): + fail(f"{variant}: blocks must be exactly 01..24") + order_counts = { + order: sum(row["order"] == order for row in rows) + for order in ("ABC", "ACB", "BAC", "BCA", "CAB", "CBA") + } + if set(order_counts.values()) != {4}: + fail(f"{variant}: six orders are not each repeated four times: {order_counts}") + position_counts = { + position: sum(row["position"] == position for row in rows) + for position in (1, 2, 3) + } + if set(position_counts.values()) != {8}: + fail(f"{variant}: launch positions are not 8/8/8: {position_counts}") + + raw_values = [float(row["submit_span_us"]) for row in rows] + center = statistics.median(raw_values) + mad = statistics.median(abs(value - center) for value in raw_values) + if mad == 0.0: + fail(f"{variant}: zero MAD cannot define an outlier threshold") + lower = center - HAMPEL_SIGMAS * HAMPEL_SCALE * mad + upper = center + HAMPEL_SIGMAS * HAMPEL_SCALE * mad + for row in rows: + value = float(row["submit_span_us"]) + row["included"] = lower <= value <= upper + row["exclusion_reason"] = "" if row["included"] else "hampel_3_scaled_mad" + clean_values = [ + float(row["submit_span_us"]) for row in rows if bool(row["included"]) + ] + + document = { + "schema": "pa_lazy_lambda_variant_independent_performance/v1", + "variant": variant, + "configuration": { + "device": 0, + "workers": 96, + "batches": 256, + "samples": 24, + "runs_per_host_launch": 1, + "launch_orders": "all six permutations, each repeated four times", + "launch_positions": {"1": 8, "2": 8, "3": 8}, + "pmu_window": "off", + "swimlane": "off-runtime", + "winner_workload": "real-compute", + }, + "outlier_rule": { + "name": "Hampel", + "median_us": center, + "mad_us": mad, + "lower_us": lower, + "upper_us": upper, + "raw_logs_retained": True, + }, + "raw": describe(raw_values), + "outlier_filtered": describe(clean_values), + "excluded_samples": [row for row in rows if not bool(row["included"])], + } + (result_dir / "summary.json").write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + with (result_dir / "samples.tsv").open("w", encoding="utf-8") as output: + output.write( + "variant\tblock\torder\tposition\tsubmit_span_us\tincluded\t" + "exclusion_reason\tsource\n" + ) + for row in sorted(rows, key=lambda item: int(item["block"])): + output.write( + f"{variant}\t{row['block']}\t{row['order']}\t{row['position']}\t" + f"{float(row['submit_span_us']):.3f}\t{row['included']}\t" + f"{row['exclusion_reason']}\t{row['source']}\n" + ) + raw = document["raw"] + clean = document["outlier_filtered"] + lines = [ + f"# {variant} device0 b256 result", + "", + "24 independent host launches, one device run per launch; PMU off, runtime swimlane off, real-compute.", + "Raw logs are retained. The primary row excludes samples outside median ± 3×1.4826×MAD.", + "", + "| View | N | Median (us) | Mean (us) | Min (us) | Max (us) | Stddev (us) |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + f"| Raw | {raw['count']} | {raw['median_submit_span_us']:.3f} | " + f"{raw['mean_submit_span_us']:.3f} | {raw['min_submit_span_us']:.3f} | " + f"{raw['max_submit_span_us']:.3f} | {raw['population_stdev_us']:.3f} |", + f"| Outlier-filtered | {clean['count']} | {clean['median_submit_span_us']:.3f} | " + f"{clean['mean_submit_span_us']:.3f} | {clean['min_submit_span_us']:.3f} | " + f"{clean['max_submit_span_us']:.3f} | {clean['population_stdev_us']:.3f} |", + "", + f"Excluded samples: {raw['count'] - clean['count']}.", + ] + (result_dir / "SUMMARY.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + print("\n".join(lines)) + + +if __name__ == "__main__": + main() diff --git a/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/verify_runtime_identity.py b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/verify_runtime_identity.py new file mode 100755 index 0000000000..786fcc487f --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/C_compete_first_lazy/verify_runtime_identity.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Prove that measured and local-rebuild AICore ELFs have identical runtime content.""" + +from __future__ import annotations + +import hashlib +import json +import struct +from dataclasses import dataclass +from pathlib import Path + + +SHT_NOBITS = 8 +SHT_SYMTAB = 2 +STT_FUNC = 2 +STT_OBJECT = 1 + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def c_string(table: bytes, offset: int) -> str: + end = table.find(b"\0", offset) + if end < 0: + raise ValueError("unterminated ELF string") + return table[offset:end].decode("utf-8") + + +@dataclass(frozen=True) +class Section: + index: int + name: str + section_type: int + flags: int + address: int + offset: int + size: int + link: int + alignment: int + entry_size: int + + +class Elf64: + def __init__(self, path: Path): + self.path = path + self.data = path.read_bytes() + header = struct.unpack_from("<16sHHIQQQIHHHHHH", self.data, 0) + if header[0][:6] != b"\x7fELF\x02\x01": + raise ValueError(f"not ELF64 little-endian: {path}") + section_offset, section_size, count, names_index = header[6], header[11], header[12], header[13] + if section_size != 64: + raise ValueError(f"unexpected section header size: {section_size}") + raw = [ + struct.unpack_from(" bytes: + if section.section_type == SHT_NOBITS: + return b"" + return self.data[section.offset : section.offset + section.size] + + def runtime_symbols(self) -> dict[str, tuple[int, int, int, int]]: + symtab = next(section for section in self.sections if section.section_type == SHT_SYMTAB) + strings = self.content(self.sections[symtab.link]) + symbols: dict[str, tuple[int, int, int, int]] = {} + for offset in range(symtab.offset, symtab.offset + symtab.size, symtab.entry_size): + name_offset, info, _, section_index, value, size = struct.unpack_from( + "= len(self.sections): + continue + section = self.sections[section_index] + if section.name not in {".text", ".rodata", ".bl_uninit"}: + continue + name = c_string(strings, name_offset) + if name: + symbols[name] = (symbol_type, info >> 4, value, size) + return symbols + + +def main() -> None: + script_dir = Path(__file__).resolve().parent + measured_dir = script_dir / "artifacts" / "measured" + rebuilt_dir = script_dir / "artifacts" / "rebuilt" + measured = Elf64(measured_dir / "pa_scheduler_kernel.o") + rebuilt = Elf64(rebuilt_dir / "pa_scheduler_kernel.o") + + content_sections = [ + ".text", + ".rodata", + ".ascend.meta.pa_scheduler_0_mix_aic", + "__CCE_KernelArgSize", + ".ascend.meta.pa_scheduler_0_mix_aiv", + ] + section_rows = [] + all_ok = True + for name in content_sections: + left = measured.by_name[name] + right = rebuilt.by_name[name] + metadata_equal = ( + left.section_type, left.flags, left.address, left.size, left.alignment + ) == ( + right.section_type, right.flags, right.address, right.size, right.alignment + ) + content_equal = measured.content(left) == rebuilt.content(right) + all_ok &= metadata_equal and content_equal + section_rows.append( + { + "name": name, + "size": left.size, + "sha256": digest(measured.content(left)), + "metadata_equal": metadata_equal, + "content_equal": content_equal, + } + ) + + nobits_rows = [] + for name in [".bl_uninit"]: + if name not in measured.by_name and name not in rebuilt.by_name: + continue + if name not in measured.by_name or name not in rebuilt.by_name: + all_ok = False + nobits_rows.append({"name": name, "layout_equal": False}) + continue + left = measured.by_name[name] + right = rebuilt.by_name[name] + layout_equal = ( + left.section_type, left.flags, left.address, left.size, left.alignment + ) == ( + right.section_type, right.flags, right.address, right.size, right.alignment + ) + all_ok &= layout_equal and left.section_type == SHT_NOBITS + nobits_rows.append( + { + "name": name, + "type": "SHT_NOBITS", + "size": left.size, + "alignment": left.alignment, + "layout_equal": layout_equal, + } + ) + + measured_symbols = measured.runtime_symbols() + rebuilt_symbols = rebuilt.runtime_symbols() + symbols_equal = measured_symbols == rebuilt_symbols + all_ok &= symbols_equal + measured_host = (measured_dir / "pa_scheduler_host").read_bytes() + rebuilt_host = (rebuilt_dir / "pa_scheduler_host").read_bytes() + host_equal = measured_host == rebuilt_host + all_ok &= host_equal + + document = { + "schema": "pa_lazy_lambda_runtime_identity/v1", + "variant": (script_dir / "VARIANT").read_text(encoding="utf-8").strip(), + "status": "PASS" if all_ok else "FAIL", + "measured_elf_sha256": digest(measured.data), + "rebuilt_elf_sha256": digest(rebuilt.data), + "full_elf_equal": measured.data == rebuilt.data, + "runtime_content_sections": section_rows, + "nobits_layout": nobits_rows, + "runtime_symbols_equal": symbols_equal, + "runtime_symbol_count": len(measured_symbols), + "host_equal": host_equal, + "host_sha256": digest(measured_host), + "note": "Measured is the exact performance binary; full ELF and runtime identities are checked explicitly.", + } + output = script_dir / "artifacts" / "runtime_identity.json" + output.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8") + if not all_ok: + raise SystemExit(f"runtime identity check failed; see {output}") + print( + f"PASS variant={document['variant']} runtime_sections={len(section_rows)} " + f"runtime_symbols={len(measured_symbols)} host_sha256={document['host_sha256']}" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/atomic_probe/lazy_lamda_sample/DISASSEMBLY_GUIDE.md b/tests/atomic_probe/lazy_lamda_sample/DISASSEMBLY_GUIDE.md new file mode 100644 index 0000000000..99cdfa7591 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/DISASSEMBLY_GUIDE.md @@ -0,0 +1,97 @@ +# final-linked 源码注释反汇编说明 + +## 交付内容 + +每份 A/B/C 包都独立保存以下文件： + +- `artifacts/measured/pa_scheduler_kernel.o`：正式性能测试实际使用的 final ELF； +- `disassembly/raw/*.asm.gz`：每个非空 `.text` `STT_FUNC` 的完整真实机器码解码； +- `disassembly/annotated/*.source.asm.gz`：对应函数的完整源码注释版； +- `disassembly/key_flow/*.source.asm`：未压缩的关键 Submit/lazy 阅读窗口； +- `disassembly/raw/manifest.tsv` 与 `gaps.tsv`：函数覆盖和 alignment/padding 缺口； +- `disassembly/published.sha256`：raw、annotated 和 key-flow 文件哈希。 + +最适合先打开的四个文件是： + +```text +A_original/disassembly/key_flow/aic_original_submit.source.asm +B_compete_first/disassembly/key_flow/aic_compete_first.source.asm +C_compete_first_lazy/disassembly/key_flow/aic_compete_first_lazy.source.asm +C_compete_first_lazy/disassembly/key_flow/aic_lazy_input_policy.source.asm +``` + +最后一个文件直接包含 `if constexpr (Lazy) { if (!won_) return; }` 的本地源码、原始注释、DWARF +定位和相邻机器指令。 + +## 真实解码链路 + +`disassemble.py` 不把十六进制或中间 IR 冒充成反汇编。它执行以下可审计步骤： + +1. 直接解析 final ELF64 little-endian section/symbol table； +2. 只选择 final `.text` 中 size 非零的 `STT_FUNC`，按最终 PC 和符号 size 截取真实函数体； +3. 使用 CANN 9.1 A5 `dav_3510` 随包解码器 + `$ASCEND_HOME_PATH/x86_64-linux/simulator/dav_3510/lib/libpem_davinci.so`； +4. 固定并校验解码器 SHA256： + `29835d2439d6dd464d34a212ad4bbd5c29af6a38465da09a6c273401d9a96dcb`； +5. 按 4/8-byte 真实编码逐字节核对 decoder 输出与 ELF 原始字节； +6. 对 RVec `.vector.thread` 使用显式 full-body range； +7. 拒绝 `UNDEF/UNKNOWN/INVALID/ILLEGAL` 等无效 mnemonic； +8. 把函数符号之外的 alignment/padding 按地址、长度和 SHA256 记录到 `gaps.tsv`，不冒充指令。 + +当前完整解码覆盖如下： + +| 形态 | 函数数 | 指令数 | 函数字节 / `.text` | +| --- | ---: | ---: | ---: | +| A | 11 | 194923 | 779700 / 780344 B（99.917%） | +| B | 17 | 136792 | 547176 / 547640 B（99.915%） | +| C | 17 | 136810 | 547248 / 547640 B（99.928%） | + +## 源码和原注释是如何加入的 + +`annotate_disassembly.py` 把 raw 文件中的每个 final PC 批量交给 GNU `addr2line -e -C`， +取得 DWARF 文件/行映射，再把 DWARF 中的编译期绝对路径映射到该包自己的 `common/` 或 `ccec/`， +读取对应源码行。 +因此输出中的普通代码和 `//` 注释确实来自随包 `.h/.cpp`，而不是人工重写的伪代码。 + +证据边界必须同时保留： + +- DWARF 只证明地址对应到某个文件/行； +- `# [DWARF] file:line` 是定位结果； +- `# > line | source` 及其上下文是脚本从本地源码复制的文本； +- 源码注释不是 ELF 内保存的注释，也不能把每条上下文注释宣称为精确拥有下一条机器指令； +- raw 机器码/助记符才是静态 codegen 的权威内容。 + +每个 annotated 文件头都重复写明这一边界，避免脱离本说明后被误读。 + +## 阅读和复现 + +查看完整压缩文件： + +```bash +zless disassembly/annotated/.source.asm.gz +zless disassembly/raw/.asm.gz +``` + +在任一变体目录内，source CANN 9.1 环境后可独立重建： + +```bash +./generate_disassembly.sh +sha256sum -c disassembly/published.sha256 +``` + +`generate_disassembly.sh` 只清空该变体自己的 `disassembly/raw` 和 `disassembly/annotated`，并始终 +从 `artifacts/measured/pa_scheduler_kernel.o` 生成全量和 key-flow 结果，以保证反汇编与正式性能日志对应。反汇编可以 +说明 codegen/布局差异，不能单独证明性能差异由 I-cache 引起；当前性能测试明确使用 PMU off， +没有 I-cache PMU 数据。 + +## 与性能复测的关系 + +当前带源码注释的反汇编是从三份包内的 `measured` final ELF 重新生成的，完整 ELF SHA256 分别为： + +- A：`76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc`； +- B：`82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b`； +- C：`8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893`。 + +24 样本/版的性能复测直接执行同一组 clean-build host/final ELF，没有修改源码或重新链接；因此 +本次只更新性能日志和统计，不需要改反汇编。若任一源码或 ELF 哈希变化，必须重新运行对应包的 +`./generate_disassembly.sh`。 diff --git a/tests/atomic_probe/lazy_lamda_sample/MANIFEST.sha256 b/tests/atomic_probe/lazy_lamda_sample/MANIFEST.sha256 new file mode 100644 index 0000000000..5d3f7aeaac --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/MANIFEST.sha256 @@ -0,0 +1,9 @@ +66137a5e7b7fffe592ad12b83f85206821d27705e7af5961bb3dd14769944d34 README.md +8814408fe943fd00b0e50e24cc463ebbb921d010b392ff6971b345182e80ebbc PERFORMANCE_COMPARISON.md +8d3565a10eacfedf950eeb233ef7d44ca894b5ccab25d3fc9d2b5f81fa6cb851 DISASSEMBLY_GUIDE.md +47ba199f41932770af5cac462cbe4cd9561b90d8f132f1b665401d03d36c5fbf analyze_performance.py +b14b3a8ec95120b7f66d539c24fb9e4438bb20ee0e74c608188e7b8959f0d449 performance_summary.json +c611ce898924a2e1e628e73384537e5a97f3b7b27f8f41777455d762536d7153 performance_samples.tsv +78d73f7fe5755178de4da5b6b597a1a36c772f8aaceb2357e938ddf58db48844 A_original/MANIFEST.sha256 +e35847d2cb201f65b82176d4d2a25905ca4a4e71fcb25f0a26f046ad2424b626 B_compete_first/MANIFEST.sha256 +9ac53b388c718a9f709ec4204fe9e4411b329c183eb78c410225b3dffc74d426 C_compete_first_lazy/MANIFEST.sha256 diff --git a/tests/atomic_probe/lazy_lamda_sample/PERFORMANCE_COMPARISON.md b/tests/atomic_probe/lazy_lamda_sample/PERFORMANCE_COMPARISON.md new file mode 100644 index 0000000000..d7d72d1569 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/PERFORMANCE_COMPARISON.md @@ -0,0 +1,122 @@ +# A/B/C 独立运行性能对比 + +## 结论 + +最终口径是 72 次独立 host 启动：A/B/C 各 24 个 device 样本，每次均为 `--runs 1`。 +去除统一 Hampel 规则识别的异常后，lazy 的直接对照 C-B 有两个读数： + +- 不考虑采集时间配对的总体中位差为 `+74.364 us / +1.986%`； +- 以相邻 A/B/C 排列块抵消时间漂移后，中位差只有 `+1.503 us / +0.040%`，C 与 B + 各自更快 `11/22` 个有效块。 + +因此，本批证据不支持 lazy lambda 存在稳定的性能收益或回退。C-B 远低于 5% 门槛，不启动 +I-cache PMU 对比，也不把静态 `.text` 差异写成 I-cache 因果。 + +B-A 的变化则稳定：去异常配对中 B 快 `206.270 us / 5.214%`，且 B 在 `22/22` 个有效块更快。 +但 A-B 同时包含 compete-first 流程、split finish/outlining 和代码布局变化，不能称为 lazy lambda +收益。C 相对 A 的去异常配对中位收益为 `144.819 us / 3.660%`。 + +## 固定测量口径 + +- 采集时间：2026-07-20 07:02:23–07:12:45 UTC； +- CCEC device0，32 AIC + 64 AIV，共 96 worker； +- 完整 `Alloc -> QK -> SF -> PV -> UP` task 流； +- b256、real-compute； +- `PA_BUILD_SUBMIT_PMU=0`，每份原始输出均显示 `[PMU-CONFIG] window=off`； +- ELF 以 `PA_BUILD_SWIMLANE=1` 构建，运行时显式 `--no-swimlane`； +- 每次 host 启动只做一个 device run，避免在同一进程中混合冷/暖运行状态； +- 六种 A/B/C 启动排列各重复四次，每版启动位置 1/2/3 均为 8/8/8； +- 72 次 host 启动全部通过 execution、semantic 和 postprocess 门禁。 + +采集 shell 中 `task-submit` 与 `npu-smi` 均不可用。依据用户明确授权直接访问 device0，本次并非 +队列隔离运行；因此可以证明配置一致和程序门禁通过，不能证明采集期间没有外部负载干扰。 + +## 异常值规则 + +原始日志不删除。统计前按每个版本自己的 24 个样本固定使用 Hampel 规则： + +`中位数 ± 3 × 1.4826 × MAD` + +越界样本不进入“去异常”总体统计；配对比较中，只要 A/B/C 对应成员之一越界，该比较涉及的 +整个排列块就不进入相应配对统计。规则和逐样本 `included` 标记可由 +`./analyze_performance.py` 复算。 + +| 形态 | Hampel 下界 / 上界 | 排除样本 | +| --- | ---: | --- | +| A original | 3854.127 / 4055.564 us | block11 `3832.087`、block23 `3788.296` us | +| B compete-first eager | 3338.151 / 4150.683 us | 无 | +| C compete-first + lazy | 3590.015 / 4066.068 us | block01 `4084.775`、block09 `4370.394` us | + +## Submit span + +| 形态 | 原始 N / 中位数 | 去异常 N / 中位数 | 去异常均值 | 去异常 min / max | 去异常标准差 | +| --- | ---: | ---: | ---: | ---: | ---: | +| A original | 24 / 3954.845 us | 22 / 3956.298 us | 3941.249 us | 3856.072 / 4000.058 us | 38.004 us | +| B compete-first eager | 24 / 3744.417 us | 24 / 3744.417 us | 3747.214 us | 3625.727 / 3866.154 us | 81.935 us | +| C compete-first + lazy | 24 / 3828.042 us | 22 / 3818.781 us | 3788.503 us | 3643.199 / 3950.280 us | 83.869 us | + +| 对比 | 去异常总体中位差 | 去异常配对中位差 | 候选更快块数 | 被排除块 | +| --- | ---: | ---: | ---: | --- | +| B - A | -211.881 us / -5.356% | -206.270 us / -5.214% | B 22/22 | 11、23 | +| C - B（lazy） | +74.364 us / +1.986% | +1.503 us / +0.040% | C 11/22 | 01、09 | +| C - A（总变化） | -137.517 us / -3.476% | -144.819 us / -3.660% | C 20/20 | 01、09、11、23 | + +负值表示候选版本更快。配对百分比使用对应 baseline 的去异常总体中位数作为分母。 + +## 为什么不配对时看起来差很多 + +设备状态在约十分钟采集窗口内并非平稳。即使六种排列和位置完全平衡，三个版本在采集前后半段 +仍发生不同方向的漂移： + +| 形态 | block01–12 原始中位数 | block13–24 原始中位数 | 后半 - 前半 | +| --- | ---: | ---: | ---: | +| A | 3939.252 us | 3956.298 us | +17.046 us | +| B | 3668.446 us | 3788.252 us | +119.806 us | +| C | 3840.722 us | 3786.717 us | -54.004 us | + +相应地，C-B 的原始配对中位差在前半段为 `+101.951 us`，后半段变成 `-6.089 us`；合并 24 块 +后为 `+8.065 us`，统一去异常后为 `+1.503 us`。这说明不按相邻时间块比较时，B 后半段变慢、 +C 后半段变快会把总体中位数拉成一个看似明确的 `+74.364 us` 差异。 + +明显长尾也有可见的动态调度特征。C 的两个高异常样本中，`frontier_flag` 中位数为 `12969`、 +`RingBp` 中位数为 `53`；C 的保留样本对应值只有 `3322.5` 和 `14`。C 内部 Submit span 与 +`frontier_flag`、`RingBp` 的 Pearson 相关系数分别为 `0.860`、`0.797`。A 的两个低异常样本则 +伴随很小的 SF `max_us`：中位数 `96.863 us`，保留样本为 `226.224 us`，A 内两者相关系数为 +`0.941`。 + +这些证据把大波动定位到运行时调度等待和 kernel 尾时延，而不是配置错误：异常日志依然全部 +通过功能门禁，PMU 和泳道配置也没有变化。由于本次是未隔离的 device0 直跑，现有证据不能继续 +区分这些动态变化是设备外部负载、频率状态还是 PA 内部竞争的哪一种组合。 + +## lazy 动态证据 + +三份包的 device0 b1 功能门禁均通过。B/C 的 `[LAZY_SAMPLE_FRONTEND]` 同时打印实测值和精确 +期望值： + +| 形态 | views | tensor args | scalar args | resets | +| --- | ---: | ---: | ---: | ---: | +| B compete-first eager | 192 | 2112 | 864 | 384 | +| C compete-first lazy | 97 | 1162 | 9 | 384 | + +C 的计数下降证明 loser 没有执行 input/scalar thunk。A 没有编译该专用 banner，因此不把源码 +推算值冒充实测值。 + +## `.text`、实测 ELF 与反汇编 + +| 形态 | final `.text` | `.text` SHA256 | 实测 final ELF SHA256 | +| --- | ---: | --- | --- | +| A original | 780344 B | `018ae7dc29ce78249b2e3bff84b0faa2e671e2448c9316719fb48bc59139d2fc` | `76c961f846c7efc89b40942c3a8113530a6da2c7bfbdc601615852f8c5b79cfc` | +| B compete-first eager | 547640 B | `8fd6209b2f0f9d1fb48d4d8a16cf27649e26071153b908e1e0e39ca107d63589` | `82a27e206ee1b2411964c0530c73adf92a2b032ac202a7c65c6b5f7d76a4571b` | +| C compete-first lazy | 547640 B | `866ed1081ebb94038a8feca87ccedf95e67aebfdaef11651c109f86672be2bdd` | `8d293c52312429d13efe89592ed705c672ceef1f2875b5e3bd25582419d37893` | + +带源码注释的反汇编是从表中当前 `artifacts/measured/pa_scheduler_kernel.o` 重新生成的，文件头记录 +完整 ELF SHA256。最终性能复测直接执行相同 clean-build host/final ELF，期间没有改源码或重编译， +所以反汇编无需因性能日志更新而修改。只有源码或 ELF 改变时才需要重新运行各包的 +`./generate_disassembly.sh`。 + +可复算数据位于： + +- `performance_summary.json`：统计、阈值、配对结果和诊断量； +- `performance_samples.tsv`：72 个样本及异常标记； +- 各包 `output/performance/raw/`：完整设备原始输出； +- 各包 `output/performance/SUMMARY.md`、`samples.tsv`、`summary.json`：单包独立结果。 diff --git a/tests/atomic_probe/lazy_lamda_sample/README.md b/tests/atomic_probe/lazy_lamda_sample/README.md new file mode 100644 index 0000000000..ce754b76f9 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/README.md @@ -0,0 +1,69 @@ +# standalone full-task lazy lambda 三版独立样例 + +本目录用于把三种 PA submit 形态作为可直接交给他人的独立样例进行静态与动态对照。三份包都覆盖 +`Alloc -> QK -> SF -> PV -> UP`，使用 32 AIC + 64 AIV；不是只改 QK。 + +根目录只保存三份包之间的公共说明。`A_original/`、`B_compete_first/`、 +`C_compete_first_lazy/` 各自包含完整源码、固定构建入口、干净构建结果、正式测试所用二进制、 +功能/性能输出以及带源码注释的反汇编；任意一份都可脱离另外两份单独复制和查看。 + +## 三份代码 + +| 包 | 固定编译形态 | 参数求值和 Submit 顺序 | +| --- | --- | --- | +| [A_original](A_original/README.md) | `original`，不定义 `PA_LAZY_SAMPLE_SHAPE_ID` | 所有 worker 在 Submit 外 eager 构参，再走原始 Submit | +| [B_compete_first](B_compete_first/README.md) | `PA_LAZY_SAMPLE_SHAPE_ID=1` | 先 Claim，再由所有 worker 同步求值所有 callback 参数 | +| [C_compete_first_lazy](C_compete_first_lazy/README.md) | `PA_LAZY_SAMPLE_SHAPE_ID=2` | 与 B 同控制流；loser 不求值 input/scalar thunk | + +A 的 eager `Build*Args` 在进入 Submit 前发生。原始 Submit 的共同前段是 +`EfDrain -> Materialize -> TensorMap retire -> Claim`；非 Alloc 随后是 winner-only Fanin、全员 +Register、winner-only Build，Alloc 则在 Claim 前 Register，winner 最后完成发布。 + +B/C 的固定顺序是： + +```text +Begin -> EfDrain -> Claim -> 调用一次 outer callback + -> Add* 内同步求值允许执行的 nested lambda + -> callback 生命周期结束 -> split finish + -> Materialize -> TensorMap retire -> winner-only Fanin + -> 全员 Register -> winner-only Build/Complete +``` + +这里没有“先保存所有 thunk、Claim 后再回放”的机制。Claim 在 outer callback 之前已经完成， +`Add*` 在 callback 内立即调用传入的临时 lambda，lambda/closure 不跨越 callback 或 split-TU 边界。 +C 只令 loser 跳过 input/scalar；output/inout 仍由所有 worker 求值，以保持本 Tier-1 样例的私有 +heap/TensorMap 状态一致。 + +## 从哪里开始看 + +- [PERFORMANCE_COMPARISON.md](PERFORMANCE_COMPARISON.md)：固定测试口径、24 样本/版统计、功能计数、 + `.text` 与结论； +- `analyze_performance.py`：校验 72 个独立运行日志并复算异常规则、总体和配对统计； +- [DISASSEMBLY_GUIDE.md](DISASSEMBLY_GUIDE.md)：真实解码、DWARF 源码注释的生成方法和证据边界； +- 每份包的 `README.md`：该包的固定宏、控制流、内容和独立复跑命令； +- C 的 `disassembly/key_flow/aic_lazy_input_policy.source.asm`：最直接的 lazy 判定源码注释与机器码； +- A/B/C 的 `output/performance/raw/`：正式 device0 b256 原始输出，不只保留汇总数字。 +- 根目录和每份包的 `MANIFEST.sha256`：公共文件与三份独立交付的完整哈希入口。 + +## 结论边界 + +正式主对照是 C-B：两者 `.text` 总尺寸相同、entry/finish 机器码相同，差别集中在 +callback/orchestration。24 个独立样本/版按统一 Hampel 规则去异常后，C-B Submit span 总体 +中位差为 `+74.364 us / +1.986%`；按相邻排列块配对后中位差只有 `+1.503 us / +0.040%`， +C/B 各自更快 11/22 块。数据存在明显时间漂移，配对结果不支持 lazy 有稳定收益或回退；差异 +低于约定的 5% 门槛，因此没有采集 I-cache PMU，也不作 I-cache 因果结论。 + +A-B 同时改变了参数求值时点、Submit 顺序、split-TU/outlining 和 `.text` 布局，只能表示整套 +compete-first 结构相对原始结构的结果，不能被称为 lazy 收益。 + +## 二进制身份说明 + +每份包同时保留： + +- `artifacts/measured/`：正式 24 样本性能日志实际执行的 host/final ELF； +- `artifacts/rebuilt/`：从该包本地源码清空专属 build 目录后生成的同一 clean-build 发布副本； +- `artifacts/runtime_identity.json`：两份发布副本的完整 ELF、host 与运行时内容校验。 + +性能测试直接执行各包的 clean `build/`；`measured/rebuilt` 当前完整 final ELF 与 host +均相同。带源码注释的反汇编从这个 `measured` ELF 生成，并在文件头记录完整 ELF 哈希；性能 +采集期间没有重新构建，因而执行二进制与反汇编直接对应。 diff --git a/tests/atomic_probe/lazy_lamda_sample/analyze_performance.py b/tests/atomic_probe/lazy_lamda_sample/analyze_performance.py new file mode 100755 index 0000000000..5dcc8153ec --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/analyze_performance.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +"""Validate and summarize the final independent-run A/B/C measurements.""" + +from __future__ import annotations + +import json +import math +import re +import statistics +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent +VARIANTS = { + "A": ("A_original", "original", "A original"), + "B": ("B_compete_first", "compete-first", "B compete-first eager"), + "C": ("C_compete_first_lazy", "compete-first-lazy", "C compete-first + lazy lambda"), +} +FILE_RE = re.compile( + r"block_(?P\d{2})_(?P[ABC]{3})_pos(?P[123])_" + r"(?Poriginal|compete-first|compete-first-lazy)\.txt$" +) +CONFIG_RE = re.compile( + r"^device=0 batches=256 tasks=1280 workers=96 runs=1 .*\bswimlane=off\b", + re.MULTILINE, +) +SUMMARY_RE = re.compile( + r"^\[SUMMARY\] runs=1 completed_runs=1 median_submit_span_us=([0-9.]+) " + r"execution_status=PASS semantic_status=PASS postprocess_status=PASS$", + re.MULTILINE, +) +FIELD_PATTERNS = { + "submit_span_us": re.compile(r"^\[METRIC\].*submit_span_us=([0-9.]+)", re.MULTILINE), + "host_launch_us": re.compile(r"^\[METRIC\].*host_launch_us=([0-9.]+)", re.MULTILINE), + "fanin_loads": re.compile(r"^\[METRIC\].*fanin_loads=(\d+)", re.MULTILINE), + "fanin_not_ready": re.compile(r"^\[ATOMIC\].*fanin_not_ready=(\d+)", re.MULTILINE), + "frontier_flag": re.compile(r"^\[ATOMIC\].*frontier_flag=(\d+)", re.MULTILINE), + "frontier_ready_fetch_max": re.compile( + r"^\[ATOMIC\].*frontier_ready_fetch_max=(\d+)", re.MULTILINE + ), + "ring_bp": re.compile(r"^\[PLACEMENT\].*RingBp=(\d+)", re.MULTILINE), + "sf_max_us": re.compile(r"^\[KERNEL\] SF .*max_us=([0-9.]+)", re.MULTILINE), +} +FLOAT_FIELDS = {"submit_span_us", "host_launch_us", "sf_max_us"} +HAMPEL_SCALE = 1.4826 +HAMPEL_SIGMAS = 3.0 + + +def fail(message: str) -> None: + raise SystemExit(message) + + +def one_match(pattern: re.Pattern[str], text: str, path: Path, field: str) -> str: + matches = pattern.findall(text) + if len(matches) != 1: + fail(f"{path}: expected one {field}, found {len(matches)}") + return matches[0] + + +def median(values: list[float | int]) -> float: + if not values: + fail("internal error: median of empty data") + return float(statistics.median(values)) + + +def pearson(rows: list[dict[str, Any]], x_name: str, y_name: str) -> float | None: + xs = [float(row[x_name]) for row in rows] + ys = [float(row[y_name]) for row in rows] + x_mean = statistics.fmean(xs) + y_mean = statistics.fmean(ys) + denominator = math.sqrt( + sum((value - x_mean) ** 2 for value in xs) + * sum((value - y_mean) ** 2 for value in ys) + ) + if denominator == 0.0: + return None + return sum( + (x_value - x_mean) * (y_value - y_mean) + for x_value, y_value in zip(xs, ys) + ) / denominator + + +def load_rows() -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for letter, (package, variant, _label) in VARIANTS.items(): + raw_dir = ROOT / package / "output" / "performance" / "raw" + paths = sorted(raw_dir.glob("*.txt")) + if len(paths) != 24: + fail(f"{variant}: expected 24 final raw files, found {len(paths)}") + for path in paths: + name_match = FILE_RE.fullmatch(path.name) + if name_match is None or name_match.group("variant") != variant: + fail(f"{variant}: unexpected raw filename {path.name}") + text = path.read_text(encoding="utf-8") + if CONFIG_RE.search(text) is None: + fail(f"{path}: fixed device0/b256/runs=1/no-swimlane config missing") + if "[WINNER-WORKLOAD] mode=real-compute " not in text: + fail(f"{path}: real-compute workload missing") + if "[PMU-CONFIG] window=off " not in text: + fail(f"{path}: PMU is not explicitly off") + summary_span = float(one_match(SUMMARY_RE, text, path, "PASS summary")) + values: dict[str, float | int] = {} + for field, pattern in FIELD_PATTERNS.items(): + value = one_match(pattern, text, path, field) + values[field] = float(value) if field in FLOAT_FIELDS else int(value) + if summary_span != values["submit_span_us"]: + fail(f"{path}: one-run SUMMARY and METRIC spans differ") + block = int(name_match.group("block")) + if not 1 <= block <= 24: + fail(f"{path}: block must be in 01..24") + rows.append( + { + "letter": letter, + "variant": variant, + "block": block, + "order": name_match.group("order"), + "position": int(name_match.group("position")), + "source": str(path.relative_to(ROOT)), + **values, + } + ) + + for block in range(1, 25): + block_rows = [row for row in rows if row["block"] == block] + if {row["letter"] for row in block_rows} != set(VARIANTS): + fail(f"block {block:02d}: missing A/B/C member") + orders = {row["order"] for row in block_rows} + if len(orders) != 1: + fail(f"block {block:02d}: variants disagree on launch order") + order = next(iter(orders)) + if sorted(order) != ["A", "B", "C"]: + fail(f"block {block:02d}: invalid launch order {order}") + for row in block_rows: + expected_position = order.index(str(row["letter"])) + 1 + if row["position"] != expected_position: + fail(f"block {block:02d}: filename position mismatch for {row['letter']}") + + order_counts = { + order: sum(row["letter"] == "A" and row["order"] == order for row in rows) + for order in ("ABC", "ACB", "BAC", "BCA", "CAB", "CBA") + } + if set(order_counts.values()) != {4}: + fail(f"six launch orders are not each repeated four times: {order_counts}") + for letter in VARIANTS: + position_counts = { + position: sum( + row["letter"] == letter and row["position"] == position for row in rows + ) + for position in (1, 2, 3) + } + if set(position_counts.values()) != {8}: + fail(f"{letter}: launch positions are not 8/8/8: {position_counts}") + return rows + + +def flag_outliers(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]: + thresholds: dict[str, dict[str, float]] = {} + for letter in VARIANTS: + variant_rows = [row for row in rows if row["letter"] == letter] + values = [float(row["submit_span_us"]) for row in variant_rows] + center = median(values) + mad = median([abs(value - center) for value in values]) + if mad == 0.0: + fail(f"{letter}: zero MAD cannot define an outlier threshold") + scaled_mad = HAMPEL_SCALE * mad + lower = center - HAMPEL_SIGMAS * scaled_mad + upper = center + HAMPEL_SIGMAS * scaled_mad + thresholds[letter] = { + "median_us": center, + "mad_us": mad, + "scaled_mad_us": scaled_mad, + "lower_us": lower, + "upper_us": upper, + } + for row in variant_rows: + value = float(row["submit_span_us"]) + row["included"] = lower <= value <= upper + row["exclusion_reason"] = "" if row["included"] else "hampel_3_scaled_mad" + return thresholds + + +def describe(rows: list[dict[str, Any]], included_only: bool) -> dict[str, Any]: + selected = [row for row in rows if not included_only or row["included"]] + values = [float(row["submit_span_us"]) for row in selected] + return { + "count": len(values), + "median_submit_span_us": median(values), + "mean_submit_span_us": statistics.fmean(values), + "min_submit_span_us": min(values), + "max_submit_span_us": max(values), + "population_stdev_us": statistics.pstdev(values), + } + + +def signal_medians(rows: list[dict[str, Any]]) -> dict[str, float] | None: + if not rows: + return None + return { + field: median([row[field] for row in rows]) + for field in ( + "submit_span_us", + "fanin_loads", + "fanin_not_ready", + "frontier_flag", + "frontier_ready_fetch_max", + "ring_bp", + "sf_max_us", + ) + } + + +def variant_statistics(rows: list[dict[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for letter, (_package, variant, label) in VARIANTS.items(): + variant_rows = [row for row in rows if row["letter"] == letter] + kept = [row for row in variant_rows if row["included"]] + excluded = [row for row in variant_rows if not row["included"]] + result[variant] = { + "letter": letter, + "label": label, + "raw": describe(variant_rows, included_only=False), + "outlier_filtered": describe(variant_rows, included_only=True), + "position_medians_us_raw": { + str(position): median( + [ + row["submit_span_us"] + for row in variant_rows + if row["position"] == position + ] + ) + for position in (1, 2, 3) + }, + "collection_half_medians_us_raw": { + "blocks_01_12": median( + [row["submit_span_us"] for row in variant_rows if row["block"] <= 12] + ), + "blocks_13_24": median( + [row["submit_span_us"] for row in variant_rows if row["block"] >= 13] + ), + }, + "kept_signal_medians": signal_medians(kept), + "excluded_signal_medians": signal_medians(excluded), + "excluded_samples": [ + { + "block": row["block"], + "position": row["position"], + "submit_span_us": row["submit_span_us"], + "source": row["source"], + } + for row in excluded + ], + } + return result + + +def comparison( + rows: list[dict[str, Any]], + stats: dict[str, Any], + candidate_letter: str, + baseline_letter: str, +) -> dict[str, Any]: + candidate_variant = VARIANTS[candidate_letter][1] + baseline_variant = VARIANTS[baseline_letter][1] + + def aggregate(filtered: bool) -> dict[str, float]: + candidate = float( + stats[candidate_variant]["outlier_filtered" if filtered else "raw"] + ["median_submit_span_us"] + ) + baseline = float( + stats[baseline_variant]["outlier_filtered" if filtered else "raw"] + ["median_submit_span_us"] + ) + delta = candidate - baseline + return {"delta_us": delta, "delta_percent": delta / baseline * 100.0} + + raw_deltas: list[float] = [] + clean_deltas: list[float] = [] + excluded_blocks: list[int] = [] + for block in range(1, 25): + candidate = next( + row + for row in rows + if row["letter"] == candidate_letter and row["block"] == block + ) + baseline = next( + row + for row in rows + if row["letter"] == baseline_letter and row["block"] == block + ) + delta = float(candidate["submit_span_us"]) - float(baseline["submit_span_us"]) + raw_deltas.append(delta) + if candidate["included"] and baseline["included"]: + clean_deltas.append(delta) + else: + excluded_blocks.append(block) + + clean_baseline_median = float( + stats[baseline_variant]["outlier_filtered"]["median_submit_span_us"] + ) + return { + "candidate": candidate_variant, + "baseline": baseline_variant, + "aggregate_raw": aggregate(False), + "aggregate_outlier_filtered": aggregate(True), + "paired_raw": { + "count": len(raw_deltas), + "median_delta_us": median(raw_deltas), + "candidate_faster_blocks": sum(delta < 0.0 for delta in raw_deltas), + }, + "paired_outlier_filtered": { + "count": len(clean_deltas), + "median_delta_us": median(clean_deltas), + "median_delta_percent_of_filtered_baseline": ( + median(clean_deltas) / clean_baseline_median * 100.0 + ), + "mean_delta_us": statistics.fmean(clean_deltas), + "candidate_faster_blocks": sum(delta < 0.0 for delta in clean_deltas), + "excluded_blocks": excluded_blocks, + }, + "paired_raw_collection_halves": { + "blocks_01_12_median_delta_us": median(raw_deltas[:12]), + "blocks_13_24_median_delta_us": median(raw_deltas[12:]), + }, + } + + +def write_samples(rows: list[dict[str, Any]]) -> None: + columns = ( + "letter", + "variant", + "block", + "order", + "position", + "submit_span_us", + "included", + "exclusion_reason", + "host_launch_us", + "fanin_loads", + "fanin_not_ready", + "frontier_flag", + "frontier_ready_fetch_max", + "ring_bp", + "sf_max_us", + "source", + ) + with (ROOT / "performance_samples.tsv").open("w", encoding="utf-8") as output: + output.write("\t".join(columns) + "\n") + for row in sorted(rows, key=lambda item: (item["block"], item["position"])): + output.write("\t".join(str(row[column]) for column in columns) + "\n") + + +def main() -> None: + rows = load_rows() + thresholds = flag_outliers(rows) + stats = variant_statistics(rows) + correlations = { + letter: { + field: pearson( + [row for row in rows if letter == "all" or row["letter"] == letter], + "submit_span_us", + field, + ) + for field in ( + "fanin_loads", + "fanin_not_ready", + "frontier_flag", + "frontier_ready_fetch_max", + "ring_bp", + "sf_max_us", + "host_launch_us", + ) + } + for letter in ("A", "B", "C", "all") + } + document = { + "schema": "pa_lazy_lambda_independent_performance/v1", + "configuration": { + "device": 0, + "workers": 96, + "batches": 256, + "samples_per_variant": 24, + "host_launches": 72, + "runs_per_host_launch": 1, + "launch_orders": "all six permutations, each repeated four times", + "launch_positions_per_variant": {"1": 8, "2": 8, "3": 8}, + "pmu_window": "off", + "swimlane": "off-runtime", + "winner_workload": "real-compute", + }, + "outlier_rule": { + "name": "Hampel", + "center": "per-variant median", + "scale": "1.4826 * per-variant MAD", + "threshold_scaled_mad": 3.0, + "raw_logs_retained": True, + "paired_policy": "exclude a block if either member is an outlier", + "thresholds": thresholds, + }, + "variants": stats, + "comparisons": { + "B_minus_A_compete_first": comparison(rows, stats, "B", "A"), + "C_minus_B_lazy_lambda": comparison(rows, stats, "C", "B"), + "C_minus_A_total": comparison(rows, stats, "C", "A"), + }, + "pearson_correlations": correlations, + } + (ROOT / "performance_summary.json").write_text( + json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + write_samples(rows) + print(json.dumps(document["comparisons"], indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tests/atomic_probe/lazy_lamda_sample/performance_samples.tsv b/tests/atomic_probe/lazy_lamda_sample/performance_samples.tsv new file mode 100644 index 0000000000..5bcf5fb5b1 --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/performance_samples.tsv @@ -0,0 +1,73 @@ +letter variant block order position submit_span_us included exclusion_reason host_launch_us fanin_loads fanin_not_ready frontier_flag frontier_ready_fetch_max ring_bp sf_max_us source +A original 1 ABC 1 3963.375 True 4840.717 31894 28174 4027 2747 25 231.412 A_original/output/performance/raw/block_01_ABC_pos1_original.txt +B compete-first 1 ABC 2 3635.16 True 4576.513 27021 23547 3246 1966 3 63.162 B_compete_first/output/performance/raw/block_01_ABC_pos2_compete-first.txt +C compete-first-lazy 1 ABC 3 4084.775 False hampel_3_scaled_mad 4990.563 29650 24919 11632 10352 47 113.194 C_compete_first_lazy/output/performance/raw/block_01_ABC_pos3_compete-first-lazy.txt +B compete-first 2 BCA 1 3650.095 True 4532.621 27321 23869 3243 1963 4 61.468 B_compete_first/output/performance/raw/block_02_BCA_pos1_compete-first.txt +C compete-first-lazy 2 BCA 2 3845.228 True 4773.478 32549 29055 3849 2569 16 249.945 C_compete_first_lazy/output/performance/raw/block_02_BCA_pos2_compete-first-lazy.txt +A original 2 BCA 3 3856.072 True 4748.418 38428 34707 3437 2157 36 130.376 A_original/output/performance/raw/block_02_BCA_pos3_original.txt +C compete-first-lazy 3 CAB 1 3655.535 True 6125.462 32564 29121 3332 2052 10 66.61 C_compete_first_lazy/output/performance/raw/block_03_CAB_pos1_compete-first-lazy.txt +A original 3 CAB 2 3870.407 True 4828.119 36534 32883 3274 1994 30 134.53 A_original/output/performance/raw/block_03_CAB_pos2_original.txt +B compete-first 3 CAB 3 3652.166 True 4566.144 26983 23506 3332 2052 3 66.399 B_compete_first/output/performance/raw/block_03_CAB_pos3_compete-first.txt +A original 4 ACB 1 3956.964 True 4960.99 38253 34543 3368 2088 38 243.774 A_original/output/performance/raw/block_04_ACB_pos1_original.txt +C compete-first-lazy 4 ACB 2 3815.964 True 4725.958 33506 30040 3194 1914 12 225.194 C_compete_first_lazy/output/performance/raw/block_04_ACB_pos2_compete-first-lazy.txt +B compete-first 4 ACB 3 3656.991 True 4596.597 26856 23415 3189 1909 1 69.523 B_compete_first/output/performance/raw/block_04_ACB_pos3_compete-first.txt +B compete-first 5 BAC 1 3730.272 True 4607.015 26251 22829 3239 1959 0 147.17 B_compete_first/output/performance/raw/block_05_BAC_pos1_compete-first.txt +A original 5 BAC 2 3989.566 True 5514.869 39529 35858 3389 2109 37 240.514 A_original/output/performance/raw/block_05_BAC_pos2_original.txt +C compete-first-lazy 5 BAC 3 3645.71 True 5229.678 32107 28634 3228 1948 9 61.252 C_compete_first_lazy/output/performance/raw/block_05_BAC_pos3_compete-first-lazy.txt +C compete-first-lazy 6 CBA 1 3836.215 True 4701.966 32630 29157 3309 2029 14 249.587 C_compete_first_lazy/output/performance/raw/block_06_CBA_pos1_compete-first-lazy.txt +B compete-first 6 CBA 2 3847.468 True 4690.89 26949 23522 3062 1782 0 255.816 B_compete_first/output/performance/raw/block_06_CBA_pos2_compete-first.txt +A original 6 CBA 3 4000.058 True 4909.12 34543 30902 3313 2033 23 259.746 A_original/output/performance/raw/block_06_CBA_pos3_original.txt +C compete-first-lazy 7 CBA 1 3751.926 True 4561.026 34745 31284 3389 2109 18 158.647 C_compete_first_lazy/output/performance/raw/block_07_CBA_pos1_compete-first-lazy.txt +B compete-first 7 CBA 2 3706.997 True 4624.574 26669 23197 3343 2063 1 112.03 B_compete_first/output/performance/raw/block_07_CBA_pos2_compete-first.txt +A original 7 CBA 3 3930.496 True 4874.213 31233 27480 4016 2736 26 174.776 A_original/output/performance/raw/block_07_CBA_pos3_original.txt +B compete-first 8 BAC 1 3837.104 True 4777.883 26623 23184 3193 1913 0 254.556 B_compete_first/output/performance/raw/block_08_BAC_pos1_compete-first.txt +A original 8 BAC 2 3903.443 True 4825.675 34306 30661 3235 1955 24 152.503 A_original/output/performance/raw/block_08_BAC_pos2_original.txt +C compete-first-lazy 8 BAC 3 3849.866 True 4705.073 31809 28377 3372 2092 8 255.461 C_compete_first_lazy/output/performance/raw/block_08_BAC_pos3_compete-first-lazy.txt +A original 9 ACB 1 3901.414 True 5107.014 33235 29500 4120 2840 35 161.111 A_original/output/performance/raw/block_09_ACB_pos1_original.txt +C compete-first-lazy 9 ACB 2 4370.394 False hampel_3_scaled_mad 6211.327 31121 25538 14306 13026 59 105.339 C_compete_first_lazy/output/performance/raw/block_09_ACB_pos2_compete-first-lazy.txt +B compete-first 9 ACB 3 3625.727 True 4961.603 26464 22988 3176 1896 2 59.553 B_compete_first/output/performance/raw/block_09_ACB_pos3_compete-first.txt +C compete-first-lazy 10 CAB 1 3821.598 True 4754.349 33447 29971 3190 1910 12 231.863 C_compete_first_lazy/output/performance/raw/block_10_CAB_pos1_compete-first-lazy.txt +A original 10 CAB 2 3948.008 True 4905.562 35157 31459 3231 1951 22 248.949 A_original/output/performance/raw/block_10_CAB_pos2_original.txt +B compete-first 10 CAB 3 3823.817 True 5051.993 27153 23678 3247 1967 1 238.224 B_compete_first/output/performance/raw/block_10_CAB_pos3_compete-first.txt +B compete-first 11 BCA 1 3679.901 True 4577.907 27047 23602 3094 1814 1 95.801 B_compete_first/output/performance/raw/block_11_BCA_pos1_compete-first.txt +C compete-first-lazy 11 BCA 2 3950.28 True 5531.069 36360 32365 6760 5480 34 76.439 C_compete_first_lazy/output/performance/raw/block_11_BCA_pos2_compete-first-lazy.txt +A original 11 BCA 3 3832.087 False hampel_3_scaled_mad 4926.323 38351 34532 4578 3298 36 131.364 A_original/output/performance/raw/block_11_BCA_pos3_original.txt +A original 12 ABC 1 3971.406 True 4864.768 37579 33915 3128 1848 32 255.715 A_original/output/performance/raw/block_12_ABC_pos1_original.txt +B compete-first 12 ABC 2 3637.016 True 4620.597 26654 23136 3377 2097 1 56.887 B_compete_first/output/performance/raw/block_12_ABC_pos2_compete-first.txt +C compete-first-lazy 12 ABC 3 3869.958 True 5215.617 33083 29640 3292 2012 13 276.419 C_compete_first_lazy/output/performance/raw/block_12_ABC_pos3_compete-first-lazy.txt +B compete-first 13 BCA 1 3753.752 True 4761.04 27031 23562 3457 2177 3 160.883 B_compete_first/output/performance/raw/block_13_BCA_pos1_compete-first.txt +C compete-first-lazy 13 BCA 2 3751.215 True 5057.378 35269 31774 3333 2053 13 159.881 C_compete_first_lazy/output/performance/raw/block_13_BCA_pos2_compete-first-lazy.txt +A original 13 BCA 3 3953.491 True 4852.575 33478 29867 3097 1817 20 246.794 A_original/output/performance/raw/block_13_BCA_pos3_original.txt +A original 14 ACB 1 3961.384 True 4900.896 41499 37808 3392 2112 39 218.183 A_original/output/performance/raw/block_14_ACB_pos1_original.txt +C compete-first-lazy 14 ACB 2 3848.951 True 8279.656 33315 29836 3363 2083 12 259.218 C_compete_first_lazy/output/performance/raw/block_14_ACB_pos2_compete-first-lazy.txt +B compete-first 14 ACB 3 3866.154 True 4786.66 27060 23613 3188 1908 1 266.041 B_compete_first/output/performance/raw/block_14_ACB_pos3_compete-first.txt +C compete-first-lazy 15 CBA 1 3858.263 True 4723.966 34190 30732 3212 1932 16 273.35 C_compete_first_lazy/output/performance/raw/block_15_CBA_pos1_compete-first-lazy.txt +B compete-first 15 CBA 2 3766.289 True 4678.773 26932 23443 3363 2083 3 160.846 B_compete_first/output/performance/raw/block_15_CBA_pos2_compete-first.txt +A original 15 CBA 3 3926.037 True 4903.135 35594 31828 3457 2177 33 175.942 A_original/output/performance/raw/block_15_CBA_pos3_original.txt +A original 16 ABC 1 3972.611 True 4845.415 41697 38030 3442 2162 45 227.113 A_original/output/performance/raw/block_16_ABC_pos1_original.txt +B compete-first 16 ABC 2 3834.848 True 4668.863 27303 23867 3189 1909 2 236.561 B_compete_first/output/performance/raw/block_16_ABC_pos2_compete-first.txt +C compete-first-lazy 16 ABC 3 3834.485 True 4679.956 34651 31220 3275 1995 14 238.105 C_compete_first_lazy/output/performance/raw/block_16_ABC_pos3_compete-first-lazy.txt +C compete-first-lazy 17 CAB 1 3742.848 True 5150.658 33883 30385 3384 2104 14 161.224 C_compete_first_lazy/output/performance/raw/block_17_CAB_pos1_compete-first-lazy.txt +A original 17 CAB 2 3956.2 True 4960.107 35544 31883 3245 1965 23 234.465 A_original/output/performance/raw/block_17_CAB_pos2_original.txt +B compete-first 17 CAB 3 3649.524 True 4638.995 26777 23313 3247 1967 1 69.083 B_compete_first/output/performance/raw/block_17_CAB_pos3_compete-first.txt +B compete-first 18 BAC 1 3848.497 True 4732.38 27019 23575 3185 1905 2 261.138 B_compete_first/output/performance/raw/block_18_BAC_pos1_compete-first.txt +A original 18 BAC 2 3918.43 True 5198.564 39851 36082 3653 2373 32 225.335 A_original/output/performance/raw/block_18_BAC_pos2_original.txt +C compete-first-lazy 18 BAC 3 3838.856 True 4749.526 36185 32734 3284 2004 16 246.848 C_compete_first_lazy/output/performance/raw/block_18_BAC_pos3_compete-first-lazy.txt +B compete-first 19 BAC 1 3807.28 True 4697.9 26759 23272 3304 2024 1 226.02 B_compete_first/output/performance/raw/block_19_BAC_pos1_compete-first.txt +A original 19 BAC 2 3960.714 True 4860.018 36373 32740 3385 2105 28 213.047 A_original/output/performance/raw/block_19_BAC_pos2_original.txt +C compete-first-lazy 19 BAC 3 3705.379 True 4637.708 37895 34336 4224 2944 26 112.765 C_compete_first_lazy/output/performance/raw/block_19_BAC_pos3_compete-first-lazy.txt +C compete-first-lazy 20 CAB 1 3762.927 True 5109.348 34560 31099 3313 2033 17 169.608 C_compete_first_lazy/output/performance/raw/block_20_CAB_pos1_compete-first-lazy.txt +A original 20 CAB 2 3876.067 True 4794.053 36242 32561 3538 2258 29 134.741 A_original/output/performance/raw/block_20_CAB_pos2_original.txt +B compete-first 20 CAB 3 3836.932 True 4753.03 26659 23161 3271 1991 2 246.828 B_compete_first/output/performance/raw/block_20_CAB_pos3_compete-first.txt +A original 21 ABC 1 3959.145 True 5190.038 38049 34416 3296 2016 35 239.966 A_original/output/performance/raw/block_21_ABC_pos1_original.txt +B compete-first 21 ABC 2 3734.9 True 5034.938 26321 22741 4212 2932 8 129.333 B_compete_first/output/performance/raw/block_21_ABC_pos2_compete-first.txt +C compete-first-lazy 21 ABC 3 3810.508 True 4808.424 31257 27808 3210 1930 2 228.054 C_compete_first_lazy/output/performance/raw/block_21_ABC_pos3_compete-first-lazy.txt +C compete-first-lazy 22 CBA 1 3643.199 True 4565.537 34212 30785 3193 1913 17 55.603 C_compete_first_lazy/output/performance/raw/block_22_CBA_pos1_compete-first-lazy.txt +B compete-first 22 CBA 2 3769.223 True 5905.797 26462 23033 3412 2132 0 182.966 B_compete_first/output/performance/raw/block_22_CBA_pos2_compete-first.txt +A original 22 CBA 3 3975.785 True 4974.285 36256 32620 3104 1824 25 258.685 A_original/output/performance/raw/block_22_CBA_pos3_original.txt +A original 23 ACB 1 3788.296 False hampel_3_scaled_mad 4783.019 39637 35970 3296 2016 38 62.361 A_original/output/performance/raw/block_23_ACB_pos1_original.txt +C compete-first-lazy 23 ACB 2 3861.894 True 4877.291 37096 33566 3781 2501 25 255.204 C_compete_first_lazy/output/performance/raw/block_23_ACB_pos2_compete-first-lazy.txt +B compete-first 23 ACB 3 3847.947 True 4711.295 27434 23953 3318 2038 3 249.729 B_compete_first/output/performance/raw/block_23_ACB_pos3_compete-first.txt +B compete-first 24 BCA 1 3735.082 True 4666.874 27255 23803 3289 2009 5 151.231 B_compete_first/output/performance/raw/block_24_BCA_pos1_compete-first.txt +C compete-first-lazy 24 BCA 2 3646.27 True 4503.558 32960 29513 3343 2063 12 63.279 C_compete_first_lazy/output/performance/raw/block_24_BCA_pos2_compete-first-lazy.txt +A original 24 BCA 3 3956.396 True 4951.339 38551 34892 3318 2038 36 216.761 A_original/output/performance/raw/block_24_BCA_pos3_original.txt diff --git a/tests/atomic_probe/lazy_lamda_sample/performance_summary.json b/tests/atomic_probe/lazy_lamda_sample/performance_summary.json new file mode 100644 index 0000000000..8ecde3277b --- /dev/null +++ b/tests/atomic_probe/lazy_lamda_sample/performance_summary.json @@ -0,0 +1,353 @@ +{ + "comparisons": { + "B_minus_A_compete_first": { + "aggregate_outlier_filtered": { + "delta_percent": -5.355536918604207, + "delta_us": -211.88099999999986 + }, + "aggregate_raw": { + "delta_percent": -5.3207767534787385, + "delta_us": -210.42849999999999 + }, + "baseline": "original", + "candidate": "compete-first", + "paired_outlier_filtered": { + "candidate_faster_blocks": 22, + "count": 22, + "excluded_blocks": [ + 11, + 23 + ], + "mean_delta_us": -195.55340909090913, + "median_delta_percent_of_filtered_baseline": -5.213699776912662, + "median_delta_us": -206.2695000000001 + }, + "paired_raw": { + "candidate_faster_blocks": 23, + "count": 24, + "median_delta_us": -202.85800000000017 + }, + "paired_raw_collection_halves": { + "blocks_01_12_median_delta_us": -220.87000000000012, + "blocks_13_24_median_delta_us": -156.59099999999967 + } + }, + "C_minus_A_total": { + "aggregate_outlier_filtered": { + "delta_percent": -3.475900955893611, + "delta_us": -137.51699999999983 + }, + "aggregate_raw": { + "delta_percent": -3.2062946580340403, + "delta_us": -126.80399999999963 + }, + "baseline": "original", + "candidate": "compete-first-lazy", + "paired_outlier_filtered": { + "candidate_faster_blocks": 20, + "count": 20, + "excluded_blocks": [ + 1, + 9, + 11, + 23 + ], + "mean_delta_us": -165.38894999999997, + "median_delta_percent_of_filtered_baseline": -3.6604547989054437, + "median_delta_us": -144.81850000000009 + }, + "paired_raw": { + "candidate_faster_blocks": 20, + "count": 24, + "median_delta_us": -132.2679999999998 + }, + "paired_raw_collection_halves": { + "blocks_01_12_median_delta_us": -113.92899999999986, + "blocks_13_24_median_delta_us": -143.38149999999996 + } + }, + "C_minus_B_lazy_lambda": { + "aggregate_outlier_filtered": { + "delta_percent": 1.9859967519643256, + "delta_us": 74.36400000000003 + }, + "aggregate_raw": { + "delta_percent": 2.233311621008033, + "delta_us": 83.62450000000035 + }, + "baseline": "compete-first", + "candidate": "compete-first-lazy", + "paired_outlier_filtered": { + "candidate_faster_blocks": 11, + "count": 22, + "excluded_blocks": [ + 1, + 9 + ], + "mean_delta_us": 30.67363636363641, + "median_delta_percent_of_filtered_baseline": 0.04013976007479747, + "median_delta_us": 1.502999999999929 + }, + "paired_raw": { + "candidate_faster_blocks": 11, + "count": 24, + "median_delta_us": 8.065499999999929 + }, + "paired_raw_collection_halves": { + "blocks_01_12_median_delta_us": 101.95100000000002, + "blocks_13_24_median_delta_us": -6.088999999999714 + } + } + }, + "configuration": { + "batches": 256, + "device": 0, + "host_launches": 72, + "launch_orders": "all six permutations, each repeated four times", + "launch_positions_per_variant": { + "1": 8, + "2": 8, + "3": 8 + }, + "pmu_window": "off", + "runs_per_host_launch": 1, + "samples_per_variant": 24, + "swimlane": "off-runtime", + "winner_workload": "real-compute", + "workers": 96 + }, + "outlier_rule": { + "center": "per-variant median", + "name": "Hampel", + "paired_policy": "exclude a block if either member is an outlier", + "raw_logs_retained": true, + "scale": "1.4826 * per-variant MAD", + "threshold_scaled_mad": 3.0, + "thresholds": { + "A": { + "lower_us": 3854.1272929000006, + "mad_us": 22.64449999999988, + "median_us": 3954.8455, + "scaled_mad_us": 33.57273569999982, + "upper_us": 4055.563707099999 + }, + "B": { + "lower_us": 3338.1505002000004, + "mad_us": 91.3409999999999, + "median_us": 3744.417, + "scaled_mad_us": 135.42216659999983, + "upper_us": 4150.6834997999995 + }, + "C": { + "lower_us": 3590.0152591000005, + "mad_us": 53.515499999999975, + "median_us": 3828.0415000000003, + "scaled_mad_us": 79.34208029999996, + "upper_us": 4066.0677409 + } + } + }, + "pearson_correlations": { + "A": { + "fanin_loads": -0.09433050697776417, + "fanin_not_ready": -0.08714393550889851, + "frontier_flag": -0.3463118229142113, + "frontier_ready_fetch_max": -0.3463118229142113, + "host_launch_us": 0.35932297170639127, + "ring_bp": -0.2136124379599777, + "sf_max_us": 0.9414224560640145 + }, + "B": { + "fanin_loads": 0.20944777206956847, + "fanin_not_ready": 0.22177434780763416, + "frontier_flag": -0.08578707062184743, + "frontier_ready_fetch_max": -0.08578707062184743, + "host_launch_us": 0.22296895421294655, + "ring_bp": -0.17648851377285255, + "sf_max_us": 0.9947120367635994 + }, + "C": { + "fanin_loads": -0.32577015855783614, + "fanin_not_ready": -0.4885848421034647, + "frontier_flag": 0.8599824611626847, + "frontier_ready_fetch_max": 0.859982461162685, + "host_launch_us": 0.2684871212573618, + "ring_bp": 0.7968034227872008, + "sf_max_us": 0.15207181023955268 + }, + "all": { + "fanin_loads": 0.4475795971640391, + "fanin_not_ready": 0.40466220663271263, + "frontier_flag": 0.5576084775903302, + "frontier_ready_fetch_max": 0.5576084775903302, + "host_launch_us": 0.27498859551967875, + "ring_bp": 0.7303519771294827, + "sf_max_us": 0.5226128059220236 + } + }, + "schema": "pa_lazy_lambda_independent_performance/v1", + "variants": { + "compete-first": { + "collection_half_medians_us_raw": { + "blocks_01_12": 3668.446, + "blocks_13_24": 3788.2515000000003 + }, + "excluded_samples": [], + "excluded_signal_medians": null, + "kept_signal_medians": { + "fanin_loads": 26940.5, + "fanin_not_ready": 23474.5, + "frontier_flag": 3247.0, + "frontier_ready_fetch_max": 1967.0, + "ring_bp": 1.5, + "sf_max_us": 156.0385, + "submit_span_us": 3744.417 + }, + "label": "B compete-first eager", + "letter": "B", + "outlier_filtered": { + "count": 24, + "max_submit_span_us": 3866.154, + "mean_submit_span_us": 3747.2142499999995, + "median_submit_span_us": 3744.417, + "min_submit_span_us": 3625.727, + "population_stdev_us": 81.93544161220773 + }, + "position_medians_us_raw": { + "1": 3744.417, + "2": 3750.5945, + "3": 3740.404 + }, + "raw": { + "count": 24, + "max_submit_span_us": 3866.154, + "mean_submit_span_us": 3747.2142499999995, + "median_submit_span_us": 3744.417, + "min_submit_span_us": 3625.727, + "population_stdev_us": 81.93544161220773 + } + }, + "compete-first-lazy": { + "collection_half_medians_us_raw": { + "blocks_01_12": 3840.7215, + "blocks_13_24": 3786.7174999999997 + }, + "excluded_samples": [ + { + "block": 1, + "position": 3, + "source": "C_compete_first_lazy/output/performance/raw/block_01_ABC_pos3_compete-first-lazy.txt", + "submit_span_us": 4084.775 + }, + { + "block": 9, + "position": 2, + "source": "C_compete_first_lazy/output/performance/raw/block_09_ACB_pos2_compete-first-lazy.txt", + "submit_span_us": 4370.394 + } + ], + "excluded_signal_medians": { + "fanin_loads": 30385.5, + "fanin_not_ready": 25228.5, + "frontier_flag": 12969.0, + "frontier_ready_fetch_max": 11689.0, + "ring_bp": 53.0, + "sf_max_us": 109.26650000000001, + "submit_span_us": 4227.5845 + }, + "kept_signal_medians": { + "fanin_loads": 33694.5, + "fanin_not_ready": 30212.5, + "frontier_flag": 3322.5, + "frontier_ready_fetch_max": 2042.5, + "ring_bp": 14.0, + "sf_max_us": 226.624, + "submit_span_us": 3818.781 + }, + "label": "C compete-first + lazy lambda", + "letter": "C", + "outlier_filtered": { + "count": 22, + "max_submit_span_us": 3950.28, + "mean_submit_span_us": 3788.503409090909, + "median_submit_span_us": 3818.781, + "min_submit_span_us": 3643.199, + "population_stdev_us": 83.86854308036808 + }, + "position_medians_us_raw": { + "1": 3757.4265, + "2": 3847.0895, + "3": 3836.6705 + }, + "raw": { + "count": 24, + "max_submit_span_us": 4370.394, + "mean_submit_span_us": 3825.0935000000004, + "median_submit_span_us": 3828.0415000000003, + "min_submit_span_us": 3643.199, + "population_stdev_us": 151.24318121103295 + } + }, + "original": { + "collection_half_medians_us_raw": { + "blocks_01_12": 3939.252, + "blocks_13_24": 3956.298 + }, + "excluded_samples": [ + { + "block": 11, + "position": 3, + "source": "A_original/output/performance/raw/block_11_BCA_pos3_original.txt", + "submit_span_us": 3832.087 + }, + { + "block": 23, + "position": 1, + "source": "A_original/output/performance/raw/block_23_ACB_pos1_original.txt", + "submit_span_us": 3788.296 + } + ], + "excluded_signal_medians": { + "fanin_loads": 38994.0, + "fanin_not_ready": 35251.0, + "frontier_flag": 3937.0, + "frontier_ready_fetch_max": 2657.0, + "ring_bp": 37.0, + "sf_max_us": 96.8625, + "submit_span_us": 3810.1915 + }, + "kept_signal_medians": { + "fanin_loads": 36314.5, + "fanin_not_ready": 32680.0, + "frontier_flag": 3376.5, + "frontier_ready_fetch_max": 2096.5, + "ring_bp": 31.0, + "sf_max_us": 226.224, + "submit_span_us": 3956.298 + }, + "label": "A original", + "letter": "A", + "outlier_filtered": { + "count": 22, + "max_submit_span_us": 4000.058, + "mean_submit_span_us": 3941.2485909090906, + "median_submit_span_us": 3956.298, + "min_submit_span_us": 3856.072, + "population_stdev_us": 38.00398063440082 + }, + "position_medians_us_raw": { + "1": 3960.2645, + "2": 3933.219, + "3": 3941.9935 + }, + "raw": { + "count": 24, + "max_submit_span_us": 4000.058, + "mean_submit_span_us": 3930.3271666666665, + "median_submit_span_us": 3954.8455, + "min_submit_span_us": 3788.296, + "population_stdev_us": 51.729562705692956 + } + } + } +} diff --git "a/tests/atomic_probe/pa_scheduler/PA-atomic\346\203\205\345\206\265\345\210\206\346\236\220.md" "b/tests/atomic_probe/pa_scheduler/PA-atomic\346\203\205\345\206\265\345\210\206\346\236\220.md" new file mode 100644 index 0000000000..b5a388c988 --- /dev/null +++ "b/tests/atomic_probe/pa_scheduler/PA-atomic\346\203\205\345\206\265\345\210\206\346\236\220.md" @@ -0,0 +1,2398 @@ +# A5 FDWIC Paged Attention Submit 原子操作与优化记录 + +## 1. 范围与当前结论 + +本文记录 `TestPagedAttentionUnroll::Case1` 在真实 A5 上的 FDWIC AICore +Submit 路径，供后续继续优化。真实 PA 生产快照日期为 2026-07-18；当前 +保留的生产优化基线为 `2c3dd1e2`，F1 负结果记录提交为 `c93c3666`。 +standalone 已完成 2026-07-25 的 S4.14b；shared Vector cursor +四分片迁址和同址八分片均通过冻结 ELF 配对门槛，当前 standalone +shared 性能基线为回退提交 `319077a9`，其运行行为与 `ee42b8c1` +一致。S4.15a shared Cube 四分片迁址虽然正确性闭合，但配对中位数 +回退 `+0.567%`，已按预登记门槛撤销。S4.16 shared Vector `8→16` +也已完成：S4.16a capacity16/active8 布局成本为 4/6 更快、 +`-12.1595us/-0.5136%`；S4.16b 虽然正确性闭合，但相对 S4.16a +仅 1/6 更快、中位数 `+2.468us/+0.1049%`，命中第一层失败门槛。 +因此未追加样本或第二层配对，S4.16a/S4.16b 已整体撤销，当前源码 +恢复 `319077a9` 的 capacity8/active8 行为。 + +范围限定为： + +- 用例：`examples/a5/fully_distributed_within_core/paged_attention_unroll/`； +- Case：`Case1`； +- runtime：`fully_distributed_within_core`； +- 平台：真实 A5，A5Sim 只用于功能回归； +- 性能口径：所有 worker 中最早的 `Submit.start` 到最晚的 `Submit.end`； +- 原子操作清单：该用例实际经过的 FDWIC Submit、执行完成和外围生命周期路径。 + +当前结论： + +- 96 个 worker（32 AIC + 64 AIV）分别回放 1280 次 Submit，共有 + 122880 个完整 Submit 事件； +- 本轮已经消除了纯单 lane 图中 146944 次无效的 BlockWon + `atomic_load(any_pub)`，在 A5 上实际对应 `atomicAdd(addr, 0)`； +- 仍在 Submit 热路径中执行且数量最大的原子操作是 Claim：固定 + 73728 次 `atomicMax`； +- 三轮最终版本的首末 Submit 中位数为 5.115620 ms，相比 + 5.642245 ms 基线下降 0.526625 ms，即 9.33%；最好单轮为 + 5.096685 ms； +- H1 又消除了默认 256 MiB heap 第一圈的 1024 次 frontier atomic load， + 十对真实 A5 的配对变化中位数为 -0.324%，最好单轮为 5.098696 ms； +- 优化没有改变通用 atomic 语义，也没有把任务推迟到最终 drain 来制造 + 表面收益。F1 的 fanin 顺序重排已经证明性能回退并撤销；下一步先精确区分 + fanin 成功/失败 load 与 frontier 重复前推，再进行单变量消减。 +- standalone `swimlane` 当前使用 schema-v5 合并排他业务阶段与 atomic + （direct Atomic 加 PollBatch）泳道；相对 schema-v4 新增 shared + Register writer-metadata detail，以及 Materialize 中 fresh-output + 发布、descriptor copy 和 flush detail。atomic flags、weighted call + 和 PollBatch ABI 沿用已验证的既有语义。 + +环境安装、编译和基线复现过程见 +[A5 FDWIC Paged Attention 安装与复现指南](../a5_fdwic_atomic_swimlane_repo.md)。 +standalone I-cache 的当前构建、采集和解读见 +[`../icache_miss_usage_guide.md`](../icache_miss_usage_guide.md)。 + +## 2. Case1 工作量与 atomic 语义 + +Case1 的关键参数是 `batch=256`、`num_heads=16`、`block_size=128`、 +`context_len=8192`。当前 orchestration 中 `N_UNROLL=64`，因此每个 batch +只有一个 block group，且 `q_loop=1`。每个 worker 回放的 task id 分布如下： + +| task id | 每 batch 操作 | 执行 lane | fanin 数 | 新分配输出 | +| ------- | ------------- | --------- | -------: | ---------- | +| `5b+0` | Alloc | 任一 worker 竞争 winner | 0 | 有 | +| `5b+1` | QK | AIC | 0 | 有 | +| `5b+2` | SF | AIV | 1 | 有 | +| `5b+3` | PV | AIC | 1 | 有 | +| `5b+4` | UP | AIV | 3 | 无，仅 INOUT | + +其中 `b=0..255`，所以每核恰好回放 1280 个 task；全局实际执行 +256 个 Alloc 和 1024 个 kernel task。 + +A5 CCEC 下的封装位于 +`src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/atomic.h`： + +| C++ 封装 | A5 指令封装 | 说明 | +| -------- | ----------- | ---- | +| `atomic_load(x)` | `atomicAdd(&x, 0)` | 不是普通 load，而是 read-modify-write | +| `atomic_exchange(x, v)` | `atomicExch(&x, v)` | 发布或重置共享状态 | +| `atomic_fetch_add(x, v)` | `atomicAdd(&x, v)` | 计数器递增 | +| `atomic_fetch_sub(x, v)` | `atomicSub(&x, v)` | joint task 完成计数 | +| `atomic_fetch_max(x, v)` | `atomicMax(&x, v)` | Claim 和 frontier 前推 | + +因此，即使源码写的是 `atomic_load`，大量 worker 读取同一个地址时仍会形成 +RMW 竞争。本文中的次数分为三类： + +- “固定”表示可以由 Case1 拓扑和源码精确推出； +- “下界”表示每次正确执行至少发生这些操作，竞争和未就绪轮询会增加次数； +- “条件”表示 trace 只能确认分支是否等待，不能直接得到 atomic 指令计数。 + +## 3. 当前原子操作分布 + +### 3.1 Claim cursor：当前最大固定项 + +调用链为 +`submit_runtime.h::dist_submit_claim_*()` -> +`submit_helpers.h::claim()` -> `atomic_fetch_max(cursor, task_id)`。 +Claim span 会在所有 worker 上记录，但只有符合 lane 的 worker 执行 +`atomicMax`；Alloc 则由全部 96 个 worker 竞争。 + +| 操作 | task 数 | 每 task 竞争 worker | 固定 atomicMax 次数 | +| ---- | ------: | ------------------: | ------------------: | +| Alloc | 256 | 96 | 24576 | +| QK | 256 | 32 AIC | 8192 | +| SF | 256 | 64 AIV | 16384 | +| PV | 256 | 32 AIC | 8192 | +| UP | 256 | 64 AIV | 16384 | +| 合计 | 1280 | - | **73728** | + +每核累计 Claim 中位数中，AIC 从基线 0.517 ms 上升到最终三轮中位数 +0.608 ms，AIV 从 0.515 ms 上升到 0.613 ms。跳过 BlockWon 轮询后 +worker 更同步，cursor 瞬时竞争反而变强。总时间仍然下降，但 Claim 已成为 +最明确的剩余原子热点。 + +### 3.2 Heap 容量保护：winner action tail + +位置为 `submit_runtime.h::dist_submit_wait_heap_capacity()`。只有 winner 且 +`output_bytes>0` 时进入检查；本 Case 每 batch 的 Alloc、QK、SF、PV 满足， +UP 不满足，所以共有 1024 次有效调用。 + +H1 后最好一轮没有 `RingBp` 事件，且默认 256 MiB heap 的 1024 次有效调用都 +命中第一圈 fast path： + +- `fatal_set()`：固定 1024 次 `atomic_load(g_dist.fatal)`； +- frontier：第一圈为 0 次；只有逻辑 heap 超过一整圈后才恢复原 + `atomic_load(g_dist.frontier)`； +- `load_task_vend(F-H)`：第一圈为 0 次；跨圈后仍按原 frontier/vend 容量 + 规则执行； +- 如果未来出现 heap backpressure，上述三项会在循环中重复，耗时会归入 + `RingBp`。 + +无等待时，这些 atomic 位于 winner 的 Build/Alloc action 中。对 kernel +Submit 来说，泳道上表现为 `Build.end` 到 `Submit.end` 的尾段。 + +### 3.3 fanin ready 与 task completion + +依赖轮询位于 `submit_core.h::drain_phase_b()`：每个 fanin 通过 +`task_flag_ready()` 执行一次 `atomic_load(cell.flag)`。QK、SF、PV、UP 的 +fanin 分别为 0、1、1、3，因此任务真正开始执行前至少有： + +~~~text +256 * (0 + 1 + 1 + 3) = 1280 次 atomic_load(flag) +~~~ + +依赖未就绪时会提前退出并在下一次 drain 重试，所以 1280 只是成功检查的 +下界，不包含失败轮询。它们发生在执行 kernel 的 drain 位置：通常是下一个 +Submit 的 `EfDrain`，有背压时可能在 `RingBp`，剩余任务则在最终 drain。 + +每个 task 完成一次，路径为 +`execute_slot()/dist_submit_complete_alloc()` -> +`complete_executed_task()` -> `advance_frontier()`： + +| 共享状态 | 操作 | 次数性质 | Case1 次数 | +| -------- | ---- | -------- | ---------: | +| `task.vend` | `atomic_exchange` | 每 task 固定一次 | 1280 | +| `task.flag` 发布 | `atomic_exchange` | 每 task 固定一次 | 1280 | +| `frontier` 初始读取 | `atomic_load` | 每 task 固定一次 | 1280 | +| 后续 `task.flag` ready | `atomic_load` | 若前推次数为 A，则准确为 A+1280 | >=2560 | +| `frontier` 前推 | `atomic_fetch_max` | 记为 A，竞争时可重复 | >=1280 | + +原因是每次 `advance_frontier()` 都以一次未就绪 flag load 结束；每次成功 +`atomic_fetch_max(frontier)` 前又恰有一次 ready flag load。因此完整成功运行中， +该处 flag load 不是笼统的“至少 1280”，而是准确的 `A + 1280`。 + +若把 fanin ready load 记为 `G`，H1 后完整 Submit + completion 路径的 atomic +总数为： + +~~~text +73728 Claim + 1024 HeapGuard fatal + G fanin ++ 1280 vend exchange + 1280 flag exchange + 1280 frontier initial load ++ A frontier atomicMax + (A + 1280) frontier flag load += 79872 + G + 2A +~~~ + +由于 `G>=1280`、`A>=1280`，硬下界为 83712 次。`G` 和 `A` 受真实调度时序 +影响，不能用静态下界代替动态样本。 + +最终最好一轮的 1024 个 kernel 中，1011 个在某个 Submit 的 `EfDrain` +中执行，0 个在 `RingBp` 中执行，13 个在本核最后一个 Submit 之后的最终 +drain 中执行。completion atomic 紧随 kernel，分布也基本相同；Alloc 的 +completion 则在 Alloc winner 的 Submit 内完成。基线对应分布是 +979 个 `EfDrain`、29 个 `RingBp`、16 个最终 drain。 + +### 3.4 BlockWon：代码仍保留，当前 Case 动态次数为零 + +BlockWon 用于同一物理 block 内至少两个 lane 联合执行一个 task。相关位置： + +- `submit_helpers.h::alloc_won_slot()`：`atomicMax(state)`； +- `submit_helpers.h::populate_won_slot_from_submit()`：`atomicExch(drained)`； +- `submit_core.h::publish_won_slot()`、`clear_won_slot_state()`： + `atomicExch(state)`； +- `submit_core.h::claim_won_lane()`：`atomicExch(drained)`； +- `submit_core.h::decrement_won_remaining_is_last()`：`atomicSub(remaining)`； +- `submit_runtime.h::publish_joint_deposits()`：`atomicExch(any_pub)`； +- `submit_core.h::drain_block_won()`、`has_pending_won()`：读取 + `any_pub/state/drained`，在 A5 上均为 `atomicAdd(addr, 0)`。 + +PA Case1 的 QK/PV 为单 AIC lane，SF/UP 为单 AIV lane，没有 joint task， +trace 中也没有 `DrainWon`。当前由 worker-local 的 +`g_fdwic_joint_submit_seen` 关闭这些轮询，所以以上 BlockWon atomic 在该 Case 的 AICore +Submit/执行路径中动态次数为 **0**；AICPU 初始化仍会重置相关状态，joint 用例也仍走完整原协议。 + +### 3.5 Submit 窗口外和非本路径 atomic + +这些操作不属于用户关注的首末 Submit 耗时，但做端到端优化时不能忽略： + +- worker 启动屏障，`core_main.h`：96 次 + `atomic_fetch_add(started_count)`，以及等待期间重复读取 + `started_count/fatal`； +- worker 最终 drain，`submit_runtime.h`：96 次 + `atomic_fetch_add(replay_done)`，以及等待期间重复读取 `replay_done`； +- AICPU 初始化，`aicpu/control_plane.h`：12 个分片 cursor、frontier、 + fatal、replay_done、started_count、32 个 BlockWon 的 `any_pub` 和 + 128 个 won slot state 均以 atomic exchange 重置；65536 个 task cell + 各重置 flag/vend，共 131072 次 atomic exchange； +- AICPU 外层生命周期，`aicpu/aicpu_executor.cpp`：线程状态使用 + `std::atomic`，并轮询 `runtime->dist.done_count`。真实 A5 的 AICore 通过 + COND 寄存器发布完成，不走 A5Sim 中的 `done_count` atomic add； +- `debug_dump.h` 中的 atomic load 只在诊断 dump 时运行；centralized + scheduler 的 completion mailbox 和 SDMA completion atomic 不属于本 + FDWIC PA 执行路径，未计入本文。 + +异常路径中的 `set_fatal()` 使用 `atomic_exchange(g_dist.fatal, 1)`；上述 +成功运行没有触发。由于 fanin/frontier/屏障轮询次数依赖时序，不能把静态 +下界当作整次运行的 atomic 总数。若需要精确动态计数，应使用 worker-local 软件 +计数或独立诊断构建；当前 PMU 事件不直接给出 atomic 条数。直接在热路径增加共享 +计数器会反过来改变竞争形态。 + +## 4. 当前优化逻辑与效果 + +### 4.1 首个 joint task 前跳过 BlockWon 轮询 + +涉及文件： + +- `dist_engine/common/worker_state.h`：增加 worker-local、单调的 + `g_fdwic_joint_submit_seen`； +- `dist_engine/aicore/core_main.h`：每次运行开始时重置为 `false`； +- `dist_engine/aicore/submit_runtime.h`：当当前 active core mask 的 + popcount 大于等于 2 时，在该 Submit 的首次 drain 前置为 `true`； +- `dist_engine/aicore/submit_core.h`：flag 为 `false` 时， + `drain_block_won()` 和 `has_pending_won()` 直接返回。 + +flag 一旦变为 `true` 就不再清零。不能只根据“当前 task 不是 joint”跳过， +因为较快 worker 可能已经发布后续 joint slot，而较慢 worker 仍在处理前一 +task。当前方案利用所有 worker 回放相同 task stream 的条件：较慢 worker +到达自己的首个 joint task 时，会在首次 drain 之前打开轮询；此后保留原 +协议，所以不会漏掉已发布或稍后发布的 won slot。 + +对纯单 lane 的 PA Case1，flag 始终为 `false`。因此删除的是： + +- 每次 Submit 开头、归入 `EfDrain` 的 AIV `any_pub` load： + `64 * 1280 = 81920` 次； +- kernel loser 在 `Replay.end` 到 `Submit.end` 尾段的 load： + QK/PV 的 `512 * 64 = 32768` 次，加 SF/UP 的 + `512 * 63 = 32256` 次，共 65024 次； +- Submit 内合计 146944 次无效 `atomicAdd(addr, 0)`。 + +所以本次 atomic 优化直接缩短的是 `EfDrain` 和 loser 的 +`Replay.end -> Submit.end` 尾段，不是 Claim。三轮 joint-skip 版本的全局 +中位数为 5.171330 ms，相对基线下降 0.470915 ms（8.35%）。AIV 每核累计 +`EfDrain` 中位数由约 0.765 ms 降至约 0.409 ms；Replay 后尾段也明显 +缩短。同时 Claim 竞争上升，抵消了一部分收益。 + +### 4.2 复用参数 tag 扫描结果 + +该项不是 atomic 优化，目标是继续减少每核重复执行的 Submit 前端工作： + +- `calculate_output_layout()` 第一次扫描 tensor tag 时同时生成 + `output_mask` 和 `register_mask`； +- materialize 用 `output_mask` 只访问 OUTPUT 参数； +- register 用保存在 `DistSubmitCtx` 中的 `register_mask` 只访问 INOUT 和 + OUTPUT_EXISTING 参数； +- 不改变参数 tag、tensor map 行为或对外接口。 + +只加入 register mask 的三轮中位数为 5.186679 ms，相比 joint-skip 的 +5.171330 ms 没有稳定的全局收益；加入 output mask 后，最终三轮中位数为 +5.115620 ms。最终版本的每核 `Materialize+Register` 累计中位数约为 +AIC 1.416 ms、AIV 1.426 ms，基线分别约为 1.453 ms、1.451 ms。 +设备未锁独占且每组只有三轮，因此 mask 的独立收益只能视为方向性证据， +不能按两组中位数差值作严格因果拆分。 + +### 4.3 性能结果和证据文件 + +| 版本 | 三轮首末 Submit（ms） | 中位数（ms） | +| ---- | --------------------- | -----------: | +| atomic baseline | 5.642245（稳定样本） | 5.642245 | +| joint polling skip | 5.171330 / 5.239468 / 5.167902 | 5.171330 | +| + register mask | 5.201349 / 5.128588 / 5.186679 | 5.186679 | +| + output/register masks | 5.115620 / 5.145057 / 5.096685 | **5.115620** | + +最终中位数比基线快 9.33%，最好单轮比基线快 9.67%。最后一个 task 的 +96-worker Submit 起点波宽从基线 463.913 us 降至最终三轮中位数 +71.919 us，表明 worker 长尾明显收敛。1024 个 kernel span 的累计时长 +从基线 32.518 ms 到最终三轮中位数 32.607 ms，未缩短，证明收益来自 +Submit 调度而非 kernel 计算变快。 + +本地证据文件： + +- 基线：`outputs/TestPagedAttentionUnroll_Case1_20260717_023809/merged_swimlane_atomic_load.json`； +- `ld_dev()+nop(100)`：`outputs/TestPagedAttentionUnroll_Case1_20260717_035341/merged_swimlane_nop100.json`； +- `ld_dev()+nop(10)`：`outputs/TestPagedAttentionUnroll_Case1_20260717_035954/merged_swimlane_nop10.json`； +- 最终最好一轮：`outputs/TestPagedAttentionUnroll_Case1_20260717_055638/merged_swimlane_best_joint_poll_skip_arg_masks_5.096685ms.json`； +- 对应原始记录均为相同目录下的 `l2_swimlane_records.json`。 + +`ld_dev()+nop(100)` 和 `ld_dev()+nop(10)` 的完整 Submit 样本分别为 +5.343592 ms 和 5.401034 ms，但它们仅用于定位 atomic load 成本，已回退， +不能作为安全方案。普通 device load 加固定 NOP 不具备 atomic RMW 的同步 +语义，也不能保证 cache 可见性或顺序；延迟长短不能修复协议正确性。 + +## 5. 后续优化顺序与验证要求 + +本节是首轮分析时给出的候选方向。实际推进采用“单变量、低风险到高风险”的 +阶段门禁，并持续记录在第 7 节；若两处顺序不同，以第 7 节已经完成验证的结论为准。 + +建议按以下顺序继续，且每次只改一个变量： + +1. **Claim cursor 竞争。** 73728 次 `atomicMax` 是最大固定项。可以研究 + 确定性 owner、分层/分片 Claim 或减少无胜算 worker 参与，但必须保持 + AIC/AIV 负载均衡、ring slot 容量和 joint placement 语义。当前四分片是按 + task id 分片，不能假设再加 shard 一定能降低同一 task 的竞争。 +2. **completion/frontier。** 研究减少重复 `atomicMax(frontier)` 和连续 flag + 扫描，例如单 advancer、批量前推或分层 frontier；最终判断必须保持“只有 + 连续完成的 task 才能释放 heap”的约束。 +3. **fanin ready 轮询。** 统计失败 load 次数，再判断是否能利用已知拓扑、 + 本地完成缓存或更少的 drain 扫描。不能跳过 acquire-ready 条件后直接执行。 +4. **heap guard。** 当前无 RingBp，优先级较低。可以缓存 frontier/vend 做 + advisory fast path，但真正覆盖 ring 前仍需可靠的共享状态确认。 +5. **非 atomic 前端。** 继续观察 tensor map、PrepareMap、参数复制及每核约 + 0.8 ms 的 inter-submit gap；这些不能由替换 atomic load 解决。 + +每个候选改动至少验证： + +- PA Case1：A5Sim 正确性、真实 A5 正确性和 10 轮以上首末 Submit A/B； +- joint mixed（AIC+AIV）和 dual-AIV 的重复 task/slot 复用； +- heap/ring 压力、依赖未就绪、不同 kernel 时长和 worker 到达顺序； +- 96 核均有 1280 个 Submit，task id 完整为 0..1279； +- `DrainWon`、`RingBp`、kernel placement 和最终 drain 数量没有异常漂移； +- 同时报告中位数、离散度、最好/最差值，并记录设备是否独占。 + +真实 PA 与历史 standalone schema-v3 泳道中的 `Build` 和 `Replay` 是 lap +marker，会覆盖前面的阶段，不能与 +`Materialize/PrepareMap/Claim/Fanin/Register` 相加。当前 standalone +schema-v4 已改为显式互斥的 `WinnerBuild/LoserReplay/AllocComplete` 尾 span； +后续文档和脚本必须先按 capture schema 选择口径，不能把历史 lap 与 v4 尾 span +混为一类。 + +## 6. 独立 PA 调度复现的对应关系 + +`pa_scheduler/` 下当前验收的 CCEC 和 CPU 用例以本文的真实 PA 路径为模型， +目标是脱离 simpler 的编译和链接依赖后，仍能单独研究 Submit 调度性能。这里的 +“脱离 simpler”不表示删减 PA 调度逻辑。当前独立模型保留： + +- Case1 的 256 batch、Alloc/QK/SF/PV/UP 五 task 拓扑和 AIC/AIV active mask； +- 96 worker 全量回放、四分片 Claim cursor 和固定 73728 次 Claim atomicMax； +- TaskArgs/Tensor/TaskPayload/DistSubmitCtx 的关键 ABI 和真实 tag 扫描； +- materialize、TensorMap retire/lookup/insert、register、fanin、slot payload； +- EfDrain、Replay、WaitForSlot、HeapGuard、flag/vend/frontier 和最终 drain； +- 单 lane Case1 的 BlockWon 动态次数为零，以及真实泳道记录格式。 + +Case1 的 standalone 依赖图由 tensor owner 与每 worker TensorMap 的 lookup 共同 +收集，不是按 task 名字直接跳过依赖：Alloc 和 QK 的 fanin 均为 0；SF 依赖 QK； +PV 依赖 SF；UP 的多个输入/原地输出按 producer 去重后依赖 Alloc、SF、PV。 +所以每 batch 的 fanin 数是 `0+0+1+1+3=5`，默认 256 batch 的严格终态断言 +要求 `fanin_edges=1280`。EfDrain 只有在这些 producer completion flag 全部 +ready 后才执行 winner 负载。 + +该图只对等 Case1 的调度依赖。真计算 workspace 用统一的受控 A/B 输入分别验证 +QK/PV matmul、SF add 和 UP mul，并按 worker-kind 写独占输出 tile；这些数值输出 +没有按 QK→SF→PV→UP 串接成 PA 的真实 tensor 数据流。因此它能验证 fanin、完成 +发布、角色路由和各类引擎算术，不能验证后继 task 消费前驱真实数值的语义。 +当前也只覆盖 Case1 的单 block group、`q_loop=1` 和全单-lane 图；通用多 group、 +多 q-loop、跨迭代更新及 joint/mixed task 的依赖数量与竞争形态均未模拟。 + +当前三后端无参数默认使用 `real-compute`：CCEC/AscendC 的 QK/PV 运行真实 Cube +matmul，SF/UP 运行真实 Vector add/mul，每轮都含 GM load、计算、GM store 和完成 +等待；CPU 使用相同 workspace、角色路由和数学运算做协议回归，不代表 A5 引擎 +时间。可控 `scalar-nop` 只作为显式兼容/校准模式保留，其历史默认值按 +本文最好真实泳道的 44.170/53.729/27.626/1.565 us 校准。两种模式都不会在 +Claim、Register、PrepareMap 或等待路径中硬补 5 ms。 + +当前真计算默认次数为 QK/SF/PV/UP=`6,28,4,1`。依据是其每 task 完整 +load/compute/store span 已接近真实 PA 的 44.170/53.729/27.626/1.565 us；它 +优先保持 per-task core work 口径，不通过增加无关 repeat 把 standalone 总时间 +硬凑到 5.1 ms。standalone 未覆盖的真实控制流、代码布局和资源竞争仍会形成 +端到端差值，不能把这部分差值反推成缺少的 kernel repeat。 + +当前严格校验覆盖 73,728 次 Claim、每 task 唯一 winner、1,024 个 kernel、 +TensorMap/heap 最终状态、fanin、flag、vend、frontier、cursor、ring placement +和每 worker 的前端操作次数。2026-07-17 三个 CCEC 独立进程首轮为 +4.846431/4.798260/4.830184 ms，中位数 4.830184 ms；AscendC 独立进程首轮为 +4.917014 ms，均为 PASS；真实最好泳道为 5.096685 ms。 +差异主要来自真实 orchestration 与 `dist_submit_impl` 跨翻译单元，而 standalone +共享实现会和固定任务图一起被编译器优化。为制造编译边界而做的强制 noinline、 +拆设备目标和全局 memory clobber 实验曾分别触发状态破坏、device exception 或 +明显 RingBp，均已回退。 + +下列四阶段是普通 runtime `--profile-phases` 的历史诊断口径；调度器中的 +WaitForSlot 协议和普通泳道仍保留。 +Claim span 和 EfDrain 每 worker 调用 1,280 次；WaitForSlot 由 1,024 个 +kernel winner 调用；HeapGuard 由每 batch 的 +Alloc/QK/SF/PV winner 调用，共 1024 次。当前代表性 CCEC 轮次的累计中位数为： + +| role | Claim | EfDrain | WaitForSlot | HeapGuard | +| ---- | ----: | ------: | ----------: | --------: | +| AIC | 470.503 us | 711.005 us | 234.063 us（全 AIC 43 次等待） | 21.349 us（约 1 次 heap 等待） | +| AIV | 533.755 us | 401.962 us | 0.067 us（无等待） | 3.723 us（约 1 次 heap 等待） | + +完整构建、参数、内存占用、冷热运行口径和脱仓复制方法见同目录 +[PA 调度器独立复现与泳道使用指南](PA调度器独立复现与泳道使用指南.md)。后续调度优化应先在该 +独立用例做协议回归和阶段定位，再回到真实 PA Case1 做最终性能确认。 + +## 7. Atomic 消减阶段日志 + +从 2026-07-17 起，每个候选必须按以下节奏更新本节：先写静态证明和直接消减 +口径，再记录 standalone 三后端结果、压力/反例结果和性能 A/B，最后写是否允许 +进入真实 PA。失败、超时和回退同样保留，不能只记录正向数据。一次只验证一个 +变量，后续阶段不得把前一阶段的间接调度变化冒充为本阶段的直接 atomic 收益。 + +| 阶段 | 单变量 | 当前状态 | +| ---- | ------ | -------- | +| H1 | HeapGuard 首圈 fast path | 正确性与真实性能完成，保留并本地提交 | +| F1 | fanin 依赖检查顺序 | standalone 性能未改善，已回退 | +| 后续 | ready cache、退避、frontier、Claim | 按风险从低到高逐项验证 | + +### 7.1 阶段 H1：HeapGuard 首圈 fast path + +#### 7.1.1 修改范围与正确性证明 + +本阶段只在 standalone 的 `HeapGuard()` 中增加以下判断；原 slow path 未改： + +```cpp +while (!IsFatal(state)) { + if (worker.heap_next <= ring) { + return true; + } + // 原 frontier/vend 容量检查。 +} +``` + +判断放在 `IsFatal()` 之后，因此仍保留每次 HeapGuard 的 fatal 原子检查。profile +版本在 fast path 返回前仍累计 HeapGuard 时间，没有改变统计调用次数。 + +该判断依赖以下由现有代码确认的不变量： + +1. 每个 worker 的 `heap_next` 从 0 开始，materialize 只按单调逻辑地址推进； +2. 只有生成物理输出地址时才对 ring 取模； +3. 每个输出按 1 KiB 对齐，单 task 输出超过 ring 会直接失败； +4. 当前 task 若跨越 ring 尾部，会先把 `task_base` 推到下一圈； +5. HeapGuard 在当前 task 完成 materialize 后执行。 + +所以在 `uint64_t` 未溢出的前提下，`heap_next <= ring` 表示所有已分配逻辑区间 +仍位于 `[0, ring)`，取模后一一对应，尚不可能覆盖旧输出。`heap_next == ring` +也安全，因为已分配区间右端为开区间；第一个真正进入第二圈的非零输出会使 +`heap_next > ring`。不能用 `output_bytes <= ring` 替代该条件，后者无法证明 +历史分配没有 wrap。 + +本阶段不顺手修改 slow path 中 `heap_next - vend` 的无符号下溢边界，也不修改 +frontier、fanin、Claim 或 completion 协议，避免把两个正确性问题混在一起。 + +#### 7.1.2 直接 atomic 消减口径 + +默认 256 batch 的 standalone 共调用 1024 次 HeapGuard。默认 256 MiB ring 下， +每个 worker 的最终状态为： + +```text +heap_next = 206,569,472 B +ring = 268,435,456 B +剩余 = 61,865,984 B +``` + +1024 次调用全部处于第一圈，直接消减如下： + +| 操作 | 修改前 | 修改后 | 直接消减 | +| ---- | -----: | -----: | -------: | +| fatal atomic load | 1024 | 1024 | 0 | +| frontier atomic load | 1024 | 0 | 固定 1024 | +| vend atomic load | `V` | 0 | 动态 `V` | + +只有 `retire = frontier - H >= 0` 时原逻辑才读取 vend。按 1280 task、`H=64` +和输出 task 分布，`0 <= V <= 972`。所以默认 workload 直接删除的是 +`1024 + V` 次 HeapGuard 原子读取，不是所有观测指标的变化量。 + +#### 7.1.3 默认配置正确性门禁 + +恢复默认 256 MiB 配置前，本阶段已经完成一次 CCEC、AscendC、CPU 的全量构建， +以及 256 batch、零 NOP、开启四阶段统计、关闭泳道大缓冲区的完整运行： + +```bash +./run.sh run all --device 0 --batches 256 --runs 1 \ + --nop-count 0 --profile-phases --no-swimlane +``` + +三个后端均为 `semantic_status=PASS`、`postprocess_status=PASS`。严格校验包括 +73,728 次 Claim、1280 个唯一 winner、1024 个 kernel、1024 次 HeapGuard、 +fanin、flag、vend、frontier、cursor、TensorMap/heap 最终状态和每 worker +前端操作次数。该结果只证明 standalone 协议，没有替代真实输出 golden。 + +#### 7.1.4 16 MiB wrap 压力与 CPU 对照 + +为覆盖首圈之外的原 slow path，曾临时把 `kHeapBytes` 从 256 MiB 改为 16 MiB +并全量重编。256 batch 的静态状态为： + +```text +ring = 16,777,216 B +最终逻辑 heap_next = 209,385,472 B +跨 ring 尾部跳转 = 12 次 +首圈 fast path = 82 次 HeapGuard +原 slow path = 942 次 HeapGuard +``` + +CCEC 和 AscendC 的 256 batch 完整语义与阶段统计均 PASS。设备结果确实进入慢路径： +CCEC 的 HeapGuard wait event 为 `629 + 3 = 632`，AscendC 为 +`542 + 6 = 548`，不是仍停留在首圈的伪压力测试。 + +CPU 结果需要单独解释：fast 版的 32、64、128 batch 均快速 PASS；256 batch +在开启阶段统计时持续运行超过 8 分钟未结束，关闭阶段统计后也未在观察窗口结束。 +为排除 fast path 回归，保持 16 MiB 和 256 batch 不变，只临时撤掉 fast path、 +重编 CPU 原版，并用统一的 120 秒门限运行；原版同样超时。由此只能得出： + +- 该 CPU synthetic-heap 模型在 16 MiB、多圈、256 batch 下存在原有的长程活性 + 或 host 线程调度问题； +- 当前证据不能把该超时归因于 H1，也不能把 CPU 256 batch 记为 PASS； +- 真实 tiny-ring 数据 golden 仍是生产迁移前不可省略的门禁。 + +对照完成后已恢复 `kHeapBytes = 256 MiB`，临时原版构建不保留为源码改动。 + +恢复后再次执行 `build all`，并用与 7.1.3 相同的 256 batch 命令完成终验； +CCEC、AscendC、CPU 再次全部 PASS。该轮 HeapGuard wait event 均为 0，符合默认 +配置全程不 wrap 的静态结论。CCEC/AscendC/CPU 的 Submit span 分别为 +3499.426 us、3523.687 us、229235.747 us；CPU 时间仅反映 host pthread 调度， +不参与 A5 性能比较。 + +#### 7.1.5 CCEC A5 standalone 十样本 A/B + +baseline 和 H1 均使用独立进程首轮，各保留 10 个具有完整 PASS 结果的有效样本； +单位为微秒： + +```text +baseline = [ + 4451.574, 4847.797, 5089.239, 5403.477, 5227.148, + 5217.446, 4929.629, 4084.562, 5268.516, 5246.626 +] + +H1 = [ + 3759.041, 4142.396, 4582.740, 4134.989, 5670.639, + 4348.529, 4503.739, 5507.320, 4286.019, 4088.046 +] +``` + +| 指标 | baseline（us） | H1（us） | 相对变化 | +| ---- | ------------: | -------: | -------: | +| 样本数 | 10 | 10 | - | +| 中位数 | 5153.3425 | 4317.2740 | -16.2238% | +| 均值 | 4976.6014 | 4502.3458 | -9.5297% | +| p90 | 5268.516 | 5507.320 | +4.5327% | +| 最小值 | 4084.562 | 3759.041 | - | +| 最大值 | 5403.477 | 5670.639 | - | + +九组可按采集顺序配对的样本中 H1 有 7 组更快，配对变化中位数为 `-14.55%`， +两组反向样本为 `+8.48%` 和 `+34.83%`。动态指标也随调度时序变化：fanin +load 中位数从 91,254 降至 66,318.5（-27.33%），但 p90 上升 11.47%； +RingBp placement 中位数从 118.5 降至 101（-14.77%），p90 下降 3.76%。 + +另有两次没有生成完整结果的 baseline 异常：首次外层 launch 约四分钟无结果后 +人工中断；后续 baseline `r9` 在 60 秒门限超时。二者均未计入耗时分布，也不能 +记为 PASS 或 FAIL；异常后设备 smoke 恢复正常。正式 A/B 必须让两边使用相同 +timeout，并把 timeout 率单独报告，避免只统计成功样本造成幸存者偏差。 + +fanin 和 RingBp 代码并未在 H1 修改，其变化幅度又远大于固定 1024 次 frontier +读取，因此只能解释为 worker 到达、依赖完成和 drain 相对时序改变后的间接效应。 +当前中心趋势显示方向性收益，但 H1 的 p90 变差且离散较大，不能声称尾延迟稳定 +改善，更不能把 16.22% 全部归因于 `1024 + V` 次直接 atomic 消减。 + +#### 7.1.6 阶段结论与下一门禁 + +H1 当前状态是“standalone 默认配置通过，允许进入真实 PA 正确性验证”，不是 +“真实 PA 已优化完成”。继续推进前按以下顺序执行： + +1. 默认 256 MiB 恢复后重建三后端并再次执行完整回归，确认临时压力常量无残留； +2. 对真实 `dist_submit_wait_heap_capacity()` 做一次单点、等价修改； +3. 运行 fully-distributed PA Case1 的 A5Sim golden； +4. 运行已有 68 KiB `AllocFillRunAhead`、`AllocHeapBackPressure` 和 MB6 heap + reclaim 用例，验证真实输出没有 premature reuse； +5. A5 正确性通过后，再做双方相同门限、独立进程、交错顺序的至少 10 轮 A/B； +6. H1 通过真实门禁后，下一项低风险候选是只调整 fanin 检查顺序；ready cache、 + backoff、frontier 合并和 Claim 参与者缩减必须分别作为后续单变量阶段。 + +standalone 使用 synthetic heap 且计算 kernel 由 NOP 模拟，它能验证控制协议、 +分支和 atomic 计数，但不能用 synthetic 地址证明真实 tensor 内容未被提前覆盖。 + +#### 7.1.7 真实 fully-distributed 迁移与正确性结果 + +standalone 门禁通过后，只在真实实现的 +`dist_submit_wait_heap_capacity()` 中加入同形判断： + +```cpp +while (!fatal_set()) { + if (ctx.self->heap_next <= ring) return true; + // 原 slow path 不变。 +} +``` + +生产实现没有 standalone 的 HeapGuard phase profile，因此没有搬入额外统计字段。 +判断仍位于 `TRACE_SPAN_BEGIN` 之后和 fatal 检查之内：原本不等待的成功路径就不会 +生成 RingBp 事件，新 fast path 的 trace 语义不变。真实函数共被 1280 个 winner +调用，UP 因 `output_bytes == 0` 在进入 atomic-bearing loop 前返回；实际消减口径 +仍是 Alloc/QK/SF/PV 的 1024 次 guard。 + +测试统一使用用户环境 `/home/q00473782/.venv`、CANN 9.1、本地 GCC 15，并按 +仓库 CI 固定 PTO-ISA 到： + +```text +ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8 +``` + +未传 SHA 时测试会把 managed clone 更新到当时的 `origin/main ea90d400`，该组合的 +CPU stub/`pto_instr.hpp` 出现大量重复定义，kernel 未能编译；这不是 H1 结果， +也没有通过修改 PTO-ISA 头文件规避。非交互 shell 还必须显式加入用户本地 +`g++-15` 路径，不能假设会自动 source `.bashrc`。 + +真实数据门禁结果如下： + +| 平台 | 用例 | 覆盖点 | 结果 | +| ---- | ---- | ------ | ---- | +| A5Sim | `AllocFillRunAhead67`，68 KiB | 首圈边界 | PASS | +| A5Sim | `AllocFillRunAhead128`，68 KiB | 跨圈 slow path | PASS | +| A5Sim | `AllocHeapBackPressure`，68 KiB | 等待、回收、真实数据 golden | PASS | +| A5Sim | PA Case1，真实 kernel | 256 batch 数值 golden | PASS | +| A5 | `AllocFillRunAhead67`，68 KiB | 首圈边界 | PASS | +| A5 | `AllocFillRunAhead128`，68 KiB | 跨圈 slow path | PASS | +| A5 | `AllocHeapBackPressure`，68 KiB | 等待、回收、真实数据 golden | PASS | +| A5 | PA Case1，真实 kernel | 256 batch 数值 golden | PASS | + +PA A5Sim 的真实 kernel orchestration 约为 38.824 s。另一次带 +`--use-example-exec-time` 的 A5Sim 调度运行也 PASS，但该选项会跳过数值 golden， +所以只作为控制流证据，不计入上表的正确性依据。 + +MB6 需要保留两个基线问题，不能写成 PASS： + +1. `Normal` 已完成底层数值运行，但 `DistRuntimeContractMixin` 随后因没有捕获到 + `[dist] DEPSIG` 而失败；当前 `src/`、`simpler_setup/` 和 runtime 构建源码中 + 查不到 `PTO_DIST_DEPSIG/DEPSIG` 实现，因此属于测试契约与当前 runtime 不匹配， + 不能靠加 shell 环境变量伪造 oracle。 +2. `Heavy` 的 8 MiB/H=64 A5Sim 压力在运行中触发 native abort。保持全部参数 + 不变、只撤销生产 H1 后，基线同样以相同调用路径 abort；所以该失败不是 H1 + 引入，但也不能作为 H1 的通过项。`FullCore36` 在 Heavy 基线已经失败后未继续跑。 + +综合现有证据，H1 已通过目标 PA 与真实 68 KiB 回卷/回收的 A5Sim+A5 golden， +可以进入真实 PA 性能 A/B。MB6 的两项既有问题作为未关闭风险保留，不能被其他 +PASS 掩盖，也不在 H1 中顺手修改测试框架或 heap slow path。 + +#### 7.1.8 真实 A5 PA Case1 十对性能 A/B + +真实性能使用提交前基线 `290dbda0` 的 detached 临时 worktree 和带 H1 的主 +worktree。两边均预先完成一次不计入统计的 warm run，随后按 +`baseline -> H1` 顺序采集 10 对独立 pytest 进程。统一条件为： + +- A5 device 0，真实 PA kernel，256 batch； +- 用户 `/home/q00473782/.venv`、CANN 9.1、GCC 15； +- PTO-ISA 固定为 `ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8`； +- 开启 L2 swimlane，不使用 `--use-example-exec-time` 或 `--skip-golden`； +- 每个进程统一 180 s timeout；双方均 10/10 PASS、0 timeout； +- 为避免自动生成数百 MiB merged JSON 影响采集周转，双方仅在测试进程内对称地 + 关闭 post-case converter；设备执行和原始 `l2_swimlane_records.json` 不变； +- 每个原始文件均有 122,880 个 Submit 事件，指标直接取最早 start 到最晚 end。 + +预热样本为 baseline 5.134950 ms、H1 5.116055 ms，只验证两边构建和设备状态， +不进入下面统计。10 对正式样本为： + +| 对次 | baseline（ms） | H1（ms） | 配对变化 | +| ---: | ------------: | -------: | -------: | +| 1 | 5.149955 | 5.301485 | +2.942% | +| 2 | 5.331087 | 5.110060 | -4.146% | +| 3 | 5.137861 | 5.098696 | -0.762% | +| 4 | 5.150200 | 5.119599 | -0.594% | +| 5 | 5.129350 | 5.112860 | -0.321% | +| 6 | 5.146475 | 5.129671 | -0.327% | +| 7 | 5.111929 | 5.105638 | -0.123% | +| 8 | 5.274224 | 5.229773 | -0.843% | +| 9 | 5.114477 | 5.137014 | +0.441% | +| 10 | 5.125086 | 5.125040 | -0.001% | + +| 统计 | baseline（ms） | H1（ms） | 相对变化 | +| ---- | ------------: | -------: | -------: | +| 中位数 | 5.142168 | 5.122320 | -0.386% | +| 均值 | 5.167064 | 5.146984 | -0.389% | +| nearest-rank p90 | 5.274224 | 5.229773 | -0.843% | +| 最小值 | 5.111929 | 5.098696 | - | +| 最大值 | 5.331087 | 5.301485 | - | +| 样本标准差 | 0.073960 | 0.065764 | - | + +H1 在 10 对中 8 胜 2 负；配对变化中位数为 -0.324%，均值为 -0.373%。第 1、2 +对分别有 +2.942% 和 -4.146% 的反向大波动，且采集顺序固定为 baseline 在前， +因此这 10 对只能支持“小幅方向性收益”，不能声称统计上已经稳定到每轮必胜。 +与 standalone 的 -16.22% 中位数不同，真实 PA 的约 0.3%～0.4% 中心改善更符合 +固定删除少量 HeapGuard atomic 的规模；standalone 的巨大间接调度变化不应外推。 + +最佳 H1 样本为第 3 对的 5.098696 ms，单独生成的泳道文件为： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260717_173313/ +merged_swimlane_heapguard_first_lap_fastpath_5.098696ms.json +``` + +原始 A/B 文件分别位于临时 baseline worktree 和主 worktree 的对应 timestamp +目录。自动 converter 被关闭的正式样本仍保留 raw JSON；上述最佳样本在统计完成后 +单独补做了 converter，未重新执行 device workload。 + +#### 7.1.9 H1 最终阶段决定 + +H1 满足保留条件：正确性证明成立，standalone 三后端默认配置 PASS，真实 +A5Sim/A5 的 68 KiB 首圈/跨圈/回收 golden 与 PA Case1 golden 全部 PASS，真实 +A5 十对中心趋势和 p90 均未回退，并且直接 atomic 数量确定下降。因此将 standalone +与生产同形改动、测试证据文档作为一个本地 commit 保存，不 push。 + +真实性能收益较小且仍有单轮反向，不把 H1 宣传成大幅优化。下一阶段 F1 只研究 +fanin 依赖检查顺序；不得同时加入 ready cache 或退避，避免失去因果归属。 + +### 7.2 阶段 F1：fanin 依赖检查顺序 + +#### 7.2.1 候选改动与静态分析 + +F1 只在 standalone `CollectFanin()` 完成原有去重和 16 项截断后，将有效 +fanin 前缀按 producer task id 降序排列。排序放在截断之后，所以不改变 +原实现选中的依赖集合、`kMaxFanin=16` 语义或 slot ABI。fanin ready 是 AND +条件，仅改变检查顺序不改变“所有 producer 都完成后才可执行”的结果。 + +PA 中只有 UP 包含 3 个 fanin，原顺序是 `[SF, PV, Alloc]`，降序后为 +`[PV, SF, Alloc]`。PV 依赖 SF，因此在其他调度状态不变时： + +- SF、PV 均未完成：两种顺序都在第 1 次 ready load 后返回； +- SF 已完成、PV 未完成：原顺序读 2 次，新顺序读 1 次； +- PV 已完成：由依赖关系可知 SF 已完成，两种顺序都成功读完 3 项。 + +所以在固定调度状态下，UP 每次失败检查的 ready atomic load 不会增加， +且在 SF 已完成而 PV 尚未完成的窗口可减少 1 次。QK 的 fanin 为 0， +SF/PV 各为 1，本改动对它们不产生重排。不过，排序本身会改变标量指令和 +worker 到达时序；真实动态调度不能由上述固定状态推导出必然性能收益。 + +#### 7.2.2 standalone 正确性与完整运行 + +F1 修改后重建 CCEC、AscendC 和 CPU 三后端，并分别运行 smoke、256 batch 零 +NOP、256 batch 当时默认 NOP。全部结果的语义断言和后处理都为 PASS；完整用例 +仍为 73,728 次 Claim、1,280 个唯一 winner、1,024 个 kernel，placement 总数为 +1,024。关键单轮结果如下： + +| 场景 | CCEC `fanin / Submit us` | AscendC | CPU | +| ---- | -----------------------: | ------: | --: | +| smoke | `6 / 34.912` | `5 / 40.142` | `6 / 156383.064` | +| 256 batch、零 NOP | `42546 / 3415.839` | `25515 / 3498.038` | `1983 / 163840.941` | +| 256 batch、当时默认 NOP | `40958 / 4019.835` | `47692 / 4283.306` | `1051412 / 185456.323` | + +CPU 只用于语义对照，不作为 A5 性能依据。单轮 fanin load 对 worker 调度非常 +敏感，不能用上表三个数值直接归因，因此又执行了独立进程的交错 A/B。 + +#### 7.2.3 CCEC 十对交错 A/B + +基线固定在 H1 提交 `2c3dd1e2`，使用 detached worktree +`/tmp/simpler-f1-baseline`。两端都用 256 batch、当时默认 NOP、关闭泳道和 phase profile； +每个样本是独立进程首轮，统一 60 s timeout。前 5 对为 baseline -> F1， +后 5 对为 F1 -> baseline。双方均 10/10 PASS、0 timeout、Claim=73,728、CAS retry=0。 + +| 指标 | baseline | F1 | 相对变化 | +| ---- | -------: | -: | -------: | +| Submit 中位数（us） | 4290.401 | 4623.944 | **+7.774%** | +| Submit 均值（us） | 4498.804 | 4684.302 | **+4.123%** | +| Submit nearest-rank p90（us） | 5303.373 | 5219.956 | -1.573% | +| Submit 样本标准差（us） | 944.376 | 475.794 | - | +| fanin load 中位数 | 67914 | 65433.5 | -3.652% | +| fanin load 均值 | 82844.5 | 72219.2 | -12.826% | +| fanin load nearest-rank p90 | 118955 | 94088 | -20.905% | + +F1 的 Submit 为 4 胜 6 负；配对相对变化中位数为 `+6.439%`，均值为 +`+8.410%`。交换顺序后，后 5 对的配对变化中位数仍为 `+3.423%`，没有把 +中心趋势变成收益。fanin load 的配对变化中位数为 `-13.582%`，但均值为 +`+0.665%`，单对范围从 `-58.082%` 到 `+89.165%`，说明其仍强烈受动态调度影响。 + +F1 的 p90 略好主要由 baseline 的 5.303 ms 和 6.640 ms 慢样本抬高；在中位数、 +均值和配对中心全部回退时，不能单独用这个 p90 宣称收益。原始日志保留在： + +```text +/tmp/f1_standalone_ab_20260718_010300/ +``` + +#### 7.2.4 阶段决定 + +F1 在固定状态下的 atomic load 不增证明成立，三后端语义也全部通过；但 +standalone 的 Submit 中位数、均值和配对中心都没有改善。因此不将这个启发式 +重排迁移到真实 `dist_submit_collect_fanin()`，也不进行真实 PA A/B。standalone +候选代码已撤回，本节保留负结果，防止后续重复同一实验。 + +### 7.3 Atomic 次数的证据边界 + +PMU 不直接提供 atomic 调用次数；atomic 次数仍以源码不变量、worker-local +软件计数和泳道 logical/physical 闭环为准；不能用其他事件计数器反推 atomic +条数。Atomic 优化的正式端到端收益仍使用无诊断构建的独立进程 A/B。 + +### 7.4 阶段 D1：精确分类 fanin 与 frontier 动态原子次数 + +#### 7.4.1 计数实现与闭合关系 + +D1 已在 standalone 公共调度器中增加 worker-local 软件计数，不新增共享 atomic： + +- fanin 每次 flag load 只递增一个分类：`ready` 或 `not_ready`； +- 每次 completion 递增一个 frontier initial load； +- ready flag 与紧随其后的 FetchMax 共用一个一一对应计数 `A`； +- 扫描遇到 not-ready flag 退出时递增 terminal load。 + +计数先保存在本核 `LocalStats`，kernel 结束时才发布到独占 `WorkerResult`。 +`WorkerResult` 在 D1 从 704 B 扩展到 768 B；分类结果和后续 atomic 泳道计数 +都先保存在 worker 私有字段，最终一次性发布，不新增共享 atomic。生产 +DistGlobal/DistCore offset 和 `LocalSlot` ABI 不变。 +三后端完整重建后，smoke 和 256 batch 当时默认 NOP 均通过全部语义断言。 + +对任一 worker，若其完成数为 `Cw`、fanin 边数为 `Ew`、失败 load 为 `Fw`，则 +逐核检查： + +~~~text +frontier_initial_w == Cw +frontier_terminal_w == Cw +fanin_ready_w >= Ew +fanin_ready_w - Ew <= 2 * Fw +~~~ + +最后一个上界来自 PA 最大 fanin 为 3：一次失败检查最多先重读两个 ready 前缀， +然后在一个 not-ready 依赖上返回。全局 `T=1280` 时还必须满足： + +~~~text +frontier_initial = frontier_terminal = T +frontier_ready = frontier_FetchMax = A >= T +frontier_flag_loads = A + T +Submit+completion ops = 79872 + G + 2A +~~~ + +其中 `G=fanin_ready+fanin_not_ready`。CCEC/AscendC 的 `A` 对应真实 A5 +atomicMax；CPU 的 FetchMax 是 load/CAS 实现，`A` 只能解释为逻辑调用数。 + +#### 7.4.2 CCEC 十轮动态基线 + +在 256 batch、当时默认 NOP、关闭泳道和 phase profile 的同一进程十轮基线中，全部 +语义断言和上述计数恒等式均 PASS： + +| 指标 | 中位数 | 均值 | nearest-rank p90 | 范围 | +| ---- | -----: | ---: | ----------------: | ---: | +| Submit（us） | 4776.940 | 4802.821 | 5335.931 | 3805.757～5535.473 | +| fanin 总 load `G` | 93201.5 | 99442.6 | 145045 | 56620～151260 | +| fanin ready | 7323.5 | 7418.0 | 9370 | 4191～9933 | +| fanin not-ready | 86675.5 | 92024.6 | 136347 | 49038～141327 | +| frontier FetchMax `A` | 15365 | 14957.8 | 18957 | 5587～20026 | +| Submit+completion ops | 203803.5 | 209230.2 | 264969 | 164508～269046 | + +日志位于： + +~~~text +tests/atomic_probe/pa_scheduler/outputs/atomic_diagnostics/ + ccec_baseline_10_20260718_023551.log +~~~ + +该十轮不是独立进程交错 A/B，因此只用于确认动态规模，不用于宣布性能收益。 +分类代码新增约 `2T+A` 次 worker 私有 scalar 增量，且结果 sidecar 扩大了一条 +cache line；本节绝对 Submit 不能与 D1 之前的二进制直接归因比较。 + +#### 7.4.3 对下一候选的约束 + +当前动态最大项不是 ready 前缀，而是 not-ready 重试：其中位数约 86676 次； +frontier helping 的额外 FetchMax 中位数为 `A-T=14085` 次。ready-prefix cache 在 +固定轮询序列下最多删除 `fanin_ready-E=6043.5` 次中位重复 load，约占 fanin 总量 +6.5%、Submit+completion ops 3.0%，预期只能是小幅候选。 + +仍先做该候选，因为它不改变依赖集合、flag 发布或跨核共享状态，正确性风险最低。 +但 probe 变短后可能在同一等待期间发起更多 not-ready 重试，所以整轮 `G` 不保证 +静态单调下降。保留门槛是：优化后 `fanin_ready == fanin_edges == 1280`，CCEC +独立进程交错十对中 fanin 总 load 配对中心下降、Submit 中心不回退；否则记录负 +结果并撤回，不迁移真实 FDWIC。standalone 第一版只验证当前单-lane PA Case1， +不能拿它的 PASS 代替 joint/BlockWon 覆盖。 + +### 7.5 阶段 O2：atomic schema-v3 泳道与计数闭环 + +本节保留 atomic ABI 在 schema-v3 中建立和验收的完整过程。当前 raw 已 +升为 schema-v4，atomic site/op/flags、PollBatch 与 weighted summary 语义不变； +atomic ABI 没有重定义。phase schema 则新增排他父区间，以真实 Submit 尾 span +替换旧 lap，将 EfDrain 改为显式 span，并禁止未使用的 `DrainWon`。 + +#### 7.5.1 观察目标与证据拆分 + +O2 先完成 atomic 观察链路，不在同一阶段继续消减 atomic。atomic 泳道回答 +“每个 site/op 执行了多少次逻辑调用”：direct Atomic 逐条给出 source-issue +或本核 return-ready bracket；显式等待区内六类 observation load 用带精确 +`call_count` 的 PollBatch 给出逻辑轮询 episode。 + +direct trace 会增加时间戳、分支和私有 record 写入，PollBatch 还会增加等待区内 +的私有累计与边界落盘，因此它是诊断运行，不能把开启 trace 后的 Submit span +直接当成正式性能基线。 + +#### 7.5.2 十五个 site、四种 op 与六类 PollBatch + +`AtomicSite` 是 raw/merged trace 的稳定编号，当前覆盖 standalone PA 公共调度器中 +所有显式共享 atomic 源码调用点。`AtomicOp` 只有 `Load/Exchange/FetchAdd/FetchMax` +四种；A5 CCEC 的 `Load` 仍是 `atomicAdd(address, 0)`，不是普通 GM load。 + +| id | AtomicSite | AtomicOp | 调度语义 | +| --: | ---------- | -------- | -------- | +| 0 | `StartupIncrement` | `FetchAdd` | worker 启动计数发布 | +| 1 | `StartupPoll` | `Load` | worker 启动屏障轮询 | +| 2 | `FatalPoll` | `Load` | 成功/异常路径 fatal 检查 | +| 3 | `FatalSet` | `Exchange` | 首次异常发布 | +| 4 | `ClaimMax` | `FetchMax` | 分片 cursor Claim | +| 5 | `FaninFlagLoad` | `Load` | 依赖 ready 检查 | +| 6 | `CompletionVendExchange` | `Exchange` | task vend 发布 | +| 7 | `CompletionFlagExchange` | `Exchange` | task flag 发布 | +| 8 | `FrontierInitialLoad` | `Load` | completion 开始时读取 frontier | +| 9 | `FrontierFlagLoad` | `Load` | 连续完成区间扫描 | +| 10 | `FrontierMax` | `FetchMax` | frontier 帮助前推 | +| 11 | `HeapFrontierLoad` | `Load` | HeapGuard slow path 读取 frontier | +| 12 | `HeapVendLoad` | `Load` | HeapGuard slow path 读取 retire vend | +| 13 | `ReplayDoneIncrement` | `FetchAdd` | worker 回放结束计数发布 | +| 14 | `ReplayDonePoll` | `Load` | 最终 drain 的 replay_done 轮询 | + +schema-v3 要求 `metadata.trace_schema_version=3` 且 `l2_swimlane_level=4`，并将 +**逻辑调用数**与**物理记录数**分开。direct Atomic 仍是一条源码调用 +对应一条同时包含 `start_cycle/end_cycle` 的记录；只有下列六类 observation load +在匹配的显式等待区内才允许聚合为 PollBatch： + +| `site_id` | site | op | +| ---: | --- | --- | +| 1 | `startup_poll` | `Load` | +| 2 | `fatal_poll` | `Load` | +| 5 | `fanin_flag_load` | `Load` | +| 11 | `heap_frontier_load` | `Load` | +| 12 | `heap_vend_load` | `Load` | +| 14 | `replay_done_poll` | `Load` | + +standalone 没有真实 PA 追加的 BlockWon site，也没有允许聚合的幂等失败 exchange； +不能照搬真实 PA 的“九类 observation load 加一类 exchange”allowlist。同一 site 在 +显式等待区外的一次性或 opportunistic 读取仍逐条记录。 + +raw 的 `auxiliary` 保存 site id。direct Atomic 的 `flags` 约定为：低 4 bit 是 op id， +bit 4 表示返回旧值是否参与后续判断，bit 5 仅对 Load 表示观察值是否为零，bit 6 +表示是否有“返回值本核可消费”依赖证据，bit 7 必须为 0；bits 8..31 仅对 direct +FetchMax 保存软件 retry 数，不能按调用次数解析。能归属任务时 `task_id` 写真实 +task id，生命周期 atomic 写 `-1`。 + +PollBatch 的 `flags` 则必须同时满足：低 4 bit 为 `Load(0)`、bit 4 为 1、bit 5/6 +为 0、bit 7 为 1，bits 8..31 保存 `1..0xFFFFFF` 的精确无符号 24 bit +`call_count`；`task_id=-1`、`function_id=-1`。达到 `0xFFFFFF` 时先落盘，再从 +1 开启下一条 batch，不能饱和后丢计数。 + +converter 把 direct Atomic、PollBatch 和 ClockBaseline 放在对应 AIC/AIV 的原 +scalar lane，atomic 不再伪装成与 scalar 并行的独立执行单元。direct 名称显式带边界： +`atomic.return_ready..#` 或 +`atomic.source_issue..#`；category 也分别为 +`atomic.return_ready`/`atomic.source_issue`，无需点开 span 即可过滤区分。direct +args 中保留 `call_count=1`、整数 `cycles`、site/op、 +`result_used`、`return_ready_observed`、`completion_boundary`、Load 的 +`value_zero` 和 FetchMax 的 `retries`。Perfetto 的浮点微秒显示不用于 +替代 raw 整数 tick。 + +PollBatch 名称为 `atomic.poll_batch..load×`，category 为 +`atomic.poll_batch`；args 还明确给出 `poll_window_cycles`、 +`batch_semantics=observation_load_calls`、 +`duration_semantics=logical_poll_episode_envelope_not_single_atomic_latency` 和 +`may_contain_interleaved_direct_atomics=true`。一个等待区可以同时累计多个 site， +所以不同 PollBatch 窗口可以重叠，窗口内也可能交错 direct Atomic。 +host 文字分析也保持两套口径：`[TRACE_ATOMIC]` 只统计 direct bracket， +`[TRACE_ATOMIC_POLL]` 只统计 episode 数、精确逻辑调用数和等待包络分布。 + +#### 7.5.3 按调用点语义区分的两种结束边界 + +不能按 `Exchange/FetchAdd/FetchMax` 指令名称一概选边界，必须看该源码 +调用点是否真正消费 atomic 返回的旧值。当前有两种口径： + +1. `source_issue_bracket`： + +~~~text +begin = get_sys_cnt() +old = atomic(...) +end = get_sys_cnt() +~~~ + +它用于返回旧值本来就不使用的发布型调用。`end` 与 `old` 无数据 +依赖，只表示源码发射包围区间；不表示返回值就绪、atomic retire 或 +其他核可见。 + +2. `return_value_ready`： + +~~~text +begin = get_sys_cnt() +old = atomic(...) +asm volatile("MOV old, old; MOV end, SYS_CNT" + : "+l"(old), "=&l"(end)) +~~~ + +该边界只用在协议本来就要判断 `old` 的 `Load/FetchMax` 调用。CCEC AIC +和 AIV 后端已确认生成紧邻的 `ATOM -> dependent MOV -> MOV SYS_CNT`； +`=&l` 防止时间戳输出与 atomic 返回寄存器重叠。它能证明 `old` 已可被 +本核 scalar 消费，仍不证明跨核全局可见。默认不加 DSB/ISB/额外 GM +地址依赖；这些操作要么本平台后端不支持，要么会明显改写被测路径。 + +standalone 的五个 `Exchange/FetchAdd` 调用点都不消费返回旧值，但共享 +新值仍由协议的后续 load 消费： + +| 发布调用点 | 旧值 | 新值消费者 | b1 热路实际情况 | +| ------------ | ---- | ------------ | ---------------- | +| `StartupIncrement` | 丢弃 | `StartupPoll` | 96 次发布，所有 worker 参与轮询 | +| `ReplayDoneIncrement` | 丢弃 | `ReplayDonePoll` | 96 次发布，所有 worker 参与轮询 | +| `CompletionFlagExchange` | 丢弃 | `FaninFlagLoad`/`FrontierFlagLoad` | 每 task 发布一次 | +| `CompletionVendExchange` | 丢弃 | wrap 后的 `HeapVendLoad` | b1 首圈 fast path 不读 vend | +| `FatalSet` | 丢弃 | `FatalPoll` | 成功 b1 不执行 `FatalSet` | + +所以强制这五处消费 `old` 会把原 no-return/发布路径改成等待返回型 +观测变体：Startup/Replay 会改变屏障到达和轮询次数，vend 会推迟 flag +发布，flag 会推迟 frontier helping。如果之后需要这种返回路径对照，必须 +做独立、单 site mask 的 A/B 诊断，不能把 B 的数据称为原 PA 热路。更贴近 +PA 的端到端 GAP 是“发布发射 -> 自然消费者首次观察到新值”；可以 +复用现有 load 记录做离线派生，但其包含互连可见性、消费者调度与轮询 +间隔，且在对外报告跨核时间差前还需单独验证各核 `SYS_CNT` 的对齐性。 + +两种 direct 边界的 `end` 都在本条 64 B trace record 写入之前取得，所以本条 +duration 不直接包含自己的 record 写入；但这次写入和附加指令会影响 +后续 atomic 到达与竞争，整轮仍是插桩运行。direct record 写入不会隐式 +关闭活跃 PollBatch，否则等待区内自然交错的发布/推进 atomic 会把一个逻辑等待 +episode 人为切碎。 + +PollBatch 使用另一种时间语义：`duration`/`poll_window_cycles` 是从该 site 在显式 +等待区内首次累计调用到边界关闭的逻辑等待 episode 包络。它不是独占 scalar 时间， +不是 `call_count` 次 atomic 延迟之和，也不是其中某次 load 的 return-ready 延迟； +不同 site 的窗口可以重叠，窗口内还可能包含 direct Atomic，不能把 PollBatch 混入 +direct 单次延迟的 median/p95。 + +以下是历史 schema-v3 边界实现复用真实 PA 规则时的约束；其中 lap helper 仍保留 +历史兼容能力，但当前 standalone schema-v4 producer 已不再调用 +`ResetTraceLap/WriteTraceLap`： + +1. 显式等待区退出时关闭与该 region 匹配的 PollBatch； +2. `TraceTimestamp` 先采 cycle，再以该 cycle 关闭全部活跃 batch，然后写 phase + begin/end； +3. `ResetTraceLap` 在推进 lap 起点前关闭，`WriteTraceLap` 在写 lap 前用同一 + `end_cycle` 关闭； +4. Kernel begin/end 都经过 `TraceTimestamp`，所以 PollBatch 不能跨入或跨出 Kernel； +5. `FlushTraceCore` 的最终关闭只作防御性兜底，不能代替上述语义边界。 + +开启 atomic trace 时，每个 worker 在最终 drain 之后额外记录两条 +`ClockBaseline`：一条是连续两次 `get_sys_cnt()`，另一条是纯寄存器依赖 +hook 后读 `SYS_CNT`。它们只给出同一二进制、同一物理核上两类边界的 +计时分辨率和固定底噪分布，使用边界是： + +- AIC 与 AIV 分开报告 median/p95/max； +- 不把某个 role 的中位数逐条从 atomic duration 中相减； +- 不因某条 atomic 接近 ClockBaseline 就声称该 atomic 没有竞争或没有等待； +- 不用 ClockBaseline 推导 atomic retire、cache 一致性或跨核可见性时刻。 + +#### 7.5.4 计数闭环与容量门禁 + +schema-v3 结果只有同时满足逻辑调用与物理记录闭环才可进入正式分析。设 +`direct_atomic_records` 是 bit 7 为 0 的物理 Atomic 条数，则逐核和全局都必须满足： + +~~~text +logical_atomic_calls = direct_atomic_records + Σ(PollBatch.call_count) +physical_atomic_records = direct_atomic_records + poll_batch_records +physical_atomic_records + = logical_atomic_calls - batched_poll_calls + poll_batch_records +~~~ + +这里 `batched_poll_calls` 是所有 PollBatch `call_count` 之和，不是 batch 条数。 +producer 的逐核 state、host 扫描 raw 行、导出的 metadata 和 converter 重算必须同时 +闭合以下七项： + +~~~text +records +atomic_records +clock_baseline_records +atomic_calls +batched_poll_calls +poll_batch_records +dropped_records +~~~ + +此外还必须满足： + +1. 每条 PollBatch 都属于六类 allowlist，flags 合法且 `call_count>0`； +2. 每个 worker 恰有 2 条 ClockBaseline，因此 96 核全局固定为 192 条； +3. 原 phase、动态 wait、物理 Atomic 和 ClockBaseline 总数与 trace header 精确相等； +4. 每核和全局 `dropped_records==0`，raw 到 merged 后物理 Atomic 条数不变； +5. 原有 Claim、winner、kernel、fanin、frontier、cursor、heap 和每 worker + 1280 Submit 语义断言仍全部 PASS。 + +trace 容量不足时必须明确报 overflow 并判该轮观察无效，不能只分析前缀，也不能 +根据静态公式补齐被丢弃的 duration。CPU pthread 启动轮询可能远多于 A5，CPU 的 +容量结果不能替代真实 A5 CCEC 的闭环；CPU/AscendC 主要用于公共 schema、编译和 +语义回归。 + +#### 7.5.5 Atomic 观察与正式性能隔离 + +Atomic 泳道是诊断构建：direct record 和 PollBatch bookkeeping 都可能改变 worker +到达与共享状态竞争，因此开启 trace 后的 Submit span 不作为无诊断性能基线。 +本节只使用 raw、producer、metadata 与 converter 的 atomic logical/physical +计数闭环，不混入其他观察机制的数据。 + +#### 7.5.6 当前实现状态、历史证据与正式重采矩阵 + +当前 schema-v4 源码完整沿用 schema-v3 建立的六类 PollBatch、精确 +`call_count`、七项 summary 闭环、两条 ClockBaseline 和 scalar lane converter。 +以下先保留 2026-07-18 schema-v3 边界修复版 CCEC b1/b256 的历史证据： + +| 样本 | winner 负载 | 总 `records` | 逻辑 `atomic_calls` | direct | 物理 Atomic | `batched_poll_calls` | PollBatch | ClockBaseline | dropped | 首末 Submit | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| b1 | `scalar-nop=0` | 4,414 | 1,031 | 613 | 850 | 418 | 237 | 192 | 0 | 54.056 us | +| b256 | `real-compute/6,28,4,1` | 967,307 | 105,580 | 103,618 | 103,883 | 1,962 | 265 | 192 | 0 | 5,774.295 us | + +两轮的 producer、raw 行与 converter 七项 summary 均闭合，96 核逐核异常数为 0： + +~~~text +b1: 613 + 418 = 1,031; 613 + 237 = 850 = 1,031 - 418 + 237 +b256: 103,618 + 1,962 = 105,580; 103,618 + 265 = 103,883 = 105,580 - 1,962 + 265 +~~~ + +b1/b256 每核记录峰值分别为 57/10,252，均低于 65,536 条固定容量。PollBatch +实际分布在 `startup_poll/fatal_poll/fanin_flag_load/replay_done_poll` 四类 site： +b1 的物理 episode/逻辑调用依次为 `96/143、42/47、2/6、97/222`，b256 为 +`96/143、42/47、16/467、111/1305`；HeapGuard 两类 allowlist 本轮为 0，不是漏插桩。 + +同核区间复核中，b1 的 237 个 PollBatch 与 4 个 Kernel、b256 的 265 个 PollBatch +与 1,024 个 Kernel 均为严格 overlap 0；分别有 3 和 31 处仅在 `end==begin` 端点 +相接，符合“边界先以同一 cycle 关闭 PollBatch，再进入/退出 Kernel”的设计。b256 +真实 Cube/Vector 计算、调度终态和全部语义断言均 PASS。产物为： + +~~~text +outputs/pa_scheduler_swimlane_20260718_182649_4060527/ccec/l2_swimlane_records.json +outputs/pa_scheduler_swimlane_20260718_182649_4060527/ccec/merged_swimlane.json +outputs/pa_scheduler_swimlane_20260718_182725_4061524/ccec/l2_swimlane_records.json +outputs/pa_scheduler_swimlane_20260718_182725_4061524/ccec/merged_swimlane.json +~~~ + +b1 是零 winner 负载的快速验收，b256 是开启 atomic 泳道的诊断运行；5.774295 ms +只证明观察构建保持目标量级，不是关闭 trace 的正式性能基线，也不能与下列历史样本 +做单轮减法归因观察开销。 + +schema-v4 于 2026-07-19 用同一 atomic ABI 完成 CCEC b1、AscendC b1 与 +CCEC b256 真机验收。v4 新增 `OrchestrationReplay/FinalDrain` 父 span，并用 +`WinnerBuild/LoserReplay/AllocComplete` 替代旧 lap；Atomic 仍由原 direct 与 +PollBatch 规则产生。三轮均为 96 核、192 条 ClockBaseline、 +`dropped=0`，Atomic weighted summary 和 raw 行逐项闭合： + +| 样本 | 总 `records` | 逻辑 `atomic_calls` | 物理 Atomic | `batched_poll_calls` | PollBatch | 首末 Submit | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| CCEC b1 | 4,602 | 1,403 | 846 | 790 | 233 | 90.741 us | +| AscendC b1 | 4,597 | 1,338 | 841 | 725 | 228 | 86.037 us | +| CCEC b256 | 964,724 | 102,324 | 101,108 | 1,471 | 255 | 5,680.749 us | + +CCEC b256 还验证了每核 1,280 个 Submit、1,024 个 Kernel 全部落入 +EfDrain 或 FinalDrain：前者 1,011 个、后者 13 个，孤儿和越界均为 0； +Submit/EfDrain/Orchestration/FinalDrain/WorkerCompletion 排他闭合。最终分析器 +还逐条验证 exclusive child 的 task 身份，以及 Alloc loser=`Claim.end`、 +非 Alloc loser=`Register.end` 的零时长锚点。完整产物为： + +~~~text +outputs/pa_scheduler_swimlane_20260719_055449_334552/ccec/ +outputs/pa_scheduler_swimlane_20260719_060409_345364/ascendc/ +outputs/pa_scheduler_swimlane_20260719_060634_347455/ccec/ +~~~ + +最终源码和相同 v4 level-4 配置连续三轮 CCEC b256 为 +5.785939/5.503109/5.381844 ms，中位数 5.503109 ms，极差 +0.404095 ms；完整导出轮比上文 v3 导出轮低 1.62%，仍小于当前自身 +轮间波动。因此当前只记录为“未观察到明显回退”，不把这个单轮差值 +当成优化收益。 + +以下是 2026-07-18 的**历史 schema-v2、逐调用、边界修复前**证据，保留用于追溯， +不作为当前 schema-v4 的物理记录规模、逻辑调用数或边界闭合验收。历史 b1 +atomic-trace-only 上板中，全部协议断言 PASS，raw 共 4959 条、 +`expected=4959`、`dropped=0`，其中逐条 Atomic 1395 条、ClockBaseline 192 条。 +Claim 当时已按 schema-v2 闭合为 `won=5/lost=283/not_attempted=192`。当时 converter +回归最终为 5/5 PASS，Atomic direct bracket 为 +`return_ready=1193/source_issue=202`。历史未入库产物为： + +~~~text +outputs/scalar_observation_final_20260718/atomic_inlineasm_ccec_b1/raw.json +outputs/scalar_observation_final_20260718/atomic_inlineasm_ccec_b1/merged_swimlane.json +~~~ + +该历史 b1 只用于说明旧边界、旧 schema 和泳道布局，不用它的插桩后 Submit span +替代 256 batch 约 5 ms 的无诊断基线。同一历史二进制的 256 batch 随后也完成一轮： +全部断言 PASS，raw/merged 均为 963368 个 span，`expected=963368`、 +`dropped=0`，Atomic 99944 条、ClockBaseline 192 条；Claim 为 +`won=1280/lost=72448/not_attempted=49152`，恰好闭环 `96*1280`。合并泳道 +不存在旧 `AIC/AIV·atomic` 线程，atomic 全部位于对应 scalar lane。本轮 +边界计数为 `return_ready=97192/source_issue=2752`，与动态 site 计数之和一致； +插桩 Submit span 为 5.209261 ms，只证明观察构建未把数量级打坏；正式 +性能仍以关闭诊断插桩的独立进程 A/B 为准。本轮未入库产物为： + +~~~text +outputs/scalar_observation_final_20260718/atomic_inlineasm_ccec_b256/raw.json +outputs/scalar_observation_final_20260718/atomic_inlineasm_ccec_b256/merged_swimlane.json +~~~ + +同一历史源码随后关闭诊断插桩做了三个独立进程，Submit 为 +3.729925/4.904346/5.563417 ms，中位数 4.904346 ms，三轮语义全部 PASS。 +这组数据说明该历史 standalone 版本复现了约 5 ms 量级，也显示了多核轮询、 +frontier helping 和 winner 分布会带来明显轮间波动；一次 5.209261 ms 插桩轮 +不能与三轮中任一轮做单样本减法后归因 trace 成本。 + +正式重采状态如下： + +| 结果 | 当前状态 | +| ---- | -------- | +| schema-v3 六类 PollBatch、flags、logical/physical 公式与拓扑的 converter 静态回归 | **19/19 PASS；不替代真机验收** | +| schema-v3 b1 的七项 summary、逐核/全局公式与 dropped | **96/96 闭合，dropped=0** | +| schema-v3 256 batch 的 direct/PollBatch raw→merged 与容量 | **已闭合，单核峰值 10,252/65,536** | +| schema-v4 排他父区间、真实 Submit 尾动作与 atomic ABI | **CCEC/AscendC b1 均通过；旧 lap=0** | +| schema-v4 CCEC b256 raw→merged→exclusive report | **964,724 条，dropped=0，六组整数 cycle 闭合** | +| 15-site schema 与六类 PollBatch allowlist | **历史 schema-v3 b1/b256 两轮 flags 全合法；实际四类有事件、HeapGuard 两类为 0** | +| 192 条 ClockBaseline 与 Kernel 边界 | **历史 schema-v3 b1/b256 两轮均闭合；严格 overlap=0** | + +此前开发过程中的探索性输出不在这里引用为正式结论。最终按同一源码依次执行： + +1. 三后端全量重建及无诊断插桩语义回归； +2. 历史 schema-v3 atomic-trace-only 已按 b1、b256 完成；当前 schema-v4 + 又按 CCEC b1、AscendC b1、CCEC b256 完成六类 allowlist、flags、七项 + summary、ClockBaseline、Kernel 包含、raw→merged 与排他报告闭合； + 后续改动仍按同一门禁复测； +3. 比较任何 atomic 优化前后时，双方使用相同观察模式、相同 winner mode/count、 + 相同 trace 配置和 + 独立进程交错 A/B；正式端到端 Submit 仍以无诊断构建复测。 + +#### 7.5.7 绝不能从 O2 声称的结论 + +即使上述重采全部通过，也不能声称： + +- 单条 source-issue 或 return-ready bracket 就是硬件 atomic retire 延迟、跨核 + 可见延迟或一致性完成时间； +- 所有 atomic duration 相加就是 Submit 中“atomic 占用时间”，或删除这些 atomic + 一定能等量缩短墙钟时间；不同核 direct bracket 会重叠，PollBatch 之间也可重叠， + poll 数还会随插桩改变； +- PollBatch duration 是单次 load 延迟、`call_count` 次延迟之和或独占 scalar 时间； + 它只表示逻辑等待 episode 包络； +- `ClockBaseline` 可以逐事件相减并得到无偏的 atomic 硬件净耗时； +- trace 开启后的 Submit span 可以直接与无 trace 基线比较并宣布性能收益； +- standalone 的 `scalar-nop` 已经真实模拟了 vector/cube task 期间 scalar 的等待状态； + NOP 本身在 scalar 上执行。CCEC `real-compute` 虽已真实激活 Cube/Vector/MTE/FIX， + 也仍不能由此推导完整真实 PA 的 scalar 等待与资源竞争； +- standalone 的 atomic 分布可以直接替代真实 FDWIC PA。迁移真实用例时仍需 + 复用已验证的最小观察代码，并重新完成计数闭环、正确性和无诊断性能 A/B。 + +#### 7.5.8 历史 schema-v2：256 batch loser ClaimMax 的定量归因 + +本节只保留 2026-07-18 边界修复前 `atomic_inlineasm_ccec_b256` 的历史定量结果。 +ClaimMax 在 schema-v3 中仍是 direct FetchMax，不会被 PollBatch 聚合；但下列数值来自 +旧二进制，只能作为后续重测的优先级假设，不能充当当前版本验收数据。该历史样本中， +Claim 三态为 1280 个 winner、 +72448 个 attempted loser 和 49152 个 role-filtered not-attempted。每个 attempted Claim +恰好包含一条 `claim_max.fetch_max`，与 73728 条 ClaimMax 精确闭环。 + +| 口径 | 次数 | median | p95 | max | 多核累计 core-work | +| ---- | ---: | ---: | ---: | ---: | ---: | +| loser ClaimMax | 72448 | 280 ns | 637 ns | 4791 ns | 24.133954 ms | +| winner ClaimMax | 1280 | 268 ns | 508 ns | 1277 ns | 0.376479 ms | +| AIC loser ClaimMax | 23808 | 269 ns | 523 ns | 3434 ns | 7.058334 ms | +| AIV loser ClaimMax | 48640 | 290 ns | 677 ns | 4791 ns | 17.075620 ms | + +loser ClaimMax 占全部 99944 条 atomic bracket core-work 的 58.6%；去掉 Submit +窗口外的 startup/replay-done 生命周期 atomic 后，它占 Submit 内 atomic bracket +core-work 的 77.7%。因此它是“atomic 内部”的第一大项，但优势主要来自 +72448 次动态调用，不是 loser 单次比 winner 慢一个数量级。 + +不能由此把整个 Replay 或整个 Submit 都归因给 ClaimMax。`Replay` 是 lap +外层区间：从 materialize 前开始，包含 Materialize、PrepareMap、Claim、Register， +到 loser 分支结束，这些嵌套 span 不能相加。72448 个 attempted-loser Replay 的 +外层 core-work 为 192.977600 ms，其中 loser ClaimMax 直接 bracket 占 12.5%； +49152 个完全没有 FetchMax 的 not-attempted Replay 仍有 93.430073 ms core-work。 + +按每核累计看，loser ClaimMax 中位数为 0.258278 ms，占每核 Claim 中位 +累计的 45.7%、Replay 外层的 8.45%、Submit envelope 的 5.0%。本轮最晚结束的 +AIV core93 上，对应数字为 0.261731/5.191702 ms，也约 5.0%。这只是直接 +可见的本核返回等待占比；真正改动 Claim 协议后还会改变 winner 到达、 +frontier/fanin 时序和竞争形态，不能简单从 5.209261 ms 中减去 0.258 ms。 + +历史阶段性结论是：ClaimMax loser 是后续 atomic 消减的第一优先级，但尚未证明 +它是整个调度的主瓶颈。开始改 Claim 后，仍需重新闭合 Claim 三态、atomic +logical/physical 次数和调度终态；端到端收益由无诊断的独立进程 A/B 确认。 + +#### 7.5.9 CCEC winner 负载从 scalar NOP 迁移到真实 Cube/Vector + +为让 winner 执行期更接近真实 Cube/Vector task，2026-07-18 先在 standalone +CCEC 增加显式 `--winner-workload real-compute`，没有迁移真实 PA。 +在提交 `e66001ff` 对应的这个历史阶段，无参数默认当时仍为 `scalar-nop`，用于 +保证旧三后端基线不静默变化；当时选择真计算而未指定次数时，QK/SF/PV/UP +使用 `6,28,4,1`。三后端闭环后当前默认已经切换为 `real-compute`，但本节后续 +数据仍按当时的显式模式和参数解读，不能追改成新默认口径。 + +实现与门禁如下： + +1. QK/PV 只在 AIC 执行完整 `128x128 float` Cube matmul，包含 + MTE2/MTE1、M、FIX、GM store 和最终完成等待； +2. SF/UP 只在 AIV 执行完整 Vector add/mul，包含 MTE2、V、MTE3、GM store + 和最终完成等待；每次 repeat 完成写回后才复用 tile； +3. 两个 GM 输入 tile 为所有 worker 只读共享，每 worker、每角色使用独占输出 + tile。host 在计时外初始化/传输，计时后逐 tile 验证 768/5/6 与 inactive + sentinel，共 12,713,984 bytes； +4. 最终 device ELF 严格限制为两个 mixed kernel GLOBAL 入口；冷路径 dispatcher + 与 Cube/add/mul 三个执行 helper 必须是非空 LOCAL 函数。这个门禁来自一次已 + 复现故障：错误暴露的 GLOBAL helper + 被 runtime 当入口启动，导致 scalar 模式也进入 Cube 路径；修正后同一 ELF 的 + scalar b1 与 real b1 均通过； +5. repeats 上限为 128，控制诊断运行规模。真计算 count 与 NOP count 互斥， + 0 次被拒绝。 + +标定不是以凑齐 5.1 ms 为目标。三个独立 b256 进程的 QK/SF/PV 中位数约为 +41.336/54.039/27.971 us，最接近真实泳道目标 44.170/53.729/27.626 us； +UP 一次完整流水约 2.5 us，已经是当前正整数下限。三轮 Submit 为 +3.808/3.555/3.706 ms，中位数 3.706 ms；最终重建后的单轮为 3.683649 ms。 +Cube/Vector 分布在物理子核并行执行，与 scalar NOP 串行占用 scalar 的到达时序 +本来不同，不能通过增加无关 repeat 把总时间硬拉回 5.1 ms。 + +最终常量 tile 只证明某个 active worker-kind 至少完成一次，因为同一 tile 会被 +后续 repeat 覆盖；不能把最终输出本身夸大成全部 repeat 的次数证明。 + +泳道 raw/merged 的 `trace_schema_version` 仍为 2；metadata 同时保存 mode、 +四类 count、迭代单位与 +QK/PV=Cube、SF/UP=Vector 映射，并增加可见的全局 capture metadata 事件。 +最终 b1 real-compute 泳道有 4964 条 raw data event；converter 产出 4965 条 +data event（增加一条 capture instant），再加 256 条 process/thread metadata， +最终 `traceEvents` 为 5221 条，`dropped=0`。转换器含旧 schema 兼容在内为 5/5 PASS。 + +当时的阶段边界：提交 `e66001ff` 只证明 CCEC standalone 的真实引擎负载、 +数值、角色与泳道闭环；当时 AscendC 真实计算和 CPU 对等算术尚未完成， +因此本阶段没有宣称三后端 winner 负载已经对等，也没有迁移真实 PA。 +后续完成情况分别记录在 7.5.10 和 7.5.11。 + +#### 7.5.10 AscendC winner 负载迁移到真实 Cube/Vector + +提交 `9aeda0dd` 在不改动 PA 调度模型的前提下，把 CCEC 已验证的 +winner workload 布局、参数解析和 host 数值校验抽到三后端共用头文件，然后 +在 AscendC mixed kernel 内分角色接入真实计算： + +1. AIC 上的 QK/PV 执行 `128x128 float` matmul，路径为 GM ND 到 + A1/B1 的 NZ 布局、A1/B1 到 A2/B2、`Mmad`、FIX 回写 GM； + `MTE2_MTE1`、`MTE1_M`、`M_FIX` 和最后的 `FIX_S` 保证 Kernel span + 包住本轮回写完成边界； +2. AIV 上的 SF/UP 执行 GM 到 UB、`Add`/`Mul`、UB 到 GM，并用 + `MTE2_V`、`V_MTE3`、`MTE3_S` 等待完整的 load/Vector/store 流水； +3. 12,713,984-byte workspace、输入 2/3、输出 768/5/6、每 worker-kind + 独占 tile 和 inactive sentinel 与 CCEC 共用同一个口径。H2D 初始化在 + launch 计时前，D2H 与数值校验在计时后，不把 host 搬运写入 Submit span。 + +首个 AIC 版本的数值闭环暴露了真实错误：A1 和 B1 虽然是不同逻辑位置， +但映射到同一块物理 L1；两个 64 KiB tile 都从地址 0 开始时，后搬入的 B +覆盖 A，因而把本应为 `2 * 3 * 128 = 768` 的结果错算成 +`3 * 3 * 128 = 1152`。将 B1 起始地址错开 64 KiB 后，整个输出 tile 恢复为 +768。右矩阵 B 在 GM 中是普通 KxN row-major，所以 L1 zN 到 L0B nZ 时设置 +分形 transpose，而不是盲目复用 A 的非转置参数。当前常量 B=3 不能单独 +证明转置差异，因此本阶段先以 CANN 布局语义修正；随后再用 7.5.12 的非均匀 +输入逐元素上板闭合，没有把常量输出冒充布局证据。 + +修正后的分层证据为： + +| 场景 | 参数 | Submit span | 数值/调度结果 | +| ---- | ---- | ----------: | ---------------- | +| b1 | count=`1,1,1,1` | 59.280 us | PASS，4 active tile + 188 sentinel tile | +| b8 | count=`1,1,1,1` | 166.426 us | PASS，32 active tile + 160 sentinel tile | +| b256 样本 1 | count=`6,28,4,1` | 3810.471 us | PASS，191 active tile + 1 sentinel tile | +| b256 样本 2 | count=`6,28,4,1` | 4828.567 us | PASS，192 active tile | +| b256 样本 3 | count=`6,28,4,1` | 3777.371 us | PASS，192 active tile | + +三个独立 b256 进程的 Submit span 中位数为 3810.471 us。相同三个样本中， +QK/SF/PV/UP 的每 task 平均 Kernel span 分别约为 41.1–41.4 / 47.3–49.7 / +27.5–28.2 / 2.5–2.7 us。这些 span 含上述引擎完成等待，不从三轮总时间反推 +未采集的流水事件。 + +b1 count=1 还完成了带 atomic 的泳道闭环：4652 条 raw data event 与静态 +期望精确一致，`dropped=0`，atomic 源码边界调用为 1088。converter 增加一条 +capture instant 后有 4653 条 data event，再加 256 条 process/thread metadata， +最终 `traceEvents=4909`。raw 和 merged 都记录 `real-compute`、`1,1,1,1` +及 QK/PV=Cube、SF/UP=Vector 映射，数值校验也同轮 PASS。 + +本阶段证明 AscendC standalone 已脱离 scalar NOP 执行真实 A5 Cube/Vector +流水，且调度语义、输出和泳道闭环一致；它不是真实 PA 迁移。 + +#### 7.5.11 CPU winner 负载补齐对等算术 + +提交 `1d3a374a` 让 CPU 后端使用同一组 workspace、角色路由、repeat +参数和输出校验：QK/PV 以三重浮点循环执行完整 `128x128` 矩阵乘， +SF/UP 执行逐元素 add/mul；每个 repeat 后保留 compiler memory boundary，避免 +因为下一轮覆盖同一 tile 而被 O3 删除。每轮重新初始化 2/3 输入和输出 +sentinel，所以 `runs>1` 不会沿用上一轮 winner 结果。 + +分层回归中，scalar-NOP b1、real-compute b1 count=`1,1,1,1` 均 PASS； +后者 Submit span 为 39.821 ms，并校验 4 active tile + 188 sentinel tile。b8 +count=`2,3,2,1` 连续两轮也均 PASS，两轮分别为 80.310/54.185 ms， +均校验 25 active tile + 167 sentinel tile。同时保留 CLI 互斥和边界门禁。在该 +历史阶段，real-compute 与 NOP override 不能混用，count 需要与 real-compute +模式一起使用，取值限制为 1..128。 +当前无参数默认已是 real-compute，因此当前 CLI 允许 count/pattern 直接覆盖默认 +真计算；显式 NOP override 会选择 scalar-nop，显式 real-compute 与 NOP override +仍然互斥。 + +CPU 阶段的“对等”只包括 PA 任务调度、AIC/AIV 角色选择、workspace 编址、 +repeat 次数和 768/5/6 数学结果。CPU pthread 调度、普通浮点循环和 x86 atomic +都不是 A5 Cube/Vector/scalar 流水；上述毫秒数据只用于证明用例实际执行完成， +不用作 A5 timing 结论。 + +至此，三后端的 winner workload 参数、workspace、角色路由、数学输出和 +泳道 metadata 口径已对齐；CCEC/AscendC 执行真实 A5 引擎负载，CPU 仅做 +调度与算术回归。这仍然是 standalone 验证，不能替代真实 PA 的计数闭环、 +正确性和无诊断性能 A/B。 + +#### 7.5.12 用非均匀输入闭合 Cube/Vector 数据布局 + +常量 A=2、B=3 能验证真实指令、角色和 GM 写回，却会同时掩盖 B 转置、 +ND/NZ stride 与分形重排错误。为把该盲区变成可重复工具，三后端公共 CLI 增加 +`--real-compute-pattern constant|layout-diagnostic`：默认仍为 `constant`，不改变 +性能基线；诊断模式只改变计时窗外的 host 输入生成和输出期望，不进入 +`SchedulerState`，也不在 CCEC/AscendC device 热路径增加分支。 + +诊断输入定义为： + +```text +A[r,c] = (r == c) ? (r + 1) : 0 +B[r,c] = 1 + ((131*r + 17*c + 7*r*c) mod 251) +``` + +QK/PV 的期望为 `(r+1)*B[r,c]`，SF 为 `A[r,c]+B[r,c]`，UP 为 +`A[r,c]*B[r,c]`。B 是非对称稠密矩阵，A 是带权对角矩阵，因此 B 转置、A 行映射、 +stride 或输出重排都会在确定元素上产生不同整数；最大结果不超过 32128，FP32 +可做逐元素精确比较而无需容差。 + +严格按 CCEC → AscendC → CPU 顺序，以 b1、count=`1,1,1,1` 上板/运行： + +| 后端 | Submit span | QK/SF/PV/UP Kernel span | 输出门禁 | +| ---- | ----------: | ----------------------- | -------- | +| CCEC | 37.682 us | 9.172 / 3.819 / 7.415 / 2.512 us | 4 active + 188 sentinel，PASS | +| AscendC | 55.041 us | 8.478 / 4.211 / 20.801 / 3.008 us | 4 active + 188 sentinel，PASS | +| CPU | 51.958 ms | 3.457 ms / 26.118 us / 3.390 ms / 17.349 us | 4 active + 188 sentinel，PASS | + +CCEC 先证明公共期望与 PTO `A*B` 语义正确；随后 AscendC 使用同一输入通过，直接 +闭合了 A1/B1 错位、普通 KxN B 的 zN→nZ 分形 transpose、ND/NZ stride 和 +FIXPIPE NZ→ND 输出。CPU 只证明 host 公式与调度路由，不把毫秒值外推到 A5。 +三后端又以 `pattern=constant` 做 b1 count1 回归，并以 `smoke all` 回归原 +scalar-NOP，均 PASS，说明诊断模式没有静默改变当时的性能默认或旧控制路径。 + +AscendC 同模式泳道位于 +`outputs/pa_scheduler_swimlane_20260718_125904_3613100/ascendc/`，raw 为 +4584 条 data event、`dropped=0`，merged 增加一条 capture instant；raw/merged +的 `metadata.winner_workload.input_pattern` 都是 `layout-diagnostic`。 +converter 对新字段保留旧 schema-v2 兼容，并新增非法 +pattern 拒绝回归，当前两种 unittest 入口均为 6/6 PASS。 + +该诊断证明一次完整 engine pipeline 的数学和布局，不单独证明同一 task 的 N 次 +repeat 都执行；不把最终覆盖同一 tile 的结果夸大成次数证明。 + +#### 7.5.13 默认切换为真负载及同口径性能验收 + +完成 CCEC→AscendC→CPU 分阶段闭环后，共享 `WinnerWorkloadOptions` +的无参数默认从 `scalar-nop` 切换为 +`real-compute/constant/6,28,4,1`。`smoke` 仍显式固定 b1/r1/scalar-nop=0； +旧命令未指定 mode 但显式给出 `--nop-count*` 时自动选择 scalar-nop。 +显式 real-compute 与 NOP override、显式 scalar-nop 与 real count/pattern 仍互斥。 + +性能验收先纠正了观察口径：早先 3.7～4.4 ms 是 `--no-swimlane`， +真实 PA 5.1 ms 是标准 L2 泳道；泳道记录不仅增加指令，还会改变 worker +到达、fanin 失败重试和 RingBp，所以不能把两者直接相减成“缺失的调度时间”。 +当前保留的 CCEC b256 真负载历史 phase-only 泳道、不开逐 atomic 的 5 个独立进程为： + +```text +5002.413 / 4875.193 / 4968.894 / 4992.477 / 4876.282 us +``` + +中位数 4,968.894 us。真实 PA 最终三轮中位数 5,115.620 us，同口径 +差 146.726 us，约 2.87%。五轮 standalone 的 QK/SF/PV/UP 每 task 均值 +中位数为 41.461/54.007/28.053/2.649 us，总 core work 已贴近真实 PA， +不通过增加 repeat 继续硬凑总时间。 + +保留的一轮历史 phase-only raw 为 +`outputs/performance_gap_20260718/standalone_ccec_real_b256_raw.json`；863,237 条 +记录全部有效，与真实 PA 863,232 条的基本阶段数完全相同，只额外有 5 条 +RingBp。这说明 standalone 已达到“独立复现约 5 ms 调度”的目标；仍然保留 +本文第 6 节的边界：它不依赖 simpler 生产代码，也不复刻真实 PA 数值数据流和 +通用多 group/joint 拓扑。当前正式 `swimlane` action 已固定合并普通阶段和 +逐 atomic 记录，因此该 863,237 条历史文件只用于同口径性能参照，不能作为当前 +合并泳道的记录数量或容量证据。 + +#### 7.5.14 排他泳道边界收敛对 atomic 口径的影响 + +2026-07-19 的后续改动只收敛普通 phase 边界和离线 residual， +没有增删 Atomic site，也没有改变 direct/PollBatch、`return_ready`/ +`source_issue` 或 logical/physical weighted 计数公式。 + +standalone loser 没有真实 Replay 计算，因此已删除每个 loser 一条的 +`LoserReplay` 过程态 phase 记录。这不是 atomic 消减：相关 Claim +FetchMax、fanin/completion/frontier 等 atomic 仍按实际调用采集；只是不再 +用一条零工作 phase 伪装 loser 尾动作。最后一个真实 child 到 +`SubmitEnd` 的时间在 Perfetto 中由离线 `submit_tail_gap` 展示，在排他 +报告中汇总为 `submit_tail_residual`；它不被命名为 loser 业务阶段。 + +schema-v4 merged 中 direct atomic 名仍编码 +`boundary/site/op/task_id`，PollBatch 名仍编码 `site/op/call_count`； +为控制数百 MiB 产物，duration 事件不再逐条复制 raw 的 `args/cat`。 +精确 flags、cycle、retry、value-zero 和 weighted 计数仍以同目录 +`l2_swimlane_records.json` 为准，不因 merged 瘦身丢失。 + +当前最新 CCEC A5 b1 为： + +~~~text +outputs/pa_scheduler_swimlane_20260719_110756_584549/ccec/ +~~~ + +该轮 4,118 条 raw 事件、`dropped=0`，全部 atomic 闭合与六类排他 +时间闭合均通过；当前 converter 生成的 merged 为 428,455 bytes。 +后续 atomic/边界迭代默认只跑 A5 b1；b256 仅用于阶段性规模/容量 +收口或明确要求的长负载结论。 + +按明确要求完成的当前生产者 CCEC b256 规模复核位于： + +~~~text +outputs/pa_scheduler_swimlane_20260719_114815_617346/ccec/ +~~~ + +该轮 839,526 条 raw、97,510 条物理 Atomic、`dropped=0`，全局 Submit +为 5,360.061 us；全部 atomic 与排他闭合通过。Submit 内部阶段间 residual +为 0，尾部 residual 为 41,008,786/433,383,588 cycle（9.4625%），Submit +间 residual 为 67,065,321/500,448,909 cycle（13.4010%）。这些数值是 +相同观测口径下的归因基线，不把边界重分类宣称为 atomic 或调度性能收益。 + +#### 7.5.15 真实 PA atomic 冷路径外提验证 + +真实 PA 将两段只服务于 level-4 的公共动作外提为设备冷函数：direct atomic 的 +record 发布，以及 PollBatch 命中后的十类遍历与落盘。Atomic 语义边界保持不变： + +- direct atomic 的 begin、真实指令、返回值地址依赖和 end 仍在原 wrapper；record + 发布只发生在 end 之后，不把 source-issue Exchange/FetchAdd 改成等待返回型； +- PollBatch 仍先在调用点取得 `end_cycle`，只把之后的遍历与发布外提；level-4 + logical call_count、episode 包络和 direct row 的区分不变； +- level gate、site/op、`result_used`、`return_ready`、ClockBaseline 和 raw ABI 均未改动。 + +真实 A5 level-4 门禁位于： + +~~~text +outputs/TestPagedAttentionUnroll_Case1_20260719_135629/ +~~~ + +该轮 107,608 条 Atomic 精确表示 115,309 次调用，其中 8,056 次轮询压缩为 +355 条 PollBatch，满足： + +~~~text +107608 = 115309 - 8056 + 355 +~~~ + +192 条 ClockBaseline、全部 site/op/`result_used`/`return_ready` 和 raw/metadata +计数通过 converter 校验，`dropped_records=0`。level-1 三轮首末 Submit 中位数为 +4.821897 ms，只用于证明保留 atomic 诊断能力后没有继续承受此前的观察回退， +不能解释为 atomic 本身的执行时间。代码体积、取指污染、外提原则和 I-cache +专用编译期剔除方案统一记录在 +[`../icache_miss_usage_guide.md`](../icache_miss_usage_guide.md)。 + +#### 7.5.16 cursor 分片 G=1/4/8/16 对照与结论边界 + +2026-07-21 在 `fdwic-swimlane-deps` 的 `8ef337c1` 上完成 A5 device0、96 workers、 +Case1/b256 对照。结论是：**standalone 稳定复现 G=1→4 的大收益，G=4→8 仍有 +约 4.6% 的配对改善，G=8→16 已无稳定增益；simpler 的四档差异都在约 0.6% +以内。** 本节只记录测试事实与口径边界，不据此修改当前默认 G=4。 + +##### 7.5.16.1 两条路径的观察口径不同 + +这组数据必须先说明泳道开关，不能把两条路径的绝对值直接相减： + +- standalone：PMU off、运行时 `--no-swimlane`。泳道记录和后处理均关闭，但设备 + 内首末 Submit 时间标记仍存在，`submit_span_us` 由这些标记直接计算； +- simpler：PMU off、`--enable-l2-swimlane 1`。从 schema-v4 level-1 raw 中 + 122,880 条 Submit 事件计算 `max(end)-min(begin)`；只跳过运行后的 converter， + 没有关闭设备泳道采集； +- 当前 simpler 关闭泳道后只能从 device-log 取得约 80 ms 的完整 replay 窗， + 不能精确还原 Submit 首末边界，因此没有拿该值冒充 Submit span； +- 所以本节只比较**各路径内部**随 G 的相对形状。standalone 与 simpler 的绝对 + 数值不是严格相同观察开销，不能解释成实现间的直接快慢关系。 + +实验构建把每类 cursor 的物理容量统一为 16，每个 shard 独占 64 B cache line， +仅用运行时 mask 选择 G=1/4/8/16；同一路径四档执行同一个 AICore ELF。本轮正式 +比较没有使用 packed 布局。本次文档提交不带实验 selector，也不修改生产或 +standalone 的固定默认值。 + +测试机没有 `task-submit`/`npu-smi`，按明确许可直接串行使用 device0；采样前确认 +没有其他 PA 测试进程，但数据仍标记为未经过队列隔离。 + +##### 7.5.16.2 standalone：G=8 是当前饱和点 + +standalone 使用 PMU off、泳道关闭、真实 winner 负载 QK/SF/PV/UP=`6,28,4,1`。 +每个样本为全新进程、`--runs 1`，四档按四种次序平衡轮换 12 组；48 轮均通过 +execution、semantic、postprocess 门禁以及逐 shard cursor、winner、fanin、 +heap/TensorMap 和数学输出校验。 + +| G | 原始中位数 | 相对 G=1 | 原始范围 | +| -: | ---------: | --------: | -------: | +| 1 | 5984.968 us | - | 5877.150～6106.107 us | +| 4 | 3760.323 us | -2224.645 us / -37.171% | 3578.988～3842.902 us | +| 8 | 3593.468 us | -2391.500 us / -39.958% | 3436.906～3967.995 us | +| 16 | 3601.595 us | -2383.373 us / -39.823% | 3399.943～5336.200 us | + +按预先固定的 `median ± 3*1.4826*MAD` 判异常；任一 G 越界就删除完整四元组， +没有只删较慢一侧。删除 block 4/7/10/11/12 后保留 7 组： + +| G | 清洗后中位数 | fanin loads 中位数 | +| -: | -------------: | -----------------: | +| 1 | 5974.589 us | 226623 | +| 4 | 3760.859 us | 29504 | +| 8 | 3625.762 us | 24601 | +| 16 | 3598.745 us | 24449 | + +清洗后的配对比较为： + +| 比较 | 更快次数 | paired 中位变化 | bootstrap 95% CI | sign test | +| ---- | -------: | ----------------: | ----------------: | --------: | +| G=4 - G=1 | 7/7 | -2188.859 us / -36.636% | [-2292.759, -2158.361] us | 0.015625 | +| G=8 - G=1 | 7/7 | -2358.668 us / -39.978% | [-2440.244, -2319.347] us | 0.015625 | +| G=16 - G=1 | 7/7 | -2388.412 us / -39.974% | [-2496.821, -2272.705] us | 0.015625 | +| G=8 - G=4 | 7/7 | -176.631 us / -4.632% | [-231.913, -135.097] us | 0.015625 | +| G=16 - G=4 | 7/7 | -204.062 us / -5.403% | [-233.349, -113.391] us | 0.015625 | +| G=16 - G=8 | 5/7 | -37.935 us / -1.043% | [-57.306, +21.706] us | 0.453125 | + +G=1→4 的墙钟改善伴随 fanin loads 从约 22.7 万降到约 3.0 万；G=4→8 继续下降 +约 4903 次并稳定改善 4.632%。G=8→16 仅再下降约 152 次，置信区间跨零，不能 +证明 G=16 稳定优于 G=8。 + +曾试跑每进程 `--runs 9`，但 G=4/8/16 的进程内后续轮次出现 3.1～6.5 ms +双峰/长尾，与既定独立进程口径不一致。该批结果整批作废，没有混入上述统计； +当前也不臆测其具体复用状态根因。 + +##### 7.5.16.3 simpler：G=8 无可辨认收益，也无可辨认回退 + +simpler 每档 3 个独立进程，12 轮 golden 均通过；每份 raw 都是 schema-v4、 +122,880 条 Submit、839,115～839,126 条总事件、`dropped_records=0`。 + +| G | 三个 Submit span | 中位数 | 相对 G=1 | +| -: | ---------------- | -----: | ---------: | +| 1 | 4697.011 / 4697.458 / 4704.788 us | 4697.458 us | - | +| 4 | 4689.221 / 4691.761 / 4754.005 us | 4691.761 us | -5.697 us / -0.121% | +| 8 | 4716.435 / 4685.210 / 4731.393 us | 4716.435 us | +18.977 us / +0.404% | +| 16 | 4775.288 / 4689.830 / 4644.471 us | 4689.830 us | -7.628 us / -0.162% | + +直接按当前默认 G=4 比较，G=8 的中位数为 `+24.674 us / +0.526%`；只有三个 +样本，且 G=16 自身跨度达到 130.817 us，因此这些方向都不能解释成确定收益或 +回退。差异远低于 5% 门槛，本轮不启动 I-cache 对比分析。 + +##### 7.5.16.4 结论边界 + +A5 standalone 数据支持把 G=8 视为后续候选：G=4→8 配对改善稳定，而 G=16 +相对 G=8 已无稳定增益。但 simpler 没有显示可辨认收益，且两条路径的泳道口径 +不同，因此不能声称 production 已获得 standalone 的 4.632% 收益，更不能用 +两种绝对时间直接支撑默认值修改。 + +按本次收口要求只提交测试记录，A5 production、A5 standalone 与 A2A3 的 +`kCursorShards` 均保持现状。若后续决定修改默认值，应在独立代码提交中重新完成 +静态构建的 ABI、功能和性能门禁。 + +#### 7.5.17 只改 final 的分层全局汇合实验 + +2026-07-21 在 standalone 中比较原始单计数器、三种二级分组和一种三级分组。 +本节最终只改 replay 尾部的 final 汇合；startup 严格恢复原始 `started_count` +flat 屏障。所有候选仍经过唯一全局根节点，分组不是取消全局同步，也不允许某组 +在其他 worker 仍可能生产 slot 时提前退出。 + +##### 7.5.17.1 为什么不改 startup + +前置探索曾同时替换 startup 和 final。清洗样本里 startup span 如下： + +| 形态 | startup span | 相对 flat | +| ---- | -----------: | --------: | +| flat | 53.125 us | - | +| two-4 | 73.000 us | +19.875 us / +37.411% | +| two-8 | 78.200 us | +25.075 us / +47.200% | +| two-16 | 84.856 us | +31.731 us / +59.730% | +| three-6x4x4 | 97.182 us | +44.057 us / +82.930% | + +startup 的工作只有“一次到达 + 等到 96”，期间没有可与等待重叠的 completion +drain。分层反而新增叶代表向 root 转发以及 root 向叶 release 的串行传播，因此 +四种分层形态全部回退。final 不同：worker 等待全局“无人继续生产 slot”时仍会 +执行 `FinalDrain`，降低单一 `replay_done` cache line 的争用有机会缩短真正的尾部。 +所以正式版本只保留 final 实验，startup 代码和计数语义固定为原始 flat；五种 +形态的 `started_count` 终值都必须是 96。 + +##### 7.5.17.2 final 的五种拓扑 + +命令行 selector 改名为 `--final-barrier`，避免继续暗示 startup 也会变化： + +| selector | 到达拓扑 | 向下放行拓扑 | +| -------- | -------- | ------------ | +| `flat` | 96 workers → 原 `replay_done` | 同一计数器达到 96 即全局放行 | +| `two-4` | 4 叶组 × 24 workers → root(4) | root → 4 个叶 release | +| `two-8` | 8 叶组 × 12 workers → root(8) | root → 8 个叶 release | +| `two-16` | 16 叶组 × 6 workers → root(16) | root → 16 个叶 release | +| `three-6x4x4` | 16 叶组 × 6 workers → 4 middle × 4 leaves → root(4) | root → 4 middle release → 16 leaf release | + +叶组使用 `block_id % G`，同一物理 block 的 1 个 AIC 和 2 个 AIV 始终在同组。 +静态 lane0 AIC 代表叶组：二级形态由 `block_id == leaf_id` 的核把叶计数转发到 +root；三级形态再由 `leaf_id == middle_id` 的四个代表继续向上转发。这里不依赖 +FetchAdd 返回值临时选 leader，避免增加另一套“谁是最后一个”的协议。 + +arrival、release、middle 和 root 各自独占 64 B；arrival 与 release 分离，等待 +release 的 add-zero 不和仍在到达的 FetchAdd 争同一 cache line。等待期间仍调用 +`FinalDrain`；每个 worker 只有同时看到本叶收到全局 release 且本核 +`occupied_count == 0` 才退出。host 精确检查 active/unused 节点终值，未使用节点 +必须保持零。 + +每个 worker 的 standalone 诊断 sidecar 新增 startup begin/end、final begin、 +global release 和 final end 五个 SYS_CNT 时间点。它们分别构造 startup span、 +final barrier span、final drain span 和完整 lifecycle span，不进入生产 DistCore ABI。 + +##### 7.5.17.3 功能和观察链路门禁 + +CPU 五种形态的 b1 功能测试全部通过。CCEC 在 device0 上逐种执行 b1，96-worker +拓扑、winner、cursor、fanin/frontier、heap/TensorMap、输出数学结果、startup=96、 +final 树计数和时间戳门禁均 PASS。AscendC 全量构建通过，并在 device0 上完成 +G=8 b1 的同一组语义与数学输出门禁。四个 standalone 泳道/PMU 分析脚本的 100 项 +`unittest` 全部通过；顶层 pytest 入口因当前环境的 `simpler_setup.parallel_scheduler` +缺失而未进入测试收集，没有将该环境问题冒充为 pytest 通过。最深的 +`three-6x4x4` 又开启逐 atomic 泳道： + +~~~text +records=4207 expected=4207 dropped=0 +logical_calls=4537 physical_records=926 +batched_poll_calls=3883 poll_batch_records=272 +926 = 4537 - 3883 + 272 +~~~ + +新增树访问全部复用正式 `ReplayDoneIncrement`/`ReplayDonePoll` site，没有藏成不可见 +普通 load/store。raw metadata 和 submit-PMU configuration 均记录 `final_barrier`。 + +##### 7.5.17.4 A5 独立进程五形态对照 + +正式口径为 96 workers、Case1/b256、real-compute `6,28,4,1`、PMU off、运行时 +`--no-swimlane`。每个样本使用全新进程且 `--runs 1`；五种形态共用同一 CCEC ELF。 +10 个 block 使用正向/反向 Latin 顺序，保证每个形态在每个执行位置恰好出现两次。 +50 次运行全部通过 execution/semantic/postprocess 门禁。 + +机器没有 `task-submit` 和 `npu-smi` 命令；设备节点和 `ascend_950_*` 驱动文件 +存在。本轮按明确许可直接串行使用 device0，不能描述成队列隔离或 `npu-smi` +预检通过。 + +50 个原始样本按形态取中位数： + +| 形态 | startup | final barrier | final drain | Submit span | lifecycle | +| ---- | ------: | ------------: | ----------: | ----------: | --------: | +| flat | 44.803 us | 389.249 us | 398.089 us | 3913.195 us | 3992.759 us | +| two-4 | 46.982 us | 330.368 us | 331.964 us | 3727.890 us | 3802.227 us | +| two-8 | 43.759 us | 325.938 us | 329.303 us | 3715.343 us | 3791.836 us | +| two-16 | 45.200 us | 314.792 us | 317.776 us | 3703.511 us | 3767.131 us | +| three-6x4x4 | 47.945 us | 326.264 us | 329.603 us | 3725.657 us | 3798.103 us | + +直接受代码改动覆盖的 `final drain` 中，G=16 原始中位数最低：相对 flat 减少 +80.313 us（-20.175%），相对 G=8/三级分别低 11.527/11.827 us。但不能把原始 +lifecycle 的 `-225.628 us / -5.651%` 直接解释成 final 收益，因为 final selector +直到最后一次 Submit 后才读取，理论上不可能改变之前的 Submit；表中 Submit 中位数 +仍相差约 210 us，已经构成明确的外部漂移负对照。 + +为剔除“明显异常”而不按最终结论挑点，对 50 个因果上应相同的 Submit span 统一使用 +`[Q1 - 3*IQR, Q3 + 3*IQR] = [3516.321, 3954.964] us` 极端异常界限。越界的 7 个 +样本为 block1/two-8、block3/flat、block3/two-16、block4/flat、block5/flat、 +block9/two-8、block10/two-4。该规则只作为敏感性检查，不伪装成完整配对统计： + +| 形态 | 保留样本 | final drain 中位数 | lifecycle 中位数 | +| ---- | -------: | -----------------: | ----------------: | +| flat | 7 | 445.491 us | 3900.457 us | +| two-4 | 9 | 334.576 us | 3801.472 us | +| two-8 | 8 | 329.303 us | 3779.048 us | +| two-16 | 9 | 321.878 us | 3766.568 us | +| three-6x4x4 | 10 | 329.603 us | 3798.103 us | + +异常敏感性检查中 G=16 的中位数最低；清洗后的完整 lifecycle 相对 flat 为 +`-133.889 us / -3.433%`，低于 5% 门槛。 + +##### 7.5.17.5 G=16 与三级树、G=8 与 G=16 复核 + +五形态数据里 G=16 方向最好，但与三级树只有约 12 us。为避免用单轮噪声选型， +又做了两轮均衡 AB/BA 复核： + +| 复核 | G=16 final drain | 三级 final drain | paired 中位差 | G=16 更快次数 | +| ---- | ----------------: | --------------: | ------------: | -------------: | +| 8 对独立进程、每进程 1 run | 323.348 us | 326.063 us | -2.715 us | 4/8 | +| 6 对独立进程、每进程 5 runs 后取中位数 | 387.857 us | 389.672 us | -1.979 us | 3/6 | + +复核只能说明 G=16 与三级树基本打平，不能证明三级层级有收益。为进一步 +判断 G=8 与 G=16，又做了 8 对均衡 AB/BA：每个独立进程运行 5 轮，先取 +进程内中位数，再做进程间对照。16 个进程均通过全部门禁。 + +| 口径 | G=8 final drain | G=16 final drain | G=16 - G=8 | +| ---- | ----------------: | -----------------: | ----------: | +| 8 个进程中位数的总体中位数 | 395.171 us | 389.113 us | -6.058 us / -1.533% | +| 8 组成对差值的中位数 | - | - | -3.885 us / -0.988% | + +G=16 只在 8 组中赢 5 组；单组差值从 `-67.374 us / -15.095%` 到 +`+64.697 us / +16.367%`，方向并不稳定。虽然两组各自取总体中位数时 G=16 +方向快 1.533%，但成对中位改善只有 0.988%，未达到“稳定快至少 1%”的收口 +选择门槛。 + +因此本轮小样本 standalone 候选当时选择 **二级 G=8**。G=16 没有证明稳定优势，却需要 +双倍的叶节点数、更多状态 cache line，并使 root 等待 16 个代表而不是 8 个。 +三级树同样没有证明比二级树更好。这个选择不否定 G=16 在若干中位数口径上 +方向略好，只是不把未达稳定性门槛的差异当成选型依据。后续更高采样量的 +独立屏障实验补足了证据，并在 §7.5.17.7 将默认选型更新为 G=16。 + +##### 7.5.17.6 `.text` 和结论边界 + +修改前同一 swimlane 构建的 device `.text` 为 547,896 B；只改 final、容纳五种 +runtime selector 的实验 ELF 为 590,648 B，增加 42,752 B（+7.803%）。`.rodata` +从 576 B 增至 620 B（+7.639%）。五种形态共用同一个 ELF，因此相互比较没有 +“五份 `.text` 大小不同”的直接混杂；但新增分支仍改变热函数布局，所以不能拿 +实验 ELF 的绝对时间与旧 547,896 B ELF 直接归因比较。 + +异常处理后的完整 lifecycle 差异低于 5%，按既定门槛不启动 I-cache PMU 对比。 +迁移到 simpler 时只实现固定二级树，不把五路 runtime selector 和 standalone +时间诊断带进生产热路径；必须重新记录 production `.text`、功能门禁和同口径性能， +再决定是否保留。standalone 多形态实验与 simpler 固定实现分两个 commit。 + +##### 7.5.17.7 200 次交错微基准将默认选型更新为 G=16 + +后续独立测试把单次 replay 的外部漂移从屏障本体中拆开：A5 单进程、 +每种拓扑先预热 10 次，再做 200 次交错采样。下表为该独立测试提供的 +p50，单位都是 device cycle；这些数据不冒充为本节前述 Case1 完整墙钟复测： + +| 拓扑 | aggregation p50 | barrier p50 | barrier 相对 flat | +| ---- | ---------------: | ----------: | ------------------: | +| flat | 29630 | 37178 | - | +| two-4 | 9626 | 28554 | -23.20% | +| two-8 | 7130 | 19898 | -46.48% | +| two-16 | 5659 | 16464 | -55.72% | +| three-6x4x4 | 6902 | 16288 | -56.19% | + +这组 200 次交错数据明确了 G=8→16 仍有屏障本体收益：`barrier p50` +从 19898 降到 16464 cycle，减少 3434 cycle / 17.26%。三级树只比 +two-16 少 176 cycle / 1.07%，当前只能视为同一档，不能断言三级确定更快。 +因此选择协议更简单、层数更少的 **two-16**：standalone 的默认 +`--final-barrier` 改为 `two-16`，`flat/two-4/two-8/three-6x4x4` 仍保留为 +明确对照选项；simpler 生产路径使用固定 G=16 二级 final 树。 + +#### 7.5.18 simpler 固定 G=16 迁移与复测 + +2026-07-21 按 §7.5.17.7 的新选型将 simpler final 汇合实现为固定 +G=16 二级树。这里没有把 standalone 的五路 selector 带进生产热路径： + +- worker 按 `block_id % 16` 到达 leaf； +- `block_id == group` 的静态 AIC 代表在本组到齐后向唯一 root 转发； +- group 0 代表在 root 到齐后发布 root release，各组代表再发布 + leaf release； +- 每组期望 worker 数和活跃组数由本轮 `layout[]` 动态推导； +- 等待全局 release 时仍执行 `drain_block_won()` 和 `drain_phase_b()`， + 退出条件仍是“全局已 release + 本核 ring 空 + 无 pending won”。 + +旧 `replay_done` cache line 原位保留，新树追加在 `DistCore[]` 之后， +不改变已有热字段和每核状态偏移。G=16 树包含 16 条 leaf arrival、 +16 条 leaf release、1 条 root arrival 和 1 条 root release，共 +34 条独立 64 B cache line / 2176 B。`DistGlobal` 总跨度从 +1007023872 B 增至 1007026048 B，standalone 的生产 ABI 镜像同步更新。 + +##### 7.5.18.1 功能和原子观察门禁 + +针对性 A5Sim/A5 runtime 构建均成功。A5Sim 的 `block_dim=1` 和 +`block_dim=36` mixed 用例均 PASS；36 blocks 会产生 108 workers，覆盖 +16 个活跃叶组以及 9/6 workers 的不均匀组大小。同一 36-block +用例开启 level-4 原子记录后 PASS，且： + +~~~text +records=5005 atomic_records=2287 dropped=0 +atomic_calls=240054009 batched_poll_calls=240052141 poll_batch_records=419 +2287 = 240054009 - 240052141 + 419 +ReplayDoneIncrement publications=141 +~~~ + +141 次发布与协议精确一致：108 次 worker leaf arrival + 16 次 leaf +代表 root arrival + 1 次 root release + 16 次 leaf release。逐核分布为 +92 个非代表核各1次、15个普通叶代表各3次、root 代表 4 次。 + +device0 的真实上限为 32 cube blocks / 64 vector blocks，所以 36-block +真机 smoke 在启动 kernel 前被 runtime 明确拒绝，没有把它记成协议样本。 +真机覆盖由下述 Case1/b256 的 32 blocks / 96 workers 完成；5 轮均 +PASS，每轮恰有 96 条 `FinalDrain` 父记录，`dropped_records=0`， +schema-v4 全部整数 cycle 闭合。 + +##### 7.5.18.2 simpler G=8 与 G=16 性能 + +口径为 A5 device0、Case1/b256、96 workers、PMU off、level-1 泳道； +G=8 和 G=16 各用 5 个独立进程。每轮的 raw `FinalDrain` 口径为 +96 条父区间的 `max(end)-min(begin)`： + +| 形态 | Submit 中位数 | worker completion 中位数 | FinalDrain 中位数 | +| ---- | -------------: | --------------------------: | --------------------: | +| 原 flat | 4865.112 us | 4929.698 us | 458.143 us | +| G=8 | 4731.252 us | 4778.583 us | 358.038 us | +| **G=16** | **4727.264 us** | **4764.885 us** | **345.774 us** | + +G=16 相对 G=8 的 Submit 只差 `-3.988 us / -0.084%`，负对照基本重合； +FinalDrain 减少 `12.264 us / 3.425%`，完整 worker completion 减少 +`13.698 us / 0.287%`。对这 10 个 G=8/G=16 Submit span 统一使用极端 +Tukey 界限，`[Q1-3*IQR, Q3+3*IQR] = [4706.716, 4753.742] us`， +没有剔除任何样本。 + +G=16 相对原 flat 的 FinalDrain 中位数为 `-112.369 us / -24.527%`， +完整 completion 为 `-164.813 us / -3.343%`。但 flat 与分层实现使用不同 +`.text` 且未交错，其 Submit 也相差 2.833%，所以不把完整墙钟差异全部 +归因于 final 树。G=8 和 G=16 的 PA 链接产物 `.text` 都是 +216912 B，`.rodata` 都是 500 B；相对原 flat 的 203344 B， +`.text` 增加 13568 B / 6.672%。 + +选型的主证据仍是 §7.5.17.7 的 200 次交错屏障微基准；本节证明 +G=16 在 simpler 完整 PA 中功能正确，且 FinalDrain 相对 G=8 继续向好。 +完整 worker completion 差异仅 0.287%，低于 5% 门槛，因此不启动 +I-cache PMU 对比。 + +#### 7.5.19 standalone shared no-wrap 删除 completion frontier helping + +2026-07-25 对照可运行参考 shared 分支后，确认 standalone 的 shared Case1 +也使用严格 no-wrap shard heap，fanin 只依赖 per-task flag，正常执行和最终 +退出均不读取连续 frontier。因此只在 `PTO_FDWIC_SHARED_MAP=1` 下编译期跳过 +`CompleteTask()` 的 `AdvanceFrontier()`；private 路径、vend → barrier → +flag 发布顺序、frontier ABI 和 AtomicSite 编号均保持不变。 + +修改前一轮 standalone shared b256 的动态计数为： + +| 项目 | 次数 | +| --- | ---: | +| frontier initial load | 1,280 | +| frontier flag load | 32,861 | +| frontier FetchMax | 31,581 | +| frontier terminal load | 1,280 | +| 三类去重后的 frontier atomic 总计 | 65,722 | + +其中 `frontier flag load = FetchMax + terminal`，所以实际完成路径删除的是 +1,280 次 initial、32,861 次 flag load 和 31,581 次 FetchMax，共 65,722 +次 atomic 调用；terminal 是 flag load 的分类，不再重复相加。修改后 A5 +shared b1/b256 四个 frontier 汇总字段均严格为 0，atomic 泳道也不再出现 +对应 site；completion vend/flag 仍完整保留。private CPU/A5 b1 的旧 +frontier 身份回归通过。 + +S4.9 提交 `e8320280` 后冻结 clean ELF，与仓内标记的 `d0042690` 冻结件 +先各预热 2 次，再跑 6 个 ABBA/BAAB block，每版 12 个正式 b256 独立进程： + +| 版本 | min | median | mean | max | 标准差 | +| --- | ---: | ---: | ---: | ---: | ---: | +| `d0042690` | 3.233ms | 7.495ms | 6.938ms | 8.562ms | 1.635ms | +| `e8320280` | 3.229ms | 3.243ms | 3.243ms | 3.265ms | 0.013ms | + +样本中位数减少 4.252ms（56.74%）；6/6 个 block 都是新版本更快，block +配对差中位数为 `-4.155ms / -56.19%`。旧版偶发 3.233ms 快样本但存在 +5～8.6ms 长路径，新版全部 12 个样本收敛到 3.229～3.265ms。结合动态 +frontier atomic 从基线取证轮的 65,722 次降为 0，可以确认该候选在本轮 +同口径配置下有效；本轮旧版长尾随修改消失，与全局前沿竞争的机制判断一致， +但这些样本不能单独证明旧版全部长尾都来自 frontier。原始日志和机器可读汇总位于： + +```text +outputs/perf_clock_pair_e8320280_vs_d0042690_20260725_090233/ +``` + +若未来 shared heap 允许回绕，必须恢复 frontier 或等价 +generation/reclaim 协议；这条保留结论不适用于可回绕实现。 + +#### 7.5.20 standalone shared Alloc 固定候选消减 ClaimMax + +2026-07-25 在 S4.9 no-wrap 基线上继续对照参考 shared 分支。参考使用 +`alloc_cursor[lane][8 shards]`，可按 task 同时选择 lane 与 shard； +standalone 为保持现有生产 ABI 仍是单层 `alloc_cursor[4]`，不能直接复制 +参考的 `task_id%3` lane 公式，否则同一 cursor 会由不同 worker 乱序推进。 + +S4.10a 候选采用“一 shard 固定一 worker”： + +```text +shard 0 -> worker 0 (block 0, AIC) +shard 1 -> worker 34 (block 1, AIV0) +shard 2 -> worker 37 (block 2, AIV1) +shard 3 -> worker 3 (block 3, AIC) +``` + +本小步只在 Alloc 的 Claim 内过滤非候选。所有 worker 仍执行 EfDrain、 +保留 Claim 边界、构造三个 Alloc Output 参数、建立 shared symbol 并进入 +generic finish；因此普通泳道、split finish、submit-PMU 和 perf-clock +边界全部不变。private 仍由 96 worker 竞争。唯一候选若执行 atomicMax +却未获胜，会立即广播 fatal，禁止把 cursor 越序错误静默解释成 replay。 + +b256 固定 Claim 由： + +| task | 修改前参与核 | 修改后参与核 | 修改后次数 | +| --- | ---: | ---: | ---: | +| Alloc | 96 | 1 | 256 | +| QK | 32 | 32 | 8,192 | +| SF | 64 | 64 | 16,384 | +| PV | 32 | 32 | 8,192 | +| UP | 64 | 64 | 16,384 | +| 合计 | - | - | **49,408** | + +相对原 73,728 次固定 Claim，精确删除 24,320 次 Alloc `ClaimMax`。CPU +shared/private b1、b256 和逐 worker owner oracle 全部通过；shared/private +CCEC swimlane、perf-clock 构建及 A5 b1、b256 均通过。shared A5 b1 +atomic raw 有 193 条 `ClaimMax`，与 +`1 + 32 + 64 + 32 + 64` 精确闭合；总 raw 4,026 条、drop=0。其余 +completion、fanin、heap、symbol、writer signature 和计算结果保持原门禁。 + +本阶段单轮 shared b256 perf-clock 为 3.300ms，当前只能作为正确性样本。 +是否比 S4.9 的冻结中位数 3.243ms 更快或更慢，必须等待 clean ELF 的独立 +进程交错配对，不能用一次运行给出收益结论。当前 `.text` 比 S4.9 增加 +256B，也要与 atomic 消减一起纳入配对解释。 + +配对已经在 clean `e83283f6` 上完成。与 `e8320280` 各运行 12 个正式 +b256 独立进程后： + +| 版本 | Claim 次数 | Submit 中位数 | +| --- | ---: | ---: | +| `e8320280` | 73,728 | 3.252ms | +| `e83283f6` | 49,408 | 3.296ms | + +6/6 个 ABBA/BAAB block 都是候选更慢，block 配对差中位数为 +`+49.152us / +1.514%`。也就是说，24,320 次 `ClaimMax` 的物理消减已经 +由计数严格证明，但没有转化成完整 Submit 收益，反而发生稳定小幅回退。 + +该 4-shard 固定 owner 候选还使 12 个正式样本的 +`max_wins_per_worker` 从基线 32～37 增至候选 75～81，并让 mixed ELF +`.text` 增加 256B；perf-clock 无法把回退唯一拆给 winner 集中或代码布局， +不能把其中任一项写成已证明的单一原因。完整日志位于： + +```text +outputs/perf_clock_pair_e83283f6_vs_e8320280_20260725_102404/ +``` + +下一步只验证参考实现还具备、当前小步尚未具备的“shared Alloc 非候选立即 +返回”。非候选仍须保留调用方的三个 Output 参数和 +`PrepareSharedTaskOutputs` 符号句柄；外层 submits 保持 5×batches， +split-finish、普通阶段和局部 PMU 调用数按真实早退精确减少。完整早退版 +与 `e83283f6` 的对照只解释早退增量，最终必须相对 `e8320280` 取得净收益； +否则整个候选方案应撤销，不能只以 atomic 次数下降作为保留理由。 + +#### 7.5.21 S4.10b 完整早退实测与整套 S4.10 撤销结论 + +S4.10b 提交 `f41e2833` 在 S4.10a 的 4-owner 规则上继续前移 shared Alloc +非候选早退。所有 worker 仍建立逻辑 task、构造三个 Output 参数并创建稳定 +shared symbol handle；只有 owner 继续执行 EfDrain、Claim 和 generic +finish。外层逻辑 Submit 仍为每核 `5×batches`，首个 Submit 起点和末个 +Submit 终点也保持不变；普通泳道、局部 PMU 和 host oracle 则按真实进入 +full path 的调用数改为 `4×batches + owned_alloc`。 + +这一步没有扩大 raw ABI。shared b1 的普通泳道实测为 3,458 条 raw、零 +drop；六类 full-path span 每类各 385 条，`ClaimMax` 仍为 193 条。b256 +claim PMU 的 full-path 总数为 98,560：92 个非 owner 核各 1,024 次， +4 个 owner 核各 1,088 次。CPU shared/private、CCEC 14 种构建组合以及 +A5 shared b1/b256 的 heap、symbol、依赖、输出 tile、split-finish 和 +逐核计数门禁均通过。这证明完整早退的功能语义和稀疏观察口径成立，但不能 +替代性能门槛。 + +clean `f41e2833` 冻结 ELF 的 `.text` 为 130,104B；相对 S4.10a +`e83283f6` 增加 768B，相对 S4.9 `e8320280` 增加 1,024B。两组冻结件 +都先各预热 2 次，再跑 6 个交替 ABBA/BAAB block，每版 12 个正式 b256 +独立进程： + +| 对照 | 基线中位数 | S4.10b 中位数 | block 配对差中位数 | block 胜负 | +| --- | ---: | ---: | ---: | ---: | +| S4.10a `e83283f6` | 3.292ms | 3.310ms | `+17.885us / +0.543%` | 1 快 / 5 慢 | +| S4.9 `e8320280` | 3.250ms | 3.315ms | `+62.879us / +1.934%` | 0 快 / 6 慢 | + +原始日志和机器可读汇总位于： + +```text +outputs/perf_clock_pair_f41e2833_vs_e83283f6_20260725_112141/ +outputs/perf_clock_pair_f41e2833_vs_e8320280_20260725_112548/ +``` + +S4.10b 相对 S4.10a 的完整早退没有收回前一小步的回退，反而继续回退； +整套 S4.10 相对 S4.9 的净回退约为 1.93%。配对还观察到固定 owner 使 +`max_wins_per_worker` 长期集中到约 73～82，而 S4.9 的一组对照为 +30～49；不同配对轮次的 fanin 和活跃 tile 也同步变化。它们只作为已证实 +的伴随现象，不能由 perf-clock 唯一拆成单一因果。 + +因此按预先声明的净收益门槛，撤销 S4.10a 与 S4.10b：恢复 shared Alloc +由 96 worker 竞争的 S4.9 路径，同时删除为稀疏 full-path 观察新增的 +converter/analyzer/PMU 分支。保留本节实验记录，避免未来仅凭“atomic 次数 +减少”再次引入同类回退。若以后重新研究唯一候选，应先采用参考实现那样的 +多 lane×多 shard 分散 owner 和匹配的 cursor ABI，作为独立架构实验重新 +建立正确性及冻结配对证据。 + +撤销后的非文档文件与 `e8320280` 逐字节相同。用户 `.venv` 下 85 项 +PMU/converter/exclusive analyzer 测试、CPU shared/private b1/b256、 +CCEC shared/private 的普通与 perf-clock 四种构建，以及 A5 shared +b1/b256、private b1 均通过。重新构建的 shared perf-clock +`.text=129,080B`，host/kernel SHA 与 S4.9 冻结件完全相同；A5 b256 +单轮为 3.233ms，Claim 恢复到 73,728，96 个 worker 均有 winner， +`max_wins_per_worker=35`。这条单轮只作为恢复身份与量级检查；S4.10 +撤销决定仍以此前 12+12 样本的冻结配对为依据。 + +#### 7.5.22 S4.11 pure INPUT publication 延迟实验 + +S4.11 没有删除 ready INPUT 的固定 atomic，只把 publication 的一次读取 +从 Collect 移到 Build；只有未就绪 INPUT 的反复 publication polling 被 +移出 Submit，在 fanin completion ready 后、Kernel 前再次确认。 +INOUT/OutputExisting 的 eager publication 与 writer FetchMax 始终不变。 +`shared_symbol_input_loads=1,280` 是 logical INPUT 数，不能冒充 physical +atomic 消减；`fanin_ready/not_ready` 也只统计 task flag。 + +两版 clean 候选分别与 S4.9 做了 12+12 正式配对： + +| 候选 | `.text` | block 胜负 | 配对差中位数 | +| --- | ---: | ---: | ---: | +| `b516409e`，resolver 内联 14 份 | 157,496B | 2 快 / 4 慢 | `+11.105us / +0.342%` | +| `6275e328`，resolver 收敛为 4 份 | 144,440B | 1 快 / 5 慢 | `+10.326us / +0.318%` | + +候选确实产生约 88～93 次 RingBp，说明提交超前实际发生；但两轮均未取得 +稳定净收益，因此不能把“轮询位置移动”写成 atomic 优化成功。本轮代码已 +完整撤销，恢复 S4.9 eager 路径。若以后在参考的 typed ref storage 和更深 +slot 容量上重试，仍需重新统计 physical atomic 与完整 Submit，不能复用 +本轮逻辑计数宣称收益。 + +撤销后 shared perf-clock host/kernel 与 `e8320280` 冻结件逐字节相同， +b256 `RingBp` 恢复为 0；因此当前 atomic 与 publication 口径也已严格 +回到 S4.9，而不是仅在源码表面删除了实验分支。 + +#### 7.5.23 S4.12a shared loser generic finish 快返实验 + +S4.12a 候选 `b2fe435f` 只让 shared Claim loser 在建立稳定输出引用后 +跳过 generic finish；Claim 候选集合、winner 分布、fanin、publication、 +completion 以及所有 physical atomic 次数都没有改变。因此它是控制流 +裁剪实验，不是 atomic 消减实验。 + +CPU、CCEC、A5 泳道和稀疏局部 PMU 均证明 winner-only 路径及其观察契约 +正确。然而与 S4.9 `e8320280` 做每版 12 个正式进程、六个 ABBA/BAAB +区组的 b256 配对后，候选区组差中位数只有 +`-0.477us / -0.015%`，区组胜负为 3:3；候选均值也只表面快约 0.043%。 +结论是噪声内中性，不能把源码调用数减少解释成 atomic 或墙钟收益。 + +本轮已完整撤销，当前 atomic 计数与 S4.9 保持一致。详细正确性门禁、样本 +分布和原始结果路径见 `shared_tensormap_record.md` 的 S4.12a 实测结论。 + +#### 7.5.24 S4.13 `3×8` Alloc cursor 消减实验与撤销结论 + +候选 `327de856` 对照参考提交 `0350b558/076f1265`，为 shared Alloc +增加独立的三 lane、每 lane 八 shard cursor，并在固定 32-block 拓扑中 +按 `lane=task%3, shard=task&7, block=shard` 固定唯一 owner。全部 +EfDrain、Claim span 和 generic finish 仍保留，因此本轮只验证 atomic +访问形状，不混入参考的 pre-EfDrain 早退。 + +b256 的物理 Claim 从 `288B=73,728` 降到 `193B=49,408`，精确删除 +24,320 次 Alloc `ClaimMax`；256 个 Alloc winner 分散为 +`16×11 + 8×10`。A5 b1 raw 中 193 条 `ClaimMax` 按 task kind 精确为 +`1/32/64/32/64`，均为 FetchMax 的 `return_ready` 边界，和 Claim +attempted 一一闭合。 + +功能与计数全部通过，但相对 S4.9 的六个 ABBA/BAAB 区组只有 3 快/3 慢， +区组差中位数为 `+3.666us / +0.113%`；候选样本中位数慢 2.789us， +均值则表面快 2.514us，方向不一致。`max_wins_per_worker` 从基线 +29～42 收敛为 27～32，说明分散 owner 生效，却没有形成稳定完整 Submit +收益。 + +按预设门槛，本轮不追加第二轮、不叠加早退，完整撤销候选并恢复 S4.9 的 +73,728 次 Claim。atomic 数量下降仍作为已证实机制保留在实验记录中，但 +不能写成当前代码的性能收益。详细 ABI、正确性门禁和逐样本路径见 +`shared_tensormap_record.md` 的 S4.13 节。 + +#### 7.5.25 S4.14a shared Vector cursor 迁址对照 + +参考分支的 `cube=8/vector=16` 和本仓旧 `G=1/4/8/16` sweep 都同时 +改变过多类 cursor，不能证明 Cube 或 Vector 的独立收益。若直接把当前 +shared Vector 从 production prefix 的四分片改成 sidecar 八分片，还会 +同时改变地址生成、页/cache 映射与分片数，仍然不是单变量。 + +S4.14 因此拆成两步。S4.14a 先在 shared-only sidecar 尾部追加物理容量 +为 8 的 `shared_vector_cursor`，但 active shards 保持 4，SF/UP 仍按 +`task_id % kSharedVectorCursorShards` 映射，与 S4.9 的 `%4` 运算形态 +一致。旧 production-prefix `vector_cursor[4]` 保持原 +offset 且 shared 运行中必须始终为 -1；private、Cube、Alloc、候选 +worker、task 图和观察边界都不变。 + +b256 的物理 atomic 数量与每线流量也保持 S4.9 原值： + +| task | 竞争核 | task 数 | ClaimMax | +| --- | ---: | ---: | ---: | +| Alloc | 96 | 256 | 24,576 | +| QK/PV | 32 AIC | 512 | 16,384 | +| SF/UP | 64 AIV | 512 | 32,768 | +| 合计 | - | 1,280 | **73,728** | + +当前四条 active Vector cursor 每条仍承担 8,192 次 ClaimMax，另外四条 +物理线保持 -1。因此本轮只量“同样四分片从 prefix 搬到 sidecar”的影响， +不是 atomic 次数消减，也不是分片优化。 + +CPU 定向测试和 b256 完整回放已经证明： + +- 每个 SF/UP 恰有 64 个 attempted 和一个 winner； +- 32,768 次 Vector FetchMax 全部命中 `task_id%4` 对应的 sidecar； +- 四条 active 高水位与四条 inactive -1、Cube/Alloc 四分片、 + heap/TensorMap/fanin/completion 全部精确闭合。 + +CCEC private/shared 14 种构建与 manifest 全部通过。A5 shared b1 atomic +最终源码对应的 raw 为 4,127 条、drop=0，其中 288 条 ClaimMax 仍为 +`96/32/64/32/64`，全部是 `return_ready`。迁址对照 +`.text=129,080B`、`.rodata=288B`，与 S4.9 大小相同；没有新增 +atomic/泳道/PMU 记录字段。 + +提交 `e24e579c` 后冻结候选 ELF，相对 S4.9 `e8320280` 各预热两次， +再跑六个交替 ABBA/BAAB 区组；每版 12 个独立 b256 正式进程。每个样本 +都保持 73,728 次 ClaimMax、96 个 active worker、RingBp=0、 +QK/SF/PV/UP 各 256 次和相同依赖签名。六个区组的候选减基线分别为 +`-5.184%/-4.908%/-5.875%/-4.949%/-6.096%/-4.872%`，候选 +6/6 更快，配对百分差中位数为 **-5.066%**，绝对差中位数为 +**-164.322us**。它满足测量前写定的“6/6 且 `<= -0.2%`”直接保留 +门槛，无需追加第二轮。 + +这里能归因的是“保持四分片时，把 shared Vector cursor 搬到 sidecar” +这一整体改动，不是分片数或 atomic 次数消减。新增 512B 位于 +`SchedulerState` 最后，未移动任何 production、control、result 或既有 +shared 字段；额外 Host 传输也不在首末 Submit 计时窗内。究竟是 cache +映射、atomic 资源隔离还是其他地址效应，当前证据不能再细分，不能把推测 +写成定论。 + +当前 standalone shared 性能基线因此前移到 `e24e579c`。S4.14b 将在完全 +相同的 sidecar 地址、物理容量、state 大小和代码骨架下，只把有效分片数 +及取模从 4 改为 8；若通过，还必须相对 S4.9 再核一次最终净收益。完整 +逐样本日志、冻结件 SHA 和 SYS_CNT 审计位于： + +```text +outputs/perf_clock_pair_e24e579c_vs_e8320280_20260725_160507/ +``` + +详细源码历史、ABI 和门禁见 `shared_tensormap_record.md` 的 S4.14 节。 + +#### 7.5.26 S4.14b shared Vector 同址八分片结果 + +S4.14b 以已保留的 `e24e579c` 为基线，只把 +`kSharedVectorCursorShards` 从 4 改为 8。sidecar 地址、物理容量、 +state 大小、初始化、Host 传输和 `%` 寻址骨架全部不变；private、 +Cube/Alloc 和其他调度协议也不变。 + +本候选不减少 atomic：b256 仍有 32,768 次 Vector ClaimMax 和 73,728 次 +全局 ClaimMax，每个 SF/UP 仍由 64 个 AIV 竞争同一条 cursor。它只把跨 +task 的每线累计流量从 8,192 次降为 4,096 次。定向测试必须逐调用证明 +FetchMax 地址为 `sidecar[task_id%8]`，完整回放和 A5 则继续证明唯一 +winner、依赖、TensorMap、heap、completion 与真实计算输出不变。 + +性能门槛在取数前固定且互斥。相对 `e24e579c` 的首轮六个 ABBA/BAAB +区组：任何语义失败、仅 0～3/6 更快或配对百分差中位数 `>=0` 时直接 +撤销；只有 6/6 更快且中位数 `<=-0.2%` 时直接保留；其余负向边界 +（4～5/6 且中位数 `<0`，或 6/6 但中位数落在 `(-0.2%,0)`）追加 +第二轮。合并十二个区组后，仅在至少 10/12 更快且中位数 `<=-0.2%` +时保留，其他结果全部撤销。 + +当前 CPU shared 定向和 b1/b256、Python 100 项、CCEC 14 种构建及 +manifest、A5 shared b1 perf-clock 与 atomic 泳道均通过。定向测试确认 +b256 每条 Vector sidecar 线恰有 4,096 次 attempted；A5 b1 泳道 raw +4,120 条、drop=0，288 条 ClaimMax 分布仍为 `96/32/64/32/64` 且 +flags 全为 `0x53`。重建 S4.14a 可逐字节复现冻结执行节；S4.14b 与其 +`.rodata` 相同、`.text` 等长，但静态常量变化经 CCEC 优化后有 50,103 +个字节位置变化。因此后续只能把性能差归给“静态八分片构建”的整体效果， +不能仅凭墙钟再拆成 atomic 竞争与代码布局两部分。 + +提交 `ee42b8c1` 后相对同址四分片 `e24e579c` 做正式配对，六区组 +候选全部更快，百分差依次为 +`-23.501%/-24.162%/-23.144%/-23.444%/-23.775%/-22.802%`， +配对中位数为 **-722.037us/-23.472%**，满足直接保留门槛，不追加 +第二轮。 + +当前版本再直接相对 S4.9 `e8320280` 复核净收益，六区组仍全部更快， +配对中位数为 **-900.894us/-27.665%**。两轮每个正式样本都保持 +73,728 次 ClaimMax、96 active、RingBp=0、相同依赖签名和四类 kernel +各 256 次，SYS_CNT 与全部语义断言闭合。 + +八分片并没有减少 fanin 工作：相对同址四分片，fanin loads 中位数反而 +从 24,041.5 增至 38,458，fanin not-ready 从 19,953.5 增至 32,410； +单核最大 winner 中位数则从 35.5 降至 18。它在更多轮询下仍明显更快， +说明收益不是“少执行后续操作”的假象；但由于静态代码布局同时变化，仍 +不能把全部收益只写成 atomic 等待下降。 + +证据目录： + +```text +outputs/perf_clock_pair_ee42b8c1_vs_e24e579c_20260725_163800/ +outputs/perf_clock_pair_ee42b8c1_vs_e8320280_20260725_164329/ +``` + +#### 7.5.27 S4.15a shared Cube 四分片迁址对照与性能否决 + +S4.15a 不直接扩张 production-prefix `cube_cursor[4]`，而是在现有 +shared-only sidecar 尾部追加容量 8、active 仍为 4 的 +`shared_cube_cursor`。shared QK/PV 继续按 `task_id%4` 路由；旧 +prefix、private、已保留的 Vector8 和 Alloc4 全部不变。sidecar 从 +4,736,192B 增至 4,736,704B，新增 512B 位于 state 末尾，不增加 +trace、PMU、span 或 atomic 记录字段。 + +尾部增长仍会改变 GM 分配长度、`scheduler_state_size` 常量及潜在 +静态代码/页布局，所以 S4.15a 配对只能评价“Cube 迁址候选整体”， +不能把差值直接等同为 atomic 硬件延迟。后续 S4.15b 才在同一地址、 +容量和 state 大小下只切 active `4→8`。 + +本候选仍固定执行 16,384 次 Cube ClaimMax。四条 active line 每条 +4,096 次 attempted，四条 inactive line attempted 为 0 且 cursor +终值保持 -1；全局 ClaimMax 仍为 73,728。定向测试必须逐调用核对 +FetchMax 地址，完整 CPU/A5 回放继续证明唯一 winner 和全部业务语义， +不能只用最终 cursor 值代替并发正确性。 + +性能仅在正确性门禁闭合、形成提交并冻结 ELF 后，相对 `ee42b8c1` +做六区组 ABBA/BAAB 配对。首轮规则预先固定：语义失败，或中位数 +`>=+0.2%` 且仅 `0～2/6` 更快时撤销；中位数 `<=-0.2%` 且 +`6/6` 更快时记为有益；绝对中位差 `<0.2%` 且 `2～4/6` 更快时 +记为中性；其他组合追加六区组。合计十二个区组时，只有 +`<=-0.2%` 且至少 `10/12` 更快才记为提升，只有绝对中位差 +`<0.2%` 且 `5～7/12` 更快才记为中性，其余全部撤销。 + +完整 ABI 数字、正确性清单和判据见 `shared_tensormap_record.md` 的 +S4.15a 节。本节不改写 S4.14a/S4.14b 的既有结果，也不提前把迁址 +描述成 atomic 次数消减或性能收益。 + +提交前验证已闭合 CPU shared b1/b256、private b1、100 项 Python +观察工具测试、CCEC private/shared 共 14 个构建与 manifest，以及 +A5 shared b1 perf-clock/合并 atomic 泳道。最新 b1 泳道 raw 为 +4,154 条、dropped=0；其中 ClaimMax 精确 288 次，AIC 的 +Alloc/QK/PV 与两个 AIV lane 的 Alloc/SF/UP 均各 32 次，全部 +`flags=0x53`。候选以历史提交 `bab00e30` 保留这些正确性证据。 + +冻结 b256 配对的六个区组差值为 +`+18.115/+22.476/+8.667/+22.822/+7.505/-5.291us`，候选只有 +1/6 区组更快；配对中位数为 **+13.391us/+0.567%**。它命中预登记的 +首轮撤销条件“中位数 `>=+0.2%` 且仅 `0～2/6` 更快”，所以不追加 +第二轮，直接撤销 S4.15a。 + +四个 warm-up 与 24 个正式样本共 28 个独立进程，每份均有 42 条 +PASS 断言；ClaimMax=73,728、active workers=96、RingBp=0、依赖签名 +`b7d985d6edb07078`、SYS_CNT 首末差和全部业务语义均闭合。基线到 +候选的 fanin loads 中位数为 `38,522→39,012`（`+490/+1.272%`）， +ready 为 `6,064→6,200`，not-ready 为 `32,460→32,814.5`； +CAS retries 均为 0，EfDrain/RingBp/FinalDrain placement 中位数均为 +`996/0/28`，没有通过少做工作制造差值。 + +两版 `.text` 同为 129,080B，但有 65,665 个字节位置不同； +`.rodata` 同为 288B 且逐字节一致。state 大小常量和编译布局同时 +变化，因此只能否决“Cube 迁址候选整体”，不能把差值解释成单条 +atomic 的纯硬件延迟。证据位于： + +```text +outputs/perf_clock_pair_bab00e30_vs_ee42b8c1_20260725_171005/ +``` + +随后由 `319077a9` 回退候选 source、ABI 和测试；当前 baseline 提交为 +`319077a9`，运行行为恢复到 `ee42b8c1`：Cube/Alloc 使用 +production-prefix 四分片，shared Vector 使用 sidecar 八分片。S4.15a +正确性成立的历史提交和取证保留，但不得再描述成当前布局或性能收益。 + +#### 7.5.28 S4.16 shared Vector `8→16` 测量前边界 + +当前回退提交 `319077a9` 的有效布局仍是 shared Vector sidecar +capacity 8、active 8。S4.16a 只把物理容量扩成 16，active 和 Claim +热路径保持 `task_id%8`；数组起点及前八条线地址不变，尾部新增八条 +cache line。sidecar 因此从 4,736,192B 增至 4,736,704B，CPU +non-split/CCEC split `SchedulerState` 分别变为 +1,011,852,672B/1,011,858,816B。 + +本阶段的 Claim 数量不变：Vector 仍为 32,768 次、全局仍为 73,728 +次；前八条线各 4,096 次 attempted，后八条线必须为零次且终值保持 +-1。容量扩张会改变完整 state 大小、GM 分配和传输长度、 +`scheduler_state_size` 常量及潜在静态代码/页布局，所以 S4.16a +只量“capacity 16、active 8 静态布局”的整体成本。它是临时布局控制， +语义通过后固定与 `319077a9` 做六区组 ABBA/BAAB 取数，但不因其单独 +快慢取消下一步，也不单独长期保留。 + +S4.16b 复用同一地址、容量和 state，只把 active 与热路径取模从 8 +改到 16；每条线变为 2,048 次 attempted，但每个 SF/UP task 的 64 路 +竞争和总 atomic 数不变。保留判据分两层： + +1. 先相对 S4.16a：首轮语义失败、仅 `0～3/6` 更快或配对中位数 + `>=0` 即失败；`6/6` 且 `<=-0.2%` 直接通过；其余负向组合补至 + 12 个区组，只有至少 `10/12` 且 `<=-0.2%` 才通过。 +2. 第一层通过后，再相对 `319077a9` 使用同一门槛验证最终净收益。 + +两层都通过才保留 capacity 16、active 16；否则 S4.16a/S4.16b 整体 +回退。旧统一 sweep 的 `G=16-G=8` 只有 5/7 更快、配对中位数 +`-37.935us/-1.043%` 且 95% 置信区间跨零，只作为收益可能已接近 +饱和的弱先验。S4.16b 也只能评价静态 Vector16 构建整体，不能把结果 +单独解释成 atomic 硬件等待变化。完整 ABI、正确性门禁和性能协议见 +`shared_tensormap_record.md` 的 S4.16 节。S4.16a 已完成冻结配对， +4/6 区组更快、中位数 `-12.1595us/-0.5136%`；它只量化临时静态 +布局整体。S4.16b 相对它仅 1/6 更快、中位数 +`+2.468us/+0.1049%`，第一层即失败；未执行第二轮或相对 +`319077a9` 的第二层。两阶段源码均已撤销，当前恢复 capacity8/active8。 + +#### 7.5.29 S5.2a shared heap 观察闭环 + +前述 0～14、15-site 表属于 O2 的 common/private 历史证据，不回写其 +编号或当时的计数。本阶段以 append-only 方式新增 shared heap 的 +15～18 四个 site： + +```text +SharedHeapVendLoad Load 5 × batches +SharedHeapCursorLoad Load 4 × batches +SharedHeapCursorReserve FetchAdd 4 × batches +SharedHeapVendAdvance FetchAdd 4 × batches +``` + +四类旧值都参与容量判断或 reservation 地址/累计进度计算，CCEC 必须标 +return-ready，不存在 source-issue 热调用。A5 shared b1 实测合计 +5/4/4/4，17 条 direct 记录全部为 return-ready；raw 4,143 条、 +dropped=0，logical/physical/PollBatch 公式和业务断言全部闭合。证据为： + +```text +outputs/pa_scheduler_shared_swimlane_20260726_034146_3041189/ +``` + +该阶段不改变 atomic 次数，也不是性能优化；它只让既有 shared heap +调用进入可见观察闭环。output `published/last_writer` 尚未接入，所以 +当前 19-site schema 仍不能称为 shared 完整 atomic 清单。完整实现边界、 +CPU/CCEC 验证和 append-only schema 说明见 +`shared_tensormap_record.md` 的 S5.2a 节。 diff --git "a/tests/atomic_probe/pa_scheduler/PA\350\260\203\345\272\246\345\231\250\347\213\254\347\253\213\345\244\215\347\216\260\344\270\216\346\263\263\351\201\223\344\275\277\347\224\250\346\214\207\345\215\227.md" "b/tests/atomic_probe/pa_scheduler/PA\350\260\203\345\272\246\345\231\250\347\213\254\347\253\213\345\244\215\347\216\260\344\270\216\346\263\263\351\201\223\344\275\277\347\224\250\346\214\207\345\215\227.md" new file mode 100644 index 0000000000..3c26eea656 --- /dev/null +++ "b/tests/atomic_probe/pa_scheduler/PA\350\260\203\345\272\246\345\231\250\347\213\254\347\253\213\345\244\215\347\216\260\344\270\216\346\263\263\351\201\223\344\275\277\347\224\250\346\214\207\345\215\227.md" @@ -0,0 +1,2130 @@ +# PA 调度器独立复现与泳道使用指南 + +## 1. 目标与边界 + +本目录复现的是 A5 FDWIC Paged Attention Case1 的 **PA Submit 调度行为**， +不是把 PA 简化成普通 NOP 并发压测。整个目录复制到 `simpler` 代码仓之外后， +不再包含对 simpler 头文件、库、Python、PyTorch 或虚拟环境的编译和链接依赖。 +泳道转换脚本也位于本目录，仅使用 Python 标准库，不 import `simpler_setup` +或任何目录外模块。 + +独立实现保留了当前 PA Case1 与调度性能相关的完整路径： + +- 32 个 AIC worker、64 个 AIV worker，物理启动比例为 1:2； +- 默认 256 个 batch，每 batch 依次提交 Alloc、QK、SF、PV、UP 五个 task； +- 96 个 worker 各自回放 1,280 次 Submit，共 122,880 次 Submit； +- Alloc 由 96 个 worker 竞争，QK/PV 由 32 个 AIC 竞争，SF/UP 由 64 个 AIV 竞争； +- private 使用 production-prefix 的 4 路 Alloc/cube/vector Claim + cursor；当前 shared 让 Vector 使用 sidecar 中全部 8 个 active + shard，Cube/Alloc 仍使用 production-prefix 4 路；两种模式都执行 + 实际 `atomicMax` Claim； +- PA 的 TaskArgs、Tensor、TaskPayload、DistSubmitCtx、DistCore/DistGlobal 关键 ABI 布局； +- tensor tag 扫描、输出 layout、materialize，以及按构建模式选择的 private + 每核有界桶环或 shared 有序桶环的 retire/lookup/insert、register mask； +- fanin 收集、winner/loser、Replay、私有 ring slot 构造和 tensor/scalar payload 拷贝； +- EfDrain、WaitForSlot、HeapGuard、completion flag、vend、frontier、最终 drain； +- 与真实 PA 相同的单 lane 优化：Case1 不执行 BlockWon 轮询； +- 与真实泳道格式对齐的阶段记录及严格的结束状态校验。 + +默认工作量固定产生 73,728 次 Claim、1,280 个 winner 和 1,024 次 kernel +执行。每次运行都会校验这些数量以及最终 TensorMap、heap、cursor、flag、vend、 +frontier 和 worker 状态，任一不符都会返回失败。 + +两种 TensorMap 构建都先执行 `EfDrain` 和 `Claim`。private 随后保持 +compete-first eager：每核构造五类完整 `TaskArgs` 并执行 per-worker +Materialize/map 前端。shared 的 Alloc/QK/SF/PV/UP 五类 task 都只有 +Claim owner 构参和 Materialize；loser 只声明稳定 output symbol，并闭合 +轻量 Submit 边界。这里的零访问从 Claim 已经判负后的 finish/replay 入口 +开始：该路径不等待或访问 TensorMap；Claim 自身仍会访问位于 shared +sidecar 的 Vector cursor，不属于该断言。CCEC 正式泳道构建将 +orchestration caller、每核 runtime state 和 noinline finish 拆分为独立 +TU；CPU 使用同一公共业务模板做协议回归。本阶段只验收 CCEC 与 CPU， +不把 AscendC 结果写进闭环证据。 + +Case1 的 task 不是五个彼此独立的占位符。standalone 会从 Tensor descriptor 的 +owner 和当前构建模式的 TensorMap 收集 producer，去重后构造下列 fanin 图： + +| task | 直接 producer | fanin 数 | +| ---- | ------------- | -------: | +| Alloc | 无 | 0 | +| QK | 无；query/key/block-table 是外部输入 | 0 | +| SF | QK | 1 | +| PV | SF | 1 | +| UP | Alloc、SF、PV；同一 producer 的多个 tensor 会去重 | 3 | + +因此每个 batch 恰有 5 条 fanin 边，默认 256 batch 恰有 1,280 条。worker 在 +EfDrain 中逐条读取 producer completion flag，只有全部 ready 才能执行 winner +负载；descriptor materialize、TensorMap lookup/register、fanin 去重、ring slot +拷贝和 completion 发布都走实际的 standalone 调度路径。 + +这里的“依赖对等”是 **Case1 调度依赖图对等**，不是 PA 数值数据流复刻。 +`real-compute` workspace 的 QK/SF/PV/UP 都读取同一组受控输入，并写入各 +worker-kind 的独占输出 tile；QK 的数值输出没有作为 SF 的真计算输入，SF/PV +的数值输出也没有继续串到后继真计算中。fanin 会真实约束执行次序，但 workspace +只校验每类 Cube/Vector 算术和角色路由。Case1 只有一个 block group 且 +`q_loop=1`；通用 PA 的多 group、多 q-loop、跨迭代数据更新以及 joint/mixed +task 依赖当前均未覆盖，不能从本用例外推其 fanin 数量或时序。 + +有意保留的替代只有两类： + +1. 三后端当前无参数默认使用 `real-compute`：CCEC 和 AscendC 在 A5 上让 QK/PV + 执行完整 Cube matmul，让 SF/UP 执行完整 Vector add/mul，并覆盖 GM load、 + 引擎计算、GM store 和完成等待；CPU 使用同一 `128x128 float` 输入、输出布局 + 执行普通浮点 matmul/add/mul，只用于回归算术、角色路由和输出闭环。 + `scalar-nop` 仍作为显式兼容/校准模式保留；历史 NOP 数据继续按当时口径解释。 +2. simpler 的 AICPU/runtime 装载链路由本目录 host runner 代替；测试关注的 + 首个 Submit 到最后一个 Submit 区间不含 AICPU 初始化和最终回收。 + +该对等范围只覆盖当前 PA Case1 的全单-lane 图，不宣称覆盖通用 FDWIC 的 +joint/mixed task。若未来加入需要两个及以上 lane 联合执行的 task，必须补回 +BlockWon 发布、claim、drain 和剩余计数协议后再谈语义对等。 + +因此，约 5 ms 是性能参考，不是通过条件。不能为了命中 5 ms 而在 Claim、 +Register、PrepareMap 或等待路径中插入虚假延时。 + +## 2. 三种实现 + +三种后端共用 `common/` 中完全相同的 PA 模型和调度器，只分别实现原子指令、 +时钟、winner 计算体和启动入口。 + +| 后端 | 启动形式 | atomic load | Claim fetch-max | 当前 winner 负载 | +| ---- | -------- | ----------- | --------------- | ---------------- | +| CCEC | 1:2 mixed AIC/AIV ELF | `atomicAdd(addr, 0)` | `atomicMax` | `scalar-nop` 或 A5 Cube/Vector 真计算 | +| AscendC | `__mix__(1, 2)` | `AtomicAdd(addr, 0)` | `AtomicMax` | `scalar-nop` 或 A5 Cube/Vector 真计算 | +| CPU | 96 个 pthread | `fetch_add(0)` | C++17 CAS loop | `scalar-nop` 或 CPU 对等浮点算术 | + +AscendC 对 64 位 vend 使用 signed `AtomicAdd(addr, 0)`。CANN 9.1 虽提供 +unsigned overload，但它在本 mixed kernel 的 64 MiB heap wrap 位置发生过 +稳定停滞；PA vend 小于 `INT64_MAX`，因此 signed add-zero 返回的位模式与比较 +语义不变。CPU 版本用于协议、边界、算术和输出检查；它没有 A5 +Cube/Vector 指令、流水线、GM 搬运和 PMU 语义，host pthread 耗时不能与 A5 比较或外推。 + +## 3. Winner 计算负载 + +### 3.1 三后端 `scalar-nop` 兼容/校准模式 + +真实 PA 最好泳道中四类 kernel 的 1 GHz counter 均值和当前 A5 NOP 校准值为： + +| Kernel | 目标时间 | `scalar-nop` 标定数 | +| ------ | -------: | ------------------: | +| QK | 44.170 us | 129,600 | +| SF | 53.729 us | 157,900 | +| PV | 27.626 us | 79,950 | +| UP | 1.565 us | 2,400 | + +Alloc 没有模拟 kernel body。NOP 数是 A5 实测校准量，不应解释为 A5 cycle 数。 +可以在运行时覆盖： + +```bash +./run.sh run ccec --winner-workload scalar-nop --nop-count 100000 +./run.sh run ccec --winner-workload scalar-nop \ + --nop-counts 129600,157900,79950,2400 +``` + +`--nop-count N` 同时设置四类 kernel；`--nop-counts` 的顺序固定为 +QK、SF、PV、UP，允许范围为 0 到 10,000,000。兼容旧命令时，显式提供 +`--nop-count*` 而不写 `--winner-workload` 也会自动选择 `scalar-nop`；新脚本 +建议像上面一样把模式写明，避免把校准样本误认成当前默认真计算。 + +### 3.2 三后端默认 `real-compute` 模式 + +三后端当前无参数运行即选择 `real-compute`。三后端共用同一组参数，可通过 +`all` 按 CCEC、AscendC、CPU 的顺序统一运行；命令中显式写出模式仍然有效： + +```bash +./run.sh run all \ + --winner-workload real-compute --batches 8 --runs 1 \ + --real-compute-count 1 --no-swimlane +``` + +也可单独选择后端： + +```bash +./run.sh run ccec --winner-workload real-compute --batches 256 --runs 1 --no-swimlane +./run.sh run ascendc --winner-workload real-compute --batches 256 --runs 1 --no-swimlane +./run.sh run cpu --winner-workload real-compute --batches 1 --runs 1 \ + --real-compute-count 1 --no-swimlane +``` + +未传 count 时，QK/SF/PV/UP 默认使用 `6,28,4,1` 次完整迭代。这组默认值来自 +CCEC 的 A5 标定；AscendC 需用自身实测解读，CPU 只复用参数含义而不复用 +性能结论。每次迭代的输入、输出形状均为 `128x128 float`，不是 scalar NOP： + +- CCEC/AscendC AIC 的 QK/PV：GM load → MTE2/MTE1 → Cube matmul → FIX + → GM store → 完成等待； +- CCEC/AscendC AIV 的 SF/UP：GM load → Vector add/mul → GM store + → 完成等待； +- CPU 的 QK/PV 执行三重循环 float matmul，SF/UP 执行 elementwise add/mul； + 这是数学与工作区布局对等，不是设备引擎对等。 + +可以统一或分类型覆盖，范围为 1 到 128： + +```bash +./run.sh run all --winner-workload real-compute --real-compute-count 1 --no-swimlane +./run.sh run ascendc --winner-workload real-compute --real-compute-counts 6,28,4,1 +``` + +`--real-compute-count*` 与 `--nop-count*` 不能混用；三后端全部使用相同的 +范围 1 至 128、QK/SF/PV/UP 顺序和互斥规则。 +真计算默认使用 `constant` 输入模式做性能测量；需要核验矩阵布局时使用： + +```bash +./run.sh run all \ + --winner-workload real-compute --real-compute-count 1 \ + --real-compute-pattern layout-diagnostic --batches 1 --runs 1 --no-swimlane +``` + +`layout-diagnostic` 使用带权对角矩阵 +`A[r,c]=(r==c ? r+1 : 0)` 和非对称稠密矩阵 +`B[r,c]=1+((131r+17c+7rc) mod 251)`。QK/PV 逐元素校验 +`(r+1)*B[r,c]`，SF/UP 分别校验 `A+B` 和 `A*B`，可直接发现 B 转置、 +ND/NZ stride、分形重排或 FIXPIPE 输出重排错误。输入生成与校验都在 Submit +计时窗外，设备执行路径和 workspace ABI 不增加 pattern 分支。 + +workspace 包含两个只读输入 tile，并为每个 worker 按其 role 对应的两个 task kind +各保留一个独占输出 tile，共 +12,713,984 bytes。CCEC/AscendC host 在计时前初始化并 H2D，计时后 D2H； +CPU 直接访问同布局的 host workspace。所有 active +worker-kind 的最终 tile 必须分别等于 QK/PV 的 768、SF 的 5、UP 的 6，未获胜 +输出必须保留 sentinel。同一 worker-kind 的 repeat 会覆盖同一 tile，因此最终常量 +结果只证明至少完成一次。CCEC 的 repeat 完整性另由受控 PMU count1→2 +精确倍增取证；AscendC 不伪造 PMU，只将官方引擎循环、输出闭环与 count1/默认 +`[KERNEL]` span 缩放合并作为证据；CPU 则在每次算术迭代后保留编译器物化边界。 +性能默认输入使用常量 2 和 3，便于与既有 768/5/6 和 PMU 标定比较；它本身 +不证明转置或 stride。正式布局闭环由上述 opt-in 诊断完成，实测见 6.4 节。 + +默认次数来自三个独立 CCEC b256 进程的标定：QK、SF、PV 中位数分别约 +41.336、54.039、27.971 us，接近真实 PA 的 44.170、53.729、27.626 us。 +UP 的一次完整 `128x128` 流水约 2.5 us，已经是正整数迭代下限；若后续要贴近 +1.565 us，应缩小 UP tile，不能用 0 次掩盖执行。真实计算下 Cube/Vector 会在 +不同物理子核并行；关闭泳道和开启标准泳道还会改变 worker 到达、fanin 重试与 +RingBp，不能把两种观察布局的绝对时间直接相减。当前 `6,28,4,1` 优先贴近 +真实 PA 的 **per-task core work**，不通过增加无关 repeat 硬凑 5.1 ms。 +同观察口径的完整验收见 6.5 节；CPU 数值不进入 A5 性能对比。 + +## 4. 本机依赖和构建 + +CCEC 与 AscendC 使用本用户安装的 CANN 9.1。非交互 shell 不保证读取 +`~/.bashrc`，复现时建议显式执行： + +```bash +cd /path/to/pa_scheduler + +source "$HOME/Ascend/cann-9.1.0-weekly-20260708/cann-9.1.0/set_env.sh" + +export CXX=/usr/bin/g++ + +./run.sh build ccec --tensormap private +./run.sh build cpu --tensormap private +``` + +当前开发门槛只要求 CCEC 与 CPU，不要求 AscendC。用户目录中从 Ubuntu +plucky 解出的 GCC 15 会生成 binutils 2.42 不认识的 `.base64` 伪指令， +standalone host/CPU 回归必须显式使用已验证的系统 GCC 13；GCC 15 仍属于 +A5sim 的独立工具链需求，不能把两种用途混为一谈。CCEC 构建会检查 1:2 +mixed 的两个入口和 metadata section；CANN 9.1 自带的 PTO 头可直接使用。 +若换用单独安装的 PTO ISA，可把 `PTO_ISA_ROOT` 指向包含 +`include/pto/common/kernel_meta.hpp` 的目录；同一 include tree 还必须具有 +`pto/pto-inst.hpp`、`pto/common/constants.hpp` 和 `pto/common/pto_tile.hpp`。 + +当前固定三条互不混算的证据链，不再生成同时夹带泳道、PMU 与性能基线 +代码的统一 CCEC ELF： + +| 构建 | 后端 | 内容 | 构建命令 | 产物目录 | +| ---- | ---- | ---- | -------- | -------- | +| `swimlane` | CCEC/AscendC/CPU | schema-v5 普通阶段、业务父区间、真实 Submit 尾动作与 atomic（direct + PollBatch）合并采集；shared Materialize 另含 task-output/copy/flush detail，Register 含 writer-metadata detail；不配置 PMU | `./run.sh build ccec --tensormap ` 或 `./run.sh build all --tensormap ` | `build///swimlane/` | +| `perf-clock` | CCEC/CPU | 编译掉泳道、atomic 观察、phase-profile、PMU 和 kernel/lifecycle 计时；每核只新增首个 Submit 起点与末个 Submit 终点两个性能边界 | `./run.sh build-perf-clock ccec\|cpu --tensormap ` | `build///perf-clock/` | +| `submit-pmu` | 仅 CCEC | 每核完整 Submit PMU，并在编译期可选一个局部阶段；当前有 `none\|claim\|efdrain\|materialize\|register` | `./run.sh build-submit-pmu ccec --tensormap ` | `build/ccec//submit-pmu//` | + +`./run.sh build all` 只构建三后端的 `swimlane` 产物；`perf-clock` +只支持当前验收范围内的 CCEC/CPU，必须逐后端构建；`submit-pmu` 必须按 +phase 另行构建。`none` 是不做局部边界读取的完整 Submit PMU 窗口， +不能替代 `perf-clock`。 + +`--tensormap private|shared` 是由 `run.sh` 消费的构建身份，不会下传为 +benchmark 运行时参数；省略时默认 `private`。它与 `swimlane`/ +`perf-clock`/`submit-pmu` 正交，并进入产物目录、CCEC manifest 以及 +host/device `magic + ABI version + mode + build variant + +sizeof(SchedulerState)` 握手。CCEC 的 swimlane/perf-clock 两件套和 +submit-pmu 四件套都必须通过 manifest 的模式、CAP、insert-turn G、变体、 +阶段和 SHA256 校验，才能启动 host。 + +shared 构建还读取 `PA_SHARED_INSERT_TURN_GROUPS`，默认 1，只接受 +1/2/4/8/16/32/64/128；private 只允许 1。该值只在构建期生效，不是 +benchmark 参数。默认 CAP=128 时，shared ABI generation 为 11， +ABI version 为 `(11<<8)|G`；host/device 握手和 schema-v3 manifest +都会拒绝不同 G 的混件。turn-G>1 只属于本阶段维护的 CPU/CCEC 后端； +AscendC 和 `all` 会在任何构建或设备动作前被拒绝。 + +`perf-clock` 的“两个时间边界”专指新增的性能观察：task 0 在 EfDrain 前 +读取一次，末 task 完成 Submit 尾动作后读取一次。shared per-slot symbol +等待和 startup 屏障仍保留时间型 watchdog；每个等待窗口先读取一次超时起点， +随后只在每 1024 次未完成轮询时复查系统计数器。它属于防止协议永久挂死 +的正确性机制，不应谎称整个 ELF 物理上只有两条 `SYS_CNT`。CPU 变体会 +逐线程断言专用性能接口恰好调用两次，但 CPU 时间只验证协议和算术， +不能作为 A5 性能证据。 + +当前冻结并已保留的性能优化止于 S4.14b：在 S0～S4.6 的构建身份、 +private/shared TensorMap 与 no-sequencer 基线上，shared no-wrap 完成 +路径已移除无消费者的 frontier helping。S4.10～S4.13 的固定 owner、 +早退、延迟解析、loser 快返和 `3×8` cursor 均已完成独立实验并因没有 +稳定净收益而撤销。S4.14a 先把 shared Vector cursor 搬到物理容量 8、 +active 仍为 4 的 sidecar；相对 S4.9 的冻结 ELF 配对为 6/6 区组更快、 +百分差中位数 `-5.066%`。S4.14b 随后在相同地址、容量、state 大小和 +寻址骨架下只改 active `4→8`，相对 S4.14a 为 6/6 更快、配对中位数 +`-23.472%`；直接相对 S4.9 的净收益为 `-27.665%`。因此 +standalone shared 冻结基线提交为 `319077a9`，其运行行为及冻结 +S4.14b 性能口径与 `ee42b8c1` 一致。 + +S4.15a 曾仿照上述拆分方法，把 shared Cube 四分片迁到 sidecar +容量 8、active 4 的新字段。候选提交 `bab00e30` 的正确性全部闭合， +但相对 `ee42b8c1` 的六区组配对只有 1/6 更快，中位数 +`+13.391us/+0.567%`，命中预登记的首轮撤销门槛且不追加第二轮。 +该候选由 `319077a9` 回退；`319077a9` 回退后的冻结基线布局与 +S4.14b `ee42b8c1` 一致。 + +S4.16 已按测量前预登记完成。S4.16a 曾把 shared Vector +物理容量从 8 扩成 16，active 仍为 8， +用来量化尾增 512B 带来的 state/GM/静态布局整体成本；它只是临时 +布局控制，不单独长期保留。CPU shared/private、CCEC shared/private +构建和 A5 shared b1 已闭合正确性；相对重建 `319077a9` 的冻结 +六区组配对为 4/6 更快、中位数 `-12.1595us/-0.5136%`。该布局成本 +结果没有取消下一步。S4.16b 随后在相同地址、容量和 state 下只把 +active 从 8 改为 16，并闭合正确性；但相对 S4.16a 仅 1/6 更快、 +中位数 `+2.468us/+0.1049%`，第一层即失败。因此没有追加第二轮， +也没有执行相对 `319077a9` 的第二层，整套 S4.16 已回退。 + +完整历史证据和互斥判据见 `shared_tensormap_record.md`。后续 shared +TensorMap 开发与阶段门禁固定覆盖 CPU/CCEC。 +`--tensormap private|shared` 都会生成对应模式的真实可执行文件；S0 用于禁止 +伪 shared 产物的临时编译门禁已在 S2 接入 shared sidecar 后删除。两种模式仍 +使用相互隔离的产物目录、manifest 和 host/device ABI 握手，不能混用镜像。 + +S0 还修复了一处独立问题：旧 CCEC PMU 配置以 +`RunConfig::reserved[4]` 访问只有四项的数组，越界落入相邻 winner workload。 +现在 PMU mode、work amount、寄存器表地址和 magic 位于独立的 64B +`PmuProbeConfig`，不再占用 `RunConfig` 尾部或覆盖业务配置。 + +### 4.1 当前 private TensorMap：128×128 有界桶环 + +S1 已把 standalone private TensorMap 从旧的 bucket linked map 同构为 +ring-per-bucket，但没有同时改构参、heap、output ref 或真实 simpler runtime： + +- 128 个 hash bucket，每桶 128 个连续 `MapEntry`，总容量仍为 16,384； +- 每桶使用普通 `uint64_t head/tail`，因为 map 仍由单 worker 独占，不使用 + atomic、per-slot `seq`、flush 或 invalidate； +- `MapEntry` 仍为 48 bytes，前 32 bytes 保存 + `(buffer_addr, lo, hi, producer)`，后 16 bytes 只保留 ABI，不提前塞入 + shared 发布状态； +- `TensorMap` 仍为 823,312 bytes，`WorkerState` 仍为 9,231,296 bytes； + `WorkerState::map`、后续 ring slot 以及 payload 的 size/offset 均未移动； +- `AdvanceTensorMap()` 用 1,024 项 `task_entry_counts` 精确推进 + `alive_floor/cleaned_upto` 并扣减 `live_count`；桶的物理 `head` 在下一次 + lookup/insert 触达该桶时由 `RetireBucket()` 惰性推进，避免每个 task + 固定扫描 128 个桶； +- lookup 扫描该桶 `[head, tail)` 内全部合法槽，只接受 + `producer >= alive_floor` 的重叠区间，并返回最大的 producer； +- insert、existing insert 和 Register 均返回成功状态。满桶时不覆写 live + 槽、不推进 tail，Submit 把失败发布为 fatal 并终止，不能静默漏掉依赖。 + +PA Case1 在 `H=64` 下全 map 最多保留 52 个 live entry，所以当前每桶 128 +槽有充分余量；这只证明当前 Case1，不是任意任务图的通用定容结论。shared +模式使用独立的并发发布、时序过滤、reclaim 与容量协议，见 4.2 节。 + +独立 ring 自测覆盖半开区间、最新 producer、`alive_floor` 边界、跨 +`task_entry_counts` 多次回绕、满桶不覆写以及固定种子 differential，并通过 +ASan/UBSan。CPU b1、CPU b256 的完整调度断言和 CCEC private 三镜像编译也已 +通过。A5 本轮使用 CCEC private、关闭泳道、`real-compute`；b1 的 S0/S1 +来自同一构建变体，Submit host span median 分别为 64.173/61.666 us。样本 +太小，只能说明没有观察到回退，不能声称 2.507 us 是稳定收益。S1 b256 +同口径单次为 3,862.246 us，只作协议和规模回归记录，不是性能基线，也不 +替代后续配对多轮性能验证。 + +### 4.2 当前 shared TensorMap：有序插入、writer history、shared heap 与 no-reclaim ring + +S3.2a 在 S3.1 的 4,735,104B output table 尾部追加 8 条 cache-line +heap cursor 和 1 条 aggregate vend，因此 `SharedTensorMapSidecar` +在 S4.9 为 4,735,680B；S4.14a 再在尾部追加 8 条物理 shared Vector +Claim cursor，S4.14b 已启用全部 8 条。S4.16a/S4.16b 曾在数组 +尾部追加并启用另外 8 条，但因性能门槛失败已整体撤销。S6.3 将 shared +`kMaxTasks` 从 1,280 扩为 4,352，使 generation-6 sidecar 增至 +11,027,648B；region ring 和 `shared_outputs` 起点不变，heap/vend/Vector +尾字段随 output table 扩容顺延。R4c 在 offset 11,027,648B 追加 +`writer_history[4352]`，每 cell 320B，共 1,392,640B；R4c sidecar 为 +12,420,288B，默认 CAP=128 的构建身份当时为 ABI generation 7。 + +R4e-a 又在该 offset 追加 `reader_done[96]`，每个 worker 独占 64B， +合计 6,144B，形成 generation-8 的 12,426,432B sidecar。R5c 保留原 +`committed_tasks` 作为 insert-turn lane 0，又在 `reader_done` 后追加 +`insert_turn_extra[7]`，形成历史 generation-9 的 12,426,880B sidecar。 +当前 generation-10 将同一尾数组扩为 `insert_turn_extra[127]`，比 +generation-9 再增加 7,680B，sidecar 为 12,434,560B。shared standalone +的 batch 输入容量同时从 256 扩到 512，因此 CPU 非 split +`SchedulerState` 为 1,019,551,552B，定义 +`PA_COMPETE_FIRST_SPLIT_FINISH` 的 CCEC split 布局为 1,019,557,696B。 +private 的 batch 上限仍为 256，sidecar 仍为 2,113,664B， +non-split/split `SchedulerState` 仍分别为 1,007,115,968B 和 +1,007,122,112B。新增 turn 控制线只追加在 shared sidecar 尾部， +不移动此前 sidecar 字段；Vector 热路径仍使用 `task_id%8`。sidecar +仍位于 standalone 控制区和 `results` 之后，不移动 `WorkerState` +或 `RunConfig`。 + +`reader_done[worker]=D` 表示该 worker 已经结束 task `[0,D]` 的全部 +ordinary-ring 读取，初值为 -1，只允许 CAS `D-1 -> D`。最慢完成值为 +`Dmin` 时，inclusive 回收候选为 `max(-1,Dmin-H)`。这与设计文档写的 +`R=min_progress-H-1` 并不矛盾：这里的下一任务进度 +`min_progress=Dmin+1`。R4e-a 建立独立状态、严格 CAS 和纯公式 CPU +门槛；R4e-b 又在隔离 ring driver 中把 exact writer turn、reader 候选和 +单调 `reclaim_upto` 发布组合起来，并以满桶慢 reader 交错证明：task 2 +读取尚未结束时 future writer 只能得到 `CapacityBlocked`，读取返回并发布 +完成前沿后才可回收 producer 0、复用对应物理槽。该 reader-gated reclaim +组合仍没有 PA/Submit 调用者；其 `active_workers` 连续前缀和 `H` 也必须是 +整个 ring 生命周期的固定权威配置，不能在已经发布回收边界后动态改变。 +当前 PA shared Submit 已复用 generic immutable writer history，但不发布 +`reader_done`，host 反向要求 96 条线保持 -1。R4e-c2b 已在独立 +shared-protocol mixed ELF 中，以定向 +`CaptureReaderSnapshot()`、单个动态 reader/reclaimer 和固定双向物理核 +建立 A5 动态证据；它没有直接调用生产 `SharedReadRegionSlot()`，也没有 +接入 PA/Submit。因此生产 ordinary lookup→reader-close 的编译器/设备顺序 +和真实 PA 全出口前沿发布仍未闭合。 + +R4e-d1 又先补上 production lookup 与合法回收并发时的 CPU 正确性边界。 +reader 保存旧 `head` 后，未来唯一 writer 可以回收低于该 reader 查询窗口 +的无关前缀，并复用对应物理槽。lookup 分别处理两种合法交错：若初始控制 +快照读到“旧 head + 新 tail”，只在二次读取的新 head 单调前进且能把 +跨度重新约束到 ring 容量内时接受；若扫描旧 cursor 时 seq 双检失败，只在 +新 head 位于初始扫描区间且已经越过失败 cursor 时跳到新 head。其余异常 +仍按协议错误拒绝。两次额外 head load 都只位于原本即将失败的异常分支， +正常 lookup 不增加 atomic。CAP=32/64/128/256/16384 的两类确定性交错均 +通过，且门槛把 `committed_tasks` 固定为 0，证明恢复只依赖 reader 前沿 +和单调 head，不偷用全局 exact turn。该阶段仍没有把 `reader_done` 接入 +PA/Submit。shared CCEC 的 AIC/AIV generic probe、正式 entry 和 1:2 +mixed ELF 已完成生成与静态链接，但没有把 CPU/编译结果冒充 A5 跨核 +可见性证据。 + +R4e-d2 再把 reader 前沿、整 task 预检和 batch append 组合成 +`SharedTryAppendReaderGatedTask()`。它不读取或推进 +`committed_tasks`，`SharedAdvanceReaderDone()` 仍由调用者在最后一次 +ordinary lookup 之后恰好执行一次，避免把不可重复的 reader close 塞进 +可重试的容量慢路。空 ordinary batch 直接返回；非空 batch 先用已经发布的 +`reclaim_upto` 预检，能写入时不扫描 96 条 reader 线，只有 +`CapacityBlocked` 才聚合 reader 前沿、刷新 reclaim 并重试。五种 CAP 的 +满环门槛均证明：慢 reader 未关闭时整批不写，关闭后同一 batch 可重试 +成功；排在满桶项之前的独立桶项不会因后项容量阻塞或 seq 损坏而部分发布， +且 `committed_tasks` 全程保持 0。该组合目前仍只由 ring CPU 门槛和 CCEC +compile probe 调用，尚未接入 generic WriterIntentSet 或 PA/Submit。 + +S4.15a 历史候选曾在末尾追加 512B Cube cursor，但已因性能门槛未通过而 +撤销，不属于当前传输布局。 + +S4.16a 的历史布局曾保持 +`shared_vector_cursor` 起点 4,735,680B、前八条线地址和热路径 +`task_id%8` 不变，只在数组尾部追加八条 inactive 物理线。 +sidecar 为 4,736,704B，CPU non-split/CCEC split `SchedulerState` +分别为 1,011,852,672B/1,011,858,816B；后八条线必须零 attempted +且终值为 -1。S4.16b 在完全相同布局中启用全部十六条线，b256 +每线 2,048 次 attempted。上述 ABI 和语义均实际验收通过，但 +S4.16b 第一层性能门槛失败；这些数字只属于历史候选，不是当前传输 +布局。 + +每 task 最多八个 fresh output，以 16B +`FdwicOutputRef` 表达 `(producer_task_id, output_slot)`，返回句柄为 +8B `SharedTaskOutputs`。当前构建身份 ABI generation 为 11。通用 +history 能力由 generation 7 引入，generation 8 追加 `reader_done`， +generation 9 追加七条 insert-turn 物理线，generation 10 再将物理容量 +扩为 128 条并把 active G 扩到 128；generation 11 将 shared 热路径的 +有序插入完成链迁到 `TaskCell::deps_prepared`：task N 只等待 +`task[N-1]`，发布完整 writer 元数据后再用 CAS 发布 `task[N]=N`。 +Claim 仍使用原有 Cube/Alloc 四分片与 shared Vector 八分片 cursor。 +generation 11 暂时保留 generation 10 的 sidecar 物理线和 manifest +字段作为历史 ABI，默认 G1 构建要求这些线保持初值，生产热路径不再访问 +它们。history、reader progress 与 per-task 插入完成链仍是不同协议能力， +不能据此声称 PA 热路径已经接入 reader-progress reclaim。 + +当前 shared 使用独立 Submit 路径。所有 worker 先 Claim；只有唯一 owner +构造本 task 参数、Materialize descriptor 和 writer delta。fresh output +descriptor 在 Materialize 尾部写入 task 独占的 +`shared_outputs[task_id]`，不占用全局有序插入区。随后 owner 等待前一 +task 的 `deps_prepared`，只在有序区内发布 ordinary/symbol writer +元数据，再发布本 task 的 `deps_prepared`；fanin lookup、Build 和执行 +均在有序区外进行。lookup 统一只接受 `producer∈[N-H,N)`。Claim 判负后 +的 loser finish/replay 不等待 per-task 插入完成字、不读取 TensorMap， +也不构造重参数，关闭轻量 Submit 后直接继续 replay；Claim 对 Vector +cursor 的访问发生在该边界之前。 +当前生产 helper 固定 `reclaim_upto=-1`，尚未把 reader-progress reclaim +接入这条路径。 + +CPU guard-page 定向测试会把 loser 的 `TaskArgs` 页改成 `PROT_NONE`，依次 +通过 Alloc/QK/SF/PV/UP 五次 split finish；任一字段读取都会立即失败。逐核 +host oracle 还按实际 wins 精确核对四类重构参次数，防止只看全局总数而漏掉 +某个 loser 回退。S4.6 另有三组定向门禁：预置 terminal `fatal` 后 winner +不得产生 heap/slot/completion/symbol 副作用；空 region 验证分别覆盖 +Local/GM 的 manual 与 ordinary writer；shared raw 从源头不生成 +`PrepareMap`，转换器和排他分析器会拒绝任何伪造的 shared +`PrepareMap` 记录。负向测试覆盖 winner 内外以及零时长记录，避免为了兼容 +旧分析器重新引入没有业务语义的 marker。 + +shared heap 使用 `task_id % 8` 的有界绝对 cursor，物理 shard span 默认为 +32MiB；b256 每 shard 精确使用 25,821,184B，不 wrap、不复用 generation。 +非空 task 分别推进 shard cursor 和 aggregate vend，零输出 task 只读取 vend。 +shared 不调用 private per-worker ring 的 HeapGuard。host 直接核对 8-shard +实际 descriptor 地址，再以 canonical 连续地址比较 private/shared 的 +normalized writer signature。 + +Materialize 只在 owner 上预留 heap 并生成本地 descriptor。拿到 insert +turn 后，owner 在同一个有序插入区间依次发布 symbol writer history/latest、 +ordinary writer entry 和本 task 的 fresh output descriptor；完成 flush 与 +存储屏障后才交出下一枚 turn。因此 `published=N` 只表示 task N 的 descriptor +和 writer 元数据已经可供后继 lookup 使用，不表示 task N 的 Build、kernel +或 completion 已完成。真正执行依赖仍由 fanin 对应的 task completion flag +约束。 + +fresh output 发布会先预检全部 slot，再预留全部 `last_writer`，最后复制并 +发布 descriptor；重复发布或后槽异常不得覆盖已经可见的 descriptor，也不能 +留下前槽的半次成功。后续 writer 通过每 task 不可变 history 记录前任， +`last_writer` CAS 是该记录的发布边界。任一发布失败都广播 terminal `fatal`； +Claim 判负后的 loser finish/replay 不读取 shared sidecar，也不等待 +insert turn；该表述不包含此前 Claim 对 Vector cursor 的访问。 + +shared symbol resolver 只接受 flags/view 字段全零的 plain ref；其他形态显式 +失败，不能被悄悄降级为普通 descriptor。由于本 task 的 INOUT writer 已在 +lookup 前发布，resolver 会从当前 `last_writer` 沿不可变 history 回退到严格 +小于 N 的最新 writer，再统一应用 `[N-H,N)` 窗口。定向测试覆盖 +`producer -> INOUT -> 后继 INPUT` 多级链，且同一窗口同时约束 symbol、 +ordinary ring 和显式 `owner_task_id`。同一 task 对同一 symbol 的重复写引用 +仍会被拒绝。 + +符号和 `manual_dep=true` 的 output view 不进入 ordinary region ring； +非 `manual_dep` 的 GM/Local writer 则在同一有序插入区间发布到 ring。当前 +PA Case1 的 `ordinary_count=0`，host 要求 bucket/slot 保持初值；generic +定向测试另外覆盖非空 ordinary delta。生产路径固定 `reclaim_upto=-1`， +满桶时整 task 在写入前失败，不覆盖 live producer。host 按计划总 task 数 +校验完整 128 线 insert-turn 终态，并将 fresh-symbol writer 投影为跨模式 +规范化签名，另按约定补入 manual view；ordinary ring 由独立结构校验， +不冒充该签名的输入。manual view 仍不是从 shared 执行态独立取证，不能用 +该签名单独证明它对等。b1 的 +dependency/normalized writer 签名为 +`5cb454393ed48dcb`/`3a3d526c9b23c3db`，b256 为 +`b7d985d6edb07078`/`556bec7ec8d0f323`。 + +以下是 S3.1 当时的历史门禁，不是当前保留路径的提交顺序或统计口径： + +| 门禁 | 结果 | +| ---- | ---- | +| CPU private/shared b1、b256 | 全部调度与终态断言 PASS | +| shared symbol、shared ring ASan/UBSan/leak | PASS | +| Python | 100 项 PASS | +| CCEC shared submit-pmu none b1 | 调度、symbol、PMU owner 恢复均 PASS | +| CCEC private b1/b256 | 73.318 us / 3.808011 ms | +| CCEC shared b1/b256（fail-closed 修正后） | 86.552 us / 27.094219 ms | +| S3.1 历史 symbol publish/INPUT-load/exchange，b1 | 8 / 5 / 3 | +| S3.1 历史 symbol publish/INPUT-load/exchange，b256 | 2,048 / 1,280 / 768 | +| S3.1 历史 shared b256 ordered 终态 | `committed_tasks=1280`，`reclaim_upto=1214` | + +S2.5 shared b256 的同类单样本为 26.556193 ms。S3.1 在正确性审计前曾取得 +23.562916 ms，但当时重复发布和后置非法引用的失败路径可能留下部分共享 +状态；补齐全量发布预检与两遍 resolver 后，最终同类单样本为 27.094219 ms。 +两者只差约 +2.03%，且都不是多轮稳定基线，不能据此宣称稳定回退或收益； +被撤销的 23.562916 ms 也不能作为有效 S3.1 基线。S3.2a 已把 Materialize +与 heap 收敛到 winner，S3.2b 又只收敛 QK/SF/PV/UP 重构参；两步分别提交， +没有把分配主体和构参主体混成一个不可归因的性能变量。 + +S4.6 提交前审查补回 winner 的 terminal-fatal 读取后，最终 CCEC +shared b1 real-compute perf-clock 单样本为 `70.279 us`。同阶段较早 ELF +的 shared b256 scalar-nop0 / real-compute 为 `3,228.844 us` / +`5,982.840 us`；private b1 real-compute / b256 scalar-nop0 为 +`70.707 us` / `3,300.478 us`。所有模式均通过完整语义断言，shared b256 +的 96 核全部活跃且 region sequencer 保持初值。b256 数字不是补回 fatal +读取后的最终 ELF，只作 convoy 消失和同量级证据；不同阶段、负载和 ELF +不能直接相减，稳定收益仍需冻结最终 ELF 后配对多轮。 + +实现中验证了一项 CCEC 约束：`[[block_local]]` runtime state 不能包含具有 +非平凡构造函数的类型。不能用 `block-local-init` 绕过该限制；当前 +`FdwicOutputRef`/`SharedTaskOutputs` 保持 trivial POD，非法引用由显式 +`InvalidSharedOutputRef()` 工厂生成，并用 `static_assert` 固化这一 ABI +前提。 + +S2 的 2,119,808B sidecar（含 96 条 per-core progress）和 S2.5 的 +2,113,664B sidecar（删除 progress）均只属于明确标注的历史阶段。S2.5 +建立的 ordered ring 原语现已被当前 owner-only ordered-insert 路径复用： +非空 ordinary delta 会执行整 task 预检与批量 append；PA Case1 默认输入 +通常产生空 ordinary delta。仍未接入的是 reader-progress reclaim，因此 +满桶只允许在写入前终止，不允许覆盖 live producer。S2/S2.5 的历史结构、 +失败实验和上板结果见 `shared_tensormap_record.md`。 + +#### shared protocol 的独立 A5 门槛 + +R4c 的通用 WriterIntentSet 为 future writer 覆盖 latest-cache 的场景追加 +task-indexed immutable history；R4d 用独立 mixed AIC/AIV ELF 验证跨物理核 +失效与回溯。R4e-c 将该门槛泛化为 shared protocol 多场景载体，并在同一 +mixed ELF 中加入 ordinary reader→reclaimer 满环复用场景。它不启动普通 +PA benchmark，也不改变 PA kernel/host。构建和运行前先 source 本用户 +CANN 9.1，再在本目录执行： + +```bash +./run.sh build-shared-protocol-litmus ccec +./run.sh shared-protocol-litmus ccec \ + --scenario history --device 0 --runs 20 +./run.sh shared-protocol-litmus ccec \ + --scenario reader-reclaim --ordering all \ + --device 0 --runs 20 +``` + +该 action 已固定为 shared-only，不能附加 `--tensormap shared`。scenario +必须显式选择，避免默认运行错误路径。history 每轮执行两个方向： + +```text +AIC writers -> AIV reader +AIV writers -> AIC reader +``` + +每个方向都先让 reader 普通读取并预热尚为零的 future-writer history 两条 +cache line，再由两个未来 writer 发布，最后调用真实 +`CollectSharedFanin()` 沿 `E -> D -> B` 回溯。host 会逐项校验预热值、 +21 条 history record 及 21 次成功 CAS、7 个 latest、最终 fanin 和控制门； +隔离断言覆盖所有 ordinary bucket 的 head/tail、96 条 `reader_done`，以及 +reader-reclaim 结果槽和 gate 未被触碰。host 内的 history 初始化与验证保持 +为独立函数，不能因共用 ACL/build 骨架而弱化断言。 + +reader-reclaim 每轮在 `AIC reader -> AIV reclaimer` 与 +`AIV reader -> AIC reclaimer` 两个方向分别执行三种 reader-close 口径： + +| ordering | reader 读取后、`reader_done:1->2` 前的约束 | 能说明什么 | +| -------- | ------------------------------------------ | ---------- | +| `compiler-clobber` | `noinline` close 调用，callee 内为空 asm memory clobber；没有 device DSB | 当前 noinline A5/CANN artifact 的最弱动态对照；PASS 不能推出 inline 热路或架构通用充分性 | +| `payload-dependency` | 五个 payload 字段进入 8 步 checksum，tied `MOV` 后让 CAS expected/desired 同时依赖该值 | scalar 已消费 payload 值的窄依赖证据；不等价于所有设备访存完成 | +| `dsb-all` | 前后 compiler clobber，中间 `DSB_ALL` | 三者中唯一具有“全部访存指令”设备完成屏障口径的兜底对照 | + +该场景把目标 bucket 构造成 CAP=128 的满环。ordinary-ring 相关切片对应一条 +可达 append 序列：task 0 写 1 条，task 1/2/3 各写 32 条，task 4 写 +31 条，`committed_tasks=5`。`H=2` 时，96 个 reader 中只有被测 reader +停在 task 1，其余均为 task 2，因此第一次 refresh 得到 `reclaim=-1`， +真实 `SharedCheckTaskAppend()` 必须返回 `CapacityBlocked`。reader 读取 +cursor 0 的完整五字段旧 payload，再按所选口径 CAS 发布 task 2；另一物理 +核直接轮询该 `reader_done`，第二次 refresh 得到 `reclaim=0`，随后真实 +check/append/commit 把同一 physical slot 复用为绝对 cursor 128，并提交 +writer task 5，使 `committed_tasks` 从 5 变为 6。`reuse-done` 只有在 +append 与 commit 都返回成功后才发布；reader 必须等到该 gate 后才验证 +自己保留的仍是 cursor 0 旧快照。 + +这里的 host 初始化只声称 ordinary-ring 相关切片对应生产可达序列；未参与 +场景的整个 `SchedulerState` 仍按隔离门槛方式清零，不能写成完整 PA 状态 +生产可达。device 侧 `CaptureReaderSnapshot()` 刻意复刻 +`SharedReadRegionSlot()` 的“seq 读取→payload 失效→五字段读取→seq +复读”顺序，却把校验推迟到复用之后，以便区分三种 ordering；因此该场景 +验证的是这条有针对性的读取序列，不能冒充直接调用了生产 +`SharedReadRegionSlot()`，也尚未证明真实 PA 已接入 reader-close。 + +该场景只覆盖一次预期 `CapacityBlocked`→reader close→成功 append/commit +的受控链；尚不覆盖 `ProtocolError`、reader CAS 失败、append/commit 失败、 +多 reader 并发 close，或复用后由新的 device reader 再读取 cursor 128。 + +构建会为 AIC/AIV 分别保留 O3 bitcode 与 textual IR；自动 use-def 门槛读取 +`.ll`。manifest 哈希 host、运行 mixed ELF `kernel.o` 和 AIC/AIV `.ll`； +`.bc` 只作为可复查中间产物保留，当前不属于 manifest 身份。自动门槛拒绝： +三个 close 函数丢失或被合并、compiler 口径混入 DSB、payload dependency +不再直接接收五个 GM load、完整 8 步 checksum 或 +checksum→tied-MOV→delta→CAS 双参数 use-def 丢失、DSB 口径不再保持 +`clobber→DSB_ALL→clobber→CAS`，以及 validator 不再用同一动态 signal +选择 `TaskCell::deps_prepared`、比较 atomic token，并只让成功 CFG 分支的 +seq/五字段检查共同决定最终返回值。textual IR 与运行 object 是同一源码和 +主体选项下的两次 CCEC 编译，因此该门槛是 compiler 证据，不是最终 object +的机器码反汇编。 + +三种 close 都刻意 `noinline`，因此除表中差异外还共享一个调用边界。该 +边界既会限制编译器跨调用重排，也会增加 call/return 延迟，可能让未决 GM +load 自然完成。因此 `compiler-clobber` PASS 只说明当前 noinline artifact +未观察到覆盖异常，不能证明真实 inline 热路径只需空 clobber；生产迁移仍须 +重新审计最后一次 ordinary lookup 到正式 reader-close 的 O3 IR。 + +`--scenario reader-reclaim --ordering all --runs 20` 共启动 120 个全新 +host 进程；也可用下面的合并命令在同一最终 artifact 上先跑 history，再跑 +reader-reclaim，共 160 个全新进程： + +```bash +./run.sh shared-protocol-litmus ccec \ + --scenario all --ordering all --device 0 --runs 20 +``` + +脚本给每个 fresh host 进程设置 60 秒上限；超时直接使整批失败，不自动重试。 +该限制只把 ACL stream 的异常停滞转成明确结果，不能用重试掩盖协议偶发问题。 + +当前冻结 artifact 先以 `runs=1` 完成 8/8 个进程，再以 `runs=20` 完成 +160/160 个进程：history 两方向共 40 个，reader-reclaim +`2 directions × 3 orderings` 共 120 个；全部为 semantic PASS、 +cleanup PASS，且未触发进程上限。该轮 manifest 的 SHA-256 为 +`1c10093a2527fd2b80da6332bf8e03bc381b5bb08f740239264d0ffc98cd240d`。 + +host 除了结果和状态转换，还反向断言未选方向、history/symbol 控制、非目标 +bucket 及其 physical slots 未被触碰。产物固定在： + +```text +build/ccec/shared/shared-protocol-litmus/ +``` + +普通 shared 构建还会把 AIC/AIV 的 generic shared-protocol probe 各自实际 +静态链接。该 probe 同时显式实例化 WriterIntentSet、`reader_done` CAS 和 +reader-based reclaim refresh，拒绝 `__multi3` 或其他未解析 device +builtin；检查后删除，不会进入正式 mixed ELF。静态链接只证明两种 CCEC +后端能生成完整设备代码，不证明 ordinary region 的跨核 +reader-progress/reclaim 可见性已经闭合。 +shared-protocol-litmus 自身虽是 CCEC mixed ELF，但没有定义 split-finish， +因此其 GM `SchedulerState` 使用当前 generation-11 non-split 大小 +1,019,551,552B。history 场景会校验 96 条 `reader_done` 始终保持 -1； +reader-reclaim 场景则要求 96 条最终均为 task 2，且只允许被测 reader +发生 `1->2`。两种场景还校验完整 128 线 insert-turn 终态。 + +当前 `shared-protocol-litmus` 固定使用 turn-G1；检查全部 128 条物理线是 +为了证明 127 条 inactive lane 保持 -1，并不表示该 litmus 覆盖其他 G。 +各 turn-G 的 CCEC 证据由主 scheduler 构建矩阵提供。 + +shared sidecar 的 atomic 当前会计入既有 Submit/业务阶段时间，但 heap +cursor/vend、symbol writer/published 等尚未逐条接入 atomic 泳道 wrapper； +所以这一版泳道不能声称完整列出了 shared 协议 atomic，也不能从现有事件拆出 +shared heap 或 symbol 单指令成本。这不影响 host 对 cursor/vend、descriptor、 +输出符号、依赖边和规范化 writer 签名的独立校验。shared raw 不生成 +`PrepareMap`；converter/analyzer 直接按 shared 的真实稀疏阶段集合闭合， +任何 `PrepareMap` 记录都会被当作协议错误。 + +## 5. 使用说明：运行、测量与泳道查看 + +以下命令均在 `pa_scheduler` 目录执行。首次使用应先按第 4 节 source CANN +环境并完成构建。主要 action 的用途如下： + +| action | 用途 | 是否生成泳道文件 | +| ------ | ---- | ---------------- | +| `build` | 构建指定后端 | 否 | +| `smoke` | 固定 b1/r1/`scalar-nop=0` 的快速语义回归 | 否，只做内存记录校验 | +| `run` | 自行控制 batch、run、winner 负载和诊断参数 | 仅显式传入 `--swimlane-json` 时生成 raw | +| `swimlane` | 单轮运行并自动生成 raw、Perfetto merged JSON 和排他闭合分析 JSON | 是 | +| `build-perf-clock` | 单独构建 CCEC 或 CPU 的低扰动首末 Submit 计时产物 | 否 | +| `perf-clock` | 强制单进程单轮、关闭全部其他观察器，输出完整 Submit 全局跨度 | 否 | +| `build-submit-pmu` | 构建指定 `none\|claim\|efdrain\|materialize\|register` 的 CCEC PMU-only ELF | 否 | +| `submit-pmu` | 单轮采集完整 Submit PMU，可选导出 JSON | 否，与泳道隔离 | + +`ccec|ascendc|cpu|all` 用于选择后端；`all` 始终按 CCEC、AscendC、CPU +的顺序执行。所有 action 都接受一次 +`--tensormap private|shared`，位置可在 backend 后的其余参数中任意放置； +两种模式均可运行，省略时仍默认 `private`。 +CCEC、AscendC 和 CPU 的 `run/swimlane` 都可使用 +`--winner-workload scalar-nop|real-compute`、 +`--real-compute-count N`、`--real-compute-counts QK,SF,PV,UP` 或 +`--real-compute-pattern constant|layout-diagnostic`；真计算 count/pattern +与 `--nop-count*` 互斥。`smoke` 有意固定 scalar-nop，不接受这些覆盖项。 + +低扰动 b1 门禁使用： + +```bash +./run.sh build-perf-clock cpu --tensormap private +./run.sh perf-clock cpu --tensormap private --batches 1 +./run.sh build-perf-clock ccec --tensormap private +./run.sh perf-clock ccec --tensormap private --batches 1 +``` + +shared 时只替换 `--tensormap shared`。`perf-clock` action 自己固定 +`--runs 1 --no-swimlane`，并拒绝泳道、atomic、phase-profile 与 PMU +参数；需要多样本时应由外层启动多个独立进程并平衡 private/shared 顺序。 + +### 5.1 首次回归 + +先做三后端快速语义回归： + +```bash +./run.sh smoke all --device 0 +``` + +通过标准是所有 `[ASSERT]` 为 `PASS`，最终同时出现： + +```text +semantic_status=PASS postprocess_status=PASS +``` + +`semantic_status` 表示 PA 调度协议和终态校验结果；`postprocess_status` 表示 +泳道读取、导出或转换等后处理结果。任何一个为 `FAIL`，进程都会返回非零。 + +### 5.2 只运行 benchmark 和文字诊断 + +再在真实 A5 上测完整 CCEC 或 AscendC： + +```bash +./run.sh run ccec \ + --device 0 --batches 256 --runs 5 \ + --analyze-swimlane + +./run.sh run ascendc \ + --device 0 --batches 256 --runs 5 \ + --profile-phases --analyze-swimlane +``` + +这两条未传 `--winner-workload`，使用当前默认 `real-compute` 和 +`6,28,4,1`。需要复现历史 NOP 基线时按 3.1 节显式传入 +`--winner-workload scalar-nop`；CPU 使用同一选项。需同配置串行回归三后端 +时，直接把 backend 换成 `all`，但 CPU 真计算只作协议/算术回归，不作 A5 性能值。 + +此模式输出指标和泳道统计，但不会自动落盘 JSON。若只关注性能且不需要完整 +泳道逐事件分析，可去掉 `--analyze-swimlane`；若也不需要记录泳道，可使用 +`--no-swimlane` 进一步节省约 384 MiB device 内存。 + +### 5.3 生成并查看泳道 + +生成可直接载入 Perfetto 的泳道文件时使用独立的 `swimlane` action： + +```bash +./run.sh swimlane ccec \ + --device 0 --batches 1 --winner-workload real-compute + +./run.sh swimlane ascendc \ + --device 0 --batches 1 --winner-workload real-compute \ + --real-compute-count 1 +``` + +`swimlane` 是唯一的正式泳道构建口径，固定合并 schema-v5 +普通阶段、业务父区间、真实 Submit 尾动作与 atomic 记录（direct +Atomic 加 PollBatch）； +无需再显式传 `--trace-atomics`。该 action 会管理 `--runs 1` 和输出路径，因此不要再传 +`--runs`、`--swimlane-json` 或 `--no-swimlane`。转换器默认使用 `python3`；如需 +固定到用户自己的 Python，可在命令前设置： + +```bash +export PYTHON=/path/to/venv/bin/python +``` + +转换器和排他分析器都只使用 Python 标准库，不需要 PyTorch， +也不需要安装 simpler Python 包。`--analyze-swimlane` 仍只控制 runner +终端中的传统分组文字统计；无论是否传它，`swimlane` action 都会生成 +排他闭合报告。 + +修改 C++ 头文件或 kernel 后必须先执行 +`./run.sh build ccec --tensormap private`， +`swimlane` action 只消费已有构建件，不会隐式重编译。日常边界迭代默认只跑 +A5 b1；b256 只用于阶段性规模/容量收口或明确指定的长负载结论。 + +该 action 固定执行一轮，产物全部位于本目录的 +`outputs/pa_scheduler__swimlane__//`。选择 `all` 时， +同一 output root 下会按顺序建立 `ccec/`、`ascendc/` 和 `cpu/` 三个子目录： + +- `l2_swimlane_records.json`：与真实 PA 相同的十列 `fdwic_events` + 权威原始件，所有字段复算以它为准； +- `merged_swimlane.json`：只用于 Perfetto 可视化。schema-v5 的 duration + 事件只保留 `ph/name/pid/tid/ts/dur` 六个必需字段，不再逐事件 + 复制 raw 中的 `args/cat`；拖入 即可查看； +- `swimlane_exclusive_analysis.json`：以原始整数 cycle 校验并汇总 + Submit、EfDrain、OrchestrationReplay、FinalDrain 和 WorkerCompletion + 排他闭合关系，并将 Submit 内和 Submit 间的 residual 按相邻边界小表 + 聚合。完整父子层级、排他角色和数值统计以该报告为准，不再逐事件 + 塞入 merged。 + +schema-v5 禁止产生历史 `Alloc/Build/Replay` lap 与未使用的 +`DrainWon`，只用 `AllocComplete/WinnerBuild` 表达真实 Submit winner +尾动作。loser 没有可单独计时的业务动作，不生成 `LoserReplay` +伪阶段；其未归因尾段只由离线 residual 展示。每个 Kernel 还必须唯一归入 EfDrain、WinnerBuild、AllocComplete 或 +FinalDrain；孤儿、越界或多重归属都会使排他分析失败。 +除 Kernel 可以执行前序 task 外，所有 Submit 前端和尾动作的 `task_id` 必须与 +包含它的 Submit 一致。 + +shared TensorMap 会分别细化 Materialize 中的 fresh-output 发布和 +Register 中的有序 writer 元数据发布。 + +Materialize 显示： + +- `materialize.before_publish_task_outputs#N`：descriptor/heap + Materialize 和 writer delta 准备； +- `materialize.publish_task_outputs#N`：精确包住 task 独占 + `SharedOutputCell` 的预检、writer 起点、descriptor 发布和 + `published` 更新；它是父 overlay，不重复加入 Submit 排他总和； +- `materialize.publish_task_outputs.copy#N`：整批复制 + `TensorDesc` 到 `shared_outputs[N].tensors[]`； +- `materialize.publish_task_outputs.flush#N`：整批 + `FlushRegion`，位于 copy 之后、`StoreBarrier`/`published` 之前； +- `materialize.publish_task_outputs.residual#N`：由父子端点离线得到的 + 预检、`last_writer`、barrier 和 published 开销； +- `materialize.after_publish_task_outputs#N`：发布完成后的 Materialize + 收尾。 + +Register 显示： + +- `register.wait_predecessor_insert#N`：等待 task N-1 发布 TensorMap + 插入完成；task 0 直接进入下一段； +- `register.publish_metadata#N`：metadata 总区间的 overlay，只用于显示 + 父子关系，不加入可加总阶段； +- `register.publish_writer_metadata#N`：资格检查，并发布 ordinary、 + symbol 和 writer 元数据； +- `register.publish_insert_completion#N`：CAS 发布 task N 的插入完成字， + 供 task N+1 的 owner 轮询。 + +设备每个成功 winner 固定写一条 `SharedRegisterPublishMetadata`，以及 +`SharedMaterializePublishTaskOutputs`、`Copy`、`Flush` 三条 +Materialize raw detail；loser 不写这些 detail。等待、Materialize +前后段、outputs residual、writer metadata 和完成发布均由父子端点离线 +还原，不记录逐次 poll。raw detail 都是 overlay，不能再次加入 Submit +排他总和。 +CCEC 在每次 turn Load 后用一条 MOV 派生互不等同的比较值和计时依赖值， +避免 O3 利用 `Ready => observed == task_id` 把后者常量传播成 task id； +AIC/AIV 优化 IR 都必须保持 +`atomic Load -> dependency fork -> Ready branch / SYS_CNT`。该处理只存在于 +swimlane 构建，不增加 DSB、GM 访问或 SYS_CNT 读取。 +`swimlane_exclusive_analysis.json` 会独立校验： + +```text +Materialize = + before_publish_task_outputs + + publish_task_outputs + + after_publish_task_outputs + +publish_task_outputs = + copy + flush + residual + +Register = wait_predecessor_insert + metadata + publish_insert_completion +``` + +当前 placement 下 `metadata == writer_metadata`，Register 中历史 +task-output 字段必须精确为零。报告同时给出整体、AIC/AIV 和逐核整数 +闭合。 + +runner 结束时会打印准确目录： + +```text +[SWIMLANE] output_root=.../outputs/pa_scheduler_private_swimlane__ +``` + +真计算泳道必须同时检查 raw/merged 顶层 +`metadata.winner_workload`；其 `mode`、`counts`、`unit`、`input_pattern` +和 `engine_mapping` +分别应为 `real-compute`、实际 QK/SF/PV/UP 次数、 +`complete_128x128_engine_pipeline_iteration`、实际输入模式以及 +`qk/pv=cube_matmul、sf=vector_add、up=vector_mul`。逻辑 +`·kernel` span 只说明 winner 执行区间，单凭轨道名称不能证明使用了硬件引擎。 + +查看步骤： + +1. 打开 ； +2. 将 `merged_swimlane.json` 拖入页面，不要拖原始的 + `l2_swimlane_records.json`； +3. 每个 `block0` 至 `block31` 是一个物理 1AIC+2AIV block； +4. `AIC`、`AIV0`、`AIV1` 轨展示 OrchestrationReplay、Submit、Claim、 + EfDrain、WinnerBuild、AllocComplete、FinalDrain、Residual 和 RingBp + 等 runtime 阶段，带 `·kernel` 的轨展示 QK、SF、PV、UP； +5. direct Atomic、PollBatch 及 ClockBaseline 固定画在对应 + `AIC/AIV` scalar lane；direct 名称显式区分 `return_ready` 和 + `source_issue`，PollBatch 单独标识逻辑轮询 episode；不生成带 `·atomic` 的 + 伪并行子轨； +6. merged 事件名保留 phase/task，atomic 名还保留 + `site/op/boundary/call_count`；需要 `func_id/core/flags/aux` 等精确字段时 + 查同目录 raw，Atomic 的解读边界见 5.6 节。 + +WaitForSlot 和 HeapGuard 没有可伪造的逐事件起止时间，因此不单独生成 Perfetto +事件；实际发生等待时，泳道中会出现 RingBp 事件。现行 CCEC 局部 PMU +只支持 Claim、EfDrain、Materialize 和 Register，使用第 5.7 节的独立 +`submit-pmu` phase ELF。 + +`outputs/` 已被 Git 忽略，生成的几十至数百 MiB 泳道文件不会被普通 +`git add` 意外纳入提交。 + +### 5.4 手工导出或重新转换 + +转换完全由本目录脚本完成，不依赖 simpler 的 Python 包或虚拟环境。已有原始 +文件也可单独转换： + +```bash +python3 ./swimlane_converter.py \ + ./outputs//ccec/l2_swimlane_records.json \ + -o ./outputs//ccec/merged_swimlane.json + +python3 ./swimlane_exclusive_analyzer.py \ + ./outputs//ccec/l2_swimlane_records.json \ + -o ./outputs//ccec/swimlane_exclusive_analysis.json +``` + +若只需要原始记录，可通过通用 `run` action 显式指定文件；导出为避免多轮覆盖 +而要求 `--runs 1`： + +```bash +mkdir -p ./outputs/manual +./run.sh run ccec --device 0 --batches 256 --runs 1 \ + --swimlane-json ./outputs/manual/l2_swimlane_records.json +``` + +手工 `--swimlane-json` 只生成 raw，不会自动生成 merged 或排他报告； +需要随后调用上面两个脚本。该参数强制要求 `--runs 1`，避免多轮 +静默覆盖同一文件。runner、converter 和 analyzer 都只在单件写完后 +原子替换各自目标；这是逐文件发布而不是三件整体事务：runner 失败时 +不启动后处理，converter 失败时保留已完成的 raw，analyzer 失败时保留 +已完成的 raw 与 merged，任何阶段都不会把半截 JSON 冒充完整产物。 + +### 5.5 CPU 回归、参数与测量口径 + +CPU 完整协议回归建议关闭大泳道缓冲区： + +```bash +./run.sh run cpu \ + --batches 256 --runs 1 --nop-count 0 \ + --profile-phases --no-swimlane +``` + +主要选项： + +- `--profile-phases`：CPU/AscendC 兼容诊断中分别统计 Claim、EfDrain、 + WaitForSlot、HeapGuard；最终 CCEC `swimlane` 构建不接受该选项， + CCEC 的独立 `submit-pmu` 局部归因只支持 Claim、EfDrain、Materialize、Register； +- `--analyze-swimlane`：读取完整记录，输出各阶段的 per-worker 累计分布以及 + EfDrain/Materialize/Claim/Register 的 per-role、per-task-kind 单事件分布； +- `--trace-atomics`：在已开启的泳道中记录 atomic 逻辑调用；direct 调用逐条记录， + 显式等待区内六类 observation load 用带精确 `call_count` 的 PollBatch 聚合。 + 建议同时传 `--analyze-swimlane` 输出按 AIC/AIV、调用点分组的分布。不能与 + `--no-swimlane` 同用； +- `--swimlane-json FILE`：流式导出原始 `fdwic_events` JSON，要求单轮运行； +- `--no-swimlane`：关闭泳道记录，不能与 `--analyze-swimlane` 或 + `--swimlane-json` 同时使用； +- `--runs N`：同一进程和同一已装载 kernel 中连续运行 N 次； +- `smoke`：1 batch、1 run、零 NOP，但仍启动全部 96 个 worker 并执行全部校验。 + +`submit_span_us` 的定义与当前 PA 基线口径一致： + +```text +96 个 worker 中最早的第一个 Submit.begin + -> 96 个 worker 中最晚的最后一个 Submit.end +``` + +它排除启动屏障和最终 drain。`host_launch_us` 另外包含 host launch、最终 drain +和 stream/thread 同步。不同版本比较时，必须使用相同构建、相同 phase +和相同泳道/PMU 开关，因为计时、记录和 PMU 边界本身都会影响竞争时序。 + +CPU/AscendC 兼容诊断中，`[PHASE]` 的每个阶段都是每 worker 在 +1,280 次 Submit 中的累计时间： + +- Claim：当前 worker 实际参与或跳过对应 lane Claim 的完整 span； +- EfDrain：每次 Submit 开头执行已就绪私有 slot 的时间； +- WaitForSlot：仅 1,024 个 kernel winner 调用，额外给出发生等待的事件数； +- HeapGuard：Alloc/QK/SF/PV 的 1,024 个输出 winner 调用，额外给出 heap + 等待事件数。 + +WaitForSlot 和 HeapGuard 的 `calls_total` 会按 winner 所在角色分布，AIC/AIV +相加必须分别等于 1,024；等待事件则对应额外的 RingBp 泳道记录。 + +每轮还会输出一行不依赖泳道的动态原子分类： + +```text +[ATOMIC] submit_completion_ops=... fanin_ready=... fanin_not_ready=... \ +frontier_initial=... frontier_flag=... frontier_ready_fetch_max=... frontier_terminal=... +``` + +- `fanin_ready/not_ready` 分别是依赖 flag 返回 1/0 的次数，两者之和等于 + `[METRIC] fanin_loads`； +- private 模式下，`frontier_initial` 是每个 completion 对 frontier 的首次 + load；shared Case1 使用严格 no-wrap heap，完成只发布 vend/flag， + `frontier_initial`、`frontier_flag`、`frontier_ready_fetch_max` 和 + `frontier_terminal` 均应为 0； +- `frontier_ready_fetch_max` 同时计数 ready flag 和紧随其后的 FetchMax，两者在 + 这条控制流中一一对应；CCEC/AscendC 上它是一条真实 A5 atomicMax，CPU 上只是 + 一次逻辑 FetchMax 调用； +- private 模式下，`frontier_terminal` 是每次扫描最终遇到的 not-ready flag， + 当前工作量下应与 completion 数相等； + `frontier_flag = ready + terminal`； +- `submit_completion_ops` 覆盖 Claim、第一圈 HeapGuard、fanin、completion 发布和 + frontier，不包含 started/replay_done 生命周期屏障。 + +shared 模式仍保留上述 frontier 字段、AtomicSite 编号和 ABI，以便与 private +共用 converter/schema。只有在 `--trace-atomics` 已开启、 +`dropped_records=0` 且 logical/physical/batch 闭合通过时，泳道没有对应 +记录才能证明热路径没有执行调用，而不是采集丢失。如果后续 shared heap +允许 wrap 或复用 task cell，必须恢复 frontier 或等价 +generation/reclaim 协议，不能沿用 no-wrap 结论。 + +这些字段在每个 worker 的私有 `LocalStats` 中递增，kernel 结束时才发布到独占 +结果区，不为诊断新增共享 atomic。它们仍会增加少量 scalar 指令，因此优化 A/B +必须使用相同的计数布局；不能把启用分类后的绝对时间直接与旧二进制比较。 + +### 5.6 合并泳道中的 atomic schema-v5 语义边界 + +正式 `swimlane` action 固定记录 standalone 调度器中的 atomic **逻辑调用**， +不只记录 winner 或慢样本。普通 direct Atomic 仍是一条源码调用对应一条物理记录； +显式等待区内允许聚合的 observation load 则用一条带精确调用次数的 PollBatch 表示。 +生成带文字分析的 CCEC 合并泳道可直接执行： + +```bash +./run.sh swimlane ccec \ + --device 0 --batches 1 \ + --analyze-swimlane +``` + +只有使用低层 `run` action 手工导出 raw 时，才需要显式传入 +`--trace-atomics`；该兼容入口不代表存在第二种 atomic-swimlane 构建。 + +schema-v5 raw 的 `metadata.trace_schema_version` 必须为 5，且顶层 +`l2_swimlane_level` 必须为 4。转换后 direct Atomic、PollBatch 和 +ClockBaseline 都放在对应 AIC/AIV 的原 scalar lane；它们属于 scalar +调度观察，不再伪装成与 scalar 并行的独立子轨。Kernel 仍放在独立计算单元轨。 +direct Atomic 事件名显式区分两种边界： + +```text +atomic.return_ready..# +atomic.source_issue..# +``` + +边界已编码在事件名中，可在 Perfetto 中按名称搜索或过滤， +无需为每条事件保留 category/args。 + +PollBatch 转换为： + +```text +atomic.poll_batch..load× +``` + +名称中的 `call_count` 是实际执行的源码 wrapper 调用次数， +不是采样或估算值。 + +当前固定 schema 共有 21 个调用点。0～14 是既有 common/private +调用点，15～18 是 shared heap 调用点，19～20 是 per-task TensorMap +插入完成链： + +| `site_id` | Perfetto `site` | `op` | 所属路径 | +| --------: | --------------- | ---- | -------- | +| 0 | `startup_increment` | `fetch_add` | 启动屏障到达计数 | +| 1 | `startup_poll` | `load` | 启动屏障轮询 | +| 2 | `fatal_poll` | `load` | fatal 状态检查 | +| 3 | `fatal_set` | `exchange` | fatal 状态发布 | +| 4 | `claim_max` | `fetch_max` | Submit lane Claim | +| 5 | `fanin_flag_load` | `load` | fanin 依赖 flag | +| 6 | `completion_vend_exchange` | `exchange` | completion vend 发布 | +| 7 | `completion_flag_exchange` | `exchange` | completion flag 发布 | +| 8 | `frontier_initial_load` | `load` | completion frontier 首次读取 | +| 9 | `frontier_flag_load` | `load` | frontier 扫描 flag | +| 10 | `frontier_max` | `fetch_max` | frontier 推进 | +| 11 | `heap_frontier_load` | `load` | HeapGuard frontier | +| 12 | `heap_vend_load` | `load` | HeapGuard vend | +| 13 | `replay_done_increment` | `fetch_add` | 回放完成屏障到达计数 | +| 14 | `replay_done_poll` | `load` | 最终 drain 中轮询回放完成 | +| 15 | `shared_heap_vend_load` | `load` | shared aggregate vend 预检 | +| 16 | `shared_heap_cursor_load` | `load` | shared 分片 cursor 预检 | +| 17 | `shared_heap_cursor_reserve` | `fetch_add` | 取得本 task 分片物理区间 | +| 18 | `shared_heap_vend_advance` | `fetch_add` | 推进并取得 aggregate vend | +| 19 | `shared_insert_predecessor_poll` | `load` | task N 等待 task N-1 的插入完成字 | +| 20 | `shared_insert_completion_publish` | `compare_exchange` | CAS 发布 task N 的插入完成字 | + +上表是源码调用点集合，不代表每轮都会出现全部 21 类事件；例如正常成功路径不应 +执行 `fatal_set`。standalone 也没有真实 PA 后续追加的 BlockWon site，不能把真实 +PA 的九类 load 加一类 exchange allowlist 照搬到这里。 + +shared heap 四个站点的返回值都参与协议判断：vend/cursor Load 用于合法性 +与容量检查，两个 FetchAdd 的旧值分别决定物理地址和累计进度。因此 CCEC +direct 记录均使用 return-ready 边界；它们不是发布后即丢弃返回值的 +source-issue 操作。PA Case1 每 batch 固定执行 5 次 vend load，以及各 4 次 +cursor load、cursor reserve 和 vend advance。output publication/last-writer +仍按后续 S5.2 小步接入，不能把当前 19-site schema 宣称为 shared 全覆盖。 + +standalone 的通用等待区只允许以下六类 observation load 在匹配窗口内进入 +PollBatch；同一 site 在等待区外的一次性或 opportunistic 读取仍是 direct Atomic： + +| `site_id` | `site` | `op` | +| --------: | ------ | ---- | +| 1 | `startup_poll` | `load` | +| 2 | `fatal_poll` | `load` | +| 5 | `fanin_flag_load` | `load` | +| 11 | `heap_frontier_load` | `load` | +| 12 | `heap_vend_load` | `load` | +| 14 | `replay_done_poll` | `load` | + +site 19 是额外的专用聚合路径：每个 task 的 predecessor Wait episode +最多写一条 PollBatch，不占用上述六类通用等待槽；task 0 没有前驱， +因此不伪造该记录。 + +一个等待区可以同时累积多个 site，所以不同 site 的 PollBatch 时间窗可以重叠； +等待区内也可以交错 direct Atomic。direct record 写入不能隐式关闭 PollBatch， +否则这些自然交错会把同一个等待 episode 人为切碎。 + +原始 `fdwic_events` 仍是十列格式。对 `phase="Atomic"` 的记录，`auxiliary` 是 +`site_id`，`flags` 使用以下 ABI： + +- direct Atomic：bit 7 为 0；低 4 bit 是 `op_id` + （Load/Exchange/FetchAdd/FetchMax/CompareExchange 依次为 + 0/1/2/3/4），bit 4 表示返回旧值参与 + 后续逻辑，bit 5 仅对 Load 表示本次读到零，bit 6 表示是否取得 + “返回值本核可消费”边界；bits 8..31 只对 direct FetchMax 表示软件 retry， + 不能解释为调用次数； +- PollBatch：低 4 bit 必须为 `load(0)`，bit 4 必须为 1，bit 5/6 必须为 0， + bit 7 必须为 1；bits 8..31 保存无符号 24 bit `call_count`，有效范围为 + `1..0xFFFFFF`。达到上限时先落盘，再从 1 开启下一条 batch，不允许饱和后 + 丢失调用数；`task_id=-1`、`function_id=-1`。 + +schema-v5 merged 只保留可视化必需的六个 duration 字段。direct 的 +`site/op/boundary/task_id` 和 PollBatch 的 `site/op/call_count` 均在名称中； +`site_id/op_id`、原始整数 cycle、flags、retry 和 value-zero 等精确值从同目录 +raw 十列记录复算。不把这些重复复制到 merged，是为了控制数百万 +事件时的文件大小和观察工具内存。 + +边界必须按源码调用点语义解读： + +- `source_issue_bracket`：返回旧值本来就丢弃的发布型调用。当前五处是 + `startup_increment/replay_done_increment/completion_vend_exchange/` + `completion_flag_exchange/fatal_set`。结束时钟与旧值无依赖，只能表示 + 源码发射包围区间。 +- `return_value_ready`：协议本来就会判断返回值的 + `Load/FetchAdd/FetchMax/Exchange`。CCEC + 在 atomic 后生成紧邻的 `dependent MOV -> MOV SYS_CNT`；这证明旧值已可被 + 本核 scalar 消费，不证明其他核已看到发布的新值。 + +两种终点都早于本条 64-byte trace record 写入。当前不为每条 atomic +加 DSB/ISB/额外 GM load；这些操作要么后端不支持，要么会明显改写 +被测路径。两种 bracket 都不能直接称为跨核可见或 atomic retire 延迟。 + +PollBatch 的 `duration`/`poll_window_cycles` 是从该 site 在显式等待区内首次累计 +调用到边界关闭的**逻辑等待 episode 包络**。它不是独占 scalar 时间，不是 +`call_count` 次 atomic 延迟之和，也不是其中任意一次 load 的单次延迟；因此不能把 +它放进 direct atomic 的 median/p95，或用 `duration/call_count` 推导单次成本。 + +边界关闭规则与 phase/lap 共用同一次 cycle 采样： + +- 显式等待区退出时关闭匹配的 PollBatch； +- `TraceTimestamp` 在写 phase begin/end 前关闭全部活跃 batch； +- schema-v5 producer 不再生成旧 `Alloc/Build/Replay` lap；历史 helper + 仍有自身关闭规则，但不得出现在当前 raw 中； +- Kernel begin/end 也通过 `TraceTimestamp`，所以 PollBatch 不能跨入或跨出 Kernel； +- 最终 flush 只作防御性兜底，不能替代上述语义边界。 + +开启该诊断时，每个 worker 在最终 drain 之后还会写两条 +`ClockBaseline`：`clock.consecutive_sys_cnt_reads` 和 +`clock.atomic_return_dependency_hook`。前者量连续时钟读，后者量纯寄存器依赖 hook +的固定底噪。全局因此恰有 `96*2=192` 条，都只是分辨率参考，不是 +可以从每条 Atomic 机械相减的校正常数。`[TRACE_ATOMIC]` 只按 AIC/AIV、site 和 op +输出 direct 事件数、源码 bracket 原始累计、中位数、p95 和最大值； +`[TRACE_ATOMIC_POLL]` 单独输出 PollBatch 的 episode 数、精确逻辑调用数和等待包络 +分布，二者不会混算。 + +schema-v5 level-4 raw 必须按逻辑调用与物理记录两套口径闭合。设 direct 物理记录数为 +`direct_atomic_records`，则逐核和全局都必须满足： + +```text +logical_atomic_calls = direct_atomic_records + Σ(PollBatch.call_count) +physical_atomic_records = direct_atomic_records + poll_batch_records +physical_atomic_records + = logical_atomic_calls - batched_poll_calls + poll_batch_records +``` + +raw metadata 的 `fdwic_summary` 七项 +`records/atomic_records/clock_baseline_records/atomic_calls/` +`batched_poll_calls/poll_batch_records/dropped_records` 必须与 producer、逐核 state、 +raw 行重算和 converter 重算逐项一致；`dropped_records` 必须为 0，每个 worker +必须恰有 2 条 ClockBaseline。raw 到 merged 后物理 Atomic 条数保持不变；正式报告 +源码调用总数必须使用 `atomic_calls`，不能使用压缩后的 `atomic_records`。 + +记录写和 PollBatch 维护本身仍会改变后续指令布局、cache、多核到达顺序、atomic +争用与轮询次数。所以不应将 bracket 或 PollBatch duration 与未插桩 Submit 时间相减， +也不能用它们计算 atomic 对 golden 的绝对占比。每核分区固定容纳 65,536 条记录； +任何容量溢出或闭合失败都使该轮 trace 无效，不能截断后继续分析。 + +#### schema-v4 排他闭合版 A5 验收 + +2026-07-19 早期曾完成 CPU/CCEC/AscendC b1 语义门禁，但该过程态 +raw 仍含现已删除的 `LoserReplay`，当前 converter 会拒绝它们。下面 +只列当前无 loser 记录的 CCEC 证据。删除无业务实体的 loser 标记后， +b256 规模样本位于： + +```text +outputs/pa_scheduler_swimlane_20260719_103435_542368/ccec/l2_swimlane_records.json +outputs/pa_scheduler_swimlane_20260719_103435_542368/ccec/merged_swimlane_thin.json +outputs/pa_scheduler_swimlane_20260719_103435_542368/ccec/swimlane_exclusive_analysis.json +``` + +该轮使用 `real-compute/6,28,4,1`，每核 1,280 个 Submit，raw +845,813 条事件、`dropped=0`，首末 Submit 为 5,326.055 us，六类整数 +cycle 闭合全部精确相等。raw 为 56,212,672 bytes；旧格式 merged 为 +248,767,986 bytes，同一 raw 经当前六字段 converter 生成的 +`merged_swimlane_thin.json` 为 138,349,686 bytes，减少 44.4%。这是离线 +可视化瘦身，不改变该轮设备采集。同目录的 +`merged_swimlane.json` 是旧胖版，查看该规模样本时应打开上述 thin 文件。 +该轮早于最终相邻边界复用，只作规模/容量门禁，不代表当前 residual +构成。 + +边界收敛后的最新日常验证只跑 CCEC b1： + +```text +outputs/pa_scheduler_swimlane_20260719_110756_584549/ccec/ +``` + +该轮 raw 4,118 条事件、`dropped=0`，全局 Submit 89.313 us，merged +428,455 bytes，六类闭合全部通过。Submit 内所有 child-to-child +gap 已为零，即 `submit_internal_residual=0`；最后一个真实 child 到 +`SubmitEnd` 的 `submit_tail_residual` 为 184,788/2,241,892 cycle +（8.2425%）。Submit 间 residual 为 155,679/2,397,571 cycle（首末 +Submit 包络的 6.493%）。Perfetto 分别用 `submit_tail_gap` 与 +`between_submit_residual` 展示两类补集，不新增 raw 记录或字段。 + +按明确要求完成的当前 CCEC b256 规模复核位于： + +```text +outputs/pa_scheduler_swimlane_20260719_114815_617346/ccec/ +``` + +该轮使用 `real-compute/6,28,4,1`，全局 Submit 为 5,360.061 us；raw +839,526 条、`dropped=0`，merged 1,085,191 条事件。raw/merged 分别为 +55,791,947/88,775,668 bytes，全部语义断言与整数闭合通过。 +Submit aggregate core-work 为 433,383,588 cycle：内部 residual 为 0， +尾部 residual 为 41,008,786 cycle（9.4625%）；逐核首末 Submit 包络为 +500,448,909 cycle，Submit 间 residual 为 67,065,321 cycle（13.4010%）。 +122,880 条 `submit_tail_gap` 与 122,784 条 `between_submit_residual` +分别精确对应 `96*1280` 和 `96*(1280-1)`，没有增加设备事件。 + +后续边界迭代默认只跑 A5 b1；b256 仅在阶段性规模/容量收口或 +明确要求时重跑。历史 level-4 b256 多轮波动证据仍保留在本文后续 +章节，不将 b1 单轮时间宣称为性能改善。 + +#### 历史：schema-v3 边界修复版 A5 验收 + +2026-07-18 已用边界修复后的同一版 standalone CCEC 依次完成 b1 与 b256 +真机重测。下列是升级到当前 schema-v4 之前的历史样本，不应修改其 +metadata 或用当前父区间口径强行解释： + +```text +outputs/pa_scheduler_swimlane_20260718_182649_4060527/ccec/ +outputs/pa_scheduler_swimlane_20260718_182725_4061524/ccec/ +``` + +每个目录都包含 `l2_swimlane_records.json` 与 `merged_swimlane.json`。raw metadata +与事件行复算结果如下；`records` 是包含普通 phase、Atomic 和 ClockBaseline 的总物理 +记录数，不能与 `atomic_records` 混用： + +| 样本 | winner 负载 | `records` | 逻辑 `atomic_calls` | direct Atomic | 物理 `atomic_records` | `batched_poll_calls` | PollBatch | 单核记录峰值 | ClockBaseline | dropped | 首末 Submit | +| ---- | ----------- | --------: | ------------------: | ------------: | --------------------: | -------------------: | --------: | -----------: | ------------: | ------: | ----------: | +| b1 | `scalar-nop=0` | 4,414 | 1,031 | 613 | 850 | 418 | 237 | 57/65,536 | 192 | 0 | 54.056 us | +| b256 | `real-compute/6,28,4,1` | 967,307 | 105,580 | 103,618 | 103,883 | 1,962 | 265 | 10,252/65,536 | 192 | 0 | 5,774.295 us | + +两轮逐核 producer state 与 raw 扫描均为 96/96 闭合、异常核为 0；全局公式展开为： + +```text +b1: +logical = 613 + 418 = 1,031 +physical = 613 + 237 = 850 = 1,031 - 418 + 237 + +b256: +logical = 103,618 + 1,962 = 105,580 +physical = 103,618 + 265 = 103,883 = 105,580 - 1,962 + 265 +``` + +PollBatch 只出现在本轮实际进入的四类 allowlist site，下面每项依次为 +`物理 episode/逻辑调用`： + +- b1：`startup_poll=96/143`、`fatal_poll=42/47`、 + `fanin_flag_load=2/6`、`replay_done_poll=97/222`； +- b256：`startup_poll=96/143`、`fatal_poll=42/47`、 + `fanin_flag_load=16/467`、`replay_done_poll=111/1305`。 + +`heap_frontier_load/heap_vend_load` 在这两轮均未进入相应 slow path，计数为 0， +不表示 allowlist 漏实现。按同一物理核区间严格检查，b1 的 237 个 PollBatch 与 +4 个 Kernel、b256 的 265 个 PollBatch 与 1,024 个 Kernel 都是严格 overlap 0； +分别有 3 和 31 处仅 `end==begin` 的端点相接。这直接验证 PollBatch 没有跨入或跨出 +Kernel。b256 的真实 Cube/Vector 计算、调度终态和全部语义断言均 PASS。 + +converter 的 schema-v3 静态回归同时为 19/19 PASS。b1 是零 winner 负载的快速验收， +b256 是开启 atomic 泳道的诊断运行；上表 Submit 只能证明当前观察构建的运行量级， +不能替代关闭 trace 的性能基线或与历史样本做单轮减法。 + +#### 历史 schema-v2 样本（仅保留旧口径） + +2026-07-18 的旧 CCEC b256 文件 +`outputs/scalar_observation_final_20260718/atomic_inlineasm_ccec_b256/raw.json` +记录了 963,368 条物理记录，其中逐条 Atomic 99,944 条、ClockBaseline 192 条， +逐核峰值 10,308/65,536，且当轮 `dropped=0`。这些数字来自引入 PollBatch 与上述 +phase/lap/Kernel 边界修复之前的 schema-v2 逐调用模型，只能用于追溯旧版观察结果； +不能拿 99,944 当作当前 schema-v4 的物理容量、逻辑调用数或闭合证据。 + +### 5.7 两类正式构建与 CCEC Submit PMU + +`swimlane` 和 `submit-pmu` 是两个独立重编译的观察产物： + +- `swimlane` 编译普通阶段和逐 atomic record，把 atomic 画在对应 + AIC/AIV scalar lane；不生成 PMU owner，也不输出 PMU JSON。 +- `submit-pmu` 编译掉泳道 record、逐 atomic wrapper、ClockBaseline、 + runtime phase-profile 和旧 cold/warm 冲刷体，只保留完整 Submit PMU + 与一个编译期 phase。 + +两者不能在同一进程同时采集。这不只是 CLI 限制：泳道/逐 atomic +代码会改变 scalar 指令布局和 I-cache 本身，将其保留在 PMU ELF 里即使 +运行时关闭 record，也会污染要观察的取指环境。 + +`none`/`claim`/`efdrain` 保持与泳道版一致的跨 TU split-finish +形状；这两个局部阶段的边界都在 finish 之前。`materialize`/ +`register` 边界位于 finish 内，当前为了使用同一份真实 +PMU context 而采用 inline-finish 诊断 ELF。因此后两者与 `none` +的代码布局并不相同，只能解释各自 ELF 内的观测结果，不能与 +`none` 机械相减或当作无扰动的阶段净值。 + +当前 `submit-pmu` 只支持： + +| phase | 编译期 ID | 局部边界 | 用途 | +| ----- | --------: | -------- | ---- | +| `none` | 0 | 不做任何中途 shadow counter 读取 | 完整 Submit 主基准，优先用于回答 AIC/AIV 每核 request/miss | +| `claim` | 1 | 每次 `Claim()` 调用前后读取 shadow counter | 验证局部归因链路，输出带观察扰动的 running read-clear 下界和保守上界 | +| `efdrain` | 2 | Submit 开头唯一的 EfDrain call-site 前后 | 归因 opportunistic drain，不包含 RingBackpressure/FinalDrain | +| `materialize` | 4 | 每次 `MaterializeTask()` 调用前后 | 归因 descriptor materialize；成功和失败出口都由同一闭合边界覆盖 | +| `register` | 5 | 每次 `RegisterOutputs()` 调用前后 | 归因输出注册；Alloc 与非 Alloc 两个互斥调用点合起来仍是每次 Submit 一次 | + +分别构建： + +```bash +./run.sh build-submit-pmu ccec none +./run.sh build-submit-pmu ccec claim +./run.sh build-submit-pmu ccec efdrain +./run.sh build-submit-pmu ccec materialize +./run.sh build-submit-pmu ccec register +``` + +上面省略了默认的 `--tensormap private`；显式写法例如： + +```bash +./run.sh build-submit-pmu ccec none --tensormap private +``` + +产物完全分开： + +```text +build/ccec/private/submit-pmu/none/ +build/ccec/private/submit-pmu/claim/ +build/ccec/private/submit-pmu/efdrain/ +build/ccec/private/submit-pmu/materialize/ +build/ccec/private/submit-pmu/register/ +``` + +每个目录都自包含同 phase 的 host、mixed kernel、PMU owner 和 dispatcher， +不得跨 mode、turn-G 或 phase 拼装。构建完成后才会原子发布八行身份头： +schema、mode、mode-id、CAP、insert-turn G、variant、phase、phase-id， +随后记录四个 SHA256；`submit-pmu` action 在启动 host 前逐项复核，缺件、 +串 mode/G/phase 或内容变化都会直接拒绝。一次正式采集示例： + +```bash +export PYTHON="$HOME/.venv/bin/python" +OUT="./outputs/submit_pmu_none_$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p "$OUT" +./run.sh submit-pmu ccec none \ + --tensormap private \ + --device 0 --batches 1 \ + --winner-workload real-compute --real-compute-counts 6,28,4,1 \ + --pmu-json "$OUT/submit_icache_raw.json" +``` + +raw 成功发布后，`run.sh` 会在同目录自动生成一份自包含的加工件： + +```text +submit_icache_raw.json # 96 核权威原始数据 +submit_icache_report.html # 浏览器直接打开的离线图表和汇总 +``` + +compete-first eager 移植后的五种 CCEC A5 b1 回归位于 +`outputs/compete_first_submit_pmu_b1_20260720/{none,claim,efdrain,materialize,register}/`。 +五轮均为 96/96 trusted，语义、真计算输出、role/triplet、owner Restore、 +phase status/time 与 primary/shadow 门禁全部通过；`none` 的 phase calls +为 0，四个局部 phase 均精确为 `96 * 5 = 480` 次。 + +HTML 中包含 AIC/AIV 的每核 request、miss、miss rate、p95、96 核散点和 +局部 phase 的 lower/upper 区间。schema-v5 的 running phase 还在页面 +最前面按 ALL/AIC/AIV 并列阶段时间、request 和 miss 占比；阶段时间 +是 `Σphase_elapsed_ticks / Σsubmit_elapsed_ticks`，request/miss 是占同一 +ELF PMU whole-gate primary 的比例区间。两者分母边界不同，不是同一 +精确分区。`none` 显示“不适用”，历史 +schema-v4 因没有阶段时间 raw 字段而显示“不可用”。报告也展示 +ALL/AIC/AIV 的逐核 PMU `total_cycles` +与 `scalar_busy`：三个响应式角色卡只选 mean 作为典型值，并补充逐核 +min/max 显示核间范围；PMU total 与 scalar busy 的极值分别独立计算，不保证 +来自同一个物理核。顶部“完整 Submit（最早开始 → 最晚结束）”是 96 核整体 +墙钟范围，不能与逐核 PMU mean 混为一个统计量。报告还展示 scalar/total 比例 +和逐核散点，并同时保留 raw cycle 与本机校准后的每核等效时间；宽表只在表格 +内部横向滚动。受控 cold/warm 同窗实测 +`1,817,457 PMU cycles / 1,101,593 ns = 1.649844 cycles/ns`；AIC/AIV +分别使用 `1.650062/1.649731 cycles/ns`，换算式为 +`time_us = cycles / cycles_per_ns / 1000`。报告将 +`total_cycles-scalar_busy` 明确标为“非 Scalar-busy 残余”。`total_cycles` +是每个物理子核在 PMU whole gate 内的 64-bit raw total，96 核求和是 core-work， +不是约 5 ms 的 Submit 墙钟；`scalar_busy` 是 CNT2 的 +`scalar_instr_busy(0x001)`。依赖返回值的 atomic 等待可以落入 scalar busy， +而 I-cache refill 的额外周期可能主要只增加 total，但 +**`total_cycles - scalar_busy` 既不能解释为 Scalar 空闲，也不能解释为 +I-cache stall**：差值还混有同步等待、Cube/Vector/MTE 等 engine 等待 +及其他非 scalar-busy 周期。2026-07-19 用本机 CANN 9.1 在 A5/DAV3510 +上依次实测 `PipeUtilization`、`PipeUtilization,MemoryDetail` 和 `Default`， +三份 `PipeUtilization.csv` 均没有 `scalar_wait_ib_time` 或 +`scalar_wait_time`；DAV3510 正式事件表也没有对应 selector/公式。因此 +当前 A5 正式可编程路径不采这两项，不套用其他产品的事件号。 +原始证据位于 `outputs/wait_ib_official_msopprof_20260719_b1_probe2/`、 +`outputs/wait_ib_official_msopprof_20260719_b1_probe3_memory_detail/` 和 +`outputs/wait_ib_official_msopprof_20260719_b1_probe4_default/`。 +报告只复用 `pmu_sidecar_analyzer.py` 已校验的统计口径；生成失败不会删除已经发布 +的 raw，但本次 action 会返回非零。 + +这里的 PMU whole gate 从 orchestration 初始化前开始，到末次 +Submit 返回后停止，包含 Submit 内的 EfDrain、Claim、当前模式实际构参 +（private 全员 eager；shared 五类 task 都是 owner-only）和 +finish，也包含 Submit 间的 `AcceptTaskOutputs()`/调用衔接，排除 FinalDrain。 +`submit_elapsed_ticks` 是每核首末 Submit 时间；顶部 +`submit_span_us` 则是 96 核共同墙钟范围。三者不能混用，详细定义见 +`../icache_miss_usage_guide.md`。 + +当前边界联动版已对 `none|claim|efdrain|materialize|register` 五个独立 +ELF 完成 A5 b1 门禁，五轮均为 96/96 有效记录并通过语义、真计算、 +phase call/time、primary/shadow 和 owner Restore。产物位于： + +```text +outputs/submit_pmu_boundary_sync_b1_20260719/{none,claim,efdrain,materialize,register}/ +``` + +该 b1 只作源码边界与工具闭合证据，不用不同 phase ELF 的单轮时间差 +宣称性能改善。 + +两类新增局部 phase 可按与 `none` 相同的参数分别运行；输出文件名应体现 phase， +避免误把不同 ELF 的结果放进同一组： + +```bash +for phase in materialize register; do + OUT="./outputs/submit_pmu_${phase}_$(date -u +%Y%m%dT%H%M%SZ)" + mkdir -p "$OUT" + ./run.sh submit-pmu ccec "$phase" \ + --device 0 --batches 1 \ + --winner-workload real-compute --real-compute-counts 6,28,4,1 \ + --pmu-json "$OUT/submit_icache_raw.json" +done +``` + +`submit-pmu` action 自己固定 `--runs 1 --no-swimlane --pmu-window submit-all`， +不要重复传入这三项，也不能传入 `--profile-phases`、 +`--trace-atomics`、`--analyze-swimlane` 或 `--swimlane-json`。 + +后续 submit-pmu 构建、门禁和边界迭代默认只跑 A5 b1；b256 只用于 +阶段性规模/容量收口或明确要求的长负载结论。 + +Submit-all PMU 整窗的权威 I-cache 主计数是从不在局部边界读取的 +`CNT6=request` 和 `CNT7=miss`。A5 b1 实测已反证 `CNT9=0x35` +可作有效计数槽：它始终为 0。因此正式 `submit-pmu` 用 +`CNT8=0x34` 作 shadow request、`CNT5=0x35` 作 shadow miss，`CNT9` +保持未使用；这会牺牲 PMU 诊断版的 `mte3_busy`，不影响标准 +`swimlane` 构建。 + +shadow 计数器是 read-to-clear：选中的局部 phase 在 begin/end 切分片段，stop 时 +再加 tail，从而软件重建 PMU whole-gate shadow whole。schema-v5 同时在 +begin read-clear 之后取阶段起点，在 end read-clear 之前先取终点；因此 +阶段时间不包含两侧 `ld_dev`，但包含每次调用两次 SYS_CNT 的观察 +扰动。`none` 没有运行中读取，必须 +在每个物理子核精确满足： + +```text +CNT8 shadow whole request == CNT6 primary whole request +CNT5 shadow whole miss == CNT7 primary whole miss +``` + +`claim/efdrain/materialize/register` 在 A5 上运行中反复 read-clear 时，shadow +可能在边界处单向少计， +因此接受条件改为逐核： + +```text +shadow request <= primary request +shadow miss <= primary miss + +request loss = primary request - shadow request +miss loss = primary miss - shadow miss + +phase request ∈ [observed request, observed request + request loss] +phase miss ∈ [observed miss, observed miss + miss loss] +``` + +区间必须逐核构造后再聚合。CNT8/CNT5 是顺序 `ld_dev` 而非原子配对快照， +不要求局部 `phase miss <= phase request`；只要求二者分别不超过对应 shadow， +上界分别不超过对应 primary。`none` 中 phase calls/begin/end/request/miss 必须 +全为 0；其余四个 phase 的每核 begin/end/calls 都必须配对，且每核 calls 固定为 +`batches * 5`，全局为 `batches * 5 * 96`。原因是每个 batch 固定提交 +Alloc/QK/SF/PV/UP 五个 task，每次 Submit 都恰好执行一次 Claim、开头 EfDrain、 +Materialize 和 Register 边界。`efdrain` 插点只允许包围 Submit 开头的专属调用， +不能插入复用的 `DrainReady()` 函数体；`materialize` 必须先保存真实返回值再关闭 +边界，保证失败出口也闭合；`register` 的 Alloc 与非 Alloc 两个源码调用点互斥， +不能误算成每次 Submit 两次。每个 running phase 还要求每核 +`phase_elapsed_ticks > 0`且不超过同核从首个 `submit_begin` 计时点 +到末个 `submit_end` 计时点的 `submit_elapsed_ticks`；前者位于首个 +`BeginCallbackSubmit()` 上下文初始化之后，后者位于末个 Submit 返回之前。 +`none` 的 phase elapsed 必须精确为 0。 +当前 Case1 中真实 TensorMap insert 工作主要发生在 +UP 的输出注册，其他 task 的 Register 可能很短或没有 insert；因此该 phase 的 +固定调用数只证明边界覆盖完整，不能解释为五类 task 拥有等量注册工作。 + +局部边界读取本身会增加 scalar 指令、改变 I-cache 布局和多核时序， +因此所有非 `none` phase 都是带边界扰动的归因结果。`none` 与每个局部 phase +是不同 ELF/不同进程；不同 phase 必须各自单独采集。不同 phase +ELF 的局部 request/miss **不可相加**，也不能与 `none` 相减后宣称 +得到了零扰动的阶段净值。这个区间只约束同一插桩 ELF、当前边界定义下的 +局部事件；它不是对应阶段在无插桩构建中的真实区间。调度语义、真计算输出和 +placement/engine 门禁都通过时，running shadow 的负差属于观测边界行为， +不得描述为 standalone scheduler 异常。 + +JSON 保留 96 条 raw record，并按 ALL/AIC/AIV 输出 authoritative whole、 +shadow loss 和 phase lower/upper。raw 中包含 `shadow_request_loss`、 +`shadow_miss_loss`、`phase_icache_requests_upper_bound` 与 +`phase_icache_misses_upper_bound`；schema-v5 还包含逐核 +`submit_elapsed_ticks`、`phase_elapsed_ticks`和 `phase_time_valid`。host 还分别 +报告 exact/bounded 核数。时间占比必须在 ALL/AIC/AIV 各自范围内按 + +```text +Σphase_elapsed_ticks / Σsubmit_elapsed_ticks +``` + +计算；分子、分母都是 1 ns/tick 的逐核 SYS_CNT core-time，不能改用 +96 核整体 `submit_span_us`，也不能平均 96 个逐核百分比。 +完整 Submit 的组内 miss rate 才按 `sum(misses)/sum(requests)` 计算，不平均 +逐核百分比；局部 lower miss/lower request 之比只是 observed read-clear +ratio，不是实际 miss rate 的数学下界。更完整的 I-cache 采集、分析、估算与排错见 +[`../icache_miss_usage_guide.md`](../icache_miss_usage_guide.md)。 + +#### 历史 PMU 校准资料（不属于当前两类正式构建） + +以下 `empty/scalar/scalar-double/icache-single`、CNT8 fix-busy 和 schema-v3 +文字保留为 2026-07-18 观察链路的建设过程与历史数据。当前 +`swimlane` 构建不提供 PMU，当前 `submit-pmu` 也只接受完整 +Submit 的 `none|claim|efdrain|materialize|register`；不应继续照抄下文的历史校准命令作为当前用法。 + +CCEC 后端提供与泳道分离的 PMU sidecar。正式取数由本目录自带的 Main AICPU +Path-A owner 配置 selector、保存并恢复 PMU 状态；kernel 在每个物理子核内门控并 +读取 `CNT_TOTAL` 和 `CNT0..8`，再写入该 worker 独占的 `WorkerResult`。该链路不需要 +目录外探针或整任务 profiler 的计数结果。完整字段为： + +| sidecar 字段 | 寄存器 / selector | 原始事件含义 | +| ------------ | ----------------- | ------------ | +| `total_cycles` | 64-bit PMU total | gate 窗口的 PMU total 原始计数 | +| `vector_busy` | `CNT0 = 0x501` | Vector pipe busy | +| `cube_busy` | `CNT1 = 0x301` | Cube pipe busy | +| `scalar_busy` | `CNT2 = 0x001` | Scalar pipe busy | +| `mte1_busy` | `CNT3 = 0x701` | MTE1 pipe busy | +| `mte2_busy` | `CNT4 = 0x202` | MTE2 pipe busy | +| `mte3_busy` | `CNT5 = 0x203` | MTE3 pipe busy | +| `icache_requests` | `CNT6 = 0x034` | I-cache request | +| `icache_misses` | `CNT7 = 0x035` | I-cache miss | +| `fix_busy` | `CNT8 = 0x714` | Fix pipe busy | + +`icache-single` 还在同一 worker 的诊断 sidecar 中保留下列配对字段；其他窗口 +将它们写零，不应解读为额外的 Submit 计数： + +| `WorkerResult` 原始字段 | 来源 | 含义 | +| ----------------------- | ---- | ---- | +| `pmu_window_ticks` | 1 GHz sys counter | cold 目标调用窗口，1 tick = 1 ns | +| `pmu_warm_total_cycles` | 64-bit PMU total | 同核 warm 对照窗口原始 total | +| `pmu_warm_window_ticks` | 1 GHz sys counter | warm 目标调用窗口 | +| `pmu_warm_icache_requests` | `CNT6 = 0x034` | 同核 warm 对照 request | +| `pmu_warm_icache_misses` | `CNT7 = 0x035` | 同核 warm 对照 miss | + +#### 历史：Main AICPU Path-A owner + +owner 已自包含在 `ccec/`：构建会同时产出 dispatcher 和 owner AArch64 SO。host +通过 CANN 9.1 已验证的 Main AICPU Path-A 完成 bootstrap 和 mode-0 注册，运行时 +调用 `simpler_aicpu_exec` 执行 Configure/Restore；不要求用户另行启动 PMU 配置进程。 +owner 对本轮会话独占的 PMU 状态先保存、再配置并读回，结束时只按成功 bitmap 从 +107 到 0 逆序恢复。owner 当前没有跨进程互斥锁；同一设备上不得并发运行 +另一个 PMU owner 会话或 `msprof` PMU 会话，否则 selector 和保存态可能互相覆盖， +本轮 Restore 也不再能代表恢复了启动前的状态。 + +host 对同一 stream 调用 `aclrtGetStreamResLimit`，当前 A5 实报 +`AIC=32/AIV=64/total=96`。owner 扫描 108 个物理 MMIO slot，只对完整读回一致的 +96 个置 bitmap，跳过 12 个不可配置 slot；32 个可用物理组都必须形成 +`1 AIC + 2 AIV` 完整 triplet。当前上板 bitmap 为： + +```text +000003ff:fff3ff7f:f7cffcff:fffdf7ff +``` + +kernel 用真实 `get_coreid()` 查 host 通过 `halResMap` 建立的 MMIO 表，不用逻辑 +`worker_id` 猜物理核。正式结果必须同时满足：九个 selector 全部匹配、96 条记录 +可信且物理核 id 唯一、worker slot/role 与物理 triplet 对应、96 个核都实际执行过 +start/stop、`miss <= request`，以及 owner Restore 成功。 + +`off` 是默认值，不建立 PMU owner 会话。五种非 off 模式中，前四种只用于 +观察链路校准；`submit-all` 是唯一正式的 Submit 取数窗口： + +| `--pmu-window` | 位置与窗口边界 | 用途 | +| -------------- | -------------- | ---- | +| `empty` | `RunScheduler` 完成后，baseline read-clear → start → stop → 末尾 snapshot | 量一段空 gate 底噪 | +| `scalar` | `RunScheduler` 完成后，一个 gate 中执行 `--pmu-scalar-nops N` | 验证 scalar/I-cache 正向敏感性 | +| `scalar-double` | `RunScheduler` 完成后，两段相同 NOP 中间只 stop/start、不读 counter | 验证暂停后继续累计 | +| `icache-single` | `RunScheduler` 完成后，每 worker 做 cold/warm 配对试验；64 KiB sweep 或目标预热在目标 gate 外 | 标定受控单次 CNT7 miss 的一阶等效时间 | +| `submit-all` | 每 worker 通过启动屏障后，在首次 PA 参数构造前 start，最后一次 Submit 返回后 stop | 累计 orchestration、Submit，以及本 worker 在窗口内执行的 NOP 或真实引擎计算 | + +`submit-all` 不含启动屏障，也不含 `replay_done`、最终 drain、末尾 ClockBaseline +或 PMU 结果发布。它是 **每 worker 从 orchestration 初始化前到本核最后一次 Submit +返回后的累计窗口**，不是全局“最早 `Submit.begin` 到最晚 `Submit.end`”墙钟 span。 + +每次 `metrics_prof_start/stop()` 都带 `PIPE_ALL` 流水屏障。该屏障用于明确门控边界， +也会收口流水并可能改变多核到达时序；因此 PMU 样本只能与相同 gate、相同负载模式、 +相同构建配置的样本比较，不能把它当成无观察开销的端到端基线。`scalar-nop` 的循环 +实际在 scalar 上执行并计入 scalar 事件；CCEC `real-compute` 则真实激活 Cube/Vector +及其搬运流水。两种口径不能混比，且 standalone sidecar 都不声称是完整真实 PA profile。 + +`icache-single` 的可执行标定命令为： + +```bash +./run.sh run ccec \ + --device 0 --batches 1 --runs 1 --nop-count 0 --no-swimlane \ + --pmu-window icache-single --pmu-icache-trials 64 +``` + +该命令用于校准并输出控制台结果，不生成正式 `submit-all` JSON。除通用 +PMU/owner 断言外，每轮还必须看到： + +```text +icache_pairs=96/96 calibrated_cores=96/96 +[ASSERT] each cold trial adds exactly one CNT7 I-cache miss PASS +``` + +门禁逐核要求 cold 的 `CNT7 == trials`、warm 的 `CNT7 == 0`，并要求 cold-warm +时间差为正。也就是说，最终系数的分母不是推测的循环次数，而是严格闭合的 +`CNT7` miss 差值。 + +#### 历史：生成旧 PMU-only JSON + +正式 sidecar 使用单轮、独立进程和唯一输出路径： + +```bash +./run.sh build ccec + +OUT="./outputs/pmu_submit_all_$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p "$OUT" +./run.sh run ccec \ + --device 0 --batches 256 --runs 1 --no-swimlane \ + --pmu-window submit-all \ + --pmu-json "$OUT/pmu_submit_all.json" +``` + +上述命令没有选择 winner 模式，采集的是当前默认 `real-compute` 和 +`6,28,4,1`。为了让 PMU 取证参数自描述，正式样本仍建议显式写出模式和次数； +例如已用于 count 倍增取证的 b8 命令： + +```bash +./run.sh run ccec \ + --device 0 --batches 8 --runs 1 --no-swimlane \ + --winner-workload real-compute --real-compute-count 1 \ + --pmu-window submit-all --pmu-json "$OUT/pmu_real_b8_count1.json" +``` + +`--pmu-json` 只允许 CCEC 的非 off 窗口并强制 `--runs 1`。正式 JSON 还要求 +`--no-swimlane`，且不能同时开启 `--trace-atomics`、`--profile-phases`、 +`--analyze-swimlane` 或 `--swimlane-json`。已有目标文件或同名 `.tmp` 会被拒绝， +不会静默覆盖。host 只在协议、96 核 PMU/owner 门禁、Restore 和 runtime 清理全部 +成功后，才把临时文件原子发布为最终 JSON。 + +`--no-swimlane` 会关闭 phase/atomic record 写和泳道后处理，但当前普通调度路径中的 +阶段 `SYS_CNT` 调用点仍存在；JSON 会显式记录“有 phase timestamp call、无 phase +record write、无 atomic trace、无 profile accumulation”。因此该模式是 PMU-only +采集口径，不等于编译期删除所有时间戳指令的零观察二进制。 + +JSON 包含： + +- `capture/configuration/validation`：采集边界、winner mode/count/unit/角色引擎映射、 + `PIPE_ALL` 门控语义、九个 selector、计数器位宽、实际 start/stop、96 条记录可信性 + 及可编程 counter 风险门槛；real-compute 还记录数值输出与 placement/引擎闭合门禁； +- `owner`：Main AICPU Path-A、配置 bitmap、AIC/AIV 数量、完整 triplet 和 Restore + 结果； +- `records`：每 worker 的物理子核 id、role、block/lane、原始 CNT0..8 和 total， + 以及本核 owner/slot/role、window started/stopped 断言； +- `summary.all/aic/aiv`：全部 96 核、32 个 AIC 和 64 个 AIV 分组后，各原始计数的 + `sum/mean/median/p95/max`。 + +控制台对应输出 `[PMU-ALL]`、`[PMU-AIC]`、`[PMU-AIV]`。I-cache miss rate 始终按 +`sum(icache_misses) / sum(icache_requests)` 计算，不平均逐核百分比。AIC 与 AIV +核数不同，比较每核强度时应看 mean/median 或 miss rate，不能直接比较两组 sum。 +`total_cycles` 是 PMU total 的原始值；96 核求和是 core-work，不是 Submit 墙钟 +时间。本机现已通过 PMU/SYS_CNT 同窗校准核实其频率，HTML 可按 +ALL/AIC/AIV 的 `1.649844/1.650062/1.649731 cycles/ns` 显示每核 +cycle-equivalent；该换算仍不能把 96 核 core-work 冒充墙钟。CNT0..CNT8 是 +32 bit，total 是 64 bit。正式门禁要求本轮最大可编程 counter 小于 `UINT32_MAX/4` +（25% 高水位），这只是缩短窗口后采用的保守风险阈值；最终 32-bit 值无法证明 +计数器没有恰好回卷一圈或多圈，因此通过该门禁也不能声称“已证明无回卷”。 + +正式观察保留三类互不混用的样本：关闭所有诊断的性能 golden、PMU-only +`submit-all` sidecar、Atomic-trace-only 泳道。PMU sidecar 只有整个窗口的每核累计， +没有可与单条 Atomic span 对齐的子窗口；不能把 AIC/AIV 平均 miss rate 回填成 +泳道中某条 atomic 的属性。优化前后需保持相同源码观察布局、winner mode/count 和 +owner 配置， +并用多个独立进程交错 A/B。 + +#### 历史：2026-07-18 上板验收样本 + +自包含 owner 的本次验收中，`empty`、`scalar 100,000` 和 +`scalar-double 2×100,000` 的 96 核 total 中位数分别约为 214、56,568 和 +112,994；三个样本均为 `96/96 trusted`，owner 为 32 AIC + 64 AIV、32 个完整 triplet， +且每个样本 Restore PASS。它们是各模式的一次上板样本，不是多轮稳定性统计；只用于 +确认空窗底噪、scalar 正向响应和双段近似倍增。 + +同一版本的 `batches=1,nop-count=0,submit-all` 单次样本也通过 96 核 start/stop、 +selector、owner/slot/role/triplet 与 Restore 门禁：all/AIC/AIV 的 total 中位数约为 +36,066.5/28,708/39,745，96 核 scalar busy 求和约 2,661,612，I-cache request/miss +求和为 210,399/30,283（约 14.3931%），host `submit_span_us` 约 47.770。该样本只 +证明 `submit-all` 观察闭环可运行；`batches=1`、零模拟计算体和单次运行都不足以 +支持 256 batch 性能归因或真实 PA 绝对结论。 + +按当时的正式命令和当时默认 PA NOP，还完成了 3 个独立进程的 256 batch +PMU-only 验收。该段保留的是切换默认模式之前的历史样本。Submit span 为 +3,688.236/4,089.057/4,673.237 us，中位数 +4,089.057 us；I-cache request 总和为 69,812,583/69,451,706/70,065,443，miss +总和为 5,854,421/5,847,256/5,830,645，miss rate 为 8.3859%/8.4192%/8.3217%。 +三轮均通过 96 核、owner/slot/role/triplet、start/stop、counter 风险门槛和 Restore， +并使用本用户 `.venv` 从 raw 记录独立重算 summary 一致。这三轮证明的是 +同配置 PMU 取数可重复；由于 gate 包含 `PIPE_ALL` 且未与无 PMU 样本交错配对， +不应将它们与约 5 ms 无诊断基线直接相减。 + +#### 历史：单次 CNT7 I-cache miss 的 scalar 一阶估算标尺 + +2026-07-18 的 96 核并发 cold/warm 配对中，15/15 轮均精确满足 +`cold CNT7 == trials`、`warm CNT7 == 0`和 `calibrated_cores=96/96`。按每轮 +`sum(cold_ticks-warm_ticks) / sum(cold_miss-warm_miss)` 计算，实测为： + +| 规模 | ALL median（range） | AIC median（range） | AIV median（range） | +| ---- | ------------------: | ------------------: | ------------------: | +| 64 trials/core × 10 | 86.596（86.532～86.792）ns/miss | 85.913（85.848～86.202）ns/miss | 86.938（86.861～87.086）ns/miss | +| 128 trials/core × 5 | 89.629（89.615～89.648）ns/miss | 92.100（91.984～92.267）ns/miss | 88.410（88.310～88.440）ns/miss | + +同一时段的 64 与 128 trials，ALL 分别约为 86.6 与 89.6 ns/miss。统一取 90 +只是便于总量级归因的保守取整，不是把两组实测改写成同一精确常数。64-trial 样本中 +AIV 略高，128-trial 样本中则 AIC 更高；角色差值方向并不稳定，不建立 +AIC/AIV 精确常数。只做总量级归因时，统一取整为： + +```text +估算的 scalar I-cache miss 时间(ns) = CNT7_I-cache_miss_total × 90 +估算的 scalar I-cache miss 时间(us) = CNT7_I-cache_miss_total × 0.09 +``` + +例如 1,000 次 miss 约为 90 us，10,000 次约为 0.9 ms。若确实需要按角色 +计算，应使用同一时段、同一运行中打印的 `[ICACHE-FORMULA-AIC/AIV]`， +不能跨时段套用上表的角色中位数。 + +`CNT7` 只报告 I-cache miss 总数，不区分 compulsory、capacity 和 conflict 原因； +三类都应计入上式。本探针用 64 KiB sweep 明确制造 capacity eviction，因此得到的 +是 96 核并发条件下、cold 相对 warm 的一阶等效 miss penalty。它适合回答“这些 +miss 大约能解释多少 scalar 时间”，不是硬件逐次给出的可加 stall；真实代码 +中的预取、miss 重叠、不同下级命中位置和并发排队都会使实际关键路径偏离简单乘积。 + +本机未入库的原始输出保留在： + +```text +tests/atomic_probe/pa_scheduler/outputs/pmu_validation/ + icache_single_64x10_20260718_085929_3232836_console.log + icache_single_128x5_20260718_090151_3235468_console.log +``` + +这里验证的是观察手段，不是 PA 优化本身。`submit-all` 仍是正式调度取数的 +唯一窗口；上述 90 ns/miss 只用于对其 CNT7 总量做一阶等效估算，不把校准 +模式扩展成 Claim、EfDrain 等多个正式局部窗口。性能 A/B 仍要保持观察布局一致， +最终端到端收益以关闭 PMU 和泳道的独立进程结果为准。 + +#### 历史：校验并聚合多轮 PMU sidecar + +`pmu_sidecar_analyzer.py` 在该历史流程中只读消费当时的 schema-v3 JSON。它不信任单轮 host 已写好的 +summary，而是从每份文件的 worker raw 记录重新计算 ALL/AIC/AIV 的 +`sum/mean/median/p95/max` 与 `sum(miss)/sum(request)`；同时检查 accepted、 +96 核 start/stop、物理核唯一性、owner membership、角色、counter 门槛和 Restore。 +任一字段不一致即拒绝整组输入。 + +同一次构建、同一观察参数的多个独立进程可直接聚合： + +```bash +source "$HOME/.venv/bin/activate" +python pmu_sidecar_analyzer.py "$OUT"/run*.json +python pmu_sidecar_analyzer.py --json "$OUT"/run*.json +``` + +工具把 device、batch、AIC/AIV 数、PMU window、selector、trace 配置和完整 +winner workload 纳入配置指纹。`submit-all` 与 `empty`、scalar-NOP 与 +real-compute 等不同口径不能混合聚合。当前 JSON 没有记录 ELF 内容哈希，所以 +调用者仍必须用独立输出目录隔离不同构建，不能只因 kernel 路径字符串相同就认为 +是同一二进制。 + +默认 `--icache-miss-ns 90` 只打印受控 cold/warm 标尺下的一阶 core-work 等效量。 +输出明确标记 `not_wall_or_additive_stall`：96 核总和不是 Submit 墙钟，逐核估算也 +可能因 miss 重叠、事件来源和真实层级差异超过窗口时间，不能据此做绝对减法。 +文本输出的 `[PRIMARY]` 以 AIV 平均 request/miss、AIV 逐核 p95 和组内 miss rate +为主；`[ACTUAL-EXPOSED-LOSS]` 在没有同语义配对 A/B 前固定报告 `UNMEASURED`， +避免把 90 ns 标尺误写成约 5 ms Submit 中已经暴露的损失。 +分析器回归可独立执行： + +```bash +python -m unittest -v \ + test_pmu_sidecar_analyzer.py \ + test_pmu_html_report.py +``` + +## 6. 当前 A5 结果与真实 PA 的差异 + +2026-07-17 `scalar-nop` 阶段的一轮代表性结果如下。这是保留的历史 +基线，不是后续 `real-compute` 数据；当时所有严格校验均为 PASS，kernel +时间使用上表附近的校准 NOP： + +| 实现 | 首轮 `submit_span_us` | EfDrain/RingBp/FinalDrain | +| ---- | --------------------: | ------------------------: | +| 真实 simpler PA 最好泳道 | 5,096.685 us | 1011 / 0 / 13 | +| standalone CCEC，3 个独立进程首轮中位数 | 4,830.184 us | 961 / 50 / 12 | +| standalone AscendC，独立进程首轮 | 4,917.014 us | 1009 / 4 / 11 | + +三次 CCEC 独立进程首轮分别为 4,846.431、4,798.260、4,830.184 us；这里 +报告中位数，不挑最好值。`--runs N` 的后续轮次会复用进程、device binary 和 +已分配内存，而真实 PA 的 5.1 ms 来自完整测试进程的一轮泳道，因此做基线 +比较时应优先比较独立进程首轮，不能把热运行中位数混作同一口径。 + +四个重点阶段的一轮 CCEC 代表值为： + +| role | Claim | EfDrain | WaitForSlot | HeapGuard | +| ---- | ----: | ------: | ----------: | --------: | +| AIC 每 worker 累计中位数 | 470.503 us | 711.005 us | 234.063 us，43 次等待 | 21.349 us，约 1 次等待 | +| AIV 每 worker 累计中位数 | 533.755 us | 401.962 us | 0.067 us，0 次等待 | 3.723 us，约 1 次等待 | + +表中阶段时长来自一轮代表值；括号内的“等待次数”则是三轮中对应角色的 +全局中位数，不是每 worker 次数。Claim 和 EfDrain 已与真实 PA 同量级； +HeapGuard 只有 0 至 3 次偶发等待。当前主要 +残差来自编译边界： +真实 orchestration 和 `dist_submit_impl` 位于不同翻译单元，TaskArgs 对 Submit +编译器是运行时数据；standalone 为了保持可复制构建，共享实现会与固定 PA +任务图一起优化，导致 PrepareMap/Register 等前端阶段的指令生成不同。CCEC +的 AIC 到达顺序还会让约 39 至 54 个 kernel 进入 WaitForSlot/RingBp；AscendC +同轮只有 4 个，说明这部分主要是后端 codegen 触发的时序放大，不是缺少完成协议。 + +已经验证过的强制 `noinline`、拆设备目标文件和全局 compiler memory clobber +分别造成状态破坏、A5 device exception 或明显 RingBp，均未保留。它们不是 +可靠的 PA 语义模拟。当前选择是保持源级协议和 ABI 对等，坦诚记录约 +0.1 至 0.4 ms 的后端/冷热差异，不用虚假 NOP 填平调度阶段。 + +### 6.1 2026-07-18 CCEC 真实计算阶段结果 + +CCEC `real-compute` 已按“构建门禁 → b1 数值 → b256 标定 → PMU 倍增 → +泳道元数据”的顺序上板闭环： + +- 最终 b256 默认 `6,28,4,1` 的 Submit 为 3,683.649 us，全部协议与 + 192 个 active output tile 数值断言通过；此前三个独立进程 Submit 为 + 3,808/3,555/3,706 ms，中位数 3.706 ms； +- b8、四类 count=1 时，Submit 窗口内 29 个 EfDrain 恰好对应 14 个非零 + AIC Cube worker 和 15 个非零 AIV Vector worker；AIC 每个 `cube_busy=8281`， + AIV 每个 `vector_busy=936/937`； +- 独立 count=2 样本的 placement 为 28 个 EfDrain、4 个 FinalDrain，14 个 AIC + 和 14 个 AIV worker 的非零计数档位分别为 16562 与 1872/1874，恰为 count=1 + 档位的两倍。获胜核会随调度变化，不能把两轮强行按同一 worker 配对； +- `submit-all` 明确排除 FinalDrain，所以 count1/count2 分别落在 FinalDrain 的 + 3/4 个 kernel 不应出现在该窗口的引擎 PMU 中。每轮都由 placement 与非零引擎 + worker 数量闭合； +- scalar-NOP 与 real-compute 使用同一个最终 ELF 分别跑 b1，避免把代码布局变化 + 误判为负载效果；最终 ELF 只暴露两个 mixed kernel 全局入口，冷路径 dispatcher + 以及 Cube/add/mul 三个执行 helper 均为非空 LOCAL 函数； +- real-compute b1 泳道 raw/merged 均记录 mode、`6,28,4,1`、完整迭代单位和 + QK/PV=Cube、SF/UP=Vector 的映射；4964 条 raw data event 转换为 4965 条 + data event（多一条 capture instant），最终 `traceEvents` 还含 256 条 + process/thread metadata，共 5221 条，且无 dropped record。 + +这组结果证明 standalone CCEC 的显式 `real-compute` 模式已不再用 scalar NOP +冒充 winner 计算，并能从数值、角色、PMU 和泳道四个方向闭环。这是 +CCEC 后端当时的阶段结论；AscendC 和 CPU 后续已分步补齐，见 6.2 和 6.3 节。 +三后端闭环仍不证明 standalone 的 3～5 ms 应等于真实 PA 的 5.1 ms； +真实引擎并行和代码生成差异都会改变调度时序。 + +### 6.2 2026-07-18 AscendC 真计算验证 + +AscendC 按 CCEC 之后独立接入同一 workspace 和参数规则。AIC 路径使用 +`DataCopy/LoadData/Mmad/FIX` 完成 `128x128` matmul，AIV 路径使用 +`DataCopy/Add/Mul` 完成 elementwise 计算；两条路径都在 task 发布前等待 GM +写回完成。分层上板结果为： + +| 场景 | `submit_span_us` | QK/SF/PV/UP `[KERNEL] mean_us` | 数值输出 | +| ---- | ---------------: | ------------------------------ | -------- | +| b1，count=`1,1,1,1` | 59.280 | 8.837 / 14.267 / 8.342 / 11.307 | 4 active + 188 sentinel，PASS | +| b8，count=`1,1,1,1` | 166.426 | 7.713 / 2.559 / 7.694 / 2.852 | 32 active + 160 sentinel，PASS | +| b256，count=`6,28,4,1` | 3810.471 | 41.232 / 48.047 / 27.940 / 2.528 | 191 active + 1 sentinel，PASS | + +b256 三个独立进程的 Submit span 为 3810.471、4828.567 和 3777.371 us， +中位数为 **3810.471 us**。三轮都通过全部 PA 协议、角色路由和真计算输出 +断言；数据离散也说明独立进程首轮需报告中位数，不能只挑最快值。 + +`[KERNEL] mean_us` 是公共调度器对每个 winner `ExecuteKernel` 前后的 +1 GHz `SYS_CNT` span 求均值；b256 每类均为 256 个样本。它覆盖本 task 的 +GM load、Cube/Vector 计算、GM store、完成等待及少量调用边界，因此是 +**完整 winner 计算 span 均值**，不是纯 Cube/Vector busy counter，也不是 CCEC +PMU 计数。AscendC 当前不伪造 CCEC-only PMU sidecar；本阶段的取证是官方指令 +接口、完整数值 tile、角色路由和上述 span 共同闭环。 + +已生成的 AscendC b1 count=1 泳道位于： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + pa_scheduler_swimlane_20260718_124326_3574886/ascendc/ + l2_swimlane_records.json + merged_swimlane.json +``` + +该轮 raw 有 4652 条 data event，merged 增加一条 capture instant 后为 4653 条 +data event，再加 256 条 process/thread metadata，`traceEvents` 合计 4909 条。 +raw 和 merged 的 `metadata.winner_workload` 均为 `mode=real-compute`、 +`counts=1,1,1,1`、`unit=complete_128x128_engine_pipeline_iteration`，且 +QK/PV、SF、UP 分别映射到 `cube_matmul`、`vector_add`、`vector_mul`。 +泳道轮次的协议、输出、record 数和 `dropped=0` 闭环全部 PASS。 + +### 6.3 2026-07-18 CPU 对等算术回归 + +CPU 在 AscendC 之后补齐了同一 `real-compute` CLI、workspace 编址、输入 +2/3、active tile 结果 768/5/6 和 inactive sentinel 校验。b1 count=1 的 +4 active + 188 sentinel 通过；b8 count=`2,3,2,1` 连续运行两轮，两轮的 +25 active + 167 sentinel 均通过，同时验证了 runs 间输出会重置为 sentinel。 + +CPU 上的 matmul/add/mul 是普通 x86 浮点循环，`[KERNEL]`、`submit_span_us` +和泳道 `·kernel` 都会被 pthread 调度、CPU cache 和编译器影响。它们只用于 +回归公共 PA 控制流与算术闭环，不得当作 A5 Cube/Vector 时间、PMU 或端到端 +性能的估计。CPU 泳道里的 `engine_mapping` 是三后端共用的逻辑 workload +映射标签，不表示 x86 上存在对应物理引擎。 + +### 6.4 2026-07-18 非均匀布局诊断 + +常量 2/3 会掩盖 `B` 转置和分形重排错误，因此在不改变 `constant` 性能输入默认的 +前提下增加了 +`--real-compute-pattern layout-diagnostic`。按 CCEC → AscendC → CPU 的顺序， +三后端均以 b1、count=`1,1,1,1` 运行同一带权对角 A 和非对称 B，并逐元素扫描 +4 个 active tile 与 188 个 inactive sentinel tile，全部 PASS： + +| 后端 | Submit span | QK/SF/PV/UP `[KERNEL] mean_us` | 结论 | +| ---- | ----------: | ------------------------------ | ---- | +| CCEC | 37.682 us | 9.172 / 3.819 / 7.415 / 2.512 | PTO Cube/Vector 数学与布局基准 PASS | +| AscendC | 55.041 us | 8.478 / 4.211 / 20.801 / 3.008 | L1 错位、B 分形转置、ND/NZ 与 FIXPIPE 闭环 PASS | +| CPU | 51.958 ms | 3.457 ms / 26.118 us / 3.390 ms / 17.349 us | host 期望公式与路由回归 PASS；不作 A5 性能数据 | + +AscendC 诊断泳道位于 +`outputs/pa_scheduler_swimlane_20260718_125904_3613100/ascendc/`：raw 为 +4584 条 data event，merged 增加一条 capture instant，`dropped=0`；raw/merged +的 `metadata.winner_workload.input_pattern` 都是 `layout-diagnostic`。CCEC 同模式的 +`submit-all` PMU sidecar 也记录该字段且 `accepted/semantic_passed=true`。 + +诊断只证明单次完整迭代的数学与数据布局闭环；默认常量性能负载仍用于稳定比较， +CCEC 的 repeat 次数证明仍以 count1→count2 的 engine PMU 精确倍增为准。 + +### 6.5 2026-07-18 默认真负载与 5 ms 口径验收 + +三后端重编后，无 workload 参数的 b1 均打印 +`mode=real-compute pattern=constant counts=6,28,4,1`，并通过 96 核、5 个 +fanin/batch、唯一 winner、角色路由、TensorMap/heap/completion 与数值输出断言。 +旧命令只给 `--nop-count 0` 时会显式打印 `mode=scalar-nop`，证明兼容 +分支没有让 NOP override 静默失效。 + +CCEC b256 关闭泳道的 3 个独立进程为 4,411.760/4,297.704/4,677.634 us， +中位数 4,411.760 us；它只用于无观察热路对比。与真实 PA 5.1 ms 比较时， +当时必须同样开启 phase-only 泳道且不开逐 atomic；standalone 5 个独立进程为： + +```text +5002.413 / 4875.193 / 4968.894 / 4992.477 / 4876.282 us +``` + +中位数为 **4,968.894 us**。真实 PA 最终三轮为 +5,115.620/5,145.057/5,096.685 us，中位数 **5,115.620 us**；同口径差值 +146.726 us，约 2.87%，已满足独立调度复现目标。五轮 standalone 的 +QK/SF/PV/UP 每 task 均值中位数为 41.461/54.007/28.053/2.649 us， +与真实 44.170/53.729/27.626/1.565 us 的总 core work 接近，不再调整 +repeat 追求逐微秒一致。 + +只保留一轮历史 phase-only 泳道原始证据： + +```text +outputs/performance_gap_20260718/standalone_ccec_real_b256_raw.json +``` + +该轮有 863,237 条记录、`dropped=0`，比真实 PA 的 863,232 条只多 5 条 +RingBp；两端的 122,880 个 Submit 与各前端阶段、1,024 个 Kernel/Fanin/Build +数量一致。这证明总体性能已接近，不等于真实 PA 数值数据流、代码生成与通用 +多 group/joint 调度已完全相同。该文件用于保留历史 5 ms 同口径性能证据， +不能冒充当前 `swimlane` action 的合并记录；当前 action 还会同时加入 Atomic 与 +ClockBaseline，并继续以逐核容量、调用数、总记录数和 `dropped=0` 闭环。 + +## 7. 内存占用和脱仓复制 + +为保持真实 DistGlobal/DistCore 偏移、65,536 个 task cell、每 worker payload +和 TensorMap，非 split 的 CPU 构建中 `WorkerResult` 为 896 bytes、 +`SchedulerState` 为 1,007,115,968 bytes。CCEC swimlane 以及使用 split-finish +的 submit-PMU 构建为了发布跨 TU 正确性诊断，`WorkerResult` 为 +960 bytes、`SchedulerState` 为 1,007,122,112 bytes，增量精确为 +`64 * 96 = 6,144` bytes。split ELF 另预留 AIC/AIV 两个 role-specific +block-local runtime state，每个精确 1,664 bytes、最终 section 合计 +3,328 bytes；它们不属于 GM `SchedulerState`。以上 `SchedulerState` 数字 +是 private 模式。R4c 的 shared sidecar 历史大小为 12,420,288 bytes； +R4e-a 追加 96 条 reader-progress cache line 后，generation-8 sidecar 为 +12,426,432 bytes。R5c 在尾部追加七条 insert-turn cache line，形成历史 +generation-9 的 12,426,880 bytes；generation-10 将该尾数组扩为 +127 条 extra line，sidecar 为 12,434,560 bytes，当前 generation-11 +保留同一物理布局但不再从热路径访问这些 turn line。shared batch 输入 +数组扩到 512 后，CPU non-split 与定义 +`PA_COMPETE_FIRST_SPLIT_FINISH` 的 CCEC 变体总大小分别为 +1,019,551,552/1,019,557,696 bytes；swimlane、perf-clock 以及 submit-PMU +none/claim/efdrain 使用后者，submit-PMU materialize/register 和独立 +shared-protocol-litmus 使用 non-split 大小。既有 production prefix 和 +`WorkerState`/`RunConfig` offset 不变；`context_lens` 后的 standalone +字段因数组扩容顺延 1,024B。S4.15a/S4.16 历史候选都曾得到 +4,736,704B，但末尾 512B 分别是 Cube cursor 和 +`shared_vector_cursor[8..15]`，且均已撤销。S4.9 的 +4,735,680B、历史 S2.5 的 2,113,664B 和 S3.1 的 4,735,104B 也都 +不是当前 shared 构建的传输或分配口径。 + +S4.16a 的 host/device `sizeof(SchedulerState)` 握手、manifest 校验和 +相对重建 `319077a9` 的正式 b256 六区组配对已经完成。不能因历史候选 +字节数碰巧相同就混用产物；冻结件及配对路径见 +`shared_tensormap_record.md`。S4.16b 沿用相同历史大小并闭合正确性， +但第一层性能门槛失败；当前源码保留 S4.14b 的八分片热路径，同时在尾部 +纳入 S6.3 的 output table 扩容、R4c history 和 R4e-a +`reader_done[96]`，不能再把当前传输长度写成“恢复 S4.14b”。 + +独立的 64 bytes PMU 配置和 64 bytes winner workload 配置各占一条 +cache line；二者都位于完整生产 DistGlobal 镜像之后，生产 DistGlobal/ +DistCore 关键偏移保持不变。 +默认泳道缓冲区另占 402,660,160 bytes；CCEC/AscendC `real-compute` 还在 device 分配 +12,713,984 bytes workspace。因此 scalar-nop+trace 的 A5 device 占用约 +1.313 GiB，real-compute+trace 约 1.325 GiB，host 侧也需分配相近内存。 +CPU 后端只在 host 侧分配相同 workspace，不存在 device 内存口径。 +`smoke` 不缩小 State；只有 `--no-swimlane` 能省去泳道缓冲区。 +256 batch 的历史 phase-only 采集约有 86.3 万条事件。删除 +`LoserReplay` 过程态后，当前 schema-v4 level-4 b256 规模门禁为 +845,813 条 raw 事件、56,212,672 bytes；同一 raw 使用旧 merged +字段布局为 248,767,986 bytes，使用当前六字段 duration 布局为 +138,349,686 bytes，减少 44.4%。按 96 核聚合而不复制每个 gap +属性的排他报告约 117 KiB。文件瘦身不改变 raw 记录写入数， +也不改变固定 trace buffer 分配；三者是独立口径。 +runner、converter 与 analyzer 都使用临时文件后原子替换自己的目标，失败时 +不会把半截文件冒充完整产物。 + +脱离 simpler 时必须复制整个目录，因为三个后端共用 `common/`： + +```bash +cp -a tests/atomic_probe/pa_scheduler /tmp/pa_scheduler +cd /tmp/pa_scheduler +./run.sh build cpu +./run.sh smoke cpu +``` + +上面两条省略 `--tensormap private`，与显式写出 private 等价。shared +模式可在复制目录后显式执行 +`./run.sh build cpu --tensormap shared`；构建身份和产物目录会保持 shared， +不会回退到 private。 + +shared insert turn 默认使用 G=1。需要构建交错候选时，通过构建环境变量 +选择 G=2/4/8/16/32/64/128；该值同时进入 host/kernel build identity +和 CCEC manifest，不能在同一组产物上运行期切换： + +```bash +PA_SHARED_INSERT_TURN_GROUPS=32 \ + ./run.sh build cpu --tensormap shared + +PA_SHARED_INSERT_TURN_GROUPS=128 \ + ./run.sh build ccec --tensormap shared + +PA_SHARED_INSERT_TURN_GROUPS=128 \ + ./run.sh perf-clock ccec --tensormap shared --batches 512 +``` + +这里的 PA-G 表示一个 batch 的 PA block-group 数；turn-G 表示 insert-turn +使用的物理控制线数。二者含义独立。turn-G 只改变同一枚有序 token 的物理 +落点，不建立多条独立 writer 插入链。 +本阶段 turn-G>1 只维护 CPU 与 CCEC；`run.sh` 会在任何构建、文件创建 +或设备动作前拒绝 AscendC 和 `all`。turn-G1 保留既有后端命令兼容性，但 +不把 AscendC 结果列入本阶段验证证据。 +standalone 各 G 顺序复用同一 variant 输出目录，因此比较时必须先构建目标 +G，再以相同环境变量消费该次 manifest 对应的 CCEC 产物；省略或写错 G +会在启动设备前被拒绝。 +CPU 的 turn-G1 保留默认文件名 `pa_scheduler_cpu`；turn-G>1 生成 +`pa_scheduler_cpu_turn_g`，`run.sh` 按环境变量选择，避免后一次构建 +覆盖前一 G 后仍把旧命令结果误认成目标配置。 + +CPU 结果只作为语义与并发正确性证据；A5 性能必须使用无诊断 +`perf-clock`。2026-07-28 已完成 B256 六轮以及同 artifact 的 B256/B512 +四轮交错矩阵；实测数据、artifact hash 和未安装 `task-submit`/`npu-smi` +的执行边界见 `shared_tensormap_record.md`。 + +CCEC/AscendC 只需再 source CANN 环境。本目录的构建脚本不会搜索 Git 根目录， +也不会引用 `simpler/src`、`simpler/examples` 或其他仓内文件。泳道转换与排他 +分析都只需 Python 3 标准库；复制后的 `./run.sh swimlane ...` 仍使用当前目录内的 +`swimlane_converter.py` 和 `swimlane_exclusive_analyzer.py`。 diff --git a/tests/atomic_probe/pa_scheduler/ascendc/build.sh b/tests/atomic_probe/pa_scheduler/ascendc/build.sh new file mode 100755 index 0000000000..d6a411f460 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ascendc/build.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +if [[ $# -gt 1 ]]; then + echo "Usage: $0 [private|shared]" >&2 + exit 1 +fi +TENSORMAP_MODE="${1:-private}" +case "$TENSORMAP_MODE" in + private) TENSORMAP_MODE_ID=0 ;; + shared) TENSORMAP_MODE_ID=1 ;; + *) + echo "Unknown TensorMap mode: $TENSORMAP_MODE (expected private|shared)" >&2 + exit 1 + ;; +esac +BUILD_DIR="$ROOT_DIR/build/ascendc/$TENSORMAP_MODE/swimlane" + +# 所有源码和公共头都从 pa_scheduler 目录解析；外部只需要用户安装的 +# CANN 工具链和运行库，不搜索 simpler 仓库根目录。 + +if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then + # 非交互 shell 不保证自动 source CANN 环境，缺失时直接失败，避免误用 + # PATH 中其他版本的 bisheng。 + echo "ASCEND_HOME_PATH is not set; source the CANN 9.1 set_env.sh first." >&2 + exit 1 +fi + +BISHENG="$ASCEND_HOME_PATH/bin/bisheng" +if [[ ! -x "$BISHENG" ]]; then + # 只接受当前 ASCEND_HOME_PATH 下的编译器，确保构建与运行库来自同一套 CANN。 + echo "bisheng is missing under ASCEND_HOME_PATH=$ASCEND_HOME_PATH" >&2 + exit 1 +fi + +# build 产物固定放回 pa_scheduler/build/ascendc；目录可重复创建，重新构建会 +# 原位替换可执行文件，不向源码目录散落中间产物。 +mkdir -p "$BUILD_DIR" + +echo "[BUILD] AscendC 1:2 mixed host + kernel executable" +# -xasc 会把本文件中的 host main 与 AscendC kernel 构建为一个可执行文件。 +# __MIX_CORE_AIC_RATION__ 的拼写来自 bisheng mixed-core ABI；值 2 必须与 +# kernel 上的 __mix__(1, 2) 一致，否则 launch metadata 不能表达 32+64 拓扑。 +# 调度器已经在协议边界显式执行 dcci，关闭自动 scalar DCCI 可避免编译器 +# 额外插入 cache 操作并扰动待测的 atomic/Submit 时序。 +"$BISHENG" -O3 -xasc \ + "$SCRIPT_DIR/pa_scheduler.asc" \ + --npu-arch=dav-3510 \ + "-DPTO_FDWIC_SHARED_MAP=$TENSORMAP_MODE_ID" \ + -D__MIX_CORE_AIC_RATION__=2 \ + -I"$ROOT_DIR/common" \ + -mllvm -cce-aicore-dcci-insert-for-scalar=false \ + -mllvm -cce-aicore-dcci-before-kernel-end=false \ + -o "$BUILD_DIR/pa_scheduler_ascendc" + +echo "[BUILD] complete: $BUILD_DIR/pa_scheduler_ascendc" diff --git a/tests/atomic_probe/pa_scheduler/ascendc/pa_scheduler.asc b/tests/atomic_probe/pa_scheduler/ascendc/pa_scheduler.asc new file mode 100644 index 0000000000..efe0b26190 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ascendc/pa_scheduler.asc @@ -0,0 +1,752 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "../common/host_support.h" +#include "../common/winner_workload_host.h" + +#include "acl/acl.h" +#include "kernel_operator.h" + +#include +#include +#include +#include +#include + +using namespace AscendC; + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_GM __gm__ +// 通过两个宏把公共调度器实例化为 AscendC 设备代码；公共头本身不依赖 +// kernel_operator.h，也不会为 AscendC 复制一套调度协议。 +#include "../common/pa_scheduler_core.h" + +namespace { + +template +__aicore__ inline void EmitNops() { +// Count 必须是编译期常量，展开后才能稳定保留对应数量的 AICore nop 指令。 +#pragma unroll + for (uint32_t index = 0; index < Count; ++index) { + asm volatile("nop"); + } +} + +__aicore__ inline void RuntimeNop(uint32_t count) { + // AscendC 的 Nop() 不能接收运行时参数，因此先按 256 指令分块， + // 再用二进制尾块拼出任意 count。两端 PIPE_ALL 屏障把模拟计算体与 + // 前后调度访存隔开；count 是本机校准量，不等同于硬件 cycle 数。 + PipeBarrier(); + while (count >= 256) { + EmitNops<256>(); + count -= 256; + } + if ((count & 128U) != 0) EmitNops<128>(); + if ((count & 64U) != 0) EmitNops<64>(); + if ((count & 32U) != 0) EmitNops<32>(); + if ((count & 16U) != 0) EmitNops<16>(); + if ((count & 8U) != 0) EmitNops<8>(); + if ((count & 4U) != 0) EmitNops<4>(); + if ((count & 2U) != 0) EmitNops<2>(); + if ((count & 1U) != 0) EmitNops<1>(); + PipeBarrier(); +} + +#if defined(__DAV_VEC__) +template +__aicore__ __attribute__((noinline)) void RunRealVectorWorkload( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + using namespace pa_scheduler::winner_workload; + // 三个 64 KiB tile 固定占用 UB 的 [0, 192 KiB)，与 CCEC 的 tile 尺寸 + // 完全一致；不构造 TPipe，避免 winner 热路径增加队列初始化/析构开销。 + LocalTensor input_a_local(TPosition::VECCALC, 0x00000, kTileElements); + LocalTensor input_b_local(TPosition::VECCALC, 0x10000, kTileElements); + LocalTensor output_local(TPosition::VECCALC, 0x20000, kTileElements); + GlobalTensor input_a_global; + GlobalTensor input_b_global; + GlobalTensor output_global; + input_a_global.SetGlobalBuffer(input_a, kTileElements); + input_b_global.SetGlobalBuffer(input_b, kTileElements); + output_global.SetGlobalBuffer(output, kTileElements); + const DataCopyParams copy_params{ + 1, static_cast(kTileBytes / 32), 0, 0 + }; + + for (uint32_t iteration = 0; iteration < repeats; ++iteration) { + DataCopy(input_a_local, input_a_global, copy_params); + DataCopy(input_b_local, input_b_global, copy_params); + SetFlag(EVENT_ID0); + WaitFlag(EVENT_ID0); + if constexpr (Multiply) { + Mul(output_local, input_a_local, input_b_local, static_cast(kTileElements)); + } else { + Add(output_local, input_a_local, input_b_local, static_cast(kTileElements)); + } + SetFlag(EVENT_ID0); + WaitFlag(EVENT_ID0); + DataCopy(output_global, output_local, copy_params); + // 返回前等待本轮 MTE3 真正完成 GM 写回；否则 Kernel span 只会包住发射。 + SetFlag(EVENT_ID7); + WaitFlag(EVENT_ID7); + } +} + +__aicore__ __attribute__((noinline)) void RealVectorAdd( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + RunRealVectorWorkload(input_a, input_b, output, repeats); +} + +__aicore__ __attribute__((noinline)) void RealVectorMul( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + RunRealVectorWorkload(input_a, input_b, output, repeats); +} +#else +__aicore__ __attribute__((noinline)) void RealCubeMatmul( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + using namespace pa_scheduler::winner_workload; + constexpr uint16_t kMFractals = kTileRows / 16; + constexpr uint16_t kKFractals = kTileCols / 8; + // CANN 9.1 arch35 Cube 示例把 unitFlag=3 定义为最终累加并触发最终写回。 + constexpr uint8_t kFinalAccumulation = 3; + // A1 与 B1 都映射到同一块物理 L1，不能像 L0A/L0B 那样都从地址 0 + // 开始。两块 64 KiB tile 显式错开，否则第二次搬入 B 会覆盖 A， + // 常量输入下会把 2*3*128 错算成 3*3*128。 + LocalTensor input_a_l1(TPosition::A1, 0, kTileElements); + LocalTensor input_b_l1(TPosition::B1, kTileBytes, kTileElements); + LocalTensor input_a_l0(TPosition::A2, 0, kTileElements); + LocalTensor input_b_l0(TPosition::B2, 0, kTileElements); + LocalTensor output_l0(TPosition::CO1, 0, kTileElements); + GlobalTensor input_a_global; + GlobalTensor input_b_global; + GlobalTensor output_global; + input_a_global.SetGlobalBuffer(input_a, kTileElements); + input_b_global.SetGlobalBuffer(input_b, kTileElements); + output_global.SetGlobalBuffer(output, kTileElements); + + Nd2NzParams nd_to_nz; + nd_to_nz.ndNum = 1; + nd_to_nz.nValue = kTileRows; + nd_to_nz.dValue = kTileCols; + nd_to_nz.srcNdMatrixStride = 0; + nd_to_nz.srcDValue = kTileCols; + nd_to_nz.dstNzC0Stride = kTileRows; + nd_to_nz.dstNzNStride = 1; + nd_to_nz.dstNzMatrixStride = 0; + LoadData2DParamsV2 a_l1_to_l0; + a_l1_to_l0.mStartPosition = 0; + a_l1_to_l0.kStartPosition = 0; + a_l1_to_l0.mStep = kMFractals; + a_l1_to_l0.kStep = kKFractals; + a_l1_to_l0.srcStride = kMFractals; + a_l1_to_l0.dstStride = kMFractals; + a_l1_to_l0.ifTranspose = false; + // GM 中的 B 是普通 KxN row-major。CANN 9.1 的 arch35 Matmul + // LoadDataToL0B 对这种右矩阵要求在 zN(L1)->nZ(L0B) 时转置分形； + // 常量 3 无法暴露这项差异，因此不能与 A 复用同一个 false 参数。 + LoadData2DParamsV2 b_l1_to_l0 = a_l1_to_l0; + b_l1_to_l0.ifTranspose = true; + MmadParams matmul; + matmul.m = kTileRows; + matmul.n = kTileCols; + matmul.k = kTileCols; + matmul.cmatrixInitVal = true; + matmul.cmatrixSource = false; + matmul.disableGemv = true; + matmul.unitFlag = kFinalAccumulation; + DataCopyCO12DstParams copy_out; + copy_out.nSize = kTileCols; + copy_out.mSize = kTileRows; + copy_out.dstStride = kTileCols; + copy_out.srcStride = kTileRows; + copy_out.quantPre = QuantMode_t::NoQuant; + copy_out.nz2ndEn = true; + copy_out.unitFlag = kFinalAccumulation; + + for (uint32_t iteration = 0; iteration < repeats; ++iteration) { + DataCopy(input_a_l1, input_a_global, nd_to_nz); + DataCopy(input_b_l1, input_b_global, nd_to_nz); + SetFlag(EVENT_ID0); + WaitFlag(EVENT_ID0); + LoadData(input_a_l0, input_a_l1, a_l1_to_l0); + LoadData(input_b_l0, input_b_l1, b_l1_to_l0); + SetFlag(EVENT_ID0); + WaitFlag(EVENT_ID0); + Mmad(output_l0, input_a_l0, input_b_l0, matmul); + SetFlag(EVENT_ID0); + WaitFlag(EVENT_ID0); + SetFixpipeNz2ndFlag(1, 1, 1); + DataCopy(output_global, output_l0, copy_out); + // FIX_S 是 task 完成边界；没有它会把 Cube 结果尚未写回 GM 的时间漏出 span。 + SetFlag(EVENT_ID7); + WaitFlag(EVENT_ID7); + } +} +#endif + +__aicore__ __attribute__((noinline)) void ExecuteRealWinnerWorkload( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState &worker, + pa_scheduler::TaskKind kind +) { + using namespace pa_scheduler::winner_workload; + const uint64_t workspace = state->winner_workload.workspace_base; + const uint32_t repeats = pa_scheduler::WorkloadCountForKind(state->winner_workload.repeats, kind); + if (state->winner_workload.version != pa_scheduler::kWinnerWorkloadConfigVersion || + workspace == 0 || state->winner_workload.workspace_bytes < kWorkspaceBytes || + worker.core_idx < 0 || static_cast(worker.core_idx) >= pa_scheduler::kWorkers || + repeats == 0 || repeats > kMaxRealComputeCount) { + return; + } +#if defined(__DAV_VEC__) + if (kind != pa_scheduler::TaskKind::Sf && kind != pa_scheduler::TaskKind::Up) return; +#else + if (kind != pa_scheduler::TaskKind::Qk && kind != pa_scheduler::TaskKind::Pv) return; +#endif + __gm__ float *input_a = reinterpret_cast<__gm__ float *>(workspace); + __gm__ float *input_b = reinterpret_cast<__gm__ float *>(workspace + kTileBytes); + const uint32_t kind_slot = + (kind == pa_scheduler::TaskKind::Pv || kind == pa_scheduler::TaskKind::Up) ? 1U : 0U; + const uint32_t output_tile = kSharedInputTiles + + static_cast(worker.core_idx) * kOutputTilesPerWorker + kind_slot; + __gm__ float *output = reinterpret_cast<__gm__ float *>( + workspace + static_cast(output_tile) * kTileBytes + ); +#if defined(__DAV_VEC__) + if (kind == pa_scheduler::TaskKind::Sf) { + RealVectorAdd(input_a, input_b, output, repeats); + } else { + RealVectorMul(input_a, input_b, output, repeats); + } +#else + RealCubeMatmul(input_a, input_b, output, repeats); +#endif +} + +struct AscendcOps { + // AscendC 路径尚未完成同构机器码核验，因此保留源码括号并在 raw flags 中 + // 明确标记为未建立 return-ready 边界。 + static constexpr bool kAtomicReturnReadyObserved = false; + + // 公共调度器把 Load 定义为“具有原子一致性的读”。这里坚持使用 + // AtomicAdd(addr, 0)，以保留真实 PA 在热点 cache line 上的竞争形态。 + __aicore__ static inline int32_t Load(__gm__ volatile int32_t *address) { + return AtomicAdd(const_cast<__gm__ int32_t *>(address), static_cast(0)); + } + + __aicore__ static inline int64_t Load(__gm__ volatile int64_t *address) { + return AtomicAdd(const_cast<__gm__ int64_t *>(address), static_cast(0)); + } + + __aicore__ static inline uint64_t Load(__gm__ volatile uint64_t *address) { + // CANN 9.1 虽接受 AtomicAdd，但该重载在本 mixed kernel + // 第一次跨过 64 MiB heap wrap 时可稳定停滞。PA vend 始终小于 + // INT64_MAX，因此对同一 64 位地址执行有符号 add-zero，不改变位模式 + // 和比较语义，同时仍保留原子读路径。 + __gm__ int64_t *signed_address = reinterpret_cast<__gm__ int64_t *>(const_cast<__gm__ uint64_t *>(address)); + return static_cast(AtomicAdd(signed_address, static_cast(0))); + } + + __aicore__ static inline int32_t Exchange(__gm__ volatile int32_t *address, int32_t value) { + // AtomicExch 公开重载使用无符号类型；这里只转换位表示，不做数值换算。 + __gm__ uint32_t *unsigned_address = reinterpret_cast<__gm__ uint32_t *>(const_cast<__gm__ int32_t *>(address)); + return static_cast(AtomicExch(unsigned_address, static_cast(value))); + } + + __aicore__ static inline int64_t Exchange(__gm__ volatile int64_t *address, int64_t value) { + __gm__ uint64_t *unsigned_address = reinterpret_cast<__gm__ uint64_t *>(const_cast<__gm__ int64_t *>(address)); + return static_cast(AtomicExch(unsigned_address, static_cast(value))); + } + + __aicore__ static inline uint64_t Exchange(__gm__ volatile uint64_t *address, uint64_t value) { + return AtomicExch(const_cast<__gm__ uint64_t *>(address), value); + } + + __aicore__ static inline int64_t FetchAdd(__gm__ volatile int64_t *address, int64_t value) { + return AtomicAdd(const_cast<__gm__ int64_t *>(address), value); + } + + __aicore__ static inline int64_t FetchMax(__gm__ volatile int64_t *address, int64_t value, uint64_t &retries) { + // A5 直接提供硬件 AtomicMax，不需要像 CPU CAS loop 那样统计软件重试。 + retries = 0; + return AtomicMax(const_cast<__gm__ int64_t *>(address), value); + } + + // A5 PA 契约中的 OUT_OF_ORDER_STORE_BARRIER 在此为空：completion 的 vend/flag + // 依靠前后 AtomicExch 协议；config、trace 与 shared TensorMap payload + // 的 cache 可见性分别由下方既有 dcci/dsb hook 处理。 + // 不额外插入全局屏障，以免改变真实 PA 的热路径。 + __aicore__ static inline void StoreBarrier() {} + + // A5 SYS_CNT 为 1 GHz；公共模型也以 1 GHz tick 记录泳道时间。 + __aicore__ static inline uint64_t Now() { return static_cast(GetSystemCycle()); } + + template + __aicore__ static inline uint64_t NowAfterAtomicResult(T value) { + (void)value; + return Now(); + } + + __aicore__ static inline void ExecuteKernel( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState &worker, + pa_scheduler::TaskKind kind, uint32_t nop_count + ) { + if (state->winner_workload.mode == + static_cast(pa_scheduler::WinnerWorkloadMode::RealCompute)) { + ExecuteRealWinnerWorkload(state, worker, kind); + return; + } + RuntimeNop(nop_count); + } + + // 当前直接 PMU MMIO 观察链路只在 CCEC 分支验证；AscendC 保持公共 hook + // 的空实现，避免静默产出一套未核实寄存器读法。 + __aicore__ static inline bool PmuWindowStart(__gm__ pa_scheduler::SchedulerState *, uint32_t) { return false; } + + __aicore__ static inline void PmuWindowStop(__gm__ pa_scheduler::SchedulerState *, uint32_t, bool) {} + + __aicore__ static inline void SpinHint() {} + + __aicore__ static inline void InvalidateRegion(__gm__ const void *address, uint64_t bytes) { + // 失效 host 写入的 standalone config 或 shared TensorMap payload， + // 再以 dsb 保证后续普通 GM 读看到发布内容；控制字仍走 atomic 路径。 + if (bytes == 0) return; + const uint64_t start = reinterpret_cast(address) & ~uint64_t{63}; + const uint64_t end = (reinterpret_cast(address) + bytes + 63) & ~uint64_t{63}; + for (uint64_t current = start; current < end; current += 64) { + dcci(reinterpret_cast<__gm__ uint8_t *>(current), SINGLE_CACHE_LINE); + } + dsb((mem_dsb_t)0); + } + + __aicore__ static inline void FlushRegion(__gm__ void *address, uint64_t bytes) { + // 普通 GM cache 内容逐 line clean-out 并以 dsb 收口：泳道记录据此 + // 对 host 可见，shared TensorMap writer 也复用同一 hook 发布 payload。 + if (bytes == 0) return; + __asm__ volatile("" ::: "memory"); + const uint64_t start = reinterpret_cast(address) & ~uint64_t{63}; + const uint64_t end = (reinterpret_cast(address) + bytes + 63) & ~uint64_t{63}; + for (uint64_t current = start; current < end; current += 64) { + dcci(reinterpret_cast<__gm__ uint8_t *>(current), SINGLE_CACHE_LINE, CACHELINE_OUT); + } + dsb((mem_dsb_t)0); + } + + __aicore__ static inline void Publish(__gm__ uint64_t *address, uint64_t value) { + // worker 结果使用 bypass-DCache 写，避免与仍驻留在标量 cache 中的旧值混合。 + WriteGmByPassDCache(address, value); + } +}; + +// ACL host API 的统一错误门：任何资源创建、拷贝、同步失败都立即带标签返回， +// 上层 main 不会在半初始化的 device 状态上继续启动 kernel。 +bool CheckAcl(aclError error, const char *label) { + if (error == ACL_SUCCESS) return true; + std::fprintf(stderr, "ACL error %d: %s\n", static_cast(error), label); + return false; +} + +} // namespace + +// 一个 mixed block 固定包含 1 个 AIC 与 2 个 AIV。host 启动 32 个 block， +// 因而形成 32 AIC + 64 AIV。编译时还必须由 build.sh 设置 +// __MIX_CORE_AIC_RATION__=2，使 binary metadata 与 __mix__(1, 2) 一致。 +__schedmode__(1) __global__ __mix__(1, 2) void pa_scheduler_ascendc(__gm__ pa_scheduler::SchedulerState *state) { +#if defined(__DAV_VEC__) + // AIV 侧 GetBlockIdx() 已展平为 0..63，平移 32 后得到公共协议中的 + // worker 32..95；AIC 侧保持原始 block 号 0..31。 + const uint32_t worker_id = pa_scheduler::kAicWorkers + static_cast(GetBlockIdx()); + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aiv); +#else + const uint32_t worker_id = static_cast(GetBlockIdx()); + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aic); +#endif +} + +// AscendC 可执行文件同时承载 host runner 和 mixed kernel。正常执行路径为： +// 解析参数 -> 初始化 ACL/stream -> 分配 GM -> 逐轮 launch/校验/后处理 -> +// 释放 GM/stream/device；初始化或传输等早期错误仍会就地返回。性能区间只包围 +// launch 与 stream synchronize。 +int32_t main(int32_t argc, char **argv) { + pa_scheduler::host::Options options; + pa_scheduler::host::WinnerWorkloadOptions workload_options; + std::vector common_argv; + if (!pa_scheduler::host::ParseWinnerWorkloadOptions( + argc, argv, &workload_options, &common_argv + )) { + return EXIT_FAILURE; + } + // false 只表示 AscendC 可执行文件内嵌 kernel、不要求 --kernel；与是否支持 + // real-compute 无关，不能为开启真计算而改成 true。 + const pa_scheduler::host::ParseStatus parse_status = pa_scheduler::host::ParseOptions( + static_cast(common_argv.size()), common_argv.data(), false, &options + ); + if (parse_status != pa_scheduler::host::ParseStatus::Ok) { + if (parse_status == pa_scheduler::host::ParseStatus::Help) { + std::fprintf( + stderr, + "AscendC winner workload options: " + "[--winner-workload scalar-nop|real-compute] " + "[--real-compute-count N | --real-compute-counts QK,SF,PV,UP] " + "[--real-compute-pattern constant|layout-diagnostic]\n" + ); + std::fprintf( + stderr, + "Default: real-compute, constant, counts=6,28,4,1; " + "scalar-nop is the calibration compatibility mode.\n" + ); + } + return parse_status == pa_scheduler::host::ParseStatus::Help ? EXIT_SUCCESS : EXIT_FAILURE; + } + if (!pa_scheduler::host::ValidateWinnerWorkloadOptions(workload_options)) { + return EXIT_FAILURE; + } + const bool real_compute = + workload_options.mode == pa_scheduler::WinnerWorkloadMode::RealCompute; + std::vector workload_image; + std::vector workload_outputs; + if (real_compute) { + pa_scheduler::host::InitializeWinnerWorkloadBuffers( + workload_options, &workload_image, &workload_outputs + ); + } + pa_scheduler::host::PrintBanner("AscendC", options); + pa_scheduler::host::PrintWinnerWorkloadConfig(workload_options, options.nops); + + // ACL 初始化和选卡必须先于 stream、GM 以及 kernel stub 的任何使用。 + if (!CheckAcl(aclInit(nullptr), "aclInit") || !CheckAcl(aclrtSetDevice(options.device), "aclrtSetDevice")) { + return EXIT_FAILURE; + } + aclrtStream stream = nullptr; + if (!CheckAcl(aclrtCreateStream(&stream), "aclrtCreateStream")) return EXIT_FAILURE; + + // SchedulerState 保留被测关键 offset、DistCore ABI 和生产总跨度；单独检查 + // 64 字节对齐，保证其中每条 atomic/cache-line 状态线不会错位。 + pa_scheduler::SchedulerState *state_device = nullptr; + if (!CheckAcl( + aclrtMalloc( + reinterpret_cast(&state_device), sizeof(pa_scheduler::SchedulerState), + ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(state)" + )) { + return EXIT_FAILURE; + } + if ((reinterpret_cast(state_device) & 63U) != 0) { + std::fprintf(stderr, "Device state is not 64-byte aligned: %p\n", static_cast(state_device)); + return EXIT_FAILURE; + } + + // 真计算只访问本目录分配的独立 GM workspace，不解引用 PA 前端用于 identity + // 建模的 synthetic tensor 地址。初始化和 H2D 都放在 launch 计时之前。 + void *workload_device = nullptr; + if (real_compute && + !CheckAcl( + aclrtMalloc( + &workload_device, pa_scheduler::winner_workload::kWorkspaceBytes, + ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(real-compute workspace)" + )) { + return EXIT_FAILURE; + } + if (real_compute && (reinterpret_cast(workload_device) & 63U) != 0) { + std::fprintf(stderr, "Real-compute workspace is not 64-byte aligned: %p\n", workload_device); + return EXIT_FAILURE; + } + + // trace 是独立 GM 区域，关闭泳道时完全不分配；开启时 header 与每个 + // worker 的固定跨度 records 共用同一块连续内存。 + void *trace_device = nullptr; + if (options.trace_enabled && + !CheckAcl( + aclrtMalloc(&trace_device, pa_scheduler::kTraceBytes, ACL_MEM_MALLOC_HUGE_FIRST), + "aclrtMalloc(swimlane trace)" + )) { + return EXIT_FAILURE; + } + if (options.trace_enabled && (reinterpret_cast(trace_device) & 63U) != 0) { + std::fprintf(stderr, "Device swimlane trace is not 64-byte aligned: %p\n", trace_device); + return EXIT_FAILURE; + } + + std::unique_ptr state(new pa_scheduler::SchedulerState); + pa_scheduler::TraceHeader trace_header{}; + std::vector spans; + std::vector startup_barrier_spans; + std::vector final_barrier_spans; + std::vector final_drain_spans; + std::vector lifecycle_spans; + bool all_passed = true; + bool postprocess_ok = true; + // 多轮复用已加载的 binary、stream 和 GM 分配，只重置协议状态与 trace + // header；因此热轮次不能冒充独立进程首轮性能。 + for (uint32_t run = 1; run <= options.runs; ++run) { + pa_scheduler::host::InitializeState(state.get(), options); + pa_scheduler::host::ConfigureTrace(state.get(), options, trace_device); + pa_scheduler::host::ConfigureWinnerWorkload( + state.get(), workload_options, workload_device + ); + if (real_compute && + !CheckAcl( + aclrtMemcpy( + workload_device, pa_scheduler::winner_workload::kWorkspaceBytes, + workload_image.data(), pa_scheduler::winner_workload::kWorkspaceBytes, + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D real-compute workspace)" + )) { + return EXIT_FAILURE; + } + if (options.trace_enabled) { + pa_scheduler::host::InitializeTraceHeader(&trace_header); + if (!CheckAcl( + aclrtMemcpy( + trace_device, sizeof(trace_header), &trace_header, sizeof(trace_header), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D swimlane header)" + )) { + return EXIT_FAILURE; + } + } + // private 只传输初始化所需的 state prefix 和 standalone controls， + // 避免每轮把约 1 GiB 的 worker 私有区从 host 全量拷入 device； + // shared map 作为 results 后的独立第四范围另行搬运。 + if (!CheckAcl( + aclrtMemcpy( + state_device, pa_scheduler::host::StatePrefixBytes(), state.get(), + pa_scheduler::host::StatePrefixBytes(), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D state prefix)" + ) || + !CheckAcl( + aclrtMemcpy( + &state_device->config, pa_scheduler::host::ControlBytes(), &state->config, + pa_scheduler::host::ControlBytes(), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D standalone controls)" + )) { + return EXIT_FAILURE; + } +#if PTO_FDWIC_SHARED_MAP + // 该 H2D 仍位于 wall_begin 之前，不改变 Submit/launch 性能边界。 + if (!CheckAcl( + aclrtMemcpy( + &state_device->shared_map, pa_scheduler::host::SharedSidecarBytes(), + &state->shared_map, pa_scheduler::host::SharedSidecarBytes(), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D shared TensorMap sidecar)" + )) { + return EXIT_FAILURE; + } +#endif + + // launch 维度是 mixed block 数而不是总 worker 数；__mix__(1, 2) + // 会从这 32 个 block 派生出全部 96 个参与者。 + const auto wall_begin = std::chrono::steady_clock::now(); + pa_scheduler_ascendc<<>>(state_device); + if (!CheckAcl(aclrtSynchronizeStream(stream), "aclrtSynchronizeStream")) return EXIT_FAILURE; + const auto wall_end = std::chrono::steady_clock::now(); + const double host_us = std::chrono::duration(wall_end - wall_begin).count(); + // kernel 完成后回读全局前缀和 96 份结果；大 worker 私有区仍留在 + // device。shared map 在结果之后单独回读；trace 也先回读小 header， + // 再按实际 count 分块读取 records。 + if (!CheckAcl( + aclrtMemcpy( + state.get(), pa_scheduler::host::StatePrefixBytes(), state_device, + pa_scheduler::host::StatePrefixBytes(), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H state prefix)" + ) || + !CheckAcl( + aclrtMemcpy( + &state->final_barrier, pa_scheduler::host::FinalBarrierStateBytes(), &state_device->final_barrier, + pa_scheduler::host::FinalBarrierStateBytes(), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H final barrier)" + ) || + !CheckAcl( + aclrtMemcpy( + state->results, pa_scheduler::host::ResultBytes(), &state_device->results[0], + pa_scheduler::host::ResultBytes(), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H worker results)" + )) { + return EXIT_FAILURE; + } +#if PTO_FDWIC_SHARED_MAP + if (!CheckAcl( + aclrtMemcpy( + &state->shared_map, pa_scheduler::host::SharedSidecarBytes(), + &state_device->shared_map, pa_scheduler::host::SharedSidecarBytes(), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H shared TensorMap sidecar)" + )) { + return EXIT_FAILURE; + } +#endif + if (real_compute && + !CheckAcl( + aclrtMemcpy( + workload_outputs.data(), + static_cast(pa_scheduler::winner_workload::kOutputTiles) * + pa_scheduler::winner_workload::kTileBytes, + static_cast(workload_device) + + pa_scheduler::winner_workload::kSharedInputTiles * + pa_scheduler::winner_workload::kTileBytes, + static_cast(pa_scheduler::winner_workload::kOutputTiles) * + pa_scheduler::winner_workload::kTileBytes, + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H real-compute outputs)" + )) { + return EXIT_FAILURE; + } + if (options.trace_enabled && + !CheckAcl( + aclrtMemcpy( + &trace_header, sizeof(trace_header), trace_device, sizeof(trace_header), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H swimlane header)" + )) { + return EXIT_FAILURE; + } + // trace buffer 按 worker 分成固定跨度；分析和 JSON 导出复用同一个 + // 逐 worker D2H 回调，不依赖 simpler 的采集器实现。 + const auto read_trace_records = + [trace_device](uint32_t worker, uint32_t count, pa_scheduler::TraceRecord *records) { + const size_t offset = + pa_scheduler::TraceRecordsOffset(worker); + return CheckAcl( + aclrtMemcpy( + records, static_cast(count) * sizeof(pa_scheduler::TraceRecord), + static_cast(trace_device) + offset, + static_cast(count) * sizeof(pa_scheduler::TraceRecord), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H swimlane records)" + ); + }; +#if PTO_FDWIC_SHARED_MAP + const auto read_submit_claim_records = + [trace_device]( + uint32_t worker, uint32_t count, + pa_scheduler::SharedSubmitClaimTraceRecord *records + ) { + const size_t offset = + pa_scheduler::TraceSubmitClaimOffset(worker); + return CheckAcl( + aclrtMemcpy( + records, + static_cast(count) * + sizeof(pa_scheduler::SharedSubmitClaimTraceRecord), + static_cast(trace_device) + offset, + static_cast(count) * + sizeof(pa_scheduler::SharedSubmitClaimTraceRecord), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H Submit/Claim endpoint records)" + ); + }; +#endif + // 语义校验是产物门禁：拓扑、cursor、flag、vend、frontier 和每 worker + // 结果全部通过后，才允许把本轮 records 导出成 raw JSON。 + const pa_scheduler::host::Metrics metrics = pa_scheduler::host::Validate( + *state, run, host_us, options.trace_enabled ? &trace_header : nullptr + ); + const bool workload_passed = + !real_compute || pa_scheduler::host::ValidateRealComputeOutputs( + *state, workload_options, workload_outputs, run + ); + all_passed &= metrics.passed && workload_passed; + spans.push_back(metrics.submit_span_us); + startup_barrier_spans.push_back(metrics.startup_barrier_span_us); + final_barrier_spans.push_back(metrics.final_barrier_span_us); + final_drain_spans.push_back(metrics.final_drain_span_us); + lifecycle_spans.push_back(metrics.lifecycle_span_us); + if (options.analyze_swimlane && + !pa_scheduler::host::AnalyzeSwimlaneRecords( + trace_header, *state, read_trace_records +#if PTO_FDWIC_SHARED_MAP + , read_submit_claim_records +#endif + )) { + postprocess_ok = false; + break; + } + if (!options.swimlane_json.empty()) { + if (!metrics.passed || !workload_passed) { + std::fprintf( + stderr, + "Skipping swimlane export because semantic or winner-workload validation failed.\n" + ); + postprocess_ok = false; + break; + } + if (!pa_scheduler::host::ExportSwimlaneRecords( +#if PTO_FDWIC_SHARED_MAP + trace_header, *state, options.swimlane_json, +#else + trace_header, options.swimlane_json, +#endif + workload_options.mode, + real_compute + ? workload_options.repeats + : pa_scheduler::WorkloadCounts{ + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + }, + real_compute + ? pa_scheduler::host::RealComputePatternName(workload_options.pattern) + : "none", + options.final_barrier_shape, options.trace_atomics, + read_trace_records +#if PTO_FDWIC_SHARED_MAP + , read_submit_claim_records +#endif + )) { + postprocess_ok = false; + break; + } + } + } + + std::printf( + "[SUMMARY] runs=%u final_shape=%s median_submit_span_us=%.3f median_startup_barrier_us=%.3f " + "median_final_barrier_us=%.3f median_final_drain_us=%.3f median_lifecycle_us=%.3f " + "semantic_status=%s postprocess_status=%s\n", + options.runs, pa_scheduler::host::FinalBarrierShapeName(options.final_barrier_shape), + pa_scheduler::host::Median(spans), pa_scheduler::host::Median(startup_barrier_spans), + pa_scheduler::host::Median(final_barrier_spans), pa_scheduler::host::Median(final_drain_spans), + pa_scheduler::host::Median(lifecycle_spans), all_passed ? "PASS" : "FAIL", postprocess_ok ? "PASS" : "FAIL" + ); + + // 进入统一清理阶段后继续累计错误而不提前返回，尽量释放本阶段持有的资源； + // 最终退出码同时包含语义、后处理和清理三类状态。 + bool cleanup_ok = true; + if (trace_device != nullptr) { + cleanup_ok &= CheckAcl(aclrtFree(trace_device), "aclrtFree(swimlane trace)"); + } + if (workload_device != nullptr) { + cleanup_ok &= CheckAcl(aclrtFree(workload_device), "aclrtFree(real-compute workspace)"); + } + cleanup_ok &= CheckAcl(aclrtFree(state_device), "aclrtFree(state)"); + cleanup_ok &= CheckAcl(aclrtDestroyStream(stream), "aclrtDestroyStream"); + cleanup_ok &= CheckAcl(aclrtResetDevice(options.device), "aclrtResetDevice"); + cleanup_ok &= CheckAcl(aclFinalize(), "aclFinalize"); + return all_passed && postprocess_ok && cleanup_ok ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/atomic_probe/pa_scheduler/ccec/build.sh b/tests/atomic_probe/pa_scheduler/ccec/build.sh new file mode 100755 index 0000000000..9763eec9e5 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/build.sh @@ -0,0 +1,879 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +# 任一工具失败、未定义变量或管道中间失败都立即终止，避免继续使用半成品 device ELF。 +set -euo pipefail + +# 所有输入和产物都从脚本自身位置解析，调用者无需位于仓库根目录。 +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +ARTIFACT_MANIFEST_NAME="pa_scheduler_artifacts.manifest" + +# mode 是 artifact 身份的一部分。run.sh 总会显式传入；直接调用 build.sh +# 未传 mode 时仍默认 private，保持原有命令可用。private/shared 分别实例化 +# 各自已接线的 TensorMap backend，manifest 会阻止两种产物交叉运行。 +TENSORMAP_MODE="private" +TENSORMAP_RING_CAP=128 +SHARED_INSERT_TURN_GROUPS="${PA_SHARED_INSERT_TURN_GROUPS:-1}" +case "$SHARED_INSERT_TURN_GROUPS" in + 1|2|4|8|16|32|64|128) ;; + *) + echo "PA_SHARED_INSERT_TURN_GROUPS must be a power of two from 1 through 128." >&2 + exit 1 + ;; +esac +if [[ "${1:-}" == "private" || "${1:-}" == "shared" ]]; then + TENSORMAP_MODE="$1" + shift +fi +case "$TENSORMAP_MODE" in + private) TENSORMAP_MODE_ID=0 ;; + shared) TENSORMAP_MODE_ID=1 ;; + *) + echo "Unknown TensorMap mode: $TENSORMAP_MODE (expected private|shared)" >&2 + exit 1 + ;; +esac +if [[ "$TENSORMAP_MODE" != "shared" && + "$SHARED_INSERT_TURN_GROUPS" != "1" ]]; then + echo "PA_SHARED_INSERT_TURN_GROUPS only applies to shared TensorMap builds." >&2 + exit 1 +fi + +# CCEC 不生成跨证据链的统一 ELF。无 variant 保持兼容并等价于 +# swimlane；perf-clock 与 submit-pmu 分别拥有独立目录和编译身份。 +BUILD_VARIANT="${1:-swimlane}" +SPLIT_FINISH=0 +COMPACT_GENERIC_TRACE=0 +case "$BUILD_VARIANT" in + swimlane) + if [[ $# -gt 1 ]]; then + echo "Usage: $0 [private|shared] [swimlane]" >&2 + exit 1 + fi + PHASE_NAME="none" + PHASE_ID=0 + BUILD_DIR="$ROOT_DIR/build/ccec/$TENSORMAP_MODE/swimlane" + COMPACT_GENERIC_TRACE="$TENSORMAP_MODE_ID" + VARIANT_DEFINES=( + "-DPTO_FDWIC_SHARED_MAP=$TENSORMAP_MODE_ID" + -DPA_BUILD_SWIMLANE=1 + -DPA_BUILD_ATOMIC_SWIMLANE=1 + "-DPA_BUILD_COMPACT_GENERIC_TRACE=$COMPACT_GENERIC_TRACE" + -DPA_BUILD_SUBMIT_PMU=0 + -DPA_BUILD_PERF_CLOCK=0 + -DPA_SUBMIT_PMU_PHASE_ID=0 + ) + SPLIT_FINISH=1 + ;; + perf-clock) + if [[ $# -gt 1 ]]; then + echo "Usage: $0 [private|shared] perf-clock" >&2 + exit 1 + fi + PHASE_NAME="none" + PHASE_ID=0 + BUILD_DIR="$ROOT_DIR/build/ccec/$TENSORMAP_MODE/perf-clock" + VARIANT_DEFINES=( + "-DPTO_FDWIC_SHARED_MAP=$TENSORMAP_MODE_ID" + -DPA_BUILD_SWIMLANE=0 + -DPA_BUILD_COMPACT_GENERIC_TRACE=0 + -DPA_BUILD_SUBMIT_PMU=0 + -DPA_BUILD_PERF_CLOCK=1 + -DPA_SUBMIT_PMU_PHASE_ID=0 + ) + # 性能基线必须保持与正式 swimlane/none 相同的 split-finish + # 调用形状，不能为减少编译工作偷偷换成 inline finish。 + SPLIT_FINISH=1 + ;; + submit-pmu) + if [[ $# -ne 2 ]]; then + echo "Usage: $0 [private|shared] submit-pmu none|claim|efdrain|materialize|register" >&2 + exit 1 + fi + PHASE_NAME="$2" + case "$PHASE_NAME" in + none) PHASE_ID=0 ;; + claim) PHASE_ID=1 ;; + efdrain) PHASE_ID=2 ;; + materialize) PHASE_ID=4 ;; + register) PHASE_ID=5 ;; + *) + echo "Unknown submit-pmu phase: $PHASE_NAME (expected none|claim|efdrain|materialize|register)" >&2 + exit 1 + ;; + esac + BUILD_DIR="$ROOT_DIR/build/ccec/$TENSORMAP_MODE/submit-pmu/$PHASE_NAME" + VARIANT_DEFINES=( + "-DPTO_FDWIC_SHARED_MAP=$TENSORMAP_MODE_ID" + -DPA_BUILD_SWIMLANE=0 + -DPA_BUILD_COMPACT_GENERIC_TRACE=0 + -DPA_BUILD_SUBMIT_PMU=1 + -DPA_BUILD_PERF_CLOCK=0 + "-DPA_SUBMIT_PMU_PHASE_ID=$PHASE_ID" + ) + # none/Claim/EfDrain 的 PMU 窗口在 finish 之前已经闭合，可以复用泳道版 + # 的跨 TU noinline finish；Materialize/Register 的窗口跨入 finish，必须 + # 保持 inline，确保边界继续操作同一份真实 PmuContext。 + if [[ "$PHASE_NAME" == "none" || "$PHASE_NAME" == "claim" || + "$PHASE_NAME" == "efdrain" ]]; then + SPLIT_FINISH=1 + fi + ;; + *) + echo "Usage: $0 [private|shared] [swimlane|perf-clock] | $0 [private|shared] submit-pmu " >&2 + exit 1 + ;; +esac +if [[ "$SPLIT_FINISH" -eq 1 ]]; then + VARIANT_DEFINES+=(-DPA_COMPETE_FIRST_SPLIT_FINISH=1) +fi + +# 物理泳道布局与传给三镜像的 compact 编译宏来自同一组 build-side +# 变量。run.sh 会按 mode/variant 独立推导并逐字段核对，避免 producer +# 和 consumer 共用同一处错误。trace-free 变体虽然不分配泳道缓冲，仍 +# 固化其编译 ABI，防止 host/kernel 交叉复用。 +if [[ "$TENSORMAP_MODE" == "shared" ]]; then + TRACE_SUBMIT_CLAIM_RECORD_BYTES=32 + TRACE_RECORDS_PER_CORE=28416 + if [[ "$COMPACT_GENERIC_TRACE" -eq 1 ]]; then + TRACE_GENERIC_RECORD_BYTES=16 + TRACE_WORKER_STRIDE_BYTES=593920 + else + TRACE_GENERIC_RECORD_BYTES=32 + TRACE_WORKER_STRIDE_BYTES=1048576 + fi +else + TRACE_GENERIC_RECORD_BYTES=32 + TRACE_SUBMIT_CLAIM_RECORD_BYTES=0 + TRACE_RECORDS_PER_CORE=65536 + TRACE_WORKER_STRIDE_BYTES=2097152 +fi + +# 正式 standalone CCEC 产物先固定使用已验证的 128×128 布局；CAP 仍 +# 显式进入三镜像编译身份和 manifest，避免默认值漂移后静默混件。 +VARIANT_DEFINES+=("-DPTO_FDWIC_TENSORMAP_RING_CAP=$TENSORMAP_RING_CAP") +VARIANT_DEFINES+=( + "-DPTO_FDWIC_SHARED_INSERT_TURN_GROUPS=$SHARED_INSERT_TURN_GROUPS" +) +echo "[BUILD] shared insert-turn groups=$SHARED_INSERT_TURN_GROUPS" + +# 编译只依赖本目录源码与用户安装的 CANN/PTO 头，不引用 pa_scheduler 目录外的 simpler 构建产物。 +if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then + echo "ASCEND_HOME_PATH is not set; source the CANN 9.1 set_env.sh first." >&2 + exit 1 +fi + +CCEC="$ASCEND_HOME_PATH/bin/ccec" +LD="$ASCEND_HOME_PATH/bin/ld.lld" +HCC="$ASCEND_HOME_PATH/tools/hcc/bin/aarch64-target-linux-gnu-g++" +CXX_BIN="${CXX:-g++}" +READELF_BIN="${READELF:-readelf}" +PTO_INCLUDE_ROOT="${PTO_ISA_ROOT:-$ASCEND_HOME_PATH/x86_64-linux}" + +# ccec/ld.lld 必须来自当前已 source 的 CANN；host 编译器和 readelf 允许用户通过环境变量替换。 +if [[ ! -x "$CCEC" || ! -x "$LD" ]]; then + echo "CCEC or ld.lld is missing under ASCEND_HOME_PATH=$ASCEND_HOME_PATH" >&2 + exit 1 +fi +if [[ "$BUILD_VARIANT" == "submit-pmu" && ! -x "$HCC" ]]; then + echo "The AICPU HCC compiler is missing under ASCEND_HOME_PATH=$ASCEND_HOME_PATH" >&2 + exit 1 +fi +if ! command -v "$READELF_BIN" >/dev/null 2>&1; then + echo "readelf is required to verify the mixed AICore ELF." >&2 + exit 1 +fi +if ! command -v sha256sum >/dev/null 2>&1; then + echo "sha256sum is required to publish the CCEC artifact manifest." >&2 + exit 1 +fi +if [[ ! -f "$PTO_INCLUDE_ROOT/include/pto/common/kernel_meta.hpp" ]]; then + echo "PTO kernel metadata header is missing under $PTO_INCLUDE_ROOT/include" >&2 + exit 1 +fi +for header in pto/pto-inst.hpp pto/common/constants.hpp pto/common/pto_tile.hpp; do + if [[ ! -f "$PTO_INCLUDE_ROOT/include/$header" ]]; then + echo "PTO real-compute header is missing: $PTO_INCLUDE_ROOT/include/$header" >&2 + exit 1 + fi +done + +mkdir -p "$BUILD_DIR" +rm -f -- "$BUILD_DIR/$ARTIFACT_MANIFEST_NAME" +if [[ "$BUILD_VARIANT" != "submit-pmu" ]]; then + # 旧构建可能在目录中残留 PMU owner；swimlane/perf-clock 主动移除 + # 不属于本变体的产物，避免 direct host 调用误加载上一版诊断 SO。 + rm -f \ + "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so" \ + "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so" +fi + +# 关闭编译器自动插入的 scalar DCCI，由 kernel.cpp 中与 PA 对齐的显式失效/回写协议负责 cache 可见性。 +# 两种架构共用这些 ABI、栈和优化参数，避免 AIC/AIV 对共享 SchedulerState 产生不同解释。 +COMMON_FLAGS=( + -c -O3 -g -x cce -Wall -std=c++17 + --cce-aicore-only + -mllvm -cce-aicore-stack-size=0x8000 + -mllvm -cce-aicore-function-stack-size=0x8000 + -mllvm -cce-aicore-record-overflow=false + -mllvm -cce-aicore-addr-transform + -mllvm -cce-aicore-dcci-insert-for-scalar=false + -mllvm -cce-aicore-dcci-before-kernel-end=false + -I"$ROOT_DIR/common" + -I"$PTO_INCLUDE_ROOT/include" + "${VARIANT_DEFINES[@]}" +) + +# 正式 shared PA entry 已实例化 ordered writer-delta 路径；reader +# progress/reclaim 与旧 WriterIntentSet 仍只保留为隔离协议原语。构建 +# 额外对这些模板做真实后端代码生成和静态链接，分别锁定 cube/vector +# 的 CcecOps、GM 地址空间、atomicCAS、DCCI 以及“无未解析 compiler +# builtin”契约。probe 不加入 DEVICE_OBJECTS， +# 检查后立即删除，因此不会改变正式 mixed ELF、I-cache 布局或运行性能。 +if [[ "$TENSORMAP_MODE" == "shared" ]]; then +( + SHARED_PROTOCOL_PROBE_AIC_OBJECT="$BUILD_DIR/.shared_protocol_probe_aic.o" + SHARED_PROTOCOL_PROBE_AIV_OBJECT="$BUILD_DIR/.shared_protocol_probe_aiv.o" + SHARED_PROTOCOL_PROBE_AIC_ELF="$BUILD_DIR/.shared_protocol_probe_aic.elf" + SHARED_PROTOCOL_PROBE_AIV_ELF="$BUILD_DIR/.shared_protocol_probe_aiv.elf" + cleanup_shared_protocol_probe() { + rm -f \ + "$SHARED_PROTOCOL_PROBE_AIC_OBJECT" \ + "$SHARED_PROTOCOL_PROBE_AIV_OBJECT" \ + "$SHARED_PROTOCOL_PROBE_AIC_ELF" \ + "$SHARED_PROTOCOL_PROBE_AIV_ELF" + } + # 独立子 shell 的 EXIT trap 不会覆盖正式 manifest 的原子发布 trap； + # 编译或链接任一步失败也会清掉隐藏 probe，不留下半成品混淆现场。 + trap cleanup_shared_protocol_probe EXIT + cleanup_shared_protocol_probe + + echo "[CHECK] CCEC AIC generic shared protocol instantiation" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DPA_BUILD_AIC \ + -o "$SHARED_PROTOCOL_PROBE_AIC_OBJECT" \ + "$SCRIPT_DIR/shared_protocol_compile_probe.cpp" + "$LD" -m aicorelinux -Ttext=0 -static \ + -o "$SHARED_PROTOCOL_PROBE_AIC_ELF" \ + "$SHARED_PROTOCOL_PROBE_AIC_OBJECT" + + echo "[CHECK] CCEC AIV generic shared protocol instantiation" + "$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DPA_BUILD_AIV \ + -o "$SHARED_PROTOCOL_PROBE_AIV_OBJECT" \ + "$SCRIPT_DIR/shared_protocol_compile_probe.cpp" + "$LD" -m aicorelinux -Ttext=0 -static \ + -o "$SHARED_PROTOCOL_PROBE_AIV_ELF" \ + "$SHARED_PROTOCOL_PROBE_AIV_OBJECT" +) +fi + +# CompeteFirstSplitRuntimeState 当前 ABI 为 1664B。只给 split 产物开启 +# block-local relocation，并按精确尺寸预留，避免影响局部 PMU 的 inline ELF。 +SPLIT_STATE_BYTES=1664 +SPLIT_FINISH_CALL_SITES=5 +if [[ "$SPLIT_FINISH" -eq 1 ]]; then + COMMON_FLAGS+=( + -mllvm -cce-block-local-relocate=true + -mllvm "-cce-block-local-reserve-size=$SPLIT_STATE_BYTES" + ) +fi + +# 同一入口源码分别面向 cube 与 vector ISA 编译，宏只选择各自的全局入口和 mixed metadata。 +echo "[BUILD] CCEC AIC entry (dav-c310-cube)" +"$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DPA_BUILD_AIC \ + -o "$BUILD_DIR/pa_scheduler_aic.o" \ + "$SCRIPT_DIR/kernel.cpp" + +echo "[BUILD] CCEC AIV entry (dav-c310-vec)" +"$CCEC" "${COMMON_FLAGS[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DPA_BUILD_AIV \ + -o "$BUILD_DIR/pa_scheduler_aiv.o" \ + "$SCRIPT_DIR/kernel.cpp" + +check_workload_dispatcher_object() { + local object_path="$1" + local expected_symbol="$2" + local wrong_role_symbol="$3" + local object_symbols + object_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$object_path")" + if ! awk -v name="$expected_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$object_symbols"; then + echo "Expected exactly one non-empty strong workload dispatcher in $object_path: $expected_symbol" >&2 + exit 1 + fi + if awk -v name="$wrong_role_symbol" \ + '$NF == name {found = 1} END {exit !found}' <<<"$object_symbols"; then + echo "Wrong-role workload dispatcher leaked into $object_path: $wrong_role_symbol" >&2 + exit 1 + fi +} +check_workload_dispatcher_object \ + "$BUILD_DIR/pa_scheduler_aic.o" \ + pa_execute_real_winner_workload_aic \ + pa_execute_real_winner_workload_aiv +check_workload_dispatcher_object \ + "$BUILD_DIR/pa_scheduler_aiv.o" \ + pa_execute_real_winner_workload_aiv \ + pa_execute_real_winner_workload_aic +echo "[CHECK] role-specific real-compute dispatchers are strong and do not cross roles" + +text_relocation_count_for_symbol() { + local object_path="$1" + local symbol_name="$2" + "$READELF_BIN" --relocs --wide "$object_path" | awk -v name="$symbol_name" ' + /^Relocation section '\''\.rela\.text'\''/ {in_text = 1; next} + /^Relocation section / {in_text = 0} + in_text { + for (column = 1; column <= NF; ++column) { + if ($column == name) { + count++ + next + } + } + } + END {print count + 0} + ' +} + +check_split_role_objects() { + local role="$1" + local wrong_role="$2" + local caller="$BUILD_DIR/pa_scheduler_${role}.o" + local runtime="$BUILD_DIR/pa_scheduler_compete_first_callback_runtime_${role}.o" + local finish="$BUILD_DIR/pa_scheduler_compete_first_callback_finish_${role}.o" + local state_symbol="pa_scheduler_compete_first_callback_state_${role}" + local finish_symbol="pa_scheduler_compete_first_callback_finish_${role}" + local orchestration_symbol="pa_scheduler_compete_first_callback_orchestration_${role}" + local entry_symbol="pa_scheduler_0_mix_${role}" + local dispatcher_symbol="pa_execute_real_winner_workload_${role}" + local caller_symbols runtime_symbols finish_symbols + caller_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$caller")" + runtime_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$runtime")" + finish_symbols="$("$READELF_BIN" --symbols --wide --sym-base=10 "$finish")" + + if ! awk -v name="$orchestration_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$caller_symbols"; then + echo "Missing unique strong compete-first orchestration in caller: $caller ($orchestration_symbol)" >&2 + exit 1 + fi + for imported in "$state_symbol" "$finish_symbol"; do + if ! awk -v name="$imported" \ + '$5 == "GLOBAL" && $7 == "UND" && $NF == name {count++} END {exit count != 1}' \ + <<<"$caller_symbols"; then + echo "Caller must import exactly one matching split symbol: $caller ($imported)" >&2 + exit 1 + fi + done + if [[ "$(text_relocation_count_for_symbol "$caller" "$finish_symbol")" -ne \ + "$SPLIT_FINISH_CALL_SITES" ]]; then + echo "Caller must contain exactly $SPLIT_FINISH_CALL_SITES all-task finish .rela.text relocations: $caller" >&2 + exit 1 + fi + if [[ "$(text_relocation_count_for_symbol "$caller" "$state_symbol")" -eq 0 ]]; then + echo "Caller must access its matching external block-local state: $caller" >&2 + exit 1 + fi + if "$READELF_BIN" --sections --wide "$caller" | awk \ + 'index($0, ".ascend.meta.") != 0 {found = 1} END {exit !found}'; then + echo "Compete-first caller object must not define launch metadata: $caller" >&2 + exit 1 + fi + + if ! awk -v name="$state_symbol" -v bytes="$SPLIT_STATE_BYTES" \ + '$4 == "OBJECT" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 == bytes {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime must own one exact-size block-local state: $runtime ($state_symbol)" >&2 + exit 1 + fi + if ! awk -v name="$entry_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime must own one non-empty mixed entry: $runtime ($entry_symbol)" >&2 + exit 1 + fi + if ! awk -v name="$orchestration_symbol" \ + '$5 == "GLOBAL" && $7 == "UND" && $NF == name {count++} END {exit count != 1}' \ + <<<"$runtime_symbols"; then + echo "Runtime must import one role-specific orchestration: $runtime ($orchestration_symbol)" >&2 + exit 1 + fi + if [[ "$(text_relocation_count_for_symbol "$runtime" "$orchestration_symbol")" -ne 1 ]]; then + echo "Runtime entry must contain exactly one orchestration call relocation: $runtime" >&2 + exit 1 + fi + local block_local_record block_local_section_index block_local_size_hex block_local_alignment + block_local_record="$( + "$READELF_BIN" --sections --wide "$runtime" | awk ' + {for (column = 1; column <= NF; ++column) { + if ($column == ".bl.uninit") { + section_index = $(column - 1) + gsub(/\[/, "", section_index) + gsub(/\]/, "", section_index) + print section_index, $(column + 4), $NF + exit + } + }} + ' + )" + read -r block_local_section_index block_local_size_hex block_local_alignment \ + <<<"$block_local_record" + if [[ -z "$block_local_section_index" || -z "$block_local_size_hex" || + $((16#$block_local_size_hex)) -ne "$SPLIT_STATE_BYTES" || + "$block_local_alignment" -ne 64 ]]; then + echo "Runtime .bl.uninit must be exactly ${SPLIT_STATE_BYTES}B and 64B aligned: $runtime" >&2 + exit 1 + fi + if ! awk -v name="$state_symbol" -v section="$block_local_section_index" \ + '$4 == "OBJECT" && $7 == section && $NF == name {count++} + END {exit count != 1}' <<<"$runtime_symbols"; then + echo "Runtime state must be defined in its exact .bl.uninit section: $runtime" >&2 + exit 1 + fi + local runtime_sections + runtime_sections="$("$READELF_BIN" --sections --wide "$runtime")" + if ! awk -v name=".ascend.meta.$entry_symbol" ' + {for (column = 1; column <= NF; ++column) { + if ($column == name) found = 1 + }} + END {exit !found} + ' <<<"$runtime_sections"; then + echo "Runtime object is missing matching mixed-entry metadata: $runtime" >&2 + exit 1 + fi + if awk -v name=".ascend.meta.pa_scheduler_0_mix_${wrong_role}" ' + {for (column = 1; column <= NF; ++column) { + if ($column == name) found = 1 + }} + END {exit !found} + ' <<<"$runtime_sections"; then + echo "Wrong-role mixed-entry metadata leaked into runtime object: $runtime" >&2 + exit 1 + fi + + if ! awk -v name="$finish_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$finish_symbols"; then + echo "Finish object must define one non-empty strong finish: $finish ($finish_symbol)" >&2 + exit 1 + fi + for imported in "$state_symbol" "$dispatcher_symbol"; do + if ! awk -v name="$imported" \ + '$5 == "GLOBAL" && $7 == "UND" && $NF == name {count++} END {exit count != 1}' \ + <<<"$finish_symbols"; then + echo "Finish object must import exactly one matching symbol: $finish ($imported)" >&2 + exit 1 + fi + if [[ "$(text_relocation_count_for_symbol "$finish" "$imported")" -eq 0 ]]; then + echo "Finish object must reference its matching imported symbol: $finish ($imported)" >&2 + exit 1 + fi + done + if "$READELF_BIN" --sections --wide "$finish" | awk \ + 'index($0, ".ascend.meta.") != 0 {found = 1} END {exit !found}'; then + echo "Compete-first finish object must not define launch metadata: $finish" >&2 + exit 1 + fi + + local forbidden symbol_table object_path + for object_path in "$caller" "$runtime" "$finish"; do + case "$object_path" in + "$caller") symbol_table="$caller_symbols" ;; + "$runtime") symbol_table="$runtime_symbols" ;; + *) symbol_table="$finish_symbols" ;; + esac + for forbidden in \ + "pa_scheduler_compete_first_callback_state_${wrong_role}" \ + "pa_scheduler_compete_first_callback_finish_${wrong_role}" \ + "pa_scheduler_compete_first_callback_orchestration_${wrong_role}" \ + "pa_execute_real_winner_workload_${wrong_role}" \ + "pa_scheduler_0_mix_${wrong_role}"; do + if awk -v name="$forbidden" \ + '$NF == name {found = 1} END {exit !found}' <<<"$symbol_table"; then + echo "Wrong-role compete-first symbol leaked into $object_path: $forbidden" >&2 + exit 1 + fi + done + done + + if awk -v name="$entry_symbol" '$NF == name {found = 1} END {exit !found}' \ + <<<"$caller_symbols"; then + echo "Compete-first caller must not own a launch entry: $caller ($entry_symbol)" >&2 + exit 1 + fi + if awk -v name="$entry_symbol" '$NF == name {found = 1} END {exit !found}' \ + <<<"$finish_symbols"; then + echo "Compete-first finish must not own a launch entry: $finish ($entry_symbol)" >&2 + exit 1 + fi + for forbidden in "$finish_symbol" "$dispatcher_symbol"; do + if awk -v name="$forbidden" '$NF == name {found = 1} END {exit !found}' \ + <<<"$runtime_symbols"; then + echo "Runtime entry/state owner contains an unexpected helper: $runtime ($forbidden)" >&2 + exit 1 + fi + done + if awk -v name="$orchestration_symbol" '$NF == name {found = 1} END {exit !found}' \ + <<<"$finish_symbols"; then + echo "Compete-first finish must not contain orchestration: $finish ($orchestration_symbol)" >&2 + exit 1 + fi +} + +if [[ "$SPLIT_FINISH" -eq 1 ]]; then + echo "[BUILD] CCEC AIC compete-first runtime/state owner" + "$CCEC" "${COMMON_FLAGS[@]}" --cce-aicore-arch=dav-c310-cube -DPA_BUILD_AIC \ + -o "$BUILD_DIR/pa_scheduler_compete_first_callback_runtime_aic.o" \ + "$SCRIPT_DIR/callback_runtime_entry.cpp" + echo "[BUILD] CCEC AIC compete-first noinline finish" + "$CCEC" "${COMMON_FLAGS[@]}" --cce-aicore-arch=dav-c310-cube -DPA_BUILD_AIC \ + -o "$BUILD_DIR/pa_scheduler_compete_first_callback_finish_aic.o" \ + "$SCRIPT_DIR/callback_finish.cpp" + echo "[BUILD] CCEC AIV compete-first runtime/state owner" + "$CCEC" "${COMMON_FLAGS[@]}" --cce-aicore-arch=dav-c310-vec -DPA_BUILD_AIV \ + -o "$BUILD_DIR/pa_scheduler_compete_first_callback_runtime_aiv.o" \ + "$SCRIPT_DIR/callback_runtime_entry.cpp" + echo "[BUILD] CCEC AIV compete-first noinline finish" + "$CCEC" "${COMMON_FLAGS[@]}" --cce-aicore-arch=dav-c310-vec -DPA_BUILD_AIV \ + -o "$BUILD_DIR/pa_scheduler_compete_first_callback_finish_aiv.o" \ + "$SCRIPT_DIR/callback_finish.cpp" + check_split_role_objects aic aiv + check_split_role_objects aiv aic + DEVICE_OBJECTS=( + "$BUILD_DIR/pa_scheduler_compete_first_callback_runtime_aic.o" + "$BUILD_DIR/pa_scheduler_aic.o" + "$BUILD_DIR/pa_scheduler_compete_first_callback_finish_aic.o" + "$BUILD_DIR/pa_scheduler_compete_first_callback_runtime_aiv.o" + "$BUILD_DIR/pa_scheduler_aiv.o" + "$BUILD_DIR/pa_scheduler_compete_first_callback_finish_aiv.o" + ) + echo "[CHECK] compete-first caller/runtime/finish role and state symbols are complete" +else + DEVICE_OBJECTS=( + "$BUILD_DIR/pa_scheduler_aic.o" + "$BUILD_DIR/pa_scheduler_aiv.o" + ) +fi + +# 静态链接把两个 device object 合成一个可由 runtime 按 1:2 比例启动的 mixed AICore ELF。 +echo "[BUILD] Static 1:2 mixed AICore ELF" +"$LD" -m aicorelinux -Ttext=0 -static \ + --version-script="$SCRIPT_DIR/pa_scheduler_device_exports.map" \ + -o "$BUILD_DIR/pa_scheduler_kernel.o" \ + "${DEVICE_OBJECTS[@]}" + +SYMBOL_TABLE="$("$READELF_BIN" --symbols --wide --sym-base=10 "$BUILD_DIR/pa_scheduler_kernel.o")" +SECTION_TABLE="$("$READELF_BIN" --sections --wide "$BUILD_DIR/pa_scheduler_kernel.o")" +# 构建成功不等于 mixed launch 可用：同时检查两个入口符号及其 metadata section，缺一即拒绝产物。 +# `set -e` 同时保证 readelf 自身失败时不会拿空字符串继续做伪检查。 +for entry in pa_scheduler_0_mix_aic pa_scheduler_0_mix_aiv; do + if ! awk -v name="$entry" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && $NF == name && $3 != "0" && $3 != "0x0" {found = 1} END {exit !found}' \ + <<<"$SYMBOL_TABLE"; then + echo "Missing non-empty defined GLOBAL mixed-kernel entry: $entry" >&2 + exit 1 + fi + if [[ "$SECTION_TABLE" != *".ascend.meta.$entry"* ]]; then + echo "Missing mixed-kernel metadata section: .ascend.meta.$entry" >&2 + exit 1 + fi +done +echo "[CHECK] both 1:2 mixed entries and metadata sections are present" + +# perf-clock 最终 ELF 必须证明最重的泳道写记录慢体已经在编译期消失。 +# 正向身份由 manifest SHA 和运行时 build_variant 双重闭合；不额外向 +# `.text` 塞 marker，避免仅用于取证的代码改变后续热函数 I-cache 对齐。 +if [[ "$BUILD_VARIANT" == "perf-clock" ]]; then + if awk \ + '$7 != "UND" && index($NF, "WritePollBatchRecordRaw") != 0 {found = 1} + END {exit !found}' <<<"$SYMBOL_TABLE"; then + echo "Swimlane record writer leaked into perf-clock AICore ELF." >&2 + exit 1 + fi + echo "[CHECK] perf-clock swimlane record writer is absent; identity uses manifest/runtime handshake" +fi + +# A5 runtime 会把已定义的 GLOBAL FUNC 当作可启动候选；最终 device ELF 只允许 +# 两个带 metadata 的 mixed 入口暴露为全局函数。任何新增 helper 都必须保持 LOCAL。 +while IFS= read -r global_func; do + case "$global_func" in + pa_scheduler_0_mix_aic|pa_scheduler_0_mix_aiv) ;; + *) + echo "Unexpected GLOBAL device function (possible kernel-entry pollution): $global_func" >&2 + exit 1 + ;; + esac +done < <(awk '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" {print $NF}' <<<"$SYMBOL_TABLE") +echo "[CHECK] only the two mixed entries are exported as GLOBAL device functions" + +# finish TU 需要调用按核型区分的真计算 dispatcher；version script 将其与 +# 底层 Cube/Vector 实体全部局部化，最终只保留两个 mixed kernel 入口。 +for workload_symbol in \ + pa_execute_real_winner_workload_aic \ + pa_execute_real_winner_workload_aiv \ + pa_real_cube_workload_aic \ + pa_real_vector_add_workload_aiv \ + pa_real_vector_mul_workload_aiv; do + workload_size="$( + awk -v name="$workload_symbol" \ + '$4 == "FUNC" && $5 == "LOCAL" && $7 != "UND" && index($NF, name) != 0 && $3 + 0 > 0 {print $3; exit}' \ + <<<"$SYMBOL_TABLE" + )" + if [[ -z "$workload_size" ]]; then + echo "Missing non-empty LOCAL CCEC real-compute workload function: $workload_symbol" >&2 + exit 1 + fi + if awk -v name="$workload_symbol" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && index($NF, name) != 0 {found = 1} END {exit !found}' \ + <<<"$SYMBOL_TABLE"; then + echo "CCEC real-compute helper must not be a GLOBAL kernel candidate: $workload_symbol" >&2 + exit 1 + fi +done +echo "[CHECK] CCEC cube/vector real-compute helpers are non-empty LOCAL functions" + +if [[ "$SPLIT_FINISH" -eq 1 ]]; then + for role in aic aiv; do + finish_symbol="pa_scheduler_compete_first_callback_finish_${role}" + orchestration_symbol="pa_scheduler_compete_first_callback_orchestration_${role}" + state_symbol="pa_scheduler_compete_first_callback_state_${role}" + for local_function in "$finish_symbol" "$orchestration_symbol"; do + if ! awk -v name="$local_function" \ + '$4 == "FUNC" && $5 == "LOCAL" && $7 != "UND" && $NF == name && $3 + 0 > 0 {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Missing unique LOCAL compete-first function: $local_function" >&2 + exit 1 + fi + done + if ! awk -v name="$state_symbol" -v bytes="$SPLIT_STATE_BYTES" \ + '$4 == "OBJECT" && $5 == "LOCAL" && $7 != "UND" && $NF == name && $3 + 0 == bytes {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Missing exact-size LOCAL compete-first state: $state_symbol" >&2 + exit 1 + fi + done + aic_state_hex="$(awk '$NF == "pa_scheduler_compete_first_callback_state_aic" {print $2; exit}' \ + <<<"$SYMBOL_TABLE")" + aiv_state_hex="$(awk '$NF == "pa_scheduler_compete_first_callback_state_aiv" {print $2; exit}' \ + <<<"$SYMBOL_TABLE")" + final_block_local_record="$( + awk '{for (column = 1; column <= NF; ++column) { + if ($column == ".bl_uninit") { + section_index = $(column - 1) + gsub(/\[/, "", section_index) + gsub(/\]/, "", section_index) + print section_index, $(column + 4), $NF + exit + } + }}' <<<"$SECTION_TABLE" + )" + read -r final_block_local_section_index final_block_local_size_hex \ + final_block_local_alignment <<<"$final_block_local_record" + if [[ -z "$aic_state_hex" || -z "$aiv_state_hex" || + $((16#$aic_state_hex)) -ne 0 || + $((16#$aiv_state_hex)) -ne "$SPLIT_STATE_BYTES" || + -z "$final_block_local_section_index" || -z "$final_block_local_size_hex" || + $((16#$final_block_local_size_hex)) -ne $((2 * SPLIT_STATE_BYTES)) || + "$final_block_local_alignment" -ne 64 ]]; then + echo "Final block-local layout must be two exact, non-overlapping 64B-aligned compete-first states." >&2 + exit 1 + fi + for state_symbol in \ + pa_scheduler_compete_first_callback_state_aic \ + pa_scheduler_compete_first_callback_state_aiv; do + if ! awk -v name="$state_symbol" -v section="$final_block_local_section_index" \ + '$4 == "OBJECT" && $7 == section && $NF == name {count++} + END {exit count != 1}' <<<"$SYMBOL_TABLE"; then + echo "Final compete-first state must be bound to the exact .bl_uninit section: $state_symbol" >&2 + exit 1 + fi + done + if [[ -n "$("$READELF_BIN" --relocs --wide "$BUILD_DIR/pa_scheduler_kernel.o" | + sed -n '/Relocation section/p')" ]]; then + echo "Final compete-first mixed ELF must not retain relocations." >&2 + exit 1 + fi + echo "[CHECK] final ELF keeps helpers LOCAL, binds two exact states, and has no relocations" +fi + +check_icache_probe_layout() { + local role="$1" + local target="pa_icache_target_${role}" + local harness="pa_icache_measure_${role}" + local thrash="pa_icache_thrash_${role}" + local target_record + local harness_record + local thrash_record + target_record="$(awk -v name="$target" '$4 == "FUNC" && index($NF, name) != 0 {print $2, $3; exit}' <<<"$SYMBOL_TABLE")" + harness_record="$(awk -v name="$harness" '$4 == "FUNC" && index($NF, name) != 0 {print $2, $3; exit}' <<<"$SYMBOL_TABLE")" + thrash_record="$(awk -v name="$thrash" '$4 == "FUNC" && index($NF, name) != 0 {print $2, $3; exit}' <<<"$SYMBOL_TABLE")" + if [[ -z "$target_record" || -z "$harness_record" || -z "$thrash_record" ]]; then + echo "Missing I-cache probe symbols for $role" >&2 + exit 1 + fi + + local target_hex target_size harness_hex harness_size thrash_hex thrash_size + read -r target_hex target_size <<<"$target_record" + read -r harness_hex harness_size <<<"$harness_record" + read -r thrash_hex thrash_size <<<"$thrash_record" + local target_address=$((16#$target_hex)) + local harness_address=$((16#$harness_hex)) + local thrash_address=$((16#$thrash_hex)) + if (( target_address % 128 != 0 || target_size == 0 || target_size > 16 )); then + echo "Invalid single-fetch-block I-cache target for $role: address=0x$target_hex size=$target_size" >&2 + exit 1 + fi + if (( thrash_size < 65536 )); then + echo "I-cache thrash body is smaller than 64 KiB for $role: size=$thrash_size" >&2 + exit 1 + fi + if (( harness_address % 128 != 0 || target_address + 128 > harness_address || + harness_address + harness_size > thrash_address )); then + echo "I-cache layout must be target -> harness -> thrash for $role" >&2 + exit 1 + fi + echo "[CHECK] $role I-cache target=0x$target_hex/$target_size harness=0x$harness_hex/$harness_size "\ + "thrash=0x$thrash_hex/$thrash_size" +} + +# 两个正式 ELF 都不携带旧 cold/warm 校准冲刷体；submit-pmu 只观察真实 +# Submit。保留上面的检查函数供历史布局取证时复核，但正式构建不调用它。 + +# PMU selector/CTRL 的所有权必须由主 aicpu_scheduler 配置并在退出前恢复。 +# standalone 目录内自带 Path-A dispatcher 与 owner：前者负责把 owner SO +# 落到设备预安装目录，后者由 mode=0 JSON 注册并通过统一入口执行命令。 +# swimlane 构建不生成 PMU owner/dispatcher；submit-pmu 则把 kernel、host、 +# owner 与 dispatcher 全部放在同一个 phase 目录，禁止跨 phase 复用。 +if [[ "$BUILD_VARIANT" == "submit-pmu" ]]; then + echo "[BUILD] self-contained AICPU PMU dispatcher" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra -Werror \ + -Wl,--build-id \ + "$SCRIPT_DIR/pmu_owner_dispatcher.cpp" \ + -o "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so" + + echo "[BUILD] self-contained AICPU PMU owner" + "$HCC" -shared -fPIC -O3 -g -std=gnu++17 -Wall -Wextra -Werror \ + -Wl,--build-id \ + "${VARIANT_DEFINES[@]}" \ + -I"$SCRIPT_DIR" \ + "$SCRIPT_DIR/pmu_owner_aicpu.cpp" \ + -o "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so" + + OWNER_HEADER="$("$READELF_BIN" --file-header "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so")" + OWNER_SYMBOLS="$("$READELF_BIN" --dyn-syms --wide "$BUILD_DIR/libpa_scheduler_pmu_owner_aicpu.so")" + DISPATCHER_HEADER="$("$READELF_BIN" --file-header "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so")" + DISPATCHER_SYMBOLS="$("$READELF_BIN" --dyn-syms --wide "$BUILD_DIR/libpa_scheduler_pmu_owner_dispatcher.so")" + if [[ "$OWNER_HEADER" != *"Type: DYN"* || + "$OWNER_HEADER" != *"Machine: AArch64"* || + "$DISPATCHER_HEADER" != *"Type: DYN"* || + "$DISPATCHER_HEADER" != *"Machine: AArch64"* ]]; then + echo "PMU dispatcher and owner must both be AArch64 shared objects." >&2 + exit 1 + fi + if [[ "$OWNER_SYMBOLS" != *" simpler_aicpu_exec"* ]]; then + echo "Missing main AICPU PMU owner entry: simpler_aicpu_exec" >&2 + exit 1 + fi + for entry in StaticTileFwkBackendKernelServer DynTileFwkBackendKernelServerInit DynTileFwkBackendKernelServer; do + if [[ "$DISPATCHER_SYMBOLS" != *" $entry"* ]]; then + echo "Missing AICPU PMU dispatcher entry: $entry" >&2 + exit 1 + fi + done + echo "[CHECK] Path-A dispatcher and main AICPU PMU owner exports are present" +fi + +# host runner 只链接用户 CANN 9.1 的 ACL/runtime，并写入同一安装目录的 rpath，运行时不需要 simpler 动态库。 +# `-Werror` 让 host API 签名或尺寸类型变化在构建期暴露，避免到上板阶段才出现参数截断。 +echo "[BUILD] CCEC host runner" +"$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror -Wno-deprecated-declarations \ + "${VARIANT_DEFINES[@]}" \ + -I"$ROOT_DIR/common" \ + -I"$ASCEND_HOME_PATH/include" \ + -I"$ASCEND_HOME_PATH/pkg_inc" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime/runtime" \ + "$SCRIPT_DIR/host.cpp" \ + -L"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -Wl,-rpath,"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -lascendcl -lruntime \ + -ldl \ + -o "$BUILD_DIR/pa_scheduler_host" + +# host 和 kernel 全部成功后才发布统一 manifest。swimlane 的两件套与 +# submit-pmu 的四件套都由 v4 十二行身份头固化 mode/variant/phase、 +# 物理泳道布局和 SHA256； +# run.sh 只消费带完整 manifest 的目录，因此中断重编不会混用新旧镜像。 +if [[ "$BUILD_VARIANT" == "submit-pmu" ]]; then + ARTIFACTS=( + pa_scheduler_host + pa_scheduler_kernel.o + libpa_scheduler_pmu_owner_aicpu.so + libpa_scheduler_pmu_owner_dispatcher.so + ) +else + ARTIFACTS=( + pa_scheduler_host + pa_scheduler_kernel.o + ) +fi +for artifact in "${ARTIFACTS[@]}"; do + if [[ ! -s "$BUILD_DIR/$artifact" ]]; then + echo "Cannot publish CCEC manifest; artifact is missing or empty: $artifact" >&2 + exit 1 + fi +done +if [[ ! -x "$BUILD_DIR/pa_scheduler_host" ]]; then + echo "Cannot publish CCEC manifest; host runner is not executable." >&2 + exit 1 +fi + +MANIFEST_PATH="$BUILD_DIR/$ARTIFACT_MANIFEST_NAME" +MANIFEST_TMP="$(mktemp "$BUILD_DIR/.${ARTIFACT_MANIFEST_NAME}.tmp.XXXXXX")" +cleanup_manifest_tmp() { + if [[ -n "${MANIFEST_TMP:-}" ]]; then + rm -f -- "$MANIFEST_TMP" + fi +} +trap cleanup_manifest_tmp EXIT +{ + printf '# schema=pa_scheduler_artifacts/v4\n' + printf '# tensormap_mode=%s\n' "$TENSORMAP_MODE" + printf '# tensormap_mode_id=%u\n' "$TENSORMAP_MODE_ID" + printf '# tensormap_ring_cap=%u\n' "$TENSORMAP_RING_CAP" + printf '# shared_insert_turn_groups=%u\n' \ + "$SHARED_INSERT_TURN_GROUPS" + printf '# generic_record_bytes=%u\n' \ + "$TRACE_GENERIC_RECORD_BYTES" + printf '# submit_claim_record_bytes=%u\n' \ + "$TRACE_SUBMIT_CLAIM_RECORD_BYTES" + printf '# records_per_core=%u\n' \ + "$TRACE_RECORDS_PER_CORE" + printf '# worker_stride_bytes=%u\n' \ + "$TRACE_WORKER_STRIDE_BYTES" + printf '# variant=%s\n' "$BUILD_VARIANT" + printf '# phase=%s\n' "$PHASE_NAME" + printf '# phase_id=%u\n' "$PHASE_ID" + (cd "$BUILD_DIR" && sha256sum "${ARTIFACTS[@]}") +} > "$MANIFEST_TMP" +mv -f -- "$MANIFEST_TMP" "$MANIFEST_PATH" +MANIFEST_TMP="" +trap - EXIT +echo "[CHECK] CCEC artifact manifest published: $MANIFEST_PATH" + +echo "[BUILD] complete: $BUILD_DIR" diff --git a/tests/atomic_probe/pa_scheduler/ccec/callback_finish.cpp b/tests/atomic_probe/pa_scheduler/ccec/callback_finish.cpp new file mode 100644 index 0000000000..92737a3656 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/callback_finish.cpp @@ -0,0 +1,46 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include +#include +#include + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" +#include "ccec_ops.h" + +using pa_scheduler_ccec::CcecOps; + +// finish 保持唯一的 cross-TU noinline 边界。入口只转交固定 POD ticket 与 +// TaskArgs；协议校验、Materialize/Register/Fanin/Build 均在公共实现中完成。 +extern "C" { +#if defined(PA_BUILD_AIC) +__attribute__((noinline)) __aicore__ uint32_t +pa_scheduler_compete_first_callback_finish_aic( + const pa_scheduler::CallbackSubmitTicket *ticket, const pa_scheduler::TaskArgs *args +) { + return pa_scheduler::FinishSplitCallbackSubmitFromRuntime(ticket, args); +} +#elif defined(PA_BUILD_AIV) +__attribute__((noinline)) __aicore__ uint32_t +pa_scheduler_compete_first_callback_finish_aiv( + const pa_scheduler::CallbackSubmitTicket *ticket, const pa_scheduler::TaskArgs *args +) { + return pa_scheduler::FinishSplitCallbackSubmitFromRuntime(ticket, args); +} +#else +#error "Compile split finish with PA_BUILD_AIC or PA_BUILD_AIV" +#endif +} diff --git a/tests/atomic_probe/pa_scheduler/ccec/callback_finish_api.h b/tests/atomic_probe/pa_scheduler/ccec/callback_finish_api.h new file mode 100644 index 0000000000..397379a4b4 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/callback_finish_api.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#ifndef TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CALLBACK_FINISH_API_H +#define TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CALLBACK_FINISH_API_H + +#if !defined(PA_COMPETE_FIRST_SPLIT_FINISH) +#error "callback_finish_api.h is only valid for split-finish artifacts" +#endif + +// runtime TU 拥有按核型区分的 block-local 实例；caller/finish 只导入对应 +// strong 符号。mixed AIC/AIV ELF 中禁止 weak 或不区分核型的通用状态。 +extern "C" { +#if defined(PA_BUILD_AIC) +[[block_local]] extern pa_scheduler::CompeteFirstSplitRuntimeState pa_scheduler_compete_first_callback_state_aic; +__aicore__ uint32_t +pa_scheduler_compete_first_callback_finish_aic(const pa_scheduler::CallbackSubmitTicket *ticket, const pa_scheduler::TaskArgs *args); +#elif defined(PA_BUILD_AIV) +[[block_local]] extern pa_scheduler::CompeteFirstSplitRuntimeState pa_scheduler_compete_first_callback_state_aiv; +__aicore__ uint32_t +pa_scheduler_compete_first_callback_finish_aiv(const pa_scheduler::CallbackSubmitTicket *ticket, const pa_scheduler::TaskArgs *args); +#else +#error "Compile split finish with PA_BUILD_AIC or PA_BUILD_AIV" +#endif +} + +#endif // TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CALLBACK_FINISH_API_H diff --git a/tests/atomic_probe/pa_scheduler/ccec/callback_runtime_entry.cpp b/tests/atomic_probe/pa_scheduler/ccec/callback_runtime_entry.cpp new file mode 100644 index 0000000000..c9c8376c5a --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/callback_runtime_entry.cpp @@ -0,0 +1,58 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" + +extern "C" { +#if defined(PA_BUILD_AIC) +// AIC 与 AIV 分别定义自己的 block-local 状态和 launch entry；不能退化成 +// 共用弱符号，否则 mixed ELF 中两个核型会误用同一套 Submit 上下文。 +[[block_local]] pa_scheduler::CompeteFirstSplitRuntimeState pa_scheduler_compete_first_callback_state_aic; +__aicore__ void +pa_scheduler_compete_first_callback_orchestration_aic( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +); +#elif defined(PA_BUILD_AIV) +[[block_local]] pa_scheduler::CompeteFirstSplitRuntimeState pa_scheduler_compete_first_callback_state_aiv; +__aicore__ void +pa_scheduler_compete_first_callback_orchestration_aiv( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +); +#else +#error "Compile split runtime entry with PA_BUILD_AIC or PA_BUILD_AIV" +#endif +} + +#if defined(PA_BUILD_AIC) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aic, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aic(__gm__ pa_scheduler::SchedulerState *state) { + // runtime entry 只负责物理核到 worker 的映射；调度主循环位于 caller TU。 + const uint32_t worker_id = static_cast(get_block_idx()); + pa_scheduler_compete_first_callback_orchestration_aic(state, worker_id); +} +#elif defined(PA_BUILD_AIV) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aiv, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aiv(__gm__ pa_scheduler::SchedulerState *state) { + // 两个 vector sub-block 展平后继续使用 standalone 的连续 worker 编号。 + const uint32_t vector_id = static_cast(get_block_idx() * get_subblockdim() + get_subblockid()); + const uint32_t worker_id = pa_scheduler::kAicWorkers + vector_id; + pa_scheduler_compete_first_callback_orchestration_aiv(state, worker_id); +} +#endif diff --git a/tests/atomic_probe/pa_scheduler/ccec/ccec_ops.h b/tests/atomic_probe/pa_scheduler/ccec/ccec_ops.h new file mode 100644 index 0000000000..c25253d222 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/ccec_ops.h @@ -0,0 +1,455 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#ifndef TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CCEC_OPS_H +#define TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CCEC_OPS_H + +// 此私有头在 CCEC/PTO 与公共调度头之后包含。kernel.cpp 每个核型只定义 +// 一次真实负载实体；split finish TU 复用按核型导出的 dispatcher 与同一份 +// inline Ops，避免复制或改写 atomic/cache/计时语义。 +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) +#include "callback_finish_api.h" +#endif + +#if defined(PA_BUILD_AIC) +extern "C" __attribute__((noinline)) __aicore__ void pa_execute_real_winner_workload_aic( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +); +#elif defined(PA_BUILD_AIV) +extern "C" __attribute__((noinline)) __aicore__ void pa_execute_real_winner_workload_aiv( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +); +#else +#error "Compile CcecOps with PA_BUILD_AIC or PA_BUILD_AIV" +#endif + +namespace pa_scheduler_ccec { + +using namespace pto; + +template +__aicore__ inline void EmitNops() { +#pragma unroll + for (uint32_t index = 0; index < Count; ++index) { + asm volatile("nop"); + } +} + +__aicore__ inline void RuntimeNop(uint32_t count) { + // 两侧全流水屏障把可调 NOP 段限定为 kernel 模拟体，避免前后调度访存进入被测计算区间。 + __builtin_cce_pipe_barrier(PIPE_ALL); + while (count >= 256) { + EmitNops<256>(); + count -= 256; + } + if ((count & 128U) != 0) EmitNops<128>(); + if ((count & 64U) != 0) EmitNops<64>(); + if ((count & 32U) != 0) EmitNops<32>(); + if ((count & 16U) != 0) EmitNops<16>(); + if ((count & 8U) != 0) EmitNops<8>(); + if ((count & 4U) != 0) EmitNops<4>(); + if ((count & 2U) != 0) EmitNops<2>(); + if ((count & 1U) != 0) EmitNops<1>(); + __builtin_cce_pipe_barrier(PIPE_ALL); +} + +#if defined(PA_CCEC_OPS_DEFINE_REAL_WORKLOAD) && defined(PA_BUILD_AIC) +// QK/PV 的首版真实负载使用完整的 128x128 float cube 路径。输入来自独立 GM +// workspace，输出属于当前 worker；每次迭代都等 FIX 写回 GM 后再复用 L0C， +// 因而函数返回就是该模拟 task 的完成边界，而不是单纯的指令发射边界。 +static __aicore__ __attribute__((noinline, used)) void pa_real_cube_workload_aic( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + constexpr int kTile = static_cast(pa_scheduler::winner_workload::kTileRows); + constexpr int kBlockAlign = C0_SIZE_BYTE / sizeof(float); + static_assert(kTile % 16 == 0, "cube M must be 16-aligned"); + static_assert(kTile % kBlockAlign == 0, "cube K/N must satisfy C0 alignment"); + + using GlobalData = GlobalTensor< + float, Shape<1, 1, 1, kTile, kTile>, + pto::Stride>; + using TileMatA = Tile< + TileType::Mat, float, kTile, kTile, BLayout::ColMajor, + kTile, kTile, SLayout::RowMajor, 512>; + using TileMatB = Tile< + TileType::Mat, float, kTile, kTile, BLayout::ColMajor, + kTile, kTile, SLayout::RowMajor, 512>; + using LeftTile = TileLeft; + using RightTile = TileRight; + using AccTile = TileAcc; + + GlobalData input_a_global(input_a); + GlobalData input_b_global(input_b); + GlobalData output_global(output); + TileMatA input_a_mat; + TileMatB input_b_mat; + LeftTile input_a_l0; + RightTile input_b_l0; + AccTile output_l0; + TASSIGN(input_a_mat, 0x0); + TASSIGN(input_b_mat, 0x20000); + TASSIGN(input_a_l0, 0x0); + TASSIGN(input_b_l0, 0x0); + TASSIGN(output_l0, 0x0); + + for (uint32_t iteration = 0; iteration < repeats; ++iteration) { + TLOAD(input_a_mat, input_a_global); + TLOAD(input_b_mat, input_b_global); + set_flag(PIPE_MTE2, PIPE_MTE1, EVENT_ID0); + wait_flag(PIPE_MTE2, PIPE_MTE1, EVENT_ID0); + TMOV(input_a_l0, input_a_mat); + TMOV(input_b_l0, input_b_mat); + set_flag(PIPE_MTE1, PIPE_M, EVENT_ID0); + wait_flag(PIPE_MTE1, PIPE_M, EVENT_ID0); + TMATMUL(output_l0, input_a_l0, input_b_l0); + set_flag(PIPE_M, PIPE_FIX, EVENT_ID0); + wait_flag(PIPE_M, PIPE_FIX, EVENT_ID0); + TSTORE(output_global, output_l0); + set_flag(PIPE_FIX, PIPE_S, EVENT_ID7); + wait_flag(PIPE_FIX, PIPE_S, EVENT_ID7); + } +} +#elif defined(PA_CCEC_OPS_DEFINE_REAL_WORKLOAD) && defined(PA_BUILD_AIV) +template +__aicore__ inline void RunRealVectorWorkload( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + constexpr int kRows = static_cast(pa_scheduler::winner_workload::kTileRows); + constexpr int kCols = static_cast(pa_scheduler::winner_workload::kTileCols); + using GlobalData = GlobalTensor< + float, Shape<1, 1, 1, kRows, kCols>, pto::Stride<1, 1, 1, kCols, 1>>; + using TileData = Tile< + TileType::Vec, float, kRows, kCols, BLayout::RowMajor, -1, -1>; + + GlobalData input_a_global(input_a); + GlobalData input_b_global(input_b); + GlobalData output_global(output); + TileData input_a_tile(kRows, kCols); + TileData input_b_tile(kRows, kCols); + TileData output_tile(kRows, kCols); + TASSIGN(input_a_tile, 0x0); + TASSIGN(input_b_tile, 0x10000); + TASSIGN(output_tile, 0x20000); + + for (uint32_t iteration = 0; iteration < repeats; ++iteration) { + TLOAD(input_a_tile, input_a_global); + TLOAD(input_b_tile, input_b_global); + set_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + wait_flag(PIPE_MTE2, PIPE_V, EVENT_ID0); + if constexpr (Multiply) { + TMUL(output_tile, input_a_tile, input_b_tile); + } else { + TADD(output_tile, input_a_tile, input_b_tile); + } + set_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + wait_flag(PIPE_V, PIPE_MTE3, EVENT_ID0); + TSTORE(output_global, output_tile); + set_flag(PIPE_MTE3, PIPE_S, EVENT_ID7); + wait_flag(PIPE_MTE3, PIPE_S, EVENT_ID7); + } +} + +static __aicore__ __attribute__((noinline, used)) void pa_real_vector_add_workload_aiv( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + RunRealVectorWorkload(input_a, input_b, output, repeats); +} + +static __aicore__ __attribute__((noinline, used)) void pa_real_vector_mul_workload_aiv( + __gm__ float *input_a, __gm__ float *input_b, __gm__ float *output, uint32_t repeats +) { + RunRealVectorWorkload(input_a, input_b, output, repeats); +} +#endif + +#if PA_BUILD_SUBMIT_PMU +struct SubmitPmuContext; +#endif + +struct CcecOps { + static constexpr bool kAtomicReturnReadyObserved = true; + + // 该适配层把平台无关调度器需要的原子、计时、NOP 和 cache 操作逐一映射到 CCEC intrinsic。 + // A5 上 PA 的共享“读取”使用 atomicAdd(addr, 0)，不是普通 GM load；这里保留其 RMW 竞争语义。 + __aicore__ static inline int32_t Load(__gm__ volatile int32_t *address) { + // atomicAdd 返回加法发生前的值；加数为 0，因此它就是本次共享读取的结果。 + return atomicAdd(const_cast<__gm__ int32_t *>(address), static_cast(0)); + } + + __aicore__ static inline int64_t Load(__gm__ volatile int64_t *address) { + return atomicAdd(const_cast<__gm__ int64_t *>(address), static_cast(0)); + } + + __aicore__ static inline uint64_t Load(__gm__ volatile uint64_t *address) { + return atomicAdd(const_cast<__gm__ uint64_t *>(address), static_cast(0)); + } + + __aicore__ static inline int32_t Exchange(__gm__ volatile int32_t *address, int32_t value) { + // atomicExch 同样返回旧值；当前 completion/fatal 发布只需要其原子写入副作用。 + return atomicExch(const_cast<__gm__ int32_t *>(address), value); + } + + __aicore__ static inline int64_t Exchange(__gm__ volatile int64_t *address, int64_t value) { + return atomicExch(const_cast<__gm__ int64_t *>(address), value); + } + + __aicore__ static inline uint64_t Exchange(__gm__ volatile uint64_t *address, uint64_t value) { + return atomicExch(const_cast<__gm__ uint64_t *>(address), value); + } + + __aicore__ static inline int64_t CompareExchange( + __gm__ volatile int64_t *address, int64_t expected, int64_t desired + ) { + // CCEC atomicCAS 与 production wrapper 同样返回操作前旧值。它只 + // 负责控制字的原子线性化；相邻 payload 的发布仍由既有 DCCI + // Flush/Invalidate 协议负责，不能把 CAS 本身解释成数据屏障。 + return atomicCAS( + const_cast<__gm__ int64_t *>(address), expected, desired + ); + } + + __aicore__ static inline int64_t FetchAdd(__gm__ volatile int64_t *address, int64_t value) { + // 返回递增前的计数；启动和 replay 屏障只关心全局累加结果，因此调用方不使用该返回值。 + return atomicAdd(const_cast<__gm__ int64_t *>(address), value); + } + + __aicore__ static inline int64_t FetchMax(__gm__ volatile int64_t *address, int64_t value, uint64_t &retries) { + // CCEC 直接生成单条硬件 atomicMax，不存在 CPU CAS 循环可观测的重试次数。 + retries = 0; + // 返回更新前的 cursor/frontier，Claim 用它判定 winner，frontier 扫描用它吸收其他核的进度。 + return atomicMax(const_cast<__gm__ int64_t *>(address), value); + } + + // PA's A5 OUT_OF_ORDER_STORE_BARRIER is intentionally a no-op; cache + // coherency is handled by the runtime's DCCI protocol. + // 这是对生产 A5 契约的刻意复刻，不是遗漏 barrier；若在这里额外插入 dsb， + // 会改变待测 Submit 热路径。completion 使用 atomic；config、trace 与 + // shared TensorMap payload 的 cache 可见性由下方既有 DCCI hook 处理。 + __aicore__ static inline void StoreBarrier() {} + + __aicore__ static inline uint64_t Now() { return static_cast(get_sys_cnt()); } + +#if PA_BUILD_PERF_CLOCK + // 专用接口只允许出现在公共调度器的首个/末个 Submit 条件分支。 + // 普通 Now 仍服务 watchdog；区分命名让源码审计不会把正确性超时读 + // 误算成新增的性能观察点。 + __aicore__ static inline uint64_t PerfClockNow() { + return static_cast(get_sys_cnt()); + } +#endif + + template + __aicore__ static inline uint64_t NowAfterAtomicResult(T value) { + static_assert(sizeof(T) == 4 || sizeof(T) == 8, "atomic dependency expects a scalar result"); + uint64_t cycle = 0; + // 同一个 inline asm 块先真正消费 atomic 返回寄存器，再读取 + // SYS_CNT；编译器不能把 t1 拆到依赖 MOV 之前。AIC/AIV 对该序列 + // 生成相同指令字节，且不增加 DSB/ISB/GM 访存。该边界仍只表示 + // 返回值已可被本核 scalar 消费，不表示跨核全局可见。 + asm volatile( + "MOV %0, %0\n" + "MOV %1, SYS_CNT\n" + : "+l"(value), "=&l"(cycle) + ); + return cycle; + } + + template + __aicore__ static inline T ForkAtomicResultForBranch( + T value, T &dependency_value + ) { + static_assert( + sizeof(T) == 4 || sizeof(T) == 8, + "atomic dependency fork expects a scalar result" + ); + T compare_value = value; + // 一个 atomic 返回值同时用于 Ready 判定和随后 SYS_CNT 取时。 + // 若直接把同一 SSA 值跨过 `observed == task_id` 分支继续传递， + // O3 会利用 true 分支事实把它常量传播成 task_id，切断真实返回 + // 依赖。这里用一条 MOV 生成两个编译器不可证明相等、硬件值完全 + // 相同的输出：compare_value 只参与分支，dependency_value 只 + // 参与 Ready 路径取时。该序列不读时钟、不访问 GM，也不加屏障。 + asm volatile( + "MOV %1, %0\n" + : "+l"(compare_value), "=&l"(dependency_value) + ); + return compare_value; + } + + __aicore__ static inline void ExecuteKernel( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState &worker, + pa_scheduler::TaskKind kind, uint32_t nop_count + ) { + const auto mode = static_cast(state->winner_workload.mode); + if (mode != pa_scheduler::WinnerWorkloadMode::RealCompute) { + RuntimeNop(nop_count); + return; + } +#if defined(PA_BUILD_AIC) + ::pa_execute_real_winner_workload_aic(state, &worker, kind); +#elif defined(PA_BUILD_AIV) + ::pa_execute_real_winner_workload_aiv(state, &worker, kind); +#endif + } + +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) + __aicore__ static inline pa_scheduler::CompeteFirstSplitRuntimeState &CompeteFirstSplitState() { +#if defined(PA_BUILD_AIC) + return ::pa_scheduler_compete_first_callback_state_aic; +#elif defined(PA_BUILD_AIV) + return ::pa_scheduler_compete_first_callback_state_aiv; +#endif + } + + __aicore__ static inline bool FinishCompeteFirstCallback( + const pa_scheduler::CallbackSubmitTicket *ticket, const pa_scheduler::TaskArgs *args + ) { +#if defined(PA_BUILD_AIC) + return ::pa_scheduler_compete_first_callback_finish_aic(ticket, args) != 0; +#elif defined(PA_BUILD_AIV) + return ::pa_scheduler_compete_first_callback_finish_aiv(ticket, args) != 0; +#endif + } +#endif + +#if PA_BUILD_SUBMIT_PMU + using PmuContext = SubmitPmuContext; + + __aicore__ static inline PmuContext PmuWindowStart( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id + ); + + __aicore__ static inline void PmuPhaseBegin(PmuContext &context); + + __aicore__ static inline void PmuPhaseEnd(PmuContext &context); + + __aicore__ static inline void PmuWindowStop( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id, PmuContext &context + ); +#else + // swimlane 产物不携带 PMU 读寄存器或门控代码；公共调度器保留同一 hook + // 形状，编译器会把这两个空实现完整消去。 + __aicore__ static inline bool PmuWindowStart( + __gm__ pa_scheduler::SchedulerState *, uint32_t + ) { return false; } + + __aicore__ static inline void PmuWindowStop( + __gm__ pa_scheduler::SchedulerState *, uint32_t, bool + ) {} +#endif + + // SPIN_WAIT_HINT is also a no-op in the real A5 inner-kernel contract. + // 同理不额外插入 nop，让等待循环保留真实 PA 内核“不主动退避”的指令成本。 + __aicore__ static inline void SpinHint() {} + + __aicore__ static inline void InvalidateRegion(__gm__ const void *address, uint64_t bytes) { + // 逐 cache line 失效并以 dsb 收口：既供 worker 读取 host 写入的 + // standalone 控制区，也供 shared TensorMap reader 观察跨核 payload。 + if (bytes == 0) return; + const uint64_t start = reinterpret_cast(address) & ~uint64_t{63}; + const uint64_t end = (reinterpret_cast(address) + bytes + 63) & ~uint64_t{63}; + for (uint64_t current = start; current < end; current += 64) { + dcci(reinterpret_cast<__gm__ uint8_t *>(current), SINGLE_CACHE_LINE); + } + dsb((mem_dsb_t)0); + // CCEC 9.1 把 DCCI/DSB 标成只读写 inaccessible memory，不能据此 + // 约束普通 GM load。后置 compiler barrier 不生成设备指令，只保证 + // 调用方在失效后读取的 header/payload 不会被合并或上提到 DCCI 前。 + __asm__ volatile("" ::: "memory"); + } + + __aicore__ static inline void FlushRegion(__gm__ void *address, uint64_t bytes) { + // 普通 GM cache 内容逐 line clean-out 并以 dsb 收口：泳道记录据此 + // 对 host 可见，shared TensorMap writer 也复用同一 hook 发布 payload。 + if (bytes == 0) return; + __asm__ volatile("" ::: "memory"); + const uint64_t start = reinterpret_cast(address) & ~uint64_t{63}; + const uint64_t end = (reinterpret_cast(address) + bytes + 63) & ~uint64_t{63}; + for (uint64_t current = start; current < end; current += 64) { + dcci(reinterpret_cast<__gm__ uint8_t *>(current), SINGLE_CACHE_LINE, CACHELINE_OUT); + } + dsb((mem_dsb_t)0); + } + + __aicore__ static inline void Publish(__gm__ uint64_t *address, uint64_t value) { + // 每核独占的 WorkerResult 用 bypass-DCache store 发布，host 同步后可直接 D2H，无需共享原子竞争。 + __builtin_cce_st_dev(value, address, 0); + } + + __aicore__ static inline void Publish(__gm__ uint32_t *address, uint32_t value) { + __builtin_cce_st_dev(value, address, 0); + } +}; + +} // namespace pa_scheduler_ccec + +#if defined(PA_CCEC_OPS_DEFINE_REAL_WORKLOAD) +// 跨 TU 只暴露按架构区分的真实负载分派；Cube/Vector 实体仍保持 LOCAL。 +// 最终 mixed ELF 由 version script 把该 strong 定义重新局部化，避免成为 kernel entry。 +#if defined(PA_BUILD_AIC) +extern "C" __attribute__((noinline, used)) __aicore__ void pa_execute_real_winner_workload_aic( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +) { +#elif defined(PA_BUILD_AIV) +extern "C" __attribute__((noinline, used)) __aicore__ void pa_execute_real_winner_workload_aiv( + __gm__ pa_scheduler::SchedulerState *state, __gm__ pa_scheduler::WorkerState *worker, + pa_scheduler::TaskKind kind +) { +#endif + const uint64_t workspace = state->winner_workload.workspace_base; + const uint32_t repeats = pa_scheduler::WorkloadCountForKind( + state->winner_workload.repeats, kind + ); + // 错版 host、截断 workspace 或越界 worker 不能继续解引用 GM。这里不额外 + // 写共享 fatal，避免在正常热路增加 atomic；host 的逐 kind sentinel/数值 + // 闭环会把这种配置错误判为失败。 + if (state->winner_workload.version != pa_scheduler::kWinnerWorkloadConfigVersion || + workspace == 0 || + state->winner_workload.workspace_bytes < pa_scheduler::winner_workload::kWorkspaceBytes || + worker->core_idx < 0 || static_cast(worker->core_idx) >= pa_scheduler::kWorkers || + repeats == 0 || repeats > pa_scheduler::winner_workload::kMaxRealComputeCount) { + return; + } +#if defined(PA_BUILD_AIC) + if (kind != pa_scheduler::TaskKind::Qk && kind != pa_scheduler::TaskKind::Pv) return; +#elif defined(PA_BUILD_AIV) + if (kind != pa_scheduler::TaskKind::Sf && kind != pa_scheduler::TaskKind::Up) return; +#endif + __gm__ float *input_a = reinterpret_cast<__gm__ float *>(workspace); + __gm__ float *input_b = reinterpret_cast<__gm__ float *>( + workspace + pa_scheduler::winner_workload::kTileBytes + ); + const uint32_t kind_slot = + (kind == pa_scheduler::TaskKind::Pv || kind == pa_scheduler::TaskKind::Up) ? 1U : 0U; + const uint32_t output_tile = + pa_scheduler::winner_workload::kSharedInputTiles + + static_cast(worker->core_idx) * + pa_scheduler::winner_workload::kOutputTilesPerWorker + + kind_slot; + __gm__ float *output = reinterpret_cast<__gm__ float *>( + workspace + static_cast(output_tile) * + pa_scheduler::winner_workload::kTileBytes + ); +#if defined(PA_BUILD_AIC) + pa_scheduler_ccec::pa_real_cube_workload_aic(input_a, input_b, output, repeats); +#elif defined(PA_BUILD_AIV) + if (kind == pa_scheduler::TaskKind::Sf) { + pa_scheduler_ccec::pa_real_vector_add_workload_aiv(input_a, input_b, output, repeats); + } else { + pa_scheduler_ccec::pa_real_vector_mul_workload_aiv(input_a, input_b, output, repeats); + } +#endif +} +#endif // PA_CCEC_OPS_DEFINE_REAL_WORKLOAD + +#endif // TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_CCEC_OPS_H diff --git a/tests/atomic_probe/pa_scheduler/ccec/host.cpp b/tests/atomic_probe/pa_scheduler/ccec/host.cpp new file mode 100644 index 0000000000..9483239c13 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/host.cpp @@ -0,0 +1,2302 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "../common/host_support.h" +#include "../common/winner_workload_host.h" +#include "pmu_owner_host.h" +#include "pmu_probe.h" + +#include "acl/acl.h" +#include "driver/ascend_hal.h" +#include "runtime/rt.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +bool CheckAcl(aclError error, const char *label) { + if (error == ACL_SUCCESS) return true; + std::fprintf(stderr, "ACL error %d: %s\n", static_cast(error), label); + return false; +} + +bool CheckRt(rtError_t error, const char *label) { + if (error == RT_ERROR_NONE) return true; + std::fprintf(stderr, "RT error %d: %s\n", static_cast(error), label); + return false; +} + +class ScopedAclDeviceAllocation { +public: + ScopedAclDeviceAllocation() = default; + ScopedAclDeviceAllocation(const ScopedAclDeviceAllocation &) = delete; + ScopedAclDeviceAllocation &operator=(const ScopedAclDeviceAllocation &) = delete; + + ~ScopedAclDeviceAllocation() { + // 早退路径没有机会汇入末尾 cleanup；这里只负责尽力释放本类新增的 + // real-compute workspace。正常路径会先 Release，再检查 aclrtFree 返回值。 + if (pointer_ != nullptr) (void)aclrtFree(pointer_); + } + + void **Address() { return &pointer_; } + void *Get() const { return pointer_; } + + void *Release() { + void *pointer = pointer_; + pointer_ = nullptr; + return pointer; + } + +private: + void *pointer_ = nullptr; +}; + +std::vector ReadBinary(const std::string &path) { + // ELF 整体保存在 vector 中直到 runtime 卸载完成，保证 rtDevBinary_t.data 在整个注册生命周期内有效。 + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) return {}; + const std::streamsize size = file.tellg(); + if (size <= 0) return {}; + std::vector data(static_cast(size)); + file.seekg(0, std::ios::beg); + if (!file.read(data.data(), size)) return {}; + return data; +} + +struct PmuOptions { + pa_scheduler::ccec_pmu::WindowMode mode = pa_scheduler::ccec_pmu::WindowMode::Off; + uint32_t scalar_nops = 100000; + uint32_t icache_trials = 64; + std::string json_path; +}; + +using pa_scheduler::host::ConfigureWinnerWorkload; +using pa_scheduler::host::ParseWinnerWorkloadOptions; +using pa_scheduler::host::ValidateRealComputeOutputs; +using pa_scheduler::host::ValidateWinnerWorkloadOptions; +using pa_scheduler::host::WinnerWorkloadModeName; +using pa_scheduler::host::WinnerWorkloadOptions; + +const char *PmuModeName(pa_scheduler::ccec_pmu::WindowMode mode) { + switch (mode) { + case pa_scheduler::ccec_pmu::WindowMode::Off: + return "off"; + case pa_scheduler::ccec_pmu::WindowMode::Empty: + return "empty"; + case pa_scheduler::ccec_pmu::WindowMode::Scalar: + return "scalar"; + case pa_scheduler::ccec_pmu::WindowMode::ScalarDouble: + return "scalar-double"; + case pa_scheduler::ccec_pmu::WindowMode::IcacheSingle: + return "icache-single"; + case pa_scheduler::ccec_pmu::WindowMode::SubmitAll: + return "submit-all"; + } + return "invalid"; +} + +bool ParsePmuOptions(int argc, char **argv, PmuOptions *pmu, std::vector *common_argv) { + // PMU 参数只属于 CCEC 验证分支；先摘出再交给三后端共享 parser，避免 CPU/AscendC 静默接受却不生效。 + bool mode_seen = false; + bool nops_seen = false; + bool icache_trials_seen = false; + bool json_seen = false; + common_argv->clear(); + common_argv->push_back(argv[0]); + for (int index = 1; index < argc; ++index) { + const std::string argument = argv[index]; + if (argument != "--pmu-window" && argument != "--pmu-scalar-nops" && + argument != "--pmu-icache-trials" && argument != "--pmu-json") { + common_argv->push_back(argv[index]); + continue; + } + if (index + 1 >= argc) { + std::fprintf(stderr, "Missing value after %s\n", argument.c_str()); + return false; + } + const char *value = argv[++index]; + if (argument == "--pmu-window") { + if (mode_seen) { + std::fprintf(stderr, "Specify --pmu-window only once.\n"); + return false; + } + const std::string name = value; + if (name == "off") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::Off; + } else if (name == "empty") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::Empty; + } else if (name == "scalar") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::Scalar; + } else if (name == "scalar-double") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::ScalarDouble; + } else if (name == "icache-single") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::IcacheSingle; + } else if (name == "submit-all") { + pmu->mode = pa_scheduler::ccec_pmu::WindowMode::SubmitAll; + } else { + std::fprintf( + stderr, + "Invalid --pmu-window value: %s " + "(expected off|empty|scalar|scalar-double|icache-single|submit-all)\n", + value + ); + return false; + } + mode_seen = true; + } else if (argument == "--pmu-scalar-nops") { + if (nops_seen || !pa_scheduler::host::ParseUint(value, 0, 10000000, &pmu->scalar_nops)) { + std::fprintf(stderr, "Invalid or duplicate --pmu-scalar-nops value: %s\n", value); + return false; + } + nops_seen = true; + } else if (argument == "--pmu-icache-trials") { + if (icache_trials_seen || + !pa_scheduler::host::ParseUint(value, 1, 10000, &pmu->icache_trials)) { + std::fprintf(stderr, "Invalid or duplicate --pmu-icache-trials value: %s\n", value); + return false; + } + icache_trials_seen = true; + } else { + if (json_seen || *value == '\0') { + std::fprintf(stderr, "Invalid or duplicate --pmu-json path: %s\n", value); + return false; + } + pmu->json_path = value; + json_seen = true; + } + } + if (nops_seen && pmu->mode != pa_scheduler::ccec_pmu::WindowMode::Scalar && + pmu->mode != pa_scheduler::ccec_pmu::WindowMode::ScalarDouble) { + std::fprintf(stderr, "--pmu-scalar-nops requires a scalar PMU window.\n"); + return false; + } + if (icache_trials_seen && pmu->mode != pa_scheduler::ccec_pmu::WindowMode::IcacheSingle) { + std::fprintf(stderr, "--pmu-icache-trials requires the icache-single PMU window.\n"); + return false; + } + return true; +} + +using HalResMapFn = int (*)(uint32_t, struct res_map_info *, unsigned long *, uint32_t *); +using HalResUnmapFn = int (*)(uint32_t, struct res_map_info *); + +struct PmuRegisterMappings { + HalResUnmapFn unmap = nullptr; + std::vector mapped_resources; + std::vector register_bases; +}; + +bool UnmapPmuRegisters(uint32_t device, PmuRegisterMappings *mappings) { + bool ok = true; + if (mappings->unmap != nullptr) { + for (auto iterator = mappings->mapped_resources.rbegin(); iterator != mappings->mapped_resources.rend(); + ++iterator) { + const int error = mappings->unmap(device, &*iterator); + if (error != 0) { + std::fprintf(stderr, "halResUnmap failed for core %u (rc=%d)\n", iterator->res_id, error); + ok = false; + } + } + } + mappings->mapped_resources.clear(); + mappings->register_bases.clear(); + return ok; +} + +bool MapPmuRegisters(uint32_t device, PmuRegisterMappings *mappings) { + using namespace pa_scheduler::ccec_pmu; + const auto map = reinterpret_cast(dlsym(RTLD_DEFAULT, "halResMap")); + mappings->unmap = reinterpret_cast(dlsym(RTLD_DEFAULT, "halResUnmap")); + if (map == nullptr || mappings->unmap == nullptr) { + std::fprintf(stderr, "halResMap/halResUnmap is unavailable in the current CANN driver process.\n"); + return false; + } + + mappings->register_bases.assign(kPhysicalSubcoreCount, 0); + mappings->mapped_resources.reserve(kPhysicalAicoreCount); + for (uint32_t aicore = 0; aicore < kPhysicalAicoreCount; ++aicore) { + res_map_info info{}; + info.target_proc_type = PROCESS_CP1; + info.res_type = RES_AICORE; + info.res_id = aicore; + unsigned long map_address = 0; + uint32_t map_bytes = kAicoreMapBytes; + const int error = map(device, &info, &map_address, &map_bytes); + if (error != 0 || map_address == 0 || map_bytes < kAicoreMapBytes) { + std::fprintf( + stderr, "halResMap failed for core %u (rc=%d address=0x%lx bytes=%u)\n", aicore, error, + map_address, map_bytes + ); + (void)UnmapPmuRegisters(device, mappings); + return false; + } + mappings->mapped_resources.push_back(info); + + // 与正式 A5 host_regs.cpp 相同：每个 die 的布局为 18 AIC，随后是 36 AIV。 + const uint32_t die = aicore / kAicorePerDie; + const uint32_t local = aicore % kAicorePerDie; + const uint32_t die_base = die * kSubcoresPerDie; + mappings->register_bases[die_base + local] = static_cast(map_address); + mappings->register_bases[die_base + kAicorePerDie + local * 2] = + static_cast(map_address) + kAivFirstOffset; + mappings->register_bases[die_base + kAicorePerDie + local * 2 + 1] = + static_cast(map_address) + kAivSecondOffset; + } + return true; +} + +void ConfigurePmu(pa_scheduler::SchedulerState *state, const PmuOptions &pmu, const void *register_table) { + using namespace pa_scheduler::ccec_pmu; + state->pmu_probe.mode = static_cast(pmu.mode); + state->pmu_probe.work_amount = + pmu.mode == WindowMode::IcacheSingle ? pmu.icache_trials : pmu.scalar_nops; + state->pmu_probe.register_table = reinterpret_cast(register_table); + state->pmu_probe.magic = pmu.mode == WindowMode::Off ? 0 : kConfigMagicValue; +} + +struct PmuAggregate { + std::vector total_cycles; + std::vector window_ticks; + std::vector submit_elapsed_ticks; + std::vector phase_elapsed_ticks; + std::vector warm_total_cycles; + std::vector warm_window_ticks; + std::vector vector_busy; + std::vector cube_busy; + std::vector scalar_busy; + std::vector mte1_busy; + std::vector mte2_busy; + std::vector icache_requests; + std::vector icache_misses; + std::vector warm_icache_requests; + std::vector warm_icache_misses; + std::vector fix_busy; + std::vector phase_calls; + std::vector phase_icache_requests; + std::vector phase_icache_misses; + std::vector shadow_icache_requests; + std::vector shadow_icache_misses; + uint32_t trusted = 0; +}; + +void AddPmuSample(const pa_scheduler::WorkerResult &result, PmuAggregate *aggregate) { + aggregate->total_cycles.push_back(result.pmu_total_cycles); + aggregate->window_ticks.push_back(result.pmu_window_ticks); + aggregate->submit_elapsed_ticks.push_back( + result.submit_end >= result.submit_begin ? result.submit_end - result.submit_begin : 0U + ); + aggregate->phase_elapsed_ticks.push_back(result.pmu_phase_elapsed_ticks); + aggregate->warm_total_cycles.push_back(result.pmu_warm_total_cycles); + aggregate->warm_window_ticks.push_back(result.pmu_warm_window_ticks); + aggregate->vector_busy.push_back(result.pmu_vector_busy); + aggregate->cube_busy.push_back(result.pmu_cube_busy); + aggregate->scalar_busy.push_back(result.pmu_scalar_busy); + aggregate->mte1_busy.push_back(result.pmu_mte1_busy); + aggregate->mte2_busy.push_back(result.pmu_mte2_busy); + aggregate->icache_requests.push_back(result.pmu_icache_requests); + aggregate->icache_misses.push_back(result.pmu_icache_misses); + aggregate->warm_icache_requests.push_back(result.pmu_warm_icache_requests); + aggregate->warm_icache_misses.push_back(result.pmu_warm_icache_misses); + aggregate->fix_busy.push_back(result.pmu_fix_busy); + aggregate->phase_calls.push_back(result.pmu_phase_calls); + aggregate->phase_icache_requests.push_back(result.pmu_phase_icache_requests); + aggregate->phase_icache_misses.push_back(result.pmu_phase_icache_misses); + aggregate->shadow_icache_requests.push_back(result.pmu_shadow_icache_requests); + aggregate->shadow_icache_misses.push_back(result.pmu_shadow_icache_misses); + aggregate->trusted += + (result.pmu_status & pa_scheduler::ccec_pmu::kStatusRequired) == + pa_scheduler::ccec_pmu::kStatusRequired && + (result.pmu_phase_status & pa_scheduler::ccec_pmu::kPhaseStatusRequired) == + pa_scheduler::ccec_pmu::kPhaseStatusRequired; +} + +bool PrintSingleIcacheAggregate( + const char *name, const PmuAggregate &aggregate, uint32_t trials_per_core +) { + const pa_scheduler::host::Uint64Distribution cold_cycles = + pa_scheduler::host::SummarizeUint64(aggregate.total_cycles); + const pa_scheduler::host::Uint64Distribution warm_cycles = + pa_scheduler::host::SummarizeUint64(aggregate.warm_total_cycles); + const pa_scheduler::host::Uint64Distribution cold_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.window_ticks); + const pa_scheduler::host::Uint64Distribution warm_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.warm_window_ticks); + const pa_scheduler::host::Uint64Distribution cold_requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution warm_requests = + pa_scheduler::host::SummarizeUint64(aggregate.warm_icache_requests); + const pa_scheduler::host::Uint64Distribution cold_misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + const pa_scheduler::host::Uint64Distribution warm_misses = + pa_scheduler::host::SummarizeUint64(aggregate.warm_icache_misses); + const int64_t cycle_delta = static_cast(cold_cycles.total) - + static_cast(warm_cycles.total); + const int64_t tick_delta = static_cast(cold_ticks.total) - + static_cast(warm_ticks.total); + const int64_t miss_delta = static_cast(cold_misses.total) - + static_cast(warm_misses.total); + const uint64_t attempted = static_cast(trials_per_core) * aggregate.total_cycles.size(); + const double misses_per_trial = attempted == 0U ? 0.0 : static_cast(miss_delta) / attempted; + const double cycles_per_miss = miss_delta <= 0 ? 0.0 : static_cast(cycle_delta) / miss_delta; + // 本用例的 get_sys_cnt 已按 1 GHz 时间基准校准，因此一个 tick 对应 1 ns。 + const double ns_per_miss = miss_delta <= 0 ? 0.0 : static_cast(tick_delta) / miss_delta; + std::printf( + "[ICACHE-SINGLE-%s] cores=%zu trials_per_core=%u attempted=%llu " + "cold_cycles=%llu warm_cycles=%llu cycle_delta=%lld cold_ticks=%llu warm_ticks=%llu " + "tick_delta=%lld cold_req=%llu warm_req=%llu cold_miss=%llu warm_miss=%llu " + "miss_delta=%lld misses_per_trial=%.6f cycles_per_miss=%.3f ns_per_miss=%.3f\n", + name, aggregate.total_cycles.size(), trials_per_core, static_cast(attempted), + static_cast(cold_cycles.total), + static_cast(warm_cycles.total), static_cast(cycle_delta), + static_cast(cold_ticks.total), + static_cast(warm_ticks.total), static_cast(tick_delta), + static_cast(cold_requests.total), + static_cast(warm_requests.total), + static_cast(cold_misses.total), + static_cast(warm_misses.total), static_cast(miss_delta), + misses_per_trial, cycles_per_miss, ns_per_miss + ); + std::printf( + "[ICACHE-FORMULA-%s] estimated_scalar_icache_time_ns = cnt7_icache_miss * %.3f\n", + name, ns_per_miss + ); + return cycle_delta > 0 && tick_delta > 0 && miss_delta == static_cast(attempted); +} + +void PrintPmuAggregate(const char *name, const PmuAggregate &aggregate) { + const pa_scheduler::host::Uint64Distribution total = + pa_scheduler::host::SummarizeUint64(aggregate.total_cycles); + const pa_scheduler::host::Uint64Distribution scalar = + pa_scheduler::host::SummarizeUint64(aggregate.scalar_busy); + const pa_scheduler::host::Uint64Distribution vector = + pa_scheduler::host::SummarizeUint64(aggregate.vector_busy); + const pa_scheduler::host::Uint64Distribution cube = + pa_scheduler::host::SummarizeUint64(aggregate.cube_busy); + const pa_scheduler::host::Uint64Distribution mte1 = + pa_scheduler::host::SummarizeUint64(aggregate.mte1_busy); + const pa_scheduler::host::Uint64Distribution mte2 = + pa_scheduler::host::SummarizeUint64(aggregate.mte2_busy); + const pa_scheduler::host::Uint64Distribution requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + const double miss_rate = requests.total == 0 ? 0.0 : 100.0 * misses.total / requests.total; + std::printf( + "[PMU-%s] cores=%zu total_sum=%llu total_median=%.1f total_p95=%llu " + "scalar_busy=%llu vector_busy=%llu cube_busy=%llu mte1_busy=%llu mte2_busy=%llu " + "icache_req=%llu icache_miss=%llu miss_rate=%.4f%%\n", + name, aggregate.total_cycles.size(), static_cast(total.total), total.median, + static_cast(total.p95), static_cast(scalar.total), + static_cast(vector.total), static_cast(cube.total), + static_cast(mte1.total), static_cast(mte2.total), + static_cast(requests.total), static_cast(misses.total), miss_rate + ); +} + +void PrintSubmitPmuPhaseAggregate(const char *name, const PmuAggregate &aggregate) { + const pa_scheduler::host::Uint64Distribution calls = + pa_scheduler::host::SummarizeUint64(aggregate.phase_calls); + const pa_scheduler::host::Uint64Distribution requests = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_requests); + const pa_scheduler::host::Uint64Distribution misses = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_misses); + const pa_scheduler::host::Uint64Distribution primary_requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution primary_misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + const pa_scheduler::host::Uint64Distribution shadow_requests = + pa_scheduler::host::SummarizeUint64(aggregate.shadow_icache_requests); + const pa_scheduler::host::Uint64Distribution shadow_misses = + pa_scheduler::host::SummarizeUint64(aggregate.shadow_icache_misses); + const pa_scheduler::host::Uint64Distribution submit_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.submit_elapsed_ticks); + const pa_scheduler::host::Uint64Distribution phase_ticks = + pa_scheduler::host::SummarizeUint64(aggregate.phase_elapsed_ticks); + const uint64_t request_loss = primary_requests.total >= shadow_requests.total + ? primary_requests.total - shadow_requests.total + : 0U; + const uint64_t miss_loss = primary_misses.total >= shadow_misses.total + ? primary_misses.total - shadow_misses.total + : 0U; + const double miss_rate = requests.total == 0U ? 0.0 : 100.0 * misses.total / requests.total; + const double phase_time_share = submit_ticks.total == 0U + ? 0.0 + : 100.0 * phase_ticks.total / submit_ticks.total; + std::printf( + "[PMU-PHASE-%s] phase=%s semantics=%s cores=%zu calls=%llu " + "icache_req=[%llu,%llu] icache_miss=[%llu,%llu] " + "observed_read_clear_ratio=%.4f%% phase_ticks=%llu submit_ticks=%llu " + "phase_time_share=%.4f%% shadow_loss=%llu/%llu\n", + name, pa_scheduler::ccec_pmu::SubmitPmuPhaseName(pa_scheduler::kCompiledSubmitPmuPhase), + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "disabled" + : "running_read_clear_lower_bound", + aggregate.phase_calls.size(), static_cast(calls.total), + static_cast(requests.total), + static_cast(requests.total + request_loss), + static_cast(misses.total), + static_cast(misses.total + miss_loss), miss_rate, + static_cast(phase_ticks.total), + static_cast(submit_ticks.total), phase_time_share, + static_cast(request_loss), + static_cast(miss_loss) + ); +} + +struct PmuValidation { + uint32_t trusted = 0; + uint32_t unique_physical_core_ids = 0; + uint32_t owner_bitmap_members = 0; + uint32_t exact_worker_slots = 0; + uint32_t physical_role_matches = 0; + uint32_t mixed_triplet_matches = 0; + uint32_t window_started = 0; + uint32_t window_stopped = 0; + uint32_t icache_pairs = 0; + uint32_t icache_calibrated_cores = 0; + uint32_t prior_snapshot_larger = 0; + uint32_t submit_engine_workers_expected = 0; + uint32_t submit_engine_workers_matched = 0; + uint32_t maximum_programmable_counter = 0; + uint32_t build_variant_matches = 0; + uint32_t phase_id_matches = 0; + uint32_t phase_status_trusted = 0; + uint32_t shadow_primary_matches = 0; + uint32_t shadow_primary_bounded = 0; + uint32_t phase_shadow_acceptable = 0; + uint32_t phase_boundary_matches = 0; + uint32_t phase_call_shape_matches = 0; + uint32_t phase_time_valid_records = 0; + uint64_t phase_calls = 0; + uint64_t expected_phase_calls = 0; + uint64_t shadow_request_abs_delta_sum = 0; + uint64_t shadow_miss_abs_delta_sum = 0; + int64_t shadow_request_signed_delta_sum = 0; + int64_t shadow_miss_signed_delta_sum = 0; + uint32_t shadow_request_abs_delta_max = 0; + uint32_t shadow_miss_abs_delta_max = 0; + bool icache_order_valid = true; + bool icache_measurement_valid = true; + bool submit_engine_observation_valid = true; + bool counter_below_risk_threshold = true; + bool phase_measurement_valid = false; + bool passed = true; +}; + +// 32-bit programmable counter 无法仅凭终值证明从未回卷。正式文件采用 25% +// 高水位作为保守拒绝阈值；它只降低风险，不把“未越线”表述成回卷证明。 +constexpr uint32_t kProgrammableCounterRiskThreshold = UINT32_MAX / 4U; + +#if PTO_FDWIC_SHARED_MAP +uint32_t ExpectedSubmitPmuPhaseCallsPerWorker( + uint32_t tasks_per_worker, uint32_t claim_wins +) { + // shared 的前端仍由每核完整重放 Claim/EfDrain；Materialize/Register + // 只在本核实际 winner 上执行。claim_wins 已由公共调度结果发布， + // 无需为 PMU 新增任何设备记录字段。 + switch (pa_scheduler::kCompiledSubmitPmuPhase) { + case pa_scheduler::SubmitPmuPhase::None: + return 0U; + case pa_scheduler::SubmitPmuPhase::Claim: + case pa_scheduler::SubmitPmuPhase::EfDrain: + return tasks_per_worker; + case pa_scheduler::SubmitPmuPhase::Materialize: + case pa_scheduler::SubmitPmuPhase::Register: + return claim_wins; + case pa_scheduler::SubmitPmuPhase::Count: + break; + } + return UINT32_MAX; +} + +uint64_t ExpectedSubmitPmuPhaseCallsTotal(uint32_t tasks_per_worker) { + switch (pa_scheduler::kCompiledSubmitPmuPhase) { + case pa_scheduler::SubmitPmuPhase::None: + return 0U; + case pa_scheduler::SubmitPmuPhase::Claim: + case pa_scheduler::SubmitPmuPhase::EfDrain: + return static_cast(tasks_per_worker) * + pa_scheduler::kWorkers; + case pa_scheduler::SubmitPmuPhase::Materialize: + case pa_scheduler::SubmitPmuPhase::Register: + // shared 每个逻辑 task 恰有一个 winner。 + return tasks_per_worker; + case pa_scheduler::SubmitPmuPhase::Count: + break; + } + return UINT64_MAX; +} +#else +uint32_t ExpectedSubmitPmuPhaseCallsPerWorker(uint32_t batches) { + // 当前所有 running phase 都覆盖每个 worker 的每次 Submit，固定为 5B。 + switch (pa_scheduler::kCompiledSubmitPmuPhase) { + case pa_scheduler::SubmitPmuPhase::None: + return 0U; + case pa_scheduler::SubmitPmuPhase::Claim: + case pa_scheduler::SubmitPmuPhase::EfDrain: + case pa_scheduler::SubmitPmuPhase::Materialize: + case pa_scheduler::SubmitPmuPhase::Register: + return batches * pa_scheduler::kTasksPerBatch; + case pa_scheduler::SubmitPmuPhase::Count: + break; + } + return UINT32_MAX; +} +#endif + +uint64_t SubmitPmuElapsedTicks( + const pa_scheduler::WorkerResult &result +) { + return result.submit_end >= result.submit_begin + ? result.submit_end - result.submit_begin + : 0U; +} + +bool SubmitPmuPhaseTimeValid( + const pa_scheduler::WorkerResult &result, + uint32_t expected_phase_calls +) { + const uint64_t submit_elapsed_ticks = + SubmitPmuElapsedTicks(result); + return + submit_elapsed_ticks != 0U && + result.pmu_phase_elapsed_ticks <= submit_elapsed_ticks && + (pa_scheduler::kCompiledSubmitPmuPhase == + pa_scheduler::SubmitPmuPhase::None + ? result.pmu_phase_elapsed_ticks == 0U + : expected_phase_calls == 0U + ? result.pmu_phase_elapsed_ticks == 0U + : result.pmu_phase_elapsed_ticks != 0U); +} + +bool ValidatePmu( + const pa_scheduler::SchedulerState &state, uint32_t run, const PmuOptions &pmu, + const WinnerWorkloadOptions &workload, + const pa_scheduler::pmu_owner::PmuOwnerControl *owner, PmuValidation *validation +) { + using namespace pa_scheduler::ccec_pmu; + if (pmu.mode == WindowMode::Off) { + *validation = PmuValidation{}; + return true; + } + +#if PTO_FDWIC_SHARED_MAP + pa_scheduler::host::SharedHostTaskPlan shared_plan; + std::string shared_plan_error; + if (!pa_scheduler::host::BuildSharedHostTaskPlan( + state, &shared_plan, &shared_plan_error + )) { + *validation = PmuValidation{}; + validation->passed = false; + std::fprintf( + stderr, + "[PMU] run=%u cannot build shared expected task plan: %s\n", + run, shared_plan_error.c_str() + ); + return false; + } +#endif + + bool seen[kPhysicalSubcoreCount] = {}; + uint32_t trusted = 0; + uint32_t unique = 0; + uint32_t owner_members = 0; + uint32_t exact_worker_slots = 0; + uint32_t physical_role_matches = 0; + uint32_t window_started = 0; + uint32_t window_stopped = 0; + uint32_t icache_pairs = 0; + uint32_t icache_calibrated_cores = 0; + uint32_t prior_larger = 0; + uint32_t submit_engine_workers_expected = 0; + uint32_t submit_engine_workers_matched = 0; + uint32_t maximum_programmable_counter = 0; + uint32_t build_variant_matches = 0; + uint32_t phase_id_matches = 0; + uint32_t phase_status_trusted = 0; + uint32_t shadow_primary_matches = 0; + uint32_t shadow_primary_bounded = 0; + uint32_t phase_shadow_acceptable = 0; + uint32_t phase_boundary_matches = 0; + uint32_t phase_call_shape_matches = 0; + uint32_t phase_time_valid_records = 0; + uint64_t phase_calls = 0; + uint64_t expected_phase_calls = 0; + uint64_t shadow_request_abs_delta_sum = 0; + uint64_t shadow_miss_abs_delta_sum = 0; + int64_t shadow_request_signed_delta_sum = 0; + int64_t shadow_miss_signed_delta_sum = 0; + uint32_t shadow_request_abs_delta_max = 0; + uint32_t shadow_miss_abs_delta_max = 0; + bool icache_order_valid = true; + uint32_t bad_printed = 0; + PmuAggregate all; + PmuAggregate aic; + PmuAggregate aiv; + for (uint32_t worker = 0; worker < pa_scheduler::kWorkers; ++worker) { + const pa_scheduler::WorkerResult &result = state.results[worker]; + const uint32_t expected_phase_calls_per_worker = +#if PTO_FDWIC_SHARED_MAP + ExpectedSubmitPmuPhaseCallsPerWorker( + shared_plan.total_tasks, + static_cast(result.claim_wins) + ); +#else + ExpectedSubmitPmuPhaseCallsPerWorker(state.config.batches); +#endif + const uint32_t status = result.pmu_status; + const uint32_t core_id = StatusCoreId(status); + const bool record_trusted = (status & kStatusRequired) == kStatusRequired; + const bool variant_matches = result.pmu_build_variant == pa_scheduler::kBuildVariantSubmitPmu; + const bool phase_id_matches_record = + result.pmu_phase_id == static_cast(pa_scheduler::kCompiledSubmitPmuPhase); + const bool phase_status_ok = + (result.pmu_phase_status & kPhaseStatusRequired) == kPhaseStatusRequired; + const bool shadow_matches = + result.pmu_shadow_icache_requests == result.pmu_icache_requests && + result.pmu_shadow_icache_misses == result.pmu_icache_misses; + const bool shadow_bounded = + result.pmu_shadow_icache_requests <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_icache_misses; + const bool phase_requires_exact_shadow = + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None; + const bool shadow_acceptable = + phase_requires_exact_shadow + ? shadow_matches + : shadow_bounded; + const uint32_t request_abs_delta = + result.pmu_shadow_icache_requests >= result.pmu_icache_requests + ? result.pmu_shadow_icache_requests - result.pmu_icache_requests + : result.pmu_icache_requests - result.pmu_shadow_icache_requests; + const uint32_t miss_abs_delta = + result.pmu_shadow_icache_misses >= result.pmu_icache_misses + ? result.pmu_shadow_icache_misses - result.pmu_icache_misses + : result.pmu_icache_misses - result.pmu_shadow_icache_misses; + const bool boundaries_match = + result.pmu_phase_begin_reads == result.pmu_phase_calls && + result.pmu_phase_end_reads == result.pmu_phase_calls; + const bool phase_call_shape_matches_record = + result.pmu_phase_calls == expected_phase_calls_per_worker; + const uint64_t submit_elapsed_ticks = + SubmitPmuElapsedTicks(result); + const bool phase_time_valid_record = + SubmitPmuPhaseTimeValid( + result, expected_phase_calls_per_worker + ); + const bool logical_aic = worker < pa_scheduler::kAicWorkers; + const bool physical_aic = pa_scheduler::pmu_owner::IsAicPhysicalSlot(core_id); + trusted += record_trusted; + build_variant_matches += variant_matches; + phase_id_matches += phase_id_matches_record; + phase_status_trusted += phase_status_ok; + shadow_primary_matches += shadow_matches; + shadow_primary_bounded += shadow_bounded; + phase_shadow_acceptable += shadow_acceptable; + phase_boundary_matches += boundaries_match; + phase_call_shape_matches += phase_call_shape_matches_record; + phase_time_valid_records += phase_time_valid_record; + phase_calls += result.pmu_phase_calls; + expected_phase_calls += expected_phase_calls_per_worker; + shadow_request_abs_delta_sum += request_abs_delta; + shadow_miss_abs_delta_sum += miss_abs_delta; + shadow_request_signed_delta_sum += + static_cast(result.pmu_shadow_icache_requests) - result.pmu_icache_requests; + shadow_miss_signed_delta_sum += + static_cast(result.pmu_shadow_icache_misses) - result.pmu_icache_misses; + shadow_request_abs_delta_max = std::max(shadow_request_abs_delta_max, request_abs_delta); + shadow_miss_abs_delta_max = std::max(shadow_miss_abs_delta_max, miss_abs_delta); + owner_members += owner != nullptr && pa_scheduler::pmu_owner::IsConfigured(*owner, core_id); + exact_worker_slots += result.worker_id == worker; + physical_role_matches += logical_aic == physical_aic; + window_started += (status & kStatusWindowStarted) != 0U; + window_stopped += (status & kStatusWindowStopped) != 0U; + icache_pairs += (status & kStatusIcachePairObserved) != 0U; + prior_larger += (status & kStatusPriorSnapshotLarger) != 0; + if (pmu.mode == WindowMode::SubmitAll && + workload.mode == pa_scheduler::WinnerWorkloadMode::RealCompute) { + const uint64_t submit_engine_tasks = + result.placement[static_cast(pa_scheduler::DrainPlace::EfDrain)] + + result.placement[static_cast(pa_scheduler::DrainPlace::RingBackpressure)]; + const uint32_t relevant_busy = logical_aic ? result.pmu_cube_busy : result.pmu_vector_busy; + const bool engine_observation_matches = + (submit_engine_tasks == 0U) == (relevant_busy == 0U); + submit_engine_workers_expected += submit_engine_tasks != 0U; + submit_engine_workers_matched += engine_observation_matches; + } + // phase 的 request/miss 由两条顺序 ld_dev 划界，局部窗口边界并不 + // 完全重合；只要求它们各自不超过完整窗口，不把 phase miss<=request + // 误设成硬门槛。完整 Submit 的 miss<=request 仍必须成立。 + icache_order_valid &= result.pmu_icache_misses <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_shadow_icache_requests && + result.pmu_shadow_icache_requests <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_icache_misses && + result.pmu_phase_icache_requests <= result.pmu_shadow_icache_requests && + result.pmu_phase_icache_misses <= result.pmu_shadow_icache_misses; + if (pmu.mode == WindowMode::IcacheSingle) { + const int64_t worker_cycle_delta = static_cast(result.pmu_total_cycles) - + static_cast(result.pmu_warm_total_cycles); + const int64_t worker_tick_delta = static_cast(result.pmu_window_ticks) - + static_cast(result.pmu_warm_window_ticks); + const int64_t worker_miss_delta = static_cast(result.pmu_icache_misses) - + static_cast(result.pmu_warm_icache_misses); + icache_calibrated_cores += worker_cycle_delta > 0 && worker_tick_delta > 0 && + worker_miss_delta == static_cast(pmu.icache_trials) && + result.pmu_warm_icache_misses == 0U && + result.pmu_icache_misses == pmu.icache_trials; + } + const uint32_t programmable[] = { + result.pmu_vector_busy, result.pmu_cube_busy, result.pmu_scalar_busy, + result.pmu_mte1_busy, result.pmu_mte2_busy, + result.pmu_icache_requests, result.pmu_icache_misses, + result.pmu_phase_icache_requests, result.pmu_phase_icache_misses, + result.pmu_shadow_icache_requests, result.pmu_shadow_icache_misses, + }; + for (uint32_t value : programmable) { + maximum_programmable_counter = std::max(maximum_programmable_counter, value); + } + if (core_id < kPhysicalSubcoreCount && !seen[core_id]) { + seen[core_id] = true; + ++unique; + } + if ((!record_trusted || !variant_matches || !phase_id_matches_record || + !phase_status_ok || !shadow_acceptable || !boundaries_match || + !phase_call_shape_matches_record || !phase_time_valid_record) && bad_printed < 8) { + std::printf( + "[PMU-BAD] worker=%u role=%llu coreid=%u status=0x%08x total=%llu scalar=%u " + "req=%u miss=%u phase_status=0x%08x phase=%u/%u calls=%u/%u boundaries=%u/%u " + "phase_ticks=%llu submit_ticks=%llu shadow=%u/%u\n", + worker, static_cast(result.role), core_id, status, + static_cast(result.pmu_total_cycles), result.pmu_scalar_busy, + result.pmu_icache_requests, result.pmu_icache_misses, + result.pmu_phase_status, result.pmu_phase_id, + static_cast(pa_scheduler::kCompiledSubmitPmuPhase), result.pmu_phase_calls, + expected_phase_calls_per_worker, + result.pmu_phase_begin_reads, result.pmu_phase_end_reads, + static_cast(result.pmu_phase_elapsed_ticks), + static_cast(submit_elapsed_ticks), + result.pmu_shadow_icache_requests, result.pmu_shadow_icache_misses + ); + ++bad_printed; + } + AddPmuSample(result, &all); + AddPmuSample(result, result.role == static_cast(pa_scheduler::CoreRole::Aic) ? &aic : &aiv); + } + + uint32_t mixed_triplet_matches = 0U; + for (uint32_t block = 0U; block < pa_scheduler::kAicWorkers; ++block) { + const uint32_t aic_id = StatusCoreId(state.results[block].pmu_status); + if (!pa_scheduler::pmu_owner::IsAicPhysicalSlot(aic_id)) continue; + const uint32_t die_base = (aic_id / pa_scheduler::pmu_owner::kSubcoresPerDie) * + pa_scheduler::pmu_owner::kSubcoresPerDie; + const uint32_t local = aic_id % pa_scheduler::pmu_owner::kSubcoresPerDie; + const uint32_t expected_aiv0 = die_base + pa_scheduler::pmu_owner::kAicPerDie + local * 2U; + const uint32_t expected_aiv1 = expected_aiv0 + 1U; + const uint32_t aiv0_id = StatusCoreId( + state.results[pa_scheduler::kAicWorkers + block * 2U].pmu_status + ); + const uint32_t aiv1_id = StatusCoreId( + state.results[pa_scheduler::kAicWorkers + block * 2U + 1U].pmu_status + ); + mixed_triplet_matches += aiv0_id == expected_aiv0 && aiv1_id == expected_aiv1; + } + + PrintPmuAggregate("ALL", all); + PrintPmuAggregate("AIC", aic); + PrintPmuAggregate("AIV", aiv); + PrintSubmitPmuPhaseAggregate("ALL", all); + PrintSubmitPmuPhaseAggregate("AIC", aic); + PrintSubmitPmuPhaseAggregate("AIV", aiv); + bool icache_measurement_ok = true; + if (pmu.mode == WindowMode::IcacheSingle) { + const bool all_ok = PrintSingleIcacheAggregate("ALL", all, pmu.icache_trials); + const bool aic_ok = PrintSingleIcacheAggregate("AIC", aic, pmu.icache_trials); + const bool aiv_ok = PrintSingleIcacheAggregate("AIV", aiv, pmu.icache_trials); + icache_measurement_ok = all_ok && aic_ok && aiv_ok && + icache_pairs == pa_scheduler::kWorkers && + icache_calibrated_cores == pa_scheduler::kWorkers; + } + const bool records_ok = trusted == pa_scheduler::kWorkers; + const bool core_ids_ok = unique == pa_scheduler::kWorkers; + const bool owner_members_ok = owner_members == pa_scheduler::kWorkers; + const bool worker_slots_ok = exact_worker_slots == pa_scheduler::kWorkers; + const bool physical_roles_ok = physical_role_matches == pa_scheduler::kWorkers; + const bool mixed_triplets_ok = mixed_triplet_matches == pa_scheduler::kAicWorkers; + const bool windows_started_ok = window_started == pa_scheduler::kWorkers; + const bool windows_stopped_ok = window_stopped == pa_scheduler::kWorkers; + const bool build_variant_ok = build_variant_matches == pa_scheduler::kWorkers; + const bool phase_id_ok = phase_id_matches == pa_scheduler::kWorkers; + const bool phase_status_ok = phase_status_trusted == pa_scheduler::kWorkers; + const bool shadow_partition_ok = phase_shadow_acceptable == pa_scheduler::kWorkers; + const bool phase_boundaries_ok = phase_boundary_matches == pa_scheduler::kWorkers; + const bool phase_call_shape_ok = phase_call_shape_matches == pa_scheduler::kWorkers; + const bool phase_time_ok = phase_time_valid_records == pa_scheduler::kWorkers; + const bool phase_calls_ok = phase_calls == expected_phase_calls; + const bool submit_engine_observation_ok = + pmu.mode != WindowMode::SubmitAll || + workload.mode != pa_scheduler::WinnerWorkloadMode::RealCompute || + submit_engine_workers_matched == pa_scheduler::kWorkers; + const bool counter_below_risk_threshold = + maximum_programmable_counter < kProgrammableCounterRiskThreshold; + std::printf( + "[PMU] run=%u window=%s calibration_scalar_nops=%u icache_trials=%u trusted=%u/%u " + "unique_coreids=%u/%u prior_larger=%u/%u icache_pairs=%u/%u calibrated_cores=%u/%u " + "programmable_max=%u headroom=%u\n", run, + PmuModeName(pmu.mode), pmu.scalar_nops, pmu.icache_trials, trusted, pa_scheduler::kWorkers, + unique, pa_scheduler::kWorkers, prior_larger, pa_scheduler::kWorkers, + icache_pairs, pa_scheduler::kWorkers, icache_calibrated_cores, pa_scheduler::kWorkers, + maximum_programmable_counter, + UINT32_MAX - maximum_programmable_counter + ); + std::printf( + "[PMU-SHADOW-DELTA] exact=%u/%u bounded=%u/%u request_abs_sum=%llu request_abs_max=%u " + "request_signed_sum=%lld miss_abs_sum=%llu miss_abs_max=%u miss_signed_sum=%lld\n", + shadow_primary_matches, pa_scheduler::kWorkers, + shadow_primary_bounded, pa_scheduler::kWorkers, + static_cast(shadow_request_abs_delta_sum), + shadow_request_abs_delta_max, static_cast(shadow_request_signed_delta_sum), + static_cast(shadow_miss_abs_delta_sum), + shadow_miss_abs_delta_max, static_cast(shadow_miss_signed_delta_sum) + ); + std::printf("[ASSERT] %-48s %s\n", "all PMU records have configured selectors and data", + records_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 96 PMU physical subcore ids are unique", + core_ids_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all PMU physical ids belong to the owner bitmap", + owner_members_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "worker result slots and ids match exactly", + worker_slots_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "logical AIC/AIV roles match physical subcores", + physical_roles_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 32 mixed blocks map to physical 1:2 triplets", + mixed_triplets_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 96 PMU windows executed start", + windows_started_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all 96 started PMU windows executed stop", + windows_stopped_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all records match submit-pmu build and phase ids", + build_variant_ok && phase_id_ok ? "PASS" : "FAIL"); + std::printf( + "[ASSERT] %-48s %s\n", + "phase shadow partitions satisfy exact-or-bounded contract", + shadow_partition_ok ? "PASS" : "FAIL" + ); + std::printf("[ASSERT] %-48s %s\n", "all phase boundaries and per-worker calls are exact", + phase_status_ok && phase_boundaries_ok && phase_call_shape_ok && phase_calls_ok + ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "all phase times fit their per-worker Submit windows", + phase_time_ok ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "I-cache misses do not exceed requests", + icache_order_valid ? "PASS" : "FAIL"); + std::printf("[ASSERT] %-48s %s\n", "programmable counters stay below 25% risk threshold", + counter_below_risk_threshold ? "PASS" : "FAIL"); + if (pmu.mode == WindowMode::SubmitAll && + workload.mode == pa_scheduler::WinnerWorkloadMode::RealCompute) { + std::printf( + "[ASSERT] %-48s %s (active_workers=%u matched_workers=%u/%u)\n", + "Submit placement has matching AIC/AIV engine PMU", + submit_engine_observation_ok ? "PASS" : "FAIL", submit_engine_workers_expected, + submit_engine_workers_matched, pa_scheduler::kWorkers + ); + } + if (pmu.mode == WindowMode::IcacheSingle) { + std::printf("[ASSERT] %-48s %s\n", "each cold trial adds exactly one CNT7 I-cache miss", + icache_measurement_ok ? "PASS" : "FAIL"); + } + validation->trusted = trusted; + validation->unique_physical_core_ids = unique; + validation->owner_bitmap_members = owner_members; + validation->exact_worker_slots = exact_worker_slots; + validation->physical_role_matches = physical_role_matches; + validation->mixed_triplet_matches = mixed_triplet_matches; + validation->window_started = window_started; + validation->window_stopped = window_stopped; + validation->icache_pairs = icache_pairs; + validation->icache_calibrated_cores = icache_calibrated_cores; + validation->prior_snapshot_larger = prior_larger; + validation->submit_engine_workers_expected = submit_engine_workers_expected; + validation->submit_engine_workers_matched = submit_engine_workers_matched; + validation->maximum_programmable_counter = maximum_programmable_counter; + validation->build_variant_matches = build_variant_matches; + validation->phase_id_matches = phase_id_matches; + validation->phase_status_trusted = phase_status_trusted; + validation->shadow_primary_matches = shadow_primary_matches; + validation->shadow_primary_bounded = shadow_primary_bounded; + validation->phase_shadow_acceptable = phase_shadow_acceptable; + validation->phase_boundary_matches = phase_boundary_matches; + validation->phase_call_shape_matches = phase_call_shape_matches; + validation->phase_time_valid_records = phase_time_valid_records; + validation->phase_calls = phase_calls; + validation->expected_phase_calls = expected_phase_calls; + validation->shadow_request_abs_delta_sum = shadow_request_abs_delta_sum; + validation->shadow_miss_abs_delta_sum = shadow_miss_abs_delta_sum; + validation->shadow_request_signed_delta_sum = shadow_request_signed_delta_sum; + validation->shadow_miss_signed_delta_sum = shadow_miss_signed_delta_sum; + validation->shadow_request_abs_delta_max = shadow_request_abs_delta_max; + validation->shadow_miss_abs_delta_max = shadow_miss_abs_delta_max; + validation->icache_order_valid = icache_order_valid; + validation->icache_measurement_valid = icache_measurement_ok; + validation->submit_engine_observation_valid = submit_engine_observation_ok; + validation->counter_below_risk_threshold = counter_below_risk_threshold; + validation->phase_measurement_valid = + build_variant_ok && phase_id_ok && phase_status_ok && shadow_partition_ok && + phase_boundaries_ok && phase_call_shape_ok && phase_calls_ok && phase_time_ok; + validation->passed = records_ok && core_ids_ok && owner_members_ok && worker_slots_ok && + physical_roles_ok && mixed_triplets_ok && windows_started_ok && windows_stopped_ok && + icache_order_valid && icache_measurement_ok && submit_engine_observation_ok && + validation->phase_measurement_valid && + counter_below_risk_threshold; + return validation->passed; +} + +void WriteJsonString(std::FILE *output, const std::string &value) { + std::fputc('"', output); + for (unsigned char character : value) { + switch (character) { + case '"': + std::fputs("\\\"", output); + break; + case '\\': + std::fputs("\\\\", output); + break; + case '\b': + std::fputs("\\b", output); + break; + case '\f': + std::fputs("\\f", output); + break; + case '\n': + std::fputs("\\n", output); + break; + case '\r': + std::fputs("\\r", output); + break; + case '\t': + std::fputs("\\t", output); + break; + default: + if (character < 0x20U) { + std::fprintf(output, "\\u%04x", static_cast(character)); + } else { + std::fputc(character, output); + } + } + } + std::fputc('"', output); +} + +void WriteMetricDistribution(std::FILE *output, const std::vector &values) { + const pa_scheduler::host::Uint64Distribution summary = pa_scheduler::host::SummarizeUint64(values); + const double mean = values.empty() ? 0.0 : static_cast(summary.total) / values.size(); + std::fprintf( + output, "{\"sum\":%llu,\"mean\":%.17g,\"median\":%.17g,\"p95\":%llu,\"max\":%llu}", + static_cast(summary.total), mean, summary.median, + static_cast(summary.p95), static_cast(summary.maximum) + ); +} + +void WritePmuAggregateJson( + std::FILE *output, const PmuAggregate &aggregate, bool icache_single +) { + (void)icache_single; + const pa_scheduler::host::Uint64Distribution requests = + pa_scheduler::host::SummarizeUint64(aggregate.icache_requests); + const pa_scheduler::host::Uint64Distribution misses = + pa_scheduler::host::SummarizeUint64(aggregate.icache_misses); + uint32_t active_cores = 0; + for (uint64_t cycles : aggregate.total_cycles) active_cores += cycles != 0; + std::fprintf( + output, "{\"cores\":%zu,\"active_cores\":%u,\"trusted_cores\":%u,\"total_cycles\":", + aggregate.total_cycles.size(), active_cores, aggregate.trusted + ); + WriteMetricDistribution(output, aggregate.total_cycles); + std::fputs(",\"vector_busy\":", output); + WriteMetricDistribution(output, aggregate.vector_busy); + std::fputs(",\"cube_busy\":", output); + WriteMetricDistribution(output, aggregate.cube_busy); + std::fputs(",\"scalar_busy\":", output); + WriteMetricDistribution(output, aggregate.scalar_busy); + std::fputs(",\"mte1_busy\":", output); + WriteMetricDistribution(output, aggregate.mte1_busy); + std::fputs(",\"mte2_busy\":", output); + WriteMetricDistribution(output, aggregate.mte2_busy); + std::fputs(",\"icache_requests\":", output); + WriteMetricDistribution(output, aggregate.icache_requests); + std::fputs(",\"icache_misses\":", output); + WriteMetricDistribution(output, aggregate.icache_misses); + std::fputs(",\"shadow_whole_icache_requests\":", output); + WriteMetricDistribution(output, aggregate.shadow_icache_requests); + std::fputs(",\"shadow_whole_icache_misses\":", output); + WriteMetricDistribution(output, aggregate.shadow_icache_misses); + std::fputs(",\"phase_calls\":", output); + WriteMetricDistribution(output, aggregate.phase_calls); + std::fputs(",\"submit_elapsed_ticks\":", output); + WriteMetricDistribution(output, aggregate.submit_elapsed_ticks); + std::fputs(",\"phase_elapsed_ticks\":", output); + WriteMetricDistribution(output, aggregate.phase_elapsed_ticks); + std::fputs(",\"phase_icache_requests\":", output); + WriteMetricDistribution(output, aggregate.phase_icache_requests); + std::fputs(",\"phase_icache_misses\":", output); + WriteMetricDistribution(output, aggregate.phase_icache_misses); + std::fputs(",\"icache_miss_rate\":", output); + if (requests.total == 0) { + std::fputs("null", output); + } else { + // 全局 miss rate 必须以总 miss/总 request 计算，不能平均逐核百分比。 + std::fprintf(output, "%.17g", static_cast(misses.total) / requests.total); + } + const pa_scheduler::host::Uint64Distribution phase_requests = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_requests); + const pa_scheduler::host::Uint64Distribution phase_misses = + pa_scheduler::host::SummarizeUint64(aggregate.phase_icache_misses); + // 两个 phase counter 是顺序 read-to-clear，下界之比不是实际 miss rate + // 的数学下界；字段名只陈述它是本次 read-clear 观察值之比。 + std::fputs(",\"phase_observed_read_clear_ratio\":", output); + if (phase_requests.total == 0U) { + std::fputs("null", output); + } else { + std::fprintf(output, "%.17g", static_cast(phase_misses.total) / phase_requests.total); + } + std::fputc('}', output); +} + +uint32_t CountConfiguredMixedTriplets(const pa_scheduler::pmu_owner::PmuOwnerControl &owner) { + uint32_t complete = 0U; + for (uint32_t die_base = 0U; + die_base < pa_scheduler::pmu_owner::kPhysicalSubcoreCount; + die_base += pa_scheduler::pmu_owner::kSubcoresPerDie) { + for (uint32_t local = 0U; local < pa_scheduler::pmu_owner::kAicPerDie; ++local) { + const uint32_t aic = die_base + local; + const uint32_t aiv0 = die_base + pa_scheduler::pmu_owner::kAicPerDie + local * 2U; + const uint32_t aiv1 = aiv0 + 1U; + complete += pa_scheduler::pmu_owner::IsConfigured(owner, aic) && + pa_scheduler::pmu_owner::IsConfigured(owner, aiv0) && + pa_scheduler::pmu_owner::IsConfigured(owner, aiv1); + } + } + return complete; +} + +bool ExportPmuJson( + const pa_scheduler::SchedulerState &state, const pa_scheduler::host::Options &options, + const PmuOptions &pmu, const WinnerWorkloadOptions &workload, + uint32_t run, double host_us, double submit_span_us, + const PmuValidation &validation, bool semantic_passed, bool workload_output_passed, + const pa_scheduler::pmu_owner::PmuOwnerControl &owner, bool restore_passed, + const std::string &output_path +) { + using namespace pa_scheduler::ccec_pmu; +#if PTO_FDWIC_SHARED_MAP + pa_scheduler::host::SharedHostTaskPlan shared_plan; + std::string shared_plan_error; + if (!pa_scheduler::host::BuildSharedHostTaskPlan( + state, &shared_plan, &shared_plan_error + )) { + std::fprintf( + stderr, "Cannot export PMU JSON without a valid shared task plan: %s\n", + shared_plan_error.c_str() + ); + return false; + } + const uint64_t expected_shared_phase_calls = + ExpectedSubmitPmuPhaseCallsTotal(shared_plan.total_tasks); + if (validation.expected_phase_calls != expected_shared_phase_calls) { + std::fprintf( + stderr, + "Cannot export PMU JSON: shared expected phase calls changed " + "between validation and export (%llu != %llu).\n", + static_cast(validation.expected_phase_calls), + static_cast(expected_shared_phase_calls) + ); + return false; + } +#endif + PmuAggregate all; + PmuAggregate aic; + PmuAggregate aiv; + uint32_t active_output_tiles = 0; + uint64_t ef_drain_kernels = 0; + uint64_t ring_backpressure_kernels = 0; + uint64_t final_drain_kernels = 0; + for (uint32_t worker = 0; worker < pa_scheduler::kWorkers; ++worker) { + const pa_scheduler::WorkerResult &result = state.results[worker]; + AddPmuSample(result, &all); + AddPmuSample(result, result.role == static_cast(pa_scheduler::CoreRole::Aic) ? &aic : &aiv); + if (result.role == static_cast(pa_scheduler::CoreRole::Aic)) { + active_output_tiles += result.kernel_counts[0] != 0; + active_output_tiles += result.kernel_counts[2] != 0; + } else if (result.role == static_cast(pa_scheduler::CoreRole::Aiv)) { + active_output_tiles += result.kernel_counts[1] != 0; + active_output_tiles += result.kernel_counts[3] != 0; + } + ef_drain_kernels += result.placement[static_cast(pa_scheduler::DrainPlace::EfDrain)]; + ring_backpressure_kernels += + result.placement[static_cast(pa_scheduler::DrainPlace::RingBackpressure)]; + final_drain_kernels += result.placement[static_cast(pa_scheduler::DrainPlace::FinalDrain)]; + } + + const auto generated = std::chrono::system_clock::now().time_since_epoch(); + const uint64_t generated_ns = static_cast( + std::chrono::duration_cast(generated).count() + ); + const std::string capture_id = "pa-pmu-" + std::to_string(generated_ns) + "-run" + std::to_string(run); + const std::string temporary_path = output_path + ".tmp"; + // 临时文件与最终文件都采用 no-replace 语义：并发采集不能截断同名 tmp， + // 也不能在最终发布时覆盖另一份已经完成的证据文件。 + const int output_fd = open(temporary_path.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0644); + std::FILE *output = output_fd < 0 ? nullptr : fdopen(output_fd, "wb"); + if (output == nullptr) { + const int open_error = errno; + if (output_fd >= 0) { + (void)close(output_fd); + (void)std::remove(temporary_path.c_str()); + } + std::fprintf( + stderr, "Cannot exclusively create PMU JSON output %s: %s\n", temporary_path.c_str(), + std::strerror(open_error) + ); + return false; + } + std::vector output_buffer(1U << 20); + std::setvbuf(output, output_buffer.data(), _IOFBF, output_buffer.size()); + + const bool submit_window = IsSubmitWindow(pmu.mode); + const bool icache_single = pmu.mode == WindowMode::IcacheSingle; + const bool real_compute = workload.mode == pa_scheduler::WinnerWorkloadMode::RealCompute; + const bool simulated_task_nops_nonzero = !real_compute && + (options.nops.qk != 0U || options.nops.sf != 0U || + options.nops.pv != 0U || options.nops.up != 0U); + const pa_scheduler::WorkloadCounts active_counts = real_compute + ? workload.repeats + : pa_scheduler::WorkloadCounts{ + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + }; + const uint32_t owner_bitmap_count = pa_scheduler::pmu_owner::CountConfigured(owner); + const uint32_t owner_complete_triplets = CountConfiguredMixedTriplets(owner); +#if PTO_FDWIC_SHARED_MAP + std::fputs("{\n\"schema\":{\"name\":\"pa_scheduler_pmu_phase_windows\",\"version\":6},\n", output); +#else + std::fputs("{\n\"schema\":{\"name\":\"pa_scheduler_pmu_phase_windows\",\"version\":5},\n", output); +#endif + std::fputs("\"capture\":{\"capture_id\":", output); + WriteJsonString(output, capture_id); + std::fprintf( + output, + ",\"generated_unix_time_ns\":%llu,\"run_index\":%u,\"accepted\":true," + "\"usable_for_same_configuration_submit_comparison\":%s," + "\"usable_as_absolute_real_pa_profile\":false,\"window_scope\":\"%s\"," + "\"pmu_probe_position\":\"%s\",\"scheduler_hot_path_included\":%s," + "\"total_sum_is_core_work_not_wall_time\":true," + "\"submit_window_excludes_final_drain\":%s," + "\"published_after_runtime_cleanup\":true,\"runtime_cleanup_passed\":true," + "\"owner_restore_passed\":%s},\n", + static_cast(generated_ns), run, + submit_window ? "true" : "false", + submit_window ? "per_worker_orchestration_to_last_submit_return" : "post_scheduler_calibration_probe", + submit_window ? "inside_RunScheduler" : "after_RunScheduler", + submit_window ? "true" : "false", submit_window ? "true" : "false", + restore_passed ? "true" : "false" + ); + std::fputs("\"configuration\":{\"kernel_path\":", output); + WriteJsonString(output, options.kernel_path); + std::fputs(",\"build_variant\":\"submit-pmu\",\"build_variant_id\":2,\"compiled_phase\":", output); + WriteJsonString(output, SubmitPmuPhaseName(pa_scheduler::kCompiledSubmitPmuPhase)); + std::fprintf( + output, ",\"compiled_phase_id\":%u", + static_cast(pa_scheduler::kCompiledSubmitPmuPhase) + ); + std::fprintf( + output, + ",\"device\":%u,\"batches\":%u,\"workers\":%u,\"aic_workers\":%u,\"aiv_workers\":%u," + "\"final_barrier\":\"%s\"," + "\"trace_enabled\":%s,\"trace_atomics\":%s,\"profile_phases\":%s," + "\"winner_workload\":{\"mode\":", + options.device, options.batches, pa_scheduler::kWorkers, pa_scheduler::kAicWorkers, + pa_scheduler::kAivWorkers, + pa_scheduler::host::FinalBarrierShapeName(options.final_barrier_shape), + options.trace_enabled ? "true" : "false", + options.trace_atomics ? "true" : "false", + options.profile_phases ? "true" : "false" + ); + WriteJsonString(output, WinnerWorkloadModeName(workload.mode)); + std::fputs(",\"input_pattern\":", output); + WriteJsonString( + output, + real_compute ? pa_scheduler::host::RealComputePatternName(workload.pattern) : "none" + ); + std::fprintf( + output, + ",\"config_version\":%u,\"counts\":{\"qk\":%u,\"sf\":%u,\"pv\":%u,\"up\":%u}," + "\"unit\":", + pa_scheduler::kWinnerWorkloadConfigVersion, active_counts.qk, active_counts.sf, + active_counts.pv, active_counts.up + ); + WriteJsonString( + output, + real_compute ? "complete_128x128_engine_pipeline_iteration" : "scalar_nop_instruction" + ); + std::fprintf( + output, + ",\"tile_rows\":%u,\"tile_cols\":%u,\"shared_input_tiles\":%u," + "\"output_tiles_per_worker\":%u,\"workspace_bytes\":%zu,\"role_mapping\":", + real_compute ? pa_scheduler::winner_workload::kTileRows : 0, + real_compute ? pa_scheduler::winner_workload::kTileCols : 0, + real_compute ? pa_scheduler::winner_workload::kSharedInputTiles : 0, + real_compute ? pa_scheduler::winner_workload::kOutputTilesPerWorker : 0, + real_compute ? pa_scheduler::winner_workload::kWorkspaceBytes : 0 + ); + if (real_compute) { + std::fputs( + "{\"qk\":\"cube_matmul\",\"pv\":\"cube_matmul\"," + "\"sf\":\"vector_add\",\"up\":\"vector_mul\"}", + output + ); + } else { + std::fputs("null", output); + } + std::fprintf( + output, ",\"engine_completion_waited_before_task_publish\":%s},\"nop_counts\":", + real_compute ? "true" : "false" + ); + if (real_compute) { + std::fputs("null", output); + } else { + std::fprintf( + output, "{\"qk\":%u,\"sf\":%u,\"pv\":%u,\"up\":%u}", + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + ); + } +#if PTO_FDWIC_SHARED_MAP + std::fputs( + ",\"tensormap_mode\":\"shared\",\"shared_context_lens\":[", + output + ); + for (uint32_t batch = 0; batch < state.config.batches; ++batch) { + std::fprintf( + output, "%s%d", batch == 0U ? "" : ",", + state.context_lens[batch] + ); + } + std::fprintf( + output, + "],\"shared_task_plan\":{\"total_groups\":%u," + "\"tasks_per_worker\":%u}", + shared_plan.total_groups, shared_plan.total_tasks + ); +#endif + std::fputs(",\"pmu_window\":", output); + WriteJsonString(output, PmuModeName(pmu.mode)); + std::fputs(",\"calibration_scalar_nops_per_segment\":", output); + if (submit_window || icache_single) { + std::fputs("null", output); + } else { + std::fprintf(output, "%u", pmu.scalar_nops); + } + std::fputs(",\"icache_single_trials_per_core\":", output); + if (icache_single) { + std::fprintf(output, "%u", pmu.icache_trials); + } else { + std::fputs("null", output); + } + std::fprintf( + output, + ",\"primary_window_segments_per_record\":1," + "\"icache_single_discarded_training_samples_per_core\":%u," + "\"icache_single_sys_counter_tick_ns\":%s," + "\"host_launch_to_sync_us\":%.17g,\"submit_span_us\":%.17g," + "\"selectors\":{\"cnt0_vector_busy\":%u,\"cnt1_cube_busy\":%u," + "\"cnt2_scalar_busy\":%u,\"cnt3_mte1_busy\":%u,\"cnt4_mte2_busy\":%u," + "\"cnt5_shadow_icache_miss\":%u,\"cnt6_primary_icache_request\":%u," + "\"cnt7_primary_icache_miss\":%u,\"cnt8_shadow_icache_request\":%u," + "\"cnt9_unused\":0},\"unavailable_metrics\":[\"mte3_busy\"]," + "\"counter_width_bits\":{\"total\":64,\"programmable\":32}," + "\"counter_wrap_not_directly_detectable\":true,\"counter_wrap_absence_proven\":false," + "\"programmable_counter_risk_threshold\":%u," + "\"gate_start_stop_have_pipe_all_barriers\":true," + "\"phase_timestamp_calls_present\":%s,\"phase_record_writes\":false," + "\"atomic_trace\":false,\"profile_accumulation\":false," + "\"phase_boundary_observation_included\":%s," + "\"phase_time_observation_included\":%s," + "\"phase_time_sys_counter_tick_ns\":1," + "\"phase_time_boundary\":\"after_begin_read_clear_to_before_end_read_clear\"," + "\"phase_time_excludes_shadow_read_overhead\":true," + "\"phase_time_includes_timestamp_overhead\":true," + "\"phase_time_share_definition\":" + "\"sum(phase_elapsed_ticks)/sum(submit_elapsed_ticks)\"," + "\"phase_time_denominator_scope\":" + "\"per_worker_first_submit_begin_to_last_submit_end\"," + "\"phase_counter_pair_snapshot_atomic\":false," + "\"primary_counters_read_at_phase_boundaries\":false," + "\"phase_shadow_partition_exact_required\":%s," + "\"phase_values_are_running_read_clear_lower_bounds\":%s," + "\"cross_phase_elf_sums_valid\":false," + "\"simulated_task_nop_mechanism_executes_on_scalar\":%s," + "\"simulated_task_nops_nonzero\":%s," + "\"icache_miss_rate_definition\":\"sum(icache_misses)/sum(icache_requests)\"},\n", + icache_single ? 2U : 0U, icache_single ? "1" : "null", + host_us, submit_span_us, kVectorBusyEvent, kCubeBusyEvent, kScalarBusyEvent, + kMte1BusyEvent, kMte2BusyEvent, kIcacheMissEvent, kIcacheRequestEvent, kIcacheMissEvent, + kIcacheRequestEvent, kProgrammableCounterRiskThreshold, + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "true" + : "false", + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? "false" + : "true", + real_compute ? "false" : "true", + simulated_task_nops_nonzero ? "true" : "false" + ); + std::fprintf( + output, + "\"validation\":{\"semantic_passed\":%s,\"pmu_passed\":%s," + "\"real_compute_output_validation_required\":%s," + "\"real_compute_output_validation_passed\":%s," + "\"real_compute_active_output_tiles\":%u," + "\"real_compute_inactive_sentinel_tiles\":%u," + "\"real_compute_output_mismatches\":%u," + "\"submit_engine_observation_valid\":%s," + "\"submit_engine_workers_expected\":%u," + "\"submit_engine_workers_matched\":%u," + "\"kernel_placement_counts\":{\"ef_drain\":%llu,\"ring_backpressure\":%llu," + "\"final_drain\":%llu},\"trusted_records\":%u," + "\"expected_records\":%u,\"unique_physical_core_ids\":%u,\"expected_unique_core_ids\":%u," + "\"owner_bitmap_member_records\":%u,\"expected_owner_bitmap_member_records\":%u," + "\"exact_worker_slot_records\":%u,\"expected_exact_worker_slot_records\":%u," + "\"physical_role_match_records\":%u,\"expected_physical_role_match_records\":%u," + "\"mixed_triplet_matches\":%u,\"expected_mixed_triplet_matches\":%u," + "\"window_started_records\":%u,\"window_stopped_records\":%u," + "\"expected_window_records\":%u,\"prior_snapshot_larger_records\":%u," + "\"icache_pair_records\":%u,\"icache_calibrated_cores\":%u," + "\"icache_measurement_valid\":%s," + "\"icache_miss_le_request\":%s,\"counter_below_risk_threshold\":%s," + "\"maximum_programmable_counter\":%u,\"programmable_counter_risk_threshold\":%u," + "\"programmable_counter_headroom\":%u," + "\"build_variant_match_records\":%u,\"phase_id_match_records\":%u," + "\"phase_status_trusted_records\":%u,\"shadow_primary_match_records\":%u," + "\"shadow_primary_bounded_records\":%u," + "\"phase_shadow_acceptable_records\":%u," + "\"shadow_request_abs_delta_sum\":%llu,\"shadow_request_abs_delta_max\":%u," + "\"shadow_request_signed_delta_sum\":%lld," + "\"shadow_miss_abs_delta_sum\":%llu,\"shadow_miss_abs_delta_max\":%u," + "\"shadow_miss_signed_delta_sum\":%lld," + "\"phase_boundary_match_records\":%u,\"phase_call_shape_match_records\":%u," + "\"phase_time_valid_records\":%u,\"phase_time_measurement_valid\":%s," + "\"phase_calls\":%llu,\"phase_expected_calls\":%llu," + "\"phase_measurement_valid\":%s},\n", + semantic_passed ? "true" : "false", validation.passed ? "true" : "false", + real_compute ? "true" : "false", + real_compute ? (workload_output_passed ? "true" : "false") : "null", + real_compute ? active_output_tiles : 0, + real_compute ? pa_scheduler::winner_workload::kOutputTiles - active_output_tiles : 0, + real_compute && !workload_output_passed ? 1U : 0U, + validation.submit_engine_observation_valid ? "true" : "false", + validation.submit_engine_workers_expected, + validation.submit_engine_workers_matched, + static_cast(ef_drain_kernels), + static_cast(ring_backpressure_kernels), + static_cast(final_drain_kernels), + validation.trusted, + pa_scheduler::kWorkers, validation.unique_physical_core_ids, pa_scheduler::kWorkers, + validation.owner_bitmap_members, pa_scheduler::kWorkers, + validation.exact_worker_slots, pa_scheduler::kWorkers, + validation.physical_role_matches, pa_scheduler::kWorkers, + validation.mixed_triplet_matches, pa_scheduler::kAicWorkers, + validation.window_started, validation.window_stopped, pa_scheduler::kWorkers, + validation.prior_snapshot_larger, validation.icache_pairs, + validation.icache_calibrated_cores, + validation.icache_measurement_valid ? "true" : "false", + validation.icache_order_valid ? "true" : "false", + validation.counter_below_risk_threshold ? "true" : "false", + validation.maximum_programmable_counter, kProgrammableCounterRiskThreshold, + UINT32_MAX - validation.maximum_programmable_counter, + validation.build_variant_matches, validation.phase_id_matches, + validation.phase_status_trusted, validation.shadow_primary_matches, + validation.shadow_primary_bounded, validation.phase_shadow_acceptable, + static_cast(validation.shadow_request_abs_delta_sum), + validation.shadow_request_abs_delta_max, + static_cast(validation.shadow_request_signed_delta_sum), + static_cast(validation.shadow_miss_abs_delta_sum), + validation.shadow_miss_abs_delta_max, + static_cast(validation.shadow_miss_signed_delta_sum), + validation.phase_boundary_matches, + validation.phase_call_shape_matches, + validation.phase_time_valid_records, + validation.phase_time_valid_records == pa_scheduler::kWorkers ? "true" : "false", + static_cast(validation.phase_calls), + static_cast(validation.expected_phase_calls), + validation.phase_measurement_valid ? "true" : "false" + ); + std::fprintf( + output, + "\"owner\":{\"mode\":\"main_aicpu_path_a\"," + "\"snapshot_phase\":\"after_configure_before_restore\"," + "\"control_magic\":%u,\"control_version\":%u,\"configure_status\":%d," + "\"configured_flag\":%u,\"configured_bitmap_count\":%u," + "\"expected\":{\"total\":%u,\"aic\":%u,\"aiv\":%u}," + "\"active\":{\"total\":%u,\"aic\":%u,\"aiv\":%u}," + "\"discovered\":{\"total\":%u,\"aic\":%u,\"aiv\":%u}," + "\"physical_slots_scanned\":%u,\"skipped_physical_slots\":%u," + "\"configured_bitmap_word_order\":\"least_significant_physical_ids_first\"," + "\"configured_bitmap_words\":[%u,%u,%u,%u]," + "\"configured_complete_mixed_triplets\":%u,\"expected_complete_mixed_triplets\":%u," + "\"configured_broken_mixed_triplets\":%u,\"restore_passed\":%s},\n", + owner.magic, owner.version, static_cast(owner.status), owner.configured, owner_bitmap_count, + owner.expected_total, owner.expected_aic, owner.expected_aiv, + owner.active_total, owner.active_aic, owner.active_aiv, + owner.discovered_total, owner.discovered_aic, owner.discovered_aiv, + pa_scheduler::pmu_owner::kPhysicalSubcoreCount, owner.skipped_total, + owner.configured_bitmap[0], owner.configured_bitmap[1], + owner.configured_bitmap[2], owner.configured_bitmap[3], + owner_complete_triplets, pa_scheduler::kAicWorkers, + owner_bitmap_count / 3U - owner_complete_triplets, restore_passed ? "true" : "false" + ); + std::fputs("\"records\":[\n", output); + for (uint32_t worker = 0; worker < pa_scheduler::kWorkers; ++worker) { + const pa_scheduler::WorkerResult &result = state.results[worker]; + const uint32_t status = result.pmu_status; + const uint32_t physical_core_id = StatusCoreId(status); + const bool primary_trusted = (status & kStatusRequired) == kStatusRequired; + const bool phase_trusted = + (result.pmu_phase_status & kPhaseStatusRequired) == kPhaseStatusRequired; + const uint64_t submit_elapsed_ticks = + SubmitPmuElapsedTicks(result); + const uint32_t expected_phase_calls = +#if PTO_FDWIC_SHARED_MAP + ExpectedSubmitPmuPhaseCallsPerWorker( + shared_plan.total_tasks, + static_cast(result.claim_wins) + ); +#else + ExpectedSubmitPmuPhaseCallsPerWorker(state.config.batches); +#endif + const bool phase_time_valid = + SubmitPmuPhaseTimeValid(result, expected_phase_calls); + const bool trusted = primary_trusted && phase_trusted && phase_time_valid; + const bool is_aic = result.role == static_cast(pa_scheduler::CoreRole::Aic); + const uint32_t vector_id = is_aic ? 0U : worker - pa_scheduler::kAicWorkers; + const uint32_t block_id = is_aic ? worker : vector_id / 2U; + const uint32_t lane = is_aic ? 0U : 1U + vector_id % 2U; + const uint32_t shadow_read_segments = result.pmu_phase_calls * 2U + 1U; + const bool owner_bitmap_member = pa_scheduler::pmu_owner::IsConfigured(owner, physical_core_id); + const bool worker_slot_exact = result.worker_id == worker; + const bool physical_role_matches = + is_aic == pa_scheduler::pmu_owner::IsAicPhysicalSlot(physical_core_id); + const bool window_started = (status & kStatusWindowStarted) != 0U; + const bool window_stopped = (status & kStatusWindowStopped) != 0U; + const bool shadow_matches = + result.pmu_shadow_icache_requests == result.pmu_icache_requests && + result.pmu_shadow_icache_misses == result.pmu_icache_misses; + const bool shadow_bounded = + result.pmu_shadow_icache_requests <= result.pmu_icache_requests && + result.pmu_shadow_icache_misses <= result.pmu_icache_misses; + const uint32_t shadow_request_loss = + result.pmu_icache_requests - result.pmu_shadow_icache_requests; + const uint32_t shadow_miss_loss = + result.pmu_icache_misses - result.pmu_shadow_icache_misses; + const uint32_t phase_request_upper = + result.pmu_phase_icache_requests + shadow_request_loss; + const uint32_t phase_miss_upper = + result.pmu_phase_icache_misses + shadow_miss_loss; + const bool boundaries_balanced = + result.pmu_phase_begin_reads == result.pmu_phase_calls && + result.pmu_phase_end_reads == result.pmu_phase_calls; + std::fprintf( + output, + "%s{\"worker_id\":%u,\"physical_core_id\":%u,\"role\":\"%s\",\"block_id\":%u," + "\"lane\":%u,\"primary_window_segments\":1,\"shadow_read_segments\":%u," + "\"window_started\":%s,\"window_stopped\":%s,\"total_cycles\":%llu,\"vector_busy\":%u," + "\"cube_busy\":%u,\"scalar_busy\":%u,\"mte1_busy\":%u,\"mte2_busy\":%u," + "\"icache_requests\":%u,\"icache_misses\":%u," + "\"build_variant_id\":%u,\"compiled_phase_id\":%u,\"phase_calls\":%u," + "\"phase_expected_calls\":%u," + "\"phase_begin_reads\":%u,\"phase_end_reads\":%u," + "\"submit_elapsed_ticks\":%llu,\"phase_elapsed_ticks\":%llu," + "\"phase_time_valid\":%s," + "\"phase_icache_requests\":%u,\"phase_icache_misses\":%u," + "\"phase_icache_requests_upper_bound\":%u," + "\"phase_icache_misses_upper_bound\":%u," + "\"shadow_whole_icache_requests\":%u,\"shadow_whole_icache_misses\":%u," + "\"shadow_matches_primary\":%s,\"shadow_not_greater_than_primary\":%s," + "\"shadow_request_loss\":%u,\"shadow_miss_loss\":%u," + "\"phase_boundaries_balanced\":%s," + "\"phase_status\":%u,\"phase_status_hex\":\"0x%08x\"," + "\"status\":%u,\"status_hex\":" + "\"0x%08x\",\"trusted\":%s,\"physical_core_id_valid\":%s,\"selectors_match\":%s," + "\"owner_bitmap_member\":%s,\"worker_slot_exact\":%s," + "\"physical_role_matches\":%s}", + worker == 0 ? "" : ",\n", worker, physical_core_id, is_aic ? "aic" : "aiv", block_id, + lane, shadow_read_segments, window_started ? "true" : "false", window_stopped ? "true" : "false", + static_cast(result.pmu_total_cycles), result.pmu_vector_busy, + result.pmu_cube_busy, result.pmu_scalar_busy, result.pmu_mte1_busy, result.pmu_mte2_busy, + result.pmu_icache_requests, result.pmu_icache_misses, + result.pmu_build_variant, result.pmu_phase_id, result.pmu_phase_calls, + expected_phase_calls, + result.pmu_phase_begin_reads, result.pmu_phase_end_reads, + static_cast(submit_elapsed_ticks), + static_cast(result.pmu_phase_elapsed_ticks), + phase_time_valid ? "true" : "false", + result.pmu_phase_icache_requests, result.pmu_phase_icache_misses, + phase_request_upper, phase_miss_upper, + result.pmu_shadow_icache_requests, result.pmu_shadow_icache_misses, + shadow_matches ? "true" : "false", shadow_bounded ? "true" : "false", + shadow_request_loss, shadow_miss_loss, boundaries_balanced ? "true" : "false", + result.pmu_phase_status, result.pmu_phase_status, + status, status, trusted ? "true" : "false", + (status & kStatusCoreIdValid) != 0 ? "true" : "false", + (status & (kStatusCnt0Selector | kStatusCnt1Selector | kStatusCnt2Selector | + kStatusCnt3Selector | kStatusCnt4Selector | kStatusCnt5Selector | + kStatusCnt6Selector | kStatusCnt7Selector | kStatusCnt8Selector)) == + (kStatusCnt0Selector | kStatusCnt1Selector | kStatusCnt2Selector | + kStatusCnt3Selector | kStatusCnt4Selector | kStatusCnt5Selector | + kStatusCnt6Selector | kStatusCnt7Selector | kStatusCnt8Selector) + ? "true" + : "false", + owner_bitmap_member ? "true" : "false", worker_slot_exact ? "true" : "false", + physical_role_matches ? "true" : "false" + ); + } + std::fputs("\n],\n\"summary\":{\"all\":", output); + WritePmuAggregateJson(output, all, icache_single); + std::fputs(",\"aic\":", output); + WritePmuAggregateJson(output, aic, icache_single); + std::fputs(",\"aiv\":", output); + WritePmuAggregateJson(output, aiv, icache_single); + std::fputs("}\n}\n", output); + + bool success = std::ferror(output) == 0; + int write_error = success ? 0 : EIO; + if (std::fflush(output) != 0) { + success = false; + write_error = errno; + } + if (success && fsync(fileno(output)) != 0) { + success = false; + write_error = errno; + } + if (std::fclose(output) != 0) { + success = false; + write_error = errno; + } + if (!success) { + std::fprintf(stderr, "Failed while writing PMU JSON output %s: %s\n", temporary_path.c_str(), + std::strerror(write_error)); + (void)std::remove(temporary_path.c_str()); + return false; + } + // 同目录 hard-link 在最终名称不存在时原子发布；EEXIST 时保留既有证据， + // 不采用会替换目标的 POSIX rename。 + if (link(temporary_path.c_str(), output_path.c_str()) != 0) { + std::fprintf( + stderr, "Cannot publish PMU JSON without replacement %s -> %s: %s\n", + temporary_path.c_str(), output_path.c_str(), + std::strerror(errno) + ); + (void)std::remove(temporary_path.c_str()); + return false; + } + if (unlink(temporary_path.c_str()) != 0) { + const int unlink_error = errno; + // 最终文件已链接但事务尚未完成；尽力撤回最终名称，避免失败返回时留下 + // 一份被调用方误认为成功发布的文件。 + (void)unlink(output_path.c_str()); + std::fprintf( + stderr, "Cannot remove PMU JSON temporary link %s: %s\n", temporary_path.c_str(), + std::strerror(unlink_error) + ); + return false; + } + std::printf("[PMU-JSON] capture_id=%s records=%u output=%s\n", capture_id.c_str(), pa_scheduler::kWorkers, + output_path.c_str()); + return true; +} + +} // namespace + +int main(int argc, char **argv) { + // 参数和 ELF 在创建 ACL 资源前完成校验，早期错误不会留下 device、stream 或 kernel handle。 + pa_scheduler::host::Options options; + PmuOptions pmu_options; + WinnerWorkloadOptions workload_options; + std::vector pmu_argv; + std::vector common_argv; + if (!ParseWinnerWorkloadOptions(argc, argv, &workload_options, &pmu_argv) || + !ParsePmuOptions( + static_cast(pmu_argv.size()), pmu_argv.data(), &pmu_options, &common_argv + )) { + return EXIT_FAILURE; + } + const pa_scheduler::host::ParseStatus parse_status = pa_scheduler::host::ParseOptions( + static_cast(common_argv.size()), common_argv.data(), true, &options + ); + if (parse_status != pa_scheduler::host::ParseStatus::Ok) { + if (parse_status == pa_scheduler::host::ParseStatus::Help) { + std::fprintf( + stderr, + "CCEC PMU options: [--pmu-window " + "off|empty|scalar|scalar-double|icache-single|submit-all] " + "[--pmu-scalar-nops N] [--pmu-icache-trials N] [--pmu-json FILE]\n" + ); + std::fprintf( + stderr, + "CCEC winner workload options: [--winner-workload scalar-nop|real-compute] " + "[--real-compute-count N | --real-compute-counts QK,SF,PV,UP] " + "[--real-compute-pattern constant|layout-diagnostic]\n" + ); + std::fprintf( + stderr, + "Default: real-compute, constant, counts=6,28,4,1; " + "scalar-nop is the calibration compatibility mode.\n" + ); + } + return parse_status == pa_scheduler::host::ParseStatus::Help ? EXIT_SUCCESS : EXIT_FAILURE; + } + if (!ValidateWinnerWorkloadOptions(workload_options)) return EXIT_FAILURE; +#if PA_BUILD_SWIMLANE + // swimlane host 与同目录 kernel 是成套产物；它不允许借旧参数重新开启 + // 已从 device ELF 编译掉的 PMU/phase-profile 路径。 + if (pmu_options.mode != pa_scheduler::ccec_pmu::WindowMode::Off || + !pmu_options.json_path.empty()) { + std::fprintf( + stderr, + "This is a swimlane build; PMU collection requires the separate submit-pmu build.\n" + ); + return EXIT_FAILURE; + } + if (options.profile_phases) { + std::fprintf( + stderr, + "--profile-phases is not part of the swimlane build; use submit-pmu phase attribution.\n" + ); + return EXIT_FAILURE; + } +#elif PA_BUILD_SUBMIT_PMU + // submit-pmu 是编译期固定 phase 的单轮诊断产物；host、kernel 与 owner + // 必须共同拒绝旧校准窗口和任何泳道/phase-profile 观察代码。 + if (pmu_options.mode != pa_scheduler::ccec_pmu::WindowMode::SubmitAll) { + std::fprintf(stderr, "The submit-pmu build requires --pmu-window submit-all.\n"); + return EXIT_FAILURE; + } + if (options.runs != 1 || options.trace_enabled || options.trace_atomics || + options.profile_phases || options.analyze_swimlane || !options.swimlane_json.empty()) { + std::fprintf( + stderr, + "submit-pmu requires one PMU-only run: --runs 1 --no-swimlane and no trace/profile options.\n" + ); + return EXIT_FAILURE; + } +#elif PA_BUILD_PERF_CLOCK + // perf-clock 是唯一决定候选净性能的低扰动构建。它只允许完整 + // Submit 首尾边界，不能借运行时参数重新打开泳道、atomic、PMU + // 或 phase-profile。 + if (pmu_options.mode != pa_scheduler::ccec_pmu::WindowMode::Off || + !pmu_options.json_path.empty()) { + std::fprintf(stderr, "The perf-clock build forbids PMU collection and PMU JSON.\n"); + return EXIT_FAILURE; + } + if (options.runs != 1 || options.trace_enabled || options.trace_atomics || + options.profile_phases || options.analyze_swimlane || + !options.swimlane_json.empty()) { + std::fprintf( + stderr, + "perf-clock requires one trace-free run: " + "--runs 1 --no-swimlane and no trace/profile options.\n" + ); + return EXIT_FAILURE; + } +#endif + if (!pmu_options.json_path.empty() && + pmu_options.mode == pa_scheduler::ccec_pmu::WindowMode::Off) { + std::fprintf(stderr, "--pmu-json requires a non-off --pmu-window.\n"); + return EXIT_FAILURE; + } + if (!pmu_options.json_path.empty() && options.runs != 1) { + // 一个 sidecar 对应一次采集，禁止多轮覆写后丢失逐轮边界。 + std::fprintf(stderr, "--pmu-json requires --runs 1 to avoid overwriting captures.\n"); + return EXIT_FAILURE; + } + if (!pmu_options.json_path.empty() && + (options.trace_enabled || options.trace_atomics || options.profile_phases || + options.analyze_swimlane || !options.swimlane_json.empty())) { + std::fprintf( + stderr, + "--pmu-json requires PMU-only collection: add --no-swimlane and do not enable " + "phase profiling, atomic tracing, swimlane analysis, or swimlane JSON.\n" + ); + return EXIT_FAILURE; + } + if (!pmu_options.json_path.empty() && + (access(pmu_options.json_path.c_str(), F_OK) == 0 || + access((pmu_options.json_path + ".tmp").c_str(), F_OK) == 0)) { + std::fprintf( + stderr, "Refusing to overwrite an existing PMU JSON or temporary file: %s\n", + pmu_options.json_path.c_str() + ); + return EXIT_FAILURE; + } + const std::vector binary_data = ReadBinary(options.kernel_path); + if (binary_data.empty()) { + std::fprintf(stderr, "Cannot read kernel binary: %s\n", options.kernel_path.c_str()); + return EXIT_FAILURE; + } + const bool real_compute = + workload_options.mode == pa_scheduler::WinnerWorkloadMode::RealCompute; + std::vector workload_image; + std::vector workload_outputs; + if (real_compute) { + pa_scheduler::host::InitializeWinnerWorkloadBuffers( + workload_options, &workload_image, &workload_outputs + ); + } + pa_scheduler::host::PrintBanner("CCEC", options); + pa_scheduler::host::PrintWinnerWorkloadConfig(workload_options, options.nops); + std::printf( + "[PMU-CONFIG] window=%s calibration_scalar_nops=%u icache_trials=%u source=direct-per-core " + "owner=main-aicpu-path-a\n", + PmuModeName(pmu_options.mode), pmu_options.scalar_nops, pmu_options.icache_trials + ); + + // 正常及后处理路径依次完成 ACL 初始化、选卡、stream/ELF/设备区创建、launch/D2H + // 和尾部清理；初始化、传输或 launch 的早期错误仍按当前实现就地返回。 + if (!CheckAcl(aclInit(nullptr), "aclInit") || !CheckAcl(aclrtSetDevice(options.device), "aclrtSetDevice")) { + return EXIT_FAILURE; + } + aclrtStream stream = nullptr; + if (!CheckAcl(aclrtCreateStream(&stream), "aclrtCreateStream")) return EXIT_FAILURE; + + rtDevBinary_t binary{RT_DEV_BINARY_MAGIC_ELF, 0, binary_data.data(), binary_data.size()}; + void *kernel_handle = nullptr; + bool registered_all = true; + // 先尝试注册带 mixed metadata 的 ELF；若 rtRegisterAllKernel 报错或未返回 handle， + // 再尝试无 tiling-key 装载。这里仅描述实际回退条件，不假设具体运行时原因。 + rtError_t register_error = rtRegisterAllKernel(&binary, &kernel_handle); + if (register_error != RT_ERROR_NONE || kernel_handle == nullptr) { + registered_all = false; + register_error = rtBinaryLoadWithoutTilingKey(binary_data.data(), binary_data.size(), &kernel_handle); + } + if (!CheckRt(register_error, "register mixed AICore ELF") || kernel_handle == nullptr) return EXIT_FAILURE; + + // SchedulerState 保留被测关键 offset、DistCore ABI 和约 1 GiB 生产总跨度； + // 使用 HUGE_FIRST 降低大块设备内存碎片风险。 + void *state_device = nullptr; + if (!CheckAcl( + aclrtMalloc(&state_device, sizeof(pa_scheduler::SchedulerState), ACL_MEM_MALLOC_HUGE_FIRST), + "aclrtMalloc(state)" + )) { + return EXIT_FAILURE; + } + if ((reinterpret_cast(state_device) & 63U) != 0) { + std::fprintf(stderr, "Device state is not 64-byte aligned: %p\n", state_device); + return EXIT_FAILURE; + } + + // 真实 PTO 负载使用独立 GM，不解引用调度器中只用于依赖建模的 synthetic tensor 地址。 + // 这里先于 PMU owner 分配；每轮 H2D 初始化虽在 owner 配置之后，但仍位于 + // launch/wall 计时之前，因此两者都不进入 Submit 性能窗口。 + ScopedAclDeviceAllocation workload_allocation; + if (real_compute && + !CheckAcl( + aclrtMalloc( + workload_allocation.Address(), pa_scheduler::winner_workload::kWorkspaceBytes, + ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(real-compute workspace)" + )) { + return EXIT_FAILURE; + } + void *workload_device = workload_allocation.Get(); + if (real_compute && (reinterpret_cast(workload_device) & 63U) != 0) { + std::fprintf(stderr, "Real-compute workspace is not 64-byte aligned: %p\n", workload_device); + return EXIT_FAILURE; + } + + // 泳道区大小由当前构建 ABI 的 kTraceBytes 决定：private 约 + // 192 MiB，shared compact full-swimlane 约 54.4 MiB。关闭泳道时 + // 不申请，也不会传递有效 base。 + // 该分配先于 PMU owner 配置，失败时不会留下需要恢复的 selector/MMIO 会话。 + void *trace_device = nullptr; + if (options.trace_enabled && + !CheckAcl( + aclrtMalloc(&trace_device, pa_scheduler::kTraceBytes, ACL_MEM_MALLOC_HUGE_FIRST), + "aclrtMalloc(swimlane trace)" + )) { + return EXIT_FAILURE; + } + if (options.trace_enabled && (reinterpret_cast(trace_device) & 63U) != 0) { + std::fprintf(stderr, "Device swimlane trace is not 64-byte aligned: %p\n", trace_device); + return EXIT_FAILURE; + } + + PmuRegisterMappings pmu_mappings; + pa_scheduler::pmu_owner::PmuOwnerSession pmu_owner; + pa_scheduler::pmu_owner::PmuOwnerControl pmu_owner_evidence{}; + bool pmu_owner_evidence_valid = false; + const void *pmu_registers_device = nullptr; + if (pmu_options.mode != pa_scheduler::ccec_pmu::WindowMode::Off) { + if (!MapPmuRegisters(options.device, &pmu_mappings)) return EXIT_FAILURE; + const std::string dispatcher_path = pa_scheduler::pmu_owner::ArtifactBesideKernel( + options.kernel_path, "libpa_scheduler_pmu_owner_dispatcher.so" + ); + const std::string owner_path = pa_scheduler::pmu_owner::ArtifactBesideKernel( + options.kernel_path, "libpa_scheduler_pmu_owner_aicpu.so" + ); + if (!pmu_owner.Initialize( + options.device, stream, dispatcher_path, owner_path, pmu_mappings.register_bases + ) || + !pmu_owner.Configure()) { + (void)pmu_owner.Finalize(); + (void)UnmapPmuRegisters(options.device, &pmu_mappings); + return EXIT_FAILURE; + } + pmu_owner_evidence = pmu_owner.Control(); + pmu_owner_evidence_valid = true; + pmu_registers_device = reinterpret_cast(pmu_owner.RegisterTableDeviceAddress()); + } + + // host shadow 保留约 1 GiB 总跨度以便按关键 offset 寻址。private 每轮 + // 只传共享前缀、控制区和结果区；shared 再单独传 results 后的 map + // sidecar，既有三个范围的大小和边界均不改变。 + std::unique_ptr state(new pa_scheduler::SchedulerState); + pa_scheduler::TraceHeader trace_header{}; + std::vector spans; + std::vector startup_barrier_spans; + std::vector final_barrier_spans; + std::vector final_drain_spans; + std::vector lifecycle_spans; + bool execution_ok = true; + bool all_passed = true; + bool postprocess_ok = true; + bool pmu_json_ready = false; + bool pmu_json_semantic_passed = false; + bool pmu_json_workload_output_passed = false; + uint32_t pmu_json_run = 0U; + double pmu_json_host_us = 0.0; + double pmu_json_submit_span_us = 0.0; + PmuValidation pmu_json_validation; + for (uint32_t run = 1; run <= options.runs; ++run) { + pa_scheduler::host::InitializeState(state.get(), options); +#if PTO_FDWIC_SHARED_MAP + pa_scheduler::host::SharedHostTaskPlan launch_plan; + pa_scheduler::host::SharedHostHeapAdmission heap_admission; + std::string admission_error; + if (!pa_scheduler::host::BuildSharedHostTaskPlan( + *state, &launch_plan, &admission_error + ) || + !pa_scheduler::host::ValidateSharedHostHeapAdmission( + launch_plan, state->heap_size, + &heap_admission, &admission_error + )) { + std::fprintf( + stderr, + "Shared launch rejected before A5 worker start: %s\n", + admission_error.c_str() + ); + execution_ok = false; + all_passed = false; + break; + } + pa_scheduler::host::PrintSharedHostHeapAdmission( + launch_plan, heap_admission + ); +#endif + pa_scheduler::host::ConfigureTrace(state.get(), options, trace_device); + ConfigurePmu(state.get(), pmu_options, pmu_registers_device); + ConfigureWinnerWorkload(state.get(), workload_options, workload_device); + if (real_compute && + !CheckAcl( + aclrtMemcpy( + workload_device, pa_scheduler::winner_workload::kWorkspaceBytes, + workload_image.data(), pa_scheduler::winner_workload::kWorkspaceBytes, + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D real-compute workspace)" + )) { + execution_ok = false; + break; + } + if (options.trace_enabled) { + // 每轮只需重置约 7 KiB header；各 worker 会从 count=0 + // 覆盖自己的物理记录区，无需清零整块 trace 缓冲。 + pa_scheduler::host::InitializeTraceHeader(&trace_header); + if (!CheckAcl( + aclrtMemcpy( + trace_device, sizeof(trace_header), &trace_header, sizeof(trace_header), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D swimlane header)" + )) { + execution_ok = false; + break; + } + } + // 为避免每轮搬运约 1 GiB，只 H2D 被测共享前缀和位于生产总跨度之后的 + // standalone 控制区； + // 每个 worker 的大块私有状态由 device kernel 自行初始化。 + if (!CheckAcl( + aclrtMemcpy( + state_device, pa_scheduler::host::StatePrefixBytes(), state.get(), + pa_scheduler::host::StatePrefixBytes(), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D state prefix)" + ) || + !CheckAcl( + aclrtMemcpy( + &static_cast(state_device)->config, + pa_scheduler::host::ControlBytes(), &state->config, pa_scheduler::host::ControlBytes(), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D standalone controls)" + )) { + execution_ok = false; + break; + } +#if PA_BUILD_PERF_CLOCK + // WorkerResult 的 PMU 尾槽通常由 submit-pmu kernel 另行发布；perf-clock + // 没有这些写。launch 前显式清零约 90 KiB 结果区，随后再要求它们保持 + // 为零，避免设备分配残值伪装成观察代码泄漏。该搬运发生在计时窗口外。 + if (!CheckAcl( + aclrtMemcpy( + &static_cast(state_device)->results[0], + pa_scheduler::host::ResultBytes(), state->results, + pa_scheduler::host::ResultBytes(), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D zeroed perf-clock results)" + )) { + execution_ok = false; + break; + } +#endif +#if PTO_FDWIC_SHARED_MAP + // shared map 位于 results 之后，不能扩大 ControlBytes 或把它重复混入 + // 每核结果范围；初始化搬运仍发生在 launch 计时开始之前。 + if (!CheckAcl( + aclrtMemcpy( + &static_cast(state_device)->shared_map, + pa_scheduler::host::SharedSidecarBytes(), &state->shared_map, + pa_scheduler::host::SharedSidecarBytes(), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D shared TensorMap sidecar)" + )) { + execution_ok = false; + break; + } +#endif + void *kernel_args[] = {state_device}; + rtArgsEx_t args_info{}; + args_info.args = kernel_args; + args_info.argsSize = sizeof(kernel_args); + rtTaskCfgInfo_t task_config{}; + // launch 维度是 32 个物理 mixed block；ELF metadata 让每个 block 同时产生 1 AIC + 2 AIV，共 96 worker。 + // wall time 在同步完成处截止，包含 launch、完整调度、最终 drain 和 stream 同步，但不包含后续 D2H/JSON。 + const auto wall_begin = std::chrono::steady_clock::now(); + if (!CheckRt( + rtKernelLaunchWithHandleV2( + kernel_handle, 0, pa_scheduler::kAicWorkers, &args_info, nullptr, stream, &task_config + ), + "rtKernelLaunchWithHandleV2" + ) || + !CheckAcl(aclrtSynchronizeStream(stream), "aclrtSynchronizeStream")) { + execution_ok = false; + break; + } + const auto wall_end = std::chrono::steady_clock::now(); + const double host_us = std::chrono::duration(wall_end - wall_begin).count(); + // D2H 同样避开约 1 GiB 的 worker arena：共享前缀用于 + // flag/vend/frontier 校验，末尾 results 单独回传；shared sidecar + // 在 results 成功回读后再作为独立范围搬回。 + if (!CheckAcl( + aclrtMemcpy( + state.get(), pa_scheduler::host::StatePrefixBytes(), state_device, + pa_scheduler::host::StatePrefixBytes(), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H state prefix)" + ) || + !CheckAcl( + aclrtMemcpy( + &state->final_barrier, pa_scheduler::host::FinalBarrierStateBytes(), + &static_cast(state_device)->final_barrier, + pa_scheduler::host::FinalBarrierStateBytes(), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H final barrier)" + ) || + !CheckAcl( + aclrtMemcpy( + state->results, pa_scheduler::host::ResultBytes(), + &static_cast(state_device)->results[0], + pa_scheduler::host::ResultBytes(), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H worker results)" + )) { + execution_ok = false; + break; + } +#if PTO_FDWIC_SHARED_MAP + if (!CheckAcl( + aclrtMemcpy( + &state->shared_map, pa_scheduler::host::SharedSidecarBytes(), + &static_cast(state_device)->shared_map, + pa_scheduler::host::SharedSidecarBytes(), ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H shared TensorMap sidecar)" + )) { + execution_ok = false; + break; + } +#endif + if (real_compute && + !CheckAcl( + aclrtMemcpy( + workload_outputs.data(), + static_cast(pa_scheduler::winner_workload::kOutputTiles) * + pa_scheduler::winner_workload::kTileBytes, + static_cast(workload_device) + + pa_scheduler::winner_workload::kSharedInputTiles * + pa_scheduler::winner_workload::kTileBytes, + static_cast(pa_scheduler::winner_workload::kOutputTiles) * + pa_scheduler::winner_workload::kTileBytes, + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H real-compute outputs)" + )) { + execution_ok = false; + break; + } + if (options.trace_enabled && + !CheckAcl( + aclrtMemcpy( + &trace_header, sizeof(trace_header), trace_device, sizeof(trace_header), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H swimlane header)" + )) { + execution_ok = false; + break; + } + // 常规校验只需 header 中的 per-worker count；真实 records 在分析或导出时才按核、按实际 count 懒加载。 + const auto read_trace_records = + [trace_device]( + uint32_t worker, uint32_t count, + pa_scheduler::TraceStorageRecord *records + ) { + // 每核记录区采用固定容量 stride；只复制 header 声明的实际 count，避免 D2H 未使用的尾部空间。 + const size_t offset = + pa_scheduler::TraceRecordsOffset(worker); + return CheckAcl( + aclrtMemcpy( + records, + static_cast(count) * + sizeof(pa_scheduler::TraceStorageRecord), + static_cast(trace_device) + offset, + static_cast(count) * + sizeof(pa_scheduler::TraceStorageRecord), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H swimlane records)" + ); + }; +#if PTO_FDWIC_SHARED_MAP + const auto read_submit_claim_records = + [trace_device]( + uint32_t worker, uint32_t count, + pa_scheduler::SharedSubmitClaimTraceRecord *records + ) { + const size_t offset = + pa_scheduler::TraceSubmitClaimOffset(worker); + return CheckAcl( + aclrtMemcpy( + records, + static_cast(count) * + sizeof(pa_scheduler::SharedSubmitClaimTraceRecord), + static_cast(trace_device) + offset, + static_cast(count) * + sizeof(pa_scheduler::SharedSubmitClaimTraceRecord), + ACL_MEMCPY_DEVICE_TO_HOST + ), + "aclrtMemcpy(D2H Submit/Claim endpoint records)" + ); + }; +#endif + // 先完成共享状态、拓扑、计数和 trace header 的语义校验，再允许 raw JSON 成为性能证据。 + const pa_scheduler::host::Metrics metrics = pa_scheduler::host::Validate( + *state, run, host_us, options.trace_enabled ? &trace_header : nullptr + ); + all_passed &= metrics.passed; + const bool workload_passed = + !real_compute || ValidateRealComputeOutputs( + *state, workload_options, workload_outputs, run + ); + all_passed &= workload_passed; + PmuValidation pmu_validation; + const bool pmu_passed = ValidatePmu( + *state, run, pmu_options, workload_options, + pmu_options.mode == pa_scheduler::ccec_pmu::WindowMode::Off ? nullptr : &pmu_owner.Control(), + &pmu_validation + ); + all_passed &= pmu_passed; + spans.push_back(metrics.submit_span_us); + startup_barrier_spans.push_back(metrics.startup_barrier_span_us); + final_barrier_spans.push_back(metrics.final_barrier_span_us); + final_drain_spans.push_back(metrics.final_drain_span_us); + lifecycle_spans.push_back(metrics.lifecycle_span_us); + if (!pmu_options.json_path.empty()) { + if (!metrics.passed || !workload_passed || !pmu_passed || !pmu_owner_evidence_valid) { + std::fprintf(stderr, "PMU JSON rejected because semantic, PMU, or owner validation failed.\n"); + postprocess_ok = false; + break; + } + pmu_json_ready = true; + pmu_json_semantic_passed = metrics.passed && workload_passed; + pmu_json_workload_output_passed = workload_passed; + pmu_json_run = run; + pmu_json_host_us = host_us; + pmu_json_submit_span_us = metrics.submit_span_us; + pmu_json_validation = pmu_validation; + } + if (options.analyze_swimlane && + !pa_scheduler::host::AnalyzeSwimlaneRecords( + trace_header, *state, read_trace_records +#if PTO_FDWIC_SHARED_MAP + , read_submit_claim_records +#endif + )) { + // 后处理错误使用 break 汇入统一 cleanup；与初始化/launch 失败的进程级立即返回语义区分开。 + postprocess_ok = false; + break; + } + if (!options.swimlane_json.empty()) { + // 只有语义校验通过才把 raw JSON 经“临时文件写完后 rename”发布， + // 避免把截断或错误调度结果误当成可用性能证据。 + if (!metrics.passed || !workload_passed) { + std::fprintf(stderr, "Skipping swimlane export because semantic validation failed.\n"); + postprocess_ok = false; + break; + } + if (!pa_scheduler::host::ExportSwimlaneRecords( +#if PTO_FDWIC_SHARED_MAP + trace_header, *state, options.swimlane_json, +#else + trace_header, options.swimlane_json, +#endif + workload_options.mode, + real_compute + ? workload_options.repeats + : pa_scheduler::WorkloadCounts{ + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + }, + real_compute + ? pa_scheduler::host::RealComputePatternName(workload_options.pattern) + : "none", + options.final_barrier_shape, options.trace_atomics, + read_trace_records +#if PTO_FDWIC_SHARED_MAP + , read_submit_claim_records +#endif + )) { + postprocess_ok = false; + break; + } + } + } + + const double median_submit_span_us = spans.empty() ? 0.0 : pa_scheduler::host::Median(spans); +#if PA_BUILD_PERF_CLOCK + std::printf( + "[SUMMARY] runs=%u completed_runs=%zu final_shape=%s " + "median_submit_span_us=%.3f lifecycle_timing=disabled " + "execution_status=%s semantic_status=%s postprocess_status=%s\n", + options.runs, spans.size(), + pa_scheduler::host::FinalBarrierShapeName(options.final_barrier_shape), + median_submit_span_us, + execution_ok ? "PASS" : "FAIL", + all_passed ? "PASS" : "FAIL", + postprocess_ok ? "PASS" : "FAIL" + ); +#else + const double median_startup_barrier_us = + startup_barrier_spans.empty() ? 0.0 : pa_scheduler::host::Median(startup_barrier_spans); + const double median_final_barrier_us = + final_barrier_spans.empty() ? 0.0 : pa_scheduler::host::Median(final_barrier_spans); + const double median_final_drain_us = + final_drain_spans.empty() ? 0.0 : pa_scheduler::host::Median(final_drain_spans); + const double median_lifecycle_us = lifecycle_spans.empty() ? 0.0 : pa_scheduler::host::Median(lifecycle_spans); + std::printf( + "[SUMMARY] runs=%u completed_runs=%zu final_shape=%s median_submit_span_us=%.3f " + "median_startup_barrier_us=%.3f median_final_barrier_us=%.3f " + "median_final_drain_us=%.3f median_lifecycle_us=%.3f " + "execution_status=%s semantic_status=%s postprocess_status=%s\n", + options.runs, spans.size(), pa_scheduler::host::FinalBarrierShapeName(options.final_barrier_shape), + median_submit_span_us, median_startup_barrier_us, median_final_barrier_us, median_final_drain_us, + median_lifecycle_us, execution_ok ? "PASS" : "FAIL", all_passed ? "PASS" : "FAIL", + postprocess_ok ? "PASS" : "FAIL" + ); +#endif + + // 后处理失败也统一走设备资源释放、ELF 卸载和 ACL 收尾，避免文件系统错误遗留运行时上下文。 + bool cleanup_ok = true; + bool pmu_owner_restore_ok = true; + // 先释放依赖当前 device/context 的大块内存，再卸载 ELF、销毁 stream，最后 reset device 与 finalize ACL。 + if (trace_device != nullptr) { + cleanup_ok &= CheckAcl(aclrtFree(trace_device), "aclrtFree(swimlane trace)"); + } + if (pmu_registers_device != nullptr) { + // owner 必须在 MMIO 映射、device context 和 ACL runtime 仍有效时恢复。 + pmu_owner_restore_ok = pmu_owner.Finalize(); + cleanup_ok &= pmu_owner_restore_ok; + cleanup_ok &= UnmapPmuRegisters(options.device, &pmu_mappings); + } + if (workload_device != nullptr) { + cleanup_ok &= CheckAcl( + aclrtFree(workload_allocation.Release()), "aclrtFree(real-compute workspace)" + ); + } + cleanup_ok &= CheckAcl(aclrtFree(state_device), "aclrtFree(state)"); + const rtError_t unload_error = + registered_all ? rtDevBinaryUnRegister(kernel_handle) : rtBinaryUnLoad(kernel_handle); + cleanup_ok &= CheckRt(unload_error, "unload mixed AICore ELF"); + cleanup_ok &= CheckAcl(aclrtDestroyStream(stream), "aclrtDestroyStream"); + cleanup_ok &= CheckAcl(aclrtResetDevice(options.device), "aclrtResetDevice"); + cleanup_ok &= CheckAcl(aclFinalize(), "aclFinalize"); + if (!pmu_options.json_path.empty()) { + if (!pmu_json_ready || !all_passed || !postprocess_ok || !cleanup_ok || !pmu_owner_restore_ok) { + std::fprintf(stderr, "PMU JSON was not published because the capture or restore transaction failed.\n"); + postprocess_ok = false; + } else if (!ExportPmuJson( + *state, options, pmu_options, workload_options, pmu_json_run, + pmu_json_host_us, pmu_json_submit_span_us, pmu_json_validation, + pmu_json_semantic_passed, pmu_json_workload_output_passed, + pmu_owner_evidence, + pmu_owner_restore_ok, pmu_options.json_path + )) { + postprocess_ok = false; + } + } + // 运行语义、后处理和资源清理三者全部成功，进程才返回成功，脚本据此决定是否继续生成 merged 泳道。 + return execution_ok && all_passed && postprocess_ok && cleanup_ok ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/atomic_probe/pa_scheduler/ccec/kernel.cpp b/tests/atomic_probe/pa_scheduler/ccec/kernel.cpp new file mode 100644 index 0000000000..73a2e61321 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/kernel.cpp @@ -0,0 +1,421 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include +#include +#include +#include + +#include "pmu_probe.h" +#include "../common/winner_workload.h" + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" + +#define PA_CCEC_OPS_DEFINE_REAL_WORKLOAD 1 +#include "ccec_ops.h" +#undef PA_CCEC_OPS_DEFINE_REAL_WORKLOAD + +using pa_scheduler_ccec::CcecOps; + +namespace { +#if PA_BUILD_SUBMIT_PMU +struct PmuSnapshot { + uint64_t total_cycles = 0; + uint32_t vector_busy = 0; + uint32_t cube_busy = 0; + uint32_t scalar_busy = 0; + uint32_t mte1_busy = 0; + uint32_t mte2_busy = 0; + uint32_t mte3_busy = 0; + uint32_t icache_requests = 0; + uint32_t icache_misses = 0; + uint32_t fix_busy = 0; + uint32_t status = 0; +}; + +struct IcacheShadowSnapshot { + uint32_t requests = 0; + uint32_t misses = 0; +}; + +} // namespace + +namespace pa_scheduler_ccec { + +struct SubmitPmuContext { + uint64_t reg_base = 0; + uint64_t shadow_requests = 0; + uint64_t shadow_misses = 0; + uint64_t phase_requests = 0; + uint64_t phase_misses = 0; + // begin 的 shadow read-clear 完成后取起点，end 的 shadow read-clear 之前 + // 取终点；累计值因此不包含两次 PMU 寄存器读取本身。 + uint64_t phase_elapsed_ticks = 0; + uint64_t phase_begin_tick = 0; + uint32_t selector_status = 0; + uint32_t phase_status = pa_scheduler::ccec_pmu::kPhaseStatusRequested; + uint32_t phase_calls = 0; + uint32_t begin_reads = 0; + uint32_t end_reads = 0; + bool started = false; + bool phase_armed = false; + bool boundary_error = false; +}; + +} // namespace pa_scheduler_ccec + +namespace { + +using pa_scheduler_ccec::SubmitPmuContext; + +template +__aicore__ inline uint32_t ReadPmuRegister(uint64_t reg_base) { + // 传给 ld_dev 的是重基址后的 __gm__ 指针；相对 offset 均落在编译器允许的 [-2048, 2047]。 + int32_t *block = reinterpret_cast(reg_base + BlockOffset); + return static_cast(ld_dev(block, static_cast(RegisterOffset - BlockOffset))); +} + +__aicore__ inline PmuSnapshot ReadObservedCounters(uint64_t reg_base) { + PmuSnapshot sample; + sample.vector_busy = ReadPmuRegister(reg_base); + sample.cube_busy = ReadPmuRegister(reg_base); + sample.scalar_busy = ReadPmuRegister(reg_base); + sample.mte1_busy = ReadPmuRegister(reg_base); + sample.mte2_busy = ReadPmuRegister(reg_base); + // submit-pmu 将 CNT5 留给 shadow I-cache miss。这里不能提前读取，否则 + // read-to-clear 会让随后的 shadow tail 漏计；MTE3 busy 在该诊断构建不可用。 + sample.mte3_busy = 0; + sample.icache_requests = ReadPmuRegister(reg_base); + sample.icache_misses = ReadPmuRegister(reg_base); + const uint64_t low = ReadPmuRegister(reg_base); + const uint64_t high = ReadPmuRegister(reg_base); + sample.total_cycles = low | (high << 32); + return sample; +} + +__aicore__ inline IcacheShadowSnapshot ReadShadowCounters(uint64_t reg_base) { + IcacheShadowSnapshot sample; + sample.requests = ReadPmuRegister(reg_base); + sample.misses = ReadPmuRegister(reg_base); + return sample; +} + +struct PmuRegisterContext { + uint64_t reg_base = 0; + uint32_t status = 0; + bool shadow_selectors = false; +}; + +__aicore__ inline PmuRegisterContext ResolvePmuRegisters(__gm__ pa_scheduler::SchedulerState *state) { + using namespace pa_scheduler::ccec_pmu; + PmuRegisterContext context; + const uint32_t physical_core_id = static_cast(get_coreid()) & kStatusCoreIdMask; + context.status = kStatusRequested | (physical_core_id << kStatusCoreIdShift); + const uint64_t table_address = state->pmu_probe.register_table; + if (state->pmu_probe.magic != kConfigMagicValue || table_address == 0 || + physical_core_id >= kPhysicalSubcoreCount) { + return context; + } + context.status |= kStatusCoreIdValid; + __gm__ const uint64_t *register_bases = reinterpret_cast<__gm__ const uint64_t *>(table_address); + context.reg_base = register_bases[physical_core_id]; + if (context.reg_base == 0) return context; + context.status |= kStatusRegMapped; + + // selector 与同 phase 目录内的 owner 逐项核对。CNT5/CNT8 分别重复 + // CNT7/CNT6；CNT9 保持正式 PIPE_UTIL 的 unused(0) 口径。 + if (ReadPmuRegister(context.reg_base) == kVectorBusyEvent) + context.status |= kStatusCnt0Selector; + if (ReadPmuRegister(context.reg_base) == kCubeBusyEvent) + context.status |= kStatusCnt1Selector; + if (ReadPmuRegister(context.reg_base) == kScalarBusyEvent) + context.status |= kStatusCnt2Selector; + if (ReadPmuRegister(context.reg_base) == kMte1BusyEvent) + context.status |= kStatusCnt3Selector; + if (ReadPmuRegister(context.reg_base) == kMte2BusyEvent) + context.status |= kStatusCnt4Selector; + const bool cnt5_ok = + ReadPmuRegister(context.reg_base) == kIcacheMissEvent; + if (cnt5_ok) + context.status |= kStatusCnt5Selector; + if (ReadPmuRegister(context.reg_base) == kIcacheRequestEvent) + context.status |= kStatusCnt6Selector; + if (ReadPmuRegister(context.reg_base) == kIcacheMissEvent) + context.status |= kStatusCnt7Selector; + const bool cnt8_ok = + ReadPmuRegister(context.reg_base) == kIcacheRequestEvent; + const bool cnt9_unused = + ReadPmuRegister(context.reg_base) == 0U; + if (cnt8_ok) + context.status |= kStatusCnt8Selector; + context.shadow_selectors = cnt5_ok && cnt8_ok && cnt9_unused; + return context; +} + +__aicore__ inline void PublishPmuSnapshot( + __gm__ pa_scheduler::WorkerResult &result, const PmuSnapshot &sample +) { + // 每核独占 sidecar 通过 bypass store 一次性发布；这些写发生在 PMU stop/read 之后， + // 不进入被导出的 Submit 窗口。 + CcecOps::Publish(&result.pmu_total_cycles, sample.total_cycles); + CcecOps::Publish(&result.pmu_scalar_busy, sample.scalar_busy); + CcecOps::Publish(&result.pmu_icache_requests, sample.icache_requests); + CcecOps::Publish(&result.pmu_icache_misses, sample.icache_misses); + CcecOps::Publish(&result.pmu_status, sample.status); + CcecOps::Publish(&result.pmu_vector_busy, sample.vector_busy); + CcecOps::Publish(&result.pmu_cube_busy, sample.cube_busy); + CcecOps::Publish(&result.pmu_mte1_busy, sample.mte1_busy); + CcecOps::Publish(&result.pmu_mte2_busy, sample.mte2_busy); + CcecOps::Publish(&result.pmu_mte3_busy, sample.mte3_busy); + // CNT8 已改作 shadow request，submit-pmu 不再发布 fix-busy。 + CcecOps::Publish(&result.pmu_fix_busy, static_cast(0)); + CcecOps::Publish(&result.pmu_warm_total_cycles, static_cast(0)); + CcecOps::Publish(&result.pmu_warm_window_ticks, static_cast(0)); +} + +__aicore__ inline bool FitsUint32(uint64_t value) { + return value <= 0xffffffffULL; +} + +__aicore__ inline void PublishSubmitPmuContext( + __gm__ pa_scheduler::WorkerResult &result, const SubmitPmuContext &context +) { + CcecOps::Publish(&result.pmu_build_variant, pa_scheduler::kBuildVariantSubmitPmu); + CcecOps::Publish( + &result.pmu_phase_id, + static_cast(pa_scheduler::kCompiledSubmitPmuPhase) + ); + CcecOps::Publish(&result.pmu_phase_calls, context.phase_calls); + CcecOps::Publish(&result.pmu_phase_status, context.phase_status); + CcecOps::Publish(&result.pmu_phase_begin_reads, context.begin_reads); + CcecOps::Publish(&result.pmu_phase_end_reads, context.end_reads); + CcecOps::Publish(&result.pmu_phase_elapsed_ticks, context.phase_elapsed_ticks); + CcecOps::Publish(&result.pmu_phase_icache_requests, static_cast(context.phase_requests)); + CcecOps::Publish(&result.pmu_phase_icache_misses, static_cast(context.phase_misses)); + CcecOps::Publish(&result.pmu_shadow_icache_requests, static_cast(context.shadow_requests)); + CcecOps::Publish(&result.pmu_shadow_icache_misses, static_cast(context.shadow_misses)); +} + +} // namespace + +namespace pa_scheduler_ccec { + +__aicore__ inline CcecOps::PmuContext CcecOps::PmuWindowStart( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +) { + using namespace pa_scheduler::ccec_pmu; + (void)worker_id; + SubmitPmuContext context; + const WindowMode mode = static_cast(state->pmu_probe.mode); + if (mode != WindowMode::SubmitAll) return context; + // Main AICPU owner 已在 launch 前配置并开启计数；先 stop + snapshot/read-clear， + // 再解析 selector，避免这些 ld_dev 污染完整 Submit 窗口。 + bisheng::cce::metrics_prof_stop(); + const PmuRegisterContext registers = ResolvePmuRegisters(state); + context.reg_base = registers.reg_base; + context.selector_status = registers.status; + if (registers.shadow_selectors) { + context.phase_status |= kPhaseStatusShadowSelectors; + } + if (context.reg_base == 0) return context; + (void)ReadObservedCounters(context.reg_base); + (void)ReadShadowCounters(context.reg_base); + bisheng::cce::metrics_prof_start(); + context.started = true; + context.phase_status |= kPhaseStatusWindowStarted; + return context; +} + +__aicore__ inline void CcecOps::PmuPhaseBegin(PmuContext &context) { + if (!context.started || context.reg_base == 0 || context.phase_armed) { + context.boundary_error = true; + return; + } + // counter 在运行中读取即清零；begin 之前的片段只进入 shadow whole。 + const IcacheShadowSnapshot sample = ReadShadowCounters(context.reg_base); + context.shadow_requests += sample.requests; + context.shadow_misses += sample.misses; + ++context.begin_reads; + context.phase_armed = true; + // get_sys_cnt() 是本机已校准为 1 ns/tick 的 A5 系统计数器。该读取位于 + // begin 的两条 ld_dev 之后，因此不会把 read-clear 成本算进阶段时间。 + context.phase_begin_tick = CcecOps::Now(); +} + +__aicore__ inline void CcecOps::PmuPhaseEnd(PmuContext &context) { + // 先取终点再读取 shadow counter，使 end 的两条 ld_dev 同样位于阶段之外。 + const uint64_t phase_end_tick = CcecOps::Now(); + if (!context.started || context.reg_base == 0 || !context.phase_armed) { + context.boundary_error = true; + return; + } + if (phase_end_tick < context.phase_begin_tick) { + context.boundary_error = true; + } else { + context.phase_elapsed_ticks += phase_end_tick - context.phase_begin_tick; + } + // end 读出的片段同时属于完整 shadow 重建与被选中的局部阶段。 + const IcacheShadowSnapshot sample = ReadShadowCounters(context.reg_base); + context.shadow_requests += sample.requests; + context.shadow_misses += sample.misses; + context.phase_requests += sample.requests; + context.phase_misses += sample.misses; + ++context.end_reads; + ++context.phase_calls; + context.phase_armed = false; + context.phase_begin_tick = 0; +} + +__aicore__ inline void CcecOps::PmuWindowStop( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id, PmuContext &context +) { + using namespace pa_scheduler::ccec_pmu; + __gm__ pa_scheduler::WorkerResult &result = state->results[worker_id]; + PmuSnapshot sample; + sample.status = context.selector_status; + if (context.started && context.reg_base != 0) { + // gate 只在整个 Submit 前后各操作一次。停止后先读从未中途清零的 + // primary counter（不含 shadow CNT5）之后，再读取 CNT8/CNT5 tail + // 完成软件重建。 + bisheng::cce::metrics_prof_stop(); + sample = ReadObservedCounters(context.reg_base); + const IcacheShadowSnapshot tail = ReadShadowCounters(context.reg_base); + context.shadow_requests += tail.requests; + context.shadow_misses += tail.misses; + sample.status = context.selector_status | kStatusWindowStarted | kStatusWindowStopped; + context.phase_status |= kPhaseStatusWindowStopped; + if (sample.total_cycles != 0) sample.status |= kStatusTotalNonzero; + } + + if (context.shadow_requests == sample.icache_requests) + context.phase_status |= kPhaseStatusShadowRequestsMatch; + if (context.shadow_misses == sample.icache_misses) + context.phase_status |= kPhaseStatusShadowMissesMatch; + if (!context.boundary_error && !context.phase_armed && + context.begin_reads == context.end_reads && context.end_reads == context.phase_calls) + context.phase_status |= kPhaseStatusBoundariesBalanced; + // 两个 shadow counter 是顺序 ld_dev，并非同一时刻的原子快照；局部 + // phase 的 miss/request 边界会错开数条指令，故不能硬性要求局部 + // miss<=request。A5 上运行中 read-to-clear 还会与同周期事件递增竞争， + // shadow 允许小于未中途读取的 primary，但绝不能反向超过它。primary- + // shadow 是该次采集可直接给出的局部分段误差包络，而不是要静默吞掉的差值。 + if (context.phase_requests <= context.shadow_requests && + context.phase_misses <= context.shadow_misses && + context.shadow_misses <= context.shadow_requests && + context.shadow_requests <= sample.icache_requests && + context.shadow_misses <= sample.icache_misses) + context.phase_status |= kPhaseStatusValuesOrdered; + if (FitsUint32(context.shadow_requests) && FitsUint32(context.shadow_misses) && + FitsUint32(context.phase_requests) && FitsUint32(context.phase_misses)) + context.phase_status |= kPhaseStatusUint32Fit; + + const bool none_shape = + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None && + context.phase_calls == 0 && context.begin_reads == 0 && context.end_reads == 0 && + context.phase_requests == 0 && context.phase_misses == 0 && + context.phase_elapsed_ticks == 0; +#if PTO_FDWIC_SHARED_MAP + // shared 的 task 数由各 batch 的 context_len 决定。Stop 时公共结果尚未 + // 发布到 state->results，因此这里只能直接读取本 worker 已完成回放的 + // local_index。Claim/EfDrain 仍严格覆盖全部逻辑 task；Materialize/ + // Register 已改为 winner-only，设备端只验证不超过任务数，host 随后 + // 用已发布的 claim_wins 做逐核精确校验。 + const uint32_t replay_task_count = + static_cast(state->workers[worker_id].local_index); + const bool running_call_shape = + pa_scheduler::kCompiledSubmitPmuPhase == + pa_scheduler::SubmitPmuPhase::Claim || + pa_scheduler::kCompiledSubmitPmuPhase == + pa_scheduler::SubmitPmuPhase::EfDrain + ? context.phase_calls == replay_task_count + : context.phase_calls <= replay_task_count; +#else + const uint32_t expected_phase_calls = + state->config.batches * pa_scheduler::kTasksPerBatch; + const bool running_call_shape = + context.phase_calls == expected_phase_calls; +#endif + const bool running_shape = + pa_scheduler::kCompiledSubmitPmuPhase != pa_scheduler::SubmitPmuPhase::None && + running_call_shape && + context.begin_reads == context.phase_calls && + context.end_reads == context.phase_calls; + if (none_shape || running_shape) + context.phase_status |= kPhaseStatusPhaseShape; + const bool phase_time_valid = + pa_scheduler::kCompiledSubmitPmuPhase == pa_scheduler::SubmitPmuPhase::None + ? context.phase_elapsed_ticks == 0 + : context.phase_calls == 0 + ? context.phase_elapsed_ticks == 0 + : context.phase_elapsed_ticks != 0; + if (phase_time_valid) + context.phase_status |= kPhaseStatusTimeValid; + + PublishPmuSnapshot(result, sample); + PublishSubmitPmuContext(result, context); +} +#endif // PA_BUILD_SUBMIT_PMU + +} // namespace + +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) && defined(PA_BUILD_AIC) +// runtime entry/state-owner TU 每次 launch 只调用一次该 orchestration；它 +// 不是 kernel entry，最终由 version script 局部化，避免污染 runtime 入口枚举。 +extern "C" __attribute__((noinline, used)) __aicore__ void +pa_scheduler_compete_first_callback_orchestration_aic( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +) { + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aic); +} +#elif defined(PA_COMPETE_FIRST_SPLIT_FINISH) && defined(PA_BUILD_AIV) +extern "C" __attribute__((noinline, used)) __aicore__ void +pa_scheduler_compete_first_callback_orchestration_aiv( + __gm__ pa_scheduler::SchedulerState *state, uint32_t worker_id +) { + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aiv); +} +#elif defined(PA_BUILD_AIC) +// 同一源码分别按 cube/vec 架构编译；metadata 声明每个物理 block 静态组合 1 个 AIC 与 2 个 AIV。 +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aic, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aic(__gm__ pa_scheduler::SchedulerState *state) { + // 32 个物理 block 的 AIC 直接使用 block_idx，形成连续 worker 0..31。 + const uint32_t worker_id = static_cast(get_block_idx()); + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aic); +} +#elif defined(PA_BUILD_AIV) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aiv, 1, 2); + +extern "C" __global__ __aicore__ void pa_scheduler_0_mix_aiv(__gm__ pa_scheduler::SchedulerState *state) { + // 每个 block 的两个 vector sub-block 展平为 vector_id=2*b+subblock，偏移 32 后形成 worker 32..95。 + const uint32_t vector_id = static_cast(get_block_idx() * get_subblockdim() + get_subblockid()); + const uint32_t worker_id = pa_scheduler::kAicWorkers + vector_id; + pa_scheduler::RunScheduler(state, worker_id, pa_scheduler::CoreRole::Aiv); +} +#else +#error "Compile with PA_BUILD_AIC or PA_BUILD_AIV" +#endif diff --git a/tests/atomic_probe/pa_scheduler/ccec/pa_scheduler_device_exports.map b/tests/atomic_probe/pa_scheduler/ccec/pa_scheduler_device_exports.map new file mode 100644 index 0000000000..b4068c90f1 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/pa_scheduler_device_exports.map @@ -0,0 +1,7 @@ +{ + global: + pa_scheduler_0_mix_aic; + pa_scheduler_0_mix_aiv; + local: + *; +}; diff --git a/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_aicpu.cpp b/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_aicpu.cpp new file mode 100644 index 0000000000..137dc71d71 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_aicpu.cpp @@ -0,0 +1,661 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "pmu_owner_control.h" +#include "pmu_owner_main_abi.h" + +#include +#include + +namespace { + +using pa_scheduler::pmu_owner::PmuOwnerControl; +using pa_scheduler::pmu_owner::PmuOwnerField; +using pa_scheduler::pmu_owner::PmuOwnerMainCommand; +using pa_scheduler::pmu_owner::PmuOwnerMainKernelArgs; +using pa_scheduler::pmu_owner::PmuOwnerStatus; +using pa_scheduler::pmu_owner::PmuSavedRegisters; + +constexpr size_t kCacheLineBytes = 64U; + +// 以下 offset 逐项核对自当前 A5 的 platform_config.h 与 onboard +// inner_platform_regs.cpp。standalone helper 在本目录重写这些常量，绝不 +// include pa_scheduler 目录外的 Simpler platform 实现。 +constexpr uint32_t kCtrl0Offset = 0x4200U; +constexpr uint32_t kCtrl1Offset = 0x2400U; +constexpr uint32_t kCounterOffsets[pa_scheduler::pmu_owner::kPmuCounterCount] = { + 0x4210U, 0x4218U, 0x4220U, 0x4228U, 0x4230U, + 0x4238U, 0x4240U, 0x4248U, 0x4250U, 0x4254U, +}; +constexpr uint32_t kSelectorOffsets[pa_scheduler::pmu_owner::kPmuCounterCount] = { + 0x2500U, 0x2504U, 0x2508U, 0x250cU, 0x2510U, + 0x2514U, 0x2518U, 0x251cU, 0x2520U, 0x2524U, +}; +constexpr uint32_t kTotalLowOffset = 0x4260U; +constexpr uint32_t kTotalHighOffset = 0x4264U; +constexpr uint32_t kStartCycleLowOffset = 0x42a0U; +constexpr uint32_t kStartCycleHighOffset = 0x42a4U; +constexpr uint32_t kStopCycleLowOffset = 0x42a8U; +constexpr uint32_t kStopCycleHighOffset = 0x42acU; +constexpr uint32_t kCtrl0Enabled = 0x7U; +constexpr uint32_t kCtrl1Enabled = 0x1U; + +static_assert( + sizeof(kCounterOffsets) / sizeof(kCounterOffsets[0]) == pa_scheduler::pmu_owner::kPmuCounterCount, + "PMU counter offset count changed" +); +static_assert( + sizeof(kSelectorOffsets) / sizeof(kSelectorOffsets[0]) == pa_scheduler::pmu_owner::kPmuCounterCount, + "PMU selector offset count changed" +); + +inline void InstructionBarrier() +{ + __asm__ volatile("isb" ::: "memory"); +} + +inline void FullSystemBarrier() +{ + __asm__ volatile("dsb sy" ::: "memory"); +} + +// Host 的 H2D/D2H 不会维护 AICPU L1。一条 Configure/Restore 命令开始前 +// 对 control 全区间执行 CIVAC，避免复用同一 GM 地址时读到上一轮 cache 内容。 +void InvalidateControl(const PmuOwnerControl *control) +{ + const uintptr_t begin = reinterpret_cast(control) & ~(kCacheLineBytes - 1U); + const uintptr_t end = + (reinterpret_cast(control) + sizeof(*control) + kCacheLineBytes - 1U) & + ~(kCacheLineBytes - 1U); + for (uintptr_t address = begin; address < end; address += kCacheLineBytes) { + __asm__ volatile("dc civac, %0" : : "r"(address) : "memory"); + } + FullSystemBarrier(); + InstructionBarrier(); +} + +// 命令完成后 clean 整个 control：不仅发布 status，也发布 bitmap、诊断以及 +// Configure 保存区。最后的 DSB/ISB 保证 runtime 同步返回后 host D2H 可见。 +void CleanControl(const PmuOwnerControl *control) +{ + const uintptr_t begin = reinterpret_cast(control) & ~(kCacheLineBytes - 1U); + const uintptr_t end = + (reinterpret_cast(control) + sizeof(*control) + kCacheLineBytes - 1U) & + ~(kCacheLineBytes - 1U); + for (uintptr_t address = begin; address < end; address += kCacheLineBytes) { + __asm__ volatile("dc cvac, %0" : : "r"(address) : "memory"); + } + FullSystemBarrier(); + InstructionBarrier(); +} + +inline volatile uint32_t *MmioPointer(uint64_t base, uint32_t offset) +{ + return reinterpret_cast(static_cast(base + offset)); +} + +inline uint32_t ReadMmio(uint64_t base, uint32_t offset) +{ + return *MmioPointer(base, offset); +} + +inline void WriteMmio(uint64_t base, uint32_t offset, uint32_t value) +{ + *MmioPointer(base, offset) = value; +} + +constexpr PmuOwnerField SelectorField(uint32_t counter) +{ + return static_cast( + static_cast(PmuOwnerField::Selector0) + counter + ); +} + +void ResetDiagnostics(PmuOwnerControl *control) +{ + control->first_failed_index = pa_scheduler::pmu_owner::kDiagnosticIndexUnset; + control->first_failed_field = static_cast(PmuOwnerField::None); + control->first_failed_observed = 0U; + control->first_failed_expected = 0U; + control->restore_failures = 0U; + control->first_restore_failed_index = pa_scheduler::pmu_owner::kDiagnosticIndexUnset; + control->first_restore_failed_field = static_cast(PmuOwnerField::None); + control->first_restore_failed_observed = 0U; + control->first_restore_failed_expected = 0U; +} + +void RecordFirstFailure( + PmuOwnerControl *control, uint32_t index, PmuOwnerField field, + uint32_t observed, uint32_t expected +) +{ + if (control->first_failed_index != pa_scheduler::pmu_owner::kDiagnosticIndexUnset) return; + control->first_failed_index = index; + control->first_failed_field = static_cast(field); + control->first_failed_observed = observed; + control->first_failed_expected = expected; +} + +void RecordRestoreFailure( + PmuOwnerControl *control, uint32_t index, PmuOwnerField field, + uint32_t observed, uint32_t expected +) +{ + ++control->restore_failures; + if (control->first_restore_failed_index != pa_scheduler::pmu_owner::kDiagnosticIndexUnset) return; + control->first_restore_failed_index = index; + control->first_restore_failed_field = static_cast(field); + control->first_restore_failed_observed = observed; + control->first_restore_failed_expected = expected; +} + +bool CheckRegister( + uint64_t base, uint32_t offset, uint32_t expected, PmuOwnerField field, + PmuOwnerField *failed_field, uint32_t *observed +) +{ + const uint32_t actual = ReadMmio(base, offset); + if (actual == expected) return true; + *failed_field = field; + *observed = actual; + return false; +} + +void SaveOne(PmuOwnerControl *control, uint64_t base, uint32_t index) +{ + PmuSavedRegisters &saved = control->saved[index]; + saved.ctrl0 = ReadMmio(base, kCtrl0Offset); + saved.ctrl1 = ReadMmio(base, kCtrl1Offset); + for (uint32_t counter = 0U; counter < pa_scheduler::pmu_owner::kPmuCounterCount; ++counter) { + saved.selectors[counter] = ReadMmio(base, kSelectorOffsets[counter]); + } + saved.start_cycle_low = ReadMmio(base, kStartCycleLowOffset); + saved.start_cycle_high = ReadMmio(base, kStartCycleHighOffset); + saved.stop_cycle_low = ReadMmio(base, kStopCycleLowOffset); + saved.stop_cycle_high = ReadMmio(base, kStopCycleHighOffset); +} + +void ConfigureOne(uint64_t base) +{ + // 先冻结 PMU，再写完整 10 槽 selector。submit-pmu 的 CNT8/CNT5 是 + // request/miss shadow；包括 unused CNT9 在内的所有槽都会在 Restore 恢复。 + WriteMmio(base, kCtrl0Offset, 0U); + WriteMmio(base, kCtrl1Offset, 0U); + for (uint32_t counter = 0U; counter < pa_scheduler::pmu_owner::kPmuCounterCount; ++counter) { + WriteMmio(base, kSelectorOffsets[counter], pa_scheduler::pmu_owner::kConfiguredSelectors[counter]); + } + + // A5 counter 是 read-to-clear。旧计数无法保存，所以先清十个事件 counter + // 和 64-bit total，之后整个 configure→restore 区间由本 owner 独占。 + for (uint32_t counter = 0U; counter < pa_scheduler::pmu_owner::kPmuCounterCount; ++counter) { + (void)ReadMmio(base, kCounterOffsets[counter]); + } + (void)ReadMmio(base, kTotalLowOffset); + (void)ReadMmio(base, kTotalHighOffset); + + WriteMmio(base, kStartCycleLowOffset, 0U); + WriteMmio(base, kStartCycleHighOffset, 0U); + WriteMmio(base, kStopCycleLowOffset, 0xffffffffU); + WriteMmio(base, kStopCycleHighOffset, 0xffffffffU); + WriteMmio(base, kCtrl0Offset, kCtrl0Enabled); + WriteMmio(base, kCtrl1Offset, kCtrl1Enabled); + // Device-nGnRnE 会维持同一区域顺序；额外 DSB 只位于 owner 冷路径，用于 + // 保证下面的逐寄存器读回发生在所有配置写真正完成之后。 + FullSystemBarrier(); +} + +bool ConfigurationMatches( + uint64_t base, PmuOwnerField *failed_field, uint32_t *observed, uint32_t *expected +) +{ + if (!CheckRegister(base, kCtrl0Offset, kCtrl0Enabled, PmuOwnerField::Ctrl0, failed_field, observed)) { + *expected = kCtrl0Enabled; + return false; + } + if (!CheckRegister(base, kCtrl1Offset, kCtrl1Enabled, PmuOwnerField::Ctrl1, failed_field, observed)) { + *expected = kCtrl1Enabled; + return false; + } + for (uint32_t counter = 0U; counter < pa_scheduler::pmu_owner::kPmuCounterCount; ++counter) { + const uint32_t selector = pa_scheduler::pmu_owner::kConfiguredSelectors[counter]; + if (!CheckRegister( + base, kSelectorOffsets[counter], selector, SelectorField(counter), failed_field, observed + )) { + *expected = selector; + return false; + } + } + if (!CheckRegister(base, kStartCycleLowOffset, 0U, PmuOwnerField::StartCycleLow, failed_field, observed)) { + *expected = 0U; + return false; + } + if (!CheckRegister(base, kStartCycleHighOffset, 0U, PmuOwnerField::StartCycleHigh, failed_field, observed)) { + *expected = 0U; + return false; + } + if (!CheckRegister( + base, kStopCycleLowOffset, 0xffffffffU, PmuOwnerField::StopCycleLow, failed_field, observed + )) { + *expected = 0xffffffffU; + return false; + } + if (!CheckRegister( + base, kStopCycleHighOffset, 0xffffffffU, PmuOwnerField::StopCycleHigh, failed_field, observed + )) { + *expected = 0xffffffffU; + return false; + } + return true; +} + +void WriteSavedConfiguration(const PmuOwnerControl &control, uint64_t base, uint32_t index) +{ + const PmuSavedRegisters &saved = control.saved[index]; + WriteMmio(base, kCtrl0Offset, 0U); + WriteMmio(base, kCtrl1Offset, 0U); + for (uint32_t counter = 0U; counter < pa_scheduler::pmu_owner::kPmuCounterCount; ++counter) { + WriteMmio(base, kSelectorOffsets[counter], saved.selectors[counter]); + } + WriteMmio(base, kStartCycleLowOffset, saved.start_cycle_low); + WriteMmio(base, kStartCycleHighOffset, saved.start_cycle_high); + WriteMmio(base, kStopCycleLowOffset, saved.stop_cycle_low); + WriteMmio(base, kStopCycleHighOffset, saved.stop_cycle_high); + // CTRL 最后恢复，避免在 selector/range 尚未回到原值时短暂恢复旧计数状态。 + WriteMmio(base, kCtrl0Offset, saved.ctrl0); + WriteMmio(base, kCtrl1Offset, saved.ctrl1); + FullSystemBarrier(); +} + +bool SavedConfigurationMatches( + const PmuOwnerControl &control, uint64_t base, uint32_t index, + PmuOwnerField *failed_field, uint32_t *observed, uint32_t *expected +) +{ + const PmuSavedRegisters &saved = control.saved[index]; + if (!CheckRegister(base, kCtrl0Offset, saved.ctrl0, PmuOwnerField::Ctrl0, failed_field, observed)) { + *expected = saved.ctrl0; + return false; + } + if (!CheckRegister(base, kCtrl1Offset, saved.ctrl1, PmuOwnerField::Ctrl1, failed_field, observed)) { + *expected = saved.ctrl1; + return false; + } + for (uint32_t counter = 0U; counter < pa_scheduler::pmu_owner::kPmuCounterCount; ++counter) { + if (!CheckRegister( + base, kSelectorOffsets[counter], saved.selectors[counter], SelectorField(counter), + failed_field, observed + )) { + *expected = saved.selectors[counter]; + return false; + } + } + if (!CheckRegister( + base, kStartCycleLowOffset, saved.start_cycle_low, PmuOwnerField::StartCycleLow, + failed_field, observed + )) { + *expected = saved.start_cycle_low; + return false; + } + if (!CheckRegister( + base, kStartCycleHighOffset, saved.start_cycle_high, PmuOwnerField::StartCycleHigh, + failed_field, observed + )) { + *expected = saved.start_cycle_high; + return false; + } + if (!CheckRegister( + base, kStopCycleLowOffset, saved.stop_cycle_low, PmuOwnerField::StopCycleLow, + failed_field, observed + )) { + *expected = saved.stop_cycle_low; + return false; + } + if (!CheckRegister( + base, kStopCycleHighOffset, saved.stop_cycle_high, PmuOwnerField::StopCycleHigh, + failed_field, observed + )) { + *expected = saved.stop_cycle_high; + return false; + } + return true; +} + +bool RestoreOne( + PmuOwnerControl *control, uint32_t index, + PmuOwnerField *failed_field, uint32_t *observed, uint32_t *expected +) +{ + const uint64_t base = control->register_bases[index]; + if (base == 0U) { + *failed_field = PmuOwnerField::RegisterBase; + *observed = 0U; + *expected = 1U; + return false; + } + WriteSavedConfiguration(*control, base, index); + return SavedConfigurationMatches(*control, base, index, failed_field, observed, expected); +} + +void IncrementRoleCount(uint32_t index, uint32_t *aic, uint32_t *aiv) +{ + if (pa_scheduler::pmu_owner::IsAicPhysicalSlot(index)) { + ++(*aic); + } else { + ++(*aiv); + } +} + +void DecrementActiveRole(PmuOwnerControl *control, uint32_t index) +{ + if (control->active_total != 0U) --control->active_total; + if (pa_scheduler::pmu_owner::IsAicPhysicalSlot(index)) { + if (control->active_aic != 0U) --control->active_aic; + } else if (control->active_aiv != 0U) { + --control->active_aiv; + } +} + +// 恢复除 skip_index 外的 owned 槽。Configure 的“当前失败槽”若当场恢复失败， +// rollback 会跳过它，确保该 bit 留给 host 随后的幂等 Restore 再次重试。 +bool RestoreOwnedBitmapExcept(PmuOwnerControl *control, uint32_t skip_index) +{ + bool all_restored = true; + for (uint32_t next = pa_scheduler::pmu_owner::kPhysicalSubcoreCount; next != 0U; --next) { + const uint32_t index = next - 1U; + if (!pa_scheduler::pmu_owner::IsConfigured(*control, index)) continue; + if (index == skip_index) continue; + + PmuOwnerField failed_field = PmuOwnerField::None; + uint32_t observed = 0U; + uint32_t expected = 0U; + if (RestoreOne(control, index, &failed_field, &observed, &expected)) { + pa_scheduler::pmu_owner::ClearConfigured(control, index); + DecrementActiveRole(control, index); + } else { + RecordRestoreFailure(control, index, failed_field, observed, expected); + all_restored = false; + } + } + return all_restored; +} + +// 只消费 owner bitmap，严格按 107→0 恢复。某槽只有在完整读回一致后才清 bit； +// 因而 Restore 失败后可再次调用，下一次只重试仍由 owner 持有的槽。 +bool RestoreConfiguredBitmap(PmuOwnerControl *control) +{ + const bool all_restored = RestoreOwnedBitmapExcept( + control, pa_scheduler::pmu_owner::kDiagnosticIndexUnset + ); + const uint32_t bitmap_count = pa_scheduler::pmu_owner::CountConfigured(*control); + return all_restored && bitmap_count == 0U && control->active_total == 0U && + control->active_aic == 0U && control->active_aiv == 0U; +} + +bool ValidateControlHeader(PmuOwnerControl *control) +{ + if (control->magic != pa_scheduler::pmu_owner::kPmuOwnerControlMagic) { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::ControlMagic, + control->magic, pa_scheduler::pmu_owner::kPmuOwnerControlMagic + ); + return false; + } + if (control->version != pa_scheduler::pmu_owner::kPmuOwnerControlVersion) { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::ControlVersion, + control->version, pa_scheduler::pmu_owner::kPmuOwnerControlVersion + ); + return false; + } + if (control->struct_bytes != sizeof(PmuOwnerControl)) { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::ControlSize, + control->struct_bytes, static_cast(sizeof(PmuOwnerControl)) + ); + return false; + } + return true; +} + +bool ValidateExpectedTopology(PmuOwnerControl *control) +{ + if (control->expected_total != pa_scheduler::pmu_owner::kExpectedSubcoreCount) { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::TotalCount, + control->expected_total, pa_scheduler::pmu_owner::kExpectedSubcoreCount + ); + return false; + } + if (control->expected_aic != pa_scheduler::pmu_owner::kExpectedAicCount) { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::AicCount, + control->expected_aic, pa_scheduler::pmu_owner::kExpectedAicCount + ); + return false; + } + if (control->expected_aiv != pa_scheduler::pmu_owner::kExpectedAivCount) { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::AivCount, + control->expected_aiv, pa_scheduler::pmu_owner::kExpectedAivCount + ); + return false; + } + return true; +} + +PmuOwnerStatus Configure(PmuOwnerControl *control) +{ + if (control->configured != 0U || control->active_total != 0U || + pa_scheduler::pmu_owner::CountConfigured(*control) != 0U) { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::State, + control->active_total, 0U + ); + return PmuOwnerStatus::AlreadyConfigured; + } + + control->active_total = 0U; + control->active_aic = 0U; + control->active_aiv = 0U; + control->discovered_total = 0U; + control->discovered_aic = 0U; + control->discovered_aiv = 0U; + control->skipped_total = 0U; + for (uint32_t word = 0U; word < pa_scheduler::pmu_owner::kConfiguredBitmapWords; ++word) { + control->configured_bitmap[word] = 0U; + } + ResetDiagnostics(control); + + for (uint32_t index = 0U; index < pa_scheduler::pmu_owner::kPhysicalSubcoreCount; ++index) { + const uint64_t base = control->register_bases[index]; + if (base == 0U) { + RecordFirstFailure(control, index, PmuOwnerField::RegisterBase, 0U, 1U); + ++control->skipped_total; + continue; + } + + SaveOne(control, base, index); + // 从保存完成到首次 MMIO 改写之间先取得所有权。即使后面的配置读回 + // 失败且当场恢复也失败，该槽仍留在 bitmap 中供后续 Restore 重试。 + pa_scheduler::pmu_owner::SetConfigured(control, index); + ++control->active_total; + IncrementRoleCount(index, &control->active_aic, &control->active_aiv); + ConfigureOne(base); + PmuOwnerField failed_field = PmuOwnerField::None; + uint32_t observed = 0U; + uint32_t expected = 0U; + if (!ConfigurationMatches(base, &failed_field, &observed, &expected)) { + RecordFirstFailure(control, index, failed_field, observed, expected); + ++control->skipped_total; + + // 配置未通过时必须当场恢复；只有原值完整读回一致才能释放该槽 + // 的所有权。若恢复失败，保留它的 bit 并回滚其余 owned 槽。 + PmuOwnerField restore_field = PmuOwnerField::None; + uint32_t restore_observed = 0U; + uint32_t restore_expected = 0U; + if (!RestoreOne(control, index, &restore_field, &restore_observed, &restore_expected)) { + RecordRestoreFailure(control, index, restore_field, restore_observed, restore_expected); + const bool rollback_ok = RestoreOwnedBitmapExcept(control, index); + control->configured = control->active_total == 0U ? 0U : 1U; + return rollback_ok ? PmuOwnerStatus::ConfigureSlotRestoreFailed : + PmuOwnerStatus::ConfigureRollbackFailed; + } + pa_scheduler::pmu_owner::ClearConfigured(control, index); + DecrementActiveRole(control, index); + continue; + } + + // discovered 只统计配置值全部读回一致的物理槽；bitmap/active 则表达 + // 更严格的“仍持有原值快照、尚未恢复”所有权，两者不可混用。 + ++control->discovered_total; + IncrementRoleCount(index, &control->discovered_aic, &control->discovered_aiv); + } + + const uint32_t bitmap_count = pa_scheduler::pmu_owner::CountConfigured(*control); + const bool topology_matches = + bitmap_count == pa_scheduler::pmu_owner::kExpectedSubcoreCount && + control->active_total == pa_scheduler::pmu_owner::kExpectedSubcoreCount && + control->active_aic == pa_scheduler::pmu_owner::kExpectedAicCount && + control->active_aiv == pa_scheduler::pmu_owner::kExpectedAivCount && + control->discovered_total == pa_scheduler::pmu_owner::kExpectedSubcoreCount && + control->discovered_aic == pa_scheduler::pmu_owner::kExpectedAicCount && + control->discovered_aiv == pa_scheduler::pmu_owner::kExpectedAivCount; + if (!topology_matches) { + if (bitmap_count != control->active_total) { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::BitmapCount, + bitmap_count, control->active_total + ); + } else if (control->active_total != pa_scheduler::pmu_owner::kExpectedSubcoreCount) { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::TotalCount, + control->active_total, pa_scheduler::pmu_owner::kExpectedSubcoreCount + ); + } else if (control->active_aic != pa_scheduler::pmu_owner::kExpectedAicCount) { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::AicCount, + control->active_aic, pa_scheduler::pmu_owner::kExpectedAicCount + ); + } else if (control->active_aiv != pa_scheduler::pmu_owner::kExpectedAivCount) { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::AivCount, + control->active_aiv, pa_scheduler::pmu_owner::kExpectedAivCount + ); + } else if (control->discovered_total != pa_scheduler::pmu_owner::kExpectedSubcoreCount) { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::TotalCount, + control->discovered_total, pa_scheduler::pmu_owner::kExpectedSubcoreCount + ); + } else if (control->discovered_aic != pa_scheduler::pmu_owner::kExpectedAicCount) { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::AicCount, + control->discovered_aic, pa_scheduler::pmu_owner::kExpectedAicCount + ); + } else { + RecordFirstFailure( + control, pa_scheduler::pmu_owner::kPhysicalSubcoreCount, PmuOwnerField::AivCount, + control->discovered_aiv, pa_scheduler::pmu_owner::kExpectedAivCount + ); + } + const bool rollback_ok = RestoreConfiguredBitmap(control); + control->configured = control->active_total == 0U ? 0U : 1U; + return rollback_ok ? PmuOwnerStatus::ConfigureCountMismatch : + PmuOwnerStatus::ConfigureRollbackFailed; + } + + control->configured = 1U; + return PmuOwnerStatus::Success; +} + +PmuOwnerStatus Restore(PmuOwnerControl *control) +{ + // 幂等恢复：bitmap 已空时，无论是一次正常 Restore 后的重入，还是 host + // 失败路径的兜底调用，都统一收口为“未持有任何 PMU 槽”。 + const uint32_t bitmap_count = pa_scheduler::pmu_owner::CountConfigured(*control); + if (bitmap_count == 0U) { + control->configured = 0U; + control->active_total = 0U; + control->active_aic = 0U; + control->active_aiv = 0U; + ResetDiagnostics(control); + return PmuOwnerStatus::Success; + } + + // 若上一次命令在更新计数字段后异常退出，bitmap 才是唯一权威所有权源。 + // Restore 先由 bitmap 重建 active 计数，再逆序恢复，避免陈旧计数阻塞清理。 + control->active_total = bitmap_count; + control->active_aic = 0U; + control->active_aiv = 0U; + for (uint32_t index = 0U; index < pa_scheduler::pmu_owner::kPhysicalSubcoreCount; ++index) { + if (pa_scheduler::pmu_owner::IsConfigured(*control, index)) { + IncrementRoleCount(index, &control->active_aic, &control->active_aiv); + } + } + ResetDiagnostics(control); + const bool restored = RestoreConfiguredBitmap(control); + control->configured = restored ? 0U : 1U; + return restored ? PmuOwnerStatus::Success : PmuOwnerStatus::RestoreFailed; +} + +// 主 aicpu_scheduler 会复制完整 152B KernelArgs；其中 runtime_args_device +// 指向跨 Configure/Restore 持续存在的 GM control。参数副本本身不需要 cache 维护。 +PmuOwnerControl *ResolveControl(const PmuOwnerMainKernelArgs *arguments) +{ + if (arguments == nullptr || arguments->runtime_args_device == 0U || + (arguments->runtime_args_device % alignof(PmuOwnerControl)) != 0U) { + return nullptr; + } + return reinterpret_cast( + static_cast(arguments->runtime_args_device) + ); +} + +void ExecuteOwnerCommand(void *argument) +{ + const auto *arguments = reinterpret_cast(argument); + PmuOwnerControl *control = ResolveControl(arguments); + if (control == nullptr) return; + + InvalidateControl(control); + // 从这一行起，即使协议校验失败也把精确业务状态 clean 回 GM。 + control->status = pa_scheduler::pmu_owner::kStatusPending; + ResetDiagnostics(control); + if (!ValidateControlHeader(control)) { + control->status = static_cast(PmuOwnerStatus::InvalidControl); + CleanControl(control); + return; + } + const auto command = static_cast(arguments->command); + if (command == PmuOwnerMainCommand::Configure && !ValidateExpectedTopology(control)) { + control->status = static_cast(PmuOwnerStatus::UnexpectedTopology); + CleanControl(control); + return; + } + + PmuOwnerStatus status = PmuOwnerStatus::InvalidArguments; + if (command == PmuOwnerMainCommand::Configure) { + status = Configure(control); + } else if (command == PmuOwnerMainCommand::Restore) { + // Restore 以 bitmap 为唯一所有权依据。即使 expected_* 诊断字段被局部 + // 覆盖，也优先尝试恢复已经保存的寄存器，避免清理被无关字段阻塞。 + status = Restore(control); + } + control->status = static_cast(status); + CleanControl(control); +} + +} // namespace + +extern "C" __attribute__((visibility("default"))) int simpler_aicpu_exec(void *argument) +{ + ExecuteOwnerCommand(argument); + return 0; +} diff --git a/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_control.h b/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_control.h new file mode 100644 index 0000000000..eb17c4c1b0 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_control.h @@ -0,0 +1,225 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_CONTROL_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_CONTROL_H_ + +#include +#include + +namespace pa_scheduler::pmu_owner { + +// 这份头文件同时供 x86 host 与 AArch64 AICPU helper 使用。所有跨端字段都采用 +// 固定宽度整数；禁止在 ABI 中放 host 指针、bool、STL 容器或编译器相关位域。 +constexpr uint32_t kPmuOwnerControlMagic = 0x504d554fU; // "PMUO" +constexpr uint32_t kPmuOwnerControlVersion = 1U; + +// DAV_3510 一共有 2 die；每个 die 依次放 18 个 AIC 和 36 个 AIV 物理槽。 +// 当前 A5 stream 实际开放 32 个 AIC 与 64 个 AIV，其余 12 个槽的 MMIO +// 读回不会匹配配置值，因此 owner 必须扫描 108 槽，最终取得 96 个可用槽。 +constexpr uint32_t kPhysicalSubcoreCount = 108U; +constexpr uint32_t kExpectedSubcoreCount = 96U; +constexpr uint32_t kExpectedAicCount = 32U; +constexpr uint32_t kExpectedAivCount = 64U; +constexpr uint32_t kAicPerDie = 18U; +constexpr uint32_t kSubcoresPerDie = 54U; +constexpr uint32_t kConfiguredBitmapWords = 4U; +constexpr uint32_t kDiagnosticIndexUnset = 0xffffffffU; + +static_assert(kExpectedAicCount + kExpectedAivCount == kExpectedSubcoreCount, "active topology count mismatch"); +static_assert(kAicPerDie * 2U == 36U, "physical AIC topology changed"); +static_assert(kSubcoresPerDie * 2U == kPhysicalSubcoreCount, "physical subcore topology changed"); + +// A5 PIPE_UTILIZATION 的正式 counter 槽位布局。submit-pmu 用 CNT8/CNT5 +// 重复配置 I-cache request/miss，作为允许中途 read-to-clear 的 shadow； +// CNT6/7 始终不在阶段边界读取，保留为完整 Submit 的权威对照。 +// +// 不能把 miss 放进 CNT9：A5 b1 实测表明 CNT9 selector 虽能回读 0x35， +// 但计数恒为 0；正式 PIPE_UTIL 表也把 CNT9 标成 unused。0x35 在独立 +// I-cache 微基准的低位 counter 已验证可计数，因此诊断构建让 CNT5 承担 +// shadow miss，并明确放弃该构建中的 MTE3 busy。 +constexpr uint32_t kPmuCounterCount = 10U; +constexpr uint32_t kConfiguredSelectors[kPmuCounterCount] = { + 0x501U, // CNT0: vector busy + 0x301U, // CNT1: cube busy + 0x001U, // CNT2: scalar busy + 0x701U, // CNT3: MTE1 busy + 0x202U, // CNT4: MTE2 busy +#if PA_BUILD_SUBMIT_PMU + 0x035U, // CNT5: shadow I-cache miss + 0x034U, // CNT6: I-cache request（完整 Submit） + 0x035U, // CNT7: I-cache miss（完整 Submit） + 0x034U, // CNT8: shadow I-cache request + 0x000U, // CNT9: A5 PIPE_UTIL 正式未使用 +#else + 0x203U, // CNT5: MTE3 busy + 0x034U, // CNT6: I-cache request + 0x035U, // CNT7: I-cache miss + 0x714U, // CNT8: fix-pipe busy + 0x000U, // CNT9: 未使用 +#endif +}; + +constexpr int32_t kStatusPending = 0x7fffffff; + +// AICPU entry 始终向 runtime 返回 0；协议结果只通过 control.status 回传， +// 从而避免一次可诊断的配置失败被 runtime 升格成整条 stream 异常。 +enum class PmuOwnerStatus : int32_t { + Success = 0, + InvalidArguments = -1, + InvalidControl = -2, + UnexpectedTopology = -3, + AlreadyConfigured = -4, + ConfigureCountMismatch = -5, + ConfigureRollbackFailed = -6, + ConfigureSlotRestoreFailed = -7, + RestoreFailed = -8, +}; + +// 首个异常寄存器使用稳定的枚举编号，host 不需要解析 AICPU 日志即可定位 +// 是基址、selector、计数范围还是 enable 控制读回不一致。 +enum class PmuOwnerField : uint32_t { + None = 0, + Arguments, + ControlMagic, + ControlVersion, + ControlSize, + State, + RegisterBase, + Ctrl0, + Ctrl1, + Selector0, + Selector1, + Selector2, + Selector3, + Selector4, + Selector5, + Selector6, + Selector7, + Selector8, + Selector9, + StartCycleLow, + StartCycleHigh, + StopCycleLow, + StopCycleHigh, + BitmapCount, + TotalCount, + AicCount, + AivCount, +}; + +// 单个物理子核被 owner 改动的完整可恢复状态恰好占一条 cache line。 +// PMU counter 是 read-to-clear，旧 counter 值无法恢复；owner 会话必须独占。 +struct alignas(64) PmuSavedRegisters { + uint32_t ctrl0; + uint32_t ctrl1; + uint32_t selectors[kPmuCounterCount]; + uint32_t start_cycle_low; + uint32_t start_cycle_high; + uint32_t stop_cycle_low; + uint32_t stop_cycle_high; +}; + +// Host 与 AICPU 共享的 owner 状态。前 128B 是命令结果和诊断，随后内嵌 +// 108 个 MMIO 基址、4-word 所有权 bitmap，以及每槽 64B 的 Configure 快照。 +// bitmap 的严格语义是“原值已保存、且 owner 可能已经改写 MMIO、但尚未 +// 完整恢复”的槽；它在 Configure 写第一项 MMIO 前置位，仅在恢复读回完整 +// 一致后清位。Restore 期间不得清零或重建 saved[]，只能按 bitmap 逆序消费。 +struct alignas(64) PmuOwnerControl { + uint32_t magic; + uint32_t version; + uint32_t struct_bytes; + volatile int32_t status; + + uint32_t configured; + uint32_t expected_total; + uint32_t expected_aic; + uint32_t expected_aiv; + + // active_* 与 bitmap 表示仍由本 owner 持有、尚未恢复的物理槽。 + uint32_t active_total; + uint32_t active_aic; + uint32_t active_aiv; + // discovered_* 保留本次 Configure 扫描结果；即使计数不匹配后回滚， + // host 仍能看到回滚前究竟探测到了多少 AIC/AIV。 + uint32_t discovered_total; + uint32_t discovered_aic; + uint32_t discovered_aiv; + uint32_t skipped_total; + + uint32_t first_failed_index; + uint32_t first_failed_field; + uint32_t first_failed_observed; + uint32_t first_failed_expected; + + uint32_t restore_failures; + uint32_t first_restore_failed_index; + uint32_t first_restore_failed_field; + uint32_t first_restore_failed_observed; + uint32_t first_restore_failed_expected; + uint32_t reserved_header[8]; + + uint64_t register_bases[kPhysicalSubcoreCount]; + // 字段名保留 configured_bitmap 以稳定 host/device ABI；失败路径中它还会 + // 临时包含“配置未通过但恢复仍待重试”的 owned 槽。 + uint32_t configured_bitmap[kConfiguredBitmapWords]; + // 让 saved[] 从新的 64B cache line 开始；该 padding 不承载协议含义。 + uint32_t reserved_bitmap[4]; + PmuSavedRegisters saved[kPhysicalSubcoreCount]; +}; + +static_assert(sizeof(PmuSavedRegisters) == 64U, "one saved PMU slot must occupy one cache line"); +static_assert(alignof(PmuSavedRegisters) == 64U, "saved PMU slot alignment changed"); +static_assert(offsetof(PmuOwnerControl, status) == 12U, "PMU owner status offset changed"); +static_assert(offsetof(PmuOwnerControl, register_bases) == 128U, "PMU owner header must occupy two cache lines"); +static_assert(offsetof(PmuOwnerControl, configured_bitmap) == 992U, "PMU owner bitmap offset changed"); +static_assert(offsetof(PmuOwnerControl, saved) == 1024U, "PMU owner saved area must be cache-line aligned"); +static_assert(sizeof(PmuOwnerControl) == 7936U, "PMU owner control ABI changed"); +static_assert(sizeof(PmuOwnerControl) % 64U == 0U, "PMU owner control must use complete cache lines"); +static_assert(alignof(PmuOwnerControl) == 64U, "PMU owner control alignment changed"); + +inline bool IsAicPhysicalSlot(uint32_t index) +{ + return index < kPhysicalSubcoreCount && (index % kSubcoresPerDie) < kAicPerDie; +} + +inline bool IsConfigured(const PmuOwnerControl &control, uint32_t index) +{ + return index < kPhysicalSubcoreCount && + (control.configured_bitmap[index / 32U] & (1U << (index % 32U))) != 0U; +} + +inline void SetConfigured(PmuOwnerControl *control, uint32_t index) +{ + if (index < kPhysicalSubcoreCount) { + control->configured_bitmap[index / 32U] |= 1U << (index % 32U); + } +} + +inline void ClearConfigured(PmuOwnerControl *control, uint32_t index) +{ + if (index < kPhysicalSubcoreCount) { + control->configured_bitmap[index / 32U] &= ~(1U << (index % 32U)); + } +} + +inline uint32_t CountConfigured(const PmuOwnerControl &control) +{ + uint32_t count = 0U; + for (uint32_t index = 0U; index < kPhysicalSubcoreCount; ++index) { + count += IsConfigured(control, index) ? 1U : 0U; + } + return count; +} + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_CONTROL_H_ diff --git a/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_dispatcher.cpp b/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_dispatcher.cpp new file mode 100644 index 0000000000..563113ead8 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_dispatcher.cpp @@ -0,0 +1,186 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +// 这个 SO 只在初始化阶段由 libaicpu_extend_kernels.so 临时加载。它在主 +// aicpu_scheduler 有权限访问的预安装目录中落盘真正的 PMU owner SO；随后 +// host 通过 mode=0 JSON 注册直接调用 owner，不会在每次命令中再经过本文件。 + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" void DlogRecord(int module_id, int level, const char *format, ...); + +namespace { + +constexpr int kDlogModuleCcecpu = 3; +constexpr int kDlogLevelError = 3; +constexpr uint64_t kFnvOffsetBasis = UINT64_C(14695981039346656037); +constexpr uint64_t kFnvPrime = UINT64_C(1099511628211); + +void Log(const char *format, ...) +{ + char buffer[1024] = {}; + va_list arguments; + va_start(arguments, format); + (void)vsnprintf(buffer, sizeof(buffer), format, arguments); + va_end(arguments); + DlogRecord(kDlogModuleCcecpu, kDlogLevelError, "[pa-pmu-dispatcher] %s", buffer); +} + +// libaicpu_extend_kernels 固定从 KernelArgs::device_args（offset 40）获取 +// DeviceArgs。后五个 qword 是本地 bootstrap 协议，offset 必须保持不变。 +struct BootstrapKernelArgs { + uint64_t unused[5]; + void *device_args; + void *runtime_args; + uint64_t regs; +}; + +struct BootstrapDeviceArgs { + uint64_t unused[12]; + uint64_t dispatcher_so_device; // offset 96，extend kernel 消费 + uint64_t dispatcher_so_bytes; // offset 104 + uint64_t device_id; // offset 112 + uint64_t owner_so_device; // offset 120，本 dispatcher 消费 + uint64_t owner_so_bytes; // offset 128 +}; + +static_assert(offsetof(BootstrapKernelArgs, device_args) == 40U, "bootstrap KernelArgs ABI changed"); +static_assert(offsetof(BootstrapDeviceArgs, dispatcher_so_device) == 96U, "dispatcher address offset changed"); +static_assert(offsetof(BootstrapDeviceArgs, dispatcher_so_bytes) == 104U, "dispatcher size offset changed"); +static_assert(offsetof(BootstrapDeviceArgs, device_id) == 112U, "device id offset changed"); +static_assert(offsetof(BootstrapDeviceArgs, owner_so_device) == 120U, "owner address offset changed"); +static_assert(offsetof(BootstrapDeviceArgs, owner_so_bytes) == 128U, "owner size offset changed"); + +// host 与 device 都对完整 SO 字节做 FNV-1a；不只散列 ELF header，避免同一 +// toolchain 产出的等长 SO 发生名字碰撞并误加载旧代码。 +uint64_t FingerprintBytes(const void *data, uint64_t bytes) +{ + const auto *input = static_cast(data); + uint64_t hash = kFnvOffsetBasis; + for (uint64_t index = 0U; index < bytes; ++index) { + hash ^= input[index]; + hash *= kFnvPrime; + } + return hash; +} + +std::string OwnerSoPath(uint64_t fingerprint, uint64_t device_id) +{ + char path[256] = {}; + (void)snprintf( + path, sizeof(path), + "/usr/lib64/aicpu_kernels/0/aicpu_kernels_device/pa_scheduler_pmu_owner_%016llx_d%llu.so", + static_cast(fingerprint), static_cast(device_id) + ); + return path; +} + +// 先写同目录临时文件，再原子 rename。目标名由内容、device 共同确定；临时 +// 名再加入进程和源地址，避免同进程并发 bootstrap 写同一个临时 inode。 +bool WriteOwnerSo(const std::string &target, const void *data, uint64_t bytes) +{ + if (data == nullptr || bytes == 0U || + bytes > static_cast(std::numeric_limits::max())) { + Log("invalid owner SO buffer: data=%p bytes=%llu", data, static_cast(bytes)); + return false; + } + + char temporary[384] = {}; + (void)snprintf( + temporary, sizeof(temporary), "%s.tmp.%d.%016llx", target.c_str(), static_cast(getpid()), + static_cast(reinterpret_cast(data)) + ); + { + std::ofstream output(temporary, std::ios::binary | std::ios::trunc); + if (!output.is_open()) { + Log("open %s failed: %s", temporary, strerror(errno)); + return false; + } + output.write(static_cast(data), static_cast(bytes)); + output.close(); + if (!output) { + Log("write %s failed", temporary); + (void)unlink(temporary); + return false; + } + } + if (chmod(temporary, 0755) != 0) { + Log("chmod %s failed: %s", temporary, strerror(errno)); + (void)unlink(temporary); + return false; + } + if (rename(temporary, target.c_str()) != 0) { + Log("rename %s -> %s failed: %s", temporary, target.c_str(), strerror(errno)); + (void)unlink(temporary); + return false; + } + return true; +} + +} // namespace + +extern "C" { + +// extend kernel 在 dlopen 后要求三个符号同时存在；本 dispatcher 只使用 Init， +// 另外两个入口保持无副作用成功返回，避免未来 runtime 的预探测变成故障。 +__attribute__((visibility("default"))) int StaticTileFwkBackendKernelServer(void *arguments) +{ + (void)arguments; + return 0; +} + +__attribute__((visibility("default"))) uint32_t DynTileFwkBackendKernelServer(void *arguments) +{ + (void)arguments; + return 0U; +} + +__attribute__((visibility("default"))) uint32_t DynTileFwkBackendKernelServerInit(void *arguments) +{ + if (arguments == nullptr) { + Log("Init received null KernelArgs"); + return 1U; + } + auto *kernel_args = static_cast(arguments); + auto *device_args = static_cast(kernel_args->device_args); + if (device_args == nullptr || device_args->owner_so_device == 0U || device_args->owner_so_bytes == 0U) { + Log( + "Init received invalid DeviceArgs: args=%p owner=%016llx bytes=%llu", device_args, + static_cast(device_args == nullptr ? 0U : device_args->owner_so_device), + static_cast(device_args == nullptr ? 0U : device_args->owner_so_bytes) + ); + return 2U; + } + + const void *owner_so = reinterpret_cast(static_cast(device_args->owner_so_device)); + const uint64_t fingerprint = FingerprintBytes(owner_so, device_args->owner_so_bytes); + const std::string target = OwnerSoPath(fingerprint, device_args->device_id); + if (!WriteOwnerSo(target, owner_so, device_args->owner_so_bytes)) return 3U; + + Log( + "installed %s (%llu bytes)", target.c_str(), + static_cast(device_args->owner_so_bytes) + ); + return 0U; +} + +} // extern "C" diff --git a/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_host.h b/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_host.h new file mode 100644 index 0000000000..68dfac042e --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_host.h @@ -0,0 +1,345 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_HOST_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_HOST_H_ + +#include "pmu_owner_control.h" +#include "pmu_owner_main_abi.h" +#include "pmu_owner_main_loader.h" + +#include "acl/acl.h" + +#include +#include +#include +#include +#include +#include + +namespace pa_scheduler::pmu_owner { + +inline bool OwnerCheckAcl(aclError error, const char *label) +{ + if (error == ACL_SUCCESS) return true; + std::fprintf(stderr, "ACL error %d: %s\n", static_cast(error), label); + return false; +} + +inline std::string ArtifactBesideKernel(const std::string &kernel_path, const char *name) +{ + const size_t slash = kernel_path.find_last_of('/'); + return slash == std::string::npos ? std::string(name) : kernel_path.substr(0, slash + 1U) + name; +} + +inline const char *OwnerFieldName(PmuOwnerField field) +{ + switch (field) { + case PmuOwnerField::None: return "none"; + case PmuOwnerField::Arguments: return "arguments"; + case PmuOwnerField::ControlMagic: return "control-magic"; + case PmuOwnerField::ControlVersion: return "control-version"; + case PmuOwnerField::ControlSize: return "control-size"; + case PmuOwnerField::State: return "state"; + case PmuOwnerField::RegisterBase: return "register-base"; + case PmuOwnerField::Ctrl0: return "ctrl0"; + case PmuOwnerField::Ctrl1: return "ctrl1"; + case PmuOwnerField::Selector0: return "selector0"; + case PmuOwnerField::Selector1: return "selector1"; + case PmuOwnerField::Selector2: return "selector2"; + case PmuOwnerField::Selector3: return "selector3"; + case PmuOwnerField::Selector4: return "selector4"; + case PmuOwnerField::Selector5: return "selector5"; + case PmuOwnerField::Selector6: return "selector6"; + case PmuOwnerField::Selector7: return "selector7"; + case PmuOwnerField::Selector8: return "selector8"; + case PmuOwnerField::Selector9: return "selector9"; + case PmuOwnerField::StartCycleLow: return "start-cycle-low"; + case PmuOwnerField::StartCycleHigh: return "start-cycle-high"; + case PmuOwnerField::StopCycleLow: return "stop-cycle-low"; + case PmuOwnerField::StopCycleHigh: return "stop-cycle-high"; + case PmuOwnerField::BitmapCount: return "bitmap-count"; + case PmuOwnerField::TotalCount: return "total-count"; + case PmuOwnerField::AicCount: return "aic-count"; + case PmuOwnerField::AivCount: return "aiv-count"; + } + return "unknown"; +} + +struct ActiveSubcoreLimits { + uint32_t aic = 0U; + uint32_t aiv = 0U; + uint32_t total = 0U; +}; + +inline bool QueryActiveSubcoreLimits(aclrtStream scheduling_stream, ActiveSubcoreLimits *limits) +{ + if (scheduling_stream == nullptr || limits == nullptr) { + std::fprintf(stderr, "Cannot query active PMU subcores with a null stream/result.\n"); + return false; + } + uint32_t aic = 0U; + uint32_t aiv = 0U; + const aclError aic_error = + aclrtGetStreamResLimit(scheduling_stream, ACL_RT_DEV_RES_CUBE_CORE, &aic); + const aclError aiv_error = + aclrtGetStreamResLimit(scheduling_stream, ACL_RT_DEV_RES_VECTOR_CORE, &aiv); + const uint64_t total = static_cast(aic) + aiv; + if (aic_error != ACL_SUCCESS || aiv_error != ACL_SUCCESS || + aic != kExpectedAicCount || aiv != kExpectedAivCount || total != kExpectedSubcoreCount) { + std::fprintf( + stderr, + "Unexpected stream PMU topology: aic_error=%d aiv_error=%d " + "aic=%u/%u aiv=%u/%u total=%llu/%u\n", + static_cast(aic_error), static_cast(aiv_error), aic, kExpectedAicCount, + aiv, kExpectedAivCount, static_cast(total), kExpectedSubcoreCount + ); + return false; + } + limits->aic = aic; + limits->aiv = aiv; + limits->total = static_cast(total); + std::printf( + "[PMU_OWNER] stream_active aic=%u aiv=%u total=%u physical_slots=%u\n", + limits->aic, limits->aiv, limits->total, kPhysicalSubcoreCount + ); + return true; +} + +// mixed launch 的一个物理 block 必须同时拥有 1 个 AIC 和相邻的 2 个 AIV。 +// 只检查 32/64 总数仍可能放过孤立 AIV；这里直接按两 die 的真实编号布局验闭包。 +inline bool ValidateConfiguredTripletTopology(const PmuOwnerControl &control) +{ + uint32_t complete_triplets = 0U; + uint32_t broken_triplets = 0U; + const uint32_t dies = kPhysicalSubcoreCount / kSubcoresPerDie; + for (uint32_t die = 0U; die < dies; ++die) { + const uint32_t die_base = die * kSubcoresPerDie; + for (uint32_t local = 0U; local < kAicPerDie; ++local) { + const bool aic = IsConfigured(control, die_base + local); + const bool aiv0 = IsConfigured(control, die_base + kAicPerDie + local * 2U); + const bool aiv1 = IsConfigured(control, die_base + kAicPerDie + local * 2U + 1U); + if (aic == aiv0 && aic == aiv1) { + complete_triplets += aic ? 1U : 0U; + } else { + ++broken_triplets; + } + } + } + const bool passed = complete_triplets == kExpectedAicCount && broken_triplets == 0U; + std::printf( + "[ASSERT] %-48s %s (complete=%u broken=%u)\n", + "PMU owner bitmap is complete 1-AIC + 2-AIV triplets", + passed ? "PASS" : "FAIL", complete_triplets, broken_triplets + ); + return passed; +} + +// owner 命令使用独立 stream，但通过 mode=0 JSON 在主 aicpu_scheduler 中执行。 +// Configure 同步完成后才允许启动 AICore；AICore 正常或异常退出后，Restore +// 都不会依赖业务 stream。MMIO 映射必须保持到 Finalize 完成之后。 +class PmuOwnerSession { +public: + PmuOwnerSession() = default; + PmuOwnerSession(const PmuOwnerSession &) = delete; + PmuOwnerSession &operator=(const PmuOwnerSession &) = delete; + + ~PmuOwnerSession() + { + if (HasResources()) (void)Finalize(); + } + + bool Initialize( + uint32_t device, aclrtStream scheduling_stream, const std::string &dispatcher_path, + const std::string &owner_path, const std::vector ®ister_bases + ) + { + if (HasResources() || register_bases.size() != kPhysicalSubcoreCount) { + std::fprintf( + stderr, "Invalid PMU owner initialization state or register table size: %zu\n", + register_bases.size() + ); + return false; + } + device_ = device; + if (!QueryActiveSubcoreLimits(scheduling_stream, &limits_)) return false; + if (!OwnerCheckAcl(aclrtCreateStream(&owner_stream_), "aclrtCreateStream(PMU owner)")) return false; + if (loader_.Initialize( + dispatcher_path, owner_path, owner_stream_, static_cast(device_) + ) != 0) { + return false; + } + if (!OwnerCheckAcl( + aclrtMalloc(&control_device_, sizeof(PmuOwnerControl), ACL_MEM_MALLOC_NORMAL_ONLY), + "aclrtMalloc(PMU owner control)" + )) { + return false; + } + if ((reinterpret_cast(control_device_) & (alignof(PmuOwnerControl) - 1U)) != 0U) { + std::fprintf(stderr, "PMU owner control is not 64-byte aligned: %p\n", control_device_); + return false; + } + + control_ = PmuOwnerControl{}; + control_.magic = kPmuOwnerControlMagic; + control_.version = kPmuOwnerControlVersion; + control_.struct_bytes = sizeof(PmuOwnerControl); + control_.status = kStatusPending; + control_.expected_total = limits_.total; + control_.expected_aic = limits_.aic; + control_.expected_aiv = limits_.aiv; + std::memcpy(control_.register_bases, register_bases.data(), sizeof(control_.register_bases)); + if (!OwnerCheckAcl( + aclrtMemcpy( + control_device_, sizeof(control_), &control_, sizeof(control_), ACL_MEMCPY_HOST_TO_DEVICE + ), + "aclrtMemcpy(H2D initial PMU owner control)" + )) { + return false; + } + ready_ = true; + return true; + } + + bool Configure() + { + if (!ready_) return false; + const bool command_ok = RunCommand(PmuOwnerMainCommand::Configure, "Configure"); + configured_ = CountConfigured(control_) != 0U; + const uint32_t bitmap_count = CountConfigured(control_); + const bool triplets_ok = ValidateConfiguredTripletTopology(control_); + const bool state_ok = control_.status == static_cast(PmuOwnerStatus::Success) && + control_.configured == 1U && control_.active_total == limits_.total && + control_.active_aic == limits_.aic && control_.active_aiv == limits_.aiv && + control_.discovered_total == limits_.total && control_.discovered_aic == limits_.aic && + control_.discovered_aiv == limits_.aiv && bitmap_count == limits_.total && + control_.skipped_total + bitmap_count == kPhysicalSubcoreCount && triplets_ok; + PrintControl("Configure", bitmap_count); + return command_ok && state_ok; + } + + bool Restore() + { + if (control_device_ == nullptr || owner_stream_ == nullptr || !loader_.IsInitialized()) { + return !configured_; + } + const bool command_ok = RunCommand(PmuOwnerMainCommand::Restore, "Restore"); + const uint32_t bitmap_count = CountConfigured(control_); + const bool state_ok = control_.status == static_cast(PmuOwnerStatus::Success) && + control_.configured == 0U && control_.active_total == 0U && + control_.active_aic == 0U && control_.active_aiv == 0U && bitmap_count == 0U; + configured_ = bitmap_count != 0U; + PrintControl("Restore", bitmap_count); + return command_ok && state_ok; + } + + bool Finalize() + { + bool ok = true; + if (control_device_ != nullptr && owner_stream_ != nullptr && loader_.IsInitialized()) { + bool restored = Restore(); + if (!restored) restored = Restore(); + ok &= restored; + } else { + ok &= !configured_; + } + ok &= loader_.Finalize() == 0; + if (control_device_ != nullptr) { + ok &= OwnerCheckAcl(aclrtFree(control_device_), "aclrtFree(PMU owner control)"); + control_device_ = nullptr; + } + if (owner_stream_ != nullptr) { + ok &= OwnerCheckAcl(aclrtDestroyStream(owner_stream_), "aclrtDestroyStream(PMU owner)"); + owner_stream_ = nullptr; + } + ready_ = false; + configured_ = false; + std::printf("[PMU_OWNER] restore_and_cleanup=%s\n", ok ? "PASS" : "FAIL"); + return ok; + } + + uint64_t RegisterTableDeviceAddress() const + { + if (control_device_ == nullptr) return 0U; + return reinterpret_cast(control_device_) + offsetof(PmuOwnerControl, register_bases); + } + + const PmuOwnerControl &Control() const { return control_; } + + bool IsConfiguredSubcore(uint32_t index) const + { + return ready_ && IsConfigured(control_, index); + } + +private: + bool HasResources() const + { + return owner_stream_ != nullptr || control_device_ != nullptr || loader_.IsInitialized(); + } + + bool RunCommand(PmuOwnerMainCommand command, const char *label) + { + const PmuOwnerMainKernelArgs arguments = MakePmuOwnerMainKernelArgs( + reinterpret_cast(control_device_), command, device_ + ); + const std::string sync_label = std::string("aclrtSynchronizeStream(PMU ") + label + ")"; + const std::string copy_label = std::string("aclrtMemcpy(D2H PMU ") + label + ")"; + if (loader_.Launch(owner_stream_, const_cast(&arguments), sizeof(arguments)) != 0 || + !OwnerCheckAcl(aclrtSynchronizeStream(owner_stream_), sync_label.c_str()) || + !OwnerCheckAcl( + aclrtMemcpy( + &control_, sizeof(control_), control_device_, sizeof(control_), ACL_MEMCPY_DEVICE_TO_HOST + ), + copy_label.c_str() + )) { + return false; + } + return true; + } + + void PrintControl(const char *command, uint32_t bitmap_count) const + { + const auto failed_field = static_cast(control_.first_failed_field); + const auto restore_field = static_cast(control_.first_restore_failed_field); + std::printf( + "[PMU_OWNER] command=%s status=%d configured=%u active=%u/%u/%u " + "discovered=%u/%u/%u bitmap=%u skipped=%u first_failed=%u:%s(%u):0x%x/0x%x " + "restore_failures=%u first_restore=%u:%s(%u):0x%x/0x%x\n", + command, static_cast(control_.status), control_.configured, + control_.active_total, control_.active_aic, control_.active_aiv, + control_.discovered_total, control_.discovered_aic, control_.discovered_aiv, + bitmap_count, control_.skipped_total, control_.first_failed_index, + OwnerFieldName(failed_field), control_.first_failed_field, + control_.first_failed_observed, control_.first_failed_expected, + control_.restore_failures, control_.first_restore_failed_index, + OwnerFieldName(restore_field), control_.first_restore_failed_field, + control_.first_restore_failed_observed, control_.first_restore_failed_expected + ); + std::printf( + "[PMU_OWNER] configured_bitmap=%08x:%08x:%08x:%08x\n", + control_.configured_bitmap[3], control_.configured_bitmap[2], + control_.configured_bitmap[1], control_.configured_bitmap[0] + ); + } + + uint32_t device_ = 0U; + aclrtStream owner_stream_ = nullptr; + MainAicpuLoader loader_; + void *control_device_ = nullptr; + PmuOwnerControl control_{}; + ActiveSubcoreLimits limits_{}; + bool ready_ = false; + bool configured_ = false; +}; + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_HOST_H_ diff --git a/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_main_abi.h b/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_main_abi.h new file mode 100644 index 0000000000..d6ba9e2239 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_main_abi.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_ABI_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_ABI_H_ + +#include +#include + +namespace pa_scheduler::pmu_owner { + +// 主 aicpu_scheduler 的统一入口 simpler_aicpu_exec 根据该命令选择配置或 +// 恢复。0 特意保留为 Invalid，避免零初始化参数意外改写 PMU 寄存器。 +enum class PmuOwnerMainCommand : uint32_t { + Invalid = 0U, + Configure = 1U, + Restore = 2U, +}; + +// 该结构逐字段复刻 A5 KernelArgs 的 152B ABI，但只使用固定宽度整数，因而 +// 不依赖 Simpler 的 DeviceArgs/Runtime C++ 类型。runtime_args_device 指向 +// PmuOwnerControl；command 位于原 enable_profiling_flag 的 offset 128。 +// 其余字段保持为零，既满足主 aicpu_scheduler 固定布局，也不引入外部依赖。 +struct PmuOwnerMainKernelArgs { + uint64_t unused[5]; // 0..39 + uint64_t device_args_device; // 40，当前 owner 不使用 + uint64_t runtime_args_device; // 48，PmuOwnerControl 的 GM 地址 + uint64_t register_bases_device; // 56，当前 control 已内嵌基址，保持为零 + uint64_t dump_data_base; // 64 + uint64_t l2_swimlane_data_base; // 72 + uint64_t pmu_data_base; // 80 + uint64_t dep_gen_data_base; // 88 + uint64_t l2_swimlane_rotation_table; // 96 + uint64_t aicore_pmu_ring_addrs; // 104 + uint64_t scope_stats_data_base; // 112 + uint32_t log_level; // 120 + uint32_t log_info_v; // 124 + uint32_t command; // 128，PmuOwnerMainCommand + uint32_t reserved_alignment; // 132 + uint64_t device_wall_data_base; // 136 + uint32_t device_id; // 144 + uint32_t force_simt_anchor; // 148 +}; + +static_assert(sizeof(PmuOwnerMainCommand) == sizeof(uint32_t), "PMU owner command ABI changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, device_args_device) == 40U, "device args offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, runtime_args_device) == 48U, "control pointer offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, register_bases_device) == 56U, "register pointer offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, command) == 128U, "PMU owner command offset changed"); +static_assert(offsetof(PmuOwnerMainKernelArgs, device_id) == 144U, "device id offset changed"); +static_assert(sizeof(PmuOwnerMainKernelArgs) == 152U, "main aicpu_scheduler KernelArgs ABI changed"); +static_assert(alignof(PmuOwnerMainKernelArgs) == 8U, "KernelArgs alignment changed"); + +inline PmuOwnerMainKernelArgs MakePmuOwnerMainKernelArgs( + uint64_t control_device, PmuOwnerMainCommand command, uint32_t device_id +) +{ + PmuOwnerMainKernelArgs arguments{}; + arguments.runtime_args_device = control_device; + arguments.command = static_cast(command); + arguments.device_id = device_id; + return arguments; +} + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_ABI_H_ diff --git a/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_main_loader.h b/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_main_loader.h new file mode 100644 index 0000000000..ea34f5fe59 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/pmu_owner_main_loader.h @@ -0,0 +1,389 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_LOADER_H_ +#define PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_LOADER_H_ + +// PMU owner 的自包含 host 装载器： +// 1. 通过 libaicpu_extend_kernels bootstrap 临时 dispatcher； +// 2. dispatcher 将 owner SO 落到主 aicpu_scheduler 的预安装目录； +// 3. 用 cpuKernelMode=0 JSON 注册 owner 的 simpler_aicpu_exec； +// 4. 后续 Configure/Restore 都用缓存的 rtFuncHandle 直接下发。 +// +// 本头文件故意不定义 owner 命令字段。Launch 接受调用方构造的完整参数块， +// 从而让装载 ABI 与 PMU 状态机 ABI 解耦，也便于先独立验证 Path-A。 + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "runtime/rt.h" +#include "runtime/runtime/rts/rts_kernel.h" + +namespace pa_scheduler::pmu_owner { + +class MainAicpuLoader { +public: + MainAicpuLoader() = default; + MainAicpuLoader(const MainAicpuLoader &) = delete; + MainAicpuLoader &operator=(const MainAicpuLoader &) = delete; + MainAicpuLoader(MainAicpuLoader &&) = delete; + MainAicpuLoader &operator=(MainAicpuLoader &&) = delete; + + ~MainAicpuLoader() { (void)Finalize(); } + + // stream 必须属于当前 device，并且调用期间当前 ACL device 不能切换。 + // 成功后 owner SO 已注册到主 aicpu_scheduler，但尚未执行任何 PMU 命令。 + int Initialize( + const std::string &dispatcher_so_path, const std::string &owner_so_path, + aclrtStream stream, int32_t device_id + ) + { + if (IsInitialized() || stream == nullptr || device_id < 0) { + return Fail("Initialize received invalid state, stream, or device id", kInvalidArgument); + } + + const std::vector dispatcher = ReadBinary(dispatcher_so_path); + const std::vector owner = ReadBinary(owner_so_path); + if (dispatcher.empty() || owner.empty()) { + std::fprintf( + stderr, "[PMU_OWNER_LOADER] cannot read dispatcher/owner: %s (%zu B), %s (%zu B)\n", + dispatcher_so_path.c_str(), dispatcher.size(), owner_so_path.c_str(), owner.size() + ); + return kFileError; + } + + device_id_ = device_id; + owner_fingerprint_ = FingerprintBytes(owner.data(), owner.size()); + owner_so_basename_ = MakeOwnerSoBasename(owner_fingerprint_, device_id_); + op_type_ = MakeOpType(owner_fingerprint_, device_id_); + + int result = Bootstrap(dispatcher, owner, stream); + if (result == 0) result = RegisterOwner(); + if (result != 0) { + (void)Finalize(); + return result; + } + return 0; + } + + // 参数块由 runtime 在 launch 时复制；调用方只需保证本函数返回前 host + // buffer 有效。参数中的 GM 指针仍必须在设备命令同步结束前保持有效。 + int Launch( + aclrtStream stream, void *kernel_arguments, size_t argument_bytes, + uint32_t aicpu_blocks = 1U + ) const + { + if (!IsInitialized() || stream == nullptr || kernel_arguments == nullptr || + argument_bytes == 0U || argument_bytes > std::numeric_limits::max() || + aicpu_blocks == 0U) { + return Fail("Launch received invalid state or arguments", kInvalidArgument); + } + + rtCpuKernelArgs_t cpu_arguments = {}; + cpu_arguments.baseArgs.args = kernel_arguments; + cpu_arguments.baseArgs.argsSize = static_cast(argument_bytes); + rtKernelLaunchCfg_t launch_config = {}; + rtLaunchKernelAttr_t launch_attribute = {}; + launch_config.attrs = &launch_attribute; + launch_config.numAttrs = 0U; + + const rtError_t result = rtsLaunchCpuKernel( + function_handle_, aicpu_blocks, static_cast(stream), + &launch_config, &cpu_arguments + ); + if (result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsLaunchCpuKernel failed: %d\n", result); + } + return static_cast(result); + } + + // Finalize 只释放 host/runtime 注册资源，不删除设备侧预安装 SO；后者按内容 + // 指纹命名，可由同一设备上的后续进程原子覆盖。 + int Finalize() + { + int result = 0; + function_handle_ = nullptr; + if (binary_handle_ != nullptr) { + const rtError_t unload_result = rtsBinaryUnload(binary_handle_); + if (unload_result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsBinaryUnload failed: %d\n", unload_result); + result = static_cast(unload_result); + } + binary_handle_ = nullptr; + } + if (!json_path_.empty()) { + if (std::remove(json_path_.c_str()) != 0 && result == 0) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] remove JSON failed: %s\n", json_path_.c_str()); + result = kFileError; + } + json_path_.clear(); + } + device_id_ = -1; + owner_fingerprint_ = 0U; + owner_so_basename_.clear(); + op_type_.clear(); + return result; + } + + bool IsInitialized() const { return binary_handle_ != nullptr && function_handle_ != nullptr; } + uint64_t OwnerFingerprint() const { return owner_fingerprint_; } + const std::string &OwnerSoBasename() const { return owner_so_basename_; } + const std::string &OpType() const { return op_type_; } + +private: + static constexpr int kInvalidArgument = -1; + static constexpr int kFileError = -2; + static constexpr int kBootstrapError = -3; + static constexpr uint64_t kFnvOffsetBasis = UINT64_C(14695981039346656037); + static constexpr uint64_t kFnvPrime = UINT64_C(1099511628211); + static constexpr const char *kOwnerFunction = "simpler_aicpu_exec"; + + struct DeviceBuffer { + void *address = nullptr; + DeviceBuffer() = default; + DeviceBuffer(const DeviceBuffer &) = delete; + DeviceBuffer &operator=(const DeviceBuffer &) = delete; + ~DeviceBuffer() + { + if (address != nullptr) (void)aclrtFree(address); + } + aclError Allocate(size_t bytes) + { + return aclrtMalloc(&address, bytes, ACL_MEM_MALLOC_HUGE_FIRST); + } + }; + + static int Fail(const char *message, int code) + { + std::fprintf(stderr, "[PMU_OWNER_LOADER] %s\n", message); + return code; + } + + static std::vector ReadBinary(const std::string &path) + { + std::ifstream input(path, std::ios::binary | std::ios::ate); + if (!input.is_open()) return {}; + const std::streampos end = input.tellg(); + if (end <= std::streampos(0) || + static_cast(end) > static_cast(std::numeric_limits::max())) { + return {}; + } + std::vector bytes(static_cast(end)); + input.seekg(0, std::ios::beg); + if (!input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size()))) { + return {}; + } + return bytes; + } + + static uint64_t FingerprintBytes(const void *data, size_t bytes) + { + const auto *input = static_cast(data); + uint64_t hash = kFnvOffsetBasis; + for (size_t index = 0U; index < bytes; ++index) { + hash ^= input[index]; + hash *= kFnvPrime; + } + return hash; + } + + static std::string MakeOwnerSoBasename(uint64_t fingerprint, int32_t device_id) + { + char name[128] = {}; + (void)snprintf( + name, sizeof(name), "pa_scheduler_pmu_owner_%016llx_d%d.so", + static_cast(fingerprint), device_id + ); + return name; + } + + static std::string MakeOpType(uint64_t fingerprint, int32_t device_id) + { + char name[160] = {}; + (void)snprintf( + name, sizeof(name), "pa_scheduler_pmu_owner_%016llx_d%d", + static_cast(fingerprint), device_id + ); + return name; + } + + int Bootstrap( + const std::vector &dispatcher, const std::vector &owner, + aclrtStream stream + ) const + { + DeviceBuffer dispatcher_device; + DeviceBuffer owner_device; + DeviceBuffer device_args; + aclError acl_result = dispatcher_device.Allocate(dispatcher.size()); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMalloc(dispatcher)", acl_result); + acl_result = aclrtMemcpy( + dispatcher_device.address, dispatcher.size(), dispatcher.data(), dispatcher.size(), + ACL_MEMCPY_HOST_TO_DEVICE + ); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMemcpy(dispatcher H2D)", acl_result); + + acl_result = owner_device.Allocate(owner.size()); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMalloc(owner)", acl_result); + acl_result = aclrtMemcpy( + owner_device.address, owner.size(), owner.data(), owner.size(), ACL_MEMCPY_HOST_TO_DEVICE + ); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMemcpy(owner H2D)", acl_result); + + constexpr size_t kDeviceArgsBytes = 160U; + uint8_t host_device_args[kDeviceArgsBytes] = {}; + const auto write_qword = [&](size_t offset, uint64_t value) { + std::memcpy(host_device_args + offset, &value, sizeof(value)); + }; + write_qword(96U, reinterpret_cast(dispatcher_device.address)); + write_qword(104U, static_cast(dispatcher.size())); + write_qword(112U, static_cast(device_id_)); + write_qword(120U, reinterpret_cast(owner_device.address)); + write_qword(128U, static_cast(owner.size())); + + acl_result = device_args.Allocate(kDeviceArgsBytes); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMalloc(bootstrap args)", acl_result); + acl_result = aclrtMemcpy( + device_args.address, kDeviceArgsBytes, host_device_args, kDeviceArgsBytes, + ACL_MEMCPY_HOST_TO_DEVICE + ); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtMemcpy(bootstrap args H2D)", acl_result); + + // k_args 总长和三个字符串 offset 与仓内已上板的 Path-A 完全一致。 + struct BootstrapArguments { + struct { + uint64_t unused[5]; + uint64_t device_args_address; + uint64_t padding[20]; + } kernel_args; + char kernel_name[32]; + char so_name[32]; + char op_name[32]; + } arguments = {}; + static_assert(offsetof(BootstrapArguments, kernel_args.device_args_address) == 40U, "bootstrap ABI changed"); + arguments.kernel_args.device_args_address = reinterpret_cast(device_args.address); + constexpr char kBootstrapKernel[] = "DynTileFwkKernelServerInit"; + constexpr char kBootstrapSo[] = "libaicpu_extend_kernels.so"; + static_assert(sizeof(kBootstrapKernel) <= sizeof(arguments.kernel_name), "bootstrap kernel name too long"); + static_assert(sizeof(kBootstrapSo) <= sizeof(arguments.so_name), "bootstrap SO name too long"); + std::memcpy(arguments.kernel_name, kBootstrapKernel, sizeof(kBootstrapKernel)); + std::memcpy(arguments.so_name, kBootstrapSo, sizeof(kBootstrapSo)); + + rtAicpuArgsEx_t runtime_arguments = {}; + runtime_arguments.args = &arguments; + runtime_arguments.argsSize = sizeof(arguments); + runtime_arguments.kernelNameAddrOffset = offsetof(BootstrapArguments, kernel_name); + runtime_arguments.soNameAddrOffset = offsetof(BootstrapArguments, so_name); + + const rtError_t launch_result = rtAicpuKernelLaunchExWithArgs( + rtKernelType_t::KERNEL_TYPE_AICPU_KFC, "AST_DYN_AICPU", 1U, + &runtime_arguments, nullptr, static_cast(stream), 0U + ); + if (launch_result != RT_ERROR_NONE) { + std::fprintf( + stderr, "[PMU_OWNER_LOADER] rtAicpuKernelLaunchExWithArgs failed: %d\n", + launch_result + ); + return static_cast(launch_result); + } + acl_result = aclrtSynchronizeStream(stream); + if (acl_result != ACL_SUCCESS) return ReportAcl("aclrtSynchronizeStream(bootstrap)", acl_result); + return 0; + } + + int RegisterOwner() + { + char path[256] = {}; + (void)snprintf( + path, sizeof(path), "/tmp/pa_scheduler_pmu_owner_%016llx_d%d_p%d_i%016llx.json", + static_cast(owner_fingerprint_), device_id_, static_cast(getpid()), + static_cast(reinterpret_cast(this)) + ); + json_path_ = path; + if (!WriteJson()) return kFileError; + + rtLoadBinaryOption_t option = {}; + option.optionId = RT_LOAD_BINARY_OPT_CPU_KERNEL_MODE; + option.value.cpuKernelMode = 0; + rtLoadBinaryConfig_t configuration = {}; + configuration.options = &option; + configuration.numOpt = 1U; + + rtError_t result = rtsBinaryLoadFromFile(json_path_.c_str(), &configuration, &binary_handle_); + if (result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsBinaryLoadFromFile failed: %d\n", result); + return static_cast(result); + } + result = rtsFuncGetByName(binary_handle_, op_type_.c_str(), &function_handle_); + if (result != RT_ERROR_NONE) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] rtsFuncGetByName(%s) failed: %d\n", op_type_.c_str(), result); + return static_cast(result); + } + return 0; + } + + bool WriteJson() const + { + std::ofstream json(json_path_, std::ios::out | std::ios::trunc); + if (!json.is_open()) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] cannot create JSON: %s\n", json_path_.c_str()); + return false; + } + // 所有动态字段仅含固定前缀、十六进制、十进制和下划线，不需要 JSON 转义。 + json << "{\n" + << " \"" << op_type_ << "\": {\n" + << " \"opInfo\": {\n" + << " \"functionName\": \"" << kOwnerFunction << "\",\n" + << " \"kernelSo\": \"" << owner_so_basename_ << "\",\n" + << " \"opKernelLib\": \"AICPUKernel\",\n" + << " \"computeCost\": \"100\",\n" + << " \"engine\": \"DNN_VM_AICPU\",\n" + << " \"flagAsync\": \"False\",\n" + << " \"flagPartial\": \"False\",\n" + << " \"userDefined\": \"False\"\n" + << " }\n" + << " }\n" + << "}\n"; + json.close(); + if (!json) { + std::fprintf(stderr, "[PMU_OWNER_LOADER] writing JSON failed: %s\n", json_path_.c_str()); + return false; + } + return true; + } + + static int ReportAcl(const char *operation, aclError result) + { + std::fprintf(stderr, "[PMU_OWNER_LOADER] %s failed: %d\n", operation, static_cast(result)); + return static_cast(result == ACL_SUCCESS ? kBootstrapError : result); + } + + int32_t device_id_ = -1; + uint64_t owner_fingerprint_ = 0U; + std::string owner_so_basename_; + std::string op_type_; + std::string json_path_; + rtBinHandle binary_handle_ = nullptr; + rtFuncHandle function_handle_ = nullptr; +}; + +} // namespace pa_scheduler::pmu_owner + +#endif // PA_SCHEDULER_CCEC_PMU_OWNER_MAIN_LOADER_H_ diff --git a/tests/atomic_probe/pa_scheduler/ccec/pmu_probe.h b/tests/atomic_probe/pa_scheduler/ccec/pmu_probe.h new file mode 100644 index 0000000000..9207ab6dfa --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/pmu_probe.h @@ -0,0 +1,169 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_CCEC_PMU_PROBE_H +#define PA_SCHEDULER_CCEC_PMU_PROBE_H + +#include + +#include "../common/pa_model.h" + +namespace pa_scheduler::ccec_pmu { + +// Empty/Scalar/ScalarDouble 在调度结束后校准门控底噪和 scalar 正向响应； +// IcacheSingle 在每核上成对累计隔离的 cold/warm 目标调用；SubmitAll 则在 +// 公共调度器 hook 内覆盖本 worker 的完整 Submit 回放窗口。 +enum class WindowMode : uint32_t { + Off = 0, + Empty = 1, + Scalar = 2, + ScalarDouble = 3, + // 保留已落远端的 I-cache 校准模式值，避免 standalone host/kernel 混用旧产物时 + // 把校准请求误解释成 Submit 窗口；新增模式只在枚举尾部扩展。 + IcacheSingle = 4, + // SubmitAll 从本 worker 的 orchestration/Submit 回放前开始，到最后一次 + // Submit 返回后停止。 + SubmitAll = 5, +}; + +inline bool IsSubmitWindow(WindowMode mode) { + return mode == WindowMode::SubmitAll; +} + +// PMU 控制已从 RunConfig 尾部拆到独立 PmuProbeConfig cache line。mode、 +// work_amount、64 位寄存器表地址和 magic 都有具名字段，避免用数组下标 +// 跨过 RunConfig 边界覆盖 winner workload。 +constexpr uint32_t kConfigMagicValue = 0x504d5531U; // "PMU1" + +// DAV_3510 有 36 个物理 AICore，每个 AICore 展开为 1 AIC + 2 AIV，共 108 个物理子核编号。 +constexpr uint32_t kPhysicalAicoreCount = 36; +constexpr uint32_t kPhysicalSubcoreCount = 108; +constexpr uint32_t kAicorePerDie = 18; +constexpr uint32_t kSubcoresPerDie = 54; +constexpr uint64_t kAivFirstOffset = 0x100000ULL; +constexpr uint64_t kAivSecondOffset = 0x200000ULL; +constexpr uint32_t kAicoreMapBytes = 0x300000U; + +// PIPE_UTILIZATION 事件由 standalone Main AICPU owner 配置，kernel 逐核读回并核对 selector。 +constexpr uint32_t kScalarBusyEvent = 0x1U; +constexpr uint32_t kIcacheRequestEvent = 0x34U; +constexpr uint32_t kIcacheMissEvent = 0x35U; +constexpr uint32_t kVectorBusyEvent = 0x501U; +constexpr uint32_t kCubeBusyEvent = 0x301U; +constexpr uint32_t kMte1BusyEvent = 0x701U; +constexpr uint32_t kMte2BusyEvent = 0x202U; +constexpr uint32_t kMte3BusyEvent = 0x203U; +constexpr uint32_t kFixBusyEvent = 0x714U; + +// DAV_3510 PMU MMIO offset。ld_dev 的立即数只有 12 bit，因此 kernel 会分别重基址到 0x2400/0x4200。 +constexpr uint32_t kSelectorBlockOffset = 0x2400U; +constexpr uint32_t kCounterBlockOffset = 0x4200U; +constexpr uint32_t kCnt2Offset = 0x4220U; +constexpr uint32_t kCnt0Offset = 0x4210U; +constexpr uint32_t kCnt1Offset = 0x4218U; +constexpr uint32_t kCnt3Offset = 0x4228U; +constexpr uint32_t kCnt4Offset = 0x4230U; +constexpr uint32_t kCnt5Offset = 0x4238U; +constexpr uint32_t kCnt6Offset = 0x4240U; +constexpr uint32_t kCnt7Offset = 0x4248U; +constexpr uint32_t kCnt8Offset = 0x4250U; +constexpr uint32_t kCnt9Offset = 0x4254U; +constexpr uint32_t kTotalLowOffset = 0x4260U; +constexpr uint32_t kTotalHighOffset = 0x4264U; +constexpr uint32_t kCnt2SelectorOffset = 0x2508U; +constexpr uint32_t kCnt0SelectorOffset = 0x2500U; +constexpr uint32_t kCnt1SelectorOffset = 0x2504U; +constexpr uint32_t kCnt3SelectorOffset = 0x250cU; +constexpr uint32_t kCnt4SelectorOffset = 0x2510U; +constexpr uint32_t kCnt5SelectorOffset = 0x2514U; +constexpr uint32_t kCnt6SelectorOffset = 0x2518U; +constexpr uint32_t kCnt7SelectorOffset = 0x251cU; +constexpr uint32_t kCnt8SelectorOffset = 0x2520U; +constexpr uint32_t kCnt9SelectorOffset = 0x2524U; + +// pmu_status 的 bits16..27 保存 get_coreid()；bits28..31 留给不参与 core id +// 解码的模式诊断。其余低位描述本条记录是否可信。 +constexpr uint32_t kStatusRequested = 1U << 0; +constexpr uint32_t kStatusRegMapped = 1U << 1; +constexpr uint32_t kStatusCoreIdValid = 1U << 2; +constexpr uint32_t kStatusCnt2Selector = 1U << 3; +constexpr uint32_t kStatusCnt6Selector = 1U << 4; +constexpr uint32_t kStatusCnt7Selector = 1U << 5; +constexpr uint32_t kStatusWindowStarted = 1U << 6; +constexpr uint32_t kStatusTotalNonzero = 1U << 7; +constexpr uint32_t kStatusPriorSnapshotLarger = 1U << 8; +constexpr uint32_t kStatusCnt0Selector = 1U << 9; +constexpr uint32_t kStatusCnt1Selector = 1U << 10; +constexpr uint32_t kStatusCnt3Selector = 1U << 11; +constexpr uint32_t kStatusCnt4Selector = 1U << 12; +constexpr uint32_t kStatusCnt5Selector = 1U << 13; +constexpr uint32_t kStatusCnt8Selector = 1U << 14; +constexpr uint32_t kStatusWindowStopped = 1U << 15; +// I-cache 配对标志不能复用 Submit start bit；StatusCoreId 只取 12 bit，故将 +// 它放在 core-id 区间之上的独立诊断位。 +constexpr uint32_t kStatusIcachePairObserved = 1U << 28; +constexpr uint32_t kStatusRequired = kStatusRequested | kStatusRegMapped | kStatusCoreIdValid | + kStatusCnt2Selector | kStatusCnt6Selector | kStatusCnt7Selector | + kStatusCnt0Selector | kStatusCnt1Selector | kStatusCnt3Selector | + kStatusCnt4Selector | kStatusCnt5Selector | kStatusCnt8Selector | + kStatusWindowStarted | kStatusWindowStopped | kStatusTotalNonzero; +constexpr uint32_t kStatusCoreIdShift = 16; +constexpr uint32_t kStatusCoreIdMask = 0x0fffU; + +// pmu_phase_status 独立于旧 pmu_status，避免与其中的物理 core-id 位域 +// 冲突。bits4/5 只记录 shadow 是否恰好等于 primary：phase=none 没有 +// 运行中 read-to-clear，host 会要求两位都成立；局部 phase 会在计数仍开启时 +// 读取 shadow，A5 实测存在同周期递增与读清竞争，因此不能把“逐次严格相等” +// 作为可信记录的共同必选位。局部 phase 的方向和误差包络由 host/raw 独立校验。 +constexpr uint32_t kPhaseStatusRequested = 1U << 0; +constexpr uint32_t kPhaseStatusShadowSelectors = 1U << 1; +constexpr uint32_t kPhaseStatusWindowStarted = 1U << 2; +constexpr uint32_t kPhaseStatusWindowStopped = 1U << 3; +constexpr uint32_t kPhaseStatusShadowRequestsMatch = 1U << 4; +constexpr uint32_t kPhaseStatusShadowMissesMatch = 1U << 5; +constexpr uint32_t kPhaseStatusBoundariesBalanced = 1U << 6; +constexpr uint32_t kPhaseStatusValuesOrdered = 1U << 7; +constexpr uint32_t kPhaseStatusUint32Fit = 1U << 8; +constexpr uint32_t kPhaseStatusPhaseShape = 1U << 9; +// none 必须保持 0 tick；运行阶段则必须确实累计到非零 SYS_CNT。阶段时间是否 +// 不超过同核首 Submit 到末 Submit 的完整区间，由拿到两端结果的 host 再校验。 +constexpr uint32_t kPhaseStatusTimeValid = 1U << 10; +constexpr uint32_t kPhaseStatusRequired = + kPhaseStatusRequested | kPhaseStatusShadowSelectors | + kPhaseStatusWindowStarted | kPhaseStatusWindowStopped | + kPhaseStatusBoundariesBalanced | kPhaseStatusValuesOrdered | + kPhaseStatusUint32Fit | kPhaseStatusPhaseShape | kPhaseStatusTimeValid; + +inline const char *SubmitPmuPhaseName(SubmitPmuPhase phase) { + switch (phase) { + case SubmitPmuPhase::None: + return "none"; + case SubmitPmuPhase::Claim: + return "claim"; + case SubmitPmuPhase::EfDrain: + return "efdrain"; + case SubmitPmuPhase::Materialize: + return "materialize"; + case SubmitPmuPhase::Register: + return "register"; + case SubmitPmuPhase::Count: + break; + } + return "invalid"; +} + +inline uint32_t StatusCoreId(uint32_t status) { + return (status >> kStatusCoreIdShift) & kStatusCoreIdMask; +} + +} // namespace pa_scheduler::ccec_pmu + +#endif // PA_SCHEDULER_CCEC_PMU_PROBE_H diff --git a/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_compile_probe.cpp b/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_compile_probe.cpp new file mode 100644 index 0000000000..5451a914c9 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_compile_probe.cpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" +#include "ccec_ops.h" + +// 该 TU 只做 shared 通用协议的设备编译器显式实例化，不参与最终 mixed +// ELF。它同时锁定 writer-intent 与 reader-progress/reclaim 使用的 CAS、 +// DCCI、GM 地址空间和引用签名，避免普通 PA kernel 尚未接线时只解析模板 +// 定义、却从未生成真实 AIC/AIV 代码。 +template pa_scheduler::SharedWriterIntentResult +pa_scheduler::PrepareSharedWriterIntentSet< + pa_scheduler_ccec::CcecOps>( + __gm__ pa_scheduler::SchedulerState *, + const pa_scheduler::TaskArgs &, + pa_scheduler::SubmitContext &, + pa_scheduler::LocalStats & +); + +template bool pa_scheduler::SharedAdvanceReaderDone< + pa_scheduler_ccec::CcecOps>( + __gm__ pa_scheduler::SharedTensorMapSidecar &, + uint32_t, int32_t +); + +template bool pa_scheduler::SharedRefreshReaderReclaimForTask< + pa_scheduler_ccec::CcecOps>( + __gm__ pa_scheduler::SharedTensorMapSidecar &, + int32_t, uint32_t, int32_t, int64_t & +); + +template pa_scheduler::SharedAppendCheck +pa_scheduler::SharedTryAppendReaderGatedTask< + pa_scheduler_ccec::CcecOps>( + __gm__ pa_scheduler::SharedTensorMapSidecar &, + const pa_scheduler::SharedRegionValue *, uint32_t, + uint32_t, int32_t +); diff --git a/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_litmus.sh b/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_litmus.sh new file mode 100755 index 0000000000..ce1676938a --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_litmus.sh @@ -0,0 +1,1393 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +BUILD_DIR="$ROOT_DIR/build/ccec/shared/shared-protocol-litmus" +MANIFEST="$BUILD_DIR/shared_protocol_litmus_artifacts.manifest" +SHARED_HEADER="$SCRIPT_DIR/shared_protocol_litmus_shared.h" +LITMUS_PROCESS_TIMEOUT_SECONDS=60 +SHARED_ABI_GENERATION="$( + sed -n \ + 's/^constexpr uint32_t kSharedAbiGeneration = \([0-9][0-9]*\);$/\1/p' \ + "$SHARED_HEADER" +)" +if [[ ! "$SHARED_ABI_GENERATION" =~ ^[0-9]+$ ]]; then + echo "Cannot read one shared ABI generation from $SHARED_HEADER." >&2 + exit 1 +fi + +usage() { + cat <<'EOF' +Usage: + ./ccec/shared_protocol_litmus.sh build + ./ccec/shared_protocol_litmus.sh run \ + --scenario history|reader-reclaim|all \ + [--ordering compiler-clobber|payload-dependency|dsb-all|all] \ + [--device N] [--runs N] + +Each selected scenario/direction/ordering tuple runs in a fresh host process. +History runs both directions with ordering "na". Reader-reclaim runs both +directions for all three orderings by default. Scenario "all" runs history +first, then all reader-reclaim orderings. + +--runs defaults to 20 and repeats every selected tuple that many times. +EOF +} + +require_toolchain() { + if [[ -z "${ASCEND_HOME_PATH:-}" ]]; then + echo "ASCEND_HOME_PATH is not set; source the user CANN 9.1 set_env.sh first." >&2 + exit 1 + fi + CCEC="$ASCEND_HOME_PATH/bin/ccec" + LD="$ASCEND_HOME_PATH/bin/ld.lld" + CXX_BIN="${CXX:-g++}" + READELF_BIN="${READELF:-readelf}" + LLVM_DIS_BIN="${LLVM_DIS:-/opt/mlir-debug/bin/llvm-dis}" + PTO_INCLUDE_ROOT="${PTO_ISA_ROOT:-$ASCEND_HOME_PATH/x86_64-linux}" + for tool in "$CCEC" "$LD"; do + if [[ ! -x "$tool" ]]; then + echo "Missing CANN tool: $tool" >&2 + exit 1 + fi + done + if [[ "$LLVM_DIS_BIN" == */* ]]; then + if [[ ! -x "$LLVM_DIS_BIN" ]]; then + echo "Missing LLVM bitcode disassembler: $LLVM_DIS_BIN" >&2 + echo "Set LLVM_DIS to an executable llvm-dis path." >&2 + exit 1 + fi + elif ! command -v "$LLVM_DIS_BIN" >/dev/null 2>&1; then + echo "Missing LLVM bitcode disassembler in PATH: $LLVM_DIS_BIN" >&2 + echo "Set LLVM_DIS to an executable llvm-dis path." >&2 + exit 1 + fi + if ! command -v "$CXX_BIN" >/dev/null 2>&1 || + ! command -v "$READELF_BIN" >/dev/null 2>&1 || + ! command -v sha256sum >/dev/null 2>&1; then + echo "shared protocol litmus requires C++, readelf, and sha256sum." >&2 + exit 1 + fi +} + +extract_ir_function() { + local ir_file="$1" + local function_name="$2" + awk -v name="$function_name" ' + !inside && /^define / && index($0, name) { + inside = 1 + } + inside { + print + } + inside && /^}/ { + exit + } + ' "$ir_file" +} + +require_ir_text() { + local block="$1" + local needle="$2" + local label="$3" + if ! grep -Fq -- "$needle" <<<"$block"; then + echo "Missing $label in optimized CCEC IR." >&2 + exit 1 + fi +} + +reject_ir_text() { + local block="$1" + local needle="$2" + local label="$3" + if grep -Fq -- "$needle" <<<"$block"; then + echo "Unexpected $label in optimized CCEC IR." >&2 + exit 1 + fi +} + +single_ir_line() { + local block="$1" + local needle="$2" + local label="$3" + local matches + matches="$( + grep -nF -- "$needle" <<<"$block" || true + )" + if [[ "$(wc -l <<<"$matches")" -ne 1 || + -z "$matches" ]]; then + echo "Expected exactly one $label in optimized CCEC IR." >&2 + exit 1 + fi + printf '%s\n' "${matches%%:*}" +} + +ssa_definition_in_block() { + local block="$1" + local ssa="$2" + local label="$3" + local matches + matches="$( + grep -F -- " $ssa = " <<<"$block" || true + )" + if [[ "$(wc -l <<<"$matches")" -ne 1 || + -z "$matches" ]]; then + echo "Expected exactly one $label SSA definition: $ssa" >&2 + exit 1 + fi + printf '%s\n' "$matches" +} + +extract_ir_basic_block() { + local function_block="$1" + local block_label="$2" + awk -v target="$block_label" ' + /^[[:alnum:]_.-]+:/ { + current = $0 + sub(/:.*/, "", current) + if (inside && current != target) { + exit + } + if (current == target) { + inside = 1 + } + } + inside { + print + } + ' <<<"$function_block" +} + +ir_instruction_block_label() { + local function_block="$1" + local instruction_needle="$2" + local label="$3" + local matches + matches="$( + awk -v needle="$instruction_needle" ' + /^[[:alnum:]_.-]+:/ { + current = $0 + sub(/:.*/, "", current) + } + index($0, needle) { + print current + } + ' <<<"$function_block" + )" + if [[ "$(wc -l <<<"$matches")" -ne 1 || + -z "$matches" ]]; then + echo "Expected exactly one $label instruction block in optimized CCEC IR." >&2 + exit 1 + fi + printf '%s\n' "$matches" +} + +unique_ir_result() { + local block="$1" + local instruction_pattern="$2" + local label="$3" + local matches + matches="$( + grep -E -- "$instruction_pattern" \ + <<<"$block" || true + )" + if [[ "$(wc -l <<<"$matches")" -ne 1 || + -z "$matches" ]]; then + echo "Expected exactly one $label instruction in optimized CCEC IR." >&2 + exit 1 + fi + local result_pattern='^[[:space:]]*(%[[:alnum:]_.-]+)[[:space:]]*=' + if [[ ! "$matches" =~ $result_pattern ]]; then + echo "Cannot identify $label SSA result in optimized CCEC IR." >&2 + exit 1 + fi + printf '%s\n' "${BASH_REMATCH[1]}" +} + +validate_reader_ordering_ir() { + local ir_file="$1" + local core_label="$2" + local entry_name + local entry_block + local compiler_block + local dependency_block + local dsb_block + local validate_block + if [[ "$core_label" == AIC ]]; then + entry_name=pa_scheduler_0_mix_aic + elif [[ "$core_label" == AIV ]]; then + entry_name=pa_scheduler_0_mix_aiv + else + echo "Unsupported shared protocol IR core label: $core_label" >&2 + exit 1 + fi + entry_block="$( + extract_ir_function "$ir_file" "$entry_name" + )" + compiler_block="$( + extract_ir_function \ + "$ir_file" "CloseReaderCompilerClobber" + )" + dependency_block="$( + extract_ir_function \ + "$ir_file" "CloseReaderPayloadDependency" + )" + dsb_block="$( + extract_ir_function "$ir_file" "CloseReaderDsbAll" + )" + validate_block="$( + extract_ir_function \ + "$ir_file" "ValidateReaderSnapshotAfterReuse" + )" + if [[ -z "$entry_block" ]]; then + echo "Missing $core_label O3 IR entry function: $entry_name" >&2 + exit 1 + fi + + local function_block + local function_name + for function_name in \ + CloseReaderCompilerClobber \ + CloseReaderPayloadDependency \ + CloseReaderDsbAll \ + ValidateReaderSnapshotAfterReuse; do + function_block="$( + extract_ir_function "$ir_file" "$function_name" + )" + if [[ -z "$function_block" ]]; then + echo "Missing $core_label O3 IR function: $function_name" >&2 + exit 1 + fi + done + + require_ir_text \ + "$compiler_block" \ + 'asm sideeffect "", "~{memory}"' \ + "$core_label compiler-only memory clobber" + require_ir_text \ + "$compiler_block" \ + '@llvm.hivm.atom.CAS.G.s64' \ + "$core_label compiler-only reader CAS" + require_ir_text \ + "$compiler_block" \ + 'i64 1, i64 2, i32 0' \ + "$core_label compiler-only constant CAS transition" + reject_ir_text \ + "$compiler_block" \ + '@llvm.hivm.DSB' \ + "$core_label compiler-only device barrier" + local compiler_clobber_line + local compiler_cas_line + compiler_clobber_line="$( + single_ir_line \ + "$compiler_block" \ + 'asm sideeffect "", "~{memory}"' \ + "$core_label compiler-only clobber" + )" + compiler_cas_line="$( + single_ir_line \ + "$compiler_block" \ + '@llvm.hivm.atom.CAS.G.s64' \ + "$core_label compiler-only CAS" + )" + if ((compiler_clobber_line >= compiler_cas_line)); then + echo "$core_label compiler clobber no longer precedes reader CAS." >&2 + exit 1 + fi + + require_ir_text \ + "$dependency_block" \ + 'asm sideeffect "MOV $0, $0", "=l,0,~{memory}"' \ + "$core_label payload tied MOV" + require_ir_text \ + "$dependency_block" \ + 'sub i32' \ + "$core_label payload dependency delta" + require_ir_text \ + "$dependency_block" \ + '@llvm.hivm.atom.CAS.G.s64' \ + "$core_label payload-dependent reader CAS" + reject_ir_text \ + "$dependency_block" \ + 'i64 1, i64 2, i32 0' \ + "$core_label payload path collapsed to constant CAS" + reject_ir_text \ + "$dependency_block" \ + '@llvm.hivm.DSB' \ + "$core_label payload path device barrier" + if ! grep -Eq \ + '@llvm\.hivm\.atom\.CAS\.G\.s64.*i64 %[[:alnum:]_.-]+, i64 %[[:alnum:]_.-]+, i32 0' \ + <<<"$dependency_block"; then + echo "$core_label payload CAS operands are not both dynamic in optimized CCEC IR." >&2 + exit 1 + fi + local dependency_header="${dependency_block%%$'\n'*}" + local ssa_pattern='(%[[:alnum:]_.-]+)' + local dependency_signature_pattern + dependency_signature_pattern="i32 noundef $ssa_pattern, i64 noundef $ssa_pattern, i64 noundef $ssa_pattern, i64 noundef $ssa_pattern, i32 noundef $ssa_pattern, i32 noundef $ssa_pattern" + if [[ ! "$dependency_header" =~ $dependency_signature_pattern ]]; then + echo "$core_label payload helper lost its worker + five scalar leaf ABI." >&2 + exit 1 + fi + local helper_buffer="${BASH_REMATCH[2]}" + local helper_lo="${BASH_REMATCH[3]}" + local helper_hi="${BASH_REMATCH[4]}" + local helper_producer="${BASH_REMATCH[5]}" + local helper_reserved="${BASH_REMATCH[6]}" + local checksum_leaves=() + local helper_value + for helper_value in \ + "$helper_buffer" "$helper_lo" "$helper_hi"; do + local low_half + local shifted_half + local high_half + low_half="$( + unique_ir_result \ + "$dependency_block" \ + " = trunc i64 $helper_value to i32" \ + "$core_label payload low half" + )" + shifted_half="$( + unique_ir_result \ + "$dependency_block" \ + " = lshr i64 $helper_value, 32" \ + "$core_label payload high-half shift" + )" + high_half="$( + unique_ir_result \ + "$dependency_block" \ + " = trunc i64 $shifted_half to i32" \ + "$core_label payload high half" + )" + checksum_leaves+=("$low_half" "$high_half") + done + checksum_leaves+=("$helper_producer" "$helper_reserved") + local checksum_steps + checksum_steps="$( + grep -Fc '16777619' <<<"$dependency_block" || true + )" + if [[ "$checksum_steps" -ne 8 ]]; then + echo "$core_label payload helper must preserve exactly eight checksum steps." >&2 + exit 1 + fi + local checksum_value="" + local checksum_leaf + local checksum_xor_pattern + local checksum_xor + for checksum_leaf in "${checksum_leaves[@]}"; do + if [[ -z "$checksum_value" ]]; then + checksum_xor_pattern=" = xor i32 ($checksum_leaf, -2128831035|-2128831035, $checksum_leaf)" + else + checksum_xor_pattern=" = xor i32 ($checksum_value, $checksum_leaf|$checksum_leaf, $checksum_value)" + fi + checksum_xor="$( + unique_ir_result \ + "$dependency_block" \ + "$checksum_xor_pattern" \ + "$core_label payload checksum XOR" + )" + checksum_value="$( + unique_ir_result \ + "$dependency_block" \ + " = mul i32 $checksum_xor, 16777619" \ + "$core_label payload checksum multiply" + )" + done + local final_checksum="$checksum_value" + local dependency_mov_instruction + dependency_mov_instruction="$( + grep -F \ + 'asm sideeffect "MOV $0, $0", "=l,0,~{memory}"' \ + <<<"$dependency_block" + )" + local dependency_mov_pattern + dependency_mov_pattern="^[[:space:]]*$ssa_pattern[[:space:]]*=.*\\(i32[[:space:]]$final_checksum\\)" + if [[ ! "$dependency_mov_instruction" =~ $dependency_mov_pattern ]]; then + echo "$core_label tied MOV no longer consumes the full payload checksum." >&2 + exit 1 + fi + local dependency_mov="${BASH_REMATCH[1]}" + local dependency_sub_instruction + dependency_sub_instruction="$( + grep -E ' = sub i32 ' <<<"$dependency_block" + )" + local dependency_sub_pattern + dependency_sub_pattern="^[[:space:]]*$ssa_pattern[[:space:]]*=[[:space:]]sub[[:space:]]i32[[:space:]]$dependency_mov,[[:space:]]$final_checksum" + if [[ ! "$dependency_sub_instruction" =~ $dependency_sub_pattern ]]; then + echo "$core_label payload dependency delta lost MOV/checksum use-def." >&2 + exit 1 + fi + local dependency_delta="${BASH_REMATCH[1]}" + local dependency_zext_instruction + dependency_zext_instruction="$( + grep -E \ + " = zext i32 $dependency_delta to i64" \ + <<<"$dependency_block" + )" + if [[ ! "$dependency_zext_instruction" =~ ^[[:space:]]*$ssa_pattern[[:space:]]*= ]]; then + echo "$core_label payload dependency delta no longer reaches i64 CAS operands." >&2 + exit 1 + fi + local dependency_delta64="${BASH_REMATCH[1]}" + local expected_instruction + local desired_instruction + expected_instruction="$( + grep -E \ + " = add .*i64 $dependency_delta64, 1" \ + <<<"$dependency_block" + )" + desired_instruction="$( + grep -E \ + " = add .*i64 $dependency_delta64, 2" \ + <<<"$dependency_block" + )" + if [[ ! "$expected_instruction" =~ ^[[:space:]]*$ssa_pattern[[:space:]]*= || + ! "$desired_instruction" =~ ^[[:space:]]*$ssa_pattern[[:space:]]*= ]]; then + echo "$core_label payload dependency no longer forms both CAS operands." >&2 + exit 1 + fi + [[ "$expected_instruction" =~ ^[[:space:]]*$ssa_pattern[[:space:]]*= ]] + local dependency_expected="${BASH_REMATCH[1]}" + [[ "$desired_instruction" =~ ^[[:space:]]*$ssa_pattern[[:space:]]*= ]] + local dependency_desired="${BASH_REMATCH[1]}" + local dependency_cas_instruction + dependency_cas_instruction="$( + grep -F '@llvm.hivm.atom.CAS.G.s64' \ + <<<"$dependency_block" + )" + if [[ "$dependency_cas_instruction" != *"i64 $dependency_expected, i64 $dependency_desired, i32 0"* ]]; then + echo "$core_label payload CAS no longer consumes both delta-derived operands." >&2 + exit 1 + fi + local dependency_mov_line + local dependency_sub_line + local dependency_cas_line + dependency_mov_line="$( + single_ir_line \ + "$dependency_block" \ + 'asm sideeffect "MOV $0, $0", "=l,0,~{memory}"' \ + "$core_label payload MOV" + )" + dependency_sub_line="$( + single_ir_line \ + "$dependency_block" \ + ' = sub i32 ' \ + "$core_label payload delta" + )" + dependency_cas_line="$( + single_ir_line \ + "$dependency_block" \ + '@llvm.hivm.atom.CAS.G.s64' \ + "$core_label payload CAS" + )" + if ((dependency_mov_line >= dependency_sub_line || + dependency_sub_line >= dependency_cas_line)); then + echo "$core_label payload MOV/delta/CAS instruction order changed." >&2 + exit 1 + fi + + require_ir_text \ + "$dsb_block" \ + '@llvm.hivm.DSB(i64 0)' \ + "$core_label DSB_ALL" + require_ir_text \ + "$dsb_block" \ + '@llvm.hivm.atom.CAS.G.s64' \ + "$core_label DSB reader CAS" + require_ir_text \ + "$dsb_block" \ + 'i64 1, i64 2, i32 0' \ + "$core_label DSB constant CAS transition" + local dsb_clobber_lines + mapfile -t dsb_clobber_lines < <( + grep -nF 'asm sideeffect "", "~{memory}"' \ + <<<"$dsb_block" | + cut -d: -f1 + ) + if [[ "${#dsb_clobber_lines[@]}" -ne 2 ]]; then + echo "$core_label DSB path must retain exactly two compiler clobbers." >&2 + exit 1 + fi + local dsb_line + local dsb_cas_line + dsb_line="$( + single_ir_line \ + "$dsb_block" '@llvm.hivm.DSB(i64 0)' \ + "$core_label DSB_ALL" + )" + dsb_cas_line="$( + single_ir_line \ + "$dsb_block" '@llvm.hivm.atom.CAS.G.s64' \ + "$core_label DSB CAS" + )" + if ((dsb_clobber_lines[0] >= dsb_line || + dsb_line >= dsb_clobber_lines[1] || + dsb_clobber_lines[1] >= dsb_cas_line)); then + echo "$core_label clobber/DSB/clobber/CAS instruction order changed." >&2 + exit 1 + fi + + require_ir_text \ + "$validate_block" \ + '@llvm.hivm.atom.ADD.G.s64' \ + "$core_label post-reuse gate reload" + local validation_calls + validation_calls="$( + grep -c \ + 'call.*ValidateReaderSnapshotAfterReuse' \ + <<<"$entry_block" || true + )" + if [[ "$validation_calls" -ne 1 ]]; then + echo "$core_label O3 IR must retain exactly one post-reuse snapshot validation call." >&2 + exit 1 + fi + local validate_header="${validate_block%%$'\n'*}" + local validate_signature_pattern + validate_signature_pattern="\\(ptr addrspace\\(1\\)[^,]*[[:space:]]$ssa_pattern, i32[^,]*[[:space:]]$ssa_pattern, ptr[^,]*[[:space:]]$ssa_pattern, i64[^,]*[[:space:]]$ssa_pattern, i64[^,]*[[:space:]]$ssa_pattern\\)" + if [[ ! "$validate_header" =~ $validate_signature_pattern ]]; then + echo "$core_label snapshot validator lost state/signal/snapshot/seq ABI." >&2 + exit 1 + fi + local validate_state="${BASH_REMATCH[1]}" + local validate_signal="${BASH_REMATCH[2]}" + local validate_snapshot="${BASH_REMATCH[3]}" + local validate_seq_before="${BASH_REMATCH[4]}" + local validate_seq_after="${BASH_REMATCH[5]}" + local validate_signal64 + validate_signal64="$( + unique_ir_result \ + "$validate_block" \ + " = zext i32 $validate_signal to i64" \ + "$core_label reuse signal extension" + )" + local validate_tasks_base + validate_tasks_base="$( + unique_ir_result \ + "$validate_block" \ + " = getelementptr inbounds %\"struct\\.pa_scheduler::SchedulerState\", ptr addrspace\\(1\\) $validate_state, i(32|64) 0, i32 6" \ + "$core_label scheduler task-array base" + )" + local validate_annotated_signal + validate_annotated_signal="$( + unique_ir_result \ + "$validate_block" \ + " = (tail )?call i64 @llvm\\.annotation\\.i64\\.p0\\(i64 $validate_signal64" \ + "$core_label annotated reuse signal" + )" + local validate_task_cell + validate_task_cell="$( + unique_ir_result \ + "$validate_block" \ + " = getelementptr inbounds \\[[0-9]+ x %\"struct\\.pa_scheduler::TaskCell\"\\], ptr addrspace\\(1\\) $validate_tasks_base, i(32|64) 0, i64 $validate_annotated_signal" \ + "$core_label signal-selected task cell" + )" + local validate_gate_pointer + validate_gate_pointer="$( + unique_ir_result \ + "$validate_block" \ + " = getelementptr inbounds %\"struct\\.pa_scheduler::TaskCell\", ptr addrspace\\(1\\) $validate_task_cell, i(32|64) 0, i32 2" \ + "$core_label TaskCell::deps_prepared pointer" + )" + local validate_token + validate_token="$( + unique_ir_result \ + "$validate_block" \ + " = (tail )?call i64 @llvm\\.hivm\\.atom\\.ADD\\.G\\.s64\\(ptr addrspace\\(1\\) $validate_gate_pointer, i64 0, i32 0\\)" \ + "$core_label same-signal gate atomic reload" + )" + local validate_compare + validate_compare="$( + unique_ir_result \ + "$validate_block" \ + " = icmp eq i64 ($validate_token, $validate_signal64|$validate_signal64, $validate_token)" \ + "$core_label gate-token comparison" + )" + local validate_branch + validate_branch="$( + grep -E \ + "br i1 $validate_compare, label %[[:alnum:]_.-]+, label %[[:alnum:]_.-]+" \ + <<<"$validate_block" || true + )" + if [[ "$(wc -l <<<"$validate_branch")" -ne 1 || + -z "$validate_branch" ]]; then + echo "$core_label snapshot validator lost its unique gate-success branch." >&2 + exit 1 + fi + local validate_branch_pattern + validate_branch_pattern="br i1 $validate_compare, label %([[:alnum:]_.-]+), label %([[:alnum:]_.-]+)" + if [[ ! "$validate_branch" =~ $validate_branch_pattern ]]; then + echo "$core_label cannot identify the gate-success basic block." >&2 + exit 1 + fi + local validate_success_label="${BASH_REMATCH[1]}" + local validate_failure_label="${BASH_REMATCH[2]}" + if [[ "$validate_success_label" == "$validate_failure_label" ]]; then + echo "$core_label gate-success and failure blocks unexpectedly alias." >&2 + exit 1 + fi + local validate_compare_block_label + validate_compare_block_label="$( + ir_instruction_block_label \ + "$validate_block" \ + "br i1 $validate_compare" \ + "$core_label gate-token branch" + )" + local validate_success_block + validate_success_block="$( + extract_ir_basic_block \ + "$validate_block" "$validate_success_label" + )" + if [[ -z "$validate_success_block" ]]; then + echo "$core_label cannot extract the gate-success basic block." >&2 + exit 1 + fi + local validate_success_header="${validate_success_block%%$'\n'*}" + local validate_success_predecessor_pattern + validate_success_predecessor_pattern="^$validate_success_label:[[:space:]]*; preds = %$validate_compare_block_label$" + if [[ ! "$validate_success_header" =~ $validate_success_predecessor_pattern ]]; then + echo "$core_label snapshot success block has an entry other than the gate-token true edge." >&2 + exit 1 + fi + + local validate_seq_or + validate_seq_or="$( + unique_ir_result \ + "$validate_success_block" \ + " = or i64 ($validate_seq_before, $validate_seq_after|$validate_seq_after, $validate_seq_before)" \ + "$core_label guarded seq pair" + )" + local validate_seq_check + validate_seq_check="$( + unique_ir_result \ + "$validate_success_block" \ + " = icmp eq i64 ($validate_seq_or, 0|0, $validate_seq_or)" \ + "$core_label guarded seq check" + )" + + local validate_payload_values=() + local validate_payload_types=(i64 i64 i64 i32 i32) + local validate_payload_expected=(30064771072 0 8 0 0) + local validate_field_pointer + local validate_field_value + validate_field_value="$( + unique_ir_result \ + "$validate_success_block" \ + " = load i64, ptr $validate_snapshot," \ + "$core_label guarded snapshot buffer load" + )" + validate_payload_values+=("$validate_field_value") + local validate_field + for validate_field in 1 2 3 4; do + validate_field_pointer="$( + unique_ir_result \ + "$validate_success_block" \ + " = getelementptr inbounds %\"struct\\.pa_scheduler::SharedRegionValue\", ptr $validate_snapshot, i(32|64) 0, i32 $validate_field" \ + "$core_label guarded snapshot field-$validate_field pointer" + )" + validate_field_value="$( + unique_ir_result \ + "$validate_success_block" \ + " = load ${validate_payload_types[validate_field]}, ptr $validate_field_pointer," \ + "$core_label guarded snapshot field-$validate_field load" + )" + validate_payload_values+=("$validate_field_value") + done + local validate_checks=("$validate_seq_check") + local validate_index + local validate_field_check + for ((validate_index = 0; + validate_index < 3; + ++validate_index)); do + validate_field_check="$( + unique_ir_result \ + "$validate_success_block" \ + " = icmp eq ${validate_payload_types[validate_index]} (${validate_payload_values[validate_index]}, ${validate_payload_expected[validate_index]}|${validate_payload_expected[validate_index]}, ${validate_payload_values[validate_index]})" \ + "$core_label guarded snapshot field-$validate_index check" + )" + validate_checks+=("$validate_field_check") + done + local validate_small_fields_or + validate_small_fields_or="$( + unique_ir_result \ + "$validate_success_block" \ + " = or i32 (${validate_payload_values[3]}, ${validate_payload_values[4]}|${validate_payload_values[4]}, ${validate_payload_values[3]})" \ + "$core_label guarded producer/reserved pair" + )" + validate_field_check="$( + unique_ir_result \ + "$validate_success_block" \ + " = icmp eq i32 ($validate_small_fields_or, 0|0, $validate_small_fields_or)" \ + "$core_label guarded producer/reserved check" + )" + validate_checks+=("$validate_field_check") + local validate_plain_load_count + validate_plain_load_count="$( + grep -Ec \ + ' = load (i64|i32), ptr ' \ + <<<"$validate_block" || true + )" + if [[ "$validate_plain_load_count" -ne 5 ]]; then + echo "$core_label snapshot validator must contain exactly five ordinary payload loads." >&2 + exit 1 + fi + + # 只允许五个叶子检查经 i1 AND 汇成成功值。这样既排除仅被 debug + # metadata 引用的“假消费”，也证明 seq/五字段确实共同决定最终返回。 + declare -A validate_masks=() + local validate_check + local validate_leaf_index + for ((validate_leaf_index = 0; + validate_leaf_index < ${#validate_checks[@]}; + ++validate_leaf_index)); do + validate_masks["${validate_checks[validate_leaf_index]}"]=$(( + 1 << validate_leaf_index + )) + done + local validate_and_lines + validate_and_lines="$( + grep -E \ + '^[[:space:]]*%[[:alnum:]_.-]+ = and i1 %[[:alnum:]_.-]+, %[[:alnum:]_.-]+' \ + <<<"$validate_success_block" || true + )" + local validate_changed=1 + local validate_and_line + local validate_and_pattern + local validate_and_result + local validate_and_left + local validate_and_right + validate_and_pattern='^[[:space:]]*(%[[:alnum:]_.-]+) = and i1 (%[[:alnum:]_.-]+), (%[[:alnum:]_.-]+)' + while [[ "$validate_changed" -eq 1 ]]; do + validate_changed=0 + while IFS= read -r validate_and_line; do + [[ -z "$validate_and_line" ]] && continue + if [[ ! "$validate_and_line" =~ $validate_and_pattern ]]; then + continue + fi + validate_and_result="${BASH_REMATCH[1]}" + validate_and_left="${BASH_REMATCH[2]}" + validate_and_right="${BASH_REMATCH[3]}" + if [[ -v "validate_masks[$validate_and_left]" && + -v "validate_masks[$validate_and_right]" ]]; then + local validate_new_mask=$(( + validate_masks["$validate_and_left"] | + validate_masks["$validate_and_right"] + )) + if [[ ! -v "validate_masks[$validate_and_result]" || + "${validate_masks[$validate_and_result]}" -ne "$validate_new_mask" ]]; then + validate_masks["$validate_and_result"]="$validate_new_mask" + validate_changed=1 + fi + fi + done <<<"$validate_and_lines" + done + local validate_full_mask=$(( + (1 << ${#validate_checks[@]}) - 1 + )) + local validate_success_value="" + local validate_mask_value + for validate_check in "${!validate_masks[@]}"; do + validate_mask_value="${validate_masks[$validate_check]}" + if [[ "$validate_mask_value" -eq "$validate_full_mask" ]]; then + if [[ -n "$validate_success_value" ]]; then + echo "$core_label snapshot validator has multiple full-check success values." >&2 + exit 1 + fi + validate_success_value="$validate_check" + fi + done + if [[ -z "$validate_success_value" ]]; then + echo "$core_label seq and payload checks no longer form one complete success value." >&2 + exit 1 + fi + local validate_merge_branch + validate_merge_branch="$( + grep -E \ + '^[[:space:]]*br label %[[:alnum:]_.-]+' \ + <<<"$validate_success_block" || true + )" + if [[ "$(wc -l <<<"$validate_merge_branch")" -ne 1 || + -z "$validate_merge_branch" ]]; then + echo "$core_label snapshot success block lost its unique merge edge." >&2 + exit 1 + fi + local validate_merge_pattern + validate_merge_pattern='br label %([[:alnum:]_.-]+)' + [[ "$validate_merge_branch" =~ $validate_merge_pattern ]] + local validate_merge_label="${BASH_REMATCH[1]}" + local validate_merge_block + validate_merge_block="$( + extract_ir_basic_block \ + "$validate_block" "$validate_merge_label" + )" + if [[ -z "$validate_merge_block" ]]; then + echo "$core_label cannot extract snapshot validation merge block." >&2 + exit 1 + fi + local validate_phi_line + validate_phi_line="$( + grep -E \ + '^[[:space:]]*%[[:alnum:]_.-]+ = phi i1 ' \ + <<<"$validate_merge_block" || true + )" + if [[ "$(wc -l <<<"$validate_phi_line")" -ne 1 || + -z "$validate_phi_line" ]]; then + echo "$core_label snapshot validator lost its unique result phi." >&2 + exit 1 + fi + local validate_phi_result_pattern + validate_phi_result_pattern='^[[:space:]]*(%[[:alnum:]_.-]+) = phi i1 ' + [[ "$validate_phi_line" =~ $validate_phi_result_pattern ]] + local validate_phi_result="${BASH_REMATCH[1]}" + local validate_phi_inputs + validate_phi_inputs="$( + grep -oE \ + '\[ [^]]+ \]' \ + <<<"$validate_phi_line" || true + )" + local validate_phi_input + local validate_phi_input_pattern + local validate_phi_value + local validate_phi_predecessor + local validate_success_input_count=0 + validate_phi_input_pattern='^\[ (false|%[[:alnum:]_.-]+), %([[:alnum:]_.-]+) \]$' + while IFS= read -r validate_phi_input; do + [[ -z "$validate_phi_input" ]] && continue + if [[ ! "$validate_phi_input" =~ $validate_phi_input_pattern ]]; then + echo "$core_label cannot parse snapshot validator phi input." >&2 + exit 1 + fi + validate_phi_value="${BASH_REMATCH[1]}" + validate_phi_predecessor="${BASH_REMATCH[2]}" + if [[ "$validate_phi_predecessor" == "$validate_success_label" ]]; then + if [[ "$validate_phi_value" != "$validate_success_value" ]]; then + echo "$core_label success edge no longer returns the complete snapshot check." >&2 + exit 1 + fi + validate_success_input_count=$(( + validate_success_input_count + 1 + )) + elif [[ "$validate_phi_value" != false ]]; then + echo "$core_label snapshot validator has a non-failure bypass edge." >&2 + exit 1 + fi + done <<<"$validate_phi_inputs" + if [[ "$validate_success_input_count" -ne 1 ]]; then + echo "$core_label snapshot validator must have exactly one success phi edge." >&2 + exit 1 + fi + local validate_return_count + validate_return_count="$( + grep -Ec \ + '^[[:space:]]*ret i1 ' \ + <<<"$validate_block" || true + )" + if [[ "$validate_return_count" -ne 1 ]]; then + echo "$core_label snapshot validator must retain exactly one boolean return." >&2 + exit 1 + fi + require_ir_text \ + "$validate_merge_block" \ + "ret i1 $validate_phi_result" \ + "$core_label complete snapshot-validation return" + + local dependency_call + dependency_call="$( + grep 'call.*CloseReaderPayloadDependency' \ + <<<"$entry_block" + )" + local dependency_call_pattern + dependency_call_pattern="CloseReaderPayloadDependency.*i32 noundef $ssa_pattern, i64 noundef $ssa_pattern, i64 noundef $ssa_pattern, i64 noundef $ssa_pattern, i32 noundef $ssa_pattern, i32 noundef $ssa_pattern" + if [[ ! "$dependency_call" =~ $dependency_call_pattern ]]; then + echo "$core_label payload close call lost five scalar arguments." >&2 + exit 1 + fi + local call_payload_values=( + "${BASH_REMATCH[2]}" + "${BASH_REMATCH[3]}" + "${BASH_REMATCH[4]}" + "${BASH_REMATCH[5]}" + "${BASH_REMATCH[6]}" + ) + local call_payload_types=(i64 i64 i64 i32 i32) + local payload_index + local payload_definition + for ((payload_index = 0; + payload_index < ${#call_payload_values[@]}; + ++payload_index)); do + payload_definition="$( + ssa_definition_in_block \ + "$entry_block" \ + "${call_payload_values[payload_index]}" \ + "$core_label captured payload" + )" + if [[ "$payload_definition" != *"load ${call_payload_types[payload_index]}, ptr addrspace(1)"* ]]; then + echo "$core_label payload close argument is not a direct GM load." >&2 + exit 1 + fi + done + + local validation_call + validation_call="$( + grep 'call.*ValidateReaderSnapshotAfterReuse' \ + <<<"$entry_block" + )" + local validation_call_pattern + validation_call_pattern="ValidateReaderSnapshotAfterReuse.*\\(ptr addrspace\\(1\\)[^,]*[[:space:]]$ssa_pattern, i32[^,]*[[:space:]]$ssa_pattern, ptr[^,]*[[:space:]]$ssa_pattern, i64[^,]*[[:space:]]$ssa_pattern, i64[^,]*[[:space:]]$ssa_pattern\\)" + if [[ ! "$validation_call" =~ $validation_call_pattern ]]; then + echo "$core_label snapshot validation call lost state/signal/snapshot/seq arguments." >&2 + exit 1 + fi + local ordering_function + local ordering_calls + for ordering_function in \ + CloseReaderCompilerClobber \ + CloseReaderPayloadDependency \ + CloseReaderDsbAll; do + ordering_calls="$( + grep -c "call.*$ordering_function" \ + <<<"$entry_block" || true + )" + if [[ "$ordering_calls" -ne 1 ]]; then + echo "$core_label O3 IR must call $ordering_function exactly once." >&2 + exit 1 + fi + done + echo "[CHECK] $core_label O3 IR preserves three reader-close paths and same-gate guarded snapshot validation" +} + +build_litmus() { + require_toolchain + mkdir -p "$BUILD_DIR" + rm -f -- \ + "$MANIFEST" \ + "$BUILD_DIR/shared_protocol_litmus_aic.bc" \ + "$BUILD_DIR/shared_protocol_litmus_aic.ll" \ + "$BUILD_DIR/shared_protocol_litmus_aiv.bc" \ + "$BUILD_DIR/shared_protocol_litmus_aiv.ll" + + local common_flags=( + -c -O3 -g -x cce -Wall -std=c++17 + --cce-aicore-only + -mllvm -cce-aicore-stack-size=0x8000 + -mllvm -cce-aicore-function-stack-size=0x8000 + -mllvm -cce-aicore-record-overflow=false + -mllvm -cce-aicore-addr-transform + -mllvm -cce-aicore-dcci-insert-for-scalar=false + -mllvm -cce-aicore-dcci-before-kernel-end=false + -DPTO_FDWIC_SHARED_MAP=1 + -DPTO_FDWIC_TENSORMAP_RING_CAP=128 + -DPA_BUILD_SWIMLANE=0 + -DPA_BUILD_SUBMIT_PMU=0 + -DPA_BUILD_PERF_CLOCK=0 + -DPA_SUBMIT_PMU_PHASE_ID=0 + -I"$ROOT_DIR/common" + -I"$PTO_INCLUDE_ROOT/include" + ) + + echo "[BUILD] shared protocol litmus AIC entry" + "$CCEC" "${common_flags[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DPA_BUILD_AIC \ + -o "$BUILD_DIR/shared_protocol_litmus_aic.o" \ + "$SCRIPT_DIR/shared_protocol_litmus_kernel.cpp" + + echo "[BUILD] shared protocol litmus AIV entry" + "$CCEC" "${common_flags[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DPA_BUILD_AIV \ + -o "$BUILD_DIR/shared_protocol_litmus_aiv.o" \ + "$SCRIPT_DIR/shared_protocol_litmus_kernel.cpp" + + echo "[BUILD] shared protocol litmus AIC optimized LLVM bitcode" + "$CCEC" "${common_flags[@]}" \ + --cce-aicore-arch=dav-c310-cube \ + -DPA_BUILD_AIC \ + -Xclang -emit-llvm-bc \ + -o "$BUILD_DIR/shared_protocol_litmus_aic.bc" \ + "$SCRIPT_DIR/shared_protocol_litmus_kernel.cpp" + "$LLVM_DIS_BIN" \ + "$BUILD_DIR/shared_protocol_litmus_aic.bc" \ + -o "$BUILD_DIR/shared_protocol_litmus_aic.ll" + + echo "[BUILD] shared protocol litmus AIV optimized LLVM bitcode" + "$CCEC" "${common_flags[@]}" \ + --cce-aicore-arch=dav-c310-vec \ + -DPA_BUILD_AIV \ + -Xclang -emit-llvm-bc \ + -o "$BUILD_DIR/shared_protocol_litmus_aiv.bc" \ + "$SCRIPT_DIR/shared_protocol_litmus_kernel.cpp" + "$LLVM_DIS_BIN" \ + "$BUILD_DIR/shared_protocol_litmus_aiv.bc" \ + -o "$BUILD_DIR/shared_protocol_litmus_aiv.ll" + + local ir_artifact + for ir_artifact in \ + "$BUILD_DIR/shared_protocol_litmus_aic.bc" \ + "$BUILD_DIR/shared_protocol_litmus_aic.ll" \ + "$BUILD_DIR/shared_protocol_litmus_aiv.bc" \ + "$BUILD_DIR/shared_protocol_litmus_aiv.ll"; do + if [[ ! -s "$ir_artifact" ]]; then + echo "CCEC shared protocol IR artifact is empty: $ir_artifact" >&2 + exit 1 + fi + done + echo "[CHECK] AIC/AIV optimized LLVM bitcode and textual IR are non-empty" + validate_reader_ordering_ir \ + "$BUILD_DIR/shared_protocol_litmus_aic.ll" AIC + validate_reader_ordering_ir \ + "$BUILD_DIR/shared_protocol_litmus_aiv.ll" AIV + + local object + for object in \ + "$BUILD_DIR/shared_protocol_litmus_aic.o" \ + "$BUILD_DIR/shared_protocol_litmus_aiv.o"; do + if "$READELF_BIN" --relocs --wide "$object" | + grep -q '__multi3'; then + echo "CCEC shared protocol path generated unsupported __multi3: $object" >&2 + exit 1 + fi + if "$READELF_BIN" --symbols --wide "$object" | + awk '$5 == "GLOBAL" && $7 == "UND" {found = 1} END {exit !found}'; then + echo "CCEC shared protocol object retains an undefined global symbol: $object" >&2 + "$READELF_BIN" --symbols --wide "$object" | + awk '$5 == "GLOBAL" && $7 == "UND" {print}' >&2 + exit 1 + fi + done + echo "[CHECK] AIC/AIV shared protocol objects need no device runtime helper" + + "$LD" -m aicorelinux -Ttext=0 -static \ + --version-script="$SCRIPT_DIR/pa_scheduler_device_exports.map" \ + -o "$BUILD_DIR/shared_protocol_litmus_kernel.o" \ + "$BUILD_DIR/shared_protocol_litmus_aic.o" \ + "$BUILD_DIR/shared_protocol_litmus_aiv.o" + + local symbols + local sections + symbols="$( + "$READELF_BIN" --symbols --wide --sym-base=10 \ + "$BUILD_DIR/shared_protocol_litmus_kernel.o" + )" + sections="$( + "$READELF_BIN" --sections --wide \ + "$BUILD_DIR/shared_protocol_litmus_kernel.o" + )" + local entry + for entry in pa_scheduler_0_mix_aic pa_scheduler_0_mix_aiv; do + if ! awk -v name="$entry" \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && + $NF == name && $3 + 0 > 0 {found = 1} + END {exit !found}' <<<"$symbols"; then + echo "Missing non-empty shared protocol mixed entry: $entry" >&2 + exit 1 + fi + if [[ "$sections" != *".ascend.meta.$entry"* ]]; then + echo "Missing shared protocol mixed metadata: .ascend.meta.$entry" >&2 + exit 1 + fi + done + if awk \ + '$4 == "FUNC" && $5 == "GLOBAL" && $7 != "UND" && + $NF != "pa_scheduler_0_mix_aic" && + $NF != "pa_scheduler_0_mix_aiv" {found = 1} + END {exit !found}' <<<"$symbols"; then + echo "Shared protocol ELF exports an unexpected GLOBAL function." >&2 + exit 1 + fi + if "$READELF_BIN" --relocs --wide \ + "$BUILD_DIR/shared_protocol_litmus_kernel.o" | + grep -q '^Relocation section'; then + echo "Shared protocol mixed ELF retains relocations." >&2 + exit 1 + fi + echo "[CHECK] shared protocol mixed ELF has two entries, metadata, and no relocations" + + echo "[BUILD] shared protocol litmus host" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror \ + -Wno-deprecated-declarations \ + -DPTO_FDWIC_SHARED_MAP=1 \ + -DPTO_FDWIC_TENSORMAP_RING_CAP=128 \ + -DPA_BUILD_SWIMLANE=0 \ + -DPA_BUILD_SUBMIT_PMU=0 \ + -DPA_BUILD_PERF_CLOCK=0 \ + -DPA_SUBMIT_PMU_PHASE_ID=0 \ + -I"$ROOT_DIR/common" \ + -I"$ASCEND_HOME_PATH/include" \ + -I"$ASCEND_HOME_PATH/pkg_inc" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime" \ + -I"$ASCEND_HOME_PATH/pkg_inc/runtime/runtime" \ + "$SCRIPT_DIR/shared_protocol_litmus_host.cpp" \ + -L"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -Wl,-rpath,"$ASCEND_HOME_PATH/x86_64-linux/lib64" \ + -lascendcl -lruntime \ + -o "$BUILD_DIR/shared_protocol_litmus_host" + + local manifest_tmp + manifest_tmp="$(mktemp "$BUILD_DIR/.shared_protocol_litmus_manifest.XXXXXX")" + trap 'rm -f -- "${manifest_tmp:-}"' EXIT + { + printf '# schema=pa_scheduler_shared_protocol_litmus/v2\n' + printf '# tensormap_mode=shared\n' + printf '# shared_abi_generation=%s\n' "$SHARED_ABI_GENERATION" + printf '# scenarios=history,reader-reclaim\n' + printf '# history_directions=aic-to-aiv,aiv-to-aic\n' + printf '# reader_reclaim_directions=aic-to-aiv,aiv-to-aic\n' + printf '# reader_reclaim_orderings=compiler-clobber,payload-dependency,dsb-all\n' + ( + cd "$BUILD_DIR" + sha256sum \ + shared_protocol_litmus_host \ + shared_protocol_litmus_kernel.o \ + shared_protocol_litmus_aic.ll \ + shared_protocol_litmus_aiv.ll + ) + } > "$manifest_tmp" + mv -f -- "$manifest_tmp" "$MANIFEST" + manifest_tmp="" + trap - EXIT + echo "[BUILD] complete: $BUILD_DIR" +} + +validate_artifacts() { + if [[ ! -x "$BUILD_DIR/shared_protocol_litmus_host" || + ! -s "$BUILD_DIR/shared_protocol_litmus_kernel.o" || + ! -s "$BUILD_DIR/shared_protocol_litmus_aic.ll" || + ! -s "$BUILD_DIR/shared_protocol_litmus_aiv.ll" || + ! -s "$MANIFEST" ]]; then + echo "Missing shared protocol litmus artifacts; run '$0 build' first." >&2 + exit 1 + fi + if ! command -v sha256sum >/dev/null 2>&1; then + echo "Shared protocol litmus validation requires sha256sum." >&2 + exit 1 + fi + + local expected_headers=( + "# schema=pa_scheduler_shared_protocol_litmus/v2" + "# tensormap_mode=shared" + "# shared_abi_generation=$SHARED_ABI_GENERATION" + "# scenarios=history,reader-reclaim" + "# history_directions=aic-to-aiv,aiv-to-aic" + "# reader_reclaim_directions=aic-to-aiv,aiv-to-aic" + "# reader_reclaim_orderings=compiler-clobber,payload-dependency,dsb-all" + ) + local expected_artifacts=( + shared_protocol_litmus_host + shared_protocol_litmus_kernel.o + shared_protocol_litmus_aic.ll + shared_protocol_litmus_aiv.ll + ) + local expected_lines=$(( + ${#expected_headers[@]} + ${#expected_artifacts[@]} + )) + local manifest_valid=true + if [[ "$(wc -l < "$MANIFEST")" -ne "$expected_lines" ]]; then + manifest_valid=false + fi + + local index + for ((index = 0; index < ${#expected_headers[@]}; ++index)); do + if [[ "$(sed -n "$((index + 1))p" "$MANIFEST")" != \ + "${expected_headers[index]}" ]]; then + manifest_valid=false + fi + done + + local digest + local artifact + local extra + local line_number + for ((index = 0; index < ${#expected_artifacts[@]}; ++index)); do + line_number=$((${#expected_headers[@]} + index + 1)) + digest="" + artifact="" + extra="" + if ! read -r digest artifact extra < <( + sed -n "${line_number}p" "$MANIFEST" + ); then + manifest_valid=false + fi + if [[ ! "$digest" =~ ^[0-9a-f]{64}$ || + "$artifact" != "${expected_artifacts[index]}" || + -n "$extra" ]]; then + manifest_valid=false + fi + done + + if [[ "$manifest_valid" != true ]]; then + echo "Shared protocol litmus manifest identity is invalid." >&2 + exit 1 + fi + ( + cd "$BUILD_DIR" + tail -n "${#expected_artifacts[@]}" "$MANIFEST" | + sha256sum --strict -c - + ) +} + +run_one_litmus_process() { + local scenario="$1" + local direction="$2" + local ordering="$3" + local device="$4" + local run="$5" + local runs="$6" + echo "[RUN] scenario=$scenario direction=$direction ordering=$ordering process=$run/$runs" + # 单进程正常包含约 1 GiB state 的 H2D/D2H，实测通常在 8 秒左右。 + # 60 秒只负责把 ACL stream 异常停滞变成明确失败，不做自动重试， + # 避免跳过一个可能属于协议本身的偶发等待。 + timeout --signal=INT --kill-after=5s \ + "${LITMUS_PROCESS_TIMEOUT_SECONDS}s" \ + "$BUILD_DIR/shared_protocol_litmus_host" \ + "$BUILD_DIR/shared_protocol_litmus_kernel.o" \ + "$scenario" "$direction" "$ordering" "$device" || { + local status=$? + echo "[FAIL] shared protocol process did not complete: " \ + "scenario=$scenario direction=$direction " \ + "ordering=$ordering process=$run/$runs status=$status" >&2 + return "$status" + } +} + +run_litmus() { + if ! command -v timeout >/dev/null 2>&1; then + echo "shared protocol litmus run requires the coreutils timeout command." >&2 + exit 1 + fi + local device=0 + local runs=20 + local scenario="" + local ordering=all + while [[ $# -gt 0 ]]; do + case "$1" in + --scenario) + if [[ $# -lt 2 || + ( "$2" != "history" && + "$2" != "reader-reclaim" && + "$2" != "all" ) ]]; then + echo "--scenario requires history, reader-reclaim, or all." >&2 + exit 1 + fi + scenario="$2" + shift 2 + ;; + --ordering) + if [[ $# -lt 2 || + ( "$2" != "compiler-clobber" && + "$2" != "payload-dependency" && + "$2" != "dsb-all" && + "$2" != "all" ) ]]; then + echo "--ordering requires compiler-clobber, payload-dependency, dsb-all, or all." >&2 + exit 1 + fi + ordering="$2" + shift 2 + ;; + --device) + if [[ $# -lt 2 || ! "$2" =~ ^[0-9]+$ ]]; then + echo "--device requires a non-negative integer." >&2 + exit 1 + fi + device="$2" + shift 2 + ;; + --runs) + if [[ $# -lt 2 || ! "$2" =~ ^[0-9]+$ || + "$2" -lt 1 || "$2" -gt 100 ]]; then + echo "--runs must be in [1, 100]." >&2 + exit 1 + fi + runs="$2" + shift 2 + ;; + *) + echo "Unknown shared protocol litmus option: $1" >&2 + usage >&2 + exit 1 + ;; + esac + done + if [[ -z "$scenario" ]]; then + echo "--scenario history, reader-reclaim, or all is required." >&2 + usage >&2 + exit 1 + fi + if [[ "$scenario" != "reader-reclaim" && "$ordering" != "all" ]]; then + echo "--ordering may be non-all only with --scenario reader-reclaim." >&2 + exit 1 + fi + + validate_artifacts + local directions=(aic-to-aiv aiv-to-aic) + local reader_orderings=( + compiler-clobber + payload-dependency + dsb-all + ) + if [[ "$scenario" == "reader-reclaim" && "$ordering" != "all" ]]; then + reader_orderings=("$ordering") + fi + + local run + local direction + local reader_ordering + local process_count=0 + for ((run = 1; run <= runs; ++run)); do + if [[ "$scenario" == "history" || "$scenario" == "all" ]]; then + for direction in "${directions[@]}"; do + run_one_litmus_process \ + history "$direction" na "$device" "$run" "$runs" + ((process_count += 1)) + done + fi + if [[ "$scenario" == "reader-reclaim" || + "$scenario" == "all" ]]; then + for reader_ordering in "${reader_orderings[@]}"; do + for direction in "${directions[@]}"; do + run_one_litmus_process \ + reader-reclaim "$direction" "$reader_ordering" \ + "$device" "$run" "$runs" + ((process_count += 1)) + done + done + fi + done + echo "[PASS] scenario=$scenario ordering=$ordering runs=$runs fresh_processes=$process_count" +} + +if [[ $# -lt 1 ]]; then + usage >&2 + exit 1 +fi + +action="$1" +shift +case "$action" in + build) + if [[ $# -ne 0 ]]; then + usage >&2 + exit 1 + fi + build_litmus + ;; + run) + run_litmus "$@" + ;; + -h|--help|help) + usage + ;; + *) + usage >&2 + exit 1 + ;; +esac diff --git a/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_litmus_host.cpp b/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_litmus_host.cpp new file mode 100644 index 0000000000..6bcce7efce --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_litmus_host.cpp @@ -0,0 +1,1260 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#define PA_DEVICE inline +#define PA_GM +#include "../common/pa_shared_tensormap.h" +#undef PA_GM +#undef PA_DEVICE +#include "shared_protocol_litmus_shared.h" + +#include "acl/acl.h" +#include "runtime/rt.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using pa_scheduler::SchedulerState; +using pa_scheduler::SharedWriterHistoryCell; +using pa_scheduler::SharedWriterHistoryRecord; +using pa_scheduler::TensorDesc; +using pa_scheduler::WorkerResult; +using pa_scheduler::shared_protocol_litmus::Control; +using pa_scheduler::shared_protocol_litmus::Direction; +using pa_scheduler::shared_protocol_litmus::HistoryChain; +using pa_scheduler::shared_protocol_litmus::ReaderOrdering; +using pa_scheduler::shared_protocol_litmus::ReaderReclaimChain; +using pa_scheduler::shared_protocol_litmus::Scenario; +using pa_scheduler::shared_protocol_litmus::kAicToAiv; +using pa_scheduler::shared_protocol_litmus::kAivToAic; +using pa_scheduler::shared_protocol_litmus::kControlMagic; +using pa_scheduler::shared_protocol_litmus::kControlVersion; +using pa_scheduler::shared_protocol_litmus::kFutureWritersStatus; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimActiveWorkers; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimAddress; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimAicToAiv; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimAivToAic; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimClosedDone; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimHeapWindow; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimHi; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimInitialDone; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimLo; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimReaderStatus; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimReplacementAddress; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimReplacementHi; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimReplacementLo; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimReclaimerStatus; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimTask; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimWriterTask; +using pa_scheduler::shared_protocol_litmus::kReaderStatus; +using pa_scheduler::shared_protocol_litmus::kResultMagic; +using pa_scheduler::shared_protocol_litmus::kSymbolCount; +using pa_scheduler::shared_protocol_litmus::kWriterBStatus; + +constexpr size_t kStatePrefixBytes = + offsetof(SchedulerState, workers); +constexpr size_t kResultBytes = + sizeof(WorkerResult) * pa_scheduler::kWorkers; +constexpr size_t kSharedSidecarBytes = + sizeof(pa_scheduler::SharedTensorMapSidecar); + +bool CheckAcl(aclError error, const char *label) { + if (error == ACL_SUCCESS) { + return true; + } + std::fprintf( + stderr, "ACL error %d: %s\n", + static_cast(error), label + ); + return false; +} + +bool CheckRt(rtError_t error, const char *label) { + if (error == RT_ERROR_NONE) { + return true; + } + std::fprintf( + stderr, "RT error %d: %s\n", + static_cast(error), label + ); + return false; +} + +std::vector ReadBinary(const std::string &path) { + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) { + return {}; + } + const std::streamsize size = file.tellg(); + if (size <= 0) { + return {}; + } + std::vector data(static_cast(size)); + file.seekg(0, std::ios::beg); + if (!file.read(data.data(), size)) { + return {}; + } + return data; +} + +SchedulerState *MapSparseState() { + int flags = MAP_PRIVATE | MAP_ANONYMOUS; +#ifdef MAP_NORESERVE + flags |= MAP_NORESERVE; +#endif + void *memory = mmap( + nullptr, sizeof(SchedulerState), + PROT_READ | PROT_WRITE, flags, -1, 0 + ); + if (memory == MAP_FAILED) { + std::perror("mmap SchedulerState"); + return nullptr; + } + return ::new (memory) SchedulerState; +} + +bool ParseDevice(const char *text, int32_t *device) { + errno = 0; + char *end = nullptr; + const long parsed = std::strtol(text, &end, 10); + if (errno != 0 || end == text || *end != '\0' || + parsed < 0 || parsed > INT32_MAX) { + return false; + } + *device = static_cast(parsed); + return true; +} + +bool ParseScenario(const char *text, Scenario *scenario) { + const std::string name = text; + if (name == "history") { + *scenario = Scenario::SymbolHistory; + return true; + } + if (name == "reader-reclaim") { + *scenario = Scenario::ReaderReclaim; + return true; + } + return false; +} + +bool ParseDirection(const char *text, Direction *direction) { + const std::string name = text; + if (name == "aic-to-aiv") { + *direction = Direction::AicToAiv; + return true; + } + if (name == "aiv-to-aic") { + *direction = Direction::AivToAic; + return true; + } + return false; +} + +bool ParseReaderOrdering( + const char *text, Scenario scenario, + ReaderOrdering *ordering +) { + const std::string name = text; + if (scenario == Scenario::SymbolHistory) { + if (name != "na") { + return false; + } + *ordering = ReaderOrdering::NotApplicable; + return true; + } + if (name == "compiler-clobber") { + *ordering = ReaderOrdering::CompilerClobber; + return true; + } + if (name == "payload-dependency") { + *ordering = ReaderOrdering::PayloadDependency; + return true; + } + if (name == "dsb-all") { + *ordering = ReaderOrdering::DsbAll; + return true; + } + return false; +} + +void SetInsertTurnsAfterTasks( + SchedulerState *state, uint32_t completed_tasks +) { + for (uint32_t lane = 0; + lane < pa_scheduler::kSharedInsertTurnCapacity; + ++lane) { + pa_scheduler::SharedInsertTurnLine( + state->shared_map, lane + ).value = + pa_scheduler::SharedInsertTurnTokenAfterTasks( + completed_tasks, lane + ); + } +} + +bool InsertTurnsMatch( + const SchedulerState &state, uint32_t completed_tasks +) { + for (uint32_t lane = 0; + lane < pa_scheduler::kSharedInsertTurnCapacity; + ++lane) { + const int64_t observed = + lane == 0 + ? state.shared_map.committed_tasks.value + : state.shared_map + .insert_turn_extra[lane - 1U].value; + if (observed != + pa_scheduler::SharedInsertTurnTokenAfterTasks( + completed_tasks, lane + )) { + return false; + } + } + return true; +} + +void ResetTaskGate(SchedulerState *state, int32_t task_id) { + pa_scheduler::TaskCell &task = + state->tasks[static_cast(task_id)]; + task.flag = 0; + task.deps_prepared = -1; +} + +void ResetHistoryGates( + SchedulerState *state, const HistoryChain &chain +) { + ResetTaskGate(state, chain.writer_b); + ResetTaskGate(state, chain.writer_d); + ResetTaskGate(state, chain.writer_e); + ResetTaskGate(state, chain.reader_past_b_signal); + ResetTaskGate(state, chain.future_done_signal); +} + +void InitializeDescriptor( + TensorDesc *tensor, const HistoryChain &chain, + uint32_t slot +) { + std::memset(tensor, 0, sizeof(*tensor)); + tensor->buffer_addr = + 0x500000000ULL + + static_cast(chain.producer) * 0x100000ULL + + static_cast(slot) * 0x10000ULL; + tensor->buffer_size = 4096; + tensor->owner_task_id = + static_cast(chain.producer); + tensor->ndims = 1; + tensor->dtype = pa_scheduler::DataType::Float32; + tensor->is_contiguous = true; + tensor->shapes[0] = 1024; + tensor->strides[0] = 1; + tensor->extent_elem_cache = 1024; +} + +void InitializeHistoryState( + SchedulerState *state, const HistoryChain &chain +) { + // 约 1 GiB shadow 由匿名稀疏映射承载；只触碰实际 H2D/D2H 的前缀、 + // results 和 shared sidecar，不为门槛制造无意义的整块主机写流量。 + std::memset(state, 0, kStatePrefixBytes); + std::memset(state->results, 0, kResultBytes); + std::memset( + &state->shared_map, 0, kSharedSidecarBytes + ); + SetInsertTurnsAfterTasks(state, 0); + state->fatal.value = 0; + state->heap_window = pa_scheduler::kHeapWindow; + for (uint32_t worker = 0; + worker < pa_scheduler::kWorkers; ++worker) { + state->shared_map.reader_done[worker].value = -1; + } + + ResetHistoryGates(state, chain); + + pa_scheduler::SharedOutputCell &cell = + state->shared_map.shared_outputs[ + static_cast(chain.producer) + ]; + for (uint32_t slot = 0; slot < kSymbolCount; ++slot) { + InitializeDescriptor( + &cell.tensors[slot], chain, slot + ); + cell.published[slot].value = chain.producer; + cell.last_writer[slot].value = chain.producer; + } +} + +int32_t ReaderFillProducer(uint32_t cursor) { + if (cursor == 0) { + return 0; + } + if (cursor <= 32) { + return 1; + } + if (cursor <= 64) { + return 2; + } + if (cursor <= 96) { + return 3; + } + return 4; +} + +void InitializeReaderReclaimState( + SchedulerState *state, + const ReaderReclaimChain &chain +) { + // 该合成镜像严格对应一条生产可达 append 序列：task0 写 1 条， + // task1/2/3 各写 32 条，task4 写 31 条，总计填满 CAP=128。 + std::memset(state, 0, kStatePrefixBytes); + std::memset(state->results, 0, kResultBytes); + std::memset( + &state->shared_map, 0, kSharedSidecarBytes + ); + SetInsertTurnsAfterTasks( + state, kReaderReclaimWriterTask + ); + state->fatal.value = 0; + state->heap_window = kReaderReclaimHeapWindow; + state->shared_map.reclaim_upto.value = -1; + + for (uint32_t slot = 0; + slot < pa_scheduler::kMapCapacity; ++slot) { + state->shared_map.slots[slot].seq.value = -1; + } + for (uint32_t worker = 0; + worker < pa_scheduler::kWorkers; ++worker) { + state->shared_map.reader_done[worker].value = + kReaderReclaimClosedDone; + } + state->shared_map.reader_done[ + chain.reader_worker + ].value = kReaderReclaimInitialDone; + + const uint32_t bucket = + pa_scheduler::TensorMapHash(kReaderReclaimAddress); + state->shared_map.buckets[bucket].head.value = 0; + state->shared_map.buckets[bucket].tail.value = + pa_scheduler::kMapBucketCapacity; + for (uint32_t cursor = 0; + cursor < pa_scheduler::kMapBucketCapacity; + ++cursor) { + pa_scheduler::SharedRegionSlot &slot = + state->shared_map.slots[ + pa_scheduler::SharedTensorMapSlotIndex( + bucket, cursor + ) + ]; + slot.payload.value.buffer_addr = + kReaderReclaimAddress; + slot.payload.value.lo = + cursor == 0 + ? kReaderReclaimLo + : 0x10000ULL + + static_cast(cursor) * 64U; + slot.payload.value.hi = + cursor == 0 + ? kReaderReclaimHi + : slot.payload.value.lo + 16U; + slot.payload.value.producer = + ReaderFillProducer(cursor); + slot.payload.value.reserved = 0; + slot.seq.value = cursor; + } + + ResetTaskGate(state, kReaderReclaimAicToAiv.blocked_signal); + ResetTaskGate( + state, kReaderReclaimAicToAiv.reuse_done_signal + ); + ResetTaskGate(state, kReaderReclaimAivToAic.blocked_signal); + ResetTaskGate( + state, kReaderReclaimAivToAic.reuse_done_signal + ); + ResetHistoryGates(state, kAicToAiv); + ResetHistoryGates(state, kAivToAic); +} + +bool Expect(bool condition, const char *label) { + std::printf( + "[ASSERT] %-62s %s\n", + label, condition ? "PASS" : "FAIL" + ); + return condition; +} + +bool ResultPrefixIsZero(const WorkerResult &result) { + return result.submit_begin == 0 && + result.submit_end == 0 && + result.finish_cycle == 0 && + result.checksum == 0 && + result.submits == 0 && + result.claim_attempts == 0 && + result.claim_wins == 0 && + result.heap_guards == 0; +} + +bool HistoryMatches( + const SharedWriterHistoryCell &history, + const HistoryChain &chain, int32_t writer, + int32_t predecessor +) { + if (history.magic != + pa_scheduler::kSharedWriterHistoryMagic || + history.writer_task != writer || + history.count != kSymbolCount || + history.reserved != 0) { + return false; + } + for (uint32_t slot = 0; slot < kSymbolCount; ++slot) { + const SharedWriterHistoryRecord &record = + history.entries[slot]; + const uint32_t expected_key = + static_cast(chain.producer) * + pa_scheduler::kSharedOutputMaxPerTask + + slot + 1U; + if (record.symbol_key != expected_key || + record.previous_writer != predecessor) { + return false; + } + } + return true; +} + +bool ResultMatches( + const WorkerResult &result, uint64_t tag, + uint64_t status, uint64_t quantity, int32_t fanin +) { + return result.submit_begin == (kResultMagic | tag) && + result.submit_end == status && + result.finish_cycle == quantity && + static_cast(result.checksum) == fanin; +} + +bool ValidateHistory( + const SchedulerState &state, const HistoryChain &chain, + const HistoryChain &inactive +) { + bool passed = true; + passed &= Expect( + state.fatal.value == 0, + "device protocol leaves fatal clear" + ); + passed &= Expect( + state.tasks[chain.writer_b].deps_prepared == + chain.writer_b && + state.tasks[chain.writer_d].deps_prepared == + chain.writer_d && + state.tasks[chain.writer_e].deps_prepared == + chain.writer_e, + "B/D/E each publish their own writer-ready gate" + ); + passed &= Expect( + state.tasks[chain.reader_past_b_signal] + .deps_prepared == + chain.reader_past_b_signal && + state.tasks[chain.future_done_signal] + .deps_prepared == + chain.future_done_signal, + "reader-past-B and future-done ordering gates both close" + ); + passed &= Expect( + state.tasks[chain.writer_b].flag == 0 && + state.tasks[chain.writer_d].flag == 0 && + state.tasks[chain.writer_e].flag == 0, + "writer-ready remains independent from kernel completion" + ); + + const WorkerResult &writer_b = + state.results[chain.writer_b_worker]; + const WorkerResult &future = + state.results[chain.future_worker]; + const WorkerResult &reader = + state.results[chain.reader_worker]; + passed &= Expect( + ResultMatches( + writer_b, chain.result_tag | 1U, + kWriterBStatus, kSymbolCount, chain.producer + ), + "B publishes seven symbol CAS operations after A" + ); + passed &= Expect( + ResultMatches( + future, chain.result_tag | 2U, + kFutureWritersStatus, 2 * kSymbolCount, + chain.writer_d + ), + "D then E publish fourteen ordered symbol CAS operations" + ); + passed &= Expect( + ResultMatches( + reader, chain.result_tag | 3U, + kReaderStatus, 1, chain.writer_b + ), + "slow C resolves one fanin and returns B after E->D->B" + ); + passed &= Expect( + reader.submits == 0 && + reader.claim_attempts == 0, + "C really prewarms both future history cache lines as zero" + ); + + const pa_scheduler::SharedOutputCell &cell = + state.shared_map.shared_outputs[ + static_cast(chain.producer) + ]; + bool latest_ok = true; + for (uint32_t slot = 0; slot < kSymbolCount; ++slot) { + latest_ok &= + cell.published[slot].value == chain.producer && + cell.last_writer[slot].value == chain.writer_e; + } + passed &= Expect( + latest_ok, + "all seven symbol latest cells advance from A to E" + ); + passed &= Expect( + HistoryMatches( + state.shared_map.writer_history[chain.writer_b], + chain, chain.writer_b, chain.producer + ) && + HistoryMatches( + state.shared_map.writer_history[chain.writer_d], + chain, chain.writer_d, chain.writer_b + ) && + HistoryMatches( + state.shared_map.writer_history[chain.writer_e], + chain, chain.writer_e, chain.writer_d + ), + "B/D/E immutable histories preserve all seven predecessors" + ); + + const SharedWriterHistoryCell &inactive_b = + state.shared_map.writer_history[inactive.writer_b]; + const SharedWriterHistoryCell &inactive_d = + state.shared_map.writer_history[inactive.writer_d]; + const SharedWriterHistoryCell &inactive_e = + state.shared_map.writer_history[inactive.writer_e]; + passed &= Expect( + inactive_b.magic == 0 && inactive_b.count == 0 && + inactive_d.magic == 0 && inactive_d.count == 0 && + inactive_e.magic == 0 && inactive_e.count == 0, + "the opposite direction remains untouched in this launch" + ); + + bool ordinary_ring_untouched = true; + for (uint32_t bucket = 0; + bucket < pa_scheduler::kMapBuckets; ++bucket) { + ordinary_ring_untouched &= + state.shared_map.buckets[bucket].head.value == 0 && + state.shared_map.buckets[bucket].tail.value == 0; + } + for (uint32_t worker = 0; + worker < pa_scheduler::kWorkers; ++worker) { + ordinary_ring_untouched &= + state.shared_map.reader_done[worker].value == -1; + } + ordinary_ring_untouched &= + InsertTurnsMatch(state, 0); + passed &= Expect( + ordinary_ring_untouched, + "symbol history leaves ring/progress/inactive turns untouched" + ); + const bool reader_reclaim_untouched = + ResultPrefixIsZero( + state.results[ + kReaderReclaimAicToAiv.reader_worker + ] + ) && + ResultPrefixIsZero( + state.results[ + kReaderReclaimAicToAiv.reclaimer_worker + ] + ) && + ResultPrefixIsZero( + state.results[ + kReaderReclaimAivToAic.reader_worker + ] + ) && + ResultPrefixIsZero( + state.results[ + kReaderReclaimAivToAic.reclaimer_worker + ] + ) && + state.tasks[ + kReaderReclaimAicToAiv.blocked_signal + ].deps_prepared == 0 && + state.tasks[ + kReaderReclaimAicToAiv.reuse_done_signal + ].deps_prepared == 0 && + state.tasks[ + kReaderReclaimAivToAic.blocked_signal + ].deps_prepared == 0 && + state.tasks[ + kReaderReclaimAivToAic.reuse_done_signal + ].deps_prepared == 0; + passed &= Expect( + reader_reclaim_untouched, + "history leaves reader-reclaim result slots and gates untouched" + ); + return passed; +} + +bool HistoryChainUntouched( + const SchedulerState &state, const HistoryChain &chain +) { + return + ResultPrefixIsZero( + state.results[chain.writer_b_worker] + ) && + ResultPrefixIsZero( + state.results[chain.future_worker] + ) && + ResultPrefixIsZero( + state.results[chain.reader_worker] + ) && + state.tasks[chain.writer_b].deps_prepared == -1 && + state.tasks[chain.writer_d].deps_prepared == -1 && + state.tasks[chain.writer_e].deps_prepared == -1 && + state.tasks[ + chain.reader_past_b_signal + ].deps_prepared == -1 && + state.tasks[ + chain.future_done_signal + ].deps_prepared == -1 && + state.shared_map.writer_history[ + chain.writer_b + ].magic == 0 && + state.shared_map.writer_history[ + chain.writer_d + ].magic == 0 && + state.shared_map.writer_history[ + chain.writer_e + ].magic == 0; +} + +bool SharedSymbolStateUntouched(const SchedulerState &state) { + for (uint32_t task = 0; + task < pa_scheduler::kMaxTasks; ++task) { + const pa_scheduler::SharedOutputCell &cell = + state.shared_map.shared_outputs[task]; + for (uint32_t slot = 0; + slot < pa_scheduler::kSharedOutputMaxPerTask; + ++slot) { + if (cell.published[slot].value != 0 || + cell.last_writer[slot].value != 0) { + return false; + } + } + const pa_scheduler::SharedWriterHistoryCell &history = + state.shared_map.writer_history[task]; + if (history.magic != 0 || history.writer_task != 0 || + history.count != 0 || history.reserved != 0) { + return false; + } + } + return true; +} + +bool ReaderResultMatches( + const WorkerResult &result, + const ReaderReclaimChain &chain, + ReaderOrdering ordering +) { + return + result.submit_begin == + (kResultMagic | chain.result_tag | 1U) && + result.submit_end == + kReaderReclaimReaderStatus && + result.finish_cycle == kReaderReclaimAddress && + result.checksum == kReaderReclaimLo && + result.submits == kReaderReclaimHi && + static_cast(result.claim_attempts) == 0 && + result.claim_wins == 0 && + result.heap_guards == + (static_cast( + static_cast(ordering) + ) << 32 | + static_cast( + kReaderReclaimClosedDone + )); +} + +bool ReclaimerResultMatches( + const WorkerResult &result, + const ReaderReclaimChain &chain +) { + return + result.submit_begin == + (kResultMagic | chain.result_tag | 2U) && + result.submit_end == + kReaderReclaimReclaimerStatus && + static_cast(result.finish_cycle) == -1 && + result.checksum == + static_cast( + pa_scheduler::SharedAppendCheck::CapacityBlocked + ) && + static_cast(result.submits) == 0 && + static_cast(result.claim_attempts) == + pa_scheduler::kMapBucketCapacity && + static_cast(result.claim_wins) == 0 && + result.heap_guards == + (static_cast(1) << 32 | + (pa_scheduler::kMapBucketCapacity + 1U)); +} + +bool ValidateReaderReclaim( + const SchedulerState &state, + const ReaderReclaimChain &chain, + const ReaderReclaimChain &inactive, + ReaderOrdering ordering +) { + bool passed = true; + const uint32_t bucket = + pa_scheduler::TensorMapHash(kReaderReclaimAddress); + const uint32_t replacement_bucket = + pa_scheduler::TensorMapHash( + kReaderReclaimReplacementAddress + ); + passed &= Expect( + bucket == replacement_bucket, + "old and replacement addresses share the target bucket" + ); + passed &= Expect( + state.fatal.value == 0, + "reader-reclaim protocol leaves fatal clear" + ); + passed &= Expect( + ReaderResultMatches( + state.results[chain.reader_worker], + chain, ordering + ), + "reader exports the complete old snapshot after reuse" + ); + passed &= Expect( + ReclaimerResultMatches( + state.results[chain.reclaimer_worker], + chain + ), + "reclaimer records blocked then allowed state transitions" + ); + + bool progress_ok = true; + for (uint32_t worker = 0; + worker < kReaderReclaimActiveWorkers; ++worker) { + progress_ok &= + state.shared_map.reader_done[worker].value == + kReaderReclaimClosedDone; + } + passed &= Expect( + progress_ok, + "selected reader advances 1->2; other 95 remain at preset task 2" + ); + passed &= Expect( + state.shared_map.reclaim_upto.value == 0 && + InsertTurnsMatch( + state, + static_cast( + kReaderReclaimWriterTask + 1 + ) + ), + "reader frontier publishes reclaim/commit and preserves inactive turns" + ); + passed &= Expect( + state.shared_map.buckets[bucket].head.value == 1 && + state.shared_map.buckets[bucket].tail.value == + pa_scheduler::kMapBucketCapacity + 1, + "one safe retirement admits exactly one wrapped append" + ); + + const pa_scheduler::SharedRegionSlot &first_slot = + state.shared_map.slots[ + pa_scheduler::SharedTensorMapSlotIndex(bucket, 0) + ]; + passed &= Expect( + first_slot.seq.value == + pa_scheduler::kMapBucketCapacity && + first_slot.payload.value.buffer_addr == + kReaderReclaimReplacementAddress && + first_slot.payload.value.lo == + kReaderReclaimReplacementLo && + first_slot.payload.value.hi == + kReaderReclaimReplacementHi && + first_slot.payload.value.producer == + kReaderReclaimWriterTask && + first_slot.payload.value.reserved == 0, + "cursor 128 replaces every mutable cursor-0 payload field" + ); + + bool surviving_slots_ok = true; + uint32_t producer_counts[5] = {}; + for (uint32_t cursor = 1; + cursor < pa_scheduler::kMapBucketCapacity; + ++cursor) { + const pa_scheduler::SharedRegionSlot &slot = + state.shared_map.slots[ + pa_scheduler::SharedTensorMapSlotIndex( + bucket, cursor + ) + ]; + const int32_t producer = + ReaderFillProducer(cursor); + surviving_slots_ok &= + slot.seq.value == cursor && + slot.payload.value.buffer_addr == + kReaderReclaimAddress && + slot.payload.value.lo == + 0x10000ULL + + static_cast(cursor) * 64U && + slot.payload.value.hi == + slot.payload.value.lo + 16U && + slot.payload.value.producer == producer && + slot.payload.value.reserved == 0; + if (producer >= 0 && producer < 5) { + ++producer_counts[ + static_cast(producer) + ]; + } + } + surviving_slots_ok &= + producer_counts[1] == 32 && + producer_counts[2] == 32 && + producer_counts[3] == 32 && + producer_counts[4] == 31; + passed &= Expect( + surviving_slots_ok, + "cursor 1..127 preserve the reachable 32/32/32/31 history" + ); + + bool other_buckets_ok = true; + for (uint32_t index = 0; + index < pa_scheduler::kMapBuckets; ++index) { + if (index == bucket) { + continue; + } + other_buckets_ok &= + state.shared_map.buckets[index].head.value == 0 && + state.shared_map.buckets[index].tail.value == 0; + } + passed &= Expect( + other_buckets_ok, + "all non-target bucket controls remain untouched" + ); + + bool other_slots_ok = true; + const uint32_t target_slot_begin = + bucket * pa_scheduler::kMapBucketCapacity; + const uint32_t target_slot_end = + target_slot_begin + pa_scheduler::kMapBucketCapacity; + for (uint32_t index = 0; + index < pa_scheduler::kMapCapacity; ++index) { + if (index >= target_slot_begin && + index < target_slot_end) { + continue; + } + const pa_scheduler::SharedRegionSlot &slot = + state.shared_map.slots[index]; + other_slots_ok &= + slot.seq.value == -1 && + slot.payload.value.buffer_addr == 0 && + slot.payload.value.lo == 0 && + slot.payload.value.hi == 0 && + slot.payload.value.producer == 0 && + slot.payload.value.reserved == 0; + } + passed &= Expect( + other_slots_ok, + "all non-target physical ring slots remain untouched" + ); + + passed &= Expect( + state.tasks[chain.blocked_signal].deps_prepared == + chain.blocked_signal && + state.tasks[ + chain.reuse_done_signal + ].deps_prepared == + chain.reuse_done_signal && + state.tasks[ + inactive.blocked_signal + ].deps_prepared == -1 && + state.tasks[ + inactive.reuse_done_signal + ].deps_prepared == -1 && + ResultPrefixIsZero( + state.results[inactive.reader_worker] + ) && + ResultPrefixIsZero( + state.results[inactive.reclaimer_worker] + ), + "opposite reader-reclaim direction remains untouched" + ); + passed &= Expect( + HistoryChainUntouched(state, kAicToAiv) && + HistoryChainUntouched(state, kAivToAic) && + SharedSymbolStateUntouched(state), + "reader-reclaim leaves all symbol controls untouched" + ); + return passed; +} + +} // namespace + +int main(int argc, char **argv) { + if (argc != 6) { + std::fprintf( + stderr, + "Usage: %s " + "history|reader-reclaim " + "aic-to-aiv|aiv-to-aic " + "na|compiler-clobber|payload-dependency|dsb-all " + "\n", + argv[0] + ); + return EXIT_FAILURE; + } + + Scenario scenario{}; + if (!ParseScenario(argv[2], &scenario)) { + std::fprintf( + stderr, "Invalid shared protocol scenario: %s\n", + argv[2] + ); + return EXIT_FAILURE; + } + Direction direction{}; + if (!ParseDirection(argv[3], &direction)) { + std::fprintf( + stderr, "Invalid shared protocol direction: %s\n", + argv[3] + ); + return EXIT_FAILURE; + } + ReaderOrdering ordering{}; + if (!ParseReaderOrdering( + argv[4], scenario, &ordering + )) { + std::fprintf( + stderr, + "Invalid ordering '%s' for scenario '%s'.\n", + argv[4], argv[2] + ); + return EXIT_FAILURE; + } + int32_t device = 0; + if (!ParseDevice(argv[5], &device)) { + std::fprintf(stderr, "Invalid device id: %s\n", argv[5]); + return EXIT_FAILURE; + } + const HistoryChain &history_chain = + direction == Direction::AicToAiv + ? kAicToAiv + : kAivToAic; + const HistoryChain &inactive_history = + direction == Direction::AicToAiv + ? kAivToAic + : kAicToAiv; + const ReaderReclaimChain &reader_chain = + direction == Direction::AicToAiv + ? kReaderReclaimAicToAiv + : kReaderReclaimAivToAic; + const ReaderReclaimChain &inactive_reader = + direction == Direction::AicToAiv + ? kReaderReclaimAivToAic + : kReaderReclaimAicToAiv; + const std::vector binary = ReadBinary(argv[1]); + if (binary.empty()) { + std::fprintf( + stderr, "Cannot read mixed AICore ELF: %s\n", argv[1] + ); + return EXIT_FAILURE; + } + + SchedulerState *host_state = MapSparseState(); + if (host_state == nullptr) { + return EXIT_FAILURE; + } + if (scenario == Scenario::SymbolHistory) { + InitializeHistoryState( + host_state, history_chain + ); + } else { + InitializeReaderReclaimState( + host_state, reader_chain + ); + } + Control host_control{}; + host_control.magic = kControlMagic; + host_control.version = kControlVersion; + host_control.scenario = static_cast(scenario); + host_control.direction = static_cast(direction); + host_control.reader_ordering = + static_cast(ordering); + host_control.launch_nonce = + scenario == Scenario::SymbolHistory + ? static_cast( + history_chain.producer + ) << 32 | + static_cast( + history_chain.reader_c + ) + : static_cast( + reader_chain.reader_worker + ) << 32 | + reader_chain.reclaimer_worker; + + bool acl_initialized = false; + bool device_set = false; + aclrtStream stream = nullptr; + void *kernel_handle = nullptr; + bool registered_all = false; + void *device_state = nullptr; + void *device_control = nullptr; + bool execution_ok = false; + + do { + if (!CheckAcl(aclInit(nullptr), "aclInit")) { + break; + } + acl_initialized = true; + if (!CheckAcl(aclrtSetDevice(device), "aclrtSetDevice")) { + break; + } + device_set = true; + if (!CheckAcl( + aclrtCreateStream(&stream), "aclrtCreateStream" + )) { + break; + } + + rtDevBinary_t device_binary{ + RT_DEV_BINARY_MAGIC_ELF, 0, + binary.data(), binary.size() + }; + rtError_t load_error = + rtRegisterAllKernel(&device_binary, &kernel_handle); + if (load_error == RT_ERROR_NONE && + kernel_handle != nullptr) { + registered_all = true; + } else { + registered_all = false; + kernel_handle = nullptr; + load_error = rtBinaryLoadWithoutTilingKey( + binary.data(), binary.size(), &kernel_handle + ); + } + if (!CheckRt( + load_error, + "load shared protocol mixed AICore ELF" + ) || + kernel_handle == nullptr) { + break; + } + + if (!CheckAcl( + aclrtMalloc( + &device_state, sizeof(SchedulerState), + ACL_MEM_MALLOC_HUGE_FIRST + ), + "aclrtMalloc(shared protocol SchedulerState)" + ) || + !CheckAcl( + aclrtMalloc( + &device_control, sizeof(Control), + ACL_MEM_MALLOC_NORMAL_ONLY + ), + "aclrtMalloc(shared protocol control)" + )) { + break; + } + if ((reinterpret_cast(device_state) & 63U) != + 0 || + (reinterpret_cast(device_control) & 63U) != + 0) { + std::fprintf( + stderr, + "Shared protocol allocations must be 64-byte aligned: " + "state=%p control=%p\n", + device_state, device_control + ); + break; + } + + if (!CheckAcl( + aclrtMemcpy( + device_state, kStatePrefixBytes, + host_state, kStatePrefixBytes, + ACL_MEMCPY_HOST_TO_DEVICE + ), + "H2D shared protocol state prefix" + ) || + !CheckAcl( + aclrtMemcpy( + &static_cast( + device_state + )->results[0], + kResultBytes, host_state->results, + kResultBytes, ACL_MEMCPY_HOST_TO_DEVICE + ), + "H2D zero shared protocol results" + ) || + !CheckAcl( + aclrtMemcpy( + &static_cast( + device_state + )->shared_map, + kSharedSidecarBytes, + &host_state->shared_map, + kSharedSidecarBytes, + ACL_MEMCPY_HOST_TO_DEVICE + ), + "H2D shared protocol sidecar" + ) || + !CheckAcl( + aclrtMemcpy( + device_control, sizeof(Control), + &host_control, sizeof(Control), + ACL_MEMCPY_HOST_TO_DEVICE + ), + "H2D shared protocol control" + )) { + break; + } + + void *kernel_args[] = { + device_state, device_control + }; + rtArgsEx_t args_info{}; + args_info.args = kernel_args; + args_info.argsSize = sizeof(kernel_args); + rtTaskCfgInfo_t task_config{}; + if (!CheckRt( + rtKernelLaunchWithHandleV2( + kernel_handle, 0, + pa_scheduler::kAicWorkers, + &args_info, nullptr, stream, + &task_config + ), + "launch shared protocol mixed AICore kernel" + ) || + !CheckAcl( + aclrtSynchronizeStream(stream), + "synchronize shared protocol mixed AICore kernel" + )) { + break; + } + + if (!CheckAcl( + aclrtMemcpy( + host_state, kStatePrefixBytes, + device_state, kStatePrefixBytes, + ACL_MEMCPY_DEVICE_TO_HOST + ), + "D2H shared protocol state prefix" + ) || + !CheckAcl( + aclrtMemcpy( + host_state->results, kResultBytes, + &static_cast( + device_state + )->results[0], + kResultBytes, ACL_MEMCPY_DEVICE_TO_HOST + ), + "D2H shared protocol results" + ) || + !CheckAcl( + aclrtMemcpy( + &host_state->shared_map, + kSharedSidecarBytes, + &static_cast( + device_state + )->shared_map, + kSharedSidecarBytes, + ACL_MEMCPY_DEVICE_TO_HOST + ), + "D2H shared protocol sidecar" + )) { + break; + } + execution_ok = + scenario == Scenario::SymbolHistory + ? ValidateHistory( + *host_state, history_chain, + inactive_history + ) + : ValidateReaderReclaim( + *host_state, reader_chain, + inactive_reader, ordering + ); + } while (false); + + bool cleanup_ok = true; + if (device_control != nullptr) { + cleanup_ok &= + CheckAcl( + aclrtFree(device_control), + "aclrtFree(shared protocol control)" + ); + } + if (device_state != nullptr) { + cleanup_ok &= + CheckAcl( + aclrtFree(device_state), + "aclrtFree(shared protocol SchedulerState)" + ); + } + if (kernel_handle != nullptr) { + const rtError_t unload_error = registered_all + ? rtDevBinaryUnRegister(kernel_handle) + : rtBinaryUnLoad(kernel_handle); + cleanup_ok &= + CheckRt( + unload_error, + "unload shared protocol mixed AICore ELF" + ); + } + if (stream != nullptr) { + cleanup_ok &= + CheckAcl( + aclrtDestroyStream(stream), + "aclrtDestroyStream" + ); + } + if (device_set) { + cleanup_ok &= + CheckAcl( + aclrtResetDevice(device), "aclrtResetDevice" + ); + } + if (acl_initialized) { + cleanup_ok &= + CheckAcl(aclFinalize(), "aclFinalize"); + } + (void)munmap(host_state, sizeof(SchedulerState)); + + std::printf( + "[SHARED-PROTOCOL-LITMUS] scenario=%s direction=%s " + "ordering=%s device=%d semantic=%s cleanup=%s\n", + argv[2], argv[3], argv[4], device, + execution_ok ? "PASS" : "FAIL", + cleanup_ok ? "PASS" : "FAIL" + ); + return execution_ok && cleanup_ok + ? EXIT_SUCCESS + : EXIT_FAILURE; +} diff --git a/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_litmus_kernel.cpp b/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_litmus_kernel.cpp new file mode 100644 index 0000000000..b58500ca6d --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_litmus_kernel.cpp @@ -0,0 +1,905 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "cce_aicore_intrinsics.h" +#include +#include + +#define PA_DEVICE __aicore__ inline +#define PA_DEVICE_NOINLINE static __aicore__ __attribute__((noinline)) +#define PA_LOOP_NOUNROLL _Pragma("clang loop unroll(disable)") +#define PA_GM __gm__ +#include "../common/pa_scheduler_core.h" +#include "ccec_ops.h" +#include "shared_protocol_litmus_shared.h" + +namespace { + +using pa_scheduler_ccec::CcecOps; +using pa_scheduler::shared_protocol_litmus::Control; +using pa_scheduler::shared_protocol_litmus::Direction; +using pa_scheduler::shared_protocol_litmus::HistoryChain; +using pa_scheduler::shared_protocol_litmus::ReaderOrdering; +using pa_scheduler::shared_protocol_litmus::ReaderReclaimChain; +using pa_scheduler::shared_protocol_litmus::Scenario; +using pa_scheduler::shared_protocol_litmus::kAicToAiv; +using pa_scheduler::shared_protocol_litmus::kAivToAic; +using pa_scheduler::shared_protocol_litmus::kControlMagic; +using pa_scheduler::shared_protocol_litmus::kControlVersion; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimActiveWorkers; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimAddress; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimAicToAiv; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimAivToAic; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimClosedDone; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimHeapWindow; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimHi; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimInitialDone; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimLo; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimReaderStatus; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimReplacementAddress; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimReplacementHi; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimReplacementLo; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimReclaimerStatus; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimTask; +using pa_scheduler::shared_protocol_litmus::kReaderReclaimWriterTask; +using pa_scheduler::shared_protocol_litmus::kResultMagic; +using pa_scheduler::shared_protocol_litmus::kSymbolCount; + +static_assert( + offsetof(pa_scheduler::SharedWriterHistoryCell, entries) + + 6 * sizeof(pa_scheduler::SharedWriterHistoryRecord) == + 64, + "the seventh history record must begin on the second cache line" +); +static_assert( + kReaderReclaimActiveWorkers == pa_scheduler::kWorkers, + "reader-reclaim litmus must scan the complete active worker prefix" +); + +__aicore__ inline pa_scheduler::FdwicOutputRef +HistoryOutputRef(const HistoryChain &chain, uint32_t slot) { + return pa_scheduler::FdwicOutputRef{ + chain.producer, + static_cast(slot), + 0, 0, 0, 0 + }; +} + +__aicore__ inline void BuildHistoryWriterArgs( + const HistoryChain &chain, pa_scheduler::TaskArgs &args +) { + pa_scheduler::ConstructTaskArgs(args); + for (uint32_t slot = 0; slot < kSymbolCount; ++slot) { + pa_scheduler::AppendSharedOutputRef( + args, HistoryOutputRef(chain, slot), + pa_scheduler::TensorArgType::Inout + ); + } +} + +__aicore__ inline bool PrepareHistoryWriter( + __gm__ pa_scheduler::SchedulerState *state, + const HistoryChain &chain, int32_t task_id, + int32_t expected_predecessor, + pa_scheduler::LocalStats &stats +) { + pa_scheduler::TaskArgs args; + BuildHistoryWriterArgs(chain, args); + pa_scheduler::SubmitContext context{}; + context.task_id = task_id; + context.won = true; + const auto result = + pa_scheduler::PrepareSharedWriterIntentSet( + state, args, context, stats + ); + return result == + pa_scheduler::SharedWriterIntentResult::Published && + context.fanin_count == 1 && + context.fanin[0] == expected_predecessor && + CcecOps::Load( + &state->tasks[ + static_cast(task_id) + ].deps_prepared + ) == task_id; +} + +__aicore__ inline void PublishHistoryResult( + __gm__ pa_scheduler::SchedulerState *state, + uint32_t worker, uint64_t tag, uint64_t status, + uint64_t quantity, int32_t fanin, + uint64_t evidence0 = 0, uint64_t evidence1 = 0 +) { + __gm__ pa_scheduler::WorkerResult &result = + state->results[worker]; + result.submit_begin = kResultMagic | tag; + result.submit_end = status; + result.finish_cycle = quantity; + result.checksum = static_cast( + static_cast(fanin) + ); + result.submits = evidence0; + result.claim_attempts = evidence1; + CcecOps::FlushRegion(&result, 64); +} + +__aicore__ inline void CaptureReaderSnapshot( + __gm__ pa_scheduler::SharedTensorMapSidecar &map, + uint32_t bucket, uint64_t cursor, + pa_scheduler::SharedRegionValue &snapshot, + int64_t &seq_before, int64_t &seq_after +) { + // 与 SharedReadRegionSlot 使用相同的 seq/失效/五字段拷贝/seq 顺序， + // 但故意把全部判断延后到 reuse-done 之后。否则 CAS 的控制流会先消费 + // payload，三种 ordering 就无法形成有意义的差异。 + __gm__ pa_scheduler::SharedRegionSlot &slot = + map.slots[ + pa_scheduler::SharedTensorMapSlotIndex( + bucket, cursor + ) + ]; + seq_before = CcecOps::Load(&slot.seq.value); + CcecOps::InvalidateRegion( + &slot.payload, sizeof(slot.payload) + ); + snapshot.buffer_addr = + slot.payload.value.buffer_addr; + snapshot.lo = slot.payload.value.lo; + snapshot.hi = slot.payload.value.hi; + snapshot.producer = + slot.payload.value.producer; + snapshot.reserved = + slot.payload.value.reserved; + seq_after = CcecOps::Load(&slot.seq.value); +} + +PA_DEVICE_NOINLINE bool ValidateReaderSnapshotAfterReuse( + __gm__ pa_scheduler::SchedulerState *state, + int32_t expected_signal, + const pa_scheduler::SharedRegionValue &snapshot, + int64_t seq_before, int64_t seq_after +) { + if (state == nullptr || expected_signal < 0 || + expected_signal >= static_cast( + pa_scheduler::kMaxTasks + )) { + return false; + } + // validator 自己对同一 reuse gate 做第二次真实 atomic load。O3 IR + // 因而能在一个 noinline 函数内证明：先由 expected_signal 算出 + // TaskCell::deps_prepared，再比较动态 token，成功分支才读取旧快照。 + const int64_t reuse_token = CcecOps::Load( + &state->tasks[ + static_cast(expected_signal) + ].deps_prepared + ); + if (reuse_token != expected_signal) { + return false; + } + // 使用 bool 位与让五个字段都在 token 成功块中实际读取，避免短路把 + // reserved 单独拆到另一个 CFG 分支后增加自动 IR 判定歧义。 + const bool seq_ok = + (seq_before == 0) & (seq_after == 0); + const bool payload_ok = + (snapshot.buffer_addr == kReaderReclaimAddress) & + (snapshot.lo == kReaderReclaimLo) & + (snapshot.hi == kReaderReclaimHi) & + (snapshot.producer == 0) & + (snapshot.reserved == 0); + return seq_ok & payload_ok; +} + +PA_DEVICE_NOINLINE bool CloseReaderCompilerClobber( + __gm__ pa_scheduler::SharedTensorMapSidecar &map, + uint32_t worker +) { + // 只约束编译器，不生成设备访存屏障；作为最弱动态对照，不能被解释为 + // ordinary payload 已在设备侧完成。 + __asm__ volatile("" ::: "memory"); + return pa_scheduler::SharedAdvanceReaderDone( + map, worker, kReaderReclaimTask + ); +} + +PA_DEVICE_NOINLINE bool CloseReaderPayloadDependency( + __gm__ pa_scheduler::SharedTensorMapSidecar &map, + uint32_t worker, uint64_t buffer_addr, + uint64_t lo, uint64_t hi, int32_t producer, + uint32_t reserved +) { + // 直接把五个 capture 标量作为 noinline 实参，而不是先在调用者中压成 + // 一个 checksum。最终 O3 IR 因而能沿函数签名审计三组 64b 和两组 + // 32b leaf 是否全部到达下面的 CAS dependency。 + uint32_t checksum = 2166136261U; + checksum = + (checksum ^ static_cast(buffer_addr)) * + 16777619U; + checksum = + (checksum ^ static_cast( + buffer_addr >> 32 + )) * + 16777619U; + checksum = + (checksum ^ static_cast(lo)) * + 16777619U; + checksum = + (checksum ^ static_cast(lo >> 32)) * + 16777619U; + checksum = + (checksum ^ static_cast(hi)) * + 16777619U; + checksum = + (checksum ^ static_cast(hi >> 32)) * + 16777619U; + checksum = + (checksum ^ static_cast(producer)) * + 16777619U; + checksum = + (checksum ^ reserved) * 16777619U; + const uint32_t saved_checksum = checksum; + uint32_t opaque_checksum = checksum; + // MOV 精确保持运行时值，但 tied output 对优化器是不透明的新值。CAS + // 的 expected/desired 都叠加该差值：正常时差值为零；若不为零， + // expected 也会偏离当前 1，CAS 因而 fail-closed，不会写入错误 task。 + asm volatile( + "MOV %0, %0" + : "+l"(opaque_checksum) + : + : "memory" + ); + const uint32_t dependency = + opaque_checksum - saved_checksum; + const int64_t expected = + static_cast(kReaderReclaimInitialDone) + + static_cast(dependency); + const int64_t desired = + static_cast(kReaderReclaimClosedDone) + + static_cast(dependency); + const int64_t observed = CcecOps::CompareExchange( + &map.reader_done[worker].value, expected, desired + ); + return dependency == 0 && observed == expected; +} + +PA_DEVICE_NOINLINE bool CloseReaderDsbAll( + __gm__ pa_scheduler::SharedTensorMapSidecar &map, + uint32_t worker +) { + // 本机 CANN 头把 mem_dsb_t(0) 定义为 DSB_ALL。两侧 compiler clobber + // 防止 CCEC 把普通 payload load 穿过设备完成屏障。 + __asm__ volatile("" ::: "memory"); + dsb((mem_dsb_t)0); + __asm__ volatile("" ::: "memory"); + return pa_scheduler::SharedAdvanceReaderDone( + map, worker, kReaderReclaimTask + ); +} + +__aicore__ inline bool CloseReaderAfterSnapshot( + __gm__ pa_scheduler::SharedTensorMapSidecar &map, + uint32_t worker, ReaderOrdering ordering, + const pa_scheduler::SharedRegionValue &snapshot +) { + if (ordering == ReaderOrdering::CompilerClobber) { + return CloseReaderCompilerClobber(map, worker); + } + if (ordering == ReaderOrdering::PayloadDependency) { + return CloseReaderPayloadDependency( + map, worker, snapshot.buffer_addr, + snapshot.lo, snapshot.hi, + snapshot.producer, snapshot.reserved + ); + } + if (ordering == ReaderOrdering::DsbAll) { + return CloseReaderDsbAll(map, worker); + } + return false; +} + +__aicore__ inline bool WaitForReaderClosed( + __gm__ pa_scheduler::SchedulerState *state, + uint32_t reader_worker, pa_scheduler::LocalStats &stats +) { + const uint64_t begin = CcecOps::Now(); + uint32_t polls = 0; + while (true) { + const int64_t observed = CcecOps::Load( + &state->shared_map.reader_done[reader_worker].value + ); + if (observed == kReaderReclaimClosedDone) { + return true; + } + if (observed != kReaderReclaimInitialDone) { + pa_scheduler::SetFatal( + state, stats, kReaderReclaimTask + ); + return false; + } + if (pa_scheduler::WatchdogExpired( + state, stats, begin, polls + )) { + return false; + } + } +} + +__aicore__ inline void PublishReaderResult( + __gm__ pa_scheduler::SchedulerState *state, + const ReaderReclaimChain &chain, uint64_t status, + const pa_scheduler::SharedRegionValue &snapshot, + ReaderOrdering ordering, int64_t closed_done +) { + __gm__ pa_scheduler::WorkerResult &result = + state->results[chain.reader_worker]; + result.submit_begin = + kResultMagic | chain.result_tag | 1U; + result.submit_end = status; + result.finish_cycle = snapshot.buffer_addr; + result.checksum = snapshot.lo; + result.submits = snapshot.hi; + result.claim_attempts = static_cast( + static_cast(snapshot.producer) + ); + result.claim_wins = snapshot.reserved; + result.heap_guards = + static_cast( + static_cast(ordering) + ) << 32 | + static_cast(closed_done); + CcecOps::FlushRegion(&result, 64); +} + +__aicore__ inline void PublishReclaimerResult( + __gm__ pa_scheduler::SchedulerState *state, + const ReaderReclaimChain &chain, uint64_t status, + int64_t blocked_reclaim, + pa_scheduler::SharedAppendCheck blocked_check, + int64_t blocked_head, int64_t blocked_tail, + int64_t allowed_reclaim, + int64_t final_head, int64_t final_tail +) { + __gm__ pa_scheduler::WorkerResult &result = + state->results[chain.reclaimer_worker]; + result.submit_begin = + kResultMagic | chain.result_tag | 2U; + result.submit_end = status; + result.finish_cycle = + static_cast(blocked_reclaim); + result.checksum = + static_cast(blocked_check); + result.submits = static_cast(blocked_head); + result.claim_attempts = + static_cast(blocked_tail); + result.claim_wins = + static_cast(allowed_reclaim); + result.heap_guards = + static_cast( + static_cast(final_head) + ) << 32 | + static_cast(final_tail); + CcecOps::FlushRegion(&result, 64); +} + +__aicore__ inline void RunWriterB( + __gm__ pa_scheduler::SchedulerState *state, + const HistoryChain &chain +) { + pa_scheduler::LocalStats stats{}; + const bool prepared = PrepareHistoryWriter( + state, chain, chain.writer_b, chain.producer, stats + ); + PublishHistoryResult( + state, chain.writer_b_worker, chain.result_tag | 1U, + prepared ? 1U : 0U, + stats.result.shared_symbol_inout_commits, + chain.producer + ); +} + +__aicore__ inline void RunFutureWriters( + __gm__ pa_scheduler::SchedulerState *state, + const HistoryChain &chain +) { + pa_scheduler::LocalStats stats{}; + const bool reader_ready = + pa_scheduler::WaitForSharedWriterReady( + state, chain.reader_past_b_signal, stats + ); + const bool d_prepared = + reader_ready && + PrepareHistoryWriter( + state, chain, chain.writer_d, + chain.writer_b, stats + ); + const bool e_prepared = + d_prepared && + PrepareHistoryWriter( + state, chain, chain.writer_e, + chain.writer_d, stats + ); + const bool signalled = + e_prepared && + pa_scheduler::PublishSharedWriterReady( + state, chain.future_done_signal + ); + if (!signalled) { + pa_scheduler::SetFatal( + state, stats, chain.writer_e + ); + } + const uint64_t status = + (reader_ready ? 1U : 0U) | + (d_prepared ? 2U : 0U) | + (e_prepared ? 4U : 0U) | + (signalled ? 8U : 0U); + PublishHistoryResult( + state, chain.future_worker, chain.result_tag | 2U, + status, stats.result.shared_symbol_inout_commits, + chain.writer_d + ); +} + +__aicore__ inline void RunSlowReader( + __gm__ pa_scheduler::SchedulerState *state, + const HistoryChain &chain +) { + pa_scheduler::LocalStats stats{}; + const bool b_ready = + pa_scheduler::WaitForSharedWriterReady( + state, chain.writer_b, stats + ); + // C 通过 B gate 后，先把未来 D/E history 的 header 与第七条 record + // 所在第二条 cache line 都以普通 GM load 预热成 host 初始化的零。 + // 预热值参与是否发布下一道 gate，保证编译器和 scalar 都必须先消费 + // 这些 load；随后 D/E 才能写回同一地址。 + __gm__ volatile uint64_t *future_d_words = + reinterpret_cast<__gm__ volatile uint64_t *>( + &state->shared_map.writer_history[ + static_cast(chain.writer_d) + ] + ); + __gm__ volatile uint64_t *future_e_words = + reinterpret_cast<__gm__ volatile uint64_t *>( + &state->shared_map.writer_history[ + static_cast(chain.writer_e) + ] + ); + uint64_t prewarm_first_line = UINT64_MAX; + uint64_t prewarm_second_line = UINT64_MAX; + if (b_ready) { + prewarm_first_line = + future_d_words[0] | future_e_words[0]; + prewarm_second_line = + future_d_words[8] | future_e_words[8]; + } + const bool prewarm_zero = + b_ready && prewarm_first_line == 0 && + prewarm_second_line == 0; + const bool passed_signal = + prewarm_zero && + pa_scheduler::PublishSharedWriterReady( + state, chain.reader_past_b_signal + ); + if (!passed_signal) { + pa_scheduler::SetFatal( + state, stats, chain.reader_c + ); + } + const bool future_ready = + passed_signal && + pa_scheduler::WaitForSharedWriterReady( + state, chain.future_done_signal, stats + ); + + pa_scheduler::TaskArgs args; + pa_scheduler::ConstructTaskArgs(args); + pa_scheduler::AppendSharedOutputRef( + args, HistoryOutputRef(chain, kSymbolCount - 1), + pa_scheduler::TensorArgType::Input + ); + int32_t fanin[pa_scheduler::kMaxFanin] = {}; + bool protocol_ok = false; + uint32_t ordinary_lookups = UINT32_MAX; + uint32_t fanin_count = 0; + if (future_ready) { + fanin_count = + pa_scheduler::CollectSharedFanin< + CcecOps, false, true + >( + state->shared_map, args, chain.reader_c, + static_cast(state->heap_window), + stats, fanin, protocol_ok, ordinary_lookups, + &state->fatal.value + ); + } + const bool resolved = + future_ready && protocol_ok && + ordinary_lookups == 0 && fanin_count == 1 && + fanin[0] == chain.writer_b; + if (!resolved) { + pa_scheduler::SetFatal( + state, stats, chain.reader_c + ); + } + const uint64_t status = + (b_ready ? 1U : 0U) | + (prewarm_zero ? 2U : 0U) | + (passed_signal ? 4U : 0U) | + (future_ready ? 8U : 0U) | + (protocol_ok ? 16U : 0U) | + (resolved ? 32U : 0U); + PublishHistoryResult( + state, chain.reader_worker, chain.result_tag | 3U, + status, fanin_count, + fanin_count == 0 ? -1 : fanin[0], + prewarm_first_line, prewarm_second_line + ); +} + +__aicore__ inline void RunHistoryParticipant( + __gm__ pa_scheduler::SchedulerState *state, + __gm__ const Control *control, uint32_t worker +) { + const HistoryChain *chain = nullptr; + const Direction direction = + static_cast(control->direction); + if (direction == Direction::AicToAiv) { + chain = &kAicToAiv; + } else if (direction == Direction::AivToAic) { + chain = &kAivToAic; + } else { + pa_scheduler::LocalStats stats{}; + pa_scheduler::SetFatal(state, stats, -1); + return; + } + + if (worker == chain->writer_b_worker) { + RunWriterB(state, *chain); + } else if (worker == chain->future_worker) { + RunFutureWriters(state, *chain); + } else if (worker == chain->reader_worker) { + RunSlowReader(state, *chain); + } +} + +__aicore__ inline void RunReaderReclaimReader( + __gm__ pa_scheduler::SchedulerState *state, + const ReaderReclaimChain &chain, + ReaderOrdering ordering +) { + pa_scheduler::LocalStats stats{}; + const bool blocked_ready = + pa_scheduler::WaitForSharedWriterReady( + state, chain.blocked_signal, stats + ); + pa_scheduler::SharedRegionValue snapshot{}; + const uint32_t bucket = + pa_scheduler::TensorMapHash(kReaderReclaimAddress); + int64_t seq_before = -2; + int64_t seq_after = -2; + if (blocked_ready) { + CaptureReaderSnapshot( + state->shared_map, bucket, 0, snapshot, + seq_before, seq_after + ); + } + const bool close_ok = + blocked_ready && + CloseReaderAfterSnapshot( + state->shared_map, chain.reader_worker, + ordering, snapshot + ); + const int64_t closed_done = CcecOps::Load( + &state->shared_map + .reader_done[chain.reader_worker].value + ); + // CAS 发布后先等 reclaimer 完成 cursor-0 复用，再消费并导出旧快照； + // 这给缺失 read->publish 顺序的对照留下确定性的覆盖窗口。 + const bool reuse_waited = + close_ok && + pa_scheduler::WaitForSharedWriterReady( + state, chain.reuse_done_signal, stats + ); + const bool reuse_done = reuse_waited; + const bool snapshot_ok = + reuse_done && + ValidateReaderSnapshotAfterReuse( + state, chain.reuse_done_signal, snapshot, + seq_before, seq_after + ); + const uint64_t status = + (blocked_ready ? 1U : 0U) | + (close_ok ? 2U : 0U) | + (closed_done == kReaderReclaimClosedDone + ? 4U + : 0U) | + (reuse_done ? 8U : 0U) | + (snapshot_ok ? 16U : 0U); + if (status != kReaderReclaimReaderStatus) { + pa_scheduler::SetFatal( + state, stats, kReaderReclaimTask + ); + } + PublishReaderResult( + state, chain, status, snapshot, ordering, + closed_done + ); +} + +__aicore__ inline void RunReaderReclaimReclaimer( + __gm__ pa_scheduler::SchedulerState *state, + const ReaderReclaimChain &chain +) { + pa_scheduler::LocalStats stats{}; + const uint32_t bucket = + pa_scheduler::TensorMapHash(kReaderReclaimAddress); + pa_scheduler::SharedRegionValue replacement{}; + replacement.buffer_addr = + kReaderReclaimReplacementAddress; + replacement.lo = kReaderReclaimReplacementLo; + replacement.hi = kReaderReclaimReplacementHi; + replacement.producer = kReaderReclaimWriterTask; + replacement.reserved = 0; + + int64_t blocked_reclaim = -2; + const bool blocked_refresh = + pa_scheduler::SharedRefreshReaderReclaimForTask< + CcecOps + >( + state->shared_map, + kReaderReclaimWriterTask, + kReaderReclaimActiveWorkers, + kReaderReclaimHeapWindow, + blocked_reclaim + ); + const pa_scheduler::SharedAppendCheck blocked_check = + blocked_refresh + ? pa_scheduler::SharedCheckTaskAppend( + state->shared_map, &replacement, 1, + blocked_reclaim + ) + : pa_scheduler::SharedAppendCheck::ProtocolError; + const int64_t blocked_head = CcecOps::Load( + &state->shared_map.buckets[bucket].head.value + ); + const int64_t blocked_tail = CcecOps::Load( + &state->shared_map.buckets[bucket].tail.value + ); + const int64_t blocked_commit = CcecOps::Load( + &state->shared_map.committed_tasks.value + ); + __gm__ pa_scheduler::SharedRegionSlot &first_slot = + state->shared_map.slots[ + pa_scheduler::SharedTensorMapSlotIndex(bucket, 0) + ]; + const int64_t blocked_seq = + CcecOps::Load(&first_slot.seq.value); + CcecOps::InvalidateRegion( + &first_slot.payload, sizeof(first_slot.payload) + ); + const uint64_t blocked_address = + first_slot.payload.value.buffer_addr; + const uint64_t blocked_lo = + first_slot.payload.value.lo; + const uint64_t blocked_hi = + first_slot.payload.value.hi; + const int32_t blocked_producer = + first_slot.payload.value.producer; + const uint32_t blocked_reserved = + first_slot.payload.value.reserved; + const bool blocked_ok = + blocked_refresh && blocked_reclaim == -1 && + blocked_check == + pa_scheduler::SharedAppendCheck::CapacityBlocked && + blocked_head == 0 && + blocked_tail == pa_scheduler::kMapBucketCapacity && + blocked_commit == kReaderReclaimWriterTask && + blocked_seq == 0 && + blocked_address == kReaderReclaimAddress && + blocked_lo == kReaderReclaimLo && + blocked_hi == kReaderReclaimHi && + blocked_producer == 0 && + blocked_reserved == 0; + if (!blocked_ok) { + pa_scheduler::SetFatal( + state, stats, kReaderReclaimWriterTask + ); + } + + // gate 只通知 reader“阻塞态已被完整取证”；随后 reclaimer 不再等待 + // 其他门值，而是直接轮询真实 reader_done，证明跨核前沿本身可见。 + const bool blocked_signalled = + blocked_ok && + pa_scheduler::PublishSharedWriterReady( + state, chain.blocked_signal + ); + const bool reader_observed = + blocked_signalled && + WaitForReaderClosed( + state, chain.reader_worker, stats + ); + + int64_t allowed_reclaim = -2; + const bool allowed_refresh = + reader_observed && + pa_scheduler::SharedRefreshReaderReclaimForTask< + CcecOps + >( + state->shared_map, + kReaderReclaimWriterTask, + kReaderReclaimActiveWorkers, + kReaderReclaimHeapWindow, + allowed_reclaim + ); + const pa_scheduler::SharedAppendCheck allowed_check = + allowed_refresh + ? pa_scheduler::SharedCheckTaskAppend( + state->shared_map, &replacement, 1, + allowed_reclaim + ) + : pa_scheduler::SharedAppendCheck::ProtocolError; + const bool appended = + allowed_check == + pa_scheduler::SharedAppendCheck::Ready && + pa_scheduler::SharedAppendPreparedTask( + state->shared_map, &replacement, 1 + ); + const bool committed = + appended && + pa_scheduler::SharedPublishTaskCommit( + state->shared_map, + kReaderReclaimWriterTask + ); + // reuse-done 只能表示完整 append+commit 已返回成功。短路依赖避免失败 + // 路径先放行 reader，也让成功路径保留从 payload Flush/seq/tail 返回值， + // 经 commit 返回值到 gate CAS 的控制链；不能把单独 gate 冒充成发布证据。 + const bool reuse_signalled = + committed && + pa_scheduler::PublishSharedWriterReady( + state, chain.reuse_done_signal + ); + const int64_t final_head = CcecOps::Load( + &state->shared_map.buckets[bucket].head.value + ); + const int64_t final_tail = CcecOps::Load( + &state->shared_map.buckets[bucket].tail.value + ); + const uint64_t status = + (blocked_refresh ? 1U : 0U) | + (blocked_ok ? 2U : 0U) | + (blocked_signalled ? 4U : 0U) | + (reader_observed ? 8U : 0U) | + (allowed_refresh && allowed_reclaim == 0 + ? 16U + : 0U) | + (appended ? 32U : 0U) | + (committed ? 64U : 0U) | + (reuse_signalled ? 128U : 0U); + if (status != kReaderReclaimReclaimerStatus) { + pa_scheduler::SetFatal( + state, stats, kReaderReclaimWriterTask + ); + } + PublishReclaimerResult( + state, chain, status, blocked_reclaim, + blocked_check, blocked_head, blocked_tail, + allowed_reclaim, final_head, final_tail + ); +} + +__aicore__ inline void RunReaderReclaimParticipant( + __gm__ pa_scheduler::SchedulerState *state, + __gm__ const Control *control, uint32_t worker +) { + const ReaderReclaimChain *chain = nullptr; + const Direction direction = + static_cast(control->direction); + if (direction == Direction::AicToAiv) { + chain = &kReaderReclaimAicToAiv; + } else if (direction == Direction::AivToAic) { + chain = &kReaderReclaimAivToAic; + } else { + pa_scheduler::LocalStats stats{}; + pa_scheduler::SetFatal(state, stats, -1); + return; + } + const ReaderOrdering ordering = + static_cast( + control->reader_ordering + ); + if (ordering != ReaderOrdering::CompilerClobber && + ordering != ReaderOrdering::PayloadDependency && + ordering != ReaderOrdering::DsbAll) { + pa_scheduler::LocalStats stats{}; + pa_scheduler::SetFatal(state, stats, -1); + return; + } + if (worker == chain->reader_worker) { + RunReaderReclaimReader( + state, *chain, ordering + ); + } else if (worker == chain->reclaimer_worker) { + RunReaderReclaimReclaimer(state, *chain); + } +} + +__aicore__ inline void RunSharedProtocolParticipant( + __gm__ pa_scheduler::SchedulerState *state, + __gm__ const Control *control, uint32_t worker +) { + CcecOps::InvalidateRegion(control, sizeof(Control)); + if (control->magic != kControlMagic || + control->version != kControlVersion) { + pa_scheduler::LocalStats stats{}; + pa_scheduler::SetFatal(state, stats, -1); + return; + } + const Scenario scenario = + static_cast(control->scenario); + if (scenario == Scenario::SymbolHistory) { + if (control->reader_ordering != + static_cast( + ReaderOrdering::NotApplicable + )) { + pa_scheduler::LocalStats stats{}; + pa_scheduler::SetFatal( + state, stats, -1 + ); + return; + } + RunHistoryParticipant(state, control, worker); + return; + } + if (scenario == Scenario::ReaderReclaim) { + RunReaderReclaimParticipant( + state, control, worker + ); + return; + } + pa_scheduler::LocalStats stats{}; + pa_scheduler::SetFatal(state, stats, -1); +} + +} // namespace + +#if defined(PA_BUILD_AIC) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aic, 1, 2); + +extern "C" __global__ __aicore__ void +pa_scheduler_0_mix_aic( + __gm__ pa_scheduler::SchedulerState *state, + __gm__ const Control *control +) { + RunSharedProtocolParticipant( + state, control, + static_cast(get_block_idx()) + ); +} +#elif defined(PA_BUILD_AIV) +PTO_SYNCALL_MIX_AIC_KERNEL_META(pa_scheduler_0_mix_aiv, 1, 2); + +extern "C" __global__ __aicore__ void +pa_scheduler_0_mix_aiv( + __gm__ pa_scheduler::SchedulerState *state, + __gm__ const Control *control +) { + const uint32_t vector_id = + static_cast( + get_block_idx() * get_subblockdim() + + get_subblockid() + ); + RunSharedProtocolParticipant( + state, control, + pa_scheduler::kAicWorkers + vector_id + ); +} +#else +#error "Compile with PA_BUILD_AIC or PA_BUILD_AIV" +#endif diff --git a/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_litmus_shared.h b/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_litmus_shared.h new file mode 100644 index 0000000000..211ba5d211 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/ccec/shared_protocol_litmus_shared.h @@ -0,0 +1,131 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ +#ifndef TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_SHARED_PROTOCOL_LITMUS_SHARED_H +#define TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_SHARED_PROTOCOL_LITMUS_SHARED_H + +#include + +namespace pa_scheduler::shared_protocol_litmus { + +constexpr uint32_t kControlMagic = 0x5350524CU; +constexpr uint32_t kControlVersion = 3; +constexpr uint32_t kSharedAbiGeneration = 11; +constexpr uint32_t kSymbolCount = 7; +constexpr uint64_t kResultMagic = 0x484953544F525900ULL; +static_assert( + pa_scheduler::kBuildIdentityAbiGeneration == + kSharedAbiGeneration, + "shared protocol litmus manifest ABI must follow the shared build identity" +); + +enum class Scenario : uint32_t { + SymbolHistory = 1, + ReaderReclaim = 2, +}; + +enum class Direction : uint32_t { + AicToAiv = 1, + AivToAic = 2, +}; + +enum class ReaderOrdering : uint32_t { + NotApplicable = 0, + CompilerClobber = 1, + PayloadDependency = 2, + DsbAll = 3, +}; + +// 单次 launch 只选择一个场景和一个方向。后续场景共享同一 mixed ELF， +// 但各自独立初始化和断言，避免不同协议同时执行后互相掩盖故障。 +// control 独占一条 GM cache line；所有 worker 在读取前显式失效该行。 +struct alignas(64) Control { + uint32_t magic; + uint32_t version; + uint32_t scenario; + uint32_t direction; + uint32_t reader_ordering; + uint32_t reserved0; + uint64_t launch_nonce; + uint64_t reserved[4]; +}; +static_assert( + sizeof(Control) == 64, + "shared protocol litmus control must occupy one cache line" +); + +struct HistoryChain { + int32_t producer; + int32_t writer_b; + int32_t reader_c; + int32_t writer_d; + int32_t writer_e; + int32_t reader_past_b_signal; + int32_t future_done_signal; + uint32_t writer_b_worker; + uint32_t future_worker; + uint32_t reader_worker; + uint64_t result_tag; +}; + +// AIC writer 使用 block0/block1，AIV reader 使用物理 block4 的第一个 +// vector 子核。三者不位于同一 mixed block，排除块内偶然共享状态。 +constexpr HistoryChain kAicToAiv{ + 10, 20, 30, 40, 50, 60, 61, + 0, 1, 40, 0x10 +}; + +// 反向使用 AIV block0/sub1、AIV block1/sub0 和 AIC block2，同样跨越 +// 不同物理 mixed block。 +constexpr HistoryChain kAivToAic{ + 110, 120, 130, 140, 150, 160, 161, + 33, 34, 2, 0x20 +}; + +constexpr uint64_t kWriterBStatus = 1; +constexpr uint64_t kFutureWritersStatus = 0x0F; +constexpr uint64_t kReaderStatus = 0x3F; + +struct ReaderReclaimChain { + uint32_t reader_worker; + uint32_t reclaimer_worker; + int32_t blocked_signal; + int32_t reuse_done_signal; + uint64_t result_tag; +}; + +// reader/reclaimer 与 history 场景使用互不重叠的 worker、result 和 task gate， +// 使 host 能反向断言未选场景完全没有执行。 +constexpr ReaderReclaimChain kReaderReclaimAicToAiv{ + 3, 42, 200, 201, 0x40 +}; +constexpr ReaderReclaimChain kReaderReclaimAivToAic{ + 35, 4, 210, 211, 0x80 +}; + +constexpr uint32_t kReaderReclaimActiveWorkers = 96; +constexpr int32_t kReaderReclaimHeapWindow = 2; +constexpr int32_t kReaderReclaimTask = 2; +constexpr int32_t kReaderReclaimWriterTask = 5; +constexpr int32_t kReaderReclaimInitialDone = 1; +constexpr int32_t kReaderReclaimClosedDone = 2; +constexpr uint64_t kReaderReclaimAddress = 0x700000000ULL; +constexpr uint64_t kReaderReclaimLo = 0; +constexpr uint64_t kReaderReclaimHi = 8; +constexpr uint64_t kReaderReclaimReplacementAddress = + 0x7000014C0ULL; +constexpr uint64_t kReaderReclaimReplacementLo = 4096; +constexpr uint64_t kReaderReclaimReplacementHi = 4128; +constexpr uint64_t kReaderReclaimReaderStatus = 0x1F; +constexpr uint64_t kReaderReclaimReclaimerStatus = 0xFF; + +} // namespace pa_scheduler::shared_protocol_litmus + +#endif // TESTS_ATOMIC_PROBE_PA_SCHEDULER_CCEC_SHARED_PROTOCOL_LITMUS_SHARED_H diff --git a/tests/atomic_probe/pa_scheduler/common/host_support.h b/tests/atomic_probe/pa_scheduler/common/host_support.h new file mode 100644 index 0000000000..581aef4bdf --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/common/host_support.h @@ -0,0 +1,5431 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_HOST_SUPPORT_H +#define PA_SCHEDULER_COMMON_HOST_SUPPORT_H + +#include "pa_model.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pa_scheduler::host { + +// shared host oracle 与 private 固定五 task 校验必须在预处理阶段彻底分叉。 +// private 的 #else 有意保留基线 token 形状，避免同一翻译单元内的 shared +// AST 改动触发 GCC IPA/内联漂移，破坏两种 TensorMap 的严格产物可比性。 +#if PTO_FDWIC_SHARED_MAP +constexpr uint32_t kHostPaBlockSize = 128; +constexpr uint32_t kHostPaBlocksPerRequest = 64; + +// 这是 host oracle 自己采用的真实 PA 上限。故意不 include/call +// pa_frontend.h::BuildSharedPaBatchPlan：host 必须从最终写入 +// SchedulerState.context_lens 的输入独立重算计划，才能发现 device plan +// 公式、累计 batch_start 或 task 元数据出错。 +constexpr uint32_t kHostSharedPaMaxContextLength = + kSharedPaMaxBlockGroups * kHostPaBlocksPerRequest * + kHostPaBlockSize; +#endif + +// 三种后端共用同一套命令行配置，保证 CPU 语义回归与 A5 上板使用完全相同的工作量。 +struct Options { + std::string kernel_path; + std::string swimlane_json; +#if PTO_FDWIC_SHARED_MAP + // 空向量表示生产默认值 8192；一个值广播到全部 batch，多个值必须 + // 与 --batches 精确等长。该开关只服务 standalone shared 语义测试， + // 不改变默认性能工作量。 + std::vector shared_context_lens; +#endif + uint32_t device = 0; + uint32_t batches = kDefaultBatches; + uint32_t runs = 5; + NopCounts nops{kDefaultQkNops, kDefaultSfNops, kDefaultPvNops, kDefaultUpNops}; + FinalBarrierShape final_barrier_shape = FinalBarrierShape::TwoLevel16; + bool profile_phases = false; + bool trace_enabled = true; + // CCEC full-swimlane 已在编译期把普通阶段、Atomic 与 DCCI 固定为同一 + // 观察合同；host 默认必须与 device ELF 一致，否则普通 run/smoke + // 会在调度入口因 atomics_enabled=false fail-closed。trace-free、 + // submit-pmu、perf-clock 和 CPU 主程序的该宏均为 0。 + bool trace_atomics = PA_BUILD_ATOMIC_SWIMLANE != 0; + bool analyze_swimlane = false; +}; + +enum class ParseStatus { + Ok, + Help, + Error, +}; + +inline bool ParseUint(const char *raw, uint32_t minimum, uint32_t maximum, uint32_t *value) { + // 要求整串都能被 strtoul 解析且结果落在给定范围内，拒绝尾随字符和溢出值， + // 避免参数被部分解析后悄悄改变工作量。 + errno = 0; + char *end = nullptr; + const unsigned long parsed = std::strtoul(raw, &end, 10); + if (errno != 0 || end == raw || *end != '\0' || parsed < minimum || parsed > maximum) { + return false; + } + *value = static_cast(parsed); + return true; +} + +inline bool ParseNopCounts(const char *raw, NopCounts *counts) { + // 四类 kernel 的 NOP 数必须一次性完整给出，顺序固定为 QK、SF、PV、UP。 + unsigned int qk = 0; + unsigned int sf = 0; + unsigned int pv = 0; + unsigned int up = 0; + char tail = '\0'; + if (std::sscanf(raw, "%u,%u,%u,%u%c", &qk, &sf, &pv, &up, &tail) != 4) { + return false; + } + constexpr uint32_t kMaxNopCount = 10000000; + if (qk > kMaxNopCount || sf > kMaxNopCount || pv > kMaxNopCount || up > kMaxNopCount) { + return false; + } + *counts = NopCounts{qk, sf, pv, up}; + return true; +} + +#if PTO_FDWIC_SHARED_MAP +inline bool ParseSharedContextLens( + const char *raw, std::vector *context_lens +) { + if (raw == nullptr || *raw == '\0' || context_lens == nullptr) { + return false; + } + std::vector parsed_values; + const char *cursor = raw; + while (*cursor != '\0') { + errno = 0; + char *end = nullptr; + const unsigned long parsed = std::strtoul(cursor, &end, 10); + if (errno != 0 || end == cursor || + parsed > kHostSharedPaMaxContextLength || + (*end != ',' && *end != '\0') || + parsed_values.size() >= kMaxBatches) { + return false; + } + parsed_values.push_back(static_cast(parsed)); + if (*end == '\0') { + break; + } + cursor = end + 1; + if (*cursor == '\0') { + return false; + } + } + *context_lens = parsed_values; + return true; +} +#endif + +inline const char *FinalBarrierShapeName(FinalBarrierShape shape) { + switch (shape) { + case FinalBarrierShape::Flat: + return "flat"; + case FinalBarrierShape::TwoLevel4: + return "two-4"; + case FinalBarrierShape::TwoLevel8: + return "two-8"; + case FinalBarrierShape::TwoLevel16: + return "two-16"; + case FinalBarrierShape::ThreeLevel6x4x4: + return "three-6x4x4"; + } + return "invalid"; +} + +inline bool ParseFinalBarrierShape(const char *raw, FinalBarrierShape *shape) { + struct Entry { + const char *name; + FinalBarrierShape shape; + }; + constexpr Entry kEntries[] = { + {"flat", FinalBarrierShape::Flat}, + {"two-4", FinalBarrierShape::TwoLevel4}, + {"two-8", FinalBarrierShape::TwoLevel8}, + {"two-16", FinalBarrierShape::TwoLevel16}, + {"three-6x4x4", FinalBarrierShape::ThreeLevel6x4x4}, + }; + for (const Entry &entry : kEntries) { + if (std::strcmp(raw, entry.name) == 0) { + *shape = entry.shape; + return true; + } + } + return false; +} + +inline void PrintUsage(const char *program, bool require_kernel) { + // require_kernel 只影响 CCEC host 的用法文本，其余 benchmark 参数在三后端完全一致。 + std::fprintf( + stderr, "Usage: %s%s [--device N] [--batches 1..%u] [--runs N] ", program, + require_kernel ? " --kernel FILE" : "", kMaxBatches + ); + std::fprintf( + stderr, + "[--nop-count N | --nop-counts QK,SF,PV,UP] [--profile-phases] [--analyze-swimlane] " + "[--trace-atomics] [--swimlane-json FILE] [--no-swimlane] " +#if PTO_FDWIC_SHARED_MAP + "[--final-barrier flat|two-4|two-8|two-16|three-6x4x4] " + "[--shared-context-lens C0[,C1...]] " + "(default: two-16" + ", shared context_len=8192" + ")\n" +#else + "[--final-barrier flat|two-4|two-8|two-16|three-6x4x4] (default: two-16)\n" +#endif + ); +} + +inline ParseStatus ParseOptions(int argc, char **argv, bool require_kernel, Options *options) { + // CCEC host 需要外部 kernel ELF；AscendC 和 CPU 的可执行文件已包含 kernel，因此不需要该参数。 + bool nop_override_seen = false; + bool swimlane_json_seen = false; +#if PTO_FDWIC_SHARED_MAP + bool shared_context_lens_seen = false; +#endif + for (int index = 1; index < argc; ++index) { + // 无值开关先处理；其余参数统一在消费下一个 argv 前检查缺值，保证错误位置明确。 + const std::string argument = argv[index]; + if (argument == "--help" || argument == "-h") { + PrintUsage(argv[0], require_kernel); + return ParseStatus::Help; + } + if (argument == "--profile-phases") { + options->profile_phases = true; + continue; + } + if (argument == "--no-swimlane") { + options->trace_enabled = false; + continue; + } + if (argument == "--trace-atomics") { + options->trace_atomics = true; + continue; + } + if (argument == "--analyze-swimlane") { + options->analyze_swimlane = true; + continue; + } + if (index + 1 >= argc) { + std::fprintf(stderr, "Missing value after %s\n", argument.c_str()); + return ParseStatus::Error; + } + const char *value = argv[++index]; + if (argument == "--kernel" && require_kernel) { + options->kernel_path = value; + } else if (argument == "--device") { + if (!ParseUint(value, 0, INT32_MAX, &options->device)) return ParseStatus::Error; + } else if (argument == "--batches") { + if (!ParseUint(value, 1, kMaxBatches, &options->batches)) return ParseStatus::Error; + } else if (argument == "--runs") { + if (!ParseUint(value, 1, 1000, &options->runs)) return ParseStatus::Error; + } else if (argument == "--final-barrier") { + if (!ParseFinalBarrierShape(value, &options->final_barrier_shape)) { + std::fprintf(stderr, "Unknown final barrier shape: %s\n", value); + return ParseStatus::Error; + } + } else if (argument == "--swimlane-json") { + if (swimlane_json_seen) { + std::fprintf(stderr, "Specify --swimlane-json only once.\n"); + return ParseStatus::Error; + } + if (*value == '\0') { + std::fprintf(stderr, "--swimlane-json requires a non-empty path.\n"); + return ParseStatus::Error; + } + options->swimlane_json = value; + swimlane_json_seen = true; + } else if (argument == "--nop-count") { + if (nop_override_seen) { + std::fprintf(stderr, "Specify only one NOP override.\n"); + return ParseStatus::Error; + } + uint32_t count = 0; + if (!ParseUint(value, 0, 10000000, &count)) return ParseStatus::Error; + options->nops = NopCounts{count, count, count, count}; + nop_override_seen = true; + } else if (argument == "--nop-counts") { + if (nop_override_seen) { + std::fprintf(stderr, "Specify only one NOP override.\n"); + return ParseStatus::Error; + } + if (!ParseNopCounts(value, &options->nops)) return ParseStatus::Error; + nop_override_seen = true; +#if PTO_FDWIC_SHARED_MAP + } else if (argument == "--shared-context-lens") { + if (shared_context_lens_seen) { + std::fprintf( + stderr, + "Specify --shared-context-lens only once.\n" + ); + return ParseStatus::Error; + } + if (!ParseSharedContextLens( + value, &options->shared_context_lens + )) { + std::fprintf( + stderr, + "--shared-context-lens requires 1..%u comma-separated " + "values in [0,%u].\n", + kMaxBatches, kHostSharedPaMaxContextLength + ); + return ParseStatus::Error; + } + shared_context_lens_seen = true; +#endif + } else { + std::fprintf(stderr, "Unknown argument: %s\n", argument.c_str()); + return ParseStatus::Error; + } + } + if (require_kernel && options->kernel_path.empty()) { + std::fprintf(stderr, "--kernel is required\n"); + return ParseStatus::Error; + } + if (options->analyze_swimlane && !options->trace_enabled) { + // 分析和导出都依赖完整 record 缓冲，不能与节省内存的 --no-swimlane 同时使用。 + std::fprintf(stderr, "--analyze-swimlane requires swimlane tracing.\n"); + return ParseStatus::Error; + } + if (options->trace_atomics && !options->trace_enabled) { + std::fprintf(stderr, "--trace-atomics cannot be combined with --no-swimlane.\n"); + return ParseStatus::Error; + } + if (!options->swimlane_json.empty() && !options->trace_enabled) { + std::fprintf(stderr, "--swimlane-json requires swimlane tracing.\n"); + return ParseStatus::Error; + } + if (!options->swimlane_json.empty() && options->runs != 1) { + // 一个文件只对应一次完整采集，禁止多轮运行反复覆盖而丢失轮次边界。 + std::fprintf(stderr, "--swimlane-json requires --runs 1 to avoid overwriting captures.\n"); + return ParseStatus::Error; + } +#if PTO_FDWIC_SHARED_MAP + if (!options->shared_context_lens.empty() && + options->shared_context_lens.size() != 1 && + options->shared_context_lens.size() != options->batches) { + std::fprintf( + stderr, + "--shared-context-lens must contain one broadcast value or " + "exactly --batches (%u) values; got %zu.\n", + options->batches, options->shared_context_lens.size() + ); + return ParseStatus::Error; + } +#endif + return ParseStatus::Ok; +} + +#if PTO_FDWIC_SHARED_MAP +struct SharedHostBatchPlan { + uint32_t batch; + uint32_t batch_start; + uint32_t group_count; + uint32_t task_count; + uint32_t block_count; + uint32_t final_up_task_id; + int32_t context_length; +}; + +struct SharedHostPlannedTask { + uint32_t task_id; + uint32_t batch; + uint32_t batch_start; + uint32_t task_offset; + uint32_t group_index; + uint32_t group_block_count; + uint64_t canonical_task_base; + uint64_t output_bytes; + TaskKind kind; + bool in_group; + bool has_following_group; + bool is_final_up; + bool is_last_in_batch; +}; + +struct SharedHostTaskPlan { + uint32_t batch_count = 0; + uint32_t total_groups = 0; + uint32_t total_tasks = 0; + uint64_t canonical_heap_bytes = 0; + uint32_t tasks_by_kind[ + static_cast(TaskKind::Count) + ] = {}; + std::vector batches; + std::vector tasks; + + const SharedHostPlannedTask *TaskAt(uint32_t task_id) const { + if (task_id >= tasks.size() || + tasks[task_id].task_id != task_id) { + return nullptr; + } + return &tasks[task_id]; + } + + const SharedHostBatchPlan *BatchAt(uint32_t batch) const { + if (batch >= batches.size() || + batches[batch].batch != batch) { + return nullptr; + } + return &batches[batch]; + } +}; + +inline uint64_t SharedHostOutputBytes( + TaskKind kind, uint32_t group_block_count +) { + switch (kind) { + case TaskKind::Alloc: + return 10240; + case TaskKind::Qk: + return static_cast(group_block_count) * 8192; + case TaskKind::Sf: + return static_cast(group_block_count) * 4096 + + 2048; + case TaskKind::Pv: + return 8192; + case TaskKind::Up: + case TaskKind::Count: + return 0; + } + return 0; +} + +inline bool BuildSharedHostTaskPlan( + const SchedulerState &state, SharedHostTaskPlan *plan, + std::string *error = nullptr +) { + if (plan == nullptr) { + if (error != nullptr) { + *error = "null output plan"; + } + return false; + } + *plan = SharedHostTaskPlan{}; + const uint32_t batches = state.config.batches; + if (batches == 0 || batches > kMaxBatches) { + if (error != nullptr) { + *error = + "batch count is outside [1," + + std::to_string(kMaxBatches) + "]"; + } + return false; + } + plan->batch_count = batches; + plan->batches.reserve(batches); + plan->tasks.reserve( + static_cast(batches) * + kSharedPaMaxTasksPerBatch + ); + + for (uint32_t batch = 0; batch < batches; ++batch) { + const int32_t context_length = state.context_lens[batch]; + if (context_length < 0 || + static_cast(context_length) > + kHostSharedPaMaxContextLength) { + if (error != nullptr) { + *error = + "context_lens[" + std::to_string(batch) + + "] is outside [0," + + std::to_string(kHostSharedPaMaxContextLength) + + "]"; + } + *plan = SharedHostTaskPlan{}; + return false; + } + const uint32_t sequence = + static_cast(context_length); + const uint32_t block_count = + (sequence + kHostPaBlockSize - 1U) / + kHostPaBlockSize; + const uint32_t group_count = + (block_count + kHostPaBlocksPerRequest - 1U) / + kHostPaBlocksPerRequest; + const uint32_t task_count = 1U + 4U * group_count; + if (group_count > kSharedPaMaxBlockGroups || + task_count > kSharedPaMaxTasksPerBatch || + plan->total_tasks > + kMaxTasks - task_count) { + if (error != nullptr) { + *error = + "shared task plan exceeds compiled capacity at batch " + + std::to_string(batch); + } + *plan = SharedHostTaskPlan{}; + return false; + } + + const uint32_t batch_start = plan->total_tasks; + const uint32_t no_final_up = UINT32_MAX; + plan->batches.push_back( + SharedHostBatchPlan{ + batch, + batch_start, + group_count, + task_count, + block_count, + group_count == 0 + ? no_final_up + : batch_start + task_count - 1U, + context_length, + } + ); + + const auto append_task = [&]( + TaskKind kind, uint32_t task_offset, + uint32_t group_index, uint32_t group_blocks, + bool in_group + ) { + const uint32_t task_id = + batch_start + task_offset; + const uint64_t output_bytes = + SharedHostOutputBytes(kind, group_blocks); + const bool has_following_group = + kind == TaskKind::Up && + group_index + 1U < group_count; + const bool is_final_up = + kind == TaskKind::Up && + group_index + 1U == group_count; + plan->tasks.push_back( + SharedHostPlannedTask{ + task_id, + batch, + batch_start, + task_offset, + group_index, + group_blocks, + plan->canonical_heap_bytes, + output_bytes, + kind, + in_group, + has_following_group, + is_final_up, + task_offset + 1U == task_count, + } + ); + plan->canonical_heap_bytes += output_bytes; + ++plan->tasks_by_kind[ + static_cast(kind) + ]; + }; + + append_task(TaskKind::Alloc, 0, 0, 0, false); + for (uint32_t group = 0; group < group_count; ++group) { + const uint32_t block_offset = + group * kHostPaBlocksPerRequest; + const uint32_t group_blocks = std::min( + kHostPaBlocksPerRequest, + block_count - block_offset + ); + const uint32_t group_offset = 1U + 4U * group; + append_task( + TaskKind::Qk, group_offset, group, + group_blocks, true + ); + append_task( + TaskKind::Sf, group_offset + 1U, group, + group_blocks, true + ); + append_task( + TaskKind::Pv, group_offset + 2U, group, + group_blocks, true + ); + append_task( + TaskKind::Up, group_offset + 3U, group, + group_blocks, true + ); + } + plan->total_groups += group_count; + plan->total_tasks += task_count; + } + + if (plan->tasks.size() != plan->total_tasks || + plan->batches.size() != plan->batch_count || + plan->tasks_by_kind[ + static_cast(TaskKind::Alloc) + ] != plan->batch_count || + plan->tasks_by_kind[ + static_cast(TaskKind::Qk) + ] != plan->total_groups || + plan->tasks_by_kind[ + static_cast(TaskKind::Sf) + ] != plan->total_groups || + plan->tasks_by_kind[ + static_cast(TaskKind::Pv) + ] != plan->total_groups || + plan->tasks_by_kind[ + static_cast(TaskKind::Up) + ] != plan->total_groups) { + if (error != nullptr) { + *error = "shared task plan internal accounting mismatch"; + } + *plan = SharedHostTaskPlan{}; + return false; + } + return true; +} + +struct SharedHostHeapAdmission { + uint64_t heap_size = 0; + uint64_t shard_span = 0; + uint64_t usable_capacity = 0; + uint64_t total_reserved_bytes = 0; + uint64_t reserved_bytes_by_shard[kSharedHeapShards] = {}; + uint32_t first_failed_task = UINT32_MAX; + uint32_t first_failed_shard = UINT32_MAX; + bool admitted = false; +}; + +inline bool ValidateSharedHostHeapAdmission( + const SharedHostTaskPlan &plan, uint64_t heap_size, + SharedHostHeapAdmission *admission, + std::string *error = nullptr +) { + if (admission == nullptr) { + if (error != nullptr) { + *error = "null shared heap admission result"; + } + return false; + } + *admission = SharedHostHeapAdmission{}; + admission->heap_size = heap_size; + if (heap_size > static_cast(INT64_MAX)) { + if (error != nullptr) { + *error = "shared heap size exceeds signed atomic range"; + } + return false; + } + admission->shard_span = + (heap_size / kSharedHeapShards) / + kOutputAlignment * kOutputAlignment; + admission->usable_capacity = + admission->shard_span * kSharedHeapShards; + + if (plan.tasks.size() != plan.total_tasks) { + if (error != nullptr) { + *error = "shared heap admission received an incomplete task plan"; + } + return false; + } + for (const SharedHostPlannedTask &task : plan.tasks) { + if (task.task_id >= kMaxTasks || + plan.TaskAt(task.task_id) != &task) { + admission->first_failed_task = task.task_id; + if (error != nullptr) { + *error = + "shared heap admission task ids are not contiguous"; + } + return false; + } + if (task.output_bytes == 0) { + continue; + } + if (task.output_bytes > + UINT64_MAX - (kOutputAlignment - 1U)) { + admission->first_failed_task = task.task_id; + if (error != nullptr) { + *error = + "shared output byte count overflows alignment"; + } + return false; + } + const uint64_t reserve = + (task.output_bytes + kOutputAlignment - 1U) / + kOutputAlignment * kOutputAlignment; + const uint32_t shard = + task.task_id % kSharedHeapShards; + admission->first_failed_task = task.task_id; + admission->first_failed_shard = shard; + if (reserve == 0 || + reserve > static_cast(INT64_MAX) || + reserve > admission->shard_span || + admission->reserved_bytes_by_shard[shard] > + admission->shard_span - reserve || + admission->total_reserved_bytes > + admission->usable_capacity - reserve || + admission->total_reserved_bytes > + static_cast(INT64_MAX) - reserve) { + if (error != nullptr) { + *error = + "shared heap capacity exceeded before worker/device start " + "at task " + std::to_string(task.task_id) + + ", shard " + std::to_string(shard); + } + return false; + } + admission->reserved_bytes_by_shard[shard] += + reserve; + admission->total_reserved_bytes += reserve; + } + if (admission->total_reserved_bytes != + plan.canonical_heap_bytes) { + if (error != nullptr) { + *error = + "shared heap admission disagrees with canonical plan bytes"; + } + return false; + } + admission->first_failed_task = UINT32_MAX; + admission->first_failed_shard = UINT32_MAX; + admission->admitted = true; + return true; +} + +inline void PrintSharedHostHeapAdmission( + const SharedHostTaskPlan &plan, + const SharedHostHeapAdmission &admission +) { + uint64_t maximum_shard_bytes = 0; + for (uint32_t shard = 0; + shard < kSharedHeapShards; ++shard) { + maximum_shard_bytes = std::max( + maximum_shard_bytes, + admission.reserved_bytes_by_shard[shard] + ); + } + std::printf( + "[HOST_HEAP_ADMISSION] batches=%u groups=%u tasks=%u " + "total_bytes=%llu max_shard_bytes=%llu " + "shard_capacity=%llu status=%s\n", + plan.batch_count, plan.total_groups, + plan.total_tasks, + static_cast( + admission.total_reserved_bytes + ), + static_cast( + maximum_shard_bytes + ), + static_cast( + admission.shard_span + ), + admission.admitted ? "PASS" : "FAIL" + ); +} +#endif + +inline void InitializeState(SchedulerState *state, const Options &options) { + // WorkerState 有意保持真实 PA 每核约 9 MiB 的布局。若 host 每轮清空全部 worker， + // 会额外触碰并拷贝近 1 GiB 内存；因此只初始化全局前缀和结果区，worker 的活跃字段 + // 由各自 kernel 在启动后复位，这也与真实 PA 的生命周期一致。 + std::memset(state, 0, offsetof(SchedulerState, workers)); + std::memset(&state->config, 0, offsetof(SchedulerState, results) - offsetof(SchedulerState, config)); + std::memset(state->results, 0, sizeof(state->results)); +#if PTO_FDWIC_SHARED_MAP + // shared_map 位于 results 之后，不属于上面的 control/result 任一范围。 + // 先把 payload、bucket 游标和保留字节清零，再建立协议要求的 -1 + // seq/reclaim sentinel；这样每轮复用同一 host/device 分配时不会继承 + // 上一轮 lap。S2.5 不再使用 per-core replay progress。 + std::memset(&state->shared_map, 0, sizeof(state->shared_map)); + state->shared_map.committed_tasks.value = 0; + for (uint32_t lane = 1; + lane < kSharedInsertTurnCapacity; ++lane) { + state->shared_map + .insert_turn_extra[lane - 1U].value = -1; + } + state->shared_map.reclaim_upto.value = -1; + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + state->shared_map.buckets[bucket].head.value = 0; + state->shared_map.buckets[bucket].tail.value = 0; + } + for (uint32_t slot = 0; slot < kMapCapacity; ++slot) { + state->shared_map.slots[slot].seq.value = -1; + } + // 每个 task 的 fresh Output 只在本轮使用一次，发布位与最后 writer 都用 + // -1 表示“尚无可消费 descriptor”。TensorDesc 区已由上方 memset 清零； + // 不对 task_id 取模，避免在本阶段提前引入 generation 语义。 + for (uint32_t task_id = 0; task_id < kMaxTasks; ++task_id) { + // task 表位于 production prefix，前面的 memset 会把该字段清零； + // shared per-task 插入完成链必须用 -1 区分“尚未发布”与 task 0 + // 已完成 TensorMap writer 元数据插入。 + state->tasks[task_id].deps_prepared = -1; + for (uint32_t slot = 0; slot < kSharedOutputMaxPerTask; ++slot) { + state->shared_map.shared_outputs[task_id].published[slot].value = -1; + state->shared_map.shared_outputs[task_id].last_writer[slot].value = -1; + } + } + // shared heap 允许不同 winner 并发推进分片 cursor 与 aggregate vend； + // 每轮仍必须从绝对零点开始，不能继承上一轮 sidecar 的终态。 + for (uint32_t shard = 0; shard < kSharedHeapShards; ++shard) { + state->shared_map.shared_heap_cursor[shard].value = 0; + } + state->shared_map.shared_heap_vend.value = 0; + // shared Claim 仍使用八条 Vector cursor；它与 per-task 插入完成链 + // 是两套独立状态，每轮都从 -1 开始。 + for (uint32_t shard = 0; shard < kSharedVectorCursorCapacity; ++shard) { + state->shared_map.shared_vector_cursor[shard].value = -1; + } + // reader_done 是“ordinary 读取已经关闭”的完成前沿，不是 task 分配 + // 游标；-1 表示本 worker 尚未关闭 task 0。R4e-a 尚未接入 PA 热路径， + // 因此真实 PA 回放后 host 还会要求 96 条线全部保持该初值。 + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + state->shared_map.reader_done[worker].value = -1; + } +#endif + state->heap_window = kHeapWindow; + state->heap_base = kSyntheticHeapBase; + state->heap_size = + options.batches > kDefaultBatches + ? kExtendedBatchHeapBytes + : kHeapBytes; + state->num_workers = kWorkers; + state->num_blocks = kAicWorkers; + state->config.batches = options.batches; + state->config.workers = kWorkers; + state->config.nops = options.nops; + state->config.profile_phases = options.profile_phases ? 1U : 0U; + state->config.final_barrier_shape = static_cast(options.final_barrier_shape); + state->config.build_identity_magic = kBuildIdentityMagic; + state->config.build_identity_abi_version = kBuildIdentityAbiVersion; + state->config.tensor_map_mode = static_cast(kCompiledTensorMapMode); + state->config.scheduler_state_size = static_cast(sizeof(SchedulerState)); + state->pmu_probe.build_variant = kCompiledBuildVariant; + for (uint32_t batch = 0; batch < options.batches; ++batch) { +#if PTO_FDWIC_SHARED_MAP + if (options.shared_context_lens.empty()) { + state->context_lens[batch] = 8192; + } else if (options.shared_context_lens.size() == 1) { + state->context_lens[batch] = + options.shared_context_lens.front(); + } else if (batch < options.shared_context_lens.size()) { + state->context_lens[batch] = + options.shared_context_lens[batch]; + } else { + // ParseOptions 会拒绝长度不匹配；这里仍用非法 sentinel + // fail closed，避免直接调用 InitializeState 的测试/后端越界。 + state->context_lens[batch] = -1; + } +#else + state->context_lens[batch] = 8192; +#endif + } + for (uint32_t shard = 0; shard < kCursorShards; ++shard) { + // -1 表示尚无 task 被 claim；task 0 的 atomicMax 因而也能正常判定唯一 winner。 + state->cube_cursor[shard].value = -1; + state->vector_cursor[shard].value = -1; + state->alloc_cursor[shard].value = -1; + } + state->frontier.value = -1; +} + +inline void ConfigureTrace(SchedulerState *state, const Options &options, const void *trace_base) { + // device 只持有裸地址和每核容量；TraceHeader/record 缓冲区由 host 单独分配并初始化。 + state->config.trace_enabled = options.trace_enabled + ? kTracePhasesEnabled | (options.trace_atomics ? kTraceAtomicsEnabled : 0U) + : 0U; + state->config.trace_base = options.trace_enabled ? reinterpret_cast(trace_base) : 0; + state->config.trace_records_per_core = options.trace_enabled ? kTraceRecordsPerCore : 0; +} + +inline void InitializeTraceHeader(TraceHeader *header) { + // version=5 表示 shared Register 必须携带嵌套的 metadata 与 task-output + // detail；core state 继续携带 weighted atomic/PollBatch 计数和权威拓扑。 + std::memset(header, 0, sizeof(*header)); + header->magic = 0x4653574cU; + header->version = 5; + header->num_cores = kWorkers; + header->records_per_core = kTraceRecordsPerCore; + header->frequency_hz = kSystemCounterHz; + header->record_size_bytes = kTraceRecordSizeBytes; +} + +inline bool EncodeCompactTraceRecord( + const TraceRecord &logical, CompactTraceRecord16 *compact +) { + if (compact == nullptr || + logical.end_cycle < logical.start_cycle || + !CompactTraceFieldsFit( + logical.task_id, logical.function_id, + static_cast(logical.phase), + logical.auxiliary + )) { + return false; + } + compact->start_cycle_low = + static_cast(logical.start_cycle); + compact->end_cycle_low = + static_cast(logical.end_cycle); + compact->flags = logical.flags; + compact->packed = PackCompactTraceFields( + logical.task_id, logical.function_id, + static_cast(logical.phase), + logical.auxiliary + ); + return true; +} + +inline bool UnfoldCompactClockAfter( + uint32_t low, uint64_t anchor, uint64_t *unfolded +) { + if (unfolded == nullptr) return false; + const uint64_t delta = static_cast( + low - static_cast(anchor) + ); + if (anchor > UINT64_MAX - delta) return false; + *unfolded = anchor + delta; + return true; +} + +inline bool UnfoldCompactClockBefore( + uint32_t low, uint64_t anchor, uint64_t *unfolded +) { + if (unfolded == nullptr) return false; + const uint64_t delta = static_cast( + static_cast(anchor) - low + ); + if (delta > anchor) return false; + *unfolded = anchor - delta; + return true; +} + +inline bool DecodeCompactTraceRecord( + const CompactTraceRecord16 &compact, + uint64_t startup_barrier_begin, uint64_t finish_cycle, + TraceRecord *logical +) { + constexpr uint64_t kCompactClockWindow = UINT64_C(1) << 32U; + if (logical == nullptr || startup_barrier_begin == 0 || + finish_cycle < startup_barrier_begin || + finish_cycle - startup_barrier_begin >= + kCompactClockWindow) { + return false; + } + + const uint32_t task_code = + compact.packed & kCompactTraceTaskMask; + const uint32_t function_code = + (compact.packed >> kCompactTraceFunctionShift) & + kCompactTraceFunctionMask; + const uint32_t phase_code = + (compact.packed >> kCompactTracePhaseShift) & + kCompactTracePhaseMask; + const uint32_t auxiliary = + (compact.packed >> kCompactTraceAuxiliaryShift) & + kCompactTraceAuxiliaryMask; + if ((task_code != kCompactTraceTaskSentinel && + task_code >= kMaxTasks) || + (function_code != kCompactTraceFunctionSentinel && + function_code > 3U) || + phase_code >= static_cast(TracePhase::Count)) { + return false; + } + + TraceRecord decoded{}; + decoded.task_id = + task_code == kCompactTraceTaskSentinel + ? -1 + : static_cast(task_code); + decoded.function_id = + function_code == kCompactTraceFunctionSentinel + ? -1 + : static_cast(function_code); + decoded.flags = compact.flags; + decoded.phase = static_cast(phase_code); + decoded.auxiliary = static_cast(auxiliary); + + const bool startup_config = + phase_code == static_cast(TracePhase::Dcci) && + auxiliary == + static_cast( + DcciSite::StartupConfigInvalidate + ); + const bool clocks_ok = startup_config + ? UnfoldCompactClockBefore( + compact.start_cycle_low, startup_barrier_begin, + &decoded.start_cycle + ) && + UnfoldCompactClockBefore( + compact.end_cycle_low, startup_barrier_begin, + &decoded.end_cycle + ) && + decoded.start_cycle <= decoded.end_cycle && + decoded.end_cycle < startup_barrier_begin + : UnfoldCompactClockAfter( + compact.start_cycle_low, startup_barrier_begin, + &decoded.start_cycle + ) && + UnfoldCompactClockAfter( + compact.end_cycle_low, startup_barrier_begin, + &decoded.end_cycle + ) && + decoded.start_cycle >= startup_barrier_begin && + decoded.start_cycle <= decoded.end_cycle && + decoded.end_cycle <= finish_cycle; + if (!clocks_ok) return false; + *logical = decoded; + return true; +} + +inline bool DecodeTraceStorageRecords( + const TraceStorageRecord *physical, uint32_t count, + uint64_t startup_barrier_begin, uint64_t finish_cycle, + TraceRecord *logical +) { + if ((count != 0 && (physical == nullptr || logical == nullptr))) { + return false; + } + for (uint32_t index = 0; index < count; ++index) { +#if PA_BUILD_COMPACT_GENERIC_TRACE + if (!DecodeCompactTraceRecord( + physical[index], startup_barrier_begin, + finish_cycle, &logical[index] + )) { + return false; + } +#else + (void)startup_barrier_begin; + (void)finish_cycle; + logical[index] = physical[index]; +#endif + } + return true; +} + +// 巨大的 WorkerState 不参与每轮 H2D/D2H。private 仍只搬前缀、控制量和 +// 结果三个既有范围；shared 额外把 results 后的 map sidecar 作为第四个 +// 独立范围搬运：private 约 2 MiB，shared 含 output/history table 约 +// 12 MiB；不能把它混入 ControlBytes/ResultBytes。 +inline constexpr size_t StatePrefixBytes() { return offsetof(SchedulerState, workers); } + +inline constexpr size_t ControlBytes() { + // control sidecar 位于为生产 DistGlobal 保留的总跨度之后，依次覆盖 + // RunConfig、独立 PMU 配置、winner workload、context 和 final barrier。 + return offsetof(SchedulerState, results) - offsetof(SchedulerState, config); +} + +inline constexpr size_t ResultBytes() { return sizeof(WorkerResult) * kWorkers; } + +inline constexpr size_t SharedSidecarBytes() { return sizeof(SharedTensorMapSidecar); } +#if PTO_FDWIC_SHARED_MAP +static_assert(SharedSidecarBytes() == 12434560, "shared TensorMap transfer size changed"); +#else +static_assert(SharedSidecarBytes() == 2113664, "private TensorMap transfer size changed"); +#endif + +inline constexpr size_t FinalBarrierStateBytes() { return sizeof(FinalBarrierState); } + +struct Metrics { + // lifecycle_* 来自跨核一致的 1 GHz SYS_CNT；host_launch_us 仍只作为包含 + // launch/synchronize 的外层参考，不与设备内分段时间混算。 + bool passed = true; + double submit_span_us = 0; + double startup_barrier_span_us = 0; + double final_barrier_span_us = 0; + double final_drain_span_us = 0; + double lifecycle_span_us = 0; +}; + +inline void Expect(bool condition, const char *label, Metrics *metrics) { + // 所有断言都继续执行，以便一次失败运行尽可能暴露完整状态，而不是遇到首错立即退出。 + std::printf("[ASSERT] %-48s %s\n", label, condition ? "PASS" : "FAIL"); + if (!condition) metrics->passed = false; +} + +inline bool FinalBarrierStateMatches(const FinalBarrierState &barrier, FinalBarrierShape shape) { + uint32_t leaf_groups = 0; + int64_t leaf_arrivals = 0; + uint32_t middle_groups = 0; + int64_t middle_arrivals = 0; + int64_t root_arrivals = 0; + switch (shape) { + case FinalBarrierShape::Flat: + break; + case FinalBarrierShape::TwoLevel4: + leaf_groups = 4; + leaf_arrivals = 24; + root_arrivals = 4; + break; + case FinalBarrierShape::TwoLevel8: + leaf_groups = 8; + leaf_arrivals = 12; + root_arrivals = 8; + break; + case FinalBarrierShape::TwoLevel16: + leaf_groups = 16; + leaf_arrivals = 6; + root_arrivals = 16; + break; + case FinalBarrierShape::ThreeLevel6x4x4: + leaf_groups = 16; + leaf_arrivals = 6; + middle_groups = 4; + middle_arrivals = 4; + root_arrivals = 4; + break; + default: + return false; + } + bool matches = true; + for (uint32_t group = 0; group < kFinalBarrierMaxLeafGroups; ++group) { + const bool active = group < leaf_groups; + matches &= barrier.leaf_arrivals[group].value == (active ? leaf_arrivals : 0); + matches &= barrier.leaf_releases[group].value == (active ? 1 : 0); + } + for (uint32_t group = 0; group < kFinalBarrierMaxMiddleGroups; ++group) { + const bool active = group < middle_groups; + matches &= barrier.middle_arrivals[group].value == (active ? middle_arrivals : 0); + matches &= barrier.middle_releases[group].value == (active ? 1 : 0); + } + matches &= barrier.root_arrival.value == root_arrivals; + matches &= barrier.root_release.value == (root_arrivals == 0 ? 0 : 1); + return matches; +} + +struct Uint64Distribution { + uint64_t total = 0; + double median = 0.0; + uint64_t p95 = 0; + uint64_t maximum = 0; +}; + +inline Uint64Distribution SummarizeUint64(std::vector values) { + // 这里按 worker 维度统计累计周期，p95 使用 nearest-rank，避免插值掩盖慢核。 + Uint64Distribution summary; + if (values.empty()) return summary; + + std::sort(values.begin(), values.end()); + for (uint64_t value : values) summary.total += value; + const size_t middle = values.size() / 2; + summary.median = (values.size() & 1U) != 0 + ? static_cast(values[middle]) + : (static_cast(values[middle - 1]) + static_cast(values[middle])) / 2.0; + const size_t p95_rank = (95U * values.size() + 99U) / 100U; + summary.p95 = values[p95_rank - 1]; + summary.maximum = values.back(); + return summary; +} + +inline void PrintPhaseDiagnostics(const SchedulerState &state) { + if (state.config.profile_phases == 0) return; + + // WaitForSlot/HeapGuard 没有各自独立命名的 TracePhase；实际发生等待时会写 + // RingBp 记录，汇总诊断则使用 WorkerResult 中的累计周期和等待次数。 + struct PhaseSpec { + ProfilePhase phase; + const char *name; + int32_t wait_event_index; + }; + const PhaseSpec phases[] = { + {ProfilePhase::Claim, "Claim", -1}, + {ProfilePhase::EfDrain, "EfDrain", -1}, + {ProfilePhase::WaitForSlot, "WaitForSlot", 0}, + {ProfilePhase::HeapGuard, "HeapGuard", 1}, + }; + const CoreRole roles[] = {CoreRole::Aic, CoreRole::Aiv}; + const char *role_names[] = {"AIC", "AIV"}; + + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + // AIC/AIV 分开统计，避免 32:64 的参与者数量差异掩盖某一类核上的长尾。 + for (const PhaseSpec &phase : phases) { + std::vector cycles; + std::vector calls; + std::vector wait_events; + const uint32_t phase_index = static_cast(phase.phase); + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const WorkerResult &result = state.results[worker]; + if (result.role != static_cast(roles[role_index])) continue; + cycles.push_back(result.phase_cycles[phase_index]); + calls.push_back(result.phase_calls[phase_index]); + wait_events.push_back( + phase.wait_event_index < 0 ? 0 : result.wait_events[static_cast(phase.wait_event_index)] + ); + } + const Uint64Distribution cycle_summary = SummarizeUint64(cycles); + const Uint64Distribution call_summary = SummarizeUint64(calls); + const Uint64Distribution wait_summary = SummarizeUint64(wait_events); + std::printf( + "[PHASE] role=%s phase=%s workers=%zu accumulated_us_median=%.3f " + "accumulated_us_p95=%.3f accumulated_us_max=%.3f calls_total=%llu " + "calls_per_worker_median=%.1f calls_per_worker_p95=%llu calls_per_worker_max=%llu " + "wait_events_total=%llu wait_events_per_worker_median=%.1f " + "wait_events_per_worker_p95=%llu wait_events_per_worker_max=%llu\n", + role_names[role_index], phase.name, cycles.size(), cycle_summary.median / 1000.0, + static_cast(cycle_summary.p95) / 1000.0, + static_cast(cycle_summary.maximum) / 1000.0, + static_cast(call_summary.total), call_summary.median, + static_cast(call_summary.p95), + static_cast(call_summary.maximum), + static_cast(wait_summary.total), wait_summary.median, + static_cast(wait_summary.p95), + static_cast(wait_summary.maximum) + ); + } + } +} + +inline const char *TracePhaseName(uint32_t phase) { + // 名称必须与 l2_swimlane_records.json 的 fdwic_events schema 保持一致。 + const char *names[] = { + "Kernel", "Alloc", "Build", "DrainWon", "Replay", "RingBp", "EfDrain", "Commit", + "Submit", "Materialize", "PrepareMap", "Claim", "Fanin", "Register", "Atomic", + "ClockBaseline", "OrchestrationReplay", "FinalDrain", "WinnerBuild", + "AllocComplete", "SharedRegisterPublishMetadata", + "SharedMaterializePublishTaskOutputs", + "SharedMaterializePublishTaskOutputsCopy", + "SharedMaterializePublishTaskOutputsFlush", + "Dcci", + }; + static_assert( + sizeof(names) / sizeof(names[0]) == static_cast(TracePhase::Count), + "TracePhaseName must cover every trace phase" + ); + return phase < sizeof(names) / sizeof(names[0]) ? names[phase] : "Unknown"; +} + +inline const char *AtomicSiteName(uint32_t site) { + // 顺序与 pa_model.h::AtomicSite 的稳定 raw ABI 完全一致。 + const char *names[] = { + "StartupIncrement", "StartupPoll", "FatalPoll", "FatalSet", "ClaimMax", + "FaninFlagLoad", "CompletionVendExchange", "CompletionFlagExchange", + "FrontierInitialLoad", "FrontierFlagLoad", "FrontierMax", "HeapFrontierLoad", + "HeapVendLoad", "ReplayDoneIncrement", "ReplayDonePoll", + "SharedHeapVendLoad", "SharedHeapCursorLoad", + "SharedHeapCursorReserve", "SharedHeapVendAdvance", + "SharedInsertPredecessorPoll", "SharedInsertCompletionPublish", + "SharedWinnerFatalGuardLoad", + "SharedMetadataFatalGuardLoad", + "SharedFaninOutputPublishedLoad", + "SharedMetadataOutputPublishedLoad", + "SharedFaninLastWriterLoad", + "SharedMetadataLastWriterLoad", + "SharedMetadataLastWriterCommit", + "SharedOutputWriterReserve", + "SharedOutputPublishedExchange", + "SharedMapLookupHeadLoad", + "SharedMapLookupTailLoad", + "SharedMapLookupSeqLoad", + "SharedMapAppendHeadLoad", + "SharedMapAppendTailLoad", + "SharedMapAppendSeqLoad", + "SharedMapAppendSeqResetExchange", + "SharedMapAppendSeqPublishExchange", + "SharedMapAppendTailExchange", + "SharedOutputRollbackExchange", + }; + static_assert( + sizeof(names) / sizeof(names[0]) == + static_cast(AtomicSite::Count), + "AtomicSiteName must cover every atomic site" + ); + return site < sizeof(names) / sizeof(names[0]) ? names[site] : "Unknown"; +} + +inline const char *AtomicOpName(uint32_t op) { + const char *names[] = { + "Load", "Exchange", "FetchAdd", "FetchMax", + "CompareExchange", + }; + return op < sizeof(names) / sizeof(names[0]) ? names[op] : "Unknown"; +} + +inline const char *DcciSiteName(uint32_t site) { + const char *names[] = { + "SharedFaninHistoryInvalidate", + "SharedWriterHistoryFlush", + "SharedOutputRollbackFlush", + "SharedOutputDescriptorFlush", + "SharedRegionReadInvalidate", + "SharedRegionAppendInvalidate", + "SharedRegionAppendFlush", + "SharedWinnerBuildDescriptorInvalidate", + "ObserverTraceExport", + "StartupConfigInvalidate", + }; + static_assert( + sizeof(names) / sizeof(names[0]) == + static_cast(DcciSite::Count), + "DcciSiteName must cover every DCCI site" + ); + return site < sizeof(names) / sizeof(names[0]) + ? names[site] + : "Unknown"; +} + +inline const char *DcciOpName(uint32_t op) { + const char *names[] = {"Invalidate", "CleanOut"}; + static_assert( + sizeof(names) / sizeof(names[0]) == + static_cast(DcciOp::Count), + "DcciOpName must cover every DCCI op" + ); + return op < sizeof(names) / sizeof(names[0]) + ? names[op] + : "Unknown"; +} + +inline AtomicOp AtomicSiteOp(AtomicSite site) { + return AtomicSiteExpectedOp(site); +} + +inline bool ValidateTraceHeader(const TraceHeader &header, const char *operation) { + // 在任何 D2H record 搬运前先验证容量和 dropped，防止损坏 header 导致 scratch 越界或导出残缺泳道。 + // 频率也要求精确为 1 GHz，否则后续 ns/us 换算即使 JSON 合法也没有性能意义。 + const bool valid = + header.magic == 0x4653574cU && + header.version == 5 && + header.num_cores == kWorkers && header.records_per_core == kTraceRecordsPerCore && + header.frequency_hz == kSystemCounterHz && + header.record_size_bytes == kTraceRecordSizeBytes; + bool core_states_valid = true; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const TraceCoreState &core = header.cores[worker]; + core_states_valid &= core.count <= kTraceRecordsPerCore; + core_states_valid &= core.dropped == 0; + core_states_valid &= core.poll_calls <= core.atomic_calls; + core_states_valid &= (core.poll_calls == 0) == (core.poll_batch_records == 0); + const uint64_t physical_atomic = + static_cast(core.atomic_calls) - + core.poll_calls + core.poll_batch_records; + core_states_valid &= physical_atomic <= core.count; + core_states_valid &= core.dcci_records <= core.dcci_calls; + core_states_valid &= (core.dcci_calls == 0) == (core.dcci_lines == 0); + core_states_valid &= (core.dcci_calls == 0) == (core.dcci_records == 0); + core_states_valid &= core.dcci_lines >= core.dcci_calls; + core_states_valid &= core.dcci_records <= core.count; + } + if (!valid || !core_states_valid) { + std::fprintf( + stderr, + "%s rejected an invalid trace header: magic=0x%08x version=%u cores=%u " + "records_per_core=%u record_size_bytes=%u frequency_hz=%llu " + "core_states_valid=%s\n", + operation, header.magic, header.version, header.num_cores, header.records_per_core, + header.record_size_bytes, + static_cast(header.frequency_hz), core_states_valid ? "yes" : "no" + ); + } + return valid && core_states_valid; +} + +struct TraceExportSummary { + uint64_t records = 0; + uint64_t atomic_records = 0; + uint64_t clock_baseline_records = 0; + uint64_t atomic_calls = 0; + uint64_t poll_calls = 0; + uint64_t poll_batch_records = 0; + uint64_t dcci_records = 0; + uint64_t dcci_calls = 0; + uint64_t dcci_lines = 0; + uint64_t dropped_records = 0; +}; + +inline bool SameTraceSummary(const TraceExportSummary &left, const TraceExportSummary &right) { + return left.records == right.records && left.atomic_records == right.atomic_records && + left.clock_baseline_records == right.clock_baseline_records && + left.atomic_calls == right.atomic_calls && left.poll_calls == right.poll_calls && + left.poll_batch_records == right.poll_batch_records && + left.dcci_records == right.dcci_records && + left.dcci_calls == right.dcci_calls && + left.dcci_lines == right.dcci_lines && + left.dropped_records == right.dropped_records; +} + +inline uint32_t AtomicRecordCallCount(const TraceRecord &record) { + return (record.flags & kAtomicPollBatch) != 0 + ? record.flags >> kAtomicPollCountShift + : 1U; +} + +inline bool AtomicRecordSchemaValid(const TraceRecord &record, bool atomic_trace_enabled) { + if (!atomic_trace_enabled || record.auxiliary >= static_cast(AtomicSite::Count)) { + return false; + } + const AtomicSite site = static_cast(record.auxiliary); +#if !PTO_FDWIC_SHARED_MAP + // private ELF 不得接受 shared-only raw site；否则混用产物或损坏记录会 + // 在 Count/op 校验均通过后被误报成合法 private atomic。 + if (AtomicSiteIsSharedOnly(site)) return false; +#endif + const uint32_t op = record.flags & kAtomicOpMask; + if (op != static_cast(AtomicSiteExpectedOp(site))) return false; + + const bool result_used = (record.flags & kAtomicResultUsed) != 0; + const bool value_zero = (record.flags & kAtomicValueZero) != 0; + const bool return_ready = (record.flags & kAtomicReturnReady) != 0; + const bool poll_batch = (record.flags & kAtomicPollBatch) != 0; + const uint32_t payload = record.flags >> kAtomicRetriesShift; + if (poll_batch) { + return AtomicSiteIsPollBatchable(site) && result_used && !value_zero && + (!return_ready || + site == AtomicSite::SharedInsertTurnPoll) && + payload > 0 && record.task_id == -1 && record.function_id == -1; + } + // insert-turn 等待只允许每个 Wait episode 一条聚合 PollBatch，禁止 + // 损坏 raw 把它伪装成逐 Load direct 记录。 + if (site == AtomicSite::SharedInsertTurnPoll) return false; + if (result_used != AtomicSiteResultUsed(site) || (return_ready && !result_used)) return false; + if (value_zero && op != static_cast(AtomicOp::Load)) return false; + if (payload != 0 && op != static_cast(AtomicOp::FetchMax)) return false; + if (site == AtomicSite::SharedInsertTurnHandoff && + record.task_id < 0) { + return false; + } + return record.function_id == -1; +} + +inline bool ClockRecordSchemaValid(const TraceRecord &record) { + const bool dependency = (record.flags & kClockAtomicDependency) != 0; + const bool dependency_applied = (record.flags & kClockAtomicDependencyApplied) != 0; + return (record.flags & ~(kClockAtomicDependency | kClockAtomicDependencyApplied)) == 0 && + (!dependency_applied || dependency) && record.task_id == -1 && + record.function_id == -1 && record.auxiliary == 0; +} + +inline uint32_t DcciRecordCallCount(const TraceRecord &record) { + return (record.flags >> kDcciCallCountShift) & + kDcciCallCountMask; +} + +inline uint32_t DcciRecordLineCount(const TraceRecord &record) { + return record.flags >> kDcciLineCountShift; +} + +inline bool DcciRecordSchemaValid(const TraceRecord &record) { + if (record.auxiliary >= + static_cast(DcciSite::Count)) { + return false; + } + const DcciSite site = + static_cast(record.auxiliary); +#if !PTO_FDWIC_SHARED_MAP + if (DcciSiteIsSharedOnly(site)) { + return false; + } +#endif + const uint32_t op_id = record.flags & kDcciOpMask; + if (op_id >= static_cast(DcciOp::Count) || + static_cast(op_id) != DcciSiteExpectedOp(site) || + (record.flags & kDcciReservedBit) != 0) { + return false; + } + const uint32_t call_count = DcciRecordCallCount(record); + const uint32_t line_count = DcciRecordLineCount(record); + if (call_count == 0 || line_count < call_count) { + return false; + } + if (site == DcciSite::ObserverTraceExport) { + return call_count == +#if PTO_FDWIC_SHARED_MAP + 3 && +#else + 2 && +#endif + (record.flags & kDcciTrailingDsb) != 0 && + record.task_id == -1 && + record.function_id == -1; + } + if (site == DcciSite::StartupConfigInvalidate) { + return call_count == 1 && + (record.flags & kDcciTrailingDsb) != 0 && + record.task_id == -1 && + record.function_id == -1; + } + return call_count == 1 && record.task_id >= 0; +} + +// shared 的真实回放边界是稀疏的：每个逻辑 task 只记录 Claim 和 Submit， +// EfDrain 由 Submit.start -> Claim.start 两个既有端点精确还原；winner +// 才继续记录 Materialize、Register、Fanin（非 Alloc）和 +// WinnerBuild/AllocComplete 子区间。每个 Materialize 父区间后依次紧跟 +// fresh-output publish 及其 copy/flush 两层 detail；每个 Register 后只 +// 跟唯一 SharedRegisterPublishMetadata。output detail 严格嵌在 +// Materialize，metadata 严格嵌在 Register，端点分别还原独占 output-cell +// 发布，以及 wait、串行 writer metadata 与 handoff。 +// PrepareMap 属于 private TensorMap，不得以零时长 marker 混入 shared raw。 +// 这里复用现有字段逐核闭合身份、顺序、次数和相邻时间边界；每个 winner +// 固定增加四条 detail（outputs + copy + flush + metadata），不增加 +// TraceRecord 字段，更不按 poll 扩张记录。 +// private 编译仍保持无约束，避免改变既有行为。 +struct SharedSparseTraceValidator { +#if PTO_FDWIC_SHARED_MAP + explicit SharedSparseTraceValidator( + const SharedHostTaskPlan *plan = nullptr + ) : plan_(plan) {} +#endif + + bool Observe(const TraceRecord &record) { +#if PTO_FDWIC_SHARED_MAP + const auto phase = static_cast(record.phase); + if (phase == TracePhase::PrepareMap || + phase == TracePhase::EfDrain) { + return false; + } + if (phase == TracePhase::Claim) { + if (state_ != State::AwaitClaim || + record.task_id != next_task_id_ || + record.end_cycle < record.start_cycle) { + return false; + } + const SharedHostPlannedTask *task = PlannedTask(record.task_id); + if (task == nullptr) { + return false; + } + task_id_ = record.task_id; + kind_ = task->kind; + claim_begin_ = record.start_cycle; + if ((record.flags & ~(kClaimWon | kClaimAttempted)) != 0 || + ((record.flags & kClaimWon) != 0 && + (record.flags & kClaimAttempted) == 0) || + record.auxiliary != + (kind_ == TaskKind::Alloc ? 1U : 0U)) { + return false; + } + winner_ = (record.flags & kClaimWon) != 0; + function_id_ = winner_ ? ExpectedFunctionId(kind_) : -1; + if (record.function_id != function_id_) { + return false; + } + previous_end_ = record.end_cycle; + ++claim_count_; + ++next_task_id_; + if (winner_) { + ++winner_count_; + if (kind_ == TaskKind::Alloc) { + ++alloc_winner_count_; + } + state_ = State::AwaitMaterialize; + } else { + state_ = State::AwaitLoserSubmit; + } + } else if (phase == TracePhase::Materialize) { + if (state_ != State::AwaitMaterialize || + !SameWinnerTask(record) || record.flags != 0 || + record.auxiliary != + (kind_ == TaskKind::Alloc ? 1U : 0U) || + record.start_cycle < previous_end_ || + record.end_cycle < record.start_cycle) { + return false; + } + materialize_begin_ = record.start_cycle; + materialize_end_ = record.end_cycle; + previous_end_ = record.end_cycle; + state_ = State::AwaitMaterializeTaskOutputs; + ++materialize_count_; + } else if ( + phase == + TracePhase::SharedMaterializePublishTaskOutputs + ) { + if (state_ != State::AwaitMaterializeTaskOutputs || + !SameWinnerTask(record) || record.flags != 0 || + record.auxiliary != 0 || + record.start_cycle < materialize_begin_ || + record.end_cycle < record.start_cycle || + record.end_cycle > materialize_end_) { + return false; + } + materialize_task_outputs_begin_ = record.start_cycle; + materialize_task_outputs_end_ = record.end_cycle; + state_ = State::AwaitMaterializeTaskOutputsCopy; + ++materialize_task_outputs_count_; + } else if ( + phase == + TracePhase::SharedMaterializePublishTaskOutputsCopy + ) { + if (state_ != State::AwaitMaterializeTaskOutputsCopy || + !SameWinnerTask(record) || record.flags != 0 || + record.auxiliary != 0 || + record.start_cycle < + materialize_task_outputs_begin_ || + record.end_cycle < record.start_cycle || + record.end_cycle > + materialize_task_outputs_end_) { + return false; + } + materialize_task_outputs_copy_end_ = record.end_cycle; + state_ = State::AwaitMaterializeTaskOutputsFlush; + ++materialize_task_outputs_copy_count_; + } else if ( + phase == + TracePhase::SharedMaterializePublishTaskOutputsFlush + ) { + if (state_ != State::AwaitMaterializeTaskOutputsFlush || + !SameWinnerTask(record) || record.flags != 0 || + record.auxiliary != 0 || + record.start_cycle != + materialize_task_outputs_copy_end_ || + record.end_cycle < record.start_cycle || + record.end_cycle > + materialize_task_outputs_end_) { + return false; + } + state_ = State::AwaitRegister; + ++materialize_task_outputs_flush_count_; + } else if (phase == TracePhase::Fanin) { + if (state_ != State::AwaitFanin || + !SameWinnerTask(record) || record.flags != 0 || + record.start_cycle != previous_end_ || + record.end_cycle < record.start_cycle) { + return false; + } + previous_end_ = record.end_cycle; + state_ = State::AwaitWinnerTail; + ++fanin_count_; + } else if (phase == TracePhase::Register) { + if (state_ != State::AwaitRegister || + !SameWinnerTask(record) || record.flags != 0 || + (kind_ == TaskKind::Alloc + ? record.auxiliary != 0 + : record.auxiliary > kMaxTaskTensors) || + record.start_cycle != previous_end_ || + record.end_cycle < record.start_cycle) { + return false; + } + register_begin_ = record.start_cycle; + register_end_ = record.end_cycle; + previous_end_ = record.end_cycle; + state_ = State::AwaitRegisterMetadata; + ++register_count_; + } else if ( + phase == TracePhase::SharedRegisterPublishMetadata + ) { + if (state_ != State::AwaitRegisterMetadata || + !SameWinnerTask(record) || record.flags != 0 || + record.auxiliary != 0 || + record.start_cycle < register_begin_ || + record.end_cycle < record.start_cycle || + record.end_cycle > register_end_) { + return false; + } + state_ = kind_ == TaskKind::Alloc + ? State::AwaitWinnerTail + : State::AwaitFanin; + ++register_metadata_count_; + } else if (phase == TracePhase::WinnerBuild || + phase == TracePhase::AllocComplete) { + const TracePhase expected = + kind_ == TaskKind::Alloc + ? TracePhase::AllocComplete + : TracePhase::WinnerBuild; + if (state_ != State::AwaitWinnerTail || + phase != expected || !SameWinnerTask(record) || + record.flags != 0 || record.auxiliary != 0 || + record.start_cycle != previous_end_ || + record.end_cycle < record.start_cycle) { + return false; + } + previous_end_ = record.end_cycle; + state_ = State::AwaitWinnerSubmit; + ++winner_tail_count_; + } else if (phase == TracePhase::Submit) { + const bool winner_submit = + state_ == State::AwaitWinnerSubmit && winner_; + const bool loser_submit = + state_ == State::AwaitLoserSubmit && !winner_; + if ((!winner_submit && !loser_submit) || + !SameTask(record) || + record.function_id != function_id_ || + record.flags != (winner_ ? kClaimWon : 0U) || + record.auxiliary != + (kind_ == TaskKind::Alloc ? 1U : 0U) || + record.start_cycle > claim_begin_ || + record.end_cycle < previous_end_) { + return false; + } + last_efdrain_begin_ = record.start_cycle; + last_efdrain_end_ = claim_begin_; + ++efdrain_count_; + state_ = State::AwaitClaim; + ++submit_count_; + } +#else + (void)record; +#endif + return true; + } + + bool Closed() const { +#if PTO_FDWIC_SHARED_MAP + return state_ == State::AwaitClaim && + efdrain_count_ == claim_count_ && + materialize_count_ == winner_count_ && + materialize_task_outputs_count_ == winner_count_ && + materialize_task_outputs_copy_count_ == + winner_count_ && + materialize_task_outputs_flush_count_ == + winner_count_ && + register_count_ == winner_count_ && + register_metadata_count_ == winner_count_ && + fanin_count_ + alloc_winner_count_ == + winner_count_ && + winner_tail_count_ == winner_count_ && + submit_count_ == claim_count_ && + (plan_ == nullptr || + claim_count_ == plan_->total_tasks); +#else + return true; +#endif + } + + uint32_t EfDrainCount() const { + return efdrain_count_; + } + + uint64_t LastEfDrainBegin() const { + return last_efdrain_begin_; + } + + uint64_t LastEfDrainEnd() const { + return last_efdrain_end_; + } + + uint32_t ClaimCount() const { + return claim_count_; + } + + uint32_t WinnerCount() const { + return winner_count_; + } + + uint32_t MaterializeCount() const { + return materialize_count_; + } + + uint32_t FaninCount() const { + return fanin_count_; + } + + uint32_t RegisterCount() const { + return register_count_; + } + + uint32_t RegisterMetadataCount() const { + return register_metadata_count_; + } + + uint32_t MaterializeTaskOutputsCount() const { + return materialize_task_outputs_count_; + } + + uint32_t MaterializeTaskOutputsCopyCount() const { + return materialize_task_outputs_copy_count_; + } + + uint32_t MaterializeTaskOutputsFlushCount() const { + return materialize_task_outputs_flush_count_; + } + + uint32_t WinnerTailCount() const { + return winner_tail_count_; + } + + uint32_t SubmitCount() const { + return submit_count_; + } + +private: +#if PTO_FDWIC_SHARED_MAP + enum class State { + AwaitClaim, + AwaitMaterialize, + AwaitMaterializeTaskOutputs, + AwaitMaterializeTaskOutputsCopy, + AwaitMaterializeTaskOutputsFlush, + AwaitFanin, + AwaitRegister, + AwaitRegisterMetadata, + AwaitWinnerTail, + AwaitWinnerSubmit, + AwaitLoserSubmit, + }; + + const SharedHostPlannedTask *PlannedTask(int32_t task_id) { + if (task_id < 0) { + return nullptr; + } + if (plan_ != nullptr) { + return plan_->TaskAt(static_cast(task_id)); + } + fallback_task_.task_id = static_cast(task_id); + fallback_task_.kind = static_cast( + static_cast(task_id) % kTasksPerBatch + ); + return &fallback_task_; + } + + static int32_t ExpectedFunctionId(TaskKind kind) { + return kind == TaskKind::Alloc + ? -1 + : static_cast( + static_cast(kind) - 1U + ); + } + + bool SameTask(const TraceRecord &record) const { + return record.task_id == task_id_; + } + + bool SameWinnerTask(const TraceRecord &record) const { + return winner_ && SameTask(record) && + record.function_id == function_id_; + } + + const SharedHostTaskPlan *plan_; + SharedHostPlannedTask fallback_task_{}; + State state_ = State::AwaitClaim; +#endif + int32_t next_task_id_ = 0; + int32_t task_id_ = -1; + int32_t function_id_ = -1; + TaskKind kind_ = TaskKind::Count; + bool winner_ = false; + uint64_t claim_begin_ = 0; + uint64_t last_efdrain_begin_ = 0; + uint64_t last_efdrain_end_ = 0; + uint64_t previous_end_ = 0; + uint64_t materialize_begin_ = 0; + uint64_t materialize_end_ = 0; + uint64_t materialize_task_outputs_begin_ = 0; + uint64_t materialize_task_outputs_end_ = 0; + uint64_t materialize_task_outputs_copy_end_ = 0; + uint64_t register_begin_ = 0; + uint64_t register_end_ = 0; + uint32_t efdrain_count_ = 0; + uint32_t claim_count_ = 0; + uint32_t winner_count_ = 0; + uint32_t alloc_winner_count_ = 0; + uint32_t materialize_count_ = 0; + uint32_t materialize_task_outputs_count_ = 0; + uint32_t materialize_task_outputs_copy_count_ = 0; + uint32_t materialize_task_outputs_flush_count_ = 0; + uint32_t fanin_count_ = 0; + uint32_t register_count_ = 0; + uint32_t register_metadata_count_ = 0; + uint32_t winner_tail_count_ = 0; + uint32_t submit_count_ = 0; +}; + +inline void ExpectedTraceTopology(uint32_t worker, int32_t *block_id, int32_t *lane) { + if (worker < kAicWorkers) { + *block_id = static_cast(worker); + *lane = 0; + return; + } + const uint32_t vector_id = worker - kAicWorkers; + *block_id = static_cast(vector_id / 2); + *lane = static_cast(1 + vector_id % 2); +} + +#if PTO_FDWIC_SHARED_MAP +inline bool SharedTraceClaimAttempted( + uint32_t worker, TaskKind kind +) { + if (kind == TaskKind::Alloc) { + return true; + } + const bool aic = worker < kAicWorkers; + return aic + ? kind == TaskKind::Qk || kind == TaskKind::Pv + : kind == TaskKind::Sf || kind == TaskKind::Up; +} + +inline int32_t SharedTraceFunctionId(TaskKind kind) { + return kind == TaskKind::Alloc + ? -1 + : static_cast( + static_cast(kind) - 1U + ); +} + +inline bool ExpandSharedTraceRecords( + uint32_t worker, + const TraceRecord *generic_records, + uint32_t generic_count, + const SharedSubmitClaimTraceRecord *submit_claim_records, + const SharedHostTaskPlan &plan, + std::vector *logical_records +) { + if (generic_records == nullptr || + submit_claim_records == nullptr || + logical_records == nullptr) { + return false; + } + logical_records->clear(); + logical_records->reserve( + static_cast(generic_count) + + 2U * plan.total_tasks + ); + uint32_t generic_index = 0; + for (uint32_t task_id = 0; + task_id < plan.total_tasks; ++task_id) { + const SharedHostPlannedTask *task = + plan.TaskAt(task_id); + const SharedSubmitClaimTraceRecord &endpoints = + submit_claim_records[task_id]; + const bool winner = + (endpoints.claim_end_and_winner & + kSharedClaimWinnerBit) != 0; + const bool attempted = + task != nullptr && + SharedTraceClaimAttempted(worker, task->kind); + const uint64_t claim_begin = + endpoints.claim_begin; + const uint64_t claim_end = + endpoints.claim_end_and_winner & + ~kSharedClaimWinnerBit; + const uint64_t submit_begin = + endpoints.submit_begin; + const uint64_t submit_end = + endpoints.submit_end; + if (task == nullptr || + submit_begin == 0 || + claim_begin == 0 || + (claim_begin & kSharedClaimWinnerBit) != 0 || + (submit_begin & kSharedClaimWinnerBit) != 0 || + (submit_end & kSharedClaimWinnerBit) != 0 || + submit_end < submit_begin || + claim_end < claim_begin || + submit_begin > claim_begin || + claim_end > submit_end || + (winner && !attempted)) { + return false; + } + + while (generic_index < generic_count && + generic_records[generic_index].end_cycle <= + claim_end) { + const TraceRecord &record = + generic_records[generic_index++]; + if (record.phase == + static_cast(TracePhase::Claim) || + record.phase == + static_cast(TracePhase::Submit) || + record.phase == + static_cast(TracePhase::EfDrain)) { + return false; + } + logical_records->push_back(record); + } + + const int32_t function_id = winner + ? SharedTraceFunctionId(task->kind) + : -1; + const uint16_t is_alloc = + task->kind == TaskKind::Alloc ? 1U : 0U; + TraceRecord claim{}; + claim.start_cycle = claim_begin; + claim.end_cycle = claim_end; + claim.task_id = static_cast(task_id); + claim.function_id = function_id; + claim.flags = + (winner ? kClaimWon : 0U) | + (attempted ? kClaimAttempted : 0U); + claim.phase = + static_cast(TracePhase::Claim); + claim.auxiliary = is_alloc; + logical_records->push_back(claim); + + while (generic_index < generic_count && + generic_records[generic_index].end_cycle <= + submit_end) { + const TraceRecord &record = + generic_records[generic_index++]; + if (record.phase == + static_cast(TracePhase::Claim) || + record.phase == + static_cast(TracePhase::Submit) || + record.phase == + static_cast(TracePhase::EfDrain)) { + return false; + } + logical_records->push_back(record); + } + + TraceRecord submit{}; + submit.start_cycle = submit_begin; + submit.end_cycle = submit_end; + submit.task_id = static_cast(task_id); + submit.function_id = function_id; + submit.flags = winner ? kClaimWon : 0U; + submit.phase = + static_cast(TracePhase::Submit); + submit.auxiliary = is_alloc; + logical_records->push_back(submit); + } + while (generic_index < generic_count) { + const TraceRecord &record = + generic_records[generic_index++]; + if (record.phase == + static_cast(TracePhase::Claim) || + record.phase == + static_cast(TracePhase::Submit) || + record.phase == + static_cast(TracePhase::EfDrain)) { + return false; + } + logical_records->push_back(record); + } + return logical_records->size() == + static_cast(generic_count) + + 2U * plan.total_tasks; +} +#endif + +template < + typename ReadRecords +#if PTO_FDWIC_SHARED_MAP + , typename ReadSubmitClaimRecords +#endif +> +inline bool ExportSwimlaneRecords( +#if PTO_FDWIC_SHARED_MAP + const TraceHeader &header, const SchedulerState &state, + const std::string &output_path, +#else + const TraceHeader &header, const std::string &output_path, +#endif + WinnerWorkloadMode workload_mode, const WorkloadCounts &workload_counts, + const char *workload_pattern, FinalBarrierShape final_barrier_shape, + bool atomic_trace_enabled, ReadRecords read_records +#if PTO_FDWIC_SHARED_MAP + , ReadSubmitClaimRecords read_submit_claim_records +#endif +) { + if (!ValidateTraceHeader(header, "swimlane export")) return false; +#if PTO_FDWIC_SHARED_MAP + SharedHostTaskPlan shared_plan; + std::string shared_plan_error; + if (!BuildSharedHostTaskPlan( + state, &shared_plan, &shared_plan_error + )) { + std::fprintf( + stderr, + "swimlane export rejected invalid shared task plan: %s\n", + shared_plan_error.c_str() + ); + return false; + } +#endif + if (workload_mode != WinnerWorkloadMode::ScalarNop && + workload_mode != WinnerWorkloadMode::RealCompute) { + std::fprintf(stderr, "swimlane export rejected invalid winner workload mode.\n"); + return false; + } + const bool real_compute = workload_mode == WinnerWorkloadMode::RealCompute; + const bool pattern_valid = workload_pattern != nullptr && + ((real_compute && + (std::strcmp(workload_pattern, "constant") == 0 || + std::strcmp(workload_pattern, "layout-diagnostic") == 0)) || + (!real_compute && std::strcmp(workload_pattern, "none") == 0)); + if (!pattern_valid) { + std::fprintf(stderr, "swimlane export rejected invalid winner workload input pattern.\n"); + return false; + } + + TraceExportSummary producer_summary; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const TraceCoreState &core = header.cores[worker]; + int32_t expected_block = -1; + int32_t expected_lane = -1; + ExpectedTraceTopology(worker, &expected_block, &expected_lane); + if (core.core_idx != static_cast(worker) || core.block_id != expected_block || + core.lane != expected_lane) { + std::fprintf( + stderr, + "swimlane export rejected worker topology: worker=%u core=%d block=%d/%d lane=%d/%d\n", + worker, core.core_idx, core.block_id, expected_block, core.lane, expected_lane + ); + return false; + } + // core.count 只统计 generic 区的物理记录；shared 的每个 + // Submit/Claim 在专用 32B 四端点区保存一条，导出后会展开成两个 + // 逻辑事件，因此 summary 必须继续保持 JSON 事件数口径。 + producer_summary.records += core.count; +#if PTO_FDWIC_SHARED_MAP + producer_summary.records += + 2ULL * shared_plan.total_tasks; +#endif + producer_summary.atomic_calls += core.atomic_calls; + producer_summary.poll_calls += core.poll_calls; + producer_summary.poll_batch_records += core.poll_batch_records; + producer_summary.dcci_records += core.dcci_records; + producer_summary.dcci_calls += core.dcci_calls; + producer_summary.dcci_lines += core.dcci_lines; + producer_summary.dropped_records += core.dropped; + if (!atomic_trace_enabled) { + if (core.atomic_calls != 0 || core.poll_calls != 0 || core.poll_batch_records != 0) { + std::fprintf( + stderr, + "phase-only swimlane worker %u unexpectedly reports atomic counters: calls=%u polls=%u batches=%u\n", + worker, core.atomic_calls, core.poll_calls, core.poll_batch_records + ); + return false; + } + continue; + } + if (core.poll_calls > core.atomic_calls || + (core.poll_calls == 0) != (core.poll_batch_records == 0)) { + std::fprintf( + stderr, + "atomic swimlane worker %u has invalid counters: calls=%u polls=%u batches=%u\n", + worker, core.atomic_calls, core.poll_calls, core.poll_batch_records + ); + return false; + } + producer_summary.atomic_records += + static_cast(core.atomic_calls) - + core.poll_calls + core.poll_batch_records; + } + producer_summary.clock_baseline_records = atomic_trace_enabled ? 2ULL * kWorkers : 0; + + // 先写同目录临时文件，全部记录写完并关闭后再 rename 替换，避免把半截 JSON + // 当成有效采集；这里没有 fsync 文件和目录，不承诺掉电后的持久化原子性。 + const std::string temporary_path = output_path + ".tmp"; + std::FILE *output = std::fopen(temporary_path.c_str(), "wb"); + if (output == nullptr) { + std::fprintf( + stderr, "Cannot open swimlane output %s: %s\n", temporary_path.c_str(), std::strerror(errno) + ); + return false; + } + + // 采用固定 1 MiB stdio 缓冲并逐核流式写出；默认 256 batch 时约 86 万条， + // 无论实际 batch 数是多少都不在 host 侧一次性聚合全部 JSON 记录。 + std::vector output_buffer(1U << 20); + std::setvbuf(output, output_buffer.data(), _IOFBF, output_buffer.size()); + std::fprintf( + output, + "{\n\"l2_swimlane_level\":%u,\n" + "\"metadata\":{\"tensormap_mode\":\"%s\"," + "\"clock_freq_hz\":%llu,\"num_cores\":%u," + "\"trace_schema_version\":%u,\"final_barrier\":\"%s\"," + "\"winner_workload\":{\"mode\":\"%s\"," + "\"counts\":{\"qk\":%u,\"sf\":%u,\"pv\":%u,\"up\":%u}," + "\"unit\":\"%s\",\"input_pattern\":\"%s\"," + "\"engine_mapping\":%s},\"core_types\":[", + atomic_trace_enabled ? 4U : 1U, + PTO_FDWIC_SHARED_MAP ? "shared" : "private", + static_cast(header.frequency_hz), kWorkers, + header.version, + FinalBarrierShapeName(final_barrier_shape), + workload_mode == WinnerWorkloadMode::RealCompute ? "real-compute" : "scalar-nop", + workload_counts.qk, workload_counts.sf, workload_counts.pv, workload_counts.up, + workload_mode == WinnerWorkloadMode::RealCompute + ? "complete_128x128_engine_pipeline_iteration" + : "scalar_nop_instruction", + workload_pattern, + workload_mode == WinnerWorkloadMode::RealCompute + ? "{\"qk\":\"cube_matmul\",\"sf\":\"vector_add\"," + "\"pv\":\"cube_matmul\",\"up\":\"vector_mul\"}" + : "null" + ); + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + std::fprintf(output, "%s\"%s\"", worker == 0 ? "" : ",", worker < kAicWorkers ? "aic" : "aiv"); + } + // schema-v5 无论是否开启 atomic 都导出 producer summary；phase-only 的 + // atomic/clock 字段为零，离线分析仍可独立证明 records 与 dropped 闭合。 + std::fprintf( + output, + "],\"fdwic_summary\":{\"records\":%llu,\"atomic_records\":%llu," + "\"clock_baseline_records\":%llu,\"atomic_calls\":%llu," + "\"batched_poll_calls\":%llu,\"poll_batch_records\":%llu," + "\"dcci_records\":%llu,\"dcci_calls\":%llu," + "\"dcci_lines\":%llu," + "\"dropped_records\":%llu}", + static_cast(producer_summary.records), + static_cast(producer_summary.atomic_records), + static_cast(producer_summary.clock_baseline_records), + static_cast(producer_summary.atomic_calls), + static_cast(producer_summary.poll_calls), + static_cast(producer_summary.poll_batch_records), + static_cast(producer_summary.dcci_records), + static_cast(producer_summary.dcci_calls), + static_cast(producer_summary.dcci_lines), + static_cast(producer_summary.dropped_records) + ); + std::fprintf( + output, + "},\n\"aicore_tasks\":[],\n\"aicpu_tasks\":[],\n" + "\"aicpu_scheduler_phases\":[],\n\"aicpu_orchestrator_phases\":[],\n\"fdwic_events\":[\n" + ); + // fdwic_events 每行固定十列：core、block、lane、task、function、phase、起止周期、flags、aux。 + + bool success = true; + bool first_record = true; + uint64_t exported_records = 0; + TraceExportSummary observed_summary; + std::vector + physical_scratch(kTraceRecordsPerCore); + std::vector + decoded_scratch(kTraceRecordsPerCore); +#if PTO_FDWIC_SHARED_MAP + std::vector + submit_claim_scratch(shared_plan.total_tasks); + std::vector logical_scratch; +#endif + constexpr int32_t kTracePhaseCount = static_cast(TracePhase::Count); + for (uint32_t worker = 0; worker < kWorkers && success; ++worker) { + // 每次只读取一个 worker 的有效区间；完整 trace 缓冲无需整体回拷。 + const uint32_t available = header.cores[worker].count; + if (available > header.records_per_core) { + std::fprintf( + stderr, "Trace core %u count %u exceeds capacity %u.\n", worker, available, + header.records_per_core + ); + success = false; + break; + } + if (available != 0 && + !read_records( + worker, available, physical_scratch.data() + )) { + success = false; + break; + } + uint64_t decode_anchor = 0; + uint64_t decode_finish = 0; +#if PA_BUILD_COMPACT_GENERIC_TRACE + decode_anchor = + state.results[worker].startup_barrier_begin; + decode_finish = state.results[worker].finish_cycle; +#endif + if (!DecodeTraceStorageRecords( + physical_scratch.data(), available, + decode_anchor, decode_finish, + decoded_scratch.data() + )) { + std::fprintf( + stderr, + "Trace core %u has an invalid compact generic " + "record stream.\n", + worker + ); + success = false; + break; + } +#if PTO_FDWIC_SHARED_MAP + if (state.results[worker].submits != + shared_plan.total_tasks) { + std::fprintf( + stderr, + "Trace core %u submit count %llu does not match " + "shared plan size %u.\n", + worker, + static_cast( + state.results[worker].submits + ), + shared_plan.total_tasks + ); + success = false; + break; + } + if (shared_plan.total_tasks != 0 && + !read_submit_claim_records( + worker, shared_plan.total_tasks, + submit_claim_scratch.data() + )) { + success = false; + break; + } + if (!ExpandSharedTraceRecords( + worker, decoded_scratch.data(), available, + submit_claim_scratch.data(), shared_plan, + &logical_scratch + )) { + std::fprintf( + stderr, + "Trace core %u has an invalid shared " + "Submit/Claim endpoint stream.\n", + worker + ); + success = false; + break; + } + const TraceRecord *records = logical_scratch.data(); + const uint32_t logical_available = + static_cast(logical_scratch.size()); +#else + const TraceRecord *records = decoded_scratch.data(); + const uint32_t logical_available = available; +#endif + const TraceCoreState &core = header.cores[worker]; + uint64_t core_atomic_calls = 0; + uint64_t core_poll_calls = 0; + uint32_t core_atomic_records = 0; + uint32_t core_poll_batch_records = 0; + uint32_t core_clock_records = 0; + uint32_t core_plain_clock_records = 0; + uint32_t core_dependency_clock_records = 0; + uint32_t core_dcci_records = 0; + uint32_t core_observer_dcci_records = 0; + uint64_t core_dcci_calls = 0; + uint64_t core_dcci_lines = 0; + bool dependency_applied = false; + bool direct_result_used_return_ready = false; + bool direct_result_used_source_issue = false; +#if PTO_FDWIC_SHARED_MAP + SharedSparseTraceValidator sparse_trace_validator( + &shared_plan + ); +#else + SharedSparseTraceValidator sparse_trace_validator; +#endif + for (uint32_t index = 0; + index < logical_available; ++index) { + const TraceRecord &record = records[index]; + const bool atomic_record = record.phase == static_cast(TracePhase::Atomic); + const bool claim_record = record.phase == static_cast(TracePhase::Claim); + const bool clock_record = record.phase == static_cast(TracePhase::ClockBaseline); + const bool dcci_record = + record.phase == + static_cast(TracePhase::Dcci); + const bool atomic_schema_valid = !atomic_record || + AtomicRecordSchemaValid(record, atomic_trace_enabled); + const bool claim_schema_valid = !claim_record || + ((record.flags & ~(kClaimWon | kClaimAttempted)) == 0 && + ((record.flags & kClaimWon) == 0 || (record.flags & kClaimAttempted) != 0) && + record.auxiliary <= 1); + const bool clock_schema_valid = !clock_record || + (atomic_trace_enabled && ClockRecordSchemaValid(record)); + const bool dcci_schema_valid = + !dcci_record || DcciRecordSchemaValid(record); + bool record_valid = record.end_cycle >= record.start_cycle && + record.phase < kTracePhaseCount && record.task_id >= -1 && + record.function_id >= -1 && atomic_schema_valid && + claim_schema_valid && clock_schema_valid && + dcci_schema_valid; + if (record_valid && !sparse_trace_validator.Observe(record)) { + record_valid = false; + } + if (!record_valid) { + std::fprintf( + stderr, + "Invalid trace record at worker=%u index=%u: phase=%u lane=%d block=%d core=%d " + "start=%llu end=%llu flags=0x%08x aux=%u\n", + worker, index, static_cast(record.phase), + core.lane, core.block_id, core.core_idx, + static_cast(record.start_cycle), + static_cast(record.end_cycle), record.flags, + static_cast(record.auxiliary) + ); + success = false; + break; + } + if (atomic_record) { + ++core_atomic_records; + const uint32_t call_count = AtomicRecordCallCount(record); + core_atomic_calls += call_count; + if ((record.flags & kAtomicPollBatch) != 0) { + core_poll_calls += call_count; + ++core_poll_batch_records; + } else if ((record.flags & kAtomicResultUsed) != 0) { + if ((record.flags & kAtomicReturnReady) != 0) { + direct_result_used_return_ready = true; + } else { + direct_result_used_source_issue = true; + } + } + } else if (clock_record) { + ++core_clock_records; + if ((record.flags & kClockAtomicDependency) != 0) { + ++core_dependency_clock_records; + dependency_applied = (record.flags & kClockAtomicDependencyApplied) != 0; + } else { + ++core_plain_clock_records; + } + } else if (dcci_record) { + ++core_dcci_records; + if (record.auxiliary == + static_cast( + DcciSite::ObserverTraceExport + )) { + ++core_observer_dcci_records; + } + core_dcci_calls += DcciRecordCallCount(record); + core_dcci_lines += DcciRecordLineCount(record); + } + std::fprintf( + output, + "%s[%d,%d,%d,%d,%d,\"%s\",%llu,%llu,%u,%u]", + first_record ? "" : ",\n", core.core_idx, core.block_id, core.lane, record.task_id, + record.function_id, TracePhaseName(static_cast(record.phase)), + static_cast(record.start_cycle), + static_cast(record.end_cycle), record.flags, + static_cast(record.auxiliary) + ); + first_record = false; + ++exported_records; + } + if (!success) break; + bool core_closed = true; + if (atomic_trace_enabled) { + const uint64_t expected_atomic_records = + static_cast(core.atomic_calls) - core.poll_calls + core.poll_batch_records; + core_closed = core_atomic_records == expected_atomic_records && + core_atomic_calls == core.atomic_calls && core_poll_calls == core.poll_calls && + core_poll_batch_records == core.poll_batch_records && core_clock_records == 2 && + core_plain_clock_records == 1 && core_dependency_clock_records == 1 && + (!dependency_applied || !direct_result_used_source_issue) && + (dependency_applied || !direct_result_used_return_ready); + } else { + core_closed = core_atomic_records == 0 && core_atomic_calls == 0 && core_poll_calls == 0 && + core_poll_batch_records == 0 && core_clock_records == 0; + } + core_closed &= sparse_trace_validator.Closed(); + core_closed &= + core_dcci_records == core.dcci_records && + core_observer_dcci_records == 1 && + core_dcci_calls == core.dcci_calls && + core_dcci_lines == core.dcci_lines; + if (!core_closed) { + std::fprintf( + stderr, + "swimlane closure failed on worker=%u: physical_atomic=%u logical_atomic=%llu/%u " + "poll_calls=%llu/%u poll_batches=%u/%u " + "dcci_records=%u/%u observer_dcci=%u/1 " + "dcci_calls=%llu/%u " + "dcci_lines=%llu/%u clock=%u plain=%u dependency=%u " + "dependency_applied=%s direct_ready=%s direct_issue=%s " + "efdrains=%u claims=%u winners=%u materializes=%u fanins=%u " + "registers=%u register_metadata=%u tails=%u submits=%u\n", + worker, core_atomic_records, static_cast(core_atomic_calls), + core.atomic_calls, static_cast(core_poll_calls), core.poll_calls, + core_poll_batch_records, core.poll_batch_records, + core_dcci_records, core.dcci_records, + core_observer_dcci_records, + static_cast(core_dcci_calls), + core.dcci_calls, + static_cast(core_dcci_lines), + core.dcci_lines, core_clock_records, + core_plain_clock_records, core_dependency_clock_records, + dependency_applied ? "yes" : "no", direct_result_used_return_ready ? "yes" : "no", + direct_result_used_source_issue ? "yes" : "no", + sparse_trace_validator.EfDrainCount(), + sparse_trace_validator.ClaimCount(), + sparse_trace_validator.WinnerCount(), + sparse_trace_validator.MaterializeCount(), + sparse_trace_validator.FaninCount(), + sparse_trace_validator.RegisterCount(), + sparse_trace_validator.RegisterMetadataCount(), + sparse_trace_validator.WinnerTailCount(), + sparse_trace_validator.SubmitCount() + ); + success = false; + break; + } + observed_summary.records += logical_available; + observed_summary.atomic_records += core_atomic_records; + observed_summary.clock_baseline_records += core_clock_records; + observed_summary.atomic_calls += core_atomic_calls; + observed_summary.poll_calls += core_poll_calls; + observed_summary.poll_batch_records += core_poll_batch_records; + observed_summary.dcci_records += core_dcci_records; + observed_summary.dcci_calls += core_dcci_calls; + observed_summary.dcci_lines += core_dcci_lines; + observed_summary.dropped_records += core.dropped; + } + if (success && !SameTraceSummary(producer_summary, observed_summary)) { + std::fprintf( + stderr, + "swimlane producer/raw summary mismatch: records=%llu/%llu atomic_records=%llu/%llu " + "atomic_calls=%llu/%llu poll_calls=%llu/%llu " + "poll_batches=%llu/%llu dcci_records=%llu/%llu " + "dcci_calls=%llu/%llu dcci_lines=%llu/%llu " + "clock=%llu/%llu\n", + static_cast(observed_summary.records), + static_cast(producer_summary.records), + static_cast(observed_summary.atomic_records), + static_cast(producer_summary.atomic_records), + static_cast(observed_summary.atomic_calls), + static_cast(producer_summary.atomic_calls), + static_cast(observed_summary.poll_calls), + static_cast(producer_summary.poll_calls), + static_cast(observed_summary.poll_batch_records), + static_cast(producer_summary.poll_batch_records), + static_cast(observed_summary.dcci_records), + static_cast(producer_summary.dcci_records), + static_cast(observed_summary.dcci_calls), + static_cast(producer_summary.dcci_calls), + static_cast(observed_summary.dcci_lines), + static_cast(producer_summary.dcci_lines), + static_cast(observed_summary.clock_baseline_records), + static_cast(producer_summary.clock_baseline_records) + ); + success = false; + } + if (success) std::fprintf(output, "\n]}\n"); + if (std::ferror(output) != 0) { + std::fprintf(stderr, "Failed while writing swimlane output %s.\n", temporary_path.c_str()); + success = false; + } + if (std::fclose(output) != 0) { + std::fprintf(stderr, "Failed to close swimlane output %s: %s\n", temporary_path.c_str(), std::strerror(errno)); + success = false; + } + if (success && std::rename(temporary_path.c_str(), output_path.c_str()) != 0) { + std::fprintf( + stderr, "Cannot finalize swimlane output %s: %s\n", output_path.c_str(), std::strerror(errno) + ); + success = false; + } + if (!success) { + std::remove(temporary_path.c_str()); + return false; + } + std::printf( + "[SWIMLANE] raw_json=%s events=%llu\n", output_path.c_str(), + static_cast(exported_records) + ); + return true; +} + +template < + typename ReadRecords +#if PTO_FDWIC_SHARED_MAP + , typename ReadSubmitClaimRecords +#endif +> +inline bool AnalyzeSwimlaneRecords( + const TraceHeader &header, const SchedulerState &state, + ReadRecords read_records +#if PTO_FDWIC_SHARED_MAP + , ReadSubmitClaimRecords read_submit_claim_records +#endif +) { + if (!ValidateTraceHeader(header, "swimlane analysis")) return false; +#if PTO_FDWIC_SHARED_MAP + SharedHostTaskPlan shared_plan; + std::string shared_plan_error; + if (!BuildSharedHostTaskPlan( + state, &shared_plan, &shared_plan_error + )) { + std::fprintf( + stderr, + "swimlane analysis rejected invalid shared task plan: %s\n", + shared_plan_error.c_str() + ); + return false; + } +#endif + + // 第一组数组统计“每个 worker 在某阶段的累计时间”；task_durations 则保留重点阶段的单事件分布。 + constexpr uint32_t kTracePhaseCount = static_cast(TracePhase::Count); + constexpr TracePhase kDetailedPhases[] = { + TracePhase::EfDrain, TracePhase::Claim, TracePhase::Materialize, TracePhase::Register, + }; + static_assert( + kDetailedPhases[0] == TracePhase::EfDrain, + "inferred EfDrain task distribution expects detail slot zero" + ); + uint64_t cycles[kWorkers][kTracePhaseCount] = {}; + uint64_t counts[kWorkers][kTracePhaseCount] = {}; +#if PTO_FDWIC_SHARED_MAP + std::vector task_durations[2][ + static_cast(TaskKind::Count) + ][sizeof(kDetailedPhases) / sizeof(kDetailedPhases[0])]; +#else + std::vector task_durations[2][kTasksPerBatch][sizeof(kDetailedPhases) / sizeof(kDetailedPhases[0])]; +#endif + std::vector atomic_durations[2][static_cast(AtomicSite::Count)]; + uint64_t atomic_return_ready_counts[2][static_cast(AtomicSite::Count)] = {}; + std::vector atomic_poll_windows[2][static_cast(AtomicSite::Count)]; + uint64_t atomic_poll_calls[2][static_cast(AtomicSite::Count)] = {}; + std::vector clock_baselines[2]; + std::vector clock_dependency_baselines[2]; + uint64_t clock_dependency_applied[2] = {}; + std::vector + physical_scratch(kTraceRecordsPerCore); + std::vector + decoded_scratch(kTraceRecordsPerCore); +#if PTO_FDWIC_SHARED_MAP + std::vector + submit_claim_scratch(shared_plan.total_tasks); + std::vector logical_scratch; +#endif + for (uint32_t worker = 0; worker < kWorkers; ++worker) { +#if PTO_FDWIC_SHARED_MAP + SharedSparseTraceValidator sparse_trace_validator( + &shared_plan + ); +#else + SharedSparseTraceValidator sparse_trace_validator; +#endif + const uint32_t available = header.cores[worker].count; + const uint32_t count = std::min(available, header.records_per_core); + if (count != 0 && + !read_records( + worker, count, physical_scratch.data() + )) { + return false; + } + uint64_t decode_anchor = 0; + uint64_t decode_finish = 0; +#if PA_BUILD_COMPACT_GENERIC_TRACE + decode_anchor = + state.results[worker].startup_barrier_begin; + decode_finish = state.results[worker].finish_cycle; +#endif + if (!DecodeTraceStorageRecords( + physical_scratch.data(), count, + decode_anchor, decode_finish, + decoded_scratch.data() + )) { + std::fprintf( + stderr, + "swimlane analysis rejected worker %u invalid " + "compact generic record stream.\n", + worker + ); + return false; + } +#if PTO_FDWIC_SHARED_MAP + if (state.results[worker].submits != + shared_plan.total_tasks) { + std::fprintf( + stderr, + "swimlane analysis rejected worker %u submit " + "count %llu; shared plan has %u tasks.\n", + worker, + static_cast( + state.results[worker].submits + ), + shared_plan.total_tasks + ); + return false; + } + if (shared_plan.total_tasks != 0 && + !read_submit_claim_records( + worker, shared_plan.total_tasks, + submit_claim_scratch.data() + )) { + return false; + } + if (!ExpandSharedTraceRecords( + worker, decoded_scratch.data(), count, + submit_claim_scratch.data(), shared_plan, + &logical_scratch + )) { + std::fprintf( + stderr, + "swimlane analysis rejected worker %u invalid " + "shared Submit/Claim endpoint stream.\n", + worker + ); + return false; + } + const TraceRecord *records = logical_scratch.data(); + const uint32_t logical_count = + static_cast(logical_scratch.size()); +#else + const TraceRecord *records = decoded_scratch.data(); + const uint32_t logical_count = count; +#endif + for (uint32_t index = 0; + index < logical_count; ++index) { + const TraceRecord &record = records[index]; + if (record.phase >= kTracePhaseCount || + record.end_cycle < record.start_cycle) { + // 分析器面对单条坏记录选择跳过；严格导出路径会直接拒绝，二者服务于不同诊断目的。 + continue; + } + if (!sparse_trace_validator.Observe(record)) { + std::fprintf( + stderr, + "swimlane analysis rejected shared sparse flow at " + "worker=%u index=%u task=%d function=%d start=%llu end=%llu " + "flags=0x%08x aux=%u\n", + worker, index, record.task_id, record.function_id, + static_cast(record.start_cycle), + static_cast(record.end_cycle), + record.flags, record.auxiliary + ); + return false; + } + const uint32_t phase = static_cast(record.phase); + const uint64_t duration = record.end_cycle - record.start_cycle; +#if PTO_FDWIC_SHARED_MAP + // shared raw 不再为 EfDrain 单写记录。Submit 在状态机中闭合时， + // 用 Submit.start -> Claim.start 恢复同一业务区间，使控制台 + // phase 聚合与 task 分布继续保持原口径。 + if (record.phase == + static_cast(TracePhase::Submit)) { + const uint64_t efdrain_begin = + sparse_trace_validator.LastEfDrainBegin(); + const uint64_t efdrain_end = + sparse_trace_validator.LastEfDrainEnd(); + if (efdrain_end < efdrain_begin) { + return false; + } + const uint64_t efdrain_duration = + efdrain_end - efdrain_begin; + const uint32_t efdrain_phase = + static_cast(TracePhase::EfDrain); + cycles[worker][efdrain_phase] += efdrain_duration; + ++counts[worker][efdrain_phase]; + const uint32_t role_index = + state.results[worker].role == + static_cast(CoreRole::Aic) + ? 0U + : 1U; + const SharedHostPlannedTask *task = + shared_plan.TaskAt( + static_cast(record.task_id) + ); + if (task == nullptr) { + return false; + } + task_durations[role_index][ + static_cast(task->kind) + ][0].push_back(efdrain_duration); + } +#endif + const bool atomic_poll_batch = + record.phase == static_cast(TracePhase::Atomic) && + (record.flags & kAtomicPollBatch) != 0; + // PollBatch 的 duration 是一次等待 episode 的包络，允许夹着其他直接 + // atomic/调度代码；不能混入“Atomic 单次括号”的累计时间或分位数。 + if (!atomic_poll_batch) { + cycles[worker][phase] += duration; + ++counts[worker][phase]; + } + if (record.phase == static_cast(TracePhase::Atomic) && + record.auxiliary < static_cast(AtomicSite::Count)) { + const uint32_t role_index = + state.results[worker].role == static_cast(CoreRole::Aic) ? 0U : 1U; + if (atomic_poll_batch) { + atomic_poll_windows[role_index][record.auxiliary].push_back(duration); + atomic_poll_calls[role_index][record.auxiliary] += AtomicRecordCallCount(record); + } else { + atomic_durations[role_index][record.auxiliary].push_back(duration); + atomic_return_ready_counts[role_index][record.auxiliary] += + (record.flags & kAtomicReturnReady) != 0; + } + } + if (record.phase == static_cast(TracePhase::ClockBaseline)) { + const uint32_t role_index = + state.results[worker].role == static_cast(CoreRole::Aic) ? 0U : 1U; + if ((record.flags & kClockAtomicDependency) != 0) { + clock_dependency_baselines[role_index].push_back(duration); + clock_dependency_applied[role_index] += + (record.flags & kClockAtomicDependencyApplied) != 0; + } else { + clock_baselines[role_index].push_back(duration); + } + } + if (record.task_id >= 0) { + const uint32_t role_index = + state.results[worker].role == static_cast(CoreRole::Aic) ? 0U : 1U; +#if PTO_FDWIC_SHARED_MAP + const SharedHostPlannedTask *task = + shared_plan.TaskAt( + static_cast(record.task_id) + ); + if (task == nullptr) { + return false; + } + const uint32_t kind = + static_cast(task->kind); +#else + const uint32_t kind = static_cast(record.task_id) % kTasksPerBatch; +#endif + for (uint32_t detail = 0; detail < sizeof(kDetailedPhases) / sizeof(kDetailedPhases[0]); ++detail) { + if (phase == static_cast(kDetailedPhases[detail])) { + task_durations[role_index][kind][detail].push_back(duration); + } + } + } + } + if (!sparse_trace_validator.Closed()) { + std::fprintf( + stderr, + "swimlane analysis rejected an unclosed shared sparse flow on " + "worker=%u: efdrains=%u claims=%u winners=%u " + "materializes=%u fanins=%u registers=%u " + "register_metadata=%u tails=%u submits=%u\n", + worker, sparse_trace_validator.EfDrainCount(), + sparse_trace_validator.ClaimCount(), + sparse_trace_validator.WinnerCount(), + sparse_trace_validator.MaterializeCount(), + sparse_trace_validator.FaninCount(), + sparse_trace_validator.RegisterCount(), + sparse_trace_validator.RegisterMetadataCount(), + sparse_trace_validator.WinnerTailCount(), + sparse_trace_validator.SubmitCount() + ); + return false; + } + } + + const CoreRole roles[] = {CoreRole::Aic, CoreRole::Aiv}; + const char *role_names[] = {"AIC", "AIV"}; + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t phase = 0; phase < kTracePhaseCount; ++phase) { + std::vector role_cycles; + uint64_t record_count = 0; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + if (state.results[worker].role != static_cast(roles[role_index])) continue; + role_cycles.push_back(cycles[worker][phase]); + record_count += counts[worker][phase]; + } + const Uint64Distribution summary = SummarizeUint64(role_cycles); + std::printf( + "[TRACE_PHASE] role=%s phase=%s records=%llu accumulated_us_median=%.3f " + "accumulated_us_p95=%.3f accumulated_us_max=%.3f\n", + role_names[role_index], TracePhaseName(phase), + static_cast(record_count), summary.median / 1000.0, + static_cast(summary.p95) / 1000.0, + static_cast(summary.maximum) / 1000.0 + ); + } + } + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + const Uint64Distribution summary = SummarizeUint64(clock_baselines[role_index]); + if (!clock_baselines[role_index].empty()) { + std::printf( + "[TRACE_CLOCK] role=%s samples=%zu definition=consecutive-sys-cnt-reads " + "median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], clock_baselines[role_index].size(), summary.median, + static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + const Uint64Distribution dependency_summary = + SummarizeUint64(clock_dependency_baselines[role_index]); + if (!clock_dependency_baselines[role_index].empty()) { + std::printf( + "[TRACE_CLOCK] role=%s samples=%zu definition=atomic-return-dependency-hook " + "dependency_applied=%llu/%zu median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], clock_dependency_baselines[role_index].size(), + static_cast(clock_dependency_applied[role_index]), + clock_dependency_baselines[role_index].size(), dependency_summary.median, + static_cast(dependency_summary.p95), + static_cast(dependency_summary.maximum) + ); + } + } + // Atomic 只报告原始括号分布，不扣除计时底噪，也不把 total_cycles + // 解释成可与 Submit 墙钟直接相加的“atomic 占比”。return-ready 只表示 + // 本核可消费返回值，不表示其他核已经观察到更新。 + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t site = 0; site < static_cast(AtomicSite::Count); ++site) { + const std::vector &durations = atomic_durations[role_index][site]; + if (durations.empty()) continue; + const Uint64Distribution summary = SummarizeUint64(durations); + const AtomicOp op = AtomicSiteOp(static_cast(site)); + const uint64_t return_ready_count = atomic_return_ready_counts[role_index][site]; + const char *boundary = return_ready_count == durations.size() + ? "return-ready" + : (return_ready_count == 0 ? "source-issue" : "mixed"); + std::printf( + "[TRACE_ATOMIC] role=%s site=%s op=%s events=%zu boundary=%s " + "return_ready=%llu/%zu bracket_cycles_total=%llu median_ns=%.1f " + "p95_ns=%llu max_ns=%llu\n", + role_names[role_index], AtomicSiteName(site), AtomicOpName(static_cast(op)), + durations.size(), boundary, static_cast(return_ready_count), + durations.size(), static_cast(summary.total), summary.median, + static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + } + // 等待聚合只报告 episode 数、精确逻辑调用数与包络分布。window 不能除以 + // calls 当作单次 atomic latency，也不能与 Submit 墙钟直接相加。 + for (uint32_t role_index = 0; role_index < 2; ++role_index) { + for (uint32_t site = 0; site < static_cast(AtomicSite::Count); ++site) { + const std::vector &windows = atomic_poll_windows[role_index][site]; + if (windows.empty()) continue; + const Uint64Distribution summary = SummarizeUint64(windows); + std::printf( + "[TRACE_ATOMIC_POLL] role=%s site=%s op=%s episodes=%zu logical_calls=%llu " + "window_definition=wait-episode-envelope median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], AtomicSiteName(site), + AtomicOpName(static_cast(AtomicSiteOp(static_cast(site)))), + windows.size(), static_cast(atomic_poll_calls[role_index][site]), + summary.median, static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + } + const char *kind_names[] = {"Alloc", "QK", "SF", "PV", "UP"}; + // 单事件统计按 role 与 task kind 展开，可区分“该 role 真实参与”与“只回放前端”的成本。 + for (uint32_t role_index = 0; role_index < 2; ++role_index) { +#if PTO_FDWIC_SHARED_MAP + for (uint32_t kind = 0; + kind < static_cast(TaskKind::Count); + ++kind) { +#else + for (uint32_t kind = 0; kind < kTasksPerBatch; ++kind) { +#endif + for (uint32_t detail = 0; detail < sizeof(kDetailedPhases) / sizeof(kDetailedPhases[0]); ++detail) { + const std::vector &durations = task_durations[role_index][kind][detail]; + const Uint64Distribution summary = SummarizeUint64(durations); + std::printf( + "[TRACE_TASK] role=%s kind=%s phase=%s events=%zu median_ns=%.1f p95_ns=%llu max_ns=%llu\n", + role_names[role_index], kind_names[kind], + TracePhaseName(static_cast(kDetailedPhases[detail])), durations.size(), summary.median, + static_cast(summary.p95), + static_cast(summary.maximum) + ); + } + } + } + return true; +} + +inline uint64_t DependencyEdgeSignatureHost( + uint32_t consumer, uint32_t producer +) { + uint64_t value = + (static_cast(consumer) << 32U) | producer; + value ^= value >> 30U; + value *= 0xBF58476D1CE4E5B9ULL; + value ^= value >> 27U; + value *= 0x94D049BB133111EBULL; + value ^= value >> 31U; + return value; +} + +#if !PTO_FDWIC_SHARED_MAP +inline uint64_t ExpectedPaDependencySignature(uint32_t batches) { + uint64_t signature = 0; + for (uint32_t batch = 0; batch < batches; ++batch) { + const uint32_t alloc = batch * kTasksPerBatch; + const uint32_t qk = alloc + 1; + const uint32_t sf = alloc + 2; + const uint32_t pv = alloc + 3; + const uint32_t up = alloc + 4; + // SF<-QK、PV<-SF；UP 的 SF max/sum 去重为 SF 一条，再依赖 + // PV 和本 batch Alloc 建立的 accumulator。BeginPaBatch 后的 Alloc + // 会替换 orchestration 中三条累计输出引用，不跨 batch 沿用前一 UP。 + signature ^= DependencyEdgeSignatureHost(sf, qk); + signature ^= DependencyEdgeSignatureHost(pv, sf); + signature ^= DependencyEdgeSignatureHost(up, sf); + signature ^= DependencyEdgeSignatureHost(up, pv); + signature ^= DependencyEdgeSignatureHost(up, alloc); + } + return signature; +} +#endif + +#if PTO_FDWIC_SHARED_MAP +inline uint64_t ExpectedPaDependencySignature( + const SharedHostTaskPlan &plan +) { + uint64_t signature = 0; + for (const SharedHostBatchPlan &batch : plan.batches) { + uint32_t accumulator_writer = batch.batch_start; + for (uint32_t group = 0; + group < batch.group_count; ++group) { + const uint32_t qk = + batch.batch_start + 1U + 4U * group; + const uint32_t sf = qk + 1U; + const uint32_t pv = qk + 2U; + const uint32_t up = qk + 3U; + // 每组始终有 SF<-QK、PV<-SF、UP<-SF/PV 三条 fresh + // 依赖。三个 accumulator symbol 在 CollectFanin 中去重为 + // 一条：首组 writer 是 Alloc，后续组 writer 是前一 UP。 + signature ^= DependencyEdgeSignatureHost(sf, qk); + signature ^= DependencyEdgeSignatureHost(pv, sf); + signature ^= DependencyEdgeSignatureHost(up, sf); + signature ^= DependencyEdgeSignatureHost(up, pv); + signature ^= + DependencyEdgeSignatureHost(up, accumulator_writer); + accumulator_writer = up; + } + } + return signature; +} +#endif + +inline uint32_t SharedTensorMapHashHost(uint64_t address) { +#if PTO_FDWIC_TENSORMAP_RING_CAP == 16384 + (void)address; + return 0; +#else + address *= 0x9E3779B97F4A7C15ULL; + return static_cast( + address >> (64 - kMapBucketShift) + ) & kMapBucketMask; +#endif +} + +inline void SharedLogicalHashWord(uint64_t *hash, uint64_t value) { + // 固定按小端字节折叠，不依赖 host struct padding；private/shared 后续 + // 都以 bucket、region 和 producer 的同一字段序列生成可比较签名。 + for (uint32_t byte = 0; byte < 8; ++byte) { + *hash ^= (value >> (byte * 8U)) & 0xFFU; + *hash *= 1099511628211ULL; + } +} + +#if PTO_FDWIC_SHARED_MAP +struct SharedTensorMapValidation { + bool protocol_ok = true; + uint64_t total_appends = 0; + uint64_t physical_entries = 0; + uint64_t logical_entries = 0; + uint64_t logical_signature = 1469598103934665603ULL; +}; + +inline int64_t SharedInsertTurnValueHost( + const SharedTensorMapSidecar &map, uint32_t lane +) { + return lane == 0 + ? map.committed_tasks.value + : map.insert_turn_extra[lane - 1U].value; +} + +inline int64_t SharedExpectedUnusedInsertTurnHost( + uint32_t lane +) { + return lane == 0 ? 0 : -1; +} + +inline SharedTensorMapValidation ValidateSharedTensorMap( + const SharedTensorMapSidecar &map, + const SharedHostTaskPlan &plan +) { + SharedTensorMapValidation validation; + for (uint32_t lane = 0; + lane < kSharedInsertTurnCapacity; ++lane) { + validation.protocol_ok &= + SharedInsertTurnValueHost(map, lane) == + SharedExpectedUnusedInsertTurnHost(lane); + } + validation.protocol_ok &= map.reclaim_upto.value == -1; + + // PA 的 fresh Output 由 shared_outputs 直接定位，唯一 ordinary + // output_view 又是 manual_dep，因此 region ring 仍为空；但每个 + // task（包括空 writer 集）都必须发布自己的 per-task 插入完成字； + // 该终态由 SchedulerState host oracle 单独逐 task 校验。 + const SharedRegionPayload zero_payload{}; + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + const int64_t head = map.buckets[bucket].head.value; + const int64_t tail = map.buckets[bucket].tail.value; + validation.protocol_ok &= head == 0 && tail == 0; + } + for (uint32_t slot = 0; slot < kMapCapacity; ++slot) { + validation.protocol_ok &= map.slots[slot].seq.value == -1; + validation.protocol_ok &= std::memcmp( + &map.slots[slot].payload, &zero_payload, + sizeof(zero_payload) + ) == 0; + } + // 每个 UP 按 max/sum/output 三个 accumulator symbol 发布不可变 + // history。首组前驱是本 batch 的 Alloc，后续组前驱是前一 UP； + // 非 UP task 与 plan 尾部必须保持空 history。 + const SharedWriterHistoryCell zero_history{}; + for (uint32_t task = 0; task < kMaxTasks; ++task) { + const SharedHostPlannedTask *planned = + plan.TaskAt(task); + const SharedWriterHistoryCell &history = + map.writer_history[task]; + if (planned == nullptr || + planned->kind != TaskKind::Up) { + validation.protocol_ok &= std::memcmp( + &history, &zero_history, + sizeof(zero_history) + ) == 0; + continue; + } + const int32_t expected_previous = + planned->group_index == 0 + ? static_cast( + planned->batch_start + ) + : static_cast(task) - 4; + validation.protocol_ok &= + history.magic == kSharedWriterHistoryMagic && + history.writer_task == static_cast(task) && + history.count == 3 && + history.reserved == 0; + bool slots_seen[3] = {}; + for (uint32_t index = 0; + index < history.count && index < 3; ++index) { + const SharedWriterHistoryRecord &record = + history.entries[index]; + const uint32_t key_base = + planned->batch_start * + kSharedOutputMaxPerTask + + 1U; + const bool key_ok = + record.symbol_key >= key_base && + record.symbol_key < key_base + 3U; + validation.protocol_ok &= + key_ok && + record.previous_writer == expected_previous; + if (key_ok) { + const uint32_t slot = + record.symbol_key - key_base; + validation.protocol_ok &= + !slots_seen[slot]; + slots_seen[slot] = true; + } + } + validation.protocol_ok &= + slots_seen[0] && slots_seen[1] && + slots_seen[2]; + } + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + validation.protocol_ok &= + map.reader_done[worker].value == -1; + } + return validation; +} +#endif + +#if PTO_FDWIC_SHARED_MAP +inline uint32_t ExpectedOutputCount(TaskKind kind) { + switch (kind) { + case TaskKind::Alloc: return 3; + case TaskKind::Qk: return 1; + case TaskKind::Sf: return 3; + case TaskKind::Pv: return 1; + case TaskKind::Up: return 0; + case TaskKind::Count: return 0; + } + return 0; +} +#else +inline uint32_t ExpectedOutputCount(uint32_t task_id) { + switch (static_cast(task_id % kTasksPerBatch)) { + case TaskKind::Alloc: return 3; + case TaskKind::Qk: return 1; + case TaskKind::Sf: return 3; + case TaskKind::Pv: return 1; + case TaskKind::Up: return 0; + case TaskKind::Count: return 0; + } + return 0; +} +#endif + +// host_support 只依赖 pa_model，不能反向 include 设备端 pa_frontend；这里保留 +// Case1 协议已固定的 descriptor 常量和 dtype 字节数，避免 host 校验引入设备代码。 +constexpr uint32_t kHostPaHeads = 16; +constexpr uint32_t kHostPaHeadDim = 128; +#if !PTO_FDWIC_SHARED_MAP +constexpr uint32_t kHostPaBlockSize = 128; +constexpr uint32_t kHostPaBlocksPerRequest = 64; +#endif +constexpr uint64_t kHostSyntheticOutputBase = 0x600000000ULL; +constexpr uint64_t kHostInvalidTaskId = UINT64_MAX; + +inline uint64_t HostElementSize(DataType dtype) { + switch (dtype) { + case DataType::Float32: + case DataType::Int32: + case DataType::Uint32: + return 4; + case DataType::Float16: + case DataType::Bfloat16: + case DataType::Int16: + case DataType::Uint16: + return 2; + case DataType::Int8: + case DataType::Uint8: + case DataType::Bool: + return 1; + case DataType::Int64: + case DataType::Uint64: + return 8; + case DataType::Count: + return 0; + } + return 0; +} + +#if PTO_FDWIC_SHARED_MAP +inline uint64_t ExpectedTaskOutputBytesForKind(TaskKind kind) { + constexpr uint64_t kOutputBytesByKind[kTasksPerBatch] = { + 10240, 524288, 264192, 8192, 0 + }; + const uint32_t index = static_cast(kind); + return index < kTasksPerBatch ? kOutputBytesByKind[index] : 0; +} +#else +inline uint64_t ExpectedTaskOutputBytes(uint32_t task_id) { + constexpr uint64_t kOutputBytesByKind[kTasksPerBatch] = { + 10240, 524288, 264192, 8192, 0 + }; + return kOutputBytesByKind[task_id % kTasksPerBatch]; +} +#endif + +#if !PTO_FDWIC_SHARED_MAP +inline uint64_t ExpectedCanonicalTaskBase(uint32_t task_id) { + constexpr uint64_t kTaskOffsetByKind[kTasksPerBatch] = { + 0, 10240, 534528, 798720, 806912 + }; + return static_cast(task_id / kTasksPerBatch) * 806912ULL + + kTaskOffsetByKind[task_id % kTasksPerBatch]; +} +#endif + +#if PTO_FDWIC_SHARED_MAP +inline uint64_t ExpectedSharedHeapShardSpan(uint64_t heap_size) { + const uint64_t raw = heap_size / kSharedHeapShards; + return raw / kOutputAlignment * kOutputAlignment; +} +#else +inline TensorDesc ExpectedCanonicalOutputDescriptor( + uint32_t task_id, uint32_t output_slot +) { + // canonical 地址保持 private 连续 heap 布局，只服务跨模式逻辑签名； + // shared 实际 descriptor 由下方 8-shard oracle 独立验证。 + const uint64_t task_base = ExpectedCanonicalTaskBase(task_id); + uint64_t output_offset = 0; + uint64_t buffer_size = 0; + uint32_t ndims = 0; + DataType dtype = DataType::Float32; + uint32_t shapes[kMaxTensorDims] = {}; + const TaskKind kind = static_cast(task_id % kTasksPerBatch); + if (kind == TaskKind::Alloc) { + if (output_slot == 0) { + buffer_size = 8192; + ndims = 2; + shapes[0] = kHostPaHeads; + shapes[1] = kHostPaHeadDim; + } else if (output_slot == 1 || output_slot == 2) { + output_offset = output_slot == 1 ? 8192 : 9216; + buffer_size = 64; + ndims = 1; + shapes[0] = kHostPaHeads; + } + } else if (kind == TaskKind::Qk && output_slot == 0) { + buffer_size = 524288; + ndims = 2; + shapes[0] = kHostPaHeads; + shapes[1] = kHostPaBlocksPerRequest * kHostPaBlockSize; + } else if (kind == TaskKind::Sf) { + if (output_slot == 0) { + buffer_size = 262144; + ndims = 2; + dtype = DataType::Bfloat16; + shapes[0] = kHostPaHeads; + shapes[1] = kHostPaBlocksPerRequest * kHostPaBlockSize; + } else if (output_slot == 1 || output_slot == 2) { + output_offset = output_slot == 1 ? 262144 : 263168; + buffer_size = 64; + ndims = 1; + shapes[0] = kHostPaHeads; + } + } else if (kind == TaskKind::Pv && output_slot == 0) { + buffer_size = 8192; + ndims = 2; + shapes[0] = kHostPaHeads; + shapes[1] = kHostPaHeadDim; + } + + TensorDesc expected{}; + expected.buffer_addr = kSyntheticHeapBase + task_base + output_offset; + expected.buffer_size = buffer_size; + expected.owner_task_id = task_id; + expected.start_offset = 0; + expected.version = 0; + expected.ndims = ndims; + expected.dtype = dtype; + expected.manual_dep = false; + expected.is_contiguous = true; + expected.child_memory = 0; + uint32_t stride = 1; + for (int32_t index = static_cast(ndims) - 1; index >= 0; --index) { + expected.strides[index] = stride; + stride *= shapes[index]; + } + expected.extent_elem_cache = stride; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + expected.shapes[index] = shapes[index]; + } + return expected; +} +#endif + +#if PTO_FDWIC_SHARED_MAP +inline TensorDesc ExpectedCanonicalOutputDescriptorForTask( + uint32_t task_id, uint32_t output_slot, TaskKind kind, + uint32_t group_block_count, uint64_t task_base +) { + uint64_t output_offset = 0; + uint64_t buffer_size = 0; + uint32_t ndims = 0; + DataType dtype = DataType::Float32; + uint32_t shapes[kMaxTensorDims] = {}; + if (kind == TaskKind::Alloc) { + if (output_slot == 0) { + buffer_size = 8192; + ndims = 2; + shapes[0] = kHostPaHeads; + shapes[1] = kHostPaHeadDim; + } else if (output_slot == 1 || output_slot == 2) { + output_offset = output_slot == 1 ? 8192 : 9216; + buffer_size = 64; + ndims = 1; + shapes[0] = kHostPaHeads; + } + } else if (kind == TaskKind::Qk && output_slot == 0) { + buffer_size = + static_cast(kHostPaHeads) * + group_block_count * kHostPaBlockSize * 4U; + ndims = 2; + shapes[0] = kHostPaHeads; + shapes[1] = group_block_count * kHostPaBlockSize; + } else if (kind == TaskKind::Sf) { + const uint64_t probabilities_size = + static_cast(kHostPaHeads) * + group_block_count * kHostPaBlockSize * 2U; + if (output_slot == 0) { + buffer_size = probabilities_size; + ndims = 2; + dtype = DataType::Bfloat16; + shapes[0] = kHostPaHeads; + shapes[1] = group_block_count * kHostPaBlockSize; + } else if (output_slot == 1 || output_slot == 2) { + output_offset = + probabilities_size + + (output_slot == 1 ? 0 : kOutputAlignment); + buffer_size = 64; + ndims = 1; + shapes[0] = kHostPaHeads; + } + } else if (kind == TaskKind::Pv && output_slot == 0) { + buffer_size = 8192; + ndims = 2; + shapes[0] = kHostPaHeads; + shapes[1] = kHostPaHeadDim; + } + + TensorDesc expected{}; + expected.buffer_addr = kSyntheticHeapBase + task_base + output_offset; + expected.buffer_size = buffer_size; + expected.owner_task_id = task_id; + expected.start_offset = 0; + expected.version = 0; + expected.ndims = ndims; + expected.dtype = dtype; + expected.manual_dep = false; + expected.is_contiguous = true; + expected.child_memory = 0; + uint32_t stride = 1; + for (int32_t index = static_cast(ndims) - 1; index >= 0; --index) { + expected.strides[index] = stride; + stride *= shapes[index]; + } + expected.extent_elem_cache = stride; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + expected.shapes[index] = shapes[index]; + } + return expected; +} + +inline TensorDesc ExpectedSharedOutputDescriptorAtBase( + const SharedHostPlannedTask &task, uint32_t output_slot, + uint64_t task_base +) { + TensorDesc expected = + ExpectedCanonicalOutputDescriptorForTask( + task.task_id, output_slot, task.kind, + task.group_block_count, + task.canonical_task_base + ); + const uint64_t output_offset = + expected.buffer_addr - kSyntheticHeapBase - + task.canonical_task_base; + expected.buffer_addr = + kSyntheticHeapBase + task_base + output_offset; + return expected; +} +#endif + +inline bool TensorDescFieldsMatch( + const TensorDesc &actual, const TensorDesc &expected +) { + if (actual.buffer_addr != expected.buffer_addr || + actual.buffer_size != expected.buffer_size || + actual.owner_task_id != expected.owner_task_id || + actual.start_offset != expected.start_offset || + actual.version != expected.version || actual.ndims != expected.ndims || + actual.dtype != expected.dtype || actual.manual_dep != expected.manual_dep || + actual.is_contiguous != expected.is_contiguous || + actual.child_memory != expected.child_memory || + actual.extent_elem_cache != expected.extent_elem_cache || + actual.ndims > kMaxTensorDims) { + return false; + } + // PA 的 descriptor 构造器只定义 [0,ndims) 的 shape/stride；payload + // arena 回绕后，inactive 维允许保留旧 task 字节。下游同样以 ndims + // 为边界，host 不能把未定义尾部要求为零并误报业务 descriptor 损坏。 + for (uint32_t index = 0; index < expected.ndims; ++index) { + if (actual.shapes[index] != expected.shapes[index] || + actual.strides[index] != expected.strides[index]) { + return false; + } + } + return true; +} + +#if PTO_FDWIC_SHARED_MAP +struct SharedOutputValidation { + bool protocol_ok = true; + uint64_t published_outputs = 0; + uint64_t allocated_bytes = 0; + uint64_t shard_bytes[kSharedHeapShards] = {}; + uint32_t first_bad_task = UINT32_MAX; + uint32_t first_bad_slot = UINT32_MAX; + const char *first_bad_reason = "none"; +}; + +struct SharedHeapInterval { + uint64_t begin; + uint64_t end; + uint32_t task_id; +}; + +inline SharedOutputValidation ValidateSharedOutputs( + const SharedTensorMapSidecar &map, + const SharedHostTaskPlan &plan, + uint64_t heap_size +) { + SharedOutputValidation validation; + const auto record = [&]( + bool condition, uint32_t task_id, + uint32_t slot, const char *reason + ) { + validation.protocol_ok &= condition; + if (!condition && + validation.first_bad_task == UINT32_MAX) { + validation.first_bad_task = task_id; + validation.first_bad_slot = slot; + validation.first_bad_reason = reason; + } + return condition; + }; + const TensorDesc zero_tensor{}; + const uint64_t shard_span = ExpectedSharedHeapShardSpan(heap_size); + std::vector intervals[kSharedHeapShards]; + for (uint32_t task_id = 0; task_id < kMaxTasks; ++task_id) { + const SharedHostPlannedTask *task = + plan.TaskAt(task_id); + const uint32_t expected_count = + task == nullptr + ? 0 + : ExpectedOutputCount(task->kind); + const SharedOutputCell &cell = map.shared_outputs[task_id]; + uint64_t task_base = 0; + if (expected_count != 0) { + const uint64_t output_bytes = task->output_bytes; + const uint32_t shard = task_id % kSharedHeapShards; + const uint64_t shard_begin = + static_cast(shard) * shard_span; + const uint64_t shard_end = shard_begin + shard_span; + const uint64_t address = cell.tensors[0].buffer_addr; + const bool address_ok = + address >= kSyntheticHeapBase && + output_bytes != 0 && + output_bytes <= shard_span; + if (address_ok) { + task_base = address - kSyntheticHeapBase; + } + const bool interval_ok = + address_ok && + task_base % kOutputAlignment == 0 && + task_base >= shard_begin && + task_base <= shard_end - output_bytes; + record( + interval_ok, task_id, UINT32_MAX, + "task heap interval" + ); + if (interval_ok) { + intervals[shard].push_back( + {task_base, task_base + output_bytes, task_id} + ); + } + } + for (uint32_t slot = 0; slot < kSharedOutputMaxPerTask; ++slot) { + const bool active = slot < expected_count; + if (!active) { + record( + cell.published[slot].value == -1, + task_id, slot, "inactive published" + ); + record( + cell.last_writer[slot].value == -1, + task_id, slot, "inactive last_writer" + ); + record( + std::memcmp( + &cell.tensors[slot], &zero_tensor, + sizeof(zero_tensor) + ) == 0, + task_id, slot, "inactive descriptor" + ); + continue; + } + int64_t expected_writer = + static_cast(task_id); + if (task->kind == TaskKind::Alloc) { + const SharedHostBatchPlan *batch = + plan.BatchAt(task->batch); + if (batch == nullptr) { + record( + false, task_id, slot, + "missing batch plan" + ); + } else if (batch->group_count != 0) { + expected_writer = + static_cast( + batch->final_up_task_id + ); + } + } + record( + cell.published[slot].value == + static_cast(task_id), + task_id, slot, "active published" + ); + record( + cell.last_writer[slot].value == expected_writer, + task_id, slot, "active last_writer" + ); + const TensorDesc expected = + ExpectedSharedOutputDescriptorAtBase( + *task, slot, task_base + ); + const bool descriptor_ok = + TensorDescFieldsMatch( + cell.tensors[slot], expected + ); + if (!descriptor_ok && + validation.first_bad_task == UINT32_MAX) { + std::printf( + "[SHARED_OUTPUT_DESCRIPTOR_FAILURE] " + "task=%u slot=%u actual={addr=%llu,size=%llu," + "owner=%llu,ndims=%u,shapes=%u/%u/%u/%u/%u," + "strides=%u/%u/%u/%u/%u} " + "expected={addr=%llu,size=%llu,owner=%llu," + "ndims=%u,shapes=%u/%u/%u/%u/%u," + "strides=%u/%u/%u/%u/%u}\n", + task_id, slot, + static_cast( + cell.tensors[slot].buffer_addr + ), + static_cast( + cell.tensors[slot].buffer_size + ), + static_cast( + cell.tensors[slot].owner_task_id + ), + cell.tensors[slot].ndims, + cell.tensors[slot].shapes[0], + cell.tensors[slot].shapes[1], + cell.tensors[slot].shapes[2], + cell.tensors[slot].shapes[3], + cell.tensors[slot].shapes[4], + cell.tensors[slot].strides[0], + cell.tensors[slot].strides[1], + cell.tensors[slot].strides[2], + cell.tensors[slot].strides[3], + cell.tensors[slot].strides[4], + static_cast( + expected.buffer_addr + ), + static_cast( + expected.buffer_size + ), + static_cast( + expected.owner_task_id + ), + expected.ndims, + expected.shapes[0], + expected.shapes[1], + expected.shapes[2], + expected.shapes[3], + expected.shapes[4], + expected.strides[0], + expected.strides[1], + expected.strides[2], + expected.strides[3], + expected.strides[4] + ); + } + record( + descriptor_ok, + task_id, slot, "active descriptor" + ); + ++validation.published_outputs; + } + } + for (uint32_t shard = 0; shard < kSharedHeapShards; ++shard) { + std::sort( + intervals[shard].begin(), intervals[shard].end(), + [](const SharedHeapInterval &left, const SharedHeapInterval &right) { + return left.begin < right.begin; + } + ); + uint64_t next = + static_cast(shard) * shard_span; + for (const SharedHeapInterval &interval : intervals[shard]) { + record( + interval.begin == next, + interval.task_id, UINT32_MAX, + "non-contiguous shard interval" + ); + next = interval.end; + } + validation.shard_bytes[shard] = + next - static_cast(shard) * shard_span; + validation.allocated_bytes += validation.shard_bytes[shard]; + } + return validation; +} +#endif + +struct NormalizedWriterEntry { + uint64_t buffer_addr; + uint64_t lo; + uint64_t hi; + uint32_t producer; +}; + +inline void AddNormalizedWriter( + std::vector buckets[kMapBuckets], + const TensorDesc &tensor, uint32_t producer +) { + const uint64_t element_size = HostElementSize(tensor.dtype); + uint64_t extent = tensor.is_contiguous ? 1 : tensor.extent_elem_cache; + if (tensor.is_contiguous) { + for (uint32_t index = 0; index < tensor.ndims; ++index) { + extent *= tensor.shapes[index]; + } + } + const uint64_t lo = tensor.start_offset * element_size; + const uint64_t hi = (tensor.start_offset + extent) * element_size; + buckets[SharedTensorMapHashHost(tensor.buffer_addr)].push_back( + {tensor.buffer_addr, lo, hi, producer} + ); +} + +inline TensorDesc ExpectedManualOutputView(uint32_t batch, uint32_t batches) { + TensorDesc view{}; + view.buffer_addr = kHostSyntheticOutputBase; + view.buffer_size = static_cast(batches) * kHostPaHeads * kHostPaHeadDim * 4; + view.owner_task_id = kHostInvalidTaskId; + view.start_offset = static_cast(batch) * kHostPaHeads * kHostPaHeadDim; + view.version = 0; + view.ndims = 2; + view.dtype = DataType::Float32; + view.manual_dep = true; + view.is_contiguous = true; + view.child_memory = 0; + view.shapes[0] = kHostPaHeads; + view.shapes[1] = kHostPaHeadDim; + view.extent_elem_cache = kHostPaHeads * kHostPaHeadDim; + view.strides[0] = kHostPaHeadDim; + view.strides[1] = 1; + return view; +} + +inline uint64_t FinishNormalizedWriterSignature( + std::vector buckets[kMapBuckets] +) { + uint64_t signature = 1469598103934665603ULL; + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + for (const NormalizedWriterEntry &entry : buckets[bucket]) { + SharedLogicalHashWord(&signature, bucket); + SharedLogicalHashWord(&signature, entry.buffer_addr); + SharedLogicalHashWord(&signature, entry.lo); + SharedLogicalHashWord(&signature, entry.hi); + SharedLogicalHashWord(&signature, entry.producer); + } + } + return signature; +} + +#if !PTO_FDWIC_SHARED_MAP +inline uint64_t ExpectedNormalizedWriterSignature( + uint32_t batches, uint32_t logical_floor +) { + std::vector by_bucket[kMapBuckets]; + for (uint32_t batch = 0; batch < batches; ++batch) { + const uint32_t alloc = batch * kTasksPerBatch; + const uint32_t up = alloc + 4; + if (up < logical_floor) { + continue; + } + // RegisterOutputs 的真实顺序是 max、sum、output、manual output_view。 + AddNormalizedWriter( + by_bucket, ExpectedCanonicalOutputDescriptor(alloc, 2), up + ); + AddNormalizedWriter( + by_bucket, ExpectedCanonicalOutputDescriptor(alloc, 1), up + ); + AddNormalizedWriter( + by_bucket, ExpectedCanonicalOutputDescriptor(alloc, 0), up + ); + AddNormalizedWriter(by_bucket, ExpectedManualOutputView(batch, batches), up); + } + return FinishNormalizedWriterSignature(by_bucket); +} +#endif + +#if PTO_FDWIC_SHARED_MAP +inline uint64_t ExpectedNormalizedWriterSignature( + const SharedHostTaskPlan &plan, uint32_t logical_floor +) { + std::vector by_bucket[kMapBuckets]; + for (const SharedHostBatchPlan &batch : plan.batches) { + if (batch.group_count == 0 || + batch.final_up_task_id < logical_floor) { + continue; + } + const SharedHostPlannedTask *alloc = + plan.TaskAt(batch.batch_start); + if (alloc == nullptr || + alloc->kind != TaskKind::Alloc) { + continue; + } + // RegisterOutputs 的真实顺序是 max、sum、output、manual + // output_view；canonical 地址由动态计划中的 task-order prefix + // 给出，不再从 task_id / 5 猜 batch。 + AddNormalizedWriter( + by_bucket, + ExpectedCanonicalOutputDescriptorForTask( + alloc->task_id, 2, alloc->kind, + alloc->group_block_count, + alloc->canonical_task_base + ), + batch.final_up_task_id + ); + AddNormalizedWriter( + by_bucket, + ExpectedCanonicalOutputDescriptorForTask( + alloc->task_id, 1, alloc->kind, + alloc->group_block_count, + alloc->canonical_task_base + ), + batch.final_up_task_id + ); + AddNormalizedWriter( + by_bucket, + ExpectedCanonicalOutputDescriptorForTask( + alloc->task_id, 0, alloc->kind, + alloc->group_block_count, + alloc->canonical_task_base + ), + batch.final_up_task_id + ); + AddNormalizedWriter( + by_bucket, + ExpectedManualOutputView( + batch.batch, plan.batch_count + ), + batch.final_up_task_id + ); + } + return FinishNormalizedWriterSignature(by_bucket); +} + +inline uint64_t SharedNormalizedWriterSignature( + const SharedTensorMapSidecar &map, + const SharedHostTaskPlan &plan, + uint32_t logical_floor +) { + std::vector by_bucket[kMapBuckets]; + for (const SharedHostBatchPlan &batch : plan.batches) { + if (batch.group_count == 0 || + batch.final_up_task_id < logical_floor) { + continue; + } + const SharedHostPlannedTask *alloc = + plan.TaskAt(batch.batch_start); + if (alloc == nullptr || + alloc->kind != TaskKind::Alloc) { + continue; + } + const SharedOutputCell &cell = + map.shared_outputs[alloc->task_id]; + // 实际 shared descriptor 的 8-shard 地址已经由 + // ValidateSharedOutputs 严格校验。跨模式签名只投影同一个业务 + // output 的 canonical(private 连续 heap)地址，不能把物理分片差异 + // 误判成 writer 拓扑差异。 + AddNormalizedWriter( + by_bucket, + ExpectedCanonicalOutputDescriptorForTask( + alloc->task_id, 2, alloc->kind, + alloc->group_block_count, + alloc->canonical_task_base + ), + static_cast(cell.last_writer[2].value) + ); + AddNormalizedWriter( + by_bucket, + ExpectedCanonicalOutputDescriptorForTask( + alloc->task_id, 1, alloc->kind, + alloc->group_block_count, + alloc->canonical_task_base + ), + static_cast(cell.last_writer[1].value) + ); + AddNormalizedWriter( + by_bucket, + ExpectedCanonicalOutputDescriptorForTask( + alloc->task_id, 0, alloc->kind, + alloc->group_block_count, + alloc->canonical_task_base + ), + static_cast(cell.last_writer[0].value) + ); + AddNormalizedWriter( + by_bucket, + ExpectedManualOutputView( + batch.batch, plan.batch_count + ), + batch.final_up_task_id + ); + } + return FinishNormalizedWriterSignature(by_bucket); +} +#endif + +#if PA_BUILD_PERF_CLOCK +inline bool PerfClockObserverFieldsAreZero(const WorkerResult &result) { + for (uint32_t kind = 0; kind < 4; ++kind) { + if (result.kernel_cycles[kind] != 0 || + result.kernel_min_cycles[kind] != 0 || + result.kernel_max_cycles[kind] != 0) { + return false; + } + } + for (uint32_t phase = 0; + phase < static_cast(ProfilePhase::Count); ++phase) { + if (result.phase_cycles[phase] != 0 || + result.phase_calls[phase] != 0) { + return false; + } + } + return result.atomic_trace_calls == 0 && + result.pmu_total_cycles == 0 && + result.pmu_scalar_busy == 0 && + result.pmu_icache_requests == 0 && + result.pmu_icache_misses == 0 && + result.pmu_status == 0 && + result.pmu_window_ticks == 0 && + result.pmu_warm_total_cycles == 0 && + result.pmu_warm_window_ticks == 0 && + result.pmu_warm_icache_requests == 0 && + result.pmu_warm_icache_misses == 0 && + result.pmu_vector_busy == 0 && + result.pmu_cube_busy == 0 && + result.pmu_mte1_busy == 0 && + result.pmu_mte2_busy == 0 && + result.pmu_mte3_busy == 0 && + result.pmu_fix_busy == 0 && + result.pmu_build_variant == 0 && + result.pmu_phase_id == 0 && + result.pmu_phase_calls == 0 && + result.pmu_phase_status == 0 && + result.pmu_phase_icache_requests == 0 && + result.pmu_phase_icache_misses == 0 && + result.pmu_shadow_icache_requests == 0 && + result.pmu_shadow_icache_misses == 0 && + result.startup_barrier_begin == 0 && + result.startup_barrier_end == 0 && + result.final_barrier_begin == 0 && + result.final_barrier_release == 0 && + result.final_barrier_end == 0; +} +#endif + +inline Metrics Validate( + const SchedulerState &state, uint32_t run, double host_us, const TraceHeader *trace_header = nullptr +) { + Metrics metrics; + // 每个 worker 都回放全部 task。Alloc 由 96 个 worker 全部执行 atomicMax Claim； + // 其余 kernel task 只有与 active role 匹配的 AIC 或 AIV 参与 Claim。 + const uint32_t batches = state.config.batches; +#if PTO_FDWIC_SHARED_MAP + SharedHostTaskPlan shared_plan; + std::string shared_plan_error; + const bool shared_plan_ok = BuildSharedHostTaskPlan( + state, &shared_plan, &shared_plan_error + ); + Expect( + shared_plan_ok, + "host independently rebuilds the final shared task plan", + &metrics + ); + if (!shared_plan_ok) { + std::fprintf( + stderr, "Invalid shared host task plan: %s\n", + shared_plan_error.c_str() + ); + } else { + std::printf( + "[HOST_PLAN] batches=%u groups=%u tasks=%u " + "kinds=Alloc:%u,QK:%u,SF:%u,PV:%u,UP:%u " + "canonical_heap_bytes=%llu\n", + shared_plan.batch_count, + shared_plan.total_groups, + shared_plan.total_tasks, + shared_plan.tasks_by_kind[ + static_cast(TaskKind::Alloc) + ], + shared_plan.tasks_by_kind[ + static_cast(TaskKind::Qk) + ], + shared_plan.tasks_by_kind[ + static_cast(TaskKind::Sf) + ], + shared_plan.tasks_by_kind[ + static_cast(TaskKind::Pv) + ], + shared_plan.tasks_by_kind[ + static_cast(TaskKind::Up) + ], + static_cast( + shared_plan.canonical_heap_bytes + ) + ); + } + const uint32_t task_count = + shared_plan_ok ? shared_plan.total_tasks : 0; + const uint32_t group_count = + shared_plan_ok ? shared_plan.total_groups : 0; +#else + const uint32_t task_count = batches * kTasksPerBatch; +#endif + const bool final_barrier_shape_valid = + state.config.final_barrier_shape <= static_cast(FinalBarrierShape::ThreeLevel6x4x4); + const auto final_barrier_shape = static_cast(state.config.final_barrier_shape); + const uint64_t expected_submits = static_cast(kWorkers) * task_count; +#if PTO_FDWIC_SHARED_MAP + const uint64_t expected_claims = + static_cast(batches) * kWorkers + + static_cast(group_count) * + (2U * kAicWorkers + 2U * kAivWorkers); +#else + const uint64_t expected_claims = + static_cast(batches) * (kWorkers + kAicWorkers + kAivWorkers + kAicWorkers + kAivWorkers); +#endif + // 上式依次对应 Alloc、QK、SF、PV、UP 的 active worker 数，默认 256 batch 时为 73728。 + + // 聚合量分为调度核心计数、kernel 分布、前端操作数和最终状态四组，便于定位语义偏差。 + uint64_t first_submit = UINT64_MAX; + uint64_t last_submit = 0; +#if !PA_BUILD_PERF_CLOCK + uint64_t first_startup_begin = UINT64_MAX; + uint64_t last_startup_end = 0; + uint64_t first_final_begin = UINT64_MAX; + uint64_t last_final_release = 0; + uint64_t last_final_end = 0; + std::vector startup_wait_ticks; + std::vector final_release_wait_ticks; + std::vector post_release_drain_ticks; +#endif + uint64_t submits = 0; + uint64_t claims = 0; + uint64_t wins = 0; + uint64_t heap_guards = 0; + uint64_t fanin_ready_loads = 0; + uint64_t fanin_not_ready_loads = 0; + uint64_t frontier_initial_loads = 0; + uint64_t frontier_updates = 0; + uint64_t frontier_terminal_loads = 0; + uint64_t atomic_trace_calls = 0; + uint64_t duplicates = 0; + uint64_t cas_retries = 0; + uint64_t joint_polls = 0; + uint64_t trace_wait_records = 0; + uint64_t wins_by_kind[5] = {}; + uint64_t kernel_counts[4] = {}; +#if !PA_BUILD_PERF_CLOCK + uint64_t kernel_cycles[4] = {}; + uint64_t kernel_min[4] = {}; + uint64_t kernel_max[4] = {}; +#endif + uint64_t placements[3] = {}; + uint64_t phase_calls[static_cast(ProfilePhase::Count)] = {}; + uint64_t context_reads = 0; + uint64_t views_created = 0; + uint64_t dynamic_create_infos = 0; + uint64_t arg_resets = 0; + uint64_t tensor_args_added = 0; + uint64_t scalar_args_added = 0; + uint64_t materialized_outputs = 0; + uint64_t map_inserts = 0; + uint64_t map_lookups = 0; + uint64_t slot_tensor_copies = 0; + uint64_t slot_scalar_copies = 0; + uint64_t fanin_edges = 0; + uint64_t dependency_signature = 0; + uint64_t shared_symbol_input_loads = 0; + uint64_t shared_symbol_inout_commits = 0; + bool worker_ids[kWorkers] = {}; + uint32_t aic_count = 0; + uint32_t aiv_count = 0; + uint32_t winning_workers = 0; + uint64_t max_worker_wins = 0; + bool worker_shape_ok = true; + bool submit_timestamps_ok = true; + bool lifecycle_timestamps_ok = true; +#if PA_BUILD_PERF_CLOCK + bool perf_clock_observer_fields_zero = true; +#endif + bool vend_values_ok = true; + bool frontend_worker_counts_ok = true; + bool final_worker_state_ok = true; + bool worker_checksums_ok = true; +#if !PTO_FDWIC_SHARED_MAP + uint64_t private_logical_map_signature = 0; +#endif + bool fanin_worker_counts_ok = true; + bool frontier_worker_counts_ok = true; + bool role_kernel_routing_ok = true; +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) + bool compete_first_split_runtime_oracle_ok = true; + const uint64_t expected_split_task_id_sum = + static_cast(task_count) * (task_count - 1U) / 2U; +#endif + + // private 按连续逻辑 heap 重建逐 task prefix；shared 只按 task_id%8 + // 重建每个 shard 的最终字节总量。并发 FetchAdd 后，某 task 获得的 + // task_base 和 aggregate vend prefix 都不再由 task_id 顺序决定。 + uint64_t expected_heap_next = 0; + bool vend_progress_bounds_ok = true; + uint32_t first_bad_vend = task_count; + uint64_t first_bad_vend_minimum = 0; + uint64_t first_bad_vend_actual = 0; + std::vector minimum_vends(task_count); +#if PTO_FDWIC_SHARED_MAP + uint64_t expected_shared_heap_cursor[kSharedHeapShards] = {}; + const uint64_t shared_heap_shard_span = + ExpectedSharedHeapShardSpan(state.heap_size); + bool shared_heap_capacity_ok = shared_heap_shard_span != 0; +#endif + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { +#if PTO_FDWIC_SHARED_MAP + const SharedHostPlannedTask *planned_task = + shared_plan.TaskAt(task_id); + const uint64_t output_bytes = + planned_task == nullptr + ? 0 + : planned_task->output_bytes; +#else + const uint64_t output_bytes = ExpectedTaskOutputBytes(task_id); +#endif +#if PTO_FDWIC_SHARED_MAP + const uint32_t shard = task_id % kSharedHeapShards; + shared_heap_capacity_ok &= planned_task != nullptr; + shared_heap_capacity_ok &= + expected_shared_heap_cursor[shard] <= shared_heap_shard_span && + output_bytes <= + shared_heap_shard_span - + std::min( + expected_shared_heap_cursor[shard], + shared_heap_shard_span + ); + expected_shared_heap_cursor[shard] += output_bytes; + expected_heap_next += output_bytes; + minimum_vends[task_id] = output_bytes; +#else + uint64_t task_base = (expected_heap_next + kOutputAlignment - 1) / kOutputAlignment * kOutputAlignment; + if (output_bytes != 0 && (task_base % state.heap_size) + output_bytes > state.heap_size) { + task_base = (task_base / state.heap_size + 1) * state.heap_size; + } + expected_heap_next = task_base + output_bytes; + minimum_vends[task_id] = expected_heap_next; +#endif + } +#if PTO_FDWIC_SHARED_MAP + bool shared_heap_state_ok = shared_heap_capacity_ok; + // 每个实际回放 task 的插入完成字最终必须恰好保存自己的 task_id； + // 未使用的 TaskCell 必须继续保持 -1。 + bool shared_per_task_insert_completions_ok = true; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + const SharedHostPlannedTask *planned_task = + shared_plan.TaskAt(task_id); + shared_per_task_insert_completions_ok &= + planned_task != nullptr && + state.tasks[task_id].deps_prepared == + static_cast(task_id); + } + for (uint32_t task_id = task_count; + task_id < kMaxTasks; ++task_id) { + shared_per_task_insert_completions_ok &= + state.tasks[task_id].deps_prepared == -1; + } + uint64_t actual_shared_cursor_sum = 0; + uint64_t expected_shared_cursor_sum = 0; + for (uint32_t shard = 0; shard < kSharedHeapShards; ++shard) { + const int64_t raw_cursor = + state.shared_map.shared_heap_cursor[shard].value; + shared_heap_state_ok &= raw_cursor >= 0; + const uint64_t actual_cursor = + raw_cursor < 0 ? 0 : static_cast(raw_cursor); + shared_heap_state_ok &= + actual_cursor == expected_shared_heap_cursor[shard]; + shared_heap_capacity_ok &= + actual_cursor <= shared_heap_shard_span; + actual_shared_cursor_sum += actual_cursor; + expected_shared_cursor_sum += expected_shared_heap_cursor[shard]; + } + const int64_t raw_shared_vend = + state.shared_map.shared_heap_vend.value; + shared_heap_state_ok &= raw_shared_vend >= 0; + const uint64_t actual_shared_vend = + raw_shared_vend < 0 ? 0 : static_cast(raw_shared_vend); + shared_heap_state_ok &= + actual_shared_vend == expected_heap_next && + actual_shared_cursor_sum == actual_shared_vend && + expected_shared_cursor_sum == expected_heap_next; + shared_heap_state_ok &= shared_heap_capacity_ok; +#endif + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + // kernel 可以晚于后续 Submit 完成，故 task vend 可以高于本 task + // reserve 后的 prefix。private 使用确定 task-order prefix；shared + // 的并发 prefix 只要求覆盖本 task 自身 reserve 且不越过最终 vend。 + if (state.tasks[task_id].vend < minimum_vends[task_id] || + state.tasks[task_id].vend > expected_heap_next) { + vend_progress_bounds_ok = false; + if (first_bad_vend == task_count) { + first_bad_vend = task_id; + first_bad_vend_minimum = minimum_vends[task_id]; + first_bad_vend_actual = state.tasks[task_id].vend; + } + } + } + // private ring 仍保留 heap window 内的四类 writer。shared fresh + // Output 已迁出 region ring，因此它的 region 摘要和 sequencer 均保持 + // 初值。expected_map_floor 只供跨模式规范化 writer 签名投影使用， + // 不能解释成 shared sidecar 实际发生过 reclaim。 +#if !PTO_FDWIC_SHARED_MAP + const uint64_t expected_private_map_live = + static_cast(kPaCase1MapEntriesPerBatch) * + std::min(batches, kPaCase1MaxLiveMapBatches); +#endif + const uint64_t expected_map_floor = task_count > kHeapWindow + 1 ? task_count - kHeapWindow - 1 : 0; +#if PTO_FDWIC_SHARED_MAP + const SharedTensorMapValidation shared_map_validation = + ValidateSharedTensorMap( + state.shared_map, shared_plan + ); + const SharedOutputValidation shared_output_validation = + ValidateSharedOutputs( + state.shared_map, shared_plan, state.heap_size + ); + if (!shared_output_validation.protocol_ok) { + std::printf( + "[SHARED_OUTPUT_FAILURE] first_bad_task=%u " + "first_bad_slot=%u reason=%s\n", + shared_output_validation.first_bad_task, + shared_output_validation.first_bad_slot, + shared_output_validation.first_bad_reason + ); + } + bool shared_output_heap_layout_ok = + shared_output_validation.protocol_ok && + shared_output_validation.allocated_bytes == expected_heap_next && + shared_output_validation.allocated_bytes == actual_shared_vend; + for (uint32_t shard = 0; shard < kSharedHeapShards; ++shard) { + const int64_t raw_cursor = + state.shared_map.shared_heap_cursor[shard].value; + shared_output_heap_layout_ok &= + shared_output_validation.shard_bytes[shard] == + expected_shared_heap_cursor[shard] && + raw_cursor >= 0 && + shared_output_validation.shard_bytes[shard] == + static_cast(raw_cursor); + } + const uint64_t shared_normalized_writer_signature = + shared_output_heap_layout_ok + ? SharedNormalizedWriterSignature( + state.shared_map, shared_plan, + static_cast(expected_map_floor) + ) + : 0; +#endif + const uint64_t expected_normalized_writer_signature = +#if PTO_FDWIC_SHARED_MAP + ExpectedNormalizedWriterSignature( + shared_plan, + static_cast(expected_map_floor) + ); +#else + ExpectedNormalizedWriterSignature( + batches, static_cast(expected_map_floor) + ); +#endif + + for (uint32_t index = 0; index < kWorkers; ++index) { + // 每核只写自己独占且按 cache line 隔离的 WorkerResult；host 在 kernel 完成后统一汇总，不引入额外 atomic。 + const WorkerResult &result = state.results[index]; + if (result.worker_id < kWorkers) { + worker_ids[result.worker_id] = true; + } else { + worker_shape_ok = false; + } + aic_count += result.role == static_cast(CoreRole::Aic); + aiv_count += result.role == static_cast(CoreRole::Aiv); + worker_shape_ok &= result.submits == task_count; + worker_shape_ok &= result.max_occupied <= kUsableSlots; + worker_shape_ok &= result.final_occupied == 0; + submit_timestamps_ok &= result.submit_begin != 0; +#if PA_BUILD_PERF_CLOCK + submit_timestamps_ok &= result.submit_end > result.submit_begin; + submit_timestamps_ok &= result.finish_cycle == result.submit_end; + lifecycle_timestamps_ok &= + result.startup_barrier_begin == 0 && + result.startup_barrier_end == 0 && + result.final_barrier_begin == 0 && + result.final_barrier_release == 0 && + result.final_barrier_end == 0; + perf_clock_observer_fields_zero &= + PerfClockObserverFieldsAreZero(result); +#else + submit_timestamps_ok &= result.submit_end >= result.submit_begin; + submit_timestamps_ok &= result.finish_cycle >= result.submit_end; + lifecycle_timestamps_ok &= result.startup_barrier_begin != 0; + lifecycle_timestamps_ok &= result.startup_barrier_end >= result.startup_barrier_begin; + lifecycle_timestamps_ok &= result.submit_begin >= result.startup_barrier_end; + lifecycle_timestamps_ok &= result.final_barrier_begin >= result.submit_end; + lifecycle_timestamps_ok &= result.final_barrier_release >= result.final_barrier_begin; + lifecycle_timestamps_ok &= result.final_barrier_end >= result.final_barrier_release; + lifecycle_timestamps_ok &= result.finish_cycle >= result.final_barrier_end; +#endif + dependency_signature ^= result.dependency_signature; + shared_symbol_input_loads += result.shared_symbol_input_loads; + shared_symbol_inout_commits += result.shared_symbol_inout_commits; + first_submit = std::min(first_submit, result.submit_begin); + last_submit = std::max(last_submit, result.submit_end); +#if !PA_BUILD_PERF_CLOCK + first_startup_begin = std::min(first_startup_begin, result.startup_barrier_begin); + last_startup_end = std::max(last_startup_end, result.startup_barrier_end); + first_final_begin = std::min(first_final_begin, result.final_barrier_begin); + last_final_release = std::max(last_final_release, result.final_barrier_release); + last_final_end = std::max(last_final_end, result.final_barrier_end); + startup_wait_ticks.push_back(result.startup_barrier_end - result.startup_barrier_begin); + final_release_wait_ticks.push_back(result.final_barrier_release - result.final_barrier_begin); + post_release_drain_ticks.push_back(result.final_barrier_end - result.final_barrier_release); +#endif + submits += result.submits; + claims += result.claim_attempts; + wins += result.claim_wins; + if (result.claim_wins != 0) ++winning_workers; + max_worker_wins = std::max(max_worker_wins, result.claim_wins); + heap_guards += result.heap_guards; + fanin_ready_loads += result.fanin_ready_loads; + fanin_not_ready_loads += result.fanin_not_ready_loads; + frontier_initial_loads += result.frontier_initial_loads; + frontier_updates += result.frontier_updates; + frontier_terminal_loads += result.frontier_terminal_loads; + atomic_trace_calls += result.atomic_trace_calls; + duplicates += result.completion_duplicates; + cas_retries += result.cas_retries; + joint_polls += result.joint_polls; + trace_wait_records += result.wait_events[0] + result.wait_events[1]; + context_reads += result.context_reads; + views_created += result.views_created; + dynamic_create_infos += result.dynamic_create_infos; + arg_resets += result.arg_resets; + tensor_args_added += result.tensor_args_added; + scalar_args_added += result.scalar_args_added; + materialized_outputs += result.materialized_outputs; + map_inserts += result.map_inserts; + map_lookups += result.map_lookups; + slot_tensor_copies += result.slot_tensor_copies; + slot_scalar_copies += result.slot_scalar_copies; + fanin_edges += result.fanin_edges; +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) + const CoreRole expected_role = index < kAicWorkers ? CoreRole::Aic : CoreRole::Aiv; + compete_first_split_runtime_oracle_ok &= result.worker_id == index; + compete_first_split_runtime_oracle_ok &= + result.compete_first_split_caller_state_address != 0; +#if PTO_FDWIC_SHARED_MAP + // shared loser 已在 caller 轻路径返回，只有本核 Claim winner 才跨 + // TU 进入完整 finish。因此 finish_calls 精确等于本核 wins； + // 没有 winner 的核从未绑定 finish TU，地址保持 0。task_id_sum + // 仍由 caller 覆盖完整 0..N-1 回放序列。 + compete_first_split_runtime_oracle_ok &= + result.compete_first_split_finish_state_address == + (result.claim_wins == 0 + ? 0 + : result.compete_first_split_caller_state_address); + compete_first_split_runtime_oracle_ok &= + result.compete_first_split_finish_calls == + result.claim_wins; +#else + compete_first_split_runtime_oracle_ok &= + result.compete_first_split_finish_state_address == + result.compete_first_split_caller_state_address; + compete_first_split_runtime_oracle_ok &= + result.compete_first_split_finish_calls == task_count; +#endif + compete_first_split_runtime_oracle_ok &= + result.compete_first_split_protocol_errors == 0; + compete_first_split_runtime_oracle_ok &= + result.compete_first_split_state_cookie == + (kCompeteFirstSplitStateCookieBase ^ static_cast(index) ^ + (static_cast(static_cast(expected_role)) << 32U)); + compete_first_split_runtime_oracle_ok &= + result.compete_first_split_task_id_sum == expected_split_task_id_sum; + compete_first_split_runtime_oracle_ok &= + result.compete_first_split_owner_worker_id == index; + compete_first_split_runtime_oracle_ok &= + result.compete_first_split_reserved == 0; +#endif +#if PTO_FDWIC_SHARED_MAP + // shared 的五类重构参和 Materialize 都必须由本核实际 wins[] + // 精确推导；loser 只声明稳定符号，任何重构参都会让这里失败。 + const uint64_t alloc_wins = result.wins[static_cast(TaskKind::Alloc)]; + const uint64_t qk_wins = result.wins[static_cast(TaskKind::Qk)]; + const uint64_t sf_wins = result.wins[static_cast(TaskKind::Sf)]; + const uint64_t pv_wins = result.wins[static_cast(TaskKind::Pv)]; + const uint64_t up_wins = result.wins[static_cast(TaskKind::Up)]; + frontend_worker_counts_ok &= result.context_reads == batches; + frontend_worker_counts_ok &= + result.views_created == qk_wins + up_wins; + frontend_worker_counts_ok &= + result.dynamic_create_infos == qk_wins + sf_wins; + frontend_worker_counts_ok &= + result.arg_resets == qk_wins + sf_wins + pv_wins + up_wins; + frontend_worker_counts_ok &= + result.tensor_args_added == + alloc_wins * 3 + + 4 * (qk_wins + sf_wins + pv_wins) + 7 * up_wins; + frontend_worker_counts_ok &= + result.scalar_args_added == + 2 * qk_wins + 3 * sf_wins + 2 * pv_wins + 2 * up_wins; + frontend_worker_counts_ok &= + result.materialized_outputs == + alloc_wins * 3 + qk_wins + sf_wins * 3 + pv_wins; + frontend_worker_counts_ok &= result.map_inserts == 0; + // shared 的权威进度是 sidecar cursor/vend。worker.heap_next 只保存 + // 该 worker 最近一次获胜时观察到的并发 aggregate prefix；不同 + // winner 的 FetchAdd 顺序不由 task_id 决定，因此不能再拿确定的 + // task-order prefix 集合核对。纯 loser 仍必须保持 0。 + const uint64_t nonzero_output_wins = + alloc_wins + qk_wins + sf_wins + pv_wins; + // WorkerResult 只按 kind 聚合 wins，不保存每个动态 group 的 + // nblocks；对 partial final group 只能重建该 worker 自身 reserve + // 的严格下界。全局逐 task output/descriptor/heap cursor 仍由 + // shared_plan 做精确校验。 + const uint64_t own_reserved_minimum = + alloc_wins * 10240ULL + + qk_wins * 8192ULL + + sf_wins * 6144ULL + + pv_wins * 8192ULL; + final_worker_state_ok &= + result.final_heap_next <= expected_heap_next && + result.final_heap_next >= own_reserved_minimum && + (result.final_heap_next == 0 || + result.final_heap_next % kOutputAlignment == 0) && + (result.claim_wins != 0 || result.final_heap_next == 0) && + (nonzero_output_wins == 0 || result.final_heap_next != 0); +#else + frontend_worker_counts_ok &= result.context_reads == batches; + frontend_worker_counts_ok &= result.views_created == static_cast(batches) * 2; + frontend_worker_counts_ok &= result.dynamic_create_infos == static_cast(batches) * 2; + frontend_worker_counts_ok &= result.arg_resets == static_cast(batches) * 4; + frontend_worker_counts_ok &= result.tensor_args_added == static_cast(batches) * 22; + frontend_worker_counts_ok &= result.scalar_args_added == static_cast(batches) * 9; + frontend_worker_counts_ok &= result.materialized_outputs == static_cast(batches) * 8; + frontend_worker_counts_ok &= result.map_inserts == static_cast(batches) * 4; + final_worker_state_ok &= result.final_heap_next == expected_heap_next; +#endif + final_worker_state_ok &= result.map_high_water == +#if PTO_FDWIC_SHARED_MAP + 0; +#else + expected_private_map_live; +#endif + final_worker_state_ok &= result.map_live_entries == +#if PTO_FDWIC_SHARED_MAP + 0; +#else + expected_private_map_live; +#endif + final_worker_state_ok &= result.map_alive_floor == +#if PTO_FDWIC_SHARED_MAP + 0; +#else + expected_map_floor; +#endif + final_worker_state_ok &= result.map_cleaned_upto == +#if PTO_FDWIC_SHARED_MAP + 0; +#else + expected_map_floor; +#endif +#if PTO_FDWIC_SHARED_MAP + // shared 的权威签名由 host 对唯一 sidecar 逐槽生成，worker 不重复 + // 扫描共享 GM，以免把验证 DCCI 成本加入 kernel 生命周期。 + worker_checksums_ok &= result.checksum == 0; +#else + if (index == 0) { + private_logical_map_signature = result.checksum; + } else { + worker_checksums_ok &= + result.checksum == + private_logical_map_signature; + } + worker_checksums_ok &= result.checksum != 0; +#endif + if (result.role == static_cast(CoreRole::Aic)) { + role_kernel_routing_ok &= result.kernel_counts[1] == 0 && result.kernel_counts[3] == 0; + } else if (result.role == static_cast(CoreRole::Aiv)) { + role_kernel_routing_ok &= result.kernel_counts[0] == 0 && result.kernel_counts[2] == 0; + } else { + role_kernel_routing_ok = false; + } +#if PTO_FDWIC_SHARED_MAP + // shared no-wrap heap 不消费连续 frontier；每核完成只发布 vend/flag。 + // 三个计数必须保持零，防止 private reclaim helping 悄悄回到热路径。 + frontier_worker_counts_ok &= + result.frontier_initial_loads == 0 && + result.frontier_updates == 0 && + result.frontier_terminal_loads == 0; +#else + const uint64_t worker_kernel_completions = result.kernel_counts[0] + result.kernel_counts[1] + + result.kernel_counts[2] + result.kernel_counts[3]; + const uint64_t worker_completions = result.wins[0] + worker_kernel_completions; + frontier_worker_counts_ok &= result.frontier_initial_loads == worker_completions; + frontier_worker_counts_ok &= result.frontier_terminal_loads == result.frontier_initial_loads; +#endif +#if PTO_FDWIC_SHARED_MAP + // shared SlotReady 会永久移除已观察为 ready 的本核私有 fanin + // 前缀；完成 flag 在单轮 kernel 内单调，因此每条真实依赖 + // 只应命中一次 ready。 + fanin_worker_counts_ok &= + result.fanin_ready_loads == result.fanin_edges; +#else + fanin_worker_counts_ok &= + result.fanin_ready_loads >= result.fanin_edges; + if (result.fanin_ready_loads >= result.fanin_edges) { + // PA 最大 fanin 为 3；每次失败检查最多先重读两个 ready 前缀，再遇到一个 not-ready。 + fanin_worker_counts_ok &= + result.fanin_ready_loads - result.fanin_edges <= 2 * result.fanin_not_ready_loads; + } +#endif + for (uint32_t kind = 0; kind < 5; ++kind) + wins_by_kind[kind] += result.wins[kind]; + for (uint32_t kind = 0; kind < 4; ++kind) { + kernel_counts[kind] += result.kernel_counts[kind]; +#if !PA_BUILD_PERF_CLOCK + kernel_cycles[kind] += result.kernel_cycles[kind]; + if (result.kernel_min_cycles[kind] != 0 && + (kernel_min[kind] == 0 || result.kernel_min_cycles[kind] < kernel_min[kind])) { + kernel_min[kind] = result.kernel_min_cycles[kind]; + } + kernel_max[kind] = std::max(kernel_max[kind], result.kernel_max_cycles[kind]); +#endif + } + for (uint32_t place = 0; place < 3; ++place) + placements[place] += result.placement[place]; + for (uint32_t phase = 0; phase < static_cast(ProfilePhase::Count); ++phase) + phase_calls[phase] += result.phase_calls[phase]; + } + for (bool seen : worker_ids) + worker_shape_ok &= seen; + + uint32_t ready_flags = 0; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + // ready flag 和 vend 是跨核 completion 的最终外部可见状态，不能只依赖 worker 私有计数判断完成。 + ready_flags += state.tasks[task_id].flag == 1; +#if PTO_FDWIC_SHARED_MAP + // 无全局 turn 时，零输出 UP 可能在任一非零 reserve 前观察到 + // aggregate vend=0。shared 不使用该值做 heap reclaim，因此 oracle + // 允许 0；有实际 output reserve 的 task 仍必须发布非零 vend。 + const SharedHostPlannedTask *planned_task = + shared_plan.TaskAt(task_id); + vend_values_ok &= + (planned_task != nullptr && + planned_task->output_bytes == 0) || + state.tasks[task_id].vend != 0; + vend_values_ok &= planned_task != nullptr; +#else + vend_values_ok &= state.tasks[task_id].vend != 0; +#endif + vend_values_ok &= state.tasks[task_id].vend % kOutputAlignment == 0; + } + const uint64_t kernel_total = kernel_counts[0] + kernel_counts[1] + kernel_counts[2] + kernel_counts[3]; + const uint64_t placement_total = placements[0] + placements[1] + placements[2]; + const uint64_t fanin_loads = fanin_ready_loads + fanin_not_ready_loads; + const uint64_t frontier_flag_loads = frontier_updates + frontier_terminal_loads; + + // 第一组断言覆盖参与者拓扑、Claim/winner、completion 和最终 drain 等调度主协议。 + Expect(aic_count == kAicWorkers && aiv_count == kAivWorkers, "participant topology is 32 AIC + 64 AIV", &metrics); + Expect( + worker_shape_ok, + kCompiledTensorMapMode == TensorMapBuildMode::Private + ? "all 96 worker markers and private rings are valid" + : "all 96 worker markers and shared-map clients are valid", + &metrics + ); +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) + Expect( + compete_first_split_runtime_oracle_ok, + "compete-first caller/finish share one role-specific block-local state", + &metrics + ); +#endif + Expect(submit_timestamps_ok, "all Submit timing markers are valid", &metrics); +#if PA_BUILD_PERF_CLOCK + Expect( + lifecycle_timestamps_ok, + "perf-clock lifecycle-only timing fields stay zero", + &metrics + ); + Expect( + perf_clock_observer_fields_zero, + "perf-clock excludes phase, atomic-trace, PMU, and kernel timing observations", + &metrics + ); + Expect( + state.config.trace_enabled == 0 && + state.config.trace_base == 0 && + state.config.trace_records_per_core == 0 && + state.config.profile_phases == 0, + "perf-clock runtime trace and phase controls stay disabled", + &metrics + ); +#else + Expect(lifecycle_timestamps_ok, "all lifecycle timing markers are valid", &metrics); +#endif + Expect(final_barrier_shape_valid, "final barrier selector is valid", &metrics); + const bool flat_final_barrier = final_barrier_shape == FinalBarrierShape::Flat; + Expect( + state.started_count.value == static_cast(kWorkers), + "startup barrier remains flat and reaches all workers", &metrics + ); + Expect(submits == expected_submits, "replay count is workers * tasks", &metrics); + Expect(claims == expected_claims, "Claim attempt count matches PA topology", &metrics); + Expect(wins == task_count, "exactly one winner per task", &metrics); +#if PTO_FDWIC_SHARED_MAP + Expect( + wins_by_kind[0] == batches && + wins_by_kind[1] == group_count && + wins_by_kind[2] == group_count && + wins_by_kind[3] == group_count && + wins_by_kind[4] == group_count, + "shared winners match Alloc + groups*(QK/SF/PV/UP)", + &metrics + ); + Expect( + kernel_total == static_cast(group_count) * 4, + "shared kernel count is four per planned group", + &metrics + ); + Expect( + kernel_counts[0] == group_count && + kernel_counts[1] == group_count && + kernel_counts[2] == group_count && + kernel_counts[3] == group_count, + "each shared kernel kind executes once per planned group", + &metrics + ); +#else + Expect( + wins_by_kind[0] == batches && wins_by_kind[1] == batches && wins_by_kind[2] == batches && + wins_by_kind[3] == batches && wins_by_kind[4] == batches, + "Alloc/QK/SF/PV/UP winners are one per batch", &metrics + ); + Expect(kernel_total == static_cast(batches) * 4, "kernel count is four per batch", &metrics); + Expect( + kernel_counts[0] == batches && kernel_counts[1] == batches && kernel_counts[2] == batches && + kernel_counts[3] == batches, + "each kernel kind executes once per batch", &metrics + ); +#endif + Expect( + role_kernel_routing_ok, + "AIC executes only QK/PV and AIV executes only SF/UP", &metrics + ); + Expect( + heap_guards == +#if PTO_FDWIC_SHARED_MAP + 0, +#else + static_cast(batches) * 4, +#endif + kCompiledTensorMapMode == TensorMapBuildMode::Private + ? "private heap guard count matches output winners" + : "shared no-wrap heap needs no private ring guard", + &metrics + ); + Expect( + fanin_worker_counts_ok && +#if PTO_FDWIC_SHARED_MAP + fanin_ready_loads == fanin_edges, +#else + fanin_ready_loads >= fanin_edges && + fanin_ready_loads - fanin_edges <= + 2 * fanin_not_ready_loads, +#endif + kCompiledTensorMapMode == TensorMapBuildMode::Shared + ? "shared fanin ready loads match each dependency edge exactly once" + : "fanin ready/failure load classification is complete", + &metrics + ); +#if PTO_FDWIC_SHARED_MAP + Expect( + frontier_worker_counts_ok && frontier_initial_loads == 0, + "shared no-wrap completion performs no frontier loads", &metrics + ); + Expect( + frontier_terminal_loads == 0 && frontier_updates == 0, + "shared no-wrap completion performs no frontier helping", &metrics + ); +#else + Expect( + frontier_worker_counts_ok && frontier_initial_loads == task_count, + "private frontier initial loads match completed tasks", &metrics + ); + Expect( + frontier_terminal_loads == task_count && frontier_updates >= task_count, + "private frontier ready/update/terminal load identity is exact", &metrics + ); +#endif + Expect(duplicates == 0, "completion flags are published once", &metrics); + Expect(ready_flags == task_count, "all task flags are ready", &metrics); + Expect( + vend_values_ok, + kCompiledTensorMapMode == TensorMapBuildMode::Private + ? "all published vend values are nonzero and aligned" + : "shared task vends are aligned and nonzero for output reservations", + &metrics + ); + Expect( + vend_progress_bounds_ok, + kCompiledTensorMapMode == TensorMapBuildMode::Private + ? "every task vend is within private worker heap progress bounds" + : "every task vend is within shared aggregate heap progress bounds", + &metrics + ); +#if PTO_FDWIC_SHARED_MAP + Expect( + state.frontier.value == -1, + "shared no-wrap frontier remains at its initial value", &metrics + ); + Expect( + shared_per_task_insert_completions_ok, + "shared per-task insert-completion words reach exact task ids", + &metrics + ); +#else + Expect( + state.frontier.value == static_cast(task_count) - 1, + "private frontier reaches the final task", &metrics + ); +#endif + Expect( + state.replay_done.value == (flat_final_barrier ? static_cast(kWorkers) : 0) && + FinalBarrierStateMatches(state.final_barrier, final_barrier_shape), + "final barrier counters match selected tree", &metrics + ); + Expect(state.fatal.value == 0, "fatal remains clear", &metrics); + Expect(placement_total == kernel_total, "EfDrain + RingBp + final placement covers every kernel", &metrics); + // joint_polls 是为未来 BlockWon 模拟预留的结果字段，当前调度路径没有递增点； + // 此断言只确认现有输出保持零，不能单独证明 active_count>=2 分支不可达。 + Expect(joint_polls == 0, "single-lane PA performs no BlockWon polling", &metrics); + // 第二组断言锁定 scalar 前端工作量，防止编译器优化或后续改动悄悄删掉 PA 模拟步骤。 + Expect( + frontend_worker_counts_ok, + kCompiledTensorMapMode == TensorMapBuildMode::Private + ? "every private worker replays the exact eager frontend counts" + : "shared winner-derived heavy args and materialize counts are exact", + &metrics + ); + const uint64_t expected_global_map_inserts = +#if PTO_FDWIC_SHARED_MAP + 0; +#else + static_cast(kWorkers) * batches * + kPaCase1MapEntriesPerBatch; +#endif + const bool global_frontend_counts_ok = +#if PTO_FDWIC_SHARED_MAP + context_reads == static_cast(kWorkers) * batches && + views_created == static_cast(group_count) * 2 && + dynamic_create_infos == + static_cast(group_count) * 2 && + arg_resets == static_cast(group_count) * 4 && + tensor_args_added == + static_cast(batches) * 3 + + static_cast(group_count) * 19 && + scalar_args_added == + static_cast(group_count) * 9 && + materialized_outputs == + static_cast(batches) * 3 + + static_cast(group_count) * 5 && + map_inserts == expected_global_map_inserts; +#else + context_reads == static_cast(kWorkers) * batches && + views_created == static_cast(kWorkers) * batches * 2 && + dynamic_create_infos == static_cast(kWorkers) * batches * 2 && + arg_resets == static_cast(kWorkers) * batches * 4 && + tensor_args_added == static_cast(kWorkers) * batches * 22 && + scalar_args_added == static_cast(kWorkers) * batches * 9 && + materialized_outputs == static_cast(kWorkers) * batches * 8 && + map_inserts == expected_global_map_inserts; +#endif + Expect( + global_frontend_counts_ok, + "global PA frontend operation totals are exact", &metrics + ); + Expect( + map_lookups == +#if PTO_FDWIC_SHARED_MAP + static_cast(group_count) * 5 && +#else + static_cast(batches) * 14 && +#endif +#if PTO_FDWIC_SHARED_MAP + slot_tensor_copies == + static_cast(group_count) * 19 && +#else + slot_tensor_copies == static_cast(batches) * 19 && +#endif +#if PTO_FDWIC_SHARED_MAP + slot_scalar_copies == + static_cast(group_count) * 9 && + fanin_edges == + static_cast(group_count) * 5, +#else + slot_scalar_copies == static_cast(batches) * 9 && + fanin_edges == static_cast(batches) * 5, +#endif + "winner-only TensorMap/symbol, slot-copy, and fanin totals are exact", &metrics + ); + Expect( + shared_symbol_input_loads == +#if PTO_FDWIC_SHARED_MAP + static_cast(group_count) * 5 && +#else + 0 && +#endif + shared_symbol_inout_commits == +#if PTO_FDWIC_SHARED_MAP + static_cast(group_count) * 3, +#else + 0, +#endif + "shared symbol INPUT-load / INOUT-writer-commit totals are exact", &metrics + ); +#if PTO_FDWIC_SHARED_MAP + std::printf( + "[SHARED_SYMBOL] published_outputs=%llu input_loads=%llu " + "inout_writer_commits=%llu\n", + static_cast( + shared_output_validation.published_outputs + ), + static_cast(shared_symbol_input_loads), + static_cast(shared_symbol_inout_commits) + ); +#endif + const uint64_t expected_dependency_signature = +#if PTO_FDWIC_SHARED_MAP + ExpectedPaDependencySignature(shared_plan); +#else + ExpectedPaDependencySignature(batches); +#endif + Expect( + dependency_signature == expected_dependency_signature, +#if PTO_FDWIC_SHARED_MAP + kCompiledTensorMapMode == TensorMapBuildMode::Private + ? "fanin dependency-edge signature matches fixed private PA" + : "fanin dependency-edge signature matches shared group chain", +#else + "fanin dependency-edge signature matches PA Case1", +#endif + &metrics + ); + std::printf( + "[DEPENDENCY] mode=%s edges=%llu signature=%016llx\n", + kCompiledTensorMapMode == TensorMapBuildMode::Private + ? "private" + : "shared", + static_cast(fanin_edges), + static_cast(dependency_signature) + ); + Expect( + final_worker_state_ok, + kCompiledTensorMapMode == TensorMapBuildMode::Private + ? "every private worker final heap and TensorMap state is exact" + : "shared worker heap snapshots are legal and TensorMap summaries are exact", + &metrics + ); +#if PTO_FDWIC_SHARED_MAP + Expect( + shared_heap_state_ok, + "shared heap cursors, vend sum, and shard capacity are exact", + &metrics + ); + std::printf( + "[SHARED_HEAP] shard_span=%llu cursors=[", + static_cast(shared_heap_shard_span) + ); + for (uint32_t shard = 0; shard < kSharedHeapShards; ++shard) { + std::printf( + "%s%lld", shard == 0 ? "" : ",", + static_cast( + state.shared_map.shared_heap_cursor[shard].value + ) + ); + } + std::printf( + "] cursor_sum=%llu vend=%lld expected_vend=%llu capacity_ok=%u\n", + static_cast(actual_shared_cursor_sum), + static_cast(state.shared_map.shared_heap_vend.value), + static_cast(expected_heap_next), + shared_heap_capacity_ok ? 1U : 0U + ); + Expect( + shared_map_validation.protocol_ok && + shared_map_validation.total_appends == 0 && + shared_map_validation.physical_entries == 0 && + shared_map_validation.logical_entries == 0 && + shared_map_validation.logical_signature == 1469598103934665603ULL, + "shared per-task insert chain, empty ordinary ring, and writer history are exact", + &metrics + ); + Expect( + shared_output_heap_layout_ok && + shared_output_validation.published_outputs == + static_cast(batches) * 3 + + static_cast(group_count) * 5, + "shared fresh-output descriptors form exact non-overlapping shard coverage", + &metrics + ); + Expect( + shared_output_heap_layout_ok && + shared_normalized_writer_signature == + expected_normalized_writer_signature, + "shared symbol projection matches canonical normalized writer signature", + &metrics + ); + std::printf( + "[TENSORMAP] mode=shared insert_order=per_task_deps_prepared " + "completed_tasks=%u legacy_turns=[%lld,%lld,%lld,%lld,%lld,%lld,%lld,%lld] " + "reclaim_upto=%lld " + "region_appends=%llu region_physical=%llu region_logical=%llu " + "region_raw_signature=%016llx normalized_writer_signature=%016llx " + "published_outputs=%llu normalized_projection_floor=%llu\n", + shared_plan.total_tasks, + static_cast( + SharedInsertTurnValueHost(state.shared_map, 0) + ), + static_cast( + SharedInsertTurnValueHost(state.shared_map, 1) + ), + static_cast( + SharedInsertTurnValueHost(state.shared_map, 2) + ), + static_cast( + SharedInsertTurnValueHost(state.shared_map, 3) + ), + static_cast( + SharedInsertTurnValueHost(state.shared_map, 4) + ), + static_cast( + SharedInsertTurnValueHost(state.shared_map, 5) + ), + static_cast( + SharedInsertTurnValueHost(state.shared_map, 6) + ), + static_cast( + SharedInsertTurnValueHost(state.shared_map, 7) + ), + static_cast(state.shared_map.reclaim_upto.value), + static_cast( + shared_map_validation.total_appends + ), + static_cast( + shared_map_validation.physical_entries + ), + static_cast( + shared_map_validation.logical_entries + ), + static_cast( + shared_map_validation.logical_signature + ), + static_cast( + shared_normalized_writer_signature + ), + static_cast( + shared_output_validation.published_outputs + ), + static_cast(expected_map_floor) + ); +#else + Expect( + private_logical_map_signature == expected_normalized_writer_signature, + "private raw TensorMap checksum matches canonical normalized writer signature", + &metrics + ); + std::printf( + "[TENSORMAP] mode=private logical_entries=%llu logical_floor=%llu " + "region_raw_signature=%016llx normalized_writer_signature=%016llx\n", + static_cast(expected_private_map_live), + static_cast(expected_map_floor), + static_cast( + private_logical_map_signature + ), + static_cast( + expected_normalized_writer_signature + ) + ); +#endif + Expect( + worker_checksums_ok, + "logical TensorMap signature publication is consistent", + &metrics + ); + + // private 三类 Claim cursor 均为 production-prefix 四分片；shared + // Vector 使用 sidecar 的八条物理线，Cube/Alloc 保持四分片。逐 task + // 重新推导每条 cursor 的最终高水位。 + int64_t expected_cube[kCursorShards] = {-1, -1, -1, -1}; +#if PTO_FDWIC_SHARED_MAP + int64_t expected_vector[kSharedVectorCursorCapacity] = { + -1, -1, -1, -1, -1, -1, -1, -1 + }; +#else + int64_t expected_vector[kCursorShards] = {-1, -1, -1, -1}; +#endif + int64_t expected_alloc[kCursorShards] = {-1, -1, -1, -1}; +#if PTO_FDWIC_SHARED_MAP + bool cursors_ok = true; +#endif + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { +#if PTO_FDWIC_SHARED_MAP + const SharedHostPlannedTask *planned_task = + shared_plan.TaskAt(task_id); + const TaskKind kind = + planned_task == nullptr + ? TaskKind::Count + : planned_task->kind; + cursors_ok &= planned_task != nullptr; +#else + const TaskKind kind = static_cast(task_id % kTasksPerBatch); +#endif + if (kind == TaskKind::Alloc) { + expected_alloc[task_id % kCursorShards] = task_id; + } else if (kind == TaskKind::Qk || kind == TaskKind::Pv) { + expected_cube[task_id % kCursorShards] = task_id; + } else { +#if PTO_FDWIC_SHARED_MAP + expected_vector[ + task_id % kSharedVectorCursorShards + ] = task_id; +#else + expected_vector[task_id % kCursorShards] = task_id; +#endif + } + } +#if !PTO_FDWIC_SHARED_MAP + bool cursors_ok = true; +#endif + for (uint32_t shard = 0; shard < kCursorShards; ++shard) { + cursors_ok &= state.cube_cursor[shard].value == expected_cube[shard]; +#if PTO_FDWIC_SHARED_MAP + // shared Vector 不应触碰旧 production-prefix vector cursor。 + cursors_ok &= state.vector_cursor[shard].value == -1; +#else + cursors_ok &= state.vector_cursor[shard].value == expected_vector[shard]; +#endif + cursors_ok &= state.alloc_cursor[shard].value == expected_alloc[shard]; + } +#if PTO_FDWIC_SHARED_MAP + for (uint32_t shard = 0; shard < kSharedVectorCursorCapacity; ++shard) { + cursors_ok &= + state.shared_map.shared_vector_cursor[shard].value == + expected_vector[shard]; + } +#endif + Expect( + cursors_ok, + "all sharded Claim cursors reach their exact final task", + &metrics + ); + + if (state.config.profile_phases != 0) { + // profile 开关关闭时这些字段允许保持零，避免把可选诊断本身变成语义门禁。 + Expect( + phase_calls[static_cast(ProfilePhase::Claim)] == expected_submits && + phase_calls[static_cast(ProfilePhase::EfDrain)] == expected_submits && + phase_calls[static_cast(ProfilePhase::WaitForSlot)] == +#if PTO_FDWIC_SHARED_MAP + static_cast(group_count) * 4 && +#else + static_cast(batches) * 4 && +#endif + phase_calls[static_cast(ProfilePhase::HeapGuard)] == +#if PTO_FDWIC_SHARED_MAP + 0, +#else + static_cast(batches) * 4, +#endif + kCompiledTensorMapMode == TensorMapBuildMode::Private + ? "private profile calls match Claim/EfDrain/WaitForSlot/HeapGuard" + : "shared profile calls match Claim/EfDrain/WaitForSlot without private HeapGuard", + &metrics + ); + } + + if (state.config.trace_enabled != 0) { + // 固定阶段记录数加上动态等待记录数，应与所有 worker 的 header count 精确相等。 + bool trace_shape_ok = trace_header != nullptr; + uint64_t physical_trace_records = 0; + uint64_t logical_trace_records = 0; + uint64_t trace_dropped = 0; + uint64_t physical_atomic_records = 0; + uint64_t batched_poll_calls = 0; + uint64_t poll_batch_records = 0; + uint64_t dcci_records = 0; + uint64_t dcci_calls = 0; + uint64_t dcci_lines = 0; + bool per_worker_trace_counts_ok = true; + if (trace_header != nullptr) { + trace_shape_ok &= trace_header->magic == 0x4653574cU; + trace_shape_ok &= trace_header->version == 5; + trace_shape_ok &= trace_header->num_cores == kWorkers; + trace_shape_ok &= trace_header->records_per_core == kTraceRecordsPerCore; + trace_shape_ok &= trace_header->frequency_hz == kSystemCounterHz; + trace_shape_ok &= trace_header->record_size_bytes == kTraceRecordSizeBytes; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const TraceCoreState &core = trace_header->cores[worker]; + physical_trace_records += core.count; + logical_trace_records += core.count; + trace_dropped += core.dropped; + trace_shape_ok &= core.count <= kTraceRecordsPerCore; + int32_t expected_block = -1; + int32_t expected_lane = -1; + ExpectedTraceTopology(worker, &expected_block, &expected_lane); + trace_shape_ok &= core.core_idx == static_cast(worker); + trace_shape_ok &= core.block_id == expected_block; + trace_shape_ok &= core.lane == expected_lane; + trace_shape_ok &= core.dcci_records <= core.dcci_calls; + trace_shape_ok &= + (core.dcci_calls == 0) == + (core.dcci_lines == 0); + trace_shape_ok &= + (core.dcci_calls == 0) == + (core.dcci_records == 0); + trace_shape_ok &= core.dcci_lines >= core.dcci_calls; + dcci_records += core.dcci_records; + dcci_calls += core.dcci_calls; + dcci_lines += core.dcci_lines; + const WorkerResult &result = state.results[worker]; +#if PTO_FDWIC_SHARED_MAP + // Submit/Claim 专用区按已完成 Submit 数写入 32B 四端点行， + // host 展开后恢复为两个 32B 逻辑事件。它不进入 + // TraceCoreState::count，但必须进入最终 JSON 事件数。 + trace_shape_ok &= result.submits <= kMaxTasks; + logical_trace_records += 2ULL * result.submits; +#endif + const uint64_t worker_kernels = result.kernel_counts[0] + result.kernel_counts[1] + + result.kernel_counts[2] + result.kernel_counts[3]; + uint64_t worker_physical_atomic = 0; + if ((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) { + trace_shape_ok &= core.atomic_calls == result.atomic_trace_calls; + trace_shape_ok &= core.poll_calls <= core.atomic_calls; + trace_shape_ok &= (core.poll_calls == 0) == (core.poll_batch_records == 0); + worker_physical_atomic = + static_cast( + core.atomic_calls + ) - core.poll_calls + + core.poll_batch_records; + physical_atomic_records += worker_physical_atomic; + batched_poll_calls += core.poll_calls; + poll_batch_records += core.poll_batch_records; + } else { + trace_shape_ok &= core.atomic_calls == 0 && core.poll_calls == 0 && + core.poll_batch_records == 0; + } + const uint64_t worker_expected = +#if PTO_FDWIC_SHARED_MAP + // core.count 只覆盖 generic 物理区：Claim/Submit 已迁到 + // 专用 32B 区，不能再计入这个物理公式。winner 追加 + // Materialize/Register/metadata/outputs/copy/flush/tail， + // 非 Alloc 再追加 Fanin。Alloc 为 7 条、普通为 8 条， + // 即 8*wins - alloc_wins。 + 8 * result.claim_wins - result.wins[0] + +#else + 6 * result.submits + 2 * result.claim_wins - result.wins[0] + +#endif + 2 * worker_kernels + result.wait_events[0] + result.wait_events[1] + 2 + + core.dcci_records + + (((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) + ? worker_physical_atomic + 2 + : 0); + per_worker_trace_counts_ok &= core.count == worker_expected; + } + } +#if PTO_FDWIC_SHARED_MAP + // generic 物理区内：每 batch 的 Alloc winner 有 7 条；每 group + // 的四个普通 winner 子区间有 32 条，四个实际 kernel 的 + // Kernel/Commit 有 8 条，合计 40 条。Claim/Submit 专用区 + // 不进入 physical_expected_trace_records。 + const uint64_t expected_shared_extra_records = + 7ULL * static_cast(batches) + + 40ULL * static_cast(group_count); + const uint64_t physical_expected_trace_records = + expected_shared_extra_records + + trace_wait_records + 2 * kWorkers + dcci_records + + (((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) + ? physical_atomic_records + 2 * kWorkers + : 0); + const uint64_t logical_expected_trace_records = + physical_expected_trace_records + + 2ULL * expected_submits; +#else + const uint64_t physical_expected_trace_records = + static_cast(batches) * (static_cast(kWorkers) * 30 + 17) + + trace_wait_records + 2 * kWorkers + dcci_records + + (((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) + ? physical_atomic_records + 2 * kWorkers + : 0); + const uint64_t logical_expected_trace_records = + physical_expected_trace_records; +#endif + // shared 每个逻辑 task固定 Claim+Submit 两条；EfDrain 由 + // Submit.start -> Claim.start 离线还原，loser 没有 + // 业务子区间；Alloc winner 追加 Materialize/Register/metadata/ + // outputs/copy/flush/AllocComplete 七条，每个普通 winner 追加 + // Materialize/Register/metadata/outputs/copy/flush/Fanin/ + // WinnerBuild 八条；每组四个实际 kernel 再各有 Kernel+Commit 两条。 + // private 仍保持既有固定六条 Submit 记录。两个父 span 每核固定 + // 增加 2 条，真实等待按运行时次数加入。 + Expect(trace_shape_ok, "swimlane header and per-worker capacities are valid", &metrics); + Expect(trace_dropped == 0, "swimlane records fit without drops", &metrics); + Expect( + physical_trace_records == + physical_expected_trace_records, + "physical generic swimlane record count matches PA phase flow", + &metrics + ); + Expect( + logical_trace_records == + logical_expected_trace_records, + "expanded logical swimlane record count matches PA phase flow", + &metrics + ); + Expect( + per_worker_trace_counts_ok, + "every worker physical generic swimlane record count is exact", + &metrics + ); + if ((state.config.trace_enabled & kTraceAtomicsEnabled) != 0) { + Expect(atomic_trace_calls != 0, "atomic trace captured source-level calls", &metrics); + } else { + Expect(atomic_trace_calls == 0, "atomic trace counters stay zero when disabled", &metrics); + } + Expect( + dcci_records >= kWorkers && + dcci_calls == + dcci_records + +#if PTO_FDWIC_SHARED_MAP + 2ULL * kWorkers && +#else + kWorkers && +#endif + dcci_lines >= dcci_calls, + "DCCI closure includes one observer export record per worker", + &metrics + ); + std::printf( + "[TRACE] physical_generic_records=%llu " + "physical_expected=%llu logical_records=%llu " + "logical_expected=%llu dropped=%llu bytes=%zu\n", + static_cast( + physical_trace_records + ), + static_cast( + physical_expected_trace_records + ), + static_cast( + logical_trace_records + ), + static_cast( + logical_expected_trace_records + ), + static_cast(trace_dropped), kTraceBytes + ); + std::printf( + "[ATOMIC_TRACE] enabled=%s logical_calls=%llu physical_records=%llu " + "batched_poll_calls=%llu poll_batch_records=%llu " + "closure=physical=logical-batched+batch_records\n", + (state.config.trace_enabled & kTraceAtomicsEnabled) != 0 ? "yes" : "no", + static_cast(atomic_trace_calls), + static_cast(physical_atomic_records), + static_cast(batched_poll_calls), + static_cast(poll_batch_records) + ); + std::printf( + "[DCCI_TRACE] logical_calls=%llu cache_lines=%llu " + "physical_records=%llu " + "closure=calls=sum(record.call_count)," + "lines=sum(record.line_count)\n", + static_cast(dcci_calls), + static_cast(dcci_lines), + static_cast(dcci_records) + ); + } + + if (first_submit != UINT64_MAX && last_submit >= first_submit) { + // 性能口径只覆盖最早 Submit.begin 到最晚 Submit.end，不含启动屏障、最终 drain 和 host 同步。 + metrics.submit_span_us = static_cast(last_submit - first_submit) / 1000.0; + } +#if PA_BUILD_PERF_CLOCK + std::printf( + "[PERF-CLOCK] run=%u global_start_tick=%llu global_end_tick=%llu " + "global_span_ticks=%llu scope=first-submit-begin-to-last-submit-end\n", + run, + static_cast(first_submit), + static_cast(last_submit), + static_cast( + first_submit == UINT64_MAX || last_submit < first_submit + ? 0 + : last_submit - first_submit + ) + ); +#else + if (first_startup_begin != UINT64_MAX && last_startup_end >= first_startup_begin && + first_final_begin != UINT64_MAX && last_final_release >= first_final_begin && + last_final_end >= first_final_begin && last_final_end >= first_startup_begin) { + metrics.startup_barrier_span_us = static_cast(last_startup_end - first_startup_begin) / 1000.0; + metrics.final_barrier_span_us = static_cast(last_final_release - first_final_begin) / 1000.0; + metrics.final_drain_span_us = static_cast(last_final_end - first_final_begin) / 1000.0; + metrics.lifecycle_span_us = static_cast(last_final_end - first_startup_begin) / 1000.0; + } + const Uint64Distribution startup_wait = SummarizeUint64(startup_wait_ticks); + const Uint64Distribution final_release_wait = SummarizeUint64(final_release_wait_ticks); + const Uint64Distribution post_release_drain = SummarizeUint64(post_release_drain_ticks); + std::printf( + "[LIFECYCLE] run=%u final_shape=%s startup_span_us=%.3f final_barrier_span_us=%.3f " + "final_drain_span_us=%.3f lifecycle_span_us=%.3f " + "worker_startup_wait_median_us=%.3f worker_startup_wait_p95_us=%.3f " + "worker_final_wait_median_us=%.3f worker_final_wait_p95_us=%.3f " + "worker_post_release_drain_median_us=%.3f worker_post_release_drain_p95_us=%.3f\n", + run, FinalBarrierShapeName(final_barrier_shape), metrics.startup_barrier_span_us, + metrics.final_barrier_span_us, metrics.final_drain_span_us, metrics.lifecycle_span_us, + startup_wait.median / 1000.0, static_cast(startup_wait.p95) / 1000.0, + final_release_wait.median / 1000.0, static_cast(final_release_wait.p95) / 1000.0, + post_release_drain.median / 1000.0, static_cast(post_release_drain.p95) / 1000.0 + ); +#endif + std::printf( + "[METRIC] run=%u submit_span_us=%.3f host_launch_us=%.3f claims=%llu fanin_loads=%llu cas_retries=%llu\n", run, + metrics.submit_span_us, host_us, static_cast(claims), + static_cast(fanin_loads), static_cast(cas_retries) + ); + const uint64_t submit_completion_ops = + claims + heap_guards + fanin_loads + 2ULL * task_count + frontier_initial_loads + + frontier_flag_loads + frontier_updates; + std::printf( + "[ATOMIC] submit_completion_ops=%llu fanin_ready=%llu fanin_not_ready=%llu frontier_initial=%llu " + "frontier_flag=%llu frontier_ready_fetch_max=%llu frontier_terminal=%llu\n", + static_cast(submit_completion_ops), + static_cast(fanin_ready_loads), + static_cast(fanin_not_ready_loads), + static_cast(frontier_initial_loads), + static_cast(frontier_flag_loads), + static_cast(frontier_updates), + static_cast(frontier_terminal_loads) + ); + std::printf( + "[WINNERS] active_workers=%u max_wins_per_worker=%llu\n", winning_workers, + static_cast(max_worker_wins) + ); + std::printf( + "[PLACEMENT] EfDrain=%llu RingBp=%llu FinalDrain=%llu\n", + static_cast(placements[static_cast(DrainPlace::EfDrain)]), + static_cast(placements[static_cast(DrainPlace::RingBackpressure)]), + static_cast(placements[static_cast(DrainPlace::FinalDrain)]) + ); + // placement 统计回答 kernel 最终在哪个 drain 点执行，与 TracePhase 的累计 span 互补。 + const char *kernel_names[] = {"QK", "SF", "PV", "UP"}; +#if !PA_BUILD_PERF_CLOCK + const uint32_t targets[] = {kTargetQkTicks, kTargetSfTicks, kTargetPvTicks, kTargetUpTicks}; +#endif + for (uint32_t kind = 0; kind < 4; ++kind) { +#if PA_BUILD_PERF_CLOCK + std::printf( + "[KERNEL] %-2s count=%llu timing=disabled-in-perf-clock\n", + kernel_names[kind], + static_cast(kernel_counts[kind]) + ); +#else + const double mean = + kernel_counts[kind] == 0 ? 0.0 : static_cast(kernel_cycles[kind]) / kernel_counts[kind]; + std::printf( + "[KERNEL] %-2s count=%llu mean_us=%.3f min_us=%.3f max_us=%.3f target_us=%.3f\n", kernel_names[kind], + static_cast(kernel_counts[kind]), mean / 1000.0, kernel_min[kind] / 1000.0, + kernel_max[kind] / 1000.0, targets[kind] / 1000.0 + ); +#endif + } + PrintPhaseDiagnostics(state); + if (!metrics.passed) { + // 失败时补充第一处未完成 task、vend 边界和 worker 进度，避免只有笼统的 ASSERT FAIL。 + uint32_t first_not_ready = task_count; + for (uint32_t task_id = 0; task_id < task_count; ++task_id) { + if (state.tasks[task_id].flag != 1) { + first_not_ready = task_id; + break; + } + } + uint64_t min_worker_submits = UINT64_MAX; + uint64_t max_worker_submits = 0; + uint32_t incomplete_workers = 0; + uint32_t occupied_workers = 0; + uint64_t max_final_occupied = 0; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const WorkerResult &result = state.results[worker]; + min_worker_submits = std::min(min_worker_submits, result.submits); + max_worker_submits = std::max(max_worker_submits, result.submits); + incomplete_workers += result.submits != task_count; + occupied_workers += result.final_occupied != 0; + max_final_occupied = std::max(max_final_occupied, result.final_occupied); + } +#if PTO_FDWIC_SHARED_MAP + std::printf( + "[FAILURE_STATE] fatal=%d frontier=%lld first_not_ready=%u first_bad_vend=%u " + "vend_minimum=%llu vend_actual=%llu shared_heap_cursors=" + "[%lld,%lld,%lld,%lld,%lld,%lld,%lld,%lld] shared_heap_vend=%lld " + "shared_heap_shard_span=%llu shared_heap_capacity_ok=%d " + "worker_submits_min=%llu worker_submits_max=%llu incomplete_workers=%u " + "final_occupied_workers=%u max_final_occupied=%llu\n", + state.fatal.value, static_cast(state.frontier.value), + first_not_ready, first_bad_vend, + static_cast(first_bad_vend_minimum), + static_cast(first_bad_vend_actual), + static_cast(state.shared_map.shared_heap_cursor[0].value), + static_cast(state.shared_map.shared_heap_cursor[1].value), + static_cast(state.shared_map.shared_heap_cursor[2].value), + static_cast(state.shared_map.shared_heap_cursor[3].value), + static_cast(state.shared_map.shared_heap_cursor[4].value), + static_cast(state.shared_map.shared_heap_cursor[5].value), + static_cast(state.shared_map.shared_heap_cursor[6].value), + static_cast(state.shared_map.shared_heap_cursor[7].value), + static_cast(state.shared_map.shared_heap_vend.value), + static_cast(shared_heap_shard_span), + shared_heap_capacity_ok ? 1 : 0, + static_cast(min_worker_submits), + static_cast(max_worker_submits), + incomplete_workers, occupied_workers, + static_cast(max_final_occupied) + ); +#else + const int64_t retire = + state.frontier.value - static_cast(kHeapWindow); + const uint64_t retire_vend = + retire >= 0 && retire < static_cast(task_count) ? state.tasks[retire].vend : 0; + std::printf( + "[FAILURE_STATE] fatal=%d frontier=%lld first_not_ready=%u first_bad_vend=%u " + "vend_minimum=%llu vend_actual=%llu retire=%lld retire_vend=%llu " + "worker_submits_min=%llu worker_submits_max=%llu incomplete_workers=%u " + "final_occupied_workers=%u max_final_occupied=%llu\n", + state.fatal.value, static_cast(state.frontier.value), first_not_ready, first_bad_vend, + static_cast(first_bad_vend_minimum), + static_cast(first_bad_vend_actual), + static_cast(retire), static_cast(retire_vend), + static_cast(min_worker_submits), + static_cast(max_worker_submits), incomplete_workers, occupied_workers, + static_cast(max_final_occupied) + ); +#endif + } + return metrics; +} + +inline double Median(std::vector values) { + // 多轮 benchmark 只报告中位数；上板基线比较仍应优先采用独立进程首轮。 + std::sort(values.begin(), values.end()); + const size_t middle = values.size() / 2; + if ((values.size() & 1U) != 0) return values[middle]; + return (values[middle - 1] + values[middle]) / 2.0; +} + +inline void PrintBanner(const char *backend, const Options &options) { + // 开始运行前完整打印工作量和大内存占用，便于确认比较口径没有混用。 + std::printf("=== Standalone PA Scheduler Benchmark: %s ===\n", backend); +#if PTO_FDWIC_SHARED_MAP + uint32_t configured_groups = 0; + for (uint32_t batch = 0; batch < options.batches; ++batch) { + const uint32_t context_length = + options.shared_context_lens.empty() + ? 8192U + : static_cast( + options.shared_context_lens.size() == 1 + ? options.shared_context_lens.front() + : batch < + options.shared_context_lens.size() + ? options.shared_context_lens[batch] + : -1 + ); + const uint32_t blocks = + (context_length + kHostPaBlockSize - 1U) / + kHostPaBlockSize; + configured_groups += + (blocks + kHostPaBlocksPerRequest - 1U) / + kHostPaBlocksPerRequest; + } + const uint32_t configured_tasks = + options.batches + 4U * configured_groups; +#endif + std::printf( + "device=%u batches=%u tasks=%u workers=%u runs=%u tensormap=%s " + "nops=%u,%u,%u,%u state_bytes=%zu " + "final_barrier=%s swimlane=%s trace_atomics=%s trace_bytes=%zu\n", options.device, +#if PTO_FDWIC_SHARED_MAP + options.batches, configured_tasks, kWorkers, options.runs, +#else + options.batches, options.batches * kTasksPerBatch, kWorkers, options.runs, +#endif + kCompiledTensorMapMode == TensorMapBuildMode::Private ? "private" : "shared", + options.nops.qk, options.nops.sf, + options.nops.pv, options.nops.up, sizeof(SchedulerState), + FinalBarrierShapeName(options.final_barrier_shape), options.trace_enabled ? "on" : "off", + options.trace_atomics ? "on" : "off", + options.trace_enabled ? kTraceBytes : 0 + ); +#if PTO_FDWIC_SHARED_MAP + std::printf( + "shared_groups=%u shared_context_lens=", + configured_groups + ); + if (options.shared_context_lens.empty()) { + std::printf("default:8192"); + } else if (options.shared_context_lens.size() == 1) { + std::printf( + "broadcast:%d", + options.shared_context_lens.front() + ); + } else { + for (size_t index = 0; + index < options.shared_context_lens.size(); ++index) { + std::printf( + "%s%d", index == 0 ? "" : ",", + options.shared_context_lens[index] + ); + } + } + std::printf("\n"); +#endif + if (!options.swimlane_json.empty()) { + std::printf("swimlane_json=%s\n", options.swimlane_json.c_str()); + } +} + +} // namespace pa_scheduler::host + +#endif // PA_SCHEDULER_COMMON_HOST_SUPPORT_H diff --git a/tests/atomic_probe/pa_scheduler/common/pa_frontend.h b/tests/atomic_probe/pa_scheduler/common/pa_frontend.h new file mode 100644 index 0000000000..32fe9ec2da --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/common/pa_frontend.h @@ -0,0 +1,1824 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_FRONTEND_H +#define PA_SCHEDULER_COMMON_PA_FRONTEND_H + +#include "pa_model.h" +#if PTO_FDWIC_SHARED_MAP +#include "pa_shared_heap.h" +#endif + +namespace pa_scheduler { + +// 这些基址只充当稳定的 tensor identity，供 descriptor、区间重叠和 heap 地址 +// 计算使用；winner workload 不解引用这些 synthetic 地址，real-compute 使用独立 +// workspace。context_lens 是唯一按真实 GM 指针读取的 PA 前端输入。 +constexpr uint64_t kInvalidTaskId = UINT64_MAX; +constexpr uint64_t kSyntheticQueryBase = 0x200000000ULL; +constexpr uint64_t kSyntheticKeyBase = 0x300000000ULL; +constexpr uint64_t kSyntheticValueBase = 0x400000000ULL; +constexpr uint64_t kSyntheticBlockTableBase = 0x500000000ULL; +constexpr uint64_t kSyntheticOutputBase = 0x600000000ULL; +constexpr uint64_t kSyntheticContextLensBase = 0x700000000ULL; +constexpr uint32_t kPaHeads = 16; +constexpr uint32_t kPaHeadDim = 128; +constexpr uint32_t kPaBlockSize = 128; +constexpr uint32_t kPaBlocksPerRequest = 64; +constexpr uint32_t kPaMaxBlocksPerRequest = 256; +#if PTO_FDWIC_SHARED_MAP +static_assert( + kSharedPaMaxBlockGroups == + kPaMaxBlocksPerRequest / kPaBlocksPerRequest, + "shared ticket group bits must cover the PA request limit" +); +#endif +constexpr uint64_t kPaScaleBits = 0x3F800000ULL; +constexpr uint32_t kSpmdLocalContextIndex = kMaxTaskTensors + kMaxTaskScalars; +constexpr uint32_t kSpmdGlobalContextIndex = kSpmdLocalContextIndex + 1; +static_assert(kMaxTaskTensors == 32, "PA frontend requires the real 32 tensor slots"); +static_assert(kMaxTaskScalars == 16, "PA frontend requires the real 16 scalar slots"); +static_assert(kSpmdLocalContextIndex == 48, "PA local-context dispatch index mismatch"); +static_assert(kSpmdGlobalContextIndex == 49, "PA global-context dispatch index mismatch"); +static_assert(kMaxFanin == 16, "PA frontend requires the real 16 fanin slots"); + +enum class TensorRefKind : uint8_t { + LocalTensor = 0, + GmTensor = 1, + CreateInfo = 2, +#if PTO_FDWIC_SHARED_MAP + SharedOutputRef = 3, +#endif +}; + +#if PTO_FDWIC_SHARED_MAP +// shared fresh Output 使用稳定的 (producer_task_id, output_slot) 符号， +// 不把某个 worker 私有 payload 中的 TensorDesc 指针传给其他 worker。 +// 后四个字段预留真实 runtime 的一维 view ABI；PA Case1 本阶段只产生 +// flags/view 全零的直接引用，resolver 对其他形态显式 fail-closed。 +struct FdwicOutputRef { + int32_t producer_task_id; + int16_t output_slot; + uint8_t flags; + uint8_t view_ndims; + uint32_t view_shape0; + uint32_t view_offset0; +}; +static_assert(sizeof(FdwicOutputRef) == 16, "FdwicOutputRef ABI size changed"); +static_assert(alignof(FdwicOutputRef) == 4, "FdwicOutputRef ABI alignment changed"); +static_assert(offsetof(FdwicOutputRef, producer_task_id) == 0, "shared output producer offset mismatch"); +static_assert(offsetof(FdwicOutputRef, output_slot) == 4, "shared output slot offset mismatch"); +static_assert(offsetof(FdwicOutputRef, flags) == 6, "shared output flags offset mismatch"); +static_assert(offsetof(FdwicOutputRef, view_ndims) == 7, "shared output view-rank offset mismatch"); +static_assert(offsetof(FdwicOutputRef, view_shape0) == 8, "shared output view-shape offset mismatch"); +static_assert(offsetof(FdwicOutputRef, view_offset0) == 12, "shared output view-offset mismatch"); +static_assert( + __is_trivially_constructible(FdwicOutputRef), + "FdwicOutputRef must remain trivial for CCEC block-local state" +); + +PA_DEVICE FdwicOutputRef InvalidSharedOutputRef() { + return FdwicOutputRef{-1, -1, 0, 0, 0, 0}; +} + +// SubmitContext 在 replay 循环中被重复使用，因此 Reset 必须同时写 task id +// 和 count；字段不能使用默认成员初始化，否则 SubmitContext 会产生非平凡 +// 构造函数，而 CCEC 禁止 [[block_local]] runtime state 带 ctor/dtor。 +struct SharedTaskOutputs { + int32_t producer_task_id; + uint32_t output_count; + + PA_DEVICE void Reset(int32_t task_id) { + producer_task_id = task_id; + output_count = 0; + } + + PA_DEVICE bool AddOutputRef(int32_t task_id, int16_t output_slot) { + if (task_id < 0 || task_id != producer_task_id || + output_count >= kSharedOutputMaxPerTask || + output_slot != static_cast(output_count)) { + return false; + } + ++output_count; + return true; + } + + PA_DEVICE bool Empty() const { return output_count == 0; } + PA_DEVICE uint32_t Size() const { return output_count; } + PA_DEVICE int32_t TaskId() const { return producer_task_id; } + + PA_DEVICE FdwicOutputRef OutputRef(uint32_t index) const { + if (index >= output_count) { + return InvalidSharedOutputRef(); + } + return FdwicOutputRef{ + producer_task_id, static_cast(index), 0, 0, 0, 0, + }; + } +}; +static_assert(sizeof(SharedTaskOutputs) == 8, "SharedTaskOutputs ABI size changed"); +static_assert(alignof(SharedTaskOutputs) == 4, "SharedTaskOutputs ABI alignment changed"); +static_assert(offsetof(SharedTaskOutputs, producer_task_id) == 0, "shared result task offset mismatch"); +static_assert(offsetof(SharedTaskOutputs, output_count) == 4, "shared result count offset mismatch"); +static_assert( + __is_trivially_constructible(SharedTaskOutputs), + "SharedTaskOutputs must remain trivial for CCEC block-local state" +); + +// pa_model.h 可能先被 CCEC 的 host-side PMU 头包含，因此这里保留明确的 +// device 版本，避免 __aicore__ 路径调用先前已实例化的 host constexpr。 +PA_DEVICE uint32_t FrontendTaskOutputCount(TaskKind kind) { + switch (kind) { + case TaskKind::Alloc: return 3; + case TaskKind::Qk: return 1; + case TaskKind::Sf: return 3; + case TaskKind::Pv: return 1; + case TaskKind::Up: + case TaskKind::Count: return 0; + } + return 0; +} + +#if PTO_FDWIC_SHARED_MAP +// 与 FrontendTaskOutputCount 相同，这些 replay-layout helper 必须在 +// pa_frontend.h 中按当前 TU 的 PA_DEVICE 身份定义。pa_model.h 可能已被 +// CCEC winner_workload 作为 host 头提前包含，不能依赖其中捕获的宏状态。 +constexpr uint64_t kSharedPaMaxContextLength = + static_cast(kPaMaxBlocksPerRequest) * kPaBlockSize; + +struct SharedPaBatchPlan { + uint32_t batch_start; + uint32_t group_count; + uint32_t task_count; +}; + +struct SharedPaPlannedTask { + TaskKind kind; + uint32_t group_index; + bool has_following_group; + bool is_last_in_batch; +}; + +PA_DEVICE bool BuildSharedPaBatchPlan( + uint64_t context_length, uint32_t batch_start, + SharedPaBatchPlan &plan +) { + // 与真实 PA 一致：Alloc 位于 group loop 之前，因此空 context 仍有 + // 一个 Alloc task；每 64 blocks 形成一组 QK/SF/PV/UP。先限制请求 + // 上限再做向上取整，避免损坏的负 int32 context 被转换后溢出。 + if (context_length > kSharedPaMaxContextLength) { + return false; + } + const uint64_t block_count = + (context_length + kPaBlockSize - 1U) / kPaBlockSize; + const uint64_t group_count = + (block_count + kPaBlocksPerRequest - 1U) / + kPaBlocksPerRequest; + const uint64_t task_count = 1U + 4U * group_count; + if (group_count > kSharedPaMaxBlockGroups || + task_count > kSharedPaMaxTasksPerBatch || + batch_start > kMaxTasks - static_cast(task_count)) { + return false; + } + plan.batch_start = batch_start; + plan.group_count = static_cast(group_count); + plan.task_count = static_cast(task_count); + return true; +} + +PA_DEVICE uint32_t SharedPaTaskOffset( + TaskKind kind, uint32_t group_index +) { + return kind == TaskKind::Alloc + ? 0U + : 1U + group_index * 4U + + (static_cast(kind) - + static_cast(TaskKind::Qk)); +} + +PA_DEVICE TaskKind SharedPaTaskKindFromOffset( + uint32_t task_offset +) { + return task_offset == 0 + ? TaskKind::Alloc + : static_cast( + 1U + ((task_offset - 1U) % 4U) + ); +} + +PA_DEVICE bool SharedPaPlannedTaskAt( + const SharedPaBatchPlan &plan, uint32_t task_offset, + SharedPaPlannedTask &task +) { + if (plan.group_count > kSharedPaMaxBlockGroups || + plan.task_count != 1U + 4U * plan.group_count || + plan.batch_start > kMaxTasks - plan.task_count || + task_offset >= plan.task_count) { + return false; + } + task.kind = SharedPaTaskKindFromOffset(task_offset); + task.group_index = + task_offset == 0 ? 0U : (task_offset - 1U) / 4U; + task.has_following_group = + task.kind == TaskKind::Up && + task.group_index + 1U < plan.group_count; + task.is_last_in_batch = task_offset + 1U == plan.task_count; + return task.kind < TaskKind::Count && + task.group_index < kSharedPaMaxBlockGroups; +} + +PA_DEVICE bool SharedPaTaskKindInBatch( + uint32_t task_id, uint32_t batch_start, + uint32_t current_group_index, TaskKind &kind +) { + if (current_group_index >= kSharedPaMaxBlockGroups || + task_id < batch_start) { + return false; + } + const uint32_t offset = task_id - batch_start; + const uint32_t visible_task_count = + 1U + 4U * (current_group_index + 1U); + if (offset >= visible_task_count) { + return false; + } + kind = SharedPaTaskKindFromOffset(offset); + return true; +} +#endif + +PA_DEVICE bool PrepareSharedTaskOutputs( + SharedTaskOutputs &outputs, int32_t task_id, TaskKind kind +) { + // loser 也必须把同一组 (producer,slot) 句柄交给本核后续 orchestration； + // 这里仅声明稳定符号，不读取 winner 私有 payload，也不物化 descriptor。 + if (task_id < 0 || outputs.TaskId() != task_id || !outputs.Empty()) { + return false; + } + const uint32_t output_count = FrontendTaskOutputCount(kind); + for (uint32_t slot = 0; slot < output_count; ++slot) { + if (!outputs.AddOutputRef( + task_id, static_cast(slot) + )) { + return false; + } + } + return outputs.Size() == output_count; +} +#endif + +// TaskArgs 同时容纳 orchestration 栈上的 descriptor、GM 中已物化的 descriptor， +// 以及尚待 Materialize 的 CreateInfo。显式 kind 保留生产 TensorRef 的地址空间分支。 +union TensorPointer { + const TensorDesc *local_tensor; + PA_GM const TensorDesc *gm_tensor; + const TensorCreateInfo *create_info; +#if PTO_FDWIC_SHARED_MAP + FdwicOutputRef output_ref; +#endif +}; + +struct TaskTensorRef { + TensorPointer pointer; + TensorRefKind kind; +}; +#if PTO_FDWIC_SHARED_MAP +static_assert(sizeof(TaskTensorRef) == 24, "shared TaskTensorRef must match the PA TensorRef ABI"); +static_assert(offsetof(TaskTensorRef, pointer) == 0, "shared TaskTensorRef pointer offset mismatch"); +static_assert(offsetof(TaskTensorRef, kind) == 16, "shared TaskTensorRef kind offset mismatch"); +#else +static_assert(sizeof(TaskTensorRef) == 16, "TaskTensorRef must match the PA TensorRef ABI"); +static_assert(offsetof(TaskTensorRef, pointer) == 0, "TaskTensorRef pointer offset mismatch"); +static_assert(offsetof(TaskTensorRef, kind) == 8, "TaskTensorRef kind offset mismatch"); +#endif + +struct PaLaunchSpec { + int16_t core_num; + bool require_sync_start; +}; +static_assert(sizeof(PaLaunchSpec) == 4, "PA launch spec ABI mismatch"); + +struct PaAsyncContext { + uint64_t completion_count; + uint64_t completion_error_code; + uint64_t completion_entries; + uint32_t completion_capacity; + uint32_t alignment_padding; + uint64_t task_token; +}; +static_assert(sizeof(PaAsyncContext) == 40, "PA async context ABI mismatch"); + +struct PaLocalContext { + int32_t block_index; + int32_t block_count; + PaAsyncContext async; +}; +// Local/GlobalContext 最终放进 RingSlot 的固定 dispatch 参数位 48/49；它们不是 +// standalone 自定义参数，offset 必须与真实 SPMD kernel 调用约定一致。 +static_assert(sizeof(PaLocalContext) == 48, "PA local context ABI mismatch"); + +struct PaGlobalContext { + int32_t sub_block_id; +}; +static_assert(sizeof(PaGlobalContext) == 4, "PA global context ABI mismatch"); + +// PTO2 profiling is enabled in the PA baseline. reset() clears all 160 bytes +// below on every QK/SF/PV/UP argument rebuild, even though Case1 does not ask +// to dump an argument. Keeping this storage and write stream matters to the +// spacing between consecutive Claim operations. +// 这段看似未使用的清零属于真实前端成本，删除会改变各 worker 到达 +// Claim 的波形与竞争强度，因此仍按生产构造/reset 顺序执行。 +struct PaDumpArgSelection { + uint64_t dump_arg_mask; + uint64_t dump_arg_index_ambiguous_mask; + uint64_t scalar_source_ptrs[kMaxTaskScalars]; + uint8_t scalar_dtypes[kMaxTaskScalars]; +}; +static_assert(sizeof(PaDumpArgSelection) == 160, "PA dump-selection ABI mismatch"); + +struct TaskArgs { + // The real TaskArgsTpl inherits its tag mixin first. TensorArgType is an + // int32 enum in the PA ABI; keeping tags first also reproduces its offsets. + // tag 数组位于对象首部不是任意排布；Materialize、fanin 与 register + // 都会重复扫描/复用这些 tag，错误 offset 会同时改变语义和前端访存成本。 + int32_t tags[kMaxTaskTensors]; + TaskTensorRef tensors[kMaxTaskTensors]; + uint64_t scalars[kMaxTaskScalars]; + int32_t tensor_count; + int32_t scalar_count; + + bool has_error; + uint64_t error_msg; + PaLaunchSpec launch_spec; + PaDumpArgSelection dump_arg_selection; + uint64_t explicit_deps; + uint32_t explicit_dep_count; + uint8_t cacheline_pad[48]; +}; +#if PTO_FDWIC_SHARED_MAP +static_assert(sizeof(TaskArgs) == 1280, "shared TaskArgs must match the PA L0TaskArgs ABI size"); +static_assert(offsetof(TaskArgs, tags) == 0, "shared TaskArgs tag offset mismatch"); +static_assert(offsetof(TaskArgs, tensors) == 128, "shared TaskArgs tensor-ref offset mismatch"); +static_assert(offsetof(TaskArgs, scalars) == 896, "shared TaskArgs scalar offset mismatch"); +static_assert(offsetof(TaskArgs, tensor_count) == 1024, "shared TaskArgs tensor-count offset mismatch"); +static_assert(offsetof(TaskArgs, scalar_count) == 1028, "shared TaskArgs scalar-count offset mismatch"); +static_assert(offsetof(TaskArgs, has_error) == 1032, "shared TaskArgs error flag offset mismatch"); +static_assert(offsetof(TaskArgs, error_msg) == 1040, "shared TaskArgs error pointer offset mismatch"); +static_assert(offsetof(TaskArgs, launch_spec) == 1048, "shared TaskArgs launch-spec offset mismatch"); +static_assert(offsetof(TaskArgs, dump_arg_selection) == 1056, "shared TaskArgs dump-selection offset mismatch"); +static_assert(offsetof(TaskArgs, explicit_deps) == 1216, "shared TaskArgs dependency pointer offset mismatch"); +static_assert(offsetof(TaskArgs, explicit_dep_count) == 1224, "shared TaskArgs dependency count offset mismatch"); +static_assert( + __is_trivially_constructible(TaskArgs), + "shared TaskArgs must not introduce implicit initialization" +); +#else +static_assert(sizeof(TaskArgs) == 1024, "TaskArgs must match the PA L0TaskArgs ABI size"); +static_assert(offsetof(TaskArgs, tags) == 0, "TaskArgs tag offset mismatch"); +static_assert(offsetof(TaskArgs, tensors) == 128, "TaskArgs tensor-ref offset mismatch"); +static_assert(offsetof(TaskArgs, scalars) == 640, "TaskArgs scalar offset mismatch"); +static_assert(offsetof(TaskArgs, tensor_count) == 768, "TaskArgs tensor-count offset mismatch"); +static_assert(offsetof(TaskArgs, scalar_count) == 772, "TaskArgs scalar-count offset mismatch"); +static_assert(offsetof(TaskArgs, has_error) == 776, "TaskArgs error flag offset mismatch"); +static_assert(offsetof(TaskArgs, error_msg) == 784, "TaskArgs error pointer offset mismatch"); +static_assert(offsetof(TaskArgs, launch_spec) == 792, "TaskArgs launch-spec offset mismatch"); +static_assert(offsetof(TaskArgs, dump_arg_selection) == 800, "TaskArgs dump-selection offset mismatch"); +static_assert(offsetof(TaskArgs, explicit_deps) == 960, "TaskArgs dependency pointer offset mismatch"); +static_assert(offsetof(TaskArgs, explicit_dep_count) == 968, "TaskArgs dependency count offset mismatch"); +#endif + +struct TaskOutputs { + uint64_t task_id; + uint32_t count; + PA_GM TensorDesc *tensors[kMaxTaskTensors]; +}; +static_assert(sizeof(TaskOutputs) == 272, "TaskOutputs must match the PA TaskOutputTensors ABI size"); +static_assert(offsetof(TaskOutputs, tensors) == 16, "TaskOutputs tensor pointer offset mismatch"); + +struct SubmitContext { + PA_GM WorkerState *self; + PA_GM TaskPayload *payload; + int32_t task_id; + int32_t tensor_count; + int32_t scalar_count; + uint32_t register_mask; + uint64_t output_bytes; + TaskOutputs result; +#if PTO_FDWIC_SHARED_MAP + SharedTaskOutputs shared_result; +#endif + int32_t fanin[kMaxFanin]; + int32_t fanin_count; + int32_t kernel_id; + bool won; + bool joint; + bool joint_init; + int32_t joint_block; + int32_t joint_slot; + int32_t joint_count; +}; +// SubmitContext 贯穿一次 Submit：Begin 绑定 task/payload，Materialize 填充输出与 +// register_mask，winner 收集 fanin 并构建 slot。它复刻 DistSubmitCtx 而非诊断结构。 +#if PTO_FDWIC_SHARED_MAP +static_assert(sizeof(SubmitContext) == 408, "shared SubmitContext must match DistSubmitCtx"); +static_assert(offsetof(SubmitContext, output_bytes) == 32, "shared SubmitContext output-byte offset mismatch"); +static_assert(offsetof(SubmitContext, result) == 40, "shared SubmitContext result offset mismatch"); +static_assert(offsetof(SubmitContext, shared_result) == 312, "shared SubmitContext result-ref offset mismatch"); +static_assert(offsetof(SubmitContext, fanin) == 320, "shared SubmitContext fanin offset mismatch"); +static_assert( + __is_trivially_constructible(SubmitContext), + "shared SubmitContext must remain trivial for CCEC block-local state" +); +#else +static_assert(sizeof(SubmitContext) == 400, "SubmitContext must match DistSubmitCtx"); +static_assert(offsetof(SubmitContext, output_bytes) == 32, "SubmitContext output-byte offset mismatch"); +static_assert(offsetof(SubmitContext, result) == 40, "SubmitContext result offset mismatch"); +static_assert(offsetof(SubmitContext, fanin) == 312, "SubmitContext fanin offset mismatch"); +#endif + +#if PTO_FDWIC_SHARED_MAP +using PaOutputHandle = FdwicOutputRef; +using OrchestrationTaskOutputs = SharedTaskOutputs; +#else +using PaOutputHandle = PA_GM TensorDesc *; +using OrchestrationTaskOutputs = TaskOutputs; +#endif + +PA_DEVICE const OrchestrationTaskOutputs &OrchestrationOutputs(const SubmitContext &context) { +#if PTO_FDWIC_SHARED_MAP + return context.shared_result; +#else + return context.result; +#endif +} + +PA_DEVICE PaOutputHandle OutputHandleAt(const OrchestrationTaskOutputs &outputs, uint32_t index) { +#if PTO_FDWIC_SHARED_MAP + return outputs.OutputRef(index); +#else + return index < outputs.count ? outputs.tensors[index] : nullptr; +#endif +} + +PA_DEVICE PaOutputHandle InvalidPaOutputHandle() { +#if PTO_FDWIC_SHARED_MAP + return InvalidSharedOutputRef(); +#else + return nullptr; +#endif +} + +struct OutputLayout { + uint64_t buffer_sizes[kMaxTaskTensors]; + uint64_t total_output_size; +}; +// 只有 tag=Output 的槽位拥有有效 buffer_sizes；总大小按 1 KiB 对齐累计，随后 +// 作为 HeapGuard 的 output_bytes 和本 worker heap_next 的推进量。 +static_assert(sizeof(OutputLayout) == 264, "OutputLayout must match DistOutputLayout"); + +// 该状态保存真实 PA orchestration 在五个 Submit 之间传递的输出 handle： +// private 为本 worker materialize payload 中的 descriptor 指针，shared 为 +// (producer_task_id, output_slot) 符号；两者经同一 facade 构建后继参数。 +struct PaOrchestrationState { + TensorDesc query; + TensorDesc key_cache; + TensorDesc value_cache; + TensorDesc block_table; + TensorDesc context_lens; + TensorDesc output; + TensorDesc query_view; + TensorDesc output_view; + + TensorCreateInfo tile_create_info; + TensorCreateInfo scalar_create_info; + TensorCreateInfo qk_create_info; + TensorCreateInfo sf_create_info; + + // The pointer is supplied by the standalone backend. On A5 it must point + // at GM so every batch performs the same descriptor-based load as PA. + PA_GM const volatile int32_t *context_lens_data; + uint64_t scale_bits; + uint64_t current_sequence; + uint64_t current_blocks; + uint64_t current_block_offset; + uint64_t current_nblocks; + uint64_t current_valid_len; + uint32_t current_batch; + + PaOutputHandle accumulated_output; + PaOutputHandle accumulated_sum; + PaOutputHandle accumulated_max; + PaOutputHandle qk_scores; + PaOutputHandle sf_probs; + PaOutputHandle sf_max; + PaOutputHandle sf_sum; + PaOutputHandle pv_output; +}; +#if PTO_FDWIC_SHARED_MAP +static_assert(sizeof(PaOrchestrationState) == 1472, "shared PA orchestration state size changed"); +static_assert( + offsetof(PaOrchestrationState, accumulated_output) == 1340, + "shared PA output-handle offset changed" +); +#else +static_assert(sizeof(PaOrchestrationState) == 1408, "private PA orchestration state size changed"); +static_assert( + offsetof(PaOrchestrationState, accumulated_output) == 1344, + "private PA output-handle offset changed" +); +#endif + +PA_DEVICE uint64_t ElementSize(DataType dtype) { + // 输入 dtype 来自已通过 PA ABI 构造的 descriptor/create-info，必须落在 Count 前； + // 输出字节数同时用于外部 tensor range 与新 Output 的 heap 大小计算。 + constexpr static uint64_t sizes[static_cast(DataType::Count)] = { + 4, 2, 4, 2, 1, 1, 2, 8, 8, 2, 4, 1, + }; + return sizes[static_cast(dtype)]; +} + +PA_DEVICE int32_t TagValue(TensorArgType tag) { return static_cast(tag); } + +PA_DEVICE TensorArgType TaskTag(const TaskArgs &args, uint32_t index) { + // index 的有效范围由 tensor_count 保证；集中转换避免各阶段对 int32 ABI tag + // 做不同解释，Materialize/CollectFanin/Register 因而共享同一分类结果。 + return static_cast(args.tags[index]); +} + +PA_DEVICE void ClearDumpArgSelection(PaDumpArgSelection &selection) { + // Volatile stores intentionally preserve the profiling-enabled PA reset + // traffic even though the standalone winner workload never consumes dump data. + // volatile 的目的不是同步，而是阻止编译器删掉这段生产基线中存在的写流量。 + volatile uint64_t *masks = &selection.dump_arg_mask; + masks[0] = 0; + masks[1] = 0; + volatile uint64_t *sources = &selection.scalar_source_ptrs[0]; + for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { + sources[index] = 0; + } + volatile uint8_t *dtypes = &selection.scalar_dtypes[0]; + for (uint32_t index = 0; index < kMaxTaskScalars; ++index) { + dtypes[index] = 0; + } +} + +PA_DEVICE void ConstructTaskArgs(TaskArgs &args) { + // TensorTagMixin::tags_{} is value-initialized by the + // real L0TaskArgs constructor. TensorRef/scalar slots remain lazy. + // 构造只初始化真实构造函数会触碰的字段，未使用的 tensor/scalar + // 槽保持惰性；整对象 memset 会引入 PA 本身没有的额外前端开销。 + volatile int32_t *tags = &args.tags[0]; + for (uint32_t index = 0; index < kMaxTaskTensors; ++index) { + tags[index] = 0; + } + args.tensor_count = 0; + args.scalar_count = 0; + args.has_error = false; + args.error_msg = 0; + args.launch_spec.core_num = 1; + args.launch_spec.require_sync_start = false; + ClearDumpArgSelection(args.dump_arg_selection); + args.explicit_deps = 0; + args.explicit_dep_count = 0; +} + +PA_DEVICE void ResetTaskArgs(TaskArgs &args) { + // reset 的输出是不含 tensor/scalar/显式依赖的新逻辑参数表，但保留已分配对象及 + // launch_spec；QK/SF/PV/UP 在同一个 1 KiB TaskArgs 上依次复用这一状态。 + args.tensor_count = 0; + args.scalar_count = 0; + ClearDumpArgSelection(args.dump_arg_selection); + args.explicit_deps = 0; + args.explicit_dep_count = 0; + args.has_error = false; + args.error_msg = 0; +} + +PA_DEVICE bool ReserveTensorArgs(TaskArgs &args, int32_t count) { + // tensor 必须先于 scalar 追加，以保持 dispatch args 的 [tensor..., scalar...] + // 排列；失败只置 has_error，不发生部分追加。 + if (args.scalar_count != 0 || count < 0 || + args.tensor_count + count > static_cast(kMaxTaskTensors)) { + args.has_error = true; + return false; + } + return true; +} + +PA_DEVICE void AppendLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { + const uint32_t index = static_cast(args.tensor_count++); + args.tensors[index].pointer.local_tensor = &tensor; + args.tensors[index].kind = TensorRefKind::LocalTensor; + args.tags[index] = TagValue(tag); +} + +PA_DEVICE void AppendGmTensor(TaskArgs &args, PA_GM const TensorDesc &tensor, TensorArgType tag) { + const uint32_t index = static_cast(args.tensor_count++); + args.tensors[index].pointer.gm_tensor = &tensor; + args.tensors[index].kind = TensorRefKind::GmTensor; + args.tags[index] = TagValue(tag); +} + +#if PTO_FDWIC_SHARED_MAP +PA_DEVICE void AppendSharedOutputRef(TaskArgs &args, FdwicOutputRef reference, TensorArgType tag) { + const uint32_t index = static_cast(args.tensor_count++); + args.tensors[index].pointer.output_ref = reference; + args.tensors[index].kind = TensorRefKind::SharedOutputRef; + args.tags[index] = TagValue(tag); +} + +PA_DEVICE bool IsValidSharedOutputRef(FdwicOutputRef reference) { + if (reference.producer_task_id < 0 || + reference.producer_task_id >= static_cast(kMaxTasks) || + reference.output_slot < 0 || + reference.output_slot >= static_cast(kSharedOutputMaxPerTask) || + (reference.flags & ~uint8_t{1}) != 0) { + return false; + } + if ((reference.flags & uint8_t{1}) == 0) { + return reference.view_ndims == 0 && reference.view_shape0 == 0 && + reference.view_offset0 == 0; + } + return reference.view_ndims == 1 && reference.view_shape0 != 0; +} + +PA_DEVICE bool IsPlainSharedOutputRef(FdwicOutputRef reference) { + return IsValidSharedOutputRef(reference) && reference.flags == 0; +} + +PA_DEVICE bool IsSharedOutputReference(const TaskTensorRef &reference) { + return reference.kind == TensorRefKind::SharedOutputRef; +} + +PA_DEVICE FdwicOutputRef SharedOutputReference(const TaskTensorRef &reference) { + return reference.pointer.output_ref; +} +#endif + +PA_DEVICE void AppendOutput(TaskArgs &args, const TensorCreateInfo &create_info) { + const uint32_t index = static_cast(args.tensor_count++); + args.tensors[index].pointer.create_info = &create_info; + args.tensors[index].kind = TensorRefKind::CreateInfo; + args.tags[index] = TagValue(TensorArgType::Output); +} + +PA_DEVICE bool ReserveScalarArgs(TaskArgs &args, int32_t count) { + // 先整体校验容量再由 AddTwo/AddThree 连续写入，保证多 scalar 操作全有或全无。 + if (count < 0 || args.scalar_count + count > static_cast(kMaxTaskScalars)) { + args.has_error = true; + return false; + } + return true; +} + +PA_DEVICE void AppendScalar(TaskArgs &args, uint64_t value) { + args.scalars[static_cast(args.scalar_count++)] = value; +} + +PA_DEVICE void AddLocalTensor(TaskArgs &args, const TensorDesc &tensor, TensorArgType tag) { + if (ReserveTensorArgs(args, 1)) AppendLocalTensor(args, tensor, tag); +} + +PA_DEVICE void AddGmTensor(TaskArgs &args, PA_GM const TensorDesc &tensor, TensorArgType tag) { + if (ReserveTensorArgs(args, 1)) AppendGmTensor(args, tensor, tag); +} + +PA_DEVICE void AddOutputHandleTensor(TaskArgs &args, PaOutputHandle handle, TensorArgType tag) { +#if PTO_FDWIC_SHARED_MAP + if (!IsValidSharedOutputRef(handle)) { + args.has_error = true; + return; + } + if (ReserveTensorArgs(args, 1)) { + AppendSharedOutputRef(args, handle, tag); + } +#else + if (handle == nullptr) { + args.has_error = true; + return; + } + AddGmTensor(args, *handle, tag); +#endif +} + +PA_DEVICE void AddOutput(TaskArgs &args, const TensorCreateInfo &create_info) { + if (ReserveTensorArgs(args, 1)) AppendOutput(args, create_info); +} + +PA_DEVICE void AddScalar(TaskArgs &args, uint64_t value) { + if (ReserveScalarArgs(args, 1)) AppendScalar(args, value); +} + +PA_DEVICE void InitCreateInfo( + TensorCreateInfo &info, const uint32_t shapes[kMaxTensorDims], uint32_t ndims, DataType dtype +) { + info.initial_value = 0; + info.has_initial_value = false; + info.reserved0 = 0; + info.start_offset = 0; + info.version = 0; + info.ndims = ndims; + info.dtype = dtype; + info.manual_dep = false; + info.is_contiguous = true; + info.child_memory = 0; + // TensorCreateInfo's real constructor only writes active dimensions. + // 只写 ndims 个 shape，保留生产构造器的写入范围，不能为方便把五维全清零。 + for (uint32_t index = 0; index < ndims; ++index) { + info.shapes[index] = shapes[index]; + } +} + +PA_DEVICE void ClearCreateInfo(TensorCreateInfo &info) { + volatile uint8_t *bytes = reinterpret_cast(&info); + for (uint32_t index = 0; index < sizeof(TensorCreateInfo); ++index) { + bytes[index] = 0; + } +} + +PA_DEVICE void InitExternalTensor( + TensorDesc &tensor, uint64_t address, const uint32_t shapes[kMaxTensorDims], uint32_t ndims, DataType dtype, + bool manual_dep +) { + // 输入为稳定 backing address、逻辑 shape 和依赖属性；输出是完整连续 descriptor， + // owner 无效表示它不是本轮 task 产生，row-major stride 从末维向前计算。 + uint64_t elements = 1; + for (uint32_t index = 0; index < ndims; ++index) { + elements *= shapes[index]; + } + tensor.buffer_addr = address; + tensor.buffer_size = elements * ElementSize(dtype); + tensor.owner_task_id = kInvalidTaskId; + tensor.start_offset = 0; + tensor.version = 0; + tensor.ndims = ndims; + tensor.dtype = dtype; + tensor.manual_dep = manual_dep; + tensor.is_contiguous = true; + tensor.child_memory = 0; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + tensor.shapes[index] = shapes[index]; + tensor.strides[index] = 0; + } + uint32_t stride = 1; + for (int32_t index = static_cast(ndims) - 1; index >= 0; --index) { + tensor.strides[index] = stride; + stride *= tensor.shapes[index]; + } + tensor.extent_elem_cache = stride; +} + +PA_DEVICE bool InitTensorFromCreateInfo( + PA_GM TensorDesc &tensor, const TensorCreateInfo &info, uint64_t address, uint64_t buffer_size +) { + tensor.buffer_addr = address; + tensor.buffer_size = buffer_size; + tensor.owner_task_id = kInvalidTaskId; + tensor.start_offset = info.start_offset; + tensor.version = info.version; + tensor.ndims = info.ndims; + tensor.dtype = info.dtype; + tensor.manual_dep = info.manual_dep; + tensor.is_contiguous = info.is_contiguous; + tensor.child_memory = info.child_memory; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + tensor.shapes[index] = info.shapes[index]; + } + uint32_t stride = 1; + for (int32_t index = static_cast(tensor.ndims) - 1; index >= 0; --index) { + tensor.strides[index] = stride; + stride *= tensor.shapes[index]; + } + tensor.extent_elem_cache = stride; + // PA can initialize the backing allocation here. Case1 never requests it; + // the standalone uses synthetic heap addresses and therefore rejects that + // unsupported path instead of writing to a fabricated GM pointer. + // Case1 的 has_initial_value 恒为 false；返回 false 是对未模拟分支的 + // 明确保护，不会在合成地址上伪造初始化写入。 + return !info.has_initial_value; +} + +PA_DEVICE uint64_t CreateInfoBytes(const TensorCreateInfo &info) { + uint64_t elements = 1; + for (uint32_t index = 0; index < info.ndims; ++index) { + elements *= info.shapes[index]; + } + return elements * ElementSize(info.dtype); +} + +template +PA_DEVICE void CopyTensorLine1(TensorDesc &destination, const Source &source) { + // view 只需复制 descriptor 第一条 cache line 的身份/shape 字段，随后由调用方 + // 覆盖 offset、shape、stride 与 extent；不做整 128-byte 拷贝以匹配 PA 写流。 + destination.buffer_addr = source.buffer_addr; + destination.buffer_size = source.buffer_size; + destination.owner_task_id = source.owner_task_id; + destination.start_offset = source.start_offset; + destination.version = source.version; + destination.ndims = source.ndims; + destination.dtype = source.dtype; + destination.manual_dep = source.manual_dep; + destination.is_contiguous = source.is_contiguous; + destination.child_memory = source.child_memory; + for (uint32_t index = 0; index < kMaxTensorDims; ++index) { + destination.shapes[index] = source.shapes[index]; + } +} + +PA_DEVICE void MakeCallbackOutputView(PaOrchestrationState &orch, uint32_t batch) { + CopyTensorLine1(orch.output_view, orch.output); + orch.output_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.output_view.ndims = 2; + orch.output_view.manual_dep = true; + orch.output_view.shapes[0] = kPaHeads; + orch.output_view.shapes[1] = kPaHeadDim; + orch.output_view.strides[0] = kPaHeadDim; + orch.output_view.strides[1] = 1; + orch.output_view.extent_elem_cache = kPaHeads * kPaHeadDim; +} + +PA_DEVICE void MakeCallbackQueryView(PaOrchestrationState &orch, uint32_t batch) { + CopyTensorLine1(orch.query_view, orch.query); + orch.query_view.start_offset = static_cast(batch) * kPaHeads * kPaHeadDim; + orch.query_view.ndims = 2; + orch.query_view.shapes[0] = kPaHeads; + orch.query_view.shapes[1] = kPaHeadDim; + orch.query_view.strides[0] = kPaHeadDim; + orch.query_view.strides[1] = 1; + orch.query_view.extent_elem_cache = kPaHeads * kPaHeadDim; +} + +PA_DEVICE uint64_t MinU64(uint64_t lhs, uint64_t rhs) { return lhs < rhs ? lhs : rhs; } + +PA_DEVICE uint64_t ReadPaContextLength(const PaOrchestrationState &orch, uint32_t batch) { + if (orch.context_lens_data == nullptr) { + // Compatibility fallback for a backend that has not yet supplied the + // 256-int GM buffer. Exact PA runs must pass a non-null pointer. + // 正式对等运行必须走下方 descriptor+stride 的 GM load；fallback + // 只用于不具备该缓冲区的兼容后端。 + return kPaBlocksPerRequest * kPaBlockSize; + } + const uint64_t flat_index = orch.context_lens.start_offset + + static_cast(batch) * orch.context_lens.strides[0]; + PA_GM const volatile int32_t *value = reinterpret_cast( + orch.context_lens.buffer_addr + flat_index * ElementSize(DataType::Int32) + ); + return static_cast(*value); +} + +PA_DEVICE void PreparePaBlockGroup(PaOrchestrationState &orch, uint64_t block_offset) { + // 输入 block_offset 位于 [0,current_blocks)；输出 nblocks 最多64，并计算最后 + // 一个 block 的有效 token 数。Case1 只有一个 group，但仍执行通用边界算术。 + orch.current_block_offset = block_offset; + orch.current_nblocks = MinU64(kPaBlocksPerRequest, orch.current_blocks - block_offset); + const uint64_t last_block_sequence_start = + (block_offset + orch.current_nblocks - 1) * kPaBlockSize; + orch.current_valid_len = MinU64(kPaBlockSize, orch.current_sequence - last_block_sequence_start); +} + +PA_DEVICE void BeginPaBatchForCallback(PaOrchestrationState &orch, uint32_t batch) { + // context GM load 与跨 task 共用算术仍在 Submit 之前；只把最终 descriptor + // 打包延后到 Claim 后的同步 callback，避免把业务数据流挪入运行时 finish。 + orch.current_batch = batch; + orch.current_sequence = ReadPaContextLength(orch, batch); + orch.current_blocks = (orch.current_sequence + kPaBlockSize - 1) / kPaBlockSize; +} + +PA_DEVICE void InitPaOrchestration( + PaOrchestrationState &orch, uint32_t batches, PA_GM const volatile int32_t *context_lens_data +) { + // 初始化只建立整轮回放共享的外部 descriptor/create-info 模板；每 batch 的 view、 + // context length、动态 QK/SF shape 和返回 descriptor 留给五阶段流按原顺序更新。 + const uint32_t query_shape[kMaxTensorDims] = {batches * kPaHeads, kPaHeadDim, 0, 0, 0}; + const uint32_t cache_shape[kMaxTensorDims] = { + batches * kPaBlocksPerRequest * kPaBlockSize, kPaHeadDim, 0, 0, 0 + }; + const uint32_t table_shape[kMaxTensorDims] = {batches, kPaMaxBlocksPerRequest, 0, 0, 0}; + const uint32_t context_shape[kMaxTensorDims] = {batches, 0, 0, 0, 0}; + InitExternalTensor(orch.query, kSyntheticQueryBase, query_shape, 2, DataType::Bfloat16, false); + InitExternalTensor(orch.key_cache, kSyntheticKeyBase, cache_shape, 2, DataType::Bfloat16, false); + InitExternalTensor(orch.value_cache, kSyntheticValueBase, cache_shape, 2, DataType::Bfloat16, false); + InitExternalTensor(orch.block_table, kSyntheticBlockTableBase, table_shape, 2, DataType::Int32, false); + const uint64_t context_address = context_lens_data == nullptr + ? kSyntheticContextLensBase + : reinterpret_cast(context_lens_data); + InitExternalTensor(orch.context_lens, context_address, context_shape, 1, DataType::Int32, false); + InitExternalTensor(orch.output, kSyntheticOutputBase, query_shape, 2, DataType::Float32, false); + + const uint32_t tile_shape[kMaxTensorDims] = {kPaHeads, kPaHeadDim, 0, 0, 0}; + const uint32_t scalar_shape[kMaxTensorDims] = {kPaHeads, 0, 0, 0, 0}; + ClearCreateInfo(orch.tile_create_info); + ClearCreateInfo(orch.scalar_create_info); + ClearCreateInfo(orch.qk_create_info); + ClearCreateInfo(orch.sf_create_info); + InitCreateInfo(orch.tile_create_info, tile_shape, 2, DataType::Float32); + InitCreateInfo(orch.scalar_create_info, scalar_shape, 1, DataType::Float32); + + // QK/SF create infos are deliberately not constructed here: in PA they are + // constructed inside the group after Alloc and QK respectively. + // 动态 shape 依赖当前 block group，提前构造既不符合业务数据流，也会 + // 把真实发生在两个 Submit 之间的前端工作错误搬到初始化阶段。 + orch.context_lens_data = context_lens_data; + orch.scale_bits = kPaScaleBits; + orch.current_sequence = 0; + orch.current_blocks = 0; + orch.current_block_offset = 0; + orch.current_nblocks = 0; + orch.current_valid_len = 0; + orch.current_batch = 0; + + orch.accumulated_output = InvalidPaOutputHandle(); + orch.accumulated_sum = InvalidPaOutputHandle(); + orch.accumulated_max = InvalidPaOutputHandle(); + orch.qk_scores = InvalidPaOutputHandle(); + orch.sf_probs = InvalidPaOutputHandle(); + orch.sf_max = InvalidPaOutputHandle(); + orch.sf_sum = InvalidPaOutputHandle(); + orch.pv_output = InvalidPaOutputHandle(); +} + +PA_DEVICE void InitPaOrchestration(PaOrchestrationState &orch, uint32_t batches) { + InitPaOrchestration(orch, batches, nullptr); +} + +struct CallbackSubmitBuildCounts { + uint32_t reset_calls; + uint32_t views_created; + uint32_t dynamic_create_infos; + uint32_t tensor_args_added; + uint32_t scalar_args_added; +}; + +// builder 只在当前 Submit 栈帧内被同步调用，任何 thunk 都不会被保存或跨 TU。 +// builder 内部不判断 winner：private 和 shared Alloc 由所有 actor 调用； +// shared QK/SF/PV/UP 由外层在 Claim 后只让 winner 调用。 +class CallbackSubmitArgsBuilder { +public: + PA_DEVICE CallbackSubmitArgsBuilder(TaskArgs &args, TaskKind kind) + : args_(args), kind_(kind), begin_calls_(0), counts_{} {} + + PA_DEVICE void Begin() { + if (++begin_calls_ != 1) { + args_.has_error = true; + return; + } + if (kind_ == TaskKind::Alloc) { + ConstructTaskArgs(args_); + } else { + ResetTaskArgs(args_); + ++counts_.reset_calls; + } + } + + PA_DEVICE void RecordView() { ++counts_.views_created; } + PA_DEVICE void RecordDynamicCreateInfo() { ++counts_.dynamic_create_infos; } + + template + PA_DEVICE void AddLocalInput(Thunk thunk) { + if (!Ready()) return; + const TensorDesc &tensor = thunk(); + AddLocalTensor(args_, tensor, TensorArgType::Input); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddGmInput(Thunk thunk) { + if (!Ready()) return; + PA_GM const TensorDesc &tensor = thunk(); + AddGmTensor(args_, tensor, TensorArgType::Input); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddOutputHandleInput(Thunk thunk) { + if (!Ready()) return; + const PaOutputHandle handle = thunk(); + AddOutputHandleTensor(args_, handle, TensorArgType::Input); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddOutput(Thunk thunk) { + if (!Ready()) return; + const TensorCreateInfo &create_info = thunk(); + pa_scheduler::AddOutput(args_, create_info); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddLocalInout(Thunk thunk) { + if (!Ready()) return; + const TensorDesc &tensor = thunk(); + AddLocalTensor(args_, tensor, TensorArgType::Inout); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddGmInout(Thunk thunk) { + if (!Ready()) return; + PA_GM const TensorDesc &tensor = thunk(); + AddGmTensor(args_, tensor, TensorArgType::Inout); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddOutputHandleInout(Thunk thunk) { + if (!Ready()) return; + const PaOutputHandle handle = thunk(); + AddOutputHandleTensor(args_, handle, TensorArgType::Inout); + if (!args_.has_error) ++counts_.tensor_args_added; + } + + template + PA_DEVICE void AddScalar(Thunk thunk) { + if (!Ready()) return; + pa_scheduler::AddScalar(args_, thunk()); + if (!args_.has_error) ++counts_.scalar_args_added; + } + + PA_DEVICE bool Valid() const { return begin_calls_ == 1 && !args_.has_error; } + PA_DEVICE const CallbackSubmitBuildCounts &Counts() const { return counts_; } + +private: + PA_DEVICE bool Ready() { + if (begin_calls_ == 1 && !args_.has_error) return true; + args_.has_error = true; + return false; + } + + TaskArgs &args_; + TaskKind kind_; + uint32_t begin_calls_; + CallbackSubmitBuildCounts counts_; +}; + +PA_DEVICE void AcceptTaskOutputs( + PaOrchestrationState &orch, TaskKind kind, const OrchestrationTaskOutputs &outputs +) { + // private 保存本 worker payload descriptor 指针；shared 保存 + // (producer_task_id, output_slot) 符号。上层五阶段 orchestration 只消费 + // PaOutputHandle，不需要在每个业务字段处分散模式宏。 + switch (kind) { + case TaskKind::Alloc: + orch.accumulated_output = OutputHandleAt(outputs, 0); + orch.accumulated_sum = OutputHandleAt(outputs, 1); + orch.accumulated_max = OutputHandleAt(outputs, 2); + break; + case TaskKind::Qk: + orch.qk_scores = OutputHandleAt(outputs, 0); + break; + case TaskKind::Sf: + orch.sf_probs = OutputHandleAt(outputs, 0); + orch.sf_max = OutputHandleAt(outputs, 1); + orch.sf_sum = OutputHandleAt(outputs, 2); + break; + case TaskKind::Pv: + orch.pv_output = OutputHandleAt(outputs, 0); + break; + default: + // UP 只更新既有 Inout，没有新 Output descriptor 需要传给下一阶段。 + break; + } +} + +PA_DEVICE PA_GM uint64_t &TensorMapBucketHead( + PA_GM TensorMap &map, uint32_t bucket +) { +#if PTO_FDWIC_TENSORMAP_RING_CAP < 128 + if (bucket >= 128U) { + return map.extra_bucket_heads[bucket - 128U]; + } +#endif + return map.bucket_heads[bucket]; +} + +PA_DEVICE PA_GM const uint64_t &TensorMapBucketHead( + PA_GM const TensorMap &map, uint32_t bucket +) { +#if PTO_FDWIC_TENSORMAP_RING_CAP < 128 + if (bucket >= 128U) { + return map.extra_bucket_heads[bucket - 128U]; + } +#endif + return map.bucket_heads[bucket]; +} + +PA_DEVICE PA_GM uint64_t &TensorMapBucketTail( + PA_GM TensorMap &map, uint32_t bucket +) { +#if PTO_FDWIC_TENSORMAP_RING_CAP < 128 + if (bucket >= 128U) { + return map.extra_bucket_tails[bucket - 128U]; + } +#endif + return map.bucket_tails[bucket]; +} + +PA_DEVICE PA_GM const uint64_t &TensorMapBucketTail( + PA_GM const TensorMap &map, uint32_t bucket +) { +#if PTO_FDWIC_TENSORMAP_RING_CAP < 128 + if (bucket >= 128U) { + return map.extra_bucket_tails[bucket - 128U]; + } +#endif + return map.bucket_tails[bucket]; +} + +PA_DEVICE void ResetTensorMap(PA_GM TensorMap &map) { + // TensorMap 完全属于当前 worker，private ring 的游标和计数都不需要 + // atomic。entry/ABI 保留区保持惰性，未落在 [head, tail) 的槽不可见。 + map.alive_floor = 0; + map.cleaned_upto = 0; + map.live_count = 0; + map.high_water = 0; + for (uint32_t index = 0; index < kMapBuckets; ++index) { + TensorMapBucketHead(map, index) = 0; + TensorMapBucketTail(map, index) = 0; + } + for (uint32_t index = 0; index < kTaskWindow; ++index) { + map.task_entry_counts[index] = 0; + } +} + +PA_DEVICE uint32_t TensorMapHash(uint64_t address) { +#if PTO_FDWIC_TENSORMAP_RING_CAP == 16384 + // CAP=16384 的隔离门槛只有一个桶；显式返回 0，避免右移 64 + // 的未定义行为。默认 128×128 分支仍生成原来的七位乘法哈希。 + (void)address; + return 0; +#else + address *= 0x9E3779B97F4A7C15ULL; + return static_cast(address >> (64 - kMapBucketShift)) & kMapBucketMask; +#endif +} + +template +PA_DEVICE void TensorByteRange(const TensorReference &tensor, uint64_t &address, uint64_t &lo, uint64_t &hi) { + // identity 先按 backing buffer 地址分桶，再用半开字节区间 [lo, hi) 判断 view + // 是否重叠。连续 tensor 由 shape 现算 extent，非连续 tensor 使用缓存 extent。 + const uint64_t element_size = ElementSize(tensor.dtype); + address = tensor.buffer_addr; + lo = tensor.start_offset * element_size; + uint64_t extent; + if (tensor.is_contiguous) { + extent = 1; + for (uint32_t index = 0; index < tensor.ndims; ++index) { + extent *= tensor.shapes[index]; + } + } else { + extent = tensor.extent_elem_cache; + } + hi = (tensor.start_offset + extent) * element_size; +} + +PA_DEVICE uint32_t TensorMapSlotIndex(uint32_t bucket, uint64_t cursor) { + return bucket * kMapBucketCapacity + + (static_cast(cursor) & kMapBucketSlotMask); +} + +PA_DEVICE void RetireBucket(PA_GM TensorMap &map, uint32_t bucket) { + // 同一 worker 按 task_id 单调 append，因此一个桶内的 producer 也单调 + // 不降。只需从最旧槽开始推进到第一个仍在 alive_floor 内的条目。 + // AdvanceTensorMap 不扫 128 个桶；lookup/insert 触达哪个桶，哪个桶才 + // 物理退休。逻辑 live_count 已在 AdvanceTensorMap 中精确扣减。 + uint64_t head = TensorMapBucketHead(map, bucket); + const uint64_t tail = TensorMapBucketTail(map, bucket); + while (head < tail) { + PA_GM const MapEntry &entry = map.entries[TensorMapSlotIndex(bucket, head)]; + if (entry.producer >= map.alive_floor) { + break; + } + ++head; + } + TensorMapBucketHead(map, bucket) = head; +} + +PA_DEVICE void AdvanceTensorMap(PA_GM TensorMap &map, uint32_t task_id, int32_t heap_window) { + // PrepareMap 在 Claim 后把存活下界推进到 task_id-H。这里按 producer + // 精确扣减 logical live_count 并清空循环计数槽；桶内物理 head 留给 + // RetireBucket 惰性推进。TensorMap 与 heap 共享 H，但前者按本 worker + // task_id 推进，后者按跨核连续 frontier 推进，二者无需同步到同一位置。 + const int32_t new_floor = static_cast(task_id) - heap_window; + if (new_floor <= map.cleaned_upto) { + if (new_floor > map.alive_floor) { + map.alive_floor = new_floor; + } + return; + } + for (int32_t id = map.cleaned_upto; id < new_floor; ++id) { + const uint32_t task_slot = static_cast(id) & kTaskWindowMask; + const uint32_t retired = map.task_entry_counts[task_slot]; + map.live_count -= retired; + map.task_entry_counts[task_slot] = 0; + } + map.cleaned_upto = new_floor; + map.alive_floor = new_floor; +} + +template +PA_DEVICE bool InsertTensor(PA_GM TensorMap &map, const TensorReference &tensor, int32_t producer) { + // 每桶独立 append。先惰性退休该桶，再检查固定容量；满时不覆写旧槽、 + // 不推进 tail/计数，明确返回 false 交由 Submit 上层转成 fatal。 + uint64_t address = 0; + uint64_t lo = 0; + uint64_t hi = 0; + TensorByteRange(tensor, address, lo, hi); + const uint32_t bucket = TensorMapHash(address); + RetireBucket(map, bucket); + const uint64_t head = TensorMapBucketHead(map, bucket); + const uint64_t tail = TensorMapBucketTail(map, bucket); + if (tail - head >= kMapBucketCapacity) { + return false; + } + + PA_GM MapEntry &entry = map.entries[TensorMapSlotIndex(bucket, tail)]; + entry.buffer_addr = address; + entry.lo = lo; + entry.hi = hi; + entry.producer = producer; + TensorMapBucketTail(map, bucket) = tail + 1; + + const uint32_t task_slot = static_cast(producer) & kTaskWindowMask; + ++map.task_entry_counts[task_slot]; + ++map.live_count; + if (map.live_count > map.high_water) { + map.high_water = map.live_count; + } + return true; +} + +template +PA_DEVICE int32_t LookupTensor(PA_GM TensorMap &map, const TensorReference &tensor) { + uint64_t address = 0; + uint64_t lo = 0; + uint64_t hi = 0; + TensorByteRange(tensor, address, lo, hi); + const uint32_t bucket = TensorMapHash(address); + RetireBucket(map, bucket); + const uint64_t head = TensorMapBucketHead(map, bucket); + const uint64_t tail = TensorMapBucketTail(map, bucket); + int32_t best = -1; + // 扫描 [head,tail) 的全部合法槽而非依赖 append 顺序提前返回；同一 + // buffer 的多个历史写者中，只接受 producer>=alive_floor 的重叠条目， + // 并取 producer 最大者，避免依赖退回旧版本。 + for (uint64_t cursor = head; cursor < tail; ++cursor) { + PA_GM const MapEntry &entry = map.entries[TensorMapSlotIndex(bucket, cursor)]; + if (entry.producer < map.alive_floor) { + continue; + } + if (entry.buffer_addr == address && lo < entry.hi && entry.lo < hi && entry.producer > best) { + best = entry.producer; + } + } + return best; +} + +PA_DEVICE uint64_t TensorOwner(const TaskTensorRef &reference) { + // CreateInfo 只会出现在 tag=Output 且在 fanin 前已被跳过；这里的输入不变量是 + // LocalTensor/GmTensor，输出为显式 owner 或 kInvalidTaskId。 + if (reference.kind == TensorRefKind::GmTensor) { + return reference.pointer.gm_tensor->owner_task_id; + } + return reference.pointer.local_tensor->owner_task_id; +} + +PA_DEVICE int32_t LookupTensorRef(PA_GM TensorMap &map, const TaskTensorRef &reference) { + // 与 TensorOwner 相同，此辅助入口只接收已存在 descriptor；返回最新重叠 producer， + // 未登记或已退休则返回 -1。 + if (reference.kind == TensorRefKind::GmTensor) { + return LookupTensor(map, *reference.pointer.gm_tensor); + } + return LookupTensor(map, *reference.pointer.local_tensor); +} + +PA_DEVICE void AddFanin(int32_t fanin[kMaxFanin], uint32_t &count, int32_t producer) { + // owner 与 TensorMap lookup 可能指向同一 producer，先去重再写固定 16 槽数组； + // Case1 的最大 fanin 为 UP 的 3，正常路径不会截断。 + if (producer < 0) { + return; + } + for (uint32_t index = 0; index < count; ++index) { + if (fanin[index] == producer) { + return; + } + } + if (count < kMaxFanin) { + fanin[count++] = producer; + } +} + +PA_DEVICE uint32_t CollectFanin( + PA_GM TensorMap &map, const TaskArgs &args, int32_t fanin[kMaxFanin] +) { + // fanin 只由 winner 收集：先吸收 descriptor 的显式 owner，再对 Input/Inout + // 查询最新重叠写者；纯 Output 尚未存在，不应成为本次 task 的输入依赖。 + uint32_t count = 0; + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = TaskTag(args, static_cast(index)); + if (tag == TensorArgType::Output) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + // Keep the two address spaces in separate control-flow arms. CCEC's + // O2/O3 backend rejects a merged pointer phi even when both arms only + // feed scalar field loads; this is also how PA's production helper is + // written. + // 分支重复是后端约束与生产写法的一部分，不应抽成一个混合地址空间指针。 + if (reference.kind == TensorRefKind::GmTensor) { + PA_GM const TensorDesc &tensor = *reference.pointer.gm_tensor; + const uint64_t owner = tensor.owner_task_id; + if (owner != kInvalidTaskId) { + AddFanin(fanin, count, static_cast(owner & 0xFFFFFFFFU)); + } + if (tag == TensorArgType::Input || tag == TensorArgType::Inout) { + AddFanin(fanin, count, LookupTensor(map, tensor)); + } + } else { + const TensorDesc &tensor = *reference.pointer.local_tensor; + const uint64_t owner = tensor.owner_task_id; + if (owner != kInvalidTaskId) { + AddFanin(fanin, count, static_cast(owner & 0xFFFFFFFFU)); + } + if (tag == TensorArgType::Input || tag == TensorArgType::Inout) { + AddFanin(fanin, count, LookupTensor(map, tensor)); + } + } + } + return count; +} + +PA_DEVICE bool InsertExistingTensor(SubmitContext &context, const TaskArgs &args, int32_t index) { + // 输入 index 来自 register_mask，故必为已有 descriptor 而非 CreateInfo；写入结果 + // 只影响 context.self 对应 worker 的 map，并把当前 task_id 登记为新的 hazard 版本。 + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind == TensorRefKind::GmTensor) { + return InsertTensor(context.self->map, *reference.pointer.gm_tensor, context.task_id); + } + return InsertTensor(context.self->map, *reference.pointer.local_tensor, context.task_id); +} + +PA_DEVICE bool RegisterOutputs(SubmitContext &context, const TaskArgs &args, bool include_existing) { + // register_mask 只覆盖 Inout/OutputExisting。新 Output 已带本次 owner；现有 + // backing buffer 的新写者则必须登记到本 worker TensorMap，供后继 task 查 hazard。 + if (!include_existing) { + return true; + } + uint32_t register_mask = context.register_mask; + for (uint32_t index = 0; register_mask != 0; ++index, register_mask >>= 1) { + if ((register_mask & 1U) != 0) { + if (!InsertExistingTensor(context, args, static_cast(index))) { + return false; + } + } + } + return true; +} + +PA_DEVICE uint64_t FrontendAlignUp(uint64_t value, uint64_t alignment) { + // alignment 在本模型中固定为2的幂1 KiB；返回逻辑 heap 地址，不做 ring 取模。 + return (value + alignment - 1) & ~(alignment - 1); +} + +#if PTO_FDWIC_SHARED_MAP +PA_DEVICE bool SharedCreateInfoBytes( + const TensorCreateInfo &info, uint64_t &bytes +) { + bytes = 0; + // shared cursor 一旦推进就不能交给另一个 task 回退。先完整校验本 + // standalone 确实模拟的连续、无初值 CreateInfo，并用除法保护每次乘法。 + if (info.ndims == 0 || info.ndims > kMaxTensorDims || + info.dtype >= DataType::Count || info.has_initial_value || + info.start_offset != 0 || !info.is_contiguous || + info.child_memory != 0) { + return false; + } + // descriptor stride 本身只有 32 bit，因此直接在 uint32_t 域做上界 + // 校验；这也避免 CCEC 为通用 64x64 溢出检测引入设备端不存在的 + // __multi3 运行库调用。 + uint32_t elements = 1; + for (uint32_t dimension = 0; dimension < info.ndims; ++dimension) { + const uint32_t extent = info.shapes[dimension]; + if (extent == 0 || elements > UINT32_MAX / extent) { + return false; + } + elements *= extent; + } + const uint64_t element_size = ElementSize(info.dtype); + if (element_size == 0) { + return false; + } + bytes = static_cast(elements) * element_size; + return true; +} +#endif + +#if PTO_FDWIC_SHARED_MAP +template +#endif +PA_DEVICE bool MaterializeTask( + PA_GM WorkerState &worker, uint32_t task_id, const TaskArgs &args, SubmitContext &context, +#if PTO_FDWIC_SHARED_MAP + PA_GM SharedTensorMapSidecar &shared_map, +#endif + uint64_t heap_base, uint64_t heap_size +#if PTO_FDWIC_SHARED_MAP + , TaskKind task_kind, uint32_t batch_start, + uint32_t group_index, + TraceContext *atomic_trace = nullptr, + WorkerResult *atomic_result = nullptr +#endif +) { + // 输入是 BeginCallbackSubmit 已绑定的 payload/context 与当前 worker.heap_next；成功输出 + // 包括本 task 的 GM TensorDesc 指针、output_bytes 和推进后的单调 heap_next。 + // 失败不得进入 slot/build 流程，由上层设置 fatal 并终止该 worker 回放。 + // compete-first 路径在这里已完成 Claim，因此不能再把“尚未 Claim”当作 + // 这个共用 helper 的前置条件。 + if (context.payload == nullptr) { + return false; + } +#if PTO_FDWIC_SHARED_MAP + if (task_id >= kMaxTasks || + args.tensor_count < 0 || + args.tensor_count > static_cast(kMaxTaskTensors) || + args.scalar_count < 0 || + args.scalar_count > static_cast(kMaxTaskScalars) || + args.has_error || + context.result.task_id != task_id || + context.result.count != 0 || + context.shared_result.TaskId() != static_cast(task_id) || + context.shared_result.Size() != FrontendTaskOutputCount( + task_kind + )) { + return false; + } + TaskKind expected_task_kind = TaskKind::Count; + if (!SharedPaTaskKindInBatch( + task_id, batch_start, group_index, + expected_task_kind + ) || + expected_task_kind != task_kind) { + return false; + } +#endif + context.tensor_count = args.tensor_count; + context.scalar_count = args.scalar_count; + context.register_mask = 0; + + // DistOutputLayout leaves non-output slots lazy and writes only the sizes + // selected by output_mask. + // 第一次 tag 扫描同时产生 output_mask/register_mask；第二次只遍历 + // Output 位，避免读取未初始化的非输出 buffer_sizes。 + OutputLayout layout; + layout.total_output_size = 0; + uint32_t output_mask = 0; + uint32_t output_count = 0; + for (int32_t index = 0; index < args.tensor_count; ++index) { +#if PTO_FDWIC_SHARED_MAP + if (args.tags[index] < static_cast(TensorArgType::Input) || + args.tags[index] > + static_cast(TensorArgType::NoDependency)) { + return false; + } +#endif + const TensorArgType tag = TaskTag(args, static_cast(index)); + if (tag == TensorArgType::Inout || tag == TensorArgType::OutputExisting) { + context.register_mask |= 1U << index; + } + if (tag != TensorArgType::Output) { +#if PTO_FDWIC_SHARED_MAP + // 先把所有后续 CollectFanin/Register 可能解引用的 active ref + // 验证完，再允许任一 shared cursor 推进。symbol 只接受当前 + // 已接入的 plain 形态，且 producer/slot 必须属于已声明输出。 + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind == TensorRefKind::LocalTensor) { + if (reference.pointer.local_tensor == nullptr) { + return false; + } + } else if (reference.kind == TensorRefKind::GmTensor) { + if (reference.pointer.gm_tensor == nullptr) { + return false; + } + } else if (reference.kind == TensorRefKind::SharedOutputRef) { + const FdwicOutputRef output_ref = + SharedOutputReference(reference); + if (!IsPlainSharedOutputRef(output_ref) || + output_ref.producer_task_id < 0 || + output_ref.producer_task_id >= + static_cast(task_id) || + (tag != TensorArgType::Input && + tag != TensorArgType::Inout && + tag != TensorArgType::OutputExisting)) { + return false; + } + TaskKind producer_kind = TaskKind::Count; + if (!SharedPaTaskKindInBatch( + static_cast( + output_ref.producer_task_id + ), + batch_start, group_index, producer_kind + ) || + output_ref.output_slot >= + static_cast( + FrontendTaskOutputCount(producer_kind) + )) { + return false; + } + } else { + return false; + } +#endif + continue; + } +#if PTO_FDWIC_SHARED_MAP + // Output 必须显式携带 CreateInfo，不能把 union 中其他地址空间的 + // 位模式误解引用；大小计算失败时也必须在触碰 heap 控制字前退出。 + if (args.tensors[index].kind != TensorRefKind::CreateInfo) { + return false; + } + if (args.tensors[index].pointer.create_info == nullptr) { + return false; + } + const TensorCreateInfo &create_info = + *args.tensors[index].pointer.create_info; + if (!SharedCreateInfoBytes( + create_info, layout.buffer_sizes[index] + )) { + return false; + } +#else + layout.buffer_sizes[index] = + CreateInfoBytes(*args.tensors[index].pointer.create_info); +#endif + output_mask |= 1U << index; + ++output_count; +#if PTO_FDWIC_SHARED_MAP + if (layout.buffer_sizes[index] > + UINT64_MAX - (kOutputAlignment - 1)) { + return false; + } +#endif + const uint64_t aligned_size = + FrontendAlignUp(layout.buffer_sizes[index], kOutputAlignment); +#if PTO_FDWIC_SHARED_MAP + if (aligned_size < layout.buffer_sizes[index] || + UINT64_MAX - layout.total_output_size < aligned_size) { + return false; + } +#endif + layout.total_output_size += aligned_size; + } +#if PTO_FDWIC_SHARED_MAP + if (output_count > kSharedOutputMaxPerTask || + output_count != context.shared_result.Size()) { + return false; + } +#else + (void)output_count; +#endif + + const uint64_t total = layout.total_output_size; +#if PTO_FDWIC_SHARED_MAP + if (total != 0 && + (heap_base == 0 || heap_base > UINT64_MAX - heap_size)) { + return false; + } + SharedHeapReservation reservation{}; + if (!ReserveSharedOutputHeap( + shared_map, task_id, total, heap_size, reservation, + atomic_trace, atomic_result + )) { + return false; + } + uint64_t task_base = reservation.task_base; + worker.heap_next = reservation.aggregate_vend; +#else + uint64_t task_base = FrontendAlignUp(worker.heap_next, kOutputAlignment); + if (total > heap_size || (total != 0 && heap_base == 0)) { + return false; + } + if (total != 0 && (task_base % heap_size) + total > heap_size) { + // 单个 task 的输出必须物理连续；若跨 ring 尾部则把逻辑 task_base 推到 + // 下一圈起点。heap_next 仍保持单调，不在这里取模。 + task_base = (task_base / heap_size + 1) * heap_size; + } +#endif + + uint64_t output_offset = 0; + // 各 Output 在同一 task_base 内按参数顺序排布；result 只收集 Output，索引与 + // TaskArgs 中非输出槽无关，而 payload 仍按原参数 index 保存 descriptor。 + for (int32_t index = 0; output_mask != 0; ++index, output_mask >>= 1) { + if ((output_mask & 1U) == 0) { + continue; + } +#if PTO_FDWIC_SHARED_MAP + const uint64_t physical = task_base + output_offset; +#else + const uint64_t physical = (task_base + output_offset) % heap_size; +#endif + PA_GM TensorDesc &tensor = context.payload->tensors[index]; + if (!InitTensorFromCreateInfo( + tensor, *args.tensors[index].pointer.create_info, heap_base + physical, layout.buffer_sizes[index] + )) { + return false; + } + tensor.owner_task_id = task_id; + const uint32_t output_ordinal = context.result.count; + context.result.tensors[output_ordinal] = &tensor; + ++context.result.count; + output_offset += FrontendAlignUp(layout.buffer_sizes[index], kOutputAlignment); + } +#if !PTO_FDWIC_SHARED_MAP + worker.heap_next = task_base + total; +#endif + context.output_bytes = total; + return true; +} + +PA_DEVICE void CopyTensorFromRef(PA_GM TensorDesc &destination, const TaskTensorRef &reference) { + // slot 必须拥有 descriptor 快照，不能保存指向 orchestration 栈对象的引用； + // 按 byte volatile copy 同时兼容 local/GM 源并保留真实 128-byte 搬运量。 + PA_GM volatile uint8_t *destination_bytes = reinterpret_cast(&destination); + if (reference.kind == TensorRefKind::GmTensor) { + PA_GM const volatile uint8_t *source_bytes = + reinterpret_cast(reference.pointer.gm_tensor); + for (uint32_t byte = 0; byte < sizeof(TensorDesc); ++byte) { + destination_bytes[byte] = source_bytes[byte]; + } + return; + } + const volatile uint8_t *source_bytes = + reinterpret_cast(reference.pointer.local_tensor); + for (uint32_t byte = 0; byte < sizeof(TensorDesc); ++byte) { + destination_bytes[byte] = source_bytes[byte]; + } +} + +PA_DEVICE void CopyGmTensor(PA_GM TensorDesc &destination, PA_GM const TensorDesc &source) { + // 新 Output 的源 descriptor 已位于 GM payload；单独入口避免把 GM 指针误走 + // local 地址空间分支，输出仍是 slot 内独立副本。 + PA_GM volatile uint8_t *destination_bytes = reinterpret_cast(&destination); + PA_GM const volatile uint8_t *source_bytes = reinterpret_cast(&source); + for (uint32_t byte = 0; byte < sizeof(TensorDesc); ++byte) { + destination_bytes[byte] = source_bytes[byte]; + } +} + +#if PTO_FDWIC_SHARED_MAP +template +PA_DEVICE void PopulateSlotPayloadImpl( +#else +PA_DEVICE void PopulateSlotPayload( +#endif + PA_GM LocalSlot &slot, const TaskArgs &args, const SubmitContext &context, const int32_t fanin[kMaxFanin], + uint32_t fanin_count, +#if PTO_FDWIC_SHARED_MAP + PA_GM SharedTensorMapSidecar *shared_map, + TraceContext *dcci_trace, +#endif + int32_t sub_block_id, bool is_multicore, int32_t won_block, + int32_t won_slot +) { + // winner 将活动 descriptor/scalar 复制进私有 slot，dispatch args 指向 slot 内 + // 副本而非 orchestration 临时对象；fanin 随 slot 保存，kernel 执行前逐 flag 检查。 + slot.tensor_count = context.tensor_count; + slot.scalar_count = context.scalar_count; + for (int32_t index = 0; index < context.tensor_count; ++index) { + if (TaskTag(args, static_cast(index)) == TensorArgType::Output) { + CopyGmTensor(slot.tensors[index], context.payload->tensors[index]); +#if PTO_FDWIC_SHARED_MAP + } else if (IsSharedOutputReference(args.tensors[index])) { + if constexpr (SharedDescriptorsDirect) { + // Materialize + Collect 已验证该 ready ref。把 invalidate/copy + // 融入既有 slot tensor 扫描，避免独立 helper 再遍历一遍 + // args；shared_map 非空是生产 BuildWinner 的内部前置条件。 + const FdwicOutputRef output_ref = + SharedOutputReference(args.tensors[index]); + PA_GM const TensorDesc &shared_tensor = + shared_map->shared_outputs[ + static_cast( + output_ref.producer_task_id + ) + ].tensors[output_ref.output_slot]; + (void)TraceConfiguredDcciInvalidate< + SharedCopyOps, SharedDescriptorsDirect + >( + dcci_trace, slot.task_id, + static_cast(slot.kind), + DcciSite::SharedWinnerBuildDescriptorInvalidate, + &shared_tensor, sizeof(shared_tensor) + ); + CopyGmTensor(slot.tensors[index], shared_tensor); + } else { + // 兼容入口仍允许调用方预先把 descriptor 放进 TaskPayload； + // 编译期布尔量保证生产 direct-to-slot 实例不产生该分支。 + CopyGmTensor( + slot.tensors[index], context.payload->tensors[index] + ); + } +#endif + } else { + CopyTensorFromRef(slot.tensors[index], args.tensors[index]); + } + slot.args[index] = static_cast(reinterpret_cast(&slot.tensors[index])); + } + for (int32_t index = 0; index < context.scalar_count; ++index) { + slot.scalars[index] = args.scalars[index]; + slot.args[context.tensor_count + index] = args.scalars[index]; + } + + PA_GM PaLocalContext &local = + *reinterpret_cast(&slot.local_context[0]); + // standalone 每个 task 只由一个 lane kernel 执行，故 block_index/count 固定0/1； + // async completion 未启用，task_token 保持 invalid，与 PA 普通同步 slot 一致。 + local.block_index = 0; + local.block_count = 1; + local.async.completion_count = 0; + local.async.completion_error_code = 0; + local.async.completion_entries = 0; + local.async.completion_capacity = 0; + local.async.task_token = kInvalidTaskId; + slot.global_context = static_cast(sub_block_id); + slot.args[kSpmdLocalContextIndex] = + static_cast(reinterpret_cast(&slot.local_context[0])); + slot.args[kSpmdGlobalContextIndex] = + static_cast(reinterpret_cast(&slot.global_context)); + slot.fanin_count = fanin_count; + // fanin 数组只复制有效前缀；执行端以 fanin_count 为边界，未使用尾部保持惰性。 + for (uint32_t index = 0; index < fanin_count; ++index) { + slot.fanin[index] = fanin[index]; + } + slot.is_multicore = is_multicore; + slot.won_block = won_block; + slot.won_slot = won_slot; +} + +#if PTO_FDWIC_SHARED_MAP +template +#endif +PA_DEVICE void BuildSlotPayload( + PA_GM LocalSlot &slot, uint32_t task_id, uint32_t function_id, uint64_t function_address, const TaskArgs &args, + const SubmitContext &context, const int32_t fanin[kMaxFanin], + uint32_t fanin_count, +#if PTO_FDWIC_SHARED_MAP + PA_GM SharedTensorMapSidecar &shared_map, + TraceContext *dcci_trace = nullptr, +#endif + int32_t sub_block_id = 0, + bool is_multicore = false, int32_t won_block = -1, int32_t won_slot = -1 +) { + // Match build_ring_slot_from_submit ordering: publish the header first, + // then copy the active descriptors/scalars and construct dispatch payload. + // slot 仅由所属 worker 消费，这里的写入次序用于复刻真实构建成本与 + // 状态机；跨核可见性由 task completion 的 flag/vend 协议承担。 + slot.occupied = true; + slot.task_id = task_id; + slot.kind = function_id; + slot.function_address = function_address; + slot.built = 1; +#if PTO_FDWIC_SHARED_MAP + PopulateSlotPayloadImpl( + slot, args, context, fanin, fanin_count, &shared_map, dcci_trace, + sub_block_id, is_multicore, won_block, won_slot + ); +#else + PopulateSlotPayload( + slot, args, context, fanin, fanin_count, sub_block_id, is_multicore, + won_block, won_slot + ); +#endif +} + +// Compatibility overload for a core that has already populated the slot +// header before calling the PA frontend. +// 该入口只补 payload，不改变既有 task/function 头；输出不变量与完整 +// BuildSlotPayload 相同，均得到 built 且可由 DrainReady 检查 fanin 的私有 slot。 +PA_DEVICE void BuildSlotPayload( + PA_GM LocalSlot &slot, const TaskArgs &args, const SubmitContext &context, const int32_t fanin[kMaxFanin], + uint32_t fanin_count +) { + slot.built = 1; +#if PTO_FDWIC_SHARED_MAP + // false 实例按旧契约从 context.payload 取 shared descriptor，编译期 + // 丢弃 direct 分支，因此内部空 map 不会产生读取或运行时判断。 + PopulateSlotPayloadImpl( + slot, args, context, fanin, fanin_count, nullptr, nullptr, + 0, false, -1, -1 + ); +#else + PopulateSlotPayload( + slot, args, context, fanin, fanin_count, 0, false, -1, -1 + ); +#endif +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_FRONTEND_H diff --git a/tests/atomic_probe/pa_scheduler/common/pa_model.h b/tests/atomic_probe/pa_scheduler/common/pa_model.h new file mode 100644 index 0000000000..799b257372 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/common/pa_model.h @@ -0,0 +1,1963 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_MODEL_H +#define PA_SCHEDULER_COMMON_PA_MODEL_H + +#include +#include + +// standalone 与真实 FDWIC 使用同一个模式宏，便于先在这里验证模式边界， +// 再把已证明的机制迁移到 runtime。构建脚本始终显式传 0/1；头文件默认 +// private，保证原有命令和既有性能基线不变。 +#ifndef PTO_FDWIC_SHARED_MAP +#define PTO_FDWIC_SHARED_MAP 0 +#endif + +#if PTO_FDWIC_SHARED_MAP != 0 && PTO_FDWIC_SHARED_MAP != 1 +#error "PTO_FDWIC_SHARED_MAP must be 0 (private) or 1 (shared)" +#endif + +// standalone 的正式产物仍固定使用已验证的 CAP=128。隔离 ring 门槛会用 +// 同一份生产 helper 重编译多个 CAP，证明“每桶连续环”没有偷写 128。 +// 这里故意采用构建期常量：hash、slot mask 与桶跨度都可被 CCEC 常量折叠； +// 运行期 auto/覆盖参数还需要静态任务图 planner，不能在本阶段冒充完成。 +#ifndef PTO_FDWIC_TENSORMAP_RING_CAP +#define PTO_FDWIC_TENSORMAP_RING_CAP 128 +#endif + +// shared writer 插入仍是一条全局 task-id 顺序链，只把相邻 token 交错 +// 放到 1/2/4/8/16/32/64/128 条独立 cache line，分散 future owner +// 的等待 load。 +// 该值是构建身份，不允许运行期改变；private 构建不会读取这些控制字。 +#ifndef PTO_FDWIC_SHARED_INSERT_TURN_GROUPS +#define PTO_FDWIC_SHARED_INSERT_TURN_GROUPS 1 +#endif + +#if PTO_FDWIC_SHARED_INSERT_TURN_GROUPS != 1 && \ + PTO_FDWIC_SHARED_INSERT_TURN_GROUPS != 2 && \ + PTO_FDWIC_SHARED_INSERT_TURN_GROUPS != 4 && \ + PTO_FDWIC_SHARED_INSERT_TURN_GROUPS != 8 && \ + PTO_FDWIC_SHARED_INSERT_TURN_GROUPS != 16 && \ + PTO_FDWIC_SHARED_INSERT_TURN_GROUPS != 32 && \ + PTO_FDWIC_SHARED_INSERT_TURN_GROUPS != 64 && \ + PTO_FDWIC_SHARED_INSERT_TURN_GROUPS != 128 +#error "PTO_FDWIC_SHARED_INSERT_TURN_GROUPS must be a power of two from 1 through 128" +#endif + +#if !PTO_FDWIC_SHARED_MAP && \ + PTO_FDWIC_SHARED_INSERT_TURN_GROUPS != 1 +#error "shared insert-turn groups only apply to shared TensorMap builds" +#endif + +// S0 的 fail-closed 门禁在 S2 接入真实 shared sidecar 后解除。模式仍由 +// 三镜像统一的构建身份与 manifest 锁定，不能把 shared 目录指向 private 实现。 + +// 三类证据链在编译期严格互斥：swimlane 保存普通阶段与 atomic 记录， +// submit-pmu 只保留 PMU 窗口，perf-clock 则只增加首个/末个 Submit +// 两个性能时间边界。未显式传宏的既有后端继续使用原有通用实现。 +#ifndef PA_BUILD_SWIMLANE +#define PA_BUILD_SWIMLANE 0 +#endif + +#ifndef PA_BUILD_SUBMIT_PMU +#define PA_BUILD_SUBMIT_PMU 0 +#endif + +#ifndef PA_BUILD_PERF_CLOCK +#define PA_BUILD_PERF_CLOCK 0 +#endif + +#ifndef PA_BUILD_ATOMIC_SWIMLANE +#define PA_BUILD_ATOMIC_SWIMLANE 0 +#endif + +#ifndef PA_BUILD_COMPACT_GENERIC_TRACE +#define PA_BUILD_COMPACT_GENERIC_TRACE 0 +#endif + +#if PA_BUILD_COMPACT_GENERIC_TRACE != 0 && \ + PA_BUILD_COMPACT_GENERIC_TRACE != 1 +#error "PA_BUILD_COMPACT_GENERIC_TRACE must be 0 or 1" +#endif + +#if (PA_BUILD_SWIMLANE + PA_BUILD_SUBMIT_PMU + PA_BUILD_PERF_CLOCK) > 1 +#error "swimlane, submit-pmu, and perf-clock builds are mutually exclusive" +#endif + +#if PA_BUILD_ATOMIC_SWIMLANE && !PA_BUILD_SWIMLANE +#error "the compile-time atomic trace specialization requires a swimlane build" +#endif + +#if PA_BUILD_COMPACT_GENERIC_TRACE && \ + (!PTO_FDWIC_SHARED_MAP || !PA_BUILD_SWIMLANE || \ + !PA_BUILD_ATOMIC_SWIMLANE) +#error "compact generic trace is restricted to shared full-swimlane builds" +#endif + +#if PA_BUILD_COMPACT_GENERIC_TRACE && \ + PTO_FDWIC_TENSORMAP_RING_CAP != 128 +#error "compact generic trace currently supports only the production CAP=128 ABI" +#endif + +// submit-pmu 与 perf-clock 都不允许把阶段泳道、atomic 包围计时或 +// phase-profile 模板带入最终 ELF。统一谓词避免各 helper 对“无 trace” +// 的理解逐渐分叉；它不代表 PMU 已开启。 +#define PA_BUILD_TRACE_FREE (PA_BUILD_SUBMIT_PMU || PA_BUILD_PERF_CLOCK) + +namespace pa_scheduler { + +enum class TensorMapBuildMode : uint32_t { + Private = 0, + Shared = 1, +}; + +constexpr TensorMapBuildMode kCompiledTensorMapMode = + static_cast(PTO_FDWIC_SHARED_MAP); +constexpr uint32_t kBuildIdentityMagic = 0x50414249U; // "PABI" +constexpr uint32_t kBuildIdentityCompactGenericTraceBit = + 1U << 31U; +#if PTO_FDWIC_SHARED_MAP +constexpr uint32_t kBuildIdentityAbiGeneration = 11; +#else +constexpr uint32_t kBuildIdentityAbiGeneration = 4; +#endif +// 默认 CAP=128 时，private 保留历史 ABI 值；shared generation 11 另把 +// active insert-turn G 编入低位。这样既避免 private AIC/AIV 入口因身份 +// 元数据多一条大立即数构造，也让 manifest v4 和 host/device 握手共同 +// 拒绝不同 G 的 shared 混件。非默认隔离变体继续把 CAP 编进 ABI。 +#if PTO_FDWIC_TENSORMAP_RING_CAP == 128 +#if PTO_FDWIC_SHARED_MAP +constexpr uint32_t kBuildIdentityAbiVersion = + (kBuildIdentityAbiGeneration << 8U) | + static_cast( + PTO_FDWIC_SHARED_INSERT_TURN_GROUPS + ) | + (PA_BUILD_COMPACT_GENERIC_TRACE + ? kBuildIdentityCompactGenericTraceBit + : 0U); +#else +constexpr uint32_t kBuildIdentityAbiVersion = + kBuildIdentityAbiGeneration; +#endif +#else +#if PTO_FDWIC_SHARED_MAP +constexpr uint32_t kBuildIdentityAbiVersion = + (kBuildIdentityAbiGeneration << 24U) | + (static_cast( + PTO_FDWIC_SHARED_INSERT_TURN_GROUPS + ) << 16U) | + static_cast(PTO_FDWIC_TENSORMAP_RING_CAP); +#else +constexpr uint32_t kBuildIdentityAbiVersion = + (kBuildIdentityAbiGeneration << 16U) | + static_cast(PTO_FDWIC_TENSORMAP_RING_CAP); +#endif +#endif +static_assert( + (kBuildIdentityAbiVersion & + kBuildIdentityCompactGenericTraceBit) == + (PA_BUILD_COMPACT_GENERIC_TRACE + ? kBuildIdentityCompactGenericTraceBit + : 0U), + "build identity does not encode the compact trace format" +); + +// private 与现有 shared 单组 Case1 每 batch 都回放五个 task。shared +// 多组请求复用一次 Alloc，随后每个 block group 增加 QK/SF/PV/UP 四 +// task；ticket 的两个 group bit 与 PA 256/64=4 组上限一致。实际组数 +// 仍由每个 batch 的 context_len 决定，不是编译期固定为四组。 +constexpr uint32_t kDefaultBatches = 256; +#if PTO_FDWIC_SHARED_MAP +// shared standalone 扩展到 512 batch，用于保持每 batch 的默认 PA-G1 +// 业务不变并把总 task 从 1,280 增至 2,560。shared 多 group 仍受独立 +// kMaxTasks 总容量约束，不能把 batch 上限误解为任意 context 都可达； +// private 继续保持原有 256-batch ABI 和测试边界。 +constexpr uint32_t kMaxBatches = 512; +#else +constexpr uint32_t kMaxBatches = kDefaultBatches; +#endif +constexpr uint32_t kTasksPerBatch = 5; +#if PTO_FDWIC_SHARED_MAP +constexpr uint32_t kSharedPaMaxBlockGroups = 4; +constexpr uint32_t kSharedPaMaxTasksPerBatch = + 1U + 4U * kSharedPaMaxBlockGroups; +// 保留现有 4,352-task output/history 物理布局；它既覆盖原 256 batch +// 的 PA-G4 最坏计划，也覆盖新增 512 batch 的默认 PA-G1 计划。 +// 512 batch 的多 group 计划若超过该总量，会由 host/device plan +// 在触碰共享状态前 fail closed。 +constexpr uint32_t kMaxTasks = + kDefaultBatches * kSharedPaMaxTasksPerBatch; +static_assert( + kSharedPaMaxTasksPerBatch == 17 && kMaxTasks == 4352 && + kMaxTasks >= kMaxBatches * kTasksPerBatch, + "shared PA task capacity no longer covers PA-G4/B256 and PA-G1/B512" +); +#else +constexpr uint32_t kMaxTasks = kMaxBatches * kTasksPerBatch; +#endif +constexpr uint32_t kTaskCellCapacity = 1U << 16; + +constexpr uint32_t kAicWorkers = 32; +constexpr uint32_t kAivWorkers = 64; +constexpr uint32_t kWorkers = kAicWorkers + kAivWorkers; +constexpr uint32_t kRuntimeMaxWorkers = 108; +constexpr uint32_t kCursorShards = 4; +// S4.14a 已建立 shared Vector 四分片迁址对照；S4.14b 继续使用相同的 +// sidecar 地址、物理容量和代码骨架，只启用此前预留的后四条物理线。 +// device 热路径仍使用同一取模表达式，唯一数值变量是 active shards +// 从4改8，以便单独归因分片收益。 +constexpr uint32_t kSharedVectorCursorCapacity = 8; +constexpr uint32_t kSharedVectorCursorShards = 8; +constexpr uint32_t kSharedVectorCursorShardMask = + kSharedVectorCursorShards - 1; +static_assert( + (kSharedVectorCursorShards & kSharedVectorCursorShardMask) == 0, + "shared Vector cursor shards must be a power of two" +); +static_assert( + kSharedVectorCursorShards <= kSharedVectorCursorCapacity, + "active shared Vector shards exceed physical capacity" +); +// shared 输出 heap 按 task_id 固定分成 8 个物理 shard。首版只做有界 +// 绝对递增分配，不在 shard 内回绕；该常量同时属于 host 地址 oracle。 +constexpr uint32_t kSharedHeapShards = 8; +constexpr uint32_t kSharedInsertTurnCapacity = 128; +constexpr uint32_t kSharedInsertTurnGroups = + static_cast( + PTO_FDWIC_SHARED_INSERT_TURN_GROUPS + ); +constexpr uint32_t kSharedInsertTurnMask = + kSharedInsertTurnGroups - 1U; +static_assert( + (kSharedInsertTurnGroups & kSharedInsertTurnMask) == 0 && + kSharedInsertTurnGroups <= kSharedInsertTurnCapacity, + "shared insert-turn groups must be a supported power of two" +); +constexpr uint32_t kFinalBarrierMaxLeafGroups = 16; +constexpr uint32_t kFinalBarrierMaxMiddleGroups = 4; +// 每个 worker 私有 ring 有 4 个物理 slot，其中 2 个为 BlockWon 协议预留； +// 单 lane Case1 虽不进入 BlockWon，普通 kernel 仍只能占用剩余 2 个 slot。 +constexpr uint32_t kPrivateSlots = 4; +constexpr uint32_t kWonReserve = 2; +constexpr uint32_t kUsableSlots = kPrivateSlots - kWonReserve; +constexpr uint32_t kMaxFanin = 16; +// H=64 同时约束 heap 可回收 frontier 和 TensorMap producer 的存活下界。 +// heap_next 使用单调逻辑地址；真正落到 256 MiB 环形 heap 时才取模，因而可判断覆盖风险。 +constexpr uint32_t kHeapWindow = 64; +constexpr uint64_t kHeapBytes = 256ULL << 20; +// B512/PA-G1 的逻辑 reservation 为 B256 的两倍，超过默认 256 MiB +// no-wrap heap。standalone 只在 batches>kDefaultBatches 时使用该扩展 +// 逻辑容量；real-compute 仍访问独立 workspace，不分配或解引用这段 +// synthetic heap。 +constexpr uint64_t kExtendedBatchHeapBytes = 512ULL << 20; +constexpr uint64_t kSyntheticHeapBase = 0x100000000ULL; +constexpr uint64_t kOutputAlignment = 1024; +constexpr uint32_t kMaxTensorDims = 5; +constexpr uint32_t kMaxTaskTensors = 32; +constexpr uint32_t kMaxTaskScalars = 16; +constexpr uint32_t kSharedOutputMaxPerTask = 8; +constexpr uint32_t kPayloadSlots = 2048; +constexpr uint32_t kPayloadMask = kPayloadSlots - 1; +constexpr uint32_t kPayloadStride = 4096; +// private/shared 统一为 ring-per-bucket。物理槽总数继续固定为旧 map 的 +// 16K；构建期 CAP 决定每桶连续槽数，桶数由 16K/CAP 推导。正式默认仍是 +// 128×128，隔离门槛另外覆盖 32×512、256×64 与 16384×1 等形态。 +// CAP 变大意味着桶更少、单桶扫描更长；CAP 变小则更易触发显式满环。 +constexpr uint32_t kMapCapacity = 16384; +constexpr uint32_t kMapBucketCapacity = + static_cast(PTO_FDWIC_TENSORMAP_RING_CAP); +constexpr uint32_t kMapBuckets = kMapCapacity / kMapBucketCapacity; + +constexpr uint32_t ConstexprLog2(uint32_t value) { + return value <= 1U ? 0U : 1U + ConstexprLog2(value >> 1U); +} + +constexpr uint32_t kMapBucketShift = ConstexprLog2(kMapBuckets); +constexpr uint32_t kMapBucketMask = kMapBuckets - 1; +constexpr uint32_t kMapBucketSlotMask = kMapBucketCapacity - 1; +constexpr uint32_t kDefaultMapBucketCapacity = 128; +constexpr uint32_t kPaCase1MapEntriesPerBatch = 4; +constexpr uint32_t kPaCase1MaxLiveMapBatches = + (kHeapWindow + 1 + kTasksPerBatch - 1) / kTasksPerBatch; +constexpr uint32_t kPaCase1MaxLiveMapEntries = + kPaCase1MapEntriesPerBatch * kPaCase1MaxLiveMapBatches; +constexpr uint32_t kTaskWindow = 1 << 10; +constexpr uint32_t kTaskWindowMask = kTaskWindow - 1; +constexpr uint64_t kSystemCounterHz = 1000000000ULL; +constexpr uint64_t kWatchdogTicks = 2 * kSystemCounterHz; +// trace_enabled 是位图而不是 bool：bit0 保持既有阶段泳道，bit1 额外开启 +// 逐条 atomic 源码括号记录。atomic 记录依赖同一份 trace buffer，因此 bit1 +// 只能与 bit0 一起配置。 +constexpr uint32_t kTracePhasesEnabled = 1U << 0; +constexpr uint32_t kTraceAtomicsEnabled = 1U << 1; +// Claim trace flags 是独立 raw ABI：bit0 表示获胜，bit1 表示已经通过 +// AIC/AIV role 路由并真正执行 atomicMax。未 attempted 的 Claim 仍保留 +// role-selection 开销，但转换器会明确标成 claim.not_attempted。 +constexpr uint32_t kClaimWon = 1U << 0; +constexpr uint32_t kClaimAttempted = 1U << 1; +// 下列 offset/size 来自真实 DistGlobal/DistCore ABI。standalone 保留被测关键字段的 +// offset、DistCore ABI 和 kRealDistGlobalBytes 总跨度；其余区域可用 opaque padding， +// 并不是对生产结构全部字段的逐一镜像。 +constexpr size_t kRealDistCoreOffset = 10043904; +constexpr size_t kRealDistGlobalBytes = 1007026048; +constexpr size_t kRealFinalBarrierBytes = 2176; +constexpr size_t kRealTasksOffset = 896; +constexpr size_t kRealFatalOffset = 4195264; +constexpr size_t kRealReplayDoneOffset = 10043776; +constexpr size_t kRealStartedCountOffset = 10043840; +// shared 把每个 task/core 固定存在的 Claim+Submit 端点移到 32B 专用区。 +// 通用区容量固定为 28,416 条：默认 32B 格式仍恰好占满 1 MiB/worker； +// CCEC full-swimlane 的 16B 格式只缩短物理 stride，不扩大事件容量。 +constexpr uint32_t kTraceLogicalRecordSizeBytes = 32; +#if PTO_FDWIC_SHARED_MAP +constexpr uint32_t kTraceSubmitClaimRecordSizeBytes = 32; +constexpr uint32_t kTraceRecordsPerCore = 28416; +constexpr uint32_t kTraceRecordSizeBytes = + PA_BUILD_COMPACT_GENERIC_TRACE ? 16U : 32U; +constexpr size_t kTraceWorkerBytes = + static_cast(kMaxTasks) * + kTraceSubmitClaimRecordSizeBytes + + static_cast(kTraceRecordsPerCore) * + kTraceRecordSizeBytes; +static_assert( + kTraceWorkerBytes == + (PA_BUILD_COMPACT_GENERIC_TRACE ? 593920U : (1U << 20)), + "shared trace worker stride changed" +); +#else +constexpr uint32_t kTraceSubmitClaimRecordSizeBytes = 0; +constexpr uint32_t kTraceRecordsPerCore = 1U << 16; +constexpr uint32_t kTraceRecordSizeBytes = 32; +constexpr size_t kTraceWorkerBytes = + static_cast(kTraceRecordsPerCore) * 32U; +#endif +static_assert( + kTraceWorkerBytes % 64U == 0, + "each trace worker partition must remain cache-line aligned" +); +static_assert((kPayloadSlots & kPayloadMask) == 0, "payload slots must be a power of two"); +static_assert( + kMapBucketCapacity >= 32 && kMapBucketCapacity <= kMapCapacity, + "standalone ring CAP must be in [32, 16384]" +); +static_assert( + (kMapBucketCapacity & (kMapBucketCapacity - 1U)) == 0, + "standalone ring CAP must be a power of two" +); +static_assert( + kMapCapacity % kMapBucketCapacity == 0, + "standalone ring CAP must divide the fixed 16K slot pool" +); +static_assert((kMapBuckets & kMapBucketMask) == 0, "map bucket count must be a power of two"); +static_assert( + (kMapBucketCapacity & kMapBucketSlotMask) == 0, + "map bucket capacity must be a power of two" +); +static_assert(kMapCapacity == 16384, "private ring must preserve the old map capacity"); +static_assert( + kPaCase1MaxLiveMapEntries <= kDefaultMapBucketCapacity, + "the default PA Case1 ring capacity no longer covers its conservative live bound" +); +static_assert(kTaskWindow > kHeapWindow, "task counters must retire before their slot is reused"); +static_assert( + kMaxTasks < kTaskCellCapacity, + "task table must cover every PA task and private frontier sentinel" +); + +// These are the measured means from the best PA A5 trace, in 1 GHz ticks. +// The scalar-NOP compatibility baseline calibrates its counts against these targets. +// 无参数默认使用 real-compute：CCEC/AscendC 执行完整 Cube/Vector 流水， +// CPU 执行对等算术；下列 NOP 常量只供显式 scalar-nop 校准。两种模式的 +// Submit、依赖、heap 与 completion 路径都不靠补时修改。target 是真实泳道 +// 均值，不是调度阶段预算。 +constexpr uint32_t kTargetQkTicks = 44170; +constexpr uint32_t kTargetSfTicks = 53729; +constexpr uint32_t kTargetPvTicks = 27626; +constexpr uint32_t kTargetUpTicks = 1565; + +// Calibrated on the local A5 with the CCEC RuntimeNop implementation. These +// counts resolve to the measured targets above; they are not cycle guesses. +constexpr uint32_t kDefaultQkNops = 129600; +constexpr uint32_t kDefaultSfNops = 157900; +constexpr uint32_t kDefaultPvNops = 79950; +constexpr uint32_t kDefaultUpNops = 2400; + +enum class CoreRole : uint32_t { + Aic = 0, + Aiv = 1, +}; + +enum class FinalBarrierShape : uint32_t { + Flat = 0, + TwoLevel4 = 1, + TwoLevel8 = 2, + TwoLevel16 = 3, + ThreeLevel6x4x4 = 4, +}; + +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) +// split caller 与 finish 通过同一份 block-local 状态协作。cookie 只用于 +// 正确性闭环：它同时编码 worker 与核型，防止 AIC/AIV 或相邻 worker 串用状态。 +constexpr uint64_t kCompeteFirstSplitStateCookieBase = 0x434653504c495400ULL; +#endif + +// private 的 task_id % 5 即 kind。shared 单组虽有相同数值布局，也统一 +// 从 ticket 元数据恢复 kind；多组布局为 Alloc + N×(QK/SF/PV/UP)， +// 不能再用 %5 推导。 +enum class TaskKind : uint32_t { + Alloc = 0, + Qk = 1, + Sf = 2, + Pv = 3, + Up = 4, + Count = 5, +}; + +// 记录 kernel 最终在哪次 drain 中落地：Submit 开头、slot/heap 背压期间，或 +// 所有 worker 回放结束后的最终清空。三者之和必须等于实际 kernel 数。 +enum class DrainPlace : uint32_t { + EfDrain = 0, + RingBackpressure = 1, + FinalDrain = 2, + Count = 3, +}; + +enum class TensorArgType : int32_t { + Input = 0, + Output = 1, + Inout = 2, + OutputExisting = 3, + NoDependency = 4, +}; +// Input 作为 kernel 输入并参与依赖、但不登记为写者；Output 由本次 Submit 在 heap 中物化； +// Inout 与 OutputExisting 还需登记进每 worker 私有 TensorMap，供后续重叠区间查询 producer。 +static_assert(sizeof(TensorArgType) == sizeof(int32_t), "TensorArgType must match the PA tag ABI"); + +enum class DataType : uint8_t { + Float32 = 0, + Float16 = 1, + Int32 = 2, + Int16 = 3, + Int8 = 4, + Uint8 = 5, + Bfloat16 = 6, + Int64 = 7, + Uint64 = 8, + Uint16 = 9, + Uint32 = 10, + Bool = 11, + Count = 12, +}; + +// ProfilePhase 是聚合计数下标，TracePhase 是原始泳道事件 ABI；二者故意分离， +// 不能假设枚举值相同。一次 trace 写入可同时归入一个不同命名的 profile 阶段。 +enum class ProfilePhase : uint32_t { + Orchestration = 0, + Submit = 1, + EfDrain = 2, + Materialize = 3, + PrepareMap = 4, + Claim = 5, + Fanin = 6, + Register = 7, + WaitForSlot = 8, + HeapGuard = 9, + Build = 10, + ReplayTail = 11, + Count = 12, +}; + +// submit-pmu 每个 ELF 只编译一个局部归因阶段。none 不做中途 counter +// 读取，是完整 Submit 的正式基线；其余阶段都在每个 worker 的五次 Submit +// 上各执行一次，因此统一按固定 5*batches 次数闭合。历史 ID=3 曾用于 +// winner-only WaitForSlot，现已退役且不复用，避免旧 raw 被误认成新阶段。 +enum class SubmitPmuPhase : uint32_t { + None = 0, + Claim = 1, + EfDrain = 2, + Materialize = 4, + Register = 5, + Count = 6, +}; + +#ifndef PA_SUBMIT_PMU_PHASE_ID +#define PA_SUBMIT_PMU_PHASE_ID 0 +#endif + +constexpr SubmitPmuPhase kCompiledSubmitPmuPhase = + static_cast(PA_SUBMIT_PMU_PHASE_ID); +constexpr uint32_t kBuildVariantSwimlane = 1U; +constexpr uint32_t kBuildVariantSubmitPmu = 2U; +constexpr uint32_t kBuildVariantPerfClock = 3U; +constexpr uint32_t kCompiledBuildVariant = +#if PA_BUILD_SUBMIT_PMU + kBuildVariantSubmitPmu; +#elif PA_BUILD_PERF_CLOCK + kBuildVariantPerfClock; +#else + kBuildVariantSwimlane; +#endif +static_assert( + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::None) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::Claim) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::EfDrain) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::Materialize) || + PA_SUBMIT_PMU_PHASE_ID == static_cast(SubmitPmuPhase::Register), + "invalid compiled submit-pmu phase" +); + +struct NopCounts { + uint32_t qk; + uint32_t sf; + uint32_t pv; + uint32_t up; +}; + +// winner 的计算负载与 NOP 校准量使用两套独立计数，禁止把同一个数字同时解释成 +// scalar 指令条数和 vector/cube 工作迭代数。首阶段只有 CCEC 实现 RealCompute； +// 该 ABI 放在公共模型中，便于后续按相同配置逐步迁移 AscendC 与 CPU。 +struct WorkloadCounts { + uint32_t qk; + uint32_t sf; + uint32_t pv; + uint32_t up; +}; +static_assert(sizeof(WorkloadCounts) == 16, "workload counts ABI changed"); + +enum class WinnerWorkloadMode : uint32_t { + ScalarNop = 0, + RealCompute = 1, +}; + +constexpr uint32_t kWinnerWorkloadConfigVersion = 1; + +// 真实计算工作区是 standalone sidecar，不属于生产 DistGlobal/DistCore ABI。 +// workspace_base 指向 host 单独申请并初始化的 GM；每个 worker 只写自己的输出片段。 +struct alignas(64) WinnerWorkloadConfig { + uint32_t mode; + uint32_t version; + WorkloadCounts repeats; + uint64_t workspace_base; + uint64_t workspace_bytes; + uint32_t reserved[6]; +}; +static_assert(sizeof(WinnerWorkloadConfig) == 64, "winner workload config must occupy one cache line"); +static_assert(offsetof(WinnerWorkloadConfig, mode) == 0, "winner workload mode offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, version) == 4, "winner workload version offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, repeats) == 8, "winner workload counts offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, workspace_base) == 24, "winner workload base offset changed"); +static_assert(offsetof(WinnerWorkloadConfig, workspace_bytes) == 32, "winner workload bytes offset changed"); + +// RunConfig 是 host 在 launch 前写、worker 启动时只读的控制 cache line。 +// 输入为 batch/NOP/诊断开关；输出不回写这里，而发布到独立 WorkerResult。 +struct alignas(64) RunConfig { + uint32_t batches; + uint32_t workers; + NopCounts nops; + uint32_t profile_phases; + uint32_t trace_enabled; + uint64_t trace_base; + uint32_t trace_records_per_core; + uint32_t final_barrier_shape; + // host 与 device 分别按自己的编译常量写入/核对这四个字段。它们占用 + // RunConfig 原有的 16B padding，不扩大热控制行，也不移动生产状态。 + uint32_t build_identity_magic; + uint32_t build_identity_abi_version; + uint32_t tensor_map_mode; + uint32_t scheduler_state_size; +}; +static_assert(sizeof(RunConfig) == 64, "RunConfig must occupy one cache line"); +static_assert(offsetof(RunConfig, build_identity_magic) == 48, "build identity offset changed"); + +// CCEC PMU 的窗口选择与寄存器表是 standalone 诊断 sidecar，不属于 +// RunConfig，也不应占用 winner workload 的字段。独占 cache line 后， +// host/kernel 可显式搬运和失效整个配置，而不会再发生 reserved[4] 越界。 +struct alignas(64) PmuProbeConfig { + uint32_t mode; + uint32_t work_amount; + uint64_t register_table; + uint32_t magic; + // 该 cache line 已随 RunConfig 一起由 host 写入并由 device 失效读取。 + // 复用首个保留槽做构建变体握手，防止绕过 manifest 后把 + // swimlane/submit-pmu/perf-clock 的 host 与 kernel 交叉运行。 + uint32_t build_variant; + uint32_t reserved[10]; +}; +static_assert(sizeof(PmuProbeConfig) == 64, "PMU probe config must occupy one cache line"); +static_assert(offsetof(PmuProbeConfig, register_table) == 8, "PMU register-table offset changed"); +static_assert(offsetof(PmuProbeConfig, magic) == 16, "PMU magic offset changed"); +static_assert(offsetof(PmuProbeConfig, build_variant) == 20, "build variant offset changed"); + +enum class TracePhase : int32_t { + Kernel = 0, + Alloc = 1, + Build = 2, + DrainWon = 3, + Replay = 4, + RingBp = 5, + EfDrain = 6, + Commit = 7, + Submit = 8, + Materialize = 9, + PrepareMap = 10, + Claim = 11, + Fanin = 12, + Register = 13, + Atomic = 14, + // 逐 atomic 诊断构建中，每个 worker 只记录一次连续两次 SYS_CNT 的 + // 空括号，用来给出同一二进制、同一物理核上的计时分辨率下限。 + ClockBaseline = 15, + // schema-v5 的父区间与真实动作区间。loser 没有可单列的真实动作， + // 其时间直接归入离线计算的 Submit residual，不占用 raw 记录。 + OrchestrationReplay = 16, + FinalDrain = 17, + WinnerBuild = 18, + AllocComplete = 19, + // shared Register 父区间内只增加这一条真实 metadata 发布边界。 + // 等待 insert turn 和把 turn 交给 N+1 的两段由父/子端点离线还原， + // 避免为每个 winner 再扩张两条 raw 记录，更不能逐 poll 记录。 + SharedRegisterPublishMetadata = 20, + // Materialize 尾部精确包住 fresh shared-output cell 的预检、writer + // 预留、descriptor flush 与 published 发布。该 cell 按 task_id + // 独占，不进入后续 ordinary/symbol 的全局串行插入区。 + SharedMaterializePublishTaskOutputs = 21, + // PublishTaskOutputs 内再拆两层：先整批 copy descriptor，再整批 + // FlushRegion。两端点仍由正式 Materialize 调用点写 raw，通用 helper + // 只回传时间戳，不自行 WriteTrace。 + SharedMaterializePublishTaskOutputsCopy = 22, + SharedMaterializePublishTaskOutputsFlush = 23, + // 区域级 DCCI 记录是 scalar 调度泳道的 overlay；它不参与 Submit + // 排他分段，也不复用 Atomic 的 flags/auxiliary 编号。 + Dcci = 24, + Count = 25, +}; + +// AtomicSite 按 standalone PA 中真实出现的源码调用点分类。编号写入 TraceRecord::auxiliary， +// 是离线泳道 schema 的一部分；追加新位置只能在 Count 前扩展，不能重排既有值。 +enum class AtomicSite : uint32_t { + StartupIncrement = 0, + StartupPoll = 1, + FatalPoll = 2, + FatalSet = 3, + ClaimMax = 4, + FaninFlagLoad = 5, + CompletionVendExchange = 6, + CompletionFlagExchange = 7, + FrontierInitialLoad = 8, + FrontierFlagLoad = 9, + FrontierMax = 10, + HeapFrontierLoad = 11, + HeapVendLoad = 12, + ReplayDoneIncrement = 13, + ReplayDonePoll = 14, + // shared heap 的预检 load 与两个返回型 FetchAdd 必须分开：前者只读 + // 全局/分片控制字，后者的旧值直接决定本 task 的物理区间。 + SharedHeapVendLoad = 15, + SharedHeapCursorLoad = 16, + SharedHeapCursorReserve = 17, + SharedHeapVendAdvance = 18, + // shared Register 的 insert-turn 等待只按一次 Wait episode 聚合， + // 不为循环内每次 Load 写 raw；handoff CAS 则保留一条 return-ready。 + SharedInsertTurnPoll = 19, + SharedInsertTurnHandoff = 20, + // shared 正式 Submit 中原先绕过 TraceAtomic* 的固定调用点。读取类按 + // 业务位置拆开，避免把 Fanin、metadata 串行区和 output 发布混成一项； + // 编号继续 append-only，旧 raw 的 0..20 语义不变。 + SharedWinnerFatalGuardLoad = 21, + SharedMetadataFatalGuardLoad = 22, + SharedFaninOutputPublishedLoad = 23, + SharedMetadataOutputPublishedLoad = 24, + SharedFaninLastWriterLoad = 25, + SharedMetadataLastWriterLoad = 26, + SharedMetadataLastWriterCommit = 27, + SharedOutputWriterReserve = 28, + SharedOutputPublishedExchange = 29, + SharedMapLookupHeadLoad = 30, + SharedMapLookupTailLoad = 31, + // ordinary-region ring 的 lookup/preflight/append 控制字。lookup 的 + // 两次 seq 双检复用同一站点；append 的 reset/publish/tail 分开， + // 便于直接看出串行 Register 中是哪一步在等待共享 cache line。 + SharedMapLookupSeqLoad = 32, + SharedMapAppendHeadLoad = 33, + SharedMapAppendTailLoad = 34, + SharedMapAppendSeqLoad = 35, + SharedMapAppendSeqResetExchange = 36, + SharedMapAppendSeqPublishExchange = 37, + SharedMapAppendTailExchange = 38, + // 仅在失败回滚中出现，返回旧值不参与协议判断。 + SharedOutputRollbackExchange = 39, + Count = 40, +}; + +// Atomic 记录 flags 的低四位保存操作种类；bit4 表示返回值参与后续判断， +// bit5 表示 Load 观察到零，bit6 表示结束时间已由返回值依赖推进到 +// return-ready 边界。schema-v3 中 bit7 区分等待区 PollBatch：此时 +// bits[31:8] 是精确调用次数；直接 FetchMax 中同一区域仍表示软件重试数。 +enum class AtomicOp : uint32_t { + Load = 0, + Exchange = 1, + FetchAdd = 2, + FetchMax = 3, + CompareExchange = 4, +}; +constexpr uint32_t kAtomicOpMask = 0x0fU; +constexpr uint32_t kAtomicResultUsed = 1U << 4; +constexpr uint32_t kAtomicValueZero = 1U << 5; +constexpr uint32_t kAtomicReturnReady = 1U << 6; +constexpr uint32_t kAtomicPollBatch = 1U << 7; +constexpr uint32_t kAtomicRetriesShift = 8; +constexpr uint32_t kAtomicPollCountShift = 8; +constexpr uint32_t kAtomicPollCountMax = 0x00ffffffU; +constexpr uint32_t kAtomicPollBatchSiteCount = 6; +static_assert(kAtomicPollBatchSiteCount <= 32, "PollBatch enable mask supports at most 32 compact indices"); + +// DCCI 与 Atomic 使用同一个 32B TraceRecord，但拥有完全独立的 raw ABI。 +// 每条记录描述一个区域原语或一组同 op 的聚合区域原语： +// - bits[1:0]：DcciOp； +// - bit2：每个区域原语末尾都执行 trailing DSB； +// - bits[6:3]：logical call_count（1..15）； +// - bit7：保留，必须为 0； +// - bits[31:8]：实际覆盖的 64B cache-line 总数。 +enum class DcciOp : uint32_t { + Invalidate = 0, + CleanOut = 1, + Count = 2, +}; + +enum class DcciSite : uint32_t { + SharedFaninHistoryInvalidate = 0, + SharedWriterHistoryFlush = 1, + SharedOutputRollbackFlush = 2, + SharedOutputDescriptorFlush = 3, + SharedRegionReadInvalidate = 4, + SharedRegionAppendInvalidate = 5, + SharedRegionAppendFlush = 6, + SharedWinnerBuildDescriptorInvalidate = 7, + // observer 自身的 records clean 与 core-state clean 必须聚合为一条 + // terminal row，不能在 FlushRegion 内递归写 trace。 + ObserverTraceExport = 8, + // RunConfig DCCI 发生在 AttachTrace 之前；正常握手成功后用已保存的 + // begin/end 补记这一条。它同时存在于 private/shared 构建。 + StartupConfigInvalidate = 9, + Count = 10, +}; + +constexpr uint32_t kDcciOpMask = 0x03U; +constexpr uint32_t kDcciTrailingDsb = 1U << 2; +constexpr uint32_t kDcciCallCountShift = 3; +constexpr uint32_t kDcciCallCountMask = 0x0fU; +constexpr uint32_t kDcciReservedBit = 1U << 7; +constexpr uint32_t kDcciLineCountShift = 8; +constexpr uint32_t kDcciLineCountMax = 0x00ffffffU; + +// 这些映射是 raw ABI 的一部分，同时被 device 聚合器与 host 闭环校验使用。 +// 0..14 与真实 PA 保持稳定；BlockWon 尚未在 standalone 中实现，不能只为 +// 编号齐全而追加没有真实调用路径的 site。 +#ifdef PA_DEVICE +#define PA_MODEL_INLINE PA_DEVICE +#else +#define PA_MODEL_INLINE inline +#endif + +PA_MODEL_INLINE constexpr AtomicOp AtomicSiteExpectedOp(AtomicSite site) { + switch (site) { + case AtomicSite::StartupIncrement: + case AtomicSite::ReplayDoneIncrement: + case AtomicSite::SharedHeapCursorReserve: + case AtomicSite::SharedHeapVendAdvance: + return AtomicOp::FetchAdd; + case AtomicSite::FatalSet: + case AtomicSite::CompletionVendExchange: + case AtomicSite::CompletionFlagExchange: + return AtomicOp::Exchange; + case AtomicSite::ClaimMax: + case AtomicSite::FrontierMax: + return AtomicOp::FetchMax; + case AtomicSite::SharedInsertTurnHandoff: + case AtomicSite::SharedMetadataLastWriterCommit: + return AtomicOp::CompareExchange; + case AtomicSite::SharedOutputWriterReserve: + return AtomicOp::FetchMax; + case AtomicSite::SharedOutputPublishedExchange: + case AtomicSite::SharedMapAppendSeqResetExchange: + case AtomicSite::SharedMapAppendSeqPublishExchange: + case AtomicSite::SharedMapAppendTailExchange: + case AtomicSite::SharedOutputRollbackExchange: + return AtomicOp::Exchange; + default: + return AtomicOp::Load; + } +} + +PA_MODEL_INLINE constexpr bool AtomicSiteResultUsed(AtomicSite site) { + switch (site) { + case AtomicSite::StartupIncrement: + case AtomicSite::FatalSet: + case AtomicSite::CompletionVendExchange: + case AtomicSite::CompletionFlagExchange: + case AtomicSite::ReplayDoneIncrement: + case AtomicSite::SharedOutputRollbackExchange: + return false; + case AtomicSite::StartupPoll: + case AtomicSite::FatalPoll: + case AtomicSite::ClaimMax: + case AtomicSite::FaninFlagLoad: + case AtomicSite::FrontierInitialLoad: + case AtomicSite::FrontierFlagLoad: + case AtomicSite::FrontierMax: + case AtomicSite::HeapFrontierLoad: + case AtomicSite::HeapVendLoad: + case AtomicSite::ReplayDonePoll: + case AtomicSite::SharedHeapVendLoad: + case AtomicSite::SharedHeapCursorLoad: + case AtomicSite::SharedHeapCursorReserve: + case AtomicSite::SharedHeapVendAdvance: + case AtomicSite::SharedInsertTurnPoll: + case AtomicSite::SharedInsertTurnHandoff: + case AtomicSite::SharedWinnerFatalGuardLoad: + case AtomicSite::SharedMetadataFatalGuardLoad: + case AtomicSite::SharedFaninOutputPublishedLoad: + case AtomicSite::SharedMetadataOutputPublishedLoad: + case AtomicSite::SharedFaninLastWriterLoad: + case AtomicSite::SharedMetadataLastWriterLoad: + case AtomicSite::SharedMetadataLastWriterCommit: + case AtomicSite::SharedOutputWriterReserve: + case AtomicSite::SharedOutputPublishedExchange: + case AtomicSite::SharedMapLookupHeadLoad: + case AtomicSite::SharedMapLookupTailLoad: + case AtomicSite::SharedMapLookupSeqLoad: + case AtomicSite::SharedMapAppendHeadLoad: + case AtomicSite::SharedMapAppendTailLoad: + case AtomicSite::SharedMapAppendSeqLoad: + case AtomicSite::SharedMapAppendSeqResetExchange: + case AtomicSite::SharedMapAppendSeqPublishExchange: + case AtomicSite::SharedMapAppendTailExchange: + return true; + case AtomicSite::Count: + return false; + } + return false; +} + +PA_MODEL_INLINE constexpr int32_t AtomicPollBatchIndex(AtomicSite site) { + switch (site) { + case AtomicSite::StartupPoll: + return 0; + case AtomicSite::FatalPoll: + return 1; + case AtomicSite::FaninFlagLoad: + return 2; + case AtomicSite::HeapFrontierLoad: + return 3; + case AtomicSite::HeapVendLoad: + return 4; + case AtomicSite::ReplayDonePoll: + return 5; + default: + return -1; + } +} + +PA_MODEL_INLINE constexpr AtomicSite AtomicPollBatchSite(uint32_t index) { + switch (index) { + case 0: + return AtomicSite::StartupPoll; + case 1: + return AtomicSite::FatalPoll; + case 2: + return AtomicSite::FaninFlagLoad; + case 3: + return AtomicSite::HeapFrontierLoad; + case 4: + return AtomicSite::HeapVendLoad; + case 5: + return AtomicSite::ReplayDonePoll; + default: + return AtomicSite::Count; + } +} + +PA_MODEL_INLINE constexpr bool AtomicSiteIsPollBatchable(AtomicSite site) { + // insert-turn 使用 Wait 已有 polls 一次性写聚合记录，不占用 + // AtomicPollBurst compact slot，也绝不进入逐调用 TraceAtomicLoad。 + return AtomicPollBatchIndex(site) >= 0 || + site == AtomicSite::SharedInsertTurnPoll; +} + +PA_MODEL_INLINE constexpr bool AtomicSiteIsSharedOnly(AtomicSite site) { + return static_cast(site) >= + static_cast(AtomicSite::SharedHeapVendLoad) && + static_cast(site) < + static_cast(AtomicSite::Count); +} + +PA_MODEL_INLINE constexpr uint32_t AtomicPollBatchMask(AtomicSite site) { + const int32_t index = AtomicPollBatchIndex(site); + return index >= 0 && index < 32 ? 1U << static_cast(index) : 0U; +} + +PA_MODEL_INLINE constexpr DcciOp DcciSiteExpectedOp(DcciSite site) { + switch (site) { + case DcciSite::SharedWriterHistoryFlush: + case DcciSite::SharedOutputRollbackFlush: + case DcciSite::SharedOutputDescriptorFlush: + case DcciSite::SharedRegionAppendFlush: + case DcciSite::ObserverTraceExport: + return DcciOp::CleanOut; + default: + return DcciOp::Invalidate; + } +} + +PA_MODEL_INLINE constexpr bool DcciSiteIsSharedOnly(DcciSite site) { + return static_cast(site) < + static_cast(DcciSite::ObserverTraceExport); +} + +static_assert( + AtomicSiteIsSharedOnly(AtomicSite::SharedInsertTurnPoll) && + AtomicSiteIsSharedOnly( + AtomicSite::SharedInsertTurnHandoff + ), + "insert-turn atomic sites must remain shared-only" +); +static_assert( + !DcciSiteIsSharedOnly(DcciSite::ObserverTraceExport), + "observer trace export must remain available in both TensorMap modes" +); +static_assert( + !DcciSiteIsSharedOnly(DcciSite::StartupConfigInvalidate), + "startup config invalidate must remain available in both TensorMap modes" +); + +#undef PA_MODEL_INLINE + +// ClockBaseline 的 bit0 区分普通连续 SYS_CNT 与后端的 atomic 返回依赖 +// 计时钩子；后者用于量化那一条依赖 MOV 自身带来的固定底噪。 +constexpr uint32_t kClockAtomicDependency = 1U << 0; +constexpr uint32_t kClockAtomicDependencyApplied = 1U << 1; + +struct alignas(64) TraceCoreState { + volatile uint32_t count; + volatile uint32_t dropped; + // logical atomic 调用数与物理记录数分开闭合：PollBatch 的一条记录可以 + // 表示多次只读轮询，physical = atomic_calls - poll_calls + batch_records。 + volatile uint32_t atomic_calls; + volatile uint32_t poll_calls; + volatile uint32_t poll_batch_records; + // 拓扑在一个 worker 分区内恒定；只在 core state 保存一份权威身份， + // 32B TraceRecord 不再为每条事件重复写入这 12B。 + volatile int32_t core_idx; + volatile int32_t block_id; + volatile int32_t lane; + // 复用 core-state 既有 32B 尾部，不扩大 header。calls/lines 是 + // logical 总量，records 是实际落盘的 Dcci row 数。 + volatile uint32_t dcci_calls; + volatile uint32_t dcci_lines; + volatile uint32_t dcci_records; + uint32_t padding[5]; +}; +// 每个 worker 独占一个计数 cache line 和一段定长 records，不需要为了写 trace +// 再引入跨核 atomic；满容量后只增加本 worker 的 dropped。 +static_assert(sizeof(TraceCoreState) == 64, "trace core state must occupy one cache line"); +static_assert(offsetof(TraceCoreState, dcci_calls) == 32, "DCCI counters must reuse the core-state tail"); +static_assert(offsetof(TraceCoreState, dcci_records) == 40, "DCCI counter layout changed"); + +struct alignas(64) TraceHeader { + uint32_t magic; + uint32_t version; + uint32_t num_cores; + uint32_t records_per_core; + uint64_t frequency_hz; + // 占用首条 header cache line 的既有 padding，不移动 cores。host/device + // 都据此拒绝把旧 64B record 缓冲误解为当前构建的物理 generic ABI。 + uint32_t record_size_bytes; + TraceCoreState cores[kRuntimeMaxWorkers]; +}; +// 本 benchmark 固定物理分配 kWorkers=96 个定长 record 分区，合法 header 也要求 +// num_cores==96；其 header/record 布局和 phase 编号可转换为真实泳道使用的 JSON。 +static_assert(offsetof(TraceHeader, record_size_bytes) == 24, "trace record-size offset changed"); +static_assert(offsetof(TraceHeader, cores) == 64, "trace core states must start at the second cache line"); +static_assert(sizeof(TraceHeader) == 6976, "trace header must match PA swimlane layout"); + +struct alignas(32) TraceRecord { + uint64_t start_cycle; + uint64_t end_cycle; + int32_t task_id; + int32_t function_id; + uint32_t flags; + uint16_t phase; + uint16_t auxiliary; +}; +// 与真实 FDWIC 的生产 record 同形：每条只保留事件自身字段，物理 +// core/block/lane 由所属分区的 TraceCoreState 在 host 导出时回填。 +static_assert(offsetof(TraceRecord, flags) == 24, "trace flags offset changed"); +static_assert(offsetof(TraceRecord, phase) == 28, "trace phase offset changed"); +static_assert(offsetof(TraceRecord, auxiliary) == 30, "trace auxiliary offset changed"); +static_assert( + sizeof(TraceRecord) == kTraceLogicalRecordSizeBytes, + "logical trace record must occupy half a cache line" +); +static_assert(alignof(TraceRecord) == 32, "trace record alignment changed"); +static_assert( + static_cast(TracePhase::Count) <= UINT16_MAX, + "trace phase does not fit the 16-bit raw ABI" +); +static_assert( + static_cast(AtomicSite::Count) <= UINT16_MAX, + "atomic site does not fit the 16-bit raw ABI" +); +static_assert( + static_cast(DcciSite::Count) <= UINT16_MAX, + "DCCI site does not fit the 16-bit raw ABI" +); + +// shared CCEC full-swimlane 的通用物理记录只保留低 32-bit 时钟与一个 +// 紧凑业务字；host 回读后恢复成上面的 32B 逻辑 TraceRecord。flags +// 完整保留 32 bit，Atomic/DCCI 的既有 raw ABI 不发生裁剪。 +struct alignas(16) CompactTraceRecord16 { + uint32_t start_cycle_low; + uint32_t end_cycle_low; + uint32_t flags; + uint32_t packed; +}; +static_assert( + sizeof(CompactTraceRecord16) == 16 && + alignof(CompactTraceRecord16) == 16, + "compact generic trace record must occupy 16 bytes" +); +static_assert( + offsetof(CompactTraceRecord16, start_cycle_low) == 0 && + offsetof(CompactTraceRecord16, end_cycle_low) == 4 && + offsetof(CompactTraceRecord16, flags) == 8 && + offsetof(CompactTraceRecord16, packed) == 12, + "compact generic trace offsets changed" +); + +constexpr uint32_t kCompactTraceTaskBits = 13; +constexpr uint32_t kCompactTraceTaskMask = + (1U << kCompactTraceTaskBits) - 1U; +constexpr uint32_t kCompactTraceTaskSentinel = + kCompactTraceTaskMask; +constexpr uint32_t kCompactTraceFunctionShift = 13; +constexpr uint32_t kCompactTraceFunctionBits = 3; +constexpr uint32_t kCompactTraceFunctionMask = + (1U << kCompactTraceFunctionBits) - 1U; +constexpr uint32_t kCompactTraceFunctionSentinel = + kCompactTraceFunctionMask; +constexpr uint32_t kCompactTracePhaseShift = 16; +constexpr uint32_t kCompactTracePhaseBits = 5; +constexpr uint32_t kCompactTracePhaseMask = + (1U << kCompactTracePhaseBits) - 1U; +constexpr uint32_t kCompactTraceAuxiliaryShift = 21; +constexpr uint32_t kCompactTraceAuxiliaryBits = 11; +constexpr uint32_t kCompactTraceAuxiliaryMask = + (1U << kCompactTraceAuxiliaryBits) - 1U; + +static_assert( + kMaxTasks <= kCompactTraceTaskSentinel, + "compact trace task field cannot encode every PA task" +); +static_assert( + static_cast(TracePhase::Count) <= + kCompactTracePhaseMask + 1U, + "compact trace phase field is too narrow" +); +static_assert( + static_cast(AtomicSite::Count) <= + kCompactTraceAuxiliaryMask + 1U && + static_cast(DcciSite::Count) <= + kCompactTraceAuxiliaryMask + 1U && + kMaxTaskTensors <= kCompactTraceAuxiliaryMask && + kMaxFanin <= kCompactTraceAuxiliaryMask, + "compact trace auxiliary field is too narrow" +); + +#ifdef PA_DEVICE +#define PA_TRACE_ABI_INLINE PA_DEVICE +#else +#define PA_TRACE_ABI_INLINE inline +#endif + +PA_TRACE_ABI_INLINE bool CompactTraceFieldsFit( + int32_t task_id, int32_t function_id, + TracePhase phase, uint32_t auxiliary +) { + return task_id >= -1 && + task_id < static_cast(kMaxTasks) && + function_id >= -1 && function_id <= 3 && + static_cast(phase) < + static_cast(TracePhase::Count) && + auxiliary <= kCompactTraceAuxiliaryMask; +} + +PA_TRACE_ABI_INLINE uint32_t PackCompactTraceFields( + int32_t task_id, int32_t function_id, + TracePhase phase, uint32_t auxiliary +) { + return + (static_cast(task_id) & + kCompactTraceTaskMask) | + ((static_cast(function_id) & + kCompactTraceFunctionMask) + << kCompactTraceFunctionShift) | + (static_cast(phase) << + kCompactTracePhaseShift) | + (auxiliary << kCompactTraceAuxiliaryShift); +} + +#undef PA_TRACE_ABI_INLINE + +#if PA_BUILD_COMPACT_GENERIC_TRACE +using TraceStorageRecord = CompactTraceRecord16; +#else +using TraceStorageRecord = TraceRecord; +#endif +static_assert( + sizeof(TraceStorageRecord) == kTraceRecordSizeBytes, + "generic trace storage size disagrees with the build identity" +); + +struct alignas(32) SharedSubmitClaimTraceRecord { + uint64_t claim_begin; + // SYS_CNT 在 watchdog 窗口内不会触及 bit63；复用该位保存 winner， + // 不为逐 task 固定存在的布尔值扩张记录。 + uint64_t claim_end_and_winner; + uint64_t submit_begin; + uint64_t submit_end; +}; +constexpr uint64_t kSharedClaimWinnerBit = 1ULL << 63; +static_assert( + sizeof(SharedSubmitClaimTraceRecord) == 32 && + alignof(SharedSubmitClaimTraceRecord) == 32, + "shared Submit/Claim record must remain 32 bytes" +); +static_assert( + offsetof(SharedSubmitClaimTraceRecord, claim_begin) == 0 && + offsetof(SharedSubmitClaimTraceRecord, claim_end_and_winner) == 8 && + offsetof(SharedSubmitClaimTraceRecord, submit_begin) == 16 && + offsetof(SharedSubmitClaimTraceRecord, submit_end) == 24, + "shared compact Submit/Claim offsets changed" +); +#if PTO_FDWIC_SHARED_MAP +static_assert( + kTraceSubmitClaimRecordSizeBytes == + sizeof(SharedSubmitClaimTraceRecord), + "shared Submit/Claim layout constant changed" +); +#endif + +constexpr size_t kTraceSubmitClaimBytesPerCore = + static_cast(kMaxTasks) * + kTraceSubmitClaimRecordSizeBytes; +constexpr size_t kTraceGenericBytesPerCore = + static_cast(kTraceRecordsPerCore) * + sizeof(TraceStorageRecord); +static_assert( + kTraceSubmitClaimBytesPerCore + + kTraceGenericBytesPerCore == + kTraceWorkerBytes, + "per-worker trace regions must exactly fill their partition" +); +constexpr size_t TraceWorkerOffset(uint32_t worker) { + return sizeof(TraceHeader) + + static_cast(worker) * kTraceWorkerBytes; +} +constexpr size_t TraceSubmitClaimOffset(uint32_t worker) { + return TraceWorkerOffset(worker); +} +constexpr size_t TraceRecordsOffset(uint32_t worker) { + return TraceWorkerOffset(worker) + + kTraceSubmitClaimBytesPerCore; +} + +constexpr size_t kTraceBytes = + sizeof(TraceHeader) + + static_cast(kWorkers) * kTraceWorkerBytes; + +struct alignas(64) AtomicLine { + volatile int64_t value; + uint8_t padding[64 - sizeof(int64_t)]; +}; +// 热点共享量各占一个 cache line，保持生产代码的地址隔离，避免 standalone +// 因伪共享额外放大 Claim/frontier/start barrier 的竞争。 +static_assert(sizeof(AtomicLine) == 64, "AtomicLine must occupy one cache line"); + +// final 分层汇合把 arrival 与 release 分到不同 cache line：等待 release +// 的 add-zero 不会反向堵塞尚未到达的 worker。最多 16 个叶组、4 个中间组； +// 未被当前形态使用的节点必须保持零并由 host 校验。startup 仍使用生产 flat 屏障。 +struct alignas(64) FinalBarrierState { + AtomicLine leaf_arrivals[kFinalBarrierMaxLeafGroups]; + AtomicLine leaf_releases[kFinalBarrierMaxLeafGroups]; + AtomicLine middle_arrivals[kFinalBarrierMaxMiddleGroups]; + AtomicLine middle_releases[kFinalBarrierMaxMiddleGroups]; + AtomicLine root_arrival; + AtomicLine root_release; +}; +static_assert(sizeof(FinalBarrierState) == 2688, "final barrier state size changed"); + +struct alignas(64) AtomicFlagLine { + volatile int32_t value; + uint8_t padding[64 - sizeof(int32_t)]; +}; +// 32-bit fatal 与 64-bit cursor 使用不同封装，但都独占 cache line；成功路径中 +// fatal 始终为零，任何写一都表示协议已终止，不能作为普通等待条件清除。 +static_assert(sizeof(AtomicFlagLine) == 64, "AtomicFlagLine must occupy one cache line"); + +struct alignas(64) TaskCell { + volatile int64_t flag; + volatile uint64_t vend; +#if PTO_FDWIC_SHARED_MAP + // shared 热路径把该字作为 per-task TensorMap 插入完成原子：初值 + // -1，task N 的唯一 Claim owner 完成 writer 元数据发布后用 CAS + // 写成 N；N+1 owner 只轮询这一字。它与 flag/vend 共处 TaskCell， + // 但当前热路径不对该 cache line 执行 DCCI。旧 writer-ready helper + // 只供隔离协议测试，不能与本热路径混用。 + volatile int64_t deps_prepared; + uint8_t padding[64 - 3 * sizeof(int64_t)]; +#else + uint8_t padding[64 - 2 * sizeof(int64_t)]; +#endif +}; +// flag 是两种模式的依赖就绪发布位；vend 是该 task 完成时 worker 的 heap +// 快照。private 还用 flag 连续推进 frontier，并由 HeapGuard 读取 +// frontier-H 对应 vend 判断环形 heap 是否可覆盖；shared no-wrap 中 vend +// 只是 aggregate-vend 快照，flag 只服务 fanin/slot，均不参与 heap 回收。 +static_assert(sizeof(TaskCell) == 64, "TaskCell must occupy one cache line"); +#if PTO_FDWIC_SHARED_MAP +static_assert( + offsetof(TaskCell, deps_prepared) == 16, + "shared task dependency-intent offset mismatch" +); +#endif + +// TensorDesc 保留真实 Tensor 的两条 64-byte 数据线。owner_task_id 表达显式生产者， +// buffer_addr + 字节区间用于 TensorMap 发现同一 backing buffer 上的读写依赖。 +struct TensorDesc { + uint64_t buffer_addr; + uint64_t buffer_size; + uint64_t owner_task_id; + uint64_t start_offset; + int32_t version; + uint32_t ndims; + DataType dtype; + bool manual_dep; + bool is_contiguous; + uint8_t child_memory; + uint32_t shapes[kMaxTensorDims]; + + uint64_t extent_elem_cache; + uint32_t strides[kMaxTensorDims]; + uint8_t padding[36]; +}; +static_assert(sizeof(TensorDesc) == 128, "TensorDesc must match the PA Tensor ABI size"); +static_assert(offsetof(TensorDesc, buffer_addr) == 0, "TensorDesc buffer offset mismatch"); +static_assert(offsetof(TensorDesc, owner_task_id) == 16, "TensorDesc owner offset mismatch"); +static_assert(offsetof(TensorDesc, start_offset) == 24, "TensorDesc view offset mismatch"); +static_assert(offsetof(TensorDesc, version) == 32, "TensorDesc version offset mismatch"); +static_assert(offsetof(TensorDesc, shapes) == 44, "TensorDesc shape offset mismatch"); +static_assert(offsetof(TensorDesc, extent_elem_cache) == 64, "TensorDesc extent offset mismatch"); +static_assert(offsetof(TensorDesc, strides) == 72, "TensorDesc stride offset mismatch"); + +struct TensorCreateInfo { + uint64_t initial_value; + bool has_initial_value; + uint8_t padding0[7]; + uint64_t reserved0; + uint64_t start_offset; + int32_t version; + uint32_t ndims; + DataType dtype; + bool manual_dep; + bool is_contiguous; + uint8_t child_memory; + uint32_t shapes[kMaxTensorDims]; +}; +// CreateInfo 只描述尚未分配的 Output；Materialize 根据形状和 dtype 计算大小， +// 再把它变成位于 worker 逻辑 heap 上的 TensorDesc。 +static_assert(sizeof(TensorCreateInfo) == 64, "TensorCreateInfo must match the PA create-info ABI size"); +static_assert(offsetof(TensorCreateInfo, start_offset) == 24, "TensorCreateInfo start offset mismatch"); +static_assert(offsetof(TensorCreateInfo, version) == 32, "TensorCreateInfo version offset mismatch"); +static_assert(offsetof(TensorCreateInfo, shapes) == 44, "TensorCreateInfo shape offset mismatch"); + +struct MapEntry { + uint64_t buffer_addr; + uint64_t lo; + uint64_t hi; + int32_t producer; + uint32_t payload_padding; + // 末 16B 只为保持 standalone/真实 PA 的既有 48B entry ABI。本阶段 + // private ring 不需要 seq，也不能提前把 shared 发布协议塞进保留区。 + uint8_t abi_reserved[16]; +}; +// bucket 与槽下标都由外层 ring 的连续布局隐式给出，不再保存 next/prev 指针。 +static_assert(sizeof(MapEntry) == 48, "MapEntry must match the PA tensor-map entry ABI"); +static_assert(alignof(MapEntry) == 8, "MapEntry alignment changed"); +static_assert(offsetof(MapEntry, producer) == 24, "MapEntry producer offset mismatch"); +static_assert(offsetof(MapEntry, abi_reserved) == 32, "MapEntry ABI reserve offset mismatch"); + +struct TensorMap { + MapEntry entries[kMapCapacity]; + // 前 128 个桶沿用默认 128×128 的原始位置。CAP=32/64 时桶数增至 + // 512/256，额外游标从原 32 KiB ABI 保留区中切出；这样默认热字段 + // offset 不动，所有 CAP 下方的 task 计数与 WorkerState 总跨度也不动。 + uint64_t bucket_heads[128]; + uint64_t bucket_tails[128]; +#if PTO_FDWIC_TENSORMAP_RING_CAP == 32 + uint64_t extra_bucket_heads[384]; + uint64_t extra_bucket_tails[384]; + uint8_t abi_reserved[24576]; +#elif PTO_FDWIC_TENSORMAP_RING_CAP == 64 + uint64_t extra_bucket_heads[128]; + uint64_t extra_bucket_tails[128]; + uint8_t abi_reserved[28672]; +#else + // CAP>=128 时逻辑桶数不超过 128，完整保留原来的 30 KiB padding。 + uint8_t abi_reserved[30720]; +#endif + // producer 退休时据此精确扣减 logical live_count；物理 bucket head + // 则由访问该桶时的 RetireBucket 惰性推进。 + uint32_t task_entry_counts[kTaskWindow]; + uint32_t live_count; + uint32_t high_water; + int32_t alive_floor; + int32_t cleaned_upto; +}; +// alive_floor 是 lookup 的权威存活下界；cleaned_upto 表示逐任务计数已 +// 精确扣减到哪里。桶头可以因惰性退休暂时落后，但不会改变逻辑 live 数。 +static_assert(sizeof(TensorMap) == 823312, "TensorMap must preserve the WorkerState ABI"); +static_assert(alignof(TensorMap) == 8, "TensorMap alignment changed"); +#if PTO_FDWIC_TENSORMAP_RING_CAP == 128 +static_assert(offsetof(TensorMap, bucket_heads) == 786432, "TensorMap head offset mismatch"); +static_assert(offsetof(TensorMap, bucket_tails) == 787456, "TensorMap tail offset mismatch"); +static_assert(offsetof(TensorMap, abi_reserved) == 788480, "TensorMap reserve offset mismatch"); +static_assert(offsetof(TensorMap, task_entry_counts) == 819200, "TensorMap task-count offset mismatch"); +static_assert(offsetof(TensorMap, live_count) == 823296, "TensorMap live-count offset mismatch"); +static_assert(offsetof(TensorMap, high_water) == 823300, "TensorMap high-water offset mismatch"); +static_assert(offsetof(TensorMap, alive_floor) == 823304, "TensorMap alive-floor offset mismatch"); +static_assert(offsetof(TensorMap, cleaned_upto) == 823308, "TensorMap cleaned offset mismatch"); +#elif PTO_FDWIC_TENSORMAP_RING_CAP == 64 +static_assert(offsetof(TensorMap, extra_bucket_heads) == 788480, "CAP64 extra-head offset mismatch"); +static_assert(offsetof(TensorMap, extra_bucket_tails) == 789504, "CAP64 extra-tail offset mismatch"); +static_assert(offsetof(TensorMap, abi_reserved) == 790528, "CAP64 reserve offset mismatch"); +static_assert(offsetof(TensorMap, task_entry_counts) == 819200, "CAP64 task-count offset mismatch"); +#elif PTO_FDWIC_TENSORMAP_RING_CAP == 32 +static_assert(offsetof(TensorMap, extra_bucket_heads) == 788480, "CAP32 extra-head offset mismatch"); +static_assert(offsetof(TensorMap, extra_bucket_tails) == 791552, "CAP32 extra-tail offset mismatch"); +static_assert(offsetof(TensorMap, abi_reserved) == 794624, "CAP32 reserve offset mismatch"); +static_assert(offsetof(TensorMap, task_entry_counts) == 819200, "CAP32 task-count offset mismatch"); +#endif + +// shared 模式只共享 region→producer 索引，不复用 private MapEntry 尾部的 +// ABI 保留字节。payload 与 seq 各占一条 cache line：普通字段写回完成后， +// 再用独立的绝对 seq 发布该 lap，reader 以 seq 双检防止槽复用 ABA。 +struct SharedRegionValue { + uint64_t buffer_addr; + uint64_t lo; + uint64_t hi; + int32_t producer; + uint32_t reserved; +}; +static_assert(sizeof(SharedRegionValue) == 32, "shared TensorMap logical value size changed"); +static_assert(offsetof(SharedRegionValue, producer) == 24, "shared producer offset mismatch"); + +struct alignas(64) SharedRegionPayload { + SharedRegionValue value; + uint8_t cacheline_padding[32]; +}; +static_assert(sizeof(SharedRegionPayload) == 64, "shared TensorMap payload must occupy one cache line"); +static_assert(alignof(SharedRegionPayload) == 64, "shared TensorMap payload alignment changed"); + +struct alignas(64) SharedRegionSlot { + SharedRegionPayload payload; + AtomicLine seq; +}; +static_assert(sizeof(SharedRegionSlot) == 128, "shared TensorMap slot must occupy two cache lines"); +static_assert(offsetof(SharedRegionSlot, seq) == 64, "shared TensorMap seq must own the second cache line"); + +struct alignas(64) SharedBucketState { + AtomicLine head; + AtomicLine tail; +}; +static_assert(sizeof(SharedBucketState) == 128, "shared TensorMap bucket controls changed"); +static_assert(offsetof(SharedBucketState, tail) == 64, "shared head/tail must not share a cache line"); + +#if PTO_FDWIC_SHARED_MAP +// fresh Output 不再借助 region map 按地址查找，而是由 +// (producer_task_id, output_slot) 直接定位。descriptor 发布位、writer 链 +// 和不可变 descriptor 分属独立 cache line 区域，避免三种访问彼此伪共享。 +// shared 最大覆盖 256 batch × 4 block group；本轮 task_id 不复用， +// 因此外层表直接寻址，不做取模，也不在这里提前引入 generation。 +struct alignas(64) SharedOutputCell { + AtomicLine published[kSharedOutputMaxPerTask]; + AtomicLine last_writer[kSharedOutputMaxPerTask]; + TensorDesc tensors[kSharedOutputMaxPerTask]; +}; +static_assert(sizeof(SharedOutputCell) == 2048, "shared output cell size changed"); +static_assert(alignof(SharedOutputCell) == 64, "shared output cell alignment changed"); +static_assert(offsetof(SharedOutputCell, published) == 0, "shared output publish offset mismatch"); +static_assert(offsetof(SharedOutputCell, last_writer) == 512, "shared output writer offset mismatch"); +static_assert(offsetof(SharedOutputCell, tensors) == 1024, "shared output tensor offset mismatch"); + +// latest writer 只是正常顺序 reader 的快取；慢 reader 若在查询前遇到 +// future writer，必须沿不可变前驱链回到严格早于自己的版本。每个 writer +// task 独占一个 history cell，record 的 writer id 由 cell 下标隐含； +// packed key 无哈希碰撞，可还原为 fresh descriptor 的 (producer, slot)。 +constexpr uint32_t kSharedWriterHistoryMaxPerTask = kMaxTaskTensors; +constexpr uint32_t kSharedWriterHistoryMagic = 0x57484953U; // "WHIS" +struct SharedWriterHistoryRecord { + uint32_t symbol_key; + int32_t previous_writer; +}; +static_assert(sizeof(SharedWriterHistoryRecord) == 8, "shared writer-history record size changed"); +static_assert(alignof(SharedWriterHistoryRecord) == 4, "shared writer-history record alignment changed"); + +struct alignas(64) SharedWriterHistoryCell { + // header 与常见的三个 PA writer record 共处首条 cache line；唯一 + // winner 一次写回这段不可变 payload，随后各 symbol 的 last_writer + // CAS 才是 reader 的发布边界，不额外增加 history atomic。 + uint32_t magic; + int32_t writer_task; + uint32_t count; + uint32_t reserved; + SharedWriterHistoryRecord entries[kSharedWriterHistoryMaxPerTask]; + uint8_t padding[48]; +}; +static_assert(sizeof(SharedWriterHistoryCell) == 320, "shared writer-history cell size changed"); +static_assert(alignof(SharedWriterHistoryCell) == 64, "shared writer-history cell alignment changed"); +static_assert( + offsetof(SharedWriterHistoryCell, entries) == 16, + "shared writer-history entries must follow their immutable header" +); +static_assert( + kSharedOutputMaxPerTask <= 8 && + kMaxTasks <= UINT32_MAX / kSharedOutputMaxPerTask, + "packed shared symbol key no longer fits uint32" +); +#endif + +struct alignas(64) SharedTensorMapSidecar { + // lane 0 保留既有 committed_tasks 地址：G=1 时它仍表示下一个允许 + // 插入 writer 元数据的 task id。G>1 时它只是交错 token 的 lane 0； + // 其余七条物理线追加在 sidecar 尾部，所有既有热点字段 offset 不动。 + AtomicLine committed_tasks; + // reclaim_upto 是可回收 producer 的 inclusive 上界，初始 -1。 + // insert-before-lookup 基线固定不回收 ordinary ring，因此保持 -1； + // reader-progress/reclaim 只由隔离测试覆盖，尚未接回当前热路径。 + AtomicLine reclaim_upto; + SharedBucketState buckets[kMapBuckets]; + SharedRegionSlot slots[kMapCapacity]; +#if PTO_FDWIC_SHARED_MAP + // 追加在既有 S2.5 region ring 之后，保持 committed/reclaim、bucket 和 + // slot 的全部 offset 不变。容量按 shared 最坏 17 task/batch 分配； + // 现有 shared sidecar H2D/D2H 按 sizeof 搬运。 + SharedOutputCell shared_outputs[kMaxTasks]; + // shared heap 控制字继续追加在 S3.1 output table 之后。每个 shard cursor + // 与全局 aggregate vend 独占 cache line，避免不同 winner 的原子更新伪共享。 + AtomicLine shared_heap_cursor[kSharedHeapShards]; + AtomicLine shared_heap_vend; + // S4.14a 的 shared-only Vector Claim cursor 追加在既有 sidecar 尾部； + // S4.14b 只启用此前已经预留的后四条物理线。production prefix 和 + // 已验证的 region/output/heap 字段仍不移动，且不宣称该地址与参考 + // DistGlobal 具有相同字节 offset。 + AtomicLine shared_vector_cursor[kSharedVectorCursorCapacity]; + // 追加在全部既有字段之后，避免为通用 writer history 移动 PA 已测 + // 热点控制字。当前 task id 在一轮内不复用，因此 history 不取模； + // 1.33 MiB 增量只影响启动期整块搬运，不改变 Submit 内旧字段地址。 + SharedWriterHistoryCell writer_history[kMaxTasks]; + // ordinary reader 的完成前沿不能复用 WorkerState::local_index：后者在 + // 读取前就会推进，且 96 个字段分散在近 1 GiB 的 per-worker arena。 + // 每个 worker 独占一条连续 cache line；初值 -1，值 N 只表示该 worker + // 已关闭 task [0,N] 的全部 ordinary-ring 读取。R4e-a 只建立状态与 + // 纯公式门槛，尚不从 PA 或通用 Submit 热路径发布该字段。 + AtomicLine reader_done[kWorkers]; + // insert_turn_extra[0..126] 对应逻辑 lane 1..127。初始值全部为 -1； + // active G 只决定前 G 条逻辑 lane 的寻址，inactive 物理线始终保持 + // -1。每个 owner（包括空写集合）发布完整元数据后，只把 baton 从 + // task N 轮换为 N+1；fanin、Build 与执行不属于这条顺序链。 + AtomicLine insert_turn_extra[ + kSharedInsertTurnCapacity - 1 + ]; +#endif +}; +#if PTO_FDWIC_SHARED_MAP +static_assert( + offsetof(SharedTensorMapSidecar, reader_done) == + offsetof(SharedTensorMapSidecar, writer_history) + + sizeof(SharedWriterHistoryCell) * kMaxTasks, + "shared reader progress must immediately follow writer history" +); +static_assert( + sizeof(SharedTensorMapSidecar) == + offsetof(SharedTensorMapSidecar, reader_done) + + sizeof(AtomicLine) * kWorkers + + sizeof(AtomicLine) * + (kSharedInsertTurnCapacity - 1), + "shared insert-turn lines must remain the sidecar tail" +); +static_assert( + offsetof(SharedTensorMapSidecar, insert_turn_extra) == + offsetof(SharedTensorMapSidecar, reader_done) + + sizeof(AtomicLine) * kWorkers, + "extra shared insert-turn lines must follow reader progress" +); +#if PTO_FDWIC_TENSORMAP_RING_CAP == 128 +static_assert(sizeof(SharedTensorMapSidecar) == 12434560, "shared TensorMap sidecar size changed"); +static_assert( + offsetof(SharedTensorMapSidecar, shared_outputs) == 2113664, + "shared output table offset mismatch" +); +static_assert( + offsetof(SharedTensorMapSidecar, shared_heap_cursor) == 11026560, + "shared heap cursor offset mismatch" +); +static_assert( + offsetof(SharedTensorMapSidecar, shared_heap_vend) == 11027072, + "shared heap vend offset mismatch" +); +static_assert( + offsetof(SharedTensorMapSidecar, shared_vector_cursor) == 11027136, + "shared Vector cursor offset mismatch" +); +static_assert( + offsetof(SharedTensorMapSidecar, writer_history) == 11027648, + "shared writer-history tail offset mismatch" +); +static_assert( + offsetof(SharedTensorMapSidecar, reader_done) == 12420288, + "shared reader-progress tail offset mismatch" +); +static_assert( + offsetof(SharedTensorMapSidecar, insert_turn_extra) == + 12426432, + "shared insert-turn tail offset mismatch" +); +#endif +#else +#if PTO_FDWIC_TENSORMAP_RING_CAP == 128 +static_assert(sizeof(SharedTensorMapSidecar) == 2113664, "private sidecar layout changed"); +#endif +#endif +static_assert(alignof(SharedTensorMapSidecar) == 64, "shared TensorMap sidecar alignment changed"); +static_assert( + offsetof(SharedTensorMapSidecar, buckets) == 128, + "shared TensorMap bucket offset mismatch" +); +static_assert( + offsetof(SharedTensorMapSidecar, slots) == + 128 + sizeof(SharedBucketState) * kMapBuckets, + "shared TensorMap slots must immediately follow all bucket controls" +); +#if PTO_FDWIC_TENSORMAP_RING_CAP == 128 +static_assert(offsetof(SharedTensorMapSidecar, slots) == 16512, "shared TensorMap slot offset changed"); +#endif + +struct TaskPayload { + TensorDesc tensors[kMaxTaskTensors]; +}; +// task_id 通过 kPayloadMask 映射到 2048 个 4 KiB payload。现有单组 B256 +// 只有 1280 个 task，不会回绕；未来 shared 多组允许回绕，但 payload +// 只活到本次 Finish 完成 descriptor 发布/slot 拷贝，后续依赖读取的是 +// shared cell 或 LocalSlot 中的副本，不能跨 Submit 保存 payload 指针。 +static_assert(sizeof(TaskPayload) == kPayloadStride, "TaskPayload must preserve the real 4 KiB task stride"); +static_assert(alignof(TaskPayload) == 8, "TaskPayload alignment must match DistTaskPayload"); +static_assert(offsetof(TaskPayload, tensors) == 0, "TaskPayload tensor offset mismatch"); + +struct LocalSlot { + // occupied 先保留容量，built 表示 payload 已按生产顺序构建；task/function + // 标识决定执行哪个 NOP 体，后续大数组则是 kernel 真正看到的参数快照。 + bool occupied; + bool built; + uint8_t header_padding[2]; + uint32_t task_id; + uint32_t kind; + uint32_t function_padding; + uint64_t function_address; + uint32_t tensor_count; + uint32_t scalar_count; + uint8_t tensor_padding[32]; + + TensorDesc tensors[kMaxTaskTensors]; + uint64_t scalars[kMaxTaskScalars]; + uint64_t args[kMaxTaskTensors + kMaxTaskScalars + 2]; + union { + struct { + uint8_t local_context[48]; + uint32_t global_context; + int32_t fanin[kMaxFanin]; + uint32_t fanin_count; + }; + // Compatibility view used by the standalone NOP payload builder. The + // first six words are the real 48-byte LocalContext; the remaining + // words overlap GlobalContext and the beginning of fanin, exactly as + // dictated by the real RingSlot offsets. + // 该视图只用于按真实 offset 填充 dispatch context，不增加另一份 + // 存储；修改其中后两字会同步覆盖 GlobalContext/fanin 的对应 ABI 字节。 + uint64_t context_words[8]; + }; + bool is_multicore; + int32_t won_block; + int32_t won_slot; +}; +// LocalSlot 是每个 winner 写入自己私有 ring 的完整 dispatch 包。fanin 在执行前 +// 逐项检查 task.flag；occupied/built 与计数共同约束最多两个普通 kernel 在途。 +static_assert(sizeof(LocalSlot) == 4824, "LocalSlot must match the PA RingSlot ABI size"); +static_assert(alignof(LocalSlot) == 8, "LocalSlot alignment must match RingSlot"); +static_assert(offsetof(LocalSlot, occupied) == 0, "LocalSlot occupied offset mismatch"); +static_assert(offsetof(LocalSlot, built) == 1, "LocalSlot built offset mismatch"); +static_assert(offsetof(LocalSlot, task_id) == 4, "LocalSlot task offset mismatch"); +static_assert(offsetof(LocalSlot, kind) == 8, "LocalSlot function-id offset mismatch"); +static_assert(offsetof(LocalSlot, function_address) == 16, "LocalSlot function address offset mismatch"); +static_assert(offsetof(LocalSlot, tensor_count) == 24, "LocalSlot tensor-count offset mismatch"); +static_assert(offsetof(LocalSlot, tensors) == 64, "LocalSlot tensor payload offset mismatch"); +static_assert(offsetof(LocalSlot, scalars) == 4160, "LocalSlot scalar payload offset mismatch"); +static_assert(offsetof(LocalSlot, args) == 4288, "LocalSlot dispatch-args offset mismatch"); +static_assert(offsetof(LocalSlot, local_context) == 4688, "LocalSlot local-context offset mismatch"); +static_assert(offsetof(LocalSlot, global_context) == 4736, "LocalSlot global-context offset mismatch"); +static_assert(offsetof(LocalSlot, fanin) == 4740, "LocalSlot fanin offset mismatch"); +static_assert(offsetof(LocalSlot, fanin_count) == 4804, "LocalSlot fanin-count offset mismatch"); +static_assert(offsetof(LocalSlot, is_multicore) == 4808, "LocalSlot multicore offset mismatch"); +static_assert(offsetof(LocalSlot, won_block) == 4812, "LocalSlot won-block offset mismatch"); +static_assert(offsetof(LocalSlot, won_slot) == 4816, "LocalSlot won-slot offset mismatch"); + +struct WorkerState { + CoreRole role; + int32_t core_idx; + int32_t block_id; + int32_t lane; + int32_t sub_block_id; + int32_t local_index; + uint64_t heap_next; + TensorMap map; + uint8_t slot_padding[16]; + LocalSlot slots[kPrivateSlots]; + uint32_t occupied_count; + uint32_t owned_total; + uint64_t swimlane_last_cycle; + uint8_t payload_padding[16]; + TaskPayload payloads[kPayloadSlots]; +}; +// 每个物理 worker 都持有独立 heap cursor、TensorMap、ring 与 task payload arena； +// 多核共享的只有 SchedulerState 前缀中的 cursor/task/frontier 等协议状态。 +static_assert(sizeof(WorkerState) == 9231296, "WorkerState must match the PA DistCore ABI size"); +static_assert(alignof(WorkerState) == 8, "WorkerState alignment must match DistCore"); +static_assert(offsetof(WorkerState, role) == 0, "WorkerState role offset mismatch"); +static_assert(offsetof(WorkerState, local_index) == 20, "WorkerState replay-index offset mismatch"); +static_assert(offsetof(WorkerState, heap_next) == 24, "WorkerState heap cursor offset mismatch"); +static_assert(offsetof(WorkerState, map) == 32, "WorkerState tensor-map offset mismatch"); +static_assert(offsetof(WorkerState, slots) == 823360, "WorkerState ring-slot offset mismatch"); +static_assert(offsetof(WorkerState, occupied_count) == 842656, "WorkerState occupancy offset mismatch"); +static_assert(offsetof(WorkerState, owned_total) == 842660, "WorkerState owned-count offset mismatch"); +static_assert(offsetof(WorkerState, swimlane_last_cycle) == 842664, "WorkerState trace clock offset mismatch"); +static_assert(offsetof(WorkerState, payloads) == 842688, "WorkerState task-payload offset mismatch"); + +struct alignas(64) WorkerResult { + // 时间边界：Submit 口径不含启动屏障和最终 drain，finish_cycle 则覆盖完整 worker 生命周期。 + uint64_t submit_begin; + uint64_t submit_end; + uint64_t finish_cycle; + uint64_t checksum; + + // 协议计数用于验证固定 Claim 拓扑及等待/依赖动态次数；joint_polls 是为未来 + // BlockWon 模拟保留的兼容字段，当前实现没有递增点，不能据其检测 joint 分支。 + uint64_t submits; + uint64_t claim_attempts; + uint64_t claim_wins; + uint64_t heap_guards; + uint64_t fanin_ready_loads; + uint64_t completion_duplicates; + uint64_t cas_retries; + uint64_t joint_polls; + + // 默认 256 batch 时 winner、kernel 分别闭合到 1280 task 和 1024 kernel； + // 非默认配置按 5*batches、4*batches 计算，placement 仍闭合到全部 kernel。 + uint64_t wins[static_cast(TaskKind::Count)]; + uint64_t kernel_counts[4]; + uint64_t kernel_cycles[4]; + uint64_t kernel_min_cycles[4]; + uint64_t kernel_max_cycles[4]; + uint64_t placement[static_cast(DrainPlace::Count)]; + uint64_t phase_cycles[static_cast(ProfilePhase::Count)]; + uint64_t phase_calls[static_cast(ProfilePhase::Count)]; + uint64_t wait_events[2]; + uint64_t wait_iterations[2]; + + // 前端工作量计数不是性能填充：private 核对全员构参/物化；shared + // 的五类 task 都核对 owner-only 重构参与 owner-only 物化。lookup、 + // slot copy 和 fanin 也按 owner 业务量闭合。 + uint64_t context_reads; + uint64_t views_created; + uint64_t dynamic_create_infos; + uint64_t arg_resets; + uint64_t tensor_args_added; + uint64_t scalar_args_added; + uint64_t materialized_outputs; + uint64_t map_inserts; + uint64_t map_lookups; + uint64_t slot_tensor_copies; + uint64_t slot_scalar_copies; + uint64_t fanin_edges; + + // private 保存逐 worker 逻辑 heap/map 终态；shared 的 heap_next 只是本核 + // 最近一次 winner 看到的 aggregate prefix，权威终态位于 shared sidecar。 + uint64_t final_heap_next; + uint64_t map_high_water; + uint64_t map_alive_floor; + uint64_t map_cleaned_upto; + uint64_t map_live_entries; + + uint64_t worker_id; + uint64_t role; + uint64_t max_occupied; + uint64_t final_occupied; + + // CCEC 标量 PMU 取证使用 WorkerResult 的诊断 sidecar，不改变生产 DistCore ABI。 + // 该诊断只在显式开启时有效；CNT2/CNT6/CNT7 分别对应 scalar busy、I-cache req/miss。 + uint64_t pmu_total_cycles; + uint32_t pmu_scalar_busy; + uint32_t pmu_icache_requests; + uint32_t pmu_icache_misses; + uint32_t pmu_status; + + // 这些计数只在 worker 私有 LocalStats 中递增，结束时一次性发布；它们把 + // 动态 fanin 重试和 private frontier helping 展开为准确次数。shared + // no-wrap 构建要求三个 frontier 计数全零；取数本身不增加共享 atomic。 + uint64_t fanin_not_ready_loads; + uint64_t frontier_initial_loads; + uint64_t frontier_updates; + uint64_t frontier_terminal_loads; + + // 仅在 trace_enabled bit1 开启时递增；每次源码 atomic 调用恰好增加一， + // host 用它与 Atomic span 数逐 worker 闭合，禁止把丢记录的泳道当成完整结果。 + uint64_t atomic_trace_calls; + + // I-cache 单 miss 探针用该槽保存 cold 窗口的 1 GHz SYS_CNT；submit-pmu + // 复用同一 64-bit 槽保存所选局部阶段的逐调用累计时间。两种构建互斥， + // 因而无需扩大当前 896B WorkerResult，也不会改变相邻 worker 的 cache-line 布局。 + union { + uint64_t pmu_window_ticks; + uint64_t pmu_phase_elapsed_ticks; + }; + uint64_t pmu_warm_total_cycles; + uint64_t pmu_warm_window_ticks; + union { + uint32_t pmu_warm_icache_requests; + uint32_t pmu_phase_begin_reads; + }; + union { + uint32_t pmu_warm_icache_misses; + uint32_t pmu_phase_end_reads; + }; + + // PIPE_UTILIZATION 已同时配置 CNT0/1/3/4/5/8；与上面的 scalar/I-cache + // 一样只保存每核原始累计值，AIC/AIV 汇总与比率统一在 host sidecar 中计算。 + // 六个 32-bit 值复用本结构既有 PMU 诊断区，不再增加 cache line。 + uint32_t pmu_vector_busy; + uint32_t pmu_cube_busy; + uint32_t pmu_mte1_busy; + uint32_t pmu_mte2_busy; + // swimlane ABI 保留该槽；submit-pmu 将物理 CNT5 改作 shadow miss， + // 因而显式发布 0，并在当前 submit-pmu schema-v5 标记 mte3_busy 不可用。 + uint32_t pmu_mte3_busy; + uint32_t pmu_fix_busy; + + // 复用 WorkerResult 原有的 32B cache-line 尾洞，不扩大 896B stride。 + // CNT6/7 是从不中途读取的权威整窗，CNT8/CNT5 是 read-to-clear shadow； + // none 在 stop 后要求逐核精确相等；运行中切片的 phase 只允许 shadow + // 单向小于 primary，并显式导出差值形成局部观测区间。 + uint32_t pmu_build_variant; + uint32_t pmu_phase_id; + uint32_t pmu_phase_calls; + uint32_t pmu_phase_status; + uint32_t pmu_phase_icache_requests; + uint32_t pmu_phase_icache_misses; + uint32_t pmu_shadow_icache_requests; + uint32_t pmu_shadow_icache_misses; + + // 生命周期屏障的 SYS_CNT 边界属于 standalone 诊断 sidecar，不进入生产 + // DistCore ABI。final release 与 final end 分开，保留“全局停止生产”和 + // “本核 drain 完成”两个不同事件。 + uint64_t startup_barrier_begin; + uint64_t startup_barrier_end; + uint64_t final_barrier_begin; + uint64_t final_barrier_release; + uint64_t final_barrier_end; + // 复用原 barrier_reserved[3] 的 24B，不扩大 WorkerResult。dependency + // signature 继续闭合 fanin 拓扑；后两项统计 shared symbol 的 + // last_writer INPUT load 和构建后 INOUT writer commit，private 构建 + // 必须保持零。 + uint64_t dependency_signature; + uint64_t shared_symbol_input_loads; + uint64_t shared_symbol_inout_commits; +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) + // split 协议诊断独占一条 cache line。普通 CPU/AscendC 与局部 PMU + // 构建不带这些字段，不改变它们的 WorkerResult ABI。shared 下 + // finish_calls 只统计跨 TU 的 winner Finish；task_id_sum 则由 caller + // 统计完整逻辑 replay，二者有意采用不同粒度。 + uint64_t compete_first_split_caller_state_address; + uint64_t compete_first_split_finish_state_address; + uint64_t compete_first_split_finish_calls; + uint64_t compete_first_split_protocol_errors; + uint64_t compete_first_split_state_cookie; + uint64_t compete_first_split_task_id_sum; + uint64_t compete_first_split_owner_worker_id; + uint64_t compete_first_split_reserved; +#endif +}; +// WorkerResult 是 standalone 尾部的诊断 sidecar，不属于真实 DistCore ABI；按 +// cache line 隔离后，各 worker 发布统计不会相互覆盖或污染被测共享状态。 +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) +static_assert(sizeof(WorkerResult) == 960, "split WorkerResult diagnostics must occupy whole cache lines"); +static_assert(offsetof(WorkerResult, compete_first_split_caller_state_address) == 896, + "split WorkerResult oracle offset mismatch"); +static_assert(offsetof(WorkerResult, compete_first_split_reserved) == 952, + "split WorkerResult oracle tail mismatch"); +#else +static_assert(sizeof(WorkerResult) == 896, "WorkerResult diagnostics must occupy whole cache lines"); +#endif +static_assert(offsetof(WorkerResult, pmu_total_cycles) == 680, "WorkerResult PMU offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_status) == 700, "WorkerResult PMU status offset mismatch"); +static_assert(offsetof(WorkerResult, fanin_not_ready_loads) == 704, "WorkerResult atomic diagnostic offset mismatch"); +static_assert(offsetof(WorkerResult, atomic_trace_calls) == 736, "WorkerResult atomic trace offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_window_ticks) == 744, "WorkerResult PMU timing offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_vector_busy) == 776, "WorkerResult extended PMU offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_build_variant) == 800, "WorkerResult submit-PMU offset mismatch"); +static_assert(offsetof(WorkerResult, pmu_shadow_icache_misses) == 828, "WorkerResult submit-PMU tail mismatch"); +static_assert(offsetof(WorkerResult, dependency_signature) == 872, "WorkerResult dependency signature offset mismatch"); +static_assert(offsetof(WorkerResult, shared_symbol_input_loads) == 880, + "WorkerResult shared symbol-load offset mismatch"); +static_assert( + offsetof(WorkerResult, shared_symbol_inout_commits) == 888, + "WorkerResult shared symbol-commit offset mismatch" +); + +// 从 cube_cursor 到 workers 结束保留关键字段 offset、DistCore ABI 和生产总字节跨度， +// 并非字段级完整镜像。RunConfig、输入 context_lens 与校验结果追加在该跨度之后， +// 因此测试控制信息不会改变被测字段 offset。 +struct alignas(64) SchedulerState { + // production prefix 的三组四分片 cursor 服务 AIC、private AIV 与 + // Alloc；shared AIV 继续使用 sidecar 尾部的 Vector cursor， + // S4.14b 启用全部八条物理线。 + // 同 task 的 eligible workers 仍竞争同一 shard，只有旧值小于 + // task_id 的调用成为 winner。 + AtomicLine cube_cursor[kCursorShards]; + AtomicLine vector_cursor[kCursorShards]; + AtomicLine alloc_cursor[kCursorShards]; + AtomicLine frontier; + int32_t heap_window; + uint8_t tasks_padding[60]; + TaskCell tasks[kTaskCellCapacity]; + // heap_base/size 描述共享物理区间。private 的 worker.heap_next 是各核 + // 独立回放的 ring 逻辑游标；shared 下它只镜像本核最近 winner 的 vend。 + uint64_t heap_base; + uint64_t heap_size; + uint64_t orchestration_args; + uint64_t runtime_state; + uint64_t runtime; + uint8_t fatal_padding[24]; + AtomicFlagLine fatal; + int32_t num_workers; + int32_t num_blocks; + // Case1 never enters BlockWon, but the inactive layout and BlockWon arena + // remains byte-for-byte reserved so every subsequent PA atomic line keeps + // its production offset. + // 此处不能因 Case1 动态次数为零而删减，否则 replay_done、started_count + // 和 DistCore 数组整体前移，便不再是对真实 PA 地址布局的等价测试。 + uint8_t layout_and_block_won[5848440]; + AtomicLine replay_done; + AtomicLine started_count; + // started_count 形成 launch 屏障。生产路径已将 final 汇合迁移到 + // DistGlobal 尾部的固定 G=16 树，replay_done 原位保留以维持后续字段 ABI。 + WorkerState workers[kRuntimeMaxWorkers]; + // 生产 DistGlobal 在 cores 后追加的固定 G=16 final 树。standalone + // 自己的五形态实验状态仍放在 controls 之后，两者不混用。 + uint8_t production_final_barrier[kRealFinalBarrierBytes]; + // Standalone-only controls live after the complete DistGlobal image. They + // therefore do not shift any cursor/task/fatal/worker address under test. + RunConfig config; + PmuProbeConfig pmu_probe; + WinnerWorkloadConfig winner_workload; + // Context lengths are the only PA input elements read by orchestration; + // keeping them in GM preserves the per-batch descriptor-based load. + // 除这些 batch 长度值外，其余 tensor 仅需稳定的合成地址来复现 + // descriptor、依赖和 heap 行为，不会解引用成真实计算数据。 + volatile int32_t context_lens[kMaxBatches]; + // standalone 的 final 分层实验状态位于完整生产 DistGlobal 镜像之后， + // 不移动任何被测生产字段；startup 继续使用生产 started_count。 + FinalBarrierState final_barrier; + WorkerResult results[kWorkers]; +#if PTO_FDWIC_SHARED_MAP + // shared 后端状态只追加在完整 production prefix、standalone controls + // 和 results 之后，不移动 RunConfig、WorkerState 或任何被测生产字段。 + SharedTensorMapSidecar shared_map; +#endif +}; +static_assert(offsetof(SchedulerState, cube_cursor) == 0, "cube cursor offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, vector_cursor) == 256, "vector cursor offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, alloc_cursor) == 512, "alloc cursor offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, frontier) == 768, "frontier offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, heap_window) == 832, "H offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, tasks) == kRealTasksOffset, "task table offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, heap_base) == 4195200, "heap base offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, heap_size) == 4195208, "heap size offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, fatal) == kRealFatalOffset, "fatal offset must match PA DistGlobal"); +static_assert(offsetof(SchedulerState, replay_done) == kRealReplayDoneOffset, "replay offset must match PA DistGlobal"); +static_assert( + offsetof(SchedulerState, started_count) == kRealStartedCountOffset, + "started-count offset must match PA DistGlobal" +); +static_assert(offsetof(SchedulerState, tasks) % 64 == 0, "task table must be cache-line aligned"); +static_assert(offsetof(SchedulerState, workers) % 64 == 0, "worker table must be cache-line aligned"); +static_assert(offsetof(SchedulerState, results) % 64 == 0, "result table must be cache-line aligned"); +static_assert(offsetof(SchedulerState, workers) == kRealDistCoreOffset, "DistCore table offset must match PA"); +static_assert(offsetof(SchedulerState, config) == kRealDistGlobalBytes, "DistGlobal byte size must match PA"); +static_assert( + offsetof(SchedulerState, pmu_probe) == kRealDistGlobalBytes + sizeof(RunConfig), + "PMU probe sidecar must follow RunConfig" +); +static_assert( + offsetof(SchedulerState, winner_workload) == + kRealDistGlobalBytes + sizeof(RunConfig) + sizeof(PmuProbeConfig), + "winner workload sidecar offset mismatch" +); +static_assert( + offsetof(SchedulerState, context_lens) == + kRealDistGlobalBytes + sizeof(RunConfig) + sizeof(PmuProbeConfig) + + sizeof(WinnerWorkloadConfig), + "context lengths must follow standalone controls" +); +static_assert(offsetof(SchedulerState, final_barrier) % 64 == 0, "final barrier must be cache-line aligned"); +#if PTO_FDWIC_SHARED_MAP +static_assert( + offsetof(SchedulerState, shared_map) == + offsetof(SchedulerState, results) + sizeof(WorkerResult) * kWorkers, + "shared TensorMap sidecar must follow the complete result array" +); +#if PTO_FDWIC_TENSORMAP_RING_CAP == 128 +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) +static_assert( + sizeof(SchedulerState) == 1019557696, + "shared split SchedulerState ABI changed" +); +#else +static_assert( + sizeof(SchedulerState) == 1019551552, + "shared non-split SchedulerState ABI changed" +); +#endif +#endif +#endif +static_assert(sizeof(SchedulerState) <= UINT32_MAX, "SchedulerState size must fit build identity"); + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_MODEL_H diff --git a/tests/atomic_probe/pa_scheduler/common/pa_scheduler_core.h b/tests/atomic_probe/pa_scheduler/common/pa_scheduler_core.h new file mode 100644 index 0000000000..5e116d4d9c --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/common/pa_scheduler_core.h @@ -0,0 +1,4533 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_SCHEDULER_CORE_H +#define PA_SCHEDULER_COMMON_PA_SCHEDULER_CORE_H + +#ifndef PA_DEVICE +#define PA_DEVICE inline +#endif + +#ifndef PA_GM +#define PA_GM +#endif + +#include "pa_frontend.h" +#if PTO_FDWIC_SHARED_MAP +#include "pa_shared_tensormap.h" +#endif +#include "pa_trace.h" + +namespace pa_scheduler { + +struct LocalStats { + WorkerResult result; + uint32_t max_occupied; +#if PTO_FDWIC_SHARED_MAP + // 只在末个 shared Submit 成功收尾时写入 task_id+1；回放结束后与 + // local_index 对照，证明 ticket 的 last bit 没有提前或遗漏。 + uint32_t declared_task_count; +#endif + TraceContext trace; +}; + +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) +// runtime-entry TU 按核型各自拥有一份 external [[block_local]] 实例。 +// orchestration caller 与 noinline finish 只交换固定 ticket/TaskArgs，Submit +// 内部的 context、统计与状态指针全部留在这份每核状态里。shared loser +// 不跨 TU：finish_calls 是 winner 数，task_id_sum 仍证明全量 caller replay。 +struct alignas(64) CompeteFirstSplitRuntimeState { + PA_GM SchedulerState *scheduler; + PA_GM WorkerState *worker; + uint32_t task_count; + uint32_t worker_id; + SubmitContext context; + LocalStats stats; + uint64_t caller_state_address; + uint64_t finish_state_address; + uint64_t finish_calls; + uint64_t protocol_errors; + uint64_t state_cookie; + uint64_t task_id_sum; + uint64_t owner_worker_id; + uint64_t reserved; +}; +static_assert(sizeof(CompeteFirstSplitRuntimeState) % 64 == 0, + "split runtime state must occupy whole cache lines"); +static_assert( + __is_trivially_constructible(CompeteFirstSplitRuntimeState) && + __has_trivial_destructor(CompeteFirstSplitRuntimeState), + "CCEC block-local split state must not require ctor/dtor" +); + +PA_DEVICE uint64_t CompeteFirstSplitStateCookie(uint32_t worker_id, CoreRole role) { + return kCompeteFirstSplitStateCookieBase ^ static_cast(worker_id) ^ + (static_cast(static_cast(role)) << 32U); +} +#endif + +// submit-pmu 的 phase 在编译期固定；非诊断构建完全不引用 Ops 的 phase +// 接口。这样公共调度代码保持一份，swimlane/CPU/AscendC 也不会多出运行时分支。 +template +PA_DEVICE void BeginSubmitPmuPhase(PmuContext &context) { +#if PA_BUILD_SUBMIT_PMU + if constexpr (kCompiledSubmitPmuPhase == Phase) { + Ops::PmuPhaseBegin(context); + } +#else + (void)context; +#endif +} + +template +PA_DEVICE void EndSubmitPmuPhase(PmuContext &context) { +#if PA_BUILD_SUBMIT_PMU + if constexpr (kCompiledSubmitPmuPhase == Phase) { + Ops::PmuPhaseEnd(context); + } +#else + (void)context; +#endif +} + +template +PA_DEVICE uint64_t TraceTimestamp(TraceContext &trace, WorkerResult &result) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)result; + return 0; +#else + (void)trace; + (void)result; + // PollBatch 只允许存在于显式 AtomicPollRegionBegin/End 区间，region + // end 已用自己的结束时钟完整收口。普通阶段边界只负责取时钟，避免 + // 每个 Submit 的四个主端点都重复读取 active_mask。 + return Ops::Now(); +#endif +} + +template +PA_DEVICE uint64_t TraceTimestampAfterAtomicResult( + TraceContext &trace, WorkerResult &result, T value +) { +#if PA_BUILD_TRACE_FREE + // submit-pmu/perf-clock/纯性能构建必须在预处理后完全没有额外时钟读取； + // value 也只用于保持模板调用形态，不在这些构建中制造返回依赖指令。 + (void)trace; + (void)result; + (void)value; + return 0; +#else + (void)trace; + (void)result; + // 与 TraceTimestamp 一样不触碰已经由显式 region 管理的 PollBatch； + // 这里只让 SYS_CNT 真正依赖 atomic 返回值。该时间表示本核 scalar + // 已能消费返回值，不宣称跨核可见。 + return Ops::NowAfterAtomicResult(value); +#endif +} + +PA_DEVICE uint32_t KindIndex(TaskKind kind) { return static_cast(kind); } + +#if PTO_FDWIC_SHARED_MAP +constexpr uint8_t kSharedPaTicketMetaPresent = 1U << 7; +constexpr uint8_t kSharedPaTicketLastSubmit = 1U << 6; +constexpr uint8_t kSharedPaTicketHasFollowing = 1U << 5; +constexpr uint8_t kSharedPaTicketKindMask = 0x07U; +constexpr uint8_t kSharedPaTicketGroupShift = 3; +constexpr uint8_t kSharedPaTicketGroupMask = 0x03U; + +struct SharedPaTaskMeta { + TaskKind kind; + uint32_t group_index; + uint32_t batch_start; + bool has_following_group; + bool is_last_submit; + bool chained_writer; +}; + +PA_DEVICE uint8_t EncodeSharedPaTaskMeta( + TaskKind kind, uint32_t group_index, bool has_following_group, + bool is_last_submit = false +) { + if (kind >= TaskKind::Count || + group_index >= kSharedPaMaxBlockGroups || + (kind == TaskKind::Alloc && + (group_index != 0 || has_following_group)) || + (kind != TaskKind::Up && has_following_group) || + (has_following_group && + group_index + 1U >= kSharedPaMaxBlockGroups) || + (is_last_submit && + (has_following_group || + (kind != TaskKind::Alloc && kind != TaskKind::Up)))) { + return 0; + } + return static_cast( + kSharedPaTicketMetaPresent | + (is_last_submit ? kSharedPaTicketLastSubmit : 0U) | + (has_following_group ? kSharedPaTicketHasFollowing : 0U) | + (group_index << kSharedPaTicketGroupShift) | + static_cast(kind) + ); +} + +PA_DEVICE bool DecodeSharedPaTaskMeta( + uint8_t encoded, uint32_t task_id, SharedPaTaskMeta &meta +) { + if ((encoded & kSharedPaTicketMetaPresent) == 0 || + task_id >= kMaxTasks) { + return false; + } + const TaskKind kind = + static_cast(encoded & kSharedPaTicketKindMask); + const uint32_t group_index = + (encoded >> kSharedPaTicketGroupShift) & + kSharedPaTicketGroupMask; + const bool has_following_group = + (encoded & kSharedPaTicketHasFollowing) != 0; + const bool is_last_submit = + (encoded & kSharedPaTicketLastSubmit) != 0; + if (kind >= TaskKind::Count || + group_index >= kSharedPaMaxBlockGroups || + (kind == TaskKind::Alloc && + (group_index != 0 || has_following_group)) || + (kind != TaskKind::Up && has_following_group) || + (has_following_group && + group_index + 1U >= kSharedPaMaxBlockGroups) || + (is_last_submit && + (has_following_group || + (kind != TaskKind::Alloc && kind != TaskKind::Up)))) { + return false; + } + const uint32_t task_offset = + SharedPaTaskOffset(kind, group_index); + if (task_id < task_offset) { + return false; + } + meta.kind = kind; + meta.group_index = group_index; + meta.batch_start = task_id - task_offset; + meta.has_following_group = has_following_group; + meta.is_last_submit = is_last_submit; + meta.chained_writer = + kind == TaskKind::Up && group_index != 0; + return true; +} +#endif + +PA_DEVICE TaskKind GetTaskKind(uint32_t task_id) { + return static_cast(task_id % kTasksPerBatch); +} + +PA_DEVICE int32_t FunctionId(TaskKind kind) { + return kind == TaskKind::Alloc ? -1 : static_cast(KindIndex(kind) - 1); +} + +#if PTO_FDWIC_SHARED_MAP +PA_DEVICE bool SharedPaFunctionIdMatches( + TaskKind kind, bool winner, int32_t function_id +) { + // Claim loser 不执行 kernel，真实 function_id 固定为 -1；winner 则 + // 必须与 ticket 中显式 kind 一致。QK/PV 的 output count 同为 1， + // 不能只靠 shared_result.Size() 间接校验。 + return function_id == (winner ? FunctionId(kind) : -1); +} +#endif + +PA_DEVICE uint64_t DependencyEdgeSignature( + uint32_t consumer, uint32_t producer +) { + // SplitMix64 finalizer 只作用于稳定的 (consumer,producer) 编码；各 + // winner 将边哈希 XOR 到本核结果，host 再跨核 XOR，因此签名与 + // winner 分布和完成顺序无关。 + uint64_t value = + (static_cast(consumer) << 32U) | producer; + value ^= value >> 30U; + value *= 0xBF58476D1CE4E5B9ULL; + value ^= value >> 27U; + value *= 0x94D049BB133111EBULL; + value ^= value >> 31U; + return value; +} + +PA_DEVICE uint32_t NopCountForKind(PA_GM const NopCounts &nops, TaskKind kind) { + switch (kind) { + case TaskKind::Qk: + return nops.qk; + case TaskKind::Sf: + return nops.sf; + case TaskKind::Pv: + return nops.pv; + case TaskKind::Up: + return nops.up; + default: + return 0; + } +} + +PA_DEVICE uint32_t WorkloadCountForKind(PA_GM const WorkloadCounts &counts, TaskKind kind) { + switch (kind) { + case TaskKind::Qk: + return counts.qk; + case TaskKind::Sf: + return counts.sf; + case TaskKind::Pv: + return counts.pv; + case TaskKind::Up: + return counts.up; + default: + return 0; + } +} + +PA_DEVICE uint32_t CountBits(uint32_t value) { + uint32_t count = 0; + while (value != 0) { + count += value & 1U; + value >>= 1; + } + return count; +} + +template +PA_DEVICE int64_t LoadLine( + PA_GM AtomicLine &line, LocalStats &stats, AtomicSite site, int32_t task_id = -1 +) { + // Ops::Load 在 A5 后端是 atomicAdd(0)；返回值是该 RMW 线性化时观察到的共享值。 + return TraceAtomicLoad(stats.trace, stats.result, task_id, site, &line.value); +} + +template +PA_DEVICE int32_t LoadLine( + PA_GM AtomicFlagLine &line, LocalStats &stats, AtomicSite site, int32_t task_id = -1 +) { + return TraceAtomicLoad(stats.trace, stats.result, task_id, site, &line.value); +} + +template +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) && PA_BUILD_SWIMLANE +PA_DEVICE_NOINLINE __attribute__((cold)) +#else +PA_DEVICE +#endif +void SetFatal( + PA_GM SchedulerState *state, LocalStats &stats, + int32_t task_id = -1 +) { + // fatal 只从 0 单调置 1，重复 Exchange 不会把其他 worker 已观察到的失败状态清除。 + TraceAtomicExchange( + stats.trace, stats.result, task_id, AtomicSite::FatalSet, &state->fatal.value, + static_cast(1) + ); +} + +template +PA_DEVICE bool IsFatal(PA_GM SchedulerState *state, LocalStats &stats, int32_t task_id = -1) { + return LoadLine(state->fatal, stats, AtomicSite::FatalPoll, task_id) != 0; +} + +template +PA_DEVICE bool WatchdogExpired( + PA_GM SchedulerState *state, LocalStats &stats, uint64_t begin, uint32_t &polls +) { + // 每 1024 次自旋才读取系统计数器，降低正常启动屏障上的计时开销；超时后向所有 worker 广播 fatal。 + ++polls; + if ((polls & 1023U) != 0 || Ops::Now() - begin <= kWatchdogTicks) { + return false; + } + SetFatal(state, stats); + return true; +} + +PA_DEVICE uint32_t TwoLevelFinalBarrierGroupCount(FinalBarrierShape shape) { + switch (shape) { + case FinalBarrierShape::TwoLevel4: + return 4; + case FinalBarrierShape::TwoLevel8: + return 8; + case FinalBarrierShape::TwoLevel16: + return 16; + default: + return 0; + } +} + +PA_DEVICE uint32_t FinalBarrierLeafGroup(FinalBarrierShape shape, PA_GM const WorkerState &worker) { + const uint32_t block = static_cast(worker.block_id); + if (shape == FinalBarrierShape::ThreeLevel6x4x4) { + return block % kFinalBarrierMaxLeafGroups; + } + return block % TwoLevelFinalBarrierGroupCount(shape); +} + +template +PA_DEVICE void PublishFinalBarrierLine(PA_GM AtomicLine &line, LocalStats &stats, AtomicSite increment_site) { + (void)TraceAtomicFetchAdd( + stats.trace, stats.result, -1, increment_site, &line.value, 1, + /*result_used=*/false + ); +} + +template +PA_DEVICE void ArriveHierarchicalFinalBarrier( + PA_GM FinalBarrierState &barrier, FinalBarrierShape shape, PA_GM const WorkerState &worker, + LocalStats &stats, AtomicSite increment_site +) { + const uint32_t leaf = FinalBarrierLeafGroup(shape, worker); + PublishFinalBarrierLine(barrier.leaf_arrivals[leaf], stats, increment_site); +} + +template +PA_DEVICE bool ProgressHierarchicalFinalBarrier( + PA_GM FinalBarrierState &barrier, FinalBarrierShape shape, PA_GM const WorkerState &worker, + LocalStats &stats, AtomicSite increment_site, AtomicSite poll_site, bool &leaf_forwarded, bool &middle_forwarded, + bool &root_released, bool &middle_released, bool &leaf_released +) { + const uint32_t block = static_cast(worker.block_id); + const uint32_t leaf = FinalBarrierLeafGroup(shape, worker); + const bool leaf_leader = worker.lane == 0 && block == leaf; + if (shape != FinalBarrierShape::ThreeLevel6x4x4) { + const uint32_t groups = TwoLevelFinalBarrierGroupCount(shape); + const int64_t workers_per_group = static_cast(kWorkers / groups); + if (leaf_leader && !leaf_forwarded && + LoadLine(barrier.leaf_arrivals[leaf], stats, poll_site) >= workers_per_group) { + PublishFinalBarrierLine(barrier.root_arrival, stats, increment_site); + leaf_forwarded = true; + } + const bool root_leader = leaf_leader && leaf == 0; + if (root_leader && !root_released && + LoadLine(barrier.root_arrival, stats, poll_site) >= static_cast(groups)) { + PublishFinalBarrierLine(barrier.root_release, stats, increment_site); + root_released = true; + } + if (leaf_leader && leaf_forwarded && !leaf_released && + LoadLine(barrier.root_release, stats, poll_site) >= 1) { + PublishFinalBarrierLine(barrier.leaf_releases[leaf], stats, increment_site); + leaf_released = true; + } + return LoadLine(barrier.leaf_releases[leaf], stats, poll_site) >= 1; + } + + constexpr int64_t kWorkersPerLeaf = 6; + constexpr int64_t kLeavesPerMiddle = 4; + constexpr int64_t kMiddleGroups = 4; + const uint32_t middle = leaf % kFinalBarrierMaxMiddleGroups; + const bool middle_leader = leaf_leader && leaf == middle; + if (leaf_leader && !leaf_forwarded && + LoadLine(barrier.leaf_arrivals[leaf], stats, poll_site) >= kWorkersPerLeaf) { + PublishFinalBarrierLine(barrier.middle_arrivals[middle], stats, increment_site); + leaf_forwarded = true; + } + if (middle_leader && leaf_forwarded && !middle_forwarded && + LoadLine(barrier.middle_arrivals[middle], stats, poll_site) >= kLeavesPerMiddle) { + PublishFinalBarrierLine(barrier.root_arrival, stats, increment_site); + middle_forwarded = true; + } + const bool global_leader = middle_leader && middle == 0; + if (global_leader && middle_forwarded && !root_released && + LoadLine(barrier.root_arrival, stats, poll_site) >= kMiddleGroups) { + PublishFinalBarrierLine(barrier.root_release, stats, increment_site); + root_released = true; + } + if (middle_leader && middle_forwarded && !middle_released && + LoadLine(barrier.root_release, stats, poll_site) >= 1) { + PublishFinalBarrierLine(barrier.middle_releases[middle], stats, increment_site); + middle_released = true; + } + if (leaf_leader && leaf_forwarded && !leaf_released && + LoadLine(barrier.middle_releases[middle], stats, poll_site) >= 1) { + PublishFinalBarrierLine(barrier.leaf_releases[leaf], stats, increment_site); + leaf_released = true; + } + return LoadLine(barrier.leaf_releases[leaf], stats, poll_site) >= 1; +} + +template +PA_DEVICE void AdvanceFrontier(PA_GM SchedulerState *state, LocalStats &stats) { + // frontier 只表示“从 task 0 开始已经连续完成”的最高 task id，不能越过尚未发布 flag 的空洞。 + // 多个完成者可以同时扫描同一段连续区间，FetchMax 保证共享 frontier 只前进、不回退。 + ++stats.result.frontier_initial_loads; + int64_t frontier = LoadLine( + state->frontier, stats, AtomicSite::FrontierInitialLoad + ); + while (true) { + const int64_t next = frontier + 1; + if (next < 0 || next >= static_cast(kTaskCellCapacity)) { + break; + } + if (TraceAtomicLoad( + stats.trace, stats.result, static_cast(next), AtomicSite::FrontierFlagLoad, + &state->tasks[next].flag + ) == 0) { + ++stats.result.frontier_terminal_loads; + break; + } + uint64_t retries = 0; + // FetchMax 返回更新前的值；若其他核已经走得更远，就从其 old 值继续扫描，避免重复从 next 起步。 + ++stats.result.frontier_updates; + const int64_t old = TraceAtomicFetchMax( + stats.trace, stats.result, static_cast(next), AtomicSite::FrontierMax, + &state->frontier.value, next, retries + ); + stats.result.cas_retries += retries; + frontier = old > next ? old : next; + } +} + +template +PA_DEVICE void CompleteTask( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, LocalStats &stats +) { + // 两种模式都先发布 vend，再经 store barrier 发布 ready flag：fanin 和 + // slot 执行以 flag 为可见性条件，不能交换顺序。private ring 还需要 + // 连续 frontier 做 heap reclaim；shared PA 使用有界 no-wrap shard， + // 依赖逐 task flag，正常完成路径不维护无消费者的全局前沿。 + TraceAtomicExchange( + stats.trace, stats.result, static_cast(task_id), AtomicSite::CompletionVendExchange, + &state->tasks[task_id].vend, worker.heap_next + ); + Ops::StoreBarrier(); + TraceAtomicExchange( + stats.trace, stats.result, static_cast(task_id), AtomicSite::CompletionFlagExchange, + &state->tasks[task_id].flag, static_cast(1) + ); +#if !PTO_FDWIC_SHARED_MAP + AdvanceFrontier(state, stats); +#endif +} + +template +PA_DEVICE bool SlotReady(PA_GM SchedulerState *state, PA_GM LocalSlot &slot, LocalStats &stats) { + // 每个 fanin flag 都是跨核共享的完成条件；在单轮 kernel 内， + // 完成值会单调保持 ready。shared 模式遇到第一个未就绪依赖时， + // 把此前已确认 ready 的前缀从本核私有 slot 中移除，避免后续 + // 每次 EfDrain 都重复 atomic-load 同一前缀；private 模式保持原逻辑。 + for (uint32_t index = 0; index < slot.fanin_count; ++index) { + const int32_t dependency = slot.fanin[index]; + if (TraceAtomicLoad( + stats.trace, stats.result, dependency, AtomicSite::FaninFlagLoad, + &state->tasks[dependency].flag + ) == 0) { + ++stats.result.fanin_not_ready_loads; +#if PTO_FDWIC_SHARED_MAP + if (index != 0) { + const uint32_t remaining = + slot.fanin_count - index; + for (uint32_t pending = 0; + pending < remaining; ++pending) { + slot.fanin[pending] = + slot.fanin[index + pending]; + } + slot.fanin_count = remaining; + } +#endif + return false; + } + ++stats.result.fanin_ready_loads; + } +#if PTO_FDWIC_SHARED_MAP + slot.fanin_count = 0; +#endif + return true; +} + +PA_DEVICE void RecordKernelCycles(LocalStats &stats, TaskKind kind, uint64_t cycles) { + const uint32_t index = KindIndex(kind) - 1; + ++stats.result.kernel_counts[index]; +#if PA_BUILD_TRACE_FREE + // 无 trace 构建仍保留四类 kernel 的正确性计数，但不把恒为零的 + // 观察时长写进热路径，更不会在 host 侧把 0 冒充 kernel 性能。 + (void)cycles; +#else + stats.result.kernel_cycles[index] += cycles; + if (stats.result.kernel_min_cycles[index] == 0 || cycles < stats.result.kernel_min_cycles[index]) { + stats.result.kernel_min_cycles[index] = cycles; + } + if (cycles > stats.result.kernel_max_cycles[index]) { + stats.result.kernel_max_cycles[index] = cycles; + } +#endif +} + +template +PA_DEVICE uint32_t DrainReady( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, DrainPlace place, LocalStats &stats +) { + // 同一套 drain 被三个位置复用：每次 Submit 开头的 EfDrain、ring 背压等待和所有 Submit 后的最终 drain。 + // slot 属于当前 worker；只有其全部跨核 fanin 已 ready 时才执行所选 winner 负载、发布完成并释放 slot。 + if (worker.occupied_count == 0) { + return 0; + } + uint32_t freed = 0; + // 一次调用遍历完本核全部私有 slot；未就绪项保留 occupied/built，已完成项立即清槽并继续扫描。 + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + PA_GM LocalSlot &slot = worker.slots[index]; + if (!slot.occupied || !slot.built || !SlotReady(state, slot, stats)) { + continue; + } + const TaskKind kind = static_cast(slot.kind + 1); + const uint64_t kernel_begin = TraceTimestamp(stats.trace, stats.result); + Ops::ExecuteKernel(state, worker, kind, NopCountForKind(state->config.nops, kind)); + const uint64_t kernel_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(slot.task_id), static_cast(slot.kind), + TracePhase::Kernel, ProfilePhase::ReplayTail, kernel_begin, kernel_end + ); + RecordKernelCycles(stats, kind, kernel_end - kernel_begin); + CompleteTask(state, worker, slot.task_id, stats); + const uint64_t commit_cycle = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(slot.task_id), static_cast(slot.kind), + TracePhase::Commit, ProfilePhase::ReplayTail, commit_cycle, commit_cycle + ); + slot.built = false; + slot.occupied = false; + --worker.occupied_count; + ++stats.result.placement[static_cast(place)]; + ++freed; + } + return freed; +} + +PA_DEVICE int32_t FindFreeSlot(PA_GM WorkerState &worker) { + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + if (!worker.slots[index].occupied) { + return static_cast(index); + } + } + return -1; +} + +template +PA_DEVICE void WaitForSlot( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, + uint32_t task_id, LocalStats &stats +#if PTO_FDWIC_SHARED_MAP + , bool &fatal_exit +#endif +) { + // 四个物理 slot 中预留两个 won slot 语义位，仅有 kUsableSlots 个可供本图使用；满时靠 drain 取得进展。 + const uint64_t wait_begin = TraceTimestamp(stats.trace, stats.result); + bool waited = false; + // 只聚合这个显式背压等待区中的 fanin 观察；每次 Submit 开头的 + // opportunistic EfDrain 仍保留逐条 Atomic，不能仅凭 site 名称全局聚合。 + const uint32_t poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, + TraceAtomicPollBatchMask(AtomicSite::FaninFlagLoad) | + TraceAtomicPollBatchMask(AtomicSite::FatalPoll) + ); + // 退出条件只有 occupied_count 重新低于可用容量；依赖尚未 ready 时 SpinHint 后继续重试。 + while (worker.occupied_count >= kUsableSlots) { + waited = true; + ++stats.result.wait_iterations[0]; + if (DrainReady( + state, worker, DrainPlace::RingBackpressure, stats + ) == 0) { +#if PTO_FDWIC_SHARED_MAP + // gate 放行后若其他 worker 报错，本核可能正被两个永远无法 + // ready 的后继 slot 顶满。每 1024 次无进展轮询一次 fatal， + // 只影响真正的背压慢路，不给正常 winner 热路增加原子读取。 + if ((stats.result.wait_iterations[0] & 1023ULL) == 0 && + IsFatal( + state, stats, static_cast(task_id) + )) { + fatal_exit = true; + break; + } +#endif + Ops::SpinHint(); + } + } + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + if (waited) { + ++stats.result.wait_events[0]; + const uint64_t wait_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::RingBp, + ProfilePhase::WaitForSlot, wait_begin, wait_end, 0, 0 + ); + } else if constexpr (Profile) { + AccumulatePhase( + stats.result, ProfilePhase::WaitForSlot, wait_begin, + TraceTimestamp(stats.trace, stats.result) + ); + } +} + +template +PA_DEVICE bool HeapGuard( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, uint64_t output_bytes, + LocalStats &stats +) { + // 只有产生新输出的 winner 需要保护环形 heap；retire=frontier-H 对应已经允许复用的最老任务 vend。 + // 等待期间也主动 drain 本核已就绪 slot，避免只自旋而阻塞能够推动 frontier 的 kernel。 + if (output_bytes == 0 || state->heap_base == 0) { + return true; + } + const uint64_t ring = state->heap_size; + ++stats.result.heap_guards; + const uint64_t wait_begin = TraceTimestamp(stats.trace, stats.result); + bool waited = false; + bool poll_region_active = false; + uint32_t poll_region = 0; + // 正常出口是 heap_next-vend 落入一个 ring；检测到不可能释放的覆盖或其他核 fatal 时返回失败。 + while (!IsFatal(state, stats, static_cast(task_id))) { + // 逻辑 heap 尚未走完第一圈时，所有物理输出区间都位于 [0, heap_next)， + // 不可能覆盖此前分配；保留上面的 fatal 原子检查后，可直接跳过 frontier/vend 读取。 + if (worker.heap_next <= ring) { + if (poll_region_active) { + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + } + if constexpr (Profile) { + AccumulatePhase( + stats.result, ProfilePhase::HeapGuard, wait_begin, + TraceTimestamp(stats.trace, stats.result) + ); + } + return true; + } + // 与真实 PA 一样，首圈 fast path 上方的 FatalPoll 仍是直接记录；只有 + // 确认进入 heap wrap 慢路径后，才开启本等待 episode 的四类观察聚合。 + if (!poll_region_active) { + poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, + TraceAtomicPollBatchMask(AtomicSite::FatalPoll) | + TraceAtomicPollBatchMask(AtomicSite::HeapFrontierLoad) | + TraceAtomicPollBatchMask(AtomicSite::HeapVendLoad) | + TraceAtomicPollBatchMask(AtomicSite::FaninFlagLoad) + ); + poll_region_active = true; + } + const int64_t frontier = LoadLine( + state->frontier, stats, AtomicSite::HeapFrontierLoad, static_cast(task_id) + ); + const int64_t retire = frontier - static_cast(state->heap_window); + const uint64_t vend = retire < 0 + ? 0 + : TraceAtomicLoad( + stats.trace, stats.result, static_cast(task_id), AtomicSite::HeapVendLoad, + &state->tasks[retire].vend + ); + if (worker.heap_next - vend <= ring) { + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + if (waited) { + ++stats.result.wait_events[1]; + const uint64_t wait_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::RingBp, + ProfilePhase::HeapGuard, wait_begin, wait_end, 0, 1 + ); + } else if constexpr (Profile) { + AccumulatePhase( + stats.result, ProfilePhase::HeapGuard, wait_begin, + TraceTimestamp(stats.trace, stats.result) + ); + } + return true; + } + if (frontier >= static_cast(task_id) - 1) { + SetFatal(state, stats, static_cast(task_id)); + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + return false; + } + waited = true; + ++stats.result.wait_iterations[1]; + if (DrainReady( + state, worker, DrainPlace::RingBackpressure, stats + ) == 0) { + Ops::SpinHint(); + } + } + if (poll_region_active) { + AtomicPollRegionEnd(stats.trace, stats.result, poll_region); + } + if (waited) { + ++stats.result.wait_events[1]; + const uint64_t wait_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, TracePhase::RingBp, + ProfilePhase::HeapGuard, wait_begin, wait_end, 0, 1 + ); + } + return false; +} + +struct ClaimOutcome { + bool attempted; + bool won; + uint64_t retries; + int32_t function_id; +}; + +template +PA_DEVICE ClaimOutcome Claim( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, TaskKind kind, + LocalStats &stats +) { + // Claim 在单调 cursor 上执行 atomicMax。private/Cube/Alloc 使用 + // production-prefix 四分片；shared Vector 使用 sidecar 中的八分片 + // cursor。同一 task 只有观察到旧值更小的竞争者获胜。 + // Alloc 由全部 96 个 worker 竞争；QK/PV 仅 32 个 AIC、 + // SF/UP 仅 64 个 AIV 发 atomicMax。 + ClaimOutcome outcome{false, false, 0, -1}; + if (task_id >= kTaskCellCapacity) { + return outcome; + } + PA_GM AtomicLine *cursor = nullptr; + if (kind == TaskKind::Alloc) { + cursor = &state->alloc_cursor[task_id % kCursorShards]; + } else { + // Mirror MixedKernels::to_active_mask(), core_mask(), popcount(), + // lane_active(), and self->role routing inside the real Claim span. + const int32_t aic_kernel = kind == TaskKind::Qk || kind == TaskKind::Pv ? FunctionId(kind) : -1; + const int32_t aiv0_kernel = kind == TaskKind::Sf || kind == TaskKind::Up ? FunctionId(kind) : -1; + const int32_t aiv1_kernel = -1; + uint8_t active_mask = 0; + if (aic_kernel >= 0) active_mask |= 1U; + if (aiv0_kernel >= 0) active_mask |= 2U; + if (aiv1_kernel >= 0) active_mask |= 4U; + const uint8_t core_mask = active_mask & 0x07U; + const int32_t active_count = __builtin_popcount(static_cast(core_mask)); + // 这里保留生产 Claim 的 lane-mask 路由边界。当前固定 PA 图按构造只生成单 lane + // 的 QK/PV 或 SF/UP；需要两个及以上 lane 协作的 joint task 本应进入 BlockWon + // 协议，本独立用例没有实现该动态路径，因此显式拒绝而不把它误当成单 lane task。 + if (active_count >= 2) { + return outcome; + } + if ((core_mask & 1U) != 0) { + if (worker.role != CoreRole::Aic) return outcome; + cursor = &state->cube_cursor[task_id % kCursorShards]; + outcome.function_id = aic_kernel; + } else if ((core_mask & 6U) != 0) { + if (worker.role != CoreRole::Aiv) return outcome; +#if PTO_FDWIC_SHARED_MAP + cursor = &state->shared_map.shared_vector_cursor[ + task_id % kSharedVectorCursorShards + ]; +#else + cursor = &state->vector_cursor[task_id % kCursorShards]; +#endif + outcome.function_id = (core_mask & 2U) != 0 ? aiv0_kernel : aiv1_kernel; + } else { + return outcome; + } + } + outcome.attempted = true; + // atomicMax 返回写入前的 cursor：old=task_id 则必须 Replay。 + const int64_t old = TraceAtomicFetchMax( + stats.trace, stats.result, static_cast(task_id), + AtomicSite::ClaimMax, &cursor->value, + static_cast(task_id), outcome.retries + ); + outcome.won = old < static_cast(task_id); + if (!outcome.won) outcome.function_id = -1; + return outcome; +} + +PA_DEVICE void RecordClaimOutcome(LocalStats &stats, TaskKind kind, const ClaimOutcome &outcome) { + if (outcome.attempted) ++stats.result.claim_attempts; + stats.result.cas_retries += outcome.retries; + if (outcome.won) { + ++stats.result.claim_wins; + ++stats.result.wins[KindIndex(kind)]; + } +} + +template +PA_DEVICE bool BuildWinner( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_id, TaskKind kind, + const TaskArgs &args, const SubmitContext &context, + const int32_t fanin[kMaxFanin], uint32_t fanin_count, LocalStats &stats +) { + // kernel winner 不在 Submit 内立即执行计算，而是把完整 payload 和 fanin 存入自己的私有 ring slot。 + // 后续 EfDrain/背压 drain/最终 drain 在依赖满足后执行它，这正是 PA 的 Submit 与执行解耦点。 +#if PTO_FDWIC_SHARED_MAP + bool slot_wait_failed = false; + WaitForSlot( + state, worker, task_id, stats, slot_wait_failed + ); + if (slot_wait_failed) { + return false; + } +#else + WaitForSlot(state, worker, task_id, stats); +#endif +#if !PTO_FDWIC_SHARED_MAP + // private heap_next 是单调 ring 坐标，必须通过 frontier/vend 防止覆盖。 + // shared S3.2 使用有界 shard cursor 且首版禁止回绕，两种坐标不能混用。 + if (!HeapGuard( + state, worker, task_id, context.output_bytes, stats + )) { + return false; + } +#endif + const int32_t slot_index = FindFreeSlot(worker); + if (slot_index < 0) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + PA_GM LocalSlot &slot = worker.slots[slot_index]; + // Match dist_submit_alloc_slot(): reserve and account the private slot + // before build_ring_slot_from_submit publishes its completed payload. + // 状态按“occupied 占位 -> built 清零 -> 计入占用 -> BuildSlotPayload”推进；后者会先 + // 置 built，再填充 payload。slot 为 worker 私有、没有跨核发布竞争，所以此处的 built + // 只是复刻生产状态机与构建成本，不承担对其他核发布完整 payload 的同步语义。 + slot.occupied = true; + slot.built = 0; + ++worker.occupied_count; + if (worker.occupied_count > stats.max_occupied) { + stats.max_occupied = worker.occupied_count; + } + const int32_t sub_block_id = worker.lane == 2 ? 1 : 0; +#if PTO_FDWIC_SHARED_MAP + BuildSlotPayload( + slot, task_id, static_cast(FunctionId(kind)), 0, args, context, fanin, fanin_count, + state->shared_map, &stats.trace, sub_block_id + ); +#else + BuildSlotPayload( + slot, task_id, static_cast(FunctionId(kind)), 0, args, + context, fanin, fanin_count, sub_block_id + ); +#endif + stats.result.slot_tensor_copies += static_cast(context.tensor_count); + stats.result.slot_scalar_copies += static_cast(context.scalar_count); + stats.result.fanin_edges += fanin_count; + return true; +} + +#if PTO_FDWIC_SHARED_MAP +PA_DEVICE bool DiscardBuiltTask( + PA_GM WorkerState &worker, uint32_t task_id +) { + // 只供“本 task 已 BuildWinner、随后 shared 封口失败”的终止路径使用。 + // slot 为本 worker 私有，可直接撤销；否则 FinalDrain 仍可能执行一个 + // 未完成 shared 最终封口的失败任务。 + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + PA_GM LocalSlot &slot = worker.slots[index]; + if (!slot.occupied || slot.task_id != task_id) { + continue; + } + const bool accounting_valid = worker.occupied_count != 0; + slot.built = false; + slot.occupied = false; + if (accounting_valid) { + --worker.occupied_count; + } + // 即使 occupied_count 本身已经损坏，终止路径也必须先清掉 slot， + // 避免 FinalDrain 执行未封口任务；返回 false 保留计数异常证据。 + return accounting_valid; + } + return false; +} + +template +PA_DEVICE bool DiscardSharedSlotsAfterReplayFatal( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, + LocalStats &stats +) { + // 只有所有 worker 都已退出 replay、不会再生产 slot 后才能调用。 + // fatal 表示本轮结果已经整体无效；此时未完成 fanin 不可能再获得 + // completion，继续 FinalDrain 只会永久自旋。保留 slot 的 task 与 + // 尚未就绪 fanin 后缀供诊断；已确认 ready 的前缀可能已被 SlotReady + // 移除。这里只清除本地执行资格与占用计数。 + if (state == nullptr || + !IsFatal(state, stats, /*task_id=*/-1)) { + return false; + } + uint32_t occupied_slots = 0; + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + occupied_slots += worker.slots[index].occupied ? 1U : 0U; + worker.slots[index].built = false; + worker.slots[index].occupied = false; + } + const bool accounting_valid = + occupied_slots == worker.occupied_count; + worker.occupied_count = 0; + return accounting_valid; +} + +// register_mask 只指向已经存在的 Local/GM descriptor。两个地址空间分支 +// 必须保持分离，避免 CCEC 把它们合并成不支持的 pointer phi。 +PA_DEVICE bool ValidateEmptySharedRegistration( + const TaskArgs &args, const SubmitContext &context +) { + uint32_t register_mask = context.register_mask; + for (int32_t index = 0; index < args.tensor_count; ++index) { + const uint32_t bit = 1U << static_cast(index); + if ((register_mask & bit) == 0) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + // shared fresh Output 由 (producer,slot) 直接寻址，不能退回 + // region map。manual_dep 的 output_view 同样不是 TensorMap 的 + // 自动 hazard，保留在 task args 但不登记。 + if (reference.kind == TensorRefKind::SharedOutputRef) { + register_mask &= ~bit; + continue; + } + if (reference.kind == TensorRefKind::GmTensor) { + PA_GM const TensorDesc &tensor = *reference.pointer.gm_tensor; + if (!tensor.manual_dep) { + return false; + } + } else if (reference.kind == TensorRefKind::LocalTensor) { + const TensorDesc &tensor = *reference.pointer.local_tensor; + if (!tensor.manual_dep) { + return false; + } + } else { + return false; + } + register_mask &= ~bit; + } + return register_mask == 0; +} + +enum class SharedWriterIntentResult : uint32_t { + NotRequired = 0, + Published = 1, + Failed = 2, +}; + +PA_DEVICE bool IsSharedWriterIntentTag(TensorArgType tag) { + return tag == TensorArgType::Inout || + tag == TensorArgType::OutputExisting; +} + +// 该扫描只回答“参数中是否存在需要自动登记的复写意图”，不依赖 PA +// TaskKind、group 或后继业务形状。manual_dep writer 由调用方显式管理， +// 不进入 shared TensorMap，也不要求 loser 等 writer-ready。 +PA_DEVICE bool InspectSharedWriterIntent( + const TaskArgs &args, bool &required +) { + required = false; + if (args.has_error || args.tensor_count < 0 || + args.tensor_count > static_cast(kMaxTaskTensors)) { + return false; + } + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = + TaskTag(args, static_cast(index)); + if (!IsSharedWriterIntentTag(tag)) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind == TensorRefKind::SharedOutputRef) { + if (!IsPlainSharedOutputRef( + SharedOutputReference(reference) + )) { + return false; + } + required = true; + continue; + } + if (reference.kind == TensorRefKind::GmTensor) { + if (reference.pointer.gm_tensor == nullptr) { + return false; + } + required |= !reference.pointer.gm_tensor->manual_dep; + continue; + } + if (reference.kind == TensorRefKind::LocalTensor) { + if (reference.pointer.local_tensor == nullptr) { + return false; + } + required |= !reference.pointer.local_tensor->manual_dep; + continue; + } + return false; + } + return true; +} + +// AddFanin 的既有接口为固定 PA Case1 静默截断到 16 条；通用 writer +// intent 不能丢失依赖，因此单独使用有返回值的严格版本。负 producer +// 表示 external input，不占 fanin。 +PA_DEVICE bool AddSharedWriterIntentFanin( + int32_t fanin[kMaxFanin], uint32_t &count, int32_t producer +) { + if (producer < 0) { + return true; + } + for (uint32_t index = 0; index < count; ++index) { + if (fanin[index] == producer) { + return true; + } + } + if (count >= kMaxFanin) { + return false; + } + fanin[count++] = producer; + return true; +} + +template +PA_DEVICE bool AddCollectedSharedFanin( + int32_t fanin[kMaxFanin], uint32_t &count, int32_t producer +) { + if constexpr (Strict) { + return AddSharedWriterIntentFanin( + fanin, count, producer + ); + } + AddFanin(fanin, count, producer); + return true; +} + +template +PA_DEVICE bool AddCollectedSharedOwner( + int32_t fanin[kMaxFanin], uint32_t &count, uint64_t owner, + int32_t reader_task, int32_t reader_lower_bound +) { + if (owner == kInvalidTaskId) { + return true; + } + const int32_t producer = + static_cast(owner & 0xFFFFFFFFU); + if constexpr (Strict) { + // 新 shared 路径只接受 [N-H,N) 内的真实前任。高 32 位非零、 + // self/future owner 都是协议错误；已经落到窗口左侧的旧 owner + // 不再形成依赖，但后续 ordinary lookup 仍可找到窗口内的新 writer。 + if (owner > static_cast(INT32_MAX) || + producer >= reader_task) { + return false; + } + if (producer < reader_lower_bound) { + return true; + } + } + return AddCollectedSharedFanin( + fanin, count, producer + ); +} + +template +PA_DEVICE bool WaitForSharedOutputPublished( + PA_GM SharedTensorMapSidecar &map, const FdwicOutputRef &output_ref, + PA_GM volatile int32_t *fatal +) { + // 前置条件：调用者已经用 IsPlainSharedOutputRef 校验 producer/slot/view + // 范围，并确认 producer_task_id 严格早于当前 consumer task。 + PA_GM volatile int64_t *published = + &map.shared_outputs[ + static_cast(output_ref.producer_task_id) + ].published[output_ref.output_slot].value; + const int64_t expected = + static_cast(output_ref.producer_task_id); + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 旧通用 output 等待仅由隔离测试使用；正式 ordered Submit 走带观察的单次校验 + int64_t observed = Ops::Load(published); + if (observed == expected) { + return true; + } + if (observed != -1) { + if (fatal != nullptr) { + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 旧通用 output 等待的隔离测试失败出口 + (void)Ops::Exchange(fatal, static_cast(1)); + } + return false; + } + + // 只在 producer 尚未发布时建立超时窗口；正常已就绪路径不增加 + // SYS_CNT。轮询对象按 (producer,slot) 分散，不再让所有依赖消费者 + // 争用同一条全局发布前沿。 + const uint64_t begin = Ops::Now(); + uint32_t polls = 0; + while (true) { + Ops::SpinHint(); + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 旧通用 output 等待的隔离测试轮询 + observed = Ops::Load(published); + if (observed == expected) { + return true; + } + if (observed != -1) { + if (fatal != nullptr) { + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 旧通用 output 等待的隔离测试失败出口 + (void)Ops::Exchange(fatal, static_cast(1)); + } + return false; + } + ++polls; + if ((polls & 1023U) != 0) { + continue; + } + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 旧通用 output 等待的隔离测试 watchdog + if (fatal != nullptr && Ops::Load(fatal) != 0) { + return false; + } + if (Ops::Now() - begin > kWatchdogTicks) { + if (fatal != nullptr) { + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 旧通用 output 等待的隔离测试超时出口 + (void)Ops::Exchange(fatal, static_cast(1)); + } + return false; + } + } +} + +// 只供已经取得本 task 有序 insert turn 的调用者使用。对任意合法引用 +// producer P < task N，P 的 fresh descriptor 发布先于 P 的 insert +// completion；逐 task predecessor completion 链又先于 N 取得 turn。因此 +// 此处若仍未观察到精确的 P，不存在继续轮询后可恢复的正常时序，只能把它 +// 视为协议错误。未取得该前提的通用路径必须继续使用上面的 Wait helper。 +template +PA_DEVICE bool CheckSharedOutputPublishedAfterInsertTurn( + PA_GM SharedTensorMapSidecar &map, + const FdwicOutputRef &output_ref, int32_t task_id, + AtomicSite site, LocalStats *stats +) { + if (!IsPlainSharedOutputRef(output_ref)) { + return false; + } + PA_GM volatile int64_t *published = + &map.shared_outputs[ + static_cast(output_ref.producer_task_id) + ].published[ + static_cast(output_ref.output_slot) + ].value; + return TraceConfiguredAtomicLoad( + stats == nullptr ? nullptr : &stats->trace, + stats == nullptr ? nullptr : &stats->result, + task_id, site, published + ) == + static_cast(output_ref.producer_task_id); +} + +PA_DEVICE bool SharedSymbolHistoryKey( + const FdwicOutputRef &output_ref, uint32_t &key +) { + if (!IsPlainSharedOutputRef(output_ref)) { + return false; + } + key = + static_cast(output_ref.producer_task_id) * + kSharedOutputMaxPerTask + + static_cast(output_ref.output_slot) + 1U; + return true; +} + +PA_DEVICE FdwicOutputRef SharedSymbolHistoryReference(uint32_t key) { + if (key == 0) { + return InvalidSharedOutputRef(); + } + --key; + const uint32_t producer = + key / kSharedOutputMaxPerTask; + const uint32_t slot = + key % kSharedOutputMaxPerTask; + if (producer >= kMaxTasks || + slot >= kSharedOutputMaxPerTask) { + return InvalidSharedOutputRef(); + } + return FdwicOutputRef{ + static_cast(producer), + static_cast(slot), + 0, 0, 0, 0 + }; +} + +// writer task 的 history cell 在对应 last_writer CAS 前完整写回，之后 +// 保持不可变。latest cache 若指向 reader 的未来 task，就按该 task 的 +// 精确 symbol key 取前驱，直到回到 reader 的过去；正常 latest +PA_DEVICE bool ResolveSharedSymbolWriterBefore( + PA_GM SharedTensorMapSidecar &map, + const FdwicOutputRef &output_ref, int32_t reader_task, + int32_t reader_lower_bound, int32_t &resolved_writer, + LocalStats &stats +) { + uint32_t symbol_key = 0; + if (!SharedSymbolHistoryKey(output_ref, symbol_key) || + reader_task <= output_ref.producer_task_id || + reader_task < 0 || reader_lower_bound < 0 || + reader_lower_bound > reader_task) { + return false; + } + PA_GM SharedOutputCell &origin = + map.shared_outputs[ + static_cast( + output_ref.producer_task_id + ) + ]; + int64_t latest = + TraceAtomicLoad( + stats.trace, stats.result, reader_task, + AtomicSite::SharedFaninLastWriterLoad, + &origin.last_writer[ + static_cast(output_ref.output_slot) + ].value + ); + uint32_t steps = 0; + while (latest >= reader_task) { + if (latest < 0 || + latest >= static_cast(kMaxTasks) || + steps++ >= kMaxTasks) { + return false; + } + PA_GM SharedWriterHistoryCell &history = + map.writer_history[static_cast(latest)]; + // latest CAS 是这份 immutable history 的发布边界。先失效首行 + // 取得 header；若该 task 有超过六个 symbol writer，再只失效 + // 余下实际使用的连续 record 行。 + (void)TraceConfiguredDcciInvalidate( + &stats.trace, reader_task, -1, + DcciSite::SharedFaninHistoryInvalidate, + &history, 64 + ); + const uint32_t count = history.count; + if (history.magic != kSharedWriterHistoryMagic || + history.writer_task != latest || + history.reserved != 0 || + count == 0 || + count > kSharedWriterHistoryMaxPerTask) { + return false; + } + const uint64_t used_bytes = + offsetof(SharedWriterHistoryCell, entries) + + static_cast(count) * + sizeof(SharedWriterHistoryRecord); + if (used_bytes > 64) { + (void)TraceConfiguredDcciInvalidate( + &stats.trace, reader_task, -1, + DcciSite::SharedFaninHistoryInvalidate, + &history.entries[6], used_bytes - 64 + ); + } + bool found = false; + int32_t previous = -1; + for (uint32_t index = 0; index < count; ++index) { + PA_GM const SharedWriterHistoryRecord &record = + history.entries[index]; + if (record.symbol_key != symbol_key) { + continue; + } + if (found) { + return false; + } + found = true; + previous = record.previous_writer; + } + if (!found || + previous < output_ref.producer_task_id || + previous >= latest) { + return false; + } + latest = previous; + } + if (latest < output_ref.producer_task_id || + latest >= reader_task) { + return false; + } + // 与 ordinary ring 使用同一半开窗口：[N-H,N)。history 仍需走到 + // 第一个 (latest); + return true; +} + +template < + typename Ops, bool ChainedWriter = false, + bool AcceptLatestWriter = false +> +PA_DEVICE uint32_t CollectSharedFanin( + PA_GM SharedTensorMapSidecar &map, const TaskArgs &args, + int32_t task_id, int32_t heap_window, LocalStats &stats, + int32_t fanin[kMaxFanin], bool &protocol_ok, + uint32_t &ordinary_lookup_count, + PA_GM volatile int32_t *fatal = nullptr, + int32_t chained_producer_task_id = -1, + int32_t expected_shared_writer = -1 +) { + static_assert( + !(ChainedWriter && AcceptLatestWriter), + "generic latest-writer lookup must not use the PA chained selector" + ); + protocol_ok = true; + ordinary_lookup_count = 0; + if (task_id < 0 || heap_window < 0 || + args.tensor_count < 0 || + args.tensor_count > static_cast(kMaxTaskTensors)) { + protocol_ok = false; + return 0; + } + const int32_t reader_lower_bound = + task_id > heap_window ? task_id - heap_window : 0; + if constexpr (ChainedWriter) { + // PA 的三个 accumulator 共用同一个 Alloc producer，后续每个 UP + // 同步推进这三个 slot。显式的 (producer,writer) 对只选择这一 + // symbol cell；本组 SF/PV 等 fresh refs 仍按自己的 producer + // 校验。selector 必须至少命中一个消费引用，不能传错后静默退化。 + if (chained_producer_task_id < 0 || + chained_producer_task_id >= expected_shared_writer || + expected_shared_writer >= task_id) { + protocol_ok = false; + return 0; + } + bool matched_chain_ref = false; + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = + TaskTag(args, static_cast(index)); + if (tag == TensorArgType::Output || + (tag != TensorArgType::Input && + tag != TensorArgType::Inout && + tag != TensorArgType::OutputExisting)) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind != TensorRefKind::SharedOutputRef) { + continue; + } + matched_chain_ref |= + SharedOutputReference(reference).producer_task_id == + chained_producer_task_id; + } + if (!matched_chain_ref) { + protocol_ok = false; + return 0; + } + } + + int32_t validated_fanin[kMaxFanin] = {}; + uint32_t validated_count = 0; + uint32_t validated_ordinary_lookups = 0; + uint32_t validated_input_loads = 0; + + // 只读取并校验，不修改 last_writer、统计或输出 fanin。旧 PA + // chained-writer 路径仍按自己的 registration/Build 边界提交； + // 独立 shared ordered-insert 路径则在进入这里前已经发布本 task + // writer history，所以 AcceptLatestWriter 必须沿 history 回退到 (index)); + if (tag == TensorArgType::Output) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind == TensorRefKind::SharedOutputRef) { + // 当前只接收普通 fresh Output；view ABI 已占位但尚未接入， + // 不能静默把带 view 的符号当成 plain descriptor 使用。 + const FdwicOutputRef output_ref = SharedOutputReference(reference); + if (!IsPlainSharedOutputRef(output_ref) || + output_ref.producer_task_id < 0 || + output_ref.producer_task_id >= task_id) { + protocol_ok = false; + return 0; + } + PA_GM SharedOutputCell &cell = + map.shared_outputs[static_cast(output_ref.producer_task_id)]; + bool output_published = false; + if constexpr (AcceptLatestWriter) { + // ordered Submit 在本 task 的 I commit 后才进入该实例； + // predecessor completion 链已经证明 producer 完成了 output + // 发布，未就绪只能立即按协议错误返回，不能重新打开轮询。 + output_published = + CheckSharedOutputPublishedAfterInsertTurn< + Ops, true + >( + map, output_ref, task_id, + AtomicSite::SharedFaninOutputPublishedLoad, + &stats + ); + } else { + output_published = + WaitForSharedOutputPublished( + map, output_ref, fatal + ); + } + if (!output_published) { + protocol_ok = false; + return 0; + } + if (tag != TensorArgType::Input && + tag != TensorArgType::Inout && + tag != TensorArgType::OutputExisting) { + protocol_ok = false; + return 0; + } + // 同一 task 对同一 symbol 最多只能有一个写引用，否则后面的 + // writer 提交会把本 task 自己误当成预期 producer。 + if (tag == TensorArgType::Inout || + tag == TensorArgType::OutputExisting) { + for (int32_t previous = 0; previous < index; ++previous) { + const TensorArgType previous_tag = + TaskTag(args, static_cast(previous)); + if (previous_tag != TensorArgType::Inout && + previous_tag != TensorArgType::OutputExisting) { + continue; + } + const TaskTensorRef &previous_ref = args.tensors[previous]; + if (previous_ref.kind != TensorRefKind::SharedOutputRef) { + continue; + } + const FdwicOutputRef previous_output = + SharedOutputReference(previous_ref); + if (previous_output.producer_task_id == + output_ref.producer_task_id && + previous_output.output_slot == + output_ref.output_slot) { + protocol_ok = false; + return 0; + } + } + } + int32_t writer = -1; + if constexpr (AcceptLatestWriter) { + // latest cell 是零开销快取；若 future writer 已经覆盖它， + // 只在这一慢路沿不可变前驱链回到 max(writer( + map, output_ref, task_id, + reader_lower_bound, writer, stats + )) { + protocol_ok = false; + return 0; + } + } else { + // PA 迁移完成前保留原来的精确 oracle：默认单组要求 + // writer==descriptor producer，ChainedWriter 只允许调用方 + // 指定的 accumulator 链。两种口径不能静默混用。 + const bool chained_ref = + ChainedWriter && + output_ref.producer_task_id == + chained_producer_task_id; + const int32_t expected_writer = + chained_ref + ? expected_shared_writer + : output_ref.producer_task_id; + writer = static_cast( + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - AcceptLatestWriter=false 的旧 PA oracle 分支不进入当前 shared scheduler + Ops::Load( + &cell.last_writer[ + output_ref.output_slot + ].value + ) + ); + if (expected_writer < output_ref.producer_task_id || + expected_writer >= task_id || + writer != expected_writer) { + protocol_ok = false; + return 0; + } + } + if (!AddCollectedSharedFanin( + validated_fanin, validated_count, + writer + )) { + protocol_ok = false; + return 0; + } + if (tag == TensorArgType::Input) { + ++validated_input_loads; + } + continue; + } + if (reference.kind == TensorRefKind::GmTensor) { + PA_GM const TensorDesc &tensor = *reference.pointer.gm_tensor; + if (tensor.manual_dep) { + continue; + } + const uint64_t owner = tensor.owner_task_id; + if (!AddCollectedSharedOwner< + AcceptLatestWriter + >( + validated_fanin, validated_count, owner, + task_id, reader_lower_bound + )) { + protocol_ok = false; + return 0; + } + if (tag == TensorArgType::Inout || + tag == TensorArgType::OutputExisting || + (tag == TensorArgType::Input && + (owner != kInvalidTaskId || AcceptLatestWriter))) { + bool lookup_ok = false; + const int32_t producer = + SharedLookupTensor( + map, tensor, task_id, heap_window, lookup_ok, + &stats.trace, &stats.result + ); + if (!lookup_ok) { + protocol_ok = false; + return 0; + } + ++validated_ordinary_lookups; + if (!AddCollectedSharedFanin< + AcceptLatestWriter + >( + validated_fanin, validated_count, producer + )) { + protocol_ok = false; + return 0; + } + } + } else if (reference.kind == TensorRefKind::LocalTensor) { + const TensorDesc &tensor = *reference.pointer.local_tensor; + if (tensor.manual_dep) { + continue; + } + const uint64_t owner = tensor.owner_task_id; + if (!AddCollectedSharedOwner< + AcceptLatestWriter + >( + validated_fanin, validated_count, owner, + task_id, reader_lower_bound + )) { + protocol_ok = false; + return 0; + } + if (tag == TensorArgType::Inout || + tag == TensorArgType::OutputExisting || + (tag == TensorArgType::Input && + (owner != kInvalidTaskId || AcceptLatestWriter))) { + bool lookup_ok = false; + const int32_t producer = + SharedLookupTensor( + map, tensor, task_id, heap_window, lookup_ok, + &stats.trace, &stats.result + ); + if (!lookup_ok) { + protocol_ok = false; + return 0; + } + ++validated_ordinary_lookups; + if (!AddCollectedSharedFanin< + AcceptLatestWriter + >( + validated_fanin, validated_count, producer + )) { + protocol_ok = false; + return 0; + } + } + } else { + protocol_ok = false; + return 0; + } + } + + // 全部引用只读校验通过后，才一次性发布统计与 fanin 结果。INPUT 次数 + // 在验证扫描中先落局部量，保留 late-failure 的 all-or-nothing 口径。 + stats.result.shared_symbol_input_loads += validated_input_loads; + ordinary_lookup_count = validated_ordinary_lookups; + for (uint32_t edge = 0; edge < validated_count; ++edge) { + fanin[edge] = validated_fanin[edge]; + } + return validated_count; +} + +template +PA_DEVICE bool PublishSharedWriterReady( + PA_GM SchedulerState *state, int32_t task_id +) { + if (state == nullptr || task_id < 0 || + task_id >= static_cast(kMaxTasks)) { + return false; + } + // writer 登记必须先于门值对 loser 可见；本 task 是否已经执行完成 + // 仍由它自己的 completion flag 表达，不能把 deps_prepared 冒充成 + // 可执行/已完成。CAS 只允许初始化 sentinel -> task_id：重复 winner + // 或错误 task-cell 复用不会先写入一个合法门值再报告失败。 + Ops::StoreBarrier(); + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 旧 writer-ready 协议只供隔离门槛，正式路径直接发布 task completion + return Ops::CompareExchange( + &state->tasks[static_cast(task_id)].deps_prepared, + static_cast(-1), + static_cast(task_id) + ) == -1; +} + +template +PA_DEVICE bool WaitForSharedWriterReady( + PA_GM SchedulerState *state, int32_t task_id, LocalStats &stats +) { + if (state == nullptr || task_id < 0 || + task_id >= static_cast(kMaxTasks)) { + return false; + } + PA_GM volatile int64_t *prepared = + &state->tasks[static_cast(task_id)].deps_prepared; + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 旧 writer-ready 等待只供隔离门槛 + int64_t observed = Ops::Load(prepared); + if (observed == task_id) { + return true; + } + if (observed != -1) { + SetFatal(state, stats, task_id); + return false; + } + + const uint64_t begin = Ops::Now(); + uint32_t polls = 0; + while (true) { + Ops::SpinHint(); + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 旧 writer-ready 等待只供隔离门槛 + observed = Ops::Load(prepared); + if (observed == task_id) { + return true; + } + if (observed != -1) { + SetFatal(state, stats, task_id); + return false; + } + ++polls; + if ((polls & 1023U) != 0) { + continue; + } + if (IsFatal(state, stats, task_id)) { + return false; + } + if (Ops::Now() - begin > kWatchdogTicks) { + SetFatal(state, stats, task_id); + return false; + } + } +} + +PA_DEVICE bool CheckedMultiplyU64ByU32( + uint64_t left, uint32_t right, uint64_t &product +) { + // CCEC 9.1 会把“UINT64_MAX/right 预检 + 64-bit 乘法”融合成 + // AICore 环境没有运行库实现的 __multi3。按 32-bit limb 展开后只需 + // 32x32->64 乘法，同时保留 generic TensorDesc 原有的 uint64 extent + // 取值域，不能为迁就编译器把合法的大 ordinary region 收窄到 32 bit。 + const uint64_t low_product = + static_cast( + static_cast(left) + ) * right; + const uint64_t high_product = + static_cast( + static_cast(left >> 32) + ) * right; + const uint64_t carry = low_product >> 32; + if (high_product > UINT32_MAX - carry) { + return false; + } + product = + ((high_product + carry) << 32) | + static_cast(low_product); + return true; +} + +template +PA_DEVICE bool MakeValidatedSharedWriterRegion( + const TensorReference &tensor, int32_t task_id, + SharedRegionValue ®ion +) { + if (task_id < 0 || tensor.dtype >= DataType::Count || + tensor.ndims == 0 || tensor.ndims > kMaxTensorDims) { + return false; + } + const uint64_t element_size = ElementSize(tensor.dtype); + if (element_size == 0) { + return false; + } + uint64_t extent = tensor.extent_elem_cache; + if (tensor.is_contiguous) { + extent = 1; + for (uint32_t dimension = 0; + dimension < tensor.ndims; ++dimension) { + const uint32_t shape = tensor.shapes[dimension]; + uint64_t next_extent = 0; + if (shape == 0 || + !CheckedMultiplyU64ByU32( + extent, shape, next_extent + )) { + return false; + } + extent = next_extent; + } + } + uint32_t element_shift = 0; + if (element_size == 8) { + element_shift = 3; + } else if (element_size == 4) { + element_shift = 2; + } else if (element_size == 2) { + element_shift = 1; + } else if (element_size != 1) { + return false; + } + if (extent == 0 || + tensor.start_offset > UINT64_MAX - extent) { + return false; + } + const uint64_t end_offset = tensor.start_offset + extent; + const uint64_t max_element_offset = + UINT64_MAX >> element_shift; + if (tensor.start_offset > max_element_offset || + end_offset > max_element_offset) { + return false; + } + region.buffer_addr = tensor.buffer_addr; + region.lo = tensor.start_offset << element_shift; + region.hi = end_offset << element_shift; + region.producer = task_id; + region.reserved = 0; + return region.lo < region.hi; +} + +template +PA_DEVICE bool ValidateOrdinarySharedWriterReference( + const TensorReference &tensor, int32_t task_id +) { + if (tensor.manual_dep) { + return true; + } + SharedRegionValue unused{}; + if (!MakeValidatedSharedWriterRegion(tensor, task_id, unused)) { + return false; + } + if (tensor.owner_task_id == kInvalidTaskId) { + return true; + } + if (tensor.owner_task_id > + static_cast(INT32_MAX)) { + return false; + } + const int32_t owner = static_cast( + tensor.owner_task_id + ); + return owner >= 0 && owner < task_id; +} + +// 在执行任一 atomic/region append 前先完成所有 writer 引用的结构校验。 +// symbol 重复 writer 会让第二次 CAS 把本 task 自己当成旧 writer，因此 +// 必须在第一项改写之前拒绝。ordinary 多 view 可以合法重叠，不在这里 +// 按地址去重。 +PA_DEVICE bool ValidateSharedWriterIntentSet( + const TaskArgs &args, int32_t task_id +) { + bool required = false; + if (task_id < 0 || + !InspectSharedWriterIntent(args, required) || + !required) { + return false; + } + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = + TaskTag(args, static_cast(index)); + if (!IsSharedWriterIntentTag(tag)) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind == TensorRefKind::SharedOutputRef) { + const FdwicOutputRef output_ref = + SharedOutputReference(reference); + if (output_ref.producer_task_id < 0 || + output_ref.producer_task_id >= task_id) { + return false; + } + for (int32_t previous = 0; previous < index; ++previous) { + if (!IsSharedWriterIntentTag( + TaskTag( + args, static_cast(previous) + ) + )) { + continue; + } + const TaskTensorRef &previous_ref = + args.tensors[previous]; + if (previous_ref.kind != + TensorRefKind::SharedOutputRef) { + continue; + } + const FdwicOutputRef previous_output = + SharedOutputReference(previous_ref); + if (previous_output.producer_task_id == + output_ref.producer_task_id && + previous_output.output_slot == + output_ref.output_slot) { + return false; + } + } + continue; + } + if (reference.kind == TensorRefKind::GmTensor) { + if (!ValidateOrdinarySharedWriterReference( + *reference.pointer.gm_tensor, task_id + )) { + return false; + } + continue; + } + if (reference.kind == TensorRefKind::LocalTensor) { + if (!ValidateOrdinarySharedWriterReference( + *reference.pointer.local_tensor, task_id + )) { + return false; + } + continue; + } + return false; + } + return true; +} + +template +PA_DEVICE bool CommitOrdinarySharedWriterIntent( + PA_GM SharedTensorMapSidecar &map, + const TensorReference &tensor, int32_t task_id, + int32_t heap_window, int32_t fanin[kMaxFanin], + uint32_t &fanin_count, LocalStats &stats +) { + if (tensor.manual_dep) { + return true; + } + if (tensor.owner_task_id != kInvalidTaskId) { + if (tensor.owner_task_id > + static_cast(INT32_MAX)) { + return false; + } + const int32_t owner = static_cast( + tensor.owner_task_id + ); + if (owner < 0 || owner >= task_id || + !AddSharedWriterIntentFanin( + fanin, fanin_count, owner + )) { + return false; + } + } + + bool lookup_ok = false; + const int32_t previous = SharedLookupTensor( + map, tensor, task_id, heap_window, lookup_ok, + &stats.trace, &stats.result + ); + if (!lookup_ok || + !AddSharedWriterIntentFanin( + fanin, fanin_count, previous + )) { + return false; + } + ++stats.result.map_lookups; + + SharedRegionValue entry{}; + if (!MakeValidatedSharedWriterRegion( + tensor, task_id, entry + )) { + return false; + } + // 通用 writer-ready 目前只证明 writer publication 的先后，尚未 + // 证明所有更早 reader 已结束。这里保持 append-only，不按 task_id + // 推进 head；容量耗尽走 terminal failure，不能用可能仍被慢 reader + // 扫描的槽换取表面上的无限回绕。 + if (SharedCheckTaskAppend( + map, &entry, 1, -1 + ) != SharedAppendCheck::Ready || + !SharedAppendPreparedEntry(map, entry)) { + return false; + } + ++stats.result.map_inserts; + return true; +} + +template +PA_DEVICE bool CommitSymbolSharedWriterIntentSet( + PA_GM SharedTensorMapSidecar &map, const TaskArgs &args, + int32_t task_id, + int32_t fanin[kMaxFanin], uint32_t &fanin_count, + LocalStats &stats, PA_GM volatile int32_t *fatal +) { + // 调用方必须保证同一 symbol 的 writer 按 task_id 单调进入本函数。 + // 独立 shared Submit 由全局 insert turn 建立这一顺序；仍保留的隔离 + // driver 则必须提供等价的唯一 ordered writer 合同。CAS 负责发现 + // 乱序或重复 owner，但不会替调用方补回已被跨越的 writer。 + if (task_id < 0 || + task_id >= static_cast(kMaxTasks)) { + return false; + } + PA_GM SharedWriterHistoryCell &history = + map.writer_history[static_cast(task_id)]; + + uint32_t count = 0; + for (int32_t index = 0; index < args.tensor_count; ++index) { + if (!IsSharedWriterIntentTag( + TaskTag(args, static_cast(index)) + )) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind != TensorRefKind::SharedOutputRef) { + continue; + } + if (count >= kSharedWriterHistoryMaxPerTask) { + return false; + } + const FdwicOutputRef output_ref = + SharedOutputReference(reference); + uint32_t symbol_key = 0; + if (!SharedSymbolHistoryKey(output_ref, symbol_key) || + !WaitForSharedOutputPublished( + map, output_ref, fatal + )) { + return false; + } + PA_GM volatile int64_t *last_writer = + &map.shared_outputs[ + static_cast( + output_ref.producer_task_id + ) + ].last_writer[ + static_cast(output_ref.output_slot) + ].value; + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - generic writer-intent helper 不进入当前 prepared ordered Submit + const int64_t previous = Ops::Load(last_writer); + if (previous < output_ref.producer_task_id || + previous >= task_id || + !AddSharedWriterIntentFanin( + fanin, fanin_count, + static_cast(previous) + )) { + return false; + } + history.entries[count].symbol_key = symbol_key; + history.entries[count].previous_writer = + static_cast(previous); + ++count; + } + if (count == 0) { + return true; + } + + history.magic = kSharedWriterHistoryMagic; + history.writer_task = task_id; + history.count = count; + history.reserved = 0; + const uint64_t history_bytes = + offsetof(SharedWriterHistoryCell, entries) + + static_cast(count) * + sizeof(SharedWriterHistoryRecord); + (void)TraceConfiguredDcciFlush( + nullptr, task_id, -1, + DcciSite::SharedWriterHistoryFlush, + &history, history_bytes + ); + Ops::StoreBarrier(); + + // last_writer CAS 是每条前驱记录的发布边界。history 已整体写回， + // 因而 reader 观察到任一 current task 后都能按 key 取到其前驱。 + for (uint32_t index = 0; index < count; ++index) { + PA_GM const SharedWriterHistoryRecord &record = + history.entries[index]; + const FdwicOutputRef output_ref = + SharedSymbolHistoryReference(record.symbol_key); + if (!IsPlainSharedOutputRef(output_ref)) { + return false; + } + PA_GM volatile int64_t *last_writer = + &map.shared_outputs[ + static_cast( + output_ref.producer_task_id + ) + ].last_writer[ + static_cast(output_ref.output_slot) + ].value; + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - generic writer-intent helper 不进入当前 prepared ordered Submit + if (Ops::CompareExchange( + last_writer, + static_cast(record.previous_writer), + static_cast(task_id) + ) != record.previous_writer) { + return false; + } + // 多 symbol 发布不是事务；若后项冲突，已经线性化的前缀不回滚。 + // 逐项计数保留故障现场，外层随后设置 fatal 且不发布 ready gate。 + ++stats.result.shared_symbol_inout_commits; + } + return true; +} + +// ordered Submit 专用入口：调用方已经在 insert turn 外完成 symbol ref +// 校验、去重和 packed-key 生成。这里不再扫描 args，也不构造随后会被 +// 丢弃的 fanin；previous writer 仍必须在取得 turn 后读取，才能写入当前 +// task 的不可变 history。通用 CommitSymbolSharedWriterIntentSet 继续保留 +// 原有等待 publication、收集 fanin 和逐项统计的合同，二者不能互换。 +template +PA_DEVICE bool CommitPreparedSymbolSharedWriterIntentSet( + PA_GM SharedTensorMapSidecar &map, + const uint32_t *symbol_keys, uint32_t symbol_count, + int32_t task_id, PA_GM volatile int32_t *fatal, + LocalStats *stats = nullptr +) { + // 正式 ordered Submit 在 task-level completion 成功后统一记录完整 + // transaction；本 helper 固定不产生逐项成功统计，避免部分 CAS 前缀 + // 与 task-level 计数混成两种口径。 + if (task_id < 0 || + task_id >= static_cast(kMaxTasks) || + symbol_count > kSharedWriterHistoryMaxPerTask || + (symbol_count != 0 && symbol_keys == nullptr)) { + return false; + } + if (symbol_count == 0) { + return true; + } + + PA_GM SharedWriterHistoryCell &history = + map.writer_history[static_cast(task_id)]; + for (uint32_t index = 0; index < symbol_count; ++index) { + const uint32_t symbol_key = symbol_keys[index]; + const FdwicOutputRef output_ref = + SharedSymbolHistoryReference(symbol_key); + if (!IsPlainSharedOutputRef(output_ref) || + output_ref.producer_task_id >= task_id || + !CheckSharedOutputPublishedAfterInsertTurn< + Ops, ObserveAtomics + >( + map, output_ref, task_id, + AtomicSite::SharedMetadataOutputPublishedLoad, + stats + )) { + if (fatal != nullptr) { + (void)TraceConfiguredAtomicExchange< + Ops, ObserveAtomics + >( + stats == nullptr ? nullptr : &stats->trace, + stats == nullptr ? nullptr : &stats->result, + task_id, AtomicSite::FatalSet, + fatal, static_cast(1), + /*result_used=*/false + ); + } + return false; + } + + PA_GM volatile int64_t *last_writer = + &map.shared_outputs[ + static_cast( + output_ref.producer_task_id + ) + ].last_writer[ + static_cast(output_ref.output_slot) + ].value; + const int64_t previous = + TraceConfiguredAtomicLoad( + stats == nullptr ? nullptr : &stats->trace, + stats == nullptr ? nullptr : &stats->result, + task_id, AtomicSite::SharedMetadataLastWriterLoad, + last_writer + ); + if (previous < output_ref.producer_task_id || + previous >= task_id) { + return false; + } + history.entries[index].symbol_key = symbol_key; + history.entries[index].previous_writer = + static_cast(previous); + } + + history.magic = kSharedWriterHistoryMagic; + history.writer_task = task_id; + history.count = symbol_count; + history.reserved = 0; + const uint64_t history_bytes = + offsetof(SharedWriterHistoryCell, entries) + + static_cast(symbol_count) * + sizeof(SharedWriterHistoryRecord); + (void)TraceConfiguredDcciFlush( + stats == nullptr ? nullptr : &stats->trace, + task_id, -1, DcciSite::SharedWriterHistoryFlush, + &history, history_bytes + ); + Ops::StoreBarrier(); + + for (uint32_t index = 0; index < symbol_count; ++index) { + PA_GM const SharedWriterHistoryRecord &record = + history.entries[index]; + const FdwicOutputRef output_ref = + SharedSymbolHistoryReference(record.symbol_key); + if (!IsPlainSharedOutputRef(output_ref)) { + return false; + } + PA_GM volatile int64_t *last_writer = + &map.shared_outputs[ + static_cast( + output_ref.producer_task_id + ) + ].last_writer[ + static_cast(output_ref.output_slot) + ].value; + if (TraceConfiguredAtomicCompareExchange< + Ops, ObserveAtomics + >( + stats == nullptr ? nullptr : &stats->trace, + stats == nullptr ? nullptr : &stats->result, + task_id, + AtomicSite::SharedMetadataLastWriterCommit, + last_writer, + static_cast(record.previous_writer), + static_cast(task_id) + ) != record.previous_writer) { + return false; + } + } + return true; +} + +// 该公共原语只处理写集合本身：读取旧 writer、发布当前 writer，并在 +// 全部 symbol/ordinary 元数据完成后放行同 task loser。它不收集纯 INPUT， +// 不做 Materialize/Build，也不发布 completion；后两者必须继续使用 +// task.flag。当前阶段锁定 A->B->慢 C->D->E 的慢 reader： +// - symbol 以 last_writer 为快取、task-indexed immutable history 为慢路； +// - 同一 symbol writer 必须按 task id 发布，乱序/部分 CAS 失败均终止整轮； +// - ordinary ring 只允许有序单追加且不回收，容量耗尽时 terminal fail。 +// 因此本函数尚未接入 PA runtime，也不能被描述成通用多版本 backend 已闭合。 +template +PA_DEVICE SharedWriterIntentResult PrepareSharedWriterIntentSet( + PA_GM SchedulerState *state, const TaskArgs &args, + SubmitContext &context, LocalStats &stats +) { + bool required = false; + if (state == nullptr || + !InspectSharedWriterIntent(args, required)) { + if (state != nullptr) { + SetFatal(state, stats, context.task_id); + } + return SharedWriterIntentResult::Failed; + } + if (!required) { + return SharedWriterIntentResult::NotRequired; + } + const int32_t task_id = context.task_id; + if (!context.won || + task_id < 0 || + task_id >= static_cast(kMaxTasks) || + context.fanin_count < 0 || + context.fanin_count > static_cast(kMaxFanin) || + !ValidateSharedWriterIntentSet(args, task_id) || + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - generic writer-intent helper 不进入当前 prepared ordered Submit + Ops::Load(&state->fatal.value) != 0) { + SetFatal(state, stats, task_id); + return SharedWriterIntentResult::Failed; + } + + int32_t intent_fanin[kMaxFanin] = {}; + uint32_t intent_fanin_count = 0; + // Writer intent 可以接在调用方已经完成的只读 fanin 解析之后。先把 + // 既有边复制进本地去重集合，全部 writer metadata 成功后再一次性 + // 回写 context；这样 PA 迁移无需保留另一套“先 Collect 再 Commit” + // 专用协议，失败路径也不会留下半更新的 context。 + for (int32_t edge = 0; edge < context.fanin_count; ++edge) { + const int32_t producer = + context.fanin[static_cast(edge)]; + if (producer < 0 || producer >= task_id || + !AddSharedWriterIntentFanin( + intent_fanin, intent_fanin_count, producer + )) { + SetFatal(state, stats, task_id); + return SharedWriterIntentResult::Failed; + } + } + if (!CommitSymbolSharedWriterIntentSet( + state->shared_map, args, task_id, + intent_fanin, intent_fanin_count, stats, + &state->fatal.value + )) { + SetFatal(state, stats, task_id); + return SharedWriterIntentResult::Failed; + } + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = + TaskTag(args, static_cast(index)); + if (!IsSharedWriterIntentTag(tag)) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + bool committed = false; + if (reference.kind == TensorRefKind::SharedOutputRef) { + // 全部 symbol history 与 latest CAS 已由上面的 batch 完成； + // 这里仅保留 ordinary 参数的原顺序提交。 + continue; + } else if (reference.kind == TensorRefKind::GmTensor) { + committed = CommitOrdinarySharedWriterIntent( + state->shared_map, *reference.pointer.gm_tensor, + task_id, static_cast(state->heap_window), + intent_fanin, intent_fanin_count, stats + ); + } else if (reference.kind == TensorRefKind::LocalTensor) { + committed = CommitOrdinarySharedWriterIntent( + state->shared_map, *reference.pointer.local_tensor, + task_id, static_cast(state->heap_window), + intent_fanin, intent_fanin_count, stats + ); + } + if (!committed) { + SetFatal(state, stats, task_id); + return SharedWriterIntentResult::Failed; + } + } + for (uint32_t edge = 0; edge < intent_fanin_count; ++edge) { + context.fanin[edge] = intent_fanin[edge]; + } + context.fanin_count = + static_cast(intent_fanin_count); + if (!PublishSharedWriterReady(state, task_id)) { + SetFatal(state, stats, task_id); + return SharedWriterIntentResult::Failed; + } + return SharedWriterIntentResult::Published; +} + +// 默认路径在本地执行状态建立后提交 INOUT writer；PA non-final UP 的 +// intent 路径允许在 fanin/registration 已验证、winner Build 前提交，以 +// 便 loser 构造下一组。FetchMax 返回旧 writer；默认实例要求精确等于 descriptor producer， +// 显式 ChainedWriter 实例按原 producer identity 选择链式 symbol，并要求 +// 其旧值精确等于调用方给出的前一 writer；其他 fresh symbol 仍匹配各自 +// producer。 +// 异常旧值即使被 FetchMax 推进也不回滚：该 RMW 已经线性化，多 symbol +// 提交不是事务，伪造负向 RMW 会抹掉故障现场。调用者随后广播 fatal， +// 整个调度不再继续消费该状态。 +template +PA_DEVICE bool CommitSharedFaninWriters( + PA_GM SharedTensorMapSidecar &map, const TaskArgs &args, + int32_t task_id, LocalStats &stats, + int32_t chained_producer_task_id = -1, + int32_t expected_shared_writer = -1 +) { + if (args.tensor_count < 0 || + args.tensor_count > static_cast(kMaxTaskTensors)) { + return false; + } + if constexpr (ChainedWriter) { + if (chained_producer_task_id < 0 || + chained_producer_task_id >= expected_shared_writer || + expected_shared_writer >= task_id) { + return false; + } + // 先验证 selector 确实命中至少一个合法 shared 写引用，再执行任何 + // FetchMax。调用参数错误不属于并发失败，不能留下半次 writer 推进。 + bool matched_chain_writer = false; + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = + TaskTag(args, static_cast(index)); + if (tag != TensorArgType::Inout && + tag != TensorArgType::OutputExisting) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind != TensorRefKind::SharedOutputRef) { + continue; + } + const FdwicOutputRef output_ref = + SharedOutputReference(reference); + if (!IsPlainSharedOutputRef(output_ref) || + output_ref.producer_task_id < 0 || + output_ref.producer_task_id >= task_id) { + return false; + } + matched_chain_writer |= + output_ref.producer_task_id == + chained_producer_task_id; + } + if (!matched_chain_writer) { + return false; + } + } + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = + TaskTag(args, static_cast(index)); + if (tag != TensorArgType::Inout && + tag != TensorArgType::OutputExisting) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind != TensorRefKind::SharedOutputRef) { + continue; + } + const FdwicOutputRef output_ref = + SharedOutputReference(reference); + if (!IsPlainSharedOutputRef(output_ref) || + output_ref.producer_task_id < 0 || + output_ref.producer_task_id >= task_id) { + return false; + } + const bool chained_ref = + ChainedWriter && + output_ref.producer_task_id == chained_producer_task_id; + const int32_t expected_writer = + chained_ref + ? expected_shared_writer + : output_ref.producer_task_id; + if (expected_writer < output_ref.producer_task_id || + expected_writer >= task_id) { + return false; + } + uint64_t retries = 0; + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 旧 PA writer-intent 路径只由隔离测试覆盖 + const int64_t observed = Ops::FetchMax( + &map.shared_outputs[ + static_cast(output_ref.producer_task_id) + ].last_writer[output_ref.output_slot].value, + static_cast(task_id), retries + ); + stats.result.cas_retries += retries; + if (observed != expected_writer) { + return false; + } + ++stats.result.shared_symbol_inout_commits; + } + return true; +} + +template +PA_DEVICE bool ValidatePaSharedWriterIntentShape( + PA_GM SchedulerState *state, const TaskArgs &args, + const SubmitContext &context, LocalStats &stats, + int32_t chained_producer_task_id = -1 +) { + const int32_t task_id = context.task_id; + if (state == nullptr || !context.won || task_id < 0 || + task_id >= static_cast(kMaxTasks) || + args.has_error || + args.tensor_count < 0 || + args.tensor_count > static_cast(kMaxTaskTensors) || + args.scalar_count < 0 || + args.scalar_count > static_cast(kMaxTaskScalars)) { + if (state != nullptr) { + SetFatal(state, stats, task_id); + } + return false; + } + + const int32_t accumulator_producer = + ChainedWriter + ? chained_producer_task_id + : task_id - 4; + bool accumulator_slots[3] = {false, false, false}; + uint32_t shared_writer_refs = 0; + uint32_t manual_dep_writer_refs = 0; + for (int32_t index = 0; index < args.tensor_count; ++index) { + const TensorArgType tag = + TaskTag(args, static_cast(index)); + if (tag != TensorArgType::Inout && + tag != TensorArgType::OutputExisting) { + continue; + } + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind == TensorRefKind::SharedOutputRef) { + const FdwicOutputRef output_ref = + SharedOutputReference(reference); + if (!IsPlainSharedOutputRef(output_ref) || + accumulator_producer < 0 || + output_ref.producer_task_id != accumulator_producer || + output_ref.output_slot < 0 || + output_ref.output_slot >= 3 || + accumulator_slots[ + static_cast(output_ref.output_slot) + ]) { + SetFatal(state, stats, task_id); + return false; + } + accumulator_slots[ + static_cast(output_ref.output_slot) + ] = true; + ++shared_writer_refs; + continue; + } + // PA UP 的真实参数还包含一个非 symbol、manual_dep 的 output + // view。这里不凭地址猜测具体 view 身份，但要求这类 writer 恰好 + // 一条；任何普通 region writer 或重复 manual-dependency writer + // 都不能借这个 PA 专用快路越过登记。 + bool manual_dep = false; + if (reference.kind == TensorRefKind::GmTensor && + reference.pointer.gm_tensor != nullptr) { + manual_dep = reference.pointer.gm_tensor->manual_dep; + } else if (reference.kind == TensorRefKind::LocalTensor && + reference.pointer.local_tensor != nullptr) { + manual_dep = reference.pointer.local_tensor->manual_dep; + } + if (!manual_dep) { + SetFatal(state, stats, task_id); + return false; + } + ++manual_dep_writer_refs; + } + // 这是 PA UP 专用快路，不是 ordinary-region writer 的通用替代品。 + // 三个 shared writer 必须正好对应 output/sum/max accumulator；缺失或 + // 多出任意一个都拒绝发布门，避免后继在 writer 状态不完整时前进。 + if (shared_writer_refs != 3 || + manual_dep_writer_refs != 1 || + !accumulator_slots[0] || + !accumulator_slots[1] || + !accumulator_slots[2]) { + SetFatal(state, stats, task_id); + return false; + } + return true; +} + +// 调用者已经完成只读 fanin 解析和 ordinary registration 校验后，登记 +// 三个 accumulator writer，再发布 deps_prepared。它位于 winner Build +// 之前；后续 Finish 必须复用 context.fanin，并在 Build 后跳过第二次 +// Commit。默认实例处理首组，ChainedWriter 处理中间组。 +template +PA_DEVICE bool CommitPaSharedWriterIntentAfterFanin( + PA_GM SchedulerState *state, const TaskArgs &args, + SubmitContext &context, LocalStats &stats, + int32_t chained_producer_task_id = -1, + int32_t expected_shared_writer = -1 +) { + const int32_t task_id = context.task_id; + if (!ValidatePaSharedWriterIntentShape( + state, args, context, stats, + chained_producer_task_id + )) { + return false; + } + if (!CommitSharedFaninWriters( + state->shared_map, args, task_id, stats, + chained_producer_task_id, expected_shared_writer + ) || + !PublishSharedWriterReady(state, task_id)) { + SetFatal(state, stats, task_id); + return false; + } + return true; +} + +// 隔离测试和无独立 Finish 阶段的调用点可一次完成 Collect + Commit + +// gate。真实 shared Finish 先自行 Collect/计入依赖签名，完成 registration +// 校验后只调用 CommitPaSharedWriterIntentAfterFanin，避免重复读 writer。 +template +PA_DEVICE bool PreparePaSharedWriterIntent( + PA_GM SchedulerState *state, const TaskArgs &args, + SubmitContext &context, LocalStats &stats, + int32_t chained_producer_task_id = -1, + int32_t expected_shared_writer = -1 +) { + const int32_t task_id = context.task_id; + if (!ValidatePaSharedWriterIntentShape( + state, args, context, stats, + chained_producer_task_id + )) { + return false; + } + + bool protocol_ok = false; + uint32_t ordinary_lookup_count = 0; + context.fanin_count = + static_cast(CollectSharedFanin( + state->shared_map, args, task_id, + static_cast(state->heap_window), stats, + context.fanin, protocol_ok, ordinary_lookup_count, + &state->fatal.value, chained_producer_task_id, + expected_shared_writer + )); + if (!protocol_ok || ordinary_lookup_count != 0) { + SetFatal(state, stats, task_id); + return false; + } + return CommitPaSharedWriterIntentAfterFanin( + state, args, context, stats, + chained_producer_task_id, expected_shared_writer + ); +} + +// fresh descriptor 的内容写入每 task 独占的 shared-output cell，并通过 +// FlushRegion 让 descriptor 与 writer 起点先于 published 可见。published +// 只表示后继可以读取 descriptor，不表示 producer 已 Build 或执行完成； +// kernel completion 仍由独立 completion flag 表达。 +template +PA_DEVICE_NOINLINE void RollbackSharedTaskOutputs( + PA_GM SharedOutputCell &cell, uint32_t output_count, + int32_t task_id = -1, LocalStats *stats = nullptr +) { + // 此入口只处理唯一 producer cell 出现非法竞争后的冷失败路径。先撤销发布位， + // 使任何非法越界 reader 都不能继续消费，再恢复 writer 与 descriptor + // 的未发布状态；正常 Submit 不执行这些额外 atomic/DCCI。 + for (uint32_t output = 0; output < output_count; ++output) { + (void)TraceOptionalAtomicExchange( + stats == nullptr ? nullptr : &stats->trace, + stats == nullptr ? nullptr : &stats->result, + task_id, AtomicSite::SharedOutputRollbackExchange, + &cell.published[output].value, static_cast(-1), + /*result_used=*/false + ); + } + for (uint32_t output = 0; output < output_count; ++output) { + (void)TraceOptionalAtomicExchange( + stats == nullptr ? nullptr : &stats->trace, + stats == nullptr ? nullptr : &stats->result, + task_id, AtomicSite::SharedOutputRollbackExchange, + &cell.last_writer[output].value, static_cast(-1), + /*result_used=*/false + ); + } + for (uint32_t output = 0; output < output_count; ++output) { + PA_GM volatile uint8_t *descriptor = + reinterpret_cast( + &cell.tensors[output] + ); + for (uint32_t byte = 0; byte < sizeof(TensorDesc); ++byte) { + descriptor[byte] = 0; + } + } + if (output_count != 0) { + (void)TraceConfiguredDcciFlush( + stats == nullptr ? nullptr : &stats->trace, + task_id, -1, DcciSite::SharedOutputRollbackFlush, + &cell.tensors[0], + static_cast(output_count) * sizeof(TensorDesc) + ); + } +} + +// 可选时间戳 out-param 只在正式 Materialize 泳道路径传入；单元测试与 +// 其它 helper 继续走默认空指针，不强制携带 LocalStats。 +template +PA_DEVICE bool PublishSharedTaskOutputs( + PA_GM SharedTensorMapSidecar &map, const SubmitContext &context, + uint32_t task_id, LocalStats *stats = nullptr, + uint64_t *copy_begin = nullptr, uint64_t *copy_end = nullptr, + uint64_t *flush_begin = nullptr, uint64_t *flush_end = nullptr +) { + if (task_id >= kMaxTasks || context.shared_result.TaskId() != static_cast(task_id) || + context.shared_result.Size() != context.result.count || + context.result.count > kSharedOutputMaxPerTask) { + return false; + } + PA_GM SharedOutputCell &cell = map.shared_outputs[task_id]; + // 先完整预检所有 slot；异常重复发布不能覆盖已对 consumer 可见的 + // descriptor，也不能让多输出 task 留下前半段控制字。 + for (uint32_t output = 0; output < context.result.count; ++output) { + PA_GM TensorDesc *source = context.result.tensors[output]; + // cell 由该 task 的唯一 Claim winner 独占；其他 task 只能在 + // published 就绪后读取，因此预检用普通 volatile GM load 即可。 + if (source == nullptr || + cell.published[output].value != -1 || + cell.last_writer[output].value != -1) { + return false; + } + } + // task-cell 唯一 winner 使预检到写入之间不存在合法竞争。仍用 + // FetchMax 预留全部 writer 控制字，并在异常旧值时撤回本次已预留项。 + for (uint32_t output = 0; output < context.result.count; ++output) { + uint64_t retries = 0; + const int64_t observed = + TraceConfiguredAtomicFetchMax( + stats == nullptr ? nullptr : &stats->trace, + stats == nullptr ? nullptr : &stats->result, + static_cast(task_id), + AtomicSite::SharedOutputWriterReserve, + &cell.last_writer[output].value, + static_cast(task_id), retries + ); + if (observed != -1) { + // atomicMax 在 observed( + stats == nullptr ? nullptr : &stats->trace, + stats == nullptr ? nullptr : &stats->result, + static_cast(task_id), + AtomicSite::SharedOutputRollbackExchange, + &cell.last_writer[output].value, observed, + /*result_used=*/false + ); + for (uint32_t previous = 0; previous < output; ++previous) { + (void)TraceConfiguredAtomicExchange< + Ops, ObserveAtomics + >( + stats == nullptr ? nullptr : &stats->trace, + stats == nullptr ? nullptr : &stats->result, + static_cast(task_id), + AtomicSite::SharedOutputRollbackExchange, + &cell.last_writer[previous].value, + static_cast(-1), + /*result_used=*/false + ); + } + return false; + } + } + // 把原先“每 slot copy 后立刻 flush”拆成两段整批动作，便于泳道单独 + // 展示 copy 与 flush；语义不变：全部 desc 写完后再统一 flush，再 + // barrier + published。零输出 task 也保留零时长边界，保证每个 + // winner 的 raw 子层数量固定。 +#if !PA_BUILD_TRACE_FREE + if (copy_begin != nullptr) { + *copy_begin = stats != nullptr + ? TraceTimestamp(stats->trace, stats->result) + : 0; + } +#else + (void)stats; + if (copy_begin != nullptr) { + *copy_begin = 0; + } +#endif + for (uint32_t output = 0; output < context.result.count; ++output) { + PA_GM TensorDesc *source = context.result.tensors[output]; + CopyGmTensor(cell.tensors[output], *source); + } +#if !PA_BUILD_TRACE_FREE + if (copy_end != nullptr || flush_begin != nullptr) { + const uint64_t boundary = stats != nullptr + ? TraceTimestamp(stats->trace, stats->result) + : 0; + if (copy_end != nullptr) { + *copy_end = boundary; + } + if (flush_begin != nullptr) { + *flush_begin = boundary; + } + } +#else + if (copy_end != nullptr) { + *copy_end = 0; + } + if (flush_begin != nullptr) { + *flush_begin = 0; + } +#endif + if (context.result.count != 0) { + const uint64_t known_begin = + flush_begin == nullptr ? 0 : *flush_begin; + const uint64_t dcci_end = + TraceConfiguredDcciFlush( + stats == nullptr ? nullptr : &stats->trace, + static_cast(task_id), -1, + DcciSite::SharedOutputDescriptorFlush, + &cell.tensors[0], + static_cast(context.result.count) * + sizeof(TensorDesc), + nullptr, known_begin + ); + if (flush_end != nullptr) { + *flush_end = dcci_end; + } + } +#if !PA_BUILD_TRACE_FREE + if (flush_end != nullptr && context.result.count == 0) { + *flush_end = stats != nullptr + ? TraceTimestamp(stats->trace, stats->result) + : 0; + } +#else + if (flush_end != nullptr) { + *flush_end = 0; + } +#endif + Ops::StoreBarrier(); + for (uint32_t output = 0; output < context.result.count; ++output) { + if (TraceConfiguredAtomicExchange( + stats == nullptr ? nullptr : &stats->trace, + stats == nullptr ? nullptr : &stats->result, + static_cast(task_id), + AtomicSite::SharedOutputPublishedExchange, + &cell.published[output].value, + static_cast(task_id), + /*result_used=*/true + ) != -1) { + RollbackSharedTaskOutputs( + cell, context.result.count, + static_cast(task_id), stats + ); + return false; + } + } + return true; +} + +template +PA_DEVICE bool PublishSharedWinnerAfterBuild( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, + const TaskArgs &args, const SubmitContext &context, + uint32_t task_id, TaskKind kind, LocalStats &stats, + bool writers_prepared = false, + bool chained_writer = false, + int32_t chained_producer_task_id = -1, + int32_t expected_shared_writer = -1 +) { + bool writers_committed = writers_prepared; + if (!writers_prepared) { + writers_committed = chained_writer + ? CommitSharedFaninWriters( + state->shared_map, args, + static_cast(task_id), stats, + chained_producer_task_id, + expected_shared_writer + ) + : CommitSharedFaninWriters( + state->shared_map, args, + static_cast(task_id), stats + ); + } + const bool outputs_published = + writers_committed && + PublishSharedTaskOutputs( + state->shared_map, context, task_id + ); + if (outputs_published) { + return true; + } + if (kind != TaskKind::Alloc) { + // BuildWinner 已经占用本 worker slot；封口失败后必须撤销， + // 防止错误路径进入 FinalDrain 并执行未完成 shared 封口的任务。 + (void)DiscardBuiltTask(worker, task_id); + } + // Alloc 的 CompleteTask 已经发布 ready flag，无法事务性撤回。该路径 + // 只可能来自 shared invariant 损坏；fatal 使整轮结果无效，不能局部 + // 回滚后继续调度。 + SetFatal(state, stats, static_cast(task_id)); + return false; +} +#endif + +// compete-first callback 跨 split finish 边界只传递这个固定 16B POD。 +// callback closure 与内部 thunk 都在 caller 中同步结束，绝不跨 TU 保存。 +struct CallbackSubmitTicket { + uint64_t submit_begin; + uint32_t task_id; + int16_t function_id; + uint8_t won; + uint8_t reserved; +}; +static_assert(sizeof(CallbackSubmitTicket) == 16, "callback ticket must remain a 16-byte POD"); +static_assert(offsetof(CallbackSubmitTicket, submit_begin) == 0, "callback ticket timestamp offset mismatch"); +static_assert(offsetof(CallbackSubmitTicket, task_id) == 8, "callback ticket task offset mismatch"); +static_assert(offsetof(CallbackSubmitTicket, function_id) == 12, "callback ticket function offset mismatch"); +static_assert(offsetof(CallbackSubmitTicket, won) == 14, "callback ticket winner offset mismatch"); + +#if PTO_FDWIC_SHARED_MAP && defined(PA_COMPETE_FIRST_SPLIT_FINISH) +constexpr uint64_t kSharedSplitTicketBindingPresent = 1ULL << 63U; + +PA_DEVICE uint64_t SharedSplitTicketBinding( + const CallbackSubmitTicket &ticket +) { + return kSharedSplitTicketBindingPresent | + (static_cast(ticket.task_id) << 8U) | + static_cast(ticket.reserved); +} + +PA_DEVICE bool ArmSharedSplitTicket( + CompeteFirstSplitRuntimeState &runtime, + const CallbackSubmitTicket &ticket +) { + // caller 在跨 TU 前保存由动态 plan 推导出的唯一 task/meta 身份； + // finish 必须逐字匹配并消费。成功 Submit 数同时充当 next task id， + // 因而重复、跳号和乱序 ticket 都不能进入 Finish body。 + if (runtime.reserved != 0 || + static_cast(ticket.task_id) != + runtime.stats.result.submits) { + return false; + } + runtime.reserved = SharedSplitTicketBinding(ticket); + return true; +} + +PA_DEVICE bool RecordSharedSplitReplayTask( + CompeteFirstSplitRuntimeState &runtime, + const CallbackSubmitTicket &ticket +) { + // task_id_sum 描述 caller 确实按 0..N-1 重放了完整前端序列， + // 与只有 winner 才跨 TU 的 finish_calls 是两条不同的协议证据。 + // loser 不再 Arm ticket，因此这一步必须留在 caller。 + if (runtime.reserved != 0 || + static_cast(ticket.task_id) != + runtime.stats.result.submits) { + return false; + } + runtime.task_id_sum += ticket.task_id; + return true; +} +#endif + +PA_DEVICE void BeginCallbackSubmit(PA_GM WorkerState &worker, SubmitContext &context) { + // Claim 必须先于 TaskArgs 构造，因此这里只建立与参数无关的 Submit 上下文； + // tensor/scalar 数量由 callback 完成后在 MaterializeTask 内写入。 + const uint32_t task_id = static_cast(worker.local_index++); + context.self = &worker; + context.payload = &worker.payloads[task_id & kPayloadMask]; + context.task_id = static_cast(task_id); + context.tensor_count = 0; + context.scalar_count = 0; + context.result.task_id = task_id; + context.result.count = 0; +#if PTO_FDWIC_SHARED_MAP + context.shared_result.Reset(static_cast(task_id)); +#endif + context.register_mask = 0; + context.output_bytes = 0; + context.fanin_count = 0; + context.kernel_id = -1; + context.won = false; + context.joint = false; + context.joint_init = false; + context.joint_block = -1; + context.joint_slot = -1; + context.joint_count = 0; +} + +#if PTO_FDWIC_SHARED_MAP +PA_DEVICE void BeginSharedCallbackSubmit( + PA_GM WorkerState &worker, SubmitContext &context +) { + // shared replay 的 96 个 actor 都要先取得同一个逻辑 task_id，但只有 + // Claim owner 会进入 Materialize/Build。loser 在 Claim 后只需要 + // task 身份与稳定 output symbol，因此这里不再为每个 replay actor + // 清零整份 408-byte SubmitContext 的 winner-only 字段。 + const uint32_t task_id = + static_cast(worker.local_index++); + context.task_id = static_cast(task_id); + context.shared_result.Reset(static_cast(task_id)); +} + +PA_DEVICE void PrepareSharedWinnerContext( + PA_GM WorkerState &worker, uint32_t task_id, + SubmitContext &context +) { + // 这些字段都只会被 shared winner 的 Materialize/Fanin/Build 消费。 + // tensor/scalar/register/output_bytes 由 MaterializeTask 在读取前覆盖； + // joint 字段属于 private BlockWon 路径，shared 单 lane PA 不读取。 + context.self = &worker; + context.payload = + &worker.payloads[task_id & kPayloadMask]; + context.result.task_id = task_id; + context.result.count = 0; + context.fanin_count = 0; +} +#endif + +#if defined(__CCE_AICORE__) || defined(__NPU_ARCH__) +#define PA_CALLBACK_LAMBDA_DEVICE __aicore__ +#else +#define PA_CALLBACK_LAMBDA_DEVICE +#endif + +template +PA_DEVICE bool BuildCallbackSubmitArgs( + PaOrchestrationState &orch, TaskArgs &args, uint32_t batch, LocalStats &stats +) { + CallbackSubmitArgsBuilder builder(args, Kind); + // 外层 callback 和所有参数 thunk 都只在这一调用点同步执行。调用者决定 + // 是否构参：private 仍全员 eager；shared 的五类 task 都只由 Claim + // owner 进入这里。 + auto callback = [&](CallbackSubmitArgsBuilder &out) PA_CALLBACK_LAMBDA_DEVICE { + out.Begin(); + if constexpr (Kind == TaskKind::Alloc) { + out.AddOutput([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.tile_create_info; + }); + out.AddOutput([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + out.AddOutput([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + } else if constexpr (Kind == TaskKind::Qk) { + out.AddLocalInput([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorDesc & { + MakeCallbackQueryView(orch, batch); + out.RecordView(); + return orch.query_view; + }); + out.AddLocalInput([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorDesc & { + return orch.key_cache; + }); + out.AddLocalInput([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorDesc & { + return orch.block_table; + }); + out.AddOutput([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorCreateInfo & { + const uint32_t score_shape[kMaxTensorDims] = { + kPaHeads, + static_cast(orch.current_nblocks * kPaBlockSize), + 0, 0, 0 + }; + InitCreateInfo(orch.qk_create_info, score_shape, 2, DataType::Float32); + out.RecordDynamicCreateInfo(); + return orch.qk_create_info; + }); + out.AddScalar([&]() PA_CALLBACK_LAMBDA_DEVICE -> uint64_t { + return orch.current_nblocks; + }); + out.AddScalar([&]() PA_CALLBACK_LAMBDA_DEVICE -> uint64_t { + return static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + + orch.current_block_offset; + }); + } else if constexpr (Kind == TaskKind::Sf) { + out.AddOutputHandleInput([&]() PA_CALLBACK_LAMBDA_DEVICE -> PaOutputHandle { + return orch.qk_scores; + }); + out.AddOutput([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorCreateInfo & { + const uint32_t probability_shape[kMaxTensorDims] = { + kPaHeads, + static_cast(orch.current_nblocks * kPaBlockSize), + 0, 0, 0 + }; + InitCreateInfo(orch.sf_create_info, probability_shape, 2, DataType::Bfloat16); + out.RecordDynamicCreateInfo(); + return orch.sf_create_info; + }); + out.AddOutput([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + out.AddOutput([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.scalar_create_info; + }); + out.AddScalar([&]() PA_CALLBACK_LAMBDA_DEVICE -> uint64_t { return orch.scale_bits; }); + out.AddScalar([&]() PA_CALLBACK_LAMBDA_DEVICE -> uint64_t { + return orch.current_nblocks; + }); + out.AddScalar([&]() PA_CALLBACK_LAMBDA_DEVICE -> uint64_t { + return orch.current_valid_len; + }); + } else if constexpr (Kind == TaskKind::Pv) { + out.AddOutputHandleInput([&]() PA_CALLBACK_LAMBDA_DEVICE -> PaOutputHandle { + return orch.sf_probs; + }); + out.AddLocalInput([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorDesc & { + return orch.value_cache; + }); + out.AddLocalInput([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorDesc & { + return orch.block_table; + }); + out.AddOutput([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorCreateInfo & { + return orch.tile_create_info; + }); + out.AddScalar([&]() PA_CALLBACK_LAMBDA_DEVICE -> uint64_t { + return orch.current_nblocks; + }); + out.AddScalar([&]() PA_CALLBACK_LAMBDA_DEVICE -> uint64_t { + return static_cast(orch.current_batch) * kPaMaxBlocksPerRequest + + orch.current_block_offset; + }); + } else { + static_assert(Kind == TaskKind::Up, "unsupported PA task kind"); + out.AddOutputHandleInput([&]() PA_CALLBACK_LAMBDA_DEVICE -> PaOutputHandle { + return orch.sf_max; + }); + out.AddOutputHandleInput([&]() PA_CALLBACK_LAMBDA_DEVICE -> PaOutputHandle { + return orch.sf_sum; + }); + out.AddOutputHandleInput([&]() PA_CALLBACK_LAMBDA_DEVICE -> PaOutputHandle { + return orch.pv_output; + }); + out.AddOutputHandleInout([&]() PA_CALLBACK_LAMBDA_DEVICE -> PaOutputHandle { + return orch.accumulated_max; + }); + out.AddOutputHandleInout([&]() PA_CALLBACK_LAMBDA_DEVICE -> PaOutputHandle { + return orch.accumulated_sum; + }); + out.AddOutputHandleInout([&]() PA_CALLBACK_LAMBDA_DEVICE -> PaOutputHandle { + return orch.accumulated_output; + }); + out.AddLocalInout([&]() PA_CALLBACK_LAMBDA_DEVICE -> const TensorDesc & { + MakeCallbackOutputView(orch, batch); + out.RecordView(); + return orch.output_view; + }); + out.AddScalar([&]() PA_CALLBACK_LAMBDA_DEVICE -> uint64_t { + return orch.current_block_offset == 0 ? 1 : 0; + }); + out.AddScalar([&]() PA_CALLBACK_LAMBDA_DEVICE -> uint64_t { + return orch.current_block_offset + orch.current_nblocks >= orch.current_blocks ? 1 : 0; + }); + } + }; + + callback(builder); + if (!builder.Valid()) return false; + const CallbackSubmitBuildCounts &counts = builder.Counts(); + stats.result.arg_resets += counts.reset_calls; + stats.result.views_created += counts.views_created; + stats.result.dynamic_create_infos += counts.dynamic_create_infos; + stats.result.tensor_args_added += counts.tensor_args_added; + stats.result.scalar_args_added += counts.scalar_args_added; + return true; +} + +#undef PA_CALLBACK_LAMBDA_DEVICE + +#if PTO_FDWIC_SHARED_MAP +template +PA_DEVICE bool CloseSharedCallbackSubmit( + PA_GM SchedulerState *state, LocalStats &stats, + const CallbackSubmitTicket &ticket, + const SharedPaTaskMeta &shared_task_meta +) { + const uint32_t task_id = ticket.task_id; + ++stats.result.submits; + + // shared 的总 task 数取决于每批 context_len。末次身份由调用者在 + // 既有 ticket bit 中显式携带，避免 96 个 worker 为了一个计时边界 + // 额外预扫整份 context_lens。 + const bool is_last_submit = shared_task_meta.is_last_submit; +#if PA_BUILD_PERF_CLOCK + // 与真实 FDWIC perf-clock 相同：只有末个逻辑 Submit 完成后才读取 + // 一次专用性能边界；loser 也必须闭合自己的全量重放窗口。 + const uint64_t submit_end = + is_last_submit ? Ops::PerfClockNow() : 0; +#elif PA_BUILD_SUBMIT_PMU + const uint64_t submit_end = is_last_submit ? Ops::Now() : 0; +#else + // 参考前端的 rt_submit_loser 仍是一次真实轻量 Submit；保留既有父 + // 区间用于覆盖稳定符号返回和轻量收尾；loser 不等待 TensorMap， + // 也不再写任何 winner-only child。 + const uint64_t submit_end = + TraceTimestamp(stats.trace, stats.result); +#endif + WriteSharedSubmitTrace( + stats.trace, stats.result, task_id, + ticket.submit_begin, submit_end + ); + if (!is_last_submit) { + return true; + } + if (stats.declared_task_count != 0) { + SetFatal( + state, stats, static_cast(task_id) + ); + return false; + } + stats.declared_task_count = task_id + 1U; + stats.result.submit_end = submit_end; + return true; +} + +template +PA_DEVICE bool FinishSharedLoserSubmit( + PA_GM SchedulerState *state, SubmitContext &context, + LocalStats &stats, const CallbackSubmitTicket &ticket +) { + SharedPaTaskMeta shared_task_meta{}; + const uint32_t task_id = ticket.task_id; + const bool valid = + ticket.won == 0 && + DecodeSharedPaTaskMeta( + ticket.reserved, task_id, shared_task_meta + ) && + SharedPaFunctionIdMatches( + shared_task_meta.kind, false, + static_cast(ticket.function_id) + ) && + context.task_id == static_cast(task_id) && + !context.won && + context.kernel_id == + static_cast(ticket.function_id) && + context.shared_result.TaskId() == + static_cast(task_id) && + context.shared_result.Size() == + FrontendTaskOutputCount(shared_task_meta.kind); + if (!valid) { + SetFatal( + state, stats, static_cast(task_id) + ); + return false; + } + + // loser 只完成本次 Submit 的轻量收尾。TensorMap 插入、前沿等待、 + // fanin lookup 与 Build 全部只属于 Claim owner；loser 不读取任何 + // TensorMap 控制字，也不再等待 writer-ready 门。 + return CloseSharedCallbackSubmit( + state, stats, ticket, shared_task_meta + ); +} + +#include "pa_shared_submit_path.h" +#endif + +template +PA_DEVICE bool FinishCallbackSubmitBody( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_count, + const TaskArgs &args, SubmitContext &context, LocalStats &stats, + PmuContext &pmu_context, const CallbackSubmitTicket &ticket +) { +#if PTO_FDWIC_SHARED_MAP + (void)task_count; + return FinishSharedWinnerSubmitBody( + state, worker, args, context, stats, pmu_context, ticket + ); +#else + const uint32_t task_id = ticket.task_id; +#if PTO_FDWIC_SHARED_MAP + SharedPaTaskMeta shared_task_meta{}; + if (!DecodeSharedPaTaskMeta( + ticket.reserved, task_id, shared_task_meta + )) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + const TaskKind kind = shared_task_meta.kind; +#else + const TaskKind kind = GetTaskKind(task_id); +#endif + const int32_t function_id = static_cast(ticket.function_id); + const bool winner = ticket.won != 0; +#if PTO_FDWIC_SHARED_MAP + if (!SharedPaFunctionIdMatches(kind, winner, function_id)) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + // shared loser 必须在 caller 的轻路径返回；跨 TU / 完整 Finish 只允许 + // winner 进入。这样 Materialize、Fanin、Register 与 Build 的边界才与 + // 实际 shared TensorMap 协议一致。 + if (!winner) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } +#endif + +#if PTO_FDWIC_SHARED_MAP + // callback 已经返回；只有 Claim winner 才把 CreateInfo 物化为 descriptor + // 并预留 shared heap。loser 已在 caller 轻路径返回，这里只处理 winner。 + // PA Case1 的普通 region 恒为空，winner 不再等待全局 exact turn； + // 跨 task 顺序只由实际消费的 (producer,slot).published 建立。 + // 删除 exact-turn 不能连带删除它成功出口的终止态检查：若其他核已经 + // 广播 fatal，本 winner 不得继续预留 heap、构建 slot 或发布 symbol。 + // 这里直接使用 Ops，不扩张 atomic 泳道记录，也不恢复任何全局前沿。 + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 该分支位于 private 外层中的不可达 shared 旧实现 + if (Ops::Load(&state->fatal.value) != 0) { + return false; + } + const uint64_t materialize_begin = + TraceTimestamp(stats.trace, stats.result); + BeginSubmitPmuPhase(pmu_context); + const bool materialized = MaterializeTask( + worker, task_id, args, context, state->shared_map, + state->heap_base, state->heap_size, + kind, shared_task_meta.batch_start, + shared_task_meta.group_index, + &stats.trace, &stats.result + ); + if (materialized) { + stats.result.materialized_outputs += context.result.count; + } + if (!materialized) { + EndSubmitPmuPhase(pmu_context); + SetFatal(state, stats, static_cast(task_id)); + return false; + } + // Materialize 只负责 shared heap reserve 与 descriptor 构造。fresh + // symbol 必须等本任务 CompleteTask/BuildWinner 成功后再封口，因此 + // 这里不能提前写 published。 + EndSubmitPmuPhase(pmu_context); + const uint64_t materialize_end = + TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Materialize, ProfilePhase::Materialize, + materialize_begin, materialize_end, 0, + kind == TaskKind::Alloc ? 1U : 0U + ); + // shared PA Case1 没有 ordinary-region PrepareMap；不再为兼容旧矩形 + // 泳道写零时长 marker。host/analyzer 直接校验 shared 稀疏真实边界。 +#else + // private 模式保持 S3.1 的 eager Materialize 与每核 heap 路径不变。 + const uint64_t materialize_begin = + TraceTimestamp(stats.trace, stats.result); + BeginSubmitPmuPhase(pmu_context); + const bool materialized = + MaterializeTask( + worker, task_id, args, context, + state->heap_base, state->heap_size + ); + if (!materialized) { + EndSubmitPmuPhase(pmu_context); + SetFatal(state, stats, static_cast(task_id)); + return false; + } + stats.result.materialized_outputs += context.result.count; + EndSubmitPmuPhase(pmu_context); + const uint64_t materialize_end = + TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Materialize, ProfilePhase::Materialize, + materialize_begin, materialize_end, 0, + kind == TaskKind::Alloc ? 1U : 0U + ); + + const uint64_t prepare_begin = materialize_end; + AdvanceTensorMap(worker.map, task_id, static_cast(state->heap_window)); + const uint64_t prepare_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::PrepareMap, ProfilePhase::PrepareMap, + prepare_begin, prepare_end, 0, kind == TaskKind::Alloc ? 1U : 0U + ); +#endif + +#if PTO_FDWIC_SHARED_MAP + bool shared_writers_prepared = false; +#endif + uint64_t register_begin = +#if PTO_FDWIC_SHARED_MAP + materialize_end; +#else + prepare_end; +#endif +#if PTO_FDWIC_SHARED_MAP + if (kind != TaskKind::Alloc) { +#else + if (kind != TaskKind::Alloc && + __builtin_expect(winner, 0)) { +#endif + const uint64_t fanin_begin = register_begin; +#if PTO_FDWIC_SHARED_MAP + bool lookup_protocol_ok = false; + uint32_t ordinary_lookup_count = 0; + if (shared_task_meta.chained_writer) { + context.fanin_count = static_cast( + CollectSharedFanin( + state->shared_map, args, + static_cast(task_id), + static_cast(state->heap_window), stats, + context.fanin, lookup_protocol_ok, + ordinary_lookup_count, &state->fatal.value, + static_cast( + shared_task_meta.batch_start + ), + static_cast(task_id) - 4 + ) + ); + } else { + context.fanin_count = static_cast( + CollectSharedFanin( + state->shared_map, args, + static_cast(task_id), + static_cast(state->heap_window), stats, + context.fanin, lookup_protocol_ok, + ordinary_lookup_count, &state->fatal.value + ) + ); + } + if (!lookup_protocol_ok) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + stats.result.map_lookups += ordinary_lookup_count; +#else + context.fanin_count = static_cast(CollectFanin(worker.map, args, context.fanin)); + stats.result.map_lookups += static_cast(args.tensor_count) - context.result.count; +#endif + for (int32_t edge = 0; edge < context.fanin_count; ++edge) { + stats.result.dependency_signature ^= + DependencyEdgeSignature( + task_id, + static_cast(context.fanin[edge]) + ); + } + const uint64_t fanin_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Fanin, ProfilePhase::Fanin, + fanin_begin, fanin_end, 0, static_cast(context.fanin_count) + ); + register_begin = fanin_end; + } + + BeginSubmitPmuPhase(pmu_context); +#if PTO_FDWIC_SHARED_MAP + // 当前 standalone 只模拟 PA Case1：fresh symbol 直接寻址， + // output_view 又是 manual_dep，ordinary region 必须严格为空。 + // 这里只读验证，不构造空 delta，也不触碰 region sequencer。 + const bool registered = + ValidateEmptySharedRegistration(args, context); +#else + const bool registered = RegisterOutputs(context, args, kind != TaskKind::Alloc); + if (registered && kind != TaskKind::Alloc) { + stats.result.map_inserts += CountBits(context.register_mask); + } +#endif + EndSubmitPmuPhase(pmu_context); + const uint64_t register_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Register, ProfilePhase::Register, + register_begin, register_end, 0, kind == TaskKind::Alloc ? 0U : 1U + ); + if (!registered) { +#if PTO_FDWIC_SHARED_MAP + // PA Case1 不接入 ordinary-region backend。非 manual-dep 的普通 + // writer 或非法 register mask 会在 region append 前失败并广播 + // fatal；此前 Materialize 和 fanin 仍可能读取 shared sidecar。 +#else + // 固定桶容量不足时，InsertTensor 没有覆写任何 live 槽。沿用现有 + // fatal 广播终止所有 worker，禁止像旧 linked map 一样静默漏登记 + // hazard、随后带着不完整 fanin 继续执行。 +#endif + SetFatal(state, stats, static_cast(task_id)); + return false; + } + +#if PTO_FDWIC_SHARED_MAP + if (shared_task_meta.has_following_group) { + // PA 的 non-final UP 固定有 SF/PV/accumulator 三条 fanin。先完成 + // registration，再登记 writer intent并放行 loser；Build 后只做 + // fresh-output 封口，绝不能重复 Collect/Commit。 + if (kind != TaskKind::Up || context.fanin_count != 3) { + SetFatal( + state, stats, static_cast(task_id) + ); + return false; + } + const bool prepared = shared_task_meta.chained_writer + ? CommitPaSharedWriterIntentAfterFanin( + state, args, context, stats, + static_cast( + shared_task_meta.batch_start + ), + static_cast(task_id) - 4 + ) + : CommitPaSharedWriterIntentAfterFanin( + state, args, context, stats + ); + if (!prepared) { + return false; + } + shared_writers_prepared = true; + } +#endif + +#if PTO_FDWIC_SHARED_MAP + { +#else + if (__builtin_expect(winner, 0)) { +#endif + const uint64_t winner_build_begin = register_end; +#if PTO_FDWIC_SHARED_MAP && \ + defined(PA_TEST_SHARED_POST_GATE_BUILD_FAILURE) + // 只供 host 96-worker 故障门槛使用：non-final UP 已完成 writer + // intent 与 deps_prepared 发布后、建立可执行 slot 前注入失败。 + // 普通 CPU/CCEC 不定义该宏，预处理后不保留调用或分支。 + if (shared_writers_prepared && + Ops::InjectSharedPostGateBuildFailure( + state, worker, task_id, kind + )) { + SetFatal( + state, stats, static_cast(task_id) + ); + return false; + } +#endif + if (kind == TaskKind::Alloc) { +#if !PTO_FDWIC_SHARED_MAP + if (!HeapGuard(state, worker, task_id, context.output_bytes, stats)) { + return false; + } +#endif + CompleteTask(state, worker, task_id, stats); + } else { + if (!BuildWinner( + state, worker, task_id, kind, args, context, context.fanin, + static_cast(context.fanin_count), stats + )) { + return false; + } + } + // 先建立可执行状态。普通/final task 随后提交本任务的 INOUT + // writer；non-final UP 已在 Build 前登记 writer intent，这里只 + // 跳过重复 Commit。fresh outputs 最后封口；后继只等待自己实际 + // 依赖的 published cell，不再经过全局 committed_tasks。 + // published 成功之后只剩观察记录与 Submit 收尾。 +#if PTO_FDWIC_SHARED_MAP + if (!PublishSharedWinnerAfterBuild( + state, worker, args, context, task_id, kind, stats, + shared_writers_prepared, + shared_task_meta.chained_writer, + static_cast( + shared_task_meta.batch_start + ), + static_cast(task_id) - 4 + )) { + return false; + } +#endif + const uint64_t winner_build_end = TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + kind == TaskKind::Alloc ? TracePhase::AllocComplete : TracePhase::WinnerBuild, + ProfilePhase::ReplayTail, winner_build_begin, winner_build_end + ); + } + +#if PTO_FDWIC_SHARED_MAP + (void)task_count; + return CloseSharedCallbackSubmit( + state, stats, ticket, shared_task_meta + ); +#else + ++stats.result.submits; +#if PA_BUILD_PERF_CLOCK + // 与真实 FDWIC perf-clock 相同：只有末个 Submit 完成全部尾动作后 + // 才采一次专用性能边界。协议 watchdog 继续使用 Ops::Now()，两者 + // 在源码上保持可审计的不同接口。 + const uint64_t submit_end = + task_id + 1 == task_count ? Ops::PerfClockNow() : 0; +#elif PA_BUILD_SUBMIT_PMU + const uint64_t submit_end = task_id + 1 == task_count ? Ops::Now() : 0; +#else + const uint64_t submit_end = TraceTimestamp(stats.trace, stats.result); +#endif + WriteTrace( + stats.trace, stats.result, static_cast(task_id), function_id, + TracePhase::Submit, ProfilePhase::Submit, + ticket.submit_begin, submit_end, winner ? 1U : 0U, kind == TaskKind::Alloc ? 1U : 0U + ); + if (task_id + 1 == task_count) stats.result.submit_end = submit_end; + return true; +#endif +#endif +} + +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) +template +PA_DEVICE uint32_t FinishSplitCallbackSubmitFromRuntime( + const CallbackSubmitTicket *ticket, const TaskArgs *args +) { + CompeteFirstSplitRuntimeState &runtime = Ops::CompeteFirstSplitState(); + const uint64_t state_address = reinterpret_cast(&runtime); + runtime.finish_state_address = state_address; +#if PTO_FDWIC_SHARED_MAP + SharedPaTaskMeta ticket_meta{}; + const bool expected_ticket_bound = + ticket != nullptr && + runtime.reserved == SharedSplitTicketBinding(*ticket); + // binding 是一次性的 caller→finish 交接状态。无论 ticket 是否 + // 合法都在读取后清零，使 fatal 路也能收敛，并让最终协议检查继续 + // 要求 runtime.reserved==0。 + runtime.reserved = 0; + bool valid = + ticket != nullptr && args != nullptr && + runtime.scheduler != nullptr && + runtime.worker != nullptr && runtime.task_count != 0 && + runtime.worker_id < kWorkers && + runtime.owner_worker_id == runtime.worker_id && + runtime.worker->core_idx == + static_cast(runtime.worker_id) && + runtime.caller_state_address == state_address && + runtime.state_cookie == CompeteFirstSplitStateCookie( + runtime.worker_id, runtime.worker->role + ) && + expected_ticket_bound && ticket->won != 0 && + static_cast(ticket->task_id) == + runtime.stats.result.submits; +#else + bool valid = ticket != nullptr && args != nullptr && runtime.scheduler != nullptr && + runtime.worker != nullptr && runtime.task_count != 0 && + runtime.worker_id < kWorkers && runtime.owner_worker_id == runtime.worker_id && + runtime.worker->core_idx == static_cast(runtime.worker_id) && + runtime.caller_state_address == state_address && + runtime.state_cookie == CompeteFirstSplitStateCookie( + runtime.worker_id, runtime.worker->role + ) && runtime.reserved == 0; +#endif + if (valid) { +#if PTO_FDWIC_SHARED_MAP + valid = ticket->task_id < runtime.task_count && + runtime.context.task_id == static_cast(ticket->task_id) && + runtime.context.kernel_id == static_cast(ticket->function_id) && + runtime.context.won == (ticket->won != 0); + if (valid) { + valid = + DecodeSharedPaTaskMeta( + ticket->reserved, ticket->task_id, ticket_meta + ) && + SharedPaFunctionIdMatches( + ticket_meta.kind, ticket->won != 0, + static_cast(ticket->function_id) + ) && + runtime.context.shared_result.TaskId() == + static_cast(ticket->task_id) && + runtime.context.shared_result.Size() == + FrontendTaskOutputCount(ticket_meta.kind); + } +#else + valid = ticket->reserved == 0 && ticket->task_id < runtime.task_count && + runtime.context.task_id == static_cast(ticket->task_id) && + runtime.context.kernel_id == static_cast(ticket->function_id) && + runtime.context.won == (ticket->won != 0); +#endif + } + ++runtime.finish_calls; +#if !PTO_FDWIC_SHARED_MAP + if (ticket != nullptr) runtime.task_id_sum += ticket->task_id; +#endif + if (!valid) { + ++runtime.protocol_errors; + if (runtime.scheduler != nullptr) { + SetFatal( + runtime.scheduler, runtime.stats, + ticket == nullptr ? -1 : static_cast(ticket->task_id) + ); + } + return 0; + } + +#if PA_BUILD_SUBMIT_PMU + // none 不需要 finish 内的局部 PMU context；Claim/EfDrain 的起止点都在 + // callback 与 finish 之前，因此也能保持 split 形状。Materialize/Register + // 边界在 finish 内，仍由构建脚本选择 inline finish 以复用同一 PmuContext。 + static_assert( + kCompiledSubmitPmuPhase == SubmitPmuPhase::None || + kCompiledSubmitPmuPhase == SubmitPmuPhase::Claim || + kCompiledSubmitPmuPhase == SubmitPmuPhase::EfDrain, + "split callback submit-PMU supports none/claim/efdrain only" + ); +#endif + bool pmu_context = false; + return FinishCallbackSubmitBody( + runtime.scheduler, *runtime.worker, runtime.task_count, *args, + runtime.context, runtime.stats, pmu_context, *ticket + ) ? 1U : 0U; +} +#endif + +template +PA_DEVICE bool SubmitCallbackTask( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, uint32_t task_count, + PaOrchestrationState &orch, TaskArgs &args, uint32_t batch, + SubmitContext &context, LocalStats &stats, PmuContext &pmu_context +#if PTO_FDWIC_SHARED_MAP + , const SharedPaBatchPlan &shared_batch_plan, + uint32_t shared_task_offset +#endif +) { +#if PTO_FDWIC_SHARED_MAP + BeginSharedCallbackSubmit(worker, context); +#else + BeginCallbackSubmit(worker, context); +#endif + const uint32_t task_id = static_cast(context.task_id); +#if PTO_FDWIC_SHARED_MAP + SharedPaPlannedTask shared_planned_task{}; + if (batch >= state->config.batches || + !SharedPaPlannedTaskAt( + shared_batch_plan, shared_task_offset, + shared_planned_task + ) || + shared_planned_task.kind != Kind || + task_id != + shared_batch_plan.batch_start + + shared_task_offset) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + const bool shared_is_last_submit = + shared_planned_task.is_last_in_batch && + batch + 1U == state->config.batches; + const uint8_t shared_task_meta = EncodeSharedPaTaskMeta( + Kind, shared_planned_task.group_index, + shared_planned_task.has_following_group, + shared_is_last_submit + ); + if (shared_task_meta == 0) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } +#endif +#if PA_BUILD_PERF_CLOCK + // dist_submit_begin/BeginCallbackSubmit 已建立本次 task_id；首个 + // Submit 的 EfDrain 前只读一次性能时钟，不为其余 Submit 递增 + // 另一份观察计数。 + const uint64_t submit_begin = task_id == 0 ? Ops::PerfClockNow() : 0; +#elif PA_BUILD_SUBMIT_PMU + const uint64_t submit_begin = task_id == 0 ? Ops::Now() : 0; +#else + const uint64_t submit_begin = TraceTimestamp(stats.trace, stats.result); +#endif + if (task_id == 0) stats.result.submit_begin = submit_begin; + + const uint64_t efdrain_begin = submit_begin; + BeginSubmitPmuPhase(pmu_context); + DrainReady(state, worker, DrainPlace::EfDrain, stats); + EndSubmitPmuPhase(pmu_context); + const uint64_t efdrain_end = TraceTimestamp(stats.trace, stats.result); +#if PTO_FDWIC_SHARED_MAP + // shared 的 EfDrain 边界与现有父子记录严格重合： + // begin = Submit.start，end = Claim.start。 + // 设备不再为它单写一条 32B raw；converter/analyzer 用这两个既有 + // 端点精确还原泳道。Profile 聚合与 submit-pmu 的原始边界保持不变。 + AccumulatePhase( + stats.result, ProfilePhase::EfDrain, + efdrain_begin, efdrain_end + ); +#else + WriteTrace( + stats.trace, stats.result, static_cast(task_id), -1, + TracePhase::EfDrain, ProfilePhase::EfDrain, efdrain_begin, efdrain_end + ); +#endif + + const uint64_t claim_begin = efdrain_end; + BeginSubmitPmuPhase(pmu_context); + const ClaimOutcome claim = + Claim(state, worker, task_id, Kind, stats); + context.won = claim.won; + context.kernel_id = claim.function_id; + RecordClaimOutcome(stats, Kind, claim); + EndSubmitPmuPhase(pmu_context); + const uint64_t claim_end = TraceTimestamp(stats.trace, stats.result); +#if PTO_FDWIC_SHARED_MAP + WriteSharedClaimTrace( + stats.trace, stats.result, task_id, + claim_begin, claim_end, claim.won + ); +#else + WriteTrace( + stats.trace, stats.result, static_cast(task_id), + claim.function_id, TracePhase::Claim, + ProfilePhase::Claim, claim_begin, claim_end, + (claim.won ? kClaimWon : 0U) | + (claim.attempted ? kClaimAttempted : 0U), + Kind == TaskKind::Alloc ? 1U : 0U + ); +#endif + +#if PTO_FDWIC_SHARED_MAP + // fresh Output 的返回值是 task/slot 符号，不依赖哪个 worker 获胜。 + // 在跨 TU finish 前为所有 replay actor 建立同一句柄集，保证 loser + // 返回后也能继续构造本核后续 task 的输入引用。 + if (!PrepareSharedTaskOutputs( + context.shared_result, static_cast(task_id), Kind + )) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } +#endif +#if PTO_FDWIC_SHARED_MAP + if (__builtin_expect(claim.won, 0)) { + // shared loser 已在上方声明稳定 output symbol；它不需要构造本 task + // 的 descriptor/scalar 参数，Alloc 也不例外。finish 的 loser + // 分支只闭合边界，不读这里留下的上一 task args。 + PrepareSharedWinnerContext( + worker, task_id, context + ); + if (!BuildCallbackSubmitArgs(orch, args, batch, stats)) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + } +#else + // private 保持所有 worker 对五个 task 的 eager 构参语义。 + if (!BuildCallbackSubmitArgs(orch, args, batch, stats)) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } +#endif + const CallbackSubmitTicket ticket{ + submit_begin, + task_id, + static_cast(claim.function_id), + static_cast(claim.won ? 1 : 0), +#if PTO_FDWIC_SHARED_MAP + shared_task_meta, +#else + 0, +#endif + }; +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) +#if PTO_FDWIC_SHARED_MAP + CompeteFirstSplitRuntimeState &split_runtime = + Ops::CompeteFirstSplitState(); + if (!RecordSharedSplitReplayTask(split_runtime, ticket)) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + if (!claim.won) { + return FinishSharedLoserSubmit( + state, context, stats, ticket + ); + } + if (!ArmSharedSplitTicket(split_runtime, ticket)) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } +#endif + (void)state; + (void)worker; + (void)task_count; + (void)context; + (void)stats; + (void)pmu_context; + return Ops::FinishCompeteFirstCallback(&ticket, &args); +#else +#if PTO_FDWIC_SHARED_MAP + if (!claim.won) { + return FinishSharedLoserSubmit( + state, context, stats, ticket + ); + } +#endif + return FinishCallbackSubmitBody( + state, worker, task_count, args, context, stats, pmu_context, ticket + ); +#endif +} + +PA_DEVICE uint32_t CountLiveMapEntries(PA_GM const TensorMap &map) { + // AdvanceTensorMap 按 producer 精确维护 logical live_count；各桶 head + // 允许惰性落后，不能再通过遍历物理槽推导逻辑存活数。 + return map.live_count; +} + +#if PTO_FDWIC_SHARED_MAP +template +PA_DEVICE uint32_t FinalizeSharedReplayTaskCount( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, + LocalStats &stats +) { + uint32_t task_count = 0; + if (worker.local_index < 0 || + static_cast(worker.local_index) > kMaxTasks) { + SetFatal(state, stats, worker.local_index); + } else { + task_count = static_cast(worker.local_index); + } + if (stats.declared_task_count != task_count) { + SetFatal( + state, stats, static_cast(task_count) + ); + } + return task_count; +} +#endif + +PA_DEVICE void LogicalTensorMapHashWord( + uint64_t &hash, uint64_t value +) { + for (uint32_t byte = 0; byte < 8; ++byte) { + hash ^= (value >> (byte * 8U)) & 0xFFU; + hash *= 1099511628211ULL; + } +} + +PA_DEVICE uint64_t PrivateLogicalTensorMapSignature( + PA_GM const TensorMap &map +) { + uint64_t hash = 1469598103934665603ULL; + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + const uint64_t head = TensorMapBucketHead(map, bucket); + const uint64_t tail = TensorMapBucketTail(map, bucket); + for (uint64_t cursor = head; cursor < tail; ++cursor) { + PA_GM const MapEntry &entry = + map.entries[TensorMapSlotIndex(bucket, cursor)]; + if (entry.producer < map.alive_floor) { + continue; + } + LogicalTensorMapHashWord(hash, bucket); + LogicalTensorMapHashWord(hash, entry.buffer_addr); + LogicalTensorMapHashWord(hash, entry.lo); + LogicalTensorMapHashWord(hash, entry.hi); + LogicalTensorMapHashWord( + hash, static_cast(entry.producer) + ); + } + } + return hash; +} + +template +PA_DEVICE void PublishResult(PA_GM WorkerResult &destination, const WorkerResult &source) { + // 每个 worker 只写自己独占、覆盖多条 cache line 的 WorkerResult 分区；逐字段 + // bypass 保证结果对 host 可见，而独立 sidecar 允许 D2H 只搬结果、不搬约 9 MiB WorkerState。 +#define PA_PUBLISH_FIELD(field) Ops::Publish(&destination.field, source.field) + PA_PUBLISH_FIELD(submit_begin); + PA_PUBLISH_FIELD(submit_end); + PA_PUBLISH_FIELD(finish_cycle); + PA_PUBLISH_FIELD(checksum); + PA_PUBLISH_FIELD(submits); + PA_PUBLISH_FIELD(claim_attempts); + PA_PUBLISH_FIELD(claim_wins); + PA_PUBLISH_FIELD(heap_guards); + PA_PUBLISH_FIELD(fanin_ready_loads); + PA_PUBLISH_FIELD(completion_duplicates); + PA_PUBLISH_FIELD(cas_retries); + PA_PUBLISH_FIELD(joint_polls); + for (uint32_t index = 0; index < static_cast(TaskKind::Count); ++index) { + Ops::Publish(&destination.wins[index], source.wins[index]); + } + for (uint32_t index = 0; index < 4; ++index) { + Ops::Publish(&destination.kernel_counts[index], source.kernel_counts[index]); + Ops::Publish(&destination.kernel_cycles[index], source.kernel_cycles[index]); + Ops::Publish(&destination.kernel_min_cycles[index], source.kernel_min_cycles[index]); + Ops::Publish(&destination.kernel_max_cycles[index], source.kernel_max_cycles[index]); + } + for (uint32_t index = 0; index < static_cast(DrainPlace::Count); ++index) { + Ops::Publish(&destination.placement[index], source.placement[index]); + } + for (uint32_t index = 0; index < static_cast(ProfilePhase::Count); ++index) { + Ops::Publish(&destination.phase_cycles[index], source.phase_cycles[index]); + Ops::Publish(&destination.phase_calls[index], source.phase_calls[index]); + } + for (uint32_t index = 0; index < 2; ++index) { + Ops::Publish(&destination.wait_events[index], source.wait_events[index]); + Ops::Publish(&destination.wait_iterations[index], source.wait_iterations[index]); + } + PA_PUBLISH_FIELD(context_reads); + PA_PUBLISH_FIELD(views_created); + PA_PUBLISH_FIELD(dynamic_create_infos); + PA_PUBLISH_FIELD(arg_resets); + PA_PUBLISH_FIELD(tensor_args_added); + PA_PUBLISH_FIELD(scalar_args_added); + PA_PUBLISH_FIELD(materialized_outputs); + PA_PUBLISH_FIELD(map_inserts); + PA_PUBLISH_FIELD(map_lookups); + PA_PUBLISH_FIELD(slot_tensor_copies); + PA_PUBLISH_FIELD(slot_scalar_copies); + PA_PUBLISH_FIELD(fanin_edges); + PA_PUBLISH_FIELD(final_heap_next); + PA_PUBLISH_FIELD(map_high_water); + PA_PUBLISH_FIELD(map_alive_floor); + PA_PUBLISH_FIELD(map_cleaned_upto); + PA_PUBLISH_FIELD(map_live_entries); + PA_PUBLISH_FIELD(worker_id); + PA_PUBLISH_FIELD(role); + PA_PUBLISH_FIELD(max_occupied); + PA_PUBLISH_FIELD(final_occupied); + PA_PUBLISH_FIELD(fanin_not_ready_loads); + PA_PUBLISH_FIELD(frontier_initial_loads); + PA_PUBLISH_FIELD(frontier_updates); + PA_PUBLISH_FIELD(frontier_terminal_loads); + PA_PUBLISH_FIELD(atomic_trace_calls); + PA_PUBLISH_FIELD(startup_barrier_begin); + PA_PUBLISH_FIELD(startup_barrier_end); + PA_PUBLISH_FIELD(final_barrier_begin); + PA_PUBLISH_FIELD(final_barrier_release); + PA_PUBLISH_FIELD(final_barrier_end); + PA_PUBLISH_FIELD(dependency_signature); + PA_PUBLISH_FIELD(shared_symbol_input_loads); + PA_PUBLISH_FIELD(shared_symbol_inout_commits); +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) + PA_PUBLISH_FIELD(compete_first_split_caller_state_address); + PA_PUBLISH_FIELD(compete_first_split_finish_state_address); + PA_PUBLISH_FIELD(compete_first_split_finish_calls); + PA_PUBLISH_FIELD(compete_first_split_protocol_errors); + PA_PUBLISH_FIELD(compete_first_split_state_cookie); + PA_PUBLISH_FIELD(compete_first_split_task_id_sum); + PA_PUBLISH_FIELD(compete_first_split_owner_worker_id); + PA_PUBLISH_FIELD(compete_first_split_reserved); +#endif +#undef PA_PUBLISH_FIELD + Ops::StoreBarrier(); +} + +template +PA_DEVICE void RunSchedulerImpl(PA_GM SchedulerState *state, uint32_t worker_id, CoreRole role) { + // 一个入口实例只拥有 state->workers[worker_id] 的私有 map/ring/payload；cursor、task cell 和屏障为跨核共享区。 + if (worker_id >= kWorkers) { + return; + } + // RunConfig、PMU 配置与 winner workload 连续占据三条独立 cache line。 + // 在解释任何可能随 TensorMap 模式变化的 WorkerState 之前，先失效 host + // 写入的控制区并核对稳定构建身份；混合 host/kernel 会置 fatal 后退出， + // 不允许继续用错误 sizeof 或模式解释 GM。 + constexpr uint64_t kStartupConfigBytes = + sizeof(state->config) + sizeof(state->pmu_probe) + + sizeof(state->winner_workload); + uint64_t startup_dcci_begin = 0; + uint64_t startup_dcci_end = 0; + CapturePreAttachDcciInvalidate( + &state->config, + kStartupConfigBytes, + startup_dcci_begin, startup_dcci_end + ); + const bool build_identity_matches = + state->config.build_identity_magic == kBuildIdentityMagic && + state->config.build_identity_abi_version == kBuildIdentityAbiVersion && + state->config.tensor_map_mode == static_cast(kCompiledTensorMapMode) && + state->config.scheduler_state_size == static_cast(sizeof(SchedulerState)) && + state->pmu_probe.build_variant == kCompiledBuildVariant; + if (!build_identity_matches) { + (void)PreAttachAtomicExchange( + &state->fatal.value, static_cast(1) + ); + return; + } + PA_GM WorkerState &worker = state->workers[worker_id]; + worker.role = role; + worker.core_idx = static_cast(worker_id); + // standalone 使用连续 worker 编号：AIC 为 0..31；AIV 为 32..95。 + // 每个物理 block b 对应 AIC(b, lane0)、AIV(32+2b, lane1)、AIV(33+2b, lane2)。 + if (role == CoreRole::Aic) { + worker.block_id = static_cast(worker_id); + worker.lane = 0; + } else { + const uint32_t vector_id = worker_id - kAicWorkers; + worker.block_id = static_cast(vector_id / 2); + worker.lane = static_cast(1 + vector_id % 2); + } + worker.sub_block_id = worker.lane == 2 ? 1 : 0; + worker.local_index = 0; + worker.heap_next = 0; +#if !PTO_FDWIC_SHARED_MAP + ResetTensorMap(worker.map); +#endif + worker.occupied_count = 0; + worker.owned_total = 0; + worker.swimlane_last_cycle = 0; + for (uint32_t index = 0; index < kPrivateSlots; ++index) { + worker.slots[index].occupied = false; + worker.slots[index].built = false; + } + +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) + CompeteFirstSplitRuntimeState &split_runtime = Ops::CompeteFirstSplitState(); + split_runtime.context = SubmitContext{}; + split_runtime.stats = LocalStats{}; + split_runtime.scheduler = state; + split_runtime.worker = &worker; + split_runtime.task_count = 0; + split_runtime.worker_id = worker_id; + split_runtime.caller_state_address = reinterpret_cast(&split_runtime); + split_runtime.finish_state_address = 0; + split_runtime.finish_calls = 0; + split_runtime.protocol_errors = 0; + split_runtime.state_cookie = CompeteFirstSplitStateCookie(worker_id, role); + split_runtime.task_id_sum = 0; + split_runtime.owner_worker_id = worker_id; + split_runtime.reserved = 0; + LocalStats &stats = split_runtime.stats; +#else + LocalStats stats{}; +#endif + stats.result.worker_id = worker_id; + stats.result.role = static_cast(role); + stats.result.checksum = 0; + stats.trace = AttachTrace(state, worker, worker_id); +#if PA_BUILD_ATOMIC_SWIMLANE + // 完整泳道产物把阶段、Atomic 与 DCCI 视为同一构建合同。入口一次性 + // 验证 host 配置与 raw header；成功后各条记录可省掉重复附着判断。 + if (stats.trace.core == nullptr || + stats.trace.records == nullptr || + stats.trace.capacity != kTraceRecordsPerCore || + !stats.trace.atomics_enabled) { + (void)PreAttachAtomicExchange( + &state->fatal.value, static_cast(1) + ); + return; + } +#endif +#if !PA_BUILD_TRACE_FREE + if (TraceStorageAttached(stats.trace)) { + const uint32_t startup_dcci_lines = + DcciRegionCacheLineCount( + &state->config, kStartupConfigBytes + ); + if (startup_dcci_lines != 0) { + (void)WriteDcciTrace( + stats.trace, -1, -1, + DcciSite::StartupConfigInvalidate, + DcciOp::Invalidate, + /*trailing_dsb=*/true, + startup_dcci_lines, + startup_dcci_begin, startup_dcci_end + ); + } else { + stats.trace.dcci_counter_overflow = true; + } + } +#else + (void)startup_dcci_begin; + (void)startup_dcci_end; +#endif + + // startup 严格保持生产 flat 语义；本实验只改变 replay 尾部的 final 汇合。 + // 96 个参与者全部完成本地状态初始化后再进入 task 0，主要用于压低启动偏斜对 + // winner 分布和 Submit 时序的干扰；atomicMax 的唯一 winner 正确性本身不依赖该屏障。 +#if PA_BUILD_PERF_CLOCK + // perf-clock 不采集生命周期诊断。启动 watchdog 仍在下一行建立自己的 + // 正确性超时起点，不能为了追求字面上的“两次 SYS_CNT”删除防挂死机制。 + stats.result.startup_barrier_begin = 0; +#else + stats.result.startup_barrier_begin = Ops::Now(); +#endif + TraceAtomicFetchAdd( + stats.trace, stats.result, -1, AtomicSite::StartupIncrement, + &state->started_count.value, 1 + ); + const uint64_t start_wait = Ops::Now(); + uint32_t start_polls = 0; + const uint32_t startup_poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, + TraceAtomicPollBatchMask(AtomicSite::StartupPoll) | + TraceAtomicPollBatchMask(AtomicSite::FatalPoll) + ); + // 全员到齐或任一核发布 fatal 即退出启动等待；watchdog 防止缺失参与者造成永久挂死。 + while (LoadLine(state->started_count, stats, AtomicSite::StartupPoll) < + static_cast(state->config.workers) && + !IsFatal(state, stats)) { + Ops::SpinHint(); + if (WatchdogExpired(state, stats, start_wait, start_polls)) { + break; + } + } + AtomicPollRegionEnd(stats.trace, stats.result, startup_poll_region); +#if PA_BUILD_PERF_CLOCK + stats.result.startup_barrier_end = 0; +#else + stats.result.startup_barrier_end = Ops::Now(); +#endif + + const uint32_t batches = state->config.batches; +#if PTO_FDWIC_SHARED_MAP + // split Finish 在 replay 过程中只需容量上限；真实 task_count 随每批 + // context_len 变化，回放结束后再由 local_index 封口。末次 Submit + // 的精确计时身份由 ticket 携带，不预扫 context_lens。 + uint32_t task_count = kMaxTasks; +#else + const uint32_t task_count = batches * kTasksPerBatch; +#endif +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) + split_runtime.task_count = task_count; +#endif + PaOrchestrationState orchestration; + TaskArgs args; +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) + SubmitContext &context = split_runtime.context; +#else + SubmitContext context; +#endif + uint64_t orchestration_begin = 0; + uint64_t orchestration_end = 0; +#if PTO_FDWIC_SHARED_MAP + bool shared_replay_window_started = false; +#endif + if (!IsFatal(state, stats)) { + // private 每批固定回放 Alloc/QK/SF/PV/UP；shared 则先按 + // context_len 建立 1+4N task plan。所有 worker 的计划相同，执行 + // lane 仍由 Claim 筛选。 + // CCEC 可在这里开启本 worker 私有 PMU 窗口；CPU/AscendC 适配层是空实现。 + // 窗口覆盖本 worker 的全部调度期：从 orchestration 初始化前开始， + // 依次包含 EfDrain、Claim、当前模式实际执行的参数构造与后续 Submit + // 阶段，到末次 Submit 返回。private 为全员 eager；shared 五类 + // task 都只由 Claim owner 构参。 + // 它与全局“首 Submit.begin～末 Submit.end”口径接近但不相同，host sidecar + // 必须按 per-worker 累计解释。泳道父边界在 PMU-only 构建中会被编译为空， + // 不应污染 Submit 取数。 + auto pmu_context = Ops::PmuWindowStart(state, worker_id); +#if PTO_FDWIC_SHARED_MAP + shared_replay_window_started = true; +#endif + orchestration_begin = TraceTimestamp(stats.trace, stats.result); + InitPaOrchestration(orchestration, batches, &state->context_lens[0]); +#if PTO_FDWIC_SHARED_MAP + bool replay_ok = true; + for (uint32_t batch = 0; + batch < batches && replay_ok; ++batch) { + BeginPaBatchForCallback(orchestration, batch); + ++stats.result.context_reads; + SharedPaBatchPlan batch_plan{}; + if (!BuildSharedPaBatchPlan( + orchestration.current_sequence, + worker.local_index, batch_plan + )) { + SetFatal( + state, stats, + static_cast(worker.local_index) + ); + break; + } + if (!SubmitCallbackTask( + state, worker, task_count, orchestration, args, + batch, context, stats, pmu_context, + batch_plan, 0 + )) { + break; + } + AcceptTaskOutputs( + orchestration, TaskKind::Alloc, + OrchestrationOutputs(context) + ); + + for (uint32_t group = 0; + group < batch_plan.group_count; ++group) { + const uint64_t block_offset = + static_cast(group) * + kPaBlocksPerRequest; + PreparePaBlockGroup(orchestration, block_offset); + if (!SubmitCallbackTask< + TaskKind::Qk, Ops, Profile + >( + state, worker, task_count, orchestration, + args, batch, context, stats, pmu_context, + batch_plan, + SharedPaTaskOffset(TaskKind::Qk, group) + )) { + replay_ok = false; + break; + } + AcceptTaskOutputs( + orchestration, TaskKind::Qk, + OrchestrationOutputs(context) + ); + + if (!SubmitCallbackTask< + TaskKind::Sf, Ops, Profile + >( + state, worker, task_count, orchestration, + args, batch, context, stats, pmu_context, + batch_plan, + SharedPaTaskOffset(TaskKind::Sf, group) + )) { + replay_ok = false; + break; + } + AcceptTaskOutputs( + orchestration, TaskKind::Sf, + OrchestrationOutputs(context) + ); + + if (!SubmitCallbackTask< + TaskKind::Pv, Ops, Profile + >( + state, worker, task_count, orchestration, + args, batch, context, stats, pmu_context, + batch_plan, + SharedPaTaskOffset(TaskKind::Pv, group) + )) { + replay_ok = false; + break; + } + AcceptTaskOutputs( + orchestration, TaskKind::Pv, + OrchestrationOutputs(context) + ); + + if (!SubmitCallbackTask< + TaskKind::Up, Ops, Profile + >( + state, worker, task_count, orchestration, + args, batch, context, stats, pmu_context, + batch_plan, + SharedPaTaskOffset(TaskKind::Up, group) + )) { + replay_ok = false; + break; + } + } + const uint32_t expected_batch_end = + batch_plan.batch_start + batch_plan.task_count; + if (replay_ok && + (worker.local_index < 0 || + static_cast(worker.local_index) != + expected_batch_end)) { + SetFatal( + state, stats, + static_cast(worker.local_index) + ); + replay_ok = false; + } + } +#else + for (uint32_t batch = 0; batch < batches; ++batch) { + BeginPaBatchForCallback(orchestration, batch); + ++stats.result.context_reads; + if (!SubmitCallbackTask( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs( + orchestration, TaskKind::Alloc, OrchestrationOutputs(context) + ); + + PreparePaBlockGroup(orchestration, 0); + if (!SubmitCallbackTask( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs( + orchestration, TaskKind::Qk, OrchestrationOutputs(context) + ); + + if (!SubmitCallbackTask( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs( + orchestration, TaskKind::Sf, OrchestrationOutputs(context) + ); + + if (!SubmitCallbackTask( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + AcceptTaskOutputs( + orchestration, TaskKind::Pv, OrchestrationOutputs(context) + ); + + if (!SubmitCallbackTask( + state, worker, task_count, orchestration, args, batch, context, stats, + pmu_context + )) { + break; + } + } +#endif +#if PTO_FDWIC_SHARED_MAP + // 先封口实际 task 数和唯一 last 身份，再停止 PMU。这样 + // missing/early/duplicate-last 不会先产出看似合法的 phase shape； + // Stop 仍在 fatal 路无条件执行，保证本核计数器完成收口。 + task_count = FinalizeSharedReplayTaskCount( + state, worker, stats + ); +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) + split_runtime.task_count = task_count; +#endif +#endif + Ops::PmuWindowStop(state, worker_id, pmu_context); + orchestration_end = TraceTimestamp(stats.trace, stats.result); + } +#if PTO_FDWIC_SHARED_MAP + // 构建身份或启动阶段已经 fatal 时 PMU 从未开启，但后续 split + // 协议仍必须使用实际的零 task 数，不能保留容量上限 kMaxTasks。 + if (!shared_replay_window_started) { + task_count = FinalizeSharedReplayTaskCount( + state, worker, stats + ); +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) + split_runtime.task_count = task_count; +#endif + } +#endif + + // replay_done 表示所有 worker 已退出回放循环（成功路径即完整提交）；之后仍需 drain 到本核 slot 为空。 + // 成功路径复用 orchestration end 作为 final drain start，使两个业务父区间 + // 首尾相接；父记录延后到 final drain 结束再写，避免记录自身落进任一业务 span。 + const uint64_t final_drain_begin = orchestration_end != 0 + ? orchestration_end + : TraceTimestamp(stats.trace, stats.result); +#if PA_BUILD_PERF_CLOCK + stats.result.final_barrier_begin = 0; +#else + stats.result.final_barrier_begin = Ops::Now(); +#endif + const auto final_barrier_shape = static_cast(state->config.final_barrier_shape); + const uint32_t final_two_level_groups = TwoLevelFinalBarrierGroupCount(final_barrier_shape); + const bool hierarchical_final_barrier = + final_barrier_shape == FinalBarrierShape::ThreeLevel6x4x4 || final_two_level_groups != 0; + if (hierarchical_final_barrier) { + ArriveHierarchicalFinalBarrier( + state->final_barrier, final_barrier_shape, worker, stats, AtomicSite::ReplayDoneIncrement + ); + } else { + PublishFinalBarrierLine(state->replay_done, stats, AtomicSite::ReplayDoneIncrement); + } + const uint32_t final_poll_region = AtomicPollRegionBegin( + stats.trace, stats.result, + TraceAtomicPollBatchMask(AtomicSite::ReplayDonePoll) | + TraceAtomicPollBatchMask(AtomicSite::FaninFlagLoad) | + TraceAtomicPollBatchMask(AtomicSite::FatalPoll) + ); + bool leaf_forwarded = false; + bool middle_forwarded = false; + bool root_released = false; + bool middle_released = false; + bool leaf_released = false; + bool global_release_observed = false; +#if PTO_FDWIC_SHARED_MAP + uint32_t final_stall_polls = 0; +#endif + while (true) { + const uint32_t freed = + DrainReady(state, worker, DrainPlace::FinalDrain, stats); + const bool all_replayed = hierarchical_final_barrier ? + ProgressHierarchicalFinalBarrier( + state->final_barrier, final_barrier_shape, worker, stats, + AtomicSite::ReplayDoneIncrement, AtomicSite::ReplayDonePoll, leaf_forwarded, + middle_forwarded, root_released, middle_released, leaf_released + ) : + LoadLine(state->replay_done, stats, AtomicSite::ReplayDonePoll) >= + static_cast(state->config.workers); + if (all_replayed && !global_release_observed) { +#if !PA_BUILD_PERF_CLOCK + stats.result.final_barrier_release = Ops::Now(); +#endif + global_release_observed = true; + } +#if PTO_FDWIC_SHARED_MAP + // final barrier 证明所有 replay actor 已停止生产。只有本轮没有 + // 释放任何 slot、且本核仍被未完成 fanin 阻塞时才按 1024 次一批 + // 探测 fatal；正常 count==0 出口不新增原子，正常跨核推进也不会 + // 被误清。fatal 时撤销本核执行资格，保证所有 worker 收敛退出。 + if (global_release_observed && freed == 0 && + worker.occupied_count != 0) { + ++final_stall_polls; + if ((final_stall_polls & 1023U) == 0) { + (void)DiscardSharedSlotsAfterReplayFatal( + state, worker, stats + ); + } + } else { + final_stall_polls = 0; + } +#endif + // 必须同时满足“无人再生产新 slot”和“本核旧 slot 全部完成”，否则继续帮助系统推进 completion。 + if (global_release_observed && worker.occupied_count == 0) { + break; + } + if (freed == 0) { + Ops::SpinHint(); + } + } + AtomicPollRegionEnd(stats.trace, stats.result, final_poll_region); +#if PA_BUILD_PERF_CLOCK + stats.result.final_barrier_end = 0; +#else + stats.result.final_barrier_end = Ops::Now(); +#endif + const uint64_t final_drain_end = TraceTimestamp(stats.trace, stats.result); + if (orchestration_begin != 0 && orchestration_end >= orchestration_begin) { + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::OrchestrationReplay, + ProfilePhase::Orchestration, orchestration_begin, orchestration_end + ); + } + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::FinalDrain, + ProfilePhase::ReplayTail, final_drain_begin, final_drain_end + ); + +#if !PA_BUILD_TRACE_FREE + if (AtomicSwimlaneEnabled(stats.trace)) { + // 两条基线都放在最终 drain 之后。第一条量连续 + // SYS_CNT，第二条量返回依赖钩子的固定成本；它们只描述计时底噪，不能 + // 从每条 atomic 中机械相减后宣称得到跨核全局可见性延迟。 + const uint64_t clock_begin = Ops::Now(); + const uint64_t clock_end = Ops::Now(); + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::ClockBaseline, + ProfilePhase::ReplayTail, clock_begin, clock_end + ); + const uint64_t dependency_begin = Ops::Now(); + const uint64_t dependency_end = Ops::NowAfterAtomicResult( + static_cast(worker_id) + ); + WriteTrace( + stats.trace, stats.result, -1, -1, TracePhase::ClockBaseline, + ProfilePhase::ReplayTail, dependency_begin, dependency_end, + kClockAtomicDependency | + (Ops::kAtomicReturnReadyObserved ? kClockAtomicDependencyApplied : 0U) + ); + } +#endif + +#if defined(PA_COMPETE_FIRST_SPLIT_FINISH) + const uint64_t expected_task_id_sum = + static_cast(task_count) * (task_count - 1U) / 2U; + // terminal fatal 可能在某个 worker 进入首个 Submit 之前已经可见。 + // 该 worker 合法地没有 finish 调用，finish TU 地址也尚未回写；不能 + // 把这种零回放收敛误报成 split 状态错配。只要开始过任一 Submit， + // 仍严格要求 caller/finish 是同一个 TLS runtime。 +#if PTO_FDWIC_SHARED_MAP + // shared 的 loser 不跨 TU；某个 worker 即使完整重放了 N 个任务，也 + // 可能一个都没赢。finish 地址是否出现只取决于本核 winner 数。 + const bool finish_state_matches = + split_runtime.finish_calls == 0 + ? split_runtime.finish_state_address == 0 + : split_runtime.finish_state_address == + split_runtime.caller_state_address; + const uint64_t expected_finish_calls = + stats.result.claim_wins; +#else + const bool finish_state_matches = + task_count == 0 + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - private split-finish 诊断不属于 standalone shared scheduler + ? Ops::Load(&state->fatal.value) != 0 && + split_runtime.finish_calls == 0 && + split_runtime.finish_state_address == 0 + : split_runtime.finish_state_address == + split_runtime.caller_state_address; + const uint64_t expected_finish_calls = task_count; +#endif + const bool split_protocol_ok = + split_runtime.scheduler == state && split_runtime.worker == &worker && + split_runtime.task_count == task_count && split_runtime.worker_id == worker_id && + split_runtime.owner_worker_id == worker_id && split_runtime.caller_state_address != 0 && + finish_state_matches && + split_runtime.finish_calls == expected_finish_calls && + split_runtime.task_id_sum == expected_task_id_sum && + split_runtime.state_cookie == CompeteFirstSplitStateCookie(worker_id, role) && + split_runtime.reserved == 0; + if (!split_protocol_ok) { + ++split_runtime.protocol_errors; + SetFatal(state, stats); + } + stats.result.compete_first_split_caller_state_address = split_runtime.caller_state_address; + stats.result.compete_first_split_finish_state_address = split_runtime.finish_state_address; + stats.result.compete_first_split_finish_calls = split_runtime.finish_calls; + stats.result.compete_first_split_protocol_errors = split_runtime.protocol_errors; + stats.result.compete_first_split_state_cookie = split_runtime.state_cookie; + stats.result.compete_first_split_task_id_sum = split_runtime.task_id_sum; + stats.result.compete_first_split_owner_worker_id = split_runtime.owner_worker_id; + stats.result.compete_first_split_reserved = split_runtime.reserved; +#endif + + // PA writes swimlane records through the ordinary GM cache and explicitly + // cleans each worker's record range before the kernel finishes. + FlushTraceCore(stats.trace, stats.result); +#if PA_BUILD_PERF_CLOCK + // 复用末个 Submit 的已保存边界满足结果有序性，不再为 worker 尾部 + // 增加一次纯诊断 SYS_CNT。 + stats.result.finish_cycle = stats.result.submit_end; +#else + stats.result.finish_cycle = Ops::Now(); +#endif + stats.result.max_occupied = stats.max_occupied; + stats.result.final_occupied = worker.occupied_count; + stats.result.final_heap_next = worker.heap_next; +#if PTO_FDWIC_SHARED_MAP + // shared 后端没有每核 map 控制字；这里发布统一逻辑窗口摘要，实际 + // bucket/head/tail/seq/payload 由 host 回读唯一 sidecar 后逐槽校验。 + // shared fresh Output 都由 shared-output table 直接寻址，manual_dep + // 的 output_view 也不进入自动 region hazard；因此 Case1 region ring + // 严格为空；四个摘要都保持零，不能再用跨模式签名比较所需的逻辑 + // floor 冒充 sidecar 实际发生过 ordered reclaim。 + stats.result.map_high_water = 0; + stats.result.map_alive_floor = 0; + stats.result.map_cleaned_upto = 0; + stats.result.map_live_entries = 0; +#else + stats.result.map_high_water = static_cast(worker.map.high_water); + stats.result.map_alive_floor = static_cast(worker.map.alive_floor); + stats.result.map_cleaned_upto = static_cast(worker.map.cleaned_upto); + stats.result.map_live_entries = CountLiveMapEntries(worker.map); + stats.result.checksum = + PrivateLogicalTensorMapSignature(worker.map); +#endif + PublishResult(state->results[worker_id], stats.result); +} + +template +PA_DEVICE void RunScheduler(PA_GM SchedulerState *state, uint32_t worker_id, CoreRole role) { + // 两个正式 CCEC 构建都不再携带旧 phase-profile 模板副本：swimlane 用 + // records 表达阶段，submit-pmu 使用独立 PMU 边界。其他后端暂时保留原 + // 运行时入口，保证公共 standalone 的 CPU/AscendC 回归不被 CCEC 构建切分影响。 +#if PA_BUILD_SWIMLANE || PA_BUILD_SUBMIT_PMU || PA_BUILD_PERF_CLOCK + RunSchedulerImpl(state, worker_id, role); +#else + // Profile 作为编译期模板参数，只在显式开启时保留阶段累计代码，关闭时不在热路径增加运行时分支。 + if (state->config.profile_phases != 0) { + RunSchedulerImpl(state, worker_id, role); + } else { + RunSchedulerImpl(state, worker_id, role); + } +#endif +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_SCHEDULER_CORE_H diff --git a/tests/atomic_probe/pa_scheduler/common/pa_shared_heap.h b/tests/atomic_probe/pa_scheduler/common/pa_shared_heap.h new file mode 100644 index 0000000000..84f32fab67 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/common/pa_shared_heap.h @@ -0,0 +1,207 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_SHARED_HEAP_H +#define PA_SCHEDULER_COMMON_PA_SHARED_HEAP_H + +#include "pa_trace.h" + +namespace pa_scheduler { + +// shared heap 的首版只验证 PA Case1 当前有界规模，不启用物理区间复用。 +// task_base 是 heap 内的物理偏移；aggregate_vend 只是所有 shard 已分配字节 +// 的全局累计值，不是任一 shard 的地址，也不能代替旧单 ring HeapGuard。 +struct SharedHeapReservation { + uint64_t task_base; + uint64_t aggregate_vend; +}; +static_assert(sizeof(SharedHeapReservation) == 16, "shared heap reservation ABI changed"); + +PA_DEVICE uint64_t SharedHeapAlignDown(uint64_t value) { + return value & ~(kOutputAlignment - 1); +} + +PA_DEVICE bool SharedHeapAligned(uint64_t value) { + return (value & (kOutputAlignment - 1)) == 0; +} + +// 单元测试默认实例化 ObserveAtomics=false，保持原有简洁 Ops 接口；真实 +// scheduler 显式选择 true 并传入本 worker 独占 trace/result。trace-free +// 构建中的 TraceAtomic* 会在编译期退化为原始 Ops，不给性能基线增加分支。 +template +PA_DEVICE int64_t SharedHeapAtomicLoad( + PA_GM volatile int64_t *address, int32_t task_id, AtomicSite site, + TraceContext *trace, WorkerResult *result +) { + if constexpr (ObserveAtomics) { + return TraceAtomicLoad( + *trace, *result, task_id, site, address + ); + } else { + (void)task_id; + (void)site; + (void)trace; + (void)result; + return Ops::Load(address); + } +} + +template +PA_DEVICE int64_t SharedHeapAtomicFetchAdd( + PA_GM volatile int64_t *address, int64_t value, int32_t task_id, + AtomicSite site, TraceContext *trace, WorkerResult *result +) { + if constexpr (ObserveAtomics) { + // 两个 FetchAdd 的旧值都决定本次 reservation，必须使用 + // return-ready 边界，不能按发布型 source-issue 观察。 + return TraceAtomicFetchAdd( + *trace, *result, task_id, site, address, value, true + ); + } else { + (void)task_id; + (void)site; + (void)trace; + (void)result; + return Ops::FetchAdd(address, value); + } +} + +// no-wrap 是本阶段的明确边界：每个 shard 的绝对 cursor 只能从 0 推进到 +// shard_span，绝不取模。FetchAdd 返回的旧 cursor 是当前 task 唯一的物理 +// 区间；合法并发 writer 可以让它不同于前置 Load 的观察值，不能因此回滚。 +// +// 容量竞争若在 FetchAdd 后才被发现，则本轮进入 terminal fatal 并保留已经 +// 推进的控制字供 host 取证。并发 allocator 绝不能用 Exchange 恢复预检 +// 快照，否则会覆盖其他 winner 的合法进度。 +template +PA_DEVICE bool ReserveSharedOutputHeap( + PA_GM SharedTensorMapSidecar &map, uint32_t task_id, uint64_t total, + uint64_t heap_size, SharedHeapReservation &reservation, + TraceContext *trace = nullptr, WorkerResult *result = nullptr +) { + reservation.task_base = 0; + reservation.aggregate_vend = 0; + + static_assert(kSharedHeapShards == 8, "standalone shared heap must use eight shards"); + static_assert( + (kSharedHeapShards & (kSharedHeapShards - 1)) == 0, + "shared heap shard count must be a power of two" + ); + static_assert( + (kOutputAlignment & (kOutputAlignment - 1)) == 0, + "shared heap alignment must be a power of two" + ); + if (task_id >= kMaxTasks || + heap_size > static_cast(INT64_MAX)) { + return false; + } + const uint64_t shard_span = + SharedHeapAlignDown(heap_size / kSharedHeapShards); + const uint64_t usable_capacity = + shard_span * kSharedHeapShards; + + const int64_t checked_vend = + SharedHeapAtomicLoad( + &map.shared_heap_vend.value, static_cast(task_id), + AtomicSite::SharedHeapVendLoad, trace, result + ); + if (checked_vend < 0) { + return false; + } + const uint64_t vend_snapshot = static_cast(checked_vend); + if (!SharedHeapAligned(vend_snapshot) || + vend_snapshot > usable_capacity) { + return false; + } + + // 零输出 task 仍需要取得当前 aggregate vend，供完成协议发布该 task 的 + // progress；但它不读取或推进任一 shard cursor，也不要求可用 heap 空间。 + if (total == 0) { + reservation.aggregate_vend = vend_snapshot; + return true; + } + if (shard_span == 0) { + return false; + } + + if (total > UINT64_MAX - (kOutputAlignment - 1)) { + return false; + } + const uint64_t reserve = + (total + kOutputAlignment - 1) & ~(kOutputAlignment - 1); + if (reserve == 0 || reserve > static_cast(INT64_MAX)) { + return false; + } + + if (reserve > shard_span || + vend_snapshot > usable_capacity - reserve || + vend_snapshot > static_cast(INT64_MAX) - reserve) { + return false; + } + + const uint32_t shard = task_id % kSharedHeapShards; + PA_GM volatile int64_t *cursor_address = + &map.shared_heap_cursor[shard].value; + const int64_t signed_cursor_before = + SharedHeapAtomicLoad( + cursor_address, static_cast(task_id), + AtomicSite::SharedHeapCursorLoad, trace, result + ); + if (signed_cursor_before < 0) { + return false; + } + const uint64_t cursor_before = + static_cast(signed_cursor_before); + if (!SharedHeapAligned(cursor_before) || + cursor_before > shard_span - reserve) { + return false; + } + + const int64_t observed_cursor = + SharedHeapAtomicFetchAdd( + cursor_address, static_cast(reserve), + static_cast(task_id), + AtomicSite::SharedHeapCursorReserve, trace, result + ); + if (observed_cursor < 0) { + return false; + } + const uint64_t cursor = static_cast(observed_cursor); + if (!SharedHeapAligned(cursor) || cursor > shard_span - reserve) { + return false; + } + + PA_GM volatile int64_t *vend_address = &map.shared_heap_vend.value; + const int64_t observed_vend = + SharedHeapAtomicFetchAdd( + vend_address, static_cast(reserve), + static_cast(task_id), + AtomicSite::SharedHeapVendAdvance, trace, result + ); + if (observed_vend < 0) { + return false; + } + const uint64_t vend = static_cast(observed_vend); + if (!SharedHeapAligned(vend) || + vend > usable_capacity - reserve || + vend > static_cast(INT64_MAX) - reserve) { + return false; + } + + reservation.task_base = + static_cast(shard) * shard_span + cursor; + reservation.aggregate_vend = vend + reserve; + return true; +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_SHARED_HEAP_H diff --git a/tests/atomic_probe/pa_scheduler/common/pa_shared_submit_path.h b/tests/atomic_probe/pa_scheduler/common/pa_shared_submit_path.h new file mode 100644 index 0000000000..307f04bf02 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/common/pa_shared_submit_path.h @@ -0,0 +1,819 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_SHARED_SUBMIT_PATH_H +#define PA_SCHEDULER_COMMON_PA_SHARED_SUBMIT_PATH_H + +// 本文件由 pa_scheduler_core.h 在 pa_scheduler 命名空间内引入。shared +// TensorMap 的 Submit 控制流独立放在这里，避免继续把两套协议塞进同一 +// 个宏分支密集的热函数；CPU 与 CCEC 仍复用相同的 Ops 和底层原语。 + +struct SharedTaskWriterDelta { + // ordinary entry 先在 owner 私有上下文中完整准备；只有拿到 task 的 + // exact insert turn 后才批量预检和发布。bucket/同桶序号及 symbol + // packed key 都在等待前计算；串行区只消费这份不可变提交计划。 + // prepared_task_id 只在全部结构检查通过后写入；Publish 阶段据此确认 + // 这份 owner-local delta 属于当前 task，不再在串行区重复扫描 args。 + SharedRegionValue ordinary_entries[kMaxTaskTensors]; + uint32_t symbol_keys[kMaxTaskTensors]; + uint16_t ordinary_buckets[kMaxTaskTensors]; + uint8_t ordinary_bucket_ordinals[kMaxTaskTensors]; + int32_t prepared_task_id; + uint32_t ordinary_count; + uint32_t symbol_count; + bool writer_intent_required; +}; +static_assert( + __is_trivially_constructible(SharedTaskWriterDelta), + "shared task writer delta must remain trivial for CCEC local state" +); +static_assert( + kMapBuckets <= 65536 && kMaxTaskTensors <= 256, + "prepared bucket and ordinal metadata no longer cover the build" +); + +PA_DEVICE bool PrepareSharedTaskWriterDelta( + const TaskArgs &args, const SubmitContext &context, + SharedTaskWriterDelta &delta +) { + delta.prepared_task_id = -1; + delta.ordinary_count = 0; + delta.symbol_count = 0; + delta.writer_intent_required = false; + const int32_t task_id = context.task_id; + if (!context.won || task_id < 0 || + task_id >= static_cast(kMaxTasks) || + args.has_error || args.tensor_count < 0 || + args.tensor_count > static_cast(kMaxTaskTensors) || + context.result.task_id != static_cast(task_id) || + context.result.count > kSharedOutputMaxPerTask) { + return false; + } + + bool writer_required = false; + if (!InspectSharedWriterIntent(args, writer_required) || + (writer_required && + !ValidateSharedWriterIntentSet(args, task_id))) { + return false; + } + uint32_t expected_register_mask = 0; + for (int32_t index = 0; + index < args.tensor_count; ++index) { + if (IsSharedWriterIntentTag( + TaskTag(args, static_cast(index)) + )) { + expected_register_mask |= + 1U << static_cast(index); + } + } + if (context.register_mask != expected_register_mask) { + return false; + } + uint32_t register_mask = context.register_mask; + for (int32_t index = 0; index < args.tensor_count; ++index) { + const uint32_t bit = + 1U << static_cast(index); + if ((register_mask & bit) == 0) { + continue; + } + const TensorArgType tag = + TaskTag(args, static_cast(index)); + if (!IsSharedWriterIntentTag(tag)) { + return false; + } + const TaskTensorRef &reference = args.tensors[index]; + if (reference.kind == TensorRefKind::SharedOutputRef) { + const FdwicOutputRef output_ref = + SharedOutputReference(reference); + uint32_t symbol_key = 0; + if (!SharedSymbolHistoryKey(output_ref, symbol_key) || + delta.symbol_count >= kMaxTaskTensors) { + return false; + } + for (uint32_t previous = 0; + previous < delta.symbol_count; ++previous) { + if (delta.symbol_keys[previous] == symbol_key) { + return false; + } + } + delta.symbol_keys[delta.symbol_count] = symbol_key; + ++delta.symbol_count; + } else if (reference.kind == TensorRefKind::GmTensor) { + PA_GM const TensorDesc &tensor = + *reference.pointer.gm_tensor; + if (!tensor.manual_dep) { + if (delta.ordinary_count >= kMaxTaskTensors || + !MakeValidatedSharedWriterRegion( + tensor, task_id, + delta.ordinary_entries[ + delta.ordinary_count + ] + )) { + return false; + } + const uint32_t bucket = TensorMapHash( + delta.ordinary_entries[ + delta.ordinary_count + ].buffer_addr + ); + uint32_t ordinal = 0; + for (uint32_t previous = 0; + previous < delta.ordinary_count; + ++previous) { + ordinal += + delta.ordinary_buckets[previous] == bucket + ? 1U + : 0U; + } + delta.ordinary_buckets[ + delta.ordinary_count + ] = static_cast(bucket); + delta.ordinary_bucket_ordinals[ + delta.ordinary_count + ] = static_cast(ordinal); + ++delta.ordinary_count; + } + } else if (reference.kind == + TensorRefKind::LocalTensor) { + const TensorDesc &tensor = + *reference.pointer.local_tensor; + if (!tensor.manual_dep) { + if (delta.ordinary_count >= kMaxTaskTensors || + !MakeValidatedSharedWriterRegion( + tensor, task_id, + delta.ordinary_entries[ + delta.ordinary_count + ] + )) { + return false; + } + const uint32_t bucket = TensorMapHash( + delta.ordinary_entries[ + delta.ordinary_count + ].buffer_addr + ); + uint32_t ordinal = 0; + for (uint32_t previous = 0; + previous < delta.ordinary_count; + ++previous) { + ordinal += + delta.ordinary_buckets[previous] == bucket + ? 1U + : 0U; + } + delta.ordinary_buckets[ + delta.ordinary_count + ] = static_cast(bucket); + delta.ordinary_bucket_ordinals[ + delta.ordinary_count + ] = static_cast(ordinal); + ++delta.ordinary_count; + } + } else { + return false; + } + register_mask &= ~bit; + } + if (register_mask != 0) { + return false; + } + // Inspect/Validate 与 delta 构造都只读取同一个 const TaskArgs；两者对 + // “是否需要自动登记”必须给出相同结论。该同步 Finish 控制流在 + // Publish 前不会修改 args，delta 也只存在于 winner 的本地栈上。 + const bool delta_requires_intent = + delta.ordinary_count != 0 || delta.symbol_count != 0; + if (writer_required != delta_requires_intent || + delta.ordinary_count + delta.symbol_count > + kMaxTaskTensors) { + return false; + } + delta.writer_intent_required = writer_required; + delta.prepared_task_id = task_id; + return true; +} + +template +PA_DEVICE bool PublishSharedTaskWriterMetadata( + PA_GM SchedulerState *state, const SubmitContext &context, + const SharedTaskWriterDelta &delta, LocalStats &stats +) { + const int32_t task_id = context.task_id; + if (state == nullptr || !context.won || task_id < 0 || + task_id >= static_cast(kMaxTasks) || + delta.prepared_task_id != task_id || + delta.ordinary_count > kMaxTaskTensors || + delta.symbol_count > kMaxTaskTensors || + delta.ordinary_count + delta.symbol_count > + kMaxTaskTensors || + delta.writer_intent_required != + (delta.ordinary_count != 0 || + delta.symbol_count != 0) || + TraceAtomicLoad( + stats.trace, stats.result, task_id, + AtomicSite::SharedMetadataFatalGuardLoad, + &state->fatal.value + ) != 0) { + if (state != nullptr) { + SetFatal(state, stats, task_id); + } + return false; + } + + // insert-before-lookup 不能用本 task 的 reader_done 回收自己仍可能 + // 消费的 N-H。首版只使用已经证明正确的 -1 前沿；容量不足明确 + // 终止，绝不覆盖 live producer 或错误推进 task turn。 + if (SharedCheckPreparedTaskAppend( + state->shared_map, delta.ordinary_entries, + delta.ordinary_buckets, + delta.ordinary_bucket_ordinals, + delta.ordinary_count, -1, task_id, + &stats.trace, &stats.result + ) != SharedAppendCheck::Ready) { + SetFatal(state, stats, task_id); + return false; + } + + if (delta.symbol_count != 0 && + !CommitPreparedSymbolSharedWriterIntentSet( + state->shared_map, delta.symbol_keys, + delta.symbol_count, task_id, &state->fatal.value, + &stats + )) { + SetFatal(state, stats, task_id); + return false; + } + if (!SharedAppendPreparedTask( + state->shared_map, delta.ordinary_entries, + delta.ordinary_buckets, delta.ordinary_count, + task_id, &stats.trace, &stats.result + )) { + SetFatal(state, stats, task_id); + return false; + } + return true; +} + +PA_DEVICE void RecordCommittedSharedTaskWriterStats( + const SharedTaskWriterDelta &delta, LocalStats &stats +) { + // 这两个字段统计已经越过 task-level completion CAS 的完整事务。 + // metadata 已写入但 CAS 失败的 terminal task 不计入成功统计；故障现场 + // 仍由 fatal、共享元数据与 CAS observed value 保留。 + stats.result.map_inserts += delta.ordinary_count; + stats.result.shared_symbol_inout_commits += delta.symbol_count; +} + +template +PA_DEVICE bool HandoffSharedTaskInsertTurn( + PA_GM SchedulerState *state, int32_t task_id, LocalStats &stats, + int64_t &cas_observed +) { + if (state == nullptr || task_id < 0 || + task_id >= static_cast(kMaxTasks)) { + cas_observed = INT64_MIN; + return false; + } + // ordinary payload 的 DCCI、symbol history/latest 与 fresh descriptor + // 都必须先于本 task 的插入完成字对 N+1 owner 可见。每个 task 使用 + // 自己的 TaskCell，不再把一枚 baton 在 G 条 sidecar 线上轮换。 + Ops::StoreBarrier(); + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: trace-free - swimlane 构建走 CaptureAtomicCompareExchange；这里只是无泳道构建和隔离测试出口 + cas_observed = Ops::CompareExchange( + &state->tasks[static_cast(task_id)] + .deps_prepared, + static_cast(-1), + static_cast(task_id) + ); + if (cas_observed != -1) { + SetFatal(state, stats, task_id); + return false; + } + return true; +} + +#if !PA_BUILD_TRACE_FREE +template +PA_DEVICE bool TraceHandoffSharedTaskInsertTurn( + PA_GM SchedulerState *state, int32_t task_id, LocalStats &stats, + int64_t &cas_observed, uint64_t &cas_trace_begin, + uint64_t &cas_trace_end +) { + cas_trace_begin = 0; + cas_trace_end = 0; + if (state == nullptr || task_id < 0 || + task_id >= static_cast(kMaxTasks)) { + cas_observed = INT64_MIN; + return false; + } + Ops::StoreBarrier(); + cas_observed = CaptureAtomicCompareExchange( + stats.trace, + &state->tasks[static_cast(task_id)] + .deps_prepared, + static_cast(-1), + static_cast(task_id), + cas_trace_begin, cas_trace_end + ); + if (cas_observed != -1) { + SetFatal(state, stats, task_id); + return false; + } + return true; +} +#endif + +template +PA_DEVICE bool WaitForSharedTaskInsertTurn( + PA_GM SchedulerState *state, int32_t task_id, + LocalStats &stats, int64_t &ready_observed +); + +template +PA_DEVICE bool PublishSharedTaskWriterDelta( + PA_GM SchedulerState *state, const SubmitContext &context, + const SharedTaskWriterDelta &delta, LocalStats &stats +) { + // fresh output cell 由本 task 的唯一 Claim winner 独占，不参与 + // ordinary/symbol 的 task-ID 串行插入。先发布 descriptor，再等待 + // predecessor；最终 deps_prepared handoff 仍同时封口两类发布。 + if (state == nullptr || !context.won || context.task_id < 0 || + context.task_id >= static_cast(kMaxTasks) || + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - generic 组合入口只供隔离测试，正式 winner 入口已使用 SharedWinnerFatalGuardLoad + Ops::Load(&state->fatal.value) != 0 || + !PublishSharedTaskOutputs( + state->shared_map, context, + static_cast(context.task_id) + )) { + if (state != nullptr) { + SetFatal(state, stats, context.task_id); + } + return false; + } + // 隔离测试和复用调用必须走与正式 Submit 相同的资格门：task 0 + // 直接进入，其余 task 只等待 N-1。Metadata helper 本身不再重复 + // atomic load，避免正式热路径把一次前驱等待测成三次原子访问。 + int64_t ignored_ready_observed = -1; + if (!WaitForSharedTaskInsertTurn( + state, context.task_id, stats, + ignored_ready_observed + )) { + RollbackSharedTaskOutputs( + state->shared_map.shared_outputs[ + static_cast(context.task_id) + ], + context.result.count, context.task_id, &stats + ); + return false; + } + if (!PublishSharedTaskWriterMetadata( + state, context, delta, stats + )) { + RollbackSharedTaskOutputs( + state->shared_map.shared_outputs[ + static_cast(context.task_id) + ], + context.result.count, context.task_id, &stats + ); + return false; + } + int64_t ignored_cas_observed = INT64_MIN; + const bool inserted = HandoffSharedTaskInsertTurn( + state, context.task_id, stats, ignored_cas_observed + ); + if (!inserted) { + RollbackSharedTaskOutputs( + state->shared_map.shared_outputs[ + static_cast(context.task_id) + ], + context.result.count, context.task_id, &stats + ); + return false; + } + RecordCommittedSharedTaskWriterStats(delta, stats); + return true; +} + +template +PA_DEVICE bool WaitForSharedTaskInsertTurn( + PA_GM SchedulerState *state, int32_t task_id, LocalStats &stats, + int64_t &ready_observed, uint64_t &load_count +) { + ready_observed = -1; + load_count = 0; + if (state == nullptr || task_id < 0 || + task_id >= static_cast(kMaxTasks)) { + return false; + } + + // task 0 没有前驱，直接进入有序插入段。它完成后仍必须把自己的 + // deps_prepared 从 -1 发布为 0，供 task 1 建立真实跨核依赖。 + if (task_id == 0) { + ready_observed = -1; + return true; + } + + PA_GM volatile int64_t *predecessor = + &state->tasks[static_cast(task_id - 1)] + .deps_prepared; + const uint64_t begin = Ops::Now(); + uint32_t polls = 0; + while (true) { + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: aggregate - 循环结束后以 SharedInsertTurnPoll 单条 PollBatch 记录精确 load_count + const int64_t observed = Ops::Load(predecessor); + int64_t compare_observed = observed; + int64_t dependency_observed = observed; +#if PA_BUILD_SWIMLANE && \ + (defined(PA_BUILD_AIC) || defined(PA_BUILD_AIV)) + // 分支判定和 SYS_CNT 依赖边界从同一个 atomic 返回寄存器派生， + // 防止 CCEC 在 ready 分支把 ready_observed 常量折叠掉。 + compare_observed = Ops::ForkAtomicResultForBranch( + observed, dependency_observed + ); +#endif + if (compare_observed == + static_cast(task_id - 1)) { + ready_observed = dependency_observed; + load_count = static_cast(polls) + 1; + return true; + } + if (compare_observed != -1) { + SetFatal(state, stats, task_id); + return false; + } + + Ops::SpinHint(); + ++polls; + if ((polls & 1023U) != 0) { + continue; + } + if (IsFatal(state, stats, task_id)) { + return false; + } + if (Ops::Now() - begin > kWatchdogTicks) { + SetFatal(state, stats, task_id); + return false; + } + } +} + +template +PA_DEVICE bool WaitForSharedTaskInsertTurn( + PA_GM SchedulerState *state, int32_t task_id, LocalStats &stats, + int64_t &ready_observed +) { + uint64_t ignored_load_count = 0; + return WaitForSharedTaskInsertTurn( + state, task_id, stats, ready_observed, + ignored_load_count + ); +} + +template +PA_DEVICE bool WaitForSharedTaskInsertTurn( + PA_GM SchedulerState *state, int32_t task_id, LocalStats &stats +) { + int64_t ignored_ready_observed = -1; + return WaitForSharedTaskInsertTurn( + state, task_id, stats, ignored_ready_observed + ); +} + +template +PA_DEVICE bool FinishSharedWinnerSubmitBody( + PA_GM SchedulerState *state, PA_GM WorkerState &worker, + const TaskArgs &args, SubmitContext &context, LocalStats &stats, + PmuContext &pmu_context, const CallbackSubmitTicket &ticket +) { + const uint32_t task_id = ticket.task_id; + SharedPaTaskMeta task_meta{}; + if (ticket.won == 0 || + !DecodeSharedPaTaskMeta(ticket.reserved, task_id, task_meta) || + !SharedPaFunctionIdMatches( + task_meta.kind, true, + static_cast(ticket.function_id) + ) || + context.task_id != static_cast(task_id) || + !context.won || + TraceAtomicLoad( + stats.trace, stats.result, + static_cast(task_id), + AtomicSite::SharedWinnerFatalGuardLoad, + &state->fatal.value + ) != 0) { + SetFatal(state, stats, static_cast(task_id)); + return false; + } + + const TaskKind kind = task_meta.kind; + const int32_t function_id = + static_cast(ticket.function_id); + + // Claim owner 先构造 descriptor 和 writer delta；这一段不查询 + // TensorMap，也不占用有序插入通道。 + const uint64_t materialize_begin = + TraceTimestamp(stats.trace, stats.result); + BeginSubmitPmuPhase( + pmu_context + ); + const bool materialized = MaterializeTask( + worker, task_id, args, context, state->shared_map, + state->heap_base, state->heap_size, + kind, task_meta.batch_start, task_meta.group_index, + &stats.trace, &stats.result + ); + if (materialized) { + stats.result.materialized_outputs += context.result.count; + } + if (!materialized) { + EndSubmitPmuPhase( + pmu_context + ); + SetFatal(state, stats, static_cast(task_id)); + return false; + } + SharedTaskWriterDelta writer_delta{}; + if (!PrepareSharedTaskWriterDelta( + args, context, writer_delta + )) { + EndSubmitPmuPhase( + pmu_context + ); + SetFatal(state, stats, static_cast(task_id)); + return false; + } +#if PA_BUILD_TRACE_FREE + const bool task_outputs_published = + PublishSharedTaskOutputs( + state->shared_map, context, task_id, &stats + ); +#else + uint64_t task_outputs_begin = + TraceTimestamp(stats.trace, stats.result); + uint64_t task_outputs_copy_begin = task_outputs_begin; + uint64_t task_outputs_copy_end = task_outputs_begin; + uint64_t task_outputs_flush_begin = task_outputs_begin; + uint64_t task_outputs_flush_end = task_outputs_begin; + const bool task_outputs_published = + PublishSharedTaskOutputs( + state->shared_map, context, task_id, &stats, + &task_outputs_copy_begin, &task_outputs_copy_end, + &task_outputs_flush_begin, &task_outputs_flush_end + ); + const uint64_t task_outputs_end = + TraceTimestamp(stats.trace, stats.result); +#endif + if (!task_outputs_published) { + EndSubmitPmuPhase( + pmu_context + ); + SetFatal(state, stats, static_cast(task_id)); + return false; + } + EndSubmitPmuPhase( + pmu_context + ); + const uint64_t materialize_end = + TraceTimestamp(stats.trace, stats.result); + + // 仅这一段全局串行：N>0 只等待 task[N-1].deps_prepared，随后插入 + // N 的 ordinary/symbol writer 元数据，再发布 task[N].deps_prepared。 + // fresh output descriptor 已在 Materialize 尾部按 task-cell 独占发布； + // 空 writer 集合也必须推进，loser 完全不参与。 + const uint64_t register_begin = materialize_end; + BeginSubmitPmuPhase( + pmu_context + ); + int64_t ready_observed = -1; + uint64_t insert_turn_load_count = 0; + const bool turn_ready = WaitForSharedTaskInsertTurn( + state, static_cast(task_id), stats, + ready_observed, insert_turn_load_count + ); + // wait_end 对最后一次返回 Ready 的 atomic Load 建立数据依赖。只在 + // swimlane 构建读取 SYS_CNT；trace-free 的 PMU/性能构建预处理后为 0。 + const uint64_t metadata_begin = + turn_ready && insert_turn_load_count != 0 + ? TraceTimestampAfterAtomicResult( + stats.trace, stats.result, ready_observed + ) + : TraceTimestamp(stats.trace, stats.result); + const bool metadata_published = + turn_ready && + PublishSharedTaskWriterMetadata( + state, context, writer_delta, stats + ); + const uint64_t metadata_end = metadata_published + ? TraceTimestamp(stats.trace, stats.result) + : metadata_begin; + + int64_t cas_observed = INT64_MIN; +#if PA_BUILD_TRACE_FREE + const bool inserted = + metadata_published && + HandoffSharedTaskInsertTurn( + state, static_cast(task_id), stats, + cas_observed + ); +#else + uint64_t cas_trace_begin = 0; + uint64_t cas_trace_end = 0; + const bool inserted = + metadata_published && + TraceHandoffSharedTaskInsertTurn( + state, static_cast(task_id), stats, + cas_observed, cas_trace_begin, cas_trace_end + ); +#endif + // 正常路径的父区间终点依赖 CAS 返回值，表示本核已经取得 + // task[N].deps_prepared 的发布结果；不加 DSB，也不把它解释成 + // N+1 已经完成读取的时刻。 + const uint64_t register_end = metadata_published + ? TraceTimestampAfterAtomicResult( + stats.trace, stats.result, cas_observed + ) + : metadata_end; + EndSubmitPmuPhase( + pmu_context + ); + // 所有端点完成后再按业务顺序写 raw，避免写 trace 本身落入 + // Materialize/Register 的测量区间。Materialize 的 output detail + // 还原独占 cell 发布；Register 只闭合 wait、writer metadata 与 + // handoff，不逐 poll 扩张记录。 + WriteTrace( + stats.trace, stats.result, static_cast(task_id), + function_id, TracePhase::Materialize, + ProfilePhase::Materialize, materialize_begin, + materialize_end, 0, + kind == TaskKind::Alloc ? 1U : 0U + ); +#if !PA_BUILD_TRACE_FREE + if (task_outputs_published) { + WriteTrace( + stats.trace, stats.result, + static_cast(task_id), function_id, + TracePhase::SharedMaterializePublishTaskOutputs, + ProfilePhase::Materialize, task_outputs_begin, + task_outputs_end + ); + WriteTrace( + stats.trace, stats.result, + static_cast(task_id), function_id, + TracePhase::SharedMaterializePublishTaskOutputsCopy, + ProfilePhase::Materialize, task_outputs_copy_begin, + task_outputs_copy_end + ); + WriteTrace( + stats.trace, stats.result, + static_cast(task_id), function_id, + TracePhase::SharedMaterializePublishTaskOutputsFlush, + ProfilePhase::Materialize, task_outputs_flush_begin, + task_outputs_flush_end + ); + } +#endif + WriteTrace( + stats.trace, stats.result, static_cast(task_id), + function_id, TracePhase::Register, + ProfilePhase::Register, register_begin, register_end, + 0, writer_delta.ordinary_count + ); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), + function_id, TracePhase::SharedRegisterPublishMetadata, + ProfilePhase::Register, metadata_begin, metadata_end + ); +#if !PA_BUILD_TRACE_FREE + // 记录动作延后到 Register 的全部时间端点之后，避免 32B raw 写入 + // 被误计入 metadata publish 或 handoff。循环内只累计本地 polls， + // 每个成功 winner 固定至多增加这一条物理 PollBatch。 + if (turn_ready && insert_turn_load_count != 0) { + (void)WriteAggregateAtomicPollBatch( + stats.trace, stats.result, + AtomicSite::SharedInsertTurnPoll, + register_begin, metadata_begin, + insert_turn_load_count, + Ops::kAtomicReturnReadyObserved + ); + } + if (metadata_published && AtomicSwimlaneEnabled(stats.trace)) { + WriteAtomicTrace( + stats.trace, stats.result, + static_cast(task_id), + AtomicSite::SharedInsertTurnHandoff, + AtomicOp::CompareExchange, + cas_trace_begin, cas_trace_end, + true, Ops::kAtomicReturnReadyObserved + ); + } +#endif + if (!inserted) { + // output cell 虽已在串行等待前短暂可见，但完成字尚未发布；失败路径 + // 恢复本 task 独占 cell，保留原有 fail-closed 终态。正常路径无额外 + // rollback 分支开销。 + RollbackSharedTaskOutputs( + state->shared_map.shared_outputs[task_id], + context.result.count, + static_cast(task_id), &stats + ); + return false; + } + // Register 的业务终点已经取完，失败分支也已经退出；成功统计因此 + // 既不污染串行区泳道，也不会把 completion CAS 失败的 metadata + // 前缀误算成已提交事务。 + RecordCommittedSharedTaskWriterStats( + writer_delta, stats + ); +#if defined(PA_TEST_SHARED_SUBMIT_HOOKS) + // 仅供 CPU 定向并发门槛暂停某个 owner；正式 CPU/CCEC 构建预处理后 + // 不保留调用。测试借此证明 N+1 的 lookup/Build 不被 N 的 Build + // 阶段串行化。 + Ops::AfterSharedTaskInsert( + state, worker, task_id + ); +#endif + + // task[N] 的插入完成字已经发布，N+1 owner 可以进入有序插入段； + // 当前 owner 的 fanin lookup、Build 和 slot 执行不再占住该链。 + uint64_t build_begin = register_end; + if (kind != TaskKind::Alloc) { + const uint64_t fanin_begin = register_end; + bool lookup_protocol_ok = false; + uint32_t ordinary_lookup_count = 0; + // 本 task 的 writer 已经进入 history/latest；lookup 必须从 + // latest 沿不可变 history 回退到 max(writer < task_id)。这同时 + // 覆盖首组与后续组，不再依赖 PA 专用 chained-writer 特判。 + context.fanin_count = static_cast( + CollectSharedFanin( + state->shared_map, args, + static_cast(task_id), + static_cast(state->heap_window), + stats, context.fanin, lookup_protocol_ok, + ordinary_lookup_count, &state->fatal.value + ) + ); + if (!lookup_protocol_ok) { + SetFatal( + state, stats, static_cast(task_id) + ); + return false; + } + stats.result.map_lookups += ordinary_lookup_count; + for (int32_t edge = 0; + edge < context.fanin_count; ++edge) { + stats.result.dependency_signature ^= + DependencyEdgeSignature( + task_id, + static_cast(context.fanin[edge]) + ); + } + const uint64_t fanin_end = + TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, + static_cast(task_id), + function_id, TracePhase::Fanin, + ProfilePhase::Fanin, fanin_begin, fanin_end, 0, + static_cast(context.fanin_count) + ); + build_begin = fanin_end; + } + if (kind == TaskKind::Alloc) { + CompleteTask(state, worker, task_id, stats); + } else if (!BuildWinner( + state, worker, task_id, kind, args, context, + context.fanin, + static_cast(context.fanin_count), + stats + )) { + return false; + } +#if defined(PA_TEST_SHARED_SUBMIT_HOOKS) + Ops::AfterSharedTaskBuild( + state, worker, task_id, kind + ); +#endif + const uint64_t build_end = + TraceTimestamp(stats.trace, stats.result); + WriteTrace( + stats.trace, stats.result, static_cast(task_id), + function_id, + kind == TaskKind::Alloc + ? TracePhase::AllocComplete + : TracePhase::WinnerBuild, + ProfilePhase::ReplayTail, build_begin, build_end + ); + + return CloseSharedCallbackSubmit( + state, stats, ticket, task_meta + ); +} + +#endif // PA_SCHEDULER_COMMON_PA_SHARED_SUBMIT_PATH_H diff --git a/tests/atomic_probe/pa_scheduler/common/pa_shared_tensormap.h b/tests/atomic_probe/pa_scheduler/common/pa_shared_tensormap.h new file mode 100644 index 0000000000..2171a0e145 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/common/pa_shared_tensormap.h @@ -0,0 +1,1020 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_SHARED_TENSORMAP_H +#define PA_SCHEDULER_COMMON_PA_SHARED_TENSORMAP_H + +#include "pa_frontend.h" + +namespace pa_scheduler { + +constexpr int64_t kSharedMapEmptySeq = -1; + +enum class SharedAppendCheck : uint32_t { + Ready = 0, + CapacityBlocked = 1, + ProtocolError = 2, +}; + +PA_DEVICE uint32_t SharedTensorMapSlotIndex(uint32_t bucket, uint64_t cursor) { + return bucket * kMapBucketCapacity + + (static_cast(cursor) & kMapBucketSlotMask); +} + +PA_DEVICE bool SharedRegionOverlaps( + const SharedRegionValue &left, const SharedRegionValue &right +) { + return left.buffer_addr == right.buffer_addr && + left.lo < right.hi && right.lo < left.hi; +} + +template +PA_DEVICE SharedRegionValue MakeSharedRegionValue( + const TensorReference &tensor, int32_t producer +) { + SharedRegionValue value{}; + TensorByteRange(tensor, value.buffer_addr, value.lo, value.hi); + value.producer = producer; + value.reserved = 0; + return value; +} + +// 读侧协议固定为：原子观察绝对 seq、失效 payload、拷出本地快照、再次 +// 原子观察同一 seq。任一检查失败都返回 false；上层不得把协议失败静默 +// 解释成“没有 producer”。 +template +PA_DEVICE bool SharedReadRegionSlot( + PA_GM SharedTensorMapSidecar &map, uint32_t bucket, uint64_t cursor, + SharedRegionValue &snapshot, int32_t task_id = -1, + TraceContext *trace = nullptr, WorkerResult *result = nullptr +) { + if (bucket >= kMapBuckets || cursor > static_cast(INT64_MAX)) { + return false; + } + PA_GM SharedRegionSlot &slot = + map.slots[SharedTensorMapSlotIndex(bucket, cursor)]; + const int64_t expected = static_cast(cursor); + if (TraceConfiguredAtomicLoad( + trace, result, task_id, + AtomicSite::SharedMapLookupSeqLoad, + &slot.seq.value + ) != expected) { + return false; + } + (void)TraceConfiguredDcciInvalidate( + trace, task_id, -1, + DcciSite::SharedRegionReadInvalidate, + &slot.payload, sizeof(slot.payload) + ); + snapshot.buffer_addr = slot.payload.value.buffer_addr; + snapshot.lo = slot.payload.value.lo; + snapshot.hi = slot.payload.value.hi; + snapshot.producer = slot.payload.value.producer; + snapshot.reserved = slot.payload.value.reserved; + if (TraceConfiguredAtomicLoad( + trace, result, task_id, + AtomicSite::SharedMapLookupSeqLoad, + &slot.seq.value + ) != expected) { + return false; + } + return snapshot.producer >= 0 && snapshot.reserved == 0 && + snapshot.lo < snapshot.hi; +} + +// lookup 的权威时间窗为 [current_task-H, current_task)。即使更快 winner +// 已经发布未来 entry，也不能把 producer>=current_task 引入本核 fanin。 +template +PA_DEVICE int32_t SharedLookupRegion( + PA_GM SharedTensorMapSidecar &map, const SharedRegionValue &query, + int32_t current_task, int32_t heap_window, bool &protocol_ok, + TraceContext *trace = nullptr, WorkerResult *result = nullptr +) { + protocol_ok = false; + if (current_task < 0 || heap_window < 0 || query.lo >= query.hi) { + return -1; + } + const uint32_t bucket = TensorMapHash(query.buffer_addr); + int64_t signed_head = + TraceConfiguredAtomicLoad( + trace, result, current_task, + AtomicSite::SharedMapLookupHeadLoad, + &map.buckets[bucket].head.value + ); + const int64_t signed_tail = + TraceConfiguredAtomicLoad( + trace, result, current_task, + AtomicSite::SharedMapLookupTailLoad, + &map.buckets[bucket].tail.value + ); + if (__builtin_expect( + signed_head < 0 || signed_tail < signed_head || + static_cast(signed_tail - signed_head) > + kMapBucketCapacity, + 0 + )) { + if (signed_head < 0 || signed_tail < 0) { + return -1; + } + // head/tail 是两个独立 control atomic。reader 先读旧 head 后， + // writer 可能合法回收前缀、复用槽并发布新 tail，从而让混合快照 + // 暂时呈现 span>CAP。异常支路只重读一次 head；唯有 head 单调 + // 前进后能把同一 tail 重新约束到容量内，才接受该快照。 + const int64_t refreshed_head = + TraceConfiguredAtomicLoad( + trace, result, current_task, + AtomicSite::SharedMapLookupHeadLoad, + &map.buckets[bucket].head.value + ); + if (refreshed_head < signed_head || + refreshed_head > signed_tail || + static_cast( + signed_tail - refreshed_head + ) > kMapBucketCapacity) { + return -1; + } + signed_head = refreshed_head; + } + + const int32_t lower = + current_task > heap_window ? current_task - heap_window : 0; + int32_t best = -1; + uint64_t cursor = static_cast(signed_head); + const uint64_t tail = static_cast(signed_tail); + while (cursor < tail) { + SharedRegionValue candidate{}; + if (__builtin_expect( + !SharedReadRegionSlot( + map, bucket, cursor, candidate, + current_task, trace, result + ), + 0 + )) { + // reader 保存旧 head 后，未来唯一 writer 仍可合法回收 + // producer < current_task-H 的无关前缀，并复用其物理槽。只有 + // head 已单调越过当前 cursor，才能把 seq 双检失败解释为这类 + // 合法复用；否则继续 fail-closed，不能吞掉真实 slot 损坏。 + const int64_t refreshed_head = + TraceConfiguredAtomicLoad( + trace, result, current_task, + AtomicSite::SharedMapLookupHeadLoad, + &map.buckets[bucket].head.value + ); + if (refreshed_head < signed_head || + refreshed_head > signed_tail || + cursor >= static_cast(refreshed_head)) { + return -1; + } + cursor = static_cast(refreshed_head); + continue; + } + if (candidate.producer >= lower && + candidate.producer < current_task && + SharedRegionOverlaps(candidate, query) && + candidate.producer > best) { + best = candidate.producer; + } + ++cursor; + } + protocol_ok = true; + return best; +} + +template < + typename Ops, bool ObserveAtomics = false, + typename TensorReference +> +PA_DEVICE int32_t SharedLookupTensor( + PA_GM SharedTensorMapSidecar &map, const TensorReference &tensor, + int32_t current_task, int32_t heap_window, bool &protocol_ok, + TraceContext *trace = nullptr, WorkerResult *result = nullptr +) { + const SharedRegionValue query = MakeSharedRegionValue(tensor, -1); + return SharedLookupRegion( + map, query, current_task, heap_window, protocol_ok, + trace, result + ); +} + +// append 至少要求唯一、按 task_id 单调推进的 writer；exact turn 与 +// writer-ready replay 都可建立 writer 顺序。真正推进 head 还必须由 +// reader_done 证明所有更早 reader 已结束；仅有 writer exact turn 并不能 +// 证明这一点。通用 intent 仍固定传 reclaim_upto=-1，R4e-a 只先建立 +// reader progress 状态与纯公式，不打开任何运行时回收。 +template +PA_DEVICE bool SharedRetireBucket( + PA_GM SharedTensorMapSidecar &map, uint32_t bucket, int64_t reclaim_upto +) { + if (bucket >= kMapBuckets || reclaim_upto < -1) { + return false; + } + PA_GM SharedBucketState &controls = map.buckets[bucket]; + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - reader 回收协议仅由 ring/litmus 门槛使用，当前 ordered Submit 固定 reclaim=-1 + const int64_t original_head = Ops::Load(&controls.head.value); + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - reader 回收协议仅由 ring/litmus 门槛使用，当前 ordered Submit 固定 reclaim=-1 + const int64_t tail = Ops::Load(&controls.tail.value); + if (original_head < 0 || tail < original_head || + static_cast(tail - original_head) > kMapBucketCapacity) { + return false; + } + + int64_t head = original_head; + while (head < tail) { + SharedRegionValue value{}; + if (!SharedReadRegionSlot( + map, bucket, static_cast(head), value + )) { + return false; + } + if (value.producer > reclaim_upto) { + break; + } + ++head; + } + if (head == original_head) { + return true; + } + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - reader 回收协议仅由 ring/litmus 门槛使用，当前 ordered Submit 固定 reclaim=-1 + const int64_t observed = Ops::Exchange(&controls.head.value, head); + return observed == original_head; +} + +#if PTO_FDWIC_SHARED_MAP +// reader_done[worker]=N 表示该 worker 已经关闭 task [0,N] 的全部 +// ordinary-ring 读取。CAS 只允许 N-1 -> N，重复、跳号和倒退都保留旧值； +// 调用方必须先证明本 task 后续不会再访问 ring，并在 A5 接线前另外闭合 +// “读取完成 -> CAS”的编译器/设备顺序。R4e-a 本身不从 PA 热路径调用。 +template +PA_DEVICE bool SharedAdvanceReaderDone( + PA_GM SharedTensorMapSidecar &map, uint32_t worker, + int32_t task_id +) { + if (worker >= kWorkers || task_id < 0 || + task_id >= static_cast(kMaxTasks)) { + return false; + } + const int64_t expected = + static_cast(task_id) - 1; + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - reader_done 协议仅由 ring/litmus 门槛使用，尚未接入当前 scheduler + return Ops::CompareExchange( + &map.reader_done[worker].value, + expected, static_cast(task_id) + ) == expected; +} + +// 对连续 active worker 前缀取最慢完成值 Dmin。窗口 H 允许未来 reader +// 查询 producer >= current_task-H；因此所有 worker 都完成到 Dmin 后， +// inclusive 安全回收上界是 max(-1,Dmin-H)。inactive worker 不参与最小值。 +// 任一非法进度都 fail-closed，且失败时不修改 candidate。 +template +PA_DEVICE bool SharedComputeReaderReclaimCandidate( + PA_GM SharedTensorMapSidecar &map, uint32_t active_workers, + int32_t heap_window, int64_t &candidate +) { + if (active_workers == 0 || active_workers > kWorkers || + heap_window < 0) { + return false; + } + int64_t minimum_done = INT64_MAX; + for (uint32_t worker = 0; worker < active_workers; ++worker) { + const int64_t done = + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - reader_done 回收候选仅由 ring/litmus 门槛使用 + Ops::Load(&map.reader_done[worker].value); + if (done < -1 || + done >= static_cast(kMaxTasks)) { + return false; + } + if (done < minimum_done) { + minimum_done = done; + } + } + int64_t computed = + minimum_done - static_cast(heap_window); + if (computed < -1) { + computed = -1; + } + candidate = computed; + return true; +} +#endif + +// 以下 current_task 公式只保留给既有单线程 ordered-ring 隔离 driver， +// 不能作为 generic 多 reader 回收依据。真实接线必须改用上面的最慢 +// reader_done 公式，并同时保持唯一 ordered append actor。 +PA_DEVICE bool SharedComputeOrderedReclaimCandidate( + int32_t current_task, int32_t heap_window, int64_t &candidate +) { + if (current_task < 0 || heap_window < 0) { + return false; + } + candidate = + static_cast(current_task) - + static_cast(heap_window) - 1; + if (candidate < -1) { + candidate = -1; + } + return true; +} + +PA_DEVICE uint32_t SharedInsertTurnLane(int32_t token) { + return static_cast(token) & + kSharedInsertTurnMask; +} + +PA_DEVICE PA_GM AtomicLine &SharedInsertTurnLine( + PA_GM SharedTensorMapSidecar &map, uint32_t lane +) { + // lane 0 保持原 committed_tasks 地址，G=1 不移动热点控制字。 + // 调用者只传 [0,kSharedInsertTurnCapacity)；active G 之外的物理线 + // 仅供初始化和 host 终态校验。 + if (lane == 0) { + return map.committed_tasks; + } + return map.insert_turn_extra[lane - 1U]; +} + +PA_DEVICE void InitializeSharedInsertTurns( + PA_GM SharedTensorMapSidecar &map +) { + map.committed_tasks.value = 0; + for (uint32_t lane = 1; + lane < kSharedInsertTurnCapacity; ++lane) { + map.insert_turn_extra[lane - 1U].value = -1; + } +} + +// completed_tasks=T 表示 token 0..T 已按序产生。每条 active lane 保存 +// 不大于 T 的最大同余 token；从未接收 token 的 lane 与全部 inactive +// lane 保持 -1。该纯公式供 host 和独立测试建立权威终态。 +constexpr int64_t SharedInsertTurnTokenAfterTasks( + uint32_t completed_tasks, uint32_t lane +) { + if (lane >= kSharedInsertTurnGroups) { + return -1; + } + if (lane > completed_tasks) { + return lane == 0 ? 0 : -1; + } + return static_cast( + completed_tasks - + ((completed_tasks - lane) & kSharedInsertTurnMask) + ); +} + +PA_DEVICE int64_t SharedInsertTurnPublishExpectedOld( + int32_t task_id +) { + const int64_t next = + static_cast(task_id) + 1; + return next >= + static_cast( + kSharedInsertTurnGroups + ) + ? next - + static_cast( + kSharedInsertTurnGroups + ) + : -1; +} + +enum class SharedInsertTurnState : uint32_t { + Ready = 0, + Pending = 1, + ProtocolError = 2, +}; + +// insert-turn 是不索引 SchedulerState::tasks 的绝对序列原语，隔离 ring +// 门槛会用它覆盖超过 kMaxTasks 的多圈 cursor，因此这里只限制 int32 +// 可表达性。生产 Submit 在触碰任何共享状态前另行要求 task_id 落在 +// [0,kMaxTasks)，不能把两层合同合并后破坏大 CAP 原语门槛。 +template +PA_DEVICE SharedInsertTurnState SharedInspectTaskTurnObserved( + PA_GM SharedTensorMapSidecar &map, int32_t current_task, + int64_t &observed +) { + if (current_task < 0) { + observed = -1; + return SharedInsertTurnState::ProtocolError; + } + const uint32_t lane = + SharedInsertTurnLane(current_task); + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 旧分组 insert-turn 协议仅由 ring/litmus 门槛使用 + observed = Ops::Load( + &SharedInsertTurnLine(map, lane).value + ); + int64_t compare_observed = observed; +#if PA_BUILD_SWIMLANE && \ + (defined(PA_BUILD_AIC) || defined(PA_BUILD_AIV)) + if constexpr (PreserveReadyDependency) { + // Ready 分支会向编译器透露 compare_observed == current_task。CCEC + // 因此只为真正需要计时的 wait 调用，从同一个 atomic 返回寄存器 + // 派生两个编译器不可等同的值：一个只作分支判定，一个通过 + // observed 带到 SYS_CNT 依赖边界。其他协议检查不承担这条 MOV。 + compare_observed = Ops::ForkAtomicResultForBranch( + observed, observed + ); + } +#endif + if (compare_observed == current_task) { + return SharedInsertTurnState::Ready; + } + if (compare_observed < -1 || + compare_observed > current_task) { + return SharedInsertTurnState::ProtocolError; + } + // lane 0 从初始化起始终保存非负的 0 mod G token；其他 lane 在 + // 第一次接收 token 前合法保持 -1。已发布旧 token 必须与本 lane + // 同余，否则表示初始化、越序写或地址计算已经损坏。 + if (compare_observed == -1) { + return lane == 0 + ? SharedInsertTurnState::ProtocolError + : SharedInsertTurnState::Pending; + } + if ((static_cast(compare_observed) & + kSharedInsertTurnMask) != lane) { + return SharedInsertTurnState::ProtocolError; + } + return SharedInsertTurnState::Pending; +} + +template +PA_DEVICE SharedInsertTurnState SharedInspectTaskTurn( + PA_GM SharedTensorMapSidecar &map, int32_t current_task +) { + int64_t ignored_observed = -1; + return SharedInspectTaskTurnObserved( + map, current_task, ignored_observed + ); +} + +template +PA_DEVICE bool SharedHasExactTaskTurn( + PA_GM SharedTensorMapSidecar &map, int32_t current_task +) { + return SharedInspectTaskTurn( + map, current_task + ) == SharedInsertTurnState::Ready; +} + +// current lane 的 N 是进入有序通道的 grant；G>1 时它会保留到下一代 +// token 覆盖，不能单独充当“尚未发布”的锁。生产合同由 Claim 保证 +// 每 task 只有一个 owner，同时在写任何 TensorMap 元数据前预检目标 +// lane 仍是本 task 应覆盖的旧 token，拒绝已经完成的重复调用。 +template +PA_DEVICE bool SharedCanPublishTaskCommit( + PA_GM SharedTensorMapSidecar &map, int32_t task_id +) { + if (task_id < 0 || task_id == INT32_MAX || + !SharedHasExactTaskTurn(map, task_id)) { + return false; + } + if (kSharedInsertTurnGroups == 1U) { + // current/next lane 是同一条线，exact N 已经同时证明 CAS 的 + // expected_old=N；避免 G=1 基线多做一次相同地址 load。 + return true; + } + const int32_t next = task_id + 1; + const uint32_t next_lane = + SharedInsertTurnLane(next); + const int64_t expected = + SharedInsertTurnPublishExpectedOld(task_id); + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 旧分组 insert-turn 预检仅由 ring/litmus 门槛使用 + return Ops::Load( + &SharedInsertTurnLine( + map, next_lane + ).value + ) == expected; +} + +template +PA_DEVICE bool SharedPublishTaskCommitAfterPreflightObserved( + PA_GM SharedTensorMapSidecar &map, int32_t task_id, + int64_t &observed +) { + if (task_id < 0 || task_id == INT32_MAX) { + observed = INT64_MIN; + return false; + } + const int32_t next = task_id + 1; + const uint32_t next_lane = + SharedInsertTurnLane(next); + const int64_t expected = + SharedInsertTurnPublishExpectedOld(task_id); + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - 旧分组 insert-turn 交接仅由 ring/litmus 门槛使用 + observed = Ops::CompareExchange( + &SharedInsertTurnLine( + map, next_lane + ).value, + expected, static_cast(next) + ); + return observed == expected; +} + +template +PA_DEVICE bool SharedPublishTaskCommitAfterPreflight( + PA_GM SharedTensorMapSidecar &map, int32_t task_id +) { + int64_t ignored_observed = INT64_MIN; + return SharedPublishTaskCommitAfterPreflightObserved( + map, task_id, ignored_observed + ); +} + +// reclaim_upto 只允许唯一 ordered append actor 单调发布。这里不从 exact +// turn 推导 actor 唯一性：调用方仍必须先完成 winner/turn 所有权收敛。 +// R4e-e 会在真实接线前单独比较 Exchange 与 CAS 的设备成本；当前保持 +// 既有单写者发布语义，不把性能选择混入 reader 正确性门槛。 +template +PA_DEVICE bool SharedPublishReclaimCandidate( + PA_GM SharedTensorMapSidecar &map, int64_t candidate, + int64_t &reclaim_upto +) { + if (candidate < -1) { + return false; + } + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - reclaim 发布仅由 ring/litmus 门槛使用，当前 ordered Submit 固定 -1 + const int64_t current = Ops::Load(&map.reclaim_upto.value); + if (current < -1 || candidate < current) { + return false; + } + if (candidate == current) { + reclaim_upto = current; + return true; + } + const int64_t observed = + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - reclaim 发布仅由 ring/litmus 门槛使用，当前 ordered Submit 固定 -1 + Ops::Exchange(&map.reclaim_upto.value, candidate); + if (observed != current) { + return false; + } + reclaim_upto = candidate; + return true; +} + +#if PTO_FDWIC_SHARED_MAP +// reader-based refresh 只组合 exact turn、最慢 reader 候选和既有单调发布； +// 不扫描 bucket、不 append，也没有 PA/Submit 调用者。exact turn 仍不等于 +// 唯一 actor，外层必须先证明只有一个 ordered append actor 进入本原语。 +// active_workers 必须是本轮固定的连续活跃前缀，heap_window 也必须在 ring +// 整个生命周期保持同一权威值；任一参数中途缩小都可能发布不可撤回的过激 +// 候选，随后再恢复真实配置已经无法找回被回收的槽。 +template +PA_DEVICE bool SharedRefreshReaderReclaimForTask( + PA_GM SharedTensorMapSidecar &map, int32_t current_task, + uint32_t active_workers, int32_t heap_window, + int64_t &reclaim_upto +) { + if (!SharedHasExactTaskTurn(map, current_task)) { + return false; + } + int64_t candidate = -1; + if (!SharedComputeReaderReclaimCandidate( + map, active_workers, heap_window, candidate + )) { + return false; + } + return SharedPublishReclaimCandidate( + map, candidate, reclaim_upto + ); +} +#endif + +// 只有 exact committed turn 的 winner 可以推进 reclaim。先验证 +// committed_tasks==current_task，再计算当前任务的 inclusive 回收边界； +// 逆序/陈旧 actor 在任何 head、tail 或 reclaim 写入前失败。该旧路径只 +// 服务单线程 ordered-ring 隔离 driver；generic 多 reader 使用上面的 +// reader-based refresh。 +template +PA_DEVICE bool SharedRefreshReclaimForTask( + PA_GM SharedTensorMapSidecar &map, int32_t current_task, + int32_t heap_window, + int64_t &reclaim_upto +) { + if (!SharedHasExactTaskTurn(map, current_task)) { + return false; + } + int64_t candidate = -1; + if (!SharedComputeOrderedReclaimCandidate( + current_task, heap_window, candidate + )) { + return false; + } + return SharedPublishReclaimCandidate( + map, candidate, reclaim_upto + ); +} + +PA_DEVICE uint32_t SharedEarlierEntriesInBucket( + const SharedRegionValue *entries, uint32_t index, uint32_t bucket +) { + uint32_t earlier = 0; + for (uint32_t previous = 0; previous < index; ++previous) { + if (TensorMapHash(entries[previous].buffer_addr) == bucket) { + ++earlier; + } + } + return earlier; +} + +// 在写任何 slot 前完成整任务容量、目标 seq 与 cursor 检查。容量不足时 +// 当前任务的 payload/seq/tail/commit 都不发布；检查期间按已发布边界推进 +// 的陈旧 head 可以保留。随后 append 失败只可能是协议破坏，调用方应 fatal。 +template +PA_DEVICE SharedAppendCheck SharedCheckTaskAppend( + PA_GM SharedTensorMapSidecar &map, const SharedRegionValue *entries, + uint32_t count, int64_t reclaim_upto +) { + if (count > kMaxTaskTensors || reclaim_upto < -1) { + return SharedAppendCheck::ProtocolError; + } + for (uint32_t index = 0; index < count; ++index) { + const SharedRegionValue &entry = entries[index]; + if (entry.producer < 0 || entry.reserved != 0 || + entry.lo >= entry.hi) { + return SharedAppendCheck::ProtocolError; + } + const uint32_t bucket = TensorMapHash(entry.buffer_addr); + // 当前 ordered Submit 固定传 -1，明确表示不回收任何合法 + // producer。该热路径无需读取旧 head slot、invalidate payload 和 + // 双检 seq；下面仍完整执行 head/tail、容量和目标 seq 的 + // fail-closed 预检。非负回收边界继续使用原 retire 协议。 + if (reclaim_upto != -1 && + !SharedRetireBucket( + map, bucket, reclaim_upto + )) { + return SharedAppendCheck::ProtocolError; + } + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - generic append preflight 仅由旧 writer-intent 与 ring/litmus 门槛使用 + const int64_t head = Ops::Load(&map.buckets[bucket].head.value); + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - generic append preflight 仅由旧 writer-intent 与 ring/litmus 门槛使用 + const int64_t tail = Ops::Load(&map.buckets[bucket].tail.value); + if (head < 0 || tail < head) { + return SharedAppendCheck::ProtocolError; + } + const uint32_t earlier = + SharedEarlierEntriesInBucket(entries, index, bucket); + const uint64_t occupied = + static_cast(tail - head) + earlier; + if (occupied >= kMapBucketCapacity) { + return SharedAppendCheck::CapacityBlocked; + } + const uint64_t cursor = static_cast(tail) + earlier; + // 该 entry 发布后 tail 必须递增；cursor==INT64_MAX 也没有 + // 可表达的 next tail，必须在触碰 slot 前拒绝。 + if (cursor >= static_cast(INT64_MAX)) { + return SharedAppendCheck::ProtocolError; + } + PA_GM SharedRegionSlot &slot = + map.slots[SharedTensorMapSlotIndex(bucket, cursor)]; + const int64_t expected_old = + cursor < kMapBucketCapacity + ? kSharedMapEmptySeq + : static_cast(cursor - kMapBucketCapacity); + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - generic append preflight 仅由旧 writer-intent 与 ring/litmus 门槛使用 + if (Ops::Load(&slot.seq.value) != expected_old) { + return SharedAppendCheck::ProtocolError; + } + } + return SharedAppendCheck::Ready; +} + +// 正式 ordered Submit 专用预检。bucket 与同 bucket 的局部序号已经由 +// winner 在等待 predecessor 前算好；取得 insert turn 后只读取共享 +// cursor/seq，不再重复 hash，也不再扫描此前 entry。prepared 数组是 +// owner-local 不可变 delta 的一部分，不能把任意外部输入直接传入这里。 +template +PA_DEVICE SharedAppendCheck SharedCheckPreparedTaskAppend( + PA_GM SharedTensorMapSidecar &map, + const SharedRegionValue *entries, const uint16_t *buckets, + const uint8_t *bucket_ordinals, uint32_t count, + int64_t reclaim_upto, int32_t task_id = -1, + TraceContext *trace = nullptr, WorkerResult *result = nullptr +) { + if (count > kMaxTaskTensors || reclaim_upto < -1 || + (count != 0 && + (entries == nullptr || buckets == nullptr || + bucket_ordinals == nullptr))) { + return SharedAppendCheck::ProtocolError; + } + for (uint32_t index = 0; index < count; ++index) { + const SharedRegionValue &entry = entries[index]; + const uint32_t bucket = buckets[index]; + const uint32_t ordinal = bucket_ordinals[index]; + if (entry.producer < 0 || entry.reserved != 0 || + entry.lo >= entry.hi || bucket >= kMapBuckets || + ordinal > index) { + return SharedAppendCheck::ProtocolError; + } + // 同一 bucket 在一份 task delta 中只需要回收一次。正式路径当前 + // 固定 -1，因而连第一次也跳过；保留非负参数是为了与 generic + // preflight 共用相同的容量/seq 合同。 + if (ordinal == 0 && reclaim_upto != -1 && + !SharedRetireBucket( + map, bucket, reclaim_upto + )) { + return SharedAppendCheck::ProtocolError; + } + const int64_t head = + TraceConfiguredAtomicLoad( + trace, result, task_id, + AtomicSite::SharedMapAppendHeadLoad, + &map.buckets[bucket].head.value + ); + const int64_t tail = + TraceConfiguredAtomicLoad( + trace, result, task_id, + AtomicSite::SharedMapAppendTailLoad, + &map.buckets[bucket].tail.value + ); + if (head < 0 || tail < head) { + return SharedAppendCheck::ProtocolError; + } + const uint64_t occupied = + static_cast(tail - head) + ordinal; + if (occupied >= kMapBucketCapacity) { + return SharedAppendCheck::CapacityBlocked; + } + const uint64_t cursor = + static_cast(tail) + ordinal; + if (cursor >= static_cast(INT64_MAX)) { + return SharedAppendCheck::ProtocolError; + } + PA_GM SharedRegionSlot &slot = + map.slots[SharedTensorMapSlotIndex(bucket, cursor)]; + const int64_t expected_old = + cursor < kMapBucketCapacity + ? kSharedMapEmptySeq + : static_cast( + cursor - kMapBucketCapacity + ); + if (TraceConfiguredAtomicLoad( + trace, result, task_id, + AtomicSite::SharedMapAppendSeqLoad, + &slot.seq.value + ) != expected_old) { + return SharedAppendCheck::ProtocolError; + } + } + return SharedAppendCheck::Ready; +} + +template +PA_DEVICE bool SharedPreflightTaskAppend( + PA_GM SharedTensorMapSidecar &map, const SharedRegionValue *entries, + uint32_t count, int64_t reclaim_upto +) { + return SharedCheckTaskAppend( + map, entries, count, reclaim_upto + ) == SharedAppendCheck::Ready; +} + +template +PA_DEVICE bool SharedAppendPreparedEntryAtBucket( + PA_GM SharedTensorMapSidecar &map, + const SharedRegionValue &entry, uint32_t bucket, + int32_t task_id = -1, TraceContext *trace = nullptr, + WorkerResult *result = nullptr +) { + if (bucket >= kMapBuckets) { + return false; + } + PA_GM SharedBucketState &controls = map.buckets[bucket]; + const int64_t head = + TraceConfiguredAtomicLoad( + trace, result, task_id, + AtomicSite::SharedMapAppendHeadLoad, + &controls.head.value + ); + const int64_t tail = + TraceConfiguredAtomicLoad( + trace, result, task_id, + AtomicSite::SharedMapAppendTailLoad, + &controls.tail.value + ); + if (head < 0 || tail < head || + static_cast(tail - head) >= kMapBucketCapacity) { + return false; + } + if (tail == INT64_MAX) { + return false; + } + const uint64_t cursor = static_cast(tail); + PA_GM SharedRegionSlot &slot = + map.slots[SharedTensorMapSlotIndex(bucket, cursor)]; + const int64_t expected_old = + cursor < kMapBucketCapacity + ? kSharedMapEmptySeq + : static_cast(cursor - kMapBucketCapacity); + const int64_t invalidated = + TraceConfiguredAtomicExchange( + trace, result, task_id, + AtomicSite::SharedMapAppendSeqResetExchange, + &slot.seq.value, kSharedMapEmptySeq, + /*result_used=*/true + ); + if (invalidated != expected_old) { + return false; + } + + (void)TraceConfiguredDcciInvalidate( + trace, task_id, -1, + DcciSite::SharedRegionAppendInvalidate, + &slot.payload, sizeof(slot.payload) + ); + slot.payload.value.buffer_addr = entry.buffer_addr; + slot.payload.value.lo = entry.lo; + slot.payload.value.hi = entry.hi; + slot.payload.value.producer = entry.producer; + slot.payload.value.reserved = 0; + // padding 不承载协议字段，也没有 reader/host 消费者；只写完整的 + // 32B value，避免为 cache-line 填充字节增加无意义的 scalar store。 + (void)TraceConfiguredDcciFlush( + trace, task_id, -1, + DcciSite::SharedRegionAppendFlush, + &slot.payload, sizeof(slot.payload) + ); + + const int64_t before_publish = + TraceConfiguredAtomicExchange( + trace, result, task_id, + AtomicSite::SharedMapAppendSeqPublishExchange, + &slot.seq.value, static_cast(cursor), + /*result_used=*/true + ); + if (before_publish != kSharedMapEmptySeq) { + return false; + } + const int64_t previous_tail = + TraceConfiguredAtomicExchange( + trace, result, task_id, + AtomicSite::SharedMapAppendTailExchange, + &controls.tail.value, tail + 1, + /*result_used=*/true + ); + return previous_tail == tail; +} + +template +PA_DEVICE bool SharedAppendPreparedEntry( + PA_GM SharedTensorMapSidecar &map, + const SharedRegionValue &entry, int32_t task_id = -1, + TraceContext *trace = nullptr, WorkerResult *result = nullptr +) { + return SharedAppendPreparedEntryAtBucket< + Ops, ObserveAtomics + >( + map, entry, TensorMapHash(entry.buffer_addr), + task_id, trace, result + ); +} + +template +PA_DEVICE bool SharedAppendPreparedTask( + PA_GM SharedTensorMapSidecar &map, const SharedRegionValue *entries, + uint32_t count, int32_t task_id = -1, + TraceContext *trace = nullptr, WorkerResult *result = nullptr +) { + for (uint32_t index = 0; index < count; ++index) { + if (!SharedAppendPreparedEntry( + map, entries[index], task_id, trace, result + )) { + return false; + } + } + return true; +} + +template +PA_DEVICE bool SharedAppendPreparedTask( + PA_GM SharedTensorMapSidecar &map, + const SharedRegionValue *entries, const uint16_t *buckets, + uint32_t count, int32_t task_id = -1, + TraceContext *trace = nullptr, WorkerResult *result = nullptr +) { + if (count > kMaxTaskTensors || + (count != 0 && + (entries == nullptr || buckets == nullptr))) { + return false; + } + for (uint32_t index = 0; index < count; ++index) { + if (!SharedAppendPreparedEntryAtBucket< + Ops, ObserveAtomics + >( + map, entries[index], buckets[index], + task_id, trace, result + )) { + return false; + } + } + return true; +} + +#if PTO_FDWIC_SHARED_MAP +// 在调用本组合前，当前 worker 必须已经完成本 task 的全部 ordinary +// lookup，并用 SharedAdvanceReaderDone() 恰好关闭一次 reader；append +// actor 还必须由外层 writer-ready replay 保证唯一且按 task id 有序。 +// +// 本组合不读取/推进 committed_tasks。正常路径先复用已发布的 +// reclaim_upto 做整 task 预检，Ready 时直接 append，不扫描 reader_done； +// 只有容量不足才按固定 active-worker 前缀刷新 reader 回收边界并重试： +// - Ready：整批预检和 append 均已成功； +// - CapacityBlocked：当前 task 没有发布 payload/seq/tail，可在其他 +// reader 前进后重试本函数；安全过期的 head/reclaim 可以保留； +// - ProtocolError：调用层必须终止本轮，不得发布 writer-ready。 +// append 阶段若遭遇预检后协议破坏，可能已经发布物理前缀；唯一 writer +// 合同下这不是合法竞争，不能尝试回滚。 +template +PA_DEVICE SharedAppendCheck SharedTryAppendReaderGatedTask( + PA_GM SharedTensorMapSidecar &map, + const SharedRegionValue *entries, uint32_t count, + uint32_t active_workers, int32_t heap_window +) { + if (entries == nullptr && count != 0) { + return SharedAppendCheck::ProtocolError; + } + // 纯 symbol writer 没有 ordinary entry，不应为一个空 batch 读取 + // reclaim 或扫描 reader 前沿。 + if (count == 0) { + return SharedAppendCheck::Ready; + } + if (active_workers == 0 || active_workers > kWorkers || + heap_window < 0) { + return SharedAppendCheck::ProtocolError; + } + int64_t reclaim_upto = + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - reader-gated append 组合尚未接入当前 scheduler + Ops::Load(&map.reclaim_upto.value); + if (reclaim_upto < -1) { + return SharedAppendCheck::ProtocolError; + } + SharedAppendCheck check = + SharedCheckTaskAppend( + map, entries, count, reclaim_upto + ); + if (check == SharedAppendCheck::ProtocolError) { + return check; + } + if (check == SharedAppendCheck::Ready) { + return SharedAppendPreparedTask( + map, entries, count + ) + ? SharedAppendCheck::Ready + : SharedAppendCheck::ProtocolError; + } + + int64_t candidate = -1; + if (!SharedComputeReaderReclaimCandidate( + map, active_workers, heap_window, candidate + )) { + return SharedAppendCheck::ProtocolError; + } + if (!SharedPublishReclaimCandidate( + map, candidate, reclaim_upto + )) { + return SharedAppendCheck::ProtocolError; + } + check = + SharedCheckTaskAppend( + map, entries, count, reclaim_upto + ); + if (check != SharedAppendCheck::Ready) { + return check; + } + return SharedAppendPreparedTask(map, entries, count) + ? SharedAppendCheck::Ready + : SharedAppendCheck::ProtocolError; +} +#endif + +template +PA_DEVICE bool SharedPublishTaskCommit( + PA_GM SharedTensorMapSidecar &map, int32_t task_id +) { + if (kSharedInsertTurnGroups == 1U) { + // 保持既有 G=1 原语恰好一次 CAS、零预检 load 的事件形状； + // CAS(N,N+1) 本身即可拒绝重复、陈旧和 future actor。 + return SharedPublishTaskCommitAfterPreflight( + map, task_id + ); + } + if (!SharedCanPublishTaskCommit( + map, task_id + )) { + return false; + } + return SharedPublishTaskCommitAfterPreflight( + map, task_id + ); +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_SHARED_TENSORMAP_H diff --git a/tests/atomic_probe/pa_scheduler/common/pa_trace.h b/tests/atomic_probe/pa_scheduler/common/pa_trace.h new file mode 100644 index 0000000000..74d2ab91ee --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/common/pa_trace.h @@ -0,0 +1,1536 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_PA_TRACE_H +#define PA_SCHEDULER_COMMON_PA_TRACE_H + +#include "pa_model.h" + +#ifndef PA_DEVICE_NOINLINE +#define PA_DEVICE_NOINLINE PA_DEVICE +#endif + +#ifndef PA_LOOP_NOUNROLL +#define PA_LOOP_NOUNROLL +#endif + +namespace pa_scheduler { + +// 记录区与真实 PA 一样直接拼在定长 Header 后面；worker_id 只选择自己的 +// records 分区，避免记录动作本身制造跨核共享写热点。 +PA_DEVICE PA_GM TraceStorageRecord *GetTraceRecords( + PA_GM TraceHeader *header, uint32_t worker_id +) { + return reinterpret_cast( + reinterpret_cast(header) + + sizeof(TraceHeader) + + static_cast(worker_id) * kTraceWorkerBytes + + kTraceSubmitClaimBytesPerCore + ); +} + +#if PTO_FDWIC_SHARED_MAP +PA_DEVICE PA_GM SharedSubmitClaimTraceRecord * +GetSharedSubmitClaimRecords(PA_GM TraceStorageRecord *records) { + // shared 每 worker 分区内，32B Submit/Claim 区紧邻在通用记录区之前。 + return reinterpret_cast< + PA_GM SharedSubmitClaimTraceRecord * + >( + reinterpret_cast(records) - + kTraceSubmitClaimBytesPerCore + ); +} +#endif + +struct AtomicPollBurst { + uint64_t start_cycle[kAtomicPollBatchSiteCount]; + uint32_t call_count[kAtomicPollBatchSiteCount]; + uint32_t active_mask; + uint32_t enabled_mask; +}; + +struct TraceContext { + PA_GM TraceCoreState *core; + PA_GM TraceStorageRecord *records; + uint32_t capacity; + bool atomics_enabled; + int32_t lane; + int32_t block_id; + int32_t core_idx; + // 本 worker 的记录槽位只由本 scalar 写。把 count/dropped 留在本地， + // kernel 末尾再一次性发布到 core-state，避免每写一条 32B record 都 + // 额外读写另一条 GM cache line。 + uint32_t record_count; + uint32_t dropped_records; + // 轮询调用数留在 worker 私有上下文，最终一次性发布到 core state； + // 等待热路不为计数再写共享/GM 状态。 + uint64_t poll_calls; + uint64_t poll_batch_records; + bool atomic_counter_overflow; + // DCCI 区域观察与 Atomic 完全独立：calls 可以因 terminal observer + // 聚合而大于 records，lines 是所有区域实际覆盖的 64B 行数。 + uint64_t dcci_calls; + uint64_t dcci_lines; + uint64_t dcci_records; + bool dcci_counter_overflow; + AtomicPollBurst poll_burst; +}; + +PA_DEVICE bool AtomicSwimlaneEnabled(const TraceContext &trace) { +#if PA_BUILD_ATOMIC_SWIMLANE + (void)trace; + return true; +#else + return trace.atomics_enabled; +#endif +} + +PA_DEVICE bool TraceStorageAttached(const TraceContext &trace) { +#if PA_BUILD_ATOMIC_SWIMLANE + // 专用完整泳道构建在调度入口只校验一次 AttachTrace 结果；通过后， + // 每条阶段/Atomic/DCCI raw 不再重复读取三个不变量。 + (void)trace; + return true; +#else + return trace.core != nullptr && trace.records != nullptr && + trace.capacity != 0; +#endif +} + +// pa_model.h 也向 host 暴露同一 raw ABI 映射，但 CCEC/AscendC 的单个 TU +// 会先以 host 语境包含该头，再实例化 device 调度器。这里保留明确的 device +// 版本，避免设备函数误调用先前已实例化的 __host__ helper。 +PA_DEVICE AtomicOp TraceAtomicSiteExpectedOp(AtomicSite site) { + switch (site) { + case AtomicSite::StartupIncrement: + case AtomicSite::ReplayDoneIncrement: + case AtomicSite::SharedHeapCursorReserve: + case AtomicSite::SharedHeapVendAdvance: + return AtomicOp::FetchAdd; + case AtomicSite::FatalSet: + case AtomicSite::CompletionVendExchange: + case AtomicSite::CompletionFlagExchange: + return AtomicOp::Exchange; + case AtomicSite::ClaimMax: + case AtomicSite::FrontierMax: + return AtomicOp::FetchMax; + case AtomicSite::SharedInsertTurnHandoff: + case AtomicSite::SharedMetadataLastWriterCommit: + return AtomicOp::CompareExchange; + case AtomicSite::SharedOutputWriterReserve: + return AtomicOp::FetchMax; + case AtomicSite::SharedOutputPublishedExchange: + case AtomicSite::SharedMapAppendSeqResetExchange: + case AtomicSite::SharedMapAppendSeqPublishExchange: + case AtomicSite::SharedMapAppendTailExchange: + case AtomicSite::SharedOutputRollbackExchange: + return AtomicOp::Exchange; + default: + return AtomicOp::Load; + } +} + +PA_DEVICE int32_t TraceAtomicPollBatchIndex(AtomicSite site) { + switch (site) { + case AtomicSite::StartupPoll: + return 0; + case AtomicSite::FatalPoll: + return 1; + case AtomicSite::FaninFlagLoad: + return 2; + case AtomicSite::HeapFrontierLoad: + return 3; + case AtomicSite::HeapVendLoad: + return 4; + case AtomicSite::ReplayDonePoll: + return 5; + default: + return -1; + } +} + +PA_DEVICE AtomicSite TraceAtomicPollBatchSite(uint32_t index) { + switch (index) { + case 0: + return AtomicSite::StartupPoll; + case 1: + return AtomicSite::FatalPoll; + case 2: + return AtomicSite::FaninFlagLoad; + case 3: + return AtomicSite::HeapFrontierLoad; + case 4: + return AtomicSite::HeapVendLoad; + case 5: + return AtomicSite::ReplayDonePoll; + default: + return AtomicSite::Count; + } +} + +PA_DEVICE bool TraceAtomicSiteIsPollBatchable(AtomicSite site) { + return TraceAtomicPollBatchIndex(site) >= 0 || + site == AtomicSite::SharedInsertTurnPoll; +} + +// kernel.cpp 会先在 host 语境经 winner_workload.h 包含 pa_model.h。 +// 与 Atomic/DCCI 映射相同，设备侧不能调用那次已经实例化的 host helper， +// 因此在本设备头中保留同一 packed ABI 的明确实现。 +PA_DEVICE bool TraceCompactFieldsFit( + int32_t task_id, int32_t function_id, + TracePhase phase, uint32_t auxiliary +) { + return task_id >= -1 && + task_id < static_cast(kMaxTasks) && + function_id >= -1 && function_id <= 3 && + static_cast(phase) < + static_cast(TracePhase::Count) && + auxiliary <= kCompactTraceAuxiliaryMask; +} + +PA_DEVICE uint32_t TracePackCompactFields( + int32_t task_id, int32_t function_id, + TracePhase phase, uint32_t auxiliary +) { + return + (static_cast(task_id) & + kCompactTraceTaskMask) | + ((static_cast(function_id) & + kCompactTraceFunctionMask) + << kCompactTraceFunctionShift) | + (static_cast(phase) << + kCompactTracePhaseShift) | + (auxiliary << kCompactTraceAuxiliaryShift); +} + +PA_DEVICE uint32_t TraceAtomicPollBatchMask(AtomicSite site) { + const int32_t index = TraceAtomicPollBatchIndex(site); + return index >= 0 && index < 32 ? 1U << static_cast(index) : 0U; +} + +// Attach 只缓存本 worker 的 header 状态、分区首址和物理 lane 信息。控制区 +// cache invalidate 在公共调度入口完成，不能随 submit-pmu 编译掉泳道而消失。 +template +PA_DEVICE TraceContext AttachTrace( + PA_GM SchedulerState *state, PA_GM const WorkerState &worker, uint32_t worker_id +) { + TraceContext trace{}; + trace.lane = worker.lane; + trace.block_id = worker.block_id; + trace.core_idx = static_cast(worker_id); +#if PA_BUILD_TRACE_FREE + // 诊断 ELF 不含 records 写入、atomic span 或 trace-only SYS_CNT；保留同一 + // TraceContext 形状只是为了复用调度协议源码。 + (void)state; + return trace; +#else + const uint64_t base = state->config.trace_base; + const uint32_t capacity = state->config.trace_records_per_core; + if ((state->config.trace_enabled & kTracePhasesEnabled) == 0 || base == 0 || capacity == 0 || + worker_id >= kWorkers) { + return trace; + } + PA_GM TraceHeader *header = reinterpret_cast(base); + if (header->magic != 0x4653574cU || header->version != 5 || + header->record_size_bytes != kTraceRecordSizeBytes || + header->records_per_core != capacity || + worker_id >= header->num_cores) { + return trace; + } + trace.core = &header->cores[worker_id]; + trace.records = GetTraceRecords(header, worker_id); + // 成功返回的不变量是 core/records/capacity 同时有效；任一前置条件失败则三者 + // 保持空值，后续 WriteTrace/FlushTraceCore 可无分支地安全退化为 no-op。 + trace.capacity = capacity; + trace.atomics_enabled = (state->config.trace_enabled & kTraceAtomicsEnabled) != 0; + // 所有 core-state 字段都等到 FlushTraceCore 一次性发布。记录期间只改 + // TraceContext 本地计数，避免启动和结束各把这条 64B GM 行弄脏一次。 + return trace; +#endif +} + +template +PA_DEVICE void WriteTrace( + TraceContext &trace, WorkerResult &result, int32_t task_id, int32_t function_id, + TracePhase trace_phase, ProfilePhase profile_phase, uint64_t start_cycle, uint64_t end_cycle, + uint32_t flags = 0, uint32_t auxiliary = 0 +); + +// CCEC 不让栈上的 TraceContext/WorkerResult 引用跨非内联调用。这里仅把 +// PollBatch 固定形状的 32-byte GM 写入抽成共享函数，以抑制各 phase 边界 +// 内联后的代码膨胀；参数只有 GM 指针与标量，局部 batch 状态仍由调用者维护。 +#if !PA_BUILD_TRACE_FREE +PA_DEVICE bool ReserveTraceRecord( + TraceContext &trace, uint32_t &slot +) { + // 调用点已经分别由 attached trace 或 atomics_enabled 门控；这里是 + // 每条 raw 都会经过的热路，只保留真正可能变化的容量判断。 + if (__builtin_expect( + trace.record_count >= trace.capacity, 0 + )) { + if (trace.dropped_records != UINT32_MAX) { + ++trace.dropped_records; + } + return false; + } + slot = trace.record_count++; + return true; +} + +PA_DEVICE void WriteGenericTraceRecordRaw( + PA_GM TraceStorageRecord *records, uint32_t slot, + uint64_t start_cycle, uint64_t end_cycle, + int32_t task_id, int32_t function_id, + TracePhase phase, uint32_t flags, uint32_t auxiliary +) { +#if PA_BUILD_COMPACT_GENERIC_TRACE + PA_GM CompactTraceRecord16 &record = records[slot]; + record.start_cycle_low = static_cast(start_cycle); + record.end_cycle_low = static_cast(end_cycle); + record.flags = flags; + record.packed = TracePackCompactFields( + task_id, function_id, phase, auxiliary + ); +#else + PA_GM TraceRecord &record = records[slot]; + record.start_cycle = start_cycle; + record.end_cycle = end_cycle; + record.task_id = task_id; + record.function_id = function_id; + record.flags = flags; + record.phase = static_cast(phase); + record.auxiliary = static_cast(auxiliary); +#endif +} + +PA_DEVICE_NOINLINE bool WritePollBatchRecordRaw( + PA_GM TraceStorageRecord *records, uint32_t slot, + uint64_t start_cycle, uint64_t end_cycle, uint32_t call_count, + uint32_t site_id, bool return_ready_end = false +) { + const AtomicSite site = static_cast(site_id); + WriteGenericTraceRecordRaw( + records, slot, start_cycle, end_cycle, + -1, -1, TracePhase::Atomic, + static_cast(TraceAtomicSiteExpectedOp(site)) | + kAtomicResultUsed | kAtomicPollBatch | + (return_ready_end ? kAtomicReturnReady : 0U) | + (call_count << kAtomicPollCountShift), + site_id + ); + return true; +} + +#endif + +PA_DEVICE DcciOp TraceDcciSiteExpectedOp(DcciSite site) { + switch (site) { + case DcciSite::SharedWriterHistoryFlush: + case DcciSite::SharedOutputRollbackFlush: + case DcciSite::SharedOutputDescriptorFlush: + case DcciSite::SharedRegionAppendFlush: + case DcciSite::ObserverTraceExport: + return DcciOp::CleanOut; + default: + return DcciOp::Invalidate; + } +} + +PA_DEVICE uint32_t DcciTraceFlags( + DcciOp op, bool trailing_dsb, + uint32_t call_count, uint32_t line_count +) { + return static_cast(op) | + (trailing_dsb ? kDcciTrailingDsb : 0U) | + (call_count << kDcciCallCountShift) | + (line_count << kDcciLineCountShift); +} + +#if !PA_BUILD_TRACE_FREE +// 返回落盘 slot；-1 表示没有可用记录位。该 helper 只写 raw，不修改 +// logical 计数，使普通单调用和 observer 两调用聚合共用同一物理格式。 +PA_DEVICE_NOINLINE int32_t WriteDcciRecordRaw( + PA_GM TraceStorageRecord *records, uint32_t slot, + int32_t task_id, int32_t function_id, + DcciSite site, DcciOp op, bool trailing_dsb, + uint32_t call_count, uint32_t line_count, + uint64_t start_cycle, uint64_t end_cycle +) { + WriteGenericTraceRecordRaw( + records, slot, start_cycle, end_cycle, + task_id, function_id, TracePhase::Dcci, + DcciTraceFlags( + op, trailing_dsb, call_count, line_count + ), + static_cast(site) + ); + return static_cast(slot); +} +#endif + +PA_DEVICE int32_t AppendDcciTrace( + TraceContext &trace, int32_t task_id, int32_t function_id, + DcciSite site, DcciOp op, bool trailing_dsb, + uint32_t call_count, uint32_t line_count, + uint64_t start_cycle, uint64_t end_cycle +) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)task_id; + (void)function_id; + (void)site; + (void)op; + (void)trailing_dsb; + (void)call_count; + (void)line_count; + (void)start_cycle; + (void)end_cycle; + return -1; +#else + if (!TraceStorageAttached(trace)) { + return -1; + } + const bool shape_valid = + static_cast(site) < + static_cast(DcciSite::Count) && + static_cast(op) < + static_cast(DcciOp::Count) && + op == TraceDcciSiteExpectedOp(site) && + call_count != 0 && call_count <= kDcciCallCountMask && + line_count >= call_count && + line_count <= kDcciLineCountMax && + end_cycle >= start_cycle +#if PA_BUILD_COMPACT_GENERIC_TRACE + && TraceCompactFieldsFit( + task_id, function_id, TracePhase::Dcci, + static_cast(site) + ) +#endif + ; + const bool counters_fit = + trace.dcci_calls <= UINT64_MAX - call_count && + trace.dcci_lines <= UINT64_MAX - line_count && + trace.dcci_records != UINT64_MAX; + if (!shape_valid || !counters_fit) { + trace.dcci_counter_overflow = true; + return -1; + } + trace.dcci_calls += call_count; + trace.dcci_lines += line_count; + uint32_t reserved_slot = 0; + if (!ReserveTraceRecord(trace, reserved_slot)) { + return -1; + } + const int32_t slot = WriteDcciRecordRaw( + trace.records, reserved_slot, + task_id, function_id, site, op, trailing_dsb, + call_count, line_count, start_cycle, end_cycle + ); + if (slot >= 0) { + ++trace.dcci_records; + } + return slot; +#endif +} + +PA_DEVICE bool WriteDcciTrace( + TraceContext &trace, int32_t task_id, int32_t function_id, + DcciSite site, DcciOp op, bool trailing_dsb, + uint32_t line_count, + uint64_t start_cycle, uint64_t end_cycle +) { + return AppendDcciTrace( + trace, task_id, function_id, site, op, + trailing_dsb, 1, line_count, + start_cycle, end_cycle + ) >= 0; +} + +template +PA_DEVICE uint32_t DcciRegionCacheLineCount( + Pointer address, uint64_t bytes +) { + if (address == nullptr || bytes == 0) { + return 0; + } + const uint64_t begin = static_cast( + reinterpret_cast(address) + ); + if (bytes > UINT64_MAX - begin || + begin + bytes > UINT64_MAX - 63U) { + return 0; + } + const uint64_t aligned_begin = begin & ~UINT64_C(63); + const uint64_t aligned_end = (begin + bytes + 63U) & ~UINT64_C(63); + const uint64_t lines = (aligned_end - aligned_begin) / 64U; + return lines == 0 || lines > kDcciLineCountMax + ? 0 + : static_cast(lines); +} + +// 区域级观察只在原 DCCI 前后各取一次时钟，并且无论区域覆盖多少条 +// cache line 都只写一条 generic 物理记录（当前构建为 16B 或 32B）。 +// begin/end 可选回传给已经存在的业务 detail span，避免同一 +// FlushRegion 为两套观察重复读取 SYS_CNT。 +template < + typename Ops, bool ObserveDcci, bool IsInvalidate, + typename Pointer +> +PA_DEVICE uint64_t TraceConfiguredDcciRegion( + TraceContext *trace, int32_t task_id, int32_t function_id, + DcciSite site, Pointer address, uint64_t bytes, + uint64_t *begin_out = nullptr, uint64_t begin_override = 0 +) { + constexpr DcciOp op = + IsInvalidate ? DcciOp::Invalidate : DcciOp::CleanOut; +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)task_id; + (void)function_id; + (void)site; + (void)begin_override; + if (begin_out != nullptr) { + *begin_out = 0; + } + if constexpr (IsInvalidate) { + Ops::InvalidateRegion(address, bytes); + } else { + Ops::FlushRegion(address, bytes); + } + return 0; +#else + if constexpr (!ObserveDcci) { + (void)trace; + (void)task_id; + (void)function_id; + (void)site; + (void)begin_override; + if (begin_out != nullptr) { + *begin_out = 0; + } + if constexpr (IsInvalidate) { + Ops::InvalidateRegion(address, bytes); + } else { + Ops::FlushRegion(address, bytes); + } + return 0; + } else { + const uint32_t line_count = + DcciRegionCacheLineCount(address, bytes); + const bool observable = + trace != nullptr && trace->core != nullptr && + trace->records != nullptr && trace->capacity != 0; + const uint64_t begin = observable + ? (begin_override != 0 ? begin_override : Ops::Now()) + : 0; + if (begin_out != nullptr) { + *begin_out = begin; + } + if constexpr (IsInvalidate) { + Ops::InvalidateRegion(address, bytes); + } else { + Ops::FlushRegion(address, bytes); + } + if (!observable) { + return 0; + } + const uint64_t end = Ops::Now(); + if (line_count == 0) { + trace->dcci_counter_overflow = true; + return end; + } + (void)WriteDcciTrace( + *trace, task_id, function_id, site, op, + /*trailing_dsb=*/true, line_count, begin, end + ); + return end; + } +#endif +} + +template +PA_DEVICE uint64_t TraceConfiguredDcciInvalidate( + TraceContext *trace, int32_t task_id, int32_t function_id, + DcciSite site, Pointer address, uint64_t bytes, + uint64_t *begin_out = nullptr, uint64_t begin_override = 0 +) { + return TraceConfiguredDcciRegion( + trace, task_id, function_id, site, + address, bytes, begin_out, begin_override + ); +} + +template +PA_DEVICE uint64_t TraceConfiguredDcciFlush( + TraceContext *trace, int32_t task_id, int32_t function_id, + DcciSite site, Pointer address, uint64_t bytes, + uint64_t *begin_out = nullptr, uint64_t begin_override = 0 +) { + return TraceConfiguredDcciRegion( + trace, task_id, function_id, site, + address, bytes, begin_out, begin_override + ); +} + +// 启动控制区必须先经 DCCI 才能读取 trace_base/trace_enabled，因此不能 +// 在原语执行时访问 TraceContext。泳道构建只保存两端点，握手成功并 +// AttachTrace 后再补写一条 StartupConfigInvalidate；无观察构建严格 +// 退化为原始 DCCI，不增加 SYS_CNT 或 cache-line 计算。 +template +PA_DEVICE void CapturePreAttachDcciInvalidate( + Pointer address, uint64_t bytes, + uint64_t &begin_cycle, uint64_t &end_cycle +) { +#if PA_BUILD_TRACE_FREE + begin_cycle = 0; + end_cycle = 0; + Ops::InvalidateRegion(address, bytes); +#else + begin_cycle = Ops::Now(); + Ops::InvalidateRegion(address, bytes); + end_cycle = Ops::Now(); +#endif +} + +// 构建身份不匹配时 trace ABI 本身不可信，不能为了记录 fatal 而解释 +// trace_base。中央化这个唯一 pre-attach atomic，仅用于 fail-closed。 +template +PA_DEVICE T PreAttachAtomicExchange( + PA_GM volatile T *address, T value +) { + return Ops::Exchange(address, value); +} + +PA_DEVICE uint32_t AtomicTraceFlags( + AtomicOp op, bool result_used, bool return_ready, bool value_zero = false, + uint64_t retries = 0 +) { + // 高 24 bit 只能容纳有限重试次数；A5 硬件 atomicMax 当前报告 0，CPU CAS + // 回归若超过范围则饱和，避免溢出覆盖低位的 op/语义标志。 + constexpr uint64_t kMaxRetries = (1ULL << (32 - kAtomicRetriesShift)) - 1; + const uint32_t encoded_retries = static_cast(retries > kMaxRetries ? kMaxRetries : retries); + return static_cast(op) | (result_used ? kAtomicResultUsed : 0U) | + (value_zero ? kAtomicValueZero : 0U) | (return_ready ? kAtomicReturnReady : 0U) | + (encoded_retries << kAtomicRetriesShift); +} + +PA_DEVICE void CountAtomicCall( + TraceContext &trace, WorkerResult &result, bool poll_batch +) { + // 单轮 task 数与 2 秒 watchdog 给出了远低于 UINT64_MAX 的物理上界； + // FlushTraceCore 仍校验 host ABI 的 UINT32 上限。PollBatch 已在 + // active call_count 中累计精确次数，统一延迟到收口时加总，避免每次 + // 轮询重复维护 atomic_calls 与 poll_calls 两个 64-bit 总数。 + (void)trace; + if (!poll_batch) { + ++result.atomic_trace_calls; + } +} + +// insert-turn 等待已经在生产循环中维护精确 polls；这里一次性把 +// pending polls + 最终 Ready Load 聚合成一条 PollBatch，避免在最热循环 +// 内为每次 Load 做 trace 分支、计数和起始状态维护。24-bit 编码不足时 +// 明确令本次 trace 闭合失败，不饱和、不拆成随轮询数增长的多条记录。 +PA_DEVICE bool WriteAggregateAtomicPollBatch( + TraceContext &trace, WorkerResult &result, AtomicSite site, + uint64_t start_cycle, uint64_t end_cycle, uint64_t call_count, + bool return_ready_end +) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)result; + (void)site; + (void)start_cycle; + (void)end_cycle; + (void)call_count; + (void)return_ready_end; + return false; +#else + if (!AtomicSwimlaneEnabled(trace)) return false; + if (!TraceAtomicSiteIsPollBatchable(site) || + TraceAtomicSiteExpectedOp(site) != AtomicOp::Load || + call_count == 0 || call_count > kAtomicPollCountMax || + end_cycle < start_cycle || + (return_ready_end && + site != AtomicSite::SharedInsertTurnPoll) || + result.atomic_trace_calls > UINT64_MAX - call_count || + trace.poll_calls > UINT64_MAX - call_count) { + trace.atomic_counter_overflow = true; + return false; + } + result.atomic_trace_calls += call_count; + trace.poll_calls += call_count; + uint32_t slot = 0; + const bool written = + ReserveTraceRecord(trace, slot) && + WritePollBatchRecordRaw( + trace.records, slot, + start_cycle, end_cycle, static_cast(call_count), + static_cast(site), return_ready_end + ); + if (written) { + if (trace.poll_batch_records == UINT64_MAX) { + trace.atomic_counter_overflow = true; + } else { + ++trace.poll_batch_records; + } + } + return written; +#endif +} + +template +PA_DEVICE_NOINLINE void AtomicPollBoundaryAtSlow( + TraceContext &trace, WorkerResult *result, uint64_t end_cycle +) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)result; + (void)end_cycle; +#else + const uint32_t active_mask = trace.poll_burst.active_mask; + // CCEC 默认会把固定 6-site 循环完整展开，再随几十个 phase 边界复制。 + // 禁止展开只控制代码体积；循环次数、site 顺序和同 cycle 关闭语义不变。 + PA_LOOP_NOUNROLL + for (uint32_t index = 0; index < kAtomicPollBatchSiteCount; ++index) { + const uint32_t bit = 1U << index; + if ((active_mask & bit) == 0) continue; + const uint32_t call_count = trace.poll_burst.call_count[index]; + if (call_count == 0 || call_count > kAtomicPollCountMax) { + trace.atomic_counter_overflow = true; + continue; + } + if (result != nullptr) { + if (result->atomic_trace_calls > + UINT64_MAX - call_count || + trace.poll_calls > UINT64_MAX - call_count) { + trace.atomic_counter_overflow = true; + continue; + } + result->atomic_trace_calls += call_count; + trace.poll_calls += call_count; + } + const AtomicSite site = TraceAtomicPollBatchSite(index); + uint32_t slot = 0; + const bool written = + ReserveTraceRecord(trace, slot) && + WritePollBatchRecordRaw( + trace.records, slot, + trace.poll_burst.start_cycle[index], end_cycle, + call_count, static_cast(site) + ); + if (written) { + if (trace.poll_batch_records == UINT64_MAX) { + trace.atomic_counter_overflow = true; + } else { + ++trace.poll_batch_records; + } + } + trace.poll_burst.call_count[index] = 0; + } + trace.poll_burst.active_mask = 0; +#endif +} + +template +PA_DEVICE void AtomicPollBoundaryAt( + TraceContext &trace, uint64_t end_cycle, + WorkerResult *result = nullptr +) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)end_cycle; + (void)result; +#else + // 绝大多数阶段边界没有活跃等待 episode；先用一个可预测分支返回， + // 只有真正需要落 PollBatch 时才进入共享慢函数，避免把固定 6-site + // 收口逻辑复制到每个 TraceTimestamp 调用点。 + if (!AtomicSwimlaneEnabled(trace) || + trace.poll_burst.active_mask == 0) { + return; + } + AtomicPollBoundaryAtSlow(trace, result, end_cycle); +#endif +} + +template +PA_DEVICE void AtomicPollBoundary(TraceContext &trace, WorkerResult &result) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)result; +#else + if (trace.poll_burst.active_mask == 0) return; + AtomicPollBoundaryAt(trace, Ops::Now(), &result); +#endif +} + +template +PA_DEVICE uint32_t AtomicPollRegionBegin( + TraceContext &trace, WorkerResult &result, uint32_t poll_batch_mask +) { + const uint32_t previous_mask = trace.poll_burst.enabled_mask; +#if PA_BUILD_TRACE_FREE + (void)result; + (void)poll_batch_mask; +#else + if (!AtomicSwimlaneEnabled(trace)) return previous_mask; + AtomicPollBoundary(trace, result); + trace.poll_burst.enabled_mask = previous_mask | poll_batch_mask; +#endif + return previous_mask; +} + +template +PA_DEVICE void AtomicPollRegionEnd( + TraceContext &trace, WorkerResult &result, uint32_t previous_mask +) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)result; + (void)previous_mask; +#else + if (!AtomicSwimlaneEnabled(trace)) return; + AtomicPollBoundary(trace, result); + trace.poll_burst.enabled_mask = previous_mask; +#endif +} + +PA_DEVICE bool AtomicPollBatchEnabled( + TraceContext &trace, AtomicSite site, AtomicOp actual_op +) { + return AtomicSwimlaneEnabled(trace) && TraceAtomicSiteIsPollBatchable(site) && + TraceAtomicSiteExpectedOp(site) == actual_op && + (trace.poll_burst.enabled_mask & TraceAtomicPollBatchMask(site)) != 0; +} + +template +PA_DEVICE void AccumulateAtomicPollCall( + TraceContext &trace, WorkerResult &result, AtomicSite site, uint64_t start_cycle +) { + const int32_t signed_index = TraceAtomicPollBatchIndex(site); + if (signed_index < 0) { + trace.atomic_counter_overflow = true; + return; + } + const uint32_t index = static_cast(signed_index); + const uint32_t bit = 1U << index; + if (__builtin_expect( + (trace.poll_burst.active_mask & bit) == 0, 0 + )) { + trace.poll_burst.start_cycle[index] = start_cycle; + trace.poll_burst.call_count[index] = 0; + trace.poll_burst.active_mask |= bit; + } + uint32_t &call_count = trace.poll_burst.call_count[index]; + ++call_count; + if (__builtin_expect( + call_count == kAtomicPollCountMax, 0 + )) { + AtomicPollBoundary(trace, result); + } +} + +template +PA_DEVICE void WriteAtomicTrace( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, AtomicOp op, + uint64_t start_cycle, uint64_t end_cycle, bool result_used, bool return_ready, + bool value_zero = false, uint64_t retries = 0 +) { + // 一次源码 atomic 只写一条同时含 start/end 的 span；结束时间先于 32B record + // 写入，因此本条区间不直接包含自己的记录写开销，但下一次竞争到达会受它影响。 + CountAtomicCall(trace, result, false); +#if PA_BUILD_TRACE_FREE + (void)task_id; + (void)site; + (void)op; + (void)start_cycle; + (void)end_cycle; + (void)result_used; + (void)return_ready; + (void)value_zero; + (void)retries; +#else + WriteTrace( + trace, result, task_id, -1, + TracePhase::Atomic, ProfilePhase::ReplayTail, + start_cycle, end_cycle, + AtomicTraceFlags( + op, result_used, return_ready, + value_zero, retries + ), + static_cast(site) + ); +#endif +} + +template +PA_DEVICE T TraceAtomicLoad( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile T *address, bool result_used = true +) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::Load(address); +#else + if (!AtomicSwimlaneEnabled(trace)) return Ops::Load(address); + const bool poll_batch = result_used && AtomicPollBatchEnabled(trace, site, AtomicOp::Load); + const int32_t poll_index = poll_batch ? TraceAtomicPollBatchIndex(site) : -1; + const bool first_in_batch = poll_batch && + (trace.poll_burst.active_mask & (1U << static_cast(poll_index))) == 0; + const uint64_t begin = !poll_batch || first_in_batch ? Ops::Now() : 0; + const T old = Ops::Load(address); + if (poll_batch) { + CountAtomicCall(trace, result, true); + AccumulateAtomicPollCall(trace, result, site, begin); + return old; + } + // CCEC 只在返回值本来就参与协议判断时插入一条依赖 MOV，再读 SYS_CNT。 + // 这样不会把未消费返回值的 RED/no-return 路径强制改成返回型 ATOM。 + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::Load, begin, end, result_used, return_ready, + old == static_cast(0) + ); + return old; +#endif +} + +template +PA_DEVICE T TraceAtomicExchange( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile T *address, T value, bool result_used = false +) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::Exchange(address, value); +#else + if (!AtomicSwimlaneEnabled(trace)) return Ops::Exchange(address, value); + const uint64_t begin = Ops::Now(); + const T old = Ops::Exchange(address, value); + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::Exchange, begin, end, result_used, return_ready + ); + return old; +#endif +} + +// Handoff 先只捕获 CAS 的 source/return-ready 边界。调用方完成成功判断并 +// 固定 Register 父区间后才写 raw，避免 32B 记录写入污染父区间，同时 +// 不把 CAS 后的比较和函数返回从 RegisterHandoffNextTurn 中删掉。 +template +PA_DEVICE int64_t CaptureAtomicCompareExchange( + TraceContext &trace, PA_GM volatile int64_t *address, + int64_t expected, int64_t desired, + uint64_t &trace_begin, uint64_t &trace_end +) { +#if PA_BUILD_TRACE_FREE + (void)trace; + trace_begin = 0; + trace_end = 0; + return Ops::CompareExchange(address, expected, desired); +#else + if (!AtomicSwimlaneEnabled(trace)) { + trace_begin = 0; + trace_end = 0; + return Ops::CompareExchange(address, expected, desired); + } + trace_begin = Ops::Now(); + const int64_t old = + Ops::CompareExchange(address, expected, desired); + trace_end = Ops::NowAfterAtomicResult(old); + return old; +#endif +} + +template +PA_DEVICE int64_t TraceAtomicCompareExchange( + TraceContext &trace, WorkerResult &result, int32_t task_id, + AtomicSite site, PA_GM volatile int64_t *address, + int64_t expected, int64_t desired, bool result_used = true +) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::CompareExchange(address, expected, desired); +#else + if (!AtomicSwimlaneEnabled(trace)) { + return Ops::CompareExchange(address, expected, desired); + } + const uint64_t begin = Ops::Now(); + const int64_t old = + Ops::CompareExchange(address, expected, desired); + const bool return_ready = + result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used + ? Ops::NowAfterAtomicResult(old) + : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, + AtomicOp::CompareExchange, begin, end, + result_used, return_ready + ); + return old; +#endif +} + +template +PA_DEVICE int64_t TraceAtomicFetchAdd( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile int64_t *address, int64_t value, bool result_used = false +) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::FetchAdd(address, value); +#else + if (!AtomicSwimlaneEnabled(trace)) return Ops::FetchAdd(address, value); + const uint64_t begin = Ops::Now(); + const int64_t old = Ops::FetchAdd(address, value); + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::FetchAdd, begin, end, result_used, return_ready + ); + return old; +#endif +} + +template +PA_DEVICE int64_t TraceAtomicFetchMax( + TraceContext &trace, WorkerResult &result, int32_t task_id, AtomicSite site, + PA_GM volatile int64_t *address, int64_t value, uint64_t &retries, bool result_used = true +) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::FetchMax(address, value, retries); +#else + if (!AtomicSwimlaneEnabled(trace)) return Ops::FetchMax(address, value, retries); + const uint64_t begin = Ops::Now(); + const int64_t old = Ops::FetchMax(address, value, retries); + const bool return_ready = result_used && Ops::kAtomicReturnReadyObserved; + const uint64_t end = result_used ? Ops::NowAfterAtomicResult(old) : Ops::Now(); + WriteAtomicTrace( + trace, result, task_id, site, AtomicOp::FetchMax, begin, end, result_used, + return_ready, false, retries + ); + return old; +#endif +} + +// 公共 shared 原语同时服务正式 scheduler 与不创建 TraceContext 的隔离 +// 单元测试。所有“可选观察”都在这一中央适配层退化为一次原始 Ops 调用； +// 业务头文件不再自行绕过 TraceAtomic*，源码门槛因而可以机械发现漏接。 +template +PA_DEVICE T TraceOptionalAtomicLoad( + TraceContext *trace, WorkerResult *result, int32_t task_id, + AtomicSite site, PA_GM volatile T *address, + bool result_used = true +) { + if (trace == nullptr || result == nullptr) { + return Ops::Load(address); + } + return TraceAtomicLoad( + *trace, *result, task_id, site, address, result_used + ); +} + +template +PA_DEVICE T TraceOptionalAtomicExchange( + TraceContext *trace, WorkerResult *result, int32_t task_id, + AtomicSite site, PA_GM volatile T *address, T value, + bool result_used = false +) { + if (trace == nullptr || result == nullptr) { + return Ops::Exchange(address, value); + } + return TraceAtomicExchange( + *trace, *result, task_id, site, address, value, + result_used + ); +} + +template +PA_DEVICE int64_t TraceOptionalAtomicCompareExchange( + TraceContext *trace, WorkerResult *result, int32_t task_id, + AtomicSite site, PA_GM volatile int64_t *address, + int64_t expected, int64_t desired, bool result_used = true +) { + if (trace == nullptr || result == nullptr) { + return Ops::CompareExchange(address, expected, desired); + } + return TraceAtomicCompareExchange( + *trace, *result, task_id, site, address, + expected, desired, result_used + ); +} + +template +PA_DEVICE int64_t TraceOptionalAtomicFetchMax( + TraceContext *trace, WorkerResult *result, int32_t task_id, + AtomicSite site, PA_GM volatile int64_t *address, + int64_t value, uint64_t &retries, bool result_used = true +) { + if (trace == nullptr || result == nullptr) { + return Ops::FetchMax(address, value, retries); + } + return TraceAtomicFetchMax( + *trace, *result, task_id, site, address, value, + retries, result_used + ); +} + +template +PA_DEVICE T TraceConfiguredAtomicLoad( + TraceContext *trace, WorkerResult *result, int32_t task_id, + AtomicSite site, PA_GM volatile T *address, + bool result_used = true +) { + if constexpr (ObserveAtomics) { + return TraceAtomicLoad( + *trace, *result, task_id, site, address, result_used + ); + } else { + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::Load(address); + } +} + +template +PA_DEVICE T TraceConfiguredAtomicExchange( + TraceContext *trace, WorkerResult *result, int32_t task_id, + AtomicSite site, PA_GM volatile T *address, T value, + bool result_used = false +) { + if constexpr (ObserveAtomics) { + return TraceAtomicExchange( + *trace, *result, task_id, site, address, value, + result_used + ); + } else { + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::Exchange(address, value); + } +} + +template +PA_DEVICE int64_t TraceConfiguredAtomicCompareExchange( + TraceContext *trace, WorkerResult *result, int32_t task_id, + AtomicSite site, PA_GM volatile int64_t *address, + int64_t expected, int64_t desired, bool result_used = true +) { + if constexpr (ObserveAtomics) { + return TraceAtomicCompareExchange( + *trace, *result, task_id, site, address, + expected, desired, result_used + ); + } else { + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::CompareExchange(address, expected, desired); + } +} + +template +PA_DEVICE int64_t TraceConfiguredAtomicFetchMax( + TraceContext *trace, WorkerResult *result, int32_t task_id, + AtomicSite site, PA_GM volatile int64_t *address, + int64_t value, uint64_t &retries, bool result_used = true +) { + if constexpr (ObserveAtomics) { + return TraceAtomicFetchMax( + *trace, *result, task_id, site, address, value, + retries, result_used + ); + } else { + (void)trace; + (void)result; + (void)task_id; + (void)site; + (void)result_used; + return Ops::FetchMax(address, value, retries); + } +} + +template +PA_DEVICE void AccumulatePhase( + WorkerResult &result, ProfilePhase phase, uint64_t start_cycle, uint64_t end_cycle +) { +#if PA_BUILD_TRACE_FREE + (void)result; + (void)phase; + (void)start_cycle; + (void)end_cycle; + return; +#else + // phase profile 与完整泳道是两套正交机制：即使关闭 records，Profile=true + // 仍会累计用户当前关注的 Claim/EfDrain/WaitForSlot/HeapGuard 四段。 + if constexpr (Profile) { + if (phase != ProfilePhase::Claim && phase != ProfilePhase::EfDrain && + phase != ProfilePhase::WaitForSlot && phase != ProfilePhase::HeapGuard) { + return; + } + const uint32_t index = static_cast(phase); + // 调用方保证 end_cycle>=start_cycle；各后端把 Now() 归一到每 tick 1 ns 的 + // 数值标度，聚合持续时间可直接相加并在 host 侧按 1000 换算为微秒。 + const uint64_t duration = end_cycle - start_cycle; + result.phase_cycles[index] += duration; + ++result.phase_calls[index]; + } +#endif +} + +#if PTO_FDWIC_SHARED_MAP +template +PA_DEVICE void WriteSharedClaimTrace( + TraceContext &trace, WorkerResult &result, uint32_t task_id, + uint64_t start_cycle, uint64_t end_cycle, bool winner +) { + AccumulatePhase( + result, ProfilePhase::Claim, start_cycle, end_cycle + ); +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)task_id; + (void)winner; +#else + if (!TraceStorageAttached(trace)) { + return; + } +#if !PA_BUILD_ATOMIC_SWIMLANE + if (task_id >= kMaxTasks) { + return; + } + if (start_cycle == 0 || end_cycle < start_cycle || + (end_cycle & kSharedClaimWinnerBit) != 0) { + trace.atomic_counter_overflow = true; + return; + } +#endif + PA_GM SharedSubmitClaimTraceRecord &record = + GetSharedSubmitClaimRecords(trace.records)[task_id]; + record.claim_begin = start_cycle; + record.claim_end_and_winner = + end_cycle | (winner ? kSharedClaimWinnerBit : 0U); +#endif +} + +template +PA_DEVICE void WriteSharedSubmitTrace( + TraceContext &trace, WorkerResult &result, uint32_t task_id, + uint64_t start_cycle, uint64_t end_cycle +) { + AccumulatePhase( + result, ProfilePhase::Submit, start_cycle, end_cycle + ); +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)task_id; +#else + if (!TraceStorageAttached(trace)) { + return; + } +#if !PA_BUILD_ATOMIC_SWIMLANE + if (task_id >= kMaxTasks) { + return; + } +#endif + PA_GM SharedSubmitClaimTraceRecord &record = + GetSharedSubmitClaimRecords(trace.records)[task_id]; +#if !PA_BUILD_ATOMIC_SWIMLANE + if (start_cycle == 0 || end_cycle < start_cycle || + (end_cycle & kSharedClaimWinnerBit) != 0) { + trace.atomic_counter_overflow = true; + return; + } +#endif + record.submit_begin = start_cycle; + record.submit_end = end_cycle; +#endif +} +#endif + +template +PA_DEVICE void WriteTrace( + TraceContext &trace, WorkerResult &result, int32_t task_id, int32_t function_id, TracePhase trace_phase, + ProfilePhase profile_phase, uint64_t start_cycle, uint64_t end_cycle, uint32_t flags, + uint32_t auxiliary +) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)result; + (void)task_id; + (void)function_id; + (void)trace_phase; + (void)profile_phase; + (void)start_cycle; + (void)end_cycle; + (void)flags; + (void)auxiliary; + return; +#else + // 每段先更新轻量 phase 统计，再按需写 32-byte 原始记录。一个分区只有对应 + // worker 写入；count/dropped 只更新 TraceContext 本地副本，不增加 GM + // 计数行写入或 atomic。 + AccumulatePhase(result, profile_phase, start_cycle, end_cycle); + if (!TraceStorageAttached(trace)) { + return; + } +#if PA_BUILD_COMPACT_GENERIC_TRACE + if (__builtin_expect( + !TraceCompactFieldsFit( + task_id, function_id, trace_phase, auxiliary + ), + 0 + )) { + if (trace.dropped_records != UINT32_MAX) { + ++trace.dropped_records; + } + return; + } +#endif + uint32_t slot = 0; + if (!ReserveTraceRecord(trace, slot)) { + return; + } + WriteGenericTraceRecordRaw( + trace.records, slot, start_cycle, end_cycle, + task_id, function_id, trace_phase, flags, auxiliary + ); +#endif +} + +template +PA_DEVICE void ResetTraceLap( + TraceContext &trace, WorkerResult &result, PA_GM WorkerState &worker +) { + // lap 是后续 Build/Replay/Alloc 等覆盖式阶段的共同起点，不代表新增嵌套 span。 + // 因此分析时不能把 lap 时长再与其中的 Materialize/Claim/Register 直接相加。 +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)result; + (void)worker; +#else + (void)result; + const uint64_t cycle = Ops::Now(); + // 与真实 FDWIC 的 TRACE_LAP_RESET 保持同一边界：等待区 PollBatch + // 只能覆盖本次逻辑轮询 episode，不能跨进下一段 lap 或计算单元执行。 + AtomicPollBoundaryAt(trace, cycle, &result); + worker.swimlane_last_cycle = cycle; +#endif +} + +template +PA_DEVICE void FlushTraceCore(TraceContext &trace, WorkerResult &result) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)result; + return; +#else + if (!TraceStorageAttached(trace)) { + return; + } + // 防御性关闭任何尚未由显式 region end 关闭的等待包；正常路径上 active_mask + // 应为 0，这里仍保证异常早退不会留下“有逻辑调用、无物理 batch”的半截采集。 + AtomicPollBoundary(trace, result); + PA_GM TraceCoreState &core = *trace.core; + if (trace.atomic_counter_overflow || result.atomic_trace_calls > UINT32_MAX || + trace.poll_calls > UINT32_MAX || trace.poll_batch_records > UINT32_MAX) { + if (trace.dropped_records != UINT32_MAX) { + ++trace.dropped_records; + } + } + // terminal row 必须先进入最终 count，随后各记录区 clean 的 + // line_count 才能精确覆盖它自身。shared 的 Submit/Claim 专用区、 + // generic 区和 core-state 各执行一次 FlushRegion，但只生成一条 + // 聚合记录，避免 observer 递归观察自己。 + const uint64_t terminal_begin = Ops::Now(); + const uint64_t final_record_count = + static_cast(trace.record_count) + 1U; + const uint64_t record_bytes = + final_record_count * sizeof(TraceStorageRecord); + const uint64_t record_lines = + (record_bytes + 63U) / 64U; +#if PTO_FDWIC_SHARED_MAP + const bool submit_claim_window_valid = + result.submit_begin != 0 && + result.submit_end >= result.submit_begin && + (result.submit_end & kSharedClaimWinnerBit) == 0; + if (result.submits > kMaxTasks || + !submit_claim_window_valid) { + if (trace.dropped_records != UINT32_MAX) { + ++trace.dropped_records; + } + } + const uint64_t submit_claim_count = + result.submits <= kMaxTasks + ? result.submits + : kMaxTasks; + const uint64_t submit_claim_bytes = + submit_claim_count * + sizeof(SharedSubmitClaimTraceRecord); + const uint64_t submit_claim_lines = + (submit_claim_bytes + 63U) / 64U; + constexpr uint32_t kObserverCallCount = 3; +#else + const uint64_t submit_claim_lines = 0; + constexpr uint32_t kObserverCallCount = 2; +#endif + int32_t terminal_slot = -1; + if (record_lines + submit_claim_lines + 1U <= + kDcciLineCountMax) { + terminal_slot = AppendDcciTrace( + trace, -1, -1, DcciSite::ObserverTraceExport, + DcciOp::CleanOut, true, + kObserverCallCount, + static_cast( + record_lines + submit_claim_lines + 1U + ), + terminal_begin, terminal_begin + ); + } else { + trace.dcci_counter_overflow = true; + } + if (trace.dcci_counter_overflow || + trace.dcci_calls > UINT32_MAX || + trace.dcci_lines > UINT32_MAX || + trace.dcci_records > UINT32_MAX) { + if (trace.dropped_records != UINT32_MAX) { + ++trace.dropped_records; + } + } + // 正常运行中第一次、也是唯一一次写这条 worker 私有 core-state 行。 + core.count = trace.record_count; + core.dropped = trace.dropped_records; + core.atomic_calls = + static_cast(result.atomic_trace_calls); + core.poll_calls = static_cast(trace.poll_calls); + core.poll_batch_records = + static_cast(trace.poll_batch_records); + core.core_idx = trace.core_idx; + core.block_id = trace.block_id; + core.lane = trace.lane; + core.dcci_calls = static_cast(trace.dcci_calls); + core.dcci_lines = static_cast(trace.dcci_lines); + core.dcci_records = static_cast(trace.dcci_records); + const uint32_t count = + trace.record_count < trace.capacity + ? trace.record_count + : trace.capacity; + // A5 侧记录经普通 GM cache 写入，kernel 结束前必须把有效 records 与最后的 + // count/dropped cache line 显式 clean，host 的 D2H 才能得到完整且自洽的快照。 + if (count != 0) { + Ops::FlushRegion( + trace.records, + static_cast(count) * + sizeof(TraceStorageRecord) + ); + } +#if PTO_FDWIC_SHARED_MAP + if (submit_claim_count != 0) { + Ops::FlushRegion( + GetSharedSubmitClaimRecords(trace.records), + submit_claim_bytes + ); + } +#endif + Ops::FlushRegion(&core, sizeof(core)); + if (terminal_slot >= 0) { + // terminal row 本身已经随 generic records clean 导出；所有 + // observer DSB 之后只能用一次 bypass-DCache store 更新 end， + // 不能再对同一 cache line 做 read-modify-write。compact 只发布 + // 4B low32，保证同一 64B 内其余三条记录保持原值。 +#if PA_BUILD_COMPACT_GENERIC_TRACE + Ops::Publish( + &trace.records[ + static_cast(terminal_slot) + ].end_cycle_low, + static_cast(Ops::Now()) + ); +#else + Ops::Publish( + &trace.records[ + static_cast(terminal_slot) + ].end_cycle, + Ops::Now() + ); +#endif + } +#endif +} + +template +PA_DEVICE uint64_t WriteTraceLap( + TraceContext &trace, PA_GM WorkerState &worker, WorkerResult &result, int32_t task_id, + int32_t function_id, TracePhase trace_phase, ProfilePhase profile_phase, + uint32_t flags = 0, uint32_t auxiliary = 0 +) { +#if PA_BUILD_TRACE_FREE + (void)trace; + (void)worker; + (void)result; + (void)task_id; + (void)function_id; + (void)trace_phase; + (void)profile_phase; + (void)flags; + (void)auxiliary; + return 0; +#else + // lap 记录区间 [上一次 Reset/WriteTraceLap, 当前时刻]，写完立即推进起点。 + // 显式 WriteTrace span 不会修改该起点，这正是生产泳道中阶段可重叠的原因。 + const uint64_t end_cycle = Ops::Now(); + // 真实 FDWIC 在 TRACE_LAP 取到结束时间后先关闭 PollBatch，再写 lap。 + // 复用同一个 end_cycle，避免额外 SYS_CNT 造成可见缝隙。 + AtomicPollBoundaryAt(trace, end_cycle, &result); + WriteTrace( + trace, result, task_id, function_id, trace_phase, profile_phase, worker.swimlane_last_cycle, end_cycle, + flags, auxiliary + ); + worker.swimlane_last_cycle = end_cycle; + return end_cycle; +#endif +} + +} // namespace pa_scheduler + +#endif // PA_SCHEDULER_COMMON_PA_TRACE_H diff --git a/tests/atomic_probe/pa_scheduler/common/winner_workload.h b/tests/atomic_probe/pa_scheduler/common/winner_workload.h new file mode 100644 index 0000000000..a3995fb50f --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/common/winner_workload.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_WINNER_WORKLOAD_H +#define PA_SCHEDULER_COMMON_WINNER_WORKLOAD_H + +#include "pa_model.h" + +#include +#include + +namespace pa_scheduler::winner_workload { + +// 三种 standalone 后端共享完全相同的 GM/host 布局。128x128 float 是 CCEC +// 已在 A5 上验证过的基本形状；后端可以采用不同指令接口，但不能改变输入、 +// 输出 tile 编址或 host 数值校验口径。 +constexpr uint32_t kTileRows = 128; +constexpr uint32_t kTileCols = 128; +constexpr size_t kTileElements = static_cast(kTileRows) * kTileCols; +constexpr size_t kTileBytes = kTileElements * sizeof(float); +constexpr uint32_t kSharedInputTiles = 2; +constexpr uint32_t kOutputTilesPerWorker = 2; +constexpr uint32_t kOutputTiles = kWorkers * kOutputTilesPerWorker; +constexpr size_t kWorkspaceTiles = kSharedInputTiles + kOutputTiles; +constexpr size_t kWorkspaceBytes = kWorkspaceTiles * kTileBytes; +constexpr float kInputAValue = 2.0F; +constexpr float kInputBValue = 3.0F; +constexpr float kExpectedAicValue = 768.0F; +constexpr float kExpectedSfValue = 5.0F; +constexpr float kExpectedUpValue = 6.0F; +constexpr float kOutputSentinel = -12345.0F; + +// 256 batch 下，即使同一 AIC 极端地拿到全部 QK/PV，128 次完整 Cube +// 迭代的 CCEC 实测 busy 上界仍低于 32-bit PMU 的 25% 门槛。其他后端也沿用 +// 此参数边界，避免相同命令在不同实现上产生不同含义。 +constexpr uint32_t kMaxRealComputeCount = 128; + +// 1 次用于最小正确性取证。默认次数来自 CCEC 的三个独立 b256 A5 进程； +// AscendC 必须重新标定后才能宣称达到同样时长，不能仅因共享默认参数便沿用 +// CCEC 的性能结论。UP 的一次完整 128x128 流水是当前正整数下限。 +constexpr WorkloadCounts kRealComputeSmokeCounts{1, 1, 1, 1}; +constexpr WorkloadCounts kDefaultRealComputeCounts{6, 28, 4, 1}; + +static_assert(kTileBytes == 65536, "real-compute tile must occupy 64 KiB"); +static_assert(kWorkspaceBytes == 12713984, "real-compute workspace size changed unexpectedly"); + +} // namespace pa_scheduler::winner_workload + +#endif // PA_SCHEDULER_COMMON_WINNER_WORKLOAD_H diff --git a/tests/atomic_probe/pa_scheduler/common/winner_workload_host.h b/tests/atomic_probe/pa_scheduler/common/winner_workload_host.h new file mode 100644 index 0000000000..0d5603ffce --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/common/winner_workload_host.h @@ -0,0 +1,380 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef PA_SCHEDULER_COMMON_WINNER_WORKLOAD_HOST_H +#define PA_SCHEDULER_COMMON_WINNER_WORKLOAD_HOST_H + +#include "host_support.h" +#include "winner_workload.h" + +#include +#include +#include + +namespace pa_scheduler::host { + +// 常量模式用于稳定的性能负载；布局诊断模式使用带权对角 A 和非对称稠密 B， +// 专门暴露 B 转置、ND/NZ stride 与输出重排错误。该选择只影响计时窗外的 +// host 输入生成和结果校验，不进入 SchedulerState，也不增加 device 热路径分支。 +enum class RealComputePattern : uint32_t { + Constant = 0, + LayoutDiagnostic = 1, +}; + +// winner 负载参数在通用 benchmark parser 前单独剥离，CCEC 可在其后继续剥离 +// PMU 参数；AscendC/CPU 则直接把剩余 argv 交给 ParseOptions。这样不把后端 +// 私有功能塞入公共 PA 参数结构，也不会复制三套互斥规则。 +struct WinnerWorkloadOptions { + WinnerWorkloadMode mode = WinnerWorkloadMode::RealCompute; + WorkloadCounts repeats = winner_workload::kDefaultRealComputeCounts; + RealComputePattern pattern = RealComputePattern::Constant; + bool counts_explicit = false; + bool pattern_explicit = false; + bool nop_override_explicit = false; +}; + +inline const char *WinnerWorkloadModeName(WinnerWorkloadMode mode) { + switch (mode) { + case WinnerWorkloadMode::ScalarNop: + return "scalar-nop"; + case WinnerWorkloadMode::RealCompute: + return "real-compute"; + } + return "invalid"; +} + +inline const char *RealComputePatternName(RealComputePattern pattern) { + switch (pattern) { + case RealComputePattern::Constant: + return "constant"; + case RealComputePattern::LayoutDiagnostic: + return "layout-diagnostic"; + } + return "invalid"; +} + +inline bool ParseWorkloadCounts(const char *raw, WorkloadCounts *counts) { + unsigned int qk = 0; + unsigned int sf = 0; + unsigned int pv = 0; + unsigned int up = 0; + char tail = '\0'; + if (std::sscanf(raw, "%u,%u,%u,%u%c", &qk, &sf, &pv, &up, &tail) != 4) return false; + const uint32_t maximum = winner_workload::kMaxRealComputeCount; + if (qk == 0 || sf == 0 || pv == 0 || up == 0 || + qk > maximum || sf > maximum || pv > maximum || up > maximum) { + return false; + } + *counts = WorkloadCounts{qk, sf, pv, up}; + return true; +} + +inline bool ParseWinnerWorkloadOptions( + int argc, char **argv, WinnerWorkloadOptions *workload, std::vector *remaining_argv +) { + bool mode_seen = false; + bool count_seen = false; + bool pattern_seen = false; + remaining_argv->clear(); + remaining_argv->push_back(argv[0]); + for (int index = 1; index < argc; ++index) { + const std::string argument = argv[index]; + if (argument == "--nop-count" || argument == "--nop-counts") { + workload->nop_override_explicit = true; + } + if (argument != "--winner-workload" && argument != "--real-compute-count" && + argument != "--real-compute-counts" && argument != "--real-compute-pattern") { + remaining_argv->push_back(argv[index]); + continue; + } + if (index + 1 >= argc) { + std::fprintf(stderr, "Missing value after %s\n", argument.c_str()); + return false; + } + const char *value = argv[++index]; + if (argument == "--winner-workload") { + if (mode_seen) { + std::fprintf(stderr, "Specify --winner-workload only once.\n"); + return false; + } + const std::string name = value; + if (name == "scalar-nop") { + workload->mode = WinnerWorkloadMode::ScalarNop; + } else if (name == "real-compute") { + workload->mode = WinnerWorkloadMode::RealCompute; + } else { + std::fprintf( + stderr, + "Invalid --winner-workload value: %s (expected scalar-nop|real-compute)\n", + value + ); + return false; + } + mode_seen = true; + continue; + } + if (argument == "--real-compute-pattern") { + if (pattern_seen) { + std::fprintf(stderr, "Specify --real-compute-pattern only once.\n"); + return false; + } + const std::string name = value; + if (name == "constant") { + workload->pattern = RealComputePattern::Constant; + } else if (name == "layout-diagnostic") { + workload->pattern = RealComputePattern::LayoutDiagnostic; + } else { + std::fprintf( + stderr, + "Invalid --real-compute-pattern value: %s " + "(expected constant|layout-diagnostic)\n", + value + ); + return false; + } + pattern_seen = true; + workload->pattern_explicit = true; + continue; + } + if (count_seen) { + std::fprintf(stderr, "Specify only one real-compute count override.\n"); + return false; + } + if (argument == "--real-compute-count") { + uint32_t count = 0; + if (!ParseUint(value, 1, winner_workload::kMaxRealComputeCount, &count)) { + std::fprintf(stderr, "Invalid --real-compute-count value: %s\n", value); + return false; + } + workload->repeats = WorkloadCounts{count, count, count, count}; + } else if (!ParseWorkloadCounts(value, &workload->repeats)) { + std::fprintf(stderr, "Invalid --real-compute-counts value: %s\n", value); + return false; + } + count_seen = true; + workload->counts_explicit = true; + } + // 无参数运行以真实 Cube/Vector 为默认。旧命令若显式给出 NOP 次数但没有 + // 指定 workload mode，则把 NOP override 本身视为选择 scalar-nop；这样 + // 既不让 --nop-count 悄悄失效，也不破坏既有标定脚本。显式指定 + // real-compute 再叠加 NOP 仍由下方互斥校验拒绝。 + if (!mode_seen && workload->nop_override_explicit) { + workload->mode = WinnerWorkloadMode::ScalarNop; + } + return true; +} + +inline bool ValidateWinnerWorkloadOptions(const WinnerWorkloadOptions &workload) { + if (workload.mode == WinnerWorkloadMode::RealCompute) { + if (workload.pattern != RealComputePattern::Constant && + workload.pattern != RealComputePattern::LayoutDiagnostic) { + std::fprintf(stderr, "Invalid real-compute input pattern.\n"); + return false; + } + if (workload.nop_override_explicit) { + std::fprintf( + stderr, + "--winner-workload real-compute cannot be combined with --nop-count or --nop-counts.\n" + ); + return false; + } + return true; + } + if (workload.counts_explicit || workload.pattern_explicit) { + std::fprintf( + stderr, + "--real-compute-count(s)/pattern requires real-compute workload mode.\n" + ); + return false; + } + return true; +} + +inline void ConfigureWinnerWorkload( + SchedulerState *state, const WinnerWorkloadOptions &workload, const void *workspace_device +) { + state->winner_workload.mode = static_cast(workload.mode); + state->winner_workload.version = kWinnerWorkloadConfigVersion; + state->winner_workload.repeats = workload.repeats; + state->winner_workload.workspace_base = reinterpret_cast(workspace_device); + state->winner_workload.workspace_bytes = + workload.mode == WinnerWorkloadMode::RealCompute ? winner_workload::kWorkspaceBytes : 0; +} + +inline float LayoutDiagnosticInputA(uint32_t row, uint32_t column) { + return row == column ? static_cast(row + 1U) : 0.0F; +} + +inline float LayoutDiagnosticInputB(uint32_t row, uint32_t column) { + const uint32_t value = + (131U * row + 17U * column + 7U * row * column) % 251U; + return static_cast(value + 1U); +} + +inline void InitializeWinnerWorkloadBuffers( + const WinnerWorkloadOptions &workload, std::vector *workspace_image, + std::vector *workspace_outputs +) { + using namespace winner_workload; + workspace_image->assign(kWorkspaceTiles * kTileElements, kOutputSentinel); + if (workload.pattern == RealComputePattern::Constant) { + std::fill_n(workspace_image->begin(), kTileElements, kInputAValue); + std::fill_n(workspace_image->begin() + kTileElements, kTileElements, kInputBValue); + } else { + for (uint32_t row = 0; row < kTileRows; ++row) { + for (uint32_t column = 0; column < kTileCols; ++column) { + const size_t element = static_cast(row) * kTileCols + column; + (*workspace_image)[element] = LayoutDiagnosticInputA(row, column); + (*workspace_image)[kTileElements + element] = + LayoutDiagnosticInputB(row, column); + } + } + } + workspace_outputs->resize(static_cast(kOutputTiles) * kTileElements); +} + +inline const char *TaskKindName(TaskKind kind) { + switch (kind) { + case TaskKind::Qk: + return "QK"; + case TaskKind::Sf: + return "SF"; + case TaskKind::Pv: + return "PV"; + case TaskKind::Up: + return "UP"; + default: + return "invalid"; + } +} + +inline float ExpectedRealComputeValue( + RealComputePattern pattern, TaskKind kind, uint32_t row, uint32_t column +) { + using namespace winner_workload; + if (pattern == RealComputePattern::Constant) { + return kind == TaskKind::Qk || kind == TaskKind::Pv + ? kExpectedAicValue + : (kind == TaskKind::Sf ? kExpectedSfValue : kExpectedUpValue); + } + const float input_a = LayoutDiagnosticInputA(row, column); + const float input_b = LayoutDiagnosticInputB(row, column); + if (kind == TaskKind::Qk || kind == TaskKind::Pv) { + // A 是带权对角矩阵，因此 A*B 的 (row,column) 只有一个非零乘积。 + return static_cast(row + 1U) * input_b; + } + return kind == TaskKind::Sf ? input_a + input_b : input_a * input_b; +} + +inline bool RealComputeActivityMatchesPlan( + const SchedulerState &state, uint32_t active_tiles +) { +#if PTO_FDWIC_SHARED_MAP + // shared G0 只有 Alloc，没有 QK/SF/PV/UP；0 个 active tile 是计划要求， + // 不能沿用 private “至少执行一个 compute kernel”的固定前提。非零 group + // 仍必须观察到 active tile，避免把漏执行 kernel 放宽成合法 G0。 + SharedHostTaskPlan plan; + if (!BuildSharedHostTaskPlan(state, &plan)) return false; + return (plan.total_groups != 0U) == (active_tiles != 0U); +#else + // private 每个合法 batch 固定包含 QK/SF/PV/UP，保持原门槛不变。 + (void)state; + return active_tiles != 0U; +#endif +} + +inline bool ValidateRealComputeOutputs( + const SchedulerState &state, const WinnerWorkloadOptions &workload, + const std::vector &outputs, uint32_t run +) { + using namespace winner_workload; + const size_t expected_elements = static_cast(kOutputTiles) * kTileElements; + if (outputs.size() != expected_elements) { + std::fprintf( + stderr, "[ASSERT] real-compute output buffer size matches workspace layout FAIL\n" + ); + return false; + } + + uint32_t active_tiles = 0; + uint32_t inactive_tiles = 0; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + const WorkerResult &result = state.results[worker]; + const bool aic = result.role == static_cast(CoreRole::Aic); + const TaskKind kinds[2] = { + aic ? TaskKind::Qk : TaskKind::Sf, + aic ? TaskKind::Pv : TaskKind::Up, + }; + for (uint32_t kind_slot = 0; kind_slot < 2; ++kind_slot) { + const TaskKind kind = kinds[kind_slot]; + const uint32_t kernel_index = static_cast(kind) - 1; + const bool active = result.kernel_counts[kernel_index] != 0; + const size_t tile_index = + static_cast(worker) * kOutputTilesPerWorker + kind_slot; + const size_t begin = tile_index * kTileElements; + for (size_t element = 0; element < kTileElements; ++element) { + const uint32_t row = static_cast(element / kTileCols); + const uint32_t column = static_cast(element % kTileCols); + const float expected = active + ? ExpectedRealComputeValue(workload.pattern, kind, row, column) + : kOutputSentinel; + if (outputs[begin + element] == expected) continue; + std::fprintf( + stderr, + "[REAL-COMPUTE-FAIL] run=%u worker=%u kind=%s element=%zu " + "expected=%.1f actual=%.9g\n", + run, worker, TaskKindName(kind), element, expected, + static_cast(outputs[begin + element]) + ); + std::fprintf( + stderr, + "[ASSERT] real-compute output tiles match role-specific engine results FAIL\n" + ); + return false; + } + active_tiles += active ? 1U : 0U; + inactive_tiles += active ? 0U : 1U; + } + } + const bool passed = + RealComputeActivityMatchesPlan(state, active_tiles) && + active_tiles + inactive_tiles == kOutputTiles; + std::printf( + "[ASSERT] %-48s %s (active_tiles=%u inactive_sentinel_tiles=%u)\n", + "real-compute output tiles match role-specific engine results", + passed ? "PASS" : "FAIL", active_tiles, inactive_tiles + ); + return passed; +} + +inline void PrintWinnerWorkloadConfig( + const WinnerWorkloadOptions &workload, const NopCounts &nops +) { + if (workload.mode == WinnerWorkloadMode::RealCompute) { + std::printf( + "[WINNER-WORKLOAD] mode=real-compute pattern=%s counts=%u,%u,%u,%u " + "unit=complete_128x128_engine_pipeline_iteration workspace_bytes=%zu\n", + RealComputePatternName(workload.pattern), + workload.repeats.qk, workload.repeats.sf, workload.repeats.pv, + workload.repeats.up, winner_workload::kWorkspaceBytes + ); + return; + } + std::printf( + "[WINNER-WORKLOAD] mode=scalar-nop counts=%u,%u,%u,%u " + "unit=scalar_nop_instruction workspace_bytes=0\n", + nops.qk, nops.sf, nops.pv, nops.up + ); +} + +} // namespace pa_scheduler::host + +#endif // PA_SCHEDULER_COMMON_WINNER_WORKLOAD_HOST_H diff --git a/tests/atomic_probe/pa_scheduler/cpu/build.sh b/tests/atomic_probe/pa_scheduler/cpu/build.sh new file mode 100755 index 0000000000..ab8e670c06 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/cpu/build.sh @@ -0,0 +1,293 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +if [[ $# -gt 2 ]]; then + echo "Usage: $0 [private|shared] [swimlane|perf-clock]" >&2 + exit 1 +fi +TENSORMAP_MODE="${1:-private}" +case "$TENSORMAP_MODE" in + private) TENSORMAP_MODE_ID=0 ;; + shared) TENSORMAP_MODE_ID=1 ;; + *) + echo "Unknown TensorMap mode: $TENSORMAP_MODE (expected private|shared)" >&2 + exit 1 + ;; +esac +BUILD_VARIANT="${2:-swimlane}" +case "$BUILD_VARIANT" in + swimlane) + VARIANT_DEFINES=( + -DPA_BUILD_SWIMLANE=0 + -DPA_BUILD_SUBMIT_PMU=0 + -DPA_BUILD_PERF_CLOCK=0 + ) + ;; + perf-clock) + VARIANT_DEFINES=( + -DPA_BUILD_SWIMLANE=0 + -DPA_BUILD_SUBMIT_PMU=0 + -DPA_BUILD_PERF_CLOCK=1 + ) + ;; + *) + echo "Unknown CPU build variant: $BUILD_VARIANT (expected swimlane|perf-clock)" >&2 + exit 1 + ;; +esac +BUILD_DIR="$ROOT_DIR/build/cpu/$TENSORMAP_MODE/$BUILD_VARIANT" +CXX_BIN="${CXX:-g++}" +TENSORMAP_RING_CAP=128 +SHARED_INSERT_TURN_GROUPS="${PA_SHARED_INSERT_TURN_GROUPS:-1}" +case "$SHARED_INSERT_TURN_GROUPS" in + 1|2|4|8|16|32|64|128) ;; + *) + echo "PA_SHARED_INSERT_TURN_GROUPS must be a power of two from 1 through 128." >&2 + exit 1 + ;; +esac +if [[ "$TENSORMAP_MODE" != "shared" && + "$SHARED_INSERT_TURN_GROUPS" != "1" ]]; then + echo "PA_SHARED_INSERT_TURN_GROUPS only applies to shared TensorMap builds." >&2 + exit 1 +fi +VARIANT_DEFINES+=( + "-DPTO_FDWIC_SHARED_INSERT_TURN_GROUPS=$SHARED_INSERT_TURN_GROUPS" +) +SCHEDULER_BINARY="pa_scheduler_cpu" +if [[ "$TENSORMAP_MODE" == "shared" && + "$SHARED_INSERT_TURN_GROUPS" != "1" ]]; then + SCHEDULER_BINARY="pa_scheduler_cpu_turn_g${SHARED_INSERT_TURN_GROUPS}" +fi + +# CPU 后端只依赖 C++17、pthread 和本目录 common/，不需要 CANN。 +# CXX 可显式指向用户目录下的 g++-15，未设置时沿用当前 PATH 中的 g++。 + +# CPU build 与设备 build 使用平行目录，便于 run.sh 根据 backend 做严格选择， +# 也避免把 host 回归二进制误当成 A5 产物。 +mkdir -p "$BUILD_DIR" + +echo "[BUILD] CPU scheduler executable" +echo "[BUILD] shared insert-turn groups=$SHARED_INSERT_TURN_GROUPS" +# -pthread 同时提供编译期线程宏和链接期 pthread 支持；严格告警用于防止 +# CPU 等价层因类型或原子接口变化而静默偏离设备端公共协议。private/shared +# 都实例化同一 scheduler，模式宏只选择各自已经接线的 TensorMap backend。 +"$CXX_BIN" -O3 -std=c++17 -pthread -Wall -Wextra -Werror \ + "-DPTO_FDWIC_SHARED_MAP=$TENSORMAP_MODE_ID" \ + "-DPTO_FDWIC_TENSORMAP_RING_CAP=$TENSORMAP_RING_CAP" \ + "${VARIANT_DEFINES[@]}" \ + -I"$ROOT_DIR/common" \ + "$SCRIPT_DIR/main.cpp" \ + -o "$BUILD_DIR/$SCHEDULER_BINARY" + +# PollBatch 是 common/ 中的设备/CPU 共用模板。这里用普通 C++17 编译器 +# 直接实例化并执行边界自测；任一断言失败都会借助 set -e 阻止构建成功。 +echo "[BUILD] atomic PollBatch boundary self-test" +"$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror \ + "-DPTO_FDWIC_SHARED_MAP=$TENSORMAP_MODE_ID" \ + -DPA_BUILD_SWIMLANE=1 \ + -I"$ROOT_DIR/common" \ + "$ROOT_DIR/test/test_atomic_poll_batch.cpp" \ + -o "$BUILD_DIR/test_atomic_poll_batch" + +echo "[TEST] atomic PollBatch boundary self-test" +"$BUILD_DIR/test_atomic_poll_batch" + +# TensorMap 独立回归不启动 96 个 worker，也不运行模拟 kernel：private +# 覆盖单线程 ring 的回收回绕与 reference 差分；shared ring 是当前 +# ordered writer-delta 的 ordinary-region 原语，隔离覆盖 seq/ABA、回收 +# 与容量预检。PA Case1 当前 ordinary entry 为零，因此这些门槛仍不能 +# 代替后面的完整 96-worker Submit 测试。 +if [[ "$TENSORMAP_MODE" == "private" ]]; then + # 同一生产 helper 在固定 16K 总槽下覆盖多组 CAP×bucket 形态； + # 正式 scheduler 仍只编默认 128，隔离门槛不冒充运行期 auto。 + for cap in 32 64 128 256 16384; do + binary="$BUILD_DIR/test_private_tensor_map_ring_cap${cap}" + echo "[BUILD] private TensorMap ring self-test CAP=$cap" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror \ + -DPTO_FDWIC_SHARED_MAP=0 \ + "-DPTO_FDWIC_TENSORMAP_RING_CAP=$cap" \ + -DPA_BUILD_SWIMLANE=1 \ + -I"$ROOT_DIR/common" \ + "$ROOT_DIR/test/test_private_tensor_map_ring.cpp" \ + -o "$binary" + + echo "[TEST] private TensorMap ring self-test CAP=$cap" + "$binary" + done +else + # shared writer 插入完成链位于 TaskCell::deps_prepared：task 0 无前驱， + # task N 只等 N-1，发布时只 CAS 自己的完成字。旧 sidecar turn 全部 + # 写入 canary 并要求零触碰；另覆盖空写、损坏值与重复发布 fatal。 + echo "[BUILD] shared per-task insert-completion self-test" + "$CXX_BIN" -O2 -std=c++17 -pthread -Wall -Wextra -Werror \ + -DPTO_FDWIC_SHARED_MAP=1 \ + "-DPTO_FDWIC_SHARED_INSERT_TURN_GROUPS=$SHARED_INSERT_TURN_GROUPS" \ + -DPA_BUILD_SWIMLANE=1 \ + -I"$ROOT_DIR/common" \ + "$ROOT_DIR/test/test_shared_insert_turn.cpp" \ + -o "$BUILD_DIR/test_shared_insert_completion" + + echo "[TEST] shared per-task insert-completion self-test" + "$BUILD_DIR/test_shared_insert_completion" + + # host 必须从最终 SchedulerState.context_lens 独立重建 shared task + # plan，不能复用 device helper 形成同错 oracle。该测试覆盖 G0/G1/G2/G4、 + # mixed 累计 batch_start、TaskAt 元数据、partial group 输出字节、writer + # dependency chain，以及测试专用 CLI 的广播/逐 batch 形式。 + echo "[BUILD] shared authoritative host task-plan self-test" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror \ + -DPTO_FDWIC_SHARED_MAP=1 \ + -DPA_BUILD_SWIMLANE=1 \ + -I"$ROOT_DIR/common" \ + "$ROOT_DIR/test/test_shared_host_task_plan.cpp" \ + -o "$BUILD_DIR/test_shared_host_task_plan" + + echo "[TEST] shared authoritative host task-plan self-test" + "$BUILD_DIR/test_shared_host_task_plan" + + for cap in 32 64 128 256 16384; do + binary="$BUILD_DIR/test_shared_tensor_map_ring_cap${cap}" + echo "[BUILD] isolated shared ordinary-region ring self-test CAP=$cap" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror \ + -DPTO_FDWIC_SHARED_MAP=1 \ + "-DPTO_FDWIC_TENSORMAP_RING_CAP=$cap" \ + -DPA_BUILD_SWIMLANE=1 \ + -I"$ROOT_DIR/common" \ + "$ROOT_DIR/test/test_shared_tensor_map_ring.cpp" \ + -o "$binary" + + echo "[TEST] isolated shared ordinary-region ring self-test CAP=$cap" + "$binary" + done + + # shared raw 只保留真实稀疏边界：所有 task 有连续 EfDrain+Claim + # 和 Submit 父区间，loser 没有业务子区间；PrepareMap 必须彻底缺席。 + echo "[BUILD] shared sparse raw-trace self-test" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror \ + -DPTO_FDWIC_SHARED_MAP=1 \ + -I"$ROOT_DIR/common" \ + "$ROOT_DIR/test/test_shared_sparse_trace.cpp" \ + -o "$BUILD_DIR/test_shared_sparse_trace" + + echo "[TEST] shared sparse raw-trace self-test" + "$BUILD_DIR/test_shared_sparse_trace" + + # CCEC full-swimlane 的 16B generic raw 仍由 host 恢复成既有 32B + # 逻辑记录。该纯主机门槛独立锁定 packed 字段、low32 前/后向回绕、 + # 生命周期拒绝和 terminal 4B 更新的 cache-line 邻值不变。 + echo "[BUILD] shared compact generic-trace codec self-test" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror \ + -DPTO_FDWIC_SHARED_MAP=1 \ + -DPA_BUILD_SWIMLANE=1 \ + -DPA_BUILD_ATOMIC_SWIMLANE=1 \ + -DPA_BUILD_COMPACT_GENERIC_TRACE=1 \ + -I"$ROOT_DIR/common" \ + "$ROOT_DIR/test/test_shared_compact_generic_trace.cpp" \ + -o "$BUILD_DIR/test_shared_compact_generic_trace" + + echo "[TEST] shared compact generic-trace codec self-test" + "$BUILD_DIR/test_shared_compact_generic_trace" + + # fresh-output symbol 与 region ring 是两条独立协议。该用例单独锁定 + # descriptor 最终封口、只读 fanin、ready descriptor 直写 slot、 + # 构建后 INOUT writer commit、失败 slot 撤销及非法引用 fail-closed， + # 避免只靠完整 96 线程回放偶然覆盖。 + echo "[BUILD] shared-output symbol self-test" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror -pthread \ + -DPTO_FDWIC_SHARED_MAP=1 \ + -DPA_BUILD_SWIMLANE=1 \ + -I"$ROOT_DIR/common" \ + "$ROOT_DIR/test/test_shared_output_symbols.cpp" \ + -o "$BUILD_DIR/test_shared_output_symbols" + + echo "[TEST] shared-output symbol self-test" + "$BUILD_DIR/test_shared_output_symbols" + + # 通用 writer-intent 门槛不使用 PA TaskKind/ticket：symbol 锁定 + # 多跳、跨 cache-line history、乱序和 partial-CAS 终止语义； + # ownerless ordinary region 锁定 A->B->C，并验证空 transaction 也 + # 推进 per-task completion，旧 sidecar turn 保持 canary。 + echo "[BUILD] generic shared writer-intent self-test" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror -pthread \ + -DPTO_FDWIC_SHARED_MAP=1 \ + "-DPTO_FDWIC_SHARED_INSERT_TURN_GROUPS=$SHARED_INSERT_TURN_GROUPS" \ + -DPA_BUILD_SWIMLANE=1 \ + -I"$ROOT_DIR/common" \ + "$ROOT_DIR/test/test_shared_writer_intent.cpp" \ + -o "$BUILD_DIR/test_shared_writer_intent" + + echo "[TEST] generic shared writer-intent self-test" + timeout --foreground 15s "$BUILD_DIR/test_shared_writer_intent" + + # shared heap 与 region/symbol 协议分开验证：锁定 8 shard、1 KiB + # 对齐、首版禁止 wrap、并发唯一分配及 terminal 容量竞争不回滚。 + echo "[BUILD] shared heap no-wrap reserve self-test" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror -pthread \ + -DPTO_FDWIC_SHARED_MAP=1 \ + -DPA_BUILD_SWIMLANE=1 \ + -I"$ROOT_DIR/common" \ + "$ROOT_DIR/test/test_shared_heap_reserve.cpp" \ + -o "$BUILD_DIR/test_shared_heap_reserve" + + echo "[TEST] shared heap no-wrap reserve self-test" + "$BUILD_DIR/test_shared_heap_reserve" + + # Claim 保持原 cursor 协议：Alloc/Cube 使用 prefix 四分片，Vector + # 使用 shared sidecar 八分片；96 worker 锁定 role 候选数、唯一 + # winner、重复 loser，并要求 Claim 不触碰 deps_prepared。 + echo "[BUILD] shared cursor Claim self-test" + "$CXX_BIN" -O2 -std=c++17 -pthread -Wall -Wextra -Werror \ + -DPTO_FDWIC_SHARED_MAP=1 \ + -DPA_BUILD_SWIMLANE=1 \ + -I"$ROOT_DIR/common" \ + "$ROOT_DIR/test/test_shared_vector_claim_cursor.cpp" \ + -o "$BUILD_DIR/test_shared_vector_claim_cursor" + + echo "[TEST] shared cursor Claim self-test" + "$BUILD_DIR/test_shared_vector_claim_cursor" + + # Materialize 在触碰 shared cursor 前必须完成数量、引用、shape/stride + # 和地址区间预检；这些 reserve 前拒绝路径不能推进 heap。FetchAdd 后 + # 才暴露的容量竞争则按 terminal 契约保留 overrun 现场。 + echo "[BUILD] shared winner materialize self-test" + "$CXX_BIN" -O2 -std=c++17 -Wall -Wextra -Werror \ + -DPTO_FDWIC_SHARED_MAP=1 \ + -DPA_BUILD_SWIMLANE=1 \ + -I"$ROOT_DIR/common" \ + "$ROOT_DIR/test/test_shared_materialize.cpp" \ + -o "$BUILD_DIR/test_shared_materialize" + + echo "[TEST] shared winner materialize self-test" + "$BUILD_DIR/test_shared_materialize" + + # 完整 96-worker Submit 逐 task 计数 cursor Claim、前驱 completion + # load 和本 task completion CAS；同时锁定 loser 零 map 访问、旧 + # sidecar turn 零触碰，以及 lookup/Build/执行仍可跨前任 Build。 + echo "[BUILD] shared ordered-insert Submit self-test" + "$CXX_BIN" -O2 -std=c++17 -pthread -Wall -Wextra -Werror \ + -DPTO_FDWIC_SHARED_MAP=1 \ + "-DPTO_FDWIC_SHARED_INSERT_TURN_GROUPS=$SHARED_INSERT_TURN_GROUPS" \ + -DPA_BUILD_SWIMLANE=1 \ + -I"$ROOT_DIR/common" \ + "$ROOT_DIR/test/test_shared_ordered_submit.cpp" \ + -o "$BUILD_DIR/test_shared_ordered_submit" + + echo "[TEST] shared ordered-insert Submit self-test" + timeout --foreground 15s "$BUILD_DIR/test_shared_ordered_submit" +fi + +# set -e 保证编译或链接失败时不会打印 complete，也不会在组合构建中继续 +# 后续步骤；只有成功退出的构建才被本脚本声明为可运行产物。 +echo "[BUILD] complete: $BUILD_DIR/$SCHEDULER_BINARY" diff --git a/tests/atomic_probe/pa_scheduler/cpu/main.cpp b/tests/atomic_probe/pa_scheduler/cpu/main.cpp new file mode 100644 index 0000000000..d647405e66 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/cpu/main.cpp @@ -0,0 +1,592 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "../common/host_support.h" +#include "../common/winner_workload_host.h" + +#define PA_DEVICE inline +#define PA_DEVICE_NOINLINE static __attribute__((noinline)) +#define PA_GM +// CPU 后端直接实例化与设备端相同的公共调度器；这里只消去 AICore 地址空间 +// 修饰符，不另写一套简化状态机，因此它可以承担协议和边界回归。 +#include "../common/pa_scheduler_core.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +#if PA_BUILD_PERF_CLOCK +thread_local uint32_t g_perf_clock_read_count = 0; +#endif + +template +inline void EmitNops() { + // 编译期 Count 配合强制展开，避免编译器把空循环折叠掉。 +#if defined(__clang__) +#pragma clang loop unroll(full) +#elif defined(__GNUC__) +#pragma GCC unroll 256 +#endif + for (uint32_t index = 0; index < Count; ++index) { + asm volatile("nop"); + } +} + +inline void RuntimeNop(uint32_t count) { + // 与 AscendC 后端采用相同的 256 + 二进制尾块分解，保证入参含义一致。 + // x86 nop 的吞吐和 CPU 线程调度都不同于 A5，所以这里只复现指令数量， + // 不能把 CPU 测得时间解释为 A5 kernel 时间。 + while (count >= 256) { + EmitNops<256>(); + count -= 256; + } + if ((count & 128U) != 0) EmitNops<128>(); + if ((count & 64U) != 0) EmitNops<64>(); + if ((count & 32U) != 0) EmitNops<32>(); + if ((count & 16U) != 0) EmitNops<16>(); + if ((count & 8U) != 0) EmitNops<8>(); + if ((count & 4U) != 0) EmitNops<4>(); + if ((count & 2U) != 0) EmitNops<2>(); + if ((count & 1U) != 0) EmitNops<1>(); +} + +// CPU 后端用相同的 128x128 float 输入、输出布局执行真实算术，以便在不依赖 +// CANN 的环境中回归任务分派和输出闭环。这里的普通 CPU 浮点循环只与数学 +// 结果对等，不冒充 A5 Cube/Vector 指令、流水线或 PMU 数据。 +__attribute__((noinline)) void RunRealMatrixWorkload( + const float *input_a, const float *input_b, float *output, uint32_t repeats +) { + using namespace pa_scheduler::winner_workload; + for (uint32_t iteration = 0; iteration < repeats; ++iteration) { + for (uint32_t row = 0; row < kTileRows; ++row) { + for (uint32_t column = 0; column < kTileCols; ++column) { + float accumulator = 0.0F; + for (uint32_t inner = 0; inner < kTileCols; ++inner) { + accumulator += input_a[static_cast(row) * kTileCols + inner] * + input_b[static_cast(inner) * kTileCols + column]; + } + output[static_cast(row) * kTileCols + column] = accumulator; + } + } + // 每轮都必须把完整结果物化到输出 tile，不能让 O3 因后续轮次覆盖同一 + // tile 而删除前一轮算术；这是编译器边界，不增加硬件 PMU 或 A5 屏障语义。 + asm volatile("" : : "r"(output) : "memory"); + } +} + +template +__attribute__((noinline)) void RunRealVectorWorkload( + const float *input_a, const float *input_b, float *output, uint32_t repeats +) { + using namespace pa_scheduler::winner_workload; + for (uint32_t iteration = 0; iteration < repeats; ++iteration) { + for (size_t element = 0; element < kTileElements; ++element) { + output[element] = Multiply ? input_a[element] * input_b[element] + : input_a[element] + input_b[element]; + } + // 与矩阵路径相同，明确保留每一次完整 elementwise 迭代。 + asm volatile("" : : "r"(output) : "memory"); + } +} + +__attribute__((noinline)) void ExecuteRealWinnerWorkload( + pa_scheduler::SchedulerState *state, pa_scheduler::WorkerState &worker, + pa_scheduler::TaskKind kind +) { + using namespace pa_scheduler; + using namespace pa_scheduler::winner_workload; + const WinnerWorkloadConfig &config = state->winner_workload; + const uint32_t repeats = WorkloadCountForKind(config.repeats, kind); + const bool role_matches = + (worker.role == CoreRole::Aic && (kind == TaskKind::Qk || kind == TaskKind::Pv)) || + (worker.role == CoreRole::Aiv && (kind == TaskKind::Sf || kind == TaskKind::Up)); + // 与设备实现采用同一组版本、范围和角色门禁；配置错误不解引用 workspace， + // host 的 active tile 数值/sentinel 校验会把本轮判为失败。 + if (config.version != kWinnerWorkloadConfigVersion || config.workspace_base == 0 || + config.workspace_bytes < kWorkspaceBytes || worker.core_idx < 0 || + static_cast(worker.core_idx) >= kWorkers || repeats == 0 || + repeats > kMaxRealComputeCount || !role_matches) { + return; + } + + float *workspace = reinterpret_cast(static_cast(config.workspace_base)); + const uint32_t kind_slot = (kind == TaskKind::Pv || kind == TaskKind::Up) ? 1U : 0U; + const size_t output_tile = + kSharedInputTiles + static_cast(worker.core_idx) * kOutputTilesPerWorker + kind_slot; + float *output = workspace + output_tile * kTileElements; + const float *input_a = workspace; + const float *input_b = workspace + kTileElements; + if (worker.role == CoreRole::Aic) { + RunRealMatrixWorkload(input_a, input_b, output, repeats); + } else if (kind == TaskKind::Sf) { + RunRealVectorWorkload(input_a, input_b, output, repeats); + } else { + RunRealVectorWorkload(input_a, input_b, output, repeats); + } +} + +struct CpuOps { + // CPU 后端只验证调度协议与 raw schema，没有建立与 A5 CCEC + // 同构的“atomic 返回值依赖 + SYS_CNT”硬件边界；因此必须标记为 + // source_issue，不能让 x86 built-in 的函数返回冒充 A5 return_ready。 + static constexpr bool kAtomicReturnReadyObserved = false; + + // 用 fetch_add(0) 模拟 A5 atomicAdd(addr, 0) 原子读，而不是退化为普通 + // CPU load。Acquire/AcqRel 只建立本 CPU 协议回归需要的发布/观察关系， + // 不模拟 A5 cache 或设备内存模型细节。 + static inline int32_t Load(volatile int32_t *address) { + return __atomic_fetch_add(address, static_cast(0), __ATOMIC_ACQUIRE); + } + + static inline int64_t Load(volatile int64_t *address) { + // 保留原子 add-zero 路径，让 96 个 pthread 仍在同一批热点地址上竞争。 + return __atomic_fetch_add(address, static_cast(0), __ATOMIC_ACQUIRE); + } + + static inline uint64_t Load(volatile uint64_t *address) { + return __atomic_fetch_add(address, static_cast(0), __ATOMIC_ACQUIRE); + } + + static inline int32_t Exchange(volatile int32_t *address, int32_t value) { + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } + + static inline int64_t Exchange(volatile int64_t *address, int64_t value) { + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } + + static inline uint64_t Exchange(volatile uint64_t *address, uint64_t value) { + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } + + static inline int64_t CompareExchange( + volatile int64_t *address, int64_t expected, int64_t desired + ) { + // 与 production atomic wrapper 保持一致：返回线性化点观察到的 + // 旧值，而不是 bool。失败时目标字保持原样，调用方据此保留现场。 + int64_t observed = expected; + (void)__atomic_compare_exchange_n( + address, &observed, desired, false, + __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE + ); + return observed; + } + + static inline int64_t FetchAdd(volatile int64_t *address, int64_t value) { + return __atomic_fetch_add(address, value, __ATOMIC_ACQ_REL); + } + + static inline int64_t FetchMax(volatile int64_t *address, int64_t value, uint64_t &retries) { + // CPU 没有直接对应本测试签名的 fetch-max，用 CAS loop 实现同一返回值 + // 语义；retries 仅用于诊断软件竞争，不能与 A5 硬件 AtomicMax 对比。 + int64_t current = __atomic_load_n(address, __ATOMIC_ACQUIRE); + retries = 0; + while (value > current) { + if (__atomic_compare_exchange_n(address, ¤t, value, true, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) { + break; + } + ++retries; + } + return current; + } + + // 前后的 Exchange 已使用 AcqRel，Publish 使用 Release，因此这里不重复 + // 插入 fence，保持与设备适配层相同的调用边界。 + static inline void StoreBarrier() {} + + // 将 steady_clock 统一换算成纳秒，数值上适配公共模型的 1 GHz tick 标度； + // 这不表示 CPU 物理时钟为 1 GHz，也不保证实际分辨率达到 1 ns。 + static inline uint64_t Now() { + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) + .count() + ); + } + +#if PA_BUILD_PERF_CLOCK + static inline void ResetPerfClockReadCount() { g_perf_clock_read_count = 0; } + + static inline uint32_t PerfClockReadCount() { return g_perf_clock_read_count; } + + static inline uint64_t PerfClockNow() { + ++g_perf_clock_read_count; + return Now(); + } +#endif + + template + static inline uint64_t NowAfterAtomicResult(T value) { + // 空 asm 让编译器保留返回值到计时点的数据依赖，不额外插入 CPU fence。 + asm volatile("" : "+r"(value)); + return Now(); + } + + static inline void ExecuteKernel( + pa_scheduler::SchedulerState *state, pa_scheduler::WorkerState &worker, pa_scheduler::TaskKind kind, + uint32_t nop_count + ) { + if (state->winner_workload.mode == + static_cast(pa_scheduler::WinnerWorkloadMode::RealCompute)) { + ExecuteRealWinnerWorkload(state, worker, kind); + return; + } + RuntimeNop(nop_count); + } + + static inline bool PmuWindowStart(pa_scheduler::SchedulerState *, uint32_t) { return false; } + + static inline void PmuWindowStop(pa_scheduler::SchedulerState *, uint32_t, bool) {} + + static inline void SpinHint() {} + + static inline void InvalidateRegion(const void *, uint64_t) { + // CPU 没有 A5 DCache line 失效指令；这里仅提供保守的本线程顺序边界， + // 接口占位但不模拟设备 cache line 行为。共享状态本身仍使用 atomic。 + std::atomic_thread_fence(std::memory_order_seq_cst); + } + + static inline void FlushRegion(void *, uint64_t) { std::atomic_thread_fence(std::memory_order_seq_cst); } + + static inline void Publish(uint64_t *address, uint64_t value) { + // Release store 对应设备端 bypass-DCache 结果发布的可见性边界。 + __atomic_store_n(address, value, __ATOMIC_RELEASE); + } +}; + +} // namespace + +// CPU runner 不承担 A5 性能对比，只负责用同一份 SchedulerState 和公共调度器 +// 做协议回归。生命周期为参数解析、host 内存准备、逐轮 96 线程执行、严格校验、 +// 可选泳道后处理，最后统一释放 trace buffer。 +int main(int argc, char **argv) { + pa_scheduler::host::Options options; + pa_scheduler::host::WinnerWorkloadOptions workload_options; + std::vector common_argv; + if (!pa_scheduler::host::ParseWinnerWorkloadOptions( + argc, argv, &workload_options, &common_argv + )) { + return EXIT_FAILURE; + } + const pa_scheduler::host::ParseStatus parse_status = pa_scheduler::host::ParseOptions( + static_cast(common_argv.size()), common_argv.data(), false, &options + ); + if (parse_status != pa_scheduler::host::ParseStatus::Ok) { + if (parse_status == pa_scheduler::host::ParseStatus::Help) { + std::fprintf( + stderr, + "CPU winner workload options: [--winner-workload scalar-nop|real-compute] " + "[--real-compute-count N | --real-compute-counts QK,SF,PV,UP] " + "[--real-compute-pattern constant|layout-diagnostic]\n" + ); + std::fprintf( + stderr, + "Default: real-compute, constant, counts=6,28,4,1; " + "scalar-nop is the calibration compatibility mode.\n" + ); + } + return parse_status == pa_scheduler::host::ParseStatus::Help ? EXIT_SUCCESS : EXIT_FAILURE; + } + if (!pa_scheduler::host::ValidateWinnerWorkloadOptions(workload_options)) { + return EXIT_FAILURE; + } +#if PA_BUILD_PERF_CLOCK + if (options.runs != 1 || options.trace_enabled || options.trace_atomics || + options.profile_phases || options.analyze_swimlane || + !options.swimlane_json.empty()) { + std::fprintf( + stderr, + "CPU perf-clock requires one trace-free run: " + "--runs 1 --no-swimlane and no trace/profile options.\n" + ); + return EXIT_FAILURE; + } +#endif + const bool real_compute = + workload_options.mode == pa_scheduler::WinnerWorkloadMode::RealCompute; + std::vector workload_image; + std::vector workload_outputs; + pa_scheduler::host::PrintBanner("CPU", options); + pa_scheduler::host::PrintWinnerWorkloadConfig(workload_options, options.nops); + std::printf( + "[NOTE] CPU scalar NOP preserves instruction count; real-compute preserves arithmetic " + "and workspace semantics. Neither represents A5 engine timing or PMU.\n" + ); + + // SchedulerState 很大，放到 heap 而不是主线程栈；trace 继续保持独立的 + // 64 字节对齐区域，以复用设备端完全相同的二进制布局。 + std::unique_ptr state(new pa_scheduler::SchedulerState); + void *trace_memory = nullptr; + if (options.trace_enabled) { + trace_memory = std::aligned_alloc(64, pa_scheduler::kTraceBytes); + if (trace_memory == nullptr) { + std::fprintf(stderr, "Cannot allocate %zu-byte swimlane trace buffer.\n", pa_scheduler::kTraceBytes); + return EXIT_FAILURE; + } + } + auto *trace_header = static_cast(trace_memory); + std::vector spans; + std::vector startup_barrier_spans; + std::vector final_barrier_spans; + std::vector final_drain_spans; + std::vector lifecycle_spans; + bool all_passed = true; + bool postprocess_ok = true; +#if PA_BUILD_PERF_CLOCK + std::atomic perf_clock_reads{0}; + std::atomic perf_clock_read_shape_ok{true}; +#endif + // 每轮复用大块 host 分配，只重置公共状态和 trace header。与设备后端一样， + // runs>1 表示同进程热运行，不等价于多个独立首轮。 + for (uint32_t run = 1; run <= options.runs; ++run) { + pa_scheduler::host::InitializeState(state.get(), options); +#if PTO_FDWIC_SHARED_MAP + pa_scheduler::host::SharedHostTaskPlan launch_plan; + pa_scheduler::host::SharedHostHeapAdmission heap_admission; + std::string admission_error; + if (!pa_scheduler::host::BuildSharedHostTaskPlan( + *state, &launch_plan, &admission_error + ) || + !pa_scheduler::host::ValidateSharedHostHeapAdmission( + launch_plan, state->heap_size, + &heap_admission, &admission_error + )) { + std::fprintf( + stderr, + "Shared launch rejected before CPU worker start: %s\n", + admission_error.c_str() + ); + all_passed = false; + break; + } + pa_scheduler::host::PrintSharedHostHeapAdmission( + launch_plan, heap_admission + ); +#endif + pa_scheduler::host::ConfigureTrace(state.get(), options, trace_memory); + if (real_compute) { + // runs>1 必须恢复所有输出 sentinel，避免上一轮 winner 的 tile 被误认 + // 为本轮结果；输入也由公共 helper 恢复成与设备后端相同的选定 pattern。 + pa_scheduler::host::InitializeWinnerWorkloadBuffers( + workload_options, &workload_image, &workload_outputs + ); + } + pa_scheduler::host::ConfigureWinnerWorkload( + state.get(), workload_options, real_compute ? workload_image.data() : nullptr + ); + if (options.trace_enabled) { + pa_scheduler::host::InitializeTraceHeader(trace_header); + } + const auto wall_begin = std::chrono::steady_clock::now(); + // 固定创建 96 个参与者：worker 0..31 扮演 AIC，32..95 扮演 AIV。 + // 每个线程仍会进入公共 started_count 屏障，再共同回放完整 task 流。 + std::vector workers; + workers.reserve(pa_scheduler::kWorkers); + for (uint32_t worker_id = 0; worker_id < pa_scheduler::kWorkers; ++worker_id) { + const pa_scheduler::CoreRole role = + worker_id < pa_scheduler::kAicWorkers ? pa_scheduler::CoreRole::Aic : pa_scheduler::CoreRole::Aiv; +#if PA_BUILD_PERF_CLOCK + workers.emplace_back([ + state_pointer = state.get(), worker_id, role, + &perf_clock_reads, &perf_clock_read_shape_ok + ]() { + CpuOps::ResetPerfClockReadCount(); + pa_scheduler::RunScheduler(state_pointer, worker_id, role); + const uint32_t reads = CpuOps::PerfClockReadCount(); + perf_clock_reads.fetch_add(reads, std::memory_order_relaxed); + if (reads != 2) { + perf_clock_read_shape_ok.store(false, std::memory_order_relaxed); + } + }); +#else + workers.emplace_back([state_pointer = state.get(), worker_id, role]() { + pa_scheduler::RunScheduler(state_pointer, worker_id, role); + }); +#endif + } + // join 是本后端的 kernel 完成屏障；所有 worker 退出后才能读取最终状态， + // 对应设备 runner 的 aclrtSynchronizeStream。 + for (std::thread &worker : workers) + worker.join(); + const auto wall_end = std::chrono::steady_clock::now(); + const double host_us = std::chrono::duration(wall_end - wall_begin).count(); + if (real_compute) { + const size_t output_begin = + static_cast(pa_scheduler::winner_workload::kSharedInputTiles) * + pa_scheduler::winner_workload::kTileElements; + std::copy_n( + workload_image.begin() + output_begin, workload_outputs.size(), + workload_outputs.begin() + ); + } + // host 内存沿用与 A5 相同的 TraceHeader + 每 worker 固定跨度 ABI； + // 分析器和 raw JSON writer 因而可以与设备后端共用同一回调接口。 + const auto read_trace_records = + [trace_memory](uint32_t worker, uint32_t count, pa_scheduler::TraceRecord *records) { + const size_t offset = + pa_scheduler::TraceRecordsOffset(worker); + std::memcpy( + records, static_cast(trace_memory) + offset, + static_cast(count) * sizeof(pa_scheduler::TraceRecord) + ); + return true; + }; +#if PTO_FDWIC_SHARED_MAP + const auto read_submit_claim_records = + [trace_memory]( + uint32_t worker, uint32_t count, + pa_scheduler::SharedSubmitClaimTraceRecord *records + ) { + const size_t offset = + pa_scheduler::TraceSubmitClaimOffset(worker); + std::memcpy( + records, + static_cast(trace_memory) + offset, + static_cast(count) * + sizeof(pa_scheduler::SharedSubmitClaimTraceRecord) + ); + return true; + }; +#endif + // 先完成严格语义校验，再允许写出；失败运行不会生成可误认成有效 + // 基线的泳道 JSON。 + const pa_scheduler::host::Metrics metrics = pa_scheduler::host::Validate( + *state, run, host_us, options.trace_enabled ? trace_header : nullptr + ); +#if PA_BUILD_PERF_CLOCK + const bool perf_clock_reads_ok = + perf_clock_read_shape_ok.load(std::memory_order_relaxed) && + perf_clock_reads.load(std::memory_order_relaxed) == 2U * pa_scheduler::kWorkers; + std::printf( + "[PERF-CLOCK] perf_boundary_reads_per_worker=2 " + "perf_boundary_total_reads=%u expected=%u status=%s\n", + perf_clock_reads.load(std::memory_order_relaxed), + 2U * pa_scheduler::kWorkers, + perf_clock_reads_ok ? "PASS" : "FAIL" + ); +#endif + const bool workload_passed = + !real_compute || pa_scheduler::host::ValidateRealComputeOutputs( + *state, workload_options, workload_outputs, run + ); + all_passed &= metrics.passed && workload_passed; +#if PA_BUILD_PERF_CLOCK + all_passed &= perf_clock_reads_ok; +#endif + spans.push_back(metrics.submit_span_us); + startup_barrier_spans.push_back(metrics.startup_barrier_span_us); + final_barrier_spans.push_back(metrics.final_barrier_span_us); + final_drain_spans.push_back(metrics.final_drain_span_us); + lifecycle_spans.push_back(metrics.lifecycle_span_us); + // 分析只打印统计，导出则写 raw JSON；两者失败都标记 postprocess， + // 与调度语义失败分开报告，便于区分协议问题和产物问题。 + if (options.analyze_swimlane && + !pa_scheduler::host::AnalyzeSwimlaneRecords( + *trace_header, *state, read_trace_records +#if PTO_FDWIC_SHARED_MAP + , read_submit_claim_records +#endif + )) { + postprocess_ok = false; + break; + } + if (!options.swimlane_json.empty()) { + if (!metrics.passed || !workload_passed) { + std::fprintf( + stderr, + "Skipping swimlane export because semantic or winner-workload validation failed.\n" + ); + postprocess_ok = false; + break; + } + if (!pa_scheduler::host::ExportSwimlaneRecords( +#if PTO_FDWIC_SHARED_MAP + *trace_header, *state, options.swimlane_json, +#else + *trace_header, options.swimlane_json, +#endif + workload_options.mode, + real_compute + ? workload_options.repeats + : pa_scheduler::WorkloadCounts{ + options.nops.qk, options.nops.sf, options.nops.pv, options.nops.up + }, + real_compute + ? pa_scheduler::host::RealComputePatternName(workload_options.pattern) + : "none", + options.final_barrier_shape, options.trace_atomics, + read_trace_records +#if PTO_FDWIC_SHARED_MAP + , read_submit_claim_records +#endif + )) { + postprocess_ok = false; + break; + } + } + } + + // host 准入可以在第一个 worker 启动前拒绝本轮，此时没有可统计样本。 + // 显式输出 0 和 completed_runs=0，避免把空 vector 交给 Median()。 + const double median_submit_span_us = + spans.empty() ? 0.0 : pa_scheduler::host::Median(spans); +#if PA_BUILD_PERF_CLOCK + std::printf( + "[SUMMARY] runs=%u completed_runs=%zu final_shape=%s " + "median_submit_span_us=%.3f " + "lifecycle_timing=disabled semantic_status=%s postprocess_status=%s\n", + options.runs, spans.size(), + pa_scheduler::host::FinalBarrierShapeName(options.final_barrier_shape), + median_submit_span_us, + all_passed ? "PASS" : "FAIL", + postprocess_ok ? "PASS" : "FAIL" + ); +#else + const double median_startup_barrier_us = + startup_barrier_spans.empty() + ? 0.0 + : pa_scheduler::host::Median(startup_barrier_spans); + const double median_final_barrier_us = + final_barrier_spans.empty() + ? 0.0 + : pa_scheduler::host::Median(final_barrier_spans); + const double median_final_drain_us = + final_drain_spans.empty() + ? 0.0 + : pa_scheduler::host::Median(final_drain_spans); + const double median_lifecycle_us = + lifecycle_spans.empty() + ? 0.0 + : pa_scheduler::host::Median(lifecycle_spans); + std::printf( + "[SUMMARY] runs=%u completed_runs=%zu final_shape=%s " + "median_submit_span_us=%.3f median_startup_barrier_us=%.3f " + "median_final_barrier_us=%.3f median_final_drain_us=%.3f median_lifecycle_us=%.3f " + "semantic_status=%s postprocess_status=%s\n", + options.runs, spans.size(), + pa_scheduler::host::FinalBarrierShapeName( + options.final_barrier_shape + ), + median_submit_span_us, median_startup_barrier_us, + median_final_barrier_us, median_final_drain_us, + median_lifecycle_us, all_passed ? "PASS" : "FAIL", + postprocess_ok ? "PASS" : "FAIL" + ); +#endif + // std::free(nullptr) 合法，因此关闭泳道时也走同一条收尾路径。 + std::free(trace_memory); + return all_passed && postprocess_ok ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/tests/atomic_probe/pa_scheduler/loser_overhead.md b/tests/atomic_probe/pa_scheduler/loser_overhead.md new file mode 100644 index 0000000000..9bdfec6dd6 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/loser_overhead.md @@ -0,0 +1,338 @@ +# Shared claim loser Submit 开销分析 + +本文回答：shared tensormap runtime 的 **claim loser** 在泳道里为何 +**Submit 父区间经常 >1µs**，并结合样本 + +`Downloads/per_task_deps_prepared_register_writer_partition_r5ij_b256_merged_swimlane.json` + +说明：**主因是固定公共外壳三段叠加**，以及其中有多少来自 **泳道图生成（插桩）本身**。 + +相关代码：`common/pa_scheduler_core.h` 的 `SubmitCallbackTask` / +`FinishSharedLoserSubmit` / `CloseSharedCallbackSubmit`。 + +--- + +## 1. 结论（先看这个） + +1. Loser **已经不做** Materialize / Register / Fanin / TensorMap / 重构参；轻路径成立。 +2. 但泳道契约仍要求每个逻辑 Submit 都有 **`EfDrain` + `Claim`**；另有 + `PrepareSharedTaskOutputs`（**只声明 `(task_id,slot)` 符号，不物化 descriptor、 + 不分配 heap**）与收尾。 +3. 在上述 R5ij b256 shared 泳道中，`claim.lost` 的 Submit 父区间 + **median ≈ 1.33 µs**，约 **89% >1µs**；去掉 EfDrain 内偶发 Kernel 后仍约 **71% >1µs**。 +4. **多数时候没有任何单项单独 >1µs**，而是： + + ```text + 空 EfDrain（~0.2 µs 中位） + + Claim（含/不含 ClaimMax，~0.7 µs 中位） + + submit_tail_gap / residual（~0.4 µs 中位） + ≈ 1.3 µs + ``` + +5. **很大一部分测量值来自泳道插桩**（多次 `SYS_CNT` / `Ops::Now()`、 + `WriteTrace`、ClaimMax 的 atomic bracket）。业务上真正不可避免的主要是 + **一次 `atomicMax`（attempted loser）+ 极薄的符号/收尾**；不能把泳道上的 + 1.3 µs 直接当成无观察热路径的净成本。 +6. `PrepareSharedTaskOutputs` 与 `WriteTrace(Submit)` **语义独立**：前者是编排 ABI + （详见 §2.1），后者是泳道/计数闭合。 + +--- + +## 2. Loser 实际执行路径 + +```text +BeginCallbackSubmit ← 在 Submit.start 之外 +── Submit 父区间 ── + EfDrain = DrainReady() ← 每个逻辑 task 必做 + Claim = role 路由 + 可能 atomicMax + PrepareSharedTaskOutputs() ← 声明 (task_id, slot) 符号 + loser 跳过 BuildCallbackSubmitArgs + FinishSharedLoserSubmit() + → CloseSharedCallbackSubmit() ← ++submits + 写 Submit 记录 +── Submit.end ── +``` + +Winner 才进入 Materialize / Register / Build。Loser 零 TensorMap 访问—— +这与定向测试（如 loser 零 map access、guard-page）一致。 + +### 2.1 `PrepareSharedTaskOutputs()`：做什么、不做什么 + +它和后面的 `CloseSharedCallbackSubmit()` → `TraceTimestamp` / `WriteTrace(Submit)` +**只是顺序相邻，语义独立**。前者不是为了配合泳道取时/落盘。 + +#### 做什么 + +只在本核 `context.shared_result`（`SharedTaskOutputs`）里登记稳定符号句柄： + +```text +(producer_task_id = 当前 task_id, output_slot = 0 .. count-1) +``` + +对应 `pa_frontend.h`：按 `FrontendTaskOutputCount(kind)` 循环 `AddOutputRef`。 +源码注释写明：loser 也必须把同一组 `(producer, slot)` 交给本核后续 +orchestration；**仅声明稳定符号**。 + +#### 为什么 loser 也要做 + +Shared 下不能把本核私有的 `TensorDesc*` 传给后续编排。Submit 的“返回值”是 +`SharedOutputRef`，identity 只由 `(task_id, slot)` 决定，**与谁 claim 赢无关**。 + +每个 worker 都按同一顺序回放全部 task。本核这次 claim 输了，后面仍要继续构参， +例如： + +```text +Alloc 输了 → 仍要用 Alloc 的 OutputRef(0/1/2) 挂后续 UP 输入 +QK 输了 → 仍要用 QK 的 OutputRef 挂 SF 的 Input +``` + +若跳过 `PrepareSharedTaskOutputs`，本核 orchestration 没有这组符号，后续 +`AppendSharedOutputRef(...)` 会断。Winner 负责真正 publish descriptor；loser +只需同一套**逻辑句柄**把图接下去。 + +`SubmitCallbackTask` 中的注释与此一致：fresh Output 返回值是 task/slot 符号， +不依赖哪个 worker 获胜；在 finish 前为所有 replay actor 建立同一句柄集。 + +#### 不做什么（常见误解） + +| 误解 | 事实 | +| --- | --- | +| 生成 output tensor descriptor | **否**。不写 `TensorDesc`，不读 shared cell | +| 通过 heap ring 分配输出缓冲 | **否**。不碰 `heap_base` / `heap_size` / vend | +| 发布 `published` / 更新 `last_writer` | **否**。那是 winner 侧 `PublishSharedTaskOutputs` 等 | +| 为了 `WriteTrace(Submit)` 才调用 | **否**。与泳道闭合无关，是编排 ABI | + +真正做 descriptor + heap 的是 **winner** 路径，例如： + +- `MaterializeTask(...)`（按 heap 分配物理输出）； +- `PublishSharedTaskOutputs(...)`（把 descriptor 写入 + `SharedOutputCell.tensors[]` 并发布）。 + +Loser 只有符号；要用地址/shape 时，后续 winner 再等 `published` 并从 shared +cell 读取 descriptor。 + +#### 和 `CloseSharedCallbackSubmit` 的分工 + +| 调用 | 目的 | +| --- | --- | +| `PrepareSharedTaskOutputs` | 填 `shared_result`，给本核后续编排用 | +| `CloseSharedCallbackSubmit` → `TraceTimestamp` + `WriteTrace(Submit)` | 计 Submit 次数、闭合泳道父区间 | + +`PrepareSharedTaskOutputs` 本身只是几次 `AddOutputRef`，成本很小；tail 里 +~0.4 µs 主要来自写 Claim 记录、取 `Submit.end` 与薄校验，而不是“物化输出”。 + +--- + +## 3. 泳道样本测量（R5ij b256） + +| 项 | 值 | +| --- | ---: | +| 模式 | shared / real-compute / schema-v5 | +| 时间单位 | µs（QK≈44 µs 可交叉验证） | +| Submit 总数 | 122,880（96×1280） | +| `claim.won` / `lost` / `not_attempted` | 1,280 / 72,448 / 49,152 | +| 完整 Submit 墙钟（首末） | ≈ 3.406 ms | + +### 3.1 Submit 父区间按 claim 结果 + +| 口径 | n | median | mean | p95 | >1µs | +| --- | ---: | ---: | ---: | ---: | ---: | +| `claim.lost` | 72,448 | **1.327** | 1.785 | 3.256 | **88.9%** | +| `not_attempted` | 49,152 | 0.909 | 1.684 | 3.042 | 45.4% | +| 上述且 EfDrain **无 Kernel** | 120,592 | **1.295** | 1.452 | 3.061 | **71.1%** | +| EfDrain **含 Kernel** 的 loser | 1,008 | **44.0** | 36.7 | 59.5 | （drain 执行） | + +仅 **0.8%** loser Submit 被前序 winner 的 Kernel 嵌进 EfDrain；去掉后仍有约七成 >1µs。 +因此日常看到的 “loser >1µs” **主要是空外壳**，不是偶发算力污染。 + +空 loser 时长直方图（无 Kernel）： + +| 区间 (µs) | 占比 | +| --- | ---: | +| [0, 1) | ~29% | +| **[1.0, 1.5)** | **~35%**（主体） | +| [1.5, 3) | ~31% | +| ≥3 | ~5.6% | + +### 3.2 空 loser 且 Submit>1µs 的三段构成（n=85,727） + +| 段 | median | mean | 约占 Submit | +| --- | ---: | ---: | ---: | +| Claim | **0.74** | 0.78 | **~45%** | +| EfDrain（空） | 0.21 | 0.52 | ~30% | +| residual（≈`submit_tail_gap`） | **0.43** | 0.44 | ~25% | + +“谁单独就能 >1µs”： + +| 模式 | 占比 | +| --- | ---: | +| **三项都 ≤1µs，但加总 >1µs** | **~65%** | +| 仅 EfDrain >1 | ~18% | +| 仅 Claim >1 | ~14% | +| residual 单独 >1 | ≈0 | + +按“哪一段最大”：Claim 主导 ~73%，EfDrain ~20%，residual ~7%。 + +`claim.lost` 中 `claim_max` 嵌在 Claim 内，约占 Claim 的一半 +（ClaimMax median ≈0.30 µs / Claim ≈0.68 µs）。`not_attempted` **没有** +ClaimMax，Submit median 仍约 0.91 µs——说明 **>1µs 不是单靠 FetchMax**。 + +--- + +## 4. 为什么说「主因是三段叠加」 + +### 4.1 字面含义 + +对空 loser，典型中位数大约是： + +```text +0.20 (EfDrain) + 0.68 (Claim) + 0.43 (tail) ≈ 1.31 µs +``` + +没有哪一段“坏到 1µs 以上才拖垮整体”；是 **三条固定必经路径的耗时相加** +越过 1µs 线。65% 的 >1µs 样本属于这种「分项都不大、总和过线」的模式。 + +### 4.2 结构原因：loser 仍走完整 Submit 外壳 + +为了 SPMD 回放与泳道闭合，每个 actor 对每个逻辑 task 仍调用同一套 +`SubmitCallbackTask` 前缀。参考路径更接近 `rt_submit_loser` 只回符号； +当前实现为了： + +- 每核 `submits == 5*batches`； +- 每个 Submit 都有 `EfDrain`/`Claim` child（`SHARED_REQUIRED_ON_EVERY_SUBMIT`）； +- loser 也要 `PrepareSharedTaskOutputs` 以便后续 orchestration 持有同一套符号； + +而把 **EfDrain + Claim + 收尾** 留在了 loser 热路径上。这三段与 TensorMap +无关，但是 **每次逻辑 task 都付一次**。 + +### 4.3 计时边界如何把成本拆进这三段（关键） + +源码顺序（泳道构建，`Profile=true`）大致是： + +```text +submit_begin / efdrain_begin = TraceTimestamp() // 或与 task0 共用 +DrainReady() // 空则几乎立刻返回 +efdrain_end = TraceTimestamp() +WriteTrace(EfDrain) // ← 落在 Claim 时间窗内！ +claim_begin = efdrain_end +Claim() → 可能 TraceAtomicFetchMax（内部再 Now×2 + 写 Atomic 记录） +claim_end = TraceTimestamp() +WriteTrace(Claim) // ← 落在 submit_tail_gap！ +PrepareSharedTaskOutputs() // ← 只填符号，非 descriptor/heap；非泳道专用 +CloseSharedCallbackSubmit(): + submit_end = TraceTimestamp() + WriteTrace(Submit) +``` + +因此泳道上看到的三段 **并不等于** 三段纯业务函数的净耗时： + +| 泳道 span | 实际装了什么 | +| --- | --- | +| **EfDrain** | `DrainReady` 本体 + **结束处一次 `Now()`**（`WriteTrace(EfDrain)` 还不在里面） | +| **Claim** | `WriteTrace(EfDrain)` + role 路由 +（lost 时）ClaimMax bracket + Claim 收尾 + **结束处一次 `Now()`** | +| **submit_tail_gap / residual** | `WriteTrace(Claim)` + `PrepareSharedTaskOutputs`（薄：仅符号）+ loser 校验/收尾 + **`Now()` 取 Submit.end** | + +这解释了： + +- 为什么空 EfDrain 中位还有 ~0.2 µs（主要是边界取时，而不是扫 slot）； +- 为什么 Claim 明显大于内嵌的 ClaimMax（另一半是 **上一段的 WriteTrace + 外壳**）； +- 为什么 residual 稳定在 ~0.4 µs 且与 `submit_tail_gap` 几乎同分布（主要是 **本段 WriteTrace + 取时 + 薄收尾**）。 + +**叠加过 1µs 的直接算术原因**：每段都带着「边界时钟 / 落盘」的固定税，三段各收一次税，总和自然落在 1–1.5 µs 的桶里。 + +--- + +## 5. 是否因为泳道图生成本身的开销？ + +### 5.1 短答 + +**是，占大头；但不等于“全部都是假的”。** + +- **泳道插桩**（多次 `Ops::Now()` / PollBatch 边界、`WriteTrace`、ClaimMax 的 + begin/end bracket）把空 loser 的测量值系统性抬到 ~1.3 µs。 +- **真实业务仍保留**：attempted loser 的一次 **`atomicMax`（ClaimMax median ~0.3 µs）**、 + 空 `DrainReady` 的函数调用、符号声明与计数闭合。这些在无泳道构建里也会存在， + 但通常远小于泳道上看到的整段 Submit 父区间。 + +### 5.2 插桩如何“制造”三段税 + +一次空 `claim.lost` Submit 在泳道构建下至少涉及： + +| 动作 | 次数（量级） | 计入哪段 | +| --- | ---: | --- | +| `TraceTimestamp` / `Now`（EfDrain 结束） | 1 | EfDrain | +| `WriteTrace(EfDrain)` | 1 | **Claim** | +| ClaimMax：`Now` 起、`NowAfterAtomic` 止 + 写 Atomic 记录 | 2+写 | Claim（且显示为 `claim_max` child） | +| `TraceTimestamp`（Claim 结束） | 1 | Claim | +| `WriteTrace(Claim)` | 1 | **tail** | +| `TraceTimestamp`（Submit 结束） | 1 | tail | +| `WriteTrace(Submit)` | 1 | Submit.end 之后（常进下一 gap / OrchestrationTail） | + +`not_attempted` 没有 ClaimMax bracket，Submit 更短（median 0.91 µs），但仍有 +EfDrain/Claim/Submit 的边界取时与两次阶段 `WriteTrace`——与“插桩税为主、 +atomic 为辅”一致。 + +### 5.3 与无观察构建的旁证 + +- 文档约定：开启泳道后 **不应**把 bracket / Submit span 与未插桩基线直接相减当绝对占比 + （插桩改变布局、到达顺序与竞争）。 +- 历史 **S4.12a** 裁掉 loser 的空 finish 外壳后，路径与观察都正确，但 + **perf-clock 墙钟中性并已撤销**——说明当时墙钟并不卡在“再少一层可裁的 finish 壳”， + 也侧面说明 **泳道上显眼的 loser 外壳 ≠ 无观察主瓶颈的同等放大**。 +- `PA_BUILD_TRACE_FREE` / perf-clock / submit-pmu 路径会去掉额外 `SYS_CNT` 与 + span 写入；要用那些构建重新量空 loser，才能得到接近热路径的净成本。 + +### 5.4 如何区分「真业务」和「泳道税」（建议口径） + +| 问题 | 建议看什么 | +| --- | --- | +| 泳道上为何 >1µs | 本文：三段叠加 + 插桩边界错位计入 | +| 无观察时 loser 还要多久 | 同业务 **perf-clock / TRACE_FREE** A/B，不要用本 merged JSON 的 1.3 µs 当净成本 | +| atomic 本身 | 看 `claim_max` child（本样本 lost median ~0.30 µs），仍含 bracket 取时 | +| 是否被 kernel 污染 | 看 EfDrain 窗口内是否有 `QK/SF/PV/UP`（本样本仅 0.8%） | + +粗分本样本空 `claim.lost`（中位量级）： + +```text +ClaimMax 硬件+bracket ~0.30 µs ← 部分真、部分观察 +Claim 内其余（含写 EfDrain 记录等）~0.38 µs ← 多为泳道落盘/外壳 +空 EfDrain span ~0.20 µs ← 多为结束取时 +tail / residual ~0.43 µs ← 多为写 Claim 记录 + 取 Submit.end + 薄收尾 + （含 PrepareSharedTaskOutputs 符号声明，非 heap 物化） +──────────────────────────────────── +Submit 父区间 ~1.3 µs +``` + +因此：**“三项叠过 1µs”的现象，在这份泳道图上 largely 是泳道生成方式导致的计量形态**； +若关掉插桩，数字会明显下降，但 **ClaimMax + 仍存在的 Submit 外壳**不会降到零。 + +--- + +## 6. 和「理想 loser 轻路径」的差距 + +| | 参考意图 | 当前 standalone shared | +| --- | --- | --- | +| Claim 后 | `rt_submit_loser` 只回符号 | 仍跑完整 Submit 父区间 | +| EfDrain | 可对空/wrong-role 更早跳过（实验项） | 每 task 必进 | +| 观察 | 无 | 每段 Now + WriteTrace，且 WriteTrace 计入下一段 | + +优化若以「泳道上 loser <1µs」为目标，会主要在打插桩税，墙钟未必动。 +若以墙钟为目标，应： + +1. 用 exclusive JSON **剔除 EfDrain 含 Kernel 的 loser**； +2. 在 **TRACE_FREE / perf-clock** 上重测空 loser； +3. 再考虑 wrong-role / 空 slot 更早跳过 EfDrain（类 S4.12b），而不是继续抠 + `FinishSharedLoserSubmit` 里已很薄的符号返回。 + +--- + +## 7. 样本与入口索引 + +| 项 | 位置 | +| --- | --- | +| 本文依据泳道 | `per_task_deps_prepared_register_writer_partition_r5ij_b256_merged_swimlane.json` | +| Submit / Claim / loser 收尾 | `common/pa_scheduler_core.h` | +| `PrepareSharedTaskOutputs` / `SharedTaskOutputs` | `common/pa_frontend.h`（§2.1） | +| Winner 物化 / 发布 descriptor | `MaterializeTask`、`PublishSharedTaskOutputs`（`pa_shared_submit_path.h` 等） | +| TraceTimestamp / ClaimMax bracket | `common/pa_scheduler_core.h`, `common/pa_trace.h` | +| §12 vs 实现差异 | `shared_tensormap_imp_analysis.md` | +| 泳道残余语义 | `swimlane_opt_anal.md`（`submit_tail_gap` 等） | diff --git a/tests/atomic_probe/pa_scheduler/pmu_html_report.py b/tests/atomic_probe/pa_scheduler/pmu_html_report.py new file mode 100644 index 0000000000..c9e5f87ab1 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/pmu_html_report.py @@ -0,0 +1,984 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""将 standalone submit-pmu raw JSON 转成可离线浏览的自包含 HTML 报告。""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import math +import os +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Sequence +from urllib.parse import quote + +try: + from .pmu_sidecar_analyzer import analyze, load_capture +except ImportError: + from pmu_sidecar_analyzer import analyze, load_capture + + +REPORT_VERSION = 5 +AIC_COLOR = "#2563eb" +AIV_COLOR = "#ea580c" +GRID_COLOR = "#cbd5e1" +TEXT_COLOR = "#334155" + +# 本机 A5 受控 cold/warm 校准：PMU cycle 与 1 ns SYS_CNT 同窗读取。 +# ALL 使用整组实测比值；AIC/AIV 使用各自分组的实测比值，避免用名义频率替代证据。 +PMU_CALIBRATION_CYCLE_DELTA = 1_817_457 +PMU_CALIBRATION_SYS_TICK_NS = 1_101_593 +DEFAULT_PMU_CYCLES_PER_NS = 1.649844 +DEFAULT_AIC_PMU_CYCLES_PER_NS = 1.650062 +DEFAULT_AIV_PMU_CYCLES_PER_NS = 1.649731 + + +def default_output_path(input_path: Path) -> Path: + """由描述性 raw 名称稳定推导报告名称。""" + + name = input_path.name + if name.endswith("_raw.json"): + return input_path.with_name(f"{name[:-len('_raw.json')]}_report.html") + if name.endswith(".json"): + return input_path.with_name(f"{name[:-len('.json')]}_report.html") + return input_path.with_name(f"{name}_report.html") + + +def _format_count(value: int | float) -> str: + return f"{value:,.0f}" + + +def _format_per_core(value: int | float) -> str: + return f"{value:,.2f}" + + +def _format_rate(value: int | float) -> str: + return f"{value * 100:.4f}%" + + +def _cycles_to_us(cycles: int | float, cycles_per_ns: float) -> float: + """按受控校准频率把 PMU cycle 换算为单核等效微秒。""" + + return float(cycles) / cycles_per_ns / 1000.0 + + +def _cycle_time_cell(cycles: int | float, cycles_per_ns: float, decimals: int = 0) -> str: + """同时保留原始 cycle 与校准后的等效时间，避免丢失原始证据。""" + + cycle_text = f"{float(cycles):,.{decimals}f}" + return ( + f'{cycle_text} cycle' + f'≈{_cycles_to_us(cycles, cycles_per_ns):,.3f} µs' + ) + + +def _escape(value: object) -> str: + return html.escape(str(value), quote=True) + + +def _raw_href(input_path: Path, output_path: Path) -> str: + relative = os.path.relpath(input_path, output_path.parent) + return quote(relative.replace(os.sep, "/"), safe="/") + + +def _group_metrics( + analysis_group: dict[str, Any], + group_records: Sequence[dict[str, Any]], + cycles_per_ns: float, +) -> dict[str, int | float]: + cores = int(analysis_group["cores"]) + total_sum = int(analysis_group["total_cycles_sum"]) + scalar_sum = int(analysis_group["scalar_busy_sum"]) + if len(group_records) != cores: + raise ValueError("PMU report group record count does not match analyzer core count") + total_values = [int(record["total_cycles"]) for record in group_records] + scalar_values = [int(record["scalar_busy"]) for record in group_records] + request_values = [int(record["icache_requests"]) for record in group_records] + miss_values = [int(record["icache_misses"]) for record in group_records] + return { + "cores": cores, + "total_sum": total_sum, + "total_min": min(total_values), + "total_per_core": total_sum / cores, + "total_max": max(total_values), + "total_per_core_us": _cycles_to_us(total_sum / cores, cycles_per_ns), + "scalar_sum": scalar_sum, + "scalar_min": min(scalar_values), + "scalar_per_core": scalar_sum / cores, + "scalar_max": max(scalar_values), + "scalar_per_core_us": _cycles_to_us(scalar_sum / cores, cycles_per_ns), + "scalar_share": 0.0 if total_sum == 0 else scalar_sum / total_sum, + "non_scalar_busy_per_core": (total_sum - scalar_sum) / cores, + "cycles_per_ns": cycles_per_ns, + "requests_min": min(request_values), + "requests_per_core": analysis_group["icache_requests_per_core"], + "requests_max": max(request_values), + "misses_min": min(miss_values), + "misses_per_core": analysis_group["icache_misses_per_core"], + "misses_max": max(miss_values), + "miss_rate": analysis_group["icache_miss_rate"], + "serial_equivalent_us": analysis_group["first_order_miss_per_core_us"], + } + + +def _plot_value(record: dict[str, Any], metric: str) -> float: + if metric == "icache_miss_rate": + requests = int(record["icache_requests"]) + return 0.0 if requests == 0 else int(record["icache_misses"]) / requests + return float(record[metric]) + + +def _plot_label(metric: str, value: float) -> str: + if metric == "icache_miss_rate": + return f"{value * 100:.2f}%" + return f"{value:,.0f}" + + +def _distribution_svg( + records: Sequence[dict[str, Any]], + metric: str, + title: str, + description: str, + cycles_per_ns_by_role: dict[str, float] | None = None, +) -> str: + """按 physical_core_id 绘制离散点；核编号只是位置，不连接成时间线。""" + + width = 1080 + height = 310 + left, right, top, bottom = 78, 28, 34, 54 + plot_width = width - left - right + plot_height = height - top - bottom + points = [(record, _plot_value(record, metric)) for record in records] + maximum = max(value for _, value in points) + maximum = maximum * 1.08 if maximum > 0 else 1.0 + + def x_position(physical_id: int) -> float: + return left + plot_width * physical_id / 107.0 + + def y_position(value: float) -> float: + return top + plot_height * (1.0 - value / maximum) + + title_id = f"plot-{metric}-title" + desc_id = f"plot-{metric}-desc" + fragments = [ + f'', + f'{_escape(title)}', + f'{_escape(description)}', + ] + for index in range(5): + ratio = index / 4 + value = maximum * (1.0 - ratio) + y = top + plot_height * ratio + fragments.append( + f'' + ) + fragments.append( + f'' + f'{_escape(_plot_label(metric, value))}' + ) + + for tick in (0, 18, 36, 54, 72, 90, 107): + x = x_position(tick) + fragments.append( + f'' + ) + fragments.append( + f'{tick}' + ) + fragments.append( + f'physical_core_id（0–107，未连接）' + ) + + for record, value in sorted(points, key=lambda item: int(item[0]["physical_core_id"])): + role = str(record["role"]) + physical_id = int(record["physical_core_id"]) + x = x_position(physical_id) + y = y_position(value) + requests = int(record["icache_requests"]) + misses = int(record["icache_misses"]) + rate = 0.0 if requests == 0 else misses / requests + tooltip = ( + f"{role.upper()} worker={record['worker_id']} physical={physical_id} " + f"block={record['block_id']} lane={record['lane']} " + f"{metric}={_plot_label(metric, value)} whole_request={requests:,} " + f"whole_miss={misses:,} whole_rate={rate * 100:.4f}%" + ) + if metric in ("total_cycles", "scalar_busy") and cycles_per_ns_by_role is not None: + tooltip += ( + f" calibrated_time={_cycles_to_us(value, cycles_per_ns_by_role[role]):,.3f}us" + ) + if role == "aic": + fragments.append( + f'' + f'{_escape(tooltip)}' + ) + else: + fragments.append( + f'{_escape(tooltip)}' + ) + fragments.append("") + return "".join(fragments) + + +def _per_core_rows( + records: Sequence[dict[str, Any]], cycles_per_ns_by_role: dict[str, float] +) -> str: + rows: list[str] = [] + for record in sorted(records, key=lambda item: int(item["physical_core_id"])): + requests = int(record["icache_requests"]) + misses = int(record["icache_misses"]) + total = int(record["total_cycles"]) + scalar = int(record["scalar_busy"]) + cycles_per_ns = cycles_per_ns_by_role[str(record["role"])] + rate = 0.0 if requests == 0 else misses / requests + scalar_share = 0.0 if total == 0 else scalar / total + exact = bool(record.get("shadow_matches_primary")) + rows.append( + f'' + f"{int(record['worker_id'])}" + f"{int(record['physical_core_id'])}" + f"{_escape(str(record['role']).upper())}" + f"{int(record['block_id'])}" + f"{int(record['lane'])}" + f"{_cycle_time_cell(total, cycles_per_ns)}" + f"{_cycle_time_cell(scalar, cycles_per_ns)}" + f"{scalar_share * 100:.4f}%" + f"{requests:,}" + f"{misses:,}" + f"{rate * 100:.4f}%" + f"{'是' if exact else '否'}" + f"{'PASS' if record.get('trusted') is True else 'FAIL'}" + "" + ) + return "".join(rows) + + +def _phase_group_row(label: str, group: dict[str, Any]) -> str: + ratio = group["phase_observed_read_clear_ratio"] + ratio_text = "—" if ratio is None else f"{ratio * 100:.4f}%" + request_share = _format_share_bounds( + group["phase_icache_request_lower_bound_share_of_submit"], + group["phase_icache_request_upper_bound_share_of_submit"], + ) + miss_share = _format_share_bounds( + group["phase_icache_miss_lower_bound_share_of_submit"], + group["phase_icache_miss_upper_bound_share_of_submit"], + ) + time_share = group.get("phase_time_share_of_submit") + time_share_text = "—" if time_share is None else f"{time_share * 100:.4f}%" + phase_time_text = ( + "—" + if group.get("phase_elapsed_per_core_us") is None + else f"{group['phase_elapsed_per_core_us']:,.3f} µs" + ) + per_call_text = ( + "—" + if group.get("phase_elapsed_per_call_ns") is None + else f"{group['phase_elapsed_per_call_ns']:,.3f} ns" + ) + return ( + "" + f"{_escape(label)}" + f"{int(group['phase_calls_sum']):,}" + f"{phase_time_text}" + f"{per_call_text}" + f"{time_share_text}" + f"{group['phase_icache_requests_lower_bound_sum']:,}..{group['phase_icache_requests_upper_bound_sum']:,}" + f"{group['phase_icache_requests_lower_bound_per_core']:,.3f}..{group['phase_icache_requests_upper_bound_per_core']:,.3f}" + f"{request_share}" + f"{group['phase_icache_misses_lower_bound_sum']:,}..{group['phase_icache_misses_upper_bound_sum']:,}" + f"{group['phase_icache_misses_lower_bound_per_core']:,.3f}..{group['phase_icache_misses_upper_bound_per_core']:,.3f}" + f"{miss_share}" + f"{int(group['shadow_request_loss_sum']):,} / {int(group['shadow_miss_loss_sum']):,}" + f"{ratio_text}" + "" + ) + + +def _format_share_bounds(lower: float | None, upper: float | None) -> str: + """将局部事件占同组完整 Submit primary 的比例格式化为区间。""" + + if lower is None or upper is None: + return "—" + return f"{lower * 100:.4f}%..{upper * 100:.4f}%" + + +def _phase_share_metric( + group_label: str, + label: str, + metric_class: str, + lower_share: float | None, + upper_share: float | None, + lower_count: int, + upper_count: int, +) -> str: + """生成一个以完整 Submit primary 为 100% 的局部占比区间条。""" + + if lower_share is None or upper_share is None: + return ( + f'
' + f'
{_escape(label)}—
' + '
完整窗口分母为 0，比例不可计算
' + "
" + ) + lower_percent = lower_share * 100.0 + upper_percent = upper_share * 100.0 + bounds_text = f"{lower_percent:.4f}%..{upper_percent:.4f}%" + aria_label = ( + f"{label} 局部占完整 Submit primary，" + f"下界 {lower_percent:.4f}%，上界 {upper_percent:.4f}%" + ) + return f""" +
+
{_escape(label)}{bounds_text}
+ +
0%50%100%
+
事件数 {lower_count:,}..{upper_count:,}
+
+""" + + +def _phase_time_metric(group_label: str, group: dict[str, Any]) -> str: + """生成所选阶段的逐核累计时间占比；该值是单点观察，不伪造上下界。""" + + share = group.get("phase_time_share_of_submit") + if share is None: + return ( + f'
' + '
阶段时间—
' + '
本次 raw 未采集阶段 SYS_CNT，不能由 I-cache 或 PMU total 反推
' + "
" + ) + percent = float(share) * 100.0 + phase_per_core_us = float(group["phase_elapsed_per_core_us"]) + per_call_ns = group.get("phase_elapsed_per_call_ns") + per_call_text = "—" if per_call_ns is None else f"{float(per_call_ns):,.3f} ns/call" + aria_label = f"{group_label} 阶段逐核累计时间占同核完整 Submit {percent:.4f}%" + return f""" +
+
阶段时间{percent:.4f}%
+ +
0%50%100%
+
平均 {phase_per_core_us:,.3f} µs/核 · {per_call_text}
+
+""" + + +def _phase_share_card(label: str, group: dict[str, Any]) -> str: + """把一个角色组的时间/request/miss 局部占比放在同一张响应式卡片中。""" + + time_metric = _phase_time_metric(label, group) + request_metric = _phase_share_metric( + label, + "I-cache request", + "request", + group["phase_icache_request_lower_bound_share_of_submit"], + group["phase_icache_request_upper_bound_share_of_submit"], + int(group["phase_icache_requests_lower_bound_sum"]), + int(group["phase_icache_requests_upper_bound_sum"]), + ) + miss_metric = _phase_share_metric( + label, + "I-cache miss", + "miss", + group["phase_icache_miss_lower_bound_share_of_submit"], + group["phase_icache_miss_upper_bound_share_of_submit"], + int(group["phase_icache_misses_lower_bound_sum"]), + int(group["phase_icache_misses_upper_bound_sum"]), + ) + role_class = "" if label == "ALL" else f" role-{label.lower()}" + return f""" +
+
{_escape(label)}{int(group['phase_calls_sum']):,} calls · {group['phase_calls_per_core']:,.0f}/核
+ {time_metric} + {request_metric} + {miss_metric} +
+""" + + +def _pmu_stat_row( + label: str, + stat: str, + total_cycles: int | float, + scalar_cycles: int | float, + cycles_per_ns: float, + decimals: int = 0, +) -> str: + """生成一行紧凑的 PMU 分布统计，同时保留 raw cycle 与校准时间。""" + + return ( + f'' + f"{_escape(label)}" + f"{_cycle_time_cell(total_cycles, cycles_per_ns, decimals)}" + f"{_cycle_time_cell(scalar_cycles, cycles_per_ns, decimals)}" + "" + ) + + +def _pmu_role_card(label: str, group: dict[str, Any]) -> str: + """按角色生成纵向分布卡片，避免横向堆叠十余列。""" + + role_name = label.lower() + role_class = "" if label == "ALL" else f" role-{role_name}" + cycles_per_ns = float(group["cycles_per_ns"]) + rows = "".join( + ( + _pmu_stat_row( + "最小值", "min", group["total_min"], group["scalar_min"], cycles_per_ns + ), + _pmu_stat_row( + "平均值", "mean", group["total_per_core"], group["scalar_per_core"], + cycles_per_ns, 2 + ), + _pmu_stat_row( + "最大值", "max", group["total_max"], group["scalar_max"], cycles_per_ns + ), + ) + ) + return f""" +
+
{_escape(label)}{_format_count(group['cores'])} 核 · {cycles_per_ns:.6f} cycles/ns
+ + + {rows} +
统计PMU totalscalar busy
+
Σscalar/Σtotal：{_format_rate(group['scalar_share'])}非 Scalar-busy 残余/核：{_cycle_time_cell(group['non_scalar_busy_per_core'], cycles_per_ns, 2)}
+
+""" + + +def render_report( + input_path: Path, + output_path: Path | None = None, + miss_penalty_ns: float = 90.0, + pmu_cycles_per_ns: float = DEFAULT_PMU_CYCLES_PER_NS, + aic_pmu_cycles_per_ns: float = DEFAULT_AIC_PMU_CYCLES_PER_NS, + aiv_pmu_cycles_per_ns: float = DEFAULT_AIV_PMU_CYCLES_PER_NS, +) -> str: + """先通过独立 analyzer 门禁，再生成一个无需网络的完整 HTML 字符串。""" + + for name, value in ( + ("pmu_cycles_per_ns", pmu_cycles_per_ns), + ("aic_pmu_cycles_per_ns", aic_pmu_cycles_per_ns), + ("aiv_pmu_cycles_per_ns", aiv_pmu_cycles_per_ns), + ): + if not math.isfinite(value) or value <= 0: + raise ValueError(f"{name} must be finite and positive") + + input_path = Path(input_path) + output_path = default_output_path(input_path) if output_path is None else Path(output_path) + # 原始件只读取一次；analyzer 与页面元数据都基于同一份私有快照，避免生成期间 + # raw 被替换后，统计值、元数据和页面 SHA256 分属不同版本。 + raw_bytes = input_path.read_bytes() + raw_digest = hashlib.sha256(raw_bytes).hexdigest() + with tempfile.TemporaryDirectory(prefix="pa-submit-pmu-report-") as snapshot_directory: + snapshot_path = Path(snapshot_directory) / input_path.name + snapshot_path.write_bytes(raw_bytes) + analysis = analyze([snapshot_path], miss_penalty_ns) + capture = load_capture(snapshot_path) + if capture.schema_version not in (4, 5, 6): + raise ValueError("HTML report requires submit-pmu schema-v4/v5/v6 input") + + data = capture.data + configuration = data["configuration"] + validation = data["validation"] + records = data["records"] + aic_records = [record for record in records if record["role"] == "aic"] + aiv_records = [record for record in records if record["role"] == "aiv"] + per_run = analysis["per_run"][0] + all_metrics = _group_metrics(per_run["groups"]["all"], records, pmu_cycles_per_ns) + aic = _group_metrics(per_run["groups"]["aic"], aic_records, aic_pmu_cycles_per_ns) + aiv = _group_metrics(per_run["groups"]["aiv"], aiv_records, aiv_pmu_cycles_per_ns) + cycles_per_ns_by_role = { + "aic": aic_pmu_cycles_per_ns, + "aiv": aiv_pmu_cycles_per_ns, + } + submit_us = float(per_run["submit_span_us"]) + aic_request_per_core = float(aic["requests_per_core"]) + aiv_request_per_core = float(aiv["requests_per_core"]) + request_delta_percent = ( + None + if aic_request_per_core == 0.0 + else (aiv_request_per_core / aic_request_per_core - 1.0) * 100.0 + ) + aic_miss_per_core = float(aic["misses_per_core"]) + aiv_miss_per_core = float(aiv["misses_per_core"]) + delta_misses = aiv_miss_per_core - aic_miss_per_core + delta_percent = ( + None if aic_miss_per_core == 0.0 else delta_misses / aic_miss_per_core * 100.0 + ) + delta_rate_pp = (float(aiv["miss_rate"]) - float(aic["miss_rate"])) * 100.0 + request_comparison = ( + "AIC request/core 为 0，AIV 相对变化不可计算" + if request_delta_percent is None + else f"AIV 的 request/core 相比 AIC {'高' if request_delta_percent >= 0 else '低'} " + f"{abs(request_delta_percent):.2f}%" + ) + miss_comparison = ( + "AIC miss/core 为 0，AIV 相对变化不可计算" + if delta_percent is None + else f"miss/core {'高' if delta_percent >= 0 else '低'} {abs(delta_percent):.2f}%" + f"（{delta_misses:+,.2f}/core）" + ) + rate_comparison = ( + f"miss rate {'高' if delta_rate_pp >= 0 else '低'} {abs(delta_rate_pp):.3f} 个百分点" + ) + total_requests = int(per_run["groups"]["all"]["icache_requests_sum"]) + total_misses = int(per_run["groups"]["all"]["icache_misses_sum"]) + phase = str(configuration["compiled_phase"]) + workload = configuration.get("winner_workload") or {} + workload_counts = workload.get("counts") or {} + shared_plan_subtitle = "" + shared_plan_card = "" + if capture.schema_version == 6: + # analyzer 已独立重建并核对过该计划；HTML 只展示通过门禁的紧凑身份， + # 不展开 B256 context 向量，避免诊断页面被输入明细淹没。 + shared_context_lens = configuration["shared_context_lens"] + shared_task_plan = configuration["shared_task_plan"] + context_min = min(shared_context_lens) if shared_context_lens else None + context_max = max(shared_context_lens) if shared_context_lens else None + context_range = ( + "context 无 batch" + if context_min is None + else f"context 最小 {context_min:,} · 最大 {context_max:,}" + ) + tasks_per_worker = int(shared_task_plan["tasks_per_worker"]) + total_groups = int(shared_task_plan["total_groups"]) + shared_plan_subtitle = f" / shared dynamic {tasks_per_worker} task/core" + shared_plan_card = ( + '\n
' + '
shared 动态 task 计划
' + f'
{tasks_per_worker:,} task/核
' + f'
group 总数 {total_groups:,} · {context_range}
' + "
" + ) + exact_records = int(validation["shadow_primary_match_records"]) + bounded_records = int(validation["shadow_primary_bounded_records"]) + workers = int(configuration["workers"]) + generated_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + raw_link = _raw_href(input_path, output_path) + + total_plot = _distribution_svg( + records, + "total_cycles", + "逐物理核 PMU total cycle", + "每核 Submit gate 内的 PMU raw total；96 核求和是 core-work，不是墙钟时间。", + cycles_per_ns_by_role=cycles_per_ns_by_role, + ) + scalar_plot = _distribution_svg( + records, + "scalar_busy", + "逐物理核 scalar busy cycle", + "CNT2 scalar_instr_busy(0x001) 的每核累计；它不包含全部等待周期。", + cycles_per_ns_by_role=cycles_per_ns_by_role, + ) + request_plot = _distribution_svg( + records, + "icache_requests", + "逐物理核 I-cache request", + "96 个实测物理子核的 request 离散分布；AIC 为圆点，AIV 为方点。", + ) + miss_plot = _distribution_svg( + records, + "icache_misses", + "逐物理核 I-cache miss", + "96 个实测物理子核的 miss 离散分布；AIC 为圆点，AIV 为方点。", + ) + rate_plot = _distribution_svg( + records, + "icache_miss_rate", + "逐物理核 I-cache miss rate", + "每核 miss/request，仅用于观察离散分布；汇总 rate 仍按总 miss 除以总 request。", + ) + per_core_rows = _per_core_rows(records, cycles_per_ns_by_role) + pmu_role_cards = "".join( + ( + _pmu_role_card("ALL", all_metrics), + _pmu_role_card("AIC", aic), + _pmu_role_card("AIV", aiv), + ) + ) + if phase == "none": + shadow_badge = f"PRIMARY ↔ SHADOW EXACT {exact_records}/{workers}" + phase_front_section = """ +
+

局部阶段总览：none

+

不适用：该 ELF 未编译局部阶段。request、miss 和阶段时间都不能作为某个局部阶段的 0% 结果；本报告只提供完整 Submit 基准。

+
+""" + phase_section = """ +
+

局部 phase：none

+

未执行 running read-clear 或阶段 SYS_CNT 边界；raw 中局部字段按契约为 0，但语义是“未选择阶段”，不是某个阶段实测为 0。

+
+""" + else: + shadow_badge = f"SHADOW BOUNDED {bounded_records}/{workers}（exact {exact_records}/{workers}）" + phase_groups = per_run["groups"] + phase_share_cards = "".join( + ( + _phase_share_card("ALL", phase_groups["all"]), + _phase_share_card("AIC", phase_groups["aic"]), + _phase_share_card("AIV", phase_groups["aiv"]), + ) + ) + phase_front_section = f""" +
+

局部阶段总览：{_escape(phase)}

+

时间占比是 Σ阶段 SYS_CNT / Σ同核首个 submit_begin 计时点到末个 submit_end 计时点 SYS_CNT，分别在 ALL/AIC/AIV 内先求和再相除。首个 submit_begin 位于 BeginCallbackSubmit 上下文初始化之后，末个 submit_end 位于返回之前。它是逐核累计 core-time 构成，不是该阶段占全局约 5 ms 墙钟的切片。request/miss 仍显示同一 ELF primary-shadow 形成的下界..上界；阶段时间是单点观察值。

+
{phase_share_cards}
+
+""" + phase_plot = _distribution_svg( + records, + "phase_icache_misses", + f"{phase} 局部 I-cache miss 观测下界", + "running read-clear 的逐核观测下界；上界还需加该核 primary-shadow residual。", + ) + phase_rows = "".join( + ( + _phase_group_row("ALL", phase_groups["all"]), + _phase_group_row("AIC", phase_groups["aic"]), + _phase_group_row("AIV", phase_groups["aiv"]), + ) + ) + phase_section = f""" +

局部阶段详细数据：{_escape(phase)}

+
+
带边界扰动的诊断区间。每次调用新增两次 SYS_CNT；时间起点位于 begin shadow read-clear 之后，终点位于 end shadow read-clear 之前，因此不包含两侧 ld_dev，但包含时间戳边界本身的扰动。request/miss 下界是直接观测值，上界是下界加本核 primary-shadow residual；不同 phase ELF 的局部值不能相加，也不能从 none 相减得到无扰动净值。
+
+ 展开 ALL / AIC / AIV 完整数字表 +
+ + + {phase_rows} +
分组calls阶段时间/core阶段时间/call阶段 core-time / Submitrequest sum 下界..上界request/core 下界..上界局部 request / Submit primarymiss sum 下界..上界miss/core 下界..上界局部 miss / Submit primaryshadow loss req/missobserved miss/request
+
+
+

逐核 phase miss 下界

+
{phase_plot}
+
+""" + + return f""" + + + + + Standalone PA Submit I-cache 报告 + + + +
+
+

Standalone PA Submit I-cache 报告

+
submit-pmu / {_escape(phase)} / batch {_escape(configuration['batches'])} / {_escape(workload.get('mode'))} / QK,SF,PV,UP={_escape(workload_counts.get('qk'))},{_escape(workload_counts.get('sf'))},{_escape(workload_counts.get('pv'))},{_escape(workload_counts.get('up'))}{shared_plan_subtitle}
+
+ RAW → SUMMARY PASS + 语义与 PMU PASS + OWNER RESTORE PASS + {_escape(shadow_badge)} +
+
+ + {phase_front_section} + +
+
完整 Submit（最早开始 → 最晚结束）
{submit_us / 1000:.6f} ms
96 核整体 Submit 耗时
+
96 核逐核 PMU total 平均值（校准）
{float(all_metrics['total_per_core_us']):,.3f} µs
最小 {_cycles_to_us(all_metrics['total_min'], pmu_cycles_per_ns):,.3f} µs · 最大 {_cycles_to_us(all_metrics['total_max'], pmu_cycles_per_ns):,.3f} µs
{_format_per_core(all_metrics['total_per_core'])} raw cycle/核
+
96 核逐核 scalar busy 平均值（校准）
{float(all_metrics['scalar_per_core_us']):,.3f} µs
最小 {_cycles_to_us(all_metrics['scalar_min'], pmu_cycles_per_ns):,.3f} µs · 最大 {_cycles_to_us(all_metrics['scalar_max'], pmu_cycles_per_ns):,.3f} µs
Σscalar/Σtotal={_format_rate(all_metrics['scalar_share'])}
+
完整 Submit primary I-cache request（96 核总和）
{total_requests:,}
逐核平均 {_format_per_core(all_metrics['requests_per_core'])}
逐核最小 {_format_count(all_metrics['requests_min'])} · 逐核最大 {_format_count(all_metrics['requests_max'])}
+
完整 Submit primary I-cache miss（96 核总和）
{total_misses:,}
逐核平均 {_format_per_core(all_metrics['misses_per_core'])}
逐核最小 {_format_count(all_metrics['misses_min'])} · 逐核最大 {_format_count(all_metrics['misses_max'])}
+
聚合 miss rate
{_format_rate(float(all_metrics['miss_rate']))}
+
实测物理子核
{workers}（32 AIC + 64 AIV）
+
Primary/Shadow
exact {exact_records}/{workers}
bounded {bounded_records}/{workers}
{shared_plan_card} +
+ +

PMU cycle 频率校准

+
+

本机受控 cold/warm 同窗证据：PMU cycle_delta = {PMU_CALIBRATION_CYCLE_DELTA:,}，1 ns SYS_CNT tick_delta = {PMU_CALIBRATION_SYS_TICK_NS:,} ns；二者相除为 {PMU_CALIBRATION_CYCLE_DELTA / PMU_CALIBRATION_SYS_TICK_NS:.6f} cycles/ns（约 1.65 GHz）。

+

当前报告换算频率：ALL = {pmu_cycles_per_ns:.6f}、AIC = {aic_pmu_cycles_per_ns:.6f}、AIV = {aiv_pmu_cycles_per_ns:.6f} cycles/ns。公式：等效 µs = PMU cycles / (cycles/ns) / 1000。

+

SYS_CNT 仍按 1 ns/tick 解释；表中的 PMU 时间是每个物理子核 gate 内 cycle 的校准等效时间。它不是 96 核 cycle 求和后的墙钟时间，也不替代独立记录的完整 Submit 墙钟。

+
+ +

PMU total 与 scalar busy

+
+
{pmu_role_cards}
+
口径：每张卡只保留最小值、平均值和最大值。total 与 scalar busy 的最小/最大值分别从各自逐核分布独立取得，不保证来自同一个物理核，也不能相减成配对差值。total 是每个物理子核在 Submit gate 内的 PMU raw total cycle；求和代表 96 核 core-work。页面始终保留 raw cycle，旁边的 µs 只按本机实测频率换算。逐核最大值用于观察慢核，但仍不等同于“最早开始至最晚结束”的完整 Submit。scalar busy 是 CNT2 scalar_instr_busy(0x001)。卡片中的“非 Scalar-busy 残余”严格等于 total−scalar busy，它不是空闲时间，也不是 I-cache stall，其中还混有同步等待、engine 等待及其他未归因周期。受控微基准已观察到依赖返回的 atomic 等待大部分进入 scalar busy，而 I-cache refill 的额外周期大部分只进入 total。当前 A5/DAV3510 正式事件表和 CANN 9.1 上板输出均不提供 scalar_wait_ib_time/scalar_wait_time，报告不会用其他产品的 selector 猜测这两项。
+

PMU total cycle

{total_plot} +

Scalar busy cycle

{scalar_plot} +
+ +

AIC 与 AIV 的 I-cache 对比

+
+
+ + + + + + +
角色核数request min/corerequest mean/corerequest max/coremiss min/coremiss mean/coremiss max/coreΣmiss/Σrequest
AIC{_format_count(aic['cores'])}{_format_count(aic['requests_min'])}{_format_per_core(aic['requests_per_core'])}{_format_count(aic['requests_max'])}{_format_count(aic['misses_min'])}{_format_per_core(aic['misses_per_core'])}{_format_count(aic['misses_max'])}{_format_rate(aic['miss_rate'])}
AIV{_format_count(aiv['cores'])}{_format_count(aiv['requests_min'])}{_format_per_core(aiv['requests_per_core'])}{_format_count(aiv['requests_max'])}{_format_count(aiv['misses_min'])}{_format_per_core(aiv['misses_per_core'])}{_format_count(aiv['misses_max'])}{_format_rate(aiv['miss_rate'])}
+
+
{_escape(request_comparison)}；{_escape(miss_comparison)}；聚合 {_escape(rate_comparison)}。
+
+ +

逐物理核分布

+
+
AIC 圆点 AIV 方点
+

I-cache request

{request_plot} +

I-cache miss

{miss_plot} +

每核 miss rate

{rate_plot} +
+ + {phase_section} + +
+

假设性 core-equivalent，不是 Submit 墙钟损失

+

AIC：{_format_per_core(aic['misses_per_core'])} miss/core × {miss_penalty_ns:.3f} ns = {float(aic['serial_equivalent_us']):,.3f} µs/core-equivalent

+

AIV：{_format_per_core(aiv['misses_per_core'])} miss/core × {miss_penalty_ns:.3f} ns = {float(aiv['serial_equivalent_us']):,.3f} µs/core-equivalent

+

该标尺不可跨 96 核相加，也不可直接从 {submit_us / 1000:.6f} ms Submit 中扣除。各核并行，miss 可能重叠或被流水/等待隐藏；实际收益必须由相同语义优化前后的 ΔSubmit 与 Δmiss/core 成对实验确认。

+
+ +

96 核精确数据

+
+
+ 展开逐核表格 +
+ + + {per_core_rows} +
workerphysicalroleblocklanePMU total
cycle / 等效 µs		scalar busy
cycle / 等效 µs		scalar/totalrequestsmissesper-core rateprimary=shadowtrusted
+
+
+

聚合 miss rate 使用 Σmiss/Σrequest，不平均逐核百分比。request/miss 的 min、mean、max 都从同组逐核 raw 计算；request 与 miss 的极值不保证来自同一个物理核。

+
+ +
+ +
capture_id：{_escape(data['capture'].get('capture_id'))}
+
SHA-256：{raw_digest}
+
生成时间：{generated_at}；生成器：pmu_html_report schema v{REPORT_VERSION}
+
+
+ + +""" + + +def write_report( + input_path: Path, + output_path: Path | None = None, + miss_penalty_ns: float = 90.0, + pmu_cycles_per_ns: float = DEFAULT_PMU_CYCLES_PER_NS, + aic_pmu_cycles_per_ns: float = DEFAULT_AIC_PMU_CYCLES_PER_NS, + aiv_pmu_cycles_per_ns: float = DEFAULT_AIV_PMU_CYCLES_PER_NS, +) -> Path: + """在完整 HTML 构造成功后原子发布，失败时不留下半截报告。""" + + input_path = Path(input_path) + output_path = default_output_path(input_path) if output_path is None else Path(output_path) + if input_path.resolve() == output_path.resolve(): + raise ValueError("HTML output path must differ from raw JSON input") + document = render_report( + input_path, + output_path, + miss_penalty_ns, + pmu_cycles_per_ns, + aic_pmu_cycles_per_ns, + aiv_pmu_cycles_per_ns, + ) + output_path.parent.mkdir(parents=True, exist_ok=True) + temporary_name: str | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix=f".{output_path.name}.", + suffix=".tmp", + dir=output_path.parent, + delete=False, + ) as temporary: + temporary.write(document) + temporary.flush() + os.fsync(temporary.fileno()) + os.fchmod(temporary.fileno(), 0o644) + temporary_name = temporary.name + os.replace(temporary_name, output_path) + finally: + if temporary_name is not None: + Path(temporary_name).unlink(missing_ok=True) + return output_path + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path, help="一份已完成的 submit-pmu raw JSON") + parser.add_argument("-o", "--output", type=Path, help="HTML 输出路径；默认由 *_raw.json 推导") + parser.add_argument( + "--icache-miss-ns", + type=float, + default=90.0, + help="受控 cold/warm 串行标尺，仅用于 core-equivalent（默认 90）", + ) + parser.add_argument( + "--pmu-cycles-per-ns", + type=float, + default=DEFAULT_PMU_CYCLES_PER_NS, + help=f"ALL PMU cycle 校准频率（默认 {DEFAULT_PMU_CYCLES_PER_NS:.6f} cycles/ns）", + ) + parser.add_argument( + "--aic-pmu-cycles-per-ns", + type=float, + default=DEFAULT_AIC_PMU_CYCLES_PER_NS, + help=f"AIC PMU cycle 校准频率（默认 {DEFAULT_AIC_PMU_CYCLES_PER_NS:.6f} cycles/ns）", + ) + parser.add_argument( + "--aiv-pmu-cycles-per-ns", + type=float, + default=DEFAULT_AIV_PMU_CYCLES_PER_NS, + help=f"AIV PMU cycle 校准频率（默认 {DEFAULT_AIV_PMU_CYCLES_PER_NS:.6f} cycles/ns）", + ) + arguments = parser.parse_args(argv) + if not math.isfinite(arguments.icache_miss_ns) or arguments.icache_miss_ns <= 0: + parser.error("--icache-miss-ns must be finite and positive") + for option, value in ( + ("--pmu-cycles-per-ns", arguments.pmu_cycles_per_ns), + ("--aic-pmu-cycles-per-ns", arguments.aic_pmu_cycles_per_ns), + ("--aiv-pmu-cycles-per-ns", arguments.aiv_pmu_cycles_per_ns), + ): + if not math.isfinite(value) or value <= 0: + parser.error(f"{option} must be finite and positive") + try: + output = write_report( + arguments.input, + arguments.output, + arguments.icache_miss_ns, + arguments.pmu_cycles_per_ns, + arguments.aic_pmu_cycles_per_ns, + arguments.aiv_pmu_cycles_per_ns, + ) + except (OSError, ValueError) as error: + print(f"PMU HTML report failed: {error}", file=sys.stderr) + return 1 + print(f"[PMU-HTML] raw={arguments.input} report={output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/atomic_probe/pa_scheduler/pmu_sidecar_analyzer.py b/tests/atomic_probe/pa_scheduler/pmu_sidecar_analyzer.py new file mode 100644 index 0000000000..042a1eb647 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/pmu_sidecar_analyzer.py @@ -0,0 +1,1869 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""校验并聚合 standalone PA 调度器的多轮 PMU JSON sidecar。 + +Host 已为单轮输出 ALL/AIC/AIV summary；本工具从每轮 96 条 worker raw 记录重新 +计算同一组统计量，再聚合多个独立进程。它不修改采集文件，也不把 PMU raw total、 +scalar busy 或 I-cache miss 的一阶估算冒充 Submit 墙钟时间。 +""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, Sequence + +SCHEMA_NAME = "pa_scheduler_pmu_phase_windows" +SCHEMA_VERSIONS = (3, 4, 5, 6) +SUBMIT_PMU_SCHEMA_VERSIONS = (4, 5, 6) +SUBMIT_PMU_TIME_SCHEMA_VERSIONS = (5, 6) +GROUP_NAMES = ("all", "aic", "aiv") +METRIC_NAMES = ( + "total_cycles", + "vector_busy", + "cube_busy", + "scalar_busy", + "mte1_busy", + "mte2_busy", + "mte3_busy", + "icache_requests", + "icache_misses", + "fix_busy", +) +SUBMIT_PMU_METRIC_NAMES = ( + "total_cycles", + "vector_busy", + "cube_busy", + "scalar_busy", + "mte1_busy", + "mte2_busy", + "icache_requests", + "icache_misses", + "shadow_whole_icache_requests", + "shadow_whole_icache_misses", + "phase_calls", + "phase_icache_requests", + "phase_icache_misses", +) +SUBMIT_PMU_V5_METRIC_NAMES = SUBMIT_PMU_METRIC_NAMES + ( + "submit_elapsed_ticks", + "phase_elapsed_ticks", +) +SUMMARY_FIELDS = ("sum", "mean", "median", "p95", "max") +SUBMIT_PMU_BUILD_VARIANT = "submit-pmu" +SUBMIT_PMU_BUILD_VARIANT_ID = 2 +PROGRAMMABLE_COUNTER_BITS = 32 +PROGRAMMABLE_COUNTER_RISK_THRESHOLD = (1 << PROGRAMMABLE_COUNTER_BITS) // 4 +SUBMIT_PMU_PHASE_IDS = { + "none": 0, + "claim": 1, + "efdrain": 2, + "materialize": 4, + "register": 5, +} +FINAL_BARRIER_SHAPES = ( + "flat", + "two-4", + "two-8", + "two-16", + "three-6x4x4", +) +TASKS_PER_BATCH = 5 +PHASE_STATUS_REQUIRED_MASK_V4 = 0x3CF +PHASE_STATUS_REQUIRED_MASK_V5 = 0x7CF + +# schema-v4/v5/v6 只描述 A5 standalone submit-pmu 正式采集，不接受由 JSON 自报的 +# 任意缩小拓扑。物理槽按每 die 18 AIC + 36 AIV 排列；当前 runtime 实际开放 +# 32 个 AIC 与 64 个 AIV，共组成 32 组 1:2 mixed triplet。 +A5_WORKERS = 96 +A5_AIC_WORKERS = 32 +A5_AIV_WORKERS = 64 +A5_PHYSICAL_SUBCORES = 108 +A5_AIC_PER_DIE = 18 +A5_SUBCORES_PER_DIE = 54 +A5_OWNER_BITMAP_WORDS = 4 +A5_OWNER_MAGIC = 0x504D554F +A5_OWNER_VERSION = 1 + +# 这些字段决定两份 sidecar 是否属于同一观察配置。动态时间、capture id 和 +# placement 分布不在其中；它们正是多轮运行允许自然变化的结果。 +CONFIG_FINGERPRINT_FIELDS = ( + "device", + "batches", + "workers", + "aic_workers", + "aiv_workers", + "final_barrier", + "pmu_window", + "selectors", + "counter_width_bits", + "phase_timestamp_calls_present", + "phase_record_writes", + "profile_accumulation", + "trace_enabled", + "atomic_trace", + "gate_start_stop_have_pipe_all_barriers", + "winner_workload", +) +SUBMIT_PMU_FINGERPRINT_FIELDS = CONFIG_FINGERPRINT_FIELDS + ( + "build_variant", + "build_variant_id", + "compiled_phase", + "compiled_phase_id", + "primary_window_segments_per_record", + "unavailable_metrics", + "phase_boundary_observation_included", + "phase_counter_pair_snapshot_atomic", + "primary_counters_read_at_phase_boundaries", + "phase_shadow_partition_exact_required", + "phase_values_are_running_read_clear_lower_bounds", + "cross_phase_elf_sums_valid", + "phase_time_observation_included", + "phase_time_sys_counter_tick_ns", + "phase_time_boundary", + "phase_time_excludes_shadow_read_overhead", + "phase_time_includes_timestamp_overhead", + "phase_time_share_definition", + "phase_time_denominator_scope", +) +SHARED_SUBMIT_PMU_FINGERPRINT_FIELDS = SUBMIT_PMU_FINGERPRINT_FIELDS + ( + "tensormap_mode", + "shared_context_lens", + "shared_task_plan", +) + + +@dataclass(frozen=True) +class Capture: + """一份已通过 raw→summary 和采集门禁的 sidecar。""" + + path: Path + data: dict[str, Any] + groups: dict[str, list[dict[str, Any]]] + fingerprint: str + schema_version: int + + +@dataclass(frozen=True) +class PhasePartitionEvidence: + """一条 raw 记录独立重算出的 shadow 分区证据。""" + + shadow_exact: bool + shadow_bounded: bool + request_abs_delta: int + request_signed_delta: int + miss_abs_delta: int + miss_signed_delta: int + expected_calls: int + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def _integer(value: Any, label: str) -> int: + # bool 是 int 的子类，但 JSON true/false 不能静默成为 PMU counter。 + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{label} must be an integer") + if value < 0: + raise ValueError(f"{label} must be non-negative") + return value + + +def _signed_integer(value: Any, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{label} must be an integer") + return value + + +def _number(value: Any, label: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{label} must be numeric") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{label} must be finite") + return result + + +def _nearest_rank_p95(values: Sequence[int]) -> int: + # 与 CCEC host 使用相同的 nearest-rank 定义：ceil(0.95*N) 对应的顺序统计量。 + ordered = sorted(values) + return ordered[math.ceil(0.95 * len(ordered)) - 1] + + +def _metric_summary(values: Sequence[int]) -> dict[str, int | float]: + _require(bool(values), "cannot summarize an empty metric") + return { + "sum": sum(values), + "mean": sum(values) / len(values), + "median": statistics.median(values), + "p95": _nearest_rank_p95(values), + "max": max(values), + } + + +def _same_number(lhs: int | float, rhs: Any) -> bool: + try: + rhs_number = _number(rhs, "summary value") + except ValueError: + return False + return math.isclose(float(lhs), rhs_number, rel_tol=1e-12, abs_tol=1e-9) + + +def _configuration_fingerprint(configuration: dict[str, Any], schema_version: int) -> str: + fields = ( + SHARED_SUBMIT_PMU_FINGERPRINT_FIELDS + if schema_version == 6 + else SUBMIT_PMU_FINGERPRINT_FIELDS + if schema_version in SUBMIT_PMU_SCHEMA_VERSIONS + else CONFIG_FINGERPRINT_FIELDS + ) + selected = {field: configuration.get(field) for field in fields} + # schema version 不是 configuration 字段，但必须进入指纹，防止 v3 历史文件与 + # v4 submit-pmu 恰好具有相同运行参数时被静默聚合。 + selected["schema_version"] = schema_version + return json.dumps(selected, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + + +def _validate_group_summary( + path: Path, + group_name: str, + records: Sequence[dict[str, Any]], + expected: Any, + metric_names: Sequence[str], + schema_version: int, +) -> None: + _require(isinstance(expected, dict), f"{path}: summary.{group_name} must be an object") + _require( + expected.get("cores") == len(records), + f"{path}: summary.{group_name}.cores does not match raw records", + ) + _require( + expected.get("trusted_cores") == len(records), + f"{path}: summary.{group_name}.trusted_cores is incomplete", + ) + _require( + expected.get("active_cores") == len(records), + f"{path}: summary.{group_name}.active_cores is incomplete", + ) + + for metric in metric_names: + values = [ + _integer(record.get(metric), f"{path}: records[{index}].{metric}") + for index, record in enumerate(records) + ] + actual = _metric_summary(values) + reported = expected.get(metric) + _require( + isinstance(reported, dict), + f"{path}: summary.{group_name}.{metric} must be an object", + ) + for field in SUMMARY_FIELDS: + if field in ("sum", "p95", "max"): + reported_value = _integer( + reported.get(field), f"{path}: summary.{group_name}.{metric}.{field}" + ) + matches = reported_value == actual[field] + else: + matches = _same_number(actual[field], reported.get(field)) + _require( + matches, + f"{path}: raw summary mismatch at {group_name}.{metric}.{field}: " + f"raw={actual[field]!r} json={reported.get(field)!r}", + ) + + requests = sum(_integer(record["icache_requests"], "icache_requests") for record in records) + misses = sum(_integer(record["icache_misses"], "icache_misses") for record in records) + _require(requests > 0, f"{path}: summary.{group_name} has zero I-cache requests") + actual_rate = misses / requests + _require( + _same_number(actual_rate, expected.get("icache_miss_rate")), + f"{path}: raw summary mismatch at {group_name}.icache_miss_rate: " + f"raw={actual_rate!r} json={expected.get('icache_miss_rate')!r}", + ) + + if schema_version in SUBMIT_PMU_SCHEMA_VERSIONS: + phase_requests = sum( + _integer(record["phase_icache_requests"], "phase_icache_requests") + for record in records + ) + phase_misses = sum( + _integer(record["phase_icache_misses"], "phase_icache_misses") + for record in records + ) + reported_rate = expected.get("phase_observed_read_clear_ratio") + if phase_requests == 0: + _require( + reported_rate is None, + f"{path}: summary.{group_name}.phase_observed_read_clear_ratio " + "must be null for zero requests", + ) + else: + actual_phase_rate = phase_misses / phase_requests + _require( + _same_number(actual_phase_rate, reported_rate), + f"{path}: raw summary mismatch at " + f"{group_name}.phase_observed_read_clear_ratio: " + f"raw={actual_phase_rate!r} json={reported_rate!r}", + ) + + +def _validate_submit_pmu_configuration( + path: Path, configuration: dict[str, Any], schema_version: int +) -> tuple[str, int]: + """校验 submit-pmu 的编译期身份、局部阶段和观察能力契约。""" + + _require( + configuration.get("build_variant") == SUBMIT_PMU_BUILD_VARIANT, + f"{path}: configuration.build_variant must be {SUBMIT_PMU_BUILD_VARIANT!r}", + ) + variant_id = _integer( + configuration.get("build_variant_id"), f"{path}: configuration.build_variant_id" + ) + _require( + variant_id == SUBMIT_PMU_BUILD_VARIANT_ID, + f"{path}: unexpected submit-pmu build_variant_id {variant_id}", + ) + phase_name = configuration.get("compiled_phase") + _require( + isinstance(phase_name, str) and phase_name in SUBMIT_PMU_PHASE_IDS, + f"{path}: unsupported configuration.compiled_phase {phase_name!r}", + ) + phase_id = _integer( + configuration.get("compiled_phase_id"), f"{path}: configuration.compiled_phase_id" + ) + _require( + phase_id == SUBMIT_PMU_PHASE_IDS[phase_name], + f"{path}: compiled phase name/id mismatch", + ) + _require( + configuration.get("pmu_window") == "submit-all", + f"{path}: submit-pmu requires pmu_window='submit-all'", + ) + final_barrier = configuration.get("final_barrier") + _require( + isinstance(final_barrier, str) + and final_barrier in FINAL_BARRIER_SHAPES, + f"{path}: unsupported configuration.final_barrier {final_barrier!r}", + ) + _require( + _integer( + configuration.get("primary_window_segments_per_record"), + f"{path}: configuration.primary_window_segments_per_record", + ) + == 1, + f"{path}: configuration.primary_window_segments_per_record must be one", + ) + _require( + configuration.get("unavailable_metrics") == ["mte3_busy"], + f"{path}: configuration.unavailable_metrics must identify mte3_busy", + ) + + counter_widths = configuration.get("counter_width_bits") + _require( + isinstance(counter_widths, dict), + f"{path}: configuration.counter_width_bits must be an object", + ) + _require( + _integer(counter_widths.get("total"), f"{path}: configuration.counter_width_bits.total") + == 64, + f"{path}: configuration.counter_width_bits.total must be 64", + ) + _require( + _integer( + counter_widths.get("programmable"), + f"{path}: configuration.counter_width_bits.programmable", + ) + == PROGRAMMABLE_COUNTER_BITS, + f"{path}: configuration.counter_width_bits.programmable must be 32", + ) + + for field in ( + "trace_enabled", + "trace_atomics", + "profile_phases", + "phase_record_writes", + "atomic_trace", + "profile_accumulation", + "primary_counters_read_at_phase_boundaries", + "cross_phase_elf_sums_valid", + ): + _require(configuration.get(field) is False, f"{path}: configuration.{field} is not false") + expected_boundary_observation = phase_name != "none" + expected_phase_timestamps = ( + schema_version in SUBMIT_PMU_TIME_SCHEMA_VERSIONS + and expected_boundary_observation + ) + _require( + configuration.get("phase_timestamp_calls_present") is expected_phase_timestamps, + f"{path}: configuration.phase_timestamp_calls_present does not match schema/phase", + ) + _require( + configuration.get("phase_boundary_observation_included") + is expected_boundary_observation, + f"{path}: configuration.phase_boundary_observation_included does not match the phase", + ) + _require( + configuration.get("phase_counter_pair_snapshot_atomic") is False, + f"{path}: configuration.phase_counter_pair_snapshot_atomic is not false", + ) + exact_partition_required = phase_name == "none" + _require( + configuration.get("phase_shadow_partition_exact_required") + is exact_partition_required, + f"{path}: configuration.phase_shadow_partition_exact_required does not match the phase", + ) + running_lower_bounds = phase_name != "none" + _require( + configuration.get("phase_values_are_running_read_clear_lower_bounds") + is running_lower_bounds, + f"{path}: configuration.phase_values_are_running_read_clear_lower_bounds does not match the phase", + ) + if schema_version in SUBMIT_PMU_TIME_SCHEMA_VERSIONS: + _require( + configuration.get("phase_time_observation_included") + is expected_boundary_observation, + f"{path}: configuration.phase_time_observation_included does not match the phase", + ) + _require( + _number( + configuration.get("phase_time_sys_counter_tick_ns"), + f"{path}: configuration.phase_time_sys_counter_tick_ns", + ) + == 1.0, + f"{path}: phase_time_sys_counter_tick_ns must be one", + ) + _require( + configuration.get("phase_time_boundary") + == "after_begin_read_clear_to_before_end_read_clear", + f"{path}: unexpected phase_time_boundary", + ) + _require( + configuration.get("phase_time_excludes_shadow_read_overhead") is True, + f"{path}: phase_time_excludes_shadow_read_overhead is not true", + ) + _require( + configuration.get("phase_time_includes_timestamp_overhead") is True, + f"{path}: phase_time_includes_timestamp_overhead is not true", + ) + _require( + configuration.get("phase_time_share_definition") + == "sum(phase_elapsed_ticks)/sum(submit_elapsed_ticks)", + f"{path}: unexpected phase_time_share_definition", + ) + _require( + configuration.get("phase_time_denominator_scope") + == "per_worker_first_submit_begin_to_last_submit_end", + f"{path}: unexpected phase_time_denominator_scope", + ) + + selectors = configuration.get("selectors") + _require(isinstance(selectors, dict), f"{path}: configuration.selectors must be an object") + expected_selectors = { + "cnt0_vector_busy": 0x501, + "cnt1_cube_busy": 0x301, + "cnt2_scalar_busy": 0x001, + "cnt3_mte1_busy": 0x701, + "cnt4_mte2_busy": 0x202, + "cnt6_primary_icache_request": 0x034, + "cnt7_primary_icache_miss": 0x035, + "cnt5_shadow_icache_miss": 0x035, + "cnt8_shadow_icache_request": 0x034, + "cnt9_unused": 0x000, + } + for field, expected in expected_selectors.items(): + _require( + _integer(selectors.get(field), f"{path}: configuration.selectors.{field}") == expected, + f"{path}: configuration.selectors.{field} is not 0x{expected:03x}", + ) + return phase_name, phase_id + + +def _is_aic_physical_slot(physical_id: int) -> bool: + return ( + physical_id < A5_PHYSICAL_SUBCORES + and physical_id % A5_SUBCORES_PER_DIE < A5_AIC_PER_DIE + ) + + +def _expected_logical_triplet(worker_id: int) -> tuple[str, int, int]: + """按 mixed launch ABI 返回 worker 的 role/block/lane。""" + + if worker_id < A5_AIC_WORKERS: + return "aic", worker_id, 0 + vector_id = worker_id - A5_AIC_WORKERS + return "aiv", vector_id // 2, 1 + vector_id % 2 + + +def _physical_aiv_pair(aic_physical_id: int) -> tuple[int, int]: + """返回一个物理 AIC 槽对应的两个 AIV 槽。""" + + die_base = (aic_physical_id // A5_SUBCORES_PER_DIE) * A5_SUBCORES_PER_DIE + local_aic = aic_physical_id % A5_SUBCORES_PER_DIE + first_aiv = die_base + A5_AIC_PER_DIE + local_aic * 2 + return first_aiv, first_aiv + 1 + + +def _require_owner_role_counts( + path: Path, label: str, value: Any, expected: tuple[int, int, int] +) -> None: + _require(isinstance(value, dict), f"{path}: owner.{label} must be an object") + for field, expected_value in zip(("total", "aic", "aiv"), expected): + actual = _integer(value.get(field), f"{path}: owner.{label}.{field}") + _require( + actual == expected_value, + f"{path}: owner.{label}.{field} must be {expected_value}", + ) + + +def _validate_submit_pmu_owner( + path: Path, owner: Any, capture: dict[str, Any] +) -> set[int]: + """独立重验 configure 后、restore 前保存的 PMU owner 快照。""" + + _require(isinstance(owner, dict), f"{path}: submit-pmu owner must be an object") + _require(owner.get("mode") == "main_aicpu_path_a", f"{path}: unexpected owner.mode") + _require( + owner.get("snapshot_phase") == "after_configure_before_restore", + f"{path}: unexpected owner.snapshot_phase", + ) + _require( + _integer(owner.get("control_magic"), f"{path}: owner.control_magic") + == A5_OWNER_MAGIC, + f"{path}: owner.control_magic mismatch", + ) + _require( + _integer(owner.get("control_version"), f"{path}: owner.control_version") + == A5_OWNER_VERSION, + f"{path}: owner.control_version mismatch", + ) + _require( + _integer(owner.get("configure_status"), f"{path}: owner.configure_status") == 0, + f"{path}: owner.configure_status is not success", + ) + _require( + _integer(owner.get("configured_flag"), f"{path}: owner.configured_flag") == 1, + f"{path}: owner.configured_flag is not one", + ) + + topology = (A5_WORKERS, A5_AIC_WORKERS, A5_AIV_WORKERS) + for label in ("expected", "active", "discovered"): + _require_owner_role_counts(path, label, owner.get(label), topology) + _require( + _integer(owner.get("physical_slots_scanned"), f"{path}: owner.physical_slots_scanned") + == A5_PHYSICAL_SUBCORES, + f"{path}: owner.physical_slots_scanned mismatch", + ) + _require( + _integer( + owner.get("skipped_physical_slots"), f"{path}: owner.skipped_physical_slots" + ) + == A5_PHYSICAL_SUBCORES - A5_WORKERS, + f"{path}: owner.skipped_physical_slots mismatch", + ) + _require( + owner.get("configured_bitmap_word_order") + == "least_significant_physical_ids_first", + f"{path}: unexpected owner.configured_bitmap_word_order", + ) + + words = owner.get("configured_bitmap_words") + _require( + isinstance(words, list) and len(words) == A5_OWNER_BITMAP_WORDS, + f"{path}: owner.configured_bitmap_words must contain four words", + ) + bitmap_words: list[int] = [] + for index, value in enumerate(words): + word = _integer(value, f"{path}: owner.configured_bitmap_words[{index}]") + _require(word <= 0xFFFFFFFF, f"{path}: owner bitmap word exceeds 32 bits") + bitmap_words.append(word) + + configured_ids = { + physical_id + for physical_id in range(A5_OWNER_BITMAP_WORDS * 32) + if bitmap_words[physical_id // 32] & (1 << (physical_id % 32)) + } + _require( + all(physical_id < A5_PHYSICAL_SUBCORES for physical_id in configured_ids), + f"{path}: owner bitmap sets a bit outside the 108 physical slots", + ) + configured_count = _integer( + owner.get("configured_bitmap_count"), f"{path}: owner.configured_bitmap_count" + ) + _require( + configured_count == len(configured_ids) == A5_WORKERS, + f"{path}: owner bitmap count is not exactly 96", + ) + configured_aic = {physical_id for physical_id in configured_ids if _is_aic_physical_slot(physical_id)} + configured_aiv = configured_ids - configured_aic + _require( + len(configured_aic) == A5_AIC_WORKERS and len(configured_aiv) == A5_AIV_WORKERS, + f"{path}: owner bitmap is not a 32 AIC / 64 AIV set", + ) + + complete_triplets = sum( + all(aiv_id in configured_aiv for aiv_id in _physical_aiv_pair(aic_id)) + for aic_id in configured_aic + ) + broken_triplets = len(configured_aic) - complete_triplets + _require( + _integer( + owner.get("configured_complete_mixed_triplets"), + f"{path}: owner.configured_complete_mixed_triplets", + ) + == complete_triplets + == A5_AIC_WORKERS, + f"{path}: owner bitmap does not contain 32 complete mixed triplets", + ) + _require( + _integer( + owner.get("expected_complete_mixed_triplets"), + f"{path}: owner.expected_complete_mixed_triplets", + ) + == A5_AIC_WORKERS, + f"{path}: owner.expected_complete_mixed_triplets mismatch", + ) + _require( + _integer( + owner.get("configured_broken_mixed_triplets"), + f"{path}: owner.configured_broken_mixed_triplets", + ) + == broken_triplets + == 0, + f"{path}: owner bitmap contains a broken mixed triplet", + ) + _require(owner.get("restore_passed") is True, f"{path}: owner.restore_passed is not true") + _require( + owner.get("restore_passed") is capture.get("owner_restore_passed"), + f"{path}: owner and capture restore results disagree", + ) + return configured_ids + + +def _validate_shared_task_plan( + path: Path, configuration: dict[str, Any], batches: int +) -> int: + """由 context lengths 独立重建 shared 每核动态 Submit 数。""" + + _require( + 1 <= batches <= 256, + f"{path}: schema-v6 configuration.batches must be in [1, 256]", + ) + _require( + configuration.get("tensormap_mode") == "shared", + f"{path}: schema-v6 requires configuration.tensormap_mode='shared'", + ) + context_lens = configuration.get("shared_context_lens") + _require( + isinstance(context_lens, list), + f"{path}: configuration.shared_context_lens must be an array", + ) + _require( + len(context_lens) == batches, + f"{path}: configuration.shared_context_lens length does not match batches", + ) + total_groups = 0 + for batch, raw_context_len in enumerate(context_lens): + context_len = _integer( + raw_context_len, + f"{path}: configuration.shared_context_lens[{batch}]", + ) + _require( + context_len <= 32768, + f"{path}: configuration.shared_context_lens[{batch}] exceeds 32768", + ) + blocks = (context_len + 127) // 128 + total_groups += (blocks + 63) // 64 + + tasks_per_worker = batches + 4 * total_groups + reported_plan = configuration.get("shared_task_plan") + _require( + isinstance(reported_plan, dict), + f"{path}: configuration.shared_task_plan must be an object", + ) + _require( + _integer( + reported_plan.get("total_groups"), + f"{path}: configuration.shared_task_plan.total_groups", + ) + == total_groups, + f"{path}: configuration.shared_task_plan.total_groups disagrees with " + "shared_context_lens", + ) + _require( + _integer( + reported_plan.get("tasks_per_worker"), + f"{path}: configuration.shared_task_plan.tasks_per_worker", + ) + == tasks_per_worker, + f"{path}: configuration.shared_task_plan.tasks_per_worker disagrees with " + "shared_context_lens", + ) + return tasks_per_worker + + +def _validate_submit_pmu_task_contract( + path: Path, + configuration: dict[str, Any], + batches: int, + schema_version: int, +) -> int | None: + """v6 使用 shared 动态计划；旧 schema 保持 private 固定 5B。""" + + if schema_version == 6: + return _validate_shared_task_plan(path, configuration, batches) + shared_fields = ( + "tensormap_mode", + "shared_context_lens", + "shared_task_plan", + ) + unexpected = [field for field in shared_fields if field in configuration] + _require( + not unexpected, + f"{path}: schema-v4/v5 cannot carry shared task identity fields: " + f"{', '.join(unexpected)}", + ) + return None + + +def _expected_submit_pmu_phase_calls( + phase_name: str, batches: int, tasks_per_worker: int | None = None +) -> int: + """重建 private 或 shared 全员阶段的逐核调用数。""" + + if phase_name == "none": + return 0 + return batches * TASKS_PER_BATCH if tasks_per_worker is None else tasks_per_worker + + +def _validate_submit_pmu_record( + path: Path, + index: int, + record: dict[str, Any], + phase_name: str, + phase_id: int, + batches: int, + schema_version: int, + tasks_per_worker: int | None = None, +) -> PhasePartitionEvidence: + """重验 raw phase 分区；运行中 read-clear 只形成上下界。""" + + prefix = f"{path}: records[{index}]" + _require( + _integer(record.get("build_variant_id"), f"{prefix}.build_variant_id") + == SUBMIT_PMU_BUILD_VARIANT_ID, + f"{prefix} build_variant_id mismatch", + ) + _require( + _integer(record.get("compiled_phase_id"), f"{prefix}.compiled_phase_id") == phase_id, + f"{prefix} compiled_phase_id mismatch", + ) + phase_status = _integer(record.get("phase_status"), f"{prefix}.phase_status") + phase_status_required = ( + PHASE_STATUS_REQUIRED_MASK_V5 + if schema_version in SUBMIT_PMU_TIME_SCHEMA_VERSIONS + else PHASE_STATUS_REQUIRED_MASK_V4 + ) + _require( + (phase_status & phase_status_required) == phase_status_required, + f"{prefix} phase_status is incomplete", + ) + reconstructed_calls = _expected_submit_pmu_phase_calls( + phase_name, batches, tasks_per_worker + ) + if schema_version == 6: + _require( + "phase_expected_calls" in record, + f"{prefix}.phase_expected_calls is required by schema-v6", + ) + reported_expected_calls = ( + _integer(record.get("phase_expected_calls"), f"{prefix}.phase_expected_calls") + if "phase_expected_calls" in record + else reconstructed_calls + ) + shared_winner_only_phase = ( + schema_version == 6 and phase_name in ("materialize", "register") + ) + if shared_winner_only_phase: + assert tasks_per_worker is not None + _require( + 0 <= reported_expected_calls <= tasks_per_worker, + f"{prefix}.phase_expected_calls exceeds the shared replay task count", + ) + expected_calls = reported_expected_calls + else: + expected_calls = reconstructed_calls + _require( + reported_expected_calls == expected_calls, + f"{prefix}.phase_expected_calls disagrees with the phase contract", + ) + + primary_requests = _integer(record.get("icache_requests"), f"{prefix}.icache_requests") + primary_misses = _integer(record.get("icache_misses"), f"{prefix}.icache_misses") + shadow_requests = _integer( + record.get("shadow_whole_icache_requests"), f"{prefix}.shadow_whole_icache_requests" + ) + shadow_misses = _integer( + record.get("shadow_whole_icache_misses"), f"{prefix}.shadow_whole_icache_misses" + ) + shadow_exact = shadow_requests == primary_requests and shadow_misses == primary_misses + shadow_bounded = shadow_requests <= primary_requests and shadow_misses <= primary_misses + _require( + shadow_misses <= shadow_requests, + f"{prefix} has shadow I-cache miss > request", + ) + _require( + record.get("shadow_matches_primary") is shadow_exact, + f"{prefix}.shadow_matches_primary disagrees with raw counters", + ) + _require( + record.get("shadow_not_greater_than_primary") is shadow_bounded, + f"{prefix}.shadow_not_greater_than_primary disagrees with raw counters", + ) + _require(shadow_bounded, f"{prefix} shadow whole exceeds the authoritative primary whole") + if phase_name == "none": + _require( + shadow_exact, + f"{prefix} a phase with zero local calls requires shadow whole to equal primary", + ) + + request_loss = primary_requests - shadow_requests + miss_loss = primary_misses - shadow_misses + _require( + _integer(record.get("shadow_request_loss"), f"{prefix}.shadow_request_loss") + == request_loss, + f"{prefix}.shadow_request_loss disagrees with raw counters", + ) + _require( + _integer(record.get("shadow_miss_loss"), f"{prefix}.shadow_miss_loss") == miss_loss, + f"{prefix}.shadow_miss_loss disagrees with raw counters", + ) + + calls = _integer(record.get("phase_calls"), f"{prefix}.phase_calls") + begin_reads = _integer(record.get("phase_begin_reads"), f"{prefix}.phase_begin_reads") + end_reads = _integer(record.get("phase_end_reads"), f"{prefix}.phase_end_reads") + _require( + record.get("phase_boundaries_balanced") is True, + f"{prefix}.phase_boundaries_balanced is not true", + ) + _require( + begin_reads == calls and end_reads == calls, + f"{prefix} phase begin/end boundaries do not match calls", + ) + primary_segments = _integer( + record.get("primary_window_segments"), f"{prefix}.primary_window_segments" + ) + shadow_segments = _integer( + record.get("shadow_read_segments"), f"{prefix}.shadow_read_segments" + ) + _require(primary_segments == 1, f"{prefix} primary_window_segments must be one") + _require( + shadow_segments == 2 * calls + 1, + f"{prefix} shadow_read_segments does not match phase boundaries plus tail", + ) + + phase_requests = _integer( + record.get("phase_icache_requests"), f"{prefix}.phase_icache_requests" + ) + phase_misses = _integer( + record.get("phase_icache_misses"), f"{prefix}.phase_icache_misses" + ) + phase_request_upper = _integer( + record.get("phase_icache_requests_upper_bound"), + f"{prefix}.phase_icache_requests_upper_bound", + ) + phase_miss_upper = _integer( + record.get("phase_icache_misses_upper_bound"), + f"{prefix}.phase_icache_misses_upper_bound", + ) + _require( + phase_requests <= shadow_requests and phase_misses <= shadow_misses, + f"{prefix} phase counters exceed the Submit shadow whole", + ) + _require( + phase_request_upper == phase_requests + request_loss, + f"{prefix}.phase_icache_requests_upper_bound is not lower plus shadow loss", + ) + _require( + phase_miss_upper == phase_misses + miss_loss, + f"{prefix}.phase_icache_misses_upper_bound is not lower plus shadow loss", + ) + _require( + phase_request_upper <= primary_requests and phase_miss_upper <= primary_misses, + f"{prefix} phase upper bound exceeds the authoritative primary whole", + ) + + _require(calls == expected_calls, f"{prefix} phase_calls does not match the phase contract") + if expected_calls == 0: + _require( + phase_requests == 0 and phase_misses == 0, + f"{prefix} a phase with zero calls must have zero phase counters", + ) + _require( + phase_request_upper == phase_requests and phase_miss_upper == phase_misses, + f"{prefix} a phase with zero calls must have zero-width bounds", + ) + if schema_version in SUBMIT_PMU_TIME_SCHEMA_VERSIONS: + submit_elapsed_ticks = _integer( + record.get("submit_elapsed_ticks"), f"{prefix}.submit_elapsed_ticks" + ) + phase_elapsed_ticks = _integer( + record.get("phase_elapsed_ticks"), f"{prefix}.phase_elapsed_ticks" + ) + _require(submit_elapsed_ticks > 0, f"{prefix}.submit_elapsed_ticks must be positive") + _require( + phase_elapsed_ticks <= submit_elapsed_ticks, + f"{prefix}.phase_elapsed_ticks exceeds its Submit interval", + ) + _require( + phase_elapsed_ticks == 0 if expected_calls == 0 else phase_elapsed_ticks > 0, + f"{prefix}.phase_elapsed_ticks does not match enabled phase calls", + ) + _require(record.get("phase_time_valid") is True, f"{prefix}.phase_time_valid is not true") + + return PhasePartitionEvidence( + shadow_exact=shadow_exact, + shadow_bounded=shadow_bounded, + request_abs_delta=abs(shadow_requests - primary_requests), + request_signed_delta=shadow_requests - primary_requests, + miss_abs_delta=abs(shadow_misses - primary_misses), + miss_signed_delta=shadow_misses - primary_misses, + expected_calls=expected_calls, + ) + + +def load_capture(path: Path) -> Capture: + """读取并完整校验历史 v3 或 submit-pmu v4/v5/v6 sidecar。""" + + with path.open("r", encoding="utf-8") as input_file: + data = json.load(input_file) + _require(isinstance(data, dict), f"{path}: capture root must be an object") + schema = data.get("schema") + _require( + isinstance(schema, dict) and schema.get("name") == SCHEMA_NAME, + f"{path}: expected schema name {SCHEMA_NAME}", + ) + schema_version = _integer(schema.get("version"), f"{path}: schema.version") + _require( + schema_version in SCHEMA_VERSIONS, + f"{path}: expected {SCHEMA_NAME} schema v3, v4, v5 or v6", + ) + + capture = data.get("capture") + configuration = data.get("configuration") + validation = data.get("validation") + records = data.get("records") + summary = data.get("summary") + _require(isinstance(capture, dict), f"{path}: capture must be an object") + _require(isinstance(configuration, dict), f"{path}: configuration must be an object") + _require(isinstance(validation, dict), f"{path}: validation must be an object") + _require(isinstance(records, list), f"{path}: records must be an array") + _require(isinstance(summary, dict), f"{path}: summary must be an object") + + workers = _integer(configuration.get("workers"), f"{path}: configuration.workers") + aic_workers = _integer(configuration.get("aic_workers"), f"{path}: configuration.aic_workers") + aiv_workers = _integer(configuration.get("aiv_workers"), f"{path}: configuration.aiv_workers") + batches = _integer(configuration.get("batches"), f"{path}: configuration.batches") + _require(workers == aic_workers + aiv_workers, f"{path}: worker role counts do not add up") + _require(len(records) == workers, f"{path}: record count does not match configuration.workers") + phase_name: str | None = None + phase_id: int | None = None + tasks_per_worker: int | None = None + if schema_version in SUBMIT_PMU_SCHEMA_VERSIONS: + _require( + (workers, aic_workers, aiv_workers) + == (A5_WORKERS, A5_AIC_WORKERS, A5_AIV_WORKERS), + f"{path}: submit-pmu schema requires the fixed 96/32/64 A5 topology", + ) + phase_name, phase_id = _validate_submit_pmu_configuration( + path, configuration, schema_version + ) + tasks_per_worker = _validate_submit_pmu_task_contract( + path, configuration, batches, schema_version + ) + else: + counter_widths = configuration.get("counter_width_bits") + _require( + isinstance(counter_widths, dict), + f"{path}: configuration.counter_width_bits must be an object", + ) + _require( + _integer( + counter_widths.get("total"), f"{path}: configuration.counter_width_bits.total" + ) + == 64, + f"{path}: configuration.counter_width_bits.total must be 64", + ) + _require( + _integer( + counter_widths.get("programmable"), + f"{path}: configuration.counter_width_bits.programmable", + ) + == PROGRAMMABLE_COUNTER_BITS, + f"{path}: configuration.counter_width_bits.programmable must be 32", + ) + + # JSON 只有在运行、PMU、owner Restore 和 runtime cleanup 全部成功后才应发布。 + # 分析器仍逐项重验，防止手工复制或未来 schema 退化绕过发布门禁。 + required_true = ( + (capture, "accepted"), + (capture, "published_after_runtime_cleanup"), + (capture, "runtime_cleanup_passed"), + (capture, "owner_restore_passed"), + (validation, "semantic_passed"), + (validation, "pmu_passed"), + (validation, "icache_measurement_valid"), + (validation, "icache_miss_le_request"), + (validation, "counter_below_risk_threshold"), + ) + for owner, field in required_true: + _require(owner.get(field) is True, f"{path}: {field} is not true") + configured_physical_ids: set[int] | None = None + if schema_version in SUBMIT_PMU_SCHEMA_VERSIONS: + _require( + validation.get("phase_measurement_valid") is True, + f"{path}: phase_measurement_valid is not true", + ) + configured_physical_ids = _validate_submit_pmu_owner(path, data.get("owner"), capture) + + expected_records = ( + "trusted_records", + "unique_physical_core_ids", + "owner_bitmap_member_records", + "exact_worker_slot_records", + "physical_role_match_records", + "window_started_records", + "window_stopped_records", + ) + for field in expected_records: + _require( + validation.get(field) == workers, + f"{path}: validation.{field} is incomplete", + ) + if schema_version in SUBMIT_PMU_SCHEMA_VERSIONS: + expected_host_counts = { + "expected_records": A5_WORKERS, + "expected_unique_core_ids": A5_WORKERS, + "expected_owner_bitmap_member_records": A5_WORKERS, + "expected_exact_worker_slot_records": A5_WORKERS, + "expected_physical_role_match_records": A5_WORKERS, + "mixed_triplet_matches": A5_AIC_WORKERS, + "expected_mixed_triplet_matches": A5_AIC_WORKERS, + "expected_window_records": A5_WORKERS, + } + for field, expected in expected_host_counts.items(): + _require( + _integer(validation.get(field), f"{path}: validation.{field}") == expected, + f"{path}: validation.{field} does not match the A5 topology", + ) + for field in ( + "build_variant_match_records", + "phase_id_match_records", + "phase_status_trusted_records", + "phase_boundary_match_records", + "phase_call_shape_match_records", + ): + _require( + validation.get(field) == workers, + f"{path}: validation.{field} is incomplete", + ) + groups: dict[str, list[dict[str, Any]]] = { + "all": records, + "aic": [record for record in records if record.get("role") == "aic"], + "aiv": [record for record in records if record.get("role") == "aiv"], + } + + worker_ids: set[int] = set() + physical_core_ids: set[int] = set() + ordered_physical_ids: list[int] = [] + phase_partition_evidence: list[PhasePartitionEvidence] = [] + for index, record in enumerate(records): + _require(isinstance(record, dict), f"{path}: records[{index}] must be an object") + worker_id = _integer(record.get("worker_id"), f"{path}: records[{index}].worker_id") + physical_id = _integer( + record.get("physical_core_id"), f"{path}: records[{index}].physical_core_id" + ) + _require(worker_id not in worker_ids, f"{path}: duplicate worker_id {worker_id}") + _require( + physical_id not in physical_core_ids, + f"{path}: duplicate physical_core_id {physical_id}", + ) + worker_ids.add(worker_id) + physical_core_ids.add(physical_id) + ordered_physical_ids.append(physical_id) + if schema_version in SUBMIT_PMU_SCHEMA_VERSIONS: + _require( + worker_id == index, + f"{path}: records[{index}].worker_id does not match its exact worker slot", + ) + expected_role, expected_block, expected_lane = _expected_logical_triplet(index) + _require( + record.get("role") == expected_role, + f"{path}: records[{index}].role does not match the logical worker topology", + ) + _require( + _integer(record.get("block_id"), f"{path}: records[{index}].block_id") + == expected_block, + f"{path}: records[{index}].block_id does not match its mixed block", + ) + _require( + _integer(record.get("lane"), f"{path}: records[{index}].lane") == expected_lane, + f"{path}: records[{index}].lane does not match its mixed lane", + ) + _require( + physical_id < A5_PHYSICAL_SUBCORES, + f"{path}: records[{index}].physical_core_id is outside the 108-slot topology", + ) + _require( + _is_aic_physical_slot(physical_id) == (expected_role == "aic"), + f"{path}: records[{index}] logical role does not match its physical slot", + ) + _require( + record.get("physical_core_id_valid") is True, + f"{path}: records[{index}].physical_core_id_valid is not true", + ) + assert configured_physical_ids is not None + _require( + physical_id in configured_physical_ids, + f"{path}: records[{index}].physical_core_id is absent from the owner bitmap", + ) + for field in ( + "trusted", + "selectors_match", + "owner_bitmap_member", + "worker_slot_exact", + "physical_role_matches", + "window_started", + "window_stopped", + ): + _require(record.get(field) is True, f"{path}: records[{index}].{field} is not true") + requests = _integer( + record.get("icache_requests"), f"{path}: records[{index}].icache_requests" + ) + misses = _integer( + record.get("icache_misses"), f"{path}: records[{index}].icache_misses" + ) + total_cycles = _integer( + record.get("total_cycles"), f"{path}: records[{index}].total_cycles" + ) + scalar_busy = _integer( + record.get("scalar_busy"), f"{path}: records[{index}].scalar_busy" + ) + _require(misses <= requests, f"{path}: records[{index}] has miss > request") + _require(total_cycles > 0, f"{path}: records[{index}] has zero total_cycles") + _require( + scalar_busy <= total_cycles, + f"{path}: records[{index}] has scalar_busy > total_cycles", + ) + programmable_fields = ( + ( + "vector_busy", + "cube_busy", + "scalar_busy", + "mte1_busy", + "mte2_busy", + "icache_requests", + "icache_misses", + "phase_icache_requests", + "phase_icache_misses", + "shadow_whole_icache_requests", + "shadow_whole_icache_misses", + ) + if schema_version in SUBMIT_PMU_SCHEMA_VERSIONS + else tuple(metric for metric in METRIC_NAMES if metric != "total_cycles") + ) + for field in programmable_fields: + value = _integer(record.get(field), f"{path}: records[{index}].{field}") + _require( + value < PROGRAMMABLE_COUNTER_RISK_THRESHOLD, + f"{path}: records[{index}].{field} reaches the 32-bit counter risk threshold", + ) + if schema_version in SUBMIT_PMU_SCHEMA_VERSIONS: + # 上面的 configuration 校验已保证二者不是 None；显式 assert 只帮助 + # 类型收窄，不替代任何 JSON 运行时门禁。 + assert phase_name is not None and phase_id is not None + phase_partition_evidence.append( + _validate_submit_pmu_record( + path, + index, + record, + phase_name, + phase_id, + batches, + schema_version, + tasks_per_worker, + ) + ) + + if schema_version in SUBMIT_PMU_SCHEMA_VERSIONS: + assert configured_physical_ids is not None + _require( + physical_core_ids == configured_physical_ids, + f"{path}: worker physical-core set does not exactly equal the owner bitmap", + ) + for block in range(A5_AIC_WORKERS): + aic_id = ordered_physical_ids[block] + expected_aiv0, expected_aiv1 = _physical_aiv_pair(aic_id) + actual_aiv0 = ordered_physical_ids[A5_AIC_WORKERS + block * 2] + actual_aiv1 = ordered_physical_ids[A5_AIC_WORKERS + block * 2 + 1] + _require( + (actual_aiv0, actual_aiv1) == (expected_aiv0, expected_aiv1), + f"{path}: mixed block {block} does not map to one physical AIC and its two AIVs", + ) + + shadow_exact_records = sum(item.shadow_exact for item in phase_partition_evidence) + shadow_bounded_records = sum(item.shadow_bounded for item in phase_partition_evidence) + phase_shadow_acceptable_records = sum( + item.shadow_exact if item.expected_calls == 0 else item.shadow_bounded + for item in phase_partition_evidence + ) + expected_phase_calls = sum(item.expected_calls for item in phase_partition_evidence) + if schema_version == 6: + assert tasks_per_worker is not None and phase_name is not None + expected_shared_total = ( + tasks_per_worker + if phase_name in ("materialize", "register") + else ( + 0 + if phase_name == "none" + else tasks_per_worker * A5_WORKERS + ) + ) + _require( + expected_phase_calls == expected_shared_total, + f"{path}: shared phase_expected_calls do not close to the " + "unique-winner/global-replay contract", + ) + _require( + shadow_bounded_records == A5_WORKERS, + f"{path}: not all shadow partitions are bounded by primary", + ) + _require( + phase_shadow_acceptable_records == A5_WORKERS, + f"{path}: zero-call phase records are not exact or active records are unbounded", + ) + assert phase_name is not None + if phase_name == "none": + _require( + shadow_exact_records == A5_WORKERS, + f"{path}: phase=none requires all shadow partitions to be exact", + ) + _require( + _integer(validation.get("phase_calls"), f"{path}: validation.phase_calls") + == expected_phase_calls, + f"{path}: validation.phase_calls does not match raw per-worker contracts", + ) + if schema_version == 6: + _require( + "phase_expected_calls" in validation, + f"{path}: validation.phase_expected_calls is required by schema-v6", + ) + if "phase_expected_calls" in validation: + _require( + _integer( + validation.get("phase_expected_calls"), + f"{path}: validation.phase_expected_calls", + ) + == expected_phase_calls, + f"{path}: validation.phase_expected_calls disagrees with raw evidence", + ) + if "phase_shadow_acceptable_records" in validation: + _require( + _integer( + validation.get("phase_shadow_acceptable_records"), + f"{path}: validation.phase_shadow_acceptable_records", + ) + == phase_shadow_acceptable_records, + f"{path}: validation.phase_shadow_acceptable_records disagrees with raw records", + ) + _require( + _integer( + validation.get("shadow_primary_match_records"), + f"{path}: validation.shadow_primary_match_records", + ) + == shadow_exact_records, + f"{path}: validation.shadow_primary_match_records disagrees with raw records", + ) + _require( + _integer( + validation.get("shadow_primary_bounded_records"), + f"{path}: validation.shadow_primary_bounded_records", + ) + == shadow_bounded_records, + f"{path}: validation.shadow_primary_bounded_records disagrees with raw records", + ) + unsigned_deltas = { + "shadow_request_abs_delta_sum": sum( + item.request_abs_delta for item in phase_partition_evidence + ), + "shadow_request_abs_delta_max": max( + item.request_abs_delta for item in phase_partition_evidence + ), + "shadow_miss_abs_delta_sum": sum( + item.miss_abs_delta for item in phase_partition_evidence + ), + "shadow_miss_abs_delta_max": max( + item.miss_abs_delta for item in phase_partition_evidence + ), + } + signed_deltas = { + "shadow_request_signed_delta_sum": sum( + item.request_signed_delta for item in phase_partition_evidence + ), + "shadow_miss_signed_delta_sum": sum( + item.miss_signed_delta for item in phase_partition_evidence + ), + } + for field, expected in unsigned_deltas.items(): + _require( + _integer(validation.get(field), f"{path}: validation.{field}") == expected, + f"{path}: validation.{field} disagrees with raw records", + ) + for field, expected in signed_deltas.items(): + _require( + _signed_integer(validation.get(field), f"{path}: validation.{field}") + == expected, + f"{path}: validation.{field} disagrees with raw records", + ) + if schema_version in SUBMIT_PMU_TIME_SCHEMA_VERSIONS: + _require( + _integer( + validation.get("phase_time_valid_records"), + f"{path}: validation.phase_time_valid_records", + ) + == A5_WORKERS, + f"{path}: validation.phase_time_valid_records is incomplete", + ) + _require( + validation.get("phase_time_measurement_valid") is True, + f"{path}: validation.phase_time_measurement_valid is not true", + ) + + _require(len(groups["aic"]) == aic_workers, f"{path}: AIC raw record count mismatch") + _require(len(groups["aiv"]) == aiv_workers, f"{path}: AIV raw record count mismatch") + + metric_names = ( + SUBMIT_PMU_V5_METRIC_NAMES + if schema_version in SUBMIT_PMU_TIME_SCHEMA_VERSIONS + else SUBMIT_PMU_METRIC_NAMES + if schema_version == 4 + else METRIC_NAMES + ) + for group_name in GROUP_NAMES: + _validate_group_summary( + path, + group_name, + groups[group_name], + summary.get(group_name), + metric_names, + schema_version, + ) + + return Capture( + path, + data, + groups, + _configuration_fingerprint(configuration, schema_version), + schema_version, + ) + + +def _median(values: Iterable[int | float]) -> int | float: + return statistics.median(list(values)) + + +def analyze(paths: Sequence[Path], miss_penalty_ns: float = 90.0) -> dict[str, Any]: + """校验同配置多轮 sidecar，并返回可序列化的跨轮汇总。""" + + _require(bool(paths), "at least one PMU JSON path is required") + _require(math.isfinite(miss_penalty_ns) and miss_penalty_ns > 0, "miss penalty must be positive") + captures = [load_capture(path) for path in paths] + schema_version = captures[0].schema_version + fingerprint = captures[0].fingerprint + for capture in captures[1:]: + _require( + capture.schema_version == schema_version, + f"{capture.path}: input schema differs from {captures[0].path}", + ) + _require( + capture.fingerprint == fingerprint, + f"{capture.path}: observation configuration differs from {captures[0].path}", + ) + + per_run: list[dict[str, Any]] = [] + for capture in captures: + configuration = capture.data["configuration"] + summary = capture.data["summary"] + row: dict[str, Any] = { + "path": str(capture.path), + "capture_id": capture.data["capture"].get("capture_id"), + "submit_span_us": _number( + configuration.get("submit_span_us"), + f"{capture.path}: configuration.submit_span_us", + ), + "groups": {}, + } + for group_name in GROUP_NAMES: + group = summary[group_name] + cores = _integer(group.get("cores"), f"summary.{group_name}.cores") + requests = _integer(group["icache_requests"].get("sum"), "icache request sum") + misses = _integer(group["icache_misses"].get("sum"), "icache miss sum") + row["groups"][group_name] = { + "cores": cores, + "icache_requests_sum": requests, + "icache_misses_sum": misses, + "icache_miss_rate": misses / requests, + "icache_requests_per_core": requests / cores, + "icache_misses_per_core": misses / cores, + "icache_misses_per_core_median": group["icache_misses"]["median"], + "icache_misses_per_core_p95": group["icache_misses"]["p95"], + "first_order_miss_core_equivalent_us": misses * miss_penalty_ns / 1000.0, + "first_order_miss_per_core_us": misses * miss_penalty_ns / cores / 1000.0, + "scalar_busy_sum": group["scalar_busy"]["sum"], + "total_cycles_sum": group["total_cycles"]["sum"], + } + if schema_version in SUBMIT_PMU_SCHEMA_VERSIONS: + raw_group = capture.groups[group_name] + phase_requests = _integer( + group["phase_icache_requests"].get("sum"), "phase I-cache request sum" + ) + phase_misses = _integer( + group["phase_icache_misses"].get("sum"), "phase I-cache miss sum" + ) + phase_calls = _integer(group["phase_calls"].get("sum"), "phase call sum") + phase_request_uppers = [ + _integer( + record.get("phase_icache_requests_upper_bound"), + "phase I-cache request upper bound", + ) + for record in raw_group + ] + phase_miss_uppers = [ + _integer( + record.get("phase_icache_misses_upper_bound"), + "phase I-cache miss upper bound", + ) + for record in raw_group + ] + request_losses = [ + _integer(record.get("shadow_request_loss"), "shadow request loss") + for record in raw_group + ] + miss_losses = [ + _integer(record.get("shadow_miss_loss"), "shadow miss loss") + for record in raw_group + ] + phase_request_upper_summary = _metric_summary(phase_request_uppers) + phase_miss_upper_summary = _metric_summary(phase_miss_uppers) + phase_request_lower_summary = _metric_summary( + [ + _integer( + record.get("phase_icache_requests"), + "phase I-cache request lower bound", + ) + for record in raw_group + ] + ) + phase_miss_lower_summary = _metric_summary( + [ + _integer( + record.get("phase_icache_misses"), + "phase I-cache miss lower bound", + ) + for record in raw_group + ] + ) + request_loss_summary = _metric_summary(request_losses) + miss_loss_summary = _metric_summary(miss_losses) + phase_request_upper_sum = sum(phase_request_uppers) + phase_miss_upper_sum = sum(phase_miss_uppers) + row["groups"][group_name].update( + { + "phase_calls_sum": phase_calls, + "phase_calls_per_core": phase_calls / cores, + "phase_icache_requests_lower_bound_sum": phase_requests, + "phase_icache_requests_upper_bound_sum": phase_request_upper_sum, + "phase_icache_misses_lower_bound_sum": phase_misses, + "phase_icache_misses_upper_bound_sum": phase_miss_upper_sum, + "phase_icache_requests_lower_bound_per_core": phase_requests / cores, + "phase_icache_requests_upper_bound_per_core": + phase_request_upper_sum / cores, + "phase_icache_misses_lower_bound_per_core": phase_misses / cores, + "phase_icache_misses_upper_bound_per_core": phase_miss_upper_sum / cores, + "phase_icache_requests_lower_bound_per_core_median": + phase_request_lower_summary["median"], + "phase_icache_requests_lower_bound_per_core_p95": + phase_request_lower_summary["p95"], + "phase_icache_requests_upper_bound_per_core_median": + phase_request_upper_summary["median"], + "phase_icache_requests_upper_bound_per_core_p95": + phase_request_upper_summary["p95"], + "phase_icache_misses_lower_bound_per_core_median": + phase_miss_lower_summary["median"], + "phase_icache_misses_lower_bound_per_core_p95": + phase_miss_lower_summary["p95"], + "phase_icache_misses_upper_bound_per_core_median": + phase_miss_upper_summary["median"], + "phase_icache_misses_upper_bound_per_core_p95": + phase_miss_upper_summary["p95"], + "shadow_request_loss_sum": sum(request_losses), + "shadow_request_loss_per_core": sum(request_losses) / cores, + "shadow_request_loss_per_core_median": request_loss_summary["median"], + "shadow_request_loss_per_core_p95": request_loss_summary["p95"], + "shadow_miss_loss_sum": sum(miss_losses), + "shadow_miss_loss_per_core": sum(miss_losses) / cores, + "shadow_miss_loss_per_core_median": miss_loss_summary["median"], + "shadow_miss_loss_per_core_p95": miss_loss_summary["p95"], + "phase_observed_read_clear_ratio": ( + None if phase_requests == 0 else phase_misses / phase_requests + ), + "phase_icache_request_lower_bound_share_of_submit": + phase_requests / requests, + "phase_icache_request_upper_bound_share_of_submit": + phase_request_upper_sum / requests, + "phase_icache_miss_lower_bound_share_of_submit": + phase_misses / misses if misses != 0 else None, + "phase_icache_miss_upper_bound_share_of_submit": + phase_miss_upper_sum / misses if misses != 0 else None, + } + ) + if schema_version in SUBMIT_PMU_TIME_SCHEMA_VERSIONS: + submit_elapsed_ticks = _integer( + group["submit_elapsed_ticks"].get("sum"), + "Submit elapsed tick sum", + ) + phase_elapsed_ticks = _integer( + group["phase_elapsed_ticks"].get("sum"), + "phase elapsed tick sum", + ) + tick_ns = _number( + configuration.get("phase_time_sys_counter_tick_ns"), + "phase SYS_CNT tick ns", + ) + _require(submit_elapsed_ticks > 0, "Submit elapsed tick sum is zero") + row["groups"][group_name].update( + { + "submit_elapsed_ticks_sum": submit_elapsed_ticks, + "submit_elapsed_per_core_us": + submit_elapsed_ticks * tick_ns / cores / 1000.0, + "phase_elapsed_ticks_sum": phase_elapsed_ticks, + "phase_elapsed_per_core_us": + phase_elapsed_ticks * tick_ns / cores / 1000.0, + "phase_elapsed_per_call_ns": ( + None + if phase_calls == 0 + else phase_elapsed_ticks * tick_ns / phase_calls + ), + "phase_time_share_of_submit": + phase_elapsed_ticks / submit_elapsed_ticks, + } + ) + per_run.append(row) + + aggregate: dict[str, Any] = { + "runs": len(per_run), + "submit_span_us": { + "median": _median(row["submit_span_us"] for row in per_run), + "min": min(row["submit_span_us"] for row in per_run), + "max": max(row["submit_span_us"] for row in per_run), + }, + "groups": {}, + } + aggregate_fields = ( + "icache_requests_sum", + "icache_misses_sum", + "icache_miss_rate", + "icache_requests_per_core", + "icache_misses_per_core", + "icache_misses_per_core_median", + "icache_misses_per_core_p95", + "first_order_miss_core_equivalent_us", + "first_order_miss_per_core_us", + "scalar_busy_sum", + "total_cycles_sum", + ) + if schema_version in SUBMIT_PMU_SCHEMA_VERSIONS: + aggregate_fields += ( + "phase_calls_sum", + "phase_calls_per_core", + "phase_icache_requests_lower_bound_sum", + "phase_icache_requests_upper_bound_sum", + "phase_icache_misses_lower_bound_sum", + "phase_icache_misses_upper_bound_sum", + "phase_icache_requests_lower_bound_per_core", + "phase_icache_requests_upper_bound_per_core", + "phase_icache_misses_lower_bound_per_core", + "phase_icache_misses_upper_bound_per_core", + "phase_icache_requests_lower_bound_per_core_median", + "phase_icache_requests_lower_bound_per_core_p95", + "phase_icache_requests_upper_bound_per_core_median", + "phase_icache_requests_upper_bound_per_core_p95", + "phase_icache_misses_lower_bound_per_core_median", + "phase_icache_misses_lower_bound_per_core_p95", + "phase_icache_misses_upper_bound_per_core_median", + "phase_icache_misses_upper_bound_per_core_p95", + "shadow_request_loss_sum", + "shadow_request_loss_per_core", + "shadow_request_loss_per_core_median", + "shadow_request_loss_per_core_p95", + "shadow_miss_loss_sum", + "shadow_miss_loss_per_core", + "shadow_miss_loss_per_core_median", + "shadow_miss_loss_per_core_p95", + "phase_observed_read_clear_ratio", + "phase_icache_request_lower_bound_share_of_submit", + "phase_icache_request_upper_bound_share_of_submit", + "phase_icache_miss_lower_bound_share_of_submit", + "phase_icache_miss_upper_bound_share_of_submit", + ) + if schema_version in SUBMIT_PMU_TIME_SCHEMA_VERSIONS: + aggregate_fields += ( + "submit_elapsed_ticks_sum", + "submit_elapsed_per_core_us", + "phase_elapsed_ticks_sum", + "phase_elapsed_per_core_us", + "phase_elapsed_per_call_ns", + "phase_time_share_of_submit", + ) + for group_name in GROUP_NAMES: + aggregate["groups"][group_name] = {} + for field in aggregate_fields: + values = [ + row["groups"][group_name][field] + for row in per_run + if row["groups"][group_name][field] is not None + ] + aggregate["groups"][group_name][field] = ( + { + "median": _median(values), + "min": min(values), + "max": max(values), + } + if values + else {"median": None, "min": None, "max": None} + ) + + configuration = captures[0].data["configuration"] + fingerprint_fields = ( + SHARED_SUBMIT_PMU_FINGERPRINT_FIELDS + if schema_version == 6 + else SUBMIT_PMU_FINGERPRINT_FIELDS + if schema_version in SUBMIT_PMU_SCHEMA_VERSIONS + else CONFIG_FINGERPRINT_FIELDS + ) + result: dict[str, Any] = { + "schema": { + "name": "pa_scheduler_pmu_multi_run_summary", + "version": ( + 4 + if schema_version == 6 + else 3 + if schema_version == 5 + else 2 + if schema_version == 4 + else 1 + ), + }, + "input_schema": {"name": SCHEMA_NAME, "version": schema_version}, + "configuration": { + field: configuration.get(field) for field in fingerprint_fields + }, + "estimation": { + "icache_miss_penalty_ns": miss_penalty_ns, + "meaning": "controlled_cold_warm_first_order_core_equivalent", + "not_wall_time": True, + "not_additive_stall_time": True, + }, + "actual_exposed_loss": { + "status": "requires_same_semantics_paired_ab", + "reason": "A5 submit-all has no verified I-cache stall-cycle counter", + "required_observations": ("delta_submit_span_us", "delta_icache_misses_per_core"), + }, + "validation": { + "raw_to_summary_all_fields_match": True, + "all_inputs_accepted_and_restored": True, + "same_observation_configuration": True, + }, + "per_run": per_run, + "aggregate": aggregate, + } + if schema_version in SUBMIT_PMU_SCHEMA_VERSIONS: + phase_enabled = configuration["compiled_phase"] != "none" + result["phase_observation"] = { + "compiled_phase": configuration["compiled_phase"], + "compiled_phase_id": configuration["compiled_phase_id"], + "enabled": phase_enabled, + "primary_whole_authoritative": True, + "phase_share_is_same_elf_submit_only": True, + "phase_boundary_observer_perturbed": phase_enabled, + "phase_counter_pair_snapshot_atomic": False, + "phase_shadow_partition_exact_required": configuration[ + "phase_shadow_partition_exact_required" + ], + "phase_values_are_running_read_clear_lower_bounds": configuration[ + "phase_values_are_running_read_clear_lower_bounds" + ], + "phase_value_semantics": ( + "disabled_zero" + if not phase_enabled + else "running_read_clear_lower_to_loss_adjusted_upper_bound" + ), + "cross_phase_elf_sums_valid": False, + "phase_time_available": schema_version in SUBMIT_PMU_TIME_SCHEMA_VERSIONS, + "phase_time_semantics": ( + "unavailable_in_schema_v4" + if schema_version == 4 + else "sum_per_call_sys_cnt_inside_shadow_read_clear_boundaries" + ), + } + return result + + +def _print_text(result: dict[str, Any]) -> None: + configuration = result["configuration"] + workload = configuration.get("winner_workload") or {} + counts = workload.get("counts") or {} + build_phase = "" + if result["input_schema"]["version"] in SUBMIT_PMU_SCHEMA_VERSIONS: + build_phase = ( + f" build={configuration.get('build_variant')}" + f" phase={configuration.get('compiled_phase')}" + ) + print( + "[CONFIG] " + f"window={configuration.get('pmu_window')} batches={configuration.get('batches')} " + f"workers={configuration.get('workers')} workload={workload.get('mode')} " + f"counts={counts.get('qk')},{counts.get('sf')},{counts.get('pv')},{counts.get('up')}" + f"{build_phase}" + ) + print("[VALIDATION] raw_to_summary=PASS accepted_restore=PASS same_configuration=PASS") + print( + "run submit_us all_miss/core all_rate " + "aic_miss/core aic_rate aiv_miss/core aiv_rate" + ) + for index, row in enumerate(result["per_run"], start=1): + all_group = row["groups"]["all"] + aic = row["groups"]["aic"] + aiv = row["groups"]["aiv"] + print( + f"{index:>3} {row['submit_span_us']:>9.3f} " + f"{all_group['icache_misses_per_core']:>13.3f} " + f"{all_group['icache_miss_rate'] * 100:>7.4f}% " + f"{aic['icache_misses_per_core']:>13.3f} {aic['icache_miss_rate'] * 100:>7.4f}% " + f"{aiv['icache_misses_per_core']:>14.3f} {aiv['icache_miss_rate'] * 100:>7.4f}%" + ) + + aggregate = result["aggregate"] + all_group = aggregate["groups"]["all"] + aic = aggregate["groups"]["aic"] + aiv = aggregate["groups"]["aiv"] + print( + "[PRIMARY] " + f"submit_us={aggregate['submit_span_us']['median']:.3f} " + f"all_miss_per_core={all_group['icache_misses_per_core']['median']:.3f} " + f"AIC_miss_per_core={aic['icache_misses_per_core']['median']:.3f} " + f"AIV_request_per_core={aiv['icache_requests_per_core']['median']:.3f} " + f"AIV_miss_per_core={aiv['icache_misses_per_core']['median']:.3f} " + f"AIV_core_miss_p95={aiv['icache_misses_per_core_p95']['median']:.3f} " + f"AIV_miss_rate={aiv['icache_miss_rate']['median'] * 100:.4f}%" + ) + if result["input_schema"]["version"] in SUBMIT_PMU_SCHEMA_VERSIONS: + phase = result["phase_observation"] + if not phase["enabled"]: + print( + "[PHASE-BOUNDS] selected=none status=DISABLED " + "request_per_core=0..0 miss_per_core=0..0 shadow_loss_per_core=request:0,miss:0 " + "shadow_partition=EXACT_DISABLED" + ) + else: + print( + "[PHASE-BOUNDS] " + f"selected={phase['compiled_phase']} semantics=RUNNING_READ_CLEAR_BOUNDS " + f"AIC_request_per_core=" + f"{aic['phase_icache_requests_lower_bound_per_core']['median']:.3f}.." + f"{aic['phase_icache_requests_upper_bound_per_core']['median']:.3f} " + f"AIC_miss_per_core=" + f"{aic['phase_icache_misses_lower_bound_per_core']['median']:.3f}.." + f"{aic['phase_icache_misses_upper_bound_per_core']['median']:.3f} " + f"AIC_shadow_loss_per_core=request:{aic['shadow_request_loss_per_core']['median']:.3f}," + f"miss:{aic['shadow_miss_loss_per_core']['median']:.3f} " + f"AIV_request_per_core=" + f"{aiv['phase_icache_requests_lower_bound_per_core']['median']:.3f}.." + f"{aiv['phase_icache_requests_upper_bound_per_core']['median']:.3f} " + f"AIV_miss_per_core=" + f"{aiv['phase_icache_misses_lower_bound_per_core']['median']:.3f}.." + f"{aiv['phase_icache_misses_upper_bound_per_core']['median']:.3f} " + f"AIV_shadow_loss_per_core=request:{aiv['shadow_request_loss_per_core']['median']:.3f}," + f"miss:{aiv['shadow_miss_loss_per_core']['median']:.3f} " + "shadow_partition=BOUNDED_NOT_EXACT_REQUIRED" + ) + if phase["phase_time_available"]: + print( + "[PHASE-TIME] " + f"selected={phase['compiled_phase']} " + f"ALL_core_time_share={all_group['phase_time_share_of_submit']['median'] * 100:.4f}% " + f"AIC_core_time_share={aic['phase_time_share_of_submit']['median'] * 100:.4f}% " + f"AIV_core_time_share={aiv['phase_time_share_of_submit']['median'] * 100:.4f}%" + ) + boundary_state = "PERTURBED" if phase["enabled"] else "DISABLED" + pair_state = "FALSE" if phase["enabled"] else "NOT_APPLICABLE" + print( + "[PERTURBATION] primary_submit_whole=AUTHORITATIVE " + f"phase_boundary_observer={boundary_state} phase_counter_pair_atomic={pair_state} " + "cross_phase_elf_sum=INVALID" + ) + print( + "[SERIAL-EQUIVALENT] " + f"penalty={result['estimation']['icache_miss_penalty_ns']:.3f}ns/miss " + f"AIC_per_core_us={aic['first_order_miss_per_core_us']['median']:.3f} " + f"AIV_per_core_us={aiv['first_order_miss_per_core_us']['median']:.3f} " + "meaning=core_equivalent_not_wall_or_additive_stall" + ) + print( + "[ACTUAL-EXPOSED-LOSS] status=UNMEASURED " + "method=requires_same_semantics_paired_AB_delta_submit_and_delta_miss" + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "inputs", nargs="+", type=Path, + help="历史 schema-v3 或 submit-pmu schema-v4/v5/v6 JSON sidecars" + ) + parser.add_argument( + "--icache-miss-ns", + type=float, + default=90.0, + help="受控 cold/warm 标尺；只用于一阶 core-work 等效估算（默认 90）", + ) + parser.add_argument("--json", action="store_true", help="输出机器可读的聚合 JSON") + arguments = parser.parse_args(argv) + try: + result = analyze(arguments.inputs, arguments.icache_miss_ns) + except (OSError, ValueError, json.JSONDecodeError) as error: + print(f"PMU sidecar analysis failed: {error}", file=sys.stderr) + return 1 + if arguments.json: + json.dump(result, sys.stdout, ensure_ascii=False, indent=2) + print() + else: + _print_text(result) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/atomic_probe/pa_scheduler/run.sh b/tests/atomic_probe/pa_scheduler/run.sh new file mode 100755 index 0000000000..07e7c252fa --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/run.sh @@ -0,0 +1,751 @@ +#!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +set -euo pipefail + +# 用脚本自身位置锚定所有构建产物、转换器和输出目录；从任意 cwd 调用都 +# 不会回退到 simpler 仓库中的同名工具。 +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +usage() { + cat <<'EOF' +Usage: + ./run.sh build ccec|ascendc|cpu|all [--tensormap private|shared] + ./run.sh run ccec|ascendc|cpu|all [--tensormap private|shared] [benchmark options] + ./run.sh smoke ccec|ascendc|cpu|all [--tensormap private|shared] [--device N] + ./run.sh swimlane ccec|ascendc|cpu|all [--tensormap private|shared] [benchmark options] + ./run.sh build-perf-clock ccec|cpu [--tensormap private|shared] + ./run.sh perf-clock ccec|cpu [--tensormap private|shared] [benchmark options] + ./run.sh build-submit-pmu ccec none|claim|efdrain|materialize|register [--tensormap private|shared] + ./run.sh submit-pmu ccec none|claim|efdrain|materialize|register [--tensormap private|shared] [benchmark options] + ./run.sh build-shared-protocol-litmus ccec + ./run.sh shared-protocol-litmus ccec --scenario history|reader-reclaim|all \ + [--ordering compiler-clobber|payload-dependency|dsb-all|all] [--device N] [--runs N] + +Build identity option (consumed by run.sh and never forwarded to the benchmark): + --tensormap private|shared (default: private) + +Shared insert-turn build identity: + PA_SHARED_INSERT_TURN_GROUPS=1|2|4|8|16|32|64|128 + (default: 1; shared only) + Use the same value when consuming a CCEC artifact; the manifest rejects a + different or omitted value instead of silently running another G. + +Benchmark options: + --device N + --batches 1..512 (shared; private remains 1..256) + --shared-context-lens C0[,C1...] (shared standalone test only; one value broadcasts, + otherwise count must equal --batches; default 8192) + --runs N + --nop-count N + --nop-counts QK,SF,PV,UP + --profile-phases + --analyze-swimlane + --trace-atomics + --swimlane-json FILE + --no-swimlane + --final-barrier flat|two-4|two-8|two-16|three-6x4x4 (default: two-16) + +CCEC-only PMU probe options (selectors are owned by the standalone Main AICPU helper): + --pmu-window off|empty|scalar|scalar-double|icache-single|submit-all + --pmu-scalar-nops N + --pmu-icache-trials N + --pmu-json FILE + +Standalone winner workload options (CCEC, AscendC, and CPU): + --winner-workload scalar-nop|real-compute + --real-compute-count N + --real-compute-counts QK,SF,PV,UP + --real-compute-pattern constant|layout-diagnostic + +real-compute is the default. Its CCEC-calibrated A5 counts are QK/SF/PV/UP=6/28/4/1; +one count is one complete 128x128 load/engine/store/completion-wait pipeline +per winner task, not a scalar NOP count. Explicit count options override those +four defaults. AscendC must be calibrated independently; CPU only preserves +the arithmetic and routing semantics and is not an A5 timing reference. +The constant pattern is the performance default. layout-diagnostic uses a +weighted diagonal A and asymmetric B to detect transpose/stride/reorder bugs. + +The swimlane action enables atomic tracing by default. For the lower-level run +action, --trace-atomics still requires swimlane tracing; add +--analyze-swimlane to print the per-role/per-site timing distributions. +--pmu-json requires --runs 1 and a non-off PMU window. For submit-pmu, a +successful raw capture also generates a self-contained HTML report beside it; +submit_icache_raw.json maps to submit_icache_report.html. PMU probe options are +CCEC-only and cannot target all. + +The submit-pmu action is a separate CCEC-only build. It fixes one PMU-only run +covering the complete Submit window. phase=none performs no internal snapshots; +phase=claim/efdrain/materialize/register reports running read-clear lower/loss-adjusted upper bounds +for one compile-time phase while CNT6/7 retain the authoritative whole-window counters. + +The swimlane action performs exactly one run and writes the raw capture, +merged Perfetto JSON, and exclusive timing analysis below this directory's +outputs/ folder. It rejects --runs, --swimlane-json, and --no-swimlane because +those are managed by the action. + +The all target always uses the requested implementation order: +CCEC, AscendC, then CPU. +EOF +} + +require_file() { + # run/smoke/swimlane 都只消费本目录 build/ 下已经生成的后端产物， + # 缺失时明确提示对应 build action，而不是临时猜测编译命令。 + if [[ ! -f "$1" ]]; then + echo "Missing build artifact: $1" >&2 + local build_prefix="" + if [[ "$3" == "shared" && "$2" == "cpu" ]]; then + build_prefix="PA_SHARED_INSERT_TURN_GROUPS=${PA_SHARED_INSERT_TURN_GROUPS:-1} " + fi + echo "Run: ${build_prefix}$0 build $2 --tensormap $3" >&2 + exit 1 + fi +} + +cpu_executable_path() { + local tensormap_mode="$1" + local variant="$2" + local binary="pa_scheduler_cpu" + local groups="${PA_SHARED_INSERT_TURN_GROUPS:-1}" + if [[ "$tensormap_mode" == "shared" && "$groups" != "1" ]]; then + binary="pa_scheduler_cpu_turn_g${groups}" + fi + printf '%s/build/cpu/%s/%s/%s\n' \ + "$SCRIPT_DIR" "$tensormap_mode" "$variant" "$binary" +} + +build_backend() { + # 后端自己的 build.sh 是唯一构建入口；all 的先后顺序由下方 BACKENDS + # 固定为 CCEC、AscendC、CPU，便于按用户要求分阶段复现。 + local backend="$1" + local tensormap_mode="$2" + case "$backend" in + ccec|ascendc|cpu) + if [[ "$backend" == "ccec" ]]; then + "$SCRIPT_DIR/ccec/build.sh" "$tensormap_mode" swimlane + else + "$SCRIPT_DIR/$backend/build.sh" "$tensormap_mode" + fi + ;; + *) + echo "Unknown backend: $1" >&2 + exit 1 + ;; + esac +} + +run_backend() { + local backend="$1" + local tensormap_mode="$2" + shift 2 + # 所有 benchmark 参数原样透传给同一套 host option parser。CCEC 额外 + # 传入本目录内的 mixed ELF，其余两个后端直接启动本地可执行文件。 + case "$backend" in + ccec) + local build_dir="$SCRIPT_DIR/build/ccec/$tensormap_mode/swimlane" + local host="$build_dir/pa_scheduler_host" + local kernel="$build_dir/pa_scheduler_kernel.o" + validate_ccec_artifacts "$tensormap_mode" swimlane none "$build_dir" + "$host" --kernel "$kernel" "$@" + ;; + ascendc) + local executable="$SCRIPT_DIR/build/ascendc/$tensormap_mode/swimlane/pa_scheduler_ascendc" + require_file "$executable" ascendc "$tensormap_mode" + "$executable" "$@" + ;; + cpu) + local executable + executable="$(cpu_executable_path "$tensormap_mode" swimlane)" + require_file "$executable" cpu "$tensormap_mode" + "$executable" "$@" + ;; + *) + echo "Unknown backend: $backend" >&2 + exit 1 + ;; + esac +} + +validate_submit_pmu_phase() { + case "$1" in + none|claim|efdrain|materialize|register) ;; + *) + echo "Unknown submit-pmu phase: $1 (expected none|claim|efdrain|materialize|register)" >&2 + exit 1 + ;; + esac +} + +ccec_artifact_failure() { + local tensormap_mode="$1" + local variant="$2" + local phase="$3" + local reason="$4" + local build_prefix="" + if [[ "$tensormap_mode" == "shared" ]]; then + build_prefix="PA_SHARED_INSERT_TURN_GROUPS=${PA_SHARED_INSERT_TURN_GROUPS:-1} " + fi + echo "Invalid CCEC artifact set for mode '$tensormap_mode', variant '$variant', phase '$phase': $reason" >&2 + if [[ "$variant" == "submit-pmu" ]]; then + echo "Run: ${build_prefix}$0 build-submit-pmu ccec $phase --tensormap $tensormap_mode" >&2 + elif [[ "$variant" == "perf-clock" ]]; then + echo "Run: ${build_prefix}$0 build-perf-clock ccec --tensormap $tensormap_mode" >&2 + else + echo "Run: ${build_prefix}$0 build ccec --tensormap $tensormap_mode" >&2 + fi + return 1 +} + +validate_ccec_artifacts() { + local tensormap_mode="$1" + local variant="$2" + local phase="$3" + local build_dir="$4" + local expected_insert_turn_groups="${PA_SHARED_INSERT_TURN_GROUPS:-1}" + case "$expected_insert_turn_groups" in + 1|2|4|8|16|32|64|128) ;; + *) + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" \ + "PA_SHARED_INSERT_TURN_GROUPS must be a power of two from 1 through 128" + return 1 + ;; + esac + local tensormap_mode_id + case "$tensormap_mode" in + private) tensormap_mode_id=0 ;; + shared) tensormap_mode_id=1 ;; + *) ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" "unsupported TensorMap mode"; return 1 ;; + esac + if [[ "$tensormap_mode" != "shared" && + "$expected_insert_turn_groups" != "1" ]]; then + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" \ + "PA_SHARED_INSERT_TURN_GROUPS only applies to shared TensorMap artifacts" + return 1 + fi + local phase_id + case "$phase" in + none) phase_id=0 ;; + claim) phase_id=1 ;; + efdrain) phase_id=2 ;; + materialize) phase_id=4 ;; + register) phase_id=5 ;; + *) ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" "unsupported phase"; return 1 ;; + esac + + local manifest_name="pa_scheduler_artifacts.manifest" + local manifest="$build_dir/$manifest_name" + local artifacts=(pa_scheduler_host pa_scheduler_kernel.o) + case "$variant" in + swimlane) + if [[ "$phase" != "none" ]]; then + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" "swimlane phase must be none" + return 1 + fi + ;; + perf-clock) + if [[ "$phase" != "none" ]]; then + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" \ + "perf-clock phase must be none" + return 1 + fi + ;; + submit-pmu) + artifacts+=(libpa_scheduler_pmu_owner_aicpu.so libpa_scheduler_pmu_owner_dispatcher.so) + ;; + *) + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" "unsupported build variant" + return 1 + ;; + esac + + # 这里独立于 build.sh 推导 consumer 期望，不能读取 manifest 再相信 + # manifest 自己。private 保留原 32B 通用区；shared 只有 full-swimlane + # 使用 16B generic record，Submit/Claim 专区始终保持 32B。 + local expected_generic_record_bytes + local expected_submit_claim_record_bytes + local expected_records_per_core + local expected_worker_stride_bytes + if [[ "$tensormap_mode" == "shared" ]]; then + expected_submit_claim_record_bytes=32 + expected_records_per_core=28416 + if [[ "$variant" == "swimlane" ]]; then + expected_generic_record_bytes=16 + expected_worker_stride_bytes=593920 + else + expected_generic_record_bytes=32 + expected_worker_stride_bytes=1048576 + fi + else + expected_generic_record_bytes=32 + expected_submit_claim_record_bytes=0 + expected_records_per_core=65536 + expected_worker_stride_bytes=2097152 + fi + + if [[ ! -x "$build_dir/${artifacts[0]}" ]]; then + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" \ + "host runner is missing, empty, or not executable" + return 1 + fi + local artifact + for artifact in "${artifacts[@]:1}"; do + if [[ ! -s "$build_dir/$artifact" ]]; then + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" \ + "artifact is missing or empty: $artifact" + return 1 + fi + done + if [[ ! -s "$manifest" ]]; then + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" \ + "ready manifest is missing or empty" + return 1 + fi + if ! command -v sha256sum >/dev/null 2>&1; then + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" "sha256sum is unavailable" + return 1 + fi + + # 十二行身份头固定 mode/CAP/insert-turn/物理泳道布局/variant/phase， + # 后续校验和行按变体精确枚举。 + # 既拒绝跨模式复用，也拒绝漏项、增项、绝对路径和重复文件。 + local manifest_lines=() + mapfile -t manifest_lines < "$manifest" + local manifest_insert_turn_groups="" + if [[ "${manifest_lines[4]:-}" == \ + "# shared_insert_turn_groups="* ]]; then + manifest_insert_turn_groups="${manifest_lines[4]#*=}" + fi + if [[ ${#manifest_lines[@]} -ne $((12 + ${#artifacts[@]})) || + "${manifest_lines[0]}" != "# schema=pa_scheduler_artifacts/v4" || + "${manifest_lines[1]}" != "# tensormap_mode=$tensormap_mode" || + "${manifest_lines[2]}" != "# tensormap_mode_id=$tensormap_mode_id" || + "${manifest_lines[3]}" != "# tensormap_ring_cap=128" || + "${manifest_lines[5]}" != "# generic_record_bytes=$expected_generic_record_bytes" || + "${manifest_lines[6]}" != "# submit_claim_record_bytes=$expected_submit_claim_record_bytes" || + "${manifest_lines[7]}" != "# records_per_core=$expected_records_per_core" || + "${manifest_lines[8]}" != "# worker_stride_bytes=$expected_worker_stride_bytes" || + "${manifest_lines[9]}" != "# variant=$variant" || + "${manifest_lines[10]}" != "# phase=$phase" || + "${manifest_lines[11]}" != "# phase_id=$phase_id" ]]; then + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" \ + "manifest schema, mode, trace layout, variant, or phase metadata does not match" + return 1 + fi + case "$manifest_insert_turn_groups" in + 1|2|4|8|16|32|64|128) ;; + *) + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" \ + "manifest shared insert-turn groups are invalid" + return 1 + ;; + esac + if [[ "$tensormap_mode" != "shared" && + "$manifest_insert_turn_groups" != "1" ]]; then + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" \ + "private artifact cannot use shared insert-turn groups" + return 1 + fi + if [[ "$manifest_insert_turn_groups" != "$expected_insert_turn_groups" ]]; then + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" \ + "manifest insert-turn groups are G=$manifest_insert_turn_groups, expected G=$expected_insert_turn_groups" + return 1 + fi + local index digest filename extra + for index in "${!artifacts[@]}"; do + digest="" + filename="" + extra="" + read -r digest filename extra <<< "${manifest_lines[index + 12]}" + if [[ ! "$digest" =~ ^[[:xdigit:]]{64}$ || + "$filename" != "${artifacts[index]}" || -n "$extra" ]]; then + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" \ + "manifest checksum entry $((index + 1)) is malformed or out of order" + return 1 + fi + done + if ! (cd "$build_dir" && sha256sum --check --strict --status "$manifest_name"); then + ccec_artifact_failure "$tensormap_mode" "$variant" "$phase" \ + "one or more artifact SHA256 values do not match" + return 1 + fi + printf '%s\n' \ + "[CHECK] CCEC artifact manifest verified: $manifest (G=$manifest_insert_turn_groups, generic=${expected_generic_record_bytes}B, submit/claim=${expected_submit_claim_record_bytes}B, records/core=$expected_records_per_core, stride=$expected_worker_stride_bytes)" +} + +reject_managed_submit_pmu_options() { + # 这些参数定义诊断 ELF 与窗口边界，必须由 action 独占；允许用户只传 + # device/batches/workload 和可选的 --pmu-json。 + for argument in "$@"; do + case "$argument" in + --kernel|--kernel=*|--runs|--runs=*|--pmu-window|--pmu-window=*|\ + --no-swimlane|--profile-phases|--trace-atomics|--analyze-swimlane|\ + --swimlane-json|--swimlane-json=*|--pmu-scalar-nops|--pmu-scalar-nops=*|\ + --pmu-icache-trials|--pmu-icache-trials=*) + echo "The submit-pmu action manages or forbids $argument." >&2 + exit 1 + ;; + esac + done +} + +run_submit_pmu() { + local tensormap_mode="$1" + local phase="$2" + shift 2 + local build_dir="$SCRIPT_DIR/build/ccec/$tensormap_mode/submit-pmu/$phase" + local host="$build_dir/pa_scheduler_host" + local kernel="$build_dir/pa_scheduler_kernel.o" + local pmu_json="" + local expect_pmu_json_value=false + local argument + # host 仍是 raw 文件的唯一写入者；这里只提取同一个路径，在 host 成功且 + # raw 已原子发布后调用独立分析器生成可视 HTML。原参数保持原样透传。 + for argument in "$@"; do + if [[ "$expect_pmu_json_value" == true ]]; then + pmu_json="$argument" + expect_pmu_json_value=false + continue + fi + case "$argument" in + --pmu-json) expect_pmu_json_value=true ;; + --pmu-json=*) pmu_json="${argument#--pmu-json=}" ;; + esac + done + validate_ccec_artifacts "$tensormap_mode" submit-pmu "$phase" "$build_dir" + "$host" --kernel "$kernel" --runs 1 --no-swimlane --pmu-window submit-all "$@" + if [[ -n "$pmu_json" ]]; then + local python_bin="${PYTHON:-python3}" + if ! command -v "$python_bin" >/dev/null 2>&1; then + echo "Python executable not found for PMU HTML report: $python_bin" >&2 + return 1 + fi + if [[ ! -f "$SCRIPT_DIR/pmu_html_report.py" ]]; then + echo "Missing local PMU HTML report generator: $SCRIPT_DIR/pmu_html_report.py" >&2 + return 1 + fi + "$python_bin" "$SCRIPT_DIR/pmu_html_report.py" "$pmu_json" + fi +} + +reject_managed_perf_clock_options() { + # perf-clock 必须保持单进程、单轮、无其他观察器。多样本由外层独立 + # 进程交错运行，不能用同一进程的热 runs 冒充独立样本。 + for argument in "$@"; do + case "$argument" in + --kernel|--kernel=*|--runs|--runs=*|--no-swimlane|\ + --profile-phases|--trace-atomics|--analyze-swimlane|\ + --swimlane-json|--swimlane-json=*|--pmu-window|--pmu-window=*|\ + --pmu-scalar-nops|--pmu-scalar-nops=*|\ + --pmu-icache-trials|--pmu-icache-trials=*|\ + --pmu-json|--pmu-json=*) + echo "The perf-clock action manages or forbids $argument." >&2 + exit 1 + ;; + esac + done +} + +run_perf_clock() { + local backend="$1" + local tensormap_mode="$2" + shift 2 + case "$backend" in + ccec) + local build_dir="$SCRIPT_DIR/build/ccec/$tensormap_mode/perf-clock" + local host="$build_dir/pa_scheduler_host" + local kernel="$build_dir/pa_scheduler_kernel.o" + validate_ccec_artifacts "$tensormap_mode" perf-clock none "$build_dir" + "$host" --kernel "$kernel" --runs 1 --no-swimlane "$@" + ;; + cpu) + local executable + executable="$(cpu_executable_path "$tensormap_mode" perf-clock)" + if [[ ! -x "$executable" ]]; then + echo "Missing CPU perf-clock artifact: $executable" >&2 + local build_prefix="" + if [[ "$tensormap_mode" == "shared" ]]; then + build_prefix="PA_SHARED_INSERT_TURN_GROUPS=${PA_SHARED_INSERT_TURN_GROUPS:-1} " + fi + echo "Run: ${build_prefix}$0 build-perf-clock cpu --tensormap $tensormap_mode" >&2 + return 1 + fi + "$executable" --runs 1 --no-swimlane "$@" + ;; + *) + echo "perf-clock supports only ccec or cpu, not '$backend'." >&2 + return 1 + ;; + esac +} + +reject_managed_swimlane_options() { + # swimlane action 必须独占轮数、raw 路径和 trace 开关，才能保证每个 + # backend 恰好对应一对 raw/merged 文件且不会发生多轮覆盖。 + for argument in "$@"; do + case "$argument" in + --runs|--swimlane-json|--no-swimlane) + echo "The swimlane action manages $argument; do not pass it explicitly." >&2 + exit 1 + ;; + esac + done +} + +reject_ccec_pmu_options_for_non_ccec() { + local backend="$1" + shift + if [[ "$backend" == "ccec" ]]; then + return + fi + + # PMU selector、校准 NOP 和导出路径只由 CCEC 分支消费；winner workload + # 已由三个后端共同解析。这里在展开 all 前拒绝 PMU，避免先启动 CCEC、 + # 再由其他后端迟到报错。 + for argument in "$@"; do + case "$argument" in + --pmu-window|--pmu-window=*|--pmu-scalar-nops|--pmu-scalar-nops=*|\ + --pmu-icache-trials|--pmu-icache-trials=*|--pmu-json|--pmu-json=*) + echo "CCEC-only option $argument is not supported by backend '$backend'." >&2 + exit 1 + ;; + esac + done +} + +consume_tensormap_option() { + TENSORMAP_MODE="private" + TENSORMAP_OPTION_SEEN=0 + TENSORMAP_REMAINING_ARGS=() + while [[ $# -gt 0 ]]; do + local value="" + case "$1" in + --tensormap) + if [[ $# -lt 2 ]]; then + echo "--tensormap requires private or shared." >&2 + exit 1 + fi + value="$2" + shift 2 + ;; + --tensormap=*) + value="${1#--tensormap=}" + shift + ;; + *) + TENSORMAP_REMAINING_ARGS+=("$1") + shift + continue + ;; + esac + if [[ "$TENSORMAP_OPTION_SEEN" -ne 0 ]]; then + echo "Specify --tensormap only once." >&2 + exit 1 + fi + case "$value" in + private|shared) TENSORMAP_MODE="$value" ;; + *) + echo "Unknown TensorMap mode: $value (expected private|shared)" >&2 + exit 1 + ;; + esac + TENSORMAP_OPTION_SEEN=1 + done +} + +validate_insert_turn_scope() { + local backend="$1" + local tensormap_mode="$2" + local groups="${PA_SHARED_INSERT_TURN_GROUPS:-1}" + case "$groups" in + 1|2|4|8|16|32|64|128) ;; + *) + echo "PA_SHARED_INSERT_TURN_GROUPS must be a power of two from 1 through 128." >&2 + exit 1 + ;; + esac + if [[ "$tensormap_mode" != "shared" && "$groups" != "1" ]]; then + echo "PA_SHARED_INSERT_TURN_GROUPS only applies to shared TensorMap." >&2 + exit 1 + fi + if [[ "$groups" != "1" && + "$backend" != "ccec" && "$backend" != "cpu" ]]; then + echo "turn-G$groups is implemented only for CPU and CCEC; backend '$backend' is outside this standalone target." >&2 + exit 1 + fi +} + +# 顶层先解释 action/backend；run 的其余参数交给共享 parser，build、smoke 和 +# swimlane 再分别处理自己的约束或默认注入项。参数不足会在创建目录前失败。 +if [[ $# -lt 2 ]]; then + usage >&2 + exit 1 +fi + +ACTION="$1" +BACKEND="$2" +shift 2 +consume_tensormap_option "$@" +set -- "${TENSORMAP_REMAINING_ARGS[@]}" + +if [[ "$BACKEND" == "all" ]]; then + # 该顺序也是组合构建、运行和泳道采集的稳定对外约定。 + BACKENDS=(ccec ascendc cpu) +else + BACKENDS=("$BACKEND") +fi + +# 后端约束必须早于 build/run/smoke/swimlane 的任何文件创建、构建或设备动作。 +validate_insert_turn_scope "$BACKEND" "$TENSORMAP_MODE" +reject_ccec_pmu_options_for_non_ccec "$BACKEND" "$@" + +case "$ACTION" in + build) + # build 只选择后端，不接收 benchmark 参数；这样编译配置不会被运行时 + # 选项暗中改变，三种实现的构建命令也保持可独立复现。 + if [[ $# -ne 0 ]]; then + echo "The build action does not accept benchmark options." >&2 + exit 1 + fi + for backend in "${BACKENDS[@]}"; do + build_backend "$backend" "$TENSORMAP_MODE" + done + ;; + run) + # run 不替用户补默认覆盖项，完整参数校验交给各后端共享的 Options parser。 + for backend in "${BACKENDS[@]}"; do + run_backend "$backend" "$TENSORMAP_MODE" "$@" + done + ;; + smoke) + # smoke 仍启动全部 96 个 worker，并显式注入 1 batch、1 run、scalar-nop=0； + # 后置用户参数仍由共享 parser 处理。它用于快速检查原子协议、拓扑和 + # 最终状态，不作为性能数据。 + for argument in "$@"; do + case "$argument" in + --batches|--batches=*|--runs|--runs=*|--nop-count|--nop-count=*|\ + --nop-counts|--nop-counts=*|--winner-workload|--winner-workload=*|\ + --real-compute-count|--real-compute-count=*|--real-compute-counts|\ + --real-compute-counts=*|--real-compute-pattern|--real-compute-pattern=*) + echo "The smoke action fixes b1/r1/scalar-nop=0; use the run action for real-compute." >&2 + exit 1 + ;; + esac + done + for backend in "${BACKENDS[@]}"; do + run_backend "$backend" "$TENSORMAP_MODE" --batches 1 --runs 1 \ + --winner-workload scalar-nop --nop-count 0 "$@" + done + ;; + swimlane) + # swimlane 是“采集 + Perfetto 转换 + 排他闭合分析”的流水线边界： + # runner 失败则不做后处理；任一后处理失败都非零退出，但保留此前 + # 已原子发布的完整文件，便于定位失败边界。 + reject_managed_swimlane_options "$@" + # 仅需要 Python 标准库；允许用户用 PYTHON 指向自己的虚拟环境， + # 但转换和排他分析脚本始终取自当前 pa_scheduler 目录。 + PYTHON_BIN="${PYTHON:-python3}" + if ! command -v "$PYTHON_BIN" >/dev/null 2>&1; then + echo "Python executable not found: $PYTHON_BIN" >&2 + exit 1 + fi + if [[ ! -f "$SCRIPT_DIR/swimlane_converter.py" ]]; then + echo "Missing local converter: $SCRIPT_DIR/swimlane_converter.py" >&2 + exit 1 + fi + if [[ ! -f "$SCRIPT_DIR/swimlane_exclusive_analyzer.py" ]]; then + echo "Missing local analyzer: $SCRIPT_DIR/swimlane_exclusive_analyzer.py" >&2 + exit 1 + fi + # UTC 秒级时间加当前 shell PID 避免并行采集目录冲突；所有产物保持 + # 在本目录 outputs/ 下，复制 pa_scheduler 后仍可原样工作。 + OUTPUT_ROOT="$SCRIPT_DIR/outputs/pa_scheduler_${TENSORMAP_MODE}_swimlane_$(date -u +%Y%m%d_%H%M%S)_$$" + mkdir -p "$OUTPUT_ROOT" + # all 模式下每个 backend 使用独立子目录，避免同名 raw/merged 互相覆盖； + # 某一后端失败后 set -e 停止，之前已完成后端的产物仍可单独检查。 + for backend in "${BACKENDS[@]}"; do + BACKEND_OUTPUT="$OUTPUT_ROOT/$backend" + RAW_JSON="$BACKEND_OUTPUT/l2_swimlane_records.json" + MERGED_JSON="$BACKEND_OUTPUT/merged_swimlane.json" + EXCLUSIVE_JSON="$BACKEND_OUTPUT/swimlane_exclusive_analysis.json" + mkdir -p "$BACKEND_OUTPUT" + # runner 先执行单轮严格语义校验并流式写 raw；成功后才依次调用 + # converter 和 analyzer。set -e 保证任一步失败即停止。 + # 用户要求泳道默认带齐逐 atomic 性能。重复传入 --trace-atomics + # 只是幂等布尔开关，不会产生两份记录；直接 run 仍可选择 phase-only。 + run_backend "$backend" "$TENSORMAP_MODE" \ + --runs 1 --trace-atomics --swimlane-json "$RAW_JSON" "$@" + "$PYTHON_BIN" "$SCRIPT_DIR/swimlane_converter.py" "$RAW_JSON" -o "$MERGED_JSON" + "$PYTHON_BIN" "$SCRIPT_DIR/swimlane_exclusive_analyzer.py" \ + "$RAW_JSON" -o "$EXCLUSIVE_JSON" + done + echo "[SWIMLANE] output_root=$OUTPUT_ROOT" + ;; + build-perf-clock) + if [[ "$BACKEND" != "ccec" && "$BACKEND" != "cpu" ]] || [[ $# -ne 0 ]]; then + echo "Usage: $0 build-perf-clock ccec|cpu [--tensormap private|shared]" >&2 + exit 1 + fi + "$SCRIPT_DIR/$BACKEND/build.sh" "$TENSORMAP_MODE" perf-clock + ;; + perf-clock) + if [[ "$BACKEND" != "ccec" && "$BACKEND" != "cpu" ]]; then + echo "Usage: $0 perf-clock ccec|cpu [--tensormap private|shared] [benchmark options]" >&2 + exit 1 + fi + reject_managed_perf_clock_options "$@" + run_perf_clock "$BACKEND" "$TENSORMAP_MODE" "$@" + ;; + build-submit-pmu) + if [[ "$BACKEND" != "ccec" || $# -ne 1 ]]; then + echo "Usage: $0 build-submit-pmu ccec none|claim|efdrain|materialize|register" >&2 + exit 1 + fi + PHASE="$1" + validate_submit_pmu_phase "$PHASE" + "$SCRIPT_DIR/ccec/build.sh" "$TENSORMAP_MODE" submit-pmu "$PHASE" + ;; + submit-pmu) + if [[ "$BACKEND" != "ccec" || $# -lt 1 ]]; then + echo "Usage: $0 submit-pmu ccec none|claim|efdrain|materialize|register [benchmark options]" >&2 + exit 1 + fi + PHASE="$1" + shift + validate_submit_pmu_phase "$PHASE" + reject_managed_submit_pmu_options "$@" + run_submit_pmu "$TENSORMAP_MODE" "$PHASE" "$@" + ;; + build-shared-protocol-litmus) + # shared protocol litmus 是独立的 shared-only A5 正确性门槛， + # 不生成普通 PA benchmark 变体，也不接受会误导产物身份的 + # TensorMap 选项。 + if [[ "$BACKEND" != "ccec" || $# -ne 0 || + "$TENSORMAP_OPTION_SEEN" -ne 0 ]]; then + echo "Usage: $0 build-shared-protocol-litmus ccec" >&2 + exit 1 + fi + "$SCRIPT_DIR/ccec/shared_protocol_litmus.sh" build + ;; + shared-protocol-litmus) + if [[ "$BACKEND" != "ccec" || + "$TENSORMAP_OPTION_SEEN" -ne 0 ]]; then + echo "Usage: $0 shared-protocol-litmus ccec --scenario history|reader-reclaim|all [--ordering compiler-clobber|payload-dependency|dsb-all|all] [--device N] [--runs N]" >&2 + exit 1 + fi + "$SCRIPT_DIR/ccec/shared_protocol_litmus.sh" run "$@" + ;; + *) + # 未知 action 不尝试推断用户意图，也不会触发任何构建或设备操作。 + echo "Unknown action: $ACTION" >&2 + usage >&2 + exit 1 + ;; +esac diff --git a/tests/atomic_probe/pa_scheduler/shared_map_ordered_single_writer_protocol.md b/tests/atomic_probe/pa_scheduler/shared_map_ordered_single_writer_protocol.md new file mode 100644 index 0000000000..ea152426ac --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/shared_map_ordered_single_writer_protocol.md @@ -0,0 +1,1347 @@ +# FDWIC Shared Map 候选方案：有序单写、多读流水 + +> 类型与状态：交互式维护的单一候选方案规格，不是当前实现说明或 +> Shared Map 通用协议。本文只把已经对齐的方案内规则写成确定项； +> 尚待实现验证收敛的内容列在第 6.7 节，不视为已完成实现。 + +本文只记录“有序单写、多读流水”这一具体候选方案。相关内容按以下 +文档边界归档： + +- 当前实现事实和 region-intent 时序放在 + [`shared_map_current_execution_model.md`][current-model]； +- INOUT 发布问题和多个候选方案的比较放在 + [`shared_tensormap_inout_publication_protocol.md`][protocol-comparison]； +- 本文只描述有序单写、多读流水方案的前提、性质和实现约束。 + +后文出现的 `H(N) -> I(N) -> L(N) -> B(N)`、`producer_task_id < N`、 +后续 insert 越过当前 lookup 等结论，都只在本文定义的方案前提下成立。 +它们不能直接用于描述当前 Shared Map，也不能直接套用到 per-core +overlay、版本化句柄或其他发布方案。 + +## 1. 候选方案前提 + +本文方案建立在以下前提上： + +1. 每个 task 都执行一次有序 heap commit 和 insert commit；对应阶段没有 + 实际工作时也执行 empty commit； +2. `next_heap_task_id` 和 `next_insert_task_id` 分别串行化 H、I，二者是 + 相互独立的原子有序游标； +3. `I(N)` 位于本 task 的 `H(N)` 之后；对 task N 执行 lookup 前，所有 + `I(K), K < N` 都已经完成； +4. 原子变量 `next_insert_task_id` 是 TensorMap 唯一的有序写锁； + lookup 不获取或修改该锁，但必须位于一次有效的 acquire-observe 之后； +5. `next_insert_task_id` 同时提供 writer 互斥、task 顺序和 insert + commit/entry 可见性发布，不再设置独立的 free/busy 锁； +6. winner N 在同一条控制流中连续执行 heap、insert、lookup 和 build； +7. lookup 按 consumer task id 查询历史视图，而不是读取物理最新 + writer。 + +这些前提不是当前实现事实；改变任一前提时，必须重新检查本文的全部 +推导结论。 + +## 2. 方案内术语和阶段 + +对 task N 定义： + +- `calc_output_layout(N)`：winner-local 准备步骤，在 H 前根据 OUTPUT + `TensorCreateInfo` 计算 output index、size、alignment/padding 和 task + 总 logical heap 需求；它不访问任何 H/I 共享状态； +- `H(N)`：使用预先算好的 output layout，从现有的单调递增 + `shared_heap_vend` 预留逻辑地址区间，并初始化 shared logical Tensor + descriptor；H 不写物理 output buffer，没有新建 OUTPUT 时执行 empty + commit； +- ref 准备：H commit 后、I 锁外解析所有 `FdwicOutputRef` 及其多维 view + metadata，再从完整 logical args 统计准确 `writer_count`； +- `I(N)`：对普通 kernel task N 的全部 writer 调用 TensorMap insert，并 + 完成有序 task commit；没有 writer 时执行 empty commit； +- `L(N)`：task N 查询 input/inout 前驱； +- `B(N)`：按既有 heap 容量/回收策略等待 task N 的预留区间可用；普通 + kernel task 把 shared logical Tensor copy 到 slot 并设置物理地址，再 + 完成 task build；`alloc_tensors` 执行不创建 kernel slot 的轻量 build； +- `E(N)`：task N fanin ready 后执行。 + +本文只使用 TensorMap entry 这一名称，不为它引入其他别名。entry 保存在 +TensorMap 自己拥有的共享 GM entry pool 中；它不引用临时 args/private +slot，也不存放在 `SharedOutputCell` 或 logical/physical heap 中。 + +执行阶段沿用以下名称： + +- heap 准备态：Claim winner 后执行 `H(N)`； +- 插入态：Claim winner 后执行 `I(N)`； +- 构建态：同一个 winner 继续执行 `L(N)` 和 `B(N)`； +- 运行态：检查 fanin，并在 ready 后执行 `E(N)`。 + +这些阶段名只区分同步范围，不表示独立工作项、队列或所有权移交。 +在正常的无故障执行假设下，同一个 winner 连续负责： + +```text +普通 kernel task: + calc_output_layout(N) -> H(N) -> resolve refs/count writers + -> I(N) -> L(N) -> B(N) + +alloc_tensors: + calc_output_layout(N) -> H(N) -> I(N) empty commit -> B_alloc(N) +``` + +loser 不执行这些阶段，也不构造完整 args；它只返回 symbolic output refs +并立即继续 replay。 + +对普通 kernel task 和 `alloc_tensors`，wrong-role 核不属于该 task 的 +候选核，因此不参与 Claim，也不执行当前 task 的 H/I/L/B。它可以在跳过 +当前 task 前 drain 本核已有的 runnable RingSlot 或已经赢得的旧工作， +以帮助系统推进；随后只根据 `task_id` 和 output 序号生成 symbolic +output refs，并继续 replay。只有 role 匹配的候选核参与 Claim，唯一 +winner 负责推进上述阶段。MIX follower 不是这种普通 wrong-role 路径， +而是按下述现有 joint/WonSlot 机制接收并执行同 block subtask。 + +MIX 的共同所有权仍采用固定 block 配对：只有 AIC 候选核参与 MIX 的 +Claim；唯一 AIC winner 同时选定该 MIX 的获胜 block，该 block 内 active +mask 对应的 AIV 核与 winner 共同执行其余 subtasks。这些 AIV 核没有参与 +Claim，也不是 Claim loser 或普通 wrong-role；它们的 subtask 所有权由 +同 block AIC winner 派生。subtask 继续通过现有 joint/WonSlot 机制从 +AIC winner 交给这些核，不为本方案重新设计一套 follower handoff。 + +MIX 的 H/I/L 是 task-level 工作，只由 AIC winner 各执行一次。H 一次 +初始化该 task 的全部 shared logical outputs；I 一次插入该 task 的全部 +TensorMap entries，并只推进一次 `next_insert_task_id`；L 一次收集整个 +MIX task 的 fanin。同 block AIV 核不重复执行 H/I/L，也不推进对应阶段 +游标；其 subtasks 后续复用 AIC winner 生成的 logical Tensors。 +MIX 不设置独立的 heap gate：AIC winner 直接复用普通 winner 的 B，在 +task-level heap 容量门通过后完成物理映射、填充 WonSlot，并构建自己的 +privateSlot；AIV 不重复执行 heap 容量检查。 + +现有 joint/WonSlot 生命周期保持不变：AIC build 时先填充但不立即发布 +WonSlot；AIC privateSlot 达到 ready 时发布 WonSlot 并执行 AIC subtask； +对应 AIV 通过 `drain_block_won()` 把自己的 subtask 搬入 privateSlot， +再按既有 privateSlot ready/execute 流程执行。原始 fanin 只保存在 AIC +privateSlot 并由 AIC 检查；AIV subtask 的 `fanin_count = 0`，AIC +ready 后对 WonSlot 的发布为 AIV 提供传递的依赖准入。该机制只需要适配 B +产出的新 logical-to-physical Tensor copy，不改变所有权、slot 状态机或 +完成计数。 + +MIX 的 WonSlot/privateSlot 推进仍采用协作式 polling，不增加中断或后台 +调度线程。每个核在以下三类检查点依次调用 +`drain_block_won_if_enabled()` 和现有 `drain_phase_b()`： + +1. 每个 replay task 的 presubmit/Claim 前；shared 路径可先用 + `dist_submit_has_drain_work()` 跳过确定无工作的调用； +2. H/I、privateSlot、WonSlot 和 heap 容量等所有阻塞等待循环的每轮； +3. replay 结束后的 drain-to-completion 循环，直到本核 privateSlot、 + 待接收 WonSlot 和 joint launch 计数全部收敛。 + +AIC winner build 完成后不额外立即执行一次 drain；其 task 在该核到达 +下一个上述检查点时进入 ready 检查。AIC 的执行 drain 发布已 ready 的 +WonSlot；对应 AIV 在自己的下一个检查点先把 WonSlot subtask 搬入 +privateSlot，再由同一检查点紧随其后的执行 drain 检查该 slot。 + +现有函数名 `drain_phase_b()` 沿用旧阶段命名，其实际行为是扫描并执行 +fanin-ready privateSlots，语义上属于本文的 E 进展，不是本文负责 heap +反压、物理映射和 build 的 `B(N)`。旧 region-intent、per-output +published wait 和 heap-shard wait 删除后，其专属 drain 调用点也随之 +删除；新 H/I/B 等待循环必须保留上述协作推进。 + +## 3. 本方案内已确认的并发规则 + +### 3.1 H、I 使用两个独立的有序游标 + +两个阶段分别按 task 顺序提交： + +```text +H(N): + wait next_heap_task_id == N + -> 分配 task N 的单调逻辑 heap 区间并初始化 shared logical Tensors + -> publish next_heap_task_id = N+1 + +I(N): + 本核已经完成 H(N) + -> wait next_insert_task_id == N + -> 插入 task N 的全部 TensorMap entries；无 entry 时不修改 TensorMap + -> publish next_insert_task_id = N+1 +``` + +`next_heap_task_id` 只约束 H 之间的顺序，`next_insert_task_id` 只约束 +I 之间的顺序。就跨 task 的准入条件而言，`H(N)` 只等待 `H(N-1)`， +`I(N)` 只等待 `I(N-1)`；`H(N) -> I(N)` 是同一个 winner 内部的 +task 内依赖。 + +`alloc_tensors` 即使含 OUTPUT，`I(N)` 也始终是 empty commit：它只原子 +推进 `next_insert_task_id`，不分配 entry、不更新 bucket head。 + +Claim 已经保证每个 task 只有一个 winner。当 `next_heap_task_id == N` +时，只有 winner N 能修改有序 heap 状态；当 +`next_insert_task_id == N` 时，只有 winner N 能修改 TensorMap。两个 +阶段游标因而分别兼任对应阶段的 task 顺序、互斥、commit/可见性发布和 +下一次权限交接，不再需要二值 free/busy 锁。 + +`next_heap_task_id` 是阶段发布游标，不等同于以 byte 为单位的 heap +分配位置。现有 `shared_heap_vend` 是 H 唯一的、只增不减的逻辑 heap +分配状态；现有 `heap_base` 仍只是固定的物理 GM heap 基址。候选方案删除 +`shared_heap_cursor[]`、heap shard 选择和 shard 内 wrap 逻辑。 + +worker 启动前，control plane 初始化 +`next_heap_task_id = 0`、`next_insert_task_id = 0`、 +`shared_heap_vend = 0`、`entry_high_water = 0`，并把全部 bucket head +初始化为 `-1`。per-task vend 初始化为 0。上述状态只在首次 worker +启动前，或确认不存在任何 H/I/L/B/E 访问的全局静默边界整体 reset； +运行中的单个 task 不得局部重置阶段游标或 allocation cursor。 + +由于 I 在全局范围内只有一个 writer，TensorMap 内部不再使用 +writer-writer 原子竞争协议：entry allocation cursor 普通递增；删除 +atomic fetch-add、`-2` bucket lock、CAS/RMW 重试和全局 insert lock。 +`bucket.head` 仍使用硬件 atomic 读写，但它只作为单字的内存级读取和 +发布原语，不再表示 bucket 被锁定，也不存在抢占或重试。entry 内容的 +可见性由 entry flush、atomic head 发布、task-level insert cursor +发布以及 reader invalidate 共同建立。 + +### 3.2 TensorMap insert 直接使用 args + +本方案中普通 kernel task 的 winner N 目标流程为： + +```text +Claim winner N + -> calc_output_layout(N) + -> 等待 next_heap_task_id == N + -> 根据预计算 layout 从 shared_heap_vend 预留 task_base/task_vend + -> 初始化 shared logical Tensor 的 storage identity 和完整几何 + -> 发布 next_heap_task_id = N+1，提交 H(N) + -> invalidate 并解析全部 FdwicOutputRef 及其多维 view metadata + -> 遍历完整 logical args，统计 I(N) 实际需要插入的 writer_count + -> 等待 next_insert_task_id == N + -> 独占 task N 的 TensorMap 写权限 + -> 按 writer_count 预检并预留连续 entry slots + -> OUTPUT: TensorMap::insert(shared logical Tensor, N) + -> INOUT/OUTPUT_EXISTING: TensorMap::insert(Tensor, N) + -> 发布 next_insert_task_id = N+1，提交 I(N) + -> lookup task N 的前驱 + -> 等待 task N 的 heap 预留区间已经可以安全复用 + -> copy logical Tensor，设置 physical buffer address + -> build task N +``` + +args 为 `OUTPUT` 提供 `TensorCreateInfo`；H 使用预计算 layout 和 +create info 初始化 shared logical Tensor，并补充单调逻辑地址。直接 +Tensor 参数已经提供 descriptor；shared OUTPUT 的 +`INPUT/INOUT/OUTPUT_EXISTING` 参数则在 H commit 后从 +`SharedOutputCell::tensors` 解析为 winner-local logical Tensor。 + +准确 `writer_count` 必须在这些 ref 解析后统计，因为解析前无法读取源 +Tensor 的 `manual_dep`。统计仍在 I 锁外，并通过 winner-local 状态传给 +I；不需要 shared 字段或单独发布。I 不构造 normalized delta，也不要求 +调用方构造额外中间对象。 +`TensorMap::insert` 直接读取 Tensor。首版仍从 Tensor 计算并保存临时 +byte range `[lo, hi)`；真实高维求交需要的完整几何字段和算法留待后续 +启用。 + +`manual_dep` 只关闭自动 TensorMap 路径。H 仍为其新 OUTPUT 完成 logical +heap reservation 和 descriptor 初始化；H 后的准确 `writer_count` 统计 +排除所有 `manual_dep` Tensor，I 不为其插入 entry。L 仍收集显式依赖和 +有效的 owner/creator 依赖，只跳过该 Tensor 的 TensorMap lookup。 + +TensorMap slot 分配、entry 初始化、bucket/history link 更新和 entry +publication 全部位于 I。B 只能修改 copy 到 private slot/payload 的 +执行 Tensor，不能修改 H 已发布并可能正被 lookup 读取的 shared logical +Tensor。任何 output buffer 写入（包括 initial-value fill）必须位于 B +的 heap 容量门之后。 + +`alloc_tensors` 使用同一 H 顺序初始化 logical Tensors，但 +`writer_count = 0`，其 I 只执行 empty commit： + +```text +alloc winner N + -> H(N) + -> 等待 next_insert_task_id == N + -> I(N) empty commit：atomic publish next_insert_task_id = N+1 + -> B_alloc(N)：heap 反压、物理映射、initial fill + -> publish task completion flag +``` + +`B_alloc` 是轻量 build，不执行 TensorMap lookup，不创建 kernel +RingSlot，也没有 E 阶段。它必须位于 I empty commit 之后，使 alloc 的 +heap 等待不会占住 insert 前缀、阻塞后续 task 的 I。 +alloc OUTPUT 是新 storage，其 creator/completion 依赖由 +`SharedOutputCell::tensors` 和 owner task id 表达；后续对该 storage 的 +INOUT/OUTPUT_EXISTING 才向 TensorMap 追加 entry。 + +候选方案保留 `SharedOutputCell::tensors`，供 +`FdwicOutputRef{producer_task_id, output_slot}` 直接取得 logical Tensor； +该表不通过 TensorMap 间接定位。SharedOutputCell 只保留 tensors，删除 +per-output `published` 和 `last_writer`。H 完整初始化 task N 的全部 +tensors 后，由 `next_heap_task_id = N+1` 一次性完成 task-level 发布。 +I 和后续 lookup 再以这些 Tensor 的 `logical_addr` 访问 writer history。 + +`FdwicOutputRef` 必须完整表达 1 至 `MAX_TENSOR_DIMS` 维的延迟 view +metadata，不能保留当前只支持一维的 `view_shape0/view_offset0` 特例： + +```cpp +struct FdwicOutputRef { + int32_t producer_task_id; + int16_t output_slot; + uint8_t flags; + uint8_t view_ndims; + uint32_t view_shapes[MAX_TENSOR_DIMS]; + uint32_t view_offsets[MAX_TENSOR_DIMS]; +}; +static_assert(sizeof(FdwicOutputRef) == 48); +``` + +无 view 时 view flag 不设置，两个数组不参与语义。第一次 `view()` 保存 +全部维度的 shape/offset；嵌套 view 要求维数一致，并逐维检查 +`new_offset[d] + new_shape[d] <= old_shape[d]`，然后执行： + +```text +view_offsets[d] += new_offset[d] +view_shapes[d] = new_shape[d] +``` + +解析时要求 `1 <= view_ndims <= MAX_TENSOR_DIMS` 且 +`view_ndims == source.ndims`，再使用完整数组调用多维 `Tensor::view`。 +实现删除全部 `ndims == 1` 断言和 `shape0/offset0` 分支，并增加二维、 +五维和嵌套 view 覆盖。这里的“延迟”只表示 view metadata 先随 +`FdwicOutputRef` 保存，之后再应用到 H 已发布的 logical Tensor；不引入 +另一种 Tensor 或 view 类型。 + +所有 ref 都在 H commit 后、I 前解析为 winner-local logical Tensor， +因此 unresolved ref 不再进入 B、RingSlot 或 WonSlot。本方案删除 +`RingSlot` 和 `BuiltSubtask` 中的 `shared_ref_mask/shared_refs[]`，同时 +删除 `dist_resolve_slot_shared_refs()`、执行前 shared-ref wait/resolve +以及对应 copy/flush/invalidate 分支。MIX AIC winner 在 B 中把已经解析并 +完成物理映射的 Tensor 写入 WonSlot，AIV 只复制 Tensor。48B +`FdwicOutputRef` 只存在于 replay/args 阶段，不占用常驻 privateSlot 或 +WonSlot payload。 + +Tensor 使用第二条 cache line 中 `strides[]` 后现有的 36B reserved 区域 +保存一个稳定 logical address。该字段使用公共 8B `MaskPointer` 值类型， +由它统一封装地址低 3 位的附加信息： + +```text +offset 92: uint8 alignment_pad[4] +offset 96: MaskPointer logical_addr +offset 104: uint8 reserved[24] +``` + +`MaskPointer::ptr()` 清除低 3 位并返回纯地址部分， +`MaskPointer::uint64()` 返回包含所有 tag 的完整 64-bit 值； +`active()` 检查指定低位。Tensor logical address 首版定义： + +```cpp +enum class TensorLogicalAddrInfo : uint8_t { + IsTensorGM = 0, +}; +``` + +这里的 `IsTensorGM` 特指由本 runtime GM ring heap 管理、需要在 B 中 +映射的 Tensor，不泛指所有位于 GM 的 external Tensor。其余两个 tag bit +首版保留。 + +external Tensor 保留已有的物理 `buffer.addr`，并以 tag 全零的 +`buffer.addr` 初始化 `logical_addr`；由于 `MaskPointer` 占用低 3 位， +external buffer base 必须至少 8B 对齐，并在创建入口检查。B 遇到 +`IsTensorGM == false` 时不修改该 Tensor copy 的 `buffer.addr`，不需要 +通过 `MaskPointer::ptr()` 恢复 external 地址。 + +新建 OUTPUT 使用 H 从 `shared_heap_vend` 分配的、至少 8B 对齐的单调 +logical byte address，并设置 `IsTensorGM`。H 发布的 logical Tensor 将 +`buffer.addr` 明确置为 0，只保留 `buffer.size`、完整几何和 +`logical_addr`；B 之前不得进行数据访问。H 从进入阶段时的 +`shared_heap_vend` 计算 task_base，把各 OUTPUT 放入该 task 的逻辑区间， +并得到包含 alignment 和 wrap padding 的 task_vend。 +`shared_heap_vend` 和 `DistTaskCell::vend` 始终保存不含 tag 的原始 byte +cursor。 + +H 为 task N 新建的每个 internal OUTPUT 设置 +`owner_task_id = N`。`FdwicOutputRef` 解析、Tensor copy 和 view 原样保留 +该 owner；它们不能把 owner 改为当前 consumer。external Tensor 继续由 +创建入口设置 invalid owner。L 使用该既有 owner/creator 关系形成依赖， +它与 TensorMap 的 writer history 是两条独立来源，最终统一去重。 + +TensorMap 的 hash 和 identity 比较使用 `logical_addr.uint64()`，使 +internal/external namespace tag 参与 identity。所有 copy 和 view 类操作 +必须原样传播完整 `MaskPointer`；view 只修改 offset、shape 和 stride。 + +本方案要求 `heap_size` 是 2 的幂，因此 B 使用 +`reinterpret_cast(logical_addr.ptr()) & (heap_size - 1)` 得到 +internal Tensor 的物理 offset，并只在 private Tensor copy 中写入 +`buffer.addr = heap_base + physical_offset`。H 必须把 ring 尾部 padding +计入 `shared_heap_vend`，保证每个要求连续存储的 OUTPUT 都满足 +`physical_offset + buffer_size <= heap_size`，不能跨物理 heap 尾部。 + +只要 `I(N)` 是一次按 task 顺序完成的提交，它完成时就意味着所有 +`I(K), K < N` 已经完成。由于 `L(N)` 紧随其后，winner N 不需要再等待 +一个独立的全局发布前缀。 + +### 3.3 每个 task 都推进两个阶段游标 + +每个 task 的 winner 都必须完成一次 H commit 和 I commit： + +```text +H(N): 有新建 OUTPUT 则预留逻辑 heap 并发布 logical Tensors，否则 empty commit + -> next_heap_task_id = N+1 +I(N): 普通 kernel task 有需跟踪的 writer 则发布全部 entries，否则 empty commit + alloc_tensors 始终执行 empty commit + -> next_insert_task_id = N+1 +``` + +否则 N+1 无法判断 N 已经经过相应阶段，流水会停止。“是否有实际阶段 +工作”和“是否推进阶段游标”是两个不同问题。 + +H 的 empty commit 不更新 `shared_heap_vend`，也不 flush +`SharedOutputCell::tensors`，但仍把当前 vend 写入 +`task_cell(N).vend`，并原子推进 `next_heap_task_id`。 + +### 3.4 `I(N)` 是整个 task 的提交点 + +一个 task 有多个 output/inout 时，winner N 在同一次写权限持有期内 +发布全部 TensorMap entries： + +```text +等待 next_insert_task_id == N + -> 预留连续 entry slots + -> 初始化 task N 的全部 entries 和 bucket links + -> 批量 flush 连续 entry 区间 + dsb + -> atomic publish 各受影响 bucket 的新 head + -> store_barrier() + -> commit I(N) + -> atomic publish next_insert_task_id = N+1 +``` + +不得在 task N 的部分 entries 发布后提前推进 +`next_insert_task_id`，也不得在同一个 task 的多个 entries 之间 +把写权限交给其他 task。 + +所有 TensorMap entries 必须先完成全部字段初始化，再执行一次覆盖该连续 +slot 区间的批量 flush。只有 flush 和 dsb 完成后，才允许发布任何受影响 +bucket 的新 head。entry 一旦发布就保持不可变，任何 reader 都不能观察到 +半初始化 entry。 + +N 的全部 entry 写入和 entry publication 都发生在 +`next_insert_task_id = N+1` 之前。该原子变量的发布是 task-level +commit：后续 task 只有 acquire-observe 到大于 N 的值后，才允许读取并 +使用 task N 的 entries。 + +task N 的多个 bucket heads 不要求同时变得可见，可以在写临界区内逐个 +发布；但此时它们可达的 task N entries 已经全部完成批量 flush。较早 +lookup 可能通过部分已更新的 heads 观察到 task N 或 N+1 的完整 entries， +但会根据 producer task id 将它们排除，因此不会使用尚未完成 task-level +commit 的 writer。 + +没有 TensorMap entry 的 empty commit（包括所有 `alloc_tensors`）和包含 +多个 entries 的 commit 使用相同的 task 级边界。区别只在于提交前是否 +需要修改 TensorMap 内容。 + +### 3.5 后续 insert 可以越过当前 lookup + +两个独立游标形成 task 间流水。`H(N-1)` 提交后，winner N 可以执行 +`H(N)`，同时 winner N-1 执行 `I(N-1)`： + +```text +Core A, winner N-1: + H(N-1) ---- I(N-1) ---- L(N-1) ---- B(N-1) + +Core B, winner N: + H(N) ------ I(N) ------ L(N) ------ B(N) +``` + +其必要偏序只有： + +```text +H(N-1) -> H(N) +I(N-1) -> I(N) +H(N) -> I(N) +``` + +因此 `H(N)` 可与 `I(N-1)` 并发；`I(N+1)` 也可与 `L(N)`/`B(N)` +并发。协议不要求 `L(N) -> I(N+1)`，这区别于把所有阶段放进 global +exact-turn 临界区的方案。 + +loser 即使立即继续 replay 并赢得 N+1，也必须在 H(N+1) 前等待 +`next_heap_task_id == N+1`，并在 I(N+1) 前等待 +`next_insert_task_id == N+1`。因此 `L(N+1)` 不可能越过 I(N)，无需再让 +task N 的 loser 等待 winner 发布 writer intent。 + +候选方案据此删除 region-intent 分支、`deps_prepared`/`PreparedDeps`、 +对应 token bits 和 shared-ref `last_writer` exchange。所有 task 统一为 +Claim 后 winner-only 构造完整 args；loser 只可操作 symbolic ref/view， +不得在 H/I 提交前读取 output Tensor metadata 或物理数据。H、I 和 heap +容量等待循环必须持续 drain 本核已有执行任务，保证阻塞期间仍可推进。 + +heap 容量和回收沿用既有策略：H 把本 task 的 task_vend 原子写入现有 +`DistTaskCell::vend`；B 使用该 task-local vend 检查回收前沿，容量不足 +时等待并协作推进执行。不能直接使用已经被未来 H 推进的全局 +`shared_heap_vend`，否则未来 reservation 可能反过来阻塞本可执行的较早 +task。该 B 反压会逐核停止 replay，因此不再规定独立的 H 相对 I/exec +最大超前量。 + +每个 task 都必须写自己的 `DistTaskCell::vend`。没有新 OUTPUT 的 task +继承当前 `shared_heap_vend`，保证 frontier 跨过连续无输出 task 时，回收 +水位仍连续。shared completion 只发布完成 flag，不再用可能已被后续 H +推进的 `DistCore::heap_next` 覆盖该 per-task vend；`heap_next` 在 +non-shared 路径中的既有用途不受影响。 + +B 对含新 OUTPUT 的 task N 使用以下反压条件： + +```text +task_vend = atomic_load(task_cell(N).vend) +loop: + F = atomic_load(frontier) + R = F - H + reclaimed_vend = (R < 0) ? 0 : atomic_load(task_cell(R).vend) + if task_vend - reclaimed_vend <= heap_size: + 允许物理映射并继续 build + + // 只在首次判断容量不足后按需推进完成前缀 + advance_frontier_until(N - 1, bounded_steps) + F = atomic_load(frontier) + R = F - H + reclaimed_vend = (R < 0) ? 0 : atomic_load(task_cell(R).vend) + if task_vend - reclaimed_vend <= heap_size: + 允许物理映射并继续 build + if F >= N - 1: + 设置 fatal：全部较早 task 已完成时该 task 仍无法放入 heap + + drain 本核已有执行任务并继续等待 +``` + +没有新 OUTPUT 的 task 跳过该容量检查。公式中的 `task_vend` 固定属于 +task N，不能在循环中改读全局 `shared_heap_vend`。所有共享标量读写仍 +通过公共 atomic 封装。普通 task completion 和 MIX 最后一个 subtask +completion 只发布 task flag，不在完成热路径调用 `advance_frontier()`； +frontier 仅在 B/B_alloc 实际发现容量不足后按上述方式懒推进。 + +`B_alloc(N)` 使用同一反压公式。容量可用后，它把 logical address 映射到 +物理 heap，并对临时 Tensor 副本执行所需的 initial-value fill；不能修改 +H 已发布的 `SharedOutputCell::tensors`。即使没有 initial fill，只要 alloc +包含新 OUTPUT，也必须先通过容量门再发布 completion flag。等待期间持续 +drain 本核已有任务。 + +当前 shared 流程在物理 heap reserve 时执行复用等待。候选方案把 logical +address 与 physical ring address 分离后，H、I、L 都不映射或写物理 +buffer，TensorMap 也只使用 logical address。因此物理 heap 的 reclaim、 +容量检查和等待可以从 insert 前整体后移到 B；B 通过容量门后才允许映射 +物理地址、执行 initial-value fill 或发布可执行 slot。 + +### 3.6 Lookup 使用 task-id 历史视图并返回多个 producer + +`L(N)` 的物理执行时间可以晚于 `I(N)`、`I(N+1)`，甚至更多未来 task +的 insert，但它的逻辑查询边界固定为 task N 之前： + +```text +producers(N, X) = + unique { W | W < N and W writes an overlapping part of X } +``` + +一个查询区域可能同时覆盖多个 task 写过的子区域，因此 lookup 允许返回 +多个 producer。每次 insert 必须追加 TensorMap entry，或者以其他方式保留 +等价的 writer 历史；后续 insert 不能覆盖并丢失仍可能形成依赖的较早 +writer。 + +首版不消除被后续 writer 完全覆盖的旧 entry。即使后续 INOUT 覆盖旧 +entry，也不能 unlink、复用或标记旧 entry 失效；lookup 仍扫描所有满足 +task-id cutoff 和相交条件的 entries，最后只按 `producer_task_id` 去重。 +因此首版允许保留可由传递依赖蕴含的冗余 fanin。现有 TensorMap 的 +`OverlapStatus::COVERED -> remove_entry()` 路径不能直接带入本共享方案； +空间覆盖消除只能作为后续独立优化。 + +本方案直接移除单值 `last_writer` 字段及其 exchange 路径。所有 +predecessor 都通过带 producer task id 的历史 entries 查询，不保留 +`last_writer` 缓存或双路径 fallback。底层实现如果需要保存版本链头 +指针，它只是定位历史 entries 的结构索引，应使用不同名称。 + +也就是说，`L(N)` 必须： + +- 只考虑 `producer_task_id < N` 的 writer； +- 排除 task N 自己在 `I(N)` 中发布的 writer； +- 排除 task N+1 及更晚 task 已经发布的 writer； +- 允许扫描到未来 task 的部分完整 entries，但不能使用它们； +- 对所有 tensor 参数、显式依赖、owner 依赖和 map 命中得到的 producer + task id 做全局去重，再占用 fanin 槽位； +- 对 `manual_dep` Tensor 保留显式依赖和 owner/creator 依赖，但跳过 + TensorMap lookup； +- 在所有 `< N` task 的 insert 已提交后才开始。 + +当前阶段沿用固定 `kMaxFanin`：去重后的 producer 总数达到上限后，后续 +唯一 producer 暂时丢弃。该行为可能丢失真实依赖边，属于已知正确性限制； +实现中的丢弃分支必须保留明确 TODO，后续改为可表达全部依赖的方案。 +先去重再检查容量可以最小化依赖边，并尽量避免重复 producer 挤占槽位。 + +因此，lookup 读取的是以 N 为 cutoff 的多 producer 逻辑历史视图，而不是 +查询时 TensorMap 中物理上最新的单一 writer。 + +### 3.7 Fatal 的热路径策略 + +shared submit 在每次 submit 入口只通过公共 atomic 封装读取一次全局 +fatal。若入口已经观察到 fatal，本核不再 Claim，也不进入 H/I/L/B， +立即返回 orchestration/replay。 + +正常 submit 的直线控制流不在 H/I/L/B 各步骤重复查询 fatal，避免所有核 +对同一 atomic cacheline 形成热竞争。本核在 layout/capacity/task-id +检查或其他中间步骤发现不可恢复错误时，只通过公共封装设置 fatal、记录 +错误，并立即退出当前 submit；设置后不得继续执行本 submit 的正常发布或 +构建副作用。 + +已经进入无界 cursor/heap/slot/WonSlot 或 tail-drain 等等待循环的核使用 +低频 fatal 检查。等待循环每轮仍只检查其本来的 progress word，并维护 +winner-local stall counter；只有 progress word 未变化且本轮协作 drain +也没有产生进展，累计达到 `kFatalPollInterval` 后，才通过公共 atomic +封装读取一次 fatal。progress word 变化或 drain 有进展时立即清零 stall +counter。 + +等待慢路径观察到 fatal 后，立即退出等待并逐层终止当前 submit/drain， +不得伪造未完成的 H/I commit 或 task completion。设置 fatal 的核不需要 +poison 阶段 cursor、frontier 或 slot。`kFatalPollInterval` 的首版取值是 +实现调优参数，必须用正常无错误 workload 验证其不会把 fatal cacheline +重新变成竞争热点，同时用 fatal injection 验证所有无界等待都能有界 +退出。 + +因此正常无等待路径只有 submit 入口的一次 fatal load；正常但发生短暂 +等待且持续有进展的路径也不产生额外 fatal load。禁止在每轮 spin 中查询 +fatal。 + +## 4. 由本方案前提推导的实现约束 + +以下约束由第 1 节的候选方案前提和第 3 节的并发规则共同导出。 +如果改用其他发布方案，这些约束可能不再成立。 + +### 4.1 TensorMap 必须支持历史查询 + +第 3.6 节已经确认 TensorMap 必须保留 writer 历史，并且不再使用单值 +`last_writer` 表达依赖关系。 + +实现通过 append-only、不可变的 TensorMap entries 和 bucket chain 保留 +历史信息。 + +TensorMap entry 的哈希身份来自 Tensor 的 `logical_addr.uint64()`；view +和同一 allocation 的后续 INOUT 因而进入同一个 bucket，internal/external +tag 也参与 identity。当前 shared ref 的 `last_writer` 字段及 exchange +路径必须删除，使 shared ref 也只从该历史索引获得 predecessor。 + +首版直接沿用并修改现有单 cacheline `SharedRegionEntry`： + +```text +MaskPointer logical_addr // 取代 buffer_addr 的物理地址语义 +lo, hi +producer_task_id +next_in_bucket // entry slot index +padding +``` + +`logical_addr` 取代当前的物理 `buffer_addr`，`[lo, hi)` 是相对该 logical +storage identity 的 byte range。lookup 首版仍按 +`logical_addr.uint64()` 相等且 +`query.lo < entry.hi && entry.lo < query.hi` 判断相交。 + +真实高维求交所需的 `start_offset`、`version`、`ndims`、`dtype`、 +`is_contiguous`、`shapes[]`、`extent_elem_cache` 和 `strides[]` 暂不作为 +实际成员参与首版逻辑；实现中在 `SharedRegionEntry` 定义旁以注释和明确 +TODO 保留，后续直接补充并启用到同一个 `SharedRegionEntry`，不再引入 +另一种 TensorMap entry 类型。现有 128B `PTO2TensorMapEntry` 只作为字段 +组织和求交算法的参考；append-only 方案不需要它的 `prev_in_bucket`、 +`next_in_task`、`prev_in_task` 等回收链字段。首版和扩展后的布局都必须用 +对应的 `sizeof`、alignment 和 offset `static_assert` 固定 ABI。 + +`SharedOutputCell::published` 也从候选结构删除。reader 不再等待 +per-output descriptor flag；它在协议规定的位置观察 task-level H/I +cursor 后，invalidate 并读取 immutable `SharedOutputCell::tensors`。 + +### 4.2 H/I 发布点和 A5 可见性实现 + +按照第 3.4 节的 task 级提交规则，`I(N)` 完成必须表示 task N 的全部 +TensorMap entries 已经可供 reader 查询。没有 entry 时，`I(N)` 完成 +表示 empty commit 已经推进 `next_insert_task_id`。 + +已确认的抽象 happens-before 关系为： + +```text +完成 task N 的逻辑 heap reservation 和 shared logical Tensors + -> publish next_heap_task_id = N+1 + -> task N+1 获得 H 权限 + +初始化 task N 的全部 TensorMap entries + -> 批量 flush entries + -> atomic publish 受影响的 bucket heads + -> publish next_insert_task_id = N+1 + -> 后续 task observe next_insert_task_id > N + -> 读取 task N 的 entries +``` + +H 使用现有 `shared_heap_vend` 的具体顺序为： + +```text +在 H 外完成 calc_output_layout(N) +等待 atomic_load(next_heap_task_id) == N +old_vend = atomic_load(shared_heap_vend) +task_vend = old_vend +若有新 OUTPUT： + 使用预计算 layout 得到 task_base/task_vend，包括 ring-tail padding + 初始化 task N 的全部 SharedOutputCell::tensors + 批量 flush Tensor descriptors + dsb + atomic_exchange(shared_heap_vend, task_vend) +atomic_exchange(task_cell(N).vend, task_vend) +store_barrier() +atomic_exchange(next_heap_task_id, N+1) +``` + +没有新 OUTPUT 时跳过 `shared_heap_vend` 更新和 Tensor descriptor flush， +但仍发布继承的 per-task vend，并执行最后的 `next_heap_task_id` empty +commit。`shared_heap_cursor[]` 不再参与地址分配或反压，并从候选结构、 +初始化和 onboard invalidate 路径中删除。 + +AICore 原子封装会忽略 `__ATOMIC_RELEASE/ACQUIRE` 参数，因此这些名称 +不能单独证明 onboard 可见性。H、I 两个发布边界都必须建立“writer +flush+dsb 后发布 cursor，reader 观察 cursor 后 invalidate+dsb 再读取” +的顺序，并通过生成代码审计和 A5 onboard litmus 验证。 + +`bucket.head` 已确定保留 atomic 读写。I 的 task 级批量发布顺序为： + +```text +为 task N 预留连续 entry slots +读取各 bucket 的旧 head，并初始化全部 entries/next_in_bucket +flush 整个连续 entry 区间 + dsb +对每个受影响 bucket 执行 atomic_exchange(head, new_head) +store_barrier() +atomic_exchange(next_insert_task_id, N+1) +``` + +同一 task 的 entries 连续分配，因此只需一次 region flush 和一次 dsb， +不再逐 entry flush，也不在 heads 或 cursor 发布后重复 flush。多个 entries +落入同一 bucket 时，先在未发布的 entries 间构造本 task 的局部链，再为 +该 bucket 发布一次最终 new head。 + +reader 使用 atomic load 读取最新的 `bucket.head`，再 invalidate 并读取 +不可变 entry 链。A5 上现有封装分别以 `atomicAdd(addr, 0)` 和 +`atomicExch` 实现这两个操作。这样无需为一个 head 标量额外执行整条 +cacheline 的 writeback/invalidate，但不能替代 entry 本体的批量 +flush/invalidate。 + +所有 atomic 操作都必须调用项目公共 atomic 封装，不能在 TensorMap、 +阶段游标或其他业务代码中直接调用 `atomicAdd`、`atomicExch` 等上板硬件 +原语。sim 和 onboard 必须使用同一组调用入口，由封装内部的 target +分支分别映射到 host atomic 与 A5 硬件原语。底层平台实现细节只允许 +出现在 atomic 封装内部。`bucket.head` 具体使用现有的 `atomic_load` +和 `atomic_exchange`。 + +这里保留 atomic 是标量发布与一致读取的实现选择，不是恢复 bucket 锁： +协议中没有 `-2` 锁定值，没有 writer-writer 竞争，也没有 CAS/RMW +重试。多个 head 更新完成后通过公共 `store_barrier()` 封装再发布 +`next_insert_task_id`；该封装的 A5 映射是否充分，以及 reader 观察 +cursor 后的精确顺序，仍需由 onboard litmus 确认。 + +### 4.3 TensorMap entry 生命周期与分配 + +首版 TensorMap 在一次 runtime/replay 生命周期内采用 append-only +entries： + +- TensorMap 容量以 entry 数量计，不以 task 数量计；使用独立的 + 编译期常量 `kSharedTensorMapCap = 1 << 16`，不与 `kFlagCap` 绑定； +- `entry_high_water` 只由当前 I writer 普通读写，使用 + H 后、I 前准备的 `writer_count` 一次性预检并预留连续 slots，不需要 + atomic； +- entry 发布后保持不可变，运行中不回收、不复用； +- 只有在确认不存在任何 I/L/B/E 访问 TensorMap 的全局静默边界，才允许 + reset 整个 map 和 allocation cursor； +- 容量不足时设置 fatal，不能等待形成反压，因为本协议中没有运行时回收 + 动作可以解除等待。 + +容量检查必须发生在 task N 写入任何 entry 或更新任何 bucket head +之前。容量足够时，I 才把 `entry_high_water` 一次推进 +`writer_count`；容量不足时，在发布任何 task N writer 前设置 fatal。 +这样不会留下只插入了部分 writer 的 task。统计发生在 H commit 后的 +winner-local ref/args 准备中，但 entry reservation 仍属于获得 TensorMap +独占写权限后的 I。 + +不能仅因某个 producer 已执行完成就立即复用其 entry。`I(N+1)` 可以与 +`L(N)` 并发，reader 可能仍在遍历旧 entry；没有 reader epoch 或等价 +安全回收协议时，复用会产生 reader/reclaimer 竞态。动态回收属于后续 +独立方案，不进入首版实现。 + +普通 kernel task 中按 insert 规则跟踪的多个 OUTPUT、INOUT 或 +OUTPUT_EXISTING 分别占用各自的 TensorMap entry；`alloc_tensors` 的 +OUTPUT 不占用 entry。首版 `SharedRegionEntry` 保持 64B，因此 +`kSharedTensorMapCap` 对应 4 MiB entry pool。即使它与当前 `kFlagCap` +数值相同，也必须定义成独立常量，不能写成 `kFlagCap` 的别名；容量单位 +始终是 entry，而不是 task。以后启用真实高维 entry 布局时，再根据实际 +entry 大小和 GM 总预算重新核算该常量，不用 +`task_count * MAX_TENSOR_ARGS` 预设最坏情况。 + +首版同样保持现有有界 task-id 生命周期：`task_id >= kFlagCap` 时直接 +设置 fatal，不在一次 runtime/replay 内对 task id 做 modulo wrap。task +cell、`SharedOutputCell` 和 TensorMap entry 中的 `producer_task_id` +因而都可以继续使用无 generation 的 task id。以后若支持 wrap，必须同时 +为这些索引和引用定义 generation/epoch 以及静默复用边界，不能只对 +task id 取模。 + +## 5. 实现顺序 + +协议语义已经完成对齐。实现前先做只读代码差异审计，再按以下依赖顺序 +修改： + +1. 公共基础：`MaskPointer`、Tensor ABI 和多维 `FdwicOutputRef`； +2. H 流水：`next_heap_task_id`、`shared_heap_vend`、per-task vend，并删除 + heap shard/cursor； +3. I/L：单写 append-only TensorMap、task-id 历史 lookup、producer + 去重和 `kMaxFanin` 已知限制 TODO； +4. B：logical-to-physical 映射、heap backpressure、惰性 frontier 和 + `B_alloc`； +5. 执行集成：删除延迟 ref/`last_writer` 路径，适配 MIX WonSlot 和 AIV + `fanin_count = 0`； +6. sim、并发压力、A5 可见性和性能验证。 + +每一阶段开始编码前，都要先列出对应文件、类型、函数、需删除路径和新增 +测试；不能只按名称机械替换。真实高维求交首版继续使用 `[lo, hi)`，在 +`SharedRegionEntry` 旁保留字段和算法 TODO，不阻塞前五阶段。 + +## 6. 验证方案 + +### 6.1 验证总原则 + +- 编译通过只说明编译门禁通过，不能作为功能正确性结论； +- 每个实现阶段根据该阶段实际代码影响、修改或新增的用例以及第 6.6 节 + 协议矩阵确定验证范围，不能用固定 smoke 代替受影响协议的专用用例； +- 每完成一个实现阶段或一次语义修复后，必须重新编译 native binding、 + A5Sim/A5 runtime 和本阶段动态生成的 orchestration/incore kernels， + 再运行本阶段全部必需用例； +- 禁止使用上一阶段、另一 commit、另一 map mode、另一编译器或另一 + PTO ISA revision 生成的 `.o`、`.so`、runtime 或 kernel 作为本阶段 + 证据； +- 验证前必须确认实际源码、native binding、runtime、动态 kernel 和 + PTO ISA checkout 属于本轮记录的同一 source state、map mode 与 + revision； +- 需要排除缓存时必须隔离 `build/cache`、`build/lib`、native binding + 和相关目标文件，确保对应 `.o`、`.so`、runtime 与 kernel 重新生成； +- 不得修改项目配置、golden、timeout、依赖规则或用例参数来掩盖失败； + 如确需改变配置，必须先把它作为独立验证变量记录并重新定义通过条件； +- 功能、并发压力、可见性 litmus 和性能/profiling 是不同证据，不能 + 互相替代； +- 所有源码检查、旧产物隔离、编译和 A5Sim 在普通 shell 中执行；所有 + 实际访问 A5 设备的操作只通过 `task-submit`； +- A5 onboard 当前仍必须通过 `task-submit`，A5Sim 不使用 + `task-submit`；当前环境没有 `npu-smi` 时，仅按已有用户授权跳过 + arch precheck，不能跳过 onboard 用例； +- 任一必需验证失败时停止当前阶段，保留 source state、构建现场、完整 + 命令和日志，先确定根因，不能用后续改动或偶然通过覆盖失败。 + +### 6.2 验证环境 + +验证统一从项目根目录使用本地 `build_runtimes.sh` 和 `run_tests.sh`。 +它们是验证工具，不属于产品实现。两个脚本必须使用完全相同的环境参数： + +- 项目根目录 `.venv` 已存在，并包含 pytest、nanobind 和用例依赖； +- 外部 shell 设置 `ASCEND_HOME_PATH`，随后 source + `$ASCEND_HOME_PATH/set_env.sh`，不另外维护一套 CANN 路径； +- `PTO_ISA_ROOT` 指向项目 `build/` 之外的独立、干净 checkout； +- `PTO_ISA_COMMIT` 与 CI revision 一致；当前默认 pin 为 + `ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8`； +- `gcc` 和 `g++` 都必须是真正的 major version 15； +- `PYTHONPATH` 把项目根目录和 `python/` 放在已有值之前； +- shared 模式设置 + `CXXFLAGS=-DPTO_FDWIC_SHARED_MAP=1`，需要 private 对照时设置为 0。 + +首次开始端到端验证且项目根目录尚不存在这两个脚本时，按本节模板各创建 +一次，然后执行： + +```bash +chmod 700 build_runtimes.sh run_tests.sh +``` + +后续源码修改、commit 切换或 shared/private 切换只重新运行脚本，不因此 +删除或重新生成脚本。只有本节脚本规范本身改变时才同步更新它们。创建时 +必须把两个模板中的 `/path/to/pto-isa` 替换为当前环境真实的、位于项目 +`build/` 外部的干净 PTO ISA checkout；两个脚本中的 +`PTO_ISA_ROOT`/`PTO_ISA_COMMIT` 必须完全相同。 + +#### 6.2.1 生成 `build_runtimes.sh` + +项目根目录的 `build_runtimes.sh` 使用以下完整内容： + +```bash +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +expected_simpler_commit="$1" +map_mode="$2" +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +venv_path="$repo_root/.venv" + +case "$map_mode" in + shared) shared_map="1" ;; + private) shared_map="0" ;; + *) + echo "Invalid map mode: $map_mode (expected shared or private)" >&2 + exit 2 + ;; +esac + +cd "$repo_root" +test "$(git rev-parse --show-toplevel)" = "$repo_root" +source "$venv_path/bin/activate" + +: "${ASCEND_HOME_PATH:?ASCEND_HOME_PATH must be set}" +source "$ASCEND_HOME_PATH/set_env.sh" + +export PTO_ISA_ROOT="/path/to/pto-isa" +export PTO_ISA_COMMIT="ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8" +export CXXFLAGS="-DPTO_FDWIC_SHARED_MAP=$shared_map" +export PYTHONPATH="$repo_root/python:$repo_root${PYTHONPATH:+:$PYTHONPATH}" + +test "$(git rev-parse HEAD)" = \ + "$(git rev-parse "$expected_simpler_commit^{commit}")" +test "$(gcc -dumpversion | cut -d. -f1)" = "15" +test "$(g++ -dumpversion | cut -d. -f1)" = "15" +test -d "$PTO_ISA_ROOT/.git" +test "$(git -C "$PTO_ISA_ROOT" rev-parse HEAD)" = "$PTO_ISA_COMMIT" +test -z "$(git -C "$PTO_ISA_ROOT" status --short)" + +stamp="$(date +%Y%m%d-%H%M%S)" +build_backup="$repo_root/../simpler-build-backup-$stamp" +binding_backup="$repo_root/../simpler-binding-backup-$stamp" + +if [ -e "$repo_root/build" ]; then + test ! -e "$build_backup" + mv "$repo_root/build" "$build_backup" +fi +mkdir -p "$repo_root/build" + +if compgen -G "$repo_root/python/_task_interface*.so" >/dev/null; then + test ! -e "$binding_backup" + mkdir -p "$binding_backup" + mv "$repo_root"/python/_task_interface*.so "$binding_backup"/ +fi + +test -z "$( + find "$repo_root/build" \ + -type f \ + \( -name '*.o' -o -name '*.so' -o -name CMakeCache.txt \) \ + -print -quit +)" + +strip_shim="$(mktemp -d /tmp/simpler-aarch64-strip.XXXXXX)" +trap 'rm -rf "$strip_shim"' EXIT +ln -s \ + "$ASCEND_HOME_PATH/tools/hcc/aarch64-target-linux-gnu/bin/strip" \ + "$strip_shim/strip" +export PATH="$strip_shim:$PATH" + +nanobind_dir="$(python -m nanobind --cmake_dir)" +cmake \ + -S "$repo_root" \ + -B "$repo_root/build/python-binding" \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython_EXECUTABLE="$(command -v python)" \ + -Dnanobind_DIR="$nanobind_dir" + +cmake \ + --build "$repo_root/build/python-binding" \ + --target _task_interface \ + -j"$(nproc)" + +test -n "$( + find "$repo_root/python" \ + -maxdepth 1 \ + -type f \ + -name '_task_interface*.so' \ + -print -quit +)" + +python "$repo_root/simpler_setup/build_runtimes.py" \ + --lib-dir "$repo_root/build/lib" \ + --cache-dir "$repo_root/build/cache" \ + --platforms a5sim a5 \ + --pto-isa-commit "$PTO_ISA_COMMIT" +``` + +该脚本只负责环境检查、旧产物隔离和干净编译，不运行测试，也不占用 +`task-submit`。同一 commit、map mode、编译器和 PTO ISA revision 下， +一次构建可供多个用例复用。 + +#### 6.2.2 生成 `run_tests.sh` + +项目根目录的 `run_tests.sh` 使用以下完整内容： + +```bash +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -lt 5 ]; then + echo "Usage: $0 [ ...]" >&2 + exit 2 +fi + +expected_simpler_commit="$1" +map_mode="$2" +platform="$3" +repeat_count="$4" +shift 4 +pytest_targets=("$@") +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +venv_path="$repo_root/.venv" + +case "$map_mode" in + shared) shared_map="1" ;; + private) shared_map="0" ;; + *) + echo "Invalid map mode: $map_mode (expected shared or private)" >&2 + exit 2 + ;; +esac + +if ! [[ "$repeat_count" =~ ^[1-9][0-9]*$ ]]; then + echo "Invalid repeat count: $repeat_count (expected a positive integer)" >&2 + exit 2 +fi + +cd "$repo_root" +test "$(git rev-parse --show-toplevel)" = "$repo_root" +source "$venv_path/bin/activate" + +: "${ASCEND_HOME_PATH:?ASCEND_HOME_PATH must be set}" +source "$ASCEND_HOME_PATH/set_env.sh" + +export PTO_ISA_ROOT="/path/to/pto-isa" +export PTO_ISA_COMMIT="ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8" +export CXXFLAGS="-DPTO_FDWIC_SHARED_MAP=$shared_map" +export PYTHONPATH="$repo_root/python:$repo_root${PYTHONPATH:+:$PYTHONPATH}" + +test "$(git rev-parse HEAD)" = \ + "$(git rev-parse "$expected_simpler_commit^{commit}")" +test "$(gcc -dumpversion | cut -d. -f1)" = "15" +test "$(g++ -dumpversion | cut -d. -f1)" = "15" +test -d "$PTO_ISA_ROOT/.git" +test "$(git -C "$PTO_ISA_ROOT" rev-parse HEAD)" = "$PTO_ISA_COMMIT" +test -z "$(git -C "$PTO_ISA_ROOT" status --short)" + +common_pytest_args=( + "${pytest_targets[@]}" + --platform "$platform" + -p no:xdist + -v + --require-pto-isa + --pto-isa-commit "$PTO_ISA_COMMIT" +) + +case "$platform" in + a5sim) + for ((iteration = 1; iteration <= repeat_count; iteration++)); do + echo "[run_tests] a5sim iteration $iteration/$repeat_count" + python -m pytest "${common_pytest_args[@]}" + done + ;; + a5) + onboard_command=( + env + "PTO_ISA_ROOT=$PTO_ISA_ROOT" + "PTO_ISA_COMMIT=$PTO_ISA_COMMIT" + "CXXFLAGS=$CXXFLAGS" + "PYTHONPATH=$PYTHONPATH" + "$venv_path/bin/python" + -m pytest + "${common_pytest_args[@]}" + --device __TASK_DEVICE__ + ) + printf -v onboard_command_string '%q ' "${onboard_command[@]}" + onboard_command_string="${onboard_command_string//__TASK_DEVICE__/'$TASK_DEVICE'}" + printf -v shell_command 'bash -ic %q' "$onboard_command_string" + for ((iteration = 1; iteration <= repeat_count; iteration++)); do + echo "[run_tests] a5 iteration $iteration/$repeat_count" + task-submit \ + --timeout 90 \ + --max-time 90 \ + --device auto \ + --device-num 1 \ + --run "$shell_command" + done + ;; + *) + echo "Invalid platform: $platform (expected a5 or a5sim)" >&2 + exit 2 + ;; +esac +``` + +该脚本接收一个或多个 pytest targets。A5Sim 直接执行 pytest；A5 在 +普通 shell 完成环境和 revision 检查，只把最终 pytest 命令交给 +`task-submit`，避免环境准备占用设备锁。 + +#### 6.2.3 脚本生成与维护规则 + +- 两个脚本只在首次需要端到端验证且文件不存在时按模板生成；不能从旧 + build、临时 runner 或 shell history 拼装； +- 若文件已经存在，先核对它与当前模板的接口和环境约束，不因普通源码 + 修改、commit 切换或 map mode 切换而重建； +- 模板规范改变时，必须同时更新两个脚本，尤其不能只更新其中一个 + `PTO_ISA_ROOT` 或 `PTO_ISA_COMMIT`； +- `PTO_ISA_ROOT` 不能位于会被构建脚本隔离的项目 `build/` 下； +- `.venv` 必须在脚本生成前准备完成，脚本不负责安装或升级 Python + dependencies； +- 外部 shell 应先设置 `ASCEND_HOME_PATH`。`set_env.sh` 把它从 + `latest` 规范化为具体 CANN 版本目录是正常行为，脚本不能要求 source + 后仍保留 `latest` 字面值； +- CANN 的 `PATH`、`LD_LIBRARY_PATH`、`CMAKE_PREFIX_PATH` 和 + `ASCEND_OPP_PATH` 等由 `set_env.sh` 提供，不在脚本中维护第二套手写 + 路径； +- 脚本把项目根目录和 `python/` 前置到 `PYTHONPATH`，但保留已有 + `PYTHONPATH` 在其后； +- `gcc` 和 `g++` 必须分别检查，不能只设置 `CXX` 或只验证 `g++`； +- 两个脚本权限固定为仅 owner 可执行的 `700`，作为本地验证工具，不随 + 产品实现提交； +- build 脚本把旧 build 和 native binding 移到项目同级、带时间戳的 + backup 目录，不静默覆盖或删除；需要清理备份时另行确认目标； +- A5 arch precheck 在 `run_tests.sh ... a5 ...` 前完成；没有 `npu-smi` + 时只有已有明确授权才能跳过 precheck，不能据此跳过 onboard 测试。 + +运行前必须检查： + +```text +simpler HEAD == 本轮声明的 expected commit +PTO ISA HEAD == PTO_ISA_COMMIT +PTO ISA worktree clean +gcc major == 15 +g++ major == 15 +构建模式 == 本轮 runner 的 shared/private 参数 +``` + +默认执行 A5 arch precheck。环境没有 `npu-smi` 时，只有在已有明确授权 +下才能跳过 precheck；该豁免不能跳过 onboard 用例。 + +### 6.3 干净构建 + +以下任一条件变化后，必须重新运行： + +```text +./build_runtimes.sh +``` + +- 修改 runtime、platform、orchestration 或 incore kernel； +- 切换 shared/private； +- 切换 commit 或 branch； +- 修改编译器、编译参数或 PTO ISA revision。 + +构建脚本必须先隔离或清理旧 `build/`、native binding 和相关 +`.o`/`.so`/CMake cache，再重新构建 `_task_interface` 以及 A5Sim/A5 +runtimes。禁止用前一阶段或另一宏模式的产物代替重新构建。构建通过只 +表示编译门禁通过，不能作为功能正确性结论。 + +同一 simpler commit、map mode、GCC/G++、编译参数和 PTO ISA revision +均未变化时，一次 runtime 构建可以供多个 pytest targets 和多轮 +A5Sim/A5 调用复用。任一项变化后都必须重新构建。 + +pytest 会动态编译 orchestration/incore kernels，因此 +`run_tests.sh` 的 map mode 必须和最近一次 `build_runtimes.sh` 完全一致。 +若先验证 shared、再验证 private、最后回到 shared 调试，三个切换点都要 +各自重新构建，不能把最后一次 private 产物当作 shared 集成状态。 + +禁止用 `python -m pip install '.[test]'` 代替本节构建流程：该命令可能 +探测并构建无关平台，也可能把 A5 专用宏传播到错误目标。Python 依赖在 +预先准备的 `.venv` 中管理，native binding 和 runtimes 只由 +`build_runtimes.sh` 生成。 + +### 6.4 用例执行入口 + +runner 接口为： + +```text +./run_tests.sh \ + \ + \ + \ + \ + [ ...] +``` + +`repeat-count` 必须是正整数。runner 在启动 pytest 前重复检查 simpler +commit、编译器、PTO ISA 和 map mode，并统一传入： + +```text +--platform +-p no:xdist +-v +--require-pto-isa +--pto-isa-commit +``` + +A5Sim 每轮直接启动一次 pytest。所有 A5 设备操作必须通过 +`task-submit`；普通 shell 只完成环境检查，然后把最终 pytest 命令交给 +设备任务，并通过 `$TASK_DEVICE` 传递 `--device`。每一轮 A5 重复必须 +重新提交一次独立 `task-submit`，不能用一个设备任务包住多轮 pytest。 +普通功能用例使用 90 秒 `timeout/max-time`；stress 或性能任务只有在 +记录轮数、预计时长和理由后才能提高。 + +多个 pytest targets 可以在一次 `run_tests.sh` 调用中传入，并共享同一 +次 runtime 构建。需要 pytest option 时直接追加在 targets 后，例如用例 +明确要求纳入 manual cases 时追加 `--manual include`。是否追加某个 +option 必须由被选用例本身决定，不能把一个用例的参数套到全部用例。 + +功能验证若定义了 golden，必须运行真实 kernel 和真实 golden。 +`--use-example-exec-time` 只允许用于明确的 sim 性能分析， +`--skip-golden` 不能用于形成正确性结论。性能任务应与功能任务分开 +调用，避免性能配置改变功能门禁语义。 + +标准调用形式为： + +```bash +COMMIT=$(git rev-parse HEAD) + +./build_runtimes.sh "$COMMIT" shared +./run_tests.sh "$COMMIT" shared a5sim 1 <本阶段 targets...> +./run_tests.sh "$COMMIT" shared a5 1 <本阶段 targets...> +``` + +是否需要 private 对照、具体 pytest targets、平台和重复轮数由本方案的 +验证矩阵决定。 + +每个实现阶段的实际执行顺序为： + +1. 固定并记录 simpler commit、map mode、GCC/G++ 和 PTO ISA revision； +2. 运行一次对应 mode 的干净 `build_runtimes.sh`； +3. 按本阶段验证矩阵选择 targets，先执行所需 A5Sim 功能/压力用例； +4. 完成 A5 precheck 后，每轮通过独立 `task-submit` 执行所需 onboard + 用例； +5. 功能通过后再单独运行本阶段要求的性能/profiling； +6. 记录每个 target 的完整命令、轮数、耗时、pass/fail 和未覆盖风险。 + +任何必需用例失败时停止当前阶段，保留 commit、构建产物和日志进行分析； +不能继续用后续改动或后续轮次的通过结果覆盖该失败。 + +### 6.5 验证结论与记录 + +- 编译通过不能代替功能验证； +- golden 功能用例必须运行真实 kernel 和真实 golden，不能用 + `--use-example-exec-time` 或 `--skip-golden` 得出功能结论； +- 性能/profiling 与功能 golden 分开运行和报告，不能互相替代； +- 失败时保留当前 commit、构建模式、日志和可重复条件，不用后续结果 + 覆盖； +- 每份结果记录 simpler commit、GCC/G++、PTO ISA revision、map mode、 + 平台、完整命令、repeat、pass/fail、耗时和未覆盖风险。 + +### 6.6 本候选方案特有的验证矩阵 + +本方案从已确认协议反推以下验证范围。代码差异审计阶段要把每一行落实为 +“可复用的现有用例”或“需要新增的最小用例”，并明确 A5Sim/A5 平台、 +运行轮数和通过条件。 + +| 范围 | 必须验证的协议性质 | +| --- | --- | +| Tensor/ABI | `sizeof`/offset/alignment；internal/external logical tag；所有 constructor、copy、view 和 fast path 完整传播 `MaskPointer`；external 8B 对齐失败 | +| H | 有/无/多个 OUTPUT 都推进 cursor；无 OUTPUT 继承 vend；descriptor 先完整发布再推进 cursor；H(N) 可与 I(N-1) 并发 | +| Ref/view | 二维、五维、嵌套 view；H 后 I 前全部解析；`manual_dep` 在解析后准确排除；无 unresolved ref 进入 slot/WonSlot | +| I | empty commit；单/多 entry；同 bucket 局部链只发布一次 head；容量 fatal 前不留下部分 task entries；业务代码只调用公共 atomic 封装 | +| L | 忽略 self/future writer；返回多个历史 producer；显式、owner 和 map producer 全局去重；`manual_dep` 跳过 map；`kMaxFanin` overflow 分支保留明确 TODO | +| B/B_alloc | ring-tail padding；只使用 task-local vend；未来 H 不反向阻塞较早 B；容量不足才惰性推进 frontier；无法容纳单 task 时 fatal；initial fill 位于容量门后 | +| MIX | 只有 AIC 执行 task-level H/I/L/B；AIV `fanin_count = 0`；WonSlot 只在 AIC 完整 fanin ready 后发布；三个协作 drain 检查点均能推进 | +| Fatal | 正常 submit 只有入口一次 atomic load；错误核设置后立即退出；cursor/heap/slot/WonSlot/tail-drain 无进展时低频检查；未提交 H/I 的 fatal 不造成永久等待或伪提交 | +| 可见性 | writer flush+dsb 后发布 cursor、reader observe 后 invalidate+dsb；调度扰动下不能读取半初始化 Tensor/entry；A5 生成代码审计和 onboard litmus | + +并发压力用例需要主动扰动 H/I/L/B 的相对进度，至少覆盖： + +- `H(N)` 与 `I(N-1)` 重叠； +- `I(N+1)` 已发布而 `L(N)` 仍在扫描； +- 多个未来 task 的 bucket heads 已可见，但 lookup cutoff 仍为 N； +- 连续 empty H/I commits； +- heap 容量长期不足时，各等待循环依靠 drain 推进并最终解除反压； +- MIX 的 AIC/AIV 到达不同 drain 检查点； +- 分别在 calc/H/I/B 和 tail-drain 注入 fatal，并让其他核预先进入对应 + 等待；记录退出迭代数、fatal load 次数以及是否出现伪 commit、 + completion、timeout 或 deadlock。 + +### 6.7 尚待实现验证收敛的事项 + +以下不再是协议选择，但必须在对应实现落地时补齐证据： + +1. 公共 barrier/cache 封装的精确 A5 映射及 H/I onboard litmus； +2. sim 和 onboard 压力轮数、随机种子、超时与可重复失败信息； +3. 以当前 Shared Map 为 baseline 的吞吐、Claim/H/I/L/B 分段耗时和 + TensorMap/heap 等待指标； +4. 真实高维 `SharedRegionEntry` 启用后的求交正确性与容量预算。 + +[current-model]: shared_tensormap_record.md +[protocol-comparison]: tensormap_inout_issue.md + +## 7. 实现进展台账 + +文档确认并开始编码后，在本节尾部按时间顺序追加所有实现进展。每个实现 +commit 对应一条独立记录；不能只在最终完成时补写汇总，也不能用后续 +结果覆盖早期失败或错误假设。 + +### 7.1 记录规则 + +- 提交时间使用带时区的 ISO 8601 格式，统一记录 Asia/Shanghai 对应的 + `+08:00` 时间； +- 记录最终 commit hash 和 subject，并标明所属实现阶段、状态 + （进行中/完成/回退）及其依赖的前序 commit； +- “前因”说明当前代码限制、触发问题或上一阶段为什么不足；“后果”说明 + 本提交改变了哪些协议性质、数据布局、控制流和兼容边界； +- 变更内容至少精确到主要文件、类型和函数；删除旧路径时同时记录删除 + 原因和替代路径； +- 每条设计结论要能回指本文对应章节，若实现中发现协议需修改，先更新 + 协议并记录原因，不能静默偏离； +- 记录完整验证环境：source state、map mode、GCC/G++、PTO ISA、 + A5Sim/A5、设备任务信息、关键编译参数和是否干净重建； +- 记录实际执行的完整 build/test 命令、pytest targets、参数、随机种子、 + repeat、样本数、warmup、timeout 和日志/产物位置； +- 功能结果不能只写“pass”：至少记录 pass/fail/skip 数、总耗时、每轮 + 结果；失败时记录错误类型、首次失败轮次、可复现频率和关键差异； +- 并发/压力结果至少记录 task 数、block/core 配置、调度扰动方式、轮数、 + 成功率、fatal/timeout/deadlock 数和最长完成时间； +- 性能结果必须提供同环境 baseline 与候选实现的原始数据或汇总数据， + 包括样本数、单位、min/median/p95/max、吞吐或延迟变化百分比；若有 + Claim/H/I/L/B、TensorMap 或 heap wait 分段数据，一并记录； +- A5 可见性验证记录 `task-submit` 任务标识、litmus 迭代数、观察到的 + outcome 计数和 forbidden outcome 是否为零； +- 未运行的必需验证必须显式写“未运行”、原因、风险和补测条件，不能 + 留空或写成通过； +- 台账采用 append-only 语义。事实订正、回退、rebase 或 squash 不静默 + 改写历史条目，而是追加订正及 old-hash -> new-hash 映射； +- 进展台账不能替代详细 commit message。非平凡实现 commit 的 message + 同样必须包含背景/根因或设计理由、关键修改和实际验证。 + +Git commit 无法在自身内容中保存自己的最终 hash。工作流采用： + +1. 实现并完成本 commit 计划内验证； +2. 在同一变更中先追加完整台账条目，hash 暂写 `待回填`； +3. 创建实现 commit 后取得最终 hash； +4. 在下一次台账更新中回填该 hash；最后一个实现 commit 由最终的 + documentation-only 台账收尾提交回填。 + +纯粹用于回填 hash 或整理台账的 documentation-only commit 不算新的实现 +进展，避免产生无限自引用；但其操作时间和用途必须写在所回填条目的 +“台账更新”字段中。 + +### 7.2 单条记录模板 + +```text +### <序号> <实现阶段/主题> + +- Commit：（） +- 提交时间： +- 状态：<进行中/完成/回退> +- 前序依赖： +- 前因与目标： + - <当前限制、问题或设计原因> +- 实现内容： + - <文件::类型/函数：行为变化> + - <删除路径及替代方案> +- 协议影响： + - <对应本文章节、保持或改变的 invariant> +- 验证环境： + - source/map mode/compiler/PTO ISA/platform/device/build state +- 验证命令： + - `<完整命令>` +- 功能结果： + - cases=, pass=, fail=, skip=, repeat=, time=<...> +- 压力/可见性结果： + - tasks/blocks/cores/seeds/iterations/outcomes/timeouts/fatals/time +- 性能结果： + - baseline/candidate/samples/unit/min/median/p95/max/delta +- 未覆盖风险与下一步： + - <未运行项、原因、风险、补测条件> +- 台账更新： + - <回填 hash 的 documentation-only commit 时间或订正映射> +``` + +### 7.3 当前状态 + +尚未开始候选方案实现；当前仅完成协议对齐和实现/验证计划归档。 diff --git a/tests/atomic_probe/pa_scheduler/shared_output_ref_lookup_analysis.md b/tests/atomic_probe/pa_scheduler/shared_output_ref_lookup_analysis.md new file mode 100644 index 0000000000..f9aa4767c3 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/shared_output_ref_lookup_analysis.md @@ -0,0 +1,387 @@ +# SharedOutputRef 依赖查询方案分析 + +## 1. 问题、结论与边界 + +本文回答： + +1. private 为什么用 `TensorDesc + TensorMap`，当前 shared 为什么对 + `SharedOutputRef` 使用 `last_writer + writer_history`？ +2. `SharedOutputRef` 能否先解析成 `TensorDesc`，再像普通 + `GmTensor/LocalTensor` 一样查询 shared ordinary TensorMap？ + +结论： + +- shared 仍然生成正常 `fanin`。symbol 是 writer 查询 key，不是 fanin。 +- `SharedOutputRef` 用于跨 worker 定位 descriptor；writer 使用 symbol + 还是 region 索引是另一项设计，没有必然绑定。 +- 解析 `SharedOutputRef` 后统一走 region TensorMap，在协议上可行。 +- “使用 region map”和“复刻 private fanin”不是同一个决定。private 把 + owner 与 region lookup 结果都加入 fanin；当前 shared symbol 只加入 + reader 之前的 latest writer。 +- 传递依赖闭合时，两种 fanin 可能得到相同计算结果，但边数量、等待对象和 + 故障敏感性不同，必须先确定业务口径。 + +核对基线为远端 `fdwic-swimlane-deps@bbd18779`；实现代码来自其父提交 +`c3aaf99f`。工作区其他未提交改动不属于本文依据。 + +### 1.1 当前状态分别承载什么 + +| 状态 | key | 回答的问题 | +| --- | --- | --- | +| shared output table | `(origin,slot)` | tensor 在哪里、什么形状 | +| `published` | `(origin,slot)` | descriptor 是否完整可读 | +| symbol writer | `(origin,slot)` | 谁最后修改了这个逻辑 tensor | +| ordinary TensorMap | `(buffer,lo,hi)` | 谁写过重叠物理区域 | + +`SharedOutputRef` 只有 `(producer_task_id, output_slot)`，没有地址、offset、 +shape 和 size。它不能直接传给 `SharedLookupTensor()`；走 region 前必须先 +取得共享 `TensorDesc`。 + +## 2. 当前代码的三条真实路径 + +### 2.1 private：owner 与 region writer 都加入 fanin + +private 输出句柄是本 worker 可解引用的 `TensorDesc *`。对 Input/Inout， +`CollectFanin()` 执行： + +```text +AddFanin(tensor.owner_task_id) +AddFanin(LookupTensor(private_map, tensor)) +``` + +`AddFanin()` 只去除相同 task id，不会因为 map writer 更新而删除旧 owner： + +```text +private fanin = unique(owner, latest_overlap_writer) +``` + +Inout/OutputExisting 随后由 `RegisterOutputs()` 把当前 task 的 region +登记进本 worker 的 map。 + +### 2.2 shared symbol：只加入 latest symbol writer + +`SharedOutputRef` 进入 `CollectSharedFanin()` 的 symbol 分支： + +```text +检查 published + -> 读取 last_writer + -> latest >= reader 时沿 writer_history 回退 + -> 得到 max(writer < reader) + -> 只把该 writer 加入 fanin +``` + +该分支不另外加入 descriptor 的 `owner_task_id`。Inout/OutputExisting 在 +ordered Register 中执行： + +```text +history[current] = previous_writer + -> flush history + -> CAS last_writer: previous -> current +``` + +### 2.3 shared ordinary：仍按物理 region 查询 + +普通 `GmTensor/LocalTensor` 当前执行： + +```text +AddFanin(owner_task_id) +AddFanin(SharedLookupTensor(shared_map, tensor)) +``` + +writer 在 ordered Register 中追加： + +```text +(buffer_addr, lo, hi, producer=current_task) +``` + +因此当前 shared 是两套 writer 索引： + +```text +SharedOutputRef -> symbol writer +GmTensor/LocalTensor -> ordinary region TensorMap +``` + +### 2.4 tag 行为对比 + +以下是当前代码事实： + +| tag | private Gm/Local | shared symbol | shared ordinary | +| --- | --- | --- | --- | +| Input | owner + lookup | symbol latest | owner + lookup | +| Inout | owner + lookup；register | previous symbol；commit | owner + lookup；append | +| OutputExisting | owner；register | previous symbol；commit | owner + lookup；append | +| Output | Materialize owner | publish descriptor/symbol | 不做 reader lookup | + +所以“改得与 private 一样”还必须明确： + +- `OutputExisting` 是否读取旧 region writer； +- resolved descriptor 的 `manual_dep` 是否关闭自动依赖； +- fanin 是 `owner + latest`，还是只保留 latest。 + +代码入口： + +- private：`pa_frontend.h::CollectFanin/RegisterOutputs` +- symbol：`pa_scheduler_core.h::ResolveSharedSymbolWriterBefore` +- shared collect：`pa_scheduler_core.h::CollectSharedFanin` +- ordered commit： + `pa_scheduler_core.h::CommitPreparedSymbolSharedWriterIntentSet` + +## 3. 用具体 task 对比 + +### 3.1 示例一：Output 经过一次 Inout + +设 tensor `X`： + +```text +symbol = (10,0) +descriptor = { buffer=A, range=[0,4096), owner_task_id=10 } +``` + +逻辑任务： + +| task | 操作 | +| --- | --- | +| T10 | `Output X` | +| T12 | `Inout X` | +| T20 | `Input X` | + +#### T12 的行为 + +| 实现 | 查询过程 | T12 fanin | writer 发布 | +| --- | --- | --- | --- | +| private | owner=10；map=NONE | `[10]` | private map 追加 producer 12 | +| shared symbol | commit 后从 history 回退 | `[10]` | history `12->10`，latest=12 | +| shared region | owner=10；map 中 self 被 `<12` 过滤 | `[10]` | shared map 追加 producer 12 | + +当前 ordered shared 在 fanin 前发布本 task writer。symbol 因而从 +`last_writer=12` 回退到 10；region 则通过 `producer < current_task` +过滤自己的 entry。 + +#### T20 的行为 + +| 实现 | owner | latest | T20 fanin | +| --- | ---: | ---: | --- | +| private | 10 | 12 | `[10,12]` | +| 当前 shared symbol | 不单独加入 | 12 | `[12]` | +| shared region/private 口径 | 10 | 12 | `[10,12]` | +| shared region/latest-only | fallback 用 | 12 | `[12]` | + +private 的 `[10,12]` 是当前源码的真实结果，不是笔误。因为 T12 已依赖 T10， +等待 T10 通常是传递冗余，但 private 仍显式保留这条边。 + +当前 shared 使用： + +```text +T20 waits T12 +T12 waits T10 +``` + +它依赖 writer 链完整，以传递关系覆盖 T10。 + +这个例子证明有两个独立选择： + +1. writer 历史存入 symbol 还是 region； +2. fanin 采用 private 的 `unique(owner,latest)`，还是 latest-only。 + +### 3.2 示例二：future writer 先物理发布 + +逻辑顺序： + +```text +T10 Output X -> T12 Inout X -> T20 Input X -> T25 Inout X +``` + +metadata commit 与较早 task 的 fanin/Build 可以流水，因此可能出现： + +```text +T12 metadata published +T25 metadata published +T20 才开始 lookup +``` + +T20 必须得到 12，不能依赖未来 T25： + +| 实现 | 如何排除 T25 | +| --- | --- | +| private | 本核处理 T20 时尚未在 private map 插入 T25 | +| shared symbol | `last_writer=25`，沿 `25->12` history 回退 | +| shared region | 扫描 12/25，但只接受 `producer < 20` | + +future 过滤两种 shared 结构都能实现。 + +### 3.3 反例：较早 writer 尚未发布 + +```text +T12 逻辑上会 Inout X，但 metadata 尚未发布 +T20 已经 lookup X +``` + +symbol 和 region 都只能看到 10，无法区分： + +```text +T12 不存在 +T12 存在但还没发布 +``` + +所以 task-id 过滤只能排除 future，不能补回 missing past。当前 +`task[N-1].deps_prepared` 链必须保证： + +```text +T20 lookup 前，T0..T19 的 writer metadata 全部完成发布 +``` + +无论选 symbol 还是 region，这条合同都不能删除。 + +### 3.4 alias 与 view 对比 + +| 场景 | symbol writer | region map | 待确认业务规则 | +| --- | --- | --- | --- | +| 不同 symbol 指向同一地址 | 两条链，不能互相发现 | 可按重叠发现 | 是否禁止跨 symbol alias | +| 同 symbol 的不重叠 view | whole-slot 保守串行 | 可区分 byte range | 是否需要 view 并行 | +| reader 横跨两个独立 writer | 若同链则传递等待 | 当前只返回最大 producer | 是否必须返回多个 fanin | + +第三种场景示例： + +```text +T12 writes X[0,4096) +T14 writes X[4096,8192) // 与 T12 没有传递依赖 +T20 reads X[0,8192) +``` + +当前 `SharedLookupTensor()` 只返回最大 producer 14。若业务允许这种图，仅等 +T14 不能证明 T12 已完成。采用 region 不自动等于 view 语义完整。 + +当前生产路径只接受 plain `SharedOutputRef`，view ABI 尚未接入；上述内容是 +选型边界，不是当前 PA Case1 已验证的收益。 + +## 4. DCCI 与操作成本对比 + +### 4.1 当前 ordinary lookup + +控制字不是普通 GM load： + +```text +head/tail/seq -> Ops::Load -> atomicAdd(address, 0) +``` + +payload 读取： + +```text +atomic load seq + -> DCCI invalidate payload + -> copy buffer/lo/hi/producer + -> atomic load seq again +``` + +writer： + +```text +write payload + -> DCCI CACHELINE_OUT + DSB + -> publish seq + -> publish tail +``` + +因此每扫描一个有效 slot，reader 都可能付出一次 payload DCCI。 + +### 4.2 当前 symbol lookup + +- `published`、`last_writer` 使用 `atomicAdd(0)`； +- `last_writer < reader` 时不读取 history； +- latest 指向 self/future 时，先 invalidate history 再回退； +- Build 复制 shared descriptor 前另行 invalidate descriptor； +- writer 先 flush descriptor/history，再发布原子控制字。 + +### 4.3 SharedOutputRef 改走 region 后 + +不能直接把共享 descriptor 引用传给 lookup。正确顺序至少是： + +```text +atomic observe published + -> DCCI invalidate shared TensorDesc + -> copy descriptor to winner-local snapshot + -> build region query + -> ordinary map lookup +``` + +当前 Build 已经 invalidate/copy descriptor。若 fanin 再做一次，会重复 DCCI； +候选实现应考虑解析一次并由 Build 复用，同时检查 local state 和 CCEC spill。 + +### 4.4 总体对比 + +| 维度 | private | 当前 shared symbol | 候选 shared region | +| --- | --- | --- | --- | +| handle | 本核 `TensorDesc*` | `SharedOutputRef` | `SharedOutputRef` | +| writer key | physical region | logical symbol | physical region | +| 示例 T20 fanin | `[10,12]` | `[12]` | 取决于 fanin 口径 | +| future 过滤 | 本核时间线 | history 回退 | `producer < N` | +| alias | 可发现 | 不跨 symbol | 可发现 | +| reader DCCI | 无跨核 payload | history/descriptor | descriptor + map slot | +| writer 成本 | 本核普通写 | history flush + CAS | entry flush + seq/tail | +| 串行区 | 无共享 map | symbol commit | region append | +| 容量 | 每核 map | task-indexed history | shared bucket ring | + +### 4.5 当前 PA Case1 的源码计数 + +```text +ordinary region inserts = 0 +ordinary map lookups = 5 * group_count +symbol INPUT loads = 5 * group_count +symbol INOUT writer commits = 3 * group_count +fanin edges = 5 * group_count +``` + +全部改为 region 后，初步模型为： + +- 每个 group 的 5 次 symbol INPUT 增加 descriptor resolve + region lookup； +- 每个 group 的 3 次 symbol writer 增加 predecessor lookup + region append； +- 移除 history flush、`last_writer` CAS 和 history 回退； +- private fanin 口径可能增加 edge 数与 dependency signature； +- shared ring 从零插入变为每个 group 至少 3 条，需重做最坏 bucket + 容量证明。 + +这是源码操作量推导，不是性能结论。symbol 通常是一次 history flush 加多个 +CAS；region 是每 entry DCCI、seq/tail 原子和后续扫描，必须在 A5 上实测。 + +## 5. 评审选项与验收 + +### 5.1 方案 + +| 方案 | writer 权威 | fanin 口径 | 主要特点 | +| --- | --- | --- | --- | +| A 保持当前 | symbol | latest-only | O(1) 快路，whole-slot，保留 history | +| B region/private | region | owner + latest | 与 private 一致，可发现 alias，边更多 | +| C region/latest | region | latest，否则 owner | 去掉 history，但不声称复刻 private | +| D 双索引 | 两者 | 需仲裁 | 双写与故障恢复复杂，不建议默认采用 | + +### 5.2 必须先回答 + +1. fanin 必须与 private 列表完全一致，还是传递依赖等价即可？ +2. 不同 `SharedOutputRef` 是否可能 alias 同一 backing region？ +3. `OutputExisting` 是否读取旧 writer？ +4. resolved descriptor 的 `manual_dep` 如何解释？ +5. 一个读取区间是否可能需要多个独立 producer？ +6. 目标是缩短 ordered Register，还是完整 Submit/makespan？ +7. ordinary ring 不回收时，B256/B512 的最坏 bucket 是否足够？ + +### 5.3 正确性与性能验收 + +正确性至少覆盖： + +- `Output -> Inout -> Input` 的逐 task 精确 fanin； +- 多次 Inout、future writer、missing past； +- 跨 symbol alias、不重叠 view、横跨多 writer reader； +- B1/B256/B512 的结果、`fatal`、依赖签名和 map 容量。 + +性能在相同提交、task 数、编译配置和设备隔离下记录： + +- trace-free 完整 Submit/makespan； +- ordered Register 执行与等待； +- descriptor resolve、fanin lookup 和 Build copy； +- history flush/CAS/invalidate； +- region DCCI、扫描 slot、seq/tail 原子及最坏 bucket occupancy； +- level-4 泳道用于解释，不替代 trace-free 主性能。 + +在 fanin、alias/view 和 `OutputExisting/manual_dep` 口径确认前，不应机械地 +把 symbol helper 替换为 `SharedLookupTensor()`，也不应让两套索引同时成为 +writer 权威。 diff --git a/tests/atomic_probe/pa_scheduler/shared_tensormap_record.md b/tests/atomic_probe/pa_scheduler/shared_tensormap_record.md new file mode 100644 index 0000000000..2ea31c5b5c --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/shared_tensormap_record.md @@ -0,0 +1,11817 @@ + + +# A5 FDWIC Shared TensorMap 分支架构审查记录 + +本文审查 +[`poursoul/simpler:fdwic-shared-tensormap`](https://github.com/poursoul/simpler/tree/fdwic-shared-tensormap) +在 A5 FDWIC runtime 中实现的 shared TensorMap 方案，并与当前分支 standalone +方案比较。本文用于后续开发决策，不表示目标分支已经合入，也不把分支文档中的 +实验记录自动当成当前分支的性能结论；当前分支 S0～S4.14b 的保留结果及 +S4.15/S4.16 正确性成立、性能否决记录另记于第 15 节。当前有效 shared +性能基线为已撤销 S4.15a 源码的 `319077a9`，其运行行为回到 +`ee42b8c1`；继续开发和验收的后端范围固定为 CPU/CCEC。 + +后续每个实现小步都必须先对照参考提交：可直接复用的机制要说明复用位置； +有意不同的顺序、ABI、失败语义或性能取舍要在本文记录证据和复核条件，不能只 +留在代码或会话里。参考代码“基本流程已跑通”本身是正向证据；暂不照搬某项 +机制只表示当前小步需要隔离变量，不表示否定其架构价值。 + +## 1. 审查快照与证据口径 + +| 项目 | 本次固定值 | +| ---- | ---------- | +| 审查日期 | 2026-07-24 | +| 目标提交 | `2866ad73b4f15a4f6fa292af5b7d546e8f972f8d` | +| 当前分支提交 | `1726a774826b20f14fbd8c8fcfb54cdbc525f49f` | +| 两分支 merge-base | `599703f5b153a3a0fd2a3516dff4efd49be3f00a` | +| shared 功能序列首提交 | `67d9d186` | +| 首提交父提交 | `f5da1a2e` | +| 重点审查差异 | `f5da1a2e..2866ad73`，42 个文件 | + +目标分支和当前分支在较早位置已经分叉。若直接审查 +`HEAD...poursoul/fdwic-shared-tensormap`，会把 131 个文件的两边独立演进都 +误算成 shared TensorMap 改动。因此本文以 shared 功能序列的父提交 +`f5da1a2e` 为功能基线，同时用当前分支 +`docs/fully_distributed_within_core.md` 第 12 章校准预期协议。 + +本文使用四种证据标记： + +- **代码事实**：可由目标提交源码直接证明； +- **本次验证**：本机执行过编译探针或 Git 检查； +- **分支记录**：目标分支文档记录的 sim、上板或性能结果，本次没有重跑； +- **审查推断**：由并发顺序和状态不变式推出，仍应由定向测试验证。 + +本次没有修改目标 worktree，没有重跑 A5/A5sim 全量用例，也没有复测目标分支 +性能。实际执行了 private/shared 布局探针和 profile-off 头文件编译探针。 + +## 2. 先给结论 + +这份实现不是“把现有 per-core TensorMap 换成一份共享 ring”，而是一套 +**性能优先的双通道依赖协议**： + +1. fresh `OUTPUT` 使用 `(producer_task_id, output_slot)` 直接索引 + `SharedOutputCell`； +2. 普通 Tensor 或既有 Tensor 的重叠写依赖使用 append-only + `SharedRegionMap`； +3. shared API 支持把 Claim 提到重参数构造之前；PA 调用点把完整参数构造放入 + `tok.won` 分支，loser 只返回符号引用。其他调用点若在 presubmit 前已经构造 + `L0TaskArgs`，并不会自动获得这部分收益。 + +这个方向有真实价值。尤其是 winner-first、PA loser 轻路径、fresh output 的 +O(1) 符号定位、显式 DCache flush/invalidate，以及 MIX follower 由 winner +统一解析输入的基础机制，都值得复用。 + +但它目前还不适合作为最终 shared TensorMap 整体移植，主要阻断点是： + +1. private/shared 会生成 ABI 不兼容的三镜像，但构建缓存身份没有包含模式； +2. writer intent 是调用方可选 API，真实 PA 的 INOUT 路径没有使用它； +3. shared heap、region map 和 task table 没有 generation、reclaim 和可证明的 + 有界复用协议； +4. shared 模式下 `fatal_set()` 恒为 false，容量错误可能退化成永久轮询； +5. shared output 每 task 物理上限为 8，但公共接口允许构造最多 32 个输出。 + +因此建议是： + +> 不整段 cherry-pick。先复用“winner-first + 符号引用 + 显式缓存维护”的设计 +> 资产，再把构建身份、写入顺序、generation/reclaim、错误传播和容量断言补成 +> 可证明的协议，最后迁移 PA。 + +## 3. 目标分支的实际逻辑架构 + +### 3.1 它是两个索引，不是一张共享 TensorMap + +```mermaid +flowchart LR + Replay[所有 worker 回放同一 Submit 流] + Claim[Presubmit / Claim] + Loser[Loser: 只返回符号引用] + Winner[当前 task winner] + Consume[收集/解析当前 task 输入] + Symbol[SharedOutputCell
task_id + output_slot] + Region[SharedRegionMap
buf_addr + byte range] + Slot[本核 RingSlot] + Kernel[AIC（Cube）/AIV（Vector）kernel] + Done[task completion flag] + + Replay --> Claim + Claim -->|lost| Loser + Claim -->|won| Winner + Winner -->|发布 fresh OUTPUT| Symbol + Winner -->|登记普通 Tensor 写区间| Region + Winner --> Consume + Symbol -->|供当前或后续 task 定位 descriptor/writer| Consume + Region -->|供当前或后续 task 查 overlap producer| Consume + Consume -->|fanin + resolved args| Slot + Slot --> Kernel + Kernel --> Done +``` + +核心状态位于 +`src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/common/state.h`： + +| 状态 | 组织方式 | 当前语义 | +| ---- | -------- | -------- | +| `shared_outputs[kFlagCap]` | task id 直达数组 | fresh output descriptor 与最新 writer | +| `SharedOutputCell::published[8]` | 每项独占 cacheline | descriptor 发布完成标志 | +| `SharedOutputCell::last_writer[8]` | 每项独占 cacheline | INPUT/INOUT 的 writer 链 | +| `SharedRegionMap::buckets` | 8192 个桶头 | 普通 Tensor 地址哈希 | +| `SharedRegionMap::entries` | 65536 个 append-only entry | 重叠 byte range 及 producer | +| `shared_heap_cursor[8]` | task id 分片 | winner 输出 heap 分配 | +| `tasks[kFlagCap]` | 65536 个 task cell | 完成、vend、依赖准备标志 | + +因此“shared map”这个名字容易造成误解： + +- fresh output 根本不进入 region map； +- region map 不是 ring，没有 per-slot `seq`，没有回收； +- task id 达到 `kFlagCap=65536` 时直接报错，不支持代际复用； +- 零输出 task 仍必须发布 task completion；它只是不需要发布 + `SharedOutputCell` 或 region delta，因为实现里没有全局 map append 前沿。 + +### 3.2 fresh output 快路径 + +producer winner 的关键顺序在 +`runtime/dist_engine/aicore/submit_core.h:668-759`： + +```text +reserve shared heap + -> for each output: + materialize Tensor descriptor + copy descriptor to SharedOutputCell.tensors[slot] + CCEC flush descriptor + DSB + fetch_max(last_writer, producer_task_id) + -> after all outputs: store_barrier + -> for each output: + release publish published[slot] = producer_task_id +``` + +在 PA 这类把参数构造放到 `tok.won` 后的调用点，loser 不构造 +`TensorCreateInfo` 和完整参数，只通过 `rt_submit_loser(tok, output_count)` +返回 `FdwicOutputRef`。`shared_symbol_smoke` 和 `submit_dependency_smoke` +部分 wrapper 在 presubmit 前已有完整 `args`，不能把这项收益外推到所有调用点。 +引用本身包含 `producer_task_id`、`output_slot` 和有限的一维 view 信息。 + +consumer winner 的行为是： + +- `INPUT`：读取 `last_writer`，无有效 writer 时回退到原 producer； +- `INOUT` / `OUTPUT_EXISTING`：对 `last_writer` 做 + `atomic_exchange(current_task_id)`，返回旧 writer 作为 fanin； +- resolve：等待指定 `published[slot]`，invalidate descriptor cacheline， + 再把 Tensor descriptor 拷入执行侧 `RingSlot` 或 `BuiltSubtask`。 + +这里的两个目标槽只有 ABI 形状相似，resolve 时机并不相同：普通 +`RingSlot` 会先 try-resolve，并允许未就绪引用留到执行前；winner follower +构造 `BuiltSubtask` 时则同步阻塞解析，当前 mask 固定为 0。 + +相较统一 region lookup，这条路径的优势明确： + +- 索引为 O(1)，没有桶扫描； +- loser 不必等待全局 task 发布前缀； +- descriptor 身份与逻辑 writer 分开； +- producer materialize 较晚时，`fetch_max` 不会覆盖已经到达的更大 writer。 + +### 3.3 普通 Tensor 的 region 慢路径 + +普通 Tensor 的地址依赖由 +`submit_core.h:854-940` 处理： + +1. 按 buffer 地址选 bucket； +2. 遍历单向 entry 链； +3. 对每个 entry 做 CCEC invalidate； +4. 在重叠区间中选择 `producer < current_task_id` 的最大 producer； +5. writer 使用 `high_water.fetch_add()` 取得永久 entry； +6. 通过 bucket 的 `-2` sentinel 独占插入，再 flush entry 并发布新桶头。 + +它是 append-only 表，不是当前文档第 12 章描述的 +ring-per-bucket： + +- lookup 窗口是所有已插入的 `< N` producer，不是 `[N-H, N)`； +- 没有 `head`、generation、per-slot `seq` 或 min-progress reclaim； +- 同一 buffer 的长写链会让 bucket 扫描不断变深； +- 容量耗尽后只能 `set_fatal()`。 + +`insert_lock` 字段和全局 lock helper 仍存在，但实际插入使用的是每 bucket +sentinel。它目前是死字段候选；应先完成 Host/AICPU/AICore 三镜像的 ABI、 +初始化和布局审计，再单独决定清理或补充用途，不能顺手删除。 + +### 3.4 region intent 是可选调用协议 + +目标分支为写依赖增加了 +`rt_presubmit_*_with_region_intent()`： + +```mermaid +sequenceDiagram + participant W as task N winner + participant L as task N losers + participant S as shared dependency state + + L->>S: wait deps_prepared[N] + W->>W: collect fanin + W->>W: register writer regions + W->>S: publish deps_prepared[N] + S-->>L: wait released; writer intent visible + W->>W: continue winner build + L->>L: continue replay +``` + +意图是防止 task N 的 losers 在 N 的 writer 尚未登记时继续跑到 task N+1。 +但该语义没有收进 runtime 的统一入口，而是由 orchestration/codegen 判断是否 +调用： + +- `submit_dependency_smoke` 的 wrapper 会扫描参数 tag 并选择 intent API； +- PA 的 QK/SF/PV/Update 都调用普通 `rt_presubmit_*`； +- PA Update 甚至在 presubmit 之后、且仅在 winner 分支内才构造四个 INOUT。 + +这意味着 PA 无法在 Claim 前依据尚未构造的参数发布 writer intent。目标分支 +计划文档也承认该能力仍依赖 codegen 显式接入。 + +### 3.5 MIX/joint 路径 + +最新代码已补上一部分 shared MIX 基础机制： + +1. anchor winner 先收集 fanin 并 resolve shared descriptor； +2. fanin ready 后才发布 follower 的 `BuiltSubtask`； +3. follower 直接消费 winner 已解析的 payload，shared follower fanin 清零； +4. `joint_launch_expected/drained` 防止最终退出早于迟到的 follower launch； +5. 所有参与 lane 完成后仍只发布一个 task completion。 + +`0f9862a2` 和 `d618589f` 修正了 winner 侧 shared-ref resolve 以及 +launch/drain 门控，但还不能称为完整闭环。目标分支文档仍列有 MIX output +publisher、MIX INOUT/OUTPUT_EXISTING、连续 MIX、容量/反压和完整 stress +缺口。准确结论是：基础 handoff 已存在，部分关键路径已修正，通用语义和压力 +验证仍未完成。 + +### 3.6 初始化和运行生命周期 + +主要生命周期路径已经接通： + +- host 在 runtime 创建/setup 路径预留并清零固定 runtime arena； +- AICPU register 重置 cursor、task cell、shared output flag、region bucket、 + heap cursor、joint counter、fatal 和 replay 状态； +- AICore attach 重置本核 slot/counter； +- AICPU 唤醒 worker 前 flush runtime arena； +- executor 结束后复位控制状态。 + +每轮 executor 正确性应依赖 AICPU 对 publication flag、bucket、high-water 等 +控制状态的复位，不能依赖“每轮一定重新执行 host 全量 memset”。陈旧 +descriptor/entry 本体没有逐项清零，但控制状态复位后，正常路径不会在重新发布 +前消费它们。error、容量耗尽和跨代复用生命周期仍没有闭环。 + +非 Submit 热区有一个明显成本：AICPU 每轮会 flush 固定 1056 MiB +（约 1.1 GB）的 `dist_global` arena。它不应混进约 5 ms Submit 指标，但会 +影响 case 启动时间。 + +## 4. 开发视图与构建边界 + +### 4.1 三镜像必须是同一模式 + +A5 FDWIC 是三个独立程序共同解释 GM ABI： + +```mermaid +flowchart TB + Config[PTO_FDWIC_SHARED_MAP
构建模式] + Host[Host runtime] + AICPU[AICPU control plane] + AICore[AICore runtime + orchestration] + GM[(DistGlobal / Runtime / Args in GM)] + + Config --> Host + Config --> AICPU + Config --> AICore + Host --> GM + AICPU --> GM + AICore --> GM +``` + +宏默认值为 0，位于 `runtime/pto_types.h:69-71`。clean build 时， +环境变量 `CXXFLAGS=-DPTO_FDWIC_SHARED_MAP=1` 原则上能传播到： + +- direct sim/host/orchestration 编译； +- A5 onboard AICore 的 CCEC 自定义命令； +- AICPU 和 runtime CMake target。 + +但它仍只是 ambient `CXXFLAGS`，不是显式的 runtime variant。任何一个镜像 +漏掉宏，都将用不同字段偏移解释同一块 GM。 + +### 4.2 private/shared ABI 实测 + +本次用目标提交真实头文件编译了两个布局探针： + +| 类型或字段 | private | shared | +| ---------- | ------: | -----: | +| `TensorRef` | 16 B | 24 B | +| `L0TaskArgs` | 1,024 B | 1,280 B | +| `L2TaskArgs` | 3,840 B | 4,864 B | +| `RingSlot` | 4,864 B | 5,440 B | +| `DistCore` | 9,231,488 B | 8,464,832 B | +| `DistGlobal` | 1,007,102,208 B | 1,080,630,976 B | +| `Runtime` | 69,952 B | 70,976 B | +| `DistCore::slots` 偏移 | 823,424 | 64 | +| `DistGlobal::heap_base` 偏移 | 4,197,504 | 143,176,000 | +| `DistGlobal::cores` 偏移 | 10,101,504 | 166,429,120 | + +变化来自： + +- shared 的 `kPrivateSlots` 从 4 增至 14； +- shared 移除 `DistCore::map`； +- shared 在 `DistGlobal` 中插入约 128 MiB output table、region map、heap + 和 joint 状态； +- `TensorRef` union 增加 `FdwicOutputRef`。 + +固定 arena `0x42000000` 只保证两种布局各自不越界，不表示 ABI 相同。若把当前 +private map 和 shared 状态机械组成一个 superset，估算会比现有 arena 多约 +62.3 MiB，因此“直接保留两套所有字段”并不是合理修法。 + +更合适的结构是固定小型控制头： + +```text +DistRuntimeHeader { + magic + abi_version + map_mode + profile_mode + state_size + backend_state_ptr +} + +backend_state_ptr -> PrivateBackendState 或 SharedBackendState +``` + +或者把两种模式作为完全独立、带 mode/ABI tag 的 artifact family。两种办法都 +比当前“字段偏移变化但无握手”可靠。 + +### 4.3 构建缓存会制造混合镜像 + +`simpler_setup/runtime_builder.py:307-329` 的 baseline runtime 缓存 stamp +只有 Git HEAD，输出目录也不包含 shared/profile 模式。与此同时， +per-callable AICore stale state 已包含 `CXXFLAGS`。 + +因此同一 commit 下先编 private、再改 `CXXFLAGS` 编 shared 时，可能出现： + +```text +baseline Host/AICPU = 旧 private artifact +per-callable AICore = 新 shared artifact +``` + +目标分支文档要求切模式前手工删除 +`build/cache/a5/{sim,onboard}`，正说明当前构建身份不完整。手工清缓存可以帮助 +实验，但不能成为架构正确性条件。 + +### 4.4 profile 与 map 模式当前不正交 + +本次对以下两个组合做了头文件编译： + +- `PTO_FDWIC_SHARED_MAP=0, PTO2_PROFILING=0`； +- `PTO_FDWIC_SHARED_MAP=1, PTO2_PROFILING=0`。 + +两者都在 `pto_types.h:906-908` 失败，`L0TaskArgs` 和 L2 Arg 对 64 取模均为 +32。原因是 `DumpArgSelection` 被条件编译去除后，固定 padding 没有同步调整。 + +这与 `docs/profiling_levels.md` 给出的 profile-off 构建方式不一致，也说明 +测试矩阵没有覆盖 `{private, shared} × {profile on, off}`。 + +## 5. 与预期架构的逐项比较 + +这里的“预期”由两部分组成： + +1. 用户要求：以宏保留 private/shared 双模式，先保证 private 基线； +2. 当前 `docs/fully_distributed_within_core.md:1272-2068` 的协议约束： + task-order 发布、`[N-H,N)` 查找、generation、progress/reclaim 和有界反压。 + +| 维度 | 原预期 | 目标分支实际实现 | 评价 | +| ---- | ------ | ---------------- | ---- | +| 默认模式 | private | 宏默认 0 | 符合 | +| 切换方式 | 明确构建模式 | ambient `CXXFLAGS` | 形式符合，工程不完整 | +| 上层 API | facade 内切后端 | public 返回类型和 submit 协议都变化 | 差异较大 | +| loser 路径 | 尽量轻 | API 支持；PA 做到 winner-only 构参 | PA 路径优于朴素预期 | +| fresh output | 可走快路径 | task/slot 直达表 | 值得保留 | +| 普通区域依赖 | 统一有界 ring | append-only bucket 链 | 不符合 | +| writer 顺序 | runtime 自动保证 task order | 可选 intent API | 不符合 | +| 发布前沿 | 零 entry task 也推进 | 没有全局 map 前沿 | 架构不同 | +| lookup 窗口 | `[N-H,N)` | 所有已发布且 `= task_id-H`，但没有记录即将覆盖的物理区间属于哪个 task/generation。 + +winner materialize 顺序可以和 task id 顺序不同，因此“当前 task 已跨 H 窗口” +不能直接证明该物理区间的上一任已经无人引用。当前只检查单 task 输出不超过 +shard span，也没有证明 H 窗口内同 shard 累计 live bytes 一定装得下。 + +对“启用 wrap 或长期复用”的实现，这是移植前 P0。若第一阶段明确禁止 wrap， +并在容量边界可靠终止，则该项可以降为通用化前 P1，但不能在没有 generation +证明时悄悄复用。 + +#### P0-4 fatal 写入后等待方仍看不到 + +`runtime_state.h:40-47` 在 shared 模式下让 `fatal_set()` 固定返回 false，而 +`set_fatal()` 仍写 `g_dist.fatal`。task cap、region cap、heap retry 或 resolve +错误发生后，其他核可能继续轮询，无法可靠结束。 + +#### P0-5 输出上限 8 与接口上限 32 不一致 + +`SharedOutputCell` 的三个数组长度都是 8，但： + +- `SharedTaskOutputs::add_output_ref()` 只检查 `MAX_TENSOR_ARGS`； +- `rt_count_outputs()` 可以返回大于 8； +- materialize 按实际 output ordinal 直接索引 shared cell。 + +第 9 个 output 起可能越界，且发生在后续 resolve 的 8 上限断言之前。 + +### 6.2 P1：通用化前补齐 + +| 风险 | 代码事实或待证点 | 建议验证 | +| ---- | ---------------- | -------- | +| task/region 容量 | task 上限 65536；region 是 65536 entry，不是 65536 task | 多区域 task 的 cap 边界 | +| external RAW | owner 无效的 external pure INPUT 确定跳过 region lookup | 查 API 禁令；无禁令则补 RAW | +| region 合并读 | lookup 只返回一个最大 producer，可能漏掉另一不重叠 writer | 两 writer + 横跨两区 reader | +| fanin 截断 | 第 17 个不同 producer 被静默丢弃 | 17 producer golden + 明确报错 | +| scalar data access | shared 下 `get/set_tensor_data` 不等待 producer | orchestration 读取前 task scalar | +| generic submit | shared 的 `dist_submit_impl()` 直接 assert | 建立统一 facade 或编译期禁用 | +| dummy submit | dummy 不分配 task id、依赖或 completion | 明确其 shared 语义 | +| view | `FdwicOutputRef::view()` 只允许 1D | 多维/嵌套 view | +| view 并行度 | `last_writer` 粒度为整个 output slot，不区分不重叠子 view | 两子 view 并行写的依赖图 | +| atomic 顺序 | CCEC wrapper 丢弃 `memorder` 参数 | 查 A5 ISA 文档并做微基准 | +| initial value | 写输出 data 后未见同等级显式 data flush | 上板 producer/consumer 可见性测试 | + +external RAW 的代码行为已经明确：owner 无效的 external pure INPUT 会跳过 +region lookup。待证的是公共 API 是否明确禁止“external Tensor 先作为 INOUT， +后作为 pure INPUT”。若没有该禁令，这就是语义缺口，不只是测试不足。 + +region map 还有一个独立的表达能力问题：假设 task A、B 分别写同一 buffer 的 +两个不重叠子区，task C 再读取横跨两区的范围；如果 A、B 之间没有传递依赖， +lookup 只返回最大 producer B，不能保证 C 同时等待 A。应在决定 region 数据 +结构前先用定向 golden 固定“一个读取区间是否允许需要多个 producer”的语义。 + +shared-ref view 则偏向保守：`last_writer` 的粒度是整个 +`(producer_task_id, output_slot)`，两个不重叠一维子 view 的写也会串在同一 +writer 链上。它通常不破坏结果，但会比普通 region byte-range 语义损失并行度。 + +### 6.3 P2：性能和工程清理 + +- `SharedRegionMap` 同桶链越长，lookup 的 invalidate + DSB 次数越多； +- shared 模式把私有 slot 从 4 增到 14，`drain_phase_b()` 的扫描地板上升； +- cursor 分片减少相邻 task 的 cacheline 冲突，但同一 task 的所有候选仍竞争 + 一个 atomic； +- 同一个 shared ref 重复出现时，descriptor resolve/invalidate 可能重复； +- shared heap 同时付出 shard cursor 和全局 vend 两次原子更新； +- 1056 MiB runtime arena 的全量 reset/flush 影响启动； +- `worker_state.h` 中巨大的 fallback `DistGlobal` 会增加 BSS/虚拟地址压力； +- 宏分散在 runtime API、state 和多个业务 orchestration 中，后续维护成本高。 + +## 7. 性能证据应该怎样解读 + +目标分支文档记录： + +- shared PA Case1 上板生成过 + `outputs/TestPagedAttentionUnroll_Case1_20260723_150651/merged_swimlane.json`； +- 脚本记录的 `global_span_us` 为 `2285.352 us`； +- 按需 EfDrain 前后约为 `2.540 ms -> 2.318 ms`； +- 错误的全局 prefix 退出方案曾把约 `2.3 ms` 劣化到约 `7.2 ms`。 + +这些是**分支记录**，不是本次验证。目标分支没有提交对应 raw/swimlane artifact， +所以本次无法重新计算事件数、统计窗口和 instrumentation 开销。 + +它也不能直接与当前分支约 5 ms 的 PA 基线相减，原因包括： + +- 两分支从较早提交分叉，runtime 和 PA orchestration 不同； +- target 使用 winner-first shared API，当前基线仍有另一套观察代码； +- 目标脚本的 `global_span_us` 是所有 `ph == "X"` 事件的最早开始到最晚结束， + 不是严格的“首个 Submit 入口到最后一个 Submit 返回”； +- 目标数字来自带 L2 swimlane 的 shared ELF，不能与当前 perf-clock、普通 + swimlane 或 atomic-swimlane ELF 互减； +- 目标记录缺少可复算 raw 证据。 + +合理表述只能是： + +> 目标分支记录表明该结构有较强性能潜力，尤其 loser 轻路径已经改变了复杂度； +> 但不能据此宣称相对当前 5 ms 基线获得确定百分比收益。 + +目标提交的多个 commit message 还分别记录了 `2285.352`、`2299.227`、 +`2330.192` 和 `2293.923 us`。这些是不同提交的单次作者记录，不是同一版本的 +重复样本，不能据此计算 median、min/max 或置信区间。 + +时钟口径也不能混用：目标泳道的 SYS_CNT raw tick 为 1 ns，换算基准为 +1 GHz；这不是 scalar core 主频。约 1.65 GHz 是本机校准得到的 PMU cycle +频率。converter 已按 raw metadata 换算 SYS_CNT，不能再用 1.65 GHz 对目标 +分支的 2.3 ms 做二次缩放。 + +迁移时仍应固定三条互不混算的证据链： + +1. `perf-clock`：决定候选保留或撤销； +2. `swimlane`：解释业务区域和 atomic 变化； +3. `submit-pmu`：辅助解释 scalar busy、I-cache request/miss。 + +三种 ELF 只做同类前后对照，不能跨构建直接相减。 + +## 8. 测试证据与证明边界 + +### 8.1 已有覆盖 + +| 测试 | 数量/平台 | 主要覆盖 | +| ---- | --------- | -------- | +| `shared_symbol_smoke` | 11 例：7 个 sim-only、4 个 manual A5 | AIC/AIV 跨角色、双输出 slot1、heap shard、dual AIV | +| `submit_dependency_smoke` | 84 例：38 sim、45 A5、1 双平台；46 manual | INOUT、overlap/view、alloc、heap、DCCI、长链 | +| `simple_orch_smoke` | 24 例：12 个 sim-only、12 个 A5-only；14 个 manual | joint、WonSlot、基础 orchestration | +| `benchmark_bgemm` | 7 例；当前只算 private/default 相关基线 | orchestration 未适配 shared 新 API，不能算 shared 覆盖 | +| PA | 3 例均声明双平台；Case2/3 为 manual | 真实业务 golden；Case1 是分支性能工作负载 | + +前三类 smoke 和 PA 调用生产 runtime，不是脱离实现的模型测试，这是优点。 +`benchmark_bgemm` 仍直接使用 shared 宏下不存在的旧 submit API，当前只能列作 +private/default 相关工作负载，不能包装成 shared 集成证据。 + +### 8.2 当前证据仍不能证明什么 + +1. Python case 没有设置或断言 `PTO_FDWIC_SHARED_MAP=1`。宏默认 0， + `shared_symbol_smoke` 还有 private fallback；测试名字本身不能证明产物是 + shared build。 +2. 可信的 shared 结果依赖外部 `CXXFLAGS` 和手工 clean cache。 +3. 许多 A5 case 标记为 manual；默认 CI 不等于执行了硬件矩阵。 +4. delayed region intent 的 mode 32/33 只有 sim，没有对应 A5 case。 +5. sim 不能证明 A5 非一致缓存、DCCI/DSB 和原子竞争语义。 +6. 没有生产协议的确定性交错单测，也没有负向容量测试。 + +至少还缺： + +- 9 个 output； +- 17 个不同 fanin； +- task/region cap 边界； +- heap shard 多次 wrap； +- stale generation ref； +- writer task 逆序到达； +- reader task 与未来 writer 逆序到达； +- 两个独立 writer 后的合并区域 reader； +- external INOUT 后 pure INPUT； +- fatal 后所有核有限时间退出； +- shared/profile 四组合构建； +- private/shared 镜像 mode 不一致时的显式拒绝。 + +## 9. 目标分支文档自身的一致性 + +两份设计文档各有价值： + +- `high_perf_shared_map_plan.md` 较准确地记录了 winner-first 目标、手工 clean + cache、task cap 和 intent 仍依赖 codegen 等限制； +- `current_shared_map_runtime_design.md` 对数据结构和性能演进写得很完整。 + +但后者不能完全作为最新代码真相： + +- 开头称 MIX 未闭环的总判断仍成立；最新两个提交缩小了 follower + resolve/launch 方面的具体缺口，但没有完成通用 MIX 语义； +- 第 9 部分引用的一批短 commit id 在当前目标提交对象库中不存在，可能来自 + rebase 前历史，无法直接追溯； +- 文档对 `last_writer` 的顺序解释默认了 writer 按 task id 到达，没有覆盖 PA + 绕过 intent 的实际调用方式； +- 性能数字只有文字记录，没有随分支提交 raw artifact。 + +后续开发文档应固定“代码提交 + 原始证据目录 + 生成命令”，避免设计说明和实际 +分支继续漂移。 + +## 10. 建议的目标架构 + +建议保留双通道思想，但把它放入一个有统一正确性底座的结构： + +```mermaid +flowchart TB + API[SubmitFacade] + Meta[轻量 SubmitMeta
role / outputs / write intent] + Lazy[Winner-only LazyArgBuilder] + Private[PrivateBackend] + Shared[SharedBackend] + Sym[SharedSymbolTable
task/slot + generation] + Reg[Ordered Region Index
bounded ring] + Prog[Progress / Reclaim] + Heap[Generation-aware Heap] + + API --> Meta + API --> Lazy + API --> Private + API --> Shared + Shared --> Sym + Shared --> Reg + Shared --> Prog + Shared --> Heap + Meta --> Reg + Lazy --> Sym +``` + +关键设计点： + +1. **统一 facade**：业务代码不直接散布 private/shared 的返回类型和流程宏。 +2. **轻量元数据与重参数分离**：Claim 前能获得 role、output_count 和写意图， + winner 后才构造完整 `L0TaskArgs`。 +3. **fresh symbol 作为优化层**：继续保留 O(1) task/slot 定位；完整 resolve + 仍可能等待 published，并执行 invalidate、DSB 和 descriptor copy。 +4. **writer 顺序由 runtime 保证**：不能让调用方选择是否正确。 +5. **有界 region backend**：使用 task-order delta + per-slot generation， + 或另一套能证明 writer 全序的协议。 +6. **统一 progress/reclaim**：region、symbol generation、heap 复用共享同一组 + 可证明的活跃窗口约束。 +7. **稳定控制 ABI**：mode/version/size 可在 Host、AICPU、AICore 启动时互检。 + +writer intent 的实现需要在开发前明确二选一： + +- **方案 A，推荐**：轻量 `SubmitMeta` 在 Claim 前表达写集合，由 runtime 按 + task id 发布 dependency delta；零 delta 也推进 sequencer。 +- **方案 B**：设计 per-object 的有序 writer CAS/版本协议，证明乱序 winner + 最终仍形成 task-id 顺序。 + +当前无条件 `atomic_exchange(last_writer)` 加可选 intent barrier 不能作为最终 +方案。 + +## 11. 分阶段移植计划 + +每一阶段独立提交，上一阶段验证通过后再进入下一阶段。实施顺序固定为： + +> 先在 `tests/atomic_probe/pa_scheduler` 完整证明，再迁移到 +> `src/a5/runtime/fully_distributed_within_core` 和真实 PA。standalone 尚未 +> 闭环时，不允许用“顺便对接真实路径”扩大修改面。 + +### 阶段 S0：standalone 构建身份和 ABI + +**状态：已完成；当时的临时 shared fail-closed 门禁已在 S2 删除。** + +- `run.sh` 增加 first-class `--tensormap private|shared`，默认 private； +- CCEC、AscendC、CPU 构建都显式传 + `PTO_FDWIC_SHARED_MAP=0/1`，产物目录包含 backend、mode 和诊断 variant； +- CCEC host、AIC/AIV、callback runtime、callback finish 必须属于同一模式； +- `RunConfig` 使用原有尾部空间保存 magic/version/mode/`sizeof(SchedulerState)`； +- CCEC swimlane、submit-pmu manifest 都记录模式并校验整套产物； +- 修复 PMU 配置使用 `reserved[4]` 越过 `RunConfig`、覆盖 + `WinnerWorkloadConfig::mode` 的问题，改为独立 cache-line sidecar。 + +**Gate**：默认 private 的 CPU b1 全断言不变；脚本语法检查通过；S0 当时 +shared backend 尚未接入，必须在编译期明确失败；故意混用模式或校验和必须在 +启动前失败。S2 接入真实 shared sidecar 后只删除临时 fail-closed，模式隔离和 +混用拒绝继续保留。 + +### 阶段 S1：standalone private map 先同构为 ring + +**状态：已完成。** + +- 保持 `TensorMap=823312 B`、`WorkerState=9231296 B`，并保持 standalone + private DistCore 中 map、ring slot 和 payload 的既有 size/offset； +- 把 private linked map 改为 128 buckets × 128 slots 的 + ring-per-bucket，总 entry 容量仍为 16,384； +- `MapEntry` 保持 48 B，后 16 B 只作 ABI 保留，本阶段不引入 shared seq； +- private 仍然每 worker 独占，不引入 atomic，不同时改构参、heap 或输出引用； +- 每桶 `head/tail` 为普通 `uint64_t`；`AdvanceTensorMap` 用逐 task 精确计数 + 推进 logical live window，`RetireBucket` 在该桶下一次 lookup/insert 时 + 惰性推进物理 head； +- lookup 扫描全部合法槽并取最大 producer，保留 + `producer >= alive_floor` 边界； +- 用统一的逻辑记录 `(buffer_addr, lo, hi, producer)` 作为后续 + private/shared 对比口径； +- insert/register 逐层返回 bool；满桶不覆写、不推进 tail，由 Submit 发布 + fatal，禁止静默漏依赖。 + +**Gate**：逐 task fanin 与旧 private 一致；Case1 保持每 batch 5 条 fanin、 +每 worker 每 batch 4 次 region insert；定向覆盖窗口边界、同地址区间重叠、 +retire、容量耗尽和复用；容量不足必须显式失败，不能静默漏依赖。当前这些 +检查均已通过，详细验证记录见第 15 节。 + +### 阶段 S2：standalone shared 有序 ring + +**状态：已完成全核强一致正确性基线；其性能问题已由后续 S2.5 定向处理。** + +第一版 shared 只切换 map 的副本数、写入主体和并发纪律，暂时保留 eager 构参和 +每核 private heap，避免在一个提交里同时改三个协议面： + +- 在完整 production prefix、standalone controls 和 `results` 后追加 64B 对齐、 + 2,119,808 bytes 的 shared sidecar，不移动 WorkerState 或既有控制/结果 offset； +- sidecar 保存连续 task commit 前沿、全局 reclaim 前沿、96 条每核 progress、 + 128 组分离 cache line 的 head/tail 和 16,384 个共享槽； +- shared 只有 winner 追加 region，零 insert task 也必须提交空 delta；这里的 + winner-only 不包括构参和 Materialize，它们在 S2 仍由所有 worker eager 执行； +- task N lookup 前所有 worker 必须观察到前 N 个 task 连续 commit；winner 完成 + 整 task append 后发布 N+1，loser 等待 N+1，随后所有 worker 精确发布 progress=N； +- lookup 只接受 `producer ∈ [max(0, N-H), N)`； +- slot 的 payload 和绝对 `seq` 各占一条 cache line。writer 先失效旧 seq， + 写 payload 并 DCache flush，再发布 seq/tail；reader 原子读 seq、invalidate + payload、拷贝快照、再次原子读 seq，双检失败即协议错误； +- reclaim 由 `min(core_progress)-H-1` 单调推动； +- 整 task 在任何写入前完成容量预检，普通容量不足保持 all-or-nothing； + overflow/fatal 对所有等待者可见，等待路径带 watchdog，并可协作 drain； +- host D2H 后独立验证 commit/progress、bucket cursor、seq/payload/hash、 + producer 和最终逻辑窗口，不依赖 device 汇总计数自证。 + +**Gate**：每 task 恰好一次 commit，最终 sequencer 等于 task_count；逆序 winner、 +慢核 progress、零 entry task、seq wrap、future/stale 过滤、tiny-cap overflow +全部通过；private/shared 的规范化 logical-map signature 和全局 dependency +signature 一致。CPU b1/b256 与 CCEC A5 b1/b256 已通过该 Gate；详细证据和 +性能边界见第 15 节。 + +### 阶段 S2.5：ordered-winner reclaim + +**状态：已完成 CPU/CCEC b1/b256 闭环；后续实现按当前开发范围不再以 +AscendC 为阶段出口。** + +S2.5 只改变 sequencer 与 reclaim，不提前混入 fresh symbol、winner-only +Materialize 或 shared heap： + +- 只有 task N 的 winner 等待 `committed_tasks == N` 并读取 shared map； + loser 不再等待 N/N+1，也不发布 per-core progress； +- winner 完成本 task lookup 后，以当前 exact turn 推导 + `reclaim_upto=max(-1,N-H-1)`；因此 sidecar 删除 96 条 progress line， + 从 2,119,808 bytes 缩减为 2,113,664 bytes； +- reclaim refresh 首先验证 exact turn，陈旧、未来或试图回退 reclaim 的 + actor 在任何共享写入前失败； +- 整 task preflight 后若仍容量不足，立即 fatal。当前 turn 的可回收边界 + 已经固定，不再等待不可能扩大该边界的慢核进度； +- 零 entry task 仍由唯一 winner 推进 commit 和 reclaim，保证 task-order + sequencer 连续。 + +**Gate**：b256 精确达到 `committed_tasks=1280`、 +`reclaim_upto=1214`、1,024 次 append、52 条逻辑 live entry；依赖签名 +`b7d985d6edb07078` 与逻辑 map 签名 `556bec7ec8d0f323` 均保持不变。 +定向测试覆盖 ordered reclaim 边界、逆序 actor、零 entry、三圈绝对 seq、 +满桶及“回收 stale 后仍满”的 all-or-nothing。验证和性能数据见第 15 节。 + +### 阶段 S3：standalone fresh symbol 与 winner-only + +S3 继续拆成两个独立提交，不能同时改变引用 ABI 和分配主体。 + +**S3.1 状态：已完成 CPU/CCEC b1/b256 闭环。** + +S3.1 已按参考实现接入 16B `FdwicOutputRef`、8B +`SharedTaskOutputs` 和每 task 2,048B 的 `SharedOutputCell`： + +- 本小步仍保留所有 worker 构参和 Materialize，只有 winner 把 8 类 fresh + output descriptor 发布到 shared cell； +- symbol resolver 只接收 plain ref；INPUT 读取 `last_writer`，三个 Alloc + INOUT 以返回旧值的 Exchange 声明当前 writer； +- symbol 与 `manual_dep=true` 的 output view 都跳过 region lookup/register， + 因而 Case1 shared region insert 已从 4/batch 精确变成 0； +- 仍保留 S2.5 ordered winner turn，避免把当前 Case1 的单 writer 事实误推广成 + 任意多个 INOUT writer 都可乱序。 + +**S3.1 Gate**：b1/b256 fresh descriptor 发布为 8/2,048，symbol INPUT +load 为 5/1,280，symbol INOUT Exchange 为 3/768，region insert 为 0/0， +fanin 仍为 5/1,280；依赖边和规范化 writer 签名保持与 private 一致。 +CPU private/shared b1/b256、定向 sanitizer、Python 100 项和 CCEC +private/shared b1/b256 均通过。此步没有引入 generation、deferred resolve、 +shared heap 或 winner-only Materialize；详细结果见第 15 节。 + +以上是 S3.1 当时的实现与门禁，不是当前协议。S4.5 已把 symbol 解析改成 +只读，把 INOUT writer 更新移动到本地执行状态成功建立之后，并用返回旧值的 +FetchMax 做构建后提交；当前统计名也已改为 `inout_writer_commits`。 + +**S3.2 状态：S3.2a shared heap/winner-only Materialize 与 S3.2b +winner-only 重构参均已分别完成 CPU/CCEC 闭环。** + +S3.2 继续拆开验证分配主体与构参主体。第一小步只把 Materialize 和堆分配 +收敛到 winner，所有 worker 仍保持 S3.1 的 eager 构参；第二小步再把 +QK/SF/PV/UP 的重参数构造收敛到 winner，Alloc 保留全核轻参数路径。 +shared heap 首版使用 8 shard 绝对递增分配，默认 b256 每 shard 需求 +25,821,184B，小于 32MiB shard span；临近 wrap 时显式失败，不能用尚未 +证明的 generation 覆盖旧 descriptor。 + +**S3.2a Gate**：private 工作量计数保持原值；shared 全局 Materialize output +从 b1/b256 的 768/196,608 精确降为 8/2,048，shared heap vend 为 +806,912/206,569,472 bytes，region insert 保持 0；CPU 定向测试和 CCEC +A5 b1 当前最终 ELF 已通过；CCEC b256 的规模证据在非法输入预检加固前已 +通过，最终 ELF 按后续只跑 b1 的约定没有重复消耗上板时间。 + +**S3.2b Gate**：shared 的 Alloc 仍保留全员 3 个静态 Output 参数， +QK/SF/PV/UP 的 reset、view/CreateInfo、tensor/scalar 添加只由 winner +执行。全局每 batch 精确闭合为 307 tensor args、9 scalar args、4 reset、 +2 view、2 dynamic CreateInfo；Materialize 仍为 8 个 output。CPU guard-page +锁定公共 split-finish loser 路径不访问上一 task 的陈旧 args；CCEC 侧 +另由 winner 分支源码审计和 b1 集成回归取证。CCEC split private/shared +均完成编译，shared split 与 inline Materialize/Register 分别通过 b1 +运行门禁。 + +### 阶段 S4：standalone CPU/CCEC 验收 + +- CPU 用于确定性交错、ABA、容量和逻辑 differential 测试，不作为 A5 性能证据； +- CCEC 先做 b1 正确性和泳道，再做一次 b256 阶段出口验证； +- 性能继续分开使用 `perf-clock`、`swimlane`、`submit-pmu` 三条证据链； +- 新增等待轮询使用聚合记录，不能把约 300 MiB raw 继续无界放大。 + +**Gate**：private/shared 产生同一 PA Case1 task/fanin/kernel/completion 拓扑， +shared 没有 future/stale 依赖、silent overflow 或永久等待；同构 `perf-clock` +结果达到可迁移标准。目标分支的 2.3 ms 只作为潜力参考，不作验收阈值。 + +#### S4.1 独立 `perf-clock` 证据链 + +2026-07-24 先补齐了此前缺失的 standalone 低扰动性能构建。它不是在 +swimlane ELF 上运行时传 `--no-swimlane`：后者仍会保留各阶段 +`TraceTimestamp` 和 atomic 包围计时。新变体使用独立 +`PA_BUILD_PERF_CLOCK=1`，并在编译期统一消去： + +- 普通阶段记录、PollBatch、atomic 开始/结束时间及返回依赖观察； +- phase-profile 累计； +- submit-PMU counter reader 与 owner； +- Kernel/Commit、startup/final lifecycle 和 ClockBaseline 计时。 + +首个 Submit 在 `BeginCallbackSubmit()` 已确定 task 0 后、EfDrain 前调用 +专用 `PerfClockNow()`；末个 Submit 在 register/winner 或 loser 尾动作、 +`submits++` 之后、返回前调用第二次。结果继续复用既有 +`WorkerResult::submit_begin/submit_end/submits`，没有扩大 trace buffer 或 +增加逐 Submit 记录。S4.1 当时的 shared exact-turn 与 startup 屏障都保留 +时间 watchdog；S4.6 已删除前者，当前只剩 per-slot symbol 等实际等待和 +startup 屏障使用同类 watchdog。每个等待窗口先读取一次超时起点，之后每 +1024 次未完成轮询复查一次。它是正确性超时，不属于新增性能观察，因而只声称 +“每核两个专用性能边界”，不声称最终 ELF 物理上只会读取两次系统时钟。 + +构建身份 ABI 从 3 升到 4，并复用 `PmuProbeConfig` 的一个保留槽加入 +`swimlane=1 / submit-pmu=2 / perf-clock=3` 握手。这样即使绕过 +`run.sh` 和 manifest，host/kernel 变体不一致也会在解释 worker 状态前 +fail-closed。CCEC perf-clock 保持现有 split-finish 形状，只生成 +host/kernel 两件套；最终 ELF 不得含 `WritePollBatchRecordRaw`，也不放置 +会改变后续热函数 I-cache 对齐的可执行 marker。CPU 使用独立产物目录，并逐线程断言专用 +性能时钟恰好调用 2 次、全局共 192 次；CPU 仍不作为 A5 性能证据。 + +本小步先按既定规则只做 b1 门禁： + +| 门禁 | 结果 | +| --- | --- | +| CPU private/shared perf-clock 严格构建 | 系统 g++ 13.3.0 下全部独立自测 PASS | +| CPU private/shared b1 real-compute | 96 核、每核 5 Submit、专用读钟 192/192、全部语义与数值断言 PASS | +| CCEC private/shared mixed ELF | split caller/runtime/finish、无 trace writer、LOCAL helper、无残留 relocation、manifest 全部 PASS | +| CCEC private b1 real-compute | 最终代码全部断言 PASS；单个结构样本 `66.620 us` | +| CCEC shared b1 real-compute | 最终代码全部断言 PASS；单个结构样本 `81.714 us` | +| 既有普通构建回归 | CPU private 与 CCEC private swimlane b1 全部断言 PASS | +| 既有 submit-PMU 回归 | CCEC private `none` 构建、96 核身份/窗口、PMU 清理与 b1 语义全部 PASS | +| 变体交叉运行负向门禁 | perf-clock host 搭配 swimlane kernel 被 ABI4 `build_variant` 握手拒绝 | + +两个 CCEC 数值来自不同 ELF 的各一个独立 b1 进程，只证明边界和产物可用， +不能据此判断 shared 快慢。正式性能结论仍需冻结两份 perf-clock ELF 后做 +平衡顺序的 b256 private/shared 配对；swimlane 与 submit-PMU 只负责解释， +不得和 perf-clock 的绝对时间互减。当前 S4 尚未因为这两个 b1 样本而宣告完成。 +本阶段只实现和验收 CCEC、CPU 两条路径，不新增 AscendC perf-clock。 + +#### S4.2 `perf-clock` 规模门禁否定全局 exact-turn + +冻结 `bb482001` 对应的 private/shared CCEC perf-clock 两份 ELF 后，先按 +`private -> shared -> shared -> private` 做独立进程交错。b1 全部语义断言 +通过： + +| 模式 | 两个独立 b1 样本 | 平均值 | +| --- | --- | ---: | +| private | `65.808 us`、`67.343 us` | `66.576 us` | +| shared | `77.823 us`、`78.893 us` | `78.358 us` | + +shared b1 平均多约 `11.783 us`，即 `17.7%`。这个结果只说明短序列差异 +方向稳定，不能替代规模门禁。 + +同一组冻结 ELF 随后只做一轮 b256 ABBA。private 两次均通过，分别为 +`3.810849 ms` 和 `4.920407 ms`；shared 两次都不是性能样本： + +| shared 样本 | watchdog 退出 | `committed_tasks` | 语义 | +| --- | ---: | ---: | --- | +| 第一次 | `2.006839 s` | 19 | FAIL | +| 第二次 | `2.008819 s` | 29 | FAIL | + +两个失败都恰好命中 2 秒 watchdog。96 个 worker 全部未完成，最快与最慢 +worker 的 Submit 进度相差 116，且很多 future winner 已经取得 Claim。 +因此这不是普通的 Claim 漏选，而是当前结构形成了 winner convoy： + +1. 分片 Claim 允许快核提前取得 future task； +2. winner 随后在原调用栈等待全局 `committed_tasks == task_id`； +3. shared loser 不等待并继续前冲，最终更多 winner 占住 worker； +4. CCEC `SpinHint()` 为空，所有 future winner 持续对同一 cache line 发 + atomic load，下一 turn 的 winner受到严重争用和饥饿； +5. Case1 的 region ring 明明恒为空，shared heap、descriptor 发布、symbol + resolve 和 register 仍被放进同一条全局串行链。 + +参考分支的高性能设计已经明确禁止该结构：fresh output 按 +`(producer_task_id, output_slot)` 独立发布，consumer 只等待自己实际消费的 +symbol；ordinary region 才进入 bucket 局部协议，不能在 shared winner +入口统一等待全局发布前缀。当前 standalone exact-turn 既与该目标冲突， +也已被 b256 反例否定，不能迁移到真实 simpler。 + +后续修正按以下独立小步推进： + +1. 先把 fresh descriptor 提前到 winner 物化后独立发布，并让 symbol + resolver 只等待实际 producer slot；保留 watchdog 和逐字段校验； +2. 把 shared heap 的 8-shard FetchAdd 改成允许合法并发 allocator， + 删除只在“全局唯一 writer”前提下成立的预检快照与回滚； +3. PA Case1 的 region entry 数必须继续严格为 0，并绕过全局 ordered + sequencer；非空 region 在 bucket 局部并发协议完成前显式拒绝； +4. 依次通过 CPU 定向测试、CCEC b1、CCEC b256 语义与签名，再重新做 + private/shared perf-clock 交错比较。 + +S4 因此保持未完成。当前优先级是修复 shared 长序列活性，而不是根据失败的 +b256 运行估算性能；本阶段范围仍只有 CCEC 与 CPU。 + +#### S4.3 fresh symbol 从全局 ordered commit 中拆出 + +第一项结构修正只拆 fresh descriptor 的发布与消费，不同时改 shared heap +或 region ring，避免把多个并发协议混进一个提交。以下顺序是 S4.3 当时的 +中间状态，随后已由 S4.5 的“构建后封口”协议替代： + +- winner 完成 descriptor 物化后，立即按 `(producer_task_id, output_slot)` + 发布 `TensorDesc`、`last_writer` 和 `published`； +- `AppendSharedTaskOrdered()` 不再代发 fresh symbol，只保留 ordinary region + delta 与旧全局 commit； +- consumer 解析真实 `FdwicOutputRef` 时先读取对应 `published` cell。已就绪 + 快路径只做一次 atomic load；只有首次观察到 `-1` 才读取 watchdog 起点， + 随后只轮询该 producer/slot，每 1,024 次复查 fatal 和 2 秒超时； +- 任意非 `-1` 且非预期 producer 的值都按协议错误 fail-closed，不能退回 + ordinary region 查找掩盖 symbol 状态破坏。 + +S4.3 当时的 descriptor 发布继续落在 `Materialize` 业务区间。泳道的 +`TracePhase::Materialize` 与 submit-PMU 的 `materialize` 起止点同步覆盖 +descriptor 构造、DCCI flush 和 `published` 原子发布，避免分段泳道已经计入 +发布、I-cache 窗口却提前关闭的边界错位。 + +CPU 新增确定性延迟发布测试：publisher 必须等 consumer 已经至少观察一次 +未发布状态后才执行发布；consumer 随后必须完成多次读取、取得正确 descriptor +与 producer fanin，且 ordinary lookup 为 0、fatal 保持 0。该测试与既有 +shared symbol、ordered ring、heap、Materialize、loser stale-args 自测全部 +通过，CPU shared b1 的依赖签名、heap 终态和规范化 writer 签名也全部闭合。 + +CCEC 使用最终代码完成以下 b1 门禁： + +| 构建 | 结果 | +| --- | --- | +| shared perf-clock real-compute | 全部语义与输出断言 PASS；单个可执行性样本 `81.801 us` | +| shared submit-PMU materialize | 96/96 PMU 边界、phase shadow、owner restore/cleanup 与全部语义断言 PASS | + +submit-PMU 样本中 `materialize` 为 480 次固定边界，phase time 占完整 Submit +PMU 窗口约 7.69%；该数字只验证新边界可采集且完整闭合，不用于判断结构收益。 +本小步仍保留 winner 入口的全局 exact-turn，因此没有声称 S4.2 的 b256 +convoy 已解决，也不运行会重复证明旧失败结构的 b256。下一小步先把 8-shard +heap 改成合法并发分配，再单独移除 PA Case1 的空 region 全局 sequencer。 +在移除 exact-turn 前还必须补齐两项协议：`last_writer` 的 per-symbol 有序 +更新，以及 producer 已发布 descriptor、但后续 fanin/register 失败时的 +terminal abort 闭环。当前 per-slot wait 不能单独替代全局 sequencer。 +本步只对 CCEC 与 CPU 做专项实现和验证，未做 AscendC 适配或验证。 + +S4.5 已进一步把 `published` 收紧为“producer Submit 已封口”，因此当前 +Materialize 只负责 reserve 和 descriptor 构造；writer 提交、turn 释放和 +最终 descriptor 发布位于 WinnerBuild/Commit 之后。当前边界与门禁见 +S4.5，不应继续用本节的 S4.3 中间顺序解释最新 ELF。 + +#### S4.4 shared 8-shard heap 建立并发合法性 + +第二项结构修正只改变 allocator、CPU 定向测试和 host oracle，暂不移动 +`FinishCallbackSubmitBody()` 中 Materialize 前的 exact-turn。这样先证明 +heap 本身允许并发，再单独处理仍依赖 task 顺序的 `last_writer` 与发布后 +失败终止协议。 + +新的 no-wrap allocator 契约为： + +1. `task_id % 8` 只决定 shard，物理 `task_base` 由该 shard cursor 的 + `FetchAdd` 返回旧值决定； +2. 前置 Load 只拒绝当时已经可见的负值、未对齐、容量耗尽等损坏状态， + 不能要求后续 FetchAdd old 与该快照相等； +3. `shared_heap_vend` 是另一条独立 FetchAdd 的全局已分配字节前缀，不是 + 物理地址，也不是 task-id prefix；cursor 与 vend 的线性化顺序可以不同； +4. 零输出 task 不执行 RMW，只读取当时 vend，因此在 empty/tiny heap 上 + 合法返回 0；多个零输出 task 也可以观察到相同 vend； +5. 可用总容量为 + `align_down(heap_size / 8, 1 KiB) * 8`，原始 heap 的分片尾部不能被 vend + 当成容量； +6. 若容量竞争在 cursor FetchAdd 后才暴露，本次返回 false 并由上层把整轮 + 置为 terminal fatal。越界 cursor 作为现场保留，绝不能 Exchange 回预检 + 快照或用负 FetchAdd 回滚，否则会覆盖其他 winner 的合法区间。 + +CPU heap 定向测试由旧的“异常 old value 必须回滚”改为真实并发契约： + +- 64 个线程使用 1～4 KiB 变长 reserve，同时覆盖 8 个 shard；排序后每个 + shard 的物理区间必须唯一、无重叠且无空洞； +- 非零 reserve 的 aggregate vend 返回值转换成全局字节区间后也必须唯一 + 且无空洞；零输出 Load 可重复同一 prefix； +- 用 `HeapInterleaveOps` 定向注入 cursor 已推进、vend 尚未推进的原子交错， + 证明两条原子序可以解耦并在静止后重新闭合； +- 用同一注入器模拟 shard 尾部容量竞争，验证失败者保留 overrun、竞争者 + 进度不被回滚； +- 额外覆盖 empty zero-output、tiny heap zero-output、excluded heap tail、 + 满 shard、负值、未对齐、溢出及 b1/b256 最终业务字节分布。 + +host 不再用 `ExpectedSharedTaskBase(task_id)` 重建并发物理地址。它从每个 +已发布 task 的首个 descriptor 反推实际 task base，验证 task 内 output +偏移/shape/size 后，结合 PA Case1 的预期 output 字节数计算逐 output 1 KiB +对齐 reserve span，再按 shard 对所有区间排序并要求从 shard 起点连续覆盖。 +当前不是从任意 descriptor 形状重新推导通用 reserve 大小；结论只覆盖 +Case1。最终必须三向闭合： + +```text +descriptor-derived task base + expected Case1 aligned span == actual cursor + == expected PA aligned reserve bytes +sum(expected Case1 aligned reserve span) == sum(cursor) + == actual aggregate vend == expected aligned reserve total +``` + +shared `TaskCell::vend` 只要求覆盖本 task 自身非零 reserve、对齐且不超过最终 +vend；allocator helper 可为 0，S4.4～S4.5 的 exact-turn 完整流程实际为 +非零；S4.6 删除 turn 后，零输出 task 的 winner 快照也可合法为 0。worker +的 `final_heap_next` 不再与 task-id prefix +集合比较：纯 loser 必须为 0，赢过非零输出 task 的 worker 必须非零且至少 +覆盖本核自身 reserve 总量。跨 private/shared 的 normalized writer signature +仍使用 canonical 连续地址，但只有实际 descriptor/cursor/vend 校验通过后 +才允许计算，不能让规范化签名掩盖物理错址。 + +当前门禁： + +- CPU private/shared 严格构建与全部公共自测 PASS； +- CPU shared b1 的 heap、descriptor、依赖、writer 签名与 real-compute + 输出全部 PASS； +- shared heap 并发定向测试在 ASan/UBSan/leak 检查下 PASS。 +- CCEC shared perf-clock mixed ELF 构建与 b1 全部断言 PASS；新 descriptor + 非重叠覆盖 oracle、cursor/vend、规范化签名和 real-compute 输出均闭合， + 最终工作树单个可执行性样本为 `81.699 us`。 + +本小步的完整运行仍被旧 exact-turn 串行保护，因此这些结果证明的是 +“allocator 与 oracle 已允许合法并发”，不声称 A5 已发生同 shard 乱序。 +b1 四个非零输出 task 还恰好落在四个不同 shard；真正移除 turn 后必须用 +CCEC b256 覆盖多个同-shard reserve，并验证运行中实际产生的任意物理次序， +不能预设单次运行必然出现乱序。本步只对 CCEC 与 CPU 做专项实现和验证， +未做 AscendC 适配或验证。 + +#### S4.5 shared symbol 构建后封口 + +第三项结构修正只处理 symbol 的业务提交点与失败闭环，仍暂时保留 exact +turn。目标是先证明“可执行状态成功建立”与“允许后继消费”之间存在唯一、 +可审计的封口顺序，再在 S4.6 删除全局 sequencer。 + +PA Case1 当前采用固定 symbol 拓扑： + +- QK0 的 fresh output 只由 SF 消费； +- SF0 只由 PV 消费； +- SF1、SF2 和 PV0 只由 UP 消费； +- Alloc0、Alloc1、Alloc2 由 UP 以 INOUT 消费； +- 当前不存在一个 fresh symbol 被多个后继 writer 连续改写的链。 + +这不是通用 shared TensorMap 保证。为防止把固定拓扑误推广，当前 resolver +要求每个 Input/Inout/OutputExisting 所见 `last_writer` 精确等于 +`FdwicOutputRef.producer_task_id`。`CollectSharedFanin()` 的第一遍只等待并 +读取 producer slot、验证引用和重复写引用、构造 fanin；全部成功后才复制 +descriptor 与提交读取统计，不再在解析阶段改写 writer。 + +winner 的正常顺序现在固定为： + +```text +Wait exact turn + -> Materialize：reserve shared heap，构造本地 descriptor + -> CollectSharedFanin：只读解析 symbol + -> PrepareSharedTaskOrdered：准备空 ordinary-region delta，不释放 turn + -> CompleteTask(Alloc) / BuildWinner(QK/SF/PV/UP) + -> CommitSharedFaninWriters：FetchMax 提交 INOUT writer + -> ReleaseSharedTaskTurn：推进 committed_tasks + -> PublishSharedTaskOutputs：复制、flush、barrier、发布 producer slot +``` + +turn 必须持有到 writer commit 完成；否则另一个 future writer 可能先推进同一 +symbol。turn 在最终 output publish 前释放是有意设计：下一 task 即使取得 +turn，也必须在自己实际依赖的 `(producer,slot).published` 上等待。因此 +`published=N` 现在明确表示 producer 的本地执行状态已经建立，且它消费的 +writer 已完成提交，而不再只是“descriptor 已物化”。 + +`CommitSharedFaninWriters()` 使用返回旧值的 FetchMax，并要求旧值精确等于 +producer。失败后不执行负向 RMW 回滚：该 RMW 已经线性化，且多 symbol +提交不是事务；盲目写回旧值会伪造原子历史并抹掉终止现场。实现保留该现场、 +广播 fatal，并使整轮结果失效。多 INOUT task 若在后项失败，前项已成功的 +commit 同样保留； +统计 `shared_symbol_inout_commits` 只计成功项，host 输出 +`inout_writer_commits`。`WorkerResult` 的 896B 大小和该字段 888 偏移不变。 + +PA Case1 的普通 region entry 必须严格为 0。Register 仍执行共享引用过滤， +但任何非零 `shared_entry_count` 都立即失败；当前 empty delta 只为下一阶段 +移除 sequencer 保留对照，不把实际 region 业务悄悄接入全局 turn。 + +构建后封口失败时，QK/SF/PV/UP 已占用的本地执行 slot 会由 +`DiscardBuiltTask()` 撤销，避免 FinalDrain 执行一个未完成 shared 封口的任务。 +Alloc 的 `CompleteTask()` 已发布 ready flag，无法事务性撤回；该路径只会由 +shared invariant 损坏触发，fatal 使整轮无效，不能局部回滚后继续调度。 +final publication 异常会清除本 producer cell 的 published/writer/descriptor， +但 writer commit 的 terminal 现场不回滚。 + +CPU 定向测试覆盖： + +- 只读解析不修改 writer，显式 commit 才推进 writer； +- 不属于当前 Case1 的 `producer -> INOUT -> 后继 INPUT` 链被显式拒绝； +- writer 缺失、future writer、多 INOUT 中途失败和重复写引用； +- 构建 slot 撤销； +- writer 提交失败、turn release 失败、turn 释放后 publication 失败和成功 + 封口四种顺序；异常 release 会恢复观测到的旧前沿，不能覆盖或倒退； +- 真实 `FinishCallbackSubmitBody()` 下的 Alloc/QK publication 故障与 UP + 第二个 writer commit 故障，分别核对 fatal、不可逆 Alloc ready、turn、 + slot、task flag、统计和保留现场。 + +最新 `test_shared_output_symbols` 与 `test_shared_loser_finish` 均通过 +ASan/UBSan/leak；CPU private/shared 严格构建和 b1 全部断言 PASS。 +CCEC shared perf-clock、submit-PMU materialize、submit-PMU register 与 +swimlane 四种 ELF 均完成构建和 A5 b1： + +| 证据链 | 结果 | +| --- | --- | +| perf-clock | 全部语义、依赖、heap、writer 与 real-compute 断言 PASS；最终冷失败加固后单次可执行性样本 `80.032 us` | +| submit-PMU materialize | 480 次边界闭合；phase time 约占 Submit `3.56%`；owner restore/cleanup PASS | +| submit-PMU register | 480 次边界闭合；phase time 约占 Submit `1.16%`；owner restore/cleanup PASS | +| swimlane | 4,265 条阶段记录、0 drop；atomic 逻辑/物理批量记录闭合；单次结构样本 `100.249 us` | + +这些数值来自不同观测 ELF，只用于证明各自边界可执行，不能相互做绝对时间 +加减，也不用于宣称 S4.5 性能收益。Materialize/Register/泳道样本采于最终 +turn-release 冷失败恢复前；该加固不改变三个业务边界，最终 CCEC 编译与 +perf-clock b1 已再次通过，但没有把旧观测数冒充成最终 ELF 的性能结果。 +S4.5 仍保留全局 exact turn 和每 task 空 region commit，因此没有声称 S4.2 +的 b256 convoy 已修复。S4.6 将只删除 PA Case1 热路径上的 +`WaitForSharedTaskTurn()`、空 region prepare/release 及对应 host 期望，再用 +CPU/CCEC b1/b256 证明 per-slot 依赖协议能够独立闭环。 + +##### 与参考分支的继承和有意差异 + +参考提交 `2866ad73` 已经跑通 shared TensorMap 基本流程，且其目标更接近 +真实 runtime；因此当前 standalone 不是另起炉灶。已经直接继承或保持同构的 +机制包括： + +- `FdwicOutputRef(producer,slot)` 直接寻址，以及每 task 物理 cell 配置 8 个 + output slot； +- `published`、`last_writer`、descriptor 三块分离且控制字独占 cache line； +- consumer 只等待自己消费的 slot，就绪后 invalidate 再复制 descriptor； +- fresh symbol 绕过 ordinary region map； +- 8 个 shared heap shard cursor 加一条 aggregate vend，并以原子分配支持 + 不同 winner 并发。 + +这些机制分别可在参考分支 +`common/state.h:305-322,356-360`、 +`common/runtime_state.h:78-80`、 +`aicore/submit_core.h:626-656,788-821` 找到；standalone 的对应实现位于 +`pa_model.h`、`pa_shared_heap.h` 和 +`pa_scheduler_core.h:875-1194`。它们是后续迁移真实 simpler 时优先复用的 +公共骨架，不因当前发现局部问题而推翻。 + +当前与参考实现存在以下有意差异： + +| 主题 | 参考提交的实际实现 | standalone 当前意见与边界 | +| --- | --- | --- | +| output 数量门禁 | `common/state.h:305-315` 的物理 cell 只有 8 个 slot，但 `runtime/pto_types.h:119-124` 的 `SharedTaskOutputs::add_output_ref()` 只按 `MAX_TENSOR_ARGS` 检查，`pto_orchestration_api.h:42-48` 又按调用方 count 循环。当前 PA 每 task 最多 3 个 output，基本流程不会触发不一致 | `pa_frontend.h:97-105` 在句柄追加时显式限制 `kSharedOutputMaxPerTask=8`，最终发布再次核对 count。迁移必须保留 API/存储同上限门禁，不能因参考 PA 当前规模没越界而照搬这一处接口缝隙 | +| output 发布时机 | `submit_core.h:743-758` 在 Materialize 内复制/flush descriptor、初始化 writer 并发布；`submit_runtime.h:524-552` 随后才进入 Register/Fanin/Build | S4.5 先采用 Build/Complete → writer FetchMax → release turn → `published`；S4.6 删除 turn 后为 Build/Complete → writer FetchMax → `published`。这样 `published` 直接表示可执行状态已建立，失败闭环更清楚；但它可能延迟 consumer，最终真实路径是否保持该顺序必须由 perf-clock 配对决定，不能仅凭 standalone 偏好否定参考快路径 | +| 引用解析位置与执行槽 ABI | 参考 `common/state.h:126-140,153-167` 在 `RingSlot` 和 `BuiltSubtask` 的 ABI 中都保留 `shared_ref_mask/shared_refs`。但当前 HEAD 只有普通 `RingSlot` 路径真正延迟解析：`submit_core.h:990-1001` 先 try-resolve，未就绪引用由 `113-121,216-218` 在执行前等待。winner follower 自 `0f9862a2` 起在 `489-499` 构造 `BuiltSubtask` 时同步调用阻塞式 resolver，并把 mask 固定为 0；不能把字段存在误写成该路径也已 deferred。实测 `RingSlot` 因 shared 形态从 4,864B 增至 5,440B。presubmit 还复用 64B task cell 中的 `deps_prepared`，并在 `DistGlobal` 增加每 worker `prepared_deps`（`common/state.h:292-296,389`） | standalone S4.8 仍在 `CollectSharedFanin()` 内完成 eager publication 等待，但已采用参考 ready-ref 的优点：验证后从 shared cell 直写既有 `LocalSlot`，不再经过 `TaskPayload`。它保持 4,824B slot ABI和 Submit 内集中失败点，却仍会阻塞 producer publication。PA Case1 是普通单-lane slot，参考 deferred resolve 仍是已跑通且很有价值的下一候选；但 standalone 不应为了表面对齐给未使用的 `BuiltSubtask` 增加冗余状态。迁移时还要同时量 Submit 缩短、GM slot 搬运/I-cache、slot 容量与执行前失败语义 | +| writer intent | `submit_runtime.h:302-311` 在 fanin 收集时用 Exchange claim writer，旧值为负则以 producer 作为 fanin；producer 在 `submit_core.h:748-750` 用 FetchMax 初始化，因此不会把已经提前写入的更大 writer 倒退。`345-385,493-507` 还提供 presubmit intent，让唯一 winner 预备依赖、loser 等待。`submit_dependency_smoke` 会调用该 API，但真实 PA 的 `paged_attention_orch.cpp:345` 当前仍调用普通 `rt_presubmit_aiv_task` | “Exchange claim + producer FetchMax”是参考代码很有价值的通用机制，允许 writer intent 早于 descriptor 发布并表达多级 writer 链；不能把“intent API/烟测可用”误写成“参考 PA 已接入该 API”。当前固定 PA Case1 先等待 published、要求 `writer==producer`，再 post-build FetchMax，限制更保守也可能产生性能差。迁移真实 simpler 时应重点配对验证参考组合，并补齐发布后失败和多 writer 顺序证明，而不是把 Case1 限制推广成通用设计 | +| symbol view | `submit_core.h:810-820,840-849` 已处理一维 view flags | standalone ABI 保留字段但 plain ref 之外显式失败；当前 Case1 的 output view 走 `manual_dep`，未提供足够业务用例证明 shared-symbol view。真实迁移需要复用参考 view 语义并增加边界测试 | +| heap 复用 | `submit_core.h:630-651` 支持 shard 取模、等待 H 窗口和最多 64 次 wrap 调整 | standalone b256 有界容量足够，先使用绝对 no-wrap cursor，避免在 generation/复用条件未证明时覆盖旧 descriptor。参考的 wrap 思路应保留为长期方案，但不能只移植取模而省略复用证明 | +| output cell 生命周期 | `runtime_state.h:78-80` 的地址计算带 `task_id & (kFlagCap-1)`，但 `submit_core.h:576-587` 拒绝单轮 `task_id >= kFlagCap`，`control_plane.h:62-68` 在每轮启动时重置全部 cell | 寻址形式不同，但参考与 standalone 当前都依赖“单轮 task id 有界”而不是 generation 实现跨代复用。standalone 直接按最多 1,280 个 task 寻址。真实长期 runtime 若要在同一轮跨 cap，双方都必须补 generation 或等价协议；不能把参考代码中的按位与本身解释成已经支持 ABA-safe 复用 | +| ordinary region | 参考 `state.h:324-343` 和 `submit_core.h:861-929` 使用全局 high-water、bucket 链和 append-only entry，无 H reclaim/绝对 seq；presubmit intent 可提前登记非 symbol INOUT。standalone `pa_model.h:730-763`、`pa_shared_tensormap.h` 使用 per-bucket head/tail ring、绝对 seq 双检和 H reclaim | standalone PA Case1 的 region entry 被严格证明为 0；S4.6 已绕过这条用不到的热路径，但不宣称保留的 ring 原型已成为通用并发 region 算法。真实迁移仍应复用参考的 symbol/region 分流思想，再根据长期容量、回收、ABA 和并发登记证据选择或重构数据结构 | +| 异常路径 | 参考 Materialize 依赖唯一 producer 等业务不变量，writer/published 原子返回值不逐项检查；且 `runtime_state.h:40-47` 在 shared 模式把 `fatal_set()` 固定为 false，即等待方不会消费已写入的 fatal | standalone 增加全量预检、terminal fatal、slot 撤销和故障注入，但不是全事务回滚：output publication 的 cell 可在冷失败清理，writer FetchMax 与 heap FetchAdd 则保留 terminal 现场。真实迁移必须先修复 fatal 广播可见性；诊断逻辑是否原样保留则应冷路径外提并由 perf-clock 判断成本 | + +因此当前结论不是“standalone 顺序优于参考实现”，而是：参考分支已经验证了 +per-slot symbol、writer intent、shared heap 和 region 分流的架构方向； +standalone 负责把当前 PA Case1 的成功/失败边界逐项证明清楚。两边发生差异 +时必须像上表一样记录证据、适用拓扑和性能复核条件。后续若数据证明参考分支 +的提前发布或 intent 路径在同等正确性门禁下更快，应回收 standalone 的保守 +中间实现，而不是为了维护已写代码拒绝更优方案。 + +#### S4.6 PA Case1 去除全局 sequencer + +本步只删除已经被证明恒为空的全局串行路径，不顺带重写 symbol writer、 +发布时机或普通 region 算法。源码变更边界为： + +- `FinishCallbackSubmitBody()` 不再调用 `WaitForSharedTaskTurn()`； +- Register 用 `ValidateEmptySharedRegistration()` 只读核对 + `SharedOutputRef` 和 `manual_dep`，发现任意非空 ordinary-region writer + 立即 fail-closed，不再构造空 `SharedRegionValue[]`； +- 删除 Case1 调用的 `PrepareSharedTaskOrdered()` / + `ReleaseSharedTaskTurn()`，构建后封口收敛为 + `CommitSharedFaninWriters()` → `PublishSharedTaskOutputs()`； +- `committed_tasks/reclaim_upto`、全部 region bucket/slot 必须保持初始化 + 状态；worker 的 map 摘要同样全部为 0； +- 通用 ordered-ring 原语和隔离自测继续保留，供未来非空 ordinary region + 研究，但不再把该自测当成 PA Case1 运行证据。 + +这项删除由固定业务 DAG 支撑，而不是把全局顺序拍脑袋换成“没有顺序”： + +```text +Alloc ────────────────┐ +QK -> SF -> PV ───────┼-> UP + └───────────┘ +``` + +host oracle 固定核对每 batch 五条依赖： +`SF←QK`、`PV←SF`、`UP←SF/PV/Alloc`；symbol resolver 又要求 +`producer_task_id < consumer_task_id`，所以当前图无环。descriptor 就绪由 +每个 `(producer,slot).published` 保证，真正 kernel 执行仍由 slot fanin +对应的 task completion flag 保证。8-shard no-wrap allocator 不依赖 task +顺序；UP 对三个 Alloc symbol 分别是唯一 writer，不需要全局 turn 排序多个 +writer。 + +泳道暂时保留每 Submit 一条 `PrepareMap`，但它是 +`begin=end=materialize_end` 的零时长结构 marker：不额外读钟、不访问 +sidecar，只为复用当前 raw schema、固定记录数和 analyzer 的 required-phase +契约。perf-clock 与 submit-PMU 不实例化该 trace 写入。若未来要彻底删除 +marker，必须同时模式化 raw metadata、host record oracle、converter 和 +analyzer；这不属于本次协议小步，不能为追求表面整洁扩大修改面。 + +CPU 定向测试和提交前只读审查新增以下直接证据： + +- `committed_tasks=0` 时，future QK task 6 可直接完成完整 split-finish、 + BuildWinner 和 output publication，task 0～5 无需先推进全局前沿； +- shared symbol writer 与 `manual_dep` Local/GM writer 通过空 region 验证， + 非 `manual_dep` Local/GM writer 和越界 register mask 在 ordinary-region + append 前失败；Materialize 和 fanin 仍可能在此前访问 shared heap 或读取 + region bucket，因此不能把该门禁误写成 sidecar 零访问； +- 预置 `fatal` 模拟另一核已经广播终止态，winner 在 heap reserve、slot、 + completion 和 symbol 任一副作用前退出；这只保留旧 exact-turn 成功出口 + 原有的终止态检查，不恢复 global sequencer，也不新增 atomic 泳道记录； +- host 复用既有 raw 扫描逐核闭合 + `Claim -> Materialize -> PrepareMap -> Submit` 身份和严格递增 task + 序列，要求 shared `PrepareMap` 的起止都锚定 matching + `Materialize.end`、flags/aux 合法且每 Submit 恰一条；尾部数量再由既有 + 逐核 Submit 数和精确记录数闭合，不增加 raw 字段。 + +当前结果： + +| 门禁 | 结果 | +| --- | --- | +| shared 严格构建及 symbol/split-finish/ring/heap/materialize 自测 | PASS | +| shared b1 scalar-nop0 / real-compute | 全部语义、descriptor、writer、heap 与计算结果 PASS | +| shared b256 scalar-nop0 | 1,280 tasks 全部完成，原 2 秒 convoy 未复现 | +| b256 symbol/依赖 | published 2,048，input loads 1,280，writer commits 768，fanin edges 1,280 | +| b256 region 终态 | committed 0，reclaim -1，bucket/slot 全空 | +| b256 heap | 8 shard 各 25,821,184B，总 vend 206,569,472B | +| private b1/b256 宏隔离 | 全部断言 PASS | +| shared b1 raw → converter → exclusive analyzer | 记录数精确、0 drop、转换与分析 PASS | +| ASan/UBSan/leak | 最新 symbol 与完整 split-finish/故障路径均 PASS | +| CCEC shared b1 real-compute | 最终 terminal-fatal 门禁 ELF 全部语义与 4 个 active tile PASS；perf-clock 单样本 70.279 us | +| CCEC shared b256 scalar-nop0 | 提交前审查前的 S4.6 ELF：96 核全部活跃，1,280 tasks 完成；perf-clock 单样本 3,228.844 us | +| CCEC shared b256 real-compute | 提交前审查前的 S4.6 ELF：192 个 active tile 全部 PASS；perf-clock 单样本 5,982.840 us | +| CCEC private b1 real-compute / b256 scalar-nop0 | 全部断言 PASS；单样本 70.707 us / 3,300.478 us | +| CCEC shared swimlane b1 | 最终 ELF 4,168 records，expected 4,168，0 drop；零时长 marker、converter/analyzer PASS | +| CCEC shared submit-PMU register b1 | 480 calls 精确；phase time share 0.8020%；owner restore/cleanup PASS | + +最终 shared 泳道位于 +`outputs/pa_scheduler_shared_swimlane_20260725_062217_2130280/ccec/`。 +`PrepareMap` 记录仍在，但所有该 phase 的 begin/end 相同；host 总记录数、 +raw marker validator、converter 与 exclusive analyzer 均闭合。其中 +converter/analyzer 负责结构配对，host validator 额外保证 shared marker +严格为零时长、匹配同 task 的 `Materialize.end`，并按每核 task 0..N-1 +顺序出现。 + +##### 与参考分支的继续对齐和暂不照搬 + +参考提交本身没有 PA 全局 exact-turn，因此 S4.6 是向参考架构靠拢：fresh +symbol 只按 per-slot 状态同步，ordinary region 走另一套结构。以下两点是 +参考实现已经跑通、且可能比 standalone 更有性能价值的机制，但本步有意不 +混入： + +1. 参考 `submit_core.h:994-1001` 允许未就绪 ref 留在 ring slot，执行前再 + resolve；standalone 当前仍在 `CollectSharedFanin()` 内阻塞 Submit。 + deferred resolve 可能缩短 Submit，但会改变 slot payload 与执行前失败 + 边界，必须另做一小步和 perf-clock 配对。 +2. 参考 consumer 在 `submit_runtime.h:302-311` 用 Exchange 声明 writer， + producer 再在 `submit_core.h:748-750` 用 FetchMax 初始化，允许 writer + intent 先于 producer publish。standalone 当前只支持 Case1 单后继 writer + 并在 Build 后提交。参考组合是通用多级 writer 链的重要候选，不能因为 + standalone 当前故障闭环更保守就忽略；也不能在没有补齐多 writer 顺序、 + fatal 可见性和发布后失败测试前直接照搬。 + +所以本步的意见差异不是反对参考方案，而是把变量隔离：先证明删除 global +sequencer 本身，再分别测 deferred resolve 和提前 writer intent。当前 +CCEC b256 已证明 A5 DCache/DCCI、per-slot 等待、writer commit 和真实 +计算能够在无全局前沿时闭合。3.229 ms/5.983 ms 与历史 exact-turn 数字来自 +不同阶段的独立 ELF，只能说明 convoy 已从结构和活性上消失，不能直接当成 +纯 sequencer 的稳定净收益；后续性能结论仍用冻结 ELF 做配对多轮。本阶段 +只覆盖 CPU 与 CCEC，不做 AscendC。 + +##### S4.7 冻结配对与后续参考机制验证顺序 + +S4.7 在 clean 提交 `dc22d076` 上连续构建 private/shared CCEC perf-clock， +随后复制为只读冻结件；循环内没有重编译。private/shared mixed ELF 的 +`.text + .rodata` 分别为 126,052B/133,208B，kernel SHA256 分别为 +`f64b87e...1c46a`/`a414beb...d7402`。冻结路径为： + +```text +outputs/perf_clock_freeze_dc22d076_20260725_065902/ +``` + +每个 batches 先运行不计入统计的 warm-up ABBA，再运行 +`ABBA/BAAB` 交替的 6 个正式 block；每个 block 各含两个 private 和 shared +独立进程，所以每模式有 12 个正式样本。56 个进程全部闭合模式、负载、 +manifest 和语义断言，正式样本结果如下： + +| batches | 模式 | 最小值 | 中位数 | 最大值 | +| ---: | --- | ---: | ---: | ---: | +| 1 | private | 62.923 us | 66.557 us | 70.749 us | +| 1 | shared | 64.641 us | 65.361 us | 70.824 us | +| 256 | private | 4,072.002 us | 5,003.790 us | 5,726.435 us | +| 256 | shared | 6,050.200 us | 7,209.016 us | 7,622.198 us | + +配对差值以每 block 的 +`mean(shared 两样本) - mean(private 两样本)` 计算。b1 的差值中位数为 +`-0.458 us`，范围 `-1.414～+0.084 us`，没有固定成本退化；b256 六个 +block 全部是 shared 更慢，差值中位数 `+2,149.766 us`，范围 +`+1,491.463～+3,000.363 us`，相对差中位数 `+43.430%`。原始 56 份日志、 +逐样本 JSON 和配对摘要保存在冻结目录的 +`runs/paired_real_compute/`。 + +因此 S4.6 的准确结论是：global convoy 和活性问题已经消失，但 shared +consumer 在 Submit 内等待 producer `published` 的规模效应仍然显著。 +`[METRIC] fanin_loads` 只统计 task completion flag 轮询，不包含 +`WaitForSharedOutputPublished()` 的 direct `Ops::Load`；b256 shared 该字段 +更小不能解释成总等待更少。 + +最新 shared b1 诊断样本中，QK/SF/PV/UP 的 Fanin 约为 +`2.031/10.708/30.913/44.673 us`。它与上述 perf-clock 使用不同 ELF， +只能定性支持 publication 等待方向，不能做数值相减。当前每 batch 发布 +8 个 descriptor、消费 5 个纯 INPUT，并提交 3 个 UP INOUT writer；b256 +对应 2,048/1,280/768。 + +冻结后的低风险到高风险顺序固定为： + +1. **ready ref 直接落 slot**：published 等待和 writer 语义不变，只把 + descriptor 的搬运从 + `shared cell -> TaskPayload -> LocalSlot` 缩成 + `shared cell -> LocalSlot`。这一步不增加 slot ABI、不改 atomic，先单独 + 判断 2,048 个 shared ref 的 128B 中间复制是否值得删除。 +2. **只延迟纯 INPUT**：shared-only `LocalSlot` 增加 mask/ref；已经 published + 的 ref 仍直接解析，未发布的纯 INPUT 才随 slot 进入执行前 resolver。 + INOUT/OUTPUT_EXISTING 保持 eager 等待和 Build 后 FetchMax，避免把 resolve + 时机与 writer 协议混成一个变量。 +3. **按数据调整在途容量**：只有 RingBackpressure 证明确有需要时才单独改变 + slot 数。standalone 当前 4 slot、实际最多 2 个可用，参考 shared 为 + 14 slot；不能与 deferred resolve 同笔修改，否则无法归因。 +4. **迁移真实 simpler**：以上边界一旦在 standalone 闭合，就复用参考分支 + 已跑通的普通 `RingSlot` resolver 骨架进入真实路径，不继续在 standalone + 扩展当前 PA 不使用的通用 MIX 或多 writer 能力。 + +deferred 版本必须新增故障门禁：fanin flag 已 ready 但 published 缺失/错误 +时立即 fatal；resolver 失败后清空 slot 且 `occupied_count` 只减一次；不得 +执行 kernel 或发布 completion；多 ref mask、非法 ref、view、slot 复用和 +延迟 publisher 都要覆盖。参考 `submit_core.h:216-219` 的 +`execute_slot()` 在 resolver 返回 false 时直接返回，而后续 drain 仍会减少 +occupied 计数，slot 本体没有在该点清除；基本 PA 成功流程不会触发，但该冷 +失败路径不能原样照搬。 + +提前 writer intent 暂不排在真实 PA 迁移之前。当前 Case1 没有多级 writer +链，提前移动 b256 的 768 次 RMW 不会减少 atomic 数量，UP 后也没有新的 +consumer 可被提前解锁。将来研究时还必须把“winner fanin 阶段的 +Exchange(last_writer)”与可选 presubmit +`deps_prepared/prepared_deps` barrier 分开；参考真实 PA 目前只用了前者， +后者主要由 smoke 接入。逆序 writer、未来 writer 越过早期 reader、多 symbol +部分提交、fatal 释放 loser 和 task-id 复用没有证明前，不能把 Case1 的单 +后继事实推广成通用协议。 + +deferred 还可能只是把工作从 Submit 移到执行/FinalDrain。后续配对除了 +`first-submit-begin -> last-submit-end` 主值，还要用不含泳道/PMU 的独立 +干净边界核对 finish/最终 drain；若 Submit 降低而后者等量增加，只能记为 +工作后移，不能记成整体性能收益。参考 `RingSlot` 从 4,864B 增至 5,440B， +每槽多 576B；相应 GM 搬运、slot 容量和 I-cache 代价都要实测，不能由源码 +大小推断。 + +### 阶段 R0：迁移真实 simpler + +只有 S0～S4.7 全部闭环后，才按已验证结构依次迁移： + +1. 真实构建身份、缓存隔离和 CPU/CCEC ABI 握手； +2. private ring 同构化； +3. shared facade 与经独立证明的 ordinary-region backend；当前 ordered + ring 只是隔离原型，不能重新接成 PA 全局 sequencer； +4. PA winner-only/fresh symbol/shared heap； +5. MIX 与 A5 非一致缓存语义； +6. PA Case1 a5sim golden； +7. PA Case1 A5 b1； +8. `perf-clock` 多轮配对，随后用 `swimlane`、`submit-pmu` 解释变化； +9. private 默认路径完整回归。 + +真实路径的每一步都应能追溯到 standalone 已通过的协议测试；不得直接 +cherry-pick 参考分支的 append-only map、可选 writer intent 或无 generation +heap。 + +## 12. 可直接复用与不应直接复用的清单 + +### 12.1 建议复用 + +- Claim-first API，以及 PA 已验证的 winner-only 重参数构造方式； +- loser 返回符号 output ref 的轻路径； +- `(task_id, output_slot)` fresh output 快路径； +- descriptor 普通写后 flush，consumer invalidate 后读取； +- `last_writer` 与 immutable descriptor 分离； +- wrong-role worker 跳过无意义 Claim； +- 空 EfDrain 按需跳过； +- MIX winner 先 resolve、再发布 follower 的基础 handoff； +- shared/private 默认隔离，private 保持默认。 + +### 12.2 不应原样复用 + +- 仅靠 ambient `CXXFLAGS` 切 ABI； +- 仅靠手工删除 cache 防止混合镜像； +- 业务 orchestration 大量散布 `#if PTO_FDWIC_SHARED_MAP`； +- 调用方可选的 `_with_region_intent` 正确性入口； +- append-only `SharedRegionMap`； +- 65536 task 的无 generation 直达大表； +- 无 generation 的 heap shard wrap； +- shared `fatal_set() == false`； +- output_count 手填且不校验； +- 第 17 个 fanin 静默丢弃； +- 没有 raw artifact 支撑的性能结论。 + +## 13. 本次验证命令摘要 + +目标分支以独立 ref 和 detached worktree 审查，没有切换当前开发分支。 + +功能差异范围： + +```bash +git diff --stat f5da1a2e..2866ad73 +git log --oneline f5da1a2e..2866ad73 +``` + +布局探针分别使用： + +```bash +g++ -std=c++17 -D__CPU_SIM ... -o /tmp/fdwic_layout_private +g++ -std=c++17 -D__CPU_SIM -DPTO_FDWIC_SHARED_MAP=1 \ + ... -o /tmp/fdwic_layout_shared +``` + +profile-off 最小复现： + +```bash +printf '#include "pto_types.h"\n' | + g++ -std=c++17 -D__CPU_SIM -DPTO2_PROFILING=0 \ + -I... -x c++ -fsyntax-only - +``` + +shared 组合再增加 `-DPTO_FDWIC_SHARED_MAP=1`。两次返回码均为 1，失败位置为 +`pto_types.h:906-908`，断言实际值均为 `(32 == 0)`。 + +## 14. S4.6 阶段决策摘要与当时尚未闭环的问题 + +截至 S4.6，standalone 已经证明： + +1. private/shared 构建身份、产物目录、manifest 和 host/device ABI 不能混用； +2. private ring 以及 shared PA Case1 的 per-slot symbol、8-shard no-wrap + heap、winner-only 构参/物化能够生成可比较的依赖和 writer 签名； +3. 固定 Case1 DAG 在没有 global sequencer 时可在 CPU/CCEC b1、b256 闭合； +4. 构建后 writer commit 与 output publication 的正常/终止边界已由故障注入 + 固化。 + +这些结论只覆盖固定 PA Case1，尤其不能改写成“任意 winner 到达顺序下的通用 +多级 writer 链已解决”。进入真实 simpler 前仍有以下独立问题： + +1. **非空 ordinary region**：现有 ordered ring 只是隔离原型，PA 热路径没有 + 接入；需在参考 append-only map、当前绝对 seq ring 或新方案之间用真实 + 容量、回收、ABA 和并发登记证据做选择。 +2. **resolve 时机**：standalone 在 Submit 的 `CollectSharedFanin()` 阻塞， + 参考实现把 shared ref 保留到 ring slot 并在执行前 resolve。后者可能更快， + 但会扩大 slot ABI并改变执行前失败边界，必须独立验证。 +3. **通用 writer intent**：当前只支持 Case1 单后继 writer；参考的 consumer + Exchange + producer FetchMax 以及 presubmit intent 能表达多级链，但还需 + 补 fatal 可见性、多个 writer 顺序和 publication 失败测试。 +4. **跨代复用**：`shared_outputs[1280]` 和 no-wrap heap 只覆盖当前单轮上限， + 没有 generation，也不能从参考代码的 task-id 取模推断已经 ABA-safe。 +5. **真实路径终止语义**：参考 shared `fatal_set()` 当前不会广播，迁移前必须 + 修正；standalone 的诊断与冷失败逻辑是否保留则由 perf-clock 判断成本。 +6. **性能定案**：先冻结 private/shared perf-clock ELF 做配对多轮，再决定 + 是否采用参考的提前发布、deferred resolve 与 writer intent；泳道和 PMU + 只解释变化，不直接与 perf-clock 数字相减。 + +因此迁移目标是复用参考分支已跑通的架构骨架，并把上述未证明部分拆成独立 +小步；不能重新接回已经被 S4.2/S4.6 否定的 PA 全局 sequencer，也不能因 +standalone 当前故障门禁更完整就排斥参考实现的高性能路径。 + +## 15. 当前分支实施记录 + +### 2026-07-24：S0 模式身份与 ABI + +本阶段只修改 `tests/atomic_probe/pa_scheduler`，没有修改 +`src/a5/runtime/fully_distributed_within_core` 或真实 PA。 + +已完成： + +- `run.sh` 增加 `--tensormap private|shared`，默认 private，并从 benchmark + 参数中消费该选项； +- CPU/CCEC 产物按 `//` 隔离； +- CCEC swimlane 与 submit-pmu 使用同一 manifest schema，固定 + mode、variant、phase 和完整运行件 SHA256； +- `RunConfig` 在原有 16B 尾部写入 magic、ABI version、mode 和 + `sizeof(SchedulerState)`，device 在解释 worker 状态前核对； +- 发现并修复旧 PMU `reserved[4]` 越界：原数组只有四项，索引 4 实际落入 + 相邻 `WinnerWorkloadConfig::mode`；现已迁到独立 64B + `PmuProbeConfig`； +- S0 当时在 shared backend 尚未接入时保留编译期门禁，不生成伪 shared + 产物；S2 已用真实 sidecar 替换该门禁，这不是当前运行限制。 + +验证结果： + +| 检查 | 结果 | +| ---- | ---- | +| CPU private build + PollBatch 自测 | PASS | +| CPU private b1 smoke | 全部语义断言 PASS | +| CCEC private swimlane 编译 | PASS | +| CCEC private submit-pmu none 编译 | PASS | +| 两类 CCEC manifest/SHA256 启动前校验 | PASS | +| S0 历史 shared CPU fail-closed，且无 executable | PASS；该门禁已在 S2 删除 | +| 重复/非法 mode、缺失 shared 产物负测 | PASS | +| standalone Python 回归（用户 `.venv`） | 100 passed | +| 四个 shell 脚本 `bash -n` | PASS | +| `git diff --check` | PASS | + +本阶段没有运行 A5/A5sim。S0 不改变 TensorMap 算法，也不声称有性能收益； +CCEC 编译只证明三镜像能够用同一模式构建。S1 已在后续阶段中完成， +见下节。 + +### 2026-07-24：S1 standalone private ring-per-bucket + +本阶段仍只修改 `tests/atomic_probe/pa_scheduler`，没有修改 +`src/a5/runtime/fully_distributed_within_core`、真实 PA 或其他 simpler +runtime 路径。 + +实现布局： + +| 项目 | S1 布局或语义 | +| ---- | ---- | +| bucket / slot | 128 buckets × 128 slots，总容量 16,384 | +| `MapEntry` | 48 B；前 32 B 为 region + producer，后 16 B 仅 ABI 保留 | +| `TensorMap` | 823,312 B，保持原 size | +| `WorkerState` | 9,231,296 B，map/slot/payload 后续 offset 不变 | +| private 并发纪律 | 单 worker 独占；普通 `uint64_t head/tail`，无 atomic/seq | +| 逻辑回收 | `task_entry_counts[1024]` 精确维护 `live_count`，同步推进 `alive_floor/cleaned_upto` | +| 物理回收 | lookup/insert 访问桶时执行 lazy `RetireBucket` | +| lookup | 扫描全部合法槽，过滤 `producer < alive_floor`，取最大重叠 producer | +| overflow | bool 逐层返回；不覆写、不推进 tail，Submit 设置 fatal | + +保持整个 `TensorMap` 大小的同时，head/tail 放在旧 entries 后；30 KiB ABI +保留区让 `task_entry_counts` 继续位于旧 `task_heads` offset，四个控制字仍 +位于原尾部 16 B。shared 所需的 per-slot seq 没有借用 `MapEntry` 保留区， +避免尚未验证的并发协议污染 private 基线。 + +`H=64` 的 PA Case1 最多只有 52 个 logical live entry，因此当前每桶 128 +槽不会溢出；这个上界只服务当前 Case1。通用图和 shared 模式仍必须根据活跃 +跨度单独证明容量，不能把本结论外推为固定配置。 + +验证结果： + +| 检查 | 结果 | +| ---- | ---- | +| private ring 独立 ABI/区间/retire/wrap/overflow/differential | PASS | +| 独立测试 ASan/UBSan | PASS | +| CPU private build + PollBatch 自测 | PASS | +| CPU private b1 smoke 全部调度语义断言 | PASS | +| CPU private b256、零计算、关闭泳道的完整语义回归 | PASS | +| CCEC private swimlane 三镜像、mixed ELF 与 manifest 编译 | PASS | +| `git diff --check` | PASS | + +A5 本轮均使用 CCEC private、关闭泳道、`real-compute`。两组 b1 来自同一 +构建变体，比较 Submit host span；这些小样本只用于检查明显回退： + +| 样本 | `submit_span_us` | +| ---- | ----: | +| S0 b1 median | 64.173 | +| S1 b1 median | 61.666 | +| S1 b256 单次 | 3,862.246 | + +b1 只有小样本，2.507 us 差值不能解释成稳定性能收益；当前只能得出 +private ring 没有表现出回退。b256 只作协议和规模回归记录，不是性能基线。 +后续是否保留性能结论仍由相同 ELF 口径的多轮配对测试决定，不能与带泳道、 +submit-pmu 或真实 PA 的绝对时间直接相减。 + +### 2026-07-24：S2 standalone shared ordered ring + +本阶段仍严格限定在 `tests/atomic_probe/pa_scheduler`，没有修改 +`src/a5/runtime/fully_distributed_within_core`、真实 PA 或其他 simpler +runtime 路径。S2 的目标是先得到可证明、可失败终止、可与 private 比较的 +shared region map 正确性基线，不在同一阶段同时改 winner-only 构参、fresh +symbol 或 shared heap。 + +#### 数据布局与发布协议 + +shared 构建在完整 production prefix、standalone controls 和 `results` 之后 +追加 `SharedTensorMapSidecar`。它为 64B 对齐、2,119,808 bytes，既有 +`WorkerState`、`RunConfig` 和结果 offset 均不移动： + +| sidecar 项 | 数量与语义 | +| ---- | ---- | +| `committed_tasks` | 连续发布的 task 数；初始 0，task N 完成后为 N+1 | +| `reclaim_upto` | 可回收 producer 的 inclusive 上界；初始 -1 | +| `core_progress` | 96 条独占 atomic cache line；每核精确记录最后观察完的 task | +| bucket state | 128 个桶；head/tail 各占独立 64B cache line | +| region slot | 128×128=16,384 个；64B payload 与 64B 绝对 `seq` 分离 | + +S2 仍保留 compete-first eager 参数构造、所有核 Materialize 和 per-worker +heap。“winner-only”只发生在共享 region 写入：每个 task 只有 winner 收集 +register entries 并追加，零 entry task 也必须发布一次空 commit。所有 worker +遵守同一 task 顺序边界： + +1. task N 的 lookup 前等待 `committed_tasks >= N`，确认前 N 个 task + 已连续发布； +2. winner 在持有 N 的 append turn 时先预检整 task 容量，再追加全部 entry， + 最后以返回值参与判断的 Exchange 把 commit 从 N 推进到 N+1； +3. loser 等待 N+1，winner 直接使用自己的发布结果；随后每个 worker 都把 + 本核 progress 从 N-1 精确推进到 N。 + +lookup 只接受 `producer ∈ [max(0, N-H), N)`，从协议上同时排除 stale 与 +future producer。slot 复用不只依赖环下标：writer 先把旧绝对 `seq` 失效， +对 payload 执行 invalidate、普通字段写和 DCache flush，再发布新 `seq` 与 +tail；reader 执行“第一次原子读 `seq` → DCache invalidate payload → 拷贝 +本地快照 → 第二次原子读 `seq`”。CCEC/AscendC 的 region hook 逐 cache line +执行 `dcci` 并以 `dsb` 收口；CPU hook 只模拟顺序。两次 seq 不等、payload +非法或 cursor 破坏都属于协议错误，不能静默当成 lookup miss。 + +reclaim candidate 按 `min(core_progress)-H-1` 计算并只允许单调推进。append +在写任何槽之前检查本 task 的全部目标桶、容量和预期旧 seq；容量暂时不足时 +等待慢核 progress，并在循环中协作 drain。预检可以先推进已经满足 reclaim +条件的 bucket head；这只清除已过期历史，不会发布当前 task 的部分 entry， +所以普通容量不足仍保持当前 task 的 all-or-nothing。协议破坏、fatal 或 +watchdog 会让运行显式失败。 + +#### host 独立校验与跨模式签名 + +host 初始化 sidecar 后将它作为独立 H2D 区段传输；运行后再独立 D2H。最终 +校验不信任 device 汇总字段，而是直接遍历 sidecar，检查： + +- `committed_tasks == task_count`，且 96 条 progress 都等于最后一个 task； +- reclaim 不回退且不越过由最终 progress 推导出的上界； +- 每桶 `0 <= tail-head <= 128`，每个 live slot 的 `seq` 与绝对 cursor + 一致； +- payload 区间、producer、reserved 和地址 hash 都合法； +- 最终逻辑窗口只保留预期 UP entries，entry 数和 producer 分布准确。 + +private 与 shared 都按 +`(bucket, buffer_addr, lo, hi, producer)` 生成同一规范化逻辑 map 签名； +每条 PA fanin 又按 `(consumer, producer)` 生成与存储布局无关的依赖边签名。 +签名不是替代逐字段断言，而是在上述独立校验之后锁定两种实现生成了同一逻辑 +结果。 + +#### 验证证据 + +| 场景 | 结果 | dependency signature | logical map signature | +| ---- | ---- | ---- | ---- | +| CPU private/shared b1 | 全部语义与终态断言 PASS | `5cb454393ed48dcb` | `3a3d526c9b23c3db` | +| CPU private/shared b256 | 全部语义与终态断言 PASS | `b7d985d6edb07078` | `556bec7ec8d0f323` | +| CCEC A5 private/shared b1 | 全部语义与终态断言 PASS | `5cb454393ed48dcb` | `3a3d526c9b23c3db` | +| CCEC A5 private/shared b256 | 全部语义与终态断言 PASS | `b7d985d6edb07078` | `556bec7ec8d0f323` | +| AscendC A5 shared b1 | 全部语义与终态断言 PASS；Submit `231.884 us` | `5cb454393ed48dcb` | `3a3d526c9b23c3db` | + +定向 shared ring 用例还覆盖零 entry ordered commit、同桶多 entry、多桶 +lookup、`[N-H,N)` 边界、慢核 reclaim、三圈 seq 复用、确定性 ABA 注入、 +满桶预检 all-or-nothing、逆序 actor 到达和逻辑 tuple differential，并通过 +ASan/UBSan。CPU 在这里证明的是原子调用顺序、状态机、容量和逻辑差分； +它不具备 A5 非一致缓存，不能证明设备 DCache/DCCI 或 A5 原子竞争行为。 +CCEC 上板 b1/b256 与 AscendC shared b1 PASS 才是当前设备缓存协议的验证证据。 + +#### 性能边界与已撤回实验 + +同源 CCEC `--no-swimlane` 构建各取一个 b256 单进程单次诊断： + +| 模式 | `submit_span` | +| ---- | ----: | +| private | 3.816830 ms | +| shared S2 | 68.796708 ms | + +这不是多轮性能统计，不能把 64.979878 ms 差值细分成某条指令的稳定成本； +但数量级差异已经足以说明 S2 这套“96 核逐 task commit 前后强一致”只能作为 +正确性基线，性能不可接受，不能直接迁移到真实 simpler。 + +曾做过一次过程实验：只让 winner 承担 commit 前等待，b1 得到 76.558 us； +同一改法在 b256 只推进到 `committed_tasks=819` 就触发 watchdog。该结果证明 +简单删除 loser 的前置顺序边界会破坏长序列活性，不能当成性能优化。实验代码 +已完整撤回，不属于 S2 提交，也不改变上述 private/shared 正确性签名。 + +S2/S2.5 shared sidecar 的原子调用会落入既有 Submit 与业务阶段 span，但还没有 +逐条接入 atomic 泳道 wrapper。因此现阶段泳道不能用来声称“shared 协议原子 +已经全量可见”；这项观察能力留到后续独立小步补齐。本阶段的正确性证据来自 +host 对 sidecar 的逐字段校验、依赖边签名和规范化 logical-map 签名。 + +### 2026-07-24：S2.5 ordered-winner reclaim + +S2 的 b1 过程实验已经提示 loser 全局等待很重，但直接让 loser 前跑后， +`min(core_progress)` 仍由最慢核决定，b256 最终停在 commit 819。S2.5 没有 +把该失败补丁原样恢复，而是重新建立只依赖 ordered winner 的回收证明： + +1. shared map 的唯一 reader/writer 是 task winner； +2. winner N 只在 `committed_tasks == N` 时进入 lookup，commit N 证明 + 所有更早 winner 已结束 map 访问； +3. winner N 完成本 task lookup 后，未来 task 的合法查询下界为 N-H； +4. 因而当前 turn 可以回收 `producer <= N-H-1`，不需要等待 loser replay + 或 96 核 progress。 + +落地改动严格限定在该证明内： + +- PrepareMap 只有 winner 等 exact turn；如果看到 commit 已越过 N，按重复或 + 迟到 actor 显式失败，不能把 `>=` 当成成功； +- loser 在 Register 不再读写 shared sidecar； +- `SharedRefreshReclaimForTask()` 先核对 exact turn，再以 signed 64-bit + 计算 `max(-1,N-H-1)`；candidate 回退、陈旧 actor 和 future actor 都在 + 任何共享写入前失败； +- 删除 `core_progress[96]`，sidecar 从 2,119,808 bytes 缩至 + 2,113,664 bytes；bucket 和 slot offset 分别变为 128、16,512； +- preflight 仍可以发布已经安全的 stale-head 回收，但容量检查失败时不发布 + 当前 task 的 payload、seq、tail 或 commit。exact turn 的回收上界已经 + 固定，容量仍不足时直接 fatal，不再无意义地 spin/drain 到 watchdog； +- 零 entry task 仍 refresh reclaim 并提交 N+1，保持 sequencer 连续。 + +#### 正确性与活性结果 + +定向 ring 用例更新为 ordered-turn 口径，覆盖： + +- `N=0`、`N=H`、`N=H+1` 和 Case1 最后 task 的 reclaim 边界； +- 零 entry commit、陈旧/future actor、reclaim 回退拒绝； +- 同桶三圈绝对 seq、ABA 注入、逆序 actor 到达； +- 满桶 all-or-nothing，以及合法回收 stale head 后仍然满的失败路径。 + +CPU private/shared b1/b256、ASan/UBSan、leak 检查和 100 项 Python 测试均 +通过。CCEC A5 shared b1/b256 的关键结果为： + +| 场景 | Submit | commit/reclaim | append/live | dependency/map signature | +| ---- | ----: | ---- | ---- | ---- | +| b1 | 74.683 us | `5 / -1` | `4 / 4` | `5cb454393ed48dcb` / `3a3d526c9b23c3db` | +| b256 | 26.556193 ms | `1280 / 1214` | `1024 / 52` | `b7d985d6edb07078` / `556bec7ec8d0f323` | + +b256 本轮观察到的 `physical_entries=142` 大于逻辑 live 52 并非泄漏： +bucket head 只在 +该桶下次被触达时惰性推进，host 仍按 `logical_floor=1215` 过滤旧 producer。 +历史 append 总数取 tail 求和，物理驻留取 `tail-head`，逻辑窗口按 producer +过滤，三者不能混用。 + +与 S2 同源 CCEC 单样本相比，b1 从 248.477 us 降到 74.683 us，b256 从 +68.796708 ms 降到 26.556193 ms，后者下降约 61.4%。这些不是多轮稳定性能 +统计，不能继续拆成单条 atomic 的稳定成本；但 b256 已完整跨过此前的 819 +停点，并保持所有逻辑签名不变，足以证明 ordered-winner reclaim 同时恢复 +活性并消除了主要结构性等待。相对 private b256 3.816830 ms 仍有明显差距， +下一阶段应转向 fresh-output symbol 与 winner-only Materialize，不把 S2.5 +误写成性能终态。 + +### 2026-07-24：S3.1 fresh-output symbol + +本阶段只在 standalone 中迁移 fresh-output symbol，没有同时改变分配主体。 +开发和验收范围明确为 CPU/CCEC。 + +#### ABI 与状态布局 + +shared 模式新增的稳定引用与返回句柄为： + +| 类型/状态 | 当前布局 | +| --- | --- | +| `FdwicOutputRef` | 16B；保存 producer、output slot 和预留的一维 view 字段 | +| `SharedTaskOutputs` | 8B；保存 producer 和连续 output count | +| `SharedOutputCell` | 2,048B；8 个 publish line、8 个 writer line、8 个 descriptor | +| `shared_outputs` | 1,280 个 cell，共 2,621,440B，按 task id 直接寻址 | + +`shared_outputs` 追加在 S2.5 的 region ring 之后，offset 为 2,113,664。 +因此 shared sidecar 从 S2.5 的 2,113,664B 扩为 4,735,104B；region ring 的 +committed/reclaim、bucket 和 slot offset 均未改变。完整 shared +`SchedulerState` 在 CPU 非 split 构建中为 1,011,851,072B，在 CCEC split +构建中为 1,011,857,216B。构建身份 ABI 版本同步推进到 2，host/device +继续核对 mode、版本和完整 state size。 + +这里没有 generation：standalone 最多 1,280 个 task，输出表不取模，也不 +复用 task id。它是本阶段明确的有界条件，不应被外推为真实 runtime 的长期 +复用方案。 + +#### 发布、解析与 writer 链 + +所有 worker 仍执行 eager 构参、Materialize 和 per-worker private heap +分配；b1/b256 的全局 Materialize output 数仍是 768/196,608。只有 ordered +winner 持有 task N 的 exact turn 后执行以下动作： + +1. 只读预检本 task 全部 output slot 的 source、`published` 和 + `last_writer`； +2. 用 FetchMax 把全部 `last_writer` 从 -1 初始化为 N； +3. 将已经物化的 descriptor 复制到 `shared_outputs[N]` 并逐个 flush； +4. store barrier 后发布每个 `published`； +5. 提交空/非空 region delta，把 `committed_tasks` 从 N 推进为 N+1。 + +预检失败时没有共享写入；预留阶段的异常旧值会撤回本 task 已预留的前槽。 +发布位 Exchange 若在 exact-turn 契约外观察到异常旧值，则冷路径撤回全部 +published/last_writer 并清空、flush 本 task descriptor。因此不同 descriptor +的重复发布和后槽异常都不会污染既有 descriptor 或留下多输出 task 的部分 +控制状态。 + +consumer 只接受 flags/view 全零的 plain `FdwicOutputRef`。第一遍先校验 +全部引用，读取每个 symbol 的 `published`，并只对 `INPUT` 读取 +`last_writer`；它不修改 writer、payload、统计或输出 fanin。全部通过后才对 +`INOUT`/`OUTPUT_EXISTING` 执行返回旧值的 Exchange、复制 descriptor，并 +一次性发布计数与 fanin。 +writer 必须处于 `[0,current_task)`，同一 task 的重复写引用会被拒绝，再通过 +既有 `AddFanin` 去重。带 view 的引用目前显式失败，没有 deferred resolve +或静默回退。 + +实现过程中实际遇到 CCEC 编译器拒绝 `[[block_local]]` runtime state 包含 +非平凡构造函数。正确修复不是使用 `block-local-init` 绕过，而是让 +`FdwicOutputRef` 和 `SharedTaskOutputs` 保持 trivial POD，以显式 +`InvalidSharedOutputRef()` 工厂构造非法值，并用 trivial +`static_assert` 固化约束。 + +#### 与 private TensorMap 的可比口径 + +PA Case1 的 fresh symbol 和 `manual_dep=true` output view 都跳过 shared +region lookup/register，所以 S3.1 的 shared region raw ring 为空： + +- b1/b256 的 region append、physical entry、logical entry 均为 0； +- raw 空表签名固定为 `14650fb0739d0383`； +- ordered commit/reclaim 继续由每 task 的空 delta 推进，b256 终态为 + `committed_tasks=1280`、`reclaim_upto=1214`。 + +private 仍在 region ring 中保存三个 Alloc writer 和 manual output view。 +因此不能直接比较两种 raw 表。host 使用同一规范化 writer 口径：shared 从 +实际回读的三个 Alloc symbol cell 投影最终 writer，再按 Case1 约定补入 +manual output view；private 使用其 raw logical writer。两者按相同 +bucket/order 计算后必须得到同一签名： + +| 规模 | dependency signature | normalized writer signature | +| --- | --- | --- | +| b1 | `5cb454393ed48dcb` | `3a3d526c9b23c3db` | +| b256 | `b7d985d6edb07078` | `556bec7ec8d0f323` | + +其中 fresh-symbol writer 来自 shared 执行终态的独立校验；manual view +则是约定补齐，不是从 shared 执行态独立取证。因此该签名证明的是 PA Case1 +已观测 fresh-symbol writer 与约定 manual-view 投影的一致性，不声称 +shared raw region 与 private raw region 具有相同物理内容，也不把它作为 +manual view 单项对等证据。 + +#### 正确性与性能结果 + +门禁结果： + +- CPU private/shared b1、b256：全部调度、输出、fanin、map 和终态断言 PASS； +- 新增 shared symbol 定向测试与 shared ring：ASan/UBSan/leak PASS； +- standalone Python：100 项 PASS； +- CCEC shared `submit-pmu none` b1：调度、symbol、PMU owner + restore/cleanup 全部 PASS； +- CCEC private/shared b1、b256：全部断言 PASS。 + +shared symbol 计数严格闭合： + +| 规模 | published outputs | INPUT loads | INOUT exchanges | +| --- | ---: | ---: | ---: | +| b1 | 8 | 5 | 3 | +| b256 | 2,048 | 1,280 | 768 | + +同一阶段取得的 CCEC 单进程单次结果为： + +| 模式 | b1 Submit | b256 Submit | +| --- | ---: | ---: | +| private | 73.318 us | 3.808011 ms | +| shared S3.1（fail-closed 修正后） | 86.552 us | 27.094219 ms | + +S2.5 shared b256 的同类单样本为 26.556193 ms。S3.1 在正确性审计前曾取得 +23.562916 ms，但当时重复发布和后置非法引用的失败路径可能留下部分共享 +状态；补齐全量发布预检与两遍 resolver 后，最终同类单样本为 +27.094219 ms。相对 S2.5 约 +2.03%，但这不是多轮稳定性能基线，不能据此 +宣称稳定回退； +被撤销的 23.562916 ms 也不能作为有效 S3.1 基线。当前所有 worker 仍重复 +Materialize 和 private heap 分配。S3.2 应继续以独立提交收敛 winner-only +Materialize 与 shared heap，不在该步夹带 generation、deferred view 或真实 +simpler 迁移。 + +### 2026-07-25：S3.2a shared heap 与 winner-only Materialize + +本小步只改变 shared 模式的输出物化和 heap 主体，所有 worker 的 +`BuildCallbackSubmitArgs` 仍保持 eager。这样可以先证明物理地址、发布顺序 +和 loser 空路径，再在下一提交单独衡量重构参收敛，避免两个性能变量混算。 +private 的 Materialize、per-worker ring heap、HeapGuard 和构参计数均未改变。 + +#### 顺序与分配协议 + +shared winner 在 Materialize 前等待 +`committed_tasks == task_id`，并一直持有 exact turn 到 descriptor 发布、 +region 空/非空 delta 写入和 commit 完成。这个顺序同时保证： + +1. 同一 task 只有 winner 读取 `TaskArgs`、写 payload 和物化 descriptor； +2. loser 仍闭合 split-finish、固定阶段和 Submit 计数，但不读取 args， + 不触碰 shared heap/map，也不发布输出； +3. 同 shard 的 task 以 task id 顺序推进，host 可以独立重建确定的物理地址； +4. consumer 在后继 task 收集 fanin 前，必定先观察到 producer 的 descriptor + 和连续 commit。 + +shared sidecar 在 S3.1 output table 后追加 8 条 cache-line cursor 和 1 条 +aggregate vend。当前 ABI 为： + +| 项目 | 数值 | +| --- | ---: | +| `shared_heap_cursor` offset | 4,735,104B | +| `shared_heap_vend` offset | 4,735,616B | +| `SharedTensorMapSidecar` | 4,735,680B | +| CPU non-split `SchedulerState` | 1,011,851,648B | +| CCEC split `SchedulerState` | 1,011,857,792B | +| 构建身份 ABI | 3 | + +heap 被均分为 8 个 1KiB 对齐的物理 shard。每个非空 output task 先在 +`task_id % 8` 的 cursor 上 FetchAdd，再推进 aggregate vend；零输出 UP +只读取当前 vend，不执行 RMW。`aggregate_vend` 是所有 shard 已分配字节之和， +不是物理地址，不能拿来替代 shard base。首版不 wrap：任何 cursor 或 vend +越界都在写入前失败。两个 FetchAdd 若观察到与 exact-turn 预检不同的旧值， +冷路径恢复预检快照并返回失败；该无条件恢复只在“没有合法并发 allocator” +的 exact-turn 契约下成立。 + +这里的回滚边界只覆盖 reserve 内部两个原子操作观测到异常旧值的情形。 +一旦 reserve 成功，后续 resolver、append 或 descriptor build 异常会设置 +整轮 terminal fatal，并保留已经推进的分配状态供 host 取证；当前协议没有 +把一次 Submit 包装成可回收 heap 的完整事务，不能把它描述为全路径回滚。 + +shared 不再调用 private `HeapGuard`。旧 guard 使用 per-worker 单调 ring +坐标，而 shared cursor/vend 属于全局分片坐标，混用会制造错误等待。host +分别验证两种地址口径： + +- shared 实际 descriptor 必须匹配 8-shard 物理地址； +- 跨 private/shared 的 normalized writer signature 仍投影到同一连续 + canonical 地址，只比较业务 writer 拓扑，不把物理分片差异误判为语义差异。 + +#### 预检与失败不污染 + +winner 在任一 cursor 推进前校验参数数量、builder error、空 result、 +Output 引用类型、CreateInfo rank/dtype/连续布局、shape/stride 上界和 +`heap_base + heap_size` 地址可表达性。shape 乘积直接按 `TensorDesc` +的 32-bit stride 上界检查；最初的通用 64-bit 乘法溢出写法会使 CCEC +生成设备 ELF 不提供的 `__multi3`，因此不能保留。 + +CPU 新增两个独立定向门禁： + +- shared heap reserve：b1/b256 精确分片、对齐、满 shard、零输出、 + vend/cursor 的负值/未对齐/越界、aggregate 容量耗尽，以及零/非零快照上的 + cursor/vend 原子异常回滚； +- shared Materialize：合法 QK 物化、shape 乘积和 stride 溢出、非法计数、 + builder error、非空 result、错误/空 CreateInfo 引用、非法 tag 和 heap + 地址溢出；所有失败都必须保持 cursor、vend 与 worker heap 快照不变。 + +host 还要求 `(claim_wins == 0) == (final_heap_next == 0)`，防止纯 loser +误写任一看似合法的 aggregate prefix 后仍通过。shared 的 profile-phase +oracle 同步改为 `HeapGuard=0`；private 仍为 `4*batches`。 + +#### 当前闭环结果 + +| 规模 | 8 个 shard cursor | aggregate vend | Materialize outputs | +| --- | --- | ---: | ---: | +| b1 | `10240,524288,264192,8192,0,0,0,0` | 806,912B | 8 | +| b256 | 每 shard 25,821,184B | 206,569,472B | 2,048 | + +CPU private/shared 严格构建、shared b1/b256、profile-phase 和新增定向用例 +全部通过；CCEC private/shared 两种 ELF 均完成链接与 manifest 校验，shared +A5 b1 的全部调度、symbol、heap、descriptor、依赖和签名断言通过。 +最终加固后的 b1 `real-compute-count=1` Submit 单样本为 92.412us，只是正确性 +运行，不与历史默认 workload 性能相减。加固非法输入预检前还执行过一次 +CCEC b256 `scalar-nop-count=1`，Submit 为 58.371217ms，8 个 cursor 均为 +25,821,184B、vend 为 206,569,472B，全部语义断言通过；后续改动只增加 +reserve 前的拒绝条件和 host 断言，按只跑 b1 的约定未把该数字冒充最终 +ELF 复测值。 + +S3.2a 当时的 atomic 泳道不是 shared 协议全量清单：exact-turn 的 load， +以及 shared heap cursor/vend 的 Load/FetchAdd/Exchange 直接调用 `Ops`， +尚未进入既有 `AtomicSite` wrapper。S4.6 已删除 exact-turn load；heap、 +symbol writer/published 仍未进入 wrapper。它们已经计入 Submit/阶段总时间 +和 host 终态校验，但现有 atomic trace 不能用于拆分这些单指令成本。 + +S3.2a 到此仍保留所有 worker eager 构参；随后由独立的 S3.2b 提交收敛 +QK/SF/PV/UP 重构参。 + +### 2026-07-25：S3.2b winner-only 重构参 + +本小步不再改 heap、Materialize、symbol、ordered commit 或 ticket ABI， +只改变 shared 的 `BuildCallbackSubmitArgs()` 调用主体： + +- Alloc 继续由所有 worker 构造 3 个静态 Output 参数，用于对齐参考 + `alloc_tensors(args)` 调用形状，并把本小步变量限定在四个重构参 task； +- QK/SF/PV/UP 只有 `claim.won` 时才执行 reset、view/CreateInfo 构造以及 + tensor/scalar 参数添加； +- private 的五类 task 仍由所有 worker eager 构参，执行路径不变； +- output symbol 继续在构参分支前由所有 worker 独立声明，故 loser 后续 + orchestration 不依赖 winner 私有 descriptor。 + +Alloc 全员构参不是 standalone 符号协议的永久要求。这里是与参考路径对齐 +和单变量验证策略，后续若要收敛 Alloc，必须另做提交和计数门禁，不能把它 +伪装成 S3.2b 的顺手修改。 + +#### 精确计数 + +设单 worker 的 Alloc/QK/SF/PV/UP 获胜次数为 `A/Q/S/P/U`，batch 数为 `B`： + +| 计数 | shared 逐核期望 | +| --- | ---: | +| context read | `B` | +| view | `Q + U` | +| dynamic CreateInfo | `Q + S` | +| reset | `Q + S + P + U` | +| tensor args | `3B + 4Q + 4S + 4P + 7U` | +| scalar args | `2Q + 3S + 2P + 2U` | +| Materialize outputs | `3A + Q + 3S + P` | + +每类 task 全局恰有 `B` 个 winner，因此 shared 全局每 batch 为 +`context=96, view=2, dynamic=2, reset=4, tensor=307, scalar=9, +materialized=8`。b1 即 `96/2/2/4/307/9/8`。host 按逐核 wins 核对， +既能防止全局总数碰巧相等，也能发现某个 loser 意外恢复构参。 + +#### stale args 与 split-finish 门禁 + +split ABI 仍要求传非空 `&args`。每批 Alloc 会先在所有 worker 上 +`ConstructTaskArgs()`，所以 heavy-task loser 传入的是生命周期有效但内容 +属于上一 task 的陈旧对象，不是空指针或悬空对象。shared finish 中 +Materialize、fanin/register 和 winner build 的每个 args 读取都由 winner +分支支配。 + +CPU 新增 guard-page 定向测试：在匿名页上建立 `TaskArgs` 后填入毒值并设为 +`PROT_NONE`，再让 Alloc/QK/SF/PV/UP 五个 loser 逐一通过真实 +`FinishSplitCallbackSubmitFromRuntime()`。测试同时要求 finish 次数为 5、 +protocol error/materialized/map insert/fatal 全为 0；任何 args 字段读取都会 +立即触发失败。约 1 GiB 的 `SchedulerState` 只用 `MAP_NORESERVE` 建立稀疏 +虚拟对象，不提交无关物理页。 + +#### 当前闭环结果 + +- CPU private/shared 严格构建和 b1 完整语义回归通过； +- guard-page split-finish loser 定向测试通过； +- CCEC private/shared split swimlane 两种 ELF 均完成 AIC/AIV 链接、符号 + 和 manifest 校验； +- shared inline-finish 的 submit-PMU Materialize/Register 两种 ELF 均完成 + 编译，并分别执行 A5 b1，PMU owner 恢复和全部语义/计数断言通过； +- shared split CCEC A5 b1 的 symbol、heap、descriptor、依赖、前端计数和 + real-compute 输出断言通过，`real-compute-count=1` Submit 单样本为 + 98.664us。 + +以上 b1 数字只证明最终 S3.2b ELF 可执行，不与 S3.2a 的单样本相减，也不 +宣称 winner-only 构参已经获得稳定性能收益。性能判断仍须使用同一 +`perf-clock` 构建做配对多轮；swimlane 与 submit-PMU 只负责解释。 + +### 2026-07-25：S4.3～S4.5 从提前发布收敛到构建后封口 + +S4.2 的 b256 活性反例证明全局 exact turn 不能成为最终架构，但不能据此把 +所有顺序一次性删除。这里按三个独立提交逐层解除旧前提： + +1. S4.3 先让 consumer 只等待实际 `(producer,slot)`，把 fresh symbol 的 + 消费从全局 committed prefix 中拆开；当时 descriptor 仍在 Materialize + 内提前发布； +2. S4.4 再把 8-shard allocator 改成真正允许 FetchAdd 合法并发，host 从 + 实际 descriptor 重建物理区间，不再按 task id 猜地址； +3. S4.5 最后把 descriptor 发布移动到本地执行状态建立、INOUT writer 提交 + 之后，使 `published` 成为 producer Submit 的最终封口。 + +S4.5 同时把 ordered append 拆成 `PrepareSharedTaskOrdered()` 与 +`ReleaseSharedTaskTurn()`。前者只准备 ordinary-region delta，后者只推进 +`committed_tasks`；这样 writer commit 仍在 exact turn 内完成，而后继 task +取得 turn 后只能等待 producer 的最终 publish。Materialize 和 Register 的 +PMU/泳道边界随业务职责同步调整：Materialize 只覆盖 reserve/descriptor， +Register 覆盖空 region prepare，不把 writer commit 和最终 publish 混进 +I-cache 局部窗口。 + +resolver 的当前契约为“全量只读校验，再复制 descriptor”；writer 的当前 +契约为“Build/Complete 成功后，用 FetchMax 返回旧值做精确 producer +提交”。早期 S3.1 的 resolver 内 Exchange 已成为历史实现，不能用其计数名 +或失败回滚语义解释当前 ELF。 + +本阶段没有扩大 `WorkerResult` 或 trace record。原字段改名为 +`shared_symbol_inout_commits` 后仍位于 888 偏移，结构总长仍为 896B；泳道 +继续复用既有 WinnerBuild/Commit 与阶段记录。失败路径通过已有 fatal 加一个 +本地 slot 撤销 helper 闭合，没有新增逐 atomic 区域字段。 + +完成的验证包括： + +- CPU shared symbol 定向测试、真实 split-finish 故障注入和 + ASan/UBSan/leak； +- CPU private/shared 严格构建及 b1，证明 shared 宏分支没有改写 private； +- CCEC shared perf-clock、submit-PMU materialize/register、swimlane 的 + mixed ELF 构建与 A5 b1； +- 96 核、8 个 descriptor、5 次 symbol input load、3 次 inout writer + commit、5 条 fanin 边、8-shard cursor/vend、规范化 writer 签名与 + real-compute 输出全部闭合。 + +提交前审查还加固了两个只在 invariant 损坏时进入的分支：ordered commit +的 Exchange 旧值不匹配时恢复原前沿，避免覆盖/倒退；本地 occupied 计数若 +已经损坏，仍先清除失败 slot 再返回计数异常。对应测试覆盖重复/越级 commit、 +turn-release 故障、损坏计数和 Alloc completion 已不可逆的 publication 故障。 + +S4.5 的结论是封口基础协议已经可用，不是最终性能结论。下一提交只处理 +PA Case1 恒为空的全局 sequencer；非空 ordinary region 的通用并发算法仍未 +实现，不能借 S4.6 的 Case1 结果宣称已经支持。 + +### 2026-07-25：S4.6 从 PA Case1 热路径移除全局 sequencer + +本阶段严格沿用 S4.5 已证明的 Build 后封口，不改发布时机、不引入参考分支 +的 deferred resolve，也不改 writer intent。唯一协议变量是删除每个 winner +对 `committed_tasks==task_id` 的等待、空 region prepare 和 turn release。 + +实现完成后，shared winner 的业务顺序为： + +```text +Materialize shared descriptor + -> CollectSharedFanin 等待并读取实际 per-slot symbol + -> ValidateEmptySharedRegistration + -> CompleteTask / BuildWinner + -> CommitSharedFaninWriters + -> PublishSharedTaskOutputs +``` + +host 不再按 task 数推导 shared reclaim；它要求 sequencer、bucket 和 slot +保持初始化状态。worker 的 `map_alive_floor/map_cleaned_upto` 也改为 0， +而跨 private/shared 的 normalized writer signature 仍使用逻辑 floor 做 +规范化投影。两者在源码注释和断言中分开，避免再次把比较坐标冒充物理 +sidecar 状态。 + +实现过程中查档发现现有 analyzer 把 `PrepareMap` 列为 required phase。 +本步没有为此新增 mode 字段或分叉 converter，而是在 shared swimlane 中 +复用 `materialize_end` 写零时长 marker。它不读取 SYS_CNT、不访问 region +控制字；perf-clock/submit-PMU 中模板实例化为 false。这样维持原 raw schema +和六条每 Submit 固定记录，也没有扩大约 400MB 的 trace buffer。 + +新增或修正的定向门禁包括： + +- future task 6 在全局前沿为 0 时完成完整 QK finish 和 publication； +- shared symbol、manual-dep Local/GM writer、非 manual-dep Local/GM + ordinary writer 与非法 register mask 的空 region 判断； +- 预置 `fatal` 模拟另一核已广播终止态，winner 在 heap reserve、slot、 + completion 和 symbol 任一副作用前退出； +- writer failure、writer 成功后 publication failure、publication preflight + failure 和成功封口均证明 sequencer 不变； +- Alloc/QK publication 与 UP 第二 writer 故障的完整 finish 继续核对 + fatal、不可逆 ready、slot 撤销和 terminal writer 现场； +- 最新 symbol 与 split-finish 用例通过 ASan/UBSan/leak。 + +CPU 回归结果： + +| 构建/负载 | 结果 | +| --- | --- | +| shared/private perf-clock 严格构建 | 公共自测全部 PASS | +| shared PrepareMap raw-marker 负向自测 | 非零时长、task 序列/身份/flags/aux 漂移、Materialize 锚点错误、缺失和重复记录均被拒绝 | +| shared b1 scalar-nop0 / real-compute | 96 worker 全部语义、descriptor、writer、heap 与计算结果 PASS | +| shared b256 scalar-nop0 | 1,280 task 完成，published/input/writer/fanin 为 2,048/1,280/768/1,280，region 保持初始化状态 | +| private b1/b256 scalar-nop0 | 全部断言 PASS，证明 shared 宏分支未改写 private | +| shared b1 raw 后处理 | 3,089/3,089 records，0 drop；零时长 PrepareMap 门禁、converter、exclusive analyzer PASS | + +A5 上板结果： + +| 构建/负载 | 结果 | +| --- | --- | +| shared perf-clock b1 real-compute | 最终 terminal-fatal 门禁 ELF 为 70.279 us，语义与 4 个 active tile PASS | +| shared perf-clock b256 scalar-nop0 | 提交前审查前的 S4.6 ELF 为 3,228.844 us，96 核活跃，全部 1,280 task PASS | +| shared perf-clock b256 real-compute | 提交前审查前的 S4.6 ELF 为 5,982.840 us，192 个 active tile PASS | +| private perf-clock b1 real-compute | 70.707 us，全部断言 PASS | +| private perf-clock b256 scalar-nop0 | 3,300.478 us，全部断言 PASS | +| shared swimlane b1 real-compute | 最终 ELF 4,168/4,168 records，0 drop，零时长 marker、转换与排他分析 PASS | +| shared submit-PMU register b1 | 480 calls，phase share 0.8020%，owner cleanup PASS | + +b256 shared 同时闭合 published 2,048、symbol input load 1,280、writer commit +768、fanin edge 1,280、8 shard 各 25,821,184B、总 vend 206,569,472B， +且 `committed=0/reclaim=-1/region=empty`。这些证据共同证明原 global +convoy 已消失，不依赖某个固定毫秒阈值。 + +参考提交没有 global sequencer，并采用 consumer Exchange + producer +FetchMax 的 writer intent。它还在普通 `RingSlot` 路径把未 resolve 的 ref +留到执行前；但当前 winner follower 会在构造 `BuiltSubtask` 时同步等待， +`shared_ref_mask` 保持 0，所以不能把 ordinary-slot 的能力外推成参考 PA +winner 路径已经 deferred。`RingSlot/BuiltSubtask` ABI 都保留 +`shared_ref_mask/shared_refs`，writer intent 另引入 +`deps_prepared/prepared_deps`；这些机制有明确性能价值，也有 GM 搬运、I-cache +和槽容量成本。当前不照搬的原因是隔离变量和补齐失败边界，不是认为参考代码 +没有价值；冻结 ELF 后应先确认实际命中的 slot 路径，再用数据决定复用并删除 +standalone 的保守过程态。 + +### 2026-07-25：S4.7 冻结 S4.6 并完成 private/shared 配对 + +提交 `dc22d076` 后工作树 clean；private/shared CCEC perf-clock 从同一 HEAD +连续重建，manifest 和 host/kernel SHA256 在冻结前、运行前与 56 轮结束后 +均通过。冻结件、完整日志和机器可读汇总位于： + +```text +outputs/perf_clock_freeze_dc22d076_20260725_065902/ +``` + +正式口径为 b1、b256 各 6 个 ABBA/BAAB block，每模式 12 个独立进程样本； +每个规模另有 4 个不计入统计的 warm-up。b1 配对差值中位数 +`-0.458 us`，没有稳定 fixed cost；b256 private/shared 样本中位数为 +`5,003.790/7,209.016 us`，六个 block 的 shared-private 差值全部为正， +配对差中位数 `+2,149.766 us`（`+43.430%`）。 + +这轮把先前不同 ELF 的单样本疑问收敛成了同一提交、冻结产物和固定负载的 +稳定结论：去 sequencer 修复了 convoy 和正确性活性，但 shared 的规模化 +Submit 仍有约 2.15ms 中位差。下一步先做不改变 atomic/slot ABI 的 ready-ref +直落 slot，再只对纯 INPUT 验证 deferred resolve；不先改 writer intent，也 +不把参考 MIX `BuiltSubtask` 的同步 resolve 误写成普通 `RingSlot` 的 +deferred 行为。 + +### 2026-07-25：S4.8 ready shared descriptor 直写 LocalSlot + +#### 参考实现中复用的机制 + +参考分支的 `dist_try_resolve_shared_output_ref()` 把已经发布的 descriptor +直接写入 `RingSlot::tensors[i]`；只有尚未发布的普通 RingSlot 引用才写入 +`shared_ref_mask/shared_refs`，留到 kernel 前解析。这一“ready ref 不经过 +submit payload”的数据路径是可直接复用的优点：descriptor 最终本来就必须 +成为 slot 的独立快照，中间 4 KiB `TaskPayload` 并不承载额外协议语义。 + +standalone 当前没有原样照搬参考代码的两个部分： + +- 本阶段不引入 deferred mask/ref，也不扩大 `LocalSlot`。先只消除 ready + 路径的冗余搬运，避免把直写收益与延迟解析、更多在途 slot 混在一起； +- 不增加参考 helper 中的第二次 `published` atomic load。当前生产链由 + `MaterializeTask()` 先校验 producer 对应业务 task 的声明 output 上限， + `CollectSharedFanin()` 再完成 acquire 等待、plain-view、物理 slot、 + `published` 和 `last_writer` 校验；在合法、非 fatal 的成功发布路径中， + fresh output descriptor 随后不再改写。紧接着进入同一 winner 的 Build + 阶段时重读控制字只增加观察不到新状态的 atomic 成本。 + +以上差异不是否定参考实现。参考代码同时支持“ready 或 deferred”两条 +RingSlot 路径，所以 try-resolve 必须自行判断 publication；standalone +S4.8 仍是纯 eager 前提，两者的 helper 责任不同。后续 S4.9 验证纯 INPUT +deferred 时，必须重新引入执行前 acquire 校验，不能把 S4.8 的前置条件 +外推到未等待的引用。 + +参考 builder 先复制普通参数和 Output，再单独覆盖 shared ref；standalone +为保持现有 builder 接口，先把 shared ref 直写 slot，再由 +`PopulateSlotPayload()` 填普通参数和 Output。两类索引互斥，正确性等价， +但写流顺序可能影响 cache 行为；是否有净收益由同一 perf-clock 配对判定， +不能仅凭少一次拷贝就预设结论。 + +参考实现和 standalone 都允许多 output 发布中的冷故障回滚此前短暂发布的 +descriptor；一旦发生，整轮由 terminal fatal 作废。S4.8 没有新增这个窗口， +旧 eager payload copy 也不能把它变成可恢复事务；额外一次 `published` 重读 +同样无法证明读后不会回滚。因此本阶段保持既有 terminal 语义，不为故障注入 +给正常热路径增加 atomic。 + +#### 单一变量与实现边界 + +S4.7 每个 shared ref 有两次 128B descriptor 搬运： + +```text +published SharedOutputCell + -> CollectSharedFanin: TaskPayload scratch + -> PopulateSlotPayload: LocalSlot +``` + +S4.8 改为： + +```text +CollectSharedFanin: 只做全量协议校验和 fanin 收集 + -> BuildWinner: invalidate published descriptor 并直写 LocalSlot + -> PopulateSlotPayload: 只把 slot descriptor 地址写入 dispatch args +``` + +PA Case1 每 batch 有 8 个 shared refs，因此删除 8 次 shared-cell 到 +TaskPayload 的 128B 中间拷贝；b256 共删除 2,048 次，即 256 KiB +descriptor 写入及其对应读取。`shared_symbol_input_loads` 在第一遍验证中 +先累计到局部量，只有所有引用成功后才一次提交；这样保留 late-failure +all-or-nothing，同时删除原来仅为提交该统计而执行的第二遍 tensor 扫描。 + +分段观察口径也随实现职责变化：剩余的一次 invalidate + descriptor copy +从 `Fanin` 移到 `WinnerBuild`。因此新泳道可能表现为 Fanin 缩短而 +WinnerBuild 增长；这只表示工作归属移动，净收益仍由无泳道、无 PMU 的 +perf-clock 完整 Submit 配对决定。 + +这一步明确没有改变： + +- `published` 等待、fanin、INOUT writer FetchMax 和最终封口顺序； +- `TaskPayload`、`LocalSlot`、`SubmitContext`、`WorkerResult` 或 trace ABI； +- `slot_tensor_copies=19*batches`、`input_loads=5*batches`、 + `writer_commits=3*batches` 的既有统计口径； +- private 构建的数据路径。 + +公共 frontend 原有的五参 compatibility builder 仍允许“调用方预填 +TaskPayload”后构建 slot。生产 `BuildWinner` 通过编译期模板参数选择 +direct-to-slot 实例；CCEC 不承担运行时布尔分支，兼容接口也没有被静默 +改写。没有增加新的 profiling 字段或 raw 记录。 + +#### 正确性门禁 + +CPU 定向测试覆盖： + +- 普通 INPUT、INOUT 和延迟 publication 均保持整个 TaskPayload poison + 不变，最终 LocalSlot descriptor 和 dispatch args 精确； +- 真实 `FinishCallbackSubmitBody -> Collect -> Register -> BuildWinner` + 正向链证明生产接线，而不是只测试独立 helper； +- “合法 INPUT 后跟非法 future ref”不会泄露局部 input 计数或 fanin； +- 原有“合法 INOUT 后跟非法 ref”继续证明 writer、payload、统计和 fanin + 均无半次提交； +- compatibility builder 仍能从预填 payload 构建正确 descriptor； +- private/shared 严格 CPU 构建、shared/private b1、symbol 与完整 + split-finish 用例均通过，两个定向用例通过 ASan/UBSan/leak。 + +CCEC shared perf-clock 完成 AIC/AIV 编译、mixed ELF 链接和 manifest 校验； +A5 b1 默认 real-compute 的 96 核语义、8/5/3 symbol 计数、5 条 fanin、 +shared heap、writer signature 和输出 tile 全部 PASS，Submit 单样本 +70.265us。该单样本只作为最终 ELF 正确性门禁；性能结论必须继续与冻结的 +S4.6 shared ELF 做同负载 ABBA/BAAB 配对，不能和 private 或历史单样本直接 +相减。 + +#### 冻结配对结果与下一步收紧 + +S4.8 代码提交为 `2514ef10`。从该 clean commit 重建 shared perf-clock， +冻结 host/kernel/manifest 后，与 `dc22d076` 的 S4.6 shared 冻结件交错 +运行。b1、b256 各先跑 4 个不计入统计的 warm-up，再跑 6 个 +ABBA/BAAB block；每模式每规模 12 个正式样本，56 个独立进程全部通过 +负载身份和完整语义门禁。冻结信息、日志和机器可读汇总位于： + +```text +outputs/perf_clock_freeze_2514ef10_20260725_074116/ +``` + +| 规模 | S4.6 中位数 | S4.8 中位数 | block 配对差中位数 | 配对差范围 | S4.8 更慢 block | +| --- | ---: | ---: | ---: | ---: | ---: | +| b1 | 66.214us | 66.332us | +0.825us（+1.244%） | -3.283～+2.418us | 4/6 | +| b256 | 7,117.583us | 7,313.766us | -121.994us（-1.654%） | -1,462.169～+1,311.427us | 2/6 | + +b1 没有稳定 fixed-cost 方向。b256 虽有 4/6 block 为 S4.8 更快，配对差 +中位数约 -122us，但极差约 2.77ms；S4.8 样本中位数更高，而样本均值 +7,233.971us 又略低于 S4.6 的 7,249.306us。波动远大于候选差值，因此只能 +记为“没有证明稳定提速，也没有证明稳定回退”，不能把机械上少 256 KiB +中转直接写成性能收益。 + +统一用 `size -A` 核对后，S4.8 mixed ELF `.text=134,456B`，S4.6 为 +132,920B，纯代码增加 1,536B；两者 `.rodata` 均为 288B，所以 +`.text + .rodata` 也同样增加 1,536B。`nm -S` 将增量定位到 split finish +AIC/AIV，分别 +752/+780B，合计 1,532B，剩余 4B 来自布局对齐。当前 +helper 在 BuildWinner 中另扫一次 tensor，随后 +`PopulateSlotPayload()` 又遍历全部 tensor。下一小步先在不改协议/ABI的 +前提下把 direct copy 折叠进既有 slot 填充扫描，单独核对 CCEC `.text` 和 +配对方向；解释这项代码布局成本后，再进入纯 INPUT deferred resolve。 + +### 2026-07-25：S4.8b 把 ready descriptor 直写融合进既有 slot 扫描 + +S4.8 已消除 `SharedOutputCell -> TaskPayload -> LocalSlot` 的中间搬运，但 +实现上仍由 `CopyValidatedSharedDescriptorsToSlot()` 先独立扫描全部 tensor， +随后 `PopulateSlotPayload()` 再扫描一次普通参数和 Output。S4.8b 不改变 +协议，只把前者的 invalidate 和 128B copy 合并到后者已有的逐 tensor 分支： + +```text +BuildWinner + -> PopulateSlotPayload 单次扫描 + 普通参数/Output：沿用 TaskPayload -> LocalSlot + ready shared ref：SharedOutputCell -> LocalSlot +``` + +生产 shared builder 以编译期模板参数选择 direct 分支，并显式接收非空 +`SharedTensorMapSidecar&`；private 保留原函数签名和实现。五参兼容入口仍 +走“调用方已填 TaskPayload”的 false 实例，`if constexpr` 在编译期删除 +shared map 访问，不增加运行时判断。这样既没有为公共接口引入可空 map +约定，也没有把 shared 模板扩散到 private 构建。 + +这一融合使每个 shared ref 的 invalidation/copy 与 slot 参数指针建立发生 +在同一索引分支。不同索引写入互不重叠，因而相对 S4.8 只改变独立扫描与 +代码布局，不改变以下内容： + +- `Materialize -> CollectSharedFanin -> Register -> BuildWinner` 的协议边界； +- acquire 等待、published/last_writer 校验、INOUT writer 和最终发布顺序； +- `TaskPayload`、`LocalSlot`、sidecar、result 和 trace ABI； +- `slot_tensor_copies`、8/5/3 shared symbol 计数和 5 条 fanin； +- compatibility builder 与 private 数据路径。 + +CPU shared/private perf-clock 严格构建和 b1 全部通过；shared symbol 的普通 +与 split-finish 用例通过 ASan/UBSan/leak。CCEC shared/private 均完成 +AIC/AIV 编译、mixed ELF 链接和 manifest 校验。shared mixed ELF 的 +`.text` 从 S4.8 的 134,456B 降至 134,200B，减少 256B；split finish +AIC/AIV 主体分别减少 96B 和 88B。A5 shared b1 默认 real-compute 为 +65.734us，96 核语义、8/5/3 symbol、5 条 fanin、heap、writer signature +和输出 tile 全部 PASS。该 b1 单样本仍只作为正确性门禁，不能据此声称 +相对 S4.8 的性能收益；冻结后的 b256 配对与参考分支同口径对照另行记录。 + +### 2026-07-25：S4.8b 与参考 shared 分支的同机近似对照 + +#### 对照来源和环境适配 + +参考实现固定为 `fdwic-shared-tensormap` 的 `2866ad73`，shared 编译开关 +查档确认是： + +```bash +CXXFLAGS='-DPTO_FDWIC_SHARED_MAP=1' +``` + +参考 checkout 原始 Case1 写死 `block_dim=36`，而本机 device 0 报告 +`cube=32/vector=64`，所以只在 `/tmp` 测试配置中临时改为 32；这也使实际 +worker 拓扑与 standalone 的 32 AIC + 64 AIV 对齐。参考提交还没有旧驱动 +`CPU_TOPO=65534` 的兼容分支，因此临时移入当前 simpler 已有的 +“ACL AICPU 核数与 OCCUPY 位图相等才允许平铺拓扑”host fallback。该补丁 +只决定 AICPU 线程可见性，不进入 AICore Submit/shared TensorMap 热路径。 + +构建过程还复用了 CANN 9.1 自带 `llvm-strip` 处理 AArch64 dispatcher， +因为参考脚本会错误地让 x86 `/usr/bin/strip` 处理该 so 并硬失败。PTO-ISA +直接复用本机完全相同的 `ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8` +checkout。以上均是让旧参考提交在本机启动的工具链或平台适配，不是性能 +实现移植。 + +参考 PA Case1 使用 batch=256、每 batch 5 个 Alloc/QK/SF/PV/UP task， +真实执行 4 类 PA kernel，并开启 level-4 L2 泳道。用例 golden 通过； +原始和合并泳道冻结在： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + reference_compare_2866ad73_20260725_082426/ +``` + +#### 时间结果和口径限制 + +standalone `d0042690` 从 clean shared perf-clock ELF 先跑 3 次 warm-up， +再跑 12 个独立 b256 real-compute 进程。正式样本的首个 Submit 起点到 +最后一个 Submit 返回为： + +| 数据 | min | median | max | mean | +| --- | ---: | ---: | ---: | ---: | +| standalone perf-clock | 4.031ms | 6.874ms | 7.721ms | 6.580ms | + +参考分支本机泳道有 97,500 条 X 事件，所有事件最早起点到最晚终点为 +2.541ms。表面上它比 standalone 中位数少 4.334ms，standalone/reference +约为 2.70 倍；参考文档此前在 device 6 的单次记录为 2.285～2.330ms， +本机这次约慢 11%。 + +这个差值只能用来判断优化方向，不能当作严格方案收益： + +- standalone 窗口是每核 task0 的 Submit 边界到 task1279 的 Submit 尾动作， + 且 perf-clock 编译期去掉泳道、atomic trace 和 PMU； +- 参考 2.541ms 是全部已记录 X 事件的 global span，既包含 Kernel、Commit + 和末端 drain，又不包含未打点的 orchestration 构参； +- 参考执行真实 PA kernel/内存流，standalone 使用 + `real-compute/6,28,4,1` 的引擎负载模型； +- 参考分支没有现成 first-to-last Submit 计时，不能把现有逐调用 + `prof_submit_task` 累计冒充 wall window。 + +尽管绝对值不能直接相减，重计算负载并不是当前 2.7 倍表面差异的合理主因。 +standalone b1 泳道的 QK/SF/PV/UP 分别约 +42.062/53.712/28.658/3.332us；参考本机 256 个同类真实 kernel 的中位数 +分别为 47.131/54.857/29.370/1.742us。三个重任务在参考中反而略慢或接近， +说明需要优先审查 scheduler 协议，而不是继续盲调 NOP/compute 次数。 + +#### 已确认的结构性差异 + +1. **参考在 shared Alloc 前先做唯一候选过滤。** + 参考实际 Claim 为 49,408 次，精确等于 + `256 + 2×256×32 + 2×256×64`。standalone 为 73,728 次，多出的 + `24,320 = 256×(96-1)` 全是 Alloc 的非候选竞争。参考在候选判断失败后 + 立即返回，既不发 Claim atomic，也不进入 Materialize/Submit 主体。 + 但参考的 orchestration 已在调用 `dist_alloc_outputs_impl()` 前构造好 + 三个 Output 参数；standalone 的全员 Alloc 轻构参对应这个调用方成本， + 不能误写成参考早退一并删除。这项可直接复用的优点首先是内部 + winner-first 过滤，不是删掉调用方参数。 + +2. **参考 shared 正常完成不推进全局 frontier。** + `complete_executed_task()` 只在 private 宏分支调用 + `advance_frontier()`；本次 PA Case1/b256 容量内的 8 个 32MiB shard + 没有触发 wrap，fanin 直接依赖 per-task flag，所以正常路径不需要连续 + 完成前沿。参考实现允许未来回绕，并在复用慢路径按需推进回收前沿。 + standalone 本次仍执行 1,280 次 initial load、32,861 次 flag load 和 + 31,581 次 FetchMax，共 65,722 个 frontier atomic，占自身 + `submit_completion_ops=176,930` 的 37.1%。这不是参考实现的细枝末节， + 而是当前 standalone 尚未剥离的 private ring 时代协议。 + +3. **参考 shared 把私有 ring 从 4 槽扩到 14 槽。** + standalone 保持 4 个 `LocalSlot`，扣除 2 个协议预留后普通任务只有 + 2 槽；参考 14 槽同样扣除 2 个预留后仍有 12 槽。参考编译期 + 移除每核 private TensorMap 后，即使 `RingSlot` 因 deferred ref 从 + 4,864B 增到 5,440B，仍为每核配置 14 槽。更深的在途窗口能减少 + RingBackpressure，并让构建与真实 kernel 更充分重叠。这部分有状态体积、 + 槽回收和执行顺序风险，不能与前两项一次合并验证。 + +4. **参考对未发布 INPUT 使用执行前 deferred resolve。** + 本机参考泳道实际出现 491 条 Resolve 区间，并完成 2,048 次 descriptor + copy/invalidate，证明 ready/deferred 两条路径都命中。standalone 仍在 + `CollectSharedFanin()` 中同步等 publication；本次 perf-clock 有 + 34,920 次 fanin load，其中 25,457 次 not-ready。standalone 随后又在 + slot 执行前检查 fanin flag，因此 eager 等待既限制提交超前，也没有替代 + 执行门禁。S4.8b 只对齐了 ready descriptor 直写，没有对齐 deferred。 + +5. **两者的 shared 输出分配方向已经接近，但状态裁剪程度不同。** + 两者都按 task shard 预留 output heap，也都让 ready descriptor 直接落 + slot。standalone 仍保留大块 per-worker private TensorMap/TaskPayload + 形态和 4 槽 ABI；参考把 shared 编译期状态作为整体重新布局。后续不能只 + 比一条 copy 指令就声称两套架构已经对等。 + +#### 对后续顺序的影响 + +本次对照改变后续优先级。纯 INPUT deferred 仍是目标，但在它之前已有两项 +更小、参考代码和本机计数都直接支持的变量： + +1. shared no-wrap PA 先停止每 task frontier helping，并把 host 断言改为 + “flag 全 ready、frontier 保持初始化”；单独做 CPU/CCEC/A5 和 frozen + perf-clock 配对； +2. shared Alloc 先只过滤 95/96 actor 的 Claim，保持调用方全员轻构参和 + 现有观察边界；验证后再单独把非候选早退前移到 EfDrain 之前； +3. 再独立比较 4→14 或分档 ring 深度，不能和 deferred 同时改； +4. 最后接纯 INPUT deferred resolve，保留 INOUT writer 等待和完整故障门禁。 + +这个顺序不是照抄参考提交：shared heap 一旦允许 wrap，frontier 或等价 +generation/reclaim 协议仍必须恢复；14 槽也必须按本机状态预算验证。复用的 +是已被参考实跑证明的主路径机制，差异和限制继续在本文件逐阶段记录。 + +### 2026-07-25：S4.9 shared no-wrap 完成路径停止推进 private frontier + +#### 采用参考机制，但明确收窄适用边界 + +参考 `2866ad73` 的 `complete_executed_task()` 在 shared 构建中不调用 +`advance_frontier()`。逐条审计 standalone 后确认，当前 shared Case1 +同样满足这一前提： + +- fanin 只 acquire-load producer 的 per-task `flag`； +- kernel 和 Alloc 完成都先发布 `vend`，经 store barrier 后发布 `flag`； +- shared Materialize 使用 8 个不回绕 shard，容量不足是 terminal failure； +- shared BuildWinner/Alloc 已不调用 private `HeapGuard`； +- 最终退出依赖 replay barrier、本核 slot 清空和 fanin ready，不读 frontier。 + +因此 S4.9 只在 `PTO_FDWIC_SHARED_MAP=1` 时编译期去掉 +`CompleteTask()` 末尾的 `AdvanceFrontier()` 调用。vend、barrier、flag、 +`AdvanceFrontier` 实现、AtomicSite 编号、`SchedulerState::frontier` 和 +`WorkerResult` ABI 全部保留；private 仍执行原协议。这里不同意把参考代码的 +“shared 不推进 frontier”无条件推广为通用结论：参考实现在 shared heap +真正回绕的慢路径仍会推进回收前沿，standalone 以后若允许 wrap 或 task-cell +复用，也必须恢复 frontier 或等价 generation/reclaim 协议。 + +host oracle 按模式闭合： + +- shared 每核和聚合的 initial/update/terminal 三类计数必须全为 0， + `state.frontier` 必须保持初值 `-1`； +- private 继续要求 initial/terminal 与完成数相等、update 不少于 task 数， + 最终 frontier 为 `task_count - 1`； +- shared 失败诊断改为输出 8 个 shard cursor、aggregate vend 和容量状态， + 不再用只对 private ring 有意义的 `frontier-H` 解释回收。 + +新增 shared 定向测试直接调用 `CompleteTask()`，分别锁定 vend、flag 已发布， +frontier 仍为 `-1`，三个 frontier 计数和 CAS retry 均为 0。没有新增 +profiling 字段或 raw 记录。 + +#### 正确性与初步性能结果 + +CPU shared/private 的 strict `-Werror` 构建和 b1/b256 均 PASS。shared +完整运行的三个 frontier 计数严格为 0；private 的旧计数身份和最终值保持 +不变。CCEC shared/private perf-clock 均完成 AIC/AIV 编译、mixed ELF 链接和 +manifest 校验。S4.9 shared perf-clock `.text` 为 129,080B，相对 +`d0042690` 冻结件的 134,200B 减少 5,120B（3.815%）；`.rodata` 大小均为 +288B，但内容会随代码布局控制表变化，不能写成逐字节相同。四个 AIC/AIV +orchestration/finish 主函数合计减少 4,880B，其余 240B 来自链接布局/对齐。 +A5 默认 real-compute： + +- shared b1 为 65.563us，private b1 为 66.801us，均通过 96 核、fanin、 + heap/TensorMap 签名和真实输出 tile 门禁； +- shared b256 单样本为 3.218ms，`frontier_initial/frontier_flag/` + `frontier_ready_fetch_max/frontier_terminal` 全为 0，全部语义门禁 PASS； +- shared b1 atomic 泳道 4,130 条 raw 记录、849 条 atomic 物理记录、零丢失， + 不再出现 FrontierInitial/FrontierFlag/FrontierMax/HeapFrontier 调用， + completion vend/flag 发布仍各 5 次；private b1 泳道继续得到 + `initial/flag/FetchMax/terminal = 5/10/5/5`，trace 闭合且零丢失。 + +#### 冻结配对和参考时间对照 + +S4.9 代码提交为 `e8320280`。从该 clean commit 重建并冻结 +host/kernel/manifest，再与仓内标记的 `d0042690` 冻结件交错运行；manifest +本身只固化 mode/variant/phase 和文件 SHA，不单独证明 Git provenance。 +两版各先预热 2 次，再跑 6 个交替 ABBA/BAAB block，每版共 12 个正式 +b256 独立进程，所有运行均通过完整语义门禁。冻结件和逐轮日志位于： + +```text +outputs/perf_clock_freeze_e8320280_20260725_090120/ +outputs/perf_clock_pair_e8320280_vs_d0042690_20260725_090233/ +``` + +| 版本 | min | median | mean | max | 标准差 | +| --- | ---: | ---: | ---: | ---: | ---: | +| `d0042690` | 3.233ms | 7.495ms | 6.938ms | 8.562ms | 1.635ms | +| `e8320280` | 3.229ms | 3.243ms | 3.243ms | 3.265ms | 0.013ms | + +样本中位数减少 4.252ms（56.74%）。6/6 个 block 均为 S4.9 更快； +每 block 两个样本先求均值后，配对差中位数为 +`-4.155ms / -56.19%`。旧版仍偶发 3.233ms 快样本，但大多数运行进入 +5～8.6ms 长路径；S4.9 的 12 个正式样本全部落在 36us 窄区间。结合基线 +取证轮记录并被删除的 65,722 次动态 frontier atomic，可以确认该候选在本轮 +同口径配置下有效，而不仅是一次幸运样本；本轮样本中的旧版长尾随修改消失， +与全局前沿竞争的机制判断一致，但 12+12 个样本不能单独证明旧版全部长尾都 +来自 frontier。 + +参考分支本机 97,500 条泳道事件的 global span 为 2.541ms。当前 S4.9 中位数 +仍多 0.702ms，standalone/reference 约 1.276 倍；相比 S4.8b 的表面 +2.70 倍已经显著靠近。这个 27.6% 剩余差值仍不是严格收益口径，因为参考值 +不是 first-to-last Submit perf-clock，并且两边 ELF、构参覆盖和执行模型不同。 +它只用于决定下一步优先级：先迁移参考分支已证明的 Alloc 唯一候选早退， +再单独评估 ring 深度和纯 INPUT deferred resolve，不能把三项一起修改。 + +### 2026-07-25：S4.10a 先收敛 shared Alloc 的 Claim 候选 + +#### 为什么没有机械照搬参考的候选公式 + +参考 `2866ad73` 使用 `alloc_cursor[3 lanes][8 shards]`，其候选是： + +```text +shard = task_id & 7 +target_lane = task_id % 3 +target_block = shard % num_blocks +``` + +同一个 `(lane, shard)` cursor 始终由同一物理 worker 按本地 task 顺序 +推进。standalone 为保持当前生产基线 ABI，仍是单层 +`alloc_cursor[4]`。如果只复制 `task_id % 3` 的 lane 规则，同一 4-shard +cursor 会被不同 worker 推进；快 worker 可能先把后续 task atomicMax 到 +cursor，慢 worker 再处理较早 task 时会观察到 `old >= task_id`，导致较早 +task 永久没有 winner。 + +因此本阶段保留 4-shard 状态布局，把每个 shard 固定绑定到一个 worker： + +| shard | block/lane | standalone worker | +| ---: | --- | ---: | +| 0 | block 0 / AIC lane 0 | 0 | +| 1 | block 1 / AIV0 lane 1 | 34 | +| 2 | block 2 / AIV1 lane 2 | 37 | +| 3 | block 3 / AIC lane 0 | 3 | + +设备拓扑固定为 32 AIC + 64 AIV，因此热路径直接使用 +`target_block=shard`、`target_lane=shard%3`，不新增 GM `num_blocks` +读取或动态除法。`static_assert(kCursorShards <= kAicWorkers)` 锁定每个 +cursor 都有物理 block owner。这个映射保留了参考实现真正重要的单调性 +不变量，但没有伪称两边 cursor ABI 已相同。 + +#### 本小步明确保留和删除的内容 + +候选判断只放在 shared `Claim()` 的 Alloc 分支内： + +- 非候选仍进入 EfDrain 和 Claim span，但 `attempted=false`，不发 + `ClaimMax`； +- 唯一候选继续对原 `alloc_cursor[task_id%4]` 执行 atomicMax； +- 全员仍建立相同 shared output handles、构造三个 Alloc Output 参数、 + 进入 generic finish 并闭合 Materialize/PrepareMap/Register/Submit； +- `submits`、split-finish 调用次数、PMU 窗口和普通泳道固定记录数不变； +- private 完全保持 96-worker Alloc 竞争。 + +固定 owner 按本地顺序执行时必须赢。若出现 `attempted && !won`，说明 cursor +继承了旧状态或已经被越序推进；把它当普通 replay 会让本 task 永久没有 +completion owner，因此 shared Alloc 在 Claim 记录闭合后立即广播 fatal。 + +本阶段没有增加 WorkerResult、TraceRecord 或 raw 字段。新增纯 CPU 定向测试 +逐 task 枚举唯一候选、锁定 `[0,34,37,3]` owner 表、验证同 shard 的 +`0→20→40` 连续 Claim 均获胜、非候选不改 cursor，并构造 `20→0` 反例证明 +为什么 candidate loss 必须终止。host 又逐 worker 复算候选次数，防止只看 +全局 49,408 而漏掉 owner 分布错误。 + +#### 正确性和单轮观测 + +CPU shared/private strict 构建及 b1/b256 全部 PASS。Claim 次数精确闭合为： + +| 模式 | b1 | b256 | +| --- | ---: | ---: | +| shared | 193 | 49,408 | +| private | 288 | 73,728 | + +shared b256 相对旧实现删除 +`73,728 - 49,408 = 24,320 = 95×256` 次 Alloc `ClaimMax`；winner、 +四个 cursor 终值、全部 completion、8-shard heap、symbol 8/5/3 计数、 +依赖签名和输出 tile 均不变。CCEC shared/private 的 swimlane 与 perf-clock +均完成 AIC/AIV 编译、mixed ELF、导出符号和 manifest 门禁。 + +A5 shared b1 atomic 泳道有 4,026 条 raw 记录、0 drop；其中 +`ClaimMax` 恰为 193 条，普通 Claim/EfDrain/Materialize/PrepareMap/ +Register/Submit 仍各 480 条，证明本小步只删 atomic，没有偷偷改变阶段 +边界。产物位于： + +```text +outputs/pa_scheduler_shared_swimlane_20260725_101155_2313002/ +``` + +A5 shared b256 的一轮普通构建为 3.417ms，perf-clock 为 3.300ms； +private b256 分别为 3.884ms 和 3.630ms。所有语义门禁通过。这些是构建与 +正确性阶段的单样本，不能与 S4.9 的 12 样本中位数直接相减。当前 shared +perf-clock `.text=129,336B`，相对 S4.9 的 129,080B 增加 256B；下一提交 +会冻结本阶段 clean ELF，与 `e8320280` 做交错配对后再判断这 24,320 次 +atomic 消减是否转化为稳定墙钟收益。 + +#### S4.10a 冻结配对：atomic 次数下降，但完整 Submit 稳定回退 + +S4.10a 实现提交为 `e83283f6`。从 clean 提交重新构建并冻结 shared +perf-clock 三件套，与 S4.9 的 `e8320280` 冻结件运行完全相同的 device 0、 +b256、`real-compute/6,28,4,1`。每版先运行 2 个不计入统计的 warm-up， +再运行 6 个交替 ABBA/BAAB block；每版共 12 个正式独立进程，全部通过 +shared heap、symbol、依赖、输出 tile 和构建身份门禁。产物和机器可读结果 +位于： + +```text +outputs/perf_clock_freeze_e83283f6_20260725_102242/ +outputs/perf_clock_pair_e83283f6_vs_e8320280_20260725_102404/ +``` + +| 版本 | 最小值 | 中位数 | 均值 | 最大值 | 样本标准差 | +| --- | ---: | ---: | ---: | ---: | ---: | +| `e8320280` | 3.217ms | 3.252ms | 3.248ms | 3.269ms | 0.017ms | +| `e83283f6` | 3.259ms | 3.296ms | 3.295ms | 3.321ms | 0.018ms | + +6/6 个 block 都是 S4.10a 更慢。每个 block 内先分别对两个样本求均值， +候选减基线的配对差中位数为 `+49.152us / +1.514%`，范围为 +`+19.811～+71.417us`。因此本轮必须把结果记为稳定回退，不能因为 +`ClaimMax` 数量更少就宣称优化有效。 + +两版的全局 task/kernel 数、依赖签名、heap/symbol 终态和数值输出门禁均 +通过；winner 落点与动态轮询量并不相同。配对日志可核实以下伴随变化： + +- `claims` 从 73,728 精确降到 49,408，少 24,320；代表原定的 atomic + 消减确实生效； +- 12 个正式样本的 `max_wins_per_worker` 从基线 32～37 增至 + 候选 75～81。S4.10a 的 4-shard 固定 owner 把 256 个 Alloc winner 集中到 + 4 个 worker，而参考 + `[lane][8]` 候选可落到最多 24 个 `(block,lane)` owner；这是两种架构 + 不能混称等价的直接证据； +- `fanin_loads` 从基线 22,958～23,349（中位数 23,181）变为候选 + 20,225～21,357（中位数 20,598）；real-compute 活跃输出 tile 则从 + 189～192 变为 186～189。它们说明固定 Alloc owner 已经改变后续任务 + winner 分布和依赖就绪时机，即使全局任务数和数值结果仍然正确； +- mixed ELF `.text` 从 129,080B 增至 129,336B。AIC/AIV orchestration + 主体分别增加 48B/120B，其余 88B 是链接布局和对齐的净变化； + `.rodata` 大小仍为 288B。 + +perf-clock 不能把 49us 唯一归因到 winner 集中或代码布局，因此这里只把 +这些数据列为已证实的伴随变化，不把相关性写成单一因果。更重要的是，本小步 +仍让 95 个非候选执行 EfDrain、普通阶段空包围和 generic finish，只消除了 +Claim atomic，没有取得参考实现“候选判断失败后立即返回”的主体裁剪。 + +下一小步继续保持单一变量：在相同 4-shard owner 规则上把 shared Alloc +非候选早退前移，删除其 EfDrain、Claim 后半段和 generic finish，同时保留 +调用方已经构造的三个 Output 参数，并为所有 actor 建立 +`PrepareSharedTaskOutputs/shared_result` 符号句柄，保证非候选返回后仍能 +为后续 QK/SF/PV/UP 建立输入引用。外层 `submits` 和 perf-clock 首末边界 +继续覆盖完整 5-task 回放；split-finish calls/task-id sum、普通 +trace/profile 和局部 PMU phase 调用数则按真实跳过的非候选精确减少，并同步 +修改 host/analyzer oracle，不增加 raw 字段。 + +完整早退版必须同时对照两条基线：与 `e83283f6` 配对只隔离早退本身的增量， +最终保留判据必须是与 `e8320280` 配对后取得净收益。若它仍不能抵消本轮 +回退，就撤销整个 S4.10 候选方案，而不是为了 atomic 数字好看而保留回退 +代码。后续若要采用参考的 24-owner 分散方式，必须单独修改 cursor ABI 和 +host oracle，不能塞进本次早退验证。 + +### 2026-07-25：S4.10b 完整早退验证及 S4.10 架构撤销 + +#### 完整早退版实际改了什么 + +S4.10b 提交 `f41e2833` 没有继续改 cursor ABI，而是在 S4.10a 的 +4-shard/4-owner 映射上验证参考分支另一项关键差异：shared Alloc 非候选在 +准备好稳定 output handle 后立即返回。具体边界为： + +1. 每个 worker 仍创建逻辑 task，外层 `submits` 仍累计 `5×batches`； +2. 调用方仍构造三个 Alloc Output 参数，所有 actor 都执行 + `PrepareSharedTaskOutputs`，因此后续 task 的 symbol 引用不依赖 owner + 是否已经跑完； +3. 非 owner 不再进入 EfDrain、Claim 和 generic finish； +4. owner 继续执行完整路径，并保持 vend、flag、barrier、heap 和真实计算 + 语义； +5. first/last perf-clock 与完整 Submit PMU 边界仍覆盖全部逻辑 Submit； + 普通泳道和局部 PMU 只记录实际 full-path 调用。 + +因此 b256 的 full-path 调用数从 122,880 降为 98,560： + +```text +92 个非 owner × 1,024 + 4 个 owner × 1,088 = 98,560 +``` + +为避免把跳过的逻辑 Alloc 伪装成有普通阶段记录，converter 和 exclusive +analyzer 一度支持精确的 shared 稀疏 task 集合；首个被跳过的 Alloc 归入 +OrchestrationSetup，后续跳过项归入 BetweenSubmitResidual。没有增加 +TraceRecord/raw 列，仅在 metadata 透传 `tensor_map_mode`。private 仍要求 +每核完整连续 task stream。 + +#### 正确性取证 + +完整早退版完成了以下门禁： + +- 95 项 converter/analyzer/PMU Python 测试全部通过； +- CPU shared/private strict 构建以及 b1/b4/b256 通过； +- CCEC shared/private 的 swimlane、perf-clock、五种 submit-PMU phase， + 共 14/14 构建通过； +- A5 shared b1 泳道有 3,458 条 raw、零 drop，EfDrain/Claim/Materialize/ + PrepareMap/Register/Submit 各 385 条，ClaimMax 为 193 条； +- A5 shared b256 claim PMU 精确得到 92 核各 1,024 次、4 核各 1,088 次， + 合计 98,560； +- shared/private 的 heap、TensorMap signature、symbol 8/5/3、依赖签名、 + split-finish、任务 id 和真实输出 tile 门禁均通过。 + +这些结果证明 early-return 和稀疏观察工具在功能上自洽。它们不证明性能 +收益，最终仍必须服从冻结 ELF 的成对墙钟结果。 + +#### 两级配对结果 + +clean `f41e2833` 冻结件 manifest 和 SHA 均在运行前复核，kernel +`.text=130,104B`。每组比较都先各预热 2 次，再做 6 个交替 +ABBA/BAAB block，每版 12 个正式 b256 独立进程： + +| 对照 | 基线 min/median/max | S4.10b min/median/max | block 配对差中位数 | 结论 | +| --- | --- | --- | ---: | --- | +| S4.10a `e83283f6` | 3.265/3.292/3.316ms | 3.257/3.310/3.336ms | `+17.885us / +0.543%` | 1/6 block 更快 | +| S4.9 `e8320280` | 3.222/3.250/3.276ms | 3.271/3.315/3.348ms | `+62.879us / +1.934%` | 0/6 block 更快 | + +原始数据： + +```text +outputs/perf_clock_pair_f41e2833_vs_e83283f6_20260725_112141/ +outputs/perf_clock_pair_f41e2833_vs_e8320280_20260725_112548/ +``` + +S4.10a 本身相对 S4.9 已回退约 49us，S4.10b 相对 S4.10a 又回退约 +18us；两段增量与最终相对 S4.9 的约 63us 回退方向和量级一致。与此同时， +`.text` 从 S4.9 的 129,080B 增至 S4.10a 的 129,336B，再增至 +S4.10b 的 130,104B；固定 4-owner 也使 winner 明显集中。代码尺寸、 +winner 分布、fanin 和活跃 tile 都是已观测到的伴随变量，但当前证据不能把 +约 63us 唯一归因于其中任何一个。 + +#### 保留与撤销决定 + +按实施前声明的规则，最终判断必须比较 S4.10b 与 S4.9，而不是用 ClaimMax +从 73,728 降至 49,408 代替墙钟收益。由于 6/6 block 稳定回退约 1.93%， +本轮撤销： + +- S4.10a 的 4-owner 固定 Alloc 候选； +- S4.10b 的非候选完整早退； +- 为该稀疏 full-path 形态增加的 converter/analyzer/PMU 分支和测试。 + +实现恢复到 S4.9 的 96-worker Alloc Claim 与完整连续观测路径。本文保留 +S4.10 的设计和失败证据，因为参考分支的唯一候选思想仍有架构价值；真正 +值得重试的方向是同步迁移多 lane×多 shard cursor、分散到更多 owner，再 +作为独立阶段验证，而不是在单层 4-shard ABI 上继续叠加局部条件分支。 + +#### 撤销后的等价性与上板回归 + +撤销完成后，除本记录和 atomic 记录外，工作树与 S4.9 `e8320280` +逐字节一致。回归结果为： + +- 用户 `.venv` 下 PMU/converter/exclusive analyzer 共 85 项测试通过； +- CPU shared/private strict 构建与 b1/b256 回放通过； +- CCEC shared/private 的普通、perf-clock 共 4/4 构建通过； +- shared perf-clock `.text=129,080B`、`.rodata=288B`，重建 + host/kernel SHA 与 `e8320280` 冻结件完全一致； +- A5 shared b1、shared b256、private b1 的真实计算与全部语义门禁通过； +- A5 shared b256 perf-clock 单轮 3.233ms，Claim=73,728， + `active_workers=96`、`max_wins_per_worker=35`，四类 frontier 计数仍为 0。 + +严格的源码与 ELF 等价证明已恢复 S4.9 实现身份；3.233ms 单轮仅验证设备上 +的性能量级，没有被拿来替代此前冻结件 12+12 样本的保留/撤销判据。 + +### 2026-07-25：S4.11 候选收敛到 pure INPUT deferred resolve + +#### 不把参考的 14 slot 当成当前 Case1 优化 + +重新核对后，参考 shared 的 14 和 standalone 的 4 都是每核 kernel 执行 +slot 数，不是 ordinary-region TensorMap ring 深度。当前 4 slot 扣除两个 +BlockWon 预留后有两个普通 slot；参考 14 slot 则有 12 个普通 slot。但当前 +Case1 和参考 PA 都只提交单 lane task，standalone 也没有实现 BlockWon +执行路径。 + +更关键的是，S4.9 冻结配对中 12/12 个正式 b256 样本均为 +`RingBp=0`。当前两个可用 slot 已经没有容量等待，把 4 改成 14 不会删除 +任何已观测开销，反而会： + +- 把 `DrainReady()` 和 `FindFreeSlot()` 的扫描上限从 4 增至 14； +- 每核增加十个 4,824B slot，并移动 WorkerState 后续字段； +- 破坏 standalone 固定的 production prefix 和 `kRealDistGlobalBytes`； +- 把状态布局、代码生成和扫描成本混进容量实验，无法解释因果。 + +因此本阶段不实现、不编译也不上板比较 4→14。只有未来接入多 lane +BlockWon、实测持续出现 RingBp，或整体迁移参考 shared DistCore ABI 时， +才重新建立独立容量实验。 + +#### pure INPUT 为什么是下一项单变量 + +S4.9 的 pure INPUT 当前走： + +```text +Submit 内等待 shared-output published +→ invalidate/copy 128B descriptor 到 LocalSlot +→ 执行前再次等待 producer task flag +→ Kernel +``` + +前一个等待只保证 descriptor 已发布，后一个 flag 才保证 producer kernel +数据完成。参考路径在 Build 时先 try-resolve：ready 就直接复制，未发布就把 +句柄随 slot 保存；drain 在 fanin flag ready 后、Kernel 前完成解析。 + +参考泳道的 491 个 Resolve task 中，若只延迟 pure INPUT，仍覆盖 490 个， +约 99.8%；共可延迟 872 个实际未就绪 pure INPUT ref，同时把 720 个 +INOUT deferred 排除在本轮之外。全量 b256 逻辑图有 1,280 个 pure INPUT +descriptor：SF 256、PV 256、UP 768。由此可在不迁移 writer intent 的前提下 +验证绝大部分调度超前价值。 + +本阶段边界固定为： + +- 只有 plain `SharedOutputRef + Input` 可 deferred； +- INOUT/OutputExisting 继续在 Submit 内等待 publication、校验 + `last_writer==producer`，并保持 + `BuildWinner → FetchMax writer commit → publish outputs`； +- deferred INPUT 仍把原 producer 加入 fanin，不能删除执行门禁； +- resolver 必须位于 fanin flag ready 后、Kernel 前； +- 非法 ref、错误 published/writer 或解析失败必须广播 fatal、只释放一次 + slot，禁止 Kernel 和 completion； +- 不改 Claim、shared heap、ordinary region、slot 数、trace/raw、PMU 和 + private 路径。 + +#### standalone 复用现有 slot 存储，不扩 production prefix + +参考 production ABI 在 RingSlot 中有独立 +`shared_ref_mask/shared_refs[32]`；机械照搬会让 standalone 每个 slot +增加约 520B，并移动固定的 WorkerState/DistGlobal 边界。这会使性能差异 +同时包含 GM 状态体积和缓存布局，不适合作为单变量验证。 + +standalone 首版采用等价协议、不同存储编码： + +- 复用 `LocalSlot.function_padding` 的 4B 保存 32-bit deferred mask； +- 某 tensor 未解析时，其 `slot.tensors[i]` 本来还没有有效 descriptor， + 用 `buffer_addr/buffer_size` 两个既有 `uint64_t` 暂存完整 16B + `FdwicOutputRef`； +- pack/unpack 使用显式位运算，不用跨类型指针别名； +- resolver 先把两个 word 解码到局部 ref，再 invalidate/copy 完整 128B + descriptor 原地覆盖；既有 `slot.args[i]=&slot.tensors[i]` 无需重建； +- 每次 Build 先清 mask，成功 resolve、正常释放和失败释放都清 mask， + 防止 slot 复用继承旧状态。 + +这样 `sizeof(LocalSlot)==4,824B`、WorkerState 全部 offset 和完整 +production prefix 保持不变。它只用于 standalone 隔离验证 deferred 协议， +不能伪称等同参考的 typed slot ABI；若候选有效，迁移真实 simpler 时应复用 +其已有 typed storage，并重新做性能配对。 + +#### 实施与保留门槛 + +先用 CPU 定向测试覆盖 ready/deferred 混合、多 mask 位、pack/unpack、 +延迟发布、错误 publication/writer、INOUT 仍 eager、slot 脏状态复用以及 +失败不执行 Kernel/Completion；再跑 CPU shared/private b1/b256 和 +CCEC shared/private 构建。A5 先做 b1/b256 语义门禁，最后冻结 clean +perf-clock ELF，与 S4.9 `e8320280` 做相同 ABBA/BAAB 配对。 + +主判据仍是完整 Submit；同时用同口径正常构建检查最终 drain/完整 launch。 +若 Submit 变短但最终 drain 等量变长，只能记录为工作后移。只有正确性闭合且 +冻结配对取得净收益才保留；否则像 S4.10 一样完整撤销。 + +#### S4.11 实现、两轮冻结配对与撤销结论 + +S4.11a 曾按上述边界完整实现 pure INPUT deferred resolve： + +- `CollectSharedFanin()` 不再在 Submit 内同步等待 pure INPUT + publication，仍保留 producer fanin； +- Build 对 ready descriptor 直接复制，对未发布引用复用 + `LocalSlot.function_padding` 与 TensorDesc 前 16B 保存 mask/ref； +- Drain 在 fanin ready 后、Kernel 前确认 publication/writer 并复制完整 + descriptor；INOUT/OutputExisting 始终保持 eager； +- `LocalSlot==4,824B`、WorkerState offset、slot 数、raw/PMU/trace ABI、 + Claim、heap 和 ordinary region 均未改变。 + +定向审查还发现并修复了失败收敛缺口：resolver 终止状态必须同时被 EfDrain、 +RingBackpressure 和 FinalDrain 消费；远端 fatal 在持续无进展时低频检查， +清槽后禁止 BuildWinner 继续建 slot。测试覆盖 ready/delayed 混合、bit31、 +越界 mask、脏 slot、非法 publication/writer、INOUT eager、成功/失败 +Drain，以及两个满 slot 依赖失败 task 的终止收敛。CPU shared/private、 +CCEC normal/perf-clock、A5 shared b1/b256 与 private b1 均通过；b256 +继续精确保持 2,048 次 output publication、1,280 个 logical INPUT、 +768 次 INOUT writer commit，以及依赖/heap/writer 签名。 + +第一版 clean 提交为 `b516409e`。它把 resolver 内联到 14 个调用点，使 +shared perf-clock mixed `.text` 从 S4.9 的 129,080B 增至 157,496B。 +与 `e8320280` 做同一 device 0、b256、`real-compute/6,28,4,1`、2 次 +warm-up/版本、6 个 ABBA/BAAB block 后： + +| 版本 | 中位数 | 均值 | 标准差 | block 胜负 | 配对差中位数 | +| --- | ---: | ---: | ---: | ---: | ---: | +| S4.9 `e8320280` | 3.247ms | 3.248ms | 0.010ms | - | - | +| S4.11a `b516409e` | 3.259ms | 3.261ms | 0.034ms | 2 快 / 4 慢 | `+11.105us / +0.342%` | + +符号与 DWARF 审计证明四个大函数和四份 +`ConvergeFatalStall` 解释 99.68% 的 `.text` 增量，其中 14 份 resolver +约占 14.6KB。S4.11b `6275e328` 因此只增加 mask guard 并把 resolver +固定 noinline；重建后精确得到 AIC/AIV orchestration/finish 共 4 份 +904B helper，perf-clock `.text` 降至 144,440B。协议没有变化，CPU/CCEC/ +A5 门禁再次通过。第二轮同口径配对为： + +| 版本 | 中位数 | 均值 | 标准差 | block 胜负 | 配对差中位数 | +| --- | ---: | ---: | ---: | ---: | ---: | +| S4.9 `e8320280` | 3.2452ms | 3.2436ms | 0.0150ms | - | - | +| S4.11b `6275e328` | 3.2451ms | 3.2484ms | 0.0334ms | 1 快 / 5 慢 | `+10.326us / +0.318%` | + +冻结件与逐样本证据位于： + +```text +outputs/perf_clock_freeze_b516409e_20260725_123109/ +outputs/perf_clock_pair_b516409e_vs_e8320280_20260725_123124/ +outputs/perf_clock_freeze_6275e328_20260725_124242/ +outputs/perf_clock_pair_6275e328_vs_e8320280_20260725_124301/ +``` + +两轮方向一致：代码体积收敛有效，但没有把 deferred 协议转化为稳定完整 +Submit 收益。候选 normal/perf-clock 单样本还稳定出现约 88～93 次 RingBp， +而 S4.9 冻结样本为 0。这证明提交确实更早把未就绪任务放进两个可用 slot， +同时把同步 publication 等待转化成了 slot 背压；它是已观测到的伴随机制， +不能据此把约 10us 回退唯一归因给 RingBp、函数调用或 I-cache。 + +按实施前声明的门槛，本提交完整撤销 S4.11 代码，恢复 S4.9 eager pure +INPUT 路径；保留本节设计、正确性补强和失败证据。若未来重试，不能再把 +deferred 当作独立必胜优化：应在参考 typed ref storage 与更深执行 slot +已经就位后，将“deferred + 容量”作为明确耦合的架构候选重新冻结配对； +也不能因为参考使用 14 slot 就在当前 eager 基线上单独扩大状态。 + +撤销后四个实现/测试文件与 `e8320280` 逐字节一致。CPU shared/private +strict 与 b256、用户 `.venv` 下 100 项观测工具测试、CCEC +shared/private normal/perf-clock 4/4 构建，以及 A5 shared b1/b256、 +private b1 real-compute 均通过。shared b256 恢复 `RingBp=0`，1,280 task +和 192 active tile 完整正确。 + +恢复后的 shared perf-clock `.text=129,080B`；当前 host/kernel 与 S4.9 +冻结件分别做 SHA256 和 `cmp`，两者均逐字节相同。A5 shared b256 普通 +构建单样本为 3.344ms，仅作恢复量级检查；S4.11 撤销依据仍是上面两轮 +12+12 冻结配对，而不是这一次运行。 + +### 2026-07-25：S4.12 先裁掉 shared loser 的空 finish 外壳 + +#### 重新核对参考调用链后的优先级修正 + +S4.10 和 S4.11 撤销后，重新逐行核对参考提交 `2866ad73` 的 +`pto_orchestration_api.h` 与 `submit_runtime.h`。参考 shared 路径把一次 +kernel submit 拆为： + +```text +rt_presubmit_task() + -> EfDrain + -> role/candidate 路由 + -> Claim + -> SubmitToken + +winner -> rt_submit_winner(tok, args) + -> Materialize/Register/Fanin/Build +loser -> rt_submit_loser(tok, output_count) + -> 只按 (task_id, output_count) 返回 SharedTaskOutputs +``` + +参考源码在 shared 宏下根本不编译 `dist_submit_loser_impl()`；loser 不会 +为了闭合一个通用接口再进入 Materialize、Register 或 split finish。其稳定 +返回值来自 task/output-slot 符号，而不是 winner 的物理 descriptor。 + +standalone 当前已经做到了“kernel winner 才构造重参数”，但 Claim loser +在 `PrepareSharedTaskOutputs()` 之后仍构造 `CallbackSubmitTicket`，调用 +`FinishCallbackSubmitBody()`。在 perf-clock 构建里，大部分 trace/PMU +模板会被编译删除，然而 generic finish 的控制流、跨 TU 调用和若干空阶段 +仍真实存在。b256 每核回放 1,280 个逻辑 task，96 核共有： + +```text +逻辑 Submit = 96 × 1,280 = 122,880 +唯一 winner = 1,280 +当前 loser finish = 122,880 - 1,280 = 121,600 +``` + +这项差异比下一步直接修改 cursor ABI 更适合作为独立候选： + +- 不改变 Claim 候选集合、winner 分布和任何 atomic 次数； +- 不改变 shared heap、symbol、fanin、slot 或 WorkerState； +- 不引入 S4.10 已观测到的 4-owner winner 集中； +- 不重上 S4.11 的 deferred 协议，也不依赖尚未证明的 slot 深度。 + +因此把原计划中的 `alloc_cursor[3][8]` 后移一阶段。这个调整来自参考源码 +调用链，而不是根据单次时间猜测。 + +#### S4.12a 的固定边界 + +本阶段只裁 shared loser 的 **Claim 之后、generic finish 之前** 的空路径： + +1. 所有 worker 仍执行现有 EfDrain 和 Claim；Alloc 仍由 96 核竞争， + QK/PV 仍由 32 个 AIC 竞争，SF/UP 仍由 64 个 AIV 竞争； +2. 每个 actor 都先建立相同的 `SharedTaskOutputs`，保证后续 orchestration + 得到完全一致的 `(task_id, output_slot)`； +3. Alloc 的三个静态 Output 参数仍由所有 actor 构造，因为参考 + `alloc_tensors(args)` 的参数构造发生在 inner submit 入口外；不能借 + loser 早退偷删这部分调用方成本； +4. kernel loser 不构重参数，并在稳定符号建立后直接做逻辑收尾；只有 + 1,280 个 winner 进入 `FinishCallbackSubmitBody()` 和 split finish； +5. 每核 `submits` 仍必须等于 `5*batches`。若末个 UP 是本核 loser， + 早退路径仍负责记录该核最后一个 Submit 的 perf-clock/PMU 结束边界； +6. private 路径逐字节保持原控制流，不共享 shared 的早退分支。 + +成功路径不为这项优化增加新的 fatal load，也不新增 raw 字段。异常路径 +仍是有限回放：任何 winner 或既有协议检查发布 fatal 后，本轮结果整体 +无效；loser 快返不能发布 descriptor、task flag 或 completion，因而不会 +把错误路径误提交为可执行任务。 + +#### 观察口径必须跟真实路径一起收敛 + +不能为了让旧工具继续看到整齐矩形而给 loser 伪造零时长 +Materialize/Register/Submit。S4.12a 的泳道应真实呈现： + +| 阶段 | b256 预计调用数 | +| --- | ---: | +| 逻辑 Submit | 122,880 | +| EfDrain | 122,880 | +| Claim span | 122,880 | +| 实际 ClaimMax | 73,728 | +| Materialize/PrepareMap/Register/Submit span | 1,280 | +| split finish | 1,280 | + +`submit-pmu none` 仍覆盖每核完整 orchestration/Submit 窗口；`claim` 和 +`efdrain` 局部变体仍覆盖所有逻辑 task；`materialize` 和 `register` +只覆盖 winner。converter/exclusive analyzer 根据已有 Claim winner +信息接受这种稀疏主体，不新增 device 记录字段。泳道空白继续归入既有 +`BetweenSubmitResidual`，不能另造 loser replay 区域扩大 raw。 + +#### 正确性、性能门禁和后续顺序 + +实现后先用 CPU 定向锁定： + +- 五类 loser 都返回正确 output symbol，且不改变 heap、publication、 + writer、slot、completion 和依赖状态； +- Alloc loser 仍精确构造三项静态 Output 参数； +- split finish 调用数和 task-id 和只等于本核 winner 集合，逻辑 submits + 仍为完整 `5*batches`； +- 末 task loser 仍闭合首末性能边界； +- private 的 eager loser finish 与原计数不变。 + +随后运行 Python converter/analyzer/PMU 测试、CPU shared/private b1/b256、 +CCEC shared/private normal/perf-clock/submit-PMU 构建以及 A5 shared +b1/b256 和 private b1。clean 实现提交冻结后，与 S4.9 `e8320280` 使用 +同一 device、两次预热、六个 ABBA/BAAB block 做 12+12 正式配对。 + +只有正确性闭合且完整 Submit 取得稳定净收益才保留 S4.12a。若无收益则 +完整撤销，不以“少了 121,600 次函数调用”替代墙钟结果。若保留，再分开 +验证： + +1. S4.12b：把 single-lane wrong-role 判断前移，减少无资格 actor 的 + Claim/EfDrain 外壳； +2. S4.13：参考的 `alloc_cursor[3 lanes][8 shards]` 与 24-owner 映射； +3. 再比较 kernel cursor 8/16 分片； +4. 最后用独立四格实验评估 eager/deferred 与 4/14 slot 的耦合关系。 + +上述四项不能合成一个提交，否则无法判断收益来自 loser 裁剪、winner +分布、cursor cacheline 冲突还是在途容量。 + +#### S4.12a 实测结论：路径正确，但墙钟中性，已撤销 + +候选提交 `b2fe435f` 按上述边界完成了 shared loser 快返。CPU guard-page +测试证明 loser 在构造稳定输出引用后不再读取 TaskArgs 后半区；Host +oracle、split finish、raw 泳道、局部 PMU 和 Python 工具也都收敛为 +winner-only 的 Materialize/PrepareMap/Register/Submit。验证覆盖： + +- 用户 `.venv` 下 114 项 converter、exclusive analyzer 和 PMU 测试； +- CPU private/shared b1、shared b256 及 perf-clock； +- CCEC private/shared 的 swimlane、perf-clock 和五种 submit-PMU， + 共 14 种构建； +- A5 shared/private b1 泳道，以及 shared Materialize/Register PMU； + b1 的五个 winner 分散在五个核，其余 91 核为合法零调用。 + +这些结果证明路径与观察契约正确，但不能替代性能门槛。候选 shared +perf-clock `.text` 从 S4.9 的 129,080B 增至 130,360B。冻结 +`e8320280` 与 `b2fe435f` 后，在同一 device 上各预热两次，再执行六个 +ABBA/BAAB 区组；每版包含 12 个独立 b256 正式进程，范围均为 +`real-compute 6,28,4,1`、`two-16`，计时边界为首个 Submit 起点到最后 +一个 Submit 返回： + +| 指标 | S4.9 `e8320280` | S4.12a `b2fe435f` | +| --- | ---: | ---: | +| 正式样本数 | 12 | 12 | +| 最小值 | 3,213.912us | 3,210.795us | +| 中位数 | 3,243.442us | 3,242.090us | +| 均值 | 3,241.991us | 3,240.591us | +| 最大值 | 3,279.896us | 3,256.256us | +| 标准差 | 16.733us | 14.042us | + +候选均值表面快 1.400us（约 0.043%），但六个区组恰好 3 快、3 慢， +区组配对差中位数仅为 `-0.477us / -0.015%`。该差异远小于样本波动， +只能判定为中性，不能宣称减少 121,600 次源码级 generic finish 调用带来 +了可测收益。原始日志与机器可读汇总位于： + +```text +outputs/perf_clock_pair_e8320280_vs_b2fe435f_20260725_135812/ +``` + +因此按预先声明的净收益门槛撤销 `b2fe435f`，同时撤销为稀疏 loser +泳道和 PMU 新增的工具分支。撤销后重新构建的 Host 与 S4.9 冻结件 SHA +一致；device ELF 的完整文件因调试/符号信息存在构建差异，但实际执行的 +`.text=129,080B` 与 `.rodata=288B` 均逐字节一致。后续不再开展 +S4.12b，因为它仍只裁剪同一类空外壳；撤销时确定的下一候选是会改变共享 +状态访问形状的 `alloc_cursor[3][8]`，其正确性、winner 分布和性能在 +下节作为独立阶段重新验证。 + +### 2026-07-25：S4.13 `3×8` Alloc cursor 功能闭合但性能中性，已撤销 + +#### 参考实现、候选边界与正确性 + +重新按提交核对参考分支后，确认 shared Alloc 的最终形态由两个提交组成： + +- `0350b558` 把 `DistGlobal::alloc_cursor` 改成三条物理 lane、每条八个 + shard，并在 Claim 中按 `task_id%3` 选择 lane； +- `076f1265` 再按 `task_id&7` 选择 shard、按 + `shard%num_blocks` 选择唯一 block，并把非候选早退前移到 EfDrain 之前。 + +S4.13 候选 `327de856` 只迁移 `3×8` cursor 和 24-owner Claim 拓扑， +不同时迁移 pre-EfDrain 早退、cube/vector cursor、slot 深度、延迟解析或 +shared heap。standalone 固定 32 个 mixed block，八个 shard 均小于 block +数，因此映射精确为： + +```text +lane = task_id % 3 +shard = task_id & 7 +block = shard +``` + +新 cursor 追加在 shared-only sidecar 尾部，既有 production prefix、 +region、output 和 heap 字段 offset 不动；这只声明访问拓扑与参考一致， +不伪称参考 `DistGlobal` 字节布局一致。全部 96 个 actor 仍保留 EfDrain、 +Claim span、参数构造、generic finish、Submit、perf-clock 和 PMU 边界， +非 owner 只在 Claim 内返回 `attempted=false`。固定 owner 若发起 FetchMax +后仍未获胜会发布 fatal，避免把状态未复位或越序推进静默解释成 Replay。 + +物理 Claim 从 `288B` 降到 `193B`，所以 b256 从 73,728 精确降到 +49,408，删除 24,320 次 Alloc `ClaimMax`。256 个 Alloc winner 分散到 +24 个 owner，其中 16 个 owner 各 11 个、8 个 owner 各 10 个；这与 +S4.10 集中到四个 owner 的候选不同。 + +新增定向测试覆盖唯一 owner、物理 worker 映射、非 owner 零 FetchMax、 +同 cursor 的 `0→120→240`、`120→0` 乱序 fatal，以及 QK/SF 仍使用原 +cube/vector cursor。功能门禁包括： + +- 用户 `.venv` 下 100 项 Python 测试； +- CPU shared/private strict 构建，shared b1/b24/b256 与 private + b1/b256 完整回放； +- CCEC shared/private 的 swimlane、perf-clock 和五种 submit-PMU， + 共 14 种构建及 manifest SHA； +- A5 shared b1 atomic 泳道、shared b24 perf-clock 和 private b1 + perf-clock。 + +shared b1 raw 共 4,034 条且零丢失；193 条 `ClaimMax` 与 Claim +attempted 一一闭合，按 Alloc/QK/SF/PV/UP 分为 `1/32/64/32/64`， +全部是 `return_ready` 边界。候选 shared perf-clock `.text=129,592B`， +比 S4.9 增加 512B，`.rodata` 同为 288B。功能取证泳道位于： + +```text +outputs/pa_scheduler_shared_swimlane_20260725_143815_2563482/ccec/ +``` + +#### 冻结配对与撤销判定 + +clean 候选冻结在： + +```text +outputs/perf_clock_freeze_327de856_20260725_145606/ +``` + +与 S4.9 `e8320280` 在同一 device 0 上各预热两次，再执行六个 +ABBA/BAAB 区组；每版 12 个正式 b256 独立进程，均使用 +`real-compute 6,28,4,1`、`two-16`，计时范围为首个 Submit 起点到最后 +一个 Submit 返回。全部样本都满足 `active_workers=96`、`RingBp=0` 和 +完整语义断言。 + +| 指标 | S4.9 `e8320280` | S4.13 `327de856` | +| --- | ---: | ---: | +| ClaimMax | 73,728 | 49,408 | +| 最小值 | 3,224.547us | 3,219.012us | +| 中位数 | 3,242.079us | 3,244.868us | +| 均值 | 3,247.788us | 3,245.274us | +| 最大值 | 3,284.128us | 3,268.426us | +| 标准差 | 19.917us | 15.907us | +| `max_wins_per_worker` 范围 | 29～42 | 27～32 | + +候选均值表面快 2.514us，但中位数慢 2.789us；六个区组恰好 3 快、3 慢， +区组配对差中位数为 `+3.666us / +0.113%`。winner 集中度和 atomic 次数 +都按设计下降，但完整 Submit 没有稳定同向收益。按实施前声明的门槛， +`0～3/6` 区组更快且配对差中位数非负时直接撤销，不启动第二轮，也不叠加 +参考的 pre-EfDrain 早退来掩盖本候选结果。 + +原始日志、逐样本表和机器可读汇总位于： + +```text +outputs/perf_clock_pair_327de856_vs_e8320280_20260725_145751/ +``` + +因此完整撤销 `327de856` 的 source、ABI、测试和当前行为文档，恢复 S4.9 +的四 shard/96-worker Alloc Claim。保留本节作为负结果：`3×8`/24-owner +确实解决了 S4.10 的 winner 集中并消减 24,320 次 return-ready FetchMax， +但在当前代码布局和调度路径中仍不足以形成可测净收益。以后若研究参考的 +pre-EfDrain 早退，必须把它作为新的独立候选相对 S4.9 冻结配对，不能把 +本轮 atomic 消减记成已保留的性能优化。 + +撤销后，非文档源码与 `e8320280` 逐字节一致；用户 `.venv` 下 100 项 +Python 测试、CPU shared strict/perf-clock 构建和 b256 完整回放再次通过。 +重建的 shared perf-clock Host 与 S4.9 冻结件 SHA 完全相同；device ELF +因调试/符号节存在构建差异，但实际执行的 `.text=129,080B` 和 +`.rodata=288B` 内容逐字节一致。因此当前运行身份确实回到 S4.9，而不是 +只在源码表面删除了候选分支。 + +### 2026-07-25：S4.14a shared Vector cursor 迁址对照 + +#### 为什么不能直接照搬参考的 `cube=8/vector=16` + +重新逐提交审查参考历史与本仓旧实验后，得到以下边界： + +- 参考 `e49f73a3` 把 cube/vector/alloc 一起从四分片改为八分片，同时还 + 修改延迟 shared ref 解析、BlockWon drain、fatal/frontier 和 trace； +- 参考 `0350b558` 再把 vector 从八分片改为十六分片，但同一提交还引入 + `alloc_cursor[3][8]` 并删除 shared completion frontier； +- 本仓 `2e92da17` 记录的 A5 sweep 也让 cube/vector/alloc 三类 cursor + 同时选择 `G=1/4/8/16`。standalone 的 `G4→G8` 虽然是 + `-176.631us/-4.632%`、7/7 配对更快，但 fanin loads 同时从 + 29,504 降到 24,601，不能把收益归因给某一类 cursor；真实 simpler + 的 `G4→G8` 只有三个样本且为 `+0.526%`，只能判定没有可辨认变化。 + +因此参考配置只证明可运行形态，不能作为独立收益证据。进一步审查还确认： +若直接把 shared Vector 从 production prefix 的四分片改成 sidecar +八分片，会同时改变 cursor 地址、页/cache 映射和分片数，仍不能把结果 +单独归因给 `4→8`。 + +S4.14 因而拆成两步。S4.14a 只做迁址对照：sidecar 预留八条物理 +cache line，但 active shards 保持 4，SF/UP 仍按 `%4` 映射。 +Cube、Alloc、private、frontier、task 图、观察器和 Claim 候选集合全部 +不变。后续 S4.14b 才在相同 sidecar 地址、物理容量和代码骨架下只把 +有效分片数和取模从 4 改为 8。 + +选择 Vector 作为后续首个分片候选，是因为 b256 中 SF/UP 固定执行： + +```text +64 AIV × 2 tasks/batch × 256 batches = 32,768 ClaimMax +``` + +而 QK/PV 的 Cube ClaimMax 为 16,384 次。S4.14a 尚未改变分片，所以 +四条 active Vector cursor 每条仍承担 8,192 次 ClaimMax，另外四条 +物理线保持 -1。未来 S4.14b 才会把每条流量降到 4,096，但即使届时也 +不会减少： + +- 总计 32,768 次 Vector ClaimMax； +- 每个 SF/UP task 的 64 路同地址竞争； +- 73,728 次全局 ClaimMax； +- winner、fanin、completion 或 kernel 数量。 + +本阶段只量同样四分片从 prefix 搬到 sidecar 的影响，不把它描述成 +atomic 次数消减或分片收益。 + +#### ABI 与实现边界 + +production prefix 的 `cube_cursor[4]`、`vector_cursor[4]` 和 +`alloc_cursor[4]` 均不移动。shared-only sidecar 尾部追加完整的 +`shared_vector_cursor[8]`，SF/UP 在 shared 编译中按 +`task_id % kSharedVectorCursorShards` 访问当前四条 active line； +这与 S4.9 的 `%4` 源码运算形态一致。private 继续使用 prefix +`vector_cursor[4]`。这样保持 private 与现有 production offset 冻结， +也为下一步同址 `4→8` 建立严格对照；不伪称 sidecar 地址等于参考 +`DistGlobal` 地址。 + +sidecar 从 4,735,680B 增至 4,736,192B，既有 region/output/heap 字段 +offset 均不变。Host 仍通过 `SharedSidecarBytes()` 整块 H2D/D2H； +每轮把新八条 cursor 初始化为 -1，终态 oracle 要求： + +- 四条 active cursor 精确等于 b1/b256 最终 task 高水位，另外四条 + 始终为 -1； +- 旧 prefix `vector_cursor[4]` 在 shared 中始终为 -1； +- Cube/Alloc 仍按原四分片达到精确终值。 + +没有新增 trace、PMU、WorkerResult、span 或 atomic 记录字段。 + +#### 候选正确性取证 + +新增 CPU 定向测试不只读取最终值，还在每次 attempted 后核对 FetchMax +地址并累计次数。它锁定： + +- task 2 与 task 14 迁址后仍同属 sidecar shard 2，证明映射仍为四分片； +- AIC 对 SF/UP 不发 FetchMax，shared Vector 不触碰旧 prefix； +- Cube/Alloc 继续访问 production-prefix 四分片； +- b256 每个 SF/UP 恰有 64 个 AIV attempted 和一个 winner，合计 + 32,768 次 Vector ClaimMax；四条 active 高水位与四条 inactive -1 + 精确闭合。 + +这里的定向用例按顺序模拟路由、地址与计数；真正的并发唯一 winner 由 +完整 CPU 96-worker b1/b256 回放验证，不能把两种证据混成一个结论。 + +当前已通过： + +- 用户 `$HOME/.venv` 下 100 项 Python converter/analyzer/PMU + 回归； +- CPU shared 全部定向测试、shared b1/b256 完整回放，以及 private + strict/perf-clock 回归； +- CCEC private/shared 的 swimlane、perf-clock 和五种 submit-PMU， + 共 14 种构建，全部 manifest 校验通过； +- A5 shared b1 perf-clock 与 atomic 泳道，heap/TensorMap、依赖签名、 + completion、真实计算输出和新 cursor oracle 全部通过。 + +b1 最终源码泳道共有 4,127 条 raw、零丢失；288 条 `ClaimMax` 仍按 +Alloc/QK/SF/PV/UP 精确分为 `96/32/64/32/64`，flags 均为 +`0x53 return_ready`。证据位于： + +```text +outputs/pa_scheduler_shared_swimlane_20260725_155729_2638154/ccec/ +``` + +迁址对照 shared perf-clock `.text=129,080B`、`.rodata=288B`，大小与 S4.9 +相同。private device 的 `.text=125,752B`、`.rodata=300B` 与已有 +`dc22d076` 冻结件内容逐字节一致；完整 ELF 只因调试节/对齐多 16B。 +private Host 当前使用用户 GCC15，而旧冻结件使用 GCC13.3，因此不把 Host +全文件差异误写成 shared 代码泄漏。 + +#### 预声明门槛与冻结配对结果 + +正确性结果本身不代表已有收益。S4.14a 提交为 `e24e579c` 后冻结 +clean shared perf-clock ELF，并相对 S4.9 `e8320280` 在 device0 上各预热 +两次，再执行六个 ABBA/BAAB 区组、每版 12 个独立 b256 正式进程。 + +测量前固定迁址对照的数值门槛。首轮六区组按候选减基线计算： + +- 任何语义失败，或配对中位数 `>= +0.2%` 且仅 `0～2/6` 区组更快： + 判回退并撤销 sidecar； +- 配对中位数 `<= -0.2%` 且 `6/6` 更快：只把收益记给迁址； +- 配对中位数落在 `(-0.2%, +0.2%)` 且 `2～4/6` 更快：判中性， + 可作为 S4.14b 的同址对照； +- 其他边界组合追加第二轮六区组。 + +若追加到十二区组，只在 `<=-0.2%` 且至少 `10/12` 更快时判改善； +只在绝对中位差 `<0.2%` 且 `5～7/12` 更快时判中性；其他组合均视为 +不适合继续叠加的迁址结果并撤销。这样不会在看到数据后临时放宽“中性”。 + +S4.14b 使用互斥六区组门槛：任何语义失败、0～3/6 更快或配对差中位数 +`>=0` 时直接撤销；6/6 更快且中位数 `<=-0.2%` 时直接保留；其余负向 +边界才追加第二轮。合并十二个区组后，仅在至少 10/12 更快且中位数 +`<=-0.2%` 时保留，其他结果全部撤销。只有 Vector `4→8` 最终保留后， +才相对新冻结基线单独测试 Cube `4→8`，再测试 Vector `8→16`。任何 +一步失败都回到上一冻结基线，不能堆叠后再猜收益来源。 + +正式六区组结果如下。表中差值均为区组内候选两次均值减去基线两次均值， +不是把两边各 12 个总体中位数直接相减： + +| 区组 | S4.9 均值 | S4.14a 均值 | 差值 | 百分差 | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 3,254.115us | 3,085.436us | -168.680us | -5.184% | +| 2 | 3,247.635us | 3,088.239us | -159.396us | -4.908% | +| 3 | 3,255.936us | 3,064.637us | -191.299us | -5.875% | +| 4 | 3,232.131us | 3,072.167us | -159.964us | -4.949% | +| 5 | 3,270.661us | 3,071.275us | -199.387us | -6.096% | +| 6 | 3,239.886us | 3,082.049us | -157.837us | -4.872% | +| 配对中位数 | - | - | **-164.322us** | **-5.066%** | + +候选 6/6 区组更快且幅度远超过预声明的 `-0.2%`，因此按门槛直接保留 +S4.14a，不追加第二轮。S4.9 的 12 个正式样本中位数为 3,249.714us， +S4.14a 为 3,073.963us；这两个总体中位数仅描述分布，不替代上表的配对 +主指标。 + +28 个独立进程日志经过二次审计，每份都有 42 条 PASS 断言，并统一满足： + +- 两版 manifest 身份均为 shared/perf-clock/none，Host 与 kernel SHA + 与各自冻结清单一致； +- 73,728 次 ClaimMax、96 个 active worker、RingBp=0； +- 依赖边 1,280、签名 `b7d985d6edb07078`，QK/SF/PV/UP 各 256 次； +- `global_end_tick - global_start_tick == global_span_ticks`，且 1ns + SYS_CNT tick 精确对应 `[METRIC] submit_span_us`； +- execution、semantic、postprocess 和真实计算输出全部 PASS。 + +证据目录： + +```text +outputs/perf_clock_freeze_e24e579c_20260725_160026/ +outputs/perf_clock_pair_e24e579c_vs_e8320280_20260725_160507/ +``` + +当前证据能归因的是“shared Vector 保持四分片并整体迁到 sidecar”，不能 +进一步断言是哪一级 cache、atomic 单元或地址映射带来的收益。新增的 512B +位于 `SchedulerState` 最后；`shared_map` 本就是最后一个字段，所以没有 +移动 production prefix、standalone controls、results 或任何既有 shared +字段。额外 H2D/D2H 字节也发生在首末 Submit 窗外。由此当前 standalone +shared 性能基线前移到 `e24e579c`。 + +下一步 S4.14b 只允许把 `kSharedVectorCursorShards` 从 4 改为 8。 +sidecar offset、物理容量、state 大小、初始化、Host 传输和寻址表达式均 +冻结不动。若 S4.14b 相对 S4.14a 通过自身门槛，仍需再相对 S4.9 核对 +最终净收益，避免把迁址容忍和分片变化叠加后只看局部比较。 + +### 2026-07-25：S4.14b shared Vector cursor 同址八分片实现与结果 + +#### 唯一运行时变量 + +S4.14a 已把 sidecar 地址效应单独量清。S4.14b 以 `e24e579c` 为冻结 +基线，只把： + +```cpp +kSharedVectorCursorShards = 4 +``` + +改为： + +```cpp +kSharedVectorCursorShards = 8 +``` + +`Claim()` 仍使用同一条 +`task_id % kSharedVectorCursorShards` 表达式。sidecar 字段 offset +4,735,680B、物理容量 8、`SchedulerState` 大小、Host 初始化和 H2D/D2H +跨度均不变；private、Cube、Alloc、heap、TensorMap、fanin、completion、 +观察器和构建身份协议也不动。因此候选 perf-clock 相对 S4.14a 的 device +热路径差异只应来自有效取模范围 4→8。 + +b256 的总调用和每 task 竞争面保持： + +```text +Vector ClaimMax = 64 AIV × 2 tasks/batch × 256 = 32,768 +全局 ClaimMax = 73,728 +每个 SF/UP = 64 个 attempted、1 个 winner +``` + +变化只在跨 task 的物理线分布：四分片时每线 8,192 次，八分片时每线 +4,096 次。单个 task 的 64 个 AIV 仍竞争同一条线，所以不能把本候选描述成 +每 task fan-in 缩小或 atomic 次数消减。 + +#### 正确性与性能门槛 + +定向测试同步改为验证 task 2 落 sidecar shard 2、task 14 落 shard 6， +并对 b256 的每一次 attempted FetchMax 核对 `task_id%8` 实际地址。 +Host 终态 oracle 从 task 序列重新推导八条高水位；旧 prefix Vector 必须 +仍全为 -1，Cube/Alloc 仍为 production-prefix 四分片。 + +候选提交前必须完成： + +- CPU shared 定向测试和 96-worker b1/b256 完整回放； +- 用户 `.venv` 下 100 项观察工具回归； +- CCEC private/shared 的 swimlane、perf-clock 和五种 submit-PMU， + manifest 全部校验； +- private device 执行节与既有冻结件一致； +- A5 shared b1 perf-clock 与 atomic 泳道，Claim 分布和全部业务 oracle + 闭合。 + +性能只使用提交后的冻结 shared perf-clock ELF，与 `e24e579c` 做相同 +device0、b256、`real-compute 6,28,4,1`、two-16 配对。首轮六区组 +门槛按以下互斥顺序执行： + +- 任何语义失败、0～3/6 更快或配对百分差中位数 `>=0`：直接撤销； +- 6/6 更快且配对百分差中位数 `<=-0.2%`：直接保留； +- 其余负向边界，即 4～5/6 且中位数 `<0`，或 6/6 但中位数在 + `(-0.2%,0)`：追加第二轮六区组。 + +合并十二个区组后，只在至少 10/12 更快且配对百分差中位数 +`<=-0.2%` 时保留，其他结果全部撤销。 + +当前正确性证据已经闭合： + +- CPU shared 定向测试显式验证 task 2/14 分别落 shard 2/6，b256 每条 + sidecar 线恰有 4,096 次 attempted；shared b1/b256 完整回放通过； +- 用户 `$HOME/.venv` 下 100 项 Python 测试通过； +- CCEC private/shared 的 14 种构建与 manifest 全部通过；shared + perf-clock `.text=129,080B`、`.rodata=288B`，与 S4.14a 大小相同； +- private perf-clock `.text=125,752B`、`.rodata=300B` 的内容 SHA + 与 `dc22d076` 冻结件逐字节一致； +- A5 shared b1 perf-clock 为 65.435us，全部语义与真实计算输出 PASS； +- 最终 atomic 泳道 raw 4,120 条、drop=0，288 条 ClaimMax 仍按 + Alloc/QK/SF/PV/UP 分为 `96/32/64/32/64`，flags 全为 + `0x53 return_ready`。 + +另做了编译确定性审计：从提交 `e24e579c` 重新导出源码并重建 +perf-clock，`.text/.rodata` 与冻结件逐字节一致，证明 CCEC 本轮可复现。 +S4.14b 的 `.rodata` 与 S4.14a 仍逐字节一致，`.text` 大小相同但有 +50,103 个字节位置变化。这不是发现了额外源码业务改动，而是静态常量 +4→8 经内联和链接后引发的编译布局连锁变化。因此正式配对衡量的是 +“生产形态的静态八分片构建”整体效果；即使出现收益，也不能只凭该数字 +把它进一步拆成 atomic 竞争收益与 I-cache/代码布局收益。若需要机制取证， +可另做同 ELF selector 辅助实验，但不能用带运行时 selector 的绝对时间 +替代静态 perf-clock 定案。 + +最终源码泳道证据位于： + +```text +outputs/pa_scheduler_shared_swimlane_20260725_163417_2671540/ccec/ +``` + +#### 冻结配对结果 + +S4.14b 提交为 `ee42b8c1` 后冻结 clean ELF。相对同址四分片 +`e24e579c` 的正式六区组如下： + +| 区组 | 四分片均值 | 八分片均值 | 差值 | 百分差 | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 3,077.091us | 2,353.955us | -723.136us | -23.501% | +| 2 | 3,095.143us | 2,347.295us | -747.847us | -24.162% | +| 3 | 3,075.267us | 2,363.523us | -711.744us | -23.144% | +| 4 | 3,075.176us | 2,354.237us | -720.939us | -23.444% | +| 5 | 3,108.386us | 2,369.363us | -739.023us | -23.775% | +| 6 | 3,060.023us | 2,362.263us | -697.760us | -22.802% | +| 配对中位数 | - | - | **-722.037us** | **-23.472%** | + +候选 6/6 更快且远超过 `<=-0.2%` 门槛，直接保留，无需追加第二轮。 +四分片 12 个正式样本中位数为 3,082.997us，八分片为 +2,356.598us；总体中位数只描述分布，正式判定仍以上表区组配对为准。 + +由于 S4.14a 本身含一次迁址，随后又把 `ee42b8c1` 直接与 S4.9 +`e8320280` 做相同六区组净收益复核： + +| 区组 | S4.9 均值 | 当前均值 | 差值 | 百分差 | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 3,256.927us | 2,354.255us | -902.672us | -27.715% | +| 2 | 3,234.991us | 2,361.163us | -873.827us | -27.012% | +| 3 | 3,253.052us | 2,350.930us | -902.122us | -27.732% | +| 4 | 3,247.256us | 2,359.753us | -887.503us | -27.331% | +| 5 | 3,257.834us | 2,358.168us | -899.666us | -27.615% | +| 6 | 3,254.939us | 2,352.250us | -902.689us | -27.733% | +| 配对中位数 | - | - | **-900.894us** | **-27.665%** | + +直接净收益同样是 6/6，更快结论不依赖把两个局部百分比相加。S4.9 +12 个正式样本中位数为 3,250.591us，当前为 2,357.489us。 + +两轮各 28 个独立进程都经过与 S4.14a 相同的二次审计：每份 42 条断言 +全部 PASS，ClaimMax=73,728、active workers=96、RingBp=0、依赖签名 +`b7d985d6edb07078`、QK/SF/PV/UP 各 256，SYS_CNT 首末差与输出 +微秒值精确一致。 + +辅助计数还表明八分片并非通过少做后续工作取巧。相对同址四分片，正式 +样本的 fanin loads 中位数从 24,041.5 增至 38,458（+59.965%）， +fanin not-ready 从 19,953.5 增至 32,410，EfDrain/FinalDrain 中位数 +从 1008/16 变为 995/29，CAS retries 两边仍为 0；同时单核最大 winner +中位数从 35.5 降到 18。也就是说候选在更多 fanin 轮询下仍明显更快， +并改变了 winner 分布。该现象支持“跨 task cursor 冲突/调度形态发生 +变化”，但结合前述 50,103 字节代码布局差异，不能把 23.472% 全部 +宣称为 atomic 硬件等待下降。 + +冻结与逐样本证据位于： + +```text +outputs/perf_clock_freeze_ee42b8c1_20260725_163655/ +outputs/perf_clock_pair_ee42b8c1_vs_e24e579c_20260725_163800/ +outputs/perf_clock_pair_ee42b8c1_vs_e8320280_20260725_164329/ +``` + +当前 standalone shared 性能基线前移到 `ee42b8c1`。下一步若测试 +Cube `4→8`，不能扩张 production-prefix `cube_cursor[4]` 并移动真实 +ABI；应先仿照 S4.14a 在 shared-only sidecar 追加容量 8、active 仍为 4 +的 Cube cursor 做迁址对照，再在同址下只改 active 4→8。 + +### 2026-07-25：S4.15a shared Cube cursor 迁址对照与性能否决 + +#### 单一变量与布局边界 + +本节在性能取数前固定候选设计和判据。当前可比较基线是已保留的 +S4.14b `ee42b8c1`：shared Vector 已使用 sidecar 中全部八条 active +cursor，Cube/Alloc 仍使用 production prefix 四分片。若直接把 prefix +`cube_cursor[4]` 扩成八条，既会移动 production ABI，又会同时改变地址和 +分片数，无法归因。 + +S4.15a 因此只做 Cube 迁址对照：在现有 sidecar 尾部追加物理容量为 8 的 +`shared_cube_cursor`，但 active shards 保持 4，shared QK/PV 仍按 +`task_id % 4` 路由。以下部分全部冻结不动： + +- production-prefix `cube_cursor[4]`、`vector_cursor[4]` 和 + `alloc_cursor[4]` 的大小、offset 与 private 路由； +- 已保留的 shared `shared_vector_cursor[8]` 地址、八分片路由与终态； +- shared heap、TensorMap、fanin、completion、task 图、winner 候选集合； +- trace、PMU、WorkerResult、span、atomic 记录和构建身份字段的布局与 + 握手协议；`scheduler_state_size` 的运行时取值必须随完整 state + 增长 512B，不能伪装成旧大小。 + +新字段紧接 `shared_vector_cursor[8]`，offset 为 4,736,192B。sidecar +从 4,736,192B 增至 4,736,704B；CPU non-split `SchedulerState` 从 +1,011,852,160B 增至 1,011,852,672B，CCEC split `SchedulerState` +从 1,011,858,304B 增至 1,011,858,816B。新增 512B 位于完整 state +末尾，不移动任何旧字段；Host 继续通过 `SharedSidecarBytes()` 整块 +初始化和传输。 + +这仍不是“只换一个 atomic 地址而其余二进制完全相同”的实验。尾部 +新增 512B 会改变 GM 分配长度，可能改变基址后的页映射；更新后的 +`scheduler_state_size` 也会作为编译期常量进入 AIC/AIV 执行节，并可能 +连带改变静态代码布局。因此配对结果只能归因于“shared Cube 四分片 +迁址候选整体”，不能直接解释成某条 atomic 的纯硬件等待差。S4.15b +必须复用本阶段同一地址、容量与 state 大小，才可继续隔离 active +分片数 `4→8` 的增量。 + +#### Atomic 数量与正确性门禁 + +b256 中 QK/PV 的 Cube ClaimMax 固定为： + +```text +32 AIC × 2 tasks/batch × 256 batches = 16,384 +``` + +迁址不减少这 16,384 次 atomic，也不改变每个 QK/PV task 的 32 路 +同地址竞争。四条 active sidecar line 各承受 4,096 次 attempted， +另外四条 inactive line 的 attempted 次数必须为 0，cursor 终值保持 +初始化值 -1。全局 ClaimMax 仍为 73,728；Vector 仍为 32,768 次并 +均分到八条 active sidecar line，Alloc 仍为 prefix 四分片。 + +候选提交前必须逐项闭合： + +- ABI 静态断言锁定新字段 offset、sidecar 和两种 `SchedulerState` + 精确大小，旧字段 offset 全部不变； +- Host 初始化全部八条 Cube 物理线为 -1；终态 oracle 要求旧 prefix + Cube 在 shared 模式始终为 -1，四条 active sidecar line 达到按 task + 序列推导的精确高水位，四条 inactive line 始终为 -1； +- CPU 定向测试逐次核对 QK/PV FetchMax 的真实地址为 + `shared_cube_cursor[task_id % 4]`，并闭合 16,384 次总量、 + active 每线 4,096 次、inactive 每线 0 次；同一 shared 定向程序 + 继续锁定 Vector8 路由与 Alloc task0 的 prefix 地址； +- CPU shared 96-worker b1/b256 完整回放、private 完整回归继续证明 + Alloc 四分片和 private 三类路由不变；用户 + `$HOME/.venv` 下观察工具测试全部通过； +- CCEC private/shared 的 swimlane、perf-clock 和五种 submit-PMU + 共 14 种构建及 manifest 全部通过，private device 执行节与既有 + 冻结件逐字节一致； +- A5 shared b1 perf-clock 与 atomic 泳道闭合全部业务 oracle， + ClaimMax 数量、分类和 `return_ready` 语义不变。 + +顺序定向测试只证明路由、地址和计数；唯一 winner 与并发闭合仍由 +96-worker 完整回放和 A5 结果证明。不得把两种证据互相替代。 + +#### 冻结配对与预声明性能判据 + +只有上述正确性门禁全部通过并形成独立提交后，才冻结 S4.15a shared +perf-clock ELF，与 `ee42b8c1` 使用同一 device0、b256、 +`real-compute 6,28,4,1`、two-16、独立进程和 ABBA/BAAB 协议配对。 +差值统一按“候选减基线”计算；swimlane、submit-PMU 或单次可执行性样本 +不参与保留判断。 + +首轮六区组按以下互斥规则判定： + +- 任何语义失败，或配对百分差中位数 `>=+0.2%` 且仅 `0～2/6` + 区组更快：撤销 S4.15a； +- 配对百分差中位数 `<=-0.2%` 且 `6/6` 更快：记为有益的 Cube + 迁址并保留； +- 配对百分差中位数绝对值 `<0.2%` 且 `2～4/6` 更快：记为中性， + 可作为后续同址分片实验的对照； +- 其余组合追加第二轮六区组，不在看到结果后修改门槛。 + +合并十二个区组后，只在中位数 `<=-0.2%` 且至少 `10/12` 更快时记为 +提升；只在中位数绝对值 `<0.2%` 且 `5～7/12` 更快时记为中性； +其他结果全部撤销。只有迁址结果被判为有益或中性，S4.15b 才允许在 +相同地址、容量、state 大小和寻址骨架下只把 Cube active shards +从 4 改为 8。 + +以上内容在测量前完成登记；下面把正确性结果单列记录，但不把 S4.14a +的历史迁址收益外推为 Cube 迁址收益，也不提前填写性能结论。 + +#### 提交前正确性与构建结果 + +S4.15a 已按上述单一变量完成实现，提交前门禁结果如下： + +- CPU shared 构建中的 ring/ABI、PrepareMap、symbol、heap、 + Cube/Vector Claim cursor、materialize、loser-finish 定向测试全部 + 通过；shared b1/b256 `real-compute 6,28,4,1` 的完整 96-worker + 回放全部通过，private b1 回归通过； +- 合并后的 Claim cursor 定向测试逐调用核对 FetchMax 地址；b256 + 精确得到 Vector 32,768 次、Cube 16,384 次，两族 active line + 均为每线 4,096 次，Cube 后四条 inactive line 为 0 次且终值 -1； +- 用户 `$HOME/.venv` 下 PMU HTML、PMU sidecar、泳道转换和 + exclusive analyzer 共 100 项 Python 测试通过； +- CCEC private/shared 各自的 swimlane、perf-clock 和 + submit-PMU none/claim/efdrain/materialize/register 共 14 个构建 + 全部通过，14 份 manifest 均完成文件哈希校验；private perf-clock + `.text` 为 125,752B、SHA256 + `94017cdbeb758c0710aec30f238b396d217e648581cc5c67f2deaaa14bca79ef`， + `.rodata` 为 300B、SHA256 + `31d12b9797d051f1529d1792055ac9f46449022118990ca65f458e41f09bbfea`， + 与 `dc22d076` 冻结件逐字节一致； +- A5 shared b1 perf-clock 完整语义通过，state 大小为 + 1,011,858,816B，Submit 冒烟值为 64.594us；该单次 b1 只证明 + 可执行性，不进入保留判据； +- A5 shared b1 合并泳道完整语义通过，raw 4,154 条、dropped=0。 + ClaimMax 共 288 次：AIC lane 的 Alloc/QK/PV 各 32 次，两个 AIV + lane 的 Alloc/SF/UP 各 32 次；全部 flags 为 `0x53`，即 FetchMax、 + 返回值参与判断且结束边界为 return-ready。依赖签名保持 + `5cb454393ed48dcb`。 + +本次 b1 泳道位于： + +```text +outputs/pa_scheduler_shared_swimlane_20260725_170632_2702522/ccec/ +``` + +至此正确性门禁闭合，候选形成历史提交 `bab00e30` 并冻结 perf-clock +ELF。上述 b1 数值和泳道记录只证明迁址候选正确、可运行；是否保留仍 +只由预登记的 b256 六/十二区组配对决定。 + +#### 冻结配对结果与回退决定 + +`bab00e30` 相对当前基线 `ee42b8c1` 各预热两次，再执行六个交替 +ABBA/BAAB 区组、每版 12 个独立正式进程。差值统一为候选减基线： + +| 区组 | 差值 | 百分差 | +| ---: | ---: | ---: | +| 1 | +18.115us | +0.767% | +| 2 | +22.476us | +0.958% | +| 3 | +8.667us | +0.367% | +| 4 | +22.822us | +0.972% | +| 5 | +7.505us | +0.319% | +| 6 | -5.291us | -0.223% | +| 配对中位数 | **+13.391us** | **+0.567%** | + +候选只有 1/6 区组更快，且配对百分差中位数 `>=+0.2%`。这精确命中 +测量前登记的首轮撤销条件“`0～2/6` 更快且中位数 +`>=+0.2%`”，因此不追加第二轮六区组，不在看到结果后放宽门槛。 + +四个 warm-up 加 24 个正式样本共 28 个独立进程，每份日志均有 42 条 +PASS 断言，并统一满足： + +- ClaimMax=73,728、active workers=96、RingBp=0； +- 依赖边 1,280、签名 `b7d985d6edb07078`，QK/SF/PV/UP 各 256； +- `global_end_tick - global_start_tick == global_span_ticks`，SYS_CNT + 与输出微秒值精确闭合； +- execution、semantic、postprocess 和 real-compute 输出全部通过。 + +候选没有通过少做后续工作制造差值。基线到候选的正式样本中位数变化为： + +| 辅助指标 | `ee42b8c1` | `bab00e30` | 变化 | +| --- | ---: | ---: | ---: | +| fanin loads | 38,522 | 39,012 | +490 / +1.272% | +| fanin ready | 6,064 | 6,200 | +136 | +| fanin not-ready | 32,460 | 32,814.5 | +354.5 | +| CAS retries | 0 | 0 | 0 | +| EfDrain / RingBp / FinalDrain | 996 / 0 / 28 | 996 / 0 / 28 | 中位数不变 | + +这些计数只说明任务量和 placement 没有缩减，不能单独解释 +`+13.391us`。两版 shared perf-clock 的 `.text` 同为 129,080B， +但有 65,665 个字节位置不同；`.rodata` 同为 288B 且逐字节一致。 +尾部 state 增长、`scheduler_state_size` 常量和 CCEC 静态布局会共同 +改变执行节，因此本次只能评价“shared Cube 四分片迁址候选整体”， +不能把回退或某一项 fanin 变化直接等同为 atomic 硬件延迟。 + +完整证据位于： + +```text +outputs/perf_clock_freeze_bab00e30_20260725_170929/ +outputs/perf_clock_pair_bab00e30_vs_ee42b8c1_20260725_171005/ +``` + +`bab00e30` 作为正确但性能未过门槛的历史实现提交保留；随后由 +`319077a9` 撤销其 source、ABI、测试和当前行为描述。当前有效 baseline +提交为 `319077a9`，运行行为恢复到 S4.14b `ee42b8c1`： + +- sidecar 4,736,192B，CPU non-split `SchedulerState` + 1,011,852,160B，CCEC split `SchedulerState` 1,011,858,304B； +- Cube/Alloc 使用 production-prefix 四分片； +- shared Vector 使用 sidecar 八分片，private 继续使用 prefix + Vector 四分片。 + +由于迁址对照已经被首轮门槛否决，不再在其上叠加 Cube 同址 +`4→8`；后续若重启 Cube 方向，必须提出新的单变量方案并重新预登记。 + +### 2026-07-25：S4.16 shared Vector `8→16` 的布局控制与性能预登记 + +#### 旧实验只能作为弱先验 + +当时的 S4.16 候选回到 shared Vector，但不能用旧实验替代该次单变量验证。本仓 +`2e92da17` 记录的旧 A5 sweep 把 Cube、Vector、Alloc 三类 cursor +一起选择 `G=1/4/8/16`，且以物理容量 16、运行时 mask 和同一个 +AICore ELF 比较四档；它不是当前 shared-only sidecar 上的静态 +Vector 构建。旧 standalone 中 `G=16-G=8` 的配对中位数为 +`-37.935us/-1.043%`，只有 5/7 组更快，95% 置信区间 +`[-57.306,+21.706]us` 跨零。该结果只提供“边际收益可能已经接近 +饱和”的弱先验，不能预填本轮方向或替代当前冻结配对。 + +参考分支同样不是独立证据：`e49f73a3` 同时改变三类 cursor 和多项 +shared 提交流程，`0350b558` 将 Vector 从 8 改到 16 时又引入 +`alloc_cursor[3][8]` 并删除 shared completion frontier。参考实现与旧 +sweep 只证明十六分片形态具备可实现性，不证明该次 shared Vector +`8→16` 的净收益。 + +#### S4.16a 只建立十六条物理线的临时布局控制 + +当前冻结基线 `319077a9` 的运行布局与 `ee42b8c1` 一致： +`shared_vector_cursor` 物理容量和 active shards 均为 8，数组起点为 +sidecar offset 4,735,680B。S4.16a 只把物理容量从 8 扩成 16， +active shards 仍保持 8；设备 Claim 热路径继续执行 +`task_id % 8`，数组起点、前八条 cache line 的地址和访问顺序全部 +不变。新增加的八条物理线只追加在现有 state 尾部，不新增 trace、 +PMU、span、atomic 记录或业务状态。 + +预登记的精确 ABI 变化如下： + +| 项目 | `319077a9` 冻结基线 | S4.16a 临时控制 | +| --- | ---: | ---: | +| Vector 物理容量 | 8 | 16 | +| Vector active shards | 8 | 8 | +| `shared_vector_cursor` 起点 | 4,735,680B | 4,735,680B | +| shared sidecar 大小 | 4,736,192B | 4,736,704B | +| CPU non-split `SchedulerState` | 1,011,852,160B | 1,011,852,672B | +| CCEC split `SchedulerState` | 1,011,858,304B | 1,011,858,816B | + +尾增 512B 不移动 production prefix、已有 region/output/heap 字段、 +前八条 Vector cursor、standalone 控制区或 `WorkerResult`。但容量变化 +仍会改变完整 state 大小、GM 分配/H2D/D2H 长度、 +`scheduler_state_size` 握手常量，并可能连带改变页映射与 CCEC 静态 +代码布局。因此 S4.16a 的配对只能量化“容量 16、active 8 的静态布局 +整体成本”，不能解释成 atomic 等待变化。 + +S4.16a 是为 S4.16b 建立同址对照的临时控制，不作为一个可独立长期保留 +的优化候选。只要语义门禁全部通过，就固定相对 `319077a9` 运行六个 +ABBA/BAAB 区组，记录扩容布局成本；无论 S4.16a 单独表现为改善、中性 +还是回退，都不据此取消或放宽后续 S4.16b。若语义失败，则不进入性能 +比较并完整回退。 + +#### Atomic 数量与正确性门禁 + +S4.16a 不改变任何 Claim 的参与者、地址映射或次数。b256 必须精确闭合： + +```text +Vector ClaimMax = 64 AIV × 2 tasks/batch × 256 batches = 32,768 +全局 ClaimMax = 73,728 +前 8 条 Vector line = 每条 4,096 次 attempted +后 8 条 Vector line = 每条 0 次 attempted，终值保持 -1 +``` + +除逐调用地址与 attempted 计数外，Host 终态 oracle 还必须重新推导全部 +十六条 cursor：前八条达到各自精确 task 高水位，后八条始终为 -1； +旧 production-prefix Vector 在 shared 模式继续保持 -1，Cube/Alloc +继续使用 production-prefix 四分片。CPU 定向测试和 shared b1/b256 +96-worker 回放负责闭合地址、计数、唯一 winner、heap/TensorMap、依赖、 +completion 和真实计算输出；private 回归必须证明其状态和路由未受影响。 + +CCEC 需要重新完成 private/shared 的 swimlane、perf-clock 和五种 +submit-PMU 构建及 manifest 身份校验；private 执行节应与冻结件一致。 +A5 shared b1 perf-clock 与合并 atomic 泳道只用于证明设备可执行、业务 +oracle、ClaimMax 分类、`return_ready` 标志和 `dropped=0` 闭合，不进入 +b256 性能保留判定。 + +#### S4.16a 正确性实测 + +S4.16a 已按上述边界实现，热路径仍按 active 8 取模，新增容量只改变 +sidecar 尾部和完整 state 大小。提交前正确性结果如下： + +- CPU shared 定向测试、b1 和 b256 全部通过；b1/b256 分别闭合 + 288/73,728 次全局 Claim，b256 的前八条 Vector line 各 4,096 次 + attempted，后八条均为 0 且终值为 -1； +- CPU private b1 仍为 1,007,115,968B，production-prefix 路由和全部 + 业务断言通过，证明 shared 尾扩展没有进入 private 状态； +- CCEC shared/private 各自的 swimlane、perf-clock 和五种 submit-PMU + 各 7 个变体、共 14 个构建全部完成，artifact manifest 校验通过； + private 当前 `.text` 为 125,752B、SHA256 + `94017cdbeb758c0710aec30f238b396d217e648581cc5c67f2deaaa14bca79ef`， + `.rodata` 为 300B、SHA256 + `31d12b9797d051f1529d1792055ac9f46449022118990ca65f458e41f09bbfea`， + 与文档保存的冻结 private 基线执行节摘要一致； +- A5 shared b1 perf-clock 在 1,011,858,816B split state 上通过执行、 + 语义和后处理门禁，功能烟测 Submit 为 `65.105us`；随后合并 atomic + 泳道得到 4,148 条 raw 记录、 + 0 丢失和 288 条 `ClaimMax`。其中 Alloc 为 96 条，QK/PV 各 32 条， + SF/UP 各 64 条，全部 direct atomic flags 为 `0x53` + （FetchMax、消费返回值、`return_ready`）。全部十六条 cursor 的 + Host 终态 oracle 同时通过。 + +A5 泳道证据位于： + +```text +outputs/pa_scheduler_shared_swimlane_20260725_174924_2741480/ccec/ +``` + +b1 只覆盖 task 0～4，不能单独证明前八条 line 的均匀 attempted 分布； +该分布由 CPU b256 的逐调用地址 oracle 闭合。泳道中的 `81.671us` +只属于带观测的 b1 功能结果，不进入性能保留判定。 + +#### S4.16a 静态布局成本配对 + +S4.15a 配对完成后，曾使用 CANN `llvm-objcopy --dump-section` 直接 +处理历史 `ee42b8c1` 和 `bab00e30` 冻结 kernel；本机工具会原地 +重写输入，导致这两份历史 manifest 后验失配。原配对和 audit 均发生 +在重写之前，其历史性能结论不受影响，但被改写的 ELF 不再用于任何 +后续配对，也没有通过重算 manifest 掩盖失配。 + +本轮从精确提交 `319077a9` 的 detached worktree 重新构建 perf-clock。 +新冻结件的完整 ELF 哈希与旧文件不同，因此不宣称二者是同一个完整 +文件；但使用 `readelf` 给出的 offset/size 和 `dd` 从输入副本提取后， +执行段逐字节命中此前登记的 S4.14b 摘要： + +```text +.text 129,080B 90f58e5715112e30f2f57d768ed257580c96ccc187d0c9b7fa4d6a1e9b05fd26 +.rodata 288B 239e997707a3090248a65626afca3cfbec89793c703ea05461bfb02789722ded +``` + +提取前后原始冻结 ELF 哈希不变，两份新冻结件的原生 manifest 均多次 +校验通过。本轮冻结和配对证据为： + +```text +outputs/perf_clock_freeze_319077a9_rebuild_20260725_180058/ +outputs/perf_clock_freeze_e719cd17_20260725_175921/ +outputs/perf_clock_pair_e719cd17_vs_319077a9_20260725_180130/ +``` + +配对继续使用 device0、b256、`real-compute 6,28,4,1`、two-16、 +PMU off 和独立进程 ABBA/BAAB。差值为 S4.16a 减 `319077a9`： + +| 区组 | 差值（us） | 差值（%） | 方向 | +| ---: | ---: | ---: | --- | +| 1 | -19.6755 | -0.8317 | S4.16a 更快 | +| 2 | -25.4500 | -1.0715 | S4.16a 更快 | +| 3 | +4.3705 | +0.1851 | S4.16a 更慢 | +| 4 | -19.6510 | -0.8290 | S4.16a 更快 | +| 5 | +11.4415 | +0.4869 | S4.16a 更慢 | +| 6 | -4.6680 | -0.1982 | S4.16a 更快 | +| 中位数 | **-12.1595** | **-0.5136** | **4/6 更快** | + +独立 audit 对 28 个进程逐日志复核：每个进程 42 条断言全部通过， +SYS_CNT 起止与 TSV 精确一致，Claim 固定 73,728、active worker 固定 +96、RingBp 与 CAS retry 均为 0，依赖签名固定 +`b7d985d6edb07078`，QK/SF/PV/UP 各执行 256 次。辅助计数存在正常 +调度时序波动，未出现业务数量或协议语义变化。 + +该结果只说明“容量 16、active 8、尾增 512B”的静态构建整体在本轮 +为 4/6 更快、中位数 `-0.5136%`。按测量前规则，S4.16a 无论改善、 +中性或回退都只是 S4.16b 的同址布局控制，不能单独长期保留，也不能 +据此取消或放宽 S4.16b 的两级门槛。 + +#### S4.16b 同址十六分片与两级保留门槛 + +S4.16b 必须复用 S4.16a 的数组起点、物理容量 16、state 大小、初始化、 +Host 传输和寻址骨架，只把 active shards 与热路径取模从 8 改为 16。 +届时 Vector ClaimMax 和全局 ClaimMax 仍分别为 32,768 与 73,728； +每个 SF/UP task 仍由 64 个 AIV 竞争同一条 cursor，变化只在跨 task +流量从八条线每条 4,096 次摊到十六条线每条 2,048 次。它不是 atomic +次数消减，也不缩小单 task 的竞争面。 + +第一层先将冻结的 S4.16b perf-clock ELF 与 S4.16a 做独立进程、 +`real-compute 6,28,4,1`、b256、同一 device 的 ABBA/BAAB 配对。 +首轮六区组按以下互斥规则判定，差值统一为 S4.16b 减 S4.16a： + +- 任一语义失败、只有 `0～3/6` 区组更快，或配对百分差中位数 + `>=0`：判定失败； +- `6/6` 更快且配对百分差中位数 `<=-0.2%`：通过第一层； +- 其余仍为负向的组合，即 `4～5/6` 更快且中位数 `<0`，或 + `6/6` 更快但中位数位于 `(-0.2%,0)`：再追加六个区组。 + +合并十二个区组后，只有至少 `10/12` 更快且配对百分差中位数 +`<=-0.2%` 才通过第一层，其他组合均判定失败，不在看到数据后修改 +门槛。 + +只有第一层通过，才把同一个 S4.16b 冻结 ELF 再与 `319077a9` 做第二层 +净收益配对；第二层复用完全相同的首六区组和必要时十二条区组规则。 +只有“S4.16b 相对 S4.16a”和“S4.16b 相对 `319077a9`”两层都通过， +才保留最终 capacity 16、active 16 的实现。任一层失败都完整撤销 +S4.16a 与 S4.16b，恢复 `319077a9` 的 capacity 8、active 8 布局； +即使 S4.16a 单独更快，也不把临时布局控制留下。 + +S4.16b 的 active 常量变化会重新触发 CCEC 内联、常量传播和静态布局 +变化，因此其结果只能评价“静态 shared Vector16 构建整体”，不能仅凭 +墙钟进一步拆分为 atomic 竞争、I-cache、页映射或 winner 到达时序中的 +某一项。以上是测量前规则；S4.16a 正确性和静态布局成本配对已经 +完成，S4.16b 的当前实现结果如下。 + +#### S4.16b 正确性实测 + +S4.16b 已按预登记的同址单变量实现：`kSharedVectorCursorCapacity` +保持 16，只把 `kSharedVectorCursorShards` 从 8 改为 16。 +`shared_vector_cursor` 起点、sidecar/state 大小、初始化、传输长度、 +production prefix、Cube/Alloc 路由和所有观测字段均未改变。 + +提交前正确性结果如下： + +- 用户目录 GCC 15 会生成当前系统汇编器不识别的 `.base64` 伪指令， + 首次 CPU build 因此失败；其后误启动的旧 CPU 二进制结果全部作废。 + 有效 CPU 证据均由显式 `CXX=/usr/bin/g++`、`set -e` 的完整重建产生； +- CPU shared 定向测试逐次证明 task 2/14 命中 shard 2/14、task 4/12 + 命中 shard 4/12；shared b1/b256 完整回放通过，b256 的 32,768 次 + Vector Claim 精确摊到十六条线，每线 2,048 次； +- CPU private b1 保持 1,007,115,968B、production-prefix + `vector_cursor[4]` 路由和全部业务断言；Python 独立测试 100 项通过； +- CCEC private 的 7 个变体全部构建并通过 manifest，perf-clock + `.text` 仍为 125,752B、 + `94017cdbeb758c0710aec30f238b396d217e648581cc5c67f2deaaa14bca79ef`， + `.rodata` 仍为 300B、 + `31d12b9797d051f1529d1792055ac9f46449022118990ca65f458e41f09bbfea`， + 与冻结 private 基线逐字节一致； +- CCEC shared 的 7 个变体曾被误并发写入同一构建目录，因此那批产物 + 全部作废；随后按 swimlane、perf-clock、五种 submit-PMU 严格串行 + 重建，源码 diff 指纹前后均为 + `4c2e9b54c594579fee2a0cafb792a054117e35168e85525f012434c32d0e13ae`， + 7 份 manifest 均独立严格校验通过； +- S4.16b shared perf-clock `.text` 仍为 129,080B，但 SHA256 变为 + `331daafefa3dcb18c047a6333da797caafdfc48f7d1cc78ea3e33b0c39966a15`； + S4.16a 为 `53f53c4d...6888a64`。`.rodata` 仍为 288B、 + `239e9977...22ded`。这证明 active 常量触发了新的 CCEC 执行代码， + 但不能仅凭代码哈希推导性能方向； +- A5 shared b1 perf-clock 在 1,011,858,816B state 上通过全部门禁， + 功能烟测 Submit 为 `65.757us`；合并 atomic 泳道得到 4,117 条 + raw 记录、0 丢失、288 条 `ClaimMax`，全部 flags 为 `0x53`， + exclusive validation 通过。 + +A5 泳道证据位于： + +```text +outputs/pa_scheduler_shared_swimlane_20260725_182035_2773375/ccec/ +``` + +b1 的 Vector task 只有 2/4，它们在 `%8` 和 `%16` 下都命中 shard 2/4， +所以这份 b1 raw 和终态不能单独证明后八条线已启用。active16 的地址 +语义由 CPU 定向测试和 b256 逐调用 oracle 证明；随后的正式 b256 +配对也再次通过十六条线终态和全部业务断言。 + +#### S4.16b 第一层性能否决与整体回退 + +正确性提交 `2e7a0c73` 的 perf-clock 冻结件为： + +```text +outputs/perf_clock_freeze_2e7a0c73_20260725_182454/ +``` + +冻结件保持 S4.16a 的 1,011,858,816B split state；kernel `.text` +为 129,080B、 +`331daafefa3dcb18c047a6333da797caafdfc48f7d1cc78ea3e33b0c39966a15`， +`.rodata` 为 288B、 +`239e997707a3090248a65626afca3cfbec89793c703ea05461bfb02789722ded`。 +原生 manifest 与提取执行段前后的原始 ELF 哈希均通过校验。 + +第一层使用 S4.16a `e719cd17` 为 base、S4.16b `2e7a0c73` 为 +candidate，继续按预登记的 device0、b256、`real-compute 6,28,4,1`、 +two-16、PMU off 和六区组 ABBA/BAAB。差值为 S4.16b 减 S4.16a： + +| 区组 | 差值（us） | 差值（%） | 方向 | +| ---: | ---: | ---: | --- | +| 1 | +0.2985 | +0.0127 | S4.16b 更慢 | +| 2 | +12.2665 | +0.5206 | S4.16b 更慢 | +| 3 | -2.0840 | -0.0883 | S4.16b 更快 | +| 4 | +3.2035 | +0.1364 | S4.16b 更慢 | +| 5 | +1.7325 | +0.0734 | S4.16b 更慢 | +| 6 | +7.6325 | +0.3247 | S4.16b 更慢 | +| 中位数 | **+2.4680** | **+0.1049** | **仅 1/6 更快** | + +证据位于： + +```text +outputs/perf_clock_pair_2e7a0c73_vs_e719cd17_20260725_182506/ +``` + +独立 audit 对 28 个进程逐日志复核：每个进程 42 条断言全部通过， +SYS_CNT 与 TSV 精确一致，Claim 固定 73,728、active worker 固定 96、 +RingBp 和 CAS retry 均为 0，依赖签名固定 +`b7d985d6edb07078`，四类 kernel 各 256 次。候选的 fanin load 中位数 +比 base 多 548.5 次，但该计数受到达时序影响，只作为伴随诊断，不把 +`+0.1049%` 因果归给某一类 load。 + +该结果同时命中预登记的两条第一层失败条件：只有 `0～3/6` 区组更快， +且配对中位数 `>=0`。因此不追加第二轮六区组，也不执行相对 +`319077a9` 的第二层净收益配对；不能因为 S4.16a 单独为 +`-0.5136%` 就留下临时容量控制。 + +S4.16a/S4.16b 的源码、ABI 和测试已整体撤销。回退后对 +`common/pa_model.h`、`common/host_support.h`、 +`test/test_shared_tensor_map_ring.cpp`、 +`test/test_shared_vector_claim_cursor.cpp` 和 `cpu/build.sh` 执行 +`git diff --exit-code 319077a9 -- ...` 无差异，当前恢复： + +```text +shared_vector_cursor capacity = 8 +shared_vector_cursor active = 8 +shared sidecar = 4,736,192B +CPU non-split state = 1,011,852,160B +CCEC split state = 1,011,858,304B +``` + +回退后再次用 `/usr/bin/g++` 完整重建并执行 CPU shared perf-clock： +全部独立自测和 b1 的 42 条调度/业务断言通过。随后用本机 CANN 9.1 +串行重建 CCEC shared perf-clock，manifest 严格校验通过；只读复制 +kernel 后按 `readelf` 偏移用 `dd` 提取执行节，原始 ELF 提取前后 +SHA256 均为 +`77dd7f2eef6ac4daf220f166b0d0aef3181478eb2d2b51255ebf1f2d05fb4df6`。 +回退件与重建 `319077a9` 冻结基线的执行节逐字节一致： + +```text +.text 129,080B 90f58e5715112e30f2f57d768ed257580c96ccc187d0c9b7fa4d6a1e9b05fd26 +.rodata 288B 239e997707a3090248a65626afca3cfbec89793c703ea05461bfb02789722ded +``` + +S4.16 的正确性提交、冻结件和负结果保留用于后续决策，但不属于当前 +运行布局或性能收益。 + +### 2026-07-25：S4.17 shared `WorkerState` 热控制字段前置预登记 + +S4.16 回退后，当前有效运行身份是 `bf7a7076`，其源码行为与 +`319077a9` 的 S4.14b 一致。下一步不继续扩大 cursor 分片，也不叠加 +Alloc cursor 迁址、output writer 去 RMW 或 descriptor 提前发布。 +S4.17 只验证参考实现 `deb2dfc3` 中一个可以独立抽出的布局假设： +shared 模式把高频访问的 worker 热控制字段移到固定头部，private 模式 +保持现有 ABI 逐字节不变。 + +当前 `WorkerState` 的关键布局为： + +| 字段 | 当前 offset | +| --- | ---: | +| `local_index` | 20B | +| `heap_next` | 24B | +| `map` | 32B | +| `slots` | 823,360B | +| `occupied_count` | 842,656B | +| `owned_total` | 842,660B | +| `swimlane_last_cycle` | 842,664B | +| `payloads` | 842,688B | + +`DrainReady()` 在每次逻辑 Submit 开头都会先读 `occupied_count`；b256 +固定执行 `96 × 1280 = 122,880` 次。当前该字段与 `local_index` 相距 +约 823KiB，而 `occupied_count == 0` 又是最常见的快速返回条件之一。 +参考实现把这些控制字段放在 map 前方，说明这不是凭空创造的新接口； +但参考实现还同时删除 shared worker 中的 private map、扩大 ring 并 +改变发布协议，这些变化不属于本轮。 + +S4.17 的 shared 候选布局预先固定如下： + +| 字段 | 候选 offset | 相对当前 | +| --- | ---: | ---: | +| `local_index` | 20B | 0 | +| `heap_next` | 24B | 0 | +| `occupied_count` | 32B | -842,624B | +| `owned_total` | 36B | -842,624B | +| `swimlane_last_cycle` | 40B | -842,624B | +| `map` | 48B | +16B | +| `slots` | 823,360B | 0 | +| `payloads` | 842,688B | 0 | + +计算依据是 `sizeof(TensorMap) == 823,312B`：`48 + 823,312 = +823,360`，因此 shared 模式的 `map` 后不再需要原来的 16B +`slot_padding`；四个 `LocalSlot` 结束于 842,656B，原来位于此处的 +16B 控制字段迁走后，以 32B `payload_padding` 保持 payload 起点不变。 +候选必须继续满足： + +- `sizeof(WorkerState) == 9,231,296B`； +- shared sidecar、CPU/CCEC `SchedulerState`、host-device 传输长度均不变； +- `slots`、`payloads`、task/cursor/frontier/sidecar 的 offset 均不变； +- shared worker 中暂时无用的 private `TensorMap` 仍然保留，本轮不借机 + 缩减 state； +- shared 构建身份版本从 v4 推进到 v5，拒绝总大小相同但字段解释不同的 + 新旧 host/kernel 混用；该握手发生在首个 Submit 之前； +- private 模式字段顺序、padding、所有 offset、ABI v4 和运行行为不变； +- Claim 地址、分片数、atomic 次数、winner、fanin、heap、依赖和 + kernel 业务逻辑均不变。 + +这是一项 shared-only 静态布局实验。即使获得墙钟收益，也只能命名为 +“`WorkerState` 热控制字段前置整体收益”，不能仅凭结果断言收益一定 +来自某一条 cache line、I-cache、页映射或单次 load 延迟。CCEC 可能因 +结构体常量 offset 改变而生成不同指令编码，这同样属于候选整体。 + +#### S4.17 正确性门槛 + +实现后先以模式化 `static_assert` 锁死上述 shared offset，并继续锁死 +private 的现有 offset。随后依次验证： + +1. 用 `/usr/bin/g++` 重建 CPU shared，运行独立布局/协议测试和 b1、 + b256 完整回放；42 条业务断言、96 个 active worker、依赖签名 + `b7d985d6edb07078`、QK/SF/PV/UP 各 256 次、三类 placement 总数及 + 终态 `occupied_count=0` 必须闭合。CPU 线程调度可能产生 RingBp， + 不能把设备历史值 0 错当成 CPU 正确性门槛； +2. 重建 CPU private 并运行 b1，确认 private 的 production-prefix + 路由、状态大小和全部业务断言未变； +3. 串行重建 CCEC private/shared 构建身份并严格校验 manifest，禁止 + 多个变体并发覆盖同一输出目录； +4. A5 shared b1 perf-clock 与合并 atomic 泳道只作为正确性证据， + atomic raw 必须 0 丢失，所有协议和业务 oracle 必须通过； +5. 任一语义、ABI、manifest 或设备断言失败，直接撤销候选，不进入 + b256 正式性能取数。 + +#### S4.17 性能口径与保留门槛 + +正式性能只比较 clean shared perf-clock ELF。基线固定使用从 +`319077a9` 重建且未被 `objcopy` 改写的冻结件： + +```text +outputs/perf_clock_freeze_319077a9_rebuild_20260725_180058/ +``` + +候选冻结时先只读复制 kernel，再按 `readelf` 给出的 section offset +用 `dd` 提取 `.text/.rodata`；提取前后原始 ELF SHA256 必须一致。 +测试固定为 device0、b256、`real-compute 6,28,4,1`、two-16、PMU off， +每版先运行 2 个不计入统计的独立进程预热，再执行 6 个交替 +ABBA/BAAB 区组，每版共 12 个正式独立进程。 + +首轮六区组按以下互斥规则判定，差值统一为候选减基线： + +- 任一语义失败、只有 0～3/6 区组更快，或配对中位数百分比 `>= 0`： + 立即撤销； +- 6/6 区组更快且配对中位数百分比 `<= -0.2%`：直接保留； +- 4～5/6 区组更快且中位数为负，或 6/6 更快但中位数落在 + `(-0.2%, 0)`：再追加 6 个交替区组； +- 十二区组累计至少 10/12 更快且中位数百分比 `<= -0.2%` 才保留， + 否则撤销。 + +当前 S4.14b 的历史正式中位数约 2,357.489us；参考分支 +`2866ad73` 的历史合并泳道全局 X-event span 约 2,540.538us。两者 +来自不同 ELF、不同观测能力和不同时间边界，不能直接相减或宣称 +standalone 已经超过参考实现，只能说明继续做边际实验的价值已经降低。 +如果 S4.17 未达到预登记门槛，standalone 的低风险布局探索到此停止， +下一阶段转入真实 simpler shared TensorMap 路径，不再为了追求单个 +standalone 数字而重复已否决方向。 + +#### S4.17 实现与正确性实测 + +S4.17 已按预登记的 shared-only 布局实现，源码只改动 +`common/pa_model.h`： + +- shared 的 `occupied_count/owned_total/swimlane_last_cycle` 分别移动到 + 32/36/40B，`map` 移到 48B； +- shared 删除 map 后的 16B padding，并以 slots 后的 32B padding 保持 + `slots=823,360B`、`payloads=842,688B`； +- `WorkerState=9,231,296B`、shared sidecar、CPU/CCEC state 和传输长度 + 均未改变； +- private 仍是 `map=32B`、`slot_padding=823,344B`、 + `occupied_count=842,656B`、`payloads=842,688B`； +- shared 构建身份为 ABI v5，private 继续为 ABI v4。头部字段、两种 + padding、热控制、map、slots、payloads 和总大小均有模式化 + `static_assert`，没有使用 union、零长数组或 packing。 + +CPU 证据均使用显式 `CXX=/usr/bin/g++` 完整重建，避免用户 GCC 15 +生成本机汇编器不识别的伪指令： + +- shared perf-clock 构建中的 PollBatch、shared ordinary-region ring、 + PrepareMap marker、shared output symbol、no-wrap heap、Vector cursor、 + Materialize 和 split-finish loser 定向测试全部通过； +- shared b1/b256 完整回放均通过全部业务断言。b256 为 96 个 active + worker、73,728 次 Claim、依赖签名 `b7d985d6edb07078`、四类 kernel + 各 256 次；CPU 宿主调度产生 + `EfDrain=572/RingBp=354/FinalDrain=98`，三者精确覆盖 1,024 个 + kernel，不能把该动态 placement 数当作 A5 性能结论； +- private perf-clock 重新构建并通过 PollBatch、private TensorMap ring + 和 b1 全部业务断言，状态仍为 1,007,115,968B； +- 本用户 `$HOME/.venv` 在按仓库文档设置 + `PYTHONPATH=$PWD:$PWD/python` 后，四个泳道/PMU 加工测试共 100 项全部 + 通过。第一次未设置项目 `PYTHONPATH` 的调用在 conftest 导入阶段即 + 失败，未进入收集，也未被记作测试证据。 + +本机 CANN 9.1 下，private/shared 的 swimlane、perf-clock 及 +none/claim/efdrain/materialize/register 五种 submit-PMU 共 14 个 CCEC +身份严格串行重建；每个目录的 mode、variant、phase 和全部 artifact +SHA 均通过 manifest `--check --strict`。private perf-clock 的执行节 +与冻结基线逐字节一致： + +```text +.text 125,752B 94017cdbeb758c0710aec30f238b396d217e648581cc5c67f2deaaa14bca79ef +.rodata 300B 31d12b9797d051f1529d1792055ac9f46449022118990ca65f458e41f09bbfea +``` + +shared 候选 perf-clock 的原始 artifact 和执行节为： + +```text +host SHA256 c1d0bb76f13ee6d8dd7d39b2f2d87ba079f8ea8d37402667f373a3bdff4ac912 +kernel SHA256 17c2ca88d5e5240d8ca917e33eca4480427e23b6f833f0f57f0af26b22b6d8a4 +.text 128,824B 598738d3540d162b0e01bed29651458a4dec879fb56adb7dc5be5ea90d913bbb +.rodata 288B 239e997707a3090248a65626afca3cfbec89793c703ea05461bfb02789722ded +``` + +相对 `319077a9`，`.text` 减少 256B 且内容变化，`.rodata` 保持逐字节 +一致。这只能证明 CCEC 为新 offset/ABI 身份生成了不同代码，不能在 +正式配对前据此推导性能方向。 + +A5 device0 shared b1 的两条正确性证据均通过： + +- perf-clock：42 条业务/观测隔离断言闭合，Submit 为 65.249us， + `RingBp=0`； +- 合并 atomic 泳道：Submit 为 81.664us，raw 4,145 条、0 丢失， + 4,453 次逻辑 atomic 调用闭合为 864 条直接物理记录和 242 条 + PollBatch 记录，严格排他分析通过。 + +泳道证据位于： + +```text +outputs/pa_scheduler_shared_swimlane_20260725_185913_2807044/ccec/ +``` + +构建身份还做了双向真实设备负测： + +1. S4.17 v5 host 配 `319077a9` v4 kernel； +2. `319077a9` v4 host 配 S4.17 v5 kernel。 + +两次都在 0 次 Submit、96 个 worker 均未进入调度时置 `fatal=1`， +进程退出码为 1；没有先按错误 offset 运行后再依赖崩溃兜底。至此 +S4.17 的源码、ABI、CPU、Python、14 种 CCEC 身份和 A5 b1 正确性门槛 +全部通过，可以形成独立正确性提交；该提交本身不代表性能候选已获保留。 + +#### S4.17 十二区组性能否决与整体回退 + +S4.17 正确性提交为： + +```text +82c0828d 优化(a5): 前置shared worker热控制字段 +``` + +该 clean commit 的 shared perf-clock 冻结件为： + +```text +outputs/perf_clock_freeze_82c0828d_20260725_191346/ +``` + +冻结时先校验原生 manifest，再只读复制 kernel，并按 `readelf` 给出的 +offset 用 `dd` 提取执行节；提取前后原始 ELF SHA256 均为 +`17c2ca88d5e5240d8ca917e33eca4480427e23b6f833f0f57f0af26b22b6d8a4`， +没有使用 `objcopy` 改写被测件。 + +第一轮严格使用预登记的 device0、b256、`real-compute 6,28,4,1`、 +two-16、PMU off、每版两次预热和六个 ABBA/BAAB 区组。结果为 4/6 +区组更快，配对中位数： + +```text +-5.117250us / -0.217197% +``` + +它既不满足 6/6 直接保留，又命中“4～5/6 且中位数为负”的追加条件， +因此不在首轮下结论，原样追加区组 7～12。首轮 28 个独立进程均经过 +深审计：每个进程恰有 42 条断言、SYS_CNT 与 TSV 精确一致、Claim +73,728、active worker 96、RingBp=0、依赖签名 +`b7d985d6edb07078`，四类 kernel 各 256 次。首轮 samples、summary、 +audit 和 diagnostics 已另存为同目录下的 `*_6blocks` 文件。 + +十二个区组的差值如下，正值表示 S4.17 更慢： + +| 区组 | 差值（us） | 差值（%） | 方向 | +| ---: | ---: | ---: | --- | +| 1 | -24.1950 | -1.0207 | S4.17 更快 | +| 2 | +1.6500 | +0.0699 | S4.17 更慢 | +| 3 | -10.2220 | -0.4344 | S4.17 更快 | +| 4 | -1.6900 | -0.0715 | S4.17 更快 | +| 5 | +6.0585 | +0.2577 | S4.17 更慢 | +| 6 | -8.5445 | -0.3629 | S4.17 更快 | +| 7 | +13.3505 | +0.5692 | S4.17 更慢 | +| 8 | +5.6315 | +0.2391 | S4.17 更慢 | +| 9 | -9.0895 | -0.3840 | S4.17 更快 | +| 10 | +1.3045 | +0.0554 | S4.17 更慢 | +| 11 | +0.1300 | +0.0055 | S4.17 更慢 | +| 12 | -17.0740 | -0.7210 | S4.17 更快 | +| 中位数 | **-0.7800** | **-0.0330** | **6 快 / 6 慢** | + +十二组的未配对分布也近乎重合：base 24 个正式样本中位数 +2,358.174us，candidate 为 2,358.205us。主判据仍是上表的成组配对， +不使用未配对中位数替代。 + +扩展取数后使用独立加固的 audit 从 `samples.tsv` 重新验证 52 个进程 +身份、warm-up/formal 顺序、每个 block 的 ABBA/BAAB position、全部 +日志和 SYS_CNT，并重新计算十二个 block 及最终中位数，再逐字段与 +`summary.json` 对照。结果仍为 6/12 更快、`-0.0330%`。因此同时未达到 +“至少 10/12 更快”和“配对中位数不高于 -0.2%”两条门槛，S4.17 +判定为中性噪声内波动，不能保留。 + +完整证据位于： + +```text +outputs/perf_clock_pair_82c0828d_vs_319077a9_20260725_191435/ +``` + +动态诊断中，candidate 的 fanin load 和 `submit_completion_ops` 中位数 +均比 base 少 476 次；但它们由 winner 到达和轮询时序决定，固定 Claim、 +业务、依赖和 kernel 数并未改变。既然成组墙钟没有稳定收益，不能把这 +一伴随变化解释成热字段前置的确定因果。 + +S4.17 的 shared 字段移动、条件 ABI v5 和新增 offset 断言已全部撤销。 +回退后的 `common/pa_model.h` 与 `bf7a7076` 逐字节无差异，当前重新恢复： + +```text +shared/private build ABI = v4 +map offset = 32B +slots offset = 823,360B +occupied_count offset = 842,656B +payloads offset = 842,688B +WorkerState = 9,231,296B +``` + +回退后再次使用 `/usr/bin/g++` 完整重建 CPU shared，全部独立自测与 +b1 业务断言通过；用本机 CANN 9.1 重建 CCEC shared perf-clock， +manifest 通过，执行节与 `319077a9` 重建冻结基线逐字节一致： + +```text +.text 129,080B 90f58e5715112e30f2f57d768ed257580c96ccc187d0c9b7fa4d6a1e9b05fd26 +.rodata 288B 239e997707a3090248a65626afca3cfbec89793c703ea05461bfb02789722ded +``` + +最后一次 A5 shared b1 perf-clock 也通过全部门禁，Submit 为 67.668us。 +S4.17 的正确性提交、冻结件和负结果继续保留用于后续决策，但其实现 +不属于当前运行布局。按照预登记规则，standalone 的低风险热布局边际 +探索到此停止，下一步转向真实 simpler shared TensorMap 路径。 + +### 2026-07-26：R0 真实路径构建身份、缓存隔离与三镜像防混用 + +本小步严格停在 R0 的第一项：只建立 private/shared 第一等构建身份、 +产物隔离和三镜像 ABI 拒绝机制，不迁移 TensorMap 数据结构或 Submit +算法。这样可以先证明后续 shared 代码不会误用 private 缓存、宏或 GM +布局；shared backend 尚未接入时，构建可以完成，但运行必须在 0 次 +Submit 前明确失败。 + +#### 显式构建身份 + +入口不再依赖参考分支使用的 ambient `CXXFLAGS`。pytest 和 standalone +入口新增： + +```text +--fdwic-tensormap private|shared +``` + +默认值为 `private`。选择会进入以下全部构建链： + +1. baseline Host、AICPU inner runtime、AICore； +2. 每 callable 的 AICore extra image； +3. orchestration TU； +4. Worker 选择和 L2 worker pool 身份； +5. submit-PMU 构建 provenance。 + +安装期 `build_runtimes.py` 也提供可重复的显式入口： + +```text +--fdwic-tensormap private +--fdwic-tensormap shared +``` + +不传时固定只构建 private，且不读取 shell 中可能残留的 +`PTO_FDWIC_TENSORMAP_MODE`；重复参数可同时预构建两族。非 A5 或非 +FDWIC runtime 永远用 private builder，不会被 shared 选择污染。 + +三类 runtime image 均收到同一个编译定义： + +```text +private -> PTO_FDWIC_SHARED_MAP=0 +shared -> PTO_FDWIC_SHARED_MAP=1 +``` + +onboard AICPU dispatcher 是进程级、模式无关的公共加载器，因此没有 +携带该宏；只给真正解释 Runtime ABI 的 inner AICPU image 传入。这一点 +与“给所有 AICPU 目标机械加宏”有意不同。 + +模式同时进入 baseline 和 `aicore-extra` 的目录： + +```text +build/cache/a5/{sim,onboard}/fully_distributed_within_core/{private,shared}/ +build/lib/a5/{sim,onboard}/fully_distributed_within_core/{private,shared}/ +``` + +baseline FDWIC cache 身份不再只有 Git HEAD，还包含实际 include/source +内容指纹和有效 compile definitions。这样当前 worktree 未提交修改、 +profile 宏变化或 private/shared 切换都不会复用旧 image。 +`aicore-extra` 会把模式宏和诊断宏合并；同名宏值冲突直接报错，禁止 +调用方用后传定义覆盖构建身份。 + +#### Runtime 稳定控制前缀与失败协议 + +`Runtime` 首字段新增一条 64B、64B 对齐的 `FdwicBuildIdentity`： + +| 字段 | 当前值或语义 | +| --- | --- | +| magic | `FDWICMAP` | +| build ABI | v1 | +| TensorMap mode | private=0，shared=1 | +| runtime bytes | 当前 Host/CPU 两种模式均为 70,080B；真实 A5 private 握手已闭合 | +| DistGlobal layout version | v1 | +| error bits | AICPU mismatch、AICore mismatch、backend unavailable | + +Host 构造时生成该身份，绑定 callable 前重新校验并清空本轮错误位。 +AICPU 在解释 `DistHandoff/PTO2Runtime` 之前校验；AICore 在进入 +`dist_core_main()` 前校验。两种负向路径分别是： + +```text +Host 与 AICPU 不一致 + -> AICPU 置 mismatch + -> 给所有 worker 发布 DIST_ABORT + -> worker 进入公共 EXIT 握手 + -> 所有 AICPU 线程返回 -1 + +Host/AICPU 一致、AICore 不一致 + -> AICore 0 发布全局 mismatch，flush + 完成屏障 + -> 全部 AICore 各自发布 FIN + -> AICPU 等齐后读取错误位 + -> 公共 EXIT 握手 + -> 所有 AICPU 线程返回 -1 +``` + +AICore 错误位只由 `s_block_idx == 0` 发布，避免 96 核争写同一 +cacheline。CCEC 使用单 owner 普通写并在错误冷路径执行 cache flush/DSB； +A5sim 使用原子 OR，避免宿主线程数据竞争。该屏障只存在于 ABI 错误路径， +不会进入正常 Submit 热路径。AICPU 各线程在 `runtime_done` 后统一重读 +错误位，不依赖未查档证明的 CANN 多 block 返回码聚合行为。 + +当前机制明确保证 private/shared **控制布局相同**时的防混用，不声称能让 +任意旧新 `Runtime` 布局互操作。身份字段以及其后的 `workers`、 +`worker_count`、AICPU launch/affinity 字段构成永久公共控制前缀： +AICore 即使发现模式不一致，也必须按该前缀完成 handshake，AICPU 才能 +发送 `DIST_ABORT/EXIT`。后续模式化 TensorMap 状态必须放在此前缀之后， +实际 shared backend 继续通过 `dist.shared_addr` 指向。 + +#### shared fail-closed + +本阶段 `kFdwicCompiledBackendReady` 仅对 private 为 true。shared 三镜像 +可以独立构建、参与 cache/ABI 负测，但同模式运行会报告： + +```text +FDWIC shared TensorMap artifact is ABI-valid but its runtime backend +is not connected yet; aborting before Submit +``` + +随后给所有 AICore 发布 `DIST_ABORT` 并正常 teardown。没有用 private +TensorMap 冒充 shared 成功，也没有为了让测试变绿提前抄入参考分支的 +append-only map、无 generation heap 或可选 writer intent。 + +#### submit-PMU 联动 + +submit-PMU provenance 从 v1 推进到 v2，新增 `tensormap_mode`，并要求： + +```text +profiled cache key 中的 mode + == provenance mode + == PTO_FDWIC_SHARED_MAP 有效定义 + == 实际 Host/inner AICPU/AICore artifact family +``` + +sidecar 同时冻结最终 AICore、AIC/AIV combined object、Host runtime 和 +实际负责 PMU owner/config/restore 的 inner AICPU runtime 的文件及 +`.text` 哈希，发布前再检查实物没有变化。HTML 也显示模式和五类实物。 +旧 raw 不做兼容分支；当前采集和加工是一体化流程，新 schema 的目标是 +拒绝把 private 诊断 image 标成 shared，或反过来。 + +#### 已完成验证 + +本用户 Python 环境统一使用 `$HOME/.venv`： + +- RuntimeBuilder、安装期预构建入口、KernelCompiler、SceneTest + cache/CLI 和 submit-PMU provenance 共 387 项非上板测试通过， + 12 项 requires-hardware 项未执行； +- Ruff lint 和 format check 全部通过； +- C++ `test_fdwic_build_identity_private/shared` 两个独立目标均构建通过， + 每个目标 4 项测试通过；测试直接包含生产 `Runtime`，约束 identity 为 + 首字段、workers/worker_count/AICPU launch 公共前缀偏移和 70,080B + 当前布局，不再只用假的 runtime size； +- A5sim private/shared baseline 三镜像均构建通过； +- A5 CANN 9.1 CCEC private/shared 的 Host、AICPU、AICore 三镜像均构建 + 通过。CCEC 实测确认普通 host inline helper 不能直接从 + `__aicore__` 调用，且该后端不能选择 32 位 GM `__atomic_fetch_or`； + 最终代码按真实 CCEC 接口分别处理，没有用 host 编译成功替代 CCEC 验证； +- A5sim PA Case1 private 在 `--use-example-exec-time` 下通过； +- 真实 A5 private CaseB1 正常 Submit 与 golden 通过；因此当前 + aarch64 Host/inner AICPU 与 CCEC AICore 对 build identity、 + Runtime 大小和公共握手前缀达成一致； +- A5sim shared 同模式在 0 Submit 前按预期返回 -1，四个 AICPU 线程均 + 返回失败，且不再输出 `orch_start=0` 的伪超长耗时； +- private Host/AICPU + shared AICore 的真实三镜像混装在所有 AICore + FIN 后明确返回 -1，没有挂死； +- private Host/AICore + shared AICPU 的真实三镜像混装发布 + `DIST_ABORT` 后明确返回 -1，没有挂死。 + +本阶段没有运行 shared A5 业务用例，因为 backend 被有意保持为 +fail-closed；在接入真实 shared facade/backend 前跑板只能重复证明 +“未实现会失败”，不能提供 TensorMap 正确性或性能证据。下一小步应先 +抽取 private/shared 共用 facade，并把 standalone 已证明的 private +ring 以不改变默认 private 行为的方式接入；shared 仍保持 fail-closed， +直到它自己的 CPU/CCEC 协议测试闭合。 + +### 2026-07-26：迁移 review 的适用边界与 standalone 后续顺序 + +新增审查材料： + +```text +shared_tensormap_swimlane_deps_migration_review.md +``` + +该文档固定审查的是 +`fdwic-swimlane-deps@1726a774 -> fdwic-shared-tensormap@351ef62e` +两条 production 分支。它提供了有价值的接口、缓存顺序、DFX 和验证 +检查表，但不能直接当成当前 standalone 的缺陷清单：本目录在该快照之后 +已经独立完成多轮实现、撤销和 A5 配对，且目标只是对等模拟 PA Case1， +不是宣称覆盖 production 的全部 joint、BlockWon 和动态拓扑。 + +#### 当前仍成立的问题 + +第一，shared atomic 观察尚未全量闭合。当前 `AtomicSite` 只有 0～14 +共 15 个 common/private 站点；以下 shared Case1 活跃操作仍直接调用 +`Ops::*`： + +- shared heap cursor/vend 的 load、reserve fetch-add 和 vend fetch-add； +- output `published` 的 probe/wait load 与 publication exchange； +- output `last_writer` 的 load、INPUT writer 更新和 INOUT writer commit。 + +因此既有 `logical == direct + PollBatch polls` 只证明“已经进入 wrapper 的 +调用自洽”，即使 `dropped=0` 也不能解释成真实 shared atomic 全覆盖。 +S3.2a 已记录这一限制，本次 review 进一步确认它应成为下一阶段的首要 +观察闭环，不再把 shared b1 的既有 atomic 数量写成协议总量。 + +第二，追加 shared site 前必须先修 PollBatch mask。当前 burst state 已经 +按 `AtomicPollBatchIndex()` 使用紧凑的 0～5 下标，但 region enable mask +仍由 `1U << raw_site_id` 构造。只要新增 site 编号达到 32，就会发生错误 +移位或启用错误站点。正确顺序是： + +```text +S5.1 enable mask 完全改用紧凑 poll index，并补 >31 site 单测 +S5.2 只追加 PA Case1 实际活跃的 shared atomic site/wrapper +S5.3 CPU closure + CCEC manifest + shared b1 atomic 泳道 +``` + +wrapper 在 DFX 关闭时必须内联回原始 `Ops`，不得新增 load、分支、屏障， +也不得把 source-issue 操作强行改成 return-ready。 + +第三，standalone submit-PMU 不能称为“纯 Submit scalar”。硬件 gate +实际从 `InitPaOrchestration()` 前开始，到本 worker 最后一次 Submit +返回后停止；窗口包含 orchestration 初始化，也包含 winner 执行的真实 +Cube/Vector workload。目前没有 linked-kernel pause/resume，也没有一份 +同 ELF 的 return-ready atomic 排除 sidecar。因此当前 total/scalar/ +I-cache 只能解释为该 **PMU gate 全窗** 的原始计数，不能冒充扣除了 +kernel、初始化和总线等待后的纯 scalar 代码成本。后续单独处理： + +1. 先把 PMU gate 与首/末 Submit 边界对齐； +2. 再在同一 ELF 内建立 linked-kernel pause/resume 计数闭环； +3. return-ready atomic 只有在同构建、同窗口、可闭合时才允许排除， + 不能拿互斥的 atomic-swimlane ELF 跨运行相减； +4. 最后才按当前 span 选择少量局部 phase，不机械复制 production 的 + Resolve/joint 阶段清单。 + +#### 已完成、不要重做 + +- private heap H1 首圈快路已经位于 fatal 检查之后并保留； +- TwoLevel16 分层 final barrier 已是 standalone 默认； +- private/shared 已由 `PTO_FDWIC_SHARED_MAP` 生成互斥 CPU/CCEC 变体， + 目录、manifest、ELF hash 和 host/kernel mode 都会闭合； +- S4.9 shared no-wrap、S4.8 descriptor 直写和 S4.14b Vector8 已通过 + A5 配对并保留； +- Alloc 固定候选/早退、pure INPUT deferred、loser shortcut、 + 24-owner Alloc、Cube sidecar、Vector16 和 WorkerState 前置均已按 + 预登记门槛否决并撤销，不能因 review 读到过程提交而重新实现。 + +review 建议“所有非 DFX 收敛后才补观察工具”不适用于本次既定开发过程。 +用户已明确要求先夯实 scalar/atomic/PMU 观察再优化，且 perf-clock、PMU +构建已经在编译期移除泳道记录路径；S4.9/S4.14 的保留或撤销也依赖这些 +证据。正确做法是继续保持三类 ELF 互不混算，而不是删除已经证明有效的 +观察基础。 + +#### 属于能力边界，不是当前 Case1 缺陷 + +- standalone 明确拒绝 `active_count >= 2`，所以没有 production + BlockWon/joint/mix_coown；若以后宣称模拟这些业务，必须另立阶段补齐； +- standalone ABI 固定 32 AIC + 64 AIV。production 工具必须支持最多 + 108 worker，不代表 PA 专用模型要为了泛化而改变当前可比拓扑； +- task id 不复用、每 task output 上限为 8、shared heap 有界 no-wrap + 都由 API 与 host oracle 显式校验，是模型约束而非静默缺陷； +- ordinary-region ring 是隔离协议测试，PA Case1 不走该热路。其 atomic + 不应伪装成当前漏采的动态调用，也不应先于活跃 output/heap 站点扩展； +- production 原有 phase 14～17 与来源 schema 冲突是未来真实路径迁移 + 风险；standalone v4 内部对这些编号已有唯一解释，不存在本目录自冲突。 + +后续因此先完成 S5.1/S5.2 的观察正确性，再处理 PMU 纯 scalar 口径； +不在同一提交里叠加性能候选，也不把 production 的八组 smoke 门禁、 +ambient `CXXFLAGS` 脚本或旧 shared ring oracle 搬进 standalone。 + +### 2026-07-26：S5.1 PollBatch 启用位改用紧凑索引 + +本阶段只修正 atomic 观察层的位图语义，不新增 shared 站点，也不修改 +调度协议。此前 `AtomicSite` 的 raw 编号同时承担稳定记录身份与 +PollBatch enable bit 两种职责；而 burst 计数数组早已按 +`AtomicPollBatchIndex()` 的 0～5 紧凑索引存放。两套索引混用会产生 +两个问题： + +1. raw site 稀疏增长后，32 位 mask 无法表达编号大于等于 32 的站点； +2. enabled mask 的 bit 与 burst 数组槽位不一致，后续扩展 shared site + 时容易启用错误的计数槽。 + +现将职责明确拆开： + +```text +AtomicSite raw id 稳定 raw ABI、站点名称和记录身份 +AtomicPollBatchIndex(site) PollBatch allowlist 内的紧凑 0～N-1 下标 +AtomicPollBatchMask(site) 仅由紧凑下标产生 enable bit +``` + +未登记或超出范围的 site 返回空 mask，不再对 raw id 执行移位； +`kAtomicPollBatchSiteCount` 同时约束不超过 32。Host constexpr 路径和 +CCEC device 路径使用相同规则，WaitForSlot、HeapGuard、startup 和 +final barrier 的 region mask 均已切换到紧凑索引。 + +本阶段补充了以下定向断言： + +- 非 PollBatch allowlist 的 `FrontierFlagLoad` mask 必须为 0，且仍走 + direct atomic 记录； +- 人工构造 raw site 40 时，compact index 为 -1、mask 为 0，不发生 + 32 位错误移位； +- raw id 为 5 的 `FaninFlagLoad` 使用 compact bit 2，证明不再把 raw + 编号直接当 bit 位。 + +验证结果： + +| 验证项 | 结果 | +| --- | --- | +| CPU private：严格告警构建、PollBatch 与 private ring 定向测试 | PASS | +| CPU shared：PollBatch、ordinary ring、PrepareMap、output/heap/Vector/materialize/loser 定向测试 | PASS | +| CPU private/shared b1 real-compute，全部运行断言 | PASS | +| CCEC private/shared swimlane 构建及产物 manifest | PASS | +| `git diff --check` | PASS | + +用户目录下抽取的 `g++-15` 在本机调用系统旧 `as` 时不识别编译器输出的 +`.base64` 汇编伪指令，因此该组合不能用于本轮 CPU 链接；这属于编译器与 +汇编器未配套，不是本次代码错误。CPU 回归使用本项目此前已验证的 +`/usr/bin/g++`，CCEC 双模式则由 CANN 9.1 工具链独立完成真实编译。 + +该修改不会增加 raw record 字段、atomic 调用、load、barrier 或热路径 +分支；它是 S5.2 追加 shared 热路径 atomic 站点前必须先完成的观察基础。 + +### 2026-07-26：S5.2a 接入 shared heap 四类 atomic 观察 + +本阶段只处理 shared Case1 已经实际执行的 heap reserve，不同时修改 +output publication/last-writer。`AtomicSite` 在既有 0～14 后追加： + +| id | site | op | 返回值用途 | b1 固定次数 | +| ---: | --- | --- | --- | ---: | +| 15 | `SharedHeapVendLoad` | Load | aggregate vend 合法性、容量与零输出进度 | 5 | +| 16 | `SharedHeapCursorLoad` | Load | 分片 cursor 合法性与 no-wrap 容量 | 4 | +| 17 | `SharedHeapCursorReserve` | FetchAdd | 旧 cursor 决定唯一物理区间 | 4 | +| 18 | `SharedHeapVendAdvance` | FetchAdd | 旧 vend 决定累计进度 | 4 | + +UP 的输出字节为 0，仍需要读取 aggregate vend，但不会读取或推进 shard +cursor；因此 vend load 是 `5 × batches`，其他三类各为 +`4 × batches`。四类返回值全部参与协议判断，CCEC 记录必须为 +return-ready；尤其两个 FetchAdd 显式传入 `result_used=true`，不能沿用 +发布型 wrapper 的默认 source-issue 口径。 + +实现上将 `stats.trace/stats.result` 从 shared `MaterializeTask` 显式传到 +`ReserveSharedOutputHeap`。单元测试继续使用模板默认 +`ObserveAtomics=false`，真实 scheduler 显式实例化 `true`。perf-clock 与 +submit-PMU 的 trace-free 构建中 `TraceAtomic*` 在编译期直接退化为原 +`Ops::*`，没有新增 load、运行时开关或 barrier。private 编译不经过该 +shared 接口；host schema 也新增模式门禁，private raw 出现 15～18 会被 +拒绝。 + +本次采用 append-only raw site registry，十列 `TraceRecord` 布局和 +schema-v4 phase 语义均未改变；0～14 的历史 raw 仍按原义转换。Python raw +当前不携带 TensorMap mode，因此 mode 交叉校验由编译态 host 与 CCEC +artifact manifest 负责，converter 只校验 site/op/result-used。该限制不 +应被描述成 Python 已独立证明模式一致。 + +验证结果： + +| 验证 | 结果 | +| --- | --- | +| converter 28 项测试，含四个新 return-ready site 与 Count 外拒绝 | PASS | +| CPU private/shared 严格告警构建与全部定向测试 | PASS | +| CPU private b1 atomic raw 中 shared-only site 行数 | 0 | +| CPU shared b1 atomic raw：5/4/4/4，业务断言、closure、dropped=0 | PASS | +| CCEC private/shared swimlane 与 artifact manifest | PASS | +| CPU/CCEC shared perf-clock；最终 ELF 无泳道 record writer | PASS | +| A5 shared b1：四站点合计 17，全部 return-ready | PASS | +| A5 shared b1：业务/协议断言、raw→merged→exclusive、dropped=0 | PASS | + +A5 b1 证据： + +```text +outputs/pa_scheduler_shared_swimlane_20260726_034146_3041189/ +``` + +该轮 raw 共 4,143 条，`logical_calls=4,517`、 +`physical_records=862`、`batched_poll_calls=3,878`、 +`poll_batch_records=223`，满足 +`physical = logical - batched + batch_records`。四个 heap site 的 +AIC/AIV 合计分别为 5/4/4/4，17 条 direct 记录的 return-ready 比例均为 +100%。诊断 Submit 为 84.578us，只用于证明 b1 观察闭环，不能当作关闭 +观察后的性能基线。 + +S5.2a 只关闭 review 中的 shared heap 漏采。`published` 快速 probe/等待、 +publication Exchange、`last_writer` load/init/commit 仍是下一小步； +在它们接入前不能宣称 shared atomic 已全量覆盖。 + +### 2026-07-26：S6.1 建立 shared 多级 writer 原语，保持现有单组热路不变 + +本阶段处理 PA 正确性边界：如果同一个 accumulator 经过 +`producer -> UP0(INOUT) -> UP1(INOUT)`，UP0 winner 必须先完成 +fanin 解析和 writer intent 登记；同 task 的 loser 在 writer-ready 发布 +前不能返回，否则它可能提前构造 UP1，并把陈旧 producer 当作前驱。 +`deps_prepared` 只表示依赖 writer 已登记，不表示 UP0 已执行或完成； +UP1 仍必须通过 fanin 等待 UP0 自己的 completion flag。 + +先澄清现有 Case1 的覆盖范围。`batches=256` 表示 256 条相互独立的 +五 task 链： + +```text +Alloc -> QK -> SF -> PV -> UP +``` + +它不是同一 batch 内的 256 个 block group；每条链的 UP 后没有继续消费 +同一 accumulator 的第二个 UP。因此原 B256 不需要执行 writer-ready +atomic，也不能用其通过来证明多级 writer 正确性。private TensorMap 更 +没有 shared output cell/`last_writer` 协议，这个问题不属于 private。 + +本次先在 shared-only common 代码建立四项公共基础： + +1. 复用 shared `TaskCell` 的 64B 内部 padding，在 offset 16 放置 + `deps_prepared`；整个 cell 大小和后续生产字段偏移不变，private + 预处理结果仍是原来的 `flag + vend + padding`。 +2. `PublishSharedWriterReady()` 在 writer intent 登记后执行 store barrier，再 + 把精确 task id 从 `-1` 发布到本 task 独占门；重复发布 fail-closed。 +3. `WaitForSharedWriterReady()` 只接受 `-1` 或精确 task id，并保留 + fatal/watchdog；错误值和超时均终止，不能把任意非负值当 ready。 +4. `CollectSharedFanin`、`CommitSharedFaninWriters` 增加编译期 + `ChainedWriter` 实例。默认实例继续要求 writer 精确等于 descriptor + producer；只有显式多级实例才要求 writer 精确等于调用方给出的前一 + writer。范围内但不是前一 writer 的陈旧值也必须拒绝。 + +定向 CPU 用例构造三个共享 accumulator 加一个 `manual_dep` output view， +验证第一阶段的三个 INOUT writer 提交、loser 在门发布前确实阻塞、门发布 +后返回、第二阶段把三个相同前驱去重为一条 fanin、第二次 writer 提交以及 +陈旧 writer 拒绝。这里的 `second_up=8` 是协议原语测试 id，不冒充当前 +主拓扑中的真实 `TaskKind::Up`；真实双 group 调度仍需下一阶段泛化 task +拓扑后覆盖。 + +一版过程草案曾提前把 `PaHasFollowingBlockGroup`、ticket byte 和 +winner/loser gate 分支接入现有 Submit。虽然 Case1 中条件恒假，CCEC +shared perf-clock 最终 ELF 的 `.text` 仍从 129,080B 增到 129,592B， +无收益地增加 512B。该草案已经撤回。最终阶段只保留 shared ABI/原语和 +协议测试，不让单组 B256 为未来拓扑执行恒假分支；重新构建后 `.text` +恢复 129,080B、`.rodata` 保持 288B。detached HEAD 与最终候选重建件 +的 `.text` 段 SHA256 还同为 `90f58e...fd26`，`.rodata` 同为 +`239e99...2ded`，证明不只是尺寸碰巧相同，而是现有 single-group +shared 性能代码段逐字节不变。 + +当前验证： + +| 项目 | 结果 | +| --- | --- | +| CPU shared 全部严格告警定向测试 | PASS | +| CPU private 全部严格告警定向测试 | PASS | +| CPU shared b1 real-compute 全部业务/协议断言 | PASS | +| shared b1 的 5 个 `deps_prepared` 均保持 `-1` | PASS | +| CCEC shared perf-clock 构建、split finish、混合 ELF 与 manifest | PASS | +| CCEC shared perf-clock `.text/.rodata` | 129,080B / 288B | +| `git diff --check` | PASS | + +private 的“无影响”另做了 HEAD/候选逐段对照，而不是只看用例通过： + +- private CPU `main.cpp` 在相同宏下的预处理输出逐字节相同，均为 + 1,441,443B，SHA256 均为 `09d35006...c013693`；最终可执行文件也 + 逐字节相同，`.text=134,616B`、`.rodata=11,984B`，两段 hash 各自 + 完全一致。 +- private CCEC/swimlane 的 AIC、AIV、两份 finish、两份 runtime 和最终 + mixed kernel 共七个对象，其 `.text/.rodata` 大小与段字节 hash 全部 + 一致。最终 mixed kernel 为 `.text=590,392B`、`.rodata=696B`。 +- CCEC device `.o` 的整文件 hash 会因 DWARF 内嵌工作树路径和新增 + shared-only 源码推移调试行号而不同；这不是 private 指令变化，不能拿 + manifest 的整对象 hash 代替代码段对照。CCEC host 整文件则仍逐字节 + 相同。 + +下一阶段必须先生成真实的同 batch 双 block-group 拓扑，再把 +writer-ready gate 接到非末组 UP 的 winner/loser Submit，并让下一组 +显式传入前一 UP task id。届时需要新增调度级 CPU/CCEC/A5 正确性用例； +在此之前不能把本阶段描述成“真实 PA 双组已跑通”。 + +### 2026-07-26：S6.2 用真实 PA 双组参数验证 shared writer intent + +本阶段继续只完善 shared 多级 writer 的正确性基础，不把尚未接通的九 +task replay 冒充成完整功能，也不让 private 或现有单组 Case1 执行未来 +协议。private TensorMap 没有 shared output cell、`last_writer` 或跨组 +writer-ready 问题，所以验收标准是生成代码零变化，而不是“性能影响很小”。 + +#### 真实混合引用暴露并修正了单参数模型 + +上一阶段的定向测试只构造了三个 INOUT，容易误以为第二组 UP 的全部 +shared 引用都应匹配前一 UP。真实 `BuildCallbackSubmitArgs()` +生成 7 个 tensor： + +```text +SF max INPUT, SF sum INPUT, PV output INPUT, +accumulated output/sum/max INOUT, manual output_view INOUT +``` + +因此第二组同时包含两类 producer： + +- 本组 SF/PV 的 fresh INPUT 必须继续匹配 task 6/7； +- 三个 accumulator 的 descriptor identity 仍为 Alloc task 0，但 + `last_writer` 必须精确匹配前一 UP task 4。 + +`ChainedWriter` 现显式接收 +`(chained_producer_task_id, expected_shared_writer)`。只有原 producer +等于 `chained_producer_task_id` 的 shared ref 才采用前一 writer，其余 +ref 仍匹配自身 producer；同时要求 +`producer < expected_writer < current_task`，且 selector 至少命中一条 +消费引用和一条写引用。传错 selector、`producer == writer`、陈旧 writer +或跳级 writer 均 fail-closed，不能静默退化成默认路径。 + +#### PA 专用 pre-Build writer-intent 契约 + +新增的 `PreparePaSharedWriterIntent()` 只供 shared、non-final UP winner +在 Build 前调用，顺序固定为： + +```text +校验真实前端状态 + -> CollectSharedFanin，结果写入 SubmitContext::fanin + -> CommitSharedFaninWriters，登记本 UP 为三个 accumulator 的 writer + -> StoreBarrier + -> 发布本 task 的 deps_prepared +``` + +该 helper 明确拒绝： + +- `args.has_error`、越界 tensor/scalar count 或非 winner context； +- shared writer 数量不是恰好三个； +- 除 `manual_dep` output_view 之外的 ordinary-region writer； +- 任何 publication、旧 writer、selector 或 ordinary lookup 异常。 + +成功只表示 fanin 已解析且 writer intent 已登记，不表示本 UP 的 kernel +已经执行；task completion 仍由原 `flag` 发布。后续 Finish 接线必须复用 +`context.fanin` 并跳过第二次 Collect/Commit，确保三个 writer FetchMax +恰好执行一次。当前阶段尚未修改 Finish，因此只把 helper 作为隔离原语 +测试，不能在主路径提前调用。 + +失败语义也按 atomic 线性化边界表述： + +- Collect 或参数校验失败时 writer 不变、gate 保持 `-1`； +- Commit 中途失败时，已经线性化的 writer 前缀保留为终止现场，未触碰 + 的后缀不变，gate 仍保持 `-1`，随后广播 fatal； +- gate 本身使用 Exchange。若旧 gate 已经异常，它会写入当前 task id 后 + 返回失败并广播 fatal；终止态不回滚。不能把这一情形写成“发布失败时 + gate 从未短暂 ready”，除非未来另行引入 CAS，而那会改变成功热路。 + +#### 当前双组测试证明什么 + +定向 CPU 测试使用真实 PA 构参器建立： + +```text +Alloc0 +group0: QK1, SF2, PV3, UP4 +group1: QK5, SF6, PV7, UP8 +``` + +它证明： + +- UP0 参数为 7 tensors / 2 scalars，`is_first=1,is_last=0`； +- loser 在 UP0 writer intent 发布前不能开始构造第二组； +- UP0 fanin 精确为 `{SF2, PV3, Alloc0}`； +- 发布 gate 后 UP0 completion flag 仍未完成； +- UP1 参数为 7 tensors / 2 scalars，`is_first=0,is_last=1`； +- UP1 fanin 精确为 `{SF6, PV7, UP4}`； +- 两组各登记三个 accumulator writer，最终三槽 writer 均为 UP8； +- 解析失败、第二条 writer 登记故障、缺少 accumulator、前端 error、 + scalar count 越界和 ordinary writer 均不会发布 gate。 + +它尚未证明默认 main loop 能运行 task 8。现有代码仍把 +`task_id % 5` 当作 TaskKind，task 8 会被误判为 PV；Materialize、split +Finish 和 host oracle 也有同类五 task 假设。下一阶段必须为 shared-only +双组测试引入显式 task kind/group 元数据，并证明 early Prepare 后 Finish +只消费已准备的 fanin，不可只在 main 中机械追加四次 Submit。 + +#### 本阶段回归 + +| 验证 | 结果 | +| --- | --- | +| CPU shared 全部严格告警定向测试 | PASS | +| CPU shared b1 real-compute，全部业务/协议断言 | PASS | +| single-group shared 的 5 个 `deps_prepared` | 全部保持 `-1` | +| CCEC shared perf-clock 构建与 manifest | PASS | +| CCEC shared single-group `.text/.rodata` | 129,080B / 288B | +| CCEC shared 段 hash | 仍为 `90f58e...fd26` / `239e99...2ded` | +| CPU private 当前/HEAD 可执行文件 | 逐字节相同，SHA256 均为 `ffa19a...fae4` | +| CCEC private perf-clock 七个 device 对象 | `.text/.rodata` 段大小与字节 hash 全部相同 | + +上述 shared 代码段与 S6.1 完全相同，证明新增 helper 和测试没有让当前 +单组 B256 多出恒假分支。private 的 CPU 整体产物和 CCEC 七个 device +对象代码/常量段也都不变；含 DWARF 的 CCEC 整对象不作为比较口径。 + +### 2026-07-26：S6.3 接通 shared 显式 Finish 协议并补齐异常收敛门槛 + +本阶段仍只在 standalone 建立双组调度的公共基础，没有把尚未完成的 +九 task main 或 host oracle 冒充成端到端结果。架构边界进一步明确为： + +- private 继续使用原五 task、`task_id % 5`、`reserved=0` 和私有 + TensorMap；shared 多组问题不能污染 private。 +- 全部 shared 构建统一携带 task kind/group 元数据。现有单组 B256 + 自然使用 `group=0,has_following=false`，因此走同一正确性基础但不发布 + writer-ready gate；不能另造一个“多组专用 ELF”回避公共代码。 +- shared 实际组数最终来自每 batch 的 `context_lens`。本阶段只接通 + ticket、Materialize、Finish 和异常收敛，主 replay 仍固定五 task。 + +#### 固定 16B ticket 内的显式 replay 身份 + +`CallbackSubmitTicket` 没有扩容，只复用最后一个 `reserved` byte： + +```text +bits 0..2 : TaskKind +bits 3..4 : group index，最多四组 +bit 5 : has-following-group，仅 non-final UP 可置位 +bit 6 : 保留，当前必须为 0 +bit 7 : shared metadata present +``` + +解码后由 +`batch_start = task_id - SharedPaTaskOffset(kind,group)` 按 ticket +自洽布局反推 batch 起点。`MaterializeTask` 不再在 shared 下使用 +`%5`，而是显式接收 `kind/batch_start/group`，并校验所有 producer 都 +落在该 batch 已经可见的 replay 前缀内。这样 task 8 会按第二组 UP +处理，不会被误判为 PV。这里还不是独立 task-plan 身份校验；下一阶段 +必须让 replay 与 host oracle 共同消费同一 plan，才能从外部证明 +`batch_start`，不能把 ticket 的自洽反推冒充成完整 plan 证明。 + +split Finish 还显式验证： + +- winner 的 `function_id` 必须等于 `FunctionId(kind)`； +- loser 的 `function_id` 必须为 `-1`； +- ticket kind 对应的 output count 必须与 `shared_result` 一致； +- metadata present/reserved/kind/group/has-following 组合全部合法。 +- `task_id` 必须小于 shared 固定容量 `kMaxTasks`，即使损坏的 + `runtime.task_count` 更大，final loser 也不能绕过 output table 边界。 + +QK 与 PV 的 output count 同为 1，因此只校验 output count 不足以证明 +kind 正确。定向用例专门构造“QK kind + PV function_id”，确认在进入 +Finish body、读取 `TaskArgs` 之前就 fail-closed。 + +#### non-final UP 的 Finish 顺序 + +shared winner 现在固定按下列顺序推进： + +```text +Materialize + -> CollectSharedFanin，并只折叠一次 dependency signature + -> ValidateEmptySharedRegistration + -> non-final UP：登记三个 accumulator writer，发布 gate + -> BuildWinner + -> 已提前登记 writer 时跳过第二次 Commit + -> fresh-output 封口 +``` + +首组 UP 使用普通 `(producer=Alloc,writer=Alloc)` 校验；第二组及后续 +UP 使用显式 `(chained_producer=Alloc,expected_writer=上一组UP)`。 +final UP 不需要放行下一组，保持 Build 后 writer commit。loser 只有在 +`has_following=true` 时等待精确 gate；放行后仍不读取 `TaskArgs`。 + +`ValidatePaSharedWriterIntentShape()` 也从“shared writer 数量为三个” +收紧为“恰好是本 batch Alloc 的 slot 0/1/2，且三槽各出现一次”，并 +要求恰好一个非 symbol、`manual_dep` 的 writer，对应真实 UP 参数中的 +output view。当前不凭地址猜测这个 view 的具体对象身份；能够证明的是 +缺失/重复 manual-dependency writer、任意三个其他 symbol、重复 slot、 +缺槽或 ordinary writer 都不能借 PA 快路发布 gate。 + +#### 容量与 shared ABI + +旧 `kMaxTasks=256×5=1280` 只能覆盖单组，b1 双组测试因 task id 很小 +不会暴露这个问题。shared 现按请求上限预留： + +```text +max groups per batch = 256 / 64 = 4 +max tasks per batch = 1 + 4 × 4 = 17 +shared kMaxTasks = 256 × 17 = 4352 +``` + +因此 `SharedTensorMapSidecar` 的 output table 扩到 4352 个 cell， +sidecar 大小从 4,736,192B 增到 11,027,648B，shared build identity ABI +从 4 升到 5。region ring、shared-output table 起点及生产前缀偏移不动， +只有其后的 heap/vector sidecar 顺延。private 仍是 1280 个 task、 +ABI 4 和原 sidecar 布局。 + +#### gate 已发布后的终止收敛 + +gate 在 non-final UP Build 前发布是下一组能正确构参的必要条件，但它 +同时形成一个失败窗口：后继 UP slot 可能已经依赖 task4；若 task4 随后 +Build/封口失败，其 completion flag 永远不会到达。仅广播 fatal 不够， +旧 `WaitForSlot` 或 `FinalDrain` 都可能永久等待。 + +shared-only 修正分两层： + +1. slot 已满且 `DrainReady()` 连续无进展时，每 1024 次才直接读取一次 + fatal。看到 fatal 后 `WaitForSlot` 返回失败，使本 worker 停止 replay； + 正常 winner 和未进入背压的路径不增加原子读取。 +2. 所有 worker 已越过 final replay barrier 后，若本核仍连续无进展， + 同样每 1024 次探测 fatal。确认失败后清除本核四个 slot 的 + `occupied/built` 和 `occupied_count`，但不调用 kernel、 + `CompleteTask` 或 placement 统计。task/fanin 内容保留用于诊断。 + +清槽只意味着“失败轮次可以退出”，绝不能解释成任务完成。故障用例仍 +要求 task4/task8 flag 为 0、kernel/placement 为 0、fatal 为 1。即使 +`occupied_count` 已损坏，helper 也先清完所有 slot 再返回计数不一致， +避免错误路径二次卡死。 + +#### 本阶段门槛结果 + +| 验证 | 结果 | +| --- | --- | +| CPU shared 全部严格告警门槛测试 | PASS | +| task4 non-final loser：门前轮询、门后仍不读 `PROT_NONE TaskArgs` | PASS | +| task8 final loser：`function_id=-1`，只从 ticket 恢复 UP | PASS | +| task4/8 winner Finish：fanin `{2,3,0}` / `{6,7,4}` | PASS | +| task4/8 单段依赖签名 | `7f405ca7dea83459` / `f772149f1ca20d6b` | +| 非法 reserved、winner/loser function、越界 task、错误 UP writer 形状 | 全部拒绝 | +| fatal 满槽退出、final-barrier 后调用的 blocked-slot 清理原语 | PASS | +| CPU shared b1 real-compute 单组业务/协议回归 | PASS | +| CCEC shared swimlane、split finish、mixed ELF、manifest | PASS | +| A5 CCEC shared b1 单组，96 worker、5 task、4 kernel | PASS | +| A5 b1 shared symbol published/input/commit | `8 / 5 / 3` | +| A5 b1 dependency signature | `5cb454393ed48dcb` | +| A5 b1 writer-ready gate | 五个 task 全部保持 `-1` | +| CPU private 与 `24eb97ed` 同命令完整 ELF | 逐字节相同，SHA256 `ffa19a...fae4` | +| CCEC private 七对象 `.text/.rodata` | 全部逐字节相同 | +| `git diff --check` | PASS | + +A5 b1 本轮无泳道、PMU 关闭，`submit_span_us=78.259`，只作为现有单组 +设备路径的正确性回归；单次数据不用于判断新 metadata 的性能代价。 + +本阶段尚未完成的边界必须继续保留： + +- `RunSchedulerImpl` 仍以 `batches * 5` 建 task_count，主循环仍只提交 + `Alloc/QK/SF/PV/UP`； +- host oracle、CCEC host 和部分 trace 分析仍存在 `%5`/五 task 假设； +- 尚未得到 b1 九 task 的 864 Submit、480 Claim、8 kernel、 + 13 published output、10 fanin/input、6 writer commit 和最终 writer=8 + 的完整闭环； +- 尚未在真实九 task 运行中注入“gate 已发布、UP0 封口失败”，当前证明 + 是公共 Finish 与终止原语的定向门槛。 + +下一阶段应集中建立 shared task plan，让 device replay 与 host oracle +共同消费同一布局，再跑完整九 task CPU/CCEC/A5。不能在现有五 task +循环后机械追加四次 Submit，也不能先改 host 期望值来掩盖 device 仍按 +`%5` 解释任务。 + +### 2026-07-26：S6.4a 建立 shared 每批 task plan 的纯函数门槛 + +本小步只建立后续 device replay 与 host oracle 共用的计划规则，不改 +当前 main 行为，也不把 `kTasksPerBatch` 从 5 改成 9。该常量继续表示 +五种 `TaskKind`；shared 每批的动态 task 数由输入 `context_len` 推导： + +```text +block_count = ceil(context_len / 128) +group_count = ceil(block_count / 64) +task_count = 1 + 4 × group_count +layout = Alloc + group_count × (QK/SF/PV/UP) +``` + +`SharedPaBatchPlan` 保存 `batch_start/group_count/task_count`， +`SharedPaPlannedTaskAt()` 从 batch 内 offset 唯一恢复 +`kind/group/has-following/is-last-in-batch`。计划先拒绝超过 PA +`256 blocks/request` 的 context 和越过 shared `kMaxTasks` 的 batch， +再执行向上取整，避免损坏的负 `int32` context 转成巨大无符号数后溢出。 + +门槛覆盖： + +- 空 context 仍有 loop 外的一个 Alloc； +- 64/128/256 blocks 分别得到 5/9/17 task； +- 64 blocks 后多一个 token 立即进入第二组； +- 最大 context 恰好可接受，超过一个 token、`UINT64_MAX` 和最后 batch + 越界全部拒绝； +- 每个 offset 的 kind/group、non-final UP gate 标记和 batch 尾身份 + 与 `SharedPaTaskOffset()` 双向一致。 + +CPU shared 严格告警全门槛通过。下一小步才让 shared `RunSchedulerImpl` +消费该计划；private 五 task replay 保持原预处理结果。 + +### 2026-07-26：S6.4b 让 shared device replay 消费动态 task plan + +本小步只收口 device replay 与跨 TU Finish 协议；host 仍是上一阶段的 +单组 oracle，因此当前只把默认 8192-token G1 跑成端到端结果，不把 +尚未合并 host plan 的 G0/G2/G4 冒充成已完成。 + +#### shared replay 从固定五 task 改为 `1+4N` + +每个 worker 对每个 batch 只读一次 `context_len`，随后调用 +`BuildSharedPaBatchPlan()`，严格按下列顺序提交： + +```text +Alloc +for group in [0, group_count): + PreparePaBlockGroup(group * 64) + QK -> SF -> PV -> UP +``` + +每批结束立即检查 `local_index == batch_start + task_count`。所有 batch +回放结束后，再用实际 `local_index` 封口 split runtime 的 task 数，并 +与末个 ticket 声明的 `task_id+1` 二次核对。private 的原五次 +`SubmitCallbackTask()` 调用完整保留在预处理 `#else` 中。 + +末次 Submit 身份没有通过 96 个 worker 预扫全部 `context_lens` 获得。 +那种做法在 B256 最坏会额外引入 `96×256=24,576` 次 GM load。当前改为 +复用 16B ticket 的 bit6： + +```text +bits 0..2 : TaskKind +bits 3..4 : group index +bit 5 : has-following-group +bit 6 : global last Submit +bit 7 : shared metadata present +``` + +因此 S6.3 中“bit6 保留且必须为 0”只描述当时 ABI5 的历史状态；从本 +阶段 ABI6 起，bit6 正式承担 global-last 身份。它只允许出现在 Alloc +或 UP，且不能与 has-following 同时出现。 + +#### metadata 只能由 plan 推导 + +第一版草案曾让五个调用点分别传入 `group/has-following/last`。只做 +编码语法检查无法证明 ticket 与动态 plan 一致，定向测试甚至可以重复 +task4、跳过 task5～7 后直接提交 task8，最后仍碰巧得到 +`declared_task_count==local_index`。该过程态没有提交。 + +收口后的 `SubmitCallbackTask()` 只接收 +`SharedPaBatchPlan + task_offset`，并在 Claim 前完成四项核对： + +- `SharedPaPlannedTaskAt()` 能恢复唯一 task； +- 恢复的 kind 必须等于模板 `TaskKind`； +- `task_id == batch_start + task_offset`； +- global-last 只能由“最后 batch 且 batch 内最后 task”推导。 + +跨 TU 前，caller 还把 `(task_id, encoded-meta)` 的一次性 binding 写入 +已有 split runtime `reserved` 字段。Finish 必须同时满足： + +- ticket 与这份 binding 完全一致； +- `task_id == 已成功 Submit 数`，即从 0 严格连续； +- context、winner/function、output count 与 ticket 一致。 + +Finish 读取后无条件清除 binding，fatal 路也不会留下过程态。最终 +split 协议继续要求 `reserved==0`。这样重复、跳号、乱序，以及跨 TU +后把合法 Alloc/UP 改成 early-last 都在读取 `TaskArgs` 前被拒绝。 + +#### PMU 与实际 task 数的收口顺序 + +shared submit-PMU 的 phase call 数不再使用 `batches×5`，而是在 Stop +时读取本 worker 已回放的 `local_index`。公共调度先完成： + +```text +actual task_count 检查 +-> declared last 检查 +-> split runtime 封口 +-> PmuWindowStop +``` + +因此 missing/early/duplicate-last 不会先产出看似合法的 phase shape； +Stop 在已经开启窗口的 fatal 路仍无条件执行。host 后续仍需用独立 +authoritative plan 对实际 task 数做外部证明，不能把 device +`local_index` 当成期望公式。 + +#### 新增与修正的门槛 + +- split loser 改为真实连续 `0..8` 两组序列，task4 等 writer-ready， + task8 为唯一 global-last； +- `[empty,empty]`、`[G1,empty]`、`[empty,G2]` 验证累计 + `batch_start` 与唯一 global-last； +- 错误 winner/loser function、非法 QK-last、合法 early-last Alloc + 的跨 TU 篡改、shared-output 容量越界全部在保护页 `TaskArgs` 前拒绝； +- task 2047→2048→4096 两次 payload 环绕，证明 2048-slot scratch + 可以复用，而已复制到 `SharedOutputCell` 与 `LocalSlot` 的 descriptor + 不被覆盖。 + +#### 本阶段结果 + +| 验证 | 结果 | +| --- | --- | +| CPU shared 全部严格告警门槛 | PASS | +| CPU shared B1/B256 默认 G1 全部业务/协议断言 | PASS | +| CCEC shared swimlane / perf-clock / submit-PMU-none 构建与 manifest | PASS | +| A5 CCEC shared B1 默认 G1 | 96 worker、5 task、4 kernel，全部断言 PASS | +| A5 B1 dependency / shared symbol | `5cb454393ed48dcb` / `8,5,3` | +| A5 B1 单次无泳道 Submit | `75.731 us`，只作正确性证据 | +| CPU private 当前/`ee0fe8c6` 完整 ELF | 逐字节相同，SHA256 均为 `ffa19a...fae4` | +| CCEC private perf-clock 七个 device 对象 | `.text/.rodata` 全部逐字节相同 | +| `git diff --check` | PASS | + +本阶段仍未声称： + +- host oracle、converter、exclusive analyzer 和 PMU host sidecar 已支持 + 动态 plan； +- G0/G2/G4/mixed 已完成 CPU/CCEC/A5 闭环； +- B256 G2/G4 可直接运行。shared heap 当前仍是 256 MiB no-wrap， + 资源准入与回收必须在后续单独处理，不能靠放宽断言掩盖容量不足。 + +### 2026-07-26：S6.4c 建立独立 host task plan 并闭合动态设备矩阵 + +本小步把“设备能按动态 plan 执行”提升为“host 能独立证明设备执行的 +就是输入要求的 plan”。host 没有 include 或调用 +`pa_frontend.h::BuildSharedPaBatchPlan()`，而是只读取 kernel 返回后的 +`SchedulerState.context_lens`，独立重算： + +```text +context_len -> block_count -> group_count +batch_start -> Alloc + groups × (QK/SF/PV/UP) +``` + +`SharedHostTaskPlan` 保存每批累计起点、group 数、逐 task kind/group、 +partial group block 数、前后组关系、输出字节和 canonical heap prefix。 +因此 device 与 host 即使把同一个 group 公式写错，也不会形成“复用同一 +helper 后一起通过”的同错 oracle。 + +#### host 动态校验范围 + +`Validate()`、raw export 和 raw analyzer 都从上述独立 plan 获取实际 +task 身份。host 现在逐项闭合： + +- `replay = 96 × total_tasks`，Claim 为 + `96 × batches + 192 × total_groups`； +- Alloc winner 数等于 batch 数，QK/SF/PV/UP winner 与 kernel 数分别 + 等于 group 数； +- fresh output、INPUT resolve、INOUT writer commit、slot tensor/scalar + copy 和 fanin edge 都按动态 group 数推导； +- 后一组 UP 的 accumulator writer 必须是上一组 UP，首组仍指向本批 + Alloc； +- descriptor 的第二维和输出字节按本组真实 block 数重建，partial + final group 不再被 64-block 固定常量掩盖； +- shared heap 八分片 cursor、aggregate vend、writer-ready gate、 + Claim cursor、逐 worker frontend 计数和动态 trace 总数均由 plan + 推导； +- normalized writer signature 使用 canonical task base，保证 G0/G1/ + G2/G4 与 private 固定五 task 的逻辑投影仍可比较。 + +`--shared-context-lens` 只用于 standalone shared 测试：一个值广播到 +全部 batch，多个值必须与 `--batches` 精确等长；未指定时仍保持生产 +默认 8192。private 编译既不接受该参数，也不携带相应容器和解析逻辑。 + +#### G0/G1/G2/G4 与 mixed 门槛 + +新增 host-only 定向用例覆盖 G0、G1、G2 partial、G2 full、G4 和 +`[G0,G1,G2,G4]` mixed，另覆盖负 context、超过四组、累计容量越界、 +CLI 广播/逐 batch/错误长度。CPU shared 完整调度的关键闭合值如下： + +| 输入 | tasks | groups | published/input/commit | fanin signature | heap bytes | +| --- | ---: | ---: | ---: | --- | ---: | +| G0：`0` | 1 | 0 | `3/0/0` | `0` | 10,240 | +| G1：`8192` | 5 | 1 | `8/5/3` | `5cb454393ed48dcb` | 806,912 | +| G2 partial：`8193` | 9 | 2 | `13/10/6` | `dda63f4f5405eaf1` | 829,440 | +| G2 full：`16384` | 9 | 2 | `13/10/6` | `dda63f4f5405eaf1` | 1,603,584 | +| G4：`32768` | 17 | 4 | `23/20/12` | `58d7a4b63aac2c4e` | 3,196,928 | +| mixed：`0,8192,8193,32768` | 32 | 7 | `47/35/21` | `6437bff09d8f8a11` | 4,843,520 | + +A5 CCEC 无泳道、scalar-nop 为零的正确性矩阵也闭合： + +| 输入 | tasks | Claim | kernel | 单次 Submit | +| --- | ---: | ---: | ---: | ---: | +| G0 | 1 | 96 | 0 | 23.882 us | +| G2 partial | 9 | 480 | 8 | 141.658 us | +| G4 | 17 | 864 | 16 | 261.723 us | +| mixed B4 | 32 | 1,728 | 28 | 347.386 us | + +四组均通过 host plan、symbol、dependency、descriptor、heap、cursor、 +gate、96-worker replay 和最终状态断言。表中时间只说明相应动态路径 +确实完成，不是 real-compute 性能基线，也不能跨构建比较。 + +#### private 身份隔离的二次收口 + +host plan 初版虽然在业务上只服务 shared，但若把等价 helper、字符串或 +临时变量留在 private 的同一翻译单元，GCC 13 的 IPA/内联决策仍会变化： +一次中间构建中 private CPU `.text` 从 `0x20dd8` 增到 `0x21568`， +`FinishCallbackSubmitBody()` 等 device 模拟函数也因 host AST 改变而 +选择了不同内联方案。这不是 private 业务语义变化，但不满足本项目的 +严格可比门槛。 + +最终处理不是放宽为“断言通过”，而是让所有 shared host 扩展在预处理 +阶段分叉，private `#else` 原样保留 `ee0fe8c6` 的 token path。收口后： + +- private CPU 完整预处理输出逐字节相同，SHA256 + `e7f71dc48ebca91ecbf8766a12296f06fafe1f684c836f95214e833c3ab79052`； +- private CPU 完整 ELF 逐字节相同，SHA256 + `ffa19a3ea82cb75a46bb4a061231158818a0610ebc3d075188b986b5ae4bfae4`； +- private CCEC perf-clock 七个 device 对象的 `.text/.rodata` + 全部逐字节相同； +- shared CPU 的预处理输出也与 host plan 初版逐字节相同，证明隔离修正 + 没有反向改变已上板的 shared 路径。 + +下一阶段只处理动态 task identity 在 converter、exclusive analyzer 和 +submit-PMU host 加工链中的传播。完成前不做性能优化，也不把固定 `%5` +的旧加工结果当作动态 shared 性能证据。 + +### 2026-07-26：S6.4d 让泳道加工恢复动态 task identity + +本小步只修 raw→merged→exclusive 加工链，不改设备采集 ABI。schema-v4 +的每条 Submit 和 Claim 原本就记录了 `is_alloc`，winner 尾动作也原本 +就记录了 `WinnerBuild.function_id` 或 `AllocComplete`。因此新增设备 +字段或逐事件 metadata 既冗余，也会继续放大接近 300 MiB 的诊断文件。 + +converter 现在从所有核的既有 Submit 记录独立恢复 task plan： + +1. 每核 task ID 必须是相同的连续 `0..N-1`； +2. 同一 task 的 Alloc 标记必须在所有核上一致； +3. task 0 必须是 Alloc； +4. 相邻 Alloc 之间必须严格构成 + `Alloc + 0..4 × (QK,SF,PV,UP)`； +5. Claim/Submit 的 Alloc 与 winner 语义必须一致； +6. winner 尾动作必须与推导出的 kind 对应： + Alloc 使用 `AllocComplete/-1`，其余使用 + `WinnerBuild/QK|SF|PV|UP function`。 + +这样固定五 task private、G0、G1、G2、G4 和 mixed 共用同一个离线 +校验机制，但不再使用全局 `task_id % 5`。exclusive analyzer 复用同一 +推导结果给 Submit 间空白命名；时间闭合公式和报告结构没有变化。动态 +mixed 中现在能正确区分： + +```text +Alloc -> Alloc 空 batch 后进入下一 batch +UP -> Alloc 一个 batch 结束后进入下一 batch +UP -> QK 同一 batch 继续下一组 +``` + +这一层只恢复 task kind 和 batch/group 边界。仅凭十列 raw，G2 partial +的 8193 与 G2 full 的 16384 具有相同 kind 流，无法恢复末组真实 block +数；本阶段不伪造这项身份。完整 context 身份会在下一步 submit-PMU +host JSON 中由独立 host plan 显式导出，并由 Python 再计算。 + +#### 回归结果 + +| 门槛 | 结果 | +| --- | --- | +| converter/exclusive 及目录内全部 Python 单测 | 104 项 PASS | +| CPU shared G0/G1/G2 partial/G2 full/G4 | raw、merged、exclusive 全部 PASS | +| CPU shared mixed `0,8192,8193,32768` | 32 task、7 group，全部 PASS | +| mixed Submit 间边界 | `Alloc->Alloc=96`、`UP->Alloc=192`、`UP->QK=384` | +| private CCEC 既有 raw 重新生成 merged | 与旧文件逐字节相同 | +| private CCEC 既有 raw 重新生成 exclusive | 与旧文件逐字节相同 | + +上述 CPU 运行只证明动态采集和离线加工闭合，不把 CPU Submit 时间当作 +A5 性能数据。设备 raw 字段数、记录数和 merged 每事件结构均未增加。 +下一小步再独立处理 submit-PMU 的 host/Python `batches×5` 固定假设。 + +### 2026-07-26：S6.4e 让 submit-PMU 使用 shared 动态 task identity + +本小步只修 standalone submit-PMU 的期望次数、raw 身份和加工链，不改 +device `WorkerResult`、PMU counter、窗口边界或逐核记录布局。shared +kernel 在 S6.4b 已经用实际回放完成后的 `local_index` 写入 phase call； +本阶段补上与它独立的 host/Python 期望值，避免继续用 private 的 +`5 × batches` 解释 G0/G2/G4。 + +#### Host raw：shared schema-v6，private 保持 schema-v5 + +shared host 在 PMU 校验和 raw 发布前分别从最终 +`SchedulerState.context_lens` 重建 `SharedHostTaskPlan`。running phase +的逐核期望调用数改为 `plan.total_tasks`，`none` 仍严格为 0；全局期望 +为逐核值乘 96。两次重建结果还会与同一轮 PMU 校验累计值交叉核对，计划 +在校验与发布之间不一致时拒绝 raw。 + +shared raw 使用 schema-v6，并在 configuration 中只增加三项 host 身份： + +```json +{ + "tensormap_mode": "shared", + "shared_context_lens": [0, 8192, 8193, 32768], + "shared_task_plan": { + "total_groups": 7, + "tasks_per_worker": 32 + } +} +``` + +完整 context 向量是区分 G2 partial/full、重算实际 task 数所必需的输入， +不是逐事件字段；B256 也只增加 256 个整数，不放大 device raw 记录。 +private 编译的原 `5 × batches` 分支和 schema-v5 JSON 保持不变，不输出 +shared 字段。 + +#### Python 不信任 host 自报 task 数 + +analyzer 对 schema-v6 重新执行独立公式： + +```text +blocks = ceil(context_len / 128) +groups = ceil(blocks / 64) +tasks_per_worker = batches + 4 × sum(groups) +``` + +它要求 batches 在 `[1,256]`、context 数量与 batches 相等、每项在 +`[0,32768]`，然后同时核对： + +- host `shared_task_plan`； +- 每核 `phase_calls/phase_expected_calls`； +- 96 核 validation 总调用数； +- v5 已有的 begin/end、shadow partition 和 phase time 契约。 + +context 向量、重建后的 plan 和 `final_barrier` 都进入多轮 fingerprint。 +同 batches 但不同 group 数不能聚合；即使 task 数相同，`1` 与 `128` +这类不同 context identity 也不能静默聚合。schema-v4/v5 继续使用固定 +五 task 规则，并拒绝携带 shared 身份字段；这样默认 G1 恰好也是五 task +时，也不能把误标为 v5 的 shared raw 当成 private。v5 与 v6 同样不能 +混合分析。schema-v6 还强制每条 record 和 validation 都携带 +`phase_expected_calls`，不把缺字段等同为旧格式兼容。 + +HTML 只显示每核 task 数、group 总数和 context 最小/最大值，不展开 +B256 的完整向量。这样可以看见动态采集身份，又不把报告变成输入转储。 + +#### 门槛与 A5 证据 + +| 门槛 | 结果 | +| --- | --- | +| 目录内全部 Python 单测 | 115 项 PASS | +| schema-v6 合成矩阵 | G0/G1/G2 partial/G2 full/G4/mixed 及元数据篡改全部闭合 | +| CCEC shared submit-PMU 构建 | none/claim/efdrain/materialize/register 全部 manifest PASS | +| CCEC private 对照构建 | none/claim manifest PASS | +| A5 shared mixed B4 Claim | 32 calls/核、3,072 calls/96 核、schema-v6、HTML PASS | +| mixed host/device plan | 4 batches、7 groups、32 tasks，Claim 1,728，28 个 kernel | +| A5 private B1 Claim | 5 calls/核、480 calls/96 核、schema-v5、HTML PASS | +| PMU owner | 两次正式样本 configure/restore/cleanup 全部 PASS | +| `git diff --check` | PASS | + +mixed A5 raw 与报告位于： + +```text +outputs/shared_dynamic_submit_pmu_v6_20260726/ + mixed_claim_icache_raw.json + mixed_claim_icache_report.html + private_g1_claim_icache_raw.json + private_g1_claim_icache_report.html +``` + +这两次单轮时间只证明采集链可执行，不能用来比较 private/shared 性能。 + +单独运行 shared G0 时，调度、动态 plan、96 次 Claim、PMU 边界和 owner +恢复全部通过，但通用 real-compute host 校验仍把“0 个计算 task，因此 +192 个输出 tile 全为 sentinel”误判为缺失 kernel，故按发布门槛没有 +生成 raw。这不是放宽 PMU 契约的理由；下一小步应让 shared G0 明确接受 +零 active tile，同时保持 private 和 shared 非零 group 的 +`active_tiles != 0` 门槛不变，然后再继续 A5 动态矩阵。 + +### 2026-07-26：S6.4f 修正 shared G0 的零 compute 输出门槛 + +S6.4e 的 G0 失败不是 scheduler 或 PMU 错误。`context_len=0` 的权威 +计划只有一个 Alloc，四类 compute task 和 kernel 都应为零；device +返回的 192 个 workload output tile 也全部保持 sentinel。真正的问题 +是通用 host 输出校验仍带着 private 固定五 task 的旧前提： + +```text +active_tiles != 0 +``` + +private 的每个合法 batch 固定包含 QK/SF/PV/UP，这个条件正确；shared +G0 则必须恰好为零。本阶段没有跳过输出校验，也没有伪造 compute task， +而是新增 host-only 的计划一致性判断： + +```text +shared: (host plan total_groups != 0) == (active_tiles != 0) +private: active_tiles != 0 +``` + +随后仍逐元素验证所有 active tile 的计算结果和所有 inactive tile 的 +sentinel，并要求 `active + inactive == 192`。因此： + +- shared G0 必须是零 active，任一被意外改写的 sentinel 仍会失败； +- shared G1/G2/G4/mixed 必须至少有一个 active tile，全零不能冒充 G0； +- private 预处理 `#else` 仍执行原来的 `active_tiles != 0`。 + +门槛测试直接覆盖 G0 的 192 个 sentinel 输出，以及 G0/G1 对 +`active_tiles=0/1` 的相反要求。验证结果： + +| 门槛 | 结果 | +| --- | --- | +| CPU shared 全部 host/协议定向门槛 | PASS | +| CPU shared G0 real-compute 完整调度 | 0 kernel、0 active、192 sentinel，PASS | +| CPU private 构建与 ring 定向门槛 | PASS | +| CCEC shared/private submit-PMU Claim 重建与 manifest | PASS | +| A5 shared G0 Claim | 1 call/核、96 calls、0 active、192 sentinel，schema-v6/HTML PASS | +| A5 shared G1 Claim | 5 calls/核、480 calls、4 active、188 sentinel，PASS | +| A5 private B1 Claim | 5 calls/核、480 calls、4 active、188 sentinel，schema-v5 PASS | +| PMU owner configure/restore/cleanup | 三次正式样本全部 PASS | + +G0 和 G1 的新证据位于： + +```text +outputs/shared_dynamic_submit_pmu_v6_20260726/ + g0_claim_icache_raw.json + g0_claim_icache_report.html + g1_claim_icache_raw.json + g1_claim_icache_report.html + private_g1_claim_after_g0_raw.json + private_g1_claim_after_g0_report.html +``` + +本阶段只改 host 校验和 host-only 定向测试，不改 device 代码、PMU 窗口 +或 raw schema。上述单轮 Submit 时间仍只作可执行性证据。 + +### 2026-07-26：S6.4g 闭合 shared 动态 task 的 A5 submit-PMU 矩阵 + +本小步不再修改设备协议，只用 S6.4e/f 已建立的 schema-v6 采集链验证 +动态计划的边界规模和五种 PMU 构建身份。所有运行都固定 device 0、 +单进程、单轮；目的仅是证明 host 独立计划、device 实际 replay、PMU +调用边界和输出结果一致，不能把下表中的单轮 Submit 时间当作正式性能 +比较。 + +#### Claim 动态规模矩阵 + +| 输入 | group | task/核 | Claim/核 | Claim/96 核 | compute kernel | active/sentinel tile | 结果 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| G0：`0` | 0 | 1 | 1 | 96 | 0 | 0/192 | PASS | +| G1：`8192` | 1 | 5 | 5 | 480 | 4 | 4/188 | PASS | +| G2 partial：`8193` | 2 | 9 | 9 | 864 | 8 | 8/184 | PASS | +| G2 full：`16384` | 2 | 9 | 9 | 864 | 8 | 8/184 | PASS | +| G4：`32768` | 4 | 17 | 17 | 1,632 | 16 | 16/176 | PASS | +| mixed：`0,8192,8193,32768` | 7 | 32 | 32 | 3,072 | 28 | 27/165 | PASS | + +mixed 的 28 个 compute kernel 与 27 个 active output tile 并不矛盾： +output tile 按 `(worker,task-kind)` 槽位保存，同一 worker 可以赢得多个 +同类 task，后一次结果复用该 worker 的同类输出槽。kernel 总数和每类 +`QK/SF/PV/UP=7` 由独立协议计数闭合；逐元素输出校验仍覆盖所有实际 +active 槽，其余 165 个槽保持 sentinel。 + +G2 partial 与 G2 full 刻意保留同一 `2 group/9 task` 拓扑。两者的 +canonical heap 分别为 829,440 和 1,603,584 bytes，证明完整 +`shared_context_lens` 身份确实参与 host 计划和 raw fingerprint，而 +不是只靠 task 数把两个输入误合并。G4 的 canonical heap 为 +3,196,928 bytes；上述三组的 heap cursor、vend、descriptor 覆盖和 +normalized writer projection 均分别闭合。 + +host 还用独立公式核对实际 fanin dependency signature：G0/G1/G2/G4/ +mixed 的预期值依次为 `0000000000000000`、`5cb454393ed48dcb`、 +`dda63f4f5405eaf1`、`58d7a4b63aac2c4e` 和 +`6437bff09d8f8a11`，所有运行的等值门槛均 PASS。该签名当前没有直接 +写进 raw；这里记录的是 host 执行过的期望值及等值断言，不能误述为从 +JSON 字段读取。 + +#### G2 partial 的五种 PMU 构建身份 + +| 编译阶段 | phase call/核 | phase call/96 核 | record | 语义/PMU/输出 | +| --- | ---: | ---: | ---: | --- | +| `none` | 0 | 0 | 96 | PASS | +| `claim` | 9 | 864 | 96 | PASS | +| `efdrain` | 9 | 864 | 96 | PASS | +| `materialize` | 9 | 864 | 96 | PASS | +| `register` | 9 | 864 | 96 | PASS | + +`none` 仍只保留完整 Submit 前后一次 PMU 开关，内部 phase 边界严格为 +零；另外四种构建在每个动态 task 上恰好进入一次对应边界，因此都是 +`9 × 96 = 864`。每份 raw 的 `phase_calls` 与 +`phase_expected_calls` 相等，96 条 record 都通过 physical owner、 +AIC/AIV triplet、begin/end、shadow lower-bound、counter headroom 和 +configure/restore/cleanup 校验。五种构建都得到 8 个有效输出和 +184 个 sentinel，说明观察阶段没有改变 G2 partial 的计算语义。 + +private 隔离对照在 shared G0/G1 运行前后各执行一次，均继续输出 +schema-v5、固定 5 call/核和 480 call/96 核；它不携带 +`tensormap_mode/shared_context_lens/shared_task_plan`，也没有被 shared +G0 的零 compute 规则放宽。 + +目录内 12 份 raw 都有对应 HTML，报告记录的 raw SHA-256 与实际文件 +逐份一致。 + +本阶段 A5 raw 与自包含 HTML 位于： + +```text +outputs/shared_dynamic_submit_pmu_v6_20260726/ + g0_claim_icache_{raw.json,report.html} + g1_claim_icache_{raw.json,report.html} + g2_partial_{none,claim,efdrain,materialize,register}_icache_{raw.json,report.html} + g2_full_claim_icache_{raw.json,report.html} + g4_claim_icache_{raw.json,report.html} + mixed_claim_icache_{raw.json,report.html} + private_g1_claim_icache_{raw.json,report.html} + private_g1_claim_after_g0_{raw.json,report.html} +``` + +至此，动态规模正确性不再依赖默认 G1 的“恰好五 task”巧合。下一阶段 +只生成真正的 shared B256 泳道：默认 256 个 8192-token batch 应恢复 +256 group、1,280 task，并要求 raw 无 dropped、converter/exclusive +按动态 task identity 闭合。该泳道只用于业务区域和 atomic 解释； +正式墙钟仍由无诊断的 perf-clock 构建单独采集。 + +### 2026-07-26：S6.4h 生成真正的 shared B256 泳道与无诊断基线 + +本阶段先从当前 HEAD 重新构建 shared `swimlane` 和 `perf-clock` 两套 +CCEC 制品，未复用早于 S6.4d～g 的旧 ELF。两套 manifest 分别固定 +`mode=shared`、`variant=swimlane/perf-clock`，运行入口在启动前重新核对 +所有制品 SHA-256。 + +#### B256 身份与协议闭合 + +正式泳道命令显式使用： + +```bash +PYTHON="$HOME/.venv/bin/python" \ +tests/atomic_probe/pa_scheduler/run.sh swimlane ccec \ + --tensormap shared --device 0 \ + --batches 256 --shared-context-lens 8192 \ + --winner-workload real-compute \ + --real-compute-counts 6,28,4,1 +``` + +运行头和独立 host plan 同时报告： + +```text +batches=256 +shared_groups=256 +tasks=1280 +kinds=Alloc:256,QK:256,SF:256,PV:256,UP:256 +``` + +这次不是依赖输出目录名推断规模。设备实际 replay、host 独立计划和离线 +analyzer 三层分别确认 1,280 个 task/核；exclusive analysis 还要求 96 +核的 task ID 都是同一个连续 `0..1279`，并恢复出 256 组 +`Alloc + QK + SF + PV + UP`。 + +关键门槛结果： + +| 项目 | 结果 | +| --- | --- | +| shared heap | vend/expected 均为 206,569,472 bytes，8 shard cursor 完全相等 | +| TensorMap symbol | published 2,048、INPUT load 1,280、INOUT writer commit 768 | +| dependency | 1,280 edges，signature `b7d985d6edb07078` | +| kernel | QK/SF/PV/UP 各 256，共 1,024 | +| 数值输出 | 192 active、0 sentinel，逐元素 PASS | +| trace | 832,263 raw records，expected 832,263，dropped 0 | +| atomic | 95,751 logical calls，90,247 physical records，轮询合批公式闭合 | +| offline | merged 1,200,808 个非 metadata 事件；exclusive validation 全项 PASS | + +merged 的 `traceEvents` 数组另含 256 条 `ph=M` metadata，因此数组总长 +是 1,201,064。这里显式区分 converter 报告的业务/诊断事件数与 JSON +数组长度，避免把两种口径误判为转换缺失。 + +exclusive 的 `capture.task_count_per_core=1280`、 +`task_ids_contiguous_and_equal_per_core=true` 和 +`validation.status=PASS` 是之后辨别 B256 的最低证据，不能再用 B1 +产物或仅凭目录名替代。 + +本次完整产物位于： + +```text +outputs/pa_scheduler_shared_swimlane_20260726_100857_3337556/ccec/ + l2_swimlane_records.json 约 53 MiB + merged_swimlane.json 约 93 MiB + swimlane_exclusive_analysis.json 约 115 KiB +``` + +#### 诊断泳道与 perf-clock 必须分开解释 + +泳道构建的 first-to-last Submit 是 4,854.651 us。它同时写普通阶段和 +atomic 记录，并生成 832,263 条设备记录，只用于解释业务区域、依赖和 +atomic，不作为权威性能基线。 + +perf-clock 构建编译期去除了泳道、atomic 观察和 PMU，只保留每核首个 +Submit 起点与最后一个 Submit 终点。相同 device 0、B256、G1 和 +`real-compute/6,28,4,1` 下，先丢弃两个独立进程预热 +`2392.876/2394.751 us`，随后六个独立正式进程为： + +```text +2382.540, 2380.577, 2389.149, 2381.226, 2356.894, 2389.874 us +``` + +正式样本最小值 2,356.894 us，中位数 2,381.883 us，均值 +2,380.043 us，最大值 2,389.874 us，样本标准差 12.028 us；六轮语义、 +输出、heap、symbol 和依赖门槛全部 PASS。当前 standalone shared PA +B256 的无诊断量级因此仍是约 2.38 ms，与此前约 2.36 ms 的判断一致。 + +单次泳道是 perf-clock 中位数的约 2.04 倍。这个差异只能说明完整诊断 +构建显著扰动热路径，不能把 2,472.768 us 逐项归因给某一种记录操作， +也不能拿两套 ELF 的绝对时间直接做候选收益。后续性能候选继续只由 +perf-clock 决定保留或撤销，泳道只解释收益落点。 + +### 2026-07-26：S6.4i 在 worker 启动前完成 shared heap 容量准入 + +S6.4a～h 已经允许 shared PA 按 `context_lens` 生成 G0～G4 动态 task +图，但设备端仍只有固定 256 MiB、8 shard、no-wrap heap。原实现只能等 +winner 在设备上执行 `FetchAdd` reserve 时发现单 shard 越界，再广播 +terminal fatal。该路径可以阻止越界写，却把一个在 host 已完全可知的 +容量错误推迟到 96 个 worker 启动之后。 + +本阶段没有放宽设备 allocator，也没有把资源判断塞回 task-plan builder。 +host 先用同一份权威 `SharedHostTaskPlan` 建图，再执行独立的 +`ValidateSharedHostHeapAdmission()`： + +```text +shard_span = floor(heap_size / 8 / 1024) * 1024 +reserve(task) = ceil(output_bytes / 1024) * 1024 +shard(task) = task_id % 8 +``` + +每个 task 只计一次 reservation，不乘 96 个 replay worker；逐 shard +累计值必须不超过 `shard_span`，总 reservation 必须不超过八个可用 +shard，且 heap、单次 reserve 和 aggregate vend 都不得越过设备 +`int64_t` atomic 的可表达范围。最终累计还必须等于 host plan 的 +`canonical_heap_bytes`。这样既能拒绝总量超限，也能拒绝“总量仍够、 +但 task 分布偏斜导致某一 shard 先满”的计划。 + +调用顺序固定为： + +```text +InitializeState +→ BuildSharedHostTaskPlan +→ ValidateSharedHostHeapAdmission +→ ConfigureTrace/PMU/workload +→ 启动 CPU threads 或 launch A5 kernel +``` + +CCEC 的 ACL 资源准备仍发生在 runner 外层，但容量失败不会启动 A5 +worker；CPU 容量失败不会创建任何 worker thread。设备端 no-wrap +reserve 校验继续保留为最终正确性防线，不依赖 host 永远无错。 + +#### 固定 256 MiB heap 的准入矩阵 + +| 输入 | B1 总 reservation | B1 最大 shard | B256 总 reservation | B256 最大 shard | B256 | +| --- | ---: | ---: | ---: | ---: | --- | +| G0 `0` | 10,240 | 10,240 | 2,621,440 | 327,680 | PASS | +| G1 `8192` | 806,912 | 524,288 | 206,569,472 | 25,821,184 | PASS | +| G2 partial `8193` | 829,440 | 524,288 | 212,336,640 | 26,542,080 | PASS | +| G2 full `16384` | 1,603,584 | 524,288 | 410,517,504 | 51,314,688 | REJECT | +| G4 `32768` | 3,196,928 | 1,048,576 | 818,413,568 | 102,301,696 | REJECT | + +单元门槛还覆盖： + +- mixed `0,8192,8193,32768` 的 4,843,520 bytes 和八个精确 shard + 累计； +- G1 B256 在恰好 206,569,472-byte heap 上通过、少 1 byte 时因 + shard span 向下对齐而拒绝； +- aggregate 仍小于 usable heap、但 task 0/8 共同压满 shard 0 的偏斜 + 计划必须拒绝； +- heap 大于 `INT64_MAX`、output 对齐加法溢出、非连续 task plan 均拒绝。 + +实际 CCEC shared perf-clock 重新构建后，A5 B1/G1 在 launch 前打印： + +```text +[HOST_HEAP_ADMISSION] batches=1 groups=1 tasks=5 +total_bytes=806912 max_shard_bytes=524288 +shard_capacity=33554432 status=PASS +``` + +随后 96 worker、5 task、4 kernel、8/5/3 symbol、5 条 dependency、 +806,912-byte vend 和真实计算结果全项 PASS。B256/G4 则在 task 1394、 +shard 2 首次超过容量时返回非零，`completed_runs=0`，没有 A5 +`HOST_PLAN`、worker 断言或 kernel 计数，证明拒绝发生在 device worker +启动前。 + +CPU 负向实跑曾暴露一个独立收尾缺陷：首轮在准入处退出时，汇总仍把 +空 `spans` 交给 `Median()`，导致准入错误之后再触发 SIGSEGV。现已让 +两种 CPU summary 显式处理 `completed_runs=0`，输出零耗时并干净返回 +`EXIT_FAILURE`；这不是用崩溃代替拒绝。 + +private CPU/CCEC 都重新构建并通过 B1。private CCEC host 中不存在 +`HOST_HEAP_ADMISSION` 和 shared reject 字符串，shared host 中两者均 +存在，说明新增准入仍被 `PTO_FDWIC_SHARED_MAP` 编译期隔离。用户已明确 +当前只维护 CPU 与 CCEC，因此本阶段不扩展 AscendC。 + +这项准入没有让 B256 G2 full/G4 “跑通”；它把固定 heap 的真实支持边界 +变成确定、可复核的 host 错误。若之后要求这两类 B256 输入运行，需要 +单独设计更大 heap、分批生命周期或 generation/reclaim 协议，不能通过 +删除设备端容量门槛解决。 + +### 2026-07-26：S6.4j 闭合 G2 放门后 Build 失败的全局收敛 + +此前已有两条分开的门槛： + +- non-final UP loser 必须等待 `deps_prepared`； +- 已经存在阻塞 slot 时，terminal fatal 会在 final barrier 后撤销执行 + 资格。 + +它们没有证明同一轮真实 96-worker scheduler 能闭合以下连续时序： + +```text +task4 winner 提交三个 accumulator writer intent +→ 发布 deps_prepared=4，放行另外 95 个 replay actor +→ 第二组 task8 完成 Build，并把三个 writer 都推进到 8 +→ task4 在 Build 前失败并广播 fatal +→ task8 因 fanin task4 未完成而禁止执行 +→ 96 worker 全部完成 final barrier 并清空在途 slot +``` + +G2 的 non-final UP 固定是 task4，final UP 是 task8。`UP` 没有 fresh +Output，因此这次故障必须准确命名为 post-gate Build failure，不能伪称 +fresh-output seal failure。 + +#### 仅测试构建可见的注入边界 + +公共 `FinishCallbackSubmitBody()` 在 +`shared_writers_prepared=true` 之后、`BuildWinner()` 之前增加一个受 +`PA_TEST_SHARED_POST_GATE_BUILD_FAILURE` 保护的 Ops hook。正式 CPU、 +CCEC、swimlane、perf-clock 和 submit-PMU 都不定义该宏，预处理后没有 +该调用和分支；宏只用于 `test_shared_loser_finish`。 + +故障 Ops 使用 thread-local split runtime 和真实 host atomic 语义启动 +32 AIC + 64 AIV。task4 hook 不会立刻失败，而是等待 Alloc task0 的三个 +`last_writer` 全部等于 8。必须三条都满足，不能只观察线性提交的第一条 +就提前广播 fatal。这个条件同时证明： + +1. task4 的 writer-ready 门已经可见； +2. 其他 actor 已进入第二组； +3. 唯一 task8 winner 已建立 slot； +4. task8 的 Build 后 writer commit 已完整结束。 + +随后 hook 对 task4 返回失败，由真实 `SetFatal`、replay break、分层 final +barrier、`DiscardSharedSlotsAfterReplayFatal()` 完成收敛。hook 自身有 +2 秒 wall-clock 取证上限，整个测试进程另由 15 秒 `timeout` 兜底；超时 +仍注入 fatal，但最终 oracle 必须失败，不能用“能退出”掩盖 task8 未到达。 + +#### 故障 oracle + +完整门槛要求： + +- 96 个 pthread 全部 join，startup=96，two-16 final barrier 精确闭合； +- hook 恰好一次，fault worker 必须是 AIV；该 worker 在 task4 失败前 + `submits=4`、split finish calls 等于该核已经取得的 Claim winner 数、 + task-id sum=10； +- `fatal=1`、`deps_prepared[4]=4`，但 task4/task8 的 flag 和 vend 都为 + 0，`deps_prepared[8]` 仍为 -1； +- Alloc 三个 writer 都等于 8，而 task8 自身无 Output 的 publication/ + writer 控制字全部保持 -1； +- 96 核四个 slot 全部 `occupied=false/built=false`、每核 + `occupied_count=0`； +- 全局不存在 task4 诊断 slot，锁定失败发生在 `BuildWinner()` 之前； +- 恰好保留一个已撤销执行资格的 task8 诊断 slot，function 为 UP， + fanin 精确为 `{6,7,4}`； +- UP kernel 数严格为 0；QK/SF/PV 的完成 flag 分别与各自 kernel 数 + 相等，全部 placement 之和等于 kernel 总数，completion duplicate 为 + 0。 + +shared loser 后续已经改为 caller 内轻量返回，不再跨 TU 进入完整 Finish。 +因此 split 尾检把两条事实分开证明：`task_id_sum` 必须覆盖该核实际重放的 +完整 `0..N-1` 序列；`finish_calls` 只等于该核 Claim winner 数。零 winner +worker 即使完成了全部逻辑 Submit，`finish_state_address` 也应保持 0； +只要至少赢过一次，caller/finish 仍必须指向同一 thread-local runtime。 +cookie、owner、task sum 和 reserved 门槛均未放宽。 + +#### 实测 + +定向二进制连续运行 30 次，30/30 通过。稳定终态为： + +```text +hook=1 +task8_seen=1 +writers=8,8,8 +kernels=2,2,2,0 +placements=6 +task4_slot=0 +task8_slot=1/1 +slots_clear=1 +split_protocol_errors=0 +``` + +QK/SF/PV 各 2 次属于允许执行的独立上游工作，不依赖 task4；UP 为 0 才是 +本故障的关键禁止条件。 + +测试宏关闭后，普通 CPU shared G2 partial `context_len=8193` 仍得到 9 +task、8 kernel、13/10/6 symbol、dependency signature +`dda63f4f5405eaf1`、829,440-byte heap 和真实计算结果全项 PASS。重新 +构建的 CCEC shared perf-clock 在 A5 上得到同一 G2 身份和全部语义门槛 +PASS，单次 first-to-last Submit 为 137.186 us。该数值只用于确认正式 +构建没有被故障 hook 污染，不作为新的性能基线。private CCEC perf-clock +也重新构建，并在 A5 上以 B1 real-compute 通过 96 worker、5 task、 +四类 kernel 各 1 次及全部输出校验。 + +最后对 private/shared 的 CCEC split-finish 翻译单元、swimlane 与 +perf-clock 产物做预处理文本和符号审计，故障宏及 hook 命中均为 0； +`PA_TEST_SHARED_POST_GATE_BUILD_FAILURE` 的唯一构建定义仍是 CPU host +self-test。由此把“测试注入能闭合 G2 故障”和“正式 CCEC/private 不带 +注入逻辑”作为两条独立证据闭合。 + +### 2026-07-26：S6.5 冻结 standalone PA 迁移基线 + +S6.4j 之后重新审查第 8、12 章、standalone 当前实现和迁移 review。 +按“真实 PA shared TensorMap 的关键调度依赖”这一既定范围，当前没有 +新的 standalone 正确性阻塞项，可以进入真实 simpler。这里的“可以迁移” +只表示 PA 的 symbol/manual-dependency 路径已经闭合，不能扩大为通用 +ordinary-region shared TensorMap 已经完成。 + +#### 迁移前已闭合的 PA 边界 + +- private/shared 构建身份、产物目录、manifest、host/device ABI 和 + CCEC 测试 hook 隔离均已闭合； +- G0～G4 动态 task plan 由 device replay、独立 host oracle 和离线工具 + 共同消费；G2 的 task4/task8 身份不再依赖 `%5` 猜测； +- non-final UP 先提交三个 accumulator writer intent，再发布 + `deps_prepared` 放行 loser；final UP 在 Build 后提交最终 writer； +- G2 task8 的 fanin 精确为 `{6,7,4}`，三个 accumulator writer 最终均为 + 8； +- task4 放门后、Build 前失败时，task8 不执行，96 worker 完成 final + barrier 并清空在途 slot； +- shared fresh output 使用稳定 `(task_id, output_slot)` 引用，只有 + winner 分配和发布 descriptor； +- 8-shard no-wrap heap 在 worker 启动前做独立准入；B256 G1 和 G2 + partial 可运行，固定 256 MiB 下的 G2 full/G4 明确拒绝。 + +ordinary-region 多版本/generation、长期 task-id 复用、heap wrap/reclaim、 +BlockWon/MIX、108-worker 泛化和任意多 writer 链继续保持后置。它们不是 +当前 PA Case1/G2 的隐式完成项，也不能为了“功能看起来更多”混入第一轮 +真实路径迁移。 + +#### 同口径 20 轮 perf-clock + +在提交 `08d40dd5` 上，private/shared 都重新构建 CCEC `perf-clock`。 +固定 device 0、B256、32 AIC + 64 AIV、two-16 和 +`real-compute 6,28,4,1`；每种模式先丢弃一个 warm-up，再运行 20 个独立 +正式进程。40/40 都闭合 1,280 task、1,024 kernel、依赖签名 +`b7d985d6edb07078`、TensorMap 投影签名 `556bec7ec8d0f323` 和真实计算 +输出。 + +| 模式 | 最小值 | 中位数 | 均值 | 最大值 | 样本标准差 | +| --- | ---: | ---: | ---: | ---: | ---: | +| private | 3,648.869 us | 4,821.200 us | 4,857.463 us | 6,599.653 us | 785.201 us | +| shared | 2,362.384 us | 2,384.792 us | 2,383.710 us | 2,416.547 us | 12.713 us | + +按中位数，shared 比 private 少 2,436.408 us，缩短 50.535%。private 的 +3,648.869 us 是通过全部语义门槛的真实原始样本，但它只是 20 轮中的低 +离群值；private 还出现 6,599.653 us，高低范围接近 2.95 ms。因此当前 +private 代表值仍采用 4.821 ms 中位数，不能用单轮最低值宣称性能收益。 +shared 20 轮范围只有约 54 us，本次对比中稳定性明显更好。 + +#### 两份真正的 B256 泳道 + +随后分别用 `swimlane` 独立诊断构建采集 private/shared。两份分析均确认 +96 核、每核连续 1,280 task、1,024 kernel、`dropped=0` 和 +`validation.status=PASS`： + +| 模式 | raw 记录 | 合并事件 | 诊断 Submit | +| --- | ---: | ---: | ---: | +| private | 841,451 | 1,209,996 | 5,258.531 us | +| shared | 832,617 | 1,201,162 | 4,759.560 us | + +诊断 Submit 只用于确认泳道身份和大致布局；两种泳道 ELF 都包含普通阶段 +及 atomic 记录，不能与 `perf-clock` 相减，也不能代替上述 20 轮性能结论。 + +加工结果和完整样本记录位于： + +```text +tests/atomic_probe/pa_scheduler/test_record/2026-07-26/ +``` + +其中 `private/shared/merged_swimlane.json` 可直接载入 Perfetto，各自的 +`swimlane_exclusive_analysis.json` 保存排他闭合结果。两份 raw 继续留在 +ignored `outputs/`，没有重复复制到 `test_record`。 + +#### 真实 simpler 的后续顺序 + +迁移继续沿用已经建立的 `PTO_FDWIC_SHARED_MAP`、三镜像身份和 fail-closed +机制，不另造模式开关。低风险到高风险顺序固定为： + +1. 先把真实默认 private TensorMap 同构为 ring-per-bucket，证明默认路径 + 行为、ABI 和性能没有回退； +2. 在同一 region/hash/overlap 语义上增加唯一 shared ring 和真实 CCEC + 缓存发布纪律； +3. 在现有 compete-first Finish 中接入 non-final INOUT writer gate； +4. 新增真实 G2 `context_len=8193` 门槛，重新证明 task4→task8 和终止收敛； +5. shared backend 通过 CPU/CCEC 门槛后才解除 fail-closed； +6. 最后再用真实 PA B1/B256 配对数据决定是否迁移 winner-only + materialize、shared symbol/heap 等 standalone 性能机制。 + +standalone 的故障宏、split runtime、固定 96-worker 测试状态、host thread +yield 和模拟负载都不得进入 production。迁移的是已经证明的协议和 oracle， +不是复制测试脚手架。 + +### 2026-07-26：S6.6 按真实 replay 形态收敛 shared loser，并复核 INOUT region intent + +S6.5 冻结时，shared 的 96 个 replay actor 仍会进入同一份完整 +`FinishCallbackSubmitBody()`；loser 虽然不执行 Materialize 的主体，却仍 +穿过完整函数外壳和矩形观察边界。这既不符合参考实现的轻量 loser 返回形态， +也会让泳道和局部 PMU 把不存在的 shared loser 重活当成业务阶段。本阶段先 +修正 standalone，不迁移真实 simpler。 + +#### 不能照抄参考 PA 的 region-intent 缺口 + +参考分支已经提供 +`rt_presubmit_*_with_region_intent()` 和 +`dist_presubmit_task_with_region_intent_impl()`，其通用协议允许 winner +先登记覆盖 writer、发布 `deps_prepared`，loser 等门后返回。但是参考分支 +真实 +`examples/a5/fully_distributed_within_core/paged_attention_unroll` +在 `FUNC_ONLINE_UPDATE` 的预提交处仍调用普通 +`rt_presubmit_aiv_task()`，没有调用 `_with_region_intent`。因此只能复用 +它已经证明的协议机制，不能把“API 存在”脑补成“PA 的跨组 INOUT 已接线”。 + +PA 的约束按业务语义固定为： + +```text +每组最后一个 UP 是 accumulator 的 INOUT writer + ├─ 后面还有组：winner 先登记新 writer 并发布 writer-ready + │ loser 等门后才返回并构造下一组 + └─ 已是最后一组：没有后继构造者，不增加 writer-ready 门 +``` + +standalone 使用动态 task plan 的 `has_following_group` 表达这一事实，不能 +用 `task_id % 5` 或固定 B1 任务数推断。winner 先完成当前 UP 的 fanin 和 +registration 校验，再由 `CommitPaSharedWriterIntentAfterFanin()` 将三个 +accumulator 的 `last_writer` 更新为本 UP，最后发布 +`deps_prepared=task_id`。loser 的 +`FinishSharedLoserSubmit()` 只在该位为真时等待门，且不读取 `TaskArgs`； +下一组最终 UP 的 fanin 因而能观察上一组 UP，而不是退回最初 Alloc。 +non-final UP 在 Build 后跳过重复 writer commit；final UP 没有门，仍在 +Build 成功后提交最终 writer。 + +两个互补定向 oracle 锁定这条协议： + +- G2 `context_len=8193` 的全 loser 回放中，task4 是 non-final INOUT， + task8 是 final INOUT；loser 在 `deps_prepared[4] == -1` 时不能返回， + 发布为 4 后才能继续，而 `deps_prepared[8]` 始终为 -1。传入的上一任务 + `TaskArgs` 页设为 `PROT_NONE`，完整重放仍通过，证明轻路径没有偷读 + winner-only 参数； +- G4 `context_len=32768` 的显式 winner 链逐组验证 + task4/8/12/16 的 accumulator fanin 分别来自 + Alloc/task4/task8/task12；task4/8/12 依次发布门，final task16 不发布； +- G4 中间 task8 在放门后、Build 前注入 fatal。hook 等到 task12 和 + task16 都已建立依赖 slot 后才失败；两者的 fanin 分别为 + `{10,11,8}`、`{14,15,12}`，但 task8/12/16 都不能执行，96 worker + 最终清空所有在途 slot。 + +这条门表示“下一任 writer 身份已经登记”，不表示对应 UP kernel 已执行或 +完成。后继 slot 仍通过 producer completion flag 保证执行依赖；不能把 +`deps_prepared` 冒充 completion。 + +当前证据也不扩大为三项未完成承诺：fatal 前已经独立就绪的后组 +QK/SF/PV 允许按 DAG 语义执行，只保证依赖失败 UP 的后继不执行；G4 +故障注入是 CPU 并发门槛，A5 目前只有当前 G2 和此前 G4 正常路径的发布 +可见性证据；损坏 gate 的重复发布仍使用 Exchange，可能先短暂放行 +waiter 再广播 fatal， +但 completion flag 会阻止其错误执行。若以后要求“非法 gate 绝不放行” +的更强异常态契约，应单独验证 CAS 发布，不能和本次热路径迁移混做。 + +#### shared replay 与稀疏观察的最终形态 + +shared 每个 actor 仍按相同 task plan 调用每次逻辑 Submit，以便 loser +取得稳定 `(task_id, output_slot)` 返回值并继续构造后续参数。但实际阶段 +改为： + +| actor / task | EfDrain | Claim | Submit 父区间 | Materialize | Fanin | Register | Build/Complete | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| loser | 1 | 1 | 1 | 0 | 0 | 0 | 0 | +| Alloc winner | 1 | 1 | 1 | 1 | 0 | 1 | 1 | +| 非 Alloc winner | 1 | 1 | 1 | 1 | 1 | 1 | 1 | + +Alloc 暂时仍由所有 actor 构造三个静态输出参数，以保持参考 +`alloc_tensors(args)` 的特殊调用形状；其 output symbol 不依赖这次构参。 +其他重任务只有 Claim winner 构造 descriptor/scalar 参数并跨 TU Finish。 +shared PA 没有 ordinary-region `PrepareMap`，因此删除原来的零时长假 +marker；private 继续保持原矩形协议和 eager 参数构造。 + +split runtime 也按真实职责拆开： + +- caller 对每个逻辑 task 累加 `task_id_sum`，证明全量 replay； +- 只有 winner arm ticket 并进入 noinline Finish， + `finish_calls == claim_wins`； +- 零 winner worker 的 `finish_state_address` 合法保持 0；赢过任务的 worker + 仍要求 caller/finish 地址完全一致。 + +raw 不增加逐事件字段。现有 `Submit` 父区间覆盖轻量 loser 返回和 +non-final UP 等门，普通 child 只记录真实执行阶段；文件级 metadata 只 +增加一次 `tensormap_mode`。converter/analyzer 按 schema v4 和模式分别 +验证：shared 禁止 `PrepareMap`、loser 禁止重阶段、winner 要求对应稀疏 +子区间；private 仍执行原契约。旧 raw 不做兼容，因为当前采集与加工是一体 +版本。 + +submit-PMU 同步采用相同边界。Claim/EfDrain 每核调用数等于 replay task +数；Materialize/Register 每核调用数等于本核 winner 数，允许为 0，但 +全局调用和必须等于 task 数。零调用记录必须满足 +`phase_elapsed_ticks=0`，同时仍是可信的完整 Submit PMU 记录。host 导出 +和运行期校验现在共用同一 `SubmitPmuPhaseTimeValid()` 判定，避免“终端 +打印 96/96 可信、JSON 却把零 winner 核标成不可信”的双口径。 + +#### 本阶段证据 + +- CPU shared 全套测试通过；G4 task8 post-gate 故障门槛额外连续运行 + 5 次均通过，稳定得到 + `deps=4:4,8:8,12:12,16:-1`、 + `flags(up4/up8/up12/up16)=1/0/0/0`、writers=`16,16,16`， + task12/task16 诊断 slot 均为 `1/1`，`slots_clear=1`； +- Python converter/analyzer/PMU 共 105 项通过； +- CCEC shared G1 泳道为 2,714/2,714 条，G2 为 4,110/4,110 条， + 两者 `dropped=0`，依赖签名分别为 + `5cb454393ed48dcb`、`dda63f4f5405eaf1`； +- private CCEC B1 重新构建并上板通过，4,149/4,149 条、 + `dropped=0`，证明 shared 稀疏分支没有污染 private； +- shared G2 的五个 submit-PMU 变体全部生成 raw 和 HTML：none 为 0 次 + 局部调用，Claim/EfDrain 各为 `9 × 96 = 864` 次， + Materialize/Register 各为 9 次全局唯一 winner 调用；五份均为 + 96/96 记录可信，Materialize/Register 各有 87 个零 winner 核且仍可信。 + 一次实测中 Materialize 为 + 4,166 request / 279 miss / 31,497 ticks，Register 为 + 147 request / 11 miss / 310 ticks。这些只证明稀疏取数合同闭合， + 不作为 B256 性能结论。 + +#### B256 配对性能 + +旧基线使用干净 detached worktree 的精确提交 `47d22e3f`，当前候选使用 +本节源码；两者重新独立构建 shared `perf-clock`。固定 CANN 9.1、用户 +`.venv`、device 0、B256、`real-compute 6,28,4,1`、two-16，关闭 PMU 和 +泳道。每版先预热 2 次，再做 6 个 ABBA/BAAB 四进程区组，A/B 各 12 个 +正式样本。 + +| 版本 | 最小值 | 中位数 | 最大值 | +| --- | ---: | ---: | ---: | +| `47d22e3f` | 2,358.457 us | 2,380.110 us | 2,400.307 us | +| 当前候选 | 2,314.435 us | 2,338.860 us | 2,367.994 us | + +六个区组的 `candidate - baseline` 依次为 +`-26.438/-45.852/-30.611/-35.463/-49.106/-47.787 us`，6/6 更快； +配对中位数为 `-40.658 us/-1.703%`，整体中位数差为 +`-41.251 us/-1.733%`。24/24 次都闭合 1,280 task、1,024 kernel、 +依赖签名 `b7d985d6edb07078`、96 active worker、`RingBp=0`、fatal +clear 和真实计算输出。 + +候选 `.text` 比基线增加 2,560B，二者仍都只有两个 GLOBAL device FUNC； +因此不能把收益解释成代码体积或符号数下降。当前最直接且与源码单变量一致的 +解释是：shared loser 不再进入完整 Finish 和 winner-only 重阶段外壳。 +这组数据达到 6/6 同向且超过 0.2% 的预设保留门槛，可以保留；它仍不能外推 +为真实 simpler 的同等收益，真实路径迁移后必须重新做同口径配对。 + +### 2026-07-26：R1a 先抽取真实 TensorMap facade，并冻结 private 机器码 + +R0 已经完成真实 private/shared 构建身份、缓存隔离和三镜像 ABI 防混用。 +进入数据结构迁移前，重新对照第 12 章、当前 production、standalone 和参考 +分支，发现不能把原计划中的“抽 facade”和“把 private 换成 128×128 ring” +混在一个提交里： + +- 当前 production private 是 8,192 个 hash bucket 共享一个 16,384-entry + pool；某一个 bucket 理论上可以占用接近整个 pool； +- standalone private 是 128 bucket × 128 slot，只有当前 PA Case1 证明 + `H=64` 时最坏同桶存活量不超过 128； +- production 允许 `PTO_DIST_H` 取 0～1,022，且一个 task 可以登记多个 + INOUT/OUTPUT_EXISTING。对任意 callable，单桶存活量可能明显超过 128， + 不能用 PA 特例替换全 runtime 容量合同； +- 参考分支没有把 private map 换成 ring；其 shared ordinary-region 也是 + 永久 append 的有锁链表，没有 head/tail、绝对 seq、generation 或 + reclaim，不能冒充第 12 章要求的有界 ring。 + +因此 R1 拆成两个可独立取证的小步。R1a 只建立模式无关 facade，逐句保留 +旧 private 算法；R1b 才会在明确 production 容量、溢出和 host-visible +错误传播之后更换存储。 + +#### facade 边界 + +原 `aicore/tensor_map.h` 拆为： + +```text +private_tensor_map.h 既有 linked private backend +tensor_map.h Submit 和 scalar data access 唯一可见的 facade +``` + +facade 固定四类上下文： + +1. `reset_worker(worker)`：初始化当前模式的 worker/backend； +2. `prepare_task(worker, task_id, H)`：按当前 task 推进回收； +3. `lookup_for_task(worker, tensor, consumer_task_id)`：shared 后端必须用 + consumer id 过滤未来 producer； +4. `insert_for_task(worker, tensor, producer_task_id, task_won)`：shared + 后端只能让唯一 winner 发布。 + +`core_state.h`、Submit 的 PrepareMap/Fanin/Register 和 +`tensor_data_access.h` 已全部改走 facade；生产 aicore 目录中只有 +`private_tensor_map.h` 和 facade 本身还能直接访问 private helper 或 +`worker.map`。当前 shared 镜像仍在零 Submit 前 fail-closed，因此只为保证 +三镜像完整编译而实例化 private backend，绝不把它解释为 shared 可运行。 + +旧 private 在 16,384-entry pool 耗尽时会静默丢弃 insert；R1a 有意不顺带 +改变这条历史语义，facade 的 bool 暂时固定返回 true。R1b 必须把容量失败 +从 backend 一路传到 Submit fatal，并最终变成 AICPU/Host 可见的非零结果； +在此之前不能把“增加一个返回值”伪称成错误传播已闭合。 + +#### 等价性证据 + +| 检查 | 结果 | +| --- | --- | +| production private retire 差分测试 | 3/3 PASS | +| 构建/cache/mode 相关 Python 单测 | 159 PASS；12 个 integration build 用例按本阶段范围排除 | +| A5sim private/shared 三镜像 | 均构建通过 | +| A5 CANN 9.1 private/shared Host、inner AICPU、AICore | 均构建通过 | +| A5sim private PA Case1 | golden PASS；使用 example-exec-time，只作功能门槛 | +| `git diff --check` | PASS | + +另外在 detached `9fc3681b` 上重新构建旧 private CCEC AICore，与 R1a 候选 +逐字节比较执行节： + +| ELF section | 大小 | 基线 SHA256 | 候选 SHA256 | +| --- | ---: | --- | --- | +| `.text` | 188,184 B | `6d5b7526cbb634f0c0220668fc25d1e3b66d102f4e36c4ccd9a042957dc671f0` | 相同 | +| `.rodata` | 220 B | `6d0b106fffc043986f3225b6e08ce09edb66b75419cd02c41b57174e309699bc` | 相同 | + +因此 R1a 不只是“源码看起来等价”：当前 private CCEC 热路径和只读数据均未 +改变。该阶段不需要跑 A5 性能，也不提升 build ABI/layout version；下一提交 +只处理 private ring 的 production 容量与错误合同，不能同时接 shared +ordinary-region、fresh symbol 或 PA region intent。 + +### 2026-07-26：S-R1 固化连续分桶环的结构边界与多 CAP 门槛 + +进入 production R1b 前，先重新核对第 12 章所说的 +`ring-per-bucket` 与 standalone 已有 `128 bucket × 128 slot`。结论是： + +- 二者不是两种数据结构。后者正是前者在 `B=128、CAP=128` 下的一组 + 具体参数； +- 规范要求的是“每个 bucket 拥有自己的连续槽段、独立单调 + `head/tail`，槽只按该桶的绝对 cursor 回绕”，而不是“一个全局 16K + 时序环再用 bucket 链把离散槽串起来”； +- 全局时序环即使总容量也是 16K，桶内仍是离散追链，不能继承连续扫描、 + 无 `next`、局部回收和 shared 无 ABA 空闲链这些设计收益； +- 因此连续 ring-per-bucket 是第 12 章明确要求的最终方向；128×128 只是 + 当前 PA 已验证的默认值，不是通用 runtime 的容量结论。 + +#### 本阶段刻意只证明结构，不冒充 `auto` + +第 12 章的 `--tensormap-ring-cap auto` 要扫描静态任务图，并按 private +的 `H` 或 shared 的 `Δ+H` 计算各桶滑动窗口峰值。standalone 当前没有 +这套 planner，真实 Host 也没有在 launch 前持有一份可直接枚举的完整动态 +任务图。因此本阶段没有新增一个“看起来像 auto”的经验公式，也没有把 +运行时 GM 配置读取塞进 Submit 热路径。 + +当前实现只增加构建期隔离参数 +`PTO_FDWIC_TENSORMAP_RING_CAP`，正式 CPU/CCEC scheduler 显式固定为 +128；private/shared ring 自测则用同一份生产 helper 依次重编译： + +| CAP | bucket 数 | 总槽数 | 主要门槛意义 | +| ---: | ---: | ---: | --- | +| 32 | 512 | 16,384 | 暴露小容量 off-by-one、额外 bucket 游标和满环路径 | +| 64 | 256 | 16,384 | 覆盖第二种额外游标布局，锁定默认/扩展区分界 | +| 128 | 128 | 16,384 | 正式 standalone 默认 ABI | +| 256 | 64 | 16,384 | 排除 helper 偷写 128 | +| 16,384 | 1 | 16,384 | 单桶覆盖完整旧 pool，并验证 B=1 hash 无右移 64 UB | + +这里固定的是总槽池 16K，CAP 与 bucket 数互为反比。CAP 越大，单桶容量越 +宽但 hash 桶越少、lookup 扫描可能越长；CAP 越小，桶更多但单桶更容易满。 +这组变体用于验证结构和错误路径，不能直接当成性能推荐。 + +private 的 `TensorMap` 在五个 CAP 变体下都保持 823,312B，`WorkerState` 仍为 +9,231,296B。默认前 128 桶的 head/tail 与全部后续字段 offset 原样保留； +CAP=32/64 多出的 384/128 组游标从原 32KiB ABI padding 中切出，并相应 +缩短剩余 padding。访问统一经过内联 bucket-control helper，默认 +CAP=128 编译分支没有额外桶判断。shared 的正式 CAP=128 sidecar size 和 +region/output/heap/vector offset 继续由原硬断言锁定；其他 CAP 只用于 +隔离算法门槛。 + +非默认 CAP 编码进 `kBuildIdentityAbiVersion`；默认 128 保留历史 ABI +值 4/6，避免只为身份元数据让 AIC/AIV 入口多一条大立即数构造并改变后续 +代码对齐。artifact manifest 升为 v2，并新增 +`tensormap_ring_cap=128`。CCEC host、AIC、AIV 必须用同一 CAP 编译， +运行时 identity、manifest schema/CAP 和整套产物 SHA 共同阻止“三镜像按 +不同桶布局解释 GM”的混件。当前没有对外暴露非 128 的正式运行产物，也 +没有声称完成第 12 章的免重编译覆盖参数。 + +#### 新增的容量与复用门槛 + +原测试已经覆盖半开区间、最大 producer、窗口回收、三圈绝对 seq、双检 +ABA、满环不覆写和整 task 预检失败不部分发布。本阶段补齐两个此前缺失的 +正向边界： + +1. shared 显式门槛中，A 桶已满时，只向独立 B 桶追加仍必须成功；A 的 + head/tail 和逻辑内容保持不变，证明容量是 per-bucket，不是误加的全局 + live gate。B=1 变体明确不执行这条不存在的命题； +2. 桶满后精确退休 `K=min(8,CAP/4)` 个旧 entry，再由同一 task 追加 K 个 + 新 entry 必须成功。head 前进 K、tail 前进 K、live 仍等于 CAP，物理 + slot 0..K-1 发布新的绝对 seq `CAP..CAP+K-1`。 + +private 的长程 wrap 用例把“必须持续成功”的窗口限制为 +`min(64,CAP-1)`；配置过小导致的显式满环由独立 overflow 用例验证。这样 +不会把 CAP=32 在 H=64 下必然不足误判成 ring 算法错误。 + +#### 当前证据 + +| 检查 | 结果 | +| --- | --- | +| private ring CAP=32/64/128/256/16384 | 5/5 PASS | +| shared ordered ring CAP=32/64/128/256/16384 | 5/5 PASS | +| CPU private 完整构建与全部门槛 | PASS，约 14.4s | +| CPU shared 完整构建与全部门槛 | PASS，约 30.0s | +| CCEC private/shared 默认 CAP=128 三镜像 | 均 PASS | +| CCEC manifest v2 mode/CAP/variant/phase/SHA | 两模式均闭合 | +| A5 B1 private/shared 默认 CAP=128 scalar-nop=0 smoke | 两模式全部语义断言 PASS | +| standalone converter/analyzer/PMU Python 回归 | 122 PASS（用户 `.venv`） | +| shell `bash -n` / `git diff --check` | PASS | + +A5 smoke 只验证 mode/CAP 身份从 manifest 到 Host、AIC、AIV 的运行闭环， +不作为性能数据。CAP=32 通过隔离测试也只表示“能正确运行或明确报满”，不 +表示它已被证明足以覆盖 PA；当前 PA Case1 的保守全 map live 上界仍是 +52，正式默认 128 满足该上界。任意任务图、production 的 +`PTO_DIST_H=0..1022` 和 shared `Δ+H` 都必须另做静态逐桶容量证明。 + +默认 private CCEC swimlane linked device ELF 另与干净 `c4c4e4c2` +重编产物做执行节逐字节比较： + +| ELF section | 大小 | 基线 / 候选 SHA256 | +| --- | ---: | --- | +| `.text` | 590,904 B | `49b50da75c356cf2e5d9f2da9ceb5d38d3e3003b446e1eaa604806cd16be1438` | +| `.rodata` | 696 B | `e75f6281546f2d140d86c1b143bce48ebab6e0952ed3051a3b81ce98a6c027b9` | + +两节均完全相同。也就是说，在这个同变体对照中，构建期参数化、 +bucket-control facade 和 CAP=16384 的 B=1 特判没有污染正式 private +128×128 swimlane device 机器码；manifest 升级发生在 host artifact +身份层，不靠向 AICore 热代码增加 marker。这一结论不外推到尚未逐节比较 +的其他 variant 或 Host 产物。 + +为回答默认 PA B256 是否发生性能变化，又从干净 `c4c4e4c2` 和当前 +tracked 修改快照分别重编同一 CCEC perf-clock variant。这里比较的仍只是 +最终 mixed device ELF 的装载执行节，不把 manifest v2 导致的 Host +artifact 变化混入热路径结论： + +| 模式 | section | 大小 | 基线 / 候选 SHA256 | +| --- | --- | ---: | --- | +| private | `.text` | 126,264 B | `a016e132247c32e36efcecb7597cfffe462f4d8a409e39b50497f7adb8d074ed` | +| private | `.rodata` | 300 B | `7c1caafcef85e369058da215f1094b36c74c5a7e5d8103f2454be5b3a4c91598` | +| shared | `.text` | 149,560 B | `d9717beb8dfd7b5a0ce2907bed00f8119e1139f4f57fa9c3784d7ecda5450f40` | +| shared | `.rodata` | 304 B | `695626dcc092dcc7e5179f28045e261c3e5a6f51c2b0277b7b2c65ef2ca72b65` | + +四组 section 的基线与候选均逐字节相同，因此本轮改动没有给默认 CAP=128 +的 private/shared perf-clock 热路径增加指令或常量。A5 B256 真计算的 +当前快速复核也全部通过语义门槛： + +- shared 三个独立进程为 `2,354.757 / 2,327.094 / 2,330.510 us`， + 中位数 `2,330.510 us`；相对改动前最近 12 样本中位数 + `2,338.860 us` 只差 `-0.357%`，未见回退； +- private 三个独立进程为 `6,118.217 / 5,357.404 / 4,505.527 us`， + 全部落在既有 20 样本 `3,648.869..6,599.653 us` 波动区间内。private + 竞争本来就有较大抖动，三样本中位数不能解释为回退或收益。 + +性能样本只用于确认没有出现新异常；“本轮未改变默认热路径代码生成”的 +直接证据是同 variant 的 `.text/.rodata` 相同，不是跨批次时间相减。 + +#### 对 production R1b 的约束 + +production 下一步不能把 128×128 数值无脑移植过去，也不能退回全局环加 +bucket 链。需要先闭合三件事： + +1. 明确真实任务图可获得的静态 region 集合、`H/Δ` 上界，以及 + manual CAP 小于可证峰值时的启动拒绝合同； +2. private 满环从 backend bool 一路传播到 Submit fatal、AICPU/Host + 非零结果；shared 还要区分可恢复反压与不可恢复容量错误； +3. 在默认产物上先证明容量布局与错误传播，再接 shared 的 per-slot seq、 + 发布/失效和有序 reclaim；不能把 fresh symbol、heap、INOUT gate 同时 + 混入这一个数据结构提交。 + +### 2026-07-26：R1b-a 先闭合 TensorMap 容量失败合同 + +在更换 production private 存储前，先处理旧 backend 已存在但被静默吞掉的 +`kMapCap=16384` 耗尽。这个小步不改变 linked/free-list 数据结构，也不把 +standalone 的 128×128 参数搬进 production；它只建立 private/shared +有界 ring 都必须遵守的失败合同： + +1. backend insert 返回 `bool`，facade 原样传播，满池时不改 TensorMap； +2. Register 任一 `OUTPUT_EXISTING/INOUT` 插入失败后，当前 task 在 + WinnerBuild 和 slot 发布前返回； +3. 失败核把既有 `local_index` 置为 `kFlagCap`。后续 Begin 复用现成 + task-cap 门禁，在 Claim 前返回 `ready=0`；不新增 DistCore 字段，也不在 + 正常 Submit 增加一次 GM latch 读取； +4. 失败冷分支持续把 post-increment 后的 `local_index` 钳回哨兵，避免错误 + orchestration 继续 Submit 时最终发生整数回绕； +5. 首个非零错误码用 A5 `atomicCAS` 发布到原 fatal cacheline 的 padding， + 再发布 fatal。AICPU 等全部 worker 完成并失效该 cacheline 后读取错误， + 所有 AICPU 线程统一返回 `-11`； +6. fatal 运行不再进入 FinalDrain，因为容量失败后的任务图已经没有完整依赖 + 闭包，继续等待只能造成挂死。 + +失败分支故意不闭合 Submit/perf-clock/submit-PMU 外层。这样即使合法图恰有 +65,536 个 Submit，失败 raw 也会被现有完整性门禁拒绝，不能混入性能基线； +已经闭合的 Register span 仍保留真实失败 task id，足够定位。 + +`DistGlobal::error_code` 复用 fatal 所在 cacheline 的 padding，未移动后续 +字段，但三镜像必须对这个布局语义达成一致，因此 +`kFdwicDistGlobalLayoutVersion` 从 1 升到 2。shared backend 仍保持零 +Submit fail-closed；本阶段只是让未来 bounded shared ring 有一条真实错误 +出口，不能解释为 shared 已经可运行。 + +#### 本阶段证据 + +| 检查 | 结果 | +| --- | --- | +| backend 最后一槽、满池无写入、retire 后复用 | PASS | +| facade 容量失败原样传播 | PASS | +| production CPU-sim Register→失败→无 Build→后续不 Claim | PASS | +| 首个错误码获胜，后续错误不覆盖容量码 | PASS | +| private/shared × A5sim/A5 三镜像构建 | 4/4 PASS | +| A5sim private PA Case1 B256 正常路径 | PASS | +| `git diff --check` | PASS | + +容量门槛 UT 直接编译 production `aicore/dist_engine.cpp`，没有复制一份 +Submit 模型。场景让 map 只剩一个物理 entry，再提交两个 +`OUTPUT_EXISTING`：第一个占用末槽、第二个失败；逐字节确认所有 task slot +未变，`occupied_count/owned_total` 未增加，随后一次 compete-first Begin +不推进 vector claim cursor。 + +本阶段仍未实现 `auto CAP`。现有 Host/AICPU 在 worker 启动前没有一份可直接 +枚举的完整 FDWIC 任务图，不能用 `8×H` 或 PA 的 52-entry 特例冒充第 12 章 +的逐桶滑窗峰值。`auto` 的可证规划接口与 private ring 迁移继续作为后续 +独立阶段。 + +### 2026-07-26：R1b-b 冻结 H 配置与精确 auto 的可实现边界 + +`H` 不是普通调试参数。private 用它推进 region retire，shared 最终需要 +`Δ+H` 作为存活窗口；错误的 H 会直接改变容量、回收和依赖正确性。旧代码用 +`strtol(e, nullptr, 10)`，会把 `"abc"` 当 0、接受尾随字符，并可能在长整型 +到 `int32_t` 的窄化后才触发断言。因此本阶段把它改成启动门槛： + +- 只接受非空 ASCII 十进制数字串； +- 闭区间固定为 `[0, kTaskWindow-2]`，当前即 `[0,1022]`； +- 拒绝空白、正负号、尾随字符、溢出和越界，输出值在失败时保持不变； +- AICPU 在修改共享 arena、配置 PMU或唤醒 AICore worker 前返回结构化 + `-12`；同一 run 的所有 AICPU 线程取得相同状态。 + +A5sim 负向 PA Case1 已实际得到： + +```text +[dist_engine] invalid PTO_DIST_H='abc'; expected ASCII decimal digits in [0, 1022] +run_prepared failed with code -12 +``` + +四个 AICPU 线程都返回 `rc=-12`，且没有进入 AICore replay。正常默认 H 的 +同一 PA Case1 B256 仍通过。 + +#### 精确 auto CAP 不能从当前 Host 状态直接算出 + +逐层核对 production 后，当前 Host/AICPU 启动前只持有 callable 元数据、 +orchestration SO 和本轮参数，没有一份可枚举的 FDWIC task/output manifest。 +真实 task id、OUTPUT 地址和 register 事件是在 96 核执行 orchestration replay +时才形成；现有 dep-gen 又是运行结束后的离线记录，不能用于启动准入。因此： + +1. 短期只能支持显式 manual CAP，并在不足时按 R1b-a 合同明确失败；不能发布 + 一个经验公式命名为 `auto`； +2. PA 这类只读取已搬运外部输入、任务图确定的 orchestration，可以在 AICPU + 已启动但 worker 尚未唤醒时，对同一 AArch64 orchestration SO 做一次 + “只规划、不执行 kernel”的回放； +3. planner 与执行必须复用同一 OUTPUT 布局和实际 TensorMap register-event + helper。fresh OUTPUT 当前并不等价于一次 map insert，机械统计全部逻辑 + output 会高估且与运行时不对等； +4. 如果 orchestration 读取本轮 kernel-produced 数据决定后续图，或直接写 + 外部 GM 产生副作用，prepass 不能安全重放，应要求 manual CAP 或显式图 + manifest，不能猜； +5. shared 在计算 `Δ+H` 前还必须有硬 runahead 上界和 + `core_progress[]`。同时 region 地址或 bucket key 必须由 producer task id + 确定；若用无序 winner 的 heap fetch-add 决定物理地址，竞态会改变 bucket， + planner 就无法证明精确峰值。 + +若 CAP 还要反过来决定本轮实际 arena 分配大小，AICPU prepass 已经太晚，只能 +再增加 Host-native planner 或两阶段 launch。当前下一步先迁移 private +ring，并保持 16K 预分配上限和明确溢出合同；planner 不与数据结构提交混做。 + +### 2026-07-26：R1b-c 将 production CAP 纳入三镜像构建身份 + +private 链表即将换成连续分桶 ring；此后 `CAP` 不只是调参值，还决定 +`bucket_count=16384/CAP`、哈希结果、槽下标及 bucket control 的解释方式。 +Host、AICPU、AICore 若使用不同 CAP，即使三份二进制都能单独加载，也会把 +同一块 GM 解释成不同的 TensorMap。因而先于 backend 迁移完成以下闭环： + +1. `PTO_FDWIC_TENSORMAP_RING_CAP` 成为显式编译定义，当前 production + 仍固定为已验证的 128，没有新增一个未经证明的运行期 `auto`； +2. RuntimeBuilder 对 Host/AICPU/AICore 统一注入 + `mode + CAP`，isolated AICore 构建禁止额外定义覆盖二者； +3. mode 继续由 private/shared 独立 artifact 目录隔离，完整编译定义继续 + 进入 source fingerprint。当前 CAP 不可配置，因此不会产生同目录下的 + 两种合法 CAP；将来若开放 CAP，必须再把它加入 artifact path 和 scene + cache key，不能只放开宏； +4. orchestration 与三镜像使用同一个 Python CAP 常量，Submit-PMU + provenance 同时封存 CAP，拒绝把不同布局的诊断 ELF 与 raw/HTML 绑定； +5. 设备侧 build identity 升为 ABI v2，并比较 + `magic/abi/mode/CAP/runtime_bytes/dist-layout` 全部字段。 + +#### 保留旧错误位 offset，而不是机械地在中间插字段 + +第一次草稿把 CAP 插到 `tensor_map_mode` 后面，导致 +`runtime_bytes/dist_layout/error_bits` 整体后移。只看“同版本三镜像”测试 +不会暴露问题，但新旧 AICore 混件时，失败方会按自己的结构布局写 +`error_bits`，AICPU 可能从另一个 offset 读取，从而漏掉 +`AicoreMismatch`。 + +最终布局把 CAP 放到 v1 的首个 reserved 字： + +| 字段 | 固定 offset | +| --- | ---: | +| `runtime_bytes` | 16 | +| `dist_global_layout_version` | 20 | +| `error_bits` | 24 | +| `tensor_map_ring_cap` | 28 | + +四个 offset 与 64B cacheline 大小均由编译期断言和 production Runtime UT +锁定。这样旧镜像即使不理解 CAP，也仍会在双方都认识的 offset 24 发布 +mismatch；ABI 版本差异负责拒绝继续执行。 + +#### 本阶段证据 + +| 检查 | 结果 | +| --- | --- | +| Python RuntimeBuilder/scene/provenance 非集成回归 | 371 PASS，12 个真实构建参数用例按范围排除 | +| private/shared build identity C++ UT | 2/2 PASS | +| private/shared × A5sim/A5 三镜像重建 | 4/4 PASS | +| A5sim private PA Case1 B256 正常路径 | PASS | +| Submit-PMU 直接脚本入口（无 `PYTHONPATH`） | `--help` PASS | +| Ruff / `git diff --check` | PASS | + +完整 Python 组合另得到 378 PASS、5 FAIL；5 个失败全部发生在既有 A2/A3 +真实构建的 `PTO2TaskPayload` 结构断言（实际 568、规范 576），与本次只作用 +于 A5 FDWIC 的 mode/CAP 单测无关，不能把它们记录成本阶段通过。 + +本阶段没有改 `DistTensorMap`、Submit 或任何设备热路径。下一提交先增加一组 +不依赖 linked/ring 内部布局的逻辑 reference 门槛，再在独立提交中替换 +private backend；不能把 shared 的 `seq`、有序 tail、全局 reclaim 或 +region-indent 混入 private 存储迁移。 + +### 2026-07-26：R1b-d 先冻结 linked/ring 共同逻辑语义 + +直接替换 private 存储后，如果测试同时从“链指针断言”改成“ring 游标断言”， +很容易让实现和测试一起改变而失去差分依据。因此本阶段只扩展 production +CPU-sim UT，不改任何 runtime 文件。新增 `LogicalReferenceMap` 只保存： + +- 独立计算的 `{buffer, byte_lo, byte_hi}` 半开区间； +- producer id； +- 单调 `alive_floor`。 + +reference 不调用 production 的 element-size、byte-range、hash、retire、 +free-list 或 lookup helper，也不描述 bucket/CAP。它只回答共同的外部语义： +相邻半开区间不重叠、不同 buffer 不相关、所有重叠版本中返回最大 producer、 +`producer < N-H` 才退休而 `producer == N-H` 仍存活。 + +四组门槛分别覆盖： + +1. 空表、半开区间首尾相接、真实重叠和不同 buffer； +2. producer 3/5/7 按真实单调顺序登记，lookup 必须返回 7，不能依赖物理 + 遍历方向返回第一个命中； +3. `N=20,H=10` 的精确边界、重复 floor 和倒退 floor 幂等； +4. 固定种子连续 12,000 task、`H=15`，每步比较刚插入、窗口内历史项和 + 随机 query，跨越十余轮 1024-task 窗口复用。 + +长程 workload 任意时刻最多只有 16 个全局存活 entry，小于计划支持的最小 +单桶 CAP=32。因此该用例中的 insert 失败只能表示实现错误，不会把未来 +per-bucket 容量差异误判为 linked/ring 语义差异。现有六个 linked 专属测试 +暂时原样保留；backend 真正迁移时再删除 free-list/next/prev 的内部断言， +新增四个逻辑测试的 reference 算法、场景和断言必须原样继续通过。ring 的 +lookup 会惰性推进物理 head，因此实际 map 参数可由 `const` 机械调整为 +可变引用；这不能被解释成允许修改 reference 口径。 + +本阶段 `test_fdwic_tensor_map_retire` 共 10 项全部通过，其中新增 12,000-task +差分约 41 ms；`git diff --check` 通过。下一阶段先替换 `DistTensorMap` +存储和 private helper，再把容量集成测试改成真实填满目标 bucket，不能在 +同一提交改变现有“多 output 前缀可已登记、失败 task 不 Build”的合同。 + +### 2026-07-26：R1b-e 将 production private TensorMap 迁移为连续分桶环 + +本阶段只替换 private TensorMap 的物理存储与回收方式，没有提前混入 shared +的跨核原子、`seq`、可见性操作或全局回收协议。这样可以先在单写者语义下证明 +环本身正确，再把后续 shared 的问题收敛到并发发布层。 + +#### 物理布局与 ABI + +production 继续保留固定的 16,384 个 `MapEntry` 物理槽，但它不是“一个全局 +时序环”。槽池按编译期 `CAP` 均匀切成连续的 per-bucket ring： + +```text +bucket_count = 16384 / CAP +slot(bucket, cursor) = bucket * CAP + (cursor & (CAP - 1)) +``` + +默认 `CAP=128`，即 128 个 bucket、每桶 128 个连续槽。CAP=32/64 时需要的 +额外 bucket head/tail 从旧 32 KiB bucket-control 区内部切出；CAP=128/256/ +16384 则只使用固定的前 128 组游标。五种 CAP 下均保持： + +- `sizeof(MapEntry) == 48`； +- `sizeof(DistTensorMap) == 823,312`； +- `sizeof(DistCore) == 9,231,296`； +- `DistCore::map/slots/occupied_count/owned_total/swimlane_last_cycle/task_payloads` + 的精确 offset 不变。 + +旧链表的 `bucket/next/prev/task-next` 字段改为 ABI reserve，旧 task-head 与 +free-list 控制区也只保留物理位置，private 热路径不再读写它们。虽然总尺寸和 +后续 offset 没变，同一批字节的解释已经不兼容，因此 build ABI 与 DistGlobal +layout identity 都从 v2 升到 v3；旧、新三镜像不能混用。 + +#### private 算法与生命周期合同 + +private `PrepareMap(N,H)` 现在只单调推进 +`alive_floor=max(alive_floor,N-H)`，不再扫描 task-head。lookup/insert 只在 +实际触达的 bucket 上从 head 开始惰性退休 +`producer < alive_floor` 的前缀；insert 在确认 +`tail-head flush payload -> 发布 seq -> 发布 tail`；本阶段只冻结这套 +地址关系，尚未加入任何 AICore DCCI/atomic 热路径。 + +默认 CAP128 sidecar 与 standalone S2.5 前缀完全同构： + +| 字段 | offset | 大小 | +| --- | ---: | ---: | +| `committed_tasks` | 0 | 64B | +| `reclaim_upto` | 64 | 64B | +| `buckets[128]` | 128 | 16,384B | +| `slots[16,384]` | 16,512 | 2,097,152B | +| sidecar 合计 | 0 | 2,113,664B | + +每个 bucket 的 `head` 与 `tail` 各占一行且相邻；每个 slot 的物理下标仍为： + +```text +bucket * CAP + (absolute_cursor & (CAP - 1)) +``` + +第一版采用 task-id exact-turn 的有序单追加者，不引入通用 MPSC `reserve`、 +桶锁、全局 free-list 或 `core_progress[]`。task N winner 完成 N 的 lookup +后，才在 `committed_tasks == N` 的轮次内 append 并推进 commit；loser 不读 +shared ring。因此 reclaim 可以直接使用 `max(-1,N-H-1)`。这与早期 +standalone S2 的异步 reader 方案不同：后者曾经需要 `core_progress[]`，S2.5 +收紧访问纪律后已经把它删除，不能把历史过程态带回 production。 + +#### 尾部追加、arena 与构建身份 + +提交 `68f51451` 冻结的 private `DistGlobal` 精确大小为 +`1,007,026,048B`。shared sidecar 只在 `final_barrier` 后、offset +`1,007,026,048` 追加： + +| artifact | `sizeof(DistGlobal)` | +| --- | ---: | +| private | 1,007,026,048B | +| shared CAP128 | 1,009,139,712B | + +固定 arena 仍为 `0x42000000 = 1,107,296,256B`，shared 默认布局剩余 +`98,156,544B`，不需要扩大 arena。private artifact 不实例化这 2MiB +sidecar，所有旧热字段、`DistCore` 和 per-core private map offset 保持原值。 + +虽然 private 物理布局没有移动，shared 的尾部解释已经改变，Host、AICPU、 +AIC 和 AIV 必须作为同一家族重新生成。因此： + +- build ABI 从 v3 升到 v4； +- `DistGlobal` layout identity 从 v3 升到 v4； +- mode 与 CAP128 继续进入三镜像身份和 artifact cache key； +- shared 的 `kFdwicCompiledBackendReady` 仍为 false，AICPU/AICore 都在零 + Submit 前拒绝运行，不能静默落到 private 语义。 + +#### AICPU 初始化边界 + +新增 reset helper 在未来真实 `dist_engine_register()` 路径中一次性设置： + +- `committed_tasks = 0`； +- `reclaim_upto = -1`； +- 所有 bucket `head = tail = 0`； +- 所有 16,384 个 slot `seq = -1`； +- payload 不清零。 + +`seq=-1` 是唯一无效哨兵；不能依赖 arena 零填充，因为首圈 slot 0 的合法 +absolute seq 正好是 0。worker reset 不得并发清空全局单副本。AICPU 随后 +沿既有路径 flush 整个 `dist_global` arena，再唤醒 worker。 + +这里必须严格限定证据：当前 shared backend-ready 门在调用 +`dist_engine_register()` **之前**就会中止，所以 shared A5 负向运行不会实际 +执行这段 reset。现阶段只能证明 helper 的行为、production AICPU 接线可 +编译，以及它位于未来 register setup 路径；正式解除门禁前还要增加 +`dist_engine_register()` 正向集成测试，不能宣称真实 shared A5 已完成初始化。 + +#### 本阶段证据 + +| 检查 | 结果 | +| --- | --- | +| private CAP128 + shared CAP32/64/128/256/16384 精确布局/reset | 6/6 PASS | +| 所有相关 FDWIC C++ 门槛 | 16/16 PASS | +| GCC15 ASAN+UBSAN 同一组 FDWIC 门槛 | 16/16 PASS，无报告 | +| private Host / inner AICPU / AIC / AIV / final AICore | 全部编译通过 | +| shared Host / inner AICPU / AIC / AIV / final AICore | 全部编译通过 | +| shared before-Submit 门禁 | 保持生效 | +| shared A5 正向初始化/Submit | 本阶段未执行，门禁仍关闭 | + +production 纯编译使用本用户 CANN 9.1、`.venv` 与 GCC15，在独立目录同时生成 +private/shared artifact。实际编译命令确认三类镜像分别收到相同的 +`PTO_FDWIC_SHARED_MAP=0/1` 与 +`PTO_FDWIC_TENSORMAP_RING_CAP=128`，AIC/AIV 分别使用 +`dav-c310-cube`/`dav-c310-vec`。这证明 sidecar 类型与 AICPU 初始化入口不是 +只在 host UT 中成立，但不替代后续真实跨核可见性验证。 + +参考分支中值得保留的是“共享控制字独占 cache line、AICPU 唤醒前建立哨兵、 +payload writeback 后再发布可见字”；没有移植其 64K +`high_water + bucket head + next` append-only 链，因为它没有 reclaim/seq， +也不是第 12 章要求的连续 ring-per-bucket。 + +下一提交只实现独立 `shared_tensor_map.h`：时序窗口 lookup、整 task +preflight、连续 append、absolute seq 双检和 exact-turn reclaim。它仍不接 +Submit、不解除门禁；算法门槛闭合后再单独接 winner-only task publish。 + +### 2026-07-26：S2a 抽取两种 backend 共用的逻辑原语 + +在移植 shared 算法前，先把与副本所有权、atomic 和 cache 可见性完全无关的 +四项逻辑从 private 实现中抽到 `tensor_map_common.h`： + +- 同一乘法高位 hash； +- Tensor 到 `[buf_addr,lo,hi)` byte range 的转换； +- `bucket * CAP + (cursor & (CAP-1))` 连续分桶下标； +- 同 buffer 半开区间重叠判断。 + +private 保留原有 `dist_private_tensor_map_*` API 作为零成本 inline wrapper， +且 lookup 继续保留原条件表达式，没有为了“形式共用”改写已经冻结的默认热 +路径。shared 下一阶段直接使用 common helper，但不会包含或复用 +`DistTensorMap`、private head/tail、alive floor 等状态。 + +五种 CAP 的 private ring、12,000-task 逻辑差分和 Submit capacity 共 6 项 +全部通过。更关键的是，使用同一 CANN/GCC15 重新编译 production private +AIC/AIV 后： + +| 入口 | S1 `.text` | S2a `.text` | 内容 SHA256 | +| --- | ---: | ---: | --- | +| AIC | `0x17188` | `0x17188` | 相同：`a8eae234f72f...` | +| AIV | `0x17518` | `0x17518` | 相同：`5dbfda402594...` | + +完整 `.o` 因源码路径/调试元数据而 hash 不同，故这里比较 ELF 声明的 +`.text` 原始字节，而不是用整文件 hash 冒充热路径证据。结果证明该抽取没有 +改变 private AIC/AIV 指令内容；后续 shared/private 的依赖差异也不会来自两份 +逐渐漂移的 hash/range 算法。 + +### 2026-07-26：S2b 独立实现 task-id 有序的 shared TensorMap 环原语 + +本阶段新增 `shared_tensor_map.h`，但仍然**不从 facade/Submit 调用**， +`kFdwicCompiledBackendReady` 继续让 shared artifact 在零 Submit 前明确 +退出。这样只验证共享环自身的状态机，不把 winner 选择、fatal 收敛或 PA +region-intent 混进同一提交。 + +#### 有序单追加者合同 + +本轮沿用 standalone S2.5 已闭合的 exact-turn 方案，而不是第 12 章早期的 +通用 MPSC `reserve`： + +```text +task N 的唯一 winner： + committed_tasks 必须恰好等于 N + -> 完成 N 的全部 lookup + -> reclaim_upto = max(-1, N-H-1) + -> 对本 task 全部 entry 做整批 preflight + -> 逐 entry 发布 payload/seq/tail + -> committed_tasks = N+1 +``` + +落后 actor 看到 `committed_tasks>N` 属于陈旧调用并报协议错误；未来 actor +看到 ` invalidate payload -> 本地 snapshot -> seq` 双检， +两次 absolute seq 必须都等于 cursor；字段还必须满足 +`producer>=0`、`reserved==0` 和 `lo invalidate payload line +-> 写 32B 逻辑值 +-> flush 独占 payload line +-> 发布 absolute seq +-> 推进该桶 tail +``` + +全部 entry 成功后才发布 task commit。首圈目标槽旧 seq 必须为 `-1`，后续 +lap 必须为 `cursor-CAP`；这使固定物理槽复用时仍能识别 ABA。该阶段提交时 +先与 standalone 一致采用 Exchange，并通过返回旧值检测 exact-turn 合同； +它留下的“写入后才发现旧值不符”问题已由紧随其后的 S2c 改成 +`expected_old -> WRITING -> absolute seq` 两段 CAS。在 S2c 闭合前没有 +接入多核 Submit。 + +#### 审查补出的边界 + +初版 CPU UT 和 production TU 的 include 顺序掩盖了头文件对 `state.h` 的 +真实依赖；最终测试先只引入平台 `inner_kernel.h` 和 +`shared_tensor_map.h`，再引入 AICPU reset 头，锁定 shared header 自身可解析。 + +另外 lookup 与所有写入口都统一拒绝 `N>=kFlagCap`，且任何环修改前完成 +该检查。写 cursor 必须严格小于 `INT64_MAX`，因为 +`cursor==INT64_MAX` 虽可表示，随后发布 `tail=cursor+1` 会发生有符号 +溢出；read-only cursor 仍允许等于该上界。 + +容量测试不只覆盖“完全没有回收的满桶”： + +1. 满桶后只安全回收 producer 0，当前 task 仍要向同桶追加两个 entry； + 结果允许 `reclaim/head` 前进，但所有 slot、tail 和 commit 不变； +2. 满桶后精确回收 K 个旧槽，并由同一 task 向同桶追加 K 个 replacement， + 验证 earlier-entry 计数和 absolute seq 的成功边界； +3. 逆序到达的 future actor 对整张 map 零修改，轮次到达后按 task id + 逐一且仅一次提交；重复到达的 stale actor 是协议错误而不是永久 Pending。 + +#### 本阶段证据 + +| 检查 | 结果 | +| --- | --- | +| CAP32/64/128/256/16384 shared ring UT | 5/5 PASS，共 13 类协议门槛 | +| 所有相关 FDWIC C++ 目标 | 21/21 PASS | +| GCC15 ASAN+UBSAN 同一组 FDWIC 目标 | 21/21 PASS，无报告 | +| 固定种子 12,000-task lookup/reference 差分 | 五种 CAP 全部 PASS | +| 修正后 shared CCEC + 显式 wrapper probe | 五种 CAP 的 AIC/AIV/final 全部链接通过 | +| private production AIC `.text` | 94,600B，SHA256 `a8eae234f72f...`，与 S1 逐字节相同 | +| private production AIV `.text` | 95,512B，SHA256 `5dbfda402594...`，与 S1 逐字节相同 | +| `git diff --check` | PASS | + +CCEC probe 显式实例化 +`lookup_region/lookup_tensor/refresh_reclaim/check_task_append/` +`append_prepared_task/publish_commit`，而不是只让 production TU 解析未使用 +的模板。五档证据位于： + +```text +/tmp/fdwic-s2-shared-final-cap-matrix-20260726/ +``` + +该目录的 combined/final 尺寸包含仓库外 probe，只能证明 CCEC +primitive/template 可实例化和链接，不能作为正常 production ELF 的代码体积 +或性能数据。private 指令对比则使用： + +```text +/tmp/fdwic-s2-private-final-text-compare-20260726/ +``` + +比较对象是 ELF `.text` 原始字节，不是会受源码路径和调试元数据影响的完整 +`.o` hash。 + +尝试构建 tests/ut/cpp 的全体无关目标时，仍会命中已记录的 A2/A3 +`PTO2TaskPayload` 结构断言：实际 offset 568、规范 576；本阶段精确构建并 +运行的 21 个 A5 FDWIC 目标全部通过，不能把全仓失败隐去，也不能把既有 +A2/A3 问题归因于本次 shared 实现。 + +当前尚未证明 A5 多核跨 cache 可见性，也没有证明 Submit winner/fatal/drain +收敛；CPU 事件账本只证明调用顺序，CCEC 只证明目标指令可以生成。下一阶段 +先用 CAS 消除状态迁移失败时的瞬态覆写，再接入 task-level +prepare/append/commit，继续保持 shared 顶层门禁。 + +### 2026-07-26：S2c 用 CAS 闭合 shared 控制字失败不覆写 + +S2b 为了直接对齐 standalone 正确性基线，所有条件状态迁移先采用 +Exchange 后检查旧值；其中 task commit 失败还会再次 Exchange 恢复。这个 +写法在正常 exact-turn 单 writer 路径不会失败，但一旦出现非法双 writer、 +状态损坏或故障注入，会先把错误值短暂发布给其他核，再发现合同不符。commit +的恢复 Exchange 还可能覆盖真正的并发值。 + +本阶段仍不接 Submit、不解除 shared 门禁，只把以下六类条件迁移改成现有 +production `atomic_compare_exchange()`： + +| 状态 | CAS | +| --- | --- | +| bucket head | `original_head -> retired_head` | +| global reclaim | `current -> candidate` | +| slot ownership | `expected_old_lap -> WRITING` | +| slot publish | `WRITING -> absolute_cursor` | +| bucket tail | `tail -> tail+1` | +| task commit | `N -> N+1` | + +CAS wrapper 返回操作时观察到的旧值，成功条件统一为 +`observed==expected`；不能把返回值当 bool。commit 失败后的恢复写已删除， +所有 mismatch 都不 retry、不 rollback。 + +#### 为什么必须增加 WRITING，而不是机械地 CAS(-1,-1) + +首圈 slot 的 `expected_old` 本来就是 `-1`。若把旧 Exchange 机械替换为： + +```text +CAS(expected_old=-1, desired=-1) +``` + +状态没有改变，两个非法 writer 都可能判断“ownership 成功”。最终增加一个 +不占额外字节的 seq 状态： + +```text +INVALID = -1 +WRITING = INT64_MIN +valid = 非负 absolute cursor +``` + +writer 必须先从旧 lap seq CAS 到 WRITING，才允许 invalidate/write/flush +payload；reader 只接受与目标 cursor 完全相等的非负 seq，因此不会消费 +WRITING。若本轮异常中止，下一 run 的 AICPU reset 会把全部 valid/WRITING/ +脏 seq 和 head/tail/commit/reclaim 一次性恢复到初态；payload 无需清零， +因为 reset 后没有任何 seq 使旧 payload 可达。 + +WRITING 只扩展已有 `sequence.v` 的值域，没有改变 sidecar 的 offset、size、 +alignment 或 CAP，因此本阶段保持 build ABI v4 和 layout v4。shared 当前 +仍在 `dist_engine_register()` 前 fail-closed，新旧算法混件都不可能进入写 +路径；等真实 Submit 接线并解除门禁时，再把 build ABI 一次提升到 v5，冻结 +可运行协议。layout 只有在物理布局变化时才提升，不能把协议版本和布局版本 +混为一谈。 + +#### CAS 不是整 task 事务：失败边界必须分层 + +CAS 只保证**发生 mismatch 的那个控制字不被本 writer 覆写**，不能让多 +slot、多 bucket append 获得自动回滚能力： + +1. reclaim/head/slot ownership CAS 失败发生在当前 entry payload 写之前。 + 目标字保留注入/竞争值，当前 entry 不 invalidate、不 flush、不推进 + seq/tail/commit；此前已证明安全的其他 head/reclaim 单调推进仍可保留。 +2. `WRITING -> cursor` 失败时，payload 已写并 flush，但 tail/commit 未推进。 +3. tail CAS 失败时，该 entry 的 payload 和 seq 已发布，task commit 未推进。 +4. task commit CAS 失败时，全部 entry 的 payload/seq/tail 已发布。 + +后三类在 exact-turn 单 writer 合同下都不是合法竞争，而是不可恢复的协议 +破坏。接入 Submit 后必须立即 fatal：不 Build 当前 task、不发布 completion、 +不允许后继越过 commit gate，所有 wait/drain 观察 fatal 后退出，AICPU 返回 +非零；下一 run 由冷启动 reset 清理现场。禁止把 seq 恢复成旧 lap、回退 +tail/head 或恢复 commit——payload 已被覆盖时回写旧 seq 会复活错误数据， +制造真正的 ABA。 + +CPU 故障门槛在每个 CAS 的线性化点前写入一个同时不同于 expected/desired +的竞争值，再执行 strong CAS，并同时断言： + +- 注入点确实命中； +- event 中 `observed` 等于竞争值； +- 目标字最终仍等于竞争值，而不是 desired； +- payload 前失败没有 invalidate/flush； +- payload 后失败只保留上述明确的部分发布边界； +- commit 失败只出现一次 CAS，不再有恢复性第二写。 + +另外单独覆盖“另一非法 writer 已持有 WRITING”的 stale-preflight 情形， +以及 production `DistSharedTensorMapAicoreOps` 的真实 CAS mismatch；后者把 +commit 预置为 2，再尝试 `CAS(0,1)`，最终必须仍为 2，可防适配器将来误退化 +成 Exchange。AICPU 重复 reset 同时覆盖 valid seq 和残留 WRITING。 + +#### A5 顺序与性能尚待真机取证 + +本机 CANN 9.1 的 CCEC 头声明并能编译 GM `atomicCAS`；production +wrapper 也已在 AIC/AIV 显式实例化。但 CCEC 实现会忽略 C++ 的 +`__ATOMIC_ACQUIRE/RELEASE` 参数，且 CANN 注释说明部分 CAS 形态可能由编译 +pass 降为软件实现。因此当前证据只说明接口、返回类型和目标代码生成成立， +不能提前声称“一定是一条硬件指令”或给出延迟结论。 + +payload 的跨核发布仍依赖已有： + +```text +payload DCCI CACHELINE_OUT -> DSB -> seq CAS -> tail CAS +``` + +而“其他核观察到 commit=N+1 后，是否必然已观察到 seq/tail”尚不能只凭 C++ +memory-order 参数证明。接入 Submit 前需要一个 A5 双核 litmus，重复验证 +writer 的 `payload/seq/tail -> commit` 与 reader 的 +`commit -> tail/seq/payload` 可见顺序；若不成立，再在 commit 前增加经过 +真机验证的设备级顺序边界，不能靠臆想接口补 barrier。 + +正常 append 的原子次数没有增加：原来的 Exchange 被一一替换为 CAS， +WRITING 复用原本的“置无效 seq”那次原子。CAS 相对 Exchange 的实际延迟仍 +需在接线后的 A5 perf-clock 中单独测量，不能用 CPU 时间推断。 + +#### 本阶段当前证据 + +| 检查 | 结果 | +| --- | --- | +| shared ring CAS/WRITING 协议门槛 | 16 类，五种 CAP 全部 PASS | +| 所有相关 FDWIC C++ 目标 | 21/21 PASS | +| GCC15 ASAN+UBSAN 同一组 FDWIC 目标 | 21/21 PASS，无报告 | +| AICPU reset：valid seq + WRITING + 非零控制字 | 五种 CAP 全部 PASS | +| production concrete CAS 成功/失败路径 | PASS | +| shared CAP32/64/128/256/16384 production CCEC | AIC/AIV/combined/final 全部 PASS | +| int64 CAS 目标对象证据 | AIC/AIV 均有 CAS/control 差分，wrapper 实例进入 final | +| private production AIC/AIV `.text` | 94,600/95,512B，与 S1 逐字节相同 | +| `git diff --check` | PASS | + +shared CCEC 证据位于： + +```text +/tmp/fdwic-s2c-shared-cas-cap-matrix-20260726/ +``` + +每档 probe 显式调用六个 concrete wrapper，AIC/AIV 对象和 final 都保留 +probe 符号。CAP128 另有同 flags、同 include 的最小 CAS/control 对象： +CAS 对象包含 `atomic_compare_exchange` 与 +`atomicCAS<(ST_L2CacheType)0>` 实例链，control 对象不含；当前 +`llvm-objdump` 对 `elf64-hiipu` 只能识别 section/符号而不能解码助记符， +所以没有把这个差分夸大成“已证明单条硬件 CAS”。 + +private 指令对比位于： + +```text +/tmp/fdwic-s2c-private-text-compare-20260726/ +``` + +AIC `.text` 为 94,600B、SHA256 `a8eae234f72f...`，AIV 为 95,512B、 +SHA256 `5dbfda402594...`，均与 S1 冻结产物逐字节相同。完整 `.o` 仍不用于 +热路径判定。真实 A5 CAS 顺序 litmus 属于接线前门槛，不会用当前纯编译结果 +替代。 + +### 2026-07-26：S2d 区分协议拒绝与不可恢复的部分发布 + +S2c 的 CAS 已经保证失败 writer 不会覆写竞争控制字，但 +`append_prepared_task()` 和 `publish_commit()` 仍只返回 `bool`。这个口径 +无法告诉 Submit：失败发生在本调用写任何当前 task payload 之前，还是已经 +写入一个或多个 slot 之后。两种情况都必须停止当前 task，但后者的 shared +sidecar 已经处于只能由下一轮 AICPU reset 清理的部分发布状态，不能重试或 +回滚。前者只说明本调用没有发布当前 task entry，不证明整个 sidecar 干净： +竞争 writer 仍可能已经破坏共享控制字，因此也必须 fatal/reset。 + +本阶段仍不接 Submit、不解除 shared 门禁，只增加两层精确结果： + +| 层次 | 结果 | 含义 | +| --- | --- | --- | +| 单 entry | `Published` | payload、absolute seq 和 tail 均已发布 | +| 单 entry | `ProtocolError` | slot ownership 前拒绝，当前 entry 未写 payload | +| 单 entry | `PartialPublish` | ownership 后的 seq 或 tail 发布失败 | +| 整 task | `Committed` | 所有 entry 和 `N -> N+1` commit 完成 | +| 整 task | `CapacityBlocked` | 整批 preflight 容量不足，未发布当前 task | +| 整 task | `ProtocolError` | 未发布任何当前 task entry 即拒绝 | +| 整 task | `PartialPublish` | 至少一个 entry 已发布，或非空 task commit 失败 | + +`dist_shared_tensor_map_publish_task_impl()` 的固定顺序为： + +```text +参数与全部 entry 只读校验 + -> refresh reclaim + -> 整 task preflight 并冻结每个 entry 的 planned cursor + -> 逐 entry 校验 tail==planned cursor 后 ownership/payload/seq/tail + -> CAS committed_tasks: N -> N+1 +``` + +第一个 entry 的 ownership CAS 失败属于纯协议拒绝；第二个及以后 entry +即使在 ownership 前失败，由于更早 entry 已经发布，也必须上报 +`PartialPublish`。零 entry task 的 commit CAS 失败没有 slot 副作用，仍是 +`ProtocolError`；非空 task 的同一失败则是 `PartialPublish`。任何部分发布 +都保留原现场，不写回旧 seq、不回退 tail、不恢复 commit。 + +preflight 与 publish 之间也不能重新按“当前 tail”选槽。否则两个非法 +same-task writer 都通过旧 preflight 后，后到者可能跟随先到者推进后的 +tail，转而发布一个从未预检的新槽，甚至错误 commit。现在 preflight 为每个 +entry 保存固定 cursor；publish 发现 tail 漂移时在 ownership 前拒绝。新增 +门槛显式在 preflight 后把 tail 从 0 改为 1，结果必须为 +`ProtocolError`，cursor 0/1 的 payload 与 seq 均保持不变，commit 仍为 0。 + +原有五档 CAP 的 driver 已切换为调用这条 production task wrapper，因此 +12,000-task 差分、容量、回收、三圈复用和零 entry commit 不再绕过新接口。 +新增故障门槛逐项覆盖： + +- 首槽 ownership CAS 失败：payload/tail/commit 均不变，返回 + `ProtocolError`； +- `WRITING -> seq` 失败：payload 已 flush，返回 `PartialPublish`； +- tail CAS 失败：payload/seq 已发布，返回 `PartialPublish`； +- 非空 task commit CAS 失败：slot/tail 已发布，返回 + `PartialPublish`； +- 零 entry commit CAS 失败：无 slot 副作用，返回 `ProtocolError`； +- 第二槽 ownership CAS 失败：第一槽保持发布、task 不 commit，返回 + `PartialPublish`。 +- preflight 后 tail 漂移：不得转到未经预检的新槽，返回 + `ProtocolError`。 + +验证结果： + +| 检查 | 结果 | +| --- | --- | +| CAP32/64/128/256/16384 shared ring UT | 5/5 PASS，每档 18 项 | +| GCC15 ASAN+UBSAN 同一五档门槛 | 5/5 PASS，无报告 | +| CCEC production wrapper 显式实例化 | AIC/AIV 均编译通过 | +| A5sim/A5 private/shared runtime artifact | 四种组合均构建通过 | +| 同口径 private `.text` 对照 | AIC/AIV 与 clean 父提交逐字节相同 | +| `git diff --check` | PASS | + +CCEC 证据位于： + +```text +/tmp/fdwic-s2d-task-result-ccec-20260726/ +``` + +private 对照使用 clean `177a09c7` detached worktree 与当前主工作树分别重建 +同一 A5 private artifact。AIC `.text` 均为 86,344B、SHA256 +`f7aacd262526...`；AIV 均为 86,704B、SHA256 `3938bd4fa2c5...`，两组 +`cmp` 均为逐字节相同。这里不与 S2c 的 94,600/95,512B 比较，因为后者来自 +另一构建变体。审计证据位于： + +```text +/tmp/fdwic-private-head-177a09c7-20260726/text-compare-safe/ +``` + +当前枚举还没有映射成 Host 错误码。真实接入时容量不足继续使用 +`PTO2_ERROR_TENSORMAP_CAPACITY`；纯协议拒绝和部分发布必须保留不同错误码， +并共同执行 fail-stop、不 Build、不 completion、所有 wait/drain 感知 fatal +后退出。该收敛属于下一阶段 production Submit 门槛，不能用本阶段的原语 +测试代替。 + +### 2026-07-26：S2e 冻结 shared Submit 的结果与错误合同 + +本阶段仍不把 shared TensorMap 接入真实 Submit，也不解除 backend-ready +门禁；只把 S2d 的 task 事务结果收敛为 Submit 和 Host 都能稳定识别的错误 +合同。先新增直接编译 production CPU-sim TU 的门槛，确认它因缺少结果处理 +入口和错误码而失败，再补充唯一映射： + +| task 发布结果 | 运行时错误码 | Submit 行为 | +| --- | ---: | --- | +| `Committed` | `0` | 继续当前 winner 流程 | +| `CapacityBlocked` | `11` | 标记本 worker 停止并置 fatal | +| `ProtocolError` | `13` | 标记本 worker 停止并置 fatal | +| `PartialPublish` | `14` | 标记本 worker 停止并置 fatal | +| 未知枚举值 | `13` | 按协议错误 fail-stop | + +`dist_submit_handle_shared_tensor_map_result()` 只完成上述映射和失败收敛，不 +执行 lookup、append、Build 或 completion。失败时把当前 worker 的 +`local_index` 设为 `kFlagCap`，再通过已有 `set_fatal_code()` 发布错误； +该函数保留“首个非零错误码获胜”合同，因此后续 worker 再发现其他失败不会 +覆盖根因。`PartialPublish` 与 `ProtocolError` 必须保留不同 code：二者都 +要求本轮停止并由下一次冷启动 reset 清理，但前者明确说明 shared sidecar +可能已有当前 task 的 payload/seq/tail 副作用，禁止把它误诊为可重试的 +容量反压。 + +新增 `test_fdwic_shared_submit_contract` 使用： + +```text +__CPU_SIM=1 +PTO_FDWIC_SHARED_MAP=1 +PTO_FDWIC_TENSORMAP_RING_CAP=128 +``` + +直接包含 production `dist_engine.cpp`，并静态断言 shared 构建身份成立、 +`kFdwicCompiledBackendReady` 仍为 false。测试覆盖成功不置 fatal、三类 +失败和未知枚举的精确映射、worker 停止标记以及 first-error-wins；后者 +预置一个非 shared 首错并遍历全部 shared 失败类型。失败前后还逐项比较 +shared commit/reclaim、bucket head/tail、slot payload/seq、task cell 和 +worker Build 计数，证明 adapter 本身不会继续改写数据面。Host 共用的状态码 +折叠函数同时锁定 `11/13/14 -> -11/-13/-14`。它与 shared ring 五档 CAP、 +private capacity、private/shared build identity 门槛共同通过，证明这一 +阶段没有用结果合同替代真正的事务测试，也没有提前让 shared artifact 进入 +尚未接线的运行路径。 + +本阶段证据： + +| 检查 | 结果 | +| --- | --- | +| 新门槛的 red-first | 缺少处理入口与错误码时按预期编译失败 | +| `test_fdwic_shared_submit_contract` | PASS | +| private capacity | PASS | +| shared ring CAP32/64/128/256/16384 | 5/5 PASS | +| private/shared build identity | 2/2 PASS，shared backend 仍 fail-closed | +| Host 可见错误值 | `-11/-13/-14` 全部 PASS | +| GCC15 ASAN+UBSAN 新门槛 | PASS，无报告 | +| production shared CCEC | AIC/AIV 均以 CAP128 shared 身份编译通过 | +| private 父提交对照 | AIC/AIV `.text` 与运行时 section 逐字节相同 | + +独立构建证据位于： + +```text +/tmp/fdwic-shared-submit-contract-coverage-20260726/ +/tmp/fdwic-s2e-submit-contract-sanitize-20260726/ +``` + +private 对照以 clean `ebe3ff28` 为父版本基线。当前与父版本的 AIC `.text` +均为 86,344B，AIV `.text` 均为 86,704B；`.text`、`.rela.text` 和运行时 +只读/数据 section 均逐字节相同。完整对象会受调试元数据影响，因此仍不拿 +整 `.o` hash 冒充热路径证据。 + +下一阶段才把 exact-turn、winner-only lookup 和整 task +`append -> commit` 接到 Kernel/Alloc/MIX 的真实 winner 路径。接线必须把 +本 helper 作为唯一结果收敛入口；任何非 `Committed` 结果都不得继续 Build +或发布 completion。 + +### 2026-07-26：R3a 接入 shared TensorMap 基础 Submit 事务 + +本阶段开始把 S2e 冻结的错误合同接入 production Submit，但仍保持 +`kFdwicCompiledBackendReady == false`。也就是说，本阶段验证的是正式 +Kernel/Alloc/joint Submit 源码在 shared 构建身份下如何访问和提交 +TensorMap，不把尚未完成多 worker、真实 PA INOUT 和跨核可见性门槛的 +artifact 提前放行。 + +#### 先固定唯一合法的访问顺序 + +shared TensorMap 不是把 private `head/tail` 换成 atomic 后让每个 replay +worker 照旧访问。当前 exact-turn 单追加者协议要求： + +```text +task N 唯一 winner + -> 等待 committed_tasks == N + -> 完成 N 的 INPUT/INOUT lookup + -> 对 N 的全部 INOUT/OUTPUT_EXISTING 做整 task preflight + -> 发布全部 payload/seq/tail + -> CAS committed_tasks: N -> N+1 + -> joint deposit（如有） + -> Build 本核 RingSlot + +task N loser + -> 不等待 shared turn + -> 不读 shared bucket/slot + -> 不发布 shared entry/commit + -> 只走既有 loser progress +``` + +`committed_tasks < N` 表示较早 task 尚未完成 map 事务，winner 在等待期间 +调用既有 `drain_block_won()/drain_phase_b()` 帮助系统前进； +`committed_tasks > N`、负值或 lookup 的 cursor/seq 不自洽均是协议错误， +不能降级成普通 miss。当前等待位于 Kernel 的 Fanin PMU 窗口、Alloc 的 +AllocComplete PMU 窗口内；后续解释分段数据时必须把这段 shared turn 等待 +算进相应阶段，不能把它误判成纯 lookup 或纯 completion 指令时间。 + +Kernel/joint winner 把 `register_mask` 中所有 INOUT/OUTPUT_EXISTING 先转换 +为固定的 `SharedTensorMapValue[MAX_TENSOR_ARGS]`，再调用一次整 task publish。 +即使 `register_mask == 0`，也必须发布零 entry 事务并推进 N+1，否则后续 +writer 会永远停在断裂的连续 commit 前沿。joint deposit 和本核 Build 均 +放在成功 commit 之后；容量、协议或部分发布失败都先经过 S2e 的唯一结果 +收敛入口，不再继续写 WonSlot 或 RingSlot。 + +Alloc 没有 ordinary-region entry，但同样占一个逻辑 task id。其 winner +必须先等待 exact turn，再提交零 entry 事务，最后才调用 +`complete_executed_task()` 发布 vend/flag/frontier。commit-ahead 等错误不得 +伪造立即完成。 + +#### 模式 facade 与 scalar 访问边界 + +`tensor_map.h` 现在把两种构建身份完全在编译期分开： + +- private 的 worker reset、`N-H` prepare、lookup 和 insert 保持原入口； +- shared worker reset/prepare 为 no-op，全局 sidecar 仍只由 AICPU setup + thread 初始化； +- shared Submit 使用单独的 + `dist_tensor_map_lookup_for_submit_winner()`，调用点位于 exact-turn gate + 之后； +- loser 不会取得这个入口，shared publish 也只接受 `ctx.won`。 + +CPU-sim 的通用 `get_tensor_data/set_tensor_data` 没有 Claim，也无法证明 +调用者是当前 exact-turn winner。它不能复用 Submit lookup 去读取 shared +map。本阶段没有脑补一套 scalar 访问协议，而是明确 fail-closed：shared +身份下设置 code 13、返回 lookup miss，并在实际读写 tensor buffer 之前 +退出；同时逐字节断言 private map 和 shared sidecar 都不被读取后写回或 +修改。CCEC 原有 scalar data access 路径不经过该 CPU-sim lookup 分支。 + +#### 集成门槛 + +新增 `test_fdwic_shared_submit_wiring`，直接在 +`__CPU_SIM=1 + PTO_FDWIC_SHARED_MAP=1 + CAP128` 下编译 production +`dist_engine.cpp`。最初只加入五个行为断言时，缺少接线的源码按预期全部 +失败；完成接线后扩展为十二类门槛： + +1. Kernel winner 从 shared map 获得 fan-in、整 task 发布 + INOUT/OUTPUT_EXISTING，并 Build 一个 RingSlot； +2. 零 region Kernel 仍推进 commit，且 bucket/slot/reclaim 不变； +3. 零 region joint winner 推进 commit 后发布一份 joint work； +4. Alloc winner 只改变 commit 后再发布 completion，shared 其余字节不变； +5. legacy 一次式 Kernel 使用同一 shared 事务边界； +6. legacy 一次式 Alloc 使用同一零 entry 事务边界； +7. loser 带 INPUT 和 OUTPUT，面对故意损坏的 shared bucket 仍不得读写 + shared，也不得触碰 private prepare/map； +8. 通用 scalar get/set 在 shared CPU-sim 下 code 13，并抑制真实 buffer + 读写； +9. winner lookup 遇到 `tail` 指向 invalid seq 时，在 publish/Build 前 + code 13； +10. Kernel 观察到 commit-ahead 时，在 Build 前 code 13； +11. Alloc 观察到 commit-ahead 时，不得发布 flag/vend/frontier； +12. joint winner 整 task 容量不足时，在 WonSlot deposit 和 RingSlot + Build 前 code 11。 + +失败门槛不仅检查 error code，还检查 `local_index == kFlagCap`、 +`committed_tasks`、task flag/vend/frontier、joint publication 和本核 +occupancy，避免把“返回了错误”误当成“没有继续产生副作用”。Alloc 零 +entry 成功门槛把 shared sidecar 除 `committed_tasks.v` 外的所有字节与调用 +前快照比较，证明它不是暗中发布垃圾 region 的伪空事务。 + +#### 冻结源码验证与代码量 + +最终冻结源码的关键 SHA256 前缀为： + +```text +tensor_map.h c045ff8e +tensor_data_access.h 68617def +submit_core.h b928f6a1 +submit_runtime.h eaf27b46 +shared wiring test 1cb238b9 +``` + +同一份源码得到以下结果： + +| 检查 | 结果 | +| --- | --- | +| GCC15 RelWithDebInfo FDWIC C++ 门槛 | 23/23 PASS | +| shared wiring 内部行为 | 12/12 PASS | +| GCC15 ASAN+UBSAN shared wiring | 12/12 PASS，无报告 | +| A5sim/A5 × private/shared production artifact | 4/4 Build complete；模式宏 0/1 与 CAP128 正确 | +| private CPU production TU 隔离 | 三个执行 section 及语义 relocation 与 `6e7e8af9` 相同 | +| private A5sim production object 隔离 | `.text/.text.exit` 及语义 relocation 相同 | +| private A5 CCEC 隔离 | AIC/AIV/final `.text` 与 raw `.rela.text` 均逐字节相同 | +| 最终格式/英文/头部/cpplint/markdown/clang-tidy | PASS | +| `git diff --check` | PASS | + +private A5 CCEC 的冻结对照为： + +| 产物 | `.text` | SHA256 前缀 | +| --- | ---: | --- | +| AIC | 86,344B | `f7aacd26` | +| AIV | 86,704B | `3938bd4f` | +| final aicore | 190,232B | `08ecb6ac` | + +A5sim private 最终 `.so` 的 raw `.text` 都是 75,955B，但有 125B 不同。 +逐项定位表明差异来自 clean worktree 与当前 worktree 的绝对源码根写入 +`.rodata` 后，RIP-local-data displacement 随本地数据地址平移；归一化 +反汇编完全相同，SHA256 都是 `50c4678d...`。因此它不是 private 执行逻辑 +变化，也没有用 raw final hash 掩盖路径重定位原因。 + +shared CCEC 则真实增加了基础事务代码： + +| 产物 | S2e | R3a | 增量 | +| --- | ---: | ---: | ---: | +| dist-engine AIC `.text` | 86,344B | 91,312B | +4,968B（+5.75%） | +| dist-engine AIV `.text` | 86,704B | 91,032B | +4,328B（+4.99%） | +| final aicore `.text` | 189,976B | 199,192B | +9,216B（+4.85%） | + +这部分不是 private 污染，而是 shared exact-turn、lookup、整 task preflight/ +publish 和错误分支进入设备产物的真实成本。后续 A5 正确性闭合后必须用 +shared perf-clock 和 I-cache 观察判断是否需要冷路径外提、缩短临时数组或 +减少模板实例，不能仅凭代码尺寸推断实际时间，也不能忽略约 4.85% 的 +I-cache 风险。 + +验证证据目录： + +```text +/tmp/fdwic-s3-basic-wiring-review-20260726/freeze-ut +/tmp/fdwic-s3-basic-wiring-review-20260726/freeze-ut-sanitize +/tmp/fdwic-s3-basic-wiring-review-20260726/freeze-artifacts +/tmp/fdwic-s3-basic-wiring-review-20260726/freeze-compare +``` + +一次无目标过滤的 C++ `all` build 仍会在既有 A2/A3 +`PTO2TaskPayload` 568B/576B 静态断言处失败；本轮没有修改 A2/A3，也没有 +把该失败隐去。上述 23/23 是显式构建和运行全部 FDWIC 目标的隔离结果。 +唯一编译 warning 是已有 `SPIN_WAIT_HINT` 在 sim +`inner_kernel.h` 与 `pto_runtime2_types.h` 的重复定义，本阶段未新增或掩盖。 + +#### 当前阶段边界 + +R3a 只闭合单 worker 可判定的基础事务顺序，下面这些仍是解除 shared 顶层 +门禁前的必做项，不能被十二类门槛替代： + +- `committed_tasks < N` 时多个真实 worker 的等待、帮助前进与恢复； +- 等待期间由其他 worker 发布 fatal 后的统一退出和 first-error-wins； +- Kernel Build 后真实执行、completion、final drain 与 Host 非零返回； +- fresh OUTPUT 的 heap/owner fan-in、连续任务复用和回收； +- joint 的 anchor/follower/last-lane 完整执行闭环； +- PA 最后一个含 INOUT task 的 region-intent Submit，保证其后的 loser + 使用覆写后的正确依赖； +- CCEC `get_tensor_data/set_tensor_data` 的同步合同；真实 PA 会调用这组 + scalar API，不能把当前只针对 CPU-sim 的 fail-closed 当成设备方案； +- A5 双核 `payload/seq/tail -> commit` 可见性 litmus； +- A5sim/A5 shared 正确性、private/shared 逻辑签名可比和 perf-clock 性能。 + +整 task 临时数组也会增加 shared winner 的 scalar stack，新增 shared +分支会扩大 CCEC `.text`。这两项先作为后续 A5 I-cache/栈和性能审计对象， +不能在基础协议尚未闭合时凭代码大小提前改写事务语义。 + +### 2026-07-26：R3b-a 固定 ordinary-region future-turn 等待与恢复 + +R3a 已经在 production `dist_submit_wait_shared_tensor_map_turn()` 中实现 +`committed_tasks < N` 时的协作式等待，但原有十二类门槛全部是单 worker +顺序调用，只能证明 exact turn 已经到达时的 lookup/publish/Build 顺序， +不能证明多个真实 replay worker 之间会等待并恢复。本小步只补这条证据， +不修改 production 热路径，也不提前解除 shared backend 总门禁。 + +新增 `test_fdwic_shared_multiworker`，以 +`__CPU_SIM=1 + PTO_FDWIC_SHARED_MAP=1 + CAP128` 直接编译 production +`dist_engine.cpp`。测试中的两个 worker 使用同一物理 block 的 AIV0/AIV1 +身份，并从相同的 task 0 开始重放： + +```text +AIV0: Begin task0，赢得 Claim，暂缓 Finish +AIV1: Begin/Finish task0，确定为 loser +AIV1: Begin task1，赢得 Claim，进入 Finish + committed_tasks 仍为 0，因此必须等待 +AIV0: Finish task0，lookup/register/commit 0 -> 1 +AIV1: 从等待恢复，解析 task1 fan-in，commit 1 -> 2，再 Build +``` + +这里没有把 AIV1 的 `local_index` 手工拨到 1。它必须先走完真实 task0 +loser 路径，再由 production Begin 自然取得 task1。task0/task1 对同一个 +INOUT region 连续读写，因此恢复后的 task1 RingSlot 还必须精确包含 +`fanin[0] == 0`；只检查最终 commit 数不足以证明等待后的 lookup 使用了 +正确版本。 + +为避免用 sleep 或“线程似乎没有返回”猜测阻塞，测试 TU 先加载正常 sim +平台定义，再只在该 TU 内把 `SPIN_WAIT_HINT()` 替换为“计数 + yield”。 +production header、A5sim/A5 artifact 和任何 private/shared 热路径源码均 +未增加 hook。主线程必须先观察到 task1 waiter 至少一次真实 spin，随后在 +释放 task0 之前逐项快照并确认： + +- `committed_tasks == 0`； +- task1 Finish 尚未返回； +- task0 和 task1 worker 都没有 Build RingSlot； +- task0 loser 没有推进 shared map。 + +释放后最终要求： + +- task0 loser ticket 为 `(task_id=0, won=0)`； +- task1 winner ticket 为 `(task_id=1, won=1)`； +- `committed_tasks == 2`，fatal/error 均为 0； +- 两个 winner 各有且仅有一个 built/occupied slot； +- task0 无 fan-in，task1 唯一 fan-in 为 task0。 + +这条门槛证明的是**通用 ordinary-region exact-turn**。它不能被误写成 +“PA 所有 Submit 都应经过全局 sequencer”：standalone PA Case1 的 fresh +symbol 热路径已经有独立的 per-output 发布协议，并明确保持 ordinary +region `committed_tasks == 0`。后续 production PA 迁移仍应让 fresh +symbol 绕过全局 region turn；只有真正访问普通 region ring 的 winner +使用本门槛所证明的等待纪律。 + +本阶段验证结果： + +| 检查 | 结果 | +| --- | --- | +| 新 production 多 worker 门槛重复运行 | 100/100 PASS | +| shared contract/wiring/multiworker 相邻回归 | 3/3 PASS | +| GCC15 ASan + UBSan | PASS，无报告 | +| `git diff --check` | PASS | + +下一小步单独处理错误收敛。当前源码仍有两个与本正常路径正交的缺口： +shared winner 在本核 slot 已满时没有 fatal 退出；已经进入 FinalDrain 的 +worker 也不会消费稍后由另一核发布的 fatal。若另一核因此跳过 FinalDrain， +前者可能永久等待不存在的 barrier release。修复必须只进入 shared 构建并 +保持 private 指令隔离，不能把本次正常等待门槛与错误路径改动合成一次提交。 + +### 2026-07-26：R3b-b 让 shared 已有等待在远端 fatal 后收敛 + +R3b-a 证明了正常的 future-turn winner 会等待并恢复，但它没有覆盖另一个 +worker 在等待期间终止整次 shared 运行的情况。重新逐个检查 production +Submit 和收尾循环后，确认有两个会永久等待的真实缺口： + +1. `dist_submit_wait_slot_capacity()` 在本核达到 + `kPrivateSlots-kWonReserve` 阈值后只尝试 drain。若已有 slot 的 fan-in + 因远端 fatal 永远不能 ready，该循环没有退出条件； +2. worker 可能先进入 `dist_submit_drain_to_completion()`，随后另一 worker + 才置 fatal。后者在 `core_main` 中观察到 fatal 后会跳过 FinalDrain， + 前者却仍等待这个缺席 worker 的 barrier arrival/release。 + +#### 改动边界 + +本阶段没有在每个 shared winner 上增加一条 fatal atomic： + +- slot-capacity 只在已经进入真实反压循环时检查 fatal；无反压路径不执行 + 新 load； +- FinalDrain 只在本轮没有释放 slot 时检查 fatal；仍有 ready work 时先 + 继续 drain，避免每个进展轮次增加 load； +- 两个检查都使用既有 `fdwic_trace_is_fatal()`，并把 + `FdwicAtomicSite::FatalPoll` 加入对应的现有 poll region。启用 atomic + 泳道时，这些 load 仍进入原有批量观测口径，不形成不可解释的裸 atomic； +- private 构建保留原 `void dist_submit_wait_slot_capacity()` 签名和原循环， + FinalDrain 也不包含新增分支。shared 构建才让 slot wait 返回 + `bool`，调用方收到 `false` 后停止 heap/joint/RingSlot Build。 + +远端 fatal 与已经开始的事务之间采用 fail-stop 合同，不新增一次昂贵的 +“事务提交后再次确认 fatal”。因此 fatal 可能发生在 task N 的 shared map +事务已经 commit、但尚未 Build 的时刻；该 task 的 +`committed_tasks == N+1` 可以保留，已有本核 slot 也不在这里强制清空。 +整次运行已经失败，AICPU/Host 必须返回首个非零错误，不能把这类中间状态 +解释成可继续执行的成功快照。此边界避免为了错误路径把每个正常 Submit +改成两阶段提交或增加第二次全局 atomic。 + +#### 红灯先行与多 worker 门槛 + +在修改 production 前，先把两个用例加到 +`test_fdwic_shared_multiworker`： + +- `RemoteFatalInterruptsSlotCapacityWait` 预置两个永远不能 ready 的本核 + built slot，让真实 task 0 winner 完成零 entry map commit 后进入 + slot-capacity wait；主线程确认 waiter 尚未返回，再发布 + `PTO2_ERROR_EXPLICIT_ORCH_FATAL`。旧实现超过 1,024 次 post-fatal spin + 后由测试 hook 确定性失败； +- `RemoteFatalInterruptsIncompleteFinalBarrier` 让一个 AIC worker 进入 + `expected=2`、实际只有一次 arrival 的真实 FinalDrain barrier；确认 + waiter 尚未返回后再发布相同 fatal。旧实现同样确定性失败，不依赖 + sleep 推测阻塞。 + +修复后第一项要求 commit 已经从 0 推进到 1，但不能新增 Build、completion +或 frontier；两条用例都要求 `local_index == kFlagCap`、首错保持不变。 +第二项还要求 leaf arrival 保持 1，root arrival/release 和 leaf release +均不得伪造。测试专用 `SPIN_WAIT_HINT` 仍只存在于该 CPU-sim 测试翻译单元， +CTEST 的 15 秒超时负责兜住任何不再经过该 hook 的意外挂死。 + +这两项只证明： + +- 本核 slot 反压在远端 fatal 后退出； +- 已经进入、但 barrier 缺少参与者的 FinalDrain 在远端 fatal 后退出。 + +它们不冒充“完整 FinalDrain 正常闭环”：fanin-blocked slot、pending joint +work、1 AIC + 2 AIV 正常 barrier、真实最后一个 Kernel 的 completion 和 +Host done 仍需后续正向联合门槛。 + +multiworker 门槛为避免测试 hook 污染记录而以 trace-disabled 身份编译； +`FatalPoll` 进入既有 PollBatch 的证据目前来自 production mask 代码审查和 +通用 PollBatch 门槛，不是本用例导出的路径级泳道。若后续把失败路径 atomic +观测也升级为发布合同，应另补 trace-enabled 隔离用例，不能拿本次功能门槛 +冒充全部 shared atomic 已经可观测。 + +#### 冻结验证 + +| 检查 | 结果 | +| --- | --- | +| 两项新增门槛的 red-first | 旧 production 均按预期触发“未消费远端 fatal”失败 | +| multiworker CTest（每次带 15 秒超时） | 100/100 PASS；每次包含正常 future-turn 和两项 fatal | +| shared contract / wiring + private capacity | 3/3 PASS | +| GCC15 ASan + UBSan multiworker | 3/3 PASS，无报告 | +| private/shared × A5sim/A5 artifact | 4/4 Build complete；shared backend 总门禁仍关闭 | +| private CPU production TU | `.text` 与父提交逐字节相同 | +| private A5sim production object | `.text` 逐字节相同；1,652 条 `.rela.text` 的 offset/type/symbol/addend 语义投影相同 | +| private A5 CCEC | AIC/AIV/final `.text` 及 AIC/AIV raw `.rela.text` 逐字节相同 | + +private A5 CCEC 冻结值继续为： + +```text +AIC .text 86,344B f7aacd262526... +AIV .text 86,704B 3938bd4fa2c5... +final .text 190,232B 08ecb6ac0e58... +``` + +A5sim clean worktree 与当前 worktree 的绝对源码根长度不同，导致断言路径 +所在 `.rodata` 的局部 `.LC*` 值和完整对象 hash 不同；执行 `.text` 的 +SHA256 均为 `655241601e3f...`，可执行 relocation 的 offset、类型、符号名 +与 addend 也完全一致。这里没有拿整 `.o` 不同掩盖 private 代码变化。 + +本阶段证据目录： + +```text +/tmp/fdwic-r3b-fatal-ut-20260726/ +/tmp/fdwic-r3b-future-sanitize-20260726/ +/tmp/fdwic-r3b-private-before-src-20260726-2218/ +/tmp/fdwic-r3b-private-before-build-20260726-2221/ +/tmp/fdwic-r3b-private-after-build-20260726-2221/ +/tmp/fdwic-r3b-artifact-compare-20260726-2228/ +``` + +下一小步应补完整 Kernel execution/completion/final-drain 正向联合门槛，再 +进入 PA 尾部 INOUT 的 region-intent。当前改动不解除 shared backend 门禁， +也不使用 CPU-sim 的 fatal 通过来替代 A5 GM 可见性和上板正确性证据。 + +### 2026-07-26：R3b-c 固定单 lane Kernel 的三 worker FinalDrain 正向闭环 + +R3b-b 的两项 FinalDrain 证据都是错误路径：它们证明 barrier 参与者缺失时 +能够退出，却不能证明正常运行中最后一个已 Build Kernel 会真正执行、发布 +completion，并让全部 worker 通过两级 barrier。当前小步只补这条正向门槛， +不修改 production。 + +#### 不手填 slot 的真实调用链 + +新用例 +`SingleLaneKernelExecutesOnceAndCompletesThroughThreeWorkerFinalDrain` +建立一个物理 block 的真实三 lane 拓扑： + +```text +core 0: AIC / block 0 / LANE_AIC +core 1: AIV / block 0 / LANE_AIV0 +core 2: AIV / block 0 / LANE_AIV1 +``` + +三核都从 `local_index=0` 重放同一个只含 AIV0 kernel 的 task 0： + +- AIV0 先走 production compete-first Begin/Finish，赢得 vector Claim； +- AIC 重放同一 task，因角色不匹配而不发起 Claim； +- AIV1 重放同一 task，发起同一 vector Claim 并成为 loser； +- 三核都通过正常 Begin 自然推进到 `local_index=1`，测试不手拨 task id。 + +winner 的 RingSlot 必须由完整 +`Materialize -> shared commit -> WinnerBuild` 路径生成。测试提供一个 +真实 `TensorCreateInfo` OUTPUT 和 4 KiB 对齐 heap；每核 materialize 后 +`heap_next=1024`，使 completion 的 vend 不再是无法区分的 0。kernel 地址 +也不是直接写进 slot：测试先构造真实 `CoreCallable`，把计数函数写进其 +`resolved_addr`，再由 production +`Runtime::func_id_to_addr_ -> resolve_kernel_addr() -> RingSlot` 解析。 + +FinalDrain 前要求： + +- kernel 调用次数为 0，排除 Submit 或 loser replay 提前执行； +- shared `committed_tasks==1`，无 fatal； +- 只有 AIV0 有一个 built task 0 slot，AIC/AIV1 ring 为空； +- slot 的 func id 和最终函数地址与 callable 一致； +- 三核 `heap_next==1024`； +- task 0 的 flag/vend 仍为 0，frontier 仍为 -1； +- `any_pub==0`，明确本用例没有暗中落入 joint/WonSlot 路径。 + +随后三个 host 线程分别设置真实 TLS `g_self`，在统一起跑门后各调用一次 +production `dist_submit_drain_to_completion()`。barrier 参数完全采用一个 +block 的 control-plane 推导结果： + +```text +leaf[0].expected = 3 +root.expected = 1 +``` + +join 后要求： + +- 计数 kernel 恰好执行一次； +- 三核 ring 全空，task 0 `flag=1`、`vend=1024`、`frontier=0`； +- shared commit 仍为 1，fatal/error 仍为 0； +- leaf arrival 为 3，root arrival/release 和 leaf release 都为 1； +- 其余 final-barrier group 仍为 0； +- 三线程都返回且没有异常，三个 worker 的 `local_index` 均保持 1。 + +这证明的是单 lane +`Build -> FinalDrain execute_slot -> completion -> hierarchical barrier` +闭环。它没有调用 CPU-sim 的 `dist_aicore_finish_worker()`：当前测试平台的 +`sim_get_reg_base()` 返回空地址，强行验证 COND/done 会引入假寄存器环境。 +若需要锁定 `core_main -> finish_worker -> COND`，应另建入口级集成门槛。 +本例的 barrier expected 也只是按 control-plane 的单 block 规则手工设置， +所以它验证 FinalDrain 使用这些值，不验证 AICPU 拓扑初始化本身。计数 kernel +虽然通过真实 callable dispatch，且 slot 中 OUTPUT 描述符必须指向本次 heap +物化出的 4B buffer，但不会检查实际算子结果；它锁定 exactly-once 和 +completion，不替代 kernel 参数 ABI/数值正确性用例。 + +#### 刻意不混入 joint/follower + +joint task 会额外引入 WonSlot deposit、follower `drain_block_won()`、 +`remaining` 最后一核发布 completion 和 WonSlot 回收。把它塞进本用例后， +失败时无法判断是 FinalDrain/barrier 还是 follower 协议错误。因此本阶段 +只证明单 lane 正常闭环；joint anchor/follower/last-lane 必须作为下一条 +独立正向门槛，不能用“三核参加了 barrier”冒充 joint 已覆盖。 + +#### 验证结果 + +| 检查 | 结果 | +| --- | --- | +| multiworker CTest（含 4 类场景） | 100/100 PASS，每次受 15 秒超时保护 | +| 新单 lane FinalDrain 定向用例 | PASS | +| GCC15 ASan + UBSan | 4/4 PASS，无报告 | +| GCC15 TSan | 4/4 PASS，无数据竞态报告 | +| production / artifact 源码 | 无修改，不产生新的 private/shared 机器码差异 | + +TSan 编译明确警告 GCC 的 ThreadSanitizer 不支持 +`__atomic_thread_fence`；因此它只能排查 host 测试中的普通 data race， +不能替代 A5 的 flush/invalidate、GM atomic 或 memory-order litmus。 + +本阶段继续复用： + +```text +/tmp/fdwic-r3b-fatal-ut-20260726/ +/tmp/fdwic-r3b-future-sanitize-20260726/ +/tmp/fdwic_ut_tsan/ +``` + +下一小步先补 joint 的 pending WonSlot、三个 lane 各执行一次、 +`remaining: 3 -> 0` 和 last-lane completion；之后再进入 PA 尾部 INOUT +region-intent，保持每个协议问题都有独立的失败定位面。 + +### 2026-07-26：R3b-d 固定 joint pending follower 与 last-lane completion + +R3b-c 只覆盖单 lane kernel。当前小步继续不修改 production，新增 +`JointThreeLanePendingFollowersDrainAndLastLaneCompletesOnce`，专门闭合一个 +真实 AIC+AIV0+AIV1 joint task 的以下链路： + +```text +三 lane Begin + -> AIV0/AIV1 loser Finish（anchor 尚未发布） + -> AIC winner Finish 并发布唯一 WonSlot + -> 原来的三条 persistent worker 线程进入 FinalDrain + -> AIV0/AIV1 从 WonSlot 搬取 pending follower + -> remaining 3 -> 2 -> 1 -> 0 + -> 最后一条 lane 清 WonSlot 并发布 task completion +``` + +#### CPU-sim TLS 踩坑与修正 + +第一版测试把三个 Begin/Finish 放在主线程，再新建三个 FinalDrain 线程。 +结果只有 AIC anchor kernel 执行，两个 follower 永远没有进入私有 RingSlot。 +检查 production 后确认这不是 shared 协议失败： + +- A5 persistent worker 的“本核见过 joint Submit”是 block-local 状态； +- CPU-sim 用 `thread_local g_fdwic_joint_submit_seen` 等价模拟； +- 新建的 FinalDrain 线程没有执行过 joint Begin，因此 + `drain_block_won()` 和 `has_pending_won()` 会按设计直接返回。 + +修正后的测试让每个物理 lane 固定在一条宿主线程上，从 Begin 一直运行到 +FinalDrain；测试不手工把 TLS 置为 true。三线程 Begin 后的 TLS 快照必须为 +true，Finish 后仍为 true，从而与真实 persistent worker 生命周期一致。 + +#### 确定性地保留 pending follower + +AIC 是三 lane active mask 的唯一 anchor。测试先让三线程都完成 Begin， +随后只放行两个 AIV loser Finish，AIC Finish 继续停在门闩前。两个 loser +虽然执行真实 `Materialize/Register/LoserReplay`，其尾部 +`drain_block_won()` 看到的 `any_pub` 仍为 0，因此发布前必须满足： + +- `committed_tasks==0`、`any_pub==0`； +- AIV0/AIV1 私有 ring 均为空； +- task flag/vend/frontier 仍为 `0/0/-1`。 + +之后才放行 AIC Finish。FinalDrain 起跑前按值保存所有关键证据，而不是在 +slot 释放后读取碰巧残留的旧字段： + +- 恰好一个 WonSlot 非 Free，且状态为 Published、`remaining==3`； +- AIC deposit 不存在；AIV0/AIV1 deposit 均为 present、drained=Free； +- 两个 follower 的 func id、真实 callable 地址、sub-block id、tensor 和 + fan-in 均与本次 joint task 一致； +- AIC 已有且仅有一个真实 multicore RingSlot，并指向同一个 WonSlot； +- AIV0/AIV1 仍没有 built/occupied RingSlot，证明 follower 仍是 pending； +- shared commit 已推进到 1，但三个 kernel 计数均为 0，completion 仍未发布。 + +扫描同时要求 `Published count==1` 和 `non-Free count==1`，防止额外泄漏一个 +Claimed WonSlot 却被“恰好找到一个 Published”掩盖。 + +#### last-lane 的中间态与终态 + +三个 callable 都是真实地从 +`Runtime::func_id_to_addr_ -> CoreCallable::resolved_addr -> RingSlot` 解析， +并在函数入口分别增加独立计数、等待各自测试门闩。只有观察到三个 kernel +均已进入后才依次放行： + +1. 放行 AIC：要求 `remaining==2` 且 AIC FinalDrain 已返回； +2. 放行 AIV0：要求 `remaining==1` 且 AIV0 FinalDrain 已返回； +3. 此时 AIV1 仍停在 kernel 内，必须同时满足 + `state=Published`、`flag/vend/frontier=0/0/-1`； +4. 最后放行 AIV1，才允许 `remaining==0`、`state=Free`、 + `flag/vend/frontier=1/1024/0`。 + +终态还要求三个 callable 各进入、退出恰好一次，AIV0/AIV1 的 +`owned_total` 各为 1，三核 ring 全空，shared commit 保持 1，fatal/error +保持 0，final barrier 精确为 leaf arrival 3、root arrival/release 1、 +leaf release 1。由此证明 completion 来自最后一个真实 lane 的 +`fetch_sub==1`，而不是测试直接写 flag 或手填 WonSlot/RingSlot。 + +#### 测试自身的正确性控制 + +- 每个阶段用 release/acquire 门闩建立明确 happens-before； +- worker 线程不调用 GTest 宏； +- 任一 Begin/Finish 阶段超时或抛异常时，先释放全部测试门闩并 join，再报告 + 失败，不能继续读取正在被 worker 修改的普通字段； +- callable/deposit/anchor slot 在 FinalDrain 前做值快照，终态不依赖 Free + slot 中的陈旧 metadata； +- 无论观察成功还是失败，join 前都无条件放开三个 kernel 门闩； +- CTest 的 15 秒超时仍兜住 production 内部不再响应测试门闩的真实挂死。 + +#### 验证结果 + +| 检查 | 结果 | +| --- | --- | +| multiworker GTest（由 CTest 运行，现含 5 类场景） | 5/5 PASS | +| joint 定向重复 | 100/100 PASS | +| GCC15 ASan + UBSan | 5/5 PASS，无报告 | +| GCC15 TSan | 5/5 PASS，无数据竞态报告 | +| production / artifact 源码 | 无修改，不产生新的 private/shared 机器码差异 | + +TSan 仍报告仓库已知的“`atomic_thread_fence` 不受 +`-fsanitize=thread` 支持”编译告警，因此这里只把它作为宿主线程普通 data +race 门槛，不把它当作 A5 GM 可见性证明。 + +下一阶段先增加真实 PA G2 的 exact-turn writer-chain oracle。当前 ordinary +region 实现已经让未来 winner 在 `committed_tasks==N` 后才 lookup，因此 +task 4 发布完成后 task 8 才能解析到最新 writer 4；此时再额外加入 +region-intent 会重复同步。只有后续同时迁移 shared-output stable symbol、 +winner-only 构参和 light loser、准备解除全局 exact-turn 时，才需要把 +“非末组 UP 的 writer-ready gate”与这些机制作为一个独立阶段引入。 + +### 2026-07-27：R3c 固定 PA G2 writer 链并补齐 manual_dep 合同 + +前述门槛只证明普通 region 事务和 joint 完成协议，没有覆盖 PA 多组 +online-update 对同一组 accumulator 的连续改写。当前阶段先构造最小真实 +G2 拓扑，再决定是否需要新增 region-intent，不能先凭推测增加一层同步。 + +#### 最小 G2 业务拓扑 + +真实 PA 在 `batch=1`、`num_heads=16`、`kv_head_num=1`、 +`block_size=128`、`context_len=8193` 时 `q_loop=1`、共有 65 个 block， +因此形成两个 group、共 9 个 task： + +```text +0 Alloc +1 QK0 -> 2 SF0 -> 3 PV0 -> 4 UP0 +5 QK1 -> 6 SF1 -> 7 PV1 -> 8 UP1 +``` + +task 4 的非末组 UP 以 INOUT 改写 `mi_update/li_update/oi`，task 8 +必须把 task 4 解析为这三个 region 的最新 writer，同时保留原始 creator 0 +和本组 QK/SF/PV 的 producer 6/7。最后一个 `out_view` 也是 INOUT，但 PA 用 +`Tensor::view(..., true)` 把它标成 `manual_dep`；其合同是 creator-only， +不单独建立 ordinary-region 边。当前 PA 的跨组执行顺序由同一 UP 的三个 +普通 accumulator writer 链保证，不能把它描述成另有一条显式 +`out_view` 依赖。 + +新用例 `PaG2LatestInoutWriterFeedsFinalUp` 直接调用 production shared +Submit。为避免复制 QK/SF/PV 算子实现，task 0、2、3、5～7 直接调用 +shared map publish 原语种入零 entry commit 来推进 exact turn，不经过 +这些 task 的完整 Submit。task 1 额外种入一条 production 不会产生的 +`out_view -> producer 1` poison entry：错误 lookup 会把 1 加入 UP +fan-in，错误 register 会把它覆盖成 4/8。task 4 和 task 8 则完整执行 +`Begin -> Fanin -> Register -> shared commit -> WinnerBuild`。它断言： + +- task 4 fan-in 精确为 producer 0、2、3； +- task 8 fan-in 精确为 producer 0、4、6、7，其中 0 是 creator lifetime， + 4 是 ordinary-region latest writer； +- 三个 accumulator 在 task 4 后的 latest producer 为 4，在 task 8 后为 + 8； +- `out_view` 的人工 latest producer 在两个时点都保持 1，且 producer 1 + 从未进入两个 UP 的 fan-in； +- shared `committed_tasks` 依次为 5、9，本核构建两个真实 RingSlot； +- private map 逐字节不变。 + +该门槛证明的是 G2 的 TensorMap writer 链，不是完整 PA 数值用例：被种下的 +commit 不执行 QK/SF/PV kernel，poison entry 也不是业务数据；本用例不验证 +这些任务的 OUTPUT 地址或数值。 + +#### 红灯揭示的公共 manual_dep 缺口 + +在 production 修正前，三个 accumulator 的 writer 链已经正确，但 +`out_view` 的人工 latest producer 在 task 4、8 后分别被错误覆盖成 4、8， +而不是保持 1。 +根因不是 shared exact-turn： + +- fan-in 的 CPU 分支已经在 lookup 前跳过 `manual_dep`； +- `calculate_output_layout()` 却把所有 INOUT/OUTPUT_EXISTING 都加入 + `register_mask`； +- private Register 和 shared 整 task publish 共用这个 mask，因此两种 + 模式都会登记按 creator-only 合同本不应进入普通 TensorMap 的 region； +- CCEC 的 GM/local fan-in 分支又缺少 CPU 已有的 `manual_dep` lookup + 过滤，真机口径还比 CPU 多做一次无意义查询。 + +修正为 Register 增加地址空间感知的 +`dist_submit_tensor_uses_manual_dependency()`：CCEC 对 GM tensor 读取 +`gm_ref().manual_dep`，对本地 tensor 读取 `ref().manual_dep`，CPU-sim +沿用 `ref()`。Fanin 已经完成 GM/local 分流，因而直接读取对应引用的 +`manual_dep`，不重复判断一次地址空间。两处都遵守 canonical 顺序： + +```text +保留 owner_task_id fan-in + -> manual_dep 时跳过普通 TensorMap lookup + -> OUTPUT 布局照常计算 + -> manual_dep 时不加入 register_mask +``` + +因此修正没有删除 creator lifetime，也没有改变 OUTPUT materialize；它只 +补齐“manual_dep 不进入普通 TensorMap”的查、插合同。private 路径会少写 +每核无意义条目；shared 的整 task publish 和 commit CAS 次数不变，只把 +UP 发布的普通 entry 从 4 个减为 3 个，并少占对应 bucket 的一个 ring +slot。本阶段没有单独测量该减法的性能收益，不把正确性修正宣称成已量化 +优化。 + +private 对称门槛直接编译 production private Submit：先给 manual_dep +INOUT 预置 creator 0 和 map producer 1，给另一个 bucket 的普通 INOUT +预置 producer 2，再由 task 3 成功 Submit。最终 RingSlot 的 fan-in 精确为 +0/2，不含 1；manual region 的 latest 仍为 1，普通 region 更新为 3，并且 +恰好构建一个 task 3 RingSlot。因此它在 CPU private 路径同时动态证明 +creator retention、lookup skip、register skip 和普通 INOUT 正常查插。 + +shared G2 的 poison entry 同样动态证明 CPU shared lookup/register skip。 +CCEC 证据则是与 canonical 源码逐支对齐、四类 artifact 编译，以及 A5 +private CaseB1 对本地 `out_view` 分支的实际执行；GM manual_dep 分支尚无 +独立动态用例。 + +#### exact-turn 与 region-intent 的当前结论 + +当前 shared Submit 要求 task N winner 只能在 +`committed_tasks == N` 时开始 lookup。于是 task 8 不可能越过尚未提交 +task 4 的事务；G2 定向用例已证明它随后取得 writer 4。此时新增 +region-intent 只会重复现有全局 turn 同步，并扩大协议和 atomic 面。 + +所以当前阶段不加 region-intent。后续迁移 stable shared-output symbol、 +winner-only 构参和 light loser，并准备放松全局 exact-turn 时，必须重新 +评估非末组 UP 的 writer-ready gate；那时 region-intent 才有独立价值。 + +#### 验证结果与边界 + +| 检查 | 结果 | +| --- | --- | +| G2 red-first | accumulator writer 链正确；manual out_view 错误登记为 4/8 | +| production private/shared CPU 门槛 | 24/24 FDWIC CTest PASS | +| G2 定向重复 | 100/100 PASS | +| private Submit 全组重复 | 100 轮 × 4 case PASS | +| GCC15 ASan + UBSan | private 4/4、shared wiring 13/13 PASS，无报告 | +| GCC15 TSan | private 4/4、shared wiring 13/13、shared multiworker 5/5 PASS，无数据竞态报告 | +| private/shared × A5sim/A5 artifact | 4/4 Build complete，CCEC GM/local 分支均编译 | +| A5 private PA CaseB1 golden | PASS | + +TSan 仍有项目既知的 `atomic_thread_fence` 不受该工具支持的编译告警，因此 +它可以排查宿主普通 data race，却不能完整建模 fence 同步，更不能替代 A5 +GM 可见性验证。shared 的 +`kFdwicCompiledBackendReady` 仍为 false，本阶段没有运行 shared A5sim/A5 +业务用例，也不把 artifact 构建成功写成 shared 上板正确性。 + +下一小步应把同一 G2 writer-chain 放入多 worker 提前到达场景，证明 future +winner 的等待、远端 task 4 commit、恢复 lookup 和最终 Build 在一个真实 +并发闭环内成立；完成后再审视解除 shared backend-ready 门禁所缺的 +A5 GM 可见性与完整 PA 条件。 + +### 2026-07-27：R3d 闭合 PA G2 future winner 的跨核 writer-chain + +R3c 在单 worker 中证明了 task 4 与 task 8 的普通 region writer 链，但其 +提交顺序由测试线程串行推进，尚未证明 future winner 提前到达时会在远端 +事务提交后恢复。R3d 不先修改 runtime，而是新增聚焦门槛 +`PaG2FutureFinalUpWaitsForRemoteFirstUpWriterCommit`，直接验证现有 +exact-turn 是否已经提供所需语义。 + +#### 并发顺序 + +测试保持 R3c 的 G2 业务依赖： + +```text +0 Alloc +1 QK0 -> 2 SF0 -> 3 PV0 -> 4 UP0 +5 QK1 -> 6 SF1 -> 7 PV1 -> 8 UP1 +``` + +具体执行顺序为： + +1. 直接提交 task 0～3 的 shared map 事务，把 + `committed_tasks` 推进到 4；task 1 仍只种入用于检查 + `manual_dep` 查、插过滤的 `out_view -> producer 1` 人工哨兵。 +2. AIV0 从 `local_index=4` 执行 task 4 的真实 + `Begin`，完成 production Claim 后暂不调用 `Finish`。其参数方向与 + PA UP 一致，并带 `is_first=1,is_last=0`。 +3. AIV1 从 `local_index=8` 执行 task 8 的真实 + `Begin/Finish`，参数带 `is_first=0,is_last=1`；task 4 与 task 8 的 + ID 都落在 + `vector_cursor[0]`，因此 cursor 依次从 `-1` 推进到 4、8，两次 + Claim 都是真实 winner Claim。 +4. task 8 在 `committed_tasks=4` 时进入 production + `dist_submit_wait_shared_tensor_map_turn()`。此时它没有返回，也不可能 + 进入后面的 Fanin、Register 或 Build。 +5. AIV0 调用真实 task 4 `Finish`，完成 Fanin、Register、shared + commit 和 WinnerBuild，把 commit 推进到 5。 +6. 测试先在 `committed_tasks=5` 时确认 task 8 仍继续执行 exact-turn + 等待且 Finish 未返回，再依次直接提交 task 5～7 的空 shared map + 事务；只有 task 7 把 + commit 推进到 8 后，task 8 才恢复 lookup，取得 task 4 的最新 writer， + 最终提交 task 8 并 Build。 + +测试没有在 task 4 之前提前提交 task 5～7，因为那会破坏 +`0..N` 连续发布合同，制造 production 不允许的事务顺序。 + +#### 精确业务断言 + +| 检查点 | 断言 | +| --- | --- | +| task 8 等待时 | `committed_tasks=4`、`vector_cursor[0]=8`、Finish 未返回 | +| task 4 提交后 | 三个 accumulator latest 都为 4，`out_view` latest 仍为 1 | +| task 4 RingSlot | fanin 精确为 `{0,2,3}`，各一次，不含 1 | +| task 8 RingSlot | fanin 精确为 `{0,4,6,7}`，各一次，不含 1 | +| 最终 shared map | 三个 accumulator latest 都为 8，`out_view` latest 仍为 1 | +| 最终提交状态 | `committed_tasks=9`、`vector_cursor[0]=8`、无 fatal/error | +| worker 状态 | AIV0/AIV1 的 `local_index` 分别为 5/9，各只 Build 一个 task | +| private 隔离 | 两个 worker 的 private TensorMap 都逐字节不变 | + +活跃 worker 期间，主线程只读取测试原子量、shared commit 和 Claim +cursor，不读取可能仍被 worker 修改的 ticket、RingSlot、`local_index` +等普通字段。任一释放异常或完成超时都会先发布 fatal，让 production wait +退出并 join；所有普通状态都在 join 后断言。这一组织方式避免用测试自身的 +数据竞态或提前析构掩盖 runtime 问题。 + +#### 证据边界 + +该用例是“G2 future-turn/writer-chain 聚焦门槛”，不是完整 G2 多核 PA： + +- R3b 的通用两 task 门槛已用从 0 开始的真实 loser replay 证明 + `local_index` 推进机制；它没有证明 G2 的 `0..8` 逐核回放。本用例手动 + 设置 4/8，只把 Claim、等待、远端提交、恢复 lookup 和 Build 放进同一 + 并发闭环，避免把两种证据混称为完整 replay。 +- task 0～3、5～7 只提交 shared map 事务，没有 Build 或执行对应 + Alloc/QK/SF/PV kernel。 +- task 4/8 使用 PA UP 相同的七个 tensor 方向和两个 + `is_first/is_last` scalar，但 tensor 地址与 producer 是测试构造；本用例 + 不覆盖 task 4/8 loser 或后续 light-loser 路径。 +- `out_view -> producer 1` 是错误查插探针，不是 PA 的真实 QK 输出。 +- 本用例不证明 96 核 replay、FinalDrain、数值 golden、A5 GM cache + 可见性或 shared A5 后端已经可放行。 + +在当前实现中，task N winner 只有等到 `committed_tasks == N` 才开始 +TensorMap lookup。R3d 已直接证明 task 8 不能越过 task 4 的事务，并会在 +远端 commit 后取得 writer 4。因此当前 exact-turn 下仍不需要额外 +region-intent；现在加入只会重复同步并扩大 atomic 面。只有后续引入 +stable shared-output、winner-only 构参和 light loser，并准备放松全局 +exact-turn 时，才重新评估非末组 UP 的 writer-ready gate。 + +这里等待的是 task 4 的 TensorMap writer 元数据事务，而不是 task 4 kernel +执行结束：production 顺序是 Register/shared commit 在 WinnerBuild 之前。 +该区分与 PA 跨组依赖一致——task 8 先取得 producer 4 作为 fanin，真正执行 +仍由 task flag 保证在 task 4 kernel 完成之后。 + +#### 验证结果 + +| 检查 | 结果 | +| --- | --- | +| 新 G2 future-turn 定向重复 | 100/100 PASS | +| shared multiworker 全组 | 6/6 PASS | +| FDWIC CPU CTest | 24/24 PASS | +| GCC15 ASan + UBSan multiworker | 6/6 PASS，无报告 | +| GCC15 TSan multiworker | 6/6 PASS，无普通数据竞态报告 | +| production 源码 | 无修改 | + +TSan 仍报告仓库既有的 `atomic_thread_fence` 不受该工具完整支持的编译告警， +所以该结果只证明宿主测试没有可见的普通 data race，不替代 A5 GM 可见性 +验证。 + +### 2026-07-27：R4a 用 CAS 加固通用 writer-ready 发布门 + +在把 PA 专用 writer 链改成通用 `WriterIntentSet` 之前，先单独加固所有 +方案都会复用的 `TaskCell::deps_prepared` 发布原语。旧实现使用 +`Exchange(-1 -> task_id)` 后再检查返回值；当 task cell 已被错误复用或 +存在重复 publisher 时，它会先把竞争值覆盖成一个合法 `task_id`，再报告 +失败。等待方可能在这个短窗口内被错误放行，且原故障值也被破坏。 + +本阶段给 standalone 的 CPU/CCEC Ops 增加与 production atomic wrapper +同口径的 `CompareExchange(int64_t)`： + +```text +observed = CAS(address, expected=-1, desired=task_id) +success = (observed == -1) +``` + +接口返回线性化点观察到的旧值，不返回 bool。CPU 使用 strong +`__atomic_compare_exchange_n`，成功/失败分别采用 AcqRel/Acquire；CCEC +直接使用本机 CANN 9.1 已声明并可编译的 GM `atomicCAS`。CCEC +CAS 只承担控制字的原子线性化，不能被描述成相邻 payload 的发布屏障； +普通 payload 仍必须沿用既有 DCCI `FlushRegion/InvalidateRegion` 协议。 + +定向门槛在竞争值已经存在时再次发布 writer-ready，并同时断言： + +- 发布返回失败； +- `deps_prepared` 保持竞争值，不能被本 task 覆盖； +- 后续 waiter 仍把错误 task id 识别为协议故障； +- 正常首次发布、重复发布、PA G2 loser 等待和 post-gate fatal 路径不变。 + +本阶段只加固通用控制门，不改变： + +- 哪些 task 需要 writer intent； +- PA `has_following_group` 的临时判定； +- symbol fanin/writer 提交； +- ordinary region ring 或其 exact-turn 原型； +- writer-ready 与 kernel completion 的职责分离。 + +因此它是通用化的前置提交，不能单独解释为 ordinary region 的 +`A -> B -> C` 竞态已经解决。 + +#### 验证结果 + +| 检查 | 结果 | +| --- | --- | +| CPU shared 全套构建与门槛 | PASS | +| writer-ready 竞争值不覆盖定向断言 | PASS | +| CCEC shared AIC/AIV entry | 编译通过 | +| CCEC split runtime / noinline finish | AIC/AIV 均编译通过 | +| CCEC mixed ELF / LOCAL helper / relocation / manifest | 全部 PASS | +| `git diff --check` | PASS | + +用户目录 GCC 15 与系统 binutils 2.42 的组合在后续独立自测目标生成 +`.base64` 汇编伪指令时失败；同一源码改用系统 GCC 13 后完整通过。这个 +工具链组合问题不属于 CAS 源码失败，也没有用来替代上述 CCEC 9.1 编译 +证据。 + +### 2026-07-27：R4b 先建立不依赖 PA 拓扑的 WriterIntentSet 语义门槛 + +R4a 只加固了 `deps_prepared` 的发布原语。R4b 开始把“某个 task 声明自己 +将复写哪些 tensor”从 PA 的 `TaskKind::Up`、group 数量、三个 accumulator +和固定 `task_id-4` 中抽离，但本阶段刻意不把新入口接入 PA runtime。原因是 +先用独立 A→B→C 门槛验证公共语义，再处理多版本历史；不能一边替换真实 +PA 热路径，一边猜测通用协议是否成立。 + +#### 本阶段公共接口 + +`InspectSharedWriterIntent()` 只检查参数方向和引用类型： + +- 非 `manual_dep` 的 `INOUT` / `OUTPUT_EXISTING` 需要自动 writer intent； +- plain shared-output symbol 和 ordinary GM/local region 使用同一入口； +- `manual_dep` writer 继续由调用方显式管理，不发布 writer-ready； +- 判定不读取 `TaskKind`、group、batch、后继 task 或 PA ticket。 + +winner 调用 `PrepareSharedWriterIntentSet()` 时： + +1. 先校验全部 writer 引用，拒绝非法 tag、空指针、非法 shape/range、 + future symbol 和同 task 重复 symbol writer； +2. 复制并去重调用方已经解析出的 fanin，因而公共入口可以接在普通 + `CollectSharedFanin()` 之后，不要求 PA 再保留一套 Commit-only 协议； +3. symbol 用 `CAS(previous_writer -> current_task)` 发布当前 writer，并把 + previous writer 加入本 task fanin； +4. ordinary region 先 lookup previous writer，再追加当前 task 的不可变 + region entry； +5. 全部 metadata 完成后才发布 `deps_prepared=task_id`。kernel completion + 仍只由 `task.flag` 表达。 + +`CollectSharedFanin<..., AcceptLatestWriter=true>` 只供新的独立门槛使用： +symbol 接受当前 cell 中落在 +`[descriptor producer,current task)` 的 writer；ownerless ordinary +`INPUT` 也查询 shared region map。默认 PA 实例仍保持原来的精确 writer +oracle，ownerless INPUT 的新增查询不会提前进入现有 PA 性能路径。 + +#### 独立 A→B→C 门槛 + +新增 `test/test_shared_writer_intent.cpp`，完全不使用 `TaskKind`、group、 +PA 参数构造器或 task ticket： + +- symbol：A 发布 fresh descriptor，B 以 INOUT 复写，C 在 B 的 + writer-ready 前阻塞；B 发布后，C 解析到 B。B 同时保留并去重调用方既有 + fanin，且 B 的 completion flag 仍为 0； +- ordinary：ownerless external region 由 A `OUTPUT_EXISTING`、B + `INOUT`、C `INPUT` 构成同一链。task id 使用 100/120/140，且 + `committed_tasks` 始终为 0，避免用连续 exact-turn 偶然掩盖接口耦合； +- manual：纯 `manual_dep` INOUT 返回 `NotRequired`，不改 region ring， + 也不发布 writer-ready。 + +CPU shared 全套门槛通过。CCEC 新增独立 compile-only TU，对真实 +`PrepareSharedWriterIntentSet` 分别做 +`dav-c310-cube` / `dav-c310-vec` 后端代码生成；输出写 `/dev/null`，不加入 +`DEVICE_OBJECTS`。最终 mixed ELF 中没有 probe 或该模板符号，因此这项 +编译取证不会改变运行期 `.text`、I-cache 布局或性能。 + +#### 审查后明确撤回的错误推论 + +初版曾尝试直接用 `current_task-H-1` 回收 ordinary ring，并用连续 writer +跨越 CAP 验证回绕。审查发现“writer 单调追加”只排除了 writer/writer +竞写，没有证明所有更早 reader 已经结束： + +```text +慢 reader M 正在扫描旧槽 + -> 快核进入 future writer N + -> N 按 task_id 推进 head 并复用槽 + -> M 的 seq 双检失败或漏掉仍应可见的 producer +``` + +因此该回收实现和门槛已在提交前撤回。R4b 的 ordinary writer 固定传 +`reclaim_upto=-1`，保持 append-only；单桶容量耗尽会 terminal fail，不用 +不成立的回收证明换取表面上的无限容量。 + +#### 当前尚未闭合的通用边界 + +R4b 证明的是“较早 writer B 不会在 reader C lookup 前缺席”，不是完整 +多版本 TensorMap： + +1. shared symbol 仍只有一个 `last_writer` cell。若 future writer D 在慢 + reader C 读取前把 cell 改成 D，C 会因 `D>=C` fail-closed；需要不可变 + writer 历史或版本化句柄，不能把单 cell 称为历史查询。 +2. ordinary ring 尚无基于最慢 reader 进度的 reclaim，且只支持由 + writer-ready replay 建立的单一有序 append actor，不支持任意 MPSC。 +3. ordinary lookup 仍只返回重叠 producer 的单一最大值；“A 写左半、 + B 写右半、C 读全区”需要 producer 集，当前 A→B→C 全重叠门槛不覆盖。 +4. CPU acquire/release 只验证宿主状态机；CCEC compile-only 只验证设备 + 接口与代码生成，尚未提供 A5 跨核 DCCI 动态证据。 +5. 新公共入口尚未从 PA 调用；现有 PA 专用 + `has_following_group` / `ChainedWriter` 路径仍是运行时权威。 + +所以下一小步不是立即删除 PA 分支，而是先选择并验证通用历史策略：symbol +必须能按 `writer < reader_task` 取最新过去版本；ordinary reclaim 必须由 +可证明的 reader progress 驱动，或在可证容量内继续 append-only。两项门槛 +闭合后，再让 PA 复用公共 WriterIntentSet，并单独删除其 task/group 特判。 + +#### 验证结果 + +| 检查 | 结果 | +| --- | --- | +| 独立 symbol/ordinary/manual A→B→C | PASS | +| 既有 fanin 保留与 previous writer 去重 | PASS | +| ownerless ordinary INPUT 查询 | 仅在显式通用模式 PASS，默认 PA 未改变 | +| CPU shared 全套构建与门槛 | PASS | +| CCEC AIC/AIV 通用模板显式实例化 | PASS | +| CCEC mixed ELF / helper / relocation / manifest | 全部 PASS | +| compile-only probe 泄漏到最终 ELF | 无 | +| ordinary reader-progress reclaim | 未实现；初版错误方案已撤回 | +| symbol future-writer 历史 | 未实现 | +| A5 跨核动态用例 | 未执行 | + +### 2026-07-27：R4c 用不可变前驱链闭合 symbol future-writer 查询 + +R4b 的 writer-ready 门只保证 B 的元数据不会在 C lookup 前缺席，不能阻止 +更快的未来 writer D 在慢 C 真正读取 `last_writer` 前把单 cell 覆盖。R4c +先把这两个问题用独立时序拆开： + +```text +A 发布 7 个 symbol + -> B 发布 writer metadata 和 writer-ready + -> 慢 C 已经越过 B gate，但停在 lookup 前 + -> D、E 依次发布并把 latest cache 从 B 推进为 E + -> C 才 lookup，必须沿 E->D->B 返回 max(writer=C` 后 fail-closed，证明 +失败来自“没有历史”，不是 writer-ready 漏门。引入不可变前驱链后，同一 +门槛转为 PASS。 + +#### 数据结构选择 + +没有把 symbol history 混入 ordinary region ring： + +- 两者的 key 域和发布协议不同，混用需要给当前单一有序 append 原型补 + MPSC reservation、publish hole 和独立容量证明； +- 默认每桶 128 槽只证明了现有 ordinary 隔离用例，host admission 没有 + 证明任意 symbol/ordinary 混合计划不会形成热桶； +- symbol 已有精确 `(descriptor producer, output slot)` 身份，不应先 + hash 再为冲突和回收增加协议面。 + +也没有复用 `SharedOutputCell::published[]` 的 56B cache-line padding。 +generic task 可以同时含 fresh `OUTPUT` 和 `INOUT`；若 history 普通 store +与 atomic `published` 共线，DCCI clean-out 需要额外证明不会把陈旧控制字 +带回 GM。这会把正确性绑在脆弱的整 task 生命周期顺序上。 + +最终在 `SharedTensorMapSidecar` 尾部追加 task-indexed table： + +```text +SharedWriterHistoryCell[4352] + header: magic, writer_task, count, reserved 16 B + record[32]: packed_symbol_key, previous_writer 256 B + alignment padding 48 B + cell total 320 B +``` + +table 共增加 `4352×320=1,392,640 B`，sidecar 从 11,027,648 B 增至 +12,420,288 B。它位于既有 `shared_vector_cursor` 之后，region ring、 +output、heap 和 Vector cursor 的所有旧 offset 均不移动。shared build +identity generation 从 6 升为 7，host 的整块 H2D/D2H 长度和 ABI 静态 +断言同步更新。 + +`packed_symbol_key=(producer_task_id×8+output_slot)+1` 是可逆的无碰撞 +编码，0 保留为无效值；record 的 writer task 由 history cell 下标隐含。 +每 task 最多 32 条，直接由 `TaskArgs` 的既有上限约束，不另造容量常量。 + +#### 发布与查询顺序 + +winner 对本 task 的全部 symbol writer 做批量处理。这里有一个不可省略 +的调用前提：同一 symbol 的 writer 必须按 task id 单调发布；迁移 shared +replay 时，后一个 writer 必须在前一个 writer-ready 之后才能进入本原语。 +CAS 能发现乱序并终止整轮，但不能补回一个已经被未来 writer 跨越的历史 +节点。 + +1. 等待 fresh descriptor 发布，读取各 symbol 的 previous writer； +2. 把 previous writer 去重并入本 task fanin； +3. 写 header 和全部不可变 record，对实际使用的连续字节做一次 + `FlushRegion`； +4. 逐 symbol 执行 `CAS(previous -> current task)`，每成功一项就累计一 + 次实际线性化的 commit； +5. ordinary metadata 也完成后，最后发布现有 `deps_prepared` gate。 + +常见 PA 三条 writer record 连同 16B header 共 40B，只清出一条 cache +line；history 不另加 count atomic。`last_writer` CAS 是对应 record 的发布 +边界：CCEC `FlushRegion` 已执行 cache-line out 和 `dsb`，reader 只有先 +原子观察到 future latest 后才会读取该 task 的 history。多 symbol CAS +不是事务：若后项冲突，已线性化的前缀保留并计数，外层设置 `fatal`，且 +不发布本 task 的 `deps_prepared`；回滚已发布的共享控制字反而会破坏故障 +现场。 + +reader 仍先读 `last_writer`： + +- `latest=reader_task`：按 latest 定位 task history，失效首行并校验 + magic/task/count；按精确 key 取 predecessor，要求 + `origin<=predecessorD`，随后 B 必须因 + `previous>=B` fail-closed，不能把 latest 倒退，也不能发布 B 的 + writer-ready。这锁定“有序 replay 是硬前提、CAS 是检测而不是修复”； +- 对含两个 symbol 的同一 task，在第一项 CAS 前人为推进第二项：第一项 + 保留为 current task，第二项保持冲突值，统计只记已线性化的一项， + `deps_prepared==-1` 且 `fatal==1`。这锁定 terminal-prefix 语义。 + +默认 PA runtime 尚未调用通用 WriterIntentSet。host 因而额外验证真实 PA +回放后的整张 history table 仍为零，防止本阶段意外改变 PA 热路径。ordinary +ring 仍保持 append-only；reader-progress reclaim、split-region producer +集合和 PA 专用分支迁移都没有被本阶段冒充完成。 + +#### 验证结果 + +| 检查 | 结果 | +| --- | --- | +| 修改前 A→B→慢 C→D 门槛 | 仅“C 应返回 B”失败，红灯定位准确 | +| 修改后 symbol A→B→慢 C→D→E | PASS，C 经两跳返回 B | +| 7 symbol 跨首条 history cache line | PASS | +| 乱序 writer fail-closed | PASS，latest 不倒退且不发布 ready | +| 多 symbol partial CAS | PASS，保留并计数成功前缀后终止 | +| ordinary/manual 既有门槛 | PASS | +| CPU shared 全套构建与自测 | PASS | +| shared ring CAP=32/64/128/256/16384 | 全部 PASS | +| CCEC AIC/AIV 通用模板显式实例化 | PASS | +| CCEC AIC/AIV entry 与 split finish | 编译通过 | +| mixed ELF / LOCAL helper / relocation / manifest | 全部 PASS | +| A5 跨核 history DCCI 动态门槛 | 尚未执行，下一独立阶段补齐 | + +### 2026-07-27：R4d 用独立 mixed ELF 闭合 A5 symbol-history 可见性 + +R4c 的 CPU 门槛只能证明不可变前驱链的状态机，CCEC compile-only probe +也只证明模板能生成 object。R4d 不修改普通 PA kernel/host，新增一套 +shared-only 的独立 mixed AIC/AIV 门槛，专门验证跨物理核 DCache 可见性。 + +#### 先修复 compile-only 掩盖的真实链接缺口 + +把通用 `PrepareSharedWriterIntentSet` 放入可启动 ELF 后，AIC/AIV +object 最初各带有 24 个 `__multi3` relocation，`ld.lld -m aicorelinux` +因符号未定义而失败。查验本机 CANN 9.1 后确认： + +- `ccec -print-libgcc-file-name` 返回的是宿主 x86_64 `libgcc.a`； +- HCC 自带的是 AArch64 `libgcc.a`，与 `aicorelinux` 明确不兼容； +- CANN 包中没有可供 AICore 静态链接的 compiler-rt/builtins； +- `-rtlib` 对 device `-c` 不生效，不能把宿主运行库错链进 mixed ELF。 + +微基准进一步把根因收敛到 CCEC 对 +“`UINT64_MAX/rhs` 溢出预检 + 64-bit 乘法”的宽乘融合，而不是普通的 +`uint64_t * uint32_t`。公共 ordinary-region 校验因此改用两次 +`u32×u32→u64` 的 limb 乘法： + +```text +left = H*2^32 + L +left*right = + (H*right + floor(L*right/2^32))*2^32 + + low32(L*right) +``` + +只有 `H*right + carry <= UINT32_MAX` 时结果可由 `uint64_t` 表达。该写法 +保留原来的 64-bit extent 取值域，没有用“限制到 `UINT32_MAX`”绕开通用 +语义。byte range 的 dtype 大小只可能为 1/2/4/8，改用带上界检查的 shift， +同样不需要宽乘 helper。CPU 定向门槛新增： + +- 恰好 `UINT32_MAX` 的连续 extent； +- 大于 `UINT32_MAX` 但仍合法的连续 extent； +- 真正超过 `UINT64_MAX` 的 shape product； +- 大于 `UINT32_MAX` 的 non-contiguous cached extent； +- dtype 缩放后的 byte-range overflow。 + +普通 shared 构建也从“模板编译到 `/dev/null`”加强为 AIC/AIV 各自真实 +静态链接后再删除 probe。这样后续任何未解析 device builtin 都会直接阻止 +构建，不能再以 compile-only PASS 冒充可执行。 + +#### 失效操作的编译器顺序 + +对 CANN 9.1 生成的 LLVM intrinsic 属性和 AIC/AIV 机器码做了单独审计。 +当前 resolver 恰好生成： + +```text +DCCI -> DSB -> load history header/record +``` + +但 DCCI/DSB 只声明访问 inaccessible memory，并不约束普通 GM load。 +最小反例中的“失效前 load、DCCI、DSB、失效后 load”会被 CCEC `-O3` +合并成零次 GM load。为此在 `CcecOps::InvalidateRegion()` 的 DSB 后增加： + +```cpp +__asm__ volatile("" ::: "memory"); +``` + +它不生成设备指令，只建立源码级 compiler ordering，保证调用方的普通 +header/payload load 不会上提到失效之前。没有给热路径增加第二个 DSB， +也没有改变 atomic 的发布/完成语义。 + +#### A5 动态门槛的精确时序 + +`ccec/history_litmus_*` 只依赖本目录的 `common/`、CCEC/ACL/runtime， +artifact 固定在 `build/ccec/shared/history-litmus/`。每个 host 进程只跑 +一个方向，两个方向分别为： + +```text +AIC block0/1 写 B/D/E -> AIV 物理 block4 的 reader C +AIV block0/1 写 B/D/E -> AIC block2 的 reader C +``` + +每个方向都使用独立 task id。host 初始化 A 的 7 个 symbol，D/E history +两条 cache line 全零；device 执行： + +```text +B 发布 7 条 history + 7 次 latest CAS + writer-ready + -> C 越过 B gate + -> C 普通 load 预热 D/E header 和 record[6] 所在第二条线，必须读到零 + -> C 发布 reader-past-B gate + -> D、E 依次发布 14 条 history/CAS + -> C 观察 future-done 后调用真实 CollectSharedFanin + -> resolver 失效 history，沿 E->D->B 返回 B +``` + +预热值直接决定 C 是否发布下一道 gate，并写入 device 独占结果行，不能被 +编译器删除或挪到 future writer 之后。host 不只看最终 sidecar，还逐项验证： + +- 预热两条线都为零； +- B/D/E 共 21 次实际 symbol CAS； +- 七个 latest 全部为 E； +- B/D/E 三份 immutable history 的 21 条 key/predecessor； +- C 的 device fanin 只有 B； +- writer-ready 全部关闭而 kernel completion flag 保持零； +- 反方向 task/history 和 ordinary region ring 均未触碰。 + +构建和运行入口为： + +```bash +./run.sh build-history-litmus ccec +./run.sh history-litmus ccec --device 0 --runs 20 +``` + +`--runs 20` 表示每个方向各启动 20 个全新 host 进程，不在同一 runtime +会话里复用偶然状态。2026-07-27 的正式上板结果为 **20×2 全部 PASS**： +40 个独立 host 进程都通过 12 项语义断言以及 ACL 资源清理；没有出现 +fatal、超时、错误 fanin、非零预热值或反方向/ordinary-ring 污染。 + +提交前同时完成以下静态与主机回归： + +| 检查 | 结果 | +| --- | --- | +| `bash -n`：top-level run、history runner、CCEC build | PASS | +| CPU shared 全套：五种 ring CAP、symbol/history、heap、Vector、materialize、loser | PASS | +| ordinary 64-bit range：合法大 extent 与 dtype/shape overflow | PASS | +| CCEC shared 正式构建：generic AIC/AIV probe 真实静态链接 | PASS | +| 正式 1:2 mixed ELF：entry/metadata/LOCAL helper/state/relocation | PASS | +| history mixed ELF：双 entry/metadata、无 `__multi3`、无未解析全局符号 | PASS | +| A5 history：AIC writers→AIV reader，20 个新进程 | PASS | +| A5 history：AIV writers→AIC reader，20 个新进程 | PASS | +| manifest/trap 收口后重建并双向各复测 1 个新进程 | PASS | +| `git diff --check` | PASS | + +CPU 首次误用本用户 plucky GCC 15 搭配系统 binutils 2.42，汇编器不识别 +编译器输出的 `.base64`；该轮在任何测试执行前即编译失败，不计作源码 +结果。CPU 有效全套回归显式使用仓库此前已验证的系统 GCC 13。CCEC device +编译与 A5 上板仍全部使用本用户 CANN 9.1，host runner 使用系统 GCC 13； +不能把 host 汇编器组合问题写成 device 协议失败。 + +提交前独立复核没有发现阻断项；两个低风险工程缺口已当场收口： + +- generic probe 放入独立子 shell，并用 `EXIT` trap 清理四个隐藏 object/ELF， + 任一步编译或链接失败都不会留下半成品，同时不覆盖后续 manifest trap； +- history runner 从被 C++ `static_assert` 绑定当前 build identity 的共享头 + 读取 ABI generation；运行前要求 manifest 精确六行、双向标签和两个固定 + artifact 名称，再验证 SHA256，避免未来 ABI 升级后标签静默漂移。 + +本门槛是当前真实 DCCI/CollectSharedFanin 路径的正向跨核可见性证据，没有 +另造“去掉 DCCI”的负向镜像，因此不把结果越界解释成 DCCI 必要性的单变量 +因果证明。 + +#### 本阶段边界 + +- 普通 PA ELF 仍未调用通用 WriterIntentSet，故其性能/I-cache 布局未被 + 该门槛改变； +- 本门槛只闭合 symbol immutable history，不冒充 ordinary region 的 + reader-progress reclaim 已完成； +- 下一步先设计 ordinary reader 进度的独立 cache-line 发布协议，再做 + A→B→慢 reader→未来 writer→回收门槛，最后才迁移 PA 并删除专用分支。 + +### 2026-07-27：R4e-a 建立独立 ordinary reader 完成前沿 + +R4b 曾尝试从有序 writer turn 直接推导 ordinary ring 的安全回收上界， +但 writer 已按 task 排序并不等于所有 reader 已经结束。R4e-a 先只解决 +“如何表达并验证 reader 已完成到哪里”这一项，不同时打开回收、改写通用 +Submit 或迁移真实 PA。 + +#### 为什么不能复用 `WorkerState::local_index` + +`local_index` 是 task 分配/回放游标，不是 reader 完成前沿。 +`BeginCallbackSubmit()` 的第一步就是： + +```cpp +const uint32_t task_id = + static_cast(worker.local_index++); +``` + +此时该 task 的 Claim、参数构造、fanin 和 ordinary-ring lookup 都还没有 +发生。把 `local_index` 或 `local_index-1` 当成 reader 完成值，会在当前 +task 仍可能读取旧 producer 时提前回收。此外，96 个 `local_index` 分散在 +约 887MiB 的 worker arena 内，是普通 `int32_t`，没有独占 cache line、 +atomic 发布或 A5 DCache 可见性合同。 + +因此 shared sidecar 尾部新增： + +```cpp +AtomicLine reader_done[kWorkers]; +``` + +每个 worker 独占 64B，初值为 -1；`reader_done[worker]=D` 只表示该 +worker 已关闭 task `[0,D]` 的全部 ordinary-ring 读取。当前普通 PA 没有 +调用发布原语，host 会反向要求所有 96 条线在运行后仍为 -1。 + +#### 单调发布与回收候选 + +`SharedAdvanceReaderDone()` 只允许单次 CAS： + +```text +expected = task_id - 1 +desired = task_id +``` + +CAS 返回 observed；只有 `observed==expected` 才成功。重复、跳号和倒退 +都失败且不覆写现值，worker/task 越界在 atomic 前拒绝。原语自身不设置 +`fatal`：以后运行期调用者必须在拥有 `SchedulerState` 和 worker 上下文的 +层次统一终止，不能让底层 helper 猜测故障归属。 + +活跃 worker 是连续前缀 `[0,active_workers)`。聚合 helper 只读取此前缀， +要求每个值都在 `[-1,kMaxTasks)`，先在局部变量中完成全量校验，成功后才 +写输出。令最慢完成值为 `Dmin`、依赖窗口为 `H`，inclusive 安全回收上界为： + +```text +candidate = max(-1, Dmin - H) +``` + +推导为：所有 worker 已完成到 `Dmin` 后，最早尚可能读取的 task 是 +`Dmin+1`；其 lookup 下界为 `Dmin+1-H`，故该下界之前的最后一个 producer +是 `Dmin-H`。设计文档使用的 +`R=min_progress-H-1` 以“下一 task 进度”为口径，而这里 +`min_progress=Dmin+1`，两式等价。示例 +`reader_done={9,5,8}, H=2` 的最小值为 5，候选精确为 3；不能沿用旧 +current-task 口径再多减 1。 + +扫描不需要同一时刻的一致快照：每条线只有对应 worker 单写并严格单调， +逐行 load 得到的混合快照至多更保守，不会把尚未发布的 reader 进度凭空 +提前。该结论只覆盖状态机；A5 上真正使用前仍必须证明 ordinary 普通读取 +不能被编译器或设备重排到 `reader_done` 发布之后。 + +#### ABI 与初始化 + +新数组严格追加在 `writer_history` 之后，不移动任何既有字段： + +| 项目 | R4e-a generation 8 阶段值 | +| --- | ---: | +| `writer_history` offset | 11,027,648B | +| `reader_done` offset | 12,420,288B | +| `reader_done[96]` | 6,144B | +| shared sidecar | 12,426,432B | +| shared non-split `SchedulerState` | 1,019,542,400B | +| shared split `SchedulerState` | 1,019,548,544B | + +默认 CAP=128 的 shared build identity 从 generation 7 升到 8； +private generation 4、2,113,664B sidecar 以及 +1,007,115,968/1,007,122,112B non-split/split `SchedulerState` 全部 +不变。host 正式初始化、ring/symbol 测试初始化和 history-litmus 专用 host +都显式写入 96 个 -1，不依赖 sidecar `memset(0)`。history manifest 从 +被 C++ `static_assert` 绑定当时 build identity 的共享头读取 generation 8， +旧 generation-7 artifact 会在启动前被拒绝。 + +#### CPU 门槛与设备边界 + +CPU 门槛复用现有 `test_shared_writer_intent.cpp` 和 +`WriterIntentTestOps`，没有再造一份 Ops 或测试 executable。它覆盖： + +- 96 条线的 -1 初值； +- `-1->0->1` 连续发布，以及重复、跳号、倒退不覆写； +- 最后一个合法 task、worker/task 参数边界； +- 三个 active reader 的最小值、inactive 脏值忽略和 active 脏值拒绝； +- 快 reader 单独前进不改变候选，慢 reader 前进才推动候选； +- 非法 worker count、负窗口、非法进度都不污染输出； +- `BeginCallbackSubmit` 真实把 task 5 的 `local_index` 推到 6 时， + `reader_done=4,H=2` 仍只得到 candidate 2。 + +本阶段的验证结果为： + +| 检查 | 结果 | +| --- | --- | +| CPU shared 五种 ring CAP 与全部公共自测 | PASS | +| CPU private 五种 ring CAP 与公共自测 | PASS，private ABI 未改变 | +| CCEC shared 既有 generic probe、AIC/AIV、split finish、mixed ELF、manifest | PASS；新 reader helper 尚未显式实例化 | +| generation-8 history mixed ELF 重建 | PASS | +| AIC writers→AIV reader，新 host 进程 1 次 | PASS | +| AIV writers→AIC reader，新 host 进程 1 次 | PASS | +| history 门槛保持 ordinary ring 与 96 条 progress 线不变 | PASS | +| `git diff --check` | PASS | + +R4d 的 generation-7 20×2 上板结果仍是当时 symbol-history 可见性的历史 +证据；本轮 1×2 只验证 ABI 尾部变化、重新生成的 artifact 身份和既有 +symbol 路径未受影响，不能冒充 reader-progress 的 A5 发布门槛。 + +#### 本阶段明确没有完成的事项 + +- 没有从 PA 或 generic Submit 发布 `reader_done`； +- 没有用 candidate 更新 `reclaim_upto`、bucket head 或复用任何 slot； +- 没有改变 `CommitOrdinarySharedWriterIntent()` 的 append-only 行为； +- 没有证明 A5 ordinary read、DCCI 与 progress CAS 的 compiler/device + 顺序； +- 没有解决 ordered append actor 与未来回收的完整组合。 + +下一阶段用满桶和慢 reader 的确定性交错证明“未关闭 reader 时绝不复用， +关闭并越过 H 后才可回收”；随后才增加独立 CCEC/A5 reader→reclaimer +可见性门槛。 + +### 2026-07-27：R4e-b 用满环交错闭合 CPU reader-gated reuse + +R4e-a 只证明 `reader_done` 的单调状态机和回收候选公式，没有把候选接到 +任何 bucket head。R4e-b 仍不接 PA/Submit，而是在 shared ordinary-ring +隔离 driver 内补上最薄的组合层，回答一个更具体的问题： + +> future writer 已拿到 ordered turn、目标桶又恰好写满时，只要仍有一个 +> active reader 没有关闭其合法 lookup 窗口，是否可能提前回收并复用慢 +> reader 正在读取的物理槽？ + +#### 最薄组合原语及其所有权边界 + +既有 `SharedRefreshReclaimForTask()` 的 `current_task-H-1` 公式只适用于 +单线程 ordered-ring 旧 driver，不能拿来替代多 reader 完成证据。本阶段 +增加： + +```cpp +SharedRefreshReaderReclaimForTask( + map, current_task, active_workers, heap_window, reclaim_upto +) +``` + +它只按顺序复用三项既有事实： + +1. `committed_tasks==current_task`，拒绝陈旧或超前 writer； +2. `SharedComputeReaderReclaimCandidate()` 对固定 active-worker 前缀取 + `max(-1,min(reader_done)-H)`； +3. 把候选单调发布到 `reclaim_upto`。 + +候选发布逻辑从旧 refresh 中原样抽为 +`SharedPublishReclaimCandidate()`，旧单线程 driver 继续走原公式，行为 +没有改变。当前发布仍使用既有单 writer `Exchange` 语义；exact turn +本身不证明 actor 唯一，调用者必须已经完成 winner/turn 所有权收敛。若 +误让多个 actor 同时发布，`Exchange` 的失败检查不能自动恢复被覆盖的 +控制字，因此本 helper 绝不能被解释为多 writer 仲裁原语。R4e-e 会在真实 +接线前单独比较 CAS 与 Exchange 的设备成本，再决定最终发布方式。 + +新 helper 不扫描 bucket、不 append、不设置 fatal，也没有 PA 调用者； +因此本阶段没有改变默认 PA 的 atomic 次数、代码布局或性能。 + +#### 确定性交错不是“先后调用”的近似 + +测试在现有 `RecordingOps::InvalidateRegion()` 增加一次性回调。回调点 +位于 `SharedReadRegionSlot()` 第一次确认绝对 `seq` 之后、复制 payload +之前；进入回调前先清空 hook，因此 future writer 的嵌套 preflight 可以 +读取同一物理槽而不会递归。它制造的是确定的源码级交错，不模拟 A5 cache +line，也不冒充 DCCI 证据。 + +五种 CAP 都使用同一组合同： + +| 参数 | 固定值或公式 | +| --- | --- | +| active reader | worker 0 和 worker 1 | +| lookup 窗口 `H` | 2 | +| 每个 fill task 的 region 数 | 固定 8，且编译期要求不超过 `kMaxTaskTensors` | +| fill task 数 | `CAP/8` | +| future writer task | `CAP/8` | +| 目标 bucket | 所有 region 使用同一 `buffer_addr` | +| replacement 数 | 8 | + +固定每 task 8 条很重要。开发中第一次把它写成 +`kMaxTaskTensors`，而当前该常量是 32；CAP=32 于是只产生一个 fill +task，快 reader 实际只能完成到 0，关闭慢 reader 的 task 2 后候选当然 +仍为 -1。这个首个失败准确暴露的是测试初始化状态与注释不一致，不是回收 +公式错误。 +修正后增加 `CAP/8>H` 的编译期断言，避免以后常量变化悄悄破坏交错前提。 +CAP=16384 时也只使用 2,048 个 task，仍小于 `kMaxTasks=4352`。 + +#### 慢 reader 未关闭时必须全量不写 + +先用真实 shared append/commit primitive 把同一 bucket 精确填到 CAP： + +```text +head=0 +tail=CAP +committed_tasks=CAP/8 +reclaim_upto=-1 +physical slot 0: seq=0, producer=0 +``` + +随后 worker 0 连续发布 `reader_done=1`，表示仍在 task 2 的 ordinary +lookup；worker 1 连续发布到 future writer 前一 task。task 2 查询 +producer 0 时，在 cursor 0 的首次 seq 检查后触发 future writer： + +```text +Dmin=min(1, future_task-1)=1 +candidate=max(-1,1-2)=-1 +``` + +future writer 已持有 exact turn，但 refresh 只能保持全局 +`reclaim_upto=-1`；整任务 preflight 必须返回 `CapacityBlocked`。回调在 +reader 仍暂停时立即记录并断言： + +- `head=0`、`tail=CAP`、`committed_tasks=future_task`； +- slot 0 仍是 `seq=0, producer=0`； +- 事件账本没有 `Exchange`、`CompareExchange` 或 payload `Flush`； +- 外层 lookup 恢复后仍返回 producer 0。 + +这里不仅检查“最终没有 commit”，还检查 payload/seq/tail/head/reclaim +均没有中间发布，避免把局部写入后回滚误判成 all-or-nothing。 + +#### lookup 返回后才允许精确回收和回绕 + +只有外层 lookup 完整返回后，worker 0 才以 CAS 把 +`reader_done:1->2`。此时： + +```text +Dmin=2 +candidate=max(-1,2-2)=0 +``` + +同一个唯一 ordered actor 发布 `reclaim_upto=0`。preflight 惰性推进 +`head:0->8`，恰好退休 task 0 的八条 region；随后八条 replacement +依次追加到绝对 cursor `[CAP,CAP+8)` 并提交 future task。最终断言： + +- `tail=CAP+8`、逻辑 bucket 大小仍为 CAP； +- cursor CAP 与 cursor 0 映射到同一 physical slot，但新绝对 + `seq=CAP`、payload 只含 future producer； +- 旧 cursor 0 读取失败，新 cursor CAP 读取成功； +- task 3、`H=2` 的 lookup 继续遵守窗口下界，不接受 producer 0。 + +最后一项只复核 lookup 时间窗，不能单独证明 producer 0 已经物理退休； +退休证据来自此前的 `head=8`、slot 0 绝对 `seq=CAP` 和旧 cursor 读取 +失败三项组合。 + +这闭合的是 CPU 状态机上的 +“慢读未关闭 -> 满桶阻塞 -> 读完发布 -> 精确回收 -> 绝对序号防 ABA” +完整链路，不是仅验证一个候选整数。 + +#### 验证结果与仍未闭合的设备事实 + +| 检查 | 结果 | +| --- | --- | +| shared ring CAP=32/64/128/256/16384 | 全部 PASS | +| 上述五档 ASan+UBSan | 全部 PASS，无越界或未定义行为 | +| CPU shared 全套公共门槛 | PASS | +| CPU private 五档 ring | PASS，无旁路回退或 shared 宏泄漏；private 不执行新 helper | +| CCEC shared AIC/AIV、split finish、mixed ELF、manifest | 构建 PASS | +| 新 reader refresh 的 CCEC 显式实例化 | 本阶段尚未增加 | +| A5 reader→reclaimer 跨核动态门槛 | 本阶段尚未执行 | + +CPU 构建和 sanitizer 使用系统 GCC 13；CCEC/device 使用本用户 CANN 9.1， +host 使用系统 GCC 13。本轮没有复用用户目录 GCC 15 与系统 binutils 2.42 +的不兼容组合。 + +R4e-b 没有证明 ordinary GM 读取一定先于 `reader_done` CAS，也没有证明 +reclaimer 在另一物理核上一定看见最新 reader 前沿。下一阶段 R4e-c 先让 +AIC/AIV 各自显式实例化新 helper，再用独立 mixed ELF 建立 +“reader 普通读取完成并发布 -> reclaimer 跨核观察 -> 允许/禁止复用”的 +A5 门槛；在该证据闭合前仍不迁移真实 PA。 + +### 2026-07-27：R4e-c1 先闭合 reader/reclaimer 的 CCEC 代码生成 + +R4e-b 的 shared 正式构建会解析 `pa_shared_tensormap.h`，但普通 PA +kernel 尚无 reader-progress 调用者，模板体即使存在签名或 device builtin +问题也可能从未实例化。R4e-c 不直接跳到动态上板，先把这一层单独闭合。 + +原 `prepare_shared_writer_intent_compile_probe.cpp` 已扩为并更名为 +`shared_protocol_compile_probe.cpp`。同一个 TU 继续显式实例化 +`PrepareSharedWriterIntentSet`，并新增： + +```cpp +SharedAdvanceReaderDone(...) +SharedRefreshReaderReclaimForTask(...) +``` + +CCEC build 对 dav-c310-cube 与 dav-c310-vec 分别生成 object，再各自用 +`ld.lld -m aicorelinux -static` 真实链接。四个隐藏 probe artifact 仍处于 +独立子 shell 的 `EXIT` trap 下，成功或失败都会删除；它们不加入正式 +`DEVICE_OBJECTS`，不会改变 PA mixed ELF、I-cache 布局或运行性能。 + +本阶段同时把 build 内的变量、清理函数和日志从 writer-intent 专名改为 +shared-protocol，避免后续把 reader helper 的构建闭环误认为只覆盖 symbol +writer。 + +验证结果： + +| 检查 | 结果 | +| --- | --- | +| `bash -n ccec/build.sh` | PASS | +| AIC generic shared-protocol object + static ELF | PASS | +| AIV generic shared-protocol object + static ELF | PASS | +| 正式 AIC/AIV entry 与 split-finish | 构建 PASS | +| 正式 1:2 mixed ELF、LOCAL helper、relocation、manifest | 全部 PASS | + +这只证明真实 CcecOps 的 GM 地址空间、CAS 签名和 reader candidate 组合在 +AIC/AIV 上都能完成代码生成与静态链接。它没有运行 reader，也没有证明 +ordinary GM load、`reader_done` CAS 和远端 reclaimer 观察之间的 +compiler/device 顺序；R4e-c2 仍必须使用独立 mixed ELF 双向上板。 + +### 2026-07-27：R4e-c2a 先把 history 门槛泛化为 shared protocol 载体 + +R4e-c2 需要新增 ordinary reader→reclaimer 场景，但原 +`history_litmus_*` 已包含约 1 GiB 稀疏 host state、ACL 生命周期、 +AIC/AIV mixed ELF 构建、artifact 哈希和跨进程运行框架。复制一套只会让 +两份初始化、清理和产物身份逻辑逐渐分叉。因此本小步只迁移基础设施，不加 +reader-reclaim 算法： + +```text +history_litmus_{shared,kernel,host}.* + -> shared_protocol_litmus_{shared,kernel,host}.* +history_litmus.sh + -> shared_protocol_litmus.sh +``` + +新 control 仍严格占一条 64B cache line，但 magic、version 和 schema 都 +重新建立身份，并显式携带 `scenario`： + +```cpp +enum class Scenario : uint32_t { + SymbolHistory = 1, +}; +``` + +本阶段只接受 `history`。host CLI、device dispatcher 和 manifest 三处 +分别拒绝缺失或非法 scenario，不能静默落入 history；direction 也改为 +scenario 无关的 `AicToAiv/AivToAic`，为下一场景复用方向枚举。原 +history 初始化与验证分别改名为 `InitializeHistoryState()` 和 +`ValidateHistory()`，算法、participant 和 12 项语义断言保持独立，没有 +合并成带 optional 分支的弱验证函数。 + +新的唯一入口和 artifact 为： + +```bash +./run.sh build-shared-protocol-litmus ccec +./run.sh shared-protocol-litmus ccec \ + --scenario history --device 0 --runs 20 +``` + +```text +build/ccec/shared/shared-protocol-litmus/ + shared_protocol_litmus_host + shared_protocol_litmus_kernel.o + shared_protocol_litmus_artifacts.manifest +``` + +旧 action 和旧脚本不保留第二份实现。R4d 的旧 commit/SHA 与 20×2 结果仍 +是当时 artifact 的历史证据；泛化后的 ELF 身份已经变化，所以后续新增 +reader-reclaim 后，history 也必须在最终同一 artifact 上重新跑 20×2， +不能直接继承旧结论。 + +本小步验证结果： + +| 检查 | 结果 | +| --- | --- | +| top-level runner 与新 litmus runner `bash -n` | PASS | +| AIC/AIV `-O3` object，无 `__multi3`/未解析全局符号 | PASS | +| mixed ELF 双入口、metadata、无额外 GLOBAL function/relocation | PASS | +| host 构建与 manifest SHA 自校验 | PASS | +| history AIC→AIV，新 host 进程 1 次、12 项语义断言与清理 | PASS | +| history AIV→AIC，新 host 进程 1 次、12 项语义断言与清理 | PASS | + +该提交只证明泛化没有改变既有 history 行为。它没有新增 reader-progress +设备证据，也没有据此宣称普通 GM payload 读取先于 `reader_done` CAS； +后者仍是 R4e-c2b 的独立场景目标。 + +### 2026-07-27：R4e-c2b 建立满环 ordinary reader→reclaimer A5 门槛 + +R4e-a 只建立 reader 完成前沿，R4e-b 只在 CPU ring driver 闭合 +“慢 reader 阻塞复用→reader close→精确退休→绝对 seq 防 ABA”，R4e-c1 +也只证明模板能在 AIC/AIV 生成并链接。真实 PA 尚未接入这些 helper，因此 +本阶段继续使用 R4e-c2a 的独立 shared-protocol mixed ELF，不跳到 PA +benchmark。 + +#### 满环镜像只复刻 production-reachable ordinary-ring 切片 + +目标 bucket 固定 CAP=128，初始 append 历史严格对应： + +```text +task 0: cursor 0 1 entry +task 1: cursor 1..32 32 entries +task 2: cursor 33..64 32 entries +task 3: cursor 65..96 32 entries +task 4: cursor 97..127 31 entries +committed_tasks = 5 +head/tail = 0/128 +``` + +合计 `1+32+32+32+31=128`，不是随意填满数组后脑补成生产状态。96 条 +`reader_done` 中，95 条由 host 预置为 task 2，只有被测 reader 停在 +task 1。`H=2` 时第一次 +`SharedRefreshReaderReclaimForTask(task=5)` 必须得到 `-1`，真实 +`SharedCheckTaskAppend()` 必须返回 `CapacityBlocked`。这只说明 +ordinary-ring 相关切片生产可达；未参与场景的整个 `SchedulerState` 仍按 +隔离门槛清零，不冒充完整 PA reset 镜像。 + +旧 cursor 0 为： + +```text +address=0x700000000, lo=0, hi=8, producer=0, reserved=0 +``` + +replacement 为： + +```text +address=0x7000014c0, lo=4096, hi=4128, producer=5, reserved=0 +``` + +两地址真实 hash 到同一个 bucket，但除协议要求恒为零的 `reserved` 外， +四个可变字段全部不同，避免只改 producer 后无法暴露其他字段的晚读。 + +reader/reclaimer 固定用两组不同物理核双向执行： + +```text +AIC reader worker 3 -> AIV reclaimer worker 42 +AIV reader worker 35 -> AIC reclaimer worker 4 +``` + +reader 先等 reclaimer 取证 `CapacityBlocked`，再读取 cursor 0。reclaimer +不借助另一个“reader 已完成”门，而是直接轮询真实 +`reader_done[reader_worker]`；观察 `1->2` 后重新 refresh，必须得到 +`reclaim=0`，随后真实 check/append/commit： + +```text +head/tail: 0/128 -> 1/129 +physical slot 0: seq 0 -> 128 +committed_tasks: 5 -> 6 +``` + +这里提交的是 writer task 5；`committed_tasks=6` 表示已提交计数前沿已经 +越过 task 5，不表示 task 6 已经提交。 + +append 的 payload `FlushRegion()` 已包含 DCCI clean-out 与 DSB，随后 +seq、tail、commit 都消费 atomic 返回值。`reuse-done` 只有在 append 与 +commit 都返回成功后才 CAS 发布，不能在失败路径提前放行 reader。 +reader 等 gate 成功后把同一个动态 signal 传给 noinline snapshot +validator；validator 自己由该 signal 计算 `TaskCell::deps_prepared` 地址， +做第二次真实 atomic load，并只在 token 等于 signal 的 CFG 成功分支读取、 +校验本地旧快照。这避免把一个可被 CCEC 常量折叠的 bool 冒充“验证一定 +发生在复用之后”的证据。 + +#### 三种 reader-close 口径 + +device 侧 `CaptureReaderSnapshot()` 与生产 `SharedReadRegionSlot()` 使用 +同形 raw 顺序： + +```text +seq_before atomic load +payload DCCI invalidate + DSB + compiler clobber +buffer_addr/lo/hi/producer/reserved 五字段普通 GM load +seq_after atomic load +``` + +但它故意不在 CAS 前判断 seq/payload，而把判断推迟到复用完成后；否则控制 +流会先消费 payload，三种 close 就失去区分意义。因此这里验证的是这条 +有针对性的 raw 序列，不是直接调用生产 `SharedReadRegionSlot()`。 + +| ordering | CAS 前约束 | 本阶段可解释口径 | +| --- | --- | --- | +| `compiler-clobber` | `noinline` close 调用，callee 内为空 asm memory clobber；无 device barrier | 当前 noinline artifact 的最弱动态对照 | +| `payload-dependency` | 五字段作为 `3×i64+2×i32` noinline 实参，8 步 FNV checksum 经 tied MOV 形成 delta，CAS expected/desired 同时依赖 delta | scalar 值已被消费的窄依赖，不是全设备访存屏障 | +| `dsb-all` | compiler clobber→`DSB_ALL`→compiler clobber | 三者中唯一有“全部访存指令”设备完成口径的兜底 | + +三个 close 都是 `noinline`；三条路径因此共享调用边界带来的 compiler +ordering 和 call/return 延迟。该边界可能让未决 GM load 自然完成，所以 +compiler-clobber 通过只能归于当前 noinline artifact，不能外推为 inline +空 clobber 充分。 + +#### 从“存在函数”收紧为 O3 use-def 门槛 + +构建为 AIC/AIV 分别增加与 object 同一源码和主体选项的 O3 +bitcode/textual IR；自动门槛检查 `.ll`。manifest 哈希 host、最终 mixed +ELF 和 AIC/AIV `.ll`，不哈希 `.bc`；bitcode 仅作为可复查中间产物保留。 +自动门槛逐核检查： + +1. compiler 路必须是唯一 clobber 先于常量 `CAS(1,2)`，且没有 DSB； +2. dependency helper ABI 必须保留三组 64b、两组 32b payload leaf； +3. dependency 调用的五个 leaf 必须是直接 `addrspace(1)` GM load； +4. 三组 64b 的上下半部和两组 32b 必须完整进入恰好 8 步 checksum； +5. 最后一轮 checksum 必须依次到 tied MOV、delta、i64 扩展、 + `1+delta/2+delta`，再成为 CAS 两个动态参数； +6. DSB 路必须精确保持 + `clobber→DSB_ALL→clobber→CAS(1,2)`； +7. validator 用同一个动态 signal 选择 `TaskCell::deps_prepared`、执行 + atomic load 并比较返回 token； +8. token 比较成功块只能由比较的 true 边进入，seq 与五字段必须是 validator + 全函数仅有的五次普通 payload load，并沿 AND 数据流共同决定唯一 result + phi 和最终返回；其他 phi 输入只能返回 false； +9. 三个 close 和一个 validator 各保留唯一真实调用。 + +额外用“成功块外增加 payload load、给成功块增加第二入口、让不完整检查 +进入 result phi、加入 `true` phi 旁路、增加额外 `ret true`”五种文本变异 +反向确认门槛都会拒绝。它仍不是最终 object 的机器码反汇编：`.ll` 与 `.o` +是相同源码/主体选项下两次独立 CCEC 编译。动态 A5 执行补的是行为证据， +不能把二者合写成精确机器指令证明。 + +#### 过程中撤回的测试构造错误 + +本小步按审计结果逐项收敛，而不是保留过程态： + +1. 初稿用 `committed_tasks=3` 配满 128 槽，无法由每 task 最多 32 条的 + append 历史到达；已改为上述 task 0..4 的 1/32/32/32/31 分布。 +2. 初稿在 reader CAS 前判断 snapshot，等于测试代码自己先消费 payload； + 已移到 cursor 128 复用并收到 reuse token 之后。 +3. 初稿 replacement 只改变 producer；已让 address/lo/hi/producer 全变。 +4. 初稿 `reuse-done` 无条件发布，可能在 append/commit 失败时让 reader + 提前验证旧值；已改为成功返回值短路依赖。 +5. 初稿只检查阻塞槽 producer；已扩大到 seq、五字段、head/tail、commit。 +6. 初稿只检查非目标 bucket 控制；已补 16K 非目标 physical slots、反方向 + reader-reclaim 结果/gate、全部 `shared_outputs.published/last_writer` + 控制字和 `writer_history` header 未触碰断言。 +7. 初稿 IR 门槛只检查 helper 形状；已收紧为上述 leaf/use-def/token + 数据流。 + +#### 验证命令、结果与结论边界 + +最终构建和正式重复使用同一个 manifest 身份： + +```bash +source "$HOME/Ascend/cann-9.1.0-weekly-20260708/cann-9.1.0/set_env.sh" +export CXX=/usr/bin/g++ +./run.sh build-shared-protocol-litmus ccec +./run.sh shared-protocol-litmus ccec \ + --scenario all --ordering all --device 0 --runs 20 +``` + +`scenario=all` 每轮先跑 history 两方向，再跑 reader-reclaim 的 +`2 directions × 3 orderings`，每个 tuple 都启动全新 host 进程。 +每个进程有 60 秒 host 上限，超时直接使整批失败且不自动重试，避免把 ACL +stream 异常停滞或协议偶发等待静默跳过。 + +正式结果只采用确认 device 0 无外部任务后的完整批次；此前与另一组 +`--device-id 0` 任务重叠的过程运行全部丢弃，不计入结论。冻结 artifact +先完成 `runs=1` 的 8/8 个进程，再完成 `runs=20`： + +| 场景 | 展开方式 | 结果 | +| --- | --- | --- | +| symbol history | 20 轮 × 2 个方向 | 40/40 PASS | +| reader-reclaim | 20 轮 × 2 个方向 × 3 种 ordering | 120/120 PASS | +| 合计 | 160 个全新 host 进程 | 160/160 semantic PASS、cleanup PASS | + +整批失败行 0，60 秒上限未触发。manifest SHA-256 为 +`1c10093a2527fd2b80da6332bf8e03bc381b5bb08f740239264d0ffc98cd240d`。 +该结果证明当前固定双向物理核和受控场景在当前 A5/CANN artifact 上重复 +成立；不扩大下面的协议结论边界。 + +即使三种 ordering 全部重复通过，也不证明 compiler-clobber 或 +payload-dependency 是 A5 架构充分条件，更不能直接决定生产方案。当前场景 +只覆盖 CAP=128、单 bucket、单 replacement、固定两组物理核，以及一次预期 +`CapacityBlocked`→close→成功 append/commit 链；不覆盖 `ProtocolError`、 +reader CAS 失败、append/commit 失败、多 reader 并发 close，或复用后的新 +device reader。 +production `SharedAdvanceReaderDone()` 仍没有 read→CAS 顺序，真实 PA 也 +尚未在每个 active worker、每个 task 的全部成功/loser/空任务出口连续发布 +前沿。下一阶段必须先在 generic ordinary flow 闭合 +“read-only lookup→reader-close→后继 append”，再讨论接入 PA。 + +### 2026-07-27：R4e-d1 消除合法前缀回收导致的两类 lookup 假失败 + +R4e-c 已证明“reader 仍可能选择的 producer 不能被复用”，但 production +`SharedLookupRegion()` 还隐含了另一个没有成立的假设：从第一次读取 bucket +`head` 到扫描结束，head 不会变化。reader task `N` 尚未关闭时，未来唯一 +writer 仍可合法回收： + +```text +producer <= N - 1 - H +``` + +而本 reader 的查询窗口下界为： + +```text +producer >= N - H +``` + +前一集合严格位于查询窗口之外，回收本身没有数据依赖错误。但 reader 保存 +旧 head 后存在两类合法交错： + +1. writer 在 reader 读取 tail 前完成回收、复用和 tail 发布，reader 得到 + “旧 head + 新 tail”，表面跨度超过 ring 容量； +2. reader 已进入旧 cursor，writer 随后发布新 head 并复用物理槽，旧 + cursor 的 absolute seq 双检失败。 + +旧实现会在第一类交错的初始容量检查，或第二类交错的 seq 双检处直接返回 +协议错误，因此都会把合法回收误报为 fatal。 + +#### 两个异常支路各自二次读取 head + +本阶段没有弱化控制跨度检查和 `SharedReadRegionSlot()` 的 seq 双检，也 +没有无条件重试。 + +初始控制快照跨度超过容量时，再对同 bucket 的 head 做一次真实 atomic +load。只有同时满足： + +```text +old_head <= new_head <= tail +tail - new_head <= CAP +``` + +才接受这个“旧 head + 新 tail”混合快照，并从 `new_head` 开始扫描。 + +slot 读取失败时同样只二次读取一次 head。只有同时满足： + +```text +head0 <= head1 <= tail0 +cursor < head1 +``` + +才说明失败 cursor 已属于合法退休的旧前缀，扫描位置直接跳到 `head1`。 +其余情况仍返回协议错误。两个额外 load 都位于原本即将失败的异常分支； +正常 lookup 不增加 atomic。 + +这依赖现有唯一 writer 发布顺序： + +```text +Exchange(head) 返回值验证成功 + -> Exchange(seq, empty) + -> 写并 flush payload + -> Exchange(seq, absolute cursor) + -> Exchange(tail) +``` + +head 单调且 absolute seq 不会把新一轮物理槽伪装成旧 cursor。测试事件 +记录进一步要求目标 head Exchange 严格早于第一次 slot seq invalidate、 +payload flush、新 seq 发布和 tail 发布，防止只凭源码顺序作结论。 + +#### 确定性交错 + +CPU 门槛用每 task 八条 region 填满同一 bucket： + +```text +cursor 0..7: producer 0 +cursor 8..15: producer 1 +... +head/tail: 0/CAP +``` + +reader 处于 task 2，`H=1`，已经连续发布 `reader_done=1`。它查询 +cursor 8 对应的 region，正确结果应为 producer 1。第一种交错让 reader +先读 `head=0`，在它读取 tail 前暂停；future writer 不读取或更新 +`committed_tasks`，只按生产 primitive 执行： + +```text +reader candidate = 0 + -> reclaim_upto = 0 + -> head 0 -> 8 + -> cursor CAP..CAP+7 复用物理槽 0..7 + -> tail CAP -> CAP+8 +``` + +reader 恢复后读取 `tail=CAP+8`，旧 head 与新 tail 的表面跨度为 +`CAP+8`；二次读取 `head=8` 后，真实跨度恢复为 CAP，并继续返回 +producer 1。 + +随后 reader 连续发布 `reader_done=2`，以 task 3 查询 cursor 16 的 +producer 2。第二种交错让 reader 在 cursor 8 第一次 seq 检查后、payload +拷贝前暂停；writer 推导 `candidate=1`，把 head 从 8 推到 16，并用 +cursor `CAP+8..CAP+15` 复用物理槽 8..15。reader 恢复后的第二次 seq +检查失败，二次读取 `head=16`，跳过退休前缀并继续返回 producer 2。 + +测试把 `committed_tasks` 在交错前重置为 0，两次回收和 append 后仍要求 +为 0，明确证明这两条 reader 恢复都不借助全局 exact-turn 前沿。 + +同一门槛最后把 cursor 16 的 seq 改成错误值但保持 head=16。此时二次 +读取 head 没有越过失败 cursor，lookup 必须继续返回协议错误；不能把 +真实损坏吞成并发回收。另一个反例直接构造 `tail-head=CAP+1` 且二次 +读取 head 不前进，同样必须返回协议错误，锁定真实控制跨度损坏不能借 +混合快照恢复。 + +#### 红灯、修正和回归 + +先固定扫描期槽复用交错、尚未修改 lookup 时，CAP=128 全套只有一项失败： + +```text +lookup skips retired cursor 0 and returns producer 1 +``` + +加入扫描期 head 判定后该项转绿。独立审查随后指出初始混合快照缺口； +只补对应测试时，CAP=128 又精确出现一项新失败： + +```text +lookup repairs old-head/new-tail snapshot and returns producer 1 +``` + +加入初始异常跨度的 head 二次判定后，该项转绿，扫描期交错仍保持通过。 +随后用系统 GCC 13、`-O2 -Wall -Wextra -Werror` 对共享普通 ring 的全部 +形态逐一重编并执行： + +| CAP | bucket 数 | 结果 | +| ---: | ---: | --- | +| 32 | 512 | PASS | +| 64 | 256 | PASS | +| 128 | 128 | PASS | +| 256 | 64 | PASS | +| 16384 | 1 | PASS | + +随后使用本用户 CANN 9.1 完成 shared CCEC 正式构建：generic protocol +probe 在 AIC/AIV 各自实例化并静态链接，正式 AIC/AIV entry、split +caller/runtime/finish、1:2 mixed ELF、LOCAL real-compute helper、零 +relocation和 artifact manifest 检查全部通过。该结果只证明新 lookup +分支可以生成并链接设备代码，不等同于跨物理核动态执行。 + +完整 shared CPU build 中，host task plan、五种 ring 容量、sparse trace、 +shared-output symbol、generic writer-intent、heap reserve、Vector Claim、 +winner materialize 和 split-finish loser 等门槛也全部通过。 + +该阶段只修正 production lookup primitive 并增加 CPU 确定性交错，没有 +接入 PA/Submit，没有改变 private 模式、ABI、字段或 atomic 数量正常路径。 +head 二次读取只发生在原本已经要失败的控制跨度或 seq 异常分支，正常 +lookup 不增加 atomic。设备当时已有其他任务，本阶段按用户要求不再尝试 +上板；因此结果不包含新的 A5 双向证据。下一小步再建立不依赖 +`committed_tasks` 的 reader-gated 整 task append 组合，并先做 CPU +容量与慢 reader 门槛。 + +### 2026-07-27：R4e-d2 建立无全局前缀的 reader-gated batch append + +R4e-d1 已让 lookup 能容忍合法旧前缀并发回收，但测试中的 future writer +仍手工串接 candidate、reclaim、preflight 和 append。R4e-d2 只在 ring +层增加一个最薄组合： + +```cpp +SharedTryAppendReaderGatedTask( + map, entries, count, active_workers, heap_window +) +``` + +它明确不调用两项旧 exact-turn 原语： + +```text +SharedRefreshReaderReclaimForTask() +SharedPublishTaskCommit() +``` + +因此不读取、不更新 `committed_tasks`。返回值继续复用已有 +`SharedAppendCheck`，避免再造一组同义状态： + +| 结果 | 本组合中的含义 | +| --- | --- | +| `Ready` | 整 task 预检通过，全部 entry 的 append 已返回成功 | +| `CapacityBlocked` | 当前 task 没有发布 payload/seq/tail，可等 reader 前进后重试 | +| `ProtocolError` | 不可重试，调用层必须终止本轮且不得发布 writer-ready | + +整 task 预检允许保留已经证明过期的 head 和单调 reclaim 更新； +`CapacityBlocked` 只保证当前 task 的 entry 没有部分发布。若预检通过后 +唯一 writer 仍遭遇 slot/seq/tail 协议破坏，逐 entry append 可能已留下 +物理前缀；这不是合法容量竞争，不能回滚，只能终止本轮。该边界没有被 +`ProtocolError` 名称掩盖。 + +#### reader close 与可重试 append 必须分开 + +`SharedAdvanceReaderDone(worker,N)` 只允许一次 CAS `N-1 -> N`。 +如果把它合并进可重试 helper，第一次 `CapacityBlocked` 后的第二次调用 +会把重复 close 误判成协议错误。固定调用顺序因此是： + +```text +完成本 task 全部 ordinary lookup + -> SharedAdvanceReaderDone(worker, task) 恰好一次 + -> 唯一有序 writer 调用 SharedTryAppendReaderGatedTask() + CapacityBlocked: 等其他 reader 前进后只重试 append helper + Ready: 才允许后续 writer metadata/gate 发布 + ProtocolError: 终止本轮 +``` + +loser/空任务以后也必须连续关闭 reader，但它们不调用 append。真实接线时 +loser 必须先 close 再等待 writer-ready，否则 winner 可能等待 loser +reader 前沿，而 loser 同时等待 winner gate，形成循环等待。本阶段没有 +接 PA/Submit，所以只冻结 ring 组合，不提前宣称出口顺序已闭合。 + +#### 96 条 reader 线只属于容量慢路 + +第一版直线组合按： + +```text +扫描 reader_done -> 发布 reclaim -> preflight -> append +``` + +实现，CPU 正确性可以通过，但每个 writer 都会无条件增加 +`active_workers` 次 atomic load；96 核真实路径会把容量慢路成本带入正常 +热路径。独立性能审查在提交前指出这个问题，最终改成： + +```text +count == 0 + -> 直接 Ready，零 shared 访问 + +count > 0 + -> 读取当前 reclaim_upto + -> 整 task preflight + Ready: 直接 batch append + ProtocolError: 直接拒绝 + CapacityBlocked: + 扫描 reader_done + -> 单调刷新 reclaim_upto + -> 第二次整 task preflight + -> Ready 才 batch append +``` + +这样纯 symbol writer 的空 ordinary batch 不读取 reclaim，普通有空间的 +ordinary writer 也不扫描 reader_done。CPU 事件门槛直接要求： + +- 96 个 actor 先在观测窗外关闭 task 0，随后 `count=0` 返回 `Ready` + 且事件列表为空； +- 空环单 entry 在 `active_workers=96` 时写入成功，96 条 + `reader_done` 地址均没有 Load； +- 只有满桶 `CapacityBlocked` 路径才读取 reader 前沿。 + +这只是避免无条件扫描，不改变容量慢路的正确性公式。`active_workers` 和 +`heap_window` 仍必须是整个 ring 生命周期的固定权威配置。 + +#### 满环、整批和反例门槛 + +五种 CAP 继续用每 task 八条 region 填满同一 bucket。fill 完成后立刻把 +`committed_tasks` 重置为 0，后续门槛全程要求它保持 0： + +```text +worker 0: reader_done=1，停在 task 2 的 cursor-0 lookup 中 +worker 1: 连续关闭到 future writer 当前 task，满足 append actor 自身 close +H=2 +batch: + [独立空桶 entry] + [满桶八条 replacement] +``` + +CAP=32/64/128/256 有多个 bucket，独立空桶 entry 故意排在满桶项之前； +CAP=16384 是单桶形态，只验证同桶整批。slow reader 第一次 seq 检查后、 +payload 拷贝前触发 future writer： + +```text +当前 reclaim=-1 + -> 第一遍 preflight 在后项得到 CapacityBlocked + -> 独立桶 tail/seq 不变 + -> 满桶 head/tail/slot/payload 不变 + -> 无 Exchange、CAS 或 Flush +``` + +lookup 完整返回后，worker 0 才关闭 task 2。重试同一 helper 时 +`candidate=0`，只回收 producer 0，batch 整体返回 `Ready`；目标桶 +`head:0->8`、`tail:CAP->CAP+8`，独立桶 entry 同时发布，而 +`committed_tasks` 仍为 0。 + +为覆盖只有一个 bucket 的 CAP=16384，五种 CAP 还共用一组同桶门槛： +task 0 只写一条，后续 task 每批最多 32 条，按 task-id 有序填满；reader +连续关闭到下一个 writer 当前 task，并令 `H=current_writer`，候选恰好 +为 0，只释放 task 0 的一个槽。同一 batch 提交两项时，第一项理论上可写、 +第二项容量不足。结果必须仍为 `CapacityBlocked`，允许 head 安全前进到 +1，但 tail、第一个可复用槽的 seq 和 payload 均不得发布，事件中不能 +出现 Flush。CAP=16384 时当前 writer 约为 513，仍远小于 +`kMaxTasks=4352`。 + +另一个反例在后项目标 slot 写入错误 seq。整 task preflight 必须返回 +`ProtocolError`，排在前面的合法 entry 仍保持空 seq/tail，事件中不得出现 +任何 publication。既有容量 all-or-nothing 门槛也已改为直接调用新组合， +不再用 exact-turn refresh 和单独 preflight 近似这条路径。 + +#### 红灯、验证与边界 + +测试先引用尚不存在的公共组合时，CAP=128 按预期只因 +`SharedTryAppendReaderGatedTask` 未声明而编译失败；补上组合后转绿。随后 +系统 GCC 13 下 CAP=32/64/128/256/16384 全部通过，完整 shared CPU build +中的 host plan、sparse trace、shared-output、generic writer-intent、heap、 +Vector、materialize 和 split-finish loser 门槛也全部通过。 + +CCEC generic compile probe 已显式实例化新组合，分别要求 AIC/AIV 后端生成 +并静态链接真实代码，正式 1:2 mixed ELF、metadata、零 relocation 和 +manifest/hash 检查也全部通过；probe 不进入正式 mixed ELF，不改变当前 +PA 的 I-cache 布局。设备已有其他任务，本阶段按用户要求不尝试上板， +不用 CPU 或 CCEC 编译结果冒充 A5 动态证据。 + +本阶段仍没有改写 `CommitOrdinarySharedWriterIntent()` 的逐项 lookup/append， +也没有把 `reader_done` 接到 winner、loser、空任务和错误核型的真实出口。 +下一提交先增加独立 `requires_writer_ready` plan/ticket 位，再把 generic +writer 拆成“收集全部 lookup/entry -> reader close -> 发布 symbol/ordinary +metadata -> writer-ready”，最后才构造真实 Claim/replay A→B→C 门槛。 + +### 2026-07-27：R5a 将发布前沿改为失败不改状态的 CAS + +后续目标已重新对齐为：`Claim(N)` 的唯一 owner 同时负责 TensorMap +插入、Build 和执行调度，但只把 TensorMap writer 元数据插入放进全局 +task-id 顺序；owner 发布插入前沿后再做 fanin lookup 和 Build，loser +直接继续 replay。该目标取代上一节末尾的逐 task loser +`writer-ready` 计划，后续不再增加对应 plan/ticket 位。 + +接入 owner 热路径前先修正既有发布原语的失败语义。旧 +`SharedPublishTaskCommit()` 无条件执行： + +```text +Exchange(N+1) -> 检查旧值 -> 失败时 Exchange(旧值) +``` + +重复、陈旧或未来 actor 会短暂把错误的 `N+1` 暴露给其他核；随后写回 +只能恢复最终值，不能撤销已经被观察到的错误前沿。新实现改成单次： + +```text +CAS(expected=N, desired=N+1) +``` + +`Claim` 仍负责证明同一 task 只有一个 owner，CAS 负责拒绝非 exact-turn +发布。失败时控制字从未改变，也不需要补偿写。CPU 事件门槛先在旧实现上 +精确得到两项失败，再要求重复和 future 发布都只产生一条 +`CompareExchange` 记录且保留原前沿；修改后 CAP=128 门槛转绿。 + +本小步只加固控制原语，没有把 `committed_tasks` 接回 PA 热路径，没有 +改变 ABI、private 模式或 TensorMap 数据布局，也没有运行 A5。 + +### 2026-07-27：R5b 建立“仅插入串行”的 shared Submit 基线 + +本阶段按照重新确认的合同，只修改 standalone，不修改 simpler 真实路径， +也不维护 AscendC。shared Submit 不再继续堆叠在 private 热函数的条件 +分支中，而是独立放到： + +```text +common/pa_shared_submit_path.h +``` + +private 路径保持原控制流；shared 的 Claim owner 进入独立 finish： + +```text +Claim(task N) + loser + -> 声明稳定 output symbol + -> 不构造重参数 + -> 不读取/等待 TensorMap + -> 关闭轻量 Submit 并继续 replay + + owner + -> 构造本 task 参数 + -> Materialize descriptor + -> 在 owner 私有状态中准备 writer delta + -> 等待 task N 的 exact insert turn + -> 发布 ordinary/symbol/fresh-output 元数据 + -> StoreBarrier + -> CAS 发布 N -> N+1 + -> fanin lookup + -> Build + -> slot/drain 执行 +``` + +这里全局串行的只有“等待 exact turn 到发布 `N+1`”这一小段。前沿一旦 +离开 N，N+1 owner 就能插入；N 的 fanin lookup、Build 和执行不再占用 +这条链。空 writer delta 也必须推进 task turn，否则后续 owner 会永久 +等待。Alloc/QK/SF/PV/UP 五类 task 均只由 Claim owner 构造重参数； +稳定 `FdwicOutputRef` 在 Claim 后按 task/slot 确定，不依赖 loser +重构参。 + +#### writer delta 与发布边界 + +`SharedTaskWriterDelta` 只保存 owner 在有序通道外准备好的 ordinary region +条目，以及本 task 是否存在 writer intent。准备函数要求 +`context.register_mask` 与全部 `Inout/OutputExisting` tag 精确相等： +缺位会漏 writer，多位会把非 writer 当成 writer，两者都在触碰共享状态前 +拒绝。 + +拿到 exact turn 后，发布顺序固定为： + +```text +整 task ordinary 容量预检 + -> symbol immutable history / latest + -> ordinary payload / seq / tail + -> fresh descriptor / published + -> StoreBarrier + -> committed_tasks: N -> N+1 +``` + +insert-before-lookup 版本暂时不能使用本 task 的 reader 前沿回收自己仍可能 +查询的 `N-H`，因此 production helper 固定 `reclaim_upto=-1`。满桶时 +fail-closed：设置 fatal，但 cursor、head/tail、slot seq/payload、 +symbol latest/history 和 fresh published 均不推进。默认 PA B256/PA-G1 +的普通 region ring 为空，因此不触发该容量边界；这不能被解释成通用 +ordinary ring 已经支持无限 task。 + +这里的 CCEC 可见性不能归功于名字抽象的 `StoreBarrier()`：当前 CCEC +实现中该接口本身为空。真实发布边界来自每类 payload 的 +`FlushRegion(DCCI + dsb)` 及其后被消费返回值的发布 atomic；最后的 +turn CAS 只能在这些具体发布动作之后执行。CPU 的 `StoreBarrier()` 是 +顺序一致 fence，只用于宿主并发模型。后续分组前沿不得省略或打乱这些 +payload 级发布边界。 + +#### fanin 统一过滤到 `[N-H,N)` + +owner 在发布自己的 writer 后才查询，因此查询必须同时做到： + +1. 排除本 task 和 future writer； +2. 允许沿 symbol immutable history 从 future/latest 回退； +3. 只保留 `producer >= N-H`。 + +当前三种 fanin 来源使用同一半开窗口： + +| 来源 | 过滤方式 | +| --- | --- | +| ordinary region ring | `SharedLookupRegion()` 选择 `[N-H,N)` 内最大重叠 producer | +| shared output symbol | `ResolveSharedSymbolWriterBefore()` 先回退到 ` Register(ordered insert) -> Fanin -> WinnerBuild +``` + +Alloc 没有 fanin lookup，因此不再生成零业务含义的 Fanin span。raw +validator、稀疏 trace 正反例和记录数公式同步到该顺序；Register 的 +`auxiliary` 表示 ordinary writer 数，Alloc 要求 0，普通 task 接受 +`[0,kMaxTaskTensors]`，不再把旧 symbol writer 数硬编码为 1。 + +CPU 开启泳道的 B1/PA-G4 实测生成 6964 条 raw record，host 期望也是 +6964，逐 worker 数量、稀疏阶段顺序和零 dropped 全部通过。对应本轮 +临时验证产物为： + +```text +outputs/pa_scheduler_shared_swimlane_20260727_171927_799726/cpu/ +``` + +该目录是本地验证产物，不进入代码提交。 + +#### 本阶段验证结果 + +最终源码下执行： + +```bash +./run.sh build cpu --tensormap shared + +./run.sh run cpu --tensormap shared \ + --batches 1 --shared-context-lens 32768 \ + --runs 1 --no-swimlane \ + --winner-workload scalar-nop --nop-count 1 + +./run.sh run cpu --tensormap shared \ + --batches 256 --shared-context-lens 8192 \ + --runs 1 --no-swimlane \ + --winner-workload scalar-nop --nop-count 1 + +./run.sh swimlane cpu --tensormap shared \ + --batches 1 --shared-context-lens 32768 \ + --winner-workload scalar-nop --nop-count 1 +``` + +全部通过。B1/PA-G4 最终 `committed_tasks=17`；B256/PA-G1 为 1280 task、 +1024 kernel，最终 `committed_tasks=1280`，96 核唯一 winner、依赖 +signature、INOUT history、heap cursor 和完成 flag 全部通过。CPU 的 +墙钟时间受宿主线程调度影响，只作为正确性执行证据，不用于推断 A5 性能。 + +使用本用户 CANN 9.1： + +```bash +source "$HOME/Ascend/cann-9.1.0-weekly-20260708/cann-9.1.0/set_env.sh" +./run.sh build ccec --tensormap shared +``` + +AIC/AIV generic protocol、正式 entry、role-specific real-compute、 +split runtime/state/finish、1:2 mixed ELF、LOCAL helper、零 relocation +和 artifact manifest 全部通过。该结果只证明 CCEC 能生成并链接当前 +路径；设备正在被其他任务使用，本阶段没有运行 A5，不能把编译成功写成 +上板成功。 + +#### R5b 预登记：先补观察，再做交错前沿 + +当前 G=1 的 `committed_tasks` 是所有 future owner 轮询的同一 atomic +地址。后续性能候选不是建立多条独立插入链，而是把同一前驱 token +交错放到 G=1/2/4/8 条 cache line： + +```text +task N 等 turn[N % G] == N +发布完成后： + next = N + 1 + CAS( + turn[next % G], + next >= G ? next - G : -1, + next + ) +``` + +G=1 必须退化为当前完全相同的 `N -> N+1`。每 task 仍只有一次发布 +CAS，插入仍严格串行；变化只是 future owner 的等待 load 分散到不同 +地址。正式实现前先给 insert-turn wait/publish 增加最小聚合观察，确认 +每 task 发布 CAS 恰好一次并能比较各 lane 的轮询量，随后再用独立提交 +验证 G=1/2/4/8。不能把独立 shard cursor 当作等价方案，因为那会允许 +不同 shard 同时修改 TensorMap，破坏本阶段刚闭合的唯一有序插入合同。 + +### 2026-07-27：R5c 交错分组同一枚插入前沿 + +R5c 实现了上面的预登记候选，但没有把 writer 插入拆成 G 条独立顺序链。 +active turn-G 只决定同一枚全局 baton 落在哪条 cache line： + +```text +初始： + turn[0] = 0 + turn[1..7] = -1 + +task N： + lane = N & (G - 1) + 等待 turn[lane] == N + +writer 元数据完整发布后： + next = N + 1 + target = next & (G - 1) + expected_old = next >= G ? next - G : -1 + CAS(turn[target], expected_old, next) +``` + +task `N+1` 的 grant 仍只能由 task N 产生，所以任何时刻只有一个 owner +能够修改 TensorMap writer 元数据。turn-G1 精确退化为原来的 +`CAS(N,N+1)`，每 task 仍只有一次发布 CAS；turn-G2/G4/G8 只是把未来 +owner 的等待 load 分散到 2/4/8 个地址。本文后续用 `PA-G4` 表示一个 +batch 的四个 PA block group，用 `turn-G4` 表示四条 insert-turn 物理线， +避免两种 G 混淆。 + +#### grant、重复调用与发布前检查 + +turn-G>1 的旧 token 会保留到同余的下一代覆盖，因此它是 grant，不是 +读取后自动失效的锁。正确性继续依赖 Claim 对每 task 只产生一个 owner； +不能为“消费 grant”再增加一次 CAS，否则每 task 会从一次发布 atomic +变为两次。 + +生产发布在写任何 TensorMap 元数据前同时检查： + +1. current lane 仍精确等于 N； +2. target lane 仍等于本轮应覆盖的 `expected_old`。 + +元数据发布结束后只执行已经预检过的目标 CAS。重复、陈旧、future 或 +损坏 token 都失败且不改控制线。turn-G1 的 publication preflight 是一次 +current-line load 加一次发布 CAS；在它之前， +`WaitForSharedTaskInsertTurn()` 还会执行至少一次等待 load。turn-G>1 +只比 turn-G1 多一次 target-line preflight load；独立低层 G1 发布原语仍 +保持单 CAS、零预检 load 的事件形状。 + +该设计不会改变 shared Submit 的业务边界： + +```text +Claim 唯一 owner + -> Materialize 与 writer delta 准备 + -> 等待并发布 writer 元数据 + -> 交出 next token + -> fanin lookup + -> Build + -> ready 检查、执行或进入本核 winning slot +``` + +loser 的零访问门槛从“只统计 sidecar Load”扩大为统计所有落在 +`SharedTensorMapSidecar` 范围内的 Load、Exchange、CAS、FetchAdd、 +FetchMax、Invalidate、Flush 和 Publish。该断言从 Claim 已经判定 loser +后的 finish 入口开始；Claim 自身的 Vector cursor 位于 sidecar，不属于 +“post-Claim loser 零访问”的范围。 + +#### 固定八线布局和构建身份 + +为保持 turn-G1 的热点地址与此前所有字段 offset，lane 0 继续使用原 +`committed_tasks`；lane 1～7 以 `insert_turn_extra[7]` 追加在 +`reader_done[96]` 后。所有 G 使用同一物理布局，inactive lane 始终为 -1： + +| 项目 | generation 9 当前值 | +| --- | ---: | +| `writer_history` offset | 11,027,648B | +| `reader_done` offset | 12,420,288B | +| `insert_turn_extra` offset | 12,426,432B | +| 七条 extra line 增量 | 448B | +| shared sidecar | 12,426,880B | +| shared non-split `SchedulerState` | 1,019,542,848B | +| shared split `SchedulerState` | 1,019,548,992B | + +private sidecar、private `SchedulerState` 以及非默认 CAP 的 private ABI +公式均保持原值。默认 CAP=128 的 shared ABI version 为 +`(9<<8)|turn-G`，turn-G1/G2/G4/G8 分别为 +`0x901/0x902/0x904/0x908`。构建方式为： + +```bash +PA_SHARED_INSERT_TURN_GROUPS=4 \ + ./run.sh build cpu --tensormap shared + +PA_SHARED_INSERT_TURN_GROUPS=8 \ + ./run.sh build ccec --tensormap shared + +PA_SHARED_INSERT_TURN_GROUPS=8 \ + ./run.sh run ccec --tensormap shared --batches 1 +``` + +CPU 正式 scheduler binary 使用构建时选择的 G；shared CPU build 还会 +独立编译所有 G 的控制原语和 96-worker 定向门槛。CCEC 每次只编译所选 G。 +CPU turn-G1 保留 `pa_scheduler_cpu`，G2/G4/G8 使用各自带 G 的文件名， +`run.sh` 按环境变量选择。CCEC 各 G 顺序复用同一 variant 目录，因此 +schema-v3 manifest 显式记录 `shared_insert_turn_groups`，run 还要求环境变量 +与 manifest 完全一致。host/kernel ABI version 也包含 G，避免只靠相同 +`sizeof` 误把不同 G 产物拼在一起。 + +#### 独立 oracle 与正确性门槛 + +完成 T 个 task 后，active lane s 保存不大于 T 的最大同余 token。完整 +公式必须先处理 inactive 和尚未收到首个 token 的 lane： + +```text +turn[s] = + -1 if s >= G or s > T + T - ((T - s) & (G - 1)) otherwise +``` + +lane 0 在 T=0 时由第二个分支得到 0。host 使用独立公式校验全部八线， +不再把 lane 0 +`committed_tasks` 单值冒充整体前沿。典型终态为： + +| 完成数 | turn-G | 八线终态 | +| ---: | ---: | --- | +| 17 | 1 | `[17,-1,-1,-1,-1,-1,-1,-1]` | +| 17 | 2 | `[16,17,-1,-1,-1,-1,-1,-1]` | +| 17 | 4 | `[16,17,14,15,-1,-1,-1,-1]` | +| 17 | 8 | `[16,17,10,11,12,13,14,15]` | +| 1280 | 1 | `[1280,-1,-1,-1,-1,-1,-1,-1]` | +| 1280 | 8 | `[1280,1273,1274,1275,1276,1277,1278,1279]` | + +新增 `test_shared_insert_turn.cpp`，在 turn-G1/G2/G4/G8 下分别覆盖脏值 +初始化、跨两轮 rollover、每步只改目标 lane、future pending、非法负值、 +错误同余、超前 token、target expected-old 不匹配和重复发布不改状态。 + +`test_shared_ordered_submit.cpp` 在四种 G 下都启动真实 96 个 pthread， +逐 task 反向核对： + +- Alloc/QK/SF/PV/UP 的 Claim attempt 数符合 96/32/64/32/64 的参与拓扑； +- 每 task 恰好一个 Claim owner； +- 每个 next token 恰好发布一次，且物理 lane 分布精确； +- task 4 交出插入 token 后暂停，task 8 仍能跨 owner 完成 lookup 与 Build； +- 独立 batch 的 task 6 kernel 能在 task 4 Build 前完成； +- post-Claim loser 对 TensorMap 的所有 Ops 访问严格为 0。 + +generic writer-intent 在 turn-G1 和 turn-G8 两端继续覆盖空写 task、 +ordinary/symbol INOUT、混合 ordinary+symbol+fresh 发布、严格 +`producer∈[N-H,N)` 和满环整批失败不改状态。CCEC shared-protocol litmus +也按完整八线初始化并校验终态，不能让 inactive lane 的错误零值被 G1 +偶然掩盖。 + +冻结前的边界审计还补了两项 fail-closed 门槛：ordinary ring 在 +`tail==INT64_MAX` 时没有可表达的下一 tail，必须在任何 slot/控制字写入前 +返回协议错误；普通 TensorDesc writer 的 `owner_task_id` 若高 32 位非零， +必须在 writer-delta 预检阶段拒绝，不能截断低位后发布元数据。两项都有 +“失败后无 TensorMap 发布”的 CPU 负例。 + +shared-protocol litmus 当前固定为 turn-G1，只承担已经建立的跨核 +memory-order 证据，不宣称覆盖 turn-G2/G4/G8。四种 G 的本阶段 CCEC 证据 +来自主 scheduler AIC/AIV、split finish、mixed ELF 和 manifest/ABI +构建矩阵；在设备可用前仍不把编译证据写成上板结果。 + +#### 验证口径和性能边界 + +CPU B256 只证明 1,280 个 task 在真实 96-thread 回放下闭合 Claim、插入、 +fanin、Build 和完成语义。CPU wall time 受宿主线程调度支配，不能用来评价 +turn-G 的 A5 性能。CCEC 构建只证明 AIC/AIV、split finish、mixed ELF 和 +host 能以同一 G 生成、链接并通过 manifest/ABI 门槛，也不能冒充上板收益。 + +turn-G>1 相比 turn-G1 每 task 多一次 target-lane 预检 load，这是旧 grant +不会自动失效时拒绝重复发布所需的正确性成本；同时它把多个 future owner +的轮询 load 分散到不同 cache line。候选成立的条件是后者减少的竞争和等待 +大于额外预检及更大数据工作集的成本，不能仅凭 atomic 次数推断净收益。 + +本阶段没有向 production raw 增加逐 poll 事件或新泳道字段。96-worker +门槛里的 Claim/publish 计数只存在于 CPU 测试，既证明唯一 owner/一次 +发布，也不扩大真实 profiling 文件。turn-G2/G4/G8 是否减少 A5 insert-turn +等待和完整 Submit 时间，必须等设备可用后用同一 `perf-clock` 构建、 +相同 PA 工作量、同设备交错多轮比较；在得到该证据前，它们只是已闭合 +正确性的性能候选，不宣称收益。 + +本阶段 turn-G2/G4/G8 只允许 CPU 与 CCEC。`run.sh` 会在任何构建、文件创建 +或设备动作前拒绝 AscendC/`all`，也会拒绝 private+G>1；因此本阶段没有 +悄悄维护或引用 AscendC 变体。 + +#### 冻结源码最终验证矩阵 + +2026-07-27 最终矩阵冻结的源码集合包括 `common/`、`ccec/`、`cpu/`、 +`test/` 下的 C/C++/shell 文件以及顶层 `run.sh`。构建前、每个 CPU +正向用例结束后和全部 CCEC 构建结束后的被测聚合 SHA256 均为： + +```text +4fd74b43afdc9588c2ef05dfd870a972d4d546135774fc2426a3e1aa6f1c884d +``` + +提交前 header 门禁只把新增 +`test/test_shared_insert_turn.cpp` 许可证注释中的 `license` 改为 +`License`；C++ token、生产源码和 CCEC 编译输入均未改变。最终暂存源码的 +逐字节聚合 SHA256 因这一个注释字符变化更新为： + +```text +8cf5053a8c1e429c918200dffe541e2b981bc9955a4ad9ae45cd5fd75fb5a642 +``` + +该最终暂存源码又完整执行一次 shared G1 CPU build；它会实际编译并运行 +turn-G1/G2/G4/G8 原语和 ordered-submit、writer-intent G1/G8、五档 CAP +ring 及其他 shared 门槛，全部 PASS。因此下表没有把行为变化藏在 +提交前修正中。CPU 使用 GCC 13.3.0；CCEC 使用本用户 CANN 9.1 weekly +2026-07-08 中的 clang/ccec 15.0.5。 + +CPU 正向矩阵对每个 shared G 都执行一次完整构建和 96-worker、B256、 +1,280-task 回放： + +```bash +PA_SHARED_INSERT_TURN_GROUPS= \ + ./run.sh build cpu --tensormap shared + +PA_SHARED_INSERT_TURN_GROUPS= \ + ./run.sh run cpu --tensormap shared \ + --batches 256 --shared-context-lens 8192 \ + --runs 1 --no-swimlane \ + --winner-workload scalar-nop --nop-count 1 +``` + +| turn-G | 实测八线终态 | normalized writer signature | 语义/后处理 | +| ---: | --- | --- | --- | +| 1 | `[1280,-1,-1,-1,-1,-1,-1,-1]` | `556bec7ec8d0f323` | PASS/PASS | +| 2 | `[1280,1279,-1,-1,-1,-1,-1,-1]` | `556bec7ec8d0f323` | PASS/PASS | +| 4 | `[1280,1277,1278,1279,-1,-1,-1,-1]` | `556bec7ec8d0f323` | PASS/PASS | +| 8 | `[1280,1273,1274,1275,1276,1277,1278,1279]` | `556bec7ec8d0f323` | PASS/PASS | + +四种 G 的完整构建都会额外运行 turn-G1/G2/G4/G8 原语门槛、 +turn-G1/G8 generic writer-intent、turn-G1/G2/G4/G8 的 96-worker +ordered-submit，以及 CAP=32/64/128/256/16384 ring 回归。关键实测值为： + +```text +loser_zero_map_access=PASS accesses=0 +release_before_build=PASS completed=17 overlap=1 kernels=4,4,4,4 +independent_kernel_overlap=PASS completed=10 kernels=2,2,2,2 +``` + +它们分别证明 Claim 判负后的 loser 对 sidecar 的全部 Ops 访问为零、 +task 4 交出 turn 后 task 8 可以跨 owner 完成 lookup/Build，以及独立 +batch 的 task 6 kernel 可以在 task 4 Build 之前完成。writer-intent +端点门槛同时覆盖空写 task、ordinary/symbol INOUT、混合三类 writer、 +`producer∈[N-H,N)`、满环整批失败不改状态、`INT64_MAX` tail 和高 32 位 +owner 的 fail-closed 负例。 + +private G1 也使用同一冻结源码完成完整构建和 B256 回放，实测 +`logical_entries=52`、`logical_floor=1215`，raw/normalized writer +signature 均为 `556bec7ec8d0f323`，语义与后处理均 PASS。它没有读取 +新增 turn line，private ABI 和正式产物名保持不变。 + +CCEC 在不启动设备的前提下完成下列构建、静态链接、mixed ELF、LOCAL +helper、零 relocation、manifest 和 SHA 门槛： + +| 模式 | turn-G | ABI version | manifest | host/kernel SHA 校验 | 结果 | +| --- | ---: | ---: | --- | --- | --- | +| shared | 1 | `0x901` / 2305 | v3、G=1 | OK/OK | PASS | +| shared | 2 | `0x902` / 2306 | v3、G=2 | OK/OK | PASS | +| shared | 4 | `0x904` / 2308 | v3、G=4 | OK/OK | PASS | +| shared | 8 | `0x908` / 2312 | v3、G=8 | OK/OK | PASS | +| private | 1 | generation 4 | v3、G=1 | OK/OK | PASS | + +对应 artifact SHA256 为： + +```text +shared G1 host=bf58a476adcc6f57b70435d8c41b935b63abb848483a24af040f0eaae23a189c + kernel=eab8afc65402361d77d44054f6f8a854767ca3b0ce3161e29e6803211b6865c8 +shared G2 host=e74f4358e5eabadc6e5eb2ee485240171cee559772b02733c1fcf32bb7bf6348 + kernel=96ed43b3044adb1a94b41503992e25bb44bb3218d70564f3383dc852676bdaac +shared G4 host=5c0070db80cd5d7f363c04ba78a8c2321b3775a1dbc110a5fe6f5fe11d7520e5 + kernel=2578fd313903408886ae626fe6074a49ace405c8eb7cafeef5b82c682ccda748 +shared G8 host=6fb9ce3f8ef970be097414e2cec72e0871fa8e61d30c3a5c0225b56368e5a024 + kernel=392f1e178a4bf2bd3ba5afe7ed917a2cdf3061eedf7d3827c8768ace644261fe +private G1 host=b63b771c42f59ccf2c05f55feede8f46a11180092fe6936f82476949edfa6f9d + kernel=1f0c524a4a3c1bc5317e8f8163a855f6625409cb24523f8d7d1f38f7abb02367 +``` + +固定布局探针在四种 shared G 下均得到 8 条物理 turn line、64B +`AtomicLine`、12,426,880B sidecar、1,019,548,992B split +`SchedulerState`，且 `committed_tasks`、`reader_done`、 +`insert_turn_extra` offset 分别为 0、12,420,288、12,426,432。 + +`shared-protocol-litmus` 只以 turn-G1 完成 CCEC build 和静态 artifact +核对；host 虽然编译成功，但本轮没有执行，也没有触发 ACL/kernel launch。 +其 manifest v2 记录 shared ABI generation 9，AIC/AIV IR、mixed kernel +和 host 四项 SHA 均通过。该证据不覆盖 turn-G2/G4/G8 的动态 memory order。 + +负向门槛均在进入后端构建、文件创建或设备动作前按预期 exit 1： + +| 请求 | 拒绝原因 | +| --- | --- | +| shared turn-G3 | G 只允许 1/2/4/8 | +| private turn-G2 | 分组前沿只属于 shared | +| AscendC turn-G2 | 本阶段只维护 CPU/CCEC | +| `all` turn-G2 | 组合后端超出本阶段范围 | +| G1 消费 G8 CCEC manifest | manifest G 与环境 G 不一致 | +| 缺少精确 G2 CPU binary | 不回退到旧名或其他 G 产物 | + +本轮没有运行 A5，因此 turn-G1/G2/G4/G8 的 A5 正确性和性能行均为 +`NOT RUN`。CPU B256 的 submit span 受宿主线程调度和模拟执行影响； +CCEC 构建也只证明代码生成、链接和产物身份。两者都不能用于比较 +turn-G 的 A5 性能，更不能据此宣称 G2/G4/G8 已获得收益。 + +### 2026-07-28：把 shared Register 拆成等待、发布与交接三段 + +前一轮 turn-G8 B256 泳道中，`Register` 的单事件中位数约为 +1.779 ms，累计 core-work 为 2,054.096 ms，占 SubmitUnion 的主要部分。 +旧边界把等待插入前沿、发布 writer 元数据和交接下一 task 混在一起， +无法判断长时间来自真实 scalar 发布代码还是有序前沿等待。 + +本轮保留原 `Register` 父区间和 `auxiliary=ordinary_count`，每个 shared +winner 只新增一条 `SharedRegisterPublishMetadata` raw 记录。四个端点形成： + +```text +Register.start + -> wait_insert_turn +SharedRegisterPublishMetadata.start + -> publish_metadata +SharedRegisterPublishMetadata.end + -> handoff_next_turn +Register.end +``` + +等待结束时间依赖最后一次返回 Ready 的 atomic Load；交接结束时间依赖 +N→N+1 CAS 的返回值。首次 AIC/AIV O3 IR 核验发现，单纯在 Ready 分支后把 +`observed` 传给 `NowAfterAtomicResult()` 并不可靠：编译器利用 +`Ready => observed == task_id`，把 SYS_CNT inline asm 的输入替换成了 +`task_id`。最终实现先用一条 MOV 从同一个 atomic 返回寄存器派生两个 +编译器不可证明相等的输出，比较值只进入 Ready 分支，独立依赖值只进入 +SYS_CNT inline asm。修正后的 AIC 与 AIV 优化 IR 均保持： + +```text +llvm.hivm.atom.ADD.G.s64 + -> asm "MOV dependency, compare" + -> compare == task_id + -> asm "MOV dependency, dependency; MOV cycle, SYS_CNT" +``` + +该序列每次 turn 轮询只增加一条 MOV，不增加逐 poll SYS_CNT、GM 访问或 +DSB；计时边界仍只表示返回值已可被本核 scalar 消费，不表示跨核全局 +可见。`PA_BUILD_TRACE_FREE` 下 fork、额外 SYS_CNT 和返回依赖均在预处理 +后完全不存在，因此 submit-PMU、perf-clock 与纯性能构建不承担该观察开销。 + +转换器直接展示中间的 `register.publish_metadata#N`，再仅用父子端点离线 +生成 `register.wait_insert_turn#N` 和 +`register.handoff_next_turn#N`。三个子段不进入 Submit 排他 child 集合； +排他分析器新增 `register_breakdown`，从 raw 整数边界逐事件、逐核和整体 +验证： + +```text +Register = wait_insert_turn + publish_metadata + handoff_next_turn +``` + +#### 正确性与规模门槛 + +- shared 每个 winner 恰好一条 detail，loser 为零； +- detail 必须与父 Register 的 core/lane/task/function 一致且完整包含于父； +- 缺失、重复、越界、身份错误和非零 flags/aux 均拒绝； +- private TensorMap 禁止出现该 shared-only phase； +- 每核记录公式更新为 + `3*submits + 5*wins - alloc_wins + kernel/wait/parent/atomic`； +- B256 的固定 shared 业务增量为 `4*batches + 28*groups`； +- 不记录任意一次 poll，只增加一条 64B detail/winner。 + +CPU shared 完整构建已通过 sparse trace、writer intent、ring，以及 +turn-G1/G2/G4/G8 的 96-worker ordered-submit 门槛。converter/analyzer +共 69 项 Python 单测通过；converter 还会独立拒绝没有对应 Claim 的孤立 +Register 父记录。CCEC turn-G8 swimlane 重新构建、mixed ELF、manifest +与 host/kernel 身份门槛通过。 + +B1 A5 先行门槛全部 PASS：5 个 winner 对应 5 条 detail， +`2,719/2,719` records、0 drop；完整 Submit 为 97.248 us。该次 B1 只作为 +正确性和记录闭合门槛，不用于推导 B256 占比。 + +#### turn-G8 B256 实测 + +完整产物位于： + +```text +outputs/pa_scheduler_shared_swimlane_20260728_012832_1184418/ccec/ +``` + +运行语义、writer signature、输出 tile、插入前沿和后处理均 PASS； +`472,067/472,067` records、0 drop，1,280 个 Register 父区间与 1,280 +个 metadata detail 精确对应。完整 Submit wall-clock 为 25.361069 ms。 +本轮相邻诊断样本在 24.792–25.361 ms 间波动；最后一次源码变化只从 +非计时协议检查中删去多余 MOV，因此不能把 0.569 ms 的单样本差异归因给 +这条修改。泳道样本用于结构和占比取证，不承担候选性能裁决。 + +下表为 1 GHz SYS_CNT 的 aggregate core-work；它不是 1.65 GHz PMU cycle， +也不是跨核 wall-clock： + +| Register 分段 | aggregate core-work | 占 Register | 占 SubmitUnion | +| --- | ---: | ---: | ---: | +| Register parent | 2,107.066118 ms | 100.000% | 90.756% | +| 等待插入轮次 | 2,085.986114 ms | 99.000% | 89.848% | +| 发布元数据 | 16.674066 ms | 0.791% | 0.718% | +| 交接下一 task | 4.405938 ms | 0.209% | 0.190% | + +1,280 个 winner 单事件分布为： + +| 分段 | median | p95 | max | +| --- | ---: | ---: | ---: | +| Register parent | 1,842.136 us | 2,027.973 us | 2,176.496 us | +| 等待插入轮次 | 1,824.032 us | 2,003.527 us | 2,163.026 us | +| 发布元数据 | 12.023 us | 23.697 us | 38.779 us | +| 交接下一 task | 2.233 us | 11.526 us | 22.267 us | + +结论已经从推测变为直接取证：当前 `Register` 长尾几乎全部来自等待有序 +插入轮次，而不是 writer 元数据发布代码或最终 CAS 交接。后续若优化 +shared B256，应优先分析 owner 到达顺序、前沿推进与等待竞争；不应先对 +只占 Register 约 1.00% 的 publish+handoff 做大范围 scalar 改写。这里的 +`wait_insert_turn` 是 Register 前段，以轮询等待为主，但还包含上一条 +Materialize trace 落盘和边界代码的少量观察开销；不应把每个 tick 都解释 +成 atomic 总线等待。 + +记录规模保持受控。相比前一份 G8 样本，raw 从 30,605,471B 变为 +30,695,570B，merged 从 60,444,501B 变为 60,779,006B；固定 +402,660,160B 设备 trace 分配没有扩大。结构性新增恰为 1,280 条 detail， +另外两段只在 merged 离线生成；其余 record 波动来自运行时 PollBatch +合并数，不是新增业务 phase。 + +### 2026-07-28：R5d 扩展 turn-G32/G64/G128，并以 B512 复核规模效应 + +本轮回答两个明确问题： + +1. 在已经有 turn-G1/G8 数据的基础上，继续测 G32/G64/G128，判断把 + future owner 的轮询 load 分散到更多 cache line 后，完整 Submit + 是否仍有收益； +2. 保持每个 batch 的 PA-G1 业务不变，仅把 batch 从 256 增至 512， + 即把总 task 从 1,280 增至 2,560，再检查各 G 的相对关系和总耗时 + 是否随 task 数近似线性增长。 + +这里的 turn-G 仍不是 G 条独立 writer 链。全局插入次序始终只有一条： +task N 完成元数据发布后，才把 task N+1 的 grant 写到 +`turn[(N+1)&(G-1)]`。G 只改变同一枚 baton 的物理落点，因而本轮数据 +不能解释为放宽 TensorMap writer 顺序。 + +#### 实现边界与 ABI + +物理 turn 容量从 8 扩到 128，构建期允许 +`G=1/2/4/8/16/32/64/128`。lane 0 继续复用 `committed_tasks`， +lane 1～127 位于尾部 `insert_turn_extra[127]`；所有 G 使用相同物理 +布局，inactive lane 初始化并保持 -1。shared ABI generation 从 9 +升级为 10，默认 CAP=128 时 ABI version 为 `(10<<8)|G`，manifest +仍把 active G 作为不可混用的构建身份。 + +当前关键布局为： + +| 项目 | generation 10 | +| --- | ---: | +| `insert_turn_extra` offset | 12,426,432B | +| shared sidecar | 12,434,560B | +| shared non-split `SchedulerState` | 1,019,551,552B | +| shared split `SchedulerState` | 1,019,557,696B | + +shared standalone 的 batch 输入容量从 256 扩到 512，但默认仍是 256； +private 继续保持 256。 +`kMaxTasks` 保留 4,352-task 物理布局：它同时容纳 B256/PA-G4 的最坏 +计划和 B512/PA-G1 的 2,560-task 计划；B512 多组计划若超过 4,352， +会在 worker/device 启动前失败，不把“batch 上限 512”误写成所有 +context 都可达。B512 的 synthetic heap 逻辑容量为 512MiB，实测 +reservation 为 413,138,944B，每个 shard 51,642,368B。real-compute +访问的是独立 workspace，不会分配或解引用这段 synthetic heap。 + +B512 首次 CPU 回放还暴露了一个 host oracle 假失败：2,048-slot payload +arena 回绕后，`ndims=1` 的合法 descriptor 会保留 inactive +`strides[1]` 的旧字节。PA 构造器和消费者都只定义、读取 +`[0,ndims)`，因此校验改为只比较 active shape/stride，同时继续拒绝 +`ndims>5`、active 字段损坏、地址/大小/owner 错误。定向测试分别证明 +inactive 尾字节不影响语义，active 维损坏仍会失败；没有为了让测试变绿 +而在设备热路径增加清零 store。 + +#### 测量口径与环境边界 + +性能构建固定为 CCEC `perf-clock`，关闭泳道、atomic trace 和 PMU； +device 0、shared TensorMap、默认 context 8192、PA-G1、 +`real-compute 6,28,4,1`、final barrier `two-16`。一条可复现命令为： + +```bash +PA_SHARED_INSERT_TURN_GROUPS= \ + ./run.sh perf-clock ccec --tensormap shared \ + --device 0 --batches <256或512> --shared-context-lens 8192 \ + --winner-workload real-compute --real-compute-counts 6,28,4,1 \ + --final-barrier two-16 +``` + +计时值是首个 Submit 起点到最后一个 Submit 终点的 wall-clock，不是 +96 核 duration 求和，也不包含泳道观察开销。环境使用 CANN +9.1 weekly 2026-07-08，CCEC 目标为 `dav-c310`。本机没有安装 +`npu-smi` 和 `task-submit`，因此测试按用户允许直接在 device 0 +无锁执行；运行结果能证明该设备上的 kernel/host 语义和计时，但本文 +不凭工具缺失臆造设备锁状态或独立确认芯片型号。 + +#### 正确性与构建门槛 + +CPU shared 构建逐项运行 G1/G2/G4/G8/G16/G32/G64/G128 的低层 +insert-turn 原语和 96-worker ordered-submit；generic writer-intent +以 G1/G128 两个端点覆盖空写、ordinary/symbol INOUT、混合 writer、 +满环失败不改状态和 owner/overflow 负例。B512/G128 完整 CPU 回放 +闭合 2,560 task、96 worker、全部业务输出和 128 条 turn 终态。 + +五档正式性能候选都通过 CCEC AIC/AIV、role-specific real-compute、 +split caller/runtime/finish、1:2 mixed ELF、LOCAL helper、零 +relocation、schema-v3 manifest 和 host/kernel SHA 校验。最终又用 +G128 artifact 运行 B1 设备门槛，manifest 精确识别 G128，5 task、 +8 个 published output、4 个 active real-compute tile 和所有语义断言 +PASS，Submit 为 80.392us；该 B1 仅证明最终产物可执行，不参与 B256/B512 +性能统计。private batch 上限收紧回原有 256 后，CPU B1 仍以原 +1,007,115,968B state 通过全部断言。 + +#### 第一轮：B256 六轮交错矩阵 + +每个 G 先运行一个不计入统计的 warm-up，再执行 6 个独立进程样本； +各轮旋转 G 顺序，避免把固定运行次序当成 G 的效果。所有 30 个正式 +样本均通过 execution、semantic、postprocess、real-compute 输出、 +writer signature 和完整 128-line turn 终态校验。 + +| turn-G | n | Submit median | mean | min～max | 相对 G1 median | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 6 | 57,116.065 us | 57,184.516 us | 56,975.478～57,565.335 us | 基线 | +| 8 | 6 | 25,020.867 us | 24,974.136 us | 24,670.165～25,103.450 us | -56.193% | +| 32 | 6 | 14,257.828 us | 14,226.053 us | 14,022.699～14,369.782 us | -75.037% | +| 64 | 6 | 11,368.688 us | 11,386.716 us | 11,323.154～11,485.594 us | -80.095% | +| 128 | 6 | 9,412.827 us | 9,405.942 us | 9,367.675～9,420.272 us | -83.520% | + +按相邻档位的 median，G32 相比 G8 约再降 43.0%，G64 相比 G32 +约再降 20.3%，G128 相比 G64 约再降 17.2%。G128 是本轮已测范围 +内的最优点；这不等于已经证明继续增加物理线仍会获益。 + +#### 第二轮：同一最终源码下的 B256/B512 配对 + +完成 B512 容量和 oracle 修正后，五种 G 全部重新构建。每轮对同一 G +相邻运行 B256/B512，四轮中交替 batch 顺序，并旋转、反转 G 顺序。 +因此下表的 B256 是最终 B512-capable artifact 的独立复测，不与上一节 +旧 artifact 的六轮样本混算： + +| turn-G | B256 median（n=4） | B512 median（n=4） | B512/B256 | B512 相对 G1 | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 57,730.850 us | 117,391.969 us | 2.0334× | 基线 | +| 8 | 25,386.679 us | 51,317.179 us | 2.0214× | -56.286% | +| 32 | 14,198.940 us | 28,649.012 us | 2.0177× | -75.595% | +| 64 | 11,494.984 us | 23,090.927 us | 2.0088× | -80.330% | +| 128 | 9,341.260 us | 18,630.268 us | 1.9944× | -84.130% | + +B512 每档的 min～max 和标准差为： + +| turn-G | mean | min～max | sample SD | +| ---: | ---: | ---: | ---: | +| 1 | 117,221.275 us | 116,498.038～117,603.127 us | 496.065 us | +| 8 | 51,278.908 us | 51,098.560～51,382.715 us | 130.252 us | +| 32 | 28,596.962 us | 28,322.863～28,766.962 us | 197.351 us | +| 64 | 23,067.335 us | 22,898.211～23,189.274 us | 122.587 us | +| 128 | 18,633.632 us | 18,600.818～18,673.176 us | 31.573 us | + +task 数精确从 1,280 翻到 2,560 后，五档中位数倍率落在 +1.9944×～2.0334×；没有出现只在 B256 偶然成立的拐点。B512 内部相邻 +档位的中位数变化为 G32/G8 -44.173%、G64/G32 -19.401%、 +G128/G64 -19.318%，G128 仍为已测最优。B512 最终闭合 +2,560 task、4,096 个 published output、413,138,944B reservation， +normalized writer signature 为 `e2107e7bc78ff5d4`。 + +#### 产物身份与静态差异 + +最终 B512-capable perf-clock artifact 的 SHA256 为： + +```text +G1 host=7fad76a055ed8656eff7ee6325800493ea7b6d5bb7c9f234fa4b745f0f4b161b + kernel=3c5312572d613d40b573de9ecf712c709c8f475c977da191702a7f9b7fec239d +G8 host=85ce78fc7c74d02e2b3969c707cb81ce6a2463c4cfc2ebce75299c67fa376ece + kernel=0e85d4cdf9a345197bedf1956c4aa40402c4919381cd73484ce79e5d6437342a +G32 host=c9655a7c2c636eb25d352f187d03a88264828dd957cdd5d89597f1ef9b4c4a5d + kernel=59f376b6034ce7f30a56eaa2dba5fb74cad06dec49b3c0d48364b3c2c124c589 +G64 host=61da81157e0e4ec39fb3395dc9054c49dcdfd2e48ad27947430bb6dbcaa80055 + kernel=29ec1ee052036ad83123b1372acb3e8b3dc3e1c6c27b8577e629f072086ae10e +G128 host=d5094f3bc4cf404ead843191b0667779fb6a27c59a8cdd2e00af4b40dc520ea1 + kernel=4f8e20e87a272a74453f74740199ec6f134b1a32f5fba4a3efcbc58e8baa294e +``` + +G1 kernel `.text/.rodata` 为 157,496/496B；G8/G32/G64/G128 均为 +158,008/496B。因而 G8 之后四档具有相同 section 大小，G32→G64→G128 +的实测差异不能归因于 `.text` 或 `.rodata` 继续增长。 + +#### 结论和边界 + +在本用例的 96-worker、单条全局 writer baton 模型下，主要代价确实来自 +future owner 对少数 turn cache line 的轮询竞争。把 token 交错到更多 +物理线没有减少 task 数、writer 发布次数或全局有序语义，却显著缩短完整 +Submit；从 B256 到 B512，收益排序和相对幅度保持稳定。当前工程判断是： +在已测集合中优先选择 G128 做后续 shared TensorMap 候选。 + +该判断只覆盖 standalone PA Case1、当前任务到达模式和 G≤128。它没有 +证明 production 主流程必然获得同等比例，也没有把下降全部归因于某一种 +硬件 atomic 事件；要迁移到主流程，仍需单独核对 state 成本、真实任务图 +到达顺序和 production 的兼容边界。 + +### 2026-07-28：R5e 用 per-task 完成字替代 insert-turn baton + +#### 目标纠正 + +本轮先撤回了一个错误过程态：不能把 Claim 的竞争地址改成 +`TaskCell::deps_prepared` 后又保留 G128 insert-turn；那既混用了 +`deps_prepared` 的职责，也没有消除真正昂贵的插入等待。最终协议保持 +Claim 不变，只替换 TensorMap 插入完成链： + +```text +Claim: + Alloc -> alloc_cursor[task % 4] + QK / PV -> cube_cursor[task % 4] + SF / UP -> shared_vector_cursor[task % 8] + +唯一 Claim owner: + Materialize + 构造 writer delta + task 0: 直接进入插入段 + task N>0: 只 atomic-load task[N-1].deps_prepared + 发布 task N 的 ordinary / symbol / fresh-output writer 元数据 + CAS task[N].deps_prepared: -1 -> N + 离开有序段 + fanin lookup -> Build -> slot / 执行 + +Claim loser: + 不读 deps_prepared + 不读 TensorMap + 直接完成轻量 Submit / replay +``` + +因此只有 writer 元数据插入保持严格 `N -> N+1`；N 发布自己的完成字后， +N+1 owner 可以开始插入，而 N 的 fanin lookup、Build 和任务执行可以与 +后续 owner 并行。lookup 仍只接受 `producer∈[N-H,N)`，不会因共享表中 +已经存在更晚 task 的信息而越界消费。 + +generation 11 暂时保留 generation 10 的 128 条 sidecar turn 物理线和 +manifest-v3 字段，默认 G1 只用于历史 ABI 身份。host 与 CPU 门槛要求 +`committed_tasks==0`、其余 127 条线均为 -1，且生产热路径对这些地址的 +atomic load/CAS 次数均为 0。彻底删除这段 sidecar 属于未来 +generation 12 / manifest-v4 的 ABI 清理，不与本轮热路径实验混做。 + +`deps_prepared` 与 `flag/vend` 共处 64B `TaskCell`，但本路径不对该 +cache line 执行 DCCI：前驱使用 atomicAdd(0) 读取，当前 task 使用 +atomicCAS 发布。writer descriptor/history/ordinary payload 各自沿用已有 +FlushRegion/DCCI 发布协议。这样不落入 `ATOMIC_USAGE_GUIDE.md` 已证实的 +“同一 dirty cache line 后续 DCCI 覆盖 atomic 新值”场景；若以后新增 +TaskCell DCCI，必须重新验证，不能沿用本结论。 + +#### 原子访问闭合 + +正式热路径只保留一次资格检查： + +- task 0 没有前驱 load； +- task N>0 的一次 Wait episode 只访问 + `task[N-1].deps_prepared`，循环次数聚合成一条 PollBatch； +- 每个 task 恰好一次 CAS 发布自己的完成字； +- metadata helper 不再重复读取前驱或自身完成字。 + +泳道沿用数字 site 19/20 和既有十列 raw ABI，但显示名改为 +`shared_insert_predecessor_poll` 与 +`shared_insert_completion_publish`；Register 三段相应为： + +```text +register.wait_predecessor_insert +register.publish_metadata +register.publish_insert_completion +``` + +task 0 仍保留第一段时间边界用于 Register 整数闭合，但不伪造 PollBatch。 +converter 对每个 winner 逐 task 校验：task 0 必须 0 条前驱 PollBatch， +task>0 必须 1 条，每个 task 必须 1 条 completion CAS。 + +#### 正确性门槛 + +CPU 定向测试覆盖并通过： + +- Alloc/QK/SF/PV/UP 原 Claim 地址、候选核数、唯一 winner 与重复 loser； +- Claim 全程不触碰 `deps_prepared`； +- task 0 零前驱读取，N 只读 N-1，CAS 只允许 `-1 -> N`； +- 260-task 顺序、pending 后唤醒、空 writer 集合推进； +- 损坏前驱、预置当前值和重复发布均 fail-closed； +- 96-worker 完整 Submit 的 Claim、每 task 完成字、业务 flag、final + writer、真实任务、barrier 与 Build/执行 overlap； +- 旧 sidecar 128 条 canary 终值和原子访问计数均保持初值。 + +完整 `./cpu/build.sh shared swimlane` 通过。converter 与 analyzer 的 +141 项 Python 回归通过。CCEC AIC/AIV generic protocol、mixed ELF、 +split finish 与 host runner 构建通过。 + +A5 B1 先行门槛的 5 个完成字精确为 0…4，旧 sidecar 为 +`[0,-1,-1,-1,-1,-1,-1,-1]`，全部业务断言、真实计算、泳道和后处理 +均 PASS。 + +#### A5 B256 泳道与性能 + +固定参数为 shared TensorMap、context 8192、real-compute +`6,28,4,1`、final barrier `two-16`、96 workers。 + +泳道结果： + +- 1,280 tasks、73,728 Claim； +- 481,198 raw records，0 drop； +- 1,279 条前驱 PollBatch； +- 1,280 条完成 CAS； +- Submit 9,091.529 us； +- execution / semantic / postprocess 全部 PASS。 + +与同口径历史 G128 泳道的 Submit 10,193.173 us 相比，新图减少 +1,101.644 us（10.8077%）。Register 前驱等待的 aggregate core-work +由 634,904,949 cycles 降到 550,131,603 cycles，减少 84,773,346 +cycles（13.3521%）；metadata 发布与完成 CAS 的记录口径保持独立。 +这与优化目标同向，但 aggregate core-work 不是 wall-clock，不能把 +84.8M cycles 直接换算成 1.10 ms 的端到端收益。 + +三次独立 trace-free `perf-clock`： + +| run | Submit | +| ---: | ---: | +| 1 | 8,406.504 us | +| 2 | 8,326.011 us | +| 3 | 8,376.102 us | +| 中位数 | 8,376.102 us | + +迁移前同设备、同业务参数的 G128 历史中位数为 9,371.635 us；新协议 +减少 995.533 us，即 10.6228%。两边构建都关闭泳道、atomic trace、PMU +和 kernel 计时；新协议 manifest 中虽然仍写默认 G1，但 G 已不参与生产 +热路径。该对照支持保留 per-task 完成链，不能外推成 simpler 真实路径 +已有同等收益。 + +归档位于： + +```text +tests/atomic_probe/pa_scheduler/test_record/2026-7-28-shard/ + per_task_deps_prepared_b256/ + manifests/per_task_deps_prepared_swimlane_artifacts.manifest + manifests/per_task_deps_prepared_perf_clock_artifacts.manifest +``` + +### 2026-07-28：R5f 细分 Register 的 writer metadata 与 task outputs + +#### 观察目标与边界 + +R5e 的 Register 只能看到前序等待、metadata 总区间和插入完成发布， +其中 metadata 仍把 writer metadata 与 `PublishSharedTaskOutputs` +混在一起。本轮不改 shared TensorMap 协议，只补充一条最小 raw +子区间，回答这两类工作各占多少： + +```text +Register +├─ register.wait_predecessor_insert +├─ SharedRegisterPublishMetadata +│ ├─ register.publish_writer_metadata +│ ├─ SharedRegisterPublishTaskOutputs +│ └─ register.publish_metadata_epilogue +└─ register.publish_insert_completion +``` + +这不是“本核私有工作”和“其他核发布”的区分。上述动作都由当前 +Claim winner 所在的 scalar 执行；writer metadata 也会访问 shared GM、 +执行 writer intent/append/symbol 发布，`PublishSharedTaskOutputs` 则专门 +发布 fresh-output cell。准确的区分是“owner 侧 writer metadata 路径” +与“fresh shared task outputs 发布路径”。 + +设备端只新增 +`TracePhase::SharedRegisterPublishTaskOutputs`，每个成功 winner 固定一条 +64B record。现有 `SharedRegisterPublishMetadata` 继续作为父 detail； +Register、metadata、outputs 的所有时间端点读取完以后，才依次写 +`R -> M -> O` raw。没有逐 helper 或逐 poll 追加记录，B256 只增加 +1,280 条有效 raw，约 80 KiB；固定 trace allocation 不变。 + +raw schema 从 4 升为 5，排他分析报告从 2 升为 3。当前采集和加工是一体 +版本，不给旧 schema-v4 猜测或补造 outputs 区间。host、converter 和 +analyzer 共同强制： + +- 每个 shared winner 恰好一个 M 和一个 O，loser 为零； +- R、M、O 的 `(core, lane, task, function)` 完全一致； +- `R.start <= M.start <= O.start <= O.end <= M.end <= R.end`； +- `M = writer metadata + task outputs + metadata epilogue`； +- `R = predecessor wait + M + insertion completion`； +- M/O 只是 overlay，不重复加入 Submit 的可加总阶段。 + +#### 正确性与构建门槛 + +CPU shared swimlane、shared trace-free、private swimlane 和 sparse-trace +负例门槛全部通过；负例覆盖 outputs 缺失、重复、越过 metadata、 +task/function 不一致和非零 payload。用户本地 Python 环境运行全部 +143 项 `pa_scheduler` Python 回归通过。CCEC shared swimlane 的 AIC/AIV、 +split runtime/finish、mixed ELF、LOCAL helper、无 relocation 和 +artifact manifest 门槛通过。 + +本轮遵照要求不运行 A5 B1，CCEC 构建后直接运行 A5 B256。 + +#### A5 B256 结果 + +固定参数仍为 shared TensorMap、context 8192、real-compute +`6,28,4,1`、final barrier `two-16`、96 workers。结果为： + +- 1,280 tasks，1,280 Register； +- metadata detail 1,280 条，task-output detail 1,280 条； +- 1,279 条前序 PollBatch，1,280 条完成 CAS； +- raw records 482,387，expected 482,387，drop 0； +- Submit 9,405.962 us； +- execution、semantic、postprocess 和 analysis validation 全部 PASS。 + +按 96 核累计的 Register core-work： + +| 区域 | cycles | 父区间占比 | +|---|---:|---:| +| Register parent | 587,885,050 | 100% | +| 前序插入完成等待 | 578,829,069 | Register 的 98.460% | +| metadata 总区间 | 8,365,557 | Register 的 1.423% | +| 插入完成发布 | 690,424 | Register 的 0.117% | +| writer metadata | 2,311,441 | metadata 的 27.630% | +| `PublishSharedTaskOutputs` | 5,963,620 | metadata 的 71.288% | +| metadata 收尾 | 90,496 | metadata 的 1.082% | + +两层整数 cycle 闭合和五段扁平闭合均精确成立。最直接的结论是： +Register 的主要累计开销仍是等待 N-1 完成插入；去掉等待后，metadata +内部以 `PublishSharedTaskOutputs` 为主，约占 71.3%。这里是 +aggregate core-work，不是 wall-clock；不能把 578.8M cycles 换算成 +Submit 可直接减少的微秒数。 + +新增边界本身包含两次取时，并新增一条 raw。新图 Submit 比 R5e 旧图 +9,091.529 us 高 314.433 us，不能据此宣称业务回退；本轮没有生成新的 +trace-free 性能样本，结论只用于 Register 内部归因。 + +归档位于： + +```text +tests/atomic_probe/pa_scheduler/test_record/2026-7-28-shard/ + per_task_deps_prepared_register_detail_b256/ + manifests/per_task_deps_prepared_register_detail_swimlane_artifacts.manifest +``` + +### 2026-07-28：R5g 细分 fresh-output descriptor copy 与 flush + +#### 观察目标与实现边界 + +R5f 已证明 `PublishSharedTaskOutputs` 是 Register metadata 内的主要非等待 +工作，但它仍把 descriptor copy、DCCI flush、writer 起点和 published +控制协议混在同一包络。本轮只细分观测，不改变输出身份和跨核可见性协议： + +```text +SharedRegisterPublishTaskOutputs +├─ SharedRegisterPublishTaskOutputsCopy +├─ SharedRegisterPublishTaskOutputsFlush +└─ residual +``` + +实现先整批把每个 128B `TensorDesc` 复制到 +`shared_outputs[task_id].tensors[]`，再对连续 descriptor 区域统一 +`FlushRegion`，随后保持原有 `StoreBarrier -> published Exchange`。 +`residual` 由父子端点离线计算，覆盖完整预检、`last_writer` 预留、 +barrier、published 和返回路径；它不是新增的设备 trace。零输出 task +仍保留零时长 copy/flush 边界，保证每个 winner 的 raw 结构固定。 + +converter 和 analyzer 同时验证： + +- 每个 shared winner 恰有一组 outputs/copy/flush，loser 为零； +- `copy.end == flush.start`，二者严格位于 outputs 父区间； +- `outputs = copy + flush + residual` 按整数 cycle 精确闭合； +- 这些 detail 都是 overlay，不进入 Submit 排他阶段的二次加总。 + +#### A5 B256 结果 + +测试参数继续固定为 shared TensorMap、context 8192、real-compute +`6,28,4,1`、final barrier `two-16`、96 workers。结果为： + +- 1,280 tasks，raw records 485,028，drop 0； +- execution、semantic、postprocess 和 analysis validation 全部 PASS； +- Submit 9,466.451 us。 + +按 96 核累计 core-work： + +| outputs 内区域 | cycles | 占 outputs | +| --- | ---: | ---: | +| `PublishSharedTaskOutputs` | 6,067,592 | 100% | +| descriptor copy | 2,094,172 | 34.514% | +| `FlushRegion` | 588,668 | 9.702% | +| residual | 3,384,752 | 55.784% | + +这里证明的是同一包络内部的相对组成。该版本比 R5f 多两条 raw 和内部取时， +不能用两个泳道的绝对 Submit 差值推导业务性能变化。 + +### 2026-07-28：R5h 将 fresh-output 发布移出全局有序插入区 + +#### 业务语义 + +fresh output 的关联键在 Claim 后已经固定为 +`(producer_task_id, output_slot)`。每个 task 只有一个 Claim winner，且 +winner 只写自己独占的 `shared_outputs[task_id]`，因此 descriptor 发布 +不依赖 ordinary region 或 symbol writer 的全局 task-ID 插入轮次。 + +最终顺序调整为： + +```text +唯一 winner: + Materialize descriptor + writer delta + PublishSharedTaskOutputs 到 task 独占 cell + copy -> flush -> StoreBarrier -> published + + task 0: 直接进入有序 writer 区 + task N: 等待 task[N-1].deps_prepared + 发布 ordinary / symbol writer metadata + CAS task[N].deps_prepared: -1 -> N + + fanin lookup -> Build -> slot / 执行 +``` + +这里没有删除 `PublishSharedTaskOutputs`，也没有减少其 128B descriptor +copy、flush、`last_writer` 或 published 控制操作；只是把这组独占 cell +工作从 Register 串行等待之后提前到 Materialize 尾部，使不同 task 的 +输出发布能够并发。后续 INOUT/OutputExisting 的 writer history 和 +`last_writer` 更新仍位于有序 writer 元数据协议内。 + +失败路径仍然 fail-closed：output 发布失败立即设置 fatal；若后续 writer +metadata 或 completion handoff 失败，则回滚本 task 独占 cell，不发布 +成功的 `deps_prepared`。`published` 只表示 descriptor 可读，不表示 +producer kernel 已执行完成；执行依赖仍由 fanin 和独立 completion flag +表达。 + +#### A5 B256 trace-free 性能 + +为避免用带观察开销的泳道判断净性能，本轮在完全关闭泳道、 +atomic trace、PMU、phase 和 kernel timing 的 `perf-clock` 构建上做 +同机交错 A/B。两侧都固定为 256 batches、1,280 tasks、96 workers、 +shared TensorMap、context 8192、real-compute `6,28,4,1` 和 final +barrier `two-16`；每个进程的运行期断言都确认这些观察能力保持关闭， +execution、semantic 和 postprocess 均为 PASS。 + +| 版本 | 四个正式样本（ms） | 中位数 | +| --- | --- | ---: | +| outputs 仍位于 Register 串行区 | 8.356157 / 8.356489 / 8.351957 / 8.355702 | 8.3559295 ms | +| outputs 移到 Materialize | 2.906899 / 2.941103 / 2.939350 / 2.951372 | 2.9402265 ms | + +中位数减少 5.4157030 ms，即下降 64.8127%，前后比为 2.8419×。after +四个样本落在 2.906899～2.951372 ms；这不是单次偶然值，也不是把 +level-4 泳道时间误写成性能。原始日志位于： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + output_publish_move_ab_20260728_104448/ +``` + +该结果是本轮“把 fresh-output 独占 cell 发布移出全局 writer 串行区”的 +主要性能证据。R5e 的 8.376102 ms 是只完成 per-task completion 链时的 +旧中间基线，不能代替最终版本约 2.94 ms 的结果。 + +#### 泳道口径与 A5 B256 结果 + +raw detail 相应更名并迁入 Materialize： + +```text +Materialize +├─ materialize.before_publish_task_outputs +├─ materialize.publish_task_outputs +│ ├─ materialize.publish_task_outputs.copy +│ ├─ materialize.publish_task_outputs.flush +│ └─ materialize.publish_task_outputs.residual +└─ materialize.after_publish_task_outputs + +Register +├─ register.wait_predecessor_insert +├─ register.publish_writer_metadata +└─ register.publish_insert_completion +``` + +同一固定参数下的最新泳道结果： + +- 1,280 tasks，raw records 486,416，drop 0； +- 每个 winner 恰有一组 Materialize outputs/copy/flush； +- Register 内三类历史 task-output detail 计数均为 0； +- execution、semantic、postprocess 和 analysis validation 全部 PASS； +- Submit 3,464.587 us。 + +按 96 核累计 core-work： + +| Materialize 区域 | cycles | 占 Materialize | +| --- | ---: | ---: | +| Materialize 总区间 | 12,743,376 | 100% | +| output 发布前 | 6,591,184 | 51.722% | +| `PublishSharedTaskOutputs` | 6,026,512 | 47.291% | +| output 发布后 | 125,680 | 0.986% | + +| outputs 内区域 | cycles | 占 outputs | +| --- | ---: | ---: | +| descriptor copy | 2,071,279 | 34.369% | +| `FlushRegion` | 603,020 | 10.006% | +| residual | 3,352,213 | 55.624% | + +| Register 区域 | cycles | 占 Register | +| --- | ---: | ---: | +| Register 总区间 | 15,678,761 | 100% | +| 前序插入完成等待 | 12,665,626 | 80.782% | +| writer metadata | 2,409,396 | 15.367% | +| 插入完成发布 | 603,739 | 3.851% | +| Register 内 task outputs | 0 | 0% | + +迁移后的 output 包络为 6,026,512 cycles，和迁移前的 6,067,592 cycles +同量级，证明工作被移动而不是被漏记。3,464.587 us 是 +level-4/atomic-trace 观测构建的端到端结果，不用于计算净收益；净性能以 +上一节四组无泳道 A/B 的 8.3559295 -> 2.9402265 ms 为准。两种独立口径 +方向一致，同时分别回答“性能是否改善”和“原工作是否被漏记”。 + +归档位于： + +```text +tests/atomic_probe/pa_scheduler/test_record/2026-7-28-shard/ + per_task_deps_prepared_task_outputs_copy_flush_b256/ + per_task_deps_prepared_materialize_task_outputs_b256/ +``` + +### 2026-07-28：R5i 消减 ordered Register 的重复工作 + +本轮先做不改变 shared TensorMap 总协议的低风险消减： + +- `PrepareSharedTaskWriterDelta()` 在串行区外冻结 task ID、ordinary 数量、 + symbol 数量和 writer-intent 一致性； +- `PublishSharedTaskWriterMetadata()` 不再重复执行 + `InspectSharedWriterIntent()`、`ValidateSharedWriterIntentSet()` 和 ordinary + producer 扫描； +- `reclaim_upto == -1` 明确走 no-retire 预检，只保留 head、tail、容量和 + 目标 seq 检查，不再读取或 invalidate 已发布的旧 payload； +- `map_inserts` 与 `shared_symbol_inout_commits` 在 + `task[N].deps_prepared` completion CAS 成功且 Register 取时结束后记账。 + +最后一项区分“metadata 已经写出”和“完整 task 插入事务已经提交”： +completion CAS 失败时保留 fatal、共享 metadata 和 observed value 作为故障 +现场，但不把该前缀计入成功统计。通用 writer-intent helper 仍保留原有 +“成功 CAS 前缀逐项计数”语义，只有正式 ordered Submit 使用延迟记账。 + +定向测试证明： + +- `reclaim_upto == -1` 仍执行 append fail-closed 预检，但不会访问旧 + payload；非法 `reclaim_upto < -1` 在共享访问前失败； +- ordinary 与 symbol metadata 已发布、completion CAS 冲突时，两项成功 + 统计均保持 0； +- 96-worker ordered Submit 继续满足 loser 零 TensorMap 访问、 + completion-before-Build 释放和独立 kernel 重叠。 + +系统 GCC 13 下的 shared CPU 全量门槛通过，包括五档 ring capacity、 +symbol history、writer-intent、heap、Claim、Materialize 和 ordered +Submit。CCEC AIC/AIV generic probe、两类入口、compete-first 拆分 TU、 +mixed ELF 与 host runner 均构建通过。该阶段尚未单独生成 A5 B256 性能 +样本，因此这里只记录结构和构建结论，不宣称已有上板收益。 + +### 2026-07-28：R5j 把静态提交计划移出 ordered Register + +第二批把只有 winner-local 输入参与的计算移到 predecessor 等待之前： + +- ordinary writer 预计算 bucket 和同 bucket 局部序号；正式 preflight + 直接使用 `tail + ordinal`，不再执行 `TensorMapHash()` 或 + `SharedEarlierEntriesInBucket()`； +- symbol writer 预计算 packed key 并在本地拒绝重复 key；取得 insert + turn 后只读取当时最新的 `last_writer`，写 history 并发布 CAS； +- 正式 symbol commit 不再分配、清零、去重和写入随后立即丢弃的 + `ignored_fanin[kMaxFanin]`。completion CAS 之后的权威 + `CollectSharedFanin()` 仍负责完整依赖收集和容量校验； +- ordinary append 增加按预计算 bucket 发布的专用入口；通用 ring helper + 继续按地址 hash，隔离测试和旧调用不改变。 + +bucket 使用 `uint16_t`，不是只按默认 128 buckets 取 `uint8_t`。原因是 +CPU ring 门槛还会编译 CAP=32、512 buckets 的真实变体；同 bucket 序号 +最多 31，继续使用 `uint8_t`。默认 CAP=128 时新增 winner-local plan 为 +32 个 symbol key、32 个 bucket 和 32 个 ordinal，不修改任何 shared ABI。 + +#### output published 单次检查的前提 + +producer P 的正式顺序为： + +```text +descriptor copy +-> FlushRegion +-> published[P][slot] atomic +-> task[P].deps_prepared completion +``` + +task completion 又按 `P -> P+1 -> ... -> N-1` 逐 task 传递。task N 只有在 +观察到 `task[N-1].deps_prepared` 后才能进入 writer metadata，因此任意合法 +`P < N` 的 published 必须已经成立。正式 prepared symbol commit 和 +`CollectSharedFanin` 据此各执行一次 atomic Load：精确命中 +即继续，未命中立即作为协议错误终止，不再打开 SYS_CNT watchdog 或 spin。 + +没有取得 insert-turn 前提的通用 `WaitForSharedOutputPublished()` 保持原样， +其延迟发布、fatal 和 watchdog 测试也全部保留。这里不是全局删除等待， +而是用两个不同 API 明确区分“状态仍可能变为合法”和“前序完成链已证明 +状态必须合法”。 + +#### 正确性和构建证据 + +CPU 定向门槛新增并通过： + +- ordinary `A,B,A,A` 计划得到 `0,0,1,2` 局部序号，prepared 与 generic + 路径生成相同逻辑 map；容量失败不产生部分发布； +- CAP=32/64/128/256/16384 五档 ring 全部通过，包含 512 buckets 与单 + bucket 两端； +- prepared symbol commit 与 ordered latest-writer lookup 都恰好读取一次 + published，`Now()` 和 `SpinHint()` 调用为 0；未发布值一次读取后立即 + 失败； +- 96-worker ordered Submit 继续满足 loser 零 map 访问、insert completion + 先于 Build 释放和独立 kernel 重叠。 + +CCEC shared swimlane 与 perf-clock 的 AIC/AIV generic probe、入口、 +compete-first split finish、mixed ELF 和 host runner 均构建通过。相对 R5i +的 swimlane 构建，AIC/AIV 入口对象大小保持 +`1,552,384 / 1,535,152 B` 不变；finish AIC/AIV 分别从 +`752,728 / 751,680 B` 降至 `741,544 / 740,560 B`，mixed kernel 从 +`2,829,232 B` 降至 `2,815,456 B`。因此当前证据没有显示预计算 plan +造成设备代码膨胀，反而因移除慢路径重复逻辑而缩小。 + +#### A5 B256 最终验证 + +最终源码重新构建了 shared G1 的 swimlane 与 perf-clock CCEC +artifact。架构预检因本机没有 `npu-smi` 而无法自动识别 silicon， +`task-submit` 也不在 PATH；沿用本仓库前序记录中的用户明确授权，在已知 +A5 的 device 0 上各执行一次无锁验证。这里诚实记录为无锁样本，不声称 +取得设备独占。 + +swimlane B256 固定 1,280 tasks、96 workers、context 8192、 +real-compute `6,28,4,1` 和 final barrier `two-16`，结果为： + +- execution、semantic、postprocess、依赖签名、fresh-output descriptor、 + writer history、per-task completion 和 real-compute 输出全部 PASS； +- 485,925 条 raw record，drop 0；exclusive analyzer 的全部闭合检查 + PASS； +- Submit 为 `3,392.893 us`。 + +与 R5h 完成后、R5i/R5j 之前的同口径泳道相比： + +| 指标 | R5h 基线 | R5i+R5j | 变化 | +| --- | ---: | ---: | ---: | +| Submit wall-clock | 3,464.587 us | 3,392.893 us | -2.069% | +| Register 累计 core-work | 15,678,761 cycles | 8,365,867 cycles | -46.642% | +| predecessor wait | 12,665,626 cycles | 6,169,187 cycles | -51.292% | +| writer metadata | 2,409,396 cycles | 1,537,416 cycles | -36.191% | +| insert completion | 603,739 cycles | 659,264 cycles | +9.197% | + +这组分解证明串行区消减确实落在 Register metadata 与连带等待上； +insert-completion 的小幅增加没有对应代码扩张，只按单次无锁样本记录， +不解释为稳定回退。两份泳道分别位于： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + pa_scheduler_shared_swimlane_20260728_105926_1598639/ccec/ + pa_scheduler_shared_swimlane_20260728_131823_1696692/ccec/ +``` + +最终源码的独立 perf-clock B256 同样全部 PASS，Submit 为 +`2,482.874 us`。它关闭泳道、atomic trace、PMU、phase 与 kernel timing； +相对 R5h 的四样本中位数 `2,940.2265 us` 方向一致，但本次只有一个无锁 +样本，不能把 `-15.555%` 当作完成交错复测后的稳定净收益。 + +本轮把 ordinary-ring visibility probe 接入 CCEC 探针集合，并直接复用 +production shared TensorMap 的 publish/read/lookup helper。两个 AIV +轮换 writer/reader；reader 在发布前以普通 scalar load 预热旧 +commit/reclaim/head/tail/seq/payload cache line，只观察 production +`committed_tasks` 交权，窗口内不增加额外 DCCI、DSB 或全核同步。 + +device 0 连续运行 20 个独立 launch，每次 394 task，覆盖空 task、同 bucket +双 entry、跨 bucket 双 entry、writer 轮换以及 CAP=128 三次以上回绕： + +- 7,880 个 task 全部完成，timeout、overshoot 和 first error 均为 0； +- 20/20 的 commit、reclaim、head、tail、sequence、payload、read 和 + lookup 精确匹配； +- A bucket 推进 394 个 entry（3.078 圈），B bucket 推进 392 个 entry + （3.062 圈）； +- `protocol_failures=0`、`semantic_failures=0`。 + +该探针证明 ordinary region 在“旧 cache line 已预热”的受控场景下，现有 +production 发布链能跨核稳定传递。它仍不覆盖 fresh-output descriptor、 +per-task completion 与单次 published Load 的完整组合，因此不能替代正式 +shared PA B256 验证；两条证据链保留各自边界。 + +### 2026-07-28：R5k 消减 shared loser 的 winner-only 上下文初始化 + +#### 先把 1～2 us 拆成真实路径 + +基线 raw： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + pa_scheduler_shared_swimlane_20260728_131823_1696692/ccec/ +``` + +该 B256 样本共有 1,280 个 winner 和 121,600 个 loser。loser 的 +Submit 中位数为 1.305 us，严格闭合为： + +| 区域 | 中位数 | loser 累计 core-work 占比 | +| --- | ---: | ---: | +| EfDrain | 0.200 us | 40.089% | +| Claim | 0.624 us | 35.292% | +| Claim.end 到 Submit.end | 0.429 us | 24.619% | + +只有 1,008 个 loser Submit（0.829%）在 EfDrain 中捎带执行真实 +kernel；它们解释 55～64 us 长尾，不能解释常见的 1～2 us。严格落在 +1～2 us 的 66,714 个 loser 中没有 kernel，也没有 EfDrain +`fanin_flag_load`；其中 78.187% 执行过 ClaimMax。 + +泳道边界还包含观察代码本身： + +- `efdrain_end` 之后写入的 64-byte EfDrain raw record 落入外层 Claim； +- ClaimMax 返回端点之后写入的 atomic record 也落入外层 Claim； +- `claim_end` 之后写入的 Claim record 落入 Claim-to-SubmitEnd 尾部； +- Submit record 在 `submit_end` 之后写入，落入下一段 SubmitTransition。 + +因此 1～2 us 不是“loser 又执行了 TensorMap/Build”。源码审计确认 +post-Claim loser 没有 TensorMap 访问、参数构造、DCCI、kernel 或新的 +atomic；它只建立稳定 output symbol，执行 split replay 记账、紧凑协议 +校验和 Submit 收尾。 + +#### 已撤回：只缩短泳道尾部、没有净性能收益的候选 + +第一版曾把 0/1/3 个稳定 output slot 改为一次性计数，并让同 TU loser +复用已校验的 batch plan，跳过 ticket meta 的再次解码。B256 泳道中 +post-Claim 中位数从 429 ns 降到 392 ns，约下降 8.6%；但 10 个独立 +perf-clock 样本没有同向变化： + +| 版本 | 中位数 | 均值 | +| --- | ---: | ---: | +| 基线 | 2,458.6665 us | 2,458.9239 us | +| direct-close 候选 | 2,461.4920 us | 2,459.9551 us | + +中位数差 `+0.115%`、均值差 `+0.042%`，均在本轮波动内。该候选只让 +带泳道的观察结果变短，没有证明 trace-free 业务执行变快，因此已经完整 +撤回，不作为性能优化保留。 + +#### 保留：shared actor 只初始化自己实际消费的字段 + +原 `BeginCallbackSubmit()` 在所有 96 个 replay actor、每个 task 上都 +清零整份 shared `SubmitContext`，其中 payload/result/fanin/joint 等字段 +只会被 1 个 Claim owner 消费。新路径拆为： + +```text +所有 shared actor： + local_index++ + context.task_id = task_id + context.shared_result.Reset(task_id) + +仅 Claim owner： + 绑定 worker/payload + 初始化 result.task_id/result.count + 初始化 fanin_count + 构造 TaskArgs，进入 Materialize/Fanin/Build +``` + +`context.won` 和 `context.kernel_id` 仍由 Claim 结果对所有 actor 明确 +覆盖；`tensor_count/scalar_count/register_mask/output_bytes` 在 winner +读取前由 `MaterializeTask()` 覆盖；joint 字段只属于 private BlockWon， +shared 单 lane PA 不读取。private 继续使用完整 +`BeginCallbackSubmit()`，没有改变原语义。 + +#### 正确性、泳道与净性能 + +shared CPU 全量门槛通过，包括 5 档 ordinary ring、symbol history、 +writer intent、heap、Claim、Materialize 和 96-worker ordered Submit。 +CPU B256 G1 的完整 host oracle、依赖签名和最终 TensorMap 投影全部 PASS。 +CCEC shared swimlane/perf-clock 的 AIC/AIV 入口、split runtime/finish、 +mixed ELF、host runner 和 manifest 校验全部通过。 + +最新 A5 B256 泳道： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + pa_scheduler_shared_swimlane_20260728_155832_1767994/ccec/ +``` + +- execution、semantic、postprocess 和 exclusive analysis 全部 PASS； +- 485,796 条 raw record，drop 0； +- Submit wall-clock 为 3,364.474 us； +- loser Submit 内部中位数仍为 1.289 us，符合本次不移动 Submit + 时间边界的预期； +- loser 后继 SubmitTransition 中位数由 399 ns 降到 335 ns， + 累计 core-work 从 67,378,398 降到 61,702,342 cycles，下降 8.424%。 + +这证明被删除的清零工作位于相邻 Submit 之间，而不是被伪装成 Claim 或 +loser tail。 + +当前源码的 10 个独立 perf-clock 样本为： + +```text +2444.000, 2428.917, 2459.165, 2417.391, 2436.721, +2457.734, 2482.708, 2432.841, 2421.812, 2436.210 us +``` + +中位数 2,436.4655 us、均值 2,441.7499 us。相对修改前同轮 10 样本， +中位数下降 22.201 us（0.903%），均值下降 17.174 us（0.698%）。 + +为避免把先后运行的状态漂移当成收益，又在同一 device 0 顺序执行 6 组 +`baseline -> candidate`： + +| 组 | 基线 | 候选 | 差值 | +| ---: | ---: | ---: | ---: | +| 1 | 2,436.254 us | 2,447.698 us | +11.444 us | +| 2 | 2,460.574 us | 2,413.855 us | -46.719 us | +| 3 | 2,485.481 us | 2,418.374 us | -67.107 us | +| 4 | 2,484.331 us | 2,437.374 us | -46.957 us | +| 5 | 2,469.693 us | 2,409.075 us | -60.618 us | +| 6 | 2,460.075 us | 2,426.713 us | -33.362 us | + +5/6 组同向改善，配对差值中位数为 `-46.838 us`；相对基线中位数 +2,465.1335 us 为 `-1.900%`。本机仍缺少 `npu-smi` 和 `task-submit`， +以上设备运行均明确属于 device 0 无锁样本；性能结论同时依赖配对方向、 +trace-free 结果和泳道中 SubmitTransition 的结构性消减，不用单次 +3.364 ms 泳道值冒充净收益。 + +#### 已撤回：把 split `task_id_sum` 移到回放封口 + +随后验证了第二个看似直接的 loser-tail 消减：保留每 task 的 +`task_id == stats.submits` 顺序断言，删除 +`runtime.task_id_sum += task_id`，并在回放结束时按实际完成 Submit 数一次 +生成 `N(N-1)/2`。从协议逻辑看，两者等价，而且逐 task 的精确 ID 比总和 +更强。 + +但 CCEC perf-clock 的同设备顺序交错结果明确回退： + +| 组 | R5k 基线 | 封口求和候选 | 差值 | +| ---: | ---: | ---: | ---: | +| 1 | 2,420.071 us | 2,491.181 us | +71.110 us | +| 2 | 2,412.474 us | 2,491.746 us | +79.272 us | +| 3 | 2,432.161 us | 2,489.907 us | +57.746 us | +| 4 | 2,419.581 us | 2,485.778 us | +66.197 us | +| 5 | 2,418.929 us | 2,498.761 us | +79.832 us | +| 6 | 2,439.582 us | 2,494.907 us | +55.325 us | + +6/6 组回退，配对差值中位数 `+68.6535 us`，相对基线中位数 +2,419.826 us 为 `+2.837%`。候选虽然少一次热路径加法，却改变了 CCEC +代码布局、活跃值和最终封口代码；实测总代价更高。该候选已经完整撤回， +继续保留原逐 task 求和。这里不能用源码指令条数推翻设备结果，也不能把 +单次带泳道的 3,291.510 us 当成反证。 + +#### 已撤回：把固定 PA Claim 改成编译期 Kind 路由 + +第四个候选为五个 `SubmitCallbackTask` 调用点分别实例化 +`ClaimForKind`，直接固化 role、function 和 cursor；动态 `Claim(kind)` +只留给隔离门槛。该实现语义和现有单 lane PA 一致，CPU 全量门槛及 CCEC +两类构建也全部通过。 + +单次 B256 泳道中，loser Claim 累计 core-work 只从 74,503,213 降到 +74,313,310 cycles，下降 0.255%；Submit 为 3,347.626 us。这个量级不足以 +单独证明净收益。随后固定同一 device 做两种顺序的 perf-clock 交错： + +- 6 组 `baseline -> candidate`：5/6 改善，配对差值中位数 + `-22.645 us`； +- 4 组 `candidate -> baseline`：0/4 改善，candidate 相对后跑 baseline + 的配对差值中位数 `+20.4365 us`； +- 合并 10 组后仅 5/10 改善，candidate-minus-baseline 中位数 + `-7.7365 us`、均值 `-6.2392 us`。 + +正反顺序结论相反，说明相邻进程约 20 us 的顺序/热状态影响已经大于 +候选本身。该版本没有达到“性能收益方向独立于运行顺序”的保留门槛， +已经完整撤回；同时保留原动态 active-mask/popcount 路由，使 standalone +继续贴近生产 Claim，而不是为了未证明的微小收益固化 PA 特例。 + +#### 已撤回：由 orchestration 直接生成稳定 output symbol + +第五个候选利用 shared fresh-output handle 只由 `(task_id, slot)` 决定 +这一事实：loser 不再初始化 `context.shared_result`，而是在每个 Submit +返回后由 orchestration 直接生成同一组稳定 symbol；winner 仍完整构造 +`shared_result`，供 Materialize、publish 与 split finish 校验使用。 + +该候选通过 shared CPU 全量门槛、CPU B256、CCEC swimlane/perf-clock +两类构建和 A5 B256 全部正确性断言。AIC/AIV perf-clock 入口分别从 +`698,120/700,072 B` 降至 `681,216/685,224 B`。带泳道样本也精确命中 +预期的 post-Claim loser 尾部： + +| loser 区域 | R5k | 候选 | 变化 | +| --- | ---: | ---: | ---: | +| post-Claim mean | 427.163 ns | 385.089 ns | -9.850% | +| post-Claim median | 408 ns | 376 ns | -7.843% | +| post-Claim p95 | 679 ns | 608 ns | -10.457% | +| Submit mean | 1,758.220 ns | 1,726.561 ns | -1.801% | +| 1～2 us Submit 占比 | 53.820% | 51.558% | -2.262 个百分点 | + +候选泳道位于： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + pa_scheduler_shared_swimlane_20260728_163004_1799490/ccec/ +``` + +但是无泳道 perf-clock 的三组对称 `baseline,candidate,candidate,baseline` +区组全部回退： + +| 区组 | 基线均值 | 候选均值 | 候选减基线 | +| ---: | ---: | ---: | ---: | +| 1 | 2,443.101 us | 2,449.460 us | +6.359 us | +| 2 | 2,442.733 us | 2,461.654 us | +18.921 us | +| 3 | 2,433.365 us | 2,459.288 us | +25.923 us | + +六个样本总体均值为 `2,439.733 -> 2,456.800 us`，回退 +`17.068 us（0.700%）`；中位数回退 `25.923 us（1.065%）`。这说明 +泳道里少掉的约 42 ns 尾部工作没有转化为端到端收益，反而改变了代码 +布局、寄存器活跃值或 orchestration 热路径成本。性能保留门槛以 +trace-free 结果为准，因此该候选已经完整撤回，不能仅凭目标区域变短 +而保留。 + +#### 已撤回：shared Claim 改为每 task 独占原子行 + +第六个候选没有减少 ClaimMax 次数，而是把 shared 的 Cube/Vector/Alloc +类型高水位 cursor 改为 `task_claim[task_id]`：每个逻辑 task 独占一条 +64-byte atomic-only cache line，private 继续使用原分片 cursor。该布局 +把 ownership、`deps_prepared` 插入完成、`flag/vend` 执行完成和 +shared-output 发布四类状态彻底分离。 + +候选在 sidecar 尾部增加 `4,352 × 64 = 278,528 B`，不移动既有热点字段； +shared ABI generation 从 11 升到 12。正确性门槛不仅验证最终值，还覆盖： + +- 五种 task 的精确候选数、每 task 唯一 winner 和合法 replay； +- 先 Claim `N+8`、再 Claim 旧 cursor 同 shard 的 `N`，两者分别获胜； +- `task_id == kMaxTasks` 在发 atomic 前拒绝； +- 旧值仅允许 `-1` 或本 task id，其他值触发 fatal； +- active/unused task claim 行和四类旧 cursor 的终态逐项核对； +- CPU B256 的完整依赖签名、TensorMap、Build/执行并发和 host oracle。 + +CPU 全量门槛、两类 CCEC 构建与 A5 B256 全部 PASS；ClaimMax 仍精确 +73,728 次。A5 泳道位于： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + pa_scheduler_shared_swimlane_20260728_165441_1825631/ccec/ +``` + +单条 ClaimMax 的改善是真实且明显的： + +| ClaimMax | R5k | per-task 候选 | 变化 | +| --- | ---: | ---: | ---: | +| mean | 376.778 ns | 297.980 ns | -20.913% | +| median | 299 ns | 264 ns | -11.706% | +| p95 | 740 ns | 455 ns | -38.514% | + +但无泳道 perf-clock 的三个对称 +`baseline,candidate,candidate,baseline` 区组全部回退： + +| 区组 | 基线均值 | 候选均值 | 候选减基线 | +| ---: | ---: | ---: | ---: | +| 1 | 2,444.538 us | 2,494.114 us | +49.576 us / +2.028% | +| 2 | 2,432.577 us | 2,499.022 us | +66.445 us / +2.731% | +| 3 | 2,446.668 us | 2,496.608 us | +49.940 us / +2.041% | + +六个样本总体均值 `2,441.261 -> 2,496.581 us`，回退 +`55.320 us（2.266%）`；中位数回退 `50.216 us（2.055%）`。同一批 +样本的 fanin loads 中位数还从 `54,241.5` 增至 `67,743.5`， +增加 `24.893%`。这与 per-task Claim 允许不同 worker 更自由地跨 task +超前、从而增加后续依赖轮询的机制一致；但当前证据只能把它写成同轮伴随 +变化，不能把全部墙钟回退唯一归因于 fanin。 + +结论是：类型高水位 cursor 的跨 task 共享确实拖慢单条 atomic，却同时 +提供了有益的进度约束。完全 per-task 化优化了局部 ClaimMax、恶化了完整 +调度，因此已经连同 ABI、状态、测试和 host oracle 修改一起撤回。以后 +若重试，应显式设计受控 run-ahead 窗口，而不是仅以 atomic 延迟更低为由 +恢复该候选。 + +#### 已撤回：shared loser 不写 winner-only context 字段 + +第七个候选只消减 shared loser 在 Claim 后对 `SubmitContext::won` 和 +`kernel_id` 的两次写回。winner 在跨 TU 前仍写入这两个字段；loser +收尾改为直接信任同一 caller 刚构造的 ticket，不再把 Claim 的 SSA +结果写入 context 后立刻读回校验。private 路径完全不变。 + +候选通过 shared CPU 全量门槛、CPU B256、private CPU 门槛、两类 CCEC +构建与 A5 B256 全部正确性断言；CPU loser 门槛还刻意给 context 留入 +陈旧 winner 值，证明 loser 收尾不再读取这两个字段。CCEC perf-clock +mixed `.text` 减少 `3,072 B`。A5 泳道位于： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + pa_scheduler_shared_swimlane_20260728_170451_1834681/ccec/ +``` + +带观测 loser 尾部仅有轻微变化，且其他区间的采集波动更大： + +| loser 区域 | R5k | 候选 | 变化 | +| --- | ---: | ---: | ---: | +| post-Claim mean | 427.163 ns | 424.074 ns | -0.723% | +| post-Claim median | 408 ns | 430 ns | +5.392% | +| Submit mean | 1,758.220 ns | 1,766.675 ns | +0.481% | + +决定去留的无观测 perf-clock 采用三组对称 +`baseline,candidate,candidate,baseline`： + +| 区组 | 基线均值 | 候选均值 | 候选减基线 | +| ---: | ---: | ---: | ---: | +| 1 | 2,434.303 us | 2,460.785 us | +26.482 us / +1.088% | +| 2 | 2,453.409 us | 2,464.873 us | +11.464 us / +0.467% | +| 3 | 2,449.121 us | 2,451.124 us | +2.003 us / +0.082% | + +三组方向全部回退。六样本总体均值从 `2,445.611 us` 增至 +`2,458.927 us`，回退 `13.316 us（0.544%）`；中位数回退 +`16.766 us（0.686%）`。因此 `.text` 更小不能作为保留依据，该候选 +已经完整撤回，winner/loser 继续共同写入并校验 `won/kernel_id`。 + +#### 已撤回：loser 不再逐次检查 split ticket 交接字 + +第八个候选把 `RecordSharedSplitReplayTask` 中每个 Submit 对 +`runtime.reserved` 的读取延后：该字只用于本核同步 caller→finish 的 +协议诊断，不参与调度；winner 的 `ArmSharedSplitTicket` 在覆盖前仍拒绝 +非零旧值，最终 split 封口仍要求它为零。若旧 winner 根本没有完成 +Finish，下一 task 还会因 `task_id != submits` 立即失败；若只剩 loser， +异常最晚在最终协议封口报出。 + +CPU 门槛覆盖了正常 B256 依赖、TensorMap 与 split 状态，并用陈旧 binding +证明后续 winner 不会覆盖旧值。CCEC perf-clock 构建和 12 次 A5 B256 +均通过全部语义断言。但三组对称 ABBA 没有形成稳定收益： + +| 区组 | 基线均值 | 候选均值 | 候选减基线 | +| ---: | ---: | ---: | ---: | +| 1 | 2,437.893 us | 2,437.335 us | -0.558 us / -0.023% | +| 2 | 2,447.406 us | 2,435.244 us | -12.162 us / -0.497% | +| 3 | 2,433.457 us | 2,448.029 us | +14.572 us / +0.599% | + +六样本总体均值 `2,439.585 -> 2,440.202 us`，候选回退 +`0.617 us（0.025%）`；中位数回退 `6.477 us（0.266%）`。方向随区组 +翻转且总体为中性，因此不能把它记录成收益。为保持异常路径尽早诊断， +候选代码和临时故障注入门槛均已撤回。 + +#### 已撤回：热路径消费已验证 batch plan 的轻量访问器 + +第九个候选针对 Submit 起点前的纯 scalar 工作：每批 +`BuildSharedPaBatchPlan` 已经校验 context、group/task 公式和容量，但 +96 个 replay actor 的每个 Submit 又通过 `SharedPaPlannedTaskAt` 重复 +检查同一 plan。候选抽出只恢复 kind/group/last 元数据的轻量访问器； +热路径仍检查 offset 上界、模板 Kind 和递增 task id，防御性接口继续供 +host oracle 与门槛测试使用。 + +只读调用图审计确认 plan 是每批栈上局部值，仅在 Build 全部检查成功并 +一次写全三个字段后，以 const 引用同步传入；Build 失败直接 fatal 并退出， +不存在未初始化或部分初始化 plan 被消费。shared CPU 全量门槛和 B256 +完整依赖/TensorMap 核对均 PASS。CCEC perf-clock 的 AIC/AIV 入口从 +`698,120/700,072 B` 降至 `695,768/699,072 B`。 + +先跑三组 `B-C-C-B` 时总体曾呈现 `-3.999 us（-0.164%）`，但换成三组 +反向 `C-B-B-C` 后变为 `+4.852 us（+0.199%）`。六组汇总如下： + +| 区组 | 基线均值 | 候选均值 | 候选减基线 | +| ---: | ---: | ---: | ---: | +| 1 | 2,447.160 us | 2,448.720 us | +1.560 us / +0.064% | +| 2 | 2,439.560 us | 2,432.793 us | -6.767 us / -0.277% | +| 3 | 2,447.746 us | 2,440.956 us | -6.789 us / -0.277% | +| 4 | 2,429.345 us | 2,448.917 us | +19.572 us / +0.806% | +| 5 | 2,442.860 us | 2,435.432 us | -7.428 us / -0.304% | +| 6 | 2,452.550 us | 2,454.960 us | +2.410 us / +0.098% | + +12 个基线与 12 个候选样本总体均值为 +`2,443.203 -> 2,443.629 us`，回退 `0.426 us（0.017%）`；中位数回退 +`1.609 us（0.066%）`。代码尺寸下降没有转化为顺序无关的端到端收益， +因此该候选已撤回，继续保留每次 Submit 的独立 plan 校验。 + +#### 已撤回：shared Claim 尝试数按 batch 聚合 + +第十个候选消减每个参与 Claim 的 replay actor 对 +`claim_attempts` 的逐次累加。shared 每批 task 形状为 +`Alloc + G×(QK,SF,PV,UP)`；每个 AIC 尝试 Alloc/QK/PV，每个 AIV +尝试 Alloc/SF/UP，因此两种角色都可由已验证 plan 精确得到 +`1 + 2G`。候选改为每批累加一次；winner、retry、每条 atomic 泳道与 +private 路径不变。 + +CPU 全量门槛通过，另以 mixed `G0/G1/G2/G4` 四批验证全局 claims 精确为 +`1,728`，完整依赖和 TensorMap 也全部 PASS。CCEC perf-clock 的 AIC/AIV +入口从 `698,120/700,072 B` 降至 `691,896/694,640 B`。但两组对称 +ABBA 都出现明显端到端回退： + +| 区组 | 基线均值 | 候选均值 | 候选减基线 | +| ---: | ---: | ---: | ---: | +| 1 | 2,437.978 us | 2,495.231 us | +57.253 us / +2.348% | +| 2 | 2,441.531 us | 2,509.778 us | +68.247 us / +2.795% | + +方向和量级已足够明确，未继续浪费第三组设备时间。该候选也会让 fatal +中途退出时的 claims 表示整批计划数、而非实际已发射数，降低错误定位 +精度。综合端到端回退和诊断语义减弱，代码已完整撤回。 + +#### 已撤回：稳定 output symbol 直接写入 count + +第十一个候选利用 `SharedTaskOutputs` 的真实布局：对象只保存 +`producer_task_id` 与 `output_count`，`OutputRef(index)` 在读取时才恢复 +句柄，并没有逐槽数组。因而 `PrepareSharedTaskOutputs` 中顺序调用 +0～3 次 `AddOutputRef` 在语义上可等价为一次经过上限校验的 count 写入。 + +shared CPU 全量门槛全部 PASS，其中 shared-output symbol、B256 依赖签名 +与 TensorMap 精确核对均覆盖该对象。CCEC perf-clock AIC/AIV 入口分别从 +`698,120/700,072 B` 降至 `696,456/698,560 B`。但两组对称 ABBA 均回退： + +| 区组 | 基线均值 | 候选均值 | 候选减基线 | +| ---: | ---: | ---: | ---: | +| 1 | 2,441.379 us | 2,459.832 us | +18.453 us / +0.756% | +| 2 | 2,438.103 us | 2,456.129 us | +18.027 us / +0.739% | + +两组方向与量级一致，无需继续消耗第三组设备时间。伴随样本中候选的 +fanin load 也系统性偏高，说明 replay actor 更快越过 output 准备后， +可能改变后续依赖轮询的相对推进节奏；现有证据不能把全部回退唯一归因于 +fanin，但足以否决候选。代码已恢复为顺序 `AddOutputRef`。 + +### 2026-07-28：R5l 消除 shared EfDrain 对 ready fanin 前缀的重复读取 + +#### 先解释泳道中的 1～2 us + +R5k B256 原始泳道中共有 121,600 次 loser Submit。按 loser 区域拆解后， +典型耗时并不是一个独立的“loser 业务函数”： + +| loser 区域 | mean | median | p95 | +| --- | ---: | ---: | ---: | +| 完整 Submit | 1,758 ns | 1,289 ns | 3,189 ns | +| Submit 开头 EfDrain | 718 ns | 210 ns | 1,812 ns | +| Claim | 613 ns | 614 ns | 1,087 ns | +| Claim 后收尾 | 427 ns | 408 ns | 679 ns | + +其中只有 1,008/121,600 次 loser EfDrain 真正取走一个已就绪 kernel， +它们解释了约 55～65 us 的长尾；常见的 1～2 us 并没有对应额外 kernel。 +逐条检查原始记录还确认，泳道写记录本身跨越了业务边界： + +- EfDrain 结束记录的写入进入后续 Claim 区间； +- Claim 的 atomic/阶段记录进入 Claim 区间； +- Claim 结束记录进入 loser 收尾； +- Submit 结束记录进入下一次 SubmitTransition。 + +因此带泳道的单次 loser 数值包含必要的观察开销，不能解释为 +“一次 atomic 竞争加一次空 EfDrain 的真实净耗时”。泳道继续用于判断 +工作落点；候选去留仍由编译期移除泳道和 atomic 观察的 perf-clock 决定。 + +#### 保留：已 ready 的 fanin 在单轮内只读取一次 + +shared 的 `LocalSlot::fanin` 是执行 winner 的本核私有待执行依赖列表； +对应 completion flag 在单轮 kernel 内只会从 0 变为 1。原 `SlotReady` +每次从索引 0 重新扫描，若依赖形态为 `[ready, ready, not-ready]`，每次 +EfDrain 都会再次对前两个 flag 执行 `atomicAdd(0)`。这部分既不是新的 +依赖判断，也不是 kernel 等待的必要工作。 + +R5l 在 shared 模式中采用如下收敛： + +1. 遇到第一个 not-ready fanin 时，移除本次已经确认 ready 的前缀， + 只保留 blocker 及其后的未检查后缀； +2. 所有 fanin ready 时将 `fanin_count` 归零； +3. private 模式保持原扫描和数组内容不变； +4. 不增加共享状态、ABI 字段、原子指令或泳道记录。 + +前向重叠复制的源索引始终大于目标索引，且 PA fanin 是小有界数组，因此 +不会覆盖尚未复制的元素。优化也不会提前执行 task：每次调用仍会先遇到并 +读取当前 blocker，只有它变为 ready 后才继续检查后缀。 + +host oracle 对 shared 收紧为 +`fanin_ready_loads == fanin_edges`，把“每条真实依赖恰好命中一次 ready” +作为正确性与性能共同门槛。独立 CPU 门槛覆盖： + +- 第 0 项未就绪时数组和计数完全不变； +- 四条依赖分三次解除时连续压缩； +- 最后一次全部 ready 后计数归零； +- 累计 `ready=4`、`not_ready=3` 精确匹配调用过程。 + +fatal 清理的诊断语义也同步说明为“保留尚未就绪的 fanin 后缀”；已经 +确认 ready 并从本核 slot 删除的前缀不再伪装成仍待处理的依赖。 + +#### 正确性与 A5 B256 结果 + +shared CPU 全量门槛、CPU B256、private CPU 全量门槛与 private b1 均 +PASS。shared CPU B256 的依赖签名仍为 `b7d985d6edb07078`， +`fanin_ready=1280 == fanin_edges=1280`；private 继续使用原来的重复扫描 +分类规则。CCEC perf-clock、swimlane 两类构建也均 PASS。 + +候选 A5 B256 泳道位于： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + pa_scheduler_shared_swimlane_20260728_174438_1874062/ccec/ +``` + +与 R5k 同为带完整阶段和 atomic 观察的 B256 样本，loser 的目标区域变化 +如下： + +| loser 区域 | R5k | R5l | 变化 | +| --- | ---: | ---: | ---: | +| Submit mean | 1,758.220 ns | 1,722.102 ns | -36.118 ns / -2.054% | +| Submit median | 1,289 ns | 1,257 ns | -32 ns / -2.483% | +| EfDrain mean | 718.365 ns | 687.718 ns | -30.647 ns / -4.266% | +| EfDrain median | 210 ns | 198 ns | -12 ns / -5.714% | +| ClaimMax atomic mean | 376.778 ns | 374.440 ns | -0.620% | + +ClaimMax 本身基本不变，收益明确落在反复调用 `SlotReady` 的 EfDrain， +而不是通过改变 Claim atomic 伪造。带泳道的 B256 还得到 +`fanin_ready=1280`、`fanin_not_ready=18,661`、总 fanin loads=19,941， +依赖、TensorMap、输出结果、73,728 次 Claim 和 1,280 个 winner 全部 +精确通过。 + +最终净性能采用六组对称 ABBA；前三组为 +`baseline,candidate,candidate,baseline`，后三组反转运行次序。六组均 +同向改善： + +| 组 | R5k 基线均值 | R5l 候选均值 | 候选减基线 | +| ---: | ---: | ---: | ---: | +| 1 | 2,435.526 us | 2,407.691 us | -27.836 us / -1.143% | +| 2 | 2,438.107 us | 2,424.129 us | -13.978 us / -0.573% | +| 3 | 2,427.100 us | 2,415.404 us | -11.696 us / -0.482% | +| 4 | 2,440.746 us | 2,405.283 us | -35.463 us / -1.453% | +| 5 | 2,448.964 us | 2,432.211 us | -16.753 us / -0.684% | +| 6 | 2,430.577 us | 2,414.200 us | -16.377 us / -0.674% | + +12 个基线与 12 个候选样本总体均值为 +`2,436.837 -> 2,416.486 us`，改善 `20.350 us（0.835%）`；中位数为 +`2,430.577 -> 2,415.203 us`，改善 `15.374 us（0.633%）`。 +`fanin_ready` 均值从 7,283.9 精确收敛到每次 1,280，下降 82.427%； +总 fanin loads 均值从 51,284.2 降到 44,278.4，下降 13.661%。 + +在加入精确 host 断言后，以最终源码重新构建并补跑的一次 A5 B256 +perf-clock 为 `2,431.319 us`，`fanin_ready=1280`、全部语义断言 PASS。 +这次优化不是要求 loser 不做 EfDrain，而是删除 EfDrain 中已经获得确定 +答案的重复原子读取；仍未 ready 的 blocker 轮询和真实 kernel drain +继续完整保留。 + +### 2026-07-29：按 1% 端到端门槛复核 R5k～R5m + +用户把总时间护栏从 2% 收紧为 1% 后，重新从远端 +`e42aba58`、R5k `9fe630bf`、R5l `9a813bc2` 和 R5m +`755397b0` 四个精确提交建立干净 worktree，并分别重建 CCEC +`perf-clock`。测试固定使用 CANN 9.1、device 0、shared TensorMap、 +G1、B256、context 8192、real-compute `6,28,4,1` 和 final barrier +`two-16`；泳道、atomic trace、PMU、phase 和 kernel timing 均在编译期 +关闭。 + +每版先运行两个独立预热进程；正式样本采用 12 个位置均衡区组，每区组 +四个版本各运行一次，四个版本分别在每个运行位置出现三次，共 48 个独立 +进程、每版 12 个正式样本。48/48 均通过 1,280 tasks、96 workers、 +73,728 claims、依赖签名 `b7d985d6edb07078`、TensorMap、real-compute +输出以及 execution/semantic/postprocess 门槛。 + +| 版本 | 样本数 | mean | median | min～max | +| --- | ---: | ---: | ---: | ---: | +| 远端基线 A | 12 | 2,457.917 us | 2,460.778 us | 2,430.524～2,477.321 us | +| R5k B | 12 | 2,436.549 us | 2,444.305 us | 2,400.860～2,455.052 us | +| R5l C | 12 | 2,414.324 us | 2,416.559 us | 2,391.268～2,430.569 us | +| R5m D | 12 | 2,461.247 us | 2,461.110 us | 2,429.795～2,493.609 us | + +相邻提交必须分别判定，不能让前两个提交的收益掩盖第三个提交的回退： + +| 相邻变化 | mean 变化 | median 变化 | 改善区组 | 结论 | +| --- | ---: | ---: | ---: | --- | +| A→B：延后 winner 上下文初始化 | -21.368 us / -0.869% | -16.473 us / -0.669% | 8/12 | 保留 | +| B→C：消除 ready fanin 重复读取 | -22.225 us / -0.912% | -27.747 us / -1.135% | 9/12 | 保留 | +| C→D：replay actor 直接恢复 symbol | +46.923 us / +1.944% | +44.552 us / +1.844% | 0/12 | 撤回 | + +R5m 的 12 个逐区组差值全部为正，均值回退的近似 95% 区间为 +`[+31.172,+62.675] us`，不是运行位置或单个异常样本造成。它虽然减少 +loser 闭合工作，但超过新的 1% 端到端门槛，因此提交 `755397b0` 连同 +对应运行时代码不再进入待推送分支。R5k 与 R5l 均为端到端改善，继续 +保留。 + +### 2026-07-29：按 PA task 类型解释 ordered Register 的性能差异 + +#### 先区分三类发布对象 + +当前 shared PA 不是把每个 output 都重复写入 ordinary TensorMap，而是按 +对象语义使用三条发布路径： + +1. Alloc/QK/SF/PV 创建的 fresh output 使用 + `(producer_task_id, output_slot)` 形式的 `SharedOutputRef`。winner 在 + Materialize 尾部把 `TensorDesc` 写入 + `shared_outputs[task].tensors[slot]`，整批执行 + `shared_output_descriptor_flush`，再经过 StoreBarrier 用 + `published[slot]` Atomic 发布。 +2. 普通 `GmTensor/LocalTensor` writer 才在 Register 中追加 ordinary + TensorMap region slot，并执行 append invalidate/flush。本次 PA B256 + 的 `ordinary_count` 始终为 0，因此不存在 ordinary TensorMap append + DCCI。 +3. UP 的三个 accumulator 是 INOUT `SharedOutputRef`，不会创建 fresh + descriptor。UP 在 Register 中写不可变 writer-history，整批执行一次 + `shared_output_ref_writer_history_flush`，再用三个 CAS 把对应 + `last_writer` 推进到当前 task。 + +因此 QK/SF/PV 的 Register 没有 TensorMap append DCCI 是协议事实，不是 +泳道漏采。它们需要的 descriptor DCCI 已经位于 Materialize。UP 需要的 +DCCI 也不是 ordinary TensorMap flush，而是 writer-history 的发布边界： +若没有这次 flush，其他核可能先观察到新的 `last_writer`，却读到尚未写回 +的旧 history。 + +consumer Build 在读取 `SharedOutputRef` descriptor 前还会执行 +`shared_winner_build_descriptor_invalidate`。完整的可见性链为： + +```text +producer 普通写 descriptor + -> descriptor DCCI clean-out + -> StoreBarrier + -> published Atomic + -> consumer 检查 published + -> descriptor DCCI invalidate + -> consumer 复制 descriptor 到执行 slot +``` + +UP 的版本链则为： + +```text +读取三个 symbol 的 last_writer + -> 普通写本 task 的三条 writer-history + -> 一次 history DCCI clean-out + -> StoreBarrier + -> 三次 last_writer CAS +``` + +#### B256 全量事件闭合 + +取证样本为： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + pa_scheduler_shared_swimlane_20260729_154301_2671167/ccec/ +``` + +这是一份带普通阶段、Atomic 和 DCCI 的观察构建，只用于区域与调用次数 +归因，不作为 trace-free 净性能结论。1,280 个 task 的 Register 记录完整 +闭合： + +| 事件 | 数量 | 说明 | +| --- | ---: | --- | +| Register parent | 1,280 | 每个 task 一个 | +| wait predecessor 子区间 | 1,280 | task 0 保留真实近零边界 | +| predecessor PollBatch | 1,279 | 全局 task 0 没有前驱 | +| writer metadata 子区间 | 1,280 | 名称中的计数只表示 ordinary entry | +| insert completion 子区间 | 1,280 | 每个 task 一个 | +| insert completion CAS | 1,280 | 每个 task 一个 | +| descriptor flush | 1,024 | Alloc/QK/SF/PV 各 256 次 | +| ordinary TensorMap append flush | 0 | 本用例没有 ordinary writer entry | +| UP writer-history flush | 256 | 每个 UP 整批一次 | +| UP output-published load | 768 | 三个 INOUT × 256 | +| UP previous-writer load | 768 | 三个 INOUT × 256 | +| UP last-writer CAS | 768 | 三个 INOUT × 256 | + +`register.publish_writer_metadata[ordinary_tensormap_entries=0]` 不能解释为 +metadata 没有工作：这个现有标签只复用 Register raw 中的 +`ordinary_count`，没有额外记录 `symbol_count`。UP 虽然也显示 ordinary +为 0，内部仍有三组 symbol writer 操作和一次 history DCCI；Atomic/DCCI +子区间才是这部分工作的直接证据。 + +#### 各 task 类型的 Register 分解 + +以下均为单 task 事件中位数，单位为微秒。`task_id % 5` 依次对应 +Alloc/QK/SF/PV/UP。 + +| task 类型 | Register | 等前驱插入 | writer metadata | completion | metadata 的主要实际工作 | +| --- | ---: | ---: | ---: | ---: | --- | +| Alloc | 71.165 | 69.991 | 0.788 | 0.388 | 空 ordinary/symbol 集合校验 | +| QK | 29.786 | 28.834 | 0.582 | 0.439 | 空 ordinary/symbol 集合校验 | +| SF | 70.162 | 68.918 | 0.777 | 0.375 | 空 ordinary/symbol 集合校验 | +| PV | 27.992 | 26.936 | 0.580 | 0.372 | 空 ordinary/symbol 集合校验 | +| UP | 80.848 | 76.170 | 4.316 | 0.388 | 三个 INOUT writer-history 与 last-writer CAS | + +所有类型的 Register 长度都主要由 +`wait_predecessor_tensormap_insert` 决定。不同类型的等待中位数反映本次 +多核回放中 owner 到达全局有序插入链的相对时机，不能当成对应 metadata +函数的纯 scalar 成本。只有 UP 的 metadata 稳定更重，可以直接归因于三个 +INOUT symbol 的版本链发布。 + +三个具体 task 可以说明泳道视觉差异： + +- task 36（QK）：Register 37.789 us，其中等前驱 36.427 us、metadata + 0.917 us、completion 0.445 us。长区间几乎全部是轮询等待。 +- task 4（UP）：Register 19.071 us，其中等前驱 13.406 us、metadata + 5.002 us、completion 0.663 us。metadata 内可见三组 load/load/CAS + 和一次 DCCI。 +- task 0（Alloc）：没有前驱，wait 只有 0.017 us，completion 只有 + 0.338 us；在较粗缩放下视觉上近似只剩 writer metadata，但原始记录并 + 未缺少另外两个子区间。 + +#### 后续优化边界 + +1. 不应为了让 QK/SF/PV 的 Register 看起来“完整”而增加无意义的 + TensorMap DCCI；这会重复发布 fresh descriptor 并污染性能。 +2. Register 长尾的第一优化对象仍是 predecessor wait 的形成原因和插入链 + 推进节奏，而不是不到 1 us 的空 metadata。 +3. UP metadata 可以单独研究 writer-history 的布局、批量发布和三个 + last-writer 控制字的访问，但必须保留“history 先可见、CAS 后发布”的 + 顺序合同。 +4. 后续若 PA 引入真实 ordinary writer，泳道必须同时出现 region append + invalidate、payload flush、seq 发布和 tail 发布；否则才应按漏采或协议 + 缺失调查。 + +### 2026-07-29：shared full-swimlane 通用记录压缩为 16B + +#### 只压缩物理存储，不改变逻辑泳道 + +本阶段针对的是观察构建写入 GM 的通用泳道记录，不改变调度协议和最终 +JSON 事件语义。shared full-swimlane 使用两种物理记录： + +- Submit/Claim 继续使用每 task 一条 32B 四端点记录，保留两者严格配对的 + 起止边界； +- 其余通用事件由 32B `TraceRecord` 压缩为 16B + `CompactTraceRecord16`，host 回读后再恢复为原来的逻辑 + `TraceRecord`。 + +16B 通用记录保存： + +- `start_cycle`、`end_cycle` 的低 32 位； +- 原有 32 位 flags； +- 由 task、function、phase 和 auxiliary 组成的 32 位打包字段。 + +host 使用每核 `startup_barrier_begin` 和 `finish_cycle` 展开低 32 位时钟。 +构建期和 host 后处理同时检查 task、function、phase、auxiliary 的可编码 +范围，以及一次调度窗口小于 `2^32` tick；任何字段越界、时钟无法展开或 +记录落在合法窗口之外都会使后处理失败，不允许静默截断。 + +压缩只用于 shared swimlane： + +| 构建 | 通用记录 | Submit/Claim | 每核通用容量 | 每核物理 stride | +| --- | ---: | ---: | ---: | ---: | +| shared swimlane | 16B | 32B | 28,416 | 593,920B | +| shared perf-clock / submit-pmu | 32B | 32B | 28,416 | 1,048,576B | +| private | 32B | 无专用区 | 65,536 | 2,097,152B | + +因此 private、perf-clock 和 submit-pmu 的编译路径不受本次压缩影响； +converter/analyzer 继续消费完全展开后的同一逻辑 schema。 + +#### 构建产物必须声明真实物理布局 + +CCEC artifact manifest 升级为 v4，并在 mode、variant、phase 之外固定 +记录： + +- `generic_record_bytes`； +- `submit_claim_record_bytes`； +- `records_per_core`； +- `worker_stride_bytes`。 + +`build.sh` 从实际传给三份 device 镜像的 compact 宏推导上述字段； +`run.sh` 则根据 mode/variant 独立计算期望值并逐字段核对。这样即使 +producer 和 consumer 某一侧修改错误，也不能因共用同一份常量而把 +16B/32B 混件误判为合法。旧 schema、字段重排、额外身份行、任一布局字段 +篡改和 artifact SHA 不一致均由独立门槛测试拒绝。 + +#### A5 B256 对称 A/B + +基线和候选从同一源码构建，唯一差异是 shared full-swimlane 通用记录为 +32B 或 16B；Submit/Claim 专用记录、逻辑事件、Atomic/DCCI 观察和所有 +正确性门槛保持一致。两版均先预热，随后运行五组 +`32B,16B,16B,32B` 的对称 ABBA，共各 10 个独立 B256 进程。每次均满足： + +- execution、semantic、postprocess 全部 PASS； +- 依赖签名为 `b7d985d6edb07078`； +- dropped records 为 0； +- Atomic/DCCI 与 Submit/Claim 闭合式全部通过。 + +| 组 | 32B 均值 | 16B 均值 | 16B 相对变化 | +| ---: | ---: | ---: | ---: | +| 1 | 2,773.395 us | 2,734.988 us | -38.407 us / -1.385% | +| 2 | 2,799.339 us | 2,740.293 us | -59.046 us / -2.109% | +| 3 | 2,798.697 us | 2,739.219 us | -59.478 us / -2.125% | +| 4 | 2,789.271 us | 2,744.362 us | -44.909 us / -1.610% | +| 5 | 2,771.184 us | 2,757.860 us | -13.324 us / -0.481% | + +汇总结果： + +| 布局 | 样本数 | mean | median | min～max | +| --- | ---: | ---: | ---: | ---: | +| 32B | 10 | 2,786.377 us | 2,789.133 us | 2,759.049～2,808.885 us | +| 16B | 10 | 2,743.345 us | 2,743.488 us | 2,726.481～2,761.876 us | + +16B 的 mean 改善 `43.033 us（1.544%）`，median 改善 +`45.645 us（1.637%）`，五组全部同向。96 核 trace 物理分配从 +`100,670,272B` 降为 `57,023,296B`，减少 +`43,646,976B（43.35%）`。这同时减少每条通用事件的 GM 写入量和完整 +trace 缓冲规模，收益达到当前 1% 端到端门槛，因此保留 16B 方案。 diff --git a/tests/atomic_probe/pa_scheduler/shared_tensormap_swimlane_deps_migration_review.md b/tests/atomic_probe/pa_scheduler/shared_tensormap_swimlane_deps_migration_review.md new file mode 100644 index 0000000000..e8221cc9c4 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/shared_tensormap_swimlane_deps_migration_review.md @@ -0,0 +1,1749 @@ +# FDWIC Shared TensorMap 与 Swimlane Deps 分支迁移审查 + +本文记录从 +`qingchuanyu/fdwic-swimlane-deps` +向 +`chenpeng/fdwic-shared-tensormap` +迁移实现的代码审查结论。 + +本文先记录两个分支分别实现了什么、哪些内容需要迁移、哪些内容需要排除； +随着逐项对齐，继续记录已经确认的取舍、shared-aware 的适配要求、按影响面 +排序的迁移阶段和每阶段验证门禁。 + +本文仍是迁移分析与执行前计划，不表示已经修改生产 runtime，也不把尚未 +确认的建议自动视为已接受。 + +## 1. 审查快照 + +| 项目 | 固定值 | +| --- | --- | +| 审查日期 | 2026-07-25 | +| 目标分支 | `chenpeng/fdwic-shared-tensormap` | +| 目标提交 | `351ef62e` | +| 来源分支 | `qingchuanyu/fdwic-swimlane-deps` | +| 来源提交 | `1726a774` | +| 两分支 Git merge-base | `599703f5b153a3a0fd2a3516dff4efd49be3f00a` | +| 目标分支 private 基线 | `f5da1a2e` | +| 来源分支 private 基线 | `bd620630` | + +`f5da1a2e` 和 `bd620630` 虽然是两个不同提交，但对应的整棵源码树完全一致。 +因此本文使用下面两个区间区分双方真正独有的实现： + +- `f5da1a2e..351ef62e`：Shared TensorMap 分支新增实现； +- `bd620630..1726a774`：Swimlane Deps 分支新增实现。 + +不能直接把两个分支 tip 做单向 diff 后按“增加/删除”理解。两个分支在相同 +private runtime 基线上分别演进，直接 diff 会把 Shared TensorMap 的新增实现 +显示成来源分支的删除。 + +直接 tip diff 共涉及约 629 个文件、`+140124/-8613` 行。其中 +`tests/atomic_probe` 在来源分支中有 509 个文件，约 10.8 万新增行，是整体 +diff 的主要组成部分。 + +本文结论基于固定 Git ref 的源码、提交历史和分支内实验记录；没有把本地 +未提交文件作为分支事实，也没有重新执行 A5 性能测试。 + +## 2. 两分支的共同基础 + +在上述两个 private 基线之前，两边已经共同具备： + +- A5 FDWIC AICPU orchestration； +- orchestration 与 AICore kernel 的联合执行； +- per-core runtime state 和 replay； +- 直接 CCEC submit； +- payload 和 output 生命周期处理； +- ring drain、fanin、task completion； +- private TensorMap、private heap 和 output layout； +- mixed AIC/AIV task 和 BlockWon 协议； +- 基础 tracing； +- private slot cache flush 优化。 + +这些内容不是本次迁移对象，不应重复搬运。 + +## 3. Shared TensorMap 分支实现内容 + +### 3.1 Shared TensorMap 数据结构 + +目标分支增加了三组 shared 状态： + +- `SharedOutputCell`：按 `(producer_task_id, output_slot)` 保存 fresh output + descriptor、发布标记和 `last_writer`； +- `SharedRegionMap`：记录普通 Tensor 地址区间及其 producer； +- shared heap：由 task winner 统一分配输出。 + +fresh output 通过 `FdwicOutputRef` 表示未物化的输出引用。普通 Tensor 和 +INOUT 重叠关系仍通过 region map 解析。 + +主要实现文件： + +- `runtime/dist_engine/common/state.h` +- `runtime/dist_engine/aicore/submit_core.h` +- `runtime/pto_types.h` + +### 3.2 Winner-first submit 协议 + +目标分支把 shared submit 拆为： + +```text +presubmit / claim + -> winner: 构参、分配、注册依赖、发布输出、提交执行 + -> loser: 只返回 symbolic output reference +``` + +PA 的重参数构造被放入 `tok.won` 分支。loser 不再重复构造动态 +`TensorCreateInfo`、完整 `L0TaskArgs`、输出 descriptor 或 shared map +记录。 + +### 3.3 Shared 引用解析和依赖 + +目标分支实现了： + +- kernel 执行前 resolve `FdwicOutputRef`； +- producer 尚未发布时等待 `published`； +- descriptor invalidate 后复制到本地执行参数； +- 用 `last_writer` 维护 fresh output 的 INPUT/INOUT writer 链； +- 用 `SharedRegionMap` 维护普通 Tensor 的重叠依赖； +- region intent 在 replay 前发布 writer 信息； +- mixed follower 由 winner 解析 shared input 后再 launch。 + +### 3.4 缓存和并发顺序 + +目标分支补充了： + +- shared descriptor 发布前的 DCCI flush； +- consumer 读取前的 invalidate； +- acquire/release publication； +- shared 状态 cache-line 隔离； +- winner readiness 和 follower launch gate； +- final drain 中的 joint-launch completion 检查。 + +### 3.5 Shared 热路径优化 + +目标分支还实现了： + +- wrong-role worker skip； +- 按需执行 EfDrain； +- shared heap 分片； +- alloc contention 过滤； +- shared resolve 热路径 guard； +- 保留 trace flags 和更精确的 trace attribution。 + +### 3.6 Shared 测试和 trace + +目标分支扩展了： + +- `simple_orch_smoke`； +- `shared_symbol_smoke`； +- `submit_dependency_smoke`； +- shared region/ref chain、长 INOUT chain、delayed registration； +- mixed AIC/AIV 和 dual-AIV 场景； +- shared resolve trace。 + +当前 shared swimlane 已使用 phase ID 14～17： + +- `Resolve = 14` +- `ResolveWait = 15` +- `ResolveInvalidate = 16` +- `ResolveCopy = 17` + +## 4. Swimlane Deps 分支实现内容 + +### 4.1 Atomic probe + +`tests/atomic_probe` 是一套独立实验资产，包含： + +- CPU、CCEC 和 AscendC atomic CAS/exchange 等探针； +- `ld_dev`、`st_dev`、DCCI、bypass dcache； +- cache-line blast/clobber； +- I-cache 和 PMU 探针； +- PA standalone scheduler/model； +- atomic swimlane 分析工具； +- lazy-lambda A/B/C 实验及原始产物。 + +它不等价于生产 runtime DFX；多数 probe 可以独立构建和运行。 + +### 4.2 Private PA 参数准备优化 + +提交 `dbbf621a` 包含两类变化： + +1. 复用一个 scope-local `L0TaskArgs`，覆盖 alloc 以及 QK、SF、PV、UP； +2. 把 `TensorRef` 默认构造改成不初始化未使用 slot。 + +分支记录的 A5 PA Case1 scalar-body 结果： + +| 版本 | 时间 | 相对原始版本 | +| --- | ---: | ---: | +| 原始四个参数容器 | 6.170876 ms | 基线 | +| 一个容器复用四个 kernel | 5.720638 ms | -7.30% | +| 再省略未使用 slot 初始化 | 5.645826 ms | 累计 -8.51% | +| alloc 也复用同一容器 | 5.577570 ms | 累计 -9.61% | + +这里的“省略未使用 slot 初始化”不是 lazy-lambda，但它与目标分支新增的 +`FdwicOutputRef` union 成员存在构造语义冲突，不能原样应用。 + +### 4.3 BlockWon 首次 joint-submit gate + +提交 `e3b748b4` 使用全局 latch，使没有出现过 joint task 的 private PA +不查询 BlockWon publication。 + +来源分支记录它消除了 146944 次无意义 atomic poll，PA 从 5.642245 ms +下降到 5.171330 ms。 + +目标分支已经有语义等价的 `g_fdwic_block_won_enabled`，并在 +`drain_block_won_if_enabled()` 中使用。因此来源分支的主体优化已经存在。 + +仍有一个残余差异：目标分支的 `has_pending_won()` 没有检查已有 gate， +而 `dist_submit_has_drain_work()` 会调用它。 + +同一提交中的 register mask 单独没有稳定收益。目标分支的 +`DistOutputLayout::output_indices` 也已经替代了 output rescan。 + +### 4.4 Private heap 首圈快速路径 + +提交 `04ec9b95` 在 private heap 的 fatal-checked 分配循环内增加 H1： + +- 当 `heap_next <= ring` 时直接返回； +- 首圈不可能发生 wrap，因此不读取 frontier/vend atomic； +- PA 中减少 1024 次 frontier load； +- 10 次 A5 配对运行中 8 次获胜，中位数约 -0.324%； +- 已验证 wrap 和 backpressure。 + +目标代码尚未包含该优化，并且对应函数已经位于 +`#if !PTO_FDWIC_SHARED_MAP` 下。 + +### 4.5 G16 两级 final barrier + +提交 `c9ea57cb` 把平坦 `replay_done` 完成线改为： + +```text +worker + -> 16 个 leaf arrival + -> root arrival + -> root release + -> 每个 leaf release + -> worker exit +``` + +worker 在等待期间仍继续 drain。新状态追加在旧 ABI 后，原 +`replay_done` 字段保留。 + +来源分支记录： + +- G16 相比 G8，FinalDrain 减少 12.264 us，即 -3.425%； +- full completion 改善 -0.287%； +- Submit 基本不变； +- 已验证 A5Sim block dimension 1/36 和 A5 PA。 + +目标分支当前仍使用平坦 final barrier，但增加了 +`joint_launches_drained_for_lane()`，迁移时必须保留 shared 退出条件。 + +### 4.6 PrepareMap 跳写候选 + +提交 `6b85d4fc` 在 `task_heads[slot]` 已经等于 `-1` 时跳过重复写入。 + +它有理论写次数、CPU differential test 和 A5Sim 验证，但提交记录明确没有 +真实 A5 A/B 数据，代码量和控制分支反而增加。 + +### 4.7 Compete-first 和 lazy-lambda + +提交 `2899cc35` 实现 compete-first eager submit： + +- 先执行 EfDrain 和 claim； +- 再执行参数 callback； +- 但所有 worker 仍构造参数。 + +分支记录的实际改善约 -0.263%，区间重叠，主要价值是建立测量阶段边界。 +目标分支的 winner-first shared API 已经让 loser 不构造参数，语义和收益路径 +都更直接。 + +提交 `ba4334d1` 对比 lazy-lambda C 和 eager B： + +- 中位数差异约 `+0.040%`； +- 22 次运行各自获胜 11 次； +- 没有可确认收益。 + +### 4.8 Atomic swimlane + +提交 `cbaf7c60`、`dbb95bb5` 及相关提交实现： + +- schema v3/v4； +- 每条 32-byte record； +- per-core trace state； +- 28 个 private atomic site、5 类 atomic op； +- 直接记录和 PollBatch 聚合； +- logical/physical atomic closure； +- `OrchestrationReplay`、`FinalDrain` 等 exclusive parent； +- `WinnerBuild`、`AllocComplete`、`LoserReplay`； +- host converter、analyzer 和 strict closure。 + +来源分支记录 PA 中有 115200 次 logical atomic call、110006 次 physical +atomic、340 个 PollBatch，且 dropped record 为 0。 + +### 4.9 PMU 和 perf-clock DFX + +来源分支还增加： + +- 每物理核 PMU owner； +- PMU save/config/restore； +- perf-clock； +- kernel aggregate； +- 多组 phase-specific profile； +- host report 和 provenance； +- ELF/profile gate。 + +这部分连同 atomic swimlane，约增加 1.3 万行 runtime、platform、host 和 +工具代码。 + +### 4.10 兼容性修改 + +来源分支包含： + +- 老 A5 driver 无 CPU topology 时的平坦 OCCUPY fallback； +- 用 `ACL_DEV_ATTR_AICPU_CORE_NUM` 校验 fallback 核数； +- 跨架构 host `strip` 兼容； +- onboard FDWIC scene 跳过 PTO tile `.text` extraction； +- BGEMM `pto::Stride` 限定和 orchestration weak symbol； +- minibench DCCI store 和 include/symbol 冲突修复。 + +其中一次提交曾加入 runtime-wide post-kernel output flush，随后已由 +`b422c48f` 完整回退。最终结论是：标量 producer kernel 应自行完成发布， +不应让通用 runtime 无条件 flush 全部输出。 + +## 5. 逐项迁移判断 + +下面的编号用于后续逐项对齐。每项以其显式“状态”或“建议”为准。 + +### 5.1 建议接收 + +#### M01：整体接收 `tests/atomic_probe` + +状态：**已确认整体接收（2026-07-25）**。 + +原因： + +- 全部为来源分支新增内容； +- probe、原始数据和分析说明可以作为独立实验资产； +- 不要求把生产 runtime atomic/PMU DFX 一并迁入。 + +`lazy_lamda_sample` 也保留，但只作为“没有收益”的实验记录，不能据此启用 +lazy-lambda 生产实现。 + +#### M02：PA 单 `L0TaskArgs` 复用 + +状态：**已确认不接收（2026-07-25）**。 + +原审查建议迁移的范围是： + +- 一个 scope-local 容器复用 QK、SF、PV、UP； +- 在生命周期安全的前提下让 alloc 也复用该容器； +- 保留 shared 模式下只有 winner 填充重参数的行为。 + +现已确认不迁移这项 PA 参数容器复用。直接把 `TensorRef()` 改成 +`= default` 本来就不属于 M02，仍单列为 C02，等待独立确认。 + +#### M03：Private heap H1 首圈快速路径 + +建议：**接收**。 + +原因： + +- 修改局部； +- 只影响 private map； +- 有真实 A5 配对数据； +- wrap/backpressure 已验证； +- 不改变 shared heap。 + +#### M04：G16 两级 final barrier + +建议：**适配后接收**。 + +当前目标分支在每个 worker 完成 orchestration replay 后执行： + +```text +每个 worker 对同一个 replay_done cacheline 执行 FetchAdd + -> 每个 worker 反复读取同一个 replay_done cacheline + -> replay_done == num_workers 后，才允许退出 final drain +``` + +36 个 worker 会同时写、读同一条全局 cacheline。G16 不是修改 task replay、 +TensorMap 或 kernel barrier，而是只替换“所有 replay worker 已经进入最终 +排空阶段”的通知方式： + +```text +worker + -> 按 block_id % 16 到达本组 leaf_arrival + -> 每组 AIC leader 等本组全部 worker 到达 + -> 16 个组最多各向 root_arrival 发布一次 + -> group 0 的 AIC leader 等所有 active group 到达 + -> 发布一次 root_release + -> 每组 leader 发布本组 leaf_release + -> 组内 worker 观察 leaf_release 后可退出 +``` + +等待 barrier 的过程中，worker 仍执行 ring drain 和 BlockWon drain。因此它 +只减少最终汇合 cacheline 的竞争，不提前停止排空，也不改变任务完成条件。 + +迁入目标分支需要修改四处： + +1. 在 `DistGlobal` 尾部追加 16 组 leaf arrival/release 和一组 root + arrival/release；保留旧 `replay_done` 字段及既有字段 offset； +2. AICPU register 时清零这些计数，并根据实际 `layout[]` 计算每个 leaf 的 + worker 数和 active group 数； +3. 把 `dist_submit_drain_to_completion()` 中对单一 `replay_done` 的 + FetchAdd/poll 换成 leaf/root/release 协议； +4. 更新 debug dump、布局断言和 final-barrier 定向测试。 + +来源补丁不能原样应用，目标分支必须保留以下差异： + +- 调用 `drain_block_won_if_enabled()`，不退回来源分支的无条件 + `drain_block_won()`； +- shared 模式退出条件仍必须包含 + `joint_launches_drained_for_lane(self)`； +- 使用目标分支当前普通 atomic helper，不引入 C01 的 atomic trace wrapper； +- `FinalBarrierState` 追加在当前 `PreparedDeps` 和 `DistCore` 之后，避免改变 + shared ABI 中已有字段 offset； +- 同时验证 private/shared、block dimension 1/36，以及目标分支支持的最大 + block dimension。 + +修改后的退出条件等价于： + +```text +本组已收到全局 release +&& 本核 ring 为空 +&& 本 lane 没有待接收 BlockWon +&&（shared）本 lane 的 joint launch 已全部 drain +``` + +它不是把“全部任务完成”降级为“本组完成”；root release 仍严格表示所有 +active group、也就是所有 replay worker 都已进入 final drain。 + +#### M05：补齐 `has_pending_won()` gate + +建议：**接收缺失的小部分**。 + +目标分支已经有 `g_fdwic_block_won_enabled`，不需要引入来源分支的重复 latch。 +只需让 `has_pending_won()` 在 gate 未开启时直接返回 false，避免按需 drain +查询继续执行 BlockWon atomic load。 + +来源分支的 8.35% 数据对应完整 gate，不应直接归因给这一处残余 guard；该 +guard 仍需在目标分支重新测量。 + +#### M06：老 A5 topology fallback + +建议：**接收**。 + +fallback 只能在 topology 不可用且 ACL AICPU core count 与 OCCUPY popcount +完全相等时启用，否则继续失败，风险有明确保护边界。 + +#### M07：必要的构建兼容修改 + +建议：**选择性接收**。 + +接收： + +- 跨架构 host `strip` 兼容； +- onboard FDWIC scene 跳过不适用的 text extraction； +- BGEMM `pto::Stride` 和 weak symbol 修复； +- 确实被接收测试需要的 include/symbol 冲突修复。 + +不接收来源分支在 `runtime_builder.py` 中重复增加的 PTO include。目标分支 +已经通过 `pto_isa_root` 和构建参数处理相同问题。 + +#### M08：`mix_coown` 测试意图 + +建议：**改写后接收**。 + +它独有地覆盖一个 task 同时包含 `1C+2V` 的 joint/co-owned 场景。目标分支 +现有测试覆盖 1C+1V 和 dual-AIV，但缺少同一 task 的 1C+2V。 + +不建议原样复制 private `rt_submit_task` 用例；应改为目标分支显式 +presubmit/winner/loser API，并增加 onboard 验证入口。 + +#### M09：旧 shared-mode 测试改用 `PTO_FDWIC_SHARED_MAP` + +状态：**已确认适配后接收（2026-07-25）**。 + +来源测试不再通过运行时环境变量 +`PTO_DIST_TENSORMAP_MODE=shared` +判断 shared 模式，统一改为目标分支的编译期 +`PTO_FDWIC_SHARED_MAP`： + +- C/C++ 测试代码使用 `#if PTO_FDWIC_SHARED_MAP` 区分协议和断言； +- 测试构建分别生成 `PTO_FDWIC_SHARED_MAP=0/1` 的 private/shared ELF； +- Python 驱动不凭旧环境变量猜测模式，而是依据构建 profile、manifest 或 + 明确的 case 参数选择对应 ELF。 + +这项改造不能只替换开关名字。MB5/MB7 及相关 helper 中针对旧 shared ring +的 head/seq/window 等断言，必须改成当前 `SharedOutputCell`、 +`SharedRegionMap`、symbolic output、publication 和 writer-chain 语义。 + +### 5.2 DFX 与需要重新设计的项目 + +#### C01：Atomic swimlane runtime DFX + +状态：**已确认接收，并且必须同时支持 private/shared（2026-07-25）**。 + +迁移顺序：**放在全部非 DFX 修改完成并稳定后再迁移**。 + +不能直接移植的原因： + +1. 目标分支 phase ID 14～17 已用于 shared resolve，来源 schema 与其冲突； +2. 来源分支的 28 个 site 只覆盖 private runtime； +3. shared 新增的下列原子状态尚未建模： + - `shared_heap_cursor` + - `shared_heap_vend` + - output `published` + - output `last_writer` + - `joint_launch_expected/drained` + - `deps_prepared` + - region-map bucket、insert sentinel 和 high-water +4. 来源 phase 顺序基于 private/compete-first，目标是 shared winner-first； +5. instrumentation 曾引入 I-cache 回退，需要配套 cold/out-of-line 修改。 + +迁移时应定义新的 shared-aware schema，而不是直接复用来源分支 v3/v4。 +详细设计见第 8 节。 + +#### C02：省略未使用 `TensorRef` slot 初始化 + +状态：**已确认不接收（2026-07-25）**。 + +来源实现使用 `TensorRef() = default`。目标分支的 union 新增 +`FdwicOutputRef`，而 `FdwicOutputRef` 带默认成员初始化，原补丁不能安全地 +机械应用。 + +该优化不是 lazy-lambda，但也不是功能依赖。来源分支的数据是在先完成 M02 +单容器复用后，再加入 C02 得到的累计结果；没有 C02 相对原始多容器实现的 +独立 A/B，也没有目标 shared 分支上的数据。M02 不接收后，C02 失去了已经 +验证过的组合上下文，却仍需要为 shared union 重新设计 active-member 和 +kind tag 的安全构造方式，当前收益证据不足以覆盖这项风险。 + +若未来 profile 单独证明 `L0TaskArgs`/`TensorRef` 默认构造是目标分支热点， +再把它作为新的独立优化重新评审，不沿用来源补丁的 `= default` 写法。 + +#### C03：PMU/perf-clock 完整工具链 + +状态：**已确认接收，并且必须同时支持 private/shared（2026-07-25）**。 + +迁移顺序：**放在 C01 之后，作为最后一组 runtime DFX 能力迁移**。 + +这套工具用于回答“PA Submit 时间为什么变化”，不提供新的调度、依赖或 +atomic 功能。它包含两层观察： + +| 观察层 | 记录内容 | 用途 | +| --- | --- | --- | +| `perf-clock` | 每核首个 Submit 开始、末个 Submit 结束的 SYS_CNT；可选聚合窗内 linked-kernel ticks/calls | 低扰动地取得每核 Submit 整窗和 kernel/residual 粗分解 | +| `submit-PMU` | A5 硬件 PMU 的 scalar/vector/cube busy、I-cache request/miss、总周期；可选一个局部 phase sidecar | 判断时间变化来自 scalar 控制、linked kernel、I-cache miss，还是某个 Submit 阶段 | + +`submit-PMU` 为减少观测扰动，不在一次运行里同时打开全部阶段，而是针对 +下面的阶段分别编译和采集 ELF： + +- ArgBuild +- EmptyBracket +- Materialize +- Claim +- Register +- SubmitTransition +- EfDrainControl +- PrepareMap +- Fanin +- WinnerBuild +- AllocComplete +- LoserReplay + +完整链路还包括： + +1. AICPU PMU owner 根据物理 core/subcore 配置硬件 counter； +2. 保存原 PMU 配置，采集后执行 readback 和 restore； +3. AICore 在整窗或选中 phase 边界读 counter，并暂停 linked kernel 所在 + 区间以区分 scalar control； +4. 每核发布固定结果和可选 phase sidecar； +5. host 校验物理核拓扑、counter selector、调用次数、owner 恢复和构建身份； +6. Python 工具汇总逐核分布、I-cache 指标、phase 占比并生成报告。 + +它与 C01 atomic swimlane 的区别是： + +- C01 记录每个 atomic site 的逻辑/物理次数、轮询聚合和时序； +- C03 读取硬件性能计数器，观察整窗或某一阶段的忙周期和 I-cache 行为； +- 二者可以联合解释同一次回退；C03 的 return-ready 扣减会复用 C01 的 + atomic site 分类和 wrapper，因此实现顺序上 C01 先于 C03； +- 完整接收 `tests/atomic_probe` 也不要求把 C03 接入生产 runtime。 + +不能原样搬运的原因： + +- 代码面大； +- profile 组合和环境 gate 多； +- 与来源分支 atomic schema、private phase 和 compete-first 历史强耦合； +- 目标分支 winner-first 后，ArgBuild、WinnerBuild、LoserReplay 等阶段边界需要 + 重新定义； +- 来源 host/report 把采集拓扑写死为 32 AIC + 64 AIV，而目标 runtime + 支持最多 108 worker，必须从本次运行拓扑动态闭合； +- 它不属于 shared TensorMap 正确运行的前置条件，所以应在功能迁移全部稳定 + 后再接入，避免 DFX 插桩掩盖功能回归。 + +详细 shared-aware 迁移设计见第 8 节。 + +### 5.3 建议排除 + +#### X01：Lazy-lambda 生产实现 + +状态：**已确认排除（2026-07-25）**。 + +原因：A/B/C 数据没有收益，且会改变 orchestration API 和构参生命周期。 + +#### X02：Compete-first eager submit + +建议：**排除**。 + +原因：收益基本持平；目标分支的 shared presubmit/winner-only 已覆盖其主要 +意图，并且 loser 路径更轻。 + +#### X03：来源分支 output/register mask + +建议：**排除**。 + +原因： + +- register mask 单独没有稳定收益； +- 目标分支已经用 `DistOutputLayout::output_indices` 避免 output 重扫； +- shared region/reference 注册语义与 private mask 不同。 + +#### X04：PrepareMap 已为 `-1` 时跳写 + +建议：**暂不接收**。 + +原因：没有真实 A5 A/B 证据，收益仅停留在理论写次数和模拟验证。 + +#### X05：Runtime-wide post-kernel output flush + +建议：**排除**。 + +原因：来源分支最终已经回退。标量 producer 应自行发布，不能让通用 runtime +为所有 output 无条件执行 flush。 + +#### X06：只为 instrumentation 回归服务的冷路径修改 + +状态：**不作为独立性能优化接收；随 C01/C03 按需接收**。 + +包括 winner/alloc branch cold hint、atomic record/pollbatch out-of-line 等。 +其中 atomic record/PollBatch out-of-line 是来源分支修复 DFX 插桩导致 +I-cache 回退的必要组成，迁移 C01 时需要按目标 private/shared 热路径重新 +验证后带入；不能脱离 C01 单独搬运，也不能机械复制 compete-first 布局的 +branch hint。 + +#### X07：来源分支已否定或回退的实验 + +建议：**排除**。 + +包括： + +- fanin reorder； +- BlockWon noinline slow path； +- cursor G8； +- 16-byte ticket； +- 其他最终未保留的负收益实验。 + +## 6. `tests/st` 额外用例判断 + +### 6.1 不应整体接收 atomic minibench + +部分最终版 minibench oracle 要求日志或环境变量： + +- `PTO_DIST_DEPSIG` +- `PTO_DIST_TENSORMAP_MODE` +- `PTO_DIST_OVERHEAD` +- `PTO_DIST_RUNAHEAD` + +这些标记在来源分支最终 runtime 中已经没有对应生产实现，原样复制会产生 +测试与 runtime 脱节。 + +### 6.2 旧 shared ring 测试改用当前编译期模式 + +MB5/MB7 依赖运行时选择： + +```text +PTO_DIST_TENSORMAP_MODE=shared +``` + +其模型是旧 shared TensorMap ring。目标分支使用编译期 +`PTO_FDWIC_SHARED_MAP`、`SharedOutputCell` 和 `SharedRegionMap`，协议不同。 + +已确认不再直接排除这些测试，而是按 M09 适配： + +- 模式判断改用 `PTO_FDWIC_SHARED_MAP`； +- private/shared 分别构建和运行； +- 删除旧 runtime-selectable ring 假设； +- 把 `test_dist_atomic_mb5_shared_map` 及相关 helper 的真值模型改为当前 + shared output/region 协议。 + +### 6.3 其他不建议原样接收的测试 + +- `vector_example`：目标分支 dependency smoke 已覆盖更完整； +- MB1/MB3/MB9 synthetic UT：复制的是旧 cursor/ring 算法，不能代表当前 + production runtime； +- `test_fdwic_tensor_map_retire`：只服务于尚无真实收益证据的 PrepareMap + 候选； +- atomic PollBatch/converter/report 单测：只在接收 C01 时需要。 + +### 6.4 可保留的测试意图 + +- `mix_coown`：按 M08 改写； +- MB2/MB8 的直接数据一致性场景：若 `atomic_probe` 和目标 smoke 仍有覆盖 + 缺口，可抽取测试意图，但不复制旧 runtime oracle。 + +## 7. 当前建议集合 + +当前已确认和仍待确认的集合为： + +```text +已确认接收： + M01 atomic_probe 整体 + M09 旧 shared-mode 测试改用 PTO_FDWIC_SHARED_MAP 并更新协议断言 + C01 atomic swimlane runtime DFX（private + shared） + C03 PMU/perf-clock 完整工具链（private + shared） + +已确认不接收： + M02 PA 单参数容器复用 + C02 未使用 TensorRef slot 的安全惰性初始化 + X01 lazy-lambda + +仍待逐项确认： + M03 private heap H1 + M04 G16 final barrier + M05 has_pending_won 现有 gate 补齐 + M06 legacy topology fallback + M07 必要构建兼容修复 + M08 改写后的 mix_coown + +排除： + X02 compete-first + X03 output/register mask + X04 PrepareMap 跳写 + X05 runtime-wide output flush + X07 已否定或回退的实验 + +绑定接收： + X06 中确属 C01/C03 插桩正确性或 I-cache 修复所需的部分 +``` + +实施约束已经确认：按影响面从小到大分阶段；每阶段通过第 10 节的完整门禁 +后才进入下一阶段；atomic 和 PMU DFX 放在所有非 DFX 阶段之后。 + +## 8. C01/C03 的 shared-aware 迁移设计 + +### 8.1 总体边界 + +C01/C03 不以 cherry-pick 来源分支提交为目标，而是复用其中已经验证的 +DFX 机制，并在目标分支当前协议上重新落点： + +- 保留目标分支现有 shared winner-first、symbolic output、 + `SharedOutputCell`、`SharedRegionMap`、joint launch 和 resolve 语义； +- 不带入 compete-first、lazy-lambda、旧 shared ring 或 private + 96-worker 假设； +- 先冻结非 DFX runtime 的最终结构，再定义 atomic site 和 PMU phase； +- C01 先于 C03。C03 需要复用 C01 对 atomic `result-used`/ + `return-ready` 的分类，才能从 scalar submit 窗中正确扣除等待返回值的 + atomic dependency 时间； +- DFX 关闭时必须能从编译产物中完全裁掉记录路径，保留一份不含 DFX + 插桩的权威性能基准 ELF。 + +来源分支的 atomic schema v4 不能直接复用。目标分支 schema v1 的 phase +14～17 已经分别是 `Resolve`、`ResolveWait`、`ResolveInvalidate` 和 +`ResolveCopy`，而来源分支把相同 ID 重新用于 `Atomic`、 +`ClockBaseline`、`OrchestrationReplay` 和 `FinalDrain`。原样搬运会让旧 +trace 被静默误解。 + +### 8.2 编译开关和 profile 正交化 + +map 模式只由编译期 `PTO_FDWIC_SHARED_MAP=0/1` 决定。DFX profile 与 map +模式正交组合，不再维护 private-only profile： + +| profile | 主要开关 | 用途 | +| --- | --- | --- | +| `off` | 所有 C01/C03 开关关闭 | 功能和性能基准 | +| `swimlane` | `PTO_FDWIC_TRACE_ENABLED=1` | 原有 phase trace | +| `atomic` | trace + atomic level | C01 atomic site/等待聚合 | +| `perf-clock` | `PTO_FDWIC_PERF_CLOCK=1` | C03 每核 Submit 整窗 | +| `perf-clock-kernel` | 再开 kernel 聚合 | Submit 窗内 linked-kernel 粗分解 | +| `submit-pmu-none` | `PTO_FDWIC_SUBMIT_PMU=1` | C03 A5 PMU 整窗 | +| `submit-pmu-` | 再指定 phase ID | 每个 ELF 只采一个局部阶段 | + +约束如下： + +- `perf-clock`、`submit-PMU` 与普通/atomic trace 互斥； +- `perf-clock` 与 `submit-PMU` 互斥； +- atomic trace 只有在 trace 编译开关开启时才能启用； +- profile 不能改变 `PTO_FDWIC_SHARED_MAP`，同一 profile 分别构建 + private/shared ELF； +- build cache key、ELF marker、输出文件名和报告 provenance 都必须包含 + map 模式、profile、phase ID 和 schema version，发现运行时/ELF 模式 + 不一致时直接失败； +- `PTO_FDWIC_TRACE_ENABLED=0` 的 ELF 中不得残留 atomic record、 + PollBatch 或 PMU phase 边界符号。 + +### 8.3 新 swimlane schema + +建议新 schema 从 v5 起步，明确与目标 v1 和来源 v4 隔离： + +1. 保留目标分支 phase 0～17 的编号和含义； +2. 从 18 开始追加 `Atomic`、`ClockBaseline`、 + `OrchestrationReplay`、`FinalDrain`、winner/loser 及 shared 专用 + phase；最终编号只在 M03～M08 取舍和非 DFX runtime 收敛后冻结； +3. 采用来源 v4 已验证的 32-byte record，把 core/lane/block 拓扑移到 + 64-byte per-core state，以便继续保留每核 64K record 而不扩大设备 + 分配； +4. header/core state 至少记录： + - schema version、record bytes、records per core； + - `PTO_FDWIC_SHARED_MAP`； + - profile/trace level； + - 本次实际 AIC/AIV/worker 拓扑； + - event count、dropped count； + - atomic logical calls、physical rows 和 PollBatch rows； + - build identity 或可与 manifest 对应的 hash； +5. host 先验证 magic/version/record bytes/map mode/profile/topology，再读 + records；任何不一致都 fail closed； +6. offline converter 显式保留目标归档 v1、来源归档 v3/v4 和新 v5 的 + 独立读取路径，不能猜测字段布局；host runtime 只接受本次 ELF 声明的 + schema。转换后的基础十列 JSON/CSV 形状保持稳定，避免下游工具全部 + 重写； +7. exclusive analyzer 保留目标分支四个 Resolve phase，并为 + `OrchestrationReplay`、`FinalDrain` 建 parent interval。shared 子阶段 + 只能在对应 parent 内出现，parent residual 由 analyzer 计算，不能用 + 重叠区间重复计时。 + +### 8.4 C01 atomic site 迁移 + +#### 8.4.1 保留与扩展的 site + +来源分支已有 28 个 private site，编号 0～27 应保留，便于复用现有 +analyzer、UT 和历史数据： + +```text +StartupIncrement, StartupPoll, FatalPoll, FatalSet, +ClaimMax, FaninFlagLoad, +CompletionVendExchange, CompletionFlagExchange, +FrontierInitialLoad, FrontierFlagLoad, FrontierMax, +HeapFrontierLoad, HeapVendLoad, +ReplayDoneIncrement, ReplayDonePoll, +WonSlotClaimMax, WonRemainingExchange, +WonLaneResetExchange, WonLaneDepositExchange, +WonStatePublishExchange, WonAnyPublishExchange, +WonAnyLoad, WonStateLoad, WonLaneClaimExchange, +WonLaneReleaseExchange, WonRemainingFetchSub, +WonStateClearExchange, WonDrainedLoad +``` + +shared 需要从 28 以后追加独立 site。下面是需要覆盖的语义集合，最终编号在 +非 DFX runtime 冻结后生成，不能先按来源 private 枚举硬套： + +| shared 位置 | 需要区分的 atomic 语义 | +| --- | --- | +| `SharedOutputCell::published` | 快速 probe load、等待 load、publish exchange | +| `SharedOutputCell::last_writer` | load、exchange、fetch-max | +| shared heap | cursor reserve fetch-add、wrap padding fetch-add、vend fetch-add | +| `SharedRegionMap` | bucket head load、sentinel wait load、claim exchange、publish exchange、high-water fetch-add | +| prepared deps | `deps_prepared` publish exchange、wait load | +| joint launch | expected fetch-add、drained fetch-add、expected load、drained load | +| final barrier | 以最终接受的 M04/现有协议分别标记 arrival、release 和 poll | + +`insert_lock` 等当前没有生产调用者的 helper 不应伪造成“已覆盖的活跃 +site”。迁移前先确认它们是删除、保留但静态未使用，还是确有调用，再决定 +是否分配 site。 + +每个 site 的 schema 元数据必须显式给出： + +- atomic op：Load、Exchange、FetchAdd、FetchMax 或 FetchSub； +- 返回值是否影响后续控制流； +- 是否允许进入 PollBatch； +- 适用模式：common、private-only 或 shared-only； +- 所属 wait region/phase。 + +#### 8.4.2 PollBatch 需要脱离 site ID + +来源实现同时要求 site ID 小于 32，并直接用 `1U << site_id` 构造 +PollBatch mask。shared site 加入后必然超过 32，这个限制必须拆开： + +- record 的 `aux` 继续保存稳定的 16-bit site ID； +- 只有可批处理的等待 site 分配紧凑 `poll_batch_index`； +- batch mask 使用该紧凑 index，而不是 raw site ID；若可批处理 site + 超过 32，再把 mask 升为 64 bit 或拆成多个 word； +- schema/tool 同时发布 `site_id -> poll_batch_index` 映射； +- 所有状态迁移成功的 RMW 仍单独记录，只有幂等失败重试或纯观察 load + 可以批处理； +- phase/parent interval 结束、site 切换和真正状态迁移前必须关闭当前 + PollBatch，保证 logical call 数精确。 + +shared 中应重点建立的 wait region 包括 output publication、region bucket +sentinel、`deps_prepared`、joint drain、heap/frontier reuse、final barrier +和 BlockWon。报告必须同时闭合： + +```text +logical atomic calls + = direct physical rows + + 所有 PollBatch 中的精确 poll_count +``` + +#### 8.4.3 shared 语义不能被 wrapper 改写 + +来源分支在 `api_glue.h` 中直接把部分普通 atomic helper 换成 trace +wrapper。目标 shared 模式的 `fatal_set()` 当前刻意返回 false；若机械换成 +来源的 `FatalPoll` load，会在 shared 下新增真实 atomic 读取并改变控制流。 + +因此 wrapper 分层必须是： + +- mode-specific helper 先保持原有语义； +- 只在该模式原本会执行 atomic 时记录对应 site； +- `set_fatal()` 的真实 exchange 可以记录，shared `fatal_set()==false` + 不伪造一次 load； +- DFX 关闭时 wrapper 内联回原 helper，不能增加额外 load、branch 或 + cache invalidate； +- AICPU 初始化/owner 的 host atomic 不进入 AICore swimlane site。 + +迁移时应增加静态检查：AICore 生产路径中的 atomic 调用必须来自已登记 +wrapper 或明确 allowlist，避免 shared 新增 atomic 漏记。静态检查只负责 +“是否分类”，动态 closure 负责“是否真正按协议执行”。 + +#### 8.4.4 I-cache 保护 + +来源 C01 曾因 inline record/PollBatch 扩大热路径而产生 I-cache 回退，随后 +才把 cold record 路径 out-of-line，并补了少量 cold branch hint。这部分是 +C01 完整性的一部分，但需要在目标协议上重做： + +- record reserve、overflow、PollBatch flush 走 cold/out-of-line； +- private loser、shared loser、wrong-role、alloc non-candidate 分别检查 + code layout，不能复制 compete-first 的 hint； +- 比较 DFX-off、phase-only、atomic 三类 ELF 的 text size、关键符号和 + PA 配对性能； +- 若 DFX-off ELF 相对迁移前仍改变热路径，C01 阶段不得通过。 + +### 8.5 C03 perf-clock 与 submit-PMU 迁移 + +#### 8.5.1 先补齐对称的 Submit 生命周期 + +来源 private runtime 的 winner/loser 都有明确结束路径；目标 shared 的 +loser wrapper 主要返回 symbolic refs，不再调用原 private loser replay。 +如果直接套来源 hooks，shared loser、wrong-role worker 和 alloc +non-candidate 会启动窗口却不关闭，PMU 报告无法闭合。 + +在 DFX 编译开启时，需要给当前 API 增加对称、但 DFX-off 为零成本的观测 +边界： + +```text +presubmit / alloc begin + -> wrong-role 或 non-candidate：对应 completion hook + -> winner：winner completion hook + -> loser：shared loser completion hook +``` + +每个 core 串行 replay，本地 profiling state 可以保存当前 task/window； +不应为了 DFX 扩大正常构建中的 `SubmitToken`。linked kernel 仍由统一 +pause/resume 排除，避免把 vector/cube 执行混入 scalar control。 + +#### 8.5.2 shared phase 重新定义 + +来源 phase 中可以保留机制，但不能沿用 private PA 的固定次数公式： + +- `ArgBuild` 在 shared 下只发生于 winner，应改名或定义为 + `WinnerArgBuild`，边界为 presubmit 返回到 winner submit 入口； +- `LoserReplay` 拆成 `PrivateLoserReplay` 和 `SharedLoserReturn`； +- wrong-role 早退单列低成本 phase，且必须关闭 Submit window； +- `PrepareMap` 在 shared 下应分为 `SharedDepsPrepare` 和 + `SharedDepsWait`； +- shared materialize 至少要能区分 heap reserve、output publish； +- region-intent 路径至少区分 lookup/register； +- 保留 `Resolve`、`ResolveWait`、`ResolveInvalidate`、 + `ResolveCopy`； +- joint/co-owned 路径需要 joint handoff/drain phase； +- `EmptyBracket` 只用于测量 PMU 边界自身开销，不代表业务 phase。 + +建议首批 shared phase 集合为： + +```text +RoleFilter / Presubmit +Claim +WinnerArgBuild +Materialize +SharedHeapReserve +SharedOutputPublish +Fanin +SharedRegionLookup +SharedRegionRegister +SharedDepsPrepare +SharedDepsWait +Register +Resolve / ResolveWait / ResolveInvalidate / ResolveCopy +WinnerBuildControl +AllocCompleteControl +SharedLoserReturn +EfDrainControl +JointDrainControl +FinalDrainControl +SubmitTransition +``` + +具体 phase 可以先合并粗粒度边界，再由 C01 atomic 分析结果决定是否拆细； +不能一开始就在一个 ELF 同时打开全部 phase。每次 `submit-PMU` 构建只选择 +一个 phase。 + +#### 8.5.3 拓扑和调用次数必须动态闭合 + +来源 `submit-PMU` 的 host/report 写死： + +```text +num_cores = 96 +aic_cores = 32 +aiv_cores = 64 +``` + +目标 runtime 的 `RUNTIME_MAX_WORKER` 是 108，并且 smoke 的 `MaxBd` 应使用 +系统实际支持的最大 block 数。迁移后： + +- AICPU owner 从本次 `layout[]`/runtime topology 得到实际参与的 + AIC/AIV/physical subcore 集合； +- header、bitmap、configured/restored count 和 triplet closure 都使用 + 实际拓扑，不能把 96 写入 JSON 模板； +- owner 配置任一核失败时设置 AICPU abort 状态，禁止 worker 在半配置 PMU + 状态下继续运行； +- 采集结束必须逐核 readback、restore，并验证 active-after-restore 为 0； +- buffer 仍按最多 108 个 worker 预留，但报告只接受本次参与者； +- block dimension 1、36 和当前设备支持的 `MaxBd` 都需要定向验证。 + +PA “每 batch 四个 Kernel + 一个 Alloc”的公式只能留在 PA case/report +层。通用 runtime 依据 manifest/runtime counters 闭合以下动态不变量： + +```text +每核 replayed submit 数 +候选 / claim attempt 数 +winner / loser / wrong-role 数 +shared phase 调用数 +output publication / region operation 数 +全局每个 task/role 恰有一个 winner +所有开始的 Submit window 都已关闭 +``` + +#### 8.5.4 buffer 与结果身份 + +`swimlane`、`perf-clock` 和 `submit-PMU` 可以复用 Runtime 到设备的 profiling +buffer plumbing，但三者的 header/version/record layout 必须独立，不能靠 +同一指针猜测类型。设备 allocation 需保留原始指针和 64-byte 对齐后的 +地址，释放原始 allocation。 + +每份结果至少携带或可追溯到： + +- simpler commit； +- PTO ISA commit； +- `PTO_FDWIC_SHARED_MAP`； +- profile 和 phase ID； +- schema version 与完整 compile definitions； +- host/AICore ELF hash； +- workload/case、batch、block dimension 和实际 topology。 + +atomic 与 PMU 不在同一 ELF 同时采集。工具通过上述 build identity 和相同 +workload signature 关联两次独立运行，不能把不匹配的捕获合并成一份报告。 + +### 8.6 主要代码影响面 + +| 层次 | C01 | C03 | +| --- | --- | --- | +| common ABI | `swimlane_types.h`、site/phase/schema | `perf_clock.h`、`submit_pmu_types.h` | +| AICore | atomic wrappers、PollBatch、phase 边界、cold record | window/phase hooks、counter read、kernel pause/resume | +| AICPU | buffer 注册/清零、拓扑 metadata | PMU owner 配置、readback、restore、abort | +| host runtime | trace level、buffer ownership、strict export | profile 选择、结果校验、动态 topology | +| build | trace-off/atomic profile | perf-clock/PMU profile 与 cache identity | +| tools/tests | schema、converter、exclusive analyzer、PollBatch UT | PMU report、provenance/closure UT、A5 定向验证 | + +这也是 C01/C03 必须放在最后的具体原因：它们横跨 common ABI、AICore、 +AICPU、host、构建和工具链，并且其观测边界依赖前面所有 runtime 取舍。 + +## 9. 按影响面从小到大的迁移阶段 + +以下是执行顺序框架，不把尚待确认的 M03～M08 自动纳入实现。某一项被 +确认不接收时，跳过对应阶段；不能因此把后面的 DFX 阶段提前到仍未收敛的 +功能阶段之前。 + +每个阶段只允许包含一类可解释的改动。每阶段结束均执行第 10 节的 +shared/private 八组强制门禁和本阶段定向验证；任一失败立即停止，不在同一 +工作树继续叠加下一阶段。 + +### 9.1 非 DFX 阶段 + +| 阶段 | 内容 | 影响面 | 本阶段额外验证 | +| --- | --- | --- | --- | +| N0 | 固定基线、编译器/PTO ISA、建立验证脚本和 DFX-off 性能基线 | 无产品改动 | 目标分支原始 tip 的八组门禁；记录 ELF/hash/PA 数据 | +| N1 | M07 中确认需要的 host strip、text extraction、BGEMM/include 构建兼容项 | 构建/host，无调度协议变化 | 干净构建、对应构建失败复现转通过、八组门禁 | +| N2 | M09 shared 测试改用 `PTO_FDWIC_SHARED_MAP`；M08 若确认则只加入改写后的 `mix_coown` 用例 | 测试层 | private/shared 分别编译；旧 ring oracle 已删除；1C+2V onboard | +| N3 | M05 `has_pending_won()` 已有 gate 的缺失 guard | 单个 AICore 查询点 | gate off 不产生 BlockWon load；gate on 功能不变 | +| N4 | M03 private heap H1 首圈快速路径 | private heap 局部热路径 | private 首圈/wrap/backpressure；shared 二进制与行为不受影响 | +| N5 | M06 legacy topology fallback | AICPU 注册与异常环境 | topology 正常/缺失、count 相等/不等四种组合；错误组合继续失败 | +| N6 | M04 G16 final barrier | 全 worker 最终排空协议 | private/shared，Bd1/Bd36/MaxBd，BlockWon/joint drain，超时和退出 closure | +| N7 | 非 DFX 收敛点 | 只做审计，不再增加功能 | 全量门禁、ABI/layout/static assert、DFX-off PA 配对基线 | + +顺序依据： + +- N1/N2 不改变 runtime 调度语义； +- N3 是单点 guard； +- N4 只影响 private heap； +- N5 触及 AICPU 注册但只在 topology 缺失时生效； +- N6 改变所有 worker 的最终同步协议，属于非 DFX 中影响最大的一项； +- N7 之后冻结 atomic site 和 PMU phase，避免 DFX 反复重编号或移动边界。 + +N2 中所有 shared 判断必须使用 `PTO_FDWIC_SHARED_MAP`。Python runner 可以 +根据明确的 build manifest/case 参数选择 ELF，但不能恢复 +`PTO_DIST_TENSORMAP_MODE` 运行时选择。 + +### 9.2 最后一组：DFX 阶段 + +只有 N0～N7 中所有实际被接受的阶段通过后，才开始下面的 DFX 尾部： + +| 阶段 | 内容 | 影响面 | 本阶段额外验证 | +| --- | --- | --- | --- | +| D0 | M01 整体引入 `tests/atomic_probe`，校验 archive/manifest；不启用 runtime 插桩 | 测试、文档和历史资产 | 文件/manifest 完整性；适用的独立 probe；八组 runtime 门禁 | +| D1 | 新 v5 schema、32B record、per-core metadata、profile gate、host strict parser；所有新记录路径编译关闭 | common ABI、host、build | v1/v3/v4/v5 parser UT；mode/profile mismatch fail；DFX-off ELF 无新符号 | +| D2 | C01 来源 0～27 site、wrapper、PollBatch 和 cold/out-of-line 机制，先覆盖 private/common 路径 | AICore 热路径、host/tools | PollBatch 精确计数；private site closure；shared common 路径不回归 | +| D3 | C01 追加 shared site、shared wait region、mode-aware analyzer | shared AICore 热路径 | output/heap/region/deps/joint/final-barrier closure；无未分类 atomic | +| D4 | C03 `perf-clock` 和可选 kernel aggregate；补齐 shared winner/loser/wrong-role/non-candidate 对称 completion hook | Submit 生命周期、低扰动计时 | 两模式窗口全部关闭；kernel ticks/calls 与 Submit 窗闭合 | +| D5 | C03 `submit-PMU-none`：owner、整窗 counter、readback/restore、abort、动态 topology | AICPU + AICore + host | A5 Bd1/Bd36/MaxBd；owner 全配置/恢复；失败注入；无半配置运行 | +| D6 | C03 private/common 单 phase ELF 和报告 | 多 profile 构建、private phase | 每个 ELF 仅一个 phase；边界/call count/PMU counter closure | +| D7 | C03 shared phase 与 shared 动态公式 | shared phase 热路径、报告 | winner/loser/wrong-role、output/region/deps/resolve/joint 各路径闭合 | +| D8 | DFX 最终收敛 | 全工具链 | atomic 与 PMU 分次配对采集、provenance 对齐、observer-overhead 审计、最终八组门禁 | + +D0 只迁移独立实验资产，不把历史二进制、原始数据或 lazy-lambda 样本当作 +当前 runtime 功能测试。`ascendc/`、`ccec/`、`cpu/` 是硬件/编译器/环境 +probe，按第 10.8 节执行，不在每个 DFX 阶段重复。 + +D1 的 schema 和 buffer plumbing 必须先证明在 DFX-off 下没有行为/布局 +回归，D2 才能落 atomic call-site。D2/D3 必须先于 D5～D7，因为 PMU +scalar-window 扣减需要最终 atomic site 的 `result-used` 分类。 + +### 9.3 每阶段提交边界 + +每个阶段至少形成一个可独立回退、可独立验证的提交；若一个阶段包含 +common ABI、device 和 host 三层，应继续拆为下列顺序，但只有整阶段全部 +通过才算完成： + +1. schema/types/tools UT； +2. host/build plumbing； +3. device runtime hooks； +4. case/报告适配； +5. 干净重编译和完整门禁。 + +禁止以“先把来源提交全部 cherry-pick，再集中修冲突”的方式实施。每次只 +从来源提交提取当前阶段需要的语义；来源提交中夹带的 compete-first、 +lazy-lambda、旧 96-worker 公式或旧 shared ring 必须在进入工作树前剔除。 + +## 10. 每阶段验证方法和范围 + +本节把 +[`tests/atomic_probe/fdwic_shared_atomic_merge_plan.md`](../../tests/atomic_probe/fdwic_shared_atomic_merge_plan.md) +中的验证方法和范围复制并改写为本计划的“阶段”术语。后续执行以本节为 +准，不需要依赖原计划的 cherry-pick 顺序。 + +### 10.1 验证总原则 + +- 编译通过不是功能验证结论。 +- 每个阶段根据代码影响和新增/修改用例确定额外验证范围。 +- 每完成一个阶段，或完成一次已确认的冲突/适配修改后，必须重新编译 + runtime 和 kernels，再运行本阶段门禁。 +- 禁止使用上一阶段的 `.o`、`.so` 或 kernel 结果代替重新编译。 +- 验证前确认源码、runtime 二进制和 PTO ISA revision 一致。 +- 需要排除缓存时，隔离 `build/cache`、`build/lib` 及相关目标文件，确保 + `.o`、`.so` 和 kernel 重新生成。 +- 不修改项目配置来绕过失败。 +- onboard 验证必须通过 `task-submit` 运行。 +- 性能配置、profiling 输出和功能 golden 不能互相替代。 + +### 10.2 Onboard 与 `task-submit` 强制规则 + +所有访问 A5 设备的操作都必须由 `task-submit` 触发；源码检查、清理、 +编译和 A5Sim 在其外执行。 + +普通功能用例统一使用： + +```text +--timeout 90 +--max-time 90 +--device auto +--device-num 1 +``` + +超时视为异常。stress 或性能任务只有在记录轮数、预计时长和理由后才能 +单独提高超时。 + +默认执行 A5 arch precheck。当前环境没有 `npu-smi` 时，只能按已有用户 +授权跳过 precheck；该豁免不跳过 onboard 用例。A5 每一轮都必须单独调用 +一次 `task-submit`，禁止用一次设备任务包住多轮 pytest 循环。A5Sim 不使用 +`task-submit`。 + +### 10.3 根目录验证脚本和环境要求 + +在项目根目录建立两个本地验证脚本： + +```text +build_runtimes.sh +run_tests.sh +``` + +它们是本地验证工具，不属于产品提交。创建后执行： + +```bash +chmod 700 build_runtimes.sh run_tests.sh +``` + +脚本只在根目录不存在时创建一次；切换阶段、commit 或 private/shared 模式 +时重新运行，不重复生成脚本。每个环境必须重新确认： + +1. `PTO_ISA_ROOT` 指向项目 `build/` 之外的独立、干净 checkout； +2. `PTO_ISA_COMMIT` 默认固定为 CI 使用的 + `ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8`，CI pin 改变时两个脚本 + 同步更新； +3. 根目录 `.venv` 存在，包含 pytest、nanobind 和用例依赖； +4. 外部 shell 已设置并加载 CANN： + + ```bash + export ASCEND_HOME_PATH=/usr/local/Ascend/ascend-toolkit/latest + source "$ASCEND_HOME_PATH/set_env.sh" + ``` + + `set_env.sh` 把变量规范化为实际版本目录是正常行为； +5. 使用 `set_env.sh` 提供的 `PATH`、`LD_LIBRARY_PATH`、 + `CMAKE_PREFIX_PATH`、`ASCEND_OPP_PATH`，不另维护一套 CANN 路径； +6. 项目根目录和 `python/` 加入 `PYTHONPATH`，已有值保留在后； +7. `gcc` 和 `g++` 都必须是真正的 15，不能只设置 `CXX` 或只检查 + `g++`； +8. A5 onboard 必须经 `task-submit`；没有 `npu-smi` 只影响已授权的 + precheck 豁免，不影响 onboard 门禁。 + +### 10.4 `build_runtimes.sh` + +脚本只负责环境检查、旧产物隔离和干净编译，不运行测试，也不占用 +`task-submit`。同一 commit、map 模式、编译器、PTO ISA revision 和 DFX +profile 下可供多个用例复用；任一维度变化都必须重新构建。 + +将 `/path/to/pto-isa` 替换为当前环境的真实路径： + +```bash +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +expected_simpler_commit="$1" +map_mode="$2" +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +venv_path="$repo_root/.venv" + +case "$map_mode" in + shared) shared_map="1" ;; + private) shared_map="0" ;; + *) + echo "Invalid map mode: $map_mode (expected shared or private)" >&2 + exit 2 + ;; +esac + +cd "$repo_root" +test "$(git rev-parse --show-toplevel)" = "$repo_root" +source "$venv_path/bin/activate" + +: "${ASCEND_HOME_PATH:?ASCEND_HOME_PATH must be set}" +source "$ASCEND_HOME_PATH/set_env.sh" + +export PTO_ISA_ROOT="/path/to/pto-isa" +export PTO_ISA_COMMIT="ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8" +export CXXFLAGS="-DPTO_FDWIC_SHARED_MAP=$shared_map" +export PYTHONPATH="$repo_root/python:$repo_root${PYTHONPATH:+:$PYTHONPATH}" + +test "$(git rev-parse HEAD)" = \ + "$(git rev-parse "$expected_simpler_commit^{commit}")" +test "$(gcc -dumpversion | cut -d. -f1)" = "15" +test "$(g++ -dumpversion | cut -d. -f1)" = "15" +test -d "$PTO_ISA_ROOT/.git" +test "$(git -C "$PTO_ISA_ROOT" rev-parse HEAD)" = "$PTO_ISA_COMMIT" +test -z "$(git -C "$PTO_ISA_ROOT" status --short)" + +stamp="$(date +%Y%m%d-%H%M%S)" +build_backup="$repo_root/../simpler-build-backup-$stamp" +binding_backup="$repo_root/../simpler-binding-backup-$stamp" + +if [ -e "$repo_root/build" ]; then + test ! -e "$build_backup" + mv "$repo_root/build" "$build_backup" +fi +mkdir -p "$repo_root/build" + +if compgen -G "$repo_root/python/_task_interface*.so" >/dev/null; then + test ! -e "$binding_backup" + mkdir -p "$binding_backup" + mv "$repo_root"/python/_task_interface*.so "$binding_backup"/ +fi + +test -z "$( + find "$repo_root/build" \ + -type f \ + \( -name '*.o' -o -name '*.so' -o -name CMakeCache.txt \) \ + -print -quit +)" + +strip_shim="$(mktemp -d /tmp/simpler-aarch64-strip.XXXXXX)" +trap 'rm -rf "$strip_shim"' EXIT +ln -s \ + "$ASCEND_HOME_PATH/tools/hcc/aarch64-target-linux-gnu/bin/strip" \ + "$strip_shim/strip" +export PATH="$strip_shim:$PATH" + +nanobind_dir="$(python -m nanobind --cmake_dir)" +cmake \ + -S "$repo_root" \ + -B "$repo_root/build/python-binding" \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython_EXECUTABLE="$(command -v python)" \ + -Dnanobind_DIR="$nanobind_dir" + +cmake \ + --build "$repo_root/build/python-binding" \ + --target _task_interface \ + -j"$(nproc)" + +test -n "$( + find "$repo_root/python" \ + -maxdepth 1 \ + -type f \ + -name '_task_interface*.so' \ + -print -quit +)" + +python "$repo_root/simpler_setup/build_runtimes.py" \ + --lib-dir "$repo_root/build/lib" \ + --cache-dir "$repo_root/build/cache" \ + --platforms a5sim a5 \ + --pto-isa-commit "$PTO_ISA_COMMIT" +``` + +DFX 阶段需要在此脚本增加一个显式 profile 参数，并把对应 compile +definitions 纳入 cache identity；不能继续只靠外部 `CXXFLAGS` 猜测 profile。 +在 D1 落地前仍使用上面的两参数权威版本。 + +### 10.5 `run_tests.sh` + +脚本接收一个或多个 pytest target。`a5sim` 直接执行 pytest；`a5` 在普通 +shell 完成环境和 revision 检查，只把最终 pytest 命令交给 +`task-submit`，避免环境准备占用设备锁。 + +`PTO_ISA_ROOT` 和 `PTO_ISA_COMMIT` 必须与 `build_runtimes.sh` 完全一致： + +```bash +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -lt 5 ]; then + echo "Usage: $0 [ ...]" >&2 + exit 2 +fi + +expected_simpler_commit="$1" +map_mode="$2" +platform="$3" +repeat_count="$4" +shift 4 +pytest_targets=("$@") +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +venv_path="$repo_root/.venv" + +case "$map_mode" in + shared) shared_map="1" ;; + private) shared_map="0" ;; + *) + echo "Invalid map mode: $map_mode (expected shared or private)" >&2 + exit 2 + ;; +esac + +if ! [[ "$repeat_count" =~ ^[1-9][0-9]*$ ]]; then + echo "Invalid repeat count: $repeat_count (expected a positive integer)" >&2 + exit 2 +fi + +cd "$repo_root" +test "$(git rev-parse --show-toplevel)" = "$repo_root" +source "$venv_path/bin/activate" + +: "${ASCEND_HOME_PATH:?ASCEND_HOME_PATH must be set}" +source "$ASCEND_HOME_PATH/set_env.sh" + +export PTO_ISA_ROOT="/path/to/pto-isa" +export PTO_ISA_COMMIT="ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8" +export CXXFLAGS="-DPTO_FDWIC_SHARED_MAP=$shared_map" +export PYTHONPATH="$repo_root/python:$repo_root${PYTHONPATH:+:$PYTHONPATH}" + +test "$(git rev-parse HEAD)" = \ + "$(git rev-parse "$expected_simpler_commit^{commit}")" +test "$(gcc -dumpversion | cut -d. -f1)" = "15" +test "$(g++ -dumpversion | cut -d. -f1)" = "15" +test -d "$PTO_ISA_ROOT/.git" +test "$(git -C "$PTO_ISA_ROOT" rev-parse HEAD)" = "$PTO_ISA_COMMIT" +test -z "$(git -C "$PTO_ISA_ROOT" status --short)" + +common_pytest_args=( + "${pytest_targets[@]}" + --platform "$platform" + -p no:xdist + -v + --require-pto-isa + --pto-isa-commit "$PTO_ISA_COMMIT" +) + +case "$platform" in + a5sim) + for ((iteration = 1; iteration <= repeat_count; iteration++)); do + echo "[run_tests] a5sim iteration $iteration/$repeat_count" + python -m pytest "${common_pytest_args[@]}" + done + ;; + a5) + onboard_command=( + env + "PTO_ISA_ROOT=$PTO_ISA_ROOT" + "PTO_ISA_COMMIT=$PTO_ISA_COMMIT" + "CXXFLAGS=$CXXFLAGS" + "PYTHONPATH=$PYTHONPATH" + "$venv_path/bin/python" + -m pytest + "${common_pytest_args[@]}" + --device __TASK_DEVICE__ + ) + printf -v onboard_command_string '%q ' "${onboard_command[@]}" + onboard_command_string="${onboard_command_string//__TASK_DEVICE__/'$TASK_DEVICE'}" + printf -v shell_command 'bash -ic %q' "$onboard_command_string" + for ((iteration = 1; iteration <= repeat_count; iteration++)); do + echo "[run_tests] a5 iteration $iteration/$repeat_count" + task-submit \ + --timeout 90 \ + --max-time 90 \ + --device auto \ + --device-num 1 \ + --run "$shell_command" + done + ;; + *) + echo "Invalid platform: $platform (expected a5 or a5sim)" >&2 + exit 2 + ;; +esac +``` + +DFX profile 引入后，`run_tests.sh` 必须读取并核对 build manifest 中的 +profile/map mode/ELF hash，不允许测试命令与已构建二进制不一致。 + +### 10.6 标准功能调用 + +下面以 shared、当前 HEAD 为例： + +```bash +COMMIT=$(git rev-parse HEAD) + +./build_runtimes.sh "$COMMIT" shared + +./run_tests.sh "$COMMIT" shared a5sim 1 \ + examples/a5/fully_distributed_within_core/simple_orch_smoke \ + examples/a5/fully_distributed_within_core/shared_symbol_smoke \ + examples/a5/fully_distributed_within_core/submit_dependency_smoke \ + --manual include + +./run_tests.sh "$COMMIT" shared a5 1 \ + examples/a5/fully_distributed_within_core/simple_orch_smoke \ + examples/a5/fully_distributed_within_core/shared_symbol_smoke \ + examples/a5/fully_distributed_within_core/submit_dependency_smoke \ + --manual include + +./run_tests.sh "$COMMIT" shared a5sim 1 \ + examples/a5/fully_distributed_within_core/paged_attention_unroll/\ +test_paged_attention_unroll.py + +./run_tests.sh "$COMMIT" shared a5 1 \ + examples/a5/fully_distributed_within_core/paged_attention_unroll/\ +test_paged_attention_unroll.py +``` + +private 阶段把以上五条命令中的 `shared` 全部替换为 `private`，并先做新的 +干净构建。新增用例可以作为额外 pytest target 传入；多个 target 可共享 +同一次匹配的 runtime 构建。 + +功能验证约束： + +- smoke 固定为 `simple_orch_smoke`、`shared_symbol_smoke` 和 + `submit_dependency_smoke`，必须传 `--manual include`； +- PA 只验证 Case1； +- PA Case3 是已知非必过项，不纳入本次阶段门禁； +- PA A5Sim 必须运行真实 kernel 和真实 golden； +- PA 功能命令不得添加 `--use-example-exec-time` 或 `--skip-golden`； +- `--use-example-exec-time` 只允许用于 sim 性能分析，其结果不能作为功能 + 正确性证据； +- `repeat-count` 必须是正整数，A5 每轮单独提交一个 `task-submit`。 + +### 10.7 重新编译规则和八组强制门禁 + +以下任一情况发生后必须重新运行匹配 map mode/profile 的干净构建： + +- 进入一个新迁移阶段； +- 解决并落地一次冲突； +- 修改 runtime、platform、orchestration 或 incore kernel； +- 在 shared/private 宏之间切换； +- 切换分支或 commit； +- 修改编译器、编译参数、DFX profile 或 PTO ISA revision。 + +不得使用 `python -m pip install '.[test]'` 建立阶段构建：它会探测并构建 +无关 a2a3sim，并可能把 A5 专用宏传播到错误平台。pytest 会动态编译 +orchestration/incore kernel，因此 runner 的 map mode/profile 必须与本次 +runtime 构建完全一致。 + +每个阶段必须全部通过以下八组门禁： + +1. shared smoke 全量 A5Sim； +2. shared smoke 全量 A5 onboard； +3. shared PA Case1 A5Sim； +4. shared PA Case1 A5 onboard； +5. private smoke 全量 A5Sim； +6. private smoke 全量 A5 onboard； +7. private PA Case1 A5Sim； +8. private PA Case1 A5 onboard。 + +执行顺序： + +1. 隔离旧产物； +2. `PTO_FDWIC_SHARED_MAP=1`，shared 干净构建； +3. shared smoke A5Sim、smoke A5、PA A5Sim、PA A5； +4. 再次隔离 shared 构建产物； +5. `PTO_FDWIC_SHARED_MAP=0`，private 干净构建； +6. private 的同四组门禁； +7. 记录八组结果和两次构建日志。 + +`MaxBd` 使用当前系统支持的最大 block 数，不写死 36。任何一组失败时停止 +迁移，保留当前 commit 和构建现场分析；不得继续下一阶段，也不得用后续 +阶段结果覆盖当前失败。 + +每阶段报告必须记录： + +- simpler commit； +- 本阶段采用的来源 commit/代码片段； +- 是否发生冲突及采用的解法； +- GCC/G++ 版本； +- PTO ISA revision； +- map mode、DFX profile、phase ID、schema version、ELF hash； +- shared/private 是否分别完成全量干净重编译； +- 八组强制门禁的逐项 pass/fail； +- 本阶段新增或修改用例的逐项 pass/fail； +- DFX 阶段的 closure、dropped records、owner restore 和 observer + overhead 结果。 + +### 10.8 新增用例和对照 worktree + +每个阶段必须检查该阶段是否新增或修改测试： + +```bash +git diff --name-status HEAD -- \ + examples/ tests/ +``` + +所有新增/修改用例都是本阶段门禁的一部分。默认在 shared/private 两种干净 +构建状态下分别执行；只有源码或设计明确声明仅适用于一种模式时才能跳过 +另一种，并在报告中记录依据和风险。 + +先在当前集成分支直接验证。只有下列结果无法判定时才建立只读对照 +worktree： + +- 用例依赖来源 commit 当时的目录、脚本或环境，无法直接运行； +- 用例失败，但无法判断是 shared 适配问题还是来源 commit 自身限制； +- 用例所需硬件、工具链或外部条件当前不可用； +- 结果与来源文档矛盾，需要确认来源 commit 的真实基线。 + +普通编译失败、已经明确的功能失败或为了节省切换时间，不构成创建对照 +worktree 的理由。对照规则： + +1. 固定到本阶段采用的确切来源 commit，不能用来源分支最新 HEAD 代替； +2. 只读对照、编译和运行，不在其中修改或提交； +3. 使用独立 `.venv`、`build/cache`、`build/lib`，不复用集成目录产物； +4. 编译器、PTO ISA、平台、设备、参数和 pytest flags 与集成分支一致； +5. onboard 对照仍通过 A5 precheck 和 `task-submit`； +6. 记录来源/集成两个 commit 的结果后移除 worktree。 + +示例： + +```bash +SOURCE_COMMIT= +CONTROL_DIR=../simpler-source-control-${SOURCE_COMMIT:0:12} + +git worktree add --detach "${CONTROL_DIR}" "${SOURCE_COMMIT}" +cd "${CONTROL_DIR}" +python3 -m venv --system-site-packages .venv +``` + +完成对照后： + +```bash +cd /path/to/simpler +git worktree remove "${CONTROL_DIR}" +``` + +若来源 commit 同样失败或无法运行，报告必须给出用例、来源 commit、双方 +错误、编译器/PTO ISA/平台/命令、缺失条件和未覆盖风险。若来源通过而集成 +失败，视为迁移回归，立即停止下一阶段。 + +### 10.9 `atomic_probe` 的验证范围 + +Atomic/PMU 阶段根据下列文档确定定向验证： + +```text +tests/atomic_probe/pa_scheduler/PA-atomic情况分析.md +tests/atomic_probe/pa_scheduler/swimlane_opt_anal.md +tests/atomic_probe/icache_miss_usage_guide.md +tests/atomic_probe/perf_opt_record.md +tests/atomic_probe/a5_fdwic_atomic_swimlane_repo.md +``` + +`tests/atomic_probe/ascendc/`、`ccec/`、`cpu/` 及封装它们的 +`test_atomic_probe.py` 只验证硬件、编译器或环境 atomic/PMU 能力，不是 +runtime 迁移功能门禁： + +- 不纳入每阶段 runtime 门禁； +- 不要求在 private/shared 下重复执行； +- 只在 D0 资产接收检查、硬件能力复核或环境归因时运行； +- 历史 raw data、预编译 artifact 和 lazy-lambda A/B/C 样本只校验 + manifest/可读性，不重解释为当前分支性能结果。 + +每个 DFX 阶段明确区分： + +- atomic 语义和依赖正确性； +- PA scheduler 功能正确性； +- shared/private smoke 回归； +- A5Sim 功能； +- A5 onboard 功能； +- 仅在必要阶段执行的 performance/profiling。 + +### 10.10 C01 定向门禁 + +D1～D3 除八组功能门禁外至少验证： + +1. schema/ABI： + - v1、v3/v4 和 v5 parser 分流； + - record/core-state size 与 64-byte partition alignment； + - phase/site/op/result-used/poll-batchable 元数据穷举； + - map mode/profile/version mismatch 必须失败； +2. PollBatch： + - 单 site、多 site、成功 RMW、flush 边界、count overflow； + - `logical = direct rows + batch poll_count`； + - `dropped == 0` 才能作为完整 capture； +3. private： + - 来源 0～27 site 的 op 和控制流保持一致； + - startup、fatal、claim、fanin、completion、frontier、heap、 + final barrier、BlockWon 均能闭合； +4. shared： + - output publication/last-writer； + - shared heap reserve/wrap/vend； + - region lookup/sentinel/claim/publish/high-water； + - deps prepare/wait； + - joint expected/drained； + - 最终采用的 final barrier； +5. 静态分类： + - AICore 生产路径无未登记 atomic； + - AICPU init/owner 和明确 allowlist 不误计入； +6. observer effect： + - DFX-off ELF 无 C01 record/PollBatch 符号； + - phase-only 不产生 atomic rows； + - atomic profile 产生完整 rows； + - 比较 text size、关键函数 layout 和 PA 配对结果； + - 若 DFX-off 性能/布局发生无法解释的变化，阶段失败。 + +A5Sim 负责功能和 schema closure；`return-ready`、真实 atomic contention、 +I-cache 和物理时序结论必须来自 A5 onboard。 + +### 10.11 C03 定向门禁 + +D4～D7 除八组功能门禁外至少验证： + +1. profile 互斥： + - trace/atomic、perf-clock、submit-PMU 不可错误共存； + - 每个 phase ELF 只包含一个 selected phase； + - build manifest 与 ELF marker 一致； +2. perf-clock： + - 每核首个 begin、末个 end 都存在且有序； + - observed submit count 等于 expected； + - linked-kernel calls/ticks 不超过整窗，residual 非负； + - shared winner、loser、wrong-role、alloc non-candidate 均关闭窗口； +3. PMU owner： + - 按实际 topology 配置，不写死 96； + - Bd1、Bd36、MaxBd 的 physical IDs/role/triplet 闭合； + - selector readback 正确； + - configured/restored 数相等，active-after-restore 为 0； + - 配置失败注入会触发 abort，worker 不开始半配置采集； +4. counter： + - I-cache miss 不大于 request； + - vector/cube/scalar/total 的关系满足所选窗口定义； + - counter 未达到 wrap 风险阈值； + - phase sidecar 可重建且位于 Submit 整窗内； + - linked-kernel pause/resume 和 atomic return-ready 扣减全部闭合； +5. shared phase： + - 逐一触发 RoleFilter、winner/loser、heap/output、region、deps、 + resolve、joint 和 final drain； + - 每个开始边界都有结束边界； + - 动态 winner/loser/role 数来自 runtime manifest/counters； + - PA 特有“五次 submit/batch”公式只在 PA report 层验证； +6. 结果身份： + - simpler/PTO ISA commit、map mode、profile、phase、schema、ELF hash、 + workload、block dimension、topology 完整； + - atomic 与 PMU 捕获只有 identity/workload 匹配时才能关联； +7. observer effect： + - `off`、`perf-clock`、`submit-pmu-none`、单 phase 分别配对采集； + - 性能结论单独记录轮数、设备和统计方法； + - 不以 profiling 数值代替功能 golden。 + +### 10.12 编译器和 PTO ISA 固定结论 + +A5Sim PA Case1 已有环境结论： + +- GCC 13 在 shared/private 和多种优化配置下稳定出现约 `1.9` 的 golden + mismatch； +- 真正 GCC/G++ 15 下，相同历史代码功能用例能够通过； +- 因此 GCC 13 的 PA A5Sim 失败作为独立环境问题处理，不归因于迁移，但也 + 不能用它替代阶段门禁。 + +A5Sim 功能基线使用真正 GCC/G++ 15。PTO ISA 固定为： + +```text +ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8 +``` + +运行前确认 checkout HEAD 精确匹配且工作区干净。 + +### 10.13 最终收敛 + +全部已确认阶段完成后： + +1. 检查最终提交顺序和每个提交的变更范围； +2. 确认未纳入临时调试、环境配置、缓存和本地验证脚本； +3. 完成 private/shared 全量 smoke； +4. 完成 PA Case1 A5Sim 与 A5 onboard； +5. 完成每个 DFX 阶段对应的 `tests/atomic_probe` 定向验证； +6. 单独报告性能环境、参数和结果，不把它当成功能结论； +7. 汇总未覆盖风险、已知限制及 CI/其他硬件待补验证。 + +### 10.14 恢复任务时的检查点 + +每次恢复迁移先执行： + +```bash +git status --short +git branch --show-current +git log --oneline --decorate -n 20 +``` + +然后确认： + +1. 当前位于约定的 integration 分支； +2. 没有未完成的 cherry-pick/rebase/冲突； +3. 本文档、本地验证脚本和环境配置没有混入产品提交； +4. 已重新计算尚未完成的阶段和确切来源 commit； +5. shared/private/DFX profile 与现存构建产物一致；不一致则先干净重建； +6. 从下一个未完成阶段继续，遇到冲突或门禁失败立即停止并记录分析。 diff --git a/tests/atomic_probe/pa_scheduler/swimlane_converter.py b/tests/atomic_probe/pa_scheduler/swimlane_converter.py new file mode 100755 index 0000000000..437638328d --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/swimlane_converter.py @@ -0,0 +1,2220 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""把 standalone PA 调度器的原始 FDWIC 记录转换为 Perfetto 泳道。 + +脚本只使用 Python 标准库和调用者给出的本地 JSON，不 import ``simpler_setup`` +或仓库外模块。输出遵循 Chrome Trace Event 格式，可直接载入 Perfetto。 +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from collections.abc import Iterator +from pathlib import Path +from typing import Any, TextIO + +# 与真实 PA swimlane converter 使用同一套阶段命名。这里保留 ringbp、 +# efdrain 等既有拼写，避免同一阶段在两类泳道中被 Perfetto 分成不同名称。 +PHASE_NAMES = { + "Kernel": "kernel", + "Alloc": "alloc", + "Build": "build", + "DrainWon": "drain_won", + "Replay": "replay", + "RingBp": "ringbp", + "EfDrain": "efdrain", + "Commit": "commit", + "Submit": "submit", + "Materialize": "materialize", + "PrepareMap": "prepare_map", + "Claim": "claim", + "Fanin": "fanin", + "Register": "register", + "Atomic": "atomic", + "Dcci": "dcci", + "ClockBaseline": "clock_baseline", + "OrchestrationReplay": "orchestration_replay", + "FinalDrain": "final_drain", + "WinnerBuild": "winner_build", + "AllocComplete": "alloc_complete", + "SharedRegisterPublishMetadata": "register.publish_metadata", + "SharedMaterializePublishTaskOutputs": ( + "materialize.publish_shared_output_descriptors" + ), + "SharedMaterializePublishTaskOutputsCopy": ( + "materialize.publish_shared_output_descriptors.copy_tensor_descs" + ), + "SharedMaterializePublishTaskOutputsFlush": ( + "materialize.publish_shared_output_descriptors.flush_tensor_descs" + ), + # 兼容迁移前已经落盘的 schema-v5 raw；新采集只会写上面的 + # SharedMaterialize* 名称，旧名称仍按其当时的 Register 归属解释。 + "SharedRegisterPublishTaskOutputs": "register.publish_task_outputs", + "SharedRegisterPublishTaskOutputsCopy": ( + "register.publish_task_outputs.copy" + ), + "SharedRegisterPublishTaskOutputsFlush": ( + "register.publish_task_outputs.flush" + ), +} +LEGACY_LAP_PHASES = {"Alloc", "Build", "Replay"} +V5_PHASES = { + "OrchestrationReplay", + "FinalDrain", + "WinnerBuild", + "AllocComplete", + "SharedRegisterPublishMetadata", + "SharedMaterializePublishTaskOutputs", + "SharedMaterializePublishTaskOutputsCopy", + "SharedMaterializePublishTaskOutputsFlush", + "SharedRegisterPublishTaskOutputs", + "SharedRegisterPublishTaskOutputsCopy", + "SharedRegisterPublishTaskOutputsFlush", + "Dcci", +} +# schema-v5 已有区间足以在离线侧取补集；这些 phase 之外的 +# Atomic、Kernel、RingBp 等是嵌套或 Overlay，不能再从 Submit 扣一次。 +V5_EXCLUSIVE_SUBMIT_PHASES = { + "EfDrain", + "Materialize", + "PrepareMap", + "Claim", + "Fanin", + "Register", + "WinnerBuild", + "AllocComplete", +} +# TaskKind/function ABI 固定为五种“类型”，但 shared TensorMap 每个 batch +# 可以有 0..4 组 QK/SF/PV/UP，运行时 task 数不再固定为五个。 +TASK_KIND_NAMES = ("Alloc", "QK", "SF", "PV", "UP") +KERNEL_NAMES = { + function_id: task_kind + for function_id, task_kind in enumerate(TASK_KIND_NAMES[1:]) +} +# 一个物理 mixed block 的三条 runtime lane：AIC、AIV0、AIV1。 +LANE_NAMES = {0: "AIC", 1: "AIV0", 2: "AIV1"} + +# Atomic raw ABI：auxiliary 存放调用点，flags 低 4 位存放操作类型。这里的 +# 数值必须与 standalone C++ AtomicSite/AtomicOp 枚举保持一致；未知值仍会 +# 以 site_/op_ 完整导出，便于识别版本不匹配，不会伪装成已知操作。 +ATOMIC_SITE_NAMES = { + 0: "startup_increment", + 1: "startup_poll", + 2: "fatal_poll", + 3: "fatal_set", + 4: "claim_max", + 5: "fanin_flag_load", + 6: "completion_vend_exchange", + 7: "completion_flag_exchange", + 8: "frontier_initial_load", + 9: "frontier_flag_load", + 10: "frontier_max", + 11: "heap_frontier_load", + 12: "heap_vend_load", + 13: "replay_done_increment", + 14: "replay_done_poll", + 15: "shared_heap_vend_load", + 16: "shared_heap_cursor_load", + 17: "shared_heap_cursor_reserve", + 18: "shared_heap_vend_advance", + 19: "shared_insert_predecessor_poll", + 20: "shared_insert_completion_publish", + 21: "shared_winner_fatal_guard_load", + 22: "shared_metadata_fatal_guard_load", + 23: "shared_output_ref_fanin_output_published_load", + 24: "shared_output_ref_metadata_output_published_load", + 25: "shared_output_ref_fanin_last_writer_load", + 26: "shared_output_ref_metadata_last_writer_load", + 27: "shared_output_ref_last_writer_commit", + 28: "shared_output_writer_reserve", + 29: "shared_output_published_exchange", + 30: "shared_tensormap_lookup_head_load", + 31: "shared_tensormap_lookup_tail_load", + 32: "shared_tensormap_lookup_seq_load", + 33: "shared_tensormap_append_head_load", + 34: "shared_tensormap_append_tail_load", + 35: "shared_tensormap_append_seq_load", + 36: "shared_tensormap_append_seq_reset_exchange", + 37: "shared_tensormap_append_seq_publish_exchange", + 38: "shared_tensormap_append_tail_exchange", + 39: "shared_output_rollback_exchange", +} +ATOMIC_OP_NAMES = { + 0: "load", + 1: "exchange", + 2: "fetch_add", + 3: "fetch_max", + 4: "compare_exchange", +} + +# schema-v3/4 的校验表必须与 standalone C++ 的稳定 AtomicSite 编号一致。 +# 0..14 是既有 common/private 站点，15..18 是 shared heap，19/20 是 +# shared Register 插入轮次的等待 Load 与交接 CAS；真实 PA 的 BlockWon +# 不属于本用例，不能为了兼容生产 converter 凭空放宽。 +ATOMIC_SITE_OP_IDS = { + 0: 2, + 1: 0, + 2: 0, + 3: 1, + 4: 3, + 5: 0, + 6: 1, + 7: 1, + 8: 0, + 9: 0, + 10: 3, + 11: 0, + 12: 0, + 13: 2, + 14: 0, + 15: 0, + 16: 0, + 17: 2, + 18: 2, + 19: 0, + 20: 4, + 21: 0, + 22: 0, + 23: 0, + 24: 0, + 25: 0, + 26: 0, + 27: 4, + 28: 3, + 29: 1, + 30: 0, + 31: 0, + 32: 0, + 33: 0, + 34: 0, + 35: 0, + 36: 1, + 37: 1, + 38: 1, + 39: 1, +} +# 这些发布型调用不消费 atomic 返回的旧值；其余 standalone site 的 +# 返回值都参与协议判断。v3 输入必须与源码语义完全一致。 +ATOMIC_RESULT_UNUSED_SITE_IDS = {0, 3, 6, 7, 13, 39} +# common/private 的六类等待 Load 与 shared Register insert-turn Load 可以 +# 合并；frontier 扫描和 Claim 即使调用很多次也必须继续保留逐调用记录。 +POLL_BATCH_SITE_OP_IDS = { + 1: 0, + 2: 0, + 5: 0, + 11: 0, + 12: 0, + 14: 0, + 19: 0, +} +SHARED_REGISTER_ATOMIC_SITE_IDS = {19, 20} +SCHEMA_V5_SHARED_ATOMIC_SITE_IDS = set(range(19, 40)) +SHARED_INSERT_TURN_POLL_SITE_ID = 19 +SHARED_INSERT_TURN_HANDOFF_SITE_ID = 20 + +ATOMIC_RESULT_USED = 1 << 4 +ATOMIC_VALUE_ZERO = 1 << 5 +ATOMIC_RETURN_READY = 1 << 6 +ATOMIC_POLL_BATCH = 1 << 7 +ATOMIC_PAYLOAD_SHIFT = 8 +ATOMIC_PAYLOAD_MASK = 0xFFFFFF + +# DCCI raw ABI 与 Atomic 独立复用 flags/aux。一次区域原语只生成一条 +# 记录；observer 最终导出把 records/core 两次 clean 聚合成一条 terminal +# 记录，因此 call_count 与物理 row 数不能混为一谈。 +DCCI_SITE_NAMES = { + 0: "shared_output_ref_fanin_history_invalidate", + 1: "shared_output_ref_writer_history_flush", + 2: "shared_output_rollback_flush", + 3: "shared_output_descriptor_flush", + 4: "shared_region_read_invalidate", + 5: "shared_region_append_invalidate", + 6: "shared_region_append_flush", + 7: "shared_winner_build_descriptor_invalidate", + 8: "observer_trace_export", + 9: "startup_config_invalidate", +} +DCCI_OP_NAMES = { + 0: "invalidate", + 1: "clean_out", +} +DCCI_SITE_OP_IDS = { + 0: 0, + 1: 1, + 2: 1, + 3: 1, + 4: 0, + 5: 0, + 6: 1, + 7: 0, + 8: 1, + 9: 0, +} +DCCI_SHARED_ONLY_SITE_IDS = set(range(8)) +DCCI_OBSERVER_SITE_ID = 8 +DCCI_STARTUP_SITE_ID = 9 +DCCI_OP_MASK = 0x3 +DCCI_TRAILING_DSB = 1 << 2 +DCCI_CALL_COUNT_SHIFT = 3 +DCCI_CALL_COUNT_MASK = 0xF +DCCI_RESERVED_BIT = 1 << 7 +DCCI_LINE_COUNT_SHIFT = 8 +DCCI_LINE_COUNT_MASK = 0xFFFFFF + + +# schema-v3 只由本目录的 standalone producer 生成；其 worker 编号与 +# 32 AIC + 64 AIV 的 mixed-block 映射是 raw ABI 的一部分，converter 不再 +# 只检查“同一 block/lane 不重复”这个弱条件。 +def _standalone_topology(core_id: int) -> tuple[int, int, str]: + if core_id < 32: + return core_id, 0, "aic" + vector_id = core_id - 32 + return vector_id // 2, 1 + vector_id % 2, "aiv" + + +# 把可转为整数的 raw 标量归一为 int，并在错误中保留精确字段路径。 +def _integer(value: Any, label: str) -> int: + # 这是兼容 JSON 数值/数值字符串的宽松归一，不负责强制原始 JSON 类型必须为 int。 + try: + return int(value) + except (TypeError, ValueError) as error: + raise ValueError(f"{label} is not an integer: {value!r}") from error + + +def _derive_v4_task_kinds( + submit_semantics: dict[tuple[int, int], tuple[bool, bool]], + num_cores: int, +) -> dict[int, int]: + """从每核 Submit 的既有 Alloc 标记恢复动态 task 类型流。 + + schema-v5 的 ``Submit.auxiliary`` 已逐核记录 ``is_alloc``，因此无需给 + 设备 raw 再增加 task-kind 字段。这里先要求 96 核（或测试给定核数） + 具有完全一致的连续 task 流，再按相邻 Alloc 边界验证每个 batch 必须是 + ``Alloc + 0..4 × (QK,SF,PV,UP)``。返回值使用稳定 TaskKind 编号： + Alloc=0，QK/SF/PV/UP=1..4。 + """ + + if num_cores <= 0: + raise ValueError(f"schema-v5 task plan requires positive num_cores, got {num_cores}") + + task_ids_by_core: dict[int, list[int]] = { + core_id: [] for core_id in range(num_cores) + } + for (core_id, task_id) in submit_semantics: + if core_id not in task_ids_by_core: + raise ValueError( + f"schema-v5 Submit task plan has out-of-range core {core_id}" + ) + if task_id < 0: + raise ValueError( + f"schema-v5 Submit task plan has negative task_id {task_id}" + ) + task_ids_by_core[core_id].append(task_id) + + reference_task_ids: list[int] | None = None + for core_id in range(num_cores): + task_ids = sorted(task_ids_by_core[core_id]) + if reference_task_ids is None: + if not task_ids or task_ids != list(range(task_ids[-1] + 1)): + raise ValueError( + "schema-v5 Submit task IDs must be contiguous 0..N-1 on every " + f"core: core={core_id} task_ids={task_ids}" + ) + reference_task_ids = task_ids + elif task_ids != reference_task_ids: + raise ValueError( + "schema-v5 Submit task IDs differ across cores: " + f"core={core_id} task_ids={task_ids}" + ) + + assert reference_task_ids is not None + alloc_by_task: dict[int, bool] = {} + for task_id in reference_task_ids: + markers = { + submit_semantics[(core_id, task_id)][1] + for core_id in range(num_cores) + } + if len(markers) != 1: + raise ValueError( + "schema-v5 Submit Alloc marker differs across cores for " + f"task {task_id}" + ) + alloc_by_task[task_id] = markers.pop() + + alloc_task_ids = [ + task_id for task_id in reference_task_ids if alloc_by_task[task_id] + ] + if not alloc_task_ids or alloc_task_ids[0] != 0: + raise ValueError("schema-v5 dynamic task plan must begin with task 0 Alloc") + + task_kind_by_id: dict[int, int] = {} + interval_ends = [*alloc_task_ids[1:], len(reference_task_ids)] + for alloc_task_id, interval_end in zip(alloc_task_ids, interval_ends): + interval_length = interval_end - alloc_task_id + payload_tasks = interval_length - 1 + if payload_tasks % 4 != 0 or not 0 <= payload_tasks // 4 <= 4: + raise ValueError( + "schema-v5 dynamic batch must contain Alloc plus 0..4 complete " + "QK/SF/PV/UP groups: " + f"alloc_task={alloc_task_id} interval_length={interval_length}" + ) + task_kind_by_id[alloc_task_id] = 0 + for offset in range(1, interval_length): + task_kind_by_id[alloc_task_id + offset] = 1 + (offset - 1) % 4 + + if set(task_kind_by_id) != set(reference_task_ids): + raise AssertionError("schema-v5 dynamic task plan derivation is incomplete") + return task_kind_by_id + + +# 读取 raw JSON，校验十列结构、字段范围与可转整数值，并返回规范化视图。 +def _load_and_validate( + input_path: Path, +) -> tuple[int, int, list[tuple[Any, ...]], dict[tuple[int, int], int], int, dict[str, Any]]: + # raw 文件沿用真实 l2_swimlane_records.json 的十列 fdwic_events ABI： + # core、block、lane、task、func、phase、start、end、flags、aux。 + with input_path.open("r", encoding="utf-8") as input_file: + data = json.load(input_file) + if not isinstance(data, dict): + raise ValueError("capture root must be a JSON object") + + # 先验证顶层 schema 和时钟元数据；时钟频率是 cycle 转时间的唯一依据， + # 不允许由 converter 根据平台名称猜测。 + level = _integer(data.get("l2_swimlane_level"), "l2_swimlane_level") + if level not in (1, 2, 3, 4): + raise ValueError(f"unsupported l2_swimlane_level: {level}") + metadata = data.get("metadata") + if not isinstance(metadata, dict): + raise ValueError("metadata must be a JSON object") + frequency_hz = _integer(metadata.get("clock_freq_hz"), "metadata.clock_freq_hz") + if frequency_hz <= 0: + raise ValueError("metadata.clock_freq_hz must be positive") + # v1 是旧 raw，Claim flags 只有 winner bit；v2 追加 attempted bit； + # v3 再加入精确计数 PollBatch；v5 追加排他父区间、真实尾动作 span + # 以及 Materialize→task outputs 与 Register→metadata detail。迁移前 + # 已落盘的 v5 Register→metadata→task outputs 仍只读兼容。v4 raw 不再接受， + # 避免缺少 task-outputs 边界的旧采集被伪装成新细分。 + # 不认识的新版本直接拒绝，避免把新 flags 按旧语义误读。 + trace_schema_version = _integer(metadata.get("trace_schema_version", 1), "metadata.trace_schema_version") + if trace_schema_version not in (1, 2, 3, 5): + raise ValueError(f"unsupported metadata.trace_schema_version: {trace_schema_version}") + if trace_schema_version == 3 and level != 4: + raise ValueError("metadata.trace_schema_version=3 requires l2_swimlane_level=4") + if trace_schema_version == 5 and level not in (1, 4): + raise ValueError( + "metadata.trace_schema_version=5 requires l2_swimlane_level=1 or 4" + ) + tensormap_mode = metadata.get("tensormap_mode") + if trace_schema_version == 5: + if tensormap_mode not in ("private", "shared"): + raise ValueError( + "metadata.tensormap_mode must be private or shared for " + "trace_schema_version=5" + ) + elif tensormap_mode is not None: + raise ValueError( + "metadata.tensormap_mode is only valid for trace_schema_version=5" + ) + num_cores = _integer(metadata.get("num_cores"), "metadata.num_cores") + if num_cores <= 0: + raise ValueError("metadata.num_cores must be positive") + core_types = metadata.get("core_types") + if not isinstance(core_types, list) or len(core_types) != num_cores: + raise ValueError("metadata.core_types length must equal metadata.num_cores") + if trace_schema_version >= 3: + if num_cores > 96: + raise ValueError("schema-v3+ standalone metadata.num_cores must not exceed 96") + for core_id, core_type in enumerate(core_types): + expected_type = _standalone_topology(core_id)[2] + if core_type != expected_type: + raise ValueError( + f"metadata.core_types[{core_id}]={core_type!r} does not match " + f"standalone topology {expected_type!r}" + ) + winner_workload = metadata.get("winner_workload") + if winner_workload is not None: + if not isinstance(winner_workload, dict): + raise ValueError("metadata.winner_workload must be a JSON object") + workload_mode = winner_workload.get("mode") + if workload_mode not in ("scalar-nop", "real-compute"): + raise ValueError("metadata.winner_workload.mode must be scalar-nop or real-compute") + workload_counts = winner_workload.get("counts") + if not isinstance(workload_counts, dict): + raise ValueError("metadata.winner_workload.counts must be a JSON object") + normalized_counts: dict[str, int] = {} + for kind in ("qk", "sf", "pv", "up"): + value = _integer( + workload_counts.get(kind), f"metadata.winner_workload.counts.{kind}" + ) + if value < 0 or (workload_mode == "real-compute" and value == 0): + raise ValueError( + f"metadata.winner_workload.counts.{kind} is invalid for {workload_mode}" + ) + normalized_counts[kind] = value + expected_unit = ( + "complete_128x128_engine_pipeline_iteration" + if workload_mode == "real-compute" + else "scalar_nop_instruction" + ) + if winner_workload.get("unit") != expected_unit: + raise ValueError( + f"metadata.winner_workload.unit must be {expected_unit!r} for {workload_mode}" + ) + # input_pattern 是 real-compute 布局诊断新增的可选元数据。旧 schema-v2 + # 文件没有该字段，仍保持可读；新采集若给出则必须与 workload 模式一致。 + input_pattern = winner_workload.get("input_pattern") + if input_pattern is not None: + valid_patterns = ( + {"constant", "layout-diagnostic"} + if workload_mode == "real-compute" + else {"none"} + ) + if input_pattern not in valid_patterns: + raise ValueError( + "metadata.winner_workload.input_pattern is invalid for " + f"{workload_mode}" + ) + engine_mapping = winner_workload.get("engine_mapping") + if workload_mode == "real-compute": + expected_mapping = { + "qk": "cube_matmul", + "sf": "vector_add", + "pv": "cube_matmul", + "up": "vector_mul", + } + if engine_mapping != expected_mapping: + raise ValueError("metadata.winner_workload.engine_mapping is invalid") + elif engine_mapping is not None: + raise ValueError("scalar-nop metadata.winner_workload.engine_mapping must be null") + # 后续 merged 顶层与 instant event 使用经过整数归一的同一份配置。 + winner_workload["counts"] = normalized_counts + + rows = data.get("fdwic_events") + if not isinstance(rows, list) or not rows: + raise ValueError("fdwic_events must be a non-empty array") + + # Perfetto metadata 需要从 (block, lane) 找回稳定的 core 编号；同一 lane + # 若在 raw 中映射到两个 core，说明采集已损坏，不能继续生成误导性泳道。 + core_by_block_lane: dict[tuple[int, int], int] = {} + base_cycle: int | None = None + observed_summary = { + "records": len(rows), + "atomic_records": 0, + "clock_baseline_records": 0, + "atomic_calls": 0, + "batched_poll_calls": 0, + "poll_batch_records": 0, + "dcci_records": 0, + "dcci_calls": 0, + "dcci_lines": 0, + # dropped 无法从已经导出的有效行反推；v3+ 必须由 producer summary + # 明确承诺为零，下面再逐字段核对。 + "dropped_records": 0, + } + v3_clock_rows: dict[int, dict[str, int | bool | None]] = { + core_id: {"plain": 0, "dependency": 0, "return_ready": None} + for core_id in range(num_cores) + } + v3_result_used_direct_rows: list[tuple[int, int, bool]] = [] + v3_insert_turn_poll_batch_rows: list[tuple[int, int, bool]] = [] + v5_observer_dcci_rows = {core_id: 0 for core_id in range(num_cores)} + v4_parent_counts: dict[int, dict[str, int]] = { + core_id: {"OrchestrationReplay": 0, "FinalDrain": 0} + for core_id in range(num_cores) + } + v4_claims: dict[tuple[int, int], tuple[bool, bool, bool]] = {} + v4_submits: set[tuple[int, int]] = set() + v4_submit_semantics: dict[tuple[int, int], tuple[bool, bool]] = {} + v4_tails: dict[tuple[int, int], tuple[str, int]] = {} + v4_materializes: dict[ + tuple[int, int], list[tuple[Any, ...]] + ] = {} + v4_registers: dict[tuple[int, int], list[tuple[Any, ...]]] = {} + v4_shared_register_details: dict[ + tuple[int, int], list[tuple[Any, ...]] + ] = {} + v4_shared_register_output_details: dict[ + tuple[int, int], list[tuple[Any, ...]] + ] = {} + v4_shared_register_output_copy_details: dict[ + tuple[int, int], list[tuple[Any, ...]] + ] = {} + v4_shared_register_output_flush_details: dict[ + tuple[int, int], list[tuple[Any, ...]] + ] = {} + v4_shared_insert_turn_polls: list[tuple[Any, ...]] = [] + v4_shared_insert_turn_handoffs: dict[ + tuple[int, int], list[tuple[Any, ...]] + ] = {} + # 逐行在写输出前检查列数、范围和可转整数的字段。任一行不满足这些约束 + # 都会整体拒绝输入，不生成缺少关键阶段的“部分可看”泳道。 + for index, row in enumerate(rows): + if not isinstance(row, (list, tuple)) or len(row) != 10: + raise ValueError(f"fdwic_events[{index}] must contain exactly 10 fields") + core_id = _integer(row[0], f"fdwic_events[{index}].core_id") + block_id = _integer(row[1], f"fdwic_events[{index}].block_id") + lane = _integer(row[2], f"fdwic_events[{index}].lane") + task_id = _integer(row[3], f"fdwic_events[{index}].task_id") + function_id = _integer(row[4], f"fdwic_events[{index}].function_id") + phase = str(row[5]) + start_cycle = _integer(row[6], f"fdwic_events[{index}].start_cycle") + end_cycle = _integer(row[7], f"fdwic_events[{index}].end_cycle") + flags = _integer(row[8], f"fdwic_events[{index}].flags") + auxiliary = _integer(row[9], f"fdwic_events[{index}].auxiliary") + if not 0 <= core_id < num_cores: + raise ValueError(f"fdwic_events[{index}] has out-of-range core_id {core_id}") + if block_id < 0: + raise ValueError(f"fdwic_events[{index}] has negative block_id {block_id}") + if lane not in LANE_NAMES: + raise ValueError(f"fdwic_events[{index}] has invalid lane {lane}") + if phase not in PHASE_NAMES: + raise ValueError(f"fdwic_events[{index}] has unknown phase {phase!r}") + if trace_schema_version == 5 and phase in LEGACY_LAP_PHASES: + raise ValueError( + f"fdwic_events[{index}] schema-v5 forbids legacy lap phase {phase!r}" + ) + if trace_schema_version == 5 and phase == "DrainWon": + raise ValueError( + f"fdwic_events[{index}] schema-v5 forbids unused legacy phase 'DrainWon'" + ) + if trace_schema_version < 5 and phase in V5_PHASES: + raise ValueError( + f"fdwic_events[{index}] phase {phase!r} requires trace_schema_version=5" + ) + if trace_schema_version >= 3: + if task_id < -1 or function_id < -1 or auxiliary < 0: + raise ValueError( + f"fdwic_events[{index}] has invalid v3+ base fields: " + f"task={task_id} func={function_id} aux={auxiliary}" + ) + if not 0 <= flags <= 0xFFFFFFFF: + raise ValueError( + f"fdwic_events[{index}] has invalid uint32 flags {flags}" + ) + expected_block, expected_lane, _ = _standalone_topology(core_id) + if block_id != expected_block or lane != expected_lane: + raise ValueError( + f"fdwic_events[{index}] block/lane={block_id}/{lane} does not match " + f"standalone topology {expected_block}/{expected_lane} for core {core_id}" + ) + if phase == "Claim" and trace_schema_version >= 2: + if flags & ~0x3 or (flags & 0x1 and not flags & 0x2): + raise ValueError( + f"fdwic_events[{index}] has invalid Claim flags 0x{flags:x}" + ) + if trace_schema_version >= 3 and auxiliary > 1: + raise ValueError( + f"fdwic_events[{index}] has invalid Claim auxiliary {auxiliary}" + ) + if phase == "Atomic": + poll_batch = bool(flags & ATOMIC_POLL_BATCH) + atomic_op = flags & 0xF + result_used = bool(flags & ATOMIC_RESULT_USED) + value_zero = bool(flags & ATOMIC_VALUE_ZERO) + return_ready = bool(flags & ATOMIC_RETURN_READY) + payload = (flags >> ATOMIC_PAYLOAD_SHIFT) & ATOMIC_PAYLOAD_MASK + if auxiliary in SCHEMA_V5_SHARED_ATOMIC_SITE_IDS and not ( + trace_schema_version == 5 and tensormap_mode == "shared" + ): + raise ValueError( + f"fdwic_events[{index}] has invalid direct Atomic " + f"site={auxiliary}: shared schema-v5 site requires " + "shared schema-v5" + ) + if ( + auxiliary == SHARED_INSERT_TURN_POLL_SITE_ID + and not poll_batch + ): + raise ValueError( + f"fdwic_events[{index}] SharedInsertTurnPoll must use PollBatch" + ) + if poll_batch: + return_ready_valid = ( + not return_ready + or auxiliary == SHARED_INSERT_TURN_POLL_SITE_ID + ) + if ( + trace_schema_version not in (3, 5) + or level != 4 + or payload == 0 + or POLL_BATCH_SITE_OP_IDS.get(auxiliary) != atomic_op + or not result_used + or value_zero + or not return_ready_valid + or task_id != -1 + or function_id != -1 + ): + raise ValueError( + f"fdwic_events[{index}] has invalid Atomic PollBatch " + f"site={auxiliary} flags=0x{flags:x}" + ) + if auxiliary == SHARED_INSERT_TURN_POLL_SITE_ID: + v3_insert_turn_poll_batch_rows.append( + (index, core_id, return_ready) + ) + elif trace_schema_version in (3, 5): + if level != 4: + raise ValueError( + f"fdwic_events[{index}] Atomic requires l2_swimlane_level=4" + ) + expected_result_used = ( + auxiliary in ATOMIC_SITE_OP_IDS + and auxiliary not in ATOMIC_RESULT_UNUSED_SITE_IDS + ) + if ( + ATOMIC_SITE_OP_IDS.get(auxiliary) != atomic_op + or result_used != expected_result_used + or (return_ready and not result_used) + or (value_zero and atomic_op != 0) + or (payload and atomic_op != 3) + or function_id != -1 + or ( + auxiliary == SHARED_INSERT_TURN_HANDOFF_SITE_ID + and task_id < 0 + ) + ): + raise ValueError( + f"fdwic_events[{index}] has invalid direct Atomic " + f"site={auxiliary} flags=0x{flags:x}" + ) + if result_used: + v3_result_used_direct_rows.append((index, core_id, return_ready)) + observed_summary["atomic_records"] += 1 + if poll_batch: + observed_summary["atomic_calls"] += payload + observed_summary["batched_poll_calls"] += payload + observed_summary["poll_batch_records"] += 1 + else: + observed_summary["atomic_calls"] += 1 + if trace_schema_version == 5 and auxiliary in ( + SHARED_INSERT_TURN_POLL_SITE_ID, + SHARED_INSERT_TURN_HANDOFF_SITE_ID, + ): + atomic_record = ( + core_id, + block_id, + lane, + task_id, + function_id, + phase, + start_cycle, + end_cycle, + flags, + auxiliary, + ) + if auxiliary == SHARED_INSERT_TURN_POLL_SITE_ID: + v4_shared_insert_turn_polls.append(atomic_record) + else: + v4_shared_insert_turn_handoffs.setdefault( + (core_id, task_id), [] + ).append(atomic_record) + elif phase == "ClockBaseline": + observed_summary["clock_baseline_records"] += 1 + if trace_schema_version in (3, 5): + if level != 4: + raise ValueError( + f"fdwic_events[{index}] ClockBaseline requires l2_swimlane_level=4" + ) + dependency = bool(flags & 0x1) + dependency_applied = bool(flags & 0x2) + if ( + flags & ~0x3 + or (dependency_applied and not dependency) + or task_id != -1 + or function_id != -1 + or auxiliary != 0 + ): + raise ValueError( + f"fdwic_events[{index}] has invalid ClockBaseline " + f"flags=0x{flags:x} auxiliary={auxiliary}" + ) + clock_state = v3_clock_rows[core_id] + if dependency: + clock_state["dependency"] = int(clock_state["dependency"]) + 1 + clock_state["return_ready"] = dependency_applied + else: + clock_state["plain"] = int(clock_state["plain"]) + 1 + elif phase == "Dcci": + if trace_schema_version != 5: + raise ValueError( + f"fdwic_events[{index}] Dcci requires trace_schema_version=5" + ) + op_id = flags & DCCI_OP_MASK + call_count = ( + flags >> DCCI_CALL_COUNT_SHIFT + ) & DCCI_CALL_COUNT_MASK + line_count = ( + flags >> DCCI_LINE_COUNT_SHIFT + ) & DCCI_LINE_COUNT_MASK + if ( + auxiliary not in DCCI_SITE_OP_IDS + or DCCI_SITE_OP_IDS[auxiliary] != op_id + or op_id not in DCCI_OP_NAMES + or flags & DCCI_RESERVED_BIT + or not flags & DCCI_TRAILING_DSB + or call_count == 0 + or line_count < call_count + ): + raise ValueError( + f"fdwic_events[{index}] has invalid Dcci " + f"site={auxiliary} flags=0x{flags:x}" + ) + if auxiliary == DCCI_OBSERVER_SITE_ID: + expected_observer_calls = ( + 3 if tensormap_mode == "shared" else 2 + ) + if ( + call_count != expected_observer_calls + or task_id != -1 + or function_id != -1 + ): + raise ValueError( + f"fdwic_events[{index}] has invalid observer Dcci fields" + ) + v5_observer_dcci_rows[core_id] += 1 + elif auxiliary == DCCI_STARTUP_SITE_ID: + if call_count != 1 or task_id != -1 or function_id != -1: + raise ValueError( + f"fdwic_events[{index}] has invalid startup Dcci fields" + ) + elif ( + tensormap_mode != "shared" + or auxiliary not in DCCI_SHARED_ONLY_SITE_IDS + or call_count != 1 + or task_id < 0 + ): + raise ValueError( + f"fdwic_events[{index}] has invalid shared Dcci fields" + ) + observed_summary["dcci_records"] += 1 + observed_summary["dcci_calls"] += call_count + observed_summary["dcci_lines"] += line_count + if trace_schema_version == 5: + task_key = (core_id, task_id) + if tensormap_mode == "shared" and phase == "PrepareMap": + raise ValueError( + f"fdwic_events[{index}] shared schema-v5 must not contain PrepareMap" + ) + if phase == "SharedRegisterPublishMetadata": + if tensormap_mode != "shared": + raise ValueError( + f"fdwic_events[{index}] SharedRegisterPublishMetadata " + "is only valid for shared TensorMap" + ) + if task_id < 0 or flags != 0 or auxiliary != 0: + raise ValueError( + f"fdwic_events[{index}] has invalid " + "SharedRegisterPublishMetadata fields" + ) + v4_shared_register_details.setdefault(task_key, []).append( + ( + core_id, + block_id, + lane, + task_id, + function_id, + phase, + start_cycle, + end_cycle, + flags, + auxiliary, + ) + ) + elif phase in ( + "SharedMaterializePublishTaskOutputs", + "SharedRegisterPublishTaskOutputs", + ): + if tensormap_mode != "shared": + raise ValueError( + f"fdwic_events[{index}] {phase} " + "is only valid for shared TensorMap" + ) + if task_id < 0 or flags != 0 or auxiliary != 0: + raise ValueError( + f"fdwic_events[{index}] has invalid {phase} fields" + ) + v4_shared_register_output_details.setdefault( + task_key, [] + ).append( + ( + core_id, + block_id, + lane, + task_id, + function_id, + phase, + start_cycle, + end_cycle, + flags, + auxiliary, + ) + ) + elif phase in ( + "SharedMaterializePublishTaskOutputsCopy", + "SharedMaterializePublishTaskOutputsFlush", + "SharedRegisterPublishTaskOutputsCopy", + "SharedRegisterPublishTaskOutputsFlush", + ): + if tensormap_mode != "shared": + raise ValueError( + f"fdwic_events[{index}] {phase} " + "is only valid for shared TensorMap" + ) + if task_id < 0 or flags != 0 or auxiliary != 0: + raise ValueError( + f"fdwic_events[{index}] has invalid {phase} fields" + ) + bucket = ( + v4_shared_register_output_copy_details + if phase.endswith("Copy") + else v4_shared_register_output_flush_details + ) + bucket.setdefault(task_key, []).append( + ( + core_id, + block_id, + lane, + task_id, + function_id, + phase, + start_cycle, + end_cycle, + flags, + auxiliary, + ) + ) + if phase in ("OrchestrationReplay", "FinalDrain"): + if task_id != -1 or function_id != -1 or flags != 0 or auxiliary != 0: + raise ValueError( + f"fdwic_events[{index}] has invalid schema-v5 parent {phase} fields" + ) + v4_parent_counts[core_id][phase] += 1 + elif phase == "Submit": + if task_id < 0 or flags > 1 or auxiliary > 1: + raise ValueError( + f"fdwic_events[{index}] has invalid schema-v5 Submit fields" + ) + if task_key in v4_submits: + raise ValueError( + f"core {core_id} has duplicate schema-v5 Submit for task {task_id}" + ) + v4_submits.add(task_key) + v4_submit_semantics[task_key] = (bool(flags & 1), bool(auxiliary)) + elif phase == "Claim": + if task_id < 0 or task_key in v4_claims: + raise ValueError( + f"core {core_id} has invalid or duplicate schema-v5 Claim for task {task_id}" + ) + v4_claims[task_key] = ( + bool(flags & 0x2), bool(flags & 0x1), bool(auxiliary) + ) + elif phase == "Materialize": + if task_id < 0: + raise ValueError( + f"fdwic_events[{index}] Materialize requires non-negative task_id" + ) + v4_materializes.setdefault(task_key, []).append( + ( + core_id, + block_id, + lane, + task_id, + function_id, + phase, + start_cycle, + end_cycle, + flags, + auxiliary, + ) + ) + elif phase == "Register": + if task_id < 0: + raise ValueError( + f"fdwic_events[{index}] Register requires non-negative task_id" + ) + v4_registers.setdefault(task_key, []).append( + ( + core_id, + block_id, + lane, + task_id, + function_id, + phase, + start_cycle, + end_cycle, + flags, + auxiliary, + ) + ) + elif phase in ("WinnerBuild", "AllocComplete"): + if task_id < 0 or flags != 0 or auxiliary != 0: + raise ValueError( + f"fdwic_events[{index}] has invalid schema-v5 tail {phase} fields" + ) + if task_key in v4_tails: + raise ValueError( + f"core {core_id} has duplicate schema-v5 tail for task {task_id}" + ) + if phase == "WinnerBuild" and function_id not in KERNEL_NAMES: + raise ValueError( + f"fdwic_events[{index}] WinnerBuild has invalid function_id " + f"{function_id}" + ) + if phase == "AllocComplete" and function_id != -1: + raise ValueError( + f"fdwic_events[{index}] AllocComplete requires function_id=-1" + ) + # task 类型必须等所有核的 Submit Alloc 标记齐全后再推导； + # 这里仅保存尾动作自己的权威 function，避免恢复固定 %5 假设。 + v4_tails[task_key] = (phase, function_id) + if start_cycle <= 0 or end_cycle < start_cycle: + raise ValueError( + f"fdwic_events[{index}] has invalid cycles start={start_cycle} end={end_cycle}" + ) + key = (block_id, lane) + previous_core = core_by_block_lane.setdefault(key, core_id) + if previous_core != core_id: + raise ValueError( + f"block {block_id} lane {lane} maps to both core {previous_core} and core {core_id}" + ) + # 所有 X 事件共同减去最早 start，既避免大整数转 float 的精度损失， + # 也让设备 SYS_CNT 的绝对值不影响 Perfetto 横轴。 + base_cycle = start_cycle if base_cycle is None else min(base_cycle, start_cycle) + rows[index] = ( + core_id, + block_id, + lane, + task_id, + function_id, + phase, + start_cycle, + end_cycle, + flags, + auxiliary, + ) + + assert base_cycle is not None + if trace_schema_version in (3, 5) and level == 4: + # 每核两条基线同时证明采集完整性和该后端是否真正应用了 + # atomic 返回值依赖；所有消费返回值的直接记录必须与本核证据一致。 + # SharedInsertTurnPoll 的最终 PollBatch 可以额外带 return_ready， + # 但它仍表示整个轮询 episode，不能解释成单次 Load 延迟。 + for core_id, clock_state in v3_clock_rows.items(): + if clock_state["plain"] != 1 or clock_state["dependency"] != 1: + raise ValueError( + f"core {core_id} requires exactly one plain and one dependency " + f"ClockBaseline: plain={clock_state['plain']} " + f"dependency={clock_state['dependency']}" + ) + for row_index, core_id, return_ready in v3_result_used_direct_rows: + expected_return_ready = bool(v3_clock_rows[core_id]["return_ready"]) + if return_ready != expected_return_ready: + raise ValueError( + f"fdwic_events[{row_index}] direct Atomic return_ready={return_ready} " + f"does not match core {core_id} ClockBaseline " + f"dependency_applied={expected_return_ready}" + ) + for row_index, core_id, return_ready in v3_insert_turn_poll_batch_rows: + expected_return_ready = bool(v3_clock_rows[core_id]["return_ready"]) + if return_ready != expected_return_ready: + raise ValueError( + f"fdwic_events[{row_index}] PollBatch " + f"return_ready={return_ready} " + f"does not match core {core_id} ClockBaseline " + f"dependency_applied={expected_return_ready}" + ) + + if trace_schema_version == 5: + for core_id, counts in v4_parent_counts.items(): + for phase, count in counts.items(): + if count != 1: + raise ValueError( + f"core {core_id} requires exactly one schema-v5 {phase}: count={count}" + ) + if set(v4_claims) != v4_submits: + raise ValueError("schema-v5 Claim keys do not match Submit keys") + task_kind_by_id = _derive_v4_task_kinds(v4_submit_semantics, num_cores) + for task_key, (attempted, won, is_alloc) in v4_claims.items(): + submit_won, submit_alloc = v4_submit_semantics[task_key] + task_kind = task_kind_by_id[task_key[1]] + expected_alloc = task_kind == 0 + if is_alloc != expected_alloc or submit_alloc != expected_alloc: + raise ValueError( + f"schema-v5 task-kind mismatch at {task_key}: " + f"expected_alloc={expected_alloc}" + ) + if won and not attempted: + raise ValueError(f"schema-v5 Claim won without attempt at {task_key}") + if submit_won != won or submit_alloc != is_alloc: + raise ValueError(f"schema-v5 Submit/Claim semantics mismatch at {task_key}") + expected_tail = ( + ("AllocComplete", -1) + if is_alloc + else ("WinnerBuild", task_kind - 1) + ) + actual_tail = v4_tails.get(task_key) + # 只为 winner 记录真实尾动作；loser 没有尾记录，其剩余时间 + # 由 Submit 的离线补集表示。 + tail_valid = actual_tail == expected_tail if won else actual_tail is None + if not tail_valid: + raise ValueError( + f"schema-v5 tail mismatch at {task_key}: " + f"expected {expected_tail if won else 'no winner tail'}, " + f"got {actual_tail}" + ) + if tensormap_mode == "shared": + materializes = v4_materializes.get(task_key, []) + parents = v4_registers.get(task_key, []) + expected_parent_count = 1 if won else 0 + if len(parents) != expected_parent_count: + raise ValueError( + "shared schema-v5 requires exactly one Register parent " + "for each winner and none for losers at " + f"{task_key}: count={len(parents)} won={won}" + ) + details = v4_shared_register_details.get(task_key, []) + if len(details) != (1 if won else 0): + raise ValueError( + "shared schema-v5 requires exactly one " + "SharedRegisterPublishMetadata for each winner and none " + f"for losers at {task_key}: count={len(details)} won={won}" + ) + if not won: + continue + parent = parents[0] + detail = details[0] + output_details = v4_shared_register_output_details.get( + task_key, [] + ) + if len(output_details) != 1: + raise ValueError( + "shared schema-v5 requires exactly one " + "SharedRegisterPublishTaskOutputs or " + "SharedMaterializePublishTaskOutputs for each winner " + "and none " + f"for losers at {task_key}: " + f"count={len(output_details)} won={won}" + ) + output_detail = output_details[0] + output_phase = str(output_detail[5]) + outputs_in_materialize = ( + output_phase == + "SharedMaterializePublishTaskOutputs" + ) + if outputs_in_materialize and len(materializes) != 1: + raise ValueError( + "shared schema-v5 requires exactly one Materialize parent " + "for each winner using Materialize task-output publication " + f"at {task_key}: count={len(materializes)}" + ) + if parent[:5] != detail[:5]: + raise ValueError( + "shared schema-v5 Register detail identity differs from " + f"its parent at {task_key}" + ) + output_parent = ( + materializes[0] + if outputs_in_materialize + else detail + ) + if output_parent[:5] != output_detail[:5]: + raise ValueError( + "shared schema-v5 task-outputs detail identity differs " + f"from {output_parent[5]} at {task_key}" + ) + parent_start, parent_end = int(parent[6]), int(parent[7]) + detail_start, detail_end = int(detail[6]), int(detail[7]) + if not ( + parent_start + <= detail_start + <= detail_end + <= parent_end + ): + raise ValueError( + "shared schema-v5 SharedRegisterPublishMetadata is outside " + f"Register parent at {task_key}" + ) + output_start = int(output_detail[6]) + output_end = int(output_detail[7]) + output_parent_start = int(output_parent[6]) + output_parent_end = int(output_parent[7]) + if not ( + output_parent_start + <= output_start + <= output_end + <= output_parent_end + ): + raise ValueError( + f"shared schema-v5 {output_phase} is outside " + f"{output_parent[5]} at {task_key}" + ) + copy_details = v4_shared_register_output_copy_details.get( + task_key, [] + ) + flush_details = v4_shared_register_output_flush_details.get( + task_key, [] + ) + if len(copy_details) != 1: + raise ValueError( + "shared schema-v5 requires exactly one " + "SharedRegisterPublishTaskOutputsCopy for each winner " + f"and none for losers at {task_key}: " + f"count={len(copy_details)} won={won}" + ) + if len(flush_details) != 1: + raise ValueError( + "shared schema-v5 requires exactly one " + "SharedRegisterPublishTaskOutputsFlush for each winner " + f"and none for losers at {task_key}: " + f"count={len(flush_details)} won={won}" + ) + copy_detail = copy_details[0] + flush_detail = flush_details[0] + expected_copy_phase = ( + "SharedMaterializePublishTaskOutputsCopy" + if outputs_in_materialize + else "SharedRegisterPublishTaskOutputsCopy" + ) + expected_flush_phase = ( + "SharedMaterializePublishTaskOutputsFlush" + if outputs_in_materialize + else "SharedRegisterPublishTaskOutputsFlush" + ) + if ( + copy_detail[5] != expected_copy_phase + or flush_detail[5] != expected_flush_phase + ): + raise ValueError( + "shared schema-v5 task-output detail families are mixed " + f"at {task_key}: parent={output_phase} " + f"copy={copy_detail[5]} flush={flush_detail[5]}" + ) + if output_detail[:5] != copy_detail[:5]: + raise ValueError( + "shared schema-v5 task-outputs copy identity differs " + f"from {output_phase} at {task_key}" + ) + if output_detail[:5] != flush_detail[:5]: + raise ValueError( + "shared schema-v5 task-outputs flush identity differs " + f"from {output_phase} at {task_key}" + ) + copy_start = int(copy_detail[6]) + copy_end = int(copy_detail[7]) + flush_start = int(flush_detail[6]) + flush_end = int(flush_detail[7]) + if not ( + output_start + <= copy_start + <= copy_end + == flush_start + <= flush_end + <= output_end + ): + raise ValueError( + "shared schema-v5 task-outputs copy/flush nesting is " + f"invalid at {task_key}: " + f"outputs=[{output_start},{output_end}) " + f"copy=[{copy_start},{copy_end}) " + f"flush=[{flush_start},{flush_end})" + ) + if level == 4: + matching_polls = [ + poll + for poll in v4_shared_insert_turn_polls + if poll[:3] == parent[:3] + and int(poll[6]) == parent_start + and int(poll[7]) == detail_start + ] + # per-task predecessor chain 中 task 0 没有前驱，不执行 + # insert-turn Load；其余 task 的 winner 各产生一条聚合 + # PollBatch。Register 的前段仍由父/detail 边界表示， + # 不能因为 task 0 没有 atomic 就删掉该闭合区间。 + expected_poll_count = 0 if task_key[1] == 0 else 1 + if len(matching_polls) != expected_poll_count: + raise ValueError( + "shared schema-v5 level4 requires exactly one " + "SharedInsertTurnPoll PollBatch for every nonzero-task " + "winner and none for task 0 on " + "Register.start->metadata.start at " + f"{task_key}: count={len(matching_polls)} " + f"expected={expected_poll_count}" + ) + handoffs = v4_shared_insert_turn_handoffs.get( + task_key, [] + ) + if len(handoffs) != 1: + raise ValueError( + "shared schema-v5 level4 requires exactly one " + "SharedInsertTurnHandoff direct CAS per winner at " + f"{task_key}: count={len(handoffs)}" + ) + handoff = handoffs[0] + handoff_start = int(handoff[6]) + handoff_end = int(handoff[7]) + if handoff[:3] != parent[:3] or not ( + detail_end + <= handoff_start + <= handoff_end + <= parent_end + ): + raise ValueError( + "shared schema-v5 SharedInsertTurnHandoff identity " + "or boundary is outside metadata.end->Register.end " + f"at {task_key}" + ) + + if tensormap_mode == "shared": + orphan_parent_keys = set(v4_registers) - set(v4_claims) + if orphan_parent_keys: + raise ValueError( + "shared schema-v5 Register parents have no matching Claim: " + f"{sorted(orphan_parent_keys)[:8]}" + ) + orphan_detail_keys = set(v4_shared_register_details) - set(v4_claims) + if orphan_detail_keys: + raise ValueError( + "shared schema-v5 Register details have no matching Claim: " + f"{sorted(orphan_detail_keys)[:8]}" + ) + orphan_output_detail_keys = ( + set(v4_shared_register_output_details) - set(v4_claims) + ) + if orphan_output_detail_keys: + raise ValueError( + "shared schema-v5 task-output details have no matching Claim: " + f"{sorted(orphan_output_detail_keys)[:8]}" + ) + orphan_copy_detail_keys = ( + set(v4_shared_register_output_copy_details) - set(v4_claims) + ) + if orphan_copy_detail_keys: + raise ValueError( + "shared schema-v5 task-output copy details have no matching " + f"Claim: {sorted(orphan_copy_detail_keys)[:8]}" + ) + orphan_flush_detail_keys = ( + set(v4_shared_register_output_flush_details) - set(v4_claims) + ) + if orphan_flush_detail_keys: + raise ValueError( + "shared schema-v5 task-output flush details have no matching " + f"Claim: {sorted(orphan_flush_detail_keys)[:8]}" + ) + for task_key, output_details in ( + v4_shared_register_output_details.items() + ): + won = v4_claims.get(task_key, (False, False, False))[1] + if len(output_details) != (1 if won else 0): + raise ValueError( + "shared schema-v5 requires exactly one " + "SharedRegisterPublishTaskOutputs for each winner and none " + f"for losers at {task_key}: " + f"count={len(output_details)} won={won}" + ) + for task_key, copy_details in ( + v4_shared_register_output_copy_details.items() + ): + won = v4_claims.get(task_key, (False, False, False))[1] + if len(copy_details) != (1 if won else 0): + raise ValueError( + "shared schema-v5 requires exactly one " + "SharedRegisterPublishTaskOutputsCopy for each winner " + f"and none for losers at {task_key}: " + f"count={len(copy_details)} won={won}" + ) + for task_key, flush_details in ( + v4_shared_register_output_flush_details.items() + ): + won = v4_claims.get(task_key, (False, False, False))[1] + if len(flush_details) != (1 if won else 0): + raise ValueError( + "shared schema-v5 requires exactly one " + "SharedRegisterPublishTaskOutputsFlush for each winner " + f"and none for losers at {task_key}: " + f"count={len(flush_details)} won={won}" + ) + if level == 4: + winner_count = sum( + won for _attempted, won, _is_alloc in v4_claims.values() + ) + nonzero_task_winner_count = sum( + 1 + for (_core_id, task_id), ( + _attempted, + won, + _is_alloc, + ) in v4_claims.items() + if won and task_id > 0 + ) + if ( + len(v4_shared_insert_turn_polls) + != nonzero_task_winner_count + ): + raise ValueError( + "shared schema-v5 level4 has orphan or duplicate " + "SharedInsertTurnPoll records: " + f"records={len(v4_shared_insert_turn_polls)} " + "expected_nonzero_task_winners=" + f"{nonzero_task_winner_count} " + f"all_winners={winner_count}" + ) + handoff_count = sum( + len(items) + for items in v4_shared_insert_turn_handoffs.values() + ) + if handoff_count != winner_count: + raise ValueError( + "shared schema-v5 level4 has orphan or duplicate " + "SharedInsertTurnHandoff records: " + f"records={handoff_count} winners={winner_count}" + ) + + if trace_schema_version >= 3: + producer_summary = metadata.get("fdwic_summary") + if not isinstance(producer_summary, dict): + raise ValueError( + "metadata.fdwic_summary is required for trace_schema_version>=3" + ) + dcci_keys = ("dcci_records", "dcci_calls", "dcci_lines") + dcci_declared = any(key in producer_summary for key in dcci_keys) + if observed_summary["dcci_records"] != 0 and not dcci_declared: + raise ValueError( + "raw Dcci records require dcci_records/dcci_calls/dcci_lines " + "in metadata.fdwic_summary" + ) + if dcci_declared: + if not all(key in producer_summary for key in dcci_keys): + raise ValueError( + "metadata.fdwic_summary must declare all three DCCI counters" + ) + if trace_schema_version != 5: + raise ValueError( + "DCCI summary counters require trace_schema_version=5" + ) + invalid = { + core_id: count + for core_id, count in v5_observer_dcci_rows.items() + if count != 1 + } + if invalid: + raise ValueError( + "schema-v5 DCCI capture requires exactly one " + "ObserverTraceExport record per core; " + f"invalid={invalid}" + ) + for key, observed_value in observed_summary.items(): + # 迁移前 schema-v5 capture 没有 DCCI 行，也没有三项 summary。 + # 一旦任一新字段出现，就必须按上面的完整合同严格闭合。 + if key in dcci_keys and not dcci_declared: + continue + producer_value = _integer( + producer_summary.get(key), f"metadata.fdwic_summary.{key}" + ) + if producer_value != observed_value: + raise ValueError( + f"metadata.fdwic_summary.{key}={producer_value} " + f"does not match raw value {observed_value}" + ) + return frequency_hz, trace_schema_version, rows, core_by_block_lane, base_cycle, metadata + + +def _restore_v5_shared_efdrain( + rows: list[tuple[Any, ...]], + trace_schema_version: int, + tensormap_mode: str | None, +) -> None: + """用 Submit.start 与 Claim.start 离线恢复 shared EfDrain。 + + 新 shared raw 不再为每个 Submit 写一条 EfDrain 记录。该区间的两端 + 已分别由 Submit 和 Claim 权威记录，离线恢复既不扩张 raw ABI，也不会 + 污染设备侧 ``fdwic_summary.records``。采集与加工使用同一份代码， + 因此 shared schema-v5 一旦出现显式 EfDrain 就直接拒绝，避免同时 + 维护设备记录与离线派生两个口径。 + """ + + if trace_schema_version != 5 or tensormap_mode != "shared": + return + + submits: dict[tuple[int, int], tuple[Any, ...]] = {} + claims: dict[tuple[int, int], tuple[Any, ...]] = {} + for row in rows: + phase = str(row[5]) + if phase == "EfDrain": + raise ValueError( + "shared schema-v5 raw must not contain explicit EfDrain" + ) + if phase not in ("Submit", "Claim"): + continue + key = (int(row[0]), int(row[3])) + bucket = submits if phase == "Submit" else claims + if key in bucket: + raise ValueError( + f"shared schema-v5 has duplicate {phase} for {key}" + ) + bucket[key] = row + + if set(claims) != set(submits): + missing = sorted(set(submits) - set(claims)) + orphan = sorted(set(claims) - set(submits)) + raise ValueError( + "shared schema-v5 EfDrain derivation requires exactly one Claim " + f"per Submit: missing={missing[:8]} orphan={orphan[:8]}" + ) + + for key in sorted(submits): + submit = submits[key] + claim = claims[key] + if tuple(int(value) for value in claim[:5]) != tuple( + int(value) for value in submit[:5] + ): + raise ValueError( + "shared schema-v5 Claim identity differs from Submit for " + f"{key}" + ) + submit_start = int(submit[6]) + submit_end = int(submit[7]) + claim_start = int(claim[6]) + claim_end = int(claim[7]) + if not ( + submit_start + <= claim_start + <= claim_end + <= submit_end + ): + raise ValueError( + "shared schema-v5 cannot derive EfDrain because Claim is " + f"outside or inverted relative to Submit at {key}: " + f"Submit=[{submit_start},{submit_end}) " + f"Claim=[{claim_start},{claim_end})" + ) + + expected = ( + int(submit[0]), + int(submit[1]), + int(submit[2]), + int(submit[3]), + # EfDrain 是 Submit 前端的 scalar 控制区，不属于某个计算 + # function；历史设备记录和新离线事件都固定使用 -1。 + -1, + "EfDrain", + submit_start, + claim_start, + 0, + 0, + ) + rows.append(expected) + + +# 写一个 Chrome Trace Event，并统一处理数组元素间的逗号。 +def _emit_event(output: TextIO, event: dict[str, Any], first: bool) -> bool: + # 逐事件写出，避免再在内存中构造一份体积可达数百 MiB 的 merged 列表。 + if not first: + output.write(",\n") + json.dump(event, output, ensure_ascii=False, separators=(",", ":")) + return False + + +def _iter_v5_residual_spans( + rows: list[tuple[Any, ...]], +) -> Iterator[tuple[int, int, int, int, int, str]]: + """只用既有 Submit/child 边界生成逐段补集，不改 raw ABI。""" + + submits_by_lane: dict[tuple[int, int], list[tuple[Any, ...]]] = {} + submit_by_task: dict[tuple[int, int, int], tuple[Any, ...]] = {} + children_by_task: dict[tuple[int, int, int], list[tuple[Any, ...]]] = {} + for row in rows: + core_id, _block_id, lane, task_id, _function_id, phase, *_rest = row + lane_key = (int(core_id), int(lane)) + task_key = (int(core_id), int(lane), int(task_id)) + if phase == "Submit": + submits_by_lane.setdefault(lane_key, []).append(row) + if task_key in submit_by_task: + raise ValueError(f"schema-v5 residual synthesis found duplicate Submit {task_key}") + submit_by_task[task_key] = row + elif phase in V5_EXCLUSIVE_SUBMIT_PHASES: + children_by_task.setdefault(task_key, []).append(row) + + orphan_child_keys = set(children_by_task) - set(submit_by_task) + if orphan_child_keys: + raise ValueError( + "schema-v5 residual synthesis found children without matching Submit: " + f"{sorted(orphan_child_keys)[:8]}" + ) + + # 先按每个 scalar lane 标记相邻 Submit 之间的真实空白。 + for lane_key in sorted(submits_by_lane): + submits = sorted( + submits_by_lane[lane_key], key=lambda row: (int(row[6]), int(row[7])) + ) + for previous, current in zip(submits, submits[1:]): + previous_end = int(previous[7]) + current_start = int(current[6]) + if current_start < previous_end: + raise ValueError(f"schema-v5 Submit spans overlap on core/lane {lane_key}") + if current_start > previous_end: + yield ( + int(previous[0]), + int(previous[1]), + int(previous[2]), + previous_end, + current_start, + "between_submit_residual", + ) + + # Submit 内部只扣除同 task 的互斥 child；每个不连续补集段 + # 单独生成一条最小 Perfetto X event，不伪造跨空白的连续区间。 + # 只有“最后一个已知 child.end -> Submit.end”使用 tail 名称； + # 前缀和 child-to-child gap 继续保留中性 residual，不冒充业务阶段。 + for task_key in sorted(submit_by_task): + submit = submit_by_task[task_key] + submit_start = int(submit[6]) + submit_end = int(submit[7]) + cursor = submit_start + children = sorted( + children_by_task.get(task_key, []), + key=lambda row: (int(row[6]), int(row[7]), str(row[5])), + ) + for child in children: + child_start = int(child[6]) + child_end = int(child[7]) + if child_start < submit_start or child_end > submit_end: + raise ValueError( + f"schema-v5 {child[5]} child is outside Submit {task_key}" + ) + if child_start < cursor: + raise ValueError( + f"schema-v5 exclusive children overlap in Submit {task_key}" + ) + if child_start > cursor: + yield ( + int(submit[0]), + int(submit[1]), + int(submit[2]), + cursor, + child_start, + "submit_residual", + ) + cursor = max(cursor, child_end) + if submit_end > cursor: + yield ( + int(submit[0]), + int(submit[1]), + int(submit[2]), + cursor, + submit_end, + # 与旧 "submit_residual" 等长，重分类后不增加 merged 字节。 + "submit_tail_gap", + ) + + +def _iter_v5_shared_register_derived_spans( + rows: list[tuple[Any, ...]], +) -> Iterator[tuple[int, int, int, int, int, str]]: + """用 Register 与 metadata 边界补出非 raw 串行段。 + + 新采集的 task outputs 已属于 Materialize，因此 Register 只合成等待、 + writer metadata 与插入完成发布。writer 名称复用 Register raw 已有的 + auxiliary，直接显示 ordinary TensorMap entry 数，不增加设备字段。 + 迁移前 raw 仍按旧 outputs 子区间恢复 metadata epilogue，保证历史 + 泳道可重放。 + """ + + parents: dict[tuple[int, int], tuple[Any, ...]] = {} + details: dict[tuple[int, int], tuple[Any, ...]] = {} + output_details: dict[tuple[int, int], tuple[Any, ...]] = {} + for row in rows: + core_id, _block_id, _lane, task_id, _function_id, phase, *_rest = row + task_key = (int(core_id), int(task_id)) + if phase == "Register": + parents[task_key] = row + elif phase == "SharedRegisterPublishMetadata": + details[task_key] = row + elif phase in ( + "SharedMaterializePublishTaskOutputs", + "SharedRegisterPublishTaskOutputs", + ): + output_details[task_key] = row + + for task_key in sorted(details): + parent = parents[task_key] + detail = details[task_key] + output_detail = output_details[task_key] + core_id, block_id, lane, task_id = ( + int(parent[0]), + int(parent[1]), + int(parent[2]), + int(parent[3]), + ) + yield ( + core_id, + block_id, + lane, + int(parent[6]), + int(detail[6]), + f"register.wait_predecessor_tensormap_insert#{task_id}", + ) + ordinary_tensormap_entries = int(parent[9]) + writer_metadata_name = ( + "register.publish_writer_metadata" + f"[ordinary_tensormap_entries={ordinary_tensormap_entries}]" + f"#{task_id}" + ) + if output_detail[5] == "SharedRegisterPublishTaskOutputs": + yield ( + core_id, + block_id, + lane, + int(detail[6]), + int(output_detail[6]), + writer_metadata_name, + ) + yield ( + core_id, + block_id, + lane, + int(output_detail[7]), + int(detail[7]), + f"register.publish_metadata_epilogue#{task_id}", + ) + else: + yield ( + core_id, + block_id, + lane, + int(detail[6]), + int(detail[7]), + writer_metadata_name, + ) + yield ( + core_id, + block_id, + lane, + int(detail[7]), + int(parent[7]), + f"register.publish_tensormap_insert_completion#{task_id}", + ) + + +def _merged_item_sort_key( + item: tuple[Any, ...], +) -> tuple[int, int, int, int, int, str]: + """按物理轨道建立父区间优先的确定性导入顺序。 + + 设备在阶段结束时才写父记录，所以 raw 的物理顺序天然是“子事件在前、 + 父区间在后”。Perfetto 的同轨 slice 建栈不能直接使用这个落盘顺序。 + 这里仅重排离线 merged：同一轨道 start 升序、end 降序，保证外层先 + 导入；完全同区间时业务 span 先于 Atomic/DCCI overlay。 + """ + + if item and item[0] == "derived": + _, _core_id, block_id, lane, start, end, name = item + return ( + int(block_id), int(lane), int(start), -int(end), + 0, str(name), + ) + ( + _core_id, + block_id, + lane, + _task_id, + _function_id, + phase_raw, + start, + end, + _flags, + _auxiliary, + ) = item + phase = PHASE_NAMES[str(phase_raw)] + thread_id = ( + int(lane) + 3 + if phase == "kernel" or phase == "commit" + else int(lane) + ) + overlay_priority = 2 if phase in ("atomic", "dcci") else 1 + return ( + int(block_id), thread_id, int(start), -int(end), + overlay_priority, str(phase_raw), + ) + + +# 完成一次 raw 到 merged 的转换，成功时返回事件数、block 数和基准 cycle。 +def convert(input_path: Path, output_path: Path) -> tuple[int, int, int]: + ( + frequency_hz, + trace_schema_version, + rows, + core_by_block_lane, + base_cycle, + capture_metadata, + ) = _load_and_validate(input_path) + _restore_v5_shared_efdrain( + rows, + trace_schema_version, + capture_metadata.get("tensormap_mode"), + ) + # 禁止原地转换；否则创建临时文件或最终 replace 时可能破坏唯一一份 raw。 + if input_path.resolve() == output_path.resolve(): + raise ValueError("input and output paths must differ") + + # 始终先写同目录临时文件，完整 flush/fsync 后再原子替换目标；转换失败 + # 时删除临时文件，不把半截 JSON 留作可加载的正式产物。 + output_path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = output_path.with_name(output_path.name + ".tmp") + # Chrome Trace Event 的 ts/dur 约定使用微秒；displayTimeUnit="ns" 只控制 + # Perfetto 的显示精度。1 GHz A5 counter 因此每 tick 对应 0.001 us。 + factor = 1_000_000.0 / float(frequency_hz) + blocks = sorted({block_id for block_id, _ in core_by_block_lane}) + # v1 raw 没有 Claim attempted bit。若同一份 capture 确实含逐 atomic + # 记录，则可以用同核、同 task 且时间被 Claim 完整包含的 + # claim_max 作为实测证据恢复 attempted；不含 atomic 时保留 unknown， + # 绝不根据 task kind 或 AIC/AIV role 在转换器中猜业务路由。 + legacy_claim_max_spans: dict[tuple[int, int, int, int], list[tuple[int, int]]] = {} + has_atomic_trace = False + if trace_schema_version == 1: + for core_id, block_id, lane, task_id, _, phase, start, end, _, auxiliary in rows: + if phase != "Atomic": + continue + has_atomic_trace = True + if auxiliary == 4: # AtomicSite::ClaimMax + key = (core_id, block_id, lane, task_id) + legacy_claim_max_spans.setdefault(key, []).append((start, end)) + # 新布局已经把 output descriptor 发布移入 Materialize。对应 Register + # metadata raw 与离线合成的 writer span 使用完全相同的边界；merged + # 只保留带 ordinary TensorMap 数量的合成事件，避免同轨同区间互相遮挡。 + # 旧 Register-placement raw 不在此集合中，仍按历史嵌套结构完整输出。 + materialize_output_tasks = { + (int(row[0]), int(row[3])) + for row in rows + if row[5] == "SharedMaterializePublishTaskOutputs" + } + # merged 的顺序是显示合同的一部分：先收集 raw 引用与离线派生 span， + # 再按物理轨道做父区间优先排序。这里只增加轻量 tuple/reference， + # 不构造数十万份 event dict；JSON 仍逐事件流式写出。 + ordered_items: list[tuple[Any, ...]] = [ + row + for row in rows + if not ( + trace_schema_version == 5 + and row[5] == "SharedRegisterPublishMetadata" + and (int(row[0]), int(row[3])) + in materialize_output_tasks + ) + ] + if trace_schema_version == 5: + ordered_items.extend( + ("derived", *span) + for span in _iter_v5_shared_register_derived_spans( + rows + ) + ) + ordered_items.extend( + ("derived", *span) + for span in _iter_v5_residual_spans(rows) + ) + ordered_items.sort(key=_merged_item_sort_key) + first = True + emitted = 0 + # 临时文件的整个生命周期都在 try 内；包括 Ctrl-C 在内的异常都会先清理 + # .tmp 再向上传播。格式/IO 错误由 main 简短报告，Ctrl-C 保留默认中断行为。 + try: + with temporary_path.open("w", encoding="utf-8") as output: + output.write('{"displayTimeUnit":"ns","metadata":') + json.dump(capture_metadata, output, ensure_ascii=False, separators=(",", ":")) + output.write(',"traceEvents":[\n') + winner_workload = capture_metadata.get("winner_workload") + if winner_workload is not None: + first = _emit_event( + output, + { + "ph": "i", + "s": "g", + "name": "pa_scheduler.capture", + "pid": 0, + "tid": 0, + "ts": 0, + "args": {"winner_workload": winner_workload}, + }, + first, + ) + emitted += 1 + # 每个物理 block 建一个 process；每条硬件 lane 再拆成 runtime + # 与 kernel 两个 thread，避免等待/提交阶段覆盖 kernel 执行条。 + for block_id in blocks: + first = _emit_event( + output, + {"ph": "M", "name": "process_name", "pid": block_id, "args": {"name": f"block{block_id}"}}, + first, + ) + first = _emit_event( + output, + { + "ph": "M", + "name": "process_sort_index", + "pid": block_id, + "args": {"sort_index": block_id}, + }, + first, + ) + for lane, lane_name in LANE_NAMES.items(): + core_id = core_by_block_lane.get((block_id, lane)) + if core_id is None: + continue + for thread_id, thread_name in ( + (lane, f"{lane_name} (core{core_id})"), + (lane + 3, f"{lane_name}·kernel (core{core_id})"), + ): + first = _emit_event( + output, + { + "ph": "M", + "name": "thread_name", + "pid": block_id, + "tid": thread_id, + "args": {"name": thread_name}, + }, + first, + ) + for item in ordered_items: + if item[0] == "derived": + ( + _derived, + _core_id, + block_id, + lane, + start, + end, + name, + ) = item + first = _emit_event( + output, + { + "ph": "X", + "name": name, + "pid": block_id, + "tid": lane, + "ts": round( + (start - base_cycle) * factor, 3 + ), + "dur": round( + (end - start) * factor, 3 + ), + }, + first, + ) + emitted += 1 + continue + row = item + core_id, block_id, lane, task_id, function_id, phase_raw, start, end, flags, auxiliary = row + phase = PHASE_NAMES[phase_raw] + # Kernel/Commit 放到 lane+3 的计算单元子泳道；Atomic/ClockBaseline + # 都是 AIC/AIV 对应 scalar 上执行的指令，必须与 runtime 阶段共用 + # lane 0..2。这样 atomic span 作为 Claim/Fanin/轮询等阶段的子区间 + # 叠加显示，不会伪装成 AIC/AIV 之外的第三类执行单元。 + if phase == "claim": + claim_attempted: bool | None + claim_attempted_source: str + if trace_schema_version >= 2: + claim_attempted = bool(flags & 0x2) + claim_attempted_source = "raw_flag" + elif has_atomic_trace: + key = (core_id, block_id, lane, task_id) + matched_claim_max = any( + atomic_start >= start and atomic_end <= end + for atomic_start, atomic_end in legacy_claim_max_spans.get(key, []) + ) + # 命中的 claim_max 能正向证明 attempted；但 v1 raw 没有 + # 显式“atomic 记录完整”元数据，未命中不能反向证明 + # not_attempted，因此保留 unknown。 + claim_attempted = True if matched_claim_max else None + claim_attempted_source = ( + "contained_claim_max" + if matched_claim_max + else "unknown_v1_without_matching_claim_max" + ) + else: + claim_attempted = None + claim_attempted_source = "unknown_v1_without_atomic_trace" + claim_won = bool(flags & 0x1) + if claim_attempted is False: + name = f"claim.not_attempted#{task_id}" + elif claim_attempted is True: + name = f"claim.{'won' if claim_won else 'lost'}#{task_id}" + else: + name = f"claim#{task_id}" + thread_id = lane + elif phase == "atomic": + atomic_site_id = auxiliary + atomic_op_id = flags & 0xF + atomic_site = ATOMIC_SITE_NAMES.get(atomic_site_id, f"site_{atomic_site_id}") + atomic_op = ATOMIC_OP_NAMES.get(atomic_op_id, f"op_{atomic_op_id}") + atomic_poll_batch = ( + trace_schema_version >= 3 and bool(flags & ATOMIC_POLL_BATCH) + ) + if atomic_poll_batch: + atomic_call_count = ( + flags >> ATOMIC_PAYLOAD_SHIFT + ) & ATOMIC_PAYLOAD_MASK + if ( + atomic_site_id + == SHARED_INSERT_TURN_POLL_SITE_ID + ): + poll_boundary_tag = ( + "return_ready" + if flags & ATOMIC_RETURN_READY + else "source_issue" + ) + name = ( + f"atomic.poll_batch.{poll_boundary_tag}." + f"{atomic_site}.{atomic_op}" + f"×{atomic_call_count}" + ) + else: + name = ( + f"atomic.poll_batch.{atomic_site}.{atomic_op}" + f"×{atomic_call_count}" + ) + else: + # 边界直接写入 span 名称，打开泳道后无需点开 args + # 就能区分“本核返回值可消费”和“只包围源码发射”。 + atomic_boundary_tag = ( + "return_ready" + if flags & ATOMIC_RETURN_READY + else "source_issue" + ) + name = ( + f"atomic.{atomic_boundary_tag}.{atomic_site}." + f"{atomic_op}#{task_id}" + ) + thread_id = lane + elif phase == "dcci": + dcci_site_id = auxiliary + dcci_op_id = flags & DCCI_OP_MASK + dcci_site = DCCI_SITE_NAMES.get( + dcci_site_id, f"site_{dcci_site_id}" + ) + dcci_op = DCCI_OP_NAMES.get( + dcci_op_id, f"op_{dcci_op_id}" + ) + dcci_call_count = ( + flags >> DCCI_CALL_COUNT_SHIFT + ) & DCCI_CALL_COUNT_MASK + dcci_line_count = ( + flags >> DCCI_LINE_COUNT_SHIFT + ) & DCCI_LINE_COUNT_MASK + name = ( + f"dcci.{dcci_site}.{dcci_op}" + f"×{dcci_call_count}.lines{dcci_line_count}" + f"#{task_id}" + ) + thread_id = lane + elif phase == "clock_baseline": + name = ( + "clock.atomic_return_dependency_hook" + if flags & 1 + else "clock.consecutive_sys_cnt_reads" + ) + thread_id = lane + elif phase == "kernel" and function_id >= 0: + name = f"{KERNEL_NAMES.get(function_id, f'f{function_id}')}#{task_id}" + thread_id = lane + 3 + elif phase == "commit": + name = f"commit#{task_id}" + thread_id = lane + 3 + elif phase in ("orchestration_replay", "final_drain"): + name = phase + thread_id = lane + else: + name = f"{phase}#{task_id}" + thread_id = lane + event = { + "ph": "X", + "name": name, + "pid": block_id, + "tid": thread_id, + "ts": round((start - base_cycle) * factor, 3), + "dur": round((end - start) * factor, 3), + "args": { + "phase": phase, + "task_id": task_id, + "func_id": function_id, + "core": core_id, + # mc 是兼容真实 merged schema 的字段名，只原样承载 flags + # bit0；它与 aux 的实际含义均需结合 phase 解读，例如 Claim + # 可表示 winner，而 Fanin/HeapGuard 的 aux 各有自己的计数语义。 + "mc": flags & 1, + "aux": auxiliary, + }, + } + if phase == "atomic": + # PollBatch 表示显式等待区内的逻辑调用次数；它的 span + # 只是 episode 包络，可能与其他 site 或直接 Atomic 交错， + # 因而绝不能伪装成一次 atomic 的 completion boundary。 + if atomic_poll_batch: + event["args"] = { + "phase": "atomic_poll_batch", + "task_id": task_id, + "func_id": function_id, + "core": core_id, + "site": atomic_site, + "site_id": atomic_site_id, + "op": atomic_op, + "op_id": atomic_op_id, + "call_count": atomic_call_count, + "poll_window_cycles": end - start, + "estimate_formula": "call_count * calibrated_atomic_cost", + "is_poll_batch": True, + "batch_semantics": "observation_load_calls", + "duration_semantics": ( + "logical_poll_episode_envelope_not_single_atomic_latency" + ), + "may_contain_interleaved_direct_atomics": True, + "flags": flags, + "execution_unit": "scalar", + } + event["cat"] = "atomic.poll_batch" + else: + # 直接 Atomic 的 flags/aux 有独立 ABI，不沿用普通 + # phase 的 mc 语义。cycles 保留原始整数，避免短 + # atomic 经微秒浮点换算后丢失 tick 精度。 + event["args"] = { + "phase": phase, + "task_id": task_id, + "func_id": function_id, + "core": core_id, + "site": atomic_site, + "site_id": atomic_site_id, + "op": atomic_op, + "op_id": atomic_op_id, + "call_count": 1, + "cycles": end - start, + "result_used": bool(flags & ATOMIC_RESULT_USED), + "return_ready_observed": bool(flags & ATOMIC_RETURN_READY), + "completion_boundary": ( + "return_value_ready" + if flags & ATOMIC_RETURN_READY + else "source_issue_bracket" + ), + "flags": flags, + "execution_unit": "scalar", + } + # 分类同样带边界，便于 Perfetto 过滤和分组；二者 + # 仍在同一 AIC/AIV scalar lane，不伪造并行执行单元。 + event["cat"] = f"atomic.{atomic_boundary_tag}" + # bit5 只对 Load 有意义；bits8..31 只对 FetchMax + # 表示饱和后的 retry 数。 + if atomic_op_id == 0: + event["args"]["value_zero"] = bool( + flags & ATOMIC_VALUE_ZERO + ) + if atomic_op_id == 3: + event["args"]["retries"] = ( + flags >> ATOMIC_PAYLOAD_SHIFT + ) & ATOMIC_PAYLOAD_MASK + elif phase == "dcci": + event["args"] = { + "phase": phase, + "task_id": task_id, + "func_id": function_id, + "core": core_id, + "site": dcci_site, + "site_id": dcci_site_id, + "op": dcci_op, + "op_id": dcci_op_id, + "call_count": dcci_call_count, + "cache_line_count": dcci_line_count, + "trailing_dsb": bool(flags & DCCI_TRAILING_DSB), + "cycles": end - start, + "execution_unit": "scalar", + "flags": flags, + } + event["cat"] = "dcci" + elif phase == "claim": + event["args"] = { + "phase": phase, + "task_id": task_id, + "func_id": function_id, + "core": core_id, + "claim_attempted": claim_attempted, + "claim_won": claim_won, + "claim_attempted_source": claim_attempted_source, + "claim_path": "alloc" if auxiliary == 1 else "kernel", + "execution_unit": "scalar", + "flags": flags, + } + event["cat"] = "scalar_scheduler" + elif phase == "clock_baseline": + dependency_hook = bool(flags & 1) + event["args"] = { + "phase": phase, + "core": core_id, + "ticks": end - start, + "clock_freq_hz": frequency_hz, + "definition": ( + "atomic-return-dependency-hook" + if dependency_hook + else "consecutive-sys-cnt-reads" + ), + "dependency_applied": bool(flags & 2) if dependency_hook else False, + "execution_unit": "scalar", + } + event["cat"] = "scalar_clock" + if trace_schema_version == 5: + # schema-v5 的 merged 只承担可视化：阶段、atomic site/op/ + # boundary、task 和 poll 次数均已编码在 name，轨道与时间由 + # pid/tid/ts/dur 给出。十列权威字段完整保留在同目录 raw， + # 不再逐事件复制近 100 MiB 的 args/cat。 + event.pop("args", None) + event.pop("cat", None) + first = _emit_event(output, event, first) + emitted += 1 + output.write("\n]}\n") + output.flush() + os.fsync(output.fileno()) + os.replace(temporary_path, output_path) + except BaseException: + temporary_path.unlink(missing_ok=True) + raise + return emitted, len(blocks), base_cycle + + +# 只解析显式 input/output，不扫描仓库 outputs，也不选择“最新”文件。 +def _parse_args() -> argparse.Namespace: + # 强制 -o 使覆盖目标可审查，避免脱仓后因 cwd 不同写到意外目录。 + parser = argparse.ArgumentParser( + description="Convert standalone PA fdwic_events JSON to a Chrome/Perfetto swimlane trace." + ) + parser.add_argument("input", type=Path, help="l2_swimlane_records.json produced by the standalone runner") + parser.add_argument("-o", "--output", type=Path, required=True, help="merged_swimlane.json output path") + return parser.parse_args() + + +# 命令行错误边界：预期的输入、格式和文件系统错误统一返回 1。 +def main() -> int: + args = _parse_args() + try: + events, blocks, base_cycle = convert(args.input, args.output) + except (OSError, ValueError, json.JSONDecodeError) as error: + # 不吞掉错误原因，但也不向普通使用者输出长 traceback；convert 已保证 + # 失败路径不会留下临时 merged 文件。 + print(f"swimlane conversion failed: {error}", file=sys.stderr) + return 1 + print( + f"[SWIMLANE] merged_json={args.output} events={events} blocks={blocks} base_cycle={base_cycle}" + ) + print(f"Open https://ui.perfetto.dev/ and load {args.output}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/atomic_probe/pa_scheduler/swimlane_exclusive_analyzer.py b/tests/atomic_probe/pa_scheduler/swimlane_exclusive_analyzer.py new file mode 100755 index 0000000000..33d008268c --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/swimlane_exclusive_analyzer.py @@ -0,0 +1,2718 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""按物理 scalar lane 生成 standalone PA Submit 的严格排他 cycle 报告。""" + +from __future__ import annotations + +import argparse +import bisect +import json +import math +import os +import sys +import tempfile +from collections import Counter, defaultdict +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +try: + # 包方式运行单测时复用同目录 converter 的完整 raw/schema 校验。 + from .swimlane_converter import ( + TASK_KIND_NAMES, + _derive_v4_task_kinds, + _load_and_validate, + _restore_v5_shared_efdrain, + _standalone_topology, + ) +except ImportError: + # 也支持直接执行本脚本，不依赖仓库安装成 Python package。 + from swimlane_converter import ( + TASK_KIND_NAMES, + _derive_v4_task_kinds, + _load_and_validate, + _restore_v5_shared_efdrain, + _standalone_topology, + ) + + +REPORT_SCHEMA_VERSION = 3 +EXPECTED_CORES = 96 +EXPECTED_AIC_CORES = 32 +EXPECTED_AIV_CORES = 64 + +# v3 的六类显式 child 保持历史口径；v4 只把 winner 的两个真实 +# 尾动作加入排他分区。loser 没有尾动作，其剩余时间属于 Submit residual。 +# Kernel 只在 EfDrain/FinalDrain 内部再次细分，不会与父区间重复相加。 +V3_EXCLUSIVE_SUBMIT_PHASES = ( + "EfDrain", + "Materialize", + "PrepareMap", + "Claim", + "Fanin", + "Register", +) +V5_TAIL_PHASES = ("WinnerBuild", "AllocComplete") +V5_EXCLUSIVE_SUBMIT_PHASES = V3_EXCLUSIVE_SUBMIT_PHASES + V5_TAIL_PHASES +PRIVATE_REQUIRED_ON_EVERY_SUBMIT = ( + "EfDrain", + "Materialize", + "PrepareMap", + "Claim", + "Register", +) +SHARED_REQUIRED_ON_EVERY_SUBMIT = ("EfDrain", "Claim") +SHARED_WINNER_ONLY_PHASES = ("Materialize", "Register") +SHARED_REGISTER_DETAIL_PHASE = "SharedRegisterPublishMetadata" +SHARED_MATERIALIZE_OUTPUT_DETAIL_PHASE = ( + "SharedMaterializePublishTaskOutputs" +) +SHARED_MATERIALIZE_OUTPUT_COPY_PHASE = ( + "SharedMaterializePublishTaskOutputsCopy" +) +SHARED_MATERIALIZE_OUTPUT_FLUSH_PHASE = ( + "SharedMaterializePublishTaskOutputsFlush" +) +LEGACY_SHARED_REGISTER_OUTPUT_DETAIL_PHASE = ( + "SharedRegisterPublishTaskOutputs" +) +LEGACY_SHARED_REGISTER_OUTPUT_COPY_PHASE = ( + "SharedRegisterPublishTaskOutputsCopy" +) +LEGACY_SHARED_REGISTER_OUTPUT_FLUSH_PHASE = ( + "SharedRegisterPublishTaskOutputsFlush" +) +SHARED_OUTPUT_DETAIL_PHASES = ( + SHARED_MATERIALIZE_OUTPUT_DETAIL_PHASE, + LEGACY_SHARED_REGISTER_OUTPUT_DETAIL_PHASE, +) +SHARED_OUTPUT_COPY_PHASES = ( + SHARED_MATERIALIZE_OUTPUT_COPY_PHASE, + LEGACY_SHARED_REGISTER_OUTPUT_COPY_PHASE, +) +SHARED_OUTPUT_FLUSH_PHASES = ( + SHARED_MATERIALIZE_OUTPUT_FLUSH_PHASE, + LEGACY_SHARED_REGISTER_OUTPUT_FLUSH_PHASE, +) +REGISTER_SERIAL_BREAKDOWN_METRICS = ( + "parent", + "register_wait_predecessor_insert", + "register_publish_metadata", + "register_publish_writer_metadata", + "register_publish_insert_completion", +) +LEGACY_REGISTER_BREAKDOWN_METRICS = ( + *REGISTER_SERIAL_BREAKDOWN_METRICS[:-1], + "register_publish_task_outputs", + "register_publish_task_outputs_copy", + "register_publish_task_outputs_flush", + "register_publish_task_outputs_residual", + "register_publish_metadata_epilogue", + "register_publish_insert_completion", +) +LEGACY_REGISTER_FLAT_PARTITION_METRICS = ( + "register_wait_predecessor_insert", + "register_publish_writer_metadata", + "register_publish_task_outputs", + "register_publish_metadata_epilogue", + "register_publish_insert_completion", +) +# 报告字段保持向后兼容；新 placement 下五个 legacy output 指标严格为 0， +# fresh-output 的非零明细只出现在 materialize_breakdown。 +REGISTER_BREAKDOWN_METRICS = LEGACY_REGISTER_BREAKDOWN_METRICS +REGISTER_FLAT_PARTITION_METRICS = ( + LEGACY_REGISTER_FLAT_PARTITION_METRICS +) +MATERIALIZE_BREAKDOWN_METRICS = ( + "parent", + "materialize_before_publish_task_outputs", + "materialize_publish_task_outputs", + "materialize_publish_task_outputs_copy", + "materialize_publish_task_outputs_flush", + "materialize_publish_task_outputs_residual", + "materialize_after_publish_task_outputs", +) +ACTOR_CYCLE_METRICS = ( + "gross", + "control", + "submit", + "efdrain_control", + "claim", + "post_claim_tail", + "post_transition", + "kernel_union", +) + +# 这些记录仍保留调用次数和 aggregate duration，但明确不进入任何闭合式。 +OVERLAY_PHASES = ( + "Atomic", + "ClockBaseline", + "Commit", + "RingBp", + "Build", + "Replay", + "Alloc", + "DrainWon", +) + +SUBMIT_PARTITION_METRICS = ( + "efdrain", + "materialize", + "prepare_map", + "claim", + "fanin", + "register", + "submit_residual", +) +V5_SUBMIT_PARTITION_METRICS = ( + *SUBMIT_PARTITION_METRICS[:-1], + "winner_build", + "alloc_complete", + "submit_residual", +) +BASE_ROLE_METRICS = ( + "submit_envelope", + "submit_union", + "between_submit_residual", + *SUBMIT_PARTITION_METRICS, + "efdrain_kernel_union", + "efdrain_control", +) +V5_PARENT_METRICS = ( + "orchestration_replay", + "orchestration_setup", + "orchestration_tail", + "final_drain", + "final_drain_kernel_union", + "final_drain_residual", + "worker_completion", +) +PHASE_TO_METRIC = { + "EfDrain": "efdrain", + "Materialize": "materialize", + "PrepareMap": "prepare_map", + "Claim": "claim", + "Fanin": "fanin", + "Register": "register", + "WinnerBuild": "winner_build", + "AllocComplete": "alloc_complete", +} +def _exclusive_phases( + trace_schema_version: int, tensormap_mode: str +) -> tuple[str, ...]: + if trace_schema_version != 5: + return V3_EXCLUSIVE_SUBMIT_PHASES + if tensormap_mode == "shared": + # shared 的 raw 已从源头禁止 PrepareMap；分析语义也不能继续把它 + # 声称为可能存在的 Submit child。 + return tuple( + phase + for phase in V5_EXCLUSIVE_SUBMIT_PHASES + if phase != "PrepareMap" + ) + return V5_EXCLUSIVE_SUBMIT_PHASES + + +def _submit_partition_metrics(trace_schema_version: int) -> tuple[str, ...]: + return ( + V5_SUBMIT_PARTITION_METRICS + if trace_schema_version == 5 + else SUBMIT_PARTITION_METRICS + ) + + +def _role_metrics(trace_schema_version: int) -> tuple[str, ...]: + if trace_schema_version == 3: + return BASE_ROLE_METRICS + return ( + "submit_envelope", + "submit_union", + "between_submit_residual", + *V5_SUBMIT_PARTITION_METRICS, + "efdrain_kernel_union", + "efdrain_control", + *V5_PARENT_METRICS, + ) + + +@dataclass(frozen=True, slots=True) +class Event: + """十列 raw ABI 的只读整数视图；row_index 用于给出可追溯错误。""" + + row_index: int + core_id: int + block_id: int + lane: int + task_id: int + function_id: int + phase: str + start_cycle: int + end_cycle: int + flags: int + auxiliary: int + + @property + def duration(self) -> int: + return self.end_cycle - self.start_cycle + + @property + def lane_key(self) -> tuple[int, int]: + # core_id 与 lane 同时作为 key，避免以后拓扑扩展时误把不同物理核合并。 + return self.core_id, self.lane + + +def _event_from_row(index: int, row: tuple[Any, ...]) -> Event: + return Event( + row_index=index, + core_id=int(row[0]), + block_id=int(row[1]), + lane=int(row[2]), + task_id=int(row[3]), + function_id=int(row[4]), + phase=str(row[5]), + start_cycle=int(row[6]), + end_cycle=int(row[7]), + flags=int(row[8]), + auxiliary=int(row[9]), + ) + + +def _overlaps(left: Event, right: Event) -> bool: + """raw span 按半开区间解释；首尾相接不算重叠，零时长 instant 不占时间。""" + + return max(left.start_cycle, right.start_cycle) < min(left.end_cycle, right.end_cycle) + + +def _interval_union_cycles(intervals: Sequence[tuple[int, int]]) -> int: + """只用整数 cycle 计算区间并集，绝不先换算成浮点微秒。""" + + if not intervals: + return 0 + ordered = sorted(intervals) + total = 0 + current_start, current_end = ordered[0] + for start, end in ordered[1:]: + if start > current_end: + total += current_end - current_start + current_start, current_end = start, end + else: + current_end = max(current_end, end) + return total + current_end - current_start + + +def _find_containing_parent( + event: Event, + parents: Sequence[Event], + parent_starts: Sequence[int], + *, + parent_name: str, +) -> Event | None: + """在已排序且互不重叠的父区间中定位唯一容器，并拒绝跨边界截断。""" + + candidate = bisect.bisect_right(parent_starts, event.start_cycle) - 1 + if candidate >= 0: + parent = parents[candidate] + if parent.start_cycle <= event.start_cycle and event.end_cycle <= parent.end_cycle: + return parent + + # 未完整包含时仍要检查相邻父区间；部分相交不能被悄悄当作“父区间外”。 + nearby = {candidate - 1, candidate, candidate + 1, candidate + 2} + for index in sorted(nearby): + if 0 <= index < len(parents) and _overlaps(event, parents[index]): + parent = parents[index] + raise ValueError( + f"row {event.row_index} {event.phase} " + f"[{event.start_cycle},{event.end_cycle}) crosses {parent_name} " + f"row {parent.row_index} [{parent.start_cycle},{parent.end_cycle})" + ) + return None + + +def _median(values: Sequence[int]) -> int | float: + ordered = sorted(values) + middle = len(ordered) // 2 + if len(ordered) % 2: + return ordered[middle] + total = ordered[middle - 1] + ordered[middle] + return total // 2 if total % 2 == 0 else total / 2 + + +def _distribution(values: Sequence[int]) -> dict[str, int | float]: + """p95 使用 nearest-rank；median 只用于横向比较，不参与整数闭合。""" + + if not values: + raise ValueError("cannot summarize an empty per-core metric") + ordered = sorted(values) + p95_index = math.ceil(0.95 * len(ordered)) - 1 + return { + "median_cycles": _median(ordered), + "p95_cycles": ordered[p95_index], + "max_cycles": ordered[-1], + } + + +def _actor_distribution(values: Sequence[int]) -> dict[str, int | float]: + """actor 汇总同时保留总量与单次分布；全部闭合仍使用整数总量。""" + + if not values: + raise ValueError("cannot summarize an empty actor class") + ordered = sorted(values) + total = sum(ordered) + p95_index = math.ceil(0.95 * len(ordered)) - 1 + return { + "sum_cycles": total, + "mean_cycles": total / len(ordered), + "median_cycles": _median(ordered), + "p95_cycles": ordered[p95_index], + } + + +def _validate_capture_identity( + trace_schema_version: int, + metadata: dict[str, Any], + events: Sequence[Event], +) -> tuple[list[str], int]: + """补足 converter 之外、排他报告必须证明的完整 96 核和 task stream 身份。""" + + if trace_schema_version not in (3, 5): + raise ValueError( + "exclusive analysis requires trace_schema_version=3 or 5 because other raw " + "does not carry producer dropped_records evidence" + ) + num_cores = int(metadata["num_cores"]) + if num_cores != EXPECTED_CORES: + raise ValueError( + f"exclusive analysis requires {EXPECTED_CORES} cores, got {num_cores}" + ) + core_types = metadata["core_types"] + expected_types = [ + "aic" if core_id < EXPECTED_AIC_CORES else "aiv" + for core_id in range(EXPECTED_CORES) + ] + if core_types != expected_types: + raise ValueError("metadata.core_types is not the complete 32 AIC + 64 AIV role map") + + summary = metadata.get("fdwic_summary") + if not isinstance(summary, dict) or int(summary.get("dropped_records", -1)) != 0: + raise ValueError("metadata.fdwic_summary.dropped_records must be exactly 0") + + observed_core_ids = {event.core_id for event in events} + expected_core_ids = set(range(EXPECTED_CORES)) + if observed_core_ids != expected_core_ids: + missing = sorted(expected_core_ids - observed_core_ids) + extra = sorted(observed_core_ids - expected_core_ids) + raise ValueError(f"raw core IDs are incomplete: missing={missing} extra={extra}") + return expected_types, num_cores + + +def _validate_and_group_submits( + events: Sequence[Event], +) -> tuple[dict[tuple[int, int], list[Event]], list[int]]: + """验证每个 core/lane 的 Submit 不重叠，且 task 0..N-1 顺序完整一致。""" + + by_lane: dict[tuple[int, int], list[Event]] = defaultdict(list) + for event in events: + if event.phase == "Submit": + if event.duration <= 0: + raise ValueError(f"row {event.row_index} Submit must have positive duration") + by_lane[event.lane_key].append(event) + + expected_lane_keys = { + (core_id, _standalone_topology(core_id)[1]) for core_id in range(EXPECTED_CORES) + } + if set(by_lane) != expected_lane_keys: + missing = sorted(expected_lane_keys - set(by_lane)) + extra = sorted(set(by_lane) - expected_lane_keys) + raise ValueError(f"Submit core/lane IDs are incomplete: missing={missing} extra={extra}") + + reference_task_ids: list[int] | None = None + for lane_key, submits in by_lane.items(): + submits.sort(key=lambda event: (event.start_cycle, event.end_cycle, event.row_index)) + for previous, current in zip(submits, submits[1:]): + if current.start_cycle < previous.end_cycle: + raise ValueError( + f"core/lane {lane_key} has overlapping Submit rows " + f"{previous.row_index} and {current.row_index}" + ) + task_ids = [event.task_id for event in submits] + if len(task_ids) != len(set(task_ids)): + raise ValueError(f"core/lane {lane_key} has duplicate Submit task IDs") + if reference_task_ids is None: + if not task_ids or task_ids != list(range(task_ids[-1] + 1)): + raise ValueError( + f"core/lane {lane_key} Submit task IDs are not contiguous 0..N-1: {task_ids}" + ) + reference_task_ids = task_ids + elif task_ids != reference_task_ids: + raise ValueError( + f"core/lane {lane_key} Submit task IDs do not match the common task stream" + ) + + assert reference_task_ids is not None + return by_lane, reference_task_ids + + +def _associate_exclusive_children( + events: Sequence[Event], + submits_by_lane: dict[tuple[int, int], list[Event]], + exclusive_phases: Sequence[str], +) -> dict[int, list[Event]]: + """把每条显式 child 严格归入同一 core/lane 上唯一包含它的 Submit。""" + + children: dict[int, list[Event]] = { + submit.row_index: [] + for submits in submits_by_lane.values() + for submit in submits + } + starts = { + lane_key: [submit.start_cycle for submit in submits] + for lane_key, submits in submits_by_lane.items() + } + exclusive_phase_set = set(exclusive_phases) + for event in events: + if event.phase not in exclusive_phase_set: + continue + parents = submits_by_lane[event.lane_key] + parent = _find_containing_parent( + event, parents, starts[event.lane_key], parent_name="Submit" + ) + if parent is None: + raise ValueError( + f"row {event.row_index} {event.phase} is outside every Submit " + f"on core/lane {event.lane_key}" + ) + # Submit 前端和真实尾动作都描述“当前 task”的 scalar 工作；仅 Kernel + # 允许在 EfDrain/FinalDrain 中执行前序 task，因此不经过这条关联路径。 + if event.task_id != parent.task_id: + raise ValueError( + f"row {event.row_index} {event.phase} task_id={event.task_id} " + f"does not match containing Submit task_id={parent.task_id}" + ) + children[parent.row_index].append(event) + return children + + +def _associate_shared_register_details( + events: Sequence[Event], + children_by_submit: dict[int, list[Event]], + trace_schema_version: int, + tensormap_mode: str, +) -> tuple[ + dict[int, Event], + dict[int, Event], + dict[int, Event], + dict[int, Event], + str, +]: + """建立 Register→metadata 与 Materialize→outputs 的严格关联。 + + ``outputs_by_owner`` 以父 row index 为键。新采集的 owner 是 + Materialize；迁移前 schema-v5 raw 的 owner 是 metadata，并通过返回的 + ``output_placement`` 显式区分，避免把历史数据悄悄套用新口径。 + """ + + registers = [ + child + for children in children_by_submit.values() + for child in children + if child.phase == "Register" + ] + materializes = [ + child + for children in children_by_submit.values() + for child in children + if child.phase == "Materialize" + ] + details = [ + event for event in events if event.phase == SHARED_REGISTER_DETAIL_PHASE + ] + output_details = [ + event + for event in events + if event.phase in SHARED_OUTPUT_DETAIL_PHASES + ] + output_copy_details = [ + event + for event in events + if event.phase in SHARED_OUTPUT_COPY_PHASES + ] + output_flush_details = [ + event + for event in events + if event.phase in SHARED_OUTPUT_FLUSH_PHASES + ] + if trace_schema_version != 5 or tensormap_mode != "shared": + if ( + details + or output_details + or output_copy_details + or output_flush_details + ): + raise ValueError( + "shared Materialize/Register details are only valid for " + "shared schema-v5" + ) + return {}, {}, {}, {}, "none" + + def _associate_unique( + nested: Sequence[Event], + parents: Sequence[Event], + *, + child_name: str, + parent_name: str, + ) -> dict[int, Event]: + parents_by_lane: dict[tuple[int, int], list[Event]] = defaultdict(list) + for parent in parents: + parents_by_lane[parent.lane_key].append(parent) + for lane_parents in parents_by_lane.values(): + lane_parents.sort( + key=lambda event: ( + event.start_cycle, + event.end_cycle, + event.row_index, + ) + ) + starts = { + lane_key: [event.start_cycle for event in lane_parents] + for lane_key, lane_parents in parents_by_lane.items() + } + association: dict[int, Event] = {} + for child in nested: + lane_parents = parents_by_lane.get(child.lane_key, []) + parent = _find_containing_parent( + child, + lane_parents, + starts.get(child.lane_key, []), + parent_name=parent_name, + ) + if parent is None: + raise ValueError( + f"row {child.row_index} {child.phase} is outside every " + f"{parent_name} on core/lane {child.lane_key}" + ) + if ( + child.core_id != parent.core_id + or child.lane != parent.lane + or child.task_id != parent.task_id + or child.function_id != parent.function_id + ): + raise ValueError( + f"row {child.row_index} {child.phase} identity does not " + f"match {parent_name} row {parent.row_index}" + ) + previous = association.setdefault(parent.row_index, child) + if previous is not child: + raise ValueError( + f"{parent_name} row {parent.row_index} has duplicate " + f"{child_name} rows {previous.row_index} and " + f"{child.row_index}" + ) + missing = [ + parent.row_index + for parent in parents + if parent.row_index not in association + ] + if missing: + raise ValueError( + "shared schema-v5 requires exactly one " + f"{child_name} per {parent_name}: " + f"missing_parent_rows={missing[:8]}" + ) + if len(association) != len(parents): + raise AssertionError( + f"shared {child_name} association is not one-to-one" + ) + return association + + detail_by_register = _associate_unique( + details, + registers, + child_name=SHARED_REGISTER_DETAIL_PHASE, + parent_name="Register", + ) + + output_placements = { + ( + "materialize" + if event.phase == SHARED_MATERIALIZE_OUTPUT_DETAIL_PHASE + else "register_legacy" + ) + for event in output_details + } + if len(output_placements) != 1: + raise ValueError( + "shared schema-v5 capture must use one task-output placement; " + f"got {sorted(output_placements)}" + ) + output_placement = next(iter(output_placements)) + output_parents = ( + materializes if output_placement == "materialize" else details + ) + output_parent_name = ( + "Materialize" + if output_placement == "materialize" + else SHARED_REGISTER_DETAIL_PHASE + ) + outputs_by_owner = _associate_unique( + output_details, + output_parents, + child_name=( + SHARED_MATERIALIZE_OUTPUT_DETAIL_PHASE + if output_placement == "materialize" + else LEGACY_SHARED_REGISTER_OUTPUT_DETAIL_PHASE + ), + parent_name=output_parent_name, + ) + + outputs_by_lane: dict[tuple[int, int], list[Event]] = defaultdict(list) + for output_detail in output_details: + outputs_by_lane[output_detail.lane_key].append(output_detail) + for lane_outputs in outputs_by_lane.values(): + lane_outputs.sort( + key=lambda event: (event.start_cycle, event.end_cycle, event.row_index) + ) + output_starts = { + lane_key: [event.start_cycle for event in lane_outputs] + for lane_key, lane_outputs in outputs_by_lane.items() + } + + def _nest_under_outputs( + children: list[Event], + phase_name: str, + association: dict[int, Event], + ) -> None: + for child in children: + lane_outputs = outputs_by_lane.get(child.lane_key, []) + parent_output = _find_containing_parent( + child, + lane_outputs, + output_starts.get(child.lane_key, []), + parent_name="task-output publication", + ) + if parent_output is None: + raise ValueError( + f"row {child.row_index} {child.phase} is outside every " + "task-output publication on core/lane " + f"{child.lane_key}" + ) + if ( + child.core_id != parent_output.core_id + or child.lane != parent_output.lane + or child.task_id != parent_output.task_id + or child.function_id != parent_output.function_id + ): + raise ValueError( + f"row {child.row_index} {child.phase} identity does not " + "match task-output publication row " + f"{parent_output.row_index}" + ) + previous = association.setdefault(parent_output.row_index, child) + if previous is not child: + raise ValueError( + "task-output publication row " + f"{parent_output.row_index} has duplicate {phase_name} rows " + f"{previous.row_index} and {child.row_index}" + ) + missing = [ + output_detail.row_index + for output_detail in output_details + if output_detail.row_index not in association + ] + if missing: + raise ValueError( + "shared schema-v5 requires exactly one " + f"{phase_name} per task-output publication: " + f"missing_output_rows={missing[:8]}" + ) + if len(association) != len(output_details): + raise AssertionError( + f"shared {phase_name} association is not one-to-one" + ) + + copies_by_outputs: dict[int, Event] = {} + flushes_by_outputs: dict[int, Event] = {} + expected_copy_phase = ( + SHARED_MATERIALIZE_OUTPUT_COPY_PHASE + if output_placement == "materialize" + else LEGACY_SHARED_REGISTER_OUTPUT_COPY_PHASE + ) + expected_flush_phase = ( + SHARED_MATERIALIZE_OUTPUT_FLUSH_PHASE + if output_placement == "materialize" + else LEGACY_SHARED_REGISTER_OUTPUT_FLUSH_PHASE + ) + if any( + event.phase != expected_copy_phase + for event in output_copy_details + ) or any( + event.phase != expected_flush_phase + for event in output_flush_details + ): + raise ValueError( + "shared schema-v5 task-output parent/copy/flush phase families " + "must not be mixed" + ) + _nest_under_outputs( + output_copy_details, + expected_copy_phase, + copies_by_outputs, + ) + _nest_under_outputs( + output_flush_details, + expected_flush_phase, + flushes_by_outputs, + ) + for output_row, copy_event in copies_by_outputs.items(): + flush_event = flushes_by_outputs[output_row] + output_event = next( + event for event in output_details if event.row_index == output_row + ) + if not ( + output_event.start_cycle + <= copy_event.start_cycle + <= copy_event.end_cycle + == flush_event.start_cycle + <= flush_event.end_cycle + <= output_event.end_cycle + ): + raise ValueError( + f"task-output publication row {output_row} " + "copy/flush nesting is invalid" + ) + return ( + detail_by_register, + outputs_by_owner, + copies_by_outputs, + flushes_by_outputs, + output_placement, + ) + + +def _build_register_breakdown( + children_by_submit: dict[int, list[Event]], + detail_by_register: dict[int, Event], + outputs_by_owner: dict[int, Event], + copies_by_outputs: dict[int, Event], + flushes_by_outputs: dict[int, Event], + output_placement: str, + num_cores: int, + trace_schema_version: int, + tensormap_mode: str, +) -> dict[str, Any] | None: + """由 raw 整数边界构造 Register 串行区排他闭合报告。""" + + if trace_schema_version != 5 or tensormap_mode != "shared": + return None + + registers_by_core: dict[int, list[Event]] = defaultdict(list) + for children in children_by_submit.values(): + for child in children: + if child.phase == "Register": + registers_by_core[child.core_id].append(child) + + per_core: list[dict[str, Any]] = [] + for core_id in range(num_cores): + metrics = {name: 0 for name in REGISTER_BREAKDOWN_METRICS} + register_count = 0 + for parent in registers_by_core.get(core_id, []): + detail = detail_by_register[parent.row_index] + wait_cycles = detail.start_cycle - parent.start_cycle + publish_cycles = detail.duration + if output_placement == "register_legacy": + output_detail = outputs_by_owner[detail.row_index] + copy_detail = copies_by_outputs[output_detail.row_index] + flush_detail = flushes_by_outputs[output_detail.row_index] + writer_metadata_cycles = ( + output_detail.start_cycle - detail.start_cycle + ) + task_outputs_cycles = output_detail.duration + copy_cycles = copy_detail.duration + flush_cycles = flush_detail.duration + outputs_residual_cycles = ( + task_outputs_cycles - copy_cycles - flush_cycles + ) + metadata_epilogue_cycles = ( + detail.end_cycle - output_detail.end_cycle + ) + else: + writer_metadata_cycles = publish_cycles + task_outputs_cycles = 0 + copy_cycles = 0 + flush_cycles = 0 + outputs_residual_cycles = 0 + metadata_epilogue_cycles = 0 + handoff_cycles = parent.end_cycle - detail.end_cycle + if min( + wait_cycles, + publish_cycles, + writer_metadata_cycles, + task_outputs_cycles, + copy_cycles, + flush_cycles, + outputs_residual_cycles, + metadata_epilogue_cycles, + handoff_cycles, + ) < 0: + raise ValueError( + f"Register row {parent.row_index} internal partition has " + "a negative raw-cycle segment" + ) + if ( + writer_metadata_cycles + + task_outputs_cycles + + metadata_epilogue_cycles + != publish_cycles + ): + raise ValueError( + f"{SHARED_REGISTER_DETAIL_PHASE} row {detail.row_index} " + "internal partition does not close in raw cycles" + ) + if ( + copy_cycles + flush_cycles + outputs_residual_cycles + != task_outputs_cycles + ): + raise ValueError( + "legacy Register task-output copy/flush residual does not " + "close in raw cycles" + ) + if wait_cycles + publish_cycles + handoff_cycles != parent.duration: + raise ValueError( + f"Register row {parent.row_index} internal partition does not " + "close in raw cycles" + ) + metrics["parent"] += parent.duration + metrics["register_wait_predecessor_insert"] += wait_cycles + metrics["register_publish_metadata"] += publish_cycles + metrics["register_publish_writer_metadata"] += ( + writer_metadata_cycles + ) + metrics["register_publish_task_outputs"] += task_outputs_cycles + metrics["register_publish_task_outputs_copy"] += copy_cycles + metrics["register_publish_task_outputs_flush"] += flush_cycles + metrics["register_publish_task_outputs_residual"] += ( + outputs_residual_cycles + ) + metrics["register_publish_metadata_epilogue"] += ( + metadata_epilogue_cycles + ) + metrics["register_publish_insert_completion"] += handoff_cycles + register_count += 1 + + metadata_children = ( + metrics["register_publish_writer_metadata"] + + metrics["register_publish_task_outputs"] + + metrics["register_publish_metadata_epilogue"] + ) + flat_children = sum( + metrics[name] for name in REGISTER_FLAT_PARTITION_METRICS + ) + outputs_children = ( + metrics["register_publish_task_outputs_copy"] + + metrics["register_publish_task_outputs_flush"] + + metrics["register_publish_task_outputs_residual"] + ) + if metadata_children != metrics["register_publish_metadata"]: + raise ValueError( + f"core {core_id} aggregate metadata partition does not close" + ) + if flat_children != metrics["parent"]: + raise ValueError( + f"core {core_id} aggregate Register partition does not close" + ) + if outputs_children != metrics["register_publish_task_outputs"]: + raise ValueError( + f"core {core_id} aggregate task-outputs partition does not close" + ) + block_id, lane, role = _standalone_topology(core_id) + per_core.append( + { + "core_id": core_id, + "block_id": block_id, + "lane": lane, + "role": role, + "register_count": register_count, + "metrics_cycles": metrics, + "closure": { + "register": { + "parent_cycles": metrics["parent"], + "flat_children_cycles": flat_children, + "exact": True, + }, + "metadata": { + "parent_cycles": metrics[ + "register_publish_metadata" + ], + "children_cycles": metadata_children, + "exact": True, + }, + "task_outputs": { + "parent_cycles": metrics[ + "register_publish_task_outputs" + ], + "children_cycles": outputs_children, + "exact": True, + }, + }, + } + ) + + aggregate = { + metric: sum(core["metrics_cycles"][metric] for core in per_core) + for metric in REGISTER_BREAKDOWN_METRICS + } + aggregate_metadata_children = ( + aggregate["register_publish_writer_metadata"] + + aggregate["register_publish_task_outputs"] + + aggregate["register_publish_metadata_epilogue"] + ) + aggregate_flat_children = sum( + aggregate[name] for name in REGISTER_FLAT_PARTITION_METRICS + ) + aggregate_outputs_children = ( + aggregate["register_publish_task_outputs_copy"] + + aggregate["register_publish_task_outputs_flush"] + + aggregate["register_publish_task_outputs_residual"] + ) + if aggregate_metadata_children != aggregate["register_publish_metadata"]: + raise AssertionError("aggregate metadata internal partition does not close") + if aggregate_flat_children != aggregate["parent"]: + raise AssertionError("aggregate Register internal partition does not close") + if ( + aggregate_outputs_children + != aggregate["register_publish_task_outputs"] + ): + raise AssertionError( + "aggregate task-outputs internal partition does not close" + ) + + role_statistics: dict[str, Any] = {} + for role, expected_count in ( + ("aic", EXPECTED_AIC_CORES), + ("aiv", EXPECTED_AIV_CORES), + ): + role_cores = [core for core in per_core if core["role"] == role] + if len(role_cores) != expected_count: + raise ValueError( + f"Register breakdown role {role} has {len(role_cores)} cores, " + f"expected {expected_count}" + ) + role_statistics[role] = { + "core_count": len(role_cores), + "metrics": { + metric: _distribution( + [core["metrics_cycles"][metric] for core in role_cores] + ) + for metric in REGISTER_BREAKDOWN_METRICS + }, + } + + return { + "semantics": { + "parent": "the existing exclusive Register span", + "register_wait_predecessor_insert": ( + "Register.start to SharedRegisterPublishMetadata.start; " + "task 0 enters directly, while task N waits for task N-1 to " + "publish its TensorMap insertion completion" + ), + "register_publish_metadata": ( + "the SharedRegisterPublishMetadata raw parent detail" + ), + "register_publish_writer_metadata": ( + "the serialized ordinary/symbol writer publication. In new " + "captures this is the complete SharedRegisterPublishMetadata " + "span; legacy captures end at task-output publication start" + ), + "register_publish_task_outputs": ( + "legacy compatibility field; exactly zero when output_placement " + "is materialize" + ), + "register_publish_task_outputs_copy": ( + "SharedRegisterPublishTaskOutputsCopy; batch TensorDesc copy " + "into shared_outputs[task].tensors[]" + ), + "register_publish_task_outputs_flush": ( + "SharedRegisterPublishTaskOutputsFlush; FlushRegion of the " + "copied descriptors before StoreBarrier/published" + ), + "register_publish_task_outputs_residual": ( + "task-outputs envelope minus copy/flush: pre-check, " + "last_writer FetchMax, StoreBarrier, and published Exchange" + ), + "register_publish_metadata_epilogue": ( + "legacy compatibility field after Register-owned output " + "publication; exactly zero in new captures" + ), + "register_publish_insert_completion": ( + "SharedRegisterPublishMetadata.end to Register.end; publish this " + "task's TensorMap insertion completion for its successor" + ), + "raw_arithmetic": "integer cycle boundaries; merged swimlane is not read", + "output_placement": output_placement, + "included_in_submit_additive_totals": { + "parent": True, + SHARED_REGISTER_DETAIL_PHASE: False, + LEGACY_SHARED_REGISTER_OUTPUT_DETAIL_PHASE: False, + LEGACY_SHARED_REGISTER_OUTPUT_COPY_PHASE: False, + LEGACY_SHARED_REGISTER_OUTPUT_FLUSH_PHASE: False, + }, + }, + "event_count": { + "metadata": len(detail_by_register), + "task_outputs": ( + len(outputs_by_owner) + if output_placement == "register_legacy" + else 0 + ), + "task_outputs_copy": ( + len(copies_by_outputs) + if output_placement == "register_legacy" + else 0 + ), + "task_outputs_flush": ( + len(flushes_by_outputs) + if output_placement == "register_legacy" + else 0 + ), + }, + "aggregate_core_work": { + "metrics_cycles": aggregate, + "closure": { + "register": { + "parent_cycles": aggregate["parent"], + "flat_children_cycles": aggregate_flat_children, + "exact": True, + }, + "metadata": { + "parent_cycles": aggregate["register_publish_metadata"], + "children_cycles": aggregate_metadata_children, + "exact": True, + }, + "task_outputs": { + "parent_cycles": aggregate[ + "register_publish_task_outputs" + ], + "children_cycles": aggregate_outputs_children, + "exact": True, + }, + }, + }, + "per_role_core_statistics": role_statistics, + "per_core": per_core, + } + + +def _build_materialize_breakdown( + children_by_submit: dict[int, list[Event]], + outputs_by_owner: dict[int, Event], + copies_by_outputs: dict[int, Event], + flushes_by_outputs: dict[int, Event], + output_placement: str, + num_cores: int, + trace_schema_version: int, + tensormap_mode: str, +) -> dict[str, Any] | None: + """闭合新路径的 Materialize→task outputs→copy/flush 两级分区。""" + + if ( + trace_schema_version != 5 + or tensormap_mode != "shared" + or output_placement != "materialize" + ): + return None + + materializes_by_core: dict[int, list[Event]] = defaultdict(list) + for children in children_by_submit.values(): + for child in children: + if child.phase == "Materialize": + materializes_by_core[child.core_id].append(child) + + per_core: list[dict[str, Any]] = [] + for core_id in range(num_cores): + metrics = { + name: 0 for name in MATERIALIZE_BREAKDOWN_METRICS + } + materialize_count = 0 + for parent in materializes_by_core.get(core_id, []): + output = outputs_by_owner[parent.row_index] + copy = copies_by_outputs[output.row_index] + flush = flushes_by_outputs[output.row_index] + before_cycles = output.start_cycle - parent.start_cycle + output_cycles = output.duration + copy_cycles = copy.duration + flush_cycles = flush.duration + output_residual_cycles = ( + output_cycles - copy_cycles - flush_cycles + ) + after_cycles = parent.end_cycle - output.end_cycle + if min( + before_cycles, + output_cycles, + copy_cycles, + flush_cycles, + output_residual_cycles, + after_cycles, + ) < 0: + raise ValueError( + f"Materialize row {parent.row_index} internal partition " + "has a negative raw-cycle segment" + ) + if ( + copy_cycles + flush_cycles + output_residual_cycles + != output_cycles + ): + raise ValueError( + f"{SHARED_MATERIALIZE_OUTPUT_DETAIL_PHASE} row " + f"{output.row_index} does not close" + ) + if ( + before_cycles + output_cycles + after_cycles + != parent.duration + ): + raise ValueError( + f"Materialize row {parent.row_index} does not close" + ) + metrics["parent"] += parent.duration + metrics[ + "materialize_before_publish_task_outputs" + ] += before_cycles + metrics["materialize_publish_task_outputs"] += output_cycles + metrics[ + "materialize_publish_task_outputs_copy" + ] += copy_cycles + metrics[ + "materialize_publish_task_outputs_flush" + ] += flush_cycles + metrics[ + "materialize_publish_task_outputs_residual" + ] += output_residual_cycles + metrics[ + "materialize_after_publish_task_outputs" + ] += after_cycles + materialize_count += 1 + + flat_children = ( + metrics["materialize_before_publish_task_outputs"] + + metrics["materialize_publish_task_outputs"] + + metrics["materialize_after_publish_task_outputs"] + ) + output_children = ( + metrics["materialize_publish_task_outputs_copy"] + + metrics["materialize_publish_task_outputs_flush"] + + metrics["materialize_publish_task_outputs_residual"] + ) + if flat_children != metrics["parent"]: + raise ValueError( + f"core {core_id} aggregate Materialize partition does not close" + ) + if ( + output_children + != metrics["materialize_publish_task_outputs"] + ): + raise ValueError( + f"core {core_id} aggregate task-output partition does not close" + ) + block_id, lane, role = _standalone_topology(core_id) + per_core.append( + { + "core_id": core_id, + "block_id": block_id, + "lane": lane, + "role": role, + "materialize_count": materialize_count, + "metrics_cycles": metrics, + "closure": { + "materialize": { + "parent_cycles": metrics["parent"], + "flat_children_cycles": flat_children, + "exact": True, + }, + "task_outputs": { + "parent_cycles": metrics[ + "materialize_publish_task_outputs" + ], + "children_cycles": output_children, + "exact": True, + }, + }, + } + ) + + aggregate = { + metric: sum(core["metrics_cycles"][metric] for core in per_core) + for metric in MATERIALIZE_BREAKDOWN_METRICS + } + aggregate_flat_children = ( + aggregate["materialize_before_publish_task_outputs"] + + aggregate["materialize_publish_task_outputs"] + + aggregate["materialize_after_publish_task_outputs"] + ) + aggregate_output_children = ( + aggregate["materialize_publish_task_outputs_copy"] + + aggregate["materialize_publish_task_outputs_flush"] + + aggregate["materialize_publish_task_outputs_residual"] + ) + if aggregate_flat_children != aggregate["parent"]: + raise AssertionError( + "aggregate Materialize internal partition does not close" + ) + if ( + aggregate_output_children + != aggregate["materialize_publish_task_outputs"] + ): + raise AssertionError( + "aggregate Materialize task-output partition does not close" + ) + + role_statistics: dict[str, Any] = {} + for role, expected_count in ( + ("aic", EXPECTED_AIC_CORES), + ("aiv", EXPECTED_AIV_CORES), + ): + role_cores = [core for core in per_core if core["role"] == role] + if len(role_cores) != expected_count: + raise ValueError( + f"Materialize breakdown role {role} has " + f"{len(role_cores)} cores, expected {expected_count}" + ) + role_statistics[role] = { + "core_count": len(role_cores), + "metrics": { + metric: _distribution( + [ + core["metrics_cycles"][metric] + for core in role_cores + ] + ) + for metric in MATERIALIZE_BREAKDOWN_METRICS + }, + } + + return { + "semantics": { + "parent": "the existing exclusive Materialize span", + "materialize_before_publish_task_outputs": ( + "Materialize.start to output-publication.start; task " + "materialization plus writer-delta preparation" + ), + "materialize_publish_task_outputs": ( + f"the unique {SHARED_MATERIALIZE_OUTPUT_DETAIL_PHASE} raw detail" + ), + "materialize_publish_task_outputs_copy": ( + f"{SHARED_MATERIALIZE_OUTPUT_COPY_PHASE}; batch TensorDesc copy " + "into shared_outputs[task].tensors[]" + ), + "materialize_publish_task_outputs_flush": ( + f"{SHARED_MATERIALIZE_OUTPUT_FLUSH_PHASE}; FlushRegion before " + "StoreBarrier/published" + ), + "materialize_publish_task_outputs_residual": ( + "output envelope minus copy/flush: pre-check, last_writer " + "FetchMax, StoreBarrier, published Exchange, and helper overhead" + ), + "materialize_after_publish_task_outputs": ( + "output-publication.end to Materialize.end; phase closure and " + "outer timestamp" + ), + "raw_arithmetic": ( + "integer cycle boundaries; merged swimlane is not read" + ), + "included_in_submit_additive_totals": { + "parent": True, + SHARED_MATERIALIZE_OUTPUT_DETAIL_PHASE: False, + SHARED_MATERIALIZE_OUTPUT_COPY_PHASE: False, + SHARED_MATERIALIZE_OUTPUT_FLUSH_PHASE: False, + }, + }, + "event_count": { + "task_outputs": len(outputs_by_owner), + "task_outputs_copy": len(copies_by_outputs), + "task_outputs_flush": len(flushes_by_outputs), + }, + "aggregate_core_work": { + "metrics_cycles": aggregate, + "closure": { + "materialize": { + "parent_cycles": aggregate["parent"], + "flat_children_cycles": aggregate_flat_children, + "exact": True, + }, + "task_outputs": { + "parent_cycles": aggregate[ + "materialize_publish_task_outputs" + ], + "children_cycles": aggregate_output_children, + "exact": True, + }, + }, + }, + "per_role_core_statistics": role_statistics, + "per_core": per_core, + } + + +def _associate_kernels_to_parents( + events: Sequence[Event], + parents_by_lane: dict[tuple[int, int], list[Event]], + *, + parent_name: str, +) -> tuple[dict[int, list[Event]], set[int]]: + """给一类互不重叠的父 span 建立 Kernel 包含关系，并返回被消费的 row ID。""" + + for parents in parents_by_lane.values(): + parents.sort(key=lambda event: (event.start_cycle, event.end_cycle, event.row_index)) + starts = { + lane_key: [event.start_cycle for event in parents] + for lane_key, parents in parents_by_lane.items() + } + kernels_by_parent: dict[int, list[Event]] = { + parent.row_index: [] + for parents in parents_by_lane.values() + for parent in parents + } + contained_rows: set[int] = set() + for kernel in (event for event in events if event.phase == "Kernel"): + parents = parents_by_lane.get(kernel.lane_key, []) + parent = _find_containing_parent( + kernel, + parents, + starts.get(kernel.lane_key, []), + parent_name=parent_name, + ) + if parent is not None: + kernels_by_parent[parent.row_index].append(kernel) + contained_rows.add(kernel.row_index) + return kernels_by_parent, contained_rows + + +def _associate_efdrain_kernels( + events: Sequence[Event], + children_by_submit: dict[int, list[Event]], +) -> tuple[dict[int, list[Event]], set[int]]: + """只把完整包含于 EfDrain 的 Kernel 纳入其 nested union。""" + + efdrains_by_lane: dict[tuple[int, int], list[Event]] = defaultdict(list) + for children in children_by_submit.values(): + for event in children: + if event.phase == "EfDrain": + efdrains_by_lane[event.lane_key].append(event) + return _associate_kernels_to_parents( + events, efdrains_by_lane, parent_name="EfDrain" + ) + + +def _associate_v4_tail_kernels( + events: Sequence[Event], + children_by_submit: dict[int, list[Event]], +) -> tuple[dict[int, list[Event]], set[int], dict[str, int]]: + """把背压期间执行的 Kernel 唯一归入 WinnerBuild/AllocComplete 父动作。""" + + tails_by_lane: dict[tuple[int, int], list[Event]] = defaultdict(list) + tail_phase_by_row: dict[int, str] = {} + for children in children_by_submit.values(): + for event in children: + if event.phase in ("WinnerBuild", "AllocComplete"): + tails_by_lane[event.lane_key].append(event) + tail_phase_by_row[event.row_index] = event.phase + kernels_by_tail, contained_rows = _associate_kernels_to_parents( + events, tails_by_lane, parent_name="Submit tail" + ) + counts = {"WinnerBuild": 0, "AllocComplete": 0} + for parent_row, kernels in kernels_by_tail.items(): + counts[tail_phase_by_row[parent_row]] += len(kernels) + return kernels_by_tail, contained_rows, counts + + +def _group_v4_parents( + events: Sequence[Event], phase: str +) -> dict[tuple[int, int], list[Event]]: + """converter 已校验每核恰一条；这里保留列表形状以复用区间定位函数。""" + + parents: dict[tuple[int, int], list[Event]] = defaultdict(list) + for event in events: + if event.phase == phase: + parents[event.lane_key].append(event) + expected_keys = { + (core_id, _standalone_topology(core_id)[1]) for core_id in range(EXPECTED_CORES) + } + if set(parents) != expected_keys or any(len(items) != 1 for items in parents.values()): + raise ValueError(f"schema-v5 requires exactly one {phase} per core/lane") + return parents + + +def _validate_submit_frontend_contract( + core_id: int, + trace_schema_version: int, + tensormap_mode: str, + submit: Event, + counts: Counter[str], +) -> None: + """按 TensorMap 模式校验 Submit 前端 child 的基数。""" + + winner = bool(submit.flags & 1) + if trace_schema_version == 5 and tensormap_mode == "shared": + for phase in SHARED_REQUIRED_ON_EVERY_SUBMIT: + if counts[phase] != 1: + raise ValueError( + f"core {core_id} task {submit.task_id} shared path requires " + f"exactly one {phase}, got {counts[phase]}" + ) + for phase in SHARED_WINNER_ONLY_PHASES: + expected_count = 1 if winner else 0 + if counts[phase] != expected_count: + raise ValueError( + f"core {core_id} task {submit.task_id} shared " + f"{'winner' if winner else 'loser'} path requires " + f"{expected_count} {phase} spans, got {counts[phase]}" + ) + if counts["PrepareMap"] != 0: + raise ValueError( + f"core {core_id} task {submit.task_id} shared path forbids PrepareMap" + ) + else: + for phase in PRIVATE_REQUIRED_ON_EVERY_SUBMIT: + if counts[phase] != 1: + raise ValueError( + f"core {core_id} task {submit.task_id} requires exactly one " + f"{phase}, got {counts[phase]}" + ) + if counts["Fanin"] > 1: + raise ValueError( + f"core {core_id} task {submit.task_id} has {counts['Fanin']} Fanin spans" + ) + + +def _analyze_core( + core_id: int, + role: str, + trace_schema_version: int, + tensormap_mode: str, + submits: Sequence[Event], + children_by_submit: dict[int, list[Event]], + kernels_by_efdrain: dict[int, list[Event]], + orchestration: Event | None, + final_drain: Event | None, + kernels_by_final_drain: dict[int, list[Event]], +) -> dict[str, Any]: + """逐 Submit 做整数闭合；v4 继续闭合两个父 span 与 worker completion。""" + + role_metric_names = _role_metrics(trace_schema_version) + submit_partition_names = _submit_partition_metrics(trace_schema_version) + metrics = {name: 0 for name in role_metric_names} + for submit in submits: + children = sorted( + children_by_submit[submit.row_index], + key=lambda event: (event.start_cycle, event.end_cycle, event.row_index), + ) + counts = Counter(event.phase for event in children) + winner = bool(submit.flags & 1) + is_alloc = bool(submit.auxiliary) + _validate_submit_frontend_contract( + core_id, + trace_schema_version, + tensormap_mode, + submit, + counts, + ) + if trace_schema_version == 5: + tail_count = sum(counts[phase] for phase in V5_TAIL_PHASES) + expected_tail_count = 1 if winner else 0 + if tail_count != expected_tail_count or any( + counts[phase] > 1 for phase in V5_TAIL_PHASES + ): + raise ValueError( + f"core {core_id} task {submit.task_id} requires " + f"{expected_tail_count} WinnerBuild/AllocComplete tails" + ) + tail = next( + (event for event in children if event.phase in V5_TAIL_PHASES), + None, + ) + if tail is not None: + latest_frontend_end = max( + event.end_cycle + for event in children + if event.phase not in V5_TAIL_PHASES + ) + if tail.start_cycle < latest_frontend_end: + raise ValueError( + f"core {core_id} task {submit.task_id} {tail.phase} must start " + "at or after every preceding frontend child" + ) + expected_tail = "AllocComplete" if is_alloc else "WinnerBuild" + if tail.phase != expected_tail: + raise ValueError( + f"core {core_id} task {submit.task_id} expected {expected_tail}, " + f"got {tail.phase}" + ) + expected_fanin_count = 1 if winner and not is_alloc else 0 + if counts["Fanin"] != expected_fanin_count: + raise ValueError( + f"core {core_id} task {submit.task_id} " + f"{'winner' if winner else 'loser'} path requires " + f"{expected_fanin_count} Fanin spans, got {counts['Fanin']}" + ) + for previous, current in zip(children, children[1:]): + if current.start_cycle < previous.end_cycle: + raise ValueError( + f"core {core_id} task {submit.task_id} has overlapping exclusive children " + f"rows {previous.row_index} ({previous.phase}) and " + f"{current.row_index} ({current.phase})" + ) + + child_cycles = 0 + efdrain: Event | None = None + for child in children: + duration = child.duration + if duration < 0: + raise AssertionError("converter accepted a negative child duration") + metrics[PHASE_TO_METRIC[child.phase]] += duration + child_cycles += duration + if child.phase == "EfDrain": + efdrain = child + residual = submit.duration - child_cycles + if residual < 0 or child_cycles + residual != submit.duration: + raise ValueError( + f"core {core_id} task {submit.task_id} Submit partition does not close in raw cycles" + ) + metrics["submit_union"] += submit.duration + metrics["submit_residual"] += residual + + assert efdrain is not None + kernel_union = _interval_union_cycles( + [ + (kernel.start_cycle, kernel.end_cycle) + for kernel in kernels_by_efdrain[efdrain.row_index] + ] + ) + control = efdrain.duration - kernel_union + if control < 0 or kernel_union + control != efdrain.duration: + raise ValueError( + f"core {core_id} task {submit.task_id} EfDrain partition does not close in raw cycles" + ) + metrics["efdrain_kernel_union"] += kernel_union + metrics["efdrain_control"] += control + + first_start = submits[0].start_cycle + last_end = submits[-1].end_cycle + between = sum( + current.start_cycle - previous.end_cycle + for previous, current in zip(submits, submits[1:]) + ) + envelope = last_end - first_start + metrics["between_submit_residual"] = between + metrics["submit_envelope"] = envelope + if metrics["submit_union"] + between != envelope: + raise ValueError(f"core {core_id} first/last Submit envelope does not close") + if sum(metrics[name] for name in submit_partition_names) != metrics["submit_union"]: + raise ValueError(f"core {core_id} aggregate Submit partition does not close") + if ( + metrics["efdrain_kernel_union"] + metrics["efdrain_control"] + != metrics["efdrain"] + ): + raise ValueError(f"core {core_id} aggregate EfDrain partition does not close") + + worker_start: int | None = None + worker_end: int | None = None + if trace_schema_version == 5: + if orchestration is None or final_drain is None: + raise ValueError(f"core {core_id} is missing schema-v5 parent spans") + if orchestration.end_cycle != final_drain.start_cycle: + raise ValueError( + f"core {core_id} OrchestrationReplay.end must equal FinalDrain.start" + ) + if not ( + orchestration.start_cycle <= first_start + and last_end <= orchestration.end_cycle + ): + raise ValueError( + f"core {core_id} Submit envelope is outside OrchestrationReplay" + ) + setup = first_start - orchestration.start_cycle + tail = orchestration.end_cycle - last_end + metrics["orchestration_setup"] = setup + metrics["orchestration_tail"] = tail + metrics["orchestration_replay"] = orchestration.duration + orchestration_children = ( + setup + metrics["submit_union"] + + metrics["between_submit_residual"] + tail + ) + if orchestration_children != orchestration.duration: + raise ValueError(f"core {core_id} OrchestrationReplay partition does not close") + + final_kernel_union = _interval_union_cycles( + [ + (kernel.start_cycle, kernel.end_cycle) + for kernel in kernels_by_final_drain[final_drain.row_index] + ] + ) + final_residual = final_drain.duration - final_kernel_union + if final_residual < 0 or final_kernel_union + final_residual != final_drain.duration: + raise ValueError(f"core {core_id} FinalDrain partition does not close") + metrics["final_drain"] = final_drain.duration + metrics["final_drain_kernel_union"] = final_kernel_union + metrics["final_drain_residual"] = final_residual + metrics["worker_completion"] = final_drain.end_cycle - orchestration.start_cycle + if ( + orchestration.duration + final_drain.duration + != metrics["worker_completion"] + ): + raise ValueError(f"core {core_id} WorkerCompletion partition does not close") + worker_start = orchestration.start_cycle + worker_end = final_drain.end_cycle + + block_id, lane, expected_role = _standalone_topology(core_id) + if role != expected_role: + raise AssertionError("role passed to _analyze_core disagrees with standalone topology") + result = { + "core_id": core_id, + "block_id": block_id, + "lane": lane, + "role": role, + "submit_count": len(submits), + "first_submit_start_cycle": first_start, + "last_submit_end_cycle": last_end, + "metrics_cycles": metrics, + } + if trace_schema_version == 5: + result["worker_completion_start_cycle"] = worker_start + result["worker_completion_end_cycle"] = worker_end + return result + + +def _build_v5_actor_closure( + submits_by_lane: dict[tuple[int, int], list[Event]], + children_by_submit: dict[int, list[Event]], + kernels_by_efdrain: dict[int, list[Event]], + orchestrations_by_lane: dict[tuple[int, int], list[Event]], + *, + core_count: int, + task_count_per_core: int, +) -> dict[str, Any]: + """按当前 task 的 Submit 加其后 transition 统计 winner/loser actor。 + + ``Submit`` 的结束点会随源码边界移动，不能单独用来衡量 loser。actor + 固定从本次 Submit.start 延伸到下一次 Submit.start；末 task 延伸到 + OrchestrationReplay.end。这样把工作从 Submit 尾搬到 transition 只会在 + 两个明细间移动，不会改变 gross/control。 + """ + + buckets: dict[str, dict[str, Any]] = { + actor_class: { + "metrics": { + metric: [] for metric in ACTOR_CYCLE_METRICS + }, + "kernel_event_count": 0, + "actors_with_kernel": 0, + } + for actor_class in ("winner", "loser") + } + orchestration_replay_cycles = 0 + orchestration_setup_cycles = 0 + actor_count = 0 + + for lane_key, submits in submits_by_lane.items(): + orchestration_rows = orchestrations_by_lane.get(lane_key, []) + if len(orchestration_rows) != 1: + raise ValueError( + "schema-v5 actor closure requires exactly one " + f"OrchestrationReplay on core/lane {lane_key}" + ) + orchestration = orchestration_rows[0] + orchestration_replay_cycles += orchestration.duration + setup = submits[0].start_cycle - orchestration.start_cycle + if setup < 0: + raise ValueError( + f"core/lane {lane_key} actor setup starts before OrchestrationReplay" + ) + orchestration_setup_cycles += setup + + for index, submit in enumerate(submits): + actor_end = ( + submits[index + 1].start_cycle + if index + 1 < len(submits) + else orchestration.end_cycle + ) + post_transition = actor_end - submit.end_cycle + gross = actor_end - submit.start_cycle + if post_transition < 0 or gross != submit.duration + post_transition: + raise ValueError( + f"core/lane {lane_key} task {submit.task_id} " + "Submit + post-transition actor window does not close" + ) + + children = children_by_submit[submit.row_index] + efdrains = [ + child for child in children if child.phase == "EfDrain" + ] + claims = [ + child for child in children if child.phase == "Claim" + ] + if len(efdrains) != 1 or len(claims) != 1: + raise ValueError( + f"core/lane {lane_key} task {submit.task_id} actor " + "requires exactly one EfDrain and one Claim" + ) + efdrain = efdrains[0] + claim = claims[0] + if efdrain.end_cycle > claim.start_cycle: + raise ValueError( + f"core/lane {lane_key} task {submit.task_id} " + "EfDrain must finish before Claim starts" + ) + + kernels = kernels_by_efdrain[efdrain.row_index] + kernel_union = _interval_union_cycles( + [ + (kernel.start_cycle, kernel.end_cycle) + for kernel in kernels + ] + ) + efdrain_control = efdrain.duration - kernel_union + post_claim_tail = ( + submit.duration - efdrain.duration - claim.duration + ) + control = gross - kernel_union + if efdrain_control < 0 or post_claim_tail < 0 or control < 0: + raise ValueError( + f"core/lane {lane_key} task {submit.task_id} " + "actor control partition has a negative component" + ) + if ( + control + != efdrain_control + + claim.duration + + post_claim_tail + + post_transition + ): + raise ValueError( + f"core/lane {lane_key} task {submit.task_id} " + "actor control partition does not close" + ) + if gross != control + kernel_union: + raise ValueError( + f"core/lane {lane_key} task {submit.task_id} " + "actor KernelUnion partition does not close" + ) + + actor_class = "winner" if submit.flags & 1 else "loser" + bucket = buckets[actor_class] + values = { + "gross": gross, + "control": control, + "submit": submit.duration, + "efdrain_control": efdrain_control, + "claim": claim.duration, + "post_claim_tail": post_claim_tail, + "post_transition": post_transition, + "kernel_union": kernel_union, + } + for metric, value in values.items(): + bucket["metrics"][metric].append(value) + bucket["kernel_event_count"] += len(kernels) + bucket["actors_with_kernel"] += int(bool(kernels)) + actor_count += 1 + + expected_actor_count = core_count * task_count_per_core + if actor_count != expected_actor_count: + raise AssertionError( + "winner/loser actor count does not match core × task identity" + ) + + actors: dict[str, Any] = {} + for actor_class, bucket in buckets.items(): + metric_values = bucket["metrics"] + actor_class_count = len(metric_values["gross"]) + metrics = { + metric: _actor_distribution(metric_values[metric]) + for metric in ACTOR_CYCLE_METRICS + } + gross_sum = int(metrics["gross"]["sum_cycles"]) + control_sum = int(metrics["control"]["sum_cycles"]) + submit_sum = int(metrics["submit"]["sum_cycles"]) + transition_sum = int( + metrics["post_transition"]["sum_cycles"] + ) + efdrain_control_sum = int( + metrics["efdrain_control"]["sum_cycles"] + ) + claim_sum = int(metrics["claim"]["sum_cycles"]) + post_claim_tail_sum = int( + metrics["post_claim_tail"]["sum_cycles"] + ) + kernel_union_sum = int( + metrics["kernel_union"]["sum_cycles"] + ) + if gross_sum != submit_sum + transition_sum: + raise AssertionError( + f"{actor_class} actor gross aggregate does not close" + ) + if ( + control_sum + != efdrain_control_sum + + claim_sum + + post_claim_tail_sum + + transition_sum + ): + raise AssertionError( + f"{actor_class} actor control aggregate does not close" + ) + if gross_sum != control_sum + kernel_union_sum: + raise AssertionError( + f"{actor_class} actor KernelUnion aggregate does not close" + ) + actors[actor_class] = { + "actor_count": actor_class_count, + "metrics_cycles": metrics, + "kernel": { + "event_count": bucket["kernel_event_count"], + "actor_count_with_kernel": bucket[ + "actors_with_kernel" + ], + "union_cycles": metrics["kernel_union"], + }, + "closure": { + "gross": { + "parent_cycles": gross_sum, + "submit_plus_post_transition_cycles": + submit_sum + transition_sum, + "exact": True, + }, + "control": { + "parent_cycles": control_sum, + "efdrain_control_plus_claim_plus_post_claim_tail_plus_" + "transition_cycles": + efdrain_control_sum + + claim_sum + + post_claim_tail_sum + + transition_sum, + "exact": True, + }, + "kernel": { + "parent_cycles": gross_sum, + "control_plus_kernel_union_cycles": + control_sum + kernel_union_sum, + "exact": True, + }, + }, + } + + winner_count = actors["winner"]["actor_count"] + loser_count = actors["loser"]["actor_count"] + winner_gross = int( + actors["winner"]["metrics_cycles"]["gross"]["sum_cycles"] + ) + loser_gross = int( + actors["loser"]["metrics_cycles"]["gross"]["sum_cycles"] + ) + winner_control = int( + actors["winner"]["metrics_cycles"]["control"]["sum_cycles"] + ) + loser_control = int( + actors["loser"]["metrics_cycles"]["control"]["sum_cycles"] + ) + winner_kernel = int( + actors["winner"]["metrics_cycles"]["kernel_union"][ + "sum_cycles" + ] + ) + loser_kernel = int( + actors["loser"]["metrics_cycles"]["kernel_union"][ + "sum_cycles" + ] + ) + gross_partition = ( + orchestration_setup_cycles + winner_gross + loser_gross + ) + control_partition = ( + orchestration_setup_cycles + + winner_control + + loser_control + + winner_kernel + + loser_kernel + ) + if gross_partition != orchestration_replay_cycles: + raise AssertionError( + "winner + loser actor gross does not close OrchestrationReplay" + ) + if control_partition != orchestration_replay_cycles: + raise AssertionError( + "winner + loser actor control does not close OrchestrationReplay" + ) + + return { + "fixed_counts": { + "core_count": core_count, + "task_count_per_core": task_count_per_core, + "expected_actor_count": expected_actor_count, + "actor_count": actor_count, + "winner_actor_count": winner_count, + "loser_actor_count": loser_count, + "winner_plus_loser_actor_count": winner_count + loser_count, + }, + "actors": actors, + "aggregate_core_work": { + "orchestration_replay_cycles": + orchestration_replay_cycles, + "orchestration_setup_cycles": + orchestration_setup_cycles, + "closure": { + "gross": { + "parent_cycles": orchestration_replay_cycles, + "setup_plus_winner_plus_loser_cycles": + gross_partition, + "exact": True, + }, + "control": { + "parent_cycles": orchestration_replay_cycles, + "setup_plus_winner_plus_loser_control_plus_" + "kernel_union_cycles": + control_partition, + "exact": True, + }, + }, + }, + "semantics": { + "fixed_counts": ( + "observed winner/loser classifications plus the exact " + "core_count × task_count total; these counts are comparison " + "populations, not a new one-winner-per-task protocol oracle" + ), + "actor_window": ( + "Submit_i.start through Submit_{i+1}.start; the final " + "task ends at OrchestrationReplay.end" + ), + "gross": "Submit duration plus post-transition duration", + "control": ( + "gross minus the interval union of Kernel events inside " + "this Submit's EfDrain" + ), + "post_claim_tail": ( + "remaining Submit control after removing EfDrain and " + "Claim; it includes all later frontend/tail work and any " + "unattributed Submit gaps" + ), + "post_transition": ( + "current Submit.end through the next Submit.start, or " + "through OrchestrationReplay.end for the final task" + ), + "distribution": ( + "sum/mean/median and nearest-rank p95 over fixed actor " + "instances" + ), + }, + } + + +def _add_residual_segment( + segments: dict[str, dict[str, int]], + key: str, + cycles: int, + role: str, +) -> None: + """只在小型汇总报告中累计空白来源，不向 raw/merged 增加逐事件字段。""" + + if cycles <= 0: + return + segment = segments.setdefault( + key, + {"event_count": 0, "cycles": 0, "aic_cycles": 0, "aiv_cycles": 0}, + ) + segment["event_count"] += 1 + segment["cycles"] += cycles + segment[f"{role}_cycles"] += cycles + + +def _residual_breakdown( + submits_by_lane: dict[tuple[int, int], list[Event]], + children_by_submit: dict[int, list[Event]], + task_kind_by_id: dict[int, int] | None, +) -> dict[str, Any]: + """按相邻既有边界聚合 Submit 内和 Submit 间空白，保持输出规模恒定。""" + + internal_segments: dict[str, dict[str, int]] = {} + tail_segments: dict[str, dict[str, int]] = {} + between_segments: dict[str, dict[str, int]] = {} + internal_total = 0 + tail_total = 0 + between_total = 0 + for (core_id, _lane), submits in submits_by_lane.items(): + role = "aic" if core_id < EXPECTED_AIC_CORES else "aiv" + for submit in submits: + cursor = submit.start_cycle + previous_phase = "SubmitBegin" + children = sorted( + children_by_submit[submit.row_index], + key=lambda event: (event.start_cycle, event.end_cycle, event.row_index), + ) + for child in children: + gap = child.start_cycle - cursor + _add_residual_segment( + internal_segments, + f"{previous_phase}->{child.phase}", + gap, + role, + ) + internal_total += max(gap, 0) + cursor = child.end_cycle + previous_phase = child.phase + tail_gap = submit.end_cycle - cursor + _add_residual_segment( + tail_segments, + f"{previous_phase}->SubmitEnd", + tail_gap, + role, + ) + tail_total += max(tail_gap, 0) + + for previous, current in zip(submits, submits[1:]): + gap = current.start_cycle - previous.end_cycle + if task_kind_by_id is None: + # schema-v3 只存在 private 固定五 task 流；保持历史报告逐字一致。 + previous_kind_id = previous.task_id % len(TASK_KIND_NAMES) + current_kind_id = current.task_id % len(TASK_KIND_NAMES) + else: + previous_kind_id = task_kind_by_id[previous.task_id] + current_kind_id = task_kind_by_id[current.task_id] + previous_kind = TASK_KIND_NAMES[previous_kind_id] + current_kind = TASK_KIND_NAMES[current_kind_id] + _add_residual_segment( + between_segments, + f"{previous_kind}->{current_kind}", + gap, + role, + ) + between_total += max(gap, 0) + + def ordered(segments: dict[str, dict[str, int]]) -> list[dict[str, Any]]: + return [ + {"boundary": key, **values} + for key, values in sorted( + segments.items(), key=lambda item: (-item[1]["cycles"], item[0]) + ) + ] + + return { + "semantics": ( + "aggregate positive gaps between existing exclusive boundaries; " + "tail remains residual and is not a standalone business phase" + ), + "submit_internal_residual": { + "total_cycles": internal_total, + "segments": ordered(internal_segments), + }, + "submit_tail_residual": { + "total_cycles": tail_total, + "segments": ordered(tail_segments), + }, + "between_submit_residual": { + "total_cycles": between_total, + "segments": ordered(between_segments), + }, + } + + +def analyze_capture(input_path: Path) -> dict[str, Any]: + """读取 schema-v3/v4 raw，完成全部门禁后返回可 JSON 序列化报告。""" + + input_path = Path(input_path) + ( + frequency_hz, + trace_schema_version, + rows, + core_by_block_lane, + _base_cycle, + metadata, + ) = _load_and_validate(input_path) + _restore_v5_shared_efdrain( + rows, + trace_schema_version, + metadata.get("tensormap_mode"), + ) + # 原始 b256 接近百万行；原地替换规范化 tuple，避免再同时保留一整份 Event + # 列表。slots 也避免每条 dataclass 单独分配 __dict__。 + for index, row in enumerate(rows): + rows[index] = _event_from_row(index, row) + events = cast(list[Event], rows) + core_types, num_cores = _validate_capture_identity( + trace_schema_version, metadata, events + ) + tensormap_mode = ( + str(metadata["tensormap_mode"]) + if trace_schema_version == 5 + else "private" + ) + if len(core_by_block_lane) != num_cores or set(core_by_block_lane.values()) != set( + range(num_cores) + ): + raise ValueError("block/lane to core mapping is incomplete") + + submits_by_lane, task_ids = _validate_and_group_submits(events) + task_kind_by_id: dict[int, int] | None = None + if trace_schema_version == 5: + # 只复用 Submit 已有 won/is_alloc 两个标量恢复动态 kind；不向 + # 300 MiB 级 raw/merged 添加逐事件字段。 + submit_semantics = { + (event.core_id, event.task_id): ( + bool(event.flags & 1), + bool(event.auxiliary), + ) + for event in events + if event.phase == "Submit" + } + task_kind_by_id = _derive_v4_task_kinds(submit_semantics, num_cores) + exclusive_phases = _exclusive_phases( + trace_schema_version, tensormap_mode + ) + submit_partition_names = _submit_partition_metrics(trace_schema_version) + role_metric_names = _role_metrics(trace_schema_version) + children_by_submit = _associate_exclusive_children( + events, submits_by_lane, exclusive_phases + ) + ( + detail_by_register, + outputs_by_owner, + copies_by_outputs, + flushes_by_outputs, + output_placement, + ) = _associate_shared_register_details( + events, + children_by_submit, + trace_schema_version, + tensormap_mode, + ) + register_breakdown = _build_register_breakdown( + children_by_submit, + detail_by_register, + outputs_by_owner, + copies_by_outputs, + flushes_by_outputs, + output_placement, + num_cores, + trace_schema_version, + tensormap_mode, + ) + materialize_breakdown = _build_materialize_breakdown( + children_by_submit, + outputs_by_owner, + copies_by_outputs, + flushes_by_outputs, + output_placement, + num_cores, + trace_schema_version, + tensormap_mode, + ) + kernels_by_efdrain, efdrain_kernel_rows = _associate_efdrain_kernels( + events, children_by_submit + ) + + orchestrations_by_lane: dict[tuple[int, int], list[Event]] = {} + final_drains_by_lane: dict[tuple[int, int], list[Event]] = {} + kernels_by_final_drain: dict[int, list[Event]] = {} + final_drain_kernel_rows: set[int] = set() + tail_kernel_rows: set[int] = set() + tail_kernel_counts = {"WinnerBuild": 0, "AllocComplete": 0} + legacy_lap_records = sum( + event.phase in {"Build", "Replay", "Alloc"} for event in events + ) + if trace_schema_version == 5: + if legacy_lap_records != 0: + raise ValueError("schema-v5 requires zero legacy Alloc/Build/Replay records") + orchestrations_by_lane = _group_v4_parents(events, "OrchestrationReplay") + final_drains_by_lane = _group_v4_parents(events, "FinalDrain") + kernels_by_final_drain, final_drain_kernel_rows = _associate_kernels_to_parents( + events, final_drains_by_lane, parent_name="FinalDrain" + ) + _kernels_by_tail, tail_kernel_rows, tail_kernel_counts = ( + _associate_v4_tail_kernels(events, children_by_submit) + ) + classified_sets = ( + efdrain_kernel_rows, + tail_kernel_rows, + final_drain_kernel_rows, + ) + if any( + left & right + for index, left in enumerate(classified_sets) + for right in classified_sets[index + 1 :] + ): + raise ValueError("one Kernel cannot belong to multiple schema-v5 parents") + + per_core = [] + for core_id in range(num_cores): + lane = _standalone_topology(core_id)[1] + per_core.append( + _analyze_core( + core_id, + core_types[core_id], + trace_schema_version, + tensormap_mode, + submits_by_lane[(core_id, lane)], + children_by_submit, + kernels_by_efdrain, + ( + orchestrations_by_lane[(core_id, lane)][0] + if trace_schema_version == 5 + else None + ), + ( + final_drains_by_lane[(core_id, lane)][0] + if trace_schema_version == 5 + else None + ), + kernels_by_final_drain, + ) + ) + + winner_loser_actor_closure = ( + _build_v5_actor_closure( + submits_by_lane, + children_by_submit, + kernels_by_efdrain, + orchestrations_by_lane, + core_count=num_cores, + task_count_per_core=len(task_ids), + ) + if trace_schema_version == 5 + else None + ) + aggregate_metrics = { + metric: sum(core["metrics_cycles"][metric] for core in per_core) + for metric in role_metric_names + } + if ( + register_breakdown is not None + and register_breakdown["aggregate_core_work"]["metrics_cycles"]["parent"] + != aggregate_metrics["register"] + ): + raise AssertionError( + "Register breakdown parent does not match the exclusive Submit Register total" + ) + if ( + materialize_breakdown is not None + and materialize_breakdown[ + "aggregate_core_work" + ]["metrics_cycles"]["parent"] + != aggregate_metrics["materialize"] + ): + raise AssertionError( + "Materialize breakdown parent does not match the exclusive " + "Submit Materialize total" + ) + residual_breakdown = _residual_breakdown( + submits_by_lane, children_by_submit, task_kind_by_id + ) + if ( + residual_breakdown["submit_internal_residual"]["total_cycles"] + + residual_breakdown["submit_tail_residual"]["total_cycles"] + != aggregate_metrics["submit_residual"] + ): + raise AssertionError("Submit internal + tail residual breakdown does not close") + if ( + residual_breakdown["between_submit_residual"]["total_cycles"] + != aggregate_metrics["between_submit_residual"] + ): + raise AssertionError("between-Submit residual boundary breakdown does not close") + # 占比只进入小型汇总报告，不给 raw/merged 逐事件增加字段。 + residual_breakdown["submit_internal_residual"]["share_of_submit_union"] = ( + residual_breakdown["submit_internal_residual"]["total_cycles"] + / aggregate_metrics["submit_union"] + ) + residual_breakdown["submit_tail_residual"]["share_of_submit_union"] = ( + residual_breakdown["submit_tail_residual"]["total_cycles"] + / aggregate_metrics["submit_union"] + ) + residual_breakdown["between_submit_residual"]["share_of_submit_envelope"] = ( + residual_breakdown["between_submit_residual"]["total_cycles"] + / aggregate_metrics["submit_envelope"] + ) + submit_partition_sum = sum( + aggregate_metrics[name] for name in submit_partition_names + ) + envelope_partition_sum = ( + aggregate_metrics["submit_union"] + + aggregate_metrics["between_submit_residual"] + ) + efdrain_partition_sum = ( + aggregate_metrics["efdrain_kernel_union"] + + aggregate_metrics["efdrain_control"] + ) + if submit_partition_sum != aggregate_metrics["submit_union"]: + raise AssertionError("per-core Submit closures did not preserve aggregate closure") + if envelope_partition_sum != aggregate_metrics["submit_envelope"]: + raise AssertionError("per-core envelope closures did not preserve aggregate closure") + if efdrain_partition_sum != aggregate_metrics["efdrain"]: + raise AssertionError("per-core EfDrain closures did not preserve aggregate closure") + + closure: dict[str, Any] = { + "submit_partition": { + "parent_cycles": aggregate_metrics["submit_union"], + "children_plus_residual_cycles": submit_partition_sum, + "exact": True, + }, + "submit_envelope": { + "parent_cycles": aggregate_metrics["submit_envelope"], + "submit_union_plus_between_cycles": envelope_partition_sum, + "exact": True, + }, + "efdrain_partition": { + "parent_cycles": aggregate_metrics["efdrain"], + "kernel_union_plus_control_cycles": efdrain_partition_sum, + "exact": True, + }, + } + if trace_schema_version == 5: + orchestration_partition_sum = ( + aggregate_metrics["orchestration_setup"] + + aggregate_metrics["submit_union"] + + aggregate_metrics["between_submit_residual"] + + aggregate_metrics["orchestration_tail"] + ) + final_drain_partition_sum = ( + aggregate_metrics["final_drain_kernel_union"] + + aggregate_metrics["final_drain_residual"] + ) + worker_completion_sum = ( + aggregate_metrics["orchestration_replay"] + + aggregate_metrics["final_drain"] + ) + if orchestration_partition_sum != aggregate_metrics["orchestration_replay"]: + raise AssertionError("aggregate OrchestrationReplay partition does not close") + if final_drain_partition_sum != aggregate_metrics["final_drain"]: + raise AssertionError("aggregate FinalDrain partition does not close") + if worker_completion_sum != aggregate_metrics["worker_completion"]: + raise AssertionError("aggregate WorkerCompletion partition does not close") + closure.update( + { + "orchestration_replay": { + "parent_cycles": aggregate_metrics["orchestration_replay"], + "setup_submit_union_between_tail_cycles": orchestration_partition_sum, + "exact": True, + }, + "final_drain": { + "parent_cycles": aggregate_metrics["final_drain"], + "kernel_union_plus_residual_cycles": final_drain_partition_sum, + "exact": True, + }, + "worker_completion": { + "parent_cycles": aggregate_metrics["worker_completion"], + "orchestration_plus_final_drain_cycles": worker_completion_sum, + "exact": True, + }, + } + ) + + role_statistics: dict[str, Any] = {} + for role, expected_count in ( + ("aic", EXPECTED_AIC_CORES), + ("aiv", EXPECTED_AIV_CORES), + ): + role_cores = [core for core in per_core if core["role"] == role] + if len(role_cores) != expected_count: + raise ValueError( + f"role {role} has {len(role_cores)} cores, expected {expected_count}" + ) + role_statistics[role] = { + "core_count": len(role_cores), + "metrics": { + metric: _distribution( + [core["metrics_cycles"][metric] for core in role_cores] + ) + for metric in role_metric_names + }, + } + + all_submits = [ + submit for submits in submits_by_lane.values() for submit in submits + ] + global_start = min(submit.start_cycle for submit in all_submits) + global_end = max(submit.end_cycle for submit in all_submits) + + overlays = {} + for phase in OVERLAY_PHASES: + phase_events = [event for event in events if event.phase == phase] + overlays[phase] = { + "event_count": len(phase_events), + "aggregate_duration_cycles": sum(event.duration for event in phase_events), + "included_in_additive_totals": False, + } + + kernels = [event for event in events if event.phase == "Kernel"] + classified_kernel_rows = ( + efdrain_kernel_rows | tail_kernel_rows | final_drain_kernel_rows + ) + orphan_kernel_count = len(kernels) - len(classified_kernel_rows) + if orphan_kernel_count < 0: + raise AssertionError("Kernel containment classification is inconsistent") + if trace_schema_version == 5 and orphan_kernel_count != 0: + orphan_rows = sorted( + event.row_index + for event in kernels + if event.row_index not in classified_kernel_rows + ) + raise ValueError( + "schema-v5 Kernel must be contained by EfDrain, WinnerBuild, " + f"AllocComplete, or FinalDrain: orphan_rows={orphan_rows[:8]}" + ) + + validation: dict[str, Any] = { + "status": "PASS", + "dropped_records": 0, + "core_ids_complete": True, + "role_map_complete": True, + "task_ids_contiguous_and_equal_per_core": True, + "exclusive_child_task_ids_match_parent": True, + "submit_non_overlapping_per_core_lane": True, + "submit_partition_exact": True, + "efdrain_partition_exact": True, + "submit_envelope_partition_exact": True, + } + if trace_schema_version == 5: + validation.update( + { + "orchestration_parent_exactly_one_per_core": True, + "final_drain_parent_exactly_one_per_core": True, + "parent_boundaries_adjacent": True, + "legacy_lap_records": 0, + "orchestration_partition_exact": True, + "final_drain_partition_exact": True, + "worker_completion_partition_exact": True, + "kernel_unique_parent_complete": True, + "winner_loser_actor_total_count_exact": True, + "winner_loser_actor_gross_partition_exact": True, + "winner_loser_actor_control_partition_exact": True, + } + ) + if tensormap_mode == "shared": + validation.update( + { + "register_publish_metadata_exactly_one_per_register": True, + "register_detail_identity_matches_parent": True, + "register_metadata_partition_exact": True, + "register_flat_partition_exact": True, + "register_details_excluded_from_submit_additive_totals": True, + "task_output_placement": output_placement, + } + ) + if output_placement == "materialize": + validation.update( + { + "materialize_publish_task_outputs_exactly_one_per_parent": True, + "materialize_task_outputs_identity_matches_parent": True, + "materialize_partition_exact": True, + "materialize_task_outputs_partition_exact": True, + "materialize_details_excluded_from_submit_additive_totals": True, + } + ) + else: + validation.update( + { + "register_publish_task_outputs_exactly_one_per_metadata": True, + "register_task_outputs_identity_matches_metadata": True, + } + ) + + semantics: dict[str, Any] = { + "cycle_arithmetic": "raw_integer_cycles", + "tensormap_mode": tensormap_mode, + "exclusive_submit_children": list(exclusive_phases), + "submit_residual": ( + "Submit minus the union of exclusive children; exactly equals " + "submit_internal_residual plus submit_tail_residual" + ), + "submit_internal_residual": ( + "unattributed prefix and child-to-child gaps inside Submit" + ), + "submit_tail_residual": ( + "unattributed suffix from the final exclusive child end to Submit end; " + "not a standalone business phase" + ), + "between_submit_residual": ( + "unattributed gap from one Submit end to the next Submit begin" + ), + "efdrain_children": ["KernelUnion", "EfDrainControl"], + "overlay_phases": list(OVERLAY_PHASES), + "overlays_are_additive": False, + "p95_method": "nearest_rank", + } + if trace_schema_version == 3: + semantics["legacy_lap_phases"] = ["Build", "Replay", "Alloc"] + else: + semantics.update( + { + "orchestration_children": [ + "OrchestrationSetup", + "SubmitUnion", + "BetweenSubmitResidual", + "OrchestrationTail", + ], + "final_drain_children": ["KernelUnion", "FinalDrainResidual"], + "worker_completion_children": ["OrchestrationReplay", "FinalDrain"], + "legacy_lap_phases_forbidden": ["Build", "Replay", "Alloc"], + "kernel_unique_parents": [ + "EfDrain", + "WinnerBuild", + "AllocComplete", + "FinalDrain", + ], + } + ) + if tensormap_mode == "shared": + semantics.update( + { + "register_internal_detail": SHARED_REGISTER_DETAIL_PHASE, + "register_internal_children": [ + "RegisterWaitInsertTurn", + "RegisterPublishWriterMetadata", + "RegisterPublishInsertCompletion", + ], + "task_output_placement": output_placement, + "register_internal_details_are_exclusive_submit_children": False, + } + ) + if output_placement == "materialize": + semantics.update( + { + "materialize_internal_output_detail": ( + SHARED_MATERIALIZE_OUTPUT_DETAIL_PHASE + ), + "materialize_internal_children": [ + "MaterializeBeforePublishTaskOutputs", + "MaterializePublishTaskOutputs", + "MaterializeAfterPublishTaskOutputs", + ], + } + ) + else: + semantics.update( + { + "register_internal_output_detail": ( + LEGACY_SHARED_REGISTER_OUTPUT_DETAIL_PHASE + ), + "register_internal_children": [ + "RegisterWaitInsertTurn", + "RegisterPublishWriterMetadata", + "RegisterPublishTaskOutputs", + "RegisterPublishMetadataEpilogue", + "RegisterPublishInsertCompletion", + ], + } + ) + + report = { + "schema_version": REPORT_SCHEMA_VERSION, + "input": str(input_path), + "capture": { + "trace_schema_version": trace_schema_version, + "tensormap_mode": tensormap_mode, + "clock_freq_hz": frequency_hz, + "core_count": num_cores, + "task_count_per_core": len(task_ids), + "event_count": len(events), + }, + "validation": validation, + "semantics": semantics, + "global_submit_makespan": { + "start_cycle": global_start, + "end_cycle": global_end, + "duration_cycles": global_end - global_start, + "duration_us": (global_end - global_start) * 1_000_000 / frequency_hz, + "semantics": "cross-core wall-clock envelope; not aggregate core-work", + }, + "aggregate_core_work": { + "metrics_cycles": aggregate_metrics, + "closure": closure, + "semantics": "sum of per-core cycles; not wall-clock duration", + }, + "materialize_breakdown": materialize_breakdown, + "register_breakdown": register_breakdown, + "residual_breakdown": residual_breakdown, + "per_role_core_statistics": role_statistics, + "kernel_containment": { + "total_events": len(kernels), + "inside_efdrain_events": len(efdrain_kernel_rows), + "inside_winner_build_events": tail_kernel_counts["WinnerBuild"], + "inside_alloc_complete_events": tail_kernel_counts["AllocComplete"], + "inside_submit_tail_events": len(tail_kernel_rows), + "inside_final_drain_events": len(final_drain_kernel_rows), + # v3 没有 FinalDrain/真实 tail 父边界，对其父区间外 Kernel + # 只能标为“无 v4 边界可分类”，不冒充已证实的孤儿。 + "orphan_events": ( + orphan_kernel_count if trace_schema_version == 5 else None + ), + "unclassified_without_v5_parent_events": ( + orphan_kernel_count if trace_schema_version == 3 else 0 + ), + }, + "overlays": overlays, + "per_core": per_core, + } + if winner_loser_actor_closure is not None: + report["winner_loser_actor_closure"] = ( + winner_loser_actor_closure + ) + return report + + +def write_analysis(input_path: Path, output_path: Path) -> Path: + """先完成全部分析，再用同目录临时文件原子发布 JSON。""" + + input_path = Path(input_path) + output_path = Path(output_path) + if input_path.resolve() == output_path.resolve(): + raise ValueError("analysis output path must differ from input raw path") + report = analyze_capture(input_path) + document = json.dumps(report, ensure_ascii=False, indent=2) + "\n" + + output_path.parent.mkdir(parents=True, exist_ok=True) + temporary_name: str | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix=f".{output_path.name}.", + suffix=".tmp", + dir=output_path.parent, + delete=False, + ) as temporary: + temporary.write(document) + temporary.flush() + os.fsync(temporary.fileno()) + os.fchmod(temporary.fileno(), 0o644) + temporary_name = temporary.name + os.replace(temporary_name, output_path) + finally: + if temporary_name is not None: + Path(temporary_name).unlink(missing_ok=True) + return output_path + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path, help="schema-v3/v4 l2_swimlane_records.json") + parser.add_argument( + "-o", + "--output", + required=True, + type=Path, + help="排他分析 JSON 输出路径", + ) + arguments = parser.parse_args(argv) + try: + output = write_analysis(arguments.input, arguments.output) + except (OSError, ValueError) as error: + print(f"exclusive swimlane analysis failed: {error}", file=sys.stderr) + return 1 + print(f"[SWIMLANE-EXCLUSIVE] input={arguments.input} output={output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/atomic_probe/pa_scheduler/swimlane_opt_anal.md b/tests/atomic_probe/pa_scheduler/swimlane_opt_anal.md new file mode 100644 index 0000000000..161bc55fb9 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/swimlane_opt_anal.md @@ -0,0 +1,1141 @@ +# PA Scheduler standalone 主执行流程与泳道全景 + +本文面向第一次接触 FDWIC PA 调度器的开发者，解释 +`tests/atomic_probe/pa_scheduler` 这条 standalone 路径从 Host 启动到任务完成的 +完整业务流程，以及每一层泳道区间究竟表示什么。 + +本文的主语始终是 **standalone PA Case1**。它复刻真实 PA 调度协议中的关键 +结构，但不是 `simpler` 主场景本身。两者差异集中列在第 9 节；除该节外，不能把 +“standalone 当前行为”自动外推成所有 `simpler` 场景的通用行为。 + +本文只使用当前源码能够证明的事实，不把历史样本数值作为业务定义，也不讨论 +I-cache、PMU 或某次性能波动。性能数据会随编译布局和运行环境变化，而这里描述的 +执行顺序、状态转移和闭合关系才是新人理解代码的稳定入口。 + +## 1. 一页看懂全流程 + +### 1.1 Host、worker 与后处理的总览 + +```text +Host + │ + ├─ 解析参数、加载 mixed ELF + ├─ 分配并初始化 SchedulerState / workload / trace + ├─ H2D：共享状态前缀、standalone 控制区、可选 workload/trace header + │ + ├─ launch 32 个 mixed block + │ └─ 每个 block 启动 1 AIC + 2 AIV，共 96 个 worker + │ + │ 每个 worker + │ ├─ 初始化自己的 WorkerState + │ ├─ StartupBarrier：等待 96 个 worker 到齐 + │ ├─ OrchestrationReplay + │ │ ├─ 初始化整轮 PA descriptor/template + │ │ └─ 对每个 batch 回放 Alloc → QK → SF → PV → UP + │ ├─ FinalDrain：等所有 worker 结束回放，并清空本核私有 slot + │ └─ DFXFinalize：flush trace、汇总并发布 WorkerResult + │ + ├─ stream synchronize + ├─ D2H：共享完成状态、WorkerResult、trace header/有效 records + ├─ 校验拓扑、计数、依赖、完成状态与 workload 输出 + └─ 校验通过后发布 raw JSON，再由脚本生成 Perfetto 和排他分析 JSON +``` + +这里最重要的三个认识是： + +1. 96 个 worker 都按相同顺序回放全部 Submit，不是把五类 Submit 预先静态分给 + 某几个核。 +2. `Claim` 只为每个全局 task 选出一个 winner；loser 仍会完成参数构造、 + descriptor 物化和本核私有依赖状态维护。 +3. kernel winner 在 Submit 内只把任务放进私有 slot。真正的 QK/SF/PV/UP + 计算通常由后续 drain 执行，因此“Submit 返回”不等于“kernel 已完成”。 + +### 1.2 一个 batch 的业务链 + +standalone 固定每个 batch 只有一个 q-loop、一个 block group，严格提交五个 +task： + +```text +BeginPaBatch + │ 读取本 batch 的 context length，计算 block 数 + ▼ +Submit Alloc ── Accept ── PreparePaBlockGroup + │ │ + │ ▼ + │ Submit QK ── Accept + │ │ + │ ▼ + │ Submit SF ── Accept + │ │ + │ ▼ + │ Submit PV ── Accept + │ │ + └──────────────────────────┴──► Submit UP +``` + +从数据依赖看，五个 task 形成下面的调度图： + +```text +Alloc ───────────────────────────────┐ + │ +QK ──► SF ──► PV ───────────────────┼──► UP + └─────────────────────────────┘ +``` + +显式 fanin 边分别是： + +- QK：0 条，输入都是外部 tensor； +- SF：依赖 QK； +- PV：依赖 SF； +- UP：依赖 Alloc、SF 和 PV，共 3 条。 + +所以每个 batch 有 5 个 fanin edge。descriptor 会在 Submit 阶段先返回给 +orchestration，真正的数据可用性由 producer task 的 completion flag 保证。 +这使 orchestration 能继续提交后继任务，而不必在每个 Submit 后同步等待计算完成。 + +### 1.3 一次 Submit 的当前顺序 + +当前 standalone 使用 compete-first eager 顺序：先做与参数无关的进度推进和 +Claim，再让所有 worker 同步构造完整参数，最后消费参数完成 Submit。 + +```text +Submit API call + ├─ BeginCallbackSubmit 分配 task_id；位于记录的 Submit.start 之前 + ├─ recorded Submit parent + │ ├─ EfDrain 尝试执行以前已 ready 的 slot + │ ├─ Claim 按 task role 竞争唯一 winner + │ ├─ [ArgBuild residual] 所有 worker 同步构造完整 TaskArgs + │ ├─ Materialize 生成输出 descriptor 和 register mask + │ ├─ PrepareMap 退休超出 H 窗口的 TensorMap 条目 + │ ├─ Fanin 仅非 Alloc winner 收集 producer task + │ ├─ Register 更新本核的 hazard producer 视图 + │ ├─ WinnerBuild/AllocComplete（仅 winner） + │ └─ [tail residual] 公共计数和 Submit.end 取时 + └─ 发布 Submit record 并返回 位于记录的 Submit.end 之后 +``` + +方括号中的 `ArgBuild residual` 是业务解释，不是当前 schema-v4 的独立 raw +phase。泳道只知道它位于 `Claim.end` 与 `Materialize.begin` 之间；该空白还包含 +Claim record 发布、Begin/Finish 衔接等成本，所以不能把整个区间都宣称为 +“纯参数构造耗时”。 + +## 2. 先认识参与者与状态 + +### 2.1 物理拓扑和 worker 编号 + +CCEC Host launch 32 个 mixed block。每个 block 产生三条运行 lane： + +| 运行 lane | worker 编号 | block/lane 映射 | 主要 task | +| --------- | ----------- | --------------- | --------- | +| AIC | `0..31` | `block=worker, lane=0` | QK、PV | +| AIV0/AIV1 | `32..95` | 两个 AIV 对应一个 block，lane 为 1/2 | SF、UP | + +因此固定拓扑是 32 AIC + 64 AIV，共 96 个 worker。所有 worker 都有自己的 +scalar 控制流、TensorMap、ring slot、heap cursor 和 task payload arena。 + +每个 task 的 Claim 参与范围不同： + +| task | 真正执行 atomicMax 的 worker | 每个 task 的 winner 数 | +| ---- | ---------------------------: | ---------------------: | +| Alloc | 全部 96 个 worker | 1 | +| QK、PV | 32 个 AIC | 各 1 | +| SF、UP | 64 个 AIV | 各 1 | + +不符合 role 的 worker 仍有一条 `Claim` span，但 `attempted=false`，不会执行 +该 task 的 `atomicMax`。 + +### 2.2 四类状态及其业务职责 + +| 状态 | 所有权 | 保存内容 | 业务作用 | +| ---- | ------ | -------- | -------- | +| `SchedulerState` 共享前缀 | 96 核共享 | Claim cursor、task cell、frontier、屏障、fatal | 建立全局唯一 winner、完成可见性和生命周期同步 | +| `WorkerState` | 每个 worker 私有 | heap cursor、TensorMap、slot、payload | 让每核独立回放同一图，并暂存自己赢得的任务 | +| `TaskCell` | 每个全局 task 一份 | `vend`、`flag` | 发布 heap 水位和完成状态，供 fanin/回收读取 | +| `PaOrchestrationState`/`TaskArgs` | 当前 worker 回放栈 | PA descriptor、动态 shape、参数列表 | 把前一 Submit 的输出接到后一 Submit 的输入 | + +共享与私有的边界非常重要： + +- Claim cursor 和 completion flag 是两种模式都使用的跨核协议；frontier + 是 private ring 回收协议，shared Case1 的严格 no-wrap heap 不消费它； +- private TensorMap 和 slot 只属于本 worker，维护它们不需要跨核原子； + shared TensorMap sidecar 是全局共享状态； +- 每个 worker 都维护同一 task stream 的私有 descriptor/producer 视图； +- kernel 的完成通过共享 `TaskCell::flag` 对所有 worker 可见。 + +### 2.3 `vend`、`flag` 和 `frontier` 分别表示什么 + +private 任务完成时按固定顺序发布： + +1. 把 winner 当前的单调 `heap_next` 写入 task 的 `vend`； +2. 执行 store barrier； +3. 把 task 的 `flag` 发布为 ready； +4. 从当前 `frontier + 1` 开始扫描连续 ready 的 task，并用 `FetchMax` + 单调推进共享 frontier。 + +shared no-wrap Case1 只执行前三步。它用每 task flag 做 fanin，可用共享 +shard cursor/vend 做容量终态校验，并且编译期禁止 heap 回绕，所以不需要在 +每次完成后维护连续 frontier。`SchedulerState::frontier` 和相关 schema +仍保留给 private；未来 shared 支持回绕时必须恢复等价的回收代际协议。 + +三者不能混为一个概念： + +- `flag(task)` 回答“这个 task 是否完成”； +- `vend(task)` 回答“完成该 task 时，该 worker 的逻辑 heap 走到了哪里”； +- `frontier` 回答“从 task 0 起连续完成到了哪里”，不能跨过中间未完成的空洞。 + +`HeapGuard` 使用 `frontier - H` 对应 task 的 `vend` 判断旧物理 heap 区间是否 +已经允许复用。`H=64` 同时也是 TensorMap producer 的存活窗口，但 TensorMap +退休进度和共享 frontier 并不要求在同一时刻相等。 + +## 3. 五类 task 分别做什么 + +### 3.1 Alloc：建立本轮累计状态 + +Alloc 构造三个 `Output`： + +- accumulated output tile； +- accumulated sum； +- accumulated max。 + +所有 worker 都会为这三个输出物化 descriptor。唯一 winner 不构建 kernel +slot，而是在 `AllocComplete` 中通过 `HeapGuard` 后立即发布 task 完成。 +orchestration 随即用 `AcceptTaskOutputs` 保存三个 descriptor，供同 batch 的 +UP 使用。 + +业务效果是“为本次 q/group 链建立累计结果的逻辑存储和 producer 身份”，不是 +执行一次计算 kernel。 + +### 3.2 QK：产生 score descriptor 和 AIC 任务 + +QK 参数包括： + +- 当前 batch/query 的 view； +- key cache 和 block table； +- 一个动态 shape 的 score `Output`； +- block 数和 block-table offset 两个 scalar。 + +32 个 AIC 竞争一个 winner。QK 输入都是外部 tensor，没有前序 task owner， +所以 winner 收集到 0 条 fanin。winner 在 `WinnerBuild` 中构造私有 slot， +score descriptor 则立即返回并被 SF 引用。 + +### 3.3 SF：等待 QK，产生 probability 和统计量 + +SF 参数包括 QK score，一个 probability `Output`、max/sum 两个 `Output`， +以及 scale、block 数和末 block 有效长度三个 scalar。 + +64 个 AIV 竞争一个 winner。score descriptor 的 owner 是 QK task，所以 SF +有 1 条 fanin。SF slot 可以先构建，但只有 QK 的 completion flag ready 后， +drain 才会执行它。 + +### 3.4 PV：等待 SF，产生新的 output tile + +PV 参数包括 SF probability、value cache、block table，一个 tile `Output`， +以及 block 数和 block-table offset 两个 scalar。 + +32 个 AIC 竞争一个 winner。PV 对 SF 有 1 条 fanin。返回的 tile descriptor +会作为 UP 的输入之一。 + +### 3.5 UP：合并本组结果并登记写 hazard + +UP 参数包括： + +- SF 的 max、sum 和 PV 的 output，三项 `Input`； +- accumulated max/sum/output 和最终 output view，四项 `Inout`； +- first-group、last-group 两个边界 scalar。 + +64 个 AIV 竞争一个 winner。UP 的显式依赖去重后是 Alloc、SF、PV 三个 task。 +它没有新的 `Output` descriptor，但四个 `Inout` 会形成 `register_mask`，并由 +每个 worker 登记到自己的 TensorMap，表示“当前 UP 是这些 backing range 的 +最新写者”。 + +### 3.6 默认 256 batch 时应看到的固定规模 + +设 batch 数为 `B`： + +| 计数 | 公式 | `B=256` | +| ---- | ---: | ------: | +| 每 worker 的 Submit | `5B` | 1,280 | +| 96 核 Submit 总记录 | `96 × 5B` | 122,880 | +| 实际 Claim atomicMax | `288B` | 73,728 | +| 全局 winner task | `5B` | 1,280 | +| AllocComplete | `B` | 256 | +| WinnerBuild / Kernel | `4B` | 1,024 | +| fanin edge | `5B` | 1,280 | + +这些是协议不变量，不是某次运行的经验均值。若这些计数不成立，应先判断为 +回放、路由、丢记录或完成协议异常，而不是直接分析性能分布。 + +## 4. Worker 的完整生命周期 + +### 4.1 Host 准备和 launch + +CCEC Host 的主要工作顺序是： + +1. 在创建 ACL 资源前解析公共参数、winner workload 参数和构建变体约束； +2. 读取 mixed AICore ELF，并完成 ACL/stream/kernel 注册； +3. 分配 `SchedulerState`、真实计算 workspace 和可选 trace 区； +4. 每轮调用 `InitializeState`，配置 trace 和 winner workload； +5. H2D 只传共享状态前缀、standalone 控制区和必要输入，不搬运每个 worker + 的完整私有 arena； +6. 以 32 个 mixed block launch，并同步等待设备完成。 + +Host 的 launch-to-sync 墙钟包含设备 launch、worker 启动、全部回放、最终 +drain、DFX 收尾和 stream 同步。它不包含 launch 前的 H2D，也不包含同步后的 +D2H/JSON。因此它不能由 `OrchestrationReplay + FinalDrain` 直接闭合。 + +### 4.2 Worker 私有状态初始化 + +每个 worker 在进入 task 0 前会： + +- 写入 role、core、block、lane、sub-block 身份； +- 把 `local_index` 和 `heap_next` 归零； +- 重置私有 TensorMap； +- 清空四个物理 slot； +- 附着当前 worker 的 trace/统计上下文。 + +四个物理 slot 中有两个保留给生产 BlockWon ABI，因此 standalone 单 lane +任务仍只有两个普通可用 slot。这个容量会触发真实的 ring backpressure，不能 +因为 standalone 不执行 joint task 就把四个 slot 全部当成普通容量。 + +### 4.3 StartupBarrier：统一回放起点 + +每个 worker 对共享 `started_count` 做一次增量，然后轮询到配置的 worker 数。 +等待期间同时检查 fatal，并由 watchdog 防止参与者缺失造成永久挂死。 + +该屏障的业务效果是压低各核进入 task 0 的启动偏斜。Claim 唯一性本身依赖 +atomicMax，不依赖屏障来保证正确性。 + +`StartupBarrier` 位于 `OrchestrationReplay` 之前，当前没有完整父 span。它可以 +作为 Atomic 发生位置观察，但不能被塞进 WorkerCompletion 的加和公式。 + +### 4.4 OrchestrationReplay：所有 worker 重放同一 PA 图 + +`OrchestrationReplay` 从 `InitPaOrchestration` 之前开始，到最后一个 Submit +返回并退出回放循环之后结束。它包含： + +1. 建立整轮共用的外部 tensor descriptor 和固定 create-info； +2. 对每个 batch 读取 context length，并计算 block 数； +3. 依次执行五个 compete-first Submit； +4. 接收每个 Submit 返回的 output descriptor，更新后继 task 的输入； +5. 执行 Submit 间的循环、view、分组和返回控制逻辑。 + +所有 worker 都走这段代码，所以每核 `task_id` 都是连续的 `0..5B-1`，且 +`task_id % 5` 固定映射为 Alloc/QK/SF/PV/UP。 + +### 4.5 FinalDrain：停止生产后清空在途任务 + +回放结束并不保证本核 slot 已空。每个 worker 先增加共享 `replay_done`，然后 +循环执行 `DrainReady(FinalDrain)`，直到同时满足： + +- `replay_done` 表明所有 worker 都已退出 orchestration replay； +- 当前 worker 的 `occupied_count == 0`。 + +第一个条件保证不会再产生新 slot，第二个条件保证本核已经执行完自己拥有的 +旧 slot。循环若没有释放任何 slot就执行 spin hint，之后继续检查 fanin 和全局 +完成状态。 + +standalone 没有 BlockWon，所以这里没有生产路径的 `has_pending_won()` 条件。 +这是与 `simpler` 主运行时的重要差异之一。 + +### 4.6 DFXFinalize：发布证据，不再属于业务完成窗口 + +FinalDrain 结束后，worker 才执行： + +- 延后写出 `OrchestrationReplay` 和 `FinalDrain` 两条父记录； +- atomic 观察构建中记录两条 `ClockBaseline`； +- flush 本核有效 trace 区； +- 汇总计数、最大 slot 占用、TensorMap 状态等； +- 把 `WorkerResult` 发布到独占的结果分区。 + +父记录故意在被测区间之后写出，避免它们自己的 GM 写入落入任一父区间。 +`DFXFinalize` 当前没有完整父 span，因此 WorkerCompletion 闭合不等于整个 worker +函数从入口到返回的闭合。 + +## 5. 一次 Submit 的逐阶段业务语义 + +### 5.1 `BeginCallbackSubmit`：建立本次提交身份 + +每个 worker 都用自己的 `local_index++` 得到 task id，并把 context 绑定到： + +- 当前 `WorkerState`； +- `task_id & payload_mask` 对应的私有 `TaskPayload`； +- 空的 result、fanin、register mask 和 kernel id。 + +这里不读取 `TaskArgs`。这是 Claim 能先于参数构造的前提，也是 Begin/Finish +之间只能同步衔接、不能嵌套另一次 Submit 的协议约束。 + +源码在 `BeginCallbackSubmit` 返回后才采集 `submit_begin`。所以这段 prologue +属于 OrchestrationReplay，却不属于 raw `Submit` 父 span：首个 task 落在 +OrchestrationSetup，其余 task 落在 BetweenSubmitResidual。 + +### 5.2 `EfDrain`：用本次 Submit 的前沿推进旧任务 + +`EfDrain` 调用 `DrainReady` 扫描本 worker 的私有 slot。对于全部 fanin flag +已 ready 的 slot，它会： + +1. 执行该 slot 对应的 winner workload； +2. 记录 `Kernel`； +3. 按 vend → flag → frontier 顺序完成任务； +4. 记录零时长 `Commit`； +5. 清空 slot 并降低 `occupied_count`。 + +因此 task `N` 的 `EfDrain` 可能执行 task `N-1` 或更早的 kernel。按外层 +Submit 的 task kind 给其中 Kernel 归类会得到错误结论，必须使用 Kernel +自己的 `task_id/function_id`。 + +如果本核没有占用 slot，`EfDrain` 是快速空路径；如果有 ready slot，它可能 +包含完整 engine workload 和完成发布。 + +### 5.3 `Claim`：决定谁负责本 task 的真实尾动作 + +Claim 先根据 task kind 生成 active role，再选择对应 cursor shard： + +- Alloc 使用 production-prefix `alloc_cursor[4]`； +- QK/PV 使用 production-prefix `cube_cursor[4]`； +- private SF/UP 使用 production-prefix `vector_cursor[4]`； +- 当前回退后的 shared SF/UP 使用 sidecar 中全部 8 个 active shard： + `shared_vector_cursor[8]`。 + +S4.15a 历史候选曾把 shared Cube 四分片迁到 sidecar，但六区组 +性能门槛未通过并已撤销；它不属于当前 Claim 路由。当前 private/shared +的 Cube 和 Alloc 均继续使用 production prefix，只有 shared Vector +使用 sidecar。 + +S4.16 历史实验只围绕 shared Vector 展开。S4.16a 建立了临时布局 +控制：把 `shared_vector_cursor` 物理容量从 8 扩成 16，但 active +保持 8，数组起点 4,735,680B、前八条 cache line +地址以及 Claim 热路径 `task_id%8` 都不变。新增八条线位于 state +尾部，sidecar 为 4,736,704B，CPU non-split/CCEC split state +为 1,011,852,672B/1,011,858,816B；后八条线零 attempted 且终值 +保持 -1。CPU/CCEC/A5 正确性已闭合；相对重建 `319077a9` 的冻结 +布局成本配对为 4/6 更快、中位数 `-12.1595us/-0.5136%`，但它不 +单独决定 S4.16b 的保留或取消。 + +S4.16b 随后在相同地址、物理容量和 state 大小下把 active 改为 16， +正确性闭合，但相对 S4.16a 仅 1/6 更快、中位数 +`+2.468us/+0.1049%`，第一层性能门槛失败并整体回退。 +无论 S4.16a 还是 S4.16b，Vector ClaimMax 都仍为 32,768 次、全局 +ClaimMax 仍为 73,728 次；前者由前八条线各承担 4,096 次，后者由 +十六条线各承担 2,048 次。每个 SF/UP task 仍是 64 个 AIV 竞争同一 +地址，所以泳道中的 Claim/Atomic 事件总数、父子层级和 +`return_ready` 解释均不因容量控制而变化。S4.16a/S4.16b 均未长期 +保留；当前恢复八分片，完整门槛与证据见 +`shared_tensormap_record.md` 的 S4.16 节。 + +符合 role 的 worker 对 cursor 执行 `atomicMax(task_id)`。返回旧值小于当前 +task id 的唯一竞争者获胜；其他参与者是 attempted loser，不符合 role 的核是 +not-attempted loser。 + +Claim 的输出包括： + +- `won`：是否是唯一 winner； +- `attempted`：是否真正执行了 atomicMax； +- `function_id`：winner 应构建的 QK/SF/PV/UP 负载类型。 + +standalone 固定 task 都是单 lane；若 active mask 同时要求两条及以上 lane， +Claim 会显式拒绝，因为本用例没有实现 BlockWon/joint 协议。 + +### 5.4 eager 参数构造：所有 worker 构建相同参数 + +Claim 返回后，同步 callback 才构造 `TaskArgs`： + +- Alloc 新建参数对象； +- QK/SF/PV/UP 复用并 reset 同一个参数对象； +- view、动态 create-info、tensor/scalar 参数都在 callback 中立即求值； +- callback 和内部 thunk 不被保存，也不会跨 Submit 生命周期执行。 + +winner 和 loser 都走完整构参路径。这保证所有 worker 的 descriptor、heap +cursor 和私有 dependency map 按相同 task stream 演进；compete-first 只改变 +Claim 与构参的先后顺序，没有改成 winner-only lazy 构参。 + +CCEC standalone 的 Begin/Finish 之间传递固定 16-byte ticket，finish 通过同一 +role-specific block-local state恢复 context，并校验 worker、task 顺序和 cookie。 +CPU/AscendC 复用相同业务函数，但不需要这条 CCEC split ABI。 + +### 5.5 `Materialize`：把逻辑 Output 变成可引用 descriptor + +`MaterializeTask` 完成四件事： + +1. 扫描参数 tag，形成 `output_mask` 和 `register_mask`； +2. 计算每个新 Output 的大小及 1 KiB 对齐后的总空间； +3. 在单调逻辑 heap 上安排连续区间，跨物理 ring 尾时跳到下一圈； +4. 在本 worker 的 task payload 中初始化 GM `TensorDesc`，写入 + `owner_task_id`，并推进 `heap_next`。 + +其输出是后继 orchestration 立即可引用的 descriptor，以及本 task 的 +`output_bytes`。它只分配和描述逻辑输出，不代表其中的数据已经由 kernel 写好。 + +所有 worker 都执行 Materialize，包括 loser。这样后续回放可以在任何 worker +上用同一个 task id 和 owner 拓扑继续构图。 + +### 5.6 `PrepareMap`：退休不再参与依赖查询的历史写者 + +`AdvanceTensorMap` 把本 worker 的 producer 存活下界推进到 `task_id - H`。 +超出窗口的 producer entry 会沿 task 链批量从地址 bucket 摘除，再放回 free +list。 + +其业务效果是限制依赖表的生命周期和容量，同时保留窗口内“同一 backing +buffer、字节区间重叠、task id 最新”的 producer 查询语义。 + +TensorMap 是 worker 私有结构，所以这个阶段不做跨核同步。它也不等价于 heap +已经可复用；物理 heap 安全仍由共享 frontier/vend 的 `HeapGuard` 判断。 + +### 5.7 `Fanin`：winner 建立执行前置条件 + +只有非 Alloc winner 执行 `CollectFanin`。它对每个非纯 Output 参数： + +1. 读取 descriptor 的显式 `owner_task_id`； +2. 对 Input/Inout 查询 TensorMap 中最新的重叠写者； +3. 对两种来源得到的 producer task id 去重。 + +最终 fanin 数组随 winner slot 保存。slot 执行前逐项读取共享 task flag，只要 +有一项未 ready，就保留该 slot 等待下次 drain。 + +QK winner 即使最终 fanin 为 0，也会有一条真实 `Fanin` span；Alloc 不执行 +该阶段；所有 loser 都不收集 fanin。 + +### 5.8 `Register`：更新后继任务看到的 hazard 版本 + +Register 使用 Materialize 生成的 `register_mask`，把 Inout 或 +OutputExisting 登记为“由当前 task 写入”。 + +当前固定图中： + +- Alloc 以 `include_existing=false` 调用，实际不插入； +- QK/SF/PV 只有新 Output，register mask 为空； +- UP 有四个 Inout，因此每个 worker 都插入四个 producer entry。 + +Register 不是 winner-only。它维护的是每个 worker 自己的未来依赖视图，而不是 +宣告 kernel 已完成。 + +### 5.9 winner 和 loser 的尾部分支 + +非 Alloc winner 进入 `WinnerBuild`： + +1. 若两个普通 slot 已满，`WaitForSlot` 循环 drain 直到有空间； +2. 对有新 Output 的任务运行 `HeapGuard`，避免覆盖仍存活的 ring 区间； +3. 预留一个私有 slot； +4. 把 active descriptor、scalar、dispatch context 和 fanin 快照复制进 slot。 + +`WinnerBuild` 的产物是一个 `occupied && built` 的待执行 slot，不是已完成 +kernel。 + +Alloc winner 进入 `AllocComplete`：它不创建 slot，在 `HeapGuard` 通过后直接 +发布 vend/flag/frontier，使本次逻辑分配成为已完成 producer。 + +standalone loser 没有额外业务动作。它只走公共 Submit 收尾，因此不能为了让 +图看起来完整而虚构 `Replay` 或 `LoserReplay` phase；剩余后缀归入 +`SubmitResidual`。 + +### 5.10 Submit 返回时已经保证了什么 + +成功路径的 Submit 返回只保证： + +- 本 worker 已完成本 task 的 descriptor 和私有依赖状态更新； +- 全局唯一 winner 已选出； +- Alloc winner 已完成，或 kernel winner 已把任务构造成待执行 slot； +- 后继 orchestration 拿到了可携带 owner 的输出 descriptor。 + +它不保证： + +- QK/SF/PV/UP kernel 已执行； +- 后继 task 的 fanin 已 ready； +- 所有 worker 已完成同一 task 的 Submit； +- heap 对更老逻辑区间已经可回收。 + +## 6. 延迟执行、背压与完成协议 + +### 6.1 `DrainReady` 有三个调用位置 + +同一套 drain 逻辑会在三个位置推进任务： + +| 位置 | 触发原因 | 泳道父位置 | +| ---- | -------- | ---------- | +| `EfDrain` | 每次新 Submit 开头顺手推进旧任务 | 当前 Submit 的 `EfDrain` | +| Ring backpressure | slot 满或 heap 暂不可复用 | `WinnerBuild`/`AllocComplete` 内部 | +| `FinalDrain` | 所有 Submit 结束后清空尾部在途任务 | `FinalDrain` | + +所以一条 Kernel 必须按实际时间包含关系归入上述唯一父位置，不能假设所有 +Kernel 都在 EfDrain，也不能把 `Kernel` 再与已包含它的父 span相加。 + +### 6.2 slot 容量为什么会形成背压 + +每个 worker 只有两个普通可用 slot。如果 winner 连续产生任务，而旧 slot 因 +fanin 未 ready 尚未释放，`WaitForSlot` 会主动调用 `DrainReady`。只有本核占用 +数降到容量以下，新的 winner payload 才能入队。 + +等待不是纯自旋：只要有依赖已满足的旧 slot，本核会执行其 kernel并推进全局 +完成状态。这保证 backpressure 路径本身也能帮助系统取得进展。 + +### 6.3 heap ring 为什么还需要另一层保护 + +slot 有空不代表输出 heap 可以安全覆盖。`heap_next` 是单调逻辑地址，真正落到 +256 MiB 物理 ring 时才取模。第一圈内不会覆盖旧区间；发生回绕后， +`HeapGuard` 读取共享 frontier 和退休 task 的 vend，确保当前 live window 不 +超过一圈。 + +若暂时不安全，`HeapGuard` 同样会 drain 本核 ready slot；若 frontier 已经追到 +当前 task 前仍无法满足容量，则发布 fatal，而不是静默覆盖数据。 + +### 6.4 为什么 FinalDrain 是完整业务的一部分 + +最后一个 Submit 返回时，最后几条 winner task 可能仍在 slot 中。若只统计首个 +Submit 到最后一个 Submit，就会漏掉这些 kernel、完成发布和跨核等待。 + +因此完整的 per-worker 业务口径必须是： + +```text +WorkerCompletion = OrchestrationReplay + FinalDrain +``` + +这也是泳道 schema-v4 新增顶层父区间的根本原因。 + +## 7. 泳道分区与“总耗时”口径 + +### 7.1 三种事件角色 + +泳道事件必须先按角色分类，才能讨论加和： + +1. **父区间**：例如 `Submit`、`OrchestrationReplay`、`FinalDrain`； +2. **排他子区间**：同一父区间内互不重叠，可与 residual 一起闭合父区间； +3. **嵌套或 Overlay**：例如 Kernel、Atomic、RingBp、Commit、ClockBaseline， + 用于定位，不得再次加到已经包含它的父区间。 + +“把所有 duration 再求和一次”会同时重复计算父子事件、多核并行时间和 Atomic +重叠观察窗口，因此没有明确的业务意义。 + +### 7.2 schema-v4 的严格层级 + +```text +WorkerCompletion 离线派生的每核业务父口径 +├─ OrchestrationReplay raw 父 span +│ ├─ OrchestrationSetup 父起点到首个 Submit +│ ├─ SubmitUnion 本核全部 Submit 的并集 +│ │ └─ 每个 Submit +│ │ ├─ EfDrain +│ │ ├─ Claim +│ │ ├─ Materialize +│ │ ├─ PrepareMap +│ │ ├─ Fanin（条件存在） +│ │ ├─ Register +│ │ ├─ WinnerBuild/AllocComplete（winner 条件存在） +│ │ ├─ SubmitInternalResidual（merged: submit_residual） +│ │ └─ SubmitTailResidual（merged: submit_tail_gap） +│ ├─ BetweenSubmitResidual 相邻 Submit 之间的精确空白 +│ └─ OrchestrationTail 最后 Submit 到父终点 +└─ FinalDrain raw 父 span + ├─ KernelUnion + └─ FinalDrainResidual + +嵌套定位：Kernel、RingBp +非加和 Overlay：Atomic、ClockBaseline、Commit 等 +``` + +其中 `EfDrain` 还能严格分成： + +```text +EfDrain = EfDrain.KernelUnion + EfDrainControl +``` + +`WinnerBuild` 或 `AllocComplete` 内若发生背压 drain，也可能包含 Kernel。当前 +排他报告会验证 Kernel 的唯一父位置，但不能把这些 Kernel 额外加到 Submit 上。 + +### 7.3 每核可以严格闭合的公式 + +所有计算先使用 raw 整数 tick，闭合后才按 metadata 中的频率换算时间： + +```text +Submit + = EfDrain + Claim + Materialize + PrepareMap + + optional(Fanin) + Register + + optional(WinnerBuild or AllocComplete) + + SubmitInternalResidual + SubmitTailResidual + +SubmitEnvelope + = SubmitUnion + BetweenSubmitResidual + +OrchestrationReplay + = OrchestrationSetup + SubmitUnion + + BetweenSubmitResidual + OrchestrationTail + +FinalDrain + = FinalDrain.KernelUnion + FinalDrainResidual + +WorkerCompletion + = OrchestrationReplay + FinalDrain +``` + +exclusive JSON 为兼容聚合口径，仍把两者之和记作 `submit_residual`： + +```text +submit_residual + = submit_internal_residual + submit_tail_residual +``` + +其中： + +- Submit 前缀及相邻 child 之间的 internal residual； +- 最后一个 child 到 Submit.end 的 tail residual。 + +residual 是“当前边界尚未单列的真实时间”，不是一个可以随意命名的业务函数。 + +### 7.4 打点边界与 record 发布归属 + +raw span 记录的是两个时间戳的差，不等于同名 helper 的无扰动净耗时。当前实现 +有意复用相邻边界以保证整数闭合，但 `WriteTrace` 本身发生在边界取时之后： + +| 区间 | start 边界 | end 边界 | 紧邻的观测归属 | +| ---- | ---------- | -------- | -------------- | +| `Submit` | `BeginCallbackSubmit` 之后 | 公共计数之后、发布父 record 之前 | prologue 在父外；父 record 写入下一段 gap | +| `EfDrain` | 与 Submit.start 相同 | `DrainReady` 返回后 | 自己的 record 写入落入后续 Claim 观察区 | +| `Claim` | 复用 EfDrain.end | atomicMax/role 路由完成后 | Claim record 和 eager 构参位于随后 residual | +| `Materialize` | callback 完成、进入 Finish 后 | `MaterializeTask` 返回后 | 自己的 record 写入落入 PrepareMap 观察区 | +| `PrepareMap` | 复用 Materialize.end | `AdvanceTensorMap` 返回后 | record 写入落入 Fanin 或 Register 观察区 | +| `Fanin`/`Register` | 复用前一业务边界 | 对应 helper 返回后 | record 写入落入下一 child 或 tail residual | +| winner tail | 复用 Register.end | build/complete 返回后 | tail record 写入落入 Submit tail residual | + +`OrchestrationReplay.end` 同时复用为 `FinalDrain.start`，两条父 record 都在 +FinalDrain 结束后才发布，因此父记录写入不属于任一业务父区间。 + +这种布局的优点是父子区间可以无浮点误差地闭合；代价是阶段值应解释为“当前 +源码边界下的观察区”，不能直接当成单个 helper 的纯函数耗时。若将来调整 mark， +必须同时验证业务边界、record 写入归属、闭合关系和插桩扰动，不能只移动标签。 + +### 7.5 三个 merged residual span 的具体业务内容 + +`between_submit_residual`、`submit_residual` 和 `submit_tail_gap` 都由 +`swimlane_converter.py` 对 schema-v4 的现有父子区间取补集生成。设备端不写 +这三个 phase，converter 也不增加新时间：三类 span 只是把原来未命名的区间按 +位置显示出来。 + +converter 只为 `end > start` 的正区间生成 Perfetto X event；若两个边界相等， +该段数学贡献为 0，merged 中不会出现一条零时长 residual。 + +exclusive analyzer 使用的名称略有不同： + +- merged `submit_residual` 对应 `submit_internal_residual`； +- merged `submit_tail_gap` 对应 `submit_tail_residual`； +- `between_submit_residual` 在两份产物中同名。 + +#### 7.5.1 `between_submit_residual`：两个 Submit 父区间之间 + +精确边界是同一物理 scalar lane 上： + +```text +previous Submit.end → next Submit.start +``` + +当前 standalone 成功路径中的公共动作顺序是： + +```text +previous Submit.end + ├─ 写 previous Submit 的 TraceRecord + ├─ 检查它是否为本核最后一个 task，并从 Finish 返回 + ├─ 回到 batch orchestration，处理前一 task 的返回结果/循环控制 + ├─ 调用下一个 SubmitCallbackTask + ├─ BeginCallbackSubmit：local_index++、绑定 payload、清 context + └─ 读取 next Submit.start +``` + +`BeginCallbackSubmit` 明确在 `Submit.start` 之前，因此属于这个 gap，而不是下一 +个 Submit。下一 Submit 的 EfDrain、Claim 和 eager 构参都在 `Submit.start` +之后，不属于 `between_submit_residual`。 + +五种 transition 的业务内容并不相同： + +- **Alloc → QK**：保存 accumulated output/sum/max 三个返回 descriptor；执行 + `PreparePaBlockGroup(0)`，计算当前 group 的 block offset、block 数和末 block + 有效长度；然后进入 QK Submit prologue。 +- **QK → SF**：保存 QK score descriptor，然后进入 SF Submit prologue。 + SF 的动态 probability create-info 和完整参数列表仍在下一 Submit 的 + Claim 后构造，不在这个 gap。 +- **SF → PV**：保存 probability、max、sum 三个 descriptor，然后进入 PV + Submit prologue。 +- **PV → UP**：保存 PV output descriptor，然后进入 UP Submit prologue。 + first/last scalar 和 output view 也在 UP Submit 内的 eager callback 构造。 +- **UP → 下一 batch 的 Alloc**：UP 没有新 Output 需要 Accept；代码退出本轮 + 五 task 顺序，递增 batch，执行 `BeginPaBatchForCallback`。这里会从 GM + `context_lens` 读取下一 batch 的真实 sequence length，计算 block 数，增加 + `context_reads` 统计，再进入 Alloc Submit prologue。 + +设 batch 数为 `B`，每核依次有 `B` 个 Alloc→QK、QK→SF、SF→PV 和 PV→UP +候选 gap，以及 `B-1` 个 UP→Alloc 候选 gap，总计 `5B-1` 个相邻 Submit +边界。当前路径有实际控制工作，通常都会形成正区间；严格的 merged event 数仍以 +`next.start > previous.end` 为准。最后一个 UP 到 `OrchestrationReplay.end` +不属于此类，而是 `OrchestrationTail`。 + +因此 `between_submit_residual` 是一个有明确边界、但内容随 transition 变化的 +orchestration 区域。它不能整体命名成“参数构造”：当前主要参数构造已经移动到 +下一 Submit 内的 compete-first eager callback。 + +#### 7.5.2 `submit_residual`：Submit 内显式 child 之间 + +converter 从 `Submit.start` 开始，按时间排序 EfDrain、Claim、Materialize、 +PrepareMap、可选 Fanin、Register 和可选 winner tail。每遇到 +`previous_child.end < next_child.start`，就为这个空白生成一条 +`submit_residual`。因此一个 Submit 理论上可以有多条同名 span，必须结合两侧 +child 名称解释。 + +按当前打点布局，成功路径中主要的正区间是： + +```text +Claim.end → Materialize.begin +``` + +它按源码顺序包含： + +1. 发布当前 `Claim` 的 TraceRecord； +2. 所有 96 个 worker 同步执行 `BuildCallbackSubmitArgs`； +3. 检查 builder 是否有效，并累计 reset/view/create-info/参数个数统计； +4. 构造固定 16-byte `CallbackSubmitTicket`； +5. CCEC 路径跨入 noinline split-finish TU，恢复对应 role 的 block-local + runtime state，校验 worker、task id、winner、cookie 和 ticket； +6. 进入 `FinishCallbackSubmitBody`，读取 `Materialize.begin`。 + +第 2 步是 compete-first eager 的主体，而且 winner、attempted loser、 +not-attempted loser 都完整执行： + +- Alloc：构造新的 `TaskArgs`，加入三个 Output； +- QK：reset，构造 query view 和动态 score create-info，加入三个 Input、一个 + Output 和两个 scalar； +- SF：reset，构造动态 probability create-info，加入一个 Input、三个 Output + 和三个 scalar； +- PV：reset，加入三个 Input、一个 Output 和两个 scalar； +- UP：reset，构造 output view，加入三个 Input、四个 Inout 和两个 scalar。 + +CPU/AscendC 不经过 CCEC 的 cross-TU ABI，但仍执行同一 builder、ticket 语义和 +Finish 入口衔接。因此这个 residual 可以描述为“Claim record + 全员 eager +ArgBuild + Begin/Finish bridge”，不能缩写成纯 `BuildCallbackSubmitArgs()` +函数耗时。 + +它不包含 Claim 的 role 路由/atomicMax，也不包含 `MaterializeTask`：前者已经在 +`Claim` child 内结束，后者从 `Materialize.begin` 才开始。若未来出现其他 +`A.end → B.start` 空白，converter 也会使用 `submit_residual` 名称，exclusive +JSON 中的 boundary 字段才是区分来源的依据。 + +#### 7.5.3 `submit_tail_gap`：最后一个 child 到 Submit.end + +converter 对每个 Submit 只把最后一个显式 child 之后的后缀命名为 +`submit_tail_gap`： + +```text +last exclusive child.end → Submit.end +``` + +起点取决于 Claim 结果： + +- 非 Alloc winner：从 `WinnerBuild.end` 开始； +- Alloc winner：从 `AllocComplete.end` 开始； +- 任意 loser：没有 winner tail，从 `Register.end` 开始。 + +三类路径的实际内容是： + +- **非 Alloc winner**：发布 `WinnerBuild` TraceRecord，增加本核 Submit 计数， + 读取 `Submit.end`；slot 等待、HeapGuard 和 payload build 已经位于 + `WinnerBuild` child 内。 +- **Alloc winner**：发布 `AllocComplete` TraceRecord，增加 Submit 计数，读取 + `Submit.end`；HeapGuard、vend/flag 发布和 frontier 推进已经位于 + `AllocComplete` child 内。 +- **loser**：发布 `Register` TraceRecord，增加 Submit 计数，读取 + `Submit.end`。standalone loser 在 Register 后没有 Replay/LoserReplay 或 + 其他调度动作，所以这段不能解释为“loser replay”。 + +winner 路径的 Register record 写入发生在 `Register.end` 之后，但由于 winner +tail 复用 `Register.end` 作为 start，它在数值上属于 WinnerBuild/AllocComplete， +不属于 `submit_tail_gap`。loser 没有该 tail child，所以同一笔 Register record +写入才落入 loser 的 `submit_tail_gap`。 + +`WriteTrace(Submit)` 发生在 `Submit.end` 取时之后，也不属于 +`submit_tail_gap`：非末次 Submit 时它落入随后的 `between_submit_residual`； +最后一个 Submit 时落入 `OrchestrationTail`。 + +所以 `submit_tail_gap` 的业务语义是“最后一个已记录 child 之后、Submit 父区间 +结束前的公共观测与 epilogue”，不是独立调度阶段。它仍必须保留，才能使每个 +Submit 在整数 tick 上严格闭合。 + +#### 7.5.4 三者在层级和加和中的关系 + +```text +OrchestrationReplay + ├─ Submit + │ ├─ explicit exclusive children + │ ├─ submit_residual 内部前缀/child 间空白，可有多段 + │ └─ submit_tail_gap 最后 child 后缀，至多一段 + └─ between_submit_residual 相邻 Submit 之间，不属于任一 Submit +``` + +排他闭合时只能这样加： + +```text +Submit + = explicit children + + submit_internal_residual + + submit_tail_residual + +OrchestrationReplay + = setup + SubmitUnion + + between_submit_residual + tail +``` + +不能把 `between_submit_residual` 再加进某个 Submit，也不能把 merged 中的 +`submit_residual` 和 exclusive JSON 聚合层的总 `submit_residual` 当成两个不同 +区域重复相加。 + +### 7.6 必须区分的四个“总时间” + +#### 单核 Submit envelope + +```text +last_submit_end(core) - first_submit_start(core) +``` + +它只覆盖该核的首末 Submit，不含 OrchestrationSetup/Tail 和 FinalDrain。 + +#### 单核 WorkerCompletion + +```text +final_drain_end(core) - orchestration_begin(core) +``` + +它是本文最完整、可严格闭合的设备业务口径，但仍不含 StartupBarrier 和 +DFXFinalize。 + +#### 跨核 Submit makespan + +```text +max(last_submit_end) - min(first_submit_start) +``` + +起点和终点可能来自不同核。它是墙钟包络，不等于任一核的阶段和，也不等于 +96 核 duration 总和。 + +#### aggregate core-work + +```text +sum(per_core_metric) +``` + +它回答“所有 scalar lane 累计投入了多少核时间”，适合比较阶段工作量分布， +不回答用户等待了多久。排他分析 JSON 明确把它与 global makespan 分开输出。 + +### 7.7 Atomic 为什么只能作为 Overlay + +Atomic span 已经位于 Claim、EfDrain、FinalDrain、AllocComplete 等父区间内。 +PollBatch 还可能用一条记录覆盖一个完整轮询 episode，而不是一条指令。 + +因此 Atomic 只能回答“原子访问发生在哪个业务区域、规模如何”，不能： + +- 再加到父阶段上； +- 从父阶段机械相减得到纯 scalar 时间； +- 用 aggregate Atomic duration 除以调用数得到单条硬件延迟； +- 直接解释全局 makespan 的关键路径。 + +## 8. 从设备记录到可验收泳道 + +### 8.1 `run.sh swimlane` 的三份产物 + +```text +device trace + │ + ▼ +l2_swimlane_records.json raw，业务证据和十列整数 ABI + ├─ swimlane_converter.py + │ └─ merged_swimlane.json Perfetto/Chrome Trace 可视化 + └─ swimlane_exclusive_analyzer.py + └─ swimlane_exclusive_analysis.json 排他闭合与统计 +``` + +`swimlane` action 固定执行一轮，并默认打开逐 Atomic 观察。它按 backend 建立 +独立输出目录，先让 runner 完成语义校验和 raw 发布，再顺序运行 converter 和 +exclusive analyzer；任一步失败都会返回非零。 + +排他分析器直接读取 raw，不以 merged 文件作为数值输入。merged 的职责是展示； +exclusive JSON 的职责是用整数区间验证加和关系。 + +### 8.2 raw 记录表达什么 + +raw 的 `fdwic_events` 每行固定十列： + +```text +core, block, lane, task, function, phase, start, end, flags, auxiliary +``` + +时间戳单位不能按平台名称猜测，只能使用同一 raw metadata 中的 +`clock_freq_hz`。当前 standalone schema-v4 支持 phase-only level 1 和带 Atomic +的 level 4；二者业务 phase 相同，观察扰动不同。 + +### 8.3 converter 做的事情 + +converter 会： + +- 校验 schema、时钟、32 AIC + 64 AIV 拓扑和 producer summary； +- 把每个 mixed block 建成一个 Perfetto process； +- 为每条硬件 lane 拆出 runtime 和 kernel 子泳道； +- 根据 raw flags 标注 claim won/lost/not-attempted； +- 只用已有父子边界合成 `between_submit_residual`、`submit_residual` 和 + `submit_tail_gap`； +- 完整写入临时文件并同步后再原子替换 merged 输出。 + +合成 residual 不改变 raw ABI，也不凭空创造业务阶段。 + +### 8.4 exclusive analyzer 的 fail-closed 门禁 + +排他报告只有在以下条件全部满足时才生成： + +- 96 个 core id、32/64 role map 和 block/lane 映射完整； +- `dropped_records == 0`； +- 每核 Submit task id 连续且 task stream 完全一致； +- 每个 Submit 恰有 EfDrain、Materialize、PrepareMap、Claim、Register； +- Fanin 和 winner tail 的条件数量与 Claim winner/Alloc 标记一致； +- 同一父区间的排他 child 不重叠，且 task id 与父 Submit 一致； +- 每核恰有一个 OrchestrationReplay 和一个 FinalDrain，二者边界相邻； +- 每条 Kernel 唯一归入 EfDrain、winner tail 或 FinalDrain； +- 所有闭合式在 raw 整数 tick 上精确相等。 + +Host 在 raw 发布前还会校验共享 flag/vend/frontier、Claim/winner 计数、每核 +结果、trace header 和真实 workload 输出。也就是说，“能打开 JSON”不是验收 +标准；producer、Host 和离线分析三层都通过才是一份可用证据。 + +### 8.5 最小复现入口 + +```bash +cd tests/atomic_probe/pa_scheduler +./run.sh build ccec +./run.sh swimlane ccec --batches 256 +``` + +CCEC 才是本文 A5 mixed-core 路径的性能证据入口。CPU backend 用于协议和算术 +语义检查，不是 A5 时序基线；不同 backend 的绝对 duration 不应互相代替。 + +## 9. 与 `simpler` 主场景的差异 + +下面比较 standalone 与当前 +`examples/a5/fully_distributed_within_core/paged_attention_unroll` 加通用 +dist runtime。这里区分“生产 runtime 具备的能力”和“PA Case1 本次实际会走的 +分支”，避免把能力差异误写成每轮必然发生的动作。 + +| 维度 | standalone PA scheduler | `simpler` 主场景 | +| ---- | ----------------------- | ---------------- | +| 目标 | 独立复现 PA Case1 的 Submit/依赖/完成协议 | 通用 dist runtime 执行真实 example orchestration 和 kernel | +| Host/入口 | 专用 Host 准备 `SchedulerState` 并 launch mixed ELF | Runtime 准备 L2 参数，各 AICore 回放 orchestration entry | +| 图规模 | 每 batch 固定一组 `Alloc+QK+SF+PV+UP` | 可有多个 q-loop 和 block group；Case1 才退化为每 batch 5 task | +| 参数与数据 | PA 形状/descriptor 拓扑接近真实路径，地址与计算 workspace 为 standalone 输入 | descriptor 指向真实输入输出，kernel 结果构成真实数值数据流 | +| 当前 PA API | compete-first eager，所有 worker 构参 | 该 PA example 也使用 compete-first eager；旧 one-shot API 仍供其他 example 使用 | +| Begin/Finish ABI | 16-byte ticket，只携带单 lane winner 所需状态 | 32-byte ticket，还携带 joint、ready、kind 等生产状态 | +| Claim 能力 | 只允许单 lane，`active_count >= 2` 显式拒绝 | 通用 runtime 支持 joint task 和 BlockWon 发布/领取 | +| loser 尾动作 | kernel loser 无额外动作，只形成 residual | 调用 `drain_block_won()` 并记录 `LoserReplay`；Case1 可快速返回 | +| EfDrain | 只 drain 本核普通私有 slot | 先处理 BlockWon，再 drain phase-B 私有 slot | +| FinalDrain | 等 all-replayed 且本核 slot 为空 | 还必须确认没有 pending BlockWon lane | +| 数值 workload | 默认用独立 workspace 运行合成的 Cube/Vector 完整流水 | 执行 example 的真实 QK/SF/PV/UP kernel 与真实 tensor | +| trace record | standalone `TraceRecord` 为 64 B | `FdwicSwimlaneRecord` 为 32 B，且多 `LoserReplay` 等生产 phase | +| 拓扑假设 | analyzer 固定验证 32 AIC + 64 AIV | worker/block 数来自 Runtime 配置，通用代码不能假设永远为 96 | + +### 9.1 图规模的具体差异 + +真实 unroll orchestration 的一轮 q-scope 是： + +```text +Alloc × 1 +for each block group: + QK → SF → PV → UP +``` + +所以一般 task 数取决于 batch、`q_loop` 和 block group 数，并不总是 `5B`。 +standalone 固定 `q_loop=1` 且每 batch 只有一个 group，才得到本文的五 task +周期。本文第 3.6 节的计数门禁只适用于这个固定输入边界。 + +### 9.2 “aicpu_orchestration_entry” 名称不能按字面误读 + +在当前 CCEC dist replay 中，每个有效 AICore worker 从 Runtime 复制 +orchestration tensor/scalar 参数，再直接调用链接进设备镜像的 +`aicpu_orchestration_entry`。这个符号名沿用 API 历史，不表示本文泳道中的 +OrchestrationReplay 是 Host/AICPU 墙钟。 + +standalone 不经过这套通用 Runtime 参数装载，而是直接在 +`RunSchedulerImpl` 中构造固定的 `PaOrchestrationState`。 + +### 9.3 数据依赖相似，但数值计算不能等同 + +standalone 对 descriptor owner、TensorMap overlap、fanin flag 和 +vend/frontier 的调度依赖是实的；QK→SF→PV→UP 的顺序也是真实 PA Case1 +拓扑。 + +但默认 `real-compute` 的 engine workload 使用独立 workspace，目的在于提供 +稳定的 AIC/AIV 计算负载和完成等待，不会把 QK 的数值输出真正送入 SF，再送入 +PV/UP。因此可以用它研究调度和取时布局，不能用它验证 PA 数值正确性或把其 +kernel duration 直接当成主场景 kernel duration。 + +### 9.4 泳道数值不能跨两条路径直接横比 + +两条路径的 record 大小、BlockWon/loser 动作、Host 入口、计算体和编译布局都 +不同。即使 phase 名相同，absolute duration 也不具备天然可比性。 + +可以复用的是： + +- `OrchestrationReplay + FinalDrain` 的顶层业务口径； +- compete-first 的 EfDrain → Claim → eager ArgBuild → Finish 顺序； +- Materialize、PrepareMap、Fanin、Register 的业务定义； +- parent/child/residual/overlay 的排他建模方法。 + +必须重新验证的是： + +- 实际 task 数和 dependency graph； +- joint/BlockWon 与 `LoserReplay` 的条件分支； +- 每条 Kernel 的唯一父位置； +- Host 端完整 makespan 的起止边界； +- 观察插桩对真实 ELF 布局和性能的影响。 + +## 10. 源码阅读地图 + +建议按以下顺序阅读，先建立业务图，再进入平台细节： + +1. [common/pa_model.h](common/pa_model.h) + + 查看拓扑常量、TaskKind、共享/私有状态、TracePhase 和 raw ABI。 + +2. [common/pa_frontend.h](common/pa_frontend.h) + + 查看 descriptor、TensorMap、Materialize、Fanin、Register 和 slot payload。 + +3. [common/pa_scheduler_core.h](common/pa_scheduler_core.h) + + 从 `RunSchedulerImpl` 进入，再读 `SubmitCallbackTask`、 + `FinishCallbackSubmitBody`、`DrainReady`、`CompleteTask`。 + +4. [ccec/callback_runtime_entry.cpp](ccec/callback_runtime_entry.cpp) 与 + [ccec/callback_finish.cpp](ccec/callback_finish.cpp) + + 查看 CCEC worker 入口和 compete-first split finish 边界。 + +5. [ccec/host.cpp](ccec/host.cpp) 与 [run.sh](run.sh) + + 查看 Host launch、D2H 校验、raw 发布和三段后处理流水线。 + +6. [swimlane_converter.py](swimlane_converter.py) 与 + [swimlane_exclusive_analyzer.py](swimlane_exclusive_analyzer.py) + + 查看 Perfetto 映射、residual 合成和整数闭合门禁。 + +对照 `simpler` 主场景时，再阅读： + +- [生产 Submit runtime](../../../src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/submit_runtime.h) +- [生产 core main](../../../src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/core_main.h) +- [compete-first API](../../../src/a5/runtime/fully_distributed_within_core/orchestration/pto_orchestration_api.h) +- [PA unroll orchestration](../../../examples/a5/fully_distributed_within_core/paged_attention_unroll/kernels/orchestration/paged_attention_orch.cpp) + +## 结论 + +standalone PA scheduler 的主执行模型可以压缩为一句话： + +> 96 个 worker 同步重放同一条 PA task stream；每个 Submit 先推进旧任务、竞争 +> 唯一 winner，再由所有 worker 完成 eager 构参与私有依赖建模；winner 只把 +> kernel 放入 slot，后续 drain 在 fanin ready 后执行并发布全局完成，最后由 +> FinalDrain 清空尾部在途任务。 + +泳道的正确加和方式也可以压缩为一句话： + +> 只在同一物理 lane、同一父区间内，对互斥 child 与 residual 做整数闭合； +> Kernel、Atomic 和多核 aggregate 不能再次加到墙钟父区间上。 + +掌握这两句话后，再去看某个 phase 的时间分布，才能区分“业务工作发生在哪里”、 +“任务何时真正完成”和“观察器把时间记到了哪里”。 diff --git a/tests/atomic_probe/pa_scheduler/tensormap_inout_issue.md b/tests/atomic_probe/pa_scheduler/tensormap_inout_issue.md new file mode 100644 index 0000000000..659fa4d350 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/tensormap_inout_issue.md @@ -0,0 +1,180 @@ +# Shared TensorMap 复写意图的正确性与高性能协议 + +> 状态：问题分析和方案探索。§2.2 描述当前实现；§4 的候选方案 +> 尚未实现，必须经过 A5 onboard 测量后才能定案。 + +本文分析 Shared TensorMap 下的一个具体竞态：较早的复写 task 已经 +Claim，但它的 writer 元数据还没有发布；另一个核却已经回放到较晚 +的 reader task，因而在 lookup 时漏掉这个 writer。 + +本文按问题语义命名，不沿用会随评审文档调整的 `P0-x` 编号。当前 +[`shared_tensormap_record.md`](shared_tensormap_record.md) 中的 P0-2 已经是 +shared heap wrap 问题；本文对应其 §3.4 的 region-intent 保证边界。 + +## 1. 结论与范围 + +设 task id 满足 `A < B < C`： + +- task A 创建 Tensor X； +- task B 以 `INOUT` 或 `OUTPUT_EXISTING` 复写 X； +- task C 以 `INPUT` 读取 X。 + +正确依赖必须是： + +```mermaid +flowchart LR + A["task A
OUTPUT X"] --> B["task B
INOUT X"] + B --> C["task C
INPUT X"] +``` + +用户描述的竞态成立，也正是当前“writer 元数据就绪”闸门要阻止的 +问题。当且仅当所有复写都强制经过这个闸门时，现有实现对这个 +场景是正确的。 + +真正值得继续研究的不是“能否加一个等待”，而是如何同时满足： + +1. C 一定依赖 B； +2. B 的执行 winner 变慢时，其他 replay core 不被整体挡住； +3. fresh `OUTPUT` / 纯 `INPUT` 主路径仍保持 shared-loser 近似 + do-nothing； +4. 不引入新的 global exact-turn 或 global publish-prefix convoy。 + +“较早 reader 读到 future writer”是另一个历史查询问题。它可以通过 +版本历史、版本化句柄或禁止相应语义独立解决，不是本文的性能主线。 + +PA 当前 `q_loop == 1` 的用例也不是本问题的证据：该参数在语义上是 +`OUTPUT`，只是当前用例标成了 `INOUT`。应先修正用例约束，再为真正的 +复写链单独设计正确性和性能测试。 + +## 2. 竞态与当前修复 + +### 2.1 没有发布约束时如何漏依赖 + +task id 只规定了逻辑顺序，不保证各 winner 完成 lookup/register 的物理 +顺序。下图中，core 0 赢得 B，core 1 输掉 B 后继续回放并赢得 C： + +```mermaid +sequenceDiagram + participant K0 as core 0 + participant K1 as core 1 + participant M as writer index + + K0->>K0: Claim B and become winner + K1->>K1: Claim B and become loser + Note over K0: B winner becomes slow + K1->>K1: Continue replay + K1->>K1: Claim C and become winner + K1->>M: Look up X before C + M-->>K1: Return writer A + K1->>K1: build C with fanin A + K0->>M: publish writer B for X + Note over K0,K1: B was published too late for C +``` + +得到的错误依赖图是： + +```mermaid +flowchart LR + A["task A"] --> B["task B"] + A --> C["task C"] + B -.-> C +``` + +图中虚线 `B -.-> C` 表示应当存在但实际没有建立的依赖。 + +C 只等待 A 的 completion flag，因而可能与 B 并行，甚至在 B 之前读 X。 +问题不在于 Tensor 数据的 cache 可见性，而在于 C 构建依赖图时根本没有 +得到 B 这条边。 + +对任意 reader N，查询结果必须满足： + +```text +producer(N, X) = max { W | W < N and W writes an overlapping part of X } +``` + +要使该公式可计算，lookup 之前必须满足以下两者之一： + +- 所有可能相交的较早 writer 意图已经发布； +- lookup 能识别“较早 writer 尚未发布”，并延后或重试。 + +只做 task-id 过滤不能修复这个场景。`producer < N` 可以排除 future +writer，却无法区分“B 不存在”和“B 存在但还没发布”。 + +### 2.2 当前“winner 发布、loser 过门”为什么正确 + +当前流程是： + +1. B 先完成 Claim，产生一个 winner 和多个 loser； +2. B winner 收集 B 自己的 fanin； +3. B winner 更新 X 的最新 writer 状态，或追加 B 的 region writer 记录； +4. B winner 以 release 语义发布“B 的依赖元数据已就绪”； +5. B loser 只有以 acquire 语义观察到就绪状态后，才能继续到 C。 + +其中第 2、3 步在两类 Tensor 上的具体逻辑不同： + +| X 的表示 | B winner 对 X 做什么 | C 如何得到 B | +| -------- | --------------------- | --------------- | +| shared 符号 ref | 将最新 writer 从 A 换成 B，A 成为 B 的 fanin | 读到最新 writer B | +| ordinary region | 查到较早重叠 writer A，再追加 B 的区间记录 | 取小于 C 的最新重叠 writer B | + +因此，任何能进入 C 的核都已经观察到 B 的 writer 元数据： + +```mermaid +sequenceDiagram + participant W as B winner + participant L as B loser + participant M as writer index + + W->>W: collect B fanin + W->>M: publish B writer metadata + W->>L: release B metadata-ready + L->>L: acquire succeeds + L->>L: Continue replay + L->>L: Claim C and become winner + L->>M: lookup X + M-->>L: latest writer = B + L->>L: build C with fanin B +``` + +这个协议发布的是“B 的依赖元数据已就绪”，不是“B 的 kernel +已执行完”。C 真正执行前仍由 fanin task flag 等待 B 完成。 + +当前 loser 在等待时会尝试 drain 已有工作，因此不是绝对空转。但是 +它仍然禁止该核继续 replay 和 Claim 后续 task；当可 drain 的工作耗尽后， +才会退化为对同一就绪标记的轮询。 + +## 3. 性能问题的本质 + +现有协议把三件不同的事绑在了 B winner 的同一个发布点上： + +| 工作 | 谁真正需要 | C 查询前必须完成吗 | +| ---- | ------------ | ---------------------- | +| 声明“B 会写 X” | 后续 X 的 reader/writer | 是 | +| 计算 B 自己的全部 fanin | B winner | 否 | +| materialize、build 和 execute B | B winner / executor | 否 | + +当前就绪标记在前两项都完成后才发布。因此一个本来只需要很小的 +writer-intent 记录，会让所有 B loser 间接等待 B 的全部 fanin 收集。 + +如果复写 task 很密集，这会形成 replay frontier： + +```text +writer B0 prepared -> all losers pass +writer B1 prepared -> all losers pass +... +``` + +主要代价是： + +- 同步范围过大：与 X 无关的核也不能越过 B； +- 关键路径过长：发布被绑定到 B winner 的 fanin 收集； +- 复写密集时，可用的 run-ahead 和 Claim 并行度被逐个截断； +- 多个 loser 可能集中读取同一个 B 元数据就绪 cache line。 + +最终方案不应用一个 global task-id publish prefix 代替当前 marker。那只是把 +“所有 loser 等 B”改成“所有后续 lookup winner 等最慢的较早 task”，无关 +Tensor 之间仍然会发生 head-of-line blocking。fresh-output exact-slot 路径更不应 +等待全局前缀。 + +同样，不能只做“先 lookup，之后看情况重试”。没有 writer announcement +或 completeness marker 时，C 无法知道自己看到的 A 是正确结果还是暂时结果。 \ No newline at end of file diff --git a/tests/atomic_probe/pa_scheduler/test/test_atomic_poll_batch.cpp b/tests/atomic_probe/pa_scheduler/test/test_atomic_poll_batch.cpp new file mode 100644 index 0000000000..cda7da8c26 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test/test_atomic_poll_batch.cpp @@ -0,0 +1,452 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include +#include + +// 该自测在普通 CPU 编译器上直接实例化 device 公共模板：只消去地址空间 +// 修饰符，不复制 PollBatch 实现，避免测试与被测代码各维护一套逻辑。 +#define PA_DEVICE inline +#define PA_GM +#include "pa_trace.h" + +namespace { + +using pa_scheduler::AtomicOp; +using pa_scheduler::AtomicPollBatchEnabled; +using pa_scheduler::AtomicPollBoundaryAt; +using pa_scheduler::AtomicPollRegionBegin; +using pa_scheduler::AtomicPollRegionEnd; +using pa_scheduler::AtomicSite; +using pa_scheduler::AccumulateAtomicPollCall; +using pa_scheduler::CaptureAtomicCompareExchange; +using pa_scheduler::TraceAtomicLoad; +using pa_scheduler::TraceAtomicPollBatchMask; +using pa_scheduler::TraceAtomicPollBatchIndex; +using pa_scheduler::TraceContext; +using pa_scheduler::TraceCoreState; +using pa_scheduler::TracePhase; +using pa_scheduler::TraceRecord; +using pa_scheduler::WorkerResult; +using pa_scheduler::WriteAggregateAtomicPollBatch; +using pa_scheduler::WriteAtomicTrace; +using pa_scheduler::kAtomicOpMask; +using pa_scheduler::kAtomicPollBatch; +using pa_scheduler::kAtomicPollCountMax; +using pa_scheduler::kAtomicPollCountShift; +using pa_scheduler::kAtomicResultUsed; +using pa_scheduler::kAtomicReturnReady; + +int g_failures = 0; + +void Expect(bool condition, const char *message) { + if (condition) return; + std::fprintf(stderr, "[FAIL] atomic PollBatch: %s\n", message); + ++g_failures; +} + +// 可控时钟让边界断言不依赖 host 调度；Load 只为验证 trace 包装器的分流， +// 不尝试在这个单线程单元测试中模拟 A5 atomicAdd(0) 的硬件时延。 +struct TestOps { + static constexpr bool kAtomicReturnReadyObserved = false; + static uint64_t now; + + static uint64_t Now() { return now++; } + + template + static uint64_t NowAfterAtomicResult(T value) { + (void)value; + return Now(); + } + + template + static T Load(volatile T *address) { + return *address; + } + + static int64_t CompareExchange( + volatile int64_t *address, int64_t expected, + int64_t desired + ) { + const int64_t observed = *address; + if (observed == expected) { + *address = desired; + } + return observed; + } +}; + +uint64_t TestOps::now = 0; + +struct ReturnReadyTestOps : TestOps { + static constexpr bool kAtomicReturnReadyObserved = true; +}; + +struct Fixture { + TraceCoreState core{}; + TraceRecord records[8]{}; + WorkerResult result{}; + TraceContext trace{}; + + Fixture() { + trace.core = &core; + trace.records = records; + trace.capacity = + static_cast(sizeof(records) / sizeof(records[0])); + trace.atomics_enabled = true; + } +}; + +void TestSplitAtMaximumCount() { + Fixture fixture; + constexpr AtomicSite kSite = AtomicSite::StartupPoll; + constexpr uint32_t kBatchIndex = 0; + constexpr uint32_t kBatchBit = 1U << kBatchIndex; + constexpr uint64_t kFirstStart = 111; + constexpr uint64_t kSecondStart = 222; + constexpr uint64_t kSecondEnd = 333; + + Expect( + TraceAtomicPollBatchIndex(kSite) == static_cast(kBatchIndex), + "StartupPoll 的 batch index 与稳定映射不一致" + ); + fixture.trace.poll_burst.active_mask = kBatchBit; + fixture.trace.poll_burst.start_cycle[kBatchIndex] = kFirstStart; + fixture.trace.poll_burst.call_count[kBatchIndex] = kAtomicPollCountMax - 1; + TestOps::now = 200; + + // 第 0xFFFFFF 次调用属于第一条记录，并在达到 24-bit 上限时立即落盘。 + AccumulateAtomicPollCall(fixture.trace, fixture.result, kSite, kFirstStart); + + Expect(fixture.trace.record_count == 1U, "达到最大计数时应立即写出第一条记录"); + Expect(fixture.trace.poll_batch_records == 1U, "第一条 PollBatch 物理记录计数错误"); + Expect(fixture.trace.poll_burst.active_mask == 0U, "达到上限后 active mask 应清零"); + Expect( + fixture.trace.poll_burst.call_count[kBatchIndex] == 0U, + "达到上限后站点调用计数应清零" + ); + Expect( + fixture.records[0].phase == static_cast(TracePhase::Atomic), + "第一条记录 phase 不是 Atomic" + ); + Expect( + fixture.records[0].auxiliary == static_cast(kSite), + "第一条记录 site 不正确" + ); + Expect(fixture.records[0].start_cycle == kFirstStart, "第一条记录起始时钟不正确"); + Expect( + fixture.records[0].end_cycle >= fixture.records[0].start_cycle, + "第一条记录的结束时钟早于起始时钟" + ); + Expect( + (fixture.records[0].flags & kAtomicPollBatch) != 0U, + "第一条记录缺少 PollBatch 标志" + ); + Expect( + fixture.records[0].flags >> kAtomicPollCountShift == kAtomicPollCountMax, + "第一条记录没有编码最大 24-bit 调用数" + ); + + // 第 0x1000000 次调用必须重新开启 count=1 的新 batch，不能饱和或丢失。 + AccumulateAtomicPollCall(fixture.trace, fixture.result, kSite, kSecondStart); + Expect(fixture.trace.poll_burst.active_mask == kBatchBit, "上限后的下一次调用没有重开 batch"); + Expect( + fixture.trace.poll_burst.call_count[kBatchIndex] == 1U, + "重开 batch 的初始调用数不是 1" + ); + AtomicPollBoundaryAt(fixture.trace, kSecondEnd); + + Expect(fixture.trace.record_count == 2U, "max+1 次调用应写出两条记录"); + Expect(fixture.trace.poll_batch_records == 2U, "max+1 次调用的物理 batch 数错误"); + Expect(fixture.trace.poll_burst.active_mask == 0U, "第二条记录关闭后 active mask 未清零"); + Expect( + fixture.trace.poll_burst.call_count[kBatchIndex] == 0U, + "第二条记录关闭后站点调用计数未清零" + ); + Expect(fixture.records[1].start_cycle == kSecondStart, "第二条记录起始时钟不正确"); + Expect(fixture.records[1].end_cycle == kSecondEnd, "第二条记录结束时钟不正确"); + Expect( + (fixture.records[1].flags & kAtomicPollBatch) != 0U, + "第二条记录缺少 PollBatch 标志" + ); + Expect( + fixture.records[1].flags >> kAtomicPollCountShift == 1U, + "第二条记录没有精确编码一次调用" + ); + const uint64_t represented_calls = + static_cast(fixture.records[0].flags >> kAtomicPollCountShift) + + static_cast(fixture.records[1].flags >> kAtomicPollCountShift); + Expect( + represented_calls == static_cast(kAtomicPollCountMax) + 1U, + "两条记录的加权调用数没有闭合到 max+1" + ); + Expect(fixture.trace.dropped_records == 0U, "边界拆批不应丢记录"); + Expect(!fixture.trace.atomic_counter_overflow, "边界拆批不应报告计数溢出"); +} + +void TestNestedRegionRestoresMask() { + Fixture fixture; + volatile int64_t startup_value = 96; + volatile int64_t fanin_value = 1; + const uint32_t startup_mask = TraceAtomicPollBatchMask(AtomicSite::StartupPoll); + const uint32_t fanin_mask = TraceAtomicPollBatchMask(AtomicSite::FaninFlagLoad); + TestOps::now = 1000; + + const uint32_t outer_previous = + AtomicPollRegionBegin(fixture.trace, fixture.result, startup_mask); + Expect(outer_previous == 0U, "最外层 region 的 previous mask 应为零"); + Expect(fixture.trace.poll_burst.enabled_mask == startup_mask, "最外层 region 未启用 startup site"); + Expect( + AtomicPollBatchEnabled(fixture.trace, AtomicSite::StartupPoll, AtomicOp::Load), + "最外层 startup site 应允许聚合" + ); + (void)TraceAtomicLoad( + fixture.trace, fixture.result, -1, AtomicSite::StartupPoll, &startup_value + ); + + // 嵌套 begin 会先关闭外层已有 batch，再把 inner mask 与外层 mask 合并。 + const uint32_t inner_previous = + AtomicPollRegionBegin(fixture.trace, fixture.result, fanin_mask); + Expect(inner_previous == startup_mask, "内层 region 没有保存外层 mask"); + Expect( + fixture.trace.poll_burst.enabled_mask == (startup_mask | fanin_mask), + "内层 region 没有合并两层 mask" + ); + Expect(fixture.trace.record_count == 1U, "内层 begin 没有关闭外层 active batch"); + (void)TraceAtomicLoad( + fixture.trace, fixture.result, -1, AtomicSite::FaninFlagLoad, &fanin_value + ); + + AtomicPollRegionEnd(fixture.trace, fixture.result, inner_previous); + Expect(fixture.trace.poll_burst.enabled_mask == startup_mask, "内层 end 没有还原外层 mask"); + Expect(fixture.trace.record_count == 2U, "内层 end 没有关闭内层 active batch"); + (void)TraceAtomicLoad( + fixture.trace, fixture.result, -1, AtomicSite::StartupPoll, &startup_value + ); + + AtomicPollRegionEnd(fixture.trace, fixture.result, outer_previous); + Expect(fixture.trace.poll_burst.enabled_mask == 0U, "最外层 end 没有还原初始 mask"); + Expect(fixture.trace.poll_burst.active_mask == 0U, "嵌套 region 结束后仍有 active batch"); + Expect(fixture.trace.record_count == 3U, "嵌套 region 应按三个边界写出三条 batch"); + Expect(fixture.trace.poll_batch_records == 3U, "嵌套 region 的物理 batch 计数错误"); + Expect(fixture.trace.poll_calls == 3U, "嵌套 region 的逻辑 poll 调用数错误"); + Expect(fixture.result.atomic_trace_calls == 3U, "嵌套 region 的逻辑 atomic 调用数错误"); + Expect( + fixture.records[0].auxiliary == static_cast(AtomicSite::StartupPoll) && + fixture.records[1].auxiliary == static_cast(AtomicSite::FaninFlagLoad) && + fixture.records[2].auxiliary == static_cast(AtomicSite::StartupPoll), + "嵌套 region 的 batch site 顺序错误" + ); + Expect(fixture.trace.dropped_records == 0U, "嵌套 region 不应丢记录"); + Expect(!fixture.trace.atomic_counter_overflow, "嵌套 region 不应报告计数溢出"); +} + +void TestNonAllowlistedSiteStaysDirect() { + Fixture fixture; + volatile int64_t frontier_flag = 7; + constexpr AtomicSite kSite = AtomicSite::FrontierFlagLoad; + const uint32_t site_mask = TraceAtomicPollBatchMask(kSite); + TestOps::now = 2000; + + Expect(site_mask == 0U, "非 PollBatch allowlist 的 site 必须得到空 compact mask"); + const uint32_t previous = + AtomicPollRegionBegin(fixture.trace, fixture.result, site_mask); + Expect( + !AtomicPollBatchEnabled(fixture.trace, kSite, AtomicOp::Load), + "非 allowlist 的 frontier scan 不得因 region mask 被聚合" + ); + const int64_t observed = TraceAtomicLoad( + fixture.trace, fixture.result, 37, kSite, &frontier_flag + ); + AtomicPollRegionEnd(fixture.trace, fixture.result, previous); + + Expect(observed == frontier_flag, "非 allowlist direct load 返回值错误"); + Expect(fixture.trace.record_count == 1U, "非 allowlist load 应写一条 direct 记录"); + Expect(fixture.trace.poll_calls == 0U, "非 allowlist load 不得增加 batched poll 调用数"); + Expect(fixture.trace.poll_batch_records == 0U, "非 allowlist load 不得写 PollBatch 记录"); + Expect(fixture.result.atomic_trace_calls == 1U, "非 allowlist direct load 的逻辑调用计数错误"); + Expect( + (fixture.records[0].flags & kAtomicPollBatch) == 0U, + "非 allowlist load 被错误标成 PollBatch" + ); + Expect(fixture.records[0].task_id == 37, "非 allowlist direct load 丢失 task 归因"); + Expect( + fixture.records[0].auxiliary == static_cast(kSite), + "非 allowlist direct load 的 site 错误" + ); + Expect(fixture.trace.poll_burst.active_mask == 0U, "非 allowlist load 不应留下 active batch"); + Expect(fixture.trace.dropped_records == 0U, "非 allowlist direct load 不应丢记录"); + Expect(!fixture.trace.atomic_counter_overflow, "非 allowlist direct load 不应报告溢出"); +} + +void TestRawSiteIdNeverBecomesTheEnableMaskBit() { + constexpr AtomicSite kHighRawSite = static_cast(40); + Expect( + TraceAtomicPollBatchIndex(kHighRawSite) == -1, + "未登记的高编号 site 不应获得 compact PollBatch index" + ); + Expect( + TraceAtomicPollBatchMask(kHighRawSite) == 0U, + "高编号 raw site 不得参与 32-bit 移位构造 enable mask" + ); + Expect( + TraceAtomicPollBatchMask(AtomicSite::FaninFlagLoad) == (1U << 2), + "Fanin 的 enable bit 必须来自 compact index 2，而不是 raw site 5" + ); + Expect( + TraceAtomicPollBatchIndex( + AtomicSite::SharedInsertTurnPoll + ) == -1 && + TraceAtomicPollBatchMask( + AtomicSite::SharedInsertTurnPoll + ) == 0, + "aggregate-only insert-turn poll 不得扩张热循环 compact state" + ); +} + +void TestAggregateInsertTurnPollBatch() { + Fixture fixture; + constexpr uint64_t kBegin = 4000; + constexpr uint64_t kEnd = 4500; + constexpr uint64_t kCalls = 37; + const bool written = WriteAggregateAtomicPollBatch( + fixture.trace, fixture.result, + AtomicSite::SharedInsertTurnPoll, + kBegin, kEnd, kCalls, true + ); + Expect(written, "insert-turn aggregate PollBatch 应写入一条记录"); + Expect(fixture.trace.record_count == 1, "aggregate PollBatch 物理记录数不是 1"); + Expect( + fixture.result.atomic_trace_calls == kCalls && + fixture.trace.poll_calls == kCalls, + "aggregate PollBatch 没有一次性累计精确 logical calls" + ); + Expect( + fixture.trace.poll_batch_records == 1, + "aggregate PollBatch 物理 batch 计数不是 1" + ); + const TraceRecord &record = fixture.records[0]; + Expect( + record.start_cycle == kBegin && + record.end_cycle == kEnd, + "aggregate PollBatch 没有复用传入的 Register/Ready 边界" + ); + Expect( + record.auxiliary == + static_cast( + AtomicSite::SharedInsertTurnPoll + ), + "aggregate PollBatch site 不正确" + ); + Expect( + (record.flags & kAtomicOpMask) == + static_cast(AtomicOp::Load) && + (record.flags & kAtomicResultUsed) != 0 && + (record.flags & kAtomicPollBatch) != 0 && + (record.flags & kAtomicReturnReady) != 0, + "aggregate PollBatch 的 Load/result/poll/return-ready 标志不闭合" + ); + Expect( + record.flags >> kAtomicPollCountShift == kCalls, + "aggregate PollBatch 未编码精确 logical call_count" + ); + + Fixture overflow; + const bool overflow_written = WriteAggregateAtomicPollBatch( + overflow.trace, overflow.result, + AtomicSite::SharedInsertTurnPoll, + kBegin, kEnd, + static_cast(kAtomicPollCountMax) + 1, + true + ); + Expect( + !overflow_written && overflow.trace.record_count == 0 && + overflow.result.atomic_trace_calls == 0 && + overflow.trace.poll_calls == 0 && + overflow.trace.atomic_counter_overflow, + "超过 24-bit 的聚合调用数必须 fail-closed，不能饱和或拆批" + ); +} + +void TestInsertTurnHandoffCompareExchange() { + Fixture fixture; + volatile int64_t token = 7; + uint64_t trace_begin = 0; + uint64_t trace_end = 0; + TestOps::now = 5000; + const int64_t observed = + CaptureAtomicCompareExchange( + fixture.trace, &token, 7, 8, + trace_begin, trace_end + ); + Expect( + observed == 7 && token == 8, + "handoff CompareExchange 返回值或目标 token 不正确" + ); + Expect( + fixture.trace.record_count == 0 && + fixture.result.atomic_trace_calls == 0, + "CAS 捕获阶段不得提前写 raw 或更新 logical counter" + ); + WriteAtomicTrace( + fixture.trace, fixture.result, 7, + AtomicSite::SharedInsertTurnHandoff, + AtomicOp::CompareExchange, + trace_begin, trace_end, true, true + ); + Expect( + fixture.trace.record_count == 1 && + fixture.result.atomic_trace_calls == 1, + "父/detail 端点固定后，handoff CAS 必须恰好写一条 direct atomic" + ); + const TraceRecord &record = fixture.records[0]; + Expect( + record.task_id == 7 && + record.auxiliary == + static_cast( + AtomicSite::SharedInsertTurnHandoff + ), + "handoff CAS 没有保留 task/site 身份" + ); + Expect( + (record.flags & kAtomicOpMask) == + static_cast( + AtomicOp::CompareExchange + ) && + (record.flags & kAtomicResultUsed) != 0 && + (record.flags & kAtomicReturnReady) != 0 && + (record.flags & kAtomicPollBatch) == 0, + "handoff CAS 的 op/result/return-ready/direct 标志不正确" + ); + Expect( + trace_end == record.end_cycle && + record.end_cycle >= record.start_cycle, + "handoff CAS 记录没有使用捕获的返回依赖边界" + ); +} + +} // namespace + +int main() { + TestSplitAtMaximumCount(); + TestNestedRegionRestoresMask(); + TestNonAllowlistedSiteStaysDirect(); + TestRawSiteIdNeverBecomesTheEnableMaskBit(); + TestAggregateInsertTurnPollBatch(); + TestInsertTurnHandoffCompareExchange(); + if (g_failures != 0) { + std::fprintf(stderr, "[FAIL] atomic PollBatch self-test failures=%d\n", g_failures); + return EXIT_FAILURE; + } + std::printf("[PASS] atomic PollBatch split/region/allowlist self-test\n"); + return EXIT_SUCCESS; +} diff --git a/tests/atomic_probe/pa_scheduler/test/test_private_tensor_map_ring.cpp b/tests/atomic_probe/pa_scheduler/test/test_private_tensor_map_ring.cpp new file mode 100644 index 0000000000..e275a7a181 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test/test_private_tensor_map_ring.cpp @@ -0,0 +1,536 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +// 本测试直接实例化 standalone 的设备公共实现，只消去地址空间修饰符； +// 不复制一份被测 ring 算法，vector reference 只保存抽象的区间/producer 语义。 +#define PA_DEVICE inline +#define PA_GM +#include "pa_scheduler_core.h" + +namespace { + +using pa_scheduler::CountLiveMapEntries; +using pa_scheduler::DataType; +using pa_scheduler::InsertTensor; +using pa_scheduler::LookupTensor; +using pa_scheduler::MapEntry; +using pa_scheduler::ResetTensorMap; +using pa_scheduler::TensorDesc; +using pa_scheduler::TensorMap; +using pa_scheduler::TensorMapSlotIndex; +using pa_scheduler::AdvanceTensorMap; +using pa_scheduler::kMapBucketCapacity; +using pa_scheduler::kMapBuckets; +using pa_scheduler::kMapCapacity; +using pa_scheduler::kMapBucketShift; +using pa_scheduler::kTaskWindow; + +static_assert(PTO_FDWIC_SHARED_MAP == 0, "this test covers only the private ring discipline"); +static_assert(kMapCapacity == kMapBuckets * kMapBucketCapacity, "private TensorMap capacity mismatch"); +static_assert(sizeof(MapEntry) == 48, "MapEntry must preserve the standalone/production-compatible ABI"); +static_assert(alignof(MapEntry) == 8, "MapEntry alignment changed"); +#if PTO_FDWIC_TENSORMAP_RING_CAP == 128 +static_assert(kMapBuckets == 128, "default private TensorMap ABI requires 128 buckets"); +static_assert(kMapBucketCapacity == 128, "default private TensorMap ABI requires 128 slots per bucket"); +static_assert(kMapBucketShift == 7, "default 128-bucket hash must consume seven high bits"); +static_assert(sizeof(TensorMap) == 823312, "TensorMap must preserve the WorkerState ABI"); +#endif +static_assert(alignof(TensorMap) == 8, "TensorMap alignment changed"); +static_assert(std::is_standard_layout::value, "MapEntry must remain a standard-layout ABI type"); +static_assert(std::is_trivially_copyable::value, "MapEntry must remain trivially copyable"); +static_assert(std::is_standard_layout::value, "TensorMap must remain a standard-layout ABI type"); + +int g_failures = 0; + +void Expect(bool condition, const char *test, const char *message) { + if (condition) { + return; + } + std::fprintf(stderr, "[FAIL] private TensorMap ring/%s: %s\n", test, message); + ++g_failures; +} + +void ExpectEqual(int64_t actual, int64_t expected, const char *test, const char *field) { + if (actual == expected) { + return; + } + std::fprintf( + stderr, "[FAIL] private TensorMap ring/%s: %s actual=%lld expected=%lld\n", + test, field, static_cast(actual), static_cast(expected) + ); + ++g_failures; +} + +uint64_t ElementBytes(DataType dtype) { + switch (dtype) { + case DataType::Float32: + case DataType::Int32: + case DataType::Uint32: + return 4; + case DataType::Float16: + case DataType::Int16: + case DataType::Bfloat16: + case DataType::Uint16: + return 2; + case DataType::Int8: + case DataType::Uint8: + case DataType::Bool: + return 1; + case DataType::Int64: + case DataType::Uint64: + return 8; + default: + std::abort(); + } +} + +TensorDesc MakeTensor( + uint64_t buffer_addr, uint64_t start_offset, uint32_t extent, DataType dtype = DataType::Float32, + bool contiguous = true +) { + TensorDesc tensor{}; + tensor.buffer_addr = buffer_addr; + tensor.buffer_size = 1ULL << 32; + tensor.owner_task_id = UINT64_MAX; + tensor.start_offset = start_offset; + tensor.version = 0; + tensor.ndims = 1; + tensor.dtype = dtype; + tensor.manual_dep = false; + tensor.is_contiguous = contiguous; + tensor.child_memory = 0; + tensor.shapes[0] = extent; + tensor.extent_elem_cache = extent; + tensor.strides[0] = 1; + return tensor; +} + +struct ByteRange { + uint64_t buffer_addr; + uint64_t lo; + uint64_t hi; +}; + +ByteRange ReferenceByteRange(const TensorDesc &tensor) { + uint64_t extent = tensor.extent_elem_cache; + if (tensor.is_contiguous) { + extent = 1; + for (uint32_t dimension = 0; dimension < tensor.ndims; ++dimension) { + extent *= tensor.shapes[dimension]; + } + } + const uint64_t element_bytes = ElementBytes(tensor.dtype); + return { + tensor.buffer_addr, + tensor.start_offset * element_bytes, + (tensor.start_offset + extent) * element_bytes, + }; +} + +uint32_t ReferenceBucket(uint64_t buffer_addr) { + // 与设计文档固定的乘法哈希一致；reference 不读取被测 ring 的 head/tail， + // 只需知道一条抽象 entry 属于哪个有界桶，才能独立判断插入是否应当失败。 +#if PTO_FDWIC_TENSORMAP_RING_CAP == 16384 + (void)buffer_addr; + return 0; +#else + const uint64_t mixed = buffer_addr * 0x9E3779B97F4A7C15ULL; + return static_cast(mixed >> (64U - kMapBucketShift)); +#endif +} + +bool Overlaps(const ByteRange &left, const ByteRange &right) { + return left.buffer_addr == right.buffer_addr && left.lo < right.hi && right.lo < left.hi; +} + +struct ReferenceEntry { + ByteRange range; + int32_t producer; + uint32_t bucket; +}; + +class ReferenceMap { +public: + void Reset() { + entries_.clear(); + alive_floor_ = 0; + } + + void Advance(uint32_t task_id, int32_t heap_window) { + const int32_t candidate = static_cast(task_id) - heap_window; + if (candidate <= alive_floor_) { + return; + } + alive_floor_ = candidate; + entries_.erase( + std::remove_if( + entries_.begin(), entries_.end(), + [&](const ReferenceEntry &entry) { return entry.producer < alive_floor_; } + ), + entries_.end() + ); + } + + bool Insert(const TensorDesc &tensor, int32_t producer) { + const ByteRange range = ReferenceByteRange(tensor); + const uint32_t bucket = ReferenceBucket(range.buffer_addr); + const size_t live_in_bucket = static_cast(std::count_if( + entries_.begin(), entries_.end(), + [&](const ReferenceEntry &entry) { return entry.bucket == bucket; } + )); + if (live_in_bucket >= kMapBucketCapacity) { + return false; + } + entries_.push_back({range, producer, bucket}); + return true; + } + + int32_t Lookup(const TensorDesc &tensor) const { + const ByteRange query = ReferenceByteRange(tensor); + int32_t best = -1; + for (const ReferenceEntry &entry : entries_) { + if (entry.producer >= alive_floor_ && Overlaps(entry.range, query)) { + best = std::max(best, entry.producer); + } + } + return best; + } + + uint32_t Count() const { return static_cast(entries_.size()); } + +private: + std::vector entries_; + int32_t alive_floor_ = 0; +}; + +std::unique_ptr NewMap() { + // TensorMap 是大对象，测试也必须遵守生产侧的 heap/GM 放置假设，避免 host + // 线程默认栈大小掩盖或制造与 ring 语义无关的失败。 + auto map = std::make_unique(); + ResetTensorMap(*map); + return map; +} + +void TestPhysicalLayoutBoundaries() { + constexpr const char *kTest = "physical-layout-boundaries"; + auto map = NewMap(); + ExpectEqual( + TensorMapSlotIndex(0, 0), 0, + kTest, "first physical slot" + ); + ExpectEqual( + TensorMapSlotIndex( + kMapBuckets - 1U, + static_cast(kMapBucketCapacity - 1U) + ), + kMapCapacity - 1U, + kTest, "last physical slot" + ); + ExpectEqual( + TensorMapSlotIndex( + kMapBuckets - 1U, + static_cast(kMapBucketCapacity) + ), + (kMapBuckets - 1U) * kMapBucketCapacity, + kTest, "last bucket first wrapped slot" + ); + + // 直接检查物理数组而不是用同一个 helper 算 expected。CAP32/64 必须 + // 跨过默认前128桶与 ABI padding 中额外游标的分界。 +#if PTO_FDWIC_TENSORMAP_RING_CAP <= 128 + TensorMapBucketHead(*map, 127U) = 17; + TensorMapBucketTail(*map, 127U) = 27; + ExpectEqual(map->bucket_heads[127], 17, kTest, "base head bucket 127"); + ExpectEqual(map->bucket_tails[127], 27, kTest, "base tail bucket 127"); +#endif +#if PTO_FDWIC_TENSORMAP_RING_CAP < 128 + TensorMapBucketHead(*map, 128U) = 18; + TensorMapBucketTail(*map, 128U) = 28; + TensorMapBucketHead(*map, kMapBuckets - 1U) = 19; + TensorMapBucketTail(*map, kMapBuckets - 1U) = 29; + ExpectEqual(map->extra_bucket_heads[0], 18, kTest, "extra head bucket 128"); + ExpectEqual(map->extra_bucket_tails[0], 28, kTest, "extra tail bucket 128"); + ExpectEqual( + map->extra_bucket_heads[kMapBuckets - 129U], 19, + kTest, "last extra head" + ); + ExpectEqual( + map->extra_bucket_tails[kMapBuckets - 129U], 29, + kTest, "last extra tail" + ); +#else + TensorMapBucketHead(*map, kMapBuckets - 1U) = 19; + TensorMapBucketTail(*map, kMapBuckets - 1U) = 29; + ExpectEqual( + map->bucket_heads[kMapBuckets - 1U], 19, + kTest, "last base head" + ); + ExpectEqual( + map->bucket_tails[kMapBuckets - 1U], 29, + kTest, "last base tail" + ); +#endif +} + +void TestEmptyAndHalfOpenIntervals() { + constexpr const char *kTest = "empty-and-half-open"; + auto map = NewMap(); + const TensorDesc left = MakeTensor(0x100000000ULL, 0, 4); + const TensorDesc touching = MakeTensor(0x100000000ULL, 4, 4); + const TensorDesc overlap = MakeTensor(0x100000000ULL, 3, 2); + + ExpectEqual(CountLiveMapEntries(*map), 0, kTest, "reset live count"); + ExpectEqual(LookupTensor(*map, left), -1, kTest, "empty lookup"); + Expect(InsertTensor(*map, left, 2), kTest, "first insert must succeed"); + ExpectEqual(LookupTensor(*map, touching), -1, kTest, "touching half-open ranges"); + ExpectEqual(LookupTensor(*map, overlap), 2, kTest, "overlapping half-open ranges"); +} + +void TestLatestProducerAndAliveFloor() { + constexpr const char *kTest = "latest-and-alive-floor"; + auto map = NewMap(); + const TensorDesc region = MakeTensor(0x200000000ULL, 8, 8); + + Expect(InsertTensor(*map, region, 3), kTest, "producer 3 insert"); + Expect(InsertTensor(*map, region, 7), kTest, "producer 7 insert"); + Expect(InsertTensor(*map, region, 5), kTest, "producer 5 insert"); + ExpectEqual(LookupTensor(*map, region), 7, kTest, "lookup must select maximum producer"); + + auto boundary_map = NewMap(); + const TensorDesc producer_9 = MakeTensor(0x300000000ULL, 0, 1); + const TensorDesc producer_10 = MakeTensor(0x300000000ULL, 2, 1); + Expect(InsertTensor(*boundary_map, producer_9, 9), kTest, "producer 9 insert"); + Expect(InsertTensor(*boundary_map, producer_10, 10), kTest, "producer 10 insert"); + + AdvanceTensorMap(*boundary_map, 20, 10); + ExpectEqual(LookupTensor(*boundary_map, producer_9), -1, kTest, "producer below alive_floor"); + ExpectEqual(LookupTensor(*boundary_map, producer_10), 10, kTest, "producer at alive_floor"); + AdvanceTensorMap(*boundary_map, 21, 10); + ExpectEqual(LookupTensor(*boundary_map, producer_10), -1, kTest, "producer after floor advances"); + ExpectEqual(CountLiveMapEntries(*boundary_map), 0, kTest, "floor retirement count"); +} + +void TestTaskWindowWrapAndMultipleLaps() { + constexpr const char *kTest = "task-window-wrap-and-multiple-laps"; + auto map = NewMap(); + const TensorDesc region = MakeTensor(0x400000000ULL, 0, 4); + // 该用例要验证“窗口内始终可追加并跨多 lap”，不是验证配置过小 + // 的 FATAL；因此窗口必须严格小于当前隔离 CAP。满环失败由下一用例 + // 独立覆盖。 + constexpr int32_t kWindow = + kMapBucketCapacity > 64U + ? 64 + : static_cast(kMapBucketCapacity - 1U); + constexpr uint32_t kLastTask = 4 * kTaskWindow + 257; + + for (uint32_t task_id = 0; task_id <= kLastTask; ++task_id) { + AdvanceTensorMap(*map, task_id, kWindow); + if (!InsertTensor(*map, region, static_cast(task_id))) { + std::fprintf( + stderr, "[FAIL] private TensorMap ring/%s: insert failed at task=%u\n", kTest, task_id + ); + ++g_failures; + return; + } + const int32_t producer = LookupTensor(*map, region); + if (producer != static_cast(task_id)) { + std::fprintf( + stderr, + "[FAIL] private TensorMap ring/%s: latest producer at task=%u actual=%d\n", + kTest, task_id, producer + ); + ++g_failures; + return; + } + const uint32_t live = CountLiveMapEntries(*map); + if (live > static_cast(kWindow + 1)) { + std::fprintf( + stderr, "[FAIL] private TensorMap ring/%s: live=%u at task=%u exceeds window\n", + kTest, live, task_id + ); + ++g_failures; + return; + } + } + + ExpectEqual( + CountLiveMapEntries(*map), static_cast(kWindow + 1), kTest, "final live window" + ); + ExpectEqual(LookupTensor(*map, region), kLastTask, kTest, "latest producer after multiple laps"); +} + +void TestFullBucketDoesNotOverwrite() { + constexpr const char *kTest = "full-bucket-no-overwrite"; + auto map = NewMap(); + std::vector existing; + existing.reserve(kMapBucketCapacity); + + for (uint32_t index = 0; index < kMapBucketCapacity; ++index) { + existing.push_back(MakeTensor(0x500000000ULL, 2ULL * index, 1)); + if (!InsertTensor(*map, existing.back(), static_cast(index))) { + std::fprintf( + stderr, "[FAIL] private TensorMap ring/%s: insert %u of %u failed\n", + kTest, index + 1, kMapBucketCapacity + ); + ++g_failures; + return; + } + } + + ExpectEqual(CountLiveMapEntries(*map), kMapBucketCapacity, kTest, "full bucket live count"); + const bool inserted = InsertTensor(*map, existing.front(), 1000); + Expect(!inserted, kTest, "CAP+1 insert must fail"); + ExpectEqual( + CountLiveMapEntries(*map), kMapBucketCapacity, kTest, "failed insert must not change count" + ); + for (uint32_t index = 0; index < existing.size(); ++index) { + const int32_t producer = LookupTensor(*map, existing[index]); + if (producer != static_cast(index)) { + std::fprintf( + stderr, + "[FAIL] private TensorMap ring/%s: failed insert changed old result index=%u actual=%d\n", + kTest, index, producer + ); + ++g_failures; + return; + } + } +} + +TensorDesc RandomTensor(std::mt19937_64 &random) { + constexpr DataType kDtypes[] = { + DataType::Float32, + DataType::Float16, + DataType::Uint8, + DataType::Int64, + }; + const uint64_t buffer_index = random() % 48; + const uint64_t buffer_addr = 0x800000000ULL + buffer_index * 0x100000ULL; + const uint64_t start_offset = random() % 96; + const uint32_t extent = 1 + static_cast(random() % 12); + const DataType dtype = kDtypes[random() % (sizeof(kDtypes) / sizeof(kDtypes[0]))]; + const bool contiguous = (random() & 3U) != 0; + return MakeTensor(buffer_addr, start_offset, extent, dtype, contiguous); +} + +void TestFixedSeedDifferential() { + constexpr const char *kTest = "fixed-seed-differential"; + constexpr uint64_t kSeed = 0x504152494E475631ULL; // "PARINGV1" + constexpr uint32_t kOperations = 12000; + constexpr int32_t kWindow = 64; + + auto map = NewMap(); + ReferenceMap reference; + reference.Reset(); + std::mt19937_64 random(kSeed); + uint32_t task_id = 0; + std::vector probes; + probes.reserve(kOperations); + + for (uint32_t operation = 0; operation < kOperations; ++operation) { + const uint32_t selector = static_cast(random() % 100); + if (selector < 28) { + task_id += 1 + static_cast(random() % 5); + AdvanceTensorMap(*map, task_id, kWindow); + reference.Advance(task_id, kWindow); + } else if (selector < 68) { + const TensorDesc tensor = RandomTensor(random); + const bool actual = InsertTensor(*map, tensor, static_cast(task_id)); + const bool expected = reference.Insert(tensor, static_cast(task_id)); + probes.push_back(tensor); + if (actual != expected) { + std::fprintf( + stderr, + "[FAIL] private TensorMap ring/%s: insert op=%u task=%u actual=%d expected=%d\n", + kTest, operation, task_id, static_cast(actual), static_cast(expected) + ); + ++g_failures; + return; + } + } else { + const TensorDesc query = RandomTensor(random); + const int32_t actual = LookupTensor(*map, query); + const int32_t expected = reference.Lookup(query); + if (actual != expected) { + std::fprintf( + stderr, + "[FAIL] private TensorMap ring/%s: lookup op=%u task=%u actual=%d expected=%d\n", + kTest, operation, task_id, actual, expected + ); + ++g_failures; + return; + } + } + + const uint32_t actual_count = CountLiveMapEntries(*map); + const uint32_t expected_count = reference.Count(); + if (actual_count != expected_count) { + std::fprintf( + stderr, + "[FAIL] private TensorMap ring/%s: count op=%u task=%u actual=%u expected=%u\n", + kTest, operation, task_id, actual_count, expected_count + ); + ++g_failures; + return; + } + + // 每个操作后都对一个既往区间做可见状态差分，而不是只比较 live + // 数量。这样 Advance 的错误退休、Insert 写错槽和满桶失败后误推进 + // 游标，都会在发生的同一步暴露，不依赖后续随机序列碰巧再次查询。 + if (!probes.empty()) { + const size_t probe_index = + (static_cast(operation) * 0x9E3779B1ULL) % probes.size(); + const int32_t actual = LookupTensor(*map, probes[probe_index]); + const int32_t expected = reference.Lookup(probes[probe_index]); + if (actual != expected) { + std::fprintf( + stderr, + "[FAIL] private TensorMap ring/%s: state op=%u task=%u probe=%zu " + "actual=%d expected=%d\n", + kTest, operation, task_id, probe_index, actual, expected + ); + ++g_failures; + return; + } + } + } +} + +} // namespace + +int main() { + TestPhysicalLayoutBoundaries(); + TestEmptyAndHalfOpenIntervals(); + TestLatestProducerAndAliveFloor(); + TestTaskWindowWrapAndMultipleLaps(); + TestFullBucketDoesNotOverwrite(); + TestFixedSeedDifferential(); + + if (g_failures != 0) { + std::fprintf(stderr, "[FAIL] private TensorMap ring: %d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf( + "[PASS] private TensorMap ring CAP=%u buckets=%u: " + "ABI, interval, reclaim, wrap, overflow, differential\n", + kMapBucketCapacity, kMapBuckets + ); + return EXIT_SUCCESS; +} diff --git a/tests/atomic_probe/pa_scheduler/test/test_shared_compact_generic_trace.cpp b/tests/atomic_probe/pa_scheduler/test/test_shared_compact_generic_trace.cpp new file mode 100644 index 0000000000..c0e9da06c3 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test/test_shared_compact_generic_trace.cpp @@ -0,0 +1,318 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include +#include +#include + +#include "host_support.h" + +namespace { + +using pa_scheduler::CompactTraceRecord16; +using pa_scheduler::DcciSite; +using pa_scheduler::TraceHeader; +using pa_scheduler::TracePhase; +using pa_scheduler::TraceRecord; +using pa_scheduler::host::DecodeCompactTraceRecord; +using pa_scheduler::host::EncodeCompactTraceRecord; +using pa_scheduler::host::InitializeTraceHeader; +using pa_scheduler::host::ValidateTraceHeader; + +int g_failures = 0; + +void Check(bool condition, const char *message) { + if (condition) return; + std::fprintf( + stderr, "[FAIL] shared compact generic trace: %s\n", + message + ); + ++g_failures; +} + +bool SameRecord( + const TraceRecord &left, const TraceRecord &right +) { + return + left.start_cycle == right.start_cycle && + left.end_cycle == right.end_cycle && + left.task_id == right.task_id && + left.function_id == right.function_id && + left.flags == right.flags && + left.phase == right.phase && + left.auxiliary == right.auxiliary; +} + +TraceRecord MakeRecord( + uint64_t start_cycle, uint64_t end_cycle, + int32_t task_id, int32_t function_id, + TracePhase phase, uint32_t flags, uint32_t auxiliary +) { + TraceRecord record{}; + record.start_cycle = start_cycle; + record.end_cycle = end_cycle; + record.task_id = task_id; + record.function_id = function_id; + record.flags = flags; + record.phase = static_cast(phase); + record.auxiliary = static_cast(auxiliary); + return record; +} + +void TestLayoutAndHeaderIdentity() { + Check( + sizeof(TraceRecord) == 32 && + sizeof(CompactTraceRecord16) == 16 && + alignof(CompactTraceRecord16) == 16, + "logical/physical record sizes must remain 32/16 bytes" + ); + Check( + pa_scheduler::kTraceRecordSizeBytes == 16 && + pa_scheduler::kTraceRecordsPerCore == 28416 && + pa_scheduler::kTraceWorkerBytes == 593920, + "compact build must keep capacity and use the exact short stride" + ); + Check( + pa_scheduler::TraceSubmitClaimOffset(0) % 64U == 0 && + pa_scheduler::TraceRecordsOffset(0) % 64U == 0 && + pa_scheduler::TraceWorkerOffset(1) - + pa_scheduler::TraceWorkerOffset(0) == + 593920U && + pa_scheduler::kTraceBytes == + sizeof(TraceHeader) + + 96U * 593920U, + "all compact worker regions must remain cache-line isolated" + ); + + TraceHeader header{}; + InitializeTraceHeader(&header); + Check( + header.record_size_bytes == 16 && + ValidateTraceHeader( + header, "compact trace ABI self-test" + ), + "initialized header must accept the 16-byte physical ABI" + ); + header.record_size_bytes = 32; + Check( + !ValidateTraceHeader( + header, "compact trace ABI negative self-test" + ), + "compact header gate must reject a 32-byte mixed artifact" + ); +} + +void TestRoundTripAndClockWrap() { + const uint64_t anchor = UINT64_C(0x00000001fffffff0); + const uint64_t finish = anchor + 0x100U; + const TraceRecord source = MakeRecord( + anchor + 0x20U, anchor + 0x50U, + static_cast(pa_scheduler::kMaxTasks - 1U), + 3, TracePhase::Materialize, UINT32_MAX, + pa_scheduler::kCompactTraceAuxiliaryMask + ); + CompactTraceRecord16 compact{}; + TraceRecord decoded{}; + Check( + EncodeCompactTraceRecord(source, &compact), + "maximum legal fields must encode" + ); + Check( + compact.start_cycle_low == 0x10U && + compact.end_cycle_low == 0x40U, + "forward timestamps must retain their wrapped low bits" + ); + Check( + DecodeCompactTraceRecord( + compact, anchor, finish, &decoded + ) && + SameRecord(source, decoded), + "maximum legal fields and a forward wrap must round-trip" + ); + + const TraceRecord sentinel = MakeRecord( + anchor + 1U, anchor + 2U, -1, -1, + TracePhase::Atomic, 0xfedcba98U, + static_cast( + pa_scheduler::AtomicSite::SharedOutputRollbackExchange + ) + ); + Check( + EncodeCompactTraceRecord(sentinel, &compact) && + DecodeCompactTraceRecord( + compact, anchor, finish, &decoded + ) && + SameRecord(sentinel, decoded), + "-1 task/function sentinels and full flags must round-trip" + ); +} + +void TestStartupBackwardWrap() { + const uint64_t anchor = UINT64_C(0x0000000200000010); + const uint64_t finish = anchor + 0x100U; + const TraceRecord source = MakeRecord( + anchor - 0x30U, anchor - 0x20U, -1, -1, + TracePhase::Dcci, 0x30cU, + static_cast( + DcciSite::StartupConfigInvalidate + ) + ); + CompactTraceRecord16 compact{}; + TraceRecord decoded{}; + Check( + EncodeCompactTraceRecord(source, &compact) && + compact.start_cycle_low == 0xffffffe0U && + compact.end_cycle_low == 0xfffffff0U && + DecodeCompactTraceRecord( + compact, anchor, finish, &decoded + ) && + SameRecord(source, decoded), + "startup DCCI must unfold backward across the low32 wrap" + ); +} + +void TestDecodeRejections() { + const uint64_t anchor = UINT64_C(0x0000000300000100); + const uint64_t finish = anchor + 0x100U; + TraceRecord source = MakeRecord( + anchor + 0x10U, anchor + 0x20U, 7, 2, + TracePhase::Register, 0x12345678U, 3 + ); + CompactTraceRecord16 compact{}; + TraceRecord decoded{}; + Check( + EncodeCompactTraceRecord(source, &compact), + "valid rejection-test seed must encode" + ); + Check( + !DecodeCompactTraceRecord(compact, 0, finish, &decoded) && + !DecodeCompactTraceRecord( + compact, anchor, anchor - 1U, &decoded + ) && + !DecodeCompactTraceRecord( + compact, anchor, + anchor + (UINT64_C(1) << 32U), &decoded + ), + "zero anchor, reversed lifecycle, and a 2^32 window must fail" + ); + + CompactTraceRecord16 invalid = compact; + invalid.packed = + (invalid.packed & ~pa_scheduler::kCompactTraceTaskMask) | + pa_scheduler::kMaxTasks; + Check( + !DecodeCompactTraceRecord( + invalid, anchor, finish, &decoded + ), + "reserved task codes must fail" + ); + invalid = compact; + invalid.packed &= + ~(pa_scheduler::kCompactTraceFunctionMask + << pa_scheduler::kCompactTraceFunctionShift); + invalid.packed |= + 4U << pa_scheduler::kCompactTraceFunctionShift; + Check( + !DecodeCompactTraceRecord( + invalid, anchor, finish, &decoded + ), + "reserved function codes must fail" + ); + invalid = compact; + invalid.packed &= + ~(pa_scheduler::kCompactTracePhaseMask + << pa_scheduler::kCompactTracePhaseShift); + invalid.packed |= + static_cast(TracePhase::Count) + << pa_scheduler::kCompactTracePhaseShift; + Check( + !DecodeCompactTraceRecord( + invalid, anchor, finish, &decoded + ), + "reserved phase codes must fail" + ); + + invalid = compact; + invalid.start_cycle_low = + static_cast(anchor + 0x30U); + invalid.end_cycle_low = + static_cast(anchor + 0x20U); + Check( + !DecodeCompactTraceRecord( + invalid, anchor, finish, &decoded + ), + "a reversed decoded interval must fail" + ); + invalid = compact; + invalid.end_cycle_low = + static_cast(finish + 1U); + Check( + !DecodeCompactTraceRecord( + invalid, anchor, finish, &decoded + ), + "an endpoint outside the worker lifecycle must fail" + ); +} + +void TestTerminalEndStoreIsolation() { + alignas(64) CompactTraceRecord16 records[4]{}; + for (uint32_t index = 0; index < 4; ++index) { + records[index].start_cycle_low = 0x1000U + index; + records[index].end_cycle_low = 0x2000U + index; + records[index].flags = 0x3000U + index; + records[index].packed = 0x4000U + index; + } + unsigned char before[sizeof(records)]{}; + std::memcpy(before, records, sizeof(records)); + records[2].end_cycle_low = 0xdeadbeefU; + + const auto *after = + reinterpret_cast(records); + const size_t changed_begin = + 2U * sizeof(CompactTraceRecord16) + + offsetof(CompactTraceRecord16, end_cycle_low); + bool isolated = true; + for (size_t byte = 0; byte < sizeof(records); ++byte) { + const bool should_change = + byte >= changed_begin && + byte < changed_begin + sizeof(uint32_t); + isolated &= should_change + ? before[byte] != after[byte] + : before[byte] == after[byte]; + } + Check( + isolated, + "terminal 32-bit end publication must not alter neighbors" + ); +} + +} // namespace + +int main() { + TestLayoutAndHeaderIdentity(); + TestRoundTripAndClockWrap(); + TestStartupBackwardWrap(); + TestDecodeRejections(); + TestTerminalEndStoreIsolation(); + if (g_failures != 0) { + std::fprintf( + stderr, + "[FAIL] shared compact generic trace: %d failure(s)\n", + g_failures + ); + return 1; + } + std::puts( + "[PASS] shared compact generic trace codec and clock gates" + ); + return 0; +} diff --git a/tests/atomic_probe/pa_scheduler/test/test_shared_heap_reserve.cpp b/tests/atomic_probe/pa_scheduler/test/test_shared_heap_reserve.cpp new file mode 100644 index 0000000000..b521466f5a --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test/test_shared_heap_reserve.cpp @@ -0,0 +1,734 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define PA_DEVICE inline +#define PA_GM +#include "pa_shared_heap.h" + +namespace { + +using namespace pa_scheduler; + +int g_failures = 0; + +void Check(bool condition, const char *message) { + if (condition) { + return; + } + std::fprintf(stderr, "[FAIL] %s\n", message); + ++g_failures; +} + +struct HeapTestOps { + static constexpr bool kAtomicReturnReadyObserved = false; + static uint64_t now; + + static uint64_t Now() { + return now++; + } + + template + static uint64_t NowAfterAtomicResult(T value) { + (void)value; + return Now(); + } + + static int64_t Load(volatile int64_t *address) { + return __atomic_load_n(address, __ATOMIC_ACQUIRE); + } + + static int64_t FetchAdd(volatile int64_t *address, int64_t value) { + return __atomic_fetch_add(address, value, __ATOMIC_ACQ_REL); + } +}; + +uint64_t HeapTestOps::now = 0; + +// 在 cursor 的前置 Load 与 FetchAdd 之间插入另一笔 reserve；它可以暂停在 +// cursor/vend 两条原子之间，也可以完整推进。helper 必须消费 FetchAdd 的 +// 真实 old value，不能要求它等于预检快照，更不能回滚并发进度。 +struct HeapInterleaveOps : HeapTestOps { + static volatile int64_t *race_cursor; + static volatile int64_t *race_vend; + static int64_t injected_delta; + static bool advance_vend; + + static int64_t FetchAdd(volatile int64_t *address, int64_t value) { + if (address == race_cursor) { + (void)__atomic_fetch_add( + address, injected_delta, __ATOMIC_ACQ_REL + ); + if (advance_vend) { + (void)__atomic_fetch_add( + race_vend, injected_delta, __ATOMIC_ACQ_REL + ); + } + race_cursor = nullptr; + } + return HeapTestOps::FetchAdd(address, value); + } +}; + +volatile int64_t *HeapInterleaveOps::race_cursor = nullptr; +volatile int64_t *HeapInterleaveOps::race_vend = nullptr; +int64_t HeapInterleaveOps::injected_delta = 0; +bool HeapInterleaveOps::advance_vend = false; + +void ResetHeapState(SharedTensorMapSidecar &map) { + for (uint32_t shard = 0; shard < kSharedHeapShards; ++shard) { + map.shared_heap_cursor[shard].value = 0; + } + map.shared_heap_vend.value = 0; +} + +struct HeapSnapshot { + int64_t cursor[kSharedHeapShards]; + int64_t vend; +}; + +HeapSnapshot Snapshot(const SharedTensorMapSidecar &map) { + HeapSnapshot snapshot{}; + for (uint32_t shard = 0; shard < kSharedHeapShards; ++shard) { + snapshot.cursor[shard] = map.shared_heap_cursor[shard].value; + } + snapshot.vend = map.shared_heap_vend.value; + return snapshot; +} + +bool SameSnapshot( + const SharedTensorMapSidecar &map, const HeapSnapshot &expected +) { + for (uint32_t shard = 0; shard < kSharedHeapShards; ++shard) { + if (map.shared_heap_cursor[shard].value != expected.cursor[shard]) { + return false; + } + } + return map.shared_heap_vend.value == expected.vend; +} + +struct HeapTraceFixture { + TraceCoreState core{}; + TraceRecord records[8]{}; + WorkerResult result{}; + TraceContext trace{}; + + HeapTraceFixture() { + trace.core = &core; + trace.records = records; + trace.capacity = + static_cast(sizeof(records) / sizeof(records[0])); + trace.atomics_enabled = true; + } +}; + +constexpr uint64_t kOutputBytes[kTasksPerBatch] = { + 10240, 524288, 264192, 8192, 0, +}; + +void TestAtomicTraceSites() { + auto map = std::make_unique(); + ResetHeapState(*map); + HeapTraceFixture nonempty; + SharedHeapReservation reservation{}; + HeapTestOps::now = 100; + + Check( + ReserveSharedOutputHeap( + *map, 0, kOutputAlignment, kHeapBytes, reservation, + &nonempty.trace, &nonempty.result + ), + "traced nonempty reservation succeeds" + ); + const AtomicSite expected_sites[] = { + AtomicSite::SharedHeapVendLoad, + AtomicSite::SharedHeapCursorLoad, + AtomicSite::SharedHeapCursorReserve, + AtomicSite::SharedHeapVendAdvance, + }; + Check(nonempty.trace.record_count == 4, "nonempty reserve writes four atomic records"); + Check( + nonempty.result.atomic_trace_calls == 4, + "nonempty reserve counts four logical atomic calls" + ); + for (uint32_t index = 0; index < 4 && index < nonempty.trace.record_count; ++index) { + const TraceRecord &record = nonempty.records[index]; + const AtomicSite site = expected_sites[index]; + Check( + record.auxiliary == static_cast(site), + "shared heap atomic site order is stable" + ); + Check( + (record.flags & kAtomicOpMask) == + static_cast(AtomicSiteExpectedOp(site)), + "shared heap atomic op matches schema" + ); + Check( + (record.flags & kAtomicResultUsed) != 0, + "every shared heap atomic return value is consumed" + ); + } + + ResetHeapState(*map); + HeapTraceFixture empty; + reservation = SharedHeapReservation{}; + HeapTestOps::now = 200; + Check( + ReserveSharedOutputHeap( + *map, 4, 0, kHeapBytes, reservation, + &empty.trace, &empty.result + ), + "traced zero-output reservation succeeds" + ); + Check( + empty.trace.record_count == 1 && + empty.records[0].auxiliary == + static_cast(AtomicSite::SharedHeapVendLoad), + "zero-output reserve observes only aggregate vend" + ); + Check( + empty.result.atomic_trace_calls == 1, + "zero-output reserve counts one logical atomic call" + ); +} + +// 串行 reference 只校验 PA Case1 的业务字节总量、默认 shard 分布和 no-wrap +// 容量；生产并发时同一 shard 内的 task 物理次序不由 task_id 决定。 +void TestPaCase(uint32_t batches) { + auto map = std::make_unique(); + ResetHeapState(*map); + uint64_t expected_cursor[kSharedHeapShards] = {}; + uint64_t expected_vend = 0; + const uint64_t shard_span = + SharedHeapAlignDown(kHeapBytes / kSharedHeapShards); + + for (uint32_t task_id = 0; task_id < batches * kTasksPerBatch; + ++task_id) { + const uint32_t shard = task_id % kSharedHeapShards; + const uint64_t total = kOutputBytes[task_id % kTasksPerBatch]; + SharedHeapReservation reservation{UINT64_MAX, UINT64_MAX}; + Check( + ReserveSharedOutputHeap( + *map, task_id, total, kHeapBytes, reservation + ), + "PA Case1 reservation succeeds" + ); + const uint64_t expected_base = + total == 0 ? 0 : shard * shard_span + expected_cursor[shard]; + Check( + reservation.task_base == expected_base, + "serial PA reference follows its current shard cursor" + ); + expected_cursor[shard] += total; + expected_vend += total; + Check( + reservation.aggregate_vend == expected_vend, + "serial PA reference aggregate vend is exact" + ); + } + + uint64_t cursor_sum = 0; + for (uint32_t shard = 0; shard < kSharedHeapShards; ++shard) { + Check( + static_cast(map->shared_heap_cursor[shard].value) == + expected_cursor[shard], + "PA Case1 final shard cursor is exact" + ); + Check( + expected_cursor[shard] <= shard_span, + "PA Case1 shard remains inside no-wrap capacity" + ); + cursor_sum += expected_cursor[shard]; + } + Check(cursor_sum == expected_vend, "sum of shard cursors equals vend"); + Check( + static_cast(map->shared_heap_vend.value) == expected_vend, + "PA Case1 final aggregate vend is exact" + ); + + if (batches == 1) { + const uint64_t expected_b1[kSharedHeapShards] = { + 10240, 524288, 264192, 8192, 0, 0, 0, 0, + }; + Check( + std::memcmp(expected_cursor, expected_b1, sizeof(expected_b1)) == 0, + "b1 shard distribution matches the PA topology" + ); + Check(expected_vend == 806912, "b1 vend is 806912 bytes"); + } else if (batches == kDefaultBatches) { + for (uint32_t shard = 0; shard < kSharedHeapShards; ++shard) { + Check( + expected_cursor[shard] == 25821184, + "b256 distributes 25821184 bytes to every shard" + ); + } + Check( + expected_vend == 206569472, + "b256 vend is 206569472 bytes" + ); + } +} + +void TestAlignmentAndShardTail() { + auto map = std::make_unique(); + ResetHeapState(*map); + const uint64_t heap_size = + kSharedHeapShards * (4096 + 512); + const uint64_t shard_span = 4096; + + SharedHeapReservation first{}; + Check( + ReserveSharedOutputHeap( + *map, 7, 1, heap_size, first + ), + "unaligned request is rounded to one KiB" + ); + Check( + first.task_base == 7 * shard_span && + first.aggregate_vend == kOutputAlignment, + "heap tail is excluded and shard base remains aligned" + ); + Check( + map->shared_heap_cursor[7].value == + static_cast(kOutputAlignment), + "rounded reservation advances only its shard" + ); + + ResetHeapState(*map); + map->shared_heap_vend.value = + static_cast(kSharedHeapShards * shard_span + kOutputAlignment); + const HeapSnapshot excluded_tail = Snapshot(*map); + SharedHeapReservation zero{}; + Check( + !ReserveSharedOutputHeap( + *map, 0, 0, heap_size, zero + ), + "aggregate vend inside the excluded heap tail is rejected" + ); + Check( + SameSnapshot(*map, excluded_tail), + "excluded-tail rejection changes no heap state" + ); +} + +void TestBoundaryZeroAndPreflightFailures() { + auto map = std::make_unique(); + ResetHeapState(*map); + const uint64_t heap_size = + kSharedHeapShards * 4096; + + SharedHeapReservation empty_zero{UINT64_MAX, UINT64_MAX}; + const HeapSnapshot empty_snapshot = Snapshot(*map); + Check( + ReserveSharedOutputHeap( + *map, 4, 0, heap_size, empty_zero + ), + "zero-output task may observe an empty aggregate vend" + ); + Check( + empty_zero.task_base == 0 && empty_zero.aggregate_vend == 0, + "empty zero-output reservation returns a zero diagnostic prefix" + ); + Check( + SameSnapshot(*map, empty_snapshot), + "empty zero-output reservation changes no heap state" + ); + + SharedHeapReservation full{}; + Check( + ReserveSharedOutputHeap( + *map, 0, 4096, heap_size, full + ), + "reservation may exactly fill one shard" + ); + const HeapSnapshot full_snapshot = Snapshot(*map); + + SharedHeapReservation zero{UINT64_MAX, UINT64_MAX}; + Check( + ReserveSharedOutputHeap( + *map, 8, 0, heap_size, zero + ), + "zero-output task succeeds when its shard is full" + ); + Check( + zero.task_base == 0 && zero.aggregate_vend == 4096, + "zero-output task returns current vend without an address" + ); + Check( + SameSnapshot(*map, full_snapshot), + "zero-output task changes no heap state" + ); + + SharedHeapReservation failed{UINT64_MAX, UINT64_MAX}; + Check( + !ReserveSharedOutputHeap( + *map, 8, 1, heap_size, failed + ), + "no-wrap helper rejects a full shard" + ); + Check( + failed.task_base == 0 && failed.aggregate_vend == 0, + "failed reservation returns a cleared result" + ); + Check( + SameSnapshot(*map, full_snapshot), + "capacity failure changes no heap state" + ); + + Check( + !ReserveSharedOutputHeap( + *map, 1, 4097, heap_size, failed + ), + "single reservation larger than shard span is rejected" + ); + Check( + SameSnapshot(*map, full_snapshot), + "oversized request changes no heap state" + ); + + Check( + !ReserveSharedOutputHeap( + *map, 1, UINT64_MAX, heap_size, failed + ), + "alignment overflow is rejected" + ); + Check( + SameSnapshot(*map, full_snapshot), + "alignment overflow changes no heap state" + ); + + Check( + !ReserveSharedOutputHeap( + *map, kMaxTasks, 0, heap_size, failed + ), + "out-of-range task id is rejected even for zero output" + ); + Check( + SameSnapshot(*map, full_snapshot), + "out-of-range task id changes no heap state" + ); + + map->shared_heap_cursor[1].value = 1; + const HeapSnapshot unaligned_cursor = Snapshot(*map); + Check( + !ReserveSharedOutputHeap( + *map, 1, 1024, heap_size, failed + ), + "unaligned cursor is rejected" + ); + Check( + SameSnapshot(*map, unaligned_cursor), + "invalid cursor preflight changes no heap state" + ); + + ResetHeapState(*map); + map->shared_heap_vend.value = -1; + const HeapSnapshot negative_vend = Snapshot(*map); + Check( + !ReserveSharedOutputHeap( + *map, 0, 1024, heap_size, failed + ), + "negative aggregate vend is rejected" + ); + Check( + SameSnapshot(*map, negative_vend), + "negative aggregate vend changes no heap state" + ); + + ResetHeapState(*map); + map->shared_heap_vend.value = 1; + const HeapSnapshot unaligned_vend = Snapshot(*map); + Check( + !ReserveSharedOutputHeap( + *map, 0, 1024, heap_size, failed + ), + "unaligned aggregate vend is rejected" + ); + Check( + SameSnapshot(*map, unaligned_vend), + "unaligned aggregate vend changes no heap state" + ); + + ResetHeapState(*map); + map->shared_heap_vend.value = + static_cast(heap_size + kOutputAlignment); + const HeapSnapshot oversized_vend = Snapshot(*map); + Check( + !ReserveSharedOutputHeap( + *map, 0, 1024, heap_size, failed + ), + "aggregate vend beyond heap capacity is rejected" + ); + Check( + SameSnapshot(*map, oversized_vend), + "oversized aggregate vend changes no heap state" + ); + + ResetHeapState(*map); + map->shared_heap_vend.value = static_cast(heap_size); + const HeapSnapshot exhausted_vend = Snapshot(*map); + Check( + !ReserveSharedOutputHeap( + *map, 1, 1024, heap_size, failed + ), + "aggregate capacity exhaustion is rejected despite free target shard" + ); + Check( + SameSnapshot(*map, exhausted_vend), + "aggregate capacity failure changes no heap state" + ); + + ResetHeapState(*map); + map->shared_heap_cursor[2].value = -1; + const HeapSnapshot negative_cursor = Snapshot(*map); + Check( + !ReserveSharedOutputHeap( + *map, 2, 1024, heap_size, failed + ), + "negative shard cursor is rejected" + ); + Check( + SameSnapshot(*map, negative_cursor), + "negative shard cursor changes no heap state" + ); + + ResetHeapState(*map); + map->shared_heap_cursor[2].value = 5120; + const HeapSnapshot oversized_cursor = Snapshot(*map); + Check( + !ReserveSharedOutputHeap( + *map, 2, 1024, heap_size, failed + ), + "shard cursor beyond its span is rejected" + ); + Check( + SameSnapshot(*map, oversized_cursor), + "oversized shard cursor changes no heap state" + ); + + ResetHeapState(*map); + const HeapSnapshot tiny_heap = Snapshot(*map); + SharedHeapReservation tiny_zero{}; + Check( + ReserveSharedOutputHeap( + *map, 4, 0, kOutputAlignment - 1, tiny_zero + ), + "zero-output task does not require one allocatable shard" + ); + Check( + tiny_zero.task_base == 0 && tiny_zero.aggregate_vend == 0 && + SameSnapshot(*map, tiny_heap), + "tiny-heap zero-output reservation preserves empty state" + ); + Check( + !ReserveSharedOutputHeap( + *map, 0, 1024, kOutputAlignment - 1, failed + ), + "heap too small to contain one aligned shard is rejected" + ); + Check( + SameSnapshot(*map, tiny_heap), + "zero-span heap failure changes no heap state" + ); +} + +void TestConcurrentReservations() { + auto map = std::make_unique(); + ResetHeapState(*map); + constexpr uint32_t kThreads = 64; + std::vector reservations(kThreads); + std::vector reserve_bytes(kThreads); + std::vector success(kThreads, 0); + std::vector workers; + workers.reserve(kThreads); + std::atomic ready{0}; + std::atomic start{false}; + + for (uint32_t task_id = 0; task_id < kThreads; ++task_id) { + reserve_bytes[task_id] = + (task_id / kSharedHeapShards % 4 + 1) * kOutputAlignment; + workers.emplace_back([&, task_id] { + ready.fetch_add(1, std::memory_order_release); + while (!start.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + success[task_id] = ReserveSharedOutputHeap( + *map, task_id, reserve_bytes[task_id], kHeapBytes, + reservations[task_id] + ) ? 1 : 0; + }); + } + while (ready.load(std::memory_order_acquire) != kThreads) { + std::this_thread::yield(); + } + start.store(true, std::memory_order_release); + for (std::thread &worker : workers) { + worker.join(); + } + + std::vector> + shard_intervals[kSharedHeapShards]; + std::vector> vend_intervals; + vend_intervals.reserve(kThreads); + const uint64_t shard_span = + SharedHeapAlignDown(kHeapBytes / kSharedHeapShards); + uint64_t expected_total = 0; + for (uint32_t task_id = 0; task_id < kThreads; ++task_id) { + Check(success[task_id] != 0, "concurrent reservation succeeds"); + const uint32_t shard = task_id % kSharedHeapShards; + const uint64_t bytes = reserve_bytes[task_id]; + shard_intervals[shard].push_back( + { + reservations[task_id].task_base, + reservations[task_id].task_base + bytes + } + ); + vend_intervals.push_back( + { + reservations[task_id].aggregate_vend - bytes, + reservations[task_id].aggregate_vend + } + ); + expected_total += bytes; + Check( + reservations[task_id].task_base >= shard * shard_span && + reservations[task_id].task_base + bytes <= + (shard + 1) * shard_span, + "concurrent reservation remains inside its assigned shard" + ); + } + for (uint32_t shard = 0; shard < kSharedHeapShards; ++shard) { + std::sort( + shard_intervals[shard].begin(), + shard_intervals[shard].end() + ); + uint64_t next = shard * shard_span; + for (const auto &interval : shard_intervals[shard]) { + Check( + interval.first == next, + "variable-size same-shard reservations are unique and gap-free" + ); + next = interval.second; + } + Check( + map->shared_heap_cursor[shard].value == + static_cast(next - shard * shard_span), + "concurrent shard cursor equals its reserved byte sum" + ); + } + std::sort(vend_intervals.begin(), vend_intervals.end()); + uint64_t next_vend = 0; + for (const auto &interval : vend_intervals) { + Check( + interval.first == next_vend, + "variable-size aggregate vend prefixes are unique and gap-free" + ); + next_vend = interval.second; + } + Check( + map->shared_heap_vend.value == + static_cast(expected_total) && + next_vend == expected_total, + "concurrent aggregate vend equals all successful reservations" + ); +} + +void TestInterleavingAndTerminalCapacityRace() { + auto map = std::make_unique(); + ResetHeapState(*map); + HeapInterleaveOps::race_cursor = &map->shared_heap_cursor[0].value; + HeapInterleaveOps::race_vend = &map->shared_heap_vend.value; + HeapInterleaveOps::injected_delta = kOutputAlignment; + HeapInterleaveOps::advance_vend = false; + SharedHeapReservation reservation{UINT64_MAX, UINT64_MAX}; + Check( + ReserveSharedOutputHeap( + *map, 0, 1024, kHeapBytes, reservation + ), + "stale preflight snapshot accepts a legal concurrent reservation" + ); + Check( + reservation.task_base == kOutputAlignment && + reservation.aggregate_vend == kOutputAlignment, + "physical cursor and aggregate vend may linearize in different orders" + ); + Check( + map->shared_heap_cursor[0].value == 2 * kOutputAlignment && + map->shared_heap_vend.value == kOutputAlignment, + "paused competitor may own a cursor interval before publishing vend" + ); + (void)__atomic_fetch_add( + &map->shared_heap_vend.value, static_cast(kOutputAlignment), + __ATOMIC_ACQ_REL + ); + Check( + map->shared_heap_cursor[0].value == 2 * kOutputAlignment && + map->shared_heap_vend.value == 2 * kOutputAlignment, + "resumed competitor closes final cursor and vend byte sums" + ); + + // 两个 caller 都从 shard 尾部看到一份余量；竞争者先占满，当前 + // FetchAdd 随后越过 no-wrap 边界。此时必须 terminal fail 并保留 + // 5KiB cursor 现场，绝不能 Exchange 回 3KiB 覆盖竞争者的合法 1KiB。 + ResetHeapState(*map); + const uint64_t heap_size = kSharedHeapShards * 4096; + map->shared_heap_cursor[0].value = 3 * kOutputAlignment; + map->shared_heap_vend.value = 3 * kOutputAlignment; + HeapInterleaveOps::race_cursor = &map->shared_heap_cursor[0].value; + HeapInterleaveOps::race_vend = &map->shared_heap_vend.value; + HeapInterleaveOps::injected_delta = kOutputAlignment; + HeapInterleaveOps::advance_vend = true; + reservation = SharedHeapReservation{UINT64_MAX, UINT64_MAX}; + Check( + !ReserveSharedOutputHeap( + *map, 0, kOutputAlignment, heap_size, reservation + ), + "capacity race fails after the competing reservation fills the shard" + ); + Check( + map->shared_heap_cursor[0].value == 5 * kOutputAlignment && + map->shared_heap_vend.value == 4 * kOutputAlignment, + "terminal capacity race preserves competitor progress and overrun evidence" + ); + Check( + reservation.task_base == 0 && reservation.aggregate_vend == 0, + "terminal capacity race returns no usable reservation" + ); +} + +} // namespace + +int main() { + TestAtomicTraceSites(); + TestPaCase(1); + TestPaCase(kDefaultBatches); + TestAlignmentAndShardTail(); + TestBoundaryZeroAndPreflightFailures(); + TestConcurrentReservations(); + TestInterleavingAndTerminalCapacityRace(); + if (g_failures != 0) { + std::fprintf( + stderr, "shared heap reserve self-test failed: %d assertion(s)\n", + g_failures + ); + return 1; + } + std::printf("shared heap reserve self-test passed\n"); + return 0; +} diff --git a/tests/atomic_probe/pa_scheduler/test/test_shared_host_task_plan.cpp b/tests/atomic_probe/pa_scheduler/test/test_shared_host_task_plan.cpp new file mode 100644 index 0000000000..ba3a265875 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test/test_shared_host_task_plan.cpp @@ -0,0 +1,702 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License Agreement"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "winner_workload_host.h" + +#include +#include +#include + +namespace { + +using namespace pa_scheduler; +using namespace pa_scheduler::host; + +bool Check(bool condition, const char *label) { + std::printf( + "[HOST_PLAN_TEST] %-56s %s\n", + label, condition ? "PASS" : "FAIL" + ); + return condition; +} + +bool SetContextsAndBuild( + SchedulerState *state, const int32_t *contexts, + uint32_t batches, SharedHostTaskPlan *plan +) { + state->config.batches = batches; + for (uint32_t batch = 0; batch < batches; ++batch) { + state->context_lens[batch] = contexts[batch]; + } + std::string error; + const bool ok = + BuildSharedHostTaskPlan(*state, plan, &error); + if (!ok) { + std::fprintf( + stderr, "BuildSharedHostTaskPlan failed: %s\n", + error.c_str() + ); + } + return ok; +} + +bool CheckSingleContext( + SchedulerState *state, int32_t context, + uint32_t expected_groups, uint32_t expected_tasks +) { + SharedHostTaskPlan plan; + bool ok = SetContextsAndBuild( + state, &context, 1, &plan + ); + ok &= plan.batch_count == 1; + ok &= plan.total_groups == expected_groups; + ok &= plan.total_tasks == expected_tasks; + ok &= plan.tasks_by_kind[ + static_cast(TaskKind::Alloc) + ] == 1; + for (uint32_t kind = + static_cast(TaskKind::Qk); + kind <= static_cast(TaskKind::Up); + ++kind) { + ok &= plan.tasks_by_kind[kind] == expected_groups; + } + const SharedHostBatchPlan *batch = plan.BatchAt(0); + ok &= batch != nullptr; + if (batch != nullptr) { + ok &= batch->batch_start == 0; + ok &= batch->group_count == expected_groups; + ok &= batch->task_count == expected_tasks; + ok &= batch->final_up_task_id == + (expected_groups == 0 + ? UINT32_MAX + : expected_tasks - 1U); + } + const SharedHostPlannedTask *first = plan.TaskAt(0); + const SharedHostPlannedTask *last = + plan.TaskAt(expected_tasks - 1U); + ok &= first != nullptr && + first->kind == TaskKind::Alloc; + ok &= last != nullptr && last->is_last_in_batch; + ok &= plan.TaskAt(expected_tasks) == nullptr; + return ok; +} + +bool CheckRealComputeActivityContract(SchedulerState *state) { + bool ok = true; + const int32_t g0_context = 0; + SharedHostTaskPlan plan; + ok &= SetContextsAndBuild( + state, &g0_context, 1, &plan + ); + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + state->results[worker] = WorkerResult{}; + state->results[worker].role = static_cast( + worker < kAicWorkers ? CoreRole::Aic : CoreRole::Aiv + ); + } + + WinnerWorkloadOptions workload; + std::vector workspace_image; + std::vector outputs; + InitializeWinnerWorkloadBuffers( + workload, &workspace_image, &outputs + ); + std::fill( + outputs.begin(), outputs.end(), + winner_workload::kOutputSentinel + ); + ok &= RealComputeActivityMatchesPlan(*state, 0); + ok &= !RealComputeActivityMatchesPlan(*state, 1); + ok &= ValidateRealComputeOutputs( + *state, workload, outputs, 1 + ); + + const int32_t g1_context = 8192; + ok &= SetContextsAndBuild( + state, &g1_context, 1, &plan + ); + ok &= !RealComputeActivityMatchesPlan(*state, 0); + ok &= RealComputeActivityMatchesPlan(*state, 1); + return ok; +} + +bool CheckTensorDescSemanticMatch() { + TensorDesc expected{}; + expected.buffer_addr = kSyntheticHeapBase; + expected.buffer_size = 64; + expected.owner_task_id = 17; + expected.ndims = 1; + expected.dtype = DataType::Float32; + expected.is_contiguous = true; + expected.shapes[0] = 16; + expected.strides[0] = 1; + expected.extent_elem_cache = 16; + + TensorDesc actual = expected; + actual.shapes[1] = 99; + actual.strides[1] = 7; + bool ok = TensorDescFieldsMatch(actual, expected); + actual.strides[0] = 2; + ok &= !TensorDescFieldsMatch(actual, expected); + actual.strides[0] = 1; + actual.ndims = kMaxTensorDims + 1U; + ok &= !TensorDescFieldsMatch(actual, expected); + return ok; +} + +bool CheckHeapAdmission(SchedulerState *state) { + bool ok = true; + const auto check_repeated = [&]( + int32_t context, uint32_t batches, + bool expected_admitted, uint64_t expected_total, + uint64_t expected_max_shard, uint64_t heap_size + ) { + std::vector contexts(batches, context); + SharedHostTaskPlan plan; + bool case_ok = SetContextsAndBuild( + state, contexts.data(), batches, &plan + ); + uint64_t planned_by_shard[kSharedHeapShards] = {}; + uint64_t planned_total = 0; + for (const SharedHostPlannedTask &task : plan.tasks) { + if (task.output_bytes == 0) { + continue; + } + const uint64_t reserve = + (task.output_bytes + kOutputAlignment - 1U) / + kOutputAlignment * kOutputAlignment; + planned_by_shard[ + task.task_id % kSharedHeapShards + ] += reserve; + planned_total += reserve; + } + uint64_t planned_max_shard = 0; + for (uint32_t shard = 0; + shard < kSharedHeapShards; ++shard) { + planned_max_shard = std::max( + planned_max_shard, planned_by_shard[shard] + ); + } + case_ok &= plan.canonical_heap_bytes == expected_total; + case_ok &= planned_total == expected_total; + case_ok &= planned_max_shard == expected_max_shard; + SharedHostHeapAdmission admission; + std::string error; + const bool admitted = case_ok && + ValidateSharedHostHeapAdmission( + plan, heap_size, &admission, &error + ); + case_ok &= admitted == expected_admitted; + case_ok &= admission.admitted == expected_admitted; + if (expected_admitted) { + uint64_t maximum_shard = 0; + for (uint32_t shard = 0; + shard < kSharedHeapShards; ++shard) { + maximum_shard = std::max( + maximum_shard, + admission.reserved_bytes_by_shard[shard] + ); + } + case_ok &= + admission.total_reserved_bytes == + expected_total; + case_ok &= maximum_shard == expected_max_shard; + case_ok &= + admission.first_failed_task == UINT32_MAX; + case_ok &= + admission.first_failed_shard == UINT32_MAX; + } else { + case_ok &= !error.empty(); + case_ok &= + admission.first_failed_task != UINT32_MAX; + case_ok &= + admission.first_failed_shard < + kSharedHeapShards; + } + return case_ok; + }; + + ok &= check_repeated( + 0, 1, true, 10240, 10240, kHeapBytes + ); + ok &= check_repeated( + 8192, 1, true, 806912, 524288, kHeapBytes + ); + ok &= check_repeated( + 8193, 1, true, 829440, 524288, kHeapBytes + ); + ok &= check_repeated( + 16384, 1, true, 1603584, 524288, kHeapBytes + ); + ok &= check_repeated( + 32768, 1, true, 3196928, 1048576, kHeapBytes + ); + ok &= check_repeated( + 0, kDefaultBatches, true, + 2621440, 327680, kHeapBytes + ); + ok &= check_repeated( + 8192, kDefaultBatches, true, + 206569472, 25821184, kHeapBytes + ); + ok &= check_repeated( + 8193, kDefaultBatches, true, + 212336640, 26542080, kHeapBytes + ); + ok &= check_repeated( + 16384, kDefaultBatches, false, + 410517504, 51314688, kHeapBytes + ); + ok &= check_repeated( + 32768, kDefaultBatches, false, + 818413568, 102301696, kHeapBytes + ); + ok &= check_repeated( + 0, kMaxBatches, true, + 5242880, 655360, kExtendedBatchHeapBytes + ); + ok &= check_repeated( + 8192, kMaxBatches, true, + 413138944, 51642368, kExtendedBatchHeapBytes + ); + + const int32_t mixed_contexts[] = { + 0, 8192, 8193, 32768, + }; + SharedHostTaskPlan mixed; + ok &= SetContextsAndBuild( + state, mixed_contexts, 4, &mixed + ); + SharedHostHeapAdmission mixed_admission; + std::string mixed_error; + ok &= ValidateSharedHostHeapAdmission( + mixed, kHeapBytes, &mixed_admission, &mixed_error + ); + const uint64_t expected_mixed_shards[ + kSharedHeapShards + ] = { + 1323008, 546816, 540672, 272384, + 1062912, 536576, 26624, 534528, + }; + ok &= mixed_admission.total_reserved_bytes == 4843520; + for (uint32_t shard = 0; + shard < kSharedHeapShards; ++shard) { + ok &= + mixed_admission.reserved_bytes_by_shard[shard] == + expected_mixed_shards[shard]; + } + + std::vector g1_contexts( + kDefaultBatches, 8192 + ); + SharedHostTaskPlan g1; + ok &= SetContextsAndBuild( + state, g1_contexts.data(), kDefaultBatches, &g1 + ); + SharedHostHeapAdmission exact; + std::string exact_error; + ok &= ValidateSharedHostHeapAdmission( + g1, 206569472, &exact, &exact_error + ); + SharedHostHeapAdmission one_byte_short; + std::string short_error; + ok &= !ValidateSharedHostHeapAdmission( + g1, 206569471, &one_byte_short, &short_error + ); + ok &= !short_error.empty(); + + std::vector extended_g1_contexts( + kMaxBatches, 8192 + ); + SharedHostTaskPlan extended_g1; + ok &= SetContextsAndBuild( + state, extended_g1_contexts.data(), + kMaxBatches, &extended_g1 + ); + ok &= extended_g1.total_tasks == 2560; + SharedHostHeapAdmission extended_exact; + std::string extended_exact_error; + ok &= ValidateSharedHostHeapAdmission( + extended_g1, 413138944, + &extended_exact, &extended_exact_error + ); + SharedHostHeapAdmission extended_short; + std::string extended_short_error; + ok &= !ValidateSharedHostHeapAdmission( + extended_g1, 413138943, + &extended_short, &extended_short_error + ); + ok &= !extended_short_error.empty(); + + // B512 只为默认 PA-G1 翻倍模型扩容；不能让 batch 上限绕过 + // 既有 4,352-task output/history 物理容量。 + state->config.batches = kMaxBatches; + for (uint32_t batch = 0; batch < kMaxBatches; ++batch) { + state->context_lens[batch] = 32768; + } + SharedHostTaskPlan extended_g4; + std::string extended_g4_error; + ok &= !BuildSharedHostTaskPlan( + *state, &extended_g4, &extended_g4_error + ); + ok &= extended_g4.total_tasks == 0; + ok &= !extended_g4_error.empty(); + + // 构造“总量仍放得下、但 task_id%8 的单个 shard 已溢出”的偏斜计划， + // 防止准入实现退化成只比较 aggregate heap。 + std::vector skew_contexts(9, 0); + SharedHostTaskPlan skew; + ok &= SetContextsAndBuild( + state, skew_contexts.data(), 9, &skew + ); + for (SharedHostPlannedTask &task : skew.tasks) { + task.output_bytes = 0; + } + const uint64_t shard_span = + (kHeapBytes / kSharedHeapShards) / + kOutputAlignment * kOutputAlignment; + skew.tasks[0].output_bytes = shard_span; + skew.tasks[8].output_bytes = kOutputAlignment; + skew.canonical_heap_bytes = + shard_span + kOutputAlignment; + SharedHostHeapAdmission skew_result; + std::string skew_error; + ok &= !ValidateSharedHostHeapAdmission( + skew, kHeapBytes, &skew_result, &skew_error + ); + ok &= skew.canonical_heap_bytes < + shard_span * kSharedHeapShards; + ok &= skew_result.first_failed_task == 8; + ok &= skew_result.first_failed_shard == 0; + ok &= !skew_error.empty(); + + SharedHostHeapAdmission signed_overflow; + std::string signed_error; + ok &= !ValidateSharedHostHeapAdmission( + g1, static_cast(INT64_MAX) + 1U, + &signed_overflow, &signed_error + ); + ok &= !signed_error.empty(); + + SharedHostTaskPlan output_overflow = g1; + output_overflow.tasks[0].output_bytes = UINT64_MAX; + SharedHostHeapAdmission output_overflow_result; + std::string output_overflow_error; + ok &= !ValidateSharedHostHeapAdmission( + output_overflow, kHeapBytes, + &output_overflow_result, &output_overflow_error + ); + ok &= output_overflow_result.first_failed_task == 0; + ok &= !output_overflow_error.empty(); + + SharedHostTaskPlan non_contiguous = g1; + non_contiguous.tasks[3].task_id = 4; + SharedHostHeapAdmission non_contiguous_result; + std::string non_contiguous_error; + ok &= !ValidateSharedHostHeapAdmission( + non_contiguous, kHeapBytes, + &non_contiguous_result, &non_contiguous_error + ); + ok &= non_contiguous_result.first_failed_task == 4; + ok &= !non_contiguous_error.empty(); + return ok; +} + +bool CheckCli() { + bool ok = true; + { + Options options; + char program[] = "host-plan-test"; + char batches_name[] = "--batches"; + char batches_value[] = "4"; + char contexts_name[] = "--shared-context-lens"; + char contexts_value[] = "0,8192,8193,32768"; + char *argv[] = { + program, batches_name, batches_value, + contexts_name, contexts_value, + }; + ok &= ParseOptions(5, argv, false, &options) == + ParseStatus::Ok; + ok &= options.batches == 4; + ok &= options.shared_context_lens.size() == 4; + ok &= options.shared_context_lens[0] == 0; + ok &= options.shared_context_lens[1] == 8192; + ok &= options.shared_context_lens[2] == 8193; + ok &= options.shared_context_lens[3] == 32768; + } + { + Options options; + char program[] = "host-plan-test"; + char batches_name[] = "--batches"; + char batches_value[] = "4"; + char contexts_name[] = "--shared-context-lens"; + char contexts_value[] = "16384"; + char *argv[] = { + program, batches_name, batches_value, + contexts_name, contexts_value, + }; + ok &= ParseOptions(5, argv, false, &options) == + ParseStatus::Ok; + ok &= options.shared_context_lens.size() == 1; + ok &= options.shared_context_lens.front() == 16384; + } + { + Options options; + char program[] = "host-plan-test"; + char batches_name[] = "--batches"; + char batches_value[] = "4"; + char contexts_name[] = "--shared-context-lens"; + char contexts_value[] = "0,8192"; + char *argv[] = { + program, batches_name, batches_value, + contexts_name, contexts_value, + }; + ok &= ParseOptions(5, argv, false, &options) == + ParseStatus::Error; + } + { + Options options; + char program[] = "host-plan-test"; + char contexts_name[] = "--shared-context-lens"; + char contexts_value[] = "32769"; + char *argv[] = { + program, contexts_name, contexts_value, + }; + ok &= ParseOptions(3, argv, false, &options) == + ParseStatus::Error; + } + return ok; +} + +} // namespace + +int main() { + std::unique_ptr state(new SchedulerState); + bool ok = true; + + ok &= Check( + CheckSingleContext(state.get(), 0, 0, 1), + "G0 context=0 -> Alloc only" + ); + ok &= Check( + CheckSingleContext(state.get(), 8192, 1, 5), + "G1 context=8192 -> Alloc + one group" + ); + ok &= Check( + CheckSingleContext(state.get(), 8193, 2, 9), + "G2 context=8193 -> full group + one-block group" + ); + ok &= Check( + CheckSingleContext(state.get(), 16384, 2, 9), + "G2 context=16384 -> two full groups" + ); + ok &= Check( + CheckSingleContext(state.get(), 32768, 4, 17), + "G4 context=32768 -> four full groups" + ); + ok &= Check( + CheckRealComputeActivityContract(state.get()), + "real-compute activity follows G0/nonzero shared plan" + ); + ok &= Check( + CheckTensorDescSemanticMatch(), + "descriptor oracle ignores inactive payload bytes but rejects active corruption" + ); + ok &= Check( + CheckHeapAdmission(state.get()), + "shared heap admission rejects over-capacity plans before workers" + ); + + const int32_t mixed_contexts[] = { + 0, 8192, 8193, 32768, + }; + SharedHostTaskPlan mixed; + bool mixed_ok = SetContextsAndBuild( + state.get(), mixed_contexts, 4, &mixed + ); + mixed_ok &= mixed.batch_count == 4; + mixed_ok &= mixed.total_groups == 7; + mixed_ok &= mixed.total_tasks == 32; + mixed_ok &= mixed.canonical_heap_bytes == 4843520; + const uint32_t expected_starts[] = {0, 1, 6, 15}; + const uint32_t expected_counts[] = {1, 5, 9, 17}; + const uint32_t expected_final_ups[] = { + UINT32_MAX, 5, 14, 31, + }; + for (uint32_t batch = 0; batch < 4; ++batch) { + const SharedHostBatchPlan *entry = + mixed.BatchAt(batch); + mixed_ok &= entry != nullptr; + if (entry != nullptr) { + mixed_ok &= + entry->batch_start == expected_starts[batch]; + mixed_ok &= + entry->task_count == expected_counts[batch]; + mixed_ok &= + entry->final_up_task_id == + expected_final_ups[batch]; + } + } + const SharedHostPlannedTask *g0_alloc = + mixed.TaskAt(0); + const SharedHostPlannedTask *g2_first_up = + mixed.TaskAt(10); + const SharedHostPlannedTask *g2_partial_qk = + mixed.TaskAt(11); + const SharedHostPlannedTask *g2_partial_sf = + mixed.TaskAt(12); + const SharedHostPlannedTask *g2_final_up = + mixed.TaskAt(14); + const SharedHostPlannedTask *g4_final_up = + mixed.TaskAt(31); + mixed_ok &= + g0_alloc != nullptr && + g0_alloc->batch == 0 && + !g0_alloc->in_group && + g0_alloc->is_last_in_batch; + mixed_ok &= + g2_first_up != nullptr && + g2_first_up->batch == 2 && + g2_first_up->group_index == 0 && + g2_first_up->has_following_group && + !g2_first_up->is_final_up; + mixed_ok &= + g2_partial_qk != nullptr && + g2_partial_qk->kind == TaskKind::Qk && + g2_partial_qk->group_index == 1 && + g2_partial_qk->group_block_count == 1 && + g2_partial_qk->output_bytes == 8192; + mixed_ok &= + g2_partial_sf != nullptr && + g2_partial_sf->kind == TaskKind::Sf && + g2_partial_sf->group_block_count == 1 && + g2_partial_sf->output_bytes == 6144; + mixed_ok &= + g2_final_up != nullptr && + g2_final_up->is_final_up && + !g2_final_up->has_following_group && + g2_final_up->is_last_in_batch; + mixed_ok &= + g4_final_up != nullptr && + g4_final_up->batch == 3 && + g4_final_up->group_index == 3 && + g4_final_up->is_final_up && + g4_final_up->is_last_in_batch; + mixed_ok &= mixed.TaskAt(32) == nullptr; + ok &= Check( + mixed_ok, + "mixed G0/G1/G2/G4 has cumulative batch starts and TaskAt metadata" + ); + + const uint64_t expected_mixed_dependency_signature = + DependencyEdgeSignatureHost(3, 2) ^ + DependencyEdgeSignatureHost(4, 3) ^ + DependencyEdgeSignatureHost(5, 3) ^ + DependencyEdgeSignatureHost(5, 4) ^ + DependencyEdgeSignatureHost(5, 1) ^ + DependencyEdgeSignatureHost(8, 7) ^ + DependencyEdgeSignatureHost(9, 8) ^ + DependencyEdgeSignatureHost(10, 8) ^ + DependencyEdgeSignatureHost(10, 9) ^ + DependencyEdgeSignatureHost(10, 6) ^ + DependencyEdgeSignatureHost(12, 11) ^ + DependencyEdgeSignatureHost(13, 12) ^ + DependencyEdgeSignatureHost(14, 12) ^ + DependencyEdgeSignatureHost(14, 13) ^ + DependencyEdgeSignatureHost(14, 10) ^ + DependencyEdgeSignatureHost(17, 16) ^ + DependencyEdgeSignatureHost(18, 17) ^ + DependencyEdgeSignatureHost(19, 17) ^ + DependencyEdgeSignatureHost(19, 18) ^ + DependencyEdgeSignatureHost(19, 15) ^ + DependencyEdgeSignatureHost(21, 20) ^ + DependencyEdgeSignatureHost(22, 21) ^ + DependencyEdgeSignatureHost(23, 21) ^ + DependencyEdgeSignatureHost(23, 22) ^ + DependencyEdgeSignatureHost(23, 19) ^ + DependencyEdgeSignatureHost(25, 24) ^ + DependencyEdgeSignatureHost(26, 25) ^ + DependencyEdgeSignatureHost(27, 25) ^ + DependencyEdgeSignatureHost(27, 26) ^ + DependencyEdgeSignatureHost(27, 23) ^ + DependencyEdgeSignatureHost(29, 28) ^ + DependencyEdgeSignatureHost(30, 29) ^ + DependencyEdgeSignatureHost(31, 29) ^ + DependencyEdgeSignatureHost(31, 30) ^ + DependencyEdgeSignatureHost(31, 27); + ok &= Check( + ExpectedPaDependencySignature(mixed) == + expected_mixed_dependency_signature, + "dependency oracle chains each later UP to the previous UP writer" + ); + + state->config.batches = 1; + state->context_lens[0] = -1; + SharedHostTaskPlan rejected; + ok &= Check( + !BuildSharedHostTaskPlan(*state, &rejected), + "negative final context_len is rejected" + ); + state->context_lens[0] = 32769; + ok &= Check( + !BuildSharedHostTaskPlan(*state, &rejected), + "context_len above four groups is rejected" + ); + state->config.batches = kMaxBatches + 1U; + ok &= Check( + !BuildSharedHostTaskPlan(*state, &rejected), + "batch count above compiled capacity is rejected" + ); + + Options default_options; + default_options.batches = 1; + InitializeState(state.get(), default_options); + ok &= Check( + state->context_lens[0] == 8192 && + state->heap_size == kHeapBytes, + "InitializeState keeps the shared default context_len and 256 MiB heap" + ); + Options extended_options; + extended_options.batches = kMaxBatches; + InitializeState(state.get(), extended_options); + ok &= Check( + state->context_lens[kMaxBatches - 1U] == 8192 && + state->heap_size == kExtendedBatchHeapBytes, + "B512 defaults to PA-G1 and the 512 MiB no-wrap heap" + ); + Options mixed_options; + mixed_options.batches = 4; + mixed_options.shared_context_lens.assign( + mixed_contexts, mixed_contexts + 4 + ); + InitializeState(state.get(), mixed_options); + bool initialized_mixed_ok = true; + for (uint32_t batch = 0; batch < 4; ++batch) { + initialized_mixed_ok &= + state->context_lens[batch] == + mixed_contexts[batch]; + } + ok &= Check( + initialized_mixed_ok, + "InitializeState writes the exact shared CLI context vector" + ); + + ok &= Check( + CheckCli(), + "shared test CLI accepts broadcast/mixed and rejects invalid lists" + ); + + std::printf( + "[HOST_PLAN_TEST] status=%s\n", + ok ? "PASS" : "FAIL" + ); + return ok ? 0 : 1; +} diff --git a/tests/atomic_probe/pa_scheduler/test/test_shared_insert_turn.cpp b/tests/atomic_probe/pa_scheduler/test/test_shared_insert_turn.cpp new file mode 100644 index 0000000000..8e2e4f490b --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test/test_shared_insert_turn.cpp @@ -0,0 +1,480 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "pa_scheduler_core.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace pa_scheduler; + +static_assert(kSharedInsertTurnCapacity == 128, "completion-chain test expects the full legacy sidecar"); + +constexpr uint32_t kSequentialTasks = 260; +using LegacyTurnSnapshot = std::array; + +int g_failures = 0; + +void Check(bool condition, const char *message) { + if (condition) { + return; + } + std::fprintf(stderr, "[FAIL] shared per-task insert completion: %s\n", message); + ++g_failures; +} + +struct CompletionTestOps { + static constexpr bool kAtomicReturnReadyObserved = false; + static inline SchedulerState *observed_state = nullptr; + static inline std::atomic load_calls{0}; + static inline std::atomic cas_calls{0}; + static inline std::atomic legacy_turn_touches{0}; + static inline std::atomic last_load_address{0}; + static inline std::atomic last_cas_address{0}; + + static bool IsLegacyTurnAddress(const volatile int64_t *address) { + if (observed_state == nullptr) { + return false; + } + for (uint32_t lane = 0; lane < kSharedInsertTurnCapacity; ++lane) { + if (address == &SharedInsertTurnLine(observed_state->shared_map, lane).value) { + return true; + } + } + return false; + } + + static int32_t Load(volatile int32_t *address) { + return __atomic_fetch_add(address, static_cast(0), __ATOMIC_ACQUIRE); + } + + static int64_t Load(volatile int64_t *address) { + load_calls.fetch_add(1, std::memory_order_relaxed); + last_load_address.store(reinterpret_cast(address), std::memory_order_relaxed); + if (IsLegacyTurnAddress(address)) { + legacy_turn_touches.fetch_add(1, std::memory_order_relaxed); + } + return __atomic_fetch_add(address, static_cast(0), __ATOMIC_ACQUIRE); + } + + static int32_t Exchange(volatile int32_t *address, int32_t value) { + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } + + static int64_t Exchange(volatile int64_t *address, int64_t value) { + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } + + static uint64_t Exchange(volatile uint64_t *address, uint64_t value) { + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } + + static int64_t CompareExchange(volatile int64_t *address, int64_t expected, int64_t desired) { + cas_calls.fetch_add(1, std::memory_order_relaxed); + last_cas_address.store(reinterpret_cast(address), std::memory_order_relaxed); + if (IsLegacyTurnAddress(address)) { + legacy_turn_touches.fetch_add(1, std::memory_order_relaxed); + } + int64_t observed = expected; + (void)__atomic_compare_exchange_n(address, &observed, desired, false, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE); + return observed; + } + + static int64_t FetchMax(volatile int64_t *address, int64_t value, uint64_t &retries) { + int64_t current = __atomic_load_n(address, __ATOMIC_ACQUIRE); + retries = 0; + while (value > current) { + if (__atomic_compare_exchange_n(address, ¤t, value, true, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) { + break; + } + ++retries; + } + return current; + } + + static void StoreBarrier() { std::atomic_thread_fence(std::memory_order_seq_cst); } + + static void FlushRegion(void *, uint64_t) { StoreBarrier(); } + + static void InvalidateRegion(const void *, uint64_t) { StoreBarrier(); } + + static uint64_t Now() { + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) + .count() + ); + } + + template + static uint64_t NowAfterAtomicResult(T value) { + asm volatile("" : "+r"(value)); + return Now(); + } + + static void SpinHint() { std::this_thread::yield(); } + + static void ResetTrace(SchedulerState &state) { + observed_state = &state; + load_calls.store(0, std::memory_order_relaxed); + cas_calls.store(0, std::memory_order_relaxed); + legacy_turn_touches.store(0, std::memory_order_relaxed); + last_load_address.store(0, std::memory_order_relaxed); + last_cas_address.store(0, std::memory_order_relaxed); + } +}; + +SchedulerState *MapSparseSchedulerState() { + int flags = MAP_PRIVATE | MAP_ANONYMOUS; +#ifdef MAP_NORESERVE + flags |= MAP_NORESERVE; +#endif + void *memory = mmap(nullptr, sizeof(SchedulerState), PROT_READ | PROT_WRITE, flags, -1, 0); + if (memory == MAP_FAILED) { + std::perror("mmap SchedulerState"); + return nullptr; + } + return ::new (memory) SchedulerState; +} + +void UnmapSparseSchedulerState(SchedulerState *state) { + if (state != nullptr) { + (void)munmap(state, sizeof(*state)); + } +} + +LegacyTurnSnapshot SeedLegacyTurns(SchedulerState &state) { + LegacyTurnSnapshot snapshot{}; + for (uint32_t lane = 0; lane < kSharedInsertTurnCapacity; ++lane) { + snapshot[lane] = -10000 - static_cast(lane); + SharedInsertTurnLine(state.shared_map, lane).value = snapshot[lane]; + } + return snapshot; +} + +bool LegacyTurnsMatch(const SchedulerState &state, const LegacyTurnSnapshot &snapshot) { + for (uint32_t lane = 0; lane < kSharedInsertTurnCapacity; ++lane) { + const volatile int64_t *address = + &SharedInsertTurnLine(const_cast(state.shared_map), lane).value; + if (__atomic_load_n(address, __ATOMIC_ACQUIRE) != snapshot[lane]) { + return false; + } + } + return true; +} + +void ResetCompletionWords(SchedulerState &state, uint32_t count) { + state.fatal.value = 0; + for (uint32_t task = 0; task < count; ++task) { + state.tasks[task].deps_prepared = -1; + } +} + +bool AddressEquals(uintptr_t observed, volatile int64_t *expected) { + return observed == reinterpret_cast(expected); +} + +void TestSequentialCompletionChain(SchedulerState &state) { + ResetCompletionWords(state, kSequentialTasks + 1U); + const LegacyTurnSnapshot legacy = SeedLegacyTurns(state); + bool exact = true; + + for (uint32_t task = 0; task < kSequentialTasks; ++task) { + CompletionTestOps::ResetTrace(state); + LocalStats wait_stats{}; + int64_t ready_observed = INT64_MIN; + uint64_t load_count = UINT64_MAX; + const bool ready = WaitForSharedTaskInsertTurn( + &state, static_cast(task), wait_stats, ready_observed, load_count + ); + exact &= ready && state.fatal.value == 0; + if (task == 0) { + exact &= ready_observed == -1 && load_count == 0 && + CompletionTestOps::load_calls.load(std::memory_order_relaxed) == 0; + } else { + exact &= ready_observed == static_cast(task - 1U) && load_count == 1 && + CompletionTestOps::load_calls.load(std::memory_order_relaxed) == 1 && + AddressEquals( + CompletionTestOps::last_load_address.load(std::memory_order_relaxed), + &state.tasks[task - 1U].deps_prepared + ); + } + exact &= CompletionTestOps::cas_calls.load(std::memory_order_relaxed) == 0 && + CompletionTestOps::legacy_turn_touches.load(std::memory_order_relaxed) == 0; + + CompletionTestOps::ResetTrace(state); + LocalStats publish_stats{}; + int64_t cas_observed = INT64_MIN; + const bool published = HandoffSharedTaskInsertTurn( + &state, static_cast(task), publish_stats, cas_observed + ); + exact &= + published && cas_observed == -1 && state.tasks[task].deps_prepared == static_cast(task) && + CompletionTestOps::cas_calls.load(std::memory_order_relaxed) == 1 && + AddressEquals( + CompletionTestOps::last_cas_address.load(std::memory_order_relaxed), &state.tasks[task].deps_prepared + ) && + CompletionTestOps::legacy_turn_touches.load(std::memory_order_relaxed) == 0 && + LegacyTurnsMatch(state, legacy); + } + + for (uint32_t task = 0; task < kSequentialTasks; ++task) { + exact &= state.tasks[task].deps_prepared == static_cast(task); + } + exact &= state.tasks[kSequentialTasks].deps_prepared == -1; + Check( + exact, "task 0 skips predecessor; every N waits only N-1, " + "publishes only N, and never touches legacy turns" + ); +} + +void TestPendingOwnerWakesOnPredecessor(SchedulerState &state) { + ResetCompletionWords(state, 2); + const LegacyTurnSnapshot legacy = SeedLegacyTurns(state); + CompletionTestOps::ResetTrace(state); + std::atomic wait_finished{false}; + bool wait_ok = false; + int64_t ready_observed = INT64_MIN; + uint64_t load_count = 0; + LocalStats wait_stats{}; + std::thread waiter([&]() { + wait_ok = WaitForSharedTaskInsertTurn(&state, 1, wait_stats, ready_observed, load_count); + wait_finished.store(true, std::memory_order_release); + }); + + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (CompletionTestOps::load_calls.load(std::memory_order_acquire) == 0 && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + const bool observed_pending = CompletionTestOps::load_calls.load(std::memory_order_acquire) != 0 && + !wait_finished.load(std::memory_order_acquire); + + LocalStats publish_stats{}; + int64_t cas_observed = INT64_MIN; + const bool published = HandoffSharedTaskInsertTurn(&state, 0, publish_stats, cas_observed); + waiter.join(); + + Check( + observed_pending && published && cas_observed == -1 && wait_ok && ready_observed == 0 && load_count >= 2 && + state.tasks[0].deps_prepared == 0 && state.tasks[1].deps_prepared == -1 && state.fatal.value == 0 && + CompletionTestOps::legacy_turn_touches.load(std::memory_order_relaxed) == 0 && + LegacyTurnsMatch(state, legacy), + "task 1 remains pending until task 0 publishes its " + "per-task completion word" + ); +} + +void TestEmptyWriterStillCompletes(SchedulerState &state) { + ResetCompletionWords(state, 2); + const LegacyTurnSnapshot legacy = SeedLegacyTurns(state); + bool exact = true; + + for (int32_t task = 0; task < 2; ++task) { + TaskArgs args; + ConstructTaskArgs(args); + SubmitContext context{}; + context.task_id = task; + context.won = true; + context.result.task_id = task; + context.shared_result.Reset(task); + SharedTaskWriterDelta delta{}; + LocalStats stats{}; + CompletionTestOps::ResetTrace(state); + exact &= PrepareSharedTaskWriterDelta(args, context, delta) && delta.ordinary_count == 0 && + !delta.writer_intent_required && + PublishSharedTaskWriterDelta(&state, context, delta, stats) && + state.tasks[static_cast(task)].deps_prepared == task && + CompletionTestOps::cas_calls.load(std::memory_order_relaxed) == 1 && + AddressEquals( + CompletionTestOps::last_cas_address.load(std::memory_order_relaxed), + &state.tasks[static_cast(task)].deps_prepared + ) && + CompletionTestOps::legacy_turn_touches.load(std::memory_order_relaxed) == 0; + } + + Check( + exact && state.fatal.value == 0 && LegacyTurnsMatch(state, legacy), + "empty metadata transactions still publish one " + "completion per task without a sidecar baton" + ); +} + +void TestOutputsPublishBeforePredecessorWait( + SchedulerState &state +) { + ResetCompletionWords(state, 2); + const LegacyTurnSnapshot legacy = SeedLegacyTurns(state); + SharedOutputCell &cell = state.shared_map.shared_outputs[1]; + cell.published[0].value = -1; + cell.last_writer[0].value = -1; + TensorDesc descriptor{}; + descriptor.buffer_addr = 0x510000000ULL; + descriptor.ndims = 1; + descriptor.shapes[0] = 16; + descriptor.strides[0] = 1; + + TaskArgs args; + ConstructTaskArgs(args); + SubmitContext context{}; + context.task_id = 1; + context.won = true; + context.result.task_id = 1; + context.result.count = 1; + context.result.tensors[0] = &descriptor; + context.shared_result.Reset(1); + const bool output_ref_ok = + context.shared_result.AddOutputRef(1, 0); + SharedTaskWriterDelta delta{}; + const bool delta_ok = + PrepareSharedTaskWriterDelta(args, context, delta); + + CompletionTestOps::ResetTrace(state); + std::atomic publish_finished{false}; + bool publish_ok = false; + LocalStats task_stats{}; + std::thread owner([&]() { + publish_ok = + PublishSharedTaskWriterDelta( + &state, context, delta, task_stats + ); + publish_finished.store(true, std::memory_order_release); + }); + + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(2); + while ( + __atomic_load_n( + &cell.published[0].value, __ATOMIC_ACQUIRE + ) != 1 && + std::chrono::steady_clock::now() < deadline + ) { + std::this_thread::yield(); + } + const bool visible_before_turn = + __atomic_load_n( + &cell.published[0].value, __ATOMIC_ACQUIRE + ) == 1 && + __atomic_load_n( + &state.tasks[1].deps_prepared, __ATOMIC_ACQUIRE + ) == -1 && + !publish_finished.load(std::memory_order_acquire); + + LocalStats predecessor_stats{}; + int64_t predecessor_observed = INT64_MIN; + const bool predecessor_published = + HandoffSharedTaskInsertTurn( + &state, 0, predecessor_stats, + predecessor_observed + ); + owner.join(); + + Check( + output_ref_ok && delta_ok && visible_before_turn && + predecessor_published && + predecessor_observed == -1 && publish_ok && + state.fatal.value == 0 && + state.tasks[0].deps_prepared == 0 && + state.tasks[1].deps_prepared == 1 && + cell.tensors[0].buffer_addr == + descriptor.buffer_addr && + LegacyTurnsMatch(state, legacy), + "fresh output is visible while task 1 still waits for task 0, " + "and deps_prepared closes only after serialized metadata" + ); +} + +void TestCorruptionAndDuplicateFailClosed(SchedulerState &state) { + bool exact = true; + + ResetCompletionWords(state, 5); + LegacyTurnSnapshot legacy = SeedLegacyTurns(state); + state.tasks[2].deps_prepared = -2; + CompletionTestOps::ResetTrace(state); + LocalStats corrupt_predecessor_stats{}; + int64_t ready_observed = INT64_MIN; + uint64_t load_count = 0; + exact &= !WaitForSharedTaskInsertTurn( + &state, 3, corrupt_predecessor_stats, ready_observed, load_count + ) && + state.fatal.value == 1 && state.tasks[2].deps_prepared == -2 && + CompletionTestOps::cas_calls.load(std::memory_order_relaxed) == 0 && + CompletionTestOps::legacy_turn_touches.load(std::memory_order_relaxed) == 0 && + LegacyTurnsMatch(state, legacy); + + ResetCompletionWords(state, 2); + legacy = SeedLegacyTurns(state); + LocalStats first_stats{}; + int64_t first_observed = INT64_MIN; + exact &= + HandoffSharedTaskInsertTurn(&state, 0, first_stats, first_observed) && first_observed == -1; + state.fatal.value = 0; + CompletionTestOps::ResetTrace(state); + LocalStats duplicate_stats{}; + int64_t duplicate_observed = INT64_MIN; + exact &= !HandoffSharedTaskInsertTurn(&state, 0, duplicate_stats, duplicate_observed) && + duplicate_observed == 0 && state.tasks[0].deps_prepared == 0 && state.fatal.value == 1 && + CompletionTestOps::cas_calls.load(std::memory_order_relaxed) == 1 && + CompletionTestOps::legacy_turn_touches.load(std::memory_order_relaxed) == 0 && + LegacyTurnsMatch(state, legacy); + + ResetCompletionWords(state, 2); + legacy = SeedLegacyTurns(state); + state.tasks[1].deps_prepared = 99; + CompletionTestOps::ResetTrace(state); + LocalStats bad_current_stats{}; + int64_t bad_current_observed = INT64_MIN; + exact &= !HandoffSharedTaskInsertTurn(&state, 1, bad_current_stats, bad_current_observed) && + bad_current_observed == 99 && state.tasks[1].deps_prepared == 99 && state.fatal.value == 1 && + CompletionTestOps::cas_calls.load(std::memory_order_relaxed) == 1 && + CompletionTestOps::legacy_turn_touches.load(std::memory_order_relaxed) == 0 && + LegacyTurnsMatch(state, legacy); + + Check( + exact, "corrupt predecessor, duplicate completion, and " + "unexpected current value all set fatal without overwrite" + ); +} + +} // namespace + +int main() { + SchedulerState *state = MapSparseSchedulerState(); + if (state == nullptr) { + return 1; + } + + TestSequentialCompletionChain(*state); + TestPendingOwnerWakesOnPredecessor(*state); + TestEmptyWriterStillCompletes(*state); + TestOutputsPublishBeforePredecessorWait(*state); + TestCorruptionAndDuplicateFailClosed(*state); + + UnmapSparseSchedulerState(state); + if (g_failures != 0) { + std::fprintf( + stderr, + "[FAIL] shared per-task insert completion tests: " + "%d failure(s)\n", + g_failures + ); + return 1; + } + std::printf("[PASS] shared per-task insert completion chain tests\n"); + return 0; +} diff --git a/tests/atomic_probe/pa_scheduler/test/test_shared_materialize.cpp b/tests/atomic_probe/pa_scheduler/test/test_shared_materialize.cpp new file mode 100644 index 0000000000..1e779a27a6 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test/test_shared_materialize.cpp @@ -0,0 +1,325 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include +#include + +#define PA_DEVICE inline +#define PA_GM +#include "pa_frontend.h" + +namespace { + +using namespace pa_scheduler; + +int g_failures = 0; + +void Check(bool condition, const char *message) { + if (condition) { + return; + } + std::fprintf(stderr, "[FAIL] %s\n", message); + ++g_failures; +} + +struct MaterializeTestOps { + static int64_t Load(volatile int64_t *address) { + return __atomic_load_n(address, __ATOMIC_ACQUIRE); + } + + static int64_t FetchAdd(volatile int64_t *address, int64_t value) { + return __atomic_fetch_add(address, value, __ATOMIC_ACQ_REL); + } + + static int64_t Exchange(volatile int64_t *address, int64_t value) { + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } +}; + +struct HeapSnapshot { + int64_t cursor[kSharedHeapShards]; + int64_t vend; + uint64_t worker_heap_next; +}; + +struct Fixture { + std::unique_ptr worker; + std::unique_ptr map; + std::unique_ptr payload; + TaskArgs args; + SubmitContext context; + TensorCreateInfo create_info; + + Fixture() + : worker(std::make_unique()), + map(std::make_unique()), + payload(std::make_unique()), + args{}, + context{}, + create_info{} { + constexpr uint32_t kTaskId = 1; + const uint32_t shape[kMaxTensorDims] = { + kPaHeads, kPaBlocksPerRequest * kPaBlockSize, 0, 0, 0 + }; + InitCreateInfo(create_info, shape, 2, DataType::Bfloat16); + args.tags[0] = static_cast(TensorArgType::Output); + args.tensors[0].pointer.create_info = &create_info; + args.tensors[0].kind = TensorRefKind::CreateInfo; + args.tensor_count = 1; + args.scalar_count = 0; + args.has_error = false; + + context.self = worker.get(); + context.payload = payload.get(); + context.task_id = static_cast(kTaskId); + context.tensor_count = 0; + context.scalar_count = 0; + context.result.task_id = kTaskId; + context.result.count = 0; + context.shared_result.Reset(static_cast(kTaskId)); + Check( + PrepareSharedTaskOutputs( + context.shared_result, static_cast(kTaskId), + TaskKind::Qk + ), + "fixture prepares the one QK output symbol" + ); + } +}; + +HeapSnapshot Snapshot(const Fixture &fixture) { + HeapSnapshot snapshot{}; + for (uint32_t shard = 0; shard < kSharedHeapShards; ++shard) { + snapshot.cursor[shard] = + fixture.map->shared_heap_cursor[shard].value; + } + snapshot.vend = fixture.map->shared_heap_vend.value; + snapshot.worker_heap_next = fixture.worker->heap_next; + return snapshot; +} + +bool SameSnapshot( + const Fixture &fixture, const HeapSnapshot &expected +) { + for (uint32_t shard = 0; shard < kSharedHeapShards; ++shard) { + if (fixture.map->shared_heap_cursor[shard].value != + expected.cursor[shard]) { + return false; + } + } + return fixture.map->shared_heap_vend.value == expected.vend && + fixture.worker->heap_next == expected.worker_heap_next; +} + +void ExpectRejectedWithoutHeapChange( + Fixture &fixture, uint64_t heap_base, uint64_t heap_size, + const char *reject_message, const char *state_message +) { + const HeapSnapshot before = Snapshot(fixture); + Check( + !MaterializeTask( + *fixture.worker, 1, fixture.args, fixture.context, *fixture.map, + heap_base, heap_size, TaskKind::Qk, 0, 0 + ), + reject_message + ); + Check(SameSnapshot(fixture, before), state_message); +} + +void TestValidQkMaterialize() { + Fixture fixture; + Check( + MaterializeTask( + *fixture.worker, 1, fixture.args, fixture.context, *fixture.map, + kSyntheticHeapBase, kHeapBytes, TaskKind::Qk, 0, 0 + ), + "valid QK output materializes" + ); + + const uint64_t bytes = + static_cast(kPaHeads) * + kPaBlocksPerRequest * kPaBlockSize * 2; + const uint64_t shard_span = + SharedHeapAlignDown(kHeapBytes / kSharedHeapShards); + Check( + fixture.map->shared_heap_cursor[1].value == + static_cast(bytes), + "valid QK advances only shard one" + ); + Check( + fixture.map->shared_heap_vend.value == static_cast(bytes) && + fixture.worker->heap_next == bytes, + "valid QK advances and snapshots the aggregate vend" + ); + Check( + fixture.context.output_bytes == bytes && + fixture.context.result.count == 1, + "valid QK returns one descriptor and exact bytes" + ); + Check( + fixture.payload->tensors[0].buffer_addr == + kSyntheticHeapBase + shard_span && + fixture.payload->tensors[0].buffer_size == bytes, + "valid QK descriptor uses the physical shard address" + ); +} + +void TestCheckedShapeAndStride() { + { + Fixture fixture; + fixture.create_info.ndims = kMaxTensorDims; + for (uint32_t dimension = 0; dimension < kMaxTensorDims; + ++dimension) { + fixture.create_info.shapes[dimension] = UINT32_MAX; + } + ExpectRejectedWithoutHeapChange( + fixture, kSyntheticHeapBase, kHeapBytes, + "shape product overflow is rejected", + "shape product overflow changes no heap state" + ); + } + { + Fixture fixture; + fixture.create_info.ndims = 2; + fixture.create_info.shapes[0] = 65536; + fixture.create_info.shapes[1] = 65536; + ExpectRejectedWithoutHeapChange( + fixture, kSyntheticHeapBase, kHeapBytes, + "uint32 descriptor stride overflow is rejected", + "stride overflow changes no heap state" + ); + } + { + Fixture fixture; + fixture.create_info.start_offset = 1; + ExpectRejectedWithoutHeapChange( + fixture, kSyntheticHeapBase, kHeapBytes, + "fresh Output start offset is rejected", + "fresh Output start offset changes no heap state" + ); + } +} + +void TestMalformedInputPreflight() { + { + Fixture fixture; + fixture.args.tensor_count = + static_cast(kMaxTaskTensors) + 1; + ExpectRejectedWithoutHeapChange( + fixture, kSyntheticHeapBase, kHeapBytes, + "oversized tensor count is rejected", + "oversized tensor count changes no heap state" + ); + } + { + Fixture fixture; + fixture.args.has_error = true; + ExpectRejectedWithoutHeapChange( + fixture, kSyntheticHeapBase, kHeapBytes, + "builder error state is rejected", + "builder error state changes no heap state" + ); + } + { + Fixture fixture; + fixture.context.result.count = 1; + ExpectRejectedWithoutHeapChange( + fixture, kSyntheticHeapBase, kHeapBytes, + "nonempty materialize result is rejected", + "nonempty result changes no heap state" + ); + } + { + Fixture fixture; + fixture.args.tensors[0].kind = TensorRefKind::LocalTensor; + ExpectRejectedWithoutHeapChange( + fixture, kSyntheticHeapBase, kHeapBytes, + "Output with a non-CreateInfo ref is rejected", + "wrong Output ref kind changes no heap state" + ); + } + { + Fixture fixture; + fixture.args.tensors[0].pointer.create_info = nullptr; + ExpectRejectedWithoutHeapChange( + fixture, kSyntheticHeapBase, kHeapBytes, + "null Output CreateInfo is rejected", + "null Output CreateInfo changes no heap state" + ); + } + { + Fixture fixture; + fixture.args.tags[0] = + static_cast(TensorArgType::NoDependency) + 1; + ExpectRejectedWithoutHeapChange( + fixture, kSyntheticHeapBase, kHeapBytes, + "invalid tensor tag is rejected", + "invalid tensor tag changes no heap state" + ); + } + { + Fixture fixture; + fixture.args.tensor_count = 2; + fixture.args.tags[1] = + static_cast(TensorArgType::Input); + fixture.args.tensors[1].kind = TensorRefKind::LocalTensor; + fixture.args.tensors[1].pointer.local_tensor = nullptr; + ExpectRejectedWithoutHeapChange( + fixture, kSyntheticHeapBase, kHeapBytes, + "null non-Output descriptor is rejected before reserve", + "null non-Output descriptor changes no heap state" + ); + } + { + Fixture fixture; + fixture.args.tensor_count = 2; + fixture.args.tags[1] = + static_cast(TensorArgType::Input); + fixture.args.tensors[1].kind = + TensorRefKind::SharedOutputRef; + fixture.args.tensors[1].pointer.output_ref = + FdwicOutputRef{1, 0, 0, 0, 0, 0}; + ExpectRejectedWithoutHeapChange( + fixture, kSyntheticHeapBase, kHeapBytes, + "future shared symbol is rejected before reserve", + "future shared symbol changes no heap state" + ); + } +} + +void TestHeapAddressOverflowPreflight() { + Fixture fixture; + ExpectRejectedWithoutHeapChange( + fixture, UINT64_MAX - 4095, kHeapBytes, + "heap base plus heap size overflow is rejected", + "heap address overflow changes no heap state" + ); +} + +} // namespace + +int main() { + TestValidQkMaterialize(); + TestCheckedShapeAndStride(); + TestMalformedInputPreflight(); + TestHeapAddressOverflowPreflight(); + if (g_failures != 0) { + std::fprintf( + stderr, + "shared materialize self-test failed: %d assertion(s)\n", + g_failures + ); + return 1; + } + std::printf("shared materialize self-test passed\n"); + return 0; +} diff --git a/tests/atomic_probe/pa_scheduler/test/test_shared_ordered_submit.cpp b/tests/atomic_probe/pa_scheduler/test/test_shared_ordered_submit.cpp new file mode 100644 index 0000000000..79931ccaa9 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test/test_shared_ordered_submit.cpp @@ -0,0 +1,982 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define PA_COMPETE_FIRST_SPLIT_FINISH 1 +#include "host_support.h" + +#define PA_DEVICE inline +#define PA_GM +#define PA_BUILD_SWIMLANE 1 +#define PA_TEST_SHARED_SUBMIT_HOOKS 1 +#include "pa_scheduler_core.h" + +namespace { + +using namespace pa_scheduler; + +enum class OrderedSubmitHookMode : uint32_t { + None = 0, + BuildOverlap = 1, + ExecutionOverlap = 2, +}; + +struct OrderedSubmitTestOps { + static constexpr bool kAtomicReturnReadyObserved = false; + static inline thread_local CompeteFirstSplitRuntimeState runtime{}; + + static inline SchedulerState *observed_state = nullptr; + static inline OrderedSubmitHookMode hook_mode = + OrderedSubmitHookMode::None; + static inline std::atomic shared_map_accesses{0}; + static inline std::atomic task4_insert_hook_calls{0}; + static inline std::atomic task8_build_hook_calls{0}; + static inline std::atomic independent_insert_hook_calls{0}; + static inline std::atomic task4_waiting_after_insert{false}; + static inline std::atomic task8_built{false}; + static inline std::atomic task8_built_before_task4_completion{false}; + static inline std::atomic task6_executed_before_task4_build{false}; + static inline std::atomic hook_timed_out{false}; + static inline std::atomic + claim_attempts_by_task[kMaxTasks]{}; + static inline std::atomic + claim_wins_by_task[kMaxTasks]{}; + static inline std::atomic + completion_loads_by_cell[kMaxTasks]{}; + static inline std::atomic + completion_cas_by_task[kMaxTasks]{}; + static inline std::atomic + completion_publish_by_task[kMaxTasks]{}; + static inline std::atomic + bad_completion_cas{0}; + static inline std::atomic + legacy_turn_atomic_accesses{0}; + + static void ResetHooks() { + observed_state = nullptr; + hook_mode = OrderedSubmitHookMode::None; + shared_map_accesses.store(0, std::memory_order_relaxed); + task4_insert_hook_calls.store(0, std::memory_order_relaxed); + task8_build_hook_calls.store(0, std::memory_order_relaxed); + independent_insert_hook_calls.store(0, std::memory_order_relaxed); + task4_waiting_after_insert.store(false, std::memory_order_relaxed); + task8_built.store(false, std::memory_order_relaxed); + task8_built_before_task4_completion.store(false, std::memory_order_relaxed); + task6_executed_before_task4_build.store(false, std::memory_order_relaxed); + hook_timed_out.store(false, std::memory_order_relaxed); + for (uint32_t task = 0; task < kMaxTasks; ++task) { + claim_attempts_by_task[task].store( + 0, std::memory_order_relaxed + ); + claim_wins_by_task[task].store( + 0, std::memory_order_relaxed + ); + completion_loads_by_cell[task].store( + 0, std::memory_order_relaxed + ); + completion_cas_by_task[task].store( + 0, std::memory_order_relaxed + ); + completion_publish_by_task[task].store( + 0, std::memory_order_relaxed + ); + } + bad_completion_cas.store( + 0, std::memory_order_relaxed + ); + legacy_turn_atomic_accesses.store( + 0, std::memory_order_relaxed + ); + } + + static CompeteFirstSplitRuntimeState & + CompeteFirstSplitState() { + return runtime; + } + + static bool FinishCompeteFirstCallback( + const CallbackSubmitTicket *ticket, + const TaskArgs *args + ) { + return FinishSplitCallbackSubmitFromRuntime< + OrderedSubmitTestOps + >(ticket, args) == 1U; + } + + static void CountSharedMapAccess( + const volatile void *address, uint64_t bytes + ) { + if (observed_state == nullptr) { + return; + } + const uintptr_t current = + reinterpret_cast(address); + const uintptr_t access_end = current + bytes; + const uintptr_t begin = reinterpret_cast( + &observed_state->shared_map + ); + const uintptr_t end = + begin + sizeof(SharedTensorMapSidecar); + if (current < end && access_end > begin) { + shared_map_accesses.fetch_add( + 1, std::memory_order_relaxed + ); + } + } + + static int32_t Load(volatile int32_t *address) { + CountSharedMapAccess(address, sizeof(*address)); + return __atomic_fetch_add(address, static_cast(0), __ATOMIC_ACQUIRE); + } + + static int64_t Load(volatile int64_t *address) { + CountSharedMapAccess(address, sizeof(*address)); + const int32_t task_cell = + CompletionTaskCell(address); + if (task_cell >= 0) { + completion_loads_by_cell[ + static_cast(task_cell) + ].fetch_add(1, std::memory_order_relaxed); + } + if (IsLegacyTurnAddress(address)) { + legacy_turn_atomic_accesses.fetch_add( + 1, std::memory_order_relaxed + ); + } + return __atomic_fetch_add(address, static_cast(0), __ATOMIC_ACQUIRE); + } + + static uint64_t Load(volatile uint64_t *address) { + CountSharedMapAccess(address, sizeof(*address)); + return __atomic_fetch_add(address, static_cast(0), __ATOMIC_ACQUIRE); + } + + static int32_t Exchange(volatile int32_t *address, int32_t value) { + CountSharedMapAccess(address, sizeof(*address)); + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } + + static int64_t Exchange(volatile int64_t *address, int64_t value) { + CountSharedMapAccess(address, sizeof(*address)); + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } + + static uint64_t Exchange(volatile uint64_t *address, uint64_t value) { + CountSharedMapAccess(address, sizeof(*address)); + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } + + static int64_t CompareExchange( + volatile int64_t *address, int64_t expected, int64_t desired + ) { + CountSharedMapAccess(address, sizeof(*address)); + const int32_t completion_task = + CompletionTaskCell(address); + if (completion_task >= 0) { + const uint32_t task = + static_cast(completion_task); + completion_cas_by_task[task].fetch_add( + 1, std::memory_order_relaxed + ); + if (expected != -1 || + desired != completion_task) { + bad_completion_cas.fetch_add( + 1, std::memory_order_relaxed + ); + } + } + if (IsLegacyTurnAddress(address)) { + legacy_turn_atomic_accesses.fetch_add( + 1, std::memory_order_relaxed + ); + } + int64_t observed = expected; + (void)__atomic_compare_exchange_n( + address, &observed, desired, false, + __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE + ); + if (completion_task >= 0 && + observed == expected && expected == -1 && + desired == completion_task) { + completion_publish_by_task[ + static_cast(completion_task) + ].fetch_add(1, std::memory_order_relaxed); + } + return observed; + } + + static int64_t FetchAdd(volatile int64_t *address, int64_t value) { + CountSharedMapAccess(address, sizeof(*address)); + return __atomic_fetch_add(address, value, __ATOMIC_ACQ_REL); + } + + static int64_t FetchMax( + volatile int64_t *address, int64_t value, uint64_t &retries + ) { + CountSharedMapAccess(address, sizeof(*address)); + const bool valid_task = + value >= 0 && + value < static_cast(kMaxTasks); + const bool claim_address = + observed_state != nullptr && valid_task && + IsClaimCursorAddress(address); + if (claim_address) { + claim_attempts_by_task[ + static_cast(value) + ].fetch_add(1, std::memory_order_relaxed); + } + int64_t current = __atomic_load_n(address, __ATOMIC_ACQUIRE); + retries = 0; + bool won = false; + while (value > current) { + if (__atomic_compare_exchange_n( + address, ¤t, value, true, + __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE + )) { + won = true; + break; + } + ++retries; + } + if (claim_address && won) { + claim_wins_by_task[ + static_cast(value) + ].fetch_add(1, std::memory_order_relaxed); + } + return current; + } + + static bool IsLegacyTurnAddress( + const volatile int64_t *address + ) { + if (observed_state == nullptr) { + return false; + } + for (uint32_t lane = 0; + lane < kSharedInsertTurnCapacity; ++lane) { + if (address == + &SharedInsertTurnLine( + observed_state->shared_map, lane + ).value) { + return true; + } + } + return false; + } + + static int32_t CompletionTaskCell( + const volatile int64_t *address + ) { + if (observed_state == nullptr) { + return -1; + } + const uintptr_t current = + reinterpret_cast(address); + const uintptr_t begin = + reinterpret_cast( + &observed_state->tasks[0].deps_prepared + ); + if (current < begin) { + return -1; + } + const uintptr_t delta = current - begin; + if (delta % sizeof(TaskCell) != 0) { + return -1; + } + const uintptr_t task = delta / sizeof(TaskCell); + return task < kMaxTasks + ? static_cast(task) + : -1; + } + + static bool IsClaimCursorAddress( + const volatile int64_t *address + ) { + if (observed_state == nullptr) { + return false; + } + for (uint32_t shard = 0; shard < kCursorShards; + ++shard) { + if (address == + &observed_state + ->cube_cursor[shard].value || + address == + &observed_state + ->alloc_cursor[shard].value) { + return true; + } + } + for (uint32_t shard = 0; + shard < kSharedVectorCursorShards; ++shard) { + if (address == + &observed_state->shared_map + .shared_vector_cursor[shard].value) { + return true; + } + } + return false; + } + + static void StoreBarrier() { + __atomic_thread_fence(__ATOMIC_SEQ_CST); + } + + static uint64_t Now() { + return static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch() + ).count() + ); + } + + template + static uint64_t NowAfterAtomicResult(T value) { + asm volatile("" : "+r"(value)); + return Now(); + } + + static void SpinHint() { + std::this_thread::yield(); + } + + static void InvalidateRegion( + const void *address, uint64_t bytes + ) { + CountSharedMapAccess(address, bytes); + __atomic_thread_fence(__ATOMIC_SEQ_CST); + } + + static void FlushRegion(void *address, uint64_t bytes) { + CountSharedMapAccess(address, bytes); + __atomic_thread_fence(__ATOMIC_SEQ_CST); + } + + static void Publish(uint64_t *address, uint64_t value) { + CountSharedMapAccess(address, sizeof(*address)); + __atomic_store_n(address, value, __ATOMIC_RELEASE); + } + + static bool PmuWindowStart(SchedulerState *, uint32_t) { + return false; + } + + static void PmuWindowStop(SchedulerState *, uint32_t, bool) {} + + static void ExecuteKernel( + SchedulerState *, WorkerState &, TaskKind, uint32_t + ) {} + + static void AfterSharedTaskInsert( + SchedulerState *state, WorkerState &, uint32_t task_id + ) { + constexpr uint32_t kPausedUp = 4; + if (hook_mode == OrderedSubmitHookMode::BuildOverlap) { + if (task_id != kPausedUp) { + return; + } + task4_insert_hook_calls.fetch_add(1, std::memory_order_relaxed); + task4_waiting_after_insert.store(true, std::memory_order_release); + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!task8_built.load(std::memory_order_acquire)) { + if (state->fatal.value != 0 || + std::chrono::steady_clock::now() >= deadline) { + hook_timed_out.store(true, std::memory_order_relaxed); + break; + } + std::this_thread::yield(); + } + return; + } + + // B2/G1 中 batch0 UP=task4，batch1 QK=task6。task4 发布插入 + // 前沿后暂停在 fanin/Build 之前；task6 只依赖 batch1 Alloc， + // 因此它的 completion flag 必须能在 task4 仍为 0 时变成 1。 + constexpr uint32_t kPausedBatch0Up = 4; + constexpr uint32_t kIndependentBatch1Qk = 6; + if (hook_mode != OrderedSubmitHookMode::ExecutionOverlap || + task_id != kPausedBatch0Up) { + return; + } + independent_insert_hook_calls.fetch_add( + 1, std::memory_order_relaxed + ); + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (Load(&state->tasks[kIndependentBatch1Qk].flag) == 0) { + if (state->fatal.value != 0 || + std::chrono::steady_clock::now() >= deadline) { + hook_timed_out.store(true, std::memory_order_relaxed); + return; + } + std::this_thread::yield(); + } + task6_executed_before_task4_build.store( + Load(&state->tasks[kPausedBatch0Up].flag) == 0, + std::memory_order_release + ); + } + + static void AfterSharedTaskBuild( + SchedulerState *state, WorkerState &, uint32_t task_id, TaskKind kind + ) { + constexpr uint32_t kPausedUp = 4; + constexpr uint32_t kFollowingUp = 8; + if (task_id != kFollowingUp || kind != TaskKind::Up) { + return; + } + task8_build_hook_calls.fetch_add(1, std::memory_order_relaxed); + const bool overlap = + task4_waiting_after_insert.load(std::memory_order_acquire) && + Load(&state->tasks[kPausedUp].flag) == 0; + task8_built_before_task4_completion.store(overlap, std::memory_order_release); + task8_built.store(true, std::memory_order_release); + } +}; + +SchedulerState *MapSchedulerState() { + void *memory = mmap( + nullptr, sizeof(SchedulerState), PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0 + ); + if (memory == MAP_FAILED) { + return nullptr; + } + return new (memory) SchedulerState{}; +} + +using LegacyTurnSnapshot = + std::array; + +LegacyTurnSnapshot SeedLegacyTurns( + SchedulerState &state +) { + LegacyTurnSnapshot snapshot{}; + for (uint32_t lane = 0; + lane < kSharedInsertTurnCapacity; ++lane) { + snapshot[lane] = + -10000 - static_cast(lane); + __atomic_store_n( + &SharedInsertTurnLine( + state.shared_map, lane + ).value, + snapshot[lane], __ATOMIC_RELEASE + ); + } + return snapshot; +} + +bool LegacyTurnsMatch( + const SchedulerState &state, + const LegacyTurnSnapshot &snapshot +) { + for (uint32_t lane = 0; + lane < kSharedInsertTurnCapacity; ++lane) { + if (__atomic_load_n( + &SharedInsertTurnLine( + const_cast( + state.shared_map + ), + lane + ).value, + __ATOMIC_ACQUIRE + ) != snapshot[lane]) { + return false; + } + } + return true; +} + +uint32_t ExpectedClaimAttempts(TaskKind kind) { + switch (kind) { + case TaskKind::Alloc: + return kWorkers; + case TaskKind::Qk: + case TaskKind::Pv: + return kAicWorkers; + case TaskKind::Sf: + case TaskKind::Up: + return kAivWorkers; + case TaskKind::Count: + return 0; + } + return 0; +} + +bool ClaimAndInsertEvidenceMatches( + const SchedulerState &state, uint32_t task_count +) { + uint32_t planned_tasks = 0; + bool exact = true; + for (uint32_t batch = 0; + batch < state.config.batches; ++batch) { + SharedPaBatchPlan plan{}; + exact &= BuildSharedPaBatchPlan( + static_cast( + state.context_lens[batch] + ), + planned_tasks, plan + ); + if (!exact) { + return false; + } + for (uint32_t offset = 0; + offset < plan.task_count; ++offset) { + SharedPaPlannedTask task{}; + exact &= SharedPaPlannedTaskAt( + plan, offset, task + ); + const uint32_t task_id = + plan.batch_start + offset; + exact &= + OrderedSubmitTestOps:: + claim_attempts_by_task[task_id].load( + std::memory_order_relaxed + ) == ExpectedClaimAttempts(task.kind); + exact &= + OrderedSubmitTestOps:: + claim_wins_by_task[task_id].load( + std::memory_order_relaxed + ) == 1; + exact &= + state.tasks[task_id].deps_prepared == + static_cast(task_id); + exact &= + OrderedSubmitTestOps:: + completion_cas_by_task[task_id] + .load(std::memory_order_relaxed) == 1; + exact &= + OrderedSubmitTestOps:: + completion_publish_by_task[task_id] + .load(std::memory_order_relaxed) == 1; + const uint32_t completion_loads = + OrderedSubmitTestOps:: + completion_loads_by_cell[task_id] + .load(std::memory_order_relaxed); + exact &= task_id + 1U < task_count + ? completion_loads != 0 + : completion_loads == 0; + } + planned_tasks += plan.task_count; + } + exact &= planned_tasks == task_count; + for (uint32_t task = task_count; + task < kMaxTasks; ++task) { + exact &= + OrderedSubmitTestOps:: + completion_cas_by_task[task].load( + std::memory_order_relaxed + ) == 0 && + OrderedSubmitTestOps:: + completion_publish_by_task[task].load( + std::memory_order_relaxed + ) == 0 && + OrderedSubmitTestOps:: + completion_loads_by_cell[task].load( + std::memory_order_relaxed + ) == 0; + } + return exact && + OrderedSubmitTestOps::bad_completion_cas.load( + std::memory_order_relaxed + ) == 0 && + OrderedSubmitTestOps:: + legacy_turn_atomic_accesses.load( + std::memory_order_relaxed + ) == 0; +} + +bool RunLoserZeroTensorMapAccessTest() { + SchedulerState *state = MapSchedulerState(); + if (state == nullptr) { + return false; + } + pa_scheduler::host::Options options; + options.batches = 1; + options.shared_context_lens = {16384}; + options.trace_enabled = false; + pa_scheduler::host::InitializeState(state, options); + pa_scheduler::host::ConfigureTrace(state, options, nullptr); + + constexpr uint32_t kTask = 4; + SubmitContext context{}; + context.task_id = static_cast(kTask); + context.won = false; + context.kernel_id = -1; + context.shared_result.Reset(static_cast(kTask)); + LocalStats stats{}; + const CallbackSubmitTicket ticket{ + 1, kTask, -1, 0, + EncodeSharedPaTaskMeta(TaskKind::Up, 0, true, false) + }; + + int64_t turns_before[kSharedInsertTurnCapacity] = {}; + for (uint32_t lane = 0; + lane < kSharedInsertTurnCapacity; ++lane) { + volatile int64_t *address = + &SharedInsertTurnLine( + state->shared_map, lane + ).value; + turns_before[lane] = + static_cast(77U + lane); + __atomic_store_n( + address, turns_before[lane], + __ATOMIC_RELEASE + ); + } + OrderedSubmitTestOps::ResetHooks(); + OrderedSubmitTestOps::observed_state = state; + const bool finished = + FinishSharedLoserSubmit( + state, context, stats, ticket + ); + bool turns_unchanged = true; + for (uint32_t lane = 0; + lane < kSharedInsertTurnCapacity; ++lane) { + turns_unchanged &= + __atomic_load_n( + &SharedInsertTurnLine( + state->shared_map, lane + ).value, + __ATOMIC_ACQUIRE + ) == turns_before[lane]; + } + const bool ok = + finished && state->fatal.value == 0 && + stats.result.submits == 1 && + stats.declared_task_count == 0 && + turns_unchanged && + state->tasks[kTask].deps_prepared == -1 && + OrderedSubmitTestOps::shared_map_accesses.load( + std::memory_order_relaxed + ) == 0; + std::printf( + "[ORDERED_SUBMIT] loser_zero_map_access=%s accesses=%llu\n", + ok ? "PASS" : "FAIL", + static_cast( + OrderedSubmitTestOps::shared_map_accesses.load( + std::memory_order_relaxed + ) + ) + ); + OrderedSubmitTestOps::observed_state = nullptr; + (void)munmap(state, sizeof(SchedulerState)); + return ok; +} + +bool RunReadyFaninPrefixCompactionTest() { + SchedulerState *state = MapSchedulerState(); + if (state == nullptr) { + return false; + } + pa_scheduler::host::Options options; + options.batches = 1; + options.shared_context_lens = {8192}; + options.trace_enabled = false; + pa_scheduler::host::InitializeState(state, options); + pa_scheduler::host::ConfigureTrace(state, options, nullptr); + + LocalSlot blocked_at_front{}; + blocked_at_front.fanin_count = 2; + blocked_at_front.fanin[0] = 4; + blocked_at_front.fanin[1] = 5; + state->tasks[4].flag = 0; + state->tasks[5].flag = 1; + LocalStats front_stats{}; + const bool front_ready = SlotReady( + state, blocked_at_front, front_stats + ); + const bool front_unchanged = + !front_ready && blocked_at_front.fanin_count == 2 && + blocked_at_front.fanin[0] == 4 && + blocked_at_front.fanin[1] == 5 && + front_stats.result.fanin_ready_loads == 0 && + front_stats.result.fanin_not_ready_loads == 1; + + LocalSlot slot{}; + slot.fanin_count = 4; + slot.fanin[0] = 0; + slot.fanin[1] = 1; + slot.fanin[2] = 2; + slot.fanin[3] = 3; + state->tasks[0].flag = 1; + state->tasks[1].flag = 0; + state->tasks[2].flag = 0; + state->tasks[3].flag = 0; + LocalStats stats{}; + const bool stage1 = + !SlotReady(state, slot, stats) && + slot.fanin_count == 3 && + slot.fanin[0] == 1 && slot.fanin[1] == 2 && + slot.fanin[2] == 3; + state->tasks[1].flag = 1; + const bool stage2 = + !SlotReady(state, slot, stats) && + slot.fanin_count == 2 && + slot.fanin[0] == 2 && slot.fanin[1] == 3; + state->tasks[2].flag = 1; + const bool stage3 = + !SlotReady(state, slot, stats) && + slot.fanin_count == 1 && slot.fanin[0] == 3; + state->tasks[3].flag = 1; + const bool final_ready = + SlotReady(state, slot, stats); + const bool ok = + front_unchanged && stage1 && stage2 && stage3 && + final_ready && slot.fanin_count == 0 && + stats.result.fanin_ready_loads == 4 && + stats.result.fanin_not_ready_loads == 3; + std::printf( + "[ORDERED_SUBMIT] ready_fanin_prefix_compaction=%s\n", + ok ? "PASS" : "FAIL" + ); + (void)munmap(state, sizeof(SchedulerState)); + return ok; +} + +bool RunInsertReleaseBeforeBuildTest() { + SchedulerState *state = MapSchedulerState(); + if (state == nullptr) { + return false; + } + pa_scheduler::host::Options options; + options.batches = 1; + options.runs = 1; + options.trace_enabled = false; + options.shared_context_lens = {kSharedPaMaxContextLength}; + options.final_barrier_shape = FinalBarrierShape::TwoLevel16; + pa_scheduler::host::InitializeState(state, options); + pa_scheduler::host::ConfigureTrace(state, options, nullptr); + const LegacyTurnSnapshot legacy = + SeedLegacyTurns(*state); + OrderedSubmitTestOps::ResetHooks(); + OrderedSubmitTestOps::hook_mode = + OrderedSubmitHookMode::BuildOverlap; + OrderedSubmitTestOps::observed_state = state; + + std::vector workers; + workers.reserve(kWorkers); + for (uint32_t worker_id = 0; worker_id < kWorkers; ++worker_id) { + const CoreRole role = + worker_id < kAicWorkers ? CoreRole::Aic : CoreRole::Aiv; + workers.emplace_back([state, worker_id, role]() { + RunScheduler(state, worker_id, role); + }); + } + for (std::thread &worker : workers) { + worker.join(); + } + + constexpr uint32_t kTaskCount = 17; + uint64_t kernel_counts[4] = {}; + bool worker_results_ok = true; + for (uint32_t worker_id = 0; worker_id < kWorkers; ++worker_id) { + const WorkerResult &result = state->results[worker_id]; + worker_results_ok &= + result.worker_id == worker_id && + result.submits == kTaskCount && + result.finish_cycle != 0 && + result.final_occupied == 0 && + result.completion_duplicates == 0; + for (uint32_t kind = 0; kind < 4; ++kind) { + kernel_counts[kind] += result.kernel_counts[kind]; + } + } + + bool all_tasks_ready = true; + bool claim_cells_match = true; + for (uint32_t task = 0; task < kTaskCount; ++task) { + all_tasks_ready &= state->tasks[task].flag == 1; + claim_cells_match &= + state->tasks[task].deps_prepared == + static_cast(task); + } + bool final_writers_ok = true; + for (uint32_t slot = 0; slot < 3; ++slot) { + final_writers_ok &= + state->shared_map.shared_outputs[0] + .last_writer[slot].value == 16; + } + + const bool overlap = + OrderedSubmitTestOps::task8_built_before_task4_completion.load( + std::memory_order_acquire + ); + const bool ok = + state->fatal.value == 0 && + LegacyTurnsMatch(*state, legacy) && + ClaimAndInsertEvidenceMatches( + *state, kTaskCount + ) && + OrderedSubmitTestOps::task4_insert_hook_calls.load( + std::memory_order_relaxed + ) == 1 && + OrderedSubmitTestOps::task8_build_hook_calls.load( + std::memory_order_relaxed + ) == 1 && + !OrderedSubmitTestOps::hook_timed_out.load( + std::memory_order_relaxed + ) && + overlap && worker_results_ok && all_tasks_ready && + claim_cells_match && final_writers_ok && + kernel_counts[0] == 4 && kernel_counts[1] == 4 && + kernel_counts[2] == 4 && kernel_counts[3] == 4 && + pa_scheduler::host::FinalBarrierStateMatches( + state->final_barrier, options.final_barrier_shape + ); + std::printf( + "[ORDERED_SUBMIT] release_before_build=%s " + "completed=%u legacy_turn0=%lld overlap=%u " + "kernels=%llu,%llu,%llu,%llu\n", + ok ? "PASS" : "FAIL", + kTaskCount, + static_cast( + state->shared_map.committed_tasks.value + ), + overlap ? 1U : 0U, + static_cast(kernel_counts[0]), + static_cast(kernel_counts[1]), + static_cast(kernel_counts[2]), + static_cast(kernel_counts[3]) + ); + OrderedSubmitTestOps::observed_state = nullptr; + (void)munmap(state, sizeof(SchedulerState)); + return ok; +} + +bool RunIndependentKernelExecutionTest() { + SchedulerState *state = MapSchedulerState(); + if (state == nullptr) { + return false; + } + pa_scheduler::host::Options options; + options.batches = 2; + options.runs = 1; + options.trace_enabled = false; + options.shared_context_lens = {8192, 8192}; + options.final_barrier_shape = FinalBarrierShape::TwoLevel16; + pa_scheduler::host::InitializeState(state, options); + pa_scheduler::host::ConfigureTrace(state, options, nullptr); + const LegacyTurnSnapshot legacy = + SeedLegacyTurns(*state); + OrderedSubmitTestOps::ResetHooks(); + OrderedSubmitTestOps::hook_mode = + OrderedSubmitHookMode::ExecutionOverlap; + OrderedSubmitTestOps::observed_state = state; + + std::vector workers; + workers.reserve(kWorkers); + for (uint32_t worker_id = 0; worker_id < kWorkers; ++worker_id) { + const CoreRole role = + worker_id < kAicWorkers ? CoreRole::Aic : CoreRole::Aiv; + workers.emplace_back([state, worker_id, role]() { + RunScheduler( + state, worker_id, role + ); + }); + } + for (std::thread &worker : workers) { + worker.join(); + } + + constexpr uint32_t kTaskCount = 10; + uint64_t kernel_counts[4] = {}; + bool worker_results_ok = true; + for (uint32_t worker_id = 0; worker_id < kWorkers; ++worker_id) { + const WorkerResult &result = state->results[worker_id]; + worker_results_ok &= + result.worker_id == worker_id && + result.submits == kTaskCount && + result.finish_cycle != 0 && + result.final_occupied == 0 && + result.completion_duplicates == 0; + for (uint32_t kind = 0; kind < 4; ++kind) { + kernel_counts[kind] += result.kernel_counts[kind]; + } + } + bool all_tasks_ready = true; + for (uint32_t task = 0; task < kTaskCount; ++task) { + all_tasks_ready &= state->tasks[task].flag == 1; + } + const bool execution_overlap = + OrderedSubmitTestOps::task6_executed_before_task4_build.load( + std::memory_order_acquire + ); + const bool ok = + state->fatal.value == 0 && + LegacyTurnsMatch(*state, legacy) && + ClaimAndInsertEvidenceMatches( + *state, kTaskCount + ) && + OrderedSubmitTestOps::independent_insert_hook_calls.load( + std::memory_order_relaxed + ) == 1 && + !OrderedSubmitTestOps::hook_timed_out.load( + std::memory_order_relaxed + ) && + execution_overlap && worker_results_ok && + all_tasks_ready && + kernel_counts[0] == 2 && kernel_counts[1] == 2 && + kernel_counts[2] == 2 && kernel_counts[3] == 2 && + pa_scheduler::host::FinalBarrierStateMatches( + state->final_barrier, options.final_barrier_shape + ); + std::printf( + "[ORDERED_SUBMIT] independent_kernel_overlap=%s " + "completed=%u legacy_turn0=%lld " + "kernels=%llu,%llu,%llu,%llu\n", + ok ? "PASS" : "FAIL", + kTaskCount, + static_cast( + state->shared_map.committed_tasks.value + ), + static_cast(kernel_counts[0]), + static_cast(kernel_counts[1]), + static_cast(kernel_counts[2]), + static_cast(kernel_counts[3]) + ); + OrderedSubmitTestOps::observed_state = nullptr; + (void)munmap(state, sizeof(SchedulerState)); + return ok; +} + +} // namespace + +int main() { + const bool loser_ok = RunLoserZeroTensorMapAccessTest(); + const bool fanin_compaction_ok = + RunReadyFaninPrefixCompactionTest(); + const bool overlap_ok = RunInsertReleaseBeforeBuildTest(); + const bool execution_ok = + RunIndependentKernelExecutionTest(); + if (!loser_ok || !fanin_compaction_ok || + !overlap_ok || !execution_ok) { + std::fprintf( + stderr, "[FAIL] shared ordered-insert Submit tests\n" + ); + return 1; + } + std::printf( + "[PASS] shared loser skips TensorMap; lookup/Build and " + "independent kernel execution cross prior owner Build\n" + ); + return 0; +} diff --git a/tests/atomic_probe/pa_scheduler/test/test_shared_output_symbols.cpp b/tests/atomic_probe/pa_scheduler/test/test_shared_output_symbols.cpp new file mode 100644 index 0000000000..4b1d7505ea --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test/test_shared_output_symbols.cpp @@ -0,0 +1,2018 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "pa_scheduler_core.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace pa_scheduler; + +int g_failures = 0; + +void Check(bool condition, const char *message) { + if (condition) { + return; + } + std::fprintf(stderr, "[FAIL] %s\n", message); + ++g_failures; +} + +// 该 Ops 只验证公共 symbol helper 的原子状态机和 descriptor 搬运。 +// fence 不模拟 A5 DCache；设备缓存可见性仍必须由 CCEC 上板门禁证明。 +struct SymbolTestOps { + static constexpr bool kAtomicReturnReadyObserved = false; + static volatile int64_t *wait_address; + static std::atomic wait_loads; + static std::atomic now_calls; + static std::atomic spin_calls; + + static int32_t Load(volatile int32_t *address) { + return __atomic_fetch_add(address, int32_t{0}, __ATOMIC_ACQUIRE); + } + + static int64_t Load(volatile int64_t *address) { + if (address == wait_address) { + wait_loads.fetch_add(1, std::memory_order_release); + } + return __atomic_fetch_add(address, int64_t{0}, __ATOMIC_ACQUIRE); + } + + static int32_t Exchange(volatile int32_t *address, int32_t value) { + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } + + static int64_t Exchange(volatile int64_t *address, int64_t value) { + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } + + static uint64_t Exchange(volatile uint64_t *address, uint64_t value) { + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } + + static int64_t CompareExchange( + volatile int64_t *address, int64_t expected, int64_t desired + ) { + int64_t observed = expected; + (void)__atomic_compare_exchange_n( + address, &observed, desired, false, + __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE + ); + return observed; + } + + static int64_t FetchMax( + volatile int64_t *address, int64_t value, uint64_t &retries + ) { + int64_t current = __atomic_load_n(address, __ATOMIC_ACQUIRE); + retries = 0; + while (current < value) { + if (__atomic_compare_exchange_n( + address, ¤t, value, true, __ATOMIC_ACQ_REL, + __ATOMIC_ACQUIRE + )) { + break; + } + ++retries; + } + return current; + } + + static void StoreBarrier() { + std::atomic_thread_fence(std::memory_order_seq_cst); + } + + static void FlushRegion(void *, uint64_t) { + std::atomic_thread_fence(std::memory_order_seq_cst); + } + + static void InvalidateRegion(const void *, uint64_t) { + std::atomic_thread_fence(std::memory_order_seq_cst); + } + + static uint64_t Now() { + now_calls.fetch_add(1, std::memory_order_relaxed); + return static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch() + ).count() + ); + } + + template + static uint64_t NowAfterAtomicResult(T value) { + asm volatile("" : "+r"(value)); + return Now(); + } + + static void SpinHint() { + spin_calls.fetch_add(1, std::memory_order_relaxed); + std::this_thread::yield(); + } +}; + +volatile int64_t *SymbolTestOps::wait_address = nullptr; +std::atomic SymbolTestOps::wait_loads{0}; +std::atomic SymbolTestOps::now_calls{0}; +std::atomic SymbolTestOps::spin_calls{0}; + +// 把第二次读钟直接推进到 watchdog 期限之后，避免用真实 2 秒等待测试 +// timeout 终止语义。 +struct ExpiredWaitOps : SymbolTestOps { + static std::atomic calls; + + static uint64_t Now() { + const uint64_t call = calls.fetch_add(1, std::memory_order_relaxed); + return call == 0 ? 0 : kWatchdogTicks + 1; + } + + static void SpinHint() {} +}; + +std::atomic ExpiredWaitOps::calls{0}; + +// 只在定向测试中模拟“预检后、atomic 执行前”出现的协议异常，覆盖正常 +// 单写者 Case1 不会命中的冷回滚分支。 +struct PublicationFaultOps : SymbolTestOps { + using SymbolTestOps::Exchange; + + static volatile int64_t *fetch_race_address; + static volatile int64_t *exchange_race_address; + + static int64_t FetchMax( + volatile int64_t *address, int64_t value, uint64_t &retries + ) { + if (address == fetch_race_address) { + __atomic_store_n(address, int64_t{-2}, __ATOMIC_RELEASE); + fetch_race_address = nullptr; + } + return SymbolTestOps::FetchMax(address, value, retries); + } + + static int64_t Exchange(volatile int64_t *address, int64_t value) { + if (address == exchange_race_address) { + __atomic_store_n(address, int64_t{7}, __ATOMIC_RELEASE); + exchange_race_address = nullptr; + } + return SymbolTestOps::Exchange(address, value); + } +}; + +volatile int64_t *PublicationFaultOps::fetch_race_address = nullptr; +volatile int64_t *PublicationFaultOps::exchange_race_address = nullptr; + +struct SealOrderOps : SymbolTestOps { + using SymbolTestOps::Exchange; + + static volatile int64_t *output_writer_address; + static volatile int64_t *published_address; + static int64_t expected_writer; + static bool order_ok; + + static int64_t Exchange(volatile int64_t *address, int64_t value) { + if (address == published_address) { + order_ok &= + __atomic_load_n(output_writer_address, __ATOMIC_ACQUIRE) == + expected_writer; + } + return SymbolTestOps::Exchange(address, value); + } +}; + +volatile int64_t *SealOrderOps::output_writer_address = nullptr; +volatile int64_t *SealOrderOps::published_address = nullptr; +int64_t SealOrderOps::expected_writer = 0; +bool SealOrderOps::order_ok = true; + +void ResetSharedState(SharedTensorMapSidecar &map) { + std::memset(&map, 0, sizeof(map)); + InitializeSharedInsertTurns(map); + map.reclaim_upto.value = -1; + for (uint32_t index = 0; index < kMapCapacity; ++index) { + map.slots[index].seq.value = -1; + } + for (uint32_t task = 0; task < kMaxTasks; ++task) { + for (uint32_t slot = 0; slot < kSharedOutputMaxPerTask; ++slot) { + map.shared_outputs[task].published[slot].value = -1; + map.shared_outputs[task].last_writer[slot].value = -1; + } + } + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + map.reader_done[worker].value = -1; + } +} + +SchedulerState *MapSparseSchedulerState() { + int flags = MAP_PRIVATE | MAP_ANONYMOUS; +#ifdef MAP_NORESERVE + flags |= MAP_NORESERVE; +#endif + void *memory = mmap( + nullptr, sizeof(SchedulerState), PROT_READ | PROT_WRITE, + flags, -1, 0 + ); + if (memory == MAP_FAILED) { + std::perror("mmap SchedulerState"); + return nullptr; + } + // SchedulerState 是 trivial 类型；匿名映射提供零页，default-init + // 只建立对象生命周期，避免为定向测试提交约 1 GiB 无关物理页。 + return ::new (memory) SchedulerState; +} + +void UnmapSparseSchedulerState(SchedulerState *state) { + if (state != nullptr) { + (void)munmap(state, sizeof(SchedulerState)); + } +} + +void TestSharedCompletionPublishesWithoutFrontier() { + SchedulerState *state = MapSparseSchedulerState(); + if (state == nullptr) { + ++g_failures; + return; + } + state->frontier.value = -1; + WorkerState &worker = state->workers[0]; + worker.heap_next = 4096; + LocalStats stats{}; + + CompleteTask(state, worker, 0, stats); + + Check( + state->tasks[0].vend == worker.heap_next && + state->tasks[0].flag == 1, + "shared completion publishes both vend and ready flag" + ); + Check( + state->frontier.value == -1, + "shared no-wrap completion leaves frontier at its initial value" + ); + Check( + stats.result.frontier_initial_loads == 0 && + stats.result.frontier_updates == 0 && + stats.result.frontier_terminal_loads == 0, + "shared no-wrap completion performs no frontier helping" + ); + Check( + stats.result.cas_retries == 0, + "shared completion adds no hidden frontier CAS retries" + ); + UnmapSparseSchedulerState(state); +} + +TensorDesc MakeTensor(uint64_t address, uint32_t owner) { + TensorDesc tensor{}; + tensor.buffer_addr = address; + tensor.buffer_size = 4096; + tensor.owner_task_id = owner; + tensor.ndims = 1; + tensor.dtype = DataType::Float32; + tensor.is_contiguous = true; + tensor.shapes[0] = 1024; + tensor.strides[0] = 1; + tensor.extent_elem_cache = 1024; + return tensor; +} + +bool SameTensor(const TensorDesc &left, const TensorDesc &right) { + return std::memcmp(&left, &right, sizeof(TensorDesc)) == 0; +} + +bool AllBytesEqual(const void *object, size_t size, unsigned char expected) { + const unsigned char *bytes = + reinterpret_cast(object); + for (size_t index = 0; index < size; ++index) { + if (bytes[index] != expected) { + return false; + } + } + return true; +} + +void TestPublishAndResolve() { + auto map = std::make_unique(); + ResetSharedState(*map); + SymbolTestOps::now_calls.store(0, std::memory_order_relaxed); + + TensorDesc first = MakeTensor(0x100000000ULL, 0); + TensorDesc second = MakeTensor(0x100001000ULL, 0); + SubmitContext producer{}; + producer.task_id = 0; + producer.result.task_id = 0; + producer.result.count = 2; + producer.result.tensors[0] = &first; + producer.result.tensors[1] = &second; + producer.shared_result.Reset(0); + Check(producer.shared_result.AddOutputRef(0, 0), "producer accepts output slot 0"); + Check(producer.shared_result.AddOutputRef(0, 1), "producer accepts output slot 1"); + + Check( + PublishSharedTaskOutputs(*map, producer, 0), + "first publication succeeds" + ); + Check(map->shared_outputs[0].published[0].value == 0, "slot 0 is published"); + Check(map->shared_outputs[0].published[1].value == 0, "slot 1 is published"); + Check(map->shared_outputs[0].last_writer[0].value == 0, "slot 0 writer starts at producer"); + Check(SameTensor(map->shared_outputs[0].tensors[0], first), "published descriptor is exact"); + TensorDesc replacement = MakeTensor(0x200000000ULL, 0); + producer.result.tensors[0] = &replacement; + Check( + !PublishSharedTaskOutputs(*map, producer, 0), + "duplicate publication fails closed" + ); + Check( + SameTensor(map->shared_outputs[0].tensors[0], first), + "failed duplicate publication cannot replace descriptor" + ); + Check( + map->shared_outputs[0].published[0].value == 0 && + map->shared_outputs[0].last_writer[0].value == 0, + "failed duplicate publication preserves control state" + ); + producer.result.tensors[0] = &first; + + TaskArgs input_args; + ConstructTaskArgs(input_args); + AppendSharedOutputRef( + input_args, producer.shared_result.OutputRef(0), TensorArgType::Input + ); + auto input_payload = std::make_unique(); + std::memset(input_payload.get(), 0xA5, sizeof(*input_payload)); + SubmitContext input_context{}; + input_context.task_id = 1; + input_context.payload = input_payload.get(); + input_context.tensor_count = input_args.tensor_count; + input_context.scalar_count = input_args.scalar_count; + LocalStats input_stats{}; + int32_t input_fanin[kMaxFanin] = {}; + bool protocol_ok = false; + uint32_t ordinary_lookups = UINT32_MAX; + const uint32_t input_count = CollectSharedFanin( + *map, input_args, 1, kHeapWindow, input_stats, + input_fanin, protocol_ok, ordinary_lookups + ); + Check(protocol_ok, "plain symbolic INPUT resolves"); + Check(input_count == 1 && input_fanin[0] == 0, "INPUT depends on producer writer"); + Check(ordinary_lookups == 0, "symbol INPUT never enters region map"); + Check( + input_stats.result.shared_symbol_input_loads == 1, + "INPUT writer load is counted" + ); + Check( + AllBytesEqual(input_payload.get(), sizeof(*input_payload), 0xA5), + "resolver leaves task payload scratch untouched" + ); + LocalSlot input_slot{}; + BuildSlotPayload( + input_slot, 1, static_cast(FunctionId(TaskKind::Qk)), 0, + input_args, input_context, input_fanin, input_count, *map + ); + Check( + SameTensor(input_slot.tensors[0], first), + "validated INPUT descriptor lands directly in LocalSlot" + ); + Check( + input_slot.args[0] == + static_cast( + reinterpret_cast(&input_slot.tensors[0]) + ), + "direct INPUT slot descriptor is wired into dispatch args" + ); + input_payload->tensors[0] = first; + LocalSlot compatibility_slot{}; + BuildSlotPayload( + compatibility_slot, input_args, input_context, input_fanin, + input_count + ); + Check( + SameTensor(compatibility_slot.tensors[0], first), + "legacy prefilled-payload builder contract remains available" + ); + + TaskArgs inout_args; + ConstructTaskArgs(inout_args); + AppendSharedOutputRef( + inout_args, producer.shared_result.OutputRef(0), TensorArgType::Inout + ); + auto inout_payload = std::make_unique(); + std::memset(inout_payload.get(), 0x5A, sizeof(*inout_payload)); + SubmitContext inout_context{}; + inout_context.task_id = 2; + inout_context.payload = inout_payload.get(); + inout_context.tensor_count = inout_args.tensor_count; + inout_context.scalar_count = inout_args.scalar_count; + LocalStats inout_stats{}; + int32_t inout_fanin[kMaxFanin] = {}; + protocol_ok = false; + ordinary_lookups = UINT32_MAX; + const uint32_t inout_count = CollectSharedFanin( + *map, inout_args, 2, kHeapWindow, inout_stats, + inout_fanin, protocol_ok, ordinary_lookups + ); + Check(protocol_ok, "plain symbolic INOUT resolves"); + Check(inout_count == 1 && inout_fanin[0] == 0, "INOUT consumes old writer"); + Check( + map->shared_outputs[0].last_writer[0].value == 0, + "read-only INOUT resolve keeps producer writer unchanged" + ); + Check( + inout_stats.result.shared_symbol_inout_commits == 0, + "read-only INOUT resolve publishes no writer statistic" + ); + Check( + AllBytesEqual(inout_payload.get(), sizeof(*inout_payload), 0x5A), + "INOUT resolver also leaves task payload scratch untouched" + ); + LocalSlot inout_slot{}; + BuildSlotPayload( + inout_slot, 2, static_cast(FunctionId(TaskKind::Sf)), 0, + inout_args, inout_context, inout_fanin, inout_count, *map + ); + Check( + SameTensor(inout_slot.tensors[0], first), + "validated INOUT descriptor lands directly in LocalSlot" + ); + Check( + CommitSharedFaninWriters( + *map, inout_args, 2, inout_stats + ), + "explicit post-build step commits the INOUT writer" + ); + Check( + map->shared_outputs[0].last_writer[0].value == 2, + "INOUT commit advances writer to current task" + ); + Check( + inout_stats.result.shared_symbol_inout_commits == 1, + "successful INOUT writer commit is counted" + ); + + // descriptor identity 仍指向最初 producer，但后继必须依赖最近一次 + // 已提交的 INOUT writer。writer-ready gate 负责阻止 loser 在该提交 + // 之前进入后继；resolver 本身据 last_writer 返回精确的新依赖。 + TaskArgs successor_args; + ConstructTaskArgs(successor_args); + AppendSharedOutputRef( + successor_args, producer.shared_result.OutputRef(0), + TensorArgType::Input + ); + LocalStats successor_stats{}; + int32_t successor_fanin[kMaxFanin] = {}; + protocol_ok = false; + ordinary_lookups = UINT32_MAX; + const uint32_t successor_count = CollectSharedFanin( + *map, successor_args, 3, kHeapWindow, + successor_stats, successor_fanin, protocol_ok, ordinary_lookups, + nullptr, 0, 2 + ); + Check(protocol_ok, "post-INOUT successor resolves the same shared descriptor"); + Check( + successor_count == 1 && successor_fanin[0] == 2, + "post-INOUT successor depends on the latest writer" + ); + Check( + SymbolTestOps::now_calls.load(std::memory_order_relaxed) == 0, + "already-published symbol fast path never reads the watchdog clock" + ); +} + +void TestPaTwoGroupWriterReadyGate() { + SchedulerState *state = MapSparseSchedulerState(); + if (state == nullptr) { + ++g_failures; + return; + } + ResetSharedState(state->shared_map); + state->fatal.value = 0; + constexpr int32_t alloc = 0; + constexpr int32_t first_sf = 2; + constexpr int32_t first_pv = 3; + constexpr int32_t first_up = 4; + constexpr int32_t second_sf = 6; + constexpr int32_t second_pv = 7; + constexpr int32_t second_up = 8; + state->tasks[first_up].deps_prepared = -1; + + const auto OutputRef = [](int32_t producer, int16_t slot) { + return FdwicOutputRef{producer, slot, 0, 0, 0, 0}; + }; + const auto SeedOutput = [&](int32_t producer, int16_t slot, uint64_t address) { + SharedOutputCell &cell = + state->shared_map.shared_outputs[ + static_cast(producer) + ]; + cell.published[static_cast(slot)].value = producer; + cell.last_writer[static_cast(slot)].value = producer; + cell.tensors[static_cast(slot)] = + MakeTensor(address, static_cast(producer)); + }; + SeedOutput(alloc, 0, 0x310000000ULL); + SeedOutput(alloc, 1, 0x310001000ULL); + SeedOutput(alloc, 2, 0x310002000ULL); + SeedOutput(first_sf, 1, 0x320001000ULL); + SeedOutput(first_sf, 2, 0x320002000ULL); + SeedOutput(first_pv, 0, 0x330000000ULL); + SeedOutput(second_sf, 1, 0x340001000ULL); + SeedOutput(second_sf, 2, 0x340002000ULL); + SeedOutput(second_pv, 0, 0x350000000ULL); + + PaOrchestrationState first_orchestration{}; + InitPaOrchestration(first_orchestration, 1, nullptr); + first_orchestration.current_batch = 0; + first_orchestration.current_sequence = + 2ULL * kPaBlocksPerRequest * kPaBlockSize; + first_orchestration.current_blocks = 2ULL * kPaBlocksPerRequest; + PreparePaBlockGroup(first_orchestration, 0); + first_orchestration.accumulated_output = OutputRef(alloc, 0); + first_orchestration.accumulated_sum = OutputRef(alloc, 1); + first_orchestration.accumulated_max = OutputRef(alloc, 2); + first_orchestration.sf_max = OutputRef(first_sf, 1); + first_orchestration.sf_sum = OutputRef(first_sf, 2); + first_orchestration.pv_output = OutputRef(first_pv, 0); + + TaskArgs first_args; + LocalStats first_build_stats{}; + Check( + BuildCallbackSubmitArgs( + first_orchestration, first_args, 0, first_build_stats + ), + "first PA group builds the real UP argument shape" + ); + Check( + first_args.tensor_count == 7 && first_args.scalar_count == 2 && + TaskTag(first_args, 0) == TensorArgType::Input && + TaskTag(first_args, 1) == TensorArgType::Input && + TaskTag(first_args, 2) == TensorArgType::Input && + TaskTag(first_args, 3) == TensorArgType::Inout && + TaskTag(first_args, 4) == TensorArgType::Inout && + TaskTag(first_args, 5) == TensorArgType::Inout && + TaskTag(first_args, 6) == TensorArgType::Inout, + "UP args contain 3 fresh INPUTs, 3 accumulator INOUTs, and output view" + ); + Check( + first_args.scalars[0] == 1 && first_args.scalars[1] == 0, + "first PA group carries is_first=1 and is_last=0" + ); + + PaOrchestrationState second_orchestration = first_orchestration; + PreparePaBlockGroup(second_orchestration, kPaBlocksPerRequest); + second_orchestration.sf_max = OutputRef(second_sf, 1); + second_orchestration.sf_sum = OutputRef(second_sf, 2); + second_orchestration.pv_output = OutputRef(second_pv, 0); + TaskArgs second_args; + std::atomic waiter_finished{false}; + std::atomic second_build_started{false}; + bool waiter_ok = false; + bool second_build_ok = false; + bool second_protocol_ok = false; + uint32_t second_ordinary_lookups = UINT32_MAX; + uint32_t second_fanin_count = 0; + int32_t second_fanin[kMaxFanin] = {}; + LocalStats waiter_stats{}; + LocalStats second_build_stats{}; + LocalStats second_stats{}; + SymbolTestOps::wait_address = &state->tasks[first_up].deps_prepared; + SymbolTestOps::wait_loads.store(0, std::memory_order_relaxed); + std::thread loser([&]() { + waiter_ok = WaitForSharedWriterReady( + state, first_up, waiter_stats + ); + if (waiter_ok) { + second_build_started.store(true, std::memory_order_release); + second_build_ok = BuildCallbackSubmitArgs( + second_orchestration, second_args, 0, second_build_stats + ); + if (second_build_ok) { + second_fanin_count = + CollectSharedFanin( + state->shared_map, second_args, second_up, + kHeapWindow, second_stats, second_fanin, + second_protocol_ok, second_ordinary_lookups, + nullptr, alloc, first_up + ); + } + } + waiter_finished.store(true, std::memory_order_release); + }); + while (SymbolTestOps::wait_loads.load(std::memory_order_acquire) == 0) { + std::this_thread::yield(); + } + for (uint32_t spin = 0; spin < 1024; ++spin) { + std::this_thread::yield(); + } + Check( + !waiter_finished.load(std::memory_order_acquire) && + !second_build_started.load(std::memory_order_acquire), + "UP loser cannot build the next group before writer intent is ready" + ); + + LocalStats first_stats{}; + SubmitContext first_context{}; + first_context.task_id = first_up; + first_context.won = true; + Check( + PreparePaSharedWriterIntent( + state, first_args, first_context, first_stats + ), + "first non-final UP prepares writer intent before winner Build" + ); + Check( + first_context.fanin_count == 3 && + first_context.fanin[0] == first_sf && + first_context.fanin[1] == first_pv && + first_context.fanin[2] == alloc && + first_stats.result.map_lookups == 0, + "first UP intent resolves SF/PV/Alloc and excludes manual output view" + ); + loser.join(); + SymbolTestOps::wait_address = nullptr; + Check( + waiter_ok && waiter_finished.load(std::memory_order_acquire) && + second_build_started.load(std::memory_order_acquire), + "UP loser observes the exact gate and then builds the second group" + ); + Check( + state->tasks[first_up].flag == 0, + "writer-ready does not impersonate the first UP completion flag" + ); + Check( + second_build_ok && second_args.tensor_count == 7 && + second_args.scalar_count == 2 && + second_args.scalars[0] == 0 && + second_args.scalars[1] == 1, + "second PA group builds real UP args with is_first=0 and is_last=1" + ); + Check( + second_protocol_ok && second_fanin_count == 3 && + second_fanin[0] == second_sf && + second_fanin[1] == second_pv && + second_fanin[2] == first_up, + "second UP keeps fresh SF/PV producers and uses first UP for accumulators" + ); + Check( + second_ordinary_lookups == 0 && + first_stats.result.shared_symbol_input_loads == 3 && + second_stats.result.shared_symbol_input_loads == 3, + "both UP intents resolve three symbolic INPUTs and no ordinary region" + ); + Check( + !PublishSharedWriterReady(state, first_up), + "duplicate writer-ready publication fails closed" + ); + constexpr int32_t wrong_gate_task = 5; + state->tasks[wrong_gate_task].deps_prepared = first_up; + Check( + !PublishSharedWriterReady( + state, wrong_gate_task + ) && + state->tasks[wrong_gate_task].deps_prepared == first_up, + "writer-ready CAS mismatch preserves the competing gate value" + ); + LocalStats wrong_gate_stats{}; + Check( + !WaitForSharedWriterReady( + state, wrong_gate_task, wrong_gate_stats + ) && state->fatal.value == 1, + "writer-ready wait rejects a different task id and broadcasts fatal" + ); + state->fatal.value = 0; + constexpr int32_t timeout_gate_task = 6; + state->tasks[timeout_gate_task].deps_prepared = -1; + ExpiredWaitOps::calls.store(0, std::memory_order_relaxed); + LocalStats timeout_gate_stats{}; + Check( + !WaitForSharedWriterReady( + state, timeout_gate_task, timeout_gate_stats + ) && state->fatal.value == 1, + "writer-ready wait times out and broadcasts fatal" + ); + state->fatal.value = 0; + + Check( + CommitSharedFaninWriters( + state->shared_map, second_args, second_up, second_stats, + alloc, first_up + ), + "second UP commits over the first UP writer" + ); + Check( + state->shared_map.shared_outputs[0].last_writer[0].value == second_up && + state->shared_map.shared_outputs[0].last_writer[1].value == second_up && + state->shared_map.shared_outputs[0].last_writer[2].value == second_up, + "all accumulator writers finish at the second UP" + ); + Check( + first_stats.result.shared_symbol_inout_commits == 3 && + second_stats.result.shared_symbol_inout_commits == 3, + "two PA groups register exactly three shared writers each" + ); + + LocalStats missing_selector_stats{}; + int32_t missing_selector_fanin[kMaxFanin] = {}; + bool missing_selector_ok = true; + uint32_t missing_selector_lookups = UINT32_MAX; + Check( + CollectSharedFanin( + state->shared_map, second_args, 12, kHeapWindow, + missing_selector_stats, missing_selector_fanin, + missing_selector_ok, missing_selector_lookups, + nullptr, 1, second_up + ) == 0 && !missing_selector_ok, + "chained resolver rejects a producer selector that matches no ref" + ); + Check( + !CommitSharedFaninWriters( + state->shared_map, second_args, 12, + missing_selector_stats, 1, second_up + ), + "chained commit rejects a selector with no writable match" + ); + LocalStats unchanged_writer_stats{}; + int32_t unchanged_writer_fanin[kMaxFanin] = {}; + bool unchanged_writer_ok = true; + uint32_t unchanged_writer_lookups = UINT32_MAX; + Check( + CollectSharedFanin( + state->shared_map, second_args, second_up, kHeapWindow, + unchanged_writer_stats, unchanged_writer_fanin, + unchanged_writer_ok, unchanged_writer_lookups, + nullptr, alloc, alloc + ) == 0 && !unchanged_writer_ok, + "chained resolver rejects producer==writer as a non-chain" + ); + Check( + !CommitSharedFaninWriters( + state->shared_map, second_args, 12, + unchanged_writer_stats, alloc, alloc + ), + "chained commit also rejects producer==writer before mutation" + ); + LocalStats invalid_expected_stats{}; + Check( + !CommitSharedFaninWriters( + state->shared_map, second_args, 12, invalid_expected_stats, + alloc, -1 + ), + "chained commit rejects an invalid expected writer before mutation" + ); + Check( + state->shared_map.shared_outputs[0].last_writer[0].value == second_up && + state->shared_map.shared_outputs[0].last_writer[1].value == second_up && + state->shared_map.shared_outputs[0].last_writer[2].value == second_up, + "invalid expected writer leaves every accumulator unchanged" + ); + + // 该测试使用真实 PA 的 task-id 顺序和 UP 参数构造，但默认主循环仍 + // 固定每 batch 五 task，GetTaskKind(8) 尚不会返回 UP。这里明确只 + // 证明双组 orchestration 参数与 writer-intent 原语，不冒充完整 replay + // 已接通。故意回退 writer，确认不能跳过前一 UP。 + for (uint32_t slot = 0; slot < 3; ++slot) { + state->shared_map.shared_outputs[alloc] + .last_writer[slot].value = first_up - 1; + } + LocalStats stale_stats{}; + int32_t stale_fanin[kMaxFanin] = {}; + bool stale_protocol_ok = true; + uint32_t stale_ordinary_lookups = UINT32_MAX; + Check( + CollectSharedFanin( + state->shared_map, second_args, second_up, kHeapWindow, + stale_stats, stale_fanin, stale_protocol_ok, + stale_ordinary_lookups, nullptr, alloc, first_up + ) == 0 && !stale_protocol_ok, + "chained resolver rejects a stale writer instead of skipping a stage" + ); + + UnmapSparseSchedulerState(state); +} + +void TestPaWriterIntentPreGateFailuresDoNotPublishGate() { + SchedulerState *state = MapSparseSchedulerState(); + if (state == nullptr) { + ++g_failures; + return; + } + ResetSharedState(state->shared_map); + state->fatal.value = 0; + state->heap_window = kHeapWindow; + constexpr int32_t producer = 0; + constexpr int32_t writer = 4; + state->tasks[writer].deps_prepared = -1; + + SharedOutputCell &cell = state->shared_map.shared_outputs[producer]; + TaskArgs damaged_writer_args; + ConstructTaskArgs(damaged_writer_args); + for (uint32_t slot = 0; slot < 3; ++slot) { + cell.published[slot].value = producer; + cell.last_writer[slot].value = producer; + cell.tensors[slot] = + MakeTensor(0x360000000ULL + slot * 0x1000ULL, producer); + AppendSharedOutputRef( + damaged_writer_args, + FdwicOutputRef{ + producer, static_cast(slot), 0, 0, 0, 0, + }, + TensorArgType::Inout + ); + } + TensorDesc manual_view = MakeTensor(0x360010000ULL, producer); + manual_view.manual_dep = true; + AddLocalTensor( + damaged_writer_args, manual_view, TensorArgType::Inout + ); + cell.last_writer[2].value = -1; + SubmitContext damaged_writer_context{}; + damaged_writer_context.task_id = writer; + damaged_writer_context.won = true; + LocalStats damaged_writer_stats{}; + Check( + !PreparePaSharedWriterIntent( + state, damaged_writer_args, damaged_writer_context, + damaged_writer_stats + ) && + state->fatal.value == 1 && + state->tasks[writer].deps_prepared == -1 && + cell.last_writer[0].value == producer && + cell.last_writer[1].value == producer && + cell.last_writer[2].value == -1, + "read-only intent validation failure changes no writer and publishes no gate" + ); + + state->fatal.value = 0; + state->tasks[writer].deps_prepared = -1; + cell.last_writer[2].value = producer; + PublicationFaultOps::fetch_race_address = &cell.last_writer[1].value; + SubmitContext partial_commit_context{}; + partial_commit_context.task_id = writer; + partial_commit_context.won = true; + LocalStats partial_commit_stats{}; + Check( + !PreparePaSharedWriterIntent( + state, damaged_writer_args, partial_commit_context, + partial_commit_stats + ) && + state->fatal.value == 1 && + state->tasks[writer].deps_prepared == -1 && + cell.last_writer[0].value == writer && + cell.last_writer[1].value == writer && + cell.last_writer[2].value == producer, + "partial writer commit keeps terminal prefix evidence but publishes no gate" + ); + PublicationFaultOps::fetch_race_address = nullptr; + + state->fatal.value = 0; + state->tasks[writer].deps_prepared = -1; + cell.last_writer[0].value = producer; + cell.last_writer[1].value = producer; + cell.last_writer[2].value = producer; + TaskArgs missing_accumulator_args = damaged_writer_args; + missing_accumulator_args.tensor_count = 2; + SubmitContext missing_accumulator_context{}; + missing_accumulator_context.task_id = writer; + missing_accumulator_context.won = true; + LocalStats missing_accumulator_stats{}; + Check( + !PreparePaSharedWriterIntent( + state, missing_accumulator_args, missing_accumulator_context, + missing_accumulator_stats + ) && + state->fatal.value == 1 && + state->tasks[writer].deps_prepared == -1 && + cell.last_writer[0].value == producer && + cell.last_writer[1].value == producer && + cell.last_writer[2].value == producer, + "missing PA accumulator rejects the fast path before mutation or gate" + ); + + state->fatal.value = 0; + state->tasks[writer].deps_prepared = -1; + TaskArgs wrong_accumulator_args = damaged_writer_args; + wrong_accumulator_args.tensors[0] + .pointer.output_ref.producer_task_id = 1; + SubmitContext wrong_accumulator_context{}; + wrong_accumulator_context.task_id = writer; + wrong_accumulator_context.won = true; + LocalStats wrong_accumulator_stats{}; + Check( + !PreparePaSharedWriterIntent( + state, wrong_accumulator_args, wrong_accumulator_context, + wrong_accumulator_stats + ) && + state->fatal.value == 1 && + state->tasks[writer].deps_prepared == -1 && + cell.last_writer[0].value == producer && + cell.last_writer[1].value == producer && + cell.last_writer[2].value == producer, + "PA writer intent rejects three refs that are not the batch accumulators" + ); + + state->fatal.value = 0; + state->tasks[writer].deps_prepared = -1; + TaskArgs duplicate_accumulator_args = damaged_writer_args; + duplicate_accumulator_args.tensors[2] + .pointer.output_ref.output_slot = 1; + SubmitContext duplicate_accumulator_context{}; + duplicate_accumulator_context.task_id = writer; + duplicate_accumulator_context.won = true; + LocalStats duplicate_accumulator_stats{}; + Check( + !PreparePaSharedWriterIntent( + state, duplicate_accumulator_args, + duplicate_accumulator_context, + duplicate_accumulator_stats + ) && + state->fatal.value == 1 && + state->tasks[writer].deps_prepared == -1 && + cell.last_writer[0].value == producer && + cell.last_writer[1].value == producer && + cell.last_writer[2].value == producer, + "PA writer intent rejects duplicate accumulator slots before mutation" + ); + + state->fatal.value = 0; + state->tasks[writer].deps_prepared = -1; + TaskArgs missing_view_args = damaged_writer_args; + missing_view_args.tensor_count = 3; + SubmitContext missing_view_context{}; + missing_view_context.task_id = writer; + missing_view_context.won = true; + LocalStats missing_view_stats{}; + Check( + !PreparePaSharedWriterIntent( + state, missing_view_args, missing_view_context, + missing_view_stats + ) && + state->fatal.value == 1 && + state->tasks[writer].deps_prepared == -1 && + cell.last_writer[0].value == producer && + cell.last_writer[1].value == producer && + cell.last_writer[2].value == producer, + "PA writer intent rejects a missing manual-dependency output view" + ); + + state->fatal.value = 0; + state->tasks[writer].deps_prepared = -1; + TaskArgs duplicate_view_args = damaged_writer_args; + AddLocalTensor( + duplicate_view_args, manual_view, TensorArgType::Inout + ); + SubmitContext duplicate_view_context{}; + duplicate_view_context.task_id = writer; + duplicate_view_context.won = true; + LocalStats duplicate_view_stats{}; + Check( + !PreparePaSharedWriterIntent( + state, duplicate_view_args, duplicate_view_context, + duplicate_view_stats + ) && + state->fatal.value == 1 && + state->tasks[writer].deps_prepared == -1 && + cell.last_writer[0].value == producer && + cell.last_writer[1].value == producer && + cell.last_writer[2].value == producer, + "PA writer intent rejects duplicate manual-dependency writers" + ); + + state->fatal.value = 0; + state->tasks[writer].deps_prepared = -1; + TaskArgs errored_args = damaged_writer_args; + errored_args.has_error = true; + SubmitContext errored_context{}; + errored_context.task_id = writer; + errored_context.won = true; + LocalStats errored_stats{}; + Check( + !PreparePaSharedWriterIntent( + state, errored_args, errored_context, errored_stats + ) && + state->fatal.value == 1 && + state->tasks[writer].deps_prepared == -1 && + cell.last_writer[0].value == producer && + cell.last_writer[1].value == producer && + cell.last_writer[2].value == producer, + "errored frontend args are rejected before writer mutation or gate" + ); + + state->fatal.value = 0; + state->tasks[writer].deps_prepared = -1; + TaskArgs invalid_scalar_args = damaged_writer_args; + invalid_scalar_args.scalar_count = + static_cast(kMaxTaskScalars) + 1; + SubmitContext invalid_scalar_context{}; + invalid_scalar_context.task_id = writer; + invalid_scalar_context.won = true; + LocalStats invalid_scalar_stats{}; + Check( + !PreparePaSharedWriterIntent( + state, invalid_scalar_args, invalid_scalar_context, + invalid_scalar_stats + ) && + state->fatal.value == 1 && + state->tasks[writer].deps_prepared == -1 && + cell.last_writer[0].value == producer && + cell.last_writer[1].value == producer && + cell.last_writer[2].value == producer, + "invalid scalar count is rejected before writer mutation or gate" + ); + + state->fatal.value = 0; + state->tasks[writer].deps_prepared = -1; + TensorDesc ordinary_writer = MakeTensor(0x370000000ULL, producer); + ordinary_writer.manual_dep = false; + TaskArgs ordinary_writer_args = damaged_writer_args; + AddLocalTensor( + ordinary_writer_args, ordinary_writer, TensorArgType::Inout + ); + SubmitContext ordinary_writer_context{}; + ordinary_writer_context.task_id = writer; + ordinary_writer_context.won = true; + LocalStats ordinary_writer_stats{}; + Check( + !PreparePaSharedWriterIntent( + state, ordinary_writer_args, ordinary_writer_context, + ordinary_writer_stats + ) && + state->fatal.value == 1 && + state->tasks[writer].deps_prepared == -1 && + cell.last_writer[0].value == producer && + cell.last_writer[1].value == producer && + cell.last_writer[2].value == producer, + "PA fast path rejects ordinary region writers before mutation or gate" + ); + + UnmapSparseSchedulerState(state); +} + +void TestWriterCommitFailuresKeepTerminalEvidence() { + auto map = std::make_unique(); + ResetSharedState(*map); + + TaskArgs args; + ConstructTaskArgs(args); + AppendSharedOutputRef( + args, FdwicOutputRef{0, 0, 0, 0, 0, 0}, + TensorArgType::Inout + ); + + LocalStats stats{}; + map->shared_outputs[0].last_writer[0].value = -1; + Check( + !CommitSharedFaninWriters(*map, args, 4, stats), + "missing producer writer rejects INOUT commit" + ); + Check( + map->shared_outputs[0].last_writer[0].value == 4, + "failed FetchMax keeps terminal over-advance evidence instead of rolling back" + ); + Check( + stats.result.shared_symbol_inout_commits == 0, + "failed writer commit is not counted as success" + ); + + map->shared_outputs[0].last_writer[0].value = 7; + Check( + !CommitSharedFaninWriters(*map, args, 4, stats), + "future writer rejects INOUT commit" + ); + Check( + map->shared_outputs[0].last_writer[0].value == 7, + "failed FetchMax does not overwrite a later writer" + ); +} + +void TestMultiWriterFailureKeepsPartialTerminalEvidence() { + auto map = std::make_unique(); + ResetSharedState(*map); + TaskArgs args; + ConstructTaskArgs(args); + for (int32_t producer = 0; producer < 3; ++producer) { + AppendSharedOutputRef( + args, FdwicOutputRef{producer, 0, 0, 0, 0, 0}, + TensorArgType::Inout + ); + map->shared_outputs[static_cast(producer)] + .last_writer[0].value = producer; + } + // 模拟只读解析完成后第二个 producer 控制字被破坏。第一个提交已经 + // 线性化，不能为了伪造事务性而回滚；第三个尚未触碰。 + map->shared_outputs[1].last_writer[0].value = -1; + LocalStats stats{}; + Check( + !CommitSharedFaninWriters(*map, args, 4, stats), + "second of three INOUT commits rejects damaged producer writer" + ); + Check( + map->shared_outputs[0].last_writer[0].value == 4 && + map->shared_outputs[1].last_writer[0].value == 4 && + map->shared_outputs[2].last_writer[0].value == 2, + "partial terminal commit preserves completed, failed, and untouched evidence" + ); + Check( + stats.result.shared_symbol_inout_commits == 1, + "only the writer committed before terminal failure is counted" + ); +} + +void TestFailedSealDiscardsBuiltTask() { + auto worker = std::make_unique(); + worker->occupied_count = 2; + worker->slots[0].task_id = 4; + worker->slots[0].occupied = true; + worker->slots[0].built = true; + worker->slots[1].task_id = 3; + worker->slots[1].occupied = true; + worker->slots[1].built = true; + + Check( + DiscardBuiltTask(*worker, 4), + "failed shared seal finds its just-built private slot" + ); + Check( + !worker->slots[0].occupied && !worker->slots[0].built && + worker->occupied_count == 1, + "failed shared seal removes only its own slot from FinalDrain" + ); + Check( + worker->slots[1].occupied && worker->slots[1].built, + "failed shared seal preserves previously built work" + ); + Check( + !DiscardBuiltTask(*worker, 9) && worker->occupied_count == 1, + "missing failed task cannot corrupt occupied accounting" + ); + worker->slots[0].task_id = 5; + worker->slots[0].occupied = true; + worker->slots[0].built = true; + worker->occupied_count = 0; + Check( + !DiscardBuiltTask(*worker, 5), + "corrupt occupied accounting remains visible to the caller" + ); + Check( + !worker->slots[0].occupied && !worker->slots[0].built, + "terminal discard clears the failed slot despite corrupt accounting" + ); +} + +void TestCase1RegistrationBypassesRegionSequencer() { + SubmitContext context{}; + + TaskArgs shared_args; + ConstructTaskArgs(shared_args); + AppendSharedOutputRef( + shared_args, FdwicOutputRef{0, 0, 0, 0, 0, 0}, + TensorArgType::Inout + ); + context.register_mask = 1; + Check( + ValidateEmptySharedRegistration(shared_args, context), + "shared symbol writer bypasses ordinary region registration" + ); + + TensorDesc manual = MakeTensor(0x360000000ULL, 0); + manual.manual_dep = true; + TaskArgs manual_args; + ConstructTaskArgs(manual_args); + AddLocalTensor(manual_args, manual, TensorArgType::Inout); + context.register_mask = 1; + Check( + ValidateEmptySharedRegistration(manual_args, context), + "manual-dependency local writer leaves the region delta empty" + ); + + TaskArgs manual_gm_args; + ConstructTaskArgs(manual_gm_args); + AddGmTensor(manual_gm_args, manual, TensorArgType::Inout); + context.register_mask = 1; + Check( + ValidateEmptySharedRegistration(manual_gm_args, context), + "manual-dependency GM writer leaves the region delta empty" + ); + + TensorDesc ordinary = manual; + ordinary.manual_dep = false; + TaskArgs ordinary_args; + ConstructTaskArgs(ordinary_args); + AddLocalTensor(ordinary_args, ordinary, TensorArgType::Inout); + context.register_mask = 1; + Check( + !ValidateEmptySharedRegistration(ordinary_args, context), + "non-manual ordinary writer fails closed instead of entering the ring" + ); + + TaskArgs ordinary_gm_args; + ConstructTaskArgs(ordinary_gm_args); + AddGmTensor(ordinary_gm_args, ordinary, TensorArgType::Inout); + context.register_mask = 1; + Check( + !ValidateEmptySharedRegistration(ordinary_gm_args, context), + "non-manual ordinary GM writer fails closed before region append" + ); + + context.register_mask = 2; + Check( + !ValidateEmptySharedRegistration(shared_args, context), + "registration mask outside active arguments fails closed" + ); +} + +void TestPostBuildSealClosesSuccessAndFailurePaths() { + { + SchedulerState *state = MapSparseSchedulerState(); + Check(state != nullptr, "writer-failure seal state maps"); + if (state != nullptr) { + ResetSharedState(state->shared_map); + state->shared_map.committed_tasks.value = 4; + WorkerState &worker = state->workers[0]; + worker.occupied_count = 1; + worker.slots[0].task_id = 4; + worker.slots[0].occupied = true; + worker.slots[0].built = true; + + TaskArgs args; + ConstructTaskArgs(args); + AppendSharedOutputRef( + args, FdwicOutputRef{0, 0, 0, 0, 0, 0}, + TensorArgType::Inout + ); + SubmitContext context{}; + context.task_id = 4; + context.result.task_id = 4; + context.shared_result.Reset(4); + LocalStats stats{}; + + Check( + !PublishSharedWinnerAfterBuild( + state, worker, args, context, 4, TaskKind::Up, stats + ), + "writer invariant failure rejects post-build seal" + ); + Check( + state->fatal.value == 1 && + state->shared_map.committed_tasks.value == 4, + "writer failure broadcasts fatal without touching region sequencer" + ); + Check( + state->shared_map.shared_outputs[0] + .last_writer[0].value == 4, + "writer failure retains terminal FetchMax evidence" + ); + Check( + !worker.slots[0].occupied && + !worker.slots[0].built && + worker.occupied_count == 0, + "writer failure removes the failed winner from FinalDrain" + ); + UnmapSparseSchedulerState(state); + } + } + + { + SchedulerState *state = MapSparseSchedulerState(); + Check(state != nullptr, "writer-then-publication-failure state maps"); + if (state != nullptr) { + ResetSharedState(state->shared_map); + state->shared_map.committed_tasks.value = 37; + state->shared_map.shared_outputs[0] + .last_writer[0].value = 0; + WorkerState &worker = state->workers[0]; + worker.occupied_count = 1; + worker.slots[0].task_id = 4; + worker.slots[0].occupied = true; + worker.slots[0].built = true; + + TaskArgs args; + ConstructTaskArgs(args); + AppendSharedOutputRef( + args, FdwicOutputRef{0, 0, 0, 0, 0, 0}, + TensorArgType::Inout + ); + SubmitContext context{}; + context.task_id = 4; + context.result.task_id = 4; + TensorDesc output = MakeTensor(0x380000000ULL, 4); + context.result.count = 1; + context.result.tensors[0] = &output; + context.shared_result.Reset(4); + Check( + context.shared_result.AddOutputRef(4, 0), + "writer-publication-failure context declares output" + ); + LocalStats stats{}; + + PublicationFaultOps::exchange_race_address = + &state->shared_map.shared_outputs[4] + .published[0].value; + Check( + !PublishSharedWinnerAfterBuild( + state, worker, args, context, 4, TaskKind::Up, stats + ), + "publication failure after writer commit rejects post-build seal" + ); + Check( + PublicationFaultOps::exchange_race_address == nullptr && + state->fatal.value == 1, + "publication failure consumes injection and broadcasts fatal" + ); + Check( + state->shared_map.committed_tasks.value == 37 && + state->shared_map.reclaim_upto.value == -1, + "post-build seal leaves the unused region sequencer untouched" + ); + Check( + state->shared_map.shared_outputs[0] + .last_writer[0].value == 4 && + stats.result.shared_symbol_inout_commits == 1, + "publication failure retains the completed writer commit" + ); + Check( + state->shared_map.shared_outputs[4] + .published[0].value == -1 && + state->shared_map.shared_outputs[4] + .last_writer[0].value == -1, + "publication failure rolls back only the producer output cell" + ); + Check( + !worker.slots[0].occupied && + !worker.slots[0].built && + worker.occupied_count == 0, + "publication failure removes the failed winner slot" + ); + UnmapSparseSchedulerState(state); + } + } + + { + SchedulerState *state = MapSparseSchedulerState(); + Check(state != nullptr, "publication-failure seal state maps"); + if (state != nullptr) { + ResetSharedState(state->shared_map); + state->shared_map.committed_tasks.value = 19; + state->shared_map.shared_outputs[1] + .last_writer[0].value = 7; + WorkerState &worker = state->workers[0]; + worker.occupied_count = 1; + worker.slots[0].task_id = 1; + worker.slots[0].occupied = true; + worker.slots[0].built = true; + + TensorDesc output = MakeTensor(0x390000000ULL, 1); + TaskArgs args; + ConstructTaskArgs(args); + SubmitContext context{}; + context.task_id = 1; + context.result.task_id = 1; + context.result.count = 1; + context.result.tensors[0] = &output; + context.shared_result.Reset(1); + Check( + context.shared_result.AddOutputRef(1, 0), + "publication-failure context declares output" + ); + LocalStats stats{}; + + Check( + !PublishSharedWinnerAfterBuild( + state, worker, args, context, 1, TaskKind::Qk, stats + ), + "publication invariant failure rejects post-build seal" + ); + Check( + state->fatal.value == 1 && + state->shared_map.committed_tasks.value == 19, + "publication failure is terminal without a global turn" + ); + Check( + state->shared_map.shared_outputs[1] + .published[0].value == -1 && + state->shared_map.shared_outputs[1] + .last_writer[0].value == 7, + "publication preflight failure exposes no new descriptor" + ); + Check( + !worker.slots[0].occupied && + worker.occupied_count == 0, + "publication failure removes the failed winner slot" + ); + UnmapSparseSchedulerState(state); + } + } + + { + SchedulerState *state = MapSparseSchedulerState(); + Check(state != nullptr, "successful seal state maps"); + if (state != nullptr) { + ResetSharedState(state->shared_map); + state->shared_map.committed_tasks.value = 23; + WorkerState &worker = state->workers[0]; + worker.occupied_count = 1; + worker.slots[0].task_id = 1; + worker.slots[0].occupied = true; + worker.slots[0].built = true; + + TensorDesc output = MakeTensor(0x3A0000000ULL, 1); + TaskArgs args; + ConstructTaskArgs(args); + SubmitContext context{}; + context.task_id = 1; + context.result.task_id = 1; + context.result.count = 1; + context.result.tensors[0] = &output; + context.shared_result.Reset(1); + Check( + context.shared_result.AddOutputRef(1, 0), + "successful seal context declares output" + ); + LocalStats stats{}; + + SealOrderOps::output_writer_address = + &state->shared_map.shared_outputs[1] + .last_writer[0].value; + SealOrderOps::published_address = + &state->shared_map.shared_outputs[1] + .published[0].value; + SealOrderOps::expected_writer = 1; + SealOrderOps::order_ok = true; + Check( + PublishSharedWinnerAfterBuild( + state, worker, args, context, 1, TaskKind::Qk, stats + ), + "valid post-build seal succeeds" + ); + Check( + SealOrderOps::order_ok, + "output writer initialization precedes published" + ); + Check( + state->fatal.value == 0 && + state->shared_map.committed_tasks.value == 23 && + state->shared_map.shared_outputs[1] + .last_writer[0].value == 1 && + state->shared_map.shared_outputs[1] + .published[0].value == 1, + "successful seal publishes output without touching sequencer" + ); + Check( + SameTensor( + state->shared_map.shared_outputs[1].tensors[0], + output + ), + "successful seal publishes the exact descriptor" + ); + Check( + worker.slots[0].occupied && + worker.slots[0].built && + worker.occupied_count == 1, + "successful seal preserves executable winner slot" + ); + SealOrderOps::output_writer_address = nullptr; + SealOrderOps::published_address = nullptr; + UnmapSparseSchedulerState(state); + } + } +} + +void TestPublicationPreflightIsAllOrNothing() { + auto map = std::make_unique(); + ResetSharedState(*map); + + TensorDesc first = MakeTensor(0x300000000ULL, 0); + TensorDesc second = MakeTensor(0x300001000ULL, 0); + SubmitContext producer{}; + producer.task_id = 0; + producer.result.task_id = 0; + producer.result.count = 2; + producer.result.tensors[0] = &first; + producer.result.tensors[1] = &second; + producer.shared_result.Reset(0); + Check(producer.shared_result.AddOutputRef(0, 0), "preflight adds slot 0"); + Check(producer.shared_result.AddOutputRef(0, 1), "preflight adds slot 1"); + + // 人为污染第二槽，验证失败发生在任何 descriptor/前槽控制字写入前。 + map->shared_outputs[0].last_writer[1].value = 7; + const TensorDesc zero{}; + Check( + !PublishSharedTaskOutputs(*map, producer, 0), + "later occupied slot rejects whole publication" + ); + Check( + map->shared_outputs[0].published[0].value == -1 && + map->shared_outputs[0].last_writer[0].value == -1, + "later-slot failure leaves earlier control state untouched" + ); + Check( + SameTensor(map->shared_outputs[0].tensors[0], zero) && + SameTensor(map->shared_outputs[0].tensors[1], zero), + "later-slot failure leaves every descriptor untouched" + ); +} + +void TestPublicationCommitFaultsRollback() { + auto map = std::make_unique(); + TensorDesc first = MakeTensor(0x350000000ULL, 0); + TensorDesc second = MakeTensor(0x350001000ULL, 0); + SubmitContext producer{}; + producer.task_id = 0; + producer.result.task_id = 0; + producer.result.count = 2; + producer.result.tensors[0] = &first; + producer.result.tensors[1] = &second; + producer.shared_result.Reset(0); + Check(producer.shared_result.AddOutputRef(0, 0), "fault test adds slot 0"); + Check(producer.shared_result.AddOutputRef(0, 1), "fault test adds slot 1"); + const TensorDesc zero{}; + + ResetSharedState(*map); + PublicationFaultOps::fetch_race_address = + &map->shared_outputs[0].last_writer[1].value; + Check( + !PublishSharedTaskOutputs(*map, producer, 0), + "FetchMax race rejects whole publication" + ); + Check( + map->shared_outputs[0].last_writer[0].value == -1 && + map->shared_outputs[0].last_writer[1].value == -2, + "FetchMax failure restores reserved and raced writer values" + ); + Check( + map->shared_outputs[0].published[0].value == -1 && + map->shared_outputs[0].published[1].value == -1 && + SameTensor(map->shared_outputs[0].tensors[0], zero) && + SameTensor(map->shared_outputs[0].tensors[1], zero), + "FetchMax failure publishes no descriptor" + ); + + ResetSharedState(*map); + PublicationFaultOps::exchange_race_address = + &map->shared_outputs[0].published[1].value; + Check( + !PublishSharedTaskOutputs(*map, producer, 0), + "published Exchange race rejects whole publication" + ); + Check( + map->shared_outputs[0].published[0].value == -1 && + map->shared_outputs[0].published[1].value == -1 && + map->shared_outputs[0].last_writer[0].value == -1 && + map->shared_outputs[0].last_writer[1].value == -1, + "published Exchange failure rolls back every control word" + ); + Check( + SameTensor(map->shared_outputs[0].tensors[0], zero) && + SameTensor(map->shared_outputs[0].tensors[1], zero), + "published Exchange failure clears flushed descriptors" + ); +} + +void TestConsumerWaitsForDelayedPublication() { + auto map = std::make_unique(); + ResetSharedState(*map); + + TensorDesc output = MakeTensor(0x380000000ULL, 0); + SubmitContext producer{}; + producer.task_id = 0; + producer.result.task_id = 0; + producer.result.count = 1; + producer.result.tensors[0] = &output; + producer.shared_result.Reset(0); + Check( + producer.shared_result.AddOutputRef(0, 0), + "delayed producer accepts output slot" + ); + + TaskArgs args; + ConstructTaskArgs(args); + AppendSharedOutputRef( + args, producer.shared_result.OutputRef(0), TensorArgType::Input + ); + auto payload = std::make_unique(); + std::memset(payload.get(), 0x3C, sizeof(*payload)); + SubmitContext consumer{}; + consumer.task_id = 1; + consumer.payload = payload.get(); + consumer.tensor_count = args.tensor_count; + consumer.scalar_count = args.scalar_count; + LocalStats stats{}; + int32_t fanin[kMaxFanin] = {}; + bool protocol_ok = false; + uint32_t ordinary_lookups = UINT32_MAX; + volatile int32_t fatal = 0; + std::atomic publish_ok{false}; + + SymbolTestOps::wait_address = + &map->shared_outputs[0].published[0].value; + SymbolTestOps::wait_loads.store(0, std::memory_order_relaxed); + SymbolTestOps::now_calls.store(0, std::memory_order_relaxed); + std::thread publisher([&] { + // 等 consumer 已经观察到未发布状态后再发布，避免把本测试退化成 + // “进入 helper 前已经 ready”的普通快路径。 + while (SymbolTestOps::wait_loads.load(std::memory_order_acquire) == 0) { + std::this_thread::yield(); + } + publish_ok.store( + PublishSharedTaskOutputs(*map, producer, 0), + std::memory_order_release + ); + }); + + const uint32_t count = CollectSharedFanin( + *map, args, 1, kHeapWindow, stats, fanin, + protocol_ok, ordinary_lookups, &fatal + ); + publisher.join(); + SymbolTestOps::wait_address = nullptr; + + Check(publish_ok.load(std::memory_order_acquire), "delayed publication succeeds"); + Check(protocol_ok && fatal == 0, "consumer waits without protocol failure"); + Check( + SymbolTestOps::wait_loads.load(std::memory_order_acquire) > 1, + "consumer performs at least one unpublished retry" + ); + Check( + SymbolTestOps::now_calls.load(std::memory_order_relaxed) >= 1, + "unpublished slow path establishes a watchdog window" + ); + Check(count == 1 && fanin[0] == 0, "delayed INPUT closes producer fanin"); + Check(ordinary_lookups == 0, "delayed symbol never enters ordinary map"); + Check( + AllBytesEqual(payload.get(), sizeof(*payload), 0x3C), + "delayed resolver leaves task payload scratch untouched" + ); + LocalSlot slot{}; + BuildSlotPayload( + slot, 1, static_cast(FunctionId(TaskKind::Qk)), 0, + args, consumer, fanin, count, *map + ); + Check( + SameTensor(slot.tensors[0], output), + "delayed descriptor lands directly in LocalSlot after publication" + ); +} + +void TestOrderedPreparedSymbolUsesSinglePublicationCheck() { + auto map = std::make_unique(); + ResetSharedState(*map); + + constexpr int32_t kProducer = 0; + constexpr int32_t kWriter = 1; + constexpr int32_t kReader = 2; + TensorDesc output = MakeTensor(0x381000000ULL, kProducer); + SubmitContext producer{}; + producer.task_id = kProducer; + producer.result.task_id = kProducer; + producer.result.count = 1; + producer.result.tensors[0] = &output; + producer.shared_result.Reset(kProducer); + Check( + producer.shared_result.AddOutputRef(kProducer, 0) && + PublishSharedTaskOutputs( + *map, producer, kProducer + ), + "ordered symbol setup publishes the producer descriptor" + ); + const FdwicOutputRef output_ref = + producer.shared_result.OutputRef(0); + uint32_t symbol_key = 0; + Check( + SharedSymbolHistoryKey(output_ref, symbol_key), + "ordered symbol setup precomputes one packed key" + ); + + volatile int32_t fatal = 0; + SymbolTestOps::wait_address = + &map->shared_outputs[kProducer].published[0].value; + SymbolTestOps::wait_loads.store(0, std::memory_order_relaxed); + SymbolTestOps::now_calls.store(0, std::memory_order_relaxed); + SymbolTestOps::spin_calls.store(0, std::memory_order_relaxed); + const bool committed = + CommitPreparedSymbolSharedWriterIntentSet( + *map, &symbol_key, 1, kWriter, &fatal + ); + SymbolTestOps::wait_address = nullptr; + + const SharedWriterHistoryCell &history = + map->writer_history[kWriter]; + Check(committed && fatal == 0, "ordered prepared symbol commit succeeds"); + Check( + SymbolTestOps::wait_loads.load(std::memory_order_relaxed) == 1, + "ordered prepared commit checks published exactly once" + ); + Check( + SymbolTestOps::now_calls.load(std::memory_order_relaxed) == 0 && + SymbolTestOps::spin_calls.load(std::memory_order_relaxed) == 0, + "ordered prepared commit never opens a watchdog or spins" + ); + Check( + history.magic == kSharedWriterHistoryMagic && + history.writer_task == kWriter && + history.count == 1 && + history.entries[0].symbol_key == symbol_key && + history.entries[0].previous_writer == kProducer && + map->shared_outputs[kProducer].last_writer[0].value == + kWriter, + "ordered prepared commit publishes the exact immutable history" + ); + TaskArgs reader_args; + ConstructTaskArgs(reader_args); + AppendSharedOutputRef( + reader_args, output_ref, TensorArgType::Input + ); + LocalStats reader_stats{}; + int32_t fanin[kMaxFanin] = {}; + bool protocol_ok = false; + uint32_t ordinary_lookups = UINT32_MAX; + SymbolTestOps::wait_address = + &map->shared_outputs[kProducer].published[0].value; + SymbolTestOps::wait_loads.store(0, std::memory_order_relaxed); + SymbolTestOps::now_calls.store(0, std::memory_order_relaxed); + SymbolTestOps::spin_calls.store(0, std::memory_order_relaxed); + const uint32_t fanin_count = + CollectSharedFanin( + *map, reader_args, kReader, kHeapWindow, + reader_stats, fanin, protocol_ok, ordinary_lookups, + &fatal + ); + SymbolTestOps::wait_address = nullptr; + Check( + protocol_ok && fanin_count == 1 && + fanin[0] == kWriter && ordinary_lookups == 0, + "ordered latest-writer lookup still resolves the prepared writer" + ); + Check( + SymbolTestOps::wait_loads.load(std::memory_order_relaxed) == 1 && + SymbolTestOps::now_calls.load(std::memory_order_relaxed) == 0 && + SymbolTestOps::spin_calls.load(std::memory_order_relaxed) == 0, + "ordered latest-writer lookup also uses one check without waiting" + ); + + ResetSharedState(*map); + fatal = 0; + SymbolTestOps::wait_address = + &map->shared_outputs[kProducer].published[0].value; + SymbolTestOps::wait_loads.store(0, std::memory_order_relaxed); + SymbolTestOps::now_calls.store(0, std::memory_order_relaxed); + SymbolTestOps::spin_calls.store(0, std::memory_order_relaxed); + const bool missing_rejected = + !CommitPreparedSymbolSharedWriterIntentSet( + *map, &symbol_key, 1, kWriter, &fatal + ); + SymbolTestOps::wait_address = nullptr; + Check( + missing_rejected && fatal == 1, + "ordered prepared commit rejects an unpublished producer immediately" + ); + Check( + SymbolTestOps::wait_loads.load(std::memory_order_relaxed) == 1 && + SymbolTestOps::now_calls.load(std::memory_order_relaxed) == 0 && + SymbolTestOps::spin_calls.load(std::memory_order_relaxed) == 0, + "unpublished ordered producer fails after one load without waiting" + ); + Check( + map->writer_history[kWriter].magic == 0 && + map->shared_outputs[kProducer].last_writer[0].value == -1, + "unpublished rejection preserves history and writer" + ); +} + +void TestPublicationWaitFailuresFailClosed() { + auto map = std::make_unique(); + const FdwicOutputRef output_ref{0, 0, 0, 0, 0, 0}; + + ResetSharedState(*map); + map->shared_outputs[0].published[0].value = 7; + volatile int32_t fatal = 0; + SymbolTestOps::now_calls.store(0, std::memory_order_relaxed); + Check( + !WaitForSharedOutputPublished( + *map, output_ref, &fatal + ), + "unexpected publication value is rejected" + ); + Check(fatal == 1, "unexpected publication value broadcasts fatal"); + Check( + SymbolTestOps::now_calls.load(std::memory_order_relaxed) == 0, + "unexpected ready value fails before opening a watchdog window" + ); + + ResetSharedState(*map); + fatal = 1; + SymbolTestOps::now_calls.store(0, std::memory_order_relaxed); + Check( + !WaitForSharedOutputPublished( + *map, output_ref, &fatal + ), + "existing fatal terminates unpublished wait" + ); + Check( + SymbolTestOps::now_calls.load(std::memory_order_relaxed) == 1, + "fatal wait exits before a watchdog recheck" + ); + + ResetSharedState(*map); + fatal = 0; + ExpiredWaitOps::calls.store(0, std::memory_order_relaxed); + Check( + !WaitForSharedOutputPublished( + *map, output_ref, &fatal + ), + "watchdog terminates permanently unpublished symbol" + ); + Check(fatal == 1, "publication watchdog broadcasts fatal"); + Check( + ExpiredWaitOps::calls.load(std::memory_order_relaxed) == 2, + "watchdog clock is read only at slow-path begin and periodic recheck" + ); +} + +void TestInvalidReferencesFailClosed() { + auto map = std::make_unique(); + ResetSharedState(*map); + auto payload = std::make_unique(); + + const FdwicOutputRef invalid_refs[] = { + FdwicOutputRef{3, 0, 0, 0, 0, 0}, + FdwicOutputRef{0, 8, 0, 0, 0, 0}, + FdwicOutputRef{0, 0, 1, 1, 16, 0}, + }; + for (const FdwicOutputRef reference : invalid_refs) { + TaskArgs args; + ConstructTaskArgs(args); + AppendSharedOutputRef(args, reference, TensorArgType::Input); + LocalStats stats{}; + int32_t fanin[kMaxFanin] = {}; + bool protocol_ok = true; + uint32_t ordinary_lookups = UINT32_MAX; + (void)CollectSharedFanin( + *map, args, 2, kHeapWindow, stats, fanin, + protocol_ok, ordinary_lookups + ); + Check(!protocol_ok, "invalid/future/view symbol fails closed"); + Check(ordinary_lookups == 0, "invalid symbol does not enter region map"); + } + + // 第一项是合法 INOUT，第二项才非法；两遍解析必须在第一遍就拒绝， + // 不能提前改写 writer、payload、统计或输出 fanin。 + TensorDesc published = MakeTensor(0x400000000ULL, 0); + SubmitContext producer{}; + producer.task_id = 0; + producer.result.task_id = 0; + producer.result.count = 1; + producer.result.tensors[0] = &published; + producer.shared_result.Reset(0); + Check(producer.shared_result.AddOutputRef(0, 0), "late-failure producer output"); + Check( + PublishSharedTaskOutputs(*map, producer, 0), + "late-failure producer publishes" + ); + + TaskArgs mixed_args; + ConstructTaskArgs(mixed_args); + AppendSharedOutputRef( + mixed_args, producer.shared_result.OutputRef(0), TensorArgType::Inout + ); + AppendSharedOutputRef( + mixed_args, FdwicOutputRef{3, 0, 0, 0, 0, 0}, + TensorArgType::Input + ); + std::memset(payload.get(), 0xA5, sizeof(*payload)); + LocalStats mixed_stats{}; + int32_t mixed_fanin[kMaxFanin]; + for (uint32_t index = 0; index < kMaxFanin; ++index) { + mixed_fanin[index] = -77; + } + bool mixed_protocol_ok = true; + uint32_t mixed_ordinary_lookups = UINT32_MAX; + (void)CollectSharedFanin( + *map, mixed_args, 2, kHeapWindow, mixed_stats, + mixed_fanin, mixed_protocol_ok, mixed_ordinary_lookups + ); + Check(!mixed_protocol_ok, "late invalid symbol rejects whole resolve"); + Check( + map->shared_outputs[0].last_writer[0].value == 0, + "late invalid symbol does not publish earlier INOUT writer" + ); + Check( + mixed_stats.result.shared_symbol_input_loads == 0 && + mixed_stats.result.shared_symbol_inout_commits == 0, + "late invalid symbol publishes no resolve statistics" + ); + Check( + mixed_fanin[0] == -77, + "late invalid symbol publishes no fanin" + ); + Check( + AllBytesEqual(payload.get(), sizeof(*payload), 0xA5), + "late invalid symbol does not touch payload scratch" + ); + + // 合法 INPUT 会先命中局部计数，随后才发现非法引用；统计与 fanin + // 必须等整批验证成功后才发布，不能把局部累计泄露成半次提交。 + TaskArgs late_input_args; + ConstructTaskArgs(late_input_args); + AppendSharedOutputRef( + late_input_args, producer.shared_result.OutputRef(0), + TensorArgType::Input + ); + AppendSharedOutputRef( + late_input_args, FdwicOutputRef{3, 0, 0, 0, 0, 0}, + TensorArgType::Input + ); + LocalStats late_input_stats{}; + int32_t late_input_fanin[kMaxFanin]; + for (uint32_t index = 0; index < kMaxFanin; ++index) { + late_input_fanin[index] = -91; + } + bool late_input_protocol_ok = true; + uint32_t late_input_ordinary_lookups = UINT32_MAX; + (void)CollectSharedFanin( + *map, late_input_args, 2, kHeapWindow, late_input_stats, + late_input_fanin, late_input_protocol_ok, + late_input_ordinary_lookups + ); + Check( + !late_input_protocol_ok, + "late invalid symbol rejects preceding valid INPUT" + ); + Check( + late_input_stats.result.shared_symbol_input_loads == 0, + "late invalid symbol publishes no partial INPUT statistic" + ); + Check( + late_input_fanin[0] == -91, + "late invalid symbol publishes no partial INPUT fanin" + ); + Check( + AllBytesEqual(payload.get(), sizeof(*payload), 0xA5), + "late invalid INPUT pair leaves payload scratch untouched" + ); + + TaskArgs duplicate_args; + ConstructTaskArgs(duplicate_args); + AppendSharedOutputRef( + duplicate_args, producer.shared_result.OutputRef(0), + TensorArgType::Inout + ); + AppendSharedOutputRef( + duplicate_args, producer.shared_result.OutputRef(0), + TensorArgType::OutputExisting + ); + LocalStats duplicate_stats{}; + int32_t duplicate_fanin[kMaxFanin] = {}; + bool duplicate_protocol_ok = true; + uint32_t duplicate_ordinary_lookups = UINT32_MAX; + (void)CollectSharedFanin( + *map, duplicate_args, 2, kHeapWindow, + duplicate_stats, duplicate_fanin, duplicate_protocol_ok, + duplicate_ordinary_lookups + ); + Check( + !duplicate_protocol_ok, + "duplicate write references to one symbol fail before commit" + ); + Check( + map->shared_outputs[0].last_writer[0].value == 0, + "duplicate write rejection preserves producer writer" + ); +} + +} // namespace + +int main() { + TestSharedCompletionPublishesWithoutFrontier(); + TestPublishAndResolve(); + TestPaTwoGroupWriterReadyGate(); + TestPaWriterIntentPreGateFailuresDoNotPublishGate(); + TestWriterCommitFailuresKeepTerminalEvidence(); + TestMultiWriterFailureKeepsPartialTerminalEvidence(); + TestFailedSealDiscardsBuiltTask(); + TestCase1RegistrationBypassesRegionSequencer(); + TestPostBuildSealClosesSuccessAndFailurePaths(); + TestPublicationPreflightIsAllOrNothing(); + TestPublicationCommitFaultsRollback(); + TestConsumerWaitsForDelayedPublication(); + TestOrderedPreparedSymbolUsesSinglePublicationCheck(); + TestPublicationWaitFailuresFailClosed(); + TestInvalidReferencesFailClosed(); + if (g_failures != 0) { + std::fprintf(stderr, "[FAIL] shared-output symbol tests: %d\n", g_failures); + return 1; + } + std::puts("[PASS] shared-output symbol publish/resolve tests"); + return 0; +} diff --git a/tests/atomic_probe/pa_scheduler/test/test_shared_sparse_trace.cpp b/tests/atomic_probe/pa_scheduler/test/test_shared_sparse_trace.cpp new file mode 100644 index 0000000000..77392b0035 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test/test_shared_sparse_trace.cpp @@ -0,0 +1,1391 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include + +#include "host_support.h" + +namespace { + +using pa_scheduler::TaskKind; +using pa_scheduler::TracePhase; +using pa_scheduler::TraceHeader; +using pa_scheduler::TraceRecord; +using pa_scheduler::SharedSubmitClaimTraceRecord; +using pa_scheduler::AtomicOp; +using pa_scheduler::AtomicSite; +using pa_scheduler::kAtomicOpMask; +using pa_scheduler::kAtomicPollBatch; +using pa_scheduler::kAtomicPollCountShift; +using pa_scheduler::kAtomicResultUsed; +using pa_scheduler::kAtomicReturnReady; +using pa_scheduler::kTraceRecordSizeBytes; +using pa_scheduler::kTraceRecordsPerCore; +using pa_scheduler::kTraceSubmitClaimBytesPerCore; +using pa_scheduler::kTraceSubmitClaimRecordSizeBytes; +using pa_scheduler::kTraceWorkerBytes; +using pa_scheduler::host::AtomicRecordSchemaValid; +using pa_scheduler::host::ExpandSharedTraceRecords; +using pa_scheduler::host::InitializeTraceHeader; +using pa_scheduler::host::SharedHostTaskPlan; +using pa_scheduler::host::SharedSparseTraceValidator; +using pa_scheduler::host::ValidateTraceHeader; + +int g_failures = 0; + +void Check(bool condition, const char *message) { + if (condition) return; + std::fprintf(stderr, "[FAIL] shared sparse trace: %s\n", message); + ++g_failures; +} + +TraceRecord MakeRecord( + TracePhase phase, int32_t task_id, int32_t function_id, + uint64_t begin, uint64_t end, uint32_t flags = 0, + uint32_t auxiliary = 0 +) { + TraceRecord record{}; + record.phase = static_cast(phase); + record.task_id = task_id; + record.function_id = function_id; + record.start_cycle = begin; + record.end_cycle = end; + record.flags = flags; + record.auxiliary = auxiliary; + return record; +} + +bool SameRecord(const TraceRecord &left, const TraceRecord &right) { + return left.start_cycle == right.start_cycle && + left.end_cycle == right.end_cycle && + left.task_id == right.task_id && + left.function_id == right.function_id && + left.flags == right.flags && + left.phase == right.phase && + left.auxiliary == right.auxiliary; +} + +void TestTraceBinaryLayoutAndHeaderGate() { + Check( + sizeof(TraceRecord) == 32 && + alignof(TraceRecord) == 32 && + kTraceRecordSizeBytes == 32, + "device trace record must remain a 32-byte half-cache-line" + ); + Check( + offsetof(TraceHeader, cores) == 64 && + sizeof(TraceHeader) == 6976, + "record-size field must not move or grow the header core array" + ); + constexpr size_t partition_bytes = + static_cast(kTraceRecordsPerCore) * + sizeof(TraceRecord); + Check( + sizeof(TraceHeader) % 64 == 0 && + partition_bytes % 64 == 0, + "header and every worker record partition must start on a cache line" + ); + Check( + (0U % 64U) + sizeof(TraceRecord) <= 64U && + (32U % 64U) + sizeof(TraceRecord) <= 64U && + 0U / 64U == 32U / 64U && + 64U / 64U != 32U / 64U, + "two trace records must fit exactly in one cache line" + ); + Check( + sizeof(SharedSubmitClaimTraceRecord) == 32 && + alignof(SharedSubmitClaimTraceRecord) == 32 && + kTraceSubmitClaimRecordSizeBytes == 32, + "shared Submit/Claim record must remain 32-byte aligned" + ); + Check( + kTraceSubmitClaimBytesPerCore == + static_cast(pa_scheduler::kMaxTasks) * + sizeof(SharedSubmitClaimTraceRecord) && + kTraceSubmitClaimBytesPerCore + + static_cast(kTraceRecordsPerCore) * + sizeof(TraceRecord) == + kTraceWorkerBytes && + kTraceWorkerBytes == (1U << 20), + "shared compact and generic regions must exactly fill one 1 MiB worker partition" + ); + Check( + pa_scheduler::TraceSubmitClaimOffset(0) % 64U == 0 && + pa_scheduler::TraceRecordsOffset(0) % 64U == 0 && + pa_scheduler::TraceRecordsOffset(0) - + pa_scheduler::TraceSubmitClaimOffset(0) == + kTraceSubmitClaimBytesPerCore && + pa_scheduler::TraceWorkerOffset(1) - + pa_scheduler::TraceWorkerOffset(0) == + kTraceWorkerBytes, + "shared compact and generic worker regions must keep cache-line isolation" + ); + + TraceHeader header{}; + InitializeTraceHeader(&header); + Check( + header.record_size_bytes == 32 && + ValidateTraceHeader(header, "trace ABI self-test"), + "initialized header must publish and accept a 32-byte raw ABI" + ); + header.record_size_bytes = 64; + Check( + !ValidateTraceHeader(header, "trace ABI negative self-test"), + "header validation must reject the old 64-byte record ABI" + ); +} + +int32_t FunctionId(TaskKind kind) { + return kind == TaskKind::Alloc + ? -1 + : static_cast(static_cast(kind) - 1U); +} + +uint32_t IsAlloc(TaskKind kind) { + return kind == TaskKind::Alloc ? 1U : 0U; +} + +struct CompactTraceWindow { + uint64_t submit_begin; + uint64_t submit_end; + uint64_t claim_begin; + uint64_t claim_end; +}; + +CompactTraceWindow WindowForTask( + uint64_t base_cycle, uint32_t task_id +) { + const uint64_t task_base = + base_cycle + static_cast(task_id) * 100U; + return CompactTraceWindow{ + task_base + 10U, + task_base + 60U, + task_base + 20U, + task_base + 25U, + }; +} + +SharedSubmitClaimTraceRecord MakeCompactRecord( + const CompactTraceWindow &window, bool winner +) { + return SharedSubmitClaimTraceRecord{ + window.claim_begin, + window.claim_end | + (winner + ? pa_scheduler::kSharedClaimWinnerBit + : 0ULL), + window.submit_begin, + window.submit_end, + }; +} + +SharedHostTaskPlan MakeCompactTracePlan() { + constexpr TaskKind kinds[] = { + TaskKind::Alloc, + TaskKind::Qk, + TaskKind::Sf, + TaskKind::Pv, + TaskKind::Up, + }; + SharedHostTaskPlan plan; + plan.total_tasks = + static_cast( + sizeof(kinds) / sizeof(kinds[0]) + ); + plan.tasks.resize(plan.total_tasks); + for (uint32_t task_id = 0; + task_id < plan.total_tasks; ++task_id) { + plan.tasks[task_id].task_id = task_id; + plan.tasks[task_id].kind = kinds[task_id]; + } + return plan; +} + +void TestSharedCompactReconstruction() { + constexpr uint32_t worker = 0; + constexpr uint64_t base_cycle = 5000; + const SharedHostTaskPlan plan = MakeCompactTracePlan(); + constexpr bool winners[] = { + true, false, false, true, false, + }; + constexpr bool attempted[] = { + true, true, false, true, false, + }; + std::vector compact( + plan.total_tasks + ); + std::vector generic; + for (uint32_t task_id = 0; + task_id < plan.total_tasks; ++task_id) { + const CompactTraceWindow window = + WindowForTask(base_cycle, task_id); + compact[task_id] = + MakeCompactRecord(window, winners[task_id]); + if (attempted[task_id]) { + generic.push_back( + MakeRecord( + TracePhase::Atomic, + static_cast(task_id), -1, + window.claim_begin + 1U, + window.claim_begin + 2U, + static_cast( + AtomicOp::FetchMax + ) | + kAtomicResultUsed | + kAtomicReturnReady, + static_cast( + AtomicSite::ClaimMax + ) + ) + ); + } + } + + std::vector logical; + Check( + ExpandSharedTraceRecords( + worker, generic.data(), + static_cast(generic.size()), + compact.data(), plan, &logical + ), + "four-endpoint records and generic ClaimMax rows reconstruct" + ); + Check( + logical.size() == + generic.size() + 2U * plan.total_tasks, + "reconstruction preserves every generic row and adds Claim/Submit" + ); + if (logical.size() != + generic.size() + 2U * plan.total_tasks) { + return; + } + + size_t index = 0; + for (uint32_t task_id = 0; + task_id < plan.total_tasks; ++task_id) { + if (attempted[task_id]) { + Check( + logical[index].phase == + static_cast( + TracePhase::Atomic + ) && + logical[index].task_id == + static_cast(task_id) && + logical[index].auxiliary == + static_cast( + AtomicSite::ClaimMax + ) && + AtomicRecordSchemaValid( + logical[index], true + ), + "generic ClaimMax remains an exact return-ready Atomic row" + ); + ++index; + } + const TraceRecord &claim = logical[index++]; + const TraceRecord &submit = logical[index++]; + const CompactTraceWindow window = + WindowForTask(base_cycle, task_id); + Check( + claim.phase == + static_cast(TracePhase::Claim) && + claim.start_cycle == window.claim_begin && + claim.end_cycle == window.claim_end && + claim.flags == + ((winners[task_id] + ? pa_scheduler::kClaimWon + : 0U) | + (attempted[task_id] + ? pa_scheduler::kClaimAttempted + : 0U)), + "Claim reconstructs absolute endpoints and role-derived attempted" + ); + Check( + submit.phase == + static_cast( + TracePhase::Submit + ) && + submit.start_cycle == window.submit_begin && + submit.end_cycle == window.submit_end && + submit.flags == + (winners[task_id] + ? pa_scheduler::kClaimWon + : 0U), + "Submit reconstructs absolute endpoints and winner" + ); + } +} + +void TestSharedCompactStageOnlyReconstruction() { + constexpr uint32_t worker = 0; + constexpr uint64_t base_cycle = 6000; + const SharedHostTaskPlan plan = MakeCompactTracePlan(); + std::vector compact( + plan.total_tasks + ); + for (uint32_t task_id = 0; + task_id < plan.total_tasks; ++task_id) { + compact[task_id] = MakeCompactRecord( + WindowForTask(base_cycle, task_id), false + ); + } + TraceRecord unused_generic{}; + std::vector logical; + Check( + ExpandSharedTraceRecords( + worker, &unused_generic, 0, + compact.data(), plan, &logical + ) && + logical.size() == 2U * plan.total_tasks, + "stage-only records expand to Claim/Submit without Atomic" + ); +} + +void TestSharedCompactGenericMergeOrder() { + constexpr uint32_t worker = 0; + constexpr uint64_t base_cycle = 7000; + const SharedHostTaskPlan plan = MakeCompactTracePlan(); + std::vector compact( + plan.total_tasks + ); + for (uint32_t task_id = 0; + task_id < plan.total_tasks; ++task_id) { + compact[task_id] = MakeCompactRecord( + WindowForTask(base_cycle, task_id), false + ); + } + const CompactTraceWindow task0 = + WindowForTask(base_cycle, 0); + std::vector generic{ + MakeRecord( + TracePhase::Atomic, 0, -1, + task0.claim_begin + 1U, + task0.claim_begin + 2U, + static_cast(AtomicOp::FetchMax) | + kAtomicResultUsed | kAtomicReturnReady, + static_cast(AtomicSite::ClaimMax) + ), + MakeRecord( + TracePhase::RingBp, 0, 9, + task0.claim_end + 1U, + task0.claim_end + 2U + ), + }; + std::vector logical; + Check( + ExpandSharedTraceRecords( + worker, generic.data(), + static_cast(generic.size()), + compact.data(), plan, &logical + ) && + logical.size() == + generic.size() + 2U * plan.total_tasks && + SameRecord(logical[0], generic[0]) && + logical[1].phase == + static_cast(TracePhase::Claim) && + SameRecord(logical[2], generic[1]) && + logical[3].phase == + static_cast(TracePhase::Submit), + "generic rows merge stably before their enclosing Claim/Submit endpoint" + ); +} + +void TestRejectsBadSharedCompactRecords() { + constexpr uint32_t worker = 0; + constexpr uint64_t base_cycle = 8000; + const SharedHostTaskPlan plan = MakeCompactTracePlan(); + std::vector valid( + plan.total_tasks + ); + for (uint32_t task_id = 0; + task_id < plan.total_tasks; ++task_id) { + valid[task_id] = MakeCompactRecord( + WindowForTask(base_cycle, task_id), false + ); + } + TraceRecord unused_generic{}; + auto rejected = [&]( + const std::vector &records + ) { + std::vector logical; + return !ExpandSharedTraceRecords( + worker, &unused_generic, 0, + records.data(), plan, &logical + ); + }; + + std::vector bad = valid; + bad[0].submit_begin = 0; + Check(rejected(bad), "missing Submit.begin is rejected"); + + bad = valid; + bad[0].claim_end_and_winner = + bad[0].submit_end + 1U; + Check(rejected(bad), "Claim outside Submit is rejected"); + + bad = valid; + bad[2].claim_end_and_winner |= + pa_scheduler::kSharedClaimWinnerBit; + Check( + rejected(bad), + "winner on a role-ineligible Claim is rejected" + ); + + bad = valid; + bad[0].submit_end |= + pa_scheduler::kSharedClaimWinnerBit; + Check( + rejected(bad), + "winner marker is forbidden in Submit endpoints" + ); + + std::vector forbidden{ + MakeRecord( + TracePhase::Claim, 0, -1, + base_cycle + 21U, base_cycle + 22U, + pa_scheduler::kClaimAttempted, 1 + ), + }; + std::vector logical; + Check( + !ExpandSharedTraceRecords( + worker, forbidden.data(), 1, + valid.data(), plan, &logical + ), + "generic stream cannot duplicate dedicated Claim rows" + ); +} + +struct TaskTraceBuilder { + SharedSparseTraceValidator &validator; + uint64_t tick = 10; + uint64_t current_submit_begin = 0; + uint64_t current_claim_begin = 0; + uint64_t last_efdrain_begin = 0; + uint64_t last_efdrain_end = 0; + + bool Begin(uint32_t task_id, TaskKind kind, bool winner, bool attempted = true) { + current_submit_begin = tick; + current_claim_begin = tick + 2; + const uint32_t flags = + (winner ? pa_scheduler::kClaimWon : 0U) | + (attempted ? pa_scheduler::kClaimAttempted : 0U); + const bool ok = validator.Observe( + MakeRecord( + TracePhase::Claim, static_cast(task_id), + winner ? FunctionId(kind) : -1, + current_claim_begin, current_claim_begin + 1, + flags, IsAlloc(kind) + ) + ); + tick = current_claim_begin + 1; + return ok; + } + + bool FinishWinner(uint32_t task_id, TaskKind kind) { + const int32_t function_id = FunctionId(kind); + const uint64_t materialize_begin = tick + 2; + bool ok = validator.Observe( + MakeRecord( + TracePhase::Materialize, static_cast(task_id), + function_id, materialize_begin, materialize_begin + 3, + 0, IsAlloc(kind) + ) + ); + ok &= validator.Observe( + MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputs, + static_cast(task_id), function_id, + materialize_begin + 1, materialize_begin + 3 + ) + ); + ok &= validator.Observe( + MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputsCopy, + static_cast(task_id), function_id, + materialize_begin + 1, materialize_begin + 2 + ) + ); + ok &= validator.Observe( + MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputsFlush, + static_cast(task_id), function_id, + materialize_begin + 2, materialize_begin + 3 + ) + ); + uint64_t previous_end = materialize_begin + 3; + ok &= validator.Observe( + MakeRecord( + TracePhase::Register, static_cast(task_id), + function_id, previous_end, previous_end + 3, 0, 0 + ) + ); + ok &= validator.Observe( + MakeRecord( + TracePhase::SharedRegisterPublishMetadata, + static_cast(task_id), function_id, + previous_end + 1, previous_end + 2 + ) + ); + previous_end += 3; + if (kind != TaskKind::Alloc) { + ok &= validator.Observe( + MakeRecord( + TracePhase::Fanin, static_cast(task_id), + function_id, previous_end, previous_end + 2, 0, 3 + ) + ); + previous_end += 2; + } + ok &= validator.Observe( + MakeRecord( + kind == TaskKind::Alloc + ? TracePhase::AllocComplete + : TracePhase::WinnerBuild, + static_cast(task_id), function_id, + previous_end, previous_end + 4 + ) + ); + previous_end += 4; + ok &= validator.Observe( + MakeRecord( + TracePhase::Submit, static_cast(task_id), + function_id, current_submit_begin, previous_end + 1, + pa_scheduler::kClaimWon, IsAlloc(kind) + ) + ); + last_efdrain_begin = current_submit_begin; + last_efdrain_end = current_claim_begin; + tick = previous_end + 1; + return ok; + } + + bool FinishLoser(uint32_t task_id, TaskKind kind) { + // loser 的 Submit 父区间只覆盖轻量返回；它不等待、不读取 + // TensorMap，也不会产生任何 winner-only 子 span。 + const bool ok = validator.Observe( + MakeRecord( + TracePhase::Submit, static_cast(task_id), -1, + current_submit_begin, tick + 1, 0, IsAlloc(kind) + ) + ); + last_efdrain_begin = current_submit_begin; + last_efdrain_end = current_claim_begin; + ++tick; + return ok; + } +}; + +bool OpenAllocWinnerMaterialize( + SharedSparseTraceValidator &validator, + uint64_t materialize_begin = 15, + uint64_t materialize_end = 18 +) { + return validator.Observe( + MakeRecord( + TracePhase::Claim, 0, -1, 12, 13, + pa_scheduler::kClaimWon | + pa_scheduler::kClaimAttempted, + 1 + ) + ) && + validator.Observe( + MakeRecord( + TracePhase::Materialize, 0, -1, + materialize_begin, materialize_end, 0, 1 + ) + ); +} + +// outputs 包络内固定 copy → flush 两层；copy.end 必须等于 flush.start。 +bool ObserveSharedMaterializeOutputNest( + SharedSparseTraceValidator &validator, + int32_t task_id, + int32_t function_id, + uint64_t outputs_begin, + uint64_t outputs_end, + uint64_t copy_end +) { + return validator.Observe( + MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputs, + task_id, function_id, outputs_begin, outputs_end + ) + ) && + validator.Observe( + MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputsCopy, + task_id, function_id, outputs_begin, copy_end + ) + ) && + validator.Observe( + MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputsFlush, + task_id, function_id, copy_end, outputs_end + ) + ); +} + +bool OpenAllocWinnerRegister( + SharedSparseTraceValidator &validator, + uint64_t register_begin = 18, + uint64_t register_end = 24 +) { + return OpenAllocWinnerMaterialize( + validator, 15, register_begin + ) && + ObserveSharedMaterializeOutputNest( + validator, 0, -1, 16, register_begin, 17 + ) && + validator.Observe( + MakeRecord( + TracePhase::Register, 0, -1, + register_begin, register_end + ) + ); +} + +void TestAcceptsSparseWinnerAndLoserFlow() { + SharedSparseTraceValidator validator; + TaskTraceBuilder trace{validator}; + Check( + trace.Begin(0, TaskKind::Alloc, true) && + trace.FinishWinner(0, TaskKind::Alloc), + "Alloc winner closes with Materialize/Register/AllocComplete/Submit" + ); + Check( + trace.Begin(1, TaskKind::Qk, false) && + trace.FinishLoser(1, TaskKind::Qk), + "attempted QK loser keeps only its Claim/Submit pair" + ); + Check( + trace.Begin(2, TaskKind::Sf, false, false) && + trace.FinishLoser(2, TaskKind::Sf), + "role-not-attempted SF loser also keeps only its Claim/Submit pair" + ); + Check( + trace.Begin(3, TaskKind::Pv, true) && + trace.FinishWinner(3, TaskKind::Pv), + "ordinary winner includes exactly one Fanin and WinnerBuild" + ); + Check( + trace.Begin(4, TaskKind::Up, false) && + trace.FinishLoser(4, TaskKind::Up), + "the final logical task may close through the loser parent" + ); + Check(validator.Closed(), "mixed sparse flow is fully closed"); + Check( + validator.EfDrainCount() == 5 && + validator.LastEfDrainBegin() == trace.last_efdrain_begin && + validator.LastEfDrainEnd() == trace.last_efdrain_end && + validator.ClaimCount() == 5 && + validator.WinnerCount() == 2 && + validator.MaterializeCount() == 2 && + validator.FaninCount() == 1 && + validator.RegisterCount() == 2 && + validator.RegisterMetadataCount() == 2 && + validator.MaterializeTaskOutputsCount() == 2 && + validator.WinnerTailCount() == 2 && + validator.SubmitCount() == 5, + "sparse flow derives every EfDrain boundary and counts only winner children" + ); +} + +void TestRejectsReplayPrefixDrift() { + { + SharedSparseTraceValidator validator; + Check( + !validator.Observe( + MakeRecord(TracePhase::EfDrain, 0, -1, 10, 12) + ), + "shared sparse raw rejects an explicit EfDrain record" + ); + } + { + SharedSparseTraceValidator validator; + Check( + !validator.Observe( + MakeRecord(TracePhase::Claim, 1, -1, 12, 13) + ), + "the first per-core task must be task 0" + ); + } + { + SharedSparseTraceValidator validator; + Check( + !validator.Observe( + MakeRecord(TracePhase::Claim, 0, -1, 13, 12) + ), + "Claim rejects an inverted time boundary" + ); + } + { + SharedSparseTraceValidator validator; + TaskTraceBuilder trace{validator}; + Check( + trace.Begin(0, TaskKind::Alloc, false) && + trace.FinishLoser(0, TaskKind::Alloc), + "task 0 loser establishes the sequence-gap test" + ); + Check( + !validator.Observe( + MakeRecord(TracePhase::Claim, 2, -1, 20, 21) + ), + "a skipped task_id is rejected after a loser" + ); + } + { + SharedSparseTraceValidator validator; + Check( + validator.Observe( + MakeRecord( + TracePhase::Claim, 0, -1, 12, 13, + pa_scheduler::kClaimAttempted, 1 + ) + ), + "valid Claim opens the derived-EfDrain inversion test" + ); + Check( + !validator.Observe( + MakeRecord( + TracePhase::Submit, 0, -1, 13, 14, 0, 1 + ) + ), + "Submit.start cannot be later than Claim.start" + ); + } +} + +void TestRejectsMissingSubmit() { + { + SharedSparseTraceValidator validator; + TaskTraceBuilder trace{validator}; + Check( + trace.Begin(0, TaskKind::Alloc, false), + "loser Claim opens the missing-Submit test" + ); + Check( + !validator.Closed() && + !validator.Observe( + MakeRecord( + TracePhase::Claim, 1, -1, 20, 21, + pa_scheduler::kClaimAttempted, 0 + ) + ), + "loser must close Submit before the next task Claim" + ); + } + { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerRegister(validator) && + validator.Observe( + MakeRecord( + TracePhase::SharedRegisterPublishMetadata, + 0, -1, 20, 22 + ) + ) && + validator.Observe( + MakeRecord( + TracePhase::AllocComplete, 0, -1, 24, 28 + ) + ), + "winner reaches the state that only permits Submit" + ); + Check( + !validator.Closed() && + !validator.Observe( + MakeRecord(TracePhase::Claim, 1, -1, 30, 31) + ), + "winner must close Submit before the next task Claim" + ); + } +} + +void TestRejectsEveryLoserOnlyForbiddenPhase() { + constexpr TracePhase forbidden[] = { + TracePhase::Materialize, + TracePhase::PrepareMap, + TracePhase::Fanin, + TracePhase::Register, + TracePhase::SharedRegisterPublishMetadata, + TracePhase::SharedMaterializePublishTaskOutputs, + TracePhase::SharedMaterializePublishTaskOutputsCopy, + TracePhase::SharedMaterializePublishTaskOutputsFlush, + TracePhase::WinnerBuild, + TracePhase::AllocComplete, + }; + for (TracePhase phase : forbidden) { + SharedSparseTraceValidator validator; + TaskTraceBuilder trace{validator}; + Check( + trace.Begin(0, TaskKind::Alloc, false), + "loser prefix is accepted before a forbidden phase" + ); + Check( + !validator.Observe( + MakeRecord(phase, 0, -1, 13, 14) + ), + "loser rejects every winner-only phase" + ); + } + { + SharedSparseTraceValidator validator; + TaskTraceBuilder trace{validator}; + Check( + trace.Begin(0, TaskKind::Alloc, false) && + trace.FinishLoser(0, TaskKind::Alloc), + "loser closes through exactly one Submit parent" + ); + Check( + !validator.Observe( + MakeRecord(TracePhase::Submit, 0, -1, 10, 15, 0, 1) + ), + "duplicate loser Submit is rejected" + ); + } +} + +void TestRejectsPrepareMapForWinnerAndOutsideSubmit() { + { + SharedSparseTraceValidator validator; + Check( + !validator.Observe( + MakeRecord(TracePhase::PrepareMap, -1, -1, 1, 1) + ), + "shared rejects PrepareMap even outside a task flow" + ); + } + { + SharedSparseTraceValidator validator; + TaskTraceBuilder trace{validator}; + Check( + trace.Begin(0, TaskKind::Alloc, true), + "Alloc winner opens the PrepareMap rejection test" + ); + Check( + validator.Observe( + MakeRecord( + TracePhase::Materialize, 0, -1, 15, 18, 0, 1 + ) + ), + "winner Materialize is accepted before the forbidden marker" + ); + Check( + !validator.Observe( + MakeRecord(TracePhase::PrepareMap, 0, -1, 18, 18) + ), + "shared winner cannot carry a zero-duration PrepareMap marker" + ); + } +} + +void TestRejectsWinnerShapeDrift() { + { + SharedSparseTraceValidator validator; + TaskTraceBuilder trace{validator}; + Check( + trace.Begin(0, TaskKind::Alloc, true), + "Alloc winner opens the Fanin rejection test" + ); + Check( + validator.Observe( + MakeRecord( + TracePhase::Materialize, 0, -1, 15, 18, 0, 1 + ) + ), + "Alloc Materialize is accepted" + ); + Check( + ObserveSharedMaterializeOutputNest( + validator, 0, -1, 16, 18, 17 + ) && + validator.Observe( + MakeRecord( + TracePhase::Register, 0, -1, 18, 20 + ) + ) && + validator.Observe( + MakeRecord( + TracePhase::SharedRegisterPublishMetadata, + 0, -1, 19, 19 + ) + ), + "Alloc Register details are accepted before the tail" + ); + Check( + !validator.Observe( + MakeRecord(TracePhase::Fanin, 0, -1, 20, 21) + ), + "Alloc winner cannot emit Fanin" + ); + } + { + SharedSparseTraceValidator validator; + TaskTraceBuilder trace{validator}; + Check( + trace.Begin(0, TaskKind::Alloc, false) && + trace.FinishLoser(0, TaskKind::Alloc) && + trace.Begin(1, TaskKind::Qk, true), + "ordinary winner opens the missing-Fanin test" + ); + Check( + validator.Observe( + MakeRecord( + TracePhase::Materialize, 1, 0, 18, 21 + ) + ), + "ordinary Materialize is accepted" + ); + Check( + ObserveSharedMaterializeOutputNest( + validator, 1, 0, 19, 21, 20 + ) && + validator.Observe( + MakeRecord(TracePhase::Register, 1, 0, 21, 24, 0, 1) + ) && + validator.Observe( + MakeRecord( + TracePhase::SharedRegisterPublishMetadata, + 1, 0, 22, 23 + ) + ), + "ordinary Register details precede Fanin" + ); + Check( + !validator.Observe( + MakeRecord(TracePhase::WinnerBuild, 1, 0, 24, 25) + ), + "ordinary winner cannot skip Fanin after Register" + ); + } + { + SharedSparseTraceValidator validator; + TaskTraceBuilder trace{validator}; + Check( + trace.Begin(0, TaskKind::Alloc, true), + "Alloc winner opens the wrong-tail test" + ); + Check( + validator.Observe( + MakeRecord( + TracePhase::Materialize, 0, -1, 15, 18, 0, 1 + ) + ) && + ObserveSharedMaterializeOutputNest( + validator, 0, -1, 16, 18, 17 + ) && + validator.Observe( + MakeRecord( + TracePhase::Register, 0, -1, 18, 20 + ) + ) && + validator.Observe( + MakeRecord( + TracePhase::SharedRegisterPublishMetadata, + 0, -1, 19, 19 + ) + ), + "Alloc winner reaches its tail" + ); + Check( + !validator.Observe( + MakeRecord(TracePhase::WinnerBuild, 0, -1, 20, 24) + ), + "Alloc winner requires AllocComplete rather than WinnerBuild" + ); + } + { + SharedSparseTraceValidator validator; + TaskTraceBuilder trace{validator}; + Check( + trace.Begin(0, TaskKind::Alloc, true), + "incomplete winner opens the closure test" + ); + Check( + !validator.Closed(), + "winner cannot close before all winner-only phases and Submit" + ); + } +} + +void TestMaterializeOutputDetailContract() { + { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerMaterialize(validator), + "Materialize parent opens the missing-output test" + ); + Check( + !validator.Observe( + MakeRecord(TracePhase::Register, 0, -1, 18, 24) + ), + "winner cannot enter Register before output publication closes" + ); + } + { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerMaterialize(validator) && + ObserveSharedMaterializeOutputNest( + validator, 0, -1, 16, 18, 17 + ), + "one output nest contained by Materialize is accepted" + ); + Check( + !validator.Observe( + MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputs, + 0, -1, 16, 18 + ) + ), + "duplicate Materialize output detail is rejected" + ); + } + { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerMaterialize(validator) && + validator.Observe( + MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputs, + 0, -1, 16, 18 + ) + ), + "output parent opens the missing-copy test" + ); + Check( + !validator.Observe( + MakeRecord(TracePhase::Register, 0, -1, 18, 24) + ), + "winner cannot omit output copy detail" + ); + } + { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerMaterialize(validator) && + validator.Observe( + MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputs, + 0, -1, 16, 18 + ) + ) && + validator.Observe( + MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputsCopy, + 0, -1, 16, 17 + ) + ), + "copy detail opens the missing-flush test" + ); + Check( + !validator.Observe( + MakeRecord(TracePhase::Register, 0, -1, 18, 24) + ), + "winner cannot omit output flush detail" + ); + } + { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerMaterialize(validator) && + validator.Observe( + MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputs, + 0, -1, 16, 18 + ) + ) && + validator.Observe( + MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputsCopy, + 0, -1, 16, 17 + ) + ), + "copy detail opens the flush-adjacency test" + ); + Check( + !validator.Observe( + MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputsFlush, + 0, -1, 18, 18 + ) + ), + "flush must start exactly at copy end" + ); + } + for (int variant = 0; variant < 4; ++variant) { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerMaterialize(validator), + "Materialize opens output containment/identity test" + ); + TraceRecord output = MakeRecord( + TracePhase::SharedMaterializePublishTaskOutputs, + 0, -1, 16, 18 + ); + if (variant == 0) output.start_cycle = 14; + if (variant == 1) output.task_id = 1; + if (variant == 2) output.function_id = 0; + if (variant == 3) { + output.flags = 1; + output.auxiliary = 1; + } + Check( + !validator.Observe(output), + "Materialize output detail rejects bad boundary, identity, or payload" + ); + } +} + +void TestRegisterMetadataDetailContract() { + { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerRegister(validator), + "Register parent opens the missing-detail test" + ); + Check( + !validator.Observe( + MakeRecord(TracePhase::AllocComplete, 0, -1, 24, 28) + ) && + !validator.Closed(), + "winner cannot omit its Register metadata detail" + ); + } + { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerRegister(validator) && + validator.Observe( + MakeRecord( + TracePhase::SharedRegisterPublishMetadata, + 0, -1, 20, 22 + ) + ), + "one contained Register metadata detail is accepted" + ); + Check( + !validator.Observe( + MakeRecord( + TracePhase::SharedRegisterPublishMetadata, + 0, -1, 20, 22 + ) + ), + "duplicate Register metadata detail is rejected" + ); + } + { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerRegister(validator), + "Register parent opens the early-boundary test" + ); + Check( + !validator.Observe( + MakeRecord( + TracePhase::SharedRegisterPublishMetadata, + 0, -1, 17, 20 + ) + ), + "Register metadata cannot begin before its parent" + ); + } + { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerRegister(validator), + "Register parent opens the late-boundary test" + ); + Check( + !validator.Observe( + MakeRecord( + TracePhase::SharedRegisterPublishMetadata, + 0, -1, 20, 25 + ) + ), + "Register metadata cannot end after its parent" + ); + } + { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerRegister(validator), + "Register parent opens the wrong-task test" + ); + Check( + !validator.Observe( + MakeRecord( + TracePhase::SharedRegisterPublishMetadata, + 1, -1, 20, 22 + ) + ), + "Register metadata must keep the parent task identity" + ); + } + { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerRegister(validator), + "Register parent opens the wrong-function test" + ); + Check( + !validator.Observe( + MakeRecord( + TracePhase::SharedRegisterPublishMetadata, + 0, 0, 20, 22 + ) + ), + "Register metadata must keep the parent function identity" + ); + } + { + SharedSparseTraceValidator validator; + Check( + OpenAllocWinnerRegister(validator), + "Register parent opens the payload-shape test" + ); + Check( + !validator.Observe( + MakeRecord( + TracePhase::SharedRegisterPublishMetadata, + 0, -1, 20, 22, 1, 1 + ) + ), + "Register metadata detail requires zero flags and auxiliary" + ); + } +} + +void TestSharedInsertTurnAtomicSchema() { + TraceRecord poll = MakeRecord( + TracePhase::Atomic, -1, -1, 100, 200, + static_cast(AtomicOp::Load) | + kAtomicResultUsed | kAtomicPollBatch | + kAtomicReturnReady | + (17U << kAtomicPollCountShift), + static_cast( + AtomicSite::SharedInsertTurnPoll + ) + ); + Check( + AtomicRecordSchemaValid(poll, true), + "shared insert-turn aggregate PollBatch schema is accepted" + ); + TraceRecord direct_poll = poll; + direct_poll.flags = + static_cast(AtomicOp::Load) | + kAtomicResultUsed | kAtomicReturnReady; + direct_poll.task_id = 3; + Check( + !AtomicRecordSchemaValid(direct_poll, true), + "shared insert-turn poll cannot masquerade as a direct Load" + ); + + TraceRecord handoff = MakeRecord( + TracePhase::Atomic, 3, -1, 200, 230, + static_cast( + AtomicOp::CompareExchange + ) | + kAtomicResultUsed | kAtomicReturnReady, + static_cast( + AtomicSite::SharedInsertTurnHandoff + ) + ); + Check( + AtomicRecordSchemaValid(handoff, true), + "shared insert-turn handoff CAS schema is accepted" + ); + TraceRecord anonymous_handoff = handoff; + anonymous_handoff.task_id = -1; + Check( + !AtomicRecordSchemaValid(anonymous_handoff, true), + "handoff CAS requires its shared winner task identity" + ); + TraceRecord wrong_handoff_op = handoff; + wrong_handoff_op.flags = + (wrong_handoff_op.flags & ~kAtomicOpMask) | + static_cast(AtomicOp::Exchange); + Check( + !AtomicRecordSchemaValid(wrong_handoff_op, true), + "handoff site rejects a non-CAS atomic op" + ); +} + +void TestPlanClosesAllLogicalTasks() { + // SchedulerState 保留真实 DistGlobal 约 1 GiB ABI，不能放在线程栈上。 + // 静态零初始化只映射本用例实际触碰的 config/context_lens 页面。 + static pa_scheduler::SchedulerState state{}; + state.config.batches = 2; + state.context_lens[0] = 0; + state.context_lens[1] = 0; + pa_scheduler::host::SharedHostTaskPlan plan; + Check( + pa_scheduler::host::BuildSharedHostTaskPlan(state, &plan), + "two-batch zero-context host plan is valid" + ); + Check( + plan.total_tasks == 2 && + plan.TaskAt(0)->kind == TaskKind::Alloc && + plan.TaskAt(1)->kind == TaskKind::Alloc, + "authoritative plan contains one Alloc per empty batch" + ); + + SharedSparseTraceValidator validator(&plan); + TaskTraceBuilder trace{validator}; + Check( + trace.Begin(0, TaskKind::Alloc, true) && + trace.FinishWinner(0, TaskKind::Alloc), + "first planned Alloc winner closes" + ); + Check( + !validator.Closed(), + "plan-aware validator rejects a truncated per-core replay" + ); + Check( + trace.Begin(1, TaskKind::Alloc, false) && + trace.FinishLoser(1, TaskKind::Alloc), + "second planned Alloc loser closes through its Submit parent" + ); + Check( + validator.Closed(), + "plan-aware validator closes only after every logical task" + ); +} + +} // namespace + +int main() { + TestTraceBinaryLayoutAndHeaderGate(); + TestSharedCompactReconstruction(); + TestSharedCompactStageOnlyReconstruction(); + TestSharedCompactGenericMergeOrder(); + TestRejectsBadSharedCompactRecords(); + TestAcceptsSparseWinnerAndLoserFlow(); + TestRejectsReplayPrefixDrift(); + TestRejectsMissingSubmit(); + TestRejectsEveryLoserOnlyForbiddenPhase(); + TestRejectsPrepareMapForWinnerAndOutsideSubmit(); + TestRejectsWinnerShapeDrift(); + TestMaterializeOutputDetailContract(); + TestRegisterMetadataDetailContract(); + TestSharedInsertTurnAtomicSchema(); + TestPlanClosesAllLogicalTasks(); + if (g_failures != 0) { + std::fprintf( + stderr, "[FAIL] shared sparse trace tests: %d\n", g_failures + ); + return 1; + } + std::printf("[PASS] shared sparse trace tests\n"); + return 0; +} diff --git a/tests/atomic_probe/pa_scheduler/test/test_shared_tensor_map_ring.cpp b/tests/atomic_probe/pa_scheduler/test/test_shared_tensor_map_ring.cpp new file mode 100644 index 0000000000..47dd8b86ce --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test/test_shared_tensor_map_ring.cpp @@ -0,0 +1,2844 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// 直接实例化 shared 生产 primitive；测试只提供 CPU 原子适配和确定性事件 +// ledger，不复制 append/lookup/reclaim 算法，也不把 x86 一致性冒充 A5 DCCI。 +#define PA_DEVICE inline +#define PA_GM +#include "pa_shared_tensormap.h" + +namespace { + +using pa_scheduler::SharedAppendPreparedTask; +using pa_scheduler::SharedAppendCheck; +using pa_scheduler::SharedAdvanceReaderDone; +using pa_scheduler::SharedBucketState; +using pa_scheduler::SharedCheckPreparedTaskAppend; +using pa_scheduler::SharedCheckTaskAppend; +using pa_scheduler::SharedComputeReaderReclaimCandidate; +using pa_scheduler::SharedComputeOrderedReclaimCandidate; +using pa_scheduler::SharedHasExactTaskTurn; +using pa_scheduler::SharedLookupRegion; +using pa_scheduler::SharedPreflightTaskAppend; +using pa_scheduler::SharedPublishReclaimCandidate; +using pa_scheduler::SharedPublishTaskCommit; +using pa_scheduler::SharedReadRegionSlot; +using pa_scheduler::SharedRefreshReaderReclaimForTask; +using pa_scheduler::SharedRefreshReclaimForTask; +using pa_scheduler::SharedRegionPayload; +using pa_scheduler::SharedRegionSlot; +using pa_scheduler::SharedRegionValue; +using pa_scheduler::SharedRetireBucket; +using pa_scheduler::SharedTensorMapSidecar; +using pa_scheduler::SharedTensorMapSlotIndex; +using pa_scheduler::SharedTryAppendReaderGatedTask; +using pa_scheduler::TensorMapHash; +using pa_scheduler::kMapBucketCapacity; +using pa_scheduler::kMapBuckets; +using pa_scheduler::kMapCapacity; +using pa_scheduler::kMaxTaskTensors; +using pa_scheduler::kSharedMapEmptySeq; + +static_assert(PTO_FDWIC_SHARED_MAP == 1, "this test must compile as the shared TensorMap mode"); +static_assert(sizeof(SharedRegionValue) == 32, "shared logical value ABI changed"); +static_assert(sizeof(SharedRegionPayload) == 64, "shared payload must occupy one cache line"); +static_assert(alignof(SharedRegionPayload) == 64, "shared payload alignment changed"); +static_assert(sizeof(SharedRegionSlot) == 128, "shared slot must occupy two cache lines"); +static_assert(offsetof(SharedRegionSlot, seq) == 64, "shared seq cache line offset changed"); +static_assert(sizeof(SharedBucketState) == 128, "shared bucket control ABI changed"); +static_assert(offsetof(SharedBucketState, tail) == 64, "shared head/tail cache lines merged"); +#if PTO_FDWIC_TENSORMAP_RING_CAP == 128 +static_assert(sizeof(SharedTensorMapSidecar) == 12434560, "shared sidecar ABI changed"); +#endif +static_assert(alignof(SharedTensorMapSidecar) == 64, "shared sidecar alignment changed"); +static_assert(offsetof(SharedTensorMapSidecar, buckets) == 128, "shared bucket offset changed"); +static_assert( + offsetof(SharedTensorMapSidecar, slots) == + offsetof(SharedTensorMapSidecar, buckets) + + sizeof(SharedBucketState) * kMapBuckets, + "shared slots must immediately follow the active bucket controls" +); +static_assert( + offsetof(SharedTensorMapSidecar, shared_outputs) == + offsetof(SharedTensorMapSidecar, slots) + + sizeof(SharedRegionSlot) * kMapCapacity, + "shared output table must immediately follow the fixed 16K slot pool" +); +static_assert( + offsetof(SharedTensorMapSidecar, reader_done) == + offsetof(SharedTensorMapSidecar, writer_history) + + sizeof(pa_scheduler::SharedWriterHistoryCell) * + pa_scheduler::kMaxTasks, + "shared reader progress must immediately follow writer history" +); +#if PTO_FDWIC_TENSORMAP_RING_CAP == 128 +static_assert(offsetof(SharedTensorMapSidecar, slots) == 16512, "default shared slot offset changed"); +static_assert(offsetof(SharedTensorMapSidecar, shared_outputs) == 2113664, "default shared output offset changed"); +static_assert(offsetof(SharedTensorMapSidecar, shared_heap_cursor) == 11026560, "default shared heap offset changed"); +static_assert( + offsetof(SharedTensorMapSidecar, shared_heap_vend) == 11027072, + "default shared heap vend offset changed" +); +static_assert( + offsetof(SharedTensorMapSidecar, shared_vector_cursor) == 11027136, + "default shared Vector cursor offset changed" +); +static_assert( + offsetof(SharedTensorMapSidecar, writer_history) == 11027648, + "default shared writer-history offset changed" +); +static_assert( + offsetof(SharedTensorMapSidecar, reader_done) == 12420288, + "default shared reader-progress offset changed" +); +#endif + +enum class EventKind : uint8_t { + Load, + Exchange, + CompareExchange, + Invalidate, + Flush, +}; + +struct Event { + EventKind kind; + const void *address; + int64_t argument; + int64_t result; +}; + +struct RecordingOps { + using InvalidateHook = void (*)(void *); + using LoadHook = void (*)(void *); + + static std::vector events; + static bool record; + static const void *mutate_payload; + static volatile int64_t *mutate_seq; + static int64_t mutate_seq_value; + static bool mutate_once; + static const void *invalidate_hook_address; + static InvalidateHook invalidate_hook; + static void *invalidate_hook_context; + static const void *load_hook_address; + static LoadHook load_hook; + static void *load_hook_context; + + static int64_t Load(volatile int64_t *address) { + // load hook 在真正取值前执行，用来固定“reader 已读旧 head、尚未 + // 读 tail”这一控制快照交错。先清空再回调，避免 writer 内部读取 + // 同一 tail 时递归。 + if (load_hook != nullptr && + ConstAddress(address) == load_hook_address) { + const LoadHook callback = load_hook; + void *const context = load_hook_context; + load_hook_address = nullptr; + load_hook = nullptr; + load_hook_context = nullptr; + callback(context); + } + // 与 CPU scheduler 相同，以 acquire atomic add-zero 表达协议观察； + // 返回值只用于正确性，不解释为 A5 atomic 完成时延。 + const int64_t value = + __atomic_fetch_add(address, static_cast(0), __ATOMIC_ACQUIRE); + if (record) { + events.push_back({EventKind::Load, ConstAddress(address), 0, value}); + } + return value; + } + + static int64_t Exchange(volatile int64_t *address, int64_t value) { + const int64_t old = __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + if (record) { + events.push_back({EventKind::Exchange, ConstAddress(address), value, old}); + } + return old; + } + + static int64_t CompareExchange( + volatile int64_t *address, int64_t expected, int64_t desired + ) { + int64_t observed = expected; + (void)__atomic_compare_exchange_n( + address, &observed, desired, false, + __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE + ); + if (record) { + events.push_back( + { + EventKind::CompareExchange, + ConstAddress(address), desired, observed, + } + ); + } + return observed; + } + + static void InvalidateRegion(const void *address, uint64_t bytes) { + if (record) { + events.push_back( + {EventKind::Invalidate, address, static_cast(bytes), 0} + ); + } + // 单次 hook 在 reader 第一次 seq 检查之后、拷贝 payload 之前执行。 + // 先清空 hook 再回调，允许回调里的 preflight 嵌套读取同一槽而不递归。 + if (invalidate_hook != nullptr && + address == invalidate_hook_address) { + const InvalidateHook callback = invalidate_hook; + void *const context = invalidate_hook_context; + invalidate_hook_address = nullptr; + invalidate_hook = nullptr; + invalidate_hook_context = nullptr; + callback(context); + } + // 该注入点只制造“第一次 seq 检查后，槽被另一 lap 复用”的确定性交错， + // 用来证明第二次 seq 检查有效；它不模拟 cache line 内容或 DCCI。 + if (mutate_once && address == mutate_payload && mutate_seq != nullptr) { + __atomic_store_n(mutate_seq, mutate_seq_value, __ATOMIC_RELEASE); + mutate_once = false; + } + std::atomic_thread_fence(std::memory_order_acquire); + } + + static void FlushRegion(void *address, uint64_t bytes) { + if (record) { + events.push_back( + {EventKind::Flush, address, static_cast(bytes), 0} + ); + } + std::atomic_thread_fence(std::memory_order_release); + } + + static void ResetEvents() { + events.clear(); + record = true; + mutate_payload = nullptr; + mutate_seq = nullptr; + mutate_seq_value = 0; + mutate_once = false; + invalidate_hook_address = nullptr; + invalidate_hook = nullptr; + invalidate_hook_context = nullptr; + load_hook_address = nullptr; + load_hook = nullptr; + load_hook_context = nullptr; + } + + static void DisableEvents() { + events.clear(); + record = false; + mutate_payload = nullptr; + mutate_seq = nullptr; + mutate_seq_value = 0; + mutate_once = false; + invalidate_hook_address = nullptr; + invalidate_hook = nullptr; + invalidate_hook_context = nullptr; + load_hook_address = nullptr; + load_hook = nullptr; + load_hook_context = nullptr; + } + +private: + static const void *ConstAddress(volatile int64_t *address) { + return const_cast(address); + } +}; + +std::vector RecordingOps::events; +bool RecordingOps::record = true; +const void *RecordingOps::mutate_payload = nullptr; +volatile int64_t *RecordingOps::mutate_seq = nullptr; +int64_t RecordingOps::mutate_seq_value = 0; +bool RecordingOps::mutate_once = false; +const void *RecordingOps::invalidate_hook_address = nullptr; +RecordingOps::InvalidateHook RecordingOps::invalidate_hook = nullptr; +void *RecordingOps::invalidate_hook_context = nullptr; +const void *RecordingOps::load_hook_address = nullptr; +RecordingOps::LoadHook RecordingOps::load_hook = nullptr; +void *RecordingOps::load_hook_context = nullptr; + +int g_failures = 0; + +void Expect(bool condition, const char *test, const char *message) { + if (condition) { + return; + } + std::fprintf(stderr, "[FAIL] shared TensorMap ring/%s: %s\n", test, message); + ++g_failures; +} + +void ExpectEqual(int64_t actual, int64_t expected, const char *test, const char *field) { + if (actual == expected) { + return; + } + std::fprintf( + stderr, "[FAIL] shared TensorMap ring/%s: %s actual=%lld expected=%lld\n", + test, field, static_cast(actual), static_cast(expected) + ); + ++g_failures; +} + +void StoreControl(volatile int64_t *address, int64_t value) { + __atomic_store_n(address, value, __ATOMIC_RELAXED); +} + +int64_t LoadControl(volatile int64_t *address) { + return __atomic_load_n(address, __ATOMIC_ACQUIRE); +} + +void ResetSharedTensorMap(SharedTensorMapSidecar &map) { + // 与 scheduler 的正式 host reset 保持相同控制字初值。payload 保持 + // 惰性：seq=-1 时任何旧字节都不可见；正式 reset 额外清零整块 + // sidecar，只是为了让 host 诊断中的保留字节也确定。 + pa_scheduler::InitializeSharedInsertTurns(map); + StoreControl(&map.reclaim_upto.value, -1); + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + StoreControl(&map.buckets[bucket].head.value, 0); + StoreControl(&map.buckets[bucket].tail.value, 0); + } + for (uint32_t slot = 0; slot < kMapCapacity; ++slot) { + StoreControl(&map.slots[slot].seq.value, kSharedMapEmptySeq); + } + // descriptor 的零值由正式 host 对整块 sidecar 的 memset 建立；独立 ring + // 用例不读取 descriptor，但仍把两组发布控制字初始化成协议要求的 -1， + // 防止后续 symbol 子测把 task 0 误判成已发布。 + for (uint32_t task = 0; task < pa_scheduler::kMaxTasks; ++task) { + for (uint32_t output = 0; output < pa_scheduler::kSharedOutputMaxPerTask; ++output) { + StoreControl(&map.shared_outputs[task].published[output].value, -1); + StoreControl(&map.shared_outputs[task].last_writer[output].value, -1); + } + map.writer_history[task].magic = 0; + map.writer_history[task].writer_task = 0; + map.writer_history[task].count = 0; + map.writer_history[task].reserved = 0; + } + for (uint32_t worker = 0; + worker < pa_scheduler::kWorkers; ++worker) { + StoreControl(&map.reader_done[worker].value, -1); + } +} + +std::unique_ptr NewMap() { + // sidecar 超过 2 MiB，必须 heap 分配，不能依赖 host 线程栈容量。 + auto map = std::unique_ptr(new SharedTensorMapSidecar); + ResetSharedTensorMap(*map); + RecordingOps::ResetEvents(); + return map; +} + +void TestPhysicalSlotBoundaries() { + constexpr const char *kTest = "physical-slot-boundaries"; + ExpectEqual( + SharedTensorMapSlotIndex(0, 0), 0, + kTest, "first physical slot" + ); + ExpectEqual( + SharedTensorMapSlotIndex( + kMapBuckets - 1U, + static_cast(kMapBucketCapacity - 1U) + ), + kMapCapacity - 1U, + kTest, "last physical slot" + ); + ExpectEqual( + SharedTensorMapSlotIndex( + kMapBuckets - 1U, + static_cast(kMapBucketCapacity) + ), + (kMapBuckets - 1U) * kMapBucketCapacity, + kTest, "last bucket first wrapped slot" + ); +} + +SharedRegionValue MakeRegion( + uint64_t buffer_addr, uint64_t lo, uint64_t hi, int32_t producer +) { + return {buffer_addr, lo, hi, producer, 0}; +} + +uint64_t FindAddressOutsideBucket(uint64_t seed, uint32_t excluded_bucket) { + if constexpr (kMapBuckets == 1) { + // 单桶变体不存在“另一个桶”，但调用方仍需要不同的 region identity。 + // 返回 seed 让语义测试继续覆盖同桶多地址；跨桶隔离由专门测试按 + // kMapBuckets>1 条件执行。 + return seed; + } + uint64_t address = seed; + while (TensorMapHash(address) == excluded_bucket) { + address += 64; + } + return address; +} + +uint64_t FindAddressInsideBucket(uint64_t seed, uint32_t wanted_bucket) { + uint64_t address = seed; + while (TensorMapHash(address) != wanted_bucket) { + address += 64; + } + return address; +} + +enum class CommitResult : uint8_t { + Pending, + Committed, + Failed, +}; + +CommitResult TryCommitTask( + SharedTensorMapSidecar &map, int32_t task_id, + const std::vector &entries, + int32_t heap_window = INT32_MAX +) { + // 这只是按公开 primitive 组合的一步式确定性 driver：未持有 task turn + // 时不触碰 reclaim/head/tail；持有 exact turn 后按 N-H-1 推进回收， + // 再做整任务预检、append 和 commit。 + if (RecordingOps::Load(&map.committed_tasks.value) != task_id) { + return CommitResult::Pending; + } + int64_t reclaim_upto = -2; + if (!SharedRefreshReclaimForTask( + map, task_id, heap_window, reclaim_upto + )) { + return CommitResult::Failed; + } + if (!SharedPreflightTaskAppend( + map, entries.data(), static_cast(entries.size()), + reclaim_upto + )) { + return CommitResult::Failed; + } + if (!SharedAppendPreparedTask( + map, entries.data(), static_cast(entries.size()) + )) { + return CommitResult::Failed; + } + if (!SharedPublishTaskCommit(map, task_id)) { + return CommitResult::Failed; + } + return CommitResult::Committed; +} + +struct SlowReaderReuseAttempt { + SharedTensorMapSidecar *map; + const SharedRegionValue *replacements; + uint32_t replacement_count; + uint32_t bucket; + uint32_t active_readers; + int32_t heap_window; + bool fired; + int64_t reclaim_upto; + SharedAppendCheck append_result; + int64_t head; + int64_t tail; + int64_t committed_tasks; + int64_t slot_seq; + int32_t slot_producer; +}; + +void AttemptReuseWhileReaderPaused(void *opaque) { + auto &attempt = + *static_cast(opaque); + attempt.fired = true; + attempt.append_result = + SharedTryAppendReaderGatedTask( + *attempt.map, attempt.replacements, + attempt.replacement_count, + attempt.active_readers, attempt.heap_window + ); + attempt.reclaim_upto = + LoadControl(&attempt.map->reclaim_upto.value); + attempt.head = + LoadControl( + &attempt.map->buckets[attempt.bucket].head.value + ); + attempt.tail = + LoadControl( + &attempt.map->buckets[attempt.bucket].tail.value + ); + attempt.committed_tasks = + LoadControl(&attempt.map->committed_tasks.value); + const uint32_t slot_index = + SharedTensorMapSlotIndex(attempt.bucket, 0); + attempt.slot_seq = + LoadControl(&attempt.map->slots[slot_index].seq.value); + attempt.slot_producer = + attempt.map->slots[slot_index].payload.value.producer; +} + +struct ConcurrentSafeRetireAttempt { + SharedTensorMapSidecar *map; + const SharedRegionValue *replacements; + uint32_t replacement_count; + uint32_t active_readers; + int32_t heap_window; + bool fired; + bool candidate_ok; + bool publish_ok; + bool append_ok; + int64_t candidate; + int64_t reclaim_upto; + SharedAppendCheck append_check; +}; + +void RetireSafePrefixWhileReaderPaused(void *opaque) { + auto &attempt = + *static_cast(opaque); + attempt.fired = true; + attempt.candidate = -2; + attempt.reclaim_upto = -2; + attempt.candidate_ok = + SharedComputeReaderReclaimCandidate( + *attempt.map, attempt.active_readers, + attempt.heap_window, attempt.candidate + ); + attempt.publish_ok = + attempt.candidate_ok && + SharedPublishReclaimCandidate( + *attempt.map, attempt.candidate, + attempt.reclaim_upto + ); + attempt.append_check = + attempt.publish_ok + ? SharedCheckTaskAppend( + *attempt.map, attempt.replacements, + attempt.replacement_count, + attempt.reclaim_upto + ) + : SharedAppendCheck::ProtocolError; + attempt.append_ok = + attempt.append_check == SharedAppendCheck::Ready && + SharedAppendPreparedTask( + *attempt.map, attempt.replacements, + attempt.replacement_count + ); +} + +struct LogicalTuple { + uint64_t buffer_addr; + uint64_t lo; + uint64_t hi; + int32_t producer; + + bool operator<(const LogicalTuple &other) const { + return std::tie(buffer_addr, lo, hi, producer) < + std::tie(other.buffer_addr, other.lo, other.hi, other.producer); + } + + bool operator==(const LogicalTuple &other) const { + return buffer_addr == other.buffer_addr && lo == other.lo && + hi == other.hi && producer == other.producer; + } +}; + +LogicalTuple ToTuple(const SharedRegionValue &value) { + return {value.buffer_addr, value.lo, value.hi, value.producer}; +} + +bool SnapshotLogicalMap( + SharedTensorMapSidecar &map, std::vector &snapshot +) { + snapshot.clear(); + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + const int64_t head = LoadControl(&map.buckets[bucket].head.value); + const int64_t tail = LoadControl(&map.buckets[bucket].tail.value); + if (head < 0 || tail < head || + static_cast(tail - head) > kMapBucketCapacity) { + return false; + } + for (int64_t cursor = head; cursor < tail; ++cursor) { + SharedRegionValue value{}; + if (!SharedReadRegionSlot( + map, bucket, static_cast(cursor), value + )) { + return false; + } + snapshot.push_back(ToTuple(value)); + } + } + std::sort(snapshot.begin(), snapshot.end()); + return true; +} + +size_t FindEvent( + EventKind kind, const void *address, size_t begin, + bool check_argument = false, int64_t argument = 0 +) { + for (size_t index = begin; index < RecordingOps::events.size(); ++index) { + const Event &event = RecordingOps::events[index]; + if (event.kind == kind && event.address == address && + (!check_argument || event.argument == argument)) { + return index; + } + } + return RecordingOps::events.size(); +} + +void TestAbiResetAndZeroEntryCommit() { + constexpr const char *kTest = "abi-reset-zero-entry"; + auto map = NewMap(); + RecordingOps::DisableEvents(); + + ExpectEqual(LoadControl(&map->committed_tasks.value), 0, kTest, "committed reset"); + ExpectEqual(LoadControl(&map->reclaim_upto.value), -1, kTest, "reclaim reset"); + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + if (LoadControl(&map->buckets[bucket].head.value) != 0 || + LoadControl(&map->buckets[bucket].tail.value) != 0) { + Expect(false, kTest, "bucket cursor reset"); + break; + } + } + for (uint32_t slot = 0; slot < kMapCapacity; ++slot) { + if (LoadControl(&map->slots[slot].seq.value) != kSharedMapEmptySeq) { + Expect(false, kTest, "slot seq reset"); + break; + } + } + for (uint32_t worker = 0; + worker < pa_scheduler::kWorkers; ++worker) { + if (LoadControl(&map->reader_done[worker].value) != -1) { + Expect(false, kTest, "reader progress reset"); + break; + } + } + for (uint32_t lane = 1; + lane < pa_scheduler::kSharedInsertTurnCapacity; + ++lane) { + if (LoadControl( + &map->insert_turn_extra[lane - 1U].value + ) != -1) { + Expect(false, kTest, "inactive insert turn reset"); + break; + } + } + + const std::vector empty; + Expect( + TryCommitTask(*map, 0, empty) == CommitResult::Committed, + kTest, "task 0 empty delta commit" + ); + Expect( + TryCommitTask(*map, 1, empty) == CommitResult::Committed, + kTest, "task 1 empty delta commit" + ); + ExpectEqual(LoadControl(&map->committed_tasks.value), 2, kTest, "empty commit sequencer"); + ExpectEqual(LoadControl(&map->reclaim_upto.value), -1, kTest, "empty commit reclaim"); + RecordingOps::ResetEvents(); + Expect( + !SharedPublishTaskCommit(*map, 1), + kTest, "repeated commit is rejected" + ); + ExpectEqual( + LoadControl(&map->committed_tasks.value), 2, kTest, + "repeated commit preserves the advanced frontier" + ); + Expect( + RecordingOps::events.size() == 1 && + RecordingOps::events[0].kind == + EventKind::CompareExchange && + RecordingOps::events[0].address == + &map->committed_tasks.value && + RecordingOps::events[0].argument == 2 && + RecordingOps::events[0].result == 2, + kTest, + "repeated commit performs one non-mutating CAS" + ); + StoreControl(&map->committed_tasks.value, 0); + RecordingOps::ResetEvents(); + Expect( + !SharedPublishTaskCommit(*map, 1), + kTest, "future commit is rejected" + ); + ExpectEqual( + LoadControl(&map->committed_tasks.value), 0, kTest, + "future commit preserves the earlier frontier" + ); + Expect( + RecordingOps::events.size() == 1 && + RecordingOps::events[0].kind == + EventKind::CompareExchange && + RecordingOps::events[0].address == + &map->committed_tasks.value && + RecordingOps::events[0].argument == 2 && + RecordingOps::events[0].result == 0, + kTest, + "future commit performs one non-mutating CAS" + ); + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + if (LoadControl(&map->buckets[bucket].tail.value) != 0) { + Expect(false, kTest, "empty commit changed a bucket tail"); + break; + } + } +} + +void TestPublicationOrderAndDoubleSeqCheck() { + constexpr const char *kTest = "publication-order-double-seq"; + auto map = NewMap(); + const SharedRegionValue entry = MakeRegion(0x100000000ULL, 0, 64, 0); + const uint32_t bucket = TensorMapHash(entry.buffer_addr); + SharedRegionSlot &slot = map->slots[SharedTensorMapSlotIndex(bucket, 0)]; + + RecordingOps::ResetEvents(); + Expect( + TryCommitTask(*map, 0, {entry}) == CommitResult::Committed, + kTest, "single entry commit" + ); + + const void *seq_address = + const_cast(&slot.seq.value); + const void *payload_address = &slot.payload; + const void *tail_address = + const_cast(&map->buckets[bucket].tail.value); + const size_t seq_invalidate = + FindEvent(EventKind::Exchange, seq_address, 0, true, kSharedMapEmptySeq); + const size_t payload_invalidate = + FindEvent(EventKind::Invalidate, payload_address, seq_invalidate + 1); + const size_t payload_flush = + FindEvent(EventKind::Flush, payload_address, payload_invalidate + 1); + const size_t seq_publish = + FindEvent(EventKind::Exchange, seq_address, payload_flush + 1, true, 0); + const size_t tail_publish = + FindEvent(EventKind::Exchange, tail_address, seq_publish + 1, true, 1); + Expect( + seq_invalidate < payload_invalidate && + payload_invalidate < payload_flush && + payload_flush < seq_publish && seq_publish < tail_publish, + kTest, "writer event order" + ); + + RecordingOps::ResetEvents(); + SharedRegionValue snapshot{}; + Expect( + SharedReadRegionSlot(*map, bucket, 0, snapshot), + kTest, "published slot read" + ); + Expect(ToTuple(snapshot) == ToTuple(entry), kTest, "published payload contents"); + const size_t first_load = FindEvent(EventKind::Load, seq_address, 0); + const size_t read_invalidate = + FindEvent(EventKind::Invalidate, payload_address, first_load + 1); + const size_t second_load = + FindEvent(EventKind::Load, seq_address, read_invalidate + 1); + Expect( + first_load < read_invalidate && read_invalidate < second_load, + kTest, "reader acquire-invalidate-double-check order" + ); + + RecordingOps::ResetEvents(); + RecordingOps::mutate_payload = payload_address; + RecordingOps::mutate_seq = &slot.seq.value; + RecordingOps::mutate_seq_value = + static_cast(kMapBucketCapacity); + RecordingOps::mutate_once = true; + SharedRegionValue raced_snapshot{}; + Expect( + !SharedReadRegionSlot( + *map, bucket, 0, raced_snapshot + ), + kTest, "second seq check rejects deterministic ABA" + ); + Expect(!RecordingOps::mutate_once, kTest, "ABA injection point reached"); +} + +void TestVersionsWindowAndMultipleBuckets() { + constexpr const char *kTest = "versions-window-buckets"; + auto map = NewMap(); + RecordingOps::DisableEvents(); + + const uint64_t versioned_address = 0x200000000ULL; + const uint32_t versioned_bucket = TensorMapHash(versioned_address); + const uint64_t stale_address = + FindAddressOutsideBucket(0x210000000ULL, versioned_bucket); + const uint32_t stale_bucket = TensorMapHash(stale_address); + const uint64_t lower_address = + FindAddressOutsideBucket(0x220000000ULL, stale_bucket); + const uint64_t future_address = + FindAddressOutsideBucket(0x230000000ULL, TensorMapHash(lower_address)); + + const std::vector task0 = { + MakeRegion(versioned_address, 0, 64, 0), + MakeRegion(versioned_address, 128, 192, 0), + MakeRegion(stale_address, 0, 64, 0), + }; + const std::vector task1 = { + MakeRegion(versioned_address, 0, 64, 1), + MakeRegion(lower_address, 0, 64, 1), + }; + const std::vector task2 = { + MakeRegion(versioned_address, 0, 64, 2), + MakeRegion(future_address, 0, 64, 2), + }; + Expect(TryCommitTask(*map, 0, task0) == CommitResult::Committed, kTest, "task 0"); + Expect(TryCommitTask(*map, 1, task1) == CommitResult::Committed, kTest, "task 1"); + Expect(TryCommitTask(*map, 2, task2) == CommitResult::Committed, kTest, "task 2"); + uint32_t expected_versioned_bucket_entries = 0; + for (const auto *task : {&task0, &task1, &task2}) { + for (const SharedRegionValue &entry : *task) { + if (TensorMapHash(entry.buffer_addr) == versioned_bucket) { + ++expected_versioned_bucket_entries; + } + } + } + ExpectEqual( + LoadControl(&map->buckets[versioned_bucket].tail.value), + expected_versioned_bucket_entries, + kTest, "same bucket multi-entry tail" + ); + + bool protocol_ok = false; + int32_t producer = SharedLookupRegion( + *map, MakeRegion(versioned_address, 16, 32, -1), 3, 2, protocol_ok + ); + Expect(protocol_ok, kTest, "versioned lookup protocol"); + ExpectEqual(producer, 2, kTest, "same address maximum producer"); + + producer = SharedLookupRegion( + *map, MakeRegion(versioned_address, 16, 32, -1), 2, 2, protocol_ok + ); + Expect(protocol_ok, kTest, "historical lookup protocol"); + ExpectEqual(producer, 1, kTest, "future producer excluded"); + + producer = SharedLookupRegion( + *map, MakeRegion(stale_address, 0, 32, -1), 3, 2, protocol_ok + ); + Expect(protocol_ok, kTest, "stale lookup protocol"); + ExpectEqual(producer, -1, kTest, "producer below N-H excluded"); + + producer = SharedLookupRegion( + *map, MakeRegion(lower_address, 0, 32, -1), 3, 2, protocol_ok + ); + Expect(protocol_ok, kTest, "lower boundary protocol"); + ExpectEqual(producer, 1, kTest, "producer at N-H accepted"); + + producer = SharedLookupRegion( + *map, MakeRegion(future_address, 0, 32, -1), 2, 2, protocol_ok + ); + Expect(protocol_ok, kTest, "upper boundary protocol"); + ExpectEqual(producer, -1, kTest, "producer at N excluded"); + + producer = SharedLookupRegion( + *map, MakeRegion(versioned_address, 64, 128, -1), 3, 3, protocol_ok + ); + Expect(protocol_ok, kTest, "half-open lookup protocol"); + ExpectEqual(producer, -1, kTest, "touching half-open ranges do not overlap"); +} + +void TestOrderedReclaimFormulaAndExactTurn() { + constexpr const char *kTest = "ordered-reclaim-exact-turn"; + int64_t candidate = -2; + Expect( + SharedComputeOrderedReclaimCandidate(0, 64, candidate), + kTest, "task 0 reclaim formula" + ); + ExpectEqual(candidate, -1, kTest, "task 0 reclaim boundary"); + Expect( + SharedComputeOrderedReclaimCandidate(64, 64, candidate), + kTest, "task H reclaim formula" + ); + ExpectEqual(candidate, -1, kTest, "task H reclaim boundary"); + Expect( + SharedComputeOrderedReclaimCandidate(65, 64, candidate), + kTest, "task H+1 reclaim formula" + ); + ExpectEqual(candidate, 0, kTest, "task H+1 inclusive reclaim"); + Expect( + SharedComputeOrderedReclaimCandidate(1279, 64, candidate), + kTest, "Case1 final task reclaim formula" + ); + ExpectEqual(candidate, 1214, kTest, "Case1 final task reclaim boundary"); + Expect( + !SharedComputeOrderedReclaimCandidate(-1, 64, candidate), + kTest, "negative task rejected" + ); + Expect( + !SharedComputeOrderedReclaimCandidate(0, -1, candidate), + kTest, "negative heap window rejected" + ); + + { + auto ahead_map = NewMap(); + StoreControl(&ahead_map->committed_tasks.value, 65); + StoreControl(&ahead_map->reclaim_upto.value, 1); + RecordingOps::ResetEvents(); + int64_t rejected = -2; + Expect( + !SharedRefreshReclaimForTask( + *ahead_map, 65, 64, rejected + ), + kTest, "reclaim state ahead of candidate rejected" + ); + ExpectEqual( + LoadControl(&ahead_map->reclaim_upto.value), 1, + kTest, "ahead reclaim state preserved" + ); + for (const Event &event : RecordingOps::events) { + if (event.kind == EventKind::Exchange) { + Expect(false, kTest, "reclaim regression issued Exchange"); + break; + } + } + } + + auto map = NewMap(); + RecordingOps::DisableEvents(); + const uint64_t address = 0x300000000ULL; + const uint32_t bucket = TensorMapHash(address); + + Expect( + TryCommitTask(*map, 0, {MakeRegion(address, 0, 32, 0)}) == + CommitResult::Committed, + kTest, "producer 0 commit" + ); + Expect( + TryCommitTask(*map, 1, {MakeRegion(address, 64, 96, 1)}) == + CommitResult::Committed, + kTest, "producer 1 commit" + ); + const std::vector empty; + Expect( + TryCommitTask(*map, 2, empty, 2) == CommitResult::Committed, + kTest, "boundary zero-entry task commit" + ); + ExpectEqual( + LoadControl(&map->committed_tasks.value), 3, + kTest, "sequencer before exact turn" + ); + ExpectEqual( + LoadControl(&map->reclaim_upto.value), -1, + kTest, "task N=H does not reclaim" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].head.value), 0, + kTest, "boundary task keeps head" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].tail.value), 2, + kTest, "boundary task keeps tail" + ); + + RecordingOps::ResetEvents(); + int64_t reclaim_upto = -2; + Expect( + !SharedRefreshReclaimForTask( + *map, 2, 2, reclaim_upto + ), + kTest, "stale actor rejected" + ); + Expect( + !SharedRefreshReclaimForTask( + *map, 4, 2, reclaim_upto + ), + kTest, "future actor rejected" + ); + for (const Event &event : RecordingOps::events) { + if (event.kind == EventKind::Exchange || + event.kind == EventKind::Flush) { + Expect(false, kTest, "out-of-turn actor changed shared state"); + break; + } + } + ExpectEqual( + LoadControl(&map->reclaim_upto.value), -1, + kTest, "out-of-turn actor preserves reclaim" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].head.value), 0, + kTest, "out-of-turn actor preserves head" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].tail.value), 2, + kTest, "out-of-turn actor preserves tail" + ); + + RecordingOps::DisableEvents(); + Expect( + SharedHasExactTaskTurn(*map, 3), + kTest, "task 3 owns exact turn" + ); + Expect( + TryCommitTask(*map, 3, empty, 2) == CommitResult::Committed, + kTest, "zero-entry task advances ordered reclaim" + ); + ExpectEqual( + LoadControl(&map->committed_tasks.value), 4, + kTest, "zero-entry task publishes commit" + ); + ExpectEqual( + LoadControl(&map->reclaim_upto.value), 0, + kTest, "zero-entry task advances inclusive reclaim" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].head.value), 0, + kTest, "zero-entry task does not scan unrelated bucket" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].tail.value), 2, + kTest, "zero-entry task does not append" + ); + + reclaim_upto = LoadControl(&map->reclaim_upto.value); + Expect( + SharedRetireBucket(*map, bucket, reclaim_upto), + kTest, "lazy retire at published boundary" + ); + ExpectEqual(LoadControl(&map->buckets[bucket].head.value), 1, kTest, "producer 0 retired"); + + bool protocol_ok = false; + const int32_t producer = SharedLookupRegion( + *map, MakeRegion(address, 64, 96, -1), 2, 2, protocol_ok + ); + Expect(protocol_ok, kTest, "surviving producer lookup protocol"); + ExpectEqual(producer, 1, kTest, "producer above reclaim boundary survives"); +} + +void TestAbsoluteSeqMultipleLapsAndAba() { + constexpr const char *kTest = "absolute-seq-multiple-laps"; + auto map = NewMap(); + RecordingOps::DisableEvents(); + const uint64_t address = 0x400000000ULL; + const uint32_t bucket = TensorMapHash(address); + constexpr uint32_t kIterations = 3 * kMapBucketCapacity + 7; + + for (uint32_t task = 0; task < kIterations; ++task) { + const CommitResult result = TryCommitTask( + *map, static_cast(task), + {MakeRegion(address, 0, 64, static_cast(task))}, + 0 + ); + if (result != CommitResult::Committed) { + std::fprintf( + stderr, "[FAIL] shared TensorMap ring/%s: commit failed task=%u\n", + kTest, task + ); + ++g_failures; + return; + } + if (task == kMapBucketCapacity || + task == 2 * kMapBucketCapacity || + task == 3 * kMapBucketCapacity) { + SharedRegionSlot &wrapped = + map->slots[SharedTensorMapSlotIndex(bucket, task)]; + ExpectEqual( + LoadControl(&wrapped.seq.value), task, kTest, + "absolute seq after physical wrap" + ); + } + } + + ExpectEqual( + LoadControl(&map->buckets[bucket].tail.value), kIterations, + kTest, "absolute tail after laps" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].head.value), kIterations - 1, + kTest, "single live entry after laps" + ); + ExpectEqual( + LoadControl(&map->reclaim_upto.value), kIterations - 2, + kTest, "three-lap ordered reclaim boundary" + ); + const uint64_t last_cursor = kIterations - 1; + SharedRegionSlot &last_slot = + map->slots[SharedTensorMapSlotIndex(bucket, last_cursor)]; + ExpectEqual( + LoadControl(&last_slot.seq.value), last_cursor, kTest, + "latest absolute seq" + ); + + bool protocol_ok = false; + const int32_t producer = SharedLookupRegion( + *map, MakeRegion(address, 0, 64, -1), + static_cast(kIterations), 1, protocol_ok + ); + Expect(protocol_ok, kTest, "post-wrap lookup protocol"); + ExpectEqual(producer, kIterations - 1, kTest, "post-wrap latest producer"); + + RecordingOps::ResetEvents(); + RecordingOps::mutate_payload = &last_slot.payload; + RecordingOps::mutate_seq = &last_slot.seq.value; + RecordingOps::mutate_seq_value = + static_cast(last_cursor + kMapBucketCapacity); + RecordingOps::mutate_once = true; + SharedRegionValue raced{}; + Expect( + !SharedReadRegionSlot( + *map, bucket, last_cursor, raced + ), + kTest, "double check rejects next-lap seq" + ); +} + +void TestCapacityFailureIsAllOrNothing() { + constexpr const char *kTest = "capacity-all-or-nothing"; + + // 空环正常写入只需既有 reclaim=-1，不应为了未来可能发生的容量反压 + // 无条件扫描全部 reader_done。事件记录锁定 fast path 没有 reader load。 + auto fast_map = NewMap(); + bool fast_readers_closed = true; + for (uint32_t worker = 0; + worker < pa_scheduler::kWorkers; ++worker) { + fast_readers_closed &= + SharedAdvanceReaderDone( + *fast_map, worker, 0 + ); + } + Expect( + fast_readers_closed, kTest, + "all fast-path actors close task 0 before append" + ); + RecordingOps::ResetEvents(); + Expect( + SharedTryAppendReaderGatedTask( + *fast_map, nullptr, 0, + pa_scheduler::kWorkers, 2 + ) == SharedAppendCheck::Ready && + RecordingOps::events.empty(), + kTest, "empty ordinary batch performs no shared access" + ); + const SharedRegionValue fast_entry = + MakeRegion(0x4F0000000ULL, 0, 8, 0); + RecordingOps::ResetEvents(); + Expect( + SharedTryAppendReaderGatedTask( + *fast_map, &fast_entry, 1, + pa_scheduler::kWorkers, 2 + ) == SharedAppendCheck::Ready, + kTest, "fast path appends without refreshing reader frontier" + ); + bool loaded_reader_progress = false; + for (const Event &event : RecordingOps::events) { + if (event.kind != EventKind::Load) { + continue; + } + for (uint32_t worker = 0; + worker < pa_scheduler::kWorkers; ++worker) { + const void *const reader_address = + const_cast( + &fast_map->reader_done[worker].value + ); + loaded_reader_progress |= + event.address == reader_address; + } + } + Expect( + !loaded_reader_progress, kTest, + "ready fast path performs zero reader-progress loads" + ); + ExpectEqual( + LoadControl(&fast_map->committed_tasks.value), + 0, kTest, + "fast reader-gated append does not use global exact turn" + ); + RecordingOps::DisableEvents(); + + auto map = NewMap(); + RecordingOps::DisableEvents(); + const uint64_t full_address = 0x500000000ULL; + const uint32_t full_bucket = TensorMapHash(full_address); + + for (uint32_t task = 0; task < kMapBucketCapacity; ++task) { + const CommitResult result = TryCommitTask( + *map, static_cast(task), + {MakeRegion( + full_address, static_cast(task) * 16, + static_cast(task) * 16 + 8, + static_cast(task) + )} + ); + if (result != CommitResult::Committed) { + std::fprintf( + stderr, "[FAIL] shared TensorMap ring/%s: fill failed task=%u\n", + kTest, task + ); + ++g_failures; + return; + } + } + + std::vector before; + Expect(SnapshotLogicalMap(*map, before), kTest, "snapshot before overflow"); + const int64_t full_head = LoadControl(&map->buckets[full_bucket].head.value); + const int64_t full_tail = LoadControl(&map->buckets[full_bucket].tail.value); + const uint64_t other_address = + FindAddressOutsideBucket(0x510000000ULL, full_bucket); + const uint32_t other_bucket = TensorMapHash(other_address); + const int64_t other_tail = LoadControl(&map->buckets[other_bucket].tail.value); + const uint32_t other_slot_index = + SharedTensorMapSlotIndex(other_bucket, static_cast(other_tail)); + const int64_t other_seq = + LoadControl(&map->slots[other_slot_index].seq.value); + + const std::vector overflowing = { + MakeRegion(other_address, 0, 8, kMapBucketCapacity), + MakeRegion(full_address, 4096, 4104, kMapBucketCapacity), + }; + // fill driver 只负责构造满桶，随后清掉它的旧 exact turn。active + // reader 尚未关闭任何 task，H=0 也只能得到 reclaim=-1。 + StoreControl(&map->committed_tasks.value, 0); + RecordingOps::ResetEvents(); + Expect( + SharedTryAppendReaderGatedTask( + *map, overflowing.data(), + static_cast(overflowing.size()), 1, 0 + ) == SharedAppendCheck::CapacityBlocked, + kTest, + "reader-gated batch rejects task if any target bucket is full" + ); + for (const Event &event : RecordingOps::events) { + if (event.kind == EventKind::Exchange || event.kind == EventKind::Flush) { + Expect(false, kTest, "capacity failure performed a state publication"); + break; + } + } + RecordingOps::DisableEvents(); + + std::vector after; + Expect(SnapshotLogicalMap(*map, after), kTest, "snapshot after overflow"); + Expect(after == before, kTest, "failed preflight preserves all logical entries"); + ExpectEqual( + LoadControl(&map->committed_tasks.value), 0, + kTest, "capacity failure does not use global exact turn" + ); + ExpectEqual( + LoadControl(&map->buckets[full_bucket].head.value), full_head, + kTest, "failed task preserves full head" + ); + ExpectEqual( + LoadControl(&map->buckets[full_bucket].tail.value), full_tail, + kTest, "failed task preserves full tail" + ); + ExpectEqual( + LoadControl(&map->buckets[other_bucket].tail.value), other_tail, + kTest, "earlier preflight entry did not partially append" + ); + ExpectEqual( + LoadControl(&map->slots[other_slot_index].seq.value), other_seq, + kTest, "earlier preflight entry did not publish seq" + ); + + // 另建生产可达的有序写入历史：task 0 只写一条，后续 task 每批最多 + // kMaxTaskTensors 条直至填满。下一 writer 的 reader 前沿只允许回收 + // producer 0，恰好空出一个槽；同桶两项仍必须整批 CapacityBlocked。 + constexpr uint32_t kOneSlotFillTasks = + 1U + + (kMapBucketCapacity - 1U + kMaxTaskTensors - 1U) / + kMaxTaskTensors; + static_assert( + kOneSlotFillTasks < pa_scheduler::kMaxTasks, + "one-slot ordered history exceeds task domain" + ); + auto one_slot_map = NewMap(); + const uint64_t one_slot_address = 0x50A000000ULL; + const uint32_t one_slot_bucket = + TensorMapHash(one_slot_address); + uint32_t one_slot_cursor = 0; + int32_t one_slot_task = 0; + while (one_slot_cursor < kMapBucketCapacity) { + const uint32_t remaining = + kMapBucketCapacity - one_slot_cursor; + const uint32_t batch_count = + one_slot_task == 0 + ? 1U + : (remaining < kMaxTaskTensors + ? remaining + : kMaxTaskTensors); + std::vector batch; + batch.reserve(batch_count); + for (uint32_t index = 0; + index < batch_count; ++index) { + const uint64_t lo = + static_cast( + one_slot_cursor + index + ) * + 16U; + batch.push_back(MakeRegion( + one_slot_address, lo, lo + 8U, + one_slot_task + )); + } + if (TryCommitTask( + *one_slot_map, one_slot_task, batch + ) != CommitResult::Committed) { + std::fprintf( + stderr, + "[FAIL] shared TensorMap ring/%s: " + "one-slot fill failed task=%d\n", + kTest, one_slot_task + ); + ++g_failures; + return; + } + one_slot_cursor += batch_count; + ++one_slot_task; + } + ExpectEqual( + one_slot_task, kOneSlotFillTasks, kTest, + "one-slot history uses the expected task count" + ); + StoreControl(&one_slot_map->committed_tasks.value, 0); + bool one_slot_reader_closed = true; + for (int32_t task = 0; + task <= one_slot_task; ++task) { + one_slot_reader_closed &= + SharedAdvanceReaderDone( + *one_slot_map, 0, task + ); + } + Expect( + one_slot_reader_closed, kTest, + "single reader closes the current ordered writer task" + ); + const uint64_t next_lo = + static_cast(kMapBucketCapacity) * 16U; + const std::vector two_for_one_slot = { + MakeRegion( + one_slot_address, next_lo, next_lo + 8U, + one_slot_task + ), + MakeRegion( + one_slot_address, next_lo + 16U, + next_lo + 24U, one_slot_task + ), + }; + RecordingOps::ResetEvents(); + Expect( + SharedTryAppendReaderGatedTask( + *one_slot_map, two_for_one_slot.data(), + static_cast(two_for_one_slot.size()), + 1, one_slot_task + ) == SharedAppendCheck::CapacityBlocked, + kTest, + "one free slot cannot partially accept a two-entry batch" + ); + bool flushed_partial_entry = false; + for (const Event &event : RecordingOps::events) { + flushed_partial_entry |= event.kind == EventKind::Flush; + } + Expect( + !flushed_partial_entry, kTest, + "one-slot retry publishes no batch payload" + ); + RecordingOps::DisableEvents(); + ExpectEqual( + LoadControl(&one_slot_map->reclaim_upto.value), + 0, kTest, + "closed reader exposes exactly producer 0" + ); + ExpectEqual( + LoadControl( + &one_slot_map->buckets[one_slot_bucket].head.value + ), + 1, kTest, + "one safe producer leaves exactly one free slot" + ); + ExpectEqual( + LoadControl( + &one_slot_map->buckets[one_slot_bucket].tail.value + ), + kMapBucketCapacity, kTest, + "blocked two-entry batch preserves tail" + ); + SharedRegionSlot &first_reusable_slot = + one_slot_map->slots[ + SharedTensorMapSlotIndex( + one_slot_bucket, kMapBucketCapacity + ) + ]; + ExpectEqual( + LoadControl(&first_reusable_slot.seq.value), + 0, kTest, + "blocked two-entry batch does not claim the one free slot" + ); + ExpectEqual( + LoadControl(&one_slot_map->committed_tasks.value), + 0, kTest, + "one-slot retry remains independent of exact turn" + ); + + // 容量反压之外，再锁定后项协议损坏同样不能让前项先发布。不同桶形态 + // 使用第二桶 cursor 0；单桶 CAP=16384 形态使用同桶 cursor 1。 + auto protocol_map = NewMap(); + const uint64_t first_address = 0x511000000ULL; + const uint32_t first_bucket = TensorMapHash(first_address); + const uint64_t second_address = + FindAddressOutsideBucket(0x512000000ULL, first_bucket); + const uint32_t second_bucket = TensorMapHash(second_address); + const uint64_t second_cursor = + second_bucket == first_bucket ? 1U : 0U; + SharedRegionSlot &bad_slot = + protocol_map->slots[ + SharedTensorMapSlotIndex( + second_bucket, second_cursor + ) + ]; + StoreControl(&bad_slot.seq.value, 7); + const std::vector invalid_batch = { + MakeRegion(first_address, 0, 8, 0), + MakeRegion(second_address, 16, 24, 0), + }; + RecordingOps::ResetEvents(); + Expect( + SharedTryAppendReaderGatedTask( + *protocol_map, invalid_batch.data(), + static_cast(invalid_batch.size()), 1, 0 + ) == SharedAppendCheck::ProtocolError, + kTest, + "later slot corruption rejects the whole reader-gated batch" + ); + for (const Event &event : RecordingOps::events) { + if (event.kind == EventKind::Exchange || + event.kind == EventKind::Flush || + event.kind == EventKind::CompareExchange) { + Expect( + false, kTest, + "protocol rejection published an earlier batch entry" + ); + break; + } + } + RecordingOps::DisableEvents(); + ExpectEqual( + LoadControl( + &protocol_map->buckets[first_bucket].tail.value + ), + 0, kTest, + "later protocol error preserves the earlier bucket tail" + ); + SharedRegionSlot &first_protocol_slot = + protocol_map->slots[ + SharedTensorMapSlotIndex(first_bucket, 0) + ]; + ExpectEqual( + LoadControl(&first_protocol_slot.seq.value), + kSharedMapEmptySeq, kTest, + "later protocol error preserves the earlier slot seq" + ); + ExpectEqual( + LoadControl(&protocol_map->committed_tasks.value), + 0, kTest, + "protocol rejection does not use global exact turn" + ); +} + +void TestSlowReaderGatesFullBucketReuse() { + constexpr const char *kTest = + "slow-reader-gates-full-bucket-reuse"; + constexpr int32_t kHeapWindow = 2; + constexpr uint32_t kActiveReaders = 2; + constexpr uint32_t kEntriesPerTask = 8; + static_assert( + kEntriesPerTask <= kMaxTaskTensors, + "reader interleave batch exceeds task tensor capacity" + ); + static_assert( + kMapBucketCapacity % kEntriesPerTask == 0, + "full-ring interleave requires whole task batches" + ); + constexpr uint32_t kFillTasks = + kMapBucketCapacity / kEntriesPerTask; + static_assert( + kFillTasks > static_cast(kHeapWindow), + "fast reader must reach the reclaim boundary" + ); + static_assert( + kFillTasks < pa_scheduler::kMaxTasks, + "full-ring interleave must stay inside reader task domain" + ); + + auto map = NewMap(); + RecordingOps::DisableEvents(); + const uint64_t address = 0x520000000ULL; + const uint32_t bucket = TensorMapHash(address); + + // 用每 task 八条合法 region 填满同一桶。即使 CAP=16384，future task + // 也只有 2048，仍处于 reader_done 的 kMaxTasks 合同内。 + for (uint32_t task = 0; task < kFillTasks; ++task) { + std::vector entries; + entries.reserve(kEntriesPerTask); + for (uint32_t index = 0; + index < kEntriesPerTask; ++index) { + const uint64_t cursor = + static_cast(task) * + kEntriesPerTask + + index; + const uint64_t lo = cursor * 32U; + entries.push_back(MakeRegion( + address, lo, lo + 8U, + static_cast(task) + )); + } + if (TryCommitTask( + *map, static_cast(task), + entries + ) != CommitResult::Committed) { + std::fprintf( + stderr, + "[FAIL] shared TensorMap ring/%s: " + "fill failed task=%u\n", + kTest, task + ); + ++g_failures; + return; + } + } + + const int32_t writer_task = + static_cast(kFillTasks); + ExpectEqual( + LoadControl(&map->committed_tasks.value), + writer_task, kTest, "fill driver reaches the future writer task" + ); + StoreControl(&map->committed_tasks.value, 0); + ExpectEqual( + LoadControl(&map->committed_tasks.value), + 0, kTest, "reader-gated writer has no global exact turn" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].head.value), + 0, kTest, "full ring starts at cursor zero" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].tail.value), + kMapBucketCapacity, kTest, "target bucket is exactly full" + ); + + // worker 0 已关闭 task 0/1，但仍在 task 2 的 ordinary lookup； + // worker 1 作为快 reader 已追到 future writer 前一 task。 + bool reader_progress_ok = true; + for (int32_t task = 0; task <= 1; ++task) { + reader_progress_ok &= + SharedAdvanceReaderDone( + *map, 0, task + ); + } + for (int32_t task = 0; task <= writer_task; ++task) { + reader_progress_ok &= + SharedAdvanceReaderDone( + *map, 1, task + ); + } + Expect( + reader_progress_ok, kTest, + "slow/fast readers publish contiguous progress" + ); + ExpectEqual( + LoadControl(&map->reader_done[0].value), + 1, kTest, "slow reader remains inside task 2" + ); + ExpectEqual( + LoadControl(&map->reader_done[1].value), + writer_task, kTest, + "writer closes its own task before reader-gated append" + ); + + const uint64_t independent_address = + FindAddressOutsideBucket(0x521000000ULL, bucket); + const uint32_t independent_bucket = + TensorMapHash(independent_address); + const int64_t independent_tail_before = + LoadControl( + &map->buckets[independent_bucket].tail.value + ); + const uint32_t independent_slot_index = + SharedTensorMapSlotIndex( + independent_bucket, + static_cast(independent_tail_before) + ); + const int64_t independent_seq_before = + LoadControl( + &map->slots[independent_slot_index].seq.value + ); + + std::vector replacements; + replacements.reserve(kEntriesPerTask + 1U); + if constexpr (kMapBuckets > 1) { + // 独立空桶 entry 故意排在满桶 batch 前。若组合 helper 错误地 + // 边检查边 append，它会在后项 CapacityBlocked 前留下部分发布。 + replacements.push_back(MakeRegion( + independent_address, 0, 8, writer_task + )); + } + for (uint32_t index = 0; + index < kEntriesPerTask; ++index) { + const uint64_t lo = + (static_cast(kMapBucketCapacity) + + index) * + 32U; + replacements.push_back(MakeRegion( + address, lo, lo + 8U, writer_task + )); + } + + SharedRegionSlot &first_slot = + map->slots[SharedTensorMapSlotIndex(bucket, 0)]; + SlowReaderReuseAttempt attempt{}; + attempt.map = map.get(); + attempt.replacements = replacements.data(); + attempt.replacement_count = + static_cast(replacements.size()); + attempt.bucket = bucket; + attempt.active_readers = kActiveReaders; + attempt.heap_window = kHeapWindow; + attempt.reclaim_upto = -2; + attempt.append_result = SharedAppendCheck::ProtocolError; + attempt.head = -2; + attempt.tail = -2; + attempt.committed_tasks = -2; + attempt.slot_seq = -2; + attempt.slot_producer = -2; + + // 把 future writer 的 refresh/preflight 精确插在 slow reader 首次 + // seq 检查之后、拷贝 cursor-0 payload 之前。 + RecordingOps::ResetEvents(); + RecordingOps::invalidate_hook_address = + &first_slot.payload; + RecordingOps::invalidate_hook = + AttemptReuseWhileReaderPaused; + RecordingOps::invalidate_hook_context = &attempt; + bool protocol_ok = false; + const int32_t producer = + SharedLookupRegion( + *map, MakeRegion(address, 0, 8, -1), + 2, kHeapWindow, protocol_ok + ); + + Expect( + protocol_ok && producer == 0, kTest, + "paused task-2 lookup still consumes producer 0" + ); + Expect(attempt.fired, kTest, "paused-reader hook fired"); + Expect( + attempt.reclaim_upto == -1, + kTest, + "slow reader keeps the global reclaim frontier at -1" + ); + Expect( + attempt.append_result == + SharedAppendCheck::CapacityBlocked, + kTest, + "reader-gated batch remains retryable while task 2 is open" + ); + ExpectEqual( + attempt.head, 0, kTest, + "blocked reuse preserves bucket head" + ); + ExpectEqual( + attempt.tail, kMapBucketCapacity, kTest, + "blocked reuse preserves bucket tail" + ); + ExpectEqual( + attempt.committed_tasks, 0, kTest, + "blocked reuse remains independent of global exact turn" + ); + ExpectEqual( + attempt.slot_seq, 0, kTest, + "blocked reuse preserves cursor-0 absolute seq" + ); + ExpectEqual( + attempt.slot_producer, 0, kTest, + "blocked reuse preserves cursor-0 payload" + ); + if constexpr (kMapBuckets > 1) { + ExpectEqual( + LoadControl( + &map->buckets[independent_bucket].tail.value + ), + independent_tail_before, kTest, + "blocked later bucket does not append the earlier entry" + ); + ExpectEqual( + LoadControl( + &map->slots[independent_slot_index].seq.value + ), + independent_seq_before, kTest, + "blocked later bucket does not publish earlier seq" + ); + } + for (const Event &event : RecordingOps::events) { + if (event.kind == EventKind::Exchange || + event.kind == EventKind::Flush || + event.kind == EventKind::CompareExchange) { + Expect( + false, kTest, + "paused-reader attempt published shared state" + ); + break; + } + } + RecordingOps::DisableEvents(); + + // lookup 已完整返回后才允许 slow reader 关闭 task 2。此时 Dmin=2， + // H=2 的 candidate 首次变成 0，producer 0 才可回收。 + Expect( + SharedAdvanceReaderDone( + *map, 0, 2 + ), + kTest, "slow reader closes task 2 after its final read" + ); + ExpectEqual( + LoadControl(&map->reader_done[0].value), + 2, kTest, "slow reader publishes its closed frontier" + ); + ExpectEqual( + LoadControl(&map->reader_done[1].value), + writer_task, kTest, + "fast reader frontier remains unchanged" + ); + Expect( + SharedTryAppendReaderGatedTask( + *map, replacements.data(), + static_cast(replacements.size()), + kActiveReaders, kHeapWindow + ) == SharedAppendCheck::Ready, + kTest, + "closed reader makes the same whole batch appendable" + ); + ExpectEqual( + LoadControl(&map->reclaim_upto.value), + 0, kTest, "closed readers publish producer-0 reclaim" + ); + + ExpectEqual( + LoadControl(&map->buckets[bucket].head.value), + kEntriesPerTask, kTest, + "all task-0 entries retire after reader close" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].tail.value), + kMapBucketCapacity + kEntriesPerTask, + kTest, "replacement batch advances tail" + ); + ExpectEqual( + LoadControl(&map->committed_tasks.value), + 0, kTest, + "successful reader-gated append does not publish exact turn" + ); + if constexpr (kMapBuckets > 1) { + ExpectEqual( + LoadControl( + &map->buckets[independent_bucket].tail.value + ), + independent_tail_before + 1, kTest, + "successful batch appends the independent first entry" + ); + } + ExpectEqual( + LoadControl(&first_slot.seq.value), + kMapBucketCapacity, kTest, + "cursor CAP reuses physical slot 0 with a new seq" + ); + ExpectEqual( + first_slot.payload.value.producer, + writer_task, kTest, + "reused slot contains only the future producer" + ); + + SharedRegionValue old_cursor{}; + Expect( + !SharedReadRegionSlot( + *map, bucket, 0, old_cursor + ), + kTest, "old cursor 0 is rejected after reuse" + ); + SharedRegionValue new_cursor{}; + Expect( + SharedReadRegionSlot( + *map, bucket, kMapBucketCapacity, + new_cursor + ) && + new_cursor.producer == writer_task, + kTest, "new cursor CAP resolves the replacement" + ); + std::vector snapshot; + Expect( + SnapshotLogicalMap(*map, snapshot), kTest, + "logical snapshot after reader-gated reuse" + ); + ExpectEqual( + snapshot.size(), + kMapBucketCapacity + + (kMapBuckets > 1 ? 1U : 0U), + kTest, + "whole batch keeps the target full and includes the independent entry" + ); + const int32_t old_region = + SharedLookupRegion( + *map, MakeRegion(address, 0, 8, -1), + 3, kHeapWindow, protocol_ok + ); + Expect( + protocol_ok && old_region == -1, kTest, + "task 3 no longer observes expired producer 0" + ); +} + +void TestLookupSkipsConcurrentlyRetiredSafePrefix() { + constexpr const char *kTest = + "lookup-skips-concurrently-retired-safe-prefix"; + constexpr int32_t kReaderTask = 2; + constexpr int32_t kHeapWindow = 1; + constexpr uint32_t kEntriesPerTask = 8; + constexpr uint32_t kActiveReaders = 1; + static_assert( + kMapBucketCapacity % kEntriesPerTask == 0, + "safe-prefix interleave requires whole task batches" + ); + constexpr uint32_t kFillTasks = + kMapBucketCapacity / kEntriesPerTask; + static_assert( + kFillTasks + 1U < pa_scheduler::kMaxTasks, + "safe-prefix writers must stay inside task domain" + ); + + auto map = NewMap(); + RecordingOps::DisableEvents(); + const uint64_t address = 0x530000000ULL; + const uint32_t bucket = TensorMapHash(address); + + // 用真实整 task append 把一桶填满：cursor 0..7 属于 producer 0， + // cursor 8..15 属于 producer 1。task 2 的窗口下界为 1，因此前八条 + // 已经失效，但 cursor 8 仍是本次查询必须返回的合法 producer。 + for (uint32_t task = 0; task < kFillTasks; ++task) { + std::vector entries; + entries.reserve(kEntriesPerTask); + for (uint32_t index = 0; + index < kEntriesPerTask; ++index) { + const uint64_t cursor = + static_cast(task) * + kEntriesPerTask + + index; + const uint64_t lo = cursor * 32U; + entries.push_back(MakeRegion( + address, lo, lo + 8U, + static_cast(task) + )); + } + if (TryCommitTask( + *map, static_cast(task), + entries + ) != CommitResult::Committed) { + std::fprintf( + stderr, + "[FAIL] shared TensorMap ring/%s: " + "fill failed task=%u\n", + kTest, task + ); + ++g_failures; + return; + } + } + + // 本门槛刻意撤掉隔离 driver 的 exact-turn 前沿。下面的合法回收只由 + // reader_done 推导，证明 lookup 的并发恢复不依赖 committed_tasks。 + StoreControl(&map->committed_tasks.value, 0); + Expect( + SharedAdvanceReaderDone( + *map, 0, 0 + ) && + SharedAdvanceReaderDone( + *map, 0, 1 + ), + kTest, "reader closes tasks before task 2" + ); + + const int32_t writer_task = + static_cast(kFillTasks); + std::vector replacements; + replacements.reserve(kEntriesPerTask); + for (uint32_t index = 0; + index < kEntriesPerTask; ++index) { + const uint64_t lo = + (static_cast(kMapBucketCapacity) + + index) * + 32U; + replacements.push_back(MakeRegion( + address, lo, lo + 8U, writer_task + )); + } + + SharedRegionSlot &first_slot = + map->slots[SharedTensorMapSlotIndex(bucket, 0)]; + ConcurrentSafeRetireAttempt attempt{}; + attempt.map = map.get(); + attempt.replacements = replacements.data(); + attempt.replacement_count = + static_cast(replacements.size()); + attempt.active_readers = kActiveReaders; + attempt.heap_window = kHeapWindow; + attempt.candidate = -2; + attempt.reclaim_upto = -2; + attempt.append_check = SharedAppendCheck::ProtocolError; + + // reader 已读旧 head=0、尚未读取 tail 时，让唯一 writer 回收 + // producer 0 并提交八条 replacement。reader 随后会读到新 + // tail=CAP+8，形成“旧 head + 新 tail”的合法混合快照；lookup 必须 + // 重读 head=8 后继续，而不能因表面跨度大于 CAP 误报协议损坏。 + RecordingOps::ResetEvents(); + RecordingOps::load_hook_address = + const_cast( + &map->buckets[bucket].tail.value + ); + RecordingOps::load_hook = + RetireSafePrefixWhileReaderPaused; + RecordingOps::load_hook_context = &attempt; + bool protocol_ok = false; + const uint64_t target_lo = + static_cast(kEntriesPerTask) * 32U; + const int32_t producer = + SharedLookupRegion( + *map, + MakeRegion( + address, target_lo, target_lo + 8U, -1 + ), + kReaderTask, kHeapWindow, protocol_ok + ); + + Expect( + attempt.fired, kTest, + "safe-retire hook fires between initial head and tail loads" + ); + Expect( + attempt.candidate_ok && attempt.candidate == 0 && + attempt.publish_ok && + attempt.reclaim_upto == 0, + kTest, "reader frontier admits only producer 0" + ); + Expect( + attempt.append_check == SharedAppendCheck::Ready && + attempt.append_ok, + kTest, "writer retires and replaces the safe prefix" + ); + Expect( + protocol_ok && producer == 1, + kTest, + "lookup repairs old-head/new-tail snapshot and returns producer 1" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].head.value), + kEntriesPerTask, kTest, + "safe retire advances head past producer 0" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].tail.value), + kMapBucketCapacity + kEntriesPerTask, + kTest, "replacement batch advances tail" + ); + ExpectEqual( + LoadControl(&map->committed_tasks.value), + 0, kTest, + "reader-gated append does not use global exact turn" + ); + ExpectEqual( + LoadControl(&first_slot.seq.value), + kMapBucketCapacity, kTest, + "physical slot 0 carries the replacement absolute seq" + ); + + size_t head_exchange = RecordingOps::events.size(); + size_t seq_invalidate = RecordingOps::events.size(); + size_t payload_flush = RecordingOps::events.size(); + size_t seq_publish = RecordingOps::events.size(); + size_t tail_publish = RecordingOps::events.size(); + const void *const head_address = + const_cast( + &map->buckets[bucket].head.value + ); + const void *const seq_address = + const_cast(&first_slot.seq.value); + const void *const tail_address = + const_cast( + &map->buckets[bucket].tail.value + ); + for (size_t index = 0; + index < RecordingOps::events.size(); ++index) { + const Event &event = RecordingOps::events[index]; + if (event.kind == EventKind::Exchange && + event.address == head_address && + event.argument == + static_cast(kEntriesPerTask) && + head_exchange == RecordingOps::events.size()) { + head_exchange = index; + } else if ( + event.kind == EventKind::Exchange && + event.address == seq_address && + event.argument == kSharedMapEmptySeq && + seq_invalidate == RecordingOps::events.size() + ) { + seq_invalidate = index; + } else if ( + event.kind == EventKind::Flush && + event.address == &first_slot.payload && + payload_flush == RecordingOps::events.size() + ) { + payload_flush = index; + } else if ( + event.kind == EventKind::Exchange && + event.address == seq_address && + event.argument == + static_cast(kMapBucketCapacity) && + seq_publish == RecordingOps::events.size() + ) { + seq_publish = index; + } else if ( + event.kind == EventKind::Exchange && + event.address == tail_address && + event.argument == + static_cast( + kMapBucketCapacity + 1U + ) && + tail_publish == RecordingOps::events.size() + ) { + tail_publish = index; + } + } + Expect( + head_exchange < seq_invalidate && + seq_invalidate < payload_flush && + payload_flush < seq_publish && + seq_publish < tail_publish, + kTest, + "head publication precedes slot reuse and tail publication" + ); + RecordingOps::DisableEvents(); + + // 再固定更晚的交错：reader task 3 已读 cursor 8 的旧绝对 seq， + // 尚未拷 payload 时，writer 回收 producer 1 并用 cursor CAP+8 + // 复用同一物理槽。第二次 seq 检查失败后，lookup 只能在新 head + // 确实越过 cursor 8 时跳到 cursor 16，并返回 producer 2。 + Expect( + SharedAdvanceReaderDone( + *map, 0, 2 + ), + kTest, "reader closes task 2 before task 3" + ); + const int32_t second_writer_task = writer_task + 1; + std::vector second_replacements; + second_replacements.reserve(kEntriesPerTask); + for (uint32_t index = 0; + index < kEntriesPerTask; ++index) { + const uint64_t lo = + (static_cast(kMapBucketCapacity) + + kEntriesPerTask + index) * + 32U; + second_replacements.push_back(MakeRegion( + address, lo, lo + 8U, second_writer_task + )); + } + ConcurrentSafeRetireAttempt second_attempt{}; + second_attempt.map = map.get(); + second_attempt.replacements = second_replacements.data(); + second_attempt.replacement_count = + static_cast(second_replacements.size()); + second_attempt.active_readers = kActiveReaders; + second_attempt.heap_window = kHeapWindow; + second_attempt.candidate = -2; + second_attempt.reclaim_upto = -2; + second_attempt.append_check = + SharedAppendCheck::ProtocolError; + + SharedRegionSlot &second_retired_slot = + map->slots[ + SharedTensorMapSlotIndex(bucket, kEntriesPerTask) + ]; + RecordingOps::ResetEvents(); + RecordingOps::invalidate_hook_address = + &second_retired_slot.payload; + RecordingOps::invalidate_hook = + RetireSafePrefixWhileReaderPaused; + RecordingOps::invalidate_hook_context = &second_attempt; + bool second_protocol_ok = false; + const uint64_t second_target_lo = + static_cast(2U * kEntriesPerTask) * 32U; + const int32_t second_producer = + SharedLookupRegion( + *map, + MakeRegion( + address, second_target_lo, + second_target_lo + 8U, -1 + ), + kReaderTask + 1, kHeapWindow, + second_protocol_ok + ); + Expect( + second_attempt.fired, kTest, + "safe-retire hook fires between slot seq checks" + ); + Expect( + second_attempt.candidate_ok && + second_attempt.candidate == 1 && + second_attempt.publish_ok && + second_attempt.reclaim_upto == 1, + kTest, "next reader frontier admits only producer 1" + ); + Expect( + second_attempt.append_check == + SharedAppendCheck::Ready && + second_attempt.append_ok, + kTest, "writer retires and replaces the next safe prefix" + ); + Expect( + second_protocol_ok && second_producer == 2, + kTest, + "lookup skips concurrently reused cursor 8 and returns producer 2" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].head.value), + 2U * kEntriesPerTask, kTest, + "second safe retire advances head past producer 1" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].tail.value), + kMapBucketCapacity + 2U * kEntriesPerTask, + kTest, "second replacement batch advances tail" + ); + ExpectEqual( + LoadControl(&map->committed_tasks.value), + 0, kTest, + "both reader-gated appends avoid global exact turn" + ); + RecordingOps::DisableEvents(); + + // 不能把任意 seq 错误都吞成并发回收。head 已停在 cursor 16 时， + // 破坏 cursor 16 的 seq；二次读取 head 没有越过它，lookup 必须继续 + // fail-closed。 + SharedRegionSlot &target_slot = + map->slots[ + SharedTensorMapSlotIndex( + bucket, 2U * kEntriesPerTask + ) + ]; + StoreControl( + &target_slot.seq.value, + static_cast(2U * kEntriesPerTask + 1U) + ); + bool corrupt_protocol_ok = true; + const int32_t corrupt_result = + SharedLookupRegion( + *map, + MakeRegion( + address, second_target_lo, + second_target_lo + 8U, -1 + ), + kReaderTask + 1, kHeapWindow, + corrupt_protocol_ok + ); + Expect( + !corrupt_protocol_ok && corrupt_result == -1, + kTest, + "seq corruption without head advance remains a protocol error" + ); + + // 控制字段真实损坏也不能借异常恢复蒙混过去。没有并发 head 前进时， + // 人为制造 CAP+1 的跨度，二次读取仍是旧 head，必须直接拒绝。 + auto oversized_map = NewMap(); + const uint32_t oversized_bucket = TensorMapHash(address); + StoreControl( + &oversized_map->buckets[oversized_bucket].tail.value, + static_cast(kMapBucketCapacity + 1U) + ); + bool oversized_protocol_ok = true; + const int32_t oversized_result = + SharedLookupRegion( + *oversized_map, + MakeRegion(address, 0, 8, -1), + 1, kHeapWindow, oversized_protocol_ok + ); + Expect( + !oversized_protocol_ok && oversized_result == -1, + kTest, + "oversized control span without head advance remains an error" + ); +} + +void TestFullBucketDoesNotBlockIndependentBucket() { + constexpr const char *kTest = "full-bucket-independent-bucket"; + if constexpr (kMapBuckets == 1) { + // CAP=16384 的 B=1 形态没有可构造的独立桶；它仍由满环、回绕和 + // 精确复用门槛覆盖，不能伪造一个“不同桶”结论。 + return; + } + + auto map = NewMap(); + RecordingOps::DisableEvents(); + const uint64_t full_address = 0x540000000ULL; + const uint32_t full_bucket = TensorMapHash(full_address); + for (uint32_t task = 0; task < kMapBucketCapacity; ++task) { + const CommitResult result = TryCommitTask( + *map, static_cast(task), + {MakeRegion( + full_address, static_cast(task) * 16, + static_cast(task) * 16 + 8, + static_cast(task) + )} + ); + if (result != CommitResult::Committed) { + Expect(false, kTest, "failed to fill the target bucket"); + return; + } + } + + const int64_t full_head = + LoadControl(&map->buckets[full_bucket].head.value); + const int64_t full_tail = + LoadControl(&map->buckets[full_bucket].tail.value); + const uint64_t independent_address = + FindAddressOutsideBucket(0x550000000ULL, full_bucket); + const uint32_t independent_bucket = + TensorMapHash(independent_address); + Expect( + independent_bucket != full_bucket, kTest, + "test address must map to an independent bucket" + ); + + const SharedRegionValue independent = MakeRegion( + independent_address, 0, 64, + static_cast(kMapBucketCapacity) + ); + Expect( + TryCommitTask( + *map, static_cast(kMapBucketCapacity), + {independent} + ) == CommitResult::Committed, + kTest, "a full bucket must not become a global capacity gate" + ); + ExpectEqual( + LoadControl(&map->buckets[full_bucket].head.value), full_head, + kTest, "independent append preserves full-bucket head" + ); + ExpectEqual( + LoadControl(&map->buckets[full_bucket].tail.value), full_tail, + kTest, "independent append preserves full-bucket tail" + ); + ExpectEqual( + LoadControl(&map->buckets[independent_bucket].tail.value), 1, + kTest, "independent bucket publishes one entry" + ); + SharedRegionSlot &slot = + map->slots[SharedTensorMapSlotIndex(independent_bucket, 0)]; + ExpectEqual( + LoadControl(&slot.seq.value), 0, + kTest, "independent bucket publishes its own absolute seq" + ); +} + +void TestFullBucketRetireAndReuseExactCapacity() { + constexpr const char *kTest = "full-bucket-retire-reuse-exact"; + auto map = NewMap(); + RecordingOps::DisableEvents(); + const uint64_t address = 0x560000000ULL; + const uint32_t bucket = TensorMapHash(address); + for (uint32_t task = 0; task < kMapBucketCapacity; ++task) { + const CommitResult result = TryCommitTask( + *map, static_cast(task), + {MakeRegion( + address, static_cast(task) * 32, + static_cast(task) * 32 + 8, + static_cast(task) + )} + ); + if (result != CommitResult::Committed) { + Expect(false, kTest, "failed to fill the target bucket"); + return; + } + } + + constexpr uint32_t kReuse = + kMapBucketCapacity / 4U < 8U + ? kMapBucketCapacity / 4U + : 8U; + static_assert(kReuse > 0, "exact reuse test requires a non-zero batch"); + std::vector replacements; + replacements.reserve(kReuse); + for (uint32_t index = 0; index < kReuse; ++index) { + const uint64_t lo = + (1ULL << 20U) + static_cast(index) * 32U; + replacements.push_back(MakeRegion( + address, lo, lo + 8U, + static_cast(kMapBucketCapacity) + )); + } + const int32_t heap_window = + static_cast(kMapBucketCapacity - kReuse); + Expect( + TryCommitTask( + *map, static_cast(kMapBucketCapacity), + replacements, heap_window + ) == CommitResult::Committed, + kTest, "retiring exactly K entries must admit exactly K replacements" + ); + ExpectEqual( + LoadControl(&map->reclaim_upto.value), kReuse - 1, + kTest, "inclusive reclaim boundary" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].head.value), kReuse, + kTest, "bucket head retires exactly K old entries" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].tail.value), + kMapBucketCapacity + kReuse, + kTest, "bucket tail appends exactly K replacement entries" + ); + for (uint32_t index = 0; index < kReuse; ++index) { + const uint64_t cursor = + static_cast(kMapBucketCapacity) + index; + SharedRegionSlot &slot = + map->slots[SharedTensorMapSlotIndex(bucket, cursor)]; + ExpectEqual( + LoadControl(&slot.seq.value), cursor, + kTest, "reused physical slot publishes the new absolute seq" + ); + ExpectEqual( + slot.payload.value.producer, kMapBucketCapacity, + kTest, "reused physical slot contains the replacement producer" + ); + } + std::vector snapshot; + Expect(SnapshotLogicalMap(*map, snapshot), kTest, "snapshot after exact reuse"); + ExpectEqual( + snapshot.size(), kMapBucketCapacity, + kTest, "exact reuse keeps the bucket logically full" + ); +} + +void TestCapacityBlockedAfterSafeRetire() { + constexpr const char *kTest = "capacity-after-safe-retire"; + auto map = NewMap(); + RecordingOps::DisableEvents(); + const uint64_t full_address = 0x580000000ULL; + const uint32_t full_bucket = TensorMapHash(full_address); + + for (uint32_t task = 0; task < kMapBucketCapacity; ++task) { + const CommitResult result = TryCommitTask( + *map, static_cast(task), + {MakeRegion( + full_address, static_cast(task) * 16, + static_cast(task) * 16 + 8, + static_cast(task) + )} + ); + if (result != CommitResult::Committed) { + std::fprintf( + stderr, "[FAIL] shared TensorMap ring/%s: fill failed task=%u\n", + kTest, task + ); + ++g_failures; + return; + } + } + + const uint64_t other_address = + FindAddressOutsideBucket(0x590000000ULL, full_bucket); + const uint32_t other_bucket = TensorMapHash(other_address); + const int64_t other_tail = + LoadControl(&map->buckets[other_bucket].tail.value); + const uint32_t other_slot_index = + SharedTensorMapSlotIndex(other_bucket, static_cast(other_tail)); + const int64_t other_seq = + LoadControl(&map->slots[other_slot_index].seq.value); + const std::vector overflowing = { + MakeRegion(other_address, 0, 8, kMapBucketCapacity), + MakeRegion(full_address, 4096, 4104, kMapBucketCapacity), + MakeRegion(full_address, 4112, 4120, kMapBucketCapacity), + }; + + RecordingOps::ResetEvents(); + int64_t reclaim_upto = -2; + Expect( + SharedRefreshReclaimForTask( + *map, kMapBucketCapacity, kMapBucketCapacity - 1, + reclaim_upto + ), + kTest, "exact turn advances reclaim to producer 0" + ); + ExpectEqual(reclaim_upto, 0, kTest, "inclusive stale boundary"); + ExpectEqual( + LoadControl(&map->reclaim_upto.value), 0, + kTest, "published stale boundary" + ); + Expect( + SharedCheckTaskAppend( + *map, overflowing.data(), + static_cast(overflowing.size()), reclaim_upto + ) == SharedAppendCheck::CapacityBlocked, + kTest, "two same-bucket entries still exceed capacity" + ); + for (const Event &event : RecordingOps::events) { + if (event.kind == EventKind::Flush) { + Expect(false, kTest, "capacity failure flushed a payload"); + break; + } + } + RecordingOps::DisableEvents(); + + ExpectEqual( + LoadControl(&map->committed_tasks.value), kMapBucketCapacity, + kTest, "failed task not committed" + ); + ExpectEqual( + LoadControl(&map->buckets[full_bucket].head.value), 1, + kTest, "only stale producer 0 retired" + ); + ExpectEqual( + LoadControl(&map->buckets[full_bucket].tail.value), kMapBucketCapacity, + kTest, "failed task did not publish full-bucket tail" + ); + ExpectEqual( + LoadControl(&map->buckets[other_bucket].tail.value), other_tail, + kTest, "earlier target bucket did not partially append" + ); + ExpectEqual( + LoadControl(&map->slots[other_slot_index].seq.value), other_seq, + kTest, "earlier target bucket did not publish seq" + ); + + std::vector after; + Expect(SnapshotLogicalMap(*map, after), kTest, "snapshot after safe retire"); + ExpectEqual( + static_cast(after.size()), + static_cast(kMapBucketCapacity - 1), + kTest, "only one stale logical entry removed" + ); + const bool published_failed_task = std::any_of( + after.begin(), after.end(), + [](const LogicalTuple &entry) { + return entry.producer == + static_cast(kMapBucketCapacity); + } + ); + Expect( + !published_failed_task, kTest, + "capacity failure published a current-task entry" + ); +} + +void TestDeterministicArrivalAndLogicalTupleDifference() { + constexpr const char *kTest = "deterministic-arrival-logical-diff"; + auto map = NewMap(); + RecordingOps::DisableEvents(); + const uint64_t address_a = 0x600000000ULL; + const uint64_t address_b = + FindAddressOutsideBucket(0x610000000ULL, TensorMapHash(address_a)); + const uint64_t address_c = + FindAddressOutsideBucket(0x620000000ULL, TensorMapHash(address_b)); + + const std::vector> deltas = { + {}, + {MakeRegion(address_a, 0, 64, 1)}, + { + MakeRegion(address_b, 0, 32, 2), + MakeRegion(address_c, 64, 96, 2), + }, + {}, + {MakeRegion(address_a, 0, 64, 4)}, + {MakeRegion(address_b, 16, 48, 5)}, + }; + constexpr uint32_t kTaskCount = 6; + constexpr uint32_t kArrivalOrder[kTaskCount] = {4, 2, 5, 3, 1, 0}; + bool committed[kTaskCount] = {}; + uint32_t commit_counts[kTaskCount] = {}; + uint32_t committed_count = 0; + uint32_t rounds = 0; + + while (committed_count < kTaskCount && rounds < kTaskCount + 1) { + for (uint32_t actor : kArrivalOrder) { + if (committed[actor]) { + continue; + } + const int64_t heads_before = [&]() { + int64_t total = 0; + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + total += LoadControl(&map->buckets[bucket].head.value); + } + return total; + }(); + const int64_t tails_before = [&]() { + int64_t total = 0; + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + total += LoadControl(&map->buckets[bucket].tail.value); + } + return total; + }(); + const int64_t reclaim_before = + LoadControl(&map->reclaim_upto.value); + const CommitResult result = TryCommitTask( + *map, static_cast(actor), deltas[actor] + ); + if (result == CommitResult::Pending) { + int64_t heads_after = 0; + int64_t tails_after = 0; + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + heads_after += + LoadControl(&map->buckets[bucket].head.value); + tails_after += + LoadControl(&map->buckets[bucket].tail.value); + } + ExpectEqual( + heads_after, heads_before, kTest, + "reverse actor preserves all heads" + ); + ExpectEqual( + tails_after, tails_before, kTest, + "reverse actor preserves all tails" + ); + ExpectEqual( + LoadControl(&map->reclaim_upto.value), + reclaim_before, kTest, + "reverse actor preserves reclaim" + ); + continue; + } + if (result == CommitResult::Failed) { + Expect(false, kTest, "in-turn commit failed"); + return; + } + committed[actor] = true; + ++commit_counts[actor]; + ++committed_count; + if (deltas[actor].empty()) { + int64_t tails_after = 0; + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + tails_after += LoadControl(&map->buckets[bucket].tail.value); + } + ExpectEqual( + tails_after, tails_before, kTest, + "zero-entry task does not append" + ); + } + } + ++rounds; + } + + ExpectEqual(committed_count, kTaskCount, kTest, "all actors eventually commit"); + ExpectEqual( + LoadControl(&map->committed_tasks.value), kTaskCount, + kTest, "final ordered sequencer" + ); + for (uint32_t task = 0; task < kTaskCount; ++task) { + if (commit_counts[task] != 1) { + Expect(false, kTest, "each task committed exactly once"); + break; + } + } + + std::vector actual; + Expect(SnapshotLogicalMap(*map, actual), kTest, "final logical snapshot"); + std::vector expected; + for (const auto &delta : deltas) { + for (const SharedRegionValue &entry : delta) { + expected.push_back(ToTuple(entry)); + } + } + std::sort(expected.begin(), expected.end()); + Expect(actual == expected, kTest, "logical tuple vector matches fixed reference"); +} + +void TestPreparedBucketPlanMatchesGenericPath() { + constexpr const char *kTest = "prepared-bucket-plan"; + auto generic_map = NewMap(); + auto prepared_map = NewMap(); + RecordingOps::DisableEvents(); + + const uint64_t address_a0 = 0x690000000ULL; + const uint32_t bucket_a = TensorMapHash(address_a0); + const uint64_t address_b = FindAddressOutsideBucket( + 0x691000000ULL, bucket_a + ); + const uint32_t bucket_b = TensorMapHash(address_b); + const uint64_t address_a1 = FindAddressInsideBucket( + 0x692000000ULL, bucket_a + ); + const uint64_t address_a2 = FindAddressInsideBucket( + address_a1 + 64, bucket_a + ); + const SharedRegionValue entries[4] = { + MakeRegion(address_a0, 0, 64, 7), + MakeRegion(address_b, 64, 128, 7), + MakeRegion(address_a1, 128, 192, 7), + MakeRegion(address_a2, 192, 256, 7), + }; + const uint16_t buckets[4] = { + static_cast(bucket_a), + static_cast(bucket_b), + static_cast(bucket_a), + static_cast(bucket_a), + }; + const uint8_t ordinals[4] = { + 0, + static_cast(kMapBuckets == 1 ? 1 : 0), + static_cast(kMapBuckets == 1 ? 2 : 1), + static_cast(kMapBuckets == 1 ? 3 : 2), + }; + + Expect( + SharedCheckTaskAppend( + *generic_map, entries, 4, -1 + ) == SharedAppendCheck::Ready && + SharedCheckPreparedTaskAppend( + *prepared_map, entries, buckets, ordinals, 4, -1 + ) == SharedAppendCheck::Ready, + kTest, "generic and prepared preflight both accept the plan" + ); + Expect( + SharedAppendPreparedTask( + *generic_map, entries, 4 + ) && + SharedAppendPreparedTask( + *prepared_map, entries, buckets, 4 + ), + kTest, "generic and prepared append both publish the plan" + ); + + std::vector generic_snapshot; + std::vector prepared_snapshot; + Expect( + SnapshotLogicalMap(*generic_map, generic_snapshot) && + SnapshotLogicalMap(*prepared_map, prepared_snapshot) && + generic_snapshot == prepared_snapshot, + kTest, "prepared plan preserves the generic logical map" + ); + ExpectEqual( + LoadControl( + &prepared_map->buckets[bucket_a].tail.value + ), + kMapBuckets == 1 ? 4 : 3, + kTest, "prepared A bucket tail" + ); + if constexpr (kMapBuckets > 1) { + ExpectEqual( + LoadControl( + &prepared_map->buckets[bucket_b].tail.value + ), + 1, kTest, "prepared B bucket tail" + ); + } + + auto blocked_map = NewMap(); + RecordingOps::DisableEvents(); + StoreControl( + &blocked_map->buckets[bucket_a].head.value, 0 + ); + StoreControl( + &blocked_map->buckets[bucket_a].tail.value, + static_cast(kMapBucketCapacity - 1) + ); + for (uint32_t cursor = 0; + cursor + 1 < kMapBucketCapacity; ++cursor) { + StoreControl( + &blocked_map + ->slots[ + SharedTensorMapSlotIndex(bucket_a, cursor) + ] + .seq.value, + static_cast(cursor) + ); + } + const int64_t tail_before = LoadControl( + &blocked_map->buckets[bucket_a].tail.value + ); + RecordingOps::ResetEvents(); + Expect( + SharedCheckPreparedTaskAppend( + *blocked_map, entries, buckets, ordinals, 4, -1 + ) == SharedAppendCheck::CapacityBlocked, + kTest, "prepared preflight rejects an over-capacity bucket" + ); + bool mutation_seen = false; + for (const Event &event : RecordingOps::events) { + mutation_seen |= + event.kind == EventKind::Exchange || + event.kind == EventKind::CompareExchange || + event.kind == EventKind::Flush || + event.kind == EventKind::Invalidate; + } + Expect( + !mutation_seen && + LoadControl( + &blocked_map->buckets[bucket_a].tail.value + ) == tail_before, + kTest, "capacity failure performs no partial publication" + ); + RecordingOps::DisableEvents(); +} + +void TestNoRetirePreflightSkipsPublishedPayload() { + constexpr const char *kTest = "no-retire-preflight"; + auto map = NewMap(); + RecordingOps::DisableEvents(); + const uint64_t address = 0x698000000ULL; + const SharedRegionValue published = + MakeRegion(address, 0, 64, 0); + const SharedRegionValue next = + MakeRegion(address, 64, 128, 1); + Expect( + TryCommitTask(*map, 0, {published}) == + CommitResult::Committed, + kTest, "seed one published entry" + ); + + const uint32_t bucket = TensorMapHash(address); + SharedRegionSlot &published_slot = + map->slots[SharedTensorMapSlotIndex(bucket, 0)]; + SharedRegionSlot &target_slot = + map->slots[SharedTensorMapSlotIndex(bucket, 1)]; + const void *published_seq = + const_cast( + &published_slot.seq.value + ); + const void *target_seq = + const_cast(&target_slot.seq.value); + const void *head = + const_cast( + &map->buckets[bucket].head.value + ); + const void *tail = + const_cast( + &map->buckets[bucket].tail.value + ); + + RecordingOps::ResetEvents(); + Expect( + SharedCheckTaskAppend( + *map, &next, 1, -1 + ) == SharedAppendCheck::Ready, + kTest, "no-reclaim preflight remains ready" + ); + Expect( + RecordingOps::events.size() == 3 && + FindEvent(EventKind::Load, head, 0) < + RecordingOps::events.size() && + FindEvent(EventKind::Load, tail, 0) < + RecordingOps::events.size() && + FindEvent(EventKind::Load, target_seq, 0) < + RecordingOps::events.size(), + kTest, + "no-reclaim preflight keeps head/tail/target-seq checks" + ); + Expect( + FindEvent(EventKind::Load, published_seq, 0) == + RecordingOps::events.size() && + FindEvent( + EventKind::Invalidate, + &published_slot.payload, 0 + ) == RecordingOps::events.size(), + kTest, + "reclaim -1 does not read or invalidate a published payload" + ); + + RecordingOps::ResetEvents(); + Expect( + SharedCheckTaskAppend( + *map, &next, 1, -2 + ) == SharedAppendCheck::ProtocolError && + RecordingOps::events.empty(), + kTest, + "invalid reclaim boundary fails before shared-state access" + ); + RecordingOps::DisableEvents(); +} + +void TestTailOverflowRejectedBeforeMutation() { + constexpr const char *kTest = "tail-overflow"; + auto map = NewMap(); + RecordingOps::DisableEvents(); + const SharedRegionValue entry = + MakeRegion(0x6A0000000ULL, 0, 64, 7); + const uint32_t bucket = + TensorMapHash(entry.buffer_addr); + StoreControl( + &map->buckets[bucket].head.value, INT64_MAX + ); + StoreControl( + &map->buckets[bucket].tail.value, INT64_MAX + ); + + RecordingOps::ResetEvents(); + Expect( + SharedCheckTaskAppend( + *map, &entry, 1, -1 + ) == SharedAppendCheck::ProtocolError, + kTest, + "preflight rejects a tail with no representable successor" + ); + Expect( + !pa_scheduler::SharedAppendPreparedEntry( + *map, entry + ), + kTest, + "append rejects a tail with no representable successor" + ); + for (const Event &event : RecordingOps::events) { + if (event.kind == EventKind::Exchange || + event.kind == EventKind::Flush) { + Expect( + false, kTest, + "overflow rejection performed a state publication" + ); + break; + } + } + RecordingOps::DisableEvents(); + ExpectEqual( + LoadControl(&map->buckets[bucket].head.value), + INT64_MAX, kTest, "overflow rejection preserves head" + ); + ExpectEqual( + LoadControl(&map->buckets[bucket].tail.value), + INT64_MAX, kTest, "overflow rejection preserves tail" + ); +} + +} // namespace + +int main() { + TestPhysicalSlotBoundaries(); + TestAbiResetAndZeroEntryCommit(); + TestPublicationOrderAndDoubleSeqCheck(); + TestVersionsWindowAndMultipleBuckets(); + TestOrderedReclaimFormulaAndExactTurn(); + TestAbsoluteSeqMultipleLapsAndAba(); + TestCapacityFailureIsAllOrNothing(); + TestSlowReaderGatesFullBucketReuse(); + TestLookupSkipsConcurrentlyRetiredSafePrefix(); + TestFullBucketDoesNotBlockIndependentBucket(); + TestFullBucketRetireAndReuseExactCapacity(); + TestCapacityBlockedAfterSafeRetire(); + TestDeterministicArrivalAndLogicalTupleDifference(); + TestPreparedBucketPlanMatchesGenericPath(); + TestNoRetirePreflightSkipsPublishedPayload(); + TestTailOverflowRejectedBeforeMutation(); + + if (g_failures != 0) { + std::fprintf(stderr, "[FAIL] shared TensorMap ring: %d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf( + "[PASS] shared TensorMap ring CAP=%u buckets=%u: " + "ABI, ordered commit, slow-reader reclaim, absolute seq, " + "concurrent safe-prefix retire, ABA, overflow, logical differential\n", + kMapBucketCapacity, kMapBuckets + ); + std::printf( + "[NOTE] CPU validates atomic/order hooks only; it does not simulate A5 DCache or DCCI.\n" + ); + return EXIT_SUCCESS; +} diff --git a/tests/atomic_probe/pa_scheduler/test/test_shared_vector_claim_cursor.cpp b/tests/atomic_probe/pa_scheduler/test/test_shared_vector_claim_cursor.cpp new file mode 100644 index 0000000000..b7064aa351 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test/test_shared_vector_claim_cursor.cpp @@ -0,0 +1,280 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define PA_DEVICE inline +#define PA_GM +#include "pa_scheduler_core.h" + +namespace { + +using namespace pa_scheduler; + +constexpr std::array kTaskKinds = { + TaskKind::Alloc, TaskKind::Qk, TaskKind::Sf, TaskKind::Pv, TaskKind::Up, +}; +constexpr std::array kTaskIds = { + 100, 101, 102, 103, 104, +}; + +int g_failures = 0; + +struct ClaimTestOps { + static constexpr bool kAtomicReturnReadyObserved = false; + static inline thread_local uint64_t fetch_max_calls = 0; + static inline thread_local volatile int64_t *last_fetch_max_address = nullptr; + + static int32_t Exchange(volatile int32_t *address, int32_t value) { + return __atomic_exchange_n(address, value, __ATOMIC_ACQ_REL); + } + + static int64_t FetchMax(volatile int64_t *address, int64_t value, uint64_t &retries) { + ++fetch_max_calls; + last_fetch_max_address = address; + int64_t current = __atomic_load_n(address, __ATOMIC_ACQUIRE); + retries = 0; + while (value > current) { + if (__atomic_compare_exchange_n(address, ¤t, value, true, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) { + break; + } + ++retries; + } + return current; + } + + static uint64_t Now() { return 0; } + + template + static uint64_t NowAfterAtomicResult(T) { + return 0; + } + + static void ResetThreadTrace() { + fetch_max_calls = 0; + last_fetch_max_address = nullptr; + } +}; + +struct ClaimEvidence { + ClaimOutcome outcome{}; + uint64_t fetch_max_calls = 0; + volatile int64_t *fetch_max_address = nullptr; +}; + +struct CursorValues { + std::array cube{}; + std::array vector{}; + std::array alloc{}; + std::array shared_vector{}; +}; + +void Check(bool condition, const char *message) { + if (condition) { + return; + } + std::fprintf(stderr, "[FAIL] %s\n", message); + ++g_failures; +} + +template +T *MapSparseObject() { + int flags = MAP_PRIVATE | MAP_ANONYMOUS; +#ifdef MAP_NORESERVE + flags |= MAP_NORESERVE; +#endif + void *memory = mmap(nullptr, sizeof(T), PROT_READ | PROT_WRITE, flags, -1, 0); + if (memory == MAP_FAILED) { + return nullptr; + } + return ::new (memory) T; +} + +void InitializeClaimCursors(SchedulerState &state, CursorValues &expected) { + expected.cube.fill(-1); + expected.vector.fill(-1); + expected.alloc.fill(-1); + expected.shared_vector.fill(-1); + for (uint32_t shard = 0; shard < kCursorShards; ++shard) { + state.cube_cursor[shard].value = -1; + state.vector_cursor[shard].value = -1; + state.alloc_cursor[shard].value = -1; + } + for (uint32_t shard = 0; shard < kSharedVectorCursorCapacity; ++shard) { + state.shared_map.shared_vector_cursor[shard].value = -1; + } +} + +volatile int64_t *ExpectedClaimAddress(SchedulerState &state, uint32_t task_id, TaskKind kind) { + if (kind == TaskKind::Alloc) { + return &state.alloc_cursor[task_id % kCursorShards].value; + } + if (kind == TaskKind::Qk || kind == TaskKind::Pv) { + return &state.cube_cursor[task_id % kCursorShards].value; + } + return &state.shared_map.shared_vector_cursor[task_id % kSharedVectorCursorShards].value; +} + +void RecordExpectedCursor(CursorValues &expected, uint32_t task_id, TaskKind kind) { + if (kind == TaskKind::Alloc) { + expected.alloc[task_id % kCursorShards] = task_id; + } else if (kind == TaskKind::Qk || kind == TaskKind::Pv) { + expected.cube[task_id % kCursorShards] = task_id; + } else { + expected.shared_vector[task_id % kSharedVectorCursorShards] = task_id; + } +} + +bool CursorsMatch(const SchedulerState &state, const CursorValues &expected) { + for (uint32_t shard = 0; shard < kCursorShards; ++shard) { + if (state.cube_cursor[shard].value != expected.cube[shard] || + state.vector_cursor[shard].value != expected.vector[shard] || + state.alloc_cursor[shard].value != expected.alloc[shard]) { + return false; + } + } + for (uint32_t shard = 0; shard < kSharedVectorCursorCapacity; ++shard) { + if (state.shared_map.shared_vector_cursor[shard].value != expected.shared_vector[shard]) { + return false; + } + } + return true; +} + +uint32_t ExpectedCandidates(TaskKind kind) { + switch (kind) { + case TaskKind::Alloc: + return kWorkers; + case TaskKind::Qk: + case TaskKind::Pv: + return kAicWorkers; + case TaskKind::Sf: + case TaskKind::Up: + return kAivWorkers; + case TaskKind::Count: + return 0; + } + return 0; +} + +bool IsCandidate(TaskKind kind, uint32_t worker_id) { + if (kind == TaskKind::Alloc) { + return true; + } + const bool is_aic = worker_id < kAicWorkers; + return kind == TaskKind::Qk || kind == TaskKind::Pv ? is_aic : !is_aic; +} + +bool RunConcurrentClaim(SchedulerState &state, uint32_t task_id, TaskKind kind) { + state.tasks[task_id].deps_prepared = -1; + state.fatal.value = 0; + std::array evidence{}; + std::atomic ready{0}; + std::atomic start{false}; + std::vector threads; + threads.reserve(kWorkers); + + for (uint32_t worker_id = 0; worker_id < kWorkers; ++worker_id) { + WorkerState &worker = state.workers[worker_id]; + worker.role = worker_id < kAicWorkers ? CoreRole::Aic : CoreRole::Aiv; + threads.emplace_back([&state, &worker, &evidence, &ready, &start, worker_id, task_id, kind]() { + ClaimTestOps::ResetThreadTrace(); + ready.fetch_add(1, std::memory_order_release); + while (!start.load(std::memory_order_acquire)) {} + LocalStats stats{}; + evidence[worker_id].outcome = Claim(&state, worker, task_id, kind, stats); + evidence[worker_id].fetch_max_calls = ClaimTestOps::fetch_max_calls; + evidence[worker_id].fetch_max_address = ClaimTestOps::last_fetch_max_address; + }); + } + + while (ready.load(std::memory_order_acquire) != kWorkers) {} + start.store(true, std::memory_order_release); + for (std::thread &thread : threads) { + thread.join(); + } + + uint32_t attempts = 0; + uint32_t winners = 0; + bool exact = true; + volatile int64_t *expected_address = ExpectedClaimAddress(state, task_id, kind); + for (uint32_t worker_id = 0; worker_id < kWorkers; ++worker_id) { + const bool candidate = IsCandidate(kind, worker_id); + const ClaimEvidence &entry = evidence[worker_id]; + attempts += entry.outcome.attempted ? 1U : 0U; + winners += entry.outcome.won ? 1U : 0U; + exact &= entry.outcome.attempted == candidate && entry.fetch_max_calls == (candidate ? 1U : 0U) && + entry.fetch_max_address == (candidate ? expected_address : nullptr); + if (entry.outcome.won) { + exact &= kind == TaskKind::Alloc ? entry.outcome.function_id == -1 : + entry.outcome.function_id == FunctionId(kind); + } else { + exact &= entry.outcome.function_id == -1; + } + } + return exact && attempts == ExpectedCandidates(kind) && winners == 1 && + *expected_address == static_cast(task_id) && state.tasks[task_id].deps_prepared == -1 && + state.fatal.value == 0; +} + +void TestAllTaskKindsUseCursorClaim() { + SchedulerState *state = MapSparseObject(); + Check(state != nullptr, "sparse shared cursor Claim fixture maps successfully"); + if (state == nullptr) { + return; + } + + CursorValues expected{}; + InitializeClaimCursors(*state, expected); + bool exact = true; + for (uint32_t index = 0; index < kTaskKinds.size(); ++index) { + exact &= RunConcurrentClaim(*state, kTaskIds[index], kTaskKinds[index]); + RecordExpectedCursor(expected, kTaskIds[index], kTaskKinds[index]); + exact &= CursorsMatch(*state, expected); + for (uint32_t observed = 0; observed <= index; ++observed) { + exact &= state->tasks[kTaskIds[observed]].deps_prepared == -1; + } + } + + WorkerState &replay_worker = state->workers[0]; + replay_worker.role = CoreRole::Aic; + ClaimTestOps::ResetThreadTrace(); + LocalStats replay_stats{}; + const ClaimOutcome replay = Claim(state, replay_worker, kTaskIds[1], TaskKind::Qk, replay_stats); + exact &= replay.attempted && !replay.won && replay.function_id == -1 && ClaimTestOps::fetch_max_calls == 1 && + ClaimTestOps::last_fetch_max_address == ExpectedClaimAddress(*state, kTaskIds[1], TaskKind::Qk) && + state->fatal.value == 0 && CursorsMatch(*state, expected); + + Check( + exact, "all task kinds keep cursor routing, exact candidates, " + "one winner, legal replay, and untouched deps_prepared" + ); + (void)munmap(state, sizeof(*state)); +} + +} // namespace + +int main() { + TestAllTaskKindsUseCursorClaim(); + if (g_failures != 0) { + std::fprintf(stderr, "[FAIL] shared cursor Claim tests: %d\n", g_failures); + return 1; + } + std::printf("[PASS] shared cursor Claim tests\n"); + return 0; +} diff --git a/tests/atomic_probe/pa_scheduler/test/test_shared_writer_intent.cpp b/tests/atomic_probe/pa_scheduler/test/test_shared_writer_intent.cpp new file mode 100644 index 0000000000..e36d76f5e6 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test/test_shared_writer_intent.cpp @@ -0,0 +1,2178 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include "pa_scheduler_core.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace pa_scheduler; + +int g_failures = 0; + +void Check(bool condition, const char *message) { + if (condition) { + return; + } + std::fprintf(stderr, "[FAIL] shared writer intent: %s\n", message); + ++g_failures; +} + +// 只验证公共 writer-intent 状态机。CPU acquire/release 保证宿主线程测试 +// 没有普通 data race，但不模拟 A5 DCache；CCEC 编译和后续 A5 litmus +// 分别承担设备接口与跨核可见性证据。 +struct WriterIntentTestOps { + static constexpr bool kAtomicReturnReadyObserved = false; + static volatile int64_t *wait_address; + static std::atomic wait_loads; + static volatile int64_t *cas_trigger_address; + static volatile int64_t *cas_conflict_address; + static int64_t cas_conflict_value; + + static int32_t Load(volatile int32_t *address) { + return __atomic_fetch_add( + address, static_cast(0), __ATOMIC_ACQUIRE + ); + } + + static int64_t Load(volatile int64_t *address) { + if (address == wait_address) { + wait_loads.fetch_add(1, std::memory_order_release); + } + return __atomic_fetch_add( + address, static_cast(0), __ATOMIC_ACQUIRE + ); + } + + static int32_t Exchange( + volatile int32_t *address, int32_t value + ) { + return __atomic_exchange_n( + address, value, __ATOMIC_ACQ_REL + ); + } + + static int64_t Exchange( + volatile int64_t *address, int64_t value + ) { + return __atomic_exchange_n( + address, value, __ATOMIC_ACQ_REL + ); + } + + static uint64_t Exchange( + volatile uint64_t *address, uint64_t value + ) { + return __atomic_exchange_n( + address, value, __ATOMIC_ACQ_REL + ); + } + + static int64_t CompareExchange( + volatile int64_t *address, int64_t expected, + int64_t desired + ) { + // 只供多 symbol terminal-prefix 门槛：在第一项真正 CAS 前， + // 原子推进第二项，稳定制造“第一项成功、第二项冲突”的时序。 + if (address == cas_trigger_address && + cas_conflict_address != nullptr) { + __atomic_store_n( + cas_conflict_address, cas_conflict_value, + __ATOMIC_RELEASE + ); + cas_trigger_address = nullptr; + cas_conflict_address = nullptr; + } + int64_t observed = expected; + (void)__atomic_compare_exchange_n( + address, &observed, desired, false, + __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE + ); + return observed; + } + + static int64_t FetchMax( + volatile int64_t *address, int64_t value, + uint64_t &retries + ) { + int64_t current = + __atomic_load_n(address, __ATOMIC_ACQUIRE); + retries = 0; + while (value > current) { + if (__atomic_compare_exchange_n( + address, ¤t, value, true, + __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE + )) { + break; + } + ++retries; + } + return current; + } + + static void StoreBarrier() { + std::atomic_thread_fence(std::memory_order_seq_cst); + } + + static void FlushRegion(void *, uint64_t) { + std::atomic_thread_fence(std::memory_order_seq_cst); + } + + static void InvalidateRegion(const void *, uint64_t) { + std::atomic_thread_fence(std::memory_order_seq_cst); + } + + static uint64_t Now() { + return static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch() + ).count() + ); + } + + template + static uint64_t NowAfterAtomicResult(T value) { + asm volatile("" : "+r"(value)); + return Now(); + } + + static void SpinHint() { std::this_thread::yield(); } +}; + +volatile int64_t *WriterIntentTestOps::wait_address = nullptr; +std::atomic WriterIntentTestOps::wait_loads{0}; +volatile int64_t *WriterIntentTestOps::cas_trigger_address = + nullptr; +volatile int64_t *WriterIntentTestOps::cas_conflict_address = + nullptr; +int64_t WriterIntentTestOps::cas_conflict_value = -1; + +SchedulerState *MapSparseSchedulerState() { + int flags = MAP_PRIVATE | MAP_ANONYMOUS; +#ifdef MAP_NORESERVE + flags |= MAP_NORESERVE; +#endif + void *memory = mmap( + nullptr, sizeof(SchedulerState), + PROT_READ | PROT_WRITE, flags, -1, 0 + ); + if (memory == MAP_FAILED) { + std::perror("mmap SchedulerState"); + return nullptr; + } + return ::new (memory) SchedulerState; +} + +void UnmapSparseSchedulerState(SchedulerState *state) { + if (state != nullptr) { + (void)munmap(state, sizeof(SchedulerState)); + } +} + +void ResetProtocolState(SchedulerState &state) { + state.fatal.value = 0; + state.heap_window = kHeapWindow; + for (uint32_t lane = 0; + lane < kSharedInsertTurnCapacity; ++lane) { + SharedInsertTurnLine( + state.shared_map, lane + ).value = -10000 - static_cast(lane); + } + state.shared_map.reclaim_upto.value = -1; + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + state.shared_map.buckets[bucket].head.value = 0; + state.shared_map.buckets[bucket].tail.value = 0; + } + for (uint32_t slot = 0; slot < kMapCapacity; ++slot) { + state.shared_map.slots[slot].seq.value = + kSharedMapEmptySeq; + } + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + state.shared_map.reader_done[worker].value = -1; + } + for (uint32_t task = 0; task < kMaxTasks; ++task) { + state.tasks[task].deps_prepared = -1; + SharedOutputCell &outputs = + state.shared_map.shared_outputs[task]; + for (uint32_t output = 0; + output < kSharedOutputMaxPerTask; ++output) { + outputs.published[output].value = -1; + outputs.last_writer[output].value = -1; + } + SharedWriterHistoryCell &history = + state.shared_map.writer_history[task]; + history.magic = 0; + history.writer_task = 0; + history.count = 0; + history.reserved = 0; + } +} + +void SetInsertCompletionsAfterTasks( + SchedulerState &state, uint32_t completed_tasks +) { + for (uint32_t task = 0; task < completed_tasks; + ++task) { + state.tasks[task].deps_prepared = + static_cast(task); + } +} + +bool InsertCompletionsMatch( + SchedulerState &state, uint32_t completed_tasks +) { + for (uint32_t task = 0; task < completed_tasks; + ++task) { + if (WriterIntentTestOps::Load( + &state.tasks[task].deps_prepared + ) != static_cast(task)) { + return false; + } + } + if (completed_tasks < kMaxTasks && + WriterIntentTestOps::Load( + &state.tasks[completed_tasks].deps_prepared + ) != -1) { + return false; + } + for (uint32_t lane = 0; + lane < kSharedInsertTurnCapacity; ++lane) { + if (WriterIntentTestOps::Load( + &SharedInsertTurnLine( + state.shared_map, lane + ).value + ) != + -10000 - static_cast(lane)) { + return false; + } + } + return true; +} + +void ResetTaskGate(SchedulerState &state, int32_t task_id) { + state.tasks[static_cast(task_id)].flag = 0; + state.tasks[static_cast(task_id)].deps_prepared = -1; +} + +TensorDesc MakeTensor( + uint64_t address, uint64_t owner = kInvalidTaskId, + bool manual_dep = false +) { + TensorDesc tensor{}; + tensor.buffer_addr = address; + tensor.buffer_size = 4096; + tensor.owner_task_id = owner; + tensor.ndims = 1; + tensor.dtype = DataType::Float32; + tensor.manual_dep = manual_dep; + tensor.is_contiguous = true; + tensor.shapes[0] = 1024; + tensor.strides[0] = 1; + tensor.extent_elem_cache = 1024; + return tensor; +} + +uint64_t FindAddressForBucket( + uint64_t begin, uint32_t wanted_bucket, + uint32_t skip_matches = 0 +) { + for (uint64_t address = begin; + address < begin + (1ULL << 28); address += 4096) { + if (TensorMapHash(address) != wanted_bucket) { + continue; + } + if (skip_matches == 0) { + return address; + } + --skip_matches; + } + return 0; +} + +bool WaitUntilObserved(std::atomic &counter) { + const auto deadline = + std::chrono::steady_clock::now() + + std::chrono::seconds(2); + while (counter.load(std::memory_order_acquire) == 0) { + if (std::chrono::steady_clock::now() >= deadline) { + return false; + } + std::this_thread::yield(); + } + return true; +} + +bool WaitUntilTrue(std::atomic &value) { + const auto deadline = + std::chrono::steady_clock::now() + + std::chrono::seconds(2); + while (!value.load(std::memory_order_acquire)) { + if (std::chrono::steady_clock::now() >= deadline) { + return false; + } + std::this_thread::yield(); + } + return true; +} + +int64_t LoadReaderDone( + SchedulerState &state, uint32_t worker +) { + return WriterIntentTestOps::Load( + &state.shared_map.reader_done[worker].value + ); +} + +void StoreReaderDone( + SchedulerState &state, uint32_t worker, int64_t value +) { + __atomic_store_n( + &state.shared_map.reader_done[worker].value, + value, __ATOMIC_RELEASE + ); +} + +void TestReaderProgressStateMachine(SchedulerState &state) { + bool reset_ok = true; + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + reset_ok &= LoadReaderDone(state, worker) == -1; + } + Check(reset_ok, "all reader progress lines start at -1"); + + Check( + SharedAdvanceReaderDone( + state.shared_map, 0, 0 + ), + "reader progress accepts the exact -1 to 0 transition" + ); + Check( + !SharedAdvanceReaderDone( + state.shared_map, 0, 0 + ) && + LoadReaderDone(state, 0) == 0, + "reader progress rejects a duplicate without overwriting" + ); + Check( + !SharedAdvanceReaderDone( + state.shared_map, 0, 2 + ) && + LoadReaderDone(state, 0) == 0, + "reader progress rejects a skipped task without overwriting" + ); + Check( + SharedAdvanceReaderDone( + state.shared_map, 0, 1 + ), + "reader progress accepts the next contiguous task" + ); + Check( + !SharedAdvanceReaderDone( + state.shared_map, 0, 0 + ) && + LoadReaderDone(state, 0) == 1, + "reader progress rejects a backward task without overwriting" + ); + + StoreReaderDone( + state, 1, static_cast(kMaxTasks) - 2 + ); + Check( + SharedAdvanceReaderDone( + state.shared_map, 1, + static_cast(kMaxTasks) - 1 + ) && + LoadReaderDone(state, 1) == + static_cast(kMaxTasks) - 1, + "reader progress accepts the last task in the shared task domain" + ); + const int64_t worker0_before = LoadReaderDone(state, 0); + Check( + !SharedAdvanceReaderDone( + state.shared_map, kWorkers, 0 + ), + "reader progress rejects an out-of-range worker" + ); + Check( + !SharedAdvanceReaderDone( + state.shared_map, 0, -1 + ), + "reader progress rejects a negative task" + ); + Check( + !SharedAdvanceReaderDone( + state.shared_map, 0, + static_cast(kMaxTasks) + ), + "reader progress rejects a task above the shared domain" + ); + Check( + LoadReaderDone(state, 0) == worker0_before, + "reader progress bounds failures do not touch state" + ); + + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + StoreReaderDone(state, worker, -1); + } + const int32_t targets[] = {9, 5, 8}; + bool sequence_ok = true; + for (uint32_t worker = 0; worker < 3; ++worker) { + for (int32_t task = 0; task <= targets[worker]; ++task) { + sequence_ok &= + SharedAdvanceReaderDone( + state.shared_map, worker, task + ); + } + } + Check( + sequence_ok, + "three active readers publish contiguous completion sequences" + ); + + // active worker 是连续前缀；前缀之外的脏值不得参与最小值。 + StoreReaderDone(state, 3, -2); + int64_t candidate = -77; + Check( + SharedComputeReaderReclaimCandidate( + state.shared_map, 3, 0, candidate + ) && + candidate == 5, + "H=0 exposes the exact active-prefix minimum" + ); + candidate = -77; + Check( + SharedComputeReaderReclaimCandidate( + state.shared_map, 3, 2, candidate + ) && + candidate == 3, + "reader minimum 5 and H=2 produce inclusive reclaim 3" + ); + candidate = -77; + Check( + !SharedComputeReaderReclaimCandidate( + state.shared_map, 4, 2, candidate + ) && + candidate == -77, + "an invalid active progress line fails without changing output" + ); + + Check( + SharedAdvanceReaderDone( + state.shared_map, 0, 10 + ) && + SharedAdvanceReaderDone( + state.shared_map, 2, 9 + ), + "faster readers may move without changing the slow frontier" + ); + candidate = -77; + Check( + SharedComputeReaderReclaimCandidate( + state.shared_map, 3, 2, candidate + ) && + candidate == 3, + "faster-reader progress leaves the slow-reader reclaim unchanged" + ); + Check( + SharedAdvanceReaderDone( + state.shared_map, 1, 6 + ), + "the slow reader advances by one task" + ); + candidate = -77; + Check( + SharedComputeReaderReclaimCandidate( + state.shared_map, 3, 2, candidate + ) && + candidate == 4, + "the reclaim candidate advances only with the slow reader" + ); + + StoreReaderDone( + state, 1, static_cast(kMaxTasks) + ); + candidate = -77; + Check( + !SharedComputeReaderReclaimCandidate( + state.shared_map, 3, 2, candidate + ) && + candidate == -77, + "reader progress above the task domain fails without output" + ); + StoreReaderDone(state, 1, 6); + candidate = -77; + Check( + !SharedComputeReaderReclaimCandidate( + state.shared_map, 0, 2, candidate + ) && + candidate == -77, + "reader reclaim rejects an empty active prefix without output" + ); + candidate = -77; + Check( + !SharedComputeReaderReclaimCandidate( + state.shared_map, kWorkers + 1, 2, candidate + ) && + candidate == -77, + "reader reclaim rejects an oversized active prefix without output" + ); + candidate = -77; + Check( + !SharedComputeReaderReclaimCandidate( + state.shared_map, 3, -1, candidate + ) && + candidate == -77, + "reader reclaim rejects a negative window without output" + ); + candidate = -77; + Check( + SharedComputeReaderReclaimCandidate( + state.shared_map, 3, INT32_MAX, candidate + ) && + candidate == -1, + "a very wide legal window clamps the reclaim candidate to -1" + ); + + // BeginCallbackSubmit 会在 task 读取前推进 local_index。即使该普通字段 + // 已到 6，task 5 的 ordinary lookup 仍可能尚未结束；真正完成前沿 4 + // 在 H=2 时只能回收到 2，不能按 local_index 推到 3/4。 + for (uint32_t worker = 0; worker < kWorkers; ++worker) { + StoreReaderDone(state, worker, -1); + } + bool local_progress_ok = true; + for (int32_t task = 0; task <= 4; ++task) { + local_progress_ok &= + SharedAdvanceReaderDone( + state.shared_map, 0, task + ); + } + Check( + local_progress_ok, + "local-index counterexample prepares reader completion" + ); + const int32_t saved_local_index = + state.workers[0].local_index; + state.workers[0].local_index = 5; + SubmitContext context{}; + BeginCallbackSubmit(state.workers[0], context); + Check( + context.task_id == 5 && + state.workers[0].local_index == 6, + "BeginCallbackSubmit advances local_index before task-5 reads" + ); + candidate = -77; + Check( + SharedComputeReaderReclaimCandidate( + state.shared_map, 1, 2, candidate + ) && + candidate == 2, + "reader reclaim is independent of the pre-read local_index" + ); + state.workers[0].local_index = 100; + candidate = -77; + Check( + SharedComputeReaderReclaimCandidate( + state.shared_map, 1, 2, candidate + ) && + candidate == 2, + "changing local_index cannot move the reader completion frontier" + ); + state.workers[0].local_index = saved_local_index; +} + +FdwicOutputRef PublishSingleTestSymbol( + SchedulerState &state, int32_t producer, uint64_t address +) { + SharedOutputCell &cell = + state.shared_map.shared_outputs[ + static_cast(producer) + ]; + cell.published[0].value = -1; + cell.last_writer[0].value = -1; + TensorDesc tensor = MakeTensor( + address, static_cast(producer) + ); + SubmitContext context{}; + context.task_id = producer; + context.result.task_id = producer; + context.result.count = 1; + context.result.tensors[0] = &tensor; + context.shared_result.Reset(producer); + const bool ref_ok = + context.shared_result.AddOutputRef(producer, 0); + Check(ref_ok, "single test symbol accepts slot 0"); + if (!ref_ok || + !PublishSharedTaskOutputs( + state.shared_map, context, producer + )) { + Check(false, "single test symbol publishes descriptor"); + return InvalidSharedOutputRef(); + } + return context.shared_result.OutputRef(0); +} + +void TestSymbolWriterIntentChain(SchedulerState &state) { + constexpr int32_t kExistingDependency = 5; + constexpr int32_t kProducer = 10; + constexpr int32_t kWriter = 20; + constexpr int32_t kReader = 30; + constexpr int32_t kFutureWriter = 40; + constexpr int32_t kFarFutureWriter = 50; + constexpr uint32_t kSymbolCount = 7; + ResetTaskGate(state, kWriter); + ResetTaskGate(state, kFutureWriter); + ResetTaskGate(state, kFarFutureWriter); + + SharedOutputCell &cell = + state.shared_map.shared_outputs[kProducer]; + for (uint32_t output = 0; output < kSymbolCount; ++output) { + cell.published[output].value = -1; + cell.last_writer[output].value = -1; + } + + TensorDesc produced[kSymbolCount] = {}; + FdwicOutputRef output_refs[kSymbolCount] = {}; + SubmitContext producer_context{}; + producer_context.task_id = kProducer; + producer_context.result.task_id = kProducer; + producer_context.result.count = + static_cast(kSymbolCount); + producer_context.shared_result.Reset(kProducer); + for (uint32_t output = 0; output < kSymbolCount; ++output) { + produced[output] = MakeTensor( + 0x410000000ULL + + static_cast(output) * 0x10000ULL, + static_cast(kProducer) + ); + producer_context.result.tensors[output] = + &produced[output]; + Check( + producer_context.shared_result.AddOutputRef( + kProducer, static_cast(output) + ), + "symbol producer accepts history-test output" + ); + output_refs[output] = + producer_context.shared_result.OutputRef(output); + } + Check( + PublishSharedTaskOutputs( + state.shared_map, producer_context, kProducer + ), + "symbol producer publishes descriptor" + ); + + const FdwicOutputRef output_ref = output_refs[0]; + TaskArgs writer_args; + ConstructTaskArgs(writer_args); + for (uint32_t output = 0; output < kSymbolCount; ++output) { + AppendSharedOutputRef( + writer_args, output_refs[output], + TensorArgType::Inout + ); + } + bool required = false; + Check( + InspectSharedWriterIntent(writer_args, required) && + required, + "symbol INOUT is a generic writer intent" + ); + + SubmitContext writer_context{}; + writer_context.task_id = kWriter; + writer_context.won = true; + writer_context.fanin_count = 2; + writer_context.fanin[0] = kExistingDependency; + writer_context.fanin[1] = kProducer; + LocalStats writer_stats{}; + + TaskArgs reader_args; + ConstructTaskArgs(reader_args); + AppendSharedOutputRef( + reader_args, output_ref, TensorArgType::Input + ); + + std::atomic reader_finished{false}; + std::atomic reader_past_writer_gate{false}; + std::atomic allow_reader_lookup{false}; + bool wait_ok = false; + bool lookup_ok = false; + uint32_t ordinary_lookups = UINT32_MAX; + uint32_t reader_fanin_count = 0; + int32_t reader_fanin[kMaxFanin] = {}; + LocalStats reader_stats{}; + WriterIntentTestOps::wait_address = + &state.tasks[kWriter].deps_prepared; + WriterIntentTestOps::wait_loads.store( + 0, std::memory_order_relaxed + ); + std::thread reader([&]() { + wait_ok = WaitForSharedWriterReady( + &state, kWriter, reader_stats + ); + if (wait_ok) { + // C 已经观察到 B 的 writer-ready，但故意停在 symbol lookup + // 之前；主线程随后让未来 writer D 覆盖 latest cache。这个 + // 时序隔离“门补齐了 B”与“查询仍能找回 B”两项不同性质。 + reader_past_writer_gate.store( + true, std::memory_order_release + ); + while (!allow_reader_lookup.load( + std::memory_order_acquire + )) { + std::this_thread::yield(); + } + reader_fanin_count = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, reader_args, kReader, + static_cast(state.heap_window), + reader_stats, reader_fanin, lookup_ok, + ordinary_lookups, &state.fatal.value + ); + } + reader_finished.store(true, std::memory_order_release); + }); + + Check( + WaitUntilObserved(WriterIntentTestOps::wait_loads), + "symbol reader reaches the delayed writer gate" + ); + Check( + !reader_finished.load(std::memory_order_acquire), + "symbol reader cannot resolve before writer metadata" + ); + const SharedWriterIntentResult prepared = + PrepareSharedWriterIntentSet( + &state, writer_args, writer_context, writer_stats + ); + + Check( + WaitUntilTrue(reader_past_writer_gate), + "symbol reader passes B gate before future D publishes" + ); + TaskArgs future_writer_args; + ConstructTaskArgs(future_writer_args); + for (uint32_t output = 0; output < kSymbolCount; ++output) { + AppendSharedOutputRef( + future_writer_args, output_refs[output], + TensorArgType::Inout + ); + } + SubmitContext future_writer_context{}; + future_writer_context.task_id = kFutureWriter; + future_writer_context.won = true; + LocalStats future_writer_stats{}; + const SharedWriterIntentResult future_prepared = + PrepareSharedWriterIntentSet( + &state, future_writer_args, + future_writer_context, future_writer_stats + ); + TaskArgs far_future_writer_args; + ConstructTaskArgs(far_future_writer_args); + for (uint32_t output = 0; output < kSymbolCount; ++output) { + AppendSharedOutputRef( + far_future_writer_args, output_refs[output], + TensorArgType::Inout + ); + } + SubmitContext far_future_writer_context{}; + far_future_writer_context.task_id = kFarFutureWriter; + far_future_writer_context.won = true; + LocalStats far_future_writer_stats{}; + const SharedWriterIntentResult far_future_prepared = + PrepareSharedWriterIntentSet( + &state, far_future_writer_args, + far_future_writer_context, far_future_writer_stats + ); + uint32_t symbol_key = 0; + uint32_t seventh_symbol_key = 0; + const bool key_ok = + SharedSymbolHistoryKey(output_ref, symbol_key) && + SharedSymbolHistoryKey( + output_refs[kSymbolCount - 1], + seventh_symbol_key + ); + const SharedWriterHistoryCell &writer_history = + state.shared_map.writer_history[kWriter]; + const SharedWriterHistoryCell &future_history = + state.shared_map.writer_history[kFutureWriter]; + const SharedWriterHistoryCell &far_future_history = + state.shared_map.writer_history[kFarFutureWriter]; + Check( + key_ok && + writer_history.magic == + kSharedWriterHistoryMagic && + writer_history.writer_task == kWriter && + writer_history.count == kSymbolCount && + writer_history.entries[0].symbol_key == + symbol_key && + writer_history.entries[0].previous_writer == + kProducer && + writer_history.entries[kSymbolCount - 1].symbol_key == + seventh_symbol_key, + "B publishes seven A predecessor records across two cache lines" + ); + Check( + future_history.magic == + kSharedWriterHistoryMagic && + future_history.writer_task == kFutureWriter && + future_history.count == kSymbolCount && + future_history.entries[0].symbol_key == + symbol_key && + future_history.entries[0].previous_writer == + kWriter, + "D publishes an immutable B predecessor record" + ); + Check( + far_future_history.magic == + kSharedWriterHistoryMagic && + far_future_history.writer_task == kFarFutureWriter && + far_future_history.count == kSymbolCount && + far_future_history.entries[0].symbol_key == + symbol_key && + far_future_history.entries[0].previous_writer == + kFutureWriter, + "E publishes an immutable D predecessor record" + ); + allow_reader_lookup.store(true, std::memory_order_release); + reader.join(); + WriterIntentTestOps::wait_address = nullptr; + + Check( + prepared == SharedWriterIntentResult::Published, + "symbol writer publishes the generic intent" + ); + Check( + writer_context.fanin_count == 2 && + writer_context.fanin[0] == kExistingDependency && + writer_context.fanin[1] == kProducer, + "symbol writer retains existing fanin and deduplicates the previous writer" + ); + Check( + future_prepared == SharedWriterIntentResult::Published && + future_writer_context.fanin_count == 1 && + future_writer_context.fanin[0] == kWriter && + state.tasks[kFutureWriter].deps_prepared == + kFutureWriter, + "future symbol writer D consumes B before publishing" + ); + Check( + far_future_prepared == + SharedWriterIntentResult::Published && + far_future_writer_context.fanin_count == 1 && + far_future_writer_context.fanin[0] == + kFutureWriter && + state.tasks[kFarFutureWriter].deps_prepared == + kFarFutureWriter, + "far-future symbol writer E consumes D before publishing" + ); + Check( + cell.last_writer[0].value == kFarFutureWriter && + state.tasks[kWriter].deps_prepared == kWriter, + "latest cache advances to E after all writer-ready gates" + ); + Check( + state.tasks[kWriter].flag == 0, + "symbol writer-ready is not kernel completion" + ); + Check( + wait_ok && lookup_ok && + reader_fanin_count == 1 && + reader_fanin[0] == kWriter && + ordinary_lookups == 0, + "slow C follows E-to-D-to-B history after latest advances" + ); +} + +void TestOutOfOrderSymbolWriterFailsClosed( + SchedulerState &state +) { + constexpr int32_t kProducer = 200; + constexpr int32_t kEarlierWriter = 220; + constexpr int32_t kLaterWriter = 240; + state.fatal.value = 0; + ResetTaskGate(state, kEarlierWriter); + ResetTaskGate(state, kLaterWriter); + const FdwicOutputRef output_ref = + PublishSingleTestSymbol( + state, kProducer, 0x440000000ULL + ); + + TaskArgs args; + ConstructTaskArgs(args); + AppendSharedOutputRef( + args, output_ref, TensorArgType::Inout + ); + SubmitContext later_context{}; + later_context.task_id = kLaterWriter; + later_context.won = true; + LocalStats later_stats{}; + const SharedWriterIntentResult later_result = + PrepareSharedWriterIntentSet( + &state, args, later_context, later_stats + ); + + SubmitContext earlier_context{}; + earlier_context.task_id = kEarlierWriter; + earlier_context.won = true; + LocalStats earlier_stats{}; + const SharedWriterIntentResult earlier_result = + PrepareSharedWriterIntentSet( + &state, args, earlier_context, earlier_stats + ); + const SharedOutputCell &cell = + state.shared_map.shared_outputs[kProducer]; + Check( + later_result == SharedWriterIntentResult::Published && + cell.last_writer[0].value == kLaterWriter, + "out-of-order setup publishes the later writer first" + ); + Check( + earlier_result == SharedWriterIntentResult::Failed && + state.fatal.value == 1 && + state.tasks[kEarlierWriter].deps_prepared == -1 && + cell.last_writer[0].value == kLaterWriter, + "skipped earlier writer fails closed without moving latest backward" + ); +} + +void TestMultiSymbolConflictKeepsTerminalPrefix( + SchedulerState &state +) { + constexpr int32_t kProducer0 = 300; + constexpr int32_t kProducer1 = 301; + constexpr int32_t kConflictingWriter = 350; + constexpr int32_t kWriter = 400; + state.fatal.value = 0; + ResetTaskGate(state, kWriter); + const FdwicOutputRef output0 = + PublishSingleTestSymbol( + state, kProducer0, 0x450000000ULL + ); + const FdwicOutputRef output1 = + PublishSingleTestSymbol( + state, kProducer1, 0x460000000ULL + ); + + TaskArgs args; + ConstructTaskArgs(args); + AppendSharedOutputRef( + args, output0, TensorArgType::Inout + ); + AppendSharedOutputRef( + args, output1, TensorArgType::Inout + ); + SubmitContext context{}; + context.task_id = kWriter; + context.won = true; + LocalStats stats{}; + volatile int64_t *first_latest = + &state.shared_map.shared_outputs[kProducer0] + .last_writer[0].value; + volatile int64_t *second_latest = + &state.shared_map.shared_outputs[kProducer1] + .last_writer[0].value; + WriterIntentTestOps::cas_trigger_address = first_latest; + WriterIntentTestOps::cas_conflict_address = second_latest; + WriterIntentTestOps::cas_conflict_value = + kConflictingWriter; + const SharedWriterIntentResult result = + PrepareSharedWriterIntentSet( + &state, args, context, stats + ); + WriterIntentTestOps::cas_trigger_address = nullptr; + WriterIntentTestOps::cas_conflict_address = nullptr; + + const SharedWriterHistoryCell &history = + state.shared_map.writer_history[kWriter]; + Check( + result == SharedWriterIntentResult::Failed && + state.fatal.value == 1 && + state.tasks[kWriter].deps_prepared == -1, + "multi-symbol CAS conflict terminates without publishing ready" + ); + Check( + *first_latest == kWriter && + *second_latest == kConflictingWriter && + stats.result.shared_symbol_inout_commits == 1, + "terminal failure preserves and counts only the linearized prefix" + ); + Check( + history.magic == kSharedWriterHistoryMagic && + history.writer_task == kWriter && + history.count == 2 && + history.entries[0].previous_writer == + kProducer0 && + history.entries[1].previous_writer == + kProducer1, + "terminal prefix retains its immutable diagnostic history" + ); +} + +void TestOrdinaryWriterIntentChain(SchedulerState &state) { + constexpr int32_t kFirstWriter = 100; + constexpr int32_t kSecondWriter = 120; + constexpr int32_t kReader = 140; + ResetTaskGate(state, kFirstWriter); + ResetTaskGate(state, kSecondWriter); + TensorDesc tensor = MakeTensor(0x420000000ULL); + + TaskArgs first_args; + ConstructTaskArgs(first_args); + AddGmTensor( + first_args, tensor, TensorArgType::OutputExisting + ); + SubmitContext first_context{}; + first_context.task_id = kFirstWriter; + first_context.won = true; + LocalStats first_stats{}; + Check( + PrepareSharedWriterIntentSet( + &state, first_args, first_context, first_stats + ) == SharedWriterIntentResult::Published, + "ordinary OUTPUT_EXISTING publishes the first writer" + ); + Check( + first_context.fanin_count == 0, + "first external ordinary writer has no predecessor" + ); + + TaskArgs second_args; + ConstructTaskArgs(second_args); + AddGmTensor( + second_args, tensor, TensorArgType::Inout + ); + SubmitContext second_context{}; + second_context.task_id = kSecondWriter; + second_context.won = true; + LocalStats second_stats{}; + + TaskArgs reader_args; + ConstructTaskArgs(reader_args); + AddGmTensor(reader_args, tensor, TensorArgType::Input); + std::atomic reader_finished{false}; + bool wait_ok = false; + bool lookup_ok = false; + uint32_t lookup_count = UINT32_MAX; + uint32_t reader_fanin_count = 0; + int32_t reader_fanin[kMaxFanin] = {}; + LocalStats reader_stats{}; + WriterIntentTestOps::wait_address = + &state.tasks[kSecondWriter].deps_prepared; + WriterIntentTestOps::wait_loads.store( + 0, std::memory_order_relaxed + ); + std::thread reader([&]() { + wait_ok = WaitForSharedWriterReady( + &state, kSecondWriter, reader_stats + ); + if (wait_ok) { + reader_fanin_count = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, reader_args, kReader, + static_cast(state.heap_window), + reader_stats, reader_fanin, lookup_ok, + lookup_count, &state.fatal.value + ); + } + reader_finished.store(true, std::memory_order_release); + }); + + Check( + WaitUntilObserved(WriterIntentTestOps::wait_loads), + "ordinary reader reaches the delayed writer gate" + ); + Check( + !reader_finished.load(std::memory_order_acquire), + "ordinary reader cannot resolve before writer metadata" + ); + const SharedWriterIntentResult prepared = + PrepareSharedWriterIntentSet( + &state, second_args, second_context, second_stats + ); + reader.join(); + WriterIntentTestOps::wait_address = nullptr; + + Check( + prepared == SharedWriterIntentResult::Published, + "ordinary INOUT publishes the second writer" + ); + Check( + second_context.fanin_count == 1 && + second_context.fanin[0] == kFirstWriter, + "ordinary INOUT consumes the first writer" + ); + Check( + state.tasks[kSecondWriter].deps_prepared == + kSecondWriter && + state.tasks[kSecondWriter].flag == 0, + "ordinary metadata-ready remains separate from completion" + ); + Check( + wait_ok && lookup_ok && + reader_fanin_count == 1 && + reader_fanin[0] == kSecondWriter && + lookup_count == 1, + "ownerless ordinary INPUT resolves the latest writer" + ); + Check( + InsertCompletionsMatch(state, 0), + "non-adjacent writer-intent preparation leaves the completion chain untouched" + ); +} + +void TestWriterDeltaRequiresExactRegisterMask() { + TensorDesc tensor = MakeTensor(0x455000000ULL); + const FdwicOutputRef output{0, 0, 0, 0, 0, 0}; + TaskArgs args; + ConstructTaskArgs(args); + AddGmTensor(args, tensor, TensorArgType::OutputExisting); + AddOutputHandleTensor(args, output, TensorArgType::Inout); + + SubmitContext context{}; + context.task_id = 1; + context.won = true; + context.result.task_id = 1; + context.shared_result.Reset(1); + SharedTaskWriterDelta delta{}; + + context.register_mask = 1; + Check( + !PrepareSharedTaskWriterDelta(args, context, delta), + "writer delta rejects a mask that omits one INOUT entry" + ); + context.register_mask = 2; + Check( + !PrepareSharedTaskWriterDelta(args, context, delta), + "writer delta rejects a mask that omits one ordinary writer" + ); + context.register_mask = 7; + Check( + !PrepareSharedTaskWriterDelta(args, context, delta), + "writer delta rejects a mask with an extra non-argument bit" + ); + context.register_mask = 3; + Check( + PrepareSharedTaskWriterDelta(args, context, delta) && + delta.prepared_task_id == context.task_id && + delta.ordinary_count == 1 && + delta.symbol_count == 1 && + delta.ordinary_buckets[0] == + TensorMapHash(tensor.buffer_addr) && + delta.ordinary_bucket_ordinals[0] == 0 && + delta.symbol_keys[0] == 1 && + delta.writer_intent_required, + "writer delta freezes task identity, writer counts, buckets and symbol keys" + ); + + TaskArgs duplicate_args; + ConstructTaskArgs(duplicate_args); + AddOutputHandleTensor( + duplicate_args, output, TensorArgType::Inout + ); + AddOutputHandleTensor( + duplicate_args, output, TensorArgType::OutputExisting + ); + context.register_mask = 3; + Check( + !PrepareSharedTaskWriterDelta( + duplicate_args, context, delta + ), + "writer delta rejects a duplicate precomputed symbol key" + ); + + // 低 32 位看似是合法前任的畸形 owner 也必须在 writer delta + // 准备阶段拒绝，不能截断成 task 0 后发布 ordinary 元数据。 + tensor.owner_task_id = uint64_t{1} << 32; + context.task_id = 1; + context.result.task_id = 1; + context.register_mask = 3; + Check( + !ValidateOrdinarySharedWriterReference( + tensor, context.task_id + ) && + !PrepareSharedTaskWriterDelta( + args, context, delta + ), + "writer delta rejects owner ids with nonzero high 32 bits" + ); +} + +void TestWriterDeltaPrecomputesInterleavedBuckets( + SchedulerState &state +) { + ResetProtocolState(state); + constexpr uint64_t kBase = 0x458000000ULL; + const uint32_t bucket_a = TensorMapHash(kBase); + const uint32_t bucket_b = + (bucket_a + 1U) % kMapBuckets; + const uint64_t address_a1 = + FindAddressForBucket(kBase + 4096, bucket_a); + const uint64_t address_a2 = + FindAddressForBucket(address_a1 + 4096, bucket_a); + const uint64_t address_b = + FindAddressForBucket(kBase + 4096, bucket_b); + Check( + address_a1 != 0 && address_a2 != 0 && + address_b != 0, + "interleaved bucket test finds distinct addresses" + ); + if (address_a1 == 0 || address_a2 == 0 || + address_b == 0) { + return; + } + + TensorDesc tensors[4] = { + MakeTensor(kBase), + MakeTensor(address_b), + MakeTensor(address_a1), + MakeTensor(address_a2) + }; + TaskArgs args; + ConstructTaskArgs(args); + for (TensorDesc &tensor : tensors) { + AddGmTensor( + args, tensor, TensorArgType::OutputExisting + ); + } + SubmitContext context{}; + context.task_id = 0; + context.won = true; + context.register_mask = 15; + context.result.task_id = 0; + context.shared_result.Reset(0); + SharedTaskWriterDelta delta{}; + LocalStats stats{}; + Check( + PrepareSharedTaskWriterDelta( + args, context, delta + ) && + delta.ordinary_count == 4 && + delta.ordinary_buckets[0] == bucket_a && + delta.ordinary_buckets[1] == bucket_b && + delta.ordinary_buckets[2] == bucket_a && + delta.ordinary_buckets[3] == bucket_a && + delta.ordinary_bucket_ordinals[0] == 0 && + delta.ordinary_bucket_ordinals[1] == 0 && + delta.ordinary_bucket_ordinals[2] == 1 && + delta.ordinary_bucket_ordinals[3] == 2, + "writer delta precomputes A,B,A,A buckets and 0,0,1,2 ordinals" + ); + Check( + PublishSharedTaskWriterDelta( + &state, context, delta, stats + ) && + state.shared_map.buckets[bucket_a].tail.value == 3 && + state.shared_map.buckets[bucket_b].tail.value == 1 && + stats.result.map_inserts == 4, + "prepared interleaved metadata publishes without reordering" + ); +} + +void TestOrderedOrdinaryInsertBeforeLookup( + SchedulerState &state +) { + ResetProtocolState(state); + TensorDesc tensor = MakeTensor(0x460000000ULL); + + TaskArgs first_args; + ConstructTaskArgs(first_args); + AddGmTensor( + first_args, tensor, TensorArgType::OutputExisting + ); + SubmitContext first_context{}; + first_context.task_id = 0; + first_context.won = true; + first_context.register_mask = 1; + first_context.result.task_id = 0; + first_context.shared_result.Reset(0); + SharedTaskWriterDelta first_delta{}; + LocalStats first_stats{}; + Check( + PrepareSharedTaskWriterDelta( + first_args, first_context, first_delta + ) && + first_delta.prepared_task_id == 0 && + first_delta.ordinary_count == 1 && + first_delta.symbol_count == 0 && + first_delta.ordinary_buckets[0] == + TensorMapHash(tensor.buffer_addr) && + first_delta.ordinary_bucket_ordinals[0] == 0 && + first_delta.writer_intent_required, + "ordered ordinary task 0 prepares one writer entry" + ); + Check( + PublishSharedTaskWriterDelta( + &state, first_context, first_delta, first_stats + ) && + InsertCompletionsMatch(state, 1) && + first_stats.result.map_inserts == 1 && + first_stats.result.shared_symbol_inout_commits == 0, + "ordered ordinary task 0 publishes before lookup" + ); + + bool lookup_ok = false; + uint32_t lookup_count = UINT32_MAX; + int32_t fanin[kMaxFanin] = {}; + const uint32_t first_fanin = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, first_args, 0, + static_cast(state.heap_window), + first_stats, fanin, lookup_ok, + lookup_count, &state.fatal.value + ); + Check( + lookup_ok && first_fanin == 0 && + lookup_count == 1, + "task 0 lookup after its own insert excludes itself" + ); + + TaskArgs second_args; + ConstructTaskArgs(second_args); + AddGmTensor(second_args, tensor, TensorArgType::Inout); + SubmitContext second_context{}; + second_context.task_id = 1; + second_context.won = true; + second_context.register_mask = 1; + second_context.result.task_id = 1; + second_context.shared_result.Reset(1); + SharedTaskWriterDelta second_delta{}; + LocalStats second_stats{}; + Check( + PrepareSharedTaskWriterDelta( + second_args, second_context, second_delta + ) && + PublishSharedTaskWriterDelta< + WriterIntentTestOps + >( + &state, second_context, second_delta, + second_stats + ) && + InsertCompletionsMatch(state, 2), + "ordered ordinary task 1 publishes its writer entry" + ); + lookup_ok = false; + lookup_count = UINT32_MAX; + const uint32_t second_fanin = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, second_args, 1, + static_cast(state.heap_window), + second_stats, fanin, lookup_ok, + lookup_count, &state.fatal.value + ); + Check( + lookup_ok && second_fanin == 1 && + fanin[0] == 0 && lookup_count == 1, + "task 1 lookup returns task 0 instead of itself" + ); + + TaskArgs reader_args; + ConstructTaskArgs(reader_args); + AddGmTensor(reader_args, tensor, TensorArgType::Input); + SubmitContext reader_context{}; + reader_context.task_id = 2; + reader_context.won = true; + reader_context.result.task_id = 2; + reader_context.shared_result.Reset(2); + SharedTaskWriterDelta reader_delta{}; + LocalStats reader_stats{}; + Check( + PrepareSharedTaskWriterDelta( + reader_args, reader_context, reader_delta + ) && + reader_delta.ordinary_count == 0 && + PublishSharedTaskWriterDelta< + WriterIntentTestOps + >( + &state, reader_context, reader_delta, + reader_stats + ) && + InsertCompletionsMatch(state, 3), + "empty writer task still publishes its per-task completion" + ); + lookup_ok = false; + lookup_count = UINT32_MAX; + const uint32_t reader_fanin = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, reader_args, 2, + static_cast(state.heap_window), + reader_stats, fanin, lookup_ok, + lookup_count, &state.fatal.value + ); + Check( + lookup_ok && reader_fanin == 1 && + fanin[0] == 1 && lookup_count == 1, + "task 2 reader resolves the newest writer below task 2" + ); +} + +void TestOrderedSymbolInsertBeforeLookup( + SchedulerState &state +) { + ResetProtocolState(state); + TensorDesc descriptor = MakeTensor(0x470000000ULL, 0); + + TaskArgs producer_args; + ConstructTaskArgs(producer_args); + SubmitContext producer_context{}; + producer_context.task_id = 0; + producer_context.won = true; + producer_context.result.task_id = 0; + producer_context.result.count = 1; + producer_context.result.tensors[0] = &descriptor; + producer_context.shared_result.Reset(0); + Check( + producer_context.shared_result.AddOutputRef(0, 0), + "fresh symbol test declares output slot 0" + ); + SharedTaskWriterDelta producer_delta{}; + LocalStats producer_stats{}; + Check( + PrepareSharedTaskWriterDelta( + producer_args, producer_context, producer_delta + ) && + PublishSharedTaskWriterDelta< + WriterIntentTestOps + >( + &state, producer_context, producer_delta, + producer_stats + ) && + state.shared_map.shared_outputs[0] + .published[0].value == 0 && + InsertCompletionsMatch(state, 1), + "fresh symbol descriptor is visible before task 0 completion" + ); + + const FdwicOutputRef output{ + 0, 0, 0, 0, 0, 0 + }; + TaskArgs writer_args; + ConstructTaskArgs(writer_args); + AddOutputHandleTensor( + writer_args, output, TensorArgType::Inout + ); + SubmitContext writer_context{}; + writer_context.task_id = 1; + writer_context.won = true; + writer_context.register_mask = 1; + writer_context.result.task_id = 1; + writer_context.shared_result.Reset(1); + SharedTaskWriterDelta writer_delta{}; + LocalStats writer_stats{}; + Check( + PrepareSharedTaskWriterDelta( + writer_args, writer_context, writer_delta + ) && + writer_delta.prepared_task_id == 1 && + writer_delta.ordinary_count == 0 && + writer_delta.symbol_count == 1 && + writer_delta.symbol_keys[0] == 1 && + writer_delta.writer_intent_required && + PublishSharedTaskWriterDelta< + WriterIntentTestOps + >( + &state, writer_context, writer_delta, + writer_stats + ) && + state.shared_map.shared_outputs[0] + .last_writer[0].value == 1 && + writer_stats.result.map_inserts == 0 && + writer_stats.result.shared_symbol_inout_commits == 1 && + InsertCompletionsMatch(state, 2), + "symbol INOUT publishes history before its own lookup" + ); + + bool lookup_ok = false; + uint32_t lookup_count = UINT32_MAX; + int32_t fanin[kMaxFanin] = {}; + const uint32_t writer_fanin = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, writer_args, 1, + static_cast(state.heap_window), + writer_stats, fanin, lookup_ok, + lookup_count, &state.fatal.value + ); + Check( + lookup_ok && writer_fanin == 1 && + fanin[0] == 0 && lookup_count == 0, + "symbol task 1 walks immutable history back to task 0" + ); + + TaskArgs reader_args; + ConstructTaskArgs(reader_args); + AddOutputHandleTensor( + reader_args, output, TensorArgType::Input + ); + SubmitContext reader_context{}; + reader_context.task_id = 2; + reader_context.won = true; + reader_context.result.task_id = 2; + reader_context.shared_result.Reset(2); + SharedTaskWriterDelta reader_delta{}; + LocalStats reader_stats{}; + Check( + PrepareSharedTaskWriterDelta( + reader_args, reader_context, reader_delta + ) && + PublishSharedTaskWriterDelta< + WriterIntentTestOps + >( + &state, reader_context, reader_delta, + reader_stats + ), + "symbol reader publishes an empty task transaction" + ); + lookup_ok = false; + lookup_count = UINT32_MAX; + const uint32_t reader_fanin = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, reader_args, 2, + static_cast(state.heap_window), + reader_stats, fanin, lookup_ok, + lookup_count, &state.fatal.value + ); + Check( + lookup_ok && reader_fanin == 1 && + fanin[0] == 1 && lookup_count == 0, + "symbol task 2 resolves task 1 as the latest prior writer" + ); +} + +void TestOrderedMixedWriterTransaction( + SchedulerState &state +) { + ResetProtocolState(state); + TensorDesc ordinary = MakeTensor(0x478000000ULL); + TensorDesc seed_output = MakeTensor(0x478100000ULL, 0); + + // task 0 同时建立 ordinary writer 与后续 INOUT 使用的 fresh + // descriptor。它先完整发布两类元数据，再把有序前沿推进到 1。 + TaskArgs seed_args; + ConstructTaskArgs(seed_args); + AddGmTensor( + seed_args, ordinary, TensorArgType::OutputExisting + ); + SubmitContext seed_context{}; + seed_context.task_id = 0; + seed_context.won = true; + seed_context.register_mask = 1; + seed_context.result.task_id = 0; + seed_context.result.count = 1; + seed_context.result.tensors[0] = &seed_output; + seed_context.shared_result.Reset(0); + Check( + seed_context.shared_result.AddOutputRef(0, 0), + "mixed transaction seed declares fresh output" + ); + SharedTaskWriterDelta seed_delta{}; + LocalStats seed_stats{}; + Check( + PrepareSharedTaskWriterDelta( + seed_args, seed_context, seed_delta + ) && + PublishSharedTaskWriterDelta< + WriterIntentTestOps + >( + &state, seed_context, seed_delta, seed_stats + ), + "mixed transaction seed publishes ordinary and fresh metadata" + ); + + const FdwicOutputRef seed_ref{0, 0, 0, 0, 0, 0}; + TensorDesc next_output = MakeTensor(0x478200000ULL, 1); + TaskArgs mixed_args; + ConstructTaskArgs(mixed_args); + AddGmTensor( + mixed_args, ordinary, TensorArgType::Inout + ); + AddOutputHandleTensor( + mixed_args, seed_ref, TensorArgType::Inout + ); + SubmitContext mixed_context{}; + mixed_context.task_id = 1; + mixed_context.won = true; + mixed_context.register_mask = 3; + mixed_context.result.task_id = 1; + mixed_context.result.count = 1; + mixed_context.result.tensors[0] = &next_output; + mixed_context.shared_result.Reset(1); + Check( + mixed_context.shared_result.AddOutputRef(1, 0), + "mixed transaction declares its own fresh output" + ); + SharedTaskWriterDelta mixed_delta{}; + LocalStats mixed_stats{}; + Check( + PrepareSharedTaskWriterDelta( + mixed_args, mixed_context, mixed_delta + ) && + mixed_delta.prepared_task_id == 1 && + mixed_delta.ordinary_count == 1 && + mixed_delta.symbol_count == 1 && + mixed_delta.writer_intent_required && + PublishSharedTaskWriterDelta< + WriterIntentTestOps + >( + &state, mixed_context, mixed_delta, + mixed_stats + ), + "one ordered transaction publishes ordinary, symbol and fresh metadata" + ); + + const SharedOutputCell &seed_cell = + state.shared_map.shared_outputs[0]; + const SharedOutputCell &mixed_cell = + state.shared_map.shared_outputs[1]; + const SharedWriterHistoryCell &history = + state.shared_map.writer_history[1]; + Check( + state.fatal.value == 0 && + InsertCompletionsMatch(state, 2) && + seed_cell.last_writer[0].value == 1 && + history.magic == kSharedWriterHistoryMagic && + history.writer_task == 1 && + history.count == 1 && + history.entries[0].symbol_key == + mixed_delta.symbol_keys[0] && + mixed_cell.published[0].value == 1 && + mixed_cell.last_writer[0].value == 1 && + mixed_stats.result.map_inserts == 1 && + mixed_stats.result.shared_symbol_inout_commits == 1 && + mixed_cell.tensors[0].buffer_addr == + next_output.buffer_addr, + "mixed transaction exposes all metadata before task 1 completion" + ); + + bool lookup_ok = false; + uint32_t lookup_count = UINT32_MAX; + int32_t fanin[kMaxFanin] = {}; + const uint32_t mixed_fanin = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, mixed_args, 1, + static_cast(state.heap_window), + mixed_stats, fanin, lookup_ok, + lookup_count, &state.fatal.value + ); + Check( + lookup_ok && mixed_fanin == 1 && + fanin[0] == 0 && lookup_count == 1, + "mixed transaction lookup deduplicates its two task-0 producers" + ); + + const FdwicOutputRef mixed_ref{1, 0, 0, 0, 0, 0}; + TaskArgs reader_args; + ConstructTaskArgs(reader_args); + AddOutputHandleTensor( + reader_args, mixed_ref, TensorArgType::Input + ); + SubmitContext reader_context{}; + reader_context.task_id = 2; + reader_context.won = true; + reader_context.result.task_id = 2; + reader_context.shared_result.Reset(2); + SharedTaskWriterDelta reader_delta{}; + LocalStats reader_stats{}; + Check( + PrepareSharedTaskWriterDelta( + reader_args, reader_context, reader_delta + ) && + PublishSharedTaskWriterDelta< + WriterIntentTestOps + >( + &state, reader_context, reader_delta, + reader_stats + ), + "mixed transaction reader publishes an empty completion" + ); + lookup_ok = false; + lookup_count = UINT32_MAX; + const uint32_t reader_fanin = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, reader_args, 2, + static_cast(state.heap_window), + reader_stats, fanin, lookup_ok, + lookup_count, &state.fatal.value + ); + Check( + lookup_ok && reader_fanin == 1 && + fanin[0] == 1 && lookup_count == 0, + "downstream reader consumes the fresh output from mixed task 1" + ); +} + +void TestStrictLatestFaninWindow(SchedulerState &state) { + ResetProtocolState(state); + constexpr int32_t kWindow = 2; + state.heap_window = kWindow; + + // symbol origin 为 task 0，当前 latest writer 为 task 2。reader 4 的 + // 左边界恰好是 2，必须接收；reader 5 的左边界是 3，必须把同一 + // writer 当作窗口外历史，而不是继续塞进 fanin。 + SharedOutputCell &symbol = state.shared_map.shared_outputs[0]; + symbol.published[0].value = 0; + symbol.last_writer[0].value = 2; + SharedWriterHistoryCell &history = + state.shared_map.writer_history[2]; + history.magic = kSharedWriterHistoryMagic; + history.writer_task = 2; + history.count = 1; + history.reserved = 0; + history.entries[0].symbol_key = 1; + history.entries[0].previous_writer = 0; + + const FdwicOutputRef output{0, 0, 0, 0, 0, 0}; + TaskArgs symbol_args; + ConstructTaskArgs(symbol_args); + AddOutputHandleTensor( + symbol_args, output, TensorArgType::Input + ); + LocalStats symbol_stats{}; + int32_t fanin[kMaxFanin] = {}; + bool protocol_ok = false; + uint32_t lookup_count = UINT32_MAX; + const uint32_t boundary_count = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, symbol_args, 4, kWindow, + symbol_stats, fanin, protocol_ok, lookup_count, + &state.fatal.value + ); + Check( + protocol_ok && boundary_count == 1 && + fanin[0] == 2 && lookup_count == 0, + "strict symbol lookup accepts producer exactly at N-H" + ); + protocol_ok = false; + lookup_count = UINT32_MAX; + const uint32_t expired_count = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, symbol_args, 5, kWindow, + symbol_stats, fanin, protocol_ok, lookup_count, + &state.fatal.value + ); + Check( + protocol_ok && expired_count == 0 && + lookup_count == 0, + "strict symbol lookup excludes producer below N-H" + ); + + // TensorDesc::owner_task_id 也属于 fanin 来源，必须使用同一窗口。 + // ordinary ring 为空，因此下面结果只由显式 owner 决定。 + TensorDesc owned = MakeTensor( + 0x480000000ULL, 1 + ); + TaskArgs ordinary_args; + ConstructTaskArgs(ordinary_args); + AddGmTensor( + ordinary_args, owned, TensorArgType::Input + ); + LocalStats ordinary_stats{}; + protocol_ok = false; + lookup_count = UINT32_MAX; + const uint32_t owner_boundary_count = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, ordinary_args, 3, kWindow, + ordinary_stats, fanin, protocol_ok, lookup_count, + &state.fatal.value + ); + Check( + protocol_ok && owner_boundary_count == 1 && + fanin[0] == 1 && lookup_count == 1, + "strict explicit owner accepts producer exactly at N-H" + ); + protocol_ok = false; + lookup_count = UINT32_MAX; + const uint32_t owner_expired_count = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, ordinary_args, 4, kWindow, + ordinary_stats, fanin, protocol_ok, lookup_count, + &state.fatal.value + ); + Check( + protocol_ok && owner_expired_count == 0 && + lookup_count == 1, + "strict explicit owner excludes producer below N-H" + ); + + owned.owner_task_id = 4; + protocol_ok = true; + lookup_count = UINT32_MAX; + const uint32_t self_count = + CollectSharedFanin< + WriterIntentTestOps, false, true + >( + state.shared_map, ordinary_args, 4, kWindow, + ordinary_stats, fanin, protocol_ok, lookup_count, + &state.fatal.value + ); + Check( + !protocol_ok && self_count == 0, + "strict explicit owner rejects self and future producer" + ); +} + +void TestOrderedPublishRejectsFullBucketAtomically( + SchedulerState &state +) { + ResetProtocolState(state); + constexpr uint64_t kAddress = 0x490000000ULL; + const uint32_t bucket = TensorMapHash(kAddress); + bool filled = true; + for (uint32_t cursor = 0; + cursor < kMapBucketCapacity; ++cursor) { + const SharedRegionValue entry{ + kAddress, 0, 4096, + static_cast(cursor), 0 + }; + filled &= + SharedAppendPreparedEntry( + state.shared_map, entry + ); + } + Check(filled, "full-bucket setup publishes exactly CAP live entries"); + + int64_t seq_before[kMapBucketCapacity] = {}; + SharedRegionValue payload_before[kMapBucketCapacity] = {}; + for (uint32_t cursor = 0; + cursor < kMapBucketCapacity; ++cursor) { + SharedRegionSlot &slot = + state.shared_map.slots[ + SharedTensorMapSlotIndex(bucket, cursor) + ]; + seq_before[cursor] = slot.seq.value; + payload_before[cursor] = slot.payload.value; + } + const int64_t head_before = + state.shared_map.buckets[bucket].head.value; + const int64_t tail_before = + state.shared_map.buckets[bucket].tail.value; + + constexpr int32_t kBlockedTask = + static_cast(kMapBucketCapacity); + SetInsertCompletionsAfterTasks( + state, + static_cast(kBlockedTask) + ); + TensorDesc tensor = MakeTensor(kAddress); + TaskArgs args; + ConstructTaskArgs(args); + AddGmTensor(args, tensor, TensorArgType::OutputExisting); + SharedOutputCell &symbol = + state.shared_map.shared_outputs[0]; + symbol.published[0].value = 0; + symbol.last_writer[0].value = 0; + const FdwicOutputRef output{0, 0, 0, 0, 0, 0}; + AddOutputHandleTensor( + args, output, TensorArgType::Inout + ); + SubmitContext context{}; + context.task_id = kBlockedTask; + context.won = true; + context.register_mask = 3; + context.result.task_id = kBlockedTask; + TensorDesc fresh = MakeTensor( + 0x491000000ULL, + static_cast(kBlockedTask) + ); + context.result.count = 1; + context.result.tensors[0] = &fresh; + context.shared_result.Reset(kBlockedTask); + Check( + context.shared_result.AddOutputRef(kBlockedTask, 0), + "blocked mixed task declares a fresh output" + ); + SharedTaskWriterDelta delta{}; + LocalStats stats{}; + Check( + PrepareSharedTaskWriterDelta(args, context, delta) && + delta.ordinary_count == 1 && + delta.writer_intent_required, + "blocked task prepares mixed ordinary and symbol writer intent" + ); + Check( + !PublishSharedTaskWriterDelta( + &state, context, delta, stats + ), + "ordered publish rejects a full live bucket" + ); + + bool slots_unchanged = true; + for (uint32_t cursor = 0; + cursor < kMapBucketCapacity; ++cursor) { + const SharedRegionSlot &slot = + state.shared_map.slots[ + SharedTensorMapSlotIndex(bucket, cursor) + ]; + slots_unchanged &= + slot.seq.value == seq_before[cursor] && + std::memcmp( + &slot.payload.value, &payload_before[cursor], + sizeof(SharedRegionValue) + ) == 0; + } + Check( + state.fatal.value == 1 && + InsertCompletionsMatch( + state, + static_cast(kBlockedTask) + ) && + state.shared_map.buckets[bucket].head.value == + head_before && + state.shared_map.buckets[bucket].tail.value == + tail_before && + slots_unchanged && + symbol.published[0].value == 0 && + symbol.last_writer[0].value == 0 && + state.shared_map.writer_history[kBlockedTask] + .magic == 0 && + state.shared_map.shared_outputs[kBlockedTask] + .published[0].value == -1 && + state.shared_map.shared_outputs[kBlockedTask] + .last_writer[0].value == -1 && + state.shared_map.shared_outputs[kBlockedTask] + .tensors[0].buffer_addr == 0 && + stats.result.map_inserts == 0, + "capacity failure keeps cursor, ring, symbol and fresh metadata unchanged" + ); +} + +void TestCommitStatsRequireCompletionCas( + SchedulerState &state +) { + ResetProtocolState(state); + SetInsertCompletionsAfterTasks(state, 1); + + TensorDesc ordinary = MakeTensor(0x492000000ULL); + TaskArgs args; + ConstructTaskArgs(args); + AddGmTensor( + args, ordinary, TensorArgType::OutputExisting + ); + SharedOutputCell &symbol = + state.shared_map.shared_outputs[0]; + symbol.published[0].value = 0; + symbol.last_writer[0].value = 0; + const FdwicOutputRef output{0, 0, 0, 0, 0, 0}; + AddOutputHandleTensor( + args, output, TensorArgType::Inout + ); + + SubmitContext context{}; + context.task_id = 1; + context.won = true; + context.register_mask = 3; + context.result.task_id = 1; + context.shared_result.Reset(1); + SharedTaskWriterDelta delta{}; + LocalStats stats{}; + Check( + PrepareSharedTaskWriterDelta( + args, context, delta + ) && + delta.prepared_task_id == 1 && + delta.ordinary_count == 1 && + delta.symbol_count == 1, + "completion failure test prepares a mixed writer delta" + ); + + // metadata 可以完整落地，但 task-level completion CAS 必须因非法旧值 + // 失败。成功统计只描述完成事务，不能把这个 terminal 前缀算进去。 + state.tasks[1].deps_prepared = 77; + const uint32_t bucket = TensorMapHash(ordinary.buffer_addr); + Check( + !PublishSharedTaskWriterDelta( + &state, context, delta, stats + ), + "completion CAS conflict rejects the ordered transaction" + ); + Check( + state.fatal.value == 1 && + state.tasks[1].deps_prepared == 77 && + state.shared_map.buckets[bucket].tail.value == 1 && + symbol.last_writer[0].value == 1 && + state.shared_map.writer_history[1].count == 1 && + stats.result.map_inserts == 0 && + stats.result.shared_symbol_inout_commits == 0, + "failed completion keeps metadata evidence but records no committed writers" + ); +} + +void TestOrdinaryWriterRangeValidation() { + SharedRegionValue region{}; + TensorDesc contiguous = MakeTensor(0x470000000ULL); + contiguous.ndims = 2; + contiguous.shapes[0] = 65535; + contiguous.shapes[1] = 65537; + Check( + MakeValidatedSharedWriterRegion( + contiguous, 200, region + ) && + region.lo == 0 && + region.hi == + static_cast(65535) * 65537 * 4, + "ordinary contiguous range accepts a uint32-representable shape product" + ); + + contiguous.shapes[0] = 65536; + contiguous.shapes[1] = 65536; + Check( + MakeValidatedSharedWriterRegion( + contiguous, 200, region + ) && + region.hi == (uint64_t{1} << 34), + "ordinary contiguous range preserves a valid extent above uint32" + ); + + contiguous.ndims = 3; + contiguous.shapes[0] = UINT32_MAX; + contiguous.shapes[1] = UINT32_MAX; + contiguous.shapes[2] = 2; + Check( + !MakeValidatedSharedWriterRegion( + contiguous, 200, region + ), + "ordinary contiguous range rejects a true uint64 shape-product overflow" + ); + + TensorDesc noncontiguous = MakeTensor(0x480000000ULL); + noncontiguous.is_contiguous = false; + noncontiguous.dtype = DataType::Uint8; + noncontiguous.extent_elem_cache = + static_cast(UINT32_MAX) + 1; + Check( + MakeValidatedSharedWriterRegion( + noncontiguous, 201, region + ) && + region.lo == 0 && + region.hi == + static_cast(UINT32_MAX) + 1, + "ordinary noncontiguous range preserves its uint64 cached extent" + ); + + noncontiguous.dtype = DataType::Uint64; + noncontiguous.start_offset = UINT64_MAX / 8; + noncontiguous.extent_elem_cache = 1; + Check( + !MakeValidatedSharedWriterRegion( + noncontiguous, 201, region + ), + "ordinary byte range rejects an end offset that overflows after dtype scaling" + ); +} + +void TestManualWriterNeedsNoGate(SchedulerState &state) { + constexpr int32_t kTask = 160; + ResetTaskGate(state, kTask); + TensorDesc manual = MakeTensor( + 0x430000000ULL, kInvalidTaskId, true + ); + const uint32_t bucket = TensorMapHash(manual.buffer_addr); + const int64_t tail_before = + state.shared_map.buckets[bucket].tail.value; + + TaskArgs args; + ConstructTaskArgs(args); + AddGmTensor(args, manual, TensorArgType::Inout); + bool required = true; + Check( + InspectSharedWriterIntent(args, required) && + !required, + "manual_dep writer is excluded from automatic intent" + ); + SubmitContext context{}; + context.task_id = kTask; + context.won = true; + LocalStats stats{}; + Check( + PrepareSharedWriterIntentSet( + &state, args, context, stats + ) == SharedWriterIntentResult::NotRequired, + "manual_dep writer returns without publishing a gate" + ); + Check( + state.tasks[kTask].deps_prepared == -1 && + state.shared_map.buckets[bucket].tail.value == + tail_before, + "manual_dep writer leaves gate and ordinary map untouched" + ); +} + +} // namespace + +int main() { + SchedulerState *state = MapSparseSchedulerState(); + if (state == nullptr) { + return 1; + } + ResetProtocolState(*state); + TestReaderProgressStateMachine(*state); + ResetProtocolState(*state); + TestSymbolWriterIntentChain(*state); + TestOrdinaryWriterIntentChain(*state); + TestWriterDeltaRequiresExactRegisterMask(); + TestWriterDeltaPrecomputesInterleavedBuckets(*state); + TestOrderedOrdinaryInsertBeforeLookup(*state); + TestOrderedSymbolInsertBeforeLookup(*state); + TestOrderedMixedWriterTransaction(*state); + TestStrictLatestFaninWindow(*state); + TestOrdinaryWriterRangeValidation(); + TestManualWriterNeedsNoGate(*state); + const bool fatal_clean = state->fatal.value == 0; + Check(fatal_clean, "all positive paths leave fatal clear"); + TestCommitStatsRequireCompletionCas(*state); + TestOrderedPublishRejectsFullBucketAtomically(*state); + TestOutOfOrderSymbolWriterFailsClosed(*state); + TestMultiSymbolConflictKeepsTerminalPrefix(*state); + UnmapSparseSchedulerState(state); + if (g_failures != 0) { + std::fprintf( + stderr, "[FAIL] shared writer intent failures=%d\n", + g_failures + ); + return 1; + } + std::printf( + "[PASS] generic shared reader progress, symbol history, " + "terminal-prefix, and ordinary writer-intent tests\n" + ); + return 0; +} diff --git a/tests/atomic_probe/pa_scheduler/test_atomic_dcci_source_coverage.py b/tests/atomic_probe/pa_scheduler/test_atomic_dcci_source_coverage.py new file mode 100644 index 0000000000..1f0c5e1f67 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_atomic_dcci_source_coverage.py @@ -0,0 +1,527 @@ +#!/usr/bin/env python3 +"""检查 standalone PA 生产头中的 atomic/DCCI 观察覆盖。 + +这个门槛刻意先于生产代码改造落地：只要 common 头文件仍直接调用受控 +Ops 原语，测试就逐条报告文件、行号和源码。以后新增调用必须接入统一 +观察封装；真正发生在 trace 建立前或仅供测试的例外，则在调用同一行或 +紧邻上一行写出以下一种标记，并说明具体原因： + + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: pretrace - <原因> + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: test-only - <原因> + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: aggregate - <原因> + // PA_ATOMIC_DCCI_SOURCE_EXEMPT: trace-free - <原因> + +不能用宽泛目录白名单掩盖新调用。唯一的结构性例外是 pa_trace.h 的中央 +观察实现，以及 pa_shared_heap.h 中 ObserveAtomics=false 的两个中央 +编译期 fallback。 +""" + +from __future__ import annotations + +import ast +import re +import unittest +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterable, List, Mapping, Sequence + + +ROOT = Path(__file__).resolve().parent +COMMON = ROOT / "common" +MODEL = COMMON / "pa_model.h" +HOST_SUPPORT = COMMON / "host_support.h" +CONVERTER = ROOT / "swimlane_converter.py" + +CONTROLLED_OPS: Sequence[str] = ( + "Load", + "Exchange", + "FetchAdd", + "FetchMax", + "CompareExchange", + "InvalidateRegion", + "FlushRegion", +) +CONTROLLED_CALL = re.compile( + r"\b(?:Ops|[A-Za-z_]\w*Ops)\s*::\s*(" + + "|".join(CONTROLLED_OPS) + + r")\s*\(" +) +EXEMPT_PREFIX = "PA_ATOMIC_DCCI_SOURCE_EXEMPT:" +EXEMPT_MARKER = re.compile( + r"//\s*PA_ATOMIC_DCCI_SOURCE_EXEMPT:\s*" + r"(pretrace|test-only|aggregate|trace-free)\s*-\s*(\S.*)$" +) + +# 这两个调用是 shared heap 的中央“无观察构建”编译期出口。精确匹配文本 +# 并校验各自只出现一次，防止把整个文件变成不受检查的区域。 +SHARED_HEAP_FALLBACKS: Mapping[str, str] = { + "Load": "return Ops::Load(address);", + "FetchAdd": "return Ops::FetchAdd(address, value);", +} + +ATOMIC_OP_IDS: Mapping[str, int] = { + "Load": 0, + "Exchange": 1, + "FetchAdd": 2, + "FetchMax": 3, + "CompareExchange": 4, +} +DCCI_OP_IDS: Mapping[str, int] = { + "Invalidate": 0, + "CleanOut": 1, +} + + +@dataclass(frozen=True) +class SourceCall: + path: Path + line: int + operation: str + text: str + + def diagnostic(self) -> str: + return f"{self.path.relative_to(ROOT)}:{self.line}: {self.text}" + + +def _strip_cpp_comments_and_literals(source: str) -> str: + """屏蔽注释与字面量，同时保留字符位置和换行，供正则计算准确行号。""" + + output = list(source) + index = 0 + state = "code" + quote = "" + while index < len(source): + char = source[index] + next_char = source[index + 1] if index + 1 < len(source) else "" + + if state == "code": + if char == "/" and next_char == "/": + output[index] = output[index + 1] = " " + index += 2 + state = "line_comment" + continue + if char == "/" and next_char == "*": + output[index] = output[index + 1] = " " + index += 2 + state = "block_comment" + continue + if char in ('"', "'"): + quote = char + output[index] = " " + index += 1 + state = "literal" + continue + index += 1 + continue + + if state == "line_comment": + if char == "\n": + state = "code" + else: + output[index] = " " + index += 1 + continue + + if state == "block_comment": + if char == "*" and next_char == "/": + output[index] = output[index + 1] = " " + index += 2 + state = "code" + else: + if char != "\n": + output[index] = " " + index += 1 + continue + + # string/character literal + if char == "\\": + output[index] = " " + if index + 1 < len(source): + if source[index + 1] != "\n": + output[index + 1] = " " + index += 2 + else: + index += 1 + continue + if char == quote: + output[index] = " " + index += 1 + state = "code" + continue + if char != "\n": + output[index] = " " + index += 1 + + return "".join(output) + + +def _source_calls(path: Path) -> List[SourceCall]: + source = path.read_text(encoding="utf-8") + searchable = _strip_cpp_comments_and_literals(source) + lines = source.splitlines() + calls: List[SourceCall] = [] + for match in CONTROLLED_CALL.finditer(searchable): + line = searchable.count("\n", 0, match.start()) + 1 + calls.append( + SourceCall( + path=path, + line=line, + operation=match.group(1), + text=lines[line - 1].strip(), + ) + ) + return calls + + +def _has_explicit_exemption(call: SourceCall) -> bool: + lines = call.path.read_text(encoding="utf-8").splitlines() + # 标记只绑定同一行或紧邻上一行，避免一个标记无意豁免整个代码块。 + for line_index in (call.line - 1, call.line - 2): + if line_index < 0: + continue + if EXEMPT_MARKER.search(lines[line_index]): + return True + return False + + +def _is_shared_heap_fallback(call: SourceCall) -> bool: + expected = SHARED_HEAP_FALLBACKS.get(call.operation) + return ( + call.path.name == "pa_shared_heap.h" + and expected is not None + and call.text == expected + ) + + +def _parse_atomic_site_enum() -> tuple[Dict[str, int], int]: + source = _strip_cpp_comments_and_literals( + MODEL.read_text(encoding="utf-8") + ) + match = re.search( + r"enum\s+class\s+AtomicSite\s*:\s*uint32_t\s*\{(.*?)\};", + source, + re.DOTALL, + ) + if match is None: + raise AssertionError(f"{MODEL}: 找不到 AtomicSite 枚举") + + sites: Dict[str, int] = {} + count: int | None = None + for item in match.group(1).split(","): + item = item.strip() + if not item: + continue + entry = re.fullmatch(r"([A-Za-z_]\w*)\s*=\s*(\d+)", item) + if entry is None: + raise AssertionError( + f"{MODEL}: 无法解析 AtomicSite 条目: {item!r}" + ) + name, raw_value = entry.groups() + value = int(raw_value) + if name == "Count": + count = value + else: + sites[name] = value + if count is None: + raise AssertionError(f"{MODEL}: AtomicSite 缺少 Count") + return sites, count + + +def _parse_host_atomic_site_names() -> List[str]: + source = HOST_SUPPORT.read_text(encoding="utf-8") + function = re.search( + r"inline\s+const\s+char\s*\*\s*AtomicSiteName\s*\(" + r".*?\)\s*\{(.*?)\n\}", + source, + re.DOTALL, + ) + if function is None: + raise AssertionError(f"{HOST_SUPPORT}: 找不到 AtomicSiteName") + names = re.search( + r"const\s+char\s*\*\s*names\s*\[\s*\]\s*=\s*\{(.*?)\};", + function.group(1), + re.DOTALL, + ) + if names is None: + raise AssertionError(f"{HOST_SUPPORT}: 找不到 AtomicSiteName::names") + return re.findall(r'"([^"]+)"', names.group(1)) + + +def _parse_python_literal_assignments( + path: Path, requested: Iterable[str] +) -> Dict[str, object]: + requested_set = set(requested) + parsed = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + values: Dict[str, object] = {} + for node in parsed.body: + if not isinstance(node, ast.Assign) or len(node.targets) != 1: + continue + target = node.targets[0] + if not isinstance(target, ast.Name) or target.id not in requested_set: + continue + values[target.id] = ast.literal_eval(node.value) + missing = requested_set - values.keys() + if missing: + raise AssertionError( + f"{path}: 缺少 Python 映射: {', '.join(sorted(missing))}" + ) + return values + + +def _parse_cpp_atomic_site_ops( + sites: Mapping[str, int] +) -> Dict[int, int]: + source = _strip_cpp_comments_and_literals( + MODEL.read_text(encoding="utf-8") + ) + function = re.search( + r"AtomicSiteExpectedOp\s*\(\s*AtomicSite\s+site\s*\)\s*\{" + r"(.*?)\n\}", + source, + re.DOTALL, + ) + if function is None: + raise AssertionError(f"{MODEL}: 找不到 AtomicSiteExpectedOp") + + # default 分支是 Load；显式 case 只覆盖其它操作。 + result = {value: ATOMIC_OP_IDS["Load"] for value in sites.values()} + groups = re.finditer( + r"((?:case\s+AtomicSite::[A-Za-z_]\w*\s*:\s*)+)" + r"return\s+AtomicOp::([A-Za-z_]\w*)\s*;", + function.group(1), + re.DOTALL, + ) + for group in groups: + operation = group.group(2) + if operation not in ATOMIC_OP_IDS: + raise AssertionError( + f"{MODEL}: 未知 AtomicOp::{operation}" + ) + for name in re.findall( + r"case\s+AtomicSite::([A-Za-z_]\w*)\s*:", + group.group(1), + ): + if name == "Count": + continue + if name not in sites: + raise AssertionError( + f"{MODEL}: AtomicSiteExpectedOp 引用了未知站点 {name}" + ) + result[sites[name]] = ATOMIC_OP_IDS[operation] + return result + + +def _parse_dcci_site_enum() -> tuple[Dict[str, int], int]: + source = _strip_cpp_comments_and_literals( + MODEL.read_text(encoding="utf-8") + ) + match = re.search( + r"enum\s+class\s+DcciSite\s*:\s*uint32_t\s*\{(.*?)\};", + source, + re.DOTALL, + ) + if match is None: + raise AssertionError(f"{MODEL}: 找不到 DcciSite 枚举") + sites: Dict[str, int] = {} + count: int | None = None + for item in match.group(1).split(","): + item = item.strip() + if not item: + continue + entry = re.fullmatch(r"([A-Za-z_]\w*)\s*=\s*(\d+)", item) + if entry is None: + raise AssertionError( + f"{MODEL}: 无法解析 DcciSite 条目: {item!r}" + ) + name, raw_value = entry.groups() + value = int(raw_value) + if name == "Count": + count = value + else: + sites[name] = value + if count is None: + raise AssertionError(f"{MODEL}: DcciSite 缺少 Count") + return sites, count + + +def _parse_host_dcci_site_names() -> List[str]: + source = HOST_SUPPORT.read_text(encoding="utf-8") + function = re.search( + r"inline\s+const\s+char\s*\*\s*DcciSiteName\s*\(" + r".*?\)\s*\{(.*?)\n\}", + source, + re.DOTALL, + ) + if function is None: + raise AssertionError(f"{HOST_SUPPORT}: 找不到 DcciSiteName") + names = re.search( + r"const\s+char\s*\*\s*names\s*\[\s*\]\s*=\s*\{(.*?)\};", + function.group(1), + re.DOTALL, + ) + if names is None: + raise AssertionError(f"{HOST_SUPPORT}: 找不到 DcciSiteName::names") + return re.findall(r'"([^"]+)"', names.group(1)) + + +def _parse_cpp_dcci_site_ops( + sites: Mapping[str, int] +) -> Dict[int, int]: + source = _strip_cpp_comments_and_literals( + MODEL.read_text(encoding="utf-8") + ) + function = re.search( + r"DcciSiteExpectedOp\s*\(\s*DcciSite\s+site\s*\)\s*\{" + r"(.*?)\n\}", + source, + re.DOTALL, + ) + if function is None: + raise AssertionError(f"{MODEL}: 找不到 DcciSiteExpectedOp") + result = { + value: DCCI_OP_IDS["Invalidate"] for value in sites.values() + } + groups = re.finditer( + r"((?:case\s+DcciSite::[A-Za-z_]\w*\s*:\s*)+)" + r"return\s+DcciOp::([A-Za-z_]\w*)\s*;", + function.group(1), + re.DOTALL, + ) + for group in groups: + operation = group.group(2) + if operation not in DCCI_OP_IDS: + raise AssertionError(f"{MODEL}: 未知 DcciOp::{operation}") + for name in re.findall( + r"case\s+DcciSite::([A-Za-z_]\w*)\s*:", + group.group(1), + ): + if name not in sites: + raise AssertionError( + f"{MODEL}: DcciSiteExpectedOp 引用了未知站点 {name}" + ) + result[sites[name]] = DCCI_OP_IDS[operation] + return result + + +class AtomicDcciSourceCoverageTest(unittest.TestCase): + maxDiff = None + + def test_production_headers_have_no_bare_controlled_ops(self) -> None: + violations: List[SourceCall] = [] + heap_fallback_counts = { + operation: 0 for operation in SHARED_HEAP_FALLBACKS + } + + for path in sorted(COMMON.glob("*.h")): + # TraceAtomic/TraceDCCI 的中央实现必须最终落到真实 Ops。 + if path.name == "pa_trace.h": + continue + for call in _source_calls(path): + if _is_shared_heap_fallback(call): + heap_fallback_counts[call.operation] += 1 + continue + if _has_explicit_exemption(call): + continue + violations.append(call) + + self.assertEqual( + heap_fallback_counts, + {operation: 1 for operation in SHARED_HEAP_FALLBACKS}, + "pa_shared_heap.h 应且只应保留两个 ObserveAtomics=false " + f"中央 fallback，实际为 {heap_fallback_counts}", + ) + if violations: + rendered = "\n".join( + f" {call.diagnostic()}" for call in violations + ) + self.fail( + "发现未接入统一观察封装的裸 atomic/DCCI 调用 " + f"({len(violations)}处)：\n{rendered}" + ) + + def test_exemption_markers_are_well_formed(self) -> None: + malformed: List[str] = [] + for path in sorted(COMMON.glob("*.h")): + for line_number, line in enumerate( + path.read_text(encoding="utf-8").splitlines(), start=1 + ): + if EXEMPT_PREFIX in line and EXEMPT_MARKER.search(line) is None: + malformed.append( + f"{path.relative_to(ROOT)}:{line_number}: {line.strip()}" + ) + if malformed: + self.fail( + "发现格式错误或缺少具体原因的源码豁免标记：\n " + + "\n ".join(malformed) + ) + + def test_atomic_site_mapping_counts_and_ops_match(self) -> None: + sites, count = _parse_atomic_site_enum() + expected_ids = list(range(count)) + self.assertEqual( + sorted(sites.values()), + expected_ids, + "AtomicSite 必须显式编号且在 [0, Count) 内连续", + ) + + host_names = _parse_host_atomic_site_names() + self.assertEqual( + len(host_names), + count, + "host AtomicSiteName 数量必须等于 AtomicSite::Count", + ) + + python_values = _parse_python_literal_assignments( + CONVERTER, ("ATOMIC_SITE_NAMES", "ATOMIC_SITE_OP_IDS") + ) + python_names = python_values["ATOMIC_SITE_NAMES"] + python_ops = python_values["ATOMIC_SITE_OP_IDS"] + self.assertIsInstance(python_names, dict) + self.assertIsInstance(python_ops, dict) + self.assertEqual( + sorted(python_names), + expected_ids, + "Python ATOMIC_SITE_NAMES 键必须完整覆盖 [0, Count)", + ) + self.assertEqual( + sorted(python_ops), + expected_ids, + "Python ATOMIC_SITE_OP_IDS 键必须完整覆盖 [0, Count)", + ) + self.assertEqual( + python_ops, + _parse_cpp_atomic_site_ops(sites), + "Python site->op 映射必须与 C++ AtomicSiteExpectedOp 一致", + ) + + def test_dcci_site_mapping_counts_and_ops_match(self) -> None: + sites, count = _parse_dcci_site_enum() + expected_ids = list(range(count)) + self.assertEqual( + sorted(sites.values()), + expected_ids, + "DcciSite 必须显式编号且在 [0, Count) 内连续", + ) + self.assertEqual( + len(_parse_host_dcci_site_names()), + count, + "host DcciSiteName 数量必须等于 DcciSite::Count", + ) + python_values = _parse_python_literal_assignments( + CONVERTER, ("DCCI_SITE_NAMES", "DCCI_SITE_OP_IDS") + ) + python_names = python_values["DCCI_SITE_NAMES"] + python_ops = python_values["DCCI_SITE_OP_IDS"] + self.assertIsInstance(python_names, dict) + self.assertIsInstance(python_ops, dict) + self.assertEqual(sorted(python_names), expected_ids) + self.assertEqual(sorted(python_ops), expected_ids) + self.assertEqual( + python_ops, + _parse_cpp_dcci_site_ops(sites), + "Python DCCI site->op 映射必须与 C++ DcciSiteExpectedOp 一致", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/atomic_probe/pa_scheduler/test_ccec_artifact_manifest.py b/tests/atomic_probe/pa_scheduler/test_ccec_artifact_manifest.py new file mode 100644 index 0000000000..5c6c30547c --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_ccec_artifact_manifest.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 + +import hashlib +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +HERE = Path(__file__).resolve().parent +RUN_SH = HERE / "run.sh" + + +class CcecArtifactManifestTest(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.run_sh = self.root / "run.sh" + shutil.copy2(RUN_SH, self.run_sh) + self.run_sh.chmod(0o755) + + def tearDown(self) -> None: + self.temporary.cleanup() + + def _publish_artifacts( + self, mode: str, variant: str, phase: str = "none" + ) -> tuple[Path, list[str]]: + if variant == "submit-pmu": + build_dir = ( + self.root / "build" / "ccec" / mode / + "submit-pmu" / phase + ) + artifacts = [ + "pa_scheduler_host", + "pa_scheduler_kernel.o", + "libpa_scheduler_pmu_owner_aicpu.so", + "libpa_scheduler_pmu_owner_dispatcher.so", + ] + else: + build_dir = ( + self.root / "build" / "ccec" / mode / variant + ) + artifacts = [ + "pa_scheduler_host", + "pa_scheduler_kernel.o", + ] + # 同一测试方法会依次破坏多个字段；每个 subTest 都从这里重新 + # 发布完整基线，覆盖上一个 subTest 留下的 manifest。 + build_dir.mkdir(parents=True, exist_ok=True) + for artifact in artifacts: + path = build_dir / artifact + if artifact == "pa_scheduler_host": + path.write_text( + "#!/usr/bin/env bash\nexit 0\n", + encoding="utf-8", + ) + path.chmod(0o755) + else: + path.write_bytes( + f"{mode}:{variant}:{phase}:{artifact}\n".encode() + ) + + if mode == "shared": + submit_claim_bytes = 32 + records_per_core = 28416 + if variant == "swimlane": + generic_bytes = 16 + worker_stride = 593920 + else: + generic_bytes = 32 + worker_stride = 1048576 + mode_id = 1 + else: + generic_bytes = 32 + submit_claim_bytes = 0 + records_per_core = 65536 + worker_stride = 2097152 + mode_id = 0 + + phase_ids = { + "none": 0, + "claim": 1, + "efdrain": 2, + "materialize": 4, + "register": 5, + } + lines = [ + "# schema=pa_scheduler_artifacts/v4", + f"# tensormap_mode={mode}", + f"# tensormap_mode_id={mode_id}", + "# tensormap_ring_cap=128", + "# shared_insert_turn_groups=1", + f"# generic_record_bytes={generic_bytes}", + f"# submit_claim_record_bytes={submit_claim_bytes}", + f"# records_per_core={records_per_core}", + f"# worker_stride_bytes={worker_stride}", + f"# variant={variant}", + f"# phase={phase}", + f"# phase_id={phase_ids[phase]}", + ] + for artifact in artifacts: + data = (build_dir / artifact).read_bytes() + lines.append( + f"{hashlib.sha256(data).hexdigest()} {artifact}" + ) + (build_dir / "pa_scheduler_artifacts.manifest").write_text( + "\n".join(lines) + "\n", + encoding="utf-8", + ) + return build_dir, lines + + def _run( + self, mode: str, variant: str, phase: str = "none" + ) -> subprocess.CompletedProcess[str]: + if variant == "swimlane": + arguments = [ + str(self.run_sh), "run", "ccec", + "--tensormap", mode, + ] + elif variant == "perf-clock": + arguments = [ + str(self.run_sh), "perf-clock", "ccec", + "--tensormap", mode, + ] + else: + arguments = [ + str(self.run_sh), "submit-pmu", "ccec", phase, + "--tensormap", mode, + ] + return subprocess.run( + arguments, + cwd=self.root, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + def test_accepts_each_unique_layout(self) -> None: + cases = [ + ("private", "swimlane", "none"), + ("shared", "swimlane", "none"), + ("shared", "perf-clock", "none"), + ("shared", "submit-pmu", "register"), + ] + for mode, variant, phase in cases: + with self.subTest( + mode=mode, variant=variant, phase=phase + ): + self._publish_artifacts(mode, variant, phase) + completed = self._run(mode, variant, phase) + self.assertEqual( + completed.returncode, + 0, + completed.stdout + completed.stderr, + ) + self.assertIn( + "CCEC artifact manifest verified", + completed.stdout, + ) + + def test_rejects_schema_and_each_layout_field_mutation(self) -> None: + mutations = { + "old schema": (0, "# schema=pa_scheduler_artifacts/v3"), + "generic bytes": (5, "# generic_record_bytes=32"), + "submit claim bytes": ( + 6, "# submit_claim_record_bytes=16" + ), + "record count": (7, "# records_per_core=65536"), + "worker stride": (8, "# worker_stride_bytes=1048576"), + } + for label, (index, replacement) in mutations.items(): + with self.subTest(label=label): + build_dir, lines = self._publish_artifacts( + "shared", "swimlane" + ) + lines[index] = replacement + ( + build_dir / + "pa_scheduler_artifacts.manifest" + ).write_text( + "\n".join(lines) + "\n", + encoding="utf-8", + ) + completed = self._run("shared", "swimlane") + self.assertNotEqual(completed.returncode, 0) + self.assertIn( + "trace layout", + completed.stderr, + ) + + def test_rejects_reordered_or_extra_identity_lines(self) -> None: + for label in ("reordered", "extra"): + with self.subTest(label=label): + build_dir, lines = self._publish_artifacts( + "shared", "swimlane" + ) + if label == "reordered": + lines[5], lines[6] = lines[6], lines[5] + else: + lines.insert(12, "# unexpected=1") + ( + build_dir / + "pa_scheduler_artifacts.manifest" + ).write_text( + "\n".join(lines) + "\n", + encoding="utf-8", + ) + completed = self._run("shared", "swimlane") + self.assertNotEqual(completed.returncode, 0) + + def test_rejects_artifact_checksum_mismatch(self) -> None: + build_dir, _ = self._publish_artifacts( + "shared", "swimlane" + ) + with (build_dir / "pa_scheduler_kernel.o").open("ab") as output: + output.write(b"tampered\n") + completed = self._run("shared", "swimlane") + self.assertNotEqual(completed.returncode, 0) + self.assertIn("SHA256", completed.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/atomic_probe/pa_scheduler/test_pmu_html_report.py b/tests/atomic_probe/pa_scheduler/test_pmu_html_report.py new file mode 100644 index 0000000000..45ed5db0a8 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_pmu_html_report.py @@ -0,0 +1,502 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +"""standalone submit-pmu HTML 可视报告的生成与失败原子性回归。""" + +from __future__ import annotations + +import json +import re +import stat +import tempfile +import unittest +from pathlib import Path + +try: + from .pmu_html_report import ( + DEFAULT_AIC_PMU_CYCLES_PER_NS, + DEFAULT_AIV_PMU_CYCLES_PER_NS, + DEFAULT_PMU_CYCLES_PER_NS, + PMU_CALIBRATION_CYCLE_DELTA, + PMU_CALIBRATION_SYS_TICK_NS, + _cycles_to_us, + _phase_share_metric, + default_output_path, + render_report, + write_report, + ) + from .pmu_sidecar_analyzer import analyze + from .test_pmu_sidecar_analyzer import _submit_pmu_capture, _submit_pmu_summary +except ImportError: + from pmu_html_report import ( + DEFAULT_AIC_PMU_CYCLES_PER_NS, + DEFAULT_AIV_PMU_CYCLES_PER_NS, + DEFAULT_PMU_CYCLES_PER_NS, + PMU_CALIBRATION_CYCLE_DELTA, + PMU_CALIBRATION_SYS_TICK_NS, + _cycles_to_us, + _phase_share_metric, + default_output_path, + render_report, + write_report, + ) + from pmu_sidecar_analyzer import analyze + from test_pmu_sidecar_analyzer import _submit_pmu_capture, _submit_pmu_summary + + +class PmuHtmlReportTest(unittest.TestCase): + def _write_capture( + self, directory: str, capture: dict, name: str = "submit_icache_raw.json" + ) -> Path: + path = Path(directory) / name + path.write_text(json.dumps(capture, ensure_ascii=False), encoding="utf-8") + return path + + def test_default_output_uses_descriptive_report_name(self) -> None: + self.assertEqual( + default_output_path(Path("result/submit_icache_raw.json")), + Path("result/submit_icache_report.html"), + ) + self.assertEqual( + default_output_path(Path("result/custom.json")), + Path("result/custom_report.html"), + ) + + def test_none_report_reuses_analyzer_values_and_has_96_rows(self) -> None: + capture = _submit_pmu_capture(phase="none") + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture) + analysis = analyze([path]) + document = render_report(path) + + aiv = analysis["per_run"][0]["groups"]["aiv"] + self.assertIn("Standalone PA Submit I-cache 报告", document) + self.assertIn(f"{aiv['icache_misses_per_core']:,.2f}", document) + self.assertIn(f"{aiv['icache_miss_rate'] * 100:.4f}%", document) + self.assertIn("PRIMARY ↔ SHADOW EXACT 96/96", document) + self.assertIn("局部 phase：none", document) + self.assertIn("局部阶段总览：none", document) + self.assertIn("不适用：该 ELF 未编译局部阶段", document) + self.assertLess( + document.index("局部阶段总览：none"), + document.index('
'), + ) + self.assertNotIn('data-metric="time"', document) + self.assertIn("PMU total 与 scalar busy", document) + self.assertIn(f"{aiv['total_cycles_sum'] / 64:,.2f}", document) + self.assertIn(f"{aiv['scalar_busy_sum'] / 64:,.2f}", document) + self.assertIn("非 Scalar-busy 残余", document) + self.assertIn("它不是空闲时间，也不是 I-cache stall", document) + self.assertIn("不提供 scalar_wait_ib_time/scalar_wait_time", document) + self.assertIn("完整 Submit（最早开始 → 最晚结束）", document) + self.assertIn("96 核逐核 PMU total 平均值（校准）", document) + self.assertNotIn("包络", document) + self.assertEqual(document.count('class="pmu-role-card"'), 3) + self.assertEqual(document.count('class="pmu-compact-table"'), 3) + self.assertEqual(document.count('data-stat="min"'), 3) + self.assertEqual(document.count('data-stat="mean"'), 3) + self.assertEqual(document.count('data-stat="max"'), 3) + self.assertNotIn('data-stat="median"', document) + self.assertNotIn('data-stat="p95"', document) + self.assertIn(".pmu-role-grid { display:grid;", document) + self.assertIn(".pmu-role-card { min-width:0;", document) + self.assertIn('
\n ', document) + self.assertGreaterEqual(document.count('class="table-scroll"'), 2) + self.assertIn("total_cycles=", document) + self.assertIn("不是 Submit 墙钟损失", document) + self.assertIn("PMU cycle 频率校准", document) + self.assertIn(f"PMU cycle_delta = {PMU_CALIBRATION_CYCLE_DELTA:,}", document) + self.assertIn( + f"SYS_CNT tick_delta = {PMU_CALIBRATION_SYS_TICK_NS:,} ns", document + ) + self.assertIn(f"ALL = {DEFAULT_PMU_CYCLES_PER_NS:.6f}", document) + self.assertIn(f"AIC = {DEFAULT_AIC_PMU_CYCLES_PER_NS:.6f}", document) + self.assertIn(f"AIV = {DEFAULT_AIV_PMU_CYCLES_PER_NS:.6f}", document) + all_group = analysis["per_run"][0]["groups"]["all"] + all_total_per_core = all_group["total_cycles_sum"] / all_group["cores"] + self.assertIn( + f'
{_cycles_to_us(all_total_per_core, DEFAULT_PMU_CYCLES_PER_NS):,.3f} µs
', + document, + ) + self.assertIn("生成器：pmu_html_report schema v5", document) + self.assertEqual(document.count('data-worker-id="'), 96) + self.assertNotIn("http://", document) + self.assertNotIn("https://", document) + self.assertNotIn(" None: + capture = _submit_pmu_capture( + phase="claim", + schema_version=6, + shared_context_lens=[0, 8192, 8193, 32768], + ) + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture) + document = render_report(path) + + self.assertIn("shared dynamic 32 task/core", document) + self.assertIn('data-shared-task-plan="true"', document) + self.assertIn("shared 动态 task 计划", document) + self.assertIn('
32 task/核
', document) + self.assertIn("group 总数 7 · context 最小 0 · 最大 32,768", document) + self.assertNotIn("shared_context_lens", document) + self.assertNotIn("[0, 8192, 8193, 32768]", document) + + def test_v5_report_does_not_show_shared_dynamic_task_plan(self) -> None: + capture = _submit_pmu_capture(phase="claim", schema_version=5) + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture) + document = render_report(path) + + self.assertNotIn("shared dynamic", document) + self.assertNotIn('data-shared-task-plan="true"', document) + self.assertNotIn("shared 动态 task 计划", document) + + def test_pmu_role_cards_derive_min_mean_max_from_validated_records(self) -> None: + capture = _submit_pmu_capture(phase="none") + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture) + document = render_report(path) + + frequencies = { + "all": DEFAULT_PMU_CYCLES_PER_NS, + "aic": DEFAULT_AIC_PMU_CYCLES_PER_NS, + "aiv": DEFAULT_AIV_PMU_CYCLES_PER_NS, + } + for group_name in ("all", "aic", "aiv"): + group_records = ( + capture["records"] + if group_name == "all" + else [record for record in capture["records"] if record["role"] == group_name] + ) + self.assertNotIn("min", capture["summary"][group_name]["total_cycles"]) + card_match = re.search( + rf'
(.*?)
', + document, + re.DOTALL, + ) + self.assertIsNotNone(card_match) + card = card_match.group(1) + cycles_per_ns = frequencies[group_name] + for metric in ("total_cycles", "scalar_busy"): + values = [int(record[metric]) for record in group_records] + for value in (min(values), max(values)): + self.assertIn(f"{value:,} cycle", card) + self.assertIn(f"≈{_cycles_to_us(value, cycles_per_ns):,.3f} µs", card) + self.assertEqual(card.count('data-stat="'), 3) + + def test_icache_overview_shows_sum_and_per_core_min_mean_max(self) -> None: + capture = _submit_pmu_capture(phase="none") + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture) + document = render_report(path) + + all_records = capture["records"] + all_requests = [int(record["icache_requests"]) for record in all_records] + all_misses = [int(record["icache_misses"]) for record in all_records] + for label, values in ( + ("request", all_requests), + ("miss", all_misses), + ): + card_match = re.search( + rf'
完整 Submit primary I-cache {label}' + rf'（96 核总和）
{sum(values):,}
' + rf'
逐核平均 {sum(values) / len(values):,.2f}
' + rf'
逐核最小 {min(values):,} · 逐核最大 {max(values):,}
', + document, + ) + self.assertIsNotNone(card_match) + + table_match = re.search( + r'
(.*?)
', document, re.DOTALL + ) + self.assertIsNotNone(table_match) + table = table_match.group(1) + self.assertIn("request min/core", table) + self.assertIn("request mean/core", table) + self.assertIn("request max/core", table) + self.assertIn("miss min/core", table) + self.assertIn("miss mean/core", table) + self.assertIn("miss max/core", table) + for role in ("aic", "aiv"): + records = [record for record in all_records if record["role"] == role] + requests = [int(record["icache_requests"]) for record in records] + misses = [int(record["icache_misses"]) for record in records] + expected_cells = ( + min(requests), + sum(requests) / len(requests), + max(requests), + min(misses), + sum(misses) / len(misses), + max(misses), + ) + row_match = re.search( + rf'{role.upper()}' + rf'{len(records):,}' + rf'{expected_cells[0]:,}' + rf'{expected_cells[1]:,.2f}' + rf'{expected_cells[2]:,}' + rf'{expected_cells[3]:,}' + rf'{expected_cells[4]:,.2f}' + rf'{expected_cells[5]:,}', + table, + ) + self.assertIsNotNone(row_match) + + self.assertNotIn("median", table.lower()) + self.assertNotIn("p95", table.lower()) + + def test_role_specific_frequency_converts_per_core_cycles(self) -> None: + capture = _submit_pmu_capture(phase="none") + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture) + document = render_report(path) + + for role, cycles_per_ns in ( + ("aic", DEFAULT_AIC_PMU_CYCLES_PER_NS), + ("aiv", DEFAULT_AIV_PMU_CYCLES_PER_NS), + ): + record = next(item for item in capture["records"] if item["role"] == role) + total = int(record["total_cycles"]) + self.assertIn( + f'{total:,} cycle' + f'≈{_cycles_to_us(total, cycles_per_ns):,.3f} µs', + document, + ) + + def test_frequency_override_changes_group_and_role_conversion(self) -> None: + capture = _submit_pmu_capture(phase="none") + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture) + document = render_report( + path, + pmu_cycles_per_ns=2.0, + aic_pmu_cycles_per_ns=4.0, + aiv_pmu_cycles_per_ns=8.0, + ) + + all_total_per_core = capture["summary"]["all"]["total_cycles"]["sum"] / 96 + self.assertIn(f'
{all_total_per_core / 2.0 / 1000:,.3f} µs
', document) + self.assertIn("ALL = 2.000000", document) + self.assertIn("AIC = 4.000000", document) + self.assertIn("AIV = 8.000000", document) + for role, cycles_per_ns in (("aic", 4.0), ("aiv", 8.0)): + record = next(item for item in capture["records"] if item["role"] == role) + self.assertIn( + f"calibrated_time={int(record['total_cycles']) / cycles_per_ns / 1000:,.3f}us", + document, + ) + + def test_invalid_pmu_frequency_is_rejected_before_publish(self) -> None: + capture = _submit_pmu_capture(phase="none") + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture) + output = default_output_path(path) + with self.assertRaisesRegex(ValueError, "pmu_cycles_per_ns must be finite and positive"): + write_report(path, pmu_cycles_per_ns=0.0) + self.assertFalse(output.exists()) + + def test_running_phase_shows_bounds_and_observer_warning(self) -> None: + capture = _submit_pmu_capture(phase="claim") + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture) + analysis = analyze([path]) + document = render_report(path) + + group = analysis["per_run"][0]["groups"]["all"] + self.assertIn("局部阶段总览：claim", document) + self.assertIn("局部阶段详细数据：claim", document) + self.assertIn("带边界扰动的诊断区间", document) + self.assertIn(f"{group['phase_calls_sum']:,}", document) + self.assertIn( + f"{group['phase_icache_misses_lower_bound_sum']:,}.." + f"{group['phase_icache_misses_upper_bound_sum']:,}", + document, + ) + self.assertLess( + document.index("局部阶段总览：claim"), + document.index('
'), + ) + self.assertEqual(document.count('class="phase-share-card"'), 3) + self.assertEqual(document.count('class="phase-share-metric"'), 9) + self.assertEqual(document.count('class="phase-share-track '), 9) + self.assertEqual(document.count('data-metric="time"'), 3) + for group_name in ("all", "aic", "aiv"): + phase_group = analysis["per_run"][0]["groups"][group_name] + time_share = phase_group["phase_time_share_of_submit"] + self.assertIn( + f'data-phase-group="{group_name}" data-metric="time" ' + f'data-time-share="{time_share:.12f}"', + document, + ) + self.assertIn(f"{time_share * 100:.4f}%", document) + self.assertIn( + f"平均 {phase_group['phase_elapsed_per_core_us']:,.3f} µs/核", + document, + ) + for metric, lower_key, upper_key in ( + ( + "request", + "phase_icache_request_lower_bound_share_of_submit", + "phase_icache_request_upper_bound_share_of_submit", + ), + ( + "miss", + "phase_icache_miss_lower_bound_share_of_submit", + "phase_icache_miss_upper_bound_share_of_submit", + ), + ): + self.assertIn( + f'data-phase-group="{group_name}" data-metric="{metric}" ' + f'data-lower-share="{phase_group[lower_key]:.12f}" ' + f'data-upper-share="{phase_group[upper_key]:.12f}"', + document, + ) + lower_percent = phase_group[lower_key] * 100.0 + upper_percent = phase_group[upper_key] * 100.0 + metric_pattern = re.compile( + rf'data-phase-group="{group_name}" data-metric="{metric}" ' + rf'data-lower-share="{phase_group[lower_key]:.12f}" ' + rf'data-upper-share="{phase_group[upper_key]:.12f}".*?' + rf'下界 {lower_percent:.4f}%，上界 {upper_percent:.4f}%.*?' + rf'class="phase-share-upper" style="width:{upper_percent:.6f}%".*?' + rf'class="phase-share-lower" style="width:{lower_percent:.6f}%".*?' + rf'class="phase-share-upper-marker" style="left:{upper_percent:.6f}%"', + re.DOTALL, + ) + self.assertRegex(document, metric_pattern) + self.assertEqual( + document.count( + '
0%50%100%
' + ), + 9, + ) + self.assertIn('class="phase-table-scroll"', document) + self.assertIn('class="phase-table"', document) + self.assertIn('
', document) + self.assertNotIn('
', document) + self.assertIn( + '
\n ', + document, + ) + self.assertIn("展开 ALL / AIC / AIV 完整数字表", document) + self.assertIn(".phase-panel { overflow:hidden; }", document) + self.assertIn( + ".phase-table-scroll { width:100%; max-width:100%; overflow-x:auto;", + document, + ) + self.assertIn('class="phase-plot-scroll"', document) + self.assertIn( + ".phase-plot-scroll { width:100%; max-width:100%; overflow-x:auto;", + document, + ) + self.assertIn("不同 phase ELF 的局部值不能相加", document) + self.assertIn("phase_icache_misses=", document) + self.assertIn( + "Σ阶段 SYS_CNT / Σ同核首个 submit_begin 计时点到末个 submit_end 计时点 SYS_CNT", + document, + ) + self.assertIn("不包含两侧 ld_dev", document) + + def test_legacy_v4_phase_time_is_explicitly_unavailable(self) -> None: + capture = _submit_pmu_capture(phase="claim", schema_version=4) + for field in ( + "phase_time_observation_included", + "phase_time_sys_counter_tick_ns", + "phase_time_boundary", + "phase_time_excludes_shadow_read_overhead", + "phase_time_includes_timestamp_overhead", + "phase_time_share_definition", + "phase_time_denominator_scope", + ): + capture["configuration"].pop(field) + capture["validation"].pop("phase_time_valid_records") + capture["validation"].pop("phase_time_measurement_valid") + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture) + document = render_report(path) + + self.assertEqual(document.count('data-metric="time" data-time-share=""'), 3) + self.assertIn("本次 raw 未采集阶段 SYS_CNT", document) + self.assertNotIn("None%", document) + self.assertNotIn("nan%", document.lower()) + + def test_phase_share_equal_bounds_keep_visible_value_and_marker(self) -> None: + fragment = _phase_share_metric("AIC", "I-cache miss", "miss", 0.25, 0.25, 10, 10) + self.assertIn("25.0000%..25.0000%", fragment) + self.assertIn('class="phase-share-upper" style="width:25.000000%"', fragment) + self.assertIn('class="phase-share-lower" style="width:25.000000%"', fragment) + self.assertIn('class="phase-share-upper-marker" style="left:25.000000%"', fragment) + + def test_phase_share_zero_denominator_is_explicitly_unavailable(self) -> None: + fragment = _phase_share_metric("AIC", "I-cache miss", "miss", None, None, 0, 0) + self.assertIn("比例不可计算", fragment) + self.assertNotIn("None%", fragment) + self.assertNotIn("nan%", fragment.lower()) + + def test_zero_aic_miss_has_an_explicit_non_dividing_comparison(self) -> None: + capture = _submit_pmu_capture(phase="none") + for record in capture["records"]: + if record["role"] == "aic": + record["icache_misses"] = 0 + record["shadow_whole_icache_misses"] = 0 + groups = { + "all": capture["records"], + "aic": [record for record in capture["records"] if record["role"] == "aic"], + "aiv": [record for record in capture["records"] if record["role"] == "aiv"], + } + capture["summary"] = { + name: _submit_pmu_summary(records) for name, records in groups.items() + } + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture) + document = render_report(path) + + self.assertIn("AIC miss/core 为 0，AIV 相对变化不可计算", document) + + def test_dynamic_strings_and_raw_link_are_escaped(self) -> None: + capture = _submit_pmu_capture(phase="none") + capture["capture"]["capture_id"] = '' + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture, "submit__raw.json") + document = render_report(path) + + self.assertNotIn('', document) + self.assertIn("<capture & "probe">", document) + self.assertIn("submit_%3Cicache%3E_raw.json", document) + self.assertIn("submit_<icache>_raw.json", document) + + def test_write_report_is_complete_readable_and_leaves_no_temp(self) -> None: + capture = _submit_pmu_capture(phase="none") + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture) + output = write_report(path) + mode = stat.S_IMODE(output.stat().st_mode) + temporary_files = list(Path(directory).glob(".*.tmp")) + + self.assertEqual(output.name, "submit_icache_report.html") + self.assertEqual(mode, 0o644) + self.assertTrue(output.read_text(encoding="utf-8").endswith("\n")) + self.assertEqual(temporary_files, []) + + def test_invalid_raw_does_not_publish_html(self) -> None: + capture = _submit_pmu_capture(phase="none") + capture["records"][0]["icache_misses"] = capture["records"][0]["icache_requests"] + 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write_capture(directory, capture) + output = default_output_path(path) + with self.assertRaisesRegex(ValueError, "miss > request|raw summary mismatch"): + write_report(path) + self.assertFalse(output.exists()) + self.assertEqual(list(Path(directory).glob(".*.tmp")), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/atomic_probe/pa_scheduler/test_pmu_sidecar_analyzer.py b/tests/atomic_probe/pa_scheduler/test_pmu_sidecar_analyzer.py new file mode 100644 index 0000000000..49b1e1a88c --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_pmu_sidecar_analyzer.py @@ -0,0 +1,1570 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +"""standalone PMU sidecar 多轮分析器的 raw 门禁与聚合回归。""" + +from __future__ import annotations + +import json +import math +import tempfile +import unittest +from pathlib import Path +from typing import Any, Sequence + +try: + from .pmu_sidecar_analyzer import ( + A5_AIC_PER_DIE, + A5_AIC_WORKERS, + A5_AIV_WORKERS, + A5_OWNER_MAGIC, + A5_OWNER_VERSION, + A5_PHYSICAL_SUBCORES, + A5_SUBCORES_PER_DIE, + A5_WORKERS, + METRIC_NAMES, + PHASE_STATUS_REQUIRED_MASK_V4, + PHASE_STATUS_REQUIRED_MASK_V5, + PROGRAMMABLE_COUNTER_RISK_THRESHOLD, + SUBMIT_PMU_METRIC_NAMES, + SUBMIT_PMU_V5_METRIC_NAMES, + TASKS_PER_BATCH, + analyze, + load_capture, + ) +except ImportError: + from pmu_sidecar_analyzer import ( + A5_AIC_PER_DIE, + A5_AIC_WORKERS, + A5_AIV_WORKERS, + A5_OWNER_MAGIC, + A5_OWNER_VERSION, + A5_PHYSICAL_SUBCORES, + A5_SUBCORES_PER_DIE, + A5_WORKERS, + METRIC_NAMES, + PHASE_STATUS_REQUIRED_MASK_V4, + PHASE_STATUS_REQUIRED_MASK_V5, + PROGRAMMABLE_COUNTER_RISK_THRESHOLD, + SUBMIT_PMU_METRIC_NAMES, + SUBMIT_PMU_V5_METRIC_NAMES, + TASKS_PER_BATCH, + analyze, + load_capture, + ) + + +def _p95(values: Sequence[int]) -> int: + ordered = sorted(values) + return ordered[math.ceil(0.95 * len(ordered)) - 1] + + +def _summary_for_metrics( + records: list[dict[str, Any]], metric_names: Sequence[str] +) -> dict[str, Any]: + result: dict[str, Any] = { + "cores": len(records), + "active_cores": len(records), + "trusted_cores": len(records), + } + for metric in metric_names: + values = [record[metric] for record in records] + result[metric] = { + "sum": sum(values), + "mean": sum(values) / len(values), + "median": sorted(values)[len(values) // 2] + if len(values) % 2 + else (sorted(values)[len(values) // 2 - 1] + sorted(values)[len(values) // 2]) / 2, + "p95": _p95(values), + "max": max(values), + } + result["icache_miss_rate"] = ( + result["icache_misses"]["sum"] / result["icache_requests"]["sum"] + ) + return result + + +def _summary(records: list[dict[str, Any]]) -> dict[str, Any]: + return _summary_for_metrics(records, METRIC_NAMES) + + +def _submit_pmu_summary( + records: list[dict[str, Any]], schema_version: int = 5 +) -> dict[str, Any]: + result = _summary_for_metrics( + records, + SUBMIT_PMU_V5_METRIC_NAMES + if schema_version in (5, 6) + else SUBMIT_PMU_METRIC_NAMES, + ) + phase_requests = result["phase_icache_requests"]["sum"] + phase_misses = result["phase_icache_misses"]["sum"] + result["phase_observed_read_clear_ratio"] = ( + None if phase_requests == 0 else phase_misses / phase_requests + ) + return result + + +def _submit_pmu_physical_id(worker_id: int) -> int: + """构造与 host mixed launch 相同的 32 组物理 1:2 triplet。""" + + if worker_id < A5_AIC_WORKERS: + block = worker_id + die_base = 0 if block < A5_AIC_WORKERS // 2 else A5_SUBCORES_PER_DIE + return die_base + block % (A5_AIC_WORKERS // 2) + vector_id = worker_id - A5_AIC_WORKERS + block = vector_id // 2 + aic_id = _submit_pmu_physical_id(block) + die_base = (aic_id // A5_SUBCORES_PER_DIE) * A5_SUBCORES_PER_DIE + local_aic = aic_id % A5_SUBCORES_PER_DIE + return die_base + A5_AIC_PER_DIE + local_aic * 2 + vector_id % 2 + + +def _submit_pmu_bitmap_words() -> list[int]: + words = [0, 0, 0, 0] + for worker_id in range(A5_WORKERS): + physical_id = _submit_pmu_physical_id(worker_id) + words[physical_id // 32] |= 1 << (physical_id % 32) + return words + + +def _capture(offset: int = 0, window: str = "submit-all") -> dict[str, Any]: + records: list[dict[str, Any]] = [] + for worker_id, role in enumerate(("aic", "aiv", "aiv")): + base = 100 + worker_id * 10 + offset + record: dict[str, Any] = { + "worker_id": worker_id, + "physical_core_id": 10 + worker_id, + "role": role, + "trusted": True, + "selectors_match": True, + "owner_bitmap_member": True, + "worker_slot_exact": True, + "physical_role_matches": True, + "window_started": True, + "window_stopped": True, + } + for metric_index, metric in enumerate(METRIC_NAMES): + record[metric] = base + metric_index + record["total_cycles"] = base + len(METRIC_NAMES) + # miss/request 取独立、直观的数值，便于断言聚合公式。 + record["icache_requests"] = 1000 + base + record["icache_misses"] = 100 + base // 10 + records.append(record) + + groups = { + "all": records, + "aic": [record for record in records if record["role"] == "aic"], + "aiv": [record for record in records if record["role"] == "aiv"], + } + return { + "schema": {"name": "pa_scheduler_pmu_phase_windows", "version": 3}, + "capture": { + "capture_id": f"capture-{offset}", + "accepted": True, + "published_after_runtime_cleanup": True, + "runtime_cleanup_passed": True, + "owner_restore_passed": True, + }, + "configuration": { + "device": 0, + "batches": 256, + "workers": 3, + "aic_workers": 1, + "aiv_workers": 2, + "pmu_window": window, + "submit_span_us": 5000.0 + offset, + "selectors": {"cnt6_icache_request": 0x034, "cnt7_icache_miss": 0x035}, + "counter_width_bits": {"total": 64, "programmable": 32}, + "phase_timestamp_calls_present": True, + "phase_record_writes": False, + "profile_accumulation": False, + "trace_enabled": False, + "atomic_trace": False, + "gate_start_stop_have_pipe_all_barriers": True, + "winner_workload": { + "mode": "real-compute", + "counts": {"qk": 6, "sf": 28, "pv": 4, "up": 1}, + "unit": "complete_128x128_engine_pipeline_iteration", + }, + }, + "validation": { + "semantic_passed": True, + "pmu_passed": True, + "icache_measurement_valid": True, + "icache_miss_le_request": True, + "counter_below_risk_threshold": True, + "trusted_records": 3, + "unique_physical_core_ids": 3, + "owner_bitmap_member_records": 3, + "exact_worker_slot_records": 3, + "physical_role_match_records": 3, + "window_started_records": 3, + "window_stopped_records": 3, + }, + "records": records, + "summary": {name: _summary(group) for name, group in groups.items()}, + } + + +def _submit_pmu_capture( + offset: int = 0, + phase: str = "claim", + schema_version: int = 5, + shared_context_lens: Sequence[int] | None = None, +) -> dict[str, Any]: + phase_ids = { + "none": 0, + "claim": 1, + "efdrain": 2, + "materialize": 4, + "register": 5, + } + phase_id = phase_ids[phase] + if schema_version == 6: + context_lens = list( + [8192, 8192] if shared_context_lens is None else shared_context_lens + ) + batches = len(context_lens) + total_groups = sum( + (((context_len + 127) // 128) + 63) // 64 + for context_len in context_lens + ) + tasks_per_worker = batches + 4 * total_groups + else: + context_lens = None + batches = 2 + total_groups = None + tasks_per_worker = batches * TASKS_PER_BATCH + records: list[dict[str, Any]] = [] + for worker_id in range(A5_WORKERS): + role = "aic" if worker_id < A5_AIC_WORKERS else "aiv" + vector_id = 0 if role == "aic" else worker_id - A5_AIC_WORKERS + block_id = worker_id if role == "aic" else vector_id // 2 + lane = 0 if role == "aic" else 1 + vector_id % 2 + base = 100 + worker_id * 10 + offset + if phase == "none": + calls_per_worker = 0 + elif schema_version == 6 and phase in ("materialize", "register"): + # shared 每个逻辑 task 恰有一个 winner；测试数据用确定性分配 + # 构造含零 winner 核的逐核稀疏调用，总和严格等于 task 数。 + calls_per_worker = ( + tasks_per_worker // A5_WORKERS + + (1 if worker_id < tasks_per_worker % A5_WORKERS else 0) + ) + else: + calls_per_worker = tasks_per_worker + primary_requests = 1000 + base + primary_misses = 100 + base // 10 + phase_requests = 0 if calls_per_worker == 0 else 100 + worker_id * 10 + offset + phase_misses = 0 if calls_per_worker == 0 else 10 + worker_id + offset // 10 + request_loss = 0 if calls_per_worker == 0 else 1 + worker_id % 3 + miss_loss = 0 if calls_per_worker == 0 else 1 + worker_id % 2 + shadow_requests = primary_requests - request_loss + shadow_misses = primary_misses - miss_loss + submit_elapsed_ticks = 4_000_000 + worker_id * 1000 + offset + phase_elapsed_ticks = ( + 0 if calls_per_worker == 0 else 200_000 + worker_id * 3000 + offset + ) + phase_status_mask = ( + PHASE_STATUS_REQUIRED_MASK_V5 + if schema_version in (5, 6) + else PHASE_STATUS_REQUIRED_MASK_V4 + ) + record: dict[str, Any] = { + "worker_id": worker_id, + "physical_core_id": _submit_pmu_physical_id(worker_id), + "role": role, + "block_id": block_id, + "lane": lane, + "trusted": True, + "physical_core_id_valid": True, + "selectors_match": True, + "owner_bitmap_member": True, + "worker_slot_exact": True, + "physical_role_matches": True, + "window_started": True, + "window_stopped": True, + "build_variant_id": 2, + "compiled_phase_id": phase_id, + "phase_status": phase_status_mask | (0x30 if calls_per_worker == 0 else 0), + "phase_calls": calls_per_worker, + "phase_expected_calls": calls_per_worker, + "phase_begin_reads": calls_per_worker, + "phase_end_reads": calls_per_worker, + "primary_window_segments": 1, + "shadow_read_segments": 2 * calls_per_worker + 1, + "phase_icache_requests": phase_requests, + "phase_icache_misses": phase_misses, + "phase_icache_requests_upper_bound": phase_requests + request_loss, + "phase_icache_misses_upper_bound": phase_misses + miss_loss, + "shadow_whole_icache_requests": shadow_requests, + "shadow_whole_icache_misses": shadow_misses, + "shadow_matches_primary": request_loss == 0 and miss_loss == 0, + "shadow_not_greater_than_primary": True, + "shadow_request_loss": request_loss, + "shadow_miss_loss": miss_loss, + "phase_boundaries_balanced": True, + } + if schema_version in (5, 6): + record.update( + { + "submit_elapsed_ticks": submit_elapsed_ticks, + "phase_elapsed_ticks": phase_elapsed_ticks, + "phase_time_valid": True, + } + ) + for metric_index, metric in enumerate(SUBMIT_PMU_METRIC_NAMES): + record.setdefault(metric, base + metric_index) + # total 是同一窗口的包络，fixture 也必须满足 scalar_busy <= total_cycles。 + record["total_cycles"] = base + len(SUBMIT_PMU_METRIC_NAMES) + record["icache_requests"] = primary_requests + record["icache_misses"] = primary_misses + # setdefault 不覆盖上面按 phase 契约填写的五个扩展字段。 + records.append(record) + + groups = { + "all": records, + "aic": [record for record in records if record["role"] == "aic"], + "aiv": [record for record in records if record["role"] == "aiv"], + } + workers = len(records) + exact_records = sum(record["shadow_matches_primary"] for record in records) + bounded_records = sum(record["shadow_not_greater_than_primary"] for record in records) + acceptable_records = sum( + record["shadow_matches_primary"] + if record["phase_expected_calls"] == 0 + else record["shadow_not_greater_than_primary"] + for record in records + ) + request_losses = [record["shadow_request_loss"] for record in records] + miss_losses = [record["shadow_miss_loss"] for record in records] + capture = { + "schema": {"name": "pa_scheduler_pmu_phase_windows", "version": schema_version}, + "capture": { + "capture_id": f"submit-pmu-{phase}-{offset}", + "accepted": True, + "published_after_runtime_cleanup": True, + "runtime_cleanup_passed": True, + "owner_restore_passed": True, + }, + "configuration": { + "build_variant": "submit-pmu", + "build_variant_id": 2, + "compiled_phase": phase, + "compiled_phase_id": phase_id, + "device": 0, + "batches": batches, + "workers": workers, + "aic_workers": A5_AIC_WORKERS, + "aiv_workers": A5_AIV_WORKERS, + "final_barrier": "two-16", + "pmu_window": "submit-all", + "primary_window_segments_per_record": 1, + "unavailable_metrics": ["mte3_busy"], + "submit_span_us": 5000.0 + offset, + "selectors": { + "cnt0_vector_busy": 0x501, + "cnt1_cube_busy": 0x301, + "cnt2_scalar_busy": 0x001, + "cnt3_mte1_busy": 0x701, + "cnt4_mte2_busy": 0x202, + "cnt5_shadow_icache_miss": 0x035, + "cnt6_primary_icache_request": 0x034, + "cnt7_primary_icache_miss": 0x035, + "cnt8_shadow_icache_request": 0x034, + "cnt9_unused": 0x000, + }, + "counter_width_bits": {"total": 64, "programmable": 32}, + "phase_timestamp_calls_present": + schema_version in (5, 6) and phase != "none", + "phase_record_writes": False, + "profile_accumulation": False, + "trace_enabled": False, + "trace_atomics": False, + "atomic_trace": False, + "profile_phases": False, + "gate_start_stop_have_pipe_all_barriers": True, + "phase_boundary_observation_included": phase != "none", + "phase_counter_pair_snapshot_atomic": False, + "primary_counters_read_at_phase_boundaries": False, + "phase_shadow_partition_exact_required": phase == "none", + "phase_values_are_running_read_clear_lower_bounds": phase != "none", + "cross_phase_elf_sums_valid": False, + "phase_time_observation_included": + schema_version in (5, 6) and phase != "none", + "phase_time_sys_counter_tick_ns": 1, + "phase_time_boundary": "after_begin_read_clear_to_before_end_read_clear", + "phase_time_excludes_shadow_read_overhead": True, + "phase_time_includes_timestamp_overhead": True, + "phase_time_share_definition": + "sum(phase_elapsed_ticks)/sum(submit_elapsed_ticks)", + "phase_time_denominator_scope": + "per_worker_first_submit_begin_to_last_submit_end", + "winner_workload": { + "mode": "real-compute", + "counts": {"qk": 6, "sf": 28, "pv": 4, "up": 1}, + "unit": "complete_128x128_engine_pipeline_iteration", + }, + }, + "validation": { + "semantic_passed": True, + "pmu_passed": True, + "icache_measurement_valid": True, + "icache_miss_le_request": True, + "counter_below_risk_threshold": True, + "phase_measurement_valid": True, + "trusted_records": workers, + "unique_physical_core_ids": workers, + "owner_bitmap_member_records": workers, + "exact_worker_slot_records": workers, + "physical_role_match_records": workers, + "window_started_records": workers, + "window_stopped_records": workers, + "build_variant_match_records": workers, + "phase_id_match_records": workers, + "phase_status_trusted_records": workers, + "shadow_primary_match_records": exact_records, + "shadow_primary_bounded_records": bounded_records, + "phase_shadow_acceptable_records": acceptable_records, + "shadow_request_abs_delta_sum": sum(request_losses), + "shadow_request_abs_delta_max": max(request_losses), + "shadow_request_signed_delta_sum": -sum(request_losses), + "shadow_miss_abs_delta_sum": sum(miss_losses), + "shadow_miss_abs_delta_max": max(miss_losses), + "shadow_miss_signed_delta_sum": -sum(miss_losses), + "phase_boundary_match_records": workers, + "phase_call_shape_match_records": workers, + "phase_time_valid_records": workers, + "phase_time_measurement_valid": True, + "phase_calls": sum(record["phase_calls"] for record in records), + "phase_expected_calls": sum(record["phase_expected_calls"] for record in records), + "expected_records": A5_WORKERS, + "expected_unique_core_ids": A5_WORKERS, + "expected_owner_bitmap_member_records": A5_WORKERS, + "expected_exact_worker_slot_records": A5_WORKERS, + "expected_physical_role_match_records": A5_WORKERS, + "mixed_triplet_matches": A5_AIC_WORKERS, + "expected_mixed_triplet_matches": A5_AIC_WORKERS, + "expected_window_records": A5_WORKERS, + }, + "owner": { + "mode": "main_aicpu_path_a", + "snapshot_phase": "after_configure_before_restore", + "control_magic": A5_OWNER_MAGIC, + "control_version": A5_OWNER_VERSION, + "configure_status": 0, + "configured_flag": 1, + "configured_bitmap_count": A5_WORKERS, + "expected": { + "total": A5_WORKERS, + "aic": A5_AIC_WORKERS, + "aiv": A5_AIV_WORKERS, + }, + "active": { + "total": A5_WORKERS, + "aic": A5_AIC_WORKERS, + "aiv": A5_AIV_WORKERS, + }, + "discovered": { + "total": A5_WORKERS, + "aic": A5_AIC_WORKERS, + "aiv": A5_AIV_WORKERS, + }, + "physical_slots_scanned": A5_PHYSICAL_SUBCORES, + "skipped_physical_slots": A5_PHYSICAL_SUBCORES - A5_WORKERS, + "configured_bitmap_word_order": "least_significant_physical_ids_first", + "configured_bitmap_words": _submit_pmu_bitmap_words(), + "configured_complete_mixed_triplets": A5_AIC_WORKERS, + "expected_complete_mixed_triplets": A5_AIC_WORKERS, + "configured_broken_mixed_triplets": 0, + "restore_passed": True, + }, + "records": records, + "summary": { + name: _submit_pmu_summary(group, schema_version) for name, group in groups.items() + }, + } + if schema_version == 6: + assert context_lens is not None and total_groups is not None + capture["configuration"].update( + { + "tensormap_mode": "shared", + "shared_context_lens": context_lens, + "shared_task_plan": { + "total_groups": total_groups, + "tasks_per_worker": tasks_per_worker, + }, + } + ) + return capture + + +class PmuSidecarAnalyzerTest(unittest.TestCase): + def _write(self, directory: str, name: str, capture: dict[str, Any]) -> Path: + path = Path(directory) / name + path.write_text(json.dumps(capture), encoding="utf-8") + return path + + def test_valid_raw_is_recomputed_and_multiple_runs_are_aggregated(self) -> None: + with tempfile.TemporaryDirectory() as directory: + first = self._write(directory, "first.json", _capture(0)) + second = self._write(directory, "second.json", _capture(20)) + result = analyze([first, second], miss_penalty_ns=90.0) + + self.assertTrue(result["validation"]["raw_to_summary_all_fields_match"]) + self.assertEqual(result["aggregate"]["runs"], 2) + self.assertEqual(result["aggregate"]["submit_span_us"]["median"], 5010.0) + expected_misses = [ + _capture(offset)["summary"]["all"]["icache_misses"]["sum"] + for offset in (0, 20) + ] + self.assertEqual( + result["aggregate"]["groups"]["all"]["icache_misses_sum"]["median"], + sum(expected_misses) / 2, + ) + self.assertTrue(result["estimation"]["not_wall_time"]) + self.assertTrue(result["estimation"]["not_additive_stall_time"]) + self.assertEqual( + result["actual_exposed_loss"]["status"], + "requires_same_semantics_paired_ab", + ) + expected_aiv_misses_per_core = [ + _capture(offset)["summary"]["aiv"]["icache_misses"]["sum"] / 2 + for offset in (0, 20) + ] + self.assertEqual( + result["aggregate"]["groups"]["aiv"]["icache_misses_per_core"]["median"], + sum(expected_aiv_misses_per_core) / 2, + ) + + def test_tampered_host_summary_is_rejected(self) -> None: + capture = _capture() + capture["summary"]["aiv"]["icache_misses"]["sum"] += 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "tampered.json", capture) + with self.assertRaisesRegex(ValueError, "raw summary mismatch"): + load_capture(path) + + def test_different_observation_configurations_cannot_be_merged(self) -> None: + with tempfile.TemporaryDirectory() as directory: + first = self._write(directory, "submit.json", _capture(window="submit-all")) + second = self._write(directory, "empty.json", _capture(window="empty")) + with self.assertRaisesRegex(ValueError, "observation configuration differs"): + analyze([first, second]) + + def test_failed_restore_is_rejected(self) -> None: + capture = _capture() + capture["capture"]["owner_restore_passed"] = False + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "restore_failed.json", capture) + with self.assertRaisesRegex(ValueError, "owner_restore_passed is not true"): + load_capture(path) + + def test_duplicate_physical_core_is_rejected(self) -> None: + capture = _capture() + capture["records"][1]["physical_core_id"] = capture["records"][0]["physical_core_id"] + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "duplicate.json", capture) + with self.assertRaisesRegex(ValueError, "duplicate physical_core_id"): + load_capture(path) + + def test_miss_greater_than_request_is_rejected_before_summary_use(self) -> None: + capture = _capture() + capture["records"][0]["icache_misses"] = capture["records"][0]["icache_requests"] + 1 + # 同步重算 summary，证明失败来自逐核物理约束，而不是 summary 不一致。 + records = capture["records"] + capture["summary"] = { + "all": _summary(records), + "aic": _summary([record for record in records if record["role"] == "aic"]), + "aiv": _summary([record for record in records if record["role"] == "aiv"]), + } + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "invalid_miss.json", capture) + with self.assertRaisesRegex(ValueError, "miss > request"): + load_capture(path) + + def test_submit_pmu_v5_claim_is_recomputed_and_aggregated(self) -> None: + with tempfile.TemporaryDirectory() as directory: + first = self._write(directory, "claim-first.json", _submit_pmu_capture(0, "claim")) + second = self._write(directory, "claim-second.json", _submit_pmu_capture(10, "claim")) + result = analyze([first, second]) + + self.assertEqual(result["input_schema"]["version"], 5) + self.assertEqual(result["schema"]["version"], 3) + self.assertEqual(result["phase_observation"]["compiled_phase"], "claim") + self.assertTrue(result["phase_observation"]["enabled"]) + self.assertFalse(result["phase_observation"]["cross_phase_elf_sums_valid"]) + self.assertEqual( + result["phase_observation"]["phase_value_semantics"], + "running_read_clear_lower_to_loss_adjusted_upper_bound", + ) + expected_aiv_phase_misses = [ + _submit_pmu_capture(offset, "claim")["summary"]["aiv"][ + "phase_icache_misses" + ]["sum"] + / A5_AIV_WORKERS + for offset in (0, 10) + ] + self.assertEqual( + result["aggregate"]["groups"]["aiv"][ + "phase_icache_misses_lower_bound_per_core" + ]["median"], + sum(expected_aiv_phase_misses) / 2, + ) + self.assertGreater( + result["aggregate"]["groups"]["aiv"][ + "phase_icache_miss_lower_bound_share_of_submit" + ]["median"], + 0, + ) + claim_aiv = result["aggregate"]["groups"]["aiv"] + self.assertGreater( + claim_aiv["phase_icache_requests_upper_bound_per_core"]["median"], + claim_aiv["phase_icache_requests_lower_bound_per_core"]["median"], + ) + self.assertGreater(claim_aiv["shadow_request_loss_per_core"]["median"], 0) + for row in result["per_run"]: + for group_name in ("all", "aic", "aiv"): + group = row["groups"][group_name] + self.assertEqual( + group["phase_icache_requests_upper_bound_sum"] + - group["phase_icache_requests_lower_bound_sum"], + group["shadow_request_loss_sum"], + ) + self.assertEqual( + group["phase_icache_misses_upper_bound_sum"] + - group["phase_icache_misses_lower_bound_sum"], + group["shadow_miss_loss_sum"], + ) + self.assertLessEqual( + group["phase_icache_requests_lower_bound_per_core_median"], + group["phase_icache_requests_upper_bound_per_core_median"], + ) + self.assertLessEqual( + group["phase_icache_misses_lower_bound_per_core_p95"], + group["phase_icache_misses_upper_bound_per_core_p95"], + ) + self.assertEqual( + group["phase_icache_request_lower_bound_share_of_submit"], + group["phase_icache_requests_lower_bound_sum"] + / group["icache_requests_sum"], + ) + self.assertEqual( + group["phase_icache_request_upper_bound_share_of_submit"], + group["phase_icache_requests_upper_bound_sum"] + / group["icache_requests_sum"], + ) + self.assertEqual( + group["phase_icache_miss_lower_bound_share_of_submit"], + group["phase_icache_misses_lower_bound_sum"] / group["icache_misses_sum"], + ) + self.assertEqual( + group["phase_icache_miss_upper_bound_share_of_submit"], + group["phase_icache_misses_upper_bound_sum"] / group["icache_misses_sum"], + ) + self.assertEqual( + group["phase_time_share_of_submit"], + group["phase_elapsed_ticks_sum"] / group["submit_elapsed_ticks_sum"], + ) + self.assertEqual( + group["phase_elapsed_per_core_us"], + group["phase_elapsed_ticks_sum"] / group["cores"] / 1000.0, + ) + first_capture = _submit_pmu_capture(0, "claim") + per_core_shares = [ + record["phase_elapsed_ticks"] / record["submit_elapsed_ticks"] + for record in first_capture["records"] + ] + weighted_share = ( + sum(record["phase_elapsed_ticks"] for record in first_capture["records"]) + / sum(record["submit_elapsed_ticks"] for record in first_capture["records"]) + ) + self.assertNotAlmostEqual( + weighted_share, sum(per_core_shares) / len(per_core_shares), places=12 + ) + self.assertLess( + _submit_pmu_capture()["validation"]["shadow_request_signed_delta_sum"], 0 + ) + + def test_submit_pmu_v6_rebuilds_shared_dynamic_task_matrix(self) -> None: + cases = ( + ("g0", [0], 0, 1), + ("g1", [8192], 1, 5), + ("g2-partial", [8193], 2, 9), + ("g2-full", [16384], 2, 9), + ("g4", [32768], 4, 17), + ("mixed", [0, 8192, 8193, 32768], 7, 32), + ) + for name, context_lens, total_groups, tasks_per_worker in cases: + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + capture = _submit_pmu_capture( + phase="claim", + schema_version=6, + shared_context_lens=context_lens, + ) + path = self._write(directory, f"{name}.json", capture) + result = analyze([path]) + + self.assertEqual(result["input_schema"]["version"], 6) + self.assertEqual(result["schema"]["version"], 4) + self.assertTrue(result["phase_observation"]["phase_time_available"]) + self.assertEqual(result["configuration"]["tensormap_mode"], "shared") + self.assertEqual( + result["configuration"]["shared_context_lens"], context_lens + ) + self.assertEqual( + result["configuration"]["shared_task_plan"], + { + "total_groups": total_groups, + "tasks_per_worker": tasks_per_worker, + }, + ) + self.assertEqual( + result["aggregate"]["groups"]["all"]["phase_calls_per_core"]["median"], + tasks_per_worker, + ) + self.assertTrue( + all( + record["phase_expected_calls"] == tasks_per_worker + for record in capture["records"] + ) + ) + self.assertIn( + "phase_elapsed_per_call_ns", + result["per_run"][0]["groups"]["all"], + ) + + def test_submit_pmu_v6_winner_only_phases_use_sparse_per_core_calls(self) -> None: + for phase in ("materialize", "register"): + with self.subTest(phase=phase), tempfile.TemporaryDirectory() as directory: + capture = _submit_pmu_capture( + phase=phase, + schema_version=6, + shared_context_lens=[8193], + ) + path = self._write(directory, f"{phase}.json", capture) + result = analyze([path]) + + calls = [ + record["phase_expected_calls"] + for record in capture["records"] + ] + self.assertEqual(sum(calls), 9) + self.assertEqual(calls.count(0), A5_WORKERS - 9) + self.assertEqual( + result["per_run"][0]["groups"]["all"]["phase_calls_sum"], + 9, + ) + + def test_submit_pmu_v6_rejects_winner_only_call_sum_without_unique_winner(self) -> None: + capture = _submit_pmu_capture( + phase="materialize", + schema_version=6, + shared_context_lens=[8193], + ) + # 同步篡改一个零 winner 核的局部 raw/summary，使逐核字段自洽， + # 但全局调用数变成 10；analyzer 必须以唯一 winner 契约拒绝。 + record = capture["records"][20] + record["phase_calls"] = 1 + record["phase_expected_calls"] = 1 + record["phase_begin_reads"] = 1 + record["phase_end_reads"] = 1 + record["shadow_read_segments"] = 3 + record["phase_icache_requests"] = 1 + record["phase_icache_misses"] = 0 + record["phase_icache_requests_upper_bound"] = 1 + record["phase_icache_misses_upper_bound"] = 0 + record["phase_elapsed_ticks"] = 1 + capture["validation"]["phase_calls"] += 1 + capture["validation"]["phase_expected_calls"] += 1 + records = capture["records"] + capture["summary"] = { + "all": _submit_pmu_summary(records, 6), + "aic": _submit_pmu_summary( + [item for item in records if item["role"] == "aic"], 6 + ), + "aiv": _submit_pmu_summary( + [item for item in records if item["role"] == "aiv"], 6 + ), + } + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "too-many-winners.json", capture) + with self.assertRaisesRegex( + ValueError, "unique-winner/global-replay contract" + ): + load_capture(path) + + def test_submit_pmu_v6_rejects_tampered_shared_task_metadata(self) -> None: + cases = ( + ( + lambda capture: capture["configuration"].__setitem__( + "tensormap_mode", "private" + ), + "tensormap_mode='shared'", + ), + ( + lambda capture: capture["configuration"]["shared_context_lens"].pop(), + "length does not match batches", + ), + ( + lambda capture: capture["configuration"]["shared_context_lens"].__setitem__( + 0, -1 + ), + r"shared_context_lens\[0\] must be non-negative", + ), + ( + lambda capture: capture["configuration"]["shared_context_lens"].__setitem__( + 0, True + ), + r"shared_context_lens\[0\] must be an integer", + ), + ( + lambda capture: capture["configuration"]["shared_context_lens"].__setitem__( + 0, 32769 + ), + r"shared_context_lens\[0\] exceeds 32768", + ), + ( + lambda capture: capture["configuration"]["shared_task_plan"].__setitem__( + "total_groups", + capture["configuration"]["shared_task_plan"]["total_groups"] + 1, + ), + "total_groups disagrees", + ), + ( + lambda capture: capture["configuration"]["shared_task_plan"].__setitem__( + "tasks_per_worker", + capture["configuration"]["shared_task_plan"]["tasks_per_worker"] + 1, + ), + "tasks_per_worker disagrees", + ), + ) + for mutate, message in cases: + with self.subTest(message=message): + capture = _submit_pmu_capture(schema_version=6) + mutate(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "tampered-shared-plan.json", capture) + with self.assertRaisesRegex(ValueError, message): + load_capture(path) + + def test_submit_pmu_v6_requires_one_to_256_batches(self) -> None: + for context_lens in ([], [0] * 257): + with self.subTest(batches=len(context_lens)): + capture = _submit_pmu_capture( + schema_version=6, shared_context_lens=context_lens + ) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "invalid-batches.json", capture) + with self.assertRaisesRegex(ValueError, r"batches must be in \[1, 256\]"): + load_capture(path) + + def test_submit_pmu_v6_rejects_record_calls_that_disagree_with_rebuilt_plan( + self, + ) -> None: + capture = _submit_pmu_capture( + schema_version=6, shared_context_lens=[0, 8192, 8193, 32768] + ) + capture["records"][0]["phase_expected_calls"] += 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "tampered-record-calls.json", capture) + with self.assertRaisesRegex(ValueError, "phase_expected_calls disagrees"): + load_capture(path) + + def test_submit_pmu_v6_requires_expected_call_evidence(self) -> None: + cases = ( + ( + lambda capture: capture["records"][0].pop("phase_expected_calls"), + r"records\[0\]\.phase_expected_calls is required", + ), + ( + lambda capture: capture["validation"].pop("phase_expected_calls"), + r"validation\.phase_expected_calls is required", + ), + ) + for mutate, message in cases: + with self.subTest(message=message): + capture = _submit_pmu_capture(schema_version=6) + mutate(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "missing-expected-calls.json", capture) + with self.assertRaisesRegex(ValueError, message): + load_capture(path) + + def test_submit_pmu_v6_plan_identity_prevents_incompatible_aggregation(self) -> None: + with tempfile.TemporaryDirectory() as directory: + g0 = self._write( + directory, + "g0.json", + _submit_pmu_capture( + schema_version=6, shared_context_lens=[0, 0] + ), + ) + g1 = self._write( + directory, + "g1.json", + _submit_pmu_capture( + schema_version=6, shared_context_lens=[8192, 8192] + ), + ) + with self.assertRaisesRegex( + ValueError, "observation configuration differs" + ): + analyze([g0, g1]) + + # 即使两组 context lengths 重建出相同 groups/tasks，也不能静默聚合。 + same_count_a = self._write( + directory, + "same-count-a.json", + _submit_pmu_capture( + schema_version=6, shared_context_lens=[1, 1] + ), + ) + same_count_b = self._write( + directory, + "same-count-b.json", + _submit_pmu_capture( + schema_version=6, shared_context_lens=[128, 128] + ), + ) + with self.assertRaisesRegex( + ValueError, "observation configuration differs" + ): + analyze([same_count_a, same_count_b]) + + def test_submit_pmu_final_barrier_is_validated_and_part_of_identity(self) -> None: + for schema_version in (5, 6): + with self.subTest(schema_version=schema_version), tempfile.TemporaryDirectory() as directory: + baseline_capture = _submit_pmu_capture(schema_version=schema_version) + different_capture = _submit_pmu_capture(schema_version=schema_version) + different_capture["configuration"]["final_barrier"] = "flat" + baseline = self._write(directory, "baseline.json", baseline_capture) + different = self._write(directory, "different.json", different_capture) + with self.assertRaisesRegex( + ValueError, "observation configuration differs" + ): + analyze([baseline, different]) + + invalid_capture = _submit_pmu_capture(schema_version=schema_version) + invalid_capture["configuration"]["final_barrier"] = "unknown" + invalid = self._write(directory, "invalid.json", invalid_capture) + with self.assertRaisesRegex(ValueError, "unsupported configuration.final_barrier"): + load_capture(invalid) + + def test_submit_pmu_v5_and_v6_contracts_are_isolated(self) -> None: + with tempfile.TemporaryDirectory() as directory: + private_v5 = self._write( + directory, "private-v5.json", _submit_pmu_capture(schema_version=5) + ) + shared_v6 = self._write( + directory, "shared-v6.json", _submit_pmu_capture(schema_version=6) + ) + private_result = analyze([private_v5]) + shared_result = analyze([shared_v6]) + with self.assertRaisesRegex(ValueError, "input schema differs"): + analyze([private_v5, shared_v6]) + + self.assertEqual( + private_result["aggregate"]["groups"]["all"]["phase_calls_per_core"][ + "median" + ], + 2 * TASKS_PER_BATCH, + ) + self.assertNotIn("tensormap_mode", private_result["configuration"]) + self.assertEqual(shared_result["configuration"]["tensormap_mode"], "shared") + self.assertTrue(shared_result["phase_observation"]["phase_time_available"]) + + def test_submit_pmu_v4_v5_reject_shared_task_identity_fields(self) -> None: + for schema_version in (4, 5): + with self.subTest(schema_version=schema_version): + capture = _submit_pmu_capture(schema_version=schema_version) + capture["configuration"].update( + { + "tensormap_mode": "shared", + "shared_context_lens": [8192, 8192], + "shared_task_plan": { + "total_groups": 2, + "tasks_per_worker": 10, + }, + } + ) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "mislabelled-shared.json", capture) + with self.assertRaisesRegex( + ValueError, + "schema-v4/v5 cannot carry shared task identity", + ): + load_capture(path) + + def test_submit_pmu_v5_none_has_zero_disabled_phase(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "none.json", _submit_pmu_capture(0, "none")) + result = analyze([path]) + + self.assertFalse(result["phase_observation"]["enabled"]) + phase = result["aggregate"]["groups"]["aiv"] + self.assertEqual(phase["phase_calls_sum"]["median"], 0) + self.assertEqual(phase["phase_icache_misses_lower_bound_per_core"]["median"], 0) + self.assertIsNone(phase["phase_observed_read_clear_ratio"]["median"]) + self.assertEqual( + phase["phase_icache_miss_lower_bound_share_of_submit"]["median"], 0 + ) + self.assertEqual( + phase["phase_icache_miss_upper_bound_share_of_submit"]["median"], 0 + ) + self.assertEqual( + phase["phase_icache_requests_lower_bound_per_core"]["median"], + phase["phase_icache_requests_upper_bound_per_core"]["median"], + ) + self.assertEqual(phase["shadow_request_loss_sum"]["median"], 0) + self.assertEqual(phase["phase_elapsed_ticks_sum"]["median"], 0) + self.assertEqual(phase["phase_time_share_of_submit"]["median"], 0) + self.assertFalse(result["phase_observation"]["enabled"]) + self.assertTrue(result["phase_observation"]["phase_time_available"]) + + def test_submit_pmu_v5_rejects_invalid_phase_time_evidence(self) -> None: + cases = ( + ( + lambda capture: capture["records"][0].__setitem__( + "phase_elapsed_ticks", capture["records"][0]["submit_elapsed_ticks"] + 1 + ), + "phase_elapsed_ticks exceeds", + ), + ( + lambda capture: capture["records"][0].pop("phase_elapsed_ticks"), + "phase_elapsed_ticks must be an integer", + ), + ( + lambda capture: capture["records"][0].__setitem__("phase_time_valid", False), + "phase_time_valid is not true", + ), + ) + for mutate, message in cases: + with self.subTest(message=message): + capture = _submit_pmu_capture(0, "claim") + mutate(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "invalid-time.json", capture) + with self.assertRaisesRegex(ValueError, message): + load_capture(path) + + def test_submit_pmu_v5_rejects_nonzero_none_phase_time(self) -> None: + capture = _submit_pmu_capture(0, "none") + capture["records"][0]["phase_elapsed_ticks"] = 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "none-time.json", capture) + with self.assertRaisesRegex(ValueError, "does not match enabled phase calls"): + load_capture(path) + + def test_submit_pmu_v4_without_phase_time_remains_readable(self) -> None: + capture = _submit_pmu_capture(0, "claim", schema_version=4) + for field in ( + "phase_time_observation_included", + "phase_time_sys_counter_tick_ns", + "phase_time_boundary", + "phase_time_excludes_shadow_read_overhead", + "phase_time_includes_timestamp_overhead", + "phase_time_share_definition", + "phase_time_denominator_scope", + ): + capture["configuration"].pop(field) + capture["validation"].pop("phase_time_valid_records") + capture["validation"].pop("phase_time_measurement_valid") + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "legacy-v4.json", capture) + result = analyze([path]) + + self.assertEqual(result["input_schema"]["version"], 4) + self.assertFalse(result["phase_observation"]["phase_time_available"]) + self.assertNotIn("phase_time_share_of_submit", result["per_run"][0]["groups"]["all"]) + + def test_submit_pmu_v5_requires_fixed_a5_topology(self) -> None: + capture = _submit_pmu_capture() + # 仍保持 workers=aic+aiv，证明拒绝原因是 submit-pmu 的固定 A5 拓扑， + # 不是原有的自报计数加和检查。 + capture["configuration"]["aic_workers"] = A5_AIC_WORKERS - 1 + capture["configuration"]["aiv_workers"] = A5_AIV_WORKERS + 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "wrong-topology.json", capture) + with self.assertRaisesRegex(ValueError, "fixed 96/32/64 A5 topology"): + load_capture(path) + + def test_submit_pmu_v5_logical_worker_triplet_is_recomputed(self) -> None: + mutations = ( + (0, "worker_id", A5_WORKERS, "exact worker slot"), + (0, "role", "aiv", "logical worker topology"), + (0, "block_id", 1, "mixed block"), + (A5_AIC_WORKERS, "lane", 2, "mixed lane"), + (0, "physical_core_id_valid", False, "physical_core_id_valid"), + ) + for record_index, field, value, message in mutations: + with self.subTest(field=field): + capture = _submit_pmu_capture() + capture["records"][record_index][field] = value + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, f"wrong-{field}.json", capture) + with self.assertRaisesRegex(ValueError, message): + load_capture(path) + + def test_submit_pmu_v5_physical_range_and_triplets_are_recomputed(self) -> None: + capture = _submit_pmu_capture() + capture["records"][0]["physical_core_id"] = A5_PHYSICAL_SUBCORES + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "out-of-range-core.json", capture) + with self.assertRaisesRegex(ValueError, "outside the 108-slot topology"): + load_capture(path) + + capture = _submit_pmu_capture() + # 交换两个 block 的 AIV0：ID 仍唯一、仍属于 owner、物理角色仍是 AIV， + # 只有逐 block 的 1:2 关系被破坏。 + first = A5_AIC_WORKERS + second = A5_AIC_WORKERS + 2 + capture["records"][first]["physical_core_id"], capture["records"][second][ + "physical_core_id" + ] = ( + capture["records"][second]["physical_core_id"], + capture["records"][first]["physical_core_id"], + ) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "crossed-aiv-triplets.json", capture) + with self.assertRaisesRegex(ValueError, "mixed block 0"): + load_capture(path) + + def test_submit_pmu_v5_owner_control_fields_are_rechecked(self) -> None: + mutations = ( + (("control_magic",), 0, "control_magic mismatch"), + (("control_version",), A5_OWNER_VERSION + 1, "control_version mismatch"), + (("configure_status",), 1, "configure_status is not success"), + (("configured_flag",), 0, "configured_flag is not one"), + (("expected", "total"), A5_WORKERS - 1, "owner.expected.total"), + (("active", "aic"), A5_AIC_WORKERS - 1, "owner.active.aic"), + (("discovered", "aiv"), A5_AIV_WORKERS - 1, "owner.discovered.aiv"), + (("physical_slots_scanned",), A5_PHYSICAL_SUBCORES - 1, "physical_slots_scanned"), + (("skipped_physical_slots",), 11, "skipped_physical_slots"), + (("configured_bitmap_count",), A5_WORKERS - 1, "bitmap count"), + (("configured_complete_mixed_triplets",), A5_AIC_WORKERS - 1, "complete mixed triplets"), + (("configured_broken_mixed_triplets",), 1, "broken mixed triplet"), + (("restore_passed",), False, "owner.restore_passed"), + ) + for keys, value, message in mutations: + with self.subTest(field=".".join(keys)): + capture = _submit_pmu_capture() + target = capture["owner"] + for key in keys[:-1]: + target = target[key] + target[keys[-1]] = value + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, f"bad-owner-{keys[-1]}.json", capture) + with self.assertRaisesRegex(ValueError, message): + load_capture(path) + + def test_submit_pmu_v5_owner_object_and_bitmap_are_rechecked(self) -> None: + capture = _submit_pmu_capture() + del capture["owner"] + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "missing-owner.json", capture) + with self.assertRaisesRegex(ValueError, "submit-pmu owner must be an object"): + load_capture(path) + + capture = _submit_pmu_capture() + capture["owner"]["configured_bitmap_words"][3] |= 1 << 31 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "owner-high-bit.json", capture) + with self.assertRaisesRegex(ValueError, "outside the 108 physical slots"): + load_capture(path) + + capture = _submit_pmu_capture() + words = capture["owner"]["configured_bitmap_words"] + # 用另一个完整 triplet 替换 block15 对应的三个位。bitmap 本身仍是 + # 96/32/64 且 32 组完整 triplet，但不再等于 worker raw 的物理 ID 集合。 + for physical_id in (15, 48, 49): + words[physical_id // 32] &= ~(1 << (physical_id % 32)) + for physical_id in (16, 50, 51): + words[physical_id // 32] |= 1 << (physical_id % 32) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "owner-record-set-mismatch.json", capture) + with self.assertRaisesRegex(ValueError, "absent from the owner bitmap"): + load_capture(path) + + def test_submit_pmu_v5_host_triplet_count_is_not_blindly_trusted(self) -> None: + capture = _submit_pmu_capture() + capture["validation"]["mixed_triplet_matches"] -= 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "host-triplet-count.json", capture) + with self.assertRaisesRegex(ValueError, "mixed_triplet_matches"): + load_capture(path) + + def test_submit_pmu_fixed_phase_calls_are_exact_per_worker(self) -> None: + for phase in ("claim", "efdrain", "materialize", "register"): + with self.subTest(phase=phase): + capture = _submit_pmu_capture(phase=phase) + first = capture["records"][0] + second = capture["records"][1] + first["phase_calls"] -= 1 + first["phase_begin_reads"] -= 1 + first["phase_end_reads"] -= 1 + first["shadow_read_segments"] -= 2 + second["phase_calls"] += 1 + second["phase_begin_reads"] += 1 + second["phase_end_reads"] += 1 + second["shadow_read_segments"] += 2 + records = capture["records"] + capture["summary"] = { + "all": _submit_pmu_summary(records), + "aic": _submit_pmu_summary(records[:A5_AIC_WORKERS]), + "aiv": _submit_pmu_summary(records[A5_AIC_WORKERS:]), + } + # 全局 calls、每条 begin/end 与 shadow segment 都保持闭合， + # 只有逐核固定流 phase 的调用契约被破坏。 + with tempfile.TemporaryDirectory() as directory: + path = self._write( + directory, f"redistributed-{phase}-calls.json", capture + ) + with self.assertRaisesRegex(ValueError, "phase_calls does not match"): + load_capture(path) + + def test_submit_pmu_efdrain_is_an_independent_phase(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "efdrain.json", _submit_pmu_capture(phase="efdrain")) + result = analyze([path]) + + self.assertEqual(result["phase_observation"]["compiled_phase"], "efdrain") + self.assertEqual( + result["aggregate"]["groups"]["all"]["phase_calls_per_core"]["median"], + 2 * TASKS_PER_BATCH, + ) + + def test_submit_pmu_none_rejects_nonzero_phase_counters(self) -> None: + capture = _submit_pmu_capture(phase="none") + capture["records"][0]["phase_icache_requests"] = 1 + capture["records"][0]["phase_icache_requests_upper_bound"] = 1 + records = capture["records"] + capture["summary"] = { + "all": _submit_pmu_summary(records), + "aic": _submit_pmu_summary(records[:A5_AIC_WORKERS]), + "aiv": _submit_pmu_summary(records[A5_AIC_WORKERS:]), + } + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "none-nonzero-phase.json", capture) + with self.assertRaisesRegex(ValueError, "zero calls must have zero"): + load_capture(path) + + def test_submit_pmu_shadow_greater_than_primary_is_rejected_from_raw(self) -> None: + capture = _submit_pmu_capture() + record = capture["records"][0] + record["shadow_whole_icache_requests"] = record["icache_requests"] + 1 + record["shadow_not_greater_than_primary"] = False + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "shadow-greater-than-primary.json", capture) + with self.assertRaisesRegex(ValueError, "shadow whole exceeds"): + load_capture(path) + + def test_submit_pmu_shadow_miss_greater_than_request_is_rejected(self) -> None: + capture = _submit_pmu_capture() + record = capture["records"][0] + record["shadow_whole_icache_requests"] = record["shadow_whole_icache_misses"] - 1 + records = capture["records"] + capture["summary"] = { + "all": _submit_pmu_summary(records), + "aic": _submit_pmu_summary(records[:A5_AIC_WORKERS]), + "aiv": _submit_pmu_summary(records[A5_AIC_WORKERS:]), + } + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "shadow-miss-greater-than-request.json", capture) + with self.assertRaisesRegex(ValueError, "shadow I-cache miss > request"): + load_capture(path) + + def test_submit_pmu_shadow_booleans_loss_and_upper_are_recomputed(self) -> None: + mutations = ( + ("shadow_matches_primary", True, "shadow_matches_primary disagrees"), + ("shadow_not_greater_than_primary", False, "shadow_not_greater_than_primary disagrees"), + ("shadow_request_loss", 999, "shadow_request_loss disagrees"), + ( + "phase_icache_requests_upper_bound", + 999, + "upper_bound is not lower plus shadow loss", + ), + ("shadow_miss_loss", 999, "shadow_miss_loss disagrees"), + ( + "phase_icache_misses_upper_bound", + 999, + "upper_bound is not lower plus shadow loss", + ), + ) + for field, value, message in mutations: + with self.subTest(field=field): + capture = _submit_pmu_capture() + capture["records"][0][field] = value + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, f"fake-{field}.json", capture) + with self.assertRaisesRegex(ValueError, message): + load_capture(path) + + def test_submit_pmu_host_shadow_counts_and_deltas_are_recomputed(self) -> None: + fields = ( + "shadow_primary_match_records", + "shadow_primary_bounded_records", + "shadow_request_abs_delta_sum", + "shadow_request_abs_delta_max", + "shadow_request_signed_delta_sum", + "shadow_miss_abs_delta_sum", + "shadow_miss_abs_delta_max", + "shadow_miss_signed_delta_sum", + ) + for field in fields: + with self.subTest(field=field): + capture = _submit_pmu_capture() + capture["validation"][field] += 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, f"tampered-{field}.json", capture) + with self.assertRaisesRegex(ValueError, field): + load_capture(path) + + def test_submit_pmu_unbalanced_phase_boundary_is_rejected_from_raw(self) -> None: + capture = _submit_pmu_capture() + capture["records"][0]["phase_end_reads"] -= 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "boundary-mismatch.json", capture) + with self.assertRaisesRegex(ValueError, "boundaries do not match calls"): + load_capture(path) + + capture = _submit_pmu_capture() + capture["records"][0]["shadow_read_segments"] -= 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "segment-mismatch.json", capture) + with self.assertRaisesRegex(ValueError, "shadow_read_segments does not match"): + load_capture(path) + + def test_submit_pmu_record_build_and_phase_ids_are_rechecked(self) -> None: + for field, value, message in ( + ("build_variant_id", 1, "build_variant_id mismatch"), + ("compiled_phase_id", 0, "compiled_phase_id mismatch"), + ): + with self.subTest(field=field): + capture = _submit_pmu_capture() + capture["records"][0][field] = value + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, f"bad-{field}.json", capture) + with self.assertRaisesRegex(ValueError, message): + load_capture(path) + + def test_submit_pmu_configuration_and_host_gate_are_rechecked(self) -> None: + cases = ( + ("build_variant", "swimlane", "configuration.build_variant"), + ("compiled_phase_id", 0, "compiled phase name/id mismatch"), + ( + "phase_boundary_observation_included", + False, + "phase_boundary_observation_included does not match", + ), + ( + "phase_counter_pair_snapshot_atomic", + True, + "phase_counter_pair_snapshot_atomic is not false", + ), + ( + "phase_shadow_partition_exact_required", + True, + "phase_shadow_partition_exact_required does not match", + ), + ( + "phase_values_are_running_read_clear_lower_bounds", + False, + "phase_values_are_running_read_clear_lower_bounds does not match", + ), + ) + for field, value, message in cases: + with self.subTest(field=field): + capture = _submit_pmu_capture() + capture["configuration"][field] = value + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, f"bad-config-{field}.json", capture) + with self.assertRaisesRegex(ValueError, message): + load_capture(path) + + capture = _submit_pmu_capture() + capture["configuration"]["selectors"]["cnt5_shadow_icache_miss"] = 0x203 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "bad-shadow-selector.json", capture) + with self.assertRaisesRegex(ValueError, "cnt5_shadow_icache_miss"): + load_capture(path) + + capture = _submit_pmu_capture() + capture["configuration"]["selectors"]["cnt2_scalar_busy"] = 0xDEAD + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "bad-scalar-selector.json", capture) + with self.assertRaisesRegex(ValueError, "cnt2_scalar_busy"): + load_capture(path) + + capture = _submit_pmu_capture() + capture["configuration"]["counter_width_bits"]["programmable"] = 64 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "bad-programmable-counter-width.json", capture) + with self.assertRaisesRegex(ValueError, "programmable must be 32"): + load_capture(path) + + capture = _submit_pmu_capture() + capture["validation"]["phase_boundary_match_records"] -= 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "bad-host-phase-gate.json", capture) + with self.assertRaisesRegex(ValueError, "phase_boundary_match_records is incomplete"): + load_capture(path) + + capture = _submit_pmu_capture() + capture["validation"]["phase_call_shape_match_records"] -= 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "bad-host-phase-call-shape.json", capture) + with self.assertRaisesRegex( + ValueError, "phase_call_shape_match_records is incomplete" + ): + load_capture(path) + + def test_submit_pmu_rejects_retired_phase_id_three(self) -> None: + capture = _submit_pmu_capture() + capture["configuration"]["compiled_phase"] = "wait-for-slot" + capture["configuration"]["compiled_phase_id"] = 3 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "retired-phase-id-three.json", capture) + with self.assertRaisesRegex(ValueError, "unsupported configuration.compiled_phase"): + load_capture(path) + + def test_submit_pmu_rejects_scalar_busy_above_total(self) -> None: + capture = _submit_pmu_capture() + capture["records"][0]["scalar_busy"] = capture["records"][0]["total_cycles"] + 1 + records = capture["records"] + capture["summary"] = { + "all": _submit_pmu_summary(records), + "aic": _submit_pmu_summary(records[:A5_AIC_WORKERS]), + "aiv": _submit_pmu_summary(records[A5_AIC_WORKERS:]), + } + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "scalar-busy-above-total.json", capture) + with self.assertRaisesRegex(ValueError, "scalar_busy > total_cycles"): + load_capture(path) + + def test_submit_pmu_rejects_zero_total_cycles(self) -> None: + capture = _submit_pmu_capture() + capture["records"][0]["total_cycles"] = 0 + capture["records"][0]["scalar_busy"] = 0 + records = capture["records"] + capture["summary"] = { + "all": _submit_pmu_summary(records), + "aic": _submit_pmu_summary(records[:A5_AIC_WORKERS]), + "aiv": _submit_pmu_summary(records[A5_AIC_WORKERS:]), + } + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "zero-total-cycles.json", capture) + with self.assertRaisesRegex(ValueError, "zero total_cycles"): + load_capture(path) + + def test_submit_pmu_recomputes_programmable_counter_risk_threshold(self) -> None: + capture = _submit_pmu_capture() + capture["records"][0]["vector_busy"] = PROGRAMMABLE_COUNTER_RISK_THRESHOLD + records = capture["records"] + capture["summary"] = { + "all": _submit_pmu_summary(records), + "aic": _submit_pmu_summary(records[:A5_AIC_WORKERS]), + "aiv": _submit_pmu_summary(records[A5_AIC_WORKERS:]), + } + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "counter-risk-threshold.json", capture) + with self.assertRaisesRegex(ValueError, "32-bit counter risk threshold"): + load_capture(path) + + def test_integer_summary_fields_require_exact_equality(self) -> None: + capture = _submit_pmu_capture() + for worker_id, record in enumerate(capture["records"]): + record["total_cycles"] = 10**12 + worker_id + records = capture["records"] + capture["summary"] = { + "all": _submit_pmu_summary(records), + "aic": _submit_pmu_summary(records[:A5_AIC_WORKERS]), + "aiv": _submit_pmu_summary(records[A5_AIC_WORKERS:]), + } + capture["summary"]["all"]["total_cycles"]["sum"] += 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "inexact-large-integer-summary.json", capture) + with self.assertRaisesRegex(ValueError, "all.total_cycles.sum"): + load_capture(path) + + def test_submit_pmu_phase_counter_order_is_rechecked(self) -> None: + # 两个 ld_dev 不是原子配对快照；边界漂移可使局部 miss 略大于 request， + # 只要两者分别不超过各自的 Submit whole 就仍是合法 raw 观察。 + capture = _submit_pmu_capture() + capture["records"][0]["phase_icache_misses"] = ( + capture["records"][0]["phase_icache_requests"] + 5 + ) + capture["records"][0]["phase_icache_misses_upper_bound"] = ( + capture["records"][0]["phase_icache_misses"] + + capture["records"][0]["shadow_miss_loss"] + ) + records = capture["records"] + capture["summary"] = { + "all": _submit_pmu_summary(records), + "aic": _submit_pmu_summary( + [record for record in records if record["role"] == "aic"] + ), + "aiv": _submit_pmu_summary( + [record for record in records if record["role"] == "aiv"] + ), + } + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "phase-miss-above-request.json", capture) + load_capture(path) + + mutations = ( + ("phase_icache_misses", 200, "phase counters exceed"), + ("phase_icache_requests", 2000, "phase counters exceed"), + ) + for field, value, message in mutations: + with self.subTest(field=field): + capture = _submit_pmu_capture() + capture["records"][0][field] = value + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, f"bad-{field}.json", capture) + with self.assertRaisesRegex(ValueError, message): + load_capture(path) + + def test_submit_pmu_phase_summary_tampering_is_rejected(self) -> None: + capture = _submit_pmu_capture() + capture["summary"]["aiv"]["phase_icache_misses"]["sum"] += 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "phase-summary-tampered.json", capture) + with self.assertRaisesRegex(ValueError, "raw summary mismatch"): + load_capture(path) + + capture = _submit_pmu_capture() + capture["summary"]["aiv"]["phase_observed_read_clear_ratio"] += 0.01 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "phase-ratio-tampered.json", capture) + with self.assertRaisesRegex(ValueError, "phase_observed_read_clear_ratio"): + load_capture(path) + + capture = _submit_pmu_capture() + capture["summary"]["aiv"]["phase_elapsed_ticks"]["sum"] += 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, "phase-time-summary-tampered.json", capture) + with self.assertRaisesRegex(ValueError, "phase_elapsed_ticks.sum"): + load_capture(path) + + def test_different_schema_or_submit_pmu_phase_cannot_be_merged(self) -> None: + with tempfile.TemporaryDirectory() as directory: + legacy = self._write(directory, "legacy.json", _capture()) + claim = self._write(directory, "claim.json", _submit_pmu_capture(0, "claim")) + none = self._write(directory, "none.json", _submit_pmu_capture(0, "none")) + with self.assertRaisesRegex(ValueError, "input schema differs"): + analyze([legacy, claim]) + with self.assertRaisesRegex(ValueError, "observation configuration differs"): + analyze([claim, none]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_alloc_complete_control_20260723_044508_icache_report.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_alloc_complete_control_20260723_044508_icache_report.html new file mode 100644 index 0000000000..6a738e322a --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_alloc_complete_control_20260723_044508_icache_report.html @@ -0,0 +1,362 @@ + + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 fdwic_submit_pmu_raw.json · schema fdwic-submit-pmu-v3 · 32 AIC + 64 AIV

+ +
+

alloc-complete-control 阶段观察（phase_id=11）

+

边界 alloc_complete_begin_to_end_excluding_linked_kernel_calls · 计数语义 discontinuous_running_read_clear_excluding_linked_kernel_calls · + 时间语义 boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls · + call_shape=dynamic_global · expected_calls=ALL 256（角色不锁定）

+ +

本报告使用 empty-bracket 估算进入 phase observed + 的局部记录代码开销。 + AIC/AIV 分别按“空区间每组记录开销 × 本阶段实际 begin/end 记录组数”计算，ALL + 再由两类核相加。参考值只从 phase observed 分子扣除该估算；原始 whole 分母保持不变， + 因为空区间没有测量 whole 窗口中的全部记录开销。所以下表的参考比例不是完整业务 + 阶段占完整调度窗口的比例，也不是精确校正。若参考值为负，表示记录代码开销估算已经 + 超过原始 observed；这只能说明该信号低于当前校准的分辨能力，不能截成零或解释成负的 + 业务事件。校准 raw：/home/q00473782/atomic/private/gpt/simpler/outputs/TestPagedAttentionUnroll_Case1_20260723_043533/fdwic_submit_pmu_raw.json · + SHA-256 56199f91545cc6a1fbf18b8bb2cfebae279cb5801516311a040d4facccf321ba。两份 provenance 已核验为同一场景、同一 Git revision 8c6b6e3a412d。

+ + +

本阶段若跨过 linked Kernel，会同时门控 SYS 时间与 PMU counter， + 因而 total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段。 + result-used return-ready atomic 依赖区间只从 SYS 边界累计值扣除，PMU counter + 仍含 atomic 指令事件；source-issue atomic + 保留在时间和计数口径内。没有 linked Kernel 的阶段不触发 PMU 停/开表； + 每次调用的分母仍是外层业务调用次数，不是 Begin/End 读数，也不是排除的 Kernel 调用数。

+ +

阶段原始 PMU 观测来自 running read-clear 的 total cycles，并按 AIC/AIV + 各自校准频率换算；提供 empty-bracket 时，页面主参考值再扣除局部记录代码开销估算。 + phase_elapsed_ticks 只是 SYS 边界闭合诊断，绝不按 1 GHz 当作阶段时间。 + Scalar busy 与 non-scalar residual（逐核先算 total−scalar）共同解释阶段 PMU total； + 未校准时只能展示 raw observed；提供 empty-bracket 后，记录代码开销估算只从 phase + observed 分子扣除，原始 whole 分母保持不变。当前数据无法得到完整业务阶段占完整调度 + 窗口的精确比例。 + linked vector/cube Kernel 从 SYS 与 PMU counter 一并门控排除；result-used return-ready atomic + 依赖区间只从 SYS 边界累计值扣除，I-cache/PMU counter 仍含其指令事件；source-issue atomic + 保留。request/miss 百分比的分母是同一 ELF、同一次采集的 Submit 整窗 primary。 + observed_plus_capture_gap = observed + (primary − shadow)； + 边界读数和插桩 bookkeeping 会进入 sample，因此“观测值”和“加全窗 capture gap”都不是原业务 + 事件数的数学上下界。不同 ELF 不能直接相减后冒充业务真值；提供 empty-bracket 时只按上方 + 明示的公式估算局部记录代码开销。request/miss 的逐核 min/max 是每核累计整个 phase + 的极值，不是逐调用极值。

+
+ + + + +
指标ALL96 核AIC32 核AIV64 核
Phase PMU total扣除记录代码开销估算后的参考值 Σ 56,816,784.374 cycles
≈ 34,437.671 µs（1.649844 cycles/ns）
原始 observed Σ 57,216,108 cycles（≈ 34,679.708 µs）− 记录代码开销估算 399,323.626 cycles
记录代码开销估算 / 原始 observed 0.698%；原始逐核 最小 0；最大 2,075,497；原始 223,500.422 cycles/call		扣除记录代码开销估算后的参考值 Σ 15,131,925.180 cycles
≈ 9,170.519 µs（1.650062 cycles/ns）
原始 observed Σ 15,274,488 cycles（≈ 9,256.918 µs）− 记录代码开销估算 142,562.820 cycles
记录代码开销估算 / 原始 observed 0.933%；原始逐核 最小 0；最大 1,349,309；原始 159,109.250 cycles/call		扣除记录代码开销估算后的参考值 Σ 41,684,859.193 cycles
≈ 25,267.670 µs（1.649731 cycles/ns）
原始 observed Σ 41,941,620 cycles（≈ 25,423.308 µs）− 记录代码开销估算 256,760.807 cycles
记录代码开销估算 / 原始 observed 0.612%；原始逐核 最小 0；最大 2,075,497；原始 262,135.125 cycles/call
Phase scalar busy扣除记录代码开销估算后的参考值 Σ 56,372,898.080 cycles
≈ 34,168.623 µs（1.649844 cycles/ns）
原始 observed Σ 56,630,433 cycles（≈ 34,324.720 µs）− 记录代码开销估算 257,534.920 cycles
记录代码开销估算 / 原始 observed 0.455%；原始逐核 最小 0；最大 2,065,174；原始 221,212.629 cycles/call		扣除记录代码开销估算后的参考值 Σ 15,022,112.064 cycles
≈ 9,103.968 µs（1.650062 cycles/ns）
原始 observed Σ 15,116,002 cycles（≈ 9,160.869 µs）− 记录代码开销估算 93,889.936 cycles
记录代码开销估算 / 原始 observed 0.621%；原始逐核 最小 0；最大 1,342,719；原始 157,458.354 cycles/call		扣除记录代码开销估算后的参考值 Σ 41,350,786.016 cycles
≈ 25,065.169 µs（1.649731 cycles/ns）
原始 observed Σ 41,514,431 cycles（≈ 25,164.364 µs）− 记录代码开销估算 163,644.984 cycles
记录代码开销估算 / 原始 observed 0.394%；原始逐核 最小 0；最大 2,065,174；原始 259,465.194 cycles/call
非 Scalar-busy 残余扣除记录代码开销估算后的参考值 Σ 443,886.294 cycles
≈ 269.047 µs（1.649844 cycles/ns）
原始 observed Σ 585,675 cycles（≈ 354.988 µs）− 记录代码开销估算 141,788.706 cycles
记录代码开销估算 / 原始 observed 24.209%；原始逐核 最小 0；最大 23,288；原始 2,287.793 cycles/call		扣除记录代码开销估算后的参考值 Σ 109,813.116 cycles
≈ 66.551 µs（1.650062 cycles/ns）
原始 observed Σ 158,486 cycles（≈ 96.049 µs）− 记录代码开销估算 48,672.884 cycles
记录代码开销估算 / 原始 observed 30.711%；原始逐核 最小 0；最大 11,646；原始 1,650.896 cycles/call		扣除记录代码开销估算后的参考值 Σ 334,073.178 cycles
≈ 202.502 µs（1.649731 cycles/ns）
原始 observed Σ 427,189 cycles（≈ 258.945 µs）− 记录代码开销估算 93,115.822 cycles
记录代码开销估算 / 原始 observed 21.797%；原始逐核 最小 0；最大 23,288；原始 2,669.931 cycles/call
阶段参考关系（raw 同格保留）扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 99.219%
参考 Non-scalar / 参考 Phase total 0.781%
参考 Phase total / 原始 whole total 6.824%；原始 observed 6.872%
参考 Phase scalar / 原始 whole scalar 7.059%；原始 observed 7.091%
whole scalar−shadow scalar：Σ 512 cycles；逐核 最小 0；最大 20		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 99.274%
参考 Non-scalar / 参考 Phase total 0.726%
参考 Phase total / 原始 whole total 5.626%；原始 observed 5.679%
参考 Phase scalar / 原始 whole scalar 5.629%；原始 observed 5.664%
whole scalar−shadow scalar：Σ 192 cycles；逐核 最小 0；最大 20		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 99.199%
参考 Non-scalar / 参考 Phase total 0.801%
参考 Phase total / 原始 whole total 7.395%；原始 observed 7.441%
参考 Phase scalar / 原始 whole scalar 7.776%；原始 observed 7.807%
whole scalar−shadow scalar：Σ 320 cycles；逐核 最小 0；最大 12
Request（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 525,452.749
原始 observed 539,738 − 记录代码开销估算 14,285.251
参考值 / 原始整窗 0.728%；原始 observed / 原始整窗 0.747%
记录代码开销估算 / 原始 observed 2.647%
原始逐核 最小 0；最大 17,122；原始整窗 72,212,561
原始 capture gap +0；加 gap 后 539,738（0.747%）		扣除记录代码开销估算后的参考值 162,277.831
原始 observed 167,486 − 记录代码开销估算 5,208.169
参考值 / 原始整窗 0.693%；原始 observed / 原始整窗 0.716%
记录代码开销估算 / 原始 observed 3.110%
原始逐核 最小 0；最大 12,416；原始整窗 23,407,344
原始 capture gap +0；加 gap 后 167,486（0.716%）		扣除记录代码开销估算后的参考值 363,174.918
原始 observed 372,252 − 记录代码开销估算 9,077.082
参考值 / 原始整窗 0.744%；原始 observed / 原始整窗 0.763%
记录代码开销估算 / 原始 observed 2.438%
原始逐核 最小 0；最大 17,122；原始整窗 48,805,217
原始 capture gap +0；加 gap 后 372,252（0.763%）
Miss（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 12,369.305
原始 observed 12,829 − 记录代码开销估算 459.695
参考值 / 原始整窗 0.930%；原始 observed / 原始整窗 0.965%
记录代码开销估算 / 原始 observed 3.583%
原始逐核 最小 0；最大 344；原始整窗 1,329,490
原始 capture gap +0；加 gap 后 12,829（0.965%）		扣除记录代码开销估算后的参考值 3,233.166
原始 observed 3,239 − 记录代码开销估算 5.834
参考值 / 原始整窗 5.650%；原始 observed / 原始整窗 5.661%
记录代码开销估算 / 原始 observed 0.180%
原始逐核 最小 0；最大 204；原始整窗 57,221
原始 capture gap +0；加 gap 后 3,239（5.661%）		扣除记录代码开销估算后的参考值 9,136.139
原始 observed 9,590 − 记录代码开销估算 453.861
参考值 / 原始整窗 0.718%；原始 observed / 原始整窗 0.754%
记录代码开销估算 / 原始 observed 4.733%
原始逐核 最小 0；最大 344；原始整窗 1,272,269
原始 capture gap +0；加 gap 后 9,590（0.754%）
SYS 边界诊断 / Begin-EndΣ 1,659,528 raw ticks
逐核 最小 0；最大 47,745；仅边界诊断
Begin / End：256 / 256
业务调用 256 次；排除 linked Kernel 调用 0 次
逐核 0–10；零调用核 7		Σ 494,259 raw ticks
逐核 最小 0；最大 35,221；仅边界诊断
Begin / End：96 / 96
业务调用 96 次；排除 linked Kernel 调用 0 次
逐核 0–10；零调用核 4		Σ 1,165,269 raw ticks
逐核 最小 0；最大 47,745；仅边界诊断
Begin / End：160 / 160
业务调用 160 次；排除 linked Kernel 调用 0 次
逐核 0–6；零调用核 3
+
+ + +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
0ca6a4f7e2b70a1b426beb36273be7a59dac8b07b9190ad7635988c2e8420d03
+
Profile / extra cache
submit-pmu-alloc-complete-control / + e3b514bbd683b612
+
Profiled cache key
TestPagedAttentionUnroll / a5 / fully_distributed_within_core / submit-pmu-alloc-complete-control
+
Source state
source-v2:8c6b6e3a412d2c7c7113c2d8f714a14eb3786f80:cf59372e4506d20850aa16f472f20a82d2b03d1c9e3064a2ff6ac26c4041f825:436dccac6ff79a84586081fad64d95bfb96afe4f5bd9fdd773bf4d541b086afd
+
Source-state stamp
/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/e3b514bbd683b612/aicore/.git_commit
+
Compile definitions
PTO_FDWIC_SUBMIT_PMU=1
PTO_FDWIC_SUBMIT_PMU_PHASE_ID=11
PTO_FDWIC_TRACE_ENABLED=0
+
+
+ + + +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
AICore final2,648,576b72aafb602e9e792c80755ddc787e8a284db3d01f73f723df10cb130c87e8e40207,18482d3132b1ee46f96038a65cab193df37721881071f37fc53ae2657cdf9f10ad8/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/aicore-extra/e3b514bbd683b612/aicore_kernel.o
AIC combined1,976,43206e202c618d87d8bc372bfb757943319e95265ba662ddac896264cd8fb3e975d96,456ee3ab9d80a7f3d4b52a7f73c44c512888f330134d790cbca628b545aac8e7ba8/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/e3b514bbd683b612/aicore/aicore_aic_combined.o
AIV combined2,237,80813cfca73c08daf970d2ee8baf152ae2d9b4f47bd09a6d3419eb3003baa53b05f110,6729593c33168230939209eeee700ec6628ddfdbac2b0ae7b382bf40f58d14f2cb8/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/e3b514bbd683b612/aicore/aicore_aiv_combined.o
Host runtime11,776,1440369d0602054bf612f86b58c294503e00486bd505bff9ee7a455994f043147cc454,6325ac381ed10be5f5d492b8072c051a177131f4caec6d6f6bc9bf50880da4bb0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so
+
+ +
+
原始全局 Submit 时间范围（墙钟） + 5,946.854 µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + 128096 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-2568302855a5a0f279b2cc37f6961498da722d27ee0461825fcea844fd4526ac9c3
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + 90.000 ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×90 ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
+
+

ALL · 96 核

+
+
Submit PMU total/core
+
均值 8,673,341.1；最小 8,056,906；最大 9,574,428 cycles
+ 等效时间 均值 5,257.067；最小 4,883.435；最大 5,803.232 µs + （按 1.649844 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 8,319,044.3；最小 7,818,310；最大 9,107,665 cycles
+ 等效时间 均值 5,042.322；最小 4,738.818；最大 5,520.319 µs； + 加权占比 95.915% +
+
非 Scalar-busy 残余/core
+
均值 354,296.8；最小 48,240；最大 669,580 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 214.746；最小 29.239；最大 405.844 µs +
+
SYS gate 边界诊断/core
+
均值 3,054,726.1；最小 2,512,850；最大 4,220,275 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,597.442；最小 5,366.295；最大 5,941.592 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 2,542,716.2；最小 1,244,936；最大 3,222,968 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 54.574% +
+
Primary I-cache request/core
最小 686,942；最大 895,916
+
Primary I-cache miss/core
最小 1,224；最大 23,126
+
加权 miss rate
1.841%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 110.160；最大 2,081.340 µs
+
+
+ +
+

AIC · 32 核

+
+
Submit PMU total/core
+
均值 8,405,481.3；最小 8,056,906；最大 8,736,621 cycles
+ 等效时间 均值 5,094.040；最小 4,882.790；最大 5,294.723 µs + （按 1.650062 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 8,340,214.4；最小 7,987,172；最大 8,670,352 cycles
+ 等效时间 均值 5,054.485；最小 4,840.528；最大 5,254.561 µs； + 加权占比 99.224% +
+
非 Scalar-busy 残余/core
+
均值 65,266.8；最小 48,240；最大 88,967 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 39.554；最小 29.235；最大 53.917 µs +
+
SYS gate 边界诊断/core
+
均值 2,743,792.6；最小 2,512,850；最大 3,795,065 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,677.873；最小 5,527.088；最大 5,903.317 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 2,934,080.8；最小 2,038,347；最大 3,222,968 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 48.324% +
+
Primary I-cache request/core
最小 697,379；最大 820,470
+
Primary I-cache miss/core
最小 1,224；最大 2,607
+
加权 miss rate
0.244%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 110.160；最大 234.630 µs
+
+
+ +
+

AIV · 64 核

+
+
Submit PMU total/core
+
均值 8,807,271.0；最小 8,231,235；最大 9,574,428 cycles
+ 等效时间 均值 5,338.610；最小 4,989.441；最大 5,803.630 µs + （按 1.649731 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 8,308,459.2；最小 7,818,310；最大 9,107,665 cycles
+ 等效时间 均值 5,036.251；最小 4,739.142；最大 5,520.697 µs； + 加权占比 94.336% +
+
非 Scalar-busy 残余/core
+
均值 498,811.8；最小 335,897；最大 669,580 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 302.359；最小 203.607；最大 405.872 µs +
+
SYS gate 边界诊断/core
+
均值 3,210,192.9；最小 2,829,467；最大 4,220,275 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,557.227；最小 5,366.295；最大 5,941.592 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 2,347,033.9；最小 1,244,936；最大 2,746,433 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 57.766% +
+
Primary I-cache request/core
最小 686,942；最大 895,916
+
Primary I-cache miss/core
最小 17,419；最大 23,126
+
加权 miss rate
2.607%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 1,567.710；最大 2,081.340 µs
+
+
+
+

逐物理核分布

+

AICAIV

+
+
+
SYS gate boundary-diagnostic ticks/core
+ + + + 4,220,275 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_submit_elapsed_ticks=2892399physical=1 logical=1 role=AIC scalar_submit_elapsed_ticks=2679618physical=2 logical=2 role=AIC scalar_submit_elapsed_ticks=2718890physical=3 logical=3 role=AIC scalar_submit_elapsed_ticks=2512850physical=4 logical=4 role=AIC scalar_submit_elapsed_ticks=2696216physical=5 logical=5 role=AIC scalar_submit_elapsed_ticks=2653375physical=6 logical=6 role=AIC scalar_submit_elapsed_ticks=2947805physical=7 logical=7 role=AIC scalar_submit_elapsed_ticks=2730021physical=8 logical=8 role=AIC scalar_submit_elapsed_ticks=2758379physical=9 logical=9 role=AIC scalar_submit_elapsed_ticks=2611360physical=10 logical=10 role=AIC scalar_submit_elapsed_ticks=2833358physical=12 logical=11 role=AIC scalar_submit_elapsed_ticks=3795065physical=13 logical=12 role=AIC scalar_submit_elapsed_ticks=2737828physical=14 logical=13 role=AIC scalar_submit_elapsed_ticks=2583400physical=15 logical=14 role=AIC scalar_submit_elapsed_ticks=2752079physical=16 logical=15 role=AIC scalar_submit_elapsed_ticks=2529985physical=54 logical=16 role=AIC scalar_submit_elapsed_ticks=2739273physical=55 logical=17 role=AIC scalar_submit_elapsed_ticks=2625852physical=56 logical=18 role=AIC scalar_submit_elapsed_ticks=3040389physical=57 logical=19 role=AIC scalar_submit_elapsed_ticks=2546668physical=58 logical=20 role=AIC scalar_submit_elapsed_ticks=2704337physical=60 logical=21 role=AIC scalar_submit_elapsed_ticks=2642349physical=61 logical=22 role=AIC scalar_submit_elapsed_ticks=2664034physical=62 logical=23 role=AIC scalar_submit_elapsed_ticks=2793728physical=63 logical=24 role=AIC scalar_submit_elapsed_ticks=2877451physical=64 logical=25 role=AIC scalar_submit_elapsed_ticks=2639334physical=65 logical=26 role=AIC scalar_submit_elapsed_ticks=2767907physical=66 logical=27 role=AIC scalar_submit_elapsed_ticks=2830955physical=67 logical=28 role=AIC scalar_submit_elapsed_ticks=2636092physical=68 logical=29 role=AIC scalar_submit_elapsed_ticks=2568030physical=69 logical=30 role=AIC scalar_submit_elapsed_ticks=2739116physical=70 logical=31 role=AIC scalar_submit_elapsed_ticks=2553221physical=18 logical=32 role=AIV scalar_submit_elapsed_ticks=3223281physical=19 logical=33 role=AIV scalar_submit_elapsed_ticks=3625439physical=20 logical=34 role=AIV scalar_submit_elapsed_ticks=3219401physical=21 logical=35 role=AIV scalar_submit_elapsed_ticks=3452379physical=22 logical=36 role=AIV scalar_submit_elapsed_ticks=3240222physical=23 logical=37 role=AIV scalar_submit_elapsed_ticks=3234367physical=24 logical=38 role=AIV scalar_submit_elapsed_ticks=3986847physical=25 logical=39 role=AIV scalar_submit_elapsed_ticks=3142483physical=26 logical=40 role=AIV scalar_submit_elapsed_ticks=3257879physical=27 logical=41 role=AIV scalar_submit_elapsed_ticks=2851224physical=28 logical=42 role=AIV scalar_submit_elapsed_ticks=3120115physical=29 logical=43 role=AIV scalar_submit_elapsed_ticks=2829467physical=30 logical=44 role=AIV scalar_submit_elapsed_ticks=3148569physical=31 logical=45 role=AIV scalar_submit_elapsed_ticks=3071512physical=32 logical=46 role=AIV scalar_submit_elapsed_ticks=3214404physical=33 logical=47 role=AIV scalar_submit_elapsed_ticks=3103419physical=34 logical=48 role=AIV scalar_submit_elapsed_ticks=4220275physical=35 logical=49 role=AIV scalar_submit_elapsed_ticks=2904001physical=36 logical=50 role=AIV scalar_submit_elapsed_ticks=3333105physical=37 logical=51 role=AIV scalar_submit_elapsed_ticks=3341778physical=38 logical=52 role=AIV scalar_submit_elapsed_ticks=3137660physical=39 logical=53 role=AIV scalar_submit_elapsed_ticks=3188448physical=42 logical=54 role=AIV scalar_submit_elapsed_ticks=3519173physical=43 logical=55 role=AIV scalar_submit_elapsed_ticks=3196419physical=44 logical=56 role=AIV scalar_submit_elapsed_ticks=3033683physical=45 logical=57 role=AIV scalar_submit_elapsed_ticks=3191699physical=46 logical=58 role=AIV scalar_submit_elapsed_ticks=2951848physical=47 logical=59 role=AIV scalar_submit_elapsed_ticks=3026378physical=48 logical=60 role=AIV scalar_submit_elapsed_ticks=3027325physical=49 logical=61 role=AIV scalar_submit_elapsed_ticks=3266718physical=50 logical=62 role=AIV scalar_submit_elapsed_ticks=3252024physical=51 logical=63 role=AIV scalar_submit_elapsed_ticks=3002447physical=72 logical=64 role=AIV scalar_submit_elapsed_ticks=3207997physical=73 logical=65 role=AIV scalar_submit_elapsed_ticks=3143937physical=74 logical=66 role=AIV scalar_submit_elapsed_ticks=3038577physical=75 logical=67 role=AIV scalar_submit_elapsed_ticks=3124356physical=76 logical=68 role=AIV scalar_submit_elapsed_ticks=3322362physical=77 logical=69 role=AIV scalar_submit_elapsed_ticks=3056185physical=78 logical=70 role=AIV scalar_submit_elapsed_ticks=3096821physical=79 logical=71 role=AIV scalar_submit_elapsed_ticks=3113062physical=80 logical=72 role=AIV scalar_submit_elapsed_ticks=3071101physical=81 logical=73 role=AIV scalar_submit_elapsed_ticks=2966426physical=84 logical=74 role=AIV scalar_submit_elapsed_ticks=3408378physical=85 logical=75 role=AIV scalar_submit_elapsed_ticks=3864104physical=86 logical=76 role=AIV scalar_submit_elapsed_ticks=2966834physical=87 logical=77 role=AIV scalar_submit_elapsed_ticks=2959273physical=88 logical=78 role=AIV scalar_submit_elapsed_ticks=3192981physical=89 logical=79 role=AIV scalar_submit_elapsed_ticks=3083103physical=90 logical=80 role=AIV scalar_submit_elapsed_ticks=3020675physical=91 logical=81 role=AIV scalar_submit_elapsed_ticks=3013636physical=92 logical=82 role=AIV scalar_submit_elapsed_ticks=3643693physical=93 logical=83 role=AIV scalar_submit_elapsed_ticks=3237245physical=94 logical=84 role=AIV scalar_submit_elapsed_ticks=2980671physical=95 logical=85 role=AIV scalar_submit_elapsed_ticks=3188306physical=96 logical=86 role=AIV scalar_submit_elapsed_ticks=3248198physical=97 logical=87 role=AIV scalar_submit_elapsed_ticks=3263605physical=98 logical=88 role=AIV scalar_submit_elapsed_ticks=3232010physical=99 logical=89 role=AIV scalar_submit_elapsed_ticks=4052519physical=100 logical=90 role=AIV scalar_submit_elapsed_ticks=3073669physical=101 logical=91 role=AIV scalar_submit_elapsed_ticks=3185697physical=102 logical=92 role=AIV scalar_submit_elapsed_ticks=3084443physical=103 logical=93 role=AIV scalar_submit_elapsed_ticks=2966116physical=104 logical=94 role=AIV scalar_submit_elapsed_ticks=3157150physical=105 logical=95 role=AIV scalar_submit_elapsed_ticks=3175224 + +
+ +
+
PMU total cycles/core
+ + + + 9,574,428 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC total_cycles=8263050physical=1 logical=1 role=AIC total_cycles=8257751physical=2 logical=2 role=AIC total_cycles=8398540physical=3 logical=3 role=AIC total_cycles=8220562physical=4 logical=4 role=AIC total_cycles=8680205physical=5 logical=5 role=AIC total_cycles=8697967physical=6 logical=6 role=AIC total_cycles=8421406physical=7 logical=7 role=AIC total_cycles=8255521physical=8 logical=8 role=AIC total_cycles=8139236physical=9 logical=9 role=AIC total_cycles=8546744physical=10 logical=10 role=AIC total_cycles=8723019physical=12 logical=11 role=AIC total_cycles=8736621physical=13 logical=12 role=AIC total_cycles=8476618physical=14 logical=13 role=AIC total_cycles=8132046physical=15 logical=14 role=AIC total_cycles=8492612physical=16 logical=15 role=AIC total_cycles=8304340physical=54 logical=16 role=AIC total_cycles=8518128physical=55 logical=17 role=AIC total_cycles=8268700physical=56 logical=18 role=AIC total_cycles=8372350physical=57 logical=19 role=AIC total_cycles=8547196physical=58 logical=20 role=AIC total_cycles=8186870physical=60 logical=21 role=AIC total_cycles=8304897physical=61 logical=22 role=AIC total_cycles=8479011physical=62 logical=23 role=AIC total_cycles=8358612physical=63 logical=24 role=AIC total_cycles=8361002physical=64 logical=25 role=AIC total_cycles=8221427physical=65 logical=26 role=AIC total_cycles=8696927physical=66 logical=27 role=AIC total_cycles=8056906physical=67 logical=28 role=AIC total_cycles=8311099physical=68 logical=29 role=AIC total_cycles=8589893physical=69 logical=30 role=AIC total_cycles=8363745physical=70 logical=31 role=AIC total_cycles=8592400physical=18 logical=32 role=AIV total_cycles=8795879physical=19 logical=33 role=AIV total_cycles=8808047physical=20 logical=34 role=AIV total_cycles=9185055physical=21 logical=35 role=AIV total_cycles=9206405physical=22 logical=36 role=AIV total_cycles=8849585physical=23 logical=37 role=AIV total_cycles=9063188physical=24 logical=38 role=AIV total_cycles=8770806physical=25 logical=39 role=AIV total_cycles=8762730physical=26 logical=40 role=AIV total_cycles=9455063physical=27 logical=41 role=AIV total_cycles=8581772physical=28 logical=42 role=AIV total_cycles=9088807physical=29 logical=43 role=AIV total_cycles=8692198physical=30 logical=44 role=AIV total_cycles=8683682physical=31 logical=45 role=AIV total_cycles=9030628physical=32 logical=46 role=AIV total_cycles=9241564physical=33 logical=47 role=AIV total_cycles=8669628physical=34 logical=48 role=AIV total_cycles=9009479physical=35 logical=49 role=AIV total_cycles=8401267physical=36 logical=50 role=AIV total_cycles=8484328physical=37 logical=51 role=AIV total_cycles=8848165physical=38 logical=52 role=AIV total_cycles=8712166physical=39 logical=53 role=AIV total_cycles=8777946physical=42 logical=54 role=AIV total_cycles=9574428physical=43 logical=55 role=AIV total_cycles=8830056physical=44 logical=56 role=AIV total_cycles=8514780physical=45 logical=57 role=AIV total_cycles=9333481physical=46 logical=58 role=AIV total_cycles=8319850physical=47 logical=59 role=AIV total_cycles=8860891physical=48 logical=60 role=AIV total_cycles=8846395physical=49 logical=61 role=AIV total_cycles=8775764physical=50 logical=62 role=AIV total_cycles=8592325physical=51 logical=63 role=AIV total_cycles=8231235physical=72 logical=64 role=AIV total_cycles=8396185physical=73 logical=65 role=AIV total_cycles=8881687physical=74 logical=66 role=AIV total_cycles=8428727physical=75 logical=67 role=AIV total_cycles=9228218physical=76 logical=68 role=AIV total_cycles=9230150physical=77 logical=69 role=AIV total_cycles=8611491physical=78 logical=70 role=AIV total_cycles=8670008physical=79 logical=71 role=AIV total_cycles=8941024physical=80 logical=72 role=AIV total_cycles=8691887physical=81 logical=73 role=AIV total_cycles=9036707physical=84 logical=74 role=AIV total_cycles=8516234physical=85 logical=75 role=AIV total_cycles=8777669physical=86 logical=76 role=AIV total_cycles=8797067physical=87 logical=77 role=AIV total_cycles=8622792physical=88 logical=78 role=AIV total_cycles=8774904physical=89 logical=79 role=AIV total_cycles=8577948physical=90 logical=80 role=AIV total_cycles=8461015physical=91 logical=81 role=AIV total_cycles=8653725physical=92 logical=82 role=AIV total_cycles=9523946physical=93 logical=83 role=AIV total_cycles=8653961physical=94 logical=84 role=AIV total_cycles=8685717physical=95 logical=85 role=AIV total_cycles=8808813physical=96 logical=86 role=AIV total_cycles=8823243physical=97 logical=87 role=AIV total_cycles=8892191physical=98 logical=88 role=AIV total_cycles=8657785physical=99 logical=89 role=AIV total_cycles=8603995physical=100 logical=90 role=AIV total_cycles=8836285physical=101 logical=91 role=AIV total_cycles=8700509physical=102 logical=92 role=AIV total_cycles=8780308physical=103 logical=93 role=AIV total_cycles=8588425physical=104 logical=94 role=AIV total_cycles=9178926physical=105 logical=95 role=AIV total_cycles=8636206 + +
+ +
+
Scalar busy cycles/core
+ + + + 9,107,665 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_busy=8192036physical=1 logical=1 role=AIC scalar_busy=8170963physical=2 logical=2 role=AIC scalar_busy=8336509physical=3 logical=3 role=AIC scalar_busy=8158127physical=4 logical=4 role=AIC scalar_busy=8619712physical=5 logical=5 role=AIC scalar_busy=8611310physical=6 logical=6 role=AIC scalar_busy=8356115physical=7 logical=7 role=AIC scalar_busy=8177209physical=8 logical=8 role=AIC scalar_busy=8076593physical=9 logical=9 role=AIC scalar_busy=8486031physical=10 logical=10 role=AIC scalar_busy=8662247physical=12 logical=11 role=AIC scalar_busy=8670352physical=13 logical=12 role=AIC scalar_busy=8420095physical=14 logical=13 role=AIC scalar_busy=8061243physical=15 logical=14 role=AIC scalar_busy=8439345physical=16 logical=15 role=AIC scalar_busy=8256100physical=54 logical=16 role=AIC scalar_busy=8451349physical=55 logical=17 role=AIC scalar_busy=8199851physical=56 logical=18 role=AIC scalar_busy=8307364physical=57 logical=19 role=AIC scalar_busy=8478144physical=58 logical=20 role=AIC scalar_busy=8120728physical=60 logical=21 role=AIC scalar_busy=8233551physical=61 logical=22 role=AIC scalar_busy=8423109physical=62 logical=23 role=AIC scalar_busy=8298130physical=63 logical=24 role=AIC scalar_busy=8296859physical=64 logical=25 role=AIC scalar_busy=8132460physical=65 logical=26 role=AIC scalar_busy=8634762physical=66 logical=27 role=AIC scalar_busy=7987172physical=67 logical=28 role=AIC scalar_busy=8257437physical=68 logical=29 role=AIC scalar_busy=8528759physical=69 logical=30 role=AIC scalar_busy=8312474physical=70 logical=31 role=AIC scalar_busy=8530726physical=18 logical=32 role=AIV scalar_busy=8194084physical=19 logical=33 role=AIV scalar_busy=8237942physical=20 logical=34 role=AIV scalar_busy=8573437physical=21 logical=35 role=AIV scalar_busy=8536825physical=22 logical=36 role=AIV scalar_busy=8372426physical=23 logical=37 role=AIV scalar_busy=8563902physical=24 logical=38 role=AIV scalar_busy=8176361physical=25 logical=39 role=AIV scalar_busy=8214545physical=26 logical=40 role=AIV scalar_busy=9058152physical=27 logical=41 role=AIV scalar_busy=8132655physical=28 logical=42 role=AIV scalar_busy=8615903physical=29 logical=43 role=AIV scalar_busy=8226835physical=30 logical=44 role=AIV scalar_busy=8220666physical=31 logical=45 role=AIV scalar_busy=8543025physical=32 logical=46 role=AIV scalar_busy=8784226physical=33 logical=47 role=AIV scalar_busy=8141602physical=34 logical=48 role=AIV scalar_busy=8568128physical=35 logical=49 role=AIV scalar_busy=7906663physical=36 logical=50 role=AIV scalar_busy=7937972physical=37 logical=51 role=AIV scalar_busy=8256642physical=38 logical=52 role=AIV scalar_busy=8230716physical=39 logical=53 role=AIV scalar_busy=8193056physical=42 logical=54 role=AIV scalar_busy=9107665physical=43 logical=55 role=AIV scalar_busy=8246273physical=44 logical=56 role=AIV scalar_busy=8027813physical=45 logical=57 role=AIV scalar_busy=8760447physical=46 logical=58 role=AIV scalar_busy=7872501physical=47 logical=59 role=AIV scalar_busy=8330533physical=48 logical=60 role=AIV scalar_busy=8386585physical=49 logical=61 role=AIV scalar_busy=8319729physical=50 logical=62 role=AIV scalar_busy=8170006physical=51 logical=63 role=AIV scalar_busy=7818310physical=72 logical=64 role=AIV scalar_busy=7938629physical=73 logical=65 role=AIV scalar_busy=8400304physical=74 logical=66 role=AIV scalar_busy=7974394physical=75 logical=67 role=AIV scalar_busy=8825117physical=76 logical=68 role=AIV scalar_busy=8727633physical=77 logical=69 role=AIV scalar_busy=8084581physical=78 logical=70 role=AIV scalar_busy=8231940physical=79 logical=71 role=AIV scalar_busy=8409946physical=80 logical=72 role=AIV scalar_busy=8169419physical=81 logical=73 role=AIV scalar_busy=8529111physical=84 logical=74 role=AIV scalar_busy=7987806physical=85 logical=75 role=AIV scalar_busy=8298372physical=86 logical=76 role=AIV scalar_busy=8461170physical=87 logical=77 role=AIV scalar_busy=8104663physical=88 logical=78 role=AIV scalar_busy=8291784physical=89 logical=79 role=AIV scalar_busy=8048199physical=90 logical=80 role=AIV scalar_busy=8073774physical=91 logical=81 role=AIV scalar_busy=8132870physical=92 logical=82 role=AIV scalar_busy=8971178physical=93 logical=83 role=AIV scalar_busy=8121107physical=94 logical=84 role=AIV scalar_busy=8245115physical=95 logical=85 role=AIV scalar_busy=8273528physical=96 logical=86 role=AIV scalar_busy=8369808physical=97 logical=87 role=AIV scalar_busy=8359238physical=98 logical=88 role=AIV scalar_busy=8178004physical=99 logical=89 role=AIV scalar_busy=8128703physical=100 logical=90 role=AIV scalar_busy=8381231physical=101 logical=91 role=AIV scalar_busy=8191801physical=102 logical=92 role=AIV scalar_busy=8323089physical=103 logical=93 role=AIV scalar_busy=8048961physical=104 logical=94 role=AIV scalar_busy=8651170physical=105 logical=95 role=AIV scalar_busy=8083118 + +
+ +
+
Primary I-cache requests/core
+ + + + 895,916 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_requests=764601physical=1 logical=1 role=AIC icache_requests=741758physical=2 logical=2 role=AIC icache_requests=699782physical=3 logical=3 role=AIC icache_requests=705465physical=4 logical=4 role=AIC icache_requests=714968physical=5 logical=5 role=AIC icache_requests=751452physical=6 logical=6 role=AIC icache_requests=810055physical=7 logical=7 role=AIC icache_requests=785509physical=8 logical=8 role=AIC icache_requests=717067physical=9 logical=9 role=AIC icache_requests=724137physical=10 logical=10 role=AIC icache_requests=702774physical=12 logical=11 role=AIC icache_requests=716985physical=13 logical=12 role=AIC icache_requests=728817physical=14 logical=13 role=AIC icache_requests=738671physical=15 logical=14 role=AIC icache_requests=697379physical=16 logical=15 role=AIC icache_requests=713098physical=54 logical=16 role=AIC icache_requests=721043physical=55 logical=17 role=AIC icache_requests=735487physical=56 logical=18 role=AIC icache_requests=707701physical=57 logical=19 role=AIC icache_requests=720201physical=58 logical=20 role=AIC icache_requests=713654physical=60 logical=21 role=AIC icache_requests=745903physical=61 logical=22 role=AIC icache_requests=704892physical=62 logical=23 role=AIC icache_requests=732742physical=63 logical=24 role=AIC icache_requests=769235physical=64 logical=25 role=AIC icache_requests=729812physical=65 logical=26 role=AIC icache_requests=739844physical=66 logical=27 role=AIC icache_requests=820470physical=67 logical=28 role=AIC icache_requests=705546physical=68 logical=29 role=AIC icache_requests=702590physical=69 logical=30 role=AIC icache_requests=726969physical=70 logical=31 role=AIC icache_requests=718737physical=18 logical=32 role=AIV icache_requests=738638physical=19 logical=33 role=AIV icache_requests=693983physical=20 logical=34 role=AIV icache_requests=776451physical=21 logical=35 role=AIV icache_requests=895916physical=22 logical=36 role=AIV icache_requests=809801physical=23 logical=37 role=AIV icache_requests=841504physical=24 logical=38 role=AIV icache_requests=734739physical=25 logical=39 role=AIV icache_requests=774493physical=26 logical=40 role=AIV icache_requests=808576physical=27 logical=41 role=AIV icache_requests=713343physical=28 logical=42 role=AIV icache_requests=742988physical=29 logical=43 role=AIV icache_requests=714235physical=30 logical=44 role=AIV icache_requests=726937physical=31 logical=45 role=AIV icache_requests=744886physical=32 logical=46 role=AIV icache_requests=785972physical=33 logical=47 role=AIV icache_requests=751659physical=34 logical=48 role=AIV icache_requests=689519physical=35 logical=49 role=AIV icache_requests=730606physical=36 logical=50 role=AIV icache_requests=715705physical=37 logical=51 role=AIV icache_requests=728539physical=38 logical=52 role=AIV icache_requests=759128physical=39 logical=53 role=AIV icache_requests=782466physical=42 logical=54 role=AIV icache_requests=875884physical=43 logical=55 role=AIV icache_requests=780005physical=44 logical=56 role=AIV icache_requests=717420physical=45 logical=57 role=AIV icache_requests=816843physical=46 logical=58 role=AIV icache_requests=717939physical=47 logical=59 role=AIV icache_requests=743509physical=48 logical=60 role=AIV icache_requests=715205physical=49 logical=61 role=AIV icache_requests=831465physical=50 logical=62 role=AIV icache_requests=699072physical=51 logical=63 role=AIV icache_requests=744947physical=72 logical=64 role=AIV icache_requests=793711physical=73 logical=65 role=AIV icache_requests=744397physical=74 logical=66 role=AIV icache_requests=714800physical=75 logical=67 role=AIV icache_requests=808868physical=76 logical=68 role=AIV icache_requests=803374physical=77 logical=69 role=AIV icache_requests=741559physical=78 logical=70 role=AIV icache_requests=761910physical=79 logical=71 role=AIV icache_requests=798874physical=80 logical=72 role=AIV icache_requests=733489physical=81 logical=73 role=AIV icache_requests=749758physical=84 logical=74 role=AIV icache_requests=800666physical=85 logical=75 role=AIV icache_requests=686942physical=86 logical=76 role=AIV icache_requests=724515physical=87 logical=77 role=AIV icache_requests=731506physical=88 logical=78 role=AIV icache_requests=708745physical=89 logical=79 role=AIV icache_requests=782937physical=90 logical=80 role=AIV icache_requests=730965physical=91 logical=81 role=AIV icache_requests=773660physical=92 logical=82 role=AIV icache_requests=781782physical=93 logical=83 role=AIV icache_requests=834036physical=94 logical=84 role=AIV icache_requests=707709physical=95 logical=85 role=AIV icache_requests=810839physical=96 logical=86 role=AIV icache_requests=817145physical=97 logical=87 role=AIV icache_requests=865119physical=98 logical=88 role=AIV icache_requests=814017physical=99 logical=89 role=AIV icache_requests=693066physical=100 logical=90 role=AIV icache_requests=735572physical=101 logical=91 role=AIV icache_requests=805921physical=102 logical=92 role=AIV icache_requests=736690physical=103 logical=93 role=AIV icache_requests=712809physical=104 logical=94 role=AIV icache_requests=780775physical=105 logical=95 role=AIV icache_requests=816688 + +
+ +
+
Primary I-cache misses/core
+ + + + 23,126 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_misses=1816physical=1 logical=1 role=AIC icache_misses=2297physical=2 logical=2 role=AIC icache_misses=1664physical=3 logical=3 role=AIC icache_misses=1727physical=4 logical=4 role=AIC icache_misses=1600physical=5 logical=5 role=AIC icache_misses=2607physical=6 logical=6 role=AIC icache_misses=1584physical=7 logical=7 role=AIC icache_misses=1913physical=8 logical=8 role=AIC icache_misses=1831physical=9 logical=9 role=AIC icache_misses=1594physical=10 logical=10 role=AIC icache_misses=1776physical=12 logical=11 role=AIC icache_misses=1957physical=13 logical=12 role=AIC icache_misses=1740physical=14 logical=13 role=AIC icache_misses=1823physical=15 logical=14 role=AIC icache_misses=1307physical=16 logical=15 role=AIC icache_misses=1367physical=54 logical=16 role=AIC icache_misses=2225physical=55 logical=17 role=AIC icache_misses=2012physical=56 logical=18 role=AIC icache_misses=1822physical=57 logical=19 role=AIC icache_misses=1919physical=58 logical=20 role=AIC icache_misses=1913physical=60 logical=21 role=AIC icache_misses=1931physical=61 logical=22 role=AIC icache_misses=1773physical=62 logical=23 role=AIC icache_misses=1655physical=63 logical=24 role=AIC icache_misses=1661physical=64 logical=25 role=AIC icache_misses=2413physical=65 logical=26 role=AIC icache_misses=1599physical=66 logical=27 role=AIC icache_misses=1818physical=67 logical=28 role=AIC icache_misses=1398physical=68 logical=29 role=AIC icache_misses=1483physical=69 logical=30 role=AIC icache_misses=1224physical=70 logical=31 role=AIC icache_misses=1772physical=18 logical=32 role=AIV icache_misses=21521physical=19 logical=33 role=AIV icache_misses=18563physical=20 logical=34 role=AIV icache_misses=21112physical=21 logical=35 role=AIV icache_misses=20692physical=22 logical=36 role=AIV icache_misses=18784physical=23 logical=37 role=AIV icache_misses=18942physical=24 logical=38 role=AIV icache_misses=20273physical=25 logical=39 role=AIV icache_misses=19933physical=26 logical=40 role=AIV icache_misses=18375physical=27 logical=41 role=AIV icache_misses=18340physical=28 logical=42 role=AIV icache_misses=20032physical=29 logical=43 role=AIV icache_misses=18394physical=30 logical=44 role=AIV icache_misses=18934physical=31 logical=45 role=AIV icache_misses=18522physical=32 logical=46 role=AIV icache_misses=18204physical=33 logical=47 role=AIV icache_misses=17784physical=34 logical=48 role=AIV icache_misses=17419physical=35 logical=49 role=AIV icache_misses=21134physical=36 logical=50 role=AIV icache_misses=21645physical=37 logical=51 role=AIV icache_misses=21084physical=38 logical=52 role=AIV icache_misses=21176physical=39 logical=53 role=AIV icache_misses=22019physical=42 logical=54 role=AIV icache_misses=17897physical=43 logical=55 role=AIV icache_misses=21498physical=44 logical=56 role=AIV icache_misses=20775physical=45 logical=57 role=AIV icache_misses=18388physical=46 logical=58 role=AIV icache_misses=20699physical=47 logical=59 role=AIV icache_misses=20630physical=48 logical=60 role=AIV icache_misses=20173physical=49 logical=61 role=AIV icache_misses=19275physical=50 logical=62 role=AIV icache_misses=19239physical=51 logical=63 role=AIV icache_misses=18298physical=72 logical=64 role=AIV icache_misses=20936physical=73 logical=65 role=AIV icache_misses=19099physical=74 logical=66 role=AIV icache_misses=20445physical=75 logical=67 role=AIV icache_misses=19205physical=76 logical=68 role=AIV icache_misses=19579physical=77 logical=69 role=AIV icache_misses=20943physical=78 logical=70 role=AIV icache_misses=18323physical=79 logical=71 role=AIV icache_misses=21879physical=80 logical=72 role=AIV icache_misses=22540physical=81 logical=73 role=AIV icache_misses=19069physical=84 logical=74 role=AIV icache_misses=23126physical=85 logical=75 role=AIV icache_misses=19058physical=86 logical=76 role=AIV icache_misses=18398physical=87 logical=77 role=AIV icache_misses=22361physical=88 logical=78 role=AIV icache_misses=20711physical=89 logical=79 role=AIV icache_misses=20529physical=90 logical=80 role=AIV icache_misses=19263physical=91 logical=81 role=AIV icache_misses=21297physical=92 logical=82 role=AIV icache_misses=20987physical=93 logical=83 role=AIV icache_misses=18121physical=94 logical=84 role=AIV icache_misses=19091physical=95 logical=85 role=AIV icache_misses=21456physical=96 logical=86 role=AIV icache_misses=18437physical=97 logical=87 role=AIV icache_misses=20534physical=98 logical=88 role=AIV icache_misses=20428physical=99 logical=89 role=AIV icache_misses=18536physical=100 logical=90 role=AIV icache_misses=19423physical=101 logical=91 role=AIV icache_misses=18606physical=102 logical=92 role=AIV icache_misses=18495physical=103 logical=93 role=AIV icache_misses=20184physical=104 logical=94 role=AIV icache_misses=20069physical=105 logical=95 role=AIV icache_misses=21387 + +
+
+

逐核原始主计数

+

报告展示 primary request/miss；phase 模式的 I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和，只用于标记全窗 capture gap；scalar shadow 同样用于核验 phase 读清重建，并显示其相对 CNT2 primary 的 capture loss，不作为第二份性能数据展示。

+
+ + + + + + + +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate90ns量尺 µs
00AIC002,892,3995,721.8408,263,0508,192,036764,6011,8160.238%163.440
11AIC102,679,6185,546.9038,257,7518,170,963741,7582,2970.310%206.730
22AIC202,718,8905,527.0888,398,5408,336,509699,7821,6640.238%149.760
33AIC302,512,8505,612.8178,220,5628,158,127705,4651,7270.245%155.430
44AIC402,696,2165,800.8978,680,2058,619,712714,9681,6000.224%144.000
55AIC502,653,3755,871.0588,697,9678,611,310751,4522,6070.347%234.630
66AIC602,947,8055,530.5148,421,4068,356,115810,0551,5840.196%142.560
77AIC702,730,0215,761.9868,255,5218,177,209785,5091,9130.244%172.170
88AIC802,758,3795,561.0318,139,2368,076,593717,0671,8310.255%164.790
99AIC902,611,3605,794.2728,546,7448,486,031724,1371,5940.220%143.460
1010AIC1002,833,3585,767.1848,723,0198,662,247702,7741,7760.253%159.840
1211AIC1103,795,0655,833.4128,736,6218,670,352716,9851,9570.273%176.130
1312AIC1202,737,8285,789.9568,476,6188,420,095728,8171,7400.239%156.600
1413AIC1302,583,4005,694.4798,132,0468,061,243738,6711,8230.247%164.070
1514AIC1402,752,0795,721.2788,492,6128,439,345697,3791,3070.187%117.630
1615AIC1502,529,9855,547.8258,304,3408,256,100713,0981,3670.192%123.030
1832AIV013,223,2815,555.5678,795,8798,194,084738,63821,5212.914%1,936.890
1933AIV023,625,4395,396.3028,808,0478,237,942693,98318,5632.675%1,670.670
2034AIV113,219,4015,736.7629,185,0558,573,437776,45121,1122.719%1,900.080
2135AIV123,452,3795,704.2389,206,4058,536,825895,91620,6922.310%1,862.280
2236AIV213,240,2225,477.3948,849,5858,372,426809,80118,7842.320%1,690.560
2337AIV223,234,3675,610.8839,063,1888,563,902841,50418,9422.251%1,704.780
2438AIV313,986,8475,379.2838,770,8068,176,361734,73920,2732.759%1,824.570
2539AIV323,142,4835,534.1468,762,7308,214,545774,49319,9332.574%1,793.970
2640AIV413,257,8795,846.1079,455,0639,058,152808,57618,3752.273%1,653.750
2741AIV422,851,2245,368.6698,581,7728,132,655713,34318,3402.571%1,650.600
2842AIV513,120,1155,840.6499,088,8078,615,903742,98820,0322.696%1,802.880
2943AIV522,829,4675,455.0438,692,1988,226,835714,23518,3942.575%1,655.460
3044AIV613,148,5695,482.8788,683,6828,220,666726,93718,9342.605%1,704.060
3145AIV623,071,5125,641.2119,030,6288,543,025744,88618,5222.487%1,666.980
3246AIV713,214,4045,875.1619,241,5648,784,226785,97218,2042.316%1,638.360
3347AIV723,103,4195,418.5358,669,6288,141,602751,65917,7842.366%1,600.560
3448AIV814,220,2755,465.2119,009,4798,568,128689,51917,4192.526%1,567.710
3549AIV822,904,0015,414.9568,401,2677,906,663730,60621,1342.893%1,902.060
3650AIV913,333,1055,422.4278,484,3287,937,972715,70521,6453.024%1,948.050
3751AIV923,341,7785,479.1398,848,1658,256,642728,53921,0842.894%1,897.560
3852AIV1013,137,6605,505.9358,712,1668,230,716759,12821,1762.790%1,905.840
3953AIV1023,188,4485,544.8068,777,9468,193,056782,46622,0192.814%1,981.710
4254AIV1113,519,1735,871.7349,574,4289,107,665875,88417,8972.043%1,610.730
4355AIV1123,196,4195,735.4678,830,0568,246,273780,00521,4982.756%1,934.820
4456AIV1213,033,6835,382.7708,514,7808,027,813717,42020,7752.896%1,869.750
4557AIV1223,191,6995,828.9229,333,4818,760,447816,84318,3882.251%1,654.920
4658AIV1312,951,8485,373.9428,319,8507,872,501717,93920,6992.883%1,862.910
4759AIV1323,026,3785,437.4978,860,8918,330,533743,50920,6302.775%1,856.700
4860AIV1413,027,3255,588.0178,846,3958,386,585715,20520,1732.821%1,815.570
4961AIV1423,266,7185,437.0718,775,7648,319,729831,46519,2752.318%1,734.750
5062AIV1513,252,0245,373.0308,592,3258,170,006699,07219,2392.752%1,731.510
5163AIV1523,002,4475,366.2958,231,2357,818,310744,94718,2982.456%1,646.820
5416AIC1602,739,2735,568.2798,518,1288,451,349721,0432,2250.309%200.250
5517AIC1702,625,8525,772.1148,268,7008,199,851735,4872,0120.274%181.080
5618AIC1803,040,3895,659.7838,372,3508,307,364707,7011,8220.257%163.980
5719AIC1902,546,6685,769.6368,547,1968,478,144720,2011,9190.266%172.710
5820AIC2002,704,3375,585.6718,186,8708,120,728713,6541,9130.268%172.170
6021AIC2102,642,3495,721.4548,304,8978,233,551745,9031,9310.259%173.790
6122AIC2202,664,0345,530.4738,479,0118,423,109704,8921,7730.252%159.570
6223AIC2302,793,7285,569.5678,358,6128,298,130732,7421,6550.226%148.950
6324AIC2402,877,4515,701.8598,361,0028,296,859769,2351,6610.216%149.490
6425AIC2502,639,3345,569.6038,221,4278,132,460729,8122,4130.331%217.170
6526AIC2602,767,9075,903.3178,696,9278,634,762739,8441,5990.216%143.910
6627AIC2702,830,9555,537.7488,056,9067,987,172820,4701,8180.222%163.620
6728AIC2802,636,0925,583.5718,311,0998,257,437705,5461,3980.198%125.820
6829AIC2902,568,0305,732.3698,589,8938,528,759702,5901,4830.211%133.470
6930AIC3002,739,1165,657.8418,363,7458,312,474726,9691,2240.168%110.160
7031AIC3102,553,2215,746.1268,592,4008,530,726718,7371,7720.247%159.480
7264AIV1613,207,9975,367.1158,396,1857,938,629793,71120,9362.638%1,884.240
7365AIV1623,143,9375,603.0358,881,6878,400,304744,39719,0992.566%1,718.910
7466AIV1713,038,5775,380.7638,428,7277,974,394714,80020,4452.860%1,840.050
7567AIV1723,124,3565,870.7899,228,2188,825,117808,86819,2052.374%1,728.450
7668AIV1813,322,3625,919.6489,230,1508,727,633803,37419,5792.437%1,762.110
7769AIV1823,056,1855,491.2348,611,4918,084,581741,55920,9432.824%1,884.870
7870AIV1913,096,8215,420.8058,670,0088,231,940761,91018,3232.405%1,649.070
7971AIV1923,113,0625,594.3178,941,0248,409,946798,87421,8792.739%1,969.110
8072AIV2013,071,1015,650.9138,691,8878,169,419733,48922,5403.073%2,028.600
8173AIV2022,966,4265,648.0489,036,7078,529,111749,75819,0692.543%1,716.210
8474AIV2113,408,3785,439.1848,516,2347,987,806800,66623,1262.888%2,081.340
8575AIV2123,864,1045,482.5488,777,6698,298,372686,94219,0582.774%1,715.220
8676AIV2212,966,8345,445.4618,797,0678,461,170724,51518,3982.539%1,655.820
8777AIV2222,959,2735,444.9598,622,7928,104,663731,50622,3613.057%2,012.490
8878AIV2313,192,9815,534.3228,774,9048,291,784708,74520,7112.922%1,863.990
8979AIV2323,083,1035,374.5848,577,9488,048,199782,93720,5292.622%1,847.610
9080AIV2413,020,6755,401.7728,461,0158,073,774730,96519,2632.635%1,733.670
9181AIV2423,013,6365,626.1738,653,7258,132,870773,66021,2972.753%1,916.730
9282AIV2513,643,6935,941.5929,523,9468,971,178781,78220,9872.685%1,888.830
9383AIV2523,237,2455,571.2458,653,9618,121,107834,03618,1212.173%1,630.890
9484AIV2612,980,6715,375.7318,685,7178,245,115707,70919,0912.698%1,718.190
9585AIV2623,188,3065,668.1198,808,8138,273,528810,83921,4562.646%1,931.040
9686AIV2713,248,1985,516.4038,823,2438,369,808817,14518,4372.256%1,659.330
9787AIV2723,263,6055,831.9888,892,1918,359,238865,11920,5342.374%1,848.060
9888AIV2813,232,0105,743.0948,657,7858,178,004814,01720,4282.510%1,838.520
9989AIV2824,052,5195,379.0608,603,9958,128,703693,06618,5362.674%1,668.240
10090AIV2913,073,6695,576.0978,836,2858,381,231735,57219,4232.641%1,748.070
10191AIV2923,185,6975,494.3708,700,5098,191,801805,92118,6062.309%1,674.540
10292AIV3013,084,4435,489.0968,780,3088,323,089736,69018,4952.511%1,664.550
10393AIV3022,966,1165,423.3988,588,4258,048,961712,80920,1842.832%1,816.560
10494AIV3113,157,1505,840.1959,178,9268,651,170780,77520,0692.570%1,806.210
10595AIV3123,175,2245,566.4058,636,2068,083,118816,68821,3872.619%1,924.830
+ + diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_arg_build_20260723_043857_icache_report.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_arg_build_20260723_043857_icache_report.html new file mode 100644 index 0000000000..4b657de1e6 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_arg_build_20260723_043857_icache_report.html @@ -0,0 +1,362 @@ + + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 fdwic_submit_pmu_raw.json · schema fdwic-submit-pmu-v3 · 32 AIC + 64 AIV

+ +
+

arg-build 阶段观察（phase_id=1）

+

边界 claim_end_to_materialize_begin · 计数语义 running_read_clear_observed_bracket · + 时间语义 boundary_diagnostic_sys_cnt_between_observers · + expected_calls_per_core=1280

+ +

本报告使用 empty-bracket 估算进入 phase observed + 的局部记录代码开销。 + AIC/AIV 分别按“空区间每组记录开销 × 本阶段实际 begin/end 记录组数”计算，ALL + 再由两类核相加。参考值只从 phase observed 分子扣除该估算；原始 whole 分母保持不变， + 因为空区间没有测量 whole 窗口中的全部记录开销。所以下表的参考比例不是完整业务 + 阶段占完整调度窗口的比例，也不是精确校正。若参考值为负，表示记录代码开销估算已经 + 超过原始 observed；这只能说明该信号低于当前校准的分辨能力，不能截成零或解释成负的 + 业务事件。校准 raw：/home/q00473782/atomic/private/gpt/simpler/outputs/TestPagedAttentionUnroll_Case1_20260723_043533/fdwic_submit_pmu_raw.json · + SHA-256 56199f91545cc6a1fbf18b8bb2cfebae279cb5801516311a040d4facccf321ba。两份 provenance 已核验为同一场景、同一 Git revision 8c6b6e3a412d。

+ + +

本阶段若跨过 linked Kernel，会同时门控 SYS 时间与 PMU counter， + 因而 total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段。 + result-used return-ready atomic 依赖区间只从 SYS 边界累计值扣除，PMU counter + 仍含 atomic 指令事件；source-issue atomic + 保留在时间和计数口径内。没有 linked Kernel 的阶段不触发 PMU 停/开表； + 每次调用的分母仍是外层业务调用次数，不是 Begin/End 读数，也不是排除的 Kernel 调用数。

+ +

阶段原始 PMU 观测来自 running read-clear 的 total cycles，并按 AIC/AIV + 各自校准频率换算；提供 empty-bracket 时，页面主参考值再扣除局部记录代码开销估算。 + phase_elapsed_ticks 只是 SYS 边界闭合诊断，绝不按 1 GHz 当作阶段时间。 + Scalar busy 与 non-scalar residual（逐核先算 total−scalar）共同解释阶段 PMU total； + 未校准时只能展示 raw observed；提供 empty-bracket 后，记录代码开销估算只从 phase + observed 分子扣除，原始 whole 分母保持不变。当前数据无法得到完整业务阶段占完整调度 + 窗口的精确比例。 + linked vector/cube Kernel 从 SYS 与 PMU counter 一并门控排除；result-used return-ready atomic + 依赖区间只从 SYS 边界累计值扣除，I-cache/PMU counter 仍含其指令事件；source-issue atomic + 保留。request/miss 百分比的分母是同一 ELF、同一次采集的 Submit 整窗 primary。 + observed_plus_capture_gap = observed + (primary − shadow)； + 边界读数和插桩 bookkeeping 会进入 sample，因此“观测值”和“加全窗 capture gap”都不是原业务 + 事件数的数学上下界。不同 ELF 不能直接相减后冒充业务真值；提供 empty-bracket 时只按上方 + 明示的公式估算局部记录代码开销。request/miss 的逐核 min/max 是每核累计整个 phase + 的极值，不是逐调用极值。

+
+ + + + +
指标ALL96 核AIC32 核AIV64 核
Phase PMU total扣除记录代码开销估算后的参考值 Σ 63,979,859.000 cycles
≈ 38,779.339 µs（1.649844 cycles/ns）
原始 observed Σ 256,268,195 cycles（≈ 155,328.743 µs）− 记录代码开销估算 192,288,336.000 cycles
记录代码开销估算 / 原始 observed 75.034%；原始逐核 最小 2,396,736；最大 2,857,155；原始 2,085.516 cycles/call		扣除记录代码开销估算后的参考值 Σ 16,327,830.000 cycles
≈ 9,895.283 µs（1.650062 cycles/ns）
原始 observed Σ 77,154,633 cycles（≈ 46,758.627 µs）− 记录代码开销估算 60,826,803.000 cycles
记录代码开销估算 / 原始 observed 78.838%；原始逐核 最小 2,396,736；最大 2,431,534；原始 1,883.658 cycles/call		扣除记录代码开销估算后的参考值 Σ 47,652,029.000 cycles
≈ 28,884.727 µs（1.649731 cycles/ns）
原始 observed Σ 179,113,562 cycles（≈ 108,571.374 µs）− 记录代码开销估算 131,461,533.000 cycles
记录代码开销估算 / 原始 observed 73.396%；原始逐核 最小 2,744,631；最大 2,857,155；原始 2,186.445 cycles/call
Phase scalar busy扣除记录代码开销估算后的参考值 Σ 47,031,713.000 cycles
≈ 28,506.764 µs（1.649844 cycles/ns）
原始 observed Σ 170,877,651 cycles（≈ 103,572.005 µs）− 记录代码开销估算 123,845,938.000 cycles
记录代码开销估算 / 原始 observed 72.476%；原始逐核 最小 1,702,639；最大 1,826,762；原始 1,390.606 cycles/call		扣除记录代码开销估算后的参考值 Σ 14,552,718.000 cycles
≈ 8,819.498 µs（1.650062 cycles/ns）
原始 observed Σ 54,612,424 cycles（≈ 33,097.195 µs）− 记录代码开销估算 40,059,706.000 cycles
记录代码开销估算 / 原始 observed 73.353%；原始逐核 最小 1,702,639；最大 1,717,426；原始 1,333.311 cycles/call		扣除记录代码开销估算后的参考值 Σ 32,478,995.000 cycles
≈ 19,687.449 µs（1.649731 cycles/ns）
原始 observed Σ 116,265,227 cycles（≈ 70,475.264 µs）− 记录代码开销估算 83,786,232.000 cycles
记录代码开销估算 / 原始 observed 72.065%；原始逐核 最小 1,809,609；最大 1,826,762；原始 1,419.253 cycles/call
非 Scalar-busy 残余扣除记录代码开销估算后的参考值 Σ 16,948,146.000 cycles
≈ 10,272.575 µs（1.649844 cycles/ns）
原始 observed Σ 85,390,544 cycles（≈ 51,756.738 µs）− 记录代码开销估算 68,442,398.000 cycles
记录代码开销估算 / 原始 observed 80.152%；原始逐核 最小 693,485；最大 1,038,153；原始 694.910 cycles/call		扣除记录代码开销估算后的参考值 Σ 1,775,112.000 cycles
≈ 1,075.785 µs（1.650062 cycles/ns）
原始 observed Σ 22,542,209 cycles（≈ 13,661.432 µs）− 记录代码开销估算 20,767,097.000 cycles
记录代码开销估算 / 原始 observed 92.125%；原始逐核 最小 693,485；最大 723,172；原始 550.347 cycles/call		扣除记录代码开销估算后的参考值 Σ 15,173,034.000 cycles
≈ 9,197.278 µs（1.649731 cycles/ns）
原始 observed Σ 62,848,335 cycles（≈ 38,096.111 µs）− 记录代码开销估算 47,675,301.000 cycles
记录代码开销估算 / 原始 observed 75.858%；原始逐核 最小 932,601；最大 1,038,153；原始 767.192 cycles/call
阶段参考关系（raw 同格保留）扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 73.510%
参考 Non-scalar / 参考 Phase total 26.490%
参考 Phase total / 原始 whole total 7.384%；原始 observed 29.577%
参考 Phase scalar / 原始 whole scalar 6.150%；原始 observed 22.345%
whole scalar−shadow scalar：Σ 245,760 cycles；逐核 最小 2,560；最大 2,560		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 89.128%
参考 Non-scalar / 参考 Phase total 10.872%
参考 Phase total / 原始 whole total 6.214%；原始 observed 29.363%
参考 Phase scalar / 原始 whole scalar 5.717%；原始 observed 21.455%
whole scalar−shadow scalar：Σ 81,920 cycles；逐核 最小 2,560；最大 2,560		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 68.159%
参考 Non-scalar / 参考 Phase total 31.841%
参考 Phase total / 原始 whole total 7.893%；原始 observed 29.670%
参考 Phase scalar / 原始 whole scalar 6.366%；原始 observed 22.790%
whole scalar−shadow scalar：Σ 163,840 cycles；逐核 最小 2,560；最大 2,560
Request（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 9,735,102.000
原始 observed 16,604,720 − 记录代码开销估算 6,869,618.000
参考值 / 原始整窗 10.592%；原始 observed / 原始整窗 18.066%
记录代码开销估算 / 原始 observed 41.371%
原始逐核 最小 169,690；最大 174,741；原始整窗 91,911,048
原始 capture gap +0；加 gap 后 16,604,720（18.066%）		扣除记录代码开销估算后的参考值 3,225,831.000
原始 observed 5,447,983 − 记录代码开销估算 2,222,152.000
参考值 / 原始整窗 10.333%；原始 observed / 原始整窗 17.452%
记录代码开销估算 / 原始 observed 40.789%
原始逐核 最小 169,690；最大 170,777；原始整窗 31,217,764
原始 capture gap +0；加 gap 后 5,447,983（17.452%）		扣除记录代码开销估算后的参考值 6,509,271.000
原始 observed 11,156,737 − 记录代码开销估算 4,647,466.000
参考值 / 原始整窗 10.725%；原始 observed / 原始整窗 18.382%
记录代码开销估算 / 原始 observed 41.656%
原始逐核 最小 173,850；最大 174,741；原始整窗 60,693,284
原始 capture gap +0；加 gap 后 11,156,737（18.382%）
Miss（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 354,547.000
原始 observed 589,413 − 记录代码开销估算 234,866.000
参考值 / 原始整窗 9.946%；原始 observed / 原始整窗 16.534%
记录代码开销估算 / 原始 observed 39.847%
原始逐核 最小 1,332；最大 8,752；原始整窗 3,564,754
原始 capture gap +0；加 gap 后 589,413（16.534%）		扣除记录代码开销估算后的参考值 45,847.000
原始 observed 48,336 − 记录代码开销估算 2,489.000
参考值 / 原始整窗 21.136%；原始 observed / 原始整窗 22.283%
记录代码开销估算 / 原始 observed 5.149%
原始逐核 最小 1,332；最大 1,848；原始整窗 216,917
原始 capture gap +0；加 gap 后 48,336（22.283%）		扣除记录代码开销估算后的参考值 308,700.000
原始 observed 541,077 − 记录代码开销估算 232,377.000
参考值 / 原始整窗 9.221%；原始 observed / 原始整窗 16.162%
记录代码开销估算 / 原始 observed 42.947%
原始逐核 最小 8,252；最大 8,752；原始整窗 3,347,837
原始 capture gap +0；加 gap 后 541,077（16.162%）
SYS 边界诊断 / Begin-EndΣ 24,072,315 raw ticks
逐核 最小 213,222；最大 273,643；仅边界诊断
Begin / End：122,880 / 122,880
业务调用 122,880 次；排除 linked Kernel 调用 0 次		Σ 7,117,255 raw ticks
逐核 最小 213,222；最大 233,149；仅边界诊断
Begin / End：40,960 / 40,960
业务调用 40,960 次；排除 linked Kernel 调用 0 次		Σ 16,955,060 raw ticks
逐核 最小 258,907；最大 273,643；仅边界诊断
Begin / End：81,920 / 81,920
业务调用 81,920 次；排除 linked Kernel 调用 0 次
+
+ + +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
ef632e23946261496a882837663ea0694efdf0bcc8b02e5bd26bca6bf58083f0
+
Profile / extra cache
submit-pmu-arg-build / + 8d3a86ebd090fe15
+
Profiled cache key
TestPagedAttentionUnroll / a5 / fully_distributed_within_core / submit-pmu-arg-build
+
Source state
source-v2:8c6b6e3a412d2c7c7113c2d8f714a14eb3786f80:d178dbac61e3fb229a41bc3694ba3c6b213f534c00fd395576cad7f369c6020d:c18fc5b58c9ffc5aba594c7924eb3df43af142e101b29f4ce65f8ea52f2b72bd
+
Source-state stamp
/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/8d3a86ebd090fe15/aicore/.git_commit
+
Compile definitions
PTO_FDWIC_SUBMIT_PMU=1
PTO_FDWIC_SUBMIT_PMU_PHASE_ID=1
PTO_FDWIC_TRACE_ENABLED=0
+
+
+ + + +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
AICore final2,676,056cefb0e3bbfb7e4dc92e874a2982b7425a94ff28c4b29ccd1ae5b0a5c922cc948213,5848f74713abd93a80ba6d120d7d08a58b2deabb96f902e281e2c59f7d2024a2557/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/aicore-extra/8d3a86ebd090fe15/aicore_kernel.o
AIC combined2,003,5128d6f664f8acaeb88858cbf02036e1e0382ae816830aa2390fa40fd7cda6c3d6c99,68876c93d59f5f45ee8743f6c857834c40c8a36f9af2086d0ab58a11a86a55019b2/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/8d3a86ebd090fe15/aicore/aicore_aic_combined.o
AIV combined2,265,1843a48256da6364a8151586feca58f5d372f1faf2db5c2cf1b6eb8dcaa8b6ff25c113,744adb36410d20c3dc22be5c9487156f0593fcaafc2659d0819102c5c7c7618d31d/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/8d3a86ebd090fe15/aicore/aicore_aiv_combined.o
Host runtime11,776,1440369d0602054bf612f86b58c294503e00486bd505bff9ee7a455994f043147cc454,6325ac381ed10be5f5d492b8072c051a177131f4caec6d6f6bc9bf50880da4bb0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so
+
+ +
+
原始全局 Submit 时间范围（墙钟） + 5,982.462 µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + 128096 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-2563486d21ca6390be8444a0be39e4b7e35f8e63d314997a2a88a32885d5cba3703
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + 90.000 ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×90 ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
+
+

ALL · 96 核

+
+
Submit PMU total/core
+
均值 9,025,516.8；最小 7,805,128；最大 9,657,322 cycles
+ 等效时间 均值 5,470.527；最小 4,730.828；最大 5,853.476 µs + （按 1.649844 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,965,713.6；最小 7,552,647；最大 8,520,574 cycles
+ 等效时间 均值 4,828.162；最小 4,577.795；最大 5,164.473 µs； + 加权占比 88.258% +
+
非 Scalar-busy 残余/core
+
均值 1,059,803.1；最小 236,956；最大 1,635,256 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 642.366；最小 143.623；最大 991.158 µs +
+
SYS gate 边界诊断/core
+
均值 5,038,735.7；最小 4,323,038；最大 5,444,064 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,804.474；最小 5,424.387；最大 5,979.850 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 765,738.4；最小 478,985；最大 1,315,639 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 86.808% +
+
Primary I-cache request/core
最小 929,826；最大 1,033,329
+
Primary I-cache miss/core
最小 5,854；最大 56,383
+
加权 miss rate
3.878%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 526.860；最大 5,074.470 µs
+
+
+ +
+

AIC · 32 核

+
+
Submit PMU total/core
+
均值 8,211,182.5；最小 7,805,128；最大 8,793,615 cycles
+ 等效时间 均值 4,976.287；最小 4,730.203；最大 5,329.263 µs + （按 1.650062 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,954,366.9；最小 7,552,647；最大 8,520,574 cycles
+ 等效时间 均值 4,820.647；最小 4,577.190；最大 5,163.790 µs； + 加权占比 96.872% +
+
非 Scalar-busy 残余/core
+
均值 256,815.6；最小 236,956；最大 278,593 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 155.640；最小 143.604；最大 168.838 µs +
+
SYS gate 边界诊断/core
+
均值 4,474,032.2；最小 4,323,038；最大 4,688,175 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,534.945；最小 5,424.387；最大 5,838.848 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 1,060,913.2；最小 894,963；最大 1,315,639 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 80.832% +
+
Primary I-cache request/core
最小 929,826；最大 1,033,329
+
Primary I-cache miss/core
最小 5,854；最大 8,244
+
加权 miss rate
0.695%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 526.860；最大 741.960 µs
+
+
+ +
+

AIV · 64 核

+
+
Submit PMU total/core
+
均值 9,432,683.9；最小 9,177,483；最大 9,657,322 cycles
+ 等效时间 均值 5,717.710；最小 5,563.018；最大 5,853.877 µs + （按 1.649731 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,971,387.0；最小 7,722,707；最大 8,184,267 cycles
+ 等效时间 均值 4,831.931；最小 4,681.192；最大 4,960.971 µs； + 加权占比 84.508% +
+
非 Scalar-busy 残余/core
+
均值 1,461,296.9；最小 1,305,114；最大 1,635,256 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 885.779；最小 791.107；最大 991.226 µs +
+
SYS gate 边界诊断/core
+
均值 5,321,087.5；最小 5,184,633；最大 5,444,064 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,939.238；最小 5,910.854；最大 5,979.850 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 618,151.0；最小 478,985；最大 765,087 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 89.592% +
+
Primary I-cache request/core
最小 931,533；最大 976,189
+
Primary I-cache miss/core
最小 48,123；最大 56,383
+
加权 miss rate
5.516%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 4,331.070；最大 5,074.470 µs
+
+
+
+

逐物理核分布

+

AICAIV

+
+
+
SYS gate boundary-diagnostic ticks/core
+ + + + 5,444,064 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_submit_elapsed_ticks=4505943physical=1 logical=1 role=AIC scalar_submit_elapsed_ticks=4323038physical=2 logical=2 role=AIC scalar_submit_elapsed_ticks=4521410physical=3 logical=3 role=AIC scalar_submit_elapsed_ticks=4405183physical=4 logical=4 role=AIC scalar_submit_elapsed_ticks=4444041physical=5 logical=5 role=AIC scalar_submit_elapsed_ticks=4350560physical=6 logical=6 role=AIC scalar_submit_elapsed_ticks=4493495physical=7 logical=7 role=AIC scalar_submit_elapsed_ticks=4385234physical=8 logical=8 role=AIC scalar_submit_elapsed_ticks=4463090physical=9 logical=9 role=AIC scalar_submit_elapsed_ticks=4523209physical=10 logical=10 role=AIC scalar_submit_elapsed_ticks=4520787physical=12 logical=11 role=AIC scalar_submit_elapsed_ticks=4403227physical=13 logical=12 role=AIC scalar_submit_elapsed_ticks=4688175physical=14 logical=13 role=AIC scalar_submit_elapsed_ticks=4467108physical=15 logical=14 role=AIC scalar_submit_elapsed_ticks=4658046physical=16 logical=15 role=AIC scalar_submit_elapsed_ticks=4401609physical=54 logical=16 role=AIC scalar_submit_elapsed_ticks=4516866physical=55 logical=17 role=AIC scalar_submit_elapsed_ticks=4346844physical=56 logical=18 role=AIC scalar_submit_elapsed_ticks=4538257physical=57 logical=19 role=AIC scalar_submit_elapsed_ticks=4345334physical=58 logical=20 role=AIC scalar_submit_elapsed_ticks=4560574physical=60 logical=21 role=AIC scalar_submit_elapsed_ticks=4493018physical=61 logical=22 role=AIC scalar_submit_elapsed_ticks=4495102physical=62 logical=23 role=AIC scalar_submit_elapsed_ticks=4385114physical=63 logical=24 role=AIC scalar_submit_elapsed_ticks=4479187physical=64 logical=25 role=AIC scalar_submit_elapsed_ticks=4382189physical=65 logical=26 role=AIC scalar_submit_elapsed_ticks=4483201physical=66 logical=27 role=AIC scalar_submit_elapsed_ticks=4450129physical=67 logical=28 role=AIC scalar_submit_elapsed_ticks=4509085physical=68 logical=29 role=AIC scalar_submit_elapsed_ticks=4634132physical=69 logical=30 role=AIC scalar_submit_elapsed_ticks=4660881physical=70 logical=31 role=AIC scalar_submit_elapsed_ticks=4334962physical=18 logical=32 role=AIV scalar_submit_elapsed_ticks=5443070physical=19 logical=33 role=AIV scalar_submit_elapsed_ticks=5385206physical=20 logical=34 role=AIV scalar_submit_elapsed_ticks=5380071physical=21 logical=35 role=AIV scalar_submit_elapsed_ticks=5341068physical=22 logical=36 role=AIV scalar_submit_elapsed_ticks=5320611physical=23 logical=37 role=AIV scalar_submit_elapsed_ticks=5231260physical=24 logical=38 role=AIV scalar_submit_elapsed_ticks=5422992physical=25 logical=39 role=AIV scalar_submit_elapsed_ticks=5277902physical=26 logical=40 role=AIV scalar_submit_elapsed_ticks=5314799physical=27 logical=41 role=AIV scalar_submit_elapsed_ticks=5207771physical=28 logical=42 role=AIV scalar_submit_elapsed_ticks=5332734physical=29 logical=43 role=AIV scalar_submit_elapsed_ticks=5218942physical=30 logical=44 role=AIV scalar_submit_elapsed_ticks=5378024physical=31 logical=45 role=AIV scalar_submit_elapsed_ticks=5282325physical=32 logical=46 role=AIV scalar_submit_elapsed_ticks=5415685physical=33 logical=47 role=AIV scalar_submit_elapsed_ticks=5355910physical=34 logical=48 role=AIV scalar_submit_elapsed_ticks=5341294physical=35 logical=49 role=AIV scalar_submit_elapsed_ticks=5184633physical=36 logical=50 role=AIV scalar_submit_elapsed_ticks=5346096physical=37 logical=51 role=AIV scalar_submit_elapsed_ticks=5363213physical=38 logical=52 role=AIV scalar_submit_elapsed_ticks=5347246physical=39 logical=53 role=AIV scalar_submit_elapsed_ticks=5262625physical=42 logical=54 role=AIV scalar_submit_elapsed_ticks=5347813physical=43 logical=55 role=AIV scalar_submit_elapsed_ticks=5337877physical=44 logical=56 role=AIV scalar_submit_elapsed_ticks=5430240physical=45 logical=57 role=AIV scalar_submit_elapsed_ticks=5305890physical=46 logical=58 role=AIV scalar_submit_elapsed_ticks=5319365physical=47 logical=59 role=AIV scalar_submit_elapsed_ticks=5289469physical=48 logical=60 role=AIV scalar_submit_elapsed_ticks=5326151physical=49 logical=61 role=AIV scalar_submit_elapsed_ticks=5303970physical=50 logical=62 role=AIV scalar_submit_elapsed_ticks=5369067physical=51 logical=63 role=AIV scalar_submit_elapsed_ticks=5359799physical=72 logical=64 role=AIV scalar_submit_elapsed_ticks=5270047physical=73 logical=65 role=AIV scalar_submit_elapsed_ticks=5190417physical=74 logical=66 role=AIV scalar_submit_elapsed_ticks=5337238physical=75 logical=67 role=AIV scalar_submit_elapsed_ticks=5199620physical=76 logical=68 role=AIV scalar_submit_elapsed_ticks=5330657physical=77 logical=69 role=AIV scalar_submit_elapsed_ticks=5216914physical=78 logical=70 role=AIV scalar_submit_elapsed_ticks=5343998physical=79 logical=71 role=AIV scalar_submit_elapsed_ticks=5196955physical=80 logical=72 role=AIV scalar_submit_elapsed_ticks=5377684physical=81 logical=73 role=AIV scalar_submit_elapsed_ticks=5319399physical=84 logical=74 role=AIV scalar_submit_elapsed_ticks=5282857physical=85 logical=75 role=AIV scalar_submit_elapsed_ticks=5293880physical=86 logical=76 role=AIV scalar_submit_elapsed_ticks=5318732physical=87 logical=77 role=AIV scalar_submit_elapsed_ticks=5236080physical=88 logical=78 role=AIV scalar_submit_elapsed_ticks=5444064physical=89 logical=79 role=AIV scalar_submit_elapsed_ticks=5309752physical=90 logical=80 role=AIV scalar_submit_elapsed_ticks=5241004physical=91 logical=81 role=AIV scalar_submit_elapsed_ticks=5302962physical=92 logical=82 role=AIV scalar_submit_elapsed_ticks=5413140physical=93 logical=83 role=AIV scalar_submit_elapsed_ticks=5365044physical=94 logical=84 role=AIV scalar_submit_elapsed_ticks=5407740physical=95 logical=85 role=AIV scalar_submit_elapsed_ticks=5267169physical=96 logical=86 role=AIV scalar_submit_elapsed_ticks=5341954physical=97 logical=87 role=AIV scalar_submit_elapsed_ticks=5266204physical=98 logical=88 role=AIV scalar_submit_elapsed_ticks=5239059physical=99 logical=89 role=AIV scalar_submit_elapsed_ticks=5388293physical=100 logical=90 role=AIV scalar_submit_elapsed_ticks=5420302physical=101 logical=91 role=AIV scalar_submit_elapsed_ticks=5435292physical=102 logical=92 role=AIV scalar_submit_elapsed_ticks=5339687physical=103 logical=93 role=AIV scalar_submit_elapsed_ticks=5266936physical=104 logical=94 role=AIV scalar_submit_elapsed_ticks=5334961physical=105 logical=95 role=AIV scalar_submit_elapsed_ticks=5308439 + +
+ +
+
PMU total cycles/core
+ + + + 9,657,322 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC total_cycles=8237432physical=1 logical=1 role=AIC total_cycles=7805128physical=2 logical=2 role=AIC total_cycles=8176346physical=3 logical=3 role=AIC total_cycles=8163237physical=4 logical=4 role=AIC total_cycles=8069041physical=5 logical=5 role=AIC total_cycles=7997667physical=6 logical=6 role=AIC total_cycles=8110251physical=7 logical=7 role=AIC total_cycles=8104558physical=8 logical=8 role=AIC total_cycles=7980589physical=9 logical=9 role=AIC total_cycles=8536310physical=10 logical=10 role=AIC total_cycles=8226080physical=12 logical=11 role=AIC total_cycles=8020119physical=13 logical=12 role=AIC total_cycles=8793615physical=14 logical=13 role=AIC total_cycles=8431896physical=15 logical=14 role=AIC total_cycles=8535897physical=16 logical=15 role=AIC total_cycles=8126109physical=54 logical=16 role=AIC total_cycles=8131916physical=55 logical=17 role=AIC total_cycles=7882063physical=56 logical=18 role=AIC total_cycles=8238374physical=57 logical=19 role=AIC total_cycles=7997681physical=58 logical=20 role=AIC total_cycles=8426426physical=60 logical=21 role=AIC total_cycles=8597411physical=61 logical=22 role=AIC total_cycles=8190907physical=62 logical=23 role=AIC total_cycles=7952361physical=63 logical=24 role=AIC total_cycles=8125931physical=64 logical=25 role=AIC total_cycles=8042420physical=65 logical=26 role=AIC total_cycles=8085215physical=66 logical=27 role=AIC total_cycles=8248525physical=67 logical=28 role=AIC total_cycles=8341036physical=68 logical=29 role=AIC total_cycles=8675832physical=69 logical=30 role=AIC total_cycles=8658662physical=70 logical=31 role=AIC total_cycles=7848805physical=18 logical=32 role=AIV total_cycles=9604588physical=19 logical=33 role=AIV total_cycles=9502376physical=20 logical=34 role=AIV total_cycles=9513583physical=21 logical=35 role=AIV total_cycles=9496802physical=22 logical=36 role=AIV total_cycles=9414723physical=23 logical=37 role=AIV total_cycles=9257384physical=24 logical=38 role=AIV total_cycles=9595893physical=25 logical=39 role=AIV total_cycles=9318991physical=26 logical=40 role=AIV total_cycles=9442259physical=27 logical=41 role=AIV total_cycles=9235863physical=28 logical=42 role=AIV total_cycles=9482364physical=29 logical=43 role=AIV total_cycles=9383031physical=30 logical=44 role=AIV total_cycles=9490295physical=31 logical=45 role=AIV total_cycles=9393933physical=32 logical=46 role=AIV total_cycles=9603672physical=33 logical=47 role=AIV total_cycles=9497166physical=34 logical=48 role=AIV total_cycles=9410515physical=35 logical=49 role=AIV total_cycles=9231523physical=36 logical=50 role=AIV total_cycles=9485865physical=37 logical=51 role=AIV total_cycles=9558407physical=38 logical=52 role=AIV total_cycles=9512151physical=39 logical=53 role=AIV total_cycles=9397867physical=42 logical=54 role=AIV total_cycles=9417725physical=43 logical=55 role=AIV total_cycles=9402590physical=44 logical=56 role=AIV total_cycles=9637266physical=45 logical=57 role=AIV total_cycles=9382529physical=46 logical=58 role=AIV total_cycles=9476378physical=47 logical=59 role=AIV total_cycles=9392703physical=48 logical=60 role=AIV total_cycles=9457337physical=49 logical=61 role=AIV total_cycles=9437800physical=50 logical=62 role=AIV total_cycles=9573981physical=51 logical=63 role=AIV total_cycles=9564232physical=72 logical=64 role=AIV total_cycles=9324499physical=73 logical=65 role=AIV total_cycles=9204388physical=74 logical=66 role=AIV total_cycles=9463730physical=75 logical=67 role=AIV total_cycles=9218936physical=76 logical=68 role=AIV total_cycles=9420722physical=77 logical=69 role=AIV total_cycles=9264001physical=78 logical=70 role=AIV total_cycles=9461790physical=79 logical=71 role=AIV total_cycles=9177483physical=80 logical=72 role=AIV total_cycles=9515745physical=81 logical=73 role=AIV total_cycles=9474002physical=84 logical=74 role=AIV total_cycles=9317095physical=85 logical=75 role=AIV total_cycles=9389460physical=86 logical=76 role=AIV total_cycles=9405821physical=87 logical=77 role=AIV total_cycles=9306622physical=88 logical=78 role=AIV total_cycles=9585612physical=89 logical=79 role=AIV total_cycles=9452491physical=90 logical=80 role=AIV total_cycles=9246289physical=91 logical=81 role=AIV total_cycles=9492341physical=92 logical=82 role=AIV total_cycles=9522431physical=93 logical=83 role=AIV total_cycles=9531182physical=94 logical=84 role=AIV total_cycles=9574761physical=95 logical=85 role=AIV total_cycles=9296979physical=96 logical=86 role=AIV total_cycles=9504272physical=97 logical=87 role=AIV total_cycles=9421768physical=98 logical=88 role=AIV total_cycles=9285754physical=99 logical=89 role=AIV total_cycles=9516312physical=100 logical=90 role=AIV total_cycles=9549783physical=101 logical=91 role=AIV total_cycles=9657322physical=102 logical=92 role=AIV total_cycles=9438476physical=103 logical=93 role=AIV total_cycles=9262849physical=104 logical=94 role=AIV total_cycles=9420884physical=105 logical=95 role=AIV total_cycles=9416179 + +
+ +
+
Scalar busy cycles/core
+ + + + 8,520,574 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_busy=7989539physical=1 logical=1 role=AIC scalar_busy=7552647physical=2 logical=2 role=AIC scalar_busy=7929502physical=3 logical=3 role=AIC scalar_busy=7912372physical=4 logical=4 role=AIC scalar_busy=7812414physical=5 logical=5 role=AIC scalar_busy=7731863physical=6 logical=6 role=AIC scalar_busy=7869773physical=7 logical=7 role=AIC scalar_busy=7849946physical=8 logical=8 role=AIC scalar_busy=7732301physical=9 logical=9 role=AIC scalar_busy=8273179physical=10 logical=10 role=AIC scalar_busy=7977270physical=12 logical=11 role=AIC scalar_busy=7763960physical=13 logical=12 role=AIC scalar_busy=8520574physical=14 logical=13 role=AIC scalar_busy=8154822physical=15 logical=14 role=AIC scalar_busy=8269452physical=16 logical=15 role=AIC scalar_busy=7870137physical=54 logical=16 role=AIC scalar_busy=7856222physical=55 logical=17 role=AIC scalar_busy=7612139physical=56 logical=18 role=AIC scalar_busy=7978698physical=57 logical=19 role=AIC scalar_busy=7727641physical=58 logical=20 role=AIC scalar_busy=8147833physical=60 logical=21 role=AIC scalar_busy=8346299physical=61 logical=22 role=AIC scalar_busy=7941074physical=62 logical=23 role=AIC scalar_busy=7696627physical=63 logical=24 role=AIC scalar_busy=7888975physical=64 logical=25 role=AIC scalar_busy=7784352physical=65 logical=26 role=AIC scalar_busy=7842899physical=66 logical=27 role=AIC scalar_busy=8003583physical=67 logical=28 role=AIC scalar_busy=8084210physical=68 logical=29 role=AIC scalar_busy=8412874physical=69 logical=30 role=AIC scalar_busy=8394937physical=70 logical=31 role=AIC scalar_busy=7611626physical=18 logical=32 role=AIV scalar_busy=8062291physical=19 logical=33 role=AIV scalar_busy=7924430physical=20 logical=34 role=AIV scalar_busy=8029670physical=21 logical=35 role=AIV scalar_busy=7861546physical=22 logical=36 role=AIV scalar_busy=8018039physical=23 logical=37 role=AIV scalar_busy=7835739physical=24 logical=38 role=AIV scalar_busy=8089447physical=25 logical=39 role=AIV scalar_busy=7782083physical=26 logical=40 role=AIV scalar_busy=8104593physical=27 logical=41 role=AIV scalar_busy=7815191physical=28 logical=42 role=AIV scalar_busy=8127162physical=29 logical=43 role=AIV scalar_busy=7956372physical=30 logical=44 role=AIV scalar_busy=8049794physical=31 logical=45 role=AIV scalar_busy=7879447physical=32 logical=46 role=AIV scalar_busy=8095421physical=33 logical=47 role=AIV scalar_busy=7897918physical=34 logical=48 role=AIV scalar_busy=8100909physical=35 logical=49 role=AIV scalar_busy=7855235physical=36 logical=50 role=AIV scalar_busy=8014081physical=37 logical=51 role=AIV scalar_busy=7971048physical=38 logical=52 role=AIV scalar_busy=8184267physical=39 logical=53 role=AIV scalar_busy=7980454physical=42 logical=54 role=AIV scalar_busy=7881613physical=43 logical=55 role=AIV scalar_busy=7790845physical=44 logical=56 role=AIV scalar_busy=8099675physical=45 logical=57 role=AIV scalar_busy=7765510physical=46 logical=58 role=AIV scalar_busy=8038743physical=47 logical=59 role=AIV scalar_busy=7897037physical=48 logical=60 role=AIV scalar_busy=8074179physical=49 logical=61 role=AIV scalar_busy=7964498physical=50 logical=62 role=AIV scalar_busy=8148507physical=51 logical=63 role=AIV scalar_busy=8101943physical=72 logical=64 role=AIV scalar_busy=8019385physical=73 logical=65 role=AIV scalar_busy=7754444physical=74 logical=66 role=AIV scalar_busy=8067096physical=75 logical=67 role=AIV scalar_busy=7831342physical=76 logical=68 role=AIV scalar_busy=8033510physical=77 logical=69 role=AIV scalar_busy=7803667physical=78 logical=70 role=AIV scalar_busy=8077839physical=79 logical=71 role=AIV scalar_busy=7722707physical=80 logical=72 role=AIV scalar_busy=7972606physical=81 logical=73 role=AIV scalar_busy=7841281physical=84 logical=74 role=AIV scalar_busy=8009300physical=85 logical=75 role=AIV scalar_busy=7949074physical=86 logical=76 role=AIV scalar_busy=8082650physical=87 logical=77 role=AIV scalar_busy=7896284physical=88 logical=78 role=AIV scalar_busy=8140966physical=89 logical=79 role=AIV scalar_busy=7921533physical=90 logical=80 role=AIV scalar_busy=7913036physical=91 logical=81 role=AIV scalar_busy=7995157physical=92 logical=82 role=AIV scalar_busy=8051897physical=93 logical=83 role=AIV scalar_busy=7953474physical=94 logical=84 role=AIV scalar_busy=8090884physical=95 logical=85 role=AIV scalar_busy=7799552physical=96 logical=86 role=AIV scalar_busy=8148637physical=97 logical=87 role=AIV scalar_busy=7964400physical=98 logical=88 role=AIV scalar_busy=7933646physical=99 logical=89 role=AIV scalar_busy=8071661physical=100 logical=90 role=AIV scalar_busy=8074210physical=101 logical=91 role=AIV scalar_busy=8034763physical=102 logical=92 role=AIV scalar_busy=8047358physical=103 logical=93 role=AIV scalar_busy=7827322physical=104 logical=94 role=AIV scalar_busy=7928697physical=105 logical=95 role=AIV scalar_busy=7812705 + +
+ +
+
Primary I-cache requests/core
+ + + + 1,033,329 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_requests=957056physical=1 logical=1 role=AIC icache_requests=949494physical=2 logical=2 role=AIC icache_requests=951678physical=3 logical=3 role=AIC icache_requests=988430physical=4 logical=4 role=AIC icache_requests=945892physical=5 logical=5 role=AIC icache_requests=975525physical=6 logical=6 role=AIC icache_requests=947366physical=7 logical=7 role=AIC icache_requests=982981physical=8 logical=8 role=AIC icache_requests=929826physical=9 logical=9 role=AIC icache_requests=1033329physical=10 logical=10 role=AIC icache_requests=948742physical=12 logical=11 role=AIC icache_requests=963741physical=13 logical=12 role=AIC icache_requests=1029412physical=14 logical=13 role=AIC icache_requests=1025649physical=15 logical=14 role=AIC icache_requests=980461physical=16 logical=15 role=AIC icache_requests=977219physical=54 logical=16 role=AIC icache_requests=948748physical=55 logical=17 role=AIC icache_requests=954727physical=56 logical=18 role=AIC icache_requests=956053physical=57 logical=19 role=AIC icache_requests=969763physical=58 logical=20 role=AIC icache_requests=985653physical=60 logical=21 role=AIC icache_requests=1032701physical=61 logical=22 role=AIC icache_requests=959046physical=62 logical=23 role=AIC icache_requests=958398physical=63 logical=24 role=AIC icache_requests=964898physical=64 logical=25 role=AIC icache_requests=978755physical=65 logical=26 role=AIC icache_requests=941395physical=66 logical=27 role=AIC icache_requests=1008836physical=67 logical=28 role=AIC icache_requests=986600physical=68 logical=29 role=AIC icache_requests=1019496physical=69 logical=30 role=AIC icache_requests=1009004physical=70 logical=31 role=AIC icache_requests=956890physical=18 logical=32 role=AIV icache_requests=946153physical=19 logical=33 role=AIV icache_requests=938373physical=20 logical=34 role=AIV icache_requests=943396physical=21 logical=35 role=AIV icache_requests=958004physical=22 logical=36 role=AIV icache_requests=948264physical=23 logical=37 role=AIV icache_requests=939621physical=24 logical=38 role=AIV icache_requests=941492physical=25 logical=39 role=AIV icache_requests=941007physical=26 logical=40 role=AIV icache_requests=943961physical=27 logical=41 role=AIV icache_requests=946160physical=28 logical=42 role=AIV icache_requests=950989physical=29 logical=43 role=AIV icache_requests=966455physical=30 logical=44 role=AIV icache_requests=938247physical=31 logical=45 role=AIV icache_requests=950748physical=32 logical=46 role=AIV icache_requests=948890physical=33 logical=47 role=AIV icache_requests=959766physical=34 logical=48 role=AIV icache_requests=931533physical=35 logical=49 role=AIV icache_requests=952236physical=36 logical=50 role=AIV icache_requests=939551physical=37 logical=51 role=AIV icache_requests=968404physical=38 logical=52 role=AIV icache_requests=950867physical=39 logical=53 role=AIV icache_requests=955620physical=42 logical=54 role=AIV icache_requests=932130physical=43 logical=55 role=AIV icache_requests=939145physical=44 logical=56 role=AIV icache_requests=949375physical=45 logical=57 role=AIV icache_requests=938219physical=46 logical=58 role=AIV icache_requests=962011physical=47 logical=59 role=AIV icache_requests=950891physical=48 logical=60 role=AIV icache_requests=952829physical=49 logical=61 role=AIV icache_requests=960266physical=50 logical=62 role=AIV icache_requests=955800physical=51 logical=63 role=AIV icache_requests=960321physical=72 logical=64 role=AIV icache_requests=947068physical=73 logical=65 role=AIV icache_requests=949417physical=74 logical=66 role=AIV icache_requests=944619physical=75 logical=67 role=AIV icache_requests=942161physical=76 logical=68 role=AIV icache_requests=935649physical=77 logical=69 role=AIV icache_requests=947102physical=78 logical=70 role=AIV icache_requests=949374physical=79 logical=71 role=AIV icache_requests=934976physical=80 logical=72 role=AIV icache_requests=942407physical=81 logical=73 role=AIV icache_requests=958564physical=84 logical=74 role=AIV icache_requests=939248physical=85 logical=75 role=AIV icache_requests=952763physical=86 logical=76 role=AIV icache_requests=943637physical=87 logical=77 role=AIV icache_requests=949089physical=88 logical=78 role=AIV icache_requests=942751physical=89 logical=79 role=AIV icache_requests=957250physical=90 logical=80 role=AIV icache_requests=937637physical=91 logical=81 role=AIV icache_requests=976189physical=92 logical=82 role=AIV icache_requests=937057physical=93 logical=83 role=AIV icache_requests=957240physical=94 logical=84 role=AIV icache_requests=946645physical=95 logical=85 role=AIV icache_requests=936744physical=96 logical=86 role=AIV icache_requests=964862physical=97 logical=87 role=AIV icache_requests=967439physical=98 logical=88 role=AIV icache_requests=949935physical=99 logical=89 role=AIV icache_requests=941895physical=100 logical=90 role=AIV icache_requests=942499physical=101 logical=91 role=AIV icache_requests=966012physical=102 logical=92 role=AIV icache_requests=946764physical=103 logical=93 role=AIV icache_requests=936431physical=104 logical=94 role=AIV icache_requests=936435physical=105 logical=95 role=AIV icache_requests=952701 + +
+ +
+
Primary I-cache misses/core
+ + + + 56,383 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_misses=6931physical=1 logical=1 role=AIC icache_misses=7530physical=2 logical=2 role=AIC icache_misses=6511physical=3 logical=3 role=AIC icache_misses=6773physical=4 logical=4 role=AIC icache_misses=7198physical=5 logical=5 role=AIC icache_misses=6800physical=6 logical=6 role=AIC icache_misses=6527physical=7 logical=7 role=AIC icache_misses=6279physical=8 logical=8 role=AIC icache_misses=6938physical=9 logical=9 role=AIC icache_misses=6933physical=10 logical=10 role=AIC icache_misses=7138physical=12 logical=11 role=AIC icache_misses=6053physical=13 logical=12 role=AIC icache_misses=6859physical=14 logical=13 role=AIC icache_misses=6554physical=15 logical=14 role=AIC icache_misses=6275physical=16 logical=15 role=AIC icache_misses=6046physical=54 logical=16 role=AIC icache_misses=8244physical=55 logical=17 role=AIC icache_misses=7067physical=56 logical=18 role=AIC icache_misses=6439physical=57 logical=19 role=AIC icache_misses=6892physical=58 logical=20 role=AIC icache_misses=6825physical=60 logical=21 role=AIC icache_misses=6046physical=61 logical=22 role=AIC icache_misses=6751physical=62 logical=23 role=AIC icache_misses=6436physical=63 logical=24 role=AIC icache_misses=6173physical=64 logical=25 role=AIC icache_misses=7184physical=65 logical=26 role=AIC icache_misses=6887physical=66 logical=27 role=AIC icache_misses=5854physical=67 logical=28 role=AIC icache_misses=6637physical=68 logical=29 role=AIC icache_misses=7215physical=69 logical=30 role=AIC icache_misses=7961physical=70 logical=31 role=AIC icache_misses=6961physical=18 logical=32 role=AIV icache_misses=54306physical=19 logical=33 role=AIV icache_misses=52696physical=20 logical=34 role=AIV icache_misses=53970physical=21 logical=35 role=AIV icache_misses=55330physical=22 logical=36 role=AIV icache_misses=50554physical=23 logical=37 role=AIV icache_misses=50024physical=24 logical=38 role=AIV icache_misses=53930physical=25 logical=39 role=AIV icache_misses=53587physical=26 logical=40 role=AIV icache_misses=51100physical=27 logical=41 role=AIV icache_misses=50433physical=28 logical=42 role=AIV icache_misses=50803physical=29 logical=43 role=AIV icache_misses=52475physical=30 logical=44 role=AIV icache_misses=52440physical=31 logical=45 role=AIV icache_misses=54445physical=32 logical=46 role=AIV icache_misses=55527physical=33 logical=47 role=AIV icache_misses=55426physical=34 logical=48 role=AIV icache_misses=48123physical=35 logical=49 role=AIV icache_misses=50897physical=36 logical=50 role=AIV icache_misses=53612physical=37 logical=51 role=AIV icache_misses=55847physical=38 logical=52 role=AIV icache_misses=50663physical=39 logical=53 role=AIV icache_misses=51428physical=42 logical=54 role=AIV icache_misses=52868physical=43 logical=55 role=AIV icache_misses=53908physical=44 logical=56 role=AIV icache_misses=55168physical=45 logical=57 role=AIV icache_misses=53393physical=46 logical=58 role=AIV icache_misses=52412physical=47 logical=59 role=AIV icache_misses=51731physical=48 logical=60 role=AIV icache_misses=51248physical=49 logical=61 role=AIV icache_misses=51829physical=50 logical=62 role=AIV icache_misses=52288physical=51 logical=63 role=AIV icache_misses=51687physical=72 logical=64 role=AIV icache_misses=50218physical=73 logical=65 role=AIV icache_misses=50883physical=74 logical=66 role=AIV icache_misses=50402physical=75 logical=67 role=AIV icache_misses=49311physical=76 logical=68 role=AIV icache_misses=49693physical=77 logical=69 role=AIV icache_misses=50155physical=78 logical=70 role=AIV icache_misses=50244physical=79 logical=71 role=AIV icache_misses=49656physical=80 logical=72 role=AIV icache_misses=53676physical=81 logical=73 role=AIV icache_misses=55719physical=84 logical=74 role=AIV icache_misses=49064physical=85 logical=75 role=AIV icache_misses=50948physical=86 logical=76 role=AIV icache_misses=50019physical=87 logical=77 role=AIV icache_misses=50664physical=88 logical=78 role=AIV icache_misses=53600physical=89 logical=79 role=AIV icache_misses=55164physical=90 logical=80 role=AIV icache_misses=49817physical=91 logical=81 role=AIV icache_misses=54071physical=92 logical=82 role=AIV icache_misses=52448physical=93 logical=83 role=AIV icache_misses=55922physical=94 logical=84 role=AIV icache_misses=54546physical=95 logical=85 role=AIV icache_misses=53162physical=96 logical=86 role=AIV icache_misses=52054physical=97 logical=87 role=AIV icache_misses=52841physical=98 logical=88 role=AIV icache_misses=50107physical=99 logical=89 role=AIV icache_misses=50670physical=100 logical=90 role=AIV icache_misses=53460physical=101 logical=91 role=AIV icache_misses=56383physical=102 logical=92 role=AIV icache_misses=51149physical=103 logical=93 role=AIV icache_misses=49168physical=104 logical=94 role=AIV icache_misses=53215physical=105 logical=95 role=AIV icache_misses=55260 + +
+
+

逐核原始主计数

+

报告展示 primary request/miss；phase 模式的 I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和，只用于标记全窗 capture gap；scalar shadow 同样用于核验 phase 读清重建，并显示其相对 CNT2 primary 的 capture loss，不作为第二份性能数据展示。

+
+ + + + + + + +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate90ns量尺 µs
00AIC004,505,9435,468.6338,237,4327,989,539957,0566,9310.724%623.790
11AIC104,323,0385,428.3267,805,1287,552,647949,4947,5300.793%677.700
22AIC204,521,4105,455.3338,176,3467,929,502951,6786,5110.684%585.990
33AIC304,405,1835,528.8758,163,2377,912,372988,4306,7730.685%609.570
44AIC404,444,0415,425.5018,069,0417,812,414945,8927,1980.761%647.820
55AIC504,350,5605,467.2577,997,6677,731,863975,5256,8000.697%612.000
66AIC604,493,4955,432.9438,110,2517,869,773947,3666,5270.689%587.430
77AIC704,385,2345,464.5458,104,5587,849,946982,9816,2790.639%565.110
88AIC804,463,0905,429.1827,980,5897,732,301929,8266,9380.746%624.420
99AIC904,523,2095,838.8488,536,3108,273,1791,033,3296,9330.671%623.970
1010AIC1004,520,7875,463.2668,226,0807,977,270948,7427,1380.752%642.420
1211AIC1104,403,2275,437.8278,020,1197,763,960963,7416,0530.628%544.770
1312AIC1204,688,1755,826.7628,793,6158,520,5741,029,4126,8590.666%617.310
1413AIC1304,467,1085,668.1858,431,8968,154,8221,025,6496,5540.639%589.860
1514AIC1404,658,0465,691.7148,535,8978,269,452980,4616,2750.640%564.750
1615AIC1504,401,6095,470.6558,126,1097,870,137977,2196,0460.619%544.140
1832AIV015,443,0705,936.3139,604,5888,062,291946,15354,3065.740%4,887.540
1933AIV025,385,2065,925.0419,502,3767,924,430938,37352,6965.616%4,742.640
2034AIV115,380,0715,933.1759,513,5838,029,670943,39653,9705.721%4,857.300
2135AIV125,341,0685,926.7629,496,8027,861,546958,00455,3305.776%4,979.700
2236AIV215,320,6115,979.1949,414,7238,018,039948,26450,5545.331%4,549.860
2337AIV225,231,2605,937.2449,257,3847,835,739939,62150,0245.324%4,502.160
2438AIV315,422,9925,931.9399,595,8938,089,447941,49253,9305.728%4,853.700
2539AIV325,277,9025,922.0519,318,9917,782,083941,00753,5875.695%4,822.830
2640AIV415,314,7995,949.0879,442,2598,104,593943,96151,1005.413%4,599.000
2741AIV425,207,7715,929.6299,235,8637,815,191946,16050,4335.330%4,538.970
2842AIV515,332,7345,914.2889,482,3648,127,162950,98950,8035.342%4,572.270
2943AIV525,218,9425,964.5249,383,0317,956,372966,45552,4755.430%4,722.750
3044AIV615,378,0245,969.0809,490,2958,049,794938,24752,4405.589%4,719.600
3145AIV625,282,3255,966.3879,393,9337,879,447950,74854,4455.727%4,900.050
3246AIV715,415,6855,939.5269,603,6728,095,421948,89055,5275.852%4,997.430
3347AIV725,355,9105,928.8219,497,1667,897,918959,76655,4265.775%4,988.340
3448AIV815,341,2945,919.5759,410,5158,100,909931,53348,1235.166%4,331.070
3549AIV825,184,6335,921.9219,231,5237,855,235952,23650,8975.345%4,580.730
3650AIV915,346,0965,917.3459,485,8658,014,081939,55153,6125.706%4,825.080
3751AIV925,363,2135,966.2289,558,4077,971,048968,40455,8475.767%5,026.230
3852AIV1015,347,2465,934.8069,512,1518,184,267950,86750,6635.328%4,559.670
3953AIV1025,262,6255,920.0659,397,8677,980,454955,62051,4285.382%4,628.520
4254AIV1115,347,8135,929.9669,417,7257,881,613932,13052,8685.672%4,758.120
4355AIV1125,337,8775,924.7539,402,5907,790,845939,14553,9085.740%4,851.720
4456AIV1215,430,2405,958.9339,637,2668,099,675949,37555,1685.811%4,965.120
4557AIV1225,305,8905,910.8549,382,5297,765,510938,21953,3935.691%4,805.370
4658AIV1315,319,3655,920.4529,476,3788,038,743962,01152,4125.448%4,717.080
4759AIV1325,289,4695,921.8749,392,7037,897,037950,89151,7315.440%4,655.790
4860AIV1415,326,1515,956.4029,457,3378,074,179952,82951,2485.379%4,612.320
4961AIV1425,303,9705,945.4789,437,8007,964,498960,26651,8295.397%4,664.610
5062AIV1515,369,0675,921.0289,573,9818,148,507955,80052,2885.471%4,705.920
5163AIV1525,359,7995,916.5909,564,2328,101,943960,32151,6875.382%4,651.830
5416AIC1604,516,8665,433.3048,131,9167,856,222948,7488,2440.869%741.960
5517AIC1704,346,8445,429.2817,882,0637,612,139954,7277,0670.740%636.030
5618AIC1804,538,2575,433.2208,238,3747,978,698956,0536,4390.673%579.510
5719AIC1904,345,3345,476.4187,997,6817,727,641969,7636,8920.711%620.280
5820AIC2004,560,5745,616.0598,426,4268,147,833985,6536,8250.692%614.250
6021AIC2104,493,0185,756.4908,597,4118,346,2991,032,7016,0460.585%544.140
6122AIC2204,495,1025,453.5448,190,9077,941,074959,0466,7510.704%607.590
6223AIC2304,385,1145,427.4707,952,3617,696,627958,3986,4360.672%579.240
6324AIC2404,479,1875,435.6618,125,9317,888,975964,8986,1730.640%555.570
6425AIC2504,382,1895,429.8778,042,4207,784,352978,7557,1840.734%646.560
6526AIC2604,483,2015,471.0838,085,2157,842,899941,3956,8870.732%619.830
6627AIC2704,450,1295,555.6948,248,5258,003,5831,008,8365,8540.580%526.860
6728AIC2804,509,0855,624.8138,341,0368,084,210986,6006,6370.673%597.330
6829AIC2904,634,1325,823.1338,675,8328,412,8741,019,4967,2150.708%649.350
6930AIC3004,660,8815,829.9678,658,6628,394,9371,009,0047,9610.789%716.490
7031AIC3104,334,9625,424.3877,848,8057,611,626956,8906,9610.727%626.490
7264AIV1615,270,0475,922.7789,324,4998,019,385947,06850,2185.302%4,519.620
7365AIV1625,190,4175,955.5049,204,3887,754,444949,41750,8835.359%4,579.470
7466AIV1715,337,2385,957.2609,463,7308,067,096944,61950,4025.336%4,536.180
7567AIV1725,199,6205,914.1439,218,9367,831,342942,16149,3115.234%4,437.990
7668AIV1815,330,6575,979.8509,420,7228,033,510935,64949,6935.311%4,472.370
7769AIV1825,216,9145,938.9879,264,0017,803,667947,10250,1555.296%4,513.950
7870AIV1915,343,9985,956.7379,461,7908,077,839949,37450,2445.292%4,521.960
7971AIV1925,196,9555,938.5709,177,4837,722,707934,97649,6565.311%4,469.040
8072AIV2015,377,6845,933.1179,515,7457,972,606942,40753,6765.696%4,830.840
8173AIV2025,319,3995,967.3239,474,0027,841,281958,56455,7195.813%5,014.710
8474AIV2115,282,8575,917.9969,317,0958,009,300939,24849,0645.224%4,415.760
8575AIV2125,293,8805,914.1359,389,4607,949,074952,76350,9485.347%4,585.320
8676AIV2215,318,7325,973.3879,405,8218,082,650943,63750,0195.301%4,501.710
8777AIV2225,236,0805,915.3249,306,6227,896,284949,08950,6645.338%4,559.760
8878AIV2315,444,0645,923.0499,585,6128,140,966942,75153,6005.685%4,824.000
8979AIV2325,309,7525,951.6189,452,4917,921,533957,25055,1645.763%4,964.760
9080AIV2415,241,0045,929.7689,246,2897,913,036937,63749,8175.313%4,483.530
9181AIV2425,302,9625,931.1249,492,3417,995,157976,18954,0715.539%4,866.390
9282AIV2515,413,1405,935.8889,522,4318,051,897937,05752,4485.597%4,720.320
9383AIV2525,365,0445,947.9379,531,1827,953,474957,24055,9225.842%5,032.980
9484AIV2615,407,7405,969.6489,574,7618,090,884946,64554,5465.762%4,909.140
9585AIV2625,267,1695,956.3569,296,9797,799,552936,74453,1625.675%4,784.580
9686AIV2715,341,9545,934.4099,504,2728,148,637964,86252,0545.395%4,684.860
9787AIV2725,266,2045,940.1019,421,7687,964,400967,43952,8415.462%4,755.690
9888AIV2815,239,0595,957.6579,285,7547,933,646949,93550,1075.275%4,509.630
9989AIV2825,388,2935,939.3069,516,3128,071,661941,89550,6705.380%4,560.300
10090AIV2915,420,3025,953.4049,549,7838,074,210942,49953,4605.672%4,811.400
10191AIV2925,435,2925,973.5259,657,3228,034,763966,01256,3835.837%5,074.470
10292AIV3015,339,6875,944.2409,438,4768,047,358946,76451,1495.403%4,603.410
10393AIV3025,266,9365,939.0169,262,8497,827,322936,43149,1685.251%4,425.120
10494AIV3115,334,9615,929.6599,420,8847,928,697936,43553,2155.683%4,789.350
10595AIV3125,308,4395,930.1159,416,1797,812,705952,70155,2605.800%4,973.400
+
+ diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_claim_20260723_043639_icache_report.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_claim_20260723_043639_icache_report.html new file mode 100644 index 0000000000..d83f1e0cb8 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_claim_20260723_043639_icache_report.html @@ -0,0 +1,362 @@ + + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 fdwic_submit_pmu_raw.json · schema fdwic-submit-pmu-v3 · 32 AIC + 64 AIV

+ +
+

claim 阶段观察（phase_id=4）

+

边界 claim_begin_to_claim_end · 计数语义 running_read_clear_observed_bracket · + 时间语义 boundary_diagnostic_sys_cnt_between_observers · + expected_calls_per_core=1280

+ +

本报告使用 empty-bracket 估算进入 phase observed + 的局部记录代码开销。 + AIC/AIV 分别按“空区间每组记录开销 × 本阶段实际 begin/end 记录组数”计算，ALL + 再由两类核相加。参考值只从 phase observed 分子扣除该估算；原始 whole 分母保持不变， + 因为空区间没有测量 whole 窗口中的全部记录开销。所以下表的参考比例不是完整业务 + 阶段占完整调度窗口的比例，也不是精确校正。若参考值为负，表示记录代码开销估算已经 + 超过原始 observed；这只能说明该信号低于当前校准的分辨能力，不能截成零或解释成负的 + 业务事件。校准 raw：/home/q00473782/atomic/private/gpt/simpler/outputs/TestPagedAttentionUnroll_Case1_20260723_043533/fdwic_submit_pmu_raw.json · + SHA-256 56199f91545cc6a1fbf18b8bb2cfebae279cb5801516311a040d4facccf321ba。两份 provenance 已核验为同一场景、同一 Git revision 8c6b6e3a412d。

+ + +

本阶段若跨过 linked Kernel，会同时门控 SYS 时间与 PMU counter， + 因而 total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段。 + result-used return-ready atomic 依赖区间只从 SYS 边界累计值扣除，PMU counter + 仍含 atomic 指令事件；source-issue atomic + 保留在时间和计数口径内。没有 linked Kernel 的阶段不触发 PMU 停/开表； + 每次调用的分母仍是外层业务调用次数，不是 Begin/End 读数，也不是排除的 Kernel 调用数。

+ +

阶段原始 PMU 观测来自 running read-clear 的 total cycles，并按 AIC/AIV + 各自校准频率换算；提供 empty-bracket 时，页面主参考值再扣除局部记录代码开销估算。 + phase_elapsed_ticks 只是 SYS 边界闭合诊断，绝不按 1 GHz 当作阶段时间。 + Scalar busy 与 non-scalar residual（逐核先算 total−scalar）共同解释阶段 PMU total； + 未校准时只能展示 raw observed；提供 empty-bracket 后，记录代码开销估算只从 phase + observed 分子扣除，原始 whole 分母保持不变。当前数据无法得到完整业务阶段占完整调度 + 窗口的精确比例。 + linked vector/cube Kernel 从 SYS 与 PMU counter 一并门控排除；result-used return-ready atomic + 依赖区间只从 SYS 边界累计值扣除，I-cache/PMU counter 仍含其指令事件；source-issue atomic + 保留。request/miss 百分比的分母是同一 ELF、同一次采集的 Submit 整窗 primary。 + observed_plus_capture_gap = observed + (primary − shadow)； + 边界读数和插桩 bookkeeping 会进入 sample，因此“观测值”和“加全窗 capture gap”都不是原业务 + 事件数的数学上下界。不同 ELF 不能直接相减后冒充业务真值；提供 empty-bracket 时只按上方 + 明示的公式估算局部记录代码开销。request/miss 的逐核 min/max 是每核累计整个 phase + 的极值，不是逐调用极值。

+
+ + + + +
指标ALL96 核AIC32 核AIV64 核
Phase PMU total扣除记录代码开销估算后的参考值 Σ 129,663,593.000 cycles
≈ 78,591.426 µs（1.649844 cycles/ns）
原始 observed Σ 321,951,929 cycles（≈ 195,140.831 µs）− 记录代码开销估算 192,288,336.000 cycles
记录代码开销估算 / 原始 observed 59.726%；原始逐核 最小 2,948,298；最大 3,599,082；原始 2,620.052 cycles/call		扣除记录代码开销估算后的参考值 Σ 34,684,258.000 cycles
≈ 21,019.973 µs（1.650062 cycles/ns）
原始 observed Σ 95,511,061 cycles（≈ 57,883.317 µs）− 记录代码开销估算 60,826,803.000 cycles
记录代码开销估算 / 原始 observed 63.686%；原始逐核 最小 2,948,298；最大 3,023,337；原始 2,331.813 cycles/call		扣除记录代码开销估算后的参考值 Σ 94,979,335.000 cycles
≈ 57,572.619 µs（1.649731 cycles/ns）
原始 observed Σ 226,440,868 cycles（≈ 137,259.267 µs）− 记录代码开销估算 131,461,533.000 cycles
记录代码开销估算 / 原始 observed 58.056%；原始逐核 最小 3,465,659；最大 3,599,082；原始 2,764.171 cycles/call
Phase scalar busy扣除记录代码开销估算后的参考值 Σ 107,038,871.000 cycles
≈ 64,878.177 µs（1.649844 cycles/ns）
原始 observed Σ 230,884,809 cycles（≈ 139,943.418 µs）− 记录代码开销估算 123,845,938.000 cycles
记录代码开销估算 / 原始 observed 53.640%；原始逐核 最小 2,209,220；最大 2,543,276；原始 1,878.945 cycles/call		扣除记录代码开销估算后的参考值 Σ 31,553,722.000 cycles
≈ 19,122.749 µs（1.650062 cycles/ns）
原始 observed Σ 71,613,428 cycles（≈ 43,400.447 µs）− 记录代码开销估算 40,059,706.000 cycles
记录代码开销估算 / 原始 observed 55.939%；原始逐核 最小 2,209,220；最大 2,273,774；原始 1,748.375 cycles/call		扣除记录代码开销估算后的参考值 Σ 75,485,149.000 cycles
≈ 45,756.035 µs（1.649731 cycles/ns）
原始 observed Σ 159,271,381 cycles（≈ 96,543.849 µs）− 记录代码开销估算 83,786,232.000 cycles
记录代码开销估算 / 原始 observed 52.606%；原始逐核 最小 2,443,961；最大 2,543,276；原始 1,944.231 cycles/call
非 Scalar-busy 残余扣除记录代码开销估算后的参考值 Σ 22,624,722.000 cycles
≈ 13,713.249 µs（1.649844 cycles/ns）
原始 observed Σ 91,067,120 cycles（≈ 55,197.413 µs）− 记录代码开销估算 68,442,398.000 cycles
记录代码开销估算 / 原始 observed 75.156%；原始逐核 最小 732,028；最大 1,077,936；原始 741.106 cycles/call		扣除记录代码开销估算后的参考值 Σ 3,130,536.000 cycles
≈ 1,897.223 µs（1.650062 cycles/ns）
原始 observed Σ 23,897,633 cycles（≈ 14,482.870 µs）− 记录代码开销估算 20,767,097.000 cycles
记录代码开销估算 / 原始 observed 86.900%；原始逐核 最小 732,028；最大 764,512；原始 583.438 cycles/call		扣除记录代码开销估算后的参考值 Σ 19,494,186.000 cycles
≈ 11,816.585 µs（1.649731 cycles/ns）
原始 observed Σ 67,169,487 cycles（≈ 40,715.418 µs）− 记录代码开销估算 47,675,301.000 cycles
记录代码开销估算 / 原始 observed 70.978%；原始逐核 最小 1,017,626；最大 1,077,936；原始 819.940 cycles/call
阶段参考关系（raw 同格保留）扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 82.551%
参考 Non-scalar / 参考 Phase total 17.449%
参考 Phase total / 原始 whole total 15.675%；原始 observed 38.921%
参考 Phase scalar / 原始 whole scalar 14.243%；原始 observed 30.723%
whole scalar−shadow scalar：Σ 245,760 cycles；逐核 最小 2,560；最大 2,560		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 90.974%
参考 Non-scalar / 参考 Phase total 9.026%
参考 Phase total / 原始 whole total 13.665%；原始 observed 37.631%
参考 Phase scalar / 原始 whole scalar 12.839%；原始 observed 29.138%
whole scalar−shadow scalar：Σ 81,920 cycles；逐核 最小 2,560；最大 2,560		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 79.475%
参考 Non-scalar / 参考 Phase total 20.525%
参考 Phase total / 原始 whole total 16.565%；原始 observed 39.492%
参考 Phase scalar / 原始 whole scalar 14.926%；原始 observed 31.493%
whole scalar−shadow scalar：Σ 163,840 cycles；逐核 最小 2,560；最大 2,560
Request（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 12,142,513.000
原始 observed 19,012,131 − 记录代码开销估算 6,869,618.000
参考值 / 原始整窗 13.152%；原始 observed / 原始整窗 20.593%
记录代码开销估算 / 原始 observed 36.133%
原始逐核 最小 192,561；最大 200,760；原始整窗 92,323,235
原始 capture gap +0；加 gap 后 19,012,131（20.593%）		扣除记录代码开销估算后的参考值 4,045,913.000
原始 observed 6,268,065 − 记录代码开销估算 2,222,152.000
参考值 / 原始整窗 12.917%；原始 observed / 原始整窗 20.012%
记录代码开销估算 / 原始 observed 35.452%
原始逐核 最小 192,561；最大 198,445；原始整窗 31,322,201
原始 capture gap +0；加 gap 后 6,268,065（20.012%）		扣除记录代码开销估算后的参考值 8,096,600.000
原始 observed 12,744,066 − 记录代码开销估算 4,647,466.000
参考值 / 原始整窗 13.273%；原始 observed / 原始整窗 20.892%
记录代码开销估算 / 原始 observed 36.468%
原始逐核 最小 197,857；最大 200,760；原始整窗 61,001,034
原始 capture gap +0；加 gap 后 12,744,066（20.892%）
Miss（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 594,211.000
原始 observed 829,077 − 记录代码开销估算 234,866.000
参考值 / 原始整窗 18.803%；原始 observed / 原始整窗 26.235%
记录代码开销估算 / 原始 observed 28.329%
原始逐核 最小 2,766；最大 12,116；原始整窗 3,160,254
原始 capture gap +0；加 gap 后 829,077（26.235%）		扣除记录代码开销估算后的参考值 92,345.000
原始 observed 94,834 − 记录代码开销估算 2,489.000
参考值 / 原始整窗 43.518%；原始 observed / 原始整窗 44.691%
记录代码开销估算 / 原始 observed 2.625%
原始逐核 最小 2,766；最大 3,449；原始整窗 212,200
原始 capture gap +0；加 gap 后 94,834（44.691%）		扣除记录代码开销估算后的参考值 501,866.000
原始 observed 734,243 − 记录代码开销估算 232,377.000
参考值 / 原始整窗 17.024%；原始 observed / 原始整窗 24.906%
记录代码开销估算 / 原始 observed 31.649%
原始逐核 最小 11,004；最大 12,116；原始整窗 2,948,054
原始 capture gap +0；加 gap 后 734,243（24.906%）
SYS 边界诊断 / Begin-EndΣ 36,441,184 raw ticks
逐核 最小 280,693；最大 443,056；仅边界诊断
Begin / End：122,880 / 122,880
业务调用 122,880 次；排除 linked Kernel 调用 0 次		Σ 9,212,340 raw ticks
逐核 最小 280,693；最大 298,137；仅边界诊断
Begin / End：40,960 / 40,960
业务调用 40,960 次；排除 linked Kernel 调用 0 次		Σ 27,228,844 raw ticks
逐核 最小 405,735；最大 443,056；仅边界诊断
Begin / End：81,920 / 81,920
业务调用 81,920 次；排除 linked Kernel 调用 0 次
+
+ + +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
5066505fbf1733398f95734491fbb3bec324d3a55cdd94e7d6105849aa46cfe4
+
Profile / extra cache
submit-pmu-claim / + 88d4ef05843d4904
+
Profiled cache key
TestPagedAttentionUnroll / a5 / fully_distributed_within_core / submit-pmu-claim
+
Source state
source-v2:8c6b6e3a412d2c7c7113c2d8f714a14eb3786f80:70877f3e89a97927da294c451930b1df13e129967111a8dc7df65033d8a286bc:3a4b1a4690ba4e8d8822b7542d6703cc542dd48c6dd73f4ea6205712ecce4b4f
+
Source-state stamp
/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/88d4ef05843d4904/aicore/.git_commit
+
Compile definitions
PTO_FDWIC_SUBMIT_PMU=1
PTO_FDWIC_SUBMIT_PMU_PHASE_ID=4
PTO_FDWIC_TRACE_ENABLED=0
+
+
+ + + +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
AICore final2,713,0240538c8653fa70b357e54b0024d0799b71c115b59f521eb916ce83fedf78d7484221,26455a2cde2f971b910e6e29264f041c4808a47a07db09460415f2ca3bdc97267cd/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/aicore-extra/88d4ef05843d4904/aicore_kernel.o
AIC combined2,042,248218be1cb59f1f8bccb47e84900032d06b8551d2564a034713cac399b910eba15103,528e611309c3d6789d97a3691a9aec5cdbbfc9d164501b3c999be8155e66a3ec204/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/88d4ef05843d4904/aicore/aicore_aic_combined.o
AIV combined2,304,7520300b82a2bb9dde65d4289e4906dd0fb7c95dbab5c5e6165da02e4a4e392c20b117,5840aae3c11bbe22e8e6ef06d1fe5884054b05726c900c6a7fced21b9a30bf73f77/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/88d4ef05843d4904/aicore/aicore_aiv_combined.o
Host runtime11,776,1440369d0602054bf612f86b58c294503e00486bd505bff9ee7a455994f043147cc454,6325ac381ed10be5f5d492b8072c051a177131f4caec6d6f6bc9bf50880da4bb0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so
+
+ +
+
原始全局 Submit 时间范围（墙钟） + 5,696.233 µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + 128096 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-25623fbcce1a20c3c359e81a0c4198ca8800730c755ccab697de32fa28187345d3e
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + 90.000 ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×90 ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
+
+

ALL · 96 核

+
+
Submit PMU total/core
+
均值 8,616,542.1；最小 7,772,732；最大 9,124,956 cycles
+ 等效时间 均值 5,222.641；最小 4,711.192；最大 5,530.799 µs + （按 1.649844 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,828,272.2；最小 7,520,926；最大 8,167,364 cycles
+ 等效时间 均值 4,744.856；最小 4,558.568；最大 4,950.386 µs； + 加权占比 90.852% +
+
非 Scalar-busy 残余/core
+
均值 788,269.9；最小 223,023；最大 1,209,184 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 477.785；最小 135.178；最大 732.908 µs +
+
SYS gate 边界诊断/core
+
均值 4,819,584.3；最小 4,305,564；最大 5,147,475 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,555.893；最小 5,336.917；最大 5,696.028 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 736,308.7；最小 497,744；最大 1,066,441 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 86.747% +
+
Primary I-cache request/core
最小 934,228；最大 1,001,034
+
Primary I-cache miss/core
最小 5,936；最大 47,438
+
加权 miss rate
3.423%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 534.240；最大 4,269.420 µs
+
+
+ +
+

AIC · 32 核

+
+
Submit PMU total/core
+
均值 7,931,545.2；最小 7,772,732；最大 8,111,615 cycles
+ 等效时间 均值 4,806.817；最小 4,710.570；最大 4,915.946 µs + （按 1.650062 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,680,352.5；最小 7,520,926；最大 7,888,592 cycles
+ 等效时间 均值 4,654.584；最小 4,557.966；最大 4,780.785 µs； + 加权占比 96.833% +
+
非 Scalar-busy 残余/core
+
均值 251,192.8；最小 223,023；最大 273,424 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 152.232；最小 135.160；最大 165.705 µs +
+
SYS gate 边界诊断/core
+
均值 4,395,841.9；最小 4,305,564；最大 4,521,531 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,363.958；最小 5,336.917；最大 5,409.252 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 968,116.3；最小 847,397；最大 1,066,441 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 81.951% +
+
Primary I-cache request/core
最小 951,535；最大 1,001,034
+
Primary I-cache miss/core
最小 5,936；最大 7,607
+
加权 miss rate
0.677%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 534.240；最大 684.630 µs
+
+
+ +
+

AIV · 64 核

+
+
Submit PMU total/core
+
均值 8,959,040.5；最小 8,705,727；最大 9,124,956 cycles
+ 等效时间 均值 5,430.607；最小 5,277.059；最大 5,531.178 µs + （按 1.649731 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,902,232.1；最小 7,637,221；最大 8,167,364 cycles
+ 等效时间 均值 4,790.012；最小 4,629.374；最大 4,950.725 µs； + 加权占比 88.204% +
+
非 Scalar-busy 残余/core
+
均值 1,056,808.5；最小 906,583；最大 1,209,184 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 640.594；最小 549.534；最大 732.958 µs +
+
SYS gate 边界诊断/core
+
均值 5,031,455.4；最小 4,877,042；最大 5,147,475 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,651.860；最小 5,621.514；最大 5,696.028 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 620,405.0；最小 497,744；最大 778,564 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 89.023% +
+
Primary I-cache request/core
最小 934,228；最大 985,878
+
Primary I-cache miss/core
最小 44,299；最大 47,438
+
加权 miss rate
4.833%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 3,986.910；最大 4,269.420 µs
+
+
+
+

逐物理核分布

+

AICAIV

+
+
+
SYS gate boundary-diagnostic ticks/core
+ + + + 5,147,475 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_submit_elapsed_ticks=4435941physical=1 logical=1 role=AIC scalar_submit_elapsed_ticks=4345196physical=2 logical=2 role=AIC scalar_submit_elapsed_ticks=4450973physical=3 logical=3 role=AIC scalar_submit_elapsed_ticks=4346628physical=4 logical=4 role=AIC scalar_submit_elapsed_ticks=4474887physical=5 logical=5 role=AIC scalar_submit_elapsed_ticks=4305564physical=6 logical=6 role=AIC scalar_submit_elapsed_ticks=4445283physical=7 logical=7 role=AIC scalar_submit_elapsed_ticks=4337465physical=8 logical=8 role=AIC scalar_submit_elapsed_ticks=4458182physical=9 logical=9 role=AIC scalar_submit_elapsed_ticks=4315191physical=10 logical=10 role=AIC scalar_submit_elapsed_ticks=4457636physical=12 logical=11 role=AIC scalar_submit_elapsed_ticks=4380467physical=13 logical=12 role=AIC scalar_submit_elapsed_ticks=4457478physical=14 logical=13 role=AIC scalar_submit_elapsed_ticks=4315728physical=15 logical=14 role=AIC scalar_submit_elapsed_ticks=4521531physical=16 logical=15 role=AIC scalar_submit_elapsed_ticks=4311722physical=54 logical=16 role=AIC scalar_submit_elapsed_ticks=4482654physical=55 logical=17 role=AIC scalar_submit_elapsed_ticks=4330795physical=56 logical=18 role=AIC scalar_submit_elapsed_ticks=4495543physical=57 logical=19 role=AIC scalar_submit_elapsed_ticks=4311434physical=58 logical=20 role=AIC scalar_submit_elapsed_ticks=4411537physical=60 logical=21 role=AIC scalar_submit_elapsed_ticks=4338629physical=61 logical=22 role=AIC scalar_submit_elapsed_ticks=4448769physical=62 logical=23 role=AIC scalar_submit_elapsed_ticks=4433109physical=63 logical=24 role=AIC scalar_submit_elapsed_ticks=4392211physical=64 logical=25 role=AIC scalar_submit_elapsed_ticks=4362003physical=65 logical=26 role=AIC scalar_submit_elapsed_ticks=4420854physical=66 logical=27 role=AIC scalar_submit_elapsed_ticks=4367218physical=67 logical=28 role=AIC scalar_submit_elapsed_ticks=4370302physical=68 logical=29 role=AIC scalar_submit_elapsed_ticks=4347552physical=69 logical=30 role=AIC scalar_submit_elapsed_ticks=4464753physical=70 logical=31 role=AIC scalar_submit_elapsed_ticks=4329706physical=18 logical=32 role=AIV scalar_submit_elapsed_ticks=5061927physical=19 logical=33 role=AIV scalar_submit_elapsed_ticks=5069774physical=20 logical=34 role=AIV scalar_submit_elapsed_ticks=5015154physical=21 logical=35 role=AIV scalar_submit_elapsed_ticks=4989579physical=22 logical=36 role=AIV scalar_submit_elapsed_ticks=5027924physical=23 logical=37 role=AIV scalar_submit_elapsed_ticks=5037655physical=24 logical=38 role=AIV scalar_submit_elapsed_ticks=5087073physical=25 logical=39 role=AIV scalar_submit_elapsed_ticks=5016407physical=26 logical=40 role=AIV scalar_submit_elapsed_ticks=5100679physical=27 logical=41 role=AIV scalar_submit_elapsed_ticks=5029695physical=28 logical=42 role=AIV scalar_submit_elapsed_ticks=5068094physical=29 logical=43 role=AIV scalar_submit_elapsed_ticks=4986258physical=30 logical=44 role=AIV scalar_submit_elapsed_ticks=5030819physical=31 logical=45 role=AIV scalar_submit_elapsed_ticks=5027588physical=32 logical=46 role=AIV scalar_submit_elapsed_ticks=5030866physical=33 logical=47 role=AIV scalar_submit_elapsed_ticks=4970216physical=34 logical=48 role=AIV scalar_submit_elapsed_ticks=5104655physical=35 logical=49 role=AIV scalar_submit_elapsed_ticks=5010319physical=36 logical=50 role=AIV scalar_submit_elapsed_ticks=5034598physical=37 logical=51 role=AIV scalar_submit_elapsed_ticks=4985213physical=38 logical=52 role=AIV scalar_submit_elapsed_ticks=5128233physical=39 logical=53 role=AIV scalar_submit_elapsed_ticks=5029912physical=42 logical=54 role=AIV scalar_submit_elapsed_ticks=5025004physical=43 logical=55 role=AIV scalar_submit_elapsed_ticks=4973221physical=44 logical=56 role=AIV scalar_submit_elapsed_ticks=5091415physical=45 logical=57 role=AIV scalar_submit_elapsed_ticks=4928348physical=46 logical=58 role=AIV scalar_submit_elapsed_ticks=5030436physical=47 logical=59 role=AIV scalar_submit_elapsed_ticks=5007528physical=48 logical=60 role=AIV scalar_submit_elapsed_ticks=5052752physical=49 logical=61 role=AIV scalar_submit_elapsed_ticks=4937603physical=50 logical=62 role=AIV scalar_submit_elapsed_ticks=5074856physical=51 logical=63 role=AIV scalar_submit_elapsed_ticks=5022592physical=72 logical=64 role=AIV scalar_submit_elapsed_ticks=4955759physical=73 logical=65 role=AIV scalar_submit_elapsed_ticks=5042715physical=74 logical=66 role=AIV scalar_submit_elapsed_ticks=5075122physical=75 logical=67 role=AIV scalar_submit_elapsed_ticks=5010686physical=76 logical=68 role=AIV scalar_submit_elapsed_ticks=5016893physical=77 logical=69 role=AIV scalar_submit_elapsed_ticks=4931327physical=78 logical=70 role=AIV scalar_submit_elapsed_ticks=5101450physical=79 logical=71 role=AIV scalar_submit_elapsed_ticks=5034999physical=80 logical=72 role=AIV scalar_submit_elapsed_ticks=5027707physical=81 logical=73 role=AIV scalar_submit_elapsed_ticks=4877042physical=84 logical=74 role=AIV scalar_submit_elapsed_ticks=5066556physical=85 logical=75 role=AIV scalar_submit_elapsed_ticks=5026570physical=86 logical=76 role=AIV scalar_submit_elapsed_ticks=5119460physical=87 logical=77 role=AIV scalar_submit_elapsed_ticks=5040291physical=88 logical=78 role=AIV scalar_submit_elapsed_ticks=5036702physical=89 logical=79 role=AIV scalar_submit_elapsed_ticks=4968815physical=90 logical=80 role=AIV scalar_submit_elapsed_ticks=5079383physical=91 logical=81 role=AIV scalar_submit_elapsed_ticks=4970275physical=92 logical=82 role=AIV scalar_submit_elapsed_ticks=5054599physical=93 logical=83 role=AIV scalar_submit_elapsed_ticks=4995230physical=94 logical=84 role=AIV scalar_submit_elapsed_ticks=5033702physical=95 logical=85 role=AIV scalar_submit_elapsed_ticks=5018908physical=96 logical=86 role=AIV scalar_submit_elapsed_ticks=5097080physical=97 logical=87 role=AIV scalar_submit_elapsed_ticks=5013390physical=98 logical=88 role=AIV scalar_submit_elapsed_ticks=5076771physical=99 logical=89 role=AIV scalar_submit_elapsed_ticks=5147475physical=100 logical=90 role=AIV scalar_submit_elapsed_ticks=5056282physical=101 logical=91 role=AIV scalar_submit_elapsed_ticks=5020566physical=102 logical=92 role=AIV scalar_submit_elapsed_ticks=5093566physical=103 logical=93 role=AIV scalar_submit_elapsed_ticks=5030106physical=104 logical=94 role=AIV scalar_submit_elapsed_ticks=5006683physical=105 logical=95 role=AIV scalar_submit_elapsed_ticks=5000645 + +
+ +
+
PMU total cycles/core
+ + + + 9,124,956 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC total_cycles=7958190physical=1 logical=1 role=AIC total_cycles=7832638physical=2 logical=2 role=AIC total_cycles=7957249physical=3 logical=3 role=AIC total_cycles=7939481physical=4 logical=4 role=AIC total_cycles=8094985physical=5 logical=5 role=AIC total_cycles=7897055physical=6 logical=6 role=AIC total_cycles=8044205physical=7 logical=7 role=AIC total_cycles=7830679physical=8 logical=8 role=AIC total_cycles=8016930physical=9 logical=9 role=AIC total_cycles=7804619physical=10 logical=10 role=AIC total_cycles=7961366physical=12 logical=11 role=AIC total_cycles=7904760physical=13 logical=12 role=AIC total_cycles=8024861physical=14 logical=13 role=AIC total_cycles=7858939physical=15 logical=14 role=AIC total_cycles=8101625physical=16 logical=15 role=AIC total_cycles=7811046physical=54 logical=16 role=AIC total_cycles=8111615physical=55 logical=17 role=AIC total_cycles=7868306physical=56 logical=18 role=AIC total_cycles=8087671physical=57 logical=19 role=AIC total_cycles=7914102physical=58 logical=20 role=AIC total_cycles=7901881physical=60 logical=21 role=AIC total_cycles=7932356physical=61 logical=22 role=AIC total_cycles=7990216physical=62 logical=23 role=AIC total_cycles=8058134physical=63 logical=24 role=AIC total_cycles=7838658physical=64 logical=25 role=AIC total_cycles=7892723physical=65 logical=26 role=AIC total_cycles=7854468physical=66 logical=27 role=AIC total_cycles=7932175physical=67 logical=28 role=AIC total_cycles=7772732physical=68 logical=29 role=AIC total_cycles=7814451physical=69 logical=30 role=AIC total_cycles=8016521physical=70 logical=31 role=AIC total_cycles=7784811physical=18 logical=32 role=AIV total_cycles=8923680physical=19 logical=33 role=AIV total_cycles=9017825physical=20 logical=34 role=AIV total_cycles=8933851physical=21 logical=35 role=AIV total_cycles=8932920physical=22 logical=36 role=AIV total_cycles=8886504physical=23 logical=37 role=AIV total_cycles=8948625physical=24 logical=38 role=AIV total_cycles=9017773physical=25 logical=39 role=AIV total_cycles=9009634physical=26 logical=40 role=AIV total_cycles=9066916physical=27 logical=41 role=AIV total_cycles=8913085physical=28 logical=42 role=AIV total_cycles=9013853physical=29 logical=43 role=AIV total_cycles=8916074physical=30 logical=44 role=AIV total_cycles=8931187physical=31 logical=45 role=AIV total_cycles=9051959physical=32 logical=46 role=AIV total_cycles=9011598physical=33 logical=47 role=AIV total_cycles=8907015physical=34 logical=48 role=AIV total_cycles=9027150physical=35 logical=49 role=AIV total_cycles=8991677physical=36 logical=50 role=AIV total_cycles=8984930physical=37 logical=51 role=AIV total_cycles=8880613physical=38 logical=52 role=AIV total_cycles=9092661physical=39 logical=53 role=AIV total_cycles=9000637physical=42 logical=54 role=AIV total_cycles=8894738physical=43 logical=55 role=AIV total_cycles=8845693physical=44 logical=56 role=AIV total_cycles=9090409physical=45 logical=57 role=AIV total_cycles=8791526physical=46 logical=58 role=AIV total_cycles=8998373physical=47 logical=59 role=AIV total_cycles=8881941physical=48 logical=60 role=AIV total_cycles=9015484physical=49 logical=61 role=AIV total_cycles=8841625physical=50 logical=62 role=AIV total_cycles=9064394physical=51 logical=63 role=AIV total_cycles=8933421physical=72 logical=64 role=AIV total_cycles=8787993physical=73 logical=65 role=AIV total_cycles=9092022physical=74 logical=66 role=AIV total_cycles=8967472physical=75 logical=67 role=AIV total_cycles=8919693physical=76 logical=68 role=AIV total_cycles=8879970physical=77 logical=69 role=AIV total_cycles=8798641physical=78 logical=70 role=AIV total_cycles=9050237physical=79 logical=71 role=AIV total_cycles=9019888physical=80 logical=72 role=AIV total_cycles=8973903physical=81 logical=73 role=AIV total_cycles=8705727physical=84 logical=74 role=AIV total_cycles=9027320physical=85 logical=75 role=AIV total_cycles=8933433physical=86 logical=76 role=AIV total_cycles=9066445physical=87 logical=77 role=AIV total_cycles=8995103physical=88 logical=78 role=AIV total_cycles=8961095physical=89 logical=79 role=AIV total_cycles=8924123physical=90 logical=80 role=AIV total_cycles=9103573physical=91 logical=81 role=AIV total_cycles=8815912physical=92 logical=82 role=AIV total_cycles=8958789physical=93 logical=83 role=AIV total_cycles=8953154physical=94 logical=84 role=AIV total_cycles=8940016physical=95 logical=85 role=AIV total_cycles=8923775physical=96 logical=86 role=AIV total_cycles=9106923physical=97 logical=87 role=AIV total_cycles=8877076physical=98 logical=88 role=AIV total_cycles=9028154physical=99 logical=89 role=AIV total_cycles=9124956physical=100 logical=90 role=AIV total_cycles=8966512physical=101 logical=91 role=AIV total_cycles=8919468physical=102 logical=92 role=AIV total_cycles=9037338physical=103 logical=93 role=AIV total_cycles=8914025physical=104 logical=94 role=AIV total_cycles=8859731physical=105 logical=95 role=AIV total_cycles=8928356 + +
+ +
+
Scalar busy cycles/core
+ + + + 8,167,364 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_busy=7704621physical=1 logical=1 role=AIC scalar_busy=7569231physical=2 logical=2 role=AIC scalar_busy=7699741physical=3 logical=3 role=AIC scalar_busy=7681800physical=4 logical=4 role=AIC scalar_busy=7824160physical=5 logical=5 role=AIC scalar_busy=7624330physical=6 logical=6 role=AIC scalar_busy=7795223physical=7 logical=7 role=AIC scalar_busy=7587800physical=8 logical=8 role=AIC scalar_busy=7762277physical=9 logical=9 role=AIC scalar_busy=7546333physical=10 logical=10 role=AIC scalar_busy=7724161physical=12 logical=11 role=AIC scalar_busy=7653459physical=13 logical=12 role=AIC scalar_busy=7780674physical=14 logical=13 role=AIC scalar_busy=7608024physical=15 logical=14 role=AIC scalar_busy=7863247physical=16 logical=15 role=AIC scalar_busy=7566892physical=54 logical=16 role=AIC scalar_busy=7888592physical=55 logical=17 role=AIC scalar_busy=7616690physical=56 logical=18 role=AIC scalar_busy=7837017physical=57 logical=19 role=AIC scalar_busy=7658588physical=58 logical=20 role=AIC scalar_busy=7649164physical=60 logical=21 role=AIC scalar_busy=7687398physical=61 logical=22 role=AIC scalar_busy=7760097physical=62 logical=23 role=AIC scalar_busy=7831110physical=63 logical=24 role=AIC scalar_busy=7590339physical=64 logical=25 role=AIC scalar_busy=7643434physical=65 logical=26 role=AIC scalar_busy=7611466physical=66 logical=27 role=AIC scalar_busy=7660941physical=67 logical=28 role=AIC scalar_busy=7520926physical=68 logical=29 role=AIC scalar_busy=7541027physical=69 logical=30 role=AIC scalar_busy=7752589physical=70 logical=31 role=AIC scalar_busy=7529928physical=18 logical=32 role=AIV scalar_busy=7852274physical=19 logical=33 role=AIV scalar_busy=7861810physical=20 logical=34 role=AIV scalar_busy=7927658physical=21 logical=35 role=AIV scalar_busy=7804457physical=22 logical=36 role=AIV scalar_busy=7860043physical=23 logical=37 role=AIV scalar_busy=7846476physical=24 logical=38 role=AIV scalar_busy=7991020physical=25 logical=39 role=AIV scalar_busy=7921491physical=26 logical=40 role=AIV scalar_busy=8020839physical=27 logical=41 role=AIV scalar_busy=7703901physical=28 logical=42 role=AIV scalar_busy=7944334physical=29 logical=43 role=AIV scalar_busy=7772159physical=30 logical=44 role=AIV scalar_busy=7950704physical=31 logical=45 role=AIV scalar_busy=7967029physical=32 logical=46 role=AIV scalar_busy=8105015physical=33 logical=47 role=AIV scalar_busy=7868272physical=34 logical=48 role=AIV scalar_busy=7976881physical=35 logical=49 role=AIV scalar_busy=7857691physical=36 logical=50 role=AIV scalar_busy=7998894physical=37 logical=51 role=AIV scalar_busy=7826181physical=38 logical=52 role=AIV scalar_busy=8044893physical=39 logical=53 role=AIV scalar_busy=7848809physical=42 logical=54 role=AIV scalar_busy=7890630physical=43 logical=55 role=AIV scalar_busy=7801692physical=44 logical=56 role=AIV scalar_busy=8167364physical=45 logical=57 role=AIV scalar_busy=7746575physical=46 logical=58 role=AIV scalar_busy=7965705physical=47 logical=59 role=AIV scalar_busy=7767440physical=48 logical=60 role=AIV scalar_busy=8095616physical=49 logical=61 role=AIV scalar_busy=7852422physical=50 logical=62 role=AIV scalar_busy=7998574physical=51 logical=63 role=AIV scalar_busy=7802776physical=72 logical=64 role=AIV scalar_busy=7801286physical=73 logical=65 role=AIV scalar_busy=7989632physical=74 logical=66 role=AIV scalar_busy=7906280physical=75 logical=67 role=AIV scalar_busy=7793657physical=76 logical=68 role=AIV scalar_busy=7883290physical=77 logical=69 role=AIV scalar_busy=7738242physical=78 logical=70 role=AIV scalar_busy=8033413physical=79 logical=71 role=AIV scalar_busy=7914547physical=80 logical=72 role=AIV scalar_busy=8044132physical=81 logical=73 role=AIV scalar_busy=7637221physical=84 logical=74 role=AIV scalar_busy=8060212physical=85 logical=75 role=AIV scalar_busy=7871422physical=86 logical=76 role=AIV scalar_busy=8007962physical=87 logical=77 role=AIV scalar_busy=7840963physical=88 logical=78 role=AIV scalar_busy=8016566physical=89 logical=79 role=AIV scalar_busy=7881082physical=90 logical=80 role=AIV scalar_busy=8089643physical=91 logical=81 role=AIV scalar_busy=7669698physical=92 logical=82 role=AIV scalar_busy=8002026physical=93 logical=83 role=AIV scalar_busy=7907104physical=94 logical=84 role=AIV scalar_busy=7903427physical=95 logical=85 role=AIV scalar_busy=7842078physical=96 logical=86 role=AIV scalar_busy=8073281physical=97 logical=87 role=AIV scalar_busy=7720387physical=98 logical=88 role=AIV scalar_busy=7921962physical=99 logical=89 role=AIV scalar_busy=7919215physical=100 logical=90 role=AIV scalar_busy=7980496physical=101 logical=91 role=AIV scalar_busy=7815731physical=102 logical=92 role=AIV scalar_busy=7981860physical=103 logical=93 role=AIV scalar_busy=7735444physical=104 logical=94 role=AIV scalar_busy=7841043physical=105 logical=95 role=AIV scalar_busy=7879926 + +
+ +
+
Primary I-cache requests/core
+ + + + 1,001,034 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_requests=967877physical=1 logical=1 role=AIC icache_requests=979306physical=2 logical=2 role=AIC icache_requests=955735physical=3 logical=3 role=AIC icache_requests=996235physical=4 logical=4 role=AIC icache_requests=983572physical=5 logical=5 role=AIC icache_requests=998811physical=6 logical=6 role=AIC icache_requests=973186physical=7 logical=7 role=AIC icache_requests=981483physical=8 logical=8 role=AIC icache_requests=970297physical=9 logical=9 role=AIC icache_requests=986851physical=10 logical=10 role=AIC icache_requests=955013physical=12 logical=11 role=AIC icache_requests=969370physical=13 logical=12 role=AIC icache_requests=981998physical=14 logical=13 role=AIC icache_requests=997877physical=15 logical=14 role=AIC icache_requests=966668physical=16 logical=15 role=AIC icache_requests=989670physical=54 logical=16 role=AIC icache_requests=982343physical=55 logical=17 role=AIC icache_requests=985742physical=56 logical=18 role=AIC icache_requests=978584physical=57 logical=19 role=AIC icache_requests=996426physical=58 logical=20 role=AIC icache_requests=970567physical=60 logical=21 role=AIC icache_requests=1001034physical=61 logical=22 role=AIC icache_requests=974226physical=62 logical=23 role=AIC icache_requests=995096physical=63 logical=24 role=AIC icache_requests=959865physical=64 logical=25 role=AIC icache_requests=989113physical=65 logical=26 role=AIC icache_requests=951535physical=66 logical=27 role=AIC icache_requests=991995physical=67 logical=28 role=AIC icache_requests=954687physical=68 logical=29 role=AIC icache_requests=978749physical=69 logical=30 role=AIC icache_requests=974686physical=70 logical=31 role=AIC icache_requests=983604physical=18 logical=32 role=AIV icache_requests=934228physical=19 logical=33 role=AIV icache_requests=946251physical=20 logical=34 role=AIV icache_requests=946289physical=21 logical=35 role=AIV icache_requests=967525physical=22 logical=36 role=AIV icache_requests=936633physical=23 logical=37 role=AIV icache_requests=939838physical=24 logical=38 role=AIV icache_requests=944031physical=25 logical=39 role=AIV icache_requests=975095physical=26 logical=40 role=AIV icache_requests=944795physical=27 logical=41 role=AIV icache_requests=942674physical=28 logical=42 role=AIV icache_requests=944243physical=29 logical=43 role=AIV icache_requests=959330physical=30 logical=44 role=AIV icache_requests=949622physical=31 logical=45 role=AIV icache_requests=976483physical=32 logical=46 role=AIV icache_requests=962058physical=33 logical=47 role=AIV icache_requests=965925physical=34 logical=48 role=AIV icache_requests=935462physical=35 logical=49 role=AIV icache_requests=969838physical=36 logical=50 role=AIV icache_requests=953872physical=37 logical=51 role=AIV icache_requests=962559physical=38 logical=52 role=AIV icache_requests=938622physical=39 logical=53 role=AIV icache_requests=959866physical=42 logical=54 role=AIV icache_requests=945874physical=43 logical=55 role=AIV icache_requests=949786physical=44 logical=56 role=AIV icache_requests=969308physical=45 logical=57 role=AIV icache_requests=951272physical=46 logical=58 role=AIV icache_requests=964155physical=47 logical=59 role=AIV icache_requests=943245physical=48 logical=60 role=AIV icache_requests=957128physical=49 logical=61 role=AIV icache_requests=949886physical=50 logical=62 role=AIV icache_requests=944273physical=51 logical=63 role=AIV icache_requests=941735physical=72 logical=64 role=AIV icache_requests=935680physical=73 logical=65 role=AIV icache_requests=985878physical=74 logical=66 role=AIV icache_requests=934286physical=75 logical=67 role=AIV icache_requests=953446physical=76 logical=68 role=AIV icache_requests=938961physical=77 logical=69 role=AIV icache_requests=944903physical=78 logical=70 role=AIV icache_requests=956176physical=79 logical=71 role=AIV icache_requests=964621physical=80 logical=72 role=AIV icache_requests=960727physical=81 logical=73 role=AIV icache_requests=945714physical=84 logical=74 role=AIV icache_requests=957741physical=85 logical=75 role=AIV icache_requests=947020physical=86 logical=76 role=AIV icache_requests=946505physical=87 logical=77 role=AIV icache_requests=962573physical=88 logical=78 role=AIV icache_requests=952501physical=89 logical=79 role=AIV icache_requests=978047physical=90 logical=80 role=AIV icache_requests=964861physical=91 logical=81 role=AIV icache_requests=951304physical=92 logical=82 role=AIV icache_requests=953774physical=93 logical=83 role=AIV icache_requests=978568physical=94 logical=84 role=AIV icache_requests=942345physical=95 logical=85 role=AIV icache_requests=959249physical=96 logical=86 role=AIV icache_requests=965305physical=97 logical=87 role=AIV icache_requests=945485physical=98 logical=88 role=AIV icache_requests=952648physical=99 logical=89 role=AIV icache_requests=943371physical=100 logical=90 role=AIV icache_requests=940504physical=101 logical=91 role=AIV icache_requests=954498physical=102 logical=92 role=AIV icache_requests=949749physical=103 logical=93 role=AIV icache_requests=949240physical=104 logical=94 role=AIV icache_requests=939911physical=105 logical=95 role=AIV icache_requests=973542 + +
+ +
+
Primary I-cache misses/core
+ + + + 47,438 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_misses=6118physical=1 logical=1 role=AIC icache_misses=7148physical=2 logical=2 role=AIC icache_misses=6092physical=3 logical=3 role=AIC icache_misses=6839physical=4 logical=4 role=AIC icache_misses=6963physical=5 logical=5 role=AIC icache_misses=6973physical=6 logical=6 role=AIC icache_misses=6571physical=7 logical=7 role=AIC icache_misses=7094physical=8 logical=8 role=AIC icache_misses=7607physical=9 logical=9 role=AIC icache_misses=7124physical=10 logical=10 role=AIC icache_misses=6364physical=12 logical=11 role=AIC icache_misses=6357physical=13 logical=12 role=AIC icache_misses=6709physical=14 logical=13 role=AIC icache_misses=6595physical=15 logical=14 role=AIC icache_misses=6230physical=16 logical=15 role=AIC icache_misses=6379physical=54 logical=16 role=AIC icache_misses=6835physical=55 logical=17 role=AIC icache_misses=6682physical=56 logical=18 role=AIC icache_misses=6267physical=57 logical=19 role=AIC icache_misses=7013physical=58 logical=20 role=AIC icache_misses=6435physical=60 logical=21 role=AIC icache_misses=6898physical=61 logical=22 role=AIC icache_misses=6044physical=62 logical=23 role=AIC icache_misses=6305physical=63 logical=24 role=AIC icache_misses=6456physical=64 logical=25 role=AIC icache_misses=7028physical=65 logical=26 role=AIC icache_misses=5936physical=66 logical=27 role=AIC icache_misses=7010physical=67 logical=28 role=AIC icache_misses=5976physical=68 logical=29 role=AIC icache_misses=6708physical=69 logical=30 role=AIC icache_misses=6512physical=70 logical=31 role=AIC icache_misses=6932physical=18 logical=32 role=AIV icache_misses=45408physical=19 logical=33 role=AIV icache_misses=45785physical=20 logical=34 role=AIV icache_misses=45929physical=21 logical=35 role=AIV icache_misses=47000physical=22 logical=36 role=AIV icache_misses=44299physical=23 logical=37 role=AIV icache_misses=45539physical=24 logical=38 role=AIV icache_misses=45574physical=25 logical=39 role=AIV icache_misses=46776physical=26 logical=40 role=AIV icache_misses=45417physical=27 logical=41 role=AIV icache_misses=45924physical=28 logical=42 role=AIV icache_misses=45599physical=29 logical=43 role=AIV icache_misses=46428physical=30 logical=44 role=AIV icache_misses=44446physical=31 logical=45 role=AIV icache_misses=46590physical=32 logical=46 role=AIV icache_misses=46481physical=33 logical=47 role=AIV icache_misses=46767physical=34 logical=48 role=AIV icache_misses=45237physical=35 logical=49 role=AIV icache_misses=46762physical=36 logical=50 role=AIV icache_misses=46508physical=37 logical=51 role=AIV icache_misses=46651physical=38 logical=52 role=AIV icache_misses=45463physical=39 logical=53 role=AIV icache_misses=46006physical=42 logical=54 role=AIV icache_misses=45942physical=43 logical=55 role=AIV icache_misses=46106physical=44 logical=56 role=AIV icache_misses=46707physical=45 logical=57 role=AIV icache_misses=46327physical=46 logical=58 role=AIV icache_misses=46129physical=47 logical=59 role=AIV icache_misses=45825physical=48 logical=60 role=AIV icache_misses=46185physical=49 logical=61 role=AIV icache_misses=46219physical=50 logical=62 role=AIV icache_misses=45702physical=51 logical=63 role=AIV icache_misses=45411physical=72 logical=64 role=AIV icache_misses=46200physical=73 logical=65 role=AIV icache_misses=47438physical=74 logical=66 role=AIV icache_misses=45366physical=75 logical=67 role=AIV icache_misses=45853physical=76 logical=68 role=AIV icache_misses=46018physical=77 logical=69 role=AIV icache_misses=46448physical=78 logical=70 role=AIV icache_misses=45811physical=79 logical=71 role=AIV icache_misses=46084physical=80 logical=72 role=AIV icache_misses=46427physical=81 logical=73 role=AIV icache_misses=46522physical=84 logical=74 role=AIV icache_misses=46222physical=85 logical=75 role=AIV icache_misses=45827physical=86 logical=76 role=AIV icache_misses=45871physical=87 logical=77 role=AIV icache_misses=46152physical=88 logical=78 role=AIV icache_misses=45906physical=89 logical=79 role=AIV icache_misses=47225physical=90 logical=80 role=AIV icache_misses=46500physical=91 logical=81 role=AIV icache_misses=46409physical=92 logical=82 role=AIV icache_misses=46297physical=93 logical=83 role=AIV icache_misses=47327physical=94 logical=84 role=AIV icache_misses=45702physical=95 logical=85 role=AIV icache_misses=46222physical=96 logical=86 role=AIV icache_misses=46445physical=97 logical=87 role=AIV icache_misses=45415physical=98 logical=88 role=AIV icache_misses=45922physical=99 logical=89 role=AIV icache_misses=45157physical=100 logical=90 role=AIV icache_misses=45597physical=101 logical=91 role=AIV icache_misses=45927physical=102 logical=92 role=AIV icache_misses=45916physical=103 logical=93 role=AIV icache_misses=46059physical=104 logical=94 role=AIV icache_misses=46032physical=105 logical=95 role=AIV icache_misses=46615 + +
+
+

逐核原始主计数

+

报告展示 primary request/miss；phase 模式的 I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和，只用于标记全窗 capture gap；scalar shadow 同样用于核验 phase 读清重建，并显示其相对 CNT2 primary 的 capture loss，不作为第二份性能数据展示。

+
+ + + + + + + +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate90ns量尺 µs
00AIC004,435,9415,364.5327,958,1907,704,621967,8776,1180.632%550.620
11AIC104,345,1965,377.1867,832,6387,569,231979,3067,1480.730%643.320
22AIC204,450,9735,357.4167,957,2497,699,741955,7356,0920.637%548.280
33AIC304,346,6285,336.9177,939,4817,681,800996,2356,8390.686%615.510
44AIC404,474,8875,347.0968,094,9857,824,160983,5726,9630.708%626.670
55AIC504,305,5645,365.1397,897,0557,624,330998,8116,9730.698%627.570
66AIC604,445,2835,361.4868,044,2057,795,223973,1866,5710.675%591.390
77AIC704,337,4655,375.6457,830,6797,587,800981,4837,0940.723%638.460
88AIC804,458,1825,387.1498,016,9307,762,277970,2977,6070.784%684.630
99AIC904,315,1915,342.7887,804,6197,546,333986,8517,1240.722%641.160
1010AIC1004,457,6365,337.3257,961,3667,724,161955,0136,3640.666%572.760
1211AIC1104,380,4675,351.7717,904,7607,653,459969,3706,3570.656%572.130
1312AIC1204,457,4785,409.2528,024,8617,780,674981,9986,7090.683%603.810
1413AIC1304,315,7285,375.7727,858,9397,608,024997,8776,5950.661%593.550
1514AIC1404,521,5315,368.9288,101,6257,863,247966,6686,2300.644%560.700
1615AIC1504,311,7225,371.4237,811,0467,566,892989,6706,3790.645%574.110
1832AIV015,061,9275,626.0668,923,6807,852,274934,22845,4084.860%4,086.720
1933AIV025,069,7745,685.7299,017,8257,861,810946,25145,7854.839%4,120.650
2034AIV115,015,1545,633.8258,933,8517,927,658946,28945,9294.854%4,133.610
2135AIV124,989,5795,692.1178,932,9207,804,457967,52547,0004.858%4,230.000
2236AIV215,027,9245,656.7528,886,5047,860,043936,63344,2994.730%3,986.910
2337AIV225,037,6555,642.7628,948,6257,846,476939,83845,5394.845%4,098.510
2438AIV315,087,0735,633.8889,017,7737,991,020944,03145,5744.828%4,101.660
2539AIV325,016,4075,634.5729,009,6347,921,491975,09546,7764.797%4,209.840
2640AIV415,100,6795,662.8179,066,9168,020,839944,79545,4174.807%4,087.530
2741AIV425,029,6955,676.3138,913,0857,703,901942,67445,9244.872%4,133.160
2842AIV515,068,0945,630.9219,013,8537,944,334944,24345,5994.829%4,103.910
2943AIV524,986,2585,679.3748,916,0747,772,159959,33046,4284.840%4,178.520
3044AIV615,030,8195,636.2628,931,1877,950,704949,62244,4464.680%4,000.140
3145AIV625,027,5885,659.4909,051,9597,967,029976,48346,5904.771%4,193.100
3246AIV715,030,8665,633.9089,011,5988,105,015962,05846,4814.831%4,183.290
3347AIV724,970,2165,623.0078,907,0157,868,272965,92546,7674.842%4,209.030
3448AIV815,104,6555,685.8039,027,1507,976,881935,46245,2374.836%4,071.330
3549AIV825,010,3195,621.5148,991,6777,857,691969,83846,7624.822%4,208.580
3650AIV915,034,5985,669.6368,984,9307,998,894953,87246,5084.876%4,185.720
3751AIV924,985,2135,657.7448,880,6137,826,181962,55946,6514.847%4,198.590
3852AIV1015,128,2335,677.8099,092,6618,044,893938,62245,4634.844%4,091.670
3953AIV1025,029,9125,680.0349,000,6377,848,809959,86646,0064.793%4,140.540
4254AIV1115,025,0045,666.2148,894,7387,890,630945,87445,9424.857%4,134.780
4355AIV1124,973,2215,637.5358,845,6937,801,692949,78646,1064.854%4,149.540
4456AIV1215,091,4155,682.8849,090,4098,167,364969,30846,7074.819%4,203.630
4557AIV1224,928,3485,659.3858,791,5267,746,575951,27246,3274.870%4,169.430
4658AIV1315,030,4365,627.7088,998,3737,965,705964,15546,1294.784%4,151.610
4759AIV1325,007,5285,658.3468,881,9417,767,440943,24545,8254.858%4,124.250
4860AIV1415,052,7525,633.7769,015,4848,095,616957,12846,1854.825%4,156.650
4961AIV1424,937,6035,633.7738,841,6257,852,422949,88646,2194.866%4,159.710
5062AIV1515,074,8565,661.5189,064,3947,998,574944,27345,7024.840%4,113.180
5163AIV1525,022,5925,635.0058,933,4217,802,776941,73545,4114.822%4,086.990
5416AIC1604,482,6545,400.1088,111,6157,888,592982,3436,8350.696%615.150
5517AIC1704,330,7955,339.8077,868,3067,616,690985,7426,6820.678%601.380
5618AIC1804,495,5435,366.4678,087,6717,837,017978,5846,2670.640%564.030
5719AIC1904,311,4345,377.8757,914,1027,658,588996,4267,0130.704%631.170
5820AIC2004,411,5375,375.7037,901,8817,649,164970,5676,4350.663%579.150
6021AIC2104,338,6295,366.8977,932,3567,687,3981,001,0346,8980.689%620.820
6122AIC2204,448,7695,381.2367,990,2167,760,097974,2266,0440.620%543.960
6223AIC2304,433,1095,340.1308,058,1347,831,110995,0966,3050.634%567.450
6324AIC2404,392,2115,339.8487,838,6587,590,339959,8656,4560.673%581.040
6425AIC2504,362,0035,378.2657,892,7237,643,434989,1137,0280.711%632.520
6526AIC2604,420,8545,360.2337,854,4687,611,466951,5355,9360.624%534.240
6627AIC2704,367,2185,370.1237,932,1757,660,941991,9957,0100.707%630.900
6728AIC2804,370,3025,348.4177,772,7327,520,926954,6875,9760.626%537.840
6829AIC2904,347,5525,349.9517,814,4517,541,027978,7496,7080.685%603.720
6930AIC3004,464,7535,375.2868,016,5217,752,589974,6866,5120.668%586.080
7031AIC3104,329,7065,346.4927,784,8117,529,928983,6046,9320.705%623.880
7264AIV1614,955,7595,646.1818,787,9937,801,286935,68046,2004.938%4,158.000
7365AIV1625,042,7155,633.4589,092,0227,989,632985,87847,4384.812%4,269.420
7466AIV1715,075,1225,651.5958,967,4727,906,280934,28645,3664.856%4,082.940
7567AIV1725,010,6865,627.9878,919,6937,793,657953,44645,8534.809%4,126.770
7668AIV1815,016,8935,651.3888,879,9707,883,290938,96146,0184.901%4,141.620
7769AIV1824,931,3275,655.7278,798,6417,738,242944,90346,4484.916%4,180.320
7870AIV1915,101,4505,656.6339,050,2378,033,413956,17645,8114.791%4,122.990
7971AIV1925,034,9995,636.8089,019,8887,914,547964,62146,0844.777%4,147.560
8072AIV2015,027,7075,663.3978,973,9038,044,132960,72746,4274.832%4,178.430
8173AIV2024,877,0425,655.6068,705,7277,637,221945,71446,5224.919%4,186.980
8474AIV2115,066,5565,639.8229,027,3208,060,212957,74146,2224.826%4,159.980
8575AIV2125,026,5705,633.0208,933,4337,871,422947,02045,8274.839%4,124.430
8676AIV2215,119,4605,661.0049,066,4458,007,962946,50545,8714.846%4,128.390
8777AIV2225,040,2915,674.3058,995,1037,840,963962,57346,1524.795%4,153.680
8878AIV2315,036,7025,649.6918,961,0958,016,566952,50145,9064.820%4,131.540
8979AIV2324,968,8155,635.6388,924,1237,881,082978,04747,2254.829%4,250.250
9080AIV2415,079,3835,638.1119,103,5738,089,643964,86146,5004.819%4,185.000
9181AIV2424,970,2755,670.7658,815,9127,669,698951,30446,4094.878%4,176.810
9282AIV2515,054,5995,649.7298,958,7898,002,026953,77446,2974.854%4,166.730
9383AIV2524,995,2305,653.1778,953,1547,907,104978,56847,3274.836%4,259.430
9484AIV2615,033,7025,688.4878,940,0167,903,427942,34545,7024.850%4,113.180
9585AIV2625,018,9085,629.4378,923,7757,842,078959,24946,2224.819%4,159.980
9686AIV2715,097,0805,639.6069,106,9238,073,281965,30546,4454.811%4,180.050
9787AIV2725,013,3905,654.1598,877,0767,720,387945,48545,4154.803%4,087.350
9888AIV2815,076,7715,641.9139,028,1547,921,962952,64845,9224.820%4,132.980
9989AIV2825,147,4755,645.2199,124,9567,919,215943,37145,1574.787%4,064.130
10090AIV2915,056,2825,655.2428,966,5127,980,496940,50445,5974.848%4,103.730
10191AIV2925,020,5665,626.3068,919,4687,815,731954,49845,9274.812%4,133.430
10292AIV3015,093,5665,648.0789,037,3387,981,860949,74945,9164.835%4,132.440
10393AIV3025,030,1065,677.2778,914,0257,735,444949,24046,0594.852%4,145.310
10494AIV3115,006,6835,696.0288,859,7317,841,043939,91146,0324.897%4,142.880
10595AIV3125,000,6455,636.7848,928,3567,879,926973,54246,6154.788%4,195.350
+
+ diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_efdrain_control_20260723_044202_icache_report.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_efdrain_control_20260723_044202_icache_report.html new file mode 100644 index 0000000000..9c365e0cfa --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_efdrain_control_20260723_044202_icache_report.html @@ -0,0 +1,362 @@ + + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 fdwic_submit_pmu_raw.json · schema fdwic-submit-pmu-v3 · 32 AIC + 64 AIV

+ +
+

efdrain-control 阶段观察（phase_id=7）

+

边界 efdrain_begin_to_end_excluding_linked_kernel_calls · 计数语义 discontinuous_running_read_clear_excluding_linked_kernel_calls · + 时间语义 boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls · + expected_calls_per_core=1280

+ +

本报告使用 empty-bracket 估算进入 phase observed + 的局部记录代码开销。 + AIC/AIV 分别按“空区间每组记录开销 × 本阶段实际 begin/end 记录组数”计算，ALL + 再由两类核相加。参考值只从 phase observed 分子扣除该估算；原始 whole 分母保持不变， + 因为空区间没有测量 whole 窗口中的全部记录开销。所以下表的参考比例不是完整业务 + 阶段占完整调度窗口的比例，也不是精确校正。若参考值为负，表示记录代码开销估算已经 + 超过原始 observed；这只能说明该信号低于当前校准的分辨能力，不能截成零或解释成负的 + 业务事件。校准 raw：/home/q00473782/atomic/private/gpt/simpler/outputs/TestPagedAttentionUnroll_Case1_20260723_043533/fdwic_submit_pmu_raw.json · + SHA-256 56199f91545cc6a1fbf18b8bb2cfebae279cb5801516311a040d4facccf321ba。两份 provenance 已核验为同一场景、同一 Git revision 8c6b6e3a412d。

+ + +

本阶段若跨过 linked Kernel，会同时门控 SYS 时间与 PMU counter， + 因而 total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段。 + result-used return-ready atomic 依赖区间只从 SYS 边界累计值扣除，PMU counter + 仍含 atomic 指令事件；source-issue atomic + 保留在时间和计数口径内。没有 linked Kernel 的阶段不触发 PMU 停/开表； + 每次调用的分母仍是外层业务调用次数，不是 Begin/End 读数，也不是排除的 Kernel 调用数。

+ +

阶段原始 PMU 观测来自 running read-clear 的 total cycles，并按 AIC/AIV + 各自校准频率换算；提供 empty-bracket 时，页面主参考值再扣除局部记录代码开销估算。 + phase_elapsed_ticks 只是 SYS 边界闭合诊断，绝不按 1 GHz 当作阶段时间。 + Scalar busy 与 non-scalar residual（逐核先算 total−scalar）共同解释阶段 PMU total； + 未校准时只能展示 raw observed；提供 empty-bracket 后，记录代码开销估算只从 phase + observed 分子扣除，原始 whole 分母保持不变。当前数据无法得到完整业务阶段占完整调度 + 窗口的精确比例。 + linked vector/cube Kernel 从 SYS 与 PMU counter 一并门控排除；result-used return-ready atomic + 依赖区间只从 SYS 边界累计值扣除，I-cache/PMU counter 仍含其指令事件；source-issue atomic + 保留。request/miss 百分比的分母是同一 ELF、同一次采集的 Submit 整窗 primary。 + observed_plus_capture_gap = observed + (primary − shadow)； + 边界读数和插桩 bookkeeping 会进入 sample，因此“观测值”和“加全窗 capture gap”都不是原业务 + 事件数的数学上下界。不同 ELF 不能直接相减后冒充业务真值；提供 empty-bracket 时只按上方 + 明示的公式估算局部记录代码开销。request/miss 的逐核 min/max 是每核累计整个 phase + 的极值，不是逐调用极值。

+
+ + + + +
指标ALL96 核AIC32 核AIV64 核
Phase PMU total扣除记录代码开销估算后的参考值 Σ 65,400,312.871 cycles
≈ 39,640.301 µs（1.649844 cycles/ns）
原始 observed Σ 259,240,439 cycles（≈ 157,130.274 µs）− 记录代码开销估算 193,840,126.129 cycles
记录代码开销估算 / 原始 observed 74.772%；原始逐核 最小 2,530,342；最大 2,918,901；原始 2,109.704 cycles/call		扣除记录代码开销估算后的参考值 Σ 23,975,836.432 cycles
≈ 14,530.264 µs（1.650062 cycles/ns）
原始 observed Σ 85,539,214 cycles（≈ 51,840.000 µs）− 记录代码开销估算 61,563,377.568 cycles
记录代码开销估算 / 原始 observed 71.971%；原始逐核 最小 2,530,342；最大 2,814,967；原始 2,088.360 cycles/call		扣除记录代码开销估算后的参考值 Σ 41,424,476.439 cycles
≈ 25,109.837 µs（1.649731 cycles/ns）
原始 observed Σ 173,701,225 cycles（≈ 105,290.635 µs）− 记录代码开销估算 132,276,748.561 cycles
记录代码开销估算 / 原始 observed 76.152%；原始逐核 最小 2,588,694；最大 2,918,901；原始 2,120.376 cycles/call
Phase scalar busy扣除记录代码开销估算后的参考值 Σ 52,687,582.172 cycles
≈ 31,934.887 µs（1.649844 cycles/ns）
原始 observed Σ 177,538,191 cycles（≈ 107,609.078 µs）− 记录代码开销估算 124,850,608.828 cycles
记录代码开销估算 / 原始 observed 70.323%；原始逐核 最小 1,697,975；最大 2,075,030；原始 1,444.809 cycles/call		扣除记录代码开销估算后的参考值 Σ 21,523,537.998 cycles
≈ 13,044.078 µs（1.650062 cycles/ns）
原始 observed Σ 62,068,342 cycles（≈ 37,615.764 µs）− 记录代码开销估算 40,544,804.002 cycles
记录代码开销估算 / 原始 observed 65.323%；原始逐核 最小 1,794,703；最大 2,075,030；原始 1,515.340 cycles/call		扣除记录代码开销估算后的参考值 Σ 31,164,044.175 cycles
≈ 18,890.379 µs（1.649731 cycles/ns）
原始 observed Σ 115,469,849 cycles（≈ 69,993.138 µs）− 记录代码开销估算 84,305,804.825 cycles
记录代码开销估算 / 原始 observed 73.011%；原始逐核 最小 1,697,975；最大 1,992,002；原始 1,409.544 cycles/call
非 Scalar-busy 残余扣除记录代码开销估算后的参考值 Σ 12,712,730.699 cycles
≈ 7,705.414 µs（1.649844 cycles/ns）
原始 observed Σ 81,702,248 cycles（≈ 49,521.196 µs）− 记录代码开销估算 68,989,517.301 cycles
记录代码开销估算 / 原始 observed 84.440%；原始逐核 最小 725,726；最大 937,867；原始 664.895 cycles/call		扣除记录代码开销估算后的参考值 Σ 2,452,298.435 cycles
≈ 1,486.186 µs（1.650062 cycles/ns）
原始 observed Σ 23,470,872 cycles（≈ 14,224.236 µs）− 记录代码开销估算 21,018,573.565 cycles
记录代码开销估算 / 原始 observed 89.552%；原始逐核 最小 725,726；最大 748,044；原始 573.019 cycles/call		扣除记录代码开销估算后的参考值 Σ 10,260,432.264 cycles
≈ 6,219.458 µs（1.649731 cycles/ns）
原始 observed Σ 58,231,376 cycles（≈ 35,297.498 µs）− 记录代码开销估算 47,970,943.736 cycles
记录代码开销估算 / 原始 observed 82.380%；原始逐核 最小 841,898；最大 937,867；原始 710.832 cycles/call
阶段参考关系（raw 同格保留）扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 80.562%
参考 Non-scalar / 参考 Phase total 19.438%
参考 Phase total / 原始 whole total 7.924%；原始 observed 31.409%
参考 Phase scalar / 原始 whole scalar 7.002%；原始 observed 23.595%
whole scalar−shadow scalar：Σ 247,768 cycles；逐核 最小 2,570；最大 2,596		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 89.772%
参考 Non-scalar / 参考 Phase total 10.228%
参考 Phase total / 原始 whole total 9.404%；原始 observed 33.552%
参考 Phase scalar / 原始 whole scalar 8.729%；原始 observed 25.171%
whole scalar−shadow scalar：Σ 82,912 cycles；逐核 最小 2,586；最大 2,596		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 75.231%
参考 Non-scalar / 参考 Phase total 24.769%
参考 Phase total / 原始 whole total 7.262%；原始 observed 30.451%
参考 Phase scalar / 原始 whole scalar 6.161%；原始 observed 22.827%
whole scalar−shadow scalar：Σ 164,856 cycles；逐核 最小 2,570；最大 2,586
Request（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 8,011,606.393
原始 observed 14,936,953 − 记录代码开销估算 6,925,346.607
参考值 / 原始整窗 8.984%；原始 observed / 原始整窗 16.750%
记录代码开销估算 / 原始 observed 46.364%
原始逐核 最小 140,297；最大 178,475；原始整窗 89,175,746
原始 capture gap +0；加 gap 后 14,936,953（16.750%）		扣除记录代码开销估算后的参考值 3,070,446.128
原始 observed 5,319,507 − 记录代码开销估算 2,249,060.872
参考值 / 原始整窗 10.227%；原始 observed / 原始整窗 17.719%
记录代码开销估算 / 原始 observed 42.279%
原始逐核 最小 151,318；最大 178,475；原始整窗 30,021,935
原始 capture gap +0；加 gap 后 5,319,507（17.719%）		扣除记录代码开销估算后的参考值 4,941,160.265
原始 observed 9,617,446 − 记录代码开销估算 4,676,285.735
参考值 / 原始整窗 8.353%；原始 observed / 原始整窗 16.258%
记录代码开销估算 / 原始 observed 48.623%
原始逐核 最小 140,297；最大 166,456；原始整窗 59,153,811
原始 capture gap +0；加 gap 后 9,617,446（16.258%）
Miss（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 287,740.850
原始 observed 524,078 − 记录代码开销估算 236,337.150
参考值 / 原始整窗 8.964%；原始 observed / 原始整窗 16.327%
记录代码开销估算 / 原始 observed 45.096%
原始逐核 最小 1,948；最大 7,856；原始整窗 3,209,861
原始 capture gap +0；加 gap 后 524,078（16.327%）		扣除记录代码开销估算后的参考值 67,186.860
原始 observed 69,706 − 记录代码开销估算 2,519.140
参考值 / 原始整窗 26.501%；原始 observed / 原始整窗 27.494%
记录代码开销估算 / 原始 observed 3.614%
原始逐核 最小 1,948；最大 2,377；原始整窗 253,528
原始 capture gap +0；加 gap 后 69,706（27.494%）		扣除记录代码开销估算后的参考值 220,553.990
原始 observed 454,372 − 记录代码开销估算 233,818.010
参考值 / 原始整窗 7.460%；原始 observed / 原始整窗 15.369%
记录代码开销估算 / 原始 observed 51.460%
原始逐核 最小 6,449；最大 7,856；原始整窗 2,956,333
原始 capture gap +0；加 gap 后 454,372（15.369%）
SYS 边界诊断 / Begin-EndΣ 20,863,410 raw ticks
逐核 最小 174,276；最大 278,930；仅边界诊断
Begin / End：123,884 / 123,884
业务调用 122,880 次；排除 linked Kernel 调用 1,004 次		Σ 7,635,465 raw ticks
逐核 最小 196,647；最大 278,930；仅边界诊断
Begin / End：41,456 / 41,456
业务调用 40,960 次；排除 linked Kernel 调用 496 次		Σ 13,227,945 raw ticks
逐核 最小 174,276；最大 265,361；仅边界诊断
Begin / End：82,428 / 82,428
业务调用 81,920 次；排除 linked Kernel 调用 508 次
+
+ + +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
66fb698587973b534b66a713f09c795c7f222e528885d08debf5d125d8d51e93
+
Profile / extra cache
submit-pmu-efdrain-control / + 88075a1848686623
+
Profiled cache key
TestPagedAttentionUnroll / a5 / fully_distributed_within_core / submit-pmu-efdrain-control
+
Source state
source-v2:8c6b6e3a412d2c7c7113c2d8f714a14eb3786f80:dbe70b248d7f6b11b5df6eb1f0ccbf4d0a6533f42811dddebd502dcbbed9a014:05e42706404d17ae251ecea9cbff9e2c026cb10b74428f37739126bf509ffd78
+
Source-state stamp
/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/88075a1848686623/aicore/.git_commit
+
Compile definitions
PTO_FDWIC_SUBMIT_PMU=1
PTO_FDWIC_SUBMIT_PMU_PHASE_ID=7
PTO_FDWIC_TRACE_ENABLED=0
+
+
+ + + +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
AICore final2,712,82449689ae673b316dbe7be81b2306a79f4199404edb2ea973c16631bd21a1b226c221,520e5d2785154866aba81bce6b0d67877d6e6e2edb7b783781f772f8cf94cc4901d/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/aicore-extra/88075a1848686623/aicore_kernel.o
AIC combined2,041,816721294f0c213fa1e8582ab742951c632264e7e7d7e7708d0d10c6bcac413e1e8103,720f3095ca8c9fdeaf4725948162406d1d486c61c40d6aa7567c22f27719638e4ed/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/88075a1848686623/aicore/aicore_aic_combined.o
AIV combined2,303,40048f3fd77cc3a6fa9cdcc89276b8ac3b8227a4a6d0a84cb984dab3798acfab8bf117,584bbd5a08e2c81cef44f5b42428de6cb7359d9fadd0c86fde3e6d80ee30f124436/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/88075a1848686623/aicore/aicore_aiv_combined.o
Host runtime11,776,1440369d0602054bf612f86b58c294503e00486bd505bff9ee7a455994f043147cc454,6325ac381ed10be5f5d492b8072c051a177131f4caec6d6f6bc9bf50880da4bb0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so
+
+ +
+
原始全局 Submit 时间范围（墙钟） + 5,669.558 µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + 128096 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-2567cd701eacb12788261320216aeb70839f9bf8b96c095559799c006fe02b0466e
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + 90.000 ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×90 ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
+
+

ALL · 96 核

+
+
Submit PMU total/core
+
均值 8,597,625.7；最小 7,731,038；最大 9,178,279 cycles
+ 等效时间 均值 5,211.175；最小 4,685.921；最大 5,563.119 µs + （按 1.649844 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,837,752.7；最小 7,473,880；最大 8,149,998 cycles
+ 等效时间 均值 4,750.602；最小 4,530.053；最大 4,939.860 µs； + 加权占比 91.162% +
+
非 Scalar-busy 残余/core
+
均值 759,873.0；最小 214,916；最大 1,188,414 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 460.573；最小 130.264；最大 720.319 µs +
+
SYS gate 边界诊断/core
+
均值 4,795,615.5；最小 4,311,365；最大 5,109,971 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,544.488；最小 5,355.024；最大 5,668.541 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 748,872.6；最小 519,890；最大 1,070,564 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 86.493% +
+
Primary I-cache request/core
最小 902,761；最大 959,565
+
Primary I-cache miss/core
最小 7,125；最大 47,666
+
加权 miss rate
3.599%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 641.250；最大 4,289.940 µs
+
+
+ +
+

AIC · 32 核

+
+
Submit PMU total/core
+
均值 7,967,140.8；最小 7,731,038；最大 8,131,614 cycles
+ 等效时间 均值 4,828.389；最小 4,685.302；最大 4,928.066 µs + （按 1.650062 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,705,748.2；最小 7,473,880；最大 7,880,170 cycles
+ 等效时间 均值 4,669.975；最小 4,529.454；最大 4,775.681 µs； + 加权占比 96.719% +
+
非 Scalar-busy 残余/core
+
均值 261,392.7；最小 214,916；最大 292,389 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 158.414；最小 130.247；最大 177.199 µs +
+
SYS gate 边界诊断/core
+
均值 4,425,774.9；最小 4,311,365；最大 4,527,260 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,385.801；最小 5,355.024；最大 5,437.909 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 960,025.9；最小 864,283；最大 1,070,564 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 82.175% +
+
Primary I-cache request/core
最小 917,723；最大 955,358
+
Primary I-cache miss/core
最小 7,125；最大 8,686
+
加权 miss rate
0.844%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 641.250；最大 781.740 µs
+
+
+ +
+

AIV · 64 核

+
+
Submit PMU total/core
+
均值 8,912,868.1；最小 8,709,090；最大 9,178,279 cycles
+ 等效时间 均值 5,402.619；最小 5,279.097；最大 5,563.500 µs + （按 1.649731 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,903,754.9；最小 7,629,685；最大 8,149,998 cycles
+ 等效时间 均值 4,790.936；最小 4,624.805；最大 4,940.198 µs； + 加权占比 88.678% +
+
非 Scalar-busy 残余/core
+
均值 1,009,113.1；最小 875,712；最大 1,188,414 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 611.683；最小 530.821；最大 720.368 µs +
+
SYS gate 边界诊断/core
+
均值 4,980,535.8；最小 4,875,119；最大 5,109,971 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,623.832；最小 5,596.466；最大 5,668.541 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 643,295.9；最小 519,890；最大 752,785 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 88.561% +
+
Primary I-cache request/core
最小 902,761；最大 959,565
+
Primary I-cache miss/core
最小 45,183；最大 47,666
+
加权 miss rate
4.998%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 4,066.470；最大 4,289.940 µs
+
+
+
+

逐物理核分布

+

AICAIV

+
+
+
SYS gate boundary-diagnostic ticks/core
+ + + + 5,109,971 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_submit_elapsed_ticks=4486886physical=1 logical=1 role=AIC scalar_submit_elapsed_ticks=4368696physical=2 logical=2 role=AIC scalar_submit_elapsed_ticks=4511395physical=3 logical=3 role=AIC scalar_submit_elapsed_ticks=4384986physical=4 logical=4 role=AIC scalar_submit_elapsed_ticks=4505402physical=5 logical=5 role=AIC scalar_submit_elapsed_ticks=4311365physical=6 logical=6 role=AIC scalar_submit_elapsed_ticks=4456183physical=7 logical=7 role=AIC scalar_submit_elapsed_ticks=4429616physical=8 logical=8 role=AIC scalar_submit_elapsed_ticks=4495958physical=9 logical=9 role=AIC scalar_submit_elapsed_ticks=4346786physical=10 logical=10 role=AIC scalar_submit_elapsed_ticks=4493851physical=12 logical=11 role=AIC scalar_submit_elapsed_ticks=4384135physical=13 logical=12 role=AIC scalar_submit_elapsed_ticks=4441582physical=14 logical=13 role=AIC scalar_submit_elapsed_ticks=4334739physical=15 logical=14 role=AIC scalar_submit_elapsed_ticks=4515523physical=16 logical=15 role=AIC scalar_submit_elapsed_ticks=4363134physical=54 logical=16 role=AIC scalar_submit_elapsed_ticks=4482645physical=55 logical=17 role=AIC scalar_submit_elapsed_ticks=4384102physical=56 logical=18 role=AIC scalar_submit_elapsed_ticks=4527260physical=57 logical=19 role=AIC scalar_submit_elapsed_ticks=4385830physical=58 logical=20 role=AIC scalar_submit_elapsed_ticks=4457350physical=60 logical=21 role=AIC scalar_submit_elapsed_ticks=4314100physical=61 logical=22 role=AIC scalar_submit_elapsed_ticks=4507653physical=62 logical=23 role=AIC scalar_submit_elapsed_ticks=4426406physical=63 logical=24 role=AIC scalar_submit_elapsed_ticks=4474393physical=64 logical=25 role=AIC scalar_submit_elapsed_ticks=4368634physical=65 logical=26 role=AIC scalar_submit_elapsed_ticks=4469735physical=66 logical=27 role=AIC scalar_submit_elapsed_ticks=4341522physical=67 logical=28 role=AIC scalar_submit_elapsed_ticks=4386814physical=68 logical=29 role=AIC scalar_submit_elapsed_ticks=4401341physical=69 logical=30 role=AIC scalar_submit_elapsed_ticks=4464422physical=70 logical=31 role=AIC scalar_submit_elapsed_ticks=4402353physical=18 logical=32 role=AIV scalar_submit_elapsed_ticks=5085537physical=19 logical=33 role=AIV scalar_submit_elapsed_ticks=5002137physical=20 logical=34 role=AIV scalar_submit_elapsed_ticks=5039938physical=21 logical=35 role=AIV scalar_submit_elapsed_ticks=5019718physical=22 logical=36 role=AIV scalar_submit_elapsed_ticks=5037539physical=23 logical=37 role=AIV scalar_submit_elapsed_ticks=5051706physical=24 logical=38 role=AIV scalar_submit_elapsed_ticks=5052988physical=25 logical=39 role=AIV scalar_submit_elapsed_ticks=4939157physical=26 logical=40 role=AIV scalar_submit_elapsed_ticks=4979762physical=27 logical=41 role=AIV scalar_submit_elapsed_ticks=4933025physical=28 logical=42 role=AIV scalar_submit_elapsed_ticks=4996853physical=29 logical=43 role=AIV scalar_submit_elapsed_ticks=4882437physical=30 logical=44 role=AIV scalar_submit_elapsed_ticks=5031492physical=31 logical=45 role=AIV scalar_submit_elapsed_ticks=4906981physical=32 logical=46 role=AIV scalar_submit_elapsed_ticks=5017709physical=33 logical=47 role=AIV scalar_submit_elapsed_ticks=4937731physical=34 logical=48 role=AIV scalar_submit_elapsed_ticks=5005303physical=35 logical=49 role=AIV scalar_submit_elapsed_ticks=4875119physical=36 logical=50 role=AIV scalar_submit_elapsed_ticks=5010206physical=37 logical=51 role=AIV scalar_submit_elapsed_ticks=4909455physical=38 logical=52 role=AIV scalar_submit_elapsed_ticks=4996202physical=39 logical=53 role=AIV scalar_submit_elapsed_ticks=4918442physical=42 logical=54 role=AIV scalar_submit_elapsed_ticks=4977841physical=43 logical=55 role=AIV scalar_submit_elapsed_ticks=4981381physical=44 logical=56 role=AIV scalar_submit_elapsed_ticks=5014382physical=45 logical=57 role=AIV scalar_submit_elapsed_ticks=4990874physical=46 logical=58 role=AIV scalar_submit_elapsed_ticks=4888684physical=47 logical=59 role=AIV scalar_submit_elapsed_ticks=4987292physical=48 logical=60 role=AIV scalar_submit_elapsed_ticks=4948741physical=49 logical=61 role=AIV scalar_submit_elapsed_ticks=4936137physical=50 logical=62 role=AIV scalar_submit_elapsed_ticks=5038193physical=51 logical=63 role=AIV scalar_submit_elapsed_ticks=4939188physical=72 logical=64 role=AIV scalar_submit_elapsed_ticks=4939525physical=73 logical=65 role=AIV scalar_submit_elapsed_ticks=4897171physical=74 logical=66 role=AIV scalar_submit_elapsed_ticks=4989450physical=75 logical=67 role=AIV scalar_submit_elapsed_ticks=4931322physical=76 logical=68 role=AIV scalar_submit_elapsed_ticks=5054745physical=77 logical=69 role=AIV scalar_submit_elapsed_ticks=4931886physical=78 logical=70 role=AIV scalar_submit_elapsed_ticks=5025098physical=79 logical=71 role=AIV scalar_submit_elapsed_ticks=4926006physical=80 logical=72 role=AIV scalar_submit_elapsed_ticks=4961422physical=81 logical=73 role=AIV scalar_submit_elapsed_ticks=4996300physical=84 logical=74 role=AIV scalar_submit_elapsed_ticks=5066550physical=85 logical=75 role=AIV scalar_submit_elapsed_ticks=4978120physical=86 logical=76 role=AIV scalar_submit_elapsed_ticks=4978253physical=87 logical=77 role=AIV scalar_submit_elapsed_ticks=4912984physical=88 logical=78 role=AIV scalar_submit_elapsed_ticks=5008679physical=89 logical=79 role=AIV scalar_submit_elapsed_ticks=4910491physical=90 logical=80 role=AIV scalar_submit_elapsed_ticks=4977303physical=91 logical=81 role=AIV scalar_submit_elapsed_ticks=4975086physical=92 logical=82 role=AIV scalar_submit_elapsed_ticks=5058519physical=93 logical=83 role=AIV scalar_submit_elapsed_ticks=4964823physical=94 logical=84 role=AIV scalar_submit_elapsed_ticks=4990745physical=95 logical=85 role=AIV scalar_submit_elapsed_ticks=4984058physical=96 logical=86 role=AIV scalar_submit_elapsed_ticks=5019935physical=97 logical=87 role=AIV scalar_submit_elapsed_ticks=5009084physical=98 logical=88 role=AIV scalar_submit_elapsed_ticks=4926601physical=99 logical=89 role=AIV scalar_submit_elapsed_ticks=5109971physical=100 logical=90 role=AIV scalar_submit_elapsed_ticks=4999702physical=101 logical=91 role=AIV scalar_submit_elapsed_ticks=5004146physical=102 logical=92 role=AIV scalar_submit_elapsed_ticks=4968007physical=103 logical=93 role=AIV scalar_submit_elapsed_ticks=4904710physical=104 logical=94 role=AIV scalar_submit_elapsed_ticks=5039164physical=105 logical=95 role=AIV scalar_submit_elapsed_ticks=4982287 + +
+ +
+
PMU total cycles/core
+ + + + 9,178,279 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC total_cycles=8085842physical=1 logical=1 role=AIC total_cycles=7848026physical=2 logical=2 role=AIC total_cycles=8109659physical=3 logical=3 role=AIC total_cycles=7932472physical=4 logical=4 role=AIC total_cycles=8106104physical=5 logical=5 role=AIC total_cycles=7895391physical=6 logical=6 role=AIC total_cycles=7948644physical=7 logical=7 role=AIC total_cycles=8045188physical=8 logical=8 role=AIC total_cycles=8040905physical=9 logical=9 role=AIC total_cycles=7943534physical=10 logical=10 role=AIC total_cycles=8051287physical=12 logical=11 role=AIC total_cycles=7876387physical=13 logical=12 role=AIC total_cycles=7925144physical=14 logical=13 role=AIC total_cycles=7852594physical=15 logical=14 role=AIC total_cycles=8049394physical=16 logical=15 role=AIC total_cycles=7913311physical=54 logical=16 role=AIC total_cycles=8049304physical=55 logical=17 role=AIC total_cycles=7961113physical=56 logical=18 role=AIC total_cycles=8112598physical=57 logical=19 role=AIC total_cycles=7965009physical=58 logical=20 role=AIC total_cycles=7977754physical=60 logical=21 role=AIC total_cycles=7731038physical=61 logical=22 role=AIC total_cycles=8131614physical=62 logical=23 role=AIC total_cycles=8031912physical=63 logical=24 role=AIC total_cycles=8025790physical=64 logical=25 role=AIC total_cycles=7847499physical=65 logical=26 role=AIC total_cycles=7999668physical=66 logical=27 role=AIC total_cycles=7806875physical=67 logical=28 role=AIC total_cycles=7812520physical=68 logical=29 role=AIC total_cycles=7906245physical=69 logical=30 role=AIC total_cycles=7995584physical=70 logical=31 role=AIC total_cycles=7970101physical=18 logical=32 role=AIV total_cycles=9069924physical=19 logical=33 role=AIV total_cycles=8896560physical=20 logical=34 role=AIV total_cycles=8974048physical=21 logical=35 role=AIV total_cycles=9036266physical=22 logical=36 role=AIV total_cycles=8989905physical=23 logical=37 role=AIV total_cycles=9087686physical=24 logical=38 role=AIV total_cycles=9003932physical=25 logical=39 role=AIV total_cycles=8868135physical=26 logical=40 role=AIV total_cycles=8899331physical=27 logical=41 role=AIV total_cycles=8869487physical=28 logical=42 role=AIV total_cycles=8918673physical=29 logical=43 role=AIV total_cycles=8799753physical=30 logical=44 role=AIV total_cycles=9041375physical=31 logical=45 role=AIV total_cycles=8841652physical=32 logical=46 role=AIV total_cycles=8956334physical=33 logical=47 role=AIV total_cycles=8844335physical=34 logical=48 role=AIV total_cycles=8899563physical=35 logical=49 role=AIV total_cycles=8739471physical=36 logical=50 role=AIV total_cycles=8985527physical=37 logical=51 role=AIV total_cycles=8754045physical=38 logical=52 role=AIV total_cycles=8915433physical=39 logical=53 role=AIV total_cycles=8863472physical=42 logical=54 role=AIV total_cycles=8887917physical=43 logical=55 role=AIV total_cycles=8899780physical=44 logical=56 role=AIV total_cycles=8918042physical=45 logical=57 role=AIV total_cycles=8970893physical=46 logical=58 role=AIV total_cycles=8753648physical=47 logical=59 role=AIV total_cycles=8934593physical=48 logical=60 role=AIV total_cycles=8803206physical=49 logical=61 role=AIV total_cycles=8860114physical=50 logical=62 role=AIV total_cycles=9065063physical=51 logical=63 role=AIV total_cycles=8886610physical=72 logical=64 role=AIV total_cycles=8879309physical=73 logical=65 role=AIV total_cycles=8800065physical=74 logical=66 role=AIV total_cycles=8926434physical=75 logical=67 role=AIV total_cycles=8861336physical=76 logical=68 role=AIV total_cycles=9066832physical=77 logical=69 role=AIV total_cycles=8834280physical=78 logical=70 role=AIV total_cycles=8964424physical=79 logical=71 role=AIV total_cycles=8856296physical=80 logical=72 role=AIV total_cycles=8819817physical=81 logical=73 role=AIV total_cycles=9058620physical=84 logical=74 role=AIV total_cycles=8997447physical=85 logical=75 role=AIV total_cycles=8888849physical=86 logical=76 role=AIV total_cycles=8895079physical=87 logical=77 role=AIV total_cycles=8823277physical=88 logical=78 role=AIV total_cycles=8890435physical=89 logical=79 role=AIV total_cycles=8825318physical=90 logical=80 role=AIV total_cycles=8962941physical=91 logical=81 role=AIV total_cycles=9004979physical=92 logical=82 role=AIV total_cycles=8977841physical=93 logical=83 role=AIV total_cycles=8884507physical=94 logical=84 role=AIV total_cycles=8881605physical=95 logical=85 role=AIV total_cycles=8972046physical=96 logical=86 role=AIV total_cycles=8903486physical=97 logical=87 role=AIV total_cycles=9045124physical=98 logical=88 role=AIV total_cycles=8768781physical=99 logical=89 role=AIV total_cycles=9178279physical=100 logical=90 role=AIV total_cycles=8832091physical=101 logical=91 role=AIV total_cycles=8895159physical=102 logical=92 role=AIV total_cycles=8836444physical=103 logical=93 role=AIV total_cycles=8709090physical=104 logical=94 role=AIV total_cycles=8993827physical=105 logical=95 role=AIV total_cycles=8954766 + +
+ +
+
Scalar busy cycles/core
+ + + + 8,149,998 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_busy=7813320physical=1 logical=1 role=AIC scalar_busy=7561830physical=2 logical=2 role=AIC scalar_busy=7819093physical=3 logical=3 role=AIC scalar_busy=7646972physical=4 logical=4 role=AIC scalar_busy=7827825physical=5 logical=5 role=AIC scalar_busy=7634616physical=6 logical=6 role=AIC scalar_busy=7691091physical=7 logical=7 role=AIC scalar_busy=7761581physical=8 logical=8 role=AIC scalar_busy=7798484physical=9 logical=9 role=AIC scalar_busy=7679330physical=10 logical=10 role=AIC scalar_busy=7809686physical=12 logical=11 role=AIC scalar_busy=7595597physical=13 logical=12 role=AIC scalar_busy=7708195physical=14 logical=13 role=AIC scalar_busy=7581989physical=15 logical=14 role=AIC scalar_busy=7809764physical=16 logical=15 role=AIC scalar_busy=7641230physical=54 logical=16 role=AIC scalar_busy=7834388physical=55 logical=17 role=AIC scalar_busy=7701487physical=56 logical=18 role=AIC scalar_busy=7862186physical=57 logical=19 role=AIC scalar_busy=7692056physical=58 logical=20 role=AIC scalar_busy=7694244physical=60 logical=21 role=AIC scalar_busy=7473880physical=61 logical=22 role=AIC scalar_busy=7880170physical=62 logical=23 role=AIC scalar_busy=7763369physical=63 logical=24 role=AIC scalar_busy=7770987physical=64 logical=25 role=AIC scalar_busy=7564879physical=65 logical=26 role=AIC scalar_busy=7771286physical=66 logical=27 role=AIC scalar_busy=7552272physical=67 logical=28 role=AIC scalar_busy=7575050physical=68 logical=29 role=AIC scalar_busy=7613856physical=69 logical=30 role=AIC scalar_busy=7739143physical=70 logical=31 role=AIC scalar_busy=7714085physical=18 logical=32 role=AIV scalar_busy=8009787physical=19 logical=33 role=AIV scalar_busy=7800650physical=20 logical=34 role=AIV scalar_busy=7939791physical=21 logical=35 role=AIV scalar_busy=7847852physical=22 logical=36 role=AIV scalar_busy=7967263physical=23 logical=37 role=AIV scalar_busy=7990265physical=24 logical=38 role=AIV scalar_busy=7951428physical=25 logical=39 role=AIV scalar_busy=7775196physical=26 logical=40 role=AIV scalar_busy=7987486physical=27 logical=41 role=AIV scalar_busy=7828131physical=28 logical=42 role=AIV scalar_busy=7930698physical=29 logical=43 role=AIV scalar_busy=7750304physical=30 logical=44 role=AIV scalar_busy=8055440physical=31 logical=45 role=AIV scalar_busy=7795901physical=32 logical=46 role=AIV scalar_busy=7981162physical=33 logical=47 role=AIV scalar_busy=7751541physical=34 logical=48 role=AIV scalar_busy=7948525physical=35 logical=49 role=AIV scalar_busy=7741998physical=36 logical=50 role=AIV scalar_busy=7972868physical=37 logical=51 role=AIV scalar_busy=7707895physical=38 logical=52 role=AIV scalar_busy=8000113physical=39 logical=53 role=AIV scalar_busy=7877512physical=42 logical=54 role=AIV scalar_busy=7902843physical=43 logical=55 role=AIV scalar_busy=7851865physical=44 logical=56 role=AIV scalar_busy=7948485physical=45 logical=57 role=AIV scalar_busy=7888104physical=46 logical=58 role=AIV scalar_busy=7833094physical=47 logical=59 role=AIV scalar_busy=7939507physical=48 logical=60 role=AIV scalar_busy=7855721physical=49 logical=61 role=AIV scalar_busy=7877140physical=50 logical=62 role=AIV scalar_busy=8149998physical=51 logical=63 role=AIV scalar_busy=7904265physical=72 logical=64 role=AIV scalar_busy=7954668physical=73 logical=65 role=AIV scalar_busy=7781490physical=74 logical=66 role=AIV scalar_busy=7995425physical=75 logical=67 role=AIV scalar_busy=7900808physical=76 logical=68 role=AIV scalar_busy=8081143physical=77 logical=69 role=AIV scalar_busy=7787397physical=78 logical=70 role=AIV scalar_busy=8056038physical=79 logical=71 role=AIV scalar_busy=7854882physical=80 logical=72 role=AIV scalar_busy=7863631physical=81 logical=73 role=AIV scalar_busy=7973139physical=84 logical=74 role=AIV scalar_busy=8048334physical=85 logical=75 role=AIV scalar_busy=7852892physical=86 logical=76 role=AIV scalar_busy=8019367physical=87 logical=77 role=AIV scalar_busy=7831084physical=88 logical=78 role=AIV scalar_busy=7934342physical=89 logical=79 role=AIV scalar_busy=7788574physical=90 logical=80 role=AIV scalar_busy=8050516physical=91 logical=81 role=AIV scalar_busy=7959759physical=92 logical=82 role=AIV scalar_busy=7975919physical=93 logical=83 role=AIV scalar_busy=7837757physical=94 logical=84 role=AIV scalar_busy=7911782physical=95 logical=85 role=AIV scalar_busy=7953594physical=96 logical=86 role=AIV scalar_busy=7925131physical=97 logical=87 role=AIV scalar_busy=7957576physical=98 logical=88 role=AIV scalar_busy=7777696physical=99 logical=89 role=AIV scalar_busy=8133992physical=100 logical=90 role=AIV scalar_busy=7849854physical=101 logical=91 role=AIV scalar_busy=7778082physical=102 logical=92 role=AIV scalar_busy=7851798physical=103 logical=93 role=AIV scalar_busy=7629685physical=104 logical=94 role=AIV scalar_busy=7924423physical=105 logical=95 role=AIV scalar_busy=7836710 + +
+ +
+
Primary I-cache requests/core
+ + + + 959,565 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_requests=938629physical=1 logical=1 role=AIC icache_requests=936212physical=2 logical=2 role=AIC icache_requests=933675physical=3 logical=3 role=AIC icache_requests=941166physical=4 logical=4 role=AIC icache_requests=936312physical=5 logical=5 role=AIC icache_requests=948399physical=6 logical=6 role=AIC icache_requests=926583physical=7 logical=7 role=AIC icache_requests=955358physical=8 logical=8 role=AIC icache_requests=929660physical=9 logical=9 role=AIC icache_requests=953317physical=10 logical=10 role=AIC icache_requests=925314physical=12 logical=11 role=AIC icache_requests=942344physical=13 logical=12 role=AIC icache_requests=922009physical=14 logical=13 role=AIC icache_requests=948178physical=15 logical=14 role=AIC icache_requests=922314physical=16 logical=15 role=AIC icache_requests=950732physical=54 logical=16 role=AIC icache_requests=932538physical=55 logical=17 role=AIC icache_requests=952678physical=56 logical=18 role=AIC icache_requests=931339physical=57 logical=19 role=AIC icache_requests=949140physical=58 logical=20 role=AIC icache_requests=926724physical=60 logical=21 role=AIC icache_requests=933540physical=61 logical=22 role=AIC icache_requests=938514physical=62 logical=23 role=AIC icache_requests=951873physical=63 logical=24 role=AIC icache_requests=935745physical=64 logical=25 role=AIC icache_requests=939290physical=65 logical=26 role=AIC icache_requests=930103physical=66 logical=27 role=AIC icache_requests=941628physical=67 logical=28 role=AIC icache_requests=917723physical=68 logical=29 role=AIC icache_requests=947258physical=69 logical=30 role=AIC icache_requests=932867physical=70 logical=31 role=AIC icache_requests=950773physical=18 logical=32 role=AIV icache_requests=919342physical=19 logical=33 role=AIV icache_requests=903967physical=20 logical=34 role=AIV icache_requests=910551physical=21 logical=35 role=AIV icache_requests=942128physical=22 logical=36 role=AIV icache_requests=918572physical=23 logical=37 role=AIV icache_requests=947028physical=24 logical=38 role=AIV icache_requests=915060physical=25 logical=39 role=AIV icache_requests=915026physical=26 logical=40 role=AIV icache_requests=912267physical=27 logical=41 role=AIV icache_requests=923427physical=28 logical=42 role=AIV icache_requests=911403physical=29 logical=43 role=AIV icache_requests=924234physical=30 logical=44 role=AIV icache_requests=932268physical=31 logical=45 role=AIV icache_requests=930551physical=32 logical=46 role=AIV icache_requests=920805physical=33 logical=47 role=AIV icache_requests=916430physical=34 logical=48 role=AIV icache_requests=905489physical=35 logical=49 role=AIV icache_requests=923425physical=36 logical=50 role=AIV icache_requests=921957physical=37 logical=51 role=AIV icache_requests=910165physical=38 logical=52 role=AIV icache_requests=923706physical=39 logical=53 role=AIV icache_requests=930716physical=42 logical=54 role=AIV icache_requests=917366physical=43 logical=55 role=AIV icache_requests=930817physical=44 logical=56 role=AIV icache_requests=911524physical=45 logical=57 role=AIV icache_requests=939461physical=46 logical=58 role=AIV icache_requests=917813physical=47 logical=59 role=AIV icache_requests=923675physical=48 logical=60 role=AIV icache_requests=906258physical=49 logical=61 role=AIV icache_requests=927119physical=50 logical=62 role=AIV icache_requests=944730physical=51 logical=63 role=AIV icache_requests=933820physical=72 logical=64 role=AIV icache_requests=926492physical=73 logical=65 role=AIV icache_requests=930756physical=74 logical=66 role=AIV icache_requests=923269physical=75 logical=67 role=AIV icache_requests=927171physical=76 logical=68 role=AIV icache_requests=931512physical=77 logical=69 role=AIV icache_requests=925273physical=78 logical=70 role=AIV icache_requests=930284physical=79 logical=71 role=AIV icache_requests=933106physical=80 logical=72 role=AIV icache_requests=908497physical=81 logical=73 role=AIV icache_requests=956780physical=84 logical=74 role=AIV icache_requests=915287physical=85 logical=75 role=AIV icache_requests=925482physical=86 logical=76 role=AIV icache_requests=917927physical=87 logical=77 role=AIV icache_requests=926827physical=88 logical=78 role=AIV icache_requests=918676physical=89 logical=79 role=AIV icache_requests=933156physical=90 logical=80 role=AIV icache_requests=931061physical=91 logical=81 role=AIV icache_requests=959565physical=92 logical=82 role=AIV icache_requests=909101physical=93 logical=83 role=AIV icache_requests=930215physical=94 logical=84 role=AIV icache_requests=914460physical=95 logical=85 role=AIV icache_requests=944933physical=96 logical=86 role=AIV icache_requests=913282physical=97 logical=87 role=AIV icache_requests=957637physical=98 logical=88 role=AIV icache_requests=915929physical=99 logical=89 role=AIV icache_requests=943603physical=100 logical=90 role=AIV icache_requests=902761physical=101 logical=91 role=AIV icache_requests=918319physical=102 logical=92 role=AIV icache_requests=907662physical=103 logical=93 role=AIV icache_requests=906814physical=104 logical=94 role=AIV icache_requests=921947physical=105 logical=95 role=AIV icache_requests=934927 + +
+ +
+
Primary I-cache misses/core
+ + + + 47,666 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_misses=7730physical=1 logical=1 role=AIC icache_misses=8126physical=2 logical=2 role=AIC icache_misses=7773physical=3 logical=3 role=AIC icache_misses=8211physical=4 logical=4 role=AIC icache_misses=7953physical=5 logical=5 role=AIC icache_misses=8119physical=6 logical=6 role=AIC icache_misses=7980physical=7 logical=7 role=AIC icache_misses=8686physical=8 logical=8 role=AIC icache_misses=7450physical=9 logical=9 role=AIC icache_misses=8197physical=10 logical=10 role=AIC icache_misses=7885physical=12 logical=11 role=AIC icache_misses=7987physical=13 logical=12 role=AIC icache_misses=7868physical=14 logical=13 role=AIC icache_misses=8085physical=15 logical=14 role=AIC icache_misses=7262physical=16 logical=15 role=AIC icache_misses=8540physical=54 logical=16 role=AIC icache_misses=7439physical=55 logical=17 role=AIC icache_misses=7986physical=56 logical=18 role=AIC icache_misses=7383physical=57 logical=19 role=AIC icache_misses=8399physical=58 logical=20 role=AIC icache_misses=8089physical=60 logical=21 role=AIC icache_misses=8074physical=61 logical=22 role=AIC icache_misses=7673physical=62 logical=23 role=AIC icache_misses=8177physical=63 logical=24 role=AIC icache_misses=7651physical=64 logical=25 role=AIC icache_misses=8359physical=65 logical=26 role=AIC icache_misses=7413physical=66 logical=27 role=AIC icache_misses=8153physical=67 logical=28 role=AIC icache_misses=7125physical=68 logical=29 role=AIC icache_misses=8035physical=69 logical=30 role=AIC icache_misses=7469physical=70 logical=31 role=AIC icache_misses=8251physical=18 logical=32 role=AIV icache_misses=45785physical=19 logical=33 role=AIV icache_misses=45238physical=20 logical=34 role=AIV icache_misses=45261physical=21 logical=35 role=AIV icache_misses=46087physical=22 logical=36 role=AIV icache_misses=45954physical=23 logical=37 role=AIV icache_misses=47063physical=24 logical=38 role=AIV icache_misses=45183physical=25 logical=39 role=AIV icache_misses=46028physical=26 logical=40 role=AIV icache_misses=46609physical=27 logical=41 role=AIV icache_misses=46654physical=28 logical=42 role=AIV icache_misses=45988physical=29 logical=43 role=AIV icache_misses=47012physical=30 logical=44 role=AIV icache_misses=46177physical=31 logical=45 role=AIV icache_misses=47019physical=32 logical=46 role=AIV icache_misses=45561physical=33 logical=47 role=AIV icache_misses=46306physical=34 logical=48 role=AIV icache_misses=45326physical=35 logical=49 role=AIV icache_misses=46581physical=36 logical=50 role=AIV icache_misses=46371physical=37 logical=51 role=AIV icache_misses=45975physical=38 logical=52 role=AIV icache_misses=46234physical=39 logical=53 role=AIV icache_misses=46385physical=42 logical=54 role=AIV icache_misses=46357physical=43 logical=55 role=AIV icache_misses=47149physical=44 logical=56 role=AIV icache_misses=46376physical=45 logical=57 role=AIV icache_misses=46302physical=46 logical=58 role=AIV icache_misses=46060physical=47 logical=59 role=AIV icache_misses=45902physical=48 logical=60 role=AIV icache_misses=45386physical=49 logical=61 role=AIV icache_misses=46077physical=50 logical=62 role=AIV icache_misses=46036physical=51 logical=63 role=AIV icache_misses=46453physical=72 logical=64 role=AIV icache_misses=45837physical=73 logical=65 role=AIV icache_misses=46319physical=74 logical=66 role=AIV icache_misses=46190physical=75 logical=67 role=AIV icache_misses=46337physical=76 logical=68 role=AIV icache_misses=45828physical=77 logical=69 role=AIV icache_misses=46424physical=78 logical=70 role=AIV icache_misses=46322physical=79 logical=71 role=AIV icache_misses=46460physical=80 logical=72 role=AIV icache_misses=45257physical=81 logical=73 role=AIV icache_misses=47253physical=84 logical=74 role=AIV icache_misses=45746physical=85 logical=75 role=AIV icache_misses=45991physical=86 logical=76 role=AIV icache_misses=45801physical=87 logical=77 role=AIV icache_misses=46607physical=88 logical=78 role=AIV icache_misses=45447physical=89 logical=79 role=AIV icache_misses=46334physical=90 logical=80 role=AIV icache_misses=46196physical=91 logical=81 role=AIV icache_misses=47666physical=92 logical=82 role=AIV icache_misses=45698physical=93 logical=83 role=AIV icache_misses=45952physical=94 logical=84 role=AIV icache_misses=46070physical=95 logical=85 role=AIV icache_misses=47012physical=96 logical=86 role=AIV icache_misses=46438physical=97 logical=87 role=AIV icache_misses=47265physical=98 logical=88 role=AIV icache_misses=46694physical=99 logical=89 role=AIV icache_misses=46819physical=100 logical=90 role=AIV icache_misses=45245physical=101 logical=91 role=AIV icache_misses=45992physical=102 logical=92 role=AIV icache_misses=45595physical=103 logical=93 role=AIV icache_misses=46241physical=104 logical=94 role=AIV icache_misses=45720physical=105 logical=95 role=AIV icache_misses=46682 + +
+
+

逐核原始主计数

+

报告展示 primary request/miss；phase 模式的 I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和，只用于标记全窗 capture gap；scalar shadow 同样用于核验 phase 读清重建，并显示其相对 CNT2 primary 的 capture loss，不作为第二份性能数据展示。

+
+ + + + + + + +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate90ns量尺 µs
00AIC004,486,8865,369.7368,085,8427,813,320938,6297,7300.824%695.700
11AIC104,368,6965,370.0667,848,0267,561,830936,2128,1260.868%731.340
22AIC204,511,3955,375.6788,109,6597,819,093933,6757,7730.833%699.570
33AIC304,384,9865,365.2797,932,4727,646,972941,1668,2110.872%738.990
44AIC404,505,4025,419.1578,106,1047,827,825936,3127,9530.849%715.770
55AIC504,311,3655,363.3127,895,3917,634,616948,3998,1190.856%730.710
66AIC604,456,1835,405.5887,948,6447,691,091926,5837,9800.861%718.200
77AIC704,429,6165,437.9098,045,1887,761,581955,3588,6860.909%781.740
88AIC804,495,9585,369.8158,040,9057,798,484929,6607,4500.801%670.500
99AIC904,346,7865,368.3907,943,5347,679,330953,3178,1970.860%737.730
1010AIC1004,493,8515,387.4588,051,2877,809,686925,3147,8850.852%709.650
1211AIC1104,384,1355,399.7487,876,3877,595,597942,3447,9870.848%718.830
1312AIC1204,441,5825,390.1207,925,1447,708,195922,0097,8680.853%708.120
1413AIC1304,334,7395,405.3037,852,5947,581,989948,1788,0850.853%727.650
1514AIC1404,515,5235,407.5678,049,3947,809,764922,3147,2620.787%653.580
1615AIC1504,363,1345,376.7817,913,3117,641,230950,7328,5400.898%768.600
1832AIV015,085,5375,612.2499,069,9248,009,787919,34245,7854.980%4,120.650
1933AIV025,002,1375,610.1948,896,5607,800,650903,96745,2385.004%4,071.420
2034AIV115,039,9385,657.8948,974,0487,939,791910,55145,2614.971%4,073.490
2135AIV125,019,7185,651.7219,036,2667,847,852942,12846,0874.892%4,147.830
2236AIV215,037,5395,668.5418,989,9057,967,263918,57245,9545.003%4,135.860
2337AIV225,051,7065,629.8719,087,6867,990,265947,02847,0634.970%4,235.670
2438AIV315,052,9885,625.2089,003,9327,951,428915,06045,1834.938%4,066.470
2539AIV324,939,1575,646.0248,868,1357,775,196915,02646,0285.030%4,142.520
2640AIV414,979,7625,615.1298,899,3317,987,486912,26746,6095.109%4,194.810
2741AIV424,933,0255,602.9068,869,4877,828,131923,42746,6545.052%4,198.860
2842AIV514,996,8535,622.0288,918,6737,930,698911,40345,9885.046%4,138.920
2943AIV524,882,4375,606.5698,799,7537,750,304924,23447,0125.087%4,231.080
3044AIV615,031,4925,598.0019,041,3758,055,440932,26846,1774.953%4,155.930
3145AIV624,906,9815,632.8278,841,6527,795,901930,55147,0195.053%4,231.710
3246AIV715,017,7095,596.4668,956,3347,981,162920,80545,5614.948%4,100.490
3347AIV724,937,7315,633.9458,844,3357,751,541916,43046,3065.053%4,167.540
3448AIV815,005,3035,609.3518,899,5637,948,525905,48945,3265.006%4,079.340
3549AIV824,875,1195,620.2508,739,4717,741,998923,42546,5815.044%4,192.290
3650AIV915,010,2065,614.2568,985,5277,972,868921,95746,3715.030%4,173.390
3751AIV924,909,4555,628.8618,754,0457,707,895910,16545,9755.051%4,137.750
3852AIV1014,996,2025,626.0318,915,4338,000,113923,70646,2345.005%4,161.060
3953AIV1024,918,4425,598.4068,863,4727,877,512930,71646,3854.984%4,174.650
4254AIV1114,977,8415,610.8468,887,9177,902,843917,36646,3575.053%4,172.130
4355AIV1124,981,3815,619.3078,899,7807,851,865930,81747,1495.065%4,243.410
4456AIV1215,014,3825,626.0408,918,0427,948,485911,52446,3765.088%4,173.840
4557AIV1224,990,8745,611.0768,970,8937,888,104939,46146,3024.929%4,167.180
4658AIV1314,888,6845,635.4968,753,6487,833,094917,81346,0605.018%4,145.400
4759AIV1324,987,2925,642.1548,934,5937,939,507923,67545,9024.969%4,131.180
4860AIV1414,948,7415,607.4028,803,2067,855,721906,25845,3865.008%4,084.740
4961AIV1424,936,1375,644.5868,860,1147,877,140927,11946,0774.970%4,146.930
5062AIV1515,038,1935,614.4889,065,0638,149,998944,73046,0364.873%4,143.240
5163AIV1524,939,1885,609.6848,886,6107,904,265933,82046,4534.975%4,180.770
5416AIC1604,482,6455,355.0248,049,3047,834,388932,5387,4390.798%669.510
5517AIC1704,384,1025,414.8247,961,1137,701,487952,6787,9860.838%718.740
5618AIC1804,527,2605,411.5118,112,5987,862,186931,3397,3830.793%664.470
5719AIC1904,385,8305,375.6927,965,0097,692,056949,1408,3990.885%755.910
5820AIC2004,457,3505,366.2447,977,7547,694,244926,7248,0890.873%728.010
6021AIC2104,314,1005,363.2847,731,0387,473,880933,5408,0740.865%726.660
6122AIC2204,507,6535,376.1218,131,6147,880,170938,5147,6730.818%690.570
6223AIC2304,426,4065,431.0098,031,9127,763,369951,8738,1770.859%735.930
6324AIC2404,474,3935,389.2508,025,7907,770,987935,7457,6510.818%688.590
6425AIC2504,368,6345,363.5457,847,4997,564,879939,2908,3590.890%752.310
6526AIC2604,469,7355,358.9987,999,6687,771,286930,1037,4130.797%667.170
6627AIC2704,341,5225,401.9127,806,8757,552,272941,6288,1530.866%733.770
6728AIC2804,386,8145,383.2017,812,5207,575,050917,7237,1250.776%641.250
6829AIC2904,401,3415,386.2327,906,2457,613,856947,2588,0350.848%723.150
6930AIC3004,464,4225,393.9977,995,5847,739,143932,8677,4690.801%672.210
7031AIC3104,402,3535,362.8747,970,1017,714,085950,7738,2510.868%742.590
7264AIV1614,939,5255,655.5468,879,3097,954,668926,49245,8374.947%4,125.330
7365AIV1624,897,1715,607.9148,800,0657,781,490930,75646,3194.976%4,168.710
7466AIV1714,989,4505,633.2498,926,4347,995,425923,26946,1905.003%4,157.100
7567AIV1724,931,3225,643.8648,861,3367,900,808927,17146,3374.998%4,170.330
7668AIV1815,054,7455,611.6289,066,8328,081,143931,51245,8284.920%4,124.520
7769AIV1824,931,8865,625.9848,834,2807,787,397925,27346,4245.017%4,178.160
7870AIV1915,025,0985,655.8128,964,4248,056,038930,28446,3224.979%4,168.980
7971AIV1924,926,0065,642.1338,856,2967,854,882933,10646,4604.979%4,181.400
8072AIV2014,961,4225,615.6488,819,8177,863,631908,49745,2574.982%4,073.130
8173AIV2024,996,3005,612.9819,058,6207,973,139956,78047,2534.939%4,252.770
8474AIV2115,066,5505,619.3578,997,4478,048,334915,28745,7464.998%4,117.140
8575AIV2124,978,1205,605.0518,888,8497,852,892925,48245,9914.969%4,139.190
8676AIV2214,978,2535,610.2548,895,0798,019,367917,92745,8014.990%4,122.090
8777AIV2224,912,9845,625.1898,823,2777,831,084926,82746,6075.029%4,194.630
8878AIV2315,008,6795,607.9638,890,4357,934,342918,67645,4474.947%4,090.230
8979AIV2324,910,4915,623.8488,825,3187,788,574933,15646,3344.965%4,170.060
9080AIV2414,977,3035,601.8978,962,9418,050,516931,06146,1964.962%4,157.640
9181AIV2424,975,0865,634.2319,004,9797,959,759959,56547,6664.967%4,289.940
9282AIV2515,058,5195,607.0918,977,8417,975,919909,10145,6985.027%4,112.820
9383AIV2524,964,8235,606.2818,884,5077,837,757930,21545,9524.940%4,135.680
9484AIV2614,990,7455,653.6788,881,6057,911,782914,46046,0705.038%4,146.300
9585AIV2624,984,0585,608.5708,972,0467,953,594944,93347,0124.975%4,231.080
9686AIV2715,019,9355,668.1468,903,4867,925,131913,28246,4385.085%4,179.420
9787AIV2725,009,0845,606.3119,045,1247,957,576957,63747,2654.936%4,253.850
9888AIV2814,926,6015,642.9738,768,7817,777,696915,92946,6945.098%4,202.460
9989AIV2825,109,9715,629.8619,178,2798,133,992943,60346,8194.962%4,213.710
10090AIV2914,999,7025,623.2938,832,0917,849,854902,76145,2455.012%4,072.050
10191AIV2925,004,1465,614.0908,895,1597,778,082918,31945,9925.008%4,139.280
10292AIV3014,968,0075,627.3548,836,4447,851,798907,66245,5955.023%4,103.550
10393AIV3024,904,7105,657.4958,709,0907,629,685906,81446,2415.099%4,161.690
10494AIV3115,039,1645,619.7878,993,8277,924,423921,94745,7204.959%4,114.800
10595AIV3124,982,2875,599.9468,954,7667,836,710934,92746,6824.993%4,201.380
+
+ diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_empty_bracket_20260723_043533_icache_report.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_empty_bracket_20260723_043533_icache_report.html new file mode 100644 index 0000000000..a98b2abf09 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_empty_bracket_20260723_043533_icache_report.html @@ -0,0 +1,359 @@ + + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 fdwic_submit_pmu_raw.json · schema fdwic-submit-pmu-v3 · 32 AIC + 64 AIV

+ +
+

empty-bracket 阶段观察（phase_id=2）

+

边界 claim_end_adjacent_empty_bracket · 计数语义 running_read_clear_empty_bracket_calibration · + 时间语义 boundary_diagnostic_outer_sys_cnt_around_adjacent_observer_pair · + expected_calls_per_core=1280

+ +

empty-bracket 用于估算每次 begin/end 紧邻执行的记录代码开销， + 不是业务 phase。 + phase PMU total/scalar 是紧邻 begin/end 对本身带来的计数开销；request/miss observed + 仍只覆盖两次 shadow read-clear 之间。SYS tick 只用来核验边界是否闭合，不参与阶段主时间换算。 + 这些结果只能用于估算进入 phase observed 的局部记录代码，不能估算完整 whole + 窗口中的全部记录开销。

+ + +

本阶段若跨过 linked Kernel，会同时门控 SYS 时间与 PMU counter， + 因而 total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段。 + result-used return-ready atomic 依赖区间只从 SYS 边界累计值扣除，PMU counter + 仍含 atomic 指令事件；source-issue atomic + 保留在时间和计数口径内。没有 linked Kernel 的阶段不触发 PMU 停/开表； + 每次调用的分母仍是外层业务调用次数，不是 Begin/End 读数，也不是排除的 Kernel 调用数。

+ +

阶段原始 PMU 观测来自 running read-clear 的 total cycles，并按 AIC/AIV + 各自校准频率换算；提供 empty-bracket 时，页面主参考值再扣除局部记录代码开销估算。 + phase_elapsed_ticks 只是 SYS 边界闭合诊断，绝不按 1 GHz 当作阶段时间。 + Scalar busy 与 non-scalar residual（逐核先算 total−scalar）共同解释阶段 PMU total； + 未校准时只能展示 raw observed；提供 empty-bracket 后，记录代码开销估算只从 phase + observed 分子扣除，原始 whole 分母保持不变。当前数据无法得到完整业务阶段占完整调度 + 窗口的精确比例。 + linked vector/cube Kernel 从 SYS 与 PMU counter 一并门控排除；result-used return-ready atomic + 依赖区间只从 SYS 边界累计值扣除，I-cache/PMU counter 仍含其指令事件；source-issue atomic + 保留。request/miss 百分比的分母是同一 ELF、同一次采集的 Submit 整窗 primary。 + observed_plus_capture_gap = observed + (primary − shadow)； + 边界读数和插桩 bookkeeping 会进入 sample，因此“观测值”和“加全窗 capture gap”都不是原业务 + 事件数的数学上下界。不同 ELF 不能直接相减后冒充业务真值；提供 empty-bracket 时只按上方 + 明示的公式估算局部记录代码开销。request/miss 的逐核 min/max 是每核累计整个 phase + 的极值，不是逐调用极值。

+
+ + + + +
指标ALL96 核AIC32 核AIV64 核
Phase PMU total原始 observed Σ 192,288,336 cycles
≈ 116,549.405 µs（1.649844 cycles/ns）
原始逐核 最小 1,898,234；最大 2,083,565；原始 1,564.846 cycles/call		原始 observed Σ 60,826,803 cycles
≈ 36,863.344 µs（1.650062 cycles/ns）
原始逐核 最小 1,898,234；最大 1,904,578；原始 1,485.029 cycles/call		原始 observed Σ 131,461,533 cycles
≈ 79,686.648 µs（1.649731 cycles/ns）
原始逐核 最小 2,017,566；最大 2,083,565；原始 1,604.755 cycles/call
Phase scalar busy原始 observed Σ 123,845,938 cycles
≈ 75,065.241 µs（1.649844 cycles/ns）
原始逐核 最小 1,251,002；最大 1,315,457；原始 1,007.861 cycles/call		原始 observed Σ 40,059,706 cycles
≈ 24,277.697 µs（1.650062 cycles/ns）
原始逐核 最小 1,251,002；最大 1,252,666；原始 978.020 cycles/call		原始 observed Σ 83,786,232 cycles
≈ 50,787.814 µs（1.649731 cycles/ns）
原始逐核 最小 1,296,538；最大 1,315,457；原始 1,022.781 cycles/call
非 Scalar-busy 残余原始 observed Σ 68,442,398 cycles
≈ 41,484.163 µs（1.649844 cycles/ns）
原始逐核 最小 647,232；最大 768,686；原始 556.986 cycles/call		原始 observed Σ 20,767,097 cycles
≈ 12,585.646 µs（1.650062 cycles/ns）
原始逐核 最小 647,232；最大 652,073；原始 507.009 cycles/call		原始 observed Σ 47,675,301 cycles
≈ 28,898.833 µs（1.649731 cycles/ns）
原始逐核 最小 714,196；最大 768,686；原始 581.974 cycles/call
原始阶段关系（非业务占比）以下均为原始 observed，不能作为业务占比
原始 Scalar / 原始 Phase total 64.406%
原始 Non-scalar / 原始 Phase total 35.594%
原始 Phase total / 原始 whole total 23.116%
原始 Phase scalar / 原始 whole scalar 16.431%
whole scalar−shadow scalar：Σ 245,760 cycles；逐核 最小 2,560；最大 2,560		以下均为原始 observed，不能作为业务占比
原始 Scalar / 原始 Phase total 65.859%
原始 Non-scalar / 原始 Phase total 34.141%
原始 Phase total / 原始 whole total 24.124%
原始 Phase scalar / 原始 whole scalar 16.149%
whole scalar−shadow scalar：Σ 81,920 cycles；逐核 最小 2,560；最大 2,560		以下均为原始 observed，不能作为业务占比
原始 Scalar / 原始 Phase total 63.734%
原始 Non-scalar / 原始 Phase total 36.266%
原始 Phase total / 原始 whole total 22.677%
原始 Phase scalar / 原始 whole scalar 16.569%
whole scalar−shadow scalar：Σ 163,840 cycles；逐核 最小 2,560；最大 2,560
Request raw observed（总数 / 逐核 min–max / 原始整窗）原始 observed 6,869,618
原始逐核 最小 69,140；最大 73,661
原始整窗 90,139,943；原始 observed / 原始整窗 7.621%（非业务占比）
原始 capture gap +0；加 gap 后 6,869,618（7.621%）		原始 observed 2,222,152
原始逐核 最小 69,140；最大 69,719
原始整窗 30,378,862；原始 observed / 原始整窗 7.315%（非业务占比）
原始 capture gap +0；加 gap 后 2,222,152（7.315%）		原始 observed 4,647,466
原始逐核 最小 70,952；最大 73,661
原始整窗 59,761,081；原始 observed / 原始整窗 7.777%（非业务占比）
原始 capture gap +0；加 gap 后 4,647,466（7.777%）
Miss raw observed（总数 / 逐核 min–max / 原始整窗）原始 observed 234,866
原始逐核 最小 41；最大 4,076
原始整窗 3,190,462；原始 observed / 原始整窗 7.362%（非业务占比）
原始 capture gap +0；加 gap 后 234,866（7.362%）		原始 observed 2,489
原始逐核 最小 41；最大 113
原始整窗 137,529；原始 observed / 原始整窗 1.810%（非业务占比）
原始 capture gap +0；加 gap 后 2,489（1.810%）		原始 observed 232,377
原始逐核 最小 2,513；最大 4,076
原始整窗 3,052,933；原始 observed / 原始整窗 7.612%（非业务占比）
原始 capture gap +0；加 gap 后 232,377（7.612%）
SYS 边界诊断 / Begin-EndΣ 160,209,998 raw ticks
逐核 最小 1,555,583；最大 1,736,354；仅边界诊断
Begin / End：122,880 / 122,880
业务调用 122,880 次；排除 linked Kernel 调用 0 次		Σ 49,863,180 raw ticks
逐核 最小 1,555,583；最大 1,560,383；仅边界诊断
Begin / End：40,960 / 40,960
业务调用 40,960 次；排除 linked Kernel 调用 0 次		Σ 110,346,818 raw ticks
逐核 最小 1,708,058；最大 1,736,354；仅边界诊断
Begin / End：81,920 / 81,920
业务调用 81,920 次；排除 linked Kernel 调用 0 次
+
+ + +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
5c03db75c56fd13cc03bec1a0ed299da6a712cd47fc1a8bb1b6597fe270e6482
+
Profile / extra cache
submit-pmu-empty-bracket / + c9e94d29047bda4b
+
Profiled cache key
TestPagedAttentionUnroll / a5 / fully_distributed_within_core / submit-pmu-empty-bracket
+
Source state
source-v2:8c6b6e3a412d2c7c7113c2d8f714a14eb3786f80:70a379ed18604dc71f2bced3e4605611a15da9fc32852b72871cfead17f51f65:e9cb5ce41f2428494aa37d7fe84ce2740501a96b635974e3093c46b7a777d847
+
Source-state stamp
/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/c9e94d29047bda4b/aicore/.git_commit
+
Compile definitions
PTO_FDWIC_SUBMIT_PMU=1
PTO_FDWIC_SUBMIT_PMU_PHASE_ID=2
PTO_FDWIC_TRACE_ENABLED=0
+
+
+ + + +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
AICore final2,676,61671743c878d9a9b57e0d6f354cfd7379609ee14fb7685a88e0b5043dfa7649f16213,0721cebadfd2bed4fe7db8587b20d287eb2bda57c4087b5cb10f2032d3343d27c81/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/aicore-extra/c9e94d29047bda4b/aicore_kernel.o
AIC combined2,003,09635588607a432297bbe78d94defc6d6636ce4a32ab37cca6ff3276beae568aa5e99,3846e01610527508cd1289d3512292c6b908d210eb1c33ae9d2c2eccc9393d9278b/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/c9e94d29047bda4b/aicore/aicore_aic_combined.o
AIV combined2,265,064da3006a49fb153cd637c4db48a08d4337f2e7977bd37ac833402463692825467113,4888d08b7bc4545fa117650513eb40b9e6bc8fd9a7e35ecbfd22cbb57c68b01ac60/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/c9e94d29047bda4b/aicore/aicore_aiv_combined.o
Host runtime11,776,1440369d0602054bf612f86b58c294503e00486bd505bff9ee7a455994f043147cc454,6325ac381ed10be5f5d492b8072c051a177131f4caec6d6f6bc9bf50880da4bb0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so
+
+ +
+
原始全局 Submit 时间范围（墙钟） + 5,755.507 µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + 128096 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-25656199f91545cc6a1fbf18b8bb2cfebae279cb5801516311a040d4facccf321ba
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + 90.000 ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×90 ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
+
+

ALL · 96 核

+
+
Submit PMU total/core
+
均值 8,665,176.4；最小 7,557,990；最大 9,310,804 cycles
+ 等效时间 均值 5,252.119；最小 4,581.033；最大 5,643.445 µs + （按 1.649844 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,851,451.6；最小 7,423,793；最大 8,189,296 cycles
+ 等效时间 均值 4,758.905；最小 4,499.694；最大 4,963.679 µs； + 加权占比 90.609% +
+
非 Scalar-busy 残余/core
+
均值 813,724.8；最小 94,684；最大 1,276,726 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 493.213；最小 57.390；最大 773.846 µs +
+
SYS gate 边界诊断/core
+
均值 4,817,561.0；最小 4,166,380；最大 5,182,194 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,585.138；最小 5,266.295；最大 5,752.086 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 767,577.0；最小 532,037；最大 1,261,279 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 86.257% +
+
Primary I-cache request/core
最小 911,138；最大 999,842
+
Primary I-cache miss/core
最小 3,517；最大 49,162
+
加权 miss rate
3.539%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 316.530；最大 4,424.580 µs
+
+
+ +
+

AIC · 32 核

+
+
Submit PMU total/core
+
均值 7,879,526.4；最小 7,557,990；最大 8,328,679 cycles
+ 等效时间 均值 4,775.291；最小 4,580.428；最大 5,047.495 µs + （按 1.650062 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,752,162.1；最小 7,423,793；最大 8,189,296 cycles
+ 等效时间 均值 4,698.104；最小 4,499.099；最大 4,963.023 µs； + 加权占比 98.384% +
+
非 Scalar-busy 残余/core
+
均值 127,364.3；最小 94,684；最大 148,511 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 77.188；最小 57.382；最大 90.003 µs +
+
SYS gate 边界诊断/core
+
均值 4,299,739.8；最小 4,166,380；最大 4,491,013 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,330.926；最小 5,266.295；最大 5,619.568 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 1,031,186.7；最小 889,619；最大 1,261,279 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 80.657% +
+
Primary I-cache request/core
最小 923,499；最大 999,842
+
Primary I-cache miss/core
最小 3,517；最大 4,793
+
加权 miss rate
0.453%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 316.530；最大 431.370 µs
+
+
+ +
+

AIV · 64 核

+
+
Submit PMU total/core
+
均值 9,058,001.4；最小 8,862,762；最大 9,310,804 cycles
+ 等效时间 均值 5,490.593；最小 5,372.247；最大 5,643.832 µs + （按 1.649731 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,901,096.3；最小 7,691,649；最大 8,139,257 cycles
+ 等效时间 均值 4,789.324；最小 4,662.366；最大 4,933.687 µs； + 加权占比 87.228% +
+
非 Scalar-busy 残余/core
+
均值 1,156,905.1；最小 1,039,712；最大 1,276,726 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 701.269；最小 630.231；最大 773.900 µs +
+
SYS gate 边界诊断/core
+
均值 5,076,471.6；最小 4,980,461；最大 5,182,194 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,712.244；最小 5,682.564；最大 5,752.086 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 635,772.2；最小 532,037；最大 711,137 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 88.870% +
+
Primary I-cache request/core
最小 911,138；最大 963,005
+
Primary I-cache miss/core
最小 46,322；最大 49,162
+
加权 miss rate
5.109%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 4,168.980；最大 4,424.580 µs
+
+
+
+

逐物理核分布

+

AICAIV

+
+
+
SYS gate boundary-diagnostic ticks/core
+ + + + 5,182,194 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_submit_elapsed_ticks=4283344physical=1 logical=1 role=AIC scalar_submit_elapsed_ticks=4306111physical=2 logical=2 role=AIC scalar_submit_elapsed_ticks=4350810physical=3 logical=3 role=AIC scalar_submit_elapsed_ticks=4166380physical=4 logical=4 role=AIC scalar_submit_elapsed_ticks=4330105physical=5 logical=5 role=AIC scalar_submit_elapsed_ticks=4303962physical=6 logical=6 role=AIC scalar_submit_elapsed_ticks=4308486physical=7 logical=7 role=AIC scalar_submit_elapsed_ticks=4201885physical=8 logical=8 role=AIC scalar_submit_elapsed_ticks=4359497physical=9 logical=9 role=AIC scalar_submit_elapsed_ticks=4205845physical=10 logical=10 role=AIC scalar_submit_elapsed_ticks=4399528physical=12 logical=11 role=AIC scalar_submit_elapsed_ticks=4257362physical=13 logical=12 role=AIC scalar_submit_elapsed_ticks=4337119physical=14 logical=13 role=AIC scalar_submit_elapsed_ticks=4330811physical=15 logical=14 role=AIC scalar_submit_elapsed_ticks=4389681physical=16 logical=15 role=AIC scalar_submit_elapsed_ticks=4244246physical=54 logical=16 role=AIC scalar_submit_elapsed_ticks=4365157physical=55 logical=17 role=AIC scalar_submit_elapsed_ticks=4259494physical=56 logical=18 role=AIC scalar_submit_elapsed_ticks=4491013physical=57 logical=19 role=AIC scalar_submit_elapsed_ticks=4167476physical=58 logical=20 role=AIC scalar_submit_elapsed_ticks=4332293physical=60 logical=21 role=AIC scalar_submit_elapsed_ticks=4274891physical=61 logical=22 role=AIC scalar_submit_elapsed_ticks=4295442physical=62 logical=23 role=AIC scalar_submit_elapsed_ticks=4286347physical=63 logical=24 role=AIC scalar_submit_elapsed_ticks=4310166physical=64 logical=25 role=AIC scalar_submit_elapsed_ticks=4358289physical=65 logical=26 role=AIC scalar_submit_elapsed_ticks=4365533physical=66 logical=27 role=AIC scalar_submit_elapsed_ticks=4182406physical=67 logical=28 role=AIC scalar_submit_elapsed_ticks=4291264physical=68 logical=29 role=AIC scalar_submit_elapsed_ticks=4290883physical=69 logical=30 role=AIC scalar_submit_elapsed_ticks=4326884physical=70 logical=31 role=AIC scalar_submit_elapsed_ticks=4218963physical=18 logical=32 role=AIV scalar_submit_elapsed_ticks=5085514physical=19 logical=33 role=AIV scalar_submit_elapsed_ticks=5134447physical=20 logical=34 role=AIV scalar_submit_elapsed_ticks=5023746physical=21 logical=35 role=AIV scalar_submit_elapsed_ticks=5008389physical=22 logical=36 role=AIV scalar_submit_elapsed_ticks=5041977physical=23 logical=37 role=AIV scalar_submit_elapsed_ticks=5106860physical=24 logical=38 role=AIV scalar_submit_elapsed_ticks=5136780physical=25 logical=39 role=AIV scalar_submit_elapsed_ticks=5011930physical=26 logical=40 role=AIV scalar_submit_elapsed_ticks=5105393physical=27 logical=41 role=AIV scalar_submit_elapsed_ticks=5039771physical=28 logical=42 role=AIV scalar_submit_elapsed_ticks=5154663physical=29 logical=43 role=AIV scalar_submit_elapsed_ticks=5046717physical=30 logical=44 role=AIV scalar_submit_elapsed_ticks=5050381physical=31 logical=45 role=AIV scalar_submit_elapsed_ticks=5018436physical=32 logical=46 role=AIV scalar_submit_elapsed_ticks=5060688physical=33 logical=47 role=AIV scalar_submit_elapsed_ticks=5101780physical=34 logical=48 role=AIV scalar_submit_elapsed_ticks=5169728physical=35 logical=49 role=AIV scalar_submit_elapsed_ticks=5031067physical=36 logical=50 role=AIV scalar_submit_elapsed_ticks=5182194physical=37 logical=51 role=AIV scalar_submit_elapsed_ticks=5031893physical=38 logical=52 role=AIV scalar_submit_elapsed_ticks=5108719physical=39 logical=53 role=AIV scalar_submit_elapsed_ticks=5107405physical=42 logical=54 role=AIV scalar_submit_elapsed_ticks=5054402physical=43 logical=55 role=AIV scalar_submit_elapsed_ticks=5080059physical=44 logical=56 role=AIV scalar_submit_elapsed_ticks=5168182physical=45 logical=57 role=AIV scalar_submit_elapsed_ticks=5031586physical=46 logical=58 role=AIV scalar_submit_elapsed_ticks=5032738physical=47 logical=59 role=AIV scalar_submit_elapsed_ticks=5068993physical=48 logical=60 role=AIV scalar_submit_elapsed_ticks=5116896physical=49 logical=61 role=AIV scalar_submit_elapsed_ticks=5061201physical=50 logical=62 role=AIV scalar_submit_elapsed_ticks=5131438physical=51 logical=63 role=AIV scalar_submit_elapsed_ticks=5070167physical=72 logical=64 role=AIV scalar_submit_elapsed_ticks=5054511physical=73 logical=65 role=AIV scalar_submit_elapsed_ticks=5028851physical=74 logical=66 role=AIV scalar_submit_elapsed_ticks=5107746physical=75 logical=67 role=AIV scalar_submit_elapsed_ticks=5037742physical=76 logical=68 role=AIV scalar_submit_elapsed_ticks=5088215physical=77 logical=69 role=AIV scalar_submit_elapsed_ticks=5086071physical=78 logical=70 role=AIV scalar_submit_elapsed_ticks=5064006physical=79 logical=71 role=AIV scalar_submit_elapsed_ticks=5058755physical=80 logical=72 role=AIV scalar_submit_elapsed_ticks=5042549physical=81 logical=73 role=AIV scalar_submit_elapsed_ticks=5002457physical=84 logical=74 role=AIV scalar_submit_elapsed_ticks=5113592physical=85 logical=75 role=AIV scalar_submit_elapsed_ticks=5091950physical=86 logical=76 role=AIV scalar_submit_elapsed_ticks=5155497physical=87 logical=77 role=AIV scalar_submit_elapsed_ticks=5058112physical=88 logical=78 role=AIV scalar_submit_elapsed_ticks=5088255physical=89 logical=79 role=AIV scalar_submit_elapsed_ticks=4980461physical=90 logical=80 role=AIV scalar_submit_elapsed_ticks=5062634physical=91 logical=81 role=AIV scalar_submit_elapsed_ticks=5000084physical=92 logical=82 role=AIV scalar_submit_elapsed_ticks=5107021physical=93 logical=83 role=AIV scalar_submit_elapsed_ticks=5039244physical=94 logical=84 role=AIV scalar_submit_elapsed_ticks=5099111physical=95 logical=85 role=AIV scalar_submit_elapsed_ticks=5064184physical=96 logical=86 role=AIV scalar_submit_elapsed_ticks=5124064physical=97 logical=87 role=AIV scalar_submit_elapsed_ticks=5102421physical=98 logical=88 role=AIV scalar_submit_elapsed_ticks=5054696physical=99 logical=89 role=AIV scalar_submit_elapsed_ticks=5159279physical=100 logical=90 role=AIV scalar_submit_elapsed_ticks=5125032physical=101 logical=91 role=AIV scalar_submit_elapsed_ticks=5113978physical=102 logical=92 role=AIV scalar_submit_elapsed_ticks=5098746physical=103 logical=93 role=AIV scalar_submit_elapsed_ticks=5085017physical=104 logical=94 role=AIV scalar_submit_elapsed_ticks=5032929physical=105 logical=95 role=AIV scalar_submit_elapsed_ticks=4992833 + +
+ +
+
PMU total cycles/core
+ + + + 9,310,804 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC total_cycles=7748349physical=1 logical=1 role=AIC total_cycles=7980684physical=2 logical=2 role=AIC total_cycles=7949115physical=3 logical=3 role=AIC total_cycles=7568999physical=4 logical=4 role=AIC total_cycles=7883030physical=5 logical=5 role=AIC total_cycles=8150329physical=6 logical=6 role=AIC total_cycles=7867439physical=7 logical=7 role=AIC total_cycles=7557990physical=8 logical=8 role=AIC total_cycles=8015393physical=9 logical=9 role=AIC total_cycles=7764216physical=10 logical=10 role=AIC total_cycles=8034078physical=12 logical=11 role=AIC total_cycles=7741245physical=13 logical=12 role=AIC total_cycles=7809234physical=14 logical=13 role=AIC total_cycles=8144023physical=15 logical=14 role=AIC total_cycles=7919247physical=16 logical=15 role=AIC total_cycles=7912368physical=54 logical=16 role=AIC total_cycles=7996957physical=55 logical=17 role=AIC total_cycles=7886202physical=56 logical=18 role=AIC total_cycles=8328679physical=57 logical=19 role=AIC total_cycles=7636172physical=58 logical=20 role=AIC total_cycles=7931935physical=60 logical=21 role=AIC total_cycles=7930019physical=61 logical=22 role=AIC total_cycles=7749532physical=62 logical=23 role=AIC total_cycles=7771910physical=63 logical=24 role=AIC total_cycles=7855826physical=64 logical=25 role=AIC total_cycles=8092712physical=65 logical=26 role=AIC total_cycles=7967850physical=66 logical=27 role=AIC total_cycles=7627670physical=67 logical=28 role=AIC total_cycles=7806912physical=68 logical=29 role=AIC total_cycles=7989361physical=69 logical=30 role=AIC total_cycles=7836005physical=70 logical=31 role=AIC total_cycles=7691363physical=18 logical=32 role=AIV total_cycles=9027680physical=19 logical=33 role=AIV total_cycles=9132657physical=20 logical=34 role=AIV total_cycles=8972273physical=21 logical=35 role=AIV total_cycles=8935618physical=22 logical=36 role=AIV total_cycles=8945278physical=23 logical=37 role=AIV total_cycles=9087371physical=24 logical=38 role=AIV total_cycles=9141310physical=25 logical=39 role=AIV total_cycles=8953214physical=26 logical=40 role=AIV total_cycles=9107888physical=27 logical=41 role=AIV total_cycles=8960800physical=28 logical=42 role=AIV total_cycles=9240137physical=29 logical=43 role=AIV total_cycles=9055444physical=30 logical=44 role=AIV total_cycles=8958657physical=31 logical=45 role=AIV total_cycles=8998138physical=32 logical=46 role=AIV total_cycles=9030572physical=33 logical=47 role=AIV total_cycles=9163923physical=34 logical=48 role=AIV total_cycles=9197643physical=35 logical=49 role=AIV total_cycles=9030896physical=36 logical=50 role=AIV total_cycles=9310804physical=37 logical=51 role=AIV total_cycles=9002508physical=38 logical=52 role=AIV total_cycles=9058986physical=39 logical=53 role=AIV total_cycles=9199477physical=42 logical=54 role=AIV total_cycles=8982626physical=43 logical=55 role=AIV total_cycles=9113781physical=44 logical=56 role=AIV total_cycles=9253226physical=45 logical=57 role=AIV total_cycles=8975277physical=46 logical=58 role=AIV total_cycles=9011091physical=47 logical=59 role=AIV total_cycles=9042238physical=48 logical=60 role=AIV total_cycles=9131876physical=49 logical=61 role=AIV total_cycles=9054644physical=50 logical=62 role=AIV total_cycles=9181806physical=51 logical=63 role=AIV total_cycles=9055752physical=72 logical=64 role=AIV total_cycles=9034455physical=73 logical=65 role=AIV total_cycles=8996893physical=74 logical=66 role=AIV total_cycles=9089765physical=75 logical=67 role=AIV total_cycles=9014039physical=76 logical=68 role=AIV total_cycles=9050201physical=77 logical=69 role=AIV total_cycles=9124430physical=78 logical=70 role=AIV total_cycles=9030232physical=79 logical=71 role=AIV total_cycles=9037499physical=80 logical=72 role=AIV total_cycles=8983708physical=81 logical=73 role=AIV total_cycles=8961781physical=84 logical=74 role=AIV total_cycles=9099285physical=85 logical=75 role=AIV total_cycles=9063583physical=86 logical=76 role=AIV total_cycles=9228797physical=87 logical=77 role=AIV total_cycles=9074494physical=88 logical=78 role=AIV total_cycles=9037590physical=89 logical=79 role=AIV total_cycles=8931411physical=90 logical=80 role=AIV total_cycles=8984842physical=91 logical=81 role=AIV total_cycles=8936274physical=92 logical=82 role=AIV total_cycles=9062696physical=93 logical=83 role=AIV total_cycles=9029322physical=94 logical=84 role=AIV total_cycles=9066783physical=95 logical=85 role=AIV total_cycles=9038192physical=96 logical=86 role=AIV total_cycles=9120146physical=97 logical=87 role=AIV total_cycles=9110401physical=98 logical=88 role=AIV total_cycles=8977649physical=99 logical=89 role=AIV total_cycles=9128468physical=100 logical=90 role=AIV total_cycles=9101495physical=101 logical=91 role=AIV total_cycles=9137943physical=102 logical=92 role=AIV total_cycles=9027847physical=103 logical=93 role=AIV total_cycles=9093689physical=104 logical=94 role=AIV total_cycles=8961827physical=105 logical=95 role=AIV total_cycles=8862762 + +
+ +
+
Scalar busy cycles/core
+ + + + 8,189,296 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_busy=7629917physical=1 logical=1 role=AIC scalar_busy=7851351physical=2 logical=2 role=AIC scalar_busy=7835214physical=3 logical=3 role=AIC scalar_busy=7450784physical=4 logical=4 role=AIC scalar_busy=7754646physical=5 logical=5 role=AIC scalar_busy=8022030physical=6 logical=6 role=AIC scalar_busy=7772755physical=7 logical=7 role=AIC scalar_busy=7423793physical=8 logical=8 role=AIC scalar_busy=7905297physical=9 logical=9 role=AIC scalar_busy=7615705physical=10 logical=10 role=AIC scalar_busy=7898420physical=12 logical=11 role=AIC scalar_busy=7606236physical=13 logical=12 role=AIC scalar_busy=7672294physical=14 logical=13 role=AIC scalar_busy=8007199physical=15 logical=14 role=AIC scalar_busy=7793214physical=16 logical=15 role=AIC scalar_busy=7785055physical=54 logical=16 role=AIC scalar_busy=7871530physical=55 logical=17 role=AIC scalar_busy=7754374physical=56 logical=18 role=AIC scalar_busy=8189296physical=57 logical=19 role=AIC scalar_busy=7507791physical=58 logical=20 role=AIC scalar_busy=7793675physical=60 logical=21 role=AIC scalar_busy=7792386physical=61 logical=22 role=AIC scalar_busy=7639180physical=62 logical=23 role=AIC scalar_busy=7635777physical=63 logical=24 role=AIC scalar_busy=7729056physical=64 logical=25 role=AIC scalar_busy=7948048physical=65 logical=26 role=AIC scalar_busy=7841751physical=66 logical=27 role=AIC scalar_busy=7507347physical=67 logical=28 role=AIC scalar_busy=7698563physical=68 logical=29 role=AIC scalar_busy=7857463physical=69 logical=30 role=AIC scalar_busy=7717689physical=70 logical=31 role=AIC scalar_busy=7561350physical=18 logical=32 role=AIV scalar_busy=7914043physical=19 logical=33 role=AIV scalar_busy=7907544physical=20 logical=34 role=AIV scalar_busy=7932561physical=21 logical=35 role=AIV scalar_busy=7703911physical=22 logical=36 role=AIV scalar_busy=7852886physical=23 logical=37 role=AIV scalar_busy=7878592physical=24 logical=38 role=AIV scalar_busy=7988916physical=25 logical=39 role=AIV scalar_busy=7782716physical=26 logical=40 role=AIV scalar_busy=7967088physical=27 logical=41 role=AIV scalar_busy=7760973physical=28 logical=42 role=AIV scalar_busy=8046951physical=29 logical=43 role=AIV scalar_busy=7800279physical=30 logical=44 role=AIV scalar_busy=7833484physical=31 logical=45 role=AIV scalar_busy=7830760physical=32 logical=46 role=AIV scalar_busy=7938994physical=33 logical=47 role=AIV scalar_busy=7960908physical=34 logical=48 role=AIV scalar_busy=8083016physical=35 logical=49 role=AIV scalar_busy=7874640physical=36 logical=50 role=AIV scalar_busy=8138715physical=37 logical=51 role=AIV scalar_busy=7801356physical=38 logical=52 role=AIV scalar_busy=7969191physical=39 logical=53 role=AIV scalar_busy=7985793physical=42 logical=54 role=AIV scalar_busy=7897445physical=43 logical=55 role=AIV scalar_busy=7967220physical=44 logical=56 role=AIV scalar_busy=8139257physical=45 logical=57 role=AIV scalar_busy=7786142physical=46 logical=58 role=AIV scalar_busy=7905300physical=47 logical=59 role=AIV scalar_busy=7830774physical=48 logical=60 role=AIV scalar_busy=8051776physical=49 logical=61 role=AIV scalar_busy=7891118physical=50 logical=62 role=AIV scalar_busy=8049541physical=51 logical=63 role=AIV scalar_busy=7840342physical=72 logical=64 role=AIV scalar_busy=7935637physical=73 logical=65 role=AIV scalar_busy=7788974physical=74 logical=66 role=AIV scalar_busy=7969994physical=75 logical=67 role=AIV scalar_busy=7843931physical=76 logical=68 role=AIV scalar_busy=7947141physical=77 logical=69 role=AIV scalar_busy=7917825physical=78 logical=70 role=AIV scalar_busy=7953062physical=79 logical=71 role=AIV scalar_busy=7858483physical=80 logical=72 role=AIV scalar_busy=7898791physical=81 logical=73 role=AIV scalar_busy=7766154physical=84 logical=74 role=AIV scalar_busy=7984959physical=85 logical=75 role=AIV scalar_busy=7857400physical=86 logical=76 role=AIV scalar_busy=8122155physical=87 logical=77 role=AIV scalar_busy=7878510physical=88 logical=78 role=AIV scalar_busy=7958632physical=89 logical=79 role=AIV scalar_busy=7754551physical=90 logical=80 role=AIV scalar_busy=7885239physical=91 logical=81 role=AIV scalar_busy=7740762physical=92 logical=82 role=AIV scalar_busy=7945574physical=93 logical=83 role=AIV scalar_busy=7874320physical=94 logical=84 role=AIV scalar_busy=7946564physical=95 logical=85 role=AIV scalar_busy=7859215physical=96 logical=86 role=AIV scalar_busy=7971907physical=97 logical=87 role=AIV scalar_busy=7833675physical=98 logical=88 role=AIV scalar_busy=7807617physical=99 logical=89 role=AIV scalar_busy=7881739physical=100 logical=90 role=AIV scalar_busy=7978443physical=101 logical=91 role=AIV scalar_busy=7912593physical=102 logical=92 role=AIV scalar_busy=7890339physical=103 logical=93 role=AIV scalar_busy=7858647physical=104 logical=94 role=AIV scalar_busy=7843452physical=105 logical=95 role=AIV scalar_busy=7691649 + +
+ +
+
Primary I-cache requests/core
+ + + + 999,842 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_requests=926010physical=1 logical=1 role=AIC icache_requests=967787physical=2 logical=2 role=AIC icache_requests=933740physical=3 logical=3 role=AIC icache_requests=938863physical=4 logical=4 role=AIC icache_requests=937021physical=5 logical=5 role=AIC icache_requests=996811physical=6 logical=6 role=AIC icache_requests=933171physical=7 logical=7 role=AIC icache_requests=934728physical=8 logical=8 role=AIC icache_requests=944449physical=9 logical=9 role=AIC icache_requests=953076physical=10 logical=10 role=AIC icache_requests=939118physical=12 logical=11 role=AIC icache_requests=941180physical=13 logical=12 role=AIC icache_requests=926034physical=14 logical=13 role=AIC icache_requests=999842physical=15 logical=14 role=AIC icache_requests=926982physical=16 logical=15 role=AIC icache_requests=966903physical=54 logical=16 role=AIC icache_requests=938942physical=55 logical=17 role=AIC icache_requests=963920physical=56 logical=18 role=AIC icache_requests=977088physical=57 logical=19 role=AIC icache_requests=939366physical=58 logical=20 role=AIC icache_requests=943581physical=60 logical=21 role=AIC icache_requests=975555physical=61 logical=22 role=AIC icache_requests=923499physical=62 logical=23 role=AIC icache_requests=939977physical=63 logical=24 role=AIC icache_requests=940267physical=64 logical=25 role=AIC icache_requests=989748physical=65 logical=26 role=AIC icache_requests=938812physical=66 logical=27 role=AIC icache_requests=943493physical=67 logical=28 role=AIC icache_requests=941122physical=68 logical=29 role=AIC icache_requests=969861physical=69 logical=30 role=AIC icache_requests=940573physical=70 logical=31 role=AIC icache_requests=947343physical=18 logical=32 role=AIV icache_requests=921684physical=19 logical=33 role=AIV icache_requests=938047physical=20 logical=34 role=AIV icache_requests=924215physical=21 logical=35 role=AIV icache_requests=933368physical=22 logical=36 role=AIV icache_requests=914445physical=23 logical=37 role=AIV icache_requests=927226physical=24 logical=38 role=AIV icache_requests=927083physical=25 logical=39 role=AIV icache_requests=934759physical=26 logical=40 role=AIV icache_requests=926305physical=27 logical=41 role=AIV icache_requests=918522physical=28 logical=42 role=AIV icache_requests=933971physical=29 logical=43 role=AIV icache_requests=944737physical=30 logical=44 role=AIV icache_requests=916199physical=31 logical=45 role=AIV icache_requests=936030physical=32 logical=46 role=AIV icache_requests=929261physical=33 logical=47 role=AIV icache_requests=961793physical=34 logical=48 role=AIV icache_requests=923491physical=35 logical=49 role=AIV icache_requests=944596physical=36 logical=50 role=AIV icache_requests=963005physical=37 logical=51 role=AIV icache_requests=937054physical=38 logical=52 role=AIV icache_requests=914578physical=39 logical=53 role=AIV icache_requests=951655physical=42 logical=54 role=AIV icache_requests=930675physical=43 logical=55 role=AIV icache_requests=955316physical=44 logical=56 role=AIV icache_requests=957931physical=45 logical=57 role=AIV icache_requests=937914physical=46 logical=58 role=AIV icache_requests=932219physical=47 logical=59 role=AIV icache_requests=929392physical=48 logical=60 role=AIV icache_requests=930980physical=49 logical=61 role=AIV icache_requests=941007physical=50 logical=62 role=AIV icache_requests=935639physical=51 logical=63 role=AIV icache_requests=928738physical=72 logical=64 role=AIV icache_requests=935644physical=73 logical=65 role=AIV icache_requests=942329physical=74 logical=66 role=AIV icache_requests=927205physical=75 logical=67 role=AIV icache_requests=928420physical=76 logical=68 role=AIV icache_requests=929106physical=77 logical=69 role=AIV icache_requests=952914physical=78 logical=70 role=AIV icache_requests=924021physical=79 logical=71 role=AIV icache_requests=936651physical=80 logical=72 role=AIV icache_requests=930497physical=81 logical=73 role=AIV icache_requests=949147physical=84 logical=74 role=AIV icache_requests=929045physical=85 logical=75 role=AIV icache_requests=927825physical=86 logical=76 role=AIV icache_requests=938532physical=87 logical=77 role=AIV icache_requests=938838physical=88 logical=78 role=AIV icache_requests=923125physical=89 logical=79 role=AIV icache_requests=942150physical=90 logical=80 role=AIV icache_requests=925086physical=91 logical=81 role=AIV icache_requests=936180physical=92 logical=82 role=AIV icache_requests=922477physical=93 logical=83 role=AIV icache_requests=946227physical=94 logical=84 role=AIV icache_requests=934328physical=95 logical=85 role=AIV icache_requests=940422physical=96 logical=86 role=AIV icache_requests=931260physical=97 logical=87 role=AIV icache_requests=936606physical=98 logical=88 role=AIV icache_requests=923961physical=99 logical=89 role=AIV icache_requests=911138physical=100 logical=90 role=AIV icache_requests=927361physical=101 logical=91 role=AIV icache_requests=948906physical=102 logical=92 role=AIV icache_requests=921822physical=103 logical=93 role=AIV icache_requests=933183physical=104 logical=94 role=AIV icache_requests=932406physical=105 logical=95 role=AIV icache_requests=932434 + +
+ +
+
Primary I-cache misses/core
+ + + + 49,162 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_misses=4213physical=1 logical=1 role=AIC icache_misses=4322physical=2 logical=2 role=AIC icache_misses=3971physical=3 logical=3 role=AIC icache_misses=4501physical=4 logical=4 role=AIC icache_misses=4374physical=5 logical=5 role=AIC icache_misses=4379physical=6 logical=6 role=AIC icache_misses=3517physical=7 logical=7 role=AIC icache_misses=4525physical=8 logical=8 role=AIC icache_misses=3959physical=9 logical=9 role=AIC icache_misses=4670physical=10 logical=10 role=AIC icache_misses=4085physical=12 logical=11 role=AIC icache_misses=4395physical=13 logical=12 role=AIC icache_misses=4286physical=14 logical=13 role=AIC icache_misses=4509physical=15 logical=14 role=AIC icache_misses=3950physical=16 logical=15 role=AIC icache_misses=4413physical=54 logical=16 role=AIC icache_misses=4143physical=55 logical=17 role=AIC icache_misses=4364physical=56 logical=18 role=AIC icache_misses=4060physical=57 logical=19 role=AIC icache_misses=4487physical=58 logical=20 role=AIC icache_misses=4170physical=60 logical=21 role=AIC icache_misses=4547physical=61 logical=22 role=AIC icache_misses=4328physical=62 logical=23 role=AIC icache_misses=4477physical=63 logical=24 role=AIC icache_misses=4440physical=64 logical=25 role=AIC icache_misses=4660physical=65 logical=26 role=AIC icache_misses=4169physical=66 logical=27 role=AIC icache_misses=4406physical=67 logical=28 role=AIC icache_misses=4019physical=68 logical=29 role=AIC icache_misses=4339physical=69 logical=30 role=AIC icache_misses=4058physical=70 logical=31 role=AIC icache_misses=4793physical=18 logical=32 role=AIV icache_misses=46611physical=19 logical=33 role=AIV icache_misses=47522physical=20 logical=34 role=AIV icache_misses=47083physical=21 logical=35 role=AIV icache_misses=49162physical=22 logical=36 role=AIV icache_misses=46354physical=23 logical=37 role=AIV icache_misses=47034physical=24 logical=38 role=AIV icache_misses=48460physical=25 logical=39 role=AIV icache_misses=47184physical=26 logical=40 role=AIV icache_misses=47514physical=27 logical=41 role=AIV icache_misses=46322physical=28 logical=42 role=AIV icache_misses=47666physical=29 logical=43 role=AIV icache_misses=48588physical=30 logical=44 role=AIV icache_misses=46961physical=31 logical=45 role=AIV icache_misses=47645physical=32 logical=46 role=AIV icache_misses=47252physical=33 logical=47 role=AIV icache_misses=49093physical=34 logical=48 role=AIV icache_misses=47076physical=35 logical=49 role=AIV icache_misses=48595physical=36 logical=50 role=AIV icache_misses=49014physical=37 logical=51 role=AIV icache_misses=48013physical=38 logical=52 role=AIV icache_misses=46820physical=39 logical=53 role=AIV icache_misses=48761physical=42 logical=54 role=AIV icache_misses=47500physical=43 logical=55 role=AIV icache_misses=48752physical=44 logical=56 role=AIV icache_misses=48797physical=45 logical=57 role=AIV icache_misses=48014physical=46 logical=58 role=AIV icache_misses=47695physical=47 logical=59 role=AIV icache_misses=47608physical=48 logical=60 role=AIV icache_misses=47163physical=49 logical=61 role=AIV icache_misses=47965physical=50 logical=62 role=AIV icache_misses=47789physical=51 logical=63 role=AIV icache_misses=47906physical=72 logical=64 role=AIV icache_misses=47349physical=73 logical=65 role=AIV icache_misses=48073physical=74 logical=66 role=AIV icache_misses=47423physical=75 logical=67 role=AIV icache_misses=47871physical=76 logical=68 role=AIV icache_misses=47591physical=77 logical=69 role=AIV icache_misses=48557physical=78 logical=70 role=AIV icache_misses=47232physical=79 logical=71 role=AIV icache_misses=47957physical=80 logical=72 role=AIV icache_misses=47651physical=81 logical=73 role=AIV icache_misses=48987physical=84 logical=74 role=AIV icache_misses=47386physical=85 logical=75 role=AIV icache_misses=47133physical=86 logical=76 role=AIV icache_misses=47959physical=87 logical=77 role=AIV icache_misses=48124physical=88 logical=78 role=AIV icache_misses=46901physical=89 logical=79 role=AIV icache_misses=48400physical=90 logical=80 role=AIV icache_misses=47496physical=91 logical=81 role=AIV icache_misses=47908physical=92 logical=82 role=AIV icache_misses=47123physical=93 logical=83 role=AIV icache_misses=48448physical=94 logical=84 role=AIV icache_misses=47532physical=95 logical=85 role=AIV icache_misses=47986physical=96 logical=86 role=AIV icache_misses=47426physical=97 logical=87 role=AIV icache_misses=47780physical=98 logical=88 role=AIV icache_misses=47094physical=99 logical=89 role=AIV icache_misses=46491physical=100 logical=90 role=AIV icache_misses=46945physical=101 logical=91 role=AIV icache_misses=48326physical=102 logical=92 role=AIV icache_misses=46806physical=103 logical=93 role=AIV icache_misses=47567physical=104 logical=94 role=AIV icache_misses=47646physical=105 logical=95 role=AIV icache_misses=47846 + +
+
+

逐核原始主计数

+

报告展示 primary request/miss；phase 模式的 I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和，只用于标记全窗 capture gap；scalar shadow 同样用于核验 phase 读清重建，并显示其相对 CNT2 primary 的 capture loss，不作为第二份性能数据展示。

+
+ + + + + + + +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate90ns量尺 µs
00AIC004,283,3445,312.9237,748,3497,629,917926,0104,2130.455%379.170
11AIC104,306,1115,283.8007,980,6847,851,351967,7874,3220.447%388.980
22AIC204,350,8105,277.2797,949,1157,835,214933,7403,9710.425%357.390
33AIC304,166,3805,312.0667,568,9997,450,784938,8634,5010.479%405.090
44AIC404,330,1055,288.8957,883,0307,754,646937,0214,3740.467%393.660
55AIC504,303,9625,474.6498,150,3298,022,030996,8114,3790.439%394.110
66AIC604,308,4865,274.1397,867,4397,772,755933,1713,5170.377%316.530
77AIC704,201,8855,309.7257,557,9907,423,793934,7284,5250.484%407.250
88AIC804,359,4975,277.9358,015,3937,905,297944,4493,9590.419%356.310
99AIC904,205,8455,308.1907,764,2167,615,705953,0764,6700.490%420.300
1010AIC1004,399,5285,310.9548,034,0787,898,420939,1184,0850.435%367.650
1211AIC1104,257,3625,289.3037,741,2457,606,236941,1804,3950.467%395.550
1312AIC1204,337,1195,295.1667,809,2347,672,294926,0344,2860.463%385.740
1413AIC1304,330,8115,447.5218,144,0238,007,199999,8424,5090.451%405.810
1514AIC1404,389,6815,279.3007,919,2477,793,214926,9823,9500.426%355.500
1615AIC1504,244,2465,366.7847,912,3687,785,055966,9034,4130.456%397.170
1832AIV015,085,5145,690.7559,027,6807,914,043921,68446,6115.057%4,194.990
1933AIV025,134,4475,704.5359,132,6577,907,544938,04747,5225.066%4,276.980
2034AIV115,023,7465,712.2658,972,2737,932,561924,21547,0835.094%4,237.470
2135AIV125,008,3895,690.5978,935,6187,703,911933,36849,1625.267%4,424.580
2236AIV215,041,9775,746.3408,945,2787,852,886914,44546,3545.069%4,171.860
2337AIV225,106,8605,727.6689,087,3717,878,592927,22647,0345.073%4,233.060
2438AIV315,136,7805,706.7009,141,3107,988,916927,08348,4605.227%4,361.400
2539AIV325,011,9305,700.3718,953,2147,782,716934,75947,1845.048%4,246.560
2640AIV415,105,3935,742.8539,107,8887,967,088926,30547,5145.129%4,276.260
2741AIV425,039,7715,704.7718,960,8007,760,973918,52246,3225.043%4,168.980
2842AIV515,154,6635,714.6469,240,1378,046,951933,97147,6665.104%4,289.940
2943AIV525,046,7175,711.7909,055,4447,800,279944,73748,5885.143%4,372.920
3044AIV615,050,3815,752.0868,958,6577,833,484916,19946,9615.126%4,226.490
3145AIV625,018,4365,729.5738,998,1387,830,760936,03047,6455.090%4,288.050
3246AIV715,060,6885,695.2139,030,5727,938,994929,26147,2525.085%4,252.680
3347AIV725,101,7805,731.5129,163,9237,960,908961,79349,0935.104%4,418.370
3448AIV815,169,7285,740.7349,197,6438,083,016923,49147,0765.098%4,236.840
3549AIV825,031,0675,697.6129,030,8967,874,640944,59648,5955.145%4,373.550
3650AIV915,182,1945,714.2319,310,8048,138,715963,00549,0145.090%4,411.260
3751AIV925,031,8935,731.7259,002,5087,801,356937,05448,0135.124%4,321.170
3852AIV1015,108,7195,707.9089,058,9867,969,191914,57846,8205.119%4,213.800
3953AIV1025,107,4055,698.5069,199,4777,985,793951,65548,7615.124%4,388.490
4254AIV1115,054,4025,726.9448,982,6267,897,445930,67547,5005.104%4,275.000
4355AIV1125,080,0595,700.4559,113,7817,967,220955,31648,7525.103%4,387.680
4456AIV1215,168,1825,731.5389,253,2268,139,257957,93148,7975.094%4,391.730
4557AIV1225,031,5865,716.8908,975,2777,786,142937,91448,0145.119%4,321.260
4658AIV1315,032,7385,686.9709,011,0917,905,300932,21947,6955.116%4,292.550
4759AIV1325,068,9935,703.6099,042,2387,830,774929,39247,6085.122%4,284.720
4860AIV1415,116,8965,706.8299,131,8768,051,776930,98047,1635.066%4,244.670
4961AIV1425,061,2015,713.1629,054,6447,891,118941,00747,9655.097%4,316.850
5062AIV1515,131,4385,733.7309,181,8068,049,541935,63947,7895.108%4,301.010
5163AIV1525,070,1675,711.1039,055,7527,840,342928,73847,9065.158%4,311.540
5416AIC1604,365,1575,287.2347,996,9577,871,530938,9424,1430.441%372.870
5517AIC1704,259,4945,326.6667,886,2027,754,374963,9204,3640.453%392.760
5618AIC1804,491,0135,511.3818,328,6798,189,296977,0884,0600.416%365.400
5719AIC1904,167,4765,301.8337,636,1727,507,791939,3664,4870.478%403.830
5820AIC2004,332,2935,335.5797,931,9357,793,675943,5814,1700.442%375.300
6021AIC2104,274,8915,337.3357,930,0197,792,386975,5554,5470.466%409.230
6122AIC2204,295,4425,272.9737,749,5327,639,180923,4994,3280.469%389.520
6223AIC2304,286,3475,288.9747,771,9107,635,777939,9774,4770.476%402.930
6324AIC2404,310,1665,296.3827,855,8267,729,056940,2674,4400.472%399.600
6425AIC2504,358,2895,619.5688,092,7127,948,048989,7484,6600.471%419.400
6526AIC2604,365,5335,328.4047,967,8507,841,751938,8124,1690.444%375.210
6627AIC2704,182,4065,294.2217,627,6707,507,347943,4934,4060.467%396.540
6728AIC2804,291,2645,266.2957,806,9127,698,563941,1224,0190.427%361.710
6829AIC2904,290,8835,440.7187,989,3617,857,463969,8614,3390.447%390.510
6930AIC3004,326,8845,280.5527,836,0057,717,689940,5734,0580.431%365.220
7031AIC3104,218,9635,292.9027,691,3637,561,350947,3434,7930.506%431.370
7264AIV1615,054,5115,695.5549,034,4557,935,637935,64447,3495.061%4,261.410
7365AIV1625,028,8515,729.7638,996,8937,788,974942,32948,0735.102%4,326.570
7466AIV1715,107,7465,730.1339,089,7657,969,994927,20547,4235.115%4,268.070
7567AIV1725,037,7425,735.9959,014,0397,843,931928,42047,8715.156%4,308.390
7668AIV1815,088,2155,703.5229,050,2017,947,141929,10647,5915.122%4,283.190
7769AIV1825,086,0715,699.5919,124,4307,917,825952,91448,5575.096%4,370.130
7870AIV1915,064,0065,746.4289,030,2327,953,062924,02147,2325.112%4,250.880
7971AIV1925,058,7555,697.3169,037,4997,858,483936,65147,9575.120%4,316.130
8072AIV2015,042,5495,720.0478,983,7087,898,791930,49747,6515.121%4,288.590
8173AIV2025,002,4575,709.1838,961,7817,766,154949,14748,9875.161%4,408.830
8474AIV2115,113,5925,732.5779,099,2857,984,959929,04547,3865.101%4,264.740
8575AIV2125,091,9505,711.2019,063,5837,857,400927,82547,1335.080%4,241.970
8676AIV2215,155,4975,711.9769,228,7978,122,155938,53247,9595.110%4,316.310
8777AIV2225,058,1125,720.6879,074,4947,878,510938,83848,1245.126%4,331.160
8878AIV2315,088,2555,695.2979,037,5907,958,632923,12546,9015.081%4,221.090
8979AIV2324,980,4615,691.3998,931,4117,754,551942,15048,4005.137%4,356.000
9080AIV2415,062,6345,719.3488,984,8427,885,239925,08647,4965.134%4,274.640
9181AIV2425,000,0845,690.5938,936,2747,740,762936,18047,9085.117%4,311.720
9282AIV2515,107,0215,710.7459,062,6967,945,574922,47747,1235.108%4,241.070
9383AIV2525,039,2445,696.1749,029,3227,874,320946,22748,4485.120%4,360.320
9484AIV2615,099,1115,715.0909,066,7837,946,564934,32847,5325.087%4,277.880
9585AIV2625,064,1845,700.5109,038,1927,859,215940,42247,9865.103%4,318.740
9686AIV2715,124,0645,701.0099,120,1467,971,907931,26047,4265.093%4,268.340
9787AIV2725,102,4215,694.2819,110,4017,833,675936,60647,7805.101%4,300.200
9888AIV2815,054,6965,715.6618,977,6497,807,617923,96147,0945.097%4,238.460
9989AIV2825,159,2795,699.2559,128,4687,881,739911,13846,4915.103%4,184.190
10090AIV2915,125,0325,682.5649,101,4957,978,443927,36146,9455.062%4,225.050
10191AIV2925,113,9785,712.1379,137,9437,912,593948,90648,3265.093%4,349.340
10292AIV3015,098,7465,692.3849,027,8477,890,339921,82246,8065.078%4,212.540
10393AIV3025,085,0175,736.0389,093,6897,858,647933,18347,5675.097%4,281.030
10494AIV3115,032,9295,704.7818,961,8277,843,452932,40647,6465.110%4,288.140
10595AIV3124,992,8335,698.7448,862,7627,691,649932,43447,8465.131%4,306.140
+
+ diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_fanin_20260723_044336_icache_report.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_fanin_20260723_044336_icache_report.html new file mode 100644 index 0000000000..c590490e14 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_fanin_20260723_044336_icache_report.html @@ -0,0 +1,362 @@ + + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 fdwic_submit_pmu_raw.json · schema fdwic-submit-pmu-v3 · 32 AIC + 64 AIV

+ +
+

fanin 阶段观察（phase_id=9）

+

边界 fanin_begin_to_fanin_end · 计数语义 running_read_clear_observed_bracket · + 时间语义 boundary_diagnostic_sys_cnt_between_observers · + call_shape=dynamic_balanced · expected_calls=ALL 1024 / AIC 512 / AIV 512

+ +

本报告使用 empty-bracket 估算进入 phase observed + 的局部记录代码开销。 + AIC/AIV 分别按“空区间每组记录开销 × 本阶段实际 begin/end 记录组数”计算，ALL + 再由两类核相加。参考值只从 phase observed 分子扣除该估算；原始 whole 分母保持不变， + 因为空区间没有测量 whole 窗口中的全部记录开销。所以下表的参考比例不是完整业务 + 阶段占完整调度窗口的比例，也不是精确校正。若参考值为负，表示记录代码开销估算已经 + 超过原始 observed；这只能说明该信号低于当前校准的分辨能力，不能截成零或解释成负的 + 业务事件。校准 raw：/home/q00473782/atomic/private/gpt/simpler/outputs/TestPagedAttentionUnroll_Case1_20260723_043533/fdwic_submit_pmu_raw.json · + SHA-256 56199f91545cc6a1fbf18b8bb2cfebae279cb5801516311a040d4facccf321ba。两份 provenance 已核验为同一场景、同一 Git revision 8c6b6e3a412d。

+ + +

本阶段若跨过 linked Kernel，会同时门控 SYS 时间与 PMU counter， + 因而 total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段。 + result-used return-ready atomic 依赖区间只从 SYS 边界累计值扣除，PMU counter + 仍含 atomic 指令事件；source-issue atomic + 保留在时间和计数口径内。没有 linked Kernel 的阶段不触发 PMU 停/开表； + 每次调用的分母仍是外层业务调用次数，不是 Begin/End 读数，也不是排除的 Kernel 调用数。

+ +

阶段原始 PMU 观测来自 running read-clear 的 total cycles，并按 AIC/AIV + 各自校准频率换算；提供 empty-bracket 时，页面主参考值再扣除局部记录代码开销估算。 + phase_elapsed_ticks 只是 SYS 边界闭合诊断，绝不按 1 GHz 当作阶段时间。 + Scalar busy 与 non-scalar residual（逐核先算 total−scalar）共同解释阶段 PMU total； + 未校准时只能展示 raw observed；提供 empty-bracket 后，记录代码开销估算只从 phase + observed 分子扣除，原始 whole 分母保持不变。当前数据无法得到完整业务阶段占完整调度 + 窗口的精确比例。 + linked vector/cube Kernel 从 SYS 与 PMU counter 一并门控排除；result-used return-ready atomic + 依赖区间只从 SYS 边界累计值扣除，I-cache/PMU counter 仍含其指令事件；source-issue atomic + 保留。request/miss 百分比的分母是同一 ELF、同一次采集的 Submit 整窗 primary。 + observed_plus_capture_gap = observed + (primary − shadow)； + 边界读数和插桩 bookkeeping 会进入 sample，因此“观测值”和“加全窗 capture gap”都不是原业务 + 事件数的数学上下界。不同 ELF 不能直接相减后冒充业务真值；提供 empty-bracket 时只按上方 + 明示的公式估算局部记录代码开销。request/miss 的逐核 min/max 是每核累计整个 phase + 的极值，不是逐调用极值。

+
+ + + + +
指标ALL96 核AIC32 核AIV64 核
Phase PMU total扣除记录代码开销估算后的参考值 Σ 1,926,456.381 cycles
≈ 1,167.660 µs（1.649844 cycles/ns）
原始 observed Σ 3,508,426 cycles（≈ 2,126.520 µs）− 记录代码开销估算 1,581,969.619 cycles
记录代码开销估算 / 原始 observed 45.091%；原始逐核 最小 13,075；最大 55,245；原始 3,426.197 cycles/call		扣除记录代码开销估算后的参考值 Σ 620,383.963 cycles
≈ 375.976 µs（1.650062 cycles/ns）
原始 observed Σ 1,380,719 cycles（≈ 836.768 µs）− 记录代码开销估算 760,335.037 cycles
记录代码开销估算 / 原始 observed 55.068%；原始逐核 最小 30,099；最大 55,123；原始 2,696.717 cycles/call		扣除记录代码开销估算后的参考值 Σ 1,306,072.419 cycles
≈ 791.688 µs（1.649731 cycles/ns）
原始 observed Σ 2,127,707 cycles（≈ 1,289.730 µs）− 记录代码开销估算 821,634.581 cycles
记录代码开销估算 / 原始 observed 38.616%；原始逐核 最小 13,075；最大 55,245；原始 4,155.678 cycles/call
Phase scalar busy扣除记录代码开销估算后的参考值 Σ 1,685,046.725 cycles
≈ 1,021.337 µs（1.649844 cycles/ns）
原始 observed Σ 2,709,457 cycles（≈ 1,642.250 µs）− 记录代码开销估算 1,024,410.275 cycles
记录代码开销估算 / 原始 observed 37.809%；原始逐核 最小 9,664；最大 44,119；原始 2,645.954 cycles/call		扣除记录代码开销估算后的参考值 Σ 545,863.675 cycles
≈ 330.814 µs（1.650062 cycles/ns）
原始 observed Σ 1,046,610 cycles（≈ 634.285 µs）− 记录代码开销估算 500,746.325 cycles
记录代码开销估算 / 原始 observed 47.845%；原始逐核 最小 22,790；最大 41,690；原始 2,044.160 cycles/call		扣除记录代码开销估算后的参考值 Σ 1,139,183.050 cycles
≈ 690.527 µs（1.649731 cycles/ns）
原始 observed Σ 1,662,847 cycles（≈ 1,007.950 µs）− 记录代码开销估算 523,663.950 cycles
记录代码开销估算 / 原始 observed 31.492%；原始逐核 最小 9,664；最大 44,119；原始 3,247.748 cycles/call
非 Scalar-busy 残余扣除记录代码开销估算后的参考值 Σ 241,409.656 cycles
≈ 146.323 µs（1.649844 cycles/ns）
原始 observed Σ 798,969 cycles（≈ 484.269 µs）− 记录代码开销估算 557,559.344 cycles
记录代码开销估算 / 原始 observed 69.785%；原始逐核 最小 3,411；最大 13,433；原始 780.243 cycles/call		扣除记录代码开销估算后的参考值 Σ 74,520.288 cycles
≈ 45.162 µs（1.650062 cycles/ns）
原始 observed Σ 334,109 cycles（≈ 202.483 µs）− 记录代码开销估算 259,588.712 cycles
记录代码开销估算 / 原始 observed 77.696%；原始逐核 最小 7,309；最大 13,433；原始 652.557 cycles/call		扣除记录代码开销估算后的参考值 Σ 166,889.369 cycles
≈ 101.162 µs（1.649731 cycles/ns）
原始 observed Σ 464,860 cycles（≈ 281.779 µs）− 记录代码开销估算 297,970.631 cycles
记录代码开销估算 / 原始 observed 64.099%；原始逐核 最小 3,411；最大 11,344；原始 907.930 cycles/call
阶段参考关系（raw 同格保留）扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 87.469%
参考 Non-scalar / 参考 Phase total 12.531%
参考 Phase total / 原始 whole total 0.295%；原始 observed 0.538%
参考 Phase scalar / 原始 whole scalar 0.280%；原始 observed 0.450%
whole scalar−shadow scalar：Σ 2,048 cycles；逐核 最小 8；最大 40		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 87.988%
参考 Non-scalar / 参考 Phase total 12.012%
参考 Phase total / 原始 whole total 0.319%；原始 observed 0.710%
参考 Phase scalar / 原始 whole scalar 0.284%；原始 observed 0.545%
whole scalar−shadow scalar：Σ 1,024 cycles；逐核 最小 24；最大 40		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 87.222%
参考 Non-scalar / 参考 Phase total 12.778%
参考 Phase total / 原始 whole total 0.285%；原始 observed 0.464%
参考 Phase scalar / 原始 whole scalar 0.277%；原始 observed 0.405%
whole scalar−shadow scalar：Σ 1,024 cycles；逐核 最小 8；最大 24
Request（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 285,209.438
原始 observed 342,033 − 记录代码开销估算 56,823.562
参考值 / 原始整窗 0.376%；原始 observed / 原始整窗 0.450%
记录代码开销估算 / 原始 observed 16.613%
原始逐核 最小 1,315；最大 5,485；原始整窗 75,933,197
原始 capture gap +0；加 gap 后 342,033（0.450%）		扣除记录代码开销估算后的参考值 112,236.100
原始 observed 140,013 − 记录代码开销估算 27,776.900
参考值 / 原始整窗 0.413%；原始 observed / 原始整窗 0.515%
记录代码开销估算 / 原始 observed 19.839%
原始逐核 最小 3,304；最大 5,485；原始整窗 27,192,061
原始 capture gap +0；加 gap 后 140,013（0.515%）		扣除记录代码开销估算后的参考值 172,973.337
原始 observed 202,020 − 记录代码开销估算 29,046.662
参考值 / 原始整窗 0.355%；原始 observed / 原始整窗 0.414%
记录代码开销估算 / 原始 observed 14.378%
原始逐核 最小 1,315；最大 5,375；原始整窗 48,741,136
原始 capture gap +0；加 gap 后 202,020（0.414%）
Miss（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 12,298.531
原始 observed 13,782 − 记录代码开销估算 1,483.469
参考值 / 原始整窗 0.527%；原始 observed / 原始整窗 0.591%
记录代码开销估算 / 原始 observed 10.764%
原始逐核 最小 37；最大 277；原始整窗 2,331,934
原始 capture gap +0；加 gap 后 13,782（0.591%）		扣除记录代码开销估算后的参考值 2,525.887
原始 observed 2,557 − 记录代码开销估算 31.113
参考值 / 原始整窗 3.141%；原始 observed / 原始整窗 3.180%
记录代码开销估算 / 原始 observed 1.217%
原始逐核 最小 37；最大 120；原始整窗 80,415
原始 capture gap +0；加 gap 后 2,557（3.180%）		扣除记录代码开销估算后的参考值 9,772.644
原始 observed 11,225 − 记录代码开销估算 1,452.356
参考值 / 原始整窗 0.434%；原始 observed / 原始整窗 0.499%
记录代码开销估算 / 原始 observed 12.939%
原始逐核 最小 83；最大 277；原始整窗 2,251,519
原始 capture gap +0；加 gap 后 11,225（0.499%）
SYS 边界诊断 / Begin-EndΣ 1,016,201 raw ticks
逐核 最小 3,253；最大 19,441；仅边界诊断
Begin / End：1,024 / 1,024
业务调用 1,024 次；排除 linked Kernel 调用 0 次
逐核 4–20；零调用核 0		Σ 320,301 raw ticks
逐核 最小 6,327；最大 13,581；仅边界诊断
Begin / End：512 / 512
业务调用 512 次；排除 linked Kernel 调用 0 次
逐核 12–20；零调用核 0		Σ 695,900 raw ticks
逐核 最小 3,253；最大 19,441；仅边界诊断
Begin / End：512 / 512
业务调用 512 次；排除 linked Kernel 调用 0 次
逐核 4–12；零调用核 0
+
+ + +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
97afeee7e4d4b005f0da4c0faa2ce735aeb128e2ffde3758ddf772b8730ca161
+
Profile / extra cache
submit-pmu-fanin / + c719ec19bb06b379
+
Profiled cache key
TestPagedAttentionUnroll / a5 / fully_distributed_within_core / submit-pmu-fanin
+
Source state
source-v2:8c6b6e3a412d2c7c7113c2d8f714a14eb3786f80:bf6096484c5d41f52ead44517d8a4f75a53b1b934b95a17a62b1c70e2148beaa:d1b14ffd842fc937447b0b562034048bf50348803b4a4fb7b8ecc36e240e4269
+
Source-state stamp
/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/c719ec19bb06b379/aicore/.git_commit
+
Compile definitions
PTO_FDWIC_SUBMIT_PMU=1
PTO_FDWIC_SUBMIT_PMU_PHASE_ID=9
PTO_FDWIC_TRACE_ENABLED=0
+
+
+ + + +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
AICore final2,602,2089eaef3d1d5cc99dcbbbd20a79173817180e80f701e78a2e87c442edbd043908d204,11219cd0bc094afe86b735949db35831585d242c978e8427b47532f8d1ea365d0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/aicore-extra/c719ec19bb06b379/aicore_kernel.o
AIC combined1,933,752081fb61d07e2d47468e24181b8e5f5025310d882dd7140dd68cfa8a88d9fa11e94,976d36952492fa76fb52584f4b6b1d16ace599fd895b854555f2327bc6ef1433a70/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/c719ec19bb06b379/aicore/aicore_aic_combined.o
AIV combined2,194,7769a6cb8f24ea8c0d1ce30bc8d58754acb52a2fb90331c17cd4b10ec3df3b533bc109,136a1524d9b8a93c20fa794cecd098fb62a939254da72f476e52319f592f7f48e47/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/c719ec19bb06b379/aicore/aicore_aiv_combined.o
Host runtime11,776,1440369d0602054bf612f86b58c294503e00486bd505bff9ee7a455994f043147cc454,6325ac381ed10be5f5d492b8072c051a177131f4caec6d6f6bc9bf50880da4bb0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so
+
+ +
+
原始全局 Submit 时间范围（墙钟） + 4,642.811 µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + 128096 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-2561404dff4a252fcc7c1593f7b84727f177f3a180e0fcc1e08b9b7cd2b0865b3fa
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + 90.000 ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×90 ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
+
+

ALL · 96 核

+
+
Submit PMU total/core
+
均值 6,798,412.7；最小 5,581,167；最大 7,356,684 cycles
+ 等效时间 均值 4,120.640；最小 3,382.845；最大 4,459.018 µs + （按 1.649844 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 6,278,028.6；最小 5,501,201；最大 6,799,495 cycles
+ 等效时间 均值 3,805.226；最小 3,334.376；最大 4,121.296 µs； + 加权占比 92.346% +
+
非 Scalar-busy 残余/core
+
均值 520,384.2；最小 65,756；最大 851,292 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 315.414；最小 39.856；最大 515.983 µs +
+
SYS gate 边界诊断/core
+
均值 3,110,526.2；最小 2,715,923；最大 3,334,908 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 4,458.012；最小 4,086.976；最大 4,642.811 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 1,347,485.6；最小 1,157,617；最大 1,602,558 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 69.774% +
+
Primary I-cache request/core
最小 740,030；最大 918,295
+
Primary I-cache miss/core
最小 1,911；最大 36,926
+
加权 miss rate
3.071%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 171.990；最大 3,323.340 µs
+
+
+ +
+

AIC · 32 核

+
+
Submit PMU total/core
+
均值 6,080,257.1；最小 5,581,167；最大 6,879,135 cycles
+ 等效时间 均值 3,684.866；最小 3,382.398；最大 4,169.016 µs + （按 1.650062 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 6,000,247.2；最小 5,501,201；最大 6,799,495 cycles
+ 等效时间 均值 3,636.377；最小 3,333.936；最大 4,120.751 µs； + 加权占比 98.684% +
+
非 Scalar-busy 残余/core
+
均值 80,009.8；最小 65,756；最大 96,032 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 48.489；最小 39.851；最大 58.199 µs +
+
SYS gate 边界诊断/core
+
均值 2,911,392.0；最小 2,715,923；最大 3,204,546 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 4,253.860；最小 4,086.976；最大 4,642.811 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 1,342,467.6；最小 1,157,617；最大 1,602,558 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 68.441% +
+
Primary I-cache request/core
最小 800,763；最大 918,295
+
Primary I-cache miss/core
最小 1,911；最大 3,228
+
加权 miss rate
0.296%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 171.990；最大 290.520 µs
+
+
+ +
+

AIV · 64 核

+
+
Submit PMU total/core
+
均值 7,157,490.6；最小 6,936,715；最大 7,356,684 cycles
+ 等效时间 均值 4,338.580；最小 4,204.755；最大 4,459.323 µs + （按 1.649731 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 6,416,919.2；最小 6,149,147；最大 6,691,809 cycles
+ 等效时间 均值 3,889.676；最小 3,727.363；最大 4,056.303 µs； + 加权占比 89.653% +
+
非 Scalar-busy 残余/core
+
均值 740,571.3；最小 614,614；最大 851,292 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 448.904；最小 372.554；最大 516.019 µs +
+
SYS gate 边界诊断/core
+
均值 3,210,093.4；最小 3,095,127；最大 3,334,908 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 4,560.088；最小 4,533.147；最大 4,597.859 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 1,349,994.6；最小 1,215,898；最大 1,475,753 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 70.395% +
+
Primary I-cache request/core
最小 740,030；最大 790,052
+
Primary I-cache miss/core
最小 33,191；最大 36,926
+
加权 miss rate
4.619%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 2,987.190；最大 3,323.340 µs
+
+
+
+

逐物理核分布

+

AICAIV

+
+
+
SYS gate boundary-diagnostic ticks/core
+ + + + 3,334,908 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_submit_elapsed_ticks=2971982physical=1 logical=1 role=AIC scalar_submit_elapsed_ticks=2911020physical=2 logical=2 role=AIC scalar_submit_elapsed_ticks=3106899physical=3 logical=3 role=AIC scalar_submit_elapsed_ticks=2932270physical=4 logical=4 role=AIC scalar_submit_elapsed_ticks=2898398physical=5 logical=5 role=AIC scalar_submit_elapsed_ticks=2772951physical=6 logical=6 role=AIC scalar_submit_elapsed_ticks=2896115physical=7 logical=7 role=AIC scalar_submit_elapsed_ticks=2715923physical=8 logical=8 role=AIC scalar_submit_elapsed_ticks=3013511physical=9 logical=9 role=AIC scalar_submit_elapsed_ticks=2918660physical=10 logical=10 role=AIC scalar_submit_elapsed_ticks=2922449physical=12 logical=11 role=AIC scalar_submit_elapsed_ticks=3075654physical=13 logical=12 role=AIC scalar_submit_elapsed_ticks=2858461physical=14 logical=13 role=AIC scalar_submit_elapsed_ticks=2904823physical=15 logical=14 role=AIC scalar_submit_elapsed_ticks=3204546physical=16 logical=15 role=AIC scalar_submit_elapsed_ticks=2840624physical=54 logical=16 role=AIC scalar_submit_elapsed_ticks=3043774physical=55 logical=17 role=AIC scalar_submit_elapsed_ticks=2818763physical=56 logical=18 role=AIC scalar_submit_elapsed_ticks=2929381physical=57 logical=19 role=AIC scalar_submit_elapsed_ticks=2793661physical=58 logical=20 role=AIC scalar_submit_elapsed_ticks=2928628physical=60 logical=21 role=AIC scalar_submit_elapsed_ticks=2852650physical=61 logical=22 role=AIC scalar_submit_elapsed_ticks=2992355physical=62 logical=23 role=AIC scalar_submit_elapsed_ticks=2797048physical=63 logical=24 role=AIC scalar_submit_elapsed_ticks=2932363physical=64 logical=25 role=AIC scalar_submit_elapsed_ticks=2812066physical=65 logical=26 role=AIC scalar_submit_elapsed_ticks=2940321physical=66 logical=27 role=AIC scalar_submit_elapsed_ticks=2931745physical=67 logical=28 role=AIC scalar_submit_elapsed_ticks=2902850physical=68 logical=29 role=AIC scalar_submit_elapsed_ticks=2808197physical=69 logical=30 role=AIC scalar_submit_elapsed_ticks=2881214physical=70 logical=31 role=AIC scalar_submit_elapsed_ticks=2855241physical=18 logical=32 role=AIV scalar_submit_elapsed_ticks=3286786physical=19 logical=33 role=AIV scalar_submit_elapsed_ticks=3253545physical=20 logical=34 role=AIV scalar_submit_elapsed_ticks=3236922physical=21 logical=35 role=AIV scalar_submit_elapsed_ticks=3334908physical=22 logical=36 role=AIV scalar_submit_elapsed_ticks=3237014physical=23 logical=37 role=AIV scalar_submit_elapsed_ticks=3184120physical=24 logical=38 role=AIV scalar_submit_elapsed_ticks=3279161physical=25 logical=39 role=AIV scalar_submit_elapsed_ticks=3206757physical=26 logical=40 role=AIV scalar_submit_elapsed_ticks=3266137physical=27 logical=41 role=AIV scalar_submit_elapsed_ticks=3191237physical=28 logical=42 role=AIV scalar_submit_elapsed_ticks=3197151physical=29 logical=43 role=AIV scalar_submit_elapsed_ticks=3218865physical=30 logical=44 role=AIV scalar_submit_elapsed_ticks=3250878physical=31 logical=45 role=AIV scalar_submit_elapsed_ticks=3172225physical=32 logical=46 role=AIV scalar_submit_elapsed_ticks=3209918physical=33 logical=47 role=AIV scalar_submit_elapsed_ticks=3162273physical=34 logical=48 role=AIV scalar_submit_elapsed_ticks=3273260physical=35 logical=49 role=AIV scalar_submit_elapsed_ticks=3095127physical=36 logical=50 role=AIV scalar_submit_elapsed_ticks=3239829physical=37 logical=51 role=AIV scalar_submit_elapsed_ticks=3184936physical=38 logical=52 role=AIV scalar_submit_elapsed_ticks=3177962physical=39 logical=53 role=AIV scalar_submit_elapsed_ticks=3151542physical=42 logical=54 role=AIV scalar_submit_elapsed_ticks=3255570physical=43 logical=55 role=AIV scalar_submit_elapsed_ticks=3219204physical=44 logical=56 role=AIV scalar_submit_elapsed_ticks=3201277physical=45 logical=57 role=AIV scalar_submit_elapsed_ticks=3229009physical=46 logical=58 role=AIV scalar_submit_elapsed_ticks=3131246physical=47 logical=59 role=AIV scalar_submit_elapsed_ticks=3211410physical=48 logical=60 role=AIV scalar_submit_elapsed_ticks=3131259physical=49 logical=61 role=AIV scalar_submit_elapsed_ticks=3121229physical=50 logical=62 role=AIV scalar_submit_elapsed_ticks=3240953physical=51 logical=63 role=AIV scalar_submit_elapsed_ticks=3166136physical=72 logical=64 role=AIV scalar_submit_elapsed_ticks=3247247physical=73 logical=65 role=AIV scalar_submit_elapsed_ticks=3101966physical=74 logical=66 role=AIV scalar_submit_elapsed_ticks=3174675physical=75 logical=67 role=AIV scalar_submit_elapsed_ticks=3150736physical=76 logical=68 role=AIV scalar_submit_elapsed_ticks=3209329physical=77 logical=69 role=AIV scalar_submit_elapsed_ticks=3169900physical=78 logical=70 role=AIV scalar_submit_elapsed_ticks=3250528physical=79 logical=71 role=AIV scalar_submit_elapsed_ticks=3188910physical=80 logical=72 role=AIV scalar_submit_elapsed_ticks=3201286physical=81 logical=73 role=AIV scalar_submit_elapsed_ticks=3112564physical=84 logical=74 role=AIV scalar_submit_elapsed_ticks=3215606physical=85 logical=75 role=AIV scalar_submit_elapsed_ticks=3216742physical=86 logical=76 role=AIV scalar_submit_elapsed_ticks=3216161physical=87 logical=77 role=AIV scalar_submit_elapsed_ticks=3127079physical=88 logical=78 role=AIV scalar_submit_elapsed_ticks=3249197physical=89 logical=79 role=AIV scalar_submit_elapsed_ticks=3177358physical=90 logical=80 role=AIV scalar_submit_elapsed_ticks=3197375physical=91 logical=81 role=AIV scalar_submit_elapsed_ticks=3193076physical=92 logical=82 role=AIV scalar_submit_elapsed_ticks=3300342physical=93 logical=83 role=AIV scalar_submit_elapsed_ticks=3173173physical=94 logical=84 role=AIV scalar_submit_elapsed_ticks=3251149physical=95 logical=85 role=AIV scalar_submit_elapsed_ticks=3199464physical=96 logical=86 role=AIV scalar_submit_elapsed_ticks=3204985physical=97 logical=87 role=AIV scalar_submit_elapsed_ticks=3191512physical=98 logical=88 role=AIV scalar_submit_elapsed_ticks=3206345physical=99 logical=89 role=AIV scalar_submit_elapsed_ticks=3295367physical=100 logical=90 role=AIV scalar_submit_elapsed_ticks=3305899physical=101 logical=91 role=AIV scalar_submit_elapsed_ticks=3232314physical=102 logical=92 role=AIV scalar_submit_elapsed_ticks=3291173physical=103 logical=93 role=AIV scalar_submit_elapsed_ticks=3270769physical=104 logical=94 role=AIV scalar_submit_elapsed_ticks=3211302physical=105 logical=95 role=AIV scalar_submit_elapsed_ticks=3194601 + +
+ +
+
PMU total cycles/core
+ + + + 7,356,684 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC total_cycles=6129552physical=1 logical=1 role=AIC total_cycles=6204113physical=2 logical=2 role=AIC total_cycles=6683106physical=3 logical=3 role=AIC total_cycles=6454294physical=4 logical=4 role=AIC total_cycles=5984443physical=5 logical=5 role=AIC total_cycles=6005435physical=6 logical=6 role=AIC total_cycles=5895775physical=7 logical=7 role=AIC total_cycles=5581167physical=8 logical=8 role=AIC total_cycles=6219835physical=9 logical=9 role=AIC total_cycles=6310169physical=10 logical=10 role=AIC total_cycles=5904540physical=12 logical=11 role=AIC total_cycles=6639698physical=13 logical=12 role=AIC total_cycles=5884600physical=14 logical=13 role=AIC total_cycles=6438803physical=15 logical=14 role=AIC total_cycles=6879135physical=16 logical=15 role=AIC total_cycles=6045253physical=54 logical=16 role=AIC total_cycles=6371585physical=55 logical=17 role=AIC total_cycles=5823522physical=56 logical=18 role=AIC total_cycles=6086460physical=57 logical=19 role=AIC total_cycles=5756338physical=58 logical=20 role=AIC total_cycles=5911239physical=60 logical=21 role=AIC total_cycles=5970236physical=61 logical=22 role=AIC total_cycles=6130599physical=62 logical=23 role=AIC total_cycles=5597313physical=63 logical=24 role=AIC total_cycles=5902362physical=64 logical=25 role=AIC total_cycles=5881587physical=65 logical=26 role=AIC total_cycles=6036213physical=66 logical=27 role=AIC total_cycles=6315842physical=67 logical=28 role=AIC total_cycles=6012580physical=68 logical=29 role=AIC total_cycles=5769893physical=69 logical=30 role=AIC total_cycles=5743157physical=70 logical=31 role=AIC total_cycles=5999382physical=18 logical=32 role=AIV total_cycles=7310198physical=19 logical=33 role=AIV total_cycles=7227860physical=20 logical=34 role=AIV total_cycles=7257547physical=21 logical=35 role=AIV total_cycles=7235534physical=22 logical=36 role=AIV total_cycles=7232457physical=23 logical=37 role=AIV total_cycles=7061195physical=24 logical=38 role=AIV total_cycles=7240175physical=25 logical=39 role=AIV total_cycles=7212761physical=26 logical=40 role=AIV total_cycles=7253692physical=27 logical=41 role=AIV total_cycles=7123194physical=28 logical=42 role=AIV total_cycles=7124607physical=29 logical=43 role=AIV total_cycles=7162650physical=30 logical=44 role=AIV total_cycles=7174163physical=31 logical=45 role=AIV total_cycles=7154810physical=32 logical=46 role=AIV total_cycles=7017733physical=33 logical=47 role=AIV total_cycles=7028378physical=34 logical=48 role=AIV total_cycles=7227720physical=35 logical=49 role=AIV total_cycles=7130155physical=36 logical=50 role=AIV total_cycles=7151739physical=37 logical=51 role=AIV total_cycles=7049426physical=38 logical=52 role=AIV total_cycles=7207720physical=39 logical=53 role=AIV total_cycles=7084452physical=42 logical=54 role=AIV total_cycles=7332455physical=43 logical=55 role=AIV total_cycles=7173657physical=44 logical=56 role=AIV total_cycles=7198457physical=45 logical=57 role=AIV total_cycles=7239224physical=46 logical=58 role=AIV total_cycles=7034565physical=47 logical=59 role=AIV total_cycles=7249097physical=48 logical=60 role=AIV total_cycles=7093394physical=49 logical=61 role=AIV total_cycles=6997405physical=50 logical=62 role=AIV total_cycles=7150420physical=51 logical=63 role=AIV total_cycles=7030252physical=72 logical=64 role=AIV total_cycles=7356684physical=73 logical=65 role=AIV total_cycles=6971626physical=74 logical=66 role=AIV total_cycles=7096785physical=75 logical=67 role=AIV total_cycles=7120615physical=76 logical=68 role=AIV total_cycles=7225240physical=77 logical=69 role=AIV total_cycles=7152508physical=78 logical=70 role=AIV total_cycles=7352674physical=79 logical=71 role=AIV total_cycles=7217139physical=80 logical=72 role=AIV total_cycles=7125176physical=81 logical=73 role=AIV total_cycles=6936715physical=84 logical=74 role=AIV total_cycles=7134613physical=85 logical=75 role=AIV total_cycles=7141363physical=86 logical=76 role=AIV total_cycles=7050565physical=87 logical=77 role=AIV total_cycles=6945991physical=88 logical=78 role=AIV total_cycles=7203367physical=89 logical=79 role=AIV total_cycles=7190672physical=90 logical=80 role=AIV total_cycles=7120354physical=91 logical=81 role=AIV total_cycles=7135883physical=92 logical=82 role=AIV total_cycles=7215245physical=93 logical=83 role=AIV total_cycles=7122643physical=94 logical=84 role=AIV total_cycles=7220369physical=95 logical=85 role=AIV total_cycles=7148686physical=96 logical=86 role=AIV total_cycles=7081618physical=97 logical=87 role=AIV total_cycles=7037917physical=98 logical=88 role=AIV total_cycles=7137844physical=99 logical=89 role=AIV total_cycles=7291554physical=100 logical=90 role=AIV total_cycles=7304281physical=101 logical=91 role=AIV total_cycles=7155906physical=102 logical=92 role=AIV total_cycles=7306297physical=103 logical=93 role=AIV total_cycles=7248520physical=104 logical=94 role=AIV total_cycles=7058668physical=105 logical=95 role=AIV total_cycles=7104787 + +
+ +
+
Scalar busy cycles/core
+ + + + 6,799,495 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_busy=6057414physical=1 logical=1 role=AIC scalar_busy=6124914physical=2 logical=2 role=AIC scalar_busy=6592161physical=3 logical=3 role=AIC scalar_busy=6379242physical=4 logical=4 role=AIC scalar_busy=5908290physical=5 logical=5 role=AIC scalar_busy=5923687physical=6 logical=6 role=AIC scalar_busy=5814402physical=7 logical=7 role=AIC scalar_busy=5501201physical=8 logical=8 role=AIC scalar_busy=6149913physical=9 logical=9 role=AIC scalar_busy=6218991physical=10 logical=10 role=AIC scalar_busy=5830937physical=12 logical=11 role=AIC scalar_busy=6544178physical=13 logical=12 role=AIC scalar_busy=5815801physical=14 logical=13 role=AIC scalar_busy=6359748physical=15 logical=14 role=AIC scalar_busy=6799495physical=16 logical=15 role=AIC scalar_busy=5958389physical=54 logical=16 role=AIC scalar_busy=6305829physical=55 logical=17 role=AIC scalar_busy=5744375physical=56 logical=18 role=AIC scalar_busy=6010924physical=57 logical=19 role=AIC scalar_busy=5664562physical=58 logical=20 role=AIC scalar_busy=5822832physical=60 logical=21 role=AIC scalar_busy=5886463physical=61 logical=22 role=AIC scalar_busy=6057394physical=62 logical=23 role=AIC scalar_busy=5522223physical=63 logical=24 role=AIC scalar_busy=5822639physical=64 logical=25 role=AIC scalar_busy=5802377physical=65 logical=26 role=AIC scalar_busy=5968332physical=66 logical=27 role=AIC scalar_busy=6228615physical=67 logical=28 role=AIC scalar_busy=5939168physical=68 logical=29 role=AIC scalar_busy=5673861physical=69 logical=30 role=AIC scalar_busy=5662619physical=70 logical=31 role=AIC scalar_busy=5916936physical=18 logical=32 role=AIV scalar_busy=6554434physical=19 logical=33 role=AIV scalar_busy=6416221physical=20 logical=34 role=AIV scalar_busy=6560663physical=21 logical=35 role=AIV scalar_busy=6506888physical=22 logical=36 role=AIV scalar_busy=6463351physical=23 logical=37 role=AIV scalar_busy=6229849physical=24 logical=38 role=AIV scalar_busy=6562152physical=25 logical=39 role=AIV scalar_busy=6418134physical=26 logical=40 role=AIV scalar_busy=6508118physical=27 logical=41 role=AIV scalar_busy=6324967physical=28 logical=42 role=AIV scalar_busy=6400428physical=29 logical=43 role=AIV scalar_busy=6327259physical=30 logical=44 role=AIV scalar_busy=6467727physical=31 logical=45 role=AIV scalar_busy=6365384physical=32 logical=46 role=AIV scalar_busy=6333783physical=33 logical=47 role=AIV scalar_busy=6241258physical=34 logical=48 role=AIV scalar_busy=6534470physical=35 logical=49 role=AIV scalar_busy=6411300physical=36 logical=50 role=AIV scalar_busy=6419364physical=37 logical=51 role=AIV scalar_busy=6262604physical=38 logical=52 role=AIV scalar_busy=6497499physical=39 logical=53 role=AIV scalar_busy=6347246physical=42 logical=54 role=AIV scalar_busy=6634262physical=43 logical=55 role=AIV scalar_busy=6399408physical=44 logical=56 role=AIV scalar_busy=6553688physical=45 logical=57 role=AIV scalar_busy=6472982physical=46 logical=58 role=AIV scalar_busy=6333221physical=47 logical=59 role=AIV scalar_busy=6498046physical=48 logical=60 role=AIV scalar_busy=6478780physical=49 logical=61 role=AIV scalar_busy=6288646physical=50 logical=62 role=AIV scalar_busy=6450343physical=51 logical=63 role=AIV scalar_busy=6239883physical=72 logical=64 role=AIV scalar_busy=6691809physical=73 logical=65 role=AIV scalar_busy=6226967physical=74 logical=66 role=AIV scalar_busy=6383356physical=75 logical=67 role=AIV scalar_busy=6358713physical=76 logical=68 role=AIV scalar_busy=6597640physical=77 logical=69 role=AIV scalar_busy=6402408physical=78 logical=70 role=AIV scalar_busy=6669755physical=79 logical=71 role=AIV scalar_busy=6454714physical=80 logical=72 role=AIV scalar_busy=6480688physical=81 logical=73 role=AIV scalar_busy=6149147physical=84 logical=74 role=AIV scalar_busy=6464024physical=85 logical=75 role=AIV scalar_busy=6427835physical=86 logical=76 role=AIV scalar_busy=6371583physical=87 logical=77 role=AIV scalar_busy=6161895physical=88 logical=78 role=AIV scalar_busy=6520015physical=89 logical=79 role=AIV scalar_busy=6372022physical=90 logical=80 role=AIV scalar_busy=6418008physical=91 logical=81 role=AIV scalar_busy=6329393physical=92 logical=82 role=AIV scalar_busy=6493437physical=93 logical=83 role=AIV scalar_busy=6373440physical=94 logical=84 role=AIV scalar_busy=6504187physical=95 logical=85 role=AIV scalar_busy=6369856physical=96 logical=86 role=AIV scalar_busy=6347635physical=97 logical=87 role=AIV scalar_busy=6253422physical=98 logical=88 role=AIV scalar_busy=6367998physical=99 logical=89 role=AIV scalar_busy=6496123physical=100 logical=90 role=AIV scalar_busy=6613479physical=101 logical=91 role=AIV scalar_busy=6304614physical=102 logical=92 role=AIV scalar_busy=6555105physical=103 logical=93 role=AIV scalar_busy=6415345physical=104 logical=94 role=AIV scalar_busy=6314162physical=105 logical=95 role=AIV scalar_busy=6291698 + +
+ +
+
Primary I-cache requests/core
+ + + + 918,295 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_requests=821402physical=1 logical=1 role=AIC icache_requests=880707physical=2 logical=2 role=AIC icache_requests=890363physical=3 logical=3 role=AIC icache_requests=899743physical=4 logical=4 role=AIC icache_requests=816057physical=5 logical=5 role=AIC icache_requests=859979physical=6 logical=6 role=AIC icache_requests=821098physical=7 logical=7 role=AIC icache_requests=823018physical=8 logical=8 role=AIC icache_requests=839621physical=9 logical=9 role=AIC icache_requests=890700physical=10 logical=10 role=AIC icache_requests=809781physical=12 logical=11 role=AIC icache_requests=914455physical=13 logical=12 role=AIC icache_requests=812975physical=14 logical=13 role=AIC icache_requests=903131physical=15 logical=14 role=AIC icache_requests=918295physical=16 logical=15 role=AIC icache_requests=862914physical=54 logical=16 role=AIC icache_requests=862246physical=55 logical=17 role=AIC icache_requests=838900physical=56 logical=18 role=AIC icache_requests=821939physical=57 logical=19 role=AIC icache_requests=836710physical=58 logical=20 role=AIC icache_requests=811840physical=60 logical=21 role=AIC icache_requests=867364physical=61 logical=22 role=AIC icache_requests=857440physical=62 logical=23 role=AIC icache_requests=800763physical=63 logical=24 role=AIC icache_requests=829376physical=64 logical=25 role=AIC icache_requests=840606physical=65 logical=26 role=AIC icache_requests=823759physical=66 logical=27 role=AIC icache_requests=895926physical=67 logical=28 role=AIC icache_requests=833708physical=68 logical=29 role=AIC icache_requests=834830physical=69 logical=30 role=AIC icache_requests=807111physical=70 logical=31 role=AIC icache_requests=865304physical=18 logical=32 role=AIV icache_requests=755691physical=19 logical=33 role=AIV icache_requests=752504physical=20 logical=34 role=AIV icache_requests=747863physical=21 logical=35 role=AIV icache_requests=740030physical=22 logical=36 role=AIV icache_requests=758403physical=23 logical=37 role=AIV icache_requests=763183physical=24 logical=38 role=AIV icache_requests=747583physical=25 logical=39 role=AIV icache_requests=770816physical=26 logical=40 role=AIV icache_requests=763026physical=27 logical=41 role=AIV icache_requests=767131physical=28 logical=42 role=AIV icache_requests=749292physical=29 logical=43 role=AIV icache_requests=790052physical=30 logical=44 role=AIV icache_requests=759480physical=31 logical=45 role=AIV icache_requests=767924physical=32 logical=46 role=AIV icache_requests=759458physical=33 logical=47 role=AIV icache_requests=756912physical=34 logical=48 role=AIV icache_requests=740188physical=35 logical=49 role=AIV icache_requests=753765physical=36 logical=50 role=AIV icache_requests=755057physical=37 logical=51 role=AIV icache_requests=761625physical=38 logical=52 role=AIV icache_requests=749819physical=39 logical=53 role=AIV icache_requests=755842physical=42 logical=54 role=AIV icache_requests=768831physical=43 logical=55 role=AIV icache_requests=774491physical=44 logical=56 role=AIV icache_requests=765737physical=45 logical=57 role=AIV icache_requests=779206physical=46 logical=58 role=AIV icache_requests=756714physical=47 logical=59 role=AIV icache_requests=780052physical=48 logical=60 role=AIV icache_requests=744870physical=49 logical=61 role=AIV icache_requests=759680physical=50 logical=62 role=AIV icache_requests=757666physical=51 logical=63 role=AIV icache_requests=752912physical=72 logical=64 role=AIV icache_requests=781078physical=73 logical=65 role=AIV icache_requests=761028physical=74 logical=66 role=AIV icache_requests=744879physical=75 logical=67 role=AIV icache_requests=767148physical=76 logical=68 role=AIV icache_requests=758457physical=77 logical=69 role=AIV icache_requests=774479physical=78 logical=70 role=AIV icache_requests=772063physical=79 logical=71 role=AIV icache_requests=783027physical=80 logical=72 role=AIV icache_requests=750578physical=81 logical=73 role=AIV icache_requests=768680physical=84 logical=74 role=AIV icache_requests=747264physical=85 logical=75 role=AIV icache_requests=746430physical=86 logical=76 role=AIV icache_requests=758090physical=87 logical=77 role=AIV icache_requests=753249physical=88 logical=78 role=AIV icache_requests=757831physical=89 logical=79 role=AIV icache_requests=784760physical=90 logical=80 role=AIV icache_requests=757982physical=91 logical=81 role=AIV icache_requests=786081physical=92 logical=82 role=AIV icache_requests=763175physical=93 logical=83 role=AIV icache_requests=765496physical=94 logical=84 role=AIV icache_requests=765153physical=95 logical=85 role=AIV icache_requests=768517physical=96 logical=86 role=AIV icache_requests=743680physical=97 logical=87 role=AIV icache_requests=764250physical=98 logical=88 role=AIV icache_requests=758976physical=99 logical=89 role=AIV icache_requests=746227physical=100 logical=90 role=AIV icache_requests=757050physical=101 logical=91 role=AIV icache_requests=773352physical=102 logical=92 role=AIV icache_requests=767354physical=103 logical=93 role=AIV icache_requests=773961physical=104 logical=94 role=AIV icache_requests=756939physical=105 logical=95 role=AIV icache_requests=778099 + +
+ +
+
Primary I-cache misses/core
+ + + + 36,926 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_misses=1985physical=1 logical=1 role=AIC icache_misses=2425physical=2 logical=2 role=AIC icache_misses=2648physical=3 logical=3 role=AIC icache_misses=2111physical=4 logical=4 role=AIC icache_misses=2335physical=5 logical=5 role=AIC icache_misses=2219physical=6 logical=6 role=AIC icache_misses=2799physical=7 logical=7 role=AIC icache_misses=2802physical=8 logical=8 role=AIC icache_misses=2372physical=9 logical=9 role=AIC icache_misses=2759physical=10 logical=10 role=AIC icache_misses=2395physical=12 logical=11 role=AIC icache_misses=2423physical=13 logical=12 role=AIC icache_misses=2411physical=14 logical=13 role=AIC icache_misses=2266physical=15 logical=14 role=AIC icache_misses=2518physical=16 logical=15 role=AIC icache_misses=3024physical=54 logical=16 role=AIC icache_misses=1911physical=55 logical=17 role=AIC icache_misses=2638physical=56 logical=18 role=AIC icache_misses=2223physical=57 logical=19 role=AIC icache_misses=3228physical=58 logical=20 role=AIC icache_misses=2956physical=60 logical=21 role=AIC icache_misses=2708physical=61 logical=22 role=AIC icache_misses=2358physical=62 logical=23 role=AIC icache_misses=2613physical=63 logical=24 role=AIC icache_misses=2336physical=64 logical=25 role=AIC icache_misses=2750physical=65 logical=26 role=AIC icache_misses=2070physical=66 logical=27 role=AIC icache_misses=2765physical=67 logical=28 role=AIC icache_misses=2080physical=68 logical=29 role=AIC icache_misses=3173physical=69 logical=30 role=AIC icache_misses=2571physical=70 logical=31 role=AIC icache_misses=2543physical=18 logical=32 role=AIV icache_misses=35164physical=19 logical=33 role=AIV icache_misses=34541physical=20 logical=34 role=AIV icache_misses=34645physical=21 logical=35 role=AIV icache_misses=33927physical=22 logical=36 role=AIV icache_misses=34807physical=23 logical=37 role=AIV icache_misses=35535physical=24 logical=38 role=AIV icache_misses=34889physical=25 logical=39 role=AIV icache_misses=36026physical=26 logical=40 role=AIV icache_misses=34892physical=27 logical=41 role=AIV icache_misses=35180physical=28 logical=42 role=AIV icache_misses=34097physical=29 logical=43 role=AIV icache_misses=36517physical=30 logical=44 role=AIV icache_misses=34843physical=31 logical=45 role=AIV icache_misses=34876physical=32 logical=46 role=AIV icache_misses=35540physical=33 logical=47 role=AIV icache_misses=35464physical=34 logical=48 role=AIV icache_misses=33191physical=35 logical=49 role=AIV icache_misses=34756physical=36 logical=50 role=AIV icache_misses=35371physical=37 logical=51 role=AIV icache_misses=35509physical=38 logical=52 role=AIV icache_misses=33884physical=39 logical=53 role=AIV icache_misses=34908physical=42 logical=54 role=AIV icache_misses=35735physical=43 logical=55 role=AIV icache_misses=35783physical=44 logical=56 role=AIV icache_misses=35716physical=45 logical=57 role=AIV icache_misses=36323physical=46 logical=58 role=AIV icache_misses=34717physical=47 logical=59 role=AIV icache_misses=35824physical=48 logical=60 role=AIV icache_misses=34616physical=49 logical=61 role=AIV icache_misses=35570physical=50 logical=62 role=AIV icache_misses=34479physical=51 logical=63 role=AIV icache_misses=34688physical=72 logical=64 role=AIV icache_misses=36191physical=73 logical=65 role=AIV icache_misses=35027physical=74 logical=66 role=AIV icache_misses=34069physical=75 logical=67 role=AIV icache_misses=35442physical=76 logical=68 role=AIV icache_misses=35126physical=77 logical=69 role=AIV icache_misses=36159physical=78 logical=70 role=AIV icache_misses=35332physical=79 logical=71 role=AIV icache_misses=35636physical=80 logical=72 role=AIV icache_misses=34483physical=81 logical=73 role=AIV icache_misses=36462physical=84 logical=74 role=AIV icache_misses=34226physical=85 logical=75 role=AIV icache_misses=34260physical=86 logical=76 role=AIV icache_misses=34355physical=87 logical=77 role=AIV icache_misses=34321physical=88 logical=78 role=AIV icache_misses=35205physical=89 logical=79 role=AIV icache_misses=36798physical=90 logical=80 role=AIV icache_misses=34932physical=91 logical=81 role=AIV icache_misses=36216physical=92 logical=82 role=AIV icache_misses=35831physical=93 logical=83 role=AIV icache_misses=36099physical=94 logical=84 role=AIV icache_misses=34942physical=95 logical=85 role=AIV icache_misses=35062physical=96 logical=86 role=AIV icache_misses=34016physical=97 logical=87 role=AIV icache_misses=35623physical=98 logical=88 role=AIV icache_misses=34959physical=99 logical=89 role=AIV icache_misses=33665physical=100 logical=90 role=AIV icache_misses=35459physical=101 logical=91 role=AIV icache_misses=36149physical=102 logical=92 role=AIV icache_misses=35374physical=103 logical=93 role=AIV icache_misses=35402physical=104 logical=94 role=AIV icache_misses=35759physical=105 logical=95 role=AIV icache_misses=36926 + +
+
+

逐核原始主计数

+

报告展示 primary request/miss；phase 模式的 I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和，只用于标记全窗 capture gap；scalar shadow 同样用于核验 phase 读清重建，并显示其相对 CNT2 primary 的 capture loss，不作为第二份性能数据展示。

+
+ + + + + + + +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate90ns量尺 µs
00AIC002,971,9824,156.3426,129,5526,057,414821,4021,9850.242%178.650
11AIC102,911,0204,350.8946,204,1136,124,914880,7072,4250.275%218.250
22AIC203,106,8994,629.9146,683,1066,592,161890,3632,6480.297%238.320
33AIC302,932,2704,388.6386,454,2946,379,242899,7432,1110.235%189.990
44AIC402,898,3984,202.1105,984,4435,908,290816,0572,3350.286%210.150
55AIC502,772,9514,116.3016,005,4355,923,687859,9792,2190.258%199.710
66AIC602,896,1154,133.1565,895,7755,814,402821,0982,7990.341%251.910
77AIC702,715,9234,101.5195,581,1675,501,201823,0182,8020.340%252.180
88AIC803,013,5114,233.7346,219,8356,149,913839,6212,3720.283%213.480
99AIC902,918,6604,519.7566,310,1696,218,991890,7002,7590.310%248.310
1010AIC1002,922,4494,122.1335,904,5405,830,937809,7812,3950.296%215.550
1211AIC1103,075,6544,642.8116,639,6986,544,178914,4552,4230.265%218.070
1312AIC1202,858,4614,091.9765,884,6005,815,801812,9752,4110.297%216.990
1413AIC1302,904,8234,392.3736,438,8036,359,748903,1312,2660.251%203.940
1514AIC1403,204,5464,608.0096,879,1356,799,495918,2952,5180.274%226.620
1615AIC1502,840,6244,292.9756,045,2535,958,389862,9143,0240.350%272.160
1832AIV013,286,7864,544.8607,310,1986,554,434755,69135,1644.653%3,164.760
1933AIV023,253,5454,548.1587,227,8606,416,221752,50434,5414.590%3,108.690
2034AIV113,236,9224,565.8367,257,5476,560,663747,86334,6454.633%3,118.050
2135AIV123,334,9084,550.8067,235,5346,506,888740,03033,9274.585%3,053.430
2236AIV213,237,0144,552.3627,232,4576,463,351758,40334,8074.590%3,132.630
2337AIV223,184,1204,557.2277,061,1956,229,849763,18335,5354.656%3,198.150
2438AIV313,279,1614,553.2557,240,1756,562,152747,58334,8894.667%3,140.010
2539AIV323,206,7574,596.7427,212,7616,418,134770,81636,0264.674%3,242.340
2640AIV413,266,1374,569.0907,253,6926,508,118763,02634,8924.573%3,140.280
2741AIV423,191,2374,543.4207,123,1946,324,967767,13135,1804.586%3,166.200
2842AIV513,197,1514,536.0737,124,6076,400,428749,29234,0974.551%3,068.730
2943AIV523,218,8654,567.4847,162,6506,327,259790,05236,5174.622%3,286.530
3044AIV613,250,8784,567.7177,174,1636,467,727759,48034,8434.588%3,135.870
3145AIV623,172,2254,557.2927,154,8106,365,384767,92434,8764.542%3,138.840
3246AIV713,209,9184,581.6387,017,7336,333,783759,45835,5404.680%3,198.600
3347AIV723,162,2734,587.1657,028,3786,241,258756,91235,4644.685%3,191.760
3448AIV813,273,2604,542.0417,227,7206,534,470740,18833,1914.484%2,987.190
3549AIV823,095,1274,540.2417,130,1556,411,300753,76534,7564.611%3,128.040
3650AIV913,239,8294,554.9817,151,7396,419,364755,05735,3714.685%3,183.390
3751AIV923,184,9364,546.1277,049,4266,262,604761,62535,5094.662%3,195.810
3852AIV1013,177,9624,586.1727,207,7206,497,499749,81933,8844.519%3,049.560
3953AIV1023,151,5424,567.8477,084,4526,347,246755,84234,9084.618%3,141.720
4254AIV1113,255,5704,563.5567,332,4556,634,262768,83135,7354.648%3,216.150
4355AIV1123,219,2044,574.0047,173,6576,399,408774,49135,7834.620%3,220.470
4456AIV1213,201,2774,533.5397,198,4576,553,688765,73735,7164.664%3,214.440
4557AIV1223,229,0094,561.3917,239,2246,472,982779,20636,3234.662%3,269.070
4658AIV1313,131,2464,539.7127,034,5656,333,221756,71434,7174.588%3,124.530
4759AIV1323,211,4104,568.8327,249,0976,498,046780,05235,8244.593%3,224.160
4860AIV1413,131,2594,572.8597,093,3946,478,780744,87034,6164.647%3,115.440
4961AIV1423,121,2294,567.4836,997,4056,288,646759,68035,5704.682%3,201.300
5062AIV1513,240,9534,554.2007,150,4206,450,343757,66634,4794.551%3,103.110
5163AIV1523,166,1364,587.2377,030,2526,239,883752,91234,6884.607%3,121.920
5416AIC1603,043,7744,261.3296,371,5856,305,829862,2461,9110.222%171.990
5517AIC1702,818,7634,126.1975,823,5225,744,375838,9002,6380.314%237.420
5618AIC1802,929,3814,247.1506,086,4606,010,924821,9392,2230.270%200.070
5719AIC1902,793,6614,139.9975,756,3385,664,562836,7103,2280.386%290.520
5820AIC2002,928,6284,164.0945,911,2395,822,832811,8402,9560.364%266.040
6021AIC2102,852,6504,281.0615,970,2365,886,463867,3642,7080.312%243.720
6122AIC2202,992,3554,359.6416,130,5996,057,394857,4402,3580.275%212.220
6223AIC2302,797,0484,125.0855,597,3135,522,223800,7632,6130.326%235.170
6324AIC2402,932,3634,103.1755,902,3625,822,639829,3762,3360.282%210.240
6425AIC2502,812,0664,110.4275,881,5875,802,377840,6062,7500.327%247.500
6526AIC2602,940,3214,097.9386,036,2135,968,332823,7592,0700.251%186.300
6627AIC2702,931,7454,534.3036,315,8426,228,615895,9262,7650.309%248.850
6728AIC2802,902,8504,086.9766,012,5805,939,168833,7082,0800.249%187.200
6829AIC2902,808,1974,131.5715,769,8935,673,861834,8303,1730.380%285.570
6930AIC3002,881,2144,115.5015,743,1575,662,619807,1112,5710.319%231.390
7031AIC3102,855,2414,256.4205,999,3825,916,936865,3042,5430.294%228.870
7264AIV1613,247,2474,577.6697,356,6846,691,809781,07836,1914.633%3,257.190
7365AIV1623,101,9664,548.3076,971,6266,226,967761,02835,0274.603%3,152.430
7466AIV1713,174,6754,575.2097,096,7856,383,356744,87934,0694.574%3,066.210
7567AIV1723,150,7364,540.6297,120,6156,358,713767,14835,4424.620%3,189.780
7668AIV1813,209,3294,544.3107,225,2406,597,640758,45735,1264.631%3,161.340
7769AIV1823,169,9004,558.0667,152,5086,402,408774,47936,1594.669%3,254.310
7870AIV1913,250,5284,571.8587,352,6746,669,755772,06335,3324.576%3,179.880
7971AIV1923,188,9104,546.9937,217,1396,454,714783,02735,6364.551%3,207.240
8072AIV2013,201,2864,535.7107,125,1766,480,688750,57834,4834.594%3,103.470
8173AIV2023,112,5644,588.3176,936,7156,149,147768,68036,4624.743%3,281.580
8474AIV2113,215,6064,542.2117,134,6136,464,024747,26434,2264.580%3,080.340
8575AIV2123,216,7424,597.8597,141,3636,427,835746,43034,2604.590%3,083.400
8676AIV2213,216,1614,547.1467,050,5656,371,583758,09034,3554.532%3,091.950
8777AIV2223,127,0794,534.8476,945,9916,161,895753,24934,3214.556%3,088.890
8878AIV2313,249,1974,533.1477,203,3676,520,015757,83135,2054.645%3,168.450
8979AIV2323,177,3584,586.1347,190,6726,372,022784,76036,7984.689%3,311.820
9080AIV2413,197,3754,592.8697,120,3546,418,008757,98234,9324.609%3,143.880
9181AIV2423,193,0764,554.4747,135,8836,329,393786,08136,2164.607%3,259.440
9282AIV2513,300,3424,542.3647,215,2456,493,437763,17535,8314.695%3,224.790
9383AIV2523,173,1734,537.2247,122,6436,373,440765,49636,0994.716%3,248.910
9484AIV2613,251,1494,542.8577,220,3696,504,187765,15334,9424.567%3,144.780
9585AIV2623,199,4644,554.5637,148,6866,369,856768,51735,0624.562%3,155.580
9686AIV2713,204,9854,562.3317,081,6186,347,635743,68034,0164.574%3,061.440
9787AIV2723,191,5124,597.3857,037,9176,253,422764,25035,6234.661%3,206.070
9888AIV2813,206,3454,550.4637,137,8446,367,998758,97634,9594.606%3,146.310
9989AIV2823,295,3674,586.0827,291,5546,496,123746,22733,6654.511%3,029.850
10090AIV2913,305,8994,543.3907,304,2816,613,479757,05035,4594.684%3,191.310
10191AIV2923,232,3144,561.6597,155,9066,304,614773,35236,1494.674%3,253.410
10292AIV3013,291,1734,547.7247,306,2976,555,105767,35435,3744.610%3,183.660
10393AIV3023,270,7694,565.3437,248,5206,415,345773,96135,4024.574%3,186.180
10494AIV3113,211,3024,552.7367,058,6686,314,162756,93935,7594.724%3,218.310
10595AIV3123,194,6014,586.4067,104,7876,291,698778,09936,9264.746%3,323.340
+
+ diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_loser_replay_20260723_044555_icache_report.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_loser_replay_20260723_044555_icache_report.html new file mode 100644 index 0000000000..98bfff1f9e --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_loser_replay_20260723_044555_icache_report.html @@ -0,0 +1,362 @@ + + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 fdwic_submit_pmu_raw.json · schema fdwic-submit-pmu-v3 · 32 AIC + 64 AIV

+ +
+

loser-replay 阶段观察（phase_id=12）

+

边界 register_end_to_drain_block_won_return · 计数语义 running_read_clear_observed_bracket · + 时间语义 boundary_diagnostic_sys_cnt_between_observers · + call_shape=dynamic_balanced · expected_calls=ALL 97280 / AIC 32256 / AIV 65024

+ +

本报告使用 empty-bracket 估算进入 phase observed + 的局部记录代码开销。 + AIC/AIV 分别按“空区间每组记录开销 × 本阶段实际 begin/end 记录组数”计算，ALL + 再由两类核相加。参考值只从 phase observed 分子扣除该估算；原始 whole 分母保持不变， + 因为空区间没有测量 whole 窗口中的全部记录开销。所以下表的参考比例不是完整业务 + 阶段占完整调度窗口的比例，也不是精确校正。若参考值为负，表示记录代码开销估算已经 + 超过原始 observed；这只能说明该信号低于当前校准的分辨能力，不能截成零或解释成负的 + 业务事件。校准 raw：/home/q00473782/atomic/private/gpt/simpler/outputs/TestPagedAttentionUnroll_Case1_20260723_043533/fdwic_submit_pmu_raw.json · + SHA-256 56199f91545cc6a1fbf18b8bb2cfebae279cb5801516311a040d4facccf321ba。两份 provenance 已核验为同一场景、同一 Git revision 8c6b6e3a412d。

+ + +

本阶段若跨过 linked Kernel，会同时门控 SYS 时间与 PMU counter， + 因而 total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段。 + result-used return-ready atomic 依赖区间只从 SYS 边界累计值扣除，PMU counter + 仍含 atomic 指令事件；source-issue atomic + 保留在时间和计数口径内。没有 linked Kernel 的阶段不触发 PMU 停/开表； + 每次调用的分母仍是外层业务调用次数，不是 Begin/End 读数，也不是排除的 Kernel 调用数。

+ +

阶段原始 PMU 观测来自 running read-clear 的 total cycles，并按 AIC/AIV + 各自校准频率换算；提供 empty-bracket 时，页面主参考值再扣除局部记录代码开销估算。 + phase_elapsed_ticks 只是 SYS 边界闭合诊断，绝不按 1 GHz 当作阶段时间。 + Scalar busy 与 non-scalar residual（逐核先算 total−scalar）共同解释阶段 PMU total； + 未校准时只能展示 raw observed；提供 empty-bracket 后，记录代码开销估算只从 phase + observed 分子扣除，原始 whole 分母保持不变。当前数据无法得到完整业务阶段占完整调度 + 窗口的精确比例。 + linked vector/cube Kernel 从 SYS 与 PMU counter 一并门控排除；result-used return-ready atomic + 依赖区间只从 SYS 边界累计值扣除，I-cache/PMU counter 仍含其指令事件；source-issue atomic + 保留。request/miss 百分比的分母是同一 ELF、同一次采集的 Submit 整窗 primary。 + observed_plus_capture_gap = observed + (primary − shadow)； + 边界读数和插桩 bookkeeping 会进入 sample，因此“观测值”和“加全窗 capture gap”都不是原业务 + 事件数的数学上下界。不同 ELF 不能直接相减后冒充业务真值；提供 empty-bracket 时只按上方 + 明示的公式估算局部记录代码开销。request/miss 的逐核 min/max 是每核累计整个 phase + 的极值，不是逐调用极值。

+
+ + + + +
指标ALL96 核AIC32 核AIV64 核
Phase PMU total扣除记录代码开销估算后的参考值 Σ 18,529,058.819 cycles
≈ 11,230.794 µs（1.649844 cycles/ns）
原始 observed Σ 170,777,758 cycles（≈ 103,511.458 µs）− 记录代码开销估算 152,248,699.181 cycles
记录代码开销估算 / 原始 observed 89.150%；原始逐核 最小 1,670,343；最大 1,856,211；原始 1,755.528 cycles/call		扣除记录代码开销估算后的参考值 Σ 5,684,185.638 cycles
≈ 3,444.832 µs（1.650062 cycles/ns）
原始 observed Σ 53,585,293 cycles（≈ 32,474.715 µs）− 记录代码开销估算 47,901,107.362 cycles
记录代码开销估算 / 原始 observed 89.392%；原始逐核 最小 1,670,343；最大 1,678,979；原始 1,661.250 cycles/call		扣除记录代码开销估算后的参考值 Σ 12,844,873.181 cycles
≈ 7,786.041 µs（1.649731 cycles/ns）
原始 observed Σ 117,192,465 cycles（≈ 71,037.318 µs）− 记录代码开销估算 104,347,591.819 cycles
记录代码开销估算 / 原始 observed 89.040%；原始逐核 最小 1,810,993；最大 1,856,211；原始 1,802.296 cycles/call
Phase scalar busy扣除记录代码开销估算后的参考值 Σ 16,387,225.875 cycles
≈ 9,932.591 µs（1.649844 cycles/ns）
原始 observed Σ 114,439,566 cycles（≈ 69,363.871 µs）− 记录代码开销估算 98,052,340.125 cycles
记录代码开销估算 / 原始 observed 85.680%；原始逐核 最小 1,154,651；最大 1,214,047；原始 1,176.394 cycles/call		扣除记录代码开销估算后的参考值 Σ 5,496,880.525 cycles
≈ 3,331.318 µs（1.650062 cycles/ns）
原始 observed Σ 37,043,899 cycles（≈ 22,450.004 µs）− 记录代码开销估算 31,547,018.475 cycles
记录代码开销估算 / 原始 observed 85.161%；原始逐核 最小 1,154,651；最大 1,161,139；原始 1,148.434 cycles/call		扣除记录代码开销估算后的参考值 Σ 10,890,345.350 cycles
≈ 6,601.286 µs（1.649731 cycles/ns）
原始 observed Σ 77,395,667 cycles（≈ 46,914.113 µs）− 记录代码开销估算 66,505,321.650 cycles
记录代码开销估算 / 原始 observed 85.929%；原始逐核 最小 1,203,214；最大 1,214,047；原始 1,190.263 cycles/call
非 Scalar-busy 残余扣除记录代码开销估算后的参考值 Σ 2,141,832.944 cycles
≈ 1,298.203 µs（1.649844 cycles/ns）
原始 observed Σ 56,338,192 cycles（≈ 34,147.587 µs）− 记录代码开销估算 54,196,359.056 cycles
记录代码开销估算 / 原始 observed 96.198%；原始逐核 最小 515,632；最大 643,226；原始 579.134 cycles/call		扣除记录代码开销估算后的参考值 Σ 187,305.113 cycles
≈ 113.514 µs（1.650062 cycles/ns）
原始 observed Σ 16,541,394 cycles（≈ 10,024.711 µs）− 记录代码开销估算 16,354,088.887 cycles
记录代码开销估算 / 原始 observed 98.868%；原始逐核 最小 515,632；最大 517,886；原始 512.816 cycles/call		扣除记录代码开销估算后的参考值 Σ 1,954,527.831 cycles
≈ 1,184.755 µs（1.649731 cycles/ns）
原始 observed Σ 39,796,798 cycles（≈ 24,123.204 µs）− 记录代码开销估算 37,842,270.169 cycles
记录代码开销估算 / 原始 observed 95.089%；原始逐核 最小 604,688；最大 643,226；原始 612.032 cycles/call
阶段参考关系（raw 同格保留）扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 88.441%
参考 Non-scalar / 参考 Phase total 11.559%
参考 Phase total / 原始 whole total 2.381%；原始 observed 21.941%
参考 Phase scalar / 原始 whole scalar 2.331%；原始 observed 16.281%
whole scalar−shadow scalar：Σ 194,560 cycles；逐核 最小 2,012；最大 2,040		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 96.705%
参考 Non-scalar / 参考 Phase total 3.295%
参考 Phase total / 原始 whole total 2.415%；原始 observed 22.763%
参考 Phase scalar / 原始 whole scalar 2.361%；原始 observed 15.914%
whole scalar−shadow scalar：Σ 64,512 cycles；逐核 最小 2,012；最大 2,022		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 84.784%
参考 Non-scalar / 参考 Phase total 15.216%
参考 Phase total / 原始 whole total 2.366%；原始 observed 21.585%
参考 Phase scalar / 原始 whole scalar 2.316%；原始 observed 16.463%
whole scalar−shadow scalar：Σ 130,048 cycles；逐核 最小 2,026；最大 2,040
Request（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 2,957,585.162
原始 observed 8,396,456 − 记录代码开销估算 5,438,870.838
参考值 / 原始整窗 3.444%；原始 observed / 原始整窗 9.778%
记录代码开销估算 / 原始 observed 64.776%
原始逐核 最小 84,633；最大 88,833；原始整窗 85,872,185
原始 capture gap +0；加 gap 后 8,396,456（9.778%）		扣除记录代码开销估算后的参考值 989,790.300
原始 observed 2,739,735 − 记录代码开销估算 1,749,944.700
参考值 / 原始整窗 3.379%；原始 observed / 原始整窗 9.354%
记录代码开销估算 / 原始 observed 63.873%
原始逐核 最小 84,633；最大 86,723；原始整窗 29,289,980
原始 capture gap +0；加 gap 后 2,739,735（9.354%）		扣除记录代码开销估算后的参考值 1,967,794.863
原始 observed 5,656,721 − 记录代码开销估算 3,688,926.137
参考值 / 原始整窗 3.478%；原始 observed / 原始整窗 9.997%
记录代码开销估算 / 原始 observed 65.213%
原始逐核 最小 88,210；最大 88,833；原始整窗 56,582,205
原始 capture gap +0；加 gap 后 5,656,721（9.997%）
Miss（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 -342.331
原始 observed 186,067 − 记录代码开销估算 186,409.331
参考值 / 原始整窗 -0.013%；原始 observed / 原始整窗 7.325%
记录代码开销估算 / 原始 observed 100.184%
原始逐核 最小 34；最大 3,083；原始整窗 2,540,177
原始 capture gap +0；加 gap 后 186,067（7.325%）		扣除记录代码开销估算后的参考值 -138.088
原始 observed 1,822 − 记录代码开销估算 1,960.088
参考值 / 原始整窗 -0.227%；原始 observed / 原始整窗 2.998%
记录代码开销估算 / 原始 observed 107.579%
原始逐核 最小 34；最大 84；原始整窗 60,767
原始 capture gap +0；加 gap 后 1,822（2.998%）		扣除记录代码开销估算后的参考值 -204.244
原始 observed 184,245 − 记录代码开销估算 184,449.244
参考值 / 原始整窗 -0.008%；原始 observed / 原始整窗 7.431%
记录代码开销估算 / 原始 observed 100.111%
原始逐核 最小 2,789；最大 3,083；原始整窗 2,479,410
原始 capture gap +0；加 gap 后 184,245（7.431%）
SYS 边界诊断 / Begin-EndΣ 6,926,924 raw ticks
逐核 最小 58,534；最大 82,670；仅边界诊断
Begin / End：97,280 / 97,280
业务调用 97,280 次；排除 linked Kernel 调用 0 次
逐核 1,006–1,020；零调用核 0		Σ 1,887,061 raw ticks
逐核 最小 58,534；最大 59,712；仅边界诊断
Begin / End：32,256 / 32,256
业务调用 32,256 次；排除 linked Kernel 调用 0 次
逐核 1,006–1,011；零调用核 0		Σ 5,039,863 raw ticks
逐核 最小 74,960；最大 82,670；仅边界诊断
Begin / End：65,024 / 65,024
业务调用 65,024 次；排除 linked Kernel 调用 0 次
逐核 1,013–1,020；零调用核 0
+
+ + +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
3ea40952de7605da13e1bd69f4300e247a832eead749272166b93f1226ea13ef
+
Profile / extra cache
submit-pmu-loser-replay / + dd46e3e7e85e933b
+
Profiled cache key
TestPagedAttentionUnroll / a5 / fully_distributed_within_core / submit-pmu-loser-replay
+
Source state
source-v2:8c6b6e3a412d2c7c7113c2d8f714a14eb3786f80:b9b4a9599d1abac1db4cb19458bb4ae8850770a25ec43b011e45c1d337b3b7fd:72f0ac618e63ba9c1bff4bdf2a52c784402f2cef8f00505aea172ece0b976f2d
+
Source-state stamp
/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/dd46e3e7e85e933b/aicore/.git_commit
+
Compile definitions
PTO_FDWIC_SUBMIT_PMU=1
PTO_FDWIC_SUBMIT_PMU_PHASE_ID=12
PTO_FDWIC_TRACE_ENABLED=0
+
+
+ + + +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
AICore final2,661,904e182a40abdfea4c7b07187eec296931ace4dcf42162175145ff8b24eadb4dead210,768eb5d51111312658cc9f7c3101b01ee0caccde709f2bda2cd4879620d5e3ea18d/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/aicore-extra/dd46e3e7e85e933b/aicore_kernel.o
AIC combined1,989,720829e9739a00dd9db4cd2ce4507197770865d6cc68d12769ac47fcfed4099616a98,176c56d4ef08f963172bab48e05798b038913473d52b3b5232bae616e5bf4b68a8c/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/dd46e3e7e85e933b/aicore/aicore_aic_combined.o
AIV combined2,251,920a8dab04ea55d8c3cfdd455cd05bd8754a595ba02021bf16f649d79b07aca9ae8112,464d6241f972bde84014bb51a00b02814f06c618f847cc798c91648c067ada01d57/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/dd46e3e7e85e933b/aicore/aicore_aiv_combined.o
Host runtime11,776,1440369d0602054bf612f86b58c294503e00486bd505bff9ee7a455994f043147cc454,6325ac381ed10be5f5d492b8072c051a177131f4caec6d6f6bc9bf50880da4bb0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so
+
+ +
+
原始全局 Submit 时间范围（墙钟） + 5,407.661 µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + 128096 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-256929d4f74a46560c1f76bcdf895eb7eb04ebf6811667237cf99dac2a9346f8c88
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + 90.000 ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×90 ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
+
+

ALL · 96 核

+
+
Submit PMU total/core
+
均值 8,107,861.2；最小 7,079,198；最大 8,755,097 cycles
+ 等效时间 均值 4,914.320；最小 4,290.829；最大 5,306.621 µs + （按 1.649844 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,321,896.4；最小 7,004,974；最大 7,883,267 cycles
+ 等效时间 均值 4,437.933；最小 4,245.840；最大 4,778.189 µs； + 加权占比 90.306% +
+
非 Scalar-busy 残余/core
+
均值 785,964.8；最小 68,544；最大 1,238,466 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 476.387；最小 41.546；最大 750.656 µs +
+
SYS gate 边界诊断/core
+
均值 4,436,814.1；最小 3,795,810；最大 4,828,952 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,248.454；最小 4,907.920；最大 5,405.875 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 811,639.8；最小 519,597；最大 1,325,353 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 84.536% +
+
Primary I-cache request/core
最小 862,393；最大 981,167
+
Primary I-cache miss/core
最小 1,343；最大 40,149
+
加权 miss rate
2.958%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 120.870；最大 3,613.410 µs
+
+
+ +
+

AIC · 32 核

+
+
Submit PMU total/core
+
均值 7,356,545.0；最小 7,079,198；最大 7,981,571 cycles
+ 等效时间 均值 4,458.345；最小 4,290.262；最大 4,837.134 µs + （按 1.650062 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,274,188.4；最小 7,004,974；最大 7,883,267 cycles
+ 等效时间 均值 4,408.433；最小 4,245.279；最大 4,777.558 µs； + 加权占比 98.880% +
+
非 Scalar-busy 残余/core
+
均值 82,356.6；最小 68,544；最大 98,453 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 49.911；最小 41.540；最大 59.666 µs +
+
SYS gate 边界诊断/core
+
均值 3,916,298.2；最小 3,795,810；最大 4,075,209 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,018.200；最小 4,907.920；最大 5,362.406 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 1,101,902.0；最小 974,932；最大 1,325,353 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 78.042% +
+
Primary I-cache request/core
最小 881,030；最大 981,167
+
Primary I-cache miss/core
最小 1,343；最大 2,542
+
加权 miss rate
0.207%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 120.870；最大 228.780 µs
+
+
+ +
+

AIV · 64 核

+
+
Submit PMU total/core
+
均值 8,483,519.2；最小 8,279,611；最大 8,755,097 cycles
+ 等效时间 均值 5,142.365；最小 5,018.764；最大 5,306.985 µs + （按 1.649731 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,345,750.4；最小 7,111,715；最大 7,673,497 cycles
+ 等效时间 均值 4,452.696；最小 4,310.833；最大 4,651.363 µs； + 加权占比 86.588% +
+
非 Scalar-busy 残余/core
+
均值 1,137,768.9；最小 1,054,190；最大 1,238,466 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 689.669；最小 639.007；最大 750.708 µs +
+
SYS gate 边界诊断/core
+
均值 4,697,072.0；最小 4,570,724；最大 4,828,952 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,363.581；最小 5,333.772；最大 5,405.875 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 666,508.7；最小 519,597；最大 793,672 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 87.573% +
+
Primary I-cache request/core
最小 862,393；最大 905,967
+
Primary I-cache miss/core
最小 37,315；最大 40,149
+
加权 miss rate
4.382%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 3,358.350；最大 3,613.410 µs
+
+
+
+

逐物理核分布

+

AICAIV

+
+
+
SYS gate boundary-diagnostic ticks/core
+ + + + 4,828,952 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_submit_elapsed_ticks=3906065physical=1 logical=1 role=AIC scalar_submit_elapsed_ticks=3889197physical=2 logical=2 role=AIC scalar_submit_elapsed_ticks=4020122physical=3 logical=3 role=AIC scalar_submit_elapsed_ticks=4075209physical=4 logical=4 role=AIC scalar_submit_elapsed_ticks=3969833physical=5 logical=5 role=AIC scalar_submit_elapsed_ticks=3795810physical=6 logical=6 role=AIC scalar_submit_elapsed_ticks=3870160physical=7 logical=7 role=AIC scalar_submit_elapsed_ticks=3861626physical=8 logical=8 role=AIC scalar_submit_elapsed_ticks=3836404physical=9 logical=9 role=AIC scalar_submit_elapsed_ticks=3897937physical=10 logical=10 role=AIC scalar_submit_elapsed_ticks=4027525physical=12 logical=11 role=AIC scalar_submit_elapsed_ticks=3865565physical=13 logical=12 role=AIC scalar_submit_elapsed_ticks=3878881physical=14 logical=13 role=AIC scalar_submit_elapsed_ticks=3875250physical=15 logical=14 role=AIC scalar_submit_elapsed_ticks=3940521physical=16 logical=15 role=AIC scalar_submit_elapsed_ticks=3867012physical=54 logical=16 role=AIC scalar_submit_elapsed_ticks=3911006physical=55 logical=17 role=AIC scalar_submit_elapsed_ticks=3853737physical=56 logical=18 role=AIC scalar_submit_elapsed_ticks=3934182physical=57 logical=19 role=AIC scalar_submit_elapsed_ticks=3912377physical=58 logical=20 role=AIC scalar_submit_elapsed_ticks=3848959physical=60 logical=21 role=AIC scalar_submit_elapsed_ticks=4019945physical=61 logical=22 role=AIC scalar_submit_elapsed_ticks=3890322physical=62 logical=23 role=AIC scalar_submit_elapsed_ticks=3958852physical=63 logical=24 role=AIC scalar_submit_elapsed_ticks=3965773physical=64 logical=25 role=AIC scalar_submit_elapsed_ticks=4005440physical=65 logical=26 role=AIC scalar_submit_elapsed_ticks=3876254physical=66 logical=27 role=AIC scalar_submit_elapsed_ticks=3863331physical=67 logical=28 role=AIC scalar_submit_elapsed_ticks=3878189physical=68 logical=29 role=AIC scalar_submit_elapsed_ticks=3884876physical=69 logical=30 role=AIC scalar_submit_elapsed_ticks=3946251physical=70 logical=31 role=AIC scalar_submit_elapsed_ticks=3994932physical=18 logical=32 role=AIV scalar_submit_elapsed_ticks=4748396physical=19 logical=33 role=AIV scalar_submit_elapsed_ticks=4718409physical=20 logical=34 role=AIV scalar_submit_elapsed_ticks=4721048physical=21 logical=35 role=AIV scalar_submit_elapsed_ticks=4670121physical=22 logical=36 role=AIV scalar_submit_elapsed_ticks=4675255physical=23 logical=37 role=AIV scalar_submit_elapsed_ticks=4677697physical=24 logical=38 role=AIV scalar_submit_elapsed_ticks=4710246physical=25 logical=39 role=AIV scalar_submit_elapsed_ticks=4673223physical=26 logical=40 role=AIV scalar_submit_elapsed_ticks=4706395physical=27 logical=41 role=AIV scalar_submit_elapsed_ticks=4700867physical=28 logical=42 role=AIV scalar_submit_elapsed_ticks=4701063physical=29 logical=43 role=AIV scalar_submit_elapsed_ticks=4601870physical=30 logical=44 role=AIV scalar_submit_elapsed_ticks=4706908physical=31 logical=45 role=AIV scalar_submit_elapsed_ticks=4721834physical=32 logical=46 role=AIV scalar_submit_elapsed_ticks=4684769physical=33 logical=47 role=AIV scalar_submit_elapsed_ticks=4660121physical=34 logical=48 role=AIV scalar_submit_elapsed_ticks=4796967physical=35 logical=49 role=AIV scalar_submit_elapsed_ticks=4636440physical=36 logical=50 role=AIV scalar_submit_elapsed_ticks=4662373physical=37 logical=51 role=AIV scalar_submit_elapsed_ticks=4699865physical=38 logical=52 role=AIV scalar_submit_elapsed_ticks=4766516physical=39 logical=53 role=AIV scalar_submit_elapsed_ticks=4675143physical=42 logical=54 role=AIV scalar_submit_elapsed_ticks=4694588physical=43 logical=55 role=AIV scalar_submit_elapsed_ticks=4674923physical=44 logical=56 role=AIV scalar_submit_elapsed_ticks=4708392physical=45 logical=57 role=AIV scalar_submit_elapsed_ticks=4702866physical=46 logical=58 role=AIV scalar_submit_elapsed_ticks=4687487physical=47 logical=59 role=AIV scalar_submit_elapsed_ticks=4663148physical=48 logical=60 role=AIV scalar_submit_elapsed_ticks=4711440physical=49 logical=61 role=AIV scalar_submit_elapsed_ticks=4629889physical=50 logical=62 role=AIV scalar_submit_elapsed_ticks=4820463physical=51 logical=63 role=AIV scalar_submit_elapsed_ticks=4704514physical=72 logical=64 role=AIV scalar_submit_elapsed_ticks=4678323physical=73 logical=65 role=AIV scalar_submit_elapsed_ticks=4757550physical=74 logical=66 role=AIV scalar_submit_elapsed_ticks=4728687physical=75 logical=67 role=AIV scalar_submit_elapsed_ticks=4738991physical=76 logical=68 role=AIV scalar_submit_elapsed_ticks=4754528physical=77 logical=69 role=AIV scalar_submit_elapsed_ticks=4727059physical=78 logical=70 role=AIV scalar_submit_elapsed_ticks=4764014physical=79 logical=71 role=AIV scalar_submit_elapsed_ticks=4644710physical=80 logical=72 role=AIV scalar_submit_elapsed_ticks=4624538physical=81 logical=73 role=AIV scalar_submit_elapsed_ticks=4644352physical=84 logical=74 role=AIV scalar_submit_elapsed_ticks=4667237physical=85 logical=75 role=AIV scalar_submit_elapsed_ticks=4753259physical=86 logical=76 role=AIV scalar_submit_elapsed_ticks=4727322physical=87 logical=77 role=AIV scalar_submit_elapsed_ticks=4697834physical=88 logical=78 role=AIV scalar_submit_elapsed_ticks=4667669physical=89 logical=79 role=AIV scalar_submit_elapsed_ticks=4602309physical=90 logical=80 role=AIV scalar_submit_elapsed_ticks=4656195physical=91 logical=81 role=AIV scalar_submit_elapsed_ticks=4678788physical=92 logical=82 role=AIV scalar_submit_elapsed_ticks=4729673physical=93 logical=83 role=AIV scalar_submit_elapsed_ticks=4672424physical=94 logical=84 role=AIV scalar_submit_elapsed_ticks=4736103physical=95 logical=85 role=AIV scalar_submit_elapsed_ticks=4688497physical=96 logical=86 role=AIV scalar_submit_elapsed_ticks=4647342physical=97 logical=87 role=AIV scalar_submit_elapsed_ticks=4690088physical=98 logical=88 role=AIV scalar_submit_elapsed_ticks=4658369physical=99 logical=89 role=AIV scalar_submit_elapsed_ticks=4828952physical=100 logical=90 role=AIV scalar_submit_elapsed_ticks=4673064physical=101 logical=91 role=AIV scalar_submit_elapsed_ticks=4752366physical=102 logical=92 role=AIV scalar_submit_elapsed_ticks=4742351physical=103 logical=93 role=AIV scalar_submit_elapsed_ticks=4687531physical=104 logical=94 role=AIV scalar_submit_elapsed_ticks=4708521physical=105 logical=95 role=AIV scalar_submit_elapsed_ticks=4570724 + +
+ +
+
PMU total cycles/core
+ + + + 8,755,097 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC total_cycles=7265187physical=1 logical=1 role=AIC total_cycles=7201751physical=2 logical=2 role=AIC total_cycles=7574861physical=3 logical=3 role=AIC total_cycles=7981571physical=4 logical=4 role=AIC total_cycles=7567744physical=5 logical=5 role=AIC total_cycles=7118801physical=6 logical=6 role=AIC total_cycles=7156702physical=7 logical=7 role=AIC total_cycles=7156156physical=8 logical=8 role=AIC total_cycles=7079198physical=9 logical=9 role=AIC total_cycles=7385277physical=10 logical=10 role=AIC total_cycles=7675277physical=12 logical=11 role=AIC total_cycles=7169718physical=13 logical=12 role=AIC total_cycles=7288286physical=14 logical=13 role=AIC total_cycles=7303779physical=15 logical=14 role=AIC total_cycles=7300841physical=16 logical=15 role=AIC total_cycles=7246153physical=54 logical=16 role=AIC total_cycles=7427440physical=55 logical=17 role=AIC total_cycles=7146968physical=56 logical=18 role=AIC total_cycles=7407080physical=57 logical=19 role=AIC total_cycles=7442940physical=58 logical=20 role=AIC total_cycles=7157516physical=60 logical=21 role=AIC total_cycles=7790362physical=61 logical=22 role=AIC total_cycles=7349817physical=62 logical=23 role=AIC total_cycles=7425947physical=63 logical=24 role=AIC total_cycles=7513237physical=64 logical=25 role=AIC total_cycles=7580197physical=65 logical=26 role=AIC total_cycles=7129768physical=66 logical=27 role=AIC total_cycles=7177935physical=67 logical=28 role=AIC total_cycles=7171305physical=68 logical=29 role=AIC total_cycles=7192992physical=69 logical=30 role=AIC total_cycles=7395258physical=70 logical=31 role=AIC total_cycles=7629376physical=18 logical=32 role=AIV total_cycles=8533203physical=19 logical=33 role=AIV total_cycles=8463908physical=20 logical=34 role=AIV total_cycles=8469373physical=21 logical=35 role=AIV total_cycles=8457278physical=22 logical=36 role=AIV total_cycles=8393489physical=23 logical=37 role=AIV total_cycles=8410238physical=24 logical=38 role=AIV total_cycles=8566907physical=25 logical=39 role=AIV total_cycles=8424466physical=26 logical=40 role=AIV total_cycles=8527503physical=27 logical=41 role=AIV total_cycles=8524002physical=28 logical=42 role=AIV total_cycles=8485402physical=29 logical=43 role=AIV total_cycles=8287393physical=30 logical=44 role=AIV total_cycles=8522877physical=31 logical=45 role=AIV total_cycles=8608763physical=32 logical=46 role=AIV total_cycles=8456914physical=33 logical=47 role=AIV total_cycles=8382390physical=34 logical=48 role=AIV total_cycles=8755097physical=35 logical=49 role=AIV total_cycles=8402639physical=36 logical=50 role=AIV total_cycles=8396554physical=37 logical=51 role=AIV total_cycles=8527940physical=38 logical=52 role=AIV total_cycles=8634990physical=39 logical=53 role=AIV total_cycles=8572087physical=42 logical=54 role=AIV total_cycles=8453089physical=43 logical=55 role=AIV total_cycles=8392996physical=44 logical=56 role=AIV total_cycles=8543173physical=45 logical=57 role=AIV total_cycles=8556735physical=46 logical=58 role=AIV total_cycles=8634656physical=47 logical=59 role=AIV total_cycles=8428533physical=48 logical=60 role=AIV total_cycles=8527862physical=49 logical=61 role=AIV total_cycles=8314229physical=50 logical=62 role=AIV total_cycles=8722155physical=51 logical=63 role=AIV total_cycles=8436456physical=72 logical=64 role=AIV total_cycles=8472274physical=73 logical=65 role=AIV total_cycles=8682441physical=74 logical=66 role=AIV total_cycles=8511736physical=75 logical=67 role=AIV total_cycles=8564970physical=76 logical=68 role=AIV total_cycles=8622058physical=77 logical=69 role=AIV total_cycles=8534315physical=78 logical=70 role=AIV total_cycles=8653409physical=79 logical=71 role=AIV total_cycles=8320725physical=80 logical=72 role=AIV total_cycles=8279611physical=81 logical=73 role=AIV total_cycles=8488360physical=84 logical=74 role=AIV total_cycles=8384563physical=85 logical=75 role=AIV total_cycles=8544211physical=86 logical=76 role=AIV total_cycles=8485570physical=87 logical=77 role=AIV total_cycles=8489716physical=88 logical=78 role=AIV total_cycles=8368560physical=89 logical=79 role=AIV total_cycles=8311848physical=90 logical=80 role=AIV total_cycles=8379497physical=91 logical=81 role=AIV total_cycles=8465110physical=92 logical=82 role=AIV total_cycles=8491813physical=93 logical=83 role=AIV total_cycles=8452226physical=94 logical=84 role=AIV total_cycles=8625909physical=95 logical=85 role=AIV total_cycles=8465857physical=96 logical=86 role=AIV total_cycles=8349630physical=97 logical=87 role=AIV total_cycles=8419836physical=98 logical=88 role=AIV total_cycles=8364659physical=99 logical=89 role=AIV total_cycles=8637378physical=100 logical=90 role=AIV total_cycles=8420147physical=101 logical=91 role=AIV total_cycles=8554225physical=102 logical=92 role=AIV total_cycles=8532883physical=103 logical=93 role=AIV total_cycles=8435125physical=104 logical=94 role=AIV total_cycles=8523721physical=105 logical=95 role=AIV total_cycles=8299552 + +
+ +
+
Scalar busy cycles/core
+ + + + 7,883,267 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_busy=7196643physical=1 logical=1 role=AIC scalar_busy=7112778physical=2 logical=2 role=AIC scalar_busy=7495386physical=3 logical=3 role=AIC scalar_busy=7883267physical=4 logical=4 role=AIC scalar_busy=7494391physical=5 logical=5 role=AIC scalar_busy=7023587physical=6 logical=6 role=AIC scalar_busy=7072411physical=7 logical=7 role=AIC scalar_busy=7080249physical=8 logical=8 role=AIC scalar_busy=7004974physical=9 logical=9 role=AIC scalar_busy=7311454physical=10 logical=10 role=AIC scalar_busy=7588156physical=12 logical=11 role=AIC scalar_busy=7085115physical=13 logical=12 role=AIC scalar_busy=7219215physical=14 logical=13 role=AIC scalar_busy=7222052physical=15 logical=14 role=AIC scalar_busy=7219833physical=16 logical=15 role=AIC scalar_busy=7158694physical=54 logical=16 role=AIC scalar_busy=7355205physical=55 logical=17 role=AIC scalar_busy=7059503physical=56 logical=18 role=AIC scalar_busy=7329708physical=57 logical=19 role=AIC scalar_busy=7362013physical=58 logical=20 role=AIC scalar_busy=7073141physical=60 logical=21 role=AIC scalar_busy=7704385physical=61 logical=22 role=AIC scalar_busy=7277292physical=62 logical=23 role=AIC scalar_busy=7345889physical=63 logical=24 role=AIC scalar_busy=7424748physical=64 logical=25 role=AIC scalar_busy=7486879physical=65 logical=26 role=AIC scalar_busy=7045326physical=66 logical=27 role=AIC scalar_busy=7079482physical=67 logical=28 role=AIC scalar_busy=7089021physical=68 logical=29 role=AIC scalar_busy=7110243physical=69 logical=30 role=AIC scalar_busy=7310020physical=70 logical=31 role=AIC scalar_busy=7552968physical=18 logical=32 role=AIV scalar_busy=7409162physical=19 logical=33 role=AIV scalar_busy=7298827physical=20 logical=34 role=AIV scalar_busy=7369332physical=21 logical=35 role=AIV scalar_busy=7258158physical=22 logical=36 role=AIV scalar_busy=7283982physical=23 logical=37 role=AIV scalar_busy=7240199physical=24 logical=38 role=AIV scalar_busy=7455938physical=25 logical=39 role=AIV scalar_busy=7232403physical=26 logical=40 role=AIV scalar_busy=7435120physical=27 logical=41 role=AIV scalar_busy=7331471physical=28 logical=42 role=AIV scalar_busy=7346940physical=29 logical=43 role=AIV scalar_busy=7111715physical=30 logical=44 role=AIV scalar_busy=7438401physical=31 logical=45 role=AIV scalar_busy=7451755physical=32 logical=46 role=AIV scalar_busy=7402724physical=33 logical=47 role=AIV scalar_busy=7226284physical=34 logical=48 role=AIV scalar_busy=7673497physical=35 logical=49 role=AIV scalar_busy=7267422physical=36 logical=50 role=AIV scalar_busy=7310155physical=37 logical=51 role=AIV scalar_busy=7355826physical=38 logical=52 role=AIV scalar_busy=7512723physical=39 logical=53 role=AIV scalar_busy=7405662physical=42 logical=54 role=AIV scalar_busy=7362556physical=43 logical=55 role=AIV scalar_busy=7252040physical=44 logical=56 role=AIV scalar_busy=7486338physical=45 logical=57 role=AIV scalar_busy=7357013physical=46 logical=58 role=AIV scalar_busy=7508029physical=47 logical=59 role=AIV scalar_busy=7257600physical=48 logical=60 role=AIV scalar_busy=7441580physical=49 logical=61 role=AIV scalar_busy=7170525physical=50 logical=62 role=AIV scalar_busy=7574363physical=51 logical=63 role=AIV scalar_busy=7247122physical=72 logical=64 role=AIV scalar_busy=7356456physical=73 logical=65 role=AIV scalar_busy=7443975physical=74 logical=66 role=AIV scalar_busy=7385589physical=75 logical=67 role=AIV scalar_busy=7359898physical=76 logical=68 role=AIV scalar_busy=7496340physical=77 logical=69 role=AIV scalar_busy=7316350physical=78 logical=70 role=AIV scalar_busy=7549833physical=79 logical=71 role=AIV scalar_busy=7139845physical=80 logical=72 role=AIV scalar_busy=7197529physical=81 logical=73 role=AIV scalar_busy=7270911physical=84 logical=74 role=AIV scalar_busy=7325678physical=85 logical=75 role=AIV scalar_busy=7381853physical=86 logical=76 role=AIV scalar_busy=7405567physical=87 logical=77 role=AIV scalar_busy=7321698physical=88 logical=78 role=AIV scalar_busy=7307832physical=89 logical=79 role=AIV scalar_busy=7150473physical=90 logical=80 role=AIV scalar_busy=7312651physical=91 logical=81 role=AIV scalar_busy=7287089physical=92 logical=82 role=AIV scalar_busy=7421048physical=93 logical=83 role=AIV scalar_busy=7316787physical=94 logical=84 role=AIV scalar_busy=7524304physical=95 logical=85 role=AIV scalar_busy=7320375physical=96 logical=86 role=AIV scalar_busy=7240182physical=97 logical=87 role=AIV scalar_busy=7198535physical=98 logical=88 role=AIV scalar_busy=7220687physical=99 logical=89 role=AIV scalar_busy=7446963physical=100 logical=90 role=AIV scalar_busy=7353275physical=101 logical=91 role=AIV scalar_busy=7342182physical=102 logical=92 role=AIV scalar_busy=7420451physical=103 logical=93 role=AIV scalar_busy=7240817physical=104 logical=94 role=AIV scalar_busy=7408879physical=105 logical=95 role=AIV scalar_busy=7189109 + +
+ +
+
Primary I-cache requests/core
+ + + + 981,167 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_requests=907211physical=1 logical=1 role=AIC icache_requests=896870physical=2 logical=2 role=AIC icache_requests=942250physical=3 logical=3 role=AIC icache_requests=981167physical=4 logical=4 role=AIC icache_requests=940520physical=5 logical=5 role=AIC icache_requests=888593physical=6 logical=6 role=AIC icache_requests=892012physical=7 logical=7 role=AIC icache_requests=904073physical=8 logical=8 role=AIC icache_requests=881030physical=9 logical=9 role=AIC icache_requests=923096physical=10 logical=10 role=AIC icache_requests=937510physical=12 logical=11 role=AIC icache_requests=883558physical=13 logical=12 role=AIC icache_requests=896297physical=14 logical=13 role=AIC icache_requests=912954physical=15 logical=14 role=AIC icache_requests=902637physical=16 logical=15 role=AIC icache_requests=909112physical=54 logical=16 role=AIC icache_requests=901891physical=55 logical=17 role=AIC icache_requests=894246physical=56 logical=18 role=AIC icache_requests=907865physical=57 logical=19 role=AIC icache_requests=932815physical=58 logical=20 role=AIC icache_requests=891538physical=60 logical=21 role=AIC icache_requests=965526physical=61 logical=22 role=AIC icache_requests=907422physical=62 logical=23 role=AIC icache_requests=928200physical=63 logical=24 role=AIC icache_requests=939336physical=64 logical=25 role=AIC icache_requests=948213physical=65 logical=26 role=AIC icache_requests=891166physical=66 logical=27 role=AIC icache_requests=895868physical=67 logical=28 role=AIC icache_requests=906220physical=68 logical=29 role=AIC icache_requests=902047physical=69 logical=30 role=AIC icache_requests=925679physical=70 logical=31 role=AIC icache_requests=953058physical=18 logical=32 role=AIV icache_requests=886830physical=19 logical=33 role=AIV icache_requests=870771physical=20 logical=34 role=AIV icache_requests=888853physical=21 logical=35 role=AIV icache_requests=890066physical=22 logical=36 role=AIV icache_requests=874683physical=23 logical=37 role=AIV icache_requests=878674physical=24 logical=38 role=AIV icache_requests=880342physical=25 logical=39 role=AIV icache_requests=885344physical=26 logical=40 role=AIV icache_requests=881173physical=27 logical=41 role=AIV icache_requests=885350physical=28 logical=42 role=AIV icache_requests=879482physical=29 logical=43 role=AIV icache_requests=864774physical=30 logical=44 role=AIV icache_requests=885752physical=31 logical=45 role=AIV icache_requests=904730physical=32 logical=46 role=AIV icache_requests=894356physical=33 logical=47 role=AIV icache_requests=879586physical=34 logical=48 role=AIV icache_requests=877898physical=35 logical=49 role=AIV icache_requests=885266physical=36 logical=50 role=AIV icache_requests=880685physical=37 logical=51 role=AIV icache_requests=884633physical=38 logical=52 role=AIV icache_requests=901228physical=39 logical=53 role=AIV icache_requests=882091physical=42 logical=54 role=AIV icache_requests=887291physical=43 logical=55 role=AIV icache_requests=878941physical=44 logical=56 role=AIV icache_requests=893264physical=45 logical=57 role=AIV icache_requests=891386physical=46 logical=58 role=AIV icache_requests=900129physical=47 logical=59 role=AIV icache_requests=868451physical=48 logical=60 role=AIV icache_requests=890998physical=49 logical=61 role=AIV icache_requests=870525physical=50 logical=62 role=AIV icache_requests=894041physical=51 logical=63 role=AIV icache_requests=866814physical=72 logical=64 role=AIV icache_requests=883508physical=73 logical=65 role=AIV icache_requests=905967physical=74 logical=66 role=AIV icache_requests=878830physical=75 logical=67 role=AIV icache_requests=893485physical=76 logical=68 role=AIV icache_requests=893751physical=77 logical=69 role=AIV icache_requests=894317physical=78 logical=70 role=AIV icache_requests=899139physical=79 logical=71 role=AIV icache_requests=869518physical=80 logical=72 role=AIV icache_requests=862393physical=81 logical=73 role=AIV icache_requests=889218physical=84 logical=74 role=AIV icache_requests=875916physical=85 logical=75 role=AIV icache_requests=882929physical=86 logical=76 role=AIV icache_requests=881082physical=87 logical=77 role=AIV icache_requests=883973physical=88 logical=78 role=AIV icache_requests=874731physical=89 logical=79 role=AIV icache_requests=877107physical=90 logical=80 role=AIV icache_requests=885792physical=91 logical=81 role=AIV icache_requests=890606physical=92 logical=82 role=AIV icache_requests=880303physical=93 logical=83 role=AIV icache_requests=893514physical=94 logical=84 role=AIV icache_requests=897950physical=95 logical=85 role=AIV icache_requests=887786physical=96 logical=86 role=AIV icache_requests=868665physical=97 logical=87 role=AIV icache_requests=882912physical=98 logical=88 role=AIV icache_requests=876810physical=99 logical=89 role=AIV icache_requests=874720physical=100 logical=90 role=AIV icache_requests=878777physical=101 logical=91 role=AIV icache_requests=893246physical=102 logical=92 role=AIV icache_requests=895229physical=103 logical=93 role=AIV icache_requests=876490physical=104 logical=94 role=AIV icache_requests=902362physical=105 logical=95 role=AIV icache_requests=866772 + +
+ +
+
Primary I-cache misses/core
+ + + + 40,149 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_misses=1421physical=1 logical=1 role=AIC icache_misses=2140physical=2 logical=2 role=AIC icache_misses=1665physical=3 logical=3 role=AIC icache_misses=2178physical=4 logical=4 role=AIC icache_misses=1343physical=5 logical=5 role=AIC icache_misses=2398physical=6 logical=6 role=AIC icache_misses=2211physical=7 logical=7 role=AIC icache_misses=1849physical=8 logical=8 role=AIC icache_misses=1865physical=9 logical=9 role=AIC icache_misses=1585physical=10 logical=10 role=AIC icache_misses=2033physical=12 logical=11 role=AIC icache_misses=1900physical=13 logical=12 role=AIC icache_misses=1585physical=14 logical=13 role=AIC icache_misses=1774physical=15 logical=14 role=AIC icache_misses=2009physical=16 logical=15 role=AIC icache_misses=2178physical=54 logical=16 role=AIC icache_misses=1765physical=55 logical=17 role=AIC icache_misses=1925physical=56 logical=18 role=AIC icache_misses=1762physical=57 logical=19 role=AIC icache_misses=1742physical=58 logical=20 role=AIC icache_misses=2043physical=60 logical=21 role=AIC icache_misses=1780physical=61 logical=22 role=AIC icache_misses=1810physical=62 logical=23 role=AIC icache_misses=1735physical=63 logical=24 role=AIC icache_misses=2163physical=64 logical=25 role=AIC icache_misses=2036physical=65 logical=26 role=AIC icache_misses=2095physical=66 logical=27 role=AIC icache_misses=2542physical=67 logical=28 role=AIC icache_misses=1918physical=68 logical=29 role=AIC icache_misses=1883physical=69 logical=30 role=AIC icache_misses=1775physical=70 logical=31 role=AIC icache_misses=1659physical=18 logical=32 role=AIV icache_misses=39124physical=19 logical=33 role=AIV icache_misses=37890physical=20 logical=34 role=AIV icache_misses=38804physical=21 logical=35 role=AIV icache_misses=38876physical=22 logical=36 role=AIV icache_misses=38002physical=23 logical=37 role=AIV icache_misses=38660physical=24 logical=38 role=AIV icache_misses=38324physical=25 logical=39 role=AIV icache_misses=38798physical=26 logical=40 role=AIV icache_misses=38321physical=27 logical=41 role=AIV icache_misses=38793physical=28 logical=42 role=AIV icache_misses=38485physical=29 logical=43 role=AIV icache_misses=37542physical=30 logical=44 role=AIV icache_misses=38654physical=31 logical=45 role=AIV icache_misses=40047physical=32 logical=46 role=AIV icache_misses=39082physical=33 logical=47 role=AIV icache_misses=38438physical=34 logical=48 role=AIV icache_misses=38579physical=35 logical=49 role=AIV icache_misses=38951physical=36 logical=50 role=AIV icache_misses=38515physical=37 logical=51 role=AIV icache_misses=39029physical=38 logical=52 role=AIV icache_misses=40149physical=39 logical=53 role=AIV icache_misses=38495physical=42 logical=54 role=AIV icache_misses=38868physical=43 logical=55 role=AIV icache_misses=38427physical=44 logical=56 role=AIV icache_misses=39254physical=45 logical=57 role=AIV icache_misses=39270physical=46 logical=58 role=AIV icache_misses=39443physical=47 logical=59 role=AIV icache_misses=37681physical=48 logical=60 role=AIV icache_misses=39269physical=49 logical=61 role=AIV icache_misses=38076physical=50 logical=62 role=AIV icache_misses=39459physical=51 logical=63 role=AIV icache_misses=37832physical=72 logical=64 role=AIV icache_misses=38744physical=73 logical=65 role=AIV icache_misses=40051physical=74 logical=66 role=AIV icache_misses=38303physical=75 logical=67 role=AIV icache_misses=39315physical=76 logical=68 role=AIV icache_misses=38985physical=77 logical=69 role=AIV icache_misses=39616physical=78 logical=70 role=AIV icache_misses=39309physical=79 logical=71 role=AIV icache_misses=38019physical=80 logical=72 role=AIV icache_misses=37315physical=81 logical=73 role=AIV icache_misses=39167physical=84 logical=74 role=AIV icache_misses=38256physical=85 logical=75 role=AIV icache_misses=39016physical=86 logical=76 role=AIV icache_misses=38620physical=87 logical=77 role=AIV icache_misses=38972physical=88 logical=78 role=AIV icache_misses=38051physical=89 logical=79 role=AIV icache_misses=38551physical=90 logical=80 role=AIV icache_misses=38782physical=91 logical=81 role=AIV icache_misses=39307physical=92 logical=82 role=AIV icache_misses=38439physical=93 logical=83 role=AIV icache_misses=39311physical=94 logical=84 role=AIV icache_misses=39409physical=95 logical=85 role=AIV icache_misses=38694physical=96 logical=86 role=AIV icache_misses=37538physical=97 logical=87 role=AIV icache_misses=39029physical=98 logical=88 role=AIV icache_misses=38626physical=99 logical=89 role=AIV icache_misses=38136physical=100 logical=90 role=AIV icache_misses=38342physical=101 logical=91 role=AIV icache_misses=39291physical=102 logical=92 role=AIV icache_misses=39362physical=103 logical=93 role=AIV icache_misses=38270physical=104 logical=94 role=AIV icache_misses=40057physical=105 logical=95 role=AIV icache_misses=37390 + +
+
+

逐核原始主计数

+

报告展示 primary request/miss；phase 模式的 I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和，只用于标记全窗 capture gap；scalar shadow 同样用于核验 phase 读清重建，并显示其相对 CNT2 primary 的 capture loss，不作为第二份性能数据展示。

+
+ + + + + + + +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate90ns量尺 µs
00AIC003,906,0654,941.9687,265,1877,196,643907,2111,4210.157%127.890
11AIC103,889,1974,957.0717,201,7517,112,778896,8702,1400.239%192.600
22AIC204,020,1225,221.4487,574,8617,495,386942,2501,6650.177%149.850
33AIC304,075,2095,362.4067,981,5717,883,267981,1672,1780.222%196.020
44AIC403,969,8335,017.2107,567,7447,494,391940,5201,3430.143%120.870
55AIC503,795,8104,932.2027,118,8017,023,587888,5932,3980.270%215.820
66AIC603,870,1604,928.6007,156,7027,072,411892,0122,2110.248%198.990
77AIC703,861,6264,951.9897,156,1567,080,249904,0731,8490.205%166.410
88AIC803,836,4044,934.7477,079,1987,004,974881,0301,8650.212%167.850
99AIC903,897,9374,935.4107,385,2777,311,454923,0961,5850.172%142.650
1010AIC1004,027,5255,261.4237,675,2777,588,156937,5102,0330.217%182.970
1211AIC1103,865,5654,925.5497,169,7187,085,115883,5581,9000.215%171.000
1312AIC1203,878,8814,922.8757,288,2867,219,215896,2971,5850.177%142.650
1413AIC1303,875,2504,999.5047,303,7797,222,052912,9541,7740.194%159.660
1514AIC1403,940,5214,915.4537,300,8417,219,833902,6372,0090.223%180.810
1615AIC1503,867,0124,907.9207,246,1537,158,694909,1122,1780.240%196.020
1832AIV014,748,3965,393.2918,533,2037,409,162886,83039,1244.412%3,521.160
1933AIV024,718,4095,350.5228,463,9087,298,827870,77137,8904.351%3,410.100
2034AIV114,721,0485,405.8758,469,3737,369,332888,85338,8044.366%3,492.360
2135AIV124,670,1215,349.2118,457,2787,258,158890,06638,8764.368%3,498.840
2236AIV214,675,2555,360.2038,393,4897,283,982874,68338,0024.345%3,420.180
2337AIV224,677,6975,370.1578,410,2387,240,199878,67438,6604.400%3,479.400
2438AIV314,710,2465,359.8428,566,9077,455,938880,34238,3244.353%3,449.160
2539AIV324,673,2235,380.6548,424,4667,232,403885,34438,7984.382%3,491.820
2640AIV414,706,3955,338.5938,527,5037,435,120881,17338,3214.349%3,448.890
2741AIV424,700,8675,337.0378,524,0027,331,471885,35038,7934.382%3,491.370
2842AIV514,701,0635,359.5308,485,4027,346,940879,48238,4854.376%3,463.650
2943AIV524,601,8705,395.5428,287,3937,111,715864,77437,5424.341%3,378.780
3044AIV614,706,9085,334.5308,522,8777,438,401885,75238,6544.364%3,478.860
3145AIV624,721,8345,339.1478,608,7637,451,755904,73040,0474.426%3,604.230
3246AIV714,684,7695,353.4428,456,9147,402,724894,35639,0824.370%3,517.380
3347AIV724,660,1215,354.4168,382,3907,226,284879,58638,4384.370%3,459.420
3448AIV814,796,9675,366.7898,755,0977,673,497877,89838,5794.394%3,472.110
3549AIV824,636,4405,365.7798,402,6397,267,422885,26638,9514.400%3,505.590
3650AIV914,662,3735,362.9688,396,5547,310,155880,68538,5154.373%3,466.350
3751AIV924,699,8655,391.2748,527,9407,355,826884,63339,0294.412%3,512.610
3852AIV1014,766,5165,354.2098,634,9907,512,723901,22840,1494.455%3,613.410
3953AIV1024,675,1435,363.9268,572,0877,405,662882,09138,4954.364%3,464.550
4254AIV1114,694,5885,349.6478,453,0897,362,556887,29138,8684.381%3,498.120
4355AIV1124,674,9235,364.2068,392,9967,252,040878,94138,4274.372%3,458.430
4456AIV1214,708,3925,350.2728,543,1737,486,338893,26439,2544.394%3,532.860
4557AIV1224,702,8665,359.4088,556,7357,357,013891,38639,2704.405%3,534.300
4658AIV1314,687,4875,355.5228,634,6567,508,029900,12939,4434.382%3,549.870
4759AIV1324,663,1485,379.4258,428,5337,257,600868,45137,6814.339%3,391.290
4860AIV1414,711,4405,341.4458,527,8627,441,580890,99839,2694.407%3,534.210
4961AIV1424,629,8895,366.5478,314,2297,170,525870,52538,0764.374%3,426.840
5062AIV1514,820,4635,348.6288,722,1557,574,363894,04139,4594.414%3,551.310
5163AIV1524,704,5145,385.1458,436,4567,247,122866,81437,8324.364%3,404.880
5416AIC1603,911,0064,946.0697,427,4407,355,205901,8911,7650.196%158.850
5517AIC1703,853,7374,941.2157,146,9687,059,503894,2461,9250.215%173.250
5618AIC1803,934,1824,910.7547,407,0807,329,708907,8651,7620.194%158.580
5719AIC1903,912,3775,056.8797,442,9407,362,013932,8151,7420.187%156.780
5820AIC2003,848,9594,937.7667,157,5167,073,141891,5382,0430.229%183.870
6021AIC2104,019,9455,345.2987,790,3627,704,385965,5261,7800.184%160.200
6122AIC2203,890,3224,927.4557,349,8177,277,292907,4221,8100.199%162.900
6223AIC2303,958,8525,069.4987,425,9477,345,889928,2001,7350.187%156.150
6324AIC2403,965,7735,079.6417,513,2377,424,748939,3362,1630.230%194.670
6425AIC2504,005,4405,223.2567,580,1977,486,879948,2132,0360.215%183.240
6526AIC2603,876,2544,909.9997,129,7687,045,326891,1662,0950.235%188.550
6627AIC2703,863,3314,941.2357,177,9357,079,482895,8682,5420.284%228.780
6728AIC2803,878,1894,912.4017,171,3057,089,021906,2201,9180.212%172.620
6829AIC2903,884,8764,927.5297,192,9927,110,243902,0471,8830.209%169.470
6930AIC3003,946,2515,065.3827,395,2587,310,020925,6791,7750.192%159.750
7031AIC3103,994,9325,272.2557,629,3767,552,968953,0581,6590.174%149.310
7264AIV1614,678,3235,355.7528,472,2747,356,456883,50838,7444.385%3,486.960
7365AIV1624,757,5505,381.6468,682,4417,443,975905,96740,0514.421%3,604.590
7466AIV1714,728,6875,379.7358,511,7367,385,589878,83038,3034.358%3,447.270
7567AIV1724,738,9915,360.1038,564,9707,359,898893,48539,3154.400%3,538.350
7668AIV1814,754,5285,341.1908,622,0587,496,340893,75138,9854.362%3,508.650
7769AIV1824,727,0595,393.1758,534,3157,316,350894,31739,6164.430%3,565.440
7870AIV1914,764,0145,363.2118,653,4097,549,833899,13939,3094.372%3,537.810
7971AIV1924,644,7105,365.1198,320,7257,139,845869,51838,0194.372%3,421.710
8072AIV2014,624,5385,341.4778,279,6117,197,529862,39337,3154.327%3,358.350
8173AIV2024,644,3525,369.3208,488,3607,270,911889,21839,1674.405%3,525.030
8474AIV2114,667,2375,356.1768,384,5637,325,678875,91638,2564.368%3,443.040
8575AIV2124,753,2595,397.5718,544,2117,381,853882,92939,0164.419%3,511.440
8676AIV2214,727,3225,362.4248,485,5707,405,567881,08238,6204.383%3,475.800
8777AIV2224,697,8345,366.3998,489,7167,321,698883,97338,9724.409%3,507.480
8878AIV2314,667,6695,341.8508,368,5607,307,832874,73138,0514.350%3,424.590
8979AIV2324,602,3095,361.2518,311,8487,150,473877,10738,5514.395%3,469.590
9080AIV2414,656,1955,354.2318,379,4977,312,651885,79238,7824.378%3,490.380
9181AIV2424,678,7885,353.3658,465,1107,287,089890,60639,3074.414%3,537.630
9282AIV2514,729,6735,367.3578,491,8137,421,048880,30338,4394.367%3,459.510
9383AIV2524,672,4245,398.8098,452,2267,316,787893,51439,3114.400%3,537.990
9484AIV2614,736,1035,345.7818,625,9097,524,304897,95039,4094.389%3,546.810
9585AIV2624,688,4975,353.4318,465,8577,320,375887,78638,6944.358%3,482.460
9686AIV2714,647,3425,333.7728,349,6307,240,182868,66537,5384.321%3,378.420
9787AIV2724,690,0885,381.0248,419,8367,198,535882,91239,0294.420%3,512.610
9888AIV2814,658,3695,399.1248,364,6597,220,687876,81038,6264.405%3,476.340
9989AIV2824,828,9525,348.5498,637,3787,446,963874,72038,1364.360%3,432.240
10090AIV2914,673,0645,376.2448,420,1477,353,275878,77738,3424.363%3,450.780
10191AIV2924,752,3665,354.9898,554,2257,342,182893,24639,2914.399%3,536.190
10292AIV3014,742,3515,397.7538,532,8837,420,451895,22939,3624.397%3,542.580
10393AIV3024,687,5315,335.9128,435,1257,240,817876,49038,2704.366%3,444.300
10494AIV3114,708,5215,392.0568,523,7217,408,879902,36240,0574.439%3,605.130
10595AIV3124,570,7245,353.1858,299,5527,189,109866,77237,3904.314%3,365.100
+
+ diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_materialize_20260723_043943_icache_report.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_materialize_20260723_043943_icache_report.html new file mode 100644 index 0000000000..a4d6162eb9 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_materialize_20260723_043943_icache_report.html @@ -0,0 +1,362 @@ + + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 fdwic_submit_pmu_raw.json · schema fdwic-submit-pmu-v3 · 32 AIC + 64 AIV

+ +
+

materialize 阶段观察（phase_id=3）

+

边界 materialize_begin_to_materialize_end · 计数语义 running_read_clear_observed_bracket · + 时间语义 boundary_diagnostic_sys_cnt_between_observers · + expected_calls_per_core=1280

+ +

本报告使用 empty-bracket 估算进入 phase observed + 的局部记录代码开销。 + AIC/AIV 分别按“空区间每组记录开销 × 本阶段实际 begin/end 记录组数”计算，ALL + 再由两类核相加。参考值只从 phase observed 分子扣除该估算；原始 whole 分母保持不变， + 因为空区间没有测量 whole 窗口中的全部记录开销。所以下表的参考比例不是完整业务 + 阶段占完整调度窗口的比例，也不是精确校正。若参考值为负，表示记录代码开销估算已经 + 超过原始 observed；这只能说明该信号低于当前校准的分辨能力，不能截成零或解释成负的 + 业务事件。校准 raw：/home/q00473782/atomic/private/gpt/simpler/outputs/TestPagedAttentionUnroll_Case1_20260723_043533/fdwic_submit_pmu_raw.json · + SHA-256 56199f91545cc6a1fbf18b8bb2cfebae279cb5801516311a040d4facccf321ba。两份 provenance 已核验为同一场景、同一 Git revision 8c6b6e3a412d。

+ + +

本阶段若跨过 linked Kernel，会同时门控 SYS 时间与 PMU counter， + 因而 total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段。 + result-used return-ready atomic 依赖区间只从 SYS 边界累计值扣除，PMU counter + 仍含 atomic 指令事件；source-issue atomic + 保留在时间和计数口径内。没有 linked Kernel 的阶段不触发 PMU 停/开表； + 每次调用的分母仍是外层业务调用次数，不是 Begin/End 读数，也不是排除的 Kernel 调用数。

+ +

阶段原始 PMU 观测来自 running read-clear 的 total cycles，并按 AIC/AIV + 各自校准频率换算；提供 empty-bracket 时，页面主参考值再扣除局部记录代码开销估算。 + phase_elapsed_ticks 只是 SYS 边界闭合诊断，绝不按 1 GHz 当作阶段时间。 + Scalar busy 与 non-scalar residual（逐核先算 total−scalar）共同解释阶段 PMU total； + 未校准时只能展示 raw observed；提供 empty-bracket 后，记录代码开销估算只从 phase + observed 分子扣除，原始 whole 分母保持不变。当前数据无法得到完整业务阶段占完整调度 + 窗口的精确比例。 + linked vector/cube Kernel 从 SYS 与 PMU counter 一并门控排除；result-used return-ready atomic + 依赖区间只从 SYS 边界累计值扣除，I-cache/PMU counter 仍含其指令事件；source-issue atomic + 保留。request/miss 百分比的分母是同一 ELF、同一次采集的 Submit 整窗 primary。 + observed_plus_capture_gap = observed + (primary − shadow)； + 边界读数和插桩 bookkeeping 会进入 sample，因此“观测值”和“加全窗 capture gap”都不是原业务 + 事件数的数学上下界。不同 ELF 不能直接相减后冒充业务真值；提供 empty-bracket 时只按上方 + 明示的公式估算局部记录代码开销。request/miss 的逐核 min/max 是每核累计整个 phase + 的极值，不是逐调用极值。

+
+ + + + +
指标ALL96 核AIC32 核AIV64 核
Phase PMU total扣除记录代码开销估算后的参考值 Σ 176,061,279.000 cycles
≈ 106,713.895 µs（1.649844 cycles/ns）
原始 observed Σ 368,349,615 cycles（≈ 223,263.299 µs）− 记录代码开销估算 192,288,336.000 cycles
记录代码开销估算 / 原始 observed 52.203%；原始逐核 最小 3,601,905；最大 4,010,884；原始 2,997.637 cycles/call		扣除记录代码开销估算后的参考值 Σ 55,567,714.000 cycles
≈ 33,676.137 µs（1.650062 cycles/ns）
原始 observed Σ 116,394,517 cycles（≈ 70,539.481 µs）− 记录代码开销估算 60,826,803.000 cycles
记录代码开销估算 / 原始 observed 52.259%；原始逐核 最小 3,601,905；最大 3,683,070；原始 2,841.663 cycles/call		扣除记录代码开销估算后的参考值 Σ 120,493,565.000 cycles
≈ 73,038.310 µs（1.649731 cycles/ns）
原始 observed Σ 251,955,098 cycles（≈ 152,724.958 µs）− 记录代码开销估算 131,461,533.000 cycles
记录代码开销估算 / 原始 observed 52.177%；原始逐核 最小 3,854,196；最大 4,010,884；原始 3,075.624 cycles/call
Phase scalar busy扣除记录代码开销估算后的参考值 Σ 170,097,672.000 cycles
≈ 103,099.246 µs（1.649844 cycles/ns）
原始 observed Σ 293,943,610 cycles（≈ 178,164.487 µs）− 记录代码开销估算 123,845,938.000 cycles
记录代码开销估算 / 原始 observed 42.133%；原始逐核 最小 2,943,032；最大 3,164,251；原始 2,392.119 cycles/call		扣除记录代码开销估算后的参考值 Σ 55,271,435.000 cycles
≈ 33,496.581 µs（1.650062 cycles/ns）
原始 observed Σ 95,331,141 cycles（≈ 57,774.278 µs）− 记录代码开销估算 40,059,706.000 cycles
记录代码开销估算 / 原始 observed 42.022%；原始逐核 最小 2,943,032；最大 3,024,557；原始 2,327.420 cycles/call		扣除记录代码开销估算后的参考值 Σ 114,826,237.000 cycles
≈ 69,603.006 µs（1.649731 cycles/ns）
原始 observed Σ 198,612,469 cycles（≈ 120,390.821 µs）− 记录代码开销估算 83,786,232.000 cycles
记录代码开销估算 / 原始 observed 42.186%；原始逐核 最小 3,063,783；最大 3,164,251；原始 2,424.469 cycles/call
非 Scalar-busy 残余扣除记录代码开销估算后的参考值 Σ 5,963,607.000 cycles
≈ 3,614.649 µs（1.649844 cycles/ns）
原始 observed Σ 74,406,005 cycles（≈ 45,098.812 µs）− 记录代码开销估算 68,442,398.000 cycles
记录代码开销估算 / 原始 observed 91.985%；原始逐核 最小 656,662；最大 886,262；原始 605.518 cycles/call		扣除记录代码开销估算后的参考值 Σ 296,279.000 cycles
≈ 179.556 µs（1.650062 cycles/ns）
原始 observed Σ 21,063,376 cycles（≈ 12,765.203 µs）− 记录代码开销估算 20,767,097.000 cycles
记录代码开销估算 / 原始 observed 98.593%；原始逐核 最小 656,662；最大 659,424；原始 514.243 cycles/call		扣除记录代码开销估算后的参考值 Σ 5,667,328.000 cycles
≈ 3,435.304 µs（1.649731 cycles/ns）
原始 observed Σ 53,342,629 cycles（≈ 32,334.138 µs）− 记录代码开销估算 47,675,301.000 cycles
记录代码开销估算 / 原始 observed 89.376%；原始逐核 最小 790,413；最大 886,262；原始 651.155 cycles/call
阶段参考关系（raw 同格保留）扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 96.613%
参考 Non-scalar / 参考 Phase total 3.387%
参考 Phase total / 原始 whole total 20.469%；原始 observed 42.825%
参考 Phase scalar / 原始 whole scalar 22.393%；原始 observed 38.698%
whole scalar−shadow scalar：Σ 245,760 cycles；逐核 最小 2,560；最大 2,560		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 99.467%
参考 Non-scalar / 参考 Phase total 0.533%
参考 Phase total / 原始 whole total 21.551%；原始 observed 45.141%
参考 Phase scalar / 原始 whole scalar 21.808%；原始 observed 37.613%
whole scalar−shadow scalar：Σ 81,920 cycles；逐核 最小 2,560；最大 2,560		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 95.297%
参考 Non-scalar / 参考 Phase total 4.703%
参考 Phase total / 原始 whole total 20.006%；原始 observed 41.834%
参考 Phase scalar / 原始 whole scalar 22.687%；原始 observed 39.241%
whole scalar−shadow scalar：Σ 163,840 cycles；逐核 最小 2,560；最大 2,560
Request（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 23,254,487.000
原始 observed 30,124,105 − 记录代码开销估算 6,869,618.000
参考值 / 原始整窗 24.748%；原始 observed / 原始整窗 32.059%
记录代码开销估算 / 原始 observed 22.804%
原始逐核 最小 311,332；最大 316,405；原始整窗 93,963,590
原始 capture gap +0；加 gap 后 30,124,105（32.059%）		扣除记录代码开销估算后的参考值 7,789,174.000
原始 observed 10,011,326 − 记录代码开销估算 2,222,152.000
参考值 / 原始整窗 24.500%；原始 observed / 原始整窗 31.489%
记录代码开销估算 / 原始 observed 22.196%
原始逐核 最小 311,744；最大 314,741；原始整窗 31,793,031
原始 capture gap +0；加 gap 后 10,011,326（31.489%）		扣除记录代码开销估算后的参考值 15,465,313.000
原始 observed 20,112,779 − 记录代码开销估算 4,647,466.000
参考值 / 原始整窗 24.876%；原始 observed / 原始整窗 32.351%
记录代码开销估算 / 原始 observed 23.107%
原始逐核 最小 311,332；最大 316,405；原始整窗 62,170,559
原始 capture gap +0；加 gap 后 20,112,779（32.351%）
Miss（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 519,859.000
原始 observed 754,725 − 记录代码开销估算 234,866.000
参考值 / 原始整窗 16.164%；原始 observed / 原始整窗 23.466%
记录代码开销估算 / 原始 observed 31.119%
原始逐核 最小 42；最大 12,093；原始整窗 3,216,252
原始 capture gap +0；加 gap 后 754,725（23.466%）		扣除记录代码开销估算后的参考值 -746.000
原始 observed 1,743 − 记录代码开销估算 2,489.000
参考值 / 原始整窗 -0.776%；原始 observed / 原始整窗 1.814%
记录代码开销估算 / 原始 observed 142.800%
原始逐核 最小 42；最大 66；原始整窗 96,089
原始 capture gap +0；加 gap 后 1,743（1.814%）		扣除记录代码开销估算后的参考值 520,605.000
原始 observed 752,982 − 记录代码开销估算 232,377.000
参考值 / 原始整窗 16.685%；原始 observed / 原始整窗 24.133%
记录代码开销估算 / 原始 observed 30.861%
原始逐核 最小 11,134；最大 12,093；原始整窗 3,120,163
原始 capture gap +0；加 gap 后 752,982（24.133%）
SYS 边界诊断 / Begin-EndΣ 99,479,891 raw ticks
逐核 最小 963,084；最大 1,106,791；仅边界诊断
Begin / End：122,880 / 122,880
业务调用 122,880 次；排除 linked Kernel 调用 0 次		Σ 31,503,078 raw ticks
逐核 最小 963,084；最大 1,011,789；仅边界诊断
Begin / End：40,960 / 40,960
业务调用 40,960 次；排除 linked Kernel 调用 0 次		Σ 67,976,813 raw ticks
逐核 最小 1,025,730；最大 1,106,791；仅边界诊断
Begin / End：81,920 / 81,920
业务调用 81,920 次；排除 linked Kernel 调用 0 次
+
+ + +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
58cfc479a86f6a7df74cc5acda83712f2fe2de1ab6a5c4f5a9f0e90b7ebeb31b
+
Profile / extra cache
submit-pmu-materialize / + b28cf51da4d4f547
+
Profiled cache key
TestPagedAttentionUnroll / a5 / fully_distributed_within_core / submit-pmu-materialize
+
Source state
source-v2:8c6b6e3a412d2c7c7113c2d8f714a14eb3786f80:38b70354824dc570fc5e2430a17e9db9caab4225cd1da0eb6bfa2ca05f92f512:feab834c8c210c6c59adcc3dc341b911d3e066fe1ceaf755cc2a00c851e32dcc
+
Source-state stamp
/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/b28cf51da4d4f547/aicore/.git_commit
+
Compile definitions
PTO_FDWIC_SUBMIT_PMU=1
PTO_FDWIC_SUBMIT_PMU_PHASE_ID=3
PTO_FDWIC_TRACE_ENABLED=0
+
+
+ + + +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
AICore final2,609,08842c60b91163de9834f48b2c322d3867f9dcdb9761f93bcf0305d0a7f42b62dd8204,88015ccd518a3f05f2d20fdd3b8937e2d75ea25debd580ea977f87c7c4436ab6382/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/aicore-extra/b28cf51da4d4f547/aicore_kernel.o
AIC combined1,941,19268e90b8ab44e82ae959ff95cdd0e8121412a2cd6ca4726ba35a97a7dec91ccf095,3846af3ffd275ea906bbd99bb4d72dac1cbf807bf61675395e06f1b3b55e3ab4430/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/b28cf51da4d4f547/aicore/aicore_aic_combined.o
AIV combined2,202,2886bac208675c83ffe311bbe6807639069115874a30b7c5abacb0b479ae0f7994e109,39207787d620537178d6ee8138e94242a525708a540779acd9e4825e321cb27cbf9/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/b28cf51da4d4f547/aicore/aicore_aiv_combined.o
Host runtime11,776,1440369d0602054bf612f86b58c294503e00486bd505bff9ee7a455994f043147cc454,6325ac381ed10be5f5d492b8072c051a177131f4caec6d6f6bc9bf50880da4bb0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so
+
+ +
+
原始全局 Submit 时间范围（墙钟） + 5,970.339 µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + 128096 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-2569e559a56fae5b0a5f340d6381967bfdbaa8dabf740c231608f2b3d74c17ddcea
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + 90.000 ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×90 ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
+
+

ALL · 96 核

+
+
Submit PMU total/core
+
均值 8,959,601.8；最小 7,815,809；最大 9,635,490 cycles
+ 等效时间 均值 5,430.575；最小 4,737.302；最大 5,840.243 µs + （按 1.649844 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,912,363.6；最小 7,664,970；最大 8,496,339 cycles
+ 等效时间 均值 4,795.825；最小 4,645.876；最大 5,149.783 µs； + 加权占比 88.312% +
+
非 Scalar-busy 残余/core
+
均值 1,047,238.2；最小 106,121；最大 1,604,361 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 634.750；最小 64.322；最大 972.432 µs +
+
SYS gate 边界诊断/core
+
均值 4,995,097.6；最小 4,304,333；最大 5,461,808 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,762.659；最小 5,346.691；最大 5,967.382 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 767,561.8；最小 492,716；最大 1,266,603 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 86.680% +
+
Primary I-cache request/core
最小 949,761；最大 1,061,563
+
Primary I-cache miss/core
最小 2,412；最大 50,488
+
加权 miss rate
3.423%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 217.080；最大 4,543.920 µs
+
+
+ +
+

AIC · 32 核

+
+
Submit PMU total/core
+
均值 8,057,662.6；最小 7,815,809；最大 8,622,183 cycles
+ 等效时间 均值 4,883.248；最小 4,736.676；最大 5,225.369 µs + （按 1.650062 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,920,304.3；最小 7,664,970；最大 8,496,339 cycles
+ 等效时间 均值 4,800.004；最小 4,645.262；最大 5,149.103 µs； + 加权占比 98.295% +
+
非 Scalar-busy 残余/core
+
均值 137,358.2；最小 106,121；最大 161,819 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 83.244；最小 64.313；最大 98.068 µs +
+
SYS gate 边界诊断/core
+
均值 4,376,585.2；最小 4,304,333；最大 4,515,878 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,434.621；最小 5,346.691；最大 5,782.287 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 1,058,035.8；最小 963,216；最大 1,266,603 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 80.532% +
+
Primary I-cache request/core
最小 968,361；最大 1,061,563
+
Primary I-cache miss/core
最小 2,412；最大 3,514
+
加权 miss rate
0.302%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 217.080；最大 316.260 µs
+
+
+ +
+

AIV · 64 核

+
+
Submit PMU total/core
+
均值 9,410,571.5；最小 9,199,428；最大 9,635,490 cycles
+ 等效时间 均值 5,704.307；最小 5,576.320；最大 5,840.643 µs + （按 1.649731 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,908,393.3；最小 7,711,681；最大 8,089,027 cycles
+ 等效时间 均值 4,793.747；最小 4,674.508；最大 4,903.240 µs； + 加权占比 84.037% +
+
非 Scalar-busy 残余/core
+
均值 1,502,178.2；最小 1,439,953；最大 1,604,361 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 910.559；最小 872.841；最大 972.499 µs +
+
SYS gate 边界诊断/core
+
均值 5,304,353.8；最小 5,199,074；最大 5,461,808 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,926.679；最小 5,898.484；最大 5,967.382 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 622,324.8；最小 492,716；最大 716,393 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 89.500% +
+
Primary I-cache request/core
最小 949,761；最大 1,010,491
+
Primary I-cache miss/core
最小 47,560；最大 50,488
+
加权 miss rate
5.019%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 4,280.400；最大 4,543.920 µs
+
+
+
+

逐物理核分布

+

AICAIV

+
+
+
SYS gate boundary-diagnostic ticks/core
+ + + + 5,461,808 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_submit_elapsed_ticks=4397903physical=1 logical=1 role=AIC scalar_submit_elapsed_ticks=4328329physical=2 logical=2 role=AIC scalar_submit_elapsed_ticks=4397068physical=3 logical=3 role=AIC scalar_submit_elapsed_ticks=4305070physical=4 logical=4 role=AIC scalar_submit_elapsed_ticks=4431813physical=5 logical=5 role=AIC scalar_submit_elapsed_ticks=4317944physical=6 logical=6 role=AIC scalar_submit_elapsed_ticks=4348209physical=7 logical=7 role=AIC scalar_submit_elapsed_ticks=4343486physical=8 logical=8 role=AIC scalar_submit_elapsed_ticks=4341241physical=9 logical=9 role=AIC scalar_submit_elapsed_ticks=4305381physical=10 logical=10 role=AIC scalar_submit_elapsed_ticks=4367518physical=12 logical=11 role=AIC scalar_submit_elapsed_ticks=4394774physical=13 logical=12 role=AIC scalar_submit_elapsed_ticks=4310493physical=14 logical=13 role=AIC scalar_submit_elapsed_ticks=4385038physical=15 logical=14 role=AIC scalar_submit_elapsed_ticks=4410038physical=16 logical=15 role=AIC scalar_submit_elapsed_ticks=4323719physical=54 logical=16 role=AIC scalar_submit_elapsed_ticks=4367600physical=55 logical=17 role=AIC scalar_submit_elapsed_ticks=4391307physical=56 logical=18 role=AIC scalar_submit_elapsed_ticks=4369569physical=57 logical=19 role=AIC scalar_submit_elapsed_ticks=4429654physical=58 logical=20 role=AIC scalar_submit_elapsed_ticks=4343229physical=60 logical=21 role=AIC scalar_submit_elapsed_ticks=4515684physical=61 logical=22 role=AIC scalar_submit_elapsed_ticks=4367555physical=62 logical=23 role=AIC scalar_submit_elapsed_ticks=4406990physical=63 logical=24 role=AIC scalar_submit_elapsed_ticks=4443083physical=64 logical=25 role=AIC scalar_submit_elapsed_ticks=4362666physical=65 logical=26 role=AIC scalar_submit_elapsed_ticks=4466468physical=66 logical=27 role=AIC scalar_submit_elapsed_ticks=4312514physical=67 logical=28 role=AIC scalar_submit_elapsed_ticks=4304333physical=68 logical=29 role=AIC scalar_submit_elapsed_ticks=4339938physical=69 logical=30 role=AIC scalar_submit_elapsed_ticks=4515878physical=70 logical=31 role=AIC scalar_submit_elapsed_ticks=4406236physical=18 logical=32 role=AIV scalar_submit_elapsed_ticks=5326927physical=19 logical=33 role=AIV scalar_submit_elapsed_ticks=5361368physical=20 logical=34 role=AIV scalar_submit_elapsed_ticks=5314620physical=21 logical=35 role=AIV scalar_submit_elapsed_ticks=5247431physical=22 logical=36 role=AIV scalar_submit_elapsed_ticks=5339009physical=23 logical=37 role=AIV scalar_submit_elapsed_ticks=5349579physical=24 logical=38 role=AIV scalar_submit_elapsed_ticks=5345819physical=25 logical=39 role=AIV scalar_submit_elapsed_ticks=5275657physical=26 logical=40 role=AIV scalar_submit_elapsed_ticks=5317690physical=27 logical=41 role=AIV scalar_submit_elapsed_ticks=5305952physical=28 logical=42 role=AIV scalar_submit_elapsed_ticks=5247519physical=29 logical=43 role=AIV scalar_submit_elapsed_ticks=5270652physical=30 logical=44 role=AIV scalar_submit_elapsed_ticks=5281572physical=31 logical=45 role=AIV scalar_submit_elapsed_ticks=5295041physical=32 logical=46 role=AIV scalar_submit_elapsed_ticks=5297250physical=33 logical=47 role=AIV scalar_submit_elapsed_ticks=5292312physical=34 logical=48 role=AIV scalar_submit_elapsed_ticks=5376888physical=35 logical=49 role=AIV scalar_submit_elapsed_ticks=5225669physical=36 logical=50 role=AIV scalar_submit_elapsed_ticks=5271790physical=37 logical=51 role=AIV scalar_submit_elapsed_ticks=5320215physical=38 logical=52 role=AIV scalar_submit_elapsed_ticks=5346440physical=39 logical=53 role=AIV scalar_submit_elapsed_ticks=5309423physical=42 logical=54 role=AIV scalar_submit_elapsed_ticks=5199074physical=43 logical=55 role=AIV scalar_submit_elapsed_ticks=5307241physical=44 logical=56 role=AIV scalar_submit_elapsed_ticks=5309779physical=45 logical=57 role=AIV scalar_submit_elapsed_ticks=5226127physical=46 logical=58 role=AIV scalar_submit_elapsed_ticks=5247836physical=47 logical=59 role=AIV scalar_submit_elapsed_ticks=5309908physical=48 logical=60 role=AIV scalar_submit_elapsed_ticks=5324613physical=49 logical=61 role=AIV scalar_submit_elapsed_ticks=5281218physical=50 logical=62 role=AIV scalar_submit_elapsed_ticks=5310230physical=51 logical=63 role=AIV scalar_submit_elapsed_ticks=5309960physical=72 logical=64 role=AIV scalar_submit_elapsed_ticks=5275832physical=73 logical=65 role=AIV scalar_submit_elapsed_ticks=5258310physical=74 logical=66 role=AIV scalar_submit_elapsed_ticks=5290401physical=75 logical=67 role=AIV scalar_submit_elapsed_ticks=5284050physical=76 logical=68 role=AIV scalar_submit_elapsed_ticks=5269120physical=77 logical=69 role=AIV scalar_submit_elapsed_ticks=5329395physical=78 logical=70 role=AIV scalar_submit_elapsed_ticks=5332999physical=79 logical=71 role=AIV scalar_submit_elapsed_ticks=5339936physical=80 logical=72 role=AIV scalar_submit_elapsed_ticks=5291881physical=81 logical=73 role=AIV scalar_submit_elapsed_ticks=5227392physical=84 logical=74 role=AIV scalar_submit_elapsed_ticks=5332506physical=85 logical=75 role=AIV scalar_submit_elapsed_ticks=5326223physical=86 logical=76 role=AIV scalar_submit_elapsed_ticks=5282537physical=87 logical=77 role=AIV scalar_submit_elapsed_ticks=5362143physical=88 logical=78 role=AIV scalar_submit_elapsed_ticks=5359597physical=89 logical=79 role=AIV scalar_submit_elapsed_ticks=5245521physical=90 logical=80 role=AIV scalar_submit_elapsed_ticks=5361456physical=91 logical=81 role=AIV scalar_submit_elapsed_ticks=5270042physical=92 logical=82 role=AIV scalar_submit_elapsed_ticks=5302058physical=93 logical=83 role=AIV scalar_submit_elapsed_ticks=5277293physical=94 logical=84 role=AIV scalar_submit_elapsed_ticks=5326280physical=95 logical=85 role=AIV scalar_submit_elapsed_ticks=5362382physical=96 logical=86 role=AIV scalar_submit_elapsed_ticks=5262869physical=97 logical=87 role=AIV scalar_submit_elapsed_ticks=5310844physical=98 logical=88 role=AIV scalar_submit_elapsed_ticks=5317895physical=99 logical=89 role=AIV scalar_submit_elapsed_ticks=5461808physical=100 logical=90 role=AIV scalar_submit_elapsed_ticks=5340541physical=101 logical=91 role=AIV scalar_submit_elapsed_ticks=5339028physical=102 logical=92 role=AIV scalar_submit_elapsed_ticks=5298061physical=103 logical=93 role=AIV scalar_submit_elapsed_ticks=5341699physical=104 logical=94 role=AIV scalar_submit_elapsed_ticks=5268050physical=105 logical=95 role=AIV scalar_submit_elapsed_ticks=5285682 + +
+ +
+
PMU total cycles/core
+ + + + 9,635,490 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC total_cycles=8143472physical=1 logical=1 role=AIC total_cycles=7856708physical=2 logical=2 role=AIC total_cycles=8076832physical=3 logical=3 role=AIC total_cycles=7862531physical=4 logical=4 role=AIC total_cycles=8332162physical=5 logical=5 role=AIC total_cycles=8034638physical=6 logical=6 role=AIC total_cycles=7995815physical=7 logical=7 role=AIC total_cycles=7891343physical=8 logical=8 role=AIC total_cycles=7913047physical=9 logical=9 role=AIC total_cycles=7815809physical=10 logical=10 role=AIC total_cycles=7916218physical=12 logical=11 role=AIC total_cycles=7956880physical=13 logical=12 role=AIC total_cycles=7918944physical=14 logical=13 role=AIC total_cycles=8232537physical=15 logical=14 role=AIC total_cycles=7990420physical=16 logical=15 role=AIC total_cycles=7863395physical=54 logical=16 role=AIC total_cycles=8030764physical=55 logical=17 role=AIC total_cycles=8036587physical=56 logical=18 role=AIC total_cycles=7999999physical=57 logical=19 role=AIC total_cycles=8300946physical=58 logical=20 role=AIC total_cycles=8020078physical=60 logical=21 role=AIC total_cycles=8622183physical=61 logical=22 role=AIC total_cycles=8004074physical=62 logical=23 role=AIC total_cycles=7976273physical=63 logical=24 role=AIC total_cycles=8287214physical=64 logical=25 role=AIC total_cycles=7976179physical=65 logical=26 role=AIC total_cycles=8347044physical=66 logical=27 role=AIC total_cycles=7915630physical=67 logical=28 role=AIC total_cycles=7868468physical=68 logical=29 role=AIC total_cycles=7871671physical=69 logical=30 role=AIC total_cycles=8594976physical=70 logical=31 role=AIC total_cycles=8192365physical=18 logical=32 role=AIV total_cycles=9410352physical=19 logical=33 role=AIV total_cycles=9475237physical=20 logical=34 role=AIV total_cycles=9469476physical=21 logical=35 role=AIV total_cycles=9316381physical=22 logical=36 role=AIV total_cycles=9496923physical=23 logical=37 role=AIV total_cycles=9449712physical=24 logical=38 role=AIV total_cycles=9481400physical=25 logical=39 role=AIV total_cycles=9379118physical=26 logical=40 role=AIV total_cycles=9421605physical=27 logical=41 role=AIV total_cycles=9376522physical=28 logical=42 role=AIV total_cycles=9301563physical=29 logical=43 role=AIV total_cycles=9374707physical=30 logical=44 role=AIV total_cycles=9307658physical=31 logical=45 role=AIV total_cycles=9382474physical=32 logical=46 role=AIV total_cycles=9428166physical=33 logical=47 role=AIV total_cycles=9424514physical=34 logical=48 role=AIV total_cycles=9519885physical=35 logical=49 role=AIV total_cycles=9251282physical=36 logical=50 role=AIV total_cycles=9390707physical=37 logical=51 role=AIV total_cycles=9491944physical=38 logical=52 role=AIV total_cycles=9439787physical=39 logical=53 role=AIV total_cycles=9453734physical=42 logical=54 role=AIV total_cycles=9199428physical=43 logical=55 role=AIV total_cycles=9391791physical=44 logical=56 role=AIV total_cycles=9454928physical=45 logical=57 role=AIV total_cycles=9229150physical=46 logical=58 role=AIV total_cycles=9369386physical=47 logical=59 role=AIV total_cycles=9441039physical=48 logical=60 role=AIV total_cycles=9494185physical=49 logical=61 role=AIV total_cycles=9380383physical=50 logical=62 role=AIV total_cycles=9396844physical=51 logical=63 role=AIV total_cycles=9336217physical=72 logical=64 role=AIV total_cycles=9362243physical=73 logical=65 role=AIV total_cycles=9350106physical=74 logical=66 role=AIV total_cycles=9406477physical=75 logical=67 role=AIV total_cycles=9346809physical=76 logical=68 role=AIV total_cycles=9290701physical=77 logical=69 role=AIV total_cycles=9490437physical=78 logical=70 role=AIV total_cycles=9480110physical=79 logical=71 role=AIV total_cycles=9507202physical=80 logical=72 role=AIV total_cycles=9424912physical=81 logical=73 role=AIV total_cycles=9291373physical=84 logical=74 role=AIV total_cycles=9487516physical=85 logical=75 role=AIV total_cycles=9377650physical=86 logical=76 role=AIV total_cycles=9385558physical=87 logical=77 role=AIV total_cycles=9546371physical=88 logical=78 role=AIV total_cycles=9549247physical=89 logical=79 role=AIV total_cycles=9319630physical=90 logical=80 role=AIV total_cycles=9607794physical=91 logical=81 role=AIV total_cycles=9301498physical=92 logical=82 role=AIV total_cycles=9389848physical=93 logical=83 role=AIV total_cycles=9367275physical=94 logical=84 role=AIV total_cycles=9392293physical=95 logical=85 role=AIV total_cycles=9564882physical=96 logical=86 role=AIV total_cycles=9306272physical=97 logical=87 role=AIV total_cycles=9407212physical=98 logical=88 role=AIV total_cycles=9439512physical=99 logical=89 role=AIV total_cycles=9635490physical=100 logical=90 role=AIV total_cycles=9474884physical=101 logical=91 role=AIV total_cycles=9477360physical=102 logical=92 role=AIV total_cycles=9398265physical=103 logical=93 role=AIV total_cycles=9437210physical=104 logical=94 role=AIV total_cycles=9343006physical=105 logical=95 role=AIV total_cycles=9380933 + +
+ +
+
Scalar busy cycles/core
+ + + + 8,496,339 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_busy=8007720physical=1 logical=1 role=AIC scalar_busy=7730010physical=2 logical=2 role=AIC scalar_busy=7938335physical=3 logical=3 role=AIC scalar_busy=7735504physical=4 logical=4 role=AIC scalar_busy=8183769physical=5 logical=5 role=AIC scalar_busy=7884059physical=6 logical=6 role=AIC scalar_busy=7867823physical=7 logical=7 role=AIC scalar_busy=7754088physical=8 logical=8 role=AIC scalar_busy=7784770physical=9 logical=9 role=AIC scalar_busy=7664970physical=10 logical=10 role=AIC scalar_busy=7754399physical=12 logical=11 role=AIC scalar_busy=7824639physical=13 logical=12 role=AIC scalar_busy=7812823physical=14 logical=13 role=AIC scalar_busy=8111972physical=15 logical=14 role=AIC scalar_busy=7851967physical=16 logical=15 role=AIC scalar_busy=7701843physical=54 logical=16 role=AIC scalar_busy=7914277physical=55 logical=17 role=AIC scalar_busy=7883637physical=56 logical=18 role=AIC scalar_busy=7870890physical=57 logical=19 role=AIC scalar_busy=8143332physical=58 logical=20 role=AIC scalar_busy=7887589physical=60 logical=21 role=AIC scalar_busy=8496339physical=61 logical=22 role=AIC scalar_busy=7856728physical=62 logical=23 role=AIC scalar_busy=7852583physical=63 logical=24 role=AIC scalar_busy=8130361physical=64 logical=25 role=AIC scalar_busy=7840479physical=65 logical=26 role=AIC scalar_busy=8205909physical=66 logical=27 role=AIC scalar_busy=7792341physical=67 logical=28 role=AIC scalar_busy=7711602physical=68 logical=29 role=AIC scalar_busy=7735817physical=69 logical=30 role=AIC scalar_busy=8446613physical=70 logical=31 role=AIC scalar_busy=8072551physical=18 logical=32 role=AIV scalar_busy=7844302physical=19 logical=33 role=AIV scalar_busy=7934234physical=20 logical=34 role=AIV scalar_busy=7924969physical=21 logical=35 role=AIV scalar_busy=7765698physical=22 logical=36 role=AIV scalar_busy=7966482physical=23 logical=37 role=AIV scalar_busy=7934375physical=24 logical=38 role=AIV scalar_busy=7950094physical=25 logical=39 role=AIV scalar_busy=7868597physical=26 logical=40 role=AIV scalar_busy=7943392physical=27 logical=41 role=AIV scalar_busy=7832236physical=28 logical=42 role=AIV scalar_busy=7814353physical=29 logical=43 role=AIV scalar_busy=7823029physical=30 logical=44 role=AIV scalar_busy=7800808physical=31 logical=45 role=AIV scalar_busy=7900970physical=32 logical=46 role=AIV scalar_busy=7963924physical=33 logical=47 role=AIV scalar_busy=7933536physical=34 logical=48 role=AIV scalar_busy=8038745physical=35 logical=49 role=AIV scalar_busy=7742513physical=36 logical=50 role=AIV scalar_busy=7909496physical=37 logical=51 role=AIV scalar_busy=8020586physical=38 logical=52 role=AIV scalar_busy=7966291physical=39 logical=53 role=AIV scalar_busy=7935128physical=42 logical=54 role=AIV scalar_busy=7711681physical=43 logical=55 role=AIV scalar_busy=7925057physical=44 logical=56 role=AIV scalar_busy=8000810physical=45 logical=57 role=AIV scalar_busy=7744676physical=46 logical=58 role=AIV scalar_busy=7844416physical=47 logical=59 role=AIV scalar_busy=7972611physical=48 logical=60 role=AIV scalar_busy=8054232physical=49 logical=61 role=AIV scalar_busy=7932810physical=50 logical=62 role=AIV scalar_busy=7899918physical=51 logical=63 role=AIV scalar_busy=7852004physical=72 logical=64 role=AIV scalar_busy=7880451physical=73 logical=65 role=AIV scalar_busy=7853799physical=74 logical=66 role=AIV scalar_busy=7936044physical=75 logical=67 role=AIV scalar_busy=7869045physical=76 logical=68 role=AIV scalar_busy=7792745physical=77 logical=69 role=AIV scalar_busy=7962894physical=78 logical=70 role=AIV scalar_busy=8018861physical=79 logical=71 role=AIV scalar_busy=8011273physical=80 logical=72 role=AIV scalar_busy=7930851physical=81 logical=73 role=AIV scalar_busy=7764092physical=84 logical=74 role=AIV scalar_busy=8034297physical=85 logical=75 role=AIV scalar_busy=7927589physical=86 logical=76 role=AIV scalar_busy=7940870physical=87 logical=77 role=AIV scalar_busy=7997669physical=88 logical=78 role=AIV scalar_busy=8089027physical=89 logical=79 role=AIV scalar_busy=7807116physical=90 logical=80 role=AIV scalar_busy=8069888physical=91 logical=81 role=AIV scalar_busy=7773429physical=92 logical=82 role=AIV scalar_busy=7928966physical=93 logical=83 role=AIV scalar_busy=7865919physical=94 logical=84 role=AIV scalar_busy=7847463physical=95 logical=85 role=AIV scalar_busy=8080841physical=96 logical=86 role=AIV scalar_busy=7831577physical=97 logical=87 role=AIV scalar_busy=7913245physical=98 logical=88 role=AIV scalar_busy=7835151physical=99 logical=89 role=AIV scalar_busy=8087316physical=100 logical=90 role=AIV scalar_busy=7976013physical=101 logical=91 role=AIV scalar_busy=7955900physical=102 logical=92 role=AIV scalar_busy=7867629physical=103 logical=93 role=AIV scalar_busy=7881067physical=104 logical=94 role=AIV scalar_busy=7805269physical=105 logical=95 role=AIV scalar_busy=7848902 + +
+ +
+
Primary I-cache requests/core
+ + + + 1,061,563 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_requests=993603physical=1 logical=1 role=AIC icache_requests=968361physical=2 logical=2 role=AIC icache_requests=992241physical=3 logical=3 role=AIC icache_requests=977824physical=4 logical=4 role=AIC icache_requests=1016174physical=5 logical=5 role=AIC icache_requests=996946physical=6 logical=6 role=AIC icache_requests=989336physical=7 logical=7 role=AIC icache_requests=977582physical=8 logical=8 role=AIC icache_requests=978971physical=9 logical=9 role=AIC icache_requests=971052physical=10 logical=10 role=AIC icache_requests=969084physical=12 logical=11 role=AIC icache_requests=970850physical=13 logical=12 role=AIC icache_requests=988114physical=14 logical=13 role=AIC icache_requests=1016466physical=15 logical=14 role=AIC icache_requests=978360physical=16 logical=15 role=AIC icache_requests=975103physical=54 logical=16 role=AIC icache_requests=984019physical=55 logical=17 role=AIC icache_requests=992639physical=56 logical=18 role=AIC icache_requests=982757physical=57 logical=19 role=AIC icache_requests=1026715physical=58 logical=20 role=AIC icache_requests=1001298physical=60 logical=21 role=AIC icache_requests=1061563physical=61 logical=22 role=AIC icache_requests=984350physical=62 logical=23 role=AIC icache_requests=976138physical=63 logical=24 role=AIC icache_requests=1021477physical=64 logical=25 role=AIC icache_requests=980504physical=65 logical=26 role=AIC icache_requests=1019929physical=66 logical=27 role=AIC icache_requests=986623physical=67 logical=28 role=AIC icache_requests=986576physical=68 logical=29 role=AIC icache_requests=981894physical=69 logical=30 role=AIC icache_requests=1041175physical=70 logical=31 role=AIC icache_requests=1005307physical=18 logical=32 role=AIV icache_requests=961549physical=19 logical=33 role=AIV icache_requests=955882physical=20 logical=34 role=AIV icache_requests=982802physical=21 logical=35 role=AIV icache_requests=965343physical=22 logical=36 role=AIV icache_requests=969330physical=23 logical=37 role=AIV icache_requests=957509physical=24 logical=38 role=AIV icache_requests=973527physical=25 logical=39 role=AIV icache_requests=966688physical=26 logical=40 role=AIV icache_requests=968826physical=27 logical=41 role=AIV icache_requests=963137physical=28 logical=42 role=AIV icache_requests=961865physical=29 logical=43 role=AIV icache_requests=971005physical=30 logical=44 role=AIV icache_requests=950100physical=31 logical=45 role=AIV icache_requests=966744physical=32 logical=46 role=AIV icache_requests=984331physical=33 logical=47 role=AIV icache_requests=979496physical=34 logical=48 role=AIV icache_requests=968195physical=35 logical=49 role=AIV icache_requests=956261physical=36 logical=50 role=AIV icache_requests=972632physical=37 logical=51 role=AIV icache_requests=982641physical=38 logical=52 role=AIV icache_requests=956747physical=39 logical=53 role=AIV icache_requests=971528physical=42 logical=54 role=AIV icache_requests=950621physical=43 logical=55 role=AIV icache_requests=971648physical=44 logical=56 role=AIV icache_requests=982771physical=45 logical=57 role=AIV icache_requests=956437physical=46 logical=58 role=AIV icache_requests=984757physical=47 logical=59 role=AIV icache_requests=966105physical=48 logical=60 role=AIV icache_requests=982105physical=49 logical=61 role=AIV icache_requests=970618physical=50 logical=62 role=AIV icache_requests=971251physical=51 logical=63 role=AIV icache_requests=949999physical=72 logical=64 role=AIV icache_requests=972761physical=73 logical=65 role=AIV icache_requests=973977physical=74 logical=66 role=AIV icache_requests=965215physical=75 logical=67 role=AIV icache_requests=960990physical=76 logical=68 role=AIV icache_requests=957081physical=77 logical=69 role=AIV icache_requests=987560physical=78 logical=70 role=AIV icache_requests=982488physical=79 logical=71 role=AIV icache_requests=982215physical=80 logical=72 role=AIV icache_requests=990631physical=81 logical=73 role=AIV icache_requests=976932physical=84 logical=74 role=AIV icache_requests=982646physical=85 logical=75 role=AIV icache_requests=949761physical=86 logical=76 role=AIV icache_requests=965205physical=87 logical=77 role=AIV icache_requests=985173physical=88 logical=78 role=AIV icache_requests=996905physical=89 logical=79 role=AIV icache_requests=973894physical=90 logical=80 role=AIV icache_requests=1010491physical=91 logical=81 role=AIV icache_requests=962345physical=92 logical=82 role=AIV icache_requests=970931physical=93 logical=83 role=AIV icache_requests=974196physical=94 logical=84 role=AIV icache_requests=972013physical=95 logical=85 role=AIV icache_requests=987230physical=96 logical=86 role=AIV icache_requests=962276physical=97 logical=87 role=AIV icache_requests=970298physical=98 logical=88 role=AIV icache_requests=985239physical=99 logical=89 role=AIV icache_requests=955340physical=100 logical=90 role=AIV icache_requests=981364physical=101 logical=91 role=AIV icache_requests=978752physical=102 logical=92 role=AIV icache_requests=965253physical=103 logical=93 role=AIV icache_requests=963942physical=104 logical=94 role=AIV icache_requests=968236physical=105 logical=95 role=AIV icache_requests=986769 + +
+ +
+
Primary I-cache misses/core
+ + + + 50,488 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_misses=2950physical=1 logical=1 role=AIC icache_misses=2913physical=2 logical=2 role=AIC icache_misses=2983physical=3 logical=3 role=AIC icache_misses=2799physical=4 logical=4 role=AIC icache_misses=2777physical=5 logical=5 role=AIC icache_misses=3073physical=6 logical=6 role=AIC icache_misses=2965physical=7 logical=7 role=AIC icache_misses=3514physical=8 logical=8 role=AIC icache_misses=2864physical=9 logical=9 role=AIC icache_misses=3274physical=10 logical=10 role=AIC icache_misses=3173physical=12 logical=11 role=AIC icache_misses=2948physical=13 logical=12 role=AIC icache_misses=3064physical=14 logical=13 role=AIC icache_misses=2412physical=15 logical=14 role=AIC icache_misses=2944physical=16 logical=15 role=AIC icache_misses=3190physical=54 logical=16 role=AIC icache_misses=2787physical=55 logical=17 role=AIC icache_misses=3087physical=56 logical=18 role=AIC icache_misses=2800physical=57 logical=19 role=AIC icache_misses=3142physical=58 logical=20 role=AIC icache_misses=2882physical=60 logical=21 role=AIC icache_misses=2726physical=61 logical=22 role=AIC icache_misses=3278physical=62 logical=23 role=AIC icache_misses=3137physical=63 logical=24 role=AIC icache_misses=3184physical=64 logical=25 role=AIC icache_misses=2898physical=65 logical=26 role=AIC icache_misses=3145physical=66 logical=27 role=AIC icache_misses=2897physical=67 logical=28 role=AIC icache_misses=3285physical=68 logical=29 role=AIC icache_misses=3017physical=69 logical=30 role=AIC icache_misses=2788physical=70 logical=31 role=AIC icache_misses=3193physical=18 logical=32 role=AIV icache_misses=48114physical=19 logical=33 role=AIV icache_misses=47970physical=20 logical=34 role=AIV icache_misses=49521physical=21 logical=35 role=AIV icache_misses=48414physical=22 logical=36 role=AIV icache_misses=48444physical=23 logical=37 role=AIV icache_misses=47838physical=24 logical=38 role=AIV icache_misses=48018physical=25 logical=39 role=AIV icache_misses=47902physical=26 logical=40 role=AIV icache_misses=48352physical=27 logical=41 role=AIV icache_misses=47806physical=28 logical=42 role=AIV icache_misses=48189physical=29 logical=43 role=AIV icache_misses=48929physical=30 logical=44 role=AIV icache_misses=48262physical=31 logical=45 role=AIV icache_misses=48736physical=32 logical=46 role=AIV icache_misses=49744physical=33 logical=47 role=AIV icache_misses=49039physical=34 logical=48 role=AIV icache_misses=48423physical=35 logical=49 role=AIV icache_misses=47880physical=36 logical=50 role=AIV icache_misses=48768physical=37 logical=51 role=AIV icache_misses=49265physical=38 logical=52 role=AIV icache_misses=47681physical=39 logical=53 role=AIV icache_misses=48413physical=42 logical=54 role=AIV icache_misses=48203physical=43 logical=55 role=AIV icache_misses=48460physical=44 logical=56 role=AIV icache_misses=48993physical=45 logical=57 role=AIV icache_misses=48336physical=46 logical=58 role=AIV icache_misses=49875physical=47 logical=59 role=AIV icache_misses=48222physical=48 logical=60 role=AIV icache_misses=49106physical=49 logical=61 role=AIV icache_misses=48744physical=50 logical=62 role=AIV icache_misses=48606physical=51 logical=63 role=AIV icache_misses=48165physical=72 logical=64 role=AIV icache_misses=49171physical=73 logical=65 role=AIV icache_misses=49366physical=74 logical=66 role=AIV icache_misses=47912physical=75 logical=67 role=AIV icache_misses=48281physical=76 logical=68 role=AIV icache_misses=48800physical=77 logical=69 role=AIV icache_misses=49617physical=78 logical=70 role=AIV icache_misses=49289physical=79 logical=71 role=AIV icache_misses=49319physical=80 logical=72 role=AIV icache_misses=49966physical=81 logical=73 role=AIV icache_misses=49337physical=84 logical=74 role=AIV icache_misses=49555physical=85 logical=75 role=AIV icache_misses=47560physical=86 logical=76 role=AIV icache_misses=48061physical=87 logical=77 role=AIV icache_misses=49537physical=88 logical=78 role=AIV icache_misses=49522physical=89 logical=79 role=AIV icache_misses=49154physical=90 logical=80 role=AIV icache_misses=50488physical=91 logical=81 role=AIV icache_misses=48591physical=92 logical=82 role=AIV icache_misses=48486physical=93 logical=83 role=AIV icache_misses=49281physical=94 logical=84 role=AIV icache_misses=49214physical=95 logical=85 role=AIV icache_misses=49308physical=96 logical=86 role=AIV icache_misses=48851physical=97 logical=87 role=AIV icache_misses=48751physical=98 logical=88 role=AIV icache_misses=49825physical=99 logical=89 role=AIV icache_misses=47619physical=100 logical=90 role=AIV icache_misses=49215physical=101 logical=91 role=AIV icache_misses=49093physical=102 logical=92 role=AIV icache_misses=47774physical=103 logical=93 role=AIV icache_misses=48433physical=104 logical=94 role=AIV icache_misses=49005physical=105 logical=95 role=AIV icache_misses=49364 + +
+
+

逐核原始主计数

+

报告展示 primary request/miss；phase 模式的 I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和，只用于标记全窗 capture gap；scalar shadow 同样用于核验 phase 读清重建，并显示其相对 CNT2 primary 的 capture loss，不作为第二份性能数据展示。

+
+ + + + + + + +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate90ns量尺 µs
00AIC004,397,9035,361.1198,143,4728,007,720993,6032,9500.297%265.500
11AIC104,328,3295,390.5777,856,7087,730,010968,3612,9130.301%262.170
22AIC204,397,0685,402.6708,076,8327,938,335992,2412,9830.301%268.470
33AIC304,305,0705,346.6917,862,5317,735,504977,8242,7990.286%251.910
44AIC404,431,8135,494.7858,332,1628,183,7691,016,1742,7770.273%249.930
55AIC504,317,9445,354.0798,034,6387,884,059996,9463,0730.308%276.570
66AIC604,348,2095,373.7907,995,8157,867,823989,3362,9650.300%266.850
77AIC704,343,4865,399.4937,891,3437,754,088977,5823,5140.359%316.260
88AIC804,341,2415,403.5617,913,0477,784,770978,9712,8640.293%257.760
99AIC904,305,3815,381.6067,815,8097,664,970971,0523,2740.337%294.660
1010AIC1004,367,5185,358.0127,916,2187,754,399969,0843,1730.327%285.570
1211AIC1104,394,7745,387.7657,956,8807,824,639970,8502,9480.304%265.320
1312AIC1204,310,4935,362.6157,918,9447,812,823988,1143,0640.310%275.760
1413AIC1304,385,0385,480.6148,232,5378,111,9721,016,4662,4120.237%217.080
1514AIC1404,410,0385,387.9367,990,4207,851,967978,3602,9440.301%264.960
1615AIC1504,323,7195,370.1087,863,3957,701,843975,1033,1900.327%287.100
1832AIV015,326,9275,925.4429,410,3527,844,302961,54948,1145.004%4,330.260
1933AIV025,361,3685,910.3259,475,2377,934,234955,88247,9705.018%4,317.300
2034AIV115,314,6205,910.5779,469,4767,924,969982,80249,5215.039%4,456.890
2135AIV125,247,4315,924.0139,316,3817,765,698965,34348,4145.015%4,357.260
2236AIV215,339,0095,924.6389,496,9237,966,482969,33048,4444.998%4,359.960
2337AIV225,349,5795,946.5689,449,7127,934,375957,50947,8384.996%4,305.420
2438AIV315,345,8195,916.3899,481,4007,950,094973,52748,0184.932%4,321.620
2539AIV325,275,6575,958.9249,379,1187,868,597966,68847,9024.955%4,311.180
2640AIV415,317,6905,935.4439,421,6057,943,392968,82648,3524.991%4,351.680
2741AIV425,305,9525,908.5009,376,5227,832,236963,13747,8064.964%4,302.540
2842AIV515,247,5195,963.9129,301,5637,814,353961,86548,1895.010%4,337.010
2943AIV525,270,6525,899.9709,374,7077,823,029971,00548,9295.039%4,403.610
3044AIV615,281,5725,910.2399,307,6587,800,808950,10048,2625.080%4,343.580
3145AIV625,295,0415,959.7909,382,4747,900,970966,74448,7365.041%4,386.240
3246AIV715,297,2505,941.2759,428,1667,963,924984,33149,7445.054%4,476.960
3347AIV725,292,3125,936.6749,424,5147,933,536979,49649,0395.007%4,413.510
3448AIV815,376,8885,938.3469,519,8858,038,745968,19548,4235.001%4,358.070
3549AIV825,225,6695,930.3139,251,2827,742,513956,26147,8805.007%4,309.200
3650AIV915,271,7905,914.9269,390,7077,909,496972,63248,7685.014%4,389.120
3751AIV925,320,2155,926.1889,491,9448,020,586982,64149,2655.014%4,433.850
3852AIV1015,346,4405,944.5609,439,7877,966,291956,74747,6814.984%4,291.290
3953AIV1025,309,4235,901.0009,453,7347,935,128971,52848,4134.983%4,357.170
4254AIV1115,199,0745,902.7709,199,4287,711,681950,62148,2035.071%4,338.270
4355AIV1125,307,2415,919.0609,391,7917,925,057971,64848,4604.987%4,361.400
4456AIV1215,309,7795,903.7929,454,9288,000,810982,77148,9934.985%4,409.370
4557AIV1225,226,1275,919.7869,229,1507,744,676956,43748,3365.054%4,350.240
4658AIV1315,247,8365,909.2969,369,3867,844,416984,75749,8755.065%4,488.750
4759AIV1325,309,9085,945.1919,441,0397,972,611966,10548,2224.991%4,339.980
4860AIV1415,324,6135,927.0299,494,1858,054,232982,10549,1065.000%4,419.540
4961AIV1425,281,2185,909.1909,380,3837,932,810970,61848,7445.022%4,386.960
5062AIV1515,310,2305,916.7699,396,8447,899,918971,25148,6065.004%4,374.540
5163AIV1525,309,9605,930.1069,336,2177,852,004949,99948,1655.070%4,334.850
5416AIC1604,367,6005,379.8988,030,7647,914,277984,0192,7870.283%250.830
5517AIC1704,391,3075,423.4378,036,5877,883,637992,6393,0870.311%277.830
5618AIC1804,369,5695,375.8727,999,9997,870,890982,7572,8000.285%252.000
5719AIC1904,429,6545,670.8948,300,9468,143,3321,026,7153,1420.306%282.780
5820AIC2004,343,2295,401.5978,020,0787,887,5891,001,2982,8820.288%259.380
6021AIC2104,515,6845,782.2878,622,1838,496,3391,061,5632,7260.257%245.340
6122AIC2204,367,5555,392.2998,004,0747,856,728984,3503,2780.333%295.020
6223AIC2304,406,9905,388.0847,976,2737,852,583976,1383,1370.321%282.330
6324AIC2404,443,0835,545.4458,287,2148,130,3611,021,4773,1840.312%286.560
6425AIC2504,362,6665,358.2917,976,1797,840,479980,5042,8980.296%260.820
6526AIC2604,466,4685,505.2068,347,0448,205,9091,019,9293,1450.308%283.050
6627AIC2704,312,5145,355.0267,915,6307,792,341986,6232,8970.294%260.730
6728AIC2804,304,3335,347.5297,868,4687,711,602986,5763,2850.333%295.650
6829AIC2904,339,9385,349.8437,871,6717,735,817981,8943,0170.307%271.530
6930AIC3004,515,8785,723.4238,594,9768,446,6131,041,1752,7880.268%250.920
7031AIC3104,406,2365,653.3208,192,3658,072,5511,005,3073,1930.318%287.370
7264AIV1615,275,8325,948.3159,362,2437,880,451972,76149,1715.055%4,425.390
7365AIV1625,258,3105,941.5119,350,1067,853,799973,97749,3665.068%4,442.940
7466AIV1715,290,4015,921.8209,406,4777,936,044965,21547,9124.964%4,312.080
7567AIV1725,284,0505,938.7809,346,8097,869,045960,99048,2815.024%4,345.290
7668AIV1815,269,1205,955.2399,290,7017,792,745957,08148,8005.099%4,392.000
7769AIV1825,329,3955,924.4839,490,4377,962,894987,56049,6175.024%4,465.530
7870AIV1915,332,9995,920.7329,480,1108,018,861982,48849,2895.017%4,436.010
7971AIV1925,339,9365,932.3659,507,2028,011,273982,21549,3195.021%4,438.710
8072AIV2015,291,8815,939.7259,424,9127,930,851990,63149,9665.044%4,496.940
8173AIV2025,227,3925,907.9149,291,3737,764,092976,93249,3375.050%4,440.330
8474AIV2115,332,5065,919.9249,487,5168,034,297982,64649,5555.043%4,459.950
8575AIV2125,326,2235,898.4849,377,6507,927,589949,76147,5605.008%4,280.400
8676AIV2215,282,5375,908.2899,385,5587,940,870965,20548,0614.979%4,325.490
8777AIV2225,362,1435,906.4289,546,3717,997,669985,17349,5375.028%4,458.330
8878AIV2315,359,5975,908.2599,549,2478,089,027996,90549,5224.968%4,456.980
8979AIV2325,245,5215,924.2029,319,6307,807,116973,89449,1545.047%4,423.860
9080AIV2415,361,4565,899.1399,607,7948,069,8881,010,49150,4884.996%4,543.920
9181AIV2425,270,0425,962.9339,301,4987,773,429962,34548,5915.049%4,373.190
9282AIV2515,302,0585,915.3479,389,8487,928,966970,93148,4864.994%4,363.740
9383AIV2525,277,2935,952.2569,367,2757,865,919974,19649,2815.059%4,435.290
9484AIV2615,326,2805,914.7089,392,2937,847,463972,01349,2145.063%4,429.260
9585AIV2625,362,3825,916.1179,564,8828,080,841987,23049,3084.995%4,437.720
9686AIV2715,262,8695,916.0039,306,2727,831,577962,27648,8515.077%4,396.590
9787AIV2725,310,8445,924.7309,407,2127,913,245970,29848,7515.024%4,387.590
9888AIV2815,317,8955,951.2709,439,5127,835,151985,23949,8255.057%4,484.250
9989AIV2825,461,8085,954.5249,635,4908,087,316955,34047,6194.985%4,285.710
10090AIV2915,340,5415,913.2719,474,8847,976,013981,36449,2155.015%4,429.350
10191AIV2925,339,0285,967.3829,477,3607,955,900978,75249,0935.016%4,418.370
10292AIV3015,298,0615,919.8249,398,2657,867,629965,25347,7744.949%4,299.660
10393AIV3025,341,6995,942.0849,437,2107,881,067963,94248,4335.024%4,358.970
10494AIV3115,268,0505,937.4049,343,0067,805,269968,23649,0055.061%4,410.450
10595AIV3125,285,6825,912.9639,380,9337,848,902986,76949,3645.003%4,442.760
+
+ diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_prepare_map_20260723_044248_icache_report.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_prepare_map_20260723_044248_icache_report.html new file mode 100644 index 0000000000..f0febcd98d --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_prepare_map_20260723_044248_icache_report.html @@ -0,0 +1,362 @@ + + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 fdwic_submit_pmu_raw.json · schema fdwic-submit-pmu-v3 · 32 AIC + 64 AIV

+ +
+

prepare-map 阶段观察（phase_id=8）

+

边界 dist_submit_prepare_map_call_entry_to_return · 计数语义 running_read_clear_observed_bracket · + 时间语义 boundary_diagnostic_sys_cnt_between_observers · + expected_calls_per_core=1280

+ +

本报告使用 empty-bracket 估算进入 phase observed + 的局部记录代码开销。 + AIC/AIV 分别按“空区间每组记录开销 × 本阶段实际 begin/end 记录组数”计算，ALL + 再由两类核相加。参考值只从 phase observed 分子扣除该估算；原始 whole 分母保持不变， + 因为空区间没有测量 whole 窗口中的全部记录开销。所以下表的参考比例不是完整业务 + 阶段占完整调度窗口的比例，也不是精确校正。若参考值为负，表示记录代码开销估算已经 + 超过原始 observed；这只能说明该信号低于当前校准的分辨能力，不能截成零或解释成负的 + 业务事件。校准 raw：/home/q00473782/atomic/private/gpt/simpler/outputs/TestPagedAttentionUnroll_Case1_20260723_043533/fdwic_submit_pmu_raw.json · + SHA-256 56199f91545cc6a1fbf18b8bb2cfebae279cb5801516311a040d4facccf321ba。两份 provenance 已核验为同一场景、同一 Git revision 8c6b6e3a412d。

+ + +

本阶段若跨过 linked Kernel，会同时门控 SYS 时间与 PMU counter， + 因而 total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段。 + result-used return-ready atomic 依赖区间只从 SYS 边界累计值扣除，PMU counter + 仍含 atomic 指令事件；source-issue atomic + 保留在时间和计数口径内。没有 linked Kernel 的阶段不触发 PMU 停/开表； + 每次调用的分母仍是外层业务调用次数，不是 Begin/End 读数，也不是排除的 Kernel 调用数。

+ +

阶段原始 PMU 观测来自 running read-clear 的 total cycles，并按 AIC/AIV + 各自校准频率换算；提供 empty-bracket 时，页面主参考值再扣除局部记录代码开销估算。 + phase_elapsed_ticks 只是 SYS 边界闭合诊断，绝不按 1 GHz 当作阶段时间。 + Scalar busy 与 non-scalar residual（逐核先算 total−scalar）共同解释阶段 PMU total； + 未校准时只能展示 raw observed；提供 empty-bracket 后，记录代码开销估算只从 phase + observed 分子扣除，原始 whole 分母保持不变。当前数据无法得到完整业务阶段占完整调度 + 窗口的精确比例。 + linked vector/cube Kernel 从 SYS 与 PMU counter 一并门控排除；result-used return-ready atomic + 依赖区间只从 SYS 边界累计值扣除，I-cache/PMU counter 仍含其指令事件；source-issue atomic + 保留。request/miss 百分比的分母是同一 ELF、同一次采集的 Submit 整窗 primary。 + observed_plus_capture_gap = observed + (primary − shadow)； + 边界读数和插桩 bookkeeping 会进入 sample，因此“观测值”和“加全窗 capture gap”都不是原业务 + 事件数的数学上下界。不同 ELF 不能直接相减后冒充业务真值；提供 empty-bracket 时只按上方 + 明示的公式估算局部记录代码开销。request/miss 的逐核 min/max 是每核累计整个 phase + 的极值，不是逐调用极值。

+
+ + + + +
指标ALL96 核AIC32 核AIV64 核
Phase PMU total扣除记录代码开销估算后的参考值 Σ 17,839,579.000 cycles
≈ 10,812.888 µs（1.649844 cycles/ns）
原始 observed Σ 210,127,915 cycles（≈ 127,362.293 µs）− 记录代码开销估算 192,288,336.000 cycles
记录代码开销估算 / 原始 observed 91.510%；原始逐核 最小 2,093,859；最大 2,368,000；原始 1,710.025 cycles/call		扣除记录代码开销估算后的参考值 Σ 7,374,161.000 cycles
≈ 4,469.021 µs（1.650062 cycles/ns）
原始 observed Σ 68,200,964 cycles（≈ 41,332.364 µs）− 记录代码开销估算 60,826,803.000 cycles
记录代码开销估算 / 原始 observed 89.188%；原始逐核 最小 2,093,859；最大 2,201,061；原始 1,665.063 cycles/call		扣除记录代码开销估算后的参考值 Σ 10,465,418.000 cycles
≈ 6,343.712 µs（1.649731 cycles/ns）
原始 observed Σ 141,926,951 cycles（≈ 86,030.359 µs）− 记录代码开销估算 131,461,533.000 cycles
记录代码开销估算 / 原始 observed 92.626%；原始逐核 最小 2,177,815；最大 2,368,000；原始 1,732.507 cycles/call
Phase scalar busy扣除记录代码开销估算后的参考值 Σ 20,173,480.000 cycles
≈ 12,227.508 µs（1.649844 cycles/ns）
原始 observed Σ 144,019,418 cycles（≈ 87,292.749 µs）− 记录代码开销估算 123,845,938.000 cycles
记录代码开销估算 / 原始 observed 85.993%；原始逐核 最小 1,441,782；最大 1,658,894；原始 1,172.033 cycles/call		扣除记录代码开销估算后的参考值 Σ 7,265,232.000 cycles
≈ 4,403.005 µs（1.650062 cycles/ns）
原始 observed Σ 47,324,938 cycles（≈ 28,680.703 µs）− 记录代码开销估算 40,059,706.000 cycles
记录代码开销估算 / 原始 observed 84.648%；原始逐核 最小 1,441,782；最大 1,548,097；原始 1,155.394 cycles/call		扣除记录代码开销估算后的参考值 Σ 12,908,248.000 cycles
≈ 7,824.456 µs（1.649731 cycles/ns）
原始 observed Σ 96,694,480 cycles（≈ 58,612.271 µs）− 记录代码开销估算 83,786,232.000 cycles
记录代码开销估算 / 原始 observed 86.650%；原始逐核 最小 1,475,591；最大 1,658,894；原始 1,180.353 cycles/call
非 Scalar-busy 残余扣除记录代码开销估算后的参考值 Σ -2,333,901.000 cycles
≈ -1,414.619 µs（1.649844 cycles/ns）
原始 observed Σ 66,108,497 cycles（≈ 40,069.544 µs）− 记录代码开销估算 68,442,398.000 cycles
记录代码开销估算 / 原始 observed 103.530%；原始逐核 最小 650,360；最大 717,275；原始 537.992 cycles/call		扣除记录代码开销估算后的参考值 Σ 108,929.000 cycles
≈ 66.015 µs（1.650062 cycles/ns）
原始 observed Σ 20,876,026 cycles（≈ 12,651.662 µs）− 记录代码开销估算 20,767,097.000 cycles
记录代码开销估算 / 原始 observed 99.478%；原始逐核 最小 650,360；最大 654,764；原始 509.669 cycles/call		扣除记录代码开销估算后的参考值 Σ -2,442,830.000 cycles
≈ -1,480.744 µs（1.649731 cycles/ns）
原始 observed Σ 45,232,471 cycles（≈ 27,418.089 µs）− 记录代码开销估算 47,675,301.000 cycles
记录代码开销估算 / 原始 observed 105.401%；原始逐核 最小 696,446；最大 717,275；原始 552.154 cycles/call
阶段参考关系（raw 同格保留）扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 113.083%
参考 Non-scalar / 参考 Phase total -13.083%
参考 Phase total / 原始 whole total 2.274%；原始 observed 26.781%
参考 Phase scalar / 原始 whole scalar 2.751%；原始 observed 19.638%
whole scalar−shadow scalar：Σ 245,760 cycles；逐核 最小 2,560；最大 2,560		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 98.523%
参考 Non-scalar / 参考 Phase total 1.477%
参考 Phase total / 原始 whole total 3.040%；原始 observed 28.116%
参考 Phase scalar / 原始 whole scalar 3.026%；原始 observed 19.711%
whole scalar−shadow scalar：Σ 81,920 cycles；逐核 最小 2,560；最大 2,560		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 123.342%
参考 Non-scalar / 参考 Phase total -23.342%
参考 Phase total / 原始 whole total 1.931%；原始 observed 26.184%
参考 Phase scalar / 原始 whole scalar 2.617%；原始 observed 19.602%
whole scalar−shadow scalar：Σ 163,840 cycles；逐核 最小 2,560；最大 2,560
Request（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 4,119,393.000
原始 observed 10,989,011 − 记录代码开销估算 6,869,618.000
参考值 / 原始整窗 4.589%；原始 observed / 原始整窗 12.242%
记录代码开销估算 / 原始 observed 62.514%
原始逐核 最小 113,877；最大 116,638；原始整窗 89,766,806
原始 capture gap +0；加 gap 后 10,989,011（12.242%）		扣除记录代码开销估算后的参考值 1,439,404.000
原始 observed 3,661,556 − 记录代码开销估算 2,222,152.000
参考值 / 原始整窗 4.789%；原始 observed / 原始整窗 12.182%
记录代码开销估算 / 原始 observed 60.689%
原始逐核 最小 113,877；最大 116,376；原始整窗 30,056,415
原始 capture gap +0；加 gap 后 3,661,556（12.182%）		扣除记录代码开销估算后的参考值 2,679,989.000
原始 observed 7,327,455 − 记录代码开销估算 4,647,466.000
参考值 / 原始整窗 4.488%；原始 observed / 原始整窗 12.272%
记录代码开销估算 / 原始 observed 63.425%
原始逐核 最小 114,110；最大 116,638；原始整窗 59,710,391
原始 capture gap +0；加 gap 后 7,327,455（12.272%）
Miss（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 -206,904.000
原始 observed 27,962 − 记录代码开销估算 234,866.000
参考值 / 原始整窗 -12.171%；原始 observed / 原始整窗 1.645%
记录代码开销估算 / 原始 observed 839.947%
原始逐核 最小 20；最大 565；原始整窗 1,699,987
原始 capture gap +0；加 gap 后 27,962（1.645%）		扣除记录代码开销估算后的参考值 -1,499.000
原始 observed 990 − 记录代码开销估算 2,489.000
参考值 / 原始整窗 -2.727%；原始 observed / 原始整窗 1.801%
记录代码开销估算 / 原始 observed 251.414%
原始逐核 最小 20；最大 41；原始整窗 54,976
原始 capture gap +0；加 gap 后 990（1.801%）		扣除记录代码开销估算后的参考值 -205,405.000
原始 observed 26,972 − 记录代码开销估算 232,377.000
参考值 / 原始整窗 -12.487%；原始 observed / 原始整窗 1.640%
记录代码开销估算 / 原始 observed 861.549%
原始逐核 最小 320；最大 565；原始整窗 1,645,011
原始 capture gap +0；加 gap 后 26,972（1.640%）
SYS 边界诊断 / Begin-EndΣ 11,407,548 raw ticks
逐核 最小 83,391；最大 212,381；仅边界诊断
Begin / End：122,880 / 122,880
业务调用 122,880 次；排除 linked Kernel 调用 0 次		Σ 3,398,978 raw ticks
逐核 最小 83,391；最大 148,176；仅边界诊断
Begin / End：40,960 / 40,960
业务调用 40,960 次；排除 linked Kernel 调用 0 次		Σ 8,008,570 raw ticks
逐核 最小 101,383；最大 212,381；仅边界诊断
Begin / End：81,920 / 81,920
业务调用 81,920 次；排除 linked Kernel 调用 0 次
+
+ + +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
42a7a6995ba387d1f2b76973f2b5c63c964940939b5ecf82e767897e9556e2eb
+
Profile / extra cache
submit-pmu-prepare-map / + a0a6c6cc86dc3ae9
+
Profiled cache key
TestPagedAttentionUnroll / a5 / fully_distributed_within_core / submit-pmu-prepare-map
+
Source state
source-v2:8c6b6e3a412d2c7c7113c2d8f714a14eb3786f80:76cb7cb81232a9da7bdb8228fbc1c2525e2880336396da2b798df8140a88b0f9:56e3fca529a2c401d074f5ada7e6ca03e01619cec1bc6cf9e4ca7f2ddbf7b041
+
Source-state stamp
/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/a0a6c6cc86dc3ae9/aicore/.git_commit
+
Compile definitions
PTO_FDWIC_SUBMIT_PMU=1
PTO_FDWIC_SUBMIT_PMU_PHASE_ID=8
PTO_FDWIC_TRACE_ENABLED=0
+
+
+ + + +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
AICore final2,584,632e09e766ee77329f2fdbb531d541f6a48457d649d9613c7aa75c0cfd7a5164490199,760e72811e43cf9c419be97ea0eee1663d089b79310ce129a097fa879b455decce4/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/aicore-extra/a0a6c6cc86dc3ae9/aicore_kernel.o
AIC combined1,915,832e39ee6b14b38324c3571c66df46ad13a0034e7fb654686a8a4dc4e23d649806f92,800a9374b27bad315944a929db793410eb1ccd6a1e4a719d20a4a075425330e80e1/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/a0a6c6cc86dc3ae9/aicore/aicore_aic_combined.o
AIV combined2,176,952d7b1dc47edc75a3dacc299f58a5918c6c0fe42d474df64b12432cdd8976abe5a106,83217970be3b954f188970b3d0d46e0b0058adb9e1fd823e8a4f1b02db7da8eb01c/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/a0a6c6cc86dc3ae9/aicore/aicore_aiv_combined.o
Host runtime11,776,1440369d0602054bf612f86b58c294503e00486bd505bff9ee7a455994f043147cc454,6325ac381ed10be5f5d492b8072c051a177131f4caec6d6f6bc9bf50880da4bb0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so
+
+ +
+
原始全局 Submit 时间范围（墙钟） + 5,399.543 µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + 128096 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-25681d8012b8c1bb2b8622bcb1b27403b625f8e525716d47ef47711738a2a548b0d
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + 90.000 ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×90 ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
+
+

ALL · 96 核

+
+
Submit PMU total/core
+
均值 8,172,967.3；最小 7,458,291；最大 8,707,394 cycles
+ 等效时间 均值 4,953.782；最小 4,520.604；最大 5,277.707 µs + （按 1.649844 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,639,382.0；最小 7,369,818；最大 7,951,594 cycles
+ 等效时间 均值 4,630.366；最小 4,466.979；最大 4,819.604 µs； + 加权占比 93.471% +
+
非 Scalar-busy 残余/core
+
均值 533,585.2；最小 66,909；最大 875,602 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 323.416；最小 40.555；最大 530.718 µs +
+
SYS gate 边界诊断/core
+
均值 4,523,696.4；最小 4,128,761；最大 4,822,363 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,288.497；最小 5,126.623；最大 5,393.848 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 764,800.2；最小 536,873；最大 1,047,150 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 85.538% +
+
Primary I-cache request/core
最小 914,282；最大 960,995
+
Primary I-cache miss/core
最小 1,456；最大 27,209
+
加权 miss rate
1.894%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 131.040；最大 2,448.810 µs
+
+
+ +
+

AIC · 32 核

+
+
Submit PMU total/core
+
均值 7,580,304.6；最小 7,458,291；最大 7,701,290 cycles
+ 等效时间 均值 4,593.951；最小 4,520.007；最大 4,667.273 µs + （按 1.650062 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,502,837.8；最小 7,369,818；最大 7,634,381 cycles
+ 等效时间 均值 4,547.004；最小 4,466.389；最大 4,626.724 µs； + 加权占比 98.978% +
+
非 Scalar-busy 残余/core
+
均值 77,466.8；最小 66,909；最大 88,473 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 46.948；最小 40.549；最大 53.618 µs +
+
SYS gate 边界诊断/core
+
均值 4,184,623.9；最小 4,128,761；最大 4,236,987 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,155.293；最小 5,126.623；最大 5,205.327 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 970,669.6；最小 897,179；最大 1,047,150 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 81.171% +
+
Primary I-cache request/core
最小 929,569；最大 953,024
+
Primary I-cache miss/core
最小 1,456；最大 1,988
+
加权 miss rate
0.183%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 131.040；最大 178.920 µs
+
+
+ +
+

AIV · 64 核

+
+
Submit PMU total/core
+
均值 8,469,298.6；最小 8,277,950；最大 8,707,394 cycles
+ 等效时间 均值 5,133.745；最小 5,017.757；最大 5,278.069 µs + （按 1.649731 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,707,654.2；最小 7,531,316；最大 7,951,594 cycles
+ 等效时间 均值 4,672.067；最小 4,565.178；最大 4,819.934 µs； + 加权占比 91.007% +
+
非 Scalar-busy 残余/core
+
均值 761,644.4；最小 678,061；最大 875,602 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 461.678；最小 411.013；最大 530.754 µs +
+
SYS gate 边界诊断/core
+
均值 4,693,232.7；最小 4,606,760；最大 4,822,363 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,355.098；最小 5,326.841；最大 5,393.848 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 661,865.5；最小 536,873；最大 766,944 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 87.640% +
+
Primary I-cache request/core
最小 914,282；最大 960,995
+
Primary I-cache miss/core
最小 24,478；最大 27,209
+
加权 miss rate
2.755%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 2,203.020；最大 2,448.810 µs
+
+
+
+

逐物理核分布

+

AICAIV

+
+
+
SYS gate boundary-diagnostic ticks/core
+ + + + 4,822,363 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_submit_elapsed_ticks=4206596physical=1 logical=1 role=AIC scalar_submit_elapsed_ticks=4179117physical=2 logical=2 role=AIC scalar_submit_elapsed_ticks=4169285physical=3 logical=3 role=AIC scalar_submit_elapsed_ticks=4159674physical=4 logical=4 role=AIC scalar_submit_elapsed_ticks=4187452physical=5 logical=5 role=AIC scalar_submit_elapsed_ticks=4137140physical=6 logical=6 role=AIC scalar_submit_elapsed_ticks=4170688physical=7 logical=7 role=AIC scalar_submit_elapsed_ticks=4176831physical=8 logical=8 role=AIC scalar_submit_elapsed_ticks=4185859physical=9 logical=9 role=AIC scalar_submit_elapsed_ticks=4160441physical=10 logical=10 role=AIC scalar_submit_elapsed_ticks=4209759physical=12 logical=11 role=AIC scalar_submit_elapsed_ticks=4206273physical=13 logical=12 role=AIC scalar_submit_elapsed_ticks=4199382physical=14 logical=13 role=AIC scalar_submit_elapsed_ticks=4153709physical=15 logical=14 role=AIC scalar_submit_elapsed_ticks=4231446physical=16 logical=15 role=AIC scalar_submit_elapsed_ticks=4197090physical=54 logical=16 role=AIC scalar_submit_elapsed_ticks=4193865physical=55 logical=17 role=AIC scalar_submit_elapsed_ticks=4199991physical=56 logical=18 role=AIC scalar_submit_elapsed_ticks=4193693physical=57 logical=19 role=AIC scalar_submit_elapsed_ticks=4130673physical=58 logical=20 role=AIC scalar_submit_elapsed_ticks=4128761physical=60 logical=21 role=AIC scalar_submit_elapsed_ticks=4212023physical=61 logical=22 role=AIC scalar_submit_elapsed_ticks=4174994physical=62 logical=23 role=AIC scalar_submit_elapsed_ticks=4236987physical=63 logical=24 role=AIC scalar_submit_elapsed_ticks=4161124physical=64 logical=25 role=AIC scalar_submit_elapsed_ticks=4178553physical=65 logical=26 role=AIC scalar_submit_elapsed_ticks=4212232physical=66 logical=27 role=AIC scalar_submit_elapsed_ticks=4186931physical=67 logical=28 role=AIC scalar_submit_elapsed_ticks=4163061physical=68 logical=29 role=AIC scalar_submit_elapsed_ticks=4230206physical=69 logical=30 role=AIC scalar_submit_elapsed_ticks=4193762physical=70 logical=31 role=AIC scalar_submit_elapsed_ticks=4180367physical=18 logical=32 role=AIV scalar_submit_elapsed_ticks=4686592physical=19 logical=33 role=AIV scalar_submit_elapsed_ticks=4763519physical=20 logical=34 role=AIV scalar_submit_elapsed_ticks=4713580physical=21 logical=35 role=AIV scalar_submit_elapsed_ticks=4745843physical=22 logical=36 role=AIV scalar_submit_elapsed_ticks=4723581physical=23 logical=37 role=AIV scalar_submit_elapsed_ticks=4744776physical=24 logical=38 role=AIV scalar_submit_elapsed_ticks=4751335physical=25 logical=39 role=AIV scalar_submit_elapsed_ticks=4756478physical=26 logical=40 role=AIV scalar_submit_elapsed_ticks=4673891physical=27 logical=41 role=AIV scalar_submit_elapsed_ticks=4674587physical=28 logical=42 role=AIV scalar_submit_elapsed_ticks=4712344physical=29 logical=43 role=AIV scalar_submit_elapsed_ticks=4666971physical=30 logical=44 role=AIV scalar_submit_elapsed_ticks=4685709physical=31 logical=45 role=AIV scalar_submit_elapsed_ticks=4678988physical=32 logical=46 role=AIV scalar_submit_elapsed_ticks=4651723physical=33 logical=47 role=AIV scalar_submit_elapsed_ticks=4682664physical=34 logical=48 role=AIV scalar_submit_elapsed_ticks=4697031physical=35 logical=49 role=AIV scalar_submit_elapsed_ticks=4671417physical=36 logical=50 role=AIV scalar_submit_elapsed_ticks=4763024physical=37 logical=51 role=AIV scalar_submit_elapsed_ticks=4727986physical=38 logical=52 role=AIV scalar_submit_elapsed_ticks=4726324physical=39 logical=53 role=AIV scalar_submit_elapsed_ticks=4659379physical=42 logical=54 role=AIV scalar_submit_elapsed_ticks=4685336physical=43 logical=55 role=AIV scalar_submit_elapsed_ticks=4641014physical=44 logical=56 role=AIV scalar_submit_elapsed_ticks=4743649physical=45 logical=57 role=AIV scalar_submit_elapsed_ticks=4735754physical=46 logical=58 role=AIV scalar_submit_elapsed_ticks=4663790physical=47 logical=59 role=AIV scalar_submit_elapsed_ticks=4792541physical=48 logical=60 role=AIV scalar_submit_elapsed_ticks=4692346physical=49 logical=61 role=AIV scalar_submit_elapsed_ticks=4697285physical=50 logical=62 role=AIV scalar_submit_elapsed_ticks=4651214physical=51 logical=63 role=AIV scalar_submit_elapsed_ticks=4672414physical=72 logical=64 role=AIV scalar_submit_elapsed_ticks=4653764physical=73 logical=65 role=AIV scalar_submit_elapsed_ticks=4627463physical=74 logical=66 role=AIV scalar_submit_elapsed_ticks=4688517physical=75 logical=67 role=AIV scalar_submit_elapsed_ticks=4621326physical=76 logical=68 role=AIV scalar_submit_elapsed_ticks=4680055physical=77 logical=69 role=AIV scalar_submit_elapsed_ticks=4687661physical=78 logical=70 role=AIV scalar_submit_elapsed_ticks=4645896physical=79 logical=71 role=AIV scalar_submit_elapsed_ticks=4610692physical=80 logical=72 role=AIV scalar_submit_elapsed_ticks=4661676physical=81 logical=73 role=AIV scalar_submit_elapsed_ticks=4693150physical=84 logical=74 role=AIV scalar_submit_elapsed_ticks=4666136physical=85 logical=75 role=AIV scalar_submit_elapsed_ticks=4720162physical=86 logical=76 role=AIV scalar_submit_elapsed_ticks=4700609physical=87 logical=77 role=AIV scalar_submit_elapsed_ticks=4647443physical=88 logical=78 role=AIV scalar_submit_elapsed_ticks=4697947physical=89 logical=79 role=AIV scalar_submit_elapsed_ticks=4705585physical=90 logical=80 role=AIV scalar_submit_elapsed_ticks=4636892physical=91 logical=81 role=AIV scalar_submit_elapsed_ticks=4642650physical=92 logical=82 role=AIV scalar_submit_elapsed_ticks=4705659physical=93 logical=83 role=AIV scalar_submit_elapsed_ticks=4704995physical=94 logical=84 role=AIV scalar_submit_elapsed_ticks=4644068physical=95 logical=85 role=AIV scalar_submit_elapsed_ticks=4753836physical=96 logical=86 role=AIV scalar_submit_elapsed_ticks=4606760physical=97 logical=87 role=AIV scalar_submit_elapsed_ticks=4700391physical=98 logical=88 role=AIV scalar_submit_elapsed_ticks=4620609physical=99 logical=89 role=AIV scalar_submit_elapsed_ticks=4822363physical=100 logical=90 role=AIV scalar_submit_elapsed_ticks=4722574physical=101 logical=91 role=AIV scalar_submit_elapsed_ticks=4770567physical=102 logical=92 role=AIV scalar_submit_elapsed_ticks=4671337physical=103 logical=93 role=AIV scalar_submit_elapsed_ticks=4695010physical=104 logical=94 role=AIV scalar_submit_elapsed_ticks=4697511physical=105 logical=95 role=AIV scalar_submit_elapsed_ticks=4730504 + +
+ +
+
PMU total cycles/core
+ + + + 8,707,394 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC total_cycles=7607964physical=1 logical=1 role=AIC total_cycles=7590125physical=2 logical=2 role=AIC total_cycles=7490479physical=3 logical=3 role=AIC total_cycles=7502004physical=4 logical=4 role=AIC total_cycles=7537953physical=5 logical=5 role=AIC total_cycles=7600621physical=6 logical=6 role=AIC total_cycles=7584079physical=7 logical=7 role=AIC total_cycles=7557068physical=8 logical=8 role=AIC total_cycles=7551586physical=9 logical=9 role=AIC total_cycles=7549380physical=10 logical=10 role=AIC total_cycles=7612816physical=12 logical=11 role=AIC total_cycles=7618778physical=13 logical=12 role=AIC total_cycles=7635114physical=14 logical=13 role=AIC total_cycles=7521869physical=15 logical=14 role=AIC total_cycles=7701290physical=16 logical=15 role=AIC total_cycles=7637459physical=54 logical=16 role=AIC total_cycles=7595193physical=55 logical=17 role=AIC total_cycles=7623492physical=56 logical=18 role=AIC total_cycles=7585565physical=57 logical=19 role=AIC total_cycles=7458291physical=58 logical=20 role=AIC total_cycles=7474946physical=60 logical=21 role=AIC total_cycles=7684678physical=61 logical=22 role=AIC total_cycles=7584989physical=62 logical=23 role=AIC total_cycles=7626087physical=63 logical=24 role=AIC total_cycles=7534382physical=64 logical=25 role=AIC total_cycles=7578700physical=65 logical=26 role=AIC total_cycles=7646380physical=66 logical=27 role=AIC total_cycles=7608001physical=67 logical=28 role=AIC total_cycles=7553612physical=68 logical=29 role=AIC total_cycles=7649230physical=69 logical=30 role=AIC total_cycles=7549095physical=70 logical=31 role=AIC total_cycles=7518521physical=18 logical=32 role=AIV total_cycles=8431460physical=19 logical=33 role=AIV total_cycles=8506704physical=20 logical=34 role=AIV total_cycles=8481278physical=21 logical=35 role=AIV total_cycles=8570439physical=22 logical=36 role=AIV total_cycles=8513862physical=23 logical=37 role=AIV total_cycles=8622762physical=24 logical=38 role=AIV total_cycles=8597865physical=25 logical=39 role=AIV total_cycles=8565861physical=26 logical=40 role=AIV total_cycles=8405195physical=27 logical=41 role=AIV total_cycles=8457005physical=28 logical=42 role=AIV total_cycles=8508257physical=29 logical=43 role=AIV total_cycles=8451471physical=30 logical=44 role=AIV total_cycles=8415229physical=31 logical=45 role=AIV total_cycles=8438371physical=32 logical=46 role=AIV total_cycles=8363711physical=33 logical=47 role=AIV total_cycles=8428653physical=34 logical=48 role=AIV total_cycles=8447977physical=35 logical=49 role=AIV total_cycles=8514996physical=36 logical=50 role=AIV total_cycles=8679151physical=37 logical=51 role=AIV total_cycles=8544797physical=38 logical=52 role=AIV total_cycles=8532603physical=39 logical=53 role=AIV total_cycles=8365342physical=42 logical=54 role=AIV total_cycles=8435052physical=43 logical=55 role=AIV total_cycles=8327509physical=44 logical=56 role=AIV total_cycles=8600966physical=45 logical=57 role=AIV total_cycles=8539891physical=46 logical=58 role=AIV total_cycles=8514923physical=47 logical=59 role=AIV total_cycles=8707394physical=48 logical=60 role=AIV total_cycles=8498035physical=49 logical=61 role=AIV total_cycles=8435406physical=50 logical=62 role=AIV total_cycles=8439079physical=51 logical=63 role=AIV total_cycles=8413860physical=72 logical=64 role=AIV total_cycles=8449840physical=73 logical=65 role=AIV total_cycles=8359619physical=74 logical=66 role=AIV total_cycles=8473982physical=75 logical=67 role=AIV total_cycles=8354449physical=76 logical=68 role=AIV total_cycles=8460087physical=77 logical=69 role=AIV total_cycles=8473221physical=78 logical=70 role=AIV total_cycles=8442216physical=79 logical=71 role=AIV total_cycles=8344460physical=80 logical=72 role=AIV total_cycles=8357935physical=81 logical=73 role=AIV total_cycles=8533454physical=84 logical=74 role=AIV total_cycles=8388689physical=85 logical=75 role=AIV total_cycles=8438918physical=86 logical=76 role=AIV total_cycles=8475080physical=87 logical=77 role=AIV total_cycles=8380030physical=88 logical=78 role=AIV total_cycles=8426520physical=89 logical=79 role=AIV total_cycles=8516656physical=90 logical=80 role=AIV total_cycles=8379220physical=91 logical=81 role=AIV total_cycles=8360738physical=92 logical=82 role=AIV total_cycles=8440777physical=93 logical=83 role=AIV total_cycles=8532666physical=94 logical=84 role=AIV total_cycles=8370832physical=95 logical=85 role=AIV total_cycles=8624999physical=96 logical=86 role=AIV total_cycles=8277950physical=97 logical=87 role=AIV total_cycles=8474966physical=98 logical=88 role=AIV total_cycles=8342412physical=99 logical=89 role=AIV total_cycles=8652219physical=100 logical=90 role=AIV total_cycles=8443098physical=101 logical=91 role=AIV total_cycles=8567311physical=102 logical=92 role=AIV total_cycles=8424607physical=103 logical=93 role=AIV total_cycles=8443698physical=104 logical=94 role=AIV total_cycles=8438197physical=105 logical=95 role=AIV total_cycles=8631160 + +
+ +
+
Scalar busy cycles/core
+ + + + 7,951,594 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_busy=7531226physical=1 logical=1 role=AIC scalar_busy=7516226physical=2 logical=2 role=AIC scalar_busy=7406211physical=3 logical=3 role=AIC scalar_busy=7425979physical=4 logical=4 role=AIC scalar_busy=7461366physical=5 logical=5 role=AIC scalar_busy=7526043physical=6 logical=6 role=AIC scalar_busy=7512202physical=7 logical=7 role=AIC scalar_busy=7484514physical=8 logical=8 role=AIC scalar_busy=7472231physical=9 logical=9 role=AIC scalar_busy=7471760physical=10 logical=10 role=AIC scalar_busy=7533540physical=12 logical=11 role=AIC scalar_busy=7545939physical=13 logical=12 role=AIC scalar_busy=7556194physical=14 logical=13 role=AIC scalar_busy=7436964physical=15 logical=14 role=AIC scalar_busy=7634381physical=16 logical=15 role=AIC scalar_busy=7556629physical=54 logical=16 role=AIC scalar_busy=7522533physical=55 logical=17 role=AIC scalar_busy=7545109physical=56 logical=18 role=AIC scalar_busy=7506397physical=57 logical=19 role=AIC scalar_busy=7369818physical=58 logical=20 role=AIC scalar_busy=7391769physical=60 logical=21 role=AIC scalar_busy=7610985physical=61 logical=22 role=AIC scalar_busy=7511678physical=62 logical=23 role=AIC scalar_busy=7544933physical=63 logical=24 role=AIC scalar_busy=7458458physical=64 logical=25 role=AIC scalar_busy=7502900physical=65 logical=26 role=AIC scalar_busy=7572820physical=66 logical=27 role=AIC scalar_busy=7526151physical=67 logical=28 role=AIC scalar_busy=7474936physical=68 logical=29 role=AIC scalar_busy=7574482physical=69 logical=30 role=AIC scalar_busy=7468961physical=70 logical=31 role=AIC scalar_busy=7437473physical=18 logical=32 role=AIV scalar_busy=7663321physical=19 logical=33 role=AIV scalar_busy=7708129physical=20 logical=34 role=AIV scalar_busy=7631580physical=21 logical=35 role=AIV scalar_busy=7694837physical=22 logical=36 role=AIV scalar_busy=7753174physical=23 logical=37 role=AIV scalar_busy=7825188physical=24 logical=38 role=AIV scalar_busy=7793011physical=25 logical=39 role=AIV scalar_busy=7755556physical=26 logical=40 role=AIV scalar_busy=7683056physical=27 logical=41 role=AIV scalar_busy=7680482physical=28 logical=42 role=AIV scalar_busy=7731786physical=29 logical=43 role=AIV scalar_busy=7667537physical=30 logical=44 role=AIV scalar_busy=7670779physical=31 logical=45 role=AIV scalar_busy=7693704physical=32 logical=46 role=AIV scalar_busy=7597279physical=33 logical=47 role=AIV scalar_busy=7666366physical=34 logical=48 role=AIV scalar_busy=7738584physical=35 logical=49 role=AIV scalar_busy=7752934physical=36 logical=50 role=AIV scalar_busy=7883994physical=37 logical=51 role=AIV scalar_busy=7783445physical=38 logical=52 role=AIV scalar_busy=7803321physical=39 logical=53 role=AIV scalar_busy=7650692physical=42 logical=54 role=AIV scalar_busy=7673236physical=43 logical=55 role=AIV scalar_busy=7601244physical=44 logical=56 role=AIV scalar_busy=7806406physical=45 logical=57 role=AIV scalar_busy=7736296physical=46 logical=58 role=AIV scalar_busy=7763919physical=47 logical=59 role=AIV scalar_busy=7951594physical=48 logical=60 role=AIV scalar_busy=7735666physical=49 logical=61 role=AIV scalar_busy=7679906physical=50 logical=62 role=AIV scalar_busy=7735511physical=51 logical=63 role=AIV scalar_busy=7715259physical=72 logical=64 role=AIV scalar_busy=7716013physical=73 logical=65 role=AIV scalar_busy=7613381physical=74 logical=66 role=AIV scalar_busy=7773723physical=75 logical=67 role=AIV scalar_busy=7655502physical=76 logical=68 role=AIV scalar_busy=7713565physical=77 logical=69 role=AIV scalar_busy=7701359physical=78 logical=70 role=AIV scalar_busy=7764155physical=79 logical=71 role=AIV scalar_busy=7637496physical=80 logical=72 role=AIV scalar_busy=7583740physical=81 logical=73 role=AIV scalar_busy=7720770physical=84 logical=74 role=AIV scalar_busy=7671262physical=85 logical=75 role=AIV scalar_busy=7706675physical=86 logical=76 role=AIV scalar_busy=7757980physical=87 logical=77 role=AIV scalar_busy=7670305physical=88 logical=78 role=AIV scalar_busy=7662830physical=89 logical=79 role=AIV scalar_busy=7738074physical=90 logical=80 role=AIV scalar_busy=7645012physical=91 logical=81 role=AIV scalar_busy=7627316physical=92 logical=82 role=AIV scalar_busy=7662447physical=93 logical=83 role=AIV scalar_busy=7734506physical=94 logical=84 role=AIV scalar_busy=7629637physical=95 logical=85 role=AIV scalar_busy=7880977physical=96 logical=86 role=AIV scalar_busy=7531316physical=97 logical=87 role=AIV scalar_busy=7712135physical=98 logical=88 role=AIV scalar_busy=7571493physical=99 logical=89 role=AIV scalar_busy=7909404physical=100 logical=90 role=AIV scalar_busy=7647916physical=101 logical=91 role=AIV scalar_busy=7745531physical=102 logical=92 role=AIV scalar_busy=7645648physical=103 logical=93 role=AIV scalar_busy=7646461physical=104 logical=94 role=AIV scalar_busy=7610053physical=105 logical=95 role=AIV scalar_busy=7775394 + +
+ +
+
Primary I-cache requests/core
+ + + + 960,995 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_requests=930878physical=1 logical=1 role=AIC icache_requests=942027physical=2 logical=2 role=AIC icache_requests=931821physical=3 logical=3 role=AIC icache_requests=932501physical=4 logical=4 role=AIC icache_requests=929569physical=5 logical=5 role=AIC icache_requests=948368physical=6 logical=6 role=AIC icache_requests=937904physical=7 logical=7 role=AIC icache_requests=937135physical=8 logical=8 role=AIC icache_requests=929894physical=9 logical=9 role=AIC icache_requests=944929physical=10 logical=10 role=AIC icache_requests=938457physical=12 logical=11 role=AIC icache_requests=935123physical=13 logical=12 role=AIC icache_requests=944666physical=14 logical=13 role=AIC icache_requests=943304physical=15 logical=14 role=AIC icache_requests=933831physical=16 logical=15 role=AIC icache_requests=943603physical=54 logical=16 role=AIC icache_requests=937937physical=55 logical=17 role=AIC icache_requests=944900physical=56 logical=18 role=AIC icache_requests=935317physical=57 logical=19 role=AIC icache_requests=932826physical=58 logical=20 role=AIC icache_requests=933864physical=60 logical=21 role=AIC icache_requests=953024physical=61 logical=22 role=AIC icache_requests=939738physical=62 logical=23 role=AIC icache_requests=936314physical=63 logical=24 role=AIC icache_requests=942020physical=64 logical=25 role=AIC icache_requests=936813physical=65 logical=26 role=AIC icache_requests=943051physical=66 logical=27 role=AIC icache_requests=946365physical=67 logical=28 role=AIC icache_requests=946030physical=68 logical=29 role=AIC icache_requests=945502physical=69 logical=30 role=AIC icache_requests=942287physical=70 logical=31 role=AIC icache_requests=936417physical=18 logical=32 role=AIV icache_requests=920740physical=19 logical=33 role=AIV icache_requests=920637physical=20 logical=34 role=AIV icache_requests=934444physical=21 logical=35 role=AIV icache_requests=948329physical=22 logical=36 role=AIV icache_requests=936049physical=23 logical=37 role=AIV icache_requests=940237physical=24 logical=38 role=AIV icache_requests=931709physical=25 logical=39 role=AIV icache_requests=930982physical=26 logical=40 role=AIV icache_requests=934572physical=27 logical=41 role=AIV icache_requests=934166physical=28 logical=42 role=AIV icache_requests=933267physical=29 logical=43 role=AIV icache_requests=939743physical=30 logical=44 role=AIV icache_requests=923984physical=31 logical=45 role=AIV icache_requests=926951physical=32 logical=46 role=AIV icache_requests=923780physical=33 logical=47 role=AIV icache_requests=926673physical=34 logical=48 role=AIV icache_requests=918067physical=35 logical=49 role=AIV icache_requests=947972physical=36 logical=50 role=AIV icache_requests=960995physical=37 logical=51 role=AIV icache_requests=935304physical=38 logical=52 role=AIV icache_requests=945143physical=39 logical=53 role=AIV icache_requests=924701physical=42 logical=54 role=AIV icache_requests=932460physical=43 logical=55 role=AIV icache_requests=918645physical=44 logical=56 role=AIV icache_requests=955306physical=45 logical=57 role=AIV icache_requests=936620physical=46 logical=58 role=AIV icache_requests=950846physical=47 logical=59 role=AIV icache_requests=955487physical=48 logical=60 role=AIV icache_requests=939665physical=49 logical=61 role=AIV icache_requests=929268physical=50 logical=62 role=AIV icache_requests=934600physical=51 logical=63 role=AIV icache_requests=927686physical=72 logical=64 role=AIV icache_requests=936905physical=73 logical=65 role=AIV icache_requests=928970physical=74 logical=66 role=AIV icache_requests=934369physical=75 logical=67 role=AIV icache_requests=923781physical=76 logical=68 role=AIV icache_requests=934218physical=77 logical=69 role=AIV icache_requests=939324physical=78 logical=70 role=AIV icache_requests=933621physical=79 logical=71 role=AIV icache_requests=923377physical=80 logical=72 role=AIV icache_requests=924495physical=81 logical=73 role=AIV icache_requests=943978physical=84 logical=74 role=AIV icache_requests=922988physical=85 logical=75 role=AIV icache_requests=914282physical=86 logical=76 role=AIV icache_requests=937929physical=87 logical=77 role=AIV icache_requests=928714physical=88 logical=78 role=AIV icache_requests=929067physical=89 logical=79 role=AIV icache_requests=934438physical=90 logical=80 role=AIV icache_requests=935613physical=91 logical=81 role=AIV icache_requests=929222physical=92 logical=82 role=AIV icache_requests=923814physical=93 logical=83 role=AIV icache_requests=941655physical=94 logical=84 role=AIV icache_requests=929175physical=95 logical=85 role=AIV icache_requests=950045physical=96 logical=86 role=AIV icache_requests=920927physical=97 logical=87 role=AIV icache_requests=934253physical=98 logical=88 role=AIV icache_requests=931866physical=99 logical=89 role=AIV icache_requests=922360physical=100 logical=90 role=AIV icache_requests=920617physical=101 logical=91 role=AIV icache_requests=937125physical=102 logical=92 role=AIV icache_requests=925840physical=103 logical=93 role=AIV icache_requests=926218physical=104 logical=94 role=AIV icache_requests=923473physical=105 logical=95 role=AIV icache_requests=948704 + +
+ +
+
Primary I-cache misses/core
+ + + + 27,209 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_misses=1752physical=1 logical=1 role=AIC icache_misses=1598physical=2 logical=2 role=AIC icache_misses=1775physical=3 logical=3 role=AIC icache_misses=1615physical=4 logical=4 role=AIC icache_misses=1732physical=5 logical=5 role=AIC icache_misses=1593physical=6 logical=6 role=AIC icache_misses=1587physical=7 logical=7 role=AIC icache_misses=1706physical=8 logical=8 role=AIC icache_misses=1854physical=9 logical=9 role=AIC icache_misses=1769physical=10 logical=10 role=AIC icache_misses=1677physical=12 logical=11 role=AIC icache_misses=1562physical=13 logical=12 role=AIC icache_misses=1740physical=14 logical=13 role=AIC icache_misses=1667physical=15 logical=14 role=AIC icache_misses=1456physical=16 logical=15 role=AIC icache_misses=1814physical=54 logical=16 role=AIC icache_misses=1657physical=55 logical=17 role=AIC icache_misses=1769physical=56 logical=18 role=AIC icache_misses=1828physical=57 logical=19 role=AIC icache_misses=1885physical=58 logical=20 role=AIC icache_misses=1988physical=60 logical=21 role=AIC icache_misses=1694physical=61 logical=22 role=AIC icache_misses=1586physical=62 logical=23 role=AIC icache_misses=1636physical=63 logical=24 role=AIC icache_misses=1729physical=64 logical=25 role=AIC icache_misses=1782physical=65 logical=26 role=AIC icache_misses=1690physical=66 logical=27 role=AIC icache_misses=1817physical=67 logical=28 role=AIC icache_misses=1750physical=68 logical=29 role=AIC icache_misses=1579physical=69 logical=30 role=AIC icache_misses=1816physical=70 logical=31 role=AIC icache_misses=1873physical=18 logical=32 role=AIV icache_misses=24667physical=19 logical=33 role=AIV icache_misses=24809physical=20 logical=34 role=AIV icache_misses=25580physical=21 logical=35 role=AIV icache_misses=26742physical=22 logical=36 role=AIV icache_misses=25729physical=23 logical=37 role=AIV icache_misses=26033physical=24 logical=38 role=AIV icache_misses=25554physical=25 logical=39 role=AIV icache_misses=25298physical=26 logical=40 role=AIV icache_misses=26209physical=27 logical=41 role=AIV icache_misses=26339physical=28 logical=42 role=AIV icache_misses=25978physical=29 logical=43 role=AIV icache_misses=26500physical=30 logical=44 role=AIV icache_misses=25000physical=31 logical=45 role=AIV icache_misses=25097physical=32 logical=46 role=AIV icache_misses=25168physical=33 logical=47 role=AIV icache_misses=25302physical=34 logical=48 role=AIV icache_misses=24478physical=35 logical=49 role=AIV icache_misses=26360physical=36 logical=50 role=AIV icache_misses=27209physical=37 logical=51 role=AIV icache_misses=25792physical=38 logical=52 role=AIV icache_misses=26640physical=39 logical=53 role=AIV icache_misses=25420physical=42 logical=54 role=AIV icache_misses=25640physical=43 logical=55 role=AIV icache_misses=25057physical=44 logical=56 role=AIV icache_misses=26926physical=45 logical=57 role=AIV icache_misses=25868physical=46 logical=58 role=AIV icache_misses=26777physical=47 logical=59 role=AIV icache_misses=26920physical=48 logical=60 role=AIV icache_misses=26123physical=49 logical=61 role=AIV icache_misses=25608physical=50 logical=62 role=AIV icache_misses=26028physical=51 logical=63 role=AIV icache_misses=25259physical=72 logical=64 role=AIV icache_misses=25787physical=73 logical=65 role=AIV icache_misses=25520physical=74 logical=66 role=AIV icache_misses=26184physical=75 logical=67 role=AIV icache_misses=25678physical=76 logical=68 role=AIV icache_misses=25621physical=77 logical=69 role=AIV icache_misses=26128physical=78 logical=70 role=AIV icache_misses=25684physical=79 logical=71 role=AIV icache_misses=25191physical=80 logical=72 role=AIV icache_misses=25224physical=81 logical=73 role=AIV icache_misses=26160physical=84 logical=74 role=AIV icache_misses=24983physical=85 logical=75 role=AIV icache_misses=24631physical=86 logical=76 role=AIV icache_misses=26358physical=87 logical=77 role=AIV icache_misses=25656physical=88 logical=78 role=AIV icache_misses=25220physical=89 logical=79 role=AIV icache_misses=25408physical=90 logical=80 role=AIV icache_misses=25786physical=91 logical=81 role=AIV icache_misses=25551physical=92 logical=82 role=AIV icache_misses=25260physical=93 logical=83 role=AIV icache_misses=26108physical=94 logical=84 role=AIV icache_misses=25297physical=95 logical=85 role=AIV icache_misses=26689physical=96 logical=86 role=AIV icache_misses=25446physical=97 logical=87 role=AIV icache_misses=25909physical=98 logical=88 role=AIV icache_misses=25957physical=99 logical=89 role=AIV icache_misses=25080physical=100 logical=90 role=AIV icache_misses=24871physical=101 logical=91 role=AIV icache_misses=25623physical=102 logical=92 role=AIV icache_misses=25199physical=103 logical=93 role=AIV icache_misses=25189physical=104 logical=94 role=AIV icache_misses=25133physical=105 logical=95 role=AIV icache_misses=26370 + +
+
+

逐核原始主计数

+

报告展示 primary request/miss；phase 模式的 I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和，只用于标记全窗 capture gap；scalar shadow 同样用于核验 phase 读清重建，并显示其相对 CNT2 primary 的 capture loss，不作为第二份性能数据展示。

+
+ + + + + + + +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate90ns量尺 µs
00AIC004,206,5965,126.6237,607,9647,531,226930,8781,7520.188%157.680
11AIC104,179,1175,156.3647,590,1257,516,226942,0271,5980.170%143.820
22AIC204,169,2855,174.5677,490,4797,406,211931,8211,7750.190%159.750
33AIC304,159,6745,138.1427,502,0047,425,979932,5011,6150.173%145.350
44AIC404,187,4525,133.4237,537,9537,461,366929,5691,7320.186%155.880
55AIC504,137,1405,169.5427,600,6217,526,043948,3681,5930.168%143.370
66AIC604,170,6885,163.6327,584,0797,512,202937,9041,5870.169%142.830
77AIC704,176,8315,146.6427,557,0687,484,514937,1351,7060.182%153.540
88AIC804,185,8595,155.0617,551,5867,472,231929,8941,8540.199%166.860
99AIC904,160,4415,171.2927,549,3807,471,760944,9291,7690.187%159.210
1010AIC1004,209,7595,143.9237,612,8167,533,540938,4571,6770.179%150.930
1211AIC1104,206,2735,162.2677,618,7787,545,939935,1231,5620.167%140.580
1312AIC1204,199,3825,164.9777,635,1147,556,194944,6661,7400.184%156.600
1413AIC1304,153,7095,140.1067,521,8697,436,964943,3041,6670.177%150.030
1514AIC1404,231,4465,128.6257,701,2907,634,381933,8311,4560.156%131.040
1615AIC1504,197,0905,203.9987,637,4597,556,629943,6031,8140.192%163.260
1832AIV014,686,5925,328.6218,431,4607,663,321920,74024,6672.679%2,220.030
1933AIV024,763,5195,375.5268,506,7047,708,129920,63724,8092.695%2,232.810
2034AIV114,713,5805,363.1938,481,2787,631,580934,44425,5802.737%2,302.200
2135AIV124,745,8435,366.9608,570,4397,694,837948,32926,7422.820%2,406.780
2236AIV214,723,5815,331.8128,513,8627,753,174936,04925,7292.749%2,315.610
2337AIV224,744,7765,343.3848,622,7627,825,188940,23726,0332.769%2,342.970
2438AIV314,751,3355,379.9798,597,8657,793,011931,70925,5542.743%2,299.860
2539AIV324,756,4785,360.8018,565,8617,755,556930,98225,2982.717%2,276.820
2640AIV414,673,8915,371.1848,405,1957,683,056934,57226,2092.804%2,358.810
2741AIV424,674,5875,351.2758,457,0057,680,482934,16626,3392.820%2,370.510
2842AIV514,712,3445,375.7658,508,2577,731,786933,26725,9782.784%2,338.020
2943AIV524,666,9715,343.9038,451,4717,667,537939,74326,5002.820%2,385.000
3044AIV614,685,7095,370.7898,415,2297,670,779923,98425,0002.706%2,250.000
3145AIV624,678,9885,334.5598,438,3717,693,704926,95125,0972.707%2,258.730
3246AIV714,651,7235,341.0618,363,7117,597,279923,78025,1682.724%2,265.120
3347AIV724,682,6645,332.1598,428,6537,666,366926,67325,3022.730%2,277.180
3448AIV814,697,0315,337.6048,447,9777,738,584918,06724,4782.666%2,203.020
3549AIV824,671,4175,382.9688,514,9967,752,934947,97226,3602.781%2,372.400
3650AIV914,763,0245,385.2118,679,1517,883,994960,99527,2092.831%2,448.810
3751AIV924,727,9865,348.2128,544,7977,783,445935,30425,7922.758%2,321.280
3852AIV1014,726,3245,343.6068,532,6037,803,321945,14326,6402.819%2,397.600
3953AIV1024,659,3795,344.7118,365,3427,650,692924,70125,4202.749%2,287.800
4254AIV1114,685,3365,339.9138,435,0527,673,236932,46025,6402.750%2,307.600
4355AIV1124,641,0145,374.9698,327,5097,601,244918,64525,0572.728%2,255.130
4456AIV1214,743,6495,390.5668,600,9667,806,406955,30626,9262.819%2,423.340
4557AIV1224,735,7545,347.9778,539,8917,736,296936,62025,8682.762%2,328.120
4658AIV1314,663,7905,336.9488,514,9237,763,919950,84626,7772.816%2,409.930
4759AIV1324,792,5415,348.3568,707,3947,951,594955,48726,9202.817%2,422.800
4860AIV1414,692,3465,375.6988,498,0357,735,666939,66526,1232.780%2,351.070
4961AIV1424,697,2855,385.7128,435,4067,679,906929,26825,6082.756%2,304.720
5062AIV1514,651,2145,336.7408,439,0797,735,511934,60026,0282.785%2,342.520
5163AIV1524,672,4145,376.1848,413,8607,715,259927,68625,2592.723%2,273.310
5416AIC1604,193,8655,172.9517,595,1937,522,533937,9371,6570.177%149.130
5517AIC1704,199,9915,140.9837,623,4927,545,109944,9001,7690.187%159.210
5618AIC1804,193,6935,141.3577,585,5657,506,397935,3171,8280.195%164.520
5719AIC1904,130,6735,137.7047,458,2917,369,818932,8261,8850.202%169.650
5820AIC2004,128,7615,175.9117,474,9467,391,769933,8641,9880.213%178.920
6021AIC2104,212,0235,171.7417,684,6787,610,985953,0241,6940.178%152.460
6122AIC2204,174,9945,129.1907,584,9897,511,678939,7381,5860.169%142.740
6223AIC2304,236,9875,156.0447,626,0877,544,933936,3141,6360.175%147.240
6324AIC2404,161,1245,131.4067,534,3827,458,458942,0201,7290.184%155.610
6425AIC2504,178,5535,186.8027,578,7007,502,900936,8131,7820.190%160.380
6526AIC2604,212,2325,160.3507,646,3807,572,820943,0511,6900.179%152.100
6627AIC2704,186,9315,149.5957,608,0017,526,151946,3651,8170.192%163.530
6728AIC2804,163,0615,126.8097,553,6127,474,936946,0301,7500.185%157.500
6829AIC2904,230,2065,167.4157,649,2307,574,482945,5021,5790.167%142.110
6930AIC3004,193,7625,205.3277,549,0957,468,961942,2871,8160.193%163.440
7031AIC3104,180,3675,136.6227,518,5217,437,473936,4171,8730.200%168.570
7264AIV1614,653,7645,393.8488,449,8407,716,013936,90525,7872.752%2,320.830
7365AIV1624,627,4635,338.3548,359,6197,613,381928,97025,5202.747%2,296.800
7466AIV1714,688,5175,357.5118,473,9827,773,723934,36926,1842.802%2,356.560
7567AIV1724,621,3265,335.8248,354,4497,655,502923,78125,6782.780%2,311.020
7668AIV1814,680,0555,346.8918,460,0877,713,565934,21825,6212.743%2,305.890
7769AIV1824,687,6615,358.8108,473,2217,701,359939,32426,1282.782%2,351.520
7870AIV1914,645,8965,336.4798,442,2167,764,155933,62125,6842.751%2,311.560
7971AIV1924,610,6925,330.3438,344,4607,637,496923,37725,1912.728%2,267.190
8072AIV2014,661,6765,336.8138,357,9357,583,740924,49525,2242.728%2,270.160
8173AIV2024,693,1505,343.6528,533,4547,720,770943,97826,1602.771%2,354.400
8474AIV2114,666,1365,353.9108,388,6897,671,262922,98824,9832.707%2,248.470
8575AIV2124,720,1625,329.9818,438,9187,706,675914,28224,6312.694%2,216.790
8676AIV2214,700,6095,359.5358,475,0807,757,980937,92926,3582.810%2,372.220
8777AIV2224,647,4435,354.2418,380,0307,670,305928,71425,6562.763%2,309.040
8878AIV2314,697,9475,377.9678,426,5207,662,830929,06725,2202.715%2,269.800
8979AIV2324,705,5855,381.6628,516,6567,738,074934,43825,4082.719%2,286.720
9080AIV2414,636,8925,355.1578,379,2207,645,012935,61325,7862.756%2,320.740
9181AIV2424,642,6505,341.8448,360,7387,627,316929,22225,5512.750%2,299.590
9282AIV2514,705,6595,388.9898,440,7777,662,447923,81425,2602.734%2,273.400
9383AIV2524,704,9955,341.1628,532,6667,734,506941,65526,1082.773%2,349.720
9484AIV2614,644,0685,345.4548,370,8327,629,637929,17525,2972.723%2,276.730
9585AIV2624,753,8365,347.8618,624,9997,880,977950,04526,6892.809%2,402.010
9686AIV2714,606,7605,340.8478,277,9507,531,316920,92725,4462.763%2,290.140
9787AIV2724,700,3915,358.0478,474,9667,712,135934,25325,9092.773%2,331.810
9888AIV2814,620,6095,387.5538,342,4127,571,493931,86625,9572.785%2,336.130
9989AIV2824,822,3635,359.2368,652,2197,909,404922,36025,0802.719%2,257.200
10090AIV2914,722,5745,337.0498,443,0987,647,916920,61724,8712.702%2,238.390
10191AIV2924,770,5675,360.6708,567,3117,745,531937,12525,6232.734%2,306.070
10292AIV3014,671,3375,326.8418,424,6077,645,648925,84025,1992.722%2,267.910
10393AIV3024,695,0105,337.6018,443,6987,646,461926,21825,1892.720%2,267.010
10494AIV3114,697,5115,386.2888,438,1977,610,053923,47325,1332.722%2,261.970
10595AIV3124,730,5045,349.5228,631,1607,775,394948,70426,3702.780%2,373.300
+
+ diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_register_20260723_044029_icache_report.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_register_20260723_044029_icache_report.html new file mode 100644 index 0000000000..615db9d814 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_register_20260723_044029_icache_report.html @@ -0,0 +1,362 @@ + + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 fdwic_submit_pmu_raw.json · schema fdwic-submit-pmu-v3 · 32 AIC + 64 AIV

+ +
+

register 阶段观察（phase_id=5）

+

边界 register_outputs_call_entry_to_return · 计数语义 running_read_clear_observed_bracket · + 时间语义 boundary_diagnostic_sys_cnt_between_observers · + expected_calls_per_core=1280

+ +

本报告使用 empty-bracket 估算进入 phase observed + 的局部记录代码开销。 + AIC/AIV 分别按“空区间每组记录开销 × 本阶段实际 begin/end 记录组数”计算，ALL + 再由两类核相加。参考值只从 phase observed 分子扣除该估算；原始 whole 分母保持不变， + 因为空区间没有测量 whole 窗口中的全部记录开销。所以下表的参考比例不是完整业务 + 阶段占完整调度窗口的比例，也不是精确校正。若参考值为负，表示记录代码开销估算已经 + 超过原始 observed；这只能说明该信号低于当前校准的分辨能力，不能截成零或解释成负的 + 业务事件。校准 raw：/home/q00473782/atomic/private/gpt/simpler/outputs/TestPagedAttentionUnroll_Case1_20260723_043533/fdwic_submit_pmu_raw.json · + SHA-256 56199f91545cc6a1fbf18b8bb2cfebae279cb5801516311a040d4facccf321ba。两份 provenance 已核验为同一场景、同一 Git revision 8c6b6e3a412d。

+ + +

本阶段若跨过 linked Kernel，会同时门控 SYS 时间与 PMU counter， + 因而 total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段。 + result-used return-ready atomic 依赖区间只从 SYS 边界累计值扣除，PMU counter + 仍含 atomic 指令事件；source-issue atomic + 保留在时间和计数口径内。没有 linked Kernel 的阶段不触发 PMU 停/开表； + 每次调用的分母仍是外层业务调用次数，不是 Begin/End 读数，也不是排除的 Kernel 调用数。

+ +

阶段原始 PMU 观测来自 running read-clear 的 total cycles，并按 AIC/AIV + 各自校准频率换算；提供 empty-bracket 时，页面主参考值再扣除局部记录代码开销估算。 + phase_elapsed_ticks 只是 SYS 边界闭合诊断，绝不按 1 GHz 当作阶段时间。 + Scalar busy 与 non-scalar residual（逐核先算 total−scalar）共同解释阶段 PMU total； + 未校准时只能展示 raw observed；提供 empty-bracket 后，记录代码开销估算只从 phase + observed 分子扣除，原始 whole 分母保持不变。当前数据无法得到完整业务阶段占完整调度 + 窗口的精确比例。 + linked vector/cube Kernel 从 SYS 与 PMU counter 一并门控排除；result-used return-ready atomic + 依赖区间只从 SYS 边界累计值扣除，I-cache/PMU counter 仍含其指令事件；source-issue atomic + 保留。request/miss 百分比的分母是同一 ELF、同一次采集的 Submit 整窗 primary。 + observed_plus_capture_gap = observed + (primary − shadow)； + 边界读数和插桩 bookkeeping 会进入 sample，因此“观测值”和“加全窗 capture gap”都不是原业务 + 事件数的数学上下界。不同 ELF 不能直接相减后冒充业务真值；提供 empty-bracket 时只按上方 + 明示的公式估算局部记录代码开销。request/miss 的逐核 min/max 是每核累计整个 phase + 的极值，不是逐调用极值。

+
+ + + + +
指标ALL96 核AIC32 核AIV64 核
Phase PMU total扣除记录代码开销估算后的参考值 Σ 56,406,147.000 cycles
≈ 34,188.776 µs（1.649844 cycles/ns）
原始 observed Σ 248,694,483 cycles（≈ 150,738.181 µs）− 记录代码开销估算 192,288,336.000 cycles
记录代码开销估算 / 原始 observed 77.319%；原始逐核 最小 2,345,551；最大 2,732,437；原始 2,023.881 cycles/call		扣除记录代码开销估算后的参考值 Σ 15,900,738.000 cycles
≈ 9,636.449 µs（1.650062 cycles/ns）
原始 observed Σ 76,727,541 cycles（≈ 46,499.793 µs）− 记录代码开销估算 60,826,803.000 cycles
记录代码开销估算 / 原始 observed 79.276%；原始逐核 最小 2,345,551；最大 2,439,083；原始 1,873.231 cycles/call		扣除记录代码开销估算后的参考值 Σ 40,505,409.000 cycles
≈ 24,552.736 µs（1.649731 cycles/ns）
原始 observed Σ 171,966,942 cycles（≈ 104,239.383 µs）− 记录代码开销估算 131,461,533.000 cycles
记录代码开销估算 / 原始 observed 76.446%；原始逐核 最小 2,635,187；最大 2,732,437；原始 2,099.206 cycles/call
Phase scalar busy扣除记录代码开销估算后的参考值 Σ 44,338,114.000 cycles
≈ 26,874.125 µs（1.649844 cycles/ns）
原始 observed Σ 168,184,052 cycles（≈ 101,939.366 µs）− 记录代码开销估算 123,845,938.000 cycles
记录代码开销估算 / 原始 observed 73.637%；原始逐核 最小 1,658,575；最大 1,816,536；原始 1,368.685 cycles/call		扣除记录代码开销估算后的参考值 Σ 14,597,384.000 cycles
≈ 8,846.567 µs（1.650062 cycles/ns）
原始 observed Σ 54,657,090 cycles（≈ 33,124.264 µs）− 记录代码开销估算 40,059,706.000 cycles
记录代码开销估算 / 原始 observed 73.293%；原始逐核 最小 1,658,575；最大 1,740,706；原始 1,334.402 cycles/call		扣除记录代码开销估算后的参考值 Σ 29,740,730.000 cycles
≈ 18,027.624 µs（1.649731 cycles/ns）
原始 observed Σ 113,526,962 cycles（≈ 68,815.438 µs）− 记录代码开销估算 83,786,232.000 cycles
记录代码开销估算 / 原始 observed 73.803%；原始逐核 最小 1,724,613；最大 1,816,536；原始 1,385.827 cycles/call
非 Scalar-busy 残余扣除记录代码开销估算后的参考值 Σ 12,068,033.000 cycles
≈ 7,314.651 µs（1.649844 cycles/ns）
原始 observed Σ 80,510,431 cycles（≈ 48,798.814 µs）− 记录代码开销估算 68,442,398.000 cycles
记录代码开销估算 / 原始 observed 85.011%；原始逐核 最小 679,724；最大 929,750；原始 655.196 cycles/call		扣除记录代码开销估算后的参考值 Σ 1,303,354.000 cycles
≈ 789.882 µs（1.650062 cycles/ns）
原始 observed Σ 22,070,451 cycles（≈ 13,375.528 µs）− 记录代码开销估算 20,767,097.000 cycles
记录代码开销估算 / 原始 observed 94.095%；原始逐核 最小 679,724；最大 701,583；原始 538.829 cycles/call		扣除记录代码开销估算后的参考值 Σ 10,764,679.000 cycles
≈ 6,525.112 µs（1.649731 cycles/ns）
原始 observed Σ 58,439,980 cycles（≈ 35,423.945 µs）− 记录代码开销估算 47,675,301.000 cycles
记录代码开销估算 / 原始 observed 81.580%；原始逐核 最小 895,222；最大 929,750；原始 713.379 cycles/call
阶段参考关系（raw 同格保留）扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 78.605%
参考 Non-scalar / 参考 Phase total 21.395%
参考 Phase total / 原始 whole total 6.769%；原始 observed 29.845%
参考 Phase scalar / 原始 whole scalar 5.962%；原始 observed 22.615%
whole scalar−shadow scalar：Σ 245,760 cycles；逐核 最小 2,560；最大 2,560		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 91.803%
参考 Non-scalar / 参考 Phase total 8.197%
参考 Phase total / 原始 whole total 6.352%；原始 observed 30.653%
参考 Phase scalar / 原始 whole scalar 5.958%；原始 observed 22.309%
whole scalar−shadow scalar：Σ 81,920 cycles；逐核 最小 2,560；最大 2,560		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 73.424%
参考 Non-scalar / 参考 Phase total 26.576%
参考 Phase total / 原始 whole total 6.948%；原始 observed 29.498%
参考 Phase scalar / 原始 whole scalar 5.964%；原始 observed 22.765%
whole scalar−shadow scalar：Σ 163,840 cycles；逐核 最小 2,560；最大 2,560
Request（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 5,560,990.000
原始 observed 12,430,608 − 记录代码开销估算 6,869,618.000
参考值 / 原始整窗 6.173%；原始 observed / 原始整窗 13.798%
记录代码开销估算 / 原始 observed 55.264%
原始逐核 最小 126,587；最大 132,112；原始整窗 90,091,095
原始 capture gap +0；加 gap 后 12,430,608（13.798%）		扣除记录代码开销估算后的参考值 1,949,540.000
原始 observed 4,171,692 − 记录代码开销估算 2,222,152.000
参考值 / 原始整窗 6.428%；原始 observed / 原始整窗 13.755%
记录代码开销估算 / 原始 observed 53.267%
原始逐核 最小 126,587；最大 132,112；原始整窗 30,327,702
原始 capture gap +0；加 gap 后 4,171,692（13.755%）		扣除记录代码开销估算后的参考值 3,611,450.000
原始 observed 8,258,916 − 记录代码开销估算 4,647,466.000
参考值 / 原始整窗 6.043%；原始 observed / 原始整窗 13.819%
记录代码开销估算 / 原始 observed 56.272%
原始逐核 最小 127,357；最大 130,347；原始整窗 59,763,393
原始 capture gap +0；加 gap 后 8,258,916（13.819%）
Miss（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 215,428.000
原始 observed 450,294 − 记录代码开销估算 234,866.000
参考值 / 原始整窗 6.116%；原始 observed / 原始整窗 12.785%
记录代码开销估算 / 原始 observed 52.158%
原始逐核 最小 695；最大 6,811；原始整窗 3,522,153
原始 capture gap +0；加 gap 后 450,294（12.785%）		扣除记录代码开销估算后的参考值 22,763.000
原始 observed 25,252 − 记录代码开销估算 2,489.000
参考值 / 原始整窗 14.151%；原始 observed / 原始整窗 15.698%
记录代码开销估算 / 原始 observed 9.857%
原始逐核 最小 695；最大 849；原始整窗 160,859
原始 capture gap +0；加 gap 后 25,252（15.698%）		扣除记录代码开销估算后的参考值 192,665.000
原始 observed 425,042 − 记录代码开销估算 232,377.000
参考值 / 原始整窗 5.732%；原始 observed / 原始整窗 12.645%
记录代码开销估算 / 原始 observed 54.672%
原始逐核 最小 6,523；最大 6,811；原始整窗 3,361,294
原始 capture gap +0；加 gap 后 425,042（12.645%）
SYS 边界诊断 / Begin-EndΣ 27,870,336 raw ticks
逐核 最小 214,334；最大 340,526；仅边界诊断
Begin / End：122,880 / 122,880
业务调用 122,880 次；排除 linked Kernel 调用 0 次		Σ 7,802,312 raw ticks
逐核 最小 214,334；最大 263,214；仅边界诊断
Begin / End：40,960 / 40,960
业务调用 40,960 次；排除 linked Kernel 调用 0 次		Σ 20,068,024 raw ticks
逐核 最小 286,670；最大 340,526；仅边界诊断
Begin / End：81,920 / 81,920
业务调用 81,920 次；排除 linked Kernel 调用 0 次
+
+ + +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
a31285d86c68c140a4a566ed3c58fdddcad6a071af9d9c2b3860ee076b1e3272
+
Profile / extra cache
submit-pmu-register / + 32c26e06ad76d186
+
Profiled cache key
TestPagedAttentionUnroll / a5 / fully_distributed_within_core / submit-pmu-register
+
Source state
source-v2:8c6b6e3a412d2c7c7113c2d8f714a14eb3786f80:4523abc45c34a46ea8b478373a78a015543d0254c1284f987c73775c4bd2b787:ff85282e42fc1b03ba04d010a0b3b793349657e8b1e321fd7b31017015f1dee9
+
Source-state stamp
/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/32c26e06ad76d186/aicore/.git_commit
+
Compile definitions
PTO_FDWIC_SUBMIT_PMU=1
PTO_FDWIC_SUBMIT_PMU_PHASE_ID=5
PTO_FDWIC_TRACE_ENABLED=0
+
+
+ + + +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
AICore final2,688,032c870328ac6fe0c4837529f6d77165a79579ff0221f7bf8b4c3e88c8cb107112d216,656f9894e8ca31c66057246b4adb96cdd12fd9bcaef6e0cbf5218e8e80b1f567aca/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/aicore-extra/32c26e06ad76d186/aicore_kernel.o
AIC combined2,015,81616dff1d0be73b82dad2b3c9cd46edf7add17b759c44037b0f99f5fc8abf99483101,1283c9cd6c6a81abaa8655a0586a48f356486bd8e53cb9f4e82a83b7cc8196f162f/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/32c26e06ad76d186/aicore/aicore_aic_combined.o
AIV combined2,277,616a59a976d24b3a7e4a1d973ef95ec3f9110dd62401954f5abca393fb540aaa9a1115,2806e80a5afe77ffd19426efad9cd0ea1525fe8fa5c9bc443227aeca490fbb1d618/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/32c26e06ad76d186/aicore/aicore_aiv_combined.o
Host runtime11,776,1440369d0602054bf612f86b58c294503e00486bd505bff9ee7a455994f043147cc454,6325ac381ed10be5f5d492b8072c051a177131f4caec6d6f6bc9bf50880da4bb0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so
+
+ +
+
原始全局 Submit 时间范围（墙钟） + 5,788.451 µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + 128096 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-256c398970e49c42b3d8d9e8001bdc5b07488da48be09d084ccca2d552a11f7f82d
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + 90.000 ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×90 ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
+
+

ALL · 96 核

+
+
Submit PMU total/core
+
均值 8,680,065.4；最小 7,645,164；最大 9,302,656 cycles
+ 等效时间 均值 5,261.143；最小 4,633.871；最大 5,638.506 µs + （按 1.649844 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,746,674.0；最小 7,484,292；最大 8,150,032 cycles
+ 等效时间 均值 4,695.398；最小 4,536.363；最大 4,939.880 µs； + 加权占比 89.247% +
+
非 Scalar-busy 残余/core
+
均值 933,391.4；最小 143,237；最大 1,445,679 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 565.745；最小 86.819；最大 876.252 µs +
+
SYS gate 边界诊断/core
+
均值 4,825,077.7；最小 4,192,812；最大 5,244,033 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,593.141；最小 5,231.758；最大 5,787.951 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 768,063.6；最小 506,845；最大 1,196,322 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 86.268% +
+
Primary I-cache request/core
最小 916,532；最大 1,002,956
+
Primary I-cache miss/core
最小 4,552；最大 54,306
+
加权 miss rate
3.910%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 409.680；最大 4,887.540 µs
+
+
+ +
+

AIC · 32 核

+
+
Submit PMU total/core
+
均值 7,822,086.9；最小 7,645,164；最大 8,316,958 cycles
+ 等效时间 均值 4,740.481；最小 4,633.259；最大 5,040.391 µs + （按 1.650062 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,656,214.4；最小 7,484,292；最大 8,150,032 cycles
+ 等效时间 均值 4,639.956；最小 4,535.764；最大 4,939.228 µs； + 加权占比 97.879% +
+
非 Scalar-busy 残余/core
+
均值 165,872.5；最小 143,237；最大 193,754 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 100.525；最小 86.807；最大 117.422 µs +
+
SYS gate 边界诊断/core
+
均值 4,253,429.9；最小 4,192,812；最大 4,397,633 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,293.564；最小 5,231.758；最大 5,577.759 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 1,040,134.1；最小 959,627；最大 1,196,322 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 80.351% +
+
Primary I-cache request/core
最小 927,340；最大 1,002,956
+
Primary I-cache miss/core
最小 4,552；最大 5,437
+
加权 miss rate
0.530%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 409.680；最大 489.330 µs
+
+
+ +
+

AIV · 64 核

+
+
Submit PMU total/core
+
均值 9,109,054.7；最小 8,866,587；最大 9,302,656 cycles
+ 等效时间 均值 5,521.539；最小 5,374.565；最大 5,638.893 µs + （按 1.649731 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 7,791,903.8；最小 7,596,886；最大 8,004,867 cycles
+ 等效时间 均值 4,723.136；最小 4,604.924；最大 4,852.226 µs； + 加权占比 85.540% +
+
非 Scalar-busy 残余/core
+
均值 1,317,150.9；最小 1,183,685；最大 1,445,679 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 798.403；最小 717.502；最大 876.312 µs +
+
SYS gate 边界诊断/core
+
均值 5,110,901.6；最小 4,993,011；最大 5,244,033 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,742.930；最小 5,712.516；最大 5,787.951 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 632,028.3；最小 506,845；最大 758,867 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 88.995% +
+
Primary I-cache request/core
最小 916,532；最大 959,307
+
Primary I-cache miss/core
最小 50,950；最大 54,306
+
加权 miss rate
5.624%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 4,585.500；最大 4,887.540 µs
+
+
+
+

逐物理核分布

+

AICAIV

+
+
+
SYS gate boundary-diagnostic ticks/core
+ + + + 5,244,033 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_submit_elapsed_ticks=4228842physical=1 logical=1 role=AIC scalar_submit_elapsed_ticks=4244081physical=2 logical=2 role=AIC scalar_submit_elapsed_ticks=4267705physical=3 logical=3 role=AIC scalar_submit_elapsed_ticks=4192812physical=4 logical=4 role=AIC scalar_submit_elapsed_ticks=4252031physical=5 logical=5 role=AIC scalar_submit_elapsed_ticks=4228501physical=6 logical=6 role=AIC scalar_submit_elapsed_ticks=4227215physical=7 logical=7 role=AIC scalar_submit_elapsed_ticks=4272716physical=8 logical=8 role=AIC scalar_submit_elapsed_ticks=4219505physical=9 logical=9 role=AIC scalar_submit_elapsed_ticks=4222972physical=10 logical=10 role=AIC scalar_submit_elapsed_ticks=4234709physical=12 logical=11 role=AIC scalar_submit_elapsed_ticks=4256287physical=13 logical=12 role=AIC scalar_submit_elapsed_ticks=4381437physical=14 logical=13 role=AIC scalar_submit_elapsed_ticks=4229851physical=15 logical=14 role=AIC scalar_submit_elapsed_ticks=4383125physical=16 logical=15 role=AIC scalar_submit_elapsed_ticks=4228835physical=54 logical=16 role=AIC scalar_submit_elapsed_ticks=4235437physical=55 logical=17 role=AIC scalar_submit_elapsed_ticks=4223088physical=56 logical=18 role=AIC scalar_submit_elapsed_ticks=4247586physical=57 logical=19 role=AIC scalar_submit_elapsed_ticks=4228282physical=58 logical=20 role=AIC scalar_submit_elapsed_ticks=4206341physical=60 logical=21 role=AIC scalar_submit_elapsed_ticks=4251033physical=61 logical=22 role=AIC scalar_submit_elapsed_ticks=4268864physical=62 logical=23 role=AIC scalar_submit_elapsed_ticks=4282614physical=63 logical=24 role=AIC scalar_submit_elapsed_ticks=4239574physical=64 logical=25 role=AIC scalar_submit_elapsed_ticks=4218483physical=65 logical=26 role=AIC scalar_submit_elapsed_ticks=4397633physical=66 logical=27 role=AIC scalar_submit_elapsed_ticks=4230562physical=67 logical=28 role=AIC scalar_submit_elapsed_ticks=4209961physical=68 logical=29 role=AIC scalar_submit_elapsed_ticks=4250447physical=69 logical=30 role=AIC scalar_submit_elapsed_ticks=4264279physical=70 logical=31 role=AIC scalar_submit_elapsed_ticks=4284949physical=18 logical=32 role=AIV scalar_submit_elapsed_ticks=5137352physical=19 logical=33 role=AIV scalar_submit_elapsed_ticks=5136139physical=20 logical=34 role=AIV scalar_submit_elapsed_ticks=5134649physical=21 logical=35 role=AIV scalar_submit_elapsed_ticks=5127657physical=22 logical=36 role=AIV scalar_submit_elapsed_ticks=5168600physical=23 logical=37 role=AIV scalar_submit_elapsed_ticks=5129535physical=24 logical=38 role=AIV scalar_submit_elapsed_ticks=5172742physical=25 logical=39 role=AIV scalar_submit_elapsed_ticks=5076200physical=26 logical=40 role=AIV scalar_submit_elapsed_ticks=5141639physical=27 logical=41 role=AIV scalar_submit_elapsed_ticks=5138477physical=28 logical=42 role=AIV scalar_submit_elapsed_ticks=5121087physical=29 logical=43 role=AIV scalar_submit_elapsed_ticks=5096688physical=30 logical=44 role=AIV scalar_submit_elapsed_ticks=5142668physical=31 logical=45 role=AIV scalar_submit_elapsed_ticks=5102914physical=32 logical=46 role=AIV scalar_submit_elapsed_ticks=5047638physical=33 logical=47 role=AIV scalar_submit_elapsed_ticks=5060554physical=34 logical=48 role=AIV scalar_submit_elapsed_ticks=5155494physical=35 logical=49 role=AIV scalar_submit_elapsed_ticks=5164635physical=36 logical=50 role=AIV scalar_submit_elapsed_ticks=5058088physical=37 logical=51 role=AIV scalar_submit_elapsed_ticks=5098216physical=38 logical=52 role=AIV scalar_submit_elapsed_ticks=5121933physical=39 logical=53 role=AIV scalar_submit_elapsed_ticks=5091403physical=42 logical=54 role=AIV scalar_submit_elapsed_ticks=5052509physical=43 logical=55 role=AIV scalar_submit_elapsed_ticks=5105902physical=44 logical=56 role=AIV scalar_submit_elapsed_ticks=5121444physical=45 logical=57 role=AIV scalar_submit_elapsed_ticks=5084080physical=46 logical=58 role=AIV scalar_submit_elapsed_ticks=5019347physical=47 logical=59 role=AIV scalar_submit_elapsed_ticks=5129912physical=48 logical=60 role=AIV scalar_submit_elapsed_ticks=4993011physical=49 logical=61 role=AIV scalar_submit_elapsed_ticks=5110583physical=50 logical=62 role=AIV scalar_submit_elapsed_ticks=5142578physical=51 logical=63 role=AIV scalar_submit_elapsed_ticks=5175379physical=72 logical=64 role=AIV scalar_submit_elapsed_ticks=5087165physical=73 logical=65 role=AIV scalar_submit_elapsed_ticks=5143371physical=74 logical=66 role=AIV scalar_submit_elapsed_ticks=5120157physical=75 logical=67 role=AIV scalar_submit_elapsed_ticks=5139239physical=76 logical=68 role=AIV scalar_submit_elapsed_ticks=5161817physical=77 logical=69 role=AIV scalar_submit_elapsed_ticks=5085939physical=78 logical=70 role=AIV scalar_submit_elapsed_ticks=5104275physical=79 logical=71 role=AIV scalar_submit_elapsed_ticks=5091759physical=80 logical=72 role=AIV scalar_submit_elapsed_ticks=5043019physical=81 logical=73 role=AIV scalar_submit_elapsed_ticks=5094720physical=84 logical=74 role=AIV scalar_submit_elapsed_ticks=5078486physical=85 logical=75 role=AIV scalar_submit_elapsed_ticks=5179759physical=86 logical=76 role=AIV scalar_submit_elapsed_ticks=5128544physical=87 logical=77 role=AIV scalar_submit_elapsed_ticks=5085209physical=88 logical=78 role=AIV scalar_submit_elapsed_ticks=5092602physical=89 logical=79 role=AIV scalar_submit_elapsed_ticks=5041161physical=90 logical=80 role=AIV scalar_submit_elapsed_ticks=5106059physical=91 logical=81 role=AIV scalar_submit_elapsed_ticks=5148639physical=92 logical=82 role=AIV scalar_submit_elapsed_ticks=5121523physical=93 logical=83 role=AIV scalar_submit_elapsed_ticks=5093112physical=94 logical=84 role=AIV scalar_submit_elapsed_ticks=5137694physical=95 logical=85 role=AIV scalar_submit_elapsed_ticks=5132587physical=96 logical=86 role=AIV scalar_submit_elapsed_ticks=5108158physical=97 logical=87 role=AIV scalar_submit_elapsed_ticks=5077109physical=98 logical=88 role=AIV scalar_submit_elapsed_ticks=5086190physical=99 logical=89 role=AIV scalar_submit_elapsed_ticks=5244033physical=100 logical=90 role=AIV scalar_submit_elapsed_ticks=5047608physical=101 logical=91 role=AIV scalar_submit_elapsed_ticks=5193448physical=102 logical=92 role=AIV scalar_submit_elapsed_ticks=5110415physical=103 logical=93 role=AIV scalar_submit_elapsed_ticks=5127779physical=104 logical=94 role=AIV scalar_submit_elapsed_ticks=5041133physical=105 logical=95 role=AIV scalar_submit_elapsed_ticks=5085943 + +
+ +
+
PMU total cycles/core
+ + + + 9,302,656 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC total_cycles=7710607physical=1 logical=1 role=AIC total_cycles=7752114physical=2 logical=2 role=AIC total_cycles=7779803physical=3 logical=3 role=AIC total_cycles=7645164physical=4 logical=4 role=AIC total_cycles=7775369physical=5 logical=5 role=AIC total_cycles=7863509physical=6 logical=6 role=AIC total_cycles=7714302physical=7 logical=7 role=AIC total_cycles=7910820physical=8 logical=8 role=AIC total_cycles=7703844physical=9 logical=9 role=AIC total_cycles=7752735physical=10 logical=10 role=AIC total_cycles=7695128physical=12 logical=11 role=AIC total_cycles=7727195physical=13 logical=12 role=AIC total_cycles=8187304physical=14 logical=13 role=AIC total_cycles=7809321physical=15 logical=14 role=AIC total_cycles=8175659physical=16 logical=15 role=AIC total_cycles=7771220physical=54 logical=16 role=AIC total_cycles=7687514physical=55 logical=17 role=AIC total_cycles=7786175physical=56 logical=18 role=AIC total_cycles=7767412physical=57 logical=19 role=AIC total_cycles=7773048physical=58 logical=20 role=AIC total_cycles=7723523physical=60 logical=21 role=AIC total_cycles=7915345physical=61 logical=22 role=AIC total_cycles=7933345physical=62 logical=23 role=AIC total_cycles=7880741physical=63 logical=24 role=AIC total_cycles=7837092physical=64 logical=25 role=AIC total_cycles=7744570physical=65 logical=26 role=AIC total_cycles=8316958physical=66 logical=27 role=AIC total_cycles=7697424physical=67 logical=28 role=AIC total_cycles=7781425physical=68 logical=29 role=AIC total_cycles=7795783physical=69 logical=30 role=AIC total_cycles=7846326physical=70 logical=31 role=AIC total_cycles=7846007physical=18 logical=32 role=AIV total_cycles=9099618physical=19 logical=33 role=AIV total_cycles=9095461physical=20 logical=34 role=AIV total_cycles=9156364physical=21 logical=35 role=AIV total_cycles=9151840physical=22 logical=36 role=AIV total_cycles=9219476physical=23 logical=37 role=AIV total_cycles=9148041physical=24 logical=38 role=AIV total_cycles=9255184physical=25 logical=39 role=AIV total_cycles=9058574physical=26 logical=40 role=AIV total_cycles=9167235physical=27 logical=41 role=AIV total_cycles=9161463physical=28 logical=42 role=AIV total_cycles=9136436physical=29 logical=43 role=AIV total_cycles=9116607physical=30 logical=44 role=AIV total_cycles=9187767physical=31 logical=45 role=AIV total_cycles=9089893physical=32 logical=46 role=AIV total_cycles=8999928physical=33 logical=47 role=AIV total_cycles=8974408physical=34 logical=48 role=AIV total_cycles=9174955physical=35 logical=49 role=AIV total_cycles=9261564physical=36 logical=50 role=AIV total_cycles=9008463physical=37 logical=51 role=AIV total_cycles=9095761physical=38 logical=52 role=AIV total_cycles=9130648physical=39 logical=53 role=AIV total_cycles=9030983physical=42 logical=54 role=AIV total_cycles=8978399physical=43 logical=55 role=AIV total_cycles=9076710physical=44 logical=56 role=AIV total_cycles=9169844physical=45 logical=57 role=AIV total_cycles=9075086physical=46 logical=58 role=AIV total_cycles=8933771physical=47 logical=59 role=AIV total_cycles=9093299physical=48 logical=60 role=AIV total_cycles=8866587physical=49 logical=61 role=AIV total_cycles=9138391physical=50 logical=62 role=AIV total_cycles=9180596physical=51 logical=63 role=AIV total_cycles=9258987physical=72 logical=64 role=AIV total_cycles=9108301physical=73 logical=65 role=AIV total_cycles=9257268physical=74 logical=66 role=AIV total_cycles=9112706physical=75 logical=67 role=AIV total_cycles=9181796physical=76 logical=68 role=AIV total_cycles=9272679physical=77 logical=69 role=AIV total_cycles=9089467physical=78 logical=70 role=AIV total_cycles=9090608physical=79 logical=71 role=AIV total_cycles=9082556physical=80 logical=72 role=AIV total_cycles=9014296physical=81 logical=73 role=AIV total_cycles=9159500physical=84 logical=74 role=AIV total_cycles=9010971physical=85 logical=75 role=AIV total_cycles=9246024physical=86 logical=76 role=AIV total_cycles=9132763physical=87 logical=77 role=AIV total_cycles=8993051physical=88 logical=78 role=AIV total_cycles=9023386physical=89 logical=79 role=AIV total_cycles=8966341physical=90 logical=80 role=AIV total_cycles=9104663physical=91 logical=81 role=AIV total_cycles=9254544physical=92 logical=82 role=AIV total_cycles=9074820physical=93 logical=83 role=AIV total_cycles=9060931physical=94 logical=84 role=AIV total_cycles=9171367physical=95 logical=85 role=AIV total_cycles=9160391physical=96 logical=86 role=AIV total_cycles=9109495physical=97 logical=87 role=AIV total_cycles=9048703physical=98 logical=88 role=AIV total_cycles=9070474physical=99 logical=89 role=AIV total_cycles=9302656physical=100 logical=90 role=AIV total_cycles=8939113physical=101 logical=91 role=AIV total_cycles=9255286physical=102 logical=92 role=AIV total_cycles=9097423physical=103 logical=93 role=AIV total_cycles=9077350physical=104 logical=94 role=AIV total_cycles=8978696physical=105 logical=95 role=AIV total_cycles=9039535 + +
+ +
+
Scalar busy cycles/core
+ + + + 8,150,032 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_busy=7549235physical=1 logical=1 role=AIC scalar_busy=7587347physical=2 logical=2 role=AIC scalar_busy=7605796physical=3 logical=3 role=AIC scalar_busy=7484292physical=4 logical=4 role=AIC scalar_busy=7596190physical=5 logical=5 role=AIC scalar_busy=7700179physical=6 logical=6 role=AIC scalar_busy=7535282physical=7 logical=7 role=AIC scalar_busy=7758719physical=8 logical=8 role=AIC scalar_busy=7528576physical=9 logical=9 role=AIC scalar_busy=7583332physical=10 logical=10 role=AIC scalar_busy=7526121physical=12 logical=11 role=AIC scalar_busy=7560893physical=13 logical=12 role=AIC scalar_busy=8028194physical=14 logical=13 role=AIC scalar_busy=7628584physical=15 logical=14 role=AIC scalar_busy=8025670physical=16 logical=15 role=AIC scalar_busy=7595654physical=54 logical=16 role=AIC scalar_busy=7540031physical=55 logical=17 role=AIC scalar_busy=7630468physical=56 logical=18 role=AIC scalar_busy=7624175physical=57 logical=19 role=AIC scalar_busy=7605601physical=58 logical=20 role=AIC scalar_busy=7568816physical=60 logical=21 role=AIC scalar_busy=7755698physical=61 logical=22 role=AIC scalar_busy=7756779physical=62 logical=23 role=AIC scalar_busy=7716679physical=63 logical=24 role=AIC scalar_busy=7672389physical=64 logical=25 role=AIC scalar_busy=7598727physical=65 logical=26 role=AIC scalar_busy=8150032physical=66 logical=27 role=AIC scalar_busy=7517274physical=67 logical=28 role=AIC scalar_busy=7587671physical=68 logical=29 role=AIC scalar_busy=7635677physical=69 logical=30 role=AIC scalar_busy=7663986physical=70 logical=31 role=AIC scalar_busy=7680795physical=18 logical=32 role=AIV scalar_busy=7751544physical=19 logical=33 role=AIV scalar_busy=7676164physical=20 logical=34 role=AIV scalar_busy=7847701physical=21 logical=35 role=AIV scalar_busy=7752896physical=22 logical=36 role=AIV scalar_busy=7884060physical=23 logical=37 role=AIV scalar_busy=7745963physical=24 logical=38 role=AIV scalar_busy=7940649physical=25 logical=39 role=AIV scalar_busy=7689527physical=26 logical=40 role=AIV scalar_busy=7851515physical=27 logical=41 role=AIV scalar_busy=7748382physical=28 logical=42 role=AIV scalar_busy=7829963physical=29 logical=43 role=AIV scalar_busy=7714117physical=30 logical=44 role=AIV scalar_busy=7881034physical=31 logical=45 role=AIV scalar_busy=7755346physical=32 logical=46 role=AIV scalar_busy=7780601physical=33 logical=47 role=AIV scalar_busy=7651763physical=34 logical=48 role=AIV scalar_busy=7881403physical=35 logical=49 role=AIV scalar_busy=7905627physical=36 logical=50 role=AIV scalar_busy=7753573physical=37 logical=51 role=AIV scalar_busy=7798835physical=38 logical=52 role=AIV scalar_busy=7845288physical=39 logical=53 role=AIV scalar_busy=7660674physical=42 logical=54 role=AIV scalar_busy=7716252physical=43 logical=55 role=AIV scalar_busy=7758525physical=44 logical=56 role=AIV scalar_busy=7973806physical=45 logical=57 role=AIV scalar_busy=7762731physical=46 logical=58 role=AIV scalar_busy=7640942physical=47 logical=59 role=AIV scalar_busy=7740317physical=48 logical=60 role=AIV scalar_busy=7682902physical=49 logical=61 role=AIV scalar_busy=7855936physical=50 logical=62 role=AIV scalar_busy=7869891physical=51 logical=63 role=AIV scalar_busy=7895020physical=72 logical=64 role=AIV scalar_busy=7862110physical=73 logical=65 role=AIV scalar_busy=7918939physical=74 logical=66 role=AIV scalar_busy=7838763physical=75 logical=67 role=AIV scalar_busy=7833799physical=76 logical=68 role=AIV scalar_busy=8004867physical=77 logical=69 role=AIV scalar_busy=7764365physical=78 logical=70 role=AIV scalar_busy=7849695physical=79 logical=71 role=AIV scalar_busy=7738892physical=80 logical=72 role=AIV scalar_busy=7793842physical=81 logical=73 role=AIV scalar_busy=7834716physical=84 logical=74 role=AIV scalar_busy=7778708physical=85 logical=75 role=AIV scalar_busy=7935263physical=86 logical=76 role=AIV scalar_busy=7877434physical=87 logical=77 role=AIV scalar_busy=7596886physical=88 logical=78 role=AIV scalar_busy=7765105physical=89 logical=79 role=AIV scalar_busy=7666986physical=90 logical=80 role=AIV scalar_busy=7824935physical=91 logical=81 role=AIV scalar_busy=7885387physical=92 logical=82 role=AIV scalar_busy=7818299physical=93 logical=83 role=AIV scalar_busy=7755185physical=94 logical=84 role=AIV scalar_busy=7866090physical=95 logical=85 role=AIV scalar_busy=7824191physical=96 logical=86 role=AIV scalar_busy=7808917physical=97 logical=87 role=AIV scalar_busy=7652254physical=98 logical=88 role=AIV scalar_busy=7699840physical=99 logical=89 role=AIV scalar_busy=7856977physical=100 logical=90 role=AIV scalar_busy=7684561physical=101 logical=91 role=AIV scalar_busy=7887384physical=102 logical=92 role=AIV scalar_busy=7744565physical=103 logical=93 role=AIV scalar_busy=7658784physical=104 logical=94 role=AIV scalar_busy=7694617physical=105 logical=95 role=AIV scalar_busy=7716540 + +
+ +
+
Primary I-cache requests/core
+ + + + 1,002,956 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_requests=930419physical=1 logical=1 role=AIC icache_requests=936103physical=2 logical=2 role=AIC icache_requests=936949physical=3 logical=3 role=AIC icache_requests=933858physical=4 logical=4 role=AIC icache_requests=933452physical=5 logical=5 role=AIC icache_requests=959288physical=6 logical=6 role=AIC icache_requests=929417physical=7 logical=7 role=AIC icache_requests=969316physical=8 logical=8 role=AIC icache_requests=932399physical=9 logical=9 role=AIC icache_requests=950272physical=10 logical=10 role=AIC icache_requests=927449physical=12 logical=11 role=AIC icache_requests=932371physical=13 logical=12 role=AIC icache_requests=993048physical=14 logical=13 role=AIC icache_requests=950097physical=15 logical=14 role=AIC icache_requests=986138physical=16 logical=15 role=AIC icache_requests=939433physical=54 logical=16 role=AIC icache_requests=927340physical=55 logical=17 role=AIC icache_requests=943573physical=56 logical=18 role=AIC icache_requests=933298physical=57 logical=19 role=AIC icache_requests=944820physical=58 logical=20 role=AIC icache_requests=939100physical=60 logical=21 role=AIC icache_requests=957615physical=61 logical=22 role=AIC icache_requests=957746physical=62 logical=23 role=AIC icache_requests=952619physical=63 logical=24 role=AIC icache_requests=961095physical=64 logical=25 role=AIC icache_requests=943260physical=65 logical=26 role=AIC icache_requests=1002956physical=66 logical=27 role=AIC icache_requests=935103physical=67 logical=28 role=AIC icache_requests=950004physical=68 logical=29 role=AIC icache_requests=942725physical=69 logical=30 role=AIC icache_requests=951144physical=70 logical=31 role=AIC icache_requests=945295physical=18 logical=32 role=AIV icache_requests=927922physical=19 logical=33 role=AIV icache_requests=916787physical=20 logical=34 role=AIV icache_requests=937058physical=21 logical=35 role=AIV icache_requests=944773physical=22 logical=36 role=AIV icache_requests=937221physical=23 logical=37 role=AIV icache_requests=933540physical=24 logical=38 role=AIV icache_requests=939084physical=25 logical=39 role=AIV icache_requests=931871physical=26 logical=40 role=AIV icache_requests=936698physical=27 logical=41 role=AIV icache_requests=934693physical=28 logical=42 role=AIV icache_requests=933295physical=29 logical=43 role=AIV icache_requests=932123physical=30 logical=44 role=AIV icache_requests=938913physical=31 logical=45 role=AIV icache_requests=935120physical=32 logical=46 role=AIV icache_requests=929638physical=33 logical=47 role=AIV icache_requests=927049physical=34 logical=48 role=AIV icache_requests=916532physical=35 logical=49 role=AIV icache_requests=947639physical=36 logical=50 role=AIV icache_requests=930181physical=37 logical=51 role=AIV icache_requests=934149physical=38 logical=52 role=AIV icache_requests=934951physical=39 logical=53 role=AIV icache_requests=919204physical=42 logical=54 role=AIV icache_requests=929202physical=43 logical=55 role=AIV icache_requests=926704physical=44 logical=56 role=AIV icache_requests=953302physical=45 logical=57 role=AIV icache_requests=936907physical=46 logical=58 role=AIV icache_requests=929237physical=47 logical=59 role=AIV icache_requests=928565physical=48 logical=60 role=AIV icache_requests=916969physical=49 logical=61 role=AIV icache_requests=944949physical=50 logical=62 role=AIV icache_requests=935645physical=51 logical=63 role=AIV icache_requests=934872physical=72 logical=64 role=AIV icache_requests=945575physical=73 logical=65 role=AIV icache_requests=958467physical=74 logical=66 role=AIV icache_requests=923825physical=75 logical=67 role=AIV icache_requests=943758physical=76 logical=68 role=AIV icache_requests=946309physical=77 logical=69 role=AIV icache_requests=936268physical=78 logical=70 role=AIV icache_requests=928160physical=79 logical=71 role=AIV icache_requests=928807physical=80 logical=72 role=AIV icache_requests=932745physical=81 logical=73 role=AIV icache_requests=959307physical=84 logical=74 role=AIV icache_requests=924348physical=85 logical=75 role=AIV icache_requests=945358physical=86 logical=76 role=AIV icache_requests=928631physical=87 logical=77 role=AIV icache_requests=922113physical=88 logical=78 role=AIV icache_requests=929295physical=89 logical=79 role=AIV icache_requests=926276physical=90 logical=80 role=AIV icache_requests=941375physical=91 logical=81 role=AIV icache_requests=951073physical=92 logical=82 role=AIV icache_requests=926062physical=93 logical=83 role=AIV icache_requests=932726physical=94 logical=84 role=AIV icache_requests=943427physical=95 logical=85 role=AIV icache_requests=937704physical=96 logical=86 role=AIV icache_requests=940159physical=97 logical=87 role=AIV icache_requests=930903physical=98 logical=88 role=AIV icache_requests=931739physical=99 logical=89 role=AIV icache_requests=918872physical=100 logical=90 role=AIV icache_requests=920514physical=101 logical=91 role=AIV icache_requests=946165physical=102 logical=92 role=AIV icache_requests=927905physical=103 logical=93 role=AIV icache_requests=922314physical=104 logical=94 role=AIV icache_requests=931010physical=105 logical=95 role=AIV icache_requests=927410 + +
+ +
+
Primary I-cache misses/core
+ + + + 54,306 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_misses=4733physical=1 logical=1 role=AIC icache_misses=5209physical=2 logical=2 role=AIC icache_misses=4815physical=3 logical=3 role=AIC icache_misses=5163physical=4 logical=4 role=AIC icache_misses=4794physical=5 logical=5 role=AIC icache_misses=4552physical=6 logical=6 role=AIC icache_misses=5208physical=7 logical=7 role=AIC icache_misses=5018physical=8 logical=8 role=AIC icache_misses=5270physical=9 logical=9 role=AIC icache_misses=5437physical=10 logical=10 role=AIC icache_misses=4865physical=12 logical=11 role=AIC icache_misses=4961physical=13 logical=12 role=AIC icache_misses=5280physical=14 logical=13 role=AIC icache_misses=5217physical=15 logical=14 role=AIC icache_misses=4925physical=16 logical=15 role=AIC icache_misses=5084physical=54 logical=16 role=AIC icache_misses=5232physical=55 logical=17 role=AIC icache_misses=4815physical=56 logical=18 role=AIC icache_misses=4870physical=57 logical=19 role=AIC icache_misses=5111physical=58 logical=20 role=AIC icache_misses=5217physical=60 logical=21 role=AIC icache_misses=4933physical=61 logical=22 role=AIC icache_misses=5209physical=62 logical=23 role=AIC icache_misses=4935physical=63 logical=24 role=AIC icache_misses=5065physical=64 logical=25 role=AIC icache_misses=4871physical=65 logical=26 role=AIC icache_misses=4648physical=66 logical=27 role=AIC icache_misses=5237physical=67 logical=28 role=AIC icache_misses=5226physical=68 logical=29 role=AIC icache_misses=4911physical=69 logical=30 role=AIC icache_misses=5037physical=70 logical=31 role=AIC icache_misses=5011physical=18 logical=32 role=AIV icache_misses=51385physical=19 logical=33 role=AIV icache_misses=50950physical=20 logical=34 role=AIV icache_misses=52838physical=21 logical=35 role=AIV icache_misses=53132physical=22 logical=36 role=AIV icache_misses=52343physical=23 logical=37 role=AIV icache_misses=52098physical=24 logical=38 role=AIV icache_misses=52555physical=25 logical=39 role=AIV icache_misses=52005physical=26 logical=40 role=AIV icache_misses=52705physical=27 logical=41 role=AIV icache_misses=52302physical=28 logical=42 role=AIV icache_misses=52274physical=29 logical=43 role=AIV icache_misses=52568physical=30 logical=44 role=AIV icache_misses=52710physical=31 logical=45 role=AIV icache_misses=52757physical=32 logical=46 role=AIV icache_misses=52215physical=33 logical=47 role=AIV icache_misses=52085physical=34 logical=48 role=AIV icache_misses=51301physical=35 logical=49 role=AIV icache_misses=53143physical=36 logical=50 role=AIV icache_misses=52033physical=37 logical=51 role=AIV icache_misses=52263physical=38 logical=52 role=AIV icache_misses=52880physical=39 logical=53 role=AIV icache_misses=51741physical=42 logical=54 role=AIV icache_misses=52390physical=43 logical=55 role=AIV icache_misses=52222physical=44 logical=56 role=AIV icache_misses=53647physical=45 logical=57 role=AIV icache_misses=53139physical=46 logical=58 role=AIV icache_misses=52699physical=47 logical=59 role=AIV icache_misses=52419physical=48 logical=60 role=AIV icache_misses=51795physical=49 logical=61 role=AIV icache_misses=53432physical=50 logical=62 role=AIV icache_misses=53060physical=51 logical=63 role=AIV icache_misses=52909physical=72 logical=64 role=AIV icache_misses=53751physical=73 logical=65 role=AIV icache_misses=54306physical=74 logical=66 role=AIV icache_misses=52319physical=75 logical=67 role=AIV icache_misses=53601physical=76 logical=68 role=AIV icache_misses=53603physical=77 logical=69 role=AIV icache_misses=52940physical=78 logical=70 role=AIV icache_misses=52519physical=79 logical=71 role=AIV icache_misses=52535physical=80 logical=72 role=AIV icache_misses=52623physical=81 logical=73 role=AIV icache_misses=54111physical=84 logical=74 role=AIV icache_misses=51843physical=85 logical=75 role=AIV icache_misses=52774physical=86 logical=76 role=AIV icache_misses=52527physical=87 logical=77 role=AIV icache_misses=52003physical=88 logical=78 role=AIV icache_misses=52215physical=89 logical=79 role=AIV icache_misses=52129physical=90 logical=80 role=AIV icache_misses=52796physical=91 logical=81 role=AIV icache_misses=53656physical=92 logical=82 role=AIV icache_misses=51976physical=93 logical=83 role=AIV icache_misses=52773physical=94 logical=84 role=AIV icache_misses=52820physical=95 logical=85 role=AIV icache_misses=52541physical=96 logical=86 role=AIV icache_misses=52649physical=97 logical=87 role=AIV icache_misses=52124physical=98 logical=88 role=AIV icache_misses=52584physical=99 logical=89 role=AIV icache_misses=51431physical=100 logical=90 role=AIV icache_misses=51462physical=101 logical=91 role=AIV icache_misses=52758physical=102 logical=92 role=AIV icache_misses=51971physical=103 logical=93 role=AIV icache_misses=51450physical=104 logical=94 role=AIV icache_misses=52321physical=105 logical=95 role=AIV icache_misses=52188 + +
+
+

逐核原始主计数

+

报告展示 primary request/miss；phase 模式的 I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和，只用于标记全窗 capture gap；scalar shadow 同样用于核验 phase 读清重建，并显示其相对 CNT2 primary 的 capture loss，不作为第二份性能数据展示。

+
+ + + + + + + +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate90ns量尺 µs
00AIC004,228,8425,308.6247,710,6077,549,235930,4194,7330.509%425.970
11AIC104,244,0815,260.0827,752,1147,587,347936,1035,2090.556%468.810
22AIC204,267,7055,242.5577,779,8037,605,796936,9494,8150.514%433.350
33AIC304,192,8125,238.7337,645,1647,484,292933,8585,1630.553%464.670
44AIC404,252,0315,239.2827,775,3697,596,190933,4524,7940.514%431.460
55AIC504,228,5015,259.7107,863,5097,700,179959,2884,5520.475%409.680
66AIC604,227,2155,258.7257,714,3027,535,282929,4175,2080.560%468.720
77AIC704,272,7165,452.7707,910,8207,758,719969,3165,0180.518%451.620
88AIC804,219,5055,269.3397,703,8447,528,576932,3995,2700.565%474.300
99AIC904,222,9725,286.4207,752,7357,583,332950,2725,4370.572%489.330
1010AIC1004,234,7095,260.2237,695,1287,526,121927,4494,8650.525%437.850
1211AIC1104,256,2875,266.8367,727,1957,560,893932,3714,9610.532%446.490
1312AIC1204,381,4375,577.7598,187,3048,028,194993,0485,2800.532%475.200
1413AIC1304,229,8515,243.9097,809,3217,628,584950,0975,2170.549%469.530
1514AIC1404,383,1255,483.4638,175,6598,025,670986,1384,9250.499%443.250
1615AIC1504,228,8355,231.7587,771,2207,595,654939,4335,0840.541%457.560
1832AIV015,137,3525,736.6699,099,6187,751,544927,92251,3855.538%4,624.650
1933AIV025,136,1395,730.4479,095,4617,676,164916,78750,9505.557%4,585.500
2034AIV115,134,6495,774.2699,156,3647,847,701937,05852,8385.639%4,755.420
2135AIV125,127,6575,772.7669,151,8407,752,896944,77353,1325.624%4,781.880
2236AIV215,168,6005,757.7789,219,4767,884,060937,22152,3435.585%4,710.870
2337AIV225,129,5355,766.5249,148,0417,745,963933,54052,0985.581%4,688.820
2438AIV315,172,7425,728.4689,255,1847,940,649939,08452,5555.596%4,729.950
2539AIV325,076,2005,762.6819,058,5747,689,527931,87152,0055.581%4,680.450
2640AIV415,141,6395,727.0559,167,2357,851,515936,69852,7055.627%4,743.450
2741AIV425,138,4775,726.6789,161,4637,748,382934,69352,3025.596%4,707.180
2842AIV515,121,0875,755.0359,136,4367,829,963933,29552,2745.601%4,704.660
2943AIV525,096,6885,743.1689,116,6077,714,117932,12352,5685.640%4,731.120
3044AIV615,142,6685,737.4439,187,7677,881,034938,91352,7105.614%4,743.900
3145AIV625,102,9145,782.9619,089,8937,755,346935,12052,7575.642%4,748.130
3246AIV715,047,6385,733.8738,999,9287,780,601929,63852,2155.617%4,699.350
3347AIV725,060,5545,764.6908,974,4087,651,763927,04952,0855.618%4,687.650
3448AIV815,155,4945,722.8799,174,9557,881,403916,53251,3015.597%4,617.090
3549AIV825,164,6355,731.5489,261,5647,905,627947,63953,1435.608%4,782.870
3650AIV915,058,0885,733.2469,008,4637,753,573930,18152,0335.594%4,682.970
3751AIV925,098,2165,735.7219,095,7617,798,835934,14952,2635.595%4,703.670
3852AIV1015,121,9335,755.7889,130,6487,845,288934,95152,8805.656%4,759.200
3953AIV1025,091,4035,746.2179,030,9837,660,674919,20451,7415.629%4,656.690
4254AIV1115,052,5095,722.9508,978,3997,716,252929,20252,3905.638%4,715.100
4355AIV1125,105,9025,724.4739,076,7107,758,525926,70452,2225.635%4,699.980
4456AIV1215,121,4445,734.9749,169,8447,973,806953,30253,6475.627%4,828.230
4557AIV1225,084,0805,777.9999,075,0867,762,731936,90753,1395.672%4,782.510
4658AIV1315,019,3475,743.5168,933,7717,640,942929,23752,6995.671%4,742.910
4759AIV1325,129,9125,734.6449,093,2997,740,317928,56552,4195.645%4,717.710
4860AIV1414,993,0115,751.8788,866,5877,682,902916,96951,7955.649%4,661.550
4961AIV1425,110,5835,765.3589,138,3917,855,936944,94953,4325.654%4,808.880
5062AIV1515,142,5785,787.9519,180,5967,869,891935,64553,0605.671%4,775.400
5163AIV1525,175,3795,728.9539,258,9877,895,020934,87252,9095.659%4,761.810
5416AIC1604,235,4375,277.3037,687,5147,540,031927,3405,2320.564%470.880
5517AIC1704,223,0885,290.1517,786,1757,630,468943,5734,8150.510%433.350
5618AIC1804,247,5865,282.0107,767,4127,624,175933,2984,8700.522%438.300
5719AIC1904,228,2825,254.7837,773,0487,605,601944,8205,1110.541%459.990
5820AIC2004,206,3415,270.2867,723,5237,568,816939,1005,2170.556%469.530
6021AIC2104,251,0335,262.1787,915,3457,755,698957,6154,9330.515%443.970
6122AIC2204,268,8645,238.6327,933,3457,756,779957,7465,2090.544%468.810
6223AIC2304,282,6145,260.1887,880,7417,716,679952,6194,9350.518%444.150
6324AIC2404,239,5745,273.4977,837,0927,672,389961,0955,0650.527%455.850
6425AIC2504,218,4835,234.8437,744,5707,598,727943,2604,8710.516%438.390
6526AIC2604,397,6335,534.6698,316,9588,150,0321,002,9564,6480.463%418.320
6627AIC2704,230,5625,268.6667,697,4247,517,274935,1035,2370.560%471.330
6728AIC2804,209,9615,288.5697,781,4257,587,671950,0045,2260.550%470.340
6829AIC2904,250,4475,266.7967,795,7837,635,677942,7254,9110.521%441.990
6930AIC3004,264,2795,266.7107,846,3267,663,986951,1445,0370.530%453.330
7031AIC3104,284,9495,244.5767,846,0077,680,795945,2955,0110.530%450.990
7264AIV1615,087,1655,743.8379,108,3017,862,110945,57553,7515.684%4,837.590
7365AIV1625,143,3715,731.0459,257,2687,918,939958,46754,3065.666%4,887.540
7466AIV1715,120,1575,741.1709,112,7067,838,763923,82552,3195.663%4,708.710
7567AIV1725,139,2395,734.7369,181,7967,833,799943,75853,6015.680%4,824.090
7668AIV1815,161,8175,737.3449,272,6798,004,867946,30953,6035.664%4,824.270
7769AIV1825,085,9395,730.5109,089,4677,764,365936,26852,9405.654%4,764.600
7870AIV1915,104,2755,728.7339,090,6087,849,695928,16052,5195.658%4,726.710
7971AIV1925,091,7595,722.4559,082,5567,738,892928,80752,5355.656%4,728.150
8072AIV2015,043,0195,736.7189,014,2967,793,842932,74552,6235.642%4,736.070
8173AIV2025,094,7205,724.9819,159,5007,834,716959,30754,1115.641%4,869.990
8474AIV2115,078,4865,734.0209,010,9717,778,708924,34851,8435.609%4,665.870
8575AIV2125,179,7595,721.7439,246,0247,935,263945,35852,7745.582%4,749.660
8676AIV2215,128,5445,753.9899,132,7637,877,434928,63152,5275.656%4,727.430
8777AIV2225,085,2095,775.7218,993,0517,596,886922,11352,0035.640%4,680.270
8878AIV2315,092,6025,739.4389,023,3867,765,105929,29552,2155.619%4,699.350
8979AIV2325,041,1615,757.8428,966,3417,666,986926,27652,1295.628%4,691.610
9080AIV2415,106,0595,742.5649,104,6637,824,935941,37552,7965.608%4,751.640
9181AIV2425,148,6395,730.2069,254,5447,885,387951,07353,6565.642%4,829.040
9282AIV2515,121,5235,771.9189,074,8207,818,299926,06251,9765.613%4,677.840
9383AIV2525,093,1125,712.5169,060,9317,755,185932,72652,7735.658%4,749.570
9484AIV2615,137,6945,729.4179,171,3677,866,090943,42752,8205.599%4,753.800
9585AIV2625,132,5875,721.4699,160,3917,824,191937,70452,5415.603%4,728.690
9686AIV2715,108,1585,745.1869,109,4957,808,917940,15952,6495.600%4,738.410
9787AIV2725,077,1095,758.2489,048,7037,652,254930,90352,1245.599%4,691.160
9888AIV2815,086,1905,721.6209,070,4747,699,840931,73952,5845.644%4,732.560
9989AIV2825,244,0335,750.8789,302,6567,856,977918,87251,4315.597%4,628.790
10090AIV2915,047,6085,742.1988,939,1137,684,561920,51451,4625.591%4,631.580
10191AIV2925,193,4485,726.8369,255,2867,887,384946,16552,7585.576%4,748.220
10292AIV3015,110,4155,737.2119,097,4237,744,565927,90551,9715.601%4,677.390
10393AIV3025,127,7795,721.2979,077,3507,658,784922,31451,4505.578%4,630.500
10494AIV3115,041,1335,770.7108,978,6967,694,617931,01052,3215.620%4,708.890
10595AIV3125,085,9435,752.3619,039,5357,716,540927,41052,1885.627%4,696.920
+
+ diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_submit_none_20260723_043811_icache_report.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_submit_none_20260723_043811_icache_report.html new file mode 100644 index 0000000000..73c674a1bc --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_submit_none_20260723_043811_icache_report.html @@ -0,0 +1,315 @@ + + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 fdwic_submit_pmu_raw.json · schema fdwic-submit-pmu-v3 · 32 AIC + 64 AIV

+ + +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
e4f66742cf9ac6726140674056fa9bc762d9ec7f1426a0b0c080c43b2328acf1
+
Profile / extra cache
submit-pmu-none / + aa43623282e2a7db
+
Profiled cache key
TestPagedAttentionUnroll / a5 / fully_distributed_within_core / submit-pmu-none
+
Source state
source-v2:8c6b6e3a412d2c7c7113c2d8f714a14eb3786f80:9254bef935704fc2249608cff008704bea9b82cde621fad345be473b84219a40:c42f14d7f4475d3ea45883e7064619dfce333ac6fd119269c2b5da73bd47096b
+
Source-state stamp
/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/aa43623282e2a7db/aicore/.git_commit
+
Compile definitions
PTO_FDWIC_SUBMIT_PMU=1
PTO_FDWIC_TRACE_ENABLED=0
+
+
+ + + +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
AICore final2,581,120ab25886c73dbb2c21f8e45f1d64405737bb51befb63fe58d5864b028badee098188,75223eb813bfcb677e08715f81cf6075352be4d98fb88cd8163435c0a89cce769dc/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/aicore-extra/aa43623282e2a7db/aicore_kernel.o
AIC combined1,891,67252f6cc5d916278a124613c9274fa00aa710c698cffe0cdeb1016f15eb514fad387,120ad3d2e4d56722c75329d421f77cf9e4f774c3e0fde1c1a715dd6cb671972b1c8/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/aa43623282e2a7db/aicore/aicore_aic_combined.o
AIV combined2,152,59272708c5ef4e70cc5516e736518b85392488e04700fbc88b592a6357ccf69ba11101,456a4a8fb8cfbcc94694b9d5e5fd235977e8ace0a728cd0645d5ddbc3bf780b4d06/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/aa43623282e2a7db/aicore/aicore_aiv_combined.o
Host runtime11,776,1440369d0602054bf612f86b58c294503e00486bd505bff9ee7a455994f043147cc454,6325ac381ed10be5f5d492b8072c051a177131f4caec6d6f6bc9bf50880da4bb0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so
+
+ +
+
原始全局 Submit 时间范围（墙钟） + 6,812.094 µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + 128096 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-2567b058c2e4ed499bf29859a1466131de7cda2eb7163110175e07ff19b4bfc7cae
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + 90.000 ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×90 ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
+
+

ALL · 96 核

+
+
Submit PMU total/core
+
均值 9,709,219.9；最小 8,929,198；最大 10,758,901 cycles
+ 等效时间 均值 5,884.932；最小 5,412.147；最大 6,521.163 µs + （按 1.649844 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 9,459,283.6；最小 8,590,434；最大 10,377,067 cycles
+ 等效时间 均值 5,733.441；最小 5,206.816；最大 6,289.726 µs； + 加权占比 97.426% +
+
非 Scalar-busy 残余/core
+
均值 249,936.2；最小 37,864；最大 455,825 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 151.491；最小 22.950；最大 276.284 µs +
+
SYS gate 边界诊断/core
+
均值 2,938,755.9；最小 2,381,818；最大 4,431,591 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 6,232.477；最小 5,791.076；最大 6,800.652 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 3,293,720.8；最小 1,692,860；最大 4,057,856 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 47.152% +
+
Primary I-cache request/core
最小 627,031；最大 919,317
+
Primary I-cache miss/core
最小 573；最大 13,700
+
加权 miss rate
1.131%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 51.570；最大 1,233.000 µs
+
+
+ +
+

AIC · 32 核

+
+
Submit PMU total/core
+
均值 9,476,082.2；最小 9,128,939；最大 9,952,568 cycles
+ 等效时间 均值 5,742.864；最小 5,532.482；最大 6,031.633 µs + （按 1.650062 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 9,434,146.2；最小 9,088,377；最大 9,913,392 cycles
+ 等效时间 均值 5,717.450；最小 5,507.900；最大 6,007.891 µs； + 加权占比 99.557% +
+
非 Scalar-busy 残余/core
+
均值 41,936.0；最小 37,864；最大 47,947 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 25.415；最小 22.947；最大 29.058 µs +
+
SYS gate 边界诊断/core
+
均值 2,654,539.3；最小 2,381,818；最大 3,331,744 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 6,345.896；最小 6,211.970；最大 6,591.535 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 3,691,357.1；最小 2,899,144；最大 4,057,856 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 41.831% +
+
Primary I-cache request/core
最小 662,570；最大 765,234
+
Primary I-cache miss/core
最小 573；最大 900
+
加权 miss rate
0.102%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 51.570；最大 81.000 µs
+
+
+ +
+

AIV · 64 核

+
+
Submit PMU total/core
+
均值 9,825,788.7；最小 8,929,198；最大 10,758,901 cycles
+ 等效时间 均值 5,955.994；最小 5,412.518；最大 6,521.609 µs + （按 1.649731 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 9,471,852.3；最小 8,590,434；最大 10,377,067 cycles
+ 等效时间 均值 5,741.453；最小 5,207.173；最大 6,290.157 µs； + 加权占比 96.398% +
+
非 Scalar-busy 残余/core
+
均值 353,936.3；最小 277,440；最大 455,825 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 214.542；最小 168.173；最大 276.303 µs +
+
SYS gate 边界诊断/core
+
均值 3,080,864.3；最小 2,656,417；最大 4,431,591 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 6,175.767；最小 5,791.076；最大 6,800.652 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 3,094,902.7；最小 1,692,860；最大 3,645,323 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 49.886% +
+
Primary I-cache request/core
最小 627,031；最大 919,317
+
Primary I-cache miss/core
最小 9,639；最大 13,700
+
加权 miss rate
1.627%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 867.510；最大 1,233.000 µs
+
+
+
+

逐物理核分布

+

AICAIV

+
+
+
SYS gate boundary-diagnostic ticks/core
+ + + + 4,431,591 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_submit_elapsed_ticks=2692971physical=1 logical=1 role=AIC scalar_submit_elapsed_ticks=2487693physical=2 logical=2 role=AIC scalar_submit_elapsed_ticks=2684231physical=3 logical=3 role=AIC scalar_submit_elapsed_ticks=2532570physical=4 logical=4 role=AIC scalar_submit_elapsed_ticks=2624514physical=5 logical=5 role=AIC scalar_submit_elapsed_ticks=2381818physical=6 logical=6 role=AIC scalar_submit_elapsed_ticks=2718223physical=7 logical=7 role=AIC scalar_submit_elapsed_ticks=2541198physical=8 logical=8 role=AIC scalar_submit_elapsed_ticks=2756596physical=9 logical=9 role=AIC scalar_submit_elapsed_ticks=2465805physical=10 logical=10 role=AIC scalar_submit_elapsed_ticks=2744674physical=12 logical=11 role=AIC scalar_submit_elapsed_ticks=3331744physical=13 logical=12 role=AIC scalar_submit_elapsed_ticks=2792106physical=14 logical=13 role=AIC scalar_submit_elapsed_ticks=2457059physical=15 logical=14 role=AIC scalar_submit_elapsed_ticks=2854452physical=16 logical=15 role=AIC scalar_submit_elapsed_ticks=2405534physical=54 logical=16 role=AIC scalar_submit_elapsed_ticks=2865468physical=55 logical=17 role=AIC scalar_submit_elapsed_ticks=2710767physical=56 logical=18 role=AIC scalar_submit_elapsed_ticks=3036364physical=57 logical=19 role=AIC scalar_submit_elapsed_ticks=2448663physical=58 logical=20 role=AIC scalar_submit_elapsed_ticks=2607695physical=60 logical=21 role=AIC scalar_submit_elapsed_ticks=2559630physical=61 logical=22 role=AIC scalar_submit_elapsed_ticks=2684257physical=62 logical=23 role=AIC scalar_submit_elapsed_ticks=2514464physical=63 logical=24 role=AIC scalar_submit_elapsed_ticks=2636555physical=64 logical=25 role=AIC scalar_submit_elapsed_ticks=2539292physical=65 logical=26 role=AIC scalar_submit_elapsed_ticks=2835262physical=66 logical=27 role=AIC scalar_submit_elapsed_ticks=2512190physical=67 logical=28 role=AIC scalar_submit_elapsed_ticks=2603294physical=68 logical=29 role=AIC scalar_submit_elapsed_ticks=2481394physical=69 logical=30 role=AIC scalar_submit_elapsed_ticks=2812390physical=70 logical=31 role=AIC scalar_submit_elapsed_ticks=2626384physical=18 logical=32 role=AIV scalar_submit_elapsed_ticks=3298991physical=19 logical=33 role=AIV scalar_submit_elapsed_ticks=3508102physical=20 logical=34 role=AIV scalar_submit_elapsed_ticks=2879754physical=21 logical=35 role=AIV scalar_submit_elapsed_ticks=3155329physical=22 logical=36 role=AIV scalar_submit_elapsed_ticks=2979681physical=23 logical=37 role=AIV scalar_submit_elapsed_ticks=2882055physical=24 logical=38 role=AIV scalar_submit_elapsed_ticks=3135947physical=25 logical=39 role=AIV scalar_submit_elapsed_ticks=2711800physical=26 logical=40 role=AIV scalar_submit_elapsed_ticks=2936228physical=27 logical=41 role=AIV scalar_submit_elapsed_ticks=3120082physical=28 logical=42 role=AIV scalar_submit_elapsed_ticks=3306871physical=29 logical=43 role=AIV scalar_submit_elapsed_ticks=3048651physical=30 logical=44 role=AIV scalar_submit_elapsed_ticks=3361370physical=31 logical=45 role=AIV scalar_submit_elapsed_ticks=3074571physical=32 logical=46 role=AIV scalar_submit_elapsed_ticks=2813320physical=33 logical=47 role=AIV scalar_submit_elapsed_ticks=3145467physical=34 logical=48 role=AIV scalar_submit_elapsed_ticks=3943454physical=35 logical=49 role=AIV scalar_submit_elapsed_ticks=2914998physical=36 logical=50 role=AIV scalar_submit_elapsed_ticks=2839116physical=37 logical=51 role=AIV scalar_submit_elapsed_ticks=2897468physical=38 logical=52 role=AIV scalar_submit_elapsed_ticks=3330220physical=39 logical=53 role=AIV scalar_submit_elapsed_ticks=2960131physical=42 logical=54 role=AIV scalar_submit_elapsed_ticks=3076624physical=43 logical=55 role=AIV scalar_submit_elapsed_ticks=2884805physical=44 logical=56 role=AIV scalar_submit_elapsed_ticks=3151439physical=45 logical=57 role=AIV scalar_submit_elapsed_ticks=2921213physical=46 logical=58 role=AIV scalar_submit_elapsed_ticks=3043601physical=47 logical=59 role=AIV scalar_submit_elapsed_ticks=3148051physical=48 logical=60 role=AIV scalar_submit_elapsed_ticks=2890407physical=49 logical=61 role=AIV scalar_submit_elapsed_ticks=2955635physical=50 logical=62 role=AIV scalar_submit_elapsed_ticks=3259038physical=51 logical=63 role=AIV scalar_submit_elapsed_ticks=3010829physical=72 logical=64 role=AIV scalar_submit_elapsed_ticks=2885749physical=73 logical=65 role=AIV scalar_submit_elapsed_ticks=2951904physical=74 logical=66 role=AIV scalar_submit_elapsed_ticks=2850007physical=75 logical=67 role=AIV scalar_submit_elapsed_ticks=2884463physical=76 logical=68 role=AIV scalar_submit_elapsed_ticks=3189880physical=77 logical=69 role=AIV scalar_submit_elapsed_ticks=2931776physical=78 logical=70 role=AIV scalar_submit_elapsed_ticks=3276336physical=79 logical=71 role=AIV scalar_submit_elapsed_ticks=3027019physical=80 logical=72 role=AIV scalar_submit_elapsed_ticks=3069187physical=81 logical=73 role=AIV scalar_submit_elapsed_ticks=2814843physical=84 logical=74 role=AIV scalar_submit_elapsed_ticks=3299633physical=85 logical=75 role=AIV scalar_submit_elapsed_ticks=4010218physical=86 logical=76 role=AIV scalar_submit_elapsed_ticks=3175294physical=87 logical=77 role=AIV scalar_submit_elapsed_ticks=2865627physical=88 logical=78 role=AIV scalar_submit_elapsed_ticks=2996600physical=89 logical=79 role=AIV scalar_submit_elapsed_ticks=3028081physical=90 logical=80 role=AIV scalar_submit_elapsed_ticks=3500545physical=91 logical=81 role=AIV scalar_submit_elapsed_ticks=2717193physical=92 logical=82 role=AIV scalar_submit_elapsed_ticks=3545805physical=93 logical=83 role=AIV scalar_submit_elapsed_ticks=2920874physical=94 logical=84 role=AIV scalar_submit_elapsed_ticks=3196690physical=95 logical=85 role=AIV scalar_submit_elapsed_ticks=2752625physical=96 logical=86 role=AIV scalar_submit_elapsed_ticks=3092943physical=97 logical=87 role=AIV scalar_submit_elapsed_ticks=3046919physical=98 logical=88 role=AIV scalar_submit_elapsed_ticks=2807564physical=99 logical=89 role=AIV scalar_submit_elapsed_ticks=4431591physical=100 logical=90 role=AIV scalar_submit_elapsed_ticks=2764669physical=101 logical=91 role=AIV scalar_submit_elapsed_ticks=2756061physical=102 logical=92 role=AIV scalar_submit_elapsed_ticks=3287997physical=103 logical=93 role=AIV scalar_submit_elapsed_ticks=3004145physical=104 logical=94 role=AIV scalar_submit_elapsed_ticks=2851410physical=105 logical=95 role=AIV scalar_submit_elapsed_ticks=2656417 + +
+ +
+
PMU total cycles/core
+ + + + 10,758,901 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC total_cycles=9497981physical=1 logical=1 role=AIC total_cycles=9403692physical=2 logical=2 role=AIC total_cycles=9385667physical=3 logical=3 role=AIC total_cycles=9467823physical=4 logical=4 role=AIC total_cycles=9440910physical=5 logical=5 role=AIC total_cycles=9675573physical=6 logical=6 role=AIC total_cycles=9245153physical=7 logical=7 role=AIC total_cycles=9435108physical=8 logical=8 role=AIC total_cycles=9217848physical=9 logical=9 role=AIC total_cycles=9348458physical=10 logical=10 role=AIC total_cycles=9425558physical=12 logical=11 role=AIC total_cycles=9481619physical=13 logical=12 role=AIC total_cycles=9694554physical=14 logical=13 role=AIC total_cycles=9759367physical=15 logical=14 role=AIC total_cycles=9341940physical=16 logical=15 role=AIC total_cycles=9441016physical=54 logical=16 role=AIC total_cycles=9742953physical=55 logical=17 role=AIC total_cycles=9561628physical=56 logical=18 role=AIC total_cycles=9128939physical=57 logical=19 role=AIC total_cycles=9512845physical=58 logical=20 role=AIC total_cycles=9264681physical=60 logical=21 role=AIC total_cycles=9372683physical=61 logical=22 role=AIC total_cycles=9248116physical=62 logical=23 role=AIC total_cycles=9292891physical=63 logical=24 role=AIC total_cycles=9734945physical=64 logical=25 role=AIC total_cycles=9757366physical=65 logical=26 role=AIC total_cycles=9952568physical=66 logical=27 role=AIC total_cycles=9436017physical=67 logical=28 role=AIC total_cycles=9350591physical=68 logical=29 role=AIC total_cycles=9543393physical=69 logical=30 role=AIC total_cycles=9758921physical=70 logical=31 role=AIC total_cycles=9313827physical=18 logical=32 role=AIV total_cycles=10492083physical=19 logical=33 role=AIV total_cycles=9894632physical=20 logical=34 role=AIV total_cycles=9996170physical=21 logical=35 role=AIV total_cycles=10758901physical=22 logical=36 role=AIV total_cycles=9203529physical=23 logical=37 role=AIV total_cycles=9214662physical=24 logical=38 role=AIV total_cycles=9708872physical=25 logical=39 role=AIV total_cycles=9683394physical=26 logical=40 role=AIV total_cycles=10009704physical=27 logical=41 role=AIV total_cycles=9969738physical=28 logical=42 role=AIV total_cycles=10427934physical=29 logical=43 role=AIV total_cycles=9496529physical=30 logical=44 role=AIV total_cycles=9200385physical=31 logical=45 role=AIV total_cycles=10522989physical=32 logical=46 role=AIV total_cycles=9602507physical=33 logical=47 role=AIV total_cycles=9611323physical=34 logical=48 role=AIV total_cycles=9945023physical=35 logical=49 role=AIV total_cycles=9600069physical=36 logical=50 role=AIV total_cycles=9610520physical=37 logical=51 role=AIV total_cycles=9559141physical=38 logical=52 role=AIV total_cycles=10455671physical=39 logical=53 role=AIV total_cycles=9507741physical=42 logical=54 role=AIV total_cycles=10520739physical=43 logical=55 role=AIV total_cycles=9993513physical=44 logical=56 role=AIV total_cycles=10064822physical=45 logical=57 role=AIV total_cycles=9700254physical=46 logical=58 role=AIV total_cycles=9265893physical=47 logical=59 role=AIV total_cycles=9348244physical=48 logical=60 role=AIV total_cycles=9907800physical=49 logical=61 role=AIV total_cycles=9942245physical=50 logical=62 role=AIV total_cycles=9756292physical=51 logical=63 role=AIV total_cycles=9509649physical=72 logical=64 role=AIV total_cycles=10073367physical=73 logical=65 role=AIV total_cycles=9949744physical=74 logical=66 role=AIV total_cycles=9279119physical=75 logical=67 role=AIV total_cycles=9872900physical=76 logical=68 role=AIV total_cycles=9629279physical=77 logical=69 role=AIV total_cycles=10038864physical=78 logical=70 role=AIV total_cycles=10078159physical=79 logical=71 role=AIV total_cycles=10424250physical=80 logical=72 role=AIV total_cycles=10354297physical=81 logical=73 role=AIV total_cycles=9719133physical=84 logical=74 role=AIV total_cycles=10109854physical=85 logical=75 role=AIV total_cycles=9632758physical=86 logical=76 role=AIV total_cycles=9570882physical=87 logical=77 role=AIV total_cycles=9781043physical=88 logical=78 role=AIV total_cycles=9570406physical=89 logical=79 role=AIV total_cycles=8929198physical=90 logical=80 role=AIV total_cycles=9813626physical=91 logical=81 role=AIV total_cycles=10003175physical=92 logical=82 role=AIV total_cycles=9629987physical=93 logical=83 role=AIV total_cycles=10050785physical=94 logical=84 role=AIV total_cycles=10277338physical=95 logical=85 role=AIV total_cycles=9819905physical=96 logical=86 role=AIV total_cycles=9831138physical=97 logical=87 role=AIV total_cycles=9492773physical=98 logical=88 role=AIV total_cycles=9041193physical=99 logical=89 role=AIV total_cycles=9917587physical=100 logical=90 role=AIV total_cycles=9696179physical=101 logical=91 role=AIV total_cycles=9674855physical=102 logical=92 role=AIV total_cycles=9605833physical=103 logical=93 role=AIV total_cycles=10239452physical=104 logical=94 role=AIV total_cycles=10332565physical=105 logical=95 role=AIV total_cycles=9929863 + +
+ +
+
Scalar busy cycles/core
+ + + + 10,377,067 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_busy=9457314physical=1 logical=1 role=AIC scalar_busy=9360947physical=2 logical=2 role=AIC scalar_busy=9342914physical=3 logical=3 role=AIC scalar_busy=9421102physical=4 logical=4 role=AIC scalar_busy=9398279physical=5 logical=5 role=AIC scalar_busy=9627626physical=6 logical=6 role=AIC scalar_busy=9206475physical=7 logical=7 role=AIC scalar_busy=9394082physical=8 logical=8 role=AIC scalar_busy=9177679physical=9 logical=9 role=AIC scalar_busy=9305606physical=10 logical=10 role=AIC scalar_busy=9387694physical=12 logical=11 role=AIC scalar_busy=9442263physical=13 logical=12 role=AIC scalar_busy=9653338physical=14 logical=13 role=AIC scalar_busy=9718995physical=15 logical=14 role=AIC scalar_busy=9302251physical=16 logical=15 role=AIC scalar_busy=9395703physical=54 logical=16 role=AIC scalar_busy=9703910physical=55 logical=17 role=AIC scalar_busy=9517188physical=56 logical=18 role=AIC scalar_busy=9088377physical=57 logical=19 role=AIC scalar_busy=9471907physical=58 logical=20 role=AIC scalar_busy=9218440physical=60 logical=21 role=AIC scalar_busy=9329383physical=61 logical=22 role=AIC scalar_busy=9205237physical=62 logical=23 role=AIC scalar_busy=9250290physical=63 logical=24 role=AIC scalar_busy=9694325physical=64 logical=25 role=AIC scalar_busy=9712010physical=65 logical=26 role=AIC scalar_busy=9913392physical=66 logical=27 role=AIC scalar_busy=9390782physical=67 logical=28 role=AIC scalar_busy=9311054physical=68 logical=29 role=AIC scalar_busy=9504170physical=69 logical=30 role=AIC scalar_busy=9717105physical=70 logical=31 role=AIC scalar_busy=9272840physical=18 logical=32 role=AIV scalar_busy=10181988physical=19 logical=33 role=AIV scalar_busy=9521072physical=20 logical=34 role=AIV scalar_busy=9636071physical=21 logical=35 role=AIV scalar_busy=10377067physical=22 logical=36 role=AIV scalar_busy=8781709physical=23 logical=37 role=AIV scalar_busy=8805815physical=24 logical=38 role=AIV scalar_busy=9331375physical=25 logical=39 role=AIV scalar_busy=9296840physical=26 logical=40 role=AIV scalar_busy=9610165physical=27 logical=41 role=AIV scalar_busy=9513913physical=28 logical=42 role=AIV scalar_busy=10093897physical=29 logical=43 role=AIV scalar_busy=9103673physical=30 logical=44 role=AIV scalar_busy=8822208physical=31 logical=45 role=AIV scalar_busy=10141790physical=32 logical=46 role=AIV scalar_busy=9276422physical=33 logical=47 role=AIV scalar_busy=9292337physical=34 logical=48 role=AIV scalar_busy=9621464physical=35 logical=49 role=AIV scalar_busy=9183971physical=36 logical=50 role=AIV scalar_busy=9275593physical=37 logical=51 role=AIV scalar_busy=9224564physical=38 logical=52 role=AIV scalar_busy=10098804physical=39 logical=53 role=AIV scalar_busy=9075857physical=42 logical=54 role=AIV scalar_busy=10197527physical=43 logical=55 role=AIV scalar_busy=9620664physical=44 logical=56 role=AIV scalar_busy=9695660physical=45 logical=57 role=AIV scalar_busy=9331122physical=46 logical=58 role=AIV scalar_busy=8829986physical=47 logical=59 role=AIV scalar_busy=8955848physical=48 logical=60 role=AIV scalar_busy=9560833physical=49 logical=61 role=AIV scalar_busy=9617891physical=50 logical=62 role=AIV scalar_busy=9371503physical=51 logical=63 role=AIV scalar_busy=9076541physical=72 logical=64 role=AIV scalar_busy=9724048physical=73 logical=65 role=AIV scalar_busy=9594307physical=74 logical=66 role=AIV scalar_busy=8915192physical=75 logical=67 role=AIV scalar_busy=9488403physical=76 logical=68 role=AIV scalar_busy=9316680physical=77 logical=69 role=AIV scalar_busy=9713695physical=78 logical=70 role=AIV scalar_busy=9716206physical=79 logical=71 role=AIV scalar_busy=10057630physical=80 logical=72 role=AIV scalar_busy=10059667physical=81 logical=73 role=AIV scalar_busy=9390266physical=84 logical=74 role=AIV scalar_busy=9800428physical=85 logical=75 role=AIV scalar_busy=9306468physical=86 logical=76 role=AIV scalar_busy=9249888physical=87 logical=77 role=AIV scalar_busy=9414898physical=88 logical=78 role=AIV scalar_busy=9279961physical=89 logical=79 role=AIV scalar_busy=8590434physical=90 logical=80 role=AIV scalar_busy=9474940physical=91 logical=81 role=AIV scalar_busy=9625926physical=92 logical=82 role=AIV scalar_busy=9351926physical=93 logical=83 role=AIV scalar_busy=9692833physical=94 logical=84 role=AIV scalar_busy=9927533physical=95 logical=85 role=AIV scalar_busy=9472586physical=96 logical=86 role=AIV scalar_busy=9520273physical=97 logical=87 role=AIV scalar_busy=9124935physical=98 logical=88 role=AIV scalar_busy=8707993physical=99 logical=89 role=AIV scalar_busy=9567020physical=100 logical=90 role=AIV scalar_busy=9418739physical=101 logical=91 role=AIV scalar_busy=9353243physical=102 logical=92 role=AIV scalar_busy=9316747physical=103 logical=93 role=AIV scalar_busy=9886972physical=104 logical=94 role=AIV scalar_busy=10013254physical=105 logical=95 role=AIV scalar_busy=9601289 + +
+ +
+
Primary I-cache requests/core
+ + + + 919,317 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_requests=691801physical=1 logical=1 role=AIC icache_requests=681329physical=2 logical=2 role=AIC icache_requests=684936physical=3 logical=3 role=AIC icache_requests=702293physical=4 logical=4 role=AIC icache_requests=689246physical=5 logical=5 role=AIC icache_requests=686583physical=6 logical=6 role=AIC icache_requests=716366physical=7 logical=7 role=AIC icache_requests=707734physical=8 logical=8 role=AIC icache_requests=716220physical=9 logical=9 role=AIC icache_requests=696610physical=10 logical=10 role=AIC icache_requests=683653physical=12 logical=11 role=AIC icache_requests=662570physical=13 logical=12 role=AIC icache_requests=744982physical=14 logical=13 role=AIC icache_requests=706725physical=15 logical=14 role=AIC icache_requests=706960physical=16 logical=15 role=AIC icache_requests=665199physical=54 logical=16 role=AIC icache_requests=759680physical=55 logical=17 role=AIC icache_requests=748451physical=56 logical=18 role=AIC icache_requests=699854physical=57 logical=19 role=AIC icache_requests=665299physical=58 logical=20 role=AIC icache_requests=680538physical=60 logical=21 role=AIC icache_requests=728899physical=61 logical=22 role=AIC icache_requests=703953physical=62 logical=23 role=AIC icache_requests=674091physical=63 logical=24 role=AIC icache_requests=692439physical=64 logical=25 role=AIC icache_requests=700403physical=65 logical=26 role=AIC icache_requests=765234physical=66 logical=27 role=AIC icache_requests=702524physical=67 logical=28 role=AIC icache_requests=690493physical=68 logical=29 role=AIC icache_requests=683595physical=69 logical=30 role=AIC icache_requests=741278physical=70 logical=31 role=AIC icache_requests=733275physical=18 logical=32 role=AIV icache_requests=740149physical=19 logical=33 role=AIV icache_requests=646612physical=20 logical=34 role=AIV icache_requests=675778physical=21 logical=35 role=AIV icache_requests=848167physical=22 logical=36 role=AIV icache_requests=698354physical=23 logical=37 role=AIV icache_requests=716434physical=24 logical=38 role=AIV icache_requests=671973physical=25 logical=39 role=AIV icache_requests=665321physical=26 logical=40 role=AIV icache_requests=679258physical=27 logical=41 role=AIV icache_requests=754625physical=28 logical=42 role=AIV icache_requests=830306physical=29 logical=43 role=AIV icache_requests=801324physical=30 logical=44 role=AIV icache_requests=640135physical=31 logical=45 role=AIV icache_requests=751849physical=32 logical=46 role=AIV icache_requests=665028physical=33 logical=47 role=AIV icache_requests=779926physical=34 logical=48 role=AIV icache_requests=627031physical=35 logical=49 role=AIV icache_requests=723274physical=36 logical=50 role=AIV icache_requests=660606physical=37 logical=51 role=AIV icache_requests=663938physical=38 logical=52 role=AIV icache_requests=829948physical=39 logical=53 role=AIV icache_requests=702920physical=42 logical=54 role=AIV icache_requests=747562physical=43 logical=55 role=AIV icache_requests=726213physical=44 logical=56 role=AIV icache_requests=735908physical=45 logical=57 role=AIV icache_requests=746209physical=46 logical=58 role=AIV icache_requests=746907physical=47 logical=59 role=AIV icache_requests=770018physical=48 logical=60 role=AIV icache_requests=663610physical=49 logical=61 role=AIV icache_requests=739939physical=50 logical=62 role=AIV icache_requests=719415physical=51 logical=63 role=AIV icache_requests=690533physical=72 logical=64 role=AIV icache_requests=695339physical=73 logical=65 role=AIV icache_requests=765234physical=74 logical=66 role=AIV icache_requests=662279physical=75 logical=67 role=AIV icache_requests=711675physical=76 logical=68 role=AIV icache_requests=781409physical=77 logical=69 role=AIV icache_requests=757096physical=78 logical=70 role=AIV icache_requests=793588physical=79 logical=71 role=AIV icache_requests=777176physical=80 logical=72 role=AIV icache_requests=764057physical=81 logical=73 role=AIV icache_requests=740126physical=84 logical=74 role=AIV icache_requests=748974physical=85 logical=75 role=AIV icache_requests=684272physical=86 logical=76 role=AIV icache_requests=771957physical=87 logical=77 role=AIV icache_requests=725894physical=88 logical=78 role=AIV icache_requests=667181physical=89 logical=79 role=AIV icache_requests=806011physical=90 logical=80 role=AIV icache_requests=919317physical=91 logical=81 role=AIV icache_requests=679879physical=92 logical=82 role=AIV icache_requests=754863physical=93 logical=83 role=AIV icache_requests=732046physical=94 logical=84 role=AIV icache_requests=785298physical=95 logical=85 role=AIV icache_requests=681687physical=96 logical=86 role=AIV icache_requests=754495physical=97 logical=87 role=AIV icache_requests=800831physical=98 logical=88 role=AIV icache_requests=684317physical=99 logical=89 role=AIV icache_requests=690110physical=100 logical=90 role=AIV icache_requests=648764physical=101 logical=91 role=AIV icache_requests=676400physical=102 logical=92 role=AIV icache_requests=830449physical=103 logical=93 role=AIV icache_requests=748179physical=104 logical=94 role=AIV icache_requests=691383physical=105 logical=95 role=AIV icache_requests=676343 + +
+ +
+
Primary I-cache misses/core
+ + + + 13,700 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_misses=658physical=1 logical=1 role=AIC icache_misses=854physical=2 logical=2 role=AIC icache_misses=712physical=3 logical=3 role=AIC icache_misses=900physical=4 logical=4 role=AIC icache_misses=770physical=5 logical=5 role=AIC icache_misses=711physical=6 logical=6 role=AIC icache_misses=573physical=7 logical=7 role=AIC icache_misses=692physical=8 logical=8 role=AIC icache_misses=682physical=9 logical=9 role=AIC icache_misses=796physical=10 logical=10 role=AIC icache_misses=635physical=12 logical=11 role=AIC icache_misses=598physical=13 logical=12 role=AIC icache_misses=749physical=14 logical=13 role=AIC icache_misses=708physical=15 logical=14 role=AIC icache_misses=703physical=16 logical=15 role=AIC icache_misses=808physical=54 logical=16 role=AIC icache_misses=689physical=55 logical=17 role=AIC icache_misses=776physical=56 logical=18 role=AIC icache_misses=741physical=57 logical=19 role=AIC icache_misses=723physical=58 logical=20 role=AIC icache_misses=738physical=60 logical=21 role=AIC icache_misses=791physical=61 logical=22 role=AIC icache_misses=663physical=62 logical=23 role=AIC icache_misses=771physical=63 logical=24 role=AIC icache_misses=727physical=64 logical=25 role=AIC icache_misses=818physical=65 logical=26 role=AIC icache_misses=654physical=66 logical=27 role=AIC icache_misses=685physical=67 logical=28 role=AIC icache_misses=672physical=68 logical=29 role=AIC icache_misses=635physical=69 logical=30 role=AIC icache_misses=688physical=70 logical=31 role=AIC icache_misses=739physical=18 logical=32 role=AIV icache_misses=11330physical=19 logical=33 role=AIV icache_misses=11199physical=20 logical=34 role=AIV icache_misses=11810physical=21 logical=35 role=AIV icache_misses=12095physical=22 logical=36 role=AIV icache_misses=12009physical=23 logical=37 role=AIV icache_misses=11396physical=24 logical=38 role=AIV icache_misses=12236physical=25 logical=39 role=AIV icache_misses=12057physical=26 logical=40 role=AIV icache_misses=12783physical=27 logical=41 role=AIV icache_misses=12217physical=28 logical=42 role=AIV icache_misses=11382physical=29 logical=43 role=AIV icache_misses=13700physical=30 logical=44 role=AIV icache_misses=10838physical=31 logical=45 role=AIV icache_misses=11748physical=32 logical=46 role=AIV icache_misses=11993physical=33 logical=47 role=AIV icache_misses=11717physical=34 logical=48 role=AIV icache_misses=9639physical=35 logical=49 role=AIV icache_misses=12077physical=36 logical=50 role=AIV icache_misses=11432physical=37 logical=51 role=AIV icache_misses=11229physical=38 logical=52 role=AIV icache_misses=12087physical=39 logical=53 role=AIV icache_misses=12618physical=42 logical=54 role=AIV icache_misses=12000physical=43 logical=55 role=AIV icache_misses=11938physical=44 logical=56 role=AIV icache_misses=12699physical=45 logical=57 role=AIV icache_misses=12514physical=46 logical=58 role=AIV icache_misses=11448physical=47 logical=59 role=AIV icache_misses=11178physical=48 logical=60 role=AIV icache_misses=11062physical=49 logical=61 role=AIV icache_misses=12098physical=50 logical=62 role=AIV icache_misses=12088physical=51 logical=63 role=AIV icache_misses=11318physical=72 logical=64 role=AIV icache_misses=12770physical=73 logical=65 role=AIV icache_misses=12352physical=74 logical=66 role=AIV icache_misses=12485physical=75 logical=67 role=AIV icache_misses=12372physical=76 logical=68 role=AIV icache_misses=11547physical=77 logical=69 role=AIV icache_misses=11737physical=78 logical=70 role=AIV icache_misses=12950physical=79 logical=71 role=AIV icache_misses=11855physical=80 logical=72 role=AIV icache_misses=11609physical=81 logical=73 role=AIV icache_misses=12064physical=84 logical=74 role=AIV icache_misses=12322physical=85 logical=75 role=AIV icache_misses=11570physical=86 logical=76 role=AIV icache_misses=10939physical=87 logical=77 role=AIV icache_misses=12028physical=88 logical=78 role=AIV icache_misses=11218physical=89 logical=79 role=AIV icache_misses=11558physical=90 logical=80 role=AIV icache_misses=12339physical=91 logical=81 role=AIV icache_misses=12656physical=92 logical=82 role=AIV icache_misses=10628physical=93 logical=83 role=AIV icache_misses=12821physical=94 logical=84 role=AIV icache_misses=12168physical=95 logical=85 role=AIV icache_misses=11744physical=96 logical=86 role=AIV icache_misses=12349physical=97 logical=87 role=AIV icache_misses=13329physical=98 logical=88 role=AIV icache_misses=11670physical=99 logical=89 role=AIV icache_misses=10602physical=100 logical=90 role=AIV icache_misses=11479physical=101 logical=91 role=AIV icache_misses=11326physical=102 logical=92 role=AIV icache_misses=10938physical=103 logical=93 role=AIV icache_misses=11417physical=104 logical=94 role=AIV icache_misses=11915physical=105 logical=95 role=AIV icache_misses=12582 + +
+
+

逐核原始主计数

+

报告展示 primary request/miss；none 模式不重复发布 shadow 原值；逐核 primary/shadow 同值由必选 status 闭环。

+
+ + + + + + + +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate90ns量尺 µs
00AIC002,692,9716,262.4989,497,9819,457,314691,8016580.095%59.220
11AIC102,487,6936,301.0619,403,6929,360,947681,3298540.125%76.860
22AIC202,684,2316,255.2179,385,6679,342,914684,9367120.104%64.080
33AIC302,532,5706,564.2589,467,8239,421,102702,2939000.128%81.000
44AIC402,624,5146,372.2009,440,9109,398,279689,2467700.112%69.300
55AIC502,381,8186,350.2189,675,5739,627,626686,5837110.104%63.990
66AIC602,718,2236,270.7249,245,1539,206,475716,3665730.080%51.570
77AIC702,541,1986,403.6419,435,1089,394,082707,7346920.098%62.280
88AIC802,756,5966,249.7269,217,8489,177,679716,2206820.095%61.380
99AIC902,465,8056,335.3209,348,4589,305,606696,6107960.114%71.640
1010AIC1002,744,6746,249.6419,425,5589,387,694683,6536350.093%57.150
1211AIC1103,331,7446,230.8889,481,6199,442,263662,5705980.090%53.820
1312AIC1202,792,1066,462.9819,694,5549,653,338744,9827490.101%67.410
1413AIC1302,457,0596,514.9159,759,3679,718,995706,7257080.100%63.720
1514AIC1402,854,4526,282.2999,341,9409,302,251706,9607030.099%63.270
1615AIC1502,405,5346,276.1269,441,0169,395,703665,1998080.121%72.720
1832AIV013,298,9916,524.06010,492,08310,181,988740,14911,3301.531%1,019.700
1933AIV023,508,1026,168.6399,894,6329,521,072646,61211,1991.732%1,007.910
2034AIV112,879,7546,119.7039,996,1709,636,071675,77811,8101.748%1,062.900
2135AIV123,155,3296,800.65210,758,90110,377,067848,16712,0951.426%1,088.550
2236AIV212,979,6815,798.0069,203,5298,781,709698,35412,0091.720%1,080.810
2337AIV222,882,0555,899.4229,214,6628,805,815716,43411,3961.591%1,025.640
2438AIV313,135,9476,161.8679,708,8729,331,375671,97312,2361.821%1,101.240
2539AIV322,711,8005,983.6649,683,3949,296,840665,32112,0571.812%1,085.130
2640AIV412,936,2286,186.96310,009,7049,610,165679,25812,7831.882%1,150.470
2741AIV423,120,0826,320.6239,969,7389,513,913754,62512,2171.619%1,099.530
2842AIV513,306,8716,590.09010,427,93410,093,897830,30611,3821.371%1,024.380
2943AIV523,048,6515,983.6429,496,5299,103,673801,32413,7001.710%1,233.000
3044AIV613,361,3705,791.0769,200,3858,822,208640,13510,8381.693%975.420
3145AIV623,074,5716,597.83410,522,98910,141,790751,84911,7481.563%1,057.320
3246AIV712,813,3205,988.4939,602,5079,276,422665,02811,9931.803%1,079.370
3347AIV723,145,4675,993.9209,611,3239,292,337779,92611,7171.502%1,054.530
3448AIV813,943,4546,028.6729,945,0239,621,464627,0319,6391.537%867.510
3549AIV822,914,9986,145.9379,600,0699,183,971723,27412,0771.670%1,086.930
3650AIV912,839,1165,937.8539,610,5209,275,593660,60611,4321.731%1,028.880
3751AIV922,897,4686,067.4009,559,1419,224,564663,93811,2291.691%1,010.610
3852AIV1013,330,2206,506.84110,455,67110,098,804829,94812,0871.456%1,087.830
3953AIV1022,960,1316,212.7209,507,7419,075,857702,92012,6181.795%1,135.620
4254AIV1113,076,6246,601.75910,520,73910,197,527747,56212,0001.605%1,080.000
4355AIV1122,884,8056,445.8149,993,5139,620,664726,21311,9381.644%1,074.420
4456AIV1213,151,4396,278.32110,064,8229,695,660735,90812,6991.726%1,142.910
4557AIV1222,921,2136,215.9259,700,2549,331,122746,20912,5141.677%1,126.260
4658AIV1313,043,6015,947.8469,265,8938,829,986746,90711,4481.533%1,030.320
4759AIV1323,148,0515,943.9809,348,2448,955,848770,01811,1781.452%1,006.020
4860AIV1412,890,4076,120.5869,907,8009,560,833663,61011,0621.667%995.580
4961AIV1422,955,6356,139.0189,942,2459,617,891739,93912,0981.635%1,088.820
5062AIV1513,259,0386,297.9159,756,2929,371,503719,41512,0881.680%1,087.920
5163AIV1523,010,8295,982.3599,509,6499,076,541690,53311,3181.639%1,018.620
5416AIC1602,865,4686,400.7539,742,9539,703,910759,6806890.091%62.010
5517AIC1702,710,7676,538.0569,561,6289,517,188748,4517760.104%69.840
5618AIC1803,036,3646,215.0449,128,9399,088,377699,8547410.106%66.690
5719AIC1902,448,6636,269.3479,512,8459,471,907665,2997230.109%65.070
5820AIC2002,607,6956,261.2769,264,6819,218,440680,5387380.108%66.420
6021AIC2102,559,6306,211.9709,372,6839,329,383728,8997910.109%71.190
6122AIC2202,684,2576,237.5069,248,1169,205,237703,9536630.094%59.670
6223AIC2302,514,4646,259.7259,292,8919,250,290674,0917710.114%69.390
6324AIC2402,636,5556,526.2179,734,9459,694,325692,4397270.105%65.430
6425AIC2502,539,2926,591.5359,757,3669,712,010700,4038180.117%73.620
6526AIC2602,835,2626,472.3699,952,5689,913,392765,2346540.085%58.860
6627AIC2702,512,1906,423.2159,436,0179,390,782702,5246850.098%61.650
6728AIC2802,603,2946,249.7079,350,5919,311,054690,4936720.097%60.480
6829AIC2902,481,3946,394.2219,543,3939,504,170683,5956350.093%57.150
6930AIC3002,812,3906,379.0909,758,9219,717,105741,2786880.093%61.920
7031AIC3102,626,3846,256.9399,313,8279,272,840733,2757390.101%66.510
7264AIV1612,885,7496,166.89410,073,3679,724,048695,33912,7701.837%1,149.300
7365AIV1622,951,9046,411.1869,949,7449,594,307765,23412,3521.614%1,111.680
7466AIV1712,850,0075,793.9709,279,1198,915,192662,27912,4851.885%1,123.650
7567AIV1722,884,4636,095.1909,872,9009,488,403711,67512,3721.738%1,113.480
7668AIV1813,189,8805,951.9679,629,2799,316,680781,40911,5471.478%1,039.230
7769AIV1822,931,7766,302.80110,038,8649,713,695757,09611,7371.550%1,056.330
7870AIV1913,276,3366,226.23510,078,1599,716,206793,58812,9501.632%1,165.500
7971AIV1923,027,0196,432.62710,424,25010,057,630777,17611,8551.525%1,066.950
8072AIV2013,069,1876,388.76410,354,29710,059,667764,05711,6091.519%1,044.810
8173AIV2022,814,8436,223.4459,719,1339,390,266740,12612,0641.630%1,085.760
8474AIV2113,299,6336,406.31610,109,8549,800,428748,97412,3221.645%1,108.980
8575AIV2124,010,2186,054.6529,632,7589,306,468684,27211,5701.691%1,041.300
8676AIV2213,175,2945,911.5759,570,8829,249,888771,95710,9391.417%984.510
8777AIV2222,865,6276,202.2449,781,0439,414,898725,89412,0281.657%1,082.520
8878AIV2312,996,6006,121.6099,570,4069,279,961667,18111,2181.681%1,009.620
8979AIV2323,028,0815,953.4588,929,1988,590,434806,01111,5581.434%1,040.220
9080AIV2413,500,5456,226.2379,813,6269,474,940919,31712,3391.342%1,110.510
9181AIV2422,717,1936,231.35310,003,1759,625,926679,87912,6561.862%1,139.040
9282AIV2513,545,8056,000.9289,629,9879,351,926754,86310,6281.408%956.520
9383AIV2522,920,8746,314.45010,050,7859,692,833732,04612,8211.751%1,153.890
9484AIV2613,196,6906,448.91210,277,3389,927,533785,29812,1681.549%1,095.120
9585AIV2622,752,6256,279.2939,819,9059,472,586681,68711,7441.723%1,056.960
9686AIV2713,092,9436,243.5189,831,1389,520,273754,49512,3491.637%1,111.410
9787AIV2723,046,9196,197.5879,492,7739,124,935800,83113,3291.664%1,199.610
9888AIV2812,807,5645,807.5909,041,1938,707,993684,31711,6701.705%1,050.300
9989AIV2824,431,5916,124.4519,917,5879,567,020690,11010,6021.536%954.180
10090AIV2912,764,6696,040.6659,696,1799,418,739648,76411,4791.769%1,033.110
10191AIV2922,756,0615,925.1779,674,8559,353,243676,40011,3261.674%1,019.340
10292AIV3013,287,9975,937.4619,605,8339,316,747830,44910,9381.317%984.420
10393AIV3023,004,1456,534.55010,239,4529,886,972748,17911,4171.526%1,027.530
10494AIV3112,851,4106,325.42410,332,56510,013,254691,38311,9151.723%1,072.350
10595AIV3122,656,4176,186.9569,929,8639,601,289676,34312,5821.860%1,132.380
+
+ diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_submit_transition_20260723_044115_icache_report.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_submit_transition_20260723_044115_icache_report.html new file mode 100644 index 0000000000..68f50d70da --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_submit_transition_20260723_044115_icache_report.html @@ -0,0 +1,362 @@ + + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 fdwic_submit_pmu_raw.json · schema fdwic-submit-pmu-v3 · 32 AIC + 64 AIV

+ +
+

submit-transition 阶段观察（phase_id=6）

+

边界 previous_submit_end_to_next_submit_begin · 计数语义 running_read_clear_observed_bracket · + 时间语义 boundary_diagnostic_sys_cnt_between_observers · + expected_calls_per_core=1279

+ +

本报告使用 empty-bracket 估算进入 phase observed + 的局部记录代码开销。 + AIC/AIV 分别按“空区间每组记录开销 × 本阶段实际 begin/end 记录组数”计算，ALL + 再由两类核相加。参考值只从 phase observed 分子扣除该估算；原始 whole 分母保持不变， + 因为空区间没有测量 whole 窗口中的全部记录开销。所以下表的参考比例不是完整业务 + 阶段占完整调度窗口的比例，也不是精确校正。若参考值为负，表示记录代码开销估算已经 + 超过原始 observed；这只能说明该信号低于当前校准的分辨能力，不能截成零或解释成负的 + 业务事件。校准 raw：/home/q00473782/atomic/private/gpt/simpler/outputs/TestPagedAttentionUnroll_Case1_20260723_043533/fdwic_submit_pmu_raw.json · + SHA-256 56199f91545cc6a1fbf18b8bb2cfebae279cb5801516311a040d4facccf321ba。两份 provenance 已核验为同一场景、同一 Git revision 8c6b6e3a412d。

+ + +

本阶段若跨过 linked Kernel，会同时门控 SYS 时间与 PMU counter， + 因而 total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段。 + result-used return-ready atomic 依赖区间只从 SYS 边界累计值扣除，PMU counter + 仍含 atomic 指令事件；source-issue atomic + 保留在时间和计数口径内。没有 linked Kernel 的阶段不触发 PMU 停/开表； + 每次调用的分母仍是外层业务调用次数，不是 Begin/End 读数，也不是排除的 Kernel 调用数。

+ +

阶段原始 PMU 观测来自 running read-clear 的 total cycles，并按 AIC/AIV + 各自校准频率换算；提供 empty-bracket 时，页面主参考值再扣除局部记录代码开销估算。 + phase_elapsed_ticks 只是 SYS 边界闭合诊断，绝不按 1 GHz 当作阶段时间。 + Scalar busy 与 non-scalar residual（逐核先算 total−scalar）共同解释阶段 PMU total； + 未校准时只能展示 raw observed；提供 empty-bracket 后，记录代码开销估算只从 phase + observed 分子扣除，原始 whole 分母保持不变。当前数据无法得到完整业务阶段占完整调度 + 窗口的精确比例。 + linked vector/cube Kernel 从 SYS 与 PMU counter 一并门控排除；result-used return-ready atomic + 依赖区间只从 SYS 边界累计值扣除，I-cache/PMU counter 仍含其指令事件；source-issue atomic + 保留。request/miss 百分比的分母是同一 ELF、同一次采集的 Submit 整窗 primary。 + observed_plus_capture_gap = observed + (primary − shadow)； + 边界读数和插桩 bookkeeping 会进入 sample，因此“观测值”和“加全窗 capture gap”都不是原业务 + 事件数的数学上下界。不同 ELF 不能直接相减后冒充业务真值；提供 empty-bracket 时只按上方 + 明示的公式估算局部记录代码开销。request/miss 的逐核 min/max 是每核累计整个 phase + 的极值，不是逐调用极值。

+
+ + + + +
指标ALL96 核AIC32 核AIV64 核
Phase PMU total扣除记录代码开销估算后的参考值 Σ 122,359,317.262 cycles
≈ 74,164.174 µs（1.649844 cycles/ns）
原始 observed Σ 314,497,428 cycles（≈ 190,622.524 µs）− 记录代码开销估算 192,138,110.738 cycles
记录代码开销估算 / 原始 observed 61.094%；原始逐核 最小 2,691,895；最大 3,631,754；原始 2,561.388 cycles/call		扣除记录代码开销估算后的参考值 Σ 26,926,937.940 cycles
≈ 16,318.743 µs（1.650062 cycles/ns）
原始 observed Σ 87,706,220 cycles（≈ 53,153.288 µs）− 记录代码开销估算 60,779,282.060 cycles
记录代码开销估算 / 原始 observed 69.299%；原始逐核 最小 2,691,895；最大 2,806,960；原始 2,142.939 cycles/call		扣除记录代码开销估算后的参考值 Σ 95,432,379.323 cycles
≈ 57,847.237 µs（1.649731 cycles/ns）
原始 observed Σ 226,791,208 cycles（≈ 137,471.629 µs）− 记录代码开销估算 131,358,828.677 cycles
记录代码开销估算 / 原始 observed 57.921%；原始逐核 最小 3,446,371；最大 3,631,754；原始 2,770.612 cycles/call
Phase scalar busy扣除记录代码开销估算后的参考值 Σ 82,841,966.639 cycles
≈ 50,212.000 µs（1.649844 cycles/ns）
原始 observed Σ 206,591,150 cycles（≈ 125,218.596 µs）− 记录代码开销估算 123,749,183.361 cycles
记录代码开销估算 / 原始 observed 59.901%；原始逐核 最小 1,982,935；最大 2,278,654；原始 1,682.558 cycles/call		扣除记录代码开销估算后的参考值 Σ 24,930,210.645 cycles
≈ 15,108.651 µs（1.650062 cycles/ns）
原始 observed Σ 64,958,620 cycles（≈ 39,367.381 µs）− 记录代码开销估算 40,028,409.355 cycles
记录代码开销估算 / 原始 observed 61.621%；原始逐核 最小 1,982,935；最大 2,076,648；原始 1,587.144 cycles/call		扣除记录代码开销估算后的参考值 Σ 57,911,755.994 cycles
≈ 35,103.757 µs（1.649731 cycles/ns）
原始 observed Σ 141,632,530 cycles（≈ 85,851.893 µs）− 记录代码开销估算 83,720,774.006 cycles
记录代码开销估算 / 原始 observed 59.111%；原始逐核 最小 2,155,867；最大 2,278,654；原始 1,730.264 cycles/call
非 Scalar-busy 残余扣除记录代码开销估算后的参考值 Σ 39,517,350.623 cycles
≈ 23,952.174 µs（1.649844 cycles/ns）
原始 observed Σ 107,906,278 cycles（≈ 65,403.928 µs）− 记录代码开销估算 68,388,927.377 cycles
记录代码开销估算 / 原始 observed 63.378%；原始逐核 最小 690,210；最大 1,411,878；原始 878.830 cycles/call		扣除记录代码开销估算后的参考值 Σ 1,996,727.295 cycles
≈ 1,210.092 µs（1.650062 cycles/ns）
原始 observed Σ 22,747,600 cycles（≈ 13,785.906 µs）− 记录代码开销估算 20,750,872.705 cycles
记录代码开销估算 / 原始 observed 91.222%；原始逐核 最小 690,210；最大 739,516；原始 555.796 cycles/call		扣除记录代码开销估算后的参考值 Σ 37,520,623.329 cycles
≈ 22,743.480 µs（1.649731 cycles/ns）
原始 observed Σ 85,158,678 cycles（≈ 51,619.736 µs）− 记录代码开销估算 47,638,054.671 cycles
记录代码开销估算 / 原始 observed 55.940%；原始逐核 最小 1,231,357；最大 1,411,878；原始 1,040.347 cycles/call
阶段参考关系（raw 同格保留）扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 67.704%
参考 Non-scalar / 参考 Phase total 32.296%
参考 Phase total / 原始 whole total 13.982%；原始 observed 35.938%
参考 Phase scalar / 原始 whole scalar 10.595%；原始 observed 26.423%
whole scalar−shadow scalar：Σ 245,568 cycles；逐核 最小 2,558；最大 2,558		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 92.585%
参考 Non-scalar / 参考 Phase total 7.415%
参考 Phase total / 原始 whole total 10.160%；原始 observed 33.092%
参考 Phase scalar / 原始 whole scalar 9.599%；原始 observed 25.011%
whole scalar−shadow scalar：Σ 81,856 cycles；逐核 最小 2,558；最大 2,558		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 60.684%
参考 Non-scalar / 参考 Phase total 39.316%
参考 Phase total / 原始 whole total 15.643%；原始 observed 37.175%
参考 Phase scalar / 原始 whole scalar 11.091%；原始 observed 27.125%
whole scalar−shadow scalar：Σ 163,712 cycles；逐核 最小 2,558；最大 2,558
Request（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 13,181,994.889
原始 observed 20,046,246 − 记录代码开销估算 6,864,251.111
参考值 / 原始整窗 14.314%；原始 observed / 原始整窗 21.768%
记录代码开销估算 / 原始 observed 34.242%
原始逐核 最小 191,239；最大 218,737；原始整窗 92,089,124
原始 capture gap +0；加 gap 后 20,046,246（21.768%）		扣除记录代码开销估算后的参考值 3,920,786.056
原始 observed 6,141,202 − 记录代码开销估算 2,220,415.944
参考值 / 原始整窗 12.514%；原始 observed / 原始整窗 19.601%
记录代码开销估算 / 原始 observed 36.156%
原始逐核 最小 191,239；最大 192,627；原始整窗 31,330,670
原始 capture gap +0；加 gap 后 6,141,202（19.601%）		扣除记录代码开销估算后的参考值 9,261,208.833
原始 observed 13,905,044 − 记录代码开销估算 4,643,835.167
参考值 / 原始整窗 15.243%；原始 observed / 原始整窗 22.886%
记录代码开销估算 / 原始 observed 33.397%
原始逐核 最小 216,672；最大 218,737；原始整窗 60,758,454
原始 capture gap +0；加 gap 后 13,905,044（22.886%）
Miss（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 991,436.489
原始 observed 1,226,119 − 记录代码开销估算 234,682.511
参考值 / 原始整窗 24.747%；原始 observed / 原始整窗 30.605%
记录代码开销估算 / 原始 observed 19.140%
原始逐核 最小 605；最大 19,639；原始整窗 4,006,223
原始 capture gap +0；加 gap 后 1,226,119（30.605%）		扣除记录代码开销估算后的参考值 25,258.945
原始 observed 27,746 − 记录代码开销估算 2,487.055
参考值 / 原始整窗 18.748%；原始 observed / 原始整窗 20.594%
记录代码开销估算 / 原始 observed 8.964%
原始逐核 最小 605；最大 1,174；原始整窗 134,727
原始 capture gap +0；加 gap 后 27,746（20.594%）		扣除记录代码开销估算后的参考值 966,177.545
原始 observed 1,198,373 − 记录代码开销估算 232,195.455
参考值 / 原始整窗 24.956%；原始 observed / 原始整窗 30.954%
记录代码开销估算 / 原始 observed 19.376%
原始逐核 最小 17,966；最大 19,639；原始整窗 3,871,496
原始 capture gap +0；加 gap 后 1,198,373（30.954%）
SYS 边界诊断 / Begin-EndΣ 60,474,015 raw ticks
逐核 最小 387,447；最大 780,684；仅边界诊断
Begin / End：122,784 / 122,784
业务调用 122,784 次；排除 linked Kernel 调用 0 次		Σ 13,424,194 raw ticks
逐核 最小 387,447；最大 454,658；仅边界诊断
Begin / End：40,928 / 40,928
业务调用 40,928 次；排除 linked Kernel 调用 0 次		Σ 47,049,821 raw ticks
逐核 最小 681,403；最大 780,684；仅边界诊断
Begin / End：81,856 / 81,856
业务调用 81,856 次；排除 linked Kernel 调用 0 次
+
+ + +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
f0a4d74fe303134935246b6b8db97bd425ff6c11ee3b3b09872f172de5157aaf
+
Profile / extra cache
submit-pmu-submit-transition / + d17b87d79477c0e1
+
Profiled cache key
TestPagedAttentionUnroll / a5 / fully_distributed_within_core / submit-pmu-submit-transition
+
Source state
source-v2:8c6b6e3a412d2c7c7113c2d8f714a14eb3786f80:d772d577a325f9eb8710e333a7bb67be668b08f44a6b980da7495a25e2794410:ef815497900b9aae59f96864c617d43eabd7349afa87b947056fab2f94617785
+
Source-state stamp
/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/d17b87d79477c0e1/aicore/.git_commit
+
Compile definitions
PTO_FDWIC_SUBMIT_PMU=1
PTO_FDWIC_SUBMIT_PMU_PHASE_ID=6
PTO_FDWIC_TRACE_ENABLED=0
+
+
+ + + +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
AICore final2,678,2483f18e5075d99579ddb7db83ef215a6fc3737eab002345cad54b6ec108f86819f213,8403a887881e570d8277819bed2a461af0e08ece0ed7ad2d2f07b2aa69ae9c6a407/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/aicore-extra/d17b87d79477c0e1/aicore_kernel.o
AIC combined2,006,6642f2c29f518c551610cb20d8a79b445026d8e2075d5e6cd99e2104d86399b073199,7925f1e433cd61d22c23e9272ab7d5a6cfd7092fa32e456f6df0c07f0c22e16ac44/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/d17b87d79477c0e1/aicore/aicore_aic_combined.o
AIV combined2,267,728b7a98f3cd618567d06de250efb2d3b9bf06a0388939ddf5c12e3a5b4e0cc9394114,000e7a8d25774140fd96766075f18df5b53d18ef8e10cc246e85cb9f433e8467e0f/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/d17b87d79477c0e1/aicore/aicore_aiv_combined.o
Host runtime11,776,1440369d0602054bf612f86b58c294503e00486bd505bff9ee7a455994f043147cc454,6325ac381ed10be5f5d492b8072c051a177131f4caec6d6f6bc9bf50880da4bb0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so
+
+ +
+
原始全局 Submit 时间范围（墙钟） + 6,052.808 µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + 128096 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-2561ad1eda60a8ec78f25150f6badde292f48d16155833905ea4e11865555a81ff7
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + 90.000 ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×90 ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
+
+

ALL · 96 核

+
+
Submit PMU total/core
+
均值 9,115,675.6；最小 7,928,790；最大 9,830,110 cycles
+ 等效时间 均值 5,525.174；最小 4,805.782；最大 5,958.206 µs + （按 1.649844 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 8,144,435.1；最小 7,748,777；最大 8,823,567 cycles
+ 等效时间 均值 4,936.488；最小 4,696.673；最大 5,348.122 µs； + 加权占比 89.345% +
+
非 Scalar-busy 残余/core
+
均值 971,240.4；最小 138,204；最大 1,586,007 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 588.686；最小 83.768；最大 961.307 µs +
+
SYS gate 边界诊断/core
+
均值 5,078,224.1；最小 4,336,142；最大 5,564,355 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,858.584；最小 5,453.500；最大 6,052.725 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 780,359.8；最小 443,032；最大 1,397,278 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 86.680% +
+
Primary I-cache request/core
最小 928,721；最大 1,069,210
+
Primary I-cache miss/core
最小 3,424；最大 62,283
+
加权 miss rate
4.350%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 308.160；最大 5,605.470 µs
+
+
+ +
+

AIC · 32 核

+
+
Submit PMU total/core
+
均值 8,282,329.6；最小 7,928,790；最大 9,023,303 cycles
+ 等效时间 均值 5,019.405；最小 4,805.147；最大 5,468.463 µs + （按 1.650062 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 8,116,326.0；最小 7,748,777；最大 8,823,567 cycles
+ 等效时间 均值 4,918.801；最小 4,696.052；最大 5,347.415 µs； + 加权占比 97.996% +
+
非 Scalar-busy 残余/core
+
均值 166,003.6；最小 138,204；最大 199,736 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 100.604；最小 83.757；最大 121.048 µs +
+
SYS gate 边界诊断/core
+
均值 4,483,095.0；最小 4,336,142；最大 4,655,447 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,575.532；最小 5,453.500；最大 6,052.725 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 1,092,437.4；最小 951,832；最大 1,397,278 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 80.407% +
+
Primary I-cache request/core
最小 935,762；最大 1,069,210
+
Primary I-cache miss/core
最小 3,424；最大 4,651
+
加权 miss rate
0.430%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 308.160；最大 418.590 µs
+
+
+ +
+

AIV · 64 核

+
+
Submit PMU total/core
+
均值 9,532,348.6；最小 9,345,182；最大 9,830,110 cycles
+ 等效时间 均值 5,778.123；最小 5,664.670；最大 5,958.614 µs + （按 1.649731 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 8,158,489.7；最小 7,893,387；最大 8,389,210 cycles
+ 等效时间 均值 4,945.345；最小 4,784.651；最大 5,085.199 µs； + 加权占比 85.587% +
+
非 Scalar-busy 残余/core
+
均值 1,373,858.9；最小 1,195,395；最大 1,586,007 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 832.778；最小 724.600；最大 961.373 µs +
+
SYS gate 边界诊断/core
+
均值 5,375,788.6；最小 5,286,117；最大 5,564,355 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 6,000.110；最小 5,968.115；最大 6,036.761 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 624,320.9；最小 443,032；最大 735,967 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 89.595% +
+
Primary I-cache request/core
最小 928,721；最大 977,135
+
Primary I-cache miss/core
最小 58,674；最大 62,283
+
加权 miss rate
6.372%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 5,280.660；最大 5,605.470 µs
+
+
+
+

逐物理核分布

+

AICAIV

+
+
+
SYS gate boundary-diagnostic ticks/core
+ + + + 5,564,355 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_submit_elapsed_ticks=4458842physical=1 logical=1 role=AIC scalar_submit_elapsed_ticks=4483209physical=2 logical=2 role=AIC scalar_submit_elapsed_ticks=4497373physical=3 logical=3 role=AIC scalar_submit_elapsed_ticks=4600969physical=4 logical=4 role=AIC scalar_submit_elapsed_ticks=4448494physical=5 logical=5 role=AIC scalar_submit_elapsed_ticks=4336142physical=6 logical=6 role=AIC scalar_submit_elapsed_ticks=4487591physical=7 logical=7 role=AIC scalar_submit_elapsed_ticks=4393965physical=8 logical=8 role=AIC scalar_submit_elapsed_ticks=4476023physical=9 logical=9 role=AIC scalar_submit_elapsed_ticks=4528344physical=10 logical=10 role=AIC scalar_submit_elapsed_ticks=4553345physical=12 logical=11 role=AIC scalar_submit_elapsed_ticks=4501098physical=13 logical=12 role=AIC scalar_submit_elapsed_ticks=4574086physical=14 logical=13 role=AIC scalar_submit_elapsed_ticks=4396442physical=15 logical=14 role=AIC scalar_submit_elapsed_ticks=4528943physical=16 logical=15 role=AIC scalar_submit_elapsed_ticks=4362754physical=54 logical=16 role=AIC scalar_submit_elapsed_ticks=4496447physical=55 logical=17 role=AIC scalar_submit_elapsed_ticks=4598987physical=56 logical=18 role=AIC scalar_submit_elapsed_ticks=4458558physical=57 logical=19 role=AIC scalar_submit_elapsed_ticks=4360838physical=58 logical=20 role=AIC scalar_submit_elapsed_ticks=4428055physical=60 logical=21 role=AIC scalar_submit_elapsed_ticks=4475601physical=61 logical=22 role=AIC scalar_submit_elapsed_ticks=4587456physical=62 logical=23 role=AIC scalar_submit_elapsed_ticks=4494631physical=63 logical=24 role=AIC scalar_submit_elapsed_ticks=4437116physical=64 logical=25 role=AIC scalar_submit_elapsed_ticks=4456214physical=65 logical=26 role=AIC scalar_submit_elapsed_ticks=4534235physical=66 logical=27 role=AIC scalar_submit_elapsed_ticks=4431527physical=67 logical=28 role=AIC scalar_submit_elapsed_ticks=4430940physical=68 logical=29 role=AIC scalar_submit_elapsed_ticks=4547519physical=69 logical=30 role=AIC scalar_submit_elapsed_ticks=4437848physical=70 logical=31 role=AIC scalar_submit_elapsed_ticks=4655447physical=18 logical=32 role=AIV scalar_submit_elapsed_ticks=5461689physical=19 logical=33 role=AIV scalar_submit_elapsed_ticks=5404321physical=20 logical=34 role=AIV scalar_submit_elapsed_ticks=5389232physical=21 logical=35 role=AIV scalar_submit_elapsed_ticks=5425386physical=22 logical=36 role=AIV scalar_submit_elapsed_ticks=5376167physical=23 logical=37 role=AIV scalar_submit_elapsed_ticks=5402215physical=24 logical=38 role=AIV scalar_submit_elapsed_ticks=5515400physical=25 logical=39 role=AIV scalar_submit_elapsed_ticks=5364350physical=26 logical=40 role=AIV scalar_submit_elapsed_ticks=5365097physical=27 logical=41 role=AIV scalar_submit_elapsed_ticks=5365031physical=28 logical=42 role=AIV scalar_submit_elapsed_ticks=5371146physical=29 logical=43 role=AIV scalar_submit_elapsed_ticks=5367805physical=30 logical=44 role=AIV scalar_submit_elapsed_ticks=5397972physical=31 logical=45 role=AIV scalar_submit_elapsed_ticks=5320146physical=32 logical=46 role=AIV scalar_submit_elapsed_ticks=5390673physical=33 logical=47 role=AIV scalar_submit_elapsed_ticks=5360942physical=34 logical=48 role=AIV scalar_submit_elapsed_ticks=5383625physical=35 logical=49 role=AIV scalar_submit_elapsed_ticks=5298998physical=36 logical=50 role=AIV scalar_submit_elapsed_ticks=5375371physical=37 logical=51 role=AIV scalar_submit_elapsed_ticks=5396589physical=38 logical=52 role=AIV scalar_submit_elapsed_ticks=5352328physical=39 logical=53 role=AIV scalar_submit_elapsed_ticks=5322583physical=42 logical=54 role=AIV scalar_submit_elapsed_ticks=5385392physical=43 logical=55 role=AIV scalar_submit_elapsed_ticks=5377119physical=44 logical=56 role=AIV scalar_submit_elapsed_ticks=5412029physical=45 logical=57 role=AIV scalar_submit_elapsed_ticks=5365785physical=46 logical=58 role=AIV scalar_submit_elapsed_ticks=5353365physical=47 logical=59 role=AIV scalar_submit_elapsed_ticks=5341237physical=48 logical=60 role=AIV scalar_submit_elapsed_ticks=5363577physical=49 logical=61 role=AIV scalar_submit_elapsed_ticks=5316559physical=50 logical=62 role=AIV scalar_submit_elapsed_ticks=5406539physical=51 logical=63 role=AIV scalar_submit_elapsed_ticks=5359288physical=72 logical=64 role=AIV scalar_submit_elapsed_ticks=5333852physical=73 logical=65 role=AIV scalar_submit_elapsed_ticks=5398605physical=74 logical=66 role=AIV scalar_submit_elapsed_ticks=5377820physical=75 logical=67 role=AIV scalar_submit_elapsed_ticks=5306389physical=76 logical=68 role=AIV scalar_submit_elapsed_ticks=5389562physical=77 logical=69 role=AIV scalar_submit_elapsed_ticks=5375800physical=78 logical=70 role=AIV scalar_submit_elapsed_ticks=5319742physical=79 logical=71 role=AIV scalar_submit_elapsed_ticks=5297869physical=80 logical=72 role=AIV scalar_submit_elapsed_ticks=5350426physical=81 logical=73 role=AIV scalar_submit_elapsed_ticks=5286117physical=84 logical=74 role=AIV scalar_submit_elapsed_ticks=5396963physical=85 logical=75 role=AIV scalar_submit_elapsed_ticks=5392911physical=86 logical=76 role=AIV scalar_submit_elapsed_ticks=5391685physical=87 logical=77 role=AIV scalar_submit_elapsed_ticks=5349358physical=88 logical=78 role=AIV scalar_submit_elapsed_ticks=5427964physical=89 logical=79 role=AIV scalar_submit_elapsed_ticks=5352783physical=90 logical=80 role=AIV scalar_submit_elapsed_ticks=5371467physical=91 logical=81 role=AIV scalar_submit_elapsed_ticks=5306224physical=92 logical=82 role=AIV scalar_submit_elapsed_ticks=5402138physical=93 logical=83 role=AIV scalar_submit_elapsed_ticks=5344633physical=94 logical=84 role=AIV scalar_submit_elapsed_ticks=5430422physical=95 logical=85 role=AIV scalar_submit_elapsed_ticks=5340226physical=96 logical=86 role=AIV scalar_submit_elapsed_ticks=5347158physical=97 logical=87 role=AIV scalar_submit_elapsed_ticks=5297351physical=98 logical=88 role=AIV scalar_submit_elapsed_ticks=5395832physical=99 logical=89 role=AIV scalar_submit_elapsed_ticks=5564355physical=100 logical=90 role=AIV scalar_submit_elapsed_ticks=5433778physical=101 logical=91 role=AIV scalar_submit_elapsed_ticks=5430115physical=102 logical=92 role=AIV scalar_submit_elapsed_ticks=5412752physical=103 logical=93 role=AIV scalar_submit_elapsed_ticks=5358885physical=104 logical=94 role=AIV scalar_submit_elapsed_ticks=5405574physical=105 logical=95 role=AIV scalar_submit_elapsed_ticks=5373758 + +
+ +
+
PMU total cycles/core
+ + + + 9,830,110 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC total_cycles=8158502physical=1 logical=1 role=AIC total_cycles=8393989physical=2 logical=2 role=AIC total_cycles=8122481physical=3 logical=3 role=AIC total_cycles=8846220physical=4 logical=4 role=AIC total_cycles=8150141physical=5 logical=5 role=AIC total_cycles=7973234physical=6 logical=6 role=AIC total_cycles=8200161physical=7 logical=7 role=AIC total_cycles=8085273physical=8 logical=8 role=AIC total_cycles=8196365physical=9 logical=9 role=AIC total_cycles=8547772physical=10 logical=10 role=AIC total_cycles=8422678physical=12 logical=11 role=AIC total_cycles=8143626physical=13 logical=12 role=AIC total_cycles=8459245physical=14 logical=13 role=AIC total_cycles=8147360physical=15 logical=14 role=AIC total_cycles=8137704physical=16 logical=15 role=AIC total_cycles=7997873physical=54 logical=16 role=AIC total_cycles=8232819physical=55 logical=17 role=AIC total_cycles=8795374physical=56 logical=18 role=AIC total_cycles=8122964physical=57 logical=19 role=AIC total_cycles=7990116physical=58 logical=20 role=AIC total_cycles=7938745physical=60 logical=21 role=AIC total_cycles=8363374physical=61 logical=22 role=AIC total_cycles=8596629physical=62 logical=23 role=AIC total_cycles=8293362physical=63 logical=24 role=AIC total_cycles=8147124physical=64 logical=25 role=AIC total_cycles=8284223physical=65 logical=26 role=AIC total_cycles=8454082physical=66 logical=27 role=AIC total_cycles=8206012physical=67 logical=28 role=AIC total_cycles=8133114physical=68 logical=29 role=AIC total_cycles=8541893physical=69 logical=30 role=AIC total_cycles=7928790physical=70 logical=31 role=AIC total_cycles=9023303physical=18 logical=32 role=AIV total_cycles=9678396physical=19 logical=33 role=AIV total_cycles=9565097physical=20 logical=34 role=AIV total_cycles=9534155physical=21 logical=35 role=AIV total_cycles=9665133physical=22 logical=36 role=AIV total_cycles=9482522physical=23 logical=37 role=AIV total_cycles=9589389physical=24 logical=38 role=AIV total_cycles=9830110physical=25 logical=39 role=AIV total_cycles=9512284physical=26 logical=40 role=AIV total_cycles=9536634physical=27 logical=41 role=AIV total_cycles=9505344physical=28 logical=42 role=AIV total_cycles=9507285physical=29 logical=43 role=AIV total_cycles=9612894physical=30 logical=44 role=AIV total_cycles=9548115physical=31 logical=45 role=AIV total_cycles=9426417physical=32 logical=46 role=AIV total_cycles=9535606physical=33 logical=47 role=AIV total_cycles=9519209physical=34 logical=48 role=AIV total_cycles=9459548physical=35 logical=49 role=AIV total_cycles=9402444physical=36 logical=50 role=AIV total_cycles=9541103physical=37 logical=51 role=AIV total_cycles=9575884physical=38 logical=52 role=AIV total_cycles=9526398physical=39 logical=53 role=AIV total_cycles=9475312physical=42 logical=54 role=AIV total_cycles=9574344physical=43 logical=55 role=AIV total_cycles=9514200physical=44 logical=56 role=AIV total_cycles=9584647physical=45 logical=57 role=AIV total_cycles=9501199physical=46 logical=58 role=AIV total_cycles=9586922physical=47 logical=59 role=AIV total_cycles=9450181physical=48 logical=60 role=AIV total_cycles=9509260physical=49 logical=61 role=AIV total_cycles=9397328physical=50 logical=62 role=AIV total_cycles=9604867physical=51 logical=63 role=AIV total_cycles=9486653physical=72 logical=64 role=AIV total_cycles=9507095physical=73 logical=65 role=AIV total_cycles=9689302physical=74 logical=66 role=AIV total_cycles=9551015physical=75 logical=67 role=AIV total_cycles=9444375physical=76 logical=68 role=AIV total_cycles=9604289physical=77 logical=69 role=AIV total_cycles=9605917physical=78 logical=70 role=AIV total_cycles=9398537physical=79 logical=71 role=AIV total_cycles=9416200physical=80 logical=72 role=AIV total_cycles=9462009physical=81 logical=73 role=AIV total_cycles=9345182physical=84 logical=74 role=AIV total_cycles=9575866physical=85 logical=75 role=AIV total_cycles=9519175physical=86 logical=76 role=AIV total_cycles=9590654physical=87 logical=77 role=AIV total_cycles=9551765physical=88 logical=78 role=AIV total_cycles=9616974physical=89 logical=79 role=AIV total_cycles=9579143physical=90 logical=80 role=AIV total_cycles=9516915physical=91 logical=81 role=AIV total_cycles=9392617physical=92 logical=82 role=AIV total_cycles=9507357physical=93 logical=83 role=AIV total_cycles=9449227physical=94 logical=84 role=AIV total_cycles=9661581physical=95 logical=85 role=AIV total_cycles=9426331physical=96 logical=86 role=AIV total_cycles=9496854physical=97 logical=87 role=AIV total_cycles=9383863physical=98 logical=88 role=AIV total_cycles=9573490physical=99 logical=89 role=AIV total_cycles=9728215physical=100 logical=90 role=AIV total_cycles=9573336physical=101 logical=91 role=AIV total_cycles=9582786physical=102 logical=92 role=AIV total_cycles=9570145physical=103 logical=93 role=AIV total_cycles=9413449physical=104 logical=94 role=AIV total_cycles=9599680physical=105 logical=95 role=AIV total_cycles=9498085 + +
+ +
+
Scalar busy cycles/core
+ + + + 8,823,567 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_busy=8009490physical=1 logical=1 role=AIC scalar_busy=8247231physical=2 logical=2 role=AIC scalar_busy=7962316physical=3 logical=3 role=AIC scalar_busy=8659435physical=4 logical=4 role=AIC scalar_busy=7998373physical=5 logical=5 role=AIC scalar_busy=7810979physical=6 logical=6 role=AIC scalar_busy=8048057physical=7 logical=7 role=AIC scalar_busy=7897869physical=8 logical=8 role=AIC scalar_busy=8051997physical=9 logical=9 role=AIC scalar_busy=8361540physical=10 logical=10 role=AIC scalar_busy=8284474physical=12 logical=11 role=AIC scalar_busy=7972014physical=13 logical=12 role=AIC scalar_busy=8293687physical=14 logical=13 role=AIC scalar_busy=7978439physical=15 logical=14 role=AIC scalar_busy=7972079physical=16 logical=15 role=AIC scalar_busy=7841185physical=54 logical=16 role=AIC scalar_busy=8084579physical=55 logical=17 role=AIC scalar_busy=8621776physical=56 logical=18 role=AIC scalar_busy=7953102physical=57 logical=19 role=AIC scalar_busy=7818367physical=58 logical=20 role=AIC scalar_busy=7748777physical=60 logical=21 role=AIC scalar_busy=8188175physical=61 logical=22 role=AIC scalar_busy=8452401physical=62 logical=23 role=AIC scalar_busy=8114415physical=63 logical=24 role=AIC scalar_busy=7987545physical=64 logical=25 role=AIC scalar_busy=8098761physical=65 logical=26 role=AIC scalar_busy=8288129physical=66 logical=27 role=AIC scalar_busy=8050671physical=67 logical=28 role=AIC scalar_busy=7988536physical=68 logical=29 role=AIC scalar_busy=8346499physical=69 logical=30 role=AIC scalar_busy=7767968physical=70 logical=31 role=AIC scalar_busy=8823567physical=18 logical=32 role=AIV scalar_busy=8235844physical=19 logical=33 role=AIV scalar_busy=8037858physical=20 logical=34 role=AIV scalar_busy=8084037physical=21 logical=35 role=AIV scalar_busy=8079126physical=22 logical=36 role=AIV scalar_busy=8120894physical=23 logical=37 role=AIV scalar_busy=8106940physical=24 logical=38 role=AIV scalar_busy=8358552physical=25 logical=39 role=AIV scalar_busy=8022070physical=26 logical=40 role=AIV scalar_busy=8265112physical=27 logical=41 role=AIV scalar_busy=8108161physical=28 logical=42 role=AIV scalar_busy=8179502physical=29 logical=43 role=AIV scalar_busy=8180688physical=30 logical=44 role=AIV scalar_busy=8231489physical=31 logical=45 role=AIV scalar_busy=8048626physical=32 logical=46 role=AIV scalar_busy=8143204physical=33 logical=47 role=AIV scalar_busy=8061552physical=34 logical=48 role=AIV scalar_busy=8177487physical=35 logical=49 role=AIV scalar_busy=8065689physical=36 logical=50 role=AIV scalar_busy=8175185physical=37 logical=51 role=AIV scalar_busy=8167764physical=38 logical=52 role=AIV scalar_busy=8272696physical=39 logical=53 role=AIV scalar_busy=8142137physical=42 logical=54 role=AIV scalar_busy=8200898physical=43 logical=55 role=AIV scalar_busy=8122208physical=44 logical=56 role=AIV scalar_busy=8211035physical=45 logical=57 role=AIV scalar_busy=8041205physical=46 logical=58 role=AIV scalar_busy=8275222physical=47 logical=59 role=AIV scalar_busy=8151909physical=48 logical=60 role=AIV scalar_busy=8217570physical=49 logical=61 role=AIV scalar_busy=8040494physical=50 logical=62 role=AIV scalar_busy=8357232physical=51 logical=63 role=AIV scalar_busy=8212186physical=72 logical=64 role=AIV scalar_busy=8240262physical=73 logical=65 role=AIV scalar_busy=8299623physical=74 logical=66 role=AIV scalar_busy=8315730physical=75 logical=67 role=AIV scalar_busy=8159963physical=76 logical=68 role=AIV scalar_busy=8296651physical=77 logical=69 role=AIV scalar_busy=8214674physical=78 logical=70 role=AIV scalar_busy=8203142physical=79 logical=71 role=AIV scalar_busy=8093202physical=80 logical=72 role=AIV scalar_busy=8074653physical=81 logical=73 role=AIV scalar_busy=7893387physical=84 logical=74 role=AIV scalar_busy=8288518physical=85 logical=75 role=AIV scalar_busy=8181875physical=86 logical=76 role=AIV scalar_busy=8331094physical=87 logical=77 role=AIV scalar_busy=8209964physical=88 logical=78 role=AIV scalar_busy=8277079physical=89 logical=79 role=AIV scalar_busy=8128673physical=90 logical=80 role=AIV scalar_busy=8188372physical=91 logical=81 role=AIV scalar_busy=8030332physical=92 logical=82 role=AIV scalar_busy=8105989physical=93 logical=83 role=AIV scalar_busy=7992834physical=94 logical=84 role=AIV scalar_busy=8286083physical=95 logical=85 role=AIV scalar_busy=8062772physical=96 logical=86 role=AIV scalar_busy=8170309physical=97 logical=87 role=AIV scalar_busy=7991187physical=98 logical=88 role=AIV scalar_busy=8204286physical=99 logical=89 role=AIV scalar_busy=8389210physical=100 logical=90 role=AIV scalar_busy=8161753physical=101 logical=91 role=AIV scalar_busy=8072222physical=102 logical=92 role=AIV scalar_busy=8151921physical=103 logical=93 role=AIV scalar_busy=7977975physical=104 logical=94 role=AIV scalar_busy=8117752physical=105 logical=95 role=AIV scalar_busy=7937282 + +
+ +
+
Primary I-cache requests/core
+ + + + 1,069,210 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_requests=960439physical=1 logical=1 role=AIC icache_requests=1003412physical=2 logical=2 role=AIC icache_requests=942360physical=3 logical=3 role=AIC icache_requests=1049481physical=4 logical=4 role=AIC icache_requests=956633physical=5 logical=5 role=AIC icache_requests=962987physical=6 logical=6 role=AIC icache_requests=955170physical=7 logical=7 role=AIC icache_requests=965583physical=8 logical=8 role=AIC icache_requests=957105physical=9 logical=9 role=AIC icache_requests=1031353physical=10 logical=10 role=AIC icache_requests=970863physical=12 logical=11 role=AIC icache_requests=947925physical=13 logical=12 role=AIC icache_requests=993891physical=14 logical=13 role=AIC icache_requests=980212physical=15 logical=14 role=AIC icache_requests=936716physical=16 logical=15 role=AIC icache_requests=960698physical=54 logical=16 role=AIC icache_requests=955702physical=55 logical=17 role=AIC icache_requests=1042047physical=56 logical=18 role=AIC icache_requests=949424physical=57 logical=19 role=AIC icache_requests=965938physical=58 logical=20 role=AIC icache_requests=939139physical=60 logical=21 role=AIC icache_requests=1004571physical=61 logical=22 role=AIC icache_requests=1010298physical=62 logical=23 role=AIC icache_requests=969827physical=63 logical=24 role=AIC icache_requests=963291physical=64 logical=25 role=AIC icache_requests=993641physical=65 logical=26 role=AIC icache_requests=994996physical=66 logical=27 role=AIC icache_requests=987104physical=67 logical=28 role=AIC icache_requests=963102physical=68 logical=29 role=AIC icache_requests=1011790physical=69 logical=30 role=AIC icache_requests=935762physical=70 logical=31 role=AIC icache_requests=1069210physical=18 logical=32 role=AIV icache_requests=943036physical=19 logical=33 role=AIV icache_requests=937651physical=20 logical=34 role=AIV icache_requests=943615physical=21 logical=35 role=AIV icache_requests=956467physical=22 logical=36 role=AIV icache_requests=933138physical=23 logical=37 role=AIV icache_requests=951156physical=24 logical=38 role=AIV icache_requests=953646physical=25 logical=39 role=AIV icache_requests=943694physical=26 logical=40 role=AIV icache_requests=948538physical=27 logical=41 role=AIV icache_requests=945656physical=28 logical=42 role=AIV icache_requests=940046physical=29 logical=43 role=AIV icache_requests=960634physical=30 logical=44 role=AIV icache_requests=942650physical=31 logical=45 role=AIV icache_requests=943334physical=32 logical=46 role=AIV icache_requests=945204physical=33 logical=47 role=AIV icache_requests=945409physical=34 logical=48 role=AIV icache_requests=929593physical=35 logical=49 role=AIV icache_requests=951154physical=36 logical=50 role=AIV icache_requests=946013physical=37 logical=51 role=AIV icache_requests=953887physical=38 logical=52 role=AIV icache_requests=955199physical=39 logical=53 role=AIV icache_requests=957045physical=42 logical=54 role=AIV icache_requests=955286physical=43 logical=55 role=AIV icache_requests=951472physical=44 logical=56 role=AIV icache_requests=958711physical=45 logical=57 role=AIV icache_requests=947146physical=46 logical=58 role=AIV icache_requests=972286physical=47 logical=59 role=AIV icache_requests=947647physical=48 logical=60 role=AIV icache_requests=950133physical=49 logical=61 role=AIV icache_requests=944670physical=50 logical=62 role=AIV icache_requests=958396physical=51 logical=63 role=AIV icache_requests=948312physical=72 logical=64 role=AIV icache_requests=962627physical=73 logical=65 role=AIV icache_requests=977135physical=74 logical=66 role=AIV icache_requests=955129physical=75 logical=67 role=AIV icache_requests=959191physical=76 logical=68 role=AIV icache_requests=949527physical=77 logical=69 role=AIV icache_requests=963560physical=78 logical=70 role=AIV icache_requests=942189physical=79 logical=71 role=AIV icache_requests=946515physical=80 logical=72 role=AIV icache_requests=940590physical=81 logical=73 role=AIV icache_requests=944974physical=84 logical=74 role=AIV icache_requests=951880physical=85 logical=75 role=AIV icache_requests=932902physical=86 logical=76 role=AIV icache_requests=956023physical=87 logical=77 role=AIV icache_requests=961393physical=88 logical=78 role=AIV icache_requests=947517physical=89 logical=79 role=AIV icache_requests=968721physical=90 logical=80 role=AIV icache_requests=956194physical=91 logical=81 role=AIV icache_requests=946326physical=92 logical=82 role=AIV icache_requests=941321physical=93 logical=83 role=AIV icache_requests=952971physical=94 logical=84 role=AIV icache_requests=963528physical=95 logical=85 role=AIV icache_requests=944293physical=96 logical=86 role=AIV icache_requests=946400physical=97 logical=87 role=AIV icache_requests=938503physical=98 logical=88 role=AIV icache_requests=961806physical=99 logical=89 role=AIV icache_requests=928721physical=100 logical=90 role=AIV icache_requests=936502physical=101 logical=91 role=AIV icache_requests=938993physical=102 logical=92 role=AIV icache_requests=946830physical=103 logical=93 role=AIV icache_requests=933095physical=104 logical=94 role=AIV icache_requests=954807physical=105 logical=95 role=AIV icache_requests=947467 + +
+ +
+
Primary I-cache misses/core
+ + + + 62,283 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_misses=3628physical=1 logical=1 role=AIC icache_misses=3424physical=2 logical=2 role=AIC icache_misses=4109physical=3 logical=3 role=AIC icache_misses=4472physical=4 logical=4 role=AIC icache_misses=4342physical=5 logical=5 role=AIC icache_misses=4395physical=6 logical=6 role=AIC icache_misses=4086physical=7 logical=7 role=AIC icache_misses=4414physical=8 logical=8 role=AIC icache_misses=3678physical=9 logical=9 role=AIC icache_misses=4335physical=10 logical=10 role=AIC icache_misses=3763physical=12 logical=11 role=AIC icache_misses=3874physical=13 logical=12 role=AIC icache_misses=3883physical=14 logical=13 role=AIC icache_misses=4347physical=15 logical=14 role=AIC icache_misses=4219physical=16 logical=15 role=AIC icache_misses=4490physical=54 logical=16 role=AIC icache_misses=4048physical=55 logical=17 role=AIC icache_misses=4501physical=56 logical=18 role=AIC icache_misses=4113physical=57 logical=19 role=AIC icache_misses=4392physical=58 logical=20 role=AIC icache_misses=4454physical=60 logical=21 role=AIC icache_misses=4410physical=61 logical=22 role=AIC icache_misses=4013physical=62 logical=23 role=AIC icache_misses=4177physical=63 logical=24 role=AIC icache_misses=4259physical=64 logical=25 role=AIC icache_misses=4509physical=65 logical=26 role=AIC icache_misses=4239physical=66 logical=27 role=AIC icache_misses=3983physical=67 logical=28 role=AIC icache_misses=4338physical=68 logical=29 role=AIC icache_misses=4533physical=69 logical=30 role=AIC icache_misses=4651physical=70 logical=31 role=AIC icache_misses=4648physical=18 logical=32 role=AIV icache_misses=59809physical=19 logical=33 role=AIV icache_misses=59562physical=20 logical=34 role=AIV icache_misses=60123physical=21 logical=35 role=AIV icache_misses=61347physical=22 logical=36 role=AIV icache_misses=59253physical=23 logical=37 role=AIV icache_misses=60644physical=24 logical=38 role=AIV icache_misses=61250physical=25 logical=39 role=AIV icache_misses=60185physical=26 logical=40 role=AIV icache_misses=60681physical=27 logical=41 role=AIV icache_misses=60666physical=28 logical=42 role=AIV icache_misses=59843physical=29 logical=43 role=AIV icache_misses=61861physical=30 logical=44 role=AIV icache_misses=60111physical=31 logical=45 role=AIV icache_misses=60253physical=32 logical=46 role=AIV icache_misses=59942physical=33 logical=47 role=AIV icache_misses=59682physical=34 logical=48 role=AIV icache_misses=59191physical=35 logical=49 role=AIV icache_misses=60703physical=36 logical=50 role=AIV icache_misses=60102physical=37 logical=51 role=AIV icache_misses=60840physical=38 logical=52 role=AIV icache_misses=60968physical=39 logical=53 role=AIV icache_misses=61199physical=42 logical=54 role=AIV icache_misses=60820physical=43 logical=55 role=AIV icache_misses=60494physical=44 logical=56 role=AIV icache_misses=60877physical=45 logical=57 role=AIV icache_misses=59597physical=46 logical=58 role=AIV icache_misses=62140physical=47 logical=59 role=AIV icache_misses=60319physical=48 logical=60 role=AIV icache_misses=59571physical=49 logical=61 role=AIV icache_misses=60078physical=50 logical=62 role=AIV icache_misses=61543physical=51 logical=63 role=AIV icache_misses=60789physical=72 logical=64 role=AIV icache_misses=61586physical=73 logical=65 role=AIV icache_misses=62283physical=74 logical=66 role=AIV icache_misses=61323physical=75 logical=67 role=AIV icache_misses=61547physical=76 logical=68 role=AIV icache_misses=60404physical=77 logical=69 role=AIV icache_misses=61552physical=78 logical=70 role=AIV icache_misses=59837physical=79 logical=71 role=AIV icache_misses=60566physical=80 logical=72 role=AIV icache_misses=59849physical=81 logical=73 role=AIV icache_misses=60026physical=84 logical=74 role=AIV icache_misses=60276physical=85 logical=75 role=AIV icache_misses=59195physical=86 logical=76 role=AIV icache_misses=61079physical=87 logical=77 role=AIV icache_misses=61435physical=88 logical=78 role=AIV icache_misses=60108physical=89 logical=79 role=AIV icache_misses=62256physical=90 logical=80 role=AIV icache_misses=60974physical=91 logical=81 role=AIV icache_misses=60192physical=92 logical=82 role=AIV icache_misses=59967physical=93 logical=83 role=AIV icache_misses=60128physical=94 logical=84 role=AIV icache_misses=61506physical=95 logical=85 role=AIV icache_misses=60186physical=96 logical=86 role=AIV icache_misses=60477physical=97 logical=87 role=AIV icache_misses=59674physical=98 logical=88 role=AIV icache_misses=61628physical=99 logical=89 role=AIV icache_misses=58674physical=100 logical=90 role=AIV icache_misses=59453physical=101 logical=91 role=AIV icache_misses=59922physical=102 logical=92 role=AIV icache_misses=60326physical=103 logical=93 role=AIV icache_misses=59074physical=104 logical=94 role=AIV icache_misses=60977physical=105 logical=95 role=AIV icache_misses=60573 + +
+
+

逐核原始主计数

+

报告展示 primary request/miss；phase 模式的 I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和，只用于标记全窗 capture gap；scalar shadow 同样用于核验 phase 读清重建，并显示其相对 CNT2 primary 的 capture loss，不作为第二份性能数据展示。

+
+ + + + + + + +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate90ns量尺 µs
00AIC004,458,8425,461.7838,158,5028,009,490960,4393,6280.378%326.520
11AIC104,483,2095,570.8768,393,9898,247,2311,003,4123,4240.341%308.160
22AIC204,497,3735,466.8418,122,4817,962,316942,3604,1090.436%369.810
33AIC304,600,9695,860.2258,846,2208,659,4351,049,4814,4720.426%402.480
44AIC404,448,4945,485.0048,150,1417,998,373956,6334,3420.454%390.780
55AIC504,336,1425,475.0877,973,2347,810,979962,9874,3950.456%395.550
66AIC604,487,5915,469.3288,200,1618,048,057955,1704,0860.428%367.740
77AIC704,393,9655,458.9258,085,2737,897,869965,5834,4140.457%397.260
88AIC804,476,0235,459.5668,196,3658,051,997957,1053,6780.384%331.020
99AIC904,528,3445,735.3628,547,7728,361,5401,031,3534,3350.420%390.150
1010AIC1004,553,3455,538.8398,422,6788,284,474970,8633,7630.388%338.670
1211AIC1104,501,0985,453.5008,143,6267,972,014947,9253,8740.409%348.660
1312AIC1204,574,0865,600.3698,459,2458,293,687993,8913,8830.391%349.470
1413AIC1304,396,4425,582.4348,147,3607,978,439980,2124,3470.443%391.230
1514AIC1404,528,9435,480.7758,137,7047,972,079936,7164,2190.450%379.710
1615AIC1504,362,7545,475.9857,997,8737,841,185960,6984,4900.467%404.100
1832AIV015,461,6896,032.2509,678,3968,235,844943,03659,8096.342%5,382.810
1933AIV025,404,3216,016.1389,565,0978,037,858937,65159,5626.352%5,360.580
2034AIV115,389,2326,000.0709,534,1558,084,037943,61560,1236.372%5,411.070
2135AIV125,425,3866,029.2729,665,1338,079,126956,46761,3476.414%5,521.230
2236AIV215,376,1676,017.6299,482,5228,120,894933,13859,2536.350%5,332.770
2337AIV225,402,2155,982.8109,589,3898,106,940951,15660,6446.376%5,457.960
2438AIV315,515,4006,024.5719,830,1108,358,552953,64661,2506.423%5,512.500
2539AIV325,364,3505,986.4569,512,2848,022,070943,69460,1856.378%5,416.650
2640AIV415,365,0976,004.8249,536,6348,265,112948,53860,6816.397%5,461.290
2741AIV425,365,0315,989.6209,505,3448,108,161945,65660,6666.415%5,459.940
2842AIV515,371,1465,981.3619,507,2858,179,502940,04659,8436.366%5,385.870
2943AIV525,367,8055,996.4299,612,8948,180,688960,63461,8616.440%5,567.490
3044AIV615,397,9726,006.2629,548,1158,231,489942,65060,1116.377%5,409.990
3145AIV625,320,1465,987.1949,426,4178,048,626943,33460,2536.387%5,422.770
3246AIV715,390,6736,001.2659,535,6068,143,204945,20459,9426.342%5,394.780
3347AIV725,360,9425,989.6889,519,2098,061,552945,40959,6826.313%5,371.380
3448AIV815,383,6256,000.1999,459,5488,177,487929,59359,1916.367%5,327.190
3549AIV825,298,9985,973.8619,402,4448,065,689951,15460,7036.382%5,463.270
3650AIV915,375,3716,004.1709,541,1038,175,185946,01360,1026.353%5,409.180
3751AIV925,396,5895,977.9319,575,8848,167,764953,88760,8406.378%5,475.600
3852AIV1015,352,3285,998.0739,526,3988,272,696955,19960,9686.383%5,487.120
3953AIV1025,322,5836,021.6979,475,3128,142,137957,04561,1996.395%5,507.910
4254AIV1115,385,3925,976.6889,574,3448,200,898955,28660,8206.367%5,473.800
4355AIV1125,377,1195,992.7859,514,2008,122,208951,47260,4946.358%5,444.460
4456AIV1215,412,0296,035.2729,584,6478,211,035958,71160,8776.350%5,478.930
4557AIV1225,365,7856,036.7619,501,1998,041,205947,14659,5976.292%5,363.730
4658AIV1315,353,3655,988.7169,586,9228,275,222972,28662,1406.391%5,592.600
4759AIV1325,341,2376,005.2579,450,1818,151,909947,64760,3196.365%5,428.710
4860AIV1415,363,5775,986.7229,509,2608,217,570950,13359,5716.270%5,361.390
4961AIV1425,316,5596,022.2459,397,3288,040,494944,67060,0786.360%5,407.020
5062AIV1515,406,5395,994.0029,604,8678,357,232958,39661,5436.421%5,538.870
5163AIV1525,359,2886,025.5449,486,6538,212,186948,31260,7896.410%5,471.010
5416AIC1604,496,4475,487.7818,232,8198,084,579955,7024,0480.424%364.320
5517AIC1704,598,9875,946.7048,795,3748,621,7761,042,0474,5010.432%405.090
5618AIC1804,458,5585,493.4098,122,9647,953,102949,4244,1130.433%370.170
5719AIC1904,360,8385,472.7167,990,1167,818,367965,9384,3920.455%395.280
5820AIC2004,428,0555,496.5857,938,7457,748,777939,1394,4540.474%400.860
6021AIC2104,475,6015,678.0138,363,3748,188,1751,004,5714,4100.439%396.900
6122AIC2204,587,4565,877.1488,596,6298,452,4011,010,2984,0130.397%361.170
6223AIC2304,494,6315,473.3238,293,3628,114,415969,8274,1770.431%375.930
6324AIC2404,437,1165,461.9788,147,1247,987,545963,2914,2590.442%383.310
6425AIC2504,456,2145,507.9318,284,2238,098,761993,6414,5090.454%405.810
6526AIC2604,534,2355,692.4168,454,0828,288,129994,9964,2390.426%381.510
6627AIC2704,431,5275,471.3628,206,0128,050,671987,1043,9830.404%358.470
6728AIC2804,430,9405,499.9828,133,1147,988,536963,1024,3380.450%390.420
6829AIC2904,547,5195,766.5948,541,8938,346,4991,011,7904,5330.448%407.970
6930AIC3004,437,8485,463.4707,928,7907,767,968935,7624,6510.497%418.590
7031AIC3104,655,4476,052.7259,023,3038,823,5671,069,2104,6480.435%418.320
7264AIV1615,333,8525,984.6269,507,0958,240,262962,62761,5866.398%5,542.740
7365AIV1625,398,6055,995.1079,689,3028,299,623977,13562,2836.374%5,605.470
7466AIV1715,377,8206,010.8489,551,0158,315,730955,12961,3236.420%5,519.070
7567AIV1725,306,3896,001.7089,444,3758,159,963959,19161,5476.417%5,539.230
7668AIV1815,389,5625,988.2819,604,2898,296,651949,52760,4046.361%5,436.360
7769AIV1825,375,8005,993.5759,605,9178,214,674963,56061,5526.388%5,539.680
7870AIV1915,319,7426,022.7489,398,5378,203,142942,18959,8376.351%5,385.330
7971AIV1925,297,8696,033.8369,416,2008,093,202946,51560,5666.399%5,450.940
8072AIV2015,350,4266,005.8569,462,0098,074,653940,59059,8496.363%5,386.410
8173AIV2025,286,1175,989.5929,345,1827,893,387944,97460,0266.352%5,402.340
8474AIV2115,396,9635,972.4969,575,8668,288,518951,88060,2766.332%5,424.840
8575AIV2125,392,9115,986.9259,519,1758,181,875932,90259,1956.345%5,327.550
8676AIV2215,391,6855,983.9469,590,6548,331,094956,02361,0796.389%5,497.110
8777AIV2225,349,3586,014.3649,551,7658,209,964961,39361,4356.390%5,529.150
8878AIV2315,427,9645,996.8669,616,9748,277,079947,51760,1086.344%5,409.720
8979AIV2325,352,7835,978.9519,579,1438,128,673968,72162,2566.427%5,603.040
9080AIV2415,371,4675,993.3349,516,9158,188,372956,19460,9746.377%5,487.660
9181AIV2425,306,2246,022.8189,392,6178,030,332946,32660,1926.361%5,417.280
9282AIV2515,402,1386,034.1459,507,3578,105,989941,32159,9676.371%5,397.030
9383AIV2525,344,6336,002.4169,449,2277,992,834952,97160,1286.310%5,411.520
9484AIV2615,430,4226,028.8439,661,5818,286,083963,52861,5066.383%5,535.540
9585AIV2625,340,2265,986.7309,426,3318,062,772944,29360,1866.374%5,416.740
9686AIV2715,347,1586,031.4279,496,8548,170,309946,40060,4776.390%5,442.930
9787AIV2725,297,3516,013.5479,383,8637,991,187938,50359,6746.358%5,370.660
9888AIV2815,395,8325,980.7759,573,4908,204,286961,80661,6286.408%5,546.520
9989AIV2825,564,3556,007.3879,728,2158,389,210928,72158,6746.318%5,280.660
10090AIV2915,433,7785,968.1159,573,3368,161,753936,50259,4536.348%5,350.770
10191AIV2925,430,1155,977.2609,582,7868,072,222938,99359,9226.382%5,392.980
10292AIV3015,412,7525,970.6539,570,1458,151,921946,83060,3266.371%5,429.340
10393AIV3025,358,8855,977.3369,413,4497,977,975933,09559,0746.331%5,316.660
10494AIV3115,405,5745,988.0359,599,6808,117,752954,80760,9776.386%5,487.930
10595AIV3125,373,7585,978.7029,498,0857,937,282947,46760,5736.393%5,451.570
+
+ diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_winner_build_control_20260723_044422_icache_report.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_winner_build_control_20260723_044422_icache_report.html new file mode 100644 index 0000000000..9256c93632 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/case1_winner_build_control_20260723_044422_icache_report.html @@ -0,0 +1,362 @@ + + + + + + 真实 FDWIC Scalar Submit PMU I-cache 报告 + + +
+

真实 FDWIC Scalar Submit PMU I-cache 报告

+

固定输入 fdwic_submit_pmu_raw.json · schema fdwic-submit-pmu-v3 · 32 AIC + 64 AIV

+ +
+

winner-build-control 阶段观察（phase_id=10）

+

边界 winner_build_begin_to_end_excluding_linked_kernel_calls · 计数语义 discontinuous_running_read_clear_excluding_linked_kernel_calls · + 时间语义 boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls · + call_shape=dynamic_balanced · expected_calls=ALL 1024 / AIC 512 / AIV 512

+ +

本报告使用 empty-bracket 估算进入 phase observed + 的局部记录代码开销。 + AIC/AIV 分别按“空区间每组记录开销 × 本阶段实际 begin/end 记录组数”计算，ALL + 再由两类核相加。参考值只从 phase observed 分子扣除该估算；原始 whole 分母保持不变， + 因为空区间没有测量 whole 窗口中的全部记录开销。所以下表的参考比例不是完整业务 + 阶段占完整调度窗口的比例，也不是精确校正。若参考值为负，表示记录代码开销估算已经 + 超过原始 observed；这只能说明该信号低于当前校准的分辨能力，不能截成零或解释成负的 + 业务事件。校准 raw：/home/q00473782/atomic/private/gpt/simpler/outputs/TestPagedAttentionUnroll_Case1_20260723_043533/fdwic_submit_pmu_raw.json · + SHA-256 56199f91545cc6a1fbf18b8bb2cfebae279cb5801516311a040d4facccf321ba。两份 provenance 已核验为同一场景、同一 Git revision 8c6b6e3a412d。

+ + +

本阶段若跨过 linked Kernel，会同时门控 SYS 时间与 PMU counter， + 因而 total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段。 + result-used return-ready atomic 依赖区间只从 SYS 边界累计值扣除，PMU counter + 仍含 atomic 指令事件；source-issue atomic + 保留在时间和计数口径内。没有 linked Kernel 的阶段不触发 PMU 停/开表； + 每次调用的分母仍是外层业务调用次数，不是 Begin/End 读数，也不是排除的 Kernel 调用数。

+ +

阶段原始 PMU 观测来自 running read-clear 的 total cycles，并按 AIC/AIV + 各自校准频率换算；提供 empty-bracket 时，页面主参考值再扣除局部记录代码开销估算。 + phase_elapsed_ticks 只是 SYS 边界闭合诊断，绝不按 1 GHz 当作阶段时间。 + Scalar busy 与 non-scalar residual（逐核先算 total−scalar）共同解释阶段 PMU total； + 未校准时只能展示 raw observed；提供 empty-bracket 后，记录代码开销估算只从 phase + observed 分子扣除，原始 whole 分母保持不变。当前数据无法得到完整业务阶段占完整调度 + 窗口的精确比例。 + linked vector/cube Kernel 从 SYS 与 PMU counter 一并门控排除；result-used return-ready atomic + 依赖区间只从 SYS 边界累计值扣除，I-cache/PMU counter 仍含其指令事件；source-issue atomic + 保留。request/miss 百分比的分母是同一 ELF、同一次采集的 Submit 整窗 primary。 + observed_plus_capture_gap = observed + (primary − shadow)； + 边界读数和插桩 bookkeeping 会进入 sample，因此“观测值”和“加全窗 capture gap”都不是原业务 + 事件数的数学上下界。不同 ELF 不能直接相减后冒充业务真值；提供 empty-bracket 时只按上方 + 明示的公式估算局部记录代码开销。request/miss 的逐核 min/max 是每核累计整个 phase + 的极值，不是逐调用极值。

+
+ + + + +
指标ALL96 核AIC32 核AIV64 核
Phase PMU total扣除记录代码开销估算后的参考值 Σ 60,658,419.285 cycles
≈ 36,766.154 µs（1.649844 cycles/ns）
原始 observed Σ 62,476,937 cycles（≈ 37,868.391 µs）− 记录代码开销估算 1,818,517.715 cycles
记录代码开销估算 / 原始 observed 2.911%；原始逐核 最小 0；最大 2,061,069；原始 61,012.634 cycles/call		扣除记录代码开销估算后的参考值 Σ 22,561,270.760 cycles
≈ 13,672.984 µs（1.650062 cycles/ns）
原始 observed Σ 23,432,983 cycles（≈ 14,201.274 µs）− 记录代码开销估算 871,712.240 cycles
记录代码开销估算 / 原始 observed 3.720%；原始逐核 最小 118,934；最大 1,809,865；原始 45,767.545 cycles/call		扣除记录代码开销估算后的参考值 Σ 38,097,148.526 cycles
≈ 23,092.946 µs（1.649731 cycles/ns）
原始 observed Σ 39,043,954 cycles（≈ 23,666.861 µs）− 记录代码开销估算 946,805.474 cycles
记录代码开销估算 / 原始 observed 2.425%；原始逐核 最小 0；最大 2,061,069；原始 76,257.723 cycles/call
Phase scalar busy扣除记录代码开销估算后的参考值 Σ 59,831,553.283 cycles
≈ 36,264.976 µs（1.649844 cycles/ns）
原始 observed Σ 61,009,092 cycles（≈ 36,978.703 µs）− 记录代码开销估算 1,177,538.717 cycles
记录代码开销估算 / 原始 observed 1.930%；原始逐核 最小 0；最大 2,039,969；原始 59,579.191 cycles/call		扣除记录代码开销估算后的参考值 Σ 22,245,908.163 cycles
≈ 13,481.862 µs（1.650062 cycles/ns）
原始 observed Σ 22,820,006 cycles（≈ 13,829.787 µs）− 记录代码开销估算 574,097.837 cycles
记录代码开销估算 / 原始 observed 2.516%；原始逐核 最小 107,836；最大 1,777,133；原始 44,570.324 cycles/call		扣除记录代码开销估算后的参考值 Σ 37,585,645.120 cycles
≈ 22,782.893 µs（1.649731 cycles/ns）
原始 observed Σ 38,189,086 cycles（≈ 23,148.675 µs）− 记录代码开销估算 603,440.880 cycles
记录代码开销估算 / 原始 observed 1.580%；原始逐核 最小 0；最大 2,039,969；原始 74,588.059 cycles/call
非 Scalar-busy 残余扣除记录代码开销估算后的参考值 Σ 826,866.003 cycles
≈ 501.178 µs（1.649844 cycles/ns）
原始 observed Σ 1,467,845 cycles（≈ 889.687 µs）− 记录代码开销估算 640,978.997 cycles
记录代码开销估算 / 原始 observed 43.668%；原始逐核 最小 0；最大 32,732；原始 1,433.442 cycles/call		扣除记录代码开销估算后的参考值 Σ 315,362.597 cycles
≈ 191.122 µs（1.650062 cycles/ns）
原始 observed Σ 612,977 cycles（≈ 371.487 µs）− 记录代码开销估算 297,614.403 cycles
记录代码开销估算 / 原始 observed 48.552%；原始逐核 最小 11,098；最大 32,732；原始 1,197.221 cycles/call		扣除记录代码开销估算后的参考值 Σ 511,503.405 cycles
≈ 310.053 µs（1.649731 cycles/ns）
原始 observed Σ 854,868 cycles（≈ 518.186 µs）− 记录代码开销估算 343,364.595 cycles
记录代码开销估算 / 原始 observed 40.166%；原始逐核 最小 0；最大 26,855；原始 1,669.664 cycles/call
阶段参考关系（raw 同格保留）扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 98.637%
参考 Non-scalar / 参考 Phase total 1.363%
参考 Phase total / 原始 whole total 7.295%；原始 observed 7.514%
参考 Phase scalar / 原始 whole scalar 7.549%；原始 observed 7.698%
whole scalar−shadow scalar：Σ 2,354 cycles；逐核 最小 0；最大 56		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 98.602%
参考 Non-scalar / 参考 Phase total 1.398%
参考 Phase total / 原始 whole total 8.473%；原始 observed 8.801%
参考 Phase scalar / 原始 whole scalar 8.407%；原始 observed 8.624%
whole scalar−shadow scalar：Σ 1,174 cycles；逐核 最小 26；最大 56		扣除局部记录代码开销估算后的参考比例
参考 Scalar / 参考 Phase total 98.657%
参考 Non-scalar / 参考 Phase total 1.343%
参考 Phase total / 原始 whole total 6.740%；原始 observed 6.907%
参考 Phase scalar / 原始 whole scalar 7.120%；原始 observed 7.234%
whole scalar−shadow scalar：Σ 1,180 cycles；逐核 最小 0；最大 30
Request（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 4,798,650.478
原始 observed 4,863,968 − 记录代码开销估算 65,317.522
参考值 / 原始整窗 6.501%；原始 observed / 原始整窗 6.589%
记录代码开销估算 / 原始 observed 1.343%
原始逐核 最小 0；最大 159,256；原始整窗 73,819,554
原始 capture gap +0；加 gap 后 4,863,968（6.589%）		扣除记录代码开销估算后的参考值 1,719,765.218
原始 observed 1,751,611 − 记录代码开销估算 31,845.782
参考值 / 原始整窗 7.182%；原始 observed / 原始整窗 7.315%
记录代码开销估算 / 原始 observed 1.818%
原始逐核 最小 21,757；最大 159,256；原始整窗 23,945,840
原始 capture gap +0；加 gap 后 1,751,611（7.315%）		扣除记录代码开销估算后的参考值 3,078,885.260
原始 observed 3,112,357 − 记录代码开销估算 33,471.740
参考值 / 原始整窗 6.173%；原始 observed / 原始整窗 6.240%
记录代码开销估算 / 原始 observed 1.075%
原始逐核 最小 0；最大 147,353；原始整窗 49,873,714
原始 capture gap +0；加 gap 后 3,112,357（6.240%）
Miss（参考主值 / raw 明细）扣除记录代码开销估算后的参考值 29,082.716
原始 observed 30,792 − 记录代码开销估算 1,709.284
参考值 / 原始整窗 1.882%；原始 observed / 原始整窗 1.993%
记录代码开销估算 / 原始 observed 5.551%
原始逐核 最小 0；最大 630；原始整窗 1,544,977
原始 capture gap +0；加 gap 后 30,792（1.993%）		扣除记录代码开销估算后的参考值 6,998.330
原始 observed 7,034 − 记录代码开销估算 35.670
参考值 / 原始整窗 18.533%；原始 observed / 原始整窗 18.628%
记录代码开销估算 / 原始 observed 0.507%
原始逐核 最小 154；最大 371；原始整窗 37,761
原始 capture gap +0；加 gap 后 7,034（18.628%）		扣除记录代码开销估算后的参考值 22,084.386
原始 observed 23,758 − 记录代码开销估算 1,673.614
参考值 / 原始整窗 1.465%；原始 observed / 原始整窗 1.576%
记录代码开销估算 / 原始 observed 7.044%
原始逐核 最小 0；最大 630；原始整窗 1,507,216
原始 capture gap +0；加 gap 后 23,758（1.576%）
SYS 边界诊断 / Begin-EndΣ 12,953,578 raw ticks
逐核 最小 0；最大 422,692；仅边界诊断
Begin / End：1,177 / 1,177
业务调用 1,024 次；排除 linked Kernel 调用 153 次
逐核 0–22；零调用核 1		Σ 4,480,543 raw ticks
逐核 最小 53,632；最大 422,692；仅边界诊断
Begin / End：587 / 587
业务调用 512 次；排除 linked Kernel 调用 75 次
逐核 12–22；零调用核 0		Σ 8,473,035 raw ticks
逐核 最小 0；最大 399,825；仅边界诊断
Begin / End：590 / 590
业务调用 512 次；排除 linked Kernel 调用 78 次
逐核 0–12；零调用核 1
+
+ + +
+

诊断构建身份

+

该 sidecar 在 case 返回后由实际加载路径冻结，并用 raw SHA 绑定；不会改写 raw， + 也不会给 AICore 热路径增加指令。

+
+
Provenance SHA-256
339c12cf200627c68516f6fa52044e17bd94742133a418d4de567d5cae35378a
+
Profile / extra cache
submit-pmu-winner-build-control / + 8b98d67f0ac0f71a
+
Profiled cache key
TestPagedAttentionUnroll / a5 / fully_distributed_within_core / submit-pmu-winner-build-control
+
Source state
source-v2:8c6b6e3a412d2c7c7113c2d8f714a14eb3786f80:c8eb04f51a89d059a063f01fd21afdab87e65e9e9b2076201812209fb83a9da5:bbaee2d61192c060196ed4f1a62878b369e5ca9a7869ac7330bb059849384ada
+
Source-state stamp
/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/8b98d67f0ac0f71a/aicore/.git_commit
+
Compile definitions
PTO_FDWIC_SUBMIT_PMU=1
PTO_FDWIC_SUBMIT_PMU_PHASE_ID=10
PTO_FDWIC_TRACE_ENABLED=0
+
+
+ + + +
实物文件大小文件 SHA-256.text 大小.text SHA-256采集时路径
AICore final2,663,7844b59f5399df35c81de5f4cc168b9e51edb2c6fd0f549070cb0fd481c2bd38e8b211,5368f0b1381c3b3358380b49ec49fb5a4560ca67e78fef35a1f34058b634fd765d8/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/aicore-extra/8b98d67f0ac0f71a/aicore_kernel.o
AIC combined1,991,808ce2b880a0b6bb7934eb234a9de8aff7928bc8e942302feef1ac1322a76427ec598,568610a05155467dfe2fcfd53f13f54f52dd760d505114999ec695a25beeb249e6e/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/8b98d67f0ac0f71a/aicore/aicore_aic_combined.o
AIV combined2,253,208636d59419500b35741fee810c55187c2bee2b29acfe34df9276c3fd30ff0f391112,72003c515266598b13e5e9006743b56e4ea03e6e736e9e7ec1b59ba41255662525b/home/q00473782/atomic/private/gpt/simpler/build/cache/a5/onboard/fully_distributed_within_core/aicore-extra/8b98d67f0ac0f71a/aicore/aicore_aiv_combined.o
Host runtime11,776,1440369d0602054bf612f86b58c294503e00486bd505bff9ee7a455994f043147cc454,6325ac381ed10be5f5d492b8072c051a177131f4caec6d6f6bc9bf50880da4bb0d9/home/q00473782/atomic/private/gpt/simpler/build/lib/a5/onboard/fully_distributed_within_core/libhost_runtime.so
+
+ +
+
原始全局 Submit 时间范围（墙钟） + 5,896.308 µs + 最早一核首个 Submit 至最晚一核末个 Submit +
+
每核 Submit 数 + 128096 核均已闭环 +
+
受信记录96 / 96owner、selector、status、拓扑均通过
+
raw SHA-256c7dbd0a7166e33c5ef79bb9227409f762aa9c7df433a2f97d94ed336fef580f6
+
+

linked vector/cube Kernel 整段从 SYS gate 边界累计值与 PMU counter + 一并门控排除。result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值 + 及命中的 phase SYS 边界诊断扣除，I-cache/PMU counter 仍包含其指令事件；source-issue atomic + 保留在时间和计数口径内。轮询和等待 flag 也仍是调度器实际开销。原始首末 SYS_CNT + 只证明窗口闭合，不参与时间占比；wall−scalar 是多种排除项与边界间隙的混合，不能称为纯 Kernel 时间。 + 90.000 ns 仅作 I-cache miss 的直觉量尺，不是 Submit 墙钟损失。 + miss 可能重叠、被流水隐藏，也可能与其他停顿共同出现；不能把 miss×90 ns + 当成可直接相减的优化收益。 + 非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall。卡片中的逐核 PMU 等效时间只解释 + 当前 ELF 的采集窗，不能与 perf-clock、swimlane 或另一个 phase ELF 相减。

+

ALL / AIC / AIV 汇总

+
+
+

ALL · 96 核

+
+
Submit PMU total/core
+
均值 8,661,575.0；最小 7,902,124；最大 9,484,919 cycles
+ 等效时间 均值 5,249.936；最小 4,789.619；最大 5,748.979 µs + （按 1.649844 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 8,255,504.6；最小 7,634,685；最大 8,840,266 cycles
+ 等效时间 均值 5,003.809；最小 4,627.519；最大 5,358.244 µs； + 加权占比 95.312% +
+
非 Scalar-busy 残余/core
+
均值 406,070.4；最小 42,065；最大 705,426 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 246.127；最小 25.496；最大 427.571 µs +
+
SYS gate 边界诊断/core
+
均值 3,089,118.4；最小 2,555,350；最大 4,456,313 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,596.072；最小 5,376.872；最大 5,892.837 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 2,506,953.2；最小 1,160,952；最大 3,198,651 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 55.202% +
+
Primary I-cache request/core
最小 694,057；最大 872,710
+
Primary I-cache miss/core
最小 972；最大 27,812
+
加权 miss rate
2.093%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 87.480；最大 2,503.080 µs
+
+
+ +
+

AIC · 32 核

+
+
Submit PMU total/core
+
均值 8,320,731.8；最小 7,902,124；最大 8,889,582 cycles
+ 等效时间 均值 5,042.678；最小 4,788.986；最大 5,387.423 µs + （按 1.650062 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 8,268,940.6；最小 7,848,037；最大 8,836,992 cycles
+ 等效时间 均值 5,011.291；最小 4,756.207；最大 5,355.551 µs； + 加权占比 99.378% +
+
非 Scalar-busy 残余/core
+
均值 51,791.2；最小 42,065；最大 67,305 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 31.387；最小 25.493；最大 40.789 µs +
+
SYS gate 边界诊断/core
+
均值 2,744,612.3；最小 2,555,350；最大 3,529,274 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,640.317；最小 5,554.967；最大 5,822.393 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 2,895,704.3；最小 2,032,485；最大 3,198,651 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 48.661% +
+
Primary I-cache request/core
最小 715,337；最大 846,868
+
Primary I-cache miss/core
最小 972；最大 1,562
+
加权 miss rate
0.158%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 87.480；最大 140.580 µs
+
+
+ +
+

AIV · 64 核

+
+
Submit PMU total/core
+
均值 8,831,996.6；最小 8,274,258；最大 9,484,919 cycles
+ 等效时间 均值 5,353.598；最小 5,015.519；最大 5,749.373 µs + （按 1.649731 cycles/ns 校准）；仍包含 atomic 指令事件 +
+
Scalar busy/core
+
均值 8,248,786.6；最小 7,634,685；最大 8,840,266 cycles
+ 等效时间 均值 5,000.080；最小 4,627.836；最大 5,358.611 µs； + 加权占比 93.397% +
+
非 Scalar-busy 残余/core
+
均值 583,210.0；最小 500,019；最大 705,426 cycles
+ 逐核先算 total−scalar；等效时间 + 均值 353.518；最小 303.091；最大 427.601 µs +
+
SYS gate 边界诊断/core
+
均值 3,261,371.5；最小 2,932,970；最大 4,456,313 raw ticks
+ 逐核累计 SYS_CNT gate 边界；linked vector/cube Kernel 与 result-used + return-ready atomic 依赖区间被扣除。它不是按 1.65 GHz 换算的 PMU 阶段时间。 +
+
原始 Submit 墙钟/core
+
均值 5,573.949；最小 5,376.872；最大 5,892.837 µs
+ 逐核首末 Submit 的 SYS_CNT 闭合证据；含 Kernel，与 PMU 校准时间是不同量。 +
+
未进入 SYS gate/core
+
均值 2,312,577.6；最小 1,160,952；最大 2,791,571 raw ticks
+ 逐核 wall−gate；混合了 linked Kernel、return-ready atomic 被扣区间 + 和门控边界间隙，不能解释成纯 Kernel 耗时；gate/墙钟加权比 + 58.511% +
+
Primary I-cache request/core
最小 694,057；最大 872,710
+
Primary I-cache miss/core
最小 18,973；最大 27,812
+
加权 miss rate
3.022%（Σmiss/Σrequest）
+
90 ns 直觉量尺/core
+
最小 1,707.570；最大 2,503.080 µs
+
+
+
+

逐物理核分布

+

AICAIV

+
+
+
SYS gate boundary-diagnostic ticks/core
+ + + + 4,456,313 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_submit_elapsed_ticks=2753276physical=1 logical=1 role=AIC scalar_submit_elapsed_ticks=2606049physical=2 logical=2 role=AIC scalar_submit_elapsed_ticks=2842921physical=3 logical=3 role=AIC scalar_submit_elapsed_ticks=2556987physical=4 logical=4 role=AIC scalar_submit_elapsed_ticks=3006036physical=5 logical=5 role=AIC scalar_submit_elapsed_ticks=2569274physical=6 logical=6 role=AIC scalar_submit_elapsed_ticks=2680127physical=7 logical=7 role=AIC scalar_submit_elapsed_ticks=2555445physical=8 logical=8 role=AIC scalar_submit_elapsed_ticks=2667420physical=9 logical=9 role=AIC scalar_submit_elapsed_ticks=2556587physical=10 logical=10 role=AIC scalar_submit_elapsed_ticks=2938909physical=12 logical=11 role=AIC scalar_submit_elapsed_ticks=3529274physical=13 logical=12 role=AIC scalar_submit_elapsed_ticks=2723671physical=14 logical=13 role=AIC scalar_submit_elapsed_ticks=2579853physical=15 logical=14 role=AIC scalar_submit_elapsed_ticks=2890988physical=16 logical=15 role=AIC scalar_submit_elapsed_ticks=2633914physical=54 logical=16 role=AIC scalar_submit_elapsed_ticks=2759698physical=55 logical=17 role=AIC scalar_submit_elapsed_ticks=2581827physical=56 logical=18 role=AIC scalar_submit_elapsed_ticks=2980954physical=57 logical=19 role=AIC scalar_submit_elapsed_ticks=2584370physical=58 logical=20 role=AIC scalar_submit_elapsed_ticks=2789311physical=60 logical=21 role=AIC scalar_submit_elapsed_ticks=2555350physical=61 logical=22 role=AIC scalar_submit_elapsed_ticks=2835866physical=62 logical=23 role=AIC scalar_submit_elapsed_ticks=2768588physical=63 logical=24 role=AIC scalar_submit_elapsed_ticks=2910646physical=64 logical=25 role=AIC scalar_submit_elapsed_ticks=2626342physical=65 logical=26 role=AIC scalar_submit_elapsed_ticks=2898290physical=66 logical=27 role=AIC scalar_submit_elapsed_ticks=2589402physical=67 logical=28 role=AIC scalar_submit_elapsed_ticks=2889052physical=68 logical=29 role=AIC scalar_submit_elapsed_ticks=2618099physical=69 logical=30 role=AIC scalar_submit_elapsed_ticks=2701464physical=70 logical=31 role=AIC scalar_submit_elapsed_ticks=2647603physical=18 logical=32 role=AIV scalar_submit_elapsed_ticks=3356880physical=19 logical=33 role=AIV scalar_submit_elapsed_ticks=3749474physical=20 logical=34 role=AIV scalar_submit_elapsed_ticks=3166695physical=21 logical=35 role=AIV scalar_submit_elapsed_ticks=3006157physical=22 logical=36 role=AIV scalar_submit_elapsed_ticks=3286482physical=23 logical=37 role=AIV scalar_submit_elapsed_ticks=3101571physical=24 logical=38 role=AIV scalar_submit_elapsed_ticks=3412756physical=25 logical=39 role=AIV scalar_submit_elapsed_ticks=2934076physical=26 logical=40 role=AIV scalar_submit_elapsed_ticks=3188396physical=27 logical=41 role=AIV scalar_submit_elapsed_ticks=3024770physical=28 logical=42 role=AIV scalar_submit_elapsed_ticks=3236834physical=29 logical=43 role=AIV scalar_submit_elapsed_ticks=3068956physical=30 logical=44 role=AIV scalar_submit_elapsed_ticks=3369518physical=31 logical=45 role=AIV scalar_submit_elapsed_ticks=3265589physical=32 logical=46 role=AIV scalar_submit_elapsed_ticks=3128255physical=33 logical=47 role=AIV scalar_submit_elapsed_ticks=3156524physical=34 logical=48 role=AIV scalar_submit_elapsed_ticks=4456313physical=35 logical=49 role=AIV scalar_submit_elapsed_ticks=3194710physical=36 logical=50 role=AIV scalar_submit_elapsed_ticks=3645144physical=37 logical=51 role=AIV scalar_submit_elapsed_ticks=3090291physical=38 logical=52 role=AIV scalar_submit_elapsed_ticks=3451564physical=39 logical=53 role=AIV scalar_submit_elapsed_ticks=3124287physical=42 logical=54 role=AIV scalar_submit_elapsed_ticks=3302435physical=43 logical=55 role=AIV scalar_submit_elapsed_ticks=3155025physical=44 logical=56 role=AIV scalar_submit_elapsed_ticks=3223689physical=45 logical=57 role=AIV scalar_submit_elapsed_ticks=3102053physical=46 logical=58 role=AIV scalar_submit_elapsed_ticks=3058664physical=47 logical=59 role=AIV scalar_submit_elapsed_ticks=3097201physical=48 logical=60 role=AIV scalar_submit_elapsed_ticks=3172667physical=49 logical=61 role=AIV scalar_submit_elapsed_ticks=3077504physical=50 logical=62 role=AIV scalar_submit_elapsed_ticks=3417480physical=51 logical=63 role=AIV scalar_submit_elapsed_ticks=3204632physical=72 logical=64 role=AIV scalar_submit_elapsed_ticks=3268734physical=73 logical=65 role=AIV scalar_submit_elapsed_ticks=3316912physical=74 logical=66 role=AIV scalar_submit_elapsed_ticks=3160824physical=75 logical=67 role=AIV scalar_submit_elapsed_ticks=3034916physical=76 logical=68 role=AIV scalar_submit_elapsed_ticks=3097568physical=77 logical=69 role=AIV scalar_submit_elapsed_ticks=3067197physical=78 logical=70 role=AIV scalar_submit_elapsed_ticks=3079325physical=79 logical=71 role=AIV scalar_submit_elapsed_ticks=2983782physical=80 logical=72 role=AIV scalar_submit_elapsed_ticks=3188364physical=81 logical=73 role=AIV scalar_submit_elapsed_ticks=2932970physical=84 logical=74 role=AIV scalar_submit_elapsed_ticks=3249150physical=85 logical=75 role=AIV scalar_submit_elapsed_ticks=4100137physical=86 logical=76 role=AIV scalar_submit_elapsed_ticks=3333880physical=87 logical=77 role=AIV scalar_submit_elapsed_ticks=3345169physical=88 logical=78 role=AIV scalar_submit_elapsed_ticks=3277818physical=89 logical=79 role=AIV scalar_submit_elapsed_ticks=3312059physical=90 logical=80 role=AIV scalar_submit_elapsed_ticks=3179319physical=91 logical=81 role=AIV scalar_submit_elapsed_ticks=3042882physical=92 logical=82 role=AIV scalar_submit_elapsed_ticks=3735218physical=93 logical=83 role=AIV scalar_submit_elapsed_ticks=3168625physical=94 logical=84 role=AIV scalar_submit_elapsed_ticks=3243642physical=95 logical=85 role=AIV scalar_submit_elapsed_ticks=3341985physical=96 logical=86 role=AIV scalar_submit_elapsed_ticks=3166723physical=97 logical=87 role=AIV scalar_submit_elapsed_ticks=3350424physical=98 logical=88 role=AIV scalar_submit_elapsed_ticks=3046604physical=99 logical=89 role=AIV scalar_submit_elapsed_ticks=4219445physical=100 logical=90 role=AIV scalar_submit_elapsed_ticks=3229705physical=101 logical=91 role=AIV scalar_submit_elapsed_ticks=3069410physical=102 logical=92 role=AIV scalar_submit_elapsed_ticks=3172717physical=103 logical=93 role=AIV scalar_submit_elapsed_ticks=3425075physical=104 logical=94 role=AIV scalar_submit_elapsed_ticks=3077056physical=105 logical=95 role=AIV scalar_submit_elapsed_ticks=3283547 + +
+ +
+
PMU total cycles/core
+ + + + 9,484,919 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC total_cycles=8088533physical=1 logical=1 role=AIC total_cycles=8539817physical=2 logical=2 role=AIC total_cycles=8275340physical=3 logical=3 role=AIC total_cycles=8410527physical=4 logical=4 role=AIC total_cycles=8549391physical=5 logical=5 role=AIC total_cycles=7997836physical=6 logical=6 role=AIC total_cycles=8259648physical=7 logical=7 role=AIC total_cycles=8696952physical=8 logical=8 role=AIC total_cycles=8487327physical=9 logical=9 role=AIC total_cycles=7966654physical=10 logical=10 role=AIC total_cycles=8530866physical=12 logical=11 role=AIC total_cycles=8427603physical=13 logical=12 role=AIC total_cycles=8407884physical=14 logical=13 role=AIC total_cycles=7902124physical=15 logical=14 role=AIC total_cycles=8301684physical=16 logical=15 role=AIC total_cycles=7935911physical=54 logical=16 role=AIC total_cycles=8536283physical=55 logical=17 role=AIC total_cycles=8350880physical=56 logical=18 role=AIC total_cycles=8457362physical=57 logical=19 role=AIC total_cycles=8115067physical=58 logical=20 role=AIC total_cycles=8433918physical=60 logical=21 role=AIC total_cycles=8183703physical=61 logical=22 role=AIC total_cycles=7985342physical=62 logical=23 role=AIC total_cycles=8428029physical=63 logical=24 role=AIC total_cycles=8889582physical=64 logical=25 role=AIC total_cycles=8096804physical=65 logical=26 role=AIC total_cycles=8422838physical=66 logical=27 role=AIC total_cycles=8494112physical=67 logical=28 role=AIC total_cycles=8105479physical=68 logical=29 role=AIC total_cycles=8318863physical=69 logical=30 role=AIC total_cycles=8464897physical=70 logical=31 role=AIC total_cycles=8202161physical=18 logical=32 role=AIV total_cycles=8543913physical=19 logical=33 role=AIV total_cycles=8676116physical=20 logical=34 role=AIV total_cycles=8836687physical=21 logical=35 role=AIV total_cycles=9114650physical=22 logical=36 role=AIV total_cycles=8277710physical=23 logical=37 role=AIV total_cycles=9003516physical=24 logical=38 role=AIV total_cycles=8914453physical=25 logical=39 role=AIV total_cycles=8467813physical=26 logical=40 role=AIV total_cycles=9269700physical=27 logical=41 role=AIV total_cycles=8861943physical=28 logical=42 role=AIV total_cycles=8937136physical=29 logical=43 role=AIV total_cycles=8618370physical=30 logical=44 role=AIV total_cycles=8419499physical=31 logical=45 role=AIV total_cycles=9139788physical=32 logical=46 role=AIV total_cycles=9132133physical=33 logical=47 role=AIV total_cycles=8807793physical=34 logical=48 role=AIV total_cycles=9088447physical=35 logical=49 role=AIV total_cycles=8815259physical=36 logical=50 role=AIV total_cycles=8609159physical=37 logical=51 role=AIV total_cycles=8596965physical=38 logical=52 role=AIV total_cycles=8879252physical=39 logical=53 role=AIV total_cycles=8961398physical=42 logical=54 role=AIV total_cycles=8888150physical=43 logical=55 role=AIV total_cycles=8274258physical=44 logical=56 role=AIV total_cycles=9282532physical=45 logical=57 role=AIV total_cycles=8712270physical=46 logical=58 role=AIV total_cycles=8888463physical=47 logical=59 role=AIV total_cycles=8619405physical=48 logical=60 role=AIV total_cycles=8516127physical=49 logical=61 role=AIV total_cycles=8652325physical=50 logical=62 role=AIV total_cycles=8805922physical=51 logical=63 role=AIV total_cycles=8993352physical=72 logical=64 role=AIV total_cycles=9047673physical=73 logical=65 role=AIV total_cycles=8646367physical=74 logical=66 role=AIV total_cycles=8913391physical=75 logical=67 role=AIV total_cycles=8324714physical=76 logical=68 role=AIV total_cycles=8983669physical=77 logical=69 role=AIV total_cycles=8624594physical=78 logical=70 role=AIV total_cycles=8577325physical=79 logical=71 role=AIV total_cycles=9023822physical=80 logical=72 role=AIV total_cycles=8715266physical=81 logical=73 role=AIV total_cycles=8797719physical=84 logical=74 role=AIV total_cycles=8824376physical=85 logical=75 role=AIV total_cycles=8995653physical=86 logical=76 role=AIV total_cycles=8785617physical=87 logical=77 role=AIV total_cycles=9484919physical=88 logical=78 role=AIV total_cycles=8688610physical=89 logical=79 role=AIV total_cycles=9009950physical=90 logical=80 role=AIV total_cycles=8815185physical=91 logical=81 role=AIV total_cycles=9043337physical=92 logical=82 role=AIV total_cycles=9076374physical=93 logical=83 role=AIV total_cycles=8394157physical=94 logical=84 role=AIV total_cycles=9200887physical=95 logical=85 role=AIV total_cycles=8984731physical=96 logical=86 role=AIV total_cycles=8519111physical=97 logical=87 role=AIV total_cycles=9202793physical=98 logical=88 role=AIV total_cycles=8853229physical=99 logical=89 role=AIV total_cycles=8948872physical=100 logical=90 role=AIV total_cycles=8786252physical=101 logical=91 role=AIV total_cycles=8919279physical=102 logical=92 role=AIV total_cycles=8739091physical=103 logical=93 role=AIV total_cycles=9235045physical=104 logical=94 role=AIV total_cycles=8570481physical=105 logical=95 role=AIV total_cycles=8910791 + +
+ +
+
Scalar busy cycles/core
+ + + + 8,840,266 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC scalar_busy=8042830physical=1 logical=1 role=AIC scalar_busy=8484976physical=2 logical=2 role=AIC scalar_busy=8223988physical=3 logical=3 role=AIC scalar_busy=8359351physical=4 logical=4 role=AIC scalar_busy=8490131physical=5 logical=5 role=AIC scalar_busy=7943464physical=6 logical=6 role=AIC scalar_busy=8212969physical=7 logical=7 role=AIC scalar_busy=8646157physical=8 logical=8 role=AIC scalar_busy=8445262physical=9 logical=9 role=AIC scalar_busy=7914180physical=10 logical=10 role=AIC scalar_busy=8484693physical=12 logical=11 role=AIC scalar_busy=8381207physical=13 logical=12 role=AIC scalar_busy=8358095physical=14 logical=13 role=AIC scalar_busy=7848037physical=15 logical=14 role=AIC scalar_busy=8245178physical=16 logical=15 role=AIC scalar_busy=7882554physical=54 logical=16 role=AIC scalar_busy=8490637physical=55 logical=17 role=AIC scalar_busy=8302087physical=56 logical=18 role=AIC scalar_busy=8410310physical=57 logical=19 role=AIC scalar_busy=8060462physical=58 logical=20 role=AIC scalar_busy=8367415physical=60 logical=21 role=AIC scalar_busy=8131196physical=61 logical=22 role=AIC scalar_busy=7925418physical=62 logical=23 role=AIC scalar_busy=8376622physical=63 logical=24 role=AIC scalar_busy=8836992physical=64 logical=25 role=AIC scalar_busy=8045982physical=65 logical=26 role=AIC scalar_busy=8374081physical=66 logical=27 role=AIC scalar_busy=8446917physical=67 logical=28 role=AIC scalar_busy=8038174physical=68 logical=29 role=AIC scalar_busy=8264852physical=69 logical=30 role=AIC scalar_busy=8413534physical=70 logical=31 role=AIC scalar_busy=8158347physical=18 logical=32 role=AIV scalar_busy=7952889physical=19 logical=33 role=AIV scalar_busy=8157472physical=20 logical=34 role=AIV scalar_busy=8295604physical=21 logical=35 role=AIV scalar_busy=8560567physical=22 logical=36 role=AIV scalar_busy=7634685physical=23 logical=37 role=AIV scalar_busy=8398621physical=24 logical=38 role=AIV scalar_busy=8414434physical=25 logical=39 role=AIV scalar_busy=7919556physical=26 logical=40 role=AIV scalar_busy=8732657physical=27 logical=41 role=AIV scalar_busy=8280047physical=28 logical=42 role=AIV scalar_busy=8365058physical=29 logical=43 role=AIV scalar_busy=8019601physical=30 logical=44 role=AIV scalar_busy=7832003physical=31 logical=45 role=AIV scalar_busy=8527029physical=32 logical=46 role=AIV scalar_busy=8581378physical=33 logical=47 role=AIV scalar_busy=8181267physical=34 logical=48 role=AIV scalar_busy=8556953physical=35 logical=49 role=AIV scalar_busy=8195587physical=36 logical=50 role=AIV scalar_busy=8036555physical=37 logical=51 role=AIV scalar_busy=7987689physical=38 logical=52 role=AIV scalar_busy=8312076physical=39 logical=53 role=AIV scalar_busy=8366456physical=42 logical=54 role=AIV scalar_busy=8297812physical=43 logical=55 role=AIV scalar_busy=7677053physical=44 logical=56 role=AIV scalar_busy=8719120physical=45 logical=57 role=AIV scalar_busy=8178204physical=46 logical=58 role=AIV scalar_busy=8284340physical=47 logical=59 role=AIV scalar_busy=8006136physical=48 logical=60 role=AIV scalar_busy=7988760physical=49 logical=61 role=AIV scalar_busy=8147478physical=50 logical=62 role=AIV scalar_busy=8185678physical=51 logical=63 role=AIV scalar_busy=8357054physical=72 logical=64 role=AIV scalar_busy=8441186physical=73 logical=65 role=AIV scalar_busy=8022698physical=74 logical=66 role=AIV scalar_busy=8327026physical=75 logical=67 role=AIV scalar_busy=7678562physical=76 logical=68 role=AIV scalar_busy=8433873physical=77 logical=69 role=AIV scalar_busy=7998472physical=78 logical=70 role=AIV scalar_busy=7997525physical=79 logical=71 role=AIV scalar_busy=8406289physical=80 logical=72 role=AIV scalar_busy=8179819physical=81 logical=73 role=AIV scalar_busy=8208934physical=84 logical=74 role=AIV scalar_busy=8286245physical=85 logical=75 role=AIV scalar_busy=8446328physical=86 logical=76 role=AIV scalar_busy=8197419physical=87 logical=77 role=AIV scalar_busy=8840266physical=88 logical=78 role=AIV scalar_busy=8145621physical=89 logical=79 role=AIV scalar_busy=8385886physical=90 logical=80 role=AIV scalar_busy=8257854physical=91 logical=81 role=AIV scalar_busy=8402005physical=92 logical=82 role=AIV scalar_busy=8519895physical=93 logical=83 role=AIV scalar_busy=7850351physical=94 logical=84 role=AIV scalar_busy=8604260physical=95 logical=85 role=AIV scalar_busy=8371537physical=96 logical=86 role=AIV scalar_busy=7994210physical=97 logical=87 role=AIV scalar_busy=8517967physical=98 logical=88 role=AIV scalar_busy=8253247physical=99 logical=89 role=AIV scalar_busy=8407992physical=100 logical=90 role=AIV scalar_busy=8220955physical=101 logical=91 role=AIV scalar_busy=8334153physical=102 logical=92 role=AIV scalar_busy=8146584physical=103 logical=93 role=AIV scalar_busy=8529619physical=104 logical=94 role=AIV scalar_busy=8005663physical=105 logical=95 role=AIV scalar_busy=8360082 + +
+ +
+
Primary I-cache requests/core
+ + + + 872,710 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_requests=727501physical=1 logical=1 role=AIC icache_requests=732968physical=2 logical=2 role=AIC icache_requests=766457physical=3 logical=3 role=AIC icache_requests=728489physical=4 logical=4 role=AIC icache_requests=846868physical=5 logical=5 role=AIC icache_requests=753661physical=6 logical=6 role=AIC icache_requests=724221physical=7 logical=7 role=AIC icache_requests=731251physical=8 logical=8 role=AIC icache_requests=715337physical=9 logical=9 role=AIC icache_requests=736887physical=10 logical=10 role=AIC icache_requests=719735physical=12 logical=11 role=AIC icache_requests=732091physical=13 logical=12 role=AIC icache_requests=734571physical=14 logical=13 role=AIC icache_requests=753522physical=15 logical=14 role=AIC icache_requests=762556physical=16 logical=15 role=AIC icache_requests=749528physical=54 logical=16 role=AIC icache_requests=723240physical=55 logical=17 role=AIC icache_requests=738563physical=56 logical=18 role=AIC icache_requests=715736physical=57 logical=19 role=AIC icache_requests=740038physical=58 logical=20 role=AIC icache_requests=766160physical=60 logical=21 role=AIC icache_requests=732155physical=61 logical=22 role=AIC icache_requests=774867physical=62 logical=23 role=AIC icache_requests=744321physical=63 logical=24 role=AIC icache_requests=808389physical=64 logical=25 role=AIC icache_requests=737075physical=65 logical=26 role=AIC icache_requests=797309physical=66 logical=27 role=AIC icache_requests=739879physical=67 logical=28 role=AIC icache_requests=808701physical=68 logical=29 role=AIC icache_requests=734243physical=69 logical=30 role=AIC icache_requests=720046physical=70 logical=31 role=AIC icache_requests=749475physical=18 logical=32 role=AIV icache_requests=825433physical=19 logical=33 role=AIV icache_requests=764005physical=20 logical=34 role=AIV icache_requests=778696physical=21 logical=35 role=AIV icache_requests=763172physical=22 logical=36 role=AIV icache_requests=797283physical=23 logical=37 role=AIV icache_requests=782631physical=24 logical=38 role=AIV icache_requests=730322physical=25 logical=39 role=AIV icache_requests=742008physical=26 logical=40 role=AIV icache_requests=776636physical=27 logical=41 role=AIV icache_requests=742419physical=28 logical=42 role=AIV icache_requests=775585physical=29 logical=43 role=AIV icache_requests=788478physical=30 logical=44 role=AIV icache_requests=779003physical=31 logical=45 role=AIV icache_requests=821363physical=32 logical=46 role=AIV icache_requests=759017physical=33 logical=47 role=AIV icache_requests=782214physical=34 logical=48 role=AIV icache_requests=706425physical=35 logical=49 role=AIV icache_requests=794251physical=36 logical=50 role=AIV icache_requests=741419physical=37 logical=51 role=AIV icache_requests=749284physical=38 logical=52 role=AIV icache_requests=870126physical=39 logical=53 role=AIV icache_requests=781527physical=42 logical=54 role=AIV icache_requests=789296physical=43 logical=55 role=AIV icache_requests=812032physical=44 logical=56 role=AIV icache_requests=786565physical=45 logical=57 role=AIV icache_requests=799384physical=46 logical=58 role=AIV icache_requests=753092physical=47 logical=59 role=AIV icache_requests=765078physical=48 logical=60 role=AIV icache_requests=770096physical=49 logical=61 role=AIV icache_requests=757956physical=50 logical=62 role=AIV icache_requests=755089physical=51 logical=63 role=AIV icache_requests=758260physical=72 logical=64 role=AIV icache_requests=806507physical=73 logical=65 role=AIV icache_requests=800768physical=74 logical=66 role=AIV icache_requests=758955physical=75 logical=67 role=AIV icache_requests=761474physical=76 logical=68 role=AIV icache_requests=735559physical=77 logical=69 role=AIV icache_requests=763397physical=78 logical=70 role=AIV icache_requests=736357physical=79 logical=71 role=AIV icache_requests=749202physical=80 logical=72 role=AIV icache_requests=791191physical=81 logical=73 role=AIV icache_requests=751642physical=84 logical=74 role=AIV icache_requests=771263physical=85 logical=75 role=AIV icache_requests=773546physical=86 logical=76 role=AIV icache_requests=808002physical=87 logical=77 role=AIV icache_requests=855264physical=88 logical=78 role=AIV icache_requests=761323physical=89 logical=79 role=AIV icache_requests=872710physical=90 logical=80 role=AIV icache_requests=781931physical=91 logical=81 role=AIV icache_requests=761726physical=92 logical=82 role=AIV icache_requests=784567physical=93 logical=83 role=AIV icache_requests=809909physical=94 logical=84 role=AIV icache_requests=771578physical=95 logical=85 role=AIV icache_requests=858932physical=96 logical=86 role=AIV icache_requests=785485physical=97 logical=87 role=AIV icache_requests=859054physical=98 logical=88 role=AIV icache_requests=743901physical=99 logical=89 role=AIV icache_requests=694057physical=100 logical=90 role=AIV icache_requests=784287physical=101 logical=91 role=AIV icache_requests=754869physical=102 logical=92 role=AIV icache_requests=746245physical=103 logical=93 role=AIV icache_requests=820317physical=104 logical=94 role=AIV icache_requests=751769physical=105 logical=95 role=AIV icache_requests=869782 + +
+ +
+
Primary I-cache misses/core
+ + + + 27,812 + 0 + physical 0 + physical 107 + physical=0 logical=0 role=AIC icache_misses=1044physical=1 logical=1 role=AIC icache_misses=1172physical=2 logical=2 role=AIC icache_misses=1104physical=3 logical=3 role=AIC icache_misses=1171physical=4 logical=4 role=AIC icache_misses=1089physical=5 logical=5 role=AIC icache_misses=1146physical=6 logical=6 role=AIC icache_misses=1077physical=7 logical=7 role=AIC icache_misses=1153physical=8 logical=8 role=AIC icache_misses=1067physical=9 logical=9 role=AIC icache_misses=1239physical=10 logical=10 role=AIC icache_misses=1117physical=12 logical=11 role=AIC icache_misses=977physical=13 logical=12 role=AIC icache_misses=1284physical=14 logical=13 role=AIC icache_misses=1305physical=15 logical=14 role=AIC icache_misses=1361physical=16 logical=15 role=AIC icache_misses=1221physical=54 logical=16 role=AIC icache_misses=1195physical=55 logical=17 role=AIC icache_misses=1090physical=56 logical=18 role=AIC icache_misses=1055physical=57 logical=19 role=AIC icache_misses=1301physical=58 logical=20 role=AIC icache_misses=1269physical=60 logical=21 role=AIC icache_misses=1370physical=61 logical=22 role=AIC icache_misses=1328physical=62 logical=23 role=AIC icache_misses=1299physical=63 logical=24 role=AIC icache_misses=1123physical=64 logical=25 role=AIC icache_misses=1192physical=65 logical=26 role=AIC icache_misses=1007physical=66 logical=27 role=AIC icache_misses=1034physical=67 logical=28 role=AIC icache_misses=1562physical=68 logical=29 role=AIC icache_misses=1332physical=69 logical=30 role=AIC icache_misses=1105physical=70 logical=31 role=AIC icache_misses=972physical=18 logical=32 role=AIV icache_misses=23993physical=19 logical=33 role=AIV icache_misses=19365physical=20 logical=34 role=AIV icache_misses=22832physical=21 logical=35 role=AIV icache_misses=22540physical=22 logical=36 role=AIV icache_misses=26875physical=23 logical=37 role=AIV icache_misses=23974physical=24 logical=38 role=AIV icache_misses=21923physical=25 logical=39 role=AIV icache_misses=22449physical=26 logical=40 role=AIV icache_misses=22574physical=27 logical=41 role=AIV icache_misses=21965physical=28 logical=42 role=AIV icache_misses=24863physical=29 logical=43 role=AIV icache_misses=22663physical=30 logical=44 role=AIV icache_misses=24244physical=31 logical=45 role=AIV icache_misses=23302physical=32 logical=46 role=AIV icache_misses=24188physical=33 logical=47 role=AIV icache_misses=25277physical=34 logical=48 role=AIV icache_misses=21091physical=35 logical=49 role=AIV icache_misses=23507physical=36 logical=50 role=AIV icache_misses=23736physical=37 logical=51 role=AIV icache_misses=23844physical=38 logical=52 role=AIV icache_misses=22791physical=39 logical=53 role=AIV icache_misses=22072physical=42 logical=54 role=AIV icache_misses=24271physical=43 logical=55 role=AIV icache_misses=22711physical=44 logical=56 role=AIV icache_misses=25218physical=45 logical=57 role=AIV icache_misses=21614physical=46 logical=58 role=AIV icache_misses=23827physical=47 logical=59 role=AIV icache_misses=22927physical=48 logical=60 role=AIV icache_misses=22496physical=49 logical=61 role=AIV icache_misses=20859physical=50 logical=62 role=AIV icache_misses=24361physical=51 logical=63 role=AIV icache_misses=23418physical=72 logical=64 role=AIV icache_misses=25087physical=73 logical=65 role=AIV icache_misses=24212physical=74 logical=66 role=AIV icache_misses=23381physical=75 logical=67 role=AIV icache_misses=24773physical=76 logical=68 role=AIV icache_misses=23128physical=77 logical=69 role=AIV icache_misses=25078physical=78 logical=70 role=AIV icache_misses=23512physical=79 logical=71 role=AIV icache_misses=23767physical=80 logical=72 role=AIV icache_misses=23121physical=81 logical=73 role=AIV icache_misses=23581physical=84 logical=74 role=AIV icache_misses=23548physical=85 logical=75 role=AIV icache_misses=22301physical=86 logical=76 role=AIV icache_misses=25210physical=87 logical=77 role=AIV icache_misses=24520physical=88 logical=78 role=AIV icache_misses=23264physical=89 logical=79 role=AIV icache_misses=24911physical=90 logical=80 role=AIV icache_misses=22598physical=91 logical=81 role=AIV icache_misses=24110physical=92 logical=82 role=AIV icache_misses=24277physical=93 logical=83 role=AIV icache_misses=22229physical=94 logical=84 role=AIV icache_misses=23503physical=95 logical=85 role=AIV icache_misses=23304physical=96 logical=86 role=AIV icache_misses=21164physical=97 logical=87 role=AIV icache_misses=27298physical=98 logical=88 role=AIV icache_misses=23425physical=99 logical=89 role=AIV icache_misses=18973physical=100 logical=90 role=AIV icache_misses=25165physical=101 logical=91 role=AIV icache_misses=24899physical=102 logical=92 role=AIV icache_misses=24664physical=103 logical=93 role=AIV icache_misses=27812physical=104 logical=94 role=AIV icache_misses=24273physical=105 logical=95 role=AIV icache_misses=24358 + +
+
+

逐核原始主计数

+

报告展示 primary request/miss；phase 模式的 I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和，只用于标记全窗 capture gap；scalar shadow 同样用于核验 phase 读清重建，并显示其相对 CNT2 primary 的 capture loss，不作为第二份性能数据展示。

+
+ + + + + + + +
PhysicalLogicalRoleBlockLaneSYS gate 诊断 raw ticks原始墙钟 µsTotal cyclesScalar busyI$ requestI$ missMiss rate90ns量尺 µs
00AIC002,753,2765,560.8958,088,5338,042,830727,5011,0440.144%93.960
11AIC102,606,0495,657.1968,539,8178,484,976732,9681,1720.160%105.480
22AIC202,842,9215,631.2938,275,3408,223,988766,4571,1040.144%99.360
33AIC302,556,9875,649.3728,410,5278,359,351728,4891,1710.161%105.390
44AIC403,006,0365,629.5378,549,3918,490,131846,8681,0890.129%98.010
55AIC502,569,2745,624.4167,997,8367,943,464753,6611,1460.152%103.140
66AIC602,680,1275,633.2068,259,6488,212,969724,2211,0770.149%96.930
77AIC702,555,4455,754.0968,696,9528,646,157731,2511,1530.158%103.770
88AIC802,667,4205,645.9288,487,3278,445,262715,3371,0670.149%96.030
99AIC902,556,5875,586.3807,966,6547,914,180736,8871,2390.168%111.510
1010AIC1002,938,9095,664.7058,530,8668,484,693719,7351,1170.155%100.530
1211AIC1103,529,2745,561.7598,427,6038,381,207732,0919770.133%87.930
1312AIC1202,723,6715,675.9218,407,8848,358,095734,5711,2840.175%115.560
1413AIC1302,579,8535,597.4597,902,1247,848,037753,5221,3050.173%117.450
1514AIC1402,890,9885,600.4718,301,6848,245,178762,5561,3610.178%122.490
1615AIC1502,633,9145,605.7387,935,9117,882,554749,5281,2210.163%109.890
1832AIV013,356,8805,537.5578,543,9137,952,889825,43323,9932.907%2,159.370
1933AIV023,749,4745,423.6278,676,1168,157,472764,00519,3652.535%1,742.850
2034AIV113,166,6955,416.9808,836,6878,295,604778,69622,8322.932%2,054.880
2135AIV123,006,1575,797.7289,114,6508,560,567763,17222,5402.953%2,028.600
2236AIV213,286,4825,381.1628,277,7107,634,685797,28326,8753.371%2,418.750
2337AIV223,101,5715,732.3199,003,5168,398,621782,63123,9743.063%2,157.660
2438AIV313,412,7565,514.6058,914,4538,414,434730,32221,9233.002%1,973.070
2539AIV322,934,0765,383.5298,467,8137,919,556742,00822,4493.025%2,020.410
2640AIV413,188,3965,892.8379,269,7008,732,657776,63622,5742.907%2,031.660
2741AIV423,024,7705,433.1768,861,9438,280,047742,41921,9652.959%1,976.850
2842AIV513,236,8345,479.1478,937,1368,365,058775,58524,8633.206%2,237.670
2943AIV523,068,9565,391.8318,618,3708,019,601788,47822,6632.874%2,039.670
3044AIV613,369,5185,376.8728,419,4997,832,003779,00324,2443.112%2,181.960
3145AIV623,265,5895,766.5199,139,7888,527,029821,36323,3022.837%2,097.180
3246AIV713,128,2555,810.4989,132,1338,581,378759,01724,1883.187%2,176.920
3347AIV723,156,5245,671.0068,807,7938,181,267782,21425,2773.231%2,274.930
3448AIV814,456,3135,617.2659,088,4478,556,953706,42521,0912.986%1,898.190
3549AIV823,194,7105,407.2228,815,2598,195,587794,25123,5072.960%2,115.630
3650AIV913,645,1445,449.9988,609,1598,036,555741,41923,7363.201%2,136.240
3751AIV923,090,2915,379.1408,596,9657,987,689749,28423,8443.182%2,145.960
3852AIV1013,451,5645,390.5118,879,2528,312,076870,12622,7912.619%2,051.190
3953AIV1023,124,2875,598.5148,961,3988,366,456781,52722,0722.824%1,986.480
4254AIV1113,302,4355,613.0868,888,1508,297,812789,29624,2713.075%2,184.390
4355AIV1123,155,0255,399.1528,274,2587,677,053812,03222,7112.797%2,043.990
4456AIV1213,223,6895,851.2249,282,5328,719,120786,56525,2183.206%2,269.620
4557AIV1223,102,0535,450.2048,712,2708,178,204799,38421,6142.704%1,945.260
4658AIV1313,058,6645,584.9328,888,4638,284,340753,09223,8273.164%2,144.430
4759AIV1323,097,2015,557.1328,619,4058,006,136765,07822,9272.997%2,063.430
4860AIV1413,172,6675,433.1118,516,1277,988,760770,09622,4962.921%2,024.640
4961AIV1423,077,5045,411.5148,652,3258,147,478757,95620,8592.752%1,877.310
5062AIV1513,417,4805,560.1738,805,9228,185,678755,08924,3613.226%2,192.490
5163AIV1523,204,6325,731.4028,993,3528,357,054758,26023,4183.088%2,107.620
5416AIC1602,759,6985,723.9918,536,2838,490,637723,2401,1950.165%107.550
5517AIC1702,581,8275,660.2578,350,8808,302,087738,5631,0900.148%98.100
5618AIC1802,980,9545,615.1808,457,3628,410,310715,7361,0550.147%94.950
5719AIC1902,584,3705,579.1218,115,0678,060,462740,0381,3010.176%117.090
5820AIC2002,789,3115,742.5928,433,9188,367,415766,1601,2690.166%114.210
6021AIC2102,555,3505,563.5018,183,7038,131,196732,1551,3700.187%123.300
6122AIC2202,835,8665,597.6947,985,3427,925,418774,8671,3280.171%119.520
6223AIC2302,768,5885,757.5398,428,0298,376,622744,3211,2990.175%116.910
6324AIC2402,910,6465,822.3938,889,5828,836,992808,3891,1230.139%101.070
6425AIC2502,626,3425,563.8498,096,8048,045,982737,0751,1920.162%107.280
6526AIC2602,898,2905,584.9348,422,8388,374,081797,3091,0070.126%90.630
6627AIC2702,589,4025,691.7138,494,1128,446,917739,8791,0340.140%93.060
6728AIC2802,889,0525,654.8558,105,4798,038,174808,7011,5620.193%140.580
6829AIC2902,618,0995,554.9678,318,8638,264,852734,2431,3320.181%119.880
6930AIC3002,701,4645,655.0598,464,8978,413,534720,0461,1050.153%99.450
7031AIC3102,647,6035,644.1148,202,1618,158,347749,4759720.130%87.480
7264AIV1613,268,7345,759.6519,047,6738,441,186806,50725,0873.111%2,257.830
7365AIV1623,316,9125,409.9818,646,3678,022,698800,76824,2123.024%2,179.080
7466AIV1713,160,8245,726.2098,913,3918,327,026758,95523,3813.081%2,104.290
7567AIV1723,034,9165,476.6908,324,7147,678,562761,47424,7733.253%2,229.570
7668AIV1813,097,5685,606.4618,983,6698,433,873735,55923,1283.144%2,081.520
7769AIV1823,067,1975,444.8468,624,5947,998,472763,39725,0783.285%2,257.020
7870AIV1913,079,3255,416.7668,577,3257,997,525736,35723,5123.193%2,116.080
7971AIV1922,983,7825,739.8529,023,8228,406,289749,20223,7673.172%2,139.030
8072AIV2013,188,3645,396.4348,715,2668,179,819791,19123,1212.922%2,080.890
8173AIV2022,932,9705,604.0868,797,7198,208,934751,64223,5813.137%2,122.290
8474AIV2113,249,1505,625.7528,824,3768,286,245771,26323,5483.053%2,119.320
8575AIV2124,100,1375,673.4618,995,6538,446,328773,54622,3012.883%2,007.090
8676AIV2213,333,8805,606.3478,785,6178,197,419808,00225,2103.120%2,268.900
8777AIV2223,345,1695,867.6719,484,9198,840,266855,26424,5202.867%2,206.800
8878AIV2313,277,8185,542.6908,688,6108,145,621761,32323,2643.056%2,093.760
8979AIV2323,312,0595,790.8909,009,9508,385,886872,71024,9112.854%2,241.990
9080AIV2413,179,3195,671.1748,815,1858,257,854781,93122,5982.890%2,033.820
9181AIV2423,042,8825,599.9239,043,3378,402,005761,72624,1103.165%2,169.900
9282AIV2513,735,2185,614.7949,076,3748,519,895784,56724,2773.094%2,184.930
9383AIV2523,168,6255,419.8378,394,1577,850,351809,90922,2292.745%2,000.610
9484AIV2613,243,6425,799.6339,200,8878,604,260771,57823,5033.046%2,115.270
9585AIV2623,341,9855,719.4908,984,7318,371,537858,93223,3042.713%2,097.360
9686AIV2713,166,7235,388.0628,519,1117,994,210785,48521,1642.694%1,904.760
9787AIV2723,350,4245,647.4469,202,7938,517,967859,05427,2983.178%2,456.820
9888AIV2813,046,6045,590.0008,853,2298,253,247743,90123,4253.149%2,108.250
9989AIV2824,219,4455,424.8498,948,8728,407,992694,05718,9732.734%1,707.570
10090AIV2913,229,7055,548.7598,786,2528,220,955784,28725,1653.209%2,264.850
10191AIV2923,069,4105,571.8068,919,2798,334,153754,86924,8993.298%2,240.910
10292AIV3013,172,7175,519.6738,739,0918,146,584746,24524,6643.305%2,219.760
10393AIV3023,425,0755,826.0469,235,0458,529,619820,31727,8123.390%2,503.080
10494AIV3113,077,0565,576.1828,570,4818,005,663751,76924,2733.229%2,184.570
10595AIV3123,283,5475,683.3078,910,7918,360,082869,78224,3582.800%2,192.220
+
+ diff --git a/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/fdwic_submit_span_overview.html b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/fdwic_submit_span_overview.html new file mode 100644 index 0000000000..8a4c523468 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_record/2026-07-23/fdwic_submit_span_overview.html @@ -0,0 +1,1548 @@ + + + + + + FDWIC Submit 全 span 证据汇总 + + +
+

FDWIC Submit 全 span 证据汇总

+

PASS · 96 核 · 每核 1,280 个 Submit · + 泳道 raw 重新执行 schema-v4 严格闭合 · 13/13 PMU raw + provenance 闭合。 + PASS 只表示两条证据链各自通过门禁，不表示它们已绑定到同一 ELF。

+
这里有两条互不混算的证据链。 + 泳道图的百分比只在同一个泳道 ELF 的排他时间树中相加；每个 Submit-PMU profile 都是独立 ELF， + phase PMU total、Scalar busy、非 Scalar-busy 残余、request 和 miss 只能使用本 ELF 自己的分母。 + 不同 ELF 的绝对值不能直接相减，各 PMU 行的占比也不能相加成 100%。业务 phase 的主显示值 + 先从 raw observed 分子中扣除 empty-bracket 估算的局部记录代码开销，再除以本 ELF 的 raw whole； + raw observed / raw whole 同格保留。empty-bracket 没有测到 whole 窗口中的全部记录工作，因此这个 + 主显示值只是更接近业务量级的参考值，不是“纯业务阶段 / 纯业务整窗”的精确占比。
+ +
+

口径与来源

+

泳道 SYS counter 频率：1,000,000,000 Hz；它用于时间戳换算， + 不是约 1.65 GHz 的 PMU cycle 频率。泳道全局 Submit 墙钟范围： + 4844.066 µs。

+

SYS 边界诊断排除了 linked Vector/Cube Kernel 和 result-used return-ready atomic 的等待区间； + PMU total、Scalar busy 与 primary I-cache 来自嵌在首末 Submit SYS closure 内且遇 linked Kernel + 会暂停的 PMU gate，但 PMU counter 仍保留 atomic 指令及等待事件。阶段原始 PMU 观测使用 + running read-clear total；Scalar busy 独立观测，非 Scalar-busy 残余逐核按 total−scalar 得到。 + 页面再用 empty-bracket 估算进入 phase 分子的局部记录代码并给出参考值；SYS phase tick 只核验 + 边界闭合。PMU/I-cache counter 仍含 atomic 指令及观察代码事件，局部 observed 不是零插桩 + 函数体的数学上下界。

+

本批 PMU 来自 1 组 revision：8c6b6e3a412d； + 每个业务 phase 与 empty-bracket 必须来自同一场景和同一 revision，才允许计算局部记录开销参考值； + 其他跨 ELF 数值仍只作明示的数量级对照。泳道 raw 没有 build provenance，页面只证明 raw SHA + 与 schema-v4 重算闭合， + 不伪称已证明当时 ELF 身份；泳道与 PMU 之间仅对齐 96 核拓扑和每核 Submit 数。

+
+ +

泳道 ELF：同一份业务时间分布

+

泳道分区展示原始业务 elapsed，不等同于纯 Scalar 时间。Kernel 是父 span 内的嵌套事件； + 当前 analyzer 能精确拆开 EfDrain 与 FinalDrain；其他 containment（本轮包括 WinnerBuild）只有事件数、 + 没有独立 Kernel union 时长。因此纯 Scalar 归因只看下方相应 PMU control ELF，不从泳道父 span 猜减。

+

分区表的 PMU 与 scalar 两列来自对应 phase 自己的独立 ELF。主值的分子是 + “raw phase observed − 按 AIC/AIV 分别估算的局部记录代码开销”，分母仍是该 ELF 的 raw whole + PMU total 或 raw whole scalar；raw 比例作为次要信息保留。它们并非泳道父区间，也不是 + submit-pmu-none，所以各行不得相加；control-only 只代表排除 linked + Kernel 后的控制路径；没有等价 phase 的行显示“—”。SubmitUnion 表专门改用每核均值： + 时间列是平均每核时间，三个占比也分别由同口径的每核均值相除；表尾四层拆分只作跨 ELF + 数量级诊断。

+ +
+

SubmitEnvelope

+
+
+ + + + + + + + + + + + + + +
区域Σ core-time占比对照AIC 每核 min–maxAIV 每核 min–max
泳道同父区间PMU 局部记录扣除参考
扣局部记录估算后的 Phase PMU / raw whole PMU；raw 比例同格保留		Scalar 局部记录扣除参考
扣局部记录估算后的 Phase scalar / raw whole scalar；raw 比例同格保留
SubmitUnion387,187.472 µs85.384%——3923.407–4019.264 µs3988.838–4117.076 µs
BetweenSubmitResidual66,278.926 µs14.616%13.982%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 35.938% · 记录估算/raw 61.094%中等依赖校准 · adjacent-submit-boundary10.595%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 26.423% · 记录估算/raw 59.901%中等依赖校准 · adjacent-submit-boundary555.018–639.839 µs692.312–795.162 µs
+
+ + +
+

SubmitUnion

+
+
+ + + + + + + + + + + + + + +
区域平均每核时间均值占比对照AIC 每核 min–maxAIV 每核 min–max
泳道每核均值 / 同父区间每核均值PMU 局部记录扣除参考
扣局部记录估算后的 Phase PMU 每核均值 / raw whole PMU 每核均值；raw 比例同格保留		Scalar 局部记录扣除参考
扣局部记录估算后的 Phase scalar 每核均值 / raw whole scalar 每核均值；raw 比例同格保留
EfDrain652.784 µs16.185%7.924%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 31.409% · 记录估算/raw 74.772%较高依赖校准 · control-only7.002%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 23.595% · 记录估算/raw 70.323%较高依赖校准 · control-only829.745–1043.623 µs421.651–599.408 µs
Materialize1,021.330 µs25.323%20.469%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 42.825% · 记录估算/raw 52.203%中等依赖校准 · same-business-boundary22.393%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 38.698% · 记录估算/raw 42.133%中等依赖校准 · same-business-boundary982.441–1044.824 µs1007.761–1079.123 µs
PrepareMap217.533 µs5.394%2.274%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 26.781% · 记录估算/raw 91.510%高度依赖校准 · same-business-boundary2.751%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 19.638% · 记录估算/raw 85.993%高度依赖校准 · same-business-boundary175.588–248.810 µs203.320–302.751 µs
Claim824.475 µs20.442%15.675%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 38.921% · 记录估算/raw 59.726%中等依赖校准 · same-business-boundary14.243%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 30.723% · 记录估算/raw 53.640%中等依赖校准 · same-business-boundary540.557–586.652 µs906.563–1005.941 µs
Fanin16.088 µs0.399%0.295%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 0.538% · 记录估算/raw 45.091%中等依赖校准 · same-business-boundary0.280%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 0.450% · 记录估算/raw 37.809%较低依赖校准 · same-business-boundary15.964–21.094 µs3.393–24.326 µs
Register504.846 µs12.517%6.769%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 29.845% · 记录估算/raw 77.319%较高依赖校准 · same-business-boundary5.962%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 22.615% · 记录估算/raw 73.637%较高依赖校准 · same-business-boundary407.881–514.917 µs468.701–570.286 µs
WinnerBuild67.174 µs1.666%7.295%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 7.514% · 记录估算/raw 2.911%较低依赖校准 · control-only7.549%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 7.698% · 记录估算/raw 1.930%较低依赖校准 · control-only73.077–145.321 µs36.526–74.041 µs
AllocComplete8.686 µs0.215%6.824%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 6.872% · 记录估算/raw 0.698%较低依赖校准 · control-only7.059%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 7.091% · 记录估算/raw 0.455%较低依赖校准 · control-only4.807–69.631 µs0.000–0.000 µs
LoserReplay128.294 µs3.181%2.381%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 21.941% · 记录估算/raw 89.150%高度依赖校准 · same-business-boundary2.331%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 16.281% · 记录估算/raw 85.680%高度依赖校准 · same-business-boundary118.306–142.011 µs117.838–141.241 µs
SubmitInternalResidual437.856 µs10.856%————
SubmitTailResidual154.136 µs3.822%————
SubmitUnion 平均每核时间合计4,033.203 µs11 个分段每核均值之和100.000%同一泳道 ELF 的严格分区闭合；不与下方独立 PMU ELF 的时间直接相减。
原始分段观测合计10 个独立 phase ELF
取数范围10 个 phase 每核均值之和9 个直接表格行 + ArgBuild 精确子段SubmitTransition 位于 SubmitUnion 外，不进入分子
PMU total raw observed20,402,206.302 cycles/core按 1.649844 cycles/ns：≈ 12,366.143 µs/coreraw / none：210.132%AIC 202.220% · AIV 213.948%
Scalar busy raw observed14,794,127.906 cycles/core按 1.649844 cycles/ns：≈ 8,966.986 µs/coreraw / none：156.398%AIC 152.913% · AIV 158.133%
非 Scalar-busy 残余 raw observed5,608,078.396 cycles/core按 1.649844 cycles/ns：≈ 3,399.157 µs/coreraw / none：2243.804%AIC 11294.549% · AIV 1707.616%
I-cache request raw observed1,231,663.781 events/coreraw / none：170.918%AIC 176.247% · AIV 168.347%
I-cache miss raw observed35,614.781 events/coreraw / none：437.029%AIC 1108.084% · AIV 416.649%

合计 9 个直接映射行和 ArgBuild 的 Claim→Materialize 精确子段；SubmitInternalResidual、SubmitTailResidual 的其余 residual 不进入分子。raw 含分段记录代码自身开销，不能直接拿来解释业务耗时。

空区间估算的记录代码开销按 AIC/AIV 分开缩放后加权
估算方法空区间每次记录开销 × 本组记录次数ALL 8,729.385 组/core · AIC 8,740.844 · AIV 8,723.656linked Kernel pause/resume 产生的额外记录也已计入
PMU total 记录开销估算13,659,690.794 cycles/core按 1.649844 cycles/ns：≈ 8,279.383 µs/core占 raw：66.952%；相对 none：140.688%AIC 67.738% · AIV 66.594%
Scalar busy 记录开销估算8,797,834.613 cycles/core按 1.649844 cycles/ns：≈ 5,332.525 µs/core占 raw：59.468%；相对 none：93.007%AIC 59.259% · AIV 59.569%
非 Scalar-busy 残余 记录开销估算4,861,856.181 cycles/core按 1.649844 cycles/ns：≈ 2,946.858 µs/core占 raw：86.694%；相对 none：1945.239%AIC 93.565% · AIV 84.001%
I-cache request 记录开销估算488,007.644 events/core占 raw：39.622%；相对 none：67.721%AIC 38.243% · AIV 40.318%
I-cache miss 记录开销估算16,674.260 events/core占 raw：46.818%；相对 none：204.610%AIC 6.652% · AIV 50.063%

empty-bracket 只在固定调用点测量一对紧邻 begin/end；这里按每个 phase 实际 begin/end 记录组数缩放。调用点、I-cache 布局和运行轮次不同，所以只能作为估算。

扣除上述估算后的参考值raw − 记录开销估算
参考值口径原始观测 − 上述记录开销估算允许为负；负值表示估算或运行波动已超过 raw它不是“真实业务值”，也不是精确校正结果
PMU total 扣除后参考值6,742,515.508 cycles/core按 1.649844 cycles/ns：≈ 4,086.759 µs/core参考值 / none：69.444%AIC 65.239% · AIV 71.472%
Scalar busy 扣除后参考值5,996,293.293 cycles/core按 1.649844 cycles/ns：≈ 3,634.461 µs/core参考值 / none：63.391%AIC 62.299% · AIV 63.934%
非 Scalar-busy 残余 扣除后参考值746,222.215 cycles/core按 1.649844 cycles/ns：≈ 452.299 µs/core参考值 / none：298.565%AIC 726.815% · AIV 273.195%
I-cache request 扣除后参考值743,656.138 events/core参考值 / none：103.197%AIC 108.844% · AIV 100.473%
I-cache miss 扣除后参考值18,940.522 events/core参考值 / none：232.419%AIC 1034.373% · AIV 208.064%

该行用于判断此前 PMU/Scalar 合计膨胀主要来自哪里；它不是同一 ELF 的闭合，不能当成已精确恢复的业务值。

submit-pmu-none每核均值对照
对照范围首个 Submit begin → 末个 Submit end包含 BetweenSubmitResidual / SubmitTransition独立 ELF；不是与分子完全同范围的一次闭合
PMU total 每核均值9,709,219.854 cycles/core按 1.649844 cycles/ns：≈ 5,884.932 µs/core对照基准：100%AIC 9,476,082.219 cycles/core · AIV 9,825,788.672 cycles/core
Scalar busy 每核均值9,459,283.625 cycles/core按 1.649844 cycles/ns：≈ 5,733.441 µs/core对照基准：100%AIC 9,434,146.188 cycles/core · AIV 9,471,852.344 cycles/core
非 Scalar-busy 残余 每核均值249,936.229 cycles/core按 1.649844 cycles/ns：≈ 151.491 µs/core对照基准：100%AIC 41,936.031 cycles/core · AIV 353,936.328 cycles/core
I-cache request 每核均值720,615.750 events/core对照基准：100%AIC 703,537.906 events/core · AIV 729,154.672 events/core
I-cache miss 每核均值8,149.302 events/core对照基准：100%AIC 720.594 events/core · AIV 11,863.656 events/core
+
+ + +
+

EfDrain

+
+
+ + + + + + + + + + + + + + +
区域Σ core-time占比对照AIC 每核 min–maxAIV 每核 min–max
泳道同父区间PMU 局部记录扣除参考
扣局部记录估算后的 Phase PMU / raw whole PMU；raw 比例同格保留		Scalar 局部记录扣除参考
扣局部记录估算后的 Phase scalar / raw whole scalar；raw 比例同格保留
EfDrainKernelUnion32,086.895 µs51.202%——367.195–710.148 µs113.969–326.572 µs
EfDrainControl30,580.414 µs48.798%7.924%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 31.409% · 记录估算/raw 74.772%较高依赖校准 · control-only7.002%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 23.595% · 记录估算/raw 70.323%较高依赖校准 · control-only295.582–474.451 µs210.889–361.879 µs
+
+ + +
外围 Worker / Orchestration / FinalDrain 闭合 + +
+

WorkerCompletion

+
+
+ + + + + + + + + + + + + + +
区域Σ core-time占比对照AIC 每核 min–maxAIV 每核 min–max
泳道同父区间PMU 局部记录扣除参考
扣局部记录估算后的 Phase PMU / raw whole PMU；raw 比例同格保留		Scalar 局部记录扣除参考
扣局部记录估算后的 Phase scalar / raw whole scalar；raw 比例同格保留
OrchestrationReplay454,607.852 µs99.345%——4556.667–4620.691 µs4778.712–4852.567 µs
FinalDrain2,996.943 µs0.655%——5.249–301.052 µs2.535–81.451 µs
+
+ + +
+

OrchestrationReplay

+
+
+ + + + + + + + + + + + + + +
区域Σ core-time占比对照AIC 每核 min–maxAIV 每核 min–max
泳道同父区间PMU 局部记录扣除参考
扣局部记录估算后的 Phase PMU / raw whole PMU；raw 比例同格保留		Scalar 局部记录扣除参考
扣局部记录估算后的 Phase scalar / raw whole scalar；raw 比例同格保留
OrchestrationSetup1,062.557 µs0.234%——10.035–13.323 µs10.151–13.400 µs
SubmitUnion387,187.472 µs85.170%——3923.407–4019.264 µs3988.838–4117.076 µs
BetweenSubmitResidual66,278.926 µs14.579%13.982%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 35.938% · 记录估算/raw 61.094%中等依赖校准 · adjacent-submit-boundary10.595%扣局部记录估算后的阶段值 / raw 整窗，仅参考raw 26.423% · 记录估算/raw 59.901%中等依赖校准 · adjacent-submit-boundary555.018–639.839 µs692.312–795.162 µs
OrchestrationTail78.897 µs0.017%——0.437–1.079 µs0.671–1.167 µs
+
+ + +
+

FinalDrain

+
+
+ + + + + + + + + + + + + + +
区域Σ core-time占比对照AIC 每核 min–maxAIV 每核 min–max
泳道同父区间PMU 局部记录扣除参考
扣局部记录估算后的 Phase PMU / raw whole PMU；raw 比例同格保留		Scalar 局部记录扣除参考
扣局部记录估算后的 Phase scalar / raw whole scalar；raw 比例同格保留
FinalDrainKernelUnion461.382 µs15.395%——0.000–57.395 µs0.000–3.029 µs
FinalDrainResidual2,535.561 µs84.605%——5.249–274.069 µs2.535–78.641 µs
+
+ +
+ +

泳道 residual 的业务边界

+
+ +
边界次数Σ core-timeAIC ticksAIV ticks
submit_internal_residual/Claim->Materialize122,88042,034.185 µs12,699,48729,334,698
submit_tail_residual/LoserReplay->SubmitEnd97,28011,312.139 µs3,265,0698,047,070
submit_tail_residual/Register->SubmitEnd24,3203,240.519 µs1,116,2172,124,302
submit_tail_residual/WinnerBuild->SubmitEnd1,024201.856 µs73,347128,509
submit_tail_residual/AllocComplete->SubmitEnd25642.514 µs42,5140
between_submit_residual/kernel->kernel73,72832,980.483 µs10,056,17522,924,308
between_submit_residual/kernel->alloc24,48021,737.225 µs5,644,08116,093,144
between_submit_residual/alloc->kernel24,57611,561.218 µs3,398,8468,162,372
+ +

泳道 Kernel containment 与非加和 overlay

+
+
+
Kernel 归属事件数
inside_efdrain_events1,002
inside_final_drain_events20
inside_winner_build_events2
total_events1,024
+
+ +
Overlay事件数累计 ticks进入分区
Atomic104,60771,981,381否
ClockBaseline1924,578否
Commit1,0240否
RingBp299,668否
DrainWon00否
+
+ +

Submit-PMU ELF：whole gate 与 SYS 边界诊断

+ +
+

submit-pmu-none：PMU whole gate 与 SYS 边界诊断

+

global 首末 Submit 仅作闭合：6812.094 µs； + 下列数值是 96 核各自累计，不是墙钟。每核先读首个 Submit start tick，再启动 PMU；末次 Submit + 先停止 PMU，再读 end tick，所以 PMU gate 嵌在首末 SYS closure 内。Scalar elapsed 累计 gate-running + SYS 段，再扣 linked Kernel 与 return-ready atomic 等待；PMU cycle 与 SYS tick 口径不同，不能直接相减。

+
+ +
+ SYS 边界闭合诊断（raw ticks）282,120,570 + AIC 每核 2,381,818–3,331,744 ticks + AIV 每核 2,656,417–4,431,591 ticks + 只核验首末 Submit 与门控边界；不参与 PMU 1.65 GHz 时间换算或阶段比例。 +
+ +
+ PMU total cyclesΣ 932,085,106 cycles · ≈ 564953.478 µs + AIC 每核 9,128,939–9,952,568 cycles（5532.482–6031.633 µs） + AIV 每核 8,929,198–10,758,901 cycles（5412.518–6521.609 µs） + +
+ +
+ Scalar busy cyclesΣ 908,091,228 cycles · ≈ 550410.359 µs + AIC 每核 9,088,377–9,913,392 cycles（5507.900–6007.891 µs） + AIV 每核 8,590,434–10,377,067 cycles（5207.173–6290.157 µs） + Scalar / PMU total：908,091,228 / 932,085,106 = 97.426% +
+ +
+ 非 Scalar-busy 残余Σ 23,993,878 cycles · ≈ 14543.119 µs + AIC 每核 37,864–47,947 cycles（22.947–29.058 µs） + AIV 每核 277,440–455,825 cycles（168.173–276.303 µs） + 逐核先算 total−scalar；残余 / PMU total：23,993,878 / 932,085,106 = 2.574%。不是空闲时间。 +
+ +
+ Primary I-cache request69,179,112 + AIC 每核 662,570–765,234 + AIV 每核 627,031–919,317 + +
+ +
+ Primary I-cache miss782,333 + AIC 每核 573–900 + AIV 每核 9,639–13,700 + +
+
+

PMU 等效时间分别按 ALL/AIC/AIV + 1.649844/1.650062/1.649731 cycles/ns 换算。

+

git 8c6b6e3a412d · AICore ELF + ab25886c73db · + 单份报告

+
+ + + +
+

11 个业务分段的记录开销拆分（含 SubmitTransition）

+

raw 合计包含分段记录代码自身开销，不能直接拿它和 + submit-pmu-none 比出“业务变慢了多少”。 + 本节另外用 empty-bracket 测得“一组 begin/end 记录代码”的开销，再按每个 phase 实际记录组数估算； + AIC 与 AIV 分开计算后才合成 ALL。扣除后的数值只供定位量级，不是精确还原的业务值。

+

下图以 none 为 100%（黑色竖线），橙色为“空区间估算的记录代码开销”， + 蓝色为“扣除上述估算后的参考值”；二者代数相加为 raw。横轴统一为 0–600%，超过部分只在图上封顶。 + empty-bracket 的调用点、代码布局及采集轮次与各 phase 不同，所以该估算不能跨 ELF 当作精确扣除。 + I-cache raw 使用 phase observed，none 使用 primary。

+
PMU total
raw 243.874%记录估算 161.302% + 扣除后 82.572%
Scalar busy
raw 179.148%记录估算 106.635% + 扣除后 72.513%
非 Scalar-busy 残余
raw 2693.528%记录估算 2230.265% + 扣除后 463.263%图形封顶 600%
I-cache request
raw 199.896%记录估算 77.643% + 扣除后 122.252%
I-cache miss
raw 593.755%记录估算 234.607% + 扣除后 359.147%
+
+ + + +
指标原始观测合计记录代码开销估算扣除后参考值submit-pmu-noneAIC / AIV 分角色比例
PMU total23,678,221.177 cycles/core≈ 14,351.794 µs/coreraw / none 243.874%15,661,129.448 cycles/core≈ 9,492.491 µs/core占 raw 66.141%8,017,091.729 cycles/core≈ 4,859.303 µs/core参考值 / none 82.572%9,709,219.854 cycles/core≈ 5,884.932 µs/core基线 100%AIC：raw 231.144% / 记录估算 157.024% / 扣除后 74.119%AIV：raw 250.012% / 记录估算 163.364% / 扣除后 86.648%
Scalar busy16,946,119.052 cycles/core≈ 10,271.346 µs/coreraw / none 179.148%10,086,888.607 cycles/core≈ 6,113.844 µs/core占 raw 59.523%6,859,230.446 cycles/core≈ 4,157.502 µs/core参考值 / none 72.513%9,459,283.625 cycles/core≈ 5,733.441 µs/core基线 100%AIC：raw 174.430% / 记录估算 103.874% / 扣除后 70.557%AIV：raw 181.497% / 记录估算 108.010% / 扣除后 73.488%
非 Scalar-busy 残余6,732,102.125 cycles/core≈ 4,080.448 µs/coreraw / none 2693.528%5,574,240.841 cycles/core≈ 3,378.647 µs/core占 raw 82.801%1,157,861.284 cycles/core≈ 701.800 µs/core参考值 / none 463.263%249,936.229 cycles/core≈ 151.491 µs/core基线 100%AIC：raw 12989.660% / 记录估算 12114.053% / 扣除后 875.607%AIV：raw 2083.561% / 记录估算 1644.727% / 扣除后 438.834%
I-cache request1,440,478.844 events/coreraw / none 199.896%559,510.259 events/core占 raw 38.842%880,968.584 events/core参考值 / none 122.252%720,615.750 events/core基线 100%AIC：raw 203.526% / 记录估算 77.266% / 扣除后 126.260%AIV：raw 198.144% / 记录估算 77.826% / 扣除后 120.319%
I-cache miss48,386.854 events/coreraw / none 593.755%19,118.869 events/core占 raw 39.513%29,267.985 events/core参考值 / none 359.147%8,149.302 events/core基线 100%AIC：raw 1228.410% / 记录估算 84.496% / 扣除后 1143.914%AIV：raw 574.480% / 记录估算 239.166% / 扣除后 335.314%
+

本组合计 10,008.385 组记录/core； + AIC 10,019.844，AIV + 10,002.656。记录组数包含 linked Kernel + pause/resume 产生的额外 begin/end；动态阶段的零调用核仍按完整 32/64 核总体计入均值。

+
+ + +

Submit-PMU ELF：各阶段独立归因

+

每张业务阶段卡都把“扣局部记录代码开销估算后的参考值”放在主位置，并同时列出 raw observed、 + 记录代码开销估算及 raw whole 分母。参考比例仍以该卡所属 ELF 的 raw whole 为分母； + empty-bracket 没有测到整窗全部观察代码，因此不能把它解释成精确业务占比。各阶段来自不同 ELF， + 也不能求和后与 submit-pmu-none 做正式判等；上方合计只用于判断数量级。若某项参考值 + 为负，表示空区间记录估算已经超过 raw observed，只说明该信号低于当前校准分辨能力；页面不截成 + 零，也不把它解释成负的业务事件。

+
+
+
+

EfDrain Scalar control

+ submit-pmu-efdrain-control
+ control-only +
+

efdrain_begin_to_end_excluding_linked_kernel_calls · calls ALL/AIC/AIV 122,880 / 40,960 / 81,920
+ calls/core AIC 1,280–1,280（零调用核 0）；AIV 1,280–1,280（零调用核 0）

+
+ +
+ Phase PMU total + 扣局部记录估算参考 Σ 65,400,312.871 cycles · ≈ + 39640.301 µs + 参考值 / raw 整窗：7.924%；raw observed / raw 整窗： + 31.409% + raw observed Σ 259,240,439 cycles − 记录代码开销估算 + 193,840,126.129 cycles；估算/raw 74.772% + raw Phase total / raw whole total：259,240,439 / 825,372,063 = 31.409% + raw AIC 每核 2,530,342–2,814,967 cycles（1533.483–1705.977 µs） + raw AIV 每核 2,588,694–2,918,901 cycles（1569.161–1769.319 µs） + +
+ + +
+ Phase scalar busy + 扣局部记录估算参考 Σ 52,687,582.172 cycles · ≈ + 31934.887 µs + 参考值 / raw 整窗：7.002%；raw observed / raw 整窗： + 23.595% + raw observed Σ 177,538,191 cycles − 记录代码开销估算 + 124,850,608.828 cycles；估算/raw 70.323% + 参考 Scalar / 参考 Phase total：80.562%；raw 68.484%raw Phase scalar / raw whole scalar：177,538,191 / 752,424,257 = 23.595% + raw AIC 每核 1,794,703–2,075,030 cycles（1087.658–1257.547 µs） + raw AIV 每核 1,697,975–1,992,002 cycles（1029.244–1207.471 µs） + +
+ + +
+ 非 Scalar-busy 残余 + 扣局部记录估算参考 Σ 12,712,730.699 cycles · ≈ + 7705.414 µs + 参考值 / raw 整窗：17.427%；raw observed / raw 整窗： + 112.001% + raw observed Σ 81,702,248 cycles − 记录代码开销估算 + 68,989,517.301 cycles；估算/raw 84.440% + 逐核先算 total−scalar；参考 residual / 参考 Phase total：19.438%；raw 31.516% + raw AIC 每核 725,726–748,044 cycles（439.817–453.343 µs） + raw AIV 每核 841,898–937,867 cycles（510.324–568.497 µs） + 它不是“空闲时间”，可能包含等待、I-cache、atomic、观察器及其他非 scalar-busy 周期。 +
+ + +
+ I-cache request observed + 扣局部记录估算参考 8,011,606.393 + 参考值 / raw 整窗：8.984% + raw observed 14,936,953 / raw 整窗 89,175,746 + = 16.750% + raw observed − 记录代码开销估算 6,925,346.607； + 估算/raw 46.364% + raw AIC 每核 151,318–178,475；raw AIV 每核 140,297–166,456 +
+ + +
+ I-cache miss observed + 扣局部记录估算参考 287,740.850 + 参考值 / raw 整窗：8.964% + raw observed 524,078 / raw 整窗 3,209,861 + = 16.327% + raw observed − 记录代码开销估算 236,337.150； + 估算/raw 45.096% + raw AIC 每核 1,948–2,377；raw AIV 每核 6,449–7,856 +
+ +
+

SYS 边界诊断： + Σ 20,863,410 raw ticks； + AIC 每核 196,647–278,930 ticks； + AIV 每核 174,276–265,361 ticks。 + 只核验 phase 边界闭合，不作为阶段主时间或占比分母。

+

Scalar shadow 诊断： + whole CNT2 scalar − CNT3 shadow scalar Σ 247,768 cycles； + AIC 每核 2,586–2,596 cycles； + AIV 每核 2,570–2,586 cycles。

+

分母只来自本 ELF；excluded linked Kernel calls=1,004； + PMU 等效时间分别按 ALL/AIC/AIV + 1.649844/1.650062/1.649731 cycles/ns 换算； + global Submit closure=5669.558 µs（不作 phase 分母）。

+

git 8c6b6e3a412d · AICore ELF + 49689ae673b3 · + 单份报告

+
+ +
+
+

Claim

+ submit-pmu-claim
+ same-business-boundary +
+

claim_begin_to_claim_end · calls ALL/AIC/AIV 122,880 / 40,960 / 81,920
+ calls/core AIC 1,280–1,280（零调用核 0）；AIV 1,280–1,280（零调用核 0）

+
+ +
+ Phase PMU total + 扣局部记录估算参考 Σ 129,663,593.000 cycles · ≈ + 78591.426 µs + 参考值 / raw 整窗：15.675%；raw observed / raw 整窗： + 38.921% + raw observed Σ 321,951,929 cycles − 记录代码开销估算 + 192,288,336.000 cycles；估算/raw 59.726% + raw Phase total / raw whole total：321,951,929 / 827,188,042 = 38.921% + raw AIC 每核 2,948,298–3,023,337 cycles（1786.780–1832.257 µs） + raw AIV 每核 3,465,659–3,599,082 cycles（2100.742–2181.617 µs） + +
+ + +
+ Phase scalar busy + 扣局部记录估算参考 Σ 107,038,871.000 cycles · ≈ + 64878.177 µs + 参考值 / raw 整窗：14.243%；raw observed / raw 整窗： + 30.723% + raw observed Σ 230,884,809 cycles − 记录代码开销估算 + 123,845,938.000 cycles；估算/raw 53.640% + 参考 Scalar / 参考 Phase total：82.551%；raw 71.714%raw Phase scalar / raw whole scalar：230,884,809 / 751,514,132 = 30.723% + raw AIC 每核 2,209,220–2,273,774 cycles（1338.871–1377.993 µs） + raw AIV 每核 2,443,961–2,543,276 cycles（1481.430–1541.631 µs） + +
+ + +
+ 非 Scalar-busy 残余 + 扣局部记录估算参考 Σ 22,624,722.000 cycles · ≈ + 13713.249 µs + 参考值 / raw 整窗：29.898%；raw observed / raw 整窗： + 120.342% + raw observed Σ 91,067,120 cycles − 记录代码开销估算 + 68,442,398.000 cycles；估算/raw 75.156% + 逐核先算 total−scalar；参考 residual / 参考 Phase total：17.449%；raw 28.286% + raw AIC 每核 732,028–764,512 cycles（443.637–463.323 µs） + raw AIV 每核 1,017,626–1,077,936 cycles（616.844–653.401 µs） + 它不是“空闲时间”，可能包含等待、I-cache、atomic、观察器及其他非 scalar-busy 周期。 +
+ + +
+ I-cache request observed + 扣局部记录估算参考 12,142,513.000 + 参考值 / raw 整窗：13.152% + raw observed 19,012,131 / raw 整窗 92,323,235 + = 20.593% + raw observed − 记录代码开销估算 6,869,618.000； + 估算/raw 36.133% + raw AIC 每核 192,561–198,445；raw AIV 每核 197,857–200,760 +
+ + +
+ I-cache miss observed + 扣局部记录估算参考 594,211.000 + 参考值 / raw 整窗：18.803% + raw observed 829,077 / raw 整窗 3,160,254 + = 26.235% + raw observed − 记录代码开销估算 234,866.000； + 估算/raw 28.329% + raw AIC 每核 2,766–3,449；raw AIV 每核 11,004–12,116 +
+ +
+

SYS 边界诊断： + Σ 36,441,184 raw ticks； + AIC 每核 280,693–298,137 ticks； + AIV 每核 405,735–443,056 ticks。 + 只核验 phase 边界闭合，不作为阶段主时间或占比分母。

+

Scalar shadow 诊断： + whole CNT2 scalar − CNT3 shadow scalar Σ 245,760 cycles； + AIC 每核 2,560–2,560 cycles； + AIV 每核 2,560–2,560 cycles。

+

分母只来自本 ELF；excluded linked Kernel calls=0； + PMU 等效时间分别按 ALL/AIC/AIV + 1.649844/1.650062/1.649731 cycles/ns 换算； + global Submit closure=5696.233 µs（不作 phase 分母）。

+

git 8c6b6e3a412d · AICore ELF + 0538c8653fa7 · + 单份报告

+
+ +
+
+

ArgBuild

+ submit-pmu-arg-build
+ residual-boundary +
+

claim_end_to_materialize_begin · calls ALL/AIC/AIV 122,880 / 40,960 / 81,920
+ calls/core AIC 1,280–1,280（零调用核 0）；AIV 1,280–1,280（零调用核 0）

+
+ +
+ Phase PMU total + 扣局部记录估算参考 Σ 63,979,859.000 cycles · ≈ + 38779.339 µs + 参考值 / raw 整窗：7.384%；raw observed / raw 整窗： + 29.577% + raw observed Σ 256,268,195 cycles − 记录代码开销估算 + 192,288,336.000 cycles；估算/raw 75.034% + raw Phase total / raw whole total：256,268,195 / 866,449,611 = 29.577% + raw AIC 每核 2,396,736–2,431,534 cycles（1452.513–1473.602 µs） + raw AIV 每核 2,744,631–2,857,155 cycles（1663.684–1731.891 µs） + +
+ + +
+ Phase scalar busy + 扣局部记录估算参考 Σ 47,031,713.000 cycles · ≈ + 28506.764 µs + 参考值 / raw 整窗：6.150%；raw observed / raw 整窗： + 22.345% + raw observed Σ 170,877,651 cycles − 记录代码开销估算 + 123,845,938.000 cycles；估算/raw 72.476% + 参考 Scalar / 参考 Phase total：73.510%；raw 66.679%raw Phase scalar / raw whole scalar：170,877,651 / 764,708,510 = 22.345% + raw AIC 每核 1,702,639–1,717,426 cycles（1031.864–1040.825 µs） + raw AIV 每核 1,809,609–1,826,762 cycles（1096.912–1107.309 µs） + +
+ + +
+ 非 Scalar-busy 残余 + 扣局部记录估算参考 Σ 16,948,146.000 cycles · ≈ + 10272.575 µs + 参考值 / raw 整窗：16.658%；raw observed / raw 整窗： + 83.929% + raw observed Σ 85,390,544 cycles − 记录代码开销估算 + 68,442,398.000 cycles；估算/raw 80.152% + 逐核先算 total−scalar；参考 residual / 参考 Phase total：26.490%；raw 33.321% + raw AIC 每核 693,485–723,172 cycles（420.278–438.270 µs） + raw AIV 每核 932,601–1,038,153 cycles（565.305–629.286 µs） + 它不是“空闲时间”，可能包含等待、I-cache、atomic、观察器及其他非 scalar-busy 周期。 +
+ + +
+ I-cache request observed + 扣局部记录估算参考 9,735,102.000 + 参考值 / raw 整窗：10.592% + raw observed 16,604,720 / raw 整窗 91,911,048 + = 18.066% + raw observed − 记录代码开销估算 6,869,618.000； + 估算/raw 41.371% + raw AIC 每核 169,690–170,777；raw AIV 每核 173,850–174,741 +
+ + +
+ I-cache miss observed + 扣局部记录估算参考 354,547.000 + 参考值 / raw 整窗：9.946% + raw observed 589,413 / raw 整窗 3,564,754 + = 16.534% + raw observed − 记录代码开销估算 234,866.000； + 估算/raw 39.847% + raw AIC 每核 1,332–1,848；raw AIV 每核 8,252–8,752 +
+ +
+

SYS 边界诊断： + Σ 24,072,315 raw ticks； + AIC 每核 213,222–233,149 ticks； + AIV 每核 258,907–273,643 ticks。 + 只核验 phase 边界闭合，不作为阶段主时间或占比分母。

+

Scalar shadow 诊断： + whole CNT2 scalar − CNT3 shadow scalar Σ 245,760 cycles； + AIC 每核 2,560–2,560 cycles； + AIV 每核 2,560–2,560 cycles。

+

分母只来自本 ELF；excluded linked Kernel calls=0； + PMU 等效时间分别按 ALL/AIC/AIV + 1.649844/1.650062/1.649731 cycles/ns 换算； + global Submit closure=5982.462 µs（不作 phase 分母）。

+

git 8c6b6e3a412d · AICore ELF + cefb0e3bbfb7 · + 单份报告

+
+ +
+
+

Materialize

+ submit-pmu-materialize
+ same-business-boundary +
+

materialize_begin_to_materialize_end · calls ALL/AIC/AIV 122,880 / 40,960 / 81,920
+ calls/core AIC 1,280–1,280（零调用核 0）；AIV 1,280–1,280（零调用核 0）

+
+ +
+ Phase PMU total + 扣局部记录估算参考 Σ 176,061,279.000 cycles · ≈ + 106713.895 µs + 参考值 / raw 整窗：20.469%；raw observed / raw 整窗： + 42.825% + raw observed Σ 368,349,615 cycles − 记录代码开销估算 + 192,288,336.000 cycles；估算/raw 52.203% + raw Phase total / raw whole total：368,349,615 / 860,121,776 = 42.825% + raw AIC 每核 3,601,905–3,683,070 cycles（2182.891–2232.080 µs） + raw AIV 每核 3,854,196–4,010,884 cycles（2336.257–2431.235 µs） + +
+ + +
+ Phase scalar busy + 扣局部记录估算参考 Σ 170,097,672.000 cycles · ≈ + 103099.246 µs + 参考值 / raw 整窗：22.393%；raw observed / raw 整窗： + 38.698% + raw observed Σ 293,943,610 cycles − 记录代码开销估算 + 123,845,938.000 cycles；估算/raw 42.133% + 参考 Scalar / 参考 Phase total：96.613%；raw 79.800%raw Phase scalar / raw whole scalar：293,943,610 / 759,586,910 = 38.698% + raw AIC 每核 2,943,032–3,024,557 cycles（1783.589–1832.996 µs） + raw AIV 每核 3,063,783–3,164,251 cycles（1857.141–1918.041 µs） + +
+ + +
+ 非 Scalar-busy 残余 + 扣局部记录估算参考 Σ 5,963,607.000 cycles · ≈ + 3614.649 µs + 参考值 / raw 整窗：5.932%；raw observed / raw 整窗： + 74.010% + raw observed Σ 74,406,005 cycles − 记录代码开销估算 + 68,442,398.000 cycles；估算/raw 91.985% + 逐核先算 total−scalar；参考 residual / 参考 Phase total：3.387%；raw 20.200% + raw AIC 每核 656,662–659,424 cycles（397.962–399.636 µs） + raw AIV 每核 790,413–886,262 cycles（479.116–537.216 µs） + 它不是“空闲时间”，可能包含等待、I-cache、atomic、观察器及其他非 scalar-busy 周期。 +
+ + +
+ I-cache request observed + 扣局部记录估算参考 23,254,487.000 + 参考值 / raw 整窗：24.748% + raw observed 30,124,105 / raw 整窗 93,963,590 + = 32.059% + raw observed − 记录代码开销估算 6,869,618.000； + 估算/raw 22.804% + raw AIC 每核 311,744–314,741；raw AIV 每核 311,332–316,405 +
+ + +
+ I-cache miss observed + 扣局部记录估算参考 519,859.000 + 参考值 / raw 整窗：16.164% + raw observed 754,725 / raw 整窗 3,216,252 + = 23.466% + raw observed − 记录代码开销估算 234,866.000； + 估算/raw 31.119% + raw AIC 每核 42–66；raw AIV 每核 11,134–12,093 +
+ +
+

SYS 边界诊断： + Σ 99,479,891 raw ticks； + AIC 每核 963,084–1,011,789 ticks； + AIV 每核 1,025,730–1,106,791 ticks。 + 只核验 phase 边界闭合，不作为阶段主时间或占比分母。

+

Scalar shadow 诊断： + whole CNT2 scalar − CNT3 shadow scalar Σ 245,760 cycles； + AIC 每核 2,560–2,560 cycles； + AIV 每核 2,560–2,560 cycles。

+

分母只来自本 ELF；excluded linked Kernel calls=0； + PMU 等效时间分别按 ALL/AIC/AIV + 1.649844/1.650062/1.649731 cycles/ns 换算； + global Submit closure=5970.339 µs（不作 phase 分母）。

+

git 8c6b6e3a412d · AICore ELF + 42c60b91163d · + 单份报告

+
+ +
+
+

PrepareMap

+ submit-pmu-prepare-map
+ same-business-boundary +
+

dist_submit_prepare_map_call_entry_to_return · calls ALL/AIC/AIV 122,880 / 40,960 / 81,920
+ calls/core AIC 1,280–1,280（零调用核 0）；AIV 1,280–1,280（零调用核 0）

+
+ +
+ Phase PMU total + 扣局部记录估算参考 Σ 17,839,579.000 cycles · ≈ + 10812.888 µs + 参考值 / raw 整窗：2.274%；raw observed / raw 整窗： + 26.781% + raw observed Σ 210,127,915 cycles − 记录代码开销估算 + 192,288,336.000 cycles；估算/raw 91.510% + raw Phase total / raw whole total：210,127,915 / 784,604,857 = 26.781% + raw AIC 每核 2,093,859–2,201,061 cycles（1268.958–1333.926 µs） + raw AIV 每核 2,177,815–2,368,000 cycles（1320.103–1435.386 µs） + +
+ + +
+ Phase scalar busy + 扣局部记录估算参考 Σ 20,173,480.000 cycles · ≈ + 12227.508 µs + 参考值 / raw 整窗：2.751%；raw observed / raw 整窗： + 19.638% + raw observed Σ 144,019,418 cycles − 记录代码开销估算 + 123,845,938.000 cycles；估算/raw 85.993% + 参考 Scalar / 参考 Phase total：113.083%；raw 68.539%raw Phase scalar / raw whole scalar：144,019,418 / 733,380,676 = 19.638% + raw AIC 每核 1,441,782–1,548,097 cycles（873.774–938.205 µs） + raw AIV 每核 1,475,591–1,658,894 cycles（894.443–1005.554 µs） + +
+ + +
+ 非 Scalar-busy 残余 + 扣局部记录估算参考 Σ -2,333,901.000 cycles · ≈ + -1414.619 µs + 参考值 / raw 整窗：-4.556%；raw observed / raw 整窗： + 129.057% + raw observed Σ 66,108,497 cycles − 记录代码开销估算 + 68,442,398.000 cycles；估算/raw 103.530% + 逐核先算 total−scalar；参考 residual / 参考 Phase total：-13.083%；raw 31.461% + raw AIC 每核 650,360–654,764 cycles（394.143–396.812 µs） + raw AIV 每核 696,446–717,275 cycles（422.157–434.783 µs） + 它不是“空闲时间”，可能包含等待、I-cache、atomic、观察器及其他非 scalar-busy 周期。 +
+ + +
+ I-cache request observed + 扣局部记录估算参考 4,119,393.000 + 参考值 / raw 整窗：4.589% + raw observed 10,989,011 / raw 整窗 89,766,806 + = 12.242% + raw observed − 记录代码开销估算 6,869,618.000； + 估算/raw 62.514% + raw AIC 每核 113,877–116,376；raw AIV 每核 114,110–116,638 +
+ + +
+ I-cache miss observed + 扣局部记录估算参考 -206,904.000 + 参考值 / raw 整窗：-12.171% + raw observed 27,962 / raw 整窗 1,699,987 + = 1.645% + raw observed − 记录代码开销估算 234,866.000； + 估算/raw 839.947% + raw AIC 每核 20–41；raw AIV 每核 320–565 +
+ +
+

SYS 边界诊断： + Σ 11,407,548 raw ticks； + AIC 每核 83,391–148,176 ticks； + AIV 每核 101,383–212,381 ticks。 + 只核验 phase 边界闭合，不作为阶段主时间或占比分母。

+

Scalar shadow 诊断： + whole CNT2 scalar − CNT3 shadow scalar Σ 245,760 cycles； + AIC 每核 2,560–2,560 cycles； + AIV 每核 2,560–2,560 cycles。

+

分母只来自本 ELF；excluded linked Kernel calls=0； + PMU 等效时间分别按 ALL/AIC/AIV + 1.649844/1.650062/1.649731 cycles/ns 换算； + global Submit closure=5399.543 µs（不作 phase 分母）。

+

git 8c6b6e3a412d · AICore ELF + e09e766ee773 · + 单份报告

+
+ +
+
+

Fanin

+ submit-pmu-fanin
+ same-business-boundary +
+

fanin_begin_to_fanin_end · calls ALL/AIC/AIV 1,024 / 512 / 512
+ calls/core AIC 12–20（零调用核 0）；AIV 4–12（零调用核 0）

+
+ +
+ Phase PMU total + 扣局部记录估算参考 Σ 1,926,456.381 cycles · ≈ + 1167.660 µs + 参考值 / raw 整窗：0.295%；raw observed / raw 整窗： + 0.538% + raw observed Σ 3,508,426 cycles − 记录代码开销估算 + 1,581,969.619 cycles；估算/raw 45.091% + raw Phase total / raw whole total：3,508,426 / 652,647,623 = 0.538% + raw AIC 每核 30,099–55,123 cycles（18.241–33.407 µs） + raw AIV 每核 13,075–55,245 cycles（7.926–33.487 µs） + +
+ + +
+ Phase scalar busy + 扣局部记录估算参考 Σ 1,685,046.725 cycles · ≈ + 1021.337 µs + 参考值 / raw 整窗：0.280%；raw observed / raw 整窗： + 0.450% + raw observed Σ 2,709,457 cycles − 记录代码开销估算 + 1,024,410.275 cycles；估算/raw 37.809% + 参考 Scalar / 参考 Phase total：87.469%；raw 77.227%raw Phase scalar / raw whole scalar：2,709,457 / 602,690,743 = 0.450% + raw AIC 每核 22,790–41,690 cycles（13.812–25.266 µs） + raw AIV 每核 9,664–44,119 cycles（5.858–26.743 µs） + +
+ + +
+ 非 Scalar-busy 残余 + 扣局部记录估算参考 Σ 241,409.656 cycles · ≈ + 146.323 µs + 参考值 / raw 整窗：0.483%；raw observed / raw 整窗： + 1.599% + raw observed Σ 798,969 cycles − 记录代码开销估算 + 557,559.344 cycles；估算/raw 69.785% + 逐核先算 total−scalar；参考 residual / 参考 Phase total：12.531%；raw 22.773% + raw AIC 每核 7,309–13,433 cycles（4.430–8.141 µs） + raw AIV 每核 3,411–11,344 cycles（2.068–6.876 µs） + 它不是“空闲时间”，可能包含等待、I-cache、atomic、观察器及其他非 scalar-busy 周期。 +
+ + +
+ I-cache request observed + 扣局部记录估算参考 285,209.438 + 参考值 / raw 整窗：0.376% + raw observed 342,033 / raw 整窗 75,933,197 + = 0.450% + raw observed − 记录代码开销估算 56,823.562； + 估算/raw 16.613% + raw AIC 每核 3,304–5,485；raw AIV 每核 1,315–5,375 +
+ + +
+ I-cache miss observed + 扣局部记录估算参考 12,298.531 + 参考值 / raw 整窗：0.527% + raw observed 13,782 / raw 整窗 2,331,934 + = 0.591% + raw observed − 记录代码开销估算 1,483.469； + 估算/raw 10.764% + raw AIC 每核 37–120；raw AIV 每核 83–277 +
+ +
+

SYS 边界诊断： + Σ 1,016,201 raw ticks； + AIC 每核 6,327–13,581 ticks； + AIV 每核 3,253–19,441 ticks。 + 只核验 phase 边界闭合，不作为阶段主时间或占比分母。

+

Scalar shadow 诊断： + whole CNT2 scalar − CNT3 shadow scalar Σ 2,048 cycles； + AIC 每核 24–40 cycles； + AIV 每核 8–24 cycles。

+

分母只来自本 ELF；excluded linked Kernel calls=0； + PMU 等效时间分别按 ALL/AIC/AIV + 1.649844/1.650062/1.649731 cycles/ns 换算； + global Submit closure=4642.811 µs（不作 phase 分母）。

+

git 8c6b6e3a412d · AICore ELF + 9eaef3d1d5cc · + 单份报告

+
+ +
+
+

Register

+ submit-pmu-register
+ same-business-boundary +
+

register_outputs_call_entry_to_return · calls ALL/AIC/AIV 122,880 / 40,960 / 81,920
+ calls/core AIC 1,280–1,280（零调用核 0）；AIV 1,280–1,280（零调用核 0）

+
+ +
+ Phase PMU total + 扣局部记录估算参考 Σ 56,406,147.000 cycles · ≈ + 34188.776 µs + 参考值 / raw 整窗：6.769%；raw observed / raw 整窗： + 29.845% + raw observed Σ 248,694,483 cycles − 记录代码开销估算 + 192,288,336.000 cycles；估算/raw 77.319% + raw Phase total / raw whole total：248,694,483 / 833,286,281 = 29.845% + raw AIC 每核 2,345,551–2,439,083 cycles（1421.493–1478.177 µs） + raw AIV 每核 2,635,187–2,732,437 cycles（1597.343–1656.292 µs） + +
+ + +
+ Phase scalar busy + 扣局部记录估算参考 Σ 44,338,114.000 cycles · ≈ + 26874.125 µs + 参考值 / raw 整窗：5.962%；raw observed / raw 整窗： + 22.615% + raw observed Σ 168,184,052 cycles − 记录代码开销估算 + 123,845,938.000 cycles；估算/raw 73.637% + 参考 Scalar / 参考 Phase total：78.605%；raw 67.627%raw Phase scalar / raw whole scalar：168,184,052 / 743,680,705 = 22.615% + raw AIC 每核 1,658,575–1,740,706 cycles（1005.159–1054.934 µs） + raw AIV 每核 1,724,613–1,816,536 cycles（1045.390–1101.110 µs） + +
+ + +
+ 非 Scalar-busy 残余 + 扣局部记录估算参考 Σ 12,068,033.000 cycles · ≈ + 7314.651 µs + 参考值 / raw 整窗：13.468%；raw observed / raw 整窗： + 89.850% + raw observed Σ 80,510,431 cycles − 记录代码开销估算 + 68,442,398.000 cycles；估算/raw 85.011% + 逐核先算 total−scalar；参考 residual / 参考 Phase total：21.395%；raw 32.373% + raw AIC 每核 679,724–701,583 cycles（411.938–425.186 µs） + raw AIV 每核 895,222–929,750 cycles（542.647–563.577 µs） + 它不是“空闲时间”，可能包含等待、I-cache、atomic、观察器及其他非 scalar-busy 周期。 +
+ + +
+ I-cache request observed + 扣局部记录估算参考 5,560,990.000 + 参考值 / raw 整窗：6.173% + raw observed 12,430,608 / raw 整窗 90,091,095 + = 13.798% + raw observed − 记录代码开销估算 6,869,618.000； + 估算/raw 55.264% + raw AIC 每核 126,587–132,112；raw AIV 每核 127,357–130,347 +
+ + +
+ I-cache miss observed + 扣局部记录估算参考 215,428.000 + 参考值 / raw 整窗：6.116% + raw observed 450,294 / raw 整窗 3,522,153 + = 12.785% + raw observed − 记录代码开销估算 234,866.000； + 估算/raw 52.158% + raw AIC 每核 695–849；raw AIV 每核 6,523–6,811 +
+ +
+

SYS 边界诊断： + Σ 27,870,336 raw ticks； + AIC 每核 214,334–263,214 ticks； + AIV 每核 286,670–340,526 ticks。 + 只核验 phase 边界闭合，不作为阶段主时间或占比分母。

+

Scalar shadow 诊断： + whole CNT2 scalar − CNT3 shadow scalar Σ 245,760 cycles； + AIC 每核 2,560–2,560 cycles； + AIV 每核 2,560–2,560 cycles。

+

分母只来自本 ELF；excluded linked Kernel calls=0； + PMU 等效时间分别按 ALL/AIC/AIV + 1.649844/1.650062/1.649731 cycles/ns 换算； + global Submit closure=5788.451 µs（不作 phase 分母）。

+

git 8c6b6e3a412d · AICore ELF + c870328ac6fe · + 单份报告

+
+ +
+
+

WinnerBuild Scalar control

+ submit-pmu-winner-build-control
+ control-only +
+

winner_build_begin_to_end_excluding_linked_kernel_calls · calls ALL/AIC/AIV 1,024 / 512 / 512
+ calls/core AIC 12–22（零调用核 0）；AIV 0–12（零调用核 1）

+
+ +
+ Phase PMU total + 扣局部记录估算参考 Σ 60,658,419.285 cycles · ≈ + 36766.154 µs + 参考值 / raw 整窗：7.295%；raw observed / raw 整窗： + 7.514% + raw observed Σ 62,476,937 cycles − 记录代码开销估算 + 1,818,517.715 cycles；估算/raw 2.911% + raw Phase total / raw whole total：62,476,937 / 831,511,201 = 7.514% + raw AIC 每核 118,934–1,809,865 cycles（72.079–1096.847 µs） + raw AIV 每核 0–2,061,069 cycles（0.000–1249.336 µs） + +
+ + +
+ Phase scalar busy + 扣局部记录估算参考 Σ 59,831,553.283 cycles · ≈ + 36264.976 µs + 参考值 / raw 整窗：7.549%；raw observed / raw 整窗： + 7.698% + raw observed Σ 61,009,092 cycles − 记录代码开销估算 + 1,177,538.717 cycles；估算/raw 1.930% + 参考 Scalar / 参考 Phase total：98.637%；raw 97.651%raw Phase scalar / raw whole scalar：61,009,092 / 792,528,440 = 7.698% + raw AIC 每核 107,836–1,777,133 cycles（65.353–1077.010 µs） + raw AIV 每核 0–2,039,969 cycles（0.000–1236.546 µs） + +
+ + +
+ 非 Scalar-busy 残余 + 扣局部记录估算参考 Σ 826,866.003 cycles · ≈ + 501.178 µs + 参考值 / raw 整窗：2.121%；raw observed / raw 整窗： + 3.765% + raw observed Σ 1,467,845 cycles − 记录代码开销估算 + 640,978.997 cycles；估算/raw 43.668% + 逐核先算 total−scalar；参考 residual / 参考 Phase total：1.363%；raw 2.349% + raw AIC 每核 11,098–32,732 cycles（6.726–19.837 µs） + raw AIV 每核 0–26,855 cycles（0.000–16.278 µs） + 它不是“空闲时间”，可能包含等待、I-cache、atomic、观察器及其他非 scalar-busy 周期。 +
+ + +
+ I-cache request observed + 扣局部记录估算参考 4,798,650.478 + 参考值 / raw 整窗：6.501% + raw observed 4,863,968 / raw 整窗 73,819,554 + = 6.589% + raw observed − 记录代码开销估算 65,317.522； + 估算/raw 1.343% + raw AIC 每核 21,757–159,256；raw AIV 每核 0–147,353 +
+ + +
+ I-cache miss observed + 扣局部记录估算参考 29,082.716 + 参考值 / raw 整窗：1.882% + raw observed 30,792 / raw 整窗 1,544,977 + = 1.993% + raw observed − 记录代码开销估算 1,709.284； + 估算/raw 5.551% + raw AIC 每核 154–371；raw AIV 每核 0–630 +
+ +
+

SYS 边界诊断： + Σ 12,953,578 raw ticks； + AIC 每核 53,632–422,692 ticks； + AIV 每核 0–399,825 ticks。 + 只核验 phase 边界闭合，不作为阶段主时间或占比分母。

+

Scalar shadow 诊断： + whole CNT2 scalar − CNT3 shadow scalar Σ 2,354 cycles； + AIC 每核 26–56 cycles； + AIV 每核 0–30 cycles。

+

分母只来自本 ELF；excluded linked Kernel calls=153； + PMU 等效时间分别按 ALL/AIC/AIV + 1.649844/1.650062/1.649731 cycles/ns 换算； + global Submit closure=5896.308 µs（不作 phase 分母）。

+

git 8c6b6e3a412d · AICore ELF + 4b59f5399df3 · + 单份报告

+
+ +
+
+

AllocComplete Scalar control

+ submit-pmu-alloc-complete-control
+ control-only +
+

alloc_complete_begin_to_end_excluding_linked_kernel_calls · calls ALL/AIC/AIV 256 / 96 / 160
+ calls/core AIC 0–10（零调用核 4）；AIV 0–6（零调用核 3）

+
+ +
+ Phase PMU total + 扣局部记录估算参考 Σ 56,816,784.374 cycles · ≈ + 34437.671 µs + 参考值 / raw 整窗：6.824%；raw observed / raw 整窗： + 6.872% + raw observed Σ 57,216,108 cycles − 记录代码开销估算 + 399,323.626 cycles；估算/raw 0.698% + raw Phase total / raw whole total：57,216,108 / 832,640,742 = 6.872% + raw AIC 每核 0–1,349,309 cycles（0.000–817.732 µs） + raw AIV 每核 0–2,075,497 cycles（0.000–1258.082 µs） + +
+ + +
+ Phase scalar busy + 扣局部记录估算参考 Σ 56,372,898.080 cycles · ≈ + 34168.623 µs + 参考值 / raw 整窗：7.059%；raw observed / raw 整窗： + 7.091% + raw observed Σ 56,630,433 cycles − 记录代码开销估算 + 257,534.920 cycles；估算/raw 0.455% + 参考 Scalar / 参考 Phase total：99.219%；raw 98.976%raw Phase scalar / raw whole scalar：56,630,433 / 798,628,250 = 7.091% + raw AIC 每核 0–1,342,719 cycles（0.000–813.739 µs） + raw AIV 每核 0–2,065,174 cycles（0.000–1251.825 µs） + +
+ + +
+ 非 Scalar-busy 残余 + 扣局部记录估算参考 Σ 443,886.294 cycles · ≈ + 269.047 µs + 参考值 / raw 整窗：1.305%；raw observed / raw 整窗： + 1.722% + raw observed Σ 585,675 cycles − 记录代码开销估算 + 141,788.706 cycles；估算/raw 24.209% + 逐核先算 total−scalar；参考 residual / 参考 Phase total：0.781%；raw 1.024% + raw AIC 每核 0–11,646 cycles（0.000–7.058 µs） + raw AIV 每核 0–23,288 cycles（0.000–14.116 µs） + 它不是“空闲时间”，可能包含等待、I-cache、atomic、观察器及其他非 scalar-busy 周期。 +
+ + +
+ I-cache request observed + 扣局部记录估算参考 525,452.749 + 参考值 / raw 整窗：0.728% + raw observed 539,738 / raw 整窗 72,212,561 + = 0.747% + raw observed − 记录代码开销估算 14,285.251； + 估算/raw 2.647% + raw AIC 每核 0–12,416；raw AIV 每核 0–17,122 +
+ + +
+ I-cache miss observed + 扣局部记录估算参考 12,369.305 + 参考值 / raw 整窗：0.930% + raw observed 12,829 / raw 整窗 1,329,490 + = 0.965% + raw observed − 记录代码开销估算 459.695； + 估算/raw 3.583% + raw AIC 每核 0–204；raw AIV 每核 0–344 +
+ +
+

SYS 边界诊断： + Σ 1,659,528 raw ticks； + AIC 每核 0–35,221 ticks； + AIV 每核 0–47,745 ticks。 + 只核验 phase 边界闭合，不作为阶段主时间或占比分母。

+

Scalar shadow 诊断： + whole CNT2 scalar − CNT3 shadow scalar Σ 512 cycles； + AIC 每核 0–20 cycles； + AIV 每核 0–12 cycles。

+

分母只来自本 ELF；excluded linked Kernel calls=0； + PMU 等效时间分别按 ALL/AIC/AIV + 1.649844/1.650062/1.649731 cycles/ns 换算； + global Submit closure=5946.854 µs（不作 phase 分母）。

+

git 8c6b6e3a412d · AICore ELF + b72aafb602e9 · + 单份报告

+
+ +
+
+

LoserReplay

+ submit-pmu-loser-replay
+ same-business-boundary +
+

register_end_to_drain_block_won_return · calls ALL/AIC/AIV 97,280 / 32,256 / 65,024
+ calls/core AIC 1,006–1,011（零调用核 0）；AIV 1,013–1,020（零调用核 0）

+
+ +
+ Phase PMU total + 扣局部记录估算参考 Σ 18,529,058.819 cycles · ≈ + 11230.794 µs + 参考值 / raw 整窗：2.381%；raw observed / raw 整窗： + 21.941% + raw observed Σ 170,777,758 cycles − 记录代码开销估算 + 152,248,699.181 cycles；估算/raw 89.150% + raw Phase total / raw whole total：170,777,758 / 778,354,672 = 21.941% + raw AIC 每核 1,670,343–1,678,979 cycles（1012.291–1017.525 µs） + raw AIV 每核 1,810,993–1,856,211 cycles（1097.750–1125.160 µs） + +
+ + +
+ Phase scalar busy + 扣局部记录估算参考 Σ 16,387,225.875 cycles · ≈ + 9932.591 µs + 参考值 / raw 整窗：2.331%；raw observed / raw 整窗： + 16.281% + raw observed Σ 114,439,566 cycles − 记录代码开销估算 + 98,052,340.125 cycles；估算/raw 85.680% + 参考 Scalar / 参考 Phase total：88.441%；raw 67.011%raw Phase scalar / raw whole scalar：114,439,566 / 702,902,051 = 16.281% + raw AIC 每核 1,154,651–1,161,139 cycles（699.762–703.694 µs） + raw AIV 每核 1,203,214–1,214,047 cycles（729.340–735.906 µs） + +
+ + +
+ 非 Scalar-busy 残余 + 扣局部记录估算参考 Σ 2,141,832.944 cycles · ≈ + 1298.203 µs + 参考值 / raw 整窗：2.839%；raw observed / raw 整窗： + 74.667% + raw observed Σ 56,338,192 cycles − 记录代码开销估算 + 54,196,359.056 cycles；估算/raw 96.198% + 逐核先算 total−scalar；参考 residual / 参考 Phase total：11.559%；raw 32.989% + raw AIC 每核 515,632–517,886 cycles（312.493–313.859 µs） + raw AIV 每核 604,688–643,226 cycles（366.537–389.898 µs） + 它不是“空闲时间”，可能包含等待、I-cache、atomic、观察器及其他非 scalar-busy 周期。 +
+ + +
+ I-cache request observed + 扣局部记录估算参考 2,957,585.162 + 参考值 / raw 整窗：3.444% + raw observed 8,396,456 / raw 整窗 85,872,185 + = 9.778% + raw observed − 记录代码开销估算 5,438,870.838； + 估算/raw 64.776% + raw AIC 每核 84,633–86,723；raw AIV 每核 88,210–88,833 +
+ + +
+ I-cache miss observed + 扣局部记录估算参考 -342.331 + 参考值 / raw 整窗：-0.013% + raw observed 186,067 / raw 整窗 2,540,177 + = 7.325% + raw observed − 记录代码开销估算 186,409.331； + 估算/raw 100.184% + raw AIC 每核 34–84；raw AIV 每核 2,789–3,083 +
+ +
+

SYS 边界诊断： + Σ 6,926,924 raw ticks； + AIC 每核 58,534–59,712 ticks； + AIV 每核 74,960–82,670 ticks。 + 只核验 phase 边界闭合，不作为阶段主时间或占比分母。

+

Scalar shadow 诊断： + whole CNT2 scalar − CNT3 shadow scalar Σ 194,560 cycles； + AIC 每核 2,012–2,022 cycles； + AIV 每核 2,026–2,040 cycles。

+

分母只来自本 ELF；excluded linked Kernel calls=0； + PMU 等效时间分别按 ALL/AIC/AIV + 1.649844/1.650062/1.649731 cycles/ns 换算； + global Submit closure=5407.661 µs（不作 phase 分母）。

+

git 8c6b6e3a412d · AICore ELF + e182a40abdfe · + 单份报告

+
+ +
+
+

SubmitTransition

+ submit-pmu-submit-transition
+ adjacent-submit-boundary +
+

previous_submit_end_to_next_submit_begin · calls ALL/AIC/AIV 122,784 / 40,928 / 81,856
+ calls/core AIC 1,279–1,279（零调用核 0）；AIV 1,279–1,279（零调用核 0）

+
+ +
+ Phase PMU total + 扣局部记录估算参考 Σ 122,359,317.262 cycles · ≈ + 74164.174 µs + 参考值 / raw 整窗：13.982%；raw observed / raw 整窗： + 35.938% + raw observed Σ 314,497,428 cycles − 记录代码开销估算 + 192,138,110.738 cycles；估算/raw 61.094% + raw Phase total / raw whole total：314,497,428 / 875,104,857 = 35.938% + raw AIC 每核 2,691,895–2,806,960 cycles（1631.390–1701.124 µs） + raw AIV 每核 3,446,371–3,631,754 cycles（2089.050–2201.422 µs） + +
+ + +
+ Phase scalar busy + 扣局部记录估算参考 Σ 82,841,966.639 cycles · ≈ + 50212.000 µs + 参考值 / raw 整窗：10.595%；raw observed / raw 整窗： + 26.423% + raw observed Σ 206,591,150 cycles − 记录代码开销估算 + 123,749,183.361 cycles；估算/raw 59.901% + 参考 Scalar / 参考 Phase total：67.704%；raw 65.689%raw Phase scalar / raw whole scalar：206,591,150 / 781,865,774 = 26.423% + raw AIC 每核 1,982,935–2,076,648 cycles（1201.734–1258.527 µs） + raw AIV 每核 2,155,867–2,278,654 cycles（1306.799–1381.228 µs） + +
+ + +
+ 非 Scalar-busy 残余 + 扣局部记录估算参考 Σ 39,517,350.623 cycles · ≈ + 23952.174 µs + 参考值 / raw 整窗：42.383%；raw observed / raw 整窗： + 115.731% + raw observed Σ 107,906,278 cycles − 记录代码开销估算 + 68,388,927.377 cycles；估算/raw 63.378% + 逐核先算 total−scalar；参考 residual / 参考 Phase total：32.296%；raw 34.311% + raw AIC 每核 690,210–739,516 cycles（418.293–448.175 µs） + raw AIV 每核 1,231,357–1,411,878 cycles（746.399–855.823 µs） + 它不是“空闲时间”，可能包含等待、I-cache、atomic、观察器及其他非 scalar-busy 周期。 +
+ + +
+ I-cache request observed + 扣局部记录估算参考 13,181,994.889 + 参考值 / raw 整窗：14.314% + raw observed 20,046,246 / raw 整窗 92,089,124 + = 21.768% + raw observed − 记录代码开销估算 6,864,251.111； + 估算/raw 34.242% + raw AIC 每核 191,239–192,627；raw AIV 每核 216,672–218,737 +
+ + +
+ I-cache miss observed + 扣局部记录估算参考 991,436.489 + 参考值 / raw 整窗：24.747% + raw observed 1,226,119 / raw 整窗 4,006,223 + = 30.605% + raw observed − 记录代码开销估算 234,682.511； + 估算/raw 19.140% + raw AIC 每核 605–1,174；raw AIV 每核 17,966–19,639 +
+ +
+

SYS 边界诊断： + Σ 60,474,015 raw ticks； + AIC 每核 387,447–454,658 ticks； + AIV 每核 681,403–780,684 ticks。 + 只核验 phase 边界闭合，不作为阶段主时间或占比分母。

+

Scalar shadow 诊断： + whole CNT2 scalar − CNT3 shadow scalar Σ 245,568 cycles； + AIC 每核 2,558–2,558 cycles； + AIV 每核 2,558–2,558 cycles。

+

分母只来自本 ELF；excluded linked Kernel calls=0； + PMU 等效时间分别按 ALL/AIC/AIV + 1.649844/1.650062/1.649731 cycles/ns 换算； + global Submit closure=6052.808 µs（不作 phase 分母）。

+

git 8c6b6e3a412d · AICore ELF + 3f18e5075d99 · + 单份报告

+
+
+ +

观察器校准（不是业务阶段）

+
+
+
+

EmptyBracket

+ submit-pmu-empty-bracket
+ observer-calibration +
+

claim_end_adjacent_empty_bracket · calls ALL/AIC/AIV 122,880 / 40,960 / 81,920
+ calls/core AIC 1,280–1,280（零调用核 0）；AIV 1,280–1,280（零调用核 0）

+
+ +
+ Phase PMU total + raw observed Σ 192,288,336 cycles · ≈ + 116549.405 µs + raw Phase total / raw whole total：192,288,336 / 831,856,934 = 23.116% + raw AIC 每核 1,898,234–1,904,578 cycles（1150.402–1154.246 µs） + raw AIV 每核 2,017,566–2,083,565 cycles（1222.967–1262.973 µs） + +
+ + +
+ Phase scalar busy + raw observed Σ 123,845,938 cycles · ≈ + 75065.241 µs + raw Scalar / raw Phase total：123,845,938 / 192,288,336 = 64.406%raw Phase scalar / raw whole scalar：123,845,938 / 753,739,352 = 16.431% + raw AIC 每核 1,251,002–1,252,666 cycles（758.155–759.163 µs） + raw AIV 每核 1,296,538–1,315,457 cycles（785.909–797.377 µs） + +
+ + +
+ 非 Scalar-busy 残余 + raw observed Σ 68,442,398 cycles · ≈ + 41484.163 µs + 逐核先算 total−scalar；raw residual / raw Phase total：68,442,398 / 192,288,336 = 35.594% + raw AIC 每核 647,232–652,073 cycles（392.247–395.181 µs） + raw AIV 每核 714,196–768,686 cycles（432.917–465.946 µs） + 它不是“空闲时间”，可能包含等待、I-cache、atomic、观察器及其他非 scalar-busy 周期。 +
+ + +
+ I-cache request observed + raw observed 6,869,618 + raw observed / raw 整窗：6,869,618 / 90,139,943 = 7.621% + raw AIC 每核 69,140–69,719；raw AIV 每核 70,952–73,661 +
+ + +
+ I-cache miss observed + raw observed 234,866 + raw observed / raw 整窗：234,866 / 3,190,462 = 7.362% + raw AIC 每核 41–113；raw AIV 每核 2,513–4,076 +
+ +
+

SYS 边界诊断： + Σ 160,209,998 raw ticks； + AIC 每核 1,555,583–1,560,383 ticks； + AIV 每核 1,708,058–1,736,354 ticks。 + 只核验 phase 边界闭合，不作为阶段主时间或占比分母。

+

Scalar shadow 诊断： + whole CNT2 scalar − CNT3 shadow scalar Σ 245,760 cycles； + AIC 每核 2,560–2,560 cycles； + AIV 每核 2,560–2,560 cycles。

+

分母只来自本 ELF；excluded linked Kernel calls=0； + PMU 等效时间分别按 ALL/AIC/AIV + 1.649844/1.650062/1.649731 cycles/ns 换算； + global Submit closure=5755.507 µs（不作 phase 分母）。

+

git 8c6b6e3a412d · AICore ELF + 71743c878d9a · + 单份报告

+
+
+ +

泳道区域与 PMU 覆盖矩阵

+
+
泳道区域PMU profile关系说明
WorkerCompletion—not-covered含 OrchestrationReplay 与 FinalDrain；没有独立 PMU 父窗
OrchestrationReplay—not-coveredPMU gate 嵌在首末 Submit SYS closure 内，不等同该父区间
OrchestrationSetup—not-covered当前没有独立 PMU phase
SubmitEnvelopesubmit-pmu-noneboundary-onlyPMU Scalar 分母另行排除 linked Kernel 与 return-ready atomic 时间
SubmitUnion—independent-components-only各 phase 是独立 ELF，不允许求和成 SubmitUnion
EfDrainsubmit-pmu-efdrain-controlcontrol-only泳道父 span 含 Kernel；PMU 只看排除 Kernel 后的 Scalar control
EfDrainKernelUnion—intentionally-excluded目标是纯 Scalar 归因，linked Kernel 从 PMU gate 排除
EfDrainControlsubmit-pmu-efdrain-controlcontrol-only与泳道 EfDrain 扣除 Kernel union 后的控制语义对应
Materializesubmit-pmu-materializesame-business-boundary独立 ELF，只可用自己的分母
PrepareMapsubmit-pmu-prepare-mapsame-business-boundary独立 ELF，只可用自己的分母
Claimsubmit-pmu-claimsame-business-boundary独立 ELF，只可用自己的分母
Faninsubmit-pmu-faninsame-business-boundary动态 winner 调用区间
Registersubmit-pmu-registersame-business-boundary独立 ELF，只可用自己的分母
WinnerBuildsubmit-pmu-winner-build-controlcontrol-only泳道父 span 可能含 Kernel；PMU 排除 linked Kernel
AllocCompletesubmit-pmu-alloc-complete-controlcontrol-only泳道父 span 可能含 Kernel；PMU 排除 linked Kernel
LoserReplaysubmit-pmu-loser-replaysame-business-boundary真实路径不执行 linked Kernel
BetweenSubmitResidualsubmit-pmu-submit-transitionadjacent-submit-boundary聚合每核相邻 Submit 的 N-1 个间隙
OrchestrationTail—not-covered当前没有独立 PMU phase
FinalDrain—outside-submit-pmu发生在末次 Submit 之后，不进入当前 PMU 窗
FinalDrainKernelUnion—outside-submit-pmu末次 Submit 后的 linked Kernel，不属于纯 Submit Scalar 分母
FinalDrainResidual—outside-submit-pmu末次 Submit 后的控制残余
SubmitResidualsubmit-pmu-arg-buildpartial仅 Claim→Materialize internal residual 有同边界 PMU；tail 未覆盖
SubmitInternalResidualsubmit-pmu-arg-buildpartial逐 segment 见下列动态覆盖项
SubmitTailResidual—not-covered逐 segment 列出但当前没有独立 PMU phase
SubmitInternalResidual/Claim->Materializesubmit-pmu-arg-buildresidual-boundaryArgBuild 同边界
SubmitTailResidual/LoserReplay->SubmitEnd—not-covered当前没有独立 PMU phase
SubmitTailResidual/Register->SubmitEnd—not-covered当前没有独立 PMU phase
SubmitTailResidual/WinnerBuild->SubmitEnd—not-covered当前没有独立 PMU phase
SubmitTailResidual/AllocComplete->SubmitEnd—not-covered当前没有独立 PMU phase
Atomic overlay—swimlane-overlayPMU 时间只扣 result-used return-ready 等待；counter 仍含 atomic 指令事件
ClockBaseline overlay—swimlane-overlay非加和泳道 overlay；当前没有独立 PMU phase
Commit overlay—swimlane-overlay非加和泳道 overlay；当前没有独立 PMU phase
RingBp overlay—swimlane-overlay非加和泳道 overlay；当前没有独立 PMU phase
DrainWon overlay—swimlane-overlay非加和泳道 overlay；当前没有独立 PMU phase
观察器校准submit-pmu-empty-bracketobserver-calibration不是业务 span，不进入任何业务分布
+ +

机器可读数据见同目录 fdwic_submit_span_overview.json。所有逐核分布只呈现 min–max； + phase extrema 是每核累计完整 phase 的极值，不是单次调用极值。

+
diff --git a/tests/atomic_probe/pa_scheduler/test_swimlane_converter.py b/tests/atomic_probe/pa_scheduler/test_swimlane_converter.py new file mode 100644 index 0000000000..0bf9faf620 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_swimlane_converter.py @@ -0,0 +1,2506 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +"""standalone 泳道转换器的最小布局回归。""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +try: + # `python -m unittest tests.atomic_probe...` 以 namespace package 导入。 + from .swimlane_converter import ( + _derive_v4_task_kinds, + _restore_v5_shared_efdrain, + convert, + ) +except ImportError: + # 也保留从本目录直接执行脚本的用法。 + from swimlane_converter import ( + _derive_v4_task_kinds, + _restore_v5_shared_efdrain, + convert, + ) + + +def _standalone_topology(core_id: int) -> tuple[int, int, str]: + """返回 standalone 固定 32 AIC + 64 AIV 拓扑中的 block/lane/type。""" + if core_id < 32: + return core_id, 0, "aic" + vector_id = core_id - 32 + return vector_id // 2, 1 + vector_id % 2, "aiv" + + +def _v3_capture( + rows: list[list[object]], + *, + num_cores: int = 1, + add_clock_baselines: bool = True, + dependency_applied: bool = True, +) -> dict[str, object]: + """构造带 producer weighted summary 的最小 schema-v3 raw。""" + all_rows = [list(row) for row in rows] + if add_clock_baselines: + dependency_flags = 0x3 if dependency_applied else 0x1 + for core_id in range(num_cores): + block_id, lane, _ = _standalone_topology(core_id) + start = 10 + core_id * 4 + all_rows.extend( + [ + [ + core_id, + block_id, + lane, + -1, + -1, + "ClockBaseline", + start, + start + 1, + 0, + 0, + ], + [ + core_id, + block_id, + lane, + -1, + -1, + "ClockBaseline", + start + 2, + start + 3, + dependency_flags, + 0, + ], + ] + ) + atomic_rows = [row for row in all_rows if row[5] == "Atomic"] + batch_rows = [row for row in atomic_rows if int(row[8]) & 0x80] + batch_calls = sum((int(row[8]) >> 8) & 0xFFFFFF for row in batch_rows) + core_types = [_standalone_topology(core_id)[2] for core_id in range(num_cores)] + summary = { + "records": len(all_rows), + "atomic_records": len(atomic_rows), + "clock_baseline_records": sum( + row[5] == "ClockBaseline" for row in all_rows + ), + "atomic_calls": len(atomic_rows) - len(batch_rows) + batch_calls, + "batched_poll_calls": batch_calls, + "poll_batch_records": len(batch_rows), + "dropped_records": 0, + } + return { + "l2_swimlane_level": 4, + "metadata": { + "clock_freq_hz": 1_000_000_000, + "num_cores": num_cores, + "trace_schema_version": 3, + "core_types": core_types, + "fdwic_summary": summary, + }, + "fdwic_events": all_rows, + } + + +def _v5_capture( + rows: list[list[object]], + *, + num_cores: int = 1, + add_parents: bool = True, + tensormap_mode: str = "private", +) -> dict[str, object]: + """构造 phase-only schema-v5 raw；调用者显式提供 Claim/Submit/尾动作。""" + all_rows = [list(row) for row in rows] + if add_parents: + for core_id in range(num_cores): + block_id, lane, _ = _standalone_topology(core_id) + submit_ends = [ + int(row[7]) + for row in all_rows + if int(row[0]) == core_id and row[5] == "Submit" + ] + orchestration_end = max([200, *submit_ends]) + all_rows.extend( + [ + [ + core_id, + block_id, + lane, + -1, + -1, + "OrchestrationReplay", + 90, + orchestration_end, + 0, + 0, + ], + [ + core_id, + block_id, + lane, + -1, + -1, + "FinalDrain", + orchestration_end, + orchestration_end + 20, + 0, + 0, + ], + ] + ) + return { + "l2_swimlane_level": 1, + "metadata": { + "clock_freq_hz": 1_000_000_000, + "num_cores": num_cores, + "trace_schema_version": 5, + "tensormap_mode": tensormap_mode, + "core_types": [ + _standalone_topology(core_id)[2] for core_id in range(num_cores) + ], + "fdwic_summary": { + "records": len(all_rows), + "atomic_records": 0, + "clock_baseline_records": 0, + "atomic_calls": 0, + "batched_poll_calls": 0, + "poll_batch_records": 0, + "dropped_records": 0, + }, + }, + "fdwic_events": all_rows, + } + + +def _refresh_summary(capture: dict[str, object]) -> None: + """按当前 raw 行重新生成 producer weighted summary。""" + + rows = capture["fdwic_events"] + metadata = capture["metadata"] + assert isinstance(rows, list) + assert isinstance(metadata, dict) + atomic_rows = [row for row in rows if row[5] == "Atomic"] + batch_rows = [row for row in atomic_rows if int(row[8]) & 0x80] + batch_calls = sum((int(row[8]) >> 8) & 0xFFFFFF for row in batch_rows) + dcci_rows = [row for row in rows if row[5] == "Dcci"] + summary = { + "records": len(rows), + "atomic_records": len(atomic_rows), + "clock_baseline_records": sum( + row[5] == "ClockBaseline" for row in rows + ), + "atomic_calls": len(atomic_rows) - len(batch_rows) + batch_calls, + "batched_poll_calls": batch_calls, + "poll_batch_records": len(batch_rows), + "dropped_records": 0, + } + if dcci_rows: + summary.update( + { + "dcci_records": len(dcci_rows), + "dcci_calls": sum( + (int(row[8]) >> 3) & 0xF for row in dcci_rows + ), + "dcci_lines": sum( + (int(row[8]) >> 8) & 0xFFFFFF + for row in dcci_rows + ), + } + ) + metadata["fdwic_summary"] = summary + + +def _v5_shared_register_atomic_capture( + *, + dependency_applied: bool = True, +) -> dict[str, object]: + """构造一批五个 winner 的 per-task predecessor-chain v5 raw。 + + task 0 无前驱，因此没有 PollBatch；task 1..4 各有一条聚合 + PollBatch；五个 task 都各自用一条 completion CAS 发布完成。 + """ + + rows: list[list[object]] = [] + for task_id in range(5): + base = 100 + task_id * 50 + is_alloc = task_id == 0 + function_id = -1 if is_alloc else task_id - 1 + rows.extend( + [ + [ + 0, + 0, + 0, + task_id, + -1, + "Claim", + base + 10, + base + 15, + 0x3, + 1 if is_alloc else 0, + ], + [ + 0, + 0, + 0, + task_id, + function_id, + "Materialize", + base + 15, + base + 20, + 0, + 1, + ], + [ + 0, + 0, + 0, + task_id, + function_id, + "Register", + base + 20, + base + 40, + 0, + 0, + ], + [ + 0, + 0, + 0, + task_id, + function_id, + "SharedRegisterPublishMetadata", + base + 24, + base + 34, + 0, + 0, + ], + [ + 0, + 0, + 0, + task_id, + function_id, + "SharedRegisterPublishTaskOutputs", + base + 29, + base + 32, + 0, + 0, + ], + [ + 0, + 0, + 0, + task_id, + function_id, + "SharedRegisterPublishTaskOutputsCopy", + base + 29, + base + 30, + 0, + 0, + ], + [ + 0, + 0, + 0, + task_id, + function_id, + "SharedRegisterPublishTaskOutputsFlush", + base + 30, + base + 32, + 0, + 0, + ], + ] + ) + if is_alloc: + rows.append( + [ + 0, + 0, + 0, + task_id, + -1, + "AllocComplete", + base + 40, + base + 45, + 0, + 0, + ] + ) + else: + rows.extend( + [ + [ + 0, + 0, + 0, + task_id, + function_id, + "Fanin", + base + 40, + base + 43, + 0, + 0, + ], + [ + 0, + 0, + 0, + task_id, + function_id, + "WinnerBuild", + base + 43, + base + 45, + 0, + 0, + ], + ] + ) + rows.append( + [ + 0, + 0, + 0, + task_id, + -1, + "Submit", + base, + base + 50, + 1, + 1 if is_alloc else 0, + ] + ) + + capture = _v5_capture(rows, tensormap_mode="shared") + capture["l2_swimlane_level"] = 4 + capture_rows = capture["fdwic_events"] + assert isinstance(capture_rows, list) + dependency_flags = 0x3 if dependency_applied else 0x1 + capture_rows.extend( + [ + [0, 0, 0, -1, -1, "ClockBaseline", 10, 11, 0, 0], + [ + 0, + 0, + 0, + -1, + -1, + "ClockBaseline", + 12, + 13, + dependency_flags, + 0, + ], + ] + ) + for task_id in range(5): + base = 100 + task_id * 50 + if task_id > 0: + # 三次 Load（两次 Pending + 最后一次 Ready）聚成一条等待 + # episode；task 0 没有前驱，不产生此记录。 + capture_rows.append( + [ + 0, + 0, + 0, + -1, + -1, + "Atomic", + base + 20, + base + 24, + (3 << 8) | 0xD0, + 19, + ] + ) + # CompareExchange(4) | result-used | return-ready。每个 task 都发布 + # 自己的 completion，包括没有前驱的 task 0。 + capture_rows.append( + [ + 0, + 0, + 0, + task_id, + -1, + "Atomic", + base + 34, + base + 40, + 0x54, + 20, + ] + ) + _refresh_summary(capture) + return capture + + +class SwimlaneConverterLayoutTest(unittest.TestCase): + def test_v5_shared_restores_efdrain_without_raw_record_growth( + self, + ) -> None: + capture = _v5_shared_register_atomic_capture() + rows = capture["fdwic_events"] + metadata = capture["metadata"] + assert isinstance(rows, list) + assert isinstance(metadata, dict) + raw_count = len(rows) + self.assertFalse(any(row[5] == "EfDrain" for row in rows)) + self.assertEqual(metadata["fdwic_summary"]["records"], raw_count) + restored_rows = [tuple(row) for row in rows] + _restore_v5_shared_efdrain(restored_rows, 5, "shared") + restored_efdrains = [ + row for row in restored_rows if row[5] == "EfDrain" + ] + self.assertEqual(len(restored_efdrains), 5) + # 五个 task 都是 winner；EfDrain 属于 scalar Submit 前端，不把 + # QK/SF/PV/UP 的 function_id 错挂到派生事件上。 + self.assertTrue(all(row[4] == -1 for row in restored_efdrains)) + + def converted_efdrains( + directory: str, + source: dict[str, object], + ) -> list[tuple[str, float, float]]: + input_path = Path(directory) / "derived.raw.json" + output_path = Path(directory) / "derived.merged.json" + input_path.write_text(json.dumps(source), encoding="utf-8") + convert(input_path, output_path) + merged = json.loads(output_path.read_text(encoding="utf-8")) + return sorted( + ( + str(event["name"]), + float(event["ts"]), + float(event["dur"]), + ) + for event in merged["traceEvents"] + if str(event.get("name", "")).startswith("efdrain#") + ) + + with tempfile.TemporaryDirectory() as directory: + derived = converted_efdrains(directory, capture) + + self.assertEqual(len(derived), 5) + self.assertEqual( + [name for name, _start, _duration in derived], + [f"efdrain#{task_id}" for task_id in range(5)], + ) + self.assertTrue( + all(duration == 0.01 for _name, _start, duration in derived) + ) + + def test_v5_shared_efdrain_derivation_rejects_invalid_evidence( + self, + ) -> None: + for label, expected_error in ( + ("missing_claim", "Claim keys do not match Submit keys"), + ("inverted_boundary", "Claim is outside or inverted"), + ("explicit_efdrain", "must not contain explicit EfDrain"), + ): + with self.subTest(label=label): + capture = _v5_shared_register_atomic_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + if label == "missing_claim": + rows[:] = [ + row + for row in rows + if not ( + row[0] == 0 + and row[3] == 0 + and row[5] == "Claim" + ) + ] + elif label == "inverted_boundary": + claim = next( + row + for row in rows + if row[0] == 0 + and row[3] == 0 + and row[5] == "Claim" + ) + claim[6:8] = [95, 99] + else: + rows.append( + [0, 0, 0, 0, -1, "EfDrain", 100, 110, 0, 0] + ) + _refresh_summary(capture) + + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text( + json.dumps(capture), encoding="utf-8" + ) + with self.assertRaisesRegex( + ValueError, expected_error + ): + convert(input_path, output_path) + + def test_v5_private_accepts_startup_and_terminal_dcci_rows(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 120, 0x3, 1], + [0, 0, 0, 0, -1, "AllocComplete", 120, 130, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 140, 1, 1], + ] + ) + rows = capture["fdwic_events"] + assert isinstance(rows, list) + # startup: invalidate + DSB, one call across three lines。 + rows.append([0, 0, 0, -1, -1, "Dcci", 92, 96, 0x30C, 9]) + # terminal observer: clean + DSB, two calls across two lines。 + rows.append([0, 0, 0, -1, -1, "Dcci", 220, 224, 0x215, 8]) + _refresh_summary(capture) + + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + convert(input_path, output_path) + merged = json.loads(output_path.read_text(encoding="utf-8")) + + names = { + event.get("name") for event in merged["traceEvents"] + } + self.assertIn( + "dcci.startup_config_invalidate.invalidate×1.lines3#-1", + names, + ) + self.assertIn( + "dcci.observer_trace_export.clean_out×2.lines2#-1", + names, + ) + summary = merged["metadata"]["fdwic_summary"] + self.assertEqual(summary["dcci_records"], 2) + self.assertEqual(summary["dcci_calls"], 3) + self.assertEqual(summary["dcci_lines"], 5) + + def test_v5_shared_accepts_three_region_terminal_dcci_row(self) -> None: + capture = _v5_shared_register_atomic_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + # shared observer 依次 clean 专用 Submit/Claim 区、通用记录区和 + # core state，因此一条聚合 row 表示三次区域原语。 + rows.extend( + [ + [0, 0, 0, -1, -1, "Dcci", 92, 96, 0x30C, 9], + [0, 0, 0, -1, -1, "Dcci", 420, 424, 0x31D, 8], + ] + ) + _refresh_summary(capture) + + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + convert(input_path, output_path) + merged = json.loads(output_path.read_text(encoding="utf-8")) + + names = { + event.get("name") for event in merged["traceEvents"] + } + self.assertIn( + "dcci.observer_trace_export.clean_out×3.lines3#-1", + names, + ) + summary = merged["metadata"]["fdwic_summary"] + self.assertEqual(summary["dcci_records"], 2) + self.assertEqual(summary["dcci_calls"], 4) + self.assertEqual(summary["dcci_lines"], 6) + + def test_v5_rejects_invalid_startup_dcci_identity(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 120, 0x3, 1], + [0, 0, 0, 0, -1, "AllocComplete", 120, 130, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 140, 1, 1], + ] + ) + rows = capture["fdwic_events"] + assert isinstance(rows, list) + rows.extend( + [ + [0, 0, 0, 0, -1, "Dcci", 92, 96, 0x30C, 9], + [0, 0, 0, -1, -1, "Dcci", 220, 224, 0x215, 8], + ] + ) + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex( + ValueError, "invalid startup Dcci fields" + ): + convert(input_path, output_path) + + def test_v4_requires_explicit_tensormap_mode(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 120, 0x3, 1], + [0, 0, 0, 0, -1, "AllocComplete", 120, 130, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 140, 1, 1], + ] + ) + metadata = capture["metadata"] + assert isinstance(metadata, dict) + metadata.pop("tensormap_mode") + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "tensormap_mode"): + convert(input_path, output_path) + + def test_v4_shared_rejects_private_prepare_map_record(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 115, 0x3, 1], + [0, 0, 0, 0, -1, "Materialize", 116, 120, 0, 1], + [0, 0, 0, 0, -1, "PrepareMap", 120, 120, 0, 1], + [0, 0, 0, 0, -1, "Register", 121, 125, 0, 0], + [0, 0, 0, 0, -1, "AllocComplete", 126, 130, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 140, 1, 1], + ], + tensormap_mode="shared", + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "must not contain PrepareMap"): + convert(input_path, output_path) + + def test_v4_shared_register_detail_splits_parent_with_one_raw_row(self) -> None: + rows = [ + [0, 0, 0, 0, -1, "Claim", 110, 115, 0x3, 1], + [0, 0, 0, 0, -1, "Materialize", 115, 120, 0, 1], + [0, 0, 0, 0, -1, "Register", 120, 140, 0, 0], + [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishMetadata", + 124, + 134, + 0, + 0, + ], + [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishTaskOutputs", + 129, + 132, + 0, + 0, + ], + [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishTaskOutputsCopy", + 129, + 130, + 0, + 0, + ], + [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishTaskOutputsFlush", + 130, + 132, + 0, + 0, + ], + [0, 0, 0, 0, -1, "AllocComplete", 140, 145, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 150, 1, 1], + ] + capture = _v5_capture(rows, tensormap_mode="shared") + raw_rows = capture["fdwic_events"] + assert isinstance(raw_rows, list) + # 两条全核 parent 由 helper 加入；Register 使用 metadata 父 detail + # 加 task-outputs 及 copy/flush 两层子 detail。 + self.assertEqual(len(raw_rows), len(rows) + 2) + self.assertEqual( + sum(row[5] == "SharedRegisterPublishMetadata" for row in raw_rows), + 1, + ) + self.assertEqual( + sum( + row[5] == "SharedRegisterPublishTaskOutputs" + for row in raw_rows + ), + 1, + ) + self.assertEqual( + sum( + row[5] == "SharedRegisterPublishTaskOutputsCopy" + for row in raw_rows + ), + 1, + ) + self.assertEqual( + sum( + row[5] == "SharedRegisterPublishTaskOutputsFlush" + for row in raw_rows + ), + 1, + ) + + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + convert(input_path, output_path) + events = json.loads(output_path.read_text(encoding="utf-8"))["traceEvents"] + + parent = next(event for event in events if event.get("name") == "register#0") + flat_child_names = ( + "register.wait_predecessor_tensormap_insert#0", + "register.publish_writer_metadata" + "[ordinary_tensormap_entries=0]#0", + "register.publish_task_outputs#0", + "register.publish_metadata_epilogue#0", + "register.publish_tensormap_insert_completion#0", + ) + nested_output_names = ( + "register.publish_task_outputs.copy#0", + "register.publish_task_outputs.flush#0", + ) + metadata_name = "register.publish_metadata#0" + children = { + event["name"]: event + for event in events + if event.get("name") + in (*flat_child_names, metadata_name, *nested_output_names) + } + self.assertEqual( + set(children), + {*flat_child_names, metadata_name, *nested_output_names}, + ) + self.assertAlmostEqual( + children["register.publish_task_outputs.copy#0"]["ts"], + children["register.publish_task_outputs#0"]["ts"], + ) + self.assertAlmostEqual( + children["register.publish_task_outputs.copy#0"]["ts"] + + children["register.publish_task_outputs.copy#0"]["dur"], + children["register.publish_task_outputs.flush#0"]["ts"], + ) + self.assertAlmostEqual( + children["register.publish_task_outputs.flush#0"]["ts"] + + children["register.publish_task_outputs.flush#0"]["dur"], + children["register.publish_task_outputs#0"]["ts"] + + children["register.publish_task_outputs#0"]["dur"], + ) + for child in children.values(): + self.assertEqual( + set(child), {"ph", "name", "pid", "tid", "ts", "dur"} + ) + self.assertEqual( + children["register.wait_predecessor_tensormap_insert#0"]["ts"], + parent["ts"], + ) + self.assertAlmostEqual( + children["register.wait_predecessor_tensormap_insert#0"]["ts"] + + children[ + "register.wait_predecessor_tensormap_insert#0" + ]["dur"], + children[ + "register.publish_writer_metadata" + "[ordinary_tensormap_entries=0]#0" + ]["ts"], + ) + self.assertAlmostEqual( + children[ + "register.publish_writer_metadata" + "[ordinary_tensormap_entries=0]#0" + ]["ts"] + + children[ + "register.publish_writer_metadata" + "[ordinary_tensormap_entries=0]#0" + ]["dur"], + children["register.publish_task_outputs#0"]["ts"], + ) + self.assertAlmostEqual( + children["register.publish_task_outputs#0"]["ts"] + + children["register.publish_task_outputs#0"]["dur"], + children["register.publish_metadata_epilogue#0"]["ts"], + ) + self.assertAlmostEqual( + children["register.publish_metadata_epilogue#0"]["ts"] + + children["register.publish_metadata_epilogue#0"]["dur"], + children["register.publish_tensormap_insert_completion#0"]["ts"], + ) + self.assertAlmostEqual( + children["register.publish_tensormap_insert_completion#0"]["ts"] + + children[ + "register.publish_tensormap_insert_completion#0" + ]["dur"], + parent["ts"] + parent["dur"], + ) + self.assertAlmostEqual( + sum(children[name]["dur"] for name in flat_child_names), + parent["dur"], + ) + self.assertAlmostEqual( + sum( + children[name]["dur"] + for name in ( + "register.publish_writer_metadata" + "[ordinary_tensormap_entries=0]#0", + "register.publish_task_outputs#0", + "register.publish_metadata_epilogue#0", + ) + ), + children[metadata_name]["dur"], + ) + + def test_v5_materialize_output_detail_leaves_register_serial_only( + self, + ) -> None: + rows = [ + [0, 0, 0, 0, -1, "Claim", 110, 115, 0x3, 1], + [0, 0, 0, 0, -1, "Materialize", 115, 125, 0, 1], + [ + 0, + 0, + 0, + 0, + -1, + "SharedMaterializePublishTaskOutputs", + 120, + 124, + 0, + 0, + ], + [ + 0, + 0, + 0, + 0, + -1, + "SharedMaterializePublishTaskOutputsCopy", + 120, + 121, + 0, + 0, + ], + [ + 0, + 0, + 0, + 0, + -1, + "SharedMaterializePublishTaskOutputsFlush", + 121, + 123, + 0, + 0, + ], + # descriptor flush 与业务 flush span 使用完全相同的端点； + # merged 必须先输出业务父区间，再输出 DCCI overlay。 + [0, 0, 0, 0, -1, "Dcci", 121, 123, 0x10D, 3], + [0, 0, 0, 0, -1, "Register", 125, 140, 0, 0], + [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishMetadata", + 129, + 135, + 0, + 0, + ], + # SharedOutputRef 的 writer commit 会消费 CAS 返回值，因此泳道 + # 必须明确显示为 return_ready；相邻 DCCI 仍归属同一个 Register + # writer metadata 子区间。 + [0, 0, 0, 0, -1, "Atomic", 130, 131, 0x54, 27], + [0, 0, 0, 0, -1, "Dcci", 131, 132, 0x10D, 1], + [0, 0, 0, 0, -1, "Atomic", 136, 137, 0x54, 20], + [0, 0, 0, 0, -1, "AllocComplete", 140, 145, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 150, 1, 1], + [0, 0, 0, -1, -1, "ClockBaseline", 10, 11, 0, 0], + [0, 0, 0, -1, -1, "ClockBaseline", 12, 13, 0x3, 0], + [0, 0, 0, -1, -1, "Dcci", 220, 224, 0x31D, 8], + ] + capture = _v5_capture(rows, tensormap_mode="shared") + capture["l2_swimlane_level"] = 4 + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + convert(input_path, output_path) + events = json.loads( + output_path.read_text(encoding="utf-8") + )["traceEvents"] + + names = {event.get("name") for event in events} + positions = { + event.get("name"): index + for index, event in enumerate(events) + } + self.assertIn( + "materialize.publish_shared_output_descriptors#0", names + ) + self.assertIn( + "materialize.publish_shared_output_descriptors" + ".copy_tensor_descs#0", + names, + ) + self.assertIn( + "materialize.publish_shared_output_descriptors" + ".flush_tensor_descs#0", + names, + ) + self.assertIn( + "register.wait_predecessor_tensormap_insert#0", names + ) + self.assertIn( + "register.publish_writer_metadata" + "[ordinary_tensormap_entries=0]#0", + names, + ) + self.assertIn( + "register.publish_tensormap_insert_completion#0", names + ) + self.assertIn( + "atomic.return_ready.shared_output_ref_last_writer_commit" + ".compare_exchange#0", + names, + ) + self.assertIn( + "dcci.shared_output_ref_writer_history_flush" + ".clean_out×1.lines1#0", + names, + ) + self.assertIn( + "dcci.shared_output_descriptor_flush" + ".clean_out×1.lines1#0", + names, + ) + self.assertLess( + positions["materialize#0"], + positions[ + "materialize.publish_shared_output_descriptors#0" + ], + ) + self.assertLess( + positions[ + "materialize.publish_shared_output_descriptors" + ".flush_tensor_descs#0" + ], + positions[ + "dcci.shared_output_descriptor_flush" + ".clean_out×1.lines1#0" + ], + ) + self.assertLess( + positions["register#0"], + positions[ + "register.publish_writer_metadata" + "[ordinary_tensormap_entries=0]#0" + ], + ) + self.assertLess( + positions[ + "register.publish_writer_metadata" + "[ordinary_tensormap_entries=0]#0" + ], + positions[ + "dcci.shared_output_ref_writer_history_flush" + ".clean_out×1.lines1#0" + ], + ) + self.assertNotIn("register.publish_metadata#0", names) + self.assertNotIn("register.publish_task_outputs#0", names) + self.assertNotIn("register.publish_metadata_epilogue#0", names) + + def test_v4_shared_register_detail_is_required_exactly_once_for_winner( + self, + ) -> None: + base_rows = [ + [0, 0, 0, 0, -1, "Claim", 110, 115, 0x3, 1], + [0, 0, 0, 0, -1, "Register", 120, 140, 0, 0], + [0, 0, 0, 0, -1, "AllocComplete", 140, 145, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 150, 1, 1], + ] + detail = [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishMetadata", + 124, + 134, + 0, + 0, + ] + cases = { + "missing": base_rows, + "duplicate": [*base_rows, detail, list(detail)], + } + for label, rows in cases.items(): + with self.subTest(label=label), tempfile.TemporaryDirectory() as directory: + capture = _v5_capture(rows, tensormap_mode="shared") + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex( + ValueError, "requires exactly one SharedRegisterPublishMetadata" + ): + convert(input_path, output_path) + + def test_v5_task_outputs_detail_is_strictly_nested_once(self) -> None: + for label in ("missing", "duplicate", "outside_metadata", "wrong_identity"): + with self.subTest(label=label), tempfile.TemporaryDirectory() as directory: + capture = _v5_shared_register_atomic_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + output_detail = next( + row + for row in rows + if row[0] == 0 + and row[3] == 0 + and row[5] == "SharedRegisterPublishTaskOutputs" + ) + if label == "missing": + rows.remove(output_detail) + elif label == "duplicate": + rows.append(list(output_detail)) + elif label == "outside_metadata": + output_detail[6] = 123 + else: + output_detail[4] = 0 + _refresh_summary(capture) + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + expected = { + "missing": "requires exactly one SharedRegisterPublishTaskOutputs", + "duplicate": "requires exactly one SharedRegisterPublishTaskOutputs", + "outside_metadata": "outside SharedRegisterPublishMetadata", + "wrong_identity": "identity differs", + }[label] + with self.assertRaisesRegex(ValueError, expected): + convert(input_path, output_path) + + def test_v5_rejects_old_schema_v4_raw(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 120, 0x3, 1], + [0, 0, 0, 0, -1, "AllocComplete", 120, 130, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 140, 1, 1], + ] + ) + metadata = capture["metadata"] + assert isinstance(metadata, dict) + metadata["trace_schema_version"] = 4 + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex( + ValueError, "unsupported metadata.trace_schema_version: 4" + ): + convert(input_path, output_path) + + def test_v4_shared_register_detail_rejects_bad_boundary_or_identity( + self, + ) -> None: + base_rows = [ + [0, 0, 0, 0, -1, "Claim", 110, 115, 0x3, 1], + [0, 0, 0, 0, -1, "Register", 120, 140, 0, 0], + [0, 0, 0, 0, -1, "AllocComplete", 140, 145, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 150, 1, 1], + ] + cases = { + "outside_parent": [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishMetadata", + 119, + 134, + 0, + 0, + ], + "different_function": [ + 0, + 0, + 0, + 0, + 0, + "SharedRegisterPublishMetadata", + 124, + 134, + 0, + 0, + ], + } + for label, detail in cases.items(): + with self.subTest(label=label), tempfile.TemporaryDirectory() as directory: + output_detail = [ + detail[0], + detail[1], + detail[2], + detail[3], + detail[4], + "SharedRegisterPublishTaskOutputs", + 129, + 132, + 0, + 0, + ] + capture = _v5_capture( + [*base_rows, detail, output_detail], + tensormap_mode="shared", + ) + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + expected = ( + "outside Register parent" + if label == "outside_parent" + else "identity differs" + ) + with self.assertRaisesRegex(ValueError, expected): + convert(input_path, output_path) + + def test_v4_rejects_shared_register_detail_in_private_mode(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 115, 0x3, 1], + [0, 0, 0, 0, -1, "Register", 120, 140, 0, 0], + [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishMetadata", + 124, + 134, + 0, + 0, + ], + [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishTaskOutputs", + 129, + 132, + 0, + 0, + ], + [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishTaskOutputs", + 129, + 132, + 0, + 0, + ], + [0, 0, 0, 0, -1, "AllocComplete", 140, 145, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 150, 1, 1], + ] + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex( + ValueError, "only valid for shared TensorMap" + ): + convert(input_path, output_path) + + def test_v4_shared_register_detail_is_forbidden_for_loser(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 115, 0x2, 1], + [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishMetadata", + 124, + 134, + 0, + 0, + ], + [0, 0, 0, 0, -1, "Submit", 100, 150, 0, 1], + ], + tensormap_mode="shared", + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "and none for losers"): + convert(input_path, output_path) + + def test_v4_shared_register_parent_is_forbidden_for_loser(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 115, 0x2, 1], + [0, 0, 0, 0, -1, "Register", 120, 140, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 150, 0, 1], + ], + tensormap_mode="shared", + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex( + ValueError, "Register parent for each winner and none for losers" + ): + convert(input_path, output_path) + + def test_v4_shared_rejects_register_parent_without_claim(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 115, 0x3, 1], + [0, 0, 0, 0, -1, "Register", 120, 140, 0, 0], + [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishMetadata", + 124, + 134, + 0, + 0, + ], + [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishTaskOutputs", + 129, + 132, + 0, + 0, + ], + [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishTaskOutputsCopy", + 129, + 130, + 0, + 0, + ], + [ + 0, + 0, + 0, + 0, + -1, + "SharedRegisterPublishTaskOutputsFlush", + 130, + 132, + 0, + 0, + ], + [0, 0, 0, 0, -1, "AllocComplete", 140, 145, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 150, 1, 1], + # task 9 没有 Claim/Submit，不能让独立 converter 静默接收。 + [0, 0, 0, 9, -1, "Register", 151, 152, 0, 0], + ], + tensormap_mode="shared", + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex( + ValueError, "Register parents have no matching Claim" + ): + convert(input_path, output_path) + + def test_v4_splits_internal_and_tail_residual_without_repeated_fields(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 120, 0x3, 1], + [0, 0, 0, 0, -1, "AllocComplete", 120, 130, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 140, 1, 1], + ] + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + convert(input_path, output_path) + merged = json.loads(output_path.read_text(encoding="utf-8")) + + self.assertEqual(merged["metadata"]["trace_schema_version"], 5) + events = merged["traceEvents"] + orchestration = next( + event for event in events if event.get("name") == "orchestration_replay" + ) + self.assertNotIn("args", orchestration) + self.assertNotIn("cat", orchestration) + residuals = [event for event in events if event.get("name") == "submit_residual"] + tails = [ + event for event in events if event.get("name") == "submit_tail_gap" + ] + self.assertEqual( + residuals, + [ + {"ph": "X", "name": "submit_residual", "pid": 0, "tid": 0, "ts": 0.01, "dur": 0.01}, + ], + ) + self.assertEqual( + tails, + [ + { + "ph": "X", + "name": "submit_tail_gap", + "pid": 0, + "tid": 0, + "ts": 0.04, + "dur": 0.01, + } + ], + ) + for event in (*residuals, *tails): + self.assertEqual(set(event), {"ph", "name", "pid", "tid", "ts", "dur"}) + + def test_v4_marks_between_submit_gap_without_loser_marker(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 120, 0x2, 1], + [0, 0, 0, 0, -1, "Submit", 100, 140, 0, 1], + [0, 0, 0, 1, -1, "Claim", 170, 180, 0x2, 0], + [0, 0, 0, 1, -1, "Submit", 160, 200, 0, 0], + [0, 0, 0, 2, -1, "Claim", 210, 220, 0x2, 0], + [0, 0, 0, 2, -1, "Submit", 200, 240, 0, 0], + [0, 0, 0, 3, -1, "Claim", 250, 260, 0x2, 0], + [0, 0, 0, 3, -1, "Submit", 240, 280, 0, 0], + [0, 0, 0, 4, -1, "Claim", 290, 300, 0x2, 0], + [0, 0, 0, 4, -1, "Submit", 280, 320, 0, 0], + ] + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + convert(input_path, output_path) + events = json.loads(output_path.read_text(encoding="utf-8"))["traceEvents"] + + internal = [event for event in events if event.get("name") == "submit_residual"] + tails = [ + event for event in events if event.get("name") == "submit_tail_gap" + ] + between = [ + event for event in events if event.get("name") == "between_submit_residual" + ] + self.assertEqual(len(internal), 5) + self.assertEqual(len(tails), 5) + self.assertEqual( + between, + [ + { + "ph": "X", + "name": "between_submit_residual", + "pid": 0, + "tid": 0, + "ts": 0.05, + "dur": 0.02, + } + ], + ) + # 每个完整 G1 task 仍只生成真实补集，不增加设备 raw 记录。 + self.assertEqual(len(internal) + len(tails) + len(between), 11) + for event in (*internal, *tails, *between): + self.assertEqual(set(event), {"ph", "name", "pid", "tid", "ts", "dur"}) + + def test_v4_rejects_legacy_lap_phase(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 120, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 140, 0, 1], + [0, 0, 0, 0, -1, "Replay", 100, 120, 0, 0], + ] + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "forbids legacy lap phase"): + convert(input_path, output_path) + + def test_v4_rejects_unused_drain_won_phase(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 120, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 140, 0, 0], + [0, 0, 0, 0, -1, "DrainWon", 121, 122, 0, 0], + ] + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "forbids unused legacy phase"): + convert(input_path, output_path) + + def test_v4_rejects_missing_winner_tail(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 120, 0x3, 1], + [0, 0, 0, 0, -1, "Submit", 100, 140, 1, 1], + ] + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "tail mismatch"): + convert(input_path, output_path) + + def test_v4_rejects_task_kind_that_disagrees_with_task_id(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 101, 102, 0, 1], + [0, 0, 0, 0, -1, "Submit", 100, 105, 0, 1], + # Submit 给出的动态 plan 是 QK；Claim 篡改成 Alloc。 + [0, 0, 0, 1, -1, "Claim", 111, 112, 0, 1], + [0, 0, 0, 1, -1, "Submit", 110, 115, 0, 0], + [0, 0, 0, 2, -1, "Claim", 121, 122, 0, 0], + [0, 0, 0, 2, -1, "Submit", 120, 125, 0, 0], + [0, 0, 0, 3, -1, "Claim", 131, 132, 0, 0], + [0, 0, 0, 3, -1, "Submit", 130, 135, 0, 0], + [0, 0, 0, 4, -1, "Claim", 141, 142, 0, 0], + [0, 0, 0, 4, -1, "Submit", 140, 145, 0, 0], + ] + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "task-kind mismatch"): + convert(input_path, output_path) + + def test_v4_derives_g0_g1_g2_g4_and_mixed_task_kinds(self) -> None: + """动态类型来自 Alloc 边界，不再把全局 task_id 按五取模。""" + + # mixed=[G0,G1,G2,G4]，各 batch 的 task 数为 1/5/9/17。 + group_counts = (0, 1, 2, 4) + alloc_task_ids: list[int] = [] + expected_kinds: dict[int, int] = {} + next_task = 0 + for group_count in group_counts: + alloc_task_ids.append(next_task) + expected_kinds[next_task] = 0 + next_task += 1 + for _group in range(group_count): + for kind_id in range(1, 5): + expected_kinds[next_task] = kind_id + next_task += 1 + + semantics = { + (core_id, task_id): ( + False, + task_id in alloc_task_ids, + ) + for core_id in range(3) + for task_id in range(next_task) + } + self.assertEqual(_derive_v4_task_kinds(semantics, 3), expected_kinds) + # G0 后紧邻下一个 batch Alloc，形成合法 Alloc->Alloc；随后 task 5 + # 是该 G1 的 UP。旧全局 task_id % 5 会把两者分别错判为 QK/Alloc。 + self.assertEqual([expected_kinds[0], expected_kinds[1]], [0, 0]) + self.assertEqual(expected_kinds[5], 4) + + def test_v4_rejects_cross_core_alloc_marker_disagreement(self) -> None: + semantics = { + (core_id, task_id): (False, task_id == 0) + for core_id in range(2) + for task_id in range(5) + } + semantics[(1, 1)] = (False, True) + with self.assertRaisesRegex(ValueError, "Alloc marker differs across cores"): + _derive_v4_task_kinds(semantics, 2) + + def test_v4_rejects_incomplete_dynamic_group(self) -> None: + semantics = { + (0, task_id): (False, task_id == 0) + for task_id in range(4) + } + with self.assertRaisesRegex(ValueError, "complete QK/SF/PV/UP groups"): + _derive_v4_task_kinds(semantics, 1) + + def test_v4_requires_both_parent_spans(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 120, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 140, 0, 1], + ], + add_parents=False, + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "requires exactly one schema-v5"): + convert(input_path, output_path) + + def test_v4_rejects_removed_loser_replay_phase(self) -> None: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 120, 0x2, 1], + [0, 0, 0, 0, -1, "LoserReplay", 120, 120, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 140, 0, 1], + ] + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "unknown phase 'LoserReplay'"): + convert(input_path, output_path) + + def test_real_compute_metadata_is_preserved_and_visible(self) -> None: + # raw 与 merged 都必须自描述真实 engine 负载；否则同名 QK/SF/PV/UP + # span 无法与历史 scalar-NOP 泳道区分。 + workload = { + "mode": "real-compute", + "counts": {"qk": 6, "sf": 28, "pv": 4, "up": 1}, + "unit": "complete_128x128_engine_pipeline_iteration", + "input_pattern": "layout-diagnostic", + "engine_mapping": { + "qk": "cube_matmul", + "sf": "vector_add", + "pv": "cube_matmul", + "up": "vector_mul", + }, + } + capture = { + "l2_swimlane_level": 1, + "metadata": { + "clock_freq_hz": 1_000_000_000, + "num_cores": 1, + "trace_schema_version": 2, + "winner_workload": workload, + "core_types": ["AIC"], + }, + "fdwic_events": [[0, 0, 0, 1, 0, "Kernel", 100, 200, 0, 0]], + } + + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + emitted, blocks, base_cycle = convert(input_path, output_path) + merged = json.loads(output_path.read_text(encoding="utf-8")) + + self.assertEqual((emitted, blocks, base_cycle), (2, 1, 100)) + self.assertEqual(merged["metadata"]["winner_workload"], workload) + capture_event = next( + event for event in merged["traceEvents"] + if event.get("name") == "pa_scheduler.capture" + ) + self.assertEqual(capture_event["args"]["winner_workload"], workload) + + def test_invalid_real_compute_input_pattern_is_rejected(self) -> None: + capture = { + "l2_swimlane_level": 1, + "metadata": { + "clock_freq_hz": 1_000_000_000, + "num_cores": 1, + "trace_schema_version": 2, + "winner_workload": { + "mode": "real-compute", + "counts": {"qk": 1, "sf": 1, "pv": 1, "up": 1}, + "unit": "complete_128x128_engine_pipeline_iteration", + "input_pattern": "unknown-layout", + "engine_mapping": { + "qk": "cube_matmul", + "sf": "vector_add", + "pv": "cube_matmul", + "up": "vector_mul", + }, + }, + "core_types": ["AIC"], + }, + "fdwic_events": [[0, 0, 0, 1, 0, "Kernel", 100, 200, 0, 0]], + } + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "input_pattern is invalid"): + convert(input_path, output_path) + self.assertFalse(output_path.exists()) + + def test_atomic_and_clock_share_the_scalar_lane(self) -> None: + # 同一 mixed block 放一条 AIC 和一条 AIV0；Atomic 是 AIC scalar + # 上 Claim 的子区间，ClockBaseline 也是 AIV0 scalar 指令，而 + # Kernel 是 AIV0 计算单元上的独立区间。 + capture = { + "l2_swimlane_level": 4, + "metadata": { + "clock_freq_hz": 1_000_000_000, + "num_cores": 2, + "trace_schema_version": 2, + "core_types": ["AIC", "AIV"], + }, + "fdwic_events": [ + # Claim flags: attempted(bit1)，本核输了所以 winner(bit0)=0。 + [0, 0, 0, 7, -1, "Claim", 100, 200, 0x2, 0], + # flags: FetchMax(3) | result-used(bit4) | return-ready(bit6)。 + [0, 0, 0, 7, -1, "Atomic", 120, 160, 0x53, 4], + # Exchange(1) 的旧值未消费，只能标 source-issue。 + [0, 0, 0, 7, -1, "Atomic", 161, 162, 0x01, 7], + # flags: dependency-hook(bit0) | dependency-applied(bit1)。 + [1, 0, 1, -1, -1, "ClockBaseline", 101, 102, 0x3, 0], + [1, 0, 1, 7, 0, "Kernel", 140, 180, 0, 0], + # 同一 AIV0 上的 role-filtered Claim，没有 atomic。 + [1, 0, 1, 8, -1, "Claim", 201, 220, 0x0, 0], + ], + } + + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + emitted, blocks, base_cycle = convert(input_path, output_path) + merged = json.loads(output_path.read_text(encoding="utf-8")) + + self.assertEqual((emitted, blocks, base_cycle), (6, 1, 100)) + events = merged["traceEvents"] + thread_names = { + (event["pid"], event["tid"]): event["args"]["name"] + for event in events + if event.get("ph") == "M" and event.get("name") == "thread_name" + } + self.assertEqual(thread_names[(0, 0)], "AIC (core0)") + self.assertEqual(thread_names[(0, 1)], "AIV0 (core1)") + self.assertEqual(thread_names[(0, 4)], "AIV0·kernel (core1)") + self.assertFalse(any("·atomic" in name for name in thread_names.values())) + + ready_atomic = next(event for event in events if event.get("cat") == "atomic.return_ready") + issue_atomic = next(event for event in events if event.get("cat") == "atomic.source_issue") + clock = next(event for event in events if event.get("cat") == "scalar_clock") + kernel = next(event for event in events if event.get("name") == "QK#7") + self.assertEqual((ready_atomic["pid"], ready_atomic["tid"]), (0, 0)) + self.assertEqual((issue_atomic["pid"], issue_atomic["tid"]), (0, 0)) + self.assertEqual((clock["pid"], clock["tid"]), (0, 1)) + self.assertEqual((kernel["pid"], kernel["tid"]), (0, 4)) + self.assertEqual( + ready_atomic["name"], "atomic.return_ready.claim_max.fetch_max#7" + ) + self.assertEqual( + issue_atomic["name"], "atomic.source_issue.completion_flag_exchange.exchange#7" + ) + self.assertEqual(ready_atomic["args"]["execution_unit"], "scalar") + self.assertEqual(issue_atomic["args"]["execution_unit"], "scalar") + self.assertEqual(clock["args"]["execution_unit"], "scalar") + self.assertEqual(ready_atomic["args"]["completion_boundary"], "return_value_ready") + self.assertEqual(issue_atomic["args"]["completion_boundary"], "source_issue_bracket") + attempted_claim = next(event for event in events if event.get("name") == "claim.lost#7") + skipped_claim = next(event for event in events if event.get("name") == "claim.not_attempted#8") + self.assertTrue(attempted_claim["args"]["claim_attempted"]) + self.assertFalse(skipped_claim["args"]["claim_attempted"]) + self.assertEqual(attempted_claim["args"]["claim_attempted_source"], "raw_flag") + + def test_v4_shared_register_atomics_are_named_on_scalar_lane(self) -> None: + capture = _v5_shared_register_atomic_capture() + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + convert(input_path, output_path) + merged = json.loads(output_path.read_text(encoding="utf-8")) + + events = merged["traceEvents"] + poll = next( + event + for event in events + if event.get("name") + == ( + "atomic.poll_batch.return_ready." + "shared_insert_predecessor_poll.load×3" + ) + ) + handoff = next( + event + for event in events + if event.get("name") + == ( + "atomic.return_ready.shared_insert_completion_publish." + "compare_exchange#0" + ) + ) + register = next( + event for event in events if event.get("name") == "register#0" + ) + self.assertEqual((poll["pid"], poll["tid"]), (0, 0)) + self.assertEqual((handoff["pid"], handoff["tid"]), (0, 0)) + self.assertEqual((register["pid"], register["tid"]), (0, 0)) + # schema-v5 为控制近 300 MiB 产物，只保留 Perfetto X 必需字段； + # poll_batch/return_ready/site/op/call_count 已完整编码在可见名称中。 + self.assertEqual( + set(poll), {"ph", "name", "pid", "tid", "ts", "dur"} + ) + self.assertEqual( + set(handoff), {"ph", "name", "pid", "tid", "ts", "dur"} + ) + summary = merged["metadata"]["fdwic_summary"] + self.assertEqual(summary["atomic_records"], 9) + self.assertEqual(summary["atomic_calls"], 17) + self.assertEqual(summary["batched_poll_calls"], 12) + self.assertEqual(summary["poll_batch_records"], 4) + thread_names = { + event["args"]["name"] + for event in events + if event.get("ph") == "M" + and event.get("name") == "thread_name" + } + self.assertFalse(any("·atomic" in name for name in thread_names)) + + def test_v4_shared_task_zero_forbids_insert_turn_poll_batch(self) -> None: + capture = _v5_shared_register_atomic_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + # task 0 的 Register.start->metadata.start 仍保留为闭合前段，但 + # task 0 没有前驱，不能伪造 SharedInsertTurnPoll。 + rows.append( + [ + 0, + 0, + 0, + -1, + -1, + "Atomic", + 120, + 124, + (1 << 8) | 0xD0, + 19, + ] + ) + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex( + ValueError, + "none for task 0.*expected=0", + ): + convert(input_path, output_path) + self.assertFalse(output_path.exists()) + + def test_v4_shared_register_atomic_schema_is_fail_closed(self) -> None: + cases = { + "poll_direct": ( + 19, + [0, 0, 0, -1, -1, "Atomic", 120, 124, 0x50, 19], + ), + "poll_wrong_op": ( + 19, + [0, 0, 0, -1, -1, "Atomic", 120, 124, (3 << 8) | 0xD1, 19], + ), + "handoff_wrong_op": ( + 20, + [0, 0, 0, 0, -1, "Atomic", 134, 140, 0x50, 20], + ), + "handoff_without_task": ( + 20, + [0, 0, 0, -1, -1, "Atomic", 134, 140, 0x54, 20], + ), + } + for label, (site_id, replacement) in cases.items(): + with self.subTest(label=label), tempfile.TemporaryDirectory() as directory: + capture = _v5_shared_register_atomic_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + row_index = next( + index + for index, row in enumerate(rows) + if row[5] == "Atomic" and row[9] == site_id + ) + rows[row_index] = replacement + _refresh_summary(capture) + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex( + ValueError, + "SharedInsertTurnPoll must use PollBatch|" + "invalid Atomic PollBatch|invalid direct Atomic", + ): + convert(input_path, output_path) + self.assertFalse(output_path.exists()) + + def test_v4_shared_register_atomic_structure_closes_per_winner( + self, + ) -> None: + cases = ( + ("missing_poll", "SharedInsertTurnPoll PollBatch"), + ("duplicate_poll", "SharedInsertTurnPoll PollBatch"), + ("poll_boundary", "SharedInsertTurnPoll PollBatch"), + ("missing_handoff", "SharedInsertTurnHandoff direct CAS"), + ("duplicate_handoff", "SharedInsertTurnHandoff direct CAS"), + ("handoff_boundary", "identity or boundary"), + ("handoff_task", "SharedInsertTurnHandoff direct CAS"), + ) + for label, expected in cases: + with self.subTest(label=label), tempfile.TemporaryDirectory() as directory: + capture = _v5_shared_register_atomic_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + poll = next( + row for row in rows if row[5] == "Atomic" and row[9] == 19 + ) + handoff = next( + row for row in rows if row[5] == "Atomic" and row[9] == 20 + ) + if label == "missing_poll": + rows.remove(poll) + elif label == "duplicate_poll": + rows.append(list(poll)) + elif label == "poll_boundary": + poll[6] = int(poll[6]) + 1 + elif label == "missing_handoff": + rows.remove(handoff) + elif label == "duplicate_handoff": + rows.append(list(handoff)) + elif label == "handoff_boundary": + handoff[6] = int(handoff[6]) - 1 + elif label == "handoff_task": + handoff[3] = 1 + else: + self.fail(f"unhandled mutation {label}") + _refresh_summary(capture) + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, expected): + convert(input_path, output_path) + self.assertFalse(output_path.exists()) + + def test_v4_shared_register_atomics_keep_cpu_source_issue_boundary( + self, + ) -> None: + capture = _v5_shared_register_atomic_capture(dependency_applied=False) + rows = capture["fdwic_events"] + assert isinstance(rows, list) + for row in rows: + if row[5] == "Atomic" and row[9] in (19, 20): + row[8] = int(row[8]) & ~0x40 + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + convert(input_path, output_path) + names = { + event["name"] + for event in json.loads( + output_path.read_text(encoding="utf-8") + )["traceEvents"] + } + + self.assertIn( + "atomic.poll_batch.source_issue.shared_insert_predecessor_poll.load×3", + names, + ) + self.assertIn( + "atomic.source_issue.shared_insert_completion_publish.compare_exchange#0", + names, + ) + + def test_v4_shared_poll_return_ready_requires_dependency_evidence(self) -> None: + capture = _v5_shared_register_atomic_capture(dependency_applied=False) + # CAS 同样要求 return_ready；先删除它，精确验证 PollBatch 自己的门禁。 + rows = capture["fdwic_events"] + assert isinstance(rows, list) + capture["fdwic_events"] = [ + row for row in rows if not (row[5] == "Atomic" and row[9] == 20) + ] + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex( + ValueError, + "PollBatch return_ready=True.*ClockBaseline", + ): + convert(input_path, output_path) + self.assertFalse(output_path.exists()) + + def test_shared_register_atomic_sites_require_shared_schema_v4(self) -> None: + cases = ( + [0, 0, 0, -1, -1, "Atomic", 100, 110, (3 << 8) | 0x90, 19], + [0, 0, 0, 0, -1, "Atomic", 100, 110, 0x54, 20], + ) + for row in cases: + with self.subTest(site=row[9]), tempfile.TemporaryDirectory() as directory: + capture = _v3_capture([row]) + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex( + ValueError, "requires shared schema-v5" + ): + convert(input_path, output_path) + self.assertFalse(output_path.exists()) + + for site_id, row in ( + (19, [0, 0, 0, -1, -1, "Atomic", 120, 124, (3 << 8) | 0xD0, 19]), + (20, [0, 0, 0, 0, -1, "Atomic", 134, 140, 0x54, 20]), + ): + with self.subTest(private_v4_site=site_id), tempfile.TemporaryDirectory() as directory: + capture = _v5_capture( + [ + [0, 0, 0, 0, -1, "Claim", 110, 115, 0x3, 1], + [0, 0, 0, 0, -1, "AllocComplete", 140, 145, 0, 0], + [0, 0, 0, 0, -1, "Submit", 100, 150, 1, 1], + ], + tensormap_mode="private", + ) + capture["l2_swimlane_level"] = 4 + rows = capture["fdwic_events"] + assert isinstance(rows, list) + rows.extend( + [ + [0, 0, 0, -1, -1, "ClockBaseline", 10, 11, 0, 0], + [0, 0, 0, -1, -1, "ClockBaseline", 12, 13, 0x3, 0], + row, + ] + ) + _refresh_summary(capture) + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex( + ValueError, "requires shared schema-v5" + ): + convert(input_path, output_path) + self.assertFalse(output_path.exists()) + + def test_v4_shared_loser_forbids_insert_turn_atomics(self) -> None: + for site_id, row in ( + (19, [0, 0, 0, -1, -1, "Atomic", 120, 124, (3 << 8) | 0xD0, 19]), + (20, [0, 0, 0, 0, -1, "Atomic", 134, 140, 0x54, 20]), + ): + with self.subTest(site=site_id), tempfile.TemporaryDirectory() as directory: + capture = _v5_capture( + [ + # 唯一 task 明确是 Alloc loser：没有 Register owner。 + [0, 0, 0, 0, -1, "Claim", 110, 115, 0x2, 1], + [0, 0, 0, 0, -1, "Submit", 100, 150, 0, 1], + ], + tensormap_mode="shared", + ) + capture["l2_swimlane_level"] = 4 + rows = capture["fdwic_events"] + assert isinstance(rows, list) + rows.extend( + [ + [0, 0, 0, -1, -1, "ClockBaseline", 10, 11, 0, 0], + [0, 0, 0, -1, -1, "ClockBaseline", 12, 13, 0x3, 0], + row, + ] + ) + _refresh_summary(capture) + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex( + ValueError, "orphan or duplicate SharedInsertTurn" + ): + convert(input_path, output_path) + self.assertFalse(output_path.exists()) + + def test_v1_claim_attempt_uses_contained_atomic_evidence(self) -> None: + # 历史 raw 没有 attempted bit。只在同一 capture 真有 claim_max 记录时 + # 恢复该语义，不根据 task_id 或 core role 猜测。 + capture = { + "l2_swimlane_level": 1, + "metadata": { + "clock_freq_hz": 1_000_000_000, + "num_cores": 1, + "core_types": ["AIC"], + }, + "fdwic_events": [ + [0, 0, 0, 1, -1, "Claim", 100, 200, 0, 0], + [0, 0, 0, 1, -1, "Atomic", 120, 160, 0x50, 4], + [0, 0, 0, 2, -1, "Claim", 210, 230, 0, 0], + ], + } + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + convert(input_path, output_path) + events = json.loads(output_path.read_text(encoding="utf-8"))["traceEvents"] + + attempted = next(event for event in events if event.get("name") == "claim.lost#1") + unknown = next(event for event in events if event.get("name") == "claim#2") + self.assertTrue(attempted["args"]["claim_attempted"]) + self.assertIsNone(unknown["args"]["claim_attempted"]) + self.assertEqual(attempted["args"]["claim_attempted_source"], "contained_claim_max") + self.assertEqual( + unknown["args"]["claim_attempted_source"], + "unknown_v1_without_matching_claim_max", + ) + + def test_v2_claim_states_do_not_require_atomic_records(self) -> None: + # v2 raw 直接携带 attempted/won，因此关闭 --trace-atomics 后仍能 + # 区分三种 Claim 状态，不依赖 converter 从业务拓扑推断。 + capture = { + "l2_swimlane_level": 1, + "metadata": { + "clock_freq_hz": 1_000_000_000, + "num_cores": 1, + "trace_schema_version": 2, + "core_types": ["AIC"], + }, + "fdwic_events": [ + [0, 0, 0, 1, -1, "Claim", 100, 110, 0x0, 0], + [0, 0, 0, 2, -1, "Claim", 120, 140, 0x2, 0], + [0, 0, 0, 3, 0, "Claim", 150, 180, 0x3, 0], + ], + } + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + convert(input_path, output_path) + events = json.loads(output_path.read_text(encoding="utf-8"))["traceEvents"] + + names = {event.get("name") for event in events} + self.assertIn("claim.not_attempted#1", names) + self.assertIn("claim.lost#2", names) + self.assertIn("claim.won#3", names) + + def test_v2_rejects_winner_without_attempt(self) -> None: + capture = { + "l2_swimlane_level": 1, + "metadata": { + "clock_freq_hz": 1_000_000_000, + "num_cores": 1, + "trace_schema_version": 2, + "core_types": ["AIC"], + }, + "fdwic_events": [[0, 0, 0, 1, 0, "Claim", 100, 110, 0x1, 0]], + } + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "invalid Claim flags"): + convert(input_path, output_path) + + def test_v3_poll_batches_preserve_exact_calls_on_scalar_lane(self) -> None: + # standalone 只允许六类显式等待区 observation load 聚合；每个 + # PollBatch 都必须保留精确 call_count，但不能伪装成单次延迟。 + sites = { + 1: "startup_poll", + 2: "fatal_poll", + 5: "fanin_flag_load", + 11: "heap_frontier_load", + 12: "heap_vend_load", + 14: "replay_done_poll", + } + call_count = 12_345 + flags = (call_count << 8) | 0x90 + for site_id, site_name in sites.items(): + with self.subTest(site=site_name), tempfile.TemporaryDirectory() as directory: + capture = _v3_capture( + [[0, 0, 0, -1, -1, "Atomic", 100, 900, flags, site_id]] + ) + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + emitted, blocks, base_cycle = convert(input_path, output_path) + merged = json.loads(output_path.read_text(encoding="utf-8")) + + self.assertEqual((emitted, blocks, base_cycle), (3, 1, 10)) + batch = next( + event + for event in merged["traceEvents"] + if event.get("cat") == "atomic.poll_batch" + ) + self.assertEqual( + batch["name"], f"atomic.poll_batch.{site_name}.load×{call_count}" + ) + self.assertEqual((batch["pid"], batch["tid"]), (0, 0)) + self.assertEqual(batch["args"]["call_count"], call_count) + self.assertEqual(batch["args"]["poll_window_cycles"], 800) + self.assertEqual( + batch["args"]["duration_semantics"], + "logical_poll_episode_envelope_not_single_atomic_latency", + ) + self.assertEqual( + batch["args"]["batch_semantics"], "observation_load_calls" + ) + self.assertTrue( + batch["args"]["may_contain_interleaved_direct_atomics"] + ) + self.assertNotIn("cycles", batch["args"]) + self.assertNotIn("completion_boundary", batch["args"]) + self.assertNotIn("return_ready_observed", batch["args"]) + self.assertEqual( + merged["metadata"]["fdwic_summary"]["atomic_calls"], + call_count, + ) + + def test_v3_poll_batch_accepts_maximum_24_bit_count(self) -> None: + call_count = 0xFFFFFF + capture = _v3_capture( + [[ + 0, + 0, + 0, + -1, + -1, + "Atomic", + 100, + 900, + (call_count << 8) | 0x90, + 1, + ]] + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + convert(input_path, output_path) + events = json.loads(output_path.read_text(encoding="utf-8"))["traceEvents"] + + batch = next(event for event in events if event.get("cat") == "atomic.poll_batch") + self.assertEqual(batch["args"]["call_count"], call_count) + + def test_v3_rejects_invalid_poll_batch_schema(self) -> None: + valid = (7 << 8) | 0x90 + cases = ( + (0x90, 5, -1, -1), # call_count=0 + ((7 << 8) | 0x91, 5, -1, -1), # observation 只能是 Load + (valid, 9, -1, -1), # frontier scan 不是显式等待区 + ((7 << 8) | 0xB0, 5, -1, -1), # batch 没有 value_zero + ((7 << 8) | 0xD0, 5, -1, -1), # batch 没有 return-ready + (valid, 5, 0, -1), # batch 不归属单个 task + (valid, 5, -1, 0), # batch 不归属 kernel function + ) + for flags, site, task_id, func_id in cases: + with self.subTest(flags=flags, site=site), tempfile.TemporaryDirectory() as directory: + capture = _v3_capture( + [[ + 0, + 0, + 0, + task_id, + func_id, + "Atomic", + 100, + 110, + flags, + site, + ]] + ) + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "invalid Atomic PollBatch"): + convert(input_path, output_path) + self.assertFalse(output_path.exists()) + + def test_v2_reserves_poll_batch_flag(self) -> None: + capture = { + "l2_swimlane_level": 4, + "metadata": { + "clock_freq_hz": 1_000_000_000, + "num_cores": 1, + "trace_schema_version": 2, + "core_types": ["aic"], + }, + "fdwic_events": [ + [0, 0, 0, -1, -1, "Atomic", 100, 110, (7 << 8) | 0x90, 5] + ], + } + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "invalid Atomic PollBatch"): + convert(input_path, output_path) + + def test_v3_rejects_invalid_direct_atomic_schema(self) -> None: + cases = ( + (0x51, 4, -1), # ClaimMax 的 op 必须是 FetchMax + (0x12, 0, -1), # StartupIncrement 不消费返回值 + (0x42, 0, -1), # 未消费返回值不能声明 return-ready + (0x73, 4, -1), # value_zero 只属于 Load + ((1 << 8) | 0x50, 1, -1), # retry payload 只属于 FetchMax + (0x50, 21, -1), # 当前 AtomicSite::Count 以外的未定义站点 + (0x53, 4, 0), # Atomic 不携带 function id + ) + for flags, site, func_id in cases: + with self.subTest(flags=flags, site=site), tempfile.TemporaryDirectory() as directory: + capture = _v3_capture( + [[0, 0, 0, 7, func_id, "Atomic", 100, 110, flags, site]] + ) + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "invalid direct Atomic"): + convert(input_path, output_path) + + def test_v3_exports_shared_heap_return_ready_sites(self) -> None: + rows = [ + [0, 0, 0, 7, -1, "Atomic", 100, 110, 0x50, 15], + [0, 0, 0, 7, -1, "Atomic", 111, 121, 0x50, 16], + [0, 0, 0, 7, -1, "Atomic", 122, 132, 0x52, 17], + [0, 0, 0, 7, -1, "Atomic", 133, 143, 0x52, 18], + ] + capture = _v3_capture(rows, dependency_applied=True) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + convert(input_path, output_path) + events = json.loads(output_path.read_text(encoding="utf-8"))[ + "traceEvents" + ] + + names = { + event["name"] + for event in events + if event.get("cat") == "atomic.return_ready" + } + self.assertEqual( + names, + { + "atomic.return_ready.shared_heap_vend_load.load#7", + "atomic.return_ready.shared_heap_cursor_load.load#7", + "atomic.return_ready.shared_heap_cursor_reserve.fetch_add#7", + "atomic.return_ready.shared_heap_vend_advance.fetch_add#7", + }, + ) + + def test_v3_direct_boundary_must_match_core_clock_baseline(self) -> None: + # baseline 声明该后端应用依赖钩子，消费返回值的直接 Atomic 却没有 + # return-ready bit；converter 必须拒绝这种自相矛盾的 raw。 + capture = _v3_capture( + [[0, 0, 0, 7, -1, "Atomic", 100, 110, 0x13, 4]], + dependency_applied=True, + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "does not match.*ClockBaseline"): + convert(input_path, output_path) + + def test_v3_source_issue_direct_boundary_matches_cpu_baseline(self) -> None: + # CPU/A5Sim 的依赖基线明确声明 dependency_applied=0；消费返回值的 + # direct span 因而保留 source-issue，不能被 converter 擅自升级。 + capture = _v3_capture( + [[0, 0, 0, 7, -1, "Atomic", 100, 110, 0x13, 4]], + dependency_applied=False, + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + convert(input_path, output_path) + events = json.loads(output_path.read_text(encoding="utf-8"))["traceEvents"] + + direct = next( + event for event in events if event.get("cat") == "atomic.source_issue" + ) + self.assertEqual(direct["args"]["call_count"], 1) + self.assertTrue(direct["args"]["result_used"]) + self.assertFalse(direct["args"]["return_ready_observed"]) + + def test_v3_rejects_invalid_clock_baseline_schema(self) -> None: + cases = ( + (0x2, -1, -1, 0), # applied 不能脱离 dependency bit + (0x4, -1, -1, 0), # 未定义 flag + (0x0, 0, -1, 0), # baseline 不归属 task + (0x0, -1, 0, 0), # baseline 不归属 function + (0x0, -1, -1, 1), # aux 必须为零 + ) + for flags, task_id, func_id, auxiliary in cases: + with self.subTest(flags=flags), tempfile.TemporaryDirectory() as directory: + capture = _v3_capture([], add_clock_baselines=False) + capture["fdwic_events"] = [ + [ + 0, + 0, + 0, + task_id, + func_id, + "ClockBaseline", + 10, + 11, + flags, + auxiliary, + ] + ] + summary = capture["metadata"]["fdwic_summary"] + summary["records"] = 1 + summary["clock_baseline_records"] = 1 + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "invalid ClockBaseline"): + convert(input_path, output_path) + + def test_v3_requires_two_clock_baselines_per_core(self) -> None: + capture = _v3_capture([], add_clock_baselines=False) + capture["fdwic_events"] = [ + [0, 0, 0, -1, -1, "ClockBaseline", 10, 11, 0, 0] + ] + summary = capture["metadata"]["fdwic_summary"] + summary["records"] = 1 + summary["clock_baseline_records"] = 1 + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "requires exactly one plain"): + convert(input_path, output_path) + + def test_v3_rejects_each_broken_weighted_summary_field(self) -> None: + rows = [ + [0, 0, 0, -1, -1, "Atomic", 100, 200, (17 << 8) | 0x90, 1], + [0, 0, 0, 4, -1, "Atomic", 210, 220, 0x53, 4], + ] + keys = ( + "records", + "atomic_records", + "clock_baseline_records", + "atomic_calls", + "batched_poll_calls", + "poll_batch_records", + "dropped_records", + ) + for key in keys: + with self.subTest(key=key), tempfile.TemporaryDirectory() as directory: + capture = _v3_capture(rows) + capture["metadata"]["fdwic_summary"][key] += 1 + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, rf"fdwic_summary\.{key}"): + convert(input_path, output_path) + + def test_v3_weighted_summary_closes_mixed_direct_and_batches(self) -> None: + rows = [ + [0, 0, 0, -1, -1, "Atomic", 100, 200, (17 << 8) | 0x90, 1], + [0, 0, 0, -1, -1, "Atomic", 201, 250, (9 << 8) | 0x90, 14], + [0, 0, 0, 4, -1, "Atomic", 251, 260, 0x53, 4], + ] + capture = _v3_capture(rows) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + emitted, _, _ = convert(input_path, output_path) + merged = json.loads(output_path.read_text(encoding="utf-8")) + + summary = merged["metadata"]["fdwic_summary"] + self.assertEqual(summary["records"], 5) + self.assertEqual(summary["atomic_records"], 3) + self.assertEqual(summary["clock_baseline_records"], 2) + self.assertEqual(summary["atomic_calls"], 27) + self.assertEqual(summary["batched_poll_calls"], 26) + self.assertEqual(summary["poll_batch_records"], 2) + self.assertEqual(summary["dropped_records"], 0) + self.assertEqual(emitted, 5) + atomic_events = [ + event + for event in merged["traceEvents"] + if str(event.get("cat", "")).startswith("atomic.") + ] + self.assertEqual(len(atomic_events), summary["atomic_records"]) + + def test_v3_requires_level4_and_producer_summary(self) -> None: + capture = _v3_capture( + [[0, 0, 0, -1, -1, "Atomic", 100, 110, (3 << 8) | 0x90, 14]] + ) + with tempfile.TemporaryDirectory() as directory: + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + + capture["l2_swimlane_level"] = 1 + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "requires l2_swimlane_level=4"): + convert(input_path, output_path) + + capture["l2_swimlane_level"] = 4 + del capture["metadata"]["fdwic_summary"] + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "fdwic_summary is required"): + convert(input_path, output_path) + + def test_v3_rejects_non_standalone_topology(self) -> None: + cases = { + "core_type": lambda capture: capture["metadata"]["core_types"].__setitem__( + 0, "aiv" + ), + "block": lambda capture: capture["fdwic_events"][0].__setitem__(1, 1), + "lane": lambda capture: capture["fdwic_events"][0].__setitem__(2, 1), + } + for name, mutate in cases.items(): + with self.subTest(field=name), tempfile.TemporaryDirectory() as directory: + capture = _v3_capture([]) + mutate(capture) + input_path = Path(directory) / "raw.json" + output_path = Path(directory) / "merged.json" + input_path.write_text(json.dumps(capture), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "does not match standalone topology"): + convert(input_path, output_path) + self.assertFalse(output_path.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/atomic_probe/pa_scheduler/test_swimlane_exclusive_analyzer.py b/tests/atomic_probe/pa_scheduler/test_swimlane_exclusive_analyzer.py new file mode 100644 index 0000000000..7bcff4c3a3 --- /dev/null +++ b/tests/atomic_probe/pa_scheduler/test_swimlane_exclusive_analyzer.py @@ -0,0 +1,1648 @@ +#!/usr/bin/env python3 +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- + +"""standalone PA Submit 排他分析器的闭合、门禁与原子发布回归。""" + +from __future__ import annotations + +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path + +try: + from .swimlane_exclusive_analyzer import ( + OVERLAY_PHASES, + SUBMIT_PARTITION_METRICS, + analyze_capture, + main, + write_analysis, + ) +except ImportError: + from swimlane_exclusive_analyzer import ( + OVERLAY_PHASES, + SUBMIT_PARTITION_METRICS, + analyze_capture, + main, + write_analysis, + ) + + +CORE_COUNT = 96 + + +def _topology(core_id: int) -> tuple[int, int, str]: + if core_id < 32: + return core_id, 0, "aic" + vector_id = core_id - 32 + return vector_id // 2, 1 + vector_id % 2, "aiv" + + +def _row( + core_id: int, + task_id: int, + phase: str, + start: int, + end: int, + *, + function_id: int = -1, + flags: int = 0, + auxiliary: int = 0, +) -> list[object]: + block_id, lane, _role = _topology(core_id) + return [ + core_id, + block_id, + lane, + task_id, + function_id, + phase, + start, + end, + flags, + auxiliary, + ] + + +def _refresh_summary(capture: dict[str, object]) -> None: + """修改 fixture 后同步 producer summary，使失败精确落在待测门禁。""" + + rows = capture["fdwic_events"] + assert isinstance(rows, list) + atomic_rows = [row for row in rows if row[5] == "Atomic"] + poll_rows = [row for row in atomic_rows if int(row[8]) & 0x80] + poll_calls = sum((int(row[8]) >> 8) & 0xFFFFFF for row in poll_rows) + metadata = capture["metadata"] + assert isinstance(metadata, dict) + old_summary = metadata.get("fdwic_summary") + dropped = int(old_summary.get("dropped_records", 0)) if isinstance(old_summary, dict) else 0 + metadata["fdwic_summary"] = { + "records": len(rows), + "atomic_records": len(atomic_rows), + "clock_baseline_records": sum(row[5] == "ClockBaseline" for row in rows), + "atomic_calls": len(atomic_rows) - len(poll_rows) + poll_calls, + "batched_poll_calls": poll_calls, + "poll_batch_records": len(poll_rows), + "dropped_records": dropped, + } + + +def _capture() -> dict[str, object]: + """构造完整 32 AIC + 64 AIV、每核两个 Submit 的 schema-v3 证据。""" + + rows: list[list[object]] = [] + for core_id in range(CORE_COUNT): + # schema-v3 每核必须有 plain/dependency 各一条。dependency_applied=1， + # 所有消费返回值的 Atomic 因而使用 return_ready 边界。 + clock_start = 10 + core_id * 4 + rows.append(_row(core_id, -1, "ClockBaseline", clock_start, clock_start + 1)) + rows.append( + _row( + core_id, + -1, + "ClockBaseline", + clock_start + 2, + clock_start + 3, + flags=0x3, + ) + ) + + base = 1000 + core_id + for task_id, submit_offset, submit_duration in ((0, 0, 100), (1, 120, 80)): + start = base + submit_offset + if task_id == 0: + # child 总计 53 cycle，SubmitResidual=47；EfDrain 内 Kernel union=10。 + efdrain = (start + 1, start + 21) + kernel = (start + 4, start + 14) + materialize = (start + 25, start + 35) + prepare = (start + 36, start + 41) + claim = (start + 43, start + 51) + fanin = (start + 52, start + 55) + register = (start + 56, start + 63) + else: + # child 总计 36 cycle，SubmitResidual=44；EfDrain 内 Kernel union=4。 + efdrain = (start + 1, start + 11) + kernel = (start + 3, start + 7) + materialize = (start + 14, start + 22) + prepare = (start + 23, start + 27) + claim = (start + 29, start + 35) + fanin = (start + 36, start + 38) + register = (start + 40, start + 46) + + rows.extend( + [ + _row(core_id, task_id, "EfDrain", *efdrain), + _row(core_id, task_id, "Kernel", *kernel, function_id=task_id), + _row(core_id, task_id, "Materialize", *materialize), + _row(core_id, task_id, "PrepareMap", *prepare), + _row( + core_id, + task_id, + "Claim", + *claim, + flags=0x2, + auxiliary=1 if task_id == 0 else 0, + ), + _row(core_id, task_id, "Fanin", *fanin, function_id=task_id), + _row(core_id, task_id, "Register", *register), + # fetch_max + result_used + return_ready，site=ClaimMax(4)。 + _row( + core_id, + task_id, + "Atomic", + claim[0] + 1, + claim[0] + 3, + flags=0x53, + auxiliary=4, + ), + _row(core_id, task_id, "Commit", start + 70, start + 70), + # 三种 lap marker 故意覆盖显式 child，用于证明它们仅为 Overlay。 + _row(core_id, task_id, "Build", start + 25, start + 65), + _row(core_id, task_id, "Replay", start + 14, start + 60), + _row(core_id, task_id, "Alloc", start + 25, start + 70), + _row(core_id, task_id, "Submit", start, start + submit_duration), + ] + ) + + capture: dict[str, object] = { + "l2_swimlane_level": 4, + "metadata": { + "clock_freq_hz": 1_000_000_000, + "num_cores": CORE_COUNT, + "trace_schema_version": 3, + "core_types": [_topology(core_id)[2] for core_id in range(CORE_COUNT)], + }, + "fdwic_events": rows, + } + _refresh_summary(capture) + return capture + + +def _append_v4_g1_tail_tasks( + rows: list[list[object]], tensormap_mode: str +) -> None: + """给历史 Alloc/QK fixture 补齐 SF/PV/UP 三个 loser。""" + + for core_id in range(CORE_COUNT): + base = 1000 + core_id + for task_id in range(2, 5): + start = base + 220 + (task_id - 2) * 100 + function_id = task_id - 1 + if tensormap_mode == "private": + rows.append( + _row( + core_id, + task_id, + "EfDrain", + start + 1, + start + 11, + ) + ) + rows.extend( + [ + _row( + core_id, + task_id, + "Kernel", + start + 3, + start + 7, + function_id=function_id, + ), + _row( + core_id, + task_id, + "Claim", + start + 14, + start + 20, + flags=0x2, + ), + ] + ) + if tensormap_mode == "private": + rows.extend( + [ + _row(core_id, task_id, "Materialize", start + 24, start + 32), + _row(core_id, task_id, "PrepareMap", start + 33, start + 37), + _row(core_id, task_id, "Register", start + 43, start + 49), + ] + ) + rows.extend( + [ + _row( + core_id, + task_id, + "Atomic", + start + 15, + start + 17, + flags=0x53, + auxiliary=4, + ), + _row(core_id, task_id, "Submit", start, start + 80), + ] + ) + + +def _skip_v4_source_phase( + tensormap_mode: str, + phase: str, + winner: bool, + task_id: int, +) -> bool: + """过滤 v3 fixture 中不属于目标 v4 Submit 路径的历史 child/overlay。""" + + return ( + phase in {"Build", "Replay", "Alloc"} + # 新 shared raw 不再写 EfDrain；分析器必须由同一 Submit.start 和 + # Claim.start 离线恢复。private fixture 继续保留显式记录。 + or (tensormap_mode == "shared" and phase == "EfDrain") + or (tensormap_mode == "shared" and phase == "PrepareMap") + or ( + tensormap_mode == "shared" + and not winner + and phase in {"Materialize", "Register"} + ) + or (phase == "Fanin" and (not winner or task_id == 0)) + ) + + +def _v4_capture(*, tensormap_mode: str = "shared") -> dict[str, object]: + """构造 shared 稀疏或 private 矩形 v4 Submit，并补齐父 span。""" + + capture = _capture() + capture["l2_swimlane_level"] = 4 + metadata = capture["metadata"] + assert isinstance(metadata, dict) + metadata["trace_schema_version"] = 5 + metadata["tensormap_mode"] = tensormap_mode + source_rows = capture["fdwic_events"] + assert isinstance(source_rows, list) + rows: list[list[object]] = [] + for original in source_rows: + row = list(original) + core_id = int(row[0]) + task_id = int(row[3]) + phase = str(row[5]) + winner = (core_id, task_id) in {(0, 0), (1, 1)} + if _skip_v4_source_phase(tensormap_mode, phase, winner, task_id): + continue + base = 1000 + core_id + submit_start = base + (0 if task_id == 0 else 120) + if phase == "Claim": + # schema-v5 跟随 compete-first eager 生产路径：Claim 先于 + # callback 构参与 Materialize。两类 task 都保留 v3 fixture + # 的 Claim 时长，只调整边界顺序。 + row[6:8] = ( + [submit_start + 25, submit_start + 33] + if task_id == 0 + else [submit_start + 14, submit_start + 20] + ) + row[8] = 0x3 if winner else 0x2 + elif phase == "Materialize": + row[6:8] = ( + [submit_start + 36, submit_start + 46] + if task_id == 0 + else [submit_start + 24, submit_start + 32] + ) + elif phase == "PrepareMap": + row[6:8] = ( + [submit_start + 47, submit_start + 52] + if task_id == 0 + else [submit_start + 33, submit_start + 37] + ) + elif phase == "Fanin": + row[6:8] = [submit_start + 40, submit_start + 42] + elif phase == "Register": + row[6:8] = ( + [submit_start + 56, submit_start + 63] + if task_id == 0 + else [submit_start + 43, submit_start + 49] + ) + elif phase == "Atomic": + row[6:8] = ( + [submit_start + 26, submit_start + 28] + if task_id == 0 + else [submit_start + 15, submit_start + 17] + ) + elif phase == "Submit": + row[8] = 1 if winner else 0 + row[9] = 1 if task_id == 0 else 0 + rows.append(row) + + if phase != "Submit": + continue + submit_start = int(row[6]) + if winner and task_id == 0: + rows.append( + _row( + core_id, + task_id, + "AllocComplete", + submit_start + 65, + submit_start + 75, + ) + ) + elif winner: + rows.append( + _row( + core_id, + task_id, + "WinnerBuild", + submit_start + 50, + submit_start + 60, + function_id=task_id % 5 - 1, + ) + ) + + if tensormap_mode == "shared": + # Register 父区间固定带 metadata 父 detail 和 task-output 子 detail。 + # 等待前驱、writer metadata、metadata 收尾和完成发布均由端点补集恢复， + # 不为这些区域继续扩张 raw。 + register_rows = [row for row in rows if row[5] == "Register"] + for register in register_rows: + start = int(register[6]) + end = int(register[7]) + publish_start = start + 1 + publish_end = end - 1 + outputs_start = start + 3 + outputs_end = end - 2 + rows.append( + _row( + int(register[0]), + int(register[3]), + "SharedRegisterPublishMetadata", + publish_start, + publish_end, + function_id=int(register[4]), + ) + ) + rows.append( + _row( + int(register[0]), + int(register[3]), + "SharedRegisterPublishTaskOutputs", + outputs_start, + outputs_end, + function_id=int(register[4]), + ) + ) + copy_end = outputs_start + max(1, (outputs_end - outputs_start) // 2) + if copy_end > outputs_end: + copy_end = outputs_end + rows.append( + _row( + int(register[0]), + int(register[3]), + "SharedRegisterPublishTaskOutputsCopy", + outputs_start, + copy_end, + function_id=int(register[4]), + ) + ) + rows.append( + _row( + int(register[0]), + int(register[3]), + "SharedRegisterPublishTaskOutputsFlush", + copy_end, + outputs_end, + function_id=int(register[4]), + ) + ) + # per-task predecessor chain 中 task 0 没有前驱，因此只有 + # task>0 的 winner 才产生 insert-turn PollBatch。 + if int(register[3]) > 0: + rows.append( + _row( + int(register[0]), + -1, + "Atomic", + start, + publish_start, + flags=(3 << 8) | 0xD0, + auxiliary=19, + ) + ) + # 每个 winner（包括 task 0）都用 completion CAS 发布本 task + # 的 metadata 已完成，使 Register 后段和父区间保持闭合。 + rows.append( + _row( + int(register[0]), + int(register[3]), + "Atomic", + publish_end, + end, + flags=0x54, + auxiliary=20, + ) + ) + + # schema-v5 动态门槛必须使用完整 G1：在历史两 task fixture 后补齐 + # SF/PV/UP 三个 loser。这样测试不会再依赖“截断到 QK 的非法 batch”。 + _append_v4_g1_tail_tasks(rows, tensormap_mode) + + for core_id in range(CORE_COUNT): + base = 1000 + core_id + rows.extend( + [ + _row(core_id, -1, "OrchestrationReplay", base - 10, base + 510), + _row(core_id, -1, "FinalDrain", base + 510, base + 550), + _row(core_id, 4, "Kernel", base + 520, base + 530, function_id=3), + ] + ) + capture["fdwic_events"] = rows + _refresh_summary(capture) + return capture + + +def _v5_materialize_output_capture() -> dict[str, object]: + """把 legacy v5 fixture 迁移成 output publication 位于 Materialize。""" + + capture = _v4_capture() + source_rows = capture["fdwic_events"] + assert isinstance(source_rows, list) + materializes = { + (int(row[0]), int(row[3])): row + for row in source_rows + if row[5] == "Materialize" + } + metadata_tasks = { + (int(row[0]), int(row[3])) + for row in source_rows + if row[5] == "SharedRegisterPublishMetadata" + } + rows = [ + row + for row in source_rows + if row[5] + not in { + "SharedRegisterPublishTaskOutputs", + "SharedRegisterPublishTaskOutputsCopy", + "SharedRegisterPublishTaskOutputsFlush", + } + ] + for task_key in sorted(metadata_tasks): + materialize = materializes[task_key] + output_start = int(materialize[7]) - 4 + output_end = int(materialize[7]) - 1 + core_id, task_id, function_id = ( + int(materialize[0]), + int(materialize[3]), + int(materialize[4]), + ) + rows.extend( + [ + _row( + core_id, + task_id, + "SharedMaterializePublishTaskOutputs", + output_start, + output_end, + function_id=function_id, + ), + _row( + core_id, + task_id, + "SharedMaterializePublishTaskOutputsCopy", + output_start, + output_start + 1, + function_id=function_id, + ), + _row( + core_id, + task_id, + "SharedMaterializePublishTaskOutputsFlush", + output_start + 1, + output_end - 1, + function_id=function_id, + ), + ] + ) + capture["fdwic_events"] = rows + _refresh_summary(capture) + return capture + + +class SwimlaneExclusiveAnalyzerTest(unittest.TestCase): + def _write(self, directory: str, capture: dict[str, object]) -> Path: + path = Path(directory) / "l2_swimlane_records.json" + path.write_text(json.dumps(capture, ensure_ascii=False), encoding="utf-8") + return path + + def test_valid_capture_closes_all_integer_partitions(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, _capture()) + report = analyze_capture(path) + + self.assertEqual(report["validation"]["status"], "PASS") + self.assertEqual(report["capture"]["core_count"], 96) + self.assertEqual(report["capture"]["task_count_per_core"], 2) + self.assertEqual(report["global_submit_makespan"]["duration_cycles"], 295) + + metrics = report["aggregate_core_work"]["metrics_cycles"] + self.assertEqual(metrics["submit_envelope"], 19_200) + self.assertEqual(metrics["submit_union"], 17_280) + self.assertEqual(metrics["between_submit_residual"], 1_920) + self.assertEqual(metrics["efdrain"], 2_880) + self.assertEqual(metrics["materialize"], 1_728) + self.assertEqual(metrics["prepare_map"], 864) + self.assertEqual(metrics["claim"], 1_344) + self.assertEqual(metrics["fanin"], 480) + self.assertEqual(metrics["register"], 1_248) + self.assertEqual(metrics["submit_residual"], 8_736) + self.assertEqual(metrics["efdrain_kernel_union"], 1_344) + self.assertEqual(metrics["efdrain_control"], 1_536) + self.assertEqual( + sum(metrics[name] for name in SUBMIT_PARTITION_METRICS), + metrics["submit_union"], + ) + self.assertEqual( + metrics["submit_union"] + metrics["between_submit_residual"], + metrics["submit_envelope"], + ) + self.assertEqual( + metrics["efdrain_kernel_union"] + metrics["efdrain_control"], + metrics["efdrain"], + ) + residual = report["residual_breakdown"] + self.assertEqual(residual["submit_internal_residual"]["total_cycles"], 1_920) + self.assertEqual(residual["submit_tail_residual"]["total_cycles"], 6_816) + self.assertEqual(residual["between_submit_residual"]["total_cycles"], 1_920) + self.assertEqual( + residual["submit_internal_residual"]["total_cycles"] + + residual["submit_tail_residual"]["total_cycles"], + metrics["submit_residual"], + ) + for section in ( + "submit_internal_residual", + "submit_tail_residual", + "between_submit_residual", + ): + segments = residual[section]["segments"] + self.assertEqual( + sum(segment["cycles"] for segment in segments), + residual[section]["total_cycles"], + ) + for segment in segments: + self.assertEqual( + segment["aic_cycles"] + segment["aiv_cycles"], + segment["cycles"], + ) + self.assertEqual(len(report["per_core"]), 96) + self.assertIsNone(report["kernel_containment"]["orphan_events"]) + self.assertEqual( + report["kernel_containment"]["unclassified_without_v5_parent_events"], + 0, + ) + + def test_v4_closes_true_tails_and_worker_parent_hierarchy(self) -> None: + capture = _v4_capture() + raw_rows = capture["fdwic_events"] + assert isinstance(raw_rows, list) + raw_event_count = len(raw_rows) + self.assertFalse(any(row[5] == "EfDrain" for row in raw_rows)) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + report = analyze_capture(path) + + self.assertEqual(report["schema_version"], 3) + self.assertEqual(report["capture"]["trace_schema_version"], 5) + self.assertEqual(report["capture"]["tensormap_mode"], "shared") + self.assertEqual( + report["capture"]["event_count"], raw_event_count + 96 * 5 + ) + self.assertEqual( + report["semantics"]["exclusive_submit_children"][-2:], + ["WinnerBuild", "AllocComplete"], + ) + self.assertNotIn( + "PrepareMap", + report["semantics"]["exclusive_submit_children"], + ) + validation = report["validation"] + self.assertIs(validation["parent_boundaries_adjacent"], True) + self.assertEqual(validation["legacy_lap_records"], 0) + self.assertIs(validation["worker_completion_partition_exact"], True) + + metrics = report["aggregate_core_work"]["metrics_cycles"] + self.assertEqual(metrics["submit_union"], 40_320) + self.assertEqual(metrics["fanin"], 2) + self.assertEqual(metrics["winner_build"], 10) + self.assertEqual(metrics["alloc_complete"], 10) + self.assertEqual(metrics["materialize"], 18) + self.assertEqual(metrics["prepare_map"], 0) + self.assertEqual(metrics["register"], 13) + self.assertEqual(metrics["efdrain"], 7_776) + self.assertEqual(metrics["submit_residual"], 29_419) + self.assertEqual(metrics["orchestration_setup"], 960) + self.assertEqual(metrics["orchestration_tail"], 960) + self.assertEqual(metrics["orchestration_replay"], 49_920) + self.assertEqual(metrics["final_drain"], 3_840) + self.assertEqual(metrics["final_drain_kernel_union"], 960) + self.assertEqual(metrics["final_drain_residual"], 2_880) + self.assertEqual(metrics["worker_completion"], 53_760) + residual = report["residual_breakdown"] + self.assertEqual(residual["submit_internal_residual"]["total_cycles"], 29) + self.assertEqual(residual["submit_tail_residual"]["total_cycles"], 29_390) + self.assertEqual(residual["between_submit_residual"]["total_cycles"], 7_680) + self.assertAlmostEqual( + residual["submit_internal_residual"]["share_of_submit_union"], + 29 / 40_320, + ) + self.assertAlmostEqual( + residual["submit_tail_residual"]["share_of_submit_union"], + 29_390 / 40_320, + ) + self.assertAlmostEqual( + residual["between_submit_residual"]["share_of_submit_envelope"], + 7_680 / 48_000, + ) + tail_boundaries = { + segment["boundary"] + for segment in residual["submit_tail_residual"]["segments"] + } + self.assertIn("Claim->SubmitEnd", tail_boundaries) + self.assertIn("AllocComplete->SubmitEnd", tail_boundaries) + self.assertIn("WinnerBuild->SubmitEnd", tail_boundaries) + self.assertTrue( + all(boundary.endswith("->SubmitEnd") for boundary in tail_boundaries) + ) + self.assertTrue( + all( + not segment["boundary"].endswith("->SubmitEnd") + for segment in residual["submit_internal_residual"]["segments"] + ) + ) + internal_boundaries = { + segment["boundary"] + for segment in residual["submit_internal_residual"]["segments"] + } + self.assertIn("Claim->Materialize", internal_boundaries) + + closure = report["aggregate_core_work"]["closure"] + for name in ( + "submit_partition", + "submit_envelope", + "efdrain_partition", + "orchestration_replay", + "final_drain", + "worker_completion", + ): + self.assertIs(closure[name]["exact"], True) + self.assertEqual(report["kernel_containment"]["inside_efdrain_events"], 480) + self.assertEqual(report["kernel_containment"]["inside_final_drain_events"], 96) + self.assertEqual(report["kernel_containment"]["orphan_events"], 0) + self.assertEqual( + report["kernel_containment"]["unclassified_without_v5_parent_events"], + 0, + ) + register = report["register_breakdown"] + self.assertIsNotNone(register) + self.assertEqual( + register["event_count"], + { + "metadata": 2, + "task_outputs": 2, + "task_outputs_copy": 2, + "task_outputs_flush": 2, + }, + ) + register_metrics = register["aggregate_core_work"]["metrics_cycles"] + # task_outputs=3 cycles 拆成 copy+flush+residual，且 residual 非负。 + self.assertEqual(register_metrics["parent"], 13) + self.assertEqual(register_metrics["register_wait_predecessor_insert"], 2) + self.assertEqual(register_metrics["register_publish_metadata"], 9) + self.assertEqual(register_metrics["register_publish_writer_metadata"], 4) + self.assertEqual(register_metrics["register_publish_task_outputs"], 3) + self.assertEqual( + register_metrics["register_publish_task_outputs_copy"] + + register_metrics["register_publish_task_outputs_flush"] + + register_metrics["register_publish_task_outputs_residual"], + register_metrics["register_publish_task_outputs"], + ) + self.assertGreaterEqual( + register_metrics["register_publish_task_outputs_residual"], 0 + ) + self.assertEqual(register_metrics["register_publish_metadata_epilogue"], 2) + self.assertEqual(register_metrics["register_publish_insert_completion"], 2) + self.assertIs( + register["aggregate_core_work"]["closure"]["register"]["exact"], + True, + ) + self.assertIs( + register["aggregate_core_work"]["closure"]["metadata"]["exact"], + True, + ) + self.assertEqual(register_metrics["parent"], metrics["register"]) + self.assertNotIn( + "SharedRegisterPublishMetadata", + report["semantics"]["exclusive_submit_children"], + ) + self.assertNotIn( + "SharedRegisterPublishTaskOutputs", + report["semantics"]["exclusive_submit_children"], + ) + self.assertIs( + report["semantics"][ + "register_internal_details_are_exclusive_submit_children" + ], + False, + ) + # detail 只拆 Register，不可作为额外 Submit child 重复相加。 + self.assertEqual( + sum( + metrics[name] + for name in ( + "efdrain", + "materialize", + "claim", + "fanin", + "register", + "winner_build", + "alloc_complete", + "submit_residual", + ) + ), + metrics["submit_union"], + ) + for core in register["per_core"]: + self.assertIs(core["closure"]["register"]["exact"], True) + self.assertIs(core["closure"]["metadata"]["exact"], True) + self.assertEqual( + core["closure"]["register"]["parent_cycles"], + core["closure"]["register"]["flat_children_cycles"], + ) + self.assertEqual( + core["closure"]["metadata"]["parent_cycles"], + core["closure"]["metadata"]["children_cycles"], + ) + self.assertEqual( + set(register["per_role_core_statistics"]), + {"aic", "aiv"}, + ) + atomic_overlay = report["overlays"]["Atomic"] + # 480 条 Claim atomic + task1 的一条 predecessor PollBatch + + # task0/task1 各一条 completion CAS。 + self.assertEqual(atomic_overlay["event_count"], 483) + self.assertEqual(atomic_overlay["aggregate_duration_cycles"], 963) + self.assertIs(atomic_overlay["included_in_additive_totals"], False) + + def test_v5_shared_rejects_any_explicit_efdrain_before_closure( + self, + ) -> None: + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + rows.append(_row(0, 0, "EfDrain", 1000, 1024)) + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex( + ValueError, "must not contain explicit EfDrain" + ): + analyze_capture(path) + + def test_v5_actor_closure_keeps_transition_work_and_removes_kernel_union( + self, + ) -> None: + baseline_capture = _v4_capture() + shifted_capture = _v4_capture() + shifted_rows = shifted_capture["fdwic_events"] + assert isinstance(shifted_rows, list) + shifted_submit = next( + row + for row in shifted_rows + if row[0] == 2 + and row[3] == 0 + and row[5] == "Submit" + ) + self.assertEqual(int(shifted_submit[8]) & 1, 0) + # 把十个 cycle 从 loser Submit 尾部搬到其后的 transition。 + # actor 的下一 Submit 起点不变，因此 gross/control 都不得下降。 + shifted_submit[7] = int(shifted_submit[7]) - 10 + _refresh_summary(shifted_capture) + + with tempfile.TemporaryDirectory() as directory: + baseline_path = self._write( + directory, baseline_capture + ) + baseline = analyze_capture(baseline_path) + shifted_path = Path(directory) / "shifted.json" + shifted_path.write_text( + json.dumps(shifted_capture, ensure_ascii=False), + encoding="utf-8", + ) + shifted = analyze_capture(shifted_path) + + actor = baseline["winner_loser_actor_closure"] + counts = actor["fixed_counts"] + self.assertEqual( + counts, + { + "core_count": 96, + "task_count_per_core": 5, + "expected_actor_count": 480, + "actor_count": 480, + "winner_actor_count": 2, + "loser_actor_count": 478, + "winner_plus_loser_actor_count": 480, + }, + ) + self.assertIn( + "not a new one-winner-per-task protocol oracle", + actor["semantics"]["fixed_counts"], + ) + winner = actor["actors"]["winner"] + loser = actor["actors"]["loser"] + self.assertEqual( + winner["metrics_cycles"]["gross"]["sum_cycles"], 220 + ) + self.assertEqual( + winner["metrics_cycles"]["control"]["sum_cycles"], 206 + ) + self.assertEqual(winner["kernel"]["event_count"], 2) + self.assertEqual( + winner["kernel"]["union_cycles"]["sum_cycles"], 14 + ) + self.assertEqual( + loser["metrics_cycles"]["gross"]["sum_cycles"], 48_740 + ) + self.assertEqual( + loser["metrics_cycles"]["control"]["sum_cycles"], 46_258 + ) + self.assertEqual(loser["kernel"]["event_count"], 478) + self.assertEqual( + loser["kernel"]["union_cycles"]["sum_cycles"], 2_482 + ) + for actor_class in ("winner", "loser"): + for metric in ("gross", "control"): + self.assertEqual( + set( + actor["actors"][actor_class][ + "metrics_cycles" + ][metric] + ), + { + "sum_cycles", + "mean_cycles", + "median_cycles", + "p95_cycles", + }, + ) + for closure in ("gross", "control", "kernel"): + self.assertIs( + actor["actors"][actor_class]["closure"][ + closure + ]["exact"], + True, + ) + self.assertIs( + actor["aggregate_core_work"]["closure"]["gross"]["exact"], + True, + ) + self.assertIs( + actor["aggregate_core_work"]["closure"]["control"][ + "exact" + ], + True, + ) + + shifted_actor = shifted["winner_loser_actor_closure"] + baseline_loser = actor["actors"]["loser"]["metrics_cycles"] + shifted_loser = shifted_actor["actors"]["loser"][ + "metrics_cycles" + ] + self.assertEqual( + shifted_loser["gross"], baseline_loser["gross"] + ) + self.assertEqual( + shifted_loser["control"], baseline_loser["control"] + ) + self.assertEqual( + shifted_loser["kernel_union"], + baseline_loser["kernel_union"], + ) + self.assertEqual( + shifted_loser["submit"]["sum_cycles"], + baseline_loser["submit"]["sum_cycles"] - 10, + ) + self.assertEqual( + shifted_loser["post_claim_tail"]["sum_cycles"], + baseline_loser["post_claim_tail"]["sum_cycles"] - 10, + ) + self.assertEqual( + shifted_loser["post_transition"]["sum_cycles"], + baseline_loser["post_transition"]["sum_cycles"] + 10, + ) + + def test_v5_moves_task_outputs_into_materialize_breakdown(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = self._write( + directory, _v5_materialize_output_capture() + ) + report = analyze_capture(path) + + self.assertEqual( + report["validation"]["task_output_placement"], + "materialize", + ) + materialize = report["materialize_breakdown"] + self.assertIsNotNone(materialize) + materialize_metrics = materialize[ + "aggregate_core_work" + ]["metrics_cycles"] + self.assertEqual( + materialize_metrics, + { + "parent": 18, + "materialize_before_publish_task_outputs": 10, + "materialize_publish_task_outputs": 6, + "materialize_publish_task_outputs_copy": 2, + "materialize_publish_task_outputs_flush": 2, + "materialize_publish_task_outputs_residual": 2, + "materialize_after_publish_task_outputs": 2, + }, + ) + self.assertIs( + materialize["aggregate_core_work"]["closure"][ + "materialize" + ]["exact"], + True, + ) + self.assertIs( + materialize["aggregate_core_work"]["closure"][ + "task_outputs" + ]["exact"], + True, + ) + + register = report["register_breakdown"] + self.assertIsNotNone(register) + register_metrics = register[ + "aggregate_core_work" + ]["metrics_cycles"] + self.assertEqual(register_metrics["parent"], 13) + self.assertEqual( + register_metrics["register_publish_metadata"], 9 + ) + self.assertEqual( + register_metrics["register_publish_writer_metadata"], 9 + ) + for metric in ( + "register_publish_task_outputs", + "register_publish_task_outputs_copy", + "register_publish_task_outputs_flush", + "register_publish_task_outputs_residual", + "register_publish_metadata_epilogue", + ): + self.assertEqual(register_metrics[metric], 0) + self.assertEqual( + report["semantics"]["materialize_internal_output_detail"], + "SharedMaterializePublishTaskOutputs", + ) + self.assertNotIn( + "register_internal_output_detail", report["semantics"] + ) + + def test_v4_shared_register_atomic_overlay_never_changes_exclusive_totals( + self, + ) -> None: + atomic_capture = _v4_capture() + phase_only_capture = _v4_capture() + phase_only_capture["l2_swimlane_level"] = 1 + phase_only_rows = phase_only_capture["fdwic_events"] + assert isinstance(phase_only_rows, list) + phase_only_capture["fdwic_events"] = [ + row + for row in phase_only_rows + if row[5] not in {"Atomic", "ClockBaseline"} + ] + _refresh_summary(phase_only_capture) + + with tempfile.TemporaryDirectory() as directory: + phase_only_path = self._write(directory, phase_only_capture) + phase_only = analyze_capture(phase_only_path) + atomic_path = Path(directory) / "atomic.json" + atomic_path.write_text( + json.dumps(atomic_capture), + encoding="utf-8", + ) + atomic = analyze_capture(atomic_path) + + self.assertEqual( + phase_only["aggregate_core_work"]["metrics_cycles"], + atomic["aggregate_core_work"]["metrics_cycles"], + ) + self.assertEqual( + phase_only["register_breakdown"], + atomic["register_breakdown"], + ) + self.assertEqual(phase_only["overlays"]["Atomic"]["event_count"], 0) + self.assertEqual(atomic["overlays"]["Atomic"]["event_count"], 483) + self.assertIs( + atomic["overlays"]["Atomic"]["included_in_additive_totals"], + False, + ) + + def test_v4_private_keeps_rectangular_frontend_contract(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = self._write( + directory, _v4_capture(tensormap_mode="private") + ) + report = analyze_capture(path) + + self.assertEqual(report["capture"]["tensormap_mode"], "private") + self.assertIn( + "PrepareMap", + report["semantics"]["exclusive_submit_children"], + ) + metrics = report["aggregate_core_work"]["metrics_cycles"] + self.assertEqual(metrics["materialize"], 4_032) + self.assertEqual(metrics["prepare_map"], 2_016) + self.assertEqual(metrics["register"], 2_976) + self.assertEqual(metrics["submit_residual"], 22_442) + self.assertIsNone(report["register_breakdown"]) + + capture = _v4_capture(tensormap_mode="private") + rows = capture["fdwic_events"] + assert isinstance(rows, list) + capture["fdwic_events"] = [ + row + for row in rows + if not (row[0] == 0 and row[3] == 1 and row[5] == "Materialize") + ] + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex( + ValueError, "requires exactly one Materialize" + ): + analyze_capture(path) + + def test_v4_shared_register_detail_is_required_exactly_once(self) -> None: + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + detail = next( + row + for row in rows + if row[0] == 0 + and row[3] == 0 + and row[5] == "SharedRegisterPublishMetadata" + ) + capture["fdwic_events"] = [row for row in rows if row is not detail] + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex( + ValueError, + "requires exactly one SharedRegisterPublishMetadata|" + "missing_register_rows", + ): + analyze_capture(path) + + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + detail = next( + row + for row in rows + if row[0] == 0 + and row[3] == 0 + and row[5] == "SharedRegisterPublishMetadata" + ) + rows.append(list(detail)) + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex( + ValueError, + "duplicate SharedRegisterPublishMetadata|" + "requires exactly one SharedRegisterPublishMetadata", + ): + analyze_capture(path) + + def test_v4_shared_register_detail_must_be_contained_and_match_identity( + self, + ) -> None: + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + detail = next( + row + for row in rows + if row[0] == 0 + and row[3] == 0 + and row[5] == "SharedRegisterPublishMetadata" + ) + detail[6] = int(detail[6]) - 4 + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex( + ValueError, + "crosses Register|outside every Register|" + "must be contained by Register|outside Register parent", + ): + analyze_capture(path) + + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + detail = next( + row + for row in rows + if row[0] == 0 + and row[3] == 0 + and row[5] == "SharedRegisterPublishMetadata" + ) + detail[4] = 0 + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex( + ValueError, + "identity does not match|must match Register identity|" + "identity differs from its parent", + ): + analyze_capture(path) + + def test_v4_private_forbids_shared_register_detail(self) -> None: + capture = _v4_capture(tensormap_mode="private") + rows = capture["fdwic_events"] + assert isinstance(rows, list) + parent = next( + row + for row in rows + if row[0] == 0 and row[3] == 0 and row[5] == "Register" + ) + rows.append( + _row( + 0, + 0, + "SharedRegisterPublishMetadata", + int(parent[6]) + 1, + int(parent[7]) - 1, + function_id=int(parent[4]), + ) + ) + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex( + ValueError, + "only valid for shared schema-v5|forbids " + "SharedRegisterPublishMetadata|only valid for shared TensorMap", + ): + analyze_capture(path) + + def test_v4_shared_loser_rejects_winner_only_frontend(self) -> None: + for phase, start, end in ( + ("Materialize", 1144, 1152), + ("Register", 1154, 1160), + ): + with self.subTest(phase=phase): + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + rows.append(_row(0, 1, phase, start, end)) + if phase == "Register": + rows.append( + _row( + 0, + 1, + "SharedRegisterPublishMetadata", + start + 1, + end - 1, + ) + ) + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex( + ValueError, + rf"shared loser path requires 0 {phase} spans|" + "requires exactly one Register parent for each winner " + "and none for losers|" + "requires exactly one SharedRegisterPublishMetadata " + "for each winner and none for losers", + ): + analyze_capture(path) + + def test_v4_shared_winner_requires_frontend_and_forbids_prepare_map(self) -> None: + for phase in ("Materialize", "Register"): + with self.subTest(missing=phase): + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + capture["fdwic_events"] = [ + row + for row in rows + if not ( + row[0] == 0 + and row[3] == 0 + and ( + row[5] == phase + or ( + phase == "Register" + and row[5] == "SharedRegisterPublishMetadata" + ) + ) + ) + ] + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex( + ValueError, + rf"shared winner path requires 1 {phase} spans|" + "requires exactly one Register parent for each winner " + "and none for losers|" + "requires exactly one SharedRegisterPublishMetadata " + "for each winner and none for losers", + ): + analyze_capture(path) + + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + rows.append(_row(0, 1, "PrepareMap", 1144, 1148)) + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex( + ValueError, "shared schema-v5 must not contain PrepareMap" + ): + analyze_capture(path) + + def test_v4_phase_only_capture_still_has_dropped_evidence_and_closes(self) -> None: + capture = _v4_capture() + capture["l2_swimlane_level"] = 1 + rows = capture["fdwic_events"] + assert isinstance(rows, list) + capture["fdwic_events"] = [ + row for row in rows if row[5] not in {"Atomic", "ClockBaseline"} + ] + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + report = analyze_capture(path) + + self.assertEqual(report["validation"]["dropped_records"], 0) + self.assertEqual(report["overlays"]["Atomic"]["event_count"], 0) + self.assertEqual(report["overlays"]["ClockBaseline"]["event_count"], 0) + self.assertIs( + report["aggregate_core_work"]["closure"]["worker_completion"]["exact"], + True, + ) + + def test_v4_parent_boundary_gap_is_rejected(self) -> None: + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + final_drain = next( + row for row in rows if row[0] == 0 and row[5] == "FinalDrain" + ) + final_drain[6] = int(final_drain[6]) + 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex( + ValueError, "OrchestrationReplay.end must equal FinalDrain.start" + ): + analyze_capture(path) + + def test_v4_submit_must_stay_inside_orchestration_parent(self) -> None: + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + orchestration = next( + row for row in rows if row[0] == 0 and row[5] == "OrchestrationReplay" + ) + orchestration[6] = 1001 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "outside OrchestrationReplay"): + analyze_capture(path) + + def test_v4_tail_cannot_precede_frontend_children(self) -> None: + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + winner_tail = next( + row + for row in rows + if row[0] == 1 and row[3] == 1 and row[5] == "WinnerBuild" + ) + winner_tail[6] = int(winner_tail[6]) - 20 + winner_tail[7] = int(winner_tail[7]) - 20 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "must start at or after"): + analyze_capture(path) + + def test_v4_loser_cannot_carry_winner_only_fanin(self) -> None: + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + rows.append(_row(0, 1, "Fanin", 1156, 1158, function_id=0)) + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "loser path requires 0 Fanin"): + analyze_capture(path) + + def test_v4_kernel_must_have_one_supported_parent(self) -> None: + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + # core0 两个 Submit 之间的 orchestration gap 不是合法 Kernel 容器。 + rows.append(_row(0, 0, "Kernel", 1105, 1110, function_id=0)) + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "Kernel must be contained"): + analyze_capture(path) + + def test_v4_kernel_inside_winner_build_is_classified(self) -> None: + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + winner_build = next( + row + for row in rows + if row[0] == 1 and row[3] == 1 and row[5] == "WinnerBuild" + ) + rows.append( + _row( + 1, + 0, + "Kernel", + int(winner_build[6]) + 2, + int(winner_build[6]) + 5, + function_id=0, + ) + ) + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + report = analyze_capture(path) + + containment = report["kernel_containment"] + self.assertEqual(containment["inside_winner_build_events"], 1) + self.assertEqual(containment["inside_submit_tail_events"], 1) + self.assertEqual(containment["orphan_events"], 0) + + def test_v4_final_drain_kernel_crossing_boundary_is_rejected(self) -> None: + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + final_drain = next( + row for row in rows if row[0] == 0 and row[5] == "FinalDrain" + ) + final_kernel = next( + row + for row in rows + if row[0] == 0 and row[5] == "Kernel" and int(row[6]) > int(final_drain[6]) + ) + final_kernel[7] = int(final_drain[7]) + 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "crosses FinalDrain"): + analyze_capture(path) + + def test_role_statistics_are_separate_from_global_and_core_work(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, _capture()) + report = analyze_capture(path) + + roles = report["per_role_core_statistics"] + self.assertEqual(roles["aic"]["core_count"], 32) + self.assertEqual(roles["aiv"]["core_count"], 64) + for role in ("aic", "aiv"): + envelope = roles[role]["metrics"]["submit_envelope"] + self.assertEqual(envelope["median_cycles"], 200) + self.assertEqual(envelope["p95_cycles"], 200) + self.assertEqual(envelope["max_cycles"], 200) + self.assertEqual( + report["global_submit_makespan"]["semantics"], + "cross-core wall-clock envelope; not aggregate core-work", + ) + self.assertEqual( + report["aggregate_core_work"]["semantics"], + "sum of per-core cycles; not wall-clock duration", + ) + + def test_all_overlay_phases_are_reported_but_never_added(self) -> None: + capture = _capture() + with tempfile.TemporaryDirectory() as directory: + baseline_path = self._write(directory, capture) + baseline = analyze_capture(baseline_path) + + # 大幅拉长所有 Atomic/Build/Replay/Alloc span；排他结果必须保持不变。 + rows = capture["fdwic_events"] + assert isinstance(rows, list) + for row in rows: + if row[5] in {"Atomic", "Build", "Replay", "Alloc"}: + row[7] = int(row[7]) + 10_000 + changed_path = Path(directory) / "changed.json" + changed_path.write_text(json.dumps(capture), encoding="utf-8") + changed = analyze_capture(changed_path) + + self.assertEqual( + baseline["aggregate_core_work"]["metrics_cycles"], + changed["aggregate_core_work"]["metrics_cycles"], + ) + self.assertEqual(set(changed["overlays"]), set(OVERLAY_PHASES)) + for phase in OVERLAY_PHASES: + self.assertIs(changed["overlays"][phase]["included_in_additive_totals"], False) + self.assertIs(changed["semantics"]["overlays_are_additive"], False) + + def test_overlapping_submits_on_same_core_lane_are_rejected(self) -> None: + capture = _capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + first_end = next( + int(row[7]) + for row in rows + if row[0] == 0 and row[3] == 0 and row[5] == "Submit" + ) + second = next( + row for row in rows if row[0] == 0 and row[3] == 1 and row[5] == "Submit" + ) + second[6] = first_end - 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "overlapping Submit"): + analyze_capture(path) + + def test_overlapping_exclusive_children_are_rejected(self) -> None: + capture = _capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + efdrain_end = next( + int(row[7]) + for row in rows + if row[0] == 0 and row[3] == 0 and row[5] == "EfDrain" + ) + materialize = next( + row for row in rows if row[0] == 0 and row[3] == 0 and row[5] == "Materialize" + ) + materialize[6] = efdrain_end - 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "overlapping exclusive children"): + analyze_capture(path) + + def test_exclusive_child_task_must_match_containing_submit(self) -> None: + # private 保留显式 EfDrain，专门隔离“child 被错误 Submit 包含”的 + # 原分析器门禁；shared 的 Claim 越界会更早被派生边界门禁拒绝。 + capture = _v4_capture(tensormap_mode="private") + rows = capture["fdwic_events"] + assert isinstance(rows, list) + first_claim = next( + row for row in rows if row[0] == 0 and row[3] == 0 and row[5] == "Claim" + ) + second_claim = next( + row for row in rows if row[0] == 0 and row[3] == 1 and row[5] == "Claim" + ) + # 只交换时间，不改 task/flags。converter 的 schema 键与 winner 语义仍 + # 合法；分析器必须拒绝 task1 Claim 被时间包含进 task0 Submit 的伪闭合。 + first_claim[6:8], second_claim[6:8] = second_claim[6:8], first_claim[6:8] + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "does not match containing Submit"): + analyze_capture(path) + + def test_v4_rejects_removed_loser_replay_phase(self) -> None: + capture = _v4_capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + rows.append(_row(2, 0, "LoserReplay", 1065, 1065)) + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "unknown phase 'LoserReplay'"): + analyze_capture(path) + + def test_kernel_crossing_efdrain_boundary_is_rejected(self) -> None: + capture = _capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + efdrain_end = next( + int(row[7]) + for row in rows + if row[0] == 0 and row[3] == 0 and row[5] == "EfDrain" + ) + kernel = next( + row for row in rows if row[0] == 0 and row[3] == 0 and row[5] == "Kernel" + ) + kernel[6] = efdrain_end - 1 + kernel[7] = efdrain_end + 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "crosses EfDrain"): + analyze_capture(path) + + def test_missing_required_child_is_rejected(self) -> None: + capture = _capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + capture["fdwic_events"] = [ + row + for row in rows + if not (row[0] == 0 and row[3] == 0 and row[5] == "Register") + ] + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "requires exactly one Register"): + analyze_capture(path) + + def test_incomplete_task_stream_is_rejected(self) -> None: + capture = _capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + capture["fdwic_events"] = [ + row for row in rows if not (row[0] == 95 and row[3] == 1) + ] + _refresh_summary(capture) + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "task IDs do not match"): + analyze_capture(path) + + def test_nonzero_dropped_records_is_rejected(self) -> None: + capture = _capture() + metadata = capture["metadata"] + assert isinstance(metadata, dict) + summary = metadata["fdwic_summary"] + assert isinstance(summary, dict) + summary["dropped_records"] = 1 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "dropped_records"): + analyze_capture(path) + + def test_old_schema_without_dropped_evidence_is_rejected(self) -> None: + capture = _capture() + capture["l2_swimlane_level"] = 1 + metadata = capture["metadata"] + assert isinstance(metadata, dict) + metadata["trace_schema_version"] = 2 + metadata.pop("fdwic_summary") + # v2 不允许 v3 ClockBaseline/Atomic flags，删去它们后应由排他分析器 + # 因缺少 dropped 证据拒绝，而不是先落入 converter 的 flags 门禁。 + rows = capture["fdwic_events"] + assert isinstance(rows, list) + capture["fdwic_events"] = [ + row for row in rows if row[5] not in {"ClockBaseline", "Atomic"} + ] + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + with self.assertRaisesRegex(ValueError, "requires trace_schema_version=3"): + analyze_capture(path) + + def test_write_is_atomic_and_failure_keeps_existing_output(self) -> None: + capture = _capture() + rows = capture["fdwic_events"] + assert isinstance(rows, list) + second = next( + row for row in rows if row[0] == 0 and row[3] == 1 and row[5] == "Submit" + ) + second[6] = 1099 + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, capture) + output = Path(directory) / "exclusive.json" + output.write_text("existing-good-output\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "overlapping Submit"): + write_analysis(path, output) + self.assertEqual(output.read_text(encoding="utf-8"), "existing-good-output\n") + self.assertEqual(list(Path(directory).glob(".exclusive.json.*.tmp")), []) + + def test_cli_requires_output_and_publishes_complete_json(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = self._write(directory, _capture()) + output = Path(directory) / "exclusive.json" + with contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as raised: + main([str(path)]) + self.assertEqual(raised.exception.code, 2) + + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(main([str(path), "-o", str(output)]), 0) + document = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual(document["validation"]["status"], "PASS") + self.assertTrue(output.read_text(encoding="utf-8").endswith("}\n")) + self.assertEqual(list(Path(directory).glob(".exclusive.json.*.tmp")), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/atomic_probe/perf_opt_record.md b/tests/atomic_probe/perf_opt_record.md new file mode 100644 index 0000000000..4f0171f5f9 --- /dev/null +++ b/tests/atomic_probe/perf_opt_record.md @@ -0,0 +1,3399 @@ +# A5 PA Submit 性能优化全过程记录 + +## 1. 文档目的与状态约定 + +本文持续记录 A5 FDWIC Paged Attention `Case1` 的 Submit 调度性能优化过程。**当前性能优化目标只包含真实 simpler PA 路径**；`tests/atomic_probe/pa_scheduler` standalone 只保留为已经完成的历史方法验证、负结果和模型边界证据，不再作为当前待办或下一阶段优化对象。目标是让后续真实 PA 优化能够从可复核的源码、提交、实测数据和产物继续推进，而不是仅保留最终结论。 + +本文使用以下状态标签： + +| 标签 | 含义 | +| --- | --- | +| **[已保留]** | 已进入当前代码路径，并完成与风险相称的正确性和性能验证 | +| **[已撤回]** | 做过实现或实验，但因语义不成立、性能回退或证据不足而不再保留 | +| **[观察工具]** | 用于建立测量能力，本身不是业务性能优化 | +| **[历史证据]** | 对当时源码和构建有效，不能自动代表当前 HEAD | +| **[受限]** | 已确认存在平台、模型或验证覆盖边界 | +| **[设计中]** | 只有经过源码核对的方案，尚无完成提交或性能结论 | +| **[验证中]** | 候选已落盘并通过部分门禁，但尚未完成真实 A5 正确性或性能裁决 | + +记录更新至 2026-07-21。当前分支为 `fdwic-swimlane-exclusive`，本阶段开始时 HEAD 为 `9f6140c1`，跟踪 `origin/fdwic-swimlane-deps`。后续每完成一个合理阶段，都应按第 12 节模板更新本文并形成一条带详细中文说明的本地提交。 + +更细的专题资料分别见： + +- [A5 FDWIC Paged Attention 安装与复现指南](a5_fdwic_atomic_swimlane_repo.md)； +- [PA 原子操作与优化记录](pa_scheduler/PA-atomic情况分析.md)； +- [PA 调度器独立复现与泳道使用指南](pa_scheduler/PA调度器独立复现与泳道使用指南.md)； +- [FDWIC 泳道排他分区与闭合分析](pa_scheduler/swimlane_opt_anal.md)； +- [I-cache Miss 采集与分析指南](icache_miss_usage_guide.md)。 + +## 2. 固定范围、环境与权威性能口径 + +### 2.1 本轮范围 + +- 真实用例：`examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py`； +- Case：`Case1`； +- runtime：`fully_distributed_within_core`； +- 平台：A5Sim 用于功能和控制流回归，真实 A5 用于性能结论； +- A5 工作核：32 个 AIC、64 个 AIV，共 96 个 worker； +- 工作量：256 batch，每 batch 依次包含 Alloc、QK、SF、PV、UP 五个 task； +- 每核 Submit：`256 * 5 = 1280`，全局 Submit：`96 * 1280 = 122880`。 + +本文不把其他 runtime、其他 PA Case、A2/A3、整段 pytest wall time 或整个 device 任务耗时混入 Submit 性能结论。 + +### 2.2 已验证环境 + +| 项目 | 固定值 | +| --- | --- | +| 设备 | `/dev/davinci0`，Ascend950PR_958b | +| Driver | `7.0.t9.0.B798`，ascendhal `7.35.23` | +| CANN | 用户目录下 9.1.0 weekly 20260708 | +| CCEC | clang 15.0.5 | +| Python | `/home/q00473782/.venv`，Python 3.12.3 | +| PyTorch | 2.6.0+cpu | +| pytest | 7.4.4 | +| GCC 15 | `/home/q00473782/.local/gcc-15/root`，15.0.1 | +| PTO-ISA | `ddafa8da9c760ecd13fe9fe2833d6ee55fb20bd8` | + +非交互 shell 不能假设自动读取 `.bashrc`。正式复测应显式 source 用户 CANN，激活本用户 `.venv`，并显式设置用户 GCC 15 的 `PATH`、`LD_LIBRARY_PATH` 和 `CXX`。完整命令以安装复现指南为准。 + +### 2.3 权威性能定义 + +本文所说的“完整 Submit 时间”默认指： + +```text +全部 worker 中最早的第一个 Submit 开始 + 到 +全部 worker 中最晚的最后一个 Submit 结束 +``` + +它排除启动屏障和 FinalDrain，不等于 pytest wall time，也不等于整个 kernel launch 的 device wall time。必须同时区分三类时间： + +1. **跨核完整 Submit 时间**：上述约 5 ms 的全局墙钟范围，是候选是否保留的最终性能口径； +2. **逐核或全核累计工作量**：某个 span 在 96 核上的时长求和，用于描述工作分布，不是 96 核共同形成的墙钟； +3. **PMU total/core**：每个物理子核 PMU gate 内的周期数，是单核周期工作量，也不是跨核完整 Submit 时间。 + +泳道原始时间使用 1 ns/tick 的 `SYS_CNT`。本机 cold/warm 校准得到 PMU 频率约 1.65 cycles/ns：ALL/AIC/AIV 分别为 1.649844/1.650062/1.649731。PMU cycle 换算不能反过来改变 `SYS_CNT` 的 1 ns/tick 定义。 + +## 3. 起始基线 + +### 3.1 环境打通与 5.6 ms 基线 + +**[已保留] `657313c9` — `fix(a5): enable paged attention on legacy drivers`** + +该提交完成 A5 平台 block 数解析和经过双重校验的 flat OCCUPY 回退，使当前旧 Driver 环境能够运行目标 PA。A5Sim 和 A5 Case1 均通过，真实 A5 level-1 泳道复现的首末 Submit 为 **5.642245 ms**；仓库更早的历史参考为 5.577570 ms。 + +基线产物： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260717_023809/ + merged_swimlane_atomic_load.json + l2_swimlane_records.json +``` + +这是第一轮业务优化的比较起点，不代表后续加入观察代码后的 ELF。 + +## 4. 提交时间线与阶段结论 + +下表按当前分支中的逻辑推进顺序列出与本轮工作直接相关的提交。详细证据和适用边界见后续各节；纯文档重命名没有单独列项。 + +| 日期 | 提交 | 类型 | 阶段摘要 | +| --- | --- | --- | --- | +| 2026-07-17 | `657313c9` | 正确性基础 | 打通旧 Driver 上的 A5 PA | +| 2026-07-17 | `ce89fae2` | 编译探针 | 建立嵌套 lambda 与跨 TU 基线 | +| 2026-07-17 | `e3b748b4` | 真实 PA 优化 | 跳过 BlockWon 无效轮询并复用参数 mask | +| 2026-07-17 | `a3f5ecc2` | 编译探针 | 隔离 inline/noinline Submit 边界 | +| 2026-07-17 | `290dbda0` | standalone | 建立 CCEC/AscendC/CPU 独立 PA 模型 | +| 2026-07-18 | `76df85ce` | 观察工具 | 隔离验证 atomic 与 I-cache PMU 归类 | +| 2026-07-18 | `04ec9b95` | 真实 PA/standalone 优化 | HeapGuard 首圈跳过冗余 atomic load | +| 2026-07-18 | `3d174a08` | 负结果 | 记录并撤回 fanin 顺序实验 | +| 2026-07-18 | `3d0aaea7` | 观察工具 | 建立 standalone 每核 scalar PMU | +| 2026-07-18 | `67407cc4` | 观察工具 | 细分 fanin/frontier 动态 atomic 次数 | +| 2026-07-18 | `8deefdef` | 观察工具 | 修正 PMU owner 活跃子核与恢复闭环 | +| 2026-07-18 | `13431a23` | 观察工具 | 建立 standalone 逐 atomic 泳道 | +| 2026-07-18 | `c93bd65d` | 观察工具 | 接通 standalone 自包含 PMU owner | +| 2026-07-18 | `640efe50` | 观察工具 | 建立单次 I-cache miss 标尺 | +| 2026-07-18 | `99971ac1` | 观察工具 | 建立 standalone Submit 全区间 PMU | +| 2026-07-18 | `c6daaeb7` | 文档证据 | 固化当时 atomic/PMU 观察口径 | +| 2026-07-18 | `5274945b` | 合并 | 吸纳 I-cache miss 标尺改动 | +| 2026-07-18 | `e66001ff` | standalone | CCEC 接入真实 Cube/Vector 负载 | +| 2026-07-18 | `9aeda0dd` | standalone | AscendC 接入真实 Cube/Vector 负载 | +| 2026-07-18 | `1d3a374a` | standalone | CPU 补齐对等算术负载 | +| 2026-07-18 | `0c9cebc3` | standalone | 增加非均匀布局诊断 | +| 2026-07-18 | `7bb118a8` | standalone | 默认切换为 `real-compute/6,28,4,1` | +| 2026-07-18 | `cbaf7c60` | 观察工具 | 真实 PA 接入 atomic/PollBatch 泳道 | +| 2026-07-18 | `44199a54` | 观察工具 | 固化 96 核 I-cache 分组分析 | +| 2026-07-18 | `187e54bc` | 观察工具 | 完善 standalone atomic 合并泳道 | +| 2026-07-18 | `5c2d39b8` | 观察工具 | 拆分 standalone `swimlane/submit-pmu` 构建 | +| 2026-07-18 | `8430f418` | 观察工具 | 增加历史 EfDrain 局部 PMU | +| 2026-07-19 | `7466e6f5` | 观察工具 | 完善局部 PMU、HTML 并退役 WaitForSlot phase | +| 2026-07-19 | `6caa269c` | 观察工具 | 收敛 standalone 排他 span 与分析器 | +| 2026-07-19 | `d2d8ce25` | standalone 优化 | 将低频 winner 调整为冷分支 | +| 2026-07-19 | `cafa9ca5` | 真实 PA 优化 | 恢复真实 PA loser 热路布局 | +| 2026-07-19 | `44367971` | 观察代码优化 | 外提 atomic 冷路径，消除取指布局回退 | +| 2026-07-19 | `14c2429f` | 文档证据 | 分离 atomic 与 I-cache 专题记录 | +| 2026-07-19 | `dbb95bb5` | 观察工具 | 将排他泳道迁入真实 FDWIC | +| 2026-07-19 | `911ecf9a` | 文档证据 | 固化排他 span 和性能分布分析 | +| 2026-07-20 | `ba4334d1` | 独立对照 | 交付 A/B/C compete-first/lazy 对照 | +| 2026-07-20 | `0d08c437` | standalone 优化 | 主路采用 compete-first eager | +| 2026-07-20 | `2899cc35` | 真实 PA 重构 | 接入 compete-first eager begin/finish | +| 2026-07-20 | `84e9d6d0` | 文档证据 | 建立真实 PA Submit 全过程记录 | +| 2026-07-20 | `8d5aa686` | 观察工具 | 建立真实 PA 低扰动 perf-clock 基线 | +| 2026-07-20 | `9f6140c1` | 观察工具 | 收口真实 PA 业务与 atomic 合并泳道 | +| 2026-07-21 | `faa370d9` | 观察工具 | 建立真实 PA `submit-pmu-none` 全窗证据链 | +| 2026-07-21 | `2a7dccee` | 观察工具 | 建立真实 PA `arg-build` 单阶段 PMU | +| 2026-07-21 | `81a1f382` | 观察工具 | 量化真实 PA running bracket 的空区间记录开销 | +| 2026-07-21 | `26cbece6` | 观察工具 | 建立真实 PA `materialize` 单阶段 PMU | +| 2026-07-21 | `d96f5a5a` | 观察工具 | 建立真实 PA `claim` 单阶段 PMU | +| 2026-07-21 | `2929b21e` | 观察工具 | 建立真实 PA `register` 单阶段 PMU | +| 2026-07-21 | `d1572c33` | 观察工具 | 建立真实 PA `submit-transition` 单阶段 PMU | +| 2026-07-21 | `53088c48` | 观察校准 | 交错量化三类真实 PA 观察构建的整体影响 | +| 2026-07-21 | `40bd6602` | 波动分析 | 收口独占设备上的 P 构建波动来源 | +| 2026-07-21 | `a58ee868` | PMU 分析 | 收口 N 构建的 Scalar/I-cache 波动载体 | +| 2026-07-21 | `a17c188a` | 观察工具 | 建立真实 PA Submit 窗内 Kernel 低容量聚合 | +| 2026-07-21 | `05397cd7` | 波动分析 | 收口 K 构建的 Kernel/residual 波动载体 | +| 2026-07-21 | `6acebc8f` | 环境取证 | 记录设备状态、低功耗接口及并发边界 | +| 2026-07-21 | `52dc04c7` | 阶段归因 | 排除 Materialize 为同 ELF 波动主载体 | +| 2026-07-21 | `7fa7399f` | 阶段归因 | 排除 Claim 为同 ELF 波动主载体 | +| 2026-07-21 | `afeb515a` | 阶段归因 | 排除 SubmitTransition 为同 ELF 波动主载体 | +| 2026-07-21 | `57aedfee` | 阶段归因 | 排除 ArgBuild 为同 ELF 波动主载体 | +| 2026-07-21 | `660bbff4` | 阶段归因 | 排除 Register 为同 ELF 波动主载体 | +| 2026-07-21 | `443a0bb3` | 观察工具 | 完善真实 PA I-cache 逐核时间加工口径 | +| 2026-07-21 | `15c54b33` | 观察工具 | 为真实 PA PMU 产物绑定构建 provenance | +| 2026-07-21 | `36547252` | 上板验证 | 闭合完整 Submit 与 Register 分段三件套 | +| 2026-07-21 | `21e0414c` | 观察工具 | 建立排除 linked Kernel 的 EfDrain-control 固定容量 PMU | +| 2026-07-21 | `77df3959` | 上板验证 | 闭合 EfDrain-control 的实际 K、N+K 与构建三件套 | +| 2026-07-21 | `159f3c4c` | 原因取数 | 形成 EfDrain-control 的首轮 Case1 AIC/AIV 稳态数据 | +| 2026-07-21 | `7131bdf5` | 负结果 | 记录并撤回 BlockWon 慢路冷外提 | +| 2026-07-21 | `0d6547b9` | 观察回归 | 补齐 I-cache 观察链与 schema-v4 atomic 组合门禁 | +| 2026-07-21 | `970e4fad` | 正确性门禁 | 直接覆盖真实 FDWIC TensorMap 清退语义 | +| 2026-07-21 | `2dc49a13` | 优化候选 | 跳过 PrepareMap 空 task-head 的冗余 GM 写回 | + +### 4.1 真实 PA 第一轮 atomic 与前端优化 + +#### 4.1.1 跳过单 lane 图无效 BlockWon 轮询并复用参数掩码 + +**[已保留] `e3b748b4` — `Update: 优化 A5 FDWIC Submit 热路径`** + +该阶段包含两类改动： + +1. 在本 worker 第一次见到 joint submit 之前，跳过无意义的 BlockWon 轮询； +2. 复用一次 tensor tag 扫描生成的 output/register mask，减少重复前端扫描。 + +PA Case1 全部 task 都是单 lane，因此第一项确定删除 Submit 内 **146944 次** 无效 `atomic_load(any_pub)`；A5 上该封装实际为 `atomicAdd(addr, 0)`，并非普通 load。删除位置主要落在每次 Submit 开头的 EfDrain 和高频 loser 的公共尾部，没有删除 Claim。 + +实测结果： + +| 版本 | 首末 Submit | +| --- | ---: | +| 初始基线 | 5.642245 ms | +| joint polling skip 三轮中位数 | 5.171330 ms | +| 加 register mask 三轮中位数 | 5.186679 ms | +| 加 output/register masks 三轮中位数 | **5.115620 ms** | + +最终相对初始基线减少 **0.526625 ms，约 9.33%**；最好单轮为 **5.096685 ms**。kernel 累计时长没有随之缩短，证据支持收益来自调度前端，而不是计算 kernel 变快。 + +最终最好产物： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260717_055638/ + merged_swimlane_best_joint_poll_skip_arg_masks_5.096685ms.json +``` + +mask 的独立收益只有三轮方向性证据，不能从组合版本中严格拆出因果比例；当前保留的是经过整体正确性回归的组合改动。 + +#### 4.1.2 用普通 load 和 NOP 替换 atomic 的定位实验 + +**[已撤回] 未形成保留提交** + +为确认 `atomic_load` 的成本量级，曾临时替换为 `ld_dev()+nop(100)` 和 `ld_dev()+nop(10)`： + +| 诊断变体 | 首末 Submit | 产物 | +| --- | ---: | --- | +| `ld_dev()+nop(100)` | 5.343592 ms | `outputs/TestPagedAttentionUnroll_Case1_20260717_035341/merged_swimlane_nop100.json` | +| `ld_dev()+nop(10)` | 5.401034 ms | `outputs/TestPagedAttentionUnroll_Case1_20260717_035954/merged_swimlane_nop10.json` | + +普通 device load 加固定 NOP 不具备 atomic RMW 的同步、可见性和顺序语义，因此这些样本只用于定位成本，源码修改已经撤回，不能作为可用优化方案。 + +#### 4.1.3 HeapGuard 首圈 fast path + +**[已保留] `04ec9b95` — `perf(a5): 跳过HeapGuard首圈冗余原子读取`** + +历史文档也使用同内容提交号 `2c3dd1e2`；当前分支可达 hash 为 `04ec9b95`。默认 256 MiB heap 下，逻辑 heap 尚未走完第一圈时不可能覆盖旧输出，因此在原 fatal 检查之内直接返回，不读取 frontier/vend。PA Case1 的 Alloc、QK、SF、PV 共确定消减 **1024 次 frontier atomic load**；跨圈 slow path 保持原协议。 + +真实 A5 十对结果： + +| 指标 | 基线 | H1 | 相对变化 | +| --- | ---: | ---: | ---: | +| 中位数 | 5.142168 ms | 5.122320 ms | -0.386% | +| 均值 | 5.167064 ms | 5.146984 ms | -0.389% | +| p90 | 5.274224 ms | 5.229773 ms | -0.843% | + +十对中 8 胜 2 负，配对变化中位数为 **-0.324%**。因此只能称为“确定减少 atomic，真实 PA 中心趋势小幅正向”，不能把 standalone 的大幅波动外推到真实 PA。最好样本为 5.098696 ms： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260717_173313/ + merged_swimlane_heapguard_first_lap_fastpath_5.098696ms.json +``` + +#### 4.1.4 fanin 检查顺序实验 + +**[已撤回] `3d174a08` — `文档(a5): 记录fanin顺序实验与回退结论`** + +F1 在 standalone 中把有效 fanin producer 按 task id 降序排列。静态分析证明固定调度状态下不会增加 ready load，实测 fanin load 也下降；但十对交错 A/B 中： + +```text +Submit 中位数：4290.401 -> 4623.944 us，+7.774% +Submit 配对变化中位数：+6.439% +``` + +fanin load 减少没有转化为 Submit 收益，说明指令布局和 worker 到达时序的间接变化不能忽略。候选代码已撤回，未迁移到真实 FDWIC。 + +### 4.2 standalone 建立与动态 atomic 取证 + +#### 4.2.1 建立三后端独立 PA 模型 + +**[观察工具] `290dbda0` — `test(a5): 增加独立 PA 调度性能复现用例`** + +在 `tests/atomic_probe/pa_scheduler` 下建立 CCEC、AscendC、CPU 三后端，保留 Case1 五 task 拓扑、96 worker 回放、四分片 Claim、TensorMap、fanin、私有 ring、WaitForSlot、HeapGuard、completion flag/vend/frontier 和最终 drain。此时 winner 计算仍以可控 NOP 为主，目标首先是闭合调度协议与 atomic 次数。 + +`ce89fae2` 和 `a3f5ecc2` 还建立了嵌套 lambda、跨 TU caller context 和 inline/noinline 边界探针。这些是后续拆分 callback/finish 时的编译行为依据，不是 PA 本身的性能收益。 + +#### 4.2.2 HeapGuard 对等修改与压力回归 + +**[已保留] `04ec9b95` 同时修改 standalone 与真实 PA** + +standalone 默认 256 MiB 配置、16 MiB 多圈压力和恢复默认值后的三后端回归均经过语义检查。16 MiB CCEC/AscendC b256 确实进入 slow path；CPU b256 在 fast 版和临时撤销 fast path 的原版中都超过观察时限，不能记为 H1 PASS 或 FAIL，详见第 9 节。 + +#### 4.2.3 fanin/frontier 软件计数 + +**[观察工具] `67407cc4` — `测试(a5): 细分PA依赖与frontier原子计数`** + +计数只写 worker 私有 `LocalStats`，结束时一次发布，不增加共享 atomic。b256 十轮动态基线为： + +| 指标 | 中位数 | +| --- | ---: | +| fanin 总 load | 93201.5 | +| fanin not-ready load | 86675.5 | +| frontier FetchMax | 15365 | +| Submit+completion atomic ops | 203803.5 | + +该结果确认主要动态项是 not-ready 重试和 frontier helping，而不是 ready 前缀。软件计数扩大了 sidecar 并增加私有 scalar 增量，因此这一版的绝对 Submit 时间不能与无计数 ELF 直接归因比较。原始日志： + +```text +tests/atomic_probe/pa_scheduler/outputs/atomic_diagnostics/ + ccec_baseline_10_20260718_023551.log +``` + +### 4.3 scalar、atomic 与 I-cache 观察工具 + +以下提交建立证据链，但不应被写成业务性能优化： + +| 提交 | 状态 | 主要内容 | +| --- | --- | --- | +| `76df85ce` | [观察工具] | 用隔离 probe 验证 atomic 与 I-cache 等待周期的 PMU 归类 | +| `3d0aaea7` | [观察工具] | 在 standalone 建立每核 scalar PMU 读取链路 | +| `8deefdef` | [观察工具] | 修正 PMU owner 的活跃物理子核配置和恢复闭环 | +| `c93bd65d` | [观察工具] | 接入自包含 Main AICPU Path-A owner，保存、配置并恢复 32 AIC+64 AIV 状态 | +| `13431a23` | [观察工具] | 给 standalone 增加逐 atomic 泳道、调用点和边界语义 | +| `640efe50` | [观察工具] | 建立隔离 cold/warm 单次 I-cache miss 一阶标尺 | +| `99971ac1` | [观察工具] | 建立每核完整 Submit PMU gate 和 PMU-only JSON | +| `c6daaeb7` | [历史证据] | 固化当时 atomic 与 Submit PMU 的边界和使用限制 | +| `5274945b` | [观察工具] | 合并单次 I-cache miss 标尺相关改动 | +| `44199a54` | [观察工具] | 固化 96 核 raw 复算、AIC/AIV 分组和多轮分析口径 | +| `187e54bc` | [观察工具] | 合并普通阶段与 atomic 泳道，并用 PollBatch 压缩等待轮询 | +| `5c2d39b8` | [观察工具] | 将 `swimlane` 与 `submit-pmu` 拆成独立重编译产物 | +| `8430f418` | [观察工具] | 增加历史 EfDrain 局部 PMU 归因 | +| `7466e6f5` | [观察工具] | 增加 Materialize/Register、HTML 报告，退役 WaitForSlot 局部 PMU | + +#### 4.3.1 atomic 时间边界 + +逐 atomic 观察不插入 DSB，也不强制原本不消费返回值的 Exchange/FetchAdd 变成等待返回型操作： + +- 真正消费返回值的 FetchMax、claim exchange 使用 `return_ready`；CCEC 通过返回值地址依赖后再读取 `SYS_CNT`，尽力使结束点晚于返回值可用； +- 不消费返回值的 Exchange/FetchAdd 使用 `source_issue`，只表示源码发射包围区间，不能解释成 atomic 已在全局完成； +- PollBatch 表示一个等待区间内多次逻辑轮询的整体时间和精确调用数，不能把其 duration 除成单次 atomic latency。 + +#### 4.3.2 PMU owner 的响应校准 + +自包含 owner 的 empty、100000 scalar NOP、2×100000 scalar NOP 三组，96 核 PMU total 中位数约为 214、56568、112994 cycles。这只证明 gate 响应和工作量近似倍增，不表示同数值的纳秒，也不是 PA Submit 基线。 + +#### 4.3.3 单次 I-cache miss 标尺 + +隔离微基准的结果为： + +| 规模 | ALL 中位数 | 轮间范围 | +| --- | ---: | ---: | +| 64 trials/core × 10 轮 | 86.596 ns/miss | 86.532～86.792 | +| 128 trials/core × 5 轮 | 89.629 ns/miss | 89.615～89.648 | + +统一使用 **90 ns/miss** 作为单核串行等效的一阶感性标尺。它不是实际 Submit 墙钟损失，不能把 `miss * 90 ns` 从约 5 ms 中直接减掉。 + +### 4.4 standalone winner 负载迁移到真实 Cube/Vector + +| 提交 | 状态 | 主要内容 | +| --- | --- | --- | +| `e66001ff` | [已保留] | CCEC QK/PV 接入真实 Cube matmul，SF/UP 接入真实 Vector add/mul | +| `9aeda0dd` | [已保留] | AscendC 接入对等 Cube/Vector 负载并修正布局 | +| `1d3a374a` | [已保留] | CPU 增加对等算术和统一路由，用于功能对照 | +| `0c9cebc3` | [已保留] | 增加非均匀输入，验证转置、ND/NZ、stride 和输出布局 | +| `7bb118a8` | [已保留] | 三后端默认切到 `real-compute/6,28,4,1`，NOP 只保留显式兼容入口 | + +最终同泳道口径下，standalone CCEC b256 五轮为： + +```text +5.002413 / 4.875193 / 4.968894 / 4.992477 / 4.876282 ms +中位数 4.968894 ms +``` + +当时真实 PA 三轮中位数为 5.115620 ms，同口径差 146.726 us，约 2.87%。这说明 standalone 达到“独立复现约 5 ms 调度”的目标，不证明两份实现的代码布局、数据流和跨核时序完全一致。历史 raw： + +```text +tests/atomic_probe/pa_scheduler/outputs/performance_gap_20260718/ + standalone_ccec_real_b256_raw.json +``` + +### 4.5 真实 PA atomic 泳道与 PollBatch + +**[观察工具] `cbaf7c60` — `工具(a5): 接通真实PA atomic泳道与精确轮询聚合`** + +该提交把 standalone 验证过的边界迁入真实 FDWIC，schema-v3 使用 32 B 紧凑 record、28 个稳定 site 和五类 atomic op，并只在允许的等待区聚合 PollBatch。当时真实 A5 PA Case1 level-4 通过： + +```text +115200 次逻辑 atomic +110006 条物理 Atomic +340 条 PollBatch +dropped_records = 0 +``` + +逻辑调用、物理记录和 PollBatch 必须满足 producer 定义的闭合公式。level-4 结果用于观察 atomic 分布，不能替代关闭诊断后的性能基线。 + +### 4.6 schema-v4 排他 span 与 raw 规模控制 + +#### 4.6.1 standalone 排他 span + +**[观察工具] `6caa269c` — `工具(a5): 收敛Submit观测边界与排他泳道分析`** + +主要变化： + +- 增加 OrchestrationReplay、FinalDrain 等父区间； +- 旧 Build/Replay/Alloc lap 改为真实 WinnerBuild/AllocComplete 尾动作； +- standalone loser 没有真实计算动作，不再为 121600 个 loser 生成伪 `LoserReplay` record； +- Submit 内和 Submit 间未覆盖时间由 converter/analyzer 使用已有边界离线求差，不增加设备 record、字段或时间戳； +- merged 只保留 Perfetto 必需字段，raw 仍是权威数据； +- Kernel 必须唯一落入 EfDrain、WinnerBuild、AllocComplete 或 FinalDrain，越界、多重归属、孤儿 Kernel 或 dropped 非零都拒绝结果。 + +阶段性 b256： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + pa_scheduler_swimlane_20260719_114815_617346/ccec/ +``` + +该轮首末 Submit 为 5.360061 ms，raw 839526 条、`dropped=0`；raw/merged 分别约 55.79/88.78 MB，全部父子关系和整数闭合通过。 + +#### 4.6.2 standalone winner 冷分支 + +**[已保留] `d2d8ce25` — `优化(a5): 将低频winner调整为冷分支`** + +每个 task 只有一个 winner，绝大多数 worker 都走 loser。给 Alloc 和普通 Submit 的重型 winner 分支增加低概率布局提示，不移动边界、不改变协议。b256 同观察口径： + +| 指标 | 基线 | 候选 | 变化 | +| --- | ---: | ---: | ---: | +| 全局首末 Submit | 5.360061 ms | 5.278401 ms | -1.52% | +| Submit 尾部未覆盖时间 | 41008786 cycles | 27155661 cycles | -33.78% | +| 完整逐核 Submit 区间累计 | 500448909 cycles | 483335683 cycles | -3.42% | + +关闭泳道后又做候选—基线—候选 ABA，每组五轮：候选中位数分别为 3.665017/3.715385 ms，基线为 3.988115 ms，分别快 8.10%/6.84%。同时记录了 `.text` 体积增长，避免只看速度不看取指代价。候选泳道： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + pa_scheduler_swimlane_20260719_123520_660296/ccec/ +``` + +#### 4.6.3 真实 PA loser 热路布局恢复 + +**[已保留] `cafa9ca5` — `优化(a5): 恢复真实PA的loser热路布局`** + +真实 PA 对等标记两处低频 winner 分支。当前 atomic 观察版基线三轮中位数从 5.631038 ms 降为 5.192087 ms，减少 0.438951 ms，约 7.80%。但历史 pre-atomic 中位数已经是 5.115620 ms，因此这一步的准确含义是： + +> 恢复 atomic 观察代码接入后发生的热路布局回退，而不是在旧 5.1 ms 基线上新增 7.8% 业务收益。 + +#### 4.6.4 外提 atomic 观察冷路径 + +**[已保留] `44367971` — `优化(a5): 外提atomic泳道冷路径消除取指回退`** + +分两步推进： + +1. 将 direct Atomic 的 record 发布外提为设备端共享 `noinline` 冷函数，保留 `begin -> atomic -> end` 在原 wrapper；三轮中位数为 5.096506 ms； +2. 保留 PollBatch 的内联 level 快速门，只把 level-4 命中后的十类遍历和落盘外提到共享 slow 函数。 + +最终代码尺寸变化： + +| 产物 | winner 冷路后 | atomic 冷路径外提后 | +| --- | ---: | ---: | +| AIC/AIV `dist_engine .text` | 347536 / 357112 B | 66768 / 67120 B | +| AIC/AIV `dist_submit_impl` | 100860 / 103676 B | 18812 / 18872 B | + +真实 A5 level-1 正式三轮： + +```text +4.821897 / 4.890447 / 4.752956 ms +中位数 4.821897 ms +``` + +相对 5.192087 ms 再下降 7.13%。该结论只能归为“消除未执行诊断代码的大量复制和热路布局回退”；当时没有同时采集专用 I-cache PMU，不能继续写成确定的 miss 降幅。 + +level-4 能力复核： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260719_135629/ + +Atomic 物理记录闭合：107608 = 115309 - 8056 + 355 +dropped_records = 0 +``` + +`14c2429f` 随后把 I-cache、代码布局和 PMU 经验集中到 I-cache 指南，PA atomic 文档只保留 atomic 语义、次数和边界。 + +#### 4.6.5 排他泳道迁入真实 FDWIC + +**[观察工具] `dbb95bb5` — `Support: 将 Submit 排他泳道分析移植到 FDWIC`** + +真实路径获得与 standalone 同类的父区间、真实尾动作、离线未覆盖时间和整数闭合分析，Atomic、Kernel 等 overlay 不参与排他加和。`911ecf9a` 进一步固定各 span 的业务含义、AIC/AIV 分布和全核工作量与墙钟末端的区别。 + +### 4.7 compete-first eager + +#### 4.7.1 A/B/C 三份独立 standalone 对照 + +**[历史证据] `ba4334d1` — `验证(pa): 交付 compete-first/lazy 三版独立对照`** + +三版分别为： + +- A：原始 Materialize-first、Submit 外 eager 构参； +- B：EfDrain、Claim 前移，全体 worker 在 Claim 后同步 eager 构造完整参数； +- C：与 B 相同控制流，仅让 loser 跳过 input/scalar thunk。 + +72 次独立 host 启动中 A/B/C 各 24 个样本，均通过语义与后处理门禁。去异常的相邻配对结果： + +| 比较 | 配对变化 | 结论 | +| --- | ---: | --- | +| B 相对 A | -206.270 us，-5.214% | 22/22 有效块更快；收益是 compete-first、split/outlining 与布局的组合 | +| C 相对 B | +1.503 us，+0.040% | 双方各 11/22；低于 5% 门槛，不支持 lazy 收益或回退 | + +因此只推进 B 的 compete-first eager，不把 C 的 lazy 视为优化，也没有为 C 启动 I-cache PMU 对比。 + +#### 4.7.2 standalone 主路采用 compete-first eager + +**[已保留] `0d08c437` — `优化(pa): standalone采用compete-first eager提交流程`** + +当前时间线变为： + +```text +EfDrain -> Claim -> 同步 eager callback 构参 + -> Materialize -> PrepareMap + -> Fanin/Register -> WinnerBuild或AllocComplete -> Submit结束 +``` + +Claim 与 Materialize 之间的构参时间使用现有边界离线求差，不新增 raw 字段。关闭泳道的 b256 五轮中位数从 3889.180 us 降到 3735.032 us，减少 154.148 us，约 **3.9635%**。这证明收益在 standalone 主路复现，不承诺真实 PA 有同一比例。 + +compete-first 移植阶段的 standalone b1 历史布局证据： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + pa_scheduler_swimlane_20260720_092021_1729726/ccec/ + l2_swimlane_records.json + merged_swimlane.json + swimlane_exclusive_analysis.json +``` + +#### 4.7.3 真实 PA 接入 compete-first eager + +**[已保留] `2899cc35` — `重构(pa): 真实路径接入compete-first eager提交流程`** + +真实路径新增显式 begin/finish API 和 32 B 同步 ticket，同时保留旧 one-shot API 及原顺序，未把新语义强加给其他调用方。所有 worker 在 Claim 后仍完整构参，未采用 lazy 跳过。 + +真实 A5 level-1 三轮： + +| 路径 | 三轮 | 中位数 | +| --- | --- | ---: | +| compete-first 最终版 | 4.843652 / 4.809211 / 4.805443 ms | 4.809211 ms | +| 原路径历史基线 | 4.821897 / 4.890447 / 4.752956 ms | 4.821897 ms | + +中位数减少 12.686 us，约 0.263%，两组三轮波动区间重叠。因此当前结论是 **真实性能基本持平**，不是稳定的 0.263% 收益。保留该接口是因为阶段顺序和业务边界更清晰，并为后续精确取证提供基础。 + +真实 level-4 权威件： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260720_104406/ + l2_swimlane_records.json + merged_swimlane.json +``` + +该轮包含 122880 个 Submit、945653 条事件、`dropped_records=0`，首末 Submit 为 5.066862 ms；atomic 闭合为： + +```text +106355 = 109392 - 3361 + 324 +``` + +它证明 compete-first 后的阶段、atomic 和离线加工一致，不是关闭观察后的净性能样本。 + +## 5. 当前保留优化汇总 + +| 优化 | 真实 PA 状态 | 当前可成立的效果结论 | +| --- | --- | --- | +| 首个 joint 前跳过 BlockWon 轮询 | [已保留] | PA 单 lane Case1 确定删除 146944 次无效 RMW；与参数 mask 的组合将 5.642245 ms 降至 5.115620 ms 中位数 | +| output/register mask 复用 | [已保留] | 组合结果正向；独立收益只具方向性，不作精确拆分 | +| HeapGuard 首圈 fast path | [已保留] | 确定删除 1024 次 frontier atomic load；真实十对配对中位 -0.324% | +| 低频 winner 冷分支 | [已保留] | 主要恢复 atomic 观察接入后的 loser 热路布局回退 | +| atomic record/PollBatch 冷路径外提 | [已保留] | 大幅缩小热函数和 `.text`，level-1 三轮中位恢复到 4.821897 ms，同时保持 level-4 闭合 | +| compete-first eager begin/finish | [已保留] | standalone -3.9635%；真实 PA 三轮只能判为基本持平 | + +这些结果来自不同历史阶段，不能把表中百分比相加得到“总收益”。当前真实路径已经同时包含这些改动，后续基线必须从当前 HEAD 重新建立。 + +## 6. 当前观察能力 + +### 6.1 真实 PA `swimlane`：业务 span 与 atomic 合并观察 + +**[观察工具，已具备]** + +- 普通业务阶段和 Atomic/PollBatch 位于同一 AIC/AIV scalar lane； +- schema-v4 以父区间和互斥子区间闭合，Kernel、Atomic 是不可加和 overlay； +- residual/未覆盖时间由离线工具使用已有相邻边界计算，不新增设备 record； +- raw 是权威数据，merged 只负责 Perfetto 可视化，exclusive analysis 负责整数闭合； +- `dropped_records != 0`、父子越界、Kernel 孤儿或 atomic 公式不闭合时，整轮无效。 + +`swimlane` 用于回答“时间落在哪个业务区域、atomic 调用次数和边界是否改变”，不作为无观察性能基线，也不直接给出 I-cache stall。 + +### 6.2 standalone `submit-pmu`：历史能力 + +**[观察工具，历史版本已具备；不是当前待办]** + +现有历史版本能在独立 CCEC ELF 中编译掉泳道和逐 atomic 记录，采集每物理子核 PMU total、scalar busy、I-cache request/miss，并生成 96 核 raw 和自包含 HTML。历史 `none/claim/efdrain/materialize/register` 数据只对当时边界和各自 ELF 有效。 + +历史 b256 `none` 一轮记录： + +| 指标 | AIC 每核 | AIV 每核 | +| --- | ---: | ---: | +| request | 408317.344 | 422480.609 | +| miss | 38664.344 | 55098.625 | +| 加权 miss/request | 9.4692% | 13.0417% | +| PMU total 等效时间 | 4527.942 us | 4294.748 us | +| scalar busy 等效时间 | 3602.744 us | 3396.667 us | + +历史产物： + +```text +tests/atomic_probe/pa_scheduler/outputs/ + submit_pmu_none_20260719_b256_final/ + submit_icache_raw.json + submit_icache_report.html +``` + +该目录可能不在当前机器保留，且从不随 Git 提交。上述数据只用于说明已经验证过的 PMU owner、逐核 raw、AIC/AIV 分组和 HTML 方法，不是当前重采 standalone 的要求，也不能把旧绝对值当成真实 PA 数据。当前工作只在真实 PA 上按当前 compete-first 代码和最新真实 span 建立新证据。 + +### 6.3 A5 当前不可获得的 `scalar_wait_ib_time` + +**[受限]** + +在本机 CANN 9.1、A5/DAV3510 上分别尝试 `PipeUtilization`、`PipeUtilization,MemoryDetail` 和 `Default` 三种正式 `msopprof` 入口，生成的 CSV 和 A5 正式事件表均没有 `scalar_wait_ib_time` 或 `scalar_wait_time`。当前不能套用 A2/A3 的事件号或字段含义。 + +对应历史目录： + +```text +tests/atomic_probe/pa_scheduler/outputs/wait_ib_official_msopprof_20260719_b1_probe2/ +tests/atomic_probe/pa_scheduler/outputs/wait_ib_official_msopprof_20260719_b1_probe3_memory_detail/ +tests/atomic_probe/pa_scheduler/outputs/wait_ib_official_msopprof_20260719_b1_probe4_default/ +``` + +因此 `PMU total - scalar_busy` 只能叫“非 Scalar-busy 残余”，不能命名为 scalar 空闲、wait vector 或 I-cache stall。 + +## 7. standalone 历史证据与真实 PA 的边界 + +**[历史证据，不是当前待办]** + +standalone 已完成其方法验证职责：证明多后端调度模型、真 Cube/Vector 负载、atomic 泳道、PMU owner、I-cache 标尺和排他区间工具可以工作。当前不再继续优化、扩充或重采 standalone；下列对等关系只用于解释为什么历史经验可以作为真实 PA 实施时的参考，以及哪些结论绝不能外推。 + +### 7.1 已对等部分 + +- 256 batch、Alloc/QK/SF/PV/UP 五 task 拓扑； +- 32 AIC + 64 AIV、每核完整回放 1280 次 Submit； +- 四分片 Claim 和固定 73728 次 Claim atomicMax； +- TaskArgs、Tensor、TaskPayload、DistSubmitCtx 的关键布局和 tag 扫描； +- TensorMap materialize、retire、lookup、insert、register mask； +- fanin、winner/loser、私有 ring slot、WaitForSlot、HeapGuard； +- completion flag、vend、frontier 和最终 drain； +- QK/PV 真 Cube、SF/UP 真 Vector 的受控计算工作量； +- 普通阶段、Atomic/PollBatch、Kernel placement 和最终状态闭合。 + +依赖不是按 task 名硬编码跳过：SF 依赖 QK，PV 依赖 SF，UP 去重后依赖 Alloc、SF、PV；每 batch fanin 边数为 5，b256 全局为 1280。 + +### 7.2 尚未对等部分 + +- 真计算 workspace 使用统一受控输入，数值没有按真实 QK→SF→PV→UP 数据流串接； +- 该历史模型只覆盖 Case1 单 block group、`q_loop=1` 和全单-lane 图； +- joint/mixed、多 group、多 q-loop 和跨迭代更新没有被完整模拟； +- synthetic heap、独立 ELF 布局和 host 启动状态与真实 simpler 不同； +- Kernel span 包含 engine launch/completion wait wrapper，不等于纯 Cube/Vector 指令时间； +- standalone 没有真实 PA 的 loser replay 业务动作，不应为追求图形对称而伪造该 span。 + +历史推进中，standalone 曾用于先验证接口、边界、计数、控制协议和候选方向，再迁真实 PA 做同构正确性与性能 A/B；它的收益比例始终不能直接外推。当前这些方法能力已经完成验证，后续真实 PA 三证据链不再把新增 standalone 实现或复测设为前置门禁。 + +## 8. 当前真实 PA 的三条互不混算证据链 + +### 8.1 `perf-clock` + +**[观察工具，已实现]** + +该阶段已经建立真实 PA 的权威低扰动性能基线。最终构建只额外定义： + +```text +PTO_FDWIC_PERF_CLOCK=1 +PTO_FDWIC_TRACE_ENABLED=0 +``` + +实现口径为： + +- **保留 `PTO2_PROFILING` 及其拥有的公开 Arg 布局/ABI**； +- 编译期去除 FDWIC 普通泳道、逐 atomic 观察和平台 PMU 路径； +- 每核只保留第一个 Submit 起点、最后一个 Submit 终点和 Submit 次数； +- 不为每个 Submit 写 record； +- 继续复用现有 host/device header 传输，但 device header 固定只有 **6976 B**，`records_per_core=0`，没有逐事件记录； +- host hook 随该构建重新编译并导出，不复用普通诊断构建的旧 host 产物； +- 候选保留或撤回最终由该构建决定。 + +第一版曾尝试设置 `PTO2_PROFILING=0`，在编译期触发 Arg 布局/cacheline `static_assert`，没有进入设备执行。该尝试已经完整撤回；这证明 `PTO2_PROFILING` 不只是可随意关闭的观察开关，不能为了减少诊断代码破坏公开 ABI。最终方案只关闭上述 FDWIC 观察路径，不能表述为“所有 PTO2 profiling 已移除”。 + +构建身份也已做双向 ELF 审计：perf ELF 含 `dist_perf_clock_expect_submits` 标记，并且不含 FDWIC swimlane、atomic 观察和平台 PMU 符号；普通 level-4 ELF 含正常泳道/atomic 符号，但不含 perf-clock 设备符号。普通构建还做了前后布局复核：旧 AICore cache 身份 `6c55004bc91e15f0` 与新 AICore cache 身份 `110ff0c62a3adcf7` 的 `.text` 均为 `0x31a50` B，两个 ELF 中 `.text` 的 96 条 FUNC symbol 记录在地址、尺寸、绑定、可见性和名称上完全一致；`.text` 仅有 AIC/AIV 两份 `aicpu_orchestration_entry` 各一处单字节从 `0xa2` 变为 `0xa9`，对应 `PTO2_SCOPE` 源码行号从 162 移到 169，原因是前置新增 7 行。没有观察到普通构建新增函数或代码尺寸膨胀；这里也不声称工具已经给出完整指令反汇编一致性。 + +在完成第 2.2 节环境准备后，最小复现命令为： + +```bash +source /home/q00473782/.venv/bin/activate +python -m pytest examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py \ + --platform a5 --case CaseB1 --manual include \ + --fdwic-profile perf-clock --rounds 1 -s -v + +python -m pytest examples/a5/fully_distributed_within_core/paged_attention_unroll/test_paged_attention_unroll.py \ + --platform a5 --case Case1 \ + --fdwic-profile perf-clock --rounds 1 -s -v +``` + +该 profile 必须独占：不要同时传 `--enable-l2-swimlane`、`--enable-pmu`、`--use-example-exec-time` 或其他诊断开关；命令行门禁会直接拒绝混用。每次只允许 `--rounds 1`，多轮基线必须由独立 pytest 进程取得。该 profile 会按源码指纹自动重编 AICore override，但不会代替安装流程重编 host runtime；新环境首次复现前必须先重建 `libhost_runtime.so`，并确认三个 `fdwic_perf_clock_host_*` hook 已导出。 + +最小 B1 两次有效 A5 闭合如下；两次都是 96 个物理子核、每核恰好 5 次 Submit： + +| 产物时间戳 | 完整 Submit | +| --- | ---: | +| `20260720_172436` | 75.347 us | +| `20260720_172615` | 73.716 us | + +Case1 golden 在 `20260720_172820` 通过，96 核均为 1280 Submit，原始 `SYS_CNT` 为 **4,489,247 ticks = 4489.247 us**。JSON 浮点输出采用默认 6 位有效数字，因此 Case1 量级显示为 `4489.25`，而 B1 仍可显示 `73.716`；后续精确分析应以 raw tick 除以 1000 为准。 + +Case1 golden 通过后，另起五个独立进程并使用 `--skip-golden` 得到本阶段干净基线；**以下五次不包含上述 golden 样本**： + +| 产物时间戳 | raw ticks | 完整 Submit | +| --- | ---: | ---: | +| `20260720_173140` | 4,495,677 | 4495.677 us | +| `20260720_173224` | 5,808,500 | 5808.500 us | +| `20260720_173307` | 5,342,774 | 5342.774 us | +| `20260720_173350` | 4,752,765 | 4752.765 us | +| `20260720_173433` | 4,823,114 | 4823.114 us | + +五次均为 96 核、每核 1280 Submit；中位数 **4823.114 us**，最小值 **4495.677 us**，最大值 **5808.500 us**。该分布是后续候选做独立进程、交错 A/B 的起点，不能只挑 4489 us 的最好值作为稳定基线。 + +还完成两项边界门禁： + +- Case2 负测试实际得到每核 576 Submit，而当前期望值为 320；host 按 fail-closed 拒绝结果且不生成成功 summary。它只证明计数不符时不会产出伪成功结论，不能用于评价 Case2 性能； +- 同一真实源码的普通 level-4 B1（`20260720_173738`）得到 480 个 Submit、完整 Submit **85.653 us**、`dropped=0`，排他闭合 `PASS`。它证明 perf-clock 的首尾边界和逐核计数与普通泳道来自同一执行语义；由于两者是不同 ELF，不能用 `85.653 - 73.716` 计算观察开销。 + +本阶段的源码、构建身份、B1、Case1、负测试和普通泳道同源边界已经闭合，并已完成独立源码审阅。后续进入真实 `swimlane` 构建复核与 `submit-pmu-none`，不在该工具阶段顺带铺开 PMU 代码。 + +### 8.2 `swimlane` + +**[观察工具，已实现]** + +保留普通阶段和 atomic 合并泳道，用于回答： + +- 收益或回退可能落在哪个业务 span； +- atomic 逻辑调用、物理记录和 PollBatch 是否变化； +- Kernel 落点、父子区间、逐核 task 连续性和记录容量是否正常。 + +它不决定候选的净性能，不与 perf-clock 的绝对时间相减。 + +本阶段没有调整设备端 span、atomic wrapper、raw ABI 或记录容量。审查发现原先 SceneTest 在 converter/analyzer 返回失败时只记录 warning，pytest 仍可能显示 PASS；而阶段序列、Kernel 唯一归属和六类整数闭合正是在该离线步骤中完成。现已将真实 A5 FDWIC level-4 成功用例改为 fail-closed：raw 缺失、converter 失败、`merged_swimlane.json` 或 `swimlane_exclusive_analysis.json` 缺失/为空都会使该用例失败；若设备执行本身已经失败，则保留原始异常，离线转换不覆盖根因。 + +普通 trace-capable AICore ELF 也增加了正向身份门禁：必须含 `fdwic_atomic_poll_boundary_slow` 与 `fdwic_swimlane_detail_record_atomic` 两个已定义观察慢体，并且不得含 `dist_perf_clock_expect_submits`。这没有新增一套等价的 swimlane profile：普通 level-0 与 level-4 仍共享同一个 trace-capable ELF，采集模式由运行时 level 决定；门禁只防止误拿 perf-clock 或不完整产物。raw 中的 `trace_schema_version=4`、`l2_swimlane_level=4` 和 atomic 元数据继续证明运行模式。 + +当前源码先用真实 B1 验证结构门禁： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260720_234158/ +``` + +该轮为 96 核、每核 5 个 Submit、4,559 条记录、`dropped=0`，所有父子关系、Kernel 归属和整数闭合均 PASS。它的完整 Submit 为 302.072 us，明显受本轮冷启动/轮询状态影响，只作为结构门禁，不替代 perf-clock 性能基线。 + +随后只运行一次当前 HEAD 的完整 Case1 并保留 golden 校验，权威第二证据链产物为： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260720_234305/ + l2_swimlane_records.json 75,397,613 B + merged_swimlane.json 182,110,972 B + swimlane_exclusive_analysis.json 121,264 B +``` + +该轮为 32 AIC + 64 AIV、每核 1,280 个 Submit、全局 122,880 个 Submit，944,874 条 raw 记录且 `dropped=0`；全局首末 Submit 为 **5,095.821 us**。Submit、Submit envelope、EfDrain、OrchestrationReplay、FinalDrain 和 WorkerCompletion 六类整数分区全部精确闭合。Atomic 物理记录、批处理轮询与逻辑调用满足： + +```text +105577 - 330 + 3899 = 109146 +``` + +其中 merged 中 `return_ready/source_issue/PollBatch` 分别为 102,495/2,752/330。Atomic 仍是不可加的 overlay；PollBatch 表示完整等待区及其精确调用次数，不能用 duration 反推单次 atomic 延迟。 + +本轮设备端仍预留固定 **201,333,568 B** trace buffer。实际 raw 没有再增加字段，但该固定容量和约 182 MB merged 进一步说明：swimlane 只在需要业务/atomic 定位时采集，不能作为权威性能基线，也不应为普通 A/B 反复生成。生产 converter/analyzer 回归与新增 fail-closed/ELF 门禁共 86 项通过。 + +### 8.3 `submit-pmu-none` 与真实 span 单阶段 PMU + +**[`submit-pmu-none`、`arg-build`、空 bracket 校准、`materialize`、`claim` 和 `register` 均已完成 A5 收口]** + +本阶段没有修改或复测 standalone，而是在真实 PA 中建立独立诊断构建。该构建在编译期去除普通泳道、atomic 观察和通用逐 task PMU ring，分为两种运行方式： + +1. **`submit-pmu-none`**：每物理子核在完整 Submit 调度期只 start/stop 一次，不做中途 shadow read-clear；输出 96 核 PMU total、scalar busy、I-cache request/miss，并按 AIC/AIV 生成 raw 与 HTML； +2. **真实 span 单阶段 PMU**：一次 ELF 只选择一个当前真实泳道区域做局部观测，仍同时保留本 ELF 自己的完整 Submit primary，局部只与本 ELF、本轮、本角色的 primary 和时间分母比较。首个 selector 为 Claim 完成到 Materialize 入口之间的 `arg-build`。 + +#### 8.3.1 已闭合的 `submit-pmu-none` + +入口为 `--fdwic-profile submit-pmu-none`，当前只接受真实 A5、FDWIC、level 2、`rounds=1`。构建保留公开 `PTO2_PROFILING` Arg ABI，但固定 `PTO_FDWIC_TRACE_ENABLED=0`、`PTO_FDWIC_SUBMIT_PMU=1`；最终 ELF 必须包含 `dist_submit_pmu_expect_submits` 和 `fdwic_submit_pmu_read_counters`，并拒绝 perf-clock、swimlane/atomic、通用 PMU ring 和通用 PMU reg-base 符号。 + +每个物理子核在 attach 时先 stop/清计数；首个 Submit 读取 1 ns `SYS_CNT` 后开启 PMU，末个 Submit stop 后读取 CNT2/CNT6/CNT7，并用 CNT8/CNT5 做 request/miss 影子复核。FinalDrain 不进入有效窗口。AICPU owner 在发布 worker 运行状态前，逐核保存并配置 PMU 控制寄存器和 selector；96 个 worker 完成后逆序恢复。正式 raw 要求 32 AIC、64 AIV、96 个唯一物理 ID、32 个完整 1:2 mixed triplet、配置/恢复 96/96、active-after-restore=0、每核 Submit 次数和窗口状态全部闭合。 + +实现过程中有两次由门禁揭示并修正的接口问题： + +1. PA orchestration 原先只在 `PTO_FDWIC_PERF_CLOCK` 条件下声明预期 Submit 数，导致 submit-PMU 首轮实测为 count `5/0`、窗口未启动；修正为 perf-clock 与 submit-PMU 复用同一真实挂点，不在 host 猜测次数。 +2. host `Runtime::workers[].physical_core_id` 是 H2D 前的 host shadow，不会在 export 前从设备 Runtime 整块回拷；真机核 1 已实证 device record 为 physical 1、host shadow 仍为 0。该无效比较已移除。逻辑核到物理核的可信关系由设备 AICPU owner 校验、每核 record、唯一集合、角色和 triplet 三层闭合，不新增冗余映射字段。 + +B1 两次独立成功采集都通过 96 核、5 Submit/core、owner restore 和报告门禁： + +| 产物 | 全局 Submit | AIC total/core mean | AIC scalar/core mean | AIV miss/core mean | +| --- | ---: | ---: | ---: | ---: | +| `..._002939` | 74.882 us | 28,778.9 cycles | 25,099.7 cycles | 129.56 | +| `..._003050` | 227.673 us | 36,999.4 cycles | 25,318.8 cycles | 129.84 | + +第二轮的 AIC total 最大值从 122,226 增至 375,185 cycles，而 AIC scalar mean 仅增加约 0.87%；AIV request/miss 也基本不变。它说明“设备独占”并不等于每核到达相位和非 scalar-busy 等待恒定，是后续波动专项归因的输入，当前不把两点样本直接写成因果结论。 + +Case1/B256 正式收口结果为： + +| 指标 | AIC | AIV | +| --- | ---: | ---: | +| core 数 | 32 | 64 | +| PMU total/core mean | 7,436,193 cycles | 7,410,246 cycles | +| scalar busy/core mean | 6,479,673 cycles | 6,761,739 cycles | +| scalar busy / total | 87.14% | 91.25% | +| I-cache request/core mean | 646,963.94 | 594,542.61 | +| I-cache miss/core mean | 1,196.88 | 16,943.17 | +| 聚合 miss/request | 0.1850% | 2.8498% | + +全局首个 Submit 到最后一个 Submit 为 **5,075.360 us**，每核 1,280 次 Submit，primary/shadow 96/96 相等；最大可编程计数 7,129,295，远小于 `0x3fffffff` 风险阈值。按 90 ns/miss 只能得到单核串行等效量级，不能把 AIV 约 1.525 ms/core 直接写成可消除的墙钟损失。raw 约 46 KB、HTML 约 77 KB，没有复用约 200 MB 的泳道 buffer。 + +提交前还用当前工作树分别回归了 B1 perf-clock 与普通 level-4：perf-clock 为 96×5 Submit、254.084 us；level-4 为 4,549 条事件、89.071 us、`dropped=0`，排他闭合 PASS。两者用于证明三种 ELF 的双向隔离和公共 Submit 挂点没有回退，绝对时间仍不得跨 ELF 相减。 + +#### 8.3.2 首个真实 selector：`submit-pmu-arg-build` + +入口为 `--fdwic-profile submit-pmu-arg-build`，编译期固定： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=1 +PTO_FDWIC_TRACE_ENABLED=0 +``` + +Kernel 与 Alloc 两条 compete-first 路径都使用同一组源码业务边界：起点在 Claim 完成后，终点在匹配 Finish 恢复并校验 ticket 后、Materialize 入口前。该区间覆盖 `dist_submit_make_ticket()`、Begin 返回、同步 eager callback 的 `build_args(args)`、Finish 重入、`dist_submit_restore_from_ticket()` 和 ticket 校验；不包含 Claim 本体和 Materialize 本体。它复用了泳道 `Claim.end -> Materialize.begin` 的业务语义，但 PMU ELF 已编译掉泳道 record，二者不是同一 ELF，也不做逐 tick 对齐。 + +`submit-pmu-none` 保持原 128 B 前缀加 `96 × 64 B` 整窗记录，共 6,272 B；`arg-build` 在其后追加 `96 × 64 B` phase sidecar，总计 12,416 B。每个 worker 仍独占 cacheline，只在窗口结束后发布一份汇总，不生成逐事件记录，也不复用约 200 MB 的泳道 buffer。CNT6/CNT7 是窗口中从不读取的整窗 primary；CNT8/CNT5 作为 running read-clear shadow，begin 样本只进入 shadow whole，end 样本同时进入 shadow whole 和 phase observed，stop 后 tail 只进入 shadow whole。 + +这里特意不把 phase request/miss 称为业务事件数的“严格下界”。counter read 与 `SYS_CNT` 边界之间仍有少量观测 bookkeeping，它们的取指会进入 observed sample；`primary - shadow` 只量化分段重建的 capture gap，不能抵消插桩自身的事件。报告因此同时展示 `observed` 与 `observed + (primary - shadow)`，二者是当前插桩 ELF 的观测值和加全窗 capture gap 后的敏感性量尺，不是原业务区间的数学上下界。phase 只提供 `SYS_CNT` 时间及 I-cache request/miss observed，不杜撰局部 PMU total、scalar busy 或 I-cache stall 时间。 + +正式 raw 只有在逐核满足以下条件时发布： + +- `phase_begin_reads == phase_end_reads == expected_submit_count`；B1 为 5，Case1 为 1,280； +- phase status 为 `0x3f`，即 requested、边界平衡、调用 shape、数值顺序、时间落在本核 Submit 内和 tail read 六项全部成立； +- phase observed 分别不超过对应 shadow，shadow 不超过 primary；本阶段实测 primary/shadow 精确相等，但通用契约仍只要求单向闭合； +- 可编程计数与每段最大读数都低于 `0x3fffffff` 风险阈值； +- phase 时间非零且不超过同核 `submit_elapsed_ticks`。 + +两轮独立 B1 结构样本分别位于： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_014154/ +outputs/TestPagedAttentionUnroll_CaseB1_20260721_014301/ +``` + +两轮均为 96 核、每核 5 次 begin/end、status `0x3f`、primary/shadow 96/96 精确相等；全局 Submit 分别为 247.205 us 和 80.702 us，phase core-time 份额分别为 6.862% 和 7.432%。这两点证明结构闭合，也再次暴露独占设备仍有明显到达/等待波动；不能拿两轮绝对时间直接形成性能结论。 + +Case1/B256 正式件为： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_014355/ + fdwic_submit_pmu_raw.json 72,811 B + fdwic_submit_pmu_report.html 79,497 B +``` + +该轮全局 Submit 为 **4,964.039 us**，96 核每核 1,280 次，共 122,880 次 begin/end 全部闭合，primary/shadow 96/96 精确相等，最大 shadow request/miss 分段为 32,136/129。按同一 ELF、同一轮聚合，`arg-build` 的 core-time、request observed、miss observed 份额分别为： + +| 角色 | core-time | request observed | miss observed | +| --- | ---: | ---: | ---: | +| ALL | 5.557% | 20.716% | 21.334% | +| AIC | 4.297% | 20.099% | 10.640% | +| AIV | 6.183% | 21.031% | 21.417% | + +这组数据说明 `arg-build` 在当前插桩 ELF 中取指观测份额高于时间份额，值得在完成空 bracket 校准后继续判断观察 bookkeeping 占比；目前不能把约 21% 直接写成原业务 I-cache 事件比例，更不能把 4.964 ms 与 none、perf-clock 或泳道绝对时间相减。 + +最终还分别回归了三条互斥证据链：`submit-pmu-none` B1 `20260721_014602` 保持 96/96 primary=shadow 且不含任何 phase 字段；perf-clock B1 `20260721_014715` 为 96×5 Submit、73.029 us；普通 level-4 B1 `20260721_014841` 为 4,560 条事件、90.275 us、`dropped=0`、排他闭合 PASS。这些结果证明 phase sidecar、reader 和状态只进入选中的 `arg-build` ELF；三轮绝对时间仍不互相相减。 + +#### 8.3.3 running bracket 空区间校准：`submit-pmu-empty-bracket` + +**[观察工具，已完成两轮 B1 与两轮 Case1 实测闭合]** + +`arg-build` 的 running read-clear 会在每次阶段 begin/end 各读一次 shadow counter。在继续扩展业务 selector 前，本阶段先回答一个基础问题：同一套 begin/end 观察器紧邻执行、其中不包任何业务体时，会在真实 A5 PA 中形成怎样的稳定时间和 I-cache 记录开销量级。入口为 `--fdwic-profile submit-pmu-empty-bracket`，编译身份固定为： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=2 +PTO_FDWIC_TRACE_ENABLED=0 +``` + +host/runtime 契约使用独立 mode `3`、phase id `2`，phase 名称和边界分别为 `empty-bracket`、`claim_end_adjacent_empty_bracket`。真实 hook 没有另造调用拓扑：Kernel 与 Alloc 的 compete-first Begin 都在现有 Claim 完成、`claim_end` 已取到的同一源码调用点，紧邻执行一次原样 generic phase begin/end；每个 Submit 恰好一对，B1 每核 5 对，Case1 每核 1,280 对。其他 profile 中该 wrapper 编译为空。 + +时间和 PMU 计数必须按两种不同边界解释： + +- generic begin 在第一次 shadow read-clear 与 begin bookkeeping 之后读取内部 `SYS_CNT`，generic end 在第二次 shadow read-clear 之前读取内部 `SYS_CNT`；若只看这两个内部 tick，空区间并没有覆盖完整的观察器调用成本； +- empty wrapper 因此保存旧累计值和 begin/end 次数，用外层两个 `SYS_CNT` 包住完整 generic begin/end 对，校验次数各恰好增加 1、状态平衡且时间无溢出，再用外层 delta 覆盖本次内部 elapsed 增量。raw 明示 `time_semantics=outer_sys_cnt_around_adjacent_begin_end_pair`； +- request/miss 仍保持原 running read-clear 口径：begin 样本只进入 whole shadow，end 样本同时进入 whole shadow 与 phase observed。外层 `SYS_CNT` 不改变这组读清边界，raw 明示 `counter_semantics=running_read_clear_empty_bracket_calibration`。 + +所以 elapsed 是“外层时间戳包住完整相邻 begin/end 对”的经验耗时，request/miss 是“两次 shadow read-clear 之间”的观察值；二者不是同一个精确指令边界。外层时间也包含两个 `SYS_CNT` 自身的底噪，不能称为观察器数学最小成本。 + +该 profile 复用 `arg-build` 已有的 phase sidecar，没有增加设备 raw 字段、逐事件 record 或泳道 ring。`submit-pmu-none` 仍为 128 B header 加 `96 × 64 B` whole record，共 6,272 B；`empty-bracket` 与其他单阶段 profile 同为再追加 `96 × 64 B` sidecar，共 12,416 B。每个 worker 最终仍只发布一份 whole 汇总和一份 phase 汇总，因此本次校准没有继续放大约 200 MB 的泳道数据。 + +正式发布继续 fail-closed：96 核必须满足 owner 配置/恢复、唯一物理核与 32 个 mixed triplet 闭合；逐核 begin/end 次数都等于预期 Submit 数，phase status 为 `0x3f`，边界平衡、shape、数值顺序、时间落在本核 Submit 内和 tail read 全部成立；phase observed 不超过 shadow，shadow 不超过 primary，计数不越风险阈值。以下四轮均为 32 AIC + 64 AIV、status `0x3f`、primary/shadow 96/96 精确相等，因而 capture gap 为 0。 + +两轮 B1 先用于结构与冷启动观察： + +| 产物 | 全局 Submit | ALL 每对 elapsed | ALL 每对 request | ALL 每对 miss | +| --- | ---: | ---: | ---: | ---: | +| `..._020932` | 257.430 us | 725.192 ns | 84.844 | 1.956 | +| `..._021100` | 303.032 us | 719.304 ns | 69.194 | 2.444 | + +两轮完整 Case1 用于稳态经验尺度： + +| 产物 | 全局 Submit | 角色 | 每对 elapsed | 每对 request | 每对 miss | +| --- | ---: | --- | ---: | ---: | ---: | +| `..._021158` | 4,972.718 us | ALL | 640.465 ns | 49.340 | 1.342 | +| 同上 | 同上 | AIC | 567.962 ns | 48.919 | 0.008 | +| 同上 | 同上 | AIV | 676.717 ns | 49.550 | 2.009 | +| `..._021311` | 4,866.126 us | ALL | 639.272 ns | 49.337 | 1.356 | +| 同上 | 同上 | AIC | 567.619 ns | 48.870 | 0.008 | +| 同上 | 同上 | AIV | 675.099 ns | 49.570 | 2.030 | + +B1 只有每核 5 对，首次进入 reader、对应调用点和相关代码布局时的冷取指占比很高；其 request 为 69.194～84.844/对，明显高于 Case1 稳定的约 49.34/对，elapsed 也从 Case1 的约 639～640 ns/对升至约 719～725 ns/对。因此 B1 继续只作为结构、次数和冷启动门禁，不能替代 Case1 的稳态观察尺度。 + +Case1 两轮还复现了稳定的角色差异：AIC 约 568 ns、48.9 request、0.008 miss/对，AIV 约 675～677 ns、49.6 request、2.01～2.03 miss/对。当前证据只证明同一观察实现对 AIC/AIV 形成不同且可复验的记录开销；它没有证明差异必然来自 reader 跨 I-cache line、某个固定冲突或 PMU 事件定义，后续若归因必须另做同构单变量证据。 + +为决定是否应在本阶段调整 reader，另对 empty 最终 ELF 做了只读核验。AIC/AIV 的 `fdwic_submit_pmu_phase_read_shadow_counters` 都只有一份、均为 92 B；两个 relocatable object 中该函数机器码逐字节相同，最终 ELF 仅因 block-local relocation 出现一个立即数字节差异。128 B line 下，AIC reader 起址行内偏移 76 B，AIV 为 120 B，两者都跨两行，所以“跨行”不能单独解释只有 AIV 约 2 miss/对。本机 CANN 9.1 的 DAV3510 模型配置显示 scalar I-cache 均为 4-way，但 AIC 为 32 KiB/64 sets、AIV 为 16 KiB/32 sets；对应 combined `.text` 又分别为 68,024 B 和 82,000 B。这些证据支持容量、角色代码和具体布局共同形成不同的记录开销，但聚合 PMU 仍不能定位到 reader 的某一条 cache line。因此本阶段保留单份 noinline reader：不为追求较小数字而 inline 复制热路径，也不以强制对齐改变整份诊断 ELF 的冲突集合。若以后局部信号确实被该量级淹没，应另做只改 reader 对齐的 empty A/B，而不是混入本次校准提交。 + +该校准不能直接从 `arg-build` 中扣除。首先，两者的时间边界不同：`arg-build` elapsed 是两侧 observer 之间的内部业务区间，而 empty elapsed 用外层 tick 包住完整 begin/end 对；二者相减会把不同对象当成同一加法模型。其次，即使 request/miss 都来自 running read-clear，两个 profile 仍是不同 ELF，代码布局、冷暖状态和前端竞争都可能改变事件数。方向上，Case1 empty 约 49 request/对，相当于同一时期 `arg-build` AIC/AIV 约 118.31/121.36 request/对的四成；AIV empty 约 2.02 miss/对，相当于 `arg-build` 约 3.80 miss/对的一半。这只说明观察器污染不可忽略，不是允许产出“扣除 empty 后的业务净 request/miss”。报告和后续结论都只保留原始 observed、capture gap 与这份经验尺度。 + +空 bracket 代码落定后又在同一工作树上串行回归了四种互斥构建：`arg-build` B1 `20260721_021930` 为 96×5 对、status `0x3f`、primary/shadow 96/96 精确相等，并带有新增的 `time_semantics`，全局 Submit 为 249.064 us；`submit-pmu-none` B1 `20260721_022026` 不含 phase 字段、primary/shadow 96/96 精确相等，全局 Submit 为 265.977 us；perf-clock B1 `20260721_022115` 为 96×5 Submit、78.230 us；普通 level-4 B1 `20260721_022221` 为 4,546 条事件、88.595 us、`dropped=0`、排他闭合 PASS。四轮只证明新增 mode、元数据和公共 Claim.end 空 wrapper 没有破坏既有证据链，B1 冷启动绝对时间仍不得跨 ELF 相减。 + +#### 8.3.4 第二个真实业务 selector：`submit-pmu-materialize` + +**[观察工具，已完成两轮 B1 与两轮 Case1 实测闭合]** + +本阶段不是从历史 `claim/efdrain/materialize/register` 名单中顺次取一个旧 phase，而是重新查看当前 compete-first 真实 PA 的最新 Case1 排他结果 `outputs/TestPagedAttentionUnroll_Case1_20260720_234305/` 后再决定。该轮 `Materialize` 为 97,467,035 aggregate core-ticks，占 `SubmitUnion` 399,604,449 ticks 的 **24.391%**，是已具备明确源码起止边界的最大业务 span；同轮 `Claim`、`EfDrain`、`Register` 分别占 19.887%、15.853% 和 12.010%。已经完成的 `arg-build` 则覆盖 `Claim.end -> Materialize.begin`，二者首尾相接但不重叠。因此这一轮选择来自最新真实布局和可复用边界，不是复活 standalone 或旧 schema 中的 selector 顺序。 + +入口为 `--fdwic-profile submit-pmu-materialize`，编译身份固定为： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=3 +PTO_FDWIC_TRACE_ENABLED=0 +``` + +host/runtime 使用独立 mode `4`、phase id `3`，phase 名称和边界分别为 `materialize`、`materialize_begin_to_materialize_end`；counter/time 口径分别为 `running_read_clear_observed_bracket` 和 `inner_sys_cnt_between_boundary_observers`。这仍是一轮只打开一个业务 phase 的诊断 ELF，不同时采 Claim、EfDrain 或 Register。 + +设备端没有另造一条近似路径，而是在四个真实入口精确打开同一个 phase： + +1. 旧 Kernel `dist_submit_impl()` 在 EfDrain 完成、进入 Materialize 前 begin； +2. 旧 Alloc `dist_alloc_tensors()` 在相同业务边界 begin； +3. compete-first Kernel finish 在 ticket 恢复和校验成功、`materialize_begin` 取时后 begin； +4. compete-first Alloc finish 在相同业务边界 begin。 + +四条入口统一调用 `dist_submit_materialize_and_prepare_map()`；唯一成功 end 位于该 helper 内部，在 `dist_submit_check_task_cap()` 和 `dist_submit_materialize_args()` 均成功返回后、泳道 `materialize_end` 取时前。因此观测区间包含 task-cap 检查、tag/output/register-mask 扫描、heap ring 布局、输出 Tensor 初始化及 `heap_next` 推进，但不包含后继 `PrepareMap`。submit-PMU 构建已编译掉泳道 record，所以旧入口 begin 与 helper 之间的 trace 宏不会给本 ELF 增加一条实际记录。 + +失败路径刻意不伪造 end：task-cap 或参数、heap、输出物化任一检查失败时，helper 直接返回，遗留的 armed phase 会使 begin/end 不平衡、调用 shape 或最终 status 闭合失败；设备发布、host 校验和 HTML 加工据此 fail-closed，不能把一个被截断的 Materialize 当成有效短样本。正常 PA Case1 中 Materialize 每个 Submit 固定执行一次，所以正式 shape 必须严格为 96 核、每核 1,280 次 begin 和 1,280 次 end，全局各 122,880 次；B1 则固定为每核 5 次、全局各 480 次。 + +该 profile 复用已有 phase sidecar，没有扩展 raw ABI：`submit-pmu-none` 仍为 128 B header 加 `96 × 64 B` whole record，共 6,272 B；`materialize` 与 `arg-build/empty-bracket` 一样只再追加 `96 × 64 B` phase record，总计 12,416 B。没有增加逐 Submit 记录、泳道字段或约 200 MB 的 trace ring。 + +两轮 B1 先验证真实挂点、固定 shape 和冷启动下的数值闭合： + +| 产物 | 全局 Submit | ALL 每次 elapsed | ALL 每次 request | ALL 每次 miss | 同 ELF 时间/request/miss 占比 | +| --- | ---: | ---: | ---: | ---: | ---: | +| `..._024533` | 82.091 us | 912.846 ns | 333.183 | 8.225 | 20.432% / 45.075% / 21.321% | +| `..._024641` | 81.741 us | 896.398 ns | 267.810 | 7.369 | 20.932% / 42.404% / 20.485% | + +两轮都是 32 AIC + 64 AIV、每核 5 次 begin/end、phase status `0x3f`，96/96 primary/shadow 精确相等，最大单段 shadow request/miss 分别为 3,068/85 和 3,040/78。B1 仍只用于结构、次数、冷启动和快速门禁，其每次 request/miss 不外推 Case1 稳态。 + +两轮完整 Case1 产物为： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_024748/ +outputs/TestPagedAttentionUnroll_Case1_20260721_024909/ +``` + +两轮均为 96 核、每核 1,280 次 begin/end、phase status `0x3f`，owner 配置/恢复、32 个 mixed triplet、固定 shape、数值顺序、phase 时间和风险阈值全部闭合；primary/shadow 96/96 精确相等，capture gap 为 0。第一轮 raw/HTML 为 72,947/80,049 B，第二轮为 72,958/80,049 B。逐角色每次调用的原始 observed 为： + +| 产物 | 全局 Submit | 角色 | 每次 elapsed | 每次 request | 每次 miss | +| --- | ---: | --- | ---: | ---: | ---: | +| `..._024748` | 4,922.142 us | ALL | 797.814 ns | 233.197 | 1.474 | +| 同上 | 同上 | AIC | 776.265 ns | 228.942 | 0.022 | +| 同上 | 同上 | AIV | 808.588 ns | 235.325 | 2.201 | +| `..._024909` | 4,851.282 us | ALL | 797.061 ns | 233.238 | 1.418 | +| 同上 | 同上 | AIC | 775.653 ns | 228.170 | 0.021 | +| 同上 | 同上 | AIV | 807.765 ns | 235.772 | 2.116 | + +“阶段占比”只使用同一 ELF、同一轮、同一角色的数据：时间分子为 phase elapsed core-time、分母为逐核首末 Submit elapsed core-time；request/miss 分子为 phase observed、分母为本轮整窗 primary。两轮结果为： + +| 产物 | 角色 | 时间占比 | request observed 占比 | miss observed 占比 | +| --- | --- | ---: | ---: | ---: | +| `..._024748` | ALL | 22.560% | 37.688% | 12.056% | +| 同上 | AIC | 22.540% | 36.532% | 10.487% | +| 同上 | AIV | 22.570% | 38.278% | 12.065% | +| `..._024909` | ALL | 22.105% | 37.741% | 11.681% | +| 同上 | AIC | 21.882% | 36.442% | 10.997% | +| 同上 | AIV | 22.214% | 38.404% | 11.685% | + +两轮约 22.1%～22.6% 的同 ELF 时间份额和约 37.7% 的 request 份额可以说明 Materialize 是当前诊断布局中的重要取指区域；它们不能直接等同于关闭插桩后的净业务成本。尤其 empty-bracket 两轮 Case1 测得的记录开销量级约为 ALL 639～640 ns、49.34 request、1.34～1.36 miss/对，而 materialize 约为 797 ns、233.2 request、1.42～1.47 miss/次。empty elapsed 用外层 tick 包住完整 observer 对，materialize elapsed 是两个 observer 内侧的业务时间；request/miss 即使都来自 running read-clear，也属于不同 ELF、不同布局和不同缓存状态。故 empty 只能提示观察器的自扰动量级不可忽略，绝不能从 materialize 中相减得到“净时间”或“净 miss”。按角色看，AIV Materialize 为 2.20/2.12 miss/次，empty-bracket 为 2.01/2.03 miss/对，也仍处在同一量级；当前结果不能证明 Materialize 业务体带来了明确的 AIV miss 增量。AIC Materialize 只有约 0.022/0.021 miss/次，同样只保留原始观测，不作跨 ELF 扣减。 + +代码落定后串行回归了五类互斥 B1 构建： + +| 构建 | 产物 | 结果 | +| --- | --- | --- | +| `submit-pmu-arg-build` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_025027/` | 96×5、status `0x3f`、primary/shadow 96/96 精确相等；257.392 us | +| `submit-pmu-empty-bracket` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_025125/` | 96×5、status `0x3f`、primary/shadow 96/96 精确相等；230.313 us | +| `submit-pmu-none` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_025214/` | 无 phase 字段、primary/shadow 96/96 精确相等；298.298 us | +| `perf-clock` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_025305/` | 96×5 Submit、ELF 身份和调用 shape 闭合；274.997 us | +| 普通 level-4 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_025353/` | 4,550 条事件、89.109 us、`dropped=0`、排他整数闭合 PASS | + +这些回归只证明新增 mode、四个 begin 挂点和统一成功 end 没有破坏既有构建身份、phase 契约或泳道加工；B1 的跨核到达/等待波动很大，五种 ELF 的绝对时间仍不得互相相减。 + +最后将 AICPU header 校验从逐个枚举旧 phase mode 等价收敛为复用 `fdwic_submit_pmu_mode_has_phase()`，使 mode 判定与公共 phase/字节数契约只有一个事实来源。该重构没有修改计数器配置、业务边界或设备 ABI；最终源码再次运行 `submit-pmu-materialize` B1： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_025856/ +``` + +结果为全局 Submit **281.494 us**、96 核各 5 次，begin/end 全局 480/480、status `0x3f`、primary/shadow 96/96 精确相等。它是最终源码状态的 materialize 回归；前两轮 B1 仍保留为最初四挂点实现的独立结构样本。 + +#### 8.3.5 第三个真实业务 selector：`submit-pmu-claim` + +**[观察工具，已完成两轮 B1 与两轮 Case1 实测闭合]** + +本阶段继续按当前真实 Case1 排他布局选择观察对象，而不是复刻历史 phase 名单。`outputs/TestPagedAttentionUnroll_Case1_20260720_234305/` 中，`Claim` 为 79,470,788 aggregate core-ticks，占 `SubmitUnion` 399,604,449 ticks 的 **19.887%**；在 `Materialize` 完成取数后，它是剩余具备固定调用 shape 和明确源码边界的最大业务 span。同轮 122,880 条 Claim 与 122,880 条 Submit 一一对应，适合继续复用现有每 Submit 一对 begin/end 的强闭合契约。 + +入口为 `--fdwic-profile submit-pmu-claim`，编译身份固定为： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=4 +PTO_FDWIC_TRACE_ENABLED=0 +``` + +host/runtime 使用独立 mode `5`、phase id `4`，名称和边界为 `claim`、`claim_begin_to_claim_end`；counter/time 口径继续使用 `running_read_clear_observed_bracket` 和 `inner_sys_cnt_between_boundary_observers`。四条真实 API 路径分别在已有泳道 Claim 业务边界内打开和关闭同一个编译期 phase： + +1. 旧 Kernel `dist_submit_impl()`：`prepare_map_end` 后 begin，包围 `dist_submit_claim(Kernel)` 和 `claim_flags` 构造，在 `claim_end` 取时前 end； +2. 旧 Alloc `dist_alloc_tensors()`：`register_end` 后 begin，以相同方式包围 Alloc Claim，在 `claim_end` 前 end； +3. compete-first Kernel begin：`efdrain_end` 后 begin，先执行属于现有 Claim span 的 `dist_submit_check_task_cap()`，再执行短路后的 Kernel Claim 和 flags，最后 end； +4. compete-first Alloc begin：同样从 `efdrain_end` 开始，包含 Alloc task-cap、Claim 和 flags，最后 end。 + +compete-first 原代码虽在声明 `claim_begin` 前计算 `ready`，但泳道的 Claim 起点本来就是更早取得的 `efdrain_end`，所以 task-cap 已经计入 Claim span。本阶段只把该计算移到 phase begin 之后，使 PMU 与现有业务边界一致，没有改变 task-cap 与 Claim 的执行顺序。四条路径的 end 都放在 `claim_flags` 形成后、泳道 `claim_end` 取时前；submit-PMU ELF 中 trace record 已编译去除，不会把 Claim/前序 record 发布混入局部计数。 + +四条边界之间都没有会绕过 phase end 的直接返回。compete-first 的 task-cap 失败会让 `ready=false` 并短路真实 Claim，但仍正常构造 flags 和关闭 phase；后续 ticket、完整 Submit 和 golden 门禁负责拒绝无效执行。旧 API 若在 Materialize/PrepareMap 阶段已经失败，则不会进入 Claim，同时也无法闭合预期 Submit 窗口。Claim helper 内部对非目标角色或无效输入返回 false，同样会回到外层统一 end。因此 phase `0x3f` 证明边界、次数、数值顺序和时间闭合，不单独证明每次都发射了 Claim atomic、也不证明 winner 协议正确；后两者仍由业务 golden 和独立泳道/atomic 证据链负责。 + +正常 PA 无论本核角色是否参与该 task 的 atomic 竞争，每个 Submit 都进入一次外层 Claim 边界，所以固定 shape 不依赖 winner 或 `claim_attempted`：B1 必须为 96 核每核 5 次 begin/end、全局各 480 次；Case1 必须为每核 1,280 次、全局各 122,880 次。该规则继续直接复用 `expected_submit_count`，没有为 Claim 增加动态次数字段。 + +设备 ABI 也没有扩容：`submit-pmu-none` 仍为 128 B header 加 `96 × 64 B` whole record，共 6,272 B；Claim 复用同一 `96 × 64 B` phase sidecar，总计 12,416 B。没有逐 Claim record、task kind、winner 或 atomic 新字段。最终 Claim CCEC image 的 `.text` 为 154,192 B，其中 `dist_engine_aic.o/.text` 与 `dist_engine_aiv.o/.text` 分别为 54,960/55,096 B；ELF 正向包含 `dist_submit_pmu_expect_submits`、`fdwic_submit_pmu_read_counters` 和每角色一份 `fdwic_submit_pmu_phase_read_shadow_counters`，并拒绝 perf-clock、普通泳道/atomic 慢体和通用 PMU ring 符号。该尺寸只描述当前 Claim 诊断 ELF，不能与其他 profile 尺寸或时间机械相减。 + +两轮 B1 先验证四条边界、固定 shape 和数值门禁： + +| 产物 | 全局 Submit | ALL 每次 elapsed | ALL 每次 request | ALL 每次 miss | 同 ELF 时间/request/miss 占比 | +| --- | ---: | ---: | ---: | ---: | ---: | +| `..._031756` | 82.413 us | 1,070.163 ns | 118.140 | 5.958 | 23.737% / 16.891% / 14.595% | +| `..._031954` | 264.184 us | 1,065.442 ns | 96.167 | 5.660 | 22.086% / 15.785% / 16.041% | + +两轮均为 32 AIC + 64 AIV、480/480 begin/end、phase status `0x3f`、96/96 primary/shadow 精确相等，最大单段 shadow request/miss 分别为 3,250/114 和 3,206/106。局部 per-call 时间相近，而跨核全局 Submit 分别为 82.413 us 和 264.184 us；这再次说明 B1 只用于结构、次数、冷启动和构建隔离，绝不能把其绝对时间或不同 ELF 的先后当作性能结论。 + +两轮完整 Case1 产物为： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_032101/ +outputs/TestPagedAttentionUnroll_Case1_20260721_032244/ +``` + +两轮均为 96 核、每核 1,280 次 begin/end、phase status `0x3f`，owner 配置/恢复、32 个 mixed triplet、固定 shape、数值顺序、phase 时间与风险阈值全部闭合；primary/shadow 96/96 精确相等，capture gap 为 0。第一轮 raw/HTML 为 72,857/80,025 B，第二轮为 72,854/80,025 B。逐角色每次调用的原始 observed 为： + +| 产物 | 全局 Submit | 角色 | 每次 elapsed | 每次 request | 每次 miss | +| --- | ---: | --- | ---: | ---: | ---: | +| `..._032101` | 4,994.863 us | ALL | 641.816 ns | 80.219 | 1.957 | +| 同上 | 同上 | AIC | 311.706 ns | 79.437 | 0.025 | +| 同上 | 同上 | AIV | 806.871 ns | 80.609 | 2.922 | +| `..._032244` | 4,704.936 us | ALL | 646.708 ns | 80.202 | 1.951 | +| 同上 | 同上 | AIC | 313.109 ns | 79.465 | 0.026 | +| 同上 | 同上 | AIV | 813.507 ns | 80.571 | 2.914 | + +阶段占比仍只在同一 ELF、同一轮、同一角色内计算：phase elapsed core-time 除以逐核首末 Submit elapsed core-time，phase request/miss observed 分别除以本轮整窗 primary。两轮结果为： + +| 产物 | 角色 | 时间占比 | request observed 占比 | miss observed 占比 | +| --- | --- | ---: | ---: | ---: | +| `..._032101` | ALL | 17.942% | 14.188% | 16.870% | +| 同上 | AIC | 8.862% | 13.636% | 4.324% | +| 同上 | AIV | 22.368% | 14.476% | 17.086% | +| `..._032244` | ALL | 18.493% | 14.182% | 16.917% | +| 同上 | AIC | 9.213% | 13.627% | 4.436% | +| 同上 | AIV | 22.940% | 14.473% | 17.133% | + +两轮逐角色 per-call 和同 ELF 占比方向稳定：AIV Claim 时间约为 AIC 的 2.6 倍，但当前证据不能把全部差异归给 atomic、I-cache 或某一条角色分支。empty-bracket Case1 测得的 ALL 记录开销量级约为 639～640 ns、49.34 request、1.34～1.36 miss/对；Claim 则约为 642～647 ns、80.2 request、1.95～1.96 miss/次。empty elapsed 是外层 tick 包住完整 observer 对，Claim elapsed 是两侧 observer 内部区间；两者又来自不同 ELF、布局和缓存状态，所以数值接近不代表 Claim 业务耗时接近零，request/miss 之差也不能当作“净 Claim 事件”。empty 仍只能作为观察器自扰动的经验尺度，不能相减。 + +Claim 内已有一条独立的 atomic 证据。最新 `_234305` 泳道中，73,728 条 `ClaimMax` FetchMax `return_ready` bracket 全部嵌套在 122,880 条 Claim 内，aggregate 为 40,222,098 core-ticks，占该泳道 ELF Claim 总量的 50.612%。其返回值被 `N > old` 真实消费，因此该边界表示返回依赖就绪，不是 source-issue；但它仍不是全系统可见性屏障。Claim submit-PMU ELF 编译掉的是 atomic 观察与落盘代码，不是实际 FetchMax，因此 phase elapsed 已经包含真实 atomic 路径及其等待，不能再把 40,222,098 ticks 加到 phase 时间，也不能跨 ELF 扣除它来制造“非 atomic Claim”。这份 overlay 只能解释为何 Claim 值得继续观察，不能单独解释 AIC/AIV 差异或直接推出可获得的优化收益。 + +Claim 代码落定后串行回归了六类互斥 B1 构建： + +| 构建 | 产物 | 结果 | +| --- | --- | --- | +| `submit-pmu-materialize` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_032445/` | 96×5、status `0x3f`、primary/shadow 96/96 精确相等；235.643 us | +| `submit-pmu-arg-build` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_032536/` | 96×5、status `0x3f`、primary/shadow 96/96 精确相等；215.137 us | +| `submit-pmu-empty-bracket` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_032624/` | 96×5、status `0x3f`、primary/shadow 96/96 精确相等；276.578 us | +| `submit-pmu-none` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_032712/` | 无 phase 字段、primary/shadow 96/96 精确相等；76.509 us | +| `perf-clock` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_032802/` | 96×5 Submit、ELF 身份和调用 shape 闭合；74.512 us | +| 普通 level-4 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_032914/` | 4,552 条事件、87.664 us、`dropped=0`、排他整数闭合 PASS | + +六轮只证明 Claim 四边界没有破坏已有 phase、none、perf-clock 与合并泳道构建；B1 的 74～277 us 绝对时间明显波动，且冷启动与跨核到达均未被单独控制，所以只作结构回归，不对差值归因，也不参与候选保留/撤销或观察代价计算。 + +#### 8.3.6 第四个真实业务 selector：`submit-pmu-register` + +**[观察工具，已完成两轮 B1、两轮 Case1 与互斥构建回归]** + +最新权威 Case1 泳道 `outputs/TestPagedAttentionUnroll_Case1_20260720_234305/` 中，Register 为 47,991,560 aggregate core-ticks，占 SubmitUnion 的 12.010%，固定 122,880 次。它是 Claim 之后仍超过约 10%、shape 固定且能映射到连续真实调用体的下一区域，因而进入单阶段 PMU；不是按 standalone 的旧 phase 清单机械补齐。 + +编译期固定： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=5 +PTO_FDWIC_TRACE_ENABLED=0 +``` + +公共协议 mode 为 `6`，phase id 为 `5`，raw name/boundary 为 `register/register_outputs_call_entry_to_return`，counter/time semantics 仍为 `running_read_clear_observed_bracket` 和 `inner_sys_cnt_between_boundary_observers`。设备继续复用 12,416 B phase ABI，没有增加逐调用、task-kind 或 insert 数字段。 + +当前最终 AICore 缓存为 `aicore-extra/32c26e06ad76d186`： + +| 文件 | SHA256 | `.text` | +| --- | --- | ---: | +| `aicore_kernel.o` | `8264a6afd39815c825e0630dcbb69d9a3492d2e26989a61136f06d5e371fb750` | 150,608 B | +| `aicore_aic_combined.o` | `fd542b9ffdb33ea480ac91527c3e361cf44ae27604d2fd9ec90fc66e1ef53306` | 68,352 B | +| `aicore_aiv_combined.o` | `9a1eeca6fc06f370c02ff3acc8d1546820a910e7272b866f3b743b91be6ce9bb` | 82,256 B | + +AIC/AIV `.text` 之和与最终 ELF 的 150,608 B 精确相等。最终 ELF 含 `dist_submit_pmu_expect_submits`、整窗 counter reader 和 AIC/AIV phase reader；perf-clock、普通泳道、逐 atomic 与通用逐 task PMU 符号均不存在。现有 raw 未内嵌 ELF SHA，上表只记录最终缓存身份，不把前几轮产物包装成逐字节 ELF 存档。 + +真实边界没有复制一条新的 Register 近似实现，而是紧贴现有三个 `dist_submit_register_outputs()` 调用点： + +1. 统一 `dist_submit_finish_kernel_tail()`，覆盖旧 Kernel 与 compete-first Kernel Finish，位于可选 Fanin 后，传 `include_existing=true`； +2. 旧 `dist_alloc_tensors()`，传 `include_existing=false`； +3. compete-first `dist_alloc_compete_first_finish()`，同样传 `include_existing=false`。 + +begin 在调用入口，end 在返回后、`TRACE_TIMESTAMP(register_end)` 前；因此刻意排除前一条 record 发布、Register 结束时间戳和 caller 衔接。它是普通泳道 Register 的核心调用体，不是 timestamp-to-timestamp span 的逐 tick 复制。三条路径在正常成功 Submit 中均恰好一次；固定 shape 和 phase status 会 fail-closed 拒绝提前返回造成的缺失边界。 + +该调用体存在重要业务混合：Kernel 的 `include_existing=true` 会按 `ctx.register_mask` 扫描并插入 existing tensor，Alloc 的 `false` 路径在 helper 入口直接返回；`register_mask=0` 的 Kernel 也可能接近空调用。当前 PA 每 batch 为 1 Alloc + 4 Kernel，但为控制观察扰动没有增加 task-kind 或逐 insert raw 字段。因此结果只能解释为 RegisterOutputs 调用体聚合，不能命名为单次 TensorMap insert 净成本。 + +两轮 B1 先闭合模式、三挂点、固定次数与数据门禁： + +| 产物 | 全局 Submit | ALL elapsed/call | ALL request/call | ALL miss/call | 结果 | +| --- | ---: | ---: | ---: | ---: | --- | +| `..._034517` | 80.904 us | 231.281 ns | 140.208 | 4.446 | 96×5、480/480、`0x3f`、primary=shadow | +| `..._034904` | 267.167 us | 236.838 ns | 123.869 | 3.956 | 96×5、480/480、`0x3f`、primary=shadow | + +B1 全局时间再次明显变化，而局部 per-call 量级相近；本阶段没有分别控制冷暖态、跨核到达或非 scalar-busy 等待，因此不把差值归给其中任何一项。这两轮只作结构证据，不用于 Register 稳态判断。 + +两轮完整 Case1 为： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_035025/ +outputs/TestPagedAttentionUnroll_Case1_20260721_035136/ +``` + +两轮都通过 96 核、每核 1,280 次、122,880/122,880 begin/end、phase status `0x3f`、owner Restore、32 个 mixed triplet、primary/shadow 精确相等、数值顺序和风险阈值门禁。逐角色原始 observed 为： + +| 产物 | 全局 Submit | 角色 | elapsed/call | request/call | miss/call | +| --- | ---: | --- | ---: | ---: | ---: | +| `..._035025` | 4,688.752 us | ALL | 187.688 ns | 86.699 | 1.352 | +| 同上 | 同上 | AIC | 187.833 ns | 88.301 | 0.045 | +| 同上 | 同上 | AIV | 187.615 ns | 85.898 | 2.005 | +| `..._035136` | 5,136.513 us | ALL | 187.879 ns | 86.517 | 1.392 | +| 同上 | 同上 | AIC | 188.787 ns | 88.166 | 0.050 | +| 同上 | 同上 | AIV | 187.425 ns | 85.692 | 2.063 | + +阶段与完整 Submit 的比率仍只在同一 raw、同一角色内计算： + +| 产物 | 角色 | 时间占比 | request observed 占比 | miss observed 占比 | +| --- | --- | ---: | ---: | ---: | +| `..._035025` | ALL | 5.249% | 14.889% | 7.132% | +| 同上 | AIC | 5.390% | 14.902% | 9.492% | +| 同上 | AIV | 5.181% | 14.882% | 7.112% | +| `..._035136` | ALL | 5.055% | 14.884% | 7.335% | +| 同上 | AIC | 5.120% | 14.844% | 10.510% | +| 同上 | AIV | 5.023% | 14.905% | 7.308% | + +两轮约 188 ns/call、86.6 request/call 的方向稳定，AIV miss 约 2.0/call、AIC 约 0.05/call。但这不支持把普通泳道 Register 的 12.010% 全部归给该调用体：泳道边界更宽且属于另一 ELF。empty-bracket 的约 639～640 ns/对也只能说明观察器自扰动不可忽略；它的外层时间口径与 Register 的内层时间不同，更不能跨 ELF 相减。当前 phase 没有局部 scalar busy，`miss × 90 ns` 也只作单核串行量级感知。 + +代码落定后串行回归了七类互斥 B1 构建： + +| 构建 | 产物 | 结果 | +| --- | --- | --- | +| `submit-pmu-claim` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035425/` | 96×5、480/480、`0x3f`、primary=shadow；254.094 us | +| `submit-pmu-materialize` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035505/` | 96×5、480/480、`0x3f`、primary=shadow；84.021 us | +| `submit-pmu-arg-build` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035545/` | 96×5、480/480、`0x3f`、primary=shadow；73.728 us | +| `submit-pmu-empty-bracket` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035625/` | 96×5、480/480、`0x3f`、primary=shadow；77.925 us | +| `submit-pmu-none` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035703/` | 无 phase 字段、primary=shadow；259.429 us | +| `perf-clock` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035744/` | 96×5 Submit、ELF 身份/shape 闭合；74.282 us | +| 普通 level-4 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035843/` | 4,550 条事件、292.159 us、`dropped=0`、排他整数闭合 PASS | + +七轮只证明新增 mode、三个挂点和报告链没有破坏旧 phase、整窗、权威基线与合并泳道构建。B1 绝对时间不参与跨 ELF 比较。Python 的 profile/report/cache/ELF 门禁共 121 项通过。 + +#### 8.3.7 第五个真实 selector：`submit-pmu-submit-transition` + +**[观察工具，已完成两轮 B1、两轮 Case1 与互斥构建回归]** + +最新权威 Case1 泳道 `outputs/TestPagedAttentionUnroll_Case1_20260720_234305/` 中，`BetweenSubmitResidual` 为 72,362,428 aggregate core-ticks，占 SubmitEnvelope 的 **15.332%**。每核 1,280 个 Submit 形成固定 1,279 个相邻间隙，全局共 122,784 个；该 shape 与源码边界都可证明，因此它成为 Register 后唯一继续实现的 selector。该阶段不是新的业务分类，只聚合观察“上一次 Submit 结束到下一次 `dist_submit_begin()` 完成”之间的返回、编排衔接和下一任务准备。 + +编译期固定： + +```text +PTO_FDWIC_SUBMIT_PMU=1 +PTO_FDWIC_SUBMIT_PMU_PHASE_ID=6 +PTO_FDWIC_TRACE_ENABLED=0 +``` + +公共协议 mode 为 `7`，phase id 为 `6`，raw name/boundary 为 `submit-transition/previous_submit_end_to_next_submit_begin`。设备没有新增业务挂点，而是在已经统一存在的 `fdwic_submit_pmu_submit_begin/end()` 内建立状态机：非末次 Submit 的 end 打开区间，下一次非首 Submit 在 `dist_submit_begin()` 后进入 begin hook 时关闭；末次 end 只停止完整 PMU 窗口，不制造没有后继的区间。由此 B1 必须为每核 `5 - 1 = 4` 次，Case1 必须为每核 `1280 - 1 = 1279` 次。 + +`fdwic_submit_pmu_expected_phase_calls()` 集中表达该差异：None 为 0，Transition 为 `N - 1`，其他 phase 仍为 N。设备 ShapeValid、C++ host 校验/raw 导出和 Python report 各自使用或复算该规则；`N <= 1` 的 Transition capture 会被拒绝。两个 64 B 逐核结构和 12,416 B 总设备 ABI 均未增加字段，也没有记录间隙类型或逐间隙事件。因此 raw 是三类相邻任务组合的加权聚合，不能从中单独还原 kernel→kernel、kernel→alloc 或 alloc→kernel。 + +最终 AICore 缓存为 `aicore-extra/d17b87d79477c0e1`： + +| 文件 | SHA256 | ELF `.text` section | +| --- | --- | ---: | +| `aicore_kernel.o` | `dff1ba7ebc40be24afd48ee478e69b82c7877338beda66db1c915203f27e7099` | 153,424 B | +| `aicore_aic_combined.o` | `84e63975e5593f2e644c4582cfe0954c34916b503f08a125bcb8e76050c09bae` | 69,608 B | +| `aicore_aiv_combined.o` | `5d284cb7db1cf8d4fed7306c88e430f924246a68b738536f217fba47a87d8bdd` | 83,792 B | + +缓存的 CCEC 命令行包含 phase 6 和 trace-off 定义；对象含整窗 reader、phase shadow reader 与 `dist_submit_pmu_expect_submits`，不含普通泳道、atomic 观察或 perf-clock 符号。`aicore_kernel.o` 的 GNU `size` text 类合计为 153,864 B，其中还包含 440 B 只读数据；上表明确记录 ELF `.text` section，避免把两种口径混写。 + +两轮 B1 先验证 N-1 状态机和三层门禁： + +| 产物 | 全局 Submit | ALL elapsed/间隙 | ALL request/间隙 | ALL miss/间隙 | 结果 | +| --- | ---: | ---: | ---: | ---: | --- | +| `..._042627` | 265.223 us | 778.982 ns | 162.406 | 10.604 | 96×4、384/384、`0x3f`、primary=shadow | +| `..._042750` | 78.873 us | 802.044 ns | 155.164 | 9.078 | 96×4、384/384、`0x3f`、primary=shadow | + +B1 全局时间仍有明显冷启动/到达波动，只作结构证据。两轮完整 Case1 为： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_042914/ +outputs/TestPagedAttentionUnroll_Case1_20260721_043036/ +``` + +两轮都通过 96 核、每核 1,280 Submit/1,279 间隙、122,784/122,784 begin/end、全窗 status `0x7ff`、phase status `0x3f`、owner Restore、primary/shadow 精确相等、数值顺序和风险阈值门禁。逐角色原始 observed 为： + +| 产物 | 全局 Submit | 角色 | elapsed/间隙 | request/间隙 | miss/间隙 | 同 ELF 时间占比 | +| --- | ---: | --- | ---: | ---: | ---: | ---: | +| `..._042914` | 4,708.545 us | ALL | 354.560 ns | 134.101 | 4.815 | 10.046% | +| 同上 | 同上 | AIC | 303.374 ns | 133.426 | 0.503 | 8.881% | +| 同上 | 同上 | AIV | 380.153 ns | 134.438 | 6.972 | 10.601% | +| `..._043036` | 4,649.434 us | ALL | 350.516 ns | 134.145 | 4.459 | 9.933% | +| 同上 | 同上 | AIC | 303.185 ns | 134.200 | 0.494 | 8.843% | +| 同上 | 同上 | AIV | 374.181 ns | 134.118 | 6.441 | 10.456% | + +两轮时间与 request 方向稳定，且 AIV 间隙时间、miss 都高于 AIC；这只能描述当前 Transition 诊断 ELF 内的聚合现象。它与泳道 `BetweenSubmitResidual` 使用同源源码边界，但泳道还发布 record，PMU 构建则在边界 observer 内部取时且改变代码布局，所以 350～355 ns/间隙不能与泳道约 589 ns/间隙逐 tick 对齐。empty-bracket 的约 640 ns/对是另一 ELF 测得的记录开销量级，也不能扣除。当前阶段不声称得到 “无观察净间隙成本”，更不把 `miss × 90 ns` 当作墙钟损失。 + +代码落定后串行回归四类互斥 B1 构建： + +| 构建 | 产物 | 结果 | +| --- | --- | --- | +| `submit-pmu-register` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_043202/` | 96×5、480/480、`0x3f`、primary=shadow；80.010 us | +| `submit-pmu-none` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_043241/` | 无 phase 字段、primary=shadow；250.164 us | +| `perf-clock` | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_043322/` | 96×5 Submit、ELF 身份/shape 闭合；312.747 us | +| 普通 level-4 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_043553/` | 4,548 条 FDWIC record、887 条 atomic record、`dropped=0`、排他整数闭合 PASS；90.406 us | + +Register 代表原有 N 次 phase，none、perf-clock 和普通泳道分别覆盖另外三条互斥构建；配合 Python profile/report/cache/ELF 门禁单测，证明共享 N-1 helper 没有把旧 phase 改成错误 shape，也没有污染其他构建。B1 绝对时间仍不参与跨 ELF 比较。 + +#### 8.3.8 selector 停止线与三条证据链分工 + +`arg-build`、`materialize`、`claim`、`register` 和 `submit-transition` 已经实现；前四项覆盖 Submit 内明确业务区域，Transition 单独覆盖相邻 Submit 间隙，empty-bracket 只提供 running 观察器经验尺度，不计入业务覆盖。完成 Transition 后重新按占比、调用 shape、边界连续性和约 640 ns/对的 empty 经验尺度筛选，没有理由继续把所有短 span 批量做成 profile： + +- EfDrain 总量为 63,350,314 core-ticks，但其中 32,102,416 ticks 是 Kernel overlay；剩余 control 约 254 ns/Submit，又混有 atomic，当前局部 PMU 指标不能把三者可靠拆开； +- PrepareMap 为 22,673,322 ticks、约 184.5 ns/次，SubmitFinalize 为 19,944,392 ticks、约 162.3 ns/次且有四类尾边界；二者都短于 empty 经验尺度； +- Fanin、WinnerBuild、AllocComplete 和真实 loser 尾动作占比更小或 shape 非统一，继续增加 profile 更可能放大观察器影响，而不是提高归因能力。 + +`submit-transition` 已按 `expected_submits - 1` 的独立 shape 契约闭合，不再是待决候选。至此停止扩张 phase，转入三类构建观察代价、独占设备波动和 perf-clock 候选优化。后续单阶段 PMU 仍只独立编译、独立运行、独立发布，只报告本 raw 内的原始 observed 和占比，不与 empty 或任何其他 selector 相减。standalone 历史实现只提供 owner、门禁和报告加工方法参考，不是当前真实 PA 的前置任务，也不能替代真实结果。 + +三条证据链最终分工为： + +| 构建 | 回答的问题 | 不能回答的问题 | +| --- | --- | --- | +| `perf-clock` | 候选是否真正缩短完整 Submit 墙钟 | 具体业务区域和 PMU 原因 | +| `swimlane` | 业务区域、atomic 次数、时序与闭合 | 无观察净性能、I-cache miss | +| `submit-pmu-none` / 单阶段 PMU | 完整窗口 AIC/AIV 每核 total/scalar/primary，以及一个真实 span 的同 ELF 时间和 request/miss observed | 跨 ELF 净阶段成本、局部 scalar busy、最终墙钟收益 | + +第 8.4 节按这一分工交错比较 perf-clock 与两类诊断构建的完整时间；只有构建级偏移超过同构建波动时，才允许讨论保留观察能力的整体影响。 + +### 8.4 三类观察构建的交错量化 + +**[观察校准，已完成；当前波动下构建级差异不可可靠分辨]** + +#### 8.4.1 固定对象与比较规则 + +本阶段冻结真实 PA 源码于 `d1572c33c942018005fa4c1f631569915f2a7e26`，只比较以下三种独立构建的完整 Submit 时间： + +| 代号 | 构建 | 唯一用途 | +| --- | --- | --- | +| P | `--fdwic-profile perf-clock` | 后续业务候选的权威低扰动时间基线 | +| S | `--enable-l2-swimlane 4` | 普通业务 span 与 atomic 合并泳道 | +| N | `--fdwic-profile submit-pmu-none` | 完整 Submit 的 AIC/AIV PMU 与 I-cache | + +三者均为独立 pytest 进程，固定 Case1、`--rounds 1 --skip-golden`。P 从 `fdwic_perf_clock_summary.json` 取整数 `global_submit_span_ticks`；S 从严格排他分析取 `global_submit_makespan.duration_cycles`；N 先经生产报告器完整校验，再取 `window.global_submit_span_ticks`。三个字段都表示跨核最早首 Submit 到最晚末 Submit，按 1 ns/tick 解释；1.65 GHz 只用于 PMU cycle，不参与这次全局时间换算。 + +三份最终 AICore 身份为： + +| 构建 | cache key | `aicore_kernel.o` SHA256 | 文件大小 | ELF `.text` | +| --- | --- | --- | ---: | ---: | +| P | `138ce601ea506665` | `1fc81e8f3169c980de3d9103e0741d5ac8ecc92a59da83c0147d197658f3f0f3` | 2,468,576 B | 132,688 B | +| S | `faf05ee682e6a57d` | `16a2f2c4688f668c41d07a8e20262e482649c28e0daf7d5c0873cd622647e586` | 2,693,320 B | 203,344 B | +| N | `aa43623282e2a7db` | `53934476e4a12f72f74da5e3035c9f066924ffbf36f1515e0baffdadad6bc3b9` | 2,513,848 B | 138,064 B | + +P 定义为 `PTO_FDWIC_PERF_CLOCK=1;PTO_FDWIC_TRACE_ENABLED=0`，N 定义为 `PTO_FDWIC_SUBMIT_PMU=1;PTO_FDWIC_TRACE_ENABLED=0`，S 不带私有 profile 定义。三者共用的 `libhost_runtime.so` SHA256 为 `5c2b6de4426e29a7aba6c65cbf83e9a3bebcc9fb19dd6bf126fa66f0aac7bd53`。测量期间 HEAD 和所有构建输入源码未变化；05:28 出现的另一个会话 standalone 说明文档工作树改动不参与编译，也不纳入本阶段提交。 + +这次比较只允许回答“整个观察构建相对 P 的时间分布是否发生可辨认偏移”。即使偏移稳定，也不能把 S-P 称为某条 record/atomic 的纯成本，不能把 N-P 称为 PMU start/stop 的纯成本，更不能用于 Claim、I-cache 或其他阶段的跨 ELF 扣减。 + +#### 8.4.2 B1 预检与扩样门禁 + +正式 Case1 前按 P→N→S 各运行一次 B1： + +| 构建 | 产物 | 完整 Submit | 结构结果 | +| --- | --- | ---: | --- | +| P | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_044523/` | 77.478 us | 96 核、每核 5 Submit、身份闭合 | +| N | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_044616/` | 77.464 us | 96 核、primary=shadow、owner Restore、validation PASS | +| S | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_044712/` | 92.890 us | 4,548 条 FDWIC record、`dropped=0`、排他整数闭合 PASS | + +B1 仍只作结构预检，不进入时间统计。正式首轮使用六个排列块： + +```text +P→S→N, S→N→P, N→P→S, P→N→S, N→S→P, S→P→N +``` + +每种构建在第一、第二、第三位置各出现两次。预先约定的接受门禁为：六个配对差中至少 5/6 与中位数同方向，且 `abs(median delta) > 2 × MAD(delta)`；否则反序补一套完整六排列。首轮 P/S/N 的 MAD 分别为 602.632/12.021/153.896 us，S-P 与 N-P 都只有 3/6 同方向，因此按规则补跑： + +```text +S→P→N, N→S→P, P→N→S, N→P→S, S→N→P, P→S→N +``` + +最终共 36 个独立 Case1 进程，每种构建 12 个样本。全部样本通过各自门禁：P 为 96 核且每核 1,280 Submit；S 均为 schema-v4、96×1,280、`dropped=0`、排他分区和整数闭合 PASS；N 均通过 profile、owner、96×1,280、primary/shadow、计数阈值与 producer/consumer 重算。没有静默丢弃或替换任何样本。 + +#### 8.4.3 12 个交错块的原始结果 + +以下所有数值均为完整 Submit 的 us；delta 只表示同一时间块中两个完整构建结果之差： + +| 块 | 顺序 | P | S | N | S-P | N-P | +| ---: | --- | ---: | ---: | ---: | ---: | ---: | +| 1 | P→S→N | 5703.409 | 5116.147 | 5137.214 | -587.262 | -566.195 | +| 2 | S→N→P | 5811.557 | 5113.538 | 5619.488 | -698.019 | -192.069 | +| 3 | N→P→S | 5386.290 | 5134.027 | 5015.682 | -252.263 | -370.608 | +| 4 | P→N→S | 4498.145 | 5103.117 | 5079.540 | +604.972 | +581.395 | +| 5 | N→S→P | 4465.933 | 5087.037 | 5740.450 | +621.104 | +1274.517 | +| 6 | S→P→N | 4784.124 | 5127.158 | 4893.280 | +343.034 | +109.156 | +| 7 | S→P→N | 5429.436 | 5092.613 | 5821.280 | -336.823 | +391.844 | +| 8 | N→S→P | 4470.463 | 5068.575 | 4467.564 | +598.112 | -2.899 | +| 9 | P→N→S | 6713.689 | 5068.551 | 5077.279 | -1645.138 | -1636.410 | +| 10 | N→P→S | 5486.364 | 5062.530 | 4651.644 | -423.834 | -834.720 | +| 11 | S→N→P | 5543.774 | 5069.357 | 4602.872 | -474.417 | -940.902 | +| 12 | P→S→N | 4563.104 | 5109.824 | 4727.538 | +546.720 | +164.434 | + +按构建自身汇总： + +| 构建 | n | 最小值 | 最大值 | 中位数 | MAD | +| --- | ---: | ---: | ---: | ---: | ---: | +| P | 12 | 4465.933 | 6713.689 | 5407.863 | 513.717 | +| S | 12 | 5062.530 | 5134.027 | 5097.865 | 23.395 | +| N | 12 | 4467.564 | 5821.280 | 5046.480 | 356.890 | + +配对统计中的“中位相对差”先按每块计算 `100 × (X/P - 1)`，再对 12 个相对差取中位数；不是用表中两个独立中位数相除： + +| 比较 | 范围 | 中位差 | MAD | 同方向 | 中位相对差 | 门禁 | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| S-P | -1645.138～+621.104 us | -294.543 us | 520.527 us | 7/12 | -5.444% | 不通过 | +| N-P | -1636.410～+1274.517 us | -97.484 us | 479.019 us | 7/12 | -1.685% | 不通过 | +| S-N | -728.667～+601.011 us | +70.961 us | 325.625 us | 7/12 | +1.412% | 仅辅助，同样不可分辨 | + +扩样后的标准是至少 10/12 同方向且差值绝对值大于 `2×MAD`。三组都只有 7/12，S-P 的 294.543 us 也小于 1041.054 us，N-P 的 97.484 us 小于 958.038 us。因此不能声称 S 或 N 相对 P 更快/更慢，也不能从三组中挑选最接近预期的样本给出 “观察代码成本”。 + +#### 8.4.4 当前能够成立的结论 + +P 的首六/后六中位数为 5085.207/5457.900 us，N 为 5108.377/4689.591 us，方向相反；S 为 5114.842/5068.966 us，变化较小。按执行位置分组时，P 的第一/第二/第三位中位数为 5133.257/5407.863/5007.118 us，N 为 4833.663/5078.409/5015.247 us，S 为 5103.075/5098.431/5085.834 us。位置本身也没有给 P/N 提供统一单调解释。 + +唯一稳定事实是：当前 S 构建自身的 12 轮分布明显窄于 P/N。但 S 使用更大的 203,344 B `.text`、高频 record 和不同同步/轮询时序，这个现象可能是观察代码改变了系统的到达分布，也可能是 P/N 暴露了某个偶发慢尾；现有跨 ELF 数据无法区分原因，更不能反向证明“泳道没有代价”。 + +因此本阶段以**负结果**收口：在当前独占设备和运行方式下，三构建完整 Submit 的交错差值被 P/N 的同构建波动淹没，无法可靠量出观察构建整体影响。下一阶段不再增加跨 ELF 样本，而是固定同一 P ELF 连续运行足量样本，分别检查完整墙钟、逐核 body、首核到达和最慢核尾部，先定位波动来自哪里；在此之前不进入业务优化 A/B。 + +### 8.5 同一 `perf-clock` ELF 的独占设备波动定位 + +**[波动定位，已完成；主统计载体为多数核 Submit body 共同伸缩，统计残差 X 表现为迁移慢尾]** + +#### 8.5.1 冻结对象与采样协议 + +第 8.4 节已经证明跨 ELF 差值会被同构建波动淹没。本阶段不再比较 P/S/N，而是只用同一份真实 PA `perf-clock` ELF 连续运行，回答波动首先落在以下哪种时间成分： + +1. 96 核首个 Submit 的到达偏斜； +2. 多数核从首个到末个 Submit 的完整 body 共同伸缩； +3. 某个慢核在多数核 body 之外额外形成的尾部。 + +冻结身份保持为： + +| 对象 | 固定值 | +| --- | --- | +| AICore cache | `138ce601ea506665` | +| `aicore_kernel.o` SHA256 | `1fc81e8f3169c980de3d9103e0741d5ac8ecc92a59da83c0147d197658f3f0f3` | +| `aicore_kernel.o` 大小 / `.text` | 2,468,576 B / 132,688 B | +| AICore 定义 | `PTO_FDWIC_PERF_CLOCK=1;PTO_FDWIC_TRACE_ENABLED=0` | +| `libhost_runtime.so` SHA256 | `5c2b6de4426e29a7aba6c65cbf83e9a3bebcc9fb19dd6bf126fa66f0aac7bd53` | + +运行期间没有修改源码、触发重编译或穿插其他 profile。开始前先运行一次 B1，只作 96 核、调用 shape 和构建身份预检；再运行一次 Case1 预热并明确排除出统计；正式样本使用 20 个连续但彼此独立的 pytest 进程，每个进程固定： + +```text +--case Case1 --fdwic-profile perf-clock --rounds 1 --skip-golden +``` + +没有使用单进程 `--rounds 20`，没有人工 sleep，也没有删除或替换快慢极值： + +| 用途 | 产物 | 结果 | +| --- | --- | --- | +| B1 预检 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_054534/` | 96×5 Submit，74.552 us，结构和身份闭合 | +| Case1 预热 | `outputs/TestPagedAttentionUnroll_Case1_20260721_054947/` | pytest PASS；不进入统计 | +| 正式 20 轮 | `outputs/TestPagedAttentionUnroll_Case1_20260721_055106/` 至 `..._060447/` | 20/20 pytest PASS | + +B1 的 74.552 us 几乎完全由 AIC core 23 的 74.484 us body 长尾决定；该核只比全局最早核晚到 0.068 us，因此 B1 也支持“长体不是起跑偏斜”的先验。但 B1 只有每核 5 次 Submit，不能与 Case1 时间混算，也不进入正式分布。 + +#### 8.5.2 raw 闭合与统计定义 + +20 份 `fdwic_perf_clock_summary.json` 全部满足： + +- schema 为 `fdwic-perf-clock-v1`，参考时钟为 1 GHz； +- 96 个唯一 core，32 AIC + 64 AIV，core/block/lane 映射闭合； +- 每核 1,280 次 Submit，单轮共 122,880 次，20 轮合计 2,457,600 次； +- 每核 `elapsed = last_submit_end - first_submit_start`； +- 全局 `span = max(last_submit_end) - min(first_submit_start)`。 + +JSON 中 `global_submit_span_us` 和 group mean 是展示用舍入值；所有计算与门禁都使用整数 tick 原值，不对舍入后的浮点字段二次运算。这里的 1 GHz 是 sys-counter 参考时钟，不是约 1.65 GHz 的 PMU/core cycle 频率。raw 本身不内嵌 ELF SHA；通过采样前后的外部构建身份关联，另行确认 AICore ELF 和 host runtime SHA 完全一致。 + +对一轮中的核 `c` 定义： + +```text +s[c] = first_submit_start +e[c] = last_submit_end +b[c] = e[c] - s[c] + +S = min(s[c]) +E = max(e[c]) +G = E - S +``` + +对该轮最晚结束核 `z`，进一步作精确分解： + +```text +A = s[z] - S # 最晚结束核相对全局首核的晚到时间 +M = median(b[c]) # 96 核完整 Submit body 的中位数 +X = b[z] - M # 最晚结束核相对中位数的额外慢尾 +G = A + M + X # 由整数 tick 派生并在每轮精确闭合 +``` + +预先固定的“主要统计载体”门禁为：`abs(Spearman(G, component)) >= 0.7`，且该 component 的 `P90-P10` 不小于 `G` 的 `P90-P10` 的 50%。这只能判断波动落在哪个可观察时间成分，不能把统计关联直接命名为 atomic、I-cache、flag 等待或频率变化。 + +#### 8.5.3 20 轮原始分解 + +下表单位均为 us；`末核` 同时给出 core 类型、block 和 lane： + +| 轮次 | 输出时间戳 | G | M | X | A | 末核 | +| ---: | --- | ---: | ---: | ---: | ---: | --- | +| 1 | `055106` | 5282.858 | 4639.647 | 634.045 | 9.166 | c28/AIC/b28/l0 | +| 2 | `055149` | 6066.913 | 5311.507 | 746.303 | 9.103 | c22/AIC/b22/l0 | +| 3 | `055232` | 4634.349 | 4397.435 | 228.024 | 8.891 | c15/AIC/b15/l0 | +| 4 | `055315` | 4898.490 | 4362.309 | 531.446 | 4.736 | c17/AIC/b17/l0 | +| 5 | `055357` | 5320.520 | 5049.764 | 268.651 | 2.105 | c16/AIC/b16/l0 | +| 6 | `055439` | 4633.401 | 4391.994 | 228.202 | 13.205 | c17/AIC/b17/l0 | +| 7 | `055522` | 4473.506 | 4377.564 | 94.329 | 1.614 | c1/AIC/b1/l0 | +| 8 | `055604` | 4509.646 | 4380.363 | 117.660 | 11.623 | c4/AIC/b4/l0 | +| 9 | `055647` | 5896.752 | 5595.158 | 299.450 | 2.144 | c77/AIV/b22/l2 | +| 10 | `055731` | 4897.879 | 4695.680 | 192.089 | 10.110 | c35/AIV/b1/l2 | +| 11 | `055813` | 5478.058 | 5172.396 | 302.890 | 2.771 | c25/AIC/b25/l0 | +| 12 | `055857` | 4653.463 | 4490.646 | 162.746 | 0.070 | c9/AIC/b9/l0 | +| 13 | `055941` | 4471.733 | 4375.400 | 88.891 | 7.442 | c77/AIV/b22/l2 | +| 14 | `060024` | 4557.543 | 4360.690 | 184.311 | 12.541 | c90/AIV/b29/l1 | +| 15 | `060110` | 5037.755 | 4675.526 | 360.102 | 2.127 | c30/AIC/b30/l0 | +| 16 | `060153` | 4764.823 | 4404.648 | 348.097 | 12.078 | c53/AIV/b10/l2 | +| 17 | `060238` | 5128.149 | 4862.399 | 265.724 | 0.026 | c40/AIV/b4/l1 | +| 18 | `060321` | 6169.966 | 5575.921 | 593.804 | 0.241 | c20/AIC/b20/l0 | +| 19 | `060404` | 4577.567 | 4381.504 | 188.969 | 7.094 | c70/AIV/b19/l1 | +| 20 | `060447` | 5860.558 | 5033.376 | 826.275 | 0.907 | c27/AIC/b27/l0 | + +#### 8.5.4 分布、相关性与门禁结果 + +主要分量的完整分布如下。MAD 为未缩放的中位绝对偏差，P10/P90 使用线性插值： + +| 指标 | 最小值 | 中位数 | 最大值 | MAD | P10 | P90 | P90-P10 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| G | 4471.733 | 4898.184 | 6169.966 | 362.658 | 4506.032 | 5913.768 | 1407.736 | +| 首核到达跨度 | 11.504 | 13.136 | 13.979 | 0.542 | 11.788 | 13.470 | 1.681 | +| M | 4360.690 | 4565.147 | 5595.158 | 188.665 | 4374.091 | 5337.948 | 963.858 | +| X | 88.891 | 267.188 | 826.275 | 87.895 | 115.327 | 645.271 | 529.944 | +| A | 0.026 | 5.915 | 13.205 | 4.002 | 0.224 | 12.124 | 11.900 | + +与 G 的关联和主要载体门禁为： + +| 序列 | Pearson | Spearman | 序列跨度 / G 跨度 | 判定 | +| --- | ---: | ---: | ---: | --- | +| M | +0.943 | +0.895 | 68.5% | PASS | +| X | +0.771 | +0.850 | 37.6% | FAIL：相关但跨度不足，只列为次要慢尾 | +| A | -0.436 | -0.395 | 0.8% | FAIL | +| 首核到达跨度 | +0.116 | +0.120 | 0.1% | FAIL | +| AIC body 中位数 | +0.943 | +0.896 | 89.8% | 辅助序列满足同阈值；不是 A/M/X 分解项 | +| AIV body 中位数 | +0.934 | +0.890 | 63.1% | 辅助序列满足同阈值；不是 A/M/X 分解项 | + +以下交叉证据排除了“只有中位数偶然同向”的解释： + +- 96 个核的 body 与 G 跨轮 Spearman 全部为正，95/96 不小于 0.7，中位数为 +0.862；相邻轮方向一致率逐核为 78.9%～94.7%，96/96 核均不低于 70%； +- body 最小值与 G 的 Spearman 为 +0.869，跨度达到 G 的 69.5%，说明连快核也随整轮共同伸缩； +- AIC 与 AIV body 中位数跨轮 Spearman 为 +0.973，不支持某类核心稳定单边拉长；AIC 中位数高于 AIV 只有 9/20 轮； +- 每轮核内“晚到程度与 body”的 Spearman 中位数仅 -0.042，范围 -0.258～+0.211，进入稍晚的核并不会系统性执行更久； +- 最晚结束核与最长 body 核 20/20 相同，但 20 轮分散到 18 个不同 core；只有 c17 和 c77 各重复两次，类型为 AIC 13 次、AIV 7 次，不支持固定物理坏核； +- 运行序号与 G 的 Spearman 仅 +0.006，没有随连续运行时间单调升高或降低的趋势。 + +#### 8.5.5 当前成立的原因层级与下一步 + +本阶段把“独占设备仍波动”收敛到两个层级： + +1. **主要统计载体是多数核完整 Submit body 的共同伸缩。** 这不是某个首核晚启动，也不是一个固定物理核偶发卡住；AIC/AIV 和绝大多数单核 body 都随整轮同向变化。 +2. **统计残差 X 表现为迁移慢尾。** X 与 G 高度相关，但 P90-P10 只有 G 的 37.6%，没有达到预设的主要载体门禁，因此只能解释次级差异，不能替代共同伸缩结论，也不能仅凭该残差声称存在一种独立物理机制。 + +“设备独占”只排除了其他用户任务抢占，不会自动固定设备频率、温度、cache、内部运行时状态或真实 Kernel/flag/atomic 的时序。当前 perf-clock raw 没有 PMU、业务 phase、Kernel 或 atomic 字段，所以还不能在以下原因之间作选择： + +- on-core PMU cycle 数本身随轮次增减； +- 每纳秒有效 PMU cycle 比例发生变化； +- scalar issue、I-cache request/miss、atomic 或 completion flag 等待变化； +- 真实 Vector/Cube Kernel 及其内部 PIPE wait 变化。 + +下一阶段先不加设备探针，也不修改业务代码：固定同一份 `submit-pmu-none` ELF，连续运行独立 Case1，在每一份同 ELF raw 内联合分析 SYS Submit elapsed、PMU total、scalar busy、I-cache request/miss。只有它能复现 P 的波动形态，才允许用这些字段解释共同伸缩；若不能复现，就明确认定 PMU ELF 改变了现象，再转向同一 swimlane ELF 的快慢轮结构或最小容量的 per-core Kernel 聚合。 + +另行查阅本机 CANN 9.1 的已安装实现，`asys profiling -r power -p <秒> -d 0` 在 Ascend950 路径会启用 `msprof --sys-lp=on`，其 low-power 数据可包含 AIC 平均频率、软件 DVFS 下发频率和 EDP 降频计数。这不是上述 20 份 raw 的结论，而且 msprof 采样不能与权威 perf-clock 混跑或跨模式相减。只有同 ELF PMU 仍不能解释波动时，才单独建立这条环境取证链。 + +### 8.6 同一 `submit-pmu-none` ELF 的 cycle、Scalar 与 I-cache 联合关联分析 + +**[PMU 波动定位，已完成；N ELF 的共同 body 变化主要落在 Scalar-busy cycle，但 N 没有完整复现 P 的 A/M/X 形态]** + +#### 8.6.1 复用样本、连续补样与构建身份 + +第 8.4 节已经留下 12 份同一 N ELF 的严格 Case1 raw。先复算这些存量后发现，PMU total 与 Scalar-busy 已有明显同向信号，但旧样本来自 P/S/N 交错排列，I-cache miss 又呈现与前序 ELF 关联的两个状态。为避免只凭受前序状态影响的 12 轮下负结论，本阶段复用存量并补充： + +1. 一次 N-only Case1 预热，明确排除出统计； +2. 8 个连续、独立的 N-only Case1 进程，中间不插入 P、S 或其他 profile； +3. 旧 12 + 新 8 合计 20 轮统一使用当前生产 `load_capture()` 重算。 + +冻结构建身份为： + +| 对象 | 固定值 | +| --- | --- | +| AICore cache | `aa43623282e2a7db` | +| `aicore_kernel.o` SHA256 | `53934476e4a12f72f74da5e3035c9f066924ffbf36f1515e0baffdadad6bc3b9` | +| `aicore_kernel.o` 大小 / `.text` | 2,513,848 B / 138,064 B | +| AICore 定义 | `PTO_FDWIC_SUBMIT_PMU=1;PTO_FDWIC_TRACE_ENABLED=0` | +| `libhost_runtime.so` SHA256 | `5c2b6de4426e29a7aba6c65cbf83e9a3bebcc9fb19dd6bf126fa66f0aac7bd53` | + +采样索引为： + +| 样本 | 目录 | +| --- | --- | +| 历史 12 轮 | `..._045212`、`..._045441`、`..._045608`、`..._050000`、`..._050228`、`..._050807`、`..._051547`、`..._051632`、`..._052029`、`..._052259`、`..._052753`、`..._053148` | +| 本次预热 | `outputs/TestPagedAttentionUnroll_Case1_20260721_062309/`，不计入统计 | +| 本次 8 轮 | `..._062430`、`..._062512`、`..._062557`、`..._062641`、`..._062725`、`..._062809`、`..._062852`、`..._062936` | + +表中的省略前缀均为 `outputs/TestPagedAttentionUnroll_Case1_20260721`。20/20 raw 通过生产消费者的 schema、mode、owner、selector、topology、96×1280、SYS 起止、status、summary、primary/shadow 与 counter 阈值全量重算；合计 1,920 条 core record、2,457,600 次 Submit，primary-shadow request/miss 差值均为 0。最大可编程计数为 8,978,154，只占风险阈值 `0x3fffffff` 的 0.836%。新增 8 份 HTML 与当前生产 `render_report(raw)` 逐字节一致。采样结束后的 AICore 和 host SHA 保持不变；raw 本身仍不内嵌 cache key 或 ELF SHA，构建身份依赖上述外部关联。 + +#### 8.6.2 20 轮原始核心指标 + +下表单位为 us。`body/core` 是 96 核逐核 SYS elapsed 的 mean；`Scalar/core` 和 `残余/core` 分别是每核 PMU `scalar_busy` 与 `total-scalar_busy`，按生产报告 ALL 口径的本机长窗系数 1.649844 cycles/ns 换算。它们是 cycle-equivalent，不是 G 的可加分区。`miss/core` 是 96 核事件数 mean： + +| 轮次 | 来源 | G | body/core | Scalar/core | 残余/core | miss/core | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| `045212` | 历史 | 5137.214 | 4638.058 | 4197.932 | 439.480 | 9253.7 | +| `045441` | 历史 | 5619.488 | 4854.759 | 4399.790 | 454.297 | 9198.4 | +| `045608` | 历史 | 5015.682 | 4333.754 | 3882.439 | 450.733 | 11621.8 | +| `050000` | 历史 | 5079.540 | 4330.985 | 3878.340 | 452.012 | 11431.9 | +| `050228` | 历史 | 5740.450 | 5260.871 | 4808.963 | 451.293 | 9258.1 | +| `050807` | 历史 | 4893.280 | 4339.138 | 3891.437 | 447.037 | 11497.0 | +| `051547` | 历史 | 5821.280 | 5253.866 | 4784.488 | 468.736 | 11545.3 | +| `051632` | 历史 | 4467.564 | 4305.209 | 3859.732 | 444.808 | 11647.9 | +| `052029` | 历史 | 5077.279 | 4577.454 | 4124.678 | 452.125 | 11530.6 | +| `052259` | 历史 | 4651.644 | 4330.944 | 3898.974 | 431.301 | 9202.2 | +| `052753` | 历史 | 4602.872 | 4339.941 | 3908.777 | 430.510 | 9083.0 | +| `053148` | 历史 | 4727.538 | 4316.797 | 3884.156 | 432.015 | 9248.5 | +| `062430` | 连续 N | 5326.155 | 4898.879 | 4437.649 | 460.560 | 11674.2 | +| `062512` | 连续 N | 5081.403 | 4494.263 | 4039.732 | 453.884 | 11627.3 | +| `062557` | 连续 N | 4500.138 | 4345.932 | 3898.434 | 446.869 | 11639.1 | +| `062641` | 连续 N | 4808.405 | 4480.150 | 4027.859 | 451.670 | 11614.8 | +| `062725` | 连续 N | 4552.993 | 4312.336 | 3859.504 | 452.236 | 11633.6 | +| `062809` | 连续 N | 5613.177 | 4729.055 | 4270.670 | 457.756 | 11699.7 | +| `062852` | 连续 N | 5058.346 | 4336.133 | 3884.605 | 450.878 | 11640.8 | +| `062936` | 连续 N | 4495.435 | 4306.473 | 3858.078 | 447.766 | 11658.3 | + +#### 8.6.3 N 与 P 的波动形态只部分一致 + +N 的 G 中位数为 5037.014 us，范围 4467.564～5821.280 us，MAD 为 347.423 us，P90-P10 为 1131.917 us。N 的起跑偏斜同样只有微秒量级，但其 A/M/X 分解与 P 不同： + +| 构建 | A：rho / 幅度占 G | M：rho / 幅度占 G | X：rho / 幅度占 G | 严格门禁结果 | +| --- | --- | --- | --- | --- | +| P 20 轮 | -0.395 / 0.8% | +0.895 / 68.5% | +0.850 / 37.6% | M 为主，X 为辅 | +| N 20 轮 | +0.152 / 1.0% | +0.863 / 47.0% | +0.776 / 50.18% | X 通过；M 的幅度略低于门槛 | + +这里比较的是各自同 ELF 内的分布形态，没有对 P/N 绝对时间作减法。新增连续 8 轮单列时仍是 M 占 43.8%、X 占 67.4%，说明 N 的慢尾增强不是旧 12 轮交错状态单独造成。N 与 P 只共同复现了“起跑不是主因、body 聚合量有同向变化、慢尾会迁移”等部分现象，**没有完整复现 P 的 M 主导、X 次要形态**。逐核 body 对 G 虽为 96/96 正相关，但只有 52/96 核的 Spearman 不低于 0.7，明显弱于 P 的 95/96；N 的最晚结束核与最长 body 核为 19/20 重合，且分散在 17 个不同 core。因此后面的 PMU 结果只能解释 N 自身，不能反向冒充 P 的完整根因。 + +#### 8.6.4 PMU total、Scalar-busy 与非 Scalar-busy 残余 + +在 N 自身内部，对每轮 96 核 mean/core 使用与 G 相同的相关性和幅度双门禁，结果为： + +| 序列 | Spearman(G, ·) | P90-P10 | 占 G 幅度 | 判定 | +| --- | ---: | ---: | ---: | --- | +| SYS body/core | +0.853 | 622.628 us | 55.0% | PASS，辅助共同 body 证据 | +| PMU total/core 等效 | +0.853 | 622.560 us | 55.0% | PASS | +| Scalar-busy/core 等效 | +0.812 | 612.624 us | 54.1% | PASS | +| 非 Scalar-busy 残余/core 等效 | +0.657 | 26.093 us | 2.3% | FAIL | + +Scalar-busy 与同轮 SYS body/core 的 Spearman 为 +0.962；留一法的 20 个子样本中，Scalar-busy 对 G 有 18/20 仍通过双门禁，对 SYS body/core 则 20/20 通过。AIC Scalar-busy 对 G 为 +0.791、幅度为 G 的 74.8%；AIV 为 +0.830，但幅度为 G 的 43.3%。若分别与同角色 SYS body 比较，AIC/AIV Scalar-busy 的 Spearman 为 +0.994/+0.899；它们的 P90-P10 也分别为 846.964/490.170 us，接近同角色 body 的 865.774/498.548 us。两组 P90-P10 不是同一轮差值，不能相除后宣称“解释比例”。两类 residual 的幅度只有 G 的 2.9%/3.1%。按同一双门禁，AIC Scalar-busy 通过，AIV 则因 43.3% 的幅度不足而未通过。 + +因此 N ELF 自身可以收敛到：**完整 body 波动表现为执行了更多 PMU total cycle，其中主要幅度落在 Scalar instruction busy cycle；非 Scalar-busy 残余不是主要幅度载体。** 该事件不是纯算术指令数：受控微基准显示依赖返回的 atomic 等待大部分会进入 Scalar-busy，而 I-cache refill 大部分不进入；当前 A5 又没有已经核验的 `scalar_wait_ib_time`，所以还不能继续把 Scalar-busy 拆成普通控制、polling、atomic 或 `wait_flag`。`total-scalar_busy` 仍只能称非 Scalar-busy 残余，不能命名为空闲、I-cache stall 或 Vector/Cube 等待。现有 raw 也没有读取独立的 Vector/Cube wait 计数，不能据此排除这类等待。 + +#### 8.6.5 有效 cycle/time 幅度不足以解释波动 + +每轮按 `sum(PMU total) / sum(SYS body ticks)` 计算的长窗有效换算比为： + +| 分组 | 最小值 | 中位数 | 最大值 | P90-P10 相对幅度 | +| --- | ---: | ---: | ---: | ---: | +| ALL | 1.649587828 | 1.649607947 | 1.649650965 | 21.35 ppm | +| AIC | 1.649581477 | 1.649609592 | 1.649655438 | 27.49 ppm | +| AIV | 1.649590009 | 1.649608471 | 1.649648649 | 18.32 ppm | + +ALL 比值的全范围只有约 0.00383%，P90-P10 只有约 0.00213%；即使按 5 ms 感知，后者也只有约 0.107 us 数量级，无法解释 1.132 ms 的 G 跨度。这里允许的结论是“本机长窗有效 PMU cycle/time 比例不是 N 波动的主要幅度来源”；total gate 位于 SYS 首尾之内、比例只是窗口平均和本机等效换算，不能把它包装成瞬时硬件核频或无条件的频率契约。 + +#### 8.6.6 I-cache miss 的前序状态特征与非归因结论 + +20 轮中，miss/core 与 G 的 Spearman 为 ALL -0.141、AIC -0.048、AIV 0.000，均不支持 I-cache miss 随慢轮增加。新增连续 N-only 8 轮的 G 仍横跨 4495.435～5613.177 us，而 AIC miss/core 仅为 855.094～1114.094，AIV 更窄至 16973.828～17049.344；尤其 AIV miss 基本固定时仍存在 1.118 ms 墙钟范围。 + +历史 12 轮还揭示了此前 I-cache 数据易飘的直接特征： + +| 前序状态 | 样本数 | AIC miss/core 范围 | AIV miss/core 范围 | G 范围 | +| --- | ---: | ---: | ---: | ---: | +| S→N | 6 | 1289.500～1302.969 | 12973.047～13241.281 | 4602.872～5740.450 us | +| P→N 或该状态延续 | 6 | 433.500～885.031 | 16852.203～17079.109 | 4467.564～5821.280 us | + +两组 miss 完全不重叠，G 却大量重叠；这只能证明前序 ELF 与后续 I-cache 状态高度关联，不能仅凭顺序杜撰具体驱逐机制。request/core 的方向也分裂：AIC 对 G 的 Spearman 为 -0.576，AIV 为 +0.818，ALL 只有 -0.110。request 是事件量，可能是更长 Scalar/poll 路径的伴随结果，不能直接相加成墙钟。 + +所以当前结论是：**I-cache miss 次数不是 N 这 20 轮共同伸缩的主要关联变量；但这不能排除 miss 延迟变化或其他缓存机制，也不表示绝对 miss 成本为零。** 已有 90 ns/miss 仍只是一阶单核串行标尺，不能乘 96 核总 miss 后冒充 Submit 墙钟损失。 + +#### 8.6.7 阶段决定 + +本阶段不继续堆 N 样本，也不立即做频率/温度采样：PMU total 与 Scalar-busy 通过 N 内统计载体门禁，而全窗平均 cycle/time 比例和 miss 次数没有表现为主要关联变量；这仍不能排除 miss 延迟或其他缓存机制。N 的全局慢尾形态又与权威 P 不完全相同，继续用 N 解释 P 会越过证据边界。 + +下一步回到最接近权威基线的低容量路径：在 `perf-clock` 构建中沿真实 `execute_slot()` 的现有 Kernel 首尾边界，只增加**每核 Kernel 累计 SYS tick 和调用次数**，不生成逐事件 record、不启用泳道或 PMU。先验证该独立变体是否复现 P 的多数核共同伸缩形态；只有复现后，才在同一变体内部判断变化主要落在真实 Vector/Cube Kernel（包含内部 PIPE wait）的墙钟，还是其余 Scalar scheduler。若 Kernel 聚合稳定而其余部分继续拉长，再进入最新真实业务 span 的 Scalar 候选分析。该变体必须独立命名和闭合，不能冒充原 P，也不能与 P 绝对时间相减。 + +### 8.7 真实 PA 的低容量 `perf-clock-kernel` 聚合变体 + +**[工具实现与 B1/Case1 结构闭合已完成；波动分布采样待下一阶段冻结提交后进行]** + +#### 8.7.1 为什么必须是独立变体 + +权威 P 只在每核首个 Submit 起点和末个 Submit 终点各读取一次 `SYS_CNT`。第 8.5、8.6 节已经证明，不能靠跨 ELF 时间差把泳道或 PMU 中看到的现象硬扣回 P；但 P 自身又没有字段可以区分真实 Kernel 与其余调度路径。为此新增独立 profile： + +```text +--fdwic-profile perf-clock-kernel +PTO_FDWIC_PERF_CLOCK=1 +PTO_FDWIC_PERF_CLOCK_KERNEL=1 +PTO_FDWIC_TRACE_ENABLED=0 +``` + +本文简称它为 K。K 使用独立 cache key、最终 ELF marker、文件名和 schema，不能覆盖或冒充 P： + +```text +fdwic_perf_clock_kernel_summary.json +schema = fdwic-perf-clock-kernel-v1 +mode = perf-clock-kernel +``` + +P 继续输出原来的 `fdwic_perf_clock_summary.json` / `fdwic-perf-clock-v1`。后续首先比较 P、K 各自同 ELF 内的波动形态；只有 K 能复现 P 的 A/M/X 结构，才允许在 K 内部用 Kernel 与剩余调度区间做二分，绝不以 K-P 绝对时间差声称观察成本或业务收益。 + +#### 8.7.2 Kernel 边界与固定 64 B 布局 + +K 沿现有泳道的真实 Kernel 边界，在 `execute_slot()` 中紧贴 `dist_aicore_call_slot_kernel(s)` 前后各读取一次 `SYS_CNT`。该区间不包含函数返回后的 `store_barrier()`、完成标志发布、frontier 推进和 Commit。它包含 linked kernel 函数内部真实执行及其显式 `wait_flag` / pipe 同步，但不能命名为“纯 Vector/Cube 指令时间”，也不能证明函数返回后不存在仍在途的异步写回。 + +计时还受逐核 perf-clock 窗口约束：只有 `first_submit_start != 0` 且 `last_submit_end == 0` 时才累计。因此，本核末个 Submit 返回后的 FinalDrain Kernel 被有意排除。这样每核才能保持精确整数关系： + +```text +elapsed_ticks += kernel_elapsed_ticks + + non_kernel_residual_ticks +``` + +这里的 `non_kernel_residual` 只是上述同核窗口的算术剩余，包含调度、同步及观察边界等尚未细分的时间，不能直接重命名为 Scalar 或空闲。 + +没有新增 sidecar，也没有逐 Kernel record。`FdwicSwimlaneCoreState` 继续保持每核独占 64 B，整个固定 header 仍为 6,976 B： + +- 原 tail 的 32 B 增加 K 专属 union 视图，保存首末 Submit、实际/期望 Submit 数和 64-bit Kernel 累计 tick； +- trace 关闭后原前 20 B 中的 `count` 保存 Kernel 调用数，`dropped` 保存聚合错误状态，`poll_batch_records` 保存 K mode=2，另外两个字段必须为 0； +- tick 逆序、64-bit 累加溢出或 32-bit 调用数溢出都会设置状态，host 拒绝发布 raw； +- P 仍要求这五个外层字段全部为 0，并要求原 mode=1/final_seen=1。P/K 互相误载时不能静默通过。 + +#### 8.7.3 Host 发布前的闭合门禁 + +K 复用原 perf-clock 的 header-only 分配和一次 64 B/core flush，不打开普通泳道、atomic 或 PMU。host 只有在以下条件全部满足后才以 `.tmp` 原子发布正式 JSON： + +1. 固定 32 AIC + 64 AIV、96 个 core 的 block/lane 拓扑完全一致； +2. `records_per_core=0`、header=6,976 B、时钟为 1 ns/tick； +3. 每核实际 Submit 数等于 orchestration 声明值，B1 为 5、Case1 为 1,280，且 K 的每核首末 tick 必须形成严格正区间； +4. K mode=2、聚合状态为 0、保留字段为 0； +5. 每核 `kernel_ticks <= elapsed`，calls/ticks 同为 0 或同为非 0； +6. 全局起止严格等于 96 核首 tick 最小值和末 tick 最大值； +7. 当前 PA 固定每 batch 一次 Alloc 和四次 Kernel Submit；无 fanin 的 QK 最迟会在后继 SF Submit 的 EfDrain 执行，所以 AIC 调用数至少为 `batch`，AIC/AIV 分别不超过 `2*batch`，总数不超过 `4*batch`。 + +第 7 条的最大值不是等式：末个 Submit 后才执行的任务属于 FinalDrain，必须从 K 中排除；QK 下界则防止 hook 失效后全零报告仍被发布。输出保留逐核整数、AIC/AIV 的 min/max/sum/mean、调用数以及 `sum(kernel_ticks)/sum(elapsed_ticks)`。最后一个比例只能称“聚合 core-time 份额”，不能当作跨核墙钟占比，也不能用全局 Submit span 减去跨核 Kernel tick 求调度时间。顶层 `min_kernel_calls_in_window` / `max_kernel_calls_in_window` 表示按 batch 推导的 **全局总调用合法范围**，不是逐核极值；逐核实际极值只在 `groups.*.kernel_calls_min/max` 中表达。 + +#### 8.7.4 B1 边界取证与 P 回归 + +最终源码的 K B1 闭合件为： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_073602/ + fdwic_perf_clock_kernel_summary.json +``` + +golden、host 门禁和 Python 末端产物契约均 PASS。96 核每核 5 次 Submit，完整 span 为 84.816 us；窗口内只出现 1 个 Kernel 调用（合法范围 1～4），位于 AIC，累计 61.289 us。AIV 的窗口内 Kernel 调用为 0。这个结果不是“漏采”：它与最近泳道中 B1 只有一个 Kernel 落在 Submit、其余任务进入 FinalDrain 的结构一致，直接证明 K 的窗口过滤生效。 + +随后用原 P 做 B1 回归： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_073736/ + fdwic_perf_clock_summary.json +``` + +该轮同样 PASS，仍输出原 schema、原字段和原文件名，不含任何 Kernel 聚合字段。P 的最终 `.text` 仍为 132,688 B，K 的 `.text` 为 138,320 B；P 只含 `dist_perf_clock_expect_submits`，K 还必须含 `dist_perf_clock_kernel_profile_marker`，两者均不含泳道、atomic 或 Submit-PMU 观察器符号。P 的完整 ELF 因调试行号变化不能只凭 SHA 断言字节相同，所以这里只把 `.text` 大小、宏门禁和最终符号表作为“热路径未编入 K hook”的证据。 + +最终构建身份如下，后续 K 分布采样必须保持不变： + +| 对象 | cache / 大小 / SHA256 | +| --- | --- | +| P AICore | `138ce601ea506665` / 2,468,896 B / `98f4e3978cd145477be1865b489f6475545ed1dce6dda9d145768894438b57d1` | +| K AICore | `e9cebfc34cbed0e7` / 2,495,544 B / `8d23407aa0062534812846b7da03f5078a4843a261f134812c95f5dbc5155060` | +| 共用 host SO | 11,644,136 B / `5f2e9af0892f64f28aded87e013a5b32425d86940b2e08cd34dad49cab0c0f9d` | + +#### 8.7.5 Case1/B256 首轮结构闭合 + +最终源码的 K Case1 闭合件为： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_073849/ + fdwic_perf_clock_kernel_summary.json +``` + +golden PASS，96×1280 Submit 与所有整数聚合均闭合： + +| 指标 | ALL | AIC | AIV | +| --- | ---: | ---: | ---: | +| 完整 Submit span | 5079.557 us（整数 tick） | - | - | +| 窗口内 Kernel calls / 合法范围 | 1013 / [256,1024] | 509 / [256,512] | 504 / [0,512] | +| Kernel core-time sum | 32.905240 ms | 18.740346 ms | 14.164894 ms | +| 非 Kernel residual core-time sum | 413.683894 ms | 132.130850 ms | 281.553044 ms | +| Kernel core-time 份额 | 7.36812% | 12.4214% | 4.7900% | +| calls/core 范围 | 3～21 | 12～21 | 3～14 | + +1013 而非 1024 表示该轮还有 11 个真实 Kernel 在对应 worker 的末次 Submit 结束后进入 FinalDrain；不能把它们补进 K 来追求“任务总数好看”。96 核均至少执行一次窗口内 Kernel，逐核 `elapsed = kernel + residual`、分组 sum 与顶层 sum 已独立从 JSON 复算通过。 + +这一轮只证明数据结构、边界和数量级可用，不能凭单样本断言 7.37% 就是权威 P 的 Kernel 墙钟比例，更不能据此直接提出 Scalar 优化。下一阶段需在冻结 K 提交后排除一次预热并连续采集足量独立 Case1，先判断 K 是否复现 P 的 M 主导、X 次要形态；同时比较 Kernel ticks、calls 和 ticks/call，避免把“更多任务迁入 Submit 窗口”误判成“单次 Kernel 变慢”。 + +K 每个已计入的 Kernel 都会新增两次 `SYS_CNT` 读取；门控判断、计数累加等开销还会落入同一 K ELF 的 residual。它们不从单轮结果中机械扣除，K 只用于同 ELF 波动载体分析，不能与 P 的字段逐项相减。 + +#### 8.7.6 当前验证记录 + +- profile/cache/compile definitions、ELF 正反门禁及 P/K JSON 末端契约定向单测：65/65 PASS； +- K B1、P B1、K Case1：golden、host raw 门禁和 Python 产物契约全部 PASS； +- Case1 JSON 的 96 核、起止、calls、Kernel tick、residual 及 AIC/AIV sum 独立复算全部闭合； +- 一次 K B1 在只重编 CMake cache、尚未通过 RuntimeBuilder 同步 `build/lib` 时仍加载旧 host SO，旧 JSON 缺少新增的全局调用下界字段，Python 末端契约按预期拒绝；该轮 `..._073315` 明确不计入有效结果。随后确认 cache 与 `build/lib` 的 host SO SHA256 同为上表值后才取得三份最终闭合件； +- 一次顺带触发的全平台 `test_runtime_builder.py` 运行中，A5 无本阶段失败，但 5 个 A2A3/A2A3sim 集成构建被该分支既有的 payload ABI 静态断言挡住；该结果不包装成本阶段 PASS，也不扩展到与真实 A5 PA 无关的修复。 + +### 8.8 同一 `perf-clock-kernel` ELF 的 Kernel、调用迁移与 residual 联合分析 + +**[波动定位，已完成；K 复现 P 的多数核共同伸缩，主要变化落在同一 K ELF 的 non-Kernel residual，不是 Kernel 数量或 Kernel core-time]** + +#### 8.8.1 冻结对象与采样协议 + +第 8.7 节提交 `a17c188a` 后没有修改源码、触发重编或穿插其他 profile。冻结身份为： + +| 对象 | 固定值 | +| --- | --- | +| commit | `a17c188a` | +| K AICore cache | `e9cebfc34cbed0e7` | +| `aicore_kernel.o` SHA256 | `8d23407aa0062534812846b7da03f5078a4843a261f134812c95f5dbc5155060` | +| `aicore_kernel.o` 大小 / `.text` | 2,495,544 B / 138,320 B | +| AICore 定义 | `PTO_FDWIC_PERF_CLOCK=1;PTO_FDWIC_PERF_CLOCK_KERNEL=1;PTO_FDWIC_TRACE_ENABLED=0` | +| `libhost_runtime.so` SHA256 | `5f2e9af0892f64f28aded87e013a5b32425d86940b2e08cd34dad49cab0c0f9d` | + +先运行一次与正式样本同参数的 Case1 预热： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_074831/ +``` + +该轮 4,480.978 us，只用于加载和预热，明确排除出统计。随后连续运行 20 个独立 pytest 进程，每个进程固定： + +```text +--case Case1 --fdwic-profile perf-clock-kernel --rounds 1 --skip-golden +``` + +没有单进程 `--rounds 20`，没有人工 sleep，没有删除快慢极值。20/20 进程 PASS，生产消费者逐份复验 1,920 条 core record、2,457,600 次 Submit；96×1280、拓扑、严格正时间窗、AIC/AIV/ALL 调用范围、逐核/分组/顶层 Kernel-residual 整数关系全部闭合。采样前后 K AICore 和 host SHA 保持不变。 + +#### 8.8.2 原始分解 + +继续严格复用第 8.5 节已经冻结的 `G=A+M+X` 定义和门禁。下表时间单位均为 us；`K mean` 与 `R mean` 分别是该轮 96 核 Kernel 和 non-Kernel residual 累计 tick 除以 96，只是逐核平均工作量，不是跨核墙钟分解： + +| 轮次 | 时间戳 | G | M | X | A | calls | K mean | R mean | +| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | `075049` | 5487.491 | 5296.392 | 179.078 | 12.021 | 999 | 342.298 | 4949.224 | +| 2 | `075146` | 4558.225 | 4395.182 | 162.861 | 0.182 | 1003 | 335.283 | 3993.544 | +| 3 | `075243` | 4801.766 | 4410.283 | 379.067 | 12.416 | 1010 | 337.500 | 4087.562 | +| 4 | `075340` | 4948.371 | 4428.318 | 516.989 | 3.065 | 1009 | 338.888 | 4144.158 | +| 5 | `075438` | 4675.254 | 4403.558 | 266.603 | 5.093 | 1008 | 339.586 | 4012.933 | +| 6 | `075537` | 4462.847 | 4374.762 | 82.975 | 5.110 | 1006 | 337.467 | 3974.188 | +| 7 | `075622` | 5850.474 | 5350.149 | 499.995 | 0.330 | 1006 | 342.335 | 4916.046 | +| 8 | `075719` | 6052.962 | 5200.194 | 839.308 | 13.461 | 1006 | 345.278 | 4878.580 | +| 9 | `075816` | 4545.193 | 4378.726 | 165.007 | 1.459 | 1009 | 337.258 | 4006.942 | +| 10 | `075921` | 4431.545 | 4374.420 | 49.245 | 7.880 | 1009 | 337.067 | 3958.067 | +| 11 | `080019` | 5107.881 | 5007.164 | 88.431 | 12.285 | 1002 | 338.038 | 4575.078 | +| 12 | `080115` | 5351.425 | 4498.411 | 839.580 | 13.434 | 1007 | 342.476 | 4353.041 | +| 13 | `080213` | 4680.613 | 4383.765 | 292.122 | 4.727 | 1011 | 338.973 | 4001.718 | +| 14 | `080309` | 5148.840 | 4856.157 | 284.580 | 8.103 | 1003 | 338.359 | 4510.296 | +| 15 | `080407` | 5723.402 | 5325.057 | 386.435 | 11.910 | 1009 | 344.642 | 4985.772 | +| 16 | `080503` | 5027.479 | 4736.360 | 289.392 | 1.726 | 998 | 340.002 | 4382.374 | +| 17 | `080602` | 6526.893 | 5702.921 | 817.173 | 6.799 | 999 | 352.976 | 5326.335 | +| 18 | `080700` | 5309.780 | 4573.770 | 728.598 | 7.412 | 1011 | 343.431 | 4270.854 | +| 19 | `080756` | 5837.718 | 5254.138 | 573.874 | 9.706 | 1001 | 351.478 | 4901.364 | +| 20 | `080901` | 4695.202 | 4449.372 | 233.099 | 12.732 | 1007 | 340.036 | 4025.199 | + +#### 8.8.3 K 是否复现 P 的波动形态 + +P10/P90 继续采用线性插值，MAD 为未缩放中位绝对偏差： + +| 指标 | 最小值 | 中位数 | 最大值 | MAD | P10 | P90 | P90-P10 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| G | 4431.545 | 5067.680 | 6526.893 | 406.119 | 4536.958 | 5870.723 | 1333.764 | +| 首核到达跨度 | 12.482 | 13.811 | 14.186 | 0.121 | 13.264 | 14.124 | 0.860 | +| M | 4374.420 | 4536.090 | 5702.921 | 161.499 | 4378.330 | 5327.566 | 949.236 | +| X | 49.245 | 290.757 | 839.580 | 165.111 | 87.886 | 819.386 | 731.501 | +| A | 0.182 | 7.646 | 13.461 | 4.478 | 1.346 | 12.802 | 11.456 | + +按第 8.5 节预先固定的“双门禁”，K 与 P 的对比如下： + +| 构建/分量 | Pearson | Spearman | P90-P10 占 G | 判定 | +| --- | ---: | ---: | ---: | --- | +| K / M | +0.917 | +0.946 | 71.2% | PASS：主要共同伸缩 | +| K / X | +0.742 | +0.755 | 54.8% | PASS：本组还有显著迁移慢尾 | +| K / A | +0.293 | +0.353 | 0.9% | FAIL | +| K / 首核到达跨度 | +0.061 | -0.021 | 0.1% | FAIL | +| P / M（第 8.5 节） | +0.943 | +0.895 | 68.5% | PASS | +| P / X（第 8.5 节） | +0.771 | +0.850 | 37.6% | 相关但幅度不足 | + +K 只**部分复现** P：最关键的 M 共同伸缩和 A 失败结论一致，因此可以在同一 K ELF 内部二分 K 自身的波动；但 K 的 X 也跨过门禁，而 P 的 X 幅度不足，不能宣称两个 ELF 的完整 A/M/X 形态相同，更不能把 K 内部归因直接外推成 P 的根因。交叉证据为： + +- 96/96 核的 body 与 G 跨轮 Spearman 不小于 0.7，中位数 +0.876； +- body 最小值与 G 的 Spearman 为 +0.922，幅度达到 G 的 79.9%； +- AIC/AIV body 中位数的跨轮 Spearman 为 +0.889； +- 每轮核内“晚到程度与 body”的 Spearman 中位数 -0.142，范围 -0.325～+0.194； +- 最晚结束核分散到 18 个不同 core，AIC/AIV 为 11/9；运行序号与 G 的 Spearman 只有 +0.304，不支持固定坏核或简单线性升温解释。 + +K 中最晚结束核与最长 body 核只有 17/20 相同，而 P 为 20/20，也再次说明 K 的尾部形态已经改变。M 与 X 的幅度比例分别以各自分布计算，不是同轮可加分量，不能把 71.2% 与 54.8% 相加成所谓“解释 126%”。 + +#### 8.8.4 同一 K ELF 内的 Kernel 与 residual + +逐轮都有精确关系： + +```text +body mean/core = Kernel mean/core + residual mean/core +``` + +但 mean/core 与 G 不是同一个跨核墙钟定义，下面只用相关性和同量纲幅度判断“波动落在哪里”，不把三者机械相减成墙钟贡献： + +| K 内部序列 | 中位数 | P90-P10 | Spearman(G, ·) | 占 G 幅度 | 双门禁 | +| --- | ---: | ---: | ---: | ---: | --- | +| body mean/core | 4654.901 us | 968.300 us | +0.941 | 72.6% | PASS | +| non-Kernel residual mean/core | 4311.948 us | 961.270 us | +0.941 | 72.1% | PASS | +| Kernel mean/core | 339.794 us | 8.659 us | +0.872 | 0.65% | FAIL：相关但幅度极小 | +| pooled Kernel 平均每调用 | 32.402 us | 0.938 us | +0.892 | 0.07% | FAIL：不是墙钟分量 | +| Kernel calls | 998～1011 次 | - | -0.392 | - | FAIL：数量与慢轮不同向 | + +角色拆分也一致：AIC/AIV residual mean/core 与 G 的 Spearman 为 +0.961/+0.937，幅度为 G 的 91.7%/66.1%，两者都通过门禁；AIC/AIV Kernel mean/core 的幅度只有 2.49%/0.34%，均不通过。逐核看，95/96 核的 residual 与 G 的 Spearman 不小于 0.7，而 96 个核没有一个 Kernel tick 序列达到 0.7。 + +总 calls 为 998～1011，与 G 的 Spearman 为 -0.392；但 AIC/AIV calls 与 G 分别为 +0.755/-0.754，方向相反。AIC pooled 单次均值约 36.30～39.28 us，AIV 约 27.54～28.42 us，而且 AIC 内部还混合 QK/PV。因此总 `ticks/call` 的同向相关至少部分受角色与任务构成变化污染，不能解释成“某个 Kernel 变慢”。 + +因此当前成立的是：**K 自身约 1.33 ms 的整轮波动主要落在 non-Kernel residual，不是窗口内 Kernel 总数量，也不由 linked-kernel mean/core 的变化幅度主导。** 这不等于 Kernel 对绝对性能没有影响，也不能反推 P 已经由 residual 主导。 + +`non_kernel_residual` 仍只是 K 窗口内的算术剩余，包含 Scalar 调度、atomic/flag 轮询与等待、Kernel 调用之间的协议路径以及 K 聚合观察开销；它不能直接重命名为 Scalar busy、atomic、I-cache miss 或频率问题。下一阶段先对 K residual 做最小侵入归因，并最终由原 P 的候选交错 A/B 决策；不能因为本轮在 K 中排除了 Kernel 变化幅度主导，就直接修改业务协议或把结论外推到 P。 + +### 8.9 独立设备状态与低功耗定性取证 + +**[取证已完成；排除约 10.24 ms 粒度的持续 DVFS/EDP 主导，不外推到更短瞬态]** + +第 8.5～8.8 节已经把 P、N、K 各自的同 ELF 波动形态摸清，但仍不能回答设备内部频率、EDP 降频或外部任务是否参与了某一批次。这里增加一次**独立、限时、只作定性判断**的设备状态取证；它不成为第四条性能证据链，也不与任何权威 perf-clock、Submit-PMU 或泳道采样同场比较。`msprof` 运行期间取得的 PA 时间一律作废。 + +#### 8.9.1 正式接口与字段语义 + +本机 CANN 9.1 的正式入口为： + +```bash +asys info -r status -d 0 +asys health -d 0 +asys profiling -r power -p -d 0 --output +``` + +本机实现确认 `asys profiling -r power` 最终调用 `msprof --sys-lp=on`。低功耗原始数据落在 `device_0/sqlite/lowpower.db` 的 `LowPower` 表；字段含义由本机 `tools/profiler/profiler_tool/analysis/viewer/stars/low_power_viewer.py` 核验： + +| 字段 | 正式含义 | 本阶段用途 | +| --- | --- | --- | +| `data7_hard` | PPU 上报的 AIC 平均频率 | 看持续频率阶跃或硬件/目标分离 | +| `data0_soft` | 软件 DVFS 下发的 AIC 频率 | 看目标频率 | +| `data2_soft` | EDP POWERBRAKE 计数 | 只看同一采样期首末增量 | +| `data3_soft`～`data5_soft` | EDP IWARNING2/1/0 计数 | 只看同一采样期首末增量 | + +该表没有 Scalar 独立频率；公开 HAL 头文件也没有 `MODULE_TYPE_SCALAR`。因此不能把 AIC 平均频率称为 Scalar 独立时钟，也不能臆造一个 Scalar 动态频率接口。N 构建中 PMU cycle 与 sys-counter 的比例仍只表示该工作负载窗口的有效 cycle/time，不是独立硬件频率读数。 + +#### 8.9.2 状态快照与告警边界 + +2026-07-21 的只读状态快照显示：设备为 `Ascend 950PR_958b V100`，功耗约 290.3～290.4 W、温度 40 C、采样时 AI Core usage 为 0%；idle 瞬时 AIC 频率为 `100, 100 MHz`。设备 health 同时为 `Alarm`，`asys health -d 0` 报告一个： + +```text +0x80b78000 | node type=SLLC | sensor type=RAS State | event state=module error +``` + +这只证明采样时存在该设备状态；当前没有证据证明 SLLC 告警导致 Submit 波动，也没有执行清告警、复位或任何配置修改。后续性能结论若仍在该状态下取得，必须保留这个环境事实，不能把它隐去或提前归因。 + +#### 8.9.3 两秒 idle probe 的原始结果 + +独立 idle 产物为： + +```text +/tmp/asys_power_idle_20260721_0815/ + asys_profiling_result_20260721082558007/ + PROF_000001_20260721082558023_03217614RIFDHMDG/ + device_0/sqlite/lowpower.db +``` + +数据库共 776 行，die 0/1/2/3 各 194 行；每个时间点四个 die 齐全。相邻时间戳的 193 个差值全部严格为 **10.240002 ms**，覆盖 **1.976320386 s**。有效 D-die 0/1 完全一致： + +| 指标 | die 0 | die 1 | +| --- | ---: | ---: | +| `data7_hard` | 首点 100 MHz，后续 193 点全为 1650 MHz | 同左 | +| `data0_soft` | 首点 100 MHz，后续 193 点全为 1650 MHz | 同左 | +| POWERBRAKE 首末增量 | 0 | 0 | +| IWARNING2/1/0 首末增量 | 0/0/0 | 0/0/0 | + +因此这条链路能看约 10 ms 以上的持续频率变化以及 EDP 计数增长，但不能解析单次约 5 ms Submit。更重要的是，纯 idle 采样也在第二个点后维持 1650 MHz，说明采集器本身或同期系统活动足以把频率拉高；不能把“采样期间为 1650 MHz”写成 PA 的独立频率结论。idle probe 只证明接口、粒度和字段可用，并且该两秒内未观察到 EDP 计数增长。 + +#### 8.9.4 K 分布存在潜在并发污染，不能继续假定独占 + +准备持续负载探针前的进程核验发现，一项 root 级系统评测从 `2026-07-21 07:17:38 UTC` 起持续运行： + +```text +python3 -m pytest test_team_evals.py ... --ascend-platform A5 ... -n 10 +``` + +它的生命周期完整覆盖 K 正式 20 轮的 `07:50:49`～`08:09:01`，其沙箱任务也在 `07:47`、`07:58`、`08:10` 等时刻陆续落盘。当前用户没有权限检查该 root 进程的设备文件描述符，因此尚未证明它实际占用 device 0；但在它退出或由外部确认不触碰 device 0 以前，同一时间段不能再写成“经过进程核验的设备独占”。这也为 K 与较早 P/N 的尾部形态不同提供了一个必须排除的环境变量，但**不是已经成立的根因**。 + +因此本阶段暂不启动新的 PA、power probe 或 phase-PMU：先等待这项潜在 A5 任务退出，再完成一次持续真实 PA 负载下的独立 power 取证。若负载期 10.24 ms 采样仍稳定在目标频率且 EDP 计数不增长，只能排除“粗粒度持续 DVFS/EDP”为主因，仍不能排除短于一个采样周期的瞬态。随后立即回到 Materialize 同 ELF 单阶段 PMU，不把低功耗采样扩张成长期路线。 + +#### 8.9.5 持续真实 PA 负载探针与路线收口 + +root 级系统评测在 `09:07 UTC` 退出。后续每个正式样本前后均未再发现该进程；本机又没有契约完整且已安装的 device-PID 枚举 CLI，所以这里准确表述为“已排除已知并发评测”，不把普通用户无法查看 root 文件描述符包装成更强的设备独占证明。 + +第一次主动负载尝试保留为失败记录：`/tmp/fdwic_power_probe_20260721_090907.log` 中的 100 轮真实 PA 本身 PASS，但探测脚本错误地只等待 `/dev/davinci0`；实际 Python 进程只持有 `/dev/davinci_manager`，所以脚本没有启动 power 采集。随后手工启动的 `/tmp/asys_power_pa_active_20260721_091152/` 已落在负载结束之后，不进入负载结论。没有用这份错位数据凑数。 + +第二次改为复用真实 PA 日志中已经存在的 AICPU 初始化标志，不再猜设备文件。20 轮无泳道、无 PMU 的 Case1 在 `09:13:14.721` 出现初始化标志，power 采集于 `09:13:15.191` 开始并持续约 3 s；测试随后完成 20/20 轮，单轮 device 时间为 77.206～84.442 ms，因此采集窗口与真实负载明确重叠。PA 时间因与 `msprof` 同场而全部作废，只保留低功耗字段： + +```text +/tmp/fdwic_power_probe_aligned_20260721_091233.log +/tmp/asys_power_pa_aligned_20260721_091233/ + asys_profiling_result_20260721091315170/ + PROF_000001_20260721091315186_03293075NQEEOOGB/ + device_0/sqlite/lowpower.db +``` + +die 0/1 各有 291 个点，290 个间隔全部为 10.240002 ms，覆盖 2.969600580 s；两 die 的 `data7_hard` 和 `data0_soft` 每一点均为 1650 MHz，POWERBRAKE 与 IWARNING2/1/0 每一点均为 0。由此可以排除该 3 s 负载窗内可被 10.24 ms 采样解析的持续频率阶跃和 EDP 计数增长。它仍不能排除单个约 5 ms Submit 内更短的瞬态，也不能证明当前 SLLC Major/module-error 告警与波动无关。至此停止扩展 power 路线，后续性能样本不再与 `asys/msprof` 同场。 + +### 8.10 同一 Materialize ELF 的阶段与其余 Submit 波动分解 + +**[阶段归因已完成；Materialize 不通过主载体门禁，波动落在同核其余 Submit]** + +#### 8.10.1 冻结对象与采样协议 + +本阶段直接复用已经闭合的 `submit-pmu-materialize`，不增加 selector、raw 字段或逐事件记录。它精确覆盖 task-cap 检查和 `dist_submit_materialize_args()`，排除后继 PrepareMap；Case1 每核固定 1,280 次 begin/end。正式取数前后身份完全一致： + +| 对象 | 固定值 | +| --- | --- | +| commit | `6acebc8fca0b053a2c014cf40ffed0dced17d1fa` | +| AICore cache | `b28cf51da4d4f547` | +| AICore 定义 | `PTO_FDWIC_SUBMIT_PMU=1;PTO_FDWIC_SUBMIT_PMU_PHASE_ID=3;PTO_FDWIC_TRACE_ENABLED=0` | +| `aicore_kernel.o` | 2,499,680 B / SHA256 `6fea16b12c3f3dd46fa8417969042fa5114e0d06ddb3c3dea0efe3d30254a8b0` | +| AIC/AIV combined | `afa5206a2a5f8cb9a7f6fa9992f8531bb04b92b21791bbdd0c8599bc1bd42584` / `31a1ef188cb27718cbba3aab7f939a7dc823975049b49c57ea05c39c5fef0061` | +| `libhost_runtime.so` | SHA256 `5f2e9af0892f64f28aded87e013a5b32425d86940b2e08cd34dad49cab0c0f9d` | + +power 探针结束后先运行一轮同参数 Case1 重新预热： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_091623/ +``` + +该轮明确排除出统计。随后连续运行 12 个彼此独立的 pytest 进程，每个进程固定 `--case Case1 --fdwic-profile submit-pmu-materialize --rounds 1 --skip-golden`；没有单进程多轮、人工 sleep 或极值删除。12/12 均 PASS： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_091706/ +outputs/TestPagedAttentionUnroll_Case1_20260721_091750/ +outputs/TestPagedAttentionUnroll_Case1_20260721_091835/ +outputs/TestPagedAttentionUnroll_Case1_20260721_091919/ +outputs/TestPagedAttentionUnroll_Case1_20260721_092002/ +outputs/TestPagedAttentionUnroll_Case1_20260721_092045/ +outputs/TestPagedAttentionUnroll_Case1_20260721_092128/ +outputs/TestPagedAttentionUnroll_Case1_20260721_092213/ +outputs/TestPagedAttentionUnroll_Case1_20260721_092258/ +outputs/TestPagedAttentionUnroll_Case1_20260721_092342/ +outputs/TestPagedAttentionUnroll_Case1_20260721_092425/ +outputs/TestPagedAttentionUnroll_Case1_20260721_092508/ +``` + +生产 `load_capture()` 逐份复验通过：12 份均为 96 核、32 AIC + 64 AIV、每核 1,280 次 Submit 和 Materialize begin/end，1,152 条逐核记录全部满足 status `0x7ff`、phase status `0x3f`、owner 配置/恢复、拓扑、计数顺序和风险阈值；primary/shadow request/miss 1,152/1,152 精确相等。12 份 HTML 也与当前 `render_report(raw)` 逐字节一致。总计覆盖 1,474,560 次 Submit/Materialize 调用。 + +#### 8.10.2 整数分解与 12 轮原始结果 + +继续沿用第 8.5 节定义。对每轮最晚结束核 `z`，再把它自己的 body 拆为： + +```text +Pz = phase_elapsed_ticks[z] +Rz = submit_elapsed_ticks[z] - Pz +G = A + Pz + Rz +``` + +`Pz/Rz` 只在同一 Materialize ELF、同一核、同一轮中相加；不会把 phase core-time 或其他 ELF 的时间拿来扣墙钟。下表时间均为 us： + +| 轮次 | 时间戳 | G | M | X | A | Pz | Rz | 最晚核 | +| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| 1 | `091706` | 5129.351 | 4584.886 | 534.023 | 10.442 | 1020.215 | 4098.694 | c18/AIC/b18/l0 | +| 2 | `091750` | 4890.960 | 4575.058 | 311.516 | 4.386 | 1005.842 | 3880.732 | c22/AIC/b22/l0 | +| 3 | `091835` | 4927.376 | 4596.597 | 330.618 | 0.161 | 970.163 | 3957.052 | c25/AIC/b25/l0 | +| 4 | `091919` | 4733.656 | 4594.046 | 129.707 | 9.904 | 1002.399 | 3721.353 | c16/AIC/b16/l0 | +| 5 | `092002` | 4665.849 | 4577.212 | 84.337 | 4.300 | 1020.395 | 3641.154 | c71/AIV/b19/l2 | +| 6 | `092045` | 4659.749 | 4579.614 | 72.424 | 7.711 | 1022.620 | 3629.418 | c41/AIV/b4/l2 | +| 7 | `092128` | 5088.129 | 4831.484 | 256.510 | 0.136 | 1032.260 | 4055.733 | c45/AIV/b6/l2 | +| 8 | `092213` | 4943.694 | 4593.721 | 344.279 | 5.693 | 1002.572 | 3935.429 | c27/AIC/b27/l0 | +| 9 | `092258` | 4653.506 | 4586.275 | 58.016 | 9.215 | 1056.466 | 3587.825 | c70/AIV/b19/l1 | +| 10 | `092342` | 4708.335 | 4595.176 | 103.877 | 9.281 | 1022.974 | 3676.080 | c72/AIV/b20/l1 | +| 11 | `092425` | 4731.198 | 4595.085 | 127.303 | 8.810 | 991.187 | 3731.201 | c11/AIC/b11/l0 | +| 12 | `092508` | 4755.224 | 4583.503 | 161.814 | 9.907 | 1010.032 | 3735.285 | c28/AIC/b28/l0 | + +12 轮最晚核分散在 12 个不同 core，AIC/AIV 为 7/5，不支持固定坏核解释。分布使用整数 tick 派生，MAD 为未缩放中位绝对偏差，P10/P90 采用线性插值： + +| 指标 | 最小值 | 中位数 | 最大值 | MAD | P10 | P90 | P90-P10 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| G | 4653.506 | 4744.440 | 5129.351 | 87.813 | 4660.359 | 5073.686 | 413.327 | +| M | 4575.058 | 4589.998 | 4831.484 | 5.836 | 4577.452 | 4596.455 | 19.003 | +| X | 58.016 | 145.760 | 534.023 | 80.540 | 73.616 | 342.913 | 269.298 | +| A | 0.136 | 8.261 | 10.442 | 1.914 | 0.575 | 9.907 | 9.332 | +| Pz | 970.163 | 1015.124 | 1056.466 | 10.917 | 992.308 | 1031.331 | 39.023 | +| Rz | 3587.825 | 3733.243 | 4098.694 | 124.621 | 3630.592 | 4045.865 | 415.273 | +| Materialize mean/core | 1020.461 | 1021.320 | 1022.028 | 0.234 | 1020.962 | 1021.957 | 0.994 | +| 其余 Submit mean/core | 3500.079 | 3523.462 | 3834.918 | 9.251 | 3506.308 | 3539.937 | 33.629 | + +#### 8.10.3 主载体门禁与结论边界 + +沿用预先固定的双门禁：`abs(Spearman(G, component)) >= 0.7`，且 component 的 `P90-P10 >= 50% * G(P90-P10)`： + +| 分量 | Spearman(G, ·) | P90-P10 | 占 G 波幅 | 判定 | +| --- | ---: | ---: | ---: | --- | +| M | +0.287 | 19.003 us | 4.60% | FAIL | +| X | +0.958 | 269.298 us | 65.15% | PASS：该 ELF 为慢尾形态 | +| A | -0.105 | 9.332 us | 2.26% | FAIL | +| Pz | -0.343 | 39.023 us | 9.44% | **FAIL：Materialize 不是主载体** | +| Rz | +0.986 | 415.273 us | 100.47% | **PASS：变化落在同核其余 Submit** | +| Materialize mean/core | +0.091 | 0.994 us | 0.24% | FAIL | +| 其余 Submit mean/core | +0.720 | 33.629 us | 8.14% | FAIL：相关但整体幅度不足 | + +因此本阶段成立的结论是：**在当前 Materialize 诊断 ELF 中，最晚核的波动不由 Materialize 携带，而由该核其余 Submit 路径携带。** Materialize 每核平均 core-time 几乎不动，12 轮 P90-P10 只有 0.994 us；每次 request 中位 233.061、范围 232.911～233.243，每次 miss 中位 1.438、范围 1.424～1.447，同样没有随 G 同向的大幅变化。它在同 ELF 中仍占约 22.47% core-time，说明绝对成本重要，但“占比大” 不能替代波动门禁，也不能据此先改固定扫描逻辑。 + +本组 G 的 P90-P10 为 413.327 us，已有可分辨波形，而 Pz 的相关性和幅度都远离门槛，因此不扩到 20 轮。更重要的是，本组由 X 而不是 M 通过门禁，没有复现权威 P 的多数核 body 共同伸缩形态；所以 Rz 结论只限定于 Materialize ELF，不能直接外推成 P 的根因或可兑现收益。下一阶段按既定顺序切换到现有 `submit-pmu-claim`，优先检查包含 return-ready atomic 的 Claim 是否携带同 ELF 波动；仍不修改业务代码。 + +### 8.11 同一 Claim ELF 的阶段与其余 Submit 波动分解 + +**[阶段归因已完成；Claim 不通过主载体门禁，角色差异不能冒充跨轮波动]** + +#### 8.11.1 B1、冻结对象与正式样本 + +本阶段复用第 8.3.5 节的 `submit-pmu-claim`，边界为 `claim_begin_to_claim_end`。它包含 task-cap 检查、实际 Claim 和 flags 构造；真实 FetchMax 返回等待位于该业务区间内，但诊断 ELF 编译掉 atomic 泳道观察代码。每个 Submit 固定进入一次外层 Claim，所以 Case1 shape 仍为每核 1,280 对 begin/end，不增加 winner 或 atomic 动态字段。 + +文档提交改变 source-v2 HEAD 后，先在新构建上运行 B1： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_093141/ +``` + +结果为 96 核各 5 次 Submit/Claim、phase id `4`、status `0x3f`、primary/shadow 96/96 精确相等，HTML 与当前 renderer 字节一致；全局 243.386 us 仍只作结构证据。随后冻结： + +| 对象 | 固定值 | +| --- | --- | +| commit | `52dc04c792cd5f6982fe7a6d1272dcc4f43231bc` | +| AICore cache | `88d4ef05843d4904` | +| AICore 定义 | `PTO_FDWIC_SUBMIT_PMU=1;PTO_FDWIC_SUBMIT_PMU_PHASE_ID=4;PTO_FDWIC_TRACE_ENABLED=0` | +| `aicore_kernel.o` | 2,590,880 B / `.text` 154,192 B / SHA256 `05f0c6e8edff1319770f9117ea17efb8c81a8822d6cbdaa199ebd339fb039703` | +| AIC/AIV combined | `fd0fa24ac1b117d4705f6405dc60497992efef17f71f268d46326e5fc2a1ac29` / `6a7bdf0221e15d750e9b531e3083caf728ad0527779e642bef41ef7f388e68f7` | +| `libhost_runtime.so` | SHA256 `5f2e9af0892f64f28aded87e013a5b32425d86940b2e08cd34dad49cab0c0f9d` | + +Case1 预热件 `..._093327` 明确排除。12 个独立正式进程仍固定 `--rounds 1 --skip-golden`，没有 sleep、极值删除或其他 profile 穿插： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_093412/ +outputs/TestPagedAttentionUnroll_Case1_20260721_093454/ +outputs/TestPagedAttentionUnroll_Case1_20260721_093536/ +outputs/TestPagedAttentionUnroll_Case1_20260721_093620/ +outputs/TestPagedAttentionUnroll_Case1_20260721_093703/ +outputs/TestPagedAttentionUnroll_Case1_20260721_093746/ +outputs/TestPagedAttentionUnroll_Case1_20260721_093829/ +outputs/TestPagedAttentionUnroll_Case1_20260721_093912/ +outputs/TestPagedAttentionUnroll_Case1_20260721_093954/ +outputs/TestPagedAttentionUnroll_Case1_20260721_094037/ +outputs/TestPagedAttentionUnroll_Case1_20260721_094121/ +outputs/TestPagedAttentionUnroll_Case1_20260721_094203/ +``` + +12/12 pytest、生产 `load_capture()` 与 HTML 精确重建全部 PASS；1,152 条逐核记录均为 96×1280、phase id `4`、phase status `0x3f`、primary=shadow，owner、拓扑、计数顺序、phase 时间和风险阈值闭合。采样前后上述 AIC/AIV/host SHA 不变，已知 root A5 评测也未重新出现。 + +#### 8.11.2 原始分解与角色差异 + +继续定义 `Pz=Claim[z]`、`Rz=body[z]-Pz`，逐轮精确满足 `G=A+Pz+Rz=A+M+X`。下表单位均为 us： + +| 轮次 | 时间戳 | G | M | X | A | Pz | Rz | 最晚核 | +| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| 1 | `093412` | 4620.233 | 4556.721 | 59.596 | 3.915 | 1024.912 | 3591.406 | c79/AIV/b23/l2 | +| 2 | `093454` | 4719.079 | 4524.864 | 181.847 | 12.368 | 415.432 | 4291.279 | c2/AIC/b2/l0 | +| 3 | `093536` | 4676.573 | 4535.911 | 140.432 | 0.230 | 370.210 | 4306.133 | c28/AIC/b28/l0 | +| 4 | `093620` | 4605.915 | 4542.673 | 62.686 | 0.556 | 1011.722 | 3593.637 | c66/AIV/b17/l1 | +| 5 | `093703` | 5015.477 | 4536.442 | 477.636 | 1.399 | 384.459 | 4629.619 | c24/AIC/b24/l0 | +| 6 | `093746` | 4952.543 | 4697.900 | 249.944 | 4.699 | 979.619 | 3968.225 | c39/AIV/b3/l2 | +| 7 | `093829` | 4657.082 | 4534.479 | 122.496 | 0.107 | 397.740 | 4259.235 | c22/AIC/b22/l0 | +| 8 | `093912` | 4676.111 | 4554.137 | 111.189 | 10.785 | 386.509 | 4278.817 | c26/AIC/b26/l0 | +| 9 | `093954` | 4726.002 | 4533.742 | 188.481 | 3.780 | 386.908 | 4335.314 | c24/AIC/b24/l0 | +| 10 | `094037` | 5238.567 | 4702.629 | 535.607 | 0.331 | 401.741 | 4836.495 | c23/AIC/b23/l0 | +| 11 | `094121` | 4725.767 | 4554.668 | 161.607 | 9.492 | 394.837 | 4321.438 | c0/AIC/b0/l0 | +| 12 | `094203` | 4633.868 | 4546.530 | 87.018 | 0.320 | 382.272 | 4251.276 | c31/AIC/b31/l0 | + +最晚核分散到 11 个 core，AIC/AIV 为 9/3。`Pz` 的 370～1,025 us 大范围主要来自最晚核角色切换：三轮 AIV 均约 980～1,025 us，九轮 AIC 均约 370～415 us。这是同一 Claim ELF 内真实存在的角色成本差异，但它不随 G 单调变化；不能因数值范围大就称 Claim 携带跨轮波动。完整分布为： + +| 指标 | 最小值 | 中位数 | 最大值 | MAD | P10 | P90 | P90-P10 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| G | 4605.915 | 4697.826 | 5238.567 | 52.351 | 4621.596 | 5009.184 | 387.587 | +| M | 4524.864 | 4544.601 | 4702.629 | 10.095 | 4533.815 | 4683.782 | 149.967 | +| X | 59.596 | 151.019 | 535.607 | 51.916 | 65.119 | 454.867 | 389.748 | +| A | 0.107 | 2.590 | 12.368 | 2.264 | 0.239 | 10.656 | 10.417 | +| Pz | 370.210 | 396.288 | 1024.912 | 12.923 | 382.491 | 1008.512 | 626.021 | +| Rz | 3591.406 | 4285.048 | 4836.495 | 43.328 | 3631.096 | 4600.189 | 969.093 | +| Claim mean/core | 778.854 | 830.892 | 856.279 | 7.354 | 813.978 | 842.240 | 28.262 | +| 其余 Submit mean/core | 3627.936 | 3676.057 | 3924.426 | 23.765 | 3642.722 | 3902.263 | 259.541 | + +#### 8.11.3 主载体门禁与阶段决定 + +| 分量 | Spearman(G, ·) | P90-P10 | 占 G 波幅 | 判定 | +| --- | ---: | ---: | ---: | --- | +| M | +0.154 | 149.967 us | 38.69% | FAIL | +| X | +0.979 | 389.748 us | 100.56% | PASS：仍为慢尾形态 | +| A | +0.175 | 10.417 us | 2.69% | FAIL | +| Pz | -0.189 | 626.021 us | 161.52% | **FAIL：幅度大但方向不相关** | +| Rz | +0.790 | 969.093 us | 250.03% | **PASS：同核其余 Submit 携带变化** | +| Claim mean/core | -0.476 | 28.262 us | 7.29% | FAIL | +| 其余 Submit mean/core | +0.448 | 259.541 us | 66.96% | FAIL：幅度够、相关性不足 | + +`Pz/Rz` 因 AIC/AIV 角色成本相反补偿，单个分量波幅可以大于 G；二者仍只按每轮同核整数关系闭合，不能把 161.52% 与 250.03% 相加。主门禁要求相关性和幅度同时成立，因此 Claim 明确失败，不能把 ClaimMax/return-ready atomic 当作本组慢轮原因。 + +Claim 在同 ELF 中的 core-time 份额中位为 18.40%，每次 request 中位 80.171、范围 80.097～80.236，每次 miss 中位 1.950、范围 1.948～1.951；这些稳定值说明 Claim 仍是绝对成本区域，但没有提供“慢轮执行了更多 Claim 取指或 miss”的证据。本组 G 有 387.587 us 的可分辨波形，Pz 相关性又远离阈值，所以不扩到 20 轮。其 M 仍未复现权威 P 的共同伸缩，结论继续严格限定在 Claim ELF。下一阶段切换到已经存在的 `submit-pmu-submit-transition`，检查相邻 Submit 之间的动态控制/回放间隙；不修改 Claim 或 atomic 业务逻辑。 + +### 8.12 同一 SubmitTransition ELF 的阶段与其余 Submit 波动分解 + +**[阶段归因已完成；相邻 Submit 间隙不通过主载体门禁]** + +#### 8.12.1 N-1 结构闭合与冻结样本 + +`submit-pmu-submit-transition` 复用统一 Submit hook，聚合上一次 Submit end 到下一次 `dist_submit_begin()` 完成之间的返回、编排衔接和下一任务准备；它不生成末次 Submit 之后的伪间隙。因此 B1 每核固定 4 次、Case1 每核固定 1,279 次，不能沿用普通 phase 的 N 次 shape，也不能从聚合 raw 中还原不同 task-kind 转换。 + +新 HEAD 的 B1 为： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_094639/ +``` + +96 核均为 5 个 Submit、4 对 transition，phase id `6`、status `0x3f`、primary/shadow 96/96 精确相等，HTML 精确重建；258.277 us 只作结构证据。冻结身份为： + +| 对象 | 固定值 | +| --- | --- | +| commit | `7fa7399f57c0bb490cb762b74a0e184c8f2bdcf4` | +| AICore cache | `d17b87d79477c0e1` | +| AICore 定义 | `PTO_FDWIC_SUBMIT_PMU=1;PTO_FDWIC_SUBMIT_PMU_PHASE_ID=6;PTO_FDWIC_TRACE_ENABLED=0` | +| `aicore_kernel.o` | 2,584,560 B / `.text` 153,424 B / SHA256 `a5f96fa4c78cfd038cf7ef2bbc1f2c811cd4a2cf1cba6466b867ce24d49543f9` | +| AIC/AIV combined | `0badb10df88e36cb0782c6e0fee3787972cf8a403a4f4f671e70f716ab6a6be5` / `754510d18b6e09a41be7c5878e7f1cdf802e7b1f60bc83d035f341ad3475b394` | +| `libhost_runtime.so` | SHA256 `5f2e9af0892f64f28aded87e013a5b32425d86940b2e08cd34dad49cab0c0f9d` | + +Case1 预热 `..._094821` 排除后，12 个独立正式进程为： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_094903/ +outputs/TestPagedAttentionUnroll_Case1_20260721_094948/ +outputs/TestPagedAttentionUnroll_Case1_20260721_095032/ +outputs/TestPagedAttentionUnroll_Case1_20260721_095117/ +outputs/TestPagedAttentionUnroll_Case1_20260721_095202/ +outputs/TestPagedAttentionUnroll_Case1_20260721_095247/ +outputs/TestPagedAttentionUnroll_Case1_20260721_095330/ +outputs/TestPagedAttentionUnroll_Case1_20260721_095415/ +outputs/TestPagedAttentionUnroll_Case1_20260721_095459/ +outputs/TestPagedAttentionUnroll_Case1_20260721_095544/ +outputs/TestPagedAttentionUnroll_Case1_20260721_095628/ +outputs/TestPagedAttentionUnroll_Case1_20260721_095712/ +``` + +12/12 pytest、生产 raw 消费和 HTML 重建均 PASS；1,152 条逐核记录全部满足 96×1280 Submit、每核 1,279 次 begin/end、phase id `6`、phase status `0x3f`、primary=shadow、owner/拓扑/计数和时间门禁，共覆盖 1,473,408 个真实相邻间隙。采样前后对象 SHA 不变，已知 root A5 评测未重新出现。 + +#### 8.12.2 原始分解与角色分层 + +继续以最晚核 `z` 定义 `Pz=Transition[z]` 和 `Rz=body[z]-Pz`。下表单位均为 us： + +| 轮次 | 时间戳 | G | M | X | A | Pz | Rz | 最晚核 | +| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| 1 | `094903` | 4706.898 | 4589.310 | 110.019 | 7.569 | 523.401 | 4175.928 | c33/AIV/b0/l2 | +| 2 | `094948` | 4772.953 | 4585.756 | 186.890 | 0.307 | 483.429 | 4289.217 | c62/AIV/b15/l1 | +| 3 | `095032` | 4944.667 | 4554.933 | 389.734 | 0.000 | 385.113 | 4559.554 | c23/AIC/b23/l0 | +| 4 | `095117` | 4665.650 | 4561.532 | 98.951 | 5.168 | 521.598 | 4138.884 | c89/AIV/b28/l2 | +| 5 | `095202` | 4984.704 | 4567.462 | 416.864 | 0.378 | 363.112 | 4621.214 | c28/AIC/b28/l0 | +| 6 | `095247` | 5136.771 | 4559.525 | 571.823 | 5.423 | 421.619 | 4709.729 | c26/AIC/b26/l0 | +| 7 | `095330` | 4685.859 | 4586.337 | 87.935 | 11.587 | 460.224 | 4214.048 | c81/AIV/b24/l2 | +| 8 | `095415` | 4907.677 | 4597.949 | 305.505 | 4.223 | 374.434 | 4529.020 | c15/AIC/b15/l0 | +| 9 | `095459` | 4965.656 | 4578.873 | 381.935 | 4.848 | 420.177 | 4540.631 | c26/AIC/b26/l0 | +| 10 | `095544` | 4706.148 | 4559.293 | 139.362 | 7.493 | 381.920 | 4316.735 | c23/AIC/b23/l0 | +| 11 | `095628` | 4758.772 | 4581.688 | 169.271 | 7.813 | 458.108 | 4292.851 | c64/AIV/b16/l1 | +| 12 | `095712` | 4643.497 | 4549.686 | 92.862 | 0.949 | 456.329 | 4186.219 | c94/AIV/b31/l1 | + +最晚核分散到 10 个 core，AIC/AIV 各 6 轮。AIC 的 `Pz` 为 363～422 us，AIV 为 456～523 us；整体 `rho=-0.531` 主要受“慢轮更常由 Pz 较短的 AIC 收尾”影响。角色内 `Spearman(G,Pz)` 只有 AIC `+0.371`、AIV `+0.257`，均不支持 transition 随慢轮增长。完整分布为： + +| 指标 | 最小值 | 中位数 | 最大值 | MAD | P10 | P90 | P90-P10 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| G | 4643.497 | 4765.863 | 5136.771 | 111.289 | 4667.671 | 4982.799 | 315.128 | +| M | 4549.686 | 4573.168 | 4597.949 | 13.406 | 4555.369 | 4589.013 | 33.644 | +| X | 87.935 | 178.080 | 571.823 | 87.682 | 93.470 | 414.151 | 320.681 | +| A | 0.000 | 5.008 | 11.587 | 2.683 | 0.314 | 7.789 | 7.474 | +| Pz | 363.112 | 438.974 | 523.401 | 49.158 | 375.183 | 517.781 | 142.599 | +| Rz | 4138.884 | 4304.793 | 4709.729 | 147.387 | 4176.957 | 4615.048 | 438.091 | +| Transition mean/core | 447.180 | 448.720 | 451.075 | 0.553 | 447.434 | 449.755 | 2.321 | +| 其余 Submit mean/core | 4047.639 | 4087.141 | 4170.705 | 16.499 | 4061.148 | 4121.627 | 60.479 | + +#### 8.12.3 门禁与阶段决定 + +| 分量 | Spearman(G, ·) | P90-P10 | 占 G 波幅 | 判定 | +| --- | ---: | ---: | ---: | --- | +| M | +0.070 | 33.644 us | 10.68% | FAIL | +| X | +0.965 | 320.681 us | 101.76% | PASS：仍为慢尾形态 | +| A | -0.343 | 7.474 us | 2.37% | FAIL | +| Pz | -0.531 | 142.599 us | 45.25% | **FAIL：相关性和幅度均不足** | +| Rz | +0.895 | 438.091 us | 139.02% | **PASS：同核其余 Submit 携带变化** | +| Transition mean/core | -0.175 | 2.321 us | 0.74% | FAIL | +| 其余 Submit mean/core | +0.399 | 60.479 us | 19.19% | FAIL | + +Pz 的幅度比例接近但仍低于 50%，相关性又只有 -0.531；角色分层后相关性更低，Transition mean/core 也几乎不动。因此当前不是样本不足导致的临界结论，不扩到 20 轮。该 ELF 内 transition core-time 份额中位为 9.90%，每间隙 request 中位 134.226、范围 134.085～134.310，每间隙 miss 中位 4.375、范围 4.278～4.553；I-cache observed 占比高不等于它携带跨轮时延，更不能换算成可相减的墙钟损失。 + +本组仍由 X 而不是 M 通过，不能外推权威 P。下一阶段依序复用现有 `submit-pmu-arg-build`，检查 Claim.end 到 Materialize.begin 的同步构参与恢复路径；不修改 transition 或编排业务。 + +### 8.13 同一 ArgBuild ELF 的阶段与其余 Submit 波动分解 + +**[阶段归因已完成；ArgBuild 不通过主载体门禁]** + +#### 8.13.1 B1、冻结对象与正式样本 + +`submit-pmu-arg-build` 的边界为 `claim_end_to_materialize_begin`：起点在真实 Claim 完成后，终点在匹配 Finish 恢复并校验 ticket 后、Materialize 入口前。它覆盖 Begin 返回、同步 eager callback 构参和 Finish 重入，不包含 Claim 与 Materialize。本阶段直接复用已经闭合的 phase id `1` 和 12,416 B 固定 ABI，没有增加 raw 字段。 + +先用 B1 验证每核 5 对边界： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_100220/ +``` + +96 核均为 5 次 Submit/ArgBuild，phase status `0x3f`、primary/shadow 精确相等，HTML 精确重建；246.495 us 只作结构证据。随后冻结对象： + +| 对象 | 固定值 | +| --- | --- | +| commit | `afeb515aaed182200d62f74f4cba0797aceeb2c2` | +| AICore cache | `8d3a86ebd090fe15` | +| AICore 定义 | `PTO_FDWIC_SUBMIT_PMU=1;PTO_FDWIC_SUBMIT_PMU_PHASE_ID=1;PTO_FDWIC_TRACE_ENABLED=0` | +| `aicore_kernel.o` | 2,567,472 B / `.text` 149,328 B / SHA256 `5172a04d0c2ffa05a511f9807b28fcf8be81d4a05783f095e036ba807238eea0` | +| AIC/AIV combined | `1dc77366f67f919594af753c5119b0c49af434cd30501c09e67be03e548cb20f` / `757c9db2528916c77443d6bcd197a5d45848ebf75c064ac874fbff52e445deb9` | +| `libhost_runtime.so` | SHA256 `5f2e9af0892f64f28aded87e013a5b32425d86940b2e08cd34dad49cab0c0f9d` | + +Case1 预热件 `..._100401` 明确排除。随后连续运行 12 个彼此独立的 pytest 进程，每个固定 `--rounds 1 --skip-golden`，没有 sleep、极值删除或其他 profile 穿插： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_100445/ +outputs/TestPagedAttentionUnroll_Case1_20260721_100529/ +outputs/TestPagedAttentionUnroll_Case1_20260721_100613/ +outputs/TestPagedAttentionUnroll_Case1_20260721_100656/ +outputs/TestPagedAttentionUnroll_Case1_20260721_100739/ +outputs/TestPagedAttentionUnroll_Case1_20260721_100824/ +outputs/TestPagedAttentionUnroll_Case1_20260721_100909/ +outputs/TestPagedAttentionUnroll_Case1_20260721_100953/ +outputs/TestPagedAttentionUnroll_Case1_20260721_101036/ +outputs/TestPagedAttentionUnroll_Case1_20260721_101121/ +outputs/TestPagedAttentionUnroll_Case1_20260721_101204/ +outputs/TestPagedAttentionUnroll_Case1_20260721_101249/ +``` + +12/12 pytest、生产 `load_capture()` 和 HTML 重建全部 PASS。1,152 条逐核记录均为 96×1,280、phase id `1`、phase status `0x3f`、owner 配置/恢复 96/96、计数低于风险阈值；primary/shadow request/miss 1,152/1,152 逐核精确相等。共覆盖 1,474,560 次真实 ArgBuild 边界。raw 能证明 profile、shape、状态、拓扑、owner 和计数闭合；同一 ELF/HEAD 仍由上述外部冻结 SHA 证明，不能倒过来从 raw 杜撰。 + +#### 8.13.2 整数分解与角色双峰 + +继续以最晚结束核 `z` 定义 `Pz=ArgBuild[z]`、`Rz=body[z]-Pz`，逐轮精确满足 `G=A+M+X=A+Pz+Rz`。下表单位均为 us： + +| 轮次 | 时间戳 | G | M | X | A | Pz | Rz | 最晚核 | +| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| 1 | `100445` | 4999.071 | 4549.461 | 439.759 | 9.851 | 198.725 | 4790.495 | c28/AIC/b28/l0 | +| 2 | `100529` | 4724.531 | 4557.943 | 160.395 | 6.193 | 292.721 | 4425.617 | c67/AIV/b17/l2 | +| 3 | `100613` | 4826.878 | 4637.162 | 178.973 | 10.743 | 276.317 | 4539.818 | c76/AIV/b22/l1 | +| 4 | `100656` | 4824.865 | 4577.124 | 235.010 | 12.731 | 199.131 | 4613.003 | c23/AIC/b23/l0 | +| 5 | `100739` | 4968.121 | 4543.257 | 412.807 | 12.057 | 199.732 | 4756.332 | c23/AIC/b23/l0 | +| 6 | `100824` | 4900.956 | 4548.284 | 342.089 | 10.583 | 198.530 | 4691.843 | c20/AIC/b20/l0 | +| 7 | `100909` | 4689.894 | 4575.105 | 100.829 | 13.959 | 279.883 | 4396.052 | c89/AIV/b28/l2 | +| 8 | `100953` | 4647.705 | 4565.437 | 71.148 | 11.120 | 286.475 | 4350.110 | c90/AIV/b29/l1 | +| 9 | `101036` | 4794.409 | 4594.467 | 199.771 | 0.171 | 198.698 | 4595.540 | c31/AIC/b31/l0 | +| 10 | `101121` | 4727.365 | 4589.965 | 129.642 | 7.758 | 298.120 | 4421.487 | c78/AIV/b23/l1 | +| 11 | `101204` | 4624.150 | 4556.818 | 66.997 | 0.335 | 198.700 | 4425.115 | c28/AIC/b28/l0 | +| 12 | `101249` | 4664.981 | 4554.060 | 100.759 | 10.162 | 198.868 | 4455.951 | c6/AIC/b6/l0 | + +最晚核分散到 10 个 core，AIC/AIV 为 7/5。`Pz` 呈现约 199 us 的 AIC 与 276～298 us 的 AIV 双峰；按角色复算 `Spearman(G,Pz)` 也只有 AIC `+0.214`、AIV `-0.100`，而对应 `Rz` 为 `+1.000/+0.900`。角色双峰不是 ArgBuild 携带跨轮波动的证据。完整分布为： + +| 指标 | 最小值 | 中位数 | 最大值 | MAD | P10 | P90 | P90-P10 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| G | 4624.150 | 4760.887 | 4999.071 | 83.450 | 4649.433 | 4961.405 | 311.972 | +| M | 4543.257 | 4561.690 | 4637.162 | 13.411 | 4548.402 | 4594.017 | 45.615 | +| X | 66.997 | 169.684 | 439.759 | 68.890 | 74.109 | 405.735 | 331.626 | +| A | 0.171 | 10.373 | 13.959 | 2.022 | 0.921 | 12.664 | 11.743 | +| Pz | 198.530 | 199.432 | 298.120 | 0.818 | 198.698 | 292.096 | 93.398 | +| Rz | 4350.110 | 4497.885 | 4790.495 | 99.744 | 4398.596 | 4749.883 | 351.288 | +| ArgBuild mean/core | 257.919 | 258.627 | 261.132 | 0.452 | 258.188 | 260.556 | 2.368 | +| 其余 Submit mean/core | 4246.566 | 4266.118 | 4387.781 | 17.411 | 4247.448 | 4328.366 | 80.918 | + +#### 8.13.3 门禁、I-cache 观察值与结论边界 + +| 分量 | Spearman(G, ·) | P90-P10 | 占 G 波幅 | 判定 | +| --- | ---: | ---: | ---: | --- | +| M | -0.210 | 45.615 us | 14.62% | FAIL | +| X | +0.972 | 331.626 us | 106.30% | PASS：迁移慢尾形态 | +| A | +0.175 | 11.743 us | 3.76% | FAIL | +| Pz | -0.259 | 93.398 us | 29.94% | **FAIL：ArgBuild 不是主载体** | +| Rz | +0.867 | 351.288 us | 112.60% | **PASS：同核其余 Submit 携带变化** | +| ArgBuild mean/core | +0.615 | 2.368 us | 0.76% | FAIL | +| 其余 Submit mean/core | -0.021 | 80.918 us | 25.94% | FAIL | + +同一 ELF 内 ArgBuild aggregate core-time 份额中位为 5.711%，范围 5.559%～5.792%。ALL 的 request/call 中位 119.891、范围 119.806～119.979，miss/call 中位 2.582、范围 2.571～2.612；AIC/AIV 的 miss/call 中位分别为 0.033/3.857。它们都是 running read-clear observed，不是无插桩净业务数；稳定的 observed 值也不能跨 ELF 换算成可直接消除的墙钟损失。 + +因此本阶段只成立一个受限结论：**当前 ArgBuild 诊断 ELF 的 311.972 us 波形不是 ArgBuild 携带，而是最晚核的非 ArgBuild 慢尾。** 本组由 X 而不是 M 通过，没有复现权威 perf-clock P 的约 1.408 ms 多数核共同伸缩；`Rz` 仍只是算术剩余，不能重命名为 atomic、flag wait、I-cache 或其他业务段。Pz 相关性和幅度都不临界，phase mean/core 波幅也只有 G 的 0.76%，所以不扩到 20 轮。下一阶段按既定顺序切换到现有 `submit-pmu-register`，完成最后一个存量 selector 的同 ELF 归因。 + +### 8.14 同一 Register ELF 的阶段与其余 Submit 波动分解 + +**[阶段归因已完成；Register 不通过主载体门禁]** + +#### 8.14.1 真实契约、冻结对象与正式样本 + +Register observer 有三个互斥源码挂点，但每个正常 Submit 只命中其中一个，因此固定 shape 是每核 `N` 对而不是 `3N` 对：B1 为 5，Case1 为 1,280。边界从真实 `dist_submit_register_outputs()` 调用入口到返回，排除普通泳道 Register 的前序记录发布、结束 timestamp 与 caller 衔接；它只能命名为 RegisterOutputs 调用体，不能冒充完整 Register timestamp-to-timestamp span 或单次 TensorMap insert。 + +新 HEAD 上先运行 B1： + +```text +outputs/TestPagedAttentionUnroll_CaseB1_20260721_102100/ +``` + +96 核各 5 次 Submit/Register，phase id `5`、status `0x3f`、primary/shadow 96/96 精确相等，HTML 精确重建；83.517 us 只作结构证据。随后冻结： + +| 对象 | 固定值 | +| --- | --- | +| commit | `57aedfee51883a5840e618037b49ac9c630d1cb6` | +| AICore cache | `32c26e06ad76d186` | +| source state | `source-v2:57aedfee51883a5840e618037b49ac9c630d1cb6:a799850ce15b3a0fcb6cd9dc6d89cc81a67f08c6d2a9a7afeb4c76b24ab75bea:ff85282e42fc1b03ba04d010a0b3b793349657e8b1e321fd7b31017015f1dee9` | +| AICore 定义 | `PTO_FDWIC_SUBMIT_PMU=1;PTO_FDWIC_SUBMIT_PMU_PHASE_ID=5;PTO_FDWIC_TRACE_ENABLED=0` | +| `aicore_kernel.o` | 2,573,816 B / `.text` 150,608 B / SHA256 `39760b286b14a3ec7a078a0d0a60981ce86ba3a58be68626df02bab056c107f1` | +| AIC/AIV combined | `78677b168454de544a708531781f00bcb1d3a2f65518affae262955f5f585933` / `8566136503bf6d7a60290a6f8776d515b13ec7469de4d4a084ced38d1ad24102` | +| `libhost_runtime.so` | SHA256 `5f2e9af0892f64f28aded87e013a5b32425d86940b2e08cd34dad49cab0c0f9d` | + +Case1 预热 `..._102246` 排除后，12 个独立正式进程全部 PASS： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_102330/ +outputs/TestPagedAttentionUnroll_Case1_20260721_102413/ +outputs/TestPagedAttentionUnroll_Case1_20260721_102458/ +outputs/TestPagedAttentionUnroll_Case1_20260721_102543/ +outputs/TestPagedAttentionUnroll_Case1_20260721_102626/ +outputs/TestPagedAttentionUnroll_Case1_20260721_102711/ +outputs/TestPagedAttentionUnroll_Case1_20260721_102754/ +outputs/TestPagedAttentionUnroll_Case1_20260721_102839/ +outputs/TestPagedAttentionUnroll_Case1_20260721_102923/ +outputs/TestPagedAttentionUnroll_Case1_20260721_103005/ +outputs/TestPagedAttentionUnroll_Case1_20260721_103049/ +outputs/TestPagedAttentionUnroll_Case1_20260721_103131/ +``` + +12/12 生产 `load_capture()` 和 HTML 重建通过；1,152 条逐核记录全部满足 96×1,280、phase id `5`、phase status `0x3f`、owner 配置/恢复、拓扑、风险阈值和计数顺序，primary/shadow 1,152/1,152 精确相等。共覆盖 1,474,560 次真实 RegisterOutputs 调用体。采样前后上述实物 SHA 和 source state 不变，已知外部评测未出现。 + +#### 8.14.2 整数分解与 12 轮结果 + +继续定义 `Pz=Register[z]`、`Rz=body[z]-Pz`，12/12 精确满足 `G=A+M+X=A+Pz+Rz`。下表单位均为 us： + +| 轮次 | 时间戳 | G | M | X | A | Pz | Rz | 最晚核 | +| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| 1 | `102330` | 5053.021 | 4784.164 | 268.581 | 0.276 | 233.630 | 4819.115 | c92/AIV/b30/l1 | +| 2 | `102413` | 4708.077 | 4587.936 | 116.475 | 3.666 | 231.358 | 4473.053 | c55/AIV/b11/l2 | +| 3 | `102458` | 4737.712 | 4593.329 | 136.797 | 7.586 | 250.913 | 4479.213 | c91/AIV/b29/l2 | +| 4 | `102543` | 4679.146 | 4594.077 | 77.060 | 8.009 | 231.034 | 4440.103 | c94/AIV/b31/l1 | +| 5 | `102626` | 4964.744 | 4608.269 | 354.148 | 2.327 | 212.237 | 4750.180 | c24/AIC/b24/l0 | +| 6 | `102711` | 5201.300 | 4590.639 | 606.702 | 3.960 | 232.572 | 4964.768 | c23/AIC/b23/l0 | +| 7 | `102754` | 5097.611 | 4818.618 | 278.777 | 0.216 | 245.937 | 4851.458 | c34/AIV/b1/l1 | +| 8 | `102839` | 4689.188 | 4604.074 | 78.356 | 6.757 | 236.377 | 4446.054 | c31/AIC/b31/l0 | +| 9 | `102923` | 4948.061 | 4612.592 | 329.669 | 5.800 | 244.657 | 4697.604 | c28/AIC/b28/l0 | +| 10 | `103005` | 4698.544 | 4598.600 | 98.540 | 1.404 | 244.352 | 4452.788 | c60/AIV/b14/l1 | +| 11 | `103049` | 4724.767 | 4595.187 | 126.730 | 2.850 | 265.833 | 4456.084 | c20/AIC/b20/l0 | +| 12 | `103131` | 4735.855 | 4604.962 | 123.287 | 7.606 | 241.715 | 4486.534 | c48/AIV/b8/l1 | + +最晚核 12 轮落在 12 个不同 core，AIC/AIV 为 5/7。完整分布为： + +| 指标 | 最小值 | 中位数 | 最大值 | MAD | P10 | P90 | P90-P10 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| G | 4679.146 | 4736.784 | 5201.300 | 52.617 | 4690.124 | 5093.152 | 403.028 | +| M | 4587.936 | 4601.337 | 4818.618 | 7.634 | 4590.908 | 4767.007 | 176.099 | +| X | 77.060 | 131.763 | 606.702 | 54.055 | 80.375 | 351.700 | 271.325 | +| A | 0.216 | 3.813 | 8.009 | 2.677 | 0.389 | 7.604 | 7.215 | +| Pz | 212.237 | 239.046 | 265.833 | 6.683 | 231.066 | 250.415 | 19.349 | +| Rz | 4440.103 | 4482.874 | 4964.768 | 39.795 | 4446.727 | 4848.224 | 401.496 | +| Register mean/core | 238.911 | 240.199 | 242.323 | 0.458 | 239.413 | 241.717 | 2.304 | +| 其余 Submit mean/core | 4297.259 | 4321.268 | 4572.133 | 20.016 | 4300.027 | 4549.033 | 249.006 | + +#### 8.14.3 门禁、AIV miss 跃迁与结论边界 + +| 分量 | Spearman(G, ·) | P90-P10 | 占 G 波幅 | 判定 | +| --- | ---: | ---: | ---: | --- | +| M | +0.378 | 176.099 us | 43.69% | FAIL | +| X | +0.937 | 271.325 us | 67.32% | PASS：迁移慢尾 | +| A | -0.476 | 7.215 us | 1.79% | FAIL | +| Pz | +0.077 | 19.349 us | 4.80% | **FAIL：Register 不是主载体** | +| Rz | +0.986 | 401.496 us | 99.62% | **PASS：同核其余 Submit 携带变化** | +| Register mean/core | +0.462 | 2.304 us | 0.57% | FAIL | +| 其余 Submit mean/core | +0.720 | 249.006 us | 61.78% | PASS：辅助 core-work 序列 | + +按最晚核角色复算，AIC/AIV 的 `Spearman(G,Pz)` 为 `-0.600/+0.571`，波幅只占各自 G 的 9.17%/4.39%；对应 Rz 为 `+1.000/+0.964`。Register 在角色内同样不携带波动。remaining mean/core 的通过说明本组除末核 X 外还存在部分多数核剩余工作联动，但它是辅助序列，不能与 X 相加成墙钟贡献，也不能继续命名为某个未测 phase。 + +Register core-time 份额中位为 5.267%，范围 5.025%～5.287%；request/call 中位 86.587，范围 86.414～86.709。miss/call 出现一个值得保留的 AIV 状态跃迁：前四轮 ALL 为 1.34～1.46、AIV 为 1.99～2.16，从第 5 轮起变为 ALL 约 1.79、AIV 约 2.66；AIC 始终约 0.047～0.053。该变化逐核 primary=shadow、构建身份和所有门禁闭合，因此不是 capture gap；但它与 G 的相关性只有 `+0.252`，Register phase 时间仍稳定，不能把 I-cache 状态变化包装成本组时延载体。这也说明正式报告必须同时展示同 ELF 的时间、request/miss 和逐核范围，而不能只看单轮 miss rate。 + +本阶段只排除当前 Register ELF 的 403.028 us 波形由 Register 主导；本组仍由 X 而不是权威 P 中的 M 主导，没有复现 P 的约 1.408 ms 宽波动。`Rz`/remaining 仍是算术剩余，不能改名为 atomic、flag wait 或 I-cache。Pz 与 mean/core 均远离门槛，所以不扩到 20 轮。至此所有存量 selector 的同 ELF 归因结束，下一步先完善报告端逐核 Submit/PMU/Scalar/非 busy 残余时间与构建 provenance，再实现固定容量的 EfDrain-control/依赖就绪观察；都不得扩展逐事件 raw。 + +### 8.15 真实 PA I-cache 报告的逐核时间闭合 + +**[观察工具已完成；不改 C++ producer、设备 ABI 或 raw schema]** + +Register 正式样本出现了“primary=shadow 全部闭合，但 AIV miss/call 中途跃迁而 phase 时间不动”的真实反例。为了避免后续继续用单轮 miss rate 或不同聚合对象解释性能，本阶段只增强生产 Python consumer 和 HTML，不修改任何 AICore/host C++： + +1. ALL/AIC/AIV 新增每核 `submit_elapsed_ticks` 的 mean/min/max，并按 1 ns/tick 显示为 us；它与顶部跨核 `global_submit_span` 明确分开； +2. PMU total、Scalar busy 和非 Scalar-busy 残余同时显示 cycles 与按各角色 1.649844/1.650062/1.649731 cycles/ns 校准的等效时间范围； +3. 残余严格先逐核计算 `total_cycles-scalar_busy` 再汇总，禁止拿来自不同核的 total/scalar 极值相减； +4. 新增逐核 `total_cycles/submit_elapsed_ticks` 的 mean/min/max，只命名为“同 ELF 长窗有效比”，不命名为瞬时频率或利用率；其典型值在最新 Register raw 中约为 ALL 1.649617、AIC 1.649614、AIV 1.649619 cycles/ns，与校准量级一致； +5. raw producer summary、`METRICS`、每核记录和 phase sidecar 全部保持不变，三个 derived summary 只存在于通过门禁后的 `SubmitPmuCapture`/HTML。 + +逐核 ratio 采用 arithmetic mean of ratios，不使用 `Σtotal/Σelapsed`；卡片纵向增加信息，不给已较宽的 phase/逐核表继续加列，原四张 SVG 也保持不变。全局说明继续强调等效时间不能与 perf-clock、swimlane 或另一个 phase ELF 相减，`total-scalar` 也不是 Scalar 空闲、I-cache stall 或 vector/cube wait。 + +单测新增专门的错法防线：让 total/scalar 极值落在不同核、构造 `mean(total_i/elapsed_i) != Σtotal/Σelapsed`、拒绝零 Submit elapsed、核对三种角色校准和 raw summary 未扩字段。生产报告 89 项、连同 profile/cache 合计 152 项 PASS，`ruff` 与 `git diff --check` 通过；最新 Register raw `outputs/TestPagedAttentionUnroll_Case1_20260721_103131/fdwic_submit_pmu_raw.json` 也已由增强后的生产 consumer 重新闭合并生成 82,656 B HTML。另从现存 raw 为 none、ArgBuild、Claim、EmptyBracket、Materialize、Register、SubmitTransition 各选最新一份重建 HTML，七份全部 PASS，大小为 79,838～82,812 B。该阶段完全发生在 case 返回后的 host 加工层，对 Submit 热路是零新增指令。 + +### 8.16 诊断 raw 与实际构建实物的 provenance 绑定 + +**[观察工具与真实 A5 B1 已闭合；不回写 raw，不给 AICore 增加指令]** + +此前正式多轮依赖文档手工冻结 ELF SHA，raw 自身不能证明“这些轮次确实使用同一构建”。本阶段没有把身份字段塞进 C++ producer 或 AICore ELF，也没有在 consumer 校验后回写权威 raw，而是增加固定第三件产物： + +```text +fdwic_submit_pmu_raw.json # C++ producer 原子发布；始终只读 +fdwic_submit_pmu_provenance.json # case 返回后由实际构建路径生成并绑定 raw SHA +fdwic_submit_pmu_report.html # 同次加载 raw + sidecar 后生成 +``` + +provenance schema 为 `fdwic-submit-pmu-provenance-v1`，至少闭合： + +1. 同一次 raw 读取冻结的固定文件名、字节数、SHA256 和 `capture.mode`； +2. 已 profile 化的 callable cache key、16 位 AICore extra cache key、严格匹配 profile 的编译宏； +3. 构建时 `.git_commit` 中的完整 `source-v2:::`，不拿采集后当前 HEAD 冒充构建身份； +4. worker 实际加载的 `build/lib/.../aicore_kernel.o`，以及 `build/cache/.../aicore/` 中的 AIC/AIV combined 和 source-state stamp；两类路径不能混用； +5. final AICore、AIC combined、AIV combined、host runtime 四件实物各自的完整文件 SHA/大小与 literal `.text` SHA/大小。 + +身份在 Submit-PMU ELF 符号门禁通过后、设备 case 开始前冻结；case 返回后再次从四个实际路径重算，任一文件或 source-state stamp 变化都拒绝发布 sidecar/HTML。scene test 按 `(class, platform, runtime, profile)` 严格查找同一 identity，缺失时在上板前拒绝，不会从 raw 自报字段反推。sidecar 自身再用 raw SHA 绑定；HTML 展示 provenance SHA、source-v2、宏和四件实物。已有无 sidecar 的历史 raw 仍可离线生成旧式报告，但新的正式采集必须自动产生三件套。 + +sidecar 与 HTML 先在同目录完整暂存，发布前后再次验证 raw 与已接受快照逐字节一致；首个/第二个最终替换失败或末次 raw 检查失败时，均恢复调用前的整对产物，不遗留半套或失配的正式文件。 + +实现中专门核实了 RuntimeBuilder 的真实目录结构：final ELF 只位于 `build/lib` 的 extra cache 目录；combined 和 `.git_commit` 位于对应的 `build/cache/.../aicore`。使用后一目录的 final 缓存文件冒充 worker 实际加载文件会造成证据错位，因此接口显式接收 output ELF 和 build directory 两条路径。`.text` 读取复用本机正式 `readelf -SW` 对 literal `.text` 的 offset/size，并单独 hash 字节范围，不把 `.rela.text` 当成正文。 + +纯 host 测试覆盖 raw 字节前后不变、sidecar/HTML 成组发布、四件实物与 `.text`、无 sidecar 历史路径，以及 raw binding、source-state/宏、schema、文件名和冻结后实物变化的 fail-closed；并覆盖首个/第二个最终替换失败、末次 raw 变化和旧产物整对恢复。report/cache 两组共 174 项 PASS，ruff 和 diff 检查通过；另用本机现存 Register 实物验证实际 `readelf` 路径，可得到 final/AIC/AIV/host `.text` 分别为 150,608/68,352/82,256/443,315 B。该工具全部运行在编译完成或 case 返回后，不会进入 Submit 性能窗口。 + +提交 `15c54b33` 后又依次完成两条真实 A5 B1： + +1. `submit-pmu-none` 位于 `outputs/TestPagedAttentionUnroll_CaseB1_20260721_111118/`。96 核均为 5 次 Submit，primary/shadow request 与 miss 逐核完全相等。raw/provenance/HTML 大小为 44,339/3,050/80,808 B，SHA256 分别为 `dbdc10c7d9cc00b79d62f96600a8a91c07470dd402677de9758eb3653572ff57`、`ebeef32f7ef26de78dcdc0d8f49d2711863adc71c28ada00341d0d26951ee235`、`9bea11a9b58e7e8af47381808687c8a5afc168e23c37d270389aa9d38282214f`； +2. `submit-pmu-register` 位于 `outputs/TestPagedAttentionUnroll_CaseB1_20260721_111427/`。96 核均闭合 5 次 Submit 和 5 对 Register begin/end，primary/shadow 逐核完全相等。三件套大小为 69,638/3,103/83,714 B，SHA256 分别为 `be122416742c28ab138db04acf23165d9c73fd90851fd7121eb0c8e303ed2d3a`、`d943be4b76c719b3513b5d030b9eaaf8a223badfb46107b74bbcda016c32db14`、`7e5e40695b529671682dc8d493bd234df2a4870bab361a67e18116be49e85896`。 + +两轮 source-v2 的 Git head 都精确指向 `15c54b33`；完整窗与 Register 的 profiled key 末项分别为 `submit-pmu-none`、`submit-pmu-register`，AICore extra key 分别为 `aa43623282e2a7db`、`32c26e06ad76d186`。因此同一进程框架下的 profile/cache 隔离已由真实 ELF 和产物旁证，而不是仅由 host mock 证明。两轮 pytest 均为 1 PASS；报告重载与重新渲染逐字节一致。 + +### 8.17 EfDrain 控制段的固定容量 I-cache 归因 + +**[观察工具、真实 A5 B1 与首轮 Case1 稳态原因取数均已闭合]** + +最新权威泳道中，完整 EfDrain 占 SubmitUnion 的 15.734%，但其中真实 KernelUnion 占 8.019%；直接包围整个 EfDrain 会把 Kernel 执行期间混入 Scalar 归因。扣除嵌套 Kernel 后的最大明确空缺是 EfDrain-control：30,754,207 aggregate core-ticks，占 SubmitUnion 的 7.715%。因此本阶段只实现这一项，不继续增加逐等待或逐事件记录。 + +四条真实 Submit 入口都在 `drain_block_won()` 前打开 phase，在 `drain_phase_b()` 后关闭。`execute_slot()` 只在 `dist_aicore_call_slot_kernel()` 紧邻前后 pause/resume；Kernel 返回后的 barrier、完成发布、frontier 和 slot 清理仍属于 control。背压与 FinalDrain 没有外层 phase，同一 helper 不会采集它们。 + +设逐核 Submit 数为 `N`、被排除的 linked-Kernel 调用数为 `K`，三层闭合公式是 `begin_reads=end_reads=N+K`，而报告 per-call 始终以外层 `N` 为分母。`K` 复用 phase record 的 `reserved[0]`，其余保留字保持 0；每核 record 继续是 64 B，总设备容量继续是 12,416 B。为避免污染其他 PMU profile，`K` 的 block-local 状态和清零写入只在 `PTO_FDWIC_SUBMIT_PMU_PHASE_ID=7` 编译；普通泳道、perf-clock、none 和其余 selector 不获得这份状态。 + +每次排除 Kernel 仍会产生四次 shadow MMIO read、两次 SYS_CNT 和少量 observer bookkeeping。完整 primary/shadow whole 会重建并包含 Kernel，局部 elapsed/request/miss 才排除 Kernel；所以它是同一诊断 ELF 内的控制段方向证据，不是零扰动业务净值，也不能跨 ELF 相减。生产 raw 新增的只是每核既有保留字对应的 `K` 字段与一项 96 核闭合结果，没有扩大设备 ABI 或改为逐事件 raw。 + +实现阶段已完成 C++ producer/consumer、CLI/profile/cache/provenance、HTML 和专属字段拒绝回归；同时在冻结 provenance 前核验实际 host ELF 的三个 hook 与精确 profile marker，旧 `libhost_runtime.so` 会在上板前直接报出重建要求，不再等到设备 init 返回 0。report/cache 共 209 项 PASS，ruff、clang-format 与 `git diff --check` 通过。 + +第一次上板在 `outputs/TestPagedAttentionUnroll_CaseB1_20260721_114357/` 于设备执行前失败：AICore override 已重编，但实际 host SO 是不含新 profile marker 的旧缓存，`fdwic_submit_pmu_host_init()` 返回 0，因此没有 raw。该失败没有被包装成设备或 Kernel 问题；补上 host ELF 能力门禁并按 `21e0414c` 重建完整 A5 FDWIC runtime 后，正式 B1 位于 `outputs/TestPagedAttentionUnroll_CaseB1_20260721_115019/`。 + +正式轮 96 核均为 5 次 Submit，实际 `ΣK=1`：95 核为 0，logical/physical core 9 的 AIC 为 1。全局 begin/end 恰为 481/481，等于 `96×5+1`；逐核也全部满足 `N+K`。phase status、专属 Kernel 排除、primary/shadow、owner Restore、拓扑、数值顺序与风险阈值均为 96/96。完整 B1 Submit 为 279.551 us；局部累计 elapsed/request/miss 为 64,751/56,187/3,414，相对本 ELF 逐核完整 Submit 累计值为 2.7441%/16.3993%/19.4021%。这些比例只证明真实不连续区间能闭合，不用于代替 Case1 稳态归因。 + +raw/provenance/HTML 大小为 72,951/3,124/84,377 B，SHA256 分别为 `79d6014e892b20823da039e3a2bea6c9761946b6c24444db71d7c61d16caca02`、`24be202086e9fda893012ca999073ceffa160aa9abba12861cee95414006e4d4`、`d8b2b4b77c243ac90e71fbb99084e7f5be7205f5d2a28ee224d4e56b9ed6c892`。provenance Git head 为 `21e0414c35ae7738a89f8994bfaf6870b733dea3`，extra key 为 `88075a1848686623`，host SHA 为 `441e54ac3d997e110de792d6597b8cf47d31a764ccf9bc63551387b9a597b919`；离线重载 raw+sidecar 后的 HTML 与正式文件逐字节一致。 + +随后从验证提交 `77df3959` 构建并运行首轮真实 Case1： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260721_115559/ +``` + +golden 与全部门禁通过，完整 Submit 为 4,840.463 us。96 核均为 1,280 Submit；实际 `ΣK=936`，逐核 3～19，AIC/AIV 分别为 429/507。begin/end 精确闭合为 123,816/123,816，即 `96×1280+936`，primary/shadow 仍为 96/96 exact。 + +| 角色 | control ns/call | 时间占比 | request/call | request 占比 | miss/call | miss 占比 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| ALL | 162.654 | 4.5920% | 82.110 | 14.0406% | 2.258 | 17.0455% | +| AIC | 199.792 | 5.7719% | 83.945 | 14.0450% | 0.110 | 28.6828% | +| AIV | 144.085 | 4.0219% | 81.193 | 14.0383% | 3.332 | 16.9316% | + +完整 Submit 的 AIC/AIV 每核平均 miss 为 493/25,187.344，miss rate 为 0.06444%/3.40227%；AIV control 每核平均 miss 为 4,264.625。按 90 ns 标尺，后者是 383.816 us/core，却高于该角色实际 control elapsed 的 184.428 us/core；因此不能把标尺当成可相减的 stall。AIC 的 control 时间又高于 AIV，而 miss/call 低两个数量级，本轮同样不支持“I-cache miss 单独主导 EfDrain-control 时间”。下一步应结合既有 atomic 泳道和控制流证据选候选，不能仅按 miss 排名改代码。 + +Case1 raw/provenance/HTML 大小为 76,269/3,124/86,600 B，SHA256 为 `6d6aa06fbf972f36fbb9260485bb5ee41f5cbb97e9246cb15a398ba2497201ce`、`b4a535ec671f7416945b5205e11268308068759e75ecc324a10ea2a57fb3fa03`、`f5751e86f503273d1b8f8e386301d1d84dfdc49ee6de45d7372199ce90a68841`；provenance Git head 为 `77df395941f86b3b546a6f50d6288fb88acb7078`，离线重渲染逐字节一致。 + +## 9. 已撤回、失败或不能外推的路线 + +### 9.1 已撤回的优化候选 + +| 路线 | 结果 | 决定 | +| --- | --- | --- | +| `ld_dev()+nop10/100` 替换 atomic | 性能用于定位，但同步语义不成立 | 撤回，不得作为优化 | +| fanin producer 降序 | fanin load 下降，Submit 中位反而 +7.774% | 撤回，不迁真实 PA | +| compete-first lazy C 版 | 相对 eager B 为 +0.040%，各 11/22 胜 | 不进入主路，不做 PMU | +| 16 B compete-first ticket | 相对同时段基线中位反向约 +0.434%，波动重叠 | 撤回 | +| pointer ticket | 首份真实 PA 样本 5.331474 ms，且引入生命周期约束 | 撤回 | +| standalone 伪 LoserReplay | loser 没有真实动作，却显著扩大 raw | 删除，改为离线未覆盖时间 | + +仓库历史中没有可追溯的 `fanin-prefix` 提交或源码痕迹；该未提交过程态已经按要求去除，本文不为它杜撰 hash、数据或收益。 + +### 9.2 正确性或平台验证的已知缺口 + +- standalone 16 MiB tiny-ring 的 CPU b256，H1 与临时恢复原实现都在观察窗口内未结束；只能认定模型存在既有长程活性或 host 调度问题，不能把它记为 H1 的 PASS/FAIL； +- MB6 `Normal` 因当前 runtime 没有测试期望的 DEPSIG 而失败； +- MB6 `Heavy` 在 H1 和基线中走同一调用路径 abort，不能归因于 H1，也不能记 PASS； +- `FullCore36` 在 Heavy 基线失败后未继续运行； +- 上述缺口没有通过顺手修改测试契约或生产协议来掩盖。 + +perf-clock 的 Case2 负测试不属于上述缺口：其每核实际 576 次与期望 320 次不符，host 已按设计拒绝并且没有输出成功 summary。这个结果只覆盖 fail-closed 门禁，不能被改写成 Case2 正确性或性能 PASS。 + +### 9.3 被证明不适合的观察路线 + +- external task-based `msprof` 的 raw counter 不受 kernel 内 start/stop 缩窗控制，不能用于 Claim、EfDrain 等局部取数； +- A5 上把 I-cache miss selector `0x35` 复制到 CNT9 时计数恒为 0，因此现行 shadow miss 使用 CNT5，并明确牺牲诊断 ELF 的 MTE3 busy； +- A5 CANN 9.1 没有 `scalar_wait_ib_time` 的正式事件或派生字段； +- 只在运行时把泳道 level 设为 0/1，不能移除 ELF 中的冷诊断代码和布局污染； +- 旧 runtime cache 不理解新增 phase 时曾只生成 Kernel/Alloc 名称，该轮已排除，不能作为有效基线； +- running phase 的 PMU read-clear 会改变布局和时序，不能将局部 ELF 与 `none` 相减得到无扰动净时间。 +- perf-clock 不能通过 `PTO2_PROFILING=0` 实现：该宏拥有公开 Arg 布局/ABI，首版尝试在 Arg cacheline `static_assert` 处失败、从未上板，并已撤回。最终实现保留该 ABI，只编译期移除 FDWIC 泳道/atomic 和平台 PMU 路径。 + +## 10. 本机证据产物索引 + +### 10.1 重要说明 + +`outputs/` 已被 Git 忽略。下列路径是本机采集证据索引，不属于源码提交，也不保证 fresh clone、其他 worktree 或后续清理后仍存在。文档中的数值必须与对应历史 commit 和采集配置一起理解，不能因为文件名存在就当作当前 HEAD 结果。 + +### 10.2 真实 PA + +| 阶段 | 路径 | 主要用途 | +| --- | --- | --- | +| 初始 atomic-load 基线 | `outputs/TestPagedAttentionUnroll_Case1_20260717_023809/merged_swimlane_atomic_load.json` | 5.642245 ms 起点 | +| NOP100 诊断 | `outputs/TestPagedAttentionUnroll_Case1_20260717_035341/merged_swimlane_nop100.json` | atomic 成本定位，已撤回 | +| NOP10 诊断 | `outputs/TestPagedAttentionUnroll_Case1_20260717_035954/merged_swimlane_nop10.json` | atomic 成本定位，已撤回 | +| 第一轮最好结果 | `outputs/TestPagedAttentionUnroll_Case1_20260717_055638/merged_swimlane_best_joint_poll_skip_arg_masks_5.096685ms.json` | joint skip + masks | +| H1 最好结果 | `outputs/TestPagedAttentionUnroll_Case1_20260717_173313/merged_swimlane_heapguard_first_lap_fastpath_5.098696ms.json` | HeapGuard 首圈 fast path | +| winner 冷路完整泳道 | `outputs/TestPagedAttentionUnroll_Case1_20260719_131116/merged_swimlane.json` | 恢复 atomic 观察接入后的布局 | +| atomic 冷路径 level-4 | `outputs/TestPagedAttentionUnroll_Case1_20260719_135629/` | Atomic/PollBatch/ClockBaseline 闭合 | +| compete-first level-1 三轮 | `outputs/TestPagedAttentionUnroll_Case1_20260720_095456/`、`..._095724/`、`..._095929/` | 真实路径三轮 A/B | +| compete-first level-4 历史样本 | `outputs/TestPagedAttentionUnroll_Case1_20260720_104406/merged_swimlane.json` | 早期真实布局、阶段与 atomic 闭合，不代表当前 HEAD | +| compete-first A5Sim | `outputs/TestPagedAttentionUnroll_Case1_20260720_104649/` | 108 核模拟回归 | +| perf-clock B1 首轮 | `outputs/TestPagedAttentionUnroll_CaseB1_20260720_172436/fdwic_perf_clock_summary.json` | 96 核、5 Submit/core，75.347 us | +| perf-clock B1 复验 | `outputs/TestPagedAttentionUnroll_CaseB1_20260720_172615/fdwic_perf_clock_summary.json` | 96 核、5 Submit/core，73.716 us | +| perf-clock Case1 golden | `outputs/TestPagedAttentionUnroll_Case1_20260720_172820/fdwic_perf_clock_summary.json` | golden PASS；4,489,247 ticks，不计入五轮基线 | +| perf-clock Case1 五轮基线 | `outputs/TestPagedAttentionUnroll_Case1_20260720_173140/`、`..._173224/`、`..._173307/`、`..._173350/`、`..._173433/` | 中位 4823.114 us，范围 4495.677～5808.500 us | +| perf-clock Case2 负测试 | `outputs/TestPagedAttentionUnroll_Case2_20260720_173035/` | 576/core 与期望 320 不符；拒绝结果，无成功 summary | +| 同源普通 level-4 B1 | `outputs/TestPagedAttentionUnroll_CaseB1_20260720_173738/` | 480 Submit、85.653 us、dropped=0、排他 PASS；不与 perf-clock 相减 | +| 当前 HEAD level-4 B1 门禁 | `outputs/TestPagedAttentionUnroll_CaseB1_20260720_234158/` | 新 fail-closed/ELF 门禁上板 PASS；仅作结构证据 | +| 当前 HEAD level-4 Case1 | `outputs/TestPagedAttentionUnroll_Case1_20260720_234305/` | 96×1280 Submit、5095.821 us、944874 records、dropped=0、全部闭合 | +| submit-PMU B1 首轮 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_002939/` | 96×5 Submit、74.882 us、raw/HTML 闭合 | +| submit-PMU B1 复验 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_003050/` | 96×5 Submit、227.673 us；AIC total 波动线索 | +| submit-PMU Case1 | `outputs/TestPagedAttentionUnroll_Case1_20260721_003335/` | 96×1280 Submit、5075.360 us；真实 AIC/AIV PMU raw/HTML | +| submit-PMU 后 perf-clock B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_004056/` | 96×5 Submit、254.084 us；ELF 隔离 PASS | +| submit-PMU 后 level-4 B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_004154/` | 4549 records、89.071 us、dropped=0、排他闭合 PASS | +| arg-build PMU B1 首轮 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_014154/` | 96×5 bracket、247.205 us、observed/capture-gap 契约闭合 | +| arg-build PMU B1 复验 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_014301/` | 96×5 bracket、80.702 us、observed/capture-gap 契约闭合 | +| arg-build PMU Case1 | `outputs/TestPagedAttentionUnroll_Case1_20260721_014355/` | 96×1280 bracket、4964.039 us、raw/HTML 全部闭合 | +| arg-build 后 none B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_014602/` | 无 phase 字段、primary=shadow 96/96 | +| arg-build 后 perf-clock B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_014715/` | 96×5 Submit、73.029 us、ELF 隔离 PASS | +| arg-build 后 level-4 B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_014841/` | 4560 records、90.275 us、dropped=0、排他闭合 PASS | +| empty-bracket PMU B1 首轮 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_020932/` | 96×5 对、257.430 us；冷启动记录开销闭合 | +| empty-bracket PMU B1 复验 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_021100/` | 96×5 对、303.032 us；冷启动记录开销闭合 | +| empty-bracket PMU Case1 首轮 | `outputs/TestPagedAttentionUnroll_Case1_20260721_021158/` | 96×1280 对、4972.718 us；ALL 640.465 ns/对 | +| empty-bracket PMU Case1 复验 | `outputs/TestPagedAttentionUnroll_Case1_20260721_021311/` | 96×1280 对、4866.126 us；ALL 639.272 ns/对，稳态尺度复现 | +| empty 后 arg-build B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_021930/` | 96×5 对、249.064 us；新旧 phase 契约闭合 | +| empty 后 none B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_022026/` | 无 phase 字段、primary=shadow 96/96 | +| empty 后 perf-clock B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_022115/` | 96×5 Submit、78.230 us、ELF 隔离 PASS | +| empty 后 level-4 B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_022221/` | 4546 records、88.595 us、dropped=0、排他闭合 PASS | +| materialize PMU B1 首轮 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_024533/` | 96×5 bracket、82.091 us、固定 shape/observed 闭合 | +| materialize PMU B1 复验 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_024641/` | 96×5 bracket、81.741 us、固定 shape/observed 闭合 | +| materialize PMU Case1 首轮 | `outputs/TestPagedAttentionUnroll_Case1_20260721_024748/` | 96×1280 bracket、4922.142 us；ALL 797.814 ns/次 | +| materialize PMU Case1 复验 | `outputs/TestPagedAttentionUnroll_Case1_20260721_024909/` | 96×1280 bracket、4851.282 us；ALL 797.061 ns/次 | +| materialize 后 arg-build B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_025027/` | 96×5、status 0x3f、primary=shadow 96/96 | +| materialize 后 empty B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_025125/` | 96×5、status 0x3f、primary=shadow 96/96 | +| materialize 后 none B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_025214/` | 无 phase 字段、primary=shadow 96/96 | +| materialize 后 perf-clock B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_025305/` | 96×5 Submit、274.997 us、ELF 隔离 PASS | +| materialize 后 level-4 B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_025353/` | 4550 records、89.109 us、dropped=0、排他闭合 PASS | +| AICPU mode 重构后 materialize B1 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_025856/` | 最终源码 480/480、status 0x3f、primary=shadow 96/96 | +| claim PMU B1 首轮 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_031756/` | 96×5 bracket、82.413 us、四边界/固定 shape 闭合 | +| claim PMU B1 复验 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_031954/` | 96×5 bracket、264.184 us；只作结构与波动样本 | +| claim PMU Case1 首轮 | `outputs/TestPagedAttentionUnroll_Case1_20260721_032101/` | 96×1280 bracket、4994.863 us；ALL 641.816 ns/次 | +| claim PMU Case1 复验 | `outputs/TestPagedAttentionUnroll_Case1_20260721_032244/` | 96×1280 bracket、4704.936 us；ALL 646.708 ns/次 | +| claim 后 materialize B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_032445/` | 96×5、status 0x3f、primary=shadow 96/96 | +| claim 后 arg-build B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_032536/` | 96×5、status 0x3f、primary=shadow 96/96 | +| claim 后 empty B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_032624/` | 96×5、status 0x3f、primary=shadow 96/96 | +| claim 后 none B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_032712/` | 无 phase 字段、primary=shadow 96/96 | +| claim 后 perf-clock B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_032802/` | 96×5 Submit、74.512 us、ELF 隔离 PASS | +| claim 后 level-4 B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_032914/` | 4552 records、87.664 us、dropped=0、排他闭合 PASS | +| register PMU B1 首轮 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_034517/` | 96×5 bracket、80.904 us、三挂点/固定 shape 闭合 | +| register PMU B1 复验 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_034904/` | 96×5 bracket、267.167 us；只作结构与波动样本 | +| register PMU Case1 首轮 | `outputs/TestPagedAttentionUnroll_Case1_20260721_035025/` | 96×1280 bracket、4688.752 us；ALL 187.688 ns/次 | +| register PMU Case1 复验 | `outputs/TestPagedAttentionUnroll_Case1_20260721_035136/` | 96×1280 bracket、5136.513 us；ALL 187.879 ns/次 | +| register 后 claim B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035425/` | 96×5、status 0x3f、primary=shadow 96/96 | +| register 后 materialize B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035505/` | 96×5、status 0x3f、primary=shadow 96/96 | +| register 后 arg-build B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035545/` | 96×5、status 0x3f、primary=shadow 96/96 | +| register 后 empty B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035625/` | 96×5、status 0x3f、primary=shadow 96/96 | +| register 后 none B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035703/` | 无 phase 字段、primary=shadow 96/96 | +| register 后 perf-clock B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035744/` | 96×5 Submit、74.282 us、ELF 隔离 PASS | +| register 后 level-4 B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_035843/` | 4550 records、292.159 us、dropped=0、排他闭合 PASS | +| submit-transition PMU B1 首轮 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_042627/` | 96×4 间隙、384/384、265.223 us；N-1 shape 闭合 | +| submit-transition PMU B1 复验 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_042750/` | 96×4 间隙、384/384、78.873 us；只作结构与波动样本 | +| submit-transition PMU Case1 首轮 | `outputs/TestPagedAttentionUnroll_Case1_20260721_042914/` | 96×1279 间隙、122784/122784、4708.545 us；ALL 354.560 ns/间隙 | +| submit-transition PMU Case1 复验 | `outputs/TestPagedAttentionUnroll_Case1_20260721_043036/` | 96×1279 间隙、122784/122784、4649.434 us；ALL 350.516 ns/间隙 | +| transition 后 register B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_043202/` | 原有 N 次 phase 仍为 96×5、480/480、`0x3f` | +| transition 后 none B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_043241/` | 无 phase 字段、primary=shadow 96/96 | +| transition 后 perf-clock B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_043322/` | 96×5 Submit、312.747 us、ELF 隔离 PASS | +| transition 后 level-4 B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_043553/` | 4548 FDWIC records、90.406 us、dropped=0、排他闭合 PASS | +| 观察代价 B1 预检 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_044523/`、`..._044616/`、`..._044712/` | P/N/S 三类身份与结构门禁闭合；不进入时间统计 | +| 观察代价首组六排列 | `outputs/TestPagedAttentionUnroll_Case1_20260721_044943/` 至 `..._050807/` | 18 个独立进程、每构建 6 样本；方向 3/6，触发扩样 | +| 观察代价反序扩样 | `outputs/TestPagedAttentionUnroll_Case1_20260721_051317/` 至 `..._053148/` | 再增 18 个独立进程；合计每构建 12 样本，差异仍不可分辨 | +| perf-clock 同 ELF 20 轮 | `outputs/TestPagedAttentionUnroll_Case1_20260721_055106/` 至 `..._060447/` | P 的 M 主导、X 次要波动证据 | +| submit-PMU-none 同 ELF 20 轮 | 历史 12 轮加 `outputs/TestPagedAttentionUnroll_Case1_20260721_062430/` 至 `..._062936/` | 仅限 N ELF 的 Scalar/I-cache 波动证据 | +| 最终 K B1 / P B1 回归 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_073602/`、`..._073736/` | K 边界与 P 未污染回归 | +| 最终 perf-clock-kernel Case1 | `outputs/TestPagedAttentionUnroll_Case1_20260721_073849/` | 1013 个窗口内 Kernel；逐核/分组/顶层整数闭合 | +| perf-clock-kernel 预热 | `outputs/TestPagedAttentionUnroll_Case1_20260721_074831/` | 4.480978 ms；明确排除出统计 | +| perf-clock-kernel 同 ELF 20 轮 | `outputs/TestPagedAttentionUnroll_Case1_20260721_075049/` 至 `..._080901/` | K 复现 P 的共同伸缩；波动主要落在 residual | +| EfDrain-control 首次旧 host 失败 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_114357/` | host 缺新 profile marker，init 前拒绝；无 raw，不作性能样本 | +| EfDrain-control 正式 B1 | `outputs/TestPagedAttentionUnroll_CaseB1_20260721_115019/` | 96×5、`ΣK=1`、481/481、primary=shadow、三件套闭合 | +| EfDrain-control Case1 | `outputs/TestPagedAttentionUnroll_Case1_20260721_115559/` | 4,840.463 us、`ΣK=936`、123816/123816、AIC/AIV 稳态原因数据 | + +### 10.3 standalone + +| 阶段 | 路径 | 主要用途 | +| --- | --- | --- | +| D1 dynamic atomic | `tests/atomic_probe/pa_scheduler/outputs/atomic_diagnostics/ccec_baseline_10_20260718_023551.log` | fanin/frontier 次数分布 | +| 历史 schema-v3 b256 | `tests/atomic_probe/pa_scheduler/outputs/pa_scheduler_swimlane_20260718_182725_4061524/ccec/` | atomic 合并泳道 5.774295 ms | +| 约 5 ms 对等样本 | `tests/atomic_probe/pa_scheduler/outputs/performance_gap_20260718/standalone_ccec_real_b256_raw.json` | 真计算同泳道比较 | +| schema-v4 b256 | `tests/atomic_probe/pa_scheduler/outputs/pa_scheduler_swimlane_20260719_114815_617346/ccec/` | 排他边界、raw 规模和整数闭合 | +| winner 冷分支 | `tests/atomic_probe/pa_scheduler/outputs/pa_scheduler_swimlane_20260719_123520_660296/ccec/` | 尾部下降和 b256 门禁 | +| compete-first b1 | `tests/atomic_probe/pa_scheduler/outputs/pa_scheduler_swimlane_20260720_092021_1729726/ccec/` | 当前 standalone 业务布局 | +| 历史 submit-pmu none | `tests/atomic_probe/pa_scheduler/outputs/submit_pmu_none_20260719_b256_final/` | 96 核 I-cache raw/HTML | +| 历史五 phase PMU | `tests/atomic_probe/pa_scheduler/outputs/submit_pmu_phase_time_v5_20260719/` | 旧边界同 ELF 比例，仅作历史 | +| 单 miss 标尺 | `tests/atomic_probe/pa_scheduler/outputs/pmu_validation/icache_single_64x10_20260718_085929_3232836_console.log`、`icache_single_128x5_20260718_090151_3235468_console.log` | 90 ns/miss 一阶标尺 | + +## 11. 禁止混算和过度解释 + +1. **不同 ELF 不相减。** `perf-clock`、`swimlane`、不同 selector 的 `submit-pmu` 都会改变代码布局；绝对时间不能机械相减。 +2. **不同观察 level 不混用。** level-4 atomic 泳道、level-1 phase 泳道和 `--no-swimlane` 不是同一性能口径。 +3. **跨核墙钟不等于累计工作量。** 96 核 span 求和、每核 PMU mean 和最早到最晚 Submit 都是不同量。 +4. **PMU cycle 不等于 SYS_CNT tick。** 前者约 1.65 cycles/ns，后者 1 ns/tick。 +5. **局部 phase 只能在本 ELF 内解释。** phase 时间、request、miss 只能除以同一进程、同一角色、同一 ELF 的完整区间；不同 phase 不相加，不与 `none` 相减。 +6. **Atomic 边界按语义解释。** `return_ready` 不是全系统可见性屏障；`source_issue` 更不能解释为完成延迟。PollBatch 是等待区间，不是单次 atomic。 +7. **90 ns/miss 不是墙钟损失。** `miss * 90 ns` 只给单核串行等效数量级；多核、预取、流水和等待会重叠。 +8. **`total - scalar_busy` 不是 scalar 空闲。** 它还可能包含 I-cache refill、atomic 等待、Vector/Cube engine 等待和其他非 scalar-busy 周期。 +9. **standalone 收益不外推真实 PA。** standalone compete-first 为 -3.9635%，真实 PA 只有波动重叠的 -0.263%，当前真实结论是基本持平。 +10. **b1 与 b256 分工不同。** b1 用于结构、边界、调用次数和快速正确性门禁；b256 只在阶段性收口时用于规模和性能结论。 +11. **不同 schema 的同名区域不直接相加。** schema-v2/v3 的 Build/Replay lap 与 schema-v4 的排他尾动作定义不同。 +12. **观察构建不能冒充净性能。** 带观察的 5.066862、5.278401、5.774295 ms 都只能解释对应观察 ELF。 +13. **中位数差不能脱离样本波动。** 三轮区间重叠时应写“基本持平”，不能只取小数点后的正向差宣布收益。 +14. **固定次数与动态次数要分开。** Claim、H1 消减等可由拓扑推出；fanin retry、frontier helping 和 barrier poll 必须以动态记录为准。 +15. **empty-bracket 不是可直接扣除的常数。** 它的 elapsed 使用外层 SYS_CNT，request/miss 使用 running read-clear 内边界，并且与业务 selector 属于不同 ELF；只能作为观察器自扰动的经验尺度，不能生成数学修正后的业务净值。 + +## 12. 后续阶段更新与提交模板 + +每个合理阶段只验证一个主要变量。完成该阶段的源码、正确性门禁、性能取数和本文记录后，再形成一条详细中文提交；不得把多个无法拆因果的候选堆进同一个性能结论。 + +建议复制以下模板： + +```markdown +### YYYY-MM-DD / 阶段代号：简短名称 + +状态：[设计中|观察工具|已保留|已撤回|受限] + +目标： +- 要回答的唯一问题； +- 预期影响的现有 span 或计数。 + +源码与构建身份： +- 基线 commit： +- 候选 commit/工作树： +- 分支： +- 构建类型：perf-clock / swimlane / submit-pmu-； +- AIC/AIV ELF 或 manifest 身份： + +单变量改动： +- 修改文件和代码语义； +- 明确未改变的协议、边界和 ABI。 + +环境与命令： +- CANN/PTO-ISA/GCC/Python； +- A5Sim/A5； +- batch、负载、观察 level、runs、timeout； +- A/B 顺序和预热规则。 + +正确性门禁： +- 语义/golden； +- 96 核、每核 Submit、task 连续性； +- winner、fanin、TensorMap、heap、frontier、placement； +- dropped、父子区间、atomic 或 PMU 闭合。 + +性能结果： +- 全部原始样本； +- 中心值、范围、timeout/失败数； +- 同构 A/B 变化； +- 不能解释的波动。 + +辅助观察： +- swimlane 变化； +- atomic 逻辑/物理次数； +- PMU total/scalar/request/miss； +- `.text` 和关键符号尺寸。 + +本机产物： +- raw： +- merged/HTML/analysis： +- 日志： + +决定： +- 保留、撤回或继续取证； +- 结论适用范围； +- 未关闭风险。 + +提交： +- hash： +- 中文主题： +- 详细正文摘要： +``` + +### 12.1 当前下一阶段：只推进真实 simpler PA + +当前只进入真实 PA 观察基础设施阶段，不继续修改 standalone，也不立即猜测新的业务优化： + +1. **[观察工具，已实现] 真实 PA `perf-clock`**：保留 PTO2 公开 Arg ABI，编译期去除 FDWIC 泳道、atomic 观察和平台 PMU；首尾 Submit、96 核逐核调用数、6976 B header、ELF 双向身份、B1、Case1 五轮基线、Case2 fail-closed 和普通 level-4 同源边界均已验证，作为后续候选保留/撤销的权威低扰动 A/B 口径； +2. **[观察工具，已实现] 真实 PA `swimlane`**：普通业务 span 与 atomic 合并采集，已补最终 ELF 正向身份、schema-v4 fail-closed、父子/Kernel/整数闭合门禁，并由当前 HEAD 的 B1 与完整 Case1 上板验证；只用于定位业务区域和 atomic 变化，不与 perf-clock 绝对时间相减； +3. **[观察工具，已实现] 真实 PA `submit-pmu-none`**：编译期去除泳道、atomic 和通用逐 task PMU，每核完整 Submit 期只开关 PMU 一次；96 核 primary/shadow、owner Restore、AIC/AIV raw/HTML、两轮 B1 和一次 Case1 均已闭合； +4. **[观察工具，六个真实 selector 与空 bracket 均已实现] 真实 PA 单阶段 PMU**：`arg-build` 已完成两轮 B1 与一次 Case1；`empty-bracket` 复用同一 12,416 B ABI，已完成两轮 B1 与两轮 Case1，量出 running begin/end 的稳态记录开销并明确外层 elapsed 与 read-clear request/miss 的不同边界；`materialize` 又按最新真实 Case1 最大明确业务 span 建立 mode 4/phase 3，完成两轮 B1、两轮 Case1、五类互斥 B1 回归和最终 AICPU mode 重构回归；`claim` 按同一真实布局的下一明确热点建立 mode 5/phase 4，四条旧/新 Kernel/Alloc 边界、固定 96×1280 shape、两轮 B1、两轮 Case1 和六类互斥 B1 回归均已闭合；`register` 又建立 mode 6/phase 5，在三个真实 RegisterOutputs 调用点固定采集 96×1280 shape，完成两轮 B1、两轮 Case1 和七类互斥 B1 回归；`submit-transition` 最后建立 mode 7/phase 6，复用统一 Submit hook，以每核 `N-1` 独立 shape 聚合相邻 Submit 间隙，两轮 B1、两轮 Case1 和四类互斥构建回归均已闭合；`efdrain-control` 最后建立 mode 8/phase 7，用四条 Submit 外层边界和 Kernel 前后 pause/resume 聚合排除 linked Kernel 的不连续 Scalar 控制片段；真实 A5 B1 已闭合 `ΣK=1`、481/481 read、96 核状态与 provenance 三件套，Case1 又闭合 `ΣK=936`、123816/123816 read 和首轮 AIC/AIV 时间/request/miss 原因数据。六个 selector 都复用 12,416 B ABI，没有增加逐调用记录；至此停止继续增加 phase，也不依赖新的 standalone 实现； +5. 结构和边界迭代先使用最小有效真实 PA 用例完成正确性门禁；只有构建身份、容量、业务边界和统计闭合后，才运行完整 Case1 b256 性能样本，避免反复生成数百 MiB profiling 文件； +6. **[观察校准，已完成但不可分辨] 三构建交错量化**：在冻结的 `d1572c33` 上先做三类 B1 预检，再执行正序和反序各六个 Case1 排列块；P/S/N 各 12 个样本全部闭合。S-P 与 N-P 都只有 7/12 同方向，且中位差绝对值小于 `2×MAD`，因此没有把跨 ELF 差值包装成观察指令成本，也不继续扩样强求单值； +7. **[波动定位，已完成] 同一 perf-clock ELF 的 20 轮独立 Case1**：20/20 raw 均按 96×1280 和整数时钟严格闭合。完整 Submit 的中位数为 4898.184 us，P90-P10 跨度为 1407.736 us；M 通过预定相关性和幅度双门禁，A 与起跑偏斜均不通过，X 只作为次要迁移慢尾。96 核中 95 核的 body 与 G 的 Spearman 不小于 0.7，AIC/AIV 中位数联动，且不存在固定最慢核或时间单调漂移； +8. **[PMU 波动定位，已完成且限制为 N ELF] 同一 submit-pmu-none ELF 的 20 轮**：复用历史 12 轮并在一次排除预热后补 8 轮连续 N-only，20/20 通过生产消费者全量闭合。N 的 Scalar-busy 是其自身每轮 ALL mean/core 波动的主要观察载体，非 Scalar-busy 残余幅度很小，全窗平均有效 cycle/time 比例只有 21.35 ppm 的 P90-P10，I-cache miss 次数不随慢轮同向；但 N 的 A/M/X 由 X 通过门禁，未复现 P 的 M 主导形态，因此没有把 N 的 Scalar/I-cache 结论外推成 P 的根因； +9. **[低容量 Kernel 聚合，结构闭合已完成] 独立 perf-clock-kernel 变体**：保持 6,976 B 固定 header 和每核 64 B，只在真实 linked-kernel 调用前后读取 SYS_CNT，并用逐核首末 Submit 状态排除 FinalDrain。K B1、P B1 和 K Case1 均通过 golden 与 host/Python 双层产物门禁；Case1 窗口内记录 1013 个 Kernel，合法总范围为 `[256,1024]`，逐核 Kernel 与 residual 整数闭合。该单轮只证明工具可用，尚未证明 K 复现 P 的波动形态； +10. **[K 波动定位，已完成] 同一 perf-clock-kernel ELF 的 20 轮独立 Case1**：20/20 生产消费者闭合，K 的 M 以 `rho=+0.946`、71.2% 幅度复现 P 的多数核共同伸缩；但 K 的 X 也通过双门禁而 P 的 X 不通过，所以只能称部分复现。K 内部 residual mean/core 以 `rho=+0.941`、72.1% 幅度通过，Kernel mean/core 虽相关但幅度只有 G 的 0.65%，calls 仅 998～1011 且与 G 负相关。由此只在 K 内部排除 Kernel 总数量或 mean/core 变化幅度主导，不能外推 P 根因；residual 也还不能未经细分就命名为 Scalar、atomic、I-cache、flag 等待或频率问题。 +11. **[环境取证，已完成] 持续真实 PA 负载 power probe**：在已知 root A5 评测退出后，用日志初始化标志对齐 20 轮真实 PA 和 3 s `asys --sys-lp`。die 0/1 各 291 个点均为 hard/soft 1650 MHz，POWERBRAKE 与三级 IWARNING 均为 0；只排除 10.24 ms 粒度的持续 DVFS/EDP，不外推到一个 Submit 内的更短瞬态。第一次错误等待 `/dev/davinci0` 的错位样本已明确作废。 +12. **[Materialize 波动归因，已完成] 同一诊断 ELF 的 12 轮独立 Case1**：12/12 生产消费者和 HTML 闭合，最晚核 Materialize `Pz` 的 `rho=-0.343`、波幅只占 G 的 9.44%，不通过；同核其余 Submit `Rz` 的 `rho=+0.986`、波幅 100.47%，通过。Materialize mean/core 的 P90-P10 仅 0.994 us。本组由 X 慢尾而不是 M 共同伸缩通过门禁，因此只排除 Materialize 为该 ELF 的波动主载体，不外推 P 根因。 +13. **[Claim 波动归因，已完成] 同一诊断 ELF 的 12 轮独立 Case1**：12/12 闭合；最晚核 Claim 因 AIC/AIV 角色切换形成 370～1,025 us 的大范围，但与 G 的 `rho=-0.189`，相关性失败；同核其余 Submit 的 `rho=+0.790` 并通过。Claim mean/core 波幅只占 G 的 7.29%，request/miss 也稳定。本组仍由 X 慢尾通过，因此不把 ClaimMax/return-ready atomic 包装成权威 P 的波动根因。 +14. **[SubmitTransition 波动归因，已完成] 同一诊断 ELF 的 12 轮独立 Case1**：每核 1,279 个相邻间隙全部闭合。最晚核 Pz 的整体 `rho=-0.531` 受 AIC/AIV 角色混合影响，角色内仅 `+0.371/+0.257`；Transition mean/core 波幅只占 G 的 0.74%。同核其余 Submit 继续通过，因此相邻 Submit 间隙也不是本 ELF 的主载体。 +15. **[ArgBuild 波动归因，已完成] 同一诊断 ELF 的 12 轮独立 Case1**：12/12 生产消费者、HTML、每核 1,280 个边界和 primary/shadow 全部闭合。AIC/AIV 的 Pz 形成约 199/276～298 us 角色双峰，但整体 `rho=-0.259`、波幅只占 G 的 29.94%，角色内也不通过；ArgBuild mean/core 波幅仅占 0.76%。同核非 ArgBuild 的 Rz 与迁移慢尾 X 通过。本组没有复现 P 的多数核共同伸缩，因此只排除当前 ArgBuild ELF 的实际波形由该 phase 主导。 +16. **[Register 波动归因，已完成] 同一诊断 ELF 的 12 轮独立 Case1**：每个 Submit 只命中三个互斥挂点之一，1,152 条逐核记录和 1,474,560 次调用体全部闭合。Register Pz 的 `rho=+0.077`、波幅只占 G 的 4.80%，mean/core 波幅仅 0.57%；同核 Rz、迁移慢尾 X 和辅助 remaining mean/core 通过。AIV miss/call 在第 5 轮发生闭合但与 G 弱相关的状态跃迁，进一步证明单轮 miss 不能替代同 ELF 时间关联。结论仍不外推到权威 P 的宽波动。 +17. **[I-cache 报告加工，已完成] 不改 raw/ABI 的逐核时间闭合**：从受信 records 派生 Submit SYS_CNT、PMU total、Scalar busy、逐核非 busy 残余和 PMU/SYS 长窗有效比的 mean/min/max；角色校准只生成当前 ELF 等效时间。测试专门拒绝 difference-of-extrema、ratio-of-sums 和零分母，相关 152 项及 ruff 通过。 +18. **[构建 provenance，真实 A5 B1 已闭合] raw 保持只读**：新增 raw SHA 绑定的独立 sidecar，记录构建时 source-v2、profile/cache/宏，以及实际 final、AIC/AIV combined 和 host 的 whole/.text 身份；构建后与 case 后双重实物检查。缺 identity、绑定或文件变化均 fail-closed；成组发布失败会恢复旧产物。相关 174 项通过；`submit-pmu-none` 与 `submit-pmu-register` 各一轮真实 A5 B1 三件套均闭合，且使用不同 profiled/extra cache key。 +19. **[EfDrain-control，B1 与首轮 Case1 已闭合]**：固定容量排除真实 linked Kernel，B1 实际 `ΣK=1`，Case1 实际 `ΣK=936`；后者完整 Submit 为 4,840.463 us，AIC/AIV control 分别为 199.792/144.085 ns/call、0.110/3.332 miss/call。AIC 时间更长但 miss 低两个数量级，因此当前证据不支持单独按 I-cache miss 选择生产优化。 + +下一步不再扩充 N、K 或已经完成归因的既有 selector，也不跨 ELF 扣减。独立构建 provenance 与 EfDrain-control 的 B1/Case1 已完成。不得恢复逐 Submit/逐等待的大 raw，也不在此基础上继续堆叠依赖就绪 selector。下一阶段只从现有泳道 atomic、排他 span、全窗 PMU 与六个 selector 的交叉证据中选择一个真实 Scalar 候选，再由 perf-clock 交错 A/B 决定保留或撤销。单阶段 observed 不机械扣除 empty，也不从单轮 I-cache 数直接提出生产优化；standalone 的绝对数继续不替代真实 PA 当前结果。 + +### 2026-07-21 / P1：BlockWon 慢路冷外提 + +状态：**[已撤回]** + +#### 目标与观察前提 + +本阶段先复核现有 I-cache 观察覆盖，再验证一个不改变 BlockWon 协议的代码布局候选。最新真实 PA 排他泳道 `outputs/TestPagedAttentionUnroll_Case1_20260721_053003/` 中，五个 Submit 内业务 selector 已覆盖 SubmitUnion 的 75.4379%；扣除 EfDrain 内真实 Kernel 后，约覆盖 82.02% 的 Scalar/control core-work。尚未直接覆盖的最大纯 Scalar 区域是 PrepareMap（6.1493%）。当前完整 Submit、六个业务 selector、empty-bracket、primary/shadow、96 核 AIC/AIV raw、构建 provenance 和 HTML 已闭合，因而不为本候选继续增加设备字段或 selector；PrepareMap 只在后续候选明确落入该区时按需补齐。 + +Case1 是单 lane 图，权威泳道中 `DrainWon=0`，但 `drain_block_won()` 仍从 EfDrain、kernel loser replay 和背压路径被高频调用。候选只把 `g_fdwic_joint_submit_seen` 门闩保留在 inline wrapper，把原 BlockWon 扫描原样放入 `noinline` 冷函数；不删除 joint 路径，不改变 atomic、slot、fanin、完成发布或 FinalDrain 语义。 + +#### 正确性与布局证据 + +候选真实 A5 B1 位于 `outputs/TestPagedAttentionUnroll_CaseB1_20260721_121714/`，golden 通过，96 核均为 5 次 Submit，完整 Submit 为 80.746 us。该轮只作结构门禁，不进入 Case1 性能统计。 + +同一 perf-clock 构建下，候选确实形成独立 `drain_block_won_joint_slow()` 符号，布局变化如下： + +| 对象 | 基线 | 候选 | 变化 | +| --- | ---: | ---: | ---: | +| AIC combined `.text` | 59,288 B | 52,960 B | -6,328 B | +| AIV combined `.text` | 73,296 B | 66,896 B | -6,400 B | +| AIC combined whole | 1,773,136 B | 1,659,096 B | -114,040 B | +| AIV combined whole | 2,032,584 B | 1,919,512 B | -113,072 B | +| final `aicore_kernel.o` | 2,469,600 B | 2,358,784 B | -110,816 B | + +这只证明冷外提减少了热调用方的重复代码，不能自动推出 I-cache 或墙钟收益。 + +#### perf-clock 五对交错 A/B + +基线 A 与候选 B 均使用独立 pytest 进程、`Case1 --fdwic-profile perf-clock --rounds 1 --skip-golden`。候选预热 `outputs/TestPagedAttentionUnroll_Case1_20260721_121858/` 明确排除。正式五对为： + +| 对次 | A 基线产物 | A/us | B 候选产物 | B/us | B-A/us | +| ---: | --- | ---: | --- | ---: | ---: | +| 1 | `..._122032` | 4,883.659 | `..._122151` | 4,460.575 | -423.084 | +| 2 | `..._122319` | 4,886.473 | `..._122439` | 5,299.185 | +412.712 | +| 3 | `..._122611` | 4,488.058 | `..._122732` | 4,489.416 | +1.358 | +| 4 | `..._122905` | 4,553.267 | `..._123040` | 5,117.292 | +564.025 | +| 5 | `..._123229` | 6,139.517 | `..._123345` | 5,377.480 | -762.037 | + +A/B 组中位数分别为 4,883.659/5,117.292 us；逐对差值中位数为 **+1.358 us（+0.0278%）**，候选 2 对更快、3 对更慢。布局缩小没有稳定兑现成完整 Submit 收益，结果仍被已知的多数核共同伸缩波动覆盖。 + +#### 决定 + +候选业务代码完整撤回，不以“ELF 更小”替代 perf-clock 结论，也不为证明预期收益继续扩样。保留本节负结果，避免后续重复进行相同的 BlockWon 冷外提。下一候选仍只改一个已由源码证明的 Scalar 冗余，并继续由完整 Submit perf-clock A/B 决定保留或撤销。 + +### 2026-07-21 / O8：补齐观察链离线组合回归 + +状态：**[观察工具，离线闭合]** + +本阶段遵守“暂停上板”的要求，没有启动 A5，也没有修改 device/host 生产代码、PMU ABI 或 raw 字段。目标是复核当前三条证据链的现有正式产物，并补上仅靠单模块合成 raw 无法覆盖的组合门禁。 + +#### 正式产物复核 + +- `outputs/TestPagedAttentionUnroll_Case1_20260721_053003/` 的 schema-v4 泳道为 96 核、每核 1,280 个 Submit、945,264 个事件且 `dropped=0`；atomic 物理记录 105,963 条、PollBatch 337 条、折叠调用 4,481 次，逻辑调用严格闭合为 `105963 - 337 + 4481 = 110107`。当前 analyzer 重算的拓扑、父区间、Kernel containment、排他分区和整数 cycle closure 全部通过。 +- Materialize、Claim、SubmitTransition、ArgBuild、Register 和 EfDrain-control 六份正式 phase raw 均通过当前严格 reader：96 个唯一物理核、32 AIC + 64 AIV、每核 1,280 个 Submit、owner 配置/恢复、primary/shadow、selector/status、风险阈值和 phase 时间/次数全部闭合；Transition 为 1,279 次/核，其他普通 phase 为 1,280 次/核，EfDrain-control 为 `N+K`，其中 `ΣK=936`、全局 begin/end 均为 123,816。 +- 历史目录中只有 Register `..._103131` 和 EfDrain-control `..._115559` 具有 provenance sidecar；Materialize `..._092508`、Claim `..._094203`、Transition `..._095712` 和 ArgBuild `..._101249` 只能证明 raw 数据闭合，不能追溯当时实际 ELF/编译宏。`..._053003` 泳道同样没有构建 sidecar。该限制如实保留，不使用后来的二进制身份反向填充历史产物。 + +#### 新增离线门禁 + +1. `test_fdwic_submit_pmu_report.py` 对 phase 1～7 参数化执行完整 raw→build identity→provenance→HTML 发布链，逐项核对 capture mode、phase ID、三个编译宏、raw 不变性、provenance SHA 和 HTML 身份展示。这样可以离线拒绝 profile/phase/cache 身份串线，不再只由 none 和 EfDrain 两个特例间接代表其余 selector。 +2. `test_fdwic_swimlane_converter.py` 在同一 schema-v4 业务 raw 中加入 ClaimMax `return_ready` direct 与 7 次 Fanin poll-batch。测试确认两类 atomic 与 Claim/Fanin 同处对应 Scalar lane，2 条物理记录加权为 8 次调用；atomic 保持非加和 overlay，加入前后的 aggregate core-work、residual breakdown 和 Submit 整数分区完全不变。 + +离线验证结果： + +```text +tests/ut/py/test_fdwic_submit_pmu_report.py +tests/ut/py/test_fdwic_swimlane_converter.py +tests/ut/py/test_scene_test_cache.py + 288 passed + +ruff check + All checks passed + +git diff --check + PASS +``` + +#### 下一单变量候选 + +源码与现有泳道交叉审计后，下一候选收敛为 PrepareMap 的空 task-head 快路，而不再继续 ActiveMask 复用或重复增加 `drain_phase_b()` caller gate。Case1 每核 1,280 个 Submit、`H=64`，会清退 1,215 个历史 task id；五任务序列中只有 UP 的四个 INOUT 建立 map 链，因而每核可静态得到 243 个非空 head 和 972 个空 head。若只在 `cur == -1` 时跳过原本再次写入 `-1` 的 GM store，96 核理论上删除 93,312 次冗余写，且影响严格落在 PrepareMap（现有 aggregate 24,512,711 ticks、占 SubmitUnion 6.1493%）。截至 O8，该候选尚未写入生产源码、没有性能结论；其后续实现、离线门禁和真实 A5 待办由 P2 单独记录。 + +### 2026-07-21 / P2：PrepareMap 空 task-head 快路 + +状态：**[验证中：离线门禁完成，待真实 A5 裁决]** + +#### 候选边界与机会量 + +真实实现位于 `src/a5/runtime/fully_distributed_within_core/runtime/dist_engine/aicore/tensor_map.h` 的 `dist_tensor_map_advance_retire()`。原逻辑读取每个已退休 task 对应的 `task_heads[slot]`；即使值已经是空链哨兵 `-1`，也会在不进入释放循环后再次向同一 GM 地址写入 `-1`。 + +提交 `2dc49a13` 只增加以下精确快路： + +```cpp +int32_t cur = self.task_heads[id & kTaskWindowMask]; +if (cur == -1) continue; +``` + +这里不能写成 `cur < 0`：原实现会把小于 `-1` 的异常负值归一为 `-1`，精确比较才保持该防御行为。`continue` 只结束当前 id；循环后的 `cleaned_upto` 和 `alive_floor` 仍统一推进，非空 task 链、bucket 双向链和 free-list 完全走原路径。每个 worker 独占自己的 `DistCore::map`，真实路径也没有第二个执行流依赖“把已经是 `-1` 的普通字段再写一次”这个物理写事件。 + +Case1 每核 1,280 次 Submit，`H=64`，实际清退 task id `0..1214`。五任务序列中只有 UP 的四个 INOUT 建立 map 链，因此清退范围内有 243 个非空 head、972 个空 head；96 核静态机会量为 **93,312 次冗余 GM store**。该数字只证明候选有真实命中机会，不等于已经取得 93,312 次写延迟之和，更不是墙钟收益。 + +#### 直接生产语义单测 + +提交 `970e4fad` 新增 no-hardware C++ 测试，直接包含生产 `tensor_map.h`，没有复用 standalone 或旧 ring-per-bucket 模型。测试将生产 retire 与优化前控制流做完整状态差分，覆盖： + +- 正常 `-1` 空 head 与异常 `-2` 归一化； +- 同 task 多 entry、bucket 头/中间摘链和 free-list 顺序； +- `kTaskWindow` 槽复用、重复 floor 和回退 floor； +- 所有未触及 entry/相邻字段的逐字节一致性。测试使用显式 `memcpy` 克隆并要求 `DistTensorMap` 可平凡复制，避免结构体 padding 导致跨编译器假失败。 + +本用户 GCC 15 的验证结果为： + +```text +test_fdwic_swimlane_poll_batch +test_fdwic_tensor_map_retire + 2/2 passed +``` + +其中 CaseB1 不能替代这项门禁：它每核只有 5 次 Submit，而 `H=64`，不会进入 retire 循环。后续真实 A5 的 CaseB1 只可作为构建冒烟；候选正确性必须至少包含一次不跳过 golden 的 Case1。 + +#### 真实 CCEC 代码生成 + +基线和候选均通过真实 PA 的 `TestPagedAttentionUnroll.compile_chip_callable("a5")`、`perf-clock` 编译门构建；该入口只编译 orchestration、四个子 kernel 和 FDWIC AICore override，没有创建 Worker、调用 ACL 或连接设备。基线绑定提交 `970e4fad`，候选使用同一提交加单文件工作树差异；source-v2 指纹分别为： + +```text +baseline 2732d4ad0ee67e920d24cc5c66d6a300c86fc9904c7ba65dc53168e8694f841d +candidate fae3c47f0bbf7562b3b1cbd264ef80b2df2c7f6750580327a64fa61a30f9c751 +``` + +使用 CANN 9.1 `dav_3510` PEM decoder 检查 AIC/AIV 合并对象中的四类真实内联入口，八处结果全部闭合： + +| 入口 | AIC 函数体 | AIV 函数体 | 控制流结论 | +| --- | ---: | ---: | --- | +| `dist_submit_impl` | 5,656→5,664 B | 5,660→5,668 B | `-1` 跳到 id 递增；其他负值跳到 reset store | +| `dist_alloc_tensors` | 5,344→5,352 B | 5,404→5,412 B | 同上 | +| `dist_submit_compete_first_finish` | 2,572→2,580 B | 2,608→2,620 B | 同上 | +| `dist_alloc_compete_first_finish` | 3,008→3,016 B | 3,024→3,044 B | 同上 | + +基线中负值分支先落到 id 递增，随后仍顺序执行 reset store；候选把 reset store 移到递增之前，并增加精确 sentinel 比较：相等分支直接落到递增，非 `-1` 的负值分支仍落到 store。由此可以确认编译器没有把源码快路重新折叠回旧行为。 + +这项改动也有可见代码布局代价：AIC combined `.text` 增加 32 B；AIV combined 和最终 ELF 的 `.text` 因对齐保持不变，但各内联函数体仍有上述增加。布局变化只证明机器码形态，不能替代 perf-clock 性能结论。 + +#### A5Sim Case1 + +候选使用本用户 `.venv`、GCC 15 和当前 PTO-ISA 跑通真实 simpler PA 的 A5Sim Case1，完整执行 256 batch 和 golden： + +```text +TestPagedAttentionUnroll::test_run + 1 passed in 81.02s +``` + +该结果证明模拟路径的 1,280 次 Submit、retire 和 TensorMap 最终输出没有回归；它不是 A5 性能数据。 + +#### 待完成的真实 A5 裁决 + +当前按要求停在上板之前，不把离线结果写成“已保留”。恢复上板后的固定顺序为： + +1. Case1 不带 `--skip-golden`，闭合真实设备正确性；B1 仅作构建冒烟； +2. 同一环境交错运行基线 A / 候选 B 的 `perf-clock` Case1，由完整 Submit 墙钟决定保留或撤回； +3. 仅当 perf-clock 有稳定收益时，再用合并泳道确认变化落在 PrepareMap，业务 span、Submit 数和 atomic 次数不变； +4. 只有仍需解释 I-cache 方向时，才增加 PrepareMap 单阶段 PMU，不预先扩张观察面。 + +### 2026-07-22 / O9：纯 scalar-code Submit-PMU ABI/schema v2 口径重建 + +状态：**[观察工具：离线、CCEC 与首轮真实 A5 v2 门禁已闭合]** + +#### 旧数据结论暂停 + +新口径要回答的是 scalar 代码本身的开销：Submit 时间分母既不含 linked Kernel，也不含已发出 atomic 后等待返回值可用的时间。旧采集和旧 scalar I-cache HTML 使用的分母与 v2 不同，因此暂停引用其绝对结论和阶段比例，不与 v2 混用。 + +linked Kernel 继续由调用前的 `metrics_prof_stop` 和返回后的 `metrics_prof_start` 排除，对应的 scalar SYS 时间段也在边界处闭合和重开；所以 linked Cube/Vector Kernel 既不进 PMU counter，也不进 scalar 时间分母。 + +#### atomic 分类与扣时方式 + +对 28 个 atomic site 做穷举分类：16 个使用返回值的 return-ready site 进入扣时，12 个只发出操作的 source-issue site 保留在 scalar 开销内。return-ready 边界是 atomic 前的 SYS_CNT 到结果数据依赖落实后的 SYS_CNT；v2 只从完整 Submit 和所属 active phase 的 scalar 时间中扣除这段 SYS bracket。 + +不在每个 atomic 周围 stop/start PMU，因为那会向每次采样注入 PMU 控制开销并扰动 PIPE_ALL；也不加 DSB，因为本地结果数据依赖已经给出 return-ready 边界，额外屏障会改写被观测路径。因此 PMU counter 仍然包含 atomic 指令及其观察钩子，v2 的“纯 scalar-code”仅指时间分母口径，不表示 counter 已排除 atomic。 + +#### 门禁与当前证据 + +核心记录 ABI 保持 64 B，没有增长；raw JSON 因必选字段、状态位和时间语义都已不兼容旧文件，同步升级为 `fdwic-submit-pmu-v2`，不伪装成旧 schema。bit 17 是 return-ready atomic 扣时有效性的 fail-closed 门禁，状态异常或 bracket 未闭合时不允许生成正式产物。当前已通过 263 项 Python 离线用例和 2 项 C++ 用例，并闭合 phase 0/10 在 AIC/AIV 下的 CCEC 编译；同一 CCEC 优化 IR 还证明 atomic 返回值进入 `MOV -> SYS_CNT` 的显式数据依赖。CANN 当前的 `llvm-objdump` 不能解码 HIIPU，因此这里不把 IR 证据误称为机器指令反汇编。这些都是离线正确性与代码生成证据，不是性能结论。 + +#### 首轮真实 A5 闭合 + +已产生四个 v2 产物： + +- none B1：`outputs/TestPagedAttentionUnroll_CaseB1_20260722_211055` +- Winner B1：`outputs/TestPagedAttentionUnroll_CaseB1_20260722_211158` +- none Case1：`outputs/TestPagedAttentionUnroll_Case1_20260722_211333` +- Winner Case1：`outputs/TestPagedAttentionUnroll_Case1_20260722_211624` + +四轮的 trusted record、linked Kernel gate 闭合、return-ready atomic 扣时有效、vector busy 为零和 cube busy 为零等正式门禁均为 `96/96`。Winner Case1 的业务 phase calls 为 `1024 = 512 AIC + 512 AIV`，排除的 53 次 linked Kernel 调用全部位于 AIC。在该 Winner Case1 的同一 ELF 内，phase scalar-code core-time 占完整 Submit 分母的比例为 ALL/AIC/AIV `3.47% / 7.93% / 1.42%`。 + +none Case1 的 `4.982069 ms` 与 Winner Case1 的 `4.599385 ms` 来自不同 ELF，不能相减或解读为优化收益，本轮不写收益结论。旧 v1 下的其余 profile 和 scalar I-cache HTML 仍需按 v2 口径全量重跑；在此之前不恢复旧数据结论。 + +#### v2 既有 profile 全量重采 + +在已记录的 none 和 Winner Case1 之外，补齐了下列 9 个 Case1 v2 产物： + +- arg：`outputs/TestPagedAttentionUnroll_Case1_20260722_212608` +- empty：`outputs/TestPagedAttentionUnroll_Case1_20260722_212653` +- materialize：`outputs/TestPagedAttentionUnroll_Case1_20260722_212739` +- claim：`outputs/TestPagedAttentionUnroll_Case1_20260722_212857` +- register：`outputs/TestPagedAttentionUnroll_Case1_20260722_212943` +- transition：`outputs/TestPagedAttentionUnroll_Case1_20260722_213030` +- efdrain：`outputs/TestPagedAttentionUnroll_Case1_20260722_213152` +- prepare：`outputs/TestPagedAttentionUnroll_Case1_20260722_213240` +- fanin：`outputs/TestPagedAttentionUnroll_Case1_20260722_213326` + +上述 9 种与 none、Winner 合计 11 种既有 Case1 profile。使用报告链本身的 `load_capture` + `load_provenance` 做机器化复验，`11/11` 全部通过；每轮 trusted record、linked Kernel gate 闭合和 return-ready atomic 扣时有效均为 `96/96`。`11/11` 的 HTML 均存在，其 I-cache 逐核摘要只显示最小值/最大值。 + +EfDrain 轮闭合了 962 次 linked Kernel 排除；Fanin 轮的动态业务 calls 为 1,024，全局 call-count 门禁闭合。每个 profile 的 phase 百分比都只属于自己的独立 ELF 及其本轮 Submit 分母，不在跨 ELF 之间求和；empty 仅用作 bracket 标定，不当作业务 phase。至此旧 v1 数据不再用于当前结论，当前口径只认 v2 全量重采产物。 + +### 2026-07-22 / O10：AllocComplete v2 阶段闭合 + +状态：**[观察工具：284 项 UT 与真实 A5 B1/Case1 v2 门禁已闭合]** + +#### 真实业务边界 + +AllocComplete 只在真实 Alloc winner 上开窗。legacy/one-shot 路径从 `Claim.end` 开始，compete-first 路径因 Claim 已在 Begin 中完成，从 Finish 内的 `Register.end` 开始；两者都在 `dist_submit_complete_alloc()` 返回后立即闭合，与 schema-v4 泳道的完整 AllocComplete 尾动作一致。 + +该阶段使用 `dynamic_global`：协议只能确定全局调用数为 `B = expected_submits / 5`，Alloc winner 由多核竞争决定，不伪造 AIC/AIV 角色公式或逐核固定次数。B1 的唯一一次调用实际落在 AIV，Case1 也出现 1 次 AIV winner，直接证明角色必须保持动态。 + +AllocComplete 内的 HeapGuard 慢路可能回收并执行 linked Kernel，因此继续复用成对 pause/resume 把这些 Kernel 从阶段时间和 counter 中排除，不把本轮观测到的零次写成协议不可能。阶段时间同时复用 O9 的 return-ready atomic SYS bracket 扣时链；PMU counter 仍含 atomic 指令事件。新模式只复用现有 phase 字段和状态门禁，核心记录与 phase sidecar 仍各为 64 B，ABI 不增长。 + +#### 离线与真实 A5 证据 + +增量后的离线回归共 284 项 UT 全部通过。真实 A5 产物为： + +- B1：`outputs/TestPagedAttentionUnroll_CaseB1_20260722_215826` +- Case1：`outputs/TestPagedAttentionUnroll_Case1_20260722_220027` + +两轮都通过 v2 严格校验和 `96/96` trusted/kernel/return-ready atomic 门禁，全局动态 call-count 也闭合。Case1 的调用数为 `256 = 255 AIC + 1 AIV`，63 个核为零调用，排除的 linked Kernel 为 0 次；阶段 scalar-code 时间为 584,689 ticks，I-cache request 为 170,344，miss 为 8,790。同一 ELF 内，它占完整 Submit scalar-code core-time 分母的 `584,689 / 296,983,746 = 0.1968758%`；该轮全局 Submit 墙钟为 `4.648447 ms`。 + +这个百分比只能在 Case1 `20260722_220027` 的同一 ELF 内解释。AllocComplete 与 none、Winner 及其他 phase 均是独立 ELF，不对其百分比求和，也不用 `4.648447 ms` 与他轮墙钟相减或写成优化收益。 + +### 2026-07-22 / O11：LoserReplay v2 阶段闭合 + +状态：**[观察工具：305 项 Python UT、2 项 C++ 用例与真实 A5 B1/Case1 v2 门禁已闭合]** + +#### 源码、泳道与动态 shape + +LoserReplay 只存在于真实 Kernel loser 分支：它与 schema-v4 泳道共用 `Register.end` 起点，在 `drain_block_won()` 返回后立即闭合，对应报告边界 `register_end_to_drain_block_won_return`。Kernel winner 进入 WinnerBuild，Alloc winner 进入 AllocComplete，Alloc loser 只留在 Submit residual，都不生成 LoserReplay。 + +当前 PA 每个 batch 有 4 个 Kernel task，每个 task 由 96 核各回放一次并且全局只有 1 个 winner；其中每 `B` 的 Kernel winner 固定为 `2B AIC + 2B AIV`。因此 LoserReplay 的精确公式是： + +- ALL：`(4 × 96 - 4)B = 380B` +- AIC：`(4 × 32 - 2)B = 126B` +- AIV：`(4 × 64 - 2)B = 254B` + +这是全局和角色总数闭合，不伪造逐核固定次数；单核实际调用数为 `4B - 该核赢得的 Kernel task 数`。B1 闭合 `380 = 126 AIC + 254 AIV`，Case1 闭合 `97,280 = 32,256 AIC + 65,024 AIV`，与先前 schema-v4 泳道原始记录一致。 + +#### atomic、Kernel 与 ABI 语义 + +`drain_block_won()` 只做 BlockWon 状态轮询、lane claim 和本地 ring slot 构造，不调用 linked Kernel；取到待回放数据时也只由 `build_ring_slot()` 物化 slot，不在 LoserReplay 内执行该 slot 中的 linked Kernel。因此本阶段的 excluded Kernel 闭合值应为 0，而不是把业务 Kernel 时间算进 scalar control。 + +函数内消费 atomic 返回值的等待继续复用 O9 的 return-ready SYS bracket 从阶段时间中扣除；不消费返回值的 source-issue atomic 仍属于 scalar 发出开销，保留在时间分母内。PMU counter 仍包含两类 atomic 指令事件。新 phase 只复用已有累加器、状态位和 sidecar 字段，核心记录与 phase sidecar 仍各为 64 B，ABI 不增长。 + +#### 离线与真实 A5 证据 + +离线回归通过 305 项 Python UT 和 2 项 C++ 用例。真实 A5 产物为： + +- B1：`outputs/TestPagedAttentionUnroll_CaseB1_20260722_222358` +- Case1：`outputs/TestPagedAttentionUnroll_Case1_20260722_222533` + +两轮的 v2 严格校验、`96/96` trusted/linked-kernel/return-ready atomic 门禁和动态全局/角色 call-count 均闭合。B1 的逐核调用数为 `3–4`，Case1 的调用数为 `97,280 = 32,256 AIC + 65,024 AIV`，逐核 `1,005–1,020`（AIC `1,005–1,013`，AIV `1,014–1,020`），excluded Kernel 为 0。Case1 的阶段 scalar-code 时间为 5,021,979 ticks，I-cache request 为 8,218,072，miss 为 453,234；在同一 ELF 内，它占完整 Submit scalar-code core-time 分母的 `5,021,979 / 394,817,462 = 1.2719749%`。 + +Case1 执行了 97,280 对 begin/end observer，每对都含 shadow counter read-clear 和边界 bookkeeping，因此这是高频观察 ELF，不是无扰动业务 ELF。局部 SYS 时间边界故意位于 begin 读取之后、end 读取之前，但观察器仍会改变整体指令流、I-cache 和墙钟。该轮全局 Submit 墙钟 `5.256747 ms` 只用于证明采集闭合，不与 none、empty 或其他 phase 的独立 ELF 相减，不把跨 ELF 百分比求和，也不写任何收益结论。 + +### 2026-07-22 / O12：真实 PA 全 span 双证据链汇总 + +状态：**[观察工具：纯离线重算与现有 13 份 v2 产物闭合；未上板]** + +#### 目标与实现边界 + +本阶段不再增加 device phase、record 字段或 raw 容量，也没有重新运行 A5。新增 +`simpler_setup/tools/fdwic_submit_span_overview.py`，把现有泳道业务时间树与 13 份独立 +Submit-PMU ELF 放进同一份离线报告，但从数据结构上禁止两条证据链混算。 + +泳道输入固定为 producer 的 `l2_swimlane_records.json`，不使用含 Perfetto 显示轨道和合成显示事件的 +`merged_swimlane.json`，也不直接信任旁边已经生成的 analysis。工具先后读取 raw SHA，并复用现有 +schema-v4 analyzer 当场重算：96 核物理拓扑、每核 1,280 个连续 Submit、父子包含、Kernel 唯一归属、 +`dropped_records=0` 以及六组整数闭合全部通过后，才生成紧凑摘要。 + +PMU 输入固定为 13 个目录中的 raw/provenance 对。每份先通过现有 `load_capture()` 的 owner、selector、 +status、拓扑、调用 shape、linked-Kernel gate 与 return-ready atomic 时间门禁，再用 +`load_provenance()` 闭合 raw SHA、profile、编译宏和构建身份；缺 mode、重复 mode、场景键不一致或输入 +生成期间变化都会拒绝发布。JSON/HTML 先成对暂存，再在输出目录独占锁下替换；第二份发布失败会恢复 +旧 pair，不留下半新半旧的正式件。 + +本次没有给最新泳道反向伪造 provenance。13 份 PMU 实际来自四组 revision： + +```text +9d3acca8 908f9adf 94172c64 d67f3f5e +``` + +13 个 `aicore_kernel` SHA 均不同；泳道 raw 又没有 build sidecar。因此 overview 明确发布 +`swimlane_to_pmu_identity_bound=false`，只证明两条证据链各自严格闭合，并仅对齐 96 核拓扑和每核 +Submit 数。mixed revision 可以并列展示，不能跨 ELF 相减、拼接或求和。 + +#### 时间与 PMU 的精确口径 + +真实源码的首末关系重新按 `submit_pmu.h` 核实：首个 Submit 先取 start tick、再启动 PMU；末个 Submit +先关闭 scalar segment、停止 PMU，再取 end tick。PMU gate 因而嵌在首末 SYS closure 内，不是比该 +closure 更宽。每次 linked Kernel 又会成对 stop/start；`scalar_submit_elapsed_ticks` 是 gate-running +SYS 段的累计，再扣 result-used return-ready atomic 等待。PMU total/scalar-busy/primary I-cache 只在 +gate 打开时计数，不含 linked Kernel，但不会为 return-ready atomic 停表，所以仍含 atomic 指令和最小 +观察 hook 的事件；source-issue atomic 继续保留在时间与 counter 中。 + +泳道 `clock_freq_hz=1,000,000,000` 是 SYS counter 的换算频率，即 1 tick=1 ns；它与 AIC/AIV 约 +1.65 GHz 的 PMU cycle 频率不是同一个量。泳道百分比只在同一 ELF 的排他 core-work 树中闭合；每张 +PMU phase 卡则直接展示 `本 ELF 分子 / 本 ELF 分母 = 占比`。页面没有跨 PMU profile 的合计字段或 +堆叠图。动态 phase 额外显示逐核 calls min/max 与零调用核数，避免把 AllocComplete AIV 的零值误读为 +一次业务调用的零耗时。 + +#### 当前泳道同 ELF 分布 + +输入为： + +```text +outputs/TestPagedAttentionUnroll_Case1_20260722_104657/l2_swimlane_records.json +``` + +schema-v4 重算得到全局 Submit 墙钟 4.844066 ms。96 核累计的 SubmitEnvelope 精确分成 SubmitUnion +`85.3839%` 与 BetweenSubmitResidual `14.6161%`；SubmitUnion 进一步精确闭合为： + +| 区域 | 同一 SubmitUnion 占比 | +| --- | ---: | +| EfDrain | 16.1853% | +| Materialize | 25.3230% | +| PrepareMap | 5.3936% | +| Claim | 20.4422% | +| Fanin | 0.3989% | +| Register | 12.5173% | +| WinnerBuild | 1.6655% | +| AllocComplete | 0.2154% | +| LoserReplay | 3.1810% | +| SubmitInternalResidual | 10.8563% | +| SubmitTailResidual | 3.8217% | + +EfDrain 内部又闭合为 KernelUnion `51.2020%` 与 control `48.7980%`。当前泳道还有 2 个 Kernel event +落在 WinnerBuild，但 analysis 没有发布该 child 的 Kernel union 时长；所以泳道表保持“原始业务 +elapsed”名称，不假装为纯 Scalar。纯 Scalar 的 EfDrain/WinnerBuild/AllocComplete 归因只看分别排除 +linked Kernel 的 PMU control ELF。 + +SubmitInternalResidual 本轮只有 `Claim->Materialize`，与 ArgBuild PMU 边界对应；SubmitTailResidual +按 LoserReplay/Register/WinnerBuild/AllocComplete 四种结尾列出，但当前没有独立 PMU phase。覆盖矩阵 +同时列出 WorkerCompletion、OrchestrationReplay、Setup/Tail、FinalDrain 及其 Kernel/Residual 子项； +未覆盖项保持未覆盖，不以 residual 名称替代业务数据。Atomic、ClockBaseline、Commit、RingBp、DrainWon +均保持非加和 overlay。 + +#### 当前 PMU 独立 ELF 结果 + +13 份 Case1 v2 raw/provenance 全部重新严格加载。`none` 的完整 Scalar 时间分母、PMU total、 +scalar-busy、primary request/miss 单列；empty-bracket 单列为 observer 校准。11 个业务 phase 的同 ELF +ALL 比例为： + +| PMU phase | Scalar 时间 | request observed | miss observed | +| --- | ---: | ---: | ---: | +| ArgBuild | 5.927% | 19.119% | 27.021% | +| Materialize | 25.044% | 32.352% | 12.503% | +| Claim | 10.626% | 20.996% | 20.683% | +| Register | 6.028% | 14.119% | 14.486% | +| SubmitTransition | 14.231% | 22.198% | 27.843% | +| EfDrainControl | 7.092% | 16.735% | 12.784% | +| PrepareMap | 3.016% | 11.785% | 1.480% | +| Fanin | 0.358% | 0.486% | 1.027% | +| WinnerBuildControl | 3.472% | 5.047% | 1.815% | +| AllocCompleteControl | 0.197% | 0.244% | 0.403% | +| LoserReplay | 1.272% | 9.723% | 11.619% | + +每一行的三个百分比只属于该行自己的 ELF。局部 request/miss 是 running read-clear observed，仍包含 +边界附近 observer/bookkeeping；即使标出 capture gap，也不是无插桩业务事件的数学上下界。上述表只 +用于找下一步应深入的区域，不是可相加的整窗解释，也不是性能收益。 + +#### 产物与离线门禁 + +正式离线加工件位于： + +```text +outputs/down/fdwic_submit_span_overview_20260722_v2/ + fdwic_submit_span_overview.json + fdwic_submit_span_overview.html +``` + +新增合成测试覆盖 schema-v4 分区闭合、13 profile 完整 cohort、缺失/重复 profile、泳道/PMU Submit +数不一致、mixed git head、所有 exclusive/residual/外围区域覆盖、零 miss 分母、HTML 只显示 min/max、 +本 ELF 分母直显、成对发布回滚和并发发布锁。真实产物则完整执行一次 75 MB 泳道 raw 重算与 13 份 +raw/provenance 加载；最终 HTML/JSON 约 46 KB/122 KB，只是离线加工件，不进入设备热路径或性能采集。 + +最终离线回归为： + +```text +test_fdwic_submit_span_overview.py +test_fdwic_submit_pmu_report.py +test_fdwic_swimlane_converter.py +test_scene_test_cache.py + 386 passed + +test_fdwic_swimlane_poll_batch + 2 passed + +ruff check / format --check（本轮新增 Python 文件） +git diff --check + PASS +``` + +### 2026-07-23 / O13：阶段计时改为 PMU total 1.65 GHz 口径 + +状态：**[观察工具：ABI/schema v3、336 项定向 UT、真实 A5 B1/Case1 及 13 个 profile 全部闭合]** + +#### 为什么不能继续用 SYS tick 冒充阶段耗时 + +v2 的 `phase_elapsed_ticks` 是观察器边界内累计的 SYS_CNT：它适合确认 begin/end 是否闭合， +但不是阶段 local PMU total，也不能回答阶段内 scalar-busy 与非 scalar-busy 周期的关系。把它按 +1 tick = 1 ns 直接呈现为阶段主时间，会把边界时间戳、观察器 bookkeeping 和 PMU 计数口径混在一起， +也无法与 whole PMU total/scalar 正确闭合。 + +v3 因此明确拆成两条数据： + +1. 阶段主时间使用 `phase_total_cycles_observed`，由唯一 TOTAL PMU counter running read-clear， + 再在软件中重建 whole total；ALL/AIC/AIV 分别按 + `1.649844/1.650062/1.649731 cycles/ns` 换算等效时间。 +2. `phase_elapsed_ticks` 只作为 raw SYS 边界闭合诊断保留。HTML 不把它换算成阶段时间，不用它做 + phase 占比，也不再出现“1 GHz phase time”的解释。 + +同一阶段还新增 `phase_scalar_busy_observed`。CNT3 配成与 CNT2 相同的 scalar selector `0x001`： +CNT2 保留 whole primary，CNT3 负责运行中 read-clear 的 phase local scalar。begin 顺序为 +TOTAL → scalar → I-cache，end 按 I-cache → scalar → TOTAL 反序读取，使 scalar 观察窗嵌套在 +total 观察窗内。逐核必须满足： + +```text +phase_scalar_busy_observed <= phase_total_cycles_observed <= whole_total_cycles +phase_scalar_busy_observed <= shadow_scalar_busy <= whole_primary_scalar_busy +``` + +报告将逐核 `phase_total-phase_scalar` 后的结果命名为“非 Scalar-busy 残余”，不命名为空闲时间或 +I-cache stall。linked vector/cube Kernel 通过 phase pause/resume 从两类 PMU counter 和 SYS 诊断中 +共同排除；result-used return-ready atomic 只从 SYS 诊断扣除，仍进入 PMU total/scalar，因此 v3 +并未伪造“去 atomic 的纯 scalar PMU 时间”。 + +#### ABI 与热路径边界 + +设备侧 phase sidecar/GM 容量仍为 64 B/核，没有逐调用增长；host raw 只增加固定 96 核字段。 +v3 复用 sidecar 原空间，布局为： + +```text +elapsed@0 u64 +total@8 u64 +request@16 u64 +miss@24 u64 +phase scalar@32 u32 +shadow scalar@36 u32 +shadow request@40 u32 +shadow miss@44 u32 +phase_id@48 u16 +status@50 u16 +begin@52 u32 +end@56 u32 +excluded_kernel_calls@60 u32 +``` + +没有为阶段计数加入 DSB 或 phase-level `PIPE_ALL`。none ELF 禁止三个 phase reader 符号；phase ELF +必须同时包含 total/scalar/I-cache 三个 reader，避免 cache 复用错误变体。host raw schema 升为 +`fdwic-submit-pmu-v3`，显式发布 PMU total、phase scalar、whole shadow scalar 和 SYS 边界诊断语义。 + +#### 真机校准 + +B1 正式闭合件： + +- none：`outputs/TestPagedAttentionUnroll_CaseB1_20260723_043155/` +- empty：`outputs/TestPagedAttentionUnroll_CaseB1_20260723_043257/` +- Claim：`outputs/TestPagedAttentionUnroll_CaseB1_20260723_043400/` + +none 的 96 个核均满足 CNT3 shadow scalar 与 CNT2 primary scalar 精确相等。phase 运行中 +read-clear 后，empty 与 Claim 的 CNT3 shadow 均相对 CNT2 primary 固定少 `2 cycles/call`； +Case1 每核 1,280 次调用时正好少 2,560 cycles，证明误差随边界次数线性出现，raw 和 HTML 已显式 +展示，未把它隐去。 + +Case1 empty： + +| 角色 | PMU total cycles/call | 按角色频率换算 | +| --- | ---: | ---: | +| ALL | 1564.846 | 948.481 ns/call | +| AIC | 1485.029 | 899.984 ns/call | +| AIV | 1604.755 | 972.737 ns/call | + +这说明当前 phase 观察器本身接近 0.95 us/call，并占 empty ELF whole PMU total 的 23.116%。 +它是当前代码布局和固定调用点下测得的每组记录代码开销，不是可跨 ELF 精确扣除的常数。 + +#### Case1 v3 全量结果 + +13 个 profile 的 raw/provenance/HTML 全部重新采集并通过严格 loader。其中 none 没有 phase， +只提供 whole-window 分母；下表列出其余 12 个独立 phase ELF 的 ALL 聚合 96-core work。 +“whole 占比”只以本行同一 ELF 的 whole PMU total 为分母，所有行禁止求和： + +| phase | PMU total us | scalar us | 非 scalar 残余 us | whole total 占比 | scalar / phase total | request 占比 | miss 占比 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| EmptyBracket | 116549.405 | 75065.241 | 41484.163 | 23.116% | 64.406% | 7.621% | 7.362% | +| EfDrainControl | 157130.274 | 107609.078 | 49521.196 | 31.409% | 68.484% | 16.750% | 16.327% | +| Claim | 195140.831 | 139943.418 | 55197.413 | 38.921% | 71.714% | 20.593% | 26.235% | +| ArgBuild | 155328.743 | 103572.005 | 51756.738 | 29.577% | 66.679% | 18.066% | 16.534% | +| Materialize | 223263.299 | 178164.487 | 45098.812 | 42.825% | 79.800% | 32.059% | 23.466% | +| PrepareMap | 127362.293 | 87292.749 | 40069.544 | 26.781% | 68.539% | 12.242% | 1.645% | +| Fanin | 2126.520 | 1642.250 | 484.269 | 0.538% | 77.227% | 0.450% | 0.591% | +| Register | 150738.181 | 101939.366 | 48798.814 | 29.845% | 67.627% | 13.798% | 12.785% | +| WinnerBuildControl | 37868.391 | 36978.703 | 889.687 | 7.514% | 97.651% | 6.589% | 1.993% | +| AllocCompleteControl | 34679.708 | 34324.720 | 354.988 | 6.872% | 98.976% | 0.747% | 0.965% | +| LoserReplay | 103511.458 | 69363.871 | 34147.587 | 21.941% | 67.011% | 9.778% | 7.325% | +| SubmitTransition | 190622.524 | 125218.596 | 65403.928 | 35.938% | 65.689% | 21.768% | 30.605% | + +产物目录： + +```text +none outputs/TestPagedAttentionUnroll_Case1_20260723_043811/ +arg-build outputs/TestPagedAttentionUnroll_Case1_20260723_043857/ +empty-bracket outputs/TestPagedAttentionUnroll_Case1_20260723_043533/ +materialize outputs/TestPagedAttentionUnroll_Case1_20260723_043943/ +claim outputs/TestPagedAttentionUnroll_Case1_20260723_043639/ +register outputs/TestPagedAttentionUnroll_Case1_20260723_044029/ +submit-transition outputs/TestPagedAttentionUnroll_Case1_20260723_044115/ +efdrain-control outputs/TestPagedAttentionUnroll_Case1_20260723_044202/ +prepare-map outputs/TestPagedAttentionUnroll_Case1_20260723_044248/ +fanin outputs/TestPagedAttentionUnroll_Case1_20260723_044336/ +winner-build outputs/TestPagedAttentionUnroll_Case1_20260723_044422/ +alloc-complete outputs/TestPagedAttentionUnroll_Case1_20260723_044508/ +loser-replay outputs/TestPagedAttentionUnroll_Case1_20260723_044555/ +``` + +最新汇总件为： + +```text +outputs/fdwic_submit_span_overview_20260723_v3/ + fdwic_submit_span_overview.json + fdwic_submit_span_overview.html +``` + +目录名 `_v3` 表示输入采用 Submit-PMU v3 cohort；overview JSON 自身的 schema 为 +`fdwic-submit-span-overview-v5`。v5 为每个业务 phase 固化了 +`recording_cost_reference`，不再允许旧 payload 缺少参考值却沿用同一 schema。 + +总览中的泳道分区表新增 PMU 与 scalar-busy 两列。早期版本直接复用对应 phase ELF 的 +`phase raw observed / 同 ELF raw whole`，该数值包含高频 begin/end 记录代码开销，只能称为 +**raw 观测比例**，不能继续标成业务阶段占比。最终统一到单 phase 报告与 overview 共用的严格参考 +公式： + +```text +recording estimate = + AIC empty 每组记录开销 × 本阶段 AIC 记录组数 + + AIV empty 每组记录开销 × 本阶段 AIV 记录组数 + +reference numerator = phase raw observed - recording estimate +reference share = reference numerator / 同 phase ELF 的 raw whole +``` + +ALL 先汇总 AIC/AIV 的分子和分母再相除，不平均两个角色的百分比。分母明确保持 raw whole，不执行 +`whole raw - recording estimate`：empty-bracket 只测得局部记录组，没有测量完整 Submit 整窗的 +全部观察成本，不能制造一个“纯业务 whole”分母。页面同时保留 raw 比例、reference share 和 +`recording estimate / phase raw`，后者用于提示两个大数相减时的校准敏感度。工具同时要求业务 +phase 与 empty-bracket provenance 的场景和 Git revision 一致；两边都无 sidecar 时会明确标为 +“只校验 raw 配置和核拓扑，revision 未证明”，不会静默宣称构建身份已闭合。 + +Materialize 的 PMU total 从 raw `42.825%` 得到 reference `20.469%`，scalar-busy 从 raw +`38.698%` 得到 reference `22.393%`；同一总览的泳道 Materialize 业务时间占比为 `25.323%`。 +三类数据来自不同计数边界和不同 ELF，只能并列判断量级。曾经把分母也扣除记录估算得到的 +`26.363%/26.756%` 已明确废弃，因为该算法虚构了未被 empty 校准覆盖的 whole 扣除量。全部业务 +phase 单报告和 overview 已使用同一公式刷新，避免单报告继续显示 raw、overview 却显示另一种 +参考口径。 + +映射只覆盖同业务边界、相邻 Submit 边界或明确 `control-only` 的行；SubmitInternalResidual +聚合、SubmitTailResidual 和 KernelUnion 等无等价 profile 的行保持“—”。 + +overview 的 11 个业务 phase 跨 ELF 合成诊断改为四层，不再把 raw observed 合计相对 none 的 +膨胀直接命名为“观测偏差”或“插桩开销”： + +1. 原始 observed 合计； +2. 空区间估算的记录代码开销，即 + `AIC/AIV 各自的 empty-bracket 每组 begin/end 开销 × 该角色实际 begin/end 记录组数`； +3. 扣除记录开销后的参考值，即第 1 层减第 2 层； +4. `submit-pmu-none` 独立基线。 + +11-phase ALL 的 PMU total 三层 phase 值为 `14.351794/9.492491/4.859303 ms/core`， +Scalar busy 为 `10.271346/6.113844/4.157502 ms/core`，非 Scalar-busy 残余为 +`4.080448/3.378647/0.701800 ms/core`；none 基线三项分别为 +`5.884932/5.733441/0.151491 ms/core`。I-cache request/miss 同样按四层展示，但保持 +events/core。 + +第 2 层只是 empty-bracket ELF 在当前代码布局和运行状态下测得的估算，不是可跨 ELF 套用的精确 +常数；第 3 层也只能用来检查量级。11 个业务 phase、empty-bracket 与 none 都是独立 ELF/进程， +还混有边界覆盖空洞、交叠、运行波动和 control-only 语义。因此不能把第 3 层冒充无记录代码下的 +业务真值，不能拿它与 none 直接相减成优化收益；none 的非 Scalar-busy 残余分母很小，相关百分比 +尤其容易放大。 + +SubmitUnion 表进一步改成全均值口径：泳道时间显示每核平均，泳道、PMU total 和 scalar-busy +三个占比都由同口径每核均值相除。表尾先将 11 个泳道分段的每核时间相加，得到 +`4033.203 us/core`，与 SubmitUnion 父区间严格闭合为 100%；随后合计 9 个直接映射行与 ArgBuild +精确 residual 子段，共 10 个 SubmitUnion phase；SubmitTransition 位于 SubmitUnion 外，明确 +不进入分子。表尾使用与全局 11-phase 相同的四层结构；真实 ALL 每核等效时间为： + +| 指标 | 原始 observed | 记录开销估算 | 扣除后的参考值 | `submit-pmu-none` | +| --- | ---: | ---: | ---: | ---: | +| PMU total | 12.366143 ms/core | 8.279383 ms/core | 4.086759 ms/core | 5.884932 ms/core | +| Scalar busy | 8.966986 ms/core | 5.332525 ms/core | 3.634461 ms/core | 5.733441 ms/core | +| 非 Scalar-busy 残余 | 3.399157 ms/core | 2.946858 ms/core | 0.452299 ms/core | 0.151491 ms/core | + +none 的范围是首个 Submit begin 到末个 Submit end，包含 BetweenSubmitResidual；10-phase 分子 +没有覆盖其余 internal residual 和 tail residual。记录开销按真实 begin/end 记录组数估算， +linked Kernel pause/resume 产生的额外记录组也计入；AIC/AIV 分开计算后再按 32/64 核合并。 +这组跨 ELF 合计只能描述“各 phase raw observed 合计减去局部记录估算”的诊断量。即使数值接近 +正式泳道 SubmitUnion，也不能据此声称恢复了完整业务阶段、纯业务 whole 或无记录代码下的严格 +闭合；phase 覆盖空洞、交叠、control-only 语义、代码布局和独立进程波动仍然存在。严格可提供的 +占比只限于上式的 `reference numerator / 本 phase ELF raw whole`，并且必须标为参考值。动态 +phase 的均值始终以全部 96 核为总体，零调用核保留为零值，不改成 active-core 或 per-call 均值。 + +该 overview 使用最新已有泳道 raw +`outputs/TestPagedAttentionUnroll_Case1_20260722_104657/l2_swimlane_records.json`， +重新闭合 schema-v4 排他树；泳道没有 provenance，因此只声明拓扑和每核 Submit 数对齐，不声明与 +13 个 v3 PMU ELF 身份相同。 + +#### 验证与构建产物教训 + +Python 定向回归： + +```text +test_fdwic_submit_pmu_report.py +test_fdwic_submit_span_overview.py +test_scene_test_cache.py + 341 passed +``` + +AICPU、Claim phase AICore、none AICore 和 host runtime 都完成真实 CANN/A5 编译；C++ clang-format、 +`ruff check` 和 `git diff --check` 通过。`ruff format --check` 仍报告两处既有且与本轮无关的 +格式差异。首次 B1 none 曾出现 96 核记录全零，最终查明不是硬件 counter +失效，而是 AICore cache 已为 v3、实际 `build/lib/.../libhost_runtime.so` 仍停留在 v2。通过正式 +`RuntimeBuilder("a5").get_binaries("fully_distributed_within_core", build=True)` 重建并落盘后， +cache 与实际加载 host SO SHA 一致，所有真机门禁随即闭合。后续 ABI 变更必须核对实际加载件， +不能只看 build cache 或把 host/device ABI 不一致误判成 PMU 行为。 diff --git a/tests/atomic_probe/run_nested_lambda.sh b/tests/atomic_probe/run_nested_lambda.sh index 2babab2a27..22b83c50e6 100755 --- a/tests/atomic_probe/run_nested_lambda.sh +++ b/tests/atomic_probe/run_nested_lambda.sh @@ -1,8 +1,16 @@ #!/usr/bin/env bash +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- # Run the minimal nested-lambda/capture/template compiler probe on one or all # of the CPU, AscendC, and pure-CCEC paths. # -# Usage: ./run_nested_lambda.sh [cpu|ascendc|ccec|ccec-cross-tu|all] +# Usage: ./run_nested_lambda.sh [cpu|ascendc|ccec|ccec-caller-capture|all] set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -33,7 +41,6 @@ run_cpu() { -I"$REPO_ROOT/src/a5/runtime/fully_distributed_within_core/orchestration" \ -I"$REPO_ROOT/src/a5/runtime" \ "$SCRIPT_DIR/cpu/nested_lambda_args_runtime_read.cpp" \ - "$SCRIPT_DIR/ccec/nested_lambda_cross_tu_runtime.cpp" \ -Wl,--gc-sections \ -o "$BUILD_DIR/nested_lambda_args_runtime_read_cpu" "$BUILD_DIR/nested_lambda_args_runtime_read_cpu" @@ -62,13 +69,13 @@ run_ccec() { "$SCRIPT_DIR/ccec/run_all.sh" nested_lambda } -run_ccec_cross_tu() { +run_ccec_caller_capture() { require_cann if [[ -z "${PTO_ISA_ROOT:-}" ]] && \ [[ -f "$ASCEND_HOME_PATH/x86_64-linux/include/pto/common/kernel_meta.hpp" ]]; then export PTO_ISA_ROOT="$ASCEND_HOME_PATH/x86_64-linux" fi - echo "=== Pure CCEC AIC cross-TU ABI path ===" + echo "=== Pure CCEC AIC inline caller-capture path ===" "$SCRIPT_DIR/ccec/run_all.sh" nested_lambda_cross_tu } @@ -82,16 +89,17 @@ ascendc) ccec) run_ccec ;; -ccec-cross-tu) - run_ccec_cross_tu +ccec-caller-capture | ccec-cross-tu) + run_ccec_caller_capture ;; all) run_cpu run_ascendc run_ccec + run_ccec_caller_capture ;; *) - echo "Usage: $0 [cpu|ascendc|ccec|ccec-cross-tu|all]" >&2 + echo "Usage: $0 [cpu|ascendc|ccec|ccec-caller-capture|all]" >&2 exit 2 ;; esac diff --git a/tests/atomic_probe/shared_preload_model_shared.h b/tests/atomic_probe/shared_preload_model_shared.h new file mode 100644 index 0000000000..549e352c4a --- /dev/null +++ b/tests/atomic_probe/shared_preload_model_shared.h @@ -0,0 +1,136 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the license. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef TESTS_ATOMIC_PROBE_SHARED_PRELOAD_MODEL_SHARED_H_ +#define TESTS_ATOMIC_PROBE_SHARED_PRELOAD_MODEL_SHARED_H_ + +#include "cache_preload_shared.h" + +#include +#include + +namespace shared_preload_model { + +#if defined(__CCE_AICORE__) +#define SHARED_PRELOAD_MODEL_FN __aicore__ +#else +#define SHARED_PRELOAD_MODEL_FN +#endif + +constexpr uint32_t kAicWorkers = 32; +constexpr uint32_t kAivWorkers = 64; +constexpr uint32_t kWorkers = kAicWorkers + kAivWorkers; +constexpr uint32_t kCacheLineBytes = 64; +constexpr uint32_t kMaxPayloadBytes = 384; +constexpr uint32_t kWriterHistoryBytes = 40; +constexpr uint32_t kOneDescriptorBytes = 128; +constexpr uint32_t kThreeDescriptorBytes = 384; +constexpr uint32_t kDCacheSamples = 11; +constexpr uint32_t kICacheSamples = 13; +constexpr uint32_t kShortGapRounds = 0; +constexpr uint32_t kOverlapGapRounds = 64; +constexpr uint32_t kICachePreloadUnits = 2; +constexpr uint32_t kControlMagic = 0x53504d44U; // "SPMD" +constexpr uint32_t kStatusComplete = 0x434f4d50U; // "COMP" + +enum class Experiment : uint32_t { + Publish = 0, + Consume = 1, + ICachePlacement = 2, +}; + +enum class Mode : uint32_t { + DCacheBaseline = 0, + DCachePreload = 1, + ICacheBaseline = 2, + ICacheCallerPreload = 3, + ICacheTargetPreload = 4, +}; + +struct alignas(64) ProbeControl { + uint32_t magic; + uint32_t experiment; + uint32_t mode; + uint32_t active_bytes; + uint32_t gap_rounds; + uint32_t sample_id; + uint32_t first_worker; + uint32_t worker_count; + uint64_t seed; + uint64_t reserved[3]; +}; + +struct alignas(64) ProbeResult { + uint64_t setup_ticks; + uint64_t issue_ticks; + uint64_t gap_ticks; + uint64_t access_ticks; + uint64_t publish_ticks; + uint64_t total_ticks; + uint64_t preparation_checksum; + uint64_t result_checksum; + uint64_t icache_immediate_status; + uint64_t icache_final_status; + + uint32_t worker_id; + uint32_t experiment; + uint32_t mode; + uint32_t active_bytes; + uint32_t gap_rounds; + uint32_t sample_id; + uint32_t status; + uint32_t reserved0; + uint64_t reserved[2]; +}; + +struct alignas(64) WorkerData { + alignas(64) uint8_t source[kMaxPayloadBytes]; + alignas(64) uint8_t destination[kMaxPayloadBytes]; +}; + +SHARED_PRELOAD_MODEL_FN constexpr uint8_t PayloadByte( + uint32_t worker, uint32_t byte +) { + return static_cast( + 0x5aU ^ (worker * 29U) ^ (byte * 17U) ^ (byte >> 3U) + ); +} + +SHARED_PRELOAD_MODEL_FN inline uint64_t PayloadChecksum( + uint32_t worker, uint32_t bytes +) { + uint64_t value = 0xcbf29ce484222325ULL; + for (uint32_t byte = 0; byte < bytes; ++byte) { + value ^= static_cast(PayloadByte(worker, byte)); + value *= 0x100000001b3ULL; + } + return value; +} + +SHARED_PRELOAD_MODEL_FN constexpr uint32_t CacheLinesForBytes( + uint32_t bytes +) { + return (bytes + kCacheLineBytes - 1U) / kCacheLineBytes; +} + +static_assert(sizeof(ProbeControl) == 64, "control must occupy one cache line"); +static_assert(sizeof(ProbeResult) == 128, "result must occupy two cache lines"); +static_assert(sizeof(WorkerData) == 768, "worker data ABI changed"); +static_assert(alignof(WorkerData) == kCacheLineBytes, "worker data must be cacheline aligned"); +static_assert(kWriterHistoryBytes == 16U + 3U * 8U, "three-symbol history byte count changed"); +static_assert(kOneDescriptorBytes == 2U * kCacheLineBytes, "one TensorDesc must occupy two lines"); +static_assert(kThreeDescriptorBytes == 6U * kCacheLineBytes, "three TensorDesc values must occupy six lines"); + +#undef SHARED_PRELOAD_MODEL_FN + +} // namespace shared_preload_model + +#endif // TESTS_ATOMIC_PROBE_SHARED_PRELOAD_MODEL_SHARED_H_ diff --git a/tests/atomic_probe/test_atomic_probe.py b/tests/atomic_probe/test_atomic_probe.py index 65a8dc22cf..b6b5f934f8 100644 --- a/tests/atomic_probe/test_atomic_probe.py +++ b/tests/atomic_probe/test_atomic_probe.py @@ -87,12 +87,19 @@ def test_a5_ccec_cacheline_probes(st_device_ids: list[int]) -> None: @pytest.mark.platforms(["a5"]) @pytest.mark.device_count(1) @pytest.mark.timeout(900) -def test_a5_ccec_nested_lambda_args_runtime_read(st_device_ids: list[int]) -> None: +def test_a5_ccec_nested_lambda_call_boundary_controls(st_device_ids: list[int]) -> None: environment = _onboard_environment(st_device_ids) - environment["ATOMIC_PROBE_MODE"] = "args-runtime-read" subprocess.run( - [str(HERE / "run_nested_lambda.sh"), "ccec-cross-tu"], - cwd=HERE, + [str(HERE / "ccec" / "run_all.sh"), "nested_lambda_cross_tu", "build"], + cwd=HERE / "ccec", env=environment, check=True, ) + for mode in ("args-runtime-read", "weak-context-materialize-0"): + environment["ATOMIC_PROBE_MODE"] = mode + subprocess.run( + [str(HERE / "ccec" / "run_all.sh"), "nested_lambda_cross_tu", "run"], + cwd=HERE / "ccec", + env=environment, + check=True, + ) diff --git a/tests/atomic_probe/test_case.md b/tests/atomic_probe/test_case.md index 4a9f330374..f29a526461 100644 --- a/tests/atomic_probe/test_case.md +++ b/tests/atomic_probe/test_case.md @@ -36,6 +36,13 @@ DCCI、`st_dev` 与 atomic 的 API 功能、隔离规则和代码评审清单见 - 2026-07-14：新增 CCEC `ld_dev_fanout_publish`。24 AIV 受控对照中，ordinary+DSB 为 `0/4416` 可见，st_dev+DSB 与 AtomicExch 均为 `4416/4416`；72 AIV 持续读压力同时破坏独立 control 对照，单列记录为高压力进展失败，不能外推为某个 data writer 的独立语义结论。 +- 2026-07-18：新增 CCEC 单 AIV `atomic_scalar_pmu`，以 EMPTY/SCALAR_CONTROL 扣除 + gate 和同构标量递推开销。三个独立会话的 8192 次 dependent `atomicAdd` 均显示： + atomic 额外 PMU total 几乎 100% 同步增加到 `scalar_instr_busy(0x1)`。 +- 2026-07-18：新增 CCEC 单 AIV `icache_scalar_pmu`，在同一静态调用点配对执行 + WARM/COLD 同一 target。三个独立会话共 33 对都是 `WARM miss=0`、 + `COLD miss=68`；COLD-WARM 只增加 48 scalar busy cycle，但增加 `2309..2312` + total cycle，证明本场景中 I-cache refill 等待的绝大多数周期不计入 scalar busy。 - 原始环境与定量结果记录在 `tests/ATOMIC_MINIBENCH_ONBOARD_LOG.md` 的 2026-07-11 与 2026-07-13 小节。 ## 权威覆盖矩阵 @@ -356,6 +363,162 @@ ATOMIC_PROBE_AIVS=24 ATOMIC_PROBE_FANOUT_LAUNCHES=3 \ tests/atomic_probe/ccec/run_all.sh ld_dev_fanout_publish ``` +## 裸 `st_dev` / `ld_dev` 单次多核同步 + +`ccec/st_dev_ld_dev_sync.cpp` 隔离测量一次一写多读同步，不复用前述 fanout +用例的 ready/control/ack 原子协议，也不执行多轮 replay。启动 `N=2..20` 个 +AIV，固定 block0 为 writer，其余 `N-1` 个 block 为 reader：writer 只对独占 +64 B line 的 signal word 执行一次裸 `st_dev`，其后不显式执行 DSB；reader +持续用 `ld_dev` 读取同一地址，直到精确观察到 `0x53594e43`。reader 使用 +20 ms 的设备侧有限超时，失败后仍参加末尾 SyncAll，避免错误路径死锁。 + +每次 launch 的执行和计时边界为： + +```text +N 个 AIV 执行初始 SyncAll（不计时） + -> 每核记录 begin + -> block0: raw st_dev(signal) + 其余核: ld_dev(signal) 直到精确命中或超时 + -> 每核记录 observe 和 final_arrive + -> N 个 AIV 执行末尾 SyncAll + -> 每核记录 end（计时结束） + -> 各核发布结果（不计时） +``` + +host 以 `overall = max(end) - min(begin)` 报告完整同步跨度，因此包含裸 +`st_dev`、所有 reader 的等待以及末尾 SyncAll，但不包含初始 SyncAll 和结果回写。 +其余诊断口径为： + +- `writer_st_dev_span = writer.observe - writer.begin`，只包围裸 `st_dev` 和取时开销； +- `last_reader_observe = max(reader.observe) - writer.begin`； +- `final_arrival_skew = max(final_arrive) - min(final_arrive)`，表示先到核等待慢核的跨度； +- `final_sync_release = max(end) - max(final_arrive)`，表示最后一个核到达后直到所有核 + 被放行并完成取时的尾部，不是 SyncAll 单条指令的纯硬件延迟； +- 末尾屏障从第一个核到达到全部放行的完整跨度为 + `max(end) - min(final_arrive)`。 + +signal 独占一条 64 B cache line，这是本协议的硬约束；其余 60 B padding 不得复用为 +flag、counter、结果或任何活跃对象，否则额外访问会改变同址 fanout 模型。每核结果和 +tail guard 也分别从后续独立 64 B line 开始。该布局由 `sizeof`、`alignof` 和 `offsetof` +静态断言约束，host 还要求 signal padding 全部保持为零。host 对每个 launch +精确检查实际 block 数、唯一 `(core_id, subblock_id)`、writer/reader 角色、目标值、 +timeout flag、时间顺序、未使用结果槽和 guard。结果槽在 `end` 之后用 `st_dev` +写回并完成发布，因此结果回写不影响被测 signal 的“裸 `st_dev`、无显式 DSB”口径。 + +2026-07-22 在 A5 device 0 直接执行；每组先预热 10 次，再采集 200 个样本。 +环境中没有 `task-submit` 和 `npu-smi`，所以两组都是未经设备锁隔离的直跑，不能 +确认采样期间不存在外部负载。两组共 400 个计量样本全部通过精确值和时间边界校验， +没有 reader timeout： + +| AIV 拓扑 | overall min / p50 / p95 / max（SYS_CNT ticks） | writer span p50 | last reader p50 / p95 | final arrival skew p50 | final release p50 / p95 | 最大 reader polls p50 / p95 | +|---|---:|---:|---:|---:|---:|---:| +| 3 核：1 写 + 2 读 | 609 / 612 / 620 / 746 | 9 | 488 / 495 | 478 | 125 / 127 | 2 / 2 | +| 20 核：1 写 + 19 读 | 722 / 2070 / 6349 / 10471 | 9 | 1912 / 6186 | 1903 | 156 / 161 | 11 / 46 | + +表中所有时间列的原始单位都是 A5 1 GHz `get_sys_cnt()` tick；数值除以 1000 +即为 μs。约 1.65 GHz 是当前 AICore PMU/core-cycle 频率，只用于换算 PMU +cycle，不能用来换算本表的 SYS_CNT tick。3 核组 +`overall` p50 为 0.612 μs，20 核组为 2.070 μs，本次配对样本中前者约为后者的 +`0.296x`，即约快 3.38 倍。裸 `st_dev` 记录跨度两组同为 9 SYS_CNT ticks，差值主要出现 +在最后一个 reader 的观察时间；这只能说明本微基准中同址 reader fanout 增加时 +等待和长尾上升，不能仅凭两组未隔离样本推导任意核数的缩放曲线或生产同步上限。 +同日相同裸写数据路径的另外两次 20 核、各 200 样本运行得到 `overall` p50 +2.659 μs 和 2.818 μs；这进一步表明无设备锁条件下存在明显会话级波动，不能把 +表中的 3.38 倍当作稳定缩放系数。 + +复现入口： + +```bash +# 默认 20 AIV +tests/atomic_probe/ccec/run_all.sh st_dev_ld_dev_sync + +# 3 AIV：1 writer + 2 readers +ATOMIC_PROBE_AIVS=3 tests/atomic_probe/ccec/run_all.sh st_dev_ld_dev_sync +``` + +## PMU 对 atomic 与 I-cache miss 等待周期的精确归类 + +`ccec/atomic_scalar_pmu.cpp` 与 `ccec/icache_scalar_pmu.cpp` 是两个独立单 AIV 微基准。 +它们共用 `pmu_probe_control.h` / `pmu_probe_aicpu.cpp` 的 108 physical sub-core +MMIO 表与 PMU 所有权协议；I-cache host 另用 `pmu_probe_host_support.h` 封装同一协议。 +AICPU helper 会保存并读回核验 +CTRL、slot 0/1/2 selector 和 START/STOP range，配置： + +| PMU 计数 | 事件 | 用途 | +|---|---:|---| +| slot 0 | `0x1` | `scalar_instr_busy` | +| slot 1 | `0x34` | `icache_req` | +| slot 2 | `0x35` | `icache_miss` | +| total | 固定总周期计数器 | PMU gate 内的总 AICore cycle | + +每个用例在待测段前后只执行 `metrics_prof_start/stop`，关窗后才用 +`ld_dev` 读取 total/scalar/request/miss，最后恢复进程进入用例前的 PMU 配置。 +它们不依赖 `msprof task-based` 的整任务 context 计数，也不与另一个 PMU session +并发执行。 + +### dependent atomic 等待计入 scalar busy + +Atomic 用例对每个 rounds 依次执行： + +1. `EMPTY`：只量 gate/read 固定开销； +2. `SCALAR_CONTROL`：使用 scalar 寄存器执行与 atomic 路径同构的 + `old/delta/checksum` 递推； +3. `DEPENDENT_ATOMIC_ADD`：`old` 改由 64-bit `atomicAdd` 返回，下一次 addend + 依赖上一次返回值，强制测量 atomic 完成延迟而不是无依赖吞吐。 + +Host 对 CONTROL/ATOMIC 复算完全相同的 checksum，并检查 atomic 终值、 +physical core id 和 `CTRL.bit0 == 0`。2026-07-18 在 A5 device 0 上的三个独立 +进程会话均执行 `8192 次 × 7 组`： + +| 会话 | `(ATOMIC-CONTROL)` 完成延迟 | PMU total cycle/op | scalar busy cycle/op | scalar / total | +|---:|---:|---:|---:|---:| +| 1 | 182.922729 ns | 301.804321 | 301.803955 | 0.999998787 | +| 2 | 251.687622 ns | 415.253540 | 415.253174 | 0.999999119 | +| 3 | 271.009888 ns | 447.110718 | 447.110352 | 0.999999181 | + +完成延迟在三个会话中处于不同档位，当前证据不足以归因；但计数归类完全一致： +**dependent `atomicAdd` 增加的 PMU total 周期几乎 100% 同步计入 +`scalar_instr_busy(0x1)`。** `get_sys_cnt` 是 1 GHz 时基，表中 PMU total/scalar +则是 AICore 核时钟 cycle，两者不能直接按同一单位比较。 + +### I-cache miss 回填等待的绝大多数周期不计入 scalar busy + +I-cache 用例的 WARM/COLD 两条路径在 PMU 开窗前汇合，窗口内只从同一个 +静态调用点执行一次同一个 target： + +- WARM 在窗外先调用一次 target； +- COLD 在窗外先执行超过 16 KiB AIV scalar I-cache 容量的 evictor； +- 最终 ELF 硬校验 target 为 `8280 B @ 0x0`、evictor 为 `32836 B @ 0x2080`， + 两者均按 128 B 对齐且区间不重叠； +- Host 逐样本复算 target/prepare checksum，并要求每对 WARM/COLD 使用同一 + physical AIV、`COLD miss > WARM miss`。 + +2026-07-18 在 A5 device 0 执行三个独立进程会话，每个会话 11 对， +交替使用 WARM,COLD 和 COLD,WARM 顺序。全部 33 对均为 +`WARM miss=0`、`COLD miss=68`，且 checksum、mode echo、PMU 关窗与恢复全部通过： + +| 会话 | WARM `total/scalar/req/miss` | COLD `total/scalar/req/miss` | total 增量 | scalar 增量 | miss 增量 | scalar / total 增量 | +|---:|---|---|---:|---:|---:|---:| +| 1 | `1068/1060/520/0` | `3377/1108/588/68` | 2309 | 48 | 68 | 2.078822% | +| 2 | `1068/1060/520/0` | `3379/1108/588/68` | 2311 | 48 | 68 | 2.077023% | +| 3 | `1068/1060/520/0` | `3380/1108/588/68` | 2312 | 48 | 68 | 2.076125% | + +因此准确结论是：**本场景中 I-cache miss 回填等待的绝大多数周期不计入 +scalar busy，但不是 scalar 增量严格为零。** 额外 68 次 miss 产生 `2309..2312` +total cycle，scalar busy 只增加 48 cycle，其余 `2261..2264` cycle 形成 +scalar-busy gap。target 实际覆盖 65 条 cache line，另 3 次 miss 与顺序预取相符； +因此 `total_delta / 68` 只能作为本窗口归一化值，不能称为单次阻塞 +I-cache miss 的精确延迟。 + +两个用例均只使用本机 CANN/PTO-ISA，不下载外部 PTO-ISA： + +```bash +source /home/q00473782/cann/cann-9.1.0/set_env.sh +cd tests/atomic_probe/ccec +./run_atomic_scalar_pmu.sh +./run_icache_scalar_pmu.sh +``` + ## 其余探针 | 文件 | 类型 | 验证内容 | @@ -371,13 +534,39 @@ ATOMIC_PROBE_AIVS=24 ATOMIC_PROBE_FANOUT_LAUNCHES=3 \ | `ascendc/st_dev_separate_line_stress.asc` / `ccec/st_dev_separate_line_stress.cpp` | regression gating | 只含分-line 数据路径；四模式覆盖两组活跃 block 与两种 allocation 内 line offset，100000 次精确终值检查 | | `ascendc/atomic_exch_same_line.asc` / `ccec/atomic_exch_same_line.cpp` | gating + control | 与 st_dev 同构的 AtomicExch 末值顺序对照；三组路径均精确通过 | | `ccec/ld_dev_fanout_publish.cpp` | regression gating + timing | 唯一 writer 以 ordinary+DSB、st_dev+DSB、AtomicExch 三种方式逐轮发布；其余全部 AIV 只用 ld_dev 读取完整序列，并记录 writer/全读者周期 | +| `ccec/atomic_scalar_pmu.cpp` | gating + PMU classification | 单 AIV dependent atomicAdd 完成延迟与同构 scalar control 对照；核实 atomic 等待是否计入 scalar busy | +| `ccec/icache_scalar_pmu.cpp` | gating + PMU classification | 单 AIV 同一 target 的 WARM/COLD I-cache 对照；核实 miss 回填等待是否计入 scalar busy | | `ascendc/dcci_atomic_stress.asc` | legacy observation | 旧的混合 stress；不再作为 DCCI selector 语义证据 | | `ccec/dcci_clean_clobber.cpp` | gating | 有序 dirty/clean line 的 dcci clobber 与 control | | `ascendc/mb2_flags_clobber.asc` | gating + observation | AtomicMax flags 无丢失；store+dcci 仅统计 | | `ascendc/mb8_dcci_seam.asc` / `ccec/dcci_seam.cpp` | gating | clean reader 的 DEFAULT/ALL/OUT/ATOMIC/no-DCCI 五模式精确对照 | | `ascendc/dcci_atomic_clobber.asc` / `ccec/dcci_atomic_clobber.cpp` | regression gating + control | 同-line 三 selector 当前明确失败；分-line 与 no-DCCI 五模式精确通过 | +| `pa_scheduler/ccec/kernel.cpp` | calibration | cold/warm 同核配对；每个 cold trial 严格增加一个 CNT7 I-cache miss，建立 scalar 时间标尺 | | `cpu/cpu_atomicity.cpp` | gating + observation | coherent CPU 同/异 cacheline 同构 control、atomic、snapshot、spinlock | +### PA I-cache 单 miss 实测数据 + +2026-07-18 在 device 0、32 AIC + 64 AIV 并发、`msprof PipeUtilization` 下， +`icache-single` 得到以下结果。时间列为多轮 `ns/miss` 中位数，括号内是最小值～最大值： + +| 配置 | 每轮 cold/warm CNT7 miss（ALL） | 严格门禁 | ALL | AIC | AIV | +|---|---:|---:|---:|---:|---:| +| 64 trials/core × 10 | 6,144 / 0（2,048 AIC + 4,096 AIV） | 10/10 PASS | 86.596（86.532～86.792） | 85.913（85.848～86.202） | 86.938（86.861～87.086） | +| 128 trials/core × 5 | 12,288 / 0（4,096 AIC + 8,192 AIV） | 5/5 PASS | 89.629（89.615～89.648） | 92.100（91.984～92.267） | 88.410（88.310～88.440） | + +两组每轮均为 `calibrated_cores=96/96`，并通过 “each cold trial adds exactly +one CNT7 I-cache miss” 断言。AIC/AIV 差值只有数 ns 且方向随运行时段变化， +因此不建立两个伪精确常数。原始日志为 +[`64×10`](pa_scheduler/outputs/pmu_validation/icache_single_64x10_20260718_085929_3232836_console.log) +和 +[`128×5`](pa_scheduler/outputs/pmu_validation/icache_single_128x5_20260718_090151_3235468_console.log)。 + +PA scalar 分析只需要数量级时，使用 `T_icache_est_ns = CNT7_miss_total * 90`；例如 +1,000 个 I-cache miss 约为 90 us。compulsory、capacity、conflict miss 都包含在 +`CNT7_miss_total` 内。该乘积是 cold/warm 校准得到的一阶等效时间，不是逐次精确 +可加的 stall；方法、角色分项和原始日志见 +[`PA调度器独立复现与泳道使用指南.md`](pa_scheduler/PA调度器独立复现与泳道使用指南.md#单次-cnt7-i-cache-miss-的-scalar-估算标尺)。 + ## 判定标准与退出码规则 1. **确定性安全契约必须 gating**：目标值、邻居值、参与核数、执行 marker 全部精确匹配；任一失败返回非零。 diff --git a/tests/atomic_probe/trace_write_preload_shared.h b/tests/atomic_probe/trace_write_preload_shared.h new file mode 100644 index 0000000000..b3c65089aa --- /dev/null +++ b/tests/atomic_probe/trace_write_preload_shared.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the license. + * ----------------------------------------------------------------------------------------------------------- + */ + +#ifndef TESTS_ATOMIC_PROBE_TRACE_WRITE_PRELOAD_SHARED_H_ +#define TESTS_ATOMIC_PROBE_TRACE_WRITE_PRELOAD_SHARED_H_ + +#include +#include + +namespace trace_write_preload { + +#if defined(__CCE_AICORE__) +#define TRACE_WRITE_PRELOAD_FN __aicore__ +#else +#define TRACE_WRITE_PRELOAD_FN +#endif + +constexpr uint32_t kAicWorkers = 32; +constexpr uint32_t kAivWorkers = 64; +constexpr uint32_t kWorkers = kAicWorkers + kAivWorkers; +constexpr uint32_t kCacheLineBytes = 64; +constexpr uint32_t kRecordBytes = 32; +constexpr uint32_t kRecordsPerLine = kCacheLineBytes / kRecordBytes; +constexpr uint32_t kWorkerStrideBytes = 128 * 1024; +constexpr uint32_t kMaxRecordsPerWorker = kWorkerStrideBytes / kRecordBytes; +constexpr uint32_t kCapacityPasses = 8; +constexpr uint32_t kControlMagic = 0x5457504cU; // "TWPL" +constexpr uint32_t kStatusComplete = 0x434f4d50U; // "COMP" + +enum class Experiment : uint32_t { + TraceWrite = 0, + CapacitySweep = 1, +}; + +struct alignas(32) TraceRecord { + uint64_t start_cycle; + uint64_t end_cycle; + int32_t task_id; + int32_t function_id; + uint32_t flags; + uint16_t phase; + uint16_t auxiliary; +}; + +struct alignas(64) ProbeControl { + uint32_t magic; + uint32_t experiment; + uint32_t first_worker; + uint32_t worker_count; + uint32_t records_per_worker; + uint32_t preload_distance_lines; + uint32_t preload_cadence_lines; + uint32_t sample_id; + uint32_t capacity_lines; + uint32_t capacity_passes; + uint32_t capacity_start_line; + uint32_t reserved0; + uint64_t seed; + uint64_t worker_stride_bytes; +}; + +struct alignas(64) ProbeResult { + uint64_t phase_begin; + uint64_t phase_split; + uint64_t phase_end; + uint64_t terminal_value; + uint32_t worker_id; + uint32_t experiment; + uint32_t records_or_lines; + uint32_t preload_distance_lines; + uint32_t preload_cadence_lines; + uint32_t sample_id; + uint32_t status; + uint32_t reserved0; +}; + +TRACE_WRITE_PRELOAD_FN constexpr uint64_t RecordStamp(uint64_t seed, uint32_t worker, uint32_t record) { + return seed + static_cast(worker) * 0x100000001ULL + static_cast(record) * 2ULL; +} + +TRACE_WRITE_PRELOAD_FN constexpr uint32_t RecordFlags(uint32_t worker, uint32_t record) { + return 0xa5000000U ^ (worker << 16U) ^ record; +} + +static_assert(sizeof(TraceRecord) == kRecordBytes, "trace record must remain exactly 32 B"); +static_assert(alignof(TraceRecord) == kRecordBytes, "trace record must remain 32 B aligned"); +static_assert(sizeof(ProbeControl) == kCacheLineBytes, "control must occupy one cache line"); +static_assert(sizeof(ProbeResult) == kCacheLineBytes, "each result must occupy one cache line"); +static_assert(kMaxRecordsPerWorker % kRecordsPerLine == 0, "worker stride must contain whole cache lines"); + +#undef TRACE_WRITE_PRELOAD_FN + +} // namespace trace_write_preload + +#endif // TESTS_ATOMIC_PROBE_TRACE_WRITE_PRELOAD_SHARED_H_ diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index f750857702..dc3c453d2a 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -434,6 +434,206 @@ add_a2a3_runtime_test(test_a2a3_scope_stats_collector a2a3/test_scope_stats_coll # --------------------------------------------------------------------------- add_a5_test(test_a5_fatal a5/test_a5_fatal.cpp) +# FDWIC's device-side PollBatch accumulator is header-only. Compile the exact +# production implementation through the A5Sim target macros so the 24-bit +# split boundary is exercised without introducing a second model. +add_a5_test(test_fdwic_swimlane_poll_batch a5/test_fdwic_swimlane_poll_batch.cpp) +target_compile_definitions(test_fdwic_swimlane_poll_batch PRIVATE __CPU_SIM=1) +target_include_directories(test_fdwic_swimlane_poll_batch BEFORE PRIVATE + ${CMAKE_SOURCE_DIR}/../../../src/a5/platform/sim/aicore + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/runtime + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/common + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime +) + +# DistTensorMap ring 是 header-only production 实现。同一源码先锁默认 +# CAP=128，再重编译四个结构边界，覆盖额外 bucket control、普通固定前缀 +# 和单桶 hash 特例；reference 只比较逻辑语义，不复制被测 ring。 +function(add_fdwic_tensor_map_ring_test name ring_cap) + add_a5_test(${name} a5/test_fdwic_tensor_map_retire.cpp) + target_compile_definitions(${name} PRIVATE + __CPU_SIM=1 + PTO_FDWIC_TENSORMAP_RING_CAP=${ring_cap} + ) + target_include_directories(${name} BEFORE PRIVATE + ${CMAKE_SOURCE_DIR}/../../../src/a5/platform/sim/aicore + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/runtime + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/common + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime + ) + set_tests_properties(${name} PROPERTIES LABELS "no_hardware") +endfunction() + +add_fdwic_tensor_map_ring_test(test_fdwic_tensor_map_retire 128) +add_fdwic_tensor_map_ring_test(test_fdwic_tensor_map_ring_cap32 32) +add_fdwic_tensor_map_ring_test(test_fdwic_tensor_map_ring_cap64 64) +add_fdwic_tensor_map_ring_test(test_fdwic_tensor_map_ring_cap256 256) +add_fdwic_tensor_map_ring_test(test_fdwic_tensor_map_ring_cap16384 16384) + +# shared sidecar 先独立验证布局和 AICPU 一次性初始化，production 顶层仍 +# fail-closed。五个 CAP 覆盖 bucket-control 长度变化，额外 private 目标 +# 锁住“只在 shared DistGlobal 尾部追加”的 ABI 边界。 +function(add_fdwic_shared_tensor_map_state_test name shared_map ring_cap) + add_a5_test(${name} a5/test_fdwic_shared_tensor_map_state.cpp) + target_compile_definitions(${name} PRIVATE + __CPU_SIM=1 + PTO_FDWIC_SHARED_MAP=${shared_map} + PTO_FDWIC_TENSORMAP_RING_CAP=${ring_cap} + ) + target_include_directories(${name} BEFORE PRIVATE + ${CMAKE_SOURCE_DIR}/../../../src/a5/platform/sim/aicore + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/runtime + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/common + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime + ) + set_tests_properties(${name} PROPERTIES LABELS "no_hardware") +endfunction() + +add_fdwic_shared_tensor_map_state_test(test_fdwic_private_tensor_map_state_layout 0 128) +add_fdwic_shared_tensor_map_state_test(test_fdwic_shared_tensor_map_state 1 128) +add_fdwic_shared_tensor_map_state_test(test_fdwic_shared_tensor_map_state_cap32 1 32) +add_fdwic_shared_tensor_map_state_test(test_fdwic_shared_tensor_map_state_cap64 1 64) +add_fdwic_shared_tensor_map_state_test(test_fdwic_shared_tensor_map_state_cap256 1 256) +add_fdwic_shared_tensor_map_state_test(test_fdwic_shared_tensor_map_state_cap16384 1 16384) + +# shared ring primitive 仍不接 Submit；五种 CAP 都直接实例化 production +# 状态机，验证 exact-turn、窗口 lookup、整 task preflight、seq 发布/复用与 +# 独立 reference 差分。 +function(add_fdwic_shared_tensor_map_ring_test name ring_cap) + add_a5_test(${name} a5/test_fdwic_shared_tensor_map_ring.cpp) + target_compile_definitions(${name} PRIVATE + __CPU_SIM=1 + PTO_FDWIC_SHARED_MAP=1 + PTO_FDWIC_TENSORMAP_RING_CAP=${ring_cap} + ) + target_include_directories(${name} BEFORE PRIVATE + ${CMAKE_SOURCE_DIR}/../../../src/a5/platform/sim/aicore + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/runtime + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/common + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime + ) + set_tests_properties(${name} PROPERTIES LABELS "no_hardware") +endfunction() + +add_fdwic_shared_tensor_map_ring_test(test_fdwic_shared_tensor_map_ring 128) +add_fdwic_shared_tensor_map_ring_test(test_fdwic_shared_tensor_map_ring_cap32 32) +add_fdwic_shared_tensor_map_ring_test(test_fdwic_shared_tensor_map_ring_cap64 64) +add_fdwic_shared_tensor_map_ring_test(test_fdwic_shared_tensor_map_ring_cap256 256) +add_fdwic_shared_tensor_map_ring_test(test_fdwic_shared_tensor_map_ring_cap16384 16384) + +# 容量错误必须在真实 Register→Build 边界中止，并让后续 eager Begin 在 +# Claim 前返回。该目标直接编译 production CPU-sim dist_engine TU，避免 +# 用测试替身重写 Submit 状态机。 +add_a5_test(test_fdwic_submit_capacity a5/test_fdwic_submit_capacity.cpp) +target_compile_definitions(test_fdwic_submit_capacity PRIVATE + __CPU_SIM=1 + PTO_FDWIC_TENSORMAP_RING_CAP=128 +) +target_include_directories(test_fdwic_submit_capacity BEFORE PRIVATE + ${CMAKE_SOURCE_DIR}/../../../src/a5/platform/sim/aicore + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/runtime + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/common + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime +) +set_tests_properties(test_fdwic_submit_capacity PROPERTIES LABELS "no_hardware") + +# The shared Submit adapter owns the mapping from task-transaction outcomes to +# structured fatal codes. This target compiles the production CPU-sim TU under +# the shared build identity without opening the AICPU backend-ready gate. +add_a5_test(test_fdwic_shared_submit_contract a5/test_fdwic_shared_submit_contract.cpp) +target_compile_definitions(test_fdwic_shared_submit_contract PRIVATE + __CPU_SIM=1 + PTO_FDWIC_SHARED_MAP=1 + PTO_FDWIC_TENSORMAP_RING_CAP=128 +) +target_include_directories(test_fdwic_shared_submit_contract BEFORE PRIVATE + ${CMAKE_SOURCE_DIR}/../../../src/a5/platform/sim/aicore + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/runtime + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/common + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime +) +set_tests_properties(test_fdwic_shared_submit_contract PROPERTIES LABELS "no_hardware") + +# This gate executes the production compete-first Submit flow under the shared +# build identity. It verifies transaction ordering at the Kernel/Alloc/joint +# boundaries instead of testing the ring primitive or result adapter alone. +add_a5_test(test_fdwic_shared_submit_wiring a5/test_fdwic_shared_submit_wiring.cpp) +target_compile_definitions(test_fdwic_shared_submit_wiring PRIVATE + __CPU_SIM=1 + PTO_FDWIC_SHARED_MAP=1 + PTO_FDWIC_TENSORMAP_RING_CAP=128 +) +target_include_directories(test_fdwic_shared_submit_wiring BEFORE PRIVATE + ${CMAKE_SOURCE_DIR}/../../../src/a5/platform/sim/aicore + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/runtime + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/common + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime +) +set_tests_properties(test_fdwic_shared_submit_wiring PROPERTIES LABELS "no_hardware") + +# This multi-worker gate compiles the same production CPU-sim Submit TU and +# proves that a future-turn winner waits for committed_tasks before fanin, +# publication, and Build. SPIN_WAIT_HINT is observed only by the test. +add_a5_test(test_fdwic_shared_multiworker a5/test_fdwic_shared_multiworker.cpp) +target_compile_definitions(test_fdwic_shared_multiworker PRIVATE + __CPU_SIM=1 + PTO_FDWIC_SHARED_MAP=1 + PTO_FDWIC_TENSORMAP_RING_CAP=128 + PTO_FDWIC_TRACE_ENABLED=0 +) +target_include_directories(test_fdwic_shared_multiworker BEFORE PRIVATE + ${CMAKE_SOURCE_DIR}/../../../src/a5/platform/sim/aicore + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/runtime + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/common + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime +) +set_tests_properties(test_fdwic_shared_multiworker PROPERTIES LABELS "no_hardware" TIMEOUT 15) + +# PTO_DIST_H 是 AICPU 控制面的纯配置输入。独立目标直接编译生产解析 +# helper，锁定仅 ASCII 十进制数字串和闭区间边界，不引入设备或 runtime。 +add_executable(test_fdwic_config_parse a5/test_fdwic_config_parse.cpp) +target_include_directories(test_fdwic_config_parse PRIVATE + ${GTEST_INCLUDE_DIRS} + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/runtime +) +target_link_libraries(test_fdwic_config_parse PRIVATE + ${GTEST_MAIN_LIB} + ${GTEST_LIB} + pthread +) +add_test(NAME test_fdwic_config_parse COMMAND test_fdwic_config_parse) +set_tests_properties(test_fdwic_config_parse PROPERTIES LABELS "no_hardware") + +# Stable Host/AICPU/AICore identity is compiled in both artifact modes. The +# shared target intentionally verifies that the first-stage backend remains +# fail-closed while its ABI identity is still well-formed. +function(add_fdwic_build_identity_test name shared_map) + add_executable(${name} a5/test_fdwic_build_identity.cpp) + target_compile_definitions(${name} PRIVATE + PTO_FDWIC_SHARED_MAP=${shared_map} + PTO_FDWIC_TENSORMAP_RING_CAP=128 + ) + target_include_directories(${name} PRIVATE + ${GTEST_INCLUDE_DIRS} + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/runtime + ${CMAKE_SOURCE_DIR}/../../../src/a5/runtime/fully_distributed_within_core/common + ${CMAKE_SOURCE_DIR}/../../../src/a5/platform/include + ${CMAKE_SOURCE_DIR}/../../../src/common/platform/include + ${CMAKE_SOURCE_DIR}/../../../src/common/task_interface + ${CMAKE_SOURCE_DIR}/../../../src/common + ) + target_link_libraries(${name} PRIVATE + ${GTEST_MAIN_LIB} + ${GTEST_LIB} + pthread + ) + add_test(NAME ${name} COMMAND ${name}) + set_tests_properties(${name} PROPERTIES LABELS "no_hardware") +endfunction() + +add_fdwic_build_identity_test(test_fdwic_build_identity_private 0) +add_fdwic_build_identity_test(test_fdwic_build_identity_shared 1) + # A5 trb runtime UTs — mirror of a2a3 trb runtime UTs, link against a5_rt_objs. # Target names carry the a5_ prefix because hierarchical/test_tensormap (and # the unprefixed a2a3 runtime targets test_scheduler_state / test_ready_queue diff --git a/tests/ut/cpp/a5/test_fdwic_build_identity.cpp b/tests/ut/cpp/a5/test_fdwic_build_identity.cpp new file mode 100644 index 0000000000..98fe6d15db --- /dev/null +++ b/tests/ut/cpp/a5/test_fdwic_build_identity.cpp @@ -0,0 +1,95 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include "runtime.h" + +namespace { + +constexpr uint32_t kRuntimeBytesForTest = sizeof(Runtime); + +// private/shared 三个镜像都要先按这段公共前缀完成握手，才能读取其后的 +// mode-specific 状态。这里直接约束生产 Runtime，而不是只用一个假的 +// runtime_bytes 数值测试 identity header。 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +TEST(FdwicBuildIdentity, RuntimeKeepsTheStableThreeImageControlPrefix) { + EXPECT_EQ(offsetof(Runtime, fdwic_build_identity), 0U); + EXPECT_EQ(offsetof(Runtime, workers), 64U); + EXPECT_EQ( + offsetof(Runtime, worker_count), + offsetof(Runtime, workers) + sizeof(Handshake) * RUNTIME_MAX_WORKER); + EXPECT_EQ(offsetof(Runtime, aicpu_thread_num), offsetof(Runtime, worker_count) + sizeof(int)); + EXPECT_EQ(offsetof(Runtime, aicpu_allowed_cpus), offsetof(Runtime, aicpu_thread_num) + sizeof(int)); + EXPECT_EQ( + offsetof(Runtime, aicpu_allowed_cpu_count), + offsetof(Runtime, aicpu_allowed_cpus) + sizeof(int32_t) * 16); + EXPECT_EQ( + offsetof(Runtime, aicpu_launch_count), + offsetof(Runtime, aicpu_allowed_cpu_count) + sizeof(int32_t)); + EXPECT_EQ(sizeof(Runtime), 70080U); +} +#pragma GCC diagnostic pop + +TEST(FdwicBuildIdentity, CompiledModeBuildsAMatchingStableLine) { + FdwicBuildIdentity identity = fdwic_make_build_identity(kRuntimeBytesForTest); + + EXPECT_EQ(sizeof(identity), 64U); + EXPECT_EQ(offsetof(FdwicBuildIdentity, runtime_bytes), 16U); + EXPECT_EQ(offsetof(FdwicBuildIdentity, dist_global_layout_version), 20U); + EXPECT_EQ(offsetof(FdwicBuildIdentity, error_bits), 24U); + EXPECT_EQ(offsetof(FdwicBuildIdentity, tensor_map_ring_cap), 28U); + EXPECT_EQ(identity.tensor_map_mode, static_cast(kFdwicCompiledTensorMapMode)); + EXPECT_EQ(identity.tensor_map_ring_cap, kFdwicTensorMapRingCap); + EXPECT_EQ(kFdwicTensorMapRingBuckets * kFdwicTensorMapRingCap, 16384U); + EXPECT_TRUE(fdwic_build_identity_matches(identity, kRuntimeBytesForTest)); +#if PTO_FDWIC_SHARED_MAP + EXPECT_FALSE(kFdwicCompiledBackendReady); +#else + EXPECT_TRUE(kFdwicCompiledBackendReady); +#endif +} + +TEST(FdwicBuildIdentity, RejectsEveryCrossImageContractField) { + FdwicBuildIdentity identity = fdwic_make_build_identity(kRuntimeBytesForTest); + + identity.magic ^= 1; + EXPECT_FALSE(fdwic_build_identity_matches(identity, kRuntimeBytesForTest)); + + identity = fdwic_make_build_identity(kRuntimeBytesForTest); + identity.abi_version += 1; + EXPECT_FALSE(fdwic_build_identity_matches(identity, kRuntimeBytesForTest)); + + identity = fdwic_make_build_identity(kRuntimeBytesForTest); + identity.tensor_map_mode ^= 1; + EXPECT_FALSE(fdwic_build_identity_matches(identity, kRuntimeBytesForTest)); + + identity = fdwic_make_build_identity(kRuntimeBytesForTest); + identity.tensor_map_ring_cap *= 2; + EXPECT_FALSE(fdwic_build_identity_matches(identity, kRuntimeBytesForTest)); + + identity = fdwic_make_build_identity(kRuntimeBytesForTest); + EXPECT_FALSE(fdwic_build_identity_matches(identity, kRuntimeBytesForTest + 64)); + + identity = fdwic_make_build_identity(kRuntimeBytesForTest); + identity.dist_global_layout_version += 1; + EXPECT_FALSE(fdwic_build_identity_matches(identity, kRuntimeBytesForTest)); +} + +TEST(FdwicBuildIdentity, DiagnosticErrorBitsDoNotChangeCompatibility) { + FdwicBuildIdentity identity = fdwic_make_build_identity(kRuntimeBytesForTest); + identity.error_bits = FdwicBuildErrorAicoreMismatch | FdwicBuildErrorBackendUnavailable; + + EXPECT_TRUE(fdwic_build_identity_matches(identity, kRuntimeBytesForTest)); +} + +} // namespace diff --git a/tests/ut/cpp/a5/test_fdwic_config_parse.cpp b/tests/ut/cpp/a5/test_fdwic_config_parse.cpp new file mode 100644 index 0000000000..fac078dfb3 --- /dev/null +++ b/tests/ut/cpp/a5/test_fdwic_config_parse.cpp @@ -0,0 +1,89 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include + +#include "dist_engine/aicpu/config_parse.h" + +namespace { + +TEST(FdwicConfigParse, AcceptsOnlyInRangeAsciiDecimalDigitStrings) { + struct ValidCase { + const char *text; + int32_t max_inclusive; + int32_t expected; + }; + constexpr ValidCase cases[] = { + {"0", 0, 0}, + {"00", 1022, 0}, + {"64", 1022, 64}, + {"00064", 1022, 64}, + {"1022", 1022, 1022}, + }; + + for (const ValidCase &test : cases) { + int32_t value = -1; + EXPECT_TRUE(dist_parse_history_window(test.text, test.max_inclusive, value)) << test.text; + EXPECT_EQ(value, test.expected) << test.text; + } +} + +TEST(FdwicConfigParse, RejectsNullEmptyWhitespaceSignsAndNonDigitsWithoutChangingOutput) { + constexpr const char *invalid[] = { + "", + " ", + " 64", + "64 ", + "\t64", + "64\n", + "+64", + "-1", + "64x", + "x64", + "6_4", + }; + + int32_t value = 777; + EXPECT_FALSE(dist_parse_history_window(nullptr, 1022, value)); + EXPECT_EQ(value, 777); + + for (const char *text : invalid) { + value = 777; + EXPECT_FALSE(dist_parse_history_window(text, 1022, value)) << text; + EXPECT_EQ(value, 777) << text; + } +} + +TEST(FdwicConfigParse, RejectsOverflowAndValuesOutsideTheConfiguredRange) { + constexpr const char *invalid[] = { + "1023", + "2147483648", + "999999999999999999999999999999999999999999999999", + }; + + for (const char *text : invalid) { + int32_t value = 777; + EXPECT_FALSE(dist_parse_history_window(text, 1022, value)) << text; + EXPECT_EQ(value, 777) << text; + } + + int32_t value = 777; + EXPECT_FALSE(dist_parse_history_window("0", -1, value)); + EXPECT_EQ(value, 777); + + value = 777; + EXPECT_FALSE(dist_parse_history_window("1", 0, value)); + EXPECT_EQ(value, 777); +} + +} // namespace diff --git a/tests/ut/cpp/a5/test_fdwic_shared_multiworker.cpp b/tests/ut/cpp/a5/test_fdwic_shared_multiworker.cpp new file mode 100644 index 0000000000..408e6e6a04 --- /dev/null +++ b/tests/ut/cpp/a5/test_fdwic_shared_multiworker.cpp @@ -0,0 +1,1399 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// The CPU-sim platform and public runtime types both provide their normal +// SPIN_WAIT_HINT definitions. Include them first, then replace only this +// translation unit's final definition with the observation hook below. +#include "inner_kernel.h" +#undef SPIN_WAIT_HINT +#include "pto_runtime2_types.h" + +namespace fdwic_shared_multiworker_test { + +std::atomic g_observed_turn_wait_spins{0}; +std::atomic g_spins_after_remote_fatal{0}; +std::atomic g_remote_fatal_published{false}; +std::atomic g_single_lane_kernel_calls{0}; +std::atomic g_joint_kernel_entered[3]{}; +std::atomic g_joint_kernel_exited[3]{}; +std::atomic g_joint_kernel_release[3]{}; +thread_local bool g_observe_turn_wait = false; +thread_local bool g_limit_spins_after_remote_fatal = false; + +constexpr uint32_t kPostFatalSpinLimit = 1024; + +void spin_wait_hint() { + if (g_observe_turn_wait) { + g_observed_turn_wait_spins.fetch_add(1, std::memory_order_release); + } + if (g_limit_spins_after_remote_fatal && g_remote_fatal_published.load(std::memory_order_acquire) && + g_spins_after_remote_fatal.fetch_add(1, std::memory_order_acq_rel) >= kPostFatalSpinLimit) { + throw std::runtime_error("production wait did not consume the remote fatal"); + } + std::this_thread::yield(); +} + +void count_single_lane_kernel(int64_t *) { g_single_lane_kernel_calls.fetch_add(1, std::memory_order_relaxed); } + +void run_joint_kernel(int32_t lane) { + g_joint_kernel_entered[lane].fetch_add(1, std::memory_order_release); + while (!g_joint_kernel_release[lane].load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + g_joint_kernel_exited[lane].fetch_add(1, std::memory_order_release); +} + +} // namespace fdwic_shared_multiworker_test + +// This is a test-only observation seam around the production wait loop. It +// does not replace Claim, TensorMap publication, fanin collection, or Build. +#undef SPIN_WAIT_HINT +#define SPIN_WAIT_HINT() ::fdwic_shared_multiworker_test::spin_wait_hint() +#include "dist_engine/aicore/dist_engine.cpp" // NOLINT(build/include) +#include "dist_engine/aicpu/shared_tensor_map_init.h" +#undef SPIN_WAIT_HINT + +namespace fdwic_shared_multiworker_test { + +void count_joint_aic_kernel(int64_t *) { run_joint_kernel(LANE_AIC); } + +void count_joint_aiv0_kernel(int64_t *) { run_joint_kernel(LANE_AIV0); } + +void count_joint_aiv1_kernel(int64_t *) { run_joint_kernel(LANE_AIV1); } + +} // namespace fdwic_shared_multiworker_test + +[[noreturn]] void assert_impl(const char *condition, const char *, int) { throw std::logic_error(condition); } + +extern "C" void aicpu_orchestration_entry(const L2TaskArgs &) {} +volatile uint8_t *sim_get_reg_base() { return nullptr; } +uint32_t sim_get_physical_core_id() { return 0; } + +Runtime::Runtime() { + for (uint64_t &address : func_id_to_addr_) { + address = 0; + } + use_example_exec_time_ = false; + for (int32_t &duration : example_exec_time_ns_) { + duration = 0; + } +} + +namespace { + +Tensor make_existing_tensor(uint64_t address) { + Tensor tensor{}; + const uint32_t shape[1] = {1}; + tensor.init_external( + reinterpret_cast(static_cast(address)), sizeof(float), shape, 1, DataType::FLOAT32, 0 + ); + return tensor; +} + +Tensor make_owned_tensor(uint64_t address, int32_t owner) { + Tensor tensor = make_existing_tensor(address); + tensor.owner_task_id = PTO2TaskId::make(0, static_cast(owner)); + return tensor; +} + +void install_test_callable( + Runtime &runtime, int32_t kernel_id, void (*kernel)(int64_t *), std::vector &storage +) { + const ArgDirection signature[] = {ArgDirection::OUT}; + storage = make_callable(signature, 1, nullptr, /*binary_size=*/0); + CoreCallable *callable = reinterpret_cast(storage.data()); + callable->set_resolved_addr(reinterpret_cast(kernel)); + runtime.func_id_to_addr_[kernel_id] = reinterpret_cast(callable); +} + +int32_t built_slot_count(const DistCore &worker) { + int32_t count = 0; + for (int32_t index = 0; index < kPrivateSlots; ++index) { + if (worker.slots[index].occupied && worker.slots[index].built) { + ++count; + } + } + return count; +} + +int32_t occupied_slot_count(const DistCore &worker) { + int32_t count = 0; + for (int32_t index = 0; index < kPrivateSlots; ++index) { + if (worker.slots[index].occupied) { + ++count; + } + } + return count; +} + +const RingSlot *find_built_slot(const DistCore &worker, int32_t task_id) { + for (int32_t index = 0; index < kPrivateSlots; ++index) { + const RingSlot &slot = worker.slots[index]; + if (slot.occupied && slot.built && slot.task_id == task_id) { + return &slot; + } + } + return nullptr; +} + +int32_t fanin_occurrences(const RingSlot &slot, int32_t producer) { + int32_t count = 0; + for (int32_t index = 0; index < slot.fanin_count; ++index) { + if (slot.fanin[index] == producer) { + ++count; + } + } + return count; +} + +class FdwicSharedMultiworkerTest : public ::testing::Test { +protected: + void SetUp() override { + static_assert(PTO_FDWIC_SHARED_MAP == 1); + static_assert(!kFdwicCompiledBackendReady); + + fdwic_shared_multiworker_test::g_observed_turn_wait_spins.store(0, std::memory_order_relaxed); + fdwic_shared_multiworker_test::g_spins_after_remote_fatal.store(0, std::memory_order_relaxed); + fdwic_shared_multiworker_test::g_remote_fatal_published.store(false, std::memory_order_relaxed); + fdwic_shared_multiworker_test::g_single_lane_kernel_calls.store(0, std::memory_order_relaxed); + for (int32_t lane = LANE_AIC; lane <= LANE_AIV1; ++lane) { + fdwic_shared_multiworker_test::g_joint_kernel_entered[lane].store(0, std::memory_order_relaxed); + fdwic_shared_multiworker_test::g_joint_kernel_exited[lane].store(0, std::memory_order_relaxed); + fdwic_shared_multiworker_test::g_joint_kernel_release[lane].store(false, std::memory_order_relaxed); + } + g_dist_ptr = &g_dist_fallback; + dist_shared_tensor_map_reset(g_dist.shared_tensor_map); + + dist_core_reset(*aic_worker_, CoreType::AIC, /*block=*/0, LANE_AIC); + dist_core_reset(*task0_worker_, CoreType::AIV, /*block=*/0, LANE_AIV0); + dist_core_reset(*task1_worker_, CoreType::AIV, /*block=*/0, LANE_AIV1); + aic_worker_->core_idx = 0; + task0_worker_->core_idx = 1; + task1_worker_->core_idx = 2; + g_self = task0_worker_.get(); + + g_dist.H = kHDefault; + g_dist.heap_base = nullptr; + g_dist.heap_size = 0; + g_dist.runtime = &runtime_; + g_dist.num_workers = 2; + g_dist.num_blocks = 1; + g_dist.frontier = -1; + g_dist.fatal = 0; + g_dist.error_code = PTO2_ERROR_NONE; + g_dist.blocks[0].any_pub = 0; + for (int32_t index = 0; index < kPrivateSlots; ++index) { + WonSlot &slot = g_dist.blocks[0].slots[index]; + slot.state.v = kWonStateFree; + slot.remaining.v = 0; + for (int32_t lane = LANE_AIC; lane <= LANE_AIV1; ++lane) { + slot.drained[lane].v = kDrainedClaimed; + slot.lane[lane].present = false; + } + } + for (int32_t shard = 0; shard < kCursorShards; ++shard) { + g_dist.cube_cursor[shard].v = -1; + g_dist.vector_cursor[shard].v = -1; + g_dist.alloc_cursor[shard].v = -1; + } + for (int32_t task = 0; task < 16; ++task) { + reset_task_cell(task); + } + for (int32_t group = 0; group < kFinalBarrierGroups; ++group) { + g_dist.final_barrier.leaf_arrivals[group].v = 0; + g_dist.final_barrier.leaf_arrivals[group].expected = 0; + g_dist.final_barrier.leaf_releases[group].v = 0; + } + g_dist.final_barrier.root_arrival.v = 0; + g_dist.final_barrier.root_arrival.expected = 0; + g_dist.final_barrier.root_release.v = 0; + g_fdwic_joint_submit_seen = false; + g_skip_exec = false; + } + + void TearDown() override { + fdwic_shared_multiworker_test::g_observe_turn_wait = false; + fdwic_shared_multiworker_test::g_limit_spins_after_remote_fatal = false; + g_self = nullptr; + g_dist_ptr = nullptr; + } + + void publish_seed_task(const SharedTensorMapValue *entries, uint32_t count, int32_t task) { + ASSERT_EQ( + dist_shared_tensor_map_publish_task(g_dist.shared_tensor_map, entries, count, task, kHDefault), + DistSharedTensorMapTaskPublishResult::Committed + ); + ASSERT_EQ(atomic_load(g_dist.shared_tensor_map.committed_tasks.v), task + 1); + } + + Runtime runtime_; + std::unique_ptr aic_worker_ = std::make_unique(); + std::unique_ptr task0_worker_ = std::make_unique(); + std::unique_ptr task1_worker_ = std::make_unique(); +}; + +TEST_F(FdwicSharedMultiworkerTest, FutureTurnWaitsForPriorCommitThenBuildsWithDependency) { + const Tensor inout = make_existing_tensor(0x600000); + L0TaskArgs task0_args; + task0_args.add_inout(inout); + L0TaskArgs task0_replay_args; + task0_replay_args.add_inout(inout); + L0TaskArgs task1_args; + task1_args.add_inout(inout); + MixedKernels mixed; + mixed.aiv0_kernel_id = 21; + + // Holding task0 between Begin and Finish is the deterministic publication + // latch: it has won Claim, but committed_tasks must remain zero. + const DistCompeteFirstTicket task0_ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(task0_ticket.task_id, 0); + ASSERT_EQ(task0_ticket.won, 1); + ASSERT_EQ(atomic_load(g_dist.shared_tensor_map.committed_tasks.v), 0); + + DistCompeteFirstTicket task0_loser_ticket{}; + DistCompeteFirstTicket task1_ticket{}; + std::atomic task1_finish_returned{false}; + std::exception_ptr task1_error; + std::thread task1_thread([&]() { + try { + g_self = task1_worker_.get(); + // AIV1 starts from the same replay position. Its task0 Claim loses + // to AIV0, then the normal per-worker sequence advances to task1. + task0_loser_ticket = dist_submit_compete_first_begin(nullptr, mixed); + (void)dist_submit_compete_first_finish(nullptr, mixed, task0_loser_ticket, task0_replay_args); + task1_ticket = dist_submit_compete_first_begin(nullptr, mixed); + fdwic_shared_multiworker_test::g_observe_turn_wait = true; + (void)dist_submit_compete_first_finish(nullptr, mixed, task1_ticket, task1_args); + fdwic_shared_multiworker_test::g_observe_turn_wait = false; + } catch (...) { + task1_error = std::current_exception(); + } + task1_finish_returned.store(true, std::memory_order_release); + g_self = nullptr; + }); + + const auto wait_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (fdwic_shared_multiworker_test::g_observed_turn_wait_spins.load(std::memory_order_acquire) == 0 && + !task1_finish_returned.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < wait_deadline) { + std::this_thread::yield(); + } + + // Snapshot the blocked state before releasing task0. These values prove + // task1 did not publish or Build while committed_tasks was behind. + const bool task1_entered_turn_wait = + fdwic_shared_multiworker_test::g_observed_turn_wait_spins.load(std::memory_order_acquire) != 0; + const bool task1_returned_before_release = task1_finish_returned.load(std::memory_order_acquire); + const int64_t committed_before_release = atomic_load(g_dist.shared_tensor_map.committed_tasks.v); + const int32_t task0_built_before_release = built_slot_count(*task0_worker_); + const int32_t task1_built_before_release = built_slot_count(*task1_worker_); + + // Publishing task0 advances the exact turn to one. The waiting production + // Finish must then collect producer 0, publish task1, and Build exactly once. + (void)dist_submit_compete_first_finish(nullptr, mixed, task0_ticket, task0_args); + task1_thread.join(); + + EXPECT_TRUE(task1_entered_turn_wait); + EXPECT_FALSE(task1_returned_before_release); + EXPECT_EQ(committed_before_release, 0); + EXPECT_EQ(task0_built_before_release, 0); + EXPECT_EQ(task1_built_before_release, 0); + + if (task1_error != nullptr) { + try { + std::rethrow_exception(task1_error); + } catch (const std::exception &error) { + FAIL() << "task1 worker threw: " << error.what(); + } catch (...) { + FAIL() << "task1 worker threw a non-standard exception"; + } + } + + EXPECT_EQ(task0_loser_ticket.task_id, 0); + EXPECT_EQ(task0_loser_ticket.won, 0); + EXPECT_EQ(task1_ticket.task_id, 1); + EXPECT_EQ(task1_ticket.won, 1); + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_NONE); + EXPECT_EQ(atomic_load(g_dist.shared_tensor_map.committed_tasks.v), 2); + + EXPECT_EQ(task0_worker_->occupied_count, 1); + EXPECT_EQ(task1_worker_->occupied_count, 1); + EXPECT_EQ(built_slot_count(*task0_worker_), 1); + EXPECT_EQ(built_slot_count(*task1_worker_), 1); + + const RingSlot *task0_slot = find_built_slot(*task0_worker_, 0); + const RingSlot *task1_slot = find_built_slot(*task1_worker_, 1); + ASSERT_NE(task0_slot, nullptr); + ASSERT_NE(task1_slot, nullptr); + EXPECT_EQ(task0_slot->fanin_count, 0); + ASSERT_EQ(task1_slot->fanin_count, 1); + EXPECT_EQ(task1_slot->fanin[0], 0); +} + +TEST_F(FdwicSharedMultiworkerTest, PaG2FutureFinalUpWaitsForRemoteFirstUpWriterCommit) { + constexpr int32_t kUpKernelId = 25; + Tensor mi_update = make_owned_tensor(0x510000, /*owner=*/0); + Tensor li_update = make_owned_tensor(0x520000, /*owner=*/0); + Tensor oi = make_owned_tensor(0x530000, /*owner=*/0); + Tensor out_view = make_existing_tensor(0x540000); + out_view.manual_dep = true; + + const Tensor group0_mi = make_owned_tensor(0x610000, /*owner=*/2); + const Tensor group0_li = make_owned_tensor(0x620000, /*owner=*/2); + const Tensor group0_oi_new = make_owned_tensor(0x630000, /*owner=*/3); + const Tensor group1_mi = make_owned_tensor(0x710000, /*owner=*/6); + const Tensor group1_li = make_owned_tensor(0x720000, /*owner=*/6); + const Tensor group1_oi_new = make_owned_tensor(0x730000, /*owner=*/7); + + // Seed only the exact-turn transactions for Alloc/QK/SF/PV. The task1 + // out_view entry is an impossible PA value used to detect an accidental + // manual_dep lookup or register; it is not a model of QK output semantics. + ASSERT_NO_FATAL_FAILURE(publish_seed_task(nullptr, 0, /*task=*/0)); + const SharedTensorMapValue manual_poison = dist_shared_tensor_map_make_value(out_view, /*producer=*/1); + ASSERT_NO_FATAL_FAILURE(publish_seed_task(&manual_poison, 1, /*task=*/1)); + ASSERT_NO_FATAL_FAILURE(publish_seed_task(nullptr, 0, /*task=*/2)); + ASSERT_NO_FATAL_FAILURE(publish_seed_task(nullptr, 0, /*task=*/3)); + + L0TaskArgs up0_args; + up0_args.add_input(group0_mi, group0_li, group0_oi_new); + up0_args.add_inout(mi_update, li_update, oi, out_view); + up0_args.add_scalar(/*is_first=*/1, /*is_last=*/0); + L0TaskArgs up1_args; + up1_args.add_input(group1_mi, group1_li, group1_oi_new); + up1_args.add_inout(mi_update, li_update, oi, out_view); + up1_args.add_scalar(/*is_first=*/0, /*is_last=*/1); + MixedKernels mixed; + mixed.aiv0_kernel_id = kUpKernelId; + + std::vector task0_private_map_before(sizeof(task0_worker_->map)); + std::vector task1_private_map_before(sizeof(task1_worker_->map)); + std::memcpy(task0_private_map_before.data(), &task0_worker_->map, task0_private_map_before.size()); + std::memcpy(task1_private_map_before.data(), &task1_worker_->map, task1_private_map_before.size()); + + // The existing sequential-replay gate proves local_index progression from + // zero. This focused gate starts directly at the two real G2 UP identities + // so both workers still perform production Claim and Finish. + task0_worker_->local_index = 4; + task1_worker_->local_index = 8; + const DistCompeteFirstTicket up0_ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(up0_ticket.task_id, 4); + ASSERT_EQ(up0_ticket.ready, 1); + ASSERT_EQ(up0_ticket.claim_attempted, 1); + ASSERT_EQ(up0_ticket.won, 1); + ASSERT_EQ(atomic_load(g_dist.vector_cursor[0].v), 4); + ASSERT_EQ(atomic_load(g_dist.shared_tensor_map.committed_tasks.v), 4); + + DistCompeteFirstTicket up1_ticket{}; + std::exception_ptr up1_error; + std::atomic up1_finish_returned{false}; + std::thread up1_thread([&]() { + try { + g_self = task1_worker_.get(); + up1_ticket = dist_submit_compete_first_begin(nullptr, mixed); + fdwic_shared_multiworker_test::g_observe_turn_wait = true; + fdwic_shared_multiworker_test::g_limit_spins_after_remote_fatal = true; + (void)dist_submit_compete_first_finish(nullptr, mixed, up1_ticket, up1_args); + } catch (...) { + up1_error = std::current_exception(); + } + fdwic_shared_multiworker_test::g_limit_spins_after_remote_fatal = false; + fdwic_shared_multiworker_test::g_observe_turn_wait = false; + up1_finish_returned.store(true, std::memory_order_release); + g_self = nullptr; + }); + + const auto wait_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (fdwic_shared_multiworker_test::g_observed_turn_wait_spins.load(std::memory_order_acquire) == 0 && + !up1_finish_returned.load(std::memory_order_acquire) && std::chrono::steady_clock::now() < wait_deadline) { + std::this_thread::yield(); + } + + // The observation hook is enabled only around task8 Finish. Together with + // committed=4 and cursor=8, a positive count proves the future winner + // reached the production exact-turn loop before task4 was released. + const bool up1_entered_turn_wait = + fdwic_shared_multiworker_test::g_observed_turn_wait_spins.load(std::memory_order_acquire) != 0; + const bool up1_returned_before_release = up1_finish_returned.load(std::memory_order_acquire); + const int64_t committed_before_release = atomic_load(g_dist.shared_tensor_map.committed_tasks.v); + const int64_t vector_cursor_before_release = atomic_load(g_dist.vector_cursor[0].v); + + std::exception_ptr release_error; + int64_t committed_after_up0 = -1; + std::array latest_after_up0 = {-1, -1, -1, -1}; + std::array protocol_after_up0 = {false, false, false, false}; + std::array filler_results = { + DistSharedTensorMapTaskPublishResult::ProtocolError, + DistSharedTensorMapTaskPublishResult::ProtocolError, + DistSharedTensorMapTaskPublishResult::ProtocolError, + }; + bool up1_waited_after_up0 = false; + bool up1_returned_after_up0 = true; + int64_t committed_while_waiting_after_up0 = -1; + + try { + // task4 performs the real fanin/register/commit/Build path. Task8 must + // remain blocked at turn 8 while tasks 5-7 are still absent. + (void)dist_submit_compete_first_finish(nullptr, mixed, up0_ticket, up0_args); + committed_after_up0 = atomic_load(g_dist.shared_tensor_map.committed_tasks.v); + if (committed_after_up0 == 5 && !fatal_set()) { + latest_after_up0[0] = dist_shared_tensor_map_lookup_tensor( + g_dist.shared_tensor_map, mi_update, 5, kHDefault, protocol_after_up0[0] + ); + latest_after_up0[1] = dist_shared_tensor_map_lookup_tensor( + g_dist.shared_tensor_map, li_update, 5, kHDefault, protocol_after_up0[1] + ); + latest_after_up0[2] = + dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, oi, 5, kHDefault, protocol_after_up0[2]); + latest_after_up0[3] = dist_shared_tensor_map_lookup_tensor( + g_dist.shared_tensor_map, out_view, 5, kHDefault, protocol_after_up0[3] + ); + } + } catch (...) { + release_error = std::current_exception(); + } + + if (release_error == nullptr && committed_after_up0 == 5 && !fatal_set()) { + // Observe another exact-turn spin while commit is already five. This + // directly proves task8 does not resume merely because task4 became + // visible; turns 5-7 must still publish before its lookup can start. + const uint32_t spins_after_up0 = + fdwic_shared_multiworker_test::g_observed_turn_wait_spins.load(std::memory_order_acquire); + const auto continued_wait_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (fdwic_shared_multiworker_test::g_observed_turn_wait_spins.load(std::memory_order_acquire) == + spins_after_up0 && + !up1_finish_returned.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < continued_wait_deadline) { + std::this_thread::yield(); + } + up1_returned_after_up0 = up1_finish_returned.load(std::memory_order_acquire); + committed_while_waiting_after_up0 = atomic_load(g_dist.shared_tensor_map.committed_tasks.v); + up1_waited_after_up0 = + fdwic_shared_multiworker_test::g_observed_turn_wait_spins.load(std::memory_order_acquire) > + spins_after_up0 && + !up1_returned_after_up0 && committed_while_waiting_after_up0 == 5; + + try { + for (int32_t task = 5; task <= 7; ++task) { + filler_results[task - 5] = + dist_shared_tensor_map_publish_task(g_dist.shared_tensor_map, nullptr, 0, task, kHDefault); + } + } catch (...) { + release_error = std::current_exception(); + } + } + + bool release_completed = release_error == nullptr && committed_after_up0 == 5; + for (const DistSharedTensorMapTaskPublishResult result : filler_results) { + release_completed &= result == DistSharedTensorMapTaskPublishResult::Committed; + } + if (!release_completed) { + set_fatal_code(PTO2_ERROR_EXPLICIT_ORCH_FATAL); + fdwic_shared_multiworker_test::g_remote_fatal_published.store(true, std::memory_order_release); + } + + const auto completion_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!up1_finish_returned.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < completion_deadline) { + std::this_thread::yield(); + } + const bool up1_timed_out = !up1_finish_returned.load(std::memory_order_acquire); + if (up1_timed_out) { + set_fatal_code(PTO2_ERROR_EXPLICIT_ORCH_FATAL); + fdwic_shared_multiworker_test::g_remote_fatal_published.store(true, std::memory_order_release); + } + up1_thread.join(); + + EXPECT_TRUE(up1_entered_turn_wait); + EXPECT_FALSE(up1_returned_before_release); + EXPECT_FALSE(up1_timed_out); + EXPECT_EQ(committed_before_release, 4); + EXPECT_EQ(vector_cursor_before_release, 8); + EXPECT_EQ(committed_after_up0, 5); + EXPECT_TRUE(up1_waited_after_up0); + EXPECT_FALSE(up1_returned_after_up0); + EXPECT_EQ(committed_while_waiting_after_up0, 5); + for (const bool protocol_ok : protocol_after_up0) { + EXPECT_TRUE(protocol_ok); + } + EXPECT_EQ(latest_after_up0[0], 4); + EXPECT_EQ(latest_after_up0[1], 4); + EXPECT_EQ(latest_after_up0[2], 4); + EXPECT_EQ(latest_after_up0[3], 1); + for (const DistSharedTensorMapTaskPublishResult result : filler_results) { + EXPECT_EQ(result, DistSharedTensorMapTaskPublishResult::Committed); + } + + if (release_error != nullptr) { + try { + std::rethrow_exception(release_error); + } catch (const std::exception &error) { + ADD_FAILURE() << "task4 release threw: " << error.what(); + } catch (...) { + ADD_FAILURE() << "task4 release threw a non-standard exception"; + } + } + if (up1_error != nullptr) { + try { + std::rethrow_exception(up1_error); + } catch (const std::exception &error) { + ADD_FAILURE() << "task8 worker threw: " << error.what(); + } catch (...) { + ADD_FAILURE() << "task8 worker threw a non-standard exception"; + } + } + + EXPECT_EQ(up1_ticket.task_id, 8); + EXPECT_EQ(up1_ticket.ready, 1); + EXPECT_EQ(up1_ticket.claim_attempted, 1); + EXPECT_EQ(up1_ticket.won, 1); + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_NONE); + EXPECT_EQ(atomic_load(g_dist.shared_tensor_map.committed_tasks.v), 9); + EXPECT_EQ(atomic_load(g_dist.vector_cursor[0].v), 8); + EXPECT_EQ(task0_worker_->local_index, 5); + EXPECT_EQ(task1_worker_->local_index, 9); + + EXPECT_EQ(built_slot_count(*task0_worker_), 1); + EXPECT_EQ(built_slot_count(*task1_worker_), 1); + const RingSlot *up0_slot = find_built_slot(*task0_worker_, 4); + const RingSlot *up1_slot = find_built_slot(*task1_worker_, 8); + ASSERT_NE(up0_slot, nullptr); + ASSERT_NE(up1_slot, nullptr); + EXPECT_EQ(up0_slot->fanin_count, 3); + EXPECT_EQ(fanin_occurrences(*up0_slot, 0), 1); + EXPECT_EQ(fanin_occurrences(*up0_slot, 1), 0); + EXPECT_EQ(fanin_occurrences(*up0_slot, 2), 1); + EXPECT_EQ(fanin_occurrences(*up0_slot, 3), 1); + EXPECT_EQ(up0_slot->scalar_count, 2); + EXPECT_EQ(up0_slot->scalars[0], 1); + EXPECT_EQ(up0_slot->scalars[1], 0); + EXPECT_EQ(up1_slot->fanin_count, 4); + EXPECT_EQ(fanin_occurrences(*up1_slot, 0), 1); + EXPECT_EQ(fanin_occurrences(*up1_slot, 1), 0); + EXPECT_EQ(fanin_occurrences(*up1_slot, 4), 1); + EXPECT_EQ(fanin_occurrences(*up1_slot, 6), 1); + EXPECT_EQ(fanin_occurrences(*up1_slot, 7), 1); + EXPECT_EQ(up1_slot->scalar_count, 2); + EXPECT_EQ(up1_slot->scalars[0], 0); + EXPECT_EQ(up1_slot->scalars[1], 1); + + bool protocol_ok = false; + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, mi_update, 9, kHDefault, protocol_ok), 8); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, li_update, 9, kHDefault, protocol_ok), 8); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, oi, 9, kHDefault, protocol_ok), 8); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, out_view, 9, kHDefault, protocol_ok), 1); + EXPECT_TRUE(protocol_ok); + + EXPECT_EQ(std::memcmp(task0_private_map_before.data(), &task0_worker_->map, task0_private_map_before.size()), 0); + EXPECT_EQ(std::memcmp(task1_private_map_before.data(), &task1_worker_->map, task1_private_map_before.size()), 0); +} + +TEST_F(FdwicSharedMultiworkerTest, RemoteFatalInterruptsSlotCapacityWait) { + // Two permanently blocked slots reach the self-owned capacity threshold. + // The current task may complete its map transaction, but it must not spin + // forever or Build after another worker publishes a terminal failure. + constexpr int32_t kBlockedProducer = 7; + for (int32_t index = 0; index < kPrivateSlots - kWonReserve; ++index) { + RingSlot &slot = task0_worker_->slots[index]; + slot.occupied = true; + slot.built = true; + slot.task_id = 100 + index; + slot.fanin_count = 1; + slot.fanin[0] = kBlockedProducer; + } + task0_worker_->occupied_count = kPrivateSlots - kWonReserve; + + L0TaskArgs args; + MixedKernels mixed; + mixed.aiv0_kernel_id = 22; + DistCompeteFirstTicket ticket{}; + std::exception_ptr worker_error; + std::atomic worker_returned{false}; + std::thread worker([&]() { + try { + g_self = task0_worker_.get(); + ticket = dist_submit_compete_first_begin(nullptr, mixed); + fdwic_shared_multiworker_test::g_observe_turn_wait = true; + fdwic_shared_multiworker_test::g_limit_spins_after_remote_fatal = true; + (void)dist_submit_compete_first_finish(nullptr, mixed, ticket, args); + fdwic_shared_multiworker_test::g_limit_spins_after_remote_fatal = false; + fdwic_shared_multiworker_test::g_observe_turn_wait = false; + } catch (...) { + worker_error = std::current_exception(); + } + worker_returned.store(true, std::memory_order_release); + g_self = nullptr; + }); + + const auto wait_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (fdwic_shared_multiworker_test::g_observed_turn_wait_spins.load(std::memory_order_acquire) == 0 && + std::chrono::steady_clock::now() < wait_deadline) { + std::this_thread::yield(); + } + const bool entered_capacity_wait = + fdwic_shared_multiworker_test::g_observed_turn_wait_spins.load(std::memory_order_acquire) != 0; + EXPECT_FALSE(worker_returned.load(std::memory_order_acquire)); + set_fatal_code(PTO2_ERROR_EXPLICIT_ORCH_FATAL); + fdwic_shared_multiworker_test::g_remote_fatal_published.store(true, std::memory_order_release); + worker.join(); + + EXPECT_TRUE(entered_capacity_wait); + EXPECT_TRUE(worker_returned.load(std::memory_order_acquire)); + if (worker_error != nullptr) { + try { + std::rethrow_exception(worker_error); + } catch (const std::exception &error) { + FAIL() << "slot-capacity waiter threw: " << error.what(); + } catch (...) { + FAIL() << "slot-capacity waiter threw a non-standard exception"; + } + } + EXPECT_EQ(ticket.task_id, 0); + EXPECT_EQ(ticket.won, 1); + EXPECT_EQ(g_dist.fatal, 1); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_EXPLICIT_ORCH_FATAL); + EXPECT_EQ(atomic_load(g_dist.shared_tensor_map.committed_tasks.v), 1); + EXPECT_EQ(task0_worker_->local_index, kFlagCap); + EXPECT_EQ(task0_worker_->occupied_count, kPrivateSlots - kWonReserve); + EXPECT_EQ(task_cell(0).flag, 0); + EXPECT_EQ(task_cell(0).vend, 0); + EXPECT_EQ(g_dist.frontier, -1); +} + +TEST_F(FdwicSharedMultiworkerTest, RemoteFatalInterruptsIncompleteFinalBarrier) { + // Only one of two expected workers enters the final barrier. A remote fatal + // means the absent worker will skip FinalDrain, so this worker must leave + // without waiting for an impossible root/leaf release. + dist_core_reset(*task0_worker_, CoreType::AIC, /*block=*/0, LANE_AIC); + task0_worker_->core_idx = 0; + g_dist.final_barrier.leaf_arrivals[0].expected = 2; + g_dist.final_barrier.root_arrival.expected = 1; + + std::exception_ptr worker_error; + std::atomic worker_returned{false}; + std::thread worker([&]() { + try { + g_self = task0_worker_.get(); + fdwic_shared_multiworker_test::g_observe_turn_wait = true; + fdwic_shared_multiworker_test::g_limit_spins_after_remote_fatal = true; + dist_submit_drain_to_completion(task0_worker_.get()); + fdwic_shared_multiworker_test::g_limit_spins_after_remote_fatal = false; + fdwic_shared_multiworker_test::g_observe_turn_wait = false; + } catch (...) { + worker_error = std::current_exception(); + } + worker_returned.store(true, std::memory_order_release); + g_self = nullptr; + }); + + const auto wait_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (fdwic_shared_multiworker_test::g_observed_turn_wait_spins.load(std::memory_order_acquire) == 0 && + std::chrono::steady_clock::now() < wait_deadline) { + std::this_thread::yield(); + } + const bool entered_final_barrier_wait = + fdwic_shared_multiworker_test::g_observed_turn_wait_spins.load(std::memory_order_acquire) != 0; + EXPECT_FALSE(worker_returned.load(std::memory_order_acquire)); + set_fatal_code(PTO2_ERROR_EXPLICIT_ORCH_FATAL); + fdwic_shared_multiworker_test::g_remote_fatal_published.store(true, std::memory_order_release); + worker.join(); + + EXPECT_TRUE(entered_final_barrier_wait); + EXPECT_TRUE(worker_returned.load(std::memory_order_acquire)); + if (worker_error != nullptr) { + try { + std::rethrow_exception(worker_error); + } catch (const std::exception &error) { + FAIL() << "final-barrier waiter threw: " << error.what(); + } catch (...) { + FAIL() << "final-barrier waiter threw a non-standard exception"; + } + } + EXPECT_EQ(g_dist.fatal, 1); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_EXPLICIT_ORCH_FATAL); + EXPECT_EQ(task0_worker_->local_index, kFlagCap); + EXPECT_EQ(g_dist.final_barrier.leaf_arrivals[0].v, 1); + EXPECT_EQ(g_dist.final_barrier.root_arrival.v, 0); + EXPECT_EQ(g_dist.final_barrier.root_release.v, 0); + EXPECT_EQ(g_dist.final_barrier.leaf_releases[0].v, 0); +} + +TEST_F(FdwicSharedMultiworkerTest, SingleLaneKernelExecutesOnceAndCompletesThroughThreeWorkerFinalDrain) { + constexpr int32_t kKernelId = 23; + alignas(PTO2_PACKED_OUTPUT_ALIGN) std::array heap{}; + const uint32_t shape[1] = {1}; + TensorCreateInfo output_info(shape, 1, DataType::FLOAT32); + L0TaskArgs args; + args.add_output(output_info); + + std::vector callable_storage; + install_test_callable( + runtime_, kKernelId, &fdwic_shared_multiworker_test::count_single_lane_kernel, callable_storage + ); + + g_dist.heap_base = heap.data(); + g_dist.heap_size = heap.size(); + g_dist.num_workers = 3; + g_dist.num_blocks = 1; + g_dist.final_barrier.leaf_arrivals[0].expected = 3; + g_dist.final_barrier.root_arrival.expected = 1; + + MixedKernels mixed; + mixed.aiv0_kernel_id = kKernelId; + + // All three physical lanes replay task 0. AIV0 wins deterministically; + // AIC is ineligible, and AIV1 observes the already-published claim. + g_self = task0_worker_.get(); + const DistCompeteFirstTicket aiv0_ticket = dist_submit_compete_first_begin(nullptr, mixed); + (void)dist_submit_compete_first_finish(nullptr, mixed, aiv0_ticket, args); + + g_self = aic_worker_.get(); + const DistCompeteFirstTicket aic_ticket = dist_submit_compete_first_begin(nullptr, mixed); + (void)dist_submit_compete_first_finish(nullptr, mixed, aic_ticket, args); + + g_self = task1_worker_.get(); + const DistCompeteFirstTicket aiv1_ticket = dist_submit_compete_first_begin(nullptr, mixed); + (void)dist_submit_compete_first_finish(nullptr, mixed, aiv1_ticket, args); + + ASSERT_EQ(aiv0_ticket.task_id, 0); + ASSERT_EQ(aiv0_ticket.won, 1); + ASSERT_EQ(aiv0_ticket.joint, 0); + ASSERT_EQ(aic_ticket.task_id, 0); + ASSERT_EQ(aic_ticket.won, 0); + ASSERT_EQ(aic_ticket.claim_attempted, 0); + ASSERT_EQ(aiv1_ticket.task_id, 0); + ASSERT_EQ(aiv1_ticket.won, 0); + ASSERT_EQ(aiv1_ticket.claim_attempted, 1); + + EXPECT_EQ(fdwic_shared_multiworker_test::g_single_lane_kernel_calls.load(std::memory_order_relaxed), 0); + EXPECT_EQ(atomic_load(g_dist.shared_tensor_map.committed_tasks.v), 1); + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_NONE); + EXPECT_EQ(g_dist.blocks[0].any_pub, 0); + EXPECT_EQ(aic_worker_->local_index, 1); + EXPECT_EQ(task0_worker_->local_index, 1); + EXPECT_EQ(task1_worker_->local_index, 1); + EXPECT_EQ(aic_worker_->heap_next, PTO2_PACKED_OUTPUT_ALIGN); + EXPECT_EQ(task0_worker_->heap_next, PTO2_PACKED_OUTPUT_ALIGN); + EXPECT_EQ(task1_worker_->heap_next, PTO2_PACKED_OUTPUT_ALIGN); + EXPECT_EQ(aic_worker_->occupied_count, 0); + ASSERT_EQ(task0_worker_->occupied_count, 1); + EXPECT_EQ(task1_worker_->occupied_count, 0); + const RingSlot *built = find_built_slot(*task0_worker_, 0); + ASSERT_NE(built, nullptr); + EXPECT_EQ(built->func_id, kKernelId); + EXPECT_EQ( + built->function_bin_addr, reinterpret_cast(&fdwic_shared_multiworker_test::count_single_lane_kernel) + ); + ASSERT_EQ(built->tensor_count, 1); + EXPECT_EQ(built->tensors[0].buffer.addr, reinterpret_cast(heap.data())); + EXPECT_EQ(built->tensors[0].buffer.size, sizeof(float)); + EXPECT_EQ(task_cell(0).flag, 0); + EXPECT_EQ(task_cell(0).vend, 0); + EXPECT_EQ(g_dist.frontier, -1); + + std::atomic ready_count{0}; + std::atomic start_drain{false}; + std::atomic aic_returned{false}; + std::atomic aiv0_returned{false}; + std::atomic aiv1_returned{false}; + std::exception_ptr aic_error; + std::exception_ptr aiv0_error; + std::exception_ptr aiv1_error; + + auto drain_worker = [&](DistCore *worker, std::atomic &returned, std::exception_ptr &error) { + g_self = worker; + ready_count.fetch_add(1, std::memory_order_release); + while (!start_drain.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + try { + dist_submit_drain_to_completion(worker); + } catch (...) { + error = std::current_exception(); + } + returned.store(true, std::memory_order_release); + g_self = nullptr; + }; + + std::thread aic_thread(drain_worker, aic_worker_.get(), std::ref(aic_returned), std::ref(aic_error)); + std::thread aiv0_thread(drain_worker, task0_worker_.get(), std::ref(aiv0_returned), std::ref(aiv0_error)); + std::thread aiv1_thread(drain_worker, task1_worker_.get(), std::ref(aiv1_returned), std::ref(aiv1_error)); + + const auto ready_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (ready_count.load(std::memory_order_acquire) != 3 && std::chrono::steady_clock::now() < ready_deadline) { + std::this_thread::yield(); + } + const bool all_workers_ready = ready_count.load(std::memory_order_acquire) == 3; + start_drain.store(true, std::memory_order_release); + aic_thread.join(); + aiv0_thread.join(); + aiv1_thread.join(); + + EXPECT_TRUE(all_workers_ready); + EXPECT_TRUE(aic_returned.load(std::memory_order_acquire)); + EXPECT_TRUE(aiv0_returned.load(std::memory_order_acquire)); + EXPECT_TRUE(aiv1_returned.load(std::memory_order_acquire)); + EXPECT_EQ(aic_error, nullptr); + EXPECT_EQ(aiv0_error, nullptr); + EXPECT_EQ(aiv1_error, nullptr); + + EXPECT_EQ(fdwic_shared_multiworker_test::g_single_lane_kernel_calls.load(std::memory_order_relaxed), 1); + EXPECT_EQ(aic_worker_->occupied_count, 0); + EXPECT_EQ(task0_worker_->occupied_count, 0); + EXPECT_EQ(task1_worker_->occupied_count, 0); + EXPECT_EQ(occupied_slot_count(*aic_worker_), 0); + EXPECT_EQ(occupied_slot_count(*task0_worker_), 0); + EXPECT_EQ(occupied_slot_count(*task1_worker_), 0); + EXPECT_EQ(built_slot_count(*aic_worker_), 0); + EXPECT_EQ(built_slot_count(*task0_worker_), 0); + EXPECT_EQ(built_slot_count(*task1_worker_), 0); + EXPECT_EQ(aic_worker_->local_index, 1); + EXPECT_EQ(task0_worker_->local_index, 1); + EXPECT_EQ(task1_worker_->local_index, 1); + EXPECT_EQ(task_cell(0).flag, 1); + EXPECT_EQ(task_cell(0).vend, PTO2_PACKED_OUTPUT_ALIGN); + EXPECT_EQ(g_dist.frontier, 0); + EXPECT_EQ(atomic_load(g_dist.shared_tensor_map.committed_tasks.v), 1); + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_NONE); + + EXPECT_EQ(g_dist.final_barrier.leaf_arrivals[0].v, 3); + EXPECT_EQ(g_dist.final_barrier.root_arrival.v, 1); + EXPECT_EQ(g_dist.final_barrier.root_release.v, 1); + EXPECT_EQ(g_dist.final_barrier.leaf_releases[0].v, 1); + for (int32_t group = 1; group < kFinalBarrierGroups; ++group) { + EXPECT_EQ(g_dist.final_barrier.leaf_arrivals[group].v, 0); + EXPECT_EQ(g_dist.final_barrier.leaf_releases[group].v, 0); + } +} + +TEST_F(FdwicSharedMultiworkerTest, JointThreeLanePendingFollowersDrainAndLastLaneCompletesOnce) { + constexpr int32_t kAicKernelId = 31; + constexpr int32_t kAiv0KernelId = 32; + constexpr int32_t kAiv1KernelId = 33; + alignas(PTO2_PACKED_OUTPUT_ALIGN) std::array heap{}; + const uint32_t shape[1] = {1}; + TensorCreateInfo output_info(shape, 1, DataType::FLOAT32); + L0TaskArgs args; + args.add_output(output_info); + + std::vector aic_callable; + std::vector aiv0_callable; + std::vector aiv1_callable; + install_test_callable(runtime_, kAicKernelId, &fdwic_shared_multiworker_test::count_joint_aic_kernel, aic_callable); + install_test_callable( + runtime_, kAiv0KernelId, &fdwic_shared_multiworker_test::count_joint_aiv0_kernel, aiv0_callable + ); + install_test_callable( + runtime_, kAiv1KernelId, &fdwic_shared_multiworker_test::count_joint_aiv1_kernel, aiv1_callable + ); + + g_dist.heap_base = heap.data(); + g_dist.heap_size = heap.size(); + g_dist.num_workers = 3; + g_dist.num_blocks = 1; + g_dist.final_barrier.leaf_arrivals[0].expected = 3; + g_dist.final_barrier.root_arrival.expected = 1; + + MixedKernels mixed; + mixed.aic_kernel_id = kAicKernelId; + mixed.aiv0_kernel_id = kAiv0KernelId; + mixed.aiv1_kernel_id = kAiv1KernelId; + + // CPU-sim keeps the "joint submit has been seen" fact in TLS. Each physical + // worker therefore stays on one host thread from Begin through FinalDrain, + // matching the persistent A5 worker lifecycle instead of manually seeding + // the test-only thread-local state. + DistCore *workers[3] = {aic_worker_.get(), task0_worker_.get(), task1_worker_.get()}; + DistCompeteFirstTicket tickets[3]{}; + std::exception_ptr errors[3]; + std::atomic allow_begin[3]{}; + std::atomic begin_done[3]{}; + std::atomic allow_finish[3]{}; + std::atomic finish_done[3]{}; + std::atomic drain_returned[3]{}; + std::atomic failed[3]{}; + std::atomic tls_after_begin[3]{}; + std::atomic tls_after_finish[3]{}; + std::atomic start_drain{false}; + std::atomic abort_workers{false}; + + auto worker_main = [&](int32_t lane) { + g_self = workers[lane]; + g_fdwic_joint_submit_seen = false; + try { + while (!allow_begin[lane].load(std::memory_order_acquire) && + !abort_workers.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + if (!abort_workers.load(std::memory_order_acquire)) { + tickets[lane] = dist_submit_compete_first_begin(nullptr, mixed); + tls_after_begin[lane].store(g_fdwic_joint_submit_seen, std::memory_order_relaxed); + } + begin_done[lane].store(true, std::memory_order_release); + + while (!allow_finish[lane].load(std::memory_order_acquire) && + !abort_workers.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + if (!abort_workers.load(std::memory_order_acquire)) { + (void)dist_submit_compete_first_finish(nullptr, mixed, tickets[lane], args); + tls_after_finish[lane].store(g_fdwic_joint_submit_seen, std::memory_order_relaxed); + } + finish_done[lane].store(true, std::memory_order_release); + + while (!start_drain.load(std::memory_order_acquire) && !abort_workers.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + if (!abort_workers.load(std::memory_order_acquire)) { + dist_submit_drain_to_completion(workers[lane]); + } + } catch (...) { + errors[lane] = std::current_exception(); + failed[lane].store(true, std::memory_order_release); + } + begin_done[lane].store(true, std::memory_order_release); + finish_done[lane].store(true, std::memory_order_release); + drain_returned[lane].store(true, std::memory_order_release); + g_self = nullptr; + }; + + std::thread aic_thread(worker_main, LANE_AIC); + std::thread aiv0_thread(worker_main, LANE_AIV0); + std::thread aiv1_thread(worker_main, LANE_AIV1); + + auto release_all_gates = [&](bool abort) { + if (abort) { + abort_workers.store(true, std::memory_order_release); + } + for (int32_t lane = LANE_AIC; lane <= LANE_AIV1; ++lane) { + allow_begin[lane].store(true, std::memory_order_release); + allow_finish[lane].store(true, std::memory_order_release); + fdwic_shared_multiworker_test::g_joint_kernel_release[lane].store(true, std::memory_order_release); + } + start_drain.store(true, std::memory_order_release); + }; + auto join_all_workers = [&]() { + if (aic_thread.joinable()) aic_thread.join(); + if (aiv0_thread.joinable()) aiv0_thread.join(); + if (aiv1_thread.joinable()) aiv1_thread.join(); + }; + auto abort_and_join = [&]() { + release_all_gates(/*abort=*/true); + join_all_workers(); + }; + + const auto protocol_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + auto wait_until = [&](const auto &predicate) { + while (!predicate() && std::chrono::steady_clock::now() < protocol_deadline) { + std::this_thread::yield(); + } + return predicate(); + }; + + // AIC is the only eligible anchor for this active mask. Begin it first to + // make the staged observation deterministic, then let both followers + // replay the same task on their persistent worker threads. + allow_begin[LANE_AIC].store(true, std::memory_order_release); + const bool aic_begin_completed = wait_until([&]() { + return begin_done[LANE_AIC].load(std::memory_order_acquire); + }); + if (!aic_begin_completed || failed[LANE_AIC].load(std::memory_order_acquire)) { + abort_and_join(); + ADD_FAILURE() << "AIC Begin did not complete successfully"; + return; + } + allow_begin[LANE_AIV0].store(true, std::memory_order_release); + allow_begin[LANE_AIV1].store(true, std::memory_order_release); + const bool all_begins_completed = wait_until([&]() { + return begin_done[LANE_AIC].load(std::memory_order_acquire) && + begin_done[LANE_AIV0].load(std::memory_order_acquire) && + begin_done[LANE_AIV1].load(std::memory_order_acquire); + }); + const bool begins_succeeded = + aic_begin_completed && all_begins_completed && !failed[LANE_AIC].load(std::memory_order_acquire) && + !failed[LANE_AIV0].load(std::memory_order_acquire) && !failed[LANE_AIV1].load(std::memory_order_acquire); + if (!begins_succeeded) { + abort_and_join(); + ADD_FAILURE() << "all three persistent workers did not complete Begin successfully"; + return; + } + + // Finish both losers before the anchor publishes. Their real loser tail + // calls drain_block_won(), but any_pub is still zero, so both deposits must + // remain pending until these same threads enter FinalDrain. + allow_finish[LANE_AIV0].store(true, std::memory_order_release); + allow_finish[LANE_AIV1].store(true, std::memory_order_release); + const bool followers_finished = wait_until([&]() { + return finish_done[LANE_AIV0].load(std::memory_order_acquire) && + finish_done[LANE_AIV1].load(std::memory_order_acquire); + }); + const bool followers_succeeded = begins_succeeded && followers_finished && + !failed[LANE_AIV0].load(std::memory_order_acquire) && + !failed[LANE_AIV1].load(std::memory_order_acquire); + if (!followers_succeeded) { + abort_and_join(); + ADD_FAILURE() << "both follower Finish calls did not complete successfully"; + return; + } + const int64_t committed_before_anchor = atomic_load(g_dist.shared_tensor_map.committed_tasks.v); + const int32_t any_pub_before_anchor = atomic_load(g_dist.blocks[0].any_pub); + const int32_t aiv0_occupied_before_anchor = task0_worker_->occupied_count; + const int32_t aiv1_occupied_before_anchor = task1_worker_->occupied_count; + const int64_t flag_before_anchor = atomic_load(task_cell(0).flag); + const uint64_t vend_before_anchor = atomic_load(task_cell(0).vend); + const int64_t frontier_before_anchor = atomic_load(g_dist.frontier); + + allow_finish[LANE_AIC].store(true, std::memory_order_release); + const bool anchor_finish_stage_reached = wait_until([&]() { + return finish_done[LANE_AIC].load(std::memory_order_acquire); + }); + const bool anchor_finished = + followers_succeeded && anchor_finish_stage_reached && !failed[LANE_AIC].load(std::memory_order_acquire); + if (!anchor_finished) { + abort_and_join(); + ADD_FAILURE() << "AIC anchor Finish did not complete successfully"; + return; + } + + struct JointDepositSnapshot { + bool present = false; + int64_t drained = -1; + int32_t func_id = INVALID_KERNEL_ID; + uint64_t function_bin_addr = 0; + int32_t sub_block_id = -1; + int32_t tensor_count = -1; + int32_t fanin_count = -1; + uint64_t tensor_addr = 0; + uint64_t tensor_size = 0; + }; + struct AnchorSlotSnapshot { + bool found = false; + bool occupied = false; + bool built = false; + bool is_multicore = false; + int32_t task_id = -1; + int32_t func_id = INVALID_KERNEL_ID; + uint64_t function_bin_addr = 0; + int32_t won_block = -1; + int32_t won_slot = -1; + int32_t tensor_count = -1; + int32_t fanin_count = -1; + uint64_t tensor_addr = 0; + uint64_t tensor_size = 0; + }; + + int32_t published_index = -1; + int32_t published_count = 0; + int32_t non_free_won_slot_count = 0; + for (int32_t index = 0; index < kPrivateSlots; ++index) { + const int64_t state = atomic_load(g_dist.blocks[0].slots[index].state.v); + if (state != kWonStateFree) { + ++non_free_won_slot_count; + } + if (state == kWonStatePublished) { + published_index = index; + ++published_count; + } + } + WonSlot *won_slot = published_index >= 0 ? &g_dist.blocks[0].slots[published_index] : nullptr; + const RingSlot *anchor_slot = find_built_slot(*aic_worker_, 0); + JointDepositSnapshot deposits[3]{}; + if (won_slot != nullptr) { + for (int32_t lane = LANE_AIC; lane <= LANE_AIV1; ++lane) { + const BuiltSubtask &source = won_slot->lane[lane]; + JointDepositSnapshot &snapshot = deposits[lane]; + snapshot.present = source.present; + snapshot.drained = atomic_load(won_slot->drained[lane].v); + snapshot.func_id = source.func_id; + snapshot.function_bin_addr = source.function_bin_addr; + snapshot.sub_block_id = source.sub_block_id; + snapshot.tensor_count = source.tensor_count; + snapshot.fanin_count = source.fanin_count; + if (source.tensor_count > 0) { + snapshot.tensor_addr = source.tensors[0].buffer.addr; + snapshot.tensor_size = source.tensors[0].buffer.size; + } + } + } + AnchorSlotSnapshot anchor_snapshot; + if (anchor_slot != nullptr) { + anchor_snapshot.found = true; + anchor_snapshot.occupied = anchor_slot->occupied; + anchor_snapshot.built = anchor_slot->built; + anchor_snapshot.is_multicore = anchor_slot->is_multicore; + anchor_snapshot.task_id = anchor_slot->task_id; + anchor_snapshot.func_id = anchor_slot->func_id; + anchor_snapshot.function_bin_addr = anchor_slot->function_bin_addr; + anchor_snapshot.won_block = anchor_slot->won_block; + anchor_snapshot.won_slot = anchor_slot->won_slot; + anchor_snapshot.tensor_count = anchor_slot->tensor_count; + anchor_snapshot.fanin_count = anchor_slot->fanin_count; + if (anchor_slot->tensor_count > 0) { + anchor_snapshot.tensor_addr = anchor_slot->tensors[0].buffer.addr; + anchor_snapshot.tensor_size = anchor_slot->tensors[0].buffer.size; + } + } + + const bool tickets_ok = begins_succeeded && tickets[LANE_AIC].task_id == 0 && tickets[LANE_AIC].won == 1 && + tickets[LANE_AIC].joint == 1 && tickets[LANE_AIC].joint_init == 1 && + tickets[LANE_AIC].claim_attempted == 1 && tickets[LANE_AIC].joint_count == 3 && + tickets[LANE_AIC].joint_block == 0 && tickets[LANE_AIC].kernel_id == kAicKernelId && + tickets[LANE_AIV0].task_id == 0 && tickets[LANE_AIV0].won == 0 && + tickets[LANE_AIV0].joint == 1 && tickets[LANE_AIV0].joint_init == 0 && + tickets[LANE_AIV0].claim_attempted == 0 && tickets[LANE_AIV0].joint_count == 3 && + tickets[LANE_AIV0].joint_block == 0 && tickets[LANE_AIV0].kernel_id == INVALID_KERNEL_ID && + tickets[LANE_AIV1].task_id == 0 && tickets[LANE_AIV1].won == 0 && + tickets[LANE_AIV1].joint == 1 && tickets[LANE_AIV1].joint_init == 0 && + tickets[LANE_AIV1].claim_attempted == 0 && tickets[LANE_AIV1].joint_count == 3 && + tickets[LANE_AIV1].joint_block == 0 && tickets[LANE_AIV1].kernel_id == INVALID_KERNEL_ID; + const bool no_worker_failed = !failed[LANE_AIC].load(std::memory_order_acquire) && + !failed[LANE_AIV0].load(std::memory_order_acquire) && + !failed[LANE_AIV1].load(std::memory_order_acquire); + const bool published_slot_ok = + won_slot != nullptr && published_count == 1 && non_free_won_slot_count == 1 && + atomic_load(won_slot->remaining.v) == 3 && !deposits[LANE_AIC].present && deposits[LANE_AIV0].present && + deposits[LANE_AIV1].present && deposits[LANE_AIV0].drained == kDrainedFree && + deposits[LANE_AIV1].drained == kDrainedFree && deposits[LANE_AIV0].func_id == kAiv0KernelId && + deposits[LANE_AIV0].function_bin_addr == + reinterpret_cast(&fdwic_shared_multiworker_test::count_joint_aiv0_kernel) && + deposits[LANE_AIV0].sub_block_id == 0 && deposits[LANE_AIV0].tensor_count == 1 && + deposits[LANE_AIV0].fanin_count == 0 && + deposits[LANE_AIV0].tensor_addr == reinterpret_cast(heap.data()) && + deposits[LANE_AIV0].tensor_size == sizeof(float) && deposits[LANE_AIV1].func_id == kAiv1KernelId && + deposits[LANE_AIV1].function_bin_addr == + reinterpret_cast(&fdwic_shared_multiworker_test::count_joint_aiv1_kernel) && + deposits[LANE_AIV1].sub_block_id == 1 && deposits[LANE_AIV1].tensor_count == 1 && + deposits[LANE_AIV1].fanin_count == 0 && + deposits[LANE_AIV1].tensor_addr == reinterpret_cast(heap.data()) && + deposits[LANE_AIV1].tensor_size == sizeof(float); + const bool anchor_slot_ok = + anchor_snapshot.found && anchor_snapshot.occupied && anchor_snapshot.built && anchor_snapshot.is_multicore && + anchor_snapshot.task_id == 0 && anchor_snapshot.func_id == kAicKernelId && + anchor_snapshot.function_bin_addr == + reinterpret_cast(&fdwic_shared_multiworker_test::count_joint_aic_kernel) && + anchor_snapshot.won_block == 0 && anchor_snapshot.won_slot == published_index && + anchor_snapshot.tensor_count == 1 && anchor_snapshot.fanin_count == 0 && + anchor_snapshot.tensor_addr == reinterpret_cast(heap.data()) && + anchor_snapshot.tensor_size == sizeof(float) && aic_worker_->occupied_count == 1 && + built_slot_count(*aic_worker_) == 1 && task0_worker_->occupied_count == 0 && + task1_worker_->occupied_count == 0 && built_slot_count(*task0_worker_) == 0 && + built_slot_count(*task1_worker_) == 0; + const bool pre_drain_ok = aic_begin_completed && all_begins_completed && followers_finished && anchor_finished && + no_worker_failed && tickets_ok && published_slot_ok && anchor_slot_ok; + if (!pre_drain_ok) { + abort_and_join(); + EXPECT_TRUE(no_worker_failed); + EXPECT_TRUE(tickets_ok); + EXPECT_EQ(published_count, 1); + EXPECT_EQ(non_free_won_slot_count, 1); + EXPECT_TRUE(published_slot_ok); + EXPECT_TRUE(anchor_slot_ok); + ADD_FAILURE() << "joint task pre-FinalDrain invariants were not established"; + return; + } + start_drain.store(true, std::memory_order_release); + + auto all_joint_kernels_entered = [&]() { + for (int32_t lane = LANE_AIC; lane <= LANE_AIV1; ++lane) { + if (fdwic_shared_multiworker_test::g_joint_kernel_entered[lane].load(std::memory_order_acquire) != 1) { + return false; + } + } + return true; + }; + const bool all_kernels_entered = pre_drain_ok && wait_until(all_joint_kernels_entered); + const int64_t remaining_before_release = won_slot != nullptr ? atomic_load(won_slot->remaining.v) : -1; + const int64_t flag_before_release = atomic_load(task_cell(0).flag); + const int64_t state_before_release = won_slot != nullptr ? atomic_load(won_slot->state.v) : -1; + + auto wait_remaining_and_returned = [&](int64_t expected, int32_t lane) { + if (won_slot == nullptr) return false; + return wait_until([&]() { + return atomic_load(won_slot->remaining.v) == expected && + drain_returned[lane].load(std::memory_order_acquire); + }); + }; + + fdwic_shared_multiworker_test::g_joint_kernel_release[LANE_AIC].store(true, std::memory_order_release); + const bool anchor_decremented_and_returned = all_kernels_entered && wait_remaining_and_returned(2, LANE_AIC); + const int64_t flag_after_anchor = atomic_load(task_cell(0).flag); + const uint64_t vend_after_anchor = atomic_load(task_cell(0).vend); + const int64_t frontier_after_anchor = atomic_load(g_dist.frontier); + const int64_t state_after_anchor = won_slot != nullptr ? atomic_load(won_slot->state.v) : -1; + const uint32_t aiv1_exited_after_anchor = + fdwic_shared_multiworker_test::g_joint_kernel_exited[LANE_AIV1].load(std::memory_order_acquire); + const bool aiv1_returned_after_anchor = drain_returned[LANE_AIV1].load(std::memory_order_acquire); + + fdwic_shared_multiworker_test::g_joint_kernel_release[LANE_AIV0].store(true, std::memory_order_release); + const bool first_follower_decremented_and_returned = + all_kernels_entered && wait_remaining_and_returned(1, LANE_AIV0); + const int64_t flag_after_first_follower = atomic_load(task_cell(0).flag); + const uint64_t vend_after_first_follower = atomic_load(task_cell(0).vend); + const int64_t frontier_after_first_follower = atomic_load(g_dist.frontier); + const int64_t state_after_first_follower = won_slot != nullptr ? atomic_load(won_slot->state.v) : -1; + const uint32_t aiv1_exited_after_first_follower = + fdwic_shared_multiworker_test::g_joint_kernel_exited[LANE_AIV1].load(std::memory_order_acquire); + const bool aiv1_returned_after_first_follower = drain_returned[LANE_AIV1].load(std::memory_order_acquire); + + // Always release every test gate before joining, including a failed + // observation, so this test cannot manufacture its own deadlock. + release_all_gates(/*abort=*/false); + join_all_workers(); + + EXPECT_TRUE(aic_begin_completed); + EXPECT_TRUE(all_begins_completed); + EXPECT_TRUE(begins_succeeded); + EXPECT_TRUE(followers_finished); + EXPECT_TRUE(followers_succeeded); + EXPECT_TRUE(anchor_finish_stage_reached); + EXPECT_TRUE(anchor_finished); + EXPECT_TRUE(no_worker_failed); + EXPECT_TRUE(tickets_ok); + EXPECT_TRUE(published_slot_ok); + EXPECT_TRUE(anchor_slot_ok); + EXPECT_TRUE(pre_drain_ok); + EXPECT_TRUE(all_kernels_entered); + EXPECT_EQ(committed_before_anchor, 0); + EXPECT_EQ(any_pub_before_anchor, 0); + EXPECT_EQ(aiv0_occupied_before_anchor, 0); + EXPECT_EQ(aiv1_occupied_before_anchor, 0); + EXPECT_EQ(flag_before_anchor, 0); + EXPECT_EQ(vend_before_anchor, 0); + EXPECT_EQ(frontier_before_anchor, -1); + + for (int32_t lane = LANE_AIC; lane <= LANE_AIV1; ++lane) { + EXPECT_TRUE(tls_after_begin[lane].load(std::memory_order_relaxed)); + EXPECT_TRUE(tls_after_finish[lane].load(std::memory_order_relaxed)); + EXPECT_TRUE(drain_returned[lane].load(std::memory_order_acquire)); + EXPECT_EQ(errors[lane], nullptr); + } + + ASSERT_NE(won_slot, nullptr); + EXPECT_EQ(atomic_load(g_dist.shared_tensor_map.committed_tasks.v), 1); + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_NONE); + EXPECT_EQ(g_dist.blocks[0].any_pub, 1); + EXPECT_EQ(remaining_before_release, 3); + EXPECT_EQ(flag_before_release, 0); + EXPECT_EQ(state_before_release, kWonStatePublished); + EXPECT_TRUE(anchor_decremented_and_returned); + EXPECT_EQ(flag_after_anchor, 0); + EXPECT_EQ(vend_after_anchor, 0); + EXPECT_EQ(frontier_after_anchor, -1); + EXPECT_EQ(state_after_anchor, kWonStatePublished); + EXPECT_EQ(aiv1_exited_after_anchor, 0); + EXPECT_FALSE(aiv1_returned_after_anchor); + EXPECT_TRUE(first_follower_decremented_and_returned); + EXPECT_EQ(flag_after_first_follower, 0); + EXPECT_EQ(vend_after_first_follower, 0); + EXPECT_EQ(frontier_after_first_follower, -1); + EXPECT_EQ(state_after_first_follower, kWonStatePublished); + EXPECT_EQ(aiv1_exited_after_first_follower, 0); + EXPECT_FALSE(aiv1_returned_after_first_follower); + + EXPECT_FALSE(deposits[LANE_AIC].present); + EXPECT_TRUE(deposits[LANE_AIV0].present); + EXPECT_TRUE(deposits[LANE_AIV1].present); + EXPECT_EQ(deposits[LANE_AIV0].func_id, kAiv0KernelId); + EXPECT_EQ(deposits[LANE_AIV1].func_id, kAiv1KernelId); + EXPECT_EQ(deposits[LANE_AIV0].sub_block_id, 0); + EXPECT_EQ(deposits[LANE_AIV1].sub_block_id, 1); + EXPECT_EQ(deposits[LANE_AIV0].tensor_count, 1); + EXPECT_EQ(deposits[LANE_AIV1].tensor_count, 1); + EXPECT_EQ(deposits[LANE_AIV0].fanin_count, 0); + EXPECT_EQ(deposits[LANE_AIV1].fanin_count, 0); + EXPECT_EQ(deposits[LANE_AIV0].tensor_addr, reinterpret_cast(heap.data())); + EXPECT_EQ(deposits[LANE_AIV1].tensor_addr, reinterpret_cast(heap.data())); + EXPECT_EQ(deposits[LANE_AIV0].tensor_size, sizeof(float)); + EXPECT_EQ(deposits[LANE_AIV1].tensor_size, sizeof(float)); + EXPECT_EQ( + deposits[LANE_AIV0].function_bin_addr, + reinterpret_cast(&fdwic_shared_multiworker_test::count_joint_aiv0_kernel) + ); + EXPECT_EQ( + deposits[LANE_AIV1].function_bin_addr, + reinterpret_cast(&fdwic_shared_multiworker_test::count_joint_aiv1_kernel) + ); + EXPECT_EQ(atomic_load(won_slot->drained[LANE_AIV0].v), kDrainedClaimed); + EXPECT_EQ(atomic_load(won_slot->drained[LANE_AIV1].v), kDrainedClaimed); + + EXPECT_TRUE(anchor_snapshot.is_multicore); + EXPECT_EQ(anchor_snapshot.func_id, kAicKernelId); + EXPECT_EQ( + anchor_snapshot.function_bin_addr, + reinterpret_cast(&fdwic_shared_multiworker_test::count_joint_aic_kernel) + ); + EXPECT_EQ(anchor_snapshot.won_block, 0); + EXPECT_EQ(anchor_snapshot.won_slot, published_index); + EXPECT_EQ(anchor_snapshot.tensor_count, 1); + EXPECT_EQ(anchor_snapshot.fanin_count, 0); + EXPECT_EQ(anchor_snapshot.tensor_addr, reinterpret_cast(heap.data())); + EXPECT_EQ(anchor_snapshot.tensor_size, sizeof(float)); + EXPECT_EQ(aic_worker_->heap_next, PTO2_PACKED_OUTPUT_ALIGN); + EXPECT_EQ(task0_worker_->heap_next, PTO2_PACKED_OUTPUT_ALIGN); + EXPECT_EQ(task1_worker_->heap_next, PTO2_PACKED_OUTPUT_ALIGN); + EXPECT_EQ(aic_worker_->owned_total, 0); + EXPECT_EQ(task0_worker_->owned_total, 1); + EXPECT_EQ(task1_worker_->owned_total, 1); + + for (int32_t lane = LANE_AIC; lane <= LANE_AIV1; ++lane) { + EXPECT_EQ(fdwic_shared_multiworker_test::g_joint_kernel_entered[lane].load(std::memory_order_acquire), 1); + EXPECT_EQ(fdwic_shared_multiworker_test::g_joint_kernel_exited[lane].load(std::memory_order_acquire), 1); + } + EXPECT_EQ(atomic_load(won_slot->remaining.v), 0); + EXPECT_EQ(atomic_load(won_slot->state.v), kWonStateFree); + EXPECT_EQ(task_cell(0).flag, 1); + EXPECT_EQ(task_cell(0).vend, PTO2_PACKED_OUTPUT_ALIGN); + EXPECT_EQ(g_dist.frontier, 0); + EXPECT_EQ(aic_worker_->occupied_count, 0); + EXPECT_EQ(task0_worker_->occupied_count, 0); + EXPECT_EQ(task1_worker_->occupied_count, 0); + EXPECT_EQ(occupied_slot_count(*aic_worker_), 0); + EXPECT_EQ(occupied_slot_count(*task0_worker_), 0); + EXPECT_EQ(occupied_slot_count(*task1_worker_), 0); + EXPECT_EQ(built_slot_count(*aic_worker_), 0); + EXPECT_EQ(built_slot_count(*task0_worker_), 0); + EXPECT_EQ(built_slot_count(*task1_worker_), 0); + EXPECT_EQ(aic_worker_->local_index, 1); + EXPECT_EQ(task0_worker_->local_index, 1); + EXPECT_EQ(task1_worker_->local_index, 1); + + EXPECT_EQ(g_dist.final_barrier.leaf_arrivals[0].v, 3); + EXPECT_EQ(g_dist.final_barrier.root_arrival.v, 1); + EXPECT_EQ(g_dist.final_barrier.root_release.v, 1); + EXPECT_EQ(g_dist.final_barrier.leaf_releases[0].v, 1); + for (int32_t group = 1; group < kFinalBarrierGroups; ++group) { + EXPECT_EQ(g_dist.final_barrier.leaf_arrivals[group].v, 0); + EXPECT_EQ(g_dist.final_barrier.leaf_releases[group].v, 0); + } +} + +} // namespace diff --git a/tests/ut/cpp/a5/test_fdwic_shared_submit_contract.cpp b/tests/ut/cpp/a5/test_fdwic_shared_submit_contract.cpp new file mode 100644 index 0000000000..20e7f61d31 --- /dev/null +++ b/tests/ut/cpp/a5/test_fdwic_shared_submit_contract.cpp @@ -0,0 +1,161 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include +#include + +#define PTO_FDWIC_TRACE_ENABLED 0 +#include "dist_engine/aicore/dist_engine.cpp" // NOLINT(build/include) + +[[noreturn]] void assert_impl(const char *condition, const char *, int) { throw std::logic_error(condition); } + +extern "C" void aicpu_orchestration_entry(const L2TaskArgs &) {} +volatile uint8_t *sim_get_reg_base() { return nullptr; } +uint32_t sim_get_physical_core_id() { return 0; } + +namespace { + +class FdwicSharedSubmitContractTest : public ::testing::Test { +protected: + void SetUp() override { + static_assert(PTO_FDWIC_SHARED_MAP == 1); + static_assert(!kFdwicCompiledBackendReady); + g_dist_ptr = &g_dist_fallback; + g_self = worker_.get(); + dist_core_reset(*worker_, CoreType::AIV, /*block=*/0, LANE_AIV0); + worker_->core_idx = 0; + g_dist.fatal = 0; + g_dist.error_code = PTO2_ERROR_NONE; + } + + void TearDown() override { + g_self = nullptr; + g_dist_ptr = nullptr; + } + + DistSubmitCtx make_context() { + DistSubmitCtx ctx{}; + ctx.self = worker_.get(); + ctx.task_id = 7; + return ctx; + } + + void seed_data_plane_markers(const DistSubmitCtx &ctx) { + g_dist.shared_tensor_map.committed_tasks.v = 19; + g_dist.shared_tensor_map.reclaim_upto.v = 11; + g_dist.shared_tensor_map.buckets[0].head.v = 3; + g_dist.shared_tensor_map.buckets[0].tail.v = 5; + g_dist.shared_tensor_map.slots[0].payload.value = SharedTensorMapValue{0x1000, 8, 32, 4, 0}; + g_dist.shared_tensor_map.slots[0].sequence.v = 17; + task_cell(ctx.task_id).flag = 23; + task_cell(ctx.task_id).vend = 29; + worker_->occupied_count = 31; + worker_->owned_total = 37; + } + + void expect_data_plane_markers_unchanged(const DistSubmitCtx &ctx) { + EXPECT_EQ(g_dist.shared_tensor_map.committed_tasks.v, 19); + EXPECT_EQ(g_dist.shared_tensor_map.reclaim_upto.v, 11); + EXPECT_EQ(g_dist.shared_tensor_map.buckets[0].head.v, 3); + EXPECT_EQ(g_dist.shared_tensor_map.buckets[0].tail.v, 5); + const SharedTensorMapValue &value = g_dist.shared_tensor_map.slots[0].payload.value; + EXPECT_EQ(value.buf_addr, 0x1000U); + EXPECT_EQ(value.lo, 8U); + EXPECT_EQ(value.hi, 32U); + EXPECT_EQ(value.producer, 4); + EXPECT_EQ(value.reserved, 0U); + EXPECT_EQ(g_dist.shared_tensor_map.slots[0].sequence.v, 17); + EXPECT_EQ(task_cell(ctx.task_id).flag, 23); + EXPECT_EQ(task_cell(ctx.task_id).vend, 29U); + EXPECT_EQ(worker_->occupied_count, 31); + EXPECT_EQ(worker_->owned_total, 37); + } + + std::unique_ptr worker_ = std::make_unique(); +}; + +TEST_F(FdwicSharedSubmitContractTest, CommittedResultDoesNotLatchFailure) { + DistSubmitCtx ctx = make_context(); + + EXPECT_EQ( + dist_submit_shared_tensor_map_error_code(DistSharedTensorMapTaskPublishResult::Committed), PTO2_ERROR_NONE + ); + EXPECT_TRUE(dist_submit_handle_shared_tensor_map_result(ctx, DistSharedTensorMapTaskPublishResult::Committed)); + EXPECT_EQ(worker_->local_index, 0); + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_NONE); +} + +struct SharedPublishFailureCase { + DistSharedTensorMapTaskPublishResult result; + int32_t error_code; +}; + +TEST_F(FdwicSharedSubmitContractTest, EveryFailureClassLatchesItsStructuredCode) { + const SharedPublishFailureCase cases[] = { + {DistSharedTensorMapTaskPublishResult::CapacityBlocked, PTO2_ERROR_TENSORMAP_CAPACITY}, + {DistSharedTensorMapTaskPublishResult::ProtocolError, PTO2_ERROR_TENSORMAP_PROTOCOL}, + {DistSharedTensorMapTaskPublishResult::PartialPublish, PTO2_ERROR_TENSORMAP_PARTIAL_PUBLISH}, + {static_cast(UINT32_MAX), PTO2_ERROR_TENSORMAP_PROTOCOL}, + }; + + for (const SharedPublishFailureCase &test_case : cases) { + g_dist.fatal = 0; + g_dist.error_code = PTO2_ERROR_NONE; + worker_->local_index = 8; + DistSubmitCtx ctx = make_context(); + seed_data_plane_markers(ctx); + + EXPECT_FALSE(dist_submit_handle_shared_tensor_map_result(ctx, test_case.result)); + EXPECT_EQ(worker_->local_index, kFlagCap); + EXPECT_EQ(g_dist.fatal, 1); + EXPECT_EQ(g_dist.error_code, test_case.error_code); + expect_data_plane_markers_unchanged(ctx); + } +} + +TEST_F(FdwicSharedSubmitContractTest, PreexistingFailureCodeRemainsAuthoritativeForEverySharedFailure) { + const DistSharedTensorMapTaskPublishResult cases[] = { + DistSharedTensorMapTaskPublishResult::CapacityBlocked, + DistSharedTensorMapTaskPublishResult::ProtocolError, + DistSharedTensorMapTaskPublishResult::PartialPublish, + static_cast(UINT32_MAX), + }; + + for (DistSharedTensorMapTaskPublishResult result : cases) { + g_dist.fatal = 1; + g_dist.error_code = PTO2_ERROR_INVALID_ARGS; + worker_->local_index = 8; + DistSubmitCtx ctx = make_context(); + + EXPECT_FALSE(dist_submit_handle_shared_tensor_map_result(ctx, result)); + EXPECT_EQ(worker_->local_index, kFlagCap); + EXPECT_EQ(g_dist.fatal, 1); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_INVALID_ARGS); + } +} + +TEST(FdwicSharedSubmitStatus, EverySharedFailureCodeIsHostVisible) { + EXPECT_EQ( + runtime_status_from_error_codes(PTO2_ERROR_TENSORMAP_CAPACITY, PTO2_ERROR_NONE), -PTO2_ERROR_TENSORMAP_CAPACITY + ); + EXPECT_EQ( + runtime_status_from_error_codes(PTO2_ERROR_TENSORMAP_PROTOCOL, PTO2_ERROR_NONE), -PTO2_ERROR_TENSORMAP_PROTOCOL + ); + EXPECT_EQ( + runtime_status_from_error_codes(PTO2_ERROR_TENSORMAP_PARTIAL_PUBLISH, PTO2_ERROR_NONE), + -PTO2_ERROR_TENSORMAP_PARTIAL_PUBLISH + ); +} + +} // namespace diff --git a/tests/ut/cpp/a5/test_fdwic_shared_submit_wiring.cpp b/tests/ut/cpp/a5/test_fdwic_shared_submit_wiring.cpp new file mode 100644 index 0000000000..22793e40d8 --- /dev/null +++ b/tests/ut/cpp/a5/test_fdwic_shared_submit_wiring.cpp @@ -0,0 +1,577 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#define PTO_FDWIC_TRACE_ENABLED 0 +#include "dist_engine/aicore/dist_engine.cpp" // NOLINT(build/include) +#include "dist_engine/aicpu/shared_tensor_map_init.h" + +[[noreturn]] void assert_impl(const char *condition, const char *, int) { throw std::logic_error(condition); } + +extern "C" void aicpu_orchestration_entry(const L2TaskArgs &) {} +volatile uint8_t *sim_get_reg_base() { return nullptr; } +uint32_t sim_get_physical_core_id() { return 0; } + +Runtime::Runtime() { + for (uint64_t &address : func_id_to_addr_) { + address = 0; + } + use_example_exec_time_ = false; + for (int32_t &duration : example_exec_time_ns_) { + duration = 0; + } +} + +namespace { + +Tensor make_existing_tensor(uint64_t address) { + Tensor tensor{}; + const uint32_t shape[1] = {1}; + tensor.init_external( + reinterpret_cast(static_cast(address)), sizeof(float), shape, 1, DataType::FLOAT32, 0 + ); + return tensor; +} + +Tensor make_owned_tensor(uint64_t address, int32_t owner) { + Tensor tensor = make_existing_tensor(address); + tensor.owner_task_id = PTO2TaskId::make(0, static_cast(owner)); + return tensor; +} + +const RingSlot *find_built_slot(const DistCore &worker, int32_t task_id) { + for (int32_t index = 0; index < kPrivateSlots; ++index) { + const RingSlot &slot = worker.slots[index]; + if (slot.occupied && slot.built && slot.task_id == task_id) { + return &slot; + } + } + return nullptr; +} + +int32_t fanin_occurrences(const RingSlot &slot, int32_t producer) { + int32_t count = 0; + for (int32_t index = 0; index < slot.fanin_count; ++index) { + if (slot.fanin[index] == producer) { + ++count; + } + } + return count; +} + +class FdwicSharedSubmitWiringTest : public ::testing::Test { +protected: + void SetUp() override { + static_assert(PTO_FDWIC_SHARED_MAP == 1); + static_assert(!kFdwicCompiledBackendReady); + g_dist_ptr = &g_dist_fallback; + g_self = worker_.get(); + reset_worker(CoreType::AIV, LANE_AIV0); + dist_shared_tensor_map_reset(g_dist.shared_tensor_map); + + g_fdwic_joint_submit_seen = false; + g_dist.H = kHDefault; + g_dist.heap_base = nullptr; + g_dist.heap_size = 0; + g_dist.runtime = &runtime_; + g_dist.num_workers = 1; + g_dist.num_blocks = 1; + g_dist.frontier = -1; + g_dist.fatal = 0; + g_dist.error_code = PTO2_ERROR_NONE; + g_dist.blocks[0].any_pub = 0; + for (int32_t index = 0; index < kPrivateSlots; ++index) { + g_dist.blocks[0].slots[index].state.v = kWonStateFree; + } + for (int32_t shard = 0; shard < kCursorShards; ++shard) { + g_dist.cube_cursor[shard].v = -1; + g_dist.vector_cursor[shard].v = -1; + g_dist.alloc_cursor[shard].v = -1; + } + for (int32_t task = 0; task < 256; ++task) { + reset_task_cell(task); + } + } + + void TearDown() override { + g_self = nullptr; + g_dist_ptr = nullptr; + } + + void reset_worker(CoreType role, int32_t lane) { + dist_core_reset(*worker_, role, /*block=*/0, lane); + worker_->core_idx = 0; + } + + void publish_seed_task(const SharedTensorMapValue *entries, uint32_t count, int32_t task, int32_t history) { + ASSERT_EQ( + dist_shared_tensor_map_publish_task(g_dist.shared_tensor_map, entries, count, task, history), + DistSharedTensorMapTaskPublishResult::Committed + ); + ASSERT_EQ(g_dist.shared_tensor_map.committed_tasks.v, task + 1); + } + + void fill_one_bucket(const Tensor &tensor, int32_t count) { + for (int32_t task = 0; task < count; ++task) { + const SharedTensorMapValue entry = dist_shared_tensor_map_make_value(tensor, task); + publish_seed_task(&entry, 1, task, kFlagCap - 1); + } + } + + std::vector snapshot_shared_tensor_map() const { + std::vector snapshot(sizeof(g_dist.shared_tensor_map)); + std::memcpy(snapshot.data(), &g_dist.shared_tensor_map, snapshot.size()); + return snapshot; + } + + Runtime runtime_; + std::unique_ptr worker_ = std::make_unique(); +}; + +TEST_F(FdwicSharedSubmitWiringTest, KernelWinnerCommitsWholeTaskAndBuildsItsSlot) { + const Tensor input = make_existing_tensor(0x100000); + const Tensor inout = make_existing_tensor(0x200000); + const Tensor output = make_existing_tensor(0x300000); + const SharedTensorMapValue seed_entries[] = { + dist_shared_tensor_map_make_value(input, 0), + dist_shared_tensor_map_make_value(inout, 0), + }; + publish_seed_task(seed_entries, 2, /*task=*/0, kHDefault); + worker_->local_index = 1; + + L0TaskArgs args; + args.add_input(input); + args.add_inout(inout); + args.add_output(output); + MixedKernels mixed; + mixed.aiv0_kernel_id = 7; + + std::vector private_map_before(sizeof(worker_->map)); + std::memcpy(private_map_before.data(), &worker_->map, private_map_before.size()); + + const DistCompeteFirstTicket ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(ticket.task_id, 1); + ASSERT_EQ(ticket.won, 1); + (void)dist_submit_compete_first_finish(nullptr, mixed, ticket, args); + + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.shared_tensor_map.committed_tasks.v, 2); + bool protocol_ok = false; + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, input, 2, kHDefault, protocol_ok), 0); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, inout, 2, kHDefault, protocol_ok), 1); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, output, 2, kHDefault, protocol_ok), 1); + EXPECT_TRUE(protocol_ok); + + ASSERT_EQ(worker_->occupied_count, 1); + EXPECT_TRUE(worker_->slots[0].built); + EXPECT_EQ(worker_->slots[0].task_id, 1); + ASSERT_EQ(worker_->slots[0].fanin_count, 1); + EXPECT_EQ(worker_->slots[0].fanin[0], 0); + EXPECT_EQ(std::memcmp(private_map_before.data(), &worker_->map, private_map_before.size()), 0); +} + +TEST_F(FdwicSharedSubmitWiringTest, PaG2LatestInoutWriterFeedsFinalUp) { + constexpr int32_t kUpKernelId = 15; + Tensor mi_update = make_owned_tensor(0x510000, /*owner=*/0); + Tensor li_update = make_owned_tensor(0x520000, /*owner=*/0); + Tensor oi = make_owned_tensor(0x530000, /*owner=*/0); + Tensor out_view = make_existing_tensor(0x540000); + out_view.manual_dep = true; + + const Tensor group0_mi = make_owned_tensor(0x610000, /*owner=*/2); + const Tensor group0_li = make_owned_tensor(0x620000, /*owner=*/2); + const Tensor group0_oi_new = make_owned_tensor(0x630000, /*owner=*/3); + const Tensor group1_mi = make_owned_tensor(0x710000, /*owner=*/6); + const Tensor group1_li = make_owned_tensor(0x720000, /*owner=*/6); + const Tensor group1_oi_new = make_owned_tensor(0x730000, /*owner=*/7); + + // Real PA G2 starts with Alloc/QK/SF/PV. Fresh runtime outputs carry their + // creator IDs but do not enter the ordinary-region TensorMap. Task 1 gets + // one impossible poison entry for out_view only in this test: a wrong + // manual_dep lookup would add producer 1 to UP fan-in, while a wrong + // register would replace its latest producer with task 4 or 8. + publish_seed_task(nullptr, 0, /*task=*/0, kHDefault); + const SharedTensorMapValue manual_poison = dist_shared_tensor_map_make_value(out_view, /*producer=*/1); + publish_seed_task(&manual_poison, 1, /*task=*/1, kHDefault); + for (int32_t task = 2; task <= 3; ++task) { + publish_seed_task(nullptr, 0, task, kHDefault); + } + worker_->local_index = 4; + + L0TaskArgs up0_args; + up0_args.add_input(group0_mi, group0_li, group0_oi_new); + up0_args.add_inout(mi_update, li_update, oi, out_view); + MixedKernels mixed; + mixed.aiv0_kernel_id = kUpKernelId; + + std::vector private_map_before(sizeof(worker_->map)); + std::memcpy(private_map_before.data(), &worker_->map, private_map_before.size()); + + const DistCompeteFirstTicket up0_ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(up0_ticket.task_id, 4); + ASSERT_EQ(up0_ticket.won, 1); + (void)dist_submit_compete_first_finish(nullptr, mixed, up0_ticket, up0_args); + + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_NONE); + EXPECT_EQ(g_dist.shared_tensor_map.committed_tasks.v, 5); + const RingSlot *up0_slot = find_built_slot(*worker_, 4); + ASSERT_NE(up0_slot, nullptr); + EXPECT_EQ(up0_slot->fanin_count, 3); + EXPECT_EQ(fanin_occurrences(*up0_slot, 0), 1); + EXPECT_EQ(fanin_occurrences(*up0_slot, 1), 0); + EXPECT_EQ(fanin_occurrences(*up0_slot, 2), 1); + EXPECT_EQ(fanin_occurrences(*up0_slot, 3), 1); + + bool protocol_ok = false; + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, mi_update, 5, kHDefault, protocol_ok), 4); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, li_update, 5, kHDefault, protocol_ok), 4); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, oi, 5, kHDefault, protocol_ok), 4); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, out_view, 5, kHDefault, protocol_ok), 1); + EXPECT_TRUE(protocol_ok); + + // QK/SF/PV of group 1 likewise advance the exact turn without publishing + // ordinary regions. The final UP must observe task 4 as the latest writer + // of all three accumulators before it publishes task 8. + for (int32_t task = 5; task <= 7; ++task) { + publish_seed_task(nullptr, 0, task, kHDefault); + } + worker_->local_index = 8; + + L0TaskArgs up1_args; + up1_args.add_input(group1_mi, group1_li, group1_oi_new); + up1_args.add_inout(mi_update, li_update, oi, out_view); + const DistCompeteFirstTicket up1_ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(up1_ticket.task_id, 8); + ASSERT_EQ(up1_ticket.won, 1); + (void)dist_submit_compete_first_finish(nullptr, mixed, up1_ticket, up1_args); + + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_NONE); + EXPECT_EQ(g_dist.shared_tensor_map.committed_tasks.v, 9); + const RingSlot *up1_slot = find_built_slot(*worker_, 8); + ASSERT_NE(up1_slot, nullptr); + EXPECT_EQ(up1_slot->fanin_count, 4); + EXPECT_EQ(fanin_occurrences(*up1_slot, 0), 1); + EXPECT_EQ(fanin_occurrences(*up1_slot, 1), 0); + EXPECT_EQ(fanin_occurrences(*up1_slot, 4), 1); + EXPECT_EQ(fanin_occurrences(*up1_slot, 6), 1); + EXPECT_EQ(fanin_occurrences(*up1_slot, 7), 1); + + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, mi_update, 9, kHDefault, protocol_ok), 8); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, li_update, 9, kHDefault, protocol_ok), 8); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, oi, 9, kHDefault, protocol_ok), 8); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, out_view, 9, kHDefault, protocol_ok), 1); + EXPECT_TRUE(protocol_ok); + + EXPECT_EQ(worker_->occupied_count, 2); + EXPECT_EQ(std::memcmp(private_map_before.data(), &worker_->map, private_map_before.size()), 0); +} + +TEST_F(FdwicSharedSubmitWiringTest, ZeroEntryKernelWinnerAdvancesCommitAndBuilds) { + L0TaskArgs args; + MixedKernels mixed; + mixed.aiv0_kernel_id = 8; + const std::vector map_before = snapshot_shared_tensor_map(); + + const DistCompeteFirstTicket ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(ticket.task_id, 0); + ASSERT_EQ(ticket.won, 1); + (void)dist_submit_compete_first_finish(nullptr, mixed, ticket, args); + + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.shared_tensor_map.committed_tasks.v, 1); + EXPECT_EQ( + std::memcmp( + map_before.data() + sizeof(g_dist.shared_tensor_map.committed_tasks.v), + reinterpret_cast(&g_dist.shared_tensor_map) + + sizeof(g_dist.shared_tensor_map.committed_tasks.v), + map_before.size() - sizeof(g_dist.shared_tensor_map.committed_tasks.v) + ), + 0 + ); + ASSERT_EQ(worker_->occupied_count, 1); + EXPECT_TRUE(worker_->slots[0].built); + EXPECT_EQ(worker_->slots[0].task_id, 0); + EXPECT_EQ(worker_->slots[0].fanin_count, 0); +} + +TEST_F(FdwicSharedSubmitWiringTest, ZeroEntryJointWinnerCommitsAndPublishesJointWork) { + reset_worker(CoreType::AIC, LANE_AIC); + L0TaskArgs args; + MixedKernels mixed; + mixed.aic_kernel_id = 12; + mixed.aiv0_kernel_id = 13; + + const DistCompeteFirstTicket ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(ticket.task_id, 0); + ASSERT_EQ(ticket.won, 1); + ASSERT_EQ(ticket.joint, 1); + ASSERT_EQ(ticket.joint_init, 1); + (void)dist_submit_compete_first_finish(nullptr, mixed, ticket, args); + + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.shared_tensor_map.committed_tasks.v, 1); + EXPECT_EQ(g_dist.blocks[0].any_pub, 1); + int32_t published = 0; + for (int32_t index = 0; index < kPrivateSlots; ++index) { + if (g_dist.blocks[0].slots[index].state.v == kWonStatePublished) { + ++published; + } + } + EXPECT_EQ(published, 1); + ASSERT_EQ(worker_->occupied_count, 1); + EXPECT_TRUE(worker_->slots[0].built); + EXPECT_TRUE(worker_->slots[0].is_multicore); + EXPECT_EQ(worker_->slots[0].task_id, 0); +} + +TEST_F(FdwicSharedSubmitWiringTest, AllocWinnerCommitsAnEmptyMapTransactionBeforeCompletion) { + L0TaskArgs args; + const std::vector map_before = snapshot_shared_tensor_map(); + + const DistCompeteFirstTicket ticket = dist_alloc_compete_first_begin(nullptr); + ASSERT_EQ(ticket.task_id, 0); + ASSERT_EQ(ticket.won, 1); + (void)dist_alloc_compete_first_finish(nullptr, ticket, args); + + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.shared_tensor_map.committed_tasks.v, 1); + EXPECT_EQ(task_cell(0).flag, 1); + EXPECT_EQ(g_dist.frontier, 0); + EXPECT_EQ( + std::memcmp( + map_before.data() + sizeof(g_dist.shared_tensor_map.committed_tasks.v), + reinterpret_cast(&g_dist.shared_tensor_map) + + sizeof(g_dist.shared_tensor_map.committed_tasks.v), + map_before.size() - sizeof(g_dist.shared_tensor_map.committed_tasks.v) + ), + 0 + ); +} + +TEST_F(FdwicSharedSubmitWiringTest, LegacyKernelUsesTheSameSharedTransactionBoundary) { + const Tensor output = make_existing_tensor(0x380000); + L0TaskArgs args; + args.add_output(output); + MixedKernels mixed; + mixed.aiv0_kernel_id = 6; + + (void)dist_submit_impl(nullptr, mixed, args); + + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.shared_tensor_map.committed_tasks.v, 1); + bool protocol_ok = false; + EXPECT_EQ(dist_shared_tensor_map_lookup_tensor(g_dist.shared_tensor_map, output, 1, kHDefault, protocol_ok), 0); + EXPECT_TRUE(protocol_ok); + ASSERT_EQ(worker_->occupied_count, 1); + EXPECT_TRUE(worker_->slots[0].built); + EXPECT_EQ(worker_->slots[0].task_id, 0); +} + +TEST_F(FdwicSharedSubmitWiringTest, LegacyAllocUsesTheSameEmptyTransactionBoundary) { + L0TaskArgs args; + + (void)dist_alloc_tensors(nullptr, args); + + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.shared_tensor_map.committed_tasks.v, 1); + EXPECT_EQ(task_cell(0).flag, 1); + EXPECT_EQ(task_cell(0).vend, 0); + EXPECT_EQ(g_dist.frontier, 0); +} + +TEST_F(FdwicSharedSubmitWiringTest, KernelLoserDoesNotReadOrWriteEitherTensorMap) { + const Tensor input = make_existing_tensor(0x400000); + const Tensor output = make_existing_tensor(0x400000); + L0TaskArgs args; + args.add_input(input); + args.add_output(output); + MixedKernels mixed; + mixed.aiv0_kernel_id = 9; + g_dist.vector_cursor[0].v = 0; + g_dist.H = 0; + + worker_->map.alive_floor = -1; + const uint32_t input_bucket = dist_tensor_map_hash(input.buffer.addr); + worker_->map.bucket_heads[input_bucket] = 0; + worker_->map.bucket_tails[input_bucket] = 1; + MapEntry &stale_entry = worker_->map.entries[dist_private_tensor_map_slot_index(input_bucket, 0)]; + stale_entry.buf_addr = input.buffer.addr; + stale_entry.lo = 0; + stale_entry.hi = sizeof(float); + stale_entry.producer = -2; + g_dist.shared_tensor_map.buckets[input_bucket].tail.v = 1; + std::vector private_map_before(sizeof(worker_->map)); + std::memcpy(private_map_before.data(), &worker_->map, private_map_before.size()); + const std::vector shared_map_before = snapshot_shared_tensor_map(); + + const DistCompeteFirstTicket ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(ticket.task_id, 0); + ASSERT_EQ(ticket.won, 0); + (void)dist_submit_compete_first_finish(nullptr, mixed, ticket, args); + + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(std::memcmp(shared_map_before.data(), &g_dist.shared_tensor_map, shared_map_before.size()), 0); + EXPECT_EQ(std::memcmp(private_map_before.data(), &worker_->map, private_map_before.size()), 0); + EXPECT_EQ(worker_->occupied_count, 0); +} + +TEST_F(FdwicSharedSubmitWiringTest, GenericScalarLookupFailsClosedWithoutReadingEitherMap) { + uint32_t scalar = 0x12345678U; + const Tensor input = make_existing_tensor(reinterpret_cast(&scalar)); + const SharedTensorMapValue entry = dist_shared_tensor_map_make_value(input, 0); + publish_seed_task(&entry, 1, /*task=*/0, kHDefault); + worker_->local_index = 1; + task_cell(0).flag = 1; + worker_->map.alive_floor = 17; + worker_->map.bucket_heads[0] = 23; + worker_->map.bucket_tails[0] = 23; + const std::vector shared_map_before = snapshot_shared_tensor_map(); + std::vector private_map_before(sizeof(worker_->map)); + std::memcpy(private_map_before.data(), &worker_->map, private_map_before.size()); + + const uint32_t indices[] = {0}; + EXPECT_EQ(dist_get_tensor_data_impl(nullptr, input, /*ndims=*/1, indices), 0); + + EXPECT_EQ(g_dist.fatal, 1); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_TENSORMAP_PROTOCOL); + EXPECT_EQ(scalar, 0x12345678U); + EXPECT_EQ(std::memcmp(shared_map_before.data(), &g_dist.shared_tensor_map, shared_map_before.size()), 0); + EXPECT_EQ(std::memcmp(private_map_before.data(), &worker_->map, private_map_before.size()), 0); + + g_dist.fatal = 0; + g_dist.error_code = PTO2_ERROR_NONE; + dist_set_tensor_data_impl(nullptr, input, /*ndims=*/1, indices, 0xA5A5A5A5U); + EXPECT_EQ(g_dist.fatal, 1); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_TENSORMAP_PROTOCOL); + EXPECT_EQ(scalar, 0x12345678U); +} + +TEST_F(FdwicSharedSubmitWiringTest, WinnerLookupProtocolFailurePrecedesPublishAndBuild) { + const Tensor input = make_existing_tensor(0x490000); + const Tensor output = make_existing_tensor(0x4A0000); + const uint32_t corrupt_bucket = dist_tensor_map_hash(input.buffer.addr); + g_dist.shared_tensor_map.buckets[corrupt_bucket].tail.v = 1; + const std::vector map_before = snapshot_shared_tensor_map(); + + L0TaskArgs args; + args.add_input(input); + args.add_output(output); + MixedKernels mixed; + mixed.aiv0_kernel_id = 10; + + const DistCompeteFirstTicket ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(ticket.task_id, 0); + ASSERT_EQ(ticket.won, 1); + (void)dist_submit_compete_first_finish(nullptr, mixed, ticket, args); + + EXPECT_EQ(g_dist.fatal, 1); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_TENSORMAP_PROTOCOL); + EXPECT_EQ(worker_->local_index, kFlagCap); + EXPECT_EQ(std::memcmp(map_before.data(), &g_dist.shared_tensor_map, map_before.size()), 0); + EXPECT_EQ(worker_->occupied_count, 0); + EXPECT_EQ(task_cell(0).flag, 0); + EXPECT_EQ(task_cell(0).vend, 0); + EXPECT_EQ(g_dist.frontier, -1); +} + +TEST_F(FdwicSharedSubmitWiringTest, CommitAheadOfWinnerIsAProtocolFailureBeforeBuild) { + g_dist.shared_tensor_map.committed_tasks.v = 1; + L0TaskArgs args; + MixedKernels mixed; + mixed.aiv0_kernel_id = 11; + + const DistCompeteFirstTicket ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(ticket.task_id, 0); + ASSERT_EQ(ticket.won, 1); + (void)dist_submit_compete_first_finish(nullptr, mixed, ticket, args); + + EXPECT_EQ(g_dist.shared_tensor_map.committed_tasks.v, 1); + EXPECT_EQ(worker_->occupied_count, 0); + EXPECT_EQ(worker_->local_index, kFlagCap); + EXPECT_EQ(g_dist.fatal, 1); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_TENSORMAP_PROTOCOL); + EXPECT_EQ(task_cell(0).flag, 0); + EXPECT_EQ(task_cell(0).vend, 0); + EXPECT_EQ(g_dist.frontier, -1); +} + +TEST_F(FdwicSharedSubmitWiringTest, AllocCommitAheadSuppressesImmediateCompletion) { + g_dist.shared_tensor_map.committed_tasks.v = 1; + L0TaskArgs args; + + const DistCompeteFirstTicket ticket = dist_alloc_compete_first_begin(nullptr); + ASSERT_EQ(ticket.task_id, 0); + ASSERT_EQ(ticket.won, 1); + (void)dist_alloc_compete_first_finish(nullptr, ticket, args); + + EXPECT_EQ(g_dist.shared_tensor_map.committed_tasks.v, 1); + EXPECT_EQ(worker_->local_index, kFlagCap); + EXPECT_EQ(g_dist.fatal, 1); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_TENSORMAP_PROTOCOL); + EXPECT_EQ(task_cell(0).flag, 0); + EXPECT_EQ(task_cell(0).vend, 0); + EXPECT_EQ(g_dist.frontier, -1); +} + +TEST_F(FdwicSharedSubmitWiringTest, JointWinnerCapacityFailurePrecedesWonDepositAndBuild) { + reset_worker(CoreType::AIC, LANE_AIC); + const Tensor output = make_existing_tensor(0x500000); + fill_one_bucket(output, kMapBucketCapacity); + g_dist.H = kFlagCap - 1; + worker_->local_index = kMapBucketCapacity; + + L0TaskArgs args; + args.add_output(output); + MixedKernels mixed; + mixed.aic_kernel_id = 13; + mixed.aiv0_kernel_id = 14; + + const DistCompeteFirstTicket ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(ticket.task_id, kMapBucketCapacity); + ASSERT_EQ(ticket.won, 1); + ASSERT_EQ(ticket.joint, 1); + ASSERT_EQ(ticket.joint_init, 1); + (void)dist_submit_compete_first_finish(nullptr, mixed, ticket, args); + + EXPECT_EQ(g_dist.shared_tensor_map.committed_tasks.v, kMapBucketCapacity); + EXPECT_EQ(g_dist.blocks[0].any_pub, 0); + for (int32_t index = 0; index < kPrivateSlots; ++index) { + EXPECT_EQ(g_dist.blocks[0].slots[index].state.v, kWonStateFree); + } + EXPECT_EQ(worker_->occupied_count, 0); + EXPECT_EQ(worker_->local_index, kFlagCap); + EXPECT_EQ(g_dist.fatal, 1); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_TENSORMAP_CAPACITY); + EXPECT_EQ(task_cell(kMapBucketCapacity).flag, 0); + EXPECT_EQ(task_cell(kMapBucketCapacity).vend, 0); + EXPECT_EQ(g_dist.frontier, -1); +} + +} // namespace diff --git a/tests/ut/cpp/a5/test_fdwic_shared_tensor_map_ring.cpp b/tests/ut/cpp/a5/test_fdwic_shared_tensor_map_ring.cpp new file mode 100644 index 0000000000..c0555ac07f --- /dev/null +++ b/tests/ut/cpp/a5/test_fdwic_shared_tensor_map_ring.cpp @@ -0,0 +1,1161 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "inner_kernel.h" +// The header must compile without dependencies supplied indirectly by AICPU +// initialization headers. +#include "dist_engine/aicore/shared_tensor_map.h" +#include "dist_engine/aicpu/shared_tensor_map_init.h" + +namespace { + +static_assert(PTO_FDWIC_SHARED_MAP == 1, "shared TensorMap ring tests require the shared artifact identity"); + +enum class EventKind : uint8_t { + Load, + CompareExchange, + Invalidate, + Flush, +}; + +struct Event { + EventKind kind; + const void *address; + int64_t expected; + int64_t desired; + int64_t observed; +}; + +const void *control_address(volatile int64_t *address) { return const_cast(address); } + +struct RecordingOps { + inline static std::vector events{}; + inline static bool record = false; + inline static const void *mutate_payload = nullptr; + inline static volatile int64_t *mutate_sequence = nullptr; + inline static int64_t mutate_sequence_value = 0; + inline static bool mutate_once = false; + inline static volatile int64_t *mutate_cas_address = nullptr; + inline static int64_t mutate_cas_expected = 0; + inline static int64_t mutate_cas_desired = 0; + inline static int64_t mutate_cas_value = 0; + inline static bool mutate_cas_once = false; + + static int64_t Load(volatile int64_t *address) { + const int64_t value = __atomic_fetch_add(address, int64_t{0}, __ATOMIC_ACQUIRE); + if (record) { + events.push_back({EventKind::Load, control_address(address), 0, 0, value}); + } + return value; + } + + static int64_t CompareExchange(volatile int64_t *address, int64_t expected, int64_t desired) { + // Deterministically inject an illegal concurrent writer immediately + // before the tested CAS linearization point. Exact-turn forbids this + // race; the injection only proves a failed CAS preserves the observed + // control word. + if (mutate_cas_once && address == mutate_cas_address && expected == mutate_cas_expected && + desired == mutate_cas_desired) { + __atomic_store_n(address, mutate_cas_value, __ATOMIC_RELEASE); + mutate_cas_once = false; + } + int64_t observed = expected; + (void)__atomic_compare_exchange_n( + address, &observed, desired, /*weak=*/false, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE + ); + if (record) { + events.push_back({EventKind::CompareExchange, control_address(address), expected, desired, observed}); + } + return observed; + } + + static void InvalidateRegion(const void *address, uint64_t bytes) { + if (record) { + events.push_back({EventKind::Invalidate, address, 0, static_cast(bytes), 0}); + } + // Reuse the slot for the next lap between the first sequence check and + // the payload snapshot, proving that the second check is required. + if (mutate_once && address == mutate_payload && mutate_sequence != nullptr) { + __atomic_store_n(mutate_sequence, mutate_sequence_value, __ATOMIC_RELEASE); + mutate_once = false; + } + } + + static void FlushRegion(void *address, uint64_t bytes) { + if (record) { + events.push_back({EventKind::Flush, address, 0, static_cast(bytes), 0}); + } + __atomic_thread_fence(__ATOMIC_SEQ_CST); + } + + static void ResetEvents(bool enable = true) { + events.clear(); + record = enable; + mutate_payload = nullptr; + mutate_sequence = nullptr; + mutate_sequence_value = 0; + mutate_once = false; + mutate_cas_address = nullptr; + mutate_cas_expected = 0; + mutate_cas_desired = 0; + mutate_cas_value = 0; + mutate_cas_once = false; + } + + static void MutateBeforeCas(volatile int64_t *address, int64_t expected, int64_t desired, int64_t competing_value) { + if (competing_value == expected || competing_value == desired) { + throw std::logic_error("CAS competing value must differ from expected and desired"); + } + mutate_cas_address = address; + mutate_cas_expected = expected; + mutate_cas_desired = desired; + mutate_cas_value = competing_value; + mutate_cas_once = true; + } +}; + +std::unique_ptr make_empty_map() { + auto map = std::make_unique(); + std::memset(map.get(), 0xa5, sizeof(*map)); + dist_shared_tensor_map_reset(*map); + RecordingOps::ResetEvents(false); + return map; +} + +SharedTensorMapValue make_region(uint64_t address, uint64_t lo, uint64_t hi, int32_t producer) { + return {address, lo, hi, producer, 0}; +} + +uint64_t find_address_in_bucket(uint32_t target_bucket, uint64_t seed) { + constexpr uint64_t kSearchLimit = 1ULL << 22; + for (uint64_t step = 0; step < kSearchLimit; ++step) { + const uint64_t candidate = seed + step * 64; + if (dist_tensor_map_hash(candidate) == target_bucket) { + return candidate; + } + } + throw std::logic_error("failed to find shared TensorMap address in requested bucket"); +} + +uint64_t find_address_outside_bucket(uint32_t excluded_bucket, uint64_t seed) { + if constexpr (kMapBuckets == 1) { + return seed; + } + uint64_t candidate = seed; + while (dist_tensor_map_hash(candidate) == excluded_bucket) { + candidate += 64; + } + return candidate; +} + +enum class DriverResult : uint8_t { + Pending, + Committed, + CapacityBlocked, + ProtocolError, + PartialPublish, +}; + +DriverResult try_commit( + SharedTensorMapState &map, int32_t task_id, const SharedTensorMapValue *entries, uint32_t count, + int32_t history = INT32_MAX +) { + if (task_id < 0 || task_id >= kFlagCap) { + return DriverResult::ProtocolError; + } + const int64_t committed = RecordingOps::Load(&map.committed_tasks.v); + if (committed < task_id) { + return DriverResult::Pending; + } + if (committed > task_id) { + return DriverResult::ProtocolError; + } + const DistSharedTensorMapTaskPublishResult result = + dist_shared_tensor_map_publish_task_impl(map, entries, count, task_id, history); + if (result == DistSharedTensorMapTaskPublishResult::Committed) { + return DriverResult::Committed; + } + if (result == DistSharedTensorMapTaskPublishResult::CapacityBlocked) { + return DriverResult::CapacityBlocked; + } + if (result == DistSharedTensorMapTaskPublishResult::PartialPublish) { + return DriverResult::PartialPublish; + } + return DriverResult::ProtocolError; +} + +DriverResult try_commit( + SharedTensorMapState &map, int32_t task_id, const std::vector &entries, + int32_t history = INT32_MAX +) { + return try_commit(map, task_id, entries.data(), static_cast(entries.size()), history); +} + +size_t find_event(EventKind kind, const void *address, size_t begin, bool check_desired = false, int64_t desired = 0) { + for (size_t index = begin; index < RecordingOps::events.size(); ++index) { + const Event &event = RecordingOps::events[index]; + if (event.kind == kind && event.address == address && (!check_desired || event.desired == desired)) { + return index; + } + } + return RecordingOps::events.size(); +} + +void expect_recorded_cas(const void *address, int64_t expected, int64_t desired, int64_t observed) { + const size_t index = find_event(EventKind::CompareExchange, address, 0, true, desired); + ASSERT_LT(index, RecordingOps::events.size()); + EXPECT_EQ(RecordingOps::events[index].expected, expected); + EXPECT_EQ(RecordingOps::events[index].desired, desired); + EXPECT_EQ(RecordingOps::events[index].observed, observed); +} + +TEST(FdwicSharedTensorMapRing, PhysicalBoundariesValueConversionAndZeroEntryCommit) { + auto map = make_empty_map(); + EXPECT_EQ(dist_shared_tensor_map_slot_index(0, 0), 0U); + EXPECT_EQ( + dist_shared_tensor_map_slot_index(kMapBuckets - 1, kMapBucketCapacity - 1), static_cast(kMapCap - 1) + ); + EXPECT_EQ( + dist_shared_tensor_map_slot_index(kMapBuckets - 1, kMapBucketCapacity), (kMapBuckets - 1) * kMapBucketCapacity + ); + + Tensor tensor{}; + tensor.buffer = {0x12340000, 1ULL << 20}; + tensor.start_offset = 3; + tensor.ndims = 1; + tensor.dtype = DataType::FLOAT32; + tensor.is_contiguous = true; + tensor.shapes[0] = 7; + tensor.extent_elem_cache = 7; + const SharedTensorMapValue value = dist_shared_tensor_map_make_value(tensor, 9); + EXPECT_EQ(value.buf_addr, tensor.buffer.addr); + EXPECT_EQ(value.lo, 12U); + EXPECT_EQ(value.hi, 40U); + EXPECT_EQ(value.producer, 9); + EXPECT_EQ(value.reserved, 0U); + + const std::vector empty; + EXPECT_EQ(try_commit(*map, 0, empty), DriverResult::Committed); + EXPECT_EQ(try_commit(*map, 1, empty), DriverResult::Committed); + EXPECT_EQ(map->committed_tasks.v, 2); + EXPECT_EQ(map->reclaim_upto.v, -1); + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + EXPECT_EQ(map->buckets[bucket].head.v, 0); + EXPECT_EQ(map->buckets[bucket].tail.v, 0); + } + + EXPECT_FALSE(dist_shared_tensor_map_publish_commit_impl(*map, 1)); + EXPECT_EQ(map->committed_tasks.v, 2); + map->committed_tasks.v = 0; + EXPECT_FALSE(dist_shared_tensor_map_publish_commit_impl(*map, 1)); + EXPECT_EQ(map->committed_tasks.v, 0); +} + +TEST(FdwicSharedTensorMapRing, PublicationOrderReaderOrderAndDoubleSequenceRejectAba) { + auto map = make_empty_map(); + const SharedTensorMapValue entry = make_region(0x100000000ULL, 0, 64, 0); + const uint32_t bucket = dist_tensor_map_hash(entry.buf_addr); + SharedTensorMapSlot &slot = map->slots[dist_shared_tensor_map_slot_index(bucket, 0)]; + + RecordingOps::ResetEvents(); + EXPECT_EQ(try_commit(*map, 0, {entry}), DriverResult::Committed); + + const void *sequence_address = control_address(&slot.sequence.v); + const void *payload_address = &slot.payload; + const void *tail_address = control_address(&map->buckets[bucket].tail.v); + const void *commit_address = control_address(&map->committed_tasks.v); + const size_t sequence_claim = + find_event(EventKind::CompareExchange, sequence_address, 0, true, kSharedTensorMapWritingSequence); + const size_t payload_invalidate = find_event(EventKind::Invalidate, payload_address, sequence_claim + 1); + const size_t payload_flush = find_event(EventKind::Flush, payload_address, payload_invalidate + 1); + const size_t sequence_publish = + find_event(EventKind::CompareExchange, sequence_address, payload_flush + 1, true, 0); + const size_t tail_publish = find_event(EventKind::CompareExchange, tail_address, sequence_publish + 1, true, 1); + const size_t task_commit = find_event(EventKind::CompareExchange, commit_address, tail_publish + 1, true, 1); + ASSERT_LT(sequence_claim, payload_invalidate); + ASSERT_LT(payload_invalidate, payload_flush); + ASSERT_LT(payload_flush, sequence_publish); + ASSERT_LT(sequence_publish, tail_publish); + ASSERT_LT(tail_publish, task_commit); + EXPECT_EQ(RecordingOps::events[sequence_claim].expected, kSharedTensorMapInvalidSequence); + EXPECT_EQ(RecordingOps::events[sequence_publish].expected, kSharedTensorMapWritingSequence); + EXPECT_EQ(RecordingOps::events[tail_publish].expected, 0); + EXPECT_EQ(RecordingOps::events[task_commit].expected, 0); + + RecordingOps::ResetEvents(); + SharedTensorMapValue snapshot{}; + ASSERT_TRUE(dist_shared_tensor_map_read_slot_impl(*map, bucket, 0, snapshot)); + EXPECT_EQ(snapshot.buf_addr, entry.buf_addr); + EXPECT_EQ(snapshot.lo, entry.lo); + EXPECT_EQ(snapshot.hi, entry.hi); + EXPECT_EQ(snapshot.producer, entry.producer); + const size_t first_load = find_event(EventKind::Load, sequence_address, 0); + const size_t read_invalidate = find_event(EventKind::Invalidate, payload_address, first_load + 1); + const size_t second_load = find_event(EventKind::Load, sequence_address, read_invalidate + 1); + ASSERT_LT(first_load, read_invalidate); + ASSERT_LT(read_invalidate, second_load); + + RecordingOps::ResetEvents(); + RecordingOps::mutate_payload = payload_address; + RecordingOps::mutate_sequence = &slot.sequence.v; + RecordingOps::mutate_sequence_value = static_cast(kMapBucketCapacity); + RecordingOps::mutate_once = true; + SharedTensorMapValue raced{}; + EXPECT_FALSE(dist_shared_tensor_map_read_slot_impl(*map, bucket, 0, raced)); + EXPECT_FALSE(RecordingOps::mutate_once); +} + +TEST(FdwicSharedTensorMapRing, ControlCasFailuresDoNotOverwriteCompetingValues) { + { + auto map = make_empty_map(); + const SharedTensorMapValue entry = make_region(0x180000000ULL, 0, 64, 0); + const uint32_t bucket = dist_tensor_map_hash(entry.buf_addr); + ASSERT_EQ(try_commit(*map, 0, {entry}), DriverResult::Committed); + const int64_t tail_before = map->buckets[bucket].tail.v; + const int64_t commit_before = map->committed_tasks.v; + const SharedTensorMapSlot slot_before = map->slots[dist_shared_tensor_map_slot_index(bucket, 0)]; + + RecordingOps::ResetEvents(); + RecordingOps::MutateBeforeCas(&map->buckets[bucket].head.v, 0, 1, 2); + EXPECT_FALSE(dist_shared_tensor_map_retire_bucket_impl(*map, bucket, 0)); + EXPECT_FALSE(RecordingOps::mutate_cas_once); + EXPECT_EQ(map->buckets[bucket].head.v, 2); + expect_recorded_cas(control_address(&map->buckets[bucket].head.v), 0, 1, 2); + EXPECT_EQ(map->buckets[bucket].tail.v, tail_before); + EXPECT_EQ(map->committed_tasks.v, commit_before); + EXPECT_EQ( + std::memcmp(&slot_before, &map->slots[dist_shared_tensor_map_slot_index(bucket, 0)], sizeof(slot_before)), 0 + ); + } + + { + auto map = make_empty_map(); + map->committed_tasks.v = 65; + RecordingOps::ResetEvents(); + RecordingOps::MutateBeforeCas(&map->reclaim_upto.v, -1, 0, 7); + int64_t reclaim_upto = -2; + EXPECT_FALSE(dist_shared_tensor_map_refresh_reclaim_impl(*map, 65, 64, reclaim_upto)); + EXPECT_FALSE(RecordingOps::mutate_cas_once); + EXPECT_EQ(reclaim_upto, -2); + EXPECT_EQ(map->reclaim_upto.v, 7); + expect_recorded_cas(control_address(&map->reclaim_upto.v), -1, 0, 7); + EXPECT_EQ(map->committed_tasks.v, 65); + } + + { + auto map = make_empty_map(); + RecordingOps::ResetEvents(); + RecordingOps::MutateBeforeCas(&map->committed_tasks.v, 0, 1, 2); + EXPECT_FALSE(dist_shared_tensor_map_publish_commit_impl(*map, 0)); + EXPECT_FALSE(RecordingOps::mutate_cas_once); + EXPECT_EQ(map->committed_tasks.v, 2); + expect_recorded_cas(control_address(&map->committed_tasks.v), 0, 1, 2); + EXPECT_EQ(map->reclaim_upto.v, -1); + } +} + +TEST(FdwicSharedTensorMapRing, SlotAcquireCasFailurePublishesNothing) { + auto map = make_empty_map(); + const SharedTensorMapValue entry = make_region(0x1a0000000ULL, 0, 64, 0); + const uint32_t bucket = dist_tensor_map_hash(entry.buf_addr); + SharedTensorMapSlot &slot = map->slots[dist_shared_tensor_map_slot_index(bucket, 0)]; + int64_t reclaim_upto = -2; + uint64_t planned_cursor = UINT64_MAX; + ASSERT_TRUE(dist_shared_tensor_map_refresh_reclaim_impl(*map, 0, 64, reclaim_upto)); + ASSERT_EQ( + dist_shared_tensor_map_check_task_append_impl(*map, &entry, 1, 0, reclaim_upto, &planned_cursor), + DistSharedTensorMapAppendCheck::Ready + ); + ASSERT_EQ(planned_cursor, 0U); + const SharedTensorMapPayloadLine payload_before = slot.payload; + + RecordingOps::ResetEvents(); + RecordingOps::MutateBeforeCas( + &slot.sequence.v, kSharedTensorMapInvalidSequence, kSharedTensorMapWritingSequence, 77 + ); + EXPECT_EQ( + dist_shared_tensor_map_publish_prepared_task_impl(*map, &entry, &planned_cursor, 1, 0), + DistSharedTensorMapTaskPublishResult::ProtocolError + ); + EXPECT_FALSE(RecordingOps::mutate_cas_once); + EXPECT_EQ(slot.sequence.v, 77); + expect_recorded_cas( + control_address(&slot.sequence.v), kSharedTensorMapInvalidSequence, kSharedTensorMapWritingSequence, 77 + ); + EXPECT_EQ(std::memcmp(&payload_before, &slot.payload, sizeof(payload_before)), 0); + EXPECT_EQ(map->buckets[bucket].tail.v, 0); + EXPECT_EQ(map->committed_tasks.v, 0); + for (const Event &event : RecordingOps::events) { + EXPECT_NE(event.kind, EventKind::Invalidate); + EXPECT_NE(event.kind, EventKind::Flush); + } + + // If two illegal writers both passed stale preflight, the first moves the + // slot to WRITING and the second must still fail the ownership CAS before + // touching payload. + auto owned_map = make_empty_map(); + SharedTensorMapSlot &owned_slot = owned_map->slots[dist_shared_tensor_map_slot_index(bucket, 0)]; + owned_slot.sequence.v = kSharedTensorMapWritingSequence; + const SharedTensorMapPayloadLine owned_payload_before = owned_slot.payload; + EXPECT_EQ( + dist_shared_tensor_map_publish_prepared_entry_impl(*owned_map, entry, 0, 0), + DistSharedTensorMapEntryPublishResult::ProtocolError + ); + EXPECT_EQ(owned_slot.sequence.v, kSharedTensorMapWritingSequence); + EXPECT_EQ(std::memcmp(&owned_payload_before, &owned_slot.payload, sizeof(owned_payload_before)), 0); + EXPECT_EQ(owned_map->buckets[bucket].tail.v, 0); +} + +TEST(FdwicSharedTensorMapRing, PreparedTaskRejectsTailDriftInsteadOfPublishingAnUncheckedSlot) { + auto map = make_empty_map(); + const SharedTensorMapValue entry = make_region(0x1b0000000ULL, 0, 64, 0); + const uint32_t bucket = dist_tensor_map_hash(entry.buf_addr); + int64_t reclaim_upto = -2; + uint64_t planned_cursor = UINT64_MAX; + ASSERT_TRUE(dist_shared_tensor_map_refresh_reclaim_impl(*map, 0, 64, reclaim_upto)); + ASSERT_EQ( + dist_shared_tensor_map_check_task_append_impl(*map, &entry, 1, 0, reclaim_upto, &planned_cursor), + DistSharedTensorMapAppendCheck::Ready + ); + ASSERT_EQ(planned_cursor, 0U); + const SharedTensorMapSlot first_before = map->slots[dist_shared_tensor_map_slot_index(bucket, 0)]; + const SharedTensorMapSlot second_before = map->slots[dist_shared_tensor_map_slot_index(bucket, 1)]; + + // Model an illegal same-task writer advancing tail after preflight. The + // prepared publisher must not silently switch from planned cursor 0 to the + // unchecked cursor 1. + map->buckets[bucket].tail.v = 1; + EXPECT_EQ( + dist_shared_tensor_map_publish_prepared_task_impl(*map, &entry, &planned_cursor, 1, 0), + DistSharedTensorMapTaskPublishResult::ProtocolError + ); + EXPECT_EQ(map->buckets[bucket].tail.v, 1); + EXPECT_EQ(map->committed_tasks.v, 0); + EXPECT_EQ( + std::memcmp(&first_before, &map->slots[dist_shared_tensor_map_slot_index(bucket, 0)], sizeof(first_before)), 0 + ); + EXPECT_EQ( + std::memcmp(&second_before, &map->slots[dist_shared_tensor_map_slot_index(bucket, 1)], sizeof(second_before)), 0 + ); +} + +TEST(FdwicSharedTensorMapRing, PostAcquireCasFailuresPreserveTheObservedControlWord) { + const SharedTensorMapValue entry = make_region(0x1c0000000ULL, 16, 80, 0); + + { + auto map = make_empty_map(); + const uint32_t bucket = dist_tensor_map_hash(entry.buf_addr); + SharedTensorMapSlot &slot = map->slots[dist_shared_tensor_map_slot_index(bucket, 0)]; + int64_t reclaim_upto = -2; + uint64_t planned_cursor = UINT64_MAX; + ASSERT_TRUE(dist_shared_tensor_map_refresh_reclaim_impl(*map, 0, 64, reclaim_upto)); + ASSERT_EQ( + dist_shared_tensor_map_check_task_append_impl( + *map, &entry, 1, 0, reclaim_upto, &planned_cursor + ), + DistSharedTensorMapAppendCheck::Ready + ); + + RecordingOps::ResetEvents(); + RecordingOps::MutateBeforeCas(&slot.sequence.v, kSharedTensorMapWritingSequence, 0, 77); + EXPECT_EQ( + dist_shared_tensor_map_publish_prepared_task_impl(*map, &entry, &planned_cursor, 1, 0), + DistSharedTensorMapTaskPublishResult::PartialPublish + ); + EXPECT_FALSE(RecordingOps::mutate_cas_once); + EXPECT_EQ(slot.sequence.v, 77); + expect_recorded_cas(control_address(&slot.sequence.v), kSharedTensorMapWritingSequence, 0, 77); + EXPECT_EQ(slot.payload.value.buf_addr, entry.buf_addr); + EXPECT_EQ(slot.payload.value.lo, entry.lo); + EXPECT_EQ(slot.payload.value.hi, entry.hi); + EXPECT_EQ(slot.payload.value.producer, entry.producer); + EXPECT_EQ(map->buckets[bucket].tail.v, 0); + EXPECT_EQ(map->committed_tasks.v, 0); + EXPECT_LT(find_event(EventKind::Flush, &slot.payload, 0), RecordingOps::events.size()); + } + + { + auto map = make_empty_map(); + const uint32_t bucket = dist_tensor_map_hash(entry.buf_addr); + SharedTensorMapSlot &slot = map->slots[dist_shared_tensor_map_slot_index(bucket, 0)]; + int64_t reclaim_upto = -2; + uint64_t planned_cursor = UINT64_MAX; + ASSERT_TRUE(dist_shared_tensor_map_refresh_reclaim_impl(*map, 0, 64, reclaim_upto)); + ASSERT_EQ( + dist_shared_tensor_map_check_task_append_impl( + *map, &entry, 1, 0, reclaim_upto, &planned_cursor + ), + DistSharedTensorMapAppendCheck::Ready + ); + + RecordingOps::ResetEvents(); + RecordingOps::MutateBeforeCas(&map->buckets[bucket].tail.v, 0, 1, 7); + EXPECT_EQ( + dist_shared_tensor_map_publish_prepared_task_impl(*map, &entry, &planned_cursor, 1, 0), + DistSharedTensorMapTaskPublishResult::PartialPublish + ); + EXPECT_FALSE(RecordingOps::mutate_cas_once); + EXPECT_EQ(slot.sequence.v, 0); + EXPECT_EQ(slot.payload.value.producer, entry.producer); + EXPECT_EQ(map->buckets[bucket].tail.v, 7); + expect_recorded_cas(control_address(&map->buckets[bucket].tail.v), 0, 1, 7); + EXPECT_EQ(map->committed_tasks.v, 0); + } + + { + auto map = make_empty_map(); + const uint32_t bucket = dist_tensor_map_hash(entry.buf_addr); + int64_t reclaim_upto = -2; + uint64_t planned_cursor = UINT64_MAX; + ASSERT_TRUE(dist_shared_tensor_map_refresh_reclaim_impl(*map, 0, 64, reclaim_upto)); + ASSERT_EQ( + dist_shared_tensor_map_check_task_append_impl( + *map, &entry, 1, 0, reclaim_upto, &planned_cursor + ), + DistSharedTensorMapAppendCheck::Ready + ); + ASSERT_EQ( + dist_shared_tensor_map_publish_prepared_entry_impl(*map, entry, 0, planned_cursor), + DistSharedTensorMapEntryPublishResult::Published + ); + + RecordingOps::ResetEvents(); + RecordingOps::MutateBeforeCas(&map->committed_tasks.v, 0, 1, 2); + EXPECT_FALSE(dist_shared_tensor_map_publish_commit_impl(*map, 0)); + EXPECT_FALSE(RecordingOps::mutate_cas_once); + EXPECT_EQ(map->committed_tasks.v, 2); + expect_recorded_cas(control_address(&map->committed_tasks.v), 0, 1, 2); + EXPECT_EQ(map->buckets[bucket].tail.v, 1); + EXPECT_EQ(map->slots[dist_shared_tensor_map_slot_index(bucket, 0)].sequence.v, 0); + size_t commit_cas_count = 0; + for (const Event &event : RecordingOps::events) { + if (event.kind == EventKind::CompareExchange && event.address == control_address(&map->committed_tasks.v)) { + ++commit_cas_count; + } + } + EXPECT_EQ(commit_cas_count, 1U); + } +} + +TEST(FdwicSharedTensorMapRing, TaskPublishDistinguishesProtocolRejectFromPartialPublication) { + const SharedTensorMapValue first = make_region(0x1e0000000ULL, 0, 64, 0); + const SharedTensorMapValue second = make_region(first.buf_addr, 128, 192, 0); + const uint32_t bucket = dist_tensor_map_hash(first.buf_addr); + + { + auto map = make_empty_map(); + SharedTensorMapSlot &slot = map->slots[dist_shared_tensor_map_slot_index(bucket, 0)]; + const SharedTensorMapPayloadLine payload_before = slot.payload; + RecordingOps::ResetEvents(); + RecordingOps::MutateBeforeCas( + &slot.sequence.v, kSharedTensorMapInvalidSequence, kSharedTensorMapWritingSequence, 77 + ); + EXPECT_EQ( + dist_shared_tensor_map_publish_task_impl(*map, &first, 1, 0, 64), + DistSharedTensorMapTaskPublishResult::ProtocolError + ); + EXPECT_EQ(std::memcmp(&payload_before, &slot.payload, sizeof(payload_before)), 0); + EXPECT_EQ(slot.sequence.v, 77); + EXPECT_EQ(map->buckets[bucket].tail.v, 0); + EXPECT_EQ(map->committed_tasks.v, 0); + } + + { + auto map = make_empty_map(); + SharedTensorMapSlot &slot = map->slots[dist_shared_tensor_map_slot_index(bucket, 0)]; + RecordingOps::ResetEvents(); + RecordingOps::MutateBeforeCas(&slot.sequence.v, kSharedTensorMapWritingSequence, 0, 77); + EXPECT_EQ( + dist_shared_tensor_map_publish_task_impl(*map, &first, 1, 0, 64), + DistSharedTensorMapTaskPublishResult::PartialPublish + ); + EXPECT_EQ(slot.sequence.v, 77); + EXPECT_EQ(slot.payload.value.producer, 0); + EXPECT_EQ(map->buckets[bucket].tail.v, 0); + EXPECT_EQ(map->committed_tasks.v, 0); + } + + { + auto map = make_empty_map(); + SharedTensorMapSlot &slot = map->slots[dist_shared_tensor_map_slot_index(bucket, 0)]; + RecordingOps::ResetEvents(); + RecordingOps::MutateBeforeCas(&map->buckets[bucket].tail.v, 0, 1, 7); + EXPECT_EQ( + dist_shared_tensor_map_publish_task_impl(*map, &first, 1, 0, 64), + DistSharedTensorMapTaskPublishResult::PartialPublish + ); + EXPECT_EQ(slot.sequence.v, 0); + EXPECT_EQ(slot.payload.value.producer, 0); + EXPECT_EQ(map->buckets[bucket].tail.v, 7); + EXPECT_EQ(map->committed_tasks.v, 0); + } + + { + auto map = make_empty_map(); + RecordingOps::ResetEvents(); + RecordingOps::MutateBeforeCas(&map->committed_tasks.v, 0, 1, 2); + EXPECT_EQ( + dist_shared_tensor_map_publish_task_impl(*map, &first, 1, 0, 64), + DistSharedTensorMapTaskPublishResult::PartialPublish + ); + EXPECT_EQ(map->buckets[bucket].tail.v, 1); + EXPECT_EQ(map->slots[dist_shared_tensor_map_slot_index(bucket, 0)].sequence.v, 0); + EXPECT_EQ(map->committed_tasks.v, 2); + } + + { + auto map = make_empty_map(); + RecordingOps::ResetEvents(); + RecordingOps::MutateBeforeCas(&map->committed_tasks.v, 0, 1, 2); + EXPECT_EQ( + dist_shared_tensor_map_publish_task_impl(*map, nullptr, 0, 0, 64), + DistSharedTensorMapTaskPublishResult::ProtocolError + ); + EXPECT_EQ(map->committed_tasks.v, 2); + for (uint32_t index = 0; index < kMapBuckets; ++index) { + EXPECT_EQ(map->buckets[index].head.v, 0); + EXPECT_EQ(map->buckets[index].tail.v, 0); + } + } + + { + auto map = make_empty_map(); + const SharedTensorMapValue entries[] = {first, second}; + SharedTensorMapSlot &second_slot = map->slots[dist_shared_tensor_map_slot_index(bucket, 1)]; + const SharedTensorMapPayloadLine second_payload_before = second_slot.payload; + RecordingOps::ResetEvents(); + RecordingOps::MutateBeforeCas( + &second_slot.sequence.v, kSharedTensorMapInvalidSequence, kSharedTensorMapWritingSequence, 77 + ); + EXPECT_EQ( + dist_shared_tensor_map_publish_task_impl(*map, entries, 2, 0, 64), + DistSharedTensorMapTaskPublishResult::PartialPublish + ); + EXPECT_EQ(map->buckets[bucket].tail.v, 1); + EXPECT_EQ(map->slots[dist_shared_tensor_map_slot_index(bucket, 0)].sequence.v, 0); + EXPECT_EQ(map->slots[dist_shared_tensor_map_slot_index(bucket, 0)].payload.value.producer, 0); + EXPECT_EQ(second_slot.sequence.v, 77); + EXPECT_EQ(std::memcmp(&second_payload_before, &second_slot.payload, sizeof(second_payload_before)), 0); + EXPECT_EQ(map->committed_tasks.v, 0); + } +} + +TEST(FdwicSharedTensorMapRing, LookupUsesHalfOpenHistoryWindowAndMaximumProducer) { + auto map = make_empty_map(); + const uint64_t versioned_address = 0x200000000ULL; + const uint32_t versioned_bucket = dist_tensor_map_hash(versioned_address); + const uint64_t stale_address = find_address_outside_bucket(versioned_bucket, 0x210000000ULL); + const uint64_t lower_address = find_address_outside_bucket(dist_tensor_map_hash(stale_address), 0x220000000ULL); + const uint64_t future_address = find_address_outside_bucket(dist_tensor_map_hash(lower_address), 0x230000000ULL); + + EXPECT_EQ( + try_commit( + *map, 0, + { + make_region(versioned_address, 0, 64, 0), + make_region(versioned_address, 128, 192, 0), + make_region(stale_address, 0, 64, 0), + } + ), + DriverResult::Committed + ); + EXPECT_EQ( + try_commit( + *map, 1, + { + make_region(versioned_address, 0, 64, 1), + make_region(lower_address, 0, 64, 1), + } + ), + DriverResult::Committed + ); + EXPECT_EQ( + try_commit( + *map, 2, + { + make_region(versioned_address, 0, 64, 2), + make_region(future_address, 0, 64, 2), + } + ), + DriverResult::Committed + ); + + bool protocol_ok = false; + EXPECT_EQ( + dist_shared_tensor_map_lookup_region_impl( + *map, make_region(versioned_address, 16, 32, -1), 3, 2, protocol_ok + ), + 2 + ); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ( + dist_shared_tensor_map_lookup_region_impl( + *map, make_region(versioned_address, 16, 32, -1), 2, 2, protocol_ok + ), + 1 + ); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ( + dist_shared_tensor_map_lookup_region_impl( + *map, make_region(stale_address, 0, 32, -1), 3, 2, protocol_ok + ), + -1 + ); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ( + dist_shared_tensor_map_lookup_region_impl( + *map, make_region(lower_address, 0, 32, -1), 3, 2, protocol_ok + ), + 1 + ); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ( + dist_shared_tensor_map_lookup_region_impl( + *map, make_region(future_address, 0, 32, -1), 2, 2, protocol_ok + ), + -1 + ); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ( + dist_shared_tensor_map_lookup_region_impl( + *map, make_region(versioned_address, 64, 128, -1), 3, 3, protocol_ok + ), + -1 + ); + EXPECT_TRUE(protocol_ok); + EXPECT_EQ( + dist_shared_tensor_map_lookup_region_impl( + *map, make_region(versioned_address, 0, 64, -1), kFlagCap, 3, protocol_ok + ), + -1 + ); + EXPECT_FALSE(protocol_ok); +} + +TEST(FdwicSharedTensorMapRing, ExactTurnAndInclusiveReclaimAdvanceMonotonically) { + int64_t candidate = -2; + ASSERT_TRUE(dist_shared_tensor_map_compute_reclaim(0, 64, candidate)); + EXPECT_EQ(candidate, -1); + ASSERT_TRUE(dist_shared_tensor_map_compute_reclaim(64, 64, candidate)); + EXPECT_EQ(candidate, -1); + ASSERT_TRUE(dist_shared_tensor_map_compute_reclaim(65, 64, candidate)); + EXPECT_EQ(candidate, 0); + ASSERT_TRUE(dist_shared_tensor_map_compute_reclaim(1279, 64, candidate)); + EXPECT_EQ(candidate, 1214); + EXPECT_FALSE(dist_shared_tensor_map_compute_reclaim(-1, 64, candidate)); + EXPECT_FALSE(dist_shared_tensor_map_compute_reclaim(0, -1, candidate)); + + auto map = make_empty_map(); + const uint64_t address = 0x300000000ULL; + const uint32_t bucket = dist_tensor_map_hash(address); + EXPECT_EQ(try_commit(*map, 0, {make_region(address, 0, 32, 0)}), DriverResult::Committed); + EXPECT_EQ(try_commit(*map, 1, {make_region(address, 64, 96, 1)}), DriverResult::Committed); + EXPECT_EQ(try_commit(*map, 2, std::vector{}, 2), DriverResult::Committed); + EXPECT_EQ(map->committed_tasks.v, 3); + EXPECT_EQ(map->reclaim_upto.v, -1); + + int64_t reclaim_upto = -2; + EXPECT_FALSE(dist_shared_tensor_map_refresh_reclaim_impl(*map, 2, 2, reclaim_upto)); + EXPECT_FALSE(dist_shared_tensor_map_refresh_reclaim_impl(*map, 4, 2, reclaim_upto)); + EXPECT_EQ(map->reclaim_upto.v, -1); + EXPECT_EQ(map->buckets[bucket].head.v, 0); + EXPECT_EQ(map->buckets[bucket].tail.v, 2); + + EXPECT_EQ(try_commit(*map, 3, std::vector{}, 2), DriverResult::Committed); + EXPECT_EQ(map->committed_tasks.v, 4); + EXPECT_EQ(map->reclaim_upto.v, 0); + EXPECT_EQ(map->buckets[bucket].head.v, 0); + ASSERT_TRUE(dist_shared_tensor_map_retire_bucket_impl(*map, bucket, map->reclaim_upto.v)); + EXPECT_EQ(map->buckets[bucket].head.v, 1); + + map->reclaim_upto.v = 2; + map->committed_tasks.v = 4; + RecordingOps::ResetEvents(); + EXPECT_FALSE(dist_shared_tensor_map_refresh_reclaim_impl(*map, 4, 2, reclaim_upto)); + EXPECT_EQ(map->reclaim_upto.v, 2); + for (const Event &event : RecordingOps::events) { + EXPECT_NE(event.kind, EventKind::CompareExchange); + } +} + +TEST(FdwicSharedTensorMapRing, SameTaskCapacityFailurePublishesNothing) { + auto map = make_empty_map(); + const uint64_t address = 0x400000000ULL; + const uint32_t bucket = dist_tensor_map_hash(address); + for (uint32_t task = 0; task + 1 < kMapBucketCapacity; ++task) { + ASSERT_EQ( + try_commit(*map, static_cast(task), {make_region(address, task * 8, task * 8 + 4, task)}), + DriverResult::Committed + ); + } + ASSERT_EQ(map->buckets[bucket].tail.v - map->buckets[bucket].head.v, kMapBucketCapacity - 1); + + const int32_t task_id = static_cast(kMapBucketCapacity - 1); + const std::vector entries = { + make_region(address, 0x100000, 0x100004, task_id), + make_region(address, 0x200000, 0x200004, task_id), + }; + std::vector before(sizeof(*map)); + std::memcpy(before.data(), map.get(), before.size()); + EXPECT_EQ(try_commit(*map, task_id, entries), DriverResult::CapacityBlocked); + EXPECT_EQ(std::memcmp(before.data(), map.get(), before.size()), 0); +} + +TEST(FdwicSharedTensorMapRing, CrossBucketCapacityFailureDoesNotPublishEarlierEntry) { + if constexpr (kMapBuckets == 1) { + GTEST_SKIP() << "single-bucket CAP16384 variant has no cross-bucket case"; + } + auto map = make_empty_map(); + const uint64_t full_address = 0x500000000ULL; + const uint32_t full_bucket = dist_tensor_map_hash(full_address); + for (uint32_t task = 0; task < kMapBucketCapacity; ++task) { + ASSERT_EQ( + try_commit( + *map, static_cast(task), + {make_region(full_address, task * 8, task * 8 + 4, static_cast(task))} + ), + DriverResult::Committed + ); + } + const uint64_t other_address = find_address_outside_bucket(full_bucket, 0x510000000ULL); + ASSERT_NE(dist_tensor_map_hash(other_address), full_bucket); + const int32_t task_id = static_cast(kMapBucketCapacity); + const std::vector entries = { + make_region(other_address, 0, 4, task_id), + make_region(full_address, 0x100000, 0x100004, task_id), + }; + std::vector before(sizeof(*map)); + std::memcpy(before.data(), map.get(), before.size()); + EXPECT_EQ(try_commit(*map, task_id, entries), DriverResult::CapacityBlocked); + EXPECT_EQ(std::memcmp(before.data(), map.get(), before.size()), 0); +} + +TEST(FdwicSharedTensorMapRing, CapacityFailureAfterRetirePublishesNoTaskData) { + auto map = make_empty_map(); + const uint64_t address = 0x580000000ULL; + const uint32_t bucket = dist_tensor_map_hash(address); + for (uint32_t task = 0; task < kMapBucketCapacity; ++task) { + ASSERT_EQ( + try_commit( + *map, static_cast(task), + {make_region(address, task * 8, task * 8 + 4, static_cast(task))} + ), + DriverResult::Committed + ); + } + + const int64_t tail_before = map->buckets[bucket].tail.v; + const int64_t commit_before = map->committed_tasks.v; + std::vector slots_before(sizeof(map->slots)); + std::memcpy(slots_before.data(), map->slots, slots_before.size()); + std::array heads_before{}; + std::array tails_before{}; + for (uint32_t index = 0; index < kMapBuckets; ++index) { + heads_before[index] = map->buckets[index].head.v; + tails_before[index] = map->buckets[index].tail.v; + } + + const int32_t task_id = static_cast(kMapBucketCapacity); + const std::vector entries = { + make_region(address, 0x100000, 0x100004, task_id), + make_region(address, 0x200000, 0x200004, task_id), + }; + // N=CAP and H=CAP-1 can reclaim only producer 0. One free slot cannot hold + // both entries from the same task. Head/reclaim may advance monotonically, + // but payload, sequence, tail, and commit must remain unpublished. + EXPECT_EQ( + try_commit(*map, task_id, entries, static_cast(kMapBucketCapacity - 1)), DriverResult::CapacityBlocked + ); + EXPECT_EQ(map->reclaim_upto.v, 0); + EXPECT_EQ(map->buckets[bucket].head.v, heads_before[bucket] + 1); + EXPECT_EQ(map->buckets[bucket].tail.v, tail_before); + EXPECT_EQ(map->committed_tasks.v, commit_before); + EXPECT_EQ(std::memcmp(slots_before.data(), map->slots, slots_before.size()), 0); + for (uint32_t index = 0; index < kMapBuckets; ++index) { + if (index != bucket) { + EXPECT_EQ(map->buckets[index].head.v, heads_before[index]); + } + EXPECT_EQ(map->buckets[index].tail.v, tails_before[index]); + } +} + +TEST(FdwicSharedTensorMapRing, FullBucketRetiresExactBatchAndReusesSameTaskSlots) { + auto map = make_empty_map(); + const uint64_t address = 0x5a0000000ULL; + const uint32_t bucket = dist_tensor_map_hash(address); + for (uint32_t task = 0; task < kMapBucketCapacity; ++task) { + ASSERT_EQ( + try_commit( + *map, static_cast(task), + {make_region(address, task * 8, task * 8 + 4, static_cast(task))} + ), + DriverResult::Committed + ); + } + + constexpr uint32_t kReuse = kMapBucketCapacity / 4U < 8U ? kMapBucketCapacity / 4U : 8U; + static_assert(kReuse > 0, "exact reuse test requires a non-zero batch"); + const int32_t task_id = static_cast(kMapBucketCapacity); + std::vector replacements; + replacements.reserve(kReuse); + for (uint32_t index = 0; index < kReuse; ++index) { + const uint64_t lo = (1ULL << 20U) + static_cast(index) * 32U; + replacements.push_back(make_region(address, lo, lo + 8U, task_id)); + } + const int32_t history = static_cast(kMapBucketCapacity - kReuse); + ASSERT_EQ(try_commit(*map, task_id, replacements, history), DriverResult::Committed); + EXPECT_EQ(map->reclaim_upto.v, static_cast(kReuse - 1)); + EXPECT_EQ(map->buckets[bucket].head.v, static_cast(kReuse)); + EXPECT_EQ(map->buckets[bucket].tail.v, static_cast(kMapBucketCapacity) + static_cast(kReuse)); + for (uint32_t index = 0; index < kReuse; ++index) { + const uint64_t cursor = static_cast(kMapBucketCapacity) + index; + const SharedTensorMapSlot &reused = map->slots[dist_shared_tensor_map_slot_index(bucket, cursor)]; + EXPECT_EQ(reused.sequence.v, static_cast(cursor)); + EXPECT_EQ(reused.payload.value.buf_addr, replacements[index].buf_addr); + EXPECT_EQ(reused.payload.value.lo, replacements[index].lo); + EXPECT_EQ(reused.payload.value.hi, replacements[index].hi); + EXPECT_EQ(reused.payload.value.producer, task_id); + EXPECT_EQ(reused.payload.value.reserved, 0U); + } +} + +TEST(FdwicSharedTensorMapRing, ReverseArrivalEventuallyCommitsInTaskOrder) { + auto map = make_empty_map(); + const SharedTensorMapValue task1 = make_region(0x5d0000000ULL, 0, 4, 1); + const SharedTensorMapValue task2 = make_region(0x5e0000000ULL, 0, 4, 2); + + std::vector before(sizeof(*map)); + std::memcpy(before.data(), map.get(), before.size()); + EXPECT_EQ(try_commit(*map, 2, {task2}), DriverResult::Pending); + EXPECT_EQ(std::memcmp(before.data(), map.get(), before.size()), 0); + EXPECT_EQ(try_commit(*map, 1, {task1}), DriverResult::Pending); + EXPECT_EQ(std::memcmp(before.data(), map.get(), before.size()), 0); + EXPECT_EQ(try_commit(*map, 0, std::vector{}), DriverResult::Committed); + + std::memcpy(before.data(), map.get(), before.size()); + EXPECT_EQ(try_commit(*map, 0, std::vector{}), DriverResult::ProtocolError); + EXPECT_EQ(std::memcmp(before.data(), map.get(), before.size()), 0); + EXPECT_EQ(try_commit(*map, 2, {task2}), DriverResult::Pending); + EXPECT_EQ(std::memcmp(before.data(), map.get(), before.size()), 0); + EXPECT_EQ(try_commit(*map, 1, {task1}), DriverResult::Committed); + EXPECT_EQ(try_commit(*map, 2, {task2}), DriverResult::Committed); + EXPECT_EQ(map->committed_tasks.v, 3); +} + +TEST(FdwicSharedTensorMapRing, ThreeLapsKeepAbsoluteSequenceAndLatestVersion) { + auto map = make_empty_map(); + const uint64_t address = 0x600000000ULL; + const uint32_t bucket = dist_tensor_map_hash(address); + const int32_t tasks = static_cast(3 * kMapBucketCapacity + 5); + for (int32_t task = 0; task < tasks; ++task) { + ASSERT_EQ(try_commit(*map, task, {make_region(address, 0, 64, task)}, 0), DriverResult::Committed) + << "task=" << task; + } + EXPECT_EQ(map->committed_tasks.v, tasks); + EXPECT_EQ(map->buckets[bucket].head.v, tasks - 1); + EXPECT_EQ(map->buckets[bucket].tail.v, tasks); + const uint64_t last_cursor = static_cast(tasks - 1); + const SharedTensorMapSlot &last = map->slots[dist_shared_tensor_map_slot_index(bucket, last_cursor)]; + EXPECT_EQ(last.sequence.v, tasks - 1); + EXPECT_EQ(last.payload.value.producer, tasks - 1); + + bool protocol_ok = false; + EXPECT_EQ( + dist_shared_tensor_map_lookup_region_impl( + *map, make_region(address, 0, 64, -1), tasks, 1, protocol_ok + ), + tasks - 1 + ); + EXPECT_TRUE(protocol_ok); +} + +TEST(FdwicSharedTensorMapRing, ProtocolErrorsRemainDistinctFromMissAndCapacity) { + auto map = make_empty_map(); + const SharedTensorMapValue valid = make_region(0x700000000ULL, 0, 64, 0); + SharedTensorMapValue wrong_producer = valid; + wrong_producer.producer = 1; + SharedTensorMapValue bad_reserved = valid; + bad_reserved.reserved = 1; + SharedTensorMapValue empty_range = valid; + empty_range.hi = empty_range.lo; + const int64_t reclaim = map->reclaim_upto.v; + + EXPECT_EQ( + dist_shared_tensor_map_check_task_append_impl(*map, &wrong_producer, 1, 0, reclaim), + DistSharedTensorMapAppendCheck::ProtocolError + ); + EXPECT_EQ( + dist_shared_tensor_map_check_task_append_impl(*map, &bad_reserved, 1, 0, reclaim), + DistSharedTensorMapAppendCheck::ProtocolError + ); + EXPECT_EQ( + dist_shared_tensor_map_check_task_append_impl(*map, &empty_range, 1, 0, reclaim), + DistSharedTensorMapAppendCheck::ProtocolError + ); + EXPECT_EQ( + dist_shared_tensor_map_check_task_append_impl(*map, nullptr, 1, 0, reclaim), + DistSharedTensorMapAppendCheck::ProtocolError + ); + EXPECT_EQ( + dist_shared_tensor_map_check_task_append_impl( + *map, &valid, static_cast(MAX_TENSOR_ARGS + 1), 0, reclaim + ), + DistSharedTensorMapAppendCheck::ProtocolError + ); + EXPECT_EQ(map->committed_tasks.v, 0); + + std::vector before(sizeof(*map)); + std::memcpy(before.data(), map.get(), before.size()); + EXPECT_EQ(try_commit(*map, kFlagCap, {make_region(valid.buf_addr, 0, 64, kFlagCap)}), DriverResult::ProtocolError); + EXPECT_EQ(std::memcmp(before.data(), map.get(), before.size()), 0); + + map->committed_tasks.v = kFlagCap; + std::memcpy(before.data(), map.get(), before.size()); + int64_t bounded_reclaim = -2; + EXPECT_FALSE(dist_shared_tensor_map_refresh_reclaim_impl(*map, kFlagCap, 64, bounded_reclaim)); + EXPECT_EQ( + dist_shared_tensor_map_check_task_append_impl(*map, &valid, 1, kFlagCap, map->reclaim_upto.v), + DistSharedTensorMapAppendCheck::ProtocolError + ); + EXPECT_EQ(std::memcmp(before.data(), map.get(), before.size()), 0); + map->committed_tasks.v = 0; + + const uint32_t bucket = dist_tensor_map_hash(valid.buf_addr); + map->buckets[bucket].head.v = INT64_MAX; + map->buckets[bucket].tail.v = INT64_MAX; + std::memcpy(before.data(), map.get(), before.size()); + EXPECT_EQ( + dist_shared_tensor_map_check_task_append_impl(*map, &valid, 1, 0, reclaim), + DistSharedTensorMapAppendCheck::ProtocolError + ); + EXPECT_EQ(std::memcmp(before.data(), map.get(), before.size()), 0); + EXPECT_EQ( + dist_shared_tensor_map_publish_prepared_entry_impl(*map, valid, 0, 0), + DistSharedTensorMapEntryPublishResult::ProtocolError + ); + EXPECT_EQ(std::memcmp(before.data(), map.get(), before.size()), 0); + map->buckets[bucket].head.v = 0; + map->buckets[bucket].tail.v = 0; + + SharedTensorMapSlot &slot = map->slots[dist_shared_tensor_map_slot_index(bucket, 0)]; + slot.sequence.v = kSharedTensorMapWritingSequence; + SharedTensorMapValue writing_snapshot{}; + EXPECT_FALSE(dist_shared_tensor_map_read_slot_impl(*map, bucket, 0, writing_snapshot)); + EXPECT_EQ( + dist_shared_tensor_map_check_task_append_impl(*map, &valid, 1, 0, reclaim), + DistSharedTensorMapAppendCheck::ProtocolError + ); + slot.sequence.v = 99; + EXPECT_EQ( + dist_shared_tensor_map_check_task_append_impl(*map, &valid, 1, 0, reclaim), + DistSharedTensorMapAppendCheck::ProtocolError + ); + slot.sequence.v = kSharedTensorMapInvalidSequence; + + ASSERT_EQ(try_commit(*map, 0, {valid}), DriverResult::Committed); + slot.payload.value.reserved = 1; + bool protocol_ok = true; + EXPECT_EQ( + dist_shared_tensor_map_lookup_region_impl( + *map, make_region(valid.buf_addr, 0, 64, -1), 1, 1, protocol_ok + ), + -1 + ); + EXPECT_FALSE(protocol_ok); +} + +struct ReferenceEntry { + SharedTensorMapValue value; +}; + +int32_t reference_lookup( + const std::vector &entries, const SharedTensorMapValue &query, int32_t current_task, int32_t history +) { + const int32_t lower = current_task > history ? current_task - history : 0; + int32_t best = -1; + for (const ReferenceEntry &entry : entries) { + if (entry.value.producer >= lower && entry.value.producer < current_task && + entry.value.buf_addr == query.buf_addr && query.lo < entry.value.hi && entry.value.lo < query.hi) { + best = std::max(best, entry.value.producer); + } + } + return best; +} + +SharedTensorMapValue make_random_region(std::mt19937_64 &random, int32_t producer) { + const uint64_t address = 0x800000000ULL + (random() % 48) * 0x100000ULL; + const uint64_t lo = (random() % 96) * 4; + const uint64_t bytes = (1 + random() % 12) * 4; + return make_region(address, lo, lo + bytes, producer); +} + +TEST(FdwicSharedTensorMapRing, FixedSeedTwelveThousandTasksMatchIndependentReference) { + constexpr uint64_t kSeed = 0x53485244544d4150ULL; // "SHRDTMAP" + constexpr int32_t kHistory = 15; + constexpr int32_t kTasks = 12000; + auto map = make_empty_map(); + std::mt19937_64 random(kSeed); + std::vector reference; + reference.reserve(kTasks); + + for (int32_t task = 0; task < kTasks; ++task) { + const SharedTensorMapValue query = make_random_region(random, -1); + bool protocol_ok = false; + const int32_t actual = + dist_shared_tensor_map_lookup_region_impl(*map, query, task, kHistory, protocol_ok); + ASSERT_TRUE(protocol_ok) << "task=" << task; + EXPECT_EQ(actual, reference_lookup(reference, query, task, kHistory)) << "task=" << task; + + const SharedTensorMapValue inserted = make_random_region(random, task); + ASSERT_EQ(try_commit(*map, task, {inserted}, kHistory), DriverResult::Committed) << "task=" << task; + reference.push_back({inserted}); + + const int32_t next_actual = + dist_shared_tensor_map_lookup_region_impl(*map, inserted, task + 1, kHistory, protocol_ok); + ASSERT_TRUE(protocol_ok) << "task=" << task; + EXPECT_EQ(next_actual, reference_lookup(reference, inserted, task + 1, kHistory)) << "task=" << task; + } +} + +TEST(FdwicSharedTensorMapRing, ConcreteAicoreOpsRoundTripUsesTheSameStateMachine) { + auto map = make_empty_map(); + const SharedTensorMapValue entry = make_region(0x900000000ULL, 16, 80, 0); + ASSERT_EQ( + dist_shared_tensor_map_publish_task(*map, &entry, 1, 0, 64), DistSharedTensorMapTaskPublishResult::Committed + ); + + bool protocol_ok = false; + EXPECT_EQ( + dist_shared_tensor_map_lookup_region(*map, make_region(entry.buf_addr, 32, 48, -1), 1, 64, protocol_ok), 0 + ); + EXPECT_TRUE(protocol_ok); + + // This also rejects an adapter that accidentally degrades CAS to Exchange: + // Exchange would overwrite the competing value 2 with desired value 1. + map->committed_tasks.v = 2; + EXPECT_FALSE(dist_shared_tensor_map_publish_commit_impl(*map, 0)); + EXPECT_EQ(map->committed_tasks.v, 2); +} + +} // namespace diff --git a/tests/ut/cpp/a5/test_fdwic_shared_tensor_map_state.cpp b/tests/ut/cpp/a5/test_fdwic_shared_tensor_map_state.cpp new file mode 100644 index 0000000000..74df48b886 --- /dev/null +++ b/tests/ut/cpp/a5/test_fdwic_shared_tensor_map_state.cpp @@ -0,0 +1,112 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include +#include +#include +#include + +#include "dist_engine/aicpu/shared_tensor_map_init.h" + +namespace { + +constexpr uint8_t kPayloadPattern = 0xa5; + +bool shared_payloads_keep_pattern(const SharedTensorMapState &state) { + for (int32_t slot = 0; slot < kMapCap; ++slot) { + const auto *bytes = reinterpret_cast(&state.slots[slot].payload); + for (size_t offset = 0; offset < sizeof(SharedTensorMapPayloadLine); ++offset) { + if (bytes[offset] != kPayloadPattern) return false; + } + } + return true; +} + +TEST(FdwicSharedTensorMapState, SidecarLayoutAppendsWithoutMovingThePrivateTail) { + constexpr size_t kExpectedStateBytes = 2 * kCacheLine + kMapBuckets * 2 * kCacheLine + kMapCap * 2 * kCacheLine; + + EXPECT_EQ(sizeof(SharedTensorMapValue), 32U); + EXPECT_EQ(offsetof(SharedTensorMapValue, buf_addr), 0U); + EXPECT_EQ(offsetof(SharedTensorMapValue, lo), 8U); + EXPECT_EQ(offsetof(SharedTensorMapValue, hi), 16U); + EXPECT_EQ(offsetof(SharedTensorMapValue, producer), 24U); + EXPECT_EQ(offsetof(SharedTensorMapValue, reserved), 28U); + EXPECT_EQ(sizeof(SharedTensorMapPayloadLine), kCacheLine); + EXPECT_EQ(sizeof(SharedTensorMapSequenceLine), kCacheLine); + EXPECT_EQ(offsetof(SharedTensorMapSequenceLine, v), 0U); + EXPECT_EQ(sizeof(SharedTensorMapSlot), 2 * kCacheLine); + EXPECT_LT(kSharedTensorMapWritingSequence, kSharedTensorMapInvalidSequence); + EXPECT_NE(kSharedTensorMapWritingSequence, kSharedTensorMapInvalidSequence); + EXPECT_EQ(sizeof(SharedTensorMapBucketState), 2 * kCacheLine); + EXPECT_EQ(sizeof(SharedTensorMapState), kExpectedStateBytes); + EXPECT_EQ(alignof(SharedTensorMapState), kCacheLine); + EXPECT_EQ(offsetof(SharedTensorMapState, committed_tasks), 0U); + EXPECT_EQ(offsetof(SharedTensorMapState, reclaim_upto), kCacheLine); + EXPECT_EQ(offsetof(SharedTensorMapState, buckets), 2 * kCacheLine); + EXPECT_EQ(offsetof(SharedTensorMapState, slots), 2 * kCacheLine + sizeof(SharedTensorMapBucketState) * kMapBuckets); + EXPECT_EQ(offsetof(SharedTensorMapBucketState, head), 0U); + EXPECT_EQ(offsetof(SharedTensorMapBucketState, tail), kCacheLine); +#if PTO_FDWIC_TENSORMAP_RING_CAP == 128 + EXPECT_EQ(offsetof(SharedTensorMapState, buckets), 128U); + EXPECT_EQ(offsetof(SharedTensorMapState, slots), 16512U); + EXPECT_EQ(sizeof(SharedTensorMapState), 2113664U); +#endif + +#if PTO_FDWIC_SHARED_MAP + EXPECT_EQ(offsetof(DistGlobal, shared_tensor_map), kFdwicSharedTensorMapOffset); + EXPECT_EQ(sizeof(DistGlobal), kFdwicSharedTensorMapOffset + sizeof(SharedTensorMapState)); +#else + EXPECT_EQ(sizeof(DistGlobal), kFdwicSharedTensorMapOffset); +#endif + EXPECT_LE(sizeof(DistGlobal), kDistEngineGlobalStateSize); +} + +#if PTO_FDWIC_SHARED_MAP +TEST(FdwicSharedTensorMapState, AicpuResetInitializesOnlyPublicationAndCursorState) { + auto state = std::make_unique(); + std::memset(state.get(), kPayloadPattern, sizeof(*state)); + + dist_shared_tensor_map_reset(*state); + + EXPECT_EQ(state->committed_tasks.v, kSharedTensorMapInitialCommit); + EXPECT_EQ(state->reclaim_upto.v, kSharedTensorMapInitialReclaim); + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + EXPECT_EQ(state->buckets[bucket].head.v, 0) << "bucket=" << bucket; + EXPECT_EQ(state->buckets[bucket].tail.v, 0) << "bucket=" << bucket; + } + for (int32_t slot = 0; slot < kMapCap; ++slot) { + EXPECT_EQ(state->slots[slot].sequence.v, kSharedTensorMapInvalidSequence) << "slot=" << slot; + } + EXPECT_TRUE(shared_payloads_keep_pattern(*state)); + + // 同一 arena 重复运行时必须清掉上一轮的绝对游标和发布 seq。 + state->committed_tasks.v = 37; + state->reclaim_upto.v = 21; + state->buckets[kMapBuckets - 1].head.v = 8; + state->buckets[kMapBuckets - 1].tail.v = 10; + state->slots[kMapCap - 2].sequence.v = 16382; + state->slots[kMapCap - 1].sequence.v = kSharedTensorMapWritingSequence; + + dist_shared_tensor_map_reset(*state); + + EXPECT_EQ(state->committed_tasks.v, kSharedTensorMapInitialCommit); + EXPECT_EQ(state->reclaim_upto.v, kSharedTensorMapInitialReclaim); + EXPECT_EQ(state->buckets[kMapBuckets - 1].head.v, 0); + EXPECT_EQ(state->buckets[kMapBuckets - 1].tail.v, 0); + EXPECT_EQ(state->slots[kMapCap - 2].sequence.v, kSharedTensorMapInvalidSequence); + EXPECT_EQ(state->slots[kMapCap - 1].sequence.v, kSharedTensorMapInvalidSequence); + EXPECT_TRUE(shared_payloads_keep_pattern(*state)); +} +#endif + +} // namespace diff --git a/tests/ut/cpp/a5/test_fdwic_submit_capacity.cpp b/tests/ut/cpp/a5/test_fdwic_submit_capacity.cpp new file mode 100644 index 0000000000..0856636ef6 --- /dev/null +++ b/tests/ut/cpp/a5/test_fdwic_submit_capacity.cpp @@ -0,0 +1,318 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +// Compile the production CPU-sim Submit implementation directly. Disable only +// diagnostics rather than copying the Claim, Register, Build, or failure state machines. +#define PTO_FDWIC_TRACE_ENABLED 0 +#include "dist_engine/aicore/dist_engine.cpp" // NOLINT(build/include) + +[[noreturn]] void assert_impl(const char *condition, const char *, int) { throw std::logic_error(condition); } + +// The production CPU-sim TU declares the real orchestration entry. These tests +// call compete-first Submit directly, so only a link definition is required. +extern "C" void aicpu_orchestration_entry(const L2TaskArgs &) {} +// The full production TU also contains unused worker-finish and platform hooks. +// These stubs satisfy linkage only and do not participate in the assertions. +volatile uint8_t *sim_get_reg_base() { return nullptr; } +uint32_t sim_get_physical_core_id() { return 0; } + +Runtime::Runtime() { + for (uint64_t &address : func_id_to_addr_) { + address = 0; + } + use_example_exec_time_ = false; + for (int32_t &duration : example_exec_time_ns_) { + duration = 0; + } +} + +namespace { + +Tensor make_existing_output(uint64_t address) { + Tensor tensor{}; + const uint32_t shape[1] = {1}; + tensor.init_external( + reinterpret_cast(static_cast(address)), sizeof(float), shape, 1, DataType::FLOAT32, 0 + ); + return tensor; +} + +Tensor make_existing_output_in_another_bucket(uint64_t address) { + const uint32_t original_bucket = dist_private_tensor_map_hash(address); + for (uint64_t candidate = address + 64; candidate < address + (1ULL << 30); candidate += 64) { + if (dist_private_tensor_map_hash(candidate) != original_bucket) { + return make_existing_output(candidate); + } + } + throw std::logic_error("failed to find a TensorMap address in another bucket"); +} + +void fill_output_bucket(DistTensorMap &map, const Tensor &tensor, uint32_t count) { + for (uint32_t index = 0; index < count; ++index) { + ASSERT_TRUE(dist_private_tensor_map_insert(map, tensor, /*producer=*/0)) + << "index=" << index << " count=" << count; + } +} + +class FdwicSubmitCapacityTest : public ::testing::Test { +protected: + void SetUp() override { + static_assert(PTO_FDWIC_SHARED_MAP == 0); + // Production worker_state.h does not bind the fallback under + // __CPU_SIM. Bind it explicitly so every entry uses real state. + g_dist_ptr = &g_dist_fallback; + g_self = worker_.get(); + g_fdwic_joint_submit_seen = false; + dist_core_reset(*worker_, CoreType::AIV, /*block=*/0, LANE_AIV0); + worker_->core_idx = 0; + + g_dist.H = kHDefault; + g_dist.heap_base = nullptr; + g_dist.heap_size = 0; + g_dist.runtime = &runtime_; + g_dist.num_workers = 1; + g_dist.num_blocks = 1; + g_dist.fatal = 0; + g_dist.error_code = PTO2_ERROR_NONE; + g_dist.blocks[0].any_pub = 0; + for (int32_t i = 0; i < kPrivateSlots; ++i) { + g_dist.blocks[0].slots[i].state.v = kWonStateFree; + } + for (int32_t shard = 0; shard < kCursorShards; ++shard) { + g_dist.cube_cursor[shard].v = -1; + g_dist.vector_cursor[shard].v = -1; + g_dist.alloc_cursor[shard].v = -1; + } + } + + void TearDown() override { + g_self = nullptr; + g_dist_ptr = nullptr; + } + + Runtime runtime_; + std::unique_ptr worker_ = std::make_unique(); +}; + +TEST_F(FdwicSubmitCapacityTest, RegisterFailureStopsBuildAndClosesFollowingClaimGate) { + // Fill the target bucket to CAP-1. The first OUTPUT_EXISTING consumes the + // last slot, and the second one reaches the exact per-bucket limit. + const Tensor first_output = make_existing_output(0x100000); + const Tensor second_output = first_output; + const uint32_t bucket = dist_private_tensor_map_hash(first_output.buffer.addr); + fill_output_bucket(worker_->map, first_output, kMapBucketCapacity - 1); + ASSERT_EQ(dist_private_tensor_map_load_head(worker_->map, bucket), 0U); + ASSERT_EQ(dist_private_tensor_map_load_tail(worker_->map, bucket), kMapBucketCapacity - 1); + L0TaskArgs args; + args.add_output(first_output, second_output); + ASSERT_EQ(args.tensor_count(), 2); + ASSERT_EQ(args.tag(0), TensorArgType::OUTPUT_EXISTING); + ASSERT_EQ(args.tag(1), TensorArgType::OUTPUT_EXISTING); + + MixedKernels mixed; + mixed.aiv0_kernel_id = 7; + + std::arrayslots)> slots_before{}; + std::memcpy(slots_before.data(), worker_->slots, slots_before.size()); + const int32_t occupied_before = worker_->occupied_count; + const int32_t owned_before = worker_->owned_total; + + const DistCompeteFirstTicket ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(ticket.ready, 1); + ASSERT_EQ(ticket.won, 1); + ASSERT_EQ(ticket.claim_attempted, 1); + ASSERT_EQ(ticket.task_id, 0); + ASSERT_EQ(worker_->local_index, 1); + ASSERT_EQ(g_dist.vector_cursor[0].v, 0); + + (void)dist_submit_compete_first_finish(nullptr, mixed, ticket, args); + + // The first region consumes the final slot and the second is not inserted. + // Register failure must return before WinnerBuild or slot allocation. + EXPECT_EQ(dist_private_tensor_map_load_head(worker_->map, bucket), 0U); + EXPECT_EQ(dist_private_tensor_map_load_tail(worker_->map, bucket), kMapBucketCapacity); + const uint32_t last_slot = dist_private_tensor_map_slot_index(bucket, kMapBucketCapacity - 1); + EXPECT_EQ(worker_->map.entries[last_slot].buf_addr, first_output.buffer.addr); + EXPECT_EQ(worker_->occupied_count, occupied_before); + EXPECT_EQ(worker_->owned_total, owned_before); + EXPECT_EQ(std::memcmp(slots_before.data(), worker_->slots, slots_before.size()), 0); + + // Failure latching reuses the task-cap sentinel. AICPU can retrieve the + // structured capacity error from the same fatal cache line. + EXPECT_EQ(worker_->local_index, kFlagCap); + EXPECT_EQ(g_dist.fatal, 1); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_TENSORMAP_CAPACITY); + set_fatal_code(PTO2_ERROR_INVALID_ARGS); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_TENSORMAP_CAPACITY); + + const int64_t claim_cursor_after_failure = g_dist.vector_cursor[0].v; + const DistCompeteFirstTicket blocked = dist_submit_compete_first_begin(nullptr, mixed); + EXPECT_EQ(blocked.task_id, kFlagCap); + EXPECT_EQ(blocked.ready, 0); + EXPECT_EQ(blocked.won, 0); + EXPECT_EQ(blocked.claim_attempted, 0); + EXPECT_EQ(worker_->local_index, kFlagCap); + EXPECT_EQ(g_dist.vector_cursor[0].v, claim_cursor_after_failure); +} + +TEST_F(FdwicSubmitCapacityTest, CrossBucketFailureKeepsTheExistingPrefixPublicationContract) { + // Register is not a whole-task transaction. Earlier outputs may be + // published before a later full ring makes the task fail without Build. + const Tensor full_output = make_existing_output(0x300000); + const Tensor prefix_output = make_existing_output_in_another_bucket(full_output.buffer.addr); + const uint32_t full_bucket = dist_private_tensor_map_hash(full_output.buffer.addr); + const uint32_t prefix_bucket = dist_private_tensor_map_hash(prefix_output.buffer.addr); + ASSERT_NE(prefix_bucket, full_bucket); + fill_output_bucket(worker_->map, full_output, kMapBucketCapacity); + + L0TaskArgs args; + args.add_output(prefix_output, full_output); + MixedKernels mixed; + mixed.aiv0_kernel_id = 9; + + std::arrayslots)> slots_before{}; + std::memcpy(slots_before.data(), worker_->slots, slots_before.size()); + const uint64_t full_head_before = dist_private_tensor_map_load_head(worker_->map, full_bucket); + const uint64_t full_tail_before = dist_private_tensor_map_load_tail(worker_->map, full_bucket); + const uint64_t prefix_tail_before = dist_private_tensor_map_load_tail(worker_->map, prefix_bucket); + + const DistCompeteFirstTicket ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(ticket.ready, 1); + ASSERT_EQ(ticket.won, 1); + ASSERT_EQ(ticket.task_id, 0); + (void)dist_submit_compete_first_finish(nullptr, mixed, ticket, args); + + EXPECT_EQ(dist_private_tensor_map_load_head(worker_->map, full_bucket), full_head_before); + EXPECT_EQ(dist_private_tensor_map_load_tail(worker_->map, full_bucket), full_tail_before); + EXPECT_EQ(dist_private_tensor_map_load_tail(worker_->map, prefix_bucket), prefix_tail_before + 1); + EXPECT_EQ(dist_private_tensor_map_lookup(worker_->map, prefix_output), 0); + EXPECT_EQ(std::memcmp(slots_before.data(), worker_->slots, slots_before.size()), 0); + EXPECT_EQ(worker_->occupied_count, 0); + EXPECT_EQ(worker_->owned_total, 0); + EXPECT_EQ(worker_->local_index, kFlagCap); + EXPECT_EQ(g_dist.fatal, 1); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_TENSORMAP_CAPACITY); +} + +TEST_F(FdwicSubmitCapacityTest, FullBucketFirstStopsBeforeAFreeLaterOutput) { + // Register follows argument order and returns on the first failure. When a + // full-bucket output comes first, no later free-bucket output is inserted. + const Tensor full_output = make_existing_output(0x500000); + const Tensor later_free_output = make_existing_output_in_another_bucket(full_output.buffer.addr); + const uint32_t full_bucket = dist_private_tensor_map_hash(full_output.buffer.addr); + const uint32_t free_bucket = dist_private_tensor_map_hash(later_free_output.buffer.addr); + ASSERT_NE(free_bucket, full_bucket); + fill_output_bucket(worker_->map, full_output, kMapBucketCapacity); + + L0TaskArgs args; + args.add_output(full_output, later_free_output); + MixedKernels mixed; + mixed.aiv0_kernel_id = 11; + + std::vector map_before(sizeof(worker_->map)); + std::memcpy(map_before.data(), &worker_->map, map_before.size()); + std::arrayslots)> slots_before{}; + std::memcpy(slots_before.data(), worker_->slots, slots_before.size()); + const int32_t occupied_before = worker_->occupied_count; + const int32_t owned_before = worker_->owned_total; + + const DistCompeteFirstTicket ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(ticket.ready, 1); + ASSERT_EQ(ticket.won, 1); + ASSERT_EQ(ticket.task_id, 0); + (void)dist_submit_compete_first_finish(nullptr, mixed, ticket, args); + + EXPECT_EQ(std::memcmp(map_before.data(), &worker_->map, map_before.size()), 0); + EXPECT_EQ(dist_private_tensor_map_load_head(worker_->map, full_bucket), 0U); + EXPECT_EQ(dist_private_tensor_map_load_tail(worker_->map, full_bucket), kMapBucketCapacity); + EXPECT_EQ(dist_private_tensor_map_load_head(worker_->map, free_bucket), 0U); + EXPECT_EQ(dist_private_tensor_map_load_tail(worker_->map, free_bucket), 0U); + EXPECT_EQ(std::memcmp(slots_before.data(), worker_->slots, slots_before.size()), 0); + EXPECT_EQ(worker_->occupied_count, occupied_before); + EXPECT_EQ(worker_->owned_total, owned_before); + EXPECT_EQ(worker_->local_index, kFlagCap); + EXPECT_EQ(g_dist.fatal, 1); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_TENSORMAP_CAPACITY); +} + +TEST_F(FdwicSubmitCapacityTest, ManualDependencyInoutKeepsCreatorButSkipsPrivateLookupAndRegister) { + Tensor manual_output = make_existing_output(0x700000); + manual_output.manual_dep = true; + manual_output.owner_task_id = PTO2TaskId::make(0, 0); + const Tensor normal_output = make_existing_output_in_another_bucket(manual_output.buffer.addr); + const uint32_t manual_bucket = dist_private_tensor_map_hash(manual_output.buffer.addr); + const uint32_t normal_bucket = dist_private_tensor_map_hash(normal_output.buffer.addr); + ASSERT_NE(manual_bucket, normal_bucket); + ASSERT_TRUE(dist_private_tensor_map_insert(worker_->map, manual_output, /*producer=*/1)); + ASSERT_TRUE(dist_private_tensor_map_insert(worker_->map, normal_output, /*producer=*/2)); + worker_->local_index = 3; + + L0TaskArgs args; + args.add_inout(manual_output, normal_output); + ASSERT_EQ(args.tag(0), TensorArgType::INOUT); + ASSERT_EQ(args.tag(1), TensorArgType::INOUT); + ASSERT_TRUE(args.tensor(0).ref().manual_dep); + ASSERT_FALSE(args.tensor(1).ref().manual_dep); + MixedKernels mixed; + mixed.aiv0_kernel_id = 13; + + const DistCompeteFirstTicket ticket = dist_submit_compete_first_begin(nullptr, mixed); + ASSERT_EQ(ticket.ready, 1); + ASSERT_EQ(ticket.won, 1); + ASSERT_EQ(ticket.task_id, 3); + (void)dist_submit_compete_first_finish(nullptr, mixed, ticket, args); + + // manual_dep retains creator 0 but neither consumes map producer 1 nor + // registers task 3. The normal INOUT consumes producer 2 and registers 3. + EXPECT_EQ(g_dist.fatal, 0); + EXPECT_EQ(g_dist.error_code, PTO2_ERROR_NONE); + EXPECT_EQ(dist_private_tensor_map_load_head(worker_->map, manual_bucket), 0U); + EXPECT_EQ(dist_private_tensor_map_load_tail(worker_->map, manual_bucket), 1U); + EXPECT_EQ(dist_private_tensor_map_lookup(worker_->map, manual_output), 1); + EXPECT_EQ(dist_private_tensor_map_load_head(worker_->map, normal_bucket), 0U); + EXPECT_EQ(dist_private_tensor_map_load_tail(worker_->map, normal_bucket), 2U); + EXPECT_EQ(dist_private_tensor_map_lookup(worker_->map, normal_output), 3); + EXPECT_EQ(worker_->occupied_count, 1); + const RingSlot *built_slot = nullptr; + for (int32_t index = 0; index < kPrivateSlots; ++index) { + const RingSlot &slot = worker_->slots[index]; + if (slot.occupied && slot.built && slot.task_id == 3) { + ASSERT_EQ(built_slot, nullptr); + built_slot = &slot; + } + } + ASSERT_NE(built_slot, nullptr); + ASSERT_EQ(built_slot->fanin_count, 2); + int32_t creator_count = 0; + int32_t manual_map_count = 0; + int32_t normal_map_count = 0; + for (int32_t index = 0; index < built_slot->fanin_count; ++index) { + creator_count += built_slot->fanin[index] == 0 ? 1 : 0; + manual_map_count += built_slot->fanin[index] == 1 ? 1 : 0; + normal_map_count += built_slot->fanin[index] == 2 ? 1 : 0; + } + EXPECT_EQ(creator_count, 1); + EXPECT_EQ(manual_map_count, 0); + EXPECT_EQ(normal_map_count, 1); + EXPECT_EQ(worker_->owned_total, 0); +} + +} // namespace diff --git a/tests/ut/cpp/a5/test_fdwic_swimlane_poll_batch.cpp b/tests/ut/cpp/a5/test_fdwic_swimlane_poll_batch.cpp new file mode 100644 index 0000000000..1d213746fb --- /dev/null +++ b/tests/ut/cpp/a5/test_fdwic_swimlane_poll_batch.cpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the LICENSE file. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include + +#include "inner_kernel.h" +#include "runtime.h" +#include "dist_engine/common/swimlane.h" + +namespace { + +TEST(FdwicAtomicSiteClassificationTest, ResultUsedBitmapIsExactlySixteenOfTwentyEightSites) { + constexpr bool kExpectedResultUsed[] = { + false, true, true, false, true, true, false, false, // 0..7 + true, true, true, true, true, false, true, true, // 8..15 + false, false, false, false, false, true, true, true, // 16..23 + false, true, false, true, // 24..27 + }; + constexpr uint32_t kSiteCount = static_cast(FdwicAtomicSite::Count); + constexpr uint32_t kExpectedSiteCount = sizeof(kExpectedResultUsed) / sizeof(kExpectedResultUsed[0]); + static_assert(kSiteCount == 28U, "FDWIC atomic site count changed"); + static_assert(kExpectedSiteCount == kSiteCount, "result-used expectation must cover every atomic site"); + + uint32_t result_used_count = 0; + for (uint32_t site_index = 0; site_index < kSiteCount; ++site_index) { + const bool result_used = fdwic_atomic_site_result_used(static_cast(site_index)); + EXPECT_EQ(result_used, kExpectedResultUsed[site_index]) << "site_index=" << site_index; + result_used_count += result_used ? 1U : 0U; + } + + EXPECT_EQ(result_used_count, 16U); + EXPECT_EQ(kSiteCount - result_used_count, 12U); +} + +class FdwicPollBatchTest : public ::testing::Test { +protected: + void SetUp() override { + core_ = {}; + records_[0] = {}; + records_[1] = {}; + g_self = reinterpret_cast(uintptr_t{1}); + g_fdwic_swimlane_level = kFdwicAtomicSwimlaneLevel; + g_fdwic_swimlane_core = &core_; + g_fdwic_swimlane_records = records_; + g_fdwic_swimlane_records_per_core = 2; + g_fdwic_atomic_poll_burst = {}; + g_fdwic_atomic_calls = 0; + g_fdwic_poll_calls = 0; + g_fdwic_poll_batch_records = 0; + g_fdwic_atomic_counter_overflow = false; + } + + void TearDown() override { + g_self = nullptr; + g_fdwic_swimlane_level = 0; + g_fdwic_swimlane_core = nullptr; + g_fdwic_swimlane_records = nullptr; + g_fdwic_swimlane_records_per_core = 0; + g_fdwic_atomic_poll_burst = {}; + } + + FdwicSwimlaneCoreState core_{}; + FdwicSwimlaneRecord records_[2]{}; +}; + +TEST_F(FdwicPollBatchTest, SplitsAtMaximum24BitCountAndReopensExactly) { + constexpr FdwicAtomicSite kSite = FdwicAtomicSite::StartupPoll; + constexpr uint32_t kBatchIndex = 0; + constexpr uint32_t kBatchBit = 1U << kBatchIndex; + constexpr uint64_t kFirstStart = 111; + constexpr uint64_t kSecondStart = 222; + constexpr uint64_t kSecondEnd = 333; + + ASSERT_EQ(fdwic_atomic_poll_batch_index(kSite), static_cast(kBatchIndex)); + g_fdwic_atomic_poll_burst.active_mask = kBatchBit; + g_fdwic_atomic_poll_burst.start_cycle[kBatchIndex] = kFirstStart; + g_fdwic_atomic_poll_burst.call_count[kBatchIndex] = kFdwicAtomicPollCountMax - 1; + + // The maximum-th call belongs to the first row and triggers an immediate + // flush. It must not be dropped or carried into the next batch. + fdwic_swimlane_accumulate_poll_call(kSite, kFirstStart); + + ASSERT_EQ(core_.count, 1U); + EXPECT_EQ(g_fdwic_poll_batch_records, 1U); + EXPECT_EQ(g_fdwic_atomic_poll_burst.active_mask, 0U); + EXPECT_EQ(g_fdwic_atomic_poll_burst.call_count[kBatchIndex], 0U); + EXPECT_EQ(records_[0].phase, static_cast(FdwicSwimlanePhase::Atomic)); + EXPECT_EQ(records_[0].aux, static_cast(kSite)); + EXPECT_EQ(records_[0].start_cycle, kFirstStart); + EXPECT_GE(records_[0].end_cycle, records_[0].start_cycle); + EXPECT_NE(records_[0].flags & kFdwicAtomicPollBatch, 0U); + EXPECT_EQ(records_[0].flags >> kFdwicAtomicPollCountShift, kFdwicAtomicPollCountMax); + + // The next call starts a fresh row at count one. Closing that row must + // preserve max+1 calls exactly across the two encoded records. + fdwic_swimlane_accumulate_poll_call(kSite, kSecondStart); + ASSERT_EQ(g_fdwic_atomic_poll_burst.active_mask, kBatchBit); + ASSERT_EQ(g_fdwic_atomic_poll_burst.call_count[kBatchIndex], 1U); + fdwic_atomic_poll_boundary_at(kSecondEnd); + + ASSERT_EQ(core_.count, 2U); + EXPECT_EQ(g_fdwic_poll_batch_records, 2U); + EXPECT_EQ(g_fdwic_atomic_poll_burst.active_mask, 0U); + EXPECT_EQ(g_fdwic_atomic_poll_burst.call_count[kBatchIndex], 0U); + EXPECT_EQ(records_[1].start_cycle, kSecondStart); + EXPECT_EQ(records_[1].end_cycle, kSecondEnd); + EXPECT_EQ(records_[1].flags >> kFdwicAtomicPollCountShift, 1U); + const uint64_t represented_calls = + (records_[0].flags >> kFdwicAtomicPollCountShift) + (records_[1].flags >> kFdwicAtomicPollCountShift); + EXPECT_EQ(represented_calls, static_cast(kFdwicAtomicPollCountMax) + 1); + EXPECT_EQ(core_.dropped, 0U); + EXPECT_FALSE(g_fdwic_atomic_counter_overflow); +} + +} // namespace diff --git a/tests/ut/cpp/a5/test_fdwic_tensor_map_retire.cpp b/tests/ut/cpp/a5/test_fdwic_tensor_map_retire.cpp new file mode 100644 index 0000000000..0ec0921990 --- /dev/null +++ b/tests/ut/cpp/a5/test_fdwic_tensor_map_retire.cpp @@ -0,0 +1,571 @@ +/* + * Copyright (c) PyPTO Contributors. + * This program is free software, you can redistribute it and/or modify it under the terms and conditions of + * CANN Open Software License Agreement Version 2.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the LICENSE file. + * ----------------------------------------------------------------------------------------------------------- + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "inner_kernel.h" +#include "dist_engine/common/state.h" +#include "dist_engine/aicore/tensor_map.h" + +[[noreturn]] void assert_impl(const char *condition, const char *, int) { throw std::logic_error(condition); } + +namespace { + +std::unique_ptr make_empty_map() { + auto map = std::make_unique(); + dist_private_tensor_map_reset(*map); + return map; +} + +std::unique_ptr clone_map_bytes(const DistTensorMap &source) { + static_assert(std::is_trivially_copyable_v); + auto clone = std::make_unique(); + std::memcpy(clone.get(), &source, sizeof(DistTensorMap)); + return clone; +} + +void expect_exact_map_state(const DistTensorMap &actual, const DistTensorMap &expected) { + // expected 是调用前的逐字节副本。失败路径必须连同未触及 entry 与 ABI + // 保留区一起保持不变，防止满环检查后仍误写 slot 或 cursor。 + EXPECT_EQ(std::memcmp(&actual, &expected, sizeof(DistTensorMap)), 0); +} + +Tensor make_region(uint64_t address) { + Tensor tensor{}; + tensor.buffer = {address, 4}; + tensor.start_offset = 0; + tensor.ndims = 1; + tensor.dtype = DataType::FLOAT32; + tensor.is_contiguous = true; + tensor.shapes[0] = 1; + tensor.extent_elem_cache = 1; + tensor.strides[0] = 1; + return tensor; +} + +Tensor make_region_in_another_bucket(uint64_t address) { + const uint32_t original_bucket = dist_private_tensor_map_hash(address); + for (uint64_t candidate = address + 64; candidate < address + (1ULL << 30); candidate += 64) { + if (dist_private_tensor_map_hash(candidate) != original_bucket) { + return make_region(candidate); + } + } + throw std::logic_error("failed to find a TensorMap address in another bucket"); +} + +Tensor make_region_in_bucket(uint32_t target_bucket, uint64_t seed) { + constexpr uint64_t kSearchSteps = 1ULL << 20; + for (uint64_t step = 0; step < kSearchSteps; ++step) { + const uint64_t candidate = seed + step * 64; + if (dist_private_tensor_map_hash(candidate) == target_bucket) { + return make_region(candidate); + } + } + throw std::logic_error("failed to find a TensorMap address in the requested bucket"); +} + +void fill_region_bucket(DistTensorMap &map, const Tensor &tensor, uint32_t count, int32_t first_producer = 0) { + for (uint32_t index = 0; index < count; ++index) { + ASSERT_TRUE(dist_private_tensor_map_insert(map, tensor, first_producer + static_cast(index))) + << "index=" << index << " count=" << count << " cap=" << kMapBucketCapacity; + } +} + +Tensor make_logical_test_tensor( + uint64_t address, uint64_t start_offset, uint32_t extent, DataType dtype = DataType::FLOAT32, + bool contiguous = true, uint32_t stride = 1 +) { + Tensor tensor{}; + tensor.buffer = {address, 1ULL << 32}; + tensor.owner_task_id = PTO2TaskId::invalid(); + tensor.start_offset = start_offset; + tensor.version = 0; + tensor.ndims = 1; + tensor.dtype = dtype; + tensor.manual_dep = false; + tensor.is_contiguous = contiguous; + tensor.child_memory = 0; + tensor.shapes[0] = extent; + tensor.strides[0] = stride; + tensor.extent_elem_cache = contiguous ? extent : 1 + static_cast(extent - 1) * stride; + return tensor; +} + +uint64_t reference_element_bytes(DataType dtype) { + // 独立列出 wire dtype 的元素宽度，不调用 production get_element_size()， + // 避免被测 byte-range helper 与 reference 共享同一个错误。 + switch (dtype) { + case DataType::FLOAT32: + case DataType::INT32: + case DataType::UINT32: + return 4; + case DataType::FLOAT16: + case DataType::INT16: + case DataType::BFLOAT16: + case DataType::UINT16: + return 2; + case DataType::INT8: + case DataType::UINT8: + case DataType::BOOL: + return 1; + case DataType::INT64: + case DataType::UINT64: + return 8; + case DataType::DATA_TYPE_NUM: + break; + } + throw std::logic_error("unexpected dtype in TensorMap logical reference"); +} + +struct LogicalByteRange { + uint64_t address; + uint64_t lo; + uint64_t hi; +}; + +LogicalByteRange reference_byte_range(const Tensor &tensor) { + uint64_t extent = tensor.extent_elem_cache; + if (tensor.is_contiguous) { + extent = 1; + for (uint32_t dimension = 0; dimension < tensor.ndims; ++dimension) { + extent *= tensor.shapes[dimension]; + } + } + const uint64_t element_bytes = reference_element_bytes(tensor.dtype); + return { + tensor.buffer.addr, + tensor.start_offset * element_bytes, + (tensor.start_offset + extent) * element_bytes, + }; +} + +bool logical_ranges_overlap(const LogicalByteRange &left, const LogicalByteRange &right) { + return left.address == right.address && left.lo < right.hi && right.lo < left.hi; +} + +struct LogicalReferenceEntry { + LogicalByteRange range; + int32_t producer; +}; + +class LogicalReferenceMap { +public: + void advance(int32_t task_id, int32_t history) { + const int32_t floor = task_id - history; + if (floor <= alive_floor_) { + return; + } + alive_floor_ = floor; + entries_.erase( + std::remove_if( + entries_.begin(), entries_.end(), + [floor](const LogicalReferenceEntry &entry) { return entry.producer < floor; } + ), + entries_.end() + ); + } + + void insert(const Tensor &tensor, int32_t producer) { + entries_.push_back({reference_byte_range(tensor), producer}); + } + + int32_t lookup(const Tensor &tensor) const { + const LogicalByteRange query = reference_byte_range(tensor); + int32_t best = -1; + for (const LogicalReferenceEntry &entry : entries_) { + if (entry.producer >= alive_floor_ && logical_ranges_overlap(entry.range, query)) { + best = std::max(best, entry.producer); + } + } + return best; + } + +private: + std::vector entries_; + int32_t alive_floor_ = 0; +}; + +void expect_logical_lookup_matches( + DistTensorMap &actual, const LogicalReferenceMap &reference, const Tensor &query, const char *context, + int32_t task_id = -1 +) { + EXPECT_EQ(dist_private_tensor_map_lookup(actual, query), reference.lookup(query)) + << "context=" << context << " task_id=" << task_id; +} + +Tensor make_random_logical_tensor(std::mt19937_64 &random) { + constexpr std::array kDtypes = { + DataType::FLOAT32, + DataType::FLOAT16, + DataType::UINT8, + DataType::INT64, + }; + const uint64_t buffer_index = random() % 48; + const uint64_t address = 0x800000000ULL + buffer_index * 0x100000ULL; + const uint64_t start_offset = random() % 96; + const uint32_t extent = 1 + static_cast(random() % 12); + const DataType dtype = kDtypes[random() % kDtypes.size()]; + const bool contiguous = (random() & 3U) != 0; + const uint32_t stride = contiguous ? 1 : 2 + static_cast(random() % 4); + return make_logical_test_tensor(address, start_offset, extent, dtype, contiguous, stride); +} + +TEST(FdwicTensorMapLogical, EmptyAndHalfOpenRangesMatchIndependentReference) { + auto actual = make_empty_map(); + LogicalReferenceMap reference; + const Tensor left = make_logical_test_tensor(0x100000000ULL, 0, 4); + const Tensor touching = make_logical_test_tensor(0x100000000ULL, 4, 4); + const Tensor overlap = make_logical_test_tensor(0x100000000ULL, 3, 2); + const Tensor different_buffer = make_logical_test_tensor(0x100100000ULL, 0, 4); + + expect_logical_lookup_matches(*actual, reference, left, "empty"); + EXPECT_EQ(reference.lookup(left), -1); + + ASSERT_TRUE(dist_private_tensor_map_insert(*actual, left, 2)); + reference.insert(left, 2); + expect_logical_lookup_matches(*actual, reference, touching, "touching-half-open"); + expect_logical_lookup_matches(*actual, reference, overlap, "overlap"); + expect_logical_lookup_matches(*actual, reference, different_buffer, "different-buffer"); + EXPECT_EQ(reference.lookup(touching), -1); + EXPECT_EQ(reference.lookup(overlap), 2); + EXPECT_EQ(reference.lookup(different_buffer), -1); +} + +TEST(FdwicTensorMapLogical, LookupSelectsMaximumOverlappingProducer) { + auto actual = make_empty_map(); + LogicalReferenceMap reference; + const uint64_t address = 0x200000000ULL; + const Tensor producer_3 = make_logical_test_tensor(address, 0, 8); + const Tensor producer_5 = make_logical_test_tensor(address, 2, 4); + const Tensor producer_7 = make_logical_test_tensor(address, 3, 2); + const Tensor query = make_logical_test_tensor(address, 3, 1); + + for (const auto &[tensor, producer] : + std::array, 3>{ + std::pair{&producer_3, 3}, + std::pair{&producer_5, 5}, + std::pair{&producer_7, 7}, + }) { + ASSERT_TRUE(dist_private_tensor_map_insert(*actual, *tensor, producer)); + reference.insert(*tensor, producer); + } + + expect_logical_lookup_matches(*actual, reference, query, "maximum-overlapping-producer"); + EXPECT_EQ(reference.lookup(query), 7); +} + +TEST(FdwicTensorMapLogical, HistoryFloorIsHalfOpenAndMonotonic) { + auto actual = make_empty_map(); + LogicalReferenceMap reference; + const Tensor producer_9 = make_logical_test_tensor(0x300000000ULL, 0, 1); + const Tensor producer_10 = make_logical_test_tensor(0x300000000ULL, 2, 1); + + ASSERT_TRUE(dist_private_tensor_map_insert(*actual, producer_9, 9)); + ASSERT_TRUE(dist_private_tensor_map_insert(*actual, producer_10, 10)); + reference.insert(producer_9, 9); + reference.insert(producer_10, 10); + + dist_private_tensor_map_advance_retire(*actual, 20, 10); + reference.advance(20, 10); + expect_logical_lookup_matches(*actual, reference, producer_9, "below-floor"); + expect_logical_lookup_matches(*actual, reference, producer_10, "at-floor"); + EXPECT_EQ(reference.lookup(producer_9), -1); + EXPECT_EQ(reference.lookup(producer_10), 10); + + dist_private_tensor_map_advance_retire(*actual, 20, 10); + reference.advance(20, 10); + dist_private_tensor_map_advance_retire(*actual, 19, 10); + reference.advance(19, 10); + expect_logical_lookup_matches(*actual, reference, producer_10, "repeated-and-lower-floor"); + EXPECT_EQ(reference.lookup(producer_10), 10); + + dist_private_tensor_map_advance_retire(*actual, 21, 10); + reference.advance(21, 10); + expect_logical_lookup_matches(*actual, reference, producer_10, "after-floor-advance"); + EXPECT_EQ(reference.lookup(producer_10), -1); +} + +TEST(FdwicTensorMapRing, SameBucketProducerContractAcceptsEqualAndIncreasingAndRejectsDecreasing) { + auto map = make_empty_map(); + const Tensor region = make_region(0x380000); + + ASSERT_TRUE(dist_private_tensor_map_insert(*map, region, 3)); + ASSERT_TRUE(dist_private_tensor_map_insert(*map, region, 3)); + ASSERT_TRUE(dist_private_tensor_map_insert(*map, region, 4)); + EXPECT_EQ(dist_private_tensor_map_lookup(*map, region), 4); + + const auto before_decreasing_insert = clone_map_bytes(*map); + EXPECT_THROW( + (void)dist_private_tensor_map_insert(*map, region, 2), + std::logic_error + ); + expect_exact_map_state(*map, *before_decreasing_insert); +} + +TEST(FdwicTensorMapLogical, FixedSeedTwelveThousandTasksMatchIndependentReference) { + constexpr uint64_t kSeed = 0x5041524F445631ULL; // "PARODV1" + constexpr int32_t kHistory = 15; + constexpr int32_t kTasks = 12000; + constexpr size_t kRecentCapacity = static_cast(kHistory + 1); + + auto actual = make_empty_map(); + LogicalReferenceMap reference; + std::mt19937_64 random(kSeed); + std::array recent{}; + size_t recent_count = 0; + + for (int32_t task_id = 0; task_id < kTasks; ++task_id) { + dist_private_tensor_map_advance_retire(*actual, task_id, kHistory); + reference.advance(task_id, kHistory); + + const size_t slot = static_cast(task_id) % recent.size(); + if (task_id >= static_cast(recent.size())) { + expect_logical_lookup_matches(*actual, reference, recent[slot], "retired-slot-before-reuse", task_id); + } + + const Tensor inserted = make_random_logical_tensor(random); + ASSERT_TRUE(dist_private_tensor_map_insert(*actual, inserted, task_id)) + << "task_id=" << task_id + << " workload keeps at most " << kRecentCapacity << " globally live entries"; + reference.insert(inserted, task_id); + recent[slot] = inserted; + recent_count = std::min(recent_count + 1, recent.size()); + + expect_logical_lookup_matches(*actual, reference, inserted, "just-inserted", task_id); + const size_t recent_slot = static_cast(random() % recent_count); + expect_logical_lookup_matches(*actual, reference, recent[recent_slot], "recent-probe", task_id); + const Tensor random_query = make_random_logical_tensor(random); + expect_logical_lookup_matches(*actual, reference, random_query, "random-query", task_id); + } +} + +TEST(FdwicTensorMapRing, ResetAndPhysicalBoundariesMatchTheFixedPool) { + auto map = make_empty_map(); + + EXPECT_EQ(sizeof(DistTensorMap), 823312U); + EXPECT_EQ(sizeof(MapEntry), 48U); + EXPECT_EQ(kMapBuckets * kMapBucketCapacity, static_cast(kMapCap)); + EXPECT_EQ(dist_private_tensor_map_slot_index(0, 0), 0U); + EXPECT_EQ( + dist_private_tensor_map_slot_index(kMapBuckets - 1, kMapBucketCapacity - 1), + static_cast(kMapCap - 1) + ); + EXPECT_EQ( + dist_private_tensor_map_slot_index(kMapBuckets - 1, kMapBucketCapacity), + (kMapBuckets - 1) * kMapBucketCapacity + ); + EXPECT_EQ(map->alive_floor, 0); + for (uint32_t bucket = 0; bucket < kMapBuckets; ++bucket) { + EXPECT_EQ(dist_private_tensor_map_load_head(*map, bucket), 0U); + EXPECT_EQ(dist_private_tensor_map_load_tail(*map, bucket), 0U); + } +} + +TEST(FdwicTensorMapRing, PrepareMapMovesOnlyTheFloorAndTouchedBucketsRetireLazily) { + auto map = make_empty_map(); + const Tensor region = make_region(0x100000); + ASSERT_TRUE(dist_private_tensor_map_insert(*map, region, 9)); + ASSERT_TRUE(dist_private_tensor_map_insert(*map, region, 10)); + const uint32_t bucket = dist_private_tensor_map_hash(region.buffer.addr); + + dist_private_tensor_map_advance_retire(*map, 20, 10); + EXPECT_EQ(map->alive_floor, 10); + EXPECT_EQ(dist_private_tensor_map_load_head(*map, bucket), 0U); + EXPECT_EQ(dist_private_tensor_map_lookup(*map, region), 10); + EXPECT_EQ(dist_private_tensor_map_load_head(*map, bucket), 1U); + + dist_private_tensor_map_advance_retire(*map, 20, 10); + dist_private_tensor_map_advance_retire(*map, 19, 10); + EXPECT_EQ(map->alive_floor, 10); + EXPECT_EQ(dist_private_tensor_map_load_head(*map, bucket), 1U); + + dist_private_tensor_map_advance_retire(*map, 21, 10); + EXPECT_EQ(map->alive_floor, 11); + EXPECT_EQ(dist_private_tensor_map_load_head(*map, bucket), 1U); + EXPECT_EQ(dist_private_tensor_map_lookup(*map, region), -1); + EXPECT_EQ(dist_private_tensor_map_load_head(*map, bucket), 2U); +} + +TEST(FdwicTensorMapRing, DifferentBuffersInTheSameBucketRemainLogicallyIndependent) { + auto map = make_empty_map(); + const Tensor first = make_region(0x180000); + const uint32_t bucket = dist_private_tensor_map_hash(first.buffer.addr); + const Tensor second = make_region_in_bucket(bucket, 0x700000000ULL); + ASSERT_NE(first.buffer.addr, second.buffer.addr); + ASSERT_EQ(dist_private_tensor_map_hash(second.buffer.addr), bucket); + + ASSERT_TRUE(dist_private_tensor_map_insert(*map, first, 7)); + ASSERT_TRUE(dist_private_tensor_map_insert(*map, second, 8)); + EXPECT_EQ(dist_private_tensor_map_lookup(*map, first), 7); + EXPECT_EQ(dist_private_tensor_map_lookup(*map, second), 8); + + ASSERT_TRUE(dist_private_tensor_map_insert(*map, first, 9)); + EXPECT_EQ(dist_private_tensor_map_lookup(*map, first), 9); + EXPECT_EQ(dist_private_tensor_map_lookup(*map, second), 8); +} + +TEST(FdwicTensorMapRing, FullyRetiredBucketReusesANonzeroCursorWithoutExposingStaleSlots) { + auto map = make_empty_map(); + const Tensor region = make_region(0x1c0000); + const uint32_t bucket = dist_private_tensor_map_hash(region.buffer.addr); + fill_region_bucket(*map, region, kMapBucketCapacity); + + dist_private_tensor_map_advance_retire( + *map, static_cast(kMapBucketCapacity), /*history=*/0 + ); + EXPECT_EQ(dist_private_tensor_map_load_head(*map, bucket), 0U); + EXPECT_EQ(dist_private_tensor_map_lookup(*map, region), -1); + EXPECT_EQ(dist_private_tensor_map_load_head(*map, bucket), kMapBucketCapacity); + EXPECT_EQ(dist_private_tensor_map_load_tail(*map, bucket), kMapBucketCapacity); + + ASSERT_TRUE( + dist_private_tensor_map_insert( + *map, region, static_cast(kMapBucketCapacity) + ) + ); + EXPECT_EQ(dist_private_tensor_map_load_head(*map, bucket), kMapBucketCapacity); + EXPECT_EQ(dist_private_tensor_map_load_tail(*map, bucket), kMapBucketCapacity + 1); + EXPECT_EQ( + map->entries[dist_private_tensor_map_slot_index(bucket, kMapBucketCapacity)].producer, + static_cast(kMapBucketCapacity) + ); + EXPECT_EQ( + dist_private_tensor_map_lookup(*map, region), + static_cast(kMapBucketCapacity) + ); +} + +TEST(FdwicTensorMapRing, FullBucketRejectsWithoutMutationAndDoesNotBlockAnotherBucket) { + auto map = make_empty_map(); + const Tensor full = make_region(0x200000); + const uint32_t full_bucket = dist_private_tensor_map_hash(full.buffer.addr); + fill_region_bucket(*map, full, kMapBucketCapacity); + EXPECT_EQ(dist_private_tensor_map_load_tail(*map, full_bucket), kMapBucketCapacity); + + const auto full_state = clone_map_bytes(*map); + EXPECT_FALSE(dist_private_tensor_map_insert(*map, full, static_cast(kMapBucketCapacity))); + expect_exact_map_state(*map, *full_state); + + if constexpr (kMapBuckets > 1) { + const uint32_t full_slot_begin = full_bucket * kMapBucketCapacity; + std::vector full_entries_before(kMapBucketCapacity); + std::memcpy( + full_entries_before.data(), &map->entries[full_slot_begin], + full_entries_before.size() * sizeof(MapEntry) + ); + const Tensor independent = make_region_in_another_bucket(full.buffer.addr); + const uint32_t independent_bucket = dist_private_tensor_map_hash(independent.buffer.addr); + ASSERT_NE(independent_bucket, full_bucket); + ASSERT_TRUE( + dist_private_tensor_map_insert(*map, independent, static_cast(kMapBucketCapacity + 1)) + ); + EXPECT_EQ(dist_private_tensor_map_load_head(*map, full_bucket), 0U); + EXPECT_EQ(dist_private_tensor_map_load_tail(*map, full_bucket), kMapBucketCapacity); + EXPECT_EQ( + std::memcmp( + full_entries_before.data(), &map->entries[full_slot_begin], + full_entries_before.size() * sizeof(MapEntry) + ), + 0 + ); + EXPECT_EQ(dist_private_tensor_map_lookup(*map, independent), static_cast(kMapBucketCapacity + 1)); + } +} + +TEST(FdwicTensorMapRing, BaseAndExtraBucketControlBoundaryIsIsolated) { + if constexpr (kMapBuckets > kMapBaseControlBuckets) { + constexpr uint32_t kLastBaseBucket = kMapBaseControlBuckets - 1; + constexpr uint32_t kFirstExtraBucket = kMapBaseControlBuckets; + const Tensor base_region = make_region_in_bucket(kLastBaseBucket, 0x500000000ULL); + const Tensor extra_region = make_region_in_bucket(kFirstExtraBucket, 0x600000000ULL); + auto map = make_empty_map(); + + ASSERT_EQ(dist_private_tensor_map_hash(base_region.buffer.addr), kLastBaseBucket); + ASSERT_EQ(dist_private_tensor_map_hash(extra_region.buffer.addr), kFirstExtraBucket); + ASSERT_TRUE(dist_private_tensor_map_insert(*map, base_region, 1)); + ASSERT_TRUE(dist_private_tensor_map_insert(*map, base_region, 2)); + ASSERT_TRUE(dist_private_tensor_map_insert(*map, extra_region, 1)); + + EXPECT_EQ(dist_private_tensor_map_load_head(*map, kLastBaseBucket), 0U); + EXPECT_EQ(dist_private_tensor_map_load_tail(*map, kLastBaseBucket), 2U); + EXPECT_EQ(dist_private_tensor_map_load_head(*map, kFirstExtraBucket), 0U); + EXPECT_EQ(dist_private_tensor_map_load_tail(*map, kFirstExtraBucket), 1U); + + const uint32_t base_slot = dist_private_tensor_map_slot_index(kLastBaseBucket, 0); + const uint32_t extra_slot = dist_private_tensor_map_slot_index(kFirstExtraBucket, 0); + ASSERT_NE(base_slot, extra_slot); + EXPECT_EQ(map->entries[base_slot].buf_addr, base_region.buffer.addr); + EXPECT_EQ(map->entries[base_slot].producer, 1); + EXPECT_EQ(map->entries[base_slot + 1].buf_addr, base_region.buffer.addr); + EXPECT_EQ(map->entries[base_slot + 1].producer, 2); + EXPECT_EQ(map->entries[extra_slot].buf_addr, extra_region.buffer.addr); + EXPECT_EQ(map->entries[extra_slot].producer, 1); + EXPECT_EQ(dist_private_tensor_map_lookup(*map, base_region), 2); + EXPECT_EQ(dist_private_tensor_map_lookup(*map, extra_region), 1); + } +} + +TEST(FdwicTensorMapRing, RetiredSlotsWrapForThreeLapsWithoutExposingOldValues) { + auto map = make_empty_map(); + const Tensor region = make_region(0x300000); + const uint32_t bucket = dist_private_tensor_map_hash(region.buffer.addr); + fill_region_bucket(*map, region, kMapBucketCapacity); + + for (uint32_t lap = 1; lap <= 3; ++lap) { + const uint32_t producer_base = lap * kMapBucketCapacity; + for (uint32_t offset = 0; offset < kMapBucketCapacity; ++offset) { + const uint32_t producer = producer_base + offset; + const uint32_t new_floor = producer - kMapBucketCapacity + 1; + dist_private_tensor_map_advance_retire(*map, static_cast(new_floor), 0); + ASSERT_TRUE(dist_private_tensor_map_insert(*map, region, static_cast(producer))) + << "lap=" << lap << " offset=" << offset; + } + + const uint64_t expected_head = static_cast(lap) * kMapBucketCapacity; + const uint64_t expected_tail = static_cast(lap + 1) * kMapBucketCapacity; + EXPECT_EQ(dist_private_tensor_map_load_head(*map, bucket), expected_head); + EXPECT_EQ(dist_private_tensor_map_load_tail(*map, bucket), expected_tail); + EXPECT_EQ( + dist_private_tensor_map_lookup(*map, region), + static_cast(expected_tail - 1) + ); + for (uint32_t slot_offset = 0; slot_offset < kMapBucketCapacity; ++slot_offset) { + const uint32_t physical_slot = dist_private_tensor_map_slot_index(bucket, slot_offset); + EXPECT_EQ(map->entries[physical_slot].buf_addr, region.buffer.addr) + << "lap=" << lap << " slot_offset=" << slot_offset; + EXPECT_EQ(map->entries[physical_slot].producer, static_cast(producer_base + slot_offset)) + << "lap=" << lap << " slot_offset=" << slot_offset; + } + } +} + +TEST(FdwicTensorMapRing, FacadePropagatesPerBucketCapacityFailureWithoutMutation) { + auto worker = std::make_unique(); + dist_tensor_map_reset_worker(*worker); + const Tensor tensor = make_region(0x400000); + fill_region_bucket(worker->map, tensor, kMapBucketCapacity); + const auto full_state = clone_map_bytes(worker->map); + + EXPECT_FALSE(dist_tensor_map_insert_for_task(*worker, tensor, 10, /*task_won=*/true)); + expect_exact_map_state(worker->map, *full_state); +} + +} // namespace diff --git a/tests/ut/py/test_build_runtimes.py b/tests/ut/py/test_build_runtimes.py new file mode 100644 index 0000000000..42f3604fa1 --- /dev/null +++ b/tests/ut/py/test_build_runtimes.py @@ -0,0 +1,137 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Tests for the install-time runtime pre-build entry point.""" + +import sys + +import pytest + + +def test_shared_environment_does_not_change_install_default(monkeypatch): + from simpler_setup import build_runtimes # noqa: PLC0415 + + runtimes = { + "a2a3": ["fully_distributed_within_core", "host_build_graph", "tensormap_and_ringbuffer"], + "a5": ["fully_distributed_within_core", "host_build_graph"], + } + monkeypatch.setattr(build_runtimes, "discover_runtimes", lambda arch: runtimes[arch]) + monkeypatch.setenv("PTO_FDWIC_TENSORMAP_MODE", "shared") + + tasks = build_runtimes._collect_runtime_build_tasks(["a2a3sim", "a5sim"], None) + + assert tasks == [ + ("a2a3sim", "fully_distributed_within_core", "private"), + ("a2a3sim", "host_build_graph", "private"), + ("a2a3sim", "tensormap_and_ringbuffer", "private"), + ("a5sim", "fully_distributed_within_core", "private"), + ("a5sim", "host_build_graph", "private"), + ] + + +def test_explicit_modes_collect_both_fdwic_artifact_families_once(monkeypatch): + from simpler_setup import build_runtimes # noqa: PLC0415 + + monkeypatch.setattr( + build_runtimes, + "discover_runtimes", + lambda _arch: ["fully_distributed_within_core", "host_build_graph"], + ) + monkeypatch.setenv("PTO_FDWIC_TENSORMAP_MODE", "shared") + + tasks = build_runtimes._collect_runtime_build_tasks( + ["a5sim"], + ["private", "shared", "private"], + ) + + assert tasks == [ + ("a5sim", "fully_distributed_within_core", "private"), + ("a5sim", "fully_distributed_within_core", "shared"), + ("a5sim", "host_build_graph", "private"), + ] + + +@pytest.mark.parametrize("modes", [[], ["typo"], "shared"]) +def test_explicit_modes_reject_invalid_api_values(modes): + from simpler_setup import build_runtimes # noqa: PLC0415 + + with pytest.raises(ValueError, match="fdwic_tensormap_modes|Invalid FDWIC TensorMap mode"): + build_runtimes._normalize_fdwic_tensormap_modes(modes) + + +def test_build_all_passes_mode_only_to_matching_runtime(tmp_path, monkeypatch): + from simpler_setup import build_runtimes # noqa: PLC0415 + + calls = [] + + class _FakeRuntimeBuilder: + _LIB_DIR = None + _CACHE_DIR = None + + def __init__(self, platform, fdwic_tensormap_mode): + self.platform = platform + self.mode = fdwic_tensormap_mode + + def ensure_simpler_log(self, build): + calls.append(("simpler_log", self.platform, self.mode, build)) + + def ensure_sim_context(self, build): + calls.append(("sim_context", self.platform, self.mode, build)) + + def get_binaries(self, runtime_name, build): + calls.append((runtime_name, self.platform, self.mode, build)) + + monkeypatch.setattr(build_runtimes, "RuntimeBuilder", _FakeRuntimeBuilder) + monkeypatch.setattr( + build_runtimes, + "discover_runtimes", + lambda _arch: ["fully_distributed_within_core", "host_build_graph"], + ) + monkeypatch.setenv("PTO_FDWIC_TENSORMAP_MODE", "shared") + + build_runtimes.build_all( + lib_dir=tmp_path / "lib", + cache_dir=tmp_path / "cache", + platforms=["a5sim"], + fdwic_tensormap_modes=["private", "shared"], + ) + + assert ("fully_distributed_within_core", "a5sim", "private", True) in calls + assert ("fully_distributed_within_core", "a5sim", "shared", True) in calls + assert calls.count(("host_build_graph", "a5sim", "private", True)) == 1 + assert ("simpler_log", "a5sim", "private", True) in calls + assert ("sim_context", "a5sim", "private", True) in calls + + +def test_cli_forwards_repeatable_fdwic_modes(tmp_path, monkeypatch): + from simpler_setup import build_runtimes # noqa: PLC0415 + + observed = {} + monkeypatch.setattr(build_runtimes, "build_all", lambda **kwargs: observed.update(kwargs)) + monkeypatch.setattr( + sys, + "argv", + [ + "build_runtimes.py", + "--lib-dir", + str(tmp_path / "lib"), + "--cache-dir", + str(tmp_path / "cache"), + "--platforms", + "a5sim", + "--fdwic-tensormap", + "private", + "--fdwic-tensormap", + "shared", + ], + ) + + build_runtimes.main() + + assert observed["platforms"] == ["a5sim"] + assert observed["fdwic_tensormap_modes"] == ["private", "shared"] diff --git a/tests/ut/py/test_fdwic_submit_pmu_report.py b/tests/ut/py/test_fdwic_submit_pmu_report.py new file mode 100644 index 0000000000..b4e1b04a30 --- /dev/null +++ b/tests/ut/py/test_fdwic_submit_pmu_report.py @@ -0,0 +1,2859 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Regression tests for the production FDWIC Submit-PMU raw-to-HTML path.""" + +from __future__ import annotations + +import hashlib +import json +import struct +from pathlib import Path +from typing import Any, Callable + +import pytest + +import simpler_setup.tools.fdwic_submit_pmu_report as report_module +from simpler_setup.fdwic_build_config import FDWIC_TENSORMAP_RING_CAP +from simpler_setup.tools.fdwic_submit_pmu_report import ( + ALLOC_COMPLETE_CAPTURE_MODE, + ARG_BUILD_CAPTURE_MODE, + CLAIM_CAPTURE_MODE, + COMMON_REQUIRED_STATUS_MASK, + DEFAULT_INPUT_NAME, + DEFAULT_OUTPUT_NAME, + DEFAULT_PROVENANCE_NAME, + EFDRAIN_CONTROL_CAPTURE_MODE, + EMPTY_BRACKET_CAPTURE_MODE, + FANIN_CAPTURE_MODE, + LOSER_REPLAY_CAPTURE_MODE, + MATERIALIZE_CAPTURE_MODE, + NONE_REQUIRED_STATUS_MASK, + PHASE_REQUIRED_STATUS_MASK, + PREPARE_MAP_CAPTURE_MODE, + REGISTER_CAPTURE_MODE, + SUBMIT_TRANSITION_CAPTURE_MODE, + WINNER_BUILD_CAPTURE_MODE, + build_phase_recording_cost_reference, + capture_build_identity, + load_capture, + load_provenance, + render_report, + write_report, + write_report_with_provenance, +) + + +def _physical_layout() -> tuple[list[int], list[int]]: + aic = [*range(16), *range(54, 70)] + aiv: list[int] = [] + for physical_id in aic: + die_base = physical_id // 54 * 54 + local = physical_id % 54 + aiv.extend((die_base + 18 + 2 * local, die_base + 18 + 2 * local + 1)) + return aic, aiv + + +def _bitmap_words(physical_ids: list[int]) -> list[int]: + words = [0, 0, 0, 0] + for physical_id in physical_ids: + words[physical_id // 32] |= 1 << (physical_id % 32) + return words + + +def _metric(values: list[int]) -> dict[str, int | float]: + return { + "sum": sum(values), + "min": min(values), + "mean": sum(values) / len(values), + "max": max(values), + } + + +def _group_summary(records: list[dict[str, Any]]) -> dict[str, Any]: + total = [record["total_cycles"] for record in records] + scalar_elapsed = [record["scalar_submit_elapsed_ticks"] for record in records] + scalar = [record["scalar_busy"] for record in records] + requests = [record["icache_requests"] for record in records] + misses = [record["icache_misses"] for record in records] + return { + "cores": len(records), + "total_cycles": _metric(total), + "scalar_submit_elapsed_ticks": _metric(scalar_elapsed), + "scalar_busy": _metric(scalar), + "icache_requests": _metric(requests), + "icache_misses": _metric(misses), + "scalar_busy_share": sum(scalar) / sum(total), + "icache_miss_rate": sum(misses) / sum(requests), + } + + +def _refresh_host_aggregates(capture: dict[str, Any]) -> None: + """Refresh only producer-owned fields after a test mutates records.""" + + records = capture["records"] + capture["summary"] = { + "all": _group_summary(records), + "aic": _group_summary(records[:32]), + "aiv": _group_summary(records[32:]), + } + first_tick = min(record["first_submit_start_tick"] for record in records) + last_tick = max(record["last_submit_end_tick"] for record in records) + capture["window"].update( + { + "global_first_submit_start_tick": first_tick, + "global_last_submit_end_tick": last_tick, + "global_submit_span_ticks": last_tick - first_tick, + "global_submit_span_us": (last_tick - first_tick) / 1_000, + } + ) + + +def _valid_capture() -> dict[str, Any]: + aic_physical, aiv_physical = _physical_layout() + records: list[dict[str, Any]] = [] + for logical_core_id in range(96): + is_aic = logical_core_id < 32 + if is_aic: + physical_core_id = aic_physical[logical_core_id] + block_id = logical_core_id + lane = 0 + role = "aic" + else: + aiv_ordinal = logical_core_id - 32 + physical_core_id = aiv_physical[aiv_ordinal] + block_id = aiv_ordinal // 2 + lane = 1 + aiv_ordinal % 2 + role = "aiv" + start = 1_000 + logical_core_id * 3 + end = 11_000 + logical_core_id * 3 + requests = 2_000 + logical_core_id + misses = 200 + logical_core_id % 5 + records.append( + { + "logical_core_id": logical_core_id, + "physical_core_id": physical_core_id, + "role": role, + "block_id": block_id, + "lane": lane, + "submit_count": 5, + "expected_submit_count": 5, + "first_submit_start_tick": start, + "last_submit_end_tick": end, + "submit_elapsed_ticks": end - start, + "scalar_submit_elapsed_ticks": end - start - 1_000, + "total_cycles": 16_000 + logical_core_id, + "scalar_busy": 12_000 + logical_core_id, + "icache_requests": requests, + "icache_misses": misses, + "status": NONE_REQUIRED_STATUS_MASK, + } + ) + + all_physical = aic_physical + aiv_physical + grouped = { + "all": records, + "aic": records[:32], + "aiv": records[32:], + } + first_tick = min(record["first_submit_start_tick"] for record in records) + last_tick = max(record["last_submit_end_tick"] for record in records) + return { + "schema": "fdwic-submit-pmu-v3", + "capture": { + "mode": "submit-pmu-none", + "window_scope": "per_core_first_submit_begin_to_last_submit_end", + "accepted": True, + "owner_restore_passed": True, + }, + "configuration": { + "num_cores": 96, + "aic_cores": 32, + "aiv_cores": 64, + "expected_submits_per_core": 5, + "sys_counter_tick_ns": 1, + "selectors": { + "cnt0_vector_busy": 0x501, + "cnt1_cube_busy": 0x301, + "cnt2_scalar_busy": 0x001, + "cnt3_shadow_scalar_busy": 0x001, + "cnt5_shadow_icache_miss": 0x035, + "cnt6_primary_icache_request": 0x034, + "cnt7_primary_icache_miss": 0x035, + "cnt8_shadow_icache_request": 0x034, + }, + "status_required_mask": NONE_REQUIRED_STATUS_MASK, + "counter_width_bits": {"total": 64, "programmable": 32}, + "programmable_counter_risk_threshold": (1 << 30) - 1, + "linked_kernel_exclusion": { + "enabled": True, + "boundary": "dist_aicore_call_slot_kernel_entry_to_return", + "gate_semantics": "metrics_prof_stop_before_call_and_start_after_return", + "time_denominator": "scalar_submit_elapsed_ticks", + "wall_tick_semantics": "first_submit_start_to_last_submit_end_closure_only", + }, + "return_ready_atomic_exclusion": { + "enabled": True, + "classification": "result_used_atomic_only", + "time_boundary": "sys_cnt_before_atomic_to_result_dependent_sys_cnt_after_return", + "counter_semantics": "pmu_counters_include_atomic_instruction_events", + "time_denominator_effect": "subtract_return_ready_atomic_elapsed", + }, + "pmu_cycles_per_ns": {"all": 1.649844, "aic": 1.650062, "aiv": 1.649731}, + }, + "owner": { + "configure_passed": True, + "restore_passed": True, + "configured_count": 96, + "configured_aic": 32, + "configured_aiv": 64, + "restored_count": 96, + "active_after_restore": 0, + "restore_failures": 0, + "configured_bitmap_words": _bitmap_words(all_physical), + "complete_mixed_triplets": 32, + }, + "window": { + "global_first_submit_start_tick": first_tick, + "global_last_submit_end_tick": last_tick, + "global_submit_span_ticks": last_tick - first_tick, + "global_submit_span_us": (last_tick - first_tick) / 1_000, + }, + "validation": { + "passed": True, + "trusted_records": 96, + "unique_physical_core_ids": 96, + "aic_records": 32, + "aiv_records": 64, + "mixed_triplets": 32, + "owner_bitmap_member_records": 96, + "status_match_records": 96, + "selector_match_records": 96, + "window_started_records": 96, + "window_stopped_records": 96, + "submit_count_closed_records": 96, + "scalar_le_total_records": 96, + "shadow_icache_primary_match_records": 96, + "shadow_scalar_primary_match_records": 96, + "icache_miss_le_request_records": 96, + "counter_below_risk_threshold_records": 96, + "linked_kernel_gate_closed_records": 96, + "scalar_submit_elapsed_valid_records": 96, + "vector_busy_zero_records": 96, + "cube_busy_zero_records": 96, + "return_ready_atomic_time_valid_records": 96, + }, + "records": records, + "summary": {name: _group_summary(group) for name, group in grouped.items()}, + } + + +def _valid_arg_build_capture() -> dict[str, Any]: + capture = _valid_capture() + capture["capture"]["mode"] = ARG_BUILD_CAPTURE_MODE + capture["configuration"]["status_required_mask"] = COMMON_REQUIRED_STATUS_MASK + capture["configuration"]["phase"] = { + "id": 1, + "name": "arg-build", + "boundary": "claim_end_to_materialize_begin", + "expected_calls_per_core": 5, + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + "pmu_observation": dict(report_module.PHASE_PMU_OBSERVATION), + } + for logical_core_id, record in enumerate(capture["records"]): + record["status"] = COMMON_REQUIRED_STATUS_MASK + record["shadow_scalar_busy"] = record["scalar_busy"] - 20 - logical_core_id % 3 + record["shadow_icache_requests"] = record["icache_requests"] - 20 - logical_core_id % 3 + record["shadow_icache_misses"] = record["icache_misses"] - 5 + record.update( + { + "phase_id": 1, + "phase_elapsed_ticks": 1_500 + logical_core_id, + "phase_total_cycles_observed": 3_000 + logical_core_id, + "phase_scalar_busy_observed": 2_100 + logical_core_id, + "phase_icache_requests_observed": 400 + logical_core_id, + "phase_icache_misses_observed": 40 + logical_core_id % 5, + "phase_begin_reads": 5, + "phase_end_reads": 5, + "phase_excluded_kernel_calls": 0, + "phase_status": PHASE_REQUIRED_STATUS_MASK, + } + ) + capture["validation"].pop("shadow_icache_primary_match_records") + capture["validation"].pop("shadow_scalar_primary_match_records") + capture["validation"].update( + { + "phase_boundary_closed_records": 96, + "phase_shape_match_records": 96, + "phase_icache_values_ordered_records": 96, + "phase_pmu_values_ordered_records": 96, + "phase_counter_reconstruction_valid_records": 96, + "phase_time_within_submit_records": 96, + "shadow_icache_primary_bounded_records": 96, + "shadow_scalar_primary_bounded_records": 96, + "phase_kernel_exclusion_closed_records": 96, + } + ) + return capture + + +def _valid_empty_bracket_capture() -> dict[str, Any]: + capture = _valid_arg_build_capture() + capture["capture"]["mode"] = EMPTY_BRACKET_CAPTURE_MODE + capture["configuration"]["phase"] = { + "id": 2, + "name": "empty-bracket", + "boundary": "claim_end_adjacent_empty_bracket", + "expected_calls_per_core": 5, + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_empty_bracket_calibration", + "time_semantics": "boundary_diagnostic_outer_sys_cnt_around_adjacent_observer_pair", + "pmu_observation": dict(report_module.PHASE_PMU_OBSERVATION), + } + for logical_core_id, record in enumerate(capture["records"]): + record["phase_id"] = 2 + record["phase_elapsed_ticks"] = 200 + logical_core_id + record["phase_icache_requests_observed"] = 30 + logical_core_id % 5 + record["phase_icache_misses_observed"] = 3 + logical_core_id % 2 + return capture + + +def _valid_materialize_capture() -> dict[str, Any]: + capture = _valid_arg_build_capture() + capture["capture"]["mode"] = MATERIALIZE_CAPTURE_MODE + capture["configuration"]["phase"] = { + "id": 3, + "name": "materialize", + "boundary": "materialize_begin_to_materialize_end", + "expected_calls_per_core": 5, + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + "pmu_observation": dict(report_module.PHASE_PMU_OBSERVATION), + } + for logical_core_id, record in enumerate(capture["records"]): + record["phase_id"] = 3 + record["phase_elapsed_ticks"] = 800 + logical_core_id + record["phase_icache_requests_observed"] = 500 + logical_core_id + record["phase_icache_misses_observed"] = 50 + logical_core_id % 5 + return capture + + +def _valid_prepare_map_capture() -> dict[str, Any]: + capture = _valid_arg_build_capture() + capture["capture"]["mode"] = PREPARE_MAP_CAPTURE_MODE + capture["configuration"]["phase"] = { + "id": 8, + "name": "prepare-map", + "boundary": "dist_submit_prepare_map_call_entry_to_return", + "expected_calls_per_core": 5, + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + "pmu_observation": dict(report_module.PHASE_PMU_OBSERVATION), + } + for logical_core_id, record in enumerate(capture["records"]): + record["phase_id"] = 8 + record["phase_elapsed_ticks"] = 300 + logical_core_id + record["phase_icache_requests_observed"] = 120 + logical_core_id + record["phase_icache_misses_observed"] = 12 + logical_core_id % 5 + return capture + + +def _valid_fanin_capture() -> dict[str, Any]: + capture = _valid_arg_build_capture() + capture["capture"]["mode"] = FANIN_CAPTURE_MODE + capture["configuration"]["phase"] = { + "id": 9, + "name": "fanin", + "boundary": "fanin_begin_to_fanin_end", + "call_shape": "dynamic_balanced", + "expected_calls": {"all": 4, "aic": 2, "aiv": 2}, + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + "pmu_observation": dict(report_module.PHASE_PMU_OBSERVATION), + } + capture["validation"]["phase_global_call_count_closed"] = True + active_calls = {0: 1, 1: 1, 32: 1, 33: 1} + for logical_core_id, record in enumerate(capture["records"]): + calls = active_calls.get(logical_core_id, 0) + record["phase_id"] = 9 + record["phase_elapsed_ticks"] = 75 * calls + record["phase_total_cycles_observed"] = 300 * calls + record["phase_scalar_busy_observed"] = 180 * calls + record["phase_icache_requests_observed"] = 40 * calls + record["phase_icache_misses_observed"] = 2 * calls + record["phase_begin_reads"] = calls + record["phase_end_reads"] = calls + return capture + + +def _valid_winner_build_capture() -> dict[str, Any]: + capture = _valid_fanin_capture() + capture["capture"]["mode"] = WINNER_BUILD_CAPTURE_MODE + capture["configuration"]["phase"].update( + { + "id": 10, + "name": "winner-build-control", + "boundary": "winner_build_begin_to_end_excluding_linked_kernel_calls", + "counter_semantics": "discontinuous_running_read_clear_excluding_linked_kernel_calls", + "time_semantics": "boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls", + "pmu_observation": dict(report_module.PHASE_PMU_OBSERVATION), + } + ) + capture["validation"]["phase_kernel_exclusion_closed_records"] = 96 + for logical_core_id, record in enumerate(capture["records"]): + excluded_kernel_calls = int(logical_core_id in {0, 32}) + record["phase_id"] = 10 + record["phase_excluded_kernel_calls"] = excluded_kernel_calls + record["phase_begin_reads"] += excluded_kernel_calls + record["phase_end_reads"] += excluded_kernel_calls + return capture + + +def _valid_alloc_complete_capture( + *, + winner_core_id: int = 0, + excluded_kernel_calls: int = 2, +) -> dict[str, Any]: + """Build one B1 AllocComplete call without constraining its AIC/AIV owner.""" + + capture = _valid_arg_build_capture() + capture["capture"]["mode"] = ALLOC_COMPLETE_CAPTURE_MODE + capture["configuration"]["phase"] = { + "id": 11, + "name": "alloc-complete-control", + "boundary": "alloc_complete_begin_to_end_excluding_linked_kernel_calls", + "call_shape": "dynamic_global", + "expected_calls": {"all": 1}, + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "discontinuous_running_read_clear_excluding_linked_kernel_calls", + "time_semantics": "boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls", + "pmu_observation": dict(report_module.PHASE_PMU_OBSERVATION), + } + capture["validation"]["phase_global_call_count_closed"] = True + for logical_core_id, record in enumerate(capture["records"]): + business_calls = int(logical_core_id == winner_core_id) + excluded_calls = excluded_kernel_calls if business_calls else 0 + record.update( + { + "phase_id": 11, + "phase_elapsed_ticks": 75 * business_calls, + "phase_total_cycles_observed": 300 * business_calls, + "phase_scalar_busy_observed": 180 * business_calls, + "phase_icache_requests_observed": 40 * business_calls, + "phase_icache_misses_observed": 2 * business_calls, + "phase_begin_reads": business_calls + excluded_calls, + "phase_end_reads": business_calls + excluded_calls, + "phase_excluded_kernel_calls": excluded_calls, + } + ) + return capture + + +def _valid_loser_replay_capture() -> dict[str, Any]: + """Build the exact B1 loser complement: four Kernel winners among 96 workers.""" + + capture = _valid_arg_build_capture() + capture["capture"]["mode"] = LOSER_REPLAY_CAPTURE_MODE + capture["configuration"]["phase"] = { + "id": 12, + "name": "loser-replay", + "boundary": "register_end_to_drain_block_won_return", + "call_shape": "dynamic_balanced", + "expected_calls": {"all": 380, "aic": 126, "aiv": 254}, + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + "pmu_observation": dict(report_module.PHASE_PMU_OBSERVATION), + } + capture["validation"]["phase_global_call_count_closed"] = True + winner_cores = {0, 1, 32, 33} + for logical_core_id, record in enumerate(capture["records"]): + calls = 3 if logical_core_id in winner_cores else 4 + record.update( + { + "phase_id": 12, + "phase_elapsed_ticks": 75 * calls, + "phase_total_cycles_observed": 300 * calls, + "phase_scalar_busy_observed": 180 * calls, + "phase_icache_requests_observed": 40 * calls, + "phase_icache_misses_observed": 2 * calls, + "phase_begin_reads": calls, + "phase_end_reads": calls, + "phase_excluded_kernel_calls": 0, + } + ) + return capture + + +def _valid_claim_capture() -> dict[str, Any]: + capture = _valid_arg_build_capture() + capture["capture"]["mode"] = CLAIM_CAPTURE_MODE + capture["configuration"]["phase"] = { + "id": 4, + "name": "claim", + "boundary": "claim_begin_to_claim_end", + "expected_calls_per_core": 5, + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + "pmu_observation": dict(report_module.PHASE_PMU_OBSERVATION), + } + for logical_core_id, record in enumerate(capture["records"]): + record["phase_id"] = 4 + record["phase_elapsed_ticks"] = 650 + logical_core_id + record["phase_icache_requests_observed"] = 200 + logical_core_id + record["phase_icache_misses_observed"] = 20 + logical_core_id % 5 + return capture + + +def _valid_register_capture() -> dict[str, Any]: + capture = _valid_arg_build_capture() + capture["capture"]["mode"] = REGISTER_CAPTURE_MODE + capture["configuration"]["phase"] = { + "id": 5, + "name": "register", + "boundary": "register_outputs_call_entry_to_return", + "expected_calls_per_core": 5, + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + "pmu_observation": dict(report_module.PHASE_PMU_OBSERVATION), + } + for logical_core_id, record in enumerate(capture["records"]): + record["phase_id"] = 5 + record["phase_elapsed_ticks"] = 400 + logical_core_id + record["phase_icache_requests_observed"] = 150 + logical_core_id + record["phase_icache_misses_observed"] = 15 + logical_core_id % 5 + return capture + + +def _valid_submit_transition_capture() -> dict[str, Any]: + capture = _valid_arg_build_capture() + capture["capture"]["mode"] = SUBMIT_TRANSITION_CAPTURE_MODE + capture["configuration"]["phase"] = { + "id": 6, + "name": "submit-transition", + "boundary": "previous_submit_end_to_next_submit_begin", + "expected_calls_per_core": 4, + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + "pmu_observation": dict(report_module.PHASE_PMU_OBSERVATION), + } + for logical_core_id, record in enumerate(capture["records"]): + record["phase_id"] = 6 + record["phase_elapsed_ticks"] = 600 + logical_core_id + record["phase_icache_requests_observed"] = 180 + logical_core_id + record["phase_icache_misses_observed"] = 18 + logical_core_id % 5 + record["phase_begin_reads"] = 4 + record["phase_end_reads"] = 4 + return capture + + +def _valid_efdrain_control_capture() -> dict[str, Any]: + capture = _valid_arg_build_capture() + capture["capture"]["mode"] = EFDRAIN_CONTROL_CAPTURE_MODE + capture["configuration"]["phase"] = { + "id": 7, + "name": "efdrain-control", + "boundary": "efdrain_begin_to_end_excluding_linked_kernel_calls", + "expected_calls_per_core": 5, + "status_required_mask": PHASE_REQUIRED_STATUS_MASK, + "counter_semantics": "discontinuous_running_read_clear_excluding_linked_kernel_calls", + "time_semantics": "boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls", + "pmu_observation": dict(report_module.PHASE_PMU_OBSERVATION), + } + capture["validation"]["phase_kernel_exclusion_closed_records"] = 96 + for logical_core_id, record in enumerate(capture["records"]): + excluded_kernel_calls = logical_core_id % 3 + record["phase_id"] = 7 + record["phase_elapsed_ticks"] = 900 + logical_core_id + record["phase_icache_requests_observed"] = 250 + logical_core_id + record["phase_icache_misses_observed"] = 25 + logical_core_id % 5 + record["phase_excluded_kernel_calls"] = excluded_kernel_calls + record["phase_begin_reads"] = 5 + excluded_kernel_calls + record["phase_end_reads"] = 5 + excluded_kernel_calls + return capture + + +def _write_capture(directory: Path, capture: dict[str, Any]) -> Path: + path = directory / DEFAULT_INPUT_NAME + path.write_text(json.dumps(capture, indent=2), encoding="utf-8") + return path + + +def _write_minimal_elf( + path: Path, + *, + code_section_name: str = ".text", + code: bytes = b"\x90\x90\xc3", +) -> bytes: + """Write a minimal ELF64 relocatable object accepted by the host readelf.""" + + elf_header_size = 64 + section_header_size = 64 + code_offset = elf_header_size + section_names = b"\0" + code_section_name.encode("ascii") + b"\0.shstrtab\0" + section_names_offset = code_offset + len(code) + section_headers_offset = (section_names_offset + len(section_names) + 7) & ~7 + section_names_name_offset = len(code_section_name) + 2 + + ident = b"\x7fELF" + bytes((2, 1, 1, 0, 0)) + b"\0" * 7 + header = struct.pack( + "<16sHHIQQQIHHHHHH", + ident, + 1, # ET_REL + 62, # EM_X86_64 + 1, + 0, + 0, + section_headers_offset, + 0, + elf_header_size, + 0, + 0, + section_header_size, + 3, + 2, + ) + null_section = bytes(section_header_size) + code_section = struct.pack( + " report_module.SubmitPmuBuildIdentity: + """Create a source-v2 identity without depending on real ELF tooling.""" + + extra_cache_key = "0123456789abcdef" + runtime_tail = Path("a5") / "onboard" / "fully_distributed_within_core" / tensormap_mode + runtime_output_directory = directory / "build" / "lib" / runtime_tail + cache_directory = directory / "build" / "cache" / runtime_tail / "aicore-extra" / extra_cache_key + aicore_build_directory = cache_directory / "aicore" + aicore_build_directory.mkdir(parents=True) + kernel = runtime_output_directory / "aicore-extra" / extra_cache_key / "aicore_kernel.o" + artifact_paths = ( + kernel, + aicore_build_directory / "aicore_aic_combined.o", + aicore_build_directory / "aicore_aiv_combined.o", + runtime_output_directory / "libhost_runtime.so", + runtime_output_directory / "libaicpu_kernel.so", + ) + for ordinal, artifact in enumerate(artifact_paths, start=1): + artifact.parent.mkdir(parents=True, exist_ok=True) + artifact.write_bytes(f"artifact-{ordinal}-header:text-{ordinal}-payload".encode()) + + compile_definitions = [ + f"PTO_FDWIC_SHARED_MAP={1 if tensormap_mode == 'shared' else 0}", + f"PTO_FDWIC_TENSORMAP_RING_CAP={FDWIC_TENSORMAP_RING_CAP}", + "PTO_FDWIC_SUBMIT_PMU=1", + ] + if profile != "submit-pmu-none": + compile_definitions.append(f"PTO_FDWIC_SUBMIT_PMU_PHASE_ID={report_module.PHASE_CONFIG_BY_MODE[profile]['id']}") + compile_definitions.append("PTO_FDWIC_TRACE_ENABLED=0") + frozen_compile_definitions = tuple(compile_definitions) + definitions_sha256 = hashlib.sha256(repr(compile_definitions).encode()).hexdigest() + source_state = f"source-v2:{'1' * 40}:{'2' * 64}:{definitions_sha256}" + (aicore_build_directory / ".git_commit").write_text(source_state, encoding="utf-8") + + def inspect_fake_artifact(path: Path | str) -> report_module.BuildArtifactIdentity: + artifact = Path(path).resolve() + data = artifact.read_bytes() + literal_text = data[data.index(b":") + 1 :] + return report_module.BuildArtifactIdentity( + path=artifact, + sha256=hashlib.sha256(data).hexdigest(), + size_bytes=len(data), + text_sha256=hashlib.sha256(literal_text).hexdigest(), + text_size_bytes=len(literal_text), + ) + + monkeypatch.setattr(report_module, "_inspect_build_artifact", inspect_fake_artifact) + return capture_build_identity( + profile=profile, + profiled_cache_key=("a5", "fdwic", tensormap_mode, profile), + aicore_extra_cache_key=extra_cache_key, + compile_definitions=frozen_compile_definitions, + aicore_kernel=kernel, + aicore_build_dir=aicore_build_directory, + host_runtime=artifact_paths[-2], + aicpu_runtime=artifact_paths[-1], + ) + + +def _publish_fake_provenance( + directory: Path, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[Path, Path, report_module.SubmitPmuBuildIdentity]: + raw_path = _write_capture(directory, _valid_capture()) + identity = _fake_build_identity(directory, monkeypatch) + output_path, provenance_path = write_report_with_provenance(raw_path, identity) + return output_path, provenance_path, identity + + +def test_inspect_build_artifact_uses_real_readelf_and_hashes_literal_text(tmp_path: Path) -> None: + literal_text = b"\x90\x66\x90\xc3\x00\xff" + artifact = tmp_path / "minimal.o" + contents = _write_minimal_elf(artifact, code=literal_text) + + identity = report_module._inspect_build_artifact(artifact) + + assert identity.path == artifact.resolve() + assert identity.size_bytes == len(contents) + assert identity.sha256 == hashlib.sha256(contents).hexdigest() + assert identity.text_size_bytes == len(literal_text) + assert identity.text_sha256 == hashlib.sha256(literal_text).hexdigest() + + +def test_inspect_build_artifact_rejects_real_elf_without_text_and_invalid_file(tmp_path: Path) -> None: + no_text = tmp_path / "no-text.o" + _write_minimal_elf(no_text, code_section_name=".code") + with pytest.raises(ValueError, match="has no literal .text section"): + report_module._inspect_build_artifact(no_text) + + invalid = tmp_path / "invalid.o" + invalid.write_bytes(b"not an ELF object") + with pytest.raises(ValueError, match="readelf failed for build artifact"): + report_module._inspect_build_artifact(invalid) + + +def test_valid_capture_is_recomputed_and_rendered(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_capture()) + + capture = load_capture(raw_path) + assert len(capture.records) == 96 + assert len(capture.groups["aic"]) == 32 + assert len(capture.groups["aiv"]) == 64 + assert all("shadow_icache_requests" not in record for record in capture.records) + assert all("shadow_icache_misses" not in record for record in capture.records) + for group_name in ("all", "aic", "aiv"): + summary = capture.summary[group_name] + assert summary["submit_elapsed_ticks"]["min"] == 10_000 + assert summary["submit_elapsed_ticks"]["mean"] == 10_000 + assert summary["submit_elapsed_ticks"]["max"] == 10_000 + assert summary["scalar_submit_elapsed_ticks"]["min"] == 9_000 + assert summary["scalar_submit_elapsed_ticks"]["mean"] == 9_000 + assert summary["scalar_submit_elapsed_ticks"]["max"] == 9_000 + assert summary["scalar_denominator_excluded_wall_ticks"]["min"] == 1_000 + assert summary["scalar_denominator_excluded_wall_ticks"]["mean"] == 1_000 + assert summary["scalar_denominator_excluded_wall_ticks"]["max"] == 1_000 + assert summary["non_scalar_busy_cycles"]["min"] == 4_000 + assert summary["non_scalar_busy_cycles"]["mean"] == 4_000 + assert summary["non_scalar_busy_cycles"]["max"] == 4_000 + document = render_report(raw_path) + assert "真实 FDWIC Scalar Submit PMU" in document + assert "32 AIC + 64 AIV" in document + assert "Σmiss/Σrequest" in document + assert "90.000 ns" in document + assert "不是 Submit 墙钟损失" in document + assert "非 Scalar-busy 残余不是空闲时间，也不是 I-cache stall" in document + assert "SYS gate 边界诊断/core" in document + assert "它不是按 1.65 GHz 换算的 PMU 阶段时间" in document + assert "PMU-total / SYS gate/core" not in document + assert "result-used return-ready atomic 的依赖区间只从 SYS gate 边界累计值" in document + assert "I-cache/PMU counter 仍包含其指令事件" in document + assert "source-issue atomic" in document + assert "不能称为纯 Kernel 时间" in document + assert "不能与 perf-clock、swimlane 或另一个 phase ELF 相减" in document + for cycles_per_ns in (1.649844, 1.650062, 1.649731): + assert f"按 {cycles_per_ns:.6f} cycles/ns 校准" in document + assert "阶段观察（phase_id=" not in document + assert "均值 16,047.5；最小 16,000；最大 16,095" in document + for group_name in ("all", "aic", "aiv"): + summary = capture.summary[group_name] + requests = summary["icache_requests"] + misses = summary["icache_misses"] + assert ( + f"
Primary I-cache request/core
最小 {requests['min']:,}；最大 {requests['max']:,}
" + ) in document + assert ( + f"
Primary I-cache miss/core
最小 {misses['min']:,}；最大 {misses['max']:,}
" + ) in document + assert ( + f"
90 ns 直觉量尺/core
\n" + f"
最小 {misses['min'] * 90 / 1_000:,.3f}；" + f"最大 {misses['max'] * 90 / 1_000:,.3f} µs
" + ) in document + assert f"{summary['icache_miss_rate']:.3%}" in document + assert document.count(" None: + capture_data = _valid_capture() + producer_keys = { + "cores", + "total_cycles", + "scalar_submit_elapsed_ticks", + "scalar_busy", + "icache_requests", + "icache_misses", + "scalar_busy_share", + "icache_miss_rate", + } + for group in capture_data["summary"].values(): + assert set(group) == producer_keys + + capture = load_capture(_write_capture(tmp_path, capture_data)) + + for group in capture.summary.values(): + assert { + "submit_elapsed_ticks", + "scalar_submit_elapsed_ticks", + "scalar_denominator_excluded_wall_ticks", + "non_scalar_busy_cycles", + } <= set(group) + + +def test_non_scalar_busy_extrema_are_computed_per_record(tmp_path: Path) -> None: + capture_data = _valid_capture() + records = capture_data["records"] + records[0]["total_cycles"] = 20_000 + records[0]["scalar_busy"] = 19_000 + records[1]["total_cycles"] = 19_000 + records[1]["scalar_busy"] = 1_000 + _refresh_host_aggregates(capture_data) + + capture = load_capture(_write_capture(tmp_path, capture_data)) + + residual = capture.summary["all"]["non_scalar_busy_cycles"] + assert residual["min"] == 1_000 + assert residual["max"] == 18_000 + assert residual["min"] != ( + capture.summary["all"]["total_cycles"]["min"] - capture.summary["all"]["scalar_busy"]["min"] + ) + assert residual["max"] != ( + capture.summary["all"]["total_cycles"]["max"] - capture.summary["all"]["scalar_busy"]["max"] + ) + + +def test_summary_does_not_derive_a_frequency_from_sys_ticks(tmp_path: Path) -> None: + capture_data = _valid_capture() + for logical_core_id, record in enumerate(capture_data["records"]): + elapsed = 10_000 if logical_core_id % 2 == 0 else 20_000 + record["last_submit_end_tick"] = record["first_submit_start_tick"] + elapsed + record["submit_elapsed_ticks"] = elapsed + record["scalar_submit_elapsed_ticks"] = elapsed + record["total_cycles"] = 10_000 if logical_core_id % 2 == 0 else 40_000 + record["scalar_busy"] = 8_000 + _refresh_host_aggregates(capture_data) + + capture = load_capture(_write_capture(tmp_path, capture_data)) + + assert all("pmu_total_cycles_per_scalar_tick" not in summary for summary in capture.summary.values()) + assert "PMU-total / SYS gate/core" not in render_report(capture.input_path) + + +def test_zero_submit_elapsed_is_rejected_before_summary(tmp_path: Path) -> None: + capture_data = _valid_capture() + record = capture_data["records"][0] + record["last_submit_end_tick"] = record["first_submit_start_tick"] + record["submit_elapsed_ticks"] = 0 + + with pytest.raises(ValueError, match=r"records\[0\]\.submit_elapsed_ticks must be an integer >= 1"): + load_capture(_write_capture(tmp_path, capture_data)) + + +def test_group_cards_use_role_specific_cycle_calibration(tmp_path: Path) -> None: + capture = load_capture(_write_capture(tmp_path, _valid_capture())) + + document = render_report(capture.input_path) + + expected_mean_us = { + "all": 16_047.5 / 1.649844 / 1_000, + "aic": 16_015.5 / 1.650062 / 1_000, + "aiv": 16_063.5 / 1.649731 / 1_000, + } + for group_name, mean_us in expected_mean_us.items(): + cycles_per_ns = capture.data["configuration"]["pmu_cycles_per_ns"][group_name] + assert f"按 {cycles_per_ns:.6f} cycles/ns 校准" in document + assert f"等效时间 均值 {mean_us:,.3f}；" in document + + +def test_valid_arg_build_capture_renders_same_elf_phase_observation_first(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_arg_build_capture()) + + capture = load_capture(raw_path) + + assert capture.phase_summary is not None + assert all("shadow_icache_requests" in record for record in capture.records) + assert all("shadow_icache_misses" in record for record in capture.records) + all_phase = capture.phase_summary["all"] + assert all_phase["phase_begin_reads"] == 96 * 5 + assert all_phase["phase_end_reads"] == 96 * 5 + assert all_phase["phase_icache_requests_observed_plus_capture_gap"]["sum"] == all_phase[ + "phase_icache_requests_observed" + ]["sum"] + sum(record["icache_requests"] - record["shadow_icache_requests"] for record in capture.records) + assert all_phase["phase_non_scalar_busy_cycles"]["sum"] == sum( + record["phase_total_cycles_observed"] - record["phase_scalar_busy_observed"] for record in capture.records + ) + assert all_phase["shadow_scalar_loss"]["sum"] == sum( + record["scalar_busy"] - record["shadow_scalar_busy"] for record in capture.records + ) + + document = render_report(raw_path) + assert document.index("arg-build 阶段观察") < document.index("全局 Submit 时间范围") + assert "claim_end_to_materialize_begin" in document + assert "running_read_clear_observed_bracket" in document + assert "boundary_diagnostic_sys_cnt_between_observers" in document + assert "Phase PMU total" in document + assert "Phase scalar busy" in document + assert "非 Scalar-busy 残余" in document + assert "指标ALL96 核" in document + assert "Phase PMU total" in document + assert "Phase scalar busy" in document + assert "SYS 边界诊断 / Begin-End" in document + assert ".phase-table table { width:100%; min-width:900px; table-layout:fixed; white-space:normal; }" in document + assert 'class="table-wrap raw-core-table"' in document + assert "" not in document + assert "原始阶段关系（非业务占比）" in document + assert "本报告没有提供 empty-bracket 校准输入" in document + assert "以下均为原始 observed，不能作为业务占比" in document + assert "原始 Phase total / 原始 whole total" in document + assert "原始 Phase scalar / 原始 whole scalar" in document + assert "Request raw observed（总数 / 逐核 min–max / 原始整窗）" in document + assert "Miss raw observed（总数 / 逐核 min–max / 原始整窗）" in document + assert "observed_plus_capture_gap = observed + (primary − shadow)" in document + assert "插桩 bookkeeping 会进入 sample" in document + assert "不是原业务" in document + assert "事件数的数学上下界" in document + assert "阶段原始 PMU 观测来自 running read-clear 的 total cycles" in document + assert "phase_elapsed_ticks 只是 SYS 边界闭合诊断" in document + assert "绝不按 1 GHz 当作阶段时间" in document + assert "result-used return-ready atomic" in document + assert "只从 SYS 边界累计值扣除" in document + assert "I-cache/PMU counter 仍含其指令事件" in document + assert "source-issue atomic" in document + assert "保留在时间和计数口径内" in document + assert "request/miss 百分比的分母是" in document + assert "同一次采集的 Submit 整窗 primary" in document + assert "cycles/call" in document + assert "是每核累计" in document + assert "不是逐调用极值" in document + assert "不同 ELF 不能直接相减" in document + assert "I-cache shadow 是 begin/end/final 全部 running read-clear 返回值之和" in document + assert "scalar shadow 同样用于核验 phase 读清重建" in document + for group_name in ("all", "aic", "aiv"): + phase = capture.phase_summary[group_name] + group_records = capture.groups[group_name] + requests = phase["phase_icache_requests_observed"] + misses = phase["phase_icache_misses_observed"] + assert phase["primary_icache_requests"] == sum(record["icache_requests"] for record in group_records) + assert phase["primary_icache_misses"] == sum(record["icache_misses"] for record in group_records) + assert phase["phase_total_share_of_pmu_total"] == pytest.approx( + sum(record["phase_total_cycles_observed"] for record in group_records) + / sum(record["total_cycles"] for record in group_records) + ) + assert phase["phase_scalar_share_of_whole_scalar"] == pytest.approx( + sum(record["phase_scalar_busy_observed"] for record in group_records) + / sum(record["scalar_busy"] for record in group_records) + ) + assert phase["phase_scalar_busy_share_of_phase_total"] == pytest.approx( + sum(record["phase_scalar_busy_observed"] for record in group_records) + / sum(record["phase_total_cycles_observed"] for record in group_records) + ) + assert f"{phase['phase_total_share_of_pmu_total']:.3%}" in document + assert f"{phase['phase_scalar_share_of_whole_scalar']:.3%}" in document + assert f"{phase['phase_scalar_busy_share_of_phase_total']:.3%}" in document + assert f"{phase['phase_request_observed_share_of_primary']:.3%}" in document + assert f"{phase['phase_request_observed_plus_capture_gap_share_of_primary']:.3%}" in document + assert f"{phase['phase_miss_observed_share_of_primary']:.3%}" in document + assert f"{phase['phase_miss_observed_plus_capture_gap_share_of_primary']:.3%}" in document + assert f"{phase['phase_total_cycles_observed_per_call']:,.3f} cycles/call" in document + assert f"{phase['phase_scalar_busy_observed_per_call']:,.3f} cycles/call" in document + assert f"{phase['phase_non_scalar_busy_cycles_per_call']:,.3f} cycles/call" in document + cycles_per_ns = capture.data["configuration"]["pmu_cycles_per_ns"][group_name] + phase_time_us = phase["phase_total_cycles_observed"]["sum"] / cycles_per_ns / 1_000 + assert f"≈ {phase_time_us:,.3f} µs（{cycles_per_ns:.6f} cycles/ns）" in document + assert f"Σ {phase['phase_elapsed_ticks']['sum']:,} raw ticks" in document + assert f"逐核 最小 {requests['min']:,}；最大 {requests['max']:,}" in document + assert f"逐核 最小 {misses['min']:,}；最大 {misses['max']:,}" in document + assert f"原始整窗 {phase['primary_icache_requests']:,}" in document + assert f"原始整窗 {phase['primary_icache_misses']:,}" in document + + +def test_valid_empty_bracket_capture_is_reported_as_observer_calibration(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_empty_bracket_capture()) + + capture = load_capture(raw_path) + + assert capture.phase_summary is not None + document = render_report(raw_path) + assert document.index("empty-bracket 阶段观察") < document.index("全局 Submit 时间范围") + assert "phase_id=2" in document + assert "claim_end_adjacent_empty_bracket" in document + assert "running_read_clear_empty_bracket_calibration" in document + assert "boundary_diagnostic_outer_sys_cnt_around_adjacent_observer_pair" in document + assert "用于估算每次 begin/end 紧邻执行的记录代码开销" in document + assert "不是业务 phase" in document + assert "phase PMU total/scalar 是紧邻 begin/end 对本身带来的计数开销" in document + assert "仍只覆盖两次 shadow read-clear 之间" in document + assert "SYS tick 只用来核验边界是否闭合" in document + assert "不能估算完整 whole" in document + for group_name in ("all", "aic", "aiv"): + phase = capture.phase_summary[group_name] + requests = phase["phase_icache_requests_observed"] + misses = phase["phase_icache_misses_observed"] + assert f"{phase['phase_total_cycles_observed_per_call']:,.3f} cycles/call" in document + assert f"逐核 最小 {requests['min']:,}；最大 {requests['max']:,}" in document + assert f"逐核 最小 {misses['min']:,}；最大 {misses['max']:,}" in document + + +def test_phase_recording_reference_subtracts_only_the_phase_numerator_by_role( + tmp_path: Path, +) -> None: + target_directory = tmp_path / "target" + empty_directory = tmp_path / "empty" + target_directory.mkdir() + empty_directory.mkdir() + target_path = _write_capture(target_directory, _valid_materialize_capture()) + empty_data = _valid_empty_bracket_capture() + for record in empty_data["records"]: + is_aic = record["role"] == "aic" + logical_core_id = record["logical_core_id"] + record["phase_total_cycles_observed"] = (900 if is_aic else 1_300) + logical_core_id + record["phase_scalar_busy_observed"] = (500 if is_aic else 800) + logical_core_id + record["phase_icache_requests_observed"] = 120 if is_aic else 180 + # Materialize 的原始 miss 是 50..54；校准值更大，用来证明参考值允许为负。 + record["phase_icache_misses_observed"] = 60 + logical_core_id % 5 + empty_path = _write_capture(empty_directory, empty_data) + + target = load_capture(target_path) + empty = load_capture(empty_path) + reference = build_phase_recording_cost_reference(target, empty) + + assert reference["exact_correction"] is False + assert reference["raw_whole_denominator_is_unchanged"] is True + assert reference["whole_recording_cost_is_not_measured"] is True + for group_name in ("aic", "aiv"): + group = reference["groups"][group_name] + target_phase = target.phase_summary[group_name] + empty_phase = empty.phase_summary[group_name] + assert target_phase is not None + assert empty_phase is not None + assert group["target_record_pairs"] == target_phase["phase_end_reads"] + assert group["empty_record_pairs"] == empty_phase["phase_end_reads"] + for metric_name, phase_field, whole_field in ( + ("pmu_total_cycles", "phase_total_cycles_observed", "total_cycles"), + ("scalar_busy_cycles", "phase_scalar_busy_observed", "scalar_busy"), + ("non_scalar_busy_cycles", "phase_non_scalar_busy_cycles", "non_scalar_busy_cycles"), + ("icache_requests", "phase_icache_requests_observed", "icache_requests"), + ("icache_misses", "phase_icache_misses_observed", "icache_misses"), + ): + metric = group["metrics"][metric_name] + raw_phase = target_phase[phase_field]["sum"] + raw_whole = target.summary[group_name][whole_field]["sum"] + empty_per_pair = empty_phase[phase_field]["sum"] / empty_phase["phase_end_reads"] + expected_cost = empty_per_pair * target_phase["phase_end_reads"] + assert metric["raw_phase_observed_sum"] == raw_phase + assert metric["raw_whole_sum"] == raw_whole + assert metric["recording_cost_estimate_sum"] == pytest.approx(expected_cost) + assert metric["after_recording_cost_reference_sum"] == pytest.approx(raw_phase - expected_cost) + assert metric["raw_phase_observed_ratio_to_raw_whole"] == pytest.approx(raw_phase / raw_whole) + assert metric["after_recording_cost_reference_ratio_to_raw_whole"] == pytest.approx( + (raw_phase - expected_cost) / raw_whole + ) + + all_group = reference["groups"]["all"] + assert all_group["all_values_are_aic_aiv_sums"] is True + for metric_name, metric in all_group["metrics"].items(): + for field in ( + "raw_phase_observed_sum", + "raw_whole_sum", + "recording_cost_estimate_sum", + "after_recording_cost_reference_sum", + ): + assert metric[field] == pytest.approx( + reference["groups"]["aic"]["metrics"][metric_name][field] + + reference["groups"]["aiv"]["metrics"][metric_name][field] + ) + miss_reference = all_group["metrics"]["icache_misses"] + assert miss_reference["after_recording_cost_reference_sum"] < 0 + assert miss_reference["after_recording_cost_reference_ratio_to_raw_whole"] < 0 + + document = render_report(target_path, calibration_input_path=empty_path) + assert "扣除记录代码开销估算后的参考值" in document + assert "参考值只从 phase observed 分子扣除该估算；原始 whole 分母保持不变" in document + assert "参考 Phase total / 原始 whole total" in document + assert "原始 observed" in document + assert "不能截成零" in document + assert f"{miss_reference['after_recording_cost_reference_sum']:,.3f}" in document + + +def test_phase_recording_reference_keeps_negative_values_and_none_for_zero_raw_whole() -> None: + metric = report_module._recording_reference_metric( + unit="events", + phase_field="phase_icache_misses_observed", + whole_field="icache_misses", + target_group={"phase_icache_misses_observed": {"sum": 3}}, + target_whole={"icache_misses": {"sum": 0}}, + empty_group={"phase_icache_misses_observed": {"sum": 5}}, + target_record_pairs=1, + empty_record_pairs=1, + ) + + assert metric["after_recording_cost_reference_sum"] == -2 + assert metric["raw_phase_observed_ratio_to_raw_whole"] is None + assert metric["after_recording_cost_reference_ratio_to_raw_whole"] is None + assert report_module._format_optional_percent(None) == "N/A" + + +def test_phase_recording_reference_provenance_requires_one_revision_and_scenario() -> None: + target = { + "build": { + "git_head": "1" * 40, + "source_state_version": "source-v2", + "profiled_cache_key": ["Case1", "a5", "fdwic", "private", MATERIALIZE_CAPTURE_MODE], + } + } + calibration = { + "build": { + "git_head": "1" * 40, + "source_state_version": "source-v2", + "profiled_cache_key": ["Case1", "a5", "fdwic", "private", EMPTY_BRACKET_CAPTURE_MODE], + } + } + + binding = report_module._validate_recording_reference_provenance(target, calibration) + assert binding == { + "verified": True, + "git_head": "1" * 40, + "source_state_version": "source-v2", + "profiled_cache_key_prefix": ["Case1", "a5", "fdwic", "private"], + } + + calibration["build"]["git_head"] = "2" * 40 + with pytest.raises(ValueError, match="git heads do not match"): + report_module._validate_recording_reference_provenance(target, calibration) + calibration["build"]["git_head"] = "1" * 40 + calibration["build"]["profiled_cache_key"][0] = "CaseB1" + with pytest.raises(ValueError, match="scenarios do not match"): + report_module._validate_recording_reference_provenance(target, calibration) + + +def test_phase_non_scalar_extrema_are_derived_per_core(tmp_path: Path) -> None: + capture_data = _valid_arg_build_capture() + capture_data["records"][0]["phase_total_cycles_observed"] = 5_000 + capture_data["records"][0]["phase_scalar_busy_observed"] = 4_900 + capture_data["records"][1]["phase_total_cycles_observed"] = 4_000 + capture_data["records"][1]["phase_scalar_busy_observed"] = 1_000 + + capture = load_capture(_write_capture(tmp_path, capture_data)) + + assert capture.phase_summary is not None + residual = capture.phase_summary["all"]["phase_non_scalar_busy_cycles"] + assert residual["min"] == 100 + assert residual["max"] == 3_000 + + +def test_empty_bracket_rejects_arg_build_phase_id(tmp_path: Path) -> None: + capture = _valid_empty_bracket_capture() + capture["records"][0]["phase_id"] = 1 + + with pytest.raises(ValueError, match=r"records\[0\]\.phase_id must equal 2"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_valid_materialize_capture_tracks_current_business_boundary(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_materialize_capture()) + + capture = load_capture(raw_path) + + assert capture.phase_summary is not None + assert capture.phase_summary["all"]["phase_begin_reads"] == 96 * 5 + document = render_report(raw_path) + assert document.index("materialize 阶段观察") < document.index("全局 Submit 时间范围") + assert "phase_id=3" in document + assert "materialize_begin_to_materialize_end" in document + assert "running_read_clear_observed_bracket" in document + assert "boundary_diagnostic_sys_cnt_between_observers" in document + assert "观察器自成本量尺，不是业务 phase" not in document + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("id", 1), + ("name", "arg-build"), + ("boundary", "claim_end_to_materialize_begin"), + ("expected_calls_per_core", 4), + ("counter_semantics", "running_read_clear_empty_bracket_calibration"), + ("time_semantics", "outer_sys_cnt_around_adjacent_begin_end_pair"), + ), +) +def test_materialize_rejects_mismatched_phase_configuration(tmp_path: Path, field: str, value: Any) -> None: + capture = _valid_materialize_capture() + capture["configuration"]["phase"][field] = value + + with pytest.raises(ValueError, match=r"configuration\.phase"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_materialize_rejects_wrong_record_phase_id(tmp_path: Path) -> None: + capture = _valid_materialize_capture() + capture["records"][0]["phase_id"] = 2 + + with pytest.raises(ValueError, match=r"records\[0\]\.phase_id must equal 3"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_valid_prepare_map_capture_tracks_call_body_boundary(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_prepare_map_capture()) + + capture = load_capture(raw_path) + + assert capture.phase_summary is not None + assert capture.phase_summary["all"]["phase_begin_reads"] == 96 * 5 + document = render_report(raw_path) + assert document.index("prepare-map 阶段观察") < document.index("全局 Submit 时间范围") + assert "phase_id=8" in document + assert "dist_submit_prepare_map_call_entry_to_return" in document + assert "running_read_clear_observed_bracket" in document + assert "boundary_diagnostic_sys_cnt_between_observers" in document + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("id", 3), + ("name", "materialize"), + ("boundary", "materialize_begin_to_materialize_end"), + ("expected_calls_per_core", 4), + ("counter_semantics", "running_read_clear_empty_bracket_calibration"), + ("time_semantics", "outer_sys_cnt_around_adjacent_begin_end_pair"), + ), +) +def test_prepare_map_rejects_mismatched_phase_configuration(tmp_path: Path, field: str, value: Any) -> None: + capture = _valid_prepare_map_capture() + capture["configuration"]["phase"][field] = value + + with pytest.raises(ValueError, match=r"configuration\.phase"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_prepare_map_rejects_wrong_record_phase_id(tmp_path: Path) -> None: + capture = _valid_prepare_map_capture() + capture["records"][0]["phase_id"] = 3 + + with pytest.raises(ValueError, match=r"records\[0\]\.phase_id must equal 8"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_valid_fanin_capture_accepts_dynamic_zero_call_cores(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_fanin_capture()) + + capture = load_capture(raw_path) + + assert capture.phase_summary is not None + assert capture.phase_summary["all"]["phase_begin_reads"] == 4 + assert capture.phase_summary["aic"]["phase_begin_reads"] == 2 + assert capture.phase_summary["aiv"]["phase_begin_reads"] == 2 + assert capture.phase_summary["all"]["phase_zero_call_cores"] == 92 + document = render_report(raw_path) + assert "fanin 阶段观察" in document + assert "phase_id=9" in document + assert "fanin_begin_to_fanin_end" in document + assert "call_shape=dynamic_balanced" in document + assert "expected_calls=ALL 4 / AIC 2 / AIV 2" in document + assert "零调用核 92" in document + + +def test_fanin_rejects_unbalanced_per_core_boundaries(tmp_path: Path) -> None: + capture = _valid_fanin_capture() + capture["records"][0]["phase_end_reads"] = 0 + + with pytest.raises(ValueError, match="dynamic phase begin/end reads must be balanced"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_fanin_rejects_wrong_global_role_call_totals(tmp_path: Path) -> None: + capture = _valid_fanin_capture() + for field in ("phase_begin_reads", "phase_end_reads"): + capture["records"][1][field] = 0 + capture["records"][34][field] = 1 + capture["records"][1]["phase_elapsed_ticks"] = 0 + capture["records"][1]["phase_total_cycles_observed"] = 0 + capture["records"][1]["phase_scalar_busy_observed"] = 0 + capture["records"][1]["phase_icache_requests_observed"] = 0 + capture["records"][1]["phase_icache_misses_observed"] = 0 + capture["records"][34]["phase_elapsed_ticks"] = 75 + capture["records"][34]["phase_total_cycles_observed"] = 300 + capture["records"][34]["phase_scalar_busy_observed"] = 180 + capture["records"][34]["phase_icache_requests_observed"] = 40 + capture["records"][34]["phase_icache_misses_observed"] = 2 + + with pytest.raises(ValueError, match="dynamic phase call totals"): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize( + "field", + ( + "phase_elapsed_ticks", + "phase_total_cycles_observed", + "phase_scalar_busy_observed", + "phase_icache_requests_observed", + "phase_icache_misses_observed", + ), +) +def test_fanin_rejects_nonzero_values_on_zero_call_core(tmp_path: Path, field: str) -> None: + capture = _valid_fanin_capture() + capture["records"][2][field] = 1 + + with pytest.raises( + ValueError, + match="zero-call dynamic phase must have zero elapsed/total/scalar/request/miss", + ): + load_capture(_write_capture(tmp_path, capture)) + + +def test_valid_winner_build_control_capture_excludes_linked_kernel_segments(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_winner_build_capture()) + + capture = load_capture(raw_path) + + assert capture.phase_summary is not None + all_phase = capture.phase_summary["all"] + assert all_phase["phase_business_calls"] == 4 + assert capture.phase_summary["aic"]["phase_business_calls"] == 2 + assert capture.phase_summary["aiv"]["phase_business_calls"] == 2 + assert all_phase["phase_begin_reads"] == 6 + assert all_phase["phase_end_reads"] == 6 + assert all_phase["phase_excluded_kernel_calls"] == 2 + assert all_phase["phase_icache_requests_observed_per_call"] == pytest.approx(40) + assert all_phase["phase_icache_misses_observed_per_call"] == pytest.approx(2) + assert all_phase["phase_calls_per_core"]["min"] == 0 + assert all_phase["phase_calls_per_core"]["max"] == 1 + assert all_phase["phase_zero_call_cores"] == 92 + document = render_report(raw_path) + assert "winner-build-control 阶段观察" in document + assert "phase_id=10" in document + assert "winner_build_begin_to_end_excluding_linked_kernel_calls" in document + assert "discontinuous_running_read_clear_excluding_linked_kernel_calls" in document + assert "boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls" in document + assert "call_shape=dynamic_balanced" in document + assert "expected_calls=ALL 4 / AIC 2 / AIV 2" in document + assert "业务调用 4 次；排除 linked Kernel 调用 2 次" in document + assert "零调用核 92" in document + + +def test_winner_build_control_global_shape_uses_business_calls_not_boundary_reads(tmp_path: Path) -> None: + capture = _valid_winner_build_capture() + capture["records"][0]["phase_excluded_kernel_calls"] += 1 + capture["records"][0]["phase_elapsed_ticks"] = 0 + capture["records"][0]["phase_total_cycles_observed"] = 0 + capture["records"][0]["phase_scalar_busy_observed"] = 0 + capture["records"][0]["phase_icache_requests_observed"] = 0 + capture["records"][0]["phase_icache_misses_observed"] = 0 + + with pytest.raises(ValueError, match="dynamic phase call totals"): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize("value", (None, -1)) +def test_winner_build_control_requires_nonnegative_excluded_kernel_calls( + tmp_path: Path, + value: int | None, +) -> None: + capture = _valid_winner_build_capture() + if value is None: + capture["records"][0].pop("phase_excluded_kernel_calls") + else: + capture["records"][0]["phase_excluded_kernel_calls"] = value + + with pytest.raises(ValueError, match=r"records\[0\]\.phase_excluded_kernel_calls must be an integer >= 0"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_winner_build_control_rejects_excluded_count_above_boundary_reads(tmp_path: Path) -> None: + capture = _valid_winner_build_capture() + capture["records"][0]["phase_excluded_kernel_calls"] = 3 + + with pytest.raises(ValueError, match="excluded Kernel calls exceed phase begin/end reads"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_winner_build_control_rejects_unbalanced_business_boundaries(tmp_path: Path) -> None: + capture = _valid_winner_build_capture() + capture["records"][0]["phase_end_reads"] -= 1 + + with pytest.raises(ValueError, match="must remain balanced after subtracting"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_winner_build_control_zero_business_calls_require_zero_observed_values(tmp_path: Path) -> None: + capture = _valid_winner_build_capture() + capture["records"][0]["phase_excluded_kernel_calls"] = capture["records"][0]["phase_begin_reads"] + + with pytest.raises(ValueError, match="zero-call dynamic phase must have zero"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_winner_build_control_requires_kernel_exclusion_validation(tmp_path: Path) -> None: + capture = _valid_winner_build_capture() + capture["validation"].pop("phase_kernel_exclusion_closed_records") + + with pytest.raises(ValueError, match=r"validation\.phase_kernel_exclusion_closed_records"): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize( + ("winner_core_id", "winner_role", "other_role"), + ((0, "aic", "aiv"), (32, "aiv", "aic")), +) +def test_valid_alloc_complete_control_accepts_role_unlocked_b1_winner( + tmp_path: Path, + winner_core_id: int, + winner_role: str, + other_role: str, +) -> None: + raw_path = _write_capture( + tmp_path, + _valid_alloc_complete_capture(winner_core_id=winner_core_id), + ) + + capture = load_capture(raw_path) + + assert capture.records[winner_core_id]["role"] == winner_role + assert capture.phase_summary is not None + all_phase = capture.phase_summary["all"] + assert all_phase["phase_business_calls"] == 1 + assert capture.phase_summary[winner_role]["phase_business_calls"] == 1 + assert capture.phase_summary[other_role]["phase_business_calls"] == 0 + assert all_phase["phase_begin_reads"] == 3 + assert all_phase["phase_end_reads"] == 3 + assert all_phase["phase_excluded_kernel_calls"] == 2 + assert all_phase["phase_icache_requests_observed_per_call"] == pytest.approx(40) + assert all_phase["phase_icache_misses_observed_per_call"] == pytest.approx(2) + assert all_phase["phase_calls_per_core"]["min"] == 0 + assert all_phase["phase_calls_per_core"]["max"] == 1 + assert all_phase["phase_zero_call_cores"] == 95 + + document = render_report(raw_path) + assert "alloc-complete-control 阶段观察" in document + assert "phase_id=11" in document + assert "alloc_complete_begin_to_end_excluding_linked_kernel_calls" in document + assert "discontinuous_running_read_clear_excluding_linked_kernel_calls" in document + assert "boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls" in document + assert "call_shape=dynamic_global" in document + assert "expected_calls=ALL 1（角色不锁定）" in document + assert "业务调用 1 次；排除 linked Kernel 调用 2 次" in document + assert "零调用核 95" in document + + +def test_alloc_complete_control_rejects_wrong_global_call_total(tmp_path: Path) -> None: + capture = _valid_alloc_complete_capture() + winner = capture["records"][0] + winner["phase_begin_reads"] = winner["phase_excluded_kernel_calls"] + winner["phase_end_reads"] = winner["phase_excluded_kernel_calls"] + winner["phase_elapsed_ticks"] = 0 + winner["phase_total_cycles_observed"] = 0 + winner["phase_scalar_busy_observed"] = 0 + winner["phase_icache_requests_observed"] = 0 + winner["phase_icache_misses_observed"] = 0 + + with pytest.raises(ValueError, match="dynamic phase call totals: global call total must equal 1"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_alloc_complete_control_rejects_per_core_call_count_above_batch_limit(tmp_path: Path) -> None: + capture = _valid_alloc_complete_capture() + winner = capture["records"][0] + winner["phase_begin_reads"] += 1 + winner["phase_end_reads"] += 1 + + with pytest.raises(ValueError, match=r"records\[0\] business phase calls exceed dynamic per-core maximum 1"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_alloc_complete_control_rejects_unbalanced_business_boundaries(tmp_path: Path) -> None: + capture = _valid_alloc_complete_capture() + capture["records"][0]["phase_end_reads"] -= 1 + + with pytest.raises(ValueError, match="must remain balanced after subtracting"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_alloc_complete_control_rejects_nonzero_values_on_zero_call_core(tmp_path: Path) -> None: + capture = _valid_alloc_complete_capture() + capture["records"][1]["phase_icache_requests_observed"] = 1 + + with pytest.raises(ValueError, match="zero-call dynamic phase must have zero"): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("id", 10), + ("name", "winner-build-control"), + ("boundary", "winner_build_begin_to_end_excluding_linked_kernel_calls"), + ("call_shape", "dynamic_balanced"), + ("expected_calls", {"all": 2}), + ("counter_semantics", "running_read_clear_observed_bracket"), + ("time_semantics", "inner_sys_cnt_between_boundary_observers"), + ), +) +def test_alloc_complete_control_rejects_mismatched_phase_configuration( + tmp_path: Path, + field: str, + value: Any, +) -> None: + capture = _valid_alloc_complete_capture() + capture["configuration"]["phase"][field] = value + + with pytest.raises(ValueError, match=r"configuration\.phase"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_alloc_complete_control_rejects_wrong_record_phase_id(tmp_path: Path) -> None: + capture = _valid_alloc_complete_capture() + capture["records"][0]["phase_id"] = 10 + + with pytest.raises(ValueError, match=r"records\[0\]\.phase_id must equal 11"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_valid_loser_replay_capture_closes_b1_role_totals_and_html(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_loser_replay_capture()) + + capture = load_capture(raw_path) + + assert capture.phase_summary is not None + all_phase = capture.phase_summary["all"] + assert all_phase["phase_business_calls"] == 380 + assert capture.phase_summary["aic"]["phase_business_calls"] == 126 + assert capture.phase_summary["aiv"]["phase_business_calls"] == 254 + assert all_phase["phase_begin_reads"] == 380 + assert all_phase["phase_end_reads"] == 380 + assert all_phase["phase_excluded_kernel_calls"] == 0 + assert all_phase["phase_icache_requests_observed_per_call"] == pytest.approx(40) + assert all_phase["phase_icache_misses_observed_per_call"] == pytest.approx(2) + assert all_phase["phase_calls_per_core"]["min"] == 3 + assert all_phase["phase_calls_per_core"]["max"] == 4 + assert all_phase["phase_zero_call_cores"] == 0 + + document = render_report(raw_path) + assert "loser-replay 阶段观察" in document + assert "phase_id=12" in document + assert "register_end_to_drain_block_won_return" in document + assert "running_read_clear_observed_bracket" in document + assert "boundary_diagnostic_sys_cnt_between_observers" in document + assert "call_shape=dynamic_balanced" in document + assert "expected_calls=ALL 380 / AIC 126 / AIV 254" in document + assert "业务调用 380 次；排除 linked Kernel 调用 0 次" in document + assert "逐核 3–4；零调用核 0" in document + + +def test_loser_replay_dynamic_formula_scales_to_b2() -> None: + assert report_module._expected_dynamic_phase_calls(LOSER_REPLAY_CAPTURE_MODE, 10) == { + "all": 760, + "aic": 252, + "aiv": 508, + } + assert report_module._dynamic_phase_max_calls_per_core(LOSER_REPLAY_CAPTURE_MODE, 10) == 8 + + +def test_loser_replay_rejects_wrong_global_call_total(tmp_path: Path) -> None: + capture = _valid_loser_replay_capture() + capture["records"][0]["phase_begin_reads"] += 1 + capture["records"][0]["phase_end_reads"] += 1 + + with pytest.raises(ValueError, match="dynamic phase call totals: global call total must equal 380"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_loser_replay_rejects_wrong_role_totals_with_closed_global_total(tmp_path: Path) -> None: + capture = _valid_loser_replay_capture() + capture["records"][2]["phase_begin_reads"] -= 1 + capture["records"][2]["phase_end_reads"] -= 1 + capture["records"][32]["phase_begin_reads"] += 1 + capture["records"][32]["phase_end_reads"] += 1 + + with pytest.raises(ValueError, match="dynamic phase call totals must equal"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_loser_replay_rejects_per_core_call_count_above_four_per_batch(tmp_path: Path) -> None: + capture = _valid_loser_replay_capture() + capture["records"][0]["phase_begin_reads"] = 5 + capture["records"][0]["phase_end_reads"] = 5 + + with pytest.raises(ValueError, match=r"records\[0\] business phase calls exceed dynamic per-core maximum 4"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_loser_replay_rejects_unbalanced_boundaries(tmp_path: Path) -> None: + capture = _valid_loser_replay_capture() + capture["records"][0]["phase_end_reads"] -= 1 + + with pytest.raises(ValueError, match="dynamic phase begin/end reads must be balanced"): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("id", 11), + ("name", "alloc-complete-control"), + ("boundary", "alloc_complete_begin_to_end_excluding_linked_kernel_calls"), + ("call_shape", "dynamic_global"), + ("expected_calls", {"all": 380}), + ("counter_semantics", "discontinuous_running_read_clear_excluding_linked_kernel_calls"), + ("time_semantics", "discontinuous_sys_cnt_control_segments_excluding_linked_kernel_calls"), + ), +) +def test_loser_replay_rejects_mismatched_phase_configuration( + tmp_path: Path, + field: str, + value: Any, +) -> None: + capture = _valid_loser_replay_capture() + capture["configuration"]["phase"][field] = value + + with pytest.raises(ValueError, match=r"configuration\.phase"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_loser_replay_rejects_wrong_record_phase_id(tmp_path: Path) -> None: + capture = _valid_loser_replay_capture() + capture["records"][0]["phase_id"] = 11 + + with pytest.raises(ValueError, match=r"records\[0\]\.phase_id must equal 12"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_valid_claim_capture_tracks_current_business_boundary(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_claim_capture()) + + capture = load_capture(raw_path) + + assert capture.phase_summary is not None + assert capture.phase_summary["all"]["phase_begin_reads"] == 96 * 5 + document = render_report(raw_path) + assert document.index("claim 阶段观察") < document.index("全局 Submit 时间范围") + assert "phase_id=4" in document + assert "claim_begin_to_claim_end" in document + assert "running_read_clear_observed_bracket" in document + assert "boundary_diagnostic_sys_cnt_between_observers" in document + assert "观察器自成本量尺，不是业务 phase" not in document + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("id", 3), + ("name", "materialize"), + ("boundary", "materialize_begin_to_materialize_end"), + ("expected_calls_per_core", 4), + ("counter_semantics", "running_read_clear_empty_bracket_calibration"), + ("time_semantics", "outer_sys_cnt_around_adjacent_begin_end_pair"), + ), +) +def test_claim_rejects_mismatched_phase_configuration(tmp_path: Path, field: str, value: Any) -> None: + capture = _valid_claim_capture() + capture["configuration"]["phase"][field] = value + + with pytest.raises(ValueError, match=r"configuration\.phase"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_claim_rejects_wrong_record_phase_id(tmp_path: Path) -> None: + capture = _valid_claim_capture() + capture["records"][0]["phase_id"] = 3 + + with pytest.raises(ValueError, match=r"records\[0\]\.phase_id must equal 4"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_valid_register_capture_tracks_call_body_boundary(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_register_capture()) + + capture = load_capture(raw_path) + + assert capture.phase_summary is not None + assert capture.phase_summary["all"]["phase_begin_reads"] == 96 * 5 + document = render_report(raw_path) + assert document.index("register 阶段观察") < document.index("全局 Submit 时间范围") + assert "phase_id=5" in document + assert "register_outputs_call_entry_to_return" in document + assert "running_read_clear_observed_bracket" in document + assert "boundary_diagnostic_sys_cnt_between_observers" in document + assert "观察器自成本量尺，不是业务 phase" not in document + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("id", 4), + ("name", "claim"), + ("boundary", "claim_begin_to_claim_end"), + ("expected_calls_per_core", 4), + ("counter_semantics", "running_read_clear_empty_bracket_calibration"), + ("time_semantics", "outer_sys_cnt_around_adjacent_begin_end_pair"), + ), +) +def test_register_rejects_mismatched_phase_configuration(tmp_path: Path, field: str, value: Any) -> None: + capture = _valid_register_capture() + capture["configuration"]["phase"][field] = value + + with pytest.raises(ValueError, match=r"configuration\.phase"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_register_rejects_wrong_record_phase_id(tmp_path: Path) -> None: + capture = _valid_register_capture() + capture["records"][0]["phase_id"] = 4 + + with pytest.raises(ValueError, match=r"records\[0\]\.phase_id must equal 5"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_valid_submit_transition_capture_uses_submit_count_minus_one(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_submit_transition_capture()) + + capture = load_capture(raw_path) + + assert capture.phase_summary is not None + assert capture.phase_summary["all"]["phase_begin_reads"] == 96 * 4 + assert capture.phase_summary["all"]["phase_end_reads"] == 96 * 4 + document = render_report(raw_path) + assert document.index("submit-transition 阶段观察") < document.index("全局 Submit 时间范围") + assert "phase_id=6" in document + assert "previous_submit_end_to_next_submit_begin" in document + assert "expected_calls_per_core=4" in document + + +def test_submit_transition_accepts_minimum_two_submit_shape(tmp_path: Path) -> None: + capture = _valid_submit_transition_capture() + capture["configuration"]["expected_submits_per_core"] = 2 + capture["configuration"]["phase"]["expected_calls_per_core"] = 1 + for record in capture["records"]: + record["submit_count"] = 2 + record["expected_submit_count"] = 2 + record["phase_begin_reads"] = 1 + record["phase_end_reads"] = 1 + + loaded = load_capture(_write_capture(tmp_path, capture)) + + assert loaded.phase_summary is not None + assert loaded.phase_summary["all"]["phase_begin_reads"] == 96 + assert loaded.phase_summary["all"]["phase_end_reads"] == 96 + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("id", 5), + ("name", "register"), + ("boundary", "register_outputs_call_entry_to_return"), + ("expected_calls_per_core", 5), + ("expected_calls_per_core", 3), + ("counter_semantics", "running_read_clear_empty_bracket_calibration"), + ("time_semantics", "outer_sys_cnt_around_adjacent_begin_end_pair"), + ), +) +def test_submit_transition_rejects_mismatched_phase_configuration(tmp_path: Path, field: str, value: Any) -> None: + capture = _valid_submit_transition_capture() + capture["configuration"]["phase"][field] = value + + with pytest.raises(ValueError, match=r"configuration\.phase"): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize("reads", [3, 5]) +def test_submit_transition_rejects_wrong_phase_call_count(tmp_path: Path, reads: int) -> None: + capture = _valid_submit_transition_capture() + capture["records"][0]["phase_begin_reads"] = reads + capture["records"][0]["phase_end_reads"] = reads + + with pytest.raises(ValueError, match="phase begin/end reads must both equal 4"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_submit_transition_rejects_wrong_record_phase_id(tmp_path: Path) -> None: + capture = _valid_submit_transition_capture() + capture["records"][0]["phase_id"] = 5 + + with pytest.raises(ValueError, match=r"records\[0\]\.phase_id must equal 6"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_submit_transition_rejects_single_submit_capture(tmp_path: Path) -> None: + capture = _valid_submit_transition_capture() + capture["configuration"]["expected_submits_per_core"] = 1 + capture["configuration"]["phase"]["expected_calls_per_core"] = 0 + + with pytest.raises(ValueError, match="requires at least two submits per core"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_valid_efdrain_control_capture_excludes_linked_kernel_segments(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_efdrain_control_capture()) + + capture = load_capture(raw_path) + + assert capture.phase_summary is not None + all_phase = capture.phase_summary["all"] + outer_calls = 96 * 5 + excluded_kernel_calls = sum(record["phase_excluded_kernel_calls"] for record in capture.records) + total_requests = sum(record["phase_icache_requests_observed"] for record in capture.records) + total_misses = sum(record["phase_icache_misses_observed"] for record in capture.records) + assert capture.data["validation"]["phase_kernel_exclusion_closed_records"] == 96 + assert all_phase["phase_excluded_kernel_calls"] == excluded_kernel_calls + assert all_phase["phase_begin_reads"] == outer_calls + excluded_kernel_calls + assert all_phase["phase_end_reads"] == outer_calls + excluded_kernel_calls + assert all_phase["phase_icache_requests_observed_per_call"] == pytest.approx(total_requests / outer_calls) + assert all_phase["phase_icache_misses_observed_per_call"] == pytest.approx(total_misses / outer_calls) + + document = render_report(raw_path) + assert "efdrain-control 阶段观察" in document + assert "phase_id=7" in document + assert "efdrain_begin_to_end_excluding_linked_kernel_calls" in document + assert "discontinuous_running_read_clear_excluding_linked_kernel_calls" in document + assert "boundary_diagnostic_discontinuous_sys_cnt_segments_excluding_linked_kernel_calls" in document + assert "total、scalar、request、miss 及 SYS 边界诊断都排除该 Kernel 整段" in document + assert "result-used return-ready atomic" in document + assert "PMU counter" in document + assert "仍含 atomic 指令事件" in document + assert "source-issue atomic" in document + assert "每次调用的分母仍是外层业务调用次数" in document + assert "SYS 边界诊断 / Begin-End" in document + assert f"业务调用 {outer_calls} 次；排除 linked Kernel 调用 {excluded_kernel_calls} 次" in document + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("id", 6), + ("name", "submit-transition"), + ("boundary", "efdrain_begin_to_end"), + ("expected_calls_per_core", 4), + ("counter_semantics", "running_read_clear_observed_bracket"), + ("time_semantics", "inner_sys_cnt_between_boundary_observers"), + ), +) +def test_efdrain_control_rejects_mismatched_phase_configuration( + tmp_path: Path, + field: str, + value: Any, +) -> None: + capture = _valid_efdrain_control_capture() + capture["configuration"]["phase"][field] = value + + with pytest.raises(ValueError, match=r"configuration\.phase"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_efdrain_control_requires_excluded_kernel_call_count(tmp_path: Path) -> None: + capture = _valid_efdrain_control_capture() + capture["records"][0].pop("phase_excluded_kernel_calls") + + with pytest.raises( + ValueError, + match=r"records\[0\]\.phase_excluded_kernel_calls must be an integer >= 0", + ): + load_capture(_write_capture(tmp_path, capture)) + + +def test_efdrain_control_rejects_negative_excluded_kernel_call_count(tmp_path: Path) -> None: + capture = _valid_efdrain_control_capture() + capture["records"][0]["phase_excluded_kernel_calls"] = -1 + + with pytest.raises( + ValueError, + match=r"records\[0\]\.phase_excluded_kernel_calls must be an integer >= 0", + ): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize("field", ("phase_begin_reads", "phase_end_reads")) +def test_efdrain_control_rejects_reads_not_closed_against_excluded_kernel_calls( + tmp_path: Path, + field: str, +) -> None: + capture = _valid_efdrain_control_capture() + capture["records"][1][field] -= 1 + + with pytest.raises( + ValueError, + match=r"records\[1\] phase begin/end reads must both equal expected calls 5 " + r"\+ excluded Kernel calls 1 = 6", + ): + load_capture(_write_capture(tmp_path, capture)) + + +def test_efdrain_control_rejects_wrong_record_phase_id(tmp_path: Path) -> None: + capture = _valid_efdrain_control_capture() + capture["records"][0]["phase_id"] = 6 + + with pytest.raises(ValueError, match=r"records\[0\]\.phase_id must equal 7"): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize("value", (None, 95)) +def test_efdrain_control_requires_closed_kernel_exclusion_validation( + tmp_path: Path, + value: int | None, +) -> None: + capture = _valid_efdrain_control_capture() + if value is None: + capture["validation"].pop("phase_kernel_exclusion_closed_records") + else: + capture["validation"]["phase_kernel_exclusion_closed_records"] = value + + with pytest.raises(ValueError, match=r"validation\.phase_kernel_exclusion_closed_records"): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize( + "capture_factory", + ( + _valid_arg_build_capture, + _valid_empty_bracket_capture, + _valid_materialize_capture, + _valid_claim_capture, + _valid_register_capture, + _valid_submit_transition_capture, + _valid_prepare_map_capture, + _valid_fanin_capture, + _valid_efdrain_control_capture, + _valid_winner_build_capture, + _valid_alloc_complete_capture, + _valid_loser_replay_capture, + ), +) +def test_all_phase_modes_require_excluded_kernel_call_count( + tmp_path: Path, + capture_factory: Callable[[], dict[str, Any]], +) -> None: + capture = capture_factory() + capture["records"][0].pop("phase_excluded_kernel_calls") + + with pytest.raises( + ValueError, + match=r"records\[0\]\.phase_excluded_kernel_calls must be an integer >= 0", + ): + load_capture(_write_capture(tmp_path, capture)) + + +def test_none_forbids_kernel_exclusion_validation(tmp_path: Path) -> None: + capture = _valid_capture() + capture["validation"]["phase_kernel_exclusion_closed_records"] = 96 + + with pytest.raises( + ValueError, + match=r"phase_kernel_exclusion_closed_records is only valid in", + ): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("name", "arg-build"), + ("boundary", "claim_end_to_materialize_begin"), + ("expected_calls_per_core", 4), + ("counter_semantics", "running_read_clear_observed_bracket"), + ("time_semantics", "inner_sys_cnt_between_boundary_observers"), + ), +) +def test_empty_bracket_rejects_mismatched_phase_configuration(tmp_path: Path, field: str, value: Any) -> None: + capture = _valid_empty_bracket_capture() + capture["configuration"]["phase"][field] = value + + with pytest.raises(ValueError, match=r"configuration\.phase"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_arg_build_rejects_unclosed_phase_boundaries(tmp_path: Path) -> None: + capture = _valid_arg_build_capture() + capture["records"][0]["phase_end_reads"] = 4 + + with pytest.raises(ValueError, match="phase begin/end reads must both equal 5"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_arg_build_rejects_shadow_counter_above_primary(tmp_path: Path) -> None: + capture = _valid_arg_build_capture() + capture["records"][0]["shadow_icache_requests"] = capture["records"][0]["icache_requests"] + 1 + + with pytest.raises(ValueError, match="shadow_icache_requests exceeds icache_requests"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_arg_build_rejects_observed_counter_above_shadow(tmp_path: Path) -> None: + capture = _valid_arg_build_capture() + capture["records"][0]["phase_icache_requests_observed"] = capture["records"][0]["shadow_icache_requests"] + 1 + + with pytest.raises(ValueError, match="phase_icache_requests_observed exceeds shadow_icache_requests"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_arg_build_requires_phase_total_and_scalar_fields(tmp_path: Path) -> None: + for field in ("phase_total_cycles_observed", "phase_scalar_busy_observed", "shadow_scalar_busy"): + capture = _valid_arg_build_capture() + capture["records"][0].pop(field) + + with pytest.raises(ValueError, match=rf"records\[0\]\.{field} must be an integer"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_arg_build_rejects_zero_whole_shadow_scalar(tmp_path: Path) -> None: + capture = _valid_arg_build_capture() + capture["records"][0]["shadow_scalar_busy"] = 0 + + with pytest.raises(ValueError, match=r"records\[0\]\.shadow_scalar_busy must be an integer >= 1"): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize( + ("field", "value_from_record", "message"), + ( + ( + "phase_scalar_busy_observed", + lambda record: record["phase_total_cycles_observed"] + 1, + "phase_scalar_busy_observed exceeds phase_total_cycles_observed", + ), + ( + "phase_total_cycles_observed", + lambda record: record["total_cycles"] + 1, + "phase_total_cycles_observed exceeds total_cycles", + ), + ( + "shadow_scalar_busy", + lambda record: record["phase_scalar_busy_observed"] - 1, + "phase_scalar_busy_observed exceeds shadow_scalar_busy", + ), + ( + "shadow_scalar_busy", + lambda record: record["scalar_busy"] + 1, + "shadow_scalar_busy exceeds scalar_busy", + ), + ), +) +def test_arg_build_rejects_invalid_phase_pmu_ordering( + tmp_path: Path, + field: str, + value_from_record: Callable[[dict[str, Any]], int], + message: str, +) -> None: + capture = _valid_arg_build_capture() + record = capture["records"][0] + record[field] = value_from_record(record) + + with pytest.raises(ValueError, match=message): + load_capture(_write_capture(tmp_path, capture)) + + +def test_arg_build_requires_positive_phase_total_for_business_calls(tmp_path: Path) -> None: + capture = _valid_arg_build_capture() + capture["records"][0]["phase_total_cycles_observed"] = 0 + capture["records"][0]["phase_scalar_busy_observed"] = 0 + + with pytest.raises(ValueError, match="non-empty phase must have positive phase_total_cycles_observed"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_arg_build_rejects_phase_time_beyond_scalar_denominator(tmp_path: Path) -> None: + capture = _valid_arg_build_capture() + capture["records"][0]["phase_elapsed_ticks"] = capture["records"][0]["scalar_submit_elapsed_ticks"] + 1 + + with pytest.raises(ValueError, match="phase_elapsed_ticks exceeds scalar_submit_elapsed_ticks"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_arg_build_rejects_incomplete_phase_status(tmp_path: Path) -> None: + capture = _valid_arg_build_capture() + capture["records"][0]["phase_status"] &= ~(1 << 5) + + with pytest.raises(ValueError, match=r"phase_status must equal 0x3ff, got 0x3df$"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_arg_build_rejects_mismatched_phase_pmu_observation_contract(tmp_path: Path) -> None: + capture = _valid_arg_build_capture() + capture["configuration"]["phase"]["pmu_observation"]["sys_cnt_role"] = "primary_phase_timing" + + with pytest.raises(ValueError, match=r"configuration\.phase"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_arg_build_rejects_mismatched_phase_configuration(tmp_path: Path) -> None: + capture = _valid_arg_build_capture() + capture["configuration"]["phase"]["boundary"] = "claim_begin_to_claim_end" + + with pytest.raises(ValueError, match=r"configuration\.phase"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_arg_build_rejects_deprecated_lower_bound_raw_field(tmp_path: Path) -> None: + capture = _valid_arg_build_capture() + record = capture["records"][0] + record["phase_icache_requests_lower_bound"] = record.pop("phase_icache_requests_observed") + + with pytest.raises(ValueError, match="must use observed phase fields"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_arg_build_rejects_deprecated_phase_time_validation_name(tmp_path: Path) -> None: + capture = _valid_arg_build_capture() + validation = capture["validation"] + validation["phase_time_bounded_records"] = validation.pop("phase_time_within_submit_records") + + with pytest.raises(ValueError, match="must use phase_time_within_submit_records"): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize( + "field", + ( + "phase_boundary_closed_records", + "phase_shape_match_records", + "phase_icache_values_ordered_records", + "phase_pmu_values_ordered_records", + "phase_counter_reconstruction_valid_records", + "phase_time_within_submit_records", + "shadow_icache_primary_bounded_records", + "shadow_scalar_primary_bounded_records", + ), +) +def test_arg_build_requires_all_producer_phase_validations(tmp_path: Path, field: str) -> None: + capture = _valid_arg_build_capture() + capture["validation"][field] = 95 + + with pytest.raises(ValueError, match=rf"validation\.{field}"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_none_rejects_phase_record_fields(tmp_path: Path) -> None: + capture = _valid_capture() + capture["records"][0]["phase_id"] = 1 + + with pytest.raises(ValueError, match="must not contain phase fields"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_none_rejects_phase_configuration(tmp_path: Path) -> None: + capture = _valid_capture() + capture["configuration"]["phase"] = {} + + with pytest.raises(ValueError, match="must not contain phase"): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("enabled", False), + ("classification", "all_atomic"), + ("time_boundary", "atomic_issue_only"), + ("counter_semantics", "pmu_counters_exclude_atomic_instruction_events"), + ("time_denominator_effect", "retain_return_ready_atomic_elapsed"), + ), +) +def test_return_ready_atomic_metadata_is_exact( + tmp_path: Path, + field: str, + value: Any, +) -> None: + capture = _valid_capture() + capture["configuration"]["return_ready_atomic_exclusion"][field] = value + + with pytest.raises(ValueError, match=r"configuration\.return_ready_atomic_exclusion"): + load_capture(_write_capture(tmp_path, capture)) + + +@pytest.mark.parametrize("value", (None, 95)) +def test_return_ready_atomic_time_validation_requires_all_cores( + tmp_path: Path, + value: int | None, +) -> None: + capture = _valid_capture() + if value is None: + capture["validation"].pop("return_ready_atomic_time_valid_records") + else: + capture["validation"]["return_ready_atomic_time_valid_records"] = value + + with pytest.raises(ValueError, match=r"validation\.return_ready_atomic_time_valid_records"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_none_status_requires_shadow_match_bit(tmp_path: Path) -> None: + capture = _valid_capture() + capture["records"][0]["status"] &= ~(1 << 18) + + with pytest.raises(ValueError, match=r"records\[0\]\.status must equal"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_phase_status_requires_return_ready_atomic_time_bit(tmp_path: Path) -> None: + capture = _valid_arg_build_capture() + capture["records"][0]["status"] &= ~(1 << 17) + + with pytest.raises(ValueError, match=r"records\[0\]\.status must equal"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_scalar_submit_elapsed_must_be_positive(tmp_path: Path) -> None: + capture = _valid_capture() + capture["records"][0]["scalar_submit_elapsed_ticks"] = 0 + + with pytest.raises( + ValueError, + match=r"records\[0\]\.scalar_submit_elapsed_ticks must be an integer >= 1", + ): + load_capture(_write_capture(tmp_path, capture)) + + +def test_scalar_submit_elapsed_must_not_exceed_wall(tmp_path: Path) -> None: + capture = _valid_capture() + capture["records"][0]["scalar_submit_elapsed_ticks"] = capture["records"][0]["submit_elapsed_ticks"] + 1 + + with pytest.raises(ValueError, match=r"scalar_submit_elapsed_ticks exceeds submit_elapsed_ticks"): + load_capture(_write_capture(tmp_path, capture)) + + +def test_write_report_uses_fixed_default_name_and_atomic_publish(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_capture()) + + output = write_report(raw_path) + + assert output == tmp_path / DEFAULT_OUTPUT_NAME + assert output.read_text(encoding="utf-8").endswith("\n") + assert not (tmp_path / f"{DEFAULT_OUTPUT_NAME}.tmp").exists() + + +def test_write_report_publishes_tmp_with_replace(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + raw_path = _write_capture(tmp_path, _valid_capture()) + real_replace = report_module.os.replace + calls: list[tuple[Path, Path]] = [] + + def record_replace(source: str | Path, destination: str | Path) -> None: + calls.append((Path(source), Path(destination))) + real_replace(source, destination) + + monkeypatch.setattr(report_module.os, "replace", record_replace) + + output = write_report(raw_path) + + assert calls == [(tmp_path / f"{DEFAULT_OUTPUT_NAME}.tmp", output)] + + +def test_provenance_publish_preserves_raw_and_closes_sidecar_and_html( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + raw_path = _write_capture(tmp_path, _valid_capture()) + raw_before = raw_path.read_bytes() + identity = _fake_build_identity(tmp_path, monkeypatch) + + output_path, provenance_path = write_report_with_provenance(raw_path, identity) + + assert output_path == tmp_path / DEFAULT_OUTPUT_NAME + assert provenance_path == tmp_path / DEFAULT_PROVENANCE_NAME + assert raw_path.read_bytes() == raw_before + assert provenance_path.read_text(encoding="utf-8").endswith("}\n") + assert output_path.read_text(encoding="utf-8").endswith("\n") + assert not (tmp_path / f"{DEFAULT_PROVENANCE_NAME}.tmp").exists() + assert not (tmp_path / f"{DEFAULT_OUTPUT_NAME}.tmp").exists() + + capture = load_capture(raw_path) + provenance, provenance_sha256 = load_provenance(provenance_path, capture) + assert provenance["binding"] == { + "raw_name": DEFAULT_INPUT_NAME, + "raw_size": len(raw_before), + "raw_sha256": hashlib.sha256(raw_before).hexdigest(), + "capture_mode": "submit-pmu-none", + } + build = provenance["build"] + assert build["profile"] == "submit-pmu-none" + assert build["tensormap_mode"] == "private" + assert build["profiled_cache_key"][-2:] == [build["tensormap_mode"], build["profile"]] + assert build["aicore_extra_cache_key"] == "0123456789abcdef" + assert build["source_state"].startswith("source-v2:") + assert build["source_state_version"] == "source-v2" + assert build["source_state_path"] == str(identity.source_state_path) + assert build["compile_definitions"] == list(identity.compile_definitions) + assert build["definitions_sha256"] == hashlib.sha256(repr(build["compile_definitions"]).encode()).hexdigest() + + frozen_artifacts = dict(identity.artifacts) + assert set(provenance["artifacts"]) == { + "aicore_kernel", + "aic_combined", + "aiv_combined", + "host_runtime", + "aicpu_runtime", + } + for name, frozen in frozen_artifacts.items(): + artifact = provenance["artifacts"][name] + assert artifact == { + "path": str(frozen.path), + "sha256": frozen.sha256, + "size_bytes": frozen.size_bytes, + "text": { + "sha256": frozen.text_sha256, + "size_bytes": frozen.text_size_bytes, + }, + } + + document = output_path.read_text(encoding="utf-8") + assert document == render_report(raw_path) + assert provenance_sha256 in document + assert "诊断构建身份" in document + for label in ( + "AICore final", + "AIC combined", + "AIV combined", + "Host runtime", + "Inner AICPU runtime", + ): + assert label in document + for frozen in frozen_artifacts.values(): + assert frozen.sha256 in document + assert frozen.text_sha256 in document + + assert not list(tmp_path.glob(".*.pending.*.tmp")) + assert not list(tmp_path.glob(".*.rollback.*.tmp")) + + +def test_shared_tensormap_provenance_closes_cache_key_definitions_and_html( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + raw_path = _write_capture(tmp_path, _valid_capture()) + identity = _fake_build_identity(tmp_path, monkeypatch, tensormap_mode="shared") + + output_path, provenance_path = write_report_with_provenance(raw_path, identity) + provenance, _ = load_provenance(provenance_path, load_capture(raw_path)) + + assert identity.tensormap_mode == "shared" + assert provenance["build"]["tensormap_mode"] == "shared" + assert provenance["build"]["profiled_cache_key"][-2:] == ["shared", "submit-pmu-none"] + assert provenance["build"]["compile_definitions"][0] == "PTO_FDWIC_SHARED_MAP=1" + assert "shared" in output_path.read_text(encoding="utf-8") + + +def test_capture_build_identity_rejects_cross_mode_aicpu_artifact_family( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + identity = _fake_build_identity(tmp_path, monkeypatch) + artifacts = dict(identity.artifacts) + private_aicpu = artifacts["aicpu_runtime"].path + shared_aicpu = private_aicpu.parent.parent / "shared" / private_aicpu.name + shared_aicpu.parent.mkdir(parents=True) + shared_aicpu.write_bytes(private_aicpu.read_bytes()) + + with pytest.raises(ValueError, match="do not belong to the selected private FDWIC artifact family"): + capture_build_identity( + profile=identity.profile, + profiled_cache_key=identity.profiled_cache_key, + aicore_extra_cache_key=identity.aicore_extra_cache_key, + compile_definitions=identity.compile_definitions, + aicore_kernel=artifacts["aicore_kernel"].path, + aicore_build_dir=identity.source_state_path.parent, + host_runtime=artifacts["host_runtime"].path, + aicpu_runtime=shared_aicpu, + ) + + +def test_efdrain_control_build_identity_uses_phase_id_7( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + identity = _fake_build_identity( + tmp_path, + monkeypatch, + profile=EFDRAIN_CONTROL_CAPTURE_MODE, + ) + + assert identity.profile == EFDRAIN_CONTROL_CAPTURE_MODE + assert identity.compile_definitions == ( + "PTO_FDWIC_SHARED_MAP=0", + "PTO_FDWIC_TENSORMAP_RING_CAP=128", + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=7", + "PTO_FDWIC_TRACE_ENABLED=0", + ) + + +@pytest.mark.parametrize( + ("profile", "capture_factory", "phase_id"), + ( + (ARG_BUILD_CAPTURE_MODE, _valid_arg_build_capture, 1), + (EMPTY_BRACKET_CAPTURE_MODE, _valid_empty_bracket_capture, 2), + (MATERIALIZE_CAPTURE_MODE, _valid_materialize_capture, 3), + (CLAIM_CAPTURE_MODE, _valid_claim_capture, 4), + (REGISTER_CAPTURE_MODE, _valid_register_capture, 5), + (SUBMIT_TRANSITION_CAPTURE_MODE, _valid_submit_transition_capture, 6), + (EFDRAIN_CONTROL_CAPTURE_MODE, _valid_efdrain_control_capture, 7), + (PREPARE_MAP_CAPTURE_MODE, _valid_prepare_map_capture, 8), + (FANIN_CAPTURE_MODE, _valid_fanin_capture, 9), + (WINNER_BUILD_CAPTURE_MODE, _valid_winner_build_capture, 10), + (ALLOC_COMPLETE_CAPTURE_MODE, _valid_alloc_complete_capture, 11), + (LOSER_REPLAY_CAPTURE_MODE, _valid_loser_replay_capture, 12), + ), +) +def test_each_phase_profile_closes_raw_identity_provenance_and_html( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + profile: str, + capture_factory: Callable[[], dict[str, Any]], + phase_id: int, +) -> None: + raw_path = _write_capture(tmp_path, capture_factory()) + raw_before = raw_path.read_bytes() + identity = _fake_build_identity(tmp_path, monkeypatch, profile=profile) + + output_path, provenance_path = write_report_with_provenance(raw_path, identity) + + capture = load_capture(raw_path) + provenance, provenance_sha256 = load_provenance(provenance_path, capture) + assert raw_path.read_bytes() == raw_before + assert capture.data["capture"]["mode"] == profile + assert provenance["binding"]["capture_mode"] == profile + assert provenance["build"]["profile"] == profile + assert provenance["build"]["compile_definitions"] == [ + "PTO_FDWIC_SHARED_MAP=0", + "PTO_FDWIC_TENSORMAP_RING_CAP=128", + "PTO_FDWIC_SUBMIT_PMU=1", + f"PTO_FDWIC_SUBMIT_PMU_PHASE_ID={phase_id}", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + document = output_path.read_text(encoding="utf-8") + assert provenance_sha256 in document + assert f"phase_id={phase_id}" in document + assert profile in document + + +@pytest.mark.parametrize("fail_on_final_replace", (1, 2)) +@pytest.mark.parametrize("old_pair_exists", (False, True)) +def test_provenance_pair_publish_failure_restores_the_exact_previous_pair( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + fail_on_final_replace: int, + old_pair_exists: bool, +) -> None: + raw_path = _write_capture(tmp_path, _valid_capture()) + raw_before = raw_path.read_bytes() + identity = _fake_build_identity(tmp_path, monkeypatch) + output_path = tmp_path / DEFAULT_OUTPUT_NAME + provenance_path = tmp_path / DEFAULT_PROVENANCE_NAME + old_output = b"old-report" if old_pair_exists else None + old_provenance = b"old-provenance" if old_pair_exists else None + if old_pair_exists: + output_path.write_bytes(old_output) + provenance_path.write_bytes(old_provenance) + + real_replace = report_module.os.replace + final_replaces = 0 + + def fail_selected_final_replace(source: str | Path, destination: str | Path) -> None: + nonlocal final_replaces + source_path = Path(source) + destination_path = Path(destination) + if ".pending." in source_path.name and destination_path in {provenance_path, output_path}: + final_replaces += 1 + if final_replaces == fail_on_final_replace: + raise OSError("injected paired publication failure") + real_replace(source, destination) + + monkeypatch.setattr(report_module.os, "replace", fail_selected_final_replace) + + with pytest.raises(OSError, match="injected paired publication failure"): + write_report_with_provenance(raw_path, identity) + + assert raw_path.read_bytes() == raw_before + if old_pair_exists: + assert output_path.read_bytes() == old_output + assert provenance_path.read_bytes() == old_provenance + else: + assert not output_path.exists() + assert not provenance_path.exists() + assert not list(tmp_path.glob(".*.pending.*.tmp")) + assert not list(tmp_path.glob(".*.rollback.*.tmp")) + + +@pytest.mark.parametrize("change_on_check", (2, 3)) +def test_provenance_pair_rejects_raw_change_before_or_after_final_replaces( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + change_on_check: int, +) -> None: + raw_path = _write_capture(tmp_path, _valid_capture()) + identity = _fake_build_identity(tmp_path, monkeypatch) + real_assert_unchanged = report_module._assert_capture_raw_unchanged + checks = 0 + + def change_raw_on_selected_check(capture: report_module.SubmitPmuCapture) -> bytes: + nonlocal checks + checks += 1 + if checks == change_on_check: + changed = bytearray(raw_path.read_bytes()) + changed[-2] = ord(" ") if changed[-2] != ord(" ") else ord("\n") + raw_path.write_bytes(changed) + return real_assert_unchanged(capture) + + monkeypatch.setattr(report_module, "_assert_capture_raw_unchanged", change_raw_on_selected_check) + + with pytest.raises(ValueError, match="raw changed after the validated capture snapshot"): + write_report_with_provenance(raw_path, identity) + + assert not (tmp_path / DEFAULT_OUTPUT_NAME).exists() + assert not (tmp_path / DEFAULT_PROVENANCE_NAME).exists() + assert not list(tmp_path.glob(".*.pending.*.tmp")) + assert not list(tmp_path.glob(".*.rollback.*.tmp")) + + +def test_render_without_provenance_keeps_the_legacy_raw_only_path(tmp_path: Path) -> None: + raw_path = _write_capture(tmp_path, _valid_capture()) + + document = render_report(raw_path) + + assert "真实 FDWIC Scalar Submit PMU" in document + assert "诊断构建身份" not in document + assert "Provenance SHA-256" not in document + + +@pytest.mark.parametrize( + ("field", "replacement"), + ( + ("raw_sha256", "0" * 64), + ("capture_mode", ARG_BUILD_CAPTURE_MODE), + ), +) +def test_provenance_rejects_raw_binding_tamper( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + field: str, + replacement: str, +) -> None: + _output_path, provenance_path, _identity = _publish_fake_provenance(tmp_path, monkeypatch) + capture = load_capture(tmp_path / DEFAULT_INPUT_NAME) + provenance = json.loads(provenance_path.read_text(encoding="utf-8")) + provenance["binding"][field] = replacement + provenance_path.write_text(json.dumps(provenance), encoding="utf-8") + + with pytest.raises(ValueError, match="provenance.binding does not match"): + load_provenance(provenance_path, capture) + + +def _change_provenance_source_state(provenance: dict[str, Any]) -> None: + fields = provenance["build"]["source_state"].split(":") + fields[1] = "3" * 40 + provenance["build"]["source_state"] = ":".join(fields) + + +def _change_provenance_definition_hash(provenance: dict[str, Any]) -> None: + mismatched_hash = "f" * 64 + fields = provenance["build"]["source_state"].split(":") + fields[3] = mismatched_hash + provenance["build"]["source_state"] = ":".join(fields) + provenance["build"]["definitions_sha256"] = mismatched_hash + + +@pytest.mark.parametrize( + ("mutation", "message"), + ( + (_change_provenance_source_state, "does not close against its profile/source state"), + (_change_provenance_definition_hash, "compile definitions do not match definitions_sha256"), + ), +) +def test_provenance_rejects_source_state_or_definition_hash_mismatch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + mutation: Callable[[dict[str, Any]], None], + message: str, +) -> None: + _output_path, provenance_path, _identity = _publish_fake_provenance(tmp_path, monkeypatch) + capture = load_capture(tmp_path / DEFAULT_INPUT_NAME) + provenance = json.loads(provenance_path.read_text(encoding="utf-8")) + mutation(provenance) + provenance_path.write_text(json.dumps(provenance), encoding="utf-8") + + with pytest.raises(ValueError, match=message): + load_provenance(provenance_path, capture) + + +@pytest.mark.parametrize("artifact_name", ("aiv_combined", "aicpu_runtime")) +def test_provenance_rejects_artifact_change_after_identity_freeze_without_publishing( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + artifact_name: str, +) -> None: + raw_path = _write_capture(tmp_path, _valid_capture()) + raw_before = raw_path.read_bytes() + identity = _fake_build_identity(tmp_path, monkeypatch) + changed_artifact = dict(identity.artifacts)[artifact_name].path + changed_artifact.write_bytes(changed_artifact.read_bytes() + b"-changed-after-freeze") + + with pytest.raises(ValueError, match=f"build artifact changed after identity freeze: {artifact_name}"): + write_report_with_provenance(raw_path, identity) + + assert raw_path.read_bytes() == raw_before + assert not (tmp_path / DEFAULT_OUTPUT_NAME).exists() + assert not (tmp_path / DEFAULT_PROVENANCE_NAME).exists() + assert not (tmp_path / f"{DEFAULT_OUTPUT_NAME}.tmp").exists() + assert not (tmp_path / f"{DEFAULT_PROVENANCE_NAME}.tmp").exists() + + +def _remove_required_provenance_field(provenance: dict[str, Any]) -> None: + provenance["build"].pop("source_state_path") + + +def _add_unexpected_provenance_field(provenance: dict[str, Any]) -> None: + provenance["artifacts"]["unexpected"] = {} + + +@pytest.mark.parametrize( + ("mutation", "message"), + ( + (_remove_required_provenance_field, "provenance.build fields do not match"), + (_add_unexpected_provenance_field, "provenance.artifacts fields do not match"), + ), +) +def test_provenance_rejects_missing_or_extra_schema_fields( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + mutation: Callable[[dict[str, Any]], None], + message: str, +) -> None: + _output_path, provenance_path, _identity = _publish_fake_provenance(tmp_path, monkeypatch) + capture = load_capture(tmp_path / DEFAULT_INPUT_NAME) + provenance = json.loads(provenance_path.read_text(encoding="utf-8")) + mutation(provenance) + provenance_path.write_text(json.dumps(provenance), encoding="utf-8") + + with pytest.raises(ValueError, match=message): + load_provenance(provenance_path, capture) + + +def test_provenance_apis_reject_wrong_output_filenames( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + raw_path = _write_capture(tmp_path, _valid_capture()) + capture = load_capture(raw_path) + identity = _fake_build_identity(tmp_path, monkeypatch) + wrong_provenance = tmp_path / "provenance.json" + wrong_provenance.write_text("{}", encoding="utf-8") + + with pytest.raises(ValueError, match=DEFAULT_PROVENANCE_NAME): + load_provenance(wrong_provenance, capture) + wrong_provenance.unlink() + + with pytest.raises(ValueError, match=DEFAULT_PROVENANCE_NAME): + write_report_with_provenance(raw_path, identity, provenance_path=wrong_provenance) + with pytest.raises(ValueError, match=DEFAULT_OUTPUT_NAME): + write_report_with_provenance(raw_path, identity, output_path=tmp_path / "report.html") + + assert not wrong_provenance.exists() + assert not (tmp_path / DEFAULT_PROVENANCE_NAME).exists() + assert not (tmp_path / DEFAULT_OUTPUT_NAME).exists() + + +def test_float_summary_accepts_the_cpp_emitters_twelve_significant_digits(tmp_path: Path) -> None: + capture = _valid_capture() + for group in capture["summary"].values(): + for metric in ("total_cycles", "scalar_busy", "icache_requests", "icache_misses"): + group[metric]["mean"] = float(f"{group[metric]['mean']:.12g}") + group["scalar_busy_share"] = float(f"{group['scalar_busy_share']:.12g}") + group["icache_miss_rate"] = float(f"{group['icache_miss_rate']:.12g}") + + loaded = load_capture(_write_capture(tmp_path, capture)) + + assert loaded.summary["all"]["cores"] == 96 + + +def test_counter_immediately_below_risk_threshold_is_accepted(tmp_path: Path) -> None: + capture = _valid_capture() + accepted_value = (1 << 30) - 2 + capture["records"][0]["icache_requests"] = accepted_value + records = capture["records"] + capture["summary"] = { + "all": _group_summary(records), + "aic": _group_summary(records[:32]), + "aiv": _group_summary(records[32:]), + } + + loaded = load_capture(_write_capture(tmp_path, capture)) + + assert loaded.records[0]["icache_requests"] == accepted_value + + +Mutation = Callable[[dict[str, Any]], None] + + +def _duplicate_physical(capture: dict[str, Any]) -> None: + capture["records"][1]["physical_core_id"] = capture["records"][0]["physical_core_id"] + + +def _wrong_role_count(capture: dict[str, Any]) -> None: + capture["records"][31]["role"] = "aiv" + + +def _broken_triplet(capture: dict[str, Any]) -> None: + records = capture["records"] + records[32]["physical_core_id"] = 50 + physical_ids = [record["physical_core_id"] for record in records] + capture["owner"]["configured_bitmap_words"] = _bitmap_words(physical_ids) + + +def _owner_not_restored(capture: dict[str, Any]) -> None: + capture["owner"]["restore_passed"] = False + + +def _owner_bitmap_mismatch(capture: dict[str, Any]) -> None: + capture["owner"]["configured_bitmap_words"][0] &= ~1 + + +def _selector_changed(capture: dict[str, Any]) -> None: + capture["configuration"]["selectors"]["cnt7_primary_icache_miss"] = 0x36 + + +def _frequency_changed(capture: dict[str, Any]) -> None: + capture["configuration"]["pmu_cycles_per_ns"]["aiv"] = 1.0 + + +def _schema_changed(capture: dict[str, Any]) -> None: + capture["schema"] = "fdwic-submit-pmu-v1" + + +def _status_missing(capture: dict[str, Any]) -> None: + capture["records"][0]["status"] &= ~(1 << 9) + + +def _submit_count_mismatch(capture: dict[str, Any]) -> None: + capture["records"][0]["submit_count"] = 4 + + +def _scalar_exceeds_total(capture: dict[str, Any]) -> None: + capture["records"][0]["scalar_busy"] = capture["records"][0]["total_cycles"] + 1 + + +def _miss_exceeds_request(capture: dict[str, Any]) -> None: + capture["records"][0]["icache_misses"] = capture["records"][0]["icache_requests"] + 1 + + +def _request_is_zero(capture: dict[str, Any]) -> None: + capture["records"][0]["icache_requests"] = 0 + capture["records"][0]["icache_misses"] = 0 + + +def _none_publishes_shadow(capture: dict[str, Any]) -> None: + capture["records"][0]["shadow_icache_requests"] = capture["records"][0]["icache_requests"] + + +def _counter_reaches_threshold(capture: dict[str, Any]) -> None: + capture["records"][0]["icache_requests"] = (1 << 30) - 1 + + +def _summary_tampered(capture: dict[str, Any]) -> None: + capture["summary"]["aiv"]["icache_misses"]["sum"] += 1 + + +def _producer_validation_failed(capture: dict[str, Any]) -> None: + capture["validation"]["passed"] = False + + +@pytest.mark.parametrize( + ("mutation", "message"), + ( + (_duplicate_physical, "physical_core_id values must be unique"), + (_wrong_role_count, "exactly 32 AIC and 64 AIV"), + (_broken_triplet, "complete 1:2 mixed triplets"), + (_owner_not_restored, "owner.restore_passed must be true"), + (_owner_bitmap_mismatch, "configured_bitmap_words must exactly match"), + (_selector_changed, "configuration.selectors"), + (_frequency_changed, "configuration.pmu_cycles_per_ns"), + (_schema_changed, "schema must equal"), + (_status_missing, "status must equal"), + (_submit_count_mismatch, "submit_count does not close"), + (_scalar_exceeds_total, "scalar_busy exceeds total_cycles"), + (_miss_exceeds_request, "icache_misses exceeds icache_requests"), + (_request_is_zero, "icache_requests must be an integer >= 1"), + (_none_publishes_shadow, "must not publish redundant shadow counters"), + (_counter_reaches_threshold, "programmable counter reaches the risk threshold"), + (_summary_tampered, "summary.aiv.icache_misses.sum"), + (_producer_validation_failed, "validation.passed"), + ), +) +def test_strict_capture_gates(tmp_path: Path, mutation: Mutation, message: str) -> None: + capture = _valid_capture() + mutation(capture) + raw_path = _write_capture(tmp_path, capture) + + with pytest.raises(ValueError, match=message): + load_capture(raw_path) + + +def test_invalid_raw_never_publishes_html(tmp_path: Path) -> None: + capture = _valid_capture() + _owner_not_restored(capture) + raw_path = _write_capture(tmp_path, capture) + + with pytest.raises(ValueError, match="owner.restore_passed must be true"): + write_report(raw_path) + + assert not (tmp_path / DEFAULT_OUTPUT_NAME).exists() + assert not (tmp_path / f"{DEFAULT_OUTPUT_NAME}.tmp").exists() + + +def test_report_requires_the_fixed_raw_filename(tmp_path: Path) -> None: + path = tmp_path / "run1.json" + path.write_text(json.dumps(_valid_capture()), encoding="utf-8") + + with pytest.raises(ValueError, match=DEFAULT_INPUT_NAME): + load_capture(path) diff --git a/tests/ut/py/test_fdwic_submit_span_overview.py b/tests/ut/py/test_fdwic_submit_span_overview.py new file mode 100644 index 0000000000..0ef64f37e3 --- /dev/null +++ b/tests/ut/py/test_fdwic_submit_span_overview.py @@ -0,0 +1,971 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the LICENSE file. +# ----------------------------------------------------------------------------------------------------------- +"""Offline closure tests for the provenance-aware FDWIC Submit span overview.""" + +from __future__ import annotations + +import copy +import hashlib +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest + +import simpler_setup.tools.fdwic_submit_span_overview as overview_module +from simpler_setup.tools.fdwic_submit_pmu_report import SubmitPmuCapture + +_PER_CORE_METRICS = { + "submit_envelope": 100, + "submit_union": 80, + "between_submit_residual": 20, + "efdrain": 20, + "materialize": 10, + "prepare_map": 5, + "claim": 7, + "fanin": 3, + "register": 8, + "winner_build": 6, + "alloc_complete": 4, + "loser_replay": 5, + "submit_residual": 12, + "efdrain_kernel_union": 8, + "efdrain_control": 12, + "orchestration_replay": 120, + "orchestration_setup": 10, + "orchestration_tail": 10, + "final_drain": 15, + "final_drain_kernel_union": 10, + "final_drain_residual": 5, + "worker_completion": 135, +} + + +def _valid_swimlane_analysis() -> dict[str, Any]: + per_core = [ + { + "core_id": core_id, + "role": "aic" if core_id < 32 else "aiv", + "submit_count": 5, + "metrics_cycles": dict(_PER_CORE_METRICS), + } + for core_id in range(96) + ] + aggregate = {metric: value * 96 for metric, value in _PER_CORE_METRICS.items()} + return { + "schema_version": overview_module.REPORT_SCHEMA_VERSION, + "validation": { + "status": "PASS", + "dropped_records": 0, + "physical_topology_complete": True, + "task_stream_contiguous_and_equal_per_core": True, + "orchestration_parent_exactly_one_per_core": True, + "final_drain_parent_exactly_one_per_core": True, + "parent_boundaries_adjacent": True, + "exclusive_children_non_overlapping": True, + "all_integer_cycle_closures_exact": True, + }, + "capture": { + "trace_schema_version": 4, + "clock_freq_hz": 1_000_000_000, + "core_count": 96, + "task_count_per_core": 5, + }, + "aggregate_core_work": { + "metrics_cycles": aggregate, + "closure": { + name: {"exact": True} + for name in ( + "submit_partition", + "submit_envelope", + "efdrain_partition", + "orchestration_replay", + "final_drain", + "worker_completion", + ) + }, + }, + "per_core": per_core, + "residual_breakdown": { + "submit_internal_residual": { + "total_cycles": 7 * 96, + "segments": [ + { + "boundary": "Claim->Materialize", + "event_count": 5 * 96, + "cycles": 4 * 96, + "aic_cycles": 4 * 32, + "aiv_cycles": 4 * 64, + }, + { + "boundary": "Register->WinnerBuild", + "event_count": 5 * 96, + "cycles": 3 * 96, + "aic_cycles": 3 * 32, + "aiv_cycles": 3 * 64, + }, + ], + }, + "submit_tail_residual": { + "total_cycles": 5 * 96, + "segments": [ + { + "boundary": "LoserReplay->SubmitEnd", + "event_count": 5 * 96, + "cycles": 5 * 96, + "aic_cycles": 5 * 32, + "aiv_cycles": 5 * 64, + } + ], + }, + "between_submit_residual": { + "total_cycles": 20 * 96, + "segments": [ + { + "boundary": "SubmitEnd->NextSubmitBegin", + "event_count": 4 * 96, + "cycles": 20 * 96, + "aic_cycles": 20 * 32, + "aiv_cycles": 20 * 64, + } + ], + }, + }, + "overlays": { + "Atomic": { + "event_count": 1, + "aggregate_duration_cycles": 2, + "included_in_additive_totals": False, + } + }, + "global_submit_makespan": {"duration_cycles": 5_000_000, "duration_us": 5_000.0}, + "global_worker_completion_makespan": {"duration_cycles": 5_100_000, "duration_us": 5_100.0}, + "kernel_containment": {"submit_contained": 1, "top_level_residual": 0}, + } + + +def _metric_summary(cores: int, base: int) -> dict[str, int | float]: + return {"sum": base * cores, "min": base, "mean": base + 0.5, "max": base + 1} + + +def _denominator_summary(cores: int) -> dict[str, Any]: + return { + "cores": cores, + "scalar_submit_elapsed_ticks": _metric_summary(cores, 10_000), + "total_cycles": _metric_summary(cores, 16_000), + "scalar_busy": _metric_summary(cores, 12_000), + "non_scalar_busy_cycles": _metric_summary(cores, 4_000), + "icache_requests": _metric_summary(cores, 2_000), + "icache_misses": _metric_summary(cores, 200), + } + + +def _phase_summary(cores: int, calls_per_core: int) -> dict[str, Any]: + return { + "cores": cores, + "phase_total_cycles_observed": _metric_summary(cores, 1_000), + "phase_scalar_busy_observed": _metric_summary(cores, 600), + "phase_non_scalar_busy_cycles": _metric_summary(cores, 400), + "shadow_scalar_loss": _metric_summary(cores, 20), + "phase_elapsed_ticks": _metric_summary(cores, 600), + "phase_icache_requests_observed": _metric_summary(cores, 180), + "phase_icache_misses_observed": _metric_summary(cores, 18), + "phase_total_share_of_pmu_total": 0.0625, + "phase_scalar_share_of_whole_scalar": 0.05, + "phase_scalar_busy_share_of_phase_total": 0.6, + "phase_request_observed_share_of_primary": 0.09, + "phase_miss_observed_share_of_primary": 0.09, + "phase_business_calls": calls_per_core * cores, + "phase_calls_per_core": _metric_summary(cores, calls_per_core), + "phase_zero_call_cores": 0, + "phase_end_reads": calls_per_core * cores, + "phase_excluded_kernel_calls": 0, + } + + +def _fake_capture(raw_path: Path, mode: str) -> SubmitPmuCapture: + raw_bytes = raw_path.read_bytes() + phase = None + phase_summary = None + if mode != overview_module.NONE_CAPTURE_MODE: + phase = { + "id": overview_module.PMU_MODE_ORDER.index(mode), + "name": mode.removeprefix("submit-pmu-"), + "boundary": f"{mode}-begin-to-end", + "counter_semantics": "running_read_clear_observed_bracket", + "time_semantics": "boundary_diagnostic_sys_cnt_between_observers", + "pmu_observation": {"boundary": "running-read-clear"}, + } + phase_summary = { + "all": _phase_summary(96, 5), + "aic": _phase_summary(32, 5), + "aiv": _phase_summary(64, 5), + } + data = { + "configuration": { + "num_cores": 96, + "aic_cores": 32, + "aiv_cores": 64, + "expected_submits_per_core": 5, + "sys_counter_tick_ns": 1, + "selectors": {"test": "fixed"}, + "linked_kernel_exclusion": {"enabled": True}, + "return_ready_atomic_exclusion": {"enabled": True}, + "counter_width_bits": {"total": 64, "programmable": 32}, + "programmable_counter_risk_threshold": (1 << 30) - 1, + "pmu_cycles_per_ns": {"all": 1.649844, "aic": 1.650062, "aiv": 1.649731}, + "phase": phase, + }, + "capture": { + "mode": mode, + "window_scope": "per_core_first_submit_begin_to_last_submit_end", + }, + "window": {"global_submit_span_us": 5_000.0}, + } + return SubmitPmuCapture( + input_path=raw_path, + raw_size=len(raw_bytes), + raw_sha256=hashlib.sha256(raw_bytes).hexdigest(), + data=data, + records=(), + groups={}, + summary={ + "all": _denominator_summary(96), + "aic": _denominator_summary(32), + "aiv": _denominator_summary(64), + }, + phase_summary=phase_summary, + ) + + +def _fake_provenance(mode: str, git_head: str = "1" * 40) -> dict[str, Any]: + return { + "build": { + "git_head": git_head, + "source_fingerprint": f"source-{mode}", + "profiled_cache_key": ["a5", "fdwic", "Case1", mode], + }, + "artifacts": { + "aicore_kernel": { + "sha256": hashlib.sha256(mode.encode()).hexdigest(), + "text": {"sha256": hashlib.sha256(f"text-{mode}".encode()).hexdigest()}, + } + }, + } + + +@dataclass +class _Evidence: + swimlane_raw: Path + analysis: dict[str, Any] + pmu_dirs: list[Path] + captures: dict[Path, SubmitPmuCapture] + provenance: dict[Path, dict[str, Any]] + + +@pytest.fixture +def evidence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> _Evidence: + swimlane_raw = tmp_path / "swimlane" / "l2_swimlane_records.json" + swimlane_raw.parent.mkdir() + swimlane_raw.write_text('{"schema_version":4}\n', encoding="utf-8") + analysis = _valid_swimlane_analysis() + + captures: dict[Path, SubmitPmuCapture] = {} + provenance: dict[Path, dict[str, Any]] = {} + pmu_dirs = [] + for mode in overview_module.PMU_MODE_ORDER: + directory = tmp_path / mode + directory.mkdir() + raw_path = directory / overview_module.DEFAULT_INPUT_NAME + raw_path.write_text(f'{{"mode":"{mode}"}}\n', encoding="utf-8") + provenance_path = directory / overview_module.DEFAULT_PROVENANCE_NAME + provenance_path.write_text(f'{{"profile":"{mode}"}}\n', encoding="utf-8") + captures[raw_path.resolve()] = _fake_capture(raw_path, mode) + provenance[provenance_path.resolve()] = _fake_provenance(mode) + pmu_dirs.append(directory) + + def fake_load_capture(path: Path | str) -> SubmitPmuCapture: + return captures[Path(path).resolve()] + + def fake_load_provenance(path: Path | str, capture: SubmitPmuCapture) -> tuple[dict[str, Any], str]: + del capture + resolved = Path(path).resolve() + return provenance[resolved], hashlib.sha256(resolved.read_bytes()).hexdigest() + + monkeypatch.setattr(overview_module, "analyze_capture", lambda path: analysis) + monkeypatch.setattr(overview_module, "load_capture", fake_load_capture) + monkeypatch.setattr(overview_module, "load_provenance", fake_load_provenance) + return _Evidence(swimlane_raw, analysis, pmu_dirs, captures, provenance) + + +def test_swimlane_analysis_requires_strict_closure_and_builds_exact_partitions(evidence: _Evidence) -> None: + payload = overview_module.build_overview(evidence.swimlane_raw, evidence.pmu_dirs) + swimlane = payload["swimlane_elf"] + + assert swimlane["core_count"] == 96 + for partition_name in ( + "submit_envelope_partition", + "submit_union_partition", + "efdrain_partition", + "orchestration_partition", + "final_drain_partition", + "worker_completion_partition", + ): + partition = swimlane[partition_name] + assert partition["exact"] is True + assert partition["parent_core_time_ticks"] == sum(row["core_time_ticks"] for row in partition["rows"]) + + broken_closure = copy.deepcopy(evidence.analysis) + broken_closure["aggregate_core_work"]["closure"]["submit_partition"]["exact"] = False + with pytest.raises(ValueError, match="aggregate closure 'submit_partition' is not exact"): + overview_module._validate_swimlane_analysis(broken_closure) + + broken_partition = copy.deepcopy(evidence.analysis) + broken_partition["aggregate_core_work"]["metrics_cycles"]["submit_union"] += 96 + for core in broken_partition["per_core"]: + core["metrics_cycles"]["submit_union"] += 1 + frequency_hz, _ = overview_module._validate_swimlane_analysis(broken_partition) + raw_size, raw_sha256 = overview_module._sha256_file(evidence.swimlane_raw) + with pytest.raises(ValueError, match="overview partition 'SubmitEnvelope' does not close"): + overview_module._summarize_swimlane( + broken_partition, + evidence.swimlane_raw, + raw_size, + raw_sha256, + frequency_hz, + ) + + +def test_complete_thirteen_profile_overview_accepts_one_revision_and_rejects_mixed_phase_calibration( + evidence: _Evidence, +) -> None: + payload = overview_module.build_overview(evidence.swimlane_raw, evidence.pmu_dirs) + + validation = payload["validation"] + assert validation["status"] == "PASS" + assert validation["pmu_profile_count"] == 13 + assert validation["pmu_profiles_complete"] is True + assert validation["git_head_uniform"] is True + assert validation["git_heads"] == ["1" * 40] + assert validation["swimlane_to_pmu_identity_bound"] is False + assert payload["submit_pmu_elfs"]["whole_window"]["capture_mode"] == overview_module.NONE_CAPTURE_MODE + assert len(payload["submit_pmu_elfs"]["phase_profiles"]) == 11 + assert payload["submit_pmu_elfs"]["calibration"]["capture_mode"] == overview_module.EMPTY_BRACKET_CAPTURE_MODE + assert payload["schema"] == "fdwic-submit-span-overview-v5" + assert payload["semantics"]["cross_elf_phase_shares_additive"] is False + assert payload["semantics"]["cross_elf_synthetic_phase_sum_is_exact_overhead"] is False + + phase_group = payload["submit_pmu_elfs"]["phase_profiles"][0]["phase"]["groups"]["all"] + assert phase_group["phase_total_cycles_observed"]["sum"] == 96_000 + assert phase_group["phase_scalar_busy_observed"]["sum"] == 57_600 + assert phase_group["phase_non_scalar_busy_cycles"]["sum"] == 38_400 + assert phase_group["phase_total_share_of_pmu_total"] == 0.0625 + assert phase_group["phase_scalar_share_of_whole_scalar"] == 0.05 + assert phase_group["phase_scalar_busy_share_of_phase_total"] == 0.6 + for profile in payload["submit_pmu_elfs"]["phase_profiles"]: + reference = profile["recording_cost_reference"] + assert reference["target_capture_mode"] == profile["capture_mode"] + assert reference["calibration_capture_mode"] == overview_module.EMPTY_BRACKET_CAPTURE_MODE + assert reference["exact_correction"] is False + assert reference["raw_whole_denominator_is_unchanged"] is True + assert reference["whole_recording_cost_is_not_measured"] is True + synthetic = payload["submit_pmu_elfs"]["synthetic_phase_sum_vs_none"] + assert synthetic["phase_profile_count"] == 11 + assert synthetic["empty_bracket_excluded_from_raw_phase_sum"] is True + assert synthetic["empty_bracket_used_as_recording_cost_calibration"] is True + assert synthetic["formal_partition_closure"] is False + assert synthetic["exact_recording_cost_correction"] is False + + mixed_mode = overview_module.LOSER_REPLAY_CAPTURE_MODE + mixed_directory = evidence.pmu_dirs[overview_module.PMU_MODE_ORDER.index(mixed_mode)] + mixed_path = (mixed_directory / overview_module.DEFAULT_PROVENANCE_NAME).resolve() + evidence.provenance[mixed_path]["build"]["git_head"] = "2" * 40 + with pytest.raises( + ValueError, + match=( + f"Submit-PMU {mixed_mode} and {overview_module.EMPTY_BRACKET_CAPTURE_MODE} " + "provenance git heads do not match" + ), + ): + overview_module.build_overview(evidence.swimlane_raw, evidence.pmu_dirs) + + +def test_missing_and_duplicate_profiles_are_rejected(evidence: _Evidence) -> None: + with pytest.raises(ValueError, match="requires exactly 13 Submit-PMU directories"): + overview_module.build_overview(evidence.swimlane_raw, evidence.pmu_dirs[:-1]) + + duplicate_dirs = [*evidence.pmu_dirs[:-1], evidence.pmu_dirs[0]] + with pytest.raises(ValueError, match=f"duplicate Submit-PMU capture mode: {overview_module.NONE_CAPTURE_MODE}"): + overview_module.build_overview(evidence.swimlane_raw, duplicate_dirs) + + +def test_pmu_task_count_must_match_swimlane(evidence: _Evidence) -> None: + mode = overview_module.CLAIM_CAPTURE_MODE + profile_directory = evidence.pmu_dirs[overview_module.PMU_MODE_ORDER.index(mode)] + raw_path = (profile_directory / overview_module.DEFAULT_INPUT_NAME).resolve() + evidence.captures[raw_path].data["configuration"]["expected_submits_per_core"] = 6 + + with pytest.raises(ValueError, match=f"Submit-PMU {mode} task count does not match the swimlane capture"): + overview_module.build_overview(evidence.swimlane_raw, evidence.pmu_dirs) + + +def test_coverage_matrix_contains_all_exclusive_spans_and_residuals(evidence: _Evidence) -> None: + payload = overview_module.build_overview(evidence.swimlane_raw, evidence.pmu_dirs) + regions = {row["swimlane_region"] for row in payload["coverage_matrix"]} + exclusive_spans = {label for label, _ in overview_module.SUBMIT_PHASES} + + assert exclusive_spans <= regions + assert "BetweenSubmitResidual" in regions + assert "SubmitInternalResidual/Claim->Materialize" in regions + assert "SubmitInternalResidual/Register->WinnerBuild" in regions + assert "SubmitTailResidual/LoserReplay->SubmitEnd" in regions + assert { + "WorkerCompletion", + "OrchestrationReplay", + "OrchestrationSetup", + "OrchestrationTail", + "SubmitUnion", + "SubmitResidual", + "EfDrainKernelUnion", + "EfDrainControl", + "FinalDrain", + "FinalDrainKernelUnion", + "FinalDrainResidual", + "Atomic overlay", + } <= regions + + +def test_html_distinguishes_formal_ratios_from_synthetic_cross_elf_diagnostic(evidence: _Evidence) -> None: + document = overview_module.render_overview(overview_module.build_overview(evidence.swimlane_raw, evidence.pmu_dirs)) + + assert all(line == line.rstrip() for line in document.splitlines()) + assert "各 PMU 行的占比也不能相加成 100%" in document + assert "业务 phase 的主显示值" in document + assert "先从 raw observed 分子中扣除" in document + assert "再除以本 ELF 的 raw whole" in document + assert "raw observed / raw whole 同格保留" in document + assert "没有测到 whole 窗口中的全部记录工作" in document + assert "只是更接近业务量级的参考值" in document + assert "不是“纯业务阶段 / 纯业务整窗”的精确占比" in document + assert "原始观测合计" in document + assert "空区间估算的记录代码开销" in document + assert "扣除上述估算后的参考值" in document + assert "不是精确校正" in document + assert "泳道同父区间" in document + assert "PMU 局部记录扣除参考" in document + assert "扣局部记录估算后的 Phase PMU / raw whole PMU" in document + assert "Scalar 局部记录扣除参考" in document + assert "扣局部记录估算后的 Phase scalar / raw whole scalar" in document + assert "raw 比例同格保留" in document + assert '' in document + assert ".partition table { min-width:920px; table-layout:fixed; }" in document + assert ".partition table { min-width:1120px; }" not in document + assert "white-space:normal; vertical-align:top; overflow-wrap:anywhere;" in document + assert "并非泳道父区间，也不是" in document + assert "AIC 每核 min–max" in document + assert "AIV 每核 min–max" in document + assert "Phase PMU total" in document + assert "Phase scalar busy" in document + assert "非 Scalar-busy 残余" in document + assert "raw Phase total / raw whole total：96,000 / 1,536,000 = 6.250%" in document + assert "参考 Scalar / 参考 Phase total：N/A；raw 60.000%" in document + assert "raw Phase scalar / raw whole scalar：57,600 / 1,152,000 = 5.000%" in document + assert "参考 residual / 参考 Phase total：N/A；raw 40.000%" in document + assert "AIC 每核 1,000–1,001 cycles（0.606–0.607 µs）" in document + assert "SYS 边界诊断" in document + assert "只核验 phase 边界闭合，不作为阶段主时间或占比分母" in document + assert "1.649844/1.650062/1.649731 cycles/ns" in document + assert "SYS 边界闭合诊断（raw ticks）" in document + assert "不参与 PMU 1.65 GHz 时间换算或阶段比例" in document + assert "不能求和后与" in document + assert "submit-pmu-none 做正式判等" in document + assert "Scalar phase core-time" not in document + assert "Observer pair empirical elapsed" not in document + assert "泳道与 PMU 之间仅对齐 96 核拓扑和每核 Submit 数" in document + for forbidden in ("median", "p95", "净性能收益", "精确观察开销"): + assert forbidden not in document + + +def test_submit_union_uses_per_core_means_and_decomposes_ten_phase_observations(evidence: _Evidence) -> None: + payload = overview_module.build_overview(evidence.swimlane_raw, evidence.pmu_dirs) + pmu = payload["submit_pmu_elfs"] + phase_by_mode = {str(item["capture_mode"]): item for item in pmu["phase_profiles"]} + phase_by_metric = { + metric: phase_by_mode[mode] for metric, mode in overview_module.PARTITION_PMU_MODE_BY_METRIC.items() + } + profiles = [phase_by_mode[mode] for mode in overview_module.SUBMIT_UNION_PMU_MODES] + diagnostic = overview_module._partition_mean_diagnostic( + payload["swimlane_elf"]["submit_union_partition"], + phase_by_metric, + profiles, + pmu["whole_window"], + payload["swimlane_elf"]["core_count"], + ) + + assert diagnostic["profiles"] == list(overview_module.SUBMIT_UNION_PMU_MODES) + assert overview_module.ARG_BUILD_CAPTURE_MODE in diagnostic["profiles"] + assert overview_module.SUBMIT_TRANSITION_CAPTURE_MODE not in diagnostic["profiles"] + assert overview_module.NONE_CAPTURE_MODE not in diagnostic["profiles"] + assert overview_module.EMPTY_BRACKET_CAPTURE_MODE not in diagnostic["profiles"] + assert diagnostic["direct_mapped_profile_count"] == 9 + assert diagnostic["unmapped_labels"] == ["SubmitInternalResidual", "SubmitTailResidual"] + decomposition = diagnostic["decomposition"] + all_group = decomposition["groups"]["all"] + all_metrics = all_group["metrics"] + assert all_group["cores"] == 96 + assert all_group["phase_record_pairs"] == 4_800 + assert all_group["phase_record_pairs_per_core"] == 50.0 + assert all_group["phase_business_calls"] == 4_800 + assert all_group["phase_business_calls_per_core"] == 50.0 + assert all_group["empty_calibration_record_pairs"] == 480 + assert all_metrics["pmu_total_cycles"] == { + "label": "PMU total", + "unit": "cycles", + "raw_observed_sum": 960_000, + "raw_observed_mean": 10_000.0, + "submit_none_sum": 1_536_000, + "submit_none_mean": 16_000.0, + "raw_observed_ratio_to_submit_none": 0.625, + "empty_cost_per_record_pair": 200.0, + "recording_cost_estimate_sum": 960_000.0, + "recording_cost_estimate_mean": 10_000.0, + "recording_cost_estimate_share_of_raw": 1.0, + "recording_cost_estimate_ratio_to_submit_none": 0.625, + "after_recording_cost_reference_sum": 0.0, + "after_recording_cost_reference_mean": 0.0, + "after_recording_cost_reference_ratio_to_submit_none": 0.0, + "phase_field": "phase_total_cycles_observed", + "submit_none_field": "pmu_total_cycles", + } + assert all_metrics["scalar_busy_cycles"]["raw_observed_sum"] == 576_000 + assert all_metrics["scalar_busy_cycles"]["submit_none_sum"] == 1_152_000 + assert all_metrics["scalar_busy_cycles"]["recording_cost_estimate_ratio_to_submit_none"] == 0.5 + assert all_metrics["scalar_busy_cycles"]["after_recording_cost_reference_sum"] == 0 + assert all_metrics["non_scalar_busy_cycles"]["recording_cost_estimate_ratio_to_submit_none"] == 1.0 + assert all_metrics["icache_requests"]["recording_cost_estimate_ratio_to_submit_none"] == 0.9 + assert all_metrics["icache_misses"]["recording_cost_estimate_ratio_to_submit_none"] == 0.9 + + changed_profiles = copy.deepcopy(profiles) + changed_profiles[0]["denominators"]["all"]["pmu_total_cycles"]["sum"] *= 10 + changed = overview_module._partition_mean_diagnostic( + payload["swimlane_elf"]["submit_union_partition"], + phase_by_metric, + changed_profiles, + pmu["whole_window"], + payload["swimlane_elf"]["core_count"], + ) + changed_metric = changed["decomposition"]["groups"]["all"]["metrics"]["pmu_total_cycles"] + assert changed_metric["raw_observed_sum"] == 960_000 + assert changed_metric["recording_cost_estimate_ratio_to_submit_none"] == 0.625 + + document = overview_module.render_overview(payload) + match = re.search( + r'
\s*

SubmitUnion

(.*?)', + document, + flags=re.DOTALL, + ) + assert match is not None + submit_union_html = match.group(1) + assert "平均每核时间" in submit_union_html + assert "均值占比对照" in submit_union_html + assert "泳道每核均值 / 同父区间每核均值" in submit_union_html + assert "扣局部记录估算后的 Phase PMU 每核均值 / raw whole PMU 每核均值" in submit_union_html + assert "扣局部记录估算后的 Phase scalar 每核均值 / raw whole scalar 每核均值" in submit_union_html + assert "Σ core-time" not in submit_union_html + assert "EfDrain0.020 µs" in submit_union_html + assert 'title="0.020 / 0.080 = 25.000%">25.000%' in submit_union_html + assert "submit-pmu-efdrain-control：参考 每核均值 0.000 / raw whole 16,000.000 = 0.000%" in submit_union_html + assert "raw 每核均值 1,000.000 / 16,000.000 = 6.250%" in submit_union_html + assert "submit-pmu-efdrain-control：参考 每核均值 0.000 / raw whole 12,000.000 = 0.000%" in submit_union_html + assert "raw 每核均值 600.000 / 12,000.000 = 5.000%" in submit_union_html + assert 'data-partition-total="per-core-mean"' in submit_union_html + assert "SubmitUnion 平均每核时间合计" in submit_union_html + assert "0.080 µs11 个分段每核均值之和" in submit_union_html + assert "100.000%" in submit_union_html + assert 'data-phase-profile-count="10"' in submit_union_html + assert "原始分段观测合计" in submit_union_html + assert "10,000.000 cycles/core" in submit_union_html + assert "空区间估算的记录代码开销" in submit_union_html + assert "扣除上述估算后的参考值" in submit_union_html + assert "0.000 cycles/core" in submit_union_html + assert "6,000.000 cycles/core" in submit_union_html + assert 'data-partition-comparison="submit-pmu-none-mean"' in submit_union_html + assert "16,000.000 cycles/core" in submit_union_html + assert "12,000.000 cycles/core" in submit_union_html + assert "包含 BetweenSubmitResidual / SubmitTransition" in submit_union_html + assert "raw 含分段记录代码自身开销，不能直接拿来解释业务耗时" in submit_union_html + assert "偏离 100%" not in submit_union_html + assert "净观察开销" not in submit_union_html + + +def test_synthetic_phase_sum_vs_none_uses_raw_metrics_not_summed_phase_shares(evidence: _Evidence) -> None: + payload = overview_module.build_overview(evidence.swimlane_raw, evidence.pmu_dirs) + diagnostic = payload["submit_pmu_elfs"]["synthetic_phase_sum_vs_none"] + all_group = diagnostic["groups"]["all"] + all_metrics = all_group["metrics"] + + assert diagnostic["included_profiles"] == [ + mode + for mode in overview_module.PMU_MODE_ORDER + if mode not in {overview_module.NONE_CAPTURE_MODE, overview_module.EMPTY_BRACKET_CAPTURE_MODE} + ] + assert all_group["cores"] == 96 + assert all_group["phase_record_pairs"] == 5_280 + assert all_group["phase_record_pairs_per_core"] == 55.0 + assert all_group["phase_business_calls"] == 5_280 + assert all_group["phase_business_calls_per_core"] == 55.0 + assert all_group["empty_calibration_record_pairs"] == 480 + assert all_metrics["pmu_total_cycles"] == { + "label": "PMU total", + "unit": "cycles", + "raw_observed_sum": 1_056_000, + "raw_observed_mean": 11_000.0, + "submit_none_sum": 1_536_000, + "submit_none_mean": 16_000.0, + "raw_observed_ratio_to_submit_none": 0.6875, + "empty_cost_per_record_pair": 200.0, + "recording_cost_estimate_sum": 1_056_000.0, + "recording_cost_estimate_mean": 11_000.0, + "recording_cost_estimate_share_of_raw": 1.0, + "recording_cost_estimate_ratio_to_submit_none": 0.6875, + "after_recording_cost_reference_sum": 0.0, + "after_recording_cost_reference_mean": 0.0, + "after_recording_cost_reference_ratio_to_submit_none": 0.0, + "phase_field": "phase_total_cycles_observed", + "submit_none_field": "pmu_total_cycles", + } + assert all_metrics["scalar_busy_cycles"]["raw_observed_sum"] == 633_600 + assert all_metrics["scalar_busy_cycles"]["submit_none_sum"] == 1_152_000 + assert all_metrics["scalar_busy_cycles"]["recording_cost_estimate_ratio_to_submit_none"] == 0.55 + assert all_metrics["scalar_busy_cycles"]["recording_cost_estimate_share_of_raw"] == 1.0 + assert all_metrics["scalar_busy_cycles"]["after_recording_cost_reference_sum"] == 0 + assert all_metrics["non_scalar_busy_cycles"]["recording_cost_estimate_ratio_to_submit_none"] == 1.1 + assert all_metrics["icache_requests"]["recording_cost_estimate_ratio_to_submit_none"] == pytest.approx(0.99) + assert all_metrics["icache_requests"]["after_recording_cost_reference_sum"] == 0 + assert all_metrics["icache_misses"]["recording_cost_estimate_ratio_to_submit_none"] == pytest.approx(0.99) + assert all_metrics["icache_misses"]["after_recording_cost_reference_sum"] == pytest.approx(0) + for group_name in ("all", "aic", "aiv"): + metrics = diagnostic["groups"][group_name]["metrics"] + for field in ( + "raw_observed_sum", + "recording_cost_estimate_sum", + "after_recording_cost_reference_sum", + ): + assert metrics["pmu_total_cycles"][field] == pytest.approx( + metrics["scalar_busy_cycles"][field] + metrics["non_scalar_busy_cycles"][field] + ) + for metric in metrics.values(): + assert metric["raw_observed_sum"] == pytest.approx( + metric["recording_cost_estimate_sum"] + metric["after_recording_cost_reference_sum"] + ) + + document = overview_module.render_overview(payload) + assert "11 个业务分段的记录开销拆分（含 SubmitTransition）" in document + assert "原始观测合计" in document + assert "空区间估算的记录代码开销" in document + assert "扣除上述估算后的参考值" in document + assert "submit-pmu-none" in document + assert "empty-bracket 不参与" not in document + assert "偏离 100%" not in document + assert "净观察开销" not in document + assert "图形封顶 600%" not in document + + +def test_recording_cost_uses_record_pairs_and_role_weighted_empty_cost(evidence: _Evidence) -> None: + summary = _phase_summary(32, 5) + summary["phase_excluded_kernel_calls"] = 7 + summary["phase_end_reads"] += 7 + summarized = overview_module._phase_group_summary(summary) + assert summarized["phase_business_calls"] == 160 + assert summarized["phase_record_pairs"] == 167 + + payload = overview_module.build_overview(evidence.swimlane_raw, evidence.pmu_dirs) + pmu = payload["submit_pmu_elfs"] + profile = copy.deepcopy(pmu["phase_profiles"][0]) + role_pairs = {"aic": 32, "aiv": 640} + role_raw_total = {"aic": 3_200, "aiv": 192_000} + role_cost = {"aic": 100.0, "aiv": 300.0} + for role in ("aic", "aiv"): + phase_group = profile["phase"]["groups"][role] + phase_group["phase_record_pairs"] = role_pairs[role] + phase_group["phase_business_calls"] = role_pairs[role] + phase_group["phase_total_cycles_observed"]["sum"] = role_raw_total[role] + reference_group = profile["recording_cost_reference"]["groups"][role] + reference_group["target_record_pairs"] = role_pairs[role] + metric = reference_group["metrics"]["pmu_total_cycles"] + metric["raw_phase_observed_sum"] = role_raw_total[role] + metric["recording_cost_estimate_sum"] = role_cost[role] * role_pairs[role] + metric["after_recording_cost_reference_sum"] = ( + metric["raw_phase_observed_sum"] - metric["recording_cost_estimate_sum"] + ) + + all_phase = profile["phase"]["groups"]["all"] + all_phase["phase_record_pairs"] = sum(role_pairs.values()) + all_phase["phase_business_calls"] = sum(role_pairs.values()) + all_phase["phase_total_cycles_observed"]["sum"] = sum(role_raw_total.values()) + all_reference = profile["recording_cost_reference"]["groups"]["all"] + all_reference["target_record_pairs"] = sum(role_pairs.values()) + all_reference_metric = all_reference["metrics"]["pmu_total_cycles"] + all_reference_metric["raw_phase_observed_sum"] = sum(role_raw_total.values()) + all_reference_metric["recording_cost_estimate_sum"] = sum( + role_cost[role] * role_pairs[role] for role in ("aic", "aiv") + ) + all_reference_metric["after_recording_cost_reference_sum"] = ( + all_reference_metric["raw_phase_observed_sum"] - all_reference_metric["recording_cost_estimate_sum"] + ) + + decomposition = overview_module._synthetic_phase_sum_vs_none( + [profile], + pmu["whole_window"], + ) + groups = decomposition["groups"] + all_metric = groups["all"]["metrics"]["pmu_total_cycles"] + aic_estimate = groups["aic"]["metrics"]["pmu_total_cycles"]["recording_cost_estimate_sum"] + aiv_estimate = groups["aiv"]["metrics"]["pmu_total_cycles"]["recording_cost_estimate_sum"] + expected_role_weighted = 100.0 * 32 + 300.0 * 640 + naive_all_average = (112_000 / 480) * 672 + + assert all_metric["recording_cost_estimate_sum"] == expected_role_weighted + assert all_metric["recording_cost_estimate_sum"] == aic_estimate + aiv_estimate + assert all_metric["recording_cost_estimate_sum"] != pytest.approx(naive_all_average) + + more_record_pairs = copy.deepcopy(profile) + more_record_pairs["phase"]["groups"]["aic"]["phase_excluded_kernel_calls"] += 8 + more_record_pairs["phase"]["groups"]["aic"]["phase_record_pairs"] += 8 + more_record_pairs["phase"]["groups"]["all"]["phase_excluded_kernel_calls"] += 8 + more_record_pairs["phase"]["groups"]["all"]["phase_record_pairs"] += 8 + for group_name in ("aic", "all"): + reference_group = more_record_pairs["recording_cost_reference"]["groups"][group_name] + reference_group["target_record_pairs"] += 8 + metric = reference_group["metrics"]["pmu_total_cycles"] + metric["raw_phase_observed_sum"] += 8 * role_cost["aic"] + metric["recording_cost_estimate_sum"] += 8 * role_cost["aic"] + increased = overview_module._synthetic_phase_sum_vs_none( + [more_record_pairs], + pmu["whole_window"], + ) + increased_estimate = increased["groups"]["all"]["metrics"]["pmu_total_cycles"]["recording_cost_estimate_sum"] + assert increased_estimate == expected_role_weighted + 8 * 100.0 + + +def test_partition_rows_join_only_semantically_matching_phase_profiles(evidence: _Evidence) -> None: + payload = overview_module.build_overview(evidence.swimlane_raw, evidence.pmu_dirs) + phases = {item["capture_mode"]: item for item in payload["submit_pmu_elfs"]["phase_profiles"]} + + def update_reference( + phase: dict[str, Any], + metric_name: str, + raw_phase_sum: int, + reference_sum: int, + ) -> None: + values = { + "all": (raw_phase_sum, reference_sum), + "aic": (raw_phase_sum / 3, reference_sum / 3), + "aiv": (raw_phase_sum * 2 / 3, reference_sum * 2 / 3), + } + for group_name, (group_raw, group_reference) in values.items(): + metric = phase["recording_cost_reference"]["groups"][group_name]["metrics"][metric_name] + raw_whole_sum = int(metric["raw_whole_sum"]) + estimate = group_raw - group_reference + metric.update( + { + "raw_phase_observed_sum": group_raw, + "raw_phase_observed_ratio_to_raw_whole": group_raw / raw_whole_sum, + "recording_cost_estimate_sum": estimate, + "recording_cost_estimate_share_of_raw_phase": estimate / group_raw, + "after_recording_cost_reference_sum": group_reference, + "after_recording_cost_reference_ratio_to_raw_whole": group_reference / raw_whole_sum, + } + ) + + claim = phases[overview_module.CLAIM_CAPTURE_MODE] + claim_all = claim["phase"]["groups"]["all"] + claim_all["phase_total_cycles_observed"]["sum"] = 192_000 + claim_all["phase_scalar_busy_observed"]["sum"] = 115_200 + claim_all["phase_total_share_of_pmu_total"] = 0.125 + claim_all["phase_scalar_share_of_whole_scalar"] = 0.1 + claim["phase"]["groups"]["aic"]["phase_total_cycles_observed"]["sum"] = 64_000 + claim["phase"]["groups"]["aiv"]["phase_total_cycles_observed"]["sum"] = 128_000 + claim["phase"]["groups"]["aic"]["phase_scalar_busy_observed"]["sum"] = 38_400 + claim["phase"]["groups"]["aiv"]["phase_scalar_busy_observed"]["sum"] = 76_800 + update_reference(claim, "pmu_total_cycles", 192_000, 96_000) + update_reference(claim, "scalar_busy_cycles", 115_200, 57_600) + + efdrain = phases[overview_module.EFDRAIN_CONTROL_CAPTURE_MODE] + efdrain_all = efdrain["phase"]["groups"]["all"] + efdrain_all["phase_total_cycles_observed"]["sum"] = 384_000 + efdrain_all["phase_scalar_busy_observed"]["sum"] = 230_400 + efdrain_all["phase_total_share_of_pmu_total"] = 0.25 + efdrain_all["phase_scalar_share_of_whole_scalar"] = 0.2 + efdrain["phase"]["groups"]["aic"]["phase_total_cycles_observed"]["sum"] = 128_000 + efdrain["phase"]["groups"]["aiv"]["phase_total_cycles_observed"]["sum"] = 256_000 + efdrain["phase"]["groups"]["aic"]["phase_scalar_busy_observed"]["sum"] = 76_800 + efdrain["phase"]["groups"]["aiv"]["phase_scalar_busy_observed"]["sum"] = 153_600 + update_reference(efdrain, "pmu_total_cycles", 384_000, 288_000) + update_reference(efdrain, "scalar_busy_cycles", 230_400, 172_800) + + document = overview_module.render_overview(payload) + + def row(metric: str) -> str: + match = re.search( + rf']*>.*?', + document, + flags=re.DOTALL, + ) + assert match is not None + return match.group(0) + + claim_row = row("claim") + assert f'data-pmu-profile="{overview_module.CLAIM_CAPTURE_MODE}"' in claim_row + assert 'data-pmu-mapping="same-business-boundary"' in claim_row + assert 'data-pmu-reference-share="6.250%"' in claim_row + assert 'data-pmu-raw-share="12.500%"' in claim_row + assert 'data-scalar-reference-share="5.000%"' in claim_row + assert 'data-scalar-raw-share="10.000%"' in claim_row + assert claim_row.count("6.250%") == 1 + assert claim_row.count("5.000%") == 1 + assert "raw 12.500%" in claim_row + assert "raw 10.000%" in claim_row + + efdrain_row = row("efdrain") + assert f'data-pmu-profile="{overview_module.EFDRAIN_CONTROL_CAPTURE_MODE}"' in efdrain_row + assert 'data-pmu-mapping="control-only"' in efdrain_row + assert 'data-pmu-reference-share="18.750%"' in efdrain_row + assert 'data-pmu-raw-share="25.000%"' in efdrain_row + assert 'data-scalar-reference-share="15.000%"' in efdrain_row + assert 'data-scalar-raw-share="20.000%"' in efdrain_row + assert "18.750%" in efdrain_row + assert "15.000%" in efdrain_row + assert "raw 25.000%" in efdrain_row + assert "raw 20.000%" in efdrain_row + + expected_links = { + "between_submit_residual": ( + overview_module.SUBMIT_TRANSITION_CAPTURE_MODE, + "adjacent-submit-boundary", + ), + "efdrain": (overview_module.EFDRAIN_CONTROL_CAPTURE_MODE, "control-only"), + "efdrain_control": (overview_module.EFDRAIN_CONTROL_CAPTURE_MODE, "control-only"), + "materialize": (overview_module.MATERIALIZE_CAPTURE_MODE, "same-business-boundary"), + "prepare_map": (overview_module.PREPARE_MAP_CAPTURE_MODE, "same-business-boundary"), + "claim": (overview_module.CLAIM_CAPTURE_MODE, "same-business-boundary"), + "fanin": (overview_module.FANIN_CAPTURE_MODE, "same-business-boundary"), + "register": (overview_module.REGISTER_CAPTURE_MODE, "same-business-boundary"), + "winner_build": (overview_module.WINNER_BUILD_CAPTURE_MODE, "control-only"), + "alloc_complete": (overview_module.ALLOC_COMPLETE_CAPTURE_MODE, "control-only"), + "loser_replay": (overview_module.LOSER_REPLAY_CAPTURE_MODE, "same-business-boundary"), + } + for metric, (capture_mode, mapping) in expected_links.items(): + linked_row = row(metric) + assert f'data-pmu-profile="{capture_mode}"' in linked_row + assert f'data-pmu-mapping="{mapping}"' in linked_row + + for metric in ( + "submit_union", + "submit_internal_residual", + "submit_tail_residual", + "efdrain_kernel_union", + "orchestration_replay", + "final_drain_kernel_union", + ): + unmapped_row = row(metric) + assert "data-pmu-profile" not in unmapped_row + assert unmapped_row.count('class="evidence-ratio evidence-na">—') == 2 + + +def test_zero_raw_whole_denominator_is_rendered_as_not_applicable(evidence: _Evidence) -> None: + payload = overview_module.build_overview(evidence.swimlane_raw, evidence.pmu_dirs) + phase = payload["submit_pmu_elfs"]["phase_profiles"][0] + for group_name in ("all", "aic", "aiv"): + reference_metric = phase["recording_cost_reference"]["groups"][group_name]["metrics"]["pmu_total_cycles"] + reference_metric["raw_phase_observed_sum"] = 0 + reference_metric["raw_whole_sum"] = 0 + reference_metric["raw_phase_observed_ratio_to_raw_whole"] = None + reference_metric["recording_cost_estimate_sum"] = 0 + reference_metric["recording_cost_estimate_share_of_raw_phase"] = None + reference_metric["after_recording_cost_reference_sum"] = 0 + reference_metric["after_recording_cost_reference_ratio_to_raw_whole"] = None + + document = overview_module.render_overview(payload) + + assert 'data-pmu-reference-share="N/A"' in document + assert 'data-pmu-raw-share="N/A"' in document + assert "N/A" in document + assert "0 / 0 = 0.000%" not in document + + +def test_partition_row_preserves_negative_reference_and_keeps_raw_secondary(evidence: _Evidence) -> None: + payload = overview_module.build_overview(evidence.swimlane_raw, evidence.pmu_dirs) + phase = next( + profile + for profile in payload["submit_pmu_elfs"]["phase_profiles"] + if profile["capture_mode"] == overview_module.EFDRAIN_CONTROL_CAPTURE_MODE + ) + for group_name in ("all", "aic", "aiv"): + metric = phase["recording_cost_reference"]["groups"][group_name]["metrics"]["pmu_total_cycles"] + raw_sum = float(metric["raw_phase_observed_sum"]) + raw_whole = float(metric["raw_whole_sum"]) + metric["recording_cost_estimate_sum"] = raw_sum * 2 + metric["recording_cost_estimate_share_of_raw_phase"] = 2.0 + metric["after_recording_cost_reference_sum"] = -raw_sum + metric["after_recording_cost_reference_ratio_to_raw_whole"] = -raw_sum / raw_whole + + document = overview_module.render_overview(payload) + match = re.search( + r']*>.*?', + document, + flags=re.DOTALL, + ) + assert match is not None + row = match.group(0) + assert 'data-pmu-reference-share="-6.250%"' in row + assert 'data-pmu-raw-share="6.250%"' in row + assert "-6.250%" in row + assert "raw 6.250%" in row + + +def test_output_lock_rejects_a_concurrent_publisher(tmp_path: Path) -> None: + lock_path = overview_module._acquire_output_lock(tmp_path) + try: + with pytest.raises(ValueError, match="another overview publication owns"): + overview_module._acquire_output_lock(tmp_path) + finally: + lock_path.unlink() + + +def test_publish_pair_rolls_back_first_file_when_second_publish_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + json_path = tmp_path / overview_module.DEFAULT_JSON_NAME + html_path = tmp_path / overview_module.DEFAULT_HTML_NAME + json_path.write_bytes(b"old-json") + html_path.write_bytes(b"old-html") + real_replace = os.replace + failed = False + + def fail_second_publish(source: Path | str, destination: Path | str) -> None: + nonlocal failed + source_path = Path(source) + destination_path = Path(destination) + if destination_path == html_path and source_path.name.endswith(".pending") and not failed: + failed = True + raise OSError("injected HTML publication failure") + real_replace(source, destination) + + monkeypatch.setattr(overview_module.os, "replace", fail_second_publish) + with pytest.raises(OSError, match="injected HTML publication failure"): + overview_module._publish_pair(json_path, "new-json", html_path, "new-html", overwrite=True) + + assert failed is True + assert json_path.read_bytes() == b"old-json" + assert html_path.read_bytes() == b"old-html" + assert not list(tmp_path.glob("*.pending")) + assert not list(tmp_path.glob("*.restore")) diff --git a/tests/ut/py/test_fdwic_swimlane_converter.py b/tests/ut/py/test_fdwic_swimlane_converter.py new file mode 100644 index 0000000000..d09e74c61e --- /dev/null +++ b/tests/ut/py/test_fdwic_swimlane_converter.py @@ -0,0 +1,830 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Atomic-event coverage for the shared FDWIC swimlane converter.""" + +import json + +import pytest + +from simpler_setup.tools.fdwic_swimlane_exclusive_analyzer import analyze_data, write_analysis_data +from simpler_setup.tools.fdwic_swimlane_schema import validate_and_partition_v4 +from simpler_setup.tools.swimlane_converter import generate_chrome_trace_json, read_perf_data + + +def _capture( + rows, + *, + trace_schema_version=None, + num_cores=1, + add_clock_baselines=True, + clock_dependency_applied=True, + level=None, +): + rows = list(rows) + if level is None: + level = 4 if trace_schema_version == 3 else 1 + if trace_schema_version in (3, 4) and level == 4 and add_clock_baselines: + dependency_flags = 0x3 if clock_dependency_applied else 0x1 + for core_id in range(num_cores): + block_id, lane = divmod(core_id, 3) + start = 10 + 4 * core_id + rows.extend( + [ + [core_id, block_id, lane, -1, -1, "ClockBaseline", start, start + 1, 0, 0], + [core_id, block_id, lane, -1, -1, "ClockBaseline", start + 2, start + 3, dependency_flags, 0], + ] + ) + metadata = { + "clock_freq_hz": 1_000_000_000, + "num_cores": num_cores, + "core_types": ["aic" if core_id % 3 == 0 else "aiv" for core_id in range(num_cores)], + } + if trace_schema_version is not None: + metadata["trace_schema_version"] = trace_schema_version + if trace_schema_version in (3, 4): + atomic_rows = [row for row in rows if row[5] == "Atomic"] + batch_rows = [row for row in atomic_rows if int(row[8]) & (1 << 7)] + batch_calls = sum((int(row[8]) >> 8) & 0xFFFFFF for row in batch_rows) + metadata["fdwic_summary"] = { + "records": len(rows), + "atomic_records": len(atomic_rows), + "clock_baseline_records": sum(row[5] == "ClockBaseline" for row in rows), + "atomic_calls": len(atomic_rows) - len(batch_rows) + batch_calls, + "batched_poll_calls": batch_calls, + "poll_batch_records": len(batch_rows), + "dropped_records": 0, + } + return { + "l2_swimlane_level": level, + "metadata": metadata, + "aicore_tasks": [], + "aicpu_tasks": [], + "aicpu_scheduler_phases": [], + "aicpu_orchestrator_phases": [], + "fdwic_events": rows, + } + + +def _convert(tmp_path, capture, *, pass_metadata=True): + raw_path = tmp_path / "l2_swimlane_records.json" + merged_path = tmp_path / "merged_swimlane.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + data = read_perf_data(raw_path) + if pass_metadata: + generate_chrome_trace_json( + data["tasks"], + merged_path, + fdwic_events=data.get("fdwic_events"), + trace_schema_version=data["trace_schema_version"], + clock_freq_hz=data["clock_freq_hz"], + fdwic_num_cores=data.get("num_cores", 0), + fdwic_core_types=data.get("core_types"), + ) + else: + generate_chrome_trace_json(data["tasks"], merged_path, fdwic_events=data.get("fdwic_events")) + return data, json.loads(merged_path.read_text(encoding="utf-8"))["traceEvents"] + + +def _v4_rows(num_cores=3): + rows = [] + for core_id in range(num_cores): + block_id, lane = divmod(core_id, 3) + offset = core_id * 1_000 + rows.extend( + [ + [core_id, block_id, lane, -1, -1, "OrchestrationReplay", 100 + offset, 300 + offset, 0, 0], + [core_id, block_id, lane, -1, -1, "FinalDrain", 300 + offset, 360 + offset, 0, 0], + # task 0 is deliberately a kernel (not standalone's task_id % 5 Alloc rule). + [core_id, block_id, lane, 0, 7, "Submit", 110 + offset, 190 + offset, 1, 0], + [core_id, block_id, lane, 0, -1, "EfDrain", 110 + offset, 120 + offset, 0, 0], + [core_id, block_id, lane, 99, 7, "Kernel", 112 + offset, 116 + offset, 0, 0], + [core_id, block_id, lane, 0, 7, "Claim", 120 + offset, 130 + offset, 3, 0], + # The callback builds eager arguments without adding a raw phase. + [core_id, block_id, lane, 0, -1, "Materialize", 135 + offset, 145 + offset, 0, 0], + [core_id, block_id, lane, 0, -1, "PrepareMap", 145 + offset, 155 + offset, 0, 0], + [core_id, block_id, lane, 0, 7, "Fanin", 155 + offset, 160 + offset, 0, 2], + [core_id, block_id, lane, 0, 7, "Register", 160 + offset, 170 + offset, 0, 1], + [core_id, block_id, lane, 0, 7, "WinnerBuild", 170 + offset, 185 + offset, 0, 0], + # DrainWon is real in production and remains a nested overlay. + [core_id, block_id, lane, 0, 7, "DrainWon", 172 + offset, 174 + offset, 1, 1], + [core_id, block_id, lane, 0, 7, "Kernel", 175 + offset, 180 + offset, 0, 0], + # task 1 is deliberately Alloc; kind comes from aux, never task_id modulo arithmetic. + [core_id, block_id, lane, 1, -1, "Submit", 200 + offset, 270 + offset, 0, 1], + [core_id, block_id, lane, 1, -1, "EfDrain", 200 + offset, 210 + offset, 0, 0], + [core_id, block_id, lane, 1, -1, "Claim", 210 + offset, 220 + offset, 2, 1], + # Alloc uses the same Claim-first callback gap as kernel Submit. + [core_id, block_id, lane, 1, -1, "Materialize", 225 + offset, 235 + offset, 0, 1], + [core_id, block_id, lane, 1, -1, "PrepareMap", 235 + offset, 240 + offset, 0, 1], + [core_id, block_id, lane, 1, -1, "Register", 240 + offset, 250 + offset, 0, 0], + # Tensor-data waits may execute a kernel between Submit calls in production. + [core_id, block_id, lane, 88, 8, "Kernel", 280 + offset, 290 + offset, 0, 0], + [core_id, block_id, lane, 77, 9, "Kernel", 310 + offset, 330 + offset, 0, 0], + ] + ) + return rows + + +@pytest.fixture +def v4_business_and_atomic_capture(): + """Combine production business spans with direct and batched-poll atomic overlays.""" + + rows = _v4_rows() + rows.extend( + [ + # ClaimMax consumes the FetchMax result, so its end is a return-ready boundary. + [0, 0, 0, 0, -1, "Atomic", 121, 128, 0x53, 4], + # Fanin polling keeps its exact logical call count without becoming exclusive work. + [0, 0, 0, -1, -1, "Atomic", 156, 159, (7 << 8) | 0x90, 5], + ] + ) + return _capture(rows, trace_schema_version=4, num_cores=3, level=4) + + +def _v4_single_path_rows(*, is_alloc, is_winner, claim_first=True, num_cores=3): + """Build one Submit per core for an exact path-order contract test.""" + + rows = [] + for core_id in range(num_cores): + block_id, lane = divmod(core_id, 3) + offset = core_id * 1_000 + function_id = -1 if is_alloc or not is_winner else 7 + submit_flags = 1 if is_winner else 0 + claim_flags = 0x2 | submit_flags + rows.extend( + [ + [core_id, block_id, lane, -1, -1, "OrchestrationReplay", 100 + offset, 300 + offset, 0, 0], + [core_id, block_id, lane, -1, -1, "FinalDrain", 300 + offset, 340 + offset, 0, 0], + [ + core_id, + block_id, + lane, + 0, + function_id, + "Submit", + 110 + offset, + 250 + offset, + submit_flags, + int(is_alloc), + ], + ] + ) + + cursor = 110 + + def add_phase(phase, *, func_id=-1, flags=0, aux=0): + nonlocal cursor + rows.append( + [ + core_id, + block_id, + lane, + 0, + func_id, + phase, + cursor + offset, + cursor + 10 + offset, + flags, + aux, + ] + ) + cursor += 10 + + add_phase("EfDrain") + if claim_first: + add_phase("Claim", func_id=function_id, flags=claim_flags, aux=int(is_alloc)) + cursor += 5 # Existing Claim/Materialize boundaries expose eager callback work as residual. + add_phase("Materialize", aux=int(is_alloc)) + add_phase("PrepareMap", aux=int(is_alloc)) + else: + # The live one-shot API deliberately retains its original order. + add_phase("Materialize", aux=int(is_alloc)) + add_phase("PrepareMap", aux=int(is_alloc)) + if is_alloc: + add_phase("Register") + add_phase("Claim", func_id=function_id, flags=claim_flags, aux=int(is_alloc)) + + if not is_alloc: + if is_winner: + add_phase("Fanin", func_id=function_id, aux=2) + add_phase("Register", func_id=function_id, aux=1) + add_phase("WinnerBuild" if is_winner else "LoserReplay", func_id=function_id) + elif claim_first: + add_phase("Register") + if is_winner: + add_phase("AllocComplete") + elif is_winner: + add_phase("AllocComplete") + return rows + + +def test_atomic_and_clock_stay_on_scalar_lane_and_preserve_atomic_count(tmp_path): + rows = [ + [0, 0, 0, 7, -1, "Claim", 100, 200, 0x2, 0], + [0, 0, 0, 7, -1, "Atomic", 120, 160, 0x53, 15], + [0, 0, 0, 7, -1, "Atomic", 161, 170, 0x54, 25], + [1, 0, 1, -1, -1, "ClockBaseline", 101, 102, 0x3, 0], + [1, 0, 1, 7, 0, "Kernel", 140, 180, 0, 0], + ] + _, events = _convert(tmp_path, _capture(rows, trace_schema_version=2, num_cores=2)) + + thread_names = { + event["tid"]: event["args"]["name"] + for event in events + if event.get("ph") == "M" and event.get("name") == "thread_name" + } + assert thread_names[0] == "AIC (core0)" + assert thread_names[1] == "AIV0 (core1)" + assert thread_names[4] == "AIV0·kernel (core1)" + assert not any("·atomic" in name for name in thread_names.values()) + + atomic_events = [event for event in events if event.get("args", {}).get("phase") == "atomic"] + clock = next(event for event in events if event.get("cat") == "scalar_clock") + kernel = next(event for event in events if event.get("name") == "f0#7") + assert len(atomic_events) == sum(row[5] == "Atomic" for row in rows) + assert all(event["tid"] == 0 for event in atomic_events) + assert clock["tid"] == 1 + assert clock["args"]["ticks"] == 1 + assert clock["args"]["clock_freq_hz"] == 1_000_000_000 + assert kernel["tid"] == 4 + + fetch_sub = next(event for event in atomic_events if event["args"]["op"] == "fetch_sub") + assert fetch_sub["name"] == "atomic.return_ready.won_remaining_fetch_sub.fetch_sub#7" + assert fetch_sub["cat"] == "atomic.return_ready" + assert fetch_sub["args"]["site_id"] == 25 + assert fetch_sub["args"]["op_id"] == 4 + assert fetch_sub["args"]["cycles"] == 9 + assert fetch_sub["args"]["call_count"] == 1 + assert isinstance(fetch_sub["args"]["cycles"], int) + assert fetch_sub["args"]["execution_unit"] == "scalar" + + +@pytest.mark.parametrize( + ("site_id", "site_name", "op_id", "op_name"), + [ + (1, "startup_poll", 0, "load"), + (2, "fatal_poll", 0, "load"), + (5, "fanin_flag_load", 0, "load"), + (11, "heap_frontier_load", 0, "load"), + (12, "heap_vend_load", 0, "load"), + (14, "replay_done_poll", 0, "load"), + (21, "won_any_load", 0, "load"), + (22, "won_state_load", 0, "load"), + (23, "won_lane_claim_exchange", 1, "exchange"), + (27, "won_drained_load", 0, "load"), + ], +) +def test_v3_poll_batch_preserves_exact_call_count_without_fake_atomic_latency( + tmp_path, site_id, site_name, op_id, op_name +): + poll_count = 12345 + flags = (poll_count << 8) | 0x80 | 0x10 | op_id + rows = [[0, 0, 0, -1, -1, "Atomic", 100, 900, flags, site_id]] + + data, events = _convert(tmp_path, _capture(rows, trace_schema_version=3)) + + assert data["trace_schema_version"] == 3 + batch = next(event for event in events if event.get("cat") == "atomic.poll_batch") + assert batch["name"] == f"atomic.poll_batch.{site_name}.{op_name}×{poll_count}" + assert batch["tid"] == 0 + assert batch["args"]["call_count"] == poll_count + assert batch["args"]["phase"] == "atomic_poll_batch" + assert batch["args"]["is_poll_batch"] is True + expected_semantics = "idempotent_failed_exchange_retries" if site_id == 23 else "observation_load_calls" + assert batch["args"]["batch_semantics"] == expected_semantics + assert batch["args"]["duration_semantics"] == "logical_poll_episode_envelope_not_single_atomic_latency" + assert batch["args"]["may_contain_interleaved_direct_atomics"] is True + assert batch["args"]["poll_window_cycles"] == 800 + assert batch["args"]["estimate_formula"] == "call_count * calibrated_atomic_cost" + assert "cycles" not in batch["args"] + assert "completion_boundary" not in batch["args"] + assert "return_ready_observed" not in batch["args"] + + +def test_v3_poll_batch_accepts_maximum_24_bit_call_count(tmp_path): + poll_count = 0xFFFFFF + flags = (poll_count << 8) | 0x90 + data, events = _convert( + tmp_path, + _capture([[0, 0, 0, -1, -1, "Atomic", 100, 900, flags, 1]], trace_schema_version=3), + ) + + batch = next(event for event in events if event.get("cat") == "atomic.poll_batch") + assert batch["args"]["call_count"] == poll_count + assert data["fdwic_summary"]["atomic_calls"] == poll_count + + +@pytest.mark.parametrize( + ("trace_schema_version", "flags", "site"), + [ + (2, (7 << 8) | 0x90, 5), # schema v2 reserves bit 7 + (3, 0x90, 5), # zero call_count + (3, (7 << 8) | 0x91, 5), # Exchange is not a batchable observation load + (3, (7 << 8) | 0x90, 9), # frontier scans are not explicit wait-region polling + (3, (7 << 8) | 0x90, 23), # failed lane-claim retries must retain their Exchange op + (3, (7 << 8) | 0x93, 15), # WonSlot FetchMax is protocol-changing, not a retry batch + (3, (7 << 8) | 0xB0, 5), # batch has no single-load value_zero meaning + (3, (7 << 8) | 0xD0, 5), # batch has no return-ready boundary + ], +) +def test_poll_batch_rejects_invalid_schema_or_flags(tmp_path, trace_schema_version, flags, site): + capture = _capture( + [[0, 0, 0, -1, -1, "Atomic", 100, 110, flags, site]], + trace_schema_version=trace_schema_version, + ) + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + + with pytest.raises(ValueError, match="invalid Atomic PollBatch"): + read_perf_data(raw_path) + + +@pytest.mark.parametrize( + ("flags", "site", "func_id"), + [ + (0x51, 4, -1), # ClaimMax is FetchMax, not Exchange. + (0x12, 0, -1), # StartupIncrement does not consume its FetchAdd result. + (0x42, 0, -1), # return_ready cannot exist without a consumed result. + (0x73, 4, -1), # value_zero is defined only for Load. + ((1 << 8) | 0x51, 23, -1), # retry payload is defined only for FetchMax. + (0x50, 28, -1), # Unknown site. + (0x53, 4, 0), # Atomic records never carry a func id. + ], +) +def test_v3_rejects_invalid_direct_atomic_schema(tmp_path, flags, site, func_id): + capture = _capture( + [[0, 0, 0, 7, func_id, "Atomic", 100, 110, flags, site]], + trace_schema_version=3, + ) + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + + with pytest.raises(ValueError, match="invalid direct Atomic"): + read_perf_data(raw_path) + + +@pytest.mark.parametrize( + ("task_id", "func_id", "flags", "aux"), + [ + (-1, -1, 0x2, 0), # applied requires the dependency bit. + (-1, -1, 0x4, 0), # Unknown flag bit. + (0, -1, 0, 0), + (-1, 0, 0, 0), + (-1, -1, 0, 99), + ], +) +def test_v3_rejects_invalid_clock_baseline_schema(tmp_path, task_id, func_id, flags, aux): + capture = _capture( + [[0, 0, 0, task_id, func_id, "ClockBaseline", 100, 110, flags, aux]], + trace_schema_version=3, + add_clock_baselines=False, + ) + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + + with pytest.raises(ValueError, match="invalid ClockBaseline"): + read_perf_data(raw_path) + + +@pytest.mark.parametrize(("start", "end"), [(-1, 10), (11, 10), (0, 1 << 64)]) +def test_v3_rejects_invalid_cycle_range(tmp_path, start, end): + capture = _capture( + [[0, 0, 0, 7, -1, "Atomic", start, end, 0x53, 4]], + trace_schema_version=3, + ) + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + + with pytest.raises(ValueError, match="invalid cycle range"): + read_perf_data(raw_path) + + +@pytest.mark.parametrize("flags", [-1, 1 << 32]) +def test_v3_rejects_non_uint32_flags(tmp_path, flags): + capture = _capture( + [[0, 0, 0, 7, -1, "Atomic", 100, 110, flags, 4]], + trace_schema_version=3, + ) + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + + with pytest.raises(ValueError, match="invalid uint32 flags"): + read_perf_data(raw_path) + + +@pytest.mark.parametrize( + ("clock_dependency_applied", "direct_flags"), + [ + (True, 0x53), # Real A5: consumed result has a return-ready boundary. + (False, 0x13), # A5Sim: source bracket only. + ], +) +def test_v3_direct_return_boundary_matches_per_core_clock_baseline(tmp_path, clock_dependency_applied, direct_flags): + capture = _capture( + [[0, 0, 0, 7, -1, "Atomic", 100, 110, direct_flags, 4]], + trace_schema_version=3, + clock_dependency_applied=clock_dependency_applied, + ) + data, _ = _convert(tmp_path, capture) + assert data["fdwic_summary"]["clock_baseline_records"] == 2 + + +def test_v3_rejects_direct_return_boundary_that_disagrees_with_clock_baseline(tmp_path): + capture = _capture( + [[0, 0, 0, 7, -1, "Atomic", 100, 110, 0x13, 4]], + trace_schema_version=3, + clock_dependency_applied=True, + ) + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + + with pytest.raises(ValueError, match="does not match.*ClockBaseline"): + read_perf_data(raw_path) + + +def test_v3_requires_two_clock_baselines_per_core(tmp_path): + capture = _capture([], trace_schema_version=3, num_cores=2, add_clock_baselines=False) + capture["fdwic_events"] = [[0, 0, 0, -1, -1, "ClockBaseline", 10, 11, 0, 0]] + capture["metadata"]["fdwic_summary"]["records"] = 1 + capture["metadata"]["fdwic_summary"]["clock_baseline_records"] = 1 + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + + with pytest.raises(ValueError, match="requires exactly one plain and one dependency ClockBaseline"): + read_perf_data(raw_path) + + +@pytest.mark.parametrize( + "summary_key", + [ + "records", + "atomic_records", + "clock_baseline_records", + "atomic_calls", + "batched_poll_calls", + "poll_batch_records", + "dropped_records", + ], +) +def test_v3_rejects_any_broken_weighted_summary_field(tmp_path, summary_key): + rows = [ + [0, 0, 0, -1, -1, "Atomic", 100, 200, (17 << 8) | 0x90, 1], + [0, 0, 0, 4, -1, "Atomic", 210, 220, 0x53, 4], + ] + capture = _capture(rows, trace_schema_version=3) + capture["metadata"]["fdwic_summary"][summary_key] += 1 + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + + with pytest.raises(ValueError, match=rf"fdwic_summary\.{summary_key}"): + read_perf_data(raw_path) + + +def test_schema_v3_requires_level4_and_weighted_summary(tmp_path): + capture = _capture([[0, 0, 0, -1, -1, "Atomic", 100, 110, (3 << 8) | 0x90, 14]], trace_schema_version=3) + capture["l2_swimlane_level"] = 1 + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + with pytest.raises(ValueError, match="requires l2_swimlane_level=4"): + read_perf_data(raw_path) + + capture["l2_swimlane_level"] = 4 + del capture["metadata"]["fdwic_summary"] + raw_path.write_text(json.dumps(capture), encoding="utf-8") + with pytest.raises(ValueError, match="fdwic_summary is required"): + read_perf_data(raw_path) + + +def test_v2_claim_flags_encode_all_three_states(tmp_path): + rows = [ + [0, 0, 0, 1, -1, "Claim", 100, 110, 0x0, 0], + [0, 0, 0, 2, -1, "Claim", 120, 140, 0x2, 0], + [0, 0, 0, 3, 0, "Claim", 150, 180, 0x3, 1], + ] + data, events = _convert(tmp_path, _capture(rows, trace_schema_version=2)) + + assert data["trace_schema_version"] == 2 + by_name = {event.get("name"): event for event in events} + assert by_name["claim.not_attempted#1"]["args"]["claim_attempted"] is False + assert by_name["claim.lost#2"]["args"]["claim_attempted"] is True + assert by_name["claim.won#3"]["args"]["claim_won"] is True + assert all( + by_name[name]["args"]["claim_attempted_source"] == "raw_flag" + for name in ("claim.not_attempted#1", "claim.lost#2", "claim.won#3") + ) + + +@pytest.mark.parametrize("flags", [0x1, 0x4]) +def test_v2_rejects_invalid_claim_flags(tmp_path, flags): + capture = _capture( + [[0, 0, 0, 1, -1, "Claim", 100, 110, flags, 0]], + trace_schema_version=2, + ) + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + + with pytest.raises(ValueError, match="invalid Claim flags"): + read_perf_data(raw_path) + + +def test_v1_claim_attempt_requires_contained_claim_max_evidence(tmp_path): + rows = [ + [0, 0, 0, 1, -1, "Claim", 100, 200, 0, 0], + [0, 0, 0, 1, -1, "Atomic", 120, 160, 0x53, 4], + [0, 0, 0, 2, -1, "Claim", 210, 230, 0, 0], + [0, 0, 0, 2, -1, "Atomic", 231, 240, 0x53, 4], + ] + data, events = _convert(tmp_path, _capture(rows), pass_metadata=False) + + assert data["trace_schema_version"] == 1 + by_name = {event.get("name"): event for event in events} + claim_max = next(event for event in events if event.get("args", {}).get("site_id") == 4) + assert claim_max["name"] == "atomic.return_ready.claim_max.fetch_max#1" + assert by_name["claim.lost#1"]["args"]["claim_attempted"] is True + assert by_name["claim.lost#1"]["args"]["claim_attempted_source"] == "contained_claim_max" + assert by_name["claim#2"]["args"]["claim_attempted"] is None + assert by_name["claim#2"]["args"]["claim_attempted_source"] == "unknown_v1_without_matching_claim_max" + + +@pytest.mark.parametrize( + ("is_alloc", "is_winner", "expected_sequence"), + [ + ( + False, + True, + ("EfDrain", "Claim", "Materialize", "PrepareMap", "Fanin", "Register", "WinnerBuild"), + ), + (False, False, ("EfDrain", "Claim", "Materialize", "PrepareMap", "Register", "LoserReplay")), + (True, True, ("EfDrain", "Claim", "Materialize", "PrepareMap", "Register", "AllocComplete")), + (True, False, ("EfDrain", "Claim", "Materialize", "PrepareMap", "Register")), + ], +) +def test_v4_accepts_claim_first_submit_paths(tmp_path, is_alloc, is_winner, expected_sequence): + capture = _capture( + _v4_single_path_rows(is_alloc=is_alloc, is_winner=is_winner), + trace_schema_version=4, + num_cores=3, + ) + raw_path = tmp_path / f"claim_first_{int(is_alloc)}_{int(is_winner)}.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + data = read_perf_data(raw_path) + + model = validate_and_partition_v4(data["fdwic_events"], data["num_cores"], data["core_types"]) + assert all( + tuple(child.phase for child in core.submits[0].children) == expected_sequence for core in model.cores + ) + + report = analyze_data(data, raw_path) + internal = report["residual_breakdown"]["submit_internal_residual"] + assert internal["total_cycles"] == 15 + assert internal["segments"] == [ + { + "boundary": "Claim->Materialize", + "event_count": 3, + "cycles": 15, + "aic_cycles": 5, + "aiv_cycles": 10, + } + ] + + +@pytest.mark.parametrize( + ("is_alloc", "expected_sequence"), + [ + ( + False, + ("EfDrain", "Materialize", "PrepareMap", "Claim", "Fanin", "Register", "WinnerBuild"), + ), + (True, ("EfDrain", "Materialize", "PrepareMap", "Register", "Claim", "AllocComplete")), + ], + ids=["kernel", "alloc"], +) +def test_v4_accepts_live_one_shot_submit_order(tmp_path, is_alloc, expected_sequence): + capture = _capture( + _v4_single_path_rows(is_alloc=is_alloc, is_winner=True, claim_first=False), + trace_schema_version=4, + num_cores=3, + ) + raw_path = tmp_path / f"one_shot_order_{int(is_alloc)}.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + data = read_perf_data(raw_path) + model = validate_and_partition_v4(data["fdwic_events"], data["num_cores"], data["core_types"]) + assert all( + tuple(child.phase for child in core.submits[0].children) == expected_sequence for core in model.cores + ) + + +def test_v4_production_hierarchy_generates_thin_events_and_exact_residuals(tmp_path): + data, events = _convert(tmp_path, _capture(_v4_rows(), trace_schema_version=4, num_cores=3)) + + assert data["trace_schema_version"] == 4 + duration_events = [event for event in events if event.get("ph") == "X"] + assert all("args" not in event and "cat" not in event for event in duration_events) + assert any(event["name"] == "orchestration_replay" for event in duration_events) + assert any(event["name"] == "final_drain" for event in duration_events) + assert any(event["name"] == "winner_build#0" for event in duration_events) + assert any(event["name"] == "drain_won#0" for event in duration_events) + assert not any(event["name"].startswith(("build#", "replay#", "alloc#")) for event in duration_events) + + residuals = [ + event + for event in duration_events + if event["name"] in {"submit_residual", "submit_tail_gap", "between_submit_residual"} + ] + assert sum(event["name"] == "submit_residual" for event in residuals) == 6 + assert sum(event["name"] == "submit_tail_gap" for event in residuals) == 6 + assert sum(event["name"] == "between_submit_residual" for event in residuals) == 3 + + +def test_v4_business_and_atomic_overlays_merge_without_changing_exclusive_partition( + tmp_path, v4_business_and_atomic_capture +): + data, events = _convert(tmp_path, v4_business_and_atomic_capture) + + claim = next(event for event in events if event.get("name") == "claim.won#0" and event["tid"] == 0) + direct = next(event for event in events if event.get("name") == "atomic.return_ready.claim_max.fetch_max#0") + poll = next(event for event in events if event.get("name") == "atomic.poll_batch.fanin_flag_load.load×7") + assert claim["pid"] == direct["pid"] == poll["pid"] == 0 + assert claim["tid"] == direct["tid"] == poll["tid"] == 0 + assert claim["ts"] <= direct["ts"] < direct["ts"] + direct["dur"] <= claim["ts"] + claim["dur"] + assert data["fdwic_summary"]["atomic_records"] == 2 + assert data["fdwic_summary"]["atomic_calls"] == 8 + assert data["fdwic_summary"]["poll_batch_records"] == 1 + + report = analyze_data(data, tmp_path / "l2_swimlane_records.json") + baseline_path = tmp_path / "business_only.json" + baseline_path.write_text( + json.dumps(_capture(_v4_rows(), trace_schema_version=4, num_cores=3, level=4)), + encoding="utf-8", + ) + baseline_report = analyze_data(read_perf_data(baseline_path), baseline_path) + + assert report["validation"]["status"] == "PASS" + assert report["aggregate_core_work"] == baseline_report["aggregate_core_work"] + assert report["residual_breakdown"] == baseline_report["residual_breakdown"] + assert report["overlays"]["Atomic"] == { + "event_count": 2, + "aggregate_duration_cycles": 10, + "included_in_additive_totals": False, + } + assert report["aggregate_core_work"]["closure"]["submit_partition"]["exact"] is True + + +def test_v4_residuals_reuse_the_combined_reader_time_origin(tmp_path): + capture = _capture(_v4_rows(), trace_schema_version=4, num_cores=3) + capture["aicore_tasks"] = [[0, 123, 0, 50, 60]] + raw_path = tmp_path / "l2_swimlane_records.json" + merged_path = tmp_path / "merged_swimlane.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + data = read_perf_data(raw_path) + + generate_chrome_trace_json( + [], + merged_path, + fdwic_events=data["fdwic_events"], + trace_schema_version=4, + clock_freq_hz=data["clock_freq_hz"], + fdwic_num_cores=data["num_cores"], + fdwic_core_types=data["core_types"], + ) + events = json.loads(merged_path.read_text(encoding="utf-8"))["traceEvents"] + + between = next( + event + for event in events + if event.get("name") == "between_submit_residual" and event["pid"] == 0 and event["tid"] == 0 + ) + assert between["ts"] == pytest.approx(0.14) + assert between["dur"] == pytest.approx(0.01) + + +def test_v4_exclusive_report_closes_production_parents_and_kernel_containment(tmp_path): + capture = _capture(_v4_rows(), trace_schema_version=4, num_cores=3) + raw_path = tmp_path / "l2_swimlane_records.json" + report_path = tmp_path / "swimlane_exclusive_analysis.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + data = read_perf_data(raw_path) + + report = analyze_data(data, raw_path) + written = write_analysis_data(data, raw_path, report_path) + + assert written == report_path + assert json.loads(report_path.read_text(encoding="utf-8"))["validation"]["status"] == "PASS" + assert report["capture"]["task_count_per_core"] == 2 + assert report["aggregate_core_work"]["closure"]["submit_partition"]["exact"] is True + assert report["aggregate_core_work"]["closure"]["orchestration_replay"]["exact"] is True + assert report["aggregate_core_work"]["closure"]["final_drain"]["exact"] is True + internal = report["residual_breakdown"]["submit_internal_residual"] + assert internal["total_cycles"] == 30 + assert internal["segments"] == [ + { + "boundary": "Claim->Materialize", + "event_count": 6, + "cycles": 30, + "aic_cycles": 10, + "aiv_cycles": 20, + } + ] + assert report["kernel_containment"]["inside_efdrain_events"] == 3 + assert report["kernel_containment"]["inside_winner_build_events"] == 3 + assert report["kernel_containment"]["inside_orchestration_residual_events"] == 3 + assert report["kernel_containment"]["inside_final_drain_events"] == 3 + assert report["overlays"]["DrainWon"]["event_count"] == 3 + assert report["overlays"]["DrainWon"]["included_in_additive_totals"] is False + + +def test_v4_reader_rejects_kernel_crossing_exclusive_boundary(tmp_path): + rows = _v4_rows() + kernel = next(row for row in rows if row[0] == 0 and row[3] == 99 and row[5] == "Kernel") + kernel[6:8] = [115, 125] + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text( + json.dumps(_capture(rows, trace_schema_version=4, num_cores=3)), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="Kernel crosses exclusive child"): + read_perf_data(raw_path) + + +def test_v4_reader_rejects_kernel_inside_submit_residual(tmp_path): + rows = _v4_rows() + kernel = next(row for row in rows if row[0] == 0 and row[3] == 99 and row[5] == "Kernel") + kernel[6:8] = [186, 189] + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text( + json.dumps(_capture(rows, trace_schema_version=4, num_cores=3)), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="Kernel is inside Submit residual"): + read_perf_data(raw_path) + + +def test_v4_supports_all_existing_fdwic_collection_levels(tmp_path): + for level in (1, 2, 3): + capture = _capture(_v4_rows(), trace_schema_version=4, num_cores=3, level=level) + raw_path = tmp_path / f"l2_swimlane_records_level{level}.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + assert read_perf_data(raw_path)["l2_swimlane_level"] == level + + +def test_v4_uses_dynamic_two_block_core_topology(tmp_path): + capture = _capture(_v4_rows(num_cores=6), trace_schema_version=4, num_cores=6) + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + + report = analyze_data(read_perf_data(raw_path), raw_path) + + assert report["capture"]["core_count"] == 6 + assert report["per_role_core_statistics"]["aic"]["core_count"] == 2 + assert report["per_role_core_statistics"]["aiv"]["core_count"] == 4 + + +@pytest.mark.parametrize("legacy_phase", ["Alloc", "Build", "Replay"]) +def test_v4_rejects_legacy_overlapping_lap_phases(tmp_path, legacy_phase): + rows = _v4_rows() + rows.append([0, 0, 0, 0, 7, legacy_phase, 170, 185, 0, 0]) + capture = _capture(rows, trace_schema_version=4, num_cores=3) + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + + with pytest.raises(ValueError, match="forbids legacy lap phase"): + read_perf_data(raw_path) + + +def test_v4_rejects_missing_parent_wrong_tail_and_overlapping_children(tmp_path): + mutations = [] + + missing_parent = [row for row in _v4_rows() if not (row[0] == 1 and row[5] == "FinalDrain")] + mutations.append((missing_parent, "exactly one OrchestrationReplay and FinalDrain")) + + wrong_tail = _v4_rows() + next(row for row in wrong_tail if row[0] == 0 and row[5] == "WinnerBuild")[5] = "AllocComplete" + mutations.append((wrong_tail, "invalid exclusive sequence")) + + overlapping = _v4_rows() + next(row for row in overlapping if row[0] == 0 and row[5] == "Materialize")[6] = 129 + mutations.append((overlapping, "overlapping exclusive children")) + + for index, (rows, message) in enumerate(mutations): + capture = _capture(rows, trace_schema_version=4, num_cores=3) + raw_path = tmp_path / f"invalid_{index}.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + with pytest.raises(ValueError, match=message): + read_perf_data(raw_path) + + +def test_v4_requires_zero_drop_producer_summary_even_without_atomic_rows(tmp_path): + capture = _capture(_v4_rows(), trace_schema_version=4, num_cores=3) + capture["metadata"]["fdwic_summary"]["dropped_records"] = 1 + raw_path = tmp_path / "l2_swimlane_records.json" + raw_path.write_text(json.dumps(capture), encoding="utf-8") + + with pytest.raises(ValueError, match=r"fdwic_summary\.dropped_records"): + read_perf_data(raw_path) diff --git a/tests/ut/py/test_kernel_compiler.py b/tests/ut/py/test_kernel_compiler.py new file mode 100644 index 0000000000..adecdc792d --- /dev/null +++ b/tests/ut/py/test_kernel_compiler.py @@ -0,0 +1,111 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +from types import SimpleNamespace + +import pytest + +from simpler_setup.kernel_compiler import KernelCompiler + + +def _compiler_with_captured_command(monkeypatch): + compiler = object.__new__(KernelCompiler) + compiler._sanitizers = "" + captured: dict[str, list[str]] = {} + + def capture(cmd, _output_path, _label, **_kwargs): + captured["cmd"] = cmd + return b"orchestration" + + monkeypatch.setattr(compiler, "_compile_to_bytes", capture) + toolchain = SimpleNamespace( + cxx_path="test-g++", + is_host=True, + get_compile_flags=lambda: ["-shared"], + ) + return compiler, toolchain, captured + + +def test_orchestration_shared_lib_forwards_each_compile_definition(monkeypatch, tmp_path): + source = tmp_path / "orch.cpp" + source.write_text("void Orchestration() {}\n") + compiler, toolchain, captured = _compiler_with_captured_command(monkeypatch) + + result = compiler._compile_orchestration_shared_lib( + str(source), + toolchain, + compile_definitions=["PTO_FDWIC_SHARED_MAP=1", "PTO_FDWIC_TRACE_ENABLED=0"], + ) + + assert result == b"orchestration" + assert "-DPTO_FDWIC_SHARED_MAP=1" in captured["cmd"] + assert "-DPTO_FDWIC_TRACE_ENABLED=0" in captured["cmd"] + + +def test_orchestration_shared_lib_omits_definitions_by_default(monkeypatch, tmp_path): + source = tmp_path / "orch.cpp" + source.write_text("void Orchestration() {}\n") + compiler, toolchain, captured = _compiler_with_captured_command(monkeypatch) + + compiler._compile_orchestration_shared_lib(str(source), toolchain) + + assert not any(argument.startswith("-D") for argument in captured["cmd"]) + + +@pytest.mark.parametrize( + "definition", + [ + "", + " ", + "\0", + "1NAME=1", + "=1", + "-DNAME=1", + ], +) +def test_orchestration_shared_lib_rejects_invalid_compile_definition(monkeypatch, tmp_path, definition): + source = tmp_path / "orch.cpp" + source.write_text("void Orchestration() {}\n") + compiler, toolchain, _captured = _compiler_with_captured_command(monkeypatch) + + with pytest.raises(ValueError, match="compile definition"): + compiler._compile_orchestration_shared_lib( + str(source), + toolchain, + compile_definitions=[definition], + ) + + +def test_compile_orchestration_forwards_compile_definitions(monkeypatch): + compiler = object.__new__(KernelCompiler) + compiler.platform = "a5sim" + compiler.host_gxx = object() + monkeypatch.setattr(compiler, "get_orchestration_include_dirs", lambda _runtime: []) + monkeypatch.setattr(compiler, "_get_orchestration_config", lambda _runtime: ([], [])) + monkeypatch.setattr( + compiler, + "_get_toolchain", + lambda _mapping: __import__("simpler_setup.toolchain", fromlist=["ToolchainType"]).ToolchainType.HOST_GXX, + ) + captured = {} + + def capture(*_args, **kwargs): + captured.update(kwargs) + return b"orchestration" + + monkeypatch.setattr(compiler, "_compile_orchestration_shared_lib", capture) + + assert ( + compiler.compile_orchestration( + "fully_distributed_within_core", + "unused.cpp", + compile_definitions=["PTO_FDWIC_SHARED_MAP=1"], + ) + == b"orchestration" + ) + assert captured["compile_definitions"] == ["PTO_FDWIC_SHARED_MAP=1"] diff --git a/tests/ut/py/test_runtime_builder.py b/tests/ut/py/test_runtime_builder.py index 96f41096d7..6feca5edd4 100644 --- a/tests/ut/py/test_runtime_builder.py +++ b/tests/ut/py/test_runtime_builder.py @@ -152,6 +152,44 @@ def test_empty_registry_shows_none(self, MockCompiler, tmp_path, monkeypatch, de builder.get_binaries("anything", build=True) +class TestFdwicTensorMapMode: + """FDWIC TensorMap mode is an explicit, narrowly scoped build identity.""" + + @patch("simpler_setup.runtime_builder.RuntimeCompiler") + def test_defaults_to_private_and_reads_environment(self, MockCompiler, monkeypatch): + from simpler_setup.runtime_builder import RuntimeBuilder # noqa: PLC0415 + + monkeypatch.delenv("PTO_FDWIC_TENSORMAP_MODE", raising=False) + assert RuntimeBuilder(platform="a5").fdwic_tensormap_mode == "private" + + monkeypatch.setenv("PTO_FDWIC_TENSORMAP_MODE", "shared") + assert RuntimeBuilder(platform="a5sim").fdwic_tensormap_mode == "shared" + assert RuntimeBuilder(platform="a5", fdwic_tensormap_mode="private").fdwic_tensormap_mode == "private" + + @patch("simpler_setup.runtime_builder.RuntimeCompiler") + @pytest.mark.parametrize("mode", ["", "unknown", "private/shared"]) + def test_rejects_invalid_mode(self, MockCompiler, mode): + from simpler_setup.runtime_builder import RuntimeBuilder # noqa: PLC0415 + + with pytest.raises(ValueError, match="Invalid FDWIC TensorMap mode"): + RuntimeBuilder(platform="a5", fdwic_tensormap_mode=mode) + + @patch("simpler_setup.runtime_builder.RuntimeCompiler") + def test_rejects_shared_on_other_architecture(self, MockCompiler): + from simpler_setup.runtime_builder import RuntimeBuilder # noqa: PLC0415 + + with pytest.raises(ValueError, match="only valid for the a5/a5sim"): + RuntimeBuilder(platform="a2a3sim", fdwic_tensormap_mode="shared") + + @patch("simpler_setup.runtime_builder.RuntimeCompiler") + def test_rejects_shared_for_other_a5_runtime(self, MockCompiler): + from simpler_setup.runtime_builder import RuntimeBuilder # noqa: PLC0415 + + builder = RuntimeBuilder(platform="a5", fdwic_tensormap_mode="shared") + with pytest.raises(ValueError, match="runtime='host_build_graph'"): + builder.get_binaries("host_build_graph") + + class TestRuntimeBuilderPtoIsaValidation: """Test PTO-ISA compatibility validation is scoped to affected runtimes.""" @@ -216,9 +254,9 @@ def _patch_runtime_root(self, monkeypatch, tmp_path): monkeypatch.setattr(rb_module, "PROJECT_ROOT", tmp_path) - def _make_runtime(self, tmp_path, test_arch): + def _make_runtime(self, tmp_path, test_arch, name="test_rt"): """Create a fake runtime with a valid build_config.py.""" - rt_dir = tmp_path / "src" / test_arch / "runtime" / "test_rt" + rt_dir = tmp_path / "src" / test_arch / "runtime" / name for sub in ["aicore", "aicpu", "host", "runtime"]: (rt_dir / sub).mkdir(parents=True) @@ -241,6 +279,11 @@ def _make_runtime(self, tmp_path, test_arch): (rt_dir / "build_config.py").write_text(config_content) return rt_dir + @staticmethod + def _set_build_roots(monkeypatch, tmp_path, RuntimeBuilder): + monkeypatch.setattr(RuntimeBuilder, "_CACHE_DIR", tmp_path / "build" / "cache") + monkeypatch.setattr(RuntimeBuilder, "_LIB_DIR", tmp_path / "build" / "lib") + @patch("simpler_setup.runtime_builder.RuntimeCompiler") def test_returns_runtime_binaries(self, MockCompiler, tmp_path, default_test_platform, test_arch): """get_binaries(build=True) returns RuntimeBinaries with three paths.""" @@ -311,6 +354,202 @@ def test_propagates_compiler_error(self, MockCompiler, tmp_path, default_test_pl with pytest.raises(RuntimeError, match="cmake failed"): builder.get_binaries("test_rt", build=True) + @patch("simpler_setup.runtime_builder.RuntimeCompiler") + def test_fdwic_mode_macro_reaches_all_three_targets(self, MockCompiler, tmp_path, monkeypatch): + from simpler_setup.runtime_builder import RuntimeBuilder # noqa: PLC0415 + + runtime = "fully_distributed_within_core" + self._make_runtime(tmp_path, "a5", runtime) + self._set_build_roots(monkeypatch, tmp_path, RuntimeBuilder) + mock_instance = MockCompiler.get_instance.return_value + mock_instance.compile.side_effect = lambda target, *a, **kw: Path(kw["output_dir"]) / f"lib{target}.so" + mock_instance.compile_simpler_log.return_value = tmp_path / "build" / "lib" / "libsimpler_log.so" + + builder = RuntimeBuilder(platform="a5", fdwic_tensormap_mode="shared") + builder.get_binaries(runtime, build=True) + + runtime_calls = [ + call for call in mock_instance.compile.call_args_list if call.args[0] in {"host", "aicpu", "aicore"} + ] + assert len(runtime_calls) == 3 + assert {tuple(call.kwargs["compile_definitions"]) for call in runtime_calls} == { + ("PTO_FDWIC_SHARED_MAP=1", "PTO_FDWIC_TENSORMAP_RING_CAP=128") + } + + @patch("simpler_setup.runtime_builder.RuntimeCompiler") + def test_private_mode_does_not_change_other_runtime_artifacts(self, MockCompiler, tmp_path, monkeypatch): + from simpler_setup.runtime_builder import RuntimeBuilder # noqa: PLC0415 + + self._make_runtime(tmp_path, "a5", "test_rt") + self._set_build_roots(monkeypatch, tmp_path, RuntimeBuilder) + mock_instance = MockCompiler.get_instance.return_value + mock_instance.compile.side_effect = lambda target, *a, **kw: Path(kw["output_dir"]) / f"lib{target}.so" + mock_instance.compile_simpler_log.return_value = tmp_path / "build" / "lib" / "libsimpler_log.so" + + RuntimeBuilder(platform="a5", fdwic_tensormap_mode="private").get_binaries("test_rt", build=True) + + runtime_calls = [ + call for call in mock_instance.compile.call_args_list if call.args[0] in {"host", "aicpu", "aicore"} + ] + assert {Path(call.kwargs["output_dir"]) for call in runtime_calls} == { + tmp_path / "build" / "lib" / "a5" / "onboard" / "test_rt" + } + assert {Path(call.kwargs["build_dir"]) for call in runtime_calls} == { + tmp_path / "build" / "cache" / "a5" / "onboard" / "test_rt" + } + assert all("compile_definitions" not in call.kwargs for call in runtime_calls) + + @patch("simpler_setup.runtime_builder.RuntimeCompiler") + def test_fdwic_private_and_shared_use_separate_output_and_cache_paths(self, MockCompiler, tmp_path, monkeypatch): + from simpler_setup.runtime_builder import RuntimeBuilder # noqa: PLC0415 + + runtime = "fully_distributed_within_core" + self._make_runtime(tmp_path, "a5", runtime) + self._set_build_roots(monkeypatch, tmp_path, RuntimeBuilder) + mock_instance = MockCompiler.get_instance.return_value + mock_instance.compile.side_effect = lambda target, *a, **kw: Path(kw["output_dir"]) / f"lib{target}.so" + mock_instance.compile_simpler_log.return_value = tmp_path / "build" / "lib" / "libsimpler_log.so" + + for mode in ("private", "shared"): + RuntimeBuilder(platform="a5", fdwic_tensormap_mode=mode).get_binaries(runtime, build=True) + + private_calls = [ + call for call in mock_instance.compile.call_args_list if Path(call.kwargs["output_dir"]).name == "private" + ] + shared_calls = [ + call for call in mock_instance.compile.call_args_list if Path(call.kwargs["output_dir"]).name == "shared" + ] + assert len(private_calls) == len(shared_calls) == 3 + assert {Path(call.kwargs["build_dir"]) for call in private_calls} == { + tmp_path / "build" / "cache" / "a5" / "onboard" / runtime / "private" + } + assert {Path(call.kwargs["build_dir"]) for call in shared_calls} == { + tmp_path / "build" / "cache" / "a5" / "onboard" / runtime / "shared" + } + assert {tuple(call.kwargs["compile_definitions"]) for call in private_calls} == { + ("PTO_FDWIC_SHARED_MAP=0", "PTO_FDWIC_TENSORMAP_RING_CAP=128") + } + assert {tuple(call.kwargs["compile_definitions"]) for call in shared_calls} == { + ("PTO_FDWIC_SHARED_MAP=1", "PTO_FDWIC_TENSORMAP_RING_CAP=128") + } + + @patch("simpler_setup.runtime_builder.RuntimeCompiler") + def test_fdwic_baseline_source_state_tracks_contents_and_mode(self, MockCompiler, tmp_path, monkeypatch): + import simpler_setup.runtime_builder as rb_module # noqa: PLC0415 + from simpler_setup.runtime_builder import RuntimeBuilder # noqa: PLC0415 + + runtime = "fully_distributed_within_core" + rt_dir = self._make_runtime(tmp_path, "a5", runtime) + source = rt_dir / "runtime" / "state.h" + source.write_text("inline constexpr int kState = 1;\n") + self._set_build_roots(monkeypatch, tmp_path, RuntimeBuilder) + mock_instance = MockCompiler.get_instance.return_value + mock_instance.compile.side_effect = lambda target, *a, **kw: Path(kw["output_dir"]) / f"lib{target}.so" + mock_instance.compile_simpler_log.return_value = tmp_path / "build" / "lib" / "libsimpler_log.so" + observed_states = [] + monkeypatch.setattr( + rb_module, + "_invalidate_cache_if_stale", + lambda _cache_dir, current_state: observed_states.append(current_state), + ) + + RuntimeBuilder(platform="a5", fdwic_tensormap_mode="private").get_binaries(runtime, build=True) + first_private_states = set(observed_states) + observed_states.clear() + source.write_text("inline constexpr int kState = 2;\n") + RuntimeBuilder(platform="a5", fdwic_tensormap_mode="private").get_binaries(runtime, build=True) + second_private_states = set(observed_states) + observed_states.clear() + RuntimeBuilder(platform="a5", fdwic_tensormap_mode="shared").get_binaries(runtime, build=True) + shared_states = set(observed_states) + + assert all(state.startswith("source-v2:") for state in first_private_states) + assert first_private_states != second_private_states + assert second_private_states != shared_states + + +class TestFdwicAicoreExtraBuild: + """Per-callable AICore images share the baseline FDWIC build identity.""" + + @staticmethod + def _make_runtime(tmp_path): + runtime = "fully_distributed_within_core" + rt_dir = tmp_path / "src" / "a5" / "runtime" / runtime + for subdir in ("aicore", "runtime"): + (rt_dir / subdir).mkdir(parents=True) + (rt_dir / "build_config.py").write_text( + textwrap.dedent("""\ + BUILD_CONFIG = { + "aicore": { + "include_dirs": ["aicore", "runtime"], + "source_dirs": ["runtime"] + } + } + """) + ) + return runtime + + @patch("simpler_setup.kernel_compiler.KernelCompiler") + @patch("simpler_setup.runtime_builder.RuntimeCompiler") + def test_mode_path_and_effective_definitions(self, MockCompiler, MockKernelCompiler, tmp_path, monkeypatch): + import simpler_setup.runtime_builder as rb_module # noqa: PLC0415 + from simpler_setup.runtime_builder import RuntimeBuilder # noqa: PLC0415 + + monkeypatch.setattr(rb_module, "PROJECT_ROOT", tmp_path) + monkeypatch.setattr(RuntimeBuilder, "_CACHE_DIR", tmp_path / "build" / "cache") + monkeypatch.setattr(RuntimeBuilder, "_LIB_DIR", tmp_path / "build" / "lib") + runtime = self._make_runtime(tmp_path) + MockKernelCompiler.return_value.get_incore_include_dirs.return_value = [] + mock_instance = MockCompiler.get_instance.return_value + mock_instance.compile.side_effect = lambda target, *a, **kw: Path(kw["output_dir"]) / "aicore.o" + extra_source = tmp_path / "callable.cpp" + extra_source.write_text("void callable() {}\n") + + result = RuntimeBuilder(platform="a5", fdwic_tensormap_mode="shared").build_aicore_with_extra_sources( + runtime, + [extra_source], + "callable-key", + compile_definitions=["PTO_FDWIC_PERF_CLOCK=1", "PTO_FDWIC_TRACE_ENABLED=0"], + ) + + call = mock_instance.compile.call_args + expected_root = tmp_path / "build" / "lib" / "a5" / "onboard" / runtime / "shared" + assert result == expected_root / "aicore-extra" / "callable-key" / "aicore.o" + assert Path(call.kwargs["build_dir"]) == ( + tmp_path / "build" / "cache" / "a5" / "onboard" / runtime / "shared" / "aicore-extra" / "callable-key" + ) + assert call.kwargs["compile_definitions"] == [ + "PTO_FDWIC_SHARED_MAP=1", + "PTO_FDWIC_TENSORMAP_RING_CAP=128", + "PTO_FDWIC_PERF_CLOCK=1", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + + @patch("simpler_setup.runtime_builder.RuntimeCompiler") + def test_rejects_conflicting_mode_definition(self, MockCompiler, tmp_path, monkeypatch): + import simpler_setup.runtime_builder as rb_module # noqa: PLC0415 + from simpler_setup.runtime_builder import RuntimeBuilder # noqa: PLC0415 + + monkeypatch.setattr(rb_module, "PROJECT_ROOT", tmp_path) + runtime = self._make_runtime(tmp_path) + builder = RuntimeBuilder(platform="a5", fdwic_tensormap_mode="shared") + + with pytest.raises(ValueError, match="Conflicting compile definitions for PTO_FDWIC_SHARED_MAP"): + builder.build_aicore_with_extra_sources( + runtime, + [], + "conflict", + compile_definitions=["PTO_FDWIC_SHARED_MAP=0"], + ) + + with pytest.raises(ValueError, match="Conflicting compile definitions for PTO_FDWIC_TENSORMAP_RING_CAP"): + builder.build_aicore_with_extra_sources( + runtime, + [], + "cap-conflict", + compile_definitions=["PTO_FDWIC_TENSORMAP_RING_CAP=64"], + ) + # --- _invalidate_cache_if_stale unit tests --- @@ -360,6 +599,40 @@ def test_clears_when_commit_unavailable(self, tmp_path): assert cache_dir.is_dir() +# --- Per-callable compile-definition tests --- + + +class TestBuildTargetCompileDefinitions: + """Per-callable CCEC profiles must reach CMake as one stable list.""" + + class _Toolchain: + is_host = False + + @staticmethod + def get_cmake_args(): + return [] + + def test_forwards_compile_definitions_to_cmake(self, tmp_path): + from simpler_setup.runtime_compiler import BuildTarget # noqa: PLC0415 + + target = BuildTarget(self._Toolchain(), str(tmp_path), "aicore_kernel.o") + args = target.gen_cmake_args( + [str(tmp_path / "include")], + [str(tmp_path / "src")], + compile_definitions=["PTO_FDWIC_PERF_CLOCK=1", "PTO_FDWIC_TRACE_ENABLED=0"], + ) + + assert ("-DCUSTOM_COMPILE_DEFINITIONS=PTO_FDWIC_PERF_CLOCK=1;PTO_FDWIC_TRACE_ENABLED=0") in args + + def test_omits_empty_compile_definitions(self, tmp_path): + from simpler_setup.runtime_compiler import BuildTarget # noqa: PLC0415 + + target = BuildTarget(self._Toolchain(), str(tmp_path), "aicore_kernel.o") + args = target.gen_cmake_args([str(tmp_path / "include")], [str(tmp_path / "src")]) + + assert not any(arg.startswith("-DCUSTOM_COMPILE_DEFINITIONS=") for arg in args) + + # --- Full integration tests (real compilation) --- diff --git a/tests/ut/py/test_scene_test_cache.py b/tests/ut/py/test_scene_test_cache.py index 4a97c410af..c6cd5d8d5d 100644 --- a/tests/ut/py/test_scene_test_cache.py +++ b/tests/ut/py/test_scene_test_cache.py @@ -20,12 +20,44 @@ from __future__ import annotations +import importlib +import json +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest from _task_interface import ArgDirection, ChipCallable # pyright: ignore[reportMissingImports] +from conftest import _configure_fdwic_profile, _configure_fdwic_tensormap + # ``simpler_setup/__init__.py`` re-exports the ``scene_test`` *decorator*, # which shadows the submodule attribute when accessed via ``simpler_setup``. # Importing the names directly from the submodule avoids that ambiguity. -from simpler_setup.scene_test import _compile_cache, clear_compile_cache +from simpler_setup.scene_test import ( + _aicore_override_cache, + _assert_fdwic_perf_clock_elf, + _assert_fdwic_submit_pmu_elf, + _assert_fdwic_submit_pmu_host_elf, + _assert_fdwic_swimlane_elf, + _compile_cache, + _convert_case_swimlane, + _fdwic_build_identity_cache, + _fdwic_compile_definitions, + _fdwic_profile, + _fdwic_tensormap_compile_definitions, + _fdwic_tensormap_mode, + _profiled_cache_key, + _render_case_fdwic_submit_pmu, + _run_swimlane_converter, + _validate_case_fdwic_perf_clock, + _validate_fdwic_tensormap_test_classes, + clear_compile_cache, + maybe_build_aicore_override, + run_class_cases, +) + +_scene_test_module = importlib.import_module("simpler_setup.scene_test") def _build_chip_callable(tag: str) -> ChipCallable: @@ -48,10 +80,1150 @@ def test_clear_compile_cache_drops_cached_chip_callables(): cleanup doesn't know about), this assertion fails. """ _compile_cache.clear() + _aicore_override_cache.clear() + _fdwic_build_identity_cache.clear() for i in range(3): _compile_cache[("t", "plat", f"rt{i}")] = _build_chip_callable(f"n{i}") + _aicore_override_cache[("t", "plat", "rt0", "private", "none")] = Path("/tmp/fake-aicore.o") + _fdwic_build_identity_cache[("t", "plat", "rt0", "private", "submit-pmu-none")] = object() assert len(_compile_cache) == 3 + assert len(_aicore_override_cache) == 1 + assert len(_fdwic_build_identity_cache) == 1 clear_compile_cache() assert _compile_cache == {} + assert _aicore_override_cache == {} + assert _fdwic_build_identity_cache == {} + + +def test_fdwic_profile_partitions_compile_cache(monkeypatch): + """TensorMap mode and every evidence profile must partition AICore images.""" + base = ("Case", "a5", "fully_distributed_within_core") + + monkeypatch.delenv("PTO_FDWIC_TENSORMAP_MODE", raising=False) + monkeypatch.delenv("PTO_FDWIC_PROFILE", raising=False) + assert _fdwic_profile() == "none" + assert _profiled_cache_key(base) == (*base, "private", "none") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "perf-clock") + assert _fdwic_profile() == "perf-clock" + assert _profiled_cache_key(base) == (*base, "private", "perf-clock") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "perf-clock-kernel") + assert _fdwic_profile() == "perf-clock-kernel" + assert _profiled_cache_key(base) == (*base, "private", "perf-clock-kernel") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-none") + assert _fdwic_profile() == "submit-pmu-none" + assert _profiled_cache_key(base) == (*base, "private", "submit-pmu-none") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-arg-build") + assert _fdwic_profile() == "submit-pmu-arg-build" + assert _profiled_cache_key(base) == (*base, "private", "submit-pmu-arg-build") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-empty-bracket") + assert _fdwic_profile() == "submit-pmu-empty-bracket" + assert _profiled_cache_key(base) == (*base, "private", "submit-pmu-empty-bracket") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-materialize") + assert _fdwic_profile() == "submit-pmu-materialize" + assert _profiled_cache_key(base) == (*base, "private", "submit-pmu-materialize") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-claim") + assert _fdwic_profile() == "submit-pmu-claim" + assert _profiled_cache_key(base) == (*base, "private", "submit-pmu-claim") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-register") + assert _fdwic_profile() == "submit-pmu-register" + assert _profiled_cache_key(base) == (*base, "private", "submit-pmu-register") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-submit-transition") + assert _fdwic_profile() == "submit-pmu-submit-transition" + assert _profiled_cache_key(base) == (*base, "private", "submit-pmu-submit-transition") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-efdrain-control") + assert _fdwic_profile() == "submit-pmu-efdrain-control" + assert _profiled_cache_key(base) == (*base, "private", "submit-pmu-efdrain-control") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-prepare-map") + assert _fdwic_profile() == "submit-pmu-prepare-map" + assert _profiled_cache_key(base) == (*base, "private", "submit-pmu-prepare-map") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-fanin") + assert _fdwic_profile() == "submit-pmu-fanin" + assert _profiled_cache_key(base) == (*base, "private", "submit-pmu-fanin") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-winner-build-control") + assert _fdwic_profile() == "submit-pmu-winner-build-control" + assert _profiled_cache_key(base) == (*base, "private", "submit-pmu-winner-build-control") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-alloc-complete-control") + assert _fdwic_profile() == "submit-pmu-alloc-complete-control" + assert _profiled_cache_key(base) == (*base, "private", "submit-pmu-alloc-complete-control") + + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-loser-replay") + assert _fdwic_profile() == "submit-pmu-loser-replay" + assert _profiled_cache_key(base) == (*base, "private", "submit-pmu-loser-replay") + + monkeypatch.setenv("PTO_FDWIC_TENSORMAP_MODE", "shared") + assert _profiled_cache_key(base) == (*base, "shared", "submit-pmu-loser-replay") + + +def test_fdwic_tensormap_mode_and_compile_definition_contract(monkeypatch): + monkeypatch.delenv("PTO_FDWIC_TENSORMAP_MODE", raising=False) + assert _fdwic_tensormap_mode() == "private" + assert _fdwic_tensormap_compile_definitions("a5", "fully_distributed_within_core") == [ + "PTO_FDWIC_SHARED_MAP=0", + "PTO_FDWIC_TENSORMAP_RING_CAP=128", + ] + + monkeypatch.setenv("PTO_FDWIC_TENSORMAP_MODE", "shared") + assert _fdwic_tensormap_mode() == "shared" + assert _fdwic_tensormap_compile_definitions("a5sim", "fully_distributed_within_core") == [ + "PTO_FDWIC_SHARED_MAP=1", + "PTO_FDWIC_TENSORMAP_RING_CAP=128", + ] + with pytest.raises(ValueError, match="only supported"): + _fdwic_tensormap_compile_definitions("a5", "host_build_graph") + + monkeypatch.setenv("PTO_FDWIC_TENSORMAP_MODE", "typo") + with pytest.raises(ValueError, match="Unsupported PTO_FDWIC_TENSORMAP_MODE"): + _fdwic_tensormap_mode() + + +def test_fdwic_evidence_profiles_have_isolated_compile_definitions(): + assert _fdwic_compile_definitions("none") is None + assert _fdwic_compile_definitions("perf-clock") == [ + "PTO_FDWIC_PERF_CLOCK=1", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("perf-clock-kernel") == [ + "PTO_FDWIC_PERF_CLOCK=1", + "PTO_FDWIC_PERF_CLOCK_KERNEL=1", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("submit-pmu-none") == [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("submit-pmu-arg-build") == [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=1", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("submit-pmu-empty-bracket") == [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=2", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("submit-pmu-materialize") == [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=3", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("submit-pmu-claim") == [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=4", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("submit-pmu-register") == [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=5", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("submit-pmu-submit-transition") == [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=6", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("submit-pmu-efdrain-control") == [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=7", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("submit-pmu-prepare-map") == [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=8", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("submit-pmu-fanin") == [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=9", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("submit-pmu-winner-build-control") == [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=10", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("submit-pmu-alloc-complete-control") == [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=11", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + assert _fdwic_compile_definitions("submit-pmu-loser-replay") == [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=12", + "PTO_FDWIC_TRACE_ENABLED=0", + ] + + +@pytest.mark.parametrize( + ("profile", "expected_compile_definitions"), + [ + ("submit-pmu-none", ["PTO_FDWIC_SUBMIT_PMU=1", "PTO_FDWIC_TRACE_ENABLED=0"]), + ( + "submit-pmu-efdrain-control", + [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=7", + "PTO_FDWIC_TRACE_ENABLED=0", + ], + ), + ( + "submit-pmu-prepare-map", + [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=8", + "PTO_FDWIC_TRACE_ENABLED=0", + ], + ), + ( + "submit-pmu-fanin", + [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=9", + "PTO_FDWIC_TRACE_ENABLED=0", + ], + ), + ( + "submit-pmu-winner-build-control", + [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=10", + "PTO_FDWIC_TRACE_ENABLED=0", + ], + ), + ( + "submit-pmu-alloc-complete-control", + [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=11", + "PTO_FDWIC_TRACE_ENABLED=0", + ], + ), + ( + "submit-pmu-loser-replay", + [ + "PTO_FDWIC_SUBMIT_PMU=1", + "PTO_FDWIC_SUBMIT_PMU_PHASE_ID=12", + "PTO_FDWIC_TRACE_ENABLED=0", + ], + ), + ], +) +def test_submit_pmu_override_registers_build_identity_after_elf_gate( + monkeypatch, tmp_path, profile, expected_compile_definitions +): + """The profiled cache key must own one identity frozen from the built files.""" + runtime = "fully_distributed_within_core" + profiled_key = ("QualifiedCase", "a5", runtime, "private", profile) + orch = tmp_path / "orch.cpp" + orch.write_text("// orchestration\n") + host = tmp_path / "libhost_runtime.so" + host.write_bytes(b"host") + aicpu = tmp_path / "libaicpu_kernel.so" + aicpu.write_bytes(b"aicpu") + order = [] + captured = {} + identity = object() + + class FakeRuntimeBuilder: + _CACHE_DIR = tmp_path / "build" / "cache" + _LIB_DIR = tmp_path / "build" / "lib" + + def __init__(self, platform, fdwic_tensormap_mode=None): + assert platform == "a5" + assert fdwic_tensormap_mode == "private" + + def build_aicore_with_extra_sources( + self, + name, + extra_sources, + cache_key, + pto_isa_root=None, + compile_definitions=None, + ): + assert name == runtime + assert extra_sources == [orch.resolve()] + order.append("build") + binary = ( + self._LIB_DIR / "a5" / "onboard" / runtime / "private" / "aicore-extra" / cache_key / "aicore_kernel.o" + ) + binary.parent.mkdir(parents=True) + binary.write_bytes(b"aicore") + return binary + + @staticmethod + def effective_compile_definitions(name, compile_definitions=None): + assert name == runtime + return [ + "PTO_FDWIC_SHARED_MAP=0", + "PTO_FDWIC_TENSORMAP_RING_CAP=128", + *(compile_definitions or []), + ] + + def get_binaries(self, name): + assert name == runtime + return SimpleNamespace(host_path=host, aicpu_path=aicpu) + + def fake_elf_gate(binary, selected_profile): + assert binary.is_file() + assert selected_profile == profile + order.append("elf-gate") + + def fake_host_elf_gate(binary, selected_profile): + assert binary == host + assert selected_profile == profile + order.append("host-elf-gate") + + def fake_capture_build_identity(**kwargs): + order.append("capture") + captured.update(kwargs) + return identity + + runtime_builder_module = importlib.import_module("simpler_setup.runtime_builder") + report_module = importlib.import_module("simpler_setup.tools.fdwic_submit_pmu_report") + monkeypatch.delenv("PTO_FDWIC_TENSORMAP_MODE", raising=False) + monkeypatch.setenv("PTO_FDWIC_PROFILE", profile) + monkeypatch.setattr(runtime_builder_module, "RuntimeBuilder", FakeRuntimeBuilder) + monkeypatch.setattr(_scene_test_module, "_assert_fdwic_submit_pmu_elf", fake_elf_gate) + monkeypatch.setattr(_scene_test_module, "_assert_fdwic_submit_pmu_host_elf", fake_host_elf_gate) + monkeypatch.setattr(report_module, "capture_build_identity", fake_capture_build_identity) + _fdwic_build_identity_cache.clear() + + binary = maybe_build_aicore_override(profiled_key, "a5", runtime, str(orch), [], pto_isa_root="/pto") + + extra_key = binary.parent.name + expected_build_dir = ( + FakeRuntimeBuilder._CACHE_DIR / "a5" / "onboard" / runtime / "private" / "aicore-extra" / extra_key / "aicore" + ) + assert order == ["build", "elf-gate", "host-elf-gate", "capture"] + assert captured == { + "profile": profile, + "profiled_cache_key": profiled_key, + "aicore_extra_cache_key": extra_key, + "compile_definitions": [ + "PTO_FDWIC_SHARED_MAP=0", + "PTO_FDWIC_TENSORMAP_RING_CAP=128", + *expected_compile_definitions, + ], + "aicore_kernel": binary, + "aicore_build_dir": expected_build_dir, + "host_runtime": host, + "aicpu_runtime": aicpu, + } + assert _fdwic_build_identity_cache == {profiled_key: identity} + _fdwic_build_identity_cache.clear() + + +def test_submit_pmu_render_publishes_bound_provenance_and_html(monkeypatch, tmp_path): + report_module = importlib.import_module("simpler_setup.tools.fdwic_submit_pmu_report") + raw = tmp_path / report_module.DEFAULT_INPUT_NAME + raw.write_text("{}") + identity = object() + called = [] + + def fake_write_report_with_provenance(input_path, build_identity, output_path): + called.append((input_path, build_identity, output_path)) + output_path.write_text("") + provenance = tmp_path / report_module.DEFAULT_PROVENANCE_NAME + provenance.write_text("{}") + return output_path, provenance + + monkeypatch.setattr(report_module, "write_report_with_provenance", fake_write_report_with_provenance) + + report = _render_case_fdwic_submit_pmu("Case", tmp_path, identity) + + assert report == tmp_path / report_module.DEFAULT_OUTPUT_NAME + assert called == [(raw, identity, report)] + + +def test_submit_pmu_run_rejects_missing_build_identity_before_case(monkeypatch): + class MissingIdentityCase: + _st_level = 2 + _st_runtime = "fully_distributed_within_core" + called = False + + @classmethod + def _run_and_validate(cls, *args, **kwargs): + cls.called = True + + monkeypatch.delenv("PTO_FDWIC_TENSORMAP_MODE", raising=False) + monkeypatch.setenv("PTO_FDWIC_PROFILE", "submit-pmu-none") + _fdwic_build_identity_cache.clear() + worker = SimpleNamespace(_config={"platform": "a5", "device_id": 0}) + + with pytest.raises(RuntimeError, match="build identity is missing"): + run_class_cases( + worker, + MissingIdentityCase(), + [{"name": "Case1", "config": {}, "params": {}}], + callable_obj=None, + sub_handles={}, + rounds=1, + skip_golden=False, + enable_l2_swimlane=0, + enable_dump_args=False, + enable_pmu=0, + enable_dep_gen=False, + enable_scope_stats=False, + ) + + assert not MissingIdentityCase.called + + +def test_submit_pmu_run_resolves_profiled_identity_for_render(monkeypatch, tmp_path): + class IdentityCase: + _st_level = 2 + _st_runtime = "fully_distributed_within_core" + + @staticmethod + def _run_and_validate(*args, **kwargs): + return None + + profile = "submit-pmu-none" + key = (IdentityCase.__qualname__, "a5", IdentityCase._st_runtime, "private", profile) + identity = object() + rendered = [] + monkeypatch.delenv("PTO_FDWIC_TENSORMAP_MODE", raising=False) + monkeypatch.setenv("PTO_FDWIC_PROFILE", profile) + monkeypatch.setattr(_scene_test_module, "_build_output_prefix", lambda _label: tmp_path) + monkeypatch.setattr( + _scene_test_module, + "_render_case_fdwic_submit_pmu", + lambda case_label, output_prefix, build_identity: rendered.append((case_label, output_prefix, build_identity)), + ) + _fdwic_build_identity_cache.clear() + _fdwic_build_identity_cache[key] = identity + worker = SimpleNamespace(_config={"platform": "a5", "device_id": 0}) + + run_class_cases( + worker, + IdentityCase(), + [{"name": "Case1", "config": {}, "params": {}}], + callable_obj=None, + sub_handles={}, + rounds=1, + skip_golden=False, + enable_l2_swimlane=0, + enable_dump_args=False, + enable_pmu=0, + enable_dep_gen=False, + enable_scope_stats=False, + ) + + assert rendered == [("IdentityCase_Case1", tmp_path, identity)] + _fdwic_build_identity_cache.clear() + + +def test_fdwic_profile_rejects_unknown_value(monkeypatch): + monkeypatch.setenv("PTO_FDWIC_PROFILE", "typo") + + with pytest.raises(ValueError, match="Unsupported PTO_FDWIC_PROFILE"): + _fdwic_profile() + + +def test_perf_clock_elf_gate_accepts_required_symbol_without_observers(monkeypatch, tmp_path): + """The final image must keep its marker and remove observer slow paths.""" + symbol_table = "37410: 0000000000001b54 68 FUNC WEAK DEFAULT 1 _Z30dist_perf_clock_expect_submitsj\n" + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=symbol_table, stderr=""), + ) + + _assert_fdwic_perf_clock_elf(tmp_path / "aicore_kernel.o") + + +def test_perf_clock_kernel_elf_gate_requires_its_dedicated_marker(monkeypatch, tmp_path): + symbol_table = ( + "37410: 0000000000001b54 68 FUNC WEAK DEFAULT 1 _Z30dist_perf_clock_expect_submitsj\n" + "37411: 0000000000001b98 8 FUNC WEAK DEFAULT 1 dist_perf_clock_kernel_profile_marker\n" + ) + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=symbol_table, stderr=""), + ) + + _assert_fdwic_perf_clock_elf(tmp_path / "aicore_kernel.o", "perf-clock-kernel") + + +def test_perf_clock_kernel_elf_gate_rejects_plain_perf_clock_image(monkeypatch, tmp_path): + symbol_table = "37410: 0000000000001b54 68 FUNC WEAK DEFAULT 1 _Z30dist_perf_clock_expect_submitsj\n" + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=symbol_table, stderr=""), + ) + + with pytest.raises(RuntimeError, match="missing defined perf-clock marker"): + _assert_fdwic_perf_clock_elf(tmp_path / "aicore_kernel.o", "perf-clock-kernel") + + +def _write_perf_clock_artifact(tmp_path: Path, profile: str) -> Path: + is_kernel = profile == "perf-clock-kernel" + output_name = "fdwic_perf_clock_kernel_summary.json" if is_kernel else "fdwic_perf_clock_summary.json" + cores = [] + for core_id in range(96): + core = { + "core_id": core_id, + "core_type": "aic" if core_id < 32 else "aiv", + "submit_count": 5, + "first_submit_start": 100 + core_id, + "last_submit_end": 200 + core_id, + "elapsed_ticks": 100, + } + if is_kernel: + has_kernel = core_id == 0 + core.update( + kernel_elapsed_ticks=10 if has_kernel else 0, + kernel_calls=1 if has_kernel else 0, + non_kernel_residual_ticks=90 if has_kernel else 100, + ) + cores.append(core) + payload = { + "schema": "fdwic-perf-clock-kernel-v1" if is_kernel else "fdwic-perf-clock-v1", + "mode": profile, + "num_cores": 96, + "aic_cores": 32, + "aiv_cores": 64, + "expected_submits_per_core": 5, + "global_first_submit_start": 100, + "global_last_submit_end": 295, + "global_submit_span_ticks": 195, + "cores": cores, + } + if is_kernel: + payload.update( + kernel_calls=1, + min_kernel_calls_in_window=1, + max_kernel_calls_in_window=4, + kernel_elapsed_ticks_sum=10, + non_kernel_residual_ticks_sum=9590, + groups={ + "aic": { + "cores": 32, + "elapsed_min_ticks": 100, + "elapsed_max_ticks": 100, + "elapsed_sum_ticks": 3200, + "kernel_min_ticks": 0, + "kernel_max_ticks": 10, + "kernel_sum_ticks": 10, + "kernel_calls_min": 0, + "kernel_calls_max": 1, + "kernel_calls_sum": 1, + "residual_min_ticks": 90, + "residual_max_ticks": 100, + "residual_sum_ticks": 3190, + }, + "aiv": { + "cores": 64, + "elapsed_min_ticks": 100, + "elapsed_max_ticks": 100, + "elapsed_sum_ticks": 6400, + "kernel_min_ticks": 0, + "kernel_max_ticks": 0, + "kernel_sum_ticks": 0, + "kernel_calls_min": 0, + "kernel_calls_max": 0, + "kernel_calls_sum": 0, + "residual_min_ticks": 100, + "residual_max_ticks": 100, + "residual_sum_ticks": 6400, + }, + }, + ) + else: + payload["groups"] = { + "aic": {"min_ticks": 100, "max_ticks": 100}, + "aiv": {"min_ticks": 100, "max_ticks": 100}, + } + artifact = tmp_path / output_name + artifact.write_text(json.dumps(payload)) + return artifact + + +@pytest.mark.parametrize("profile", ["perf-clock", "perf-clock-kernel"]) +def test_perf_clock_case_artifact_contract_accepts_exact_closure(tmp_path, profile): + artifact = _write_perf_clock_artifact(tmp_path, profile) + + assert _validate_case_fdwic_perf_clock("Case", tmp_path, profile) == artifact + + +def test_perf_clock_kernel_case_artifact_contract_rejects_wrong_integer_aggregate(tmp_path): + artifact = _write_perf_clock_artifact(tmp_path, "perf-clock-kernel") + payload = json.loads(artifact.read_text()) + payload["kernel_elapsed_ticks_sum"] += 1 + artifact.write_text(json.dumps(payload)) + + with pytest.raises(RuntimeError, match="invalid perf-clock-kernel integer aggregates"): + _validate_case_fdwic_perf_clock("Case", tmp_path, "perf-clock-kernel") + + +def test_perf_clock_case_artifact_contract_rejects_wrong_schema(tmp_path): + artifact = _write_perf_clock_artifact(tmp_path, "perf-clock") + payload = json.loads(artifact.read_text()) + payload["schema"] = "wrong" + artifact.write_text(json.dumps(payload)) + + with pytest.raises(RuntimeError, match="invalid perf-clock artifact contract"): + _validate_case_fdwic_perf_clock("Case", tmp_path, "perf-clock") + + +def test_perf_clock_kernel_case_artifact_contract_rejects_zero_length_core_window(tmp_path): + artifact = _write_perf_clock_artifact(tmp_path, "perf-clock-kernel") + payload = json.loads(artifact.read_text()) + core = payload["cores"][1] + core["last_submit_end"] = core["first_submit_start"] + core["elapsed_ticks"] = 0 + core["non_kernel_residual_ticks"] = 0 + artifact.write_text(json.dumps(payload)) + + with pytest.raises(RuntimeError, match="elapsed tick closure failed"): + _validate_case_fdwic_perf_clock("Case", tmp_path, "perf-clock-kernel") + + +def test_perf_clock_kernel_case_artifact_contract_recomputes_pa_call_range(tmp_path): + artifact = _write_perf_clock_artifact(tmp_path, "perf-clock-kernel") + payload = json.loads(artifact.read_text()) + payload["expected_submits_per_core"] = 1280 + for core in payload["cores"]: + core["submit_count"] = 1280 + artifact.write_text(json.dumps(payload)) + + with pytest.raises(RuntimeError, match="global Kernel call range closure failed"): + _validate_case_fdwic_perf_clock("Case", tmp_path, "perf-clock-kernel") + + +def test_perf_clock_kernel_case_artifact_contract_rejects_too_few_actual_calls(tmp_path): + artifact = _write_perf_clock_artifact(tmp_path, "perf-clock-kernel") + payload = json.loads(artifact.read_text()) + payload["expected_submits_per_core"] = 1280 + payload["min_kernel_calls_in_window"] = 256 + payload["max_kernel_calls_in_window"] = 1024 + for core in payload["cores"]: + core["submit_count"] = 1280 + artifact.write_text(json.dumps(payload)) + + with pytest.raises(RuntimeError, match="global Kernel call range closure failed"): + _validate_case_fdwic_perf_clock("Case", tmp_path, "perf-clock-kernel") + + +@pytest.mark.parametrize( + ("symbol_table", "message"), + [ + ("", "missing defined perf-clock marker"), + ( + "12: 0000000000000000 0 FUNC GLOBAL DEFAULT UND _Z30dist_perf_clock_expect_submitsj\n", + "missing defined perf-clock marker", + ), + ( + "37410: 0000000000001b54 68 FUNC WEAK DEFAULT 1 " + "_Z30dist_perf_clock_expect_submitsj\n" + "42: 0000000000000048 8 OBJECT LOCAL DEFAULT 17 g_fdwic_atomic_record_index\n", + r"profiling symbol\(s\) still present", + ), + ( + "37410: 0000000000001b54 68 FUNC WEAK DEFAULT 1 " + "_Z30dist_perf_clock_expect_submitsj\n" + "43: 0000000000000080 8 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_read_counters\n", + r"profiling symbol\(s\) still present", + ), + ( + "37410: 0000000000001b54 68 FUNC WEAK DEFAULT 1 " + "_Z30dist_perf_clock_expect_submitsj\n" + "44: 0000000000000088 8 OBJECT LOCAL DEFAULT 17 g_fdwic_submit_pmu_reg_base\n", + r"profiling symbol\(s\) still present", + ), + ( + "37410: 0000000000001b54 68 FUNC WEAK DEFAULT 1 " + "_Z30dist_perf_clock_expect_submitsj\n" + "37411: 0000000000001b98 8 FUNC WEAK DEFAULT 1 dist_perf_clock_kernel_profile_marker\n", + r"profiling symbol\(s\) still present", + ), + ], +) +def test_perf_clock_elf_gate_rejects_incomplete_image(monkeypatch, tmp_path, symbol_table, message): + """A missing marker or residual observer state must fail closed.""" + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=symbol_table, stderr=""), + ) + + with pytest.raises(RuntimeError, match=message): + _assert_fdwic_perf_clock_elf(tmp_path / "aicore_kernel.o") + + +def test_perf_clock_elf_gate_reports_readelf_failure(monkeypatch, tmp_path): + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=1, stdout="", stderr="not an ELF"), + ) + + with pytest.raises(RuntimeError, match="readelf failed.*not an ELF"): + _assert_fdwic_perf_clock_elf(tmp_path / "aicore_kernel.o") + + +def test_submit_pmu_elf_gate_accepts_only_whole_window_observer(monkeypatch, tmp_path): + symbol_table = ( + "10: 0000000000001000 64 FUNC WEAK DEFAULT 1 dist_submit_pmu_expect_submits\n" + "11: 0000000000001040 96 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_read_counters\n" + ) + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=symbol_table, stderr=""), + ) + + _assert_fdwic_submit_pmu_elf(tmp_path / "aicore_kernel.o", "submit-pmu-none") + + +@pytest.mark.parametrize( + "profile", + [ + "submit-pmu-arg-build", + "submit-pmu-empty-bracket", + "submit-pmu-materialize", + "submit-pmu-claim", + "submit-pmu-register", + "submit-pmu-submit-transition", + "submit-pmu-efdrain-control", + "submit-pmu-prepare-map", + "submit-pmu-fanin", + "submit-pmu-winner-build-control", + "submit-pmu-alloc-complete-control", + "submit-pmu-loser-replay", + ], +) +def test_submit_pmu_phase_elf_gate_requires_all_running_phase_readers(monkeypatch, tmp_path, profile): + symbol_table = ( + "10: 0000000000001000 64 FUNC WEAK DEFAULT 1 dist_submit_pmu_expect_submits\n" + "11: 0000000000001040 96 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_read_counters\n" + "12: 00000000000010a0 64 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_phase_read_shadow_counters\n" + "13: 00000000000010e0 64 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_phase_read_scalar_shadow\n" + "14: 0000000000001120 64 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_phase_read_total_shadow\n" + ) + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=symbol_table, stderr=""), + ) + + _assert_fdwic_submit_pmu_elf(tmp_path / "aicore_kernel.o", profile) + + +@pytest.mark.parametrize( + ("symbol_table", "message"), + [ + ("", "missing defined submit-pmu marker"), + ( + "1: 0 0 FUNC GLOBAL DEFAULT UND dist_submit_pmu_expect_submits\n" + "2: 0 1 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_read_counters\n", + "missing defined submit-pmu marker", + ), + ( + "1: 0 1 FUNC WEAK DEFAULT 1 dist_submit_pmu_expect_submits\n" + "2: 0 1 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_read_counters\n" + "3: 0 1 FUNC LOCAL DEFAULT 1 fdwic_swimlane_detail_record_atomic\n", + r"unrelated profiling symbol\(s\) still present", + ), + ( + "1: 0 1 FUNC WEAK DEFAULT 1 dist_submit_pmu_expect_submits\n" + "2: 0 1 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_read_counters\n" + "3: 0 1 FUNC WEAK DEFAULT 1 get_aicore_pmu_ring\n", + r"unrelated profiling symbol\(s\) still present", + ), + ( + "1: 0 1 FUNC WEAK DEFAULT 1 dist_submit_pmu_expect_submits\n" + "2: 0 1 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_read_counters\n" + "3: 0 1 FUNC WEAK DEFAULT 1 get_aicore_pmu_reg_base\n", + r"unrelated profiling symbol\(s\) still present", + ), + ( + "1: 0 1 FUNC WEAK DEFAULT 1 dist_submit_pmu_expect_submits\n" + "2: 0 1 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_read_counters\n" + "3: 0 1 FUNC WEAK DEFAULT 1 dist_perf_clock_expect_submits\n", + r"unrelated profiling symbol\(s\) still present", + ), + ], +) +def test_submit_pmu_elf_gate_rejects_incomplete_or_mixed_image(monkeypatch, tmp_path, symbol_table, message): + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=symbol_table, stderr=""), + ) + + with pytest.raises(RuntimeError, match=message): + _assert_fdwic_submit_pmu_elf(tmp_path / "aicore_kernel.o", "submit-pmu-none") + + +@pytest.mark.parametrize( + "profile", + [ + "submit-pmu-arg-build", + "submit-pmu-empty-bracket", + "submit-pmu-materialize", + "submit-pmu-claim", + "submit-pmu-register", + "submit-pmu-submit-transition", + "submit-pmu-efdrain-control", + "submit-pmu-prepare-map", + "submit-pmu-fanin", + "submit-pmu-winner-build-control", + "submit-pmu-alloc-complete-control", + "submit-pmu-loser-replay", + ], +) +def test_submit_pmu_phase_elf_gate_rejects_incomplete_running_phase_readers(monkeypatch, tmp_path, profile): + symbol_table = ( + "10: 0000000000001000 64 FUNC WEAK DEFAULT 1 dist_submit_pmu_expect_submits\n" + "11: 0000000000001040 96 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_read_counters\n" + "12: 00000000000010a0 64 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_phase_read_shadow_counters\n" + ) + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=symbol_table, stderr=""), + ) + + with pytest.raises(RuntimeError, match="missing defined submit-pmu marker"): + _assert_fdwic_submit_pmu_elf(tmp_path / "aicore_kernel.o", profile) + + +@pytest.mark.parametrize( + "phase_reader", + ( + "fdwic_submit_pmu_phase_read_shadow_counters", + "fdwic_submit_pmu_phase_read_scalar_shadow", + "fdwic_submit_pmu_phase_read_total_shadow", + ), +) +def test_submit_pmu_none_elf_gate_rejects_every_running_phase_reader(monkeypatch, tmp_path, phase_reader): + symbol_table = ( + "10: 0000000000001000 64 FUNC WEAK DEFAULT 1 dist_submit_pmu_expect_submits\n" + "11: 0000000000001040 96 FUNC LOCAL DEFAULT 1 fdwic_submit_pmu_read_counters\n" + f"12: 00000000000010a0 64 FUNC LOCAL DEFAULT 1 {phase_reader}\n" + ) + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=symbol_table, stderr=""), + ) + + with pytest.raises(RuntimeError, match=r"unrelated profiling symbol\(s\) still present"): + _assert_fdwic_submit_pmu_elf(tmp_path / "aicore_kernel.o", "submit-pmu-none") + + +@pytest.mark.parametrize( + "profile", + ( + "submit-pmu-efdrain-control", + "submit-pmu-prepare-map", + "submit-pmu-fanin", + "submit-pmu-winner-build-control", + "submit-pmu-alloc-complete-control", + "submit-pmu-loser-replay", + ), +) +def test_submit_pmu_host_elf_gate_accepts_exact_profile_and_hooks(monkeypatch, tmp_path, profile): + host_runtime = tmp_path / "libhost_runtime.so" + host_runtime.write_bytes(b"prefix\0" + profile.encode() + b"\0suffix") + symbol_table = ( + "10: 0000000000001000 64 FUNC GLOBAL DEFAULT 1 fdwic_submit_pmu_host_init\n" + "11: 0000000000001040 64 FUNC GLOBAL DEFAULT 1 fdwic_submit_pmu_host_export\n" + "12: 0000000000001080 64 FUNC GLOBAL DEFAULT 1 fdwic_submit_pmu_host_finalize\n" + ) + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=symbol_table, stderr=""), + ) + + _assert_fdwic_submit_pmu_host_elf(host_runtime, profile) + + +@pytest.mark.parametrize( + ("image", "symbol_table", "message"), + ( + ( + b"submit-pmu-submit-transition\0", + "10: 0 1 FUNC GLOBAL DEFAULT 1 fdwic_submit_pmu_host_init\n" + "11: 0 1 FUNC GLOBAL DEFAULT 1 fdwic_submit_pmu_host_export\n" + "12: 0 1 FUNC GLOBAL DEFAULT 1 fdwic_submit_pmu_host_finalize\n", + "missing exact profile marker", + ), + ( + b"submit-pmu-efdrain-control\0", + "10: 0 1 FUNC GLOBAL DEFAULT 1 fdwic_submit_pmu_host_init\n", + "missing defined host hook", + ), + ), +) +def test_submit_pmu_host_elf_gate_rejects_stale_or_incomplete_runtime( + monkeypatch, tmp_path, image, symbol_table, message +): + host_runtime = tmp_path / "libhost_runtime.so" + host_runtime.write_bytes(image) + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=symbol_table, stderr=""), + ) + + with pytest.raises(RuntimeError, match=message): + _assert_fdwic_submit_pmu_host_elf(host_runtime, "submit-pmu-efdrain-control") + + +def test_swimlane_elf_gate_accepts_merged_phase_atomic_observer(monkeypatch, tmp_path): + symbol_table = ( + "2796: 0000000000012d8c 4080 FUNC LOCAL DEFAULT 1 " + "_ZN12_GLOBAL__N_131fdwic_atomic_poll_boundary_slowEm\n" + "3010: 0000000000013d7c 344 FUNC LOCAL DEFAULT 1 " + "_ZN12_GLOBAL__N_135fdwic_swimlane_detail_record_atomicEv\n" + ) + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=symbol_table, stderr=""), + ) + + _assert_fdwic_swimlane_elf(tmp_path / "aicore_kernel.o") + + +@pytest.mark.parametrize( + ("symbol_table", "message"), + [ + ("", "missing defined swimlane observer"), + ( + "1: 0 1 FUNC LOCAL DEFAULT 1 fdwic_atomic_poll_boundary_slow\n" + "2: 0 1 FUNC LOCAL DEFAULT 1 fdwic_swimlane_detail_record_atomic\n" + "3: 0 1 FUNC WEAK DEFAULT 1 dist_perf_clock_expect_submits\n", + "isolated-profile symbol.*leaked", + ), + ( + "1: 0 1 FUNC LOCAL DEFAULT 1 fdwic_atomic_poll_boundary_slow\n" + "2: 0 1 FUNC LOCAL DEFAULT 1 fdwic_swimlane_detail_record_atomic\n" + "3: 0 1 FUNC WEAK DEFAULT 1 dist_submit_pmu_expect_submits\n", + "isolated-profile symbol.*leaked", + ), + ( + "1: 0 1 FUNC LOCAL DEFAULT 1 fdwic_atomic_poll_boundary_slow\n" + "2: 0 1 FUNC LOCAL DEFAULT 1 fdwic_swimlane_detail_record_atomic\n" + "3: 0 1 FUNC WEAK DEFAULT 1 get_fdwic_submit_pmu_reg_base\n", + "isolated-profile symbol.*leaked", + ), + ], +) +def test_swimlane_elf_gate_rejects_wrong_image(monkeypatch, tmp_path, symbol_table, message): + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout=symbol_table, stderr=""), + ) + + with pytest.raises(RuntimeError, match=message): + _assert_fdwic_swimlane_elf(tmp_path / "aicore_kernel.o") + + +class _FakePytestConfig: + def __init__(self, **options): + self.options = { + "--fdwic-tensormap": "private", + "--fdwic-profile": "submit-pmu-none", + "--platform": "a5", + "--runtime": "fully_distributed_within_core", + "--level": 2, + "--rounds": 1, + **options, + } + + def getoption(self, option, default=None): + return self.options.get(option, default) + + +@pytest.mark.parametrize( + "profile", + ( + "submit-pmu-none", + "submit-pmu-efdrain-control", + "submit-pmu-prepare-map", + "submit-pmu-fanin", + "submit-pmu-winner-build-control", + "submit-pmu-alloc-complete-control", + "submit-pmu-loser-replay", + ), +) +def test_submit_pmu_profile_publishes_environment(monkeypatch, profile): + monkeypatch.delenv("PTO_FDWIC_PROFILE", raising=False) + + _configure_fdwic_profile(_FakePytestConfig(**{"--fdwic-profile": profile})) + + assert _fdwic_profile() == profile + + +def test_perf_clock_kernel_profile_publishes_environment(monkeypatch): + monkeypatch.delenv("PTO_FDWIC_PROFILE", raising=False) + + _configure_fdwic_profile(_FakePytestConfig(**{"--fdwic-profile": "perf-clock-kernel"})) + + assert _fdwic_profile() == "perf-clock-kernel" + + +def test_shared_tensormap_mode_publishes_environment(monkeypatch): + monkeypatch.delenv("PTO_FDWIC_TENSORMAP_MODE", raising=False) + + _configure_fdwic_tensormap(_FakePytestConfig(**{"--fdwic-tensormap": "shared"})) + + assert _fdwic_tensormap_mode() == "shared" + + +@pytest.mark.parametrize( + ("options", "message"), + [ + ({"--platform": "a2a3sim"}, "requires --platform a5 or a5sim"), + ({"--runtime": "host_build_graph"}, "only supports runtime fully_distributed_within_core"), + ({"--level": 3}, "only supports SceneTest level 2"), + ], +) +def test_shared_tensormap_mode_rejects_wrong_execution_scope(options, message): + with pytest.raises(pytest.UsageError, match=message): + _configure_fdwic_tensormap(_FakePytestConfig(**{"--fdwic-tensormap": "shared", **options})) + + +def test_standalone_shared_tensormap_rejects_mixed_runtime_or_l3_classes(): + fdwic_l2 = type( + "FdwicL2", + (), + {"_st_level": 2, "_st_runtime": "fully_distributed_within_core"}, + ) + fdwic_l3 = type( + "FdwicL3", + (), + {"_st_level": 3, "_st_runtime": "fully_distributed_within_core"}, + ) + other_l2 = type("OtherL2", (), {"_st_level": 2, "_st_runtime": "host_build_graph"}) + + _validate_fdwic_tensormap_test_classes("private", [fdwic_l3, other_l2]) + _validate_fdwic_tensormap_test_classes("shared", [fdwic_l2]) + with pytest.raises(ValueError, match=r"FdwicL3, OtherL2"): + _validate_fdwic_tensormap_test_classes("shared", [fdwic_l2, other_l2, fdwic_l3]) + + +@pytest.mark.parametrize( + ("options", "message"), + [ + ({"--platform": "a5sim"}, "requires --platform a5"), + ({"--runtime": "tensormap_and_ringbuffer"}, "only supports runtime fully_distributed_within_core"), + ({"--level": 3}, "only supports SceneTest level 2"), + ({"--rounds": 2}, "requires --rounds 1"), + ], +) +def test_submit_pmu_profile_rejects_wrong_execution_scope(options, message): + with pytest.raises(pytest.UsageError, match=message): + _configure_fdwic_profile(_FakePytestConfig(**options)) + + +@pytest.mark.parametrize( + "option", + [ + "--enable-l2-swimlane", + "--dump-args", + "--enable-pmu", + "--enable-dep-gen", + "--enable-scope-stats", + "--enable-device-log-timing", + "--enable-swimlane-overhead", + "--use-example-exec-time", + ], +) +def test_submit_pmu_profile_rejects_other_diagnostics(option): + with pytest.raises(pytest.UsageError, match=option): + _configure_fdwic_profile(_FakePytestConfig(**{option: 1})) + + +def test_strict_fdwic_v4_converter_requires_closure_artifacts(monkeypatch, tmp_path): + raw = tmp_path / "l2_swimlane_records.json" + raw.write_text("{}") + monkeypatch.setattr( + _scene_test_module.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace(returncode=0, stdout="converted", stderr=""), + ) + + with pytest.raises(RuntimeError, match="did not publish required artifact"): + _run_swimlane_converter(input_path=raw, strict_fdwic_v4=True) + + (tmp_path / "merged_swimlane.json").write_text("{}") + (tmp_path / "swimlane_exclusive_analysis.json").write_text("{}") + _run_swimlane_converter(input_path=raw, strict_fdwic_v4=True) + + +def test_strict_fdwic_v4_converter_propagates_validation_failure(monkeypatch, tmp_path): + raw = tmp_path / "l2_swimlane_records.json" + raw.write_text("{}") + + def fail(*args, **kwargs): + raise subprocess.CalledProcessError(1, args[0], stderr="integer closure failed") + + monkeypatch.setattr(_scene_test_module.subprocess, "run", fail) + + with pytest.raises(RuntimeError, match="closure validation failed.*integer closure failed"): + _run_swimlane_converter(input_path=raw, strict_fdwic_v4=True) + + +def test_strict_fdwic_v4_case_requires_raw_artifact(tmp_path): + with pytest.raises(RuntimeError, match="required FDWIC schema-v4 raw artifact was not produced"): + _convert_case_swimlane("Case", tmp_path, strict_fdwic_v4=True) + + +def test_device_failure_keeps_original_error_and_disables_strict_conversion(monkeypatch, tmp_path): + class FailingCase: + _st_level = 2 + _st_runtime = "fully_distributed_within_core" + + @staticmethod + def _run_and_validate(*args, **kwargs): + raise ValueError("device execution failed") + + worker = SimpleNamespace(_config={"platform": "a5", "device_id": 0}) + strict_values = [] + monkeypatch.setattr(_scene_test_module, "_build_output_prefix", lambda _label: tmp_path) + monkeypatch.setattr( + _scene_test_module, + "_convert_case_swimlane", + lambda *args, strict_fdwic_v4=False, **kwargs: strict_values.append(strict_fdwic_v4), + ) + + with pytest.raises(ValueError, match="device execution failed"): + run_class_cases( + worker, + FailingCase(), + [{"name": "Case1", "config": {}, "params": {}}], + callable_obj=None, + sub_handles={}, + rounds=1, + skip_golden=False, + enable_l2_swimlane=4, + enable_dump_args=False, + enable_pmu=0, + enable_dep_gen=False, + enable_scope_stats=False, + ) + + assert strict_values == [False]