ci_sim.yml

# Copyright (c) 2026 Huawei Technologies Co., Ltd.
# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
# CANN Open Software License Agreement Version 2.0 (the "License").
# Please refer to the License for details. You may not use this file except in compliance with the License.
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
# See LICENSE in the root of the software repository for the full text of the License.

name: ci-sim

on:
  pull_request:
  # Nightly self-hosted simulator validation (GitHub cron is UTC).
  schedule:
    - cron: "0 14 * * *"
  workflow_dispatch:

permissions:
  contents: read
  pull-requests: read

jobs:
  pr-context:
    if: ${{ github.event_name == 'pull_request' && github.run_attempt == 1 }}
    runs-on: ubuntu-22.04
    steps:
      - name: Write PR context
        env:
          PR_NUMBER: ${{ github.event.pull_request.number }}
          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p pr-context
          printf '%s\n' "${PR_NUMBER}" > pr-context/pr-number
          printf '%s\n' "${PR_HEAD_SHA}" > pr-context/pr-head-sha

      - name: Upload PR context
        uses: actions/upload-artifact@v4
        with:
          name: pr-context
          path: pr-context
          if-no-files-found: error

  detect-vpto-sim-changes:
    runs-on: ubuntu-22.04
    outputs:
      should_run: ${{ steps.decide.outputs.should_run }}
    steps:
      - name: Detect PR changes relevant to VPTO SIM validation
        if: ${{ github.event_name == 'pull_request' }}
        id: filter
        uses: dorny/paths-filter@v3
        with:
          token: ${{ github.token }}
          filters: |
            relevant:
              - '.github/workflows/ci_sim.yml'
              - 'scripts/sim_dsl.sh'
              - 'test/vpto/**'
              - 'test/tilelang_st/**'
              - 'test/dsl-st/**'
              - 'test/tilelib-st/**'
              - 'ptodsl/**'
              - 'tools/ptoas/**'
              - 'lib/TileOps/**'
              - 'include/PTO/IR/VPTO*.td'
              - 'lib/PTO/IR/VPTO.cpp'
              - 'include/PTO/Transforms/Passes.h'
              - 'include/PTO/Transforms/Passes.td'
              - 'include/PTO/Transforms/*VPTO*'
              - 'include/PTO/Transforms/**/*VPTO*'
              - 'lib/PTO/Transforms/CMakeLists.txt'
              - 'lib/PTO/Transforms/ExpandTileOp.cpp'
              - 'lib/PTO/Transforms/FoldTileBufIntrinsics.cpp'
              - 'lib/PTO/Transforms/PTOToEmitC.cpp'
              - 'lib/PTO/Transforms/*VPTO*'
              - 'lib/PTO/Transforms/**/*VPTO*'

      - name: Decide whether to run VPTO SIM validation
        id: decide
        shell: bash
        env:
          EVENT_NAME: ${{ github.event_name }}
          PR_RELEVANT: ${{ steps.filter.outputs.relevant || 'false' }}
        run: |
          set -euo pipefail
          if [[ "${EVENT_NAME}" != "pull_request" ]]; then
            echo "should_run=true" >> "${GITHUB_OUTPUT}"
            echo "Running vpto-sim-validation on ${EVENT_NAME}"
          elif [[ "${PR_RELEVANT}" == "true" ]]; then
            echo "should_run=true" >> "${GITHUB_OUTPUT}"
            echo "Running vpto-sim-validation because the PR touched VPTO-related paths"
          else
            echo "should_run=false" >> "${GITHUB_OUTPUT}"
            echo "Skipping vpto-sim-validation because the PR did not touch VPTO-related paths"
          fi

  vpto-sim-validation:
    needs: detect-vpto-sim-changes
    runs-on: [self-hosted, Linux, X64, label-1]
    timeout-minutes: 300
    concurrency:
      group: vpto-sim-${{ github.event.pull_request.number || github.ref }}
      cancel-in-progress: true
    if: >-
      ${{
        needs.detect-vpto-sim-changes.outputs.should_run == 'true'
      }}
    env:
      LLVM_REPO: https://github.com/vpto-dev/llvm-project.git
      LLVM_REF: feature-vpto-llvm21
      PTOAS_VENV: ${{ github.workspace }}/.work/ptoas-sim-venv
      VPTO_SIM_WORKSPACE: ${{ github.workspace }}/.work/vpto-sim-ci
      TILELIB_ST_WORKSPACE: ${{ github.workspace }}/.work/tilelib-st-ci
      PYPTO_REF: ef6ce7cd8bd33b4c93b58dc34830a20b145736ac
      PYPTO_WORKSPACE: ${{ github.workspace }}/.work/pypto-ci
      PYPTO_RUN_WORKSPACE: ${{ github.workspace }}/.work/pypto-run-ci
      # GitHub pin containing the two-argument Soft SYNCALL ABI from f24f7b7
      # and the follow-up CPU-Sim duplicate-stub fix. The GitCode mirror carries
      # the ABI as d56d42d within the separate ce3262e3 remote-validation pin.
      PTO_ISA_COMMIT: a8168c6ca8ae22b06e4ea440506f0a87e446e9a2
      PTO_ISA_ROOT: ${{ github.workspace }}/.work/pto-isa-ci
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
          persist-credentials: false

      - name: Resolve simulator environment
        shell: bash
        run: |
          set -euo pipefail

          detect_ascend_home() {
            for d in \
              "${ASCEND_HOME_PATH:-}" \
              /usr/local/Ascend/cann \
              /usr/local/Ascend/cann-* \
              /usr/local/Ascend/ascend-toolkit/latest
            do
              [[ -n "${d}" && -d "${d}" ]] || continue
              printf '%s\n' "${d}"
              return 0
            done
            return 1
          }

          ASCEND_HOME_PATH_DETECTED="$(detect_ascend_home || true)"
          if [[ -z "${ASCEND_HOME_PATH_DETECTED}" ]]; then
            echo "ERROR: failed to detect ASCEND_HOME_PATH on self-hosted runner" >&2
            exit 1
          fi

          echo "ASCEND_HOME_PATH=${ASCEND_HOME_PATH_DETECTED}" >> "${GITHUB_ENV}"
          echo "PTOAS_BIN=${PTOAS_VENV}/bin/ptoas" >> "${GITHUB_ENV}"

      - name: Ensure runner dependencies
        shell: bash
        run: |
          set -euo pipefail
          missing_tools=()
          for tool in python3 git cmake ninja make; do
            if ! command -v "${tool}" >/dev/null 2>&1; then
              missing_tools+=("${tool}")
            fi
          done

          if [[ "${#missing_tools[@]}" -gt 0 ]]; then
            if command -v sudo >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then
              sudo apt-get update
              sudo apt-get install -y python3 python3-pip python3-venv git cmake ninja-build make
            else
              echo "ERROR: missing required tools on self-hosted runner: ${missing_tools[*]}" >&2
              echo "ERROR: automatic installation requires sudo + apt-get" >&2
              exit 1
            fi
          fi

          if ! python3 -m ensurepip --version >/dev/null 2>&1; then
            if command -v sudo >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then
              sudo apt-get update
              sudo apt-get install -y python3-pip python3-venv
            else
              echo "ERROR: python3-venv is required on self-hosted runner" >&2
              exit 1
            fi
          fi

      - name: Configure ccache
        shell: bash
        run: |
          set -euo pipefail
          if ! command -v ccache >/dev/null 2>&1; then
            if command -v sudo >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then
              sudo apt-get update
              sudo apt-get install -y ccache
            else
              echo "ccache is unavailable; continuing without a compiler launcher"
              exit 0
            fi
          fi

          ccache --version
          echo "CMAKE_C_COMPILER_LAUNCHER=ccache" >> "${GITHUB_ENV}"
          echo "CMAKE_CXX_COMPILER_LAUNCHER=ccache" >> "${GITHUB_ENV}"

      - name: Resolve LLVM directories
        shell: bash
        run: |
          set -euo pipefail
          echo "LLVM_ROOT=${RUNNER_TOOL_CACHE}/llvm-project" >> "${GITHUB_ENV}"
          echo "LLVM_DIR=${RUNNER_TOOL_CACHE}/llvm-project/llvm/build-assert" >> "${GITHUB_ENV}"

      - name: Resolve PTODSL build directories
        shell: bash
        run: |
          set -euo pipefail
          echo "PTO_DSL_ST_BUILD_DIR=${GITHUB_WORKSPACE}/build-ptodsl" >> "${GITHUB_ENV}"

      - name: Resolve LLVM cache key
        id: llvm-cache-key
        shell: bash
        run: |
          set -euo pipefail
          # Resolve to a Git object that ls-remote can handle: either a tag
          # (LLVM_TAG) or a branch head (LLVM_REF).  Only one is expected.
          ref="${LLVM_TAG:-${LLVM_REF}}"
          sha="$(git ls-remote "${LLVM_REPO}" "${ref}" | awk '{print $1}')"
          if [[ -z "${sha}" ]]; then
            echo "ERROR: failed to resolve ${LLVM_REPO} ${ref}" >&2
            exit 1
          fi
          echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
          echo "key=llvm-build-${sha}-assert-v2" >> "${GITHUB_OUTPUT}"

      - name: Restore LLVM build cache
        id: llvm-cache
        continue-on-error: true
        uses: actions/cache/restore@v4
        with:
          path: ${{ env.LLVM_DIR }}
          key: ${{ steps.llvm-cache-key.outputs.key }}

      - name: Clean CI work dirs
        shell: bash
        run: |
          set -euo pipefail
          rm -rf "${GITHUB_WORKSPACE}/build"
          rm -rf "${GITHUB_WORKSPACE}/build-ptodsl"
          rm -rf "${PTOAS_VENV}"
          rm -rf "${VPTO_SIM_WORKSPACE}"
          rm -rf "${TILELIB_ST_WORKSPACE}"
          rm -rf "${PYPTO_WORKSPACE}"
          rm -rf "${PYPTO_RUN_WORKSPACE}"
          rm -rf "${PTO_ISA_ROOT}"
          rm -rf "${GITHUB_WORKSPACE}/.work/camodel"

      - name: Create PTOAS simulator environment
        shell: bash
        run: |
          set -euo pipefail
          python3 -m venv "${PTOAS_VENV}"
          "${PTOAS_VENV}/bin/python" -m pip install --upgrade pip
          "${PTOAS_VENV}/bin/pip" install \
            'scikit-build-core>=0.12.2,<2' 'pybind11<3' \
            nanobind numpy ml-dtypes

      - name: Prepare LLVM source
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p "${LLVM_ROOT}"
          cd "${LLVM_ROOT}"

          if [ ! -d .git ]; then
            git init
            git remote add origin "${LLVM_REPO}"
          fi

          git remote set-url origin "${LLVM_REPO}"
          sha="${{ steps.llvm-cache-key.outputs.sha }}"
          git fetch --depth 1 origin "${sha}"
          git checkout --force "${sha}"

      - name: Build LLVM/MLIR
        if: steps.llvm-cache.outputs.cache-hit != 'true'
        shell: bash
        run: |
          set -euo pipefail
          cd "${LLVM_ROOT}"
          # Pin system compilers.  Self-hosted runners may have conda GCC on
          # PATH; letting CMake autodetect it produces build errors or ABI
          # mismatches with system libraries.
          export CC=gcc
          export CXX=g++
          # Clean the build directory so that stale generated files from a
          # previous run (e.g. _smt_ops_gen.py left behind when the ref
          # changed) do not leak into the fresh build.
          rm -rf "${LLVM_DIR}"
          cmake -G Ninja -S llvm -B "${LLVM_DIR}" \
            -DLLVM_ENABLE_PROJECTS="mlir;clang" \
            -DBUILD_SHARED_LIBS=ON \
            -DLLVM_ENABLE_ASSERTIONS=ON \
            -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
            -DPython3_EXECUTABLE="${PTOAS_VENV}/bin/python" \
            -DPython_EXECUTABLE="${PTOAS_VENV}/bin/python" \
            -Dpybind11_DIR="$("${PTOAS_VENV}/bin/python" -m pybind11 --cmakedir)" \
            -Dnanobind_DIR="$("${PTOAS_VENV}/bin/python" -m nanobind --cmake_dir)" \
            -DCMAKE_BUILD_TYPE=Release \
            -DLLVM_TARGETS_TO_BUILD="host"

          ninja -C "${LLVM_DIR}"

      - name: Save LLVM build cache
        if: steps.llvm-cache.outputs.cache-hit != 'true'
        continue-on-error: true
        uses: actions/cache/save@v4
        with:
          path: ${{ env.LLVM_DIR }}
          key: ${{ steps.llvm-cache-key.outputs.key }}

      - name: Install PTOAS editable
        shell: bash
        run: |
          set -euo pipefail
          # Pin system compilers for the same reason as LLVM above.
          export CC=gcc
          export CXX=g++
          LLVM_BUILD_DIR="${LLVM_DIR}" \
            "${PTOAS_VENV}/bin/python" -m pip install --editable . \
              --no-build-isolation --no-deps \
              --config-settings="build-dir=${GITHUB_WORKSPACE}/build"
          PATH="${PTOAS_VENV}/bin:${PATH}" \
          PYTHON="${PTOAS_VENV}/bin/python" \
            bash docker/test_wheel_imports.sh

      - name: Checkout PyPTO
        uses: actions/checkout@v4
        with:
          repository: hw-native-sys/pypto
          ref: ${{ env.PYPTO_REF }}
          path: ${{ env.PYPTO_WORKSPACE }}
          fetch-depth: 1
          submodules: recursive
          persist-credentials: false

      - name: Checkout PTO-ISA
        uses: actions/checkout@v4
        with:
          repository: hw-native-sys/pto-isa
          ref: ${{ env.PTO_ISA_COMMIT }}
          path: ${{ env.PTO_ISA_ROOT }}
          fetch-depth: 1
          persist-credentials: false

      - name: Ensure GCC 15 for PyPTO simulator
        shell: bash
        run: |
          set -euo pipefail

          compiler_major() {
            "$1" -dumpfullversion -dumpversion | cut -d. -f1
          }

          if command -v g++-15 >/dev/null 2>&1; then
            if [[ "$(compiler_major "$(command -v g++-15)")" != "15" ]]; then
              echo "ERROR: g++-15 exists but is not GCC 15: $(g++-15 --version | head -1)" >&2
              exit 1
            fi
            if ! command -v gcc-15 >/dev/null 2>&1; then
              echo "ERROR: g++-15 exists but gcc-15 is missing." >&2
              exit 1
            fi
            exit 0
          fi

          if ! command -v x86_64-conda-linux-gnu-g++ >/dev/null 2>&1 || \
             ! command -v x86_64-conda-linux-gnu-gcc >/dev/null 2>&1 || \
             [[ "$(compiler_major "$(command -v x86_64-conda-linux-gnu-g++)")" != "15" ]]; then
            if ! command -v conda >/dev/null 2>&1; then
              echo "ERROR: PyPTO simulator smoke requires GCC/G++ 15 and conda is unavailable to install it." >&2
              exit 1
            fi
            conda install -y -c conda-forge gcc_linux-64=15 gxx_linux-64=15
            hash -r
          fi

          mkdir -p "${PYPTO_WORKSPACE}/.work/bin"
          ln -sf "$(command -v x86_64-conda-linux-gnu-g++)" "${PYPTO_WORKSPACE}/.work/bin/g++-15"
          ln -sf "$(command -v x86_64-conda-linux-gnu-gcc)" "${PYPTO_WORKSPACE}/.work/bin/gcc-15"
          echo "${PYPTO_WORKSPACE}/.work/bin" >> "${GITHUB_PATH}"

      - name: Prepare PyPTO Python dependencies
        shell: bash
        run: |
          set -euo pipefail

          python3 -m pip install --upgrade pip
          python3 -m pip install --index-url https://download.pytorch.org/whl/cpu torch
          python3 -m pip install scikit-build-core nanobind ninja pytest cloudpickle

      - name: Install PyPTO frontend and runtime
        shell: bash
        env:
          PTO_ISA_ROOT: ${{ env.PTO_ISA_ROOT }}
        run: |
          set -euo pipefail
          rm -rf "${PYPTO_WORKSPACE}/build" "${PYPTO_WORKSPACE}/_skbuild" "${PYPTO_WORKSPACE}/runtime/build"
          python3 -m pip install --no-build-isolation --no-deps "${PYPTO_WORKSPACE}"
          # ci_sim only runs PyPTO simulator tests. Keep the runtime install from
          # auto-detecting onboard platforms from the runner's CANN environment.
          env -u ASCEND_HOME_PATH python3 -m pip install --no-build-isolation --no-deps "${PYPTO_WORKSPACE}/runtime"

          python3 - <<'PY'
          from pathlib import Path

          from simpler_setup.environment import PROJECT_ROOT
          from simpler_setup.kernel_compiler import KernelCompiler

          required = PROJECT_ROOT / "src" / "common" / "task_interface" / "arg_direction.h"
          if not required.is_file():
              raise SystemExit(f"missing required simpler runtime header: {required}")

          include_dirs = KernelCompiler(platform="a5sim").get_orchestration_include_dirs(
              "tensormap_and_ringbuffer"
          )
          if str(required.parent) not in include_dirs:
              raise SystemExit(
                  "simpler orchestration include dirs do not contain "
                  f"{required.parent}: {include_dirs}"
              )

          print(f"simpler PROJECT_ROOT={PROJECT_ROOT}")
          print(f"simpler arg_direction.h={required}")
          PY

      - name: Run PyPTO PTOAS end-to-end simulator smoke
        shell: bash
        env:
          PTOAS_ROOT: ${{ github.workspace }}/build/tools/ptoas
          PTO_ISA_ROOT: ${{ env.PTO_ISA_ROOT }}
          TORCH_DEVICE_BACKEND_AUTOLOAD: "0"
        run: |
          set -euo pipefail
          mkdir -p "${PYPTO_RUN_WORKSPACE}"
          cd "${PYPTO_WORKSPACE}"

          run_pypto_pytest() {
            local platform="$1"
            local suite_name="$2"
            local kernels_dir="${PYPTO_RUN_WORKSPACE}/${suite_name}_${platform}"
            local log_file="${PYPTO_RUN_WORKSPACE}/${suite_name}_${platform}.log"
            shift 2

            python3 -m pytest "$@" \
              -v \
              --platform="${platform}" \
              --save-kernels \
              --kernels-dir="${kernels_dir}" \
              2>&1 | tee "${log_file}"
          }

          # All suites below compile through PyPTO's PTO backend and run ptoas.
          # They are split into separate pytest processes to avoid backend global-state conflicts.
          run_pypto_core_smoke() {
            local platform="$1"
            local cross_core="tests/st/runtime/cross_core/test_cross_core.py::TestCrossCore::test_tpush_tpop_v2c_updown[${platform}]"
            local pypto_smoke_tests=(
              tests/st/examples/00_hello_world/test_hello_world.py
              tests/st/examples/02_intermediate/test_softmax.py::TestTileSoftmax::test_tile_softmax
              tests/st/examples/02_intermediate/test_rms_norm.py::TestRMSNormCore::test_rms_norm_core
              tests/st/runtime/ops/test_elementwise.py
              tests/st/runtime/ops/test_assemble.py
              tests/st/runtime/framework_and_models/test_jit.py::TestJITExecution::test_cache_hit_reuses_compiled_program
              tests/st/runtime/framework_and_models/test_jit.py::TestJITDynamicBatch::test_one_artifact_serves_multiple_batches
              tests/st/runtime/framework_and_models/test_compiled_program.py::TestManualWorkerExtraction::test_block_dim_override_runs
              "${cross_core}"
            )
            # The a5sim self-hosted environment does not ship the PyPTO
            # tensormap_and_ringbuffer runtime required by this dyn-orch case.
            # Keep the coverage on a2a3sim instead of failing unrelated PTOAS PRs.
            if [[ "${platform}" != "a5sim" ]]; then
              local dyn_valid_shape="tests/st/runtime/control_flow/test_dyn_orch_shape.py::TestDynOrchShapeOperations::test_dyn_orch_valid_shape_add[shape0-valid_shape0-${platform}]"
              pypto_smoke_tests+=("${dyn_valid_shape}")
            fi
            run_pypto_pytest "${platform}" core_ptoas "${pypto_smoke_tests[@]}"
          }

          run_pypto_fa_ptoas_smoke() {
            local platform="$1"
            local pypto_fa_tests=(
              tests/st/runtime/ops/test_cast.py::TestCast::test_tile_cast_col_major_narrow
              tests/st/runtime/framework_and_models/test_paged_attention.py::TestPagedAttentionKernels::test_qk_matmul_ptoas[16-128-128]
              tests/st/runtime/framework_and_models/test_paged_attention.py::TestPagedAttentionKernels::test_softmax_prepare_ptoas[16-128]
              tests/st/runtime/framework_and_models/test_paged_attention.py::TestPagedAttentionKernels::test_softmax_prepare_unaligned_ptoas[16-128-100]
              tests/st/runtime/framework_and_models/test_paged_attention.py::TestPagedAttentionKernels::test_pv_matmul_ptoas[16-128-128]
              tests/st/runtime/framework_and_models/test_paged_attention.py::TestPagedAttentionKernels::test_online_update_ptoas[16-128-0-1]
            )
            run_pypto_pytest "${platform}" fa_ptoas "${pypto_fa_tests[@]}"
          }

          run_pypto_int8_codegen_smoke() {
            local platform="$1"
            local pypto_int8_tests=(
              tests/st/codegen/dsl/test_batch_matmul_pipeline.py::test_no_mat_to_mat_tmov
            )
            run_pypto_pytest "${platform}" int8_ptoas_codegen "${pypto_int8_tests[@]}"
          }

          run_pypto_core_smoke a5sim
          run_pypto_fa_ptoas_smoke a5sim
          run_pypto_core_smoke a2a3sim
          run_pypto_fa_ptoas_smoke a2a3sim
          run_pypto_int8_codegen_smoke a2a3sim

      - name: Prepare quiet camodel
        shell: bash
        run: |
          set -euo pipefail
          readarray -t camodel_candidates < <(
            find "${ASCEND_HOME_PATH}" -type d -path '*/simulator/dav_3510/lib' | sort
          )
          if [[ "${#camodel_candidates[@]}" -eq 0 ]]; then
            echo "ERROR: cannot find dav_3510 camodel lib under ${ASCEND_HOME_PATH}" >&2
            exit 1
          fi
          SIM_LIB_DIR="$(python3 scripts/prepare_quiet_camodel.py \
            --source-dir "${camodel_candidates[0]}" \
            --output-dir "${GITHUB_WORKSPACE}/.work/camodel")"
          echo "SIM_LIB_DIR=${SIM_LIB_DIR}" >> "${GITHUB_ENV}"
          echo "SIM_LIB_DIR=${SIM_LIB_DIR}"

      - name: Resolve PTODSL Python
        shell: bash
        run: |
          set -euo pipefail
          source "${ASCEND_HOME_PATH}/bin/setenv.bash"

          add_python_candidate() {
            local candidate="$1"
            local resolved
            [[ -n "${candidate}" ]] || return 0
            if [[ "${candidate}" != */* ]]; then
              resolved="$(command -v "${candidate}" 2>/dev/null || true)"
            else
              resolved="${candidate}"
            fi
            [[ -n "${resolved}" && -x "${resolved}" ]] || return 0
            resolved="$(readlink -f "${resolved}")"
            case ":${PYTHON_CANDIDATES[*]}:" in
              *":${resolved}:"*) return 0 ;;
            esac
            PYTHON_CANDIDATES+=("${resolved}")
          }

          has_torch_npu_packages() {
            "$1" - <<'PY'
          import importlib.util

          missing = [
              name for name in ("torch", "torch_npu")
              if importlib.util.find_spec(name) is None
          ]
          raise SystemExit(1 if missing else 0)
          PY
          }

          probe_ptodsl_runtime_python() {
            TORCH_DEVICE_BACKEND_AUTOLOAD=0 "$1" - <<'PY'
          import sys
          import nanobind
          import numpy
          import pybind11
          import pytest
          import xdist
          import yaml
          import torch
          import torch_npu  # noqa: F401

          print(sys.executable)
          print(f"python {sys.version_info.major}.{sys.version_info.minor}")
          print("torch", torch.__version__)
          print("torch_npu", getattr(torch_npu, "__version__", "unknown"))
          print("numpy", numpy.__version__)
          print("nanobind", getattr(nanobind, "__version__", "unknown"))
          print("pybind11", pybind11.__version__)
          print("pytest", pytest.__version__)
          print("pytest-xdist", xdist.__version__)
          PY
          }

          missing_ptodsl_python_deps() {
            "$1" - <<'PY'
          import importlib.util

          deps = [
              ("setuptools", "setuptools"),
              ("wheel", "wheel"),
              ("scikit_build_core", "scikit-build-core>=0.12.2,<2"),
              ("nanobind", "nanobind"),
              ("numpy", "numpy"),
              ("ml_dtypes", "ml-dtypes"),
              ("pytest", "pytest"),
              ("xdist", "pytest-xdist"),
              ("yaml", "PyYAML"),
          ]

          missing = [
              requirement for module_name, requirement in deps
              if importlib.util.find_spec(module_name) is None
          ]

          try:
              import pybind11
              version = tuple(int(part) for part in pybind11.__version__.split(".")[:1])
              if version >= (3,):
                  missing.append("pybind11<3")
          except Exception:
              missing.append("pybind11<3")

          print(" ".join(missing))
          PY
          }

          PYTHON_CANDIDATES=()
          if [[ -n "${PTO_DSL_ST_PYTHON_BIN:-}" ]]; then
            add_python_candidate "${PTO_DSL_ST_PYTHON_BIN}"
          else
            add_python_candidate python3
            shopt -s nullglob
            for candidate in \
              /home/*/miniconda3/bin/python3 \
              /home/*/miniconda3/bin/python \
              /home/*/anaconda3/bin/python3 \
              /home/*/anaconda3/bin/python \
              /home/*/miniconda3/envs/*/bin/python \
              /home/*/anaconda3/envs/*/bin/python \
              /opt/conda/envs/*/bin/python
            do
              add_python_candidate "${candidate}"
            done
            shopt -u nullglob
          fi

          SELECTED_BASE_PYTHON=""
          for candidate in "${PYTHON_CANDIDATES[@]}"; do
            echo "Probing PTODSL base Python package presence: ${candidate}"
            if has_torch_npu_packages "${candidate}"; then
              SELECTED_BASE_PYTHON="${candidate}"
              break
            fi
          done

          if [[ -z "${SELECTED_BASE_PYTHON}" ]]; then
            echo "ERROR: PTODSL DSL ST requires an existing Python runtime with torch and torch_npu." >&2
            echo "ERROR: this workflow intentionally does not install torch or torch_npu on every run." >&2
            echo "ERROR: set PTO_DSL_ST_PYTHON_BIN to a compatible pre-installed interpreter." >&2
            exit 1
          fi

          PYTHON_ABI_TAG="$("${SELECTED_BASE_PYTHON}" - <<'PY'
          import sys
          print(f"py{sys.version_info.major}{sys.version_info.minor}")
          PY
          )"
          BASE_HASH="$(printf '%s' "${SELECTED_BASE_PYTHON}" | sha256sum | cut -c1-12)"
          PYTHON_TAG="${PYTHON_ABI_TAG}-${BASE_HASH}"
          PTODSL_PYTHON_ROOT="${RUNNER_TOOL_CACHE}/ptodsl-python/${PYTHON_TAG}"
          if [[ ! -x "${PTODSL_PYTHON_ROOT}/bin/python" ]]; then
            rm -rf "${PTODSL_PYTHON_ROOT}"
            "${SELECTED_BASE_PYTHON}" -m venv --system-site-packages "${PTODSL_PYTHON_ROOT}"
          fi

          PTODSL_PYTHON="${PTODSL_PYTHON_ROOT}/bin/python"
          if ! "${PTODSL_PYTHON}" -m pip --version >/dev/null 2>&1; then
            "${PTODSL_PYTHON}" -m ensurepip --upgrade
          fi
          missing_deps="$(missing_ptodsl_python_deps "${PTODSL_PYTHON}")"
          if [[ -n "${missing_deps}" ]]; then
            "${PTODSL_PYTHON}" -m pip install ${missing_deps}
          fi

          probe_ptodsl_runtime_python "${PTODSL_PYTHON}"

          PTO_DSL_ST_LLVM_DIR="${RUNNER_TOOL_CACHE}/llvm-project/llvm/build-assert-${PYTHON_TAG}"
          PTO_DSL_ST_BIN_DIR="$(dirname "${PTODSL_PYTHON}")"
          echo "PTO_DSL_ST_BASE_PYTHON=${SELECTED_BASE_PYTHON}" >> "${GITHUB_ENV}"
          echo "PTO_DSL_ST_PYTHON_BIN=${PTODSL_PYTHON}" >> "${GITHUB_ENV}"
          echo "PTO_DSL_ST_BIN_DIR=${PTO_DSL_ST_BIN_DIR}" >> "${GITHUB_ENV}"
          echo "PTO_DSL_ST_PTOAS_BIN=${PTO_DSL_ST_BIN_DIR}/ptoas" >> "${GITHUB_ENV}"
          echo "PTO_DSL_ST_PYTHON_TAG=${PYTHON_TAG}" >> "${GITHUB_ENV}"
          echo "PTO_DSL_ST_LLVM_DIR=${PTO_DSL_ST_LLVM_DIR}" >> "${GITHUB_ENV}"

      - name: Run TileOp ST CI
        # Temporarily keep the legacy-named TileOp ST harness in smoke mode for every trigger.
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p "${TILELIB_ST_WORKSPACE}"
          export LLVM_BUILD_DIR="${LLVM_DIR}"
          ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \
          PTOAS_BIN="${PTOAS_BIN}" \
          bash test/tilelang_st/script/run_ci.sh -r sim -v a5 \
            --build-jobs 32 --jobs 64 --smoke \
            2>&1 | tee "${TILELIB_ST_WORKSPACE}/run_ci.log"

      - name: Restore PTODSL LLVM build cache
        id: ptodsl-llvm-cache
        continue-on-error: true
        uses: actions/cache/restore@v4
        with:
          path: ${{ env.PTO_DSL_ST_LLVM_DIR }}
          key: llvm-build-${{ steps.llvm-cache-key.outputs.sha }}-assert-${{ env.PTO_DSL_ST_PYTHON_TAG }}-v2

      - name: Build PTODSL LLVM/MLIR
        if: steps.ptodsl-llvm-cache.outputs.cache-hit != 'true'
        shell: bash
        run: |
          set -euo pipefail
          cd "${LLVM_ROOT}"
          export CC=gcc
          export CXX=g++
          PTODSL_PYTHON_PREFIX="$(dirname "$(dirname "${PTO_DSL_ST_PYTHON_BIN}")")"
          export VIRTUAL_ENV="${PTODSL_PYTHON_PREFIX}"
          export PATH="${PTODSL_PYTHON_PREFIX}/bin:${PATH}"
          PYBIND11_CMAKE_DIR="$("${PTO_DSL_ST_PYTHON_BIN}" -m pybind11 --cmakedir)"
          NANOBIND_CMAKE_DIR="$("${PTO_DSL_ST_PYTHON_BIN}" -m nanobind --cmake_dir)"
          PYTHON_INCLUDE_DIR="$("${PTO_DSL_ST_PYTHON_BIN}" - <<'PY'
          import sysconfig
          print(sysconfig.get_path("include"))
          PY
          )"
          PYTHON_LIBRARY="$("${PTO_DSL_ST_PYTHON_BIN}" - <<'PY'
          import sysconfig
          print(sysconfig.get_config_var("LIBDIR") + "/" + sysconfig.get_config_var("LDLIBRARY"))
          PY
          )"
          rm -rf "${PTO_DSL_ST_LLVM_DIR}"
          cmake -G Ninja -S llvm -B "${PTO_DSL_ST_LLVM_DIR}" \
            -DLLVM_ENABLE_PROJECTS="mlir;clang" \
            -DBUILD_SHARED_LIBS=ON \
            -DLLVM_ENABLE_ASSERTIONS=ON \
            -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
            -DPython3_EXECUTABLE="${PTO_DSL_ST_PYTHON_BIN}" \
            -DPython_EXECUTABLE="${PTO_DSL_ST_PYTHON_BIN}" \
            -DPython3_ROOT_DIR="${PTODSL_PYTHON_PREFIX}" \
            -DPython_ROOT_DIR="${PTODSL_PYTHON_PREFIX}" \
            -DPython3_INCLUDE_DIR="${PYTHON_INCLUDE_DIR}" \
            -DPython_INCLUDE_DIR="${PYTHON_INCLUDE_DIR}" \
            -DPython3_LIBRARY="${PYTHON_LIBRARY}" \
            -DPython_LIBRARY="${PYTHON_LIBRARY}" \
            -DPython3_FIND_STRATEGY=LOCATION \
            -DPython_FIND_STRATEGY=LOCATION \
            -DPython3_FIND_VIRTUALENV=ONLY \
            -DPython_FIND_VIRTUALENV=ONLY \
            -DPython3_FIND_REGISTRY=NEVER \
            -DPython_FIND_REGISTRY=NEVER \
            -DPython3_FIND_FRAMEWORK=NEVER \
            -DPython_FIND_FRAMEWORK=NEVER \
            -Dpybind11_DIR="${PYBIND11_CMAKE_DIR}" \
            -Dnanobind_DIR="${NANOBIND_CMAKE_DIR}" \
            -DCMAKE_PREFIX_PATH="${PYBIND11_CMAKE_DIR};${NANOBIND_CMAKE_DIR};${PTODSL_PYTHON_PREFIX}" \
            -DCMAKE_BUILD_TYPE=Release \
            -DLLVM_TARGETS_TO_BUILD="host"

          ninja -C "${PTO_DSL_ST_LLVM_DIR}"

      - name: Save PTODSL LLVM build cache
        if: steps.ptodsl-llvm-cache.outputs.cache-hit != 'true'
        continue-on-error: true
        uses: actions/cache/save@v4
        with:
          path: ${{ env.PTO_DSL_ST_LLVM_DIR }}
          key: llvm-build-${{ steps.llvm-cache-key.outputs.sha }}-assert-${{ env.PTO_DSL_ST_PYTHON_TAG }}-v2

      - name: Install PTODSL PTOAS editable
        shell: bash
        run: |
          set -euo pipefail
          rm -rf "${PTO_DSL_ST_BUILD_DIR}"
          export CC=gcc
          export CXX=g++
          LLVM_BUILD_DIR="${PTO_DSL_ST_LLVM_DIR}" \
            "${PTO_DSL_ST_PYTHON_BIN}" -m pip install --editable . \
              --no-build-isolation --no-deps \
              --config-settings="build-dir=${PTO_DSL_ST_BUILD_DIR}"

      - name: Probe PTODSL runtime Python
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p "${TILELIB_ST_WORKSPACE}"
          export TORCH_DEVICE_BACKEND_AUTOLOAD=0
          source "${ASCEND_HOME_PATH}/bin/setenv.bash"

          probe_ptodsl_python() {
            "${PTO_DSL_ST_PYTHON_BIN}" - <<'PY'
          import sys
          import ptoas.mlir.ir  # noqa: F401
          from ptodsl import pto  # noqa: F401
          from ptoas.mlir.dialects import pto as _pto  # noqa: F401

          print(sys.executable)
          print("ptodsl/ptoas.mlir import ok")
          PY
          }

          probe_torch_npu_python() {
            "${PTO_DSL_ST_PYTHON_BIN}" - <<'PY'
          import numpy
          import torch
          import torch_npu  # noqa: F401

          print("torch", torch.__version__)
          print("torch_npu", getattr(torch_npu, "__version__", "unknown"))
          print("numpy", numpy.__version__)
          PY
          }

          PROBE_LOG="${TILELIB_ST_WORKSPACE}/ptodsl-python-probe.log"
          : > "${PROBE_LOG}"
          echo "Probing PTODSL DSL ST Python: ${PTO_DSL_ST_PYTHON_BIN}" | tee -a "${PROBE_LOG}"
          echo "Probing PTODSL DSL ST ptoas: ${PTO_DSL_ST_PTOAS_BIN}" | tee -a "${PROBE_LOG}"
          if [[ ! -x "${PTO_DSL_ST_PTOAS_BIN}" ]]; then
            echo "ERROR: installed PTODSL ptoas entry is missing: ${PTO_DSL_ST_PTOAS_BIN}" | tee -a "${PROBE_LOG}" >&2
            exit 1
          fi
          if ! probe_ptodsl_python >> "${PROBE_LOG}" 2>&1; then
            cat "${PROBE_LOG}" >&2
            echo "ERROR: selected PTODSL Python cannot import ptodsl and MLIR PTO bindings." >&2
            exit 1
          fi
          if ! probe_torch_npu_python >> "${PROBE_LOG}" 2>&1; then
            cat "${PROBE_LOG}" >&2
            echo "ERROR: selected PTODSL Python cannot import torch, torch_npu, and numpy." >&2
            exit 1
          fi
          "${PTO_DSL_ST_PTOAS_BIN}" --version >> "${PROBE_LOG}" 2>&1

          cat "${PROBE_LOG}"

      - name: Run VPTO SIM validation
        if: ${{ true }}
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p "${VPTO_SIM_WORKSPACE}"
          export PYTHON_BIN="${PTO_DSL_ST_PYTHON_BIN}"
          export PTO_PYTHON_BIN="${PTO_DSL_ST_PYTHON_BIN}"
          export PATH="${PTO_DSL_ST_BIN_DIR}:${PATH}"
          export TORCH_DEVICE_BACKEND_AUTOLOAD=0
          source "${ASCEND_HOME_PATH}/bin/setenv.bash"

          WORK_SPACE="${VPTO_SIM_WORKSPACE}" \
          ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \
          PTOAS_BIN="${PTO_DSL_ST_PTOAS_BIN}" \
          DEVICE=SIM \
          JOBS="${JOBS:-32}" \
          bash test/vpto/scripts/run_host_vpto_validation_parallel.sh

      - name: Run TileLib ST CI
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p "${TILELIB_ST_WORKSPACE}"
          export PYTHON_BIN="${PTO_DSL_ST_PYTHON_BIN}"
          export PTO_PYTHON_BIN="${PTO_DSL_ST_PYTHON_BIN}"
          export PATH="${PTO_DSL_ST_BIN_DIR}:${PATH}"
          export TORCH_DEVICE_BACKEND_AUTOLOAD=0
          source "${ASCEND_HOME_PATH}/bin/setenv.bash"

          TILELIB_ST_OUTPUT_ROOT="${TILELIB_ST_WORKSPACE}/tilelib-st-a5" \
          ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \
          PTOAS_BIN="${PTO_DSL_ST_PTOAS_BIN}" \
          scripts/sim_dsl.sh test/tilelib-st/run_tilelib_st.py -- test/tilelib-st/a5 \
            2>&1 | tee "${TILELIB_ST_WORKSPACE}/tilelib-st-a5.log"

      - name: Run PTODSL ST CI
        shell: bash
        run: |
          set -euo pipefail
          mkdir -p "${TILELIB_ST_WORKSPACE}"
          export PYTHON_BIN="${PTO_DSL_ST_PYTHON_BIN}"
          export PTO_PYTHON_BIN="${PTO_DSL_ST_PYTHON_BIN}"
          export PATH="${PTO_DSL_ST_BIN_DIR}:${PATH}"
          export TORCH_DEVICE_BACKEND_AUTOLOAD=0
          source "${ASCEND_HOME_PATH}/bin/setenv.bash"

          ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \
          PTOAS_BIN="${PTO_DSL_ST_PTOAS_BIN}" \
          scripts/sim_dsl.sh test/dsl-st \
            2>&1 | tee "${TILELIB_ST_WORKSPACE}/ptodsl-dsl-st.log"

      - name: Upload TileLib ST logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: tilelib-st-ci-${{ github.run_id }}
          path: |
            ${{ env.TILELIB_ST_WORKSPACE }}/run_ci.log
            ${{ env.TILELIB_ST_WORKSPACE }}/ptodsl-dsl-st.log
            ${{ env.TILELIB_ST_WORKSPACE }}/tilelib-st-a5.log
            ${{ env.TILELIB_ST_WORKSPACE }}/tilelib-st-a5/**
            ${{ env.TILELIB_ST_WORKSPACE }}/ptodsl-python-probe.log
          if-no-files-found: warn

      - name: Upload VPTO SIM logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: vpto-sim-validation-${{ github.run_id }}
          path: |
            ${{ env.VPTO_SIM_WORKSPACE }}/parallel-runner.log
            ${{ env.VPTO_SIM_WORKSPACE }}/parallel-summary.tsv
          if-no-files-found: warn

      - name: Upload PyPTO SIM logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: pypto-sim-smoke-${{ github.run_id }}
          path: |
            ${{ env.PYPTO_RUN_WORKSPACE }}/*.log
          if-no-files-found: warn