From 9059a60fa931fb4567d971362a51597c65f44725 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Wed, 22 Jul 2026 16:26:56 +0800 Subject: [PATCH 01/32] Refactor Python packaging around scikit-build-core --- .github/workflows/build_wheel.yml | 28 +- .github/workflows/build_wheel_mac.yml | 35 +- .github/workflows/ci.yml | 15 +- .github/workflows/ci_sim.yml | 14 +- CMakeLists.txt | 51 ++ _ptoas_build_backend.py | 375 -------------- docker/Dockerfile | 27 +- docker/collect_ptoas_dist.sh | 88 ++-- docker/collect_ptoas_dist_mac.sh | 77 ++- docker/copy_ptoas_deps.sh | 148 ------ docker/create_wheel.sh | 309 ------------ docker/setup.py | 54 --- docker/setup_mac.py | 64 --- docker/test_wheel_imports.sh | 28 +- docker/validate_wheel_payload.py | 22 +- docs/designs/ptoas-python-launcher-layout.md | 154 +++--- lib/Bindings/Python/CMakeLists.txt | 2 + packaging/ptoas-vmi/pyproject.toml | 60 +++ ptodsl/CMakeLists.txt | 21 +- ptodsl/ptoas/__init__.py | 3 +- ptodsl/ptoas/_cli.py | 100 ++++ ptodsl/ptoas/_launcher.py | 21 - ptodsl/ptoas/_runtime_entry.py | 144 ------ ptodsl/ptoas_wheel_bootstrap.py | 190 -------- ptodsl/tests/CMakeLists.txt | 1 + ptodsl/tests/test_docs_as_test.py | 18 +- .../tests/test_pipe_surface_sample_compile.py | 19 +- ptodsl/tests/test_ptoas_cli.py | 119 +++++ ptodsl/tests/test_ptoas_frontend_verify.py | 18 +- ptodsl/tests/test_ptoas_launcher_shim.py | 45 -- ptodsl/tests/test_ptoas_runtime_entry.py | 249 ---------- ptodsl/tests/test_ptoas_tree_wrapper.py | 459 +++--------------- ptodsl/tests/test_ptoas_wheel_bootstrap.py | 319 ------------ pyproject.toml | 59 ++- quick_install.sh | 37 +- test/python/test_docker_runtime_packaging.py | 69 +-- test/python/test_ptoas_build_backend.py | 126 ----- test/python/test_scikit_build_config.py | 120 +++++ test/python/test_validate_wheel_payload.py | 82 ++-- tools/ptoas/CMakeLists.txt | 90 ++-- tools/ptoas/NativeModule.cpp | 37 ++ tools/ptoas/driver.cpp | 8 +- tools/ptoas/ptoas.h | 4 +- tools/ptoas/ptoas_wrapper.py | 118 +---- 44 files changed, 964 insertions(+), 3063 deletions(-) delete mode 100644 _ptoas_build_backend.py delete mode 100644 docker/copy_ptoas_deps.sh delete mode 100755 docker/create_wheel.sh delete mode 100644 docker/setup.py delete mode 100644 docker/setup_mac.py create mode 100644 packaging/ptoas-vmi/pyproject.toml create mode 100644 ptodsl/ptoas/_cli.py delete mode 100644 ptodsl/ptoas/_launcher.py delete mode 100644 ptodsl/ptoas/_runtime_entry.py delete mode 100644 ptodsl/ptoas_wheel_bootstrap.py create mode 100644 ptodsl/tests/test_ptoas_cli.py delete mode 100644 ptodsl/tests/test_ptoas_launcher_shim.py delete mode 100644 ptodsl/tests/test_ptoas_runtime_entry.py delete mode 100644 ptodsl/tests/test_ptoas_wheel_bootstrap.py delete mode 100644 test/python/test_ptoas_build_backend.py create mode 100644 test/python/test_scikit_build_config.py create mode 100644 tools/ptoas/NativeModule.cpp diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 98a8ac0c9f..02e776ac42 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -130,12 +130,13 @@ jobs: export PATH="${PY_PATH}/bin:$PATH" # LLVM/MLIR Python bindings are not yet compatible with pybind11 3.x # (see pybind11 static_assert on def_property + keep_alive). - pip install --no-cache-dir numpy 'pybind11<3' nanobind setuptools wheel auditwheel + pip install --no-cache-dir numpy 'pybind11<3' nanobind 'scikit-build-core>=0.12.2,<2' auditwheel - - name: Test PEP 517 build backend + - name: Validate PEP 517 project metadata run: | export PATH="${PY_PATH}/bin:$PATH" - PYTHONPATH="$GITHUB_WORKSPACE" python test/python/test_ptoas_build_backend.py + python -m scikit_build_core.build project-table + (cd packaging/ptoas-vmi && python -m scikit_build_core.build project-table) - name: Set build directories run: | @@ -215,16 +216,19 @@ jobs: CMAKE_BUILD_PARALLEL_LEVEL: "2" run: | export PATH="${PY_PATH}/bin:$PATH" + if [ "${PTOAS_PYTHON_PACKAGE_NAME}" = "ptoas-vmi" ]; then + PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}/packaging/ptoas-vmi" + else + PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}" + fi + rm -rf "${PTO_SOURCE_DIR}/build/wheel-dist" + mkdir -p "${PTO_SOURCE_DIR}/build/wheel-dist" PTOAS_RELEASE_VERSION_OVERRIDE="${PTOAS_CLI_VERSION}" \ - pip install . --no-build-isolation - - - name: Create Python wheel - if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - run: | - export PATH="${PY_PATH}/bin:$PATH" - export PTOAS_PYTHON_PACKAGE_NAME="${PTOAS_PYTHON_PACKAGE_NAME}" - export PTOAS_PYTHON_PACKAGE_VERSION="${PTOAS_VERSION}" - bash $PTO_SOURCE_DIR/docker/create_wheel.sh + SKBUILD_BUILD_DIR="${PTO_BUILD_DIR}" \ + python -m pip wheel "${PTOAS_WHEEL_SOURCE_DIR}" \ + --no-build-isolation --no-deps \ + --wheel-dir "${PTO_SOURCE_DIR}/build/wheel-dist" + cmake --install "${PTO_BUILD_DIR}" --prefix "${PTO_INSTALL_DIR}" - name: Validate wheel payload and launcher contract if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' diff --git a/.github/workflows/build_wheel_mac.yml b/.github/workflows/build_wheel_mac.yml index 9948a32759..a473cc9c4c 100644 --- a/.github/workflows/build_wheel_mac.yml +++ b/.github/workflows/build_wheel_mac.yml @@ -133,7 +133,7 @@ jobs: run: | # LLVM/MLIR Python bindings are not yet compatible with pybind11 3.x # (see pybind11 static_assert on def_property + keep_alive). - pip install --no-cache-dir numpy 'pybind11<3' nanobind setuptools wheel delocate + pip install --no-cache-dir numpy 'pybind11<3' nanobind 'scikit-build-core>=0.12.2,<2' delocate - name: Set build directories run: | @@ -209,26 +209,27 @@ jobs: - name: Build PTOAS run: | - PTOAS_RELEASE_VERSION_OVERRIDE="${PTOAS_CLI_VERSION}" \ - pip install . --no-build-isolation - - - name: Create Python wheel - if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - run: | - export PTO_WHEEL_DIST_DIR=$PTO_SOURCE_DIR/build/wheel-dist - export PTOAS_PYTHON_PACKAGE_NAME="${PTOAS_PYTHON_PACKAGE_NAME}" - export PTOAS_PYTHON_PACKAGE_VERSION="${PTOAS_VERSION}" + if [ "${PTOAS_PYTHON_PACKAGE_NAME}" = "ptoas-vmi" ]; then + PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}/packaging/ptoas-vmi" + else + PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}" + fi if [ "${{ matrix.arch }}" = "x86_64" ]; then TARGET_ARCH="x86_64" else TARGET_ARCH="arm64" fi - # setup-python may provide a universal2 interpreter on macOS. - # Force a single-arch wheel tag to match the matrix target and delocate checks. - export WHEEL_PLAT_NAME=$(python "$PTO_SOURCE_DIR/docker/get_macos_wheel_plat_name.py" "$TARGET_ARCH") - bash $PTO_SOURCE_DIR/docker/create_wheel.sh + export _PYTHON_HOST_PLATFORM="$(python "$PTO_SOURCE_DIR/docker/get_macos_wheel_plat_name.py" "$TARGET_ARCH")" + rm -rf "${PTO_SOURCE_DIR}/build/wheel-dist" + mkdir -p "${PTO_SOURCE_DIR}/build/wheel-dist" + PTOAS_RELEASE_VERSION_OVERRIDE="${PTOAS_CLI_VERSION}" \ + SKBUILD_BUILD_DIR="${PTO_BUILD_DIR}" \ + python -m pip wheel "${PTOAS_WHEEL_SOURCE_DIR}" \ + --no-build-isolation --no-deps \ + --wheel-dir "${PTO_SOURCE_DIR}/build/wheel-dist" + cmake --install "${PTO_BUILD_DIR}" --prefix "${PTO_INSTALL_DIR}" shopt -s nullglob - built_wheels=("$PTO_WHEEL_DIST_DIR"/ptoas*.whl) + built_wheels=("$PTO_SOURCE_DIR"/build/wheel-dist/ptoas*.whl) if [ "${#built_wheels[@]}" -gt 0 ]; then printf 'Built wheel file: %s\n' "$(basename "${built_wheels[0]}")" fi @@ -354,8 +355,8 @@ jobs: tar -xzf "$GITHUB_WORKSPACE/ptoas-bin-macos-${{ matrix.arch }}.tar.gz" -C "$TEST_DIR/extracted" test -x "$TEST_DIR/extracted/bin/ptoas" - test -f "$TEST_DIR/extracted/ptoas/_runtime_entry.py" - test -f "$TEST_DIR/extracted/ptoas_wheel_bootstrap.py" + test -f "$TEST_DIR/extracted/ptoas/_cli.py" + compgen -G "$TEST_DIR/extracted/ptoas/_native*.so" >/dev/null env -u PYTHONPATH -u DYLD_LIBRARY_PATH -u LD_LIBRARY_PATH \ "$TEST_DIR/extracted/bin/ptoas" --version diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad780e9c0a..8e3eb1f92a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -133,7 +133,7 @@ jobs: libedit-dev zlib1g-dev libxml2-dev libzstd-dev python3 -m pip install --upgrade pip # LLVM/MLIR Python bindings are not yet compatible with pybind11 3.x. - python3 -m pip install setuptools wheel 'pybind11<3' nanobind numpy + python3 -m pip install 'scikit-build-core>=0.12.2,<2' 'pybind11<3' nanobind numpy - name: Define payload paths shell: bash @@ -269,7 +269,7 @@ jobs: rm -rf "${PTOAS_VENV}" python3 -m venv "${PTOAS_VENV}" "${PTOAS_VENV}/bin/python" -m pip install --upgrade pip - "${PTOAS_VENV}/bin/pip" install setuptools wheel 'pybind11<3' nanobind numpy ml-dtypes + "${PTOAS_VENV}/bin/pip" install 'scikit-build-core>=0.12.2,<2' 'pybind11<3' nanobind numpy ml-dtypes - name: Build PTOAS wheel run: | @@ -278,13 +278,14 @@ jobs: rm -rf "${PTO_INSTALL_DIR}" rm -rf "${PTO_WHEELHOUSE}" mkdir -p "${PTO_WHEELHOUSE}" - # LLVM_BUILD_DIR is the env var read by the build backend (_ptoas_build_backend.py). + # scikit-build-core reuses this build tree so the following CTest and + # lit steps exercise the exact sources used to produce the wheel. LLVM_BUILD_DIR="${LLVM_DIR}" \ - PTO_BUILD_DIR="${PTO_BUILD_DIR}" \ - CMAKE_C_COMPILER="${PTOAS_CMAKE_C_COMPILER}" \ - CMAKE_CXX_COMPILER="${PTOAS_CMAKE_CXX_COMPILER}" \ - PTO_INSTALL_DIR="${PTO_INSTALL_DIR}" \ + SKBUILD_BUILD_DIR="${PTO_BUILD_DIR}" \ + CC="${PTOAS_CMAKE_C_COMPILER}" \ + CXX="${PTOAS_CMAKE_CXX_COMPILER}" \ "${PTOAS_VENV}/bin/python" -m pip wheel . --no-build-isolation --no-deps --wheel-dir "${PTO_WHEELHOUSE}" + cmake --install "${PTO_BUILD_DIR}" --prefix "${PTO_INSTALL_DIR}" - name: Install PTOAS wheel into test venv shell: bash diff --git a/.github/workflows/ci_sim.yml b/.github/workflows/ci_sim.yml index 795e077919..000446d8af 100644 --- a/.github/workflows/ci_sim.yml +++ b/.github/workflows/ci_sim.yml @@ -191,14 +191,14 @@ jobs: need_pip_install=0 python3 -c "import numpy" >/dev/null 2>&1 || need_pip_install=1 - python3 -c "import setuptools, wheel" >/dev/null 2>&1 || need_pip_install=1 + python3 -c "import scikit_build_core" >/dev/null 2>&1 || need_pip_install=1 python3 -m pybind11 --cmakedir >/dev/null 2>&1 || need_pip_install=1 python3 -m nanobind --cmake_dir >/dev/null 2>&1 || need_pip_install=1 python3 -c "import ml_dtypes" >/dev/null 2>&1 || need_pip_install=1 if [[ "${need_pip_install}" -eq 1 ]]; then python3 -m pip install --upgrade pip - python3 -m pip install setuptools wheel 'pybind11<3' nanobind numpy ml-dtypes + python3 -m pip install 'scikit-build-core>=0.12.2,<2' 'pybind11<3' nanobind numpy ml-dtypes fi - name: Resolve LLVM directories @@ -323,11 +323,10 @@ jobs: # Pin system compilers for the same reason as LLVM above. export CC=gcc export CXX=g++ - # LLVM_BUILD_DIR / PTO_INSTALL_DIR are read by the build backend - # while producing the wheel payload and staging native runtime bits. LLVM_BUILD_DIR="${LLVM_DIR}" \ - PTO_INSTALL_DIR="${PTO_INSTALL_DIR}" \ + SKBUILD_BUILD_DIR="${GITHUB_WORKSPACE}/build" \ python3 -m pip wheel . --no-build-isolation --no-deps --wheel-dir "${PTO_WHEELHOUSE}" + cmake --install "${GITHUB_WORKSPACE}/build" --prefix "${PTO_INSTALL_DIR}" - name: Create PTOAS simulator venv shell: bash @@ -609,6 +608,7 @@ jobs: deps = [ ("setuptools", "setuptools"), ("wheel", "wheel"), + ("scikit_build_core", "scikit-build-core>=0.12.2,<2"), ("nanobind", "nanobind"), ("numpy", "numpy"), ("ml_dtypes", "ml-dtypes"), @@ -795,9 +795,9 @@ jobs: export CC=gcc export CXX=g++ LLVM_BUILD_DIR="${PTO_DSL_ST_LLVM_DIR}" \ - PTO_BUILD_DIR="${PTO_DSL_ST_BUILD_DIR}" \ - PTO_INSTALL_DIR="${PTO_DSL_ST_INSTALL_DIR}" \ + SKBUILD_BUILD_DIR="${PTO_DSL_ST_BUILD_DIR}" \ "${PTO_DSL_ST_PYTHON_BIN}" -m pip wheel . --no-build-isolation --no-deps --wheel-dir "${PTO_DSL_ST_WHEELHOUSE}" + cmake --install "${PTO_DSL_ST_BUILD_DIR}" --prefix "${PTO_DSL_ST_INSTALL_DIR}" - name: Install PTODSL PTOAS wheel shell: bash diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ad8933b5a..2a195bb99c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,11 +76,42 @@ function(ptoas_preseed_compilers_from_llvm_cache) endif() endfunction() +# Standard Python build backends pass environment variables through to CMake +# but do not synthesize project-specific cache arguments. Accept the existing +# LLVM_BUILD_DIR convention directly so `pip wheel .` and `pip install .` can +# use the same configured LLVM tree as developer CMake builds. +if((NOT DEFINED LLVM_DIR OR LLVM_DIR STREQUAL "") + AND DEFINED ENV{LLVM_BUILD_DIR} + AND NOT "$ENV{LLVM_BUILD_DIR}" STREQUAL "") + set(LLVM_DIR "$ENV{LLVM_BUILD_DIR}/lib/cmake/llvm" CACHE PATH + "LLVM package directory derived from LLVM_BUILD_DIR") +endif() +if((NOT DEFINED MLIR_DIR OR MLIR_DIR STREQUAL "") + AND DEFINED ENV{LLVM_BUILD_DIR} + AND NOT "$ENV{LLVM_BUILD_DIR}" STREQUAL "") + set(MLIR_DIR "$ENV{LLVM_BUILD_DIR}/lib/cmake/mlir" CACHE PATH + "MLIR package directory derived from LLVM_BUILD_DIR") +endif() +if((NOT DEFINED MLIR_PYTHON_PACKAGE_DIR + OR MLIR_PYTHON_PACKAGE_DIR STREQUAL "") + AND DEFINED ENV{LLVM_BUILD_DIR} + AND NOT "$ENV{LLVM_BUILD_DIR}" STREQUAL "") + set(MLIR_PYTHON_PACKAGE_DIR + "$ENV{LLVM_BUILD_DIR}/tools/mlir/python_packages/mlir_core" + CACHE PATH "MLIR Python package derived from LLVM_BUILD_DIR") +endif() + ptoas_preseed_compilers_from_llvm_cache() project(ptoas VERSION 0.55) set(PTOAS_VMI_VERSION "0.1.4") +# Wheel builds carry the same Linux hardening policy as release CMake builds. +# Including the cache settings here keeps the policy backend-independent. +if(SKBUILD AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/LinuxHardeningCache.cmake") +endif() + option(PTOAS_VALIDATE_CMAKE4_FETCHCONTENT_COMPAT "Validate CMake 4 compatibility for legacy FetchContent users" OFF) if(PTOAS_VALIDATE_CMAKE4_FETCHCONTENT_COMPAT) @@ -244,6 +275,25 @@ if(PTO_ENABLE_PYTHON_BINDING) find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) find_package(pybind11 CONFIG REQUIRED) + # A wheel must include the base MLIR Python package in addition to PTOAS's + # generated dialect modules. Normal CMake installs continue to use the LLVM + # installation supplied by the user and therefore do not duplicate it. + if(SKBUILD) + if(NOT MLIR_PYTHON_PACKAGE_DIR + OR NOT EXISTS "${MLIR_PYTHON_PACKAGE_DIR}/mlir") + message(FATAL_ERROR + "MLIR_PYTHON_PACKAGE_DIR must contain the MLIR Python package when " + "building a PTOAS wheel") + endif() + install( + DIRECTORY "${MLIR_PYTHON_PACKAGE_DIR}/mlir" + DESTINATION "." + COMPONENT PTOAS_Python + PATTERN "__pycache__" EXCLUDE + PATTERN "*.pyc" EXCLUDE + ) + endif() + # 设置 Python 扩展的输出目录，方便调试 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/python/pto) @@ -282,6 +332,7 @@ if(BUILD_TESTING) "PYTHONPATH=${_pto_python_test_pythonpath}" "LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:${_llvm_root}/lib:${CMAKE_INSTALL_PREFIX}/lib:$ENV{LD_LIBRARY_PATH}" "PATH=${CMAKE_BINARY_DIR}/tools/ptoas:${CMAKE_INSTALL_PREFIX}/bin:${_pto_python_bin_dir}:$ENV{PATH}" + "PTOAS_BIN=${CMAKE_BINARY_DIR}/tools/ptoas/ptoas" ) add_subdirectory(ptodsl/tests) diff --git a/_ptoas_build_backend.py b/_ptoas_build_backend.py deleted file mode 100644 index 1dfea891db..0000000000 --- a/_ptoas_build_backend.py +++ /dev/null @@ -1,375 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -""" -PEP 517 build backend for ptoas. - -Runs the CMake build (assuming LLVM is already built), then delegates -wheel packaging to docker/create_wheel.sh. - -Environment variables (all optional): - LLVM_BUILD_DIR Path to LLVM build dir - (default: /llvm-workspace/llvm-project/build-shared) - PTO_BUILD_DIR Path to PTOAS build dir (default: /build) - PTO_INSTALL_DIR Install prefix (default: /install) - CMAKE_C_COMPILER Optional C compiler passed through to CMake - CMAKE_CXX_COMPILER Optional C++ compiler passed through to CMake - PTOAS_PYTHON_PACKAGE_NAME Wheel project-name override - PTOAS_PYTHON_PACKAGE_VERSION Wheel version override -""" -from __future__ import annotations - -import base64 -import glob -import hashlib -import io -import os -import re -import shutil -import subprocess -import sys -import zipfile -from pathlib import Path - -_REPO = Path(__file__).parent.resolve() - -def _find_llvm_dir(): - """Return an LLVM install or build-tree prefix, resolving in order: - - 1. ``LLVM_BUILD_DIR`` / ``LLVM_INSTALL_DIR`` env vars - 2. Auto-detect common install locations by probing ``lib/cmake/llvm`` - 3. Default build-tree path - """ - for key in ("LLVM_BUILD_DIR", "LLVM_INSTALL_DIR"): - if key in os.environ: - return Path(os.environ[key]) - - for cand in ("/usr/local/llvm", "/usr/local/Ascend/latest/compiler", - "/opt/llvm"): - if (Path(cand) / "lib" / "cmake" / "llvm").is_dir(): - return Path(cand) - - return Path("/llvm-workspace/llvm-project/build-shared") - - -_LLVM_BUILD_DIR = _find_llvm_dir() -_PTO_INSTALL_DIR = Path( - os.environ.get("PTO_INSTALL_DIR", str(_REPO / "install")) -) -_BUILD_DIR = Path(os.environ.get("PTO_BUILD_DIR", str(_REPO / "build"))) -_PTODSL_SOURCE_ROOT = _REPO / "ptodsl" -_MLIR_PY_PKG = None -if "MLIR_PYTHON_PACKAGE_DIR" in os.environ: - _MLIR_PY_PKG = Path(os.environ["MLIR_PYTHON_PACKAGE_DIR"]) -elif "LLVM_INSTALL_DIR" in os.environ: - _MLIR_PY_PKG = Path(os.environ["LLVM_INSTALL_DIR"]) / "python_packages" / "mlir_core" -else: - _installed = _LLVM_BUILD_DIR / "python_packages" / "mlir_core" - _build_tree = _LLVM_BUILD_DIR / "tools" / "mlir" / "python_packages" / "mlir_core" - _MLIR_PY_PKG = _installed if _installed.exists() else _build_tree -_WHEEL_DIST_DIR = _BUILD_DIR / "wheel-dist" -_PROJECT_VERSION_RE = re.compile( - r"project\s*\(\s*ptoas\s+VERSION\s+([0-9]+\.[0-9]+)\s*\)" -) -_PACKAGE_NAME_RE = re.compile( - r"^[A-Za-z0-9]+(?:[-_.][A-Za-z0-9]+)*$" -) - - -def _default_ptoas_package_name() -> str: - package_name = os.environ.get("PTOAS_PYTHON_PACKAGE_NAME", "").strip() - if not package_name: - return "ptoas" - if not _PACKAGE_NAME_RE.fullmatch(package_name): - raise RuntimeError( - "invalid PTOAS Python package name override " - f"{package_name!r}; expected a normalized project name" - ) - return package_name - - -def _wheel_distribution_name(package_name: str) -> str: - return re.sub(r"[-_.]+", "_", package_name) - - -def _assert_installed_ptodsl_payload() -> None: - """Fail fast if the root install tree did not stage the PTODSL package.""" - installed_init = _PTO_INSTALL_DIR / "ptodsl" / "__init__.py" - if installed_init.exists(): - return - raise RuntimeError( - "PTODSL is missing from the PTOAS install tree. " - f"Expected to find {installed_init}. " - "Root CMake install rules must stage the public 'ptodsl' package " - "before wheel assembly or installed-environment validation." - ) - - -def _assert_installed_ptoas_shared_module() -> None: - installed_shared_module = _PTO_INSTALL_DIR / "lib" / "ptoas.so" - if ( - installed_shared_module.is_file() - and installed_shared_module.stat().st_size > 0 - ): - return - raise RuntimeError( - "PTOAS shared launcher module is missing or empty in the PTOAS install tree. " - f"Expected to find {installed_shared_module}. " - "Wheel assembly now packages the shared module from the install tree." - ) - - -def _assert_editable_ptodsl_source() -> None: - """Fail fast if the editable install cannot point at the PTODSL source.""" - source_init = _PTODSL_SOURCE_ROOT / "ptodsl" / "__init__.py" - if source_init.exists(): - return - raise RuntimeError( - "PTODSL editable source is missing from the repository checkout. " - f"Expected to find {source_init}. " - "Root editable installs must expose the in-repo PTODSL sources." - ) - - -def _default_ptoas_version() -> str: - version = os.environ.get("PTOAS_PYTHON_PACKAGE_VERSION", "").strip() - if version: - return version - cmake_file = _REPO / "CMakeLists.txt" - match = _PROJECT_VERSION_RE.search(cmake_file.read_text(encoding="utf-8")) - if not match: - raise RuntimeError(f"could not find PTOAS version in {cmake_file}") - return match.group(1) - - -def get_requires_for_build_wheel(config_settings=None): - return ["setuptools>=68", "wheel", "pybind11<3"] - - -def get_requires_for_build_editable(config_settings=None): - return ["setuptools>=68", "wheel", "pybind11<3"] - - -def get_requires_for_build_sdist(config_settings=None): - return [] - - -def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None): - """Return wheel metadata without running the full build.""" - import email.message - - package_name = _default_ptoas_package_name() - dist_name = _wheel_distribution_name(package_name) - version = _default_ptoas_version() - dist_info = Path(metadata_directory) / f"{dist_name}-{version}.dist-info" - dist_info.mkdir(parents=True, exist_ok=True) - - meta = email.message.Message() - meta["Metadata-Version"] = "2.1" - meta["Name"] = package_name - meta["Version"] = version - meta["Summary"] = "PTO Assembler & Optimizer" - meta["Requires-Python"] = ">=3.9" - meta["License"] = "Apache-2.0" - meta["Requires-Dist"] = "numpy" - (dist_info / "METADATA").write_text(str(meta)) - (dist_info / "WHEEL").write_text( - "Wheel-Version: 1.0\nGenerator: _ptoas_build_backend\n" - "Root-Is-Purelib: True\nTag: py3-none-any\n" - ) - return dist_info.name - - -prepare_metadata_for_build_editable = prepare_metadata_for_build_wheel - - -def build_sdist(sdist_directory, config_settings=None): - raise NotImplementedError( - "ptoas does not support sdist. Use `pip install .` to build a wheel." - ) - - -def _should_use_linux_hardening_cache() -> bool: - return sys.platform.startswith("linux") - - -def _cmake_configure_and_build(skip_install=False): - """CMake configure + build + install.""" - _BUILD_DIR.mkdir(parents=True, exist_ok=True) - - pybind11_dir = subprocess.check_output( - [sys.executable, "-m", "pybind11", "--cmakedir"], text=True - ).strip() - - cmake_cmd = [ - "cmake", "-GNinja", - f"-S{_REPO}", f"-B{_BUILD_DIR}", - "-DCMAKE_BUILD_TYPE=Release", - "-DPTO_ENABLE_PYTHON_BINDING=ON", - f"-DLLVM_DIR={_LLVM_BUILD_DIR}/lib/cmake/llvm", - f"-DMLIR_DIR={_LLVM_BUILD_DIR}/lib/cmake/mlir", - f"-DPython3_ROOT_DIR={sys.prefix}", - f"-DPython3_EXECUTABLE={sys.executable}", - "-DPython3_FIND_STRATEGY=LOCATION", - f"-Dpybind11_DIR={pybind11_dir}", - f"-DMLIR_PYTHON_PACKAGE_DIR={_MLIR_PY_PKG}", - f"-DCMAKE_INSTALL_PREFIX={_PTO_INSTALL_DIR}", - ] - - for compiler_var in ("CMAKE_C_COMPILER", "CMAKE_CXX_COMPILER"): - compiler = os.environ.get(compiler_var, "").strip() - if compiler: - cmake_cmd.append(f"-D{compiler_var}={compiler}") - - release_version = os.environ.get("PTOAS_RELEASE_VERSION_OVERRIDE", "") - if release_version: - cmake_cmd.append(f"-DPTOAS_RELEASE_VERSION_OVERRIDE={release_version}") - - hardening_cache = _REPO / "cmake" / "LinuxHardeningCache.cmake" - if _should_use_linux_hardening_cache() and hardening_cache.exists(): - cmake_cmd.insert(1, f"-C{hardening_cache}") - - subprocess.check_call(cmake_cmd) - subprocess.check_call(["cmake", "--build", str(_BUILD_DIR)]) - if not skip_install: - subprocess.check_call( - ["cmake", "--build", str(_BUILD_DIR), "--target", "install"] - ) - _assert_installed_ptodsl_payload() - _assert_installed_ptoas_shared_module() - - -def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): - _cmake_configure_and_build() - - package_name = _default_ptoas_package_name() - dist_name = _wheel_distribution_name(package_name) - env = os.environ.copy() - env.update({ - "PTO_SOURCE_DIR": str(_REPO), - "PTO_INSTALL_DIR": str(_PTO_INSTALL_DIR), - "PTO_BUILD_DIR": str(_BUILD_DIR), - "LLVM_BUILD_DIR": str(_LLVM_BUILD_DIR), - "PTO_WHEEL_DIST_DIR": str(_WHEEL_DIST_DIR), - # Keep wheel packaging on the same interpreter pip used to invoke the - # backend. This avoids drifting to a different `python3` on PATH in - # conda/self-hosted CI environments. - "PYTHON": sys.executable, - }) - subprocess.check_call( - ["bash", str(_REPO / "docker" / "create_wheel.sh")], - env=env, - ) - - wheels = sorted( - glob.glob(str(_WHEEL_DIST_DIR / f"{dist_name}-*.whl")), - key=os.path.getmtime, - ) - if not wheels: - raise RuntimeError( - f"No {dist_name}-*.whl found in {_WHEEL_DIST_DIR} after build." - ) - - wheel_path = Path(wheels[-1]) - dest = Path(wheel_directory) / wheel_path.name - shutil.copy2(wheel_path, dest) - return dest.name - - -def build_editable(wheel_directory, config_settings=None, metadata_directory=None): - """PEP 660 editable install. - - Builds the C++ extensions in-place, then produces a minimal wheel that - installs a .pth file pointing sys.path at the local PTODSL sources plus - the installed/runtime build outputs. No files are copied into - site-packages except the .pth file itself. - """ - _cmake_configure_and_build(skip_install=True) - _assert_editable_ptodsl_source() - - package_name = _default_ptoas_package_name() - dist_name = _wheel_distribution_name(package_name) - version = _default_ptoas_version() - - # Paths that must be on sys.path for the package to be importable - pth_paths = [ - # mlir.* namespace + _pto.so (installed there by CMake) - str(_MLIR_PY_PKG), - # Prefer the repository PTODSL sources so editable installs pick up - # local Python edits instead of staged/install-tree copies. - str(_PTODSL_SOURCE_ROOT), - # Handwritten Python sources (pto/dialects/pto.py, etc.). - str(_REPO / "python"), - # Installed PTOAS runtime overlay (TileOps/resources when present). - str(_PTO_INSTALL_DIR), - # Generated files (_pto.so, _pto_ops_gen.py) under mlir/ namespace. - str(_BUILD_DIR / "python"), - ] - - # Use executable .pth content so editable installs can deterministically - # prepend the repository PTODSL sources ahead of pre-existing build-tree - # entries in the caller's Python environment. - pth_content = ( - "import sys; " - f"pth_paths = {pth_paths!r}; " - "[sys.path.remove(path) for path in pth_paths if path in sys.path]; " - "sys.path[:0] = pth_paths\n" - ) - pth_filename = f"{dist_name}-editable.pth" - - # ---- Build the editable wheel (a zip with .pth + dist-info) ---- - tag = f"py3-none-any" - wheel_name = f"{dist_name}-{version}-{tag}.whl" - wheel_path = Path(wheel_directory) / wheel_name - - dist_info_dir = f"{dist_name}-{version}.dist-info" - - def _sha256_record(data: bytes) -> str: - digest = hashlib.sha256(data).digest() - b64 = base64.urlsafe_b64encode(digest).rstrip(b"=").decode() - return f"sha256={b64}" - - pth_bytes = pth_content.encode() - wheel_meta = ( - "Wheel-Version: 1.0\n" - "Generator: _ptoas_build_backend\n" - "Root-Is-Purelib: True\n" - f"Tag: {tag}\n" - "Build: editable\n" - ).encode() - entry_points = ( - "[console_scripts]\n" - "ptoas=ptoas_wheel_bootstrap:main\n" - ).encode() - metadata_content = ( - "Metadata-Version: 2.1\n" - f"Name: {package_name}\n" - f"Version: {version}\n" - "Summary: PTO Assembler & Optimizer\n" - "Requires-Python: >=3.9\n" - "License: Apache-2.0\n" - "Requires-Dist: numpy\n" - ).encode() - - record_lines = [ - f"{pth_filename},{_sha256_record(pth_bytes)},{len(pth_bytes)}", - f"{dist_info_dir}/WHEEL,{_sha256_record(wheel_meta)},{len(wheel_meta)}", - f"{dist_info_dir}/entry_points.txt,{_sha256_record(entry_points)},{len(entry_points)}", - f"{dist_info_dir}/METADATA,{_sha256_record(metadata_content)},{len(metadata_content)}", - f"{dist_info_dir}/RECORD,,", - ] - record_content = "\n".join(record_lines).encode() - - with zipfile.ZipFile(wheel_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: - zf.writestr(pth_filename, pth_bytes) - zf.writestr(f"{dist_info_dir}/WHEEL", wheel_meta) - zf.writestr(f"{dist_info_dir}/entry_points.txt", entry_points) - zf.writestr(f"{dist_info_dir}/METADATA", metadata_content) - zf.writestr(f"{dist_info_dir}/RECORD", record_content) - - return wheel_name diff --git a/docker/Dockerfile b/docker/Dockerfile index f04c7a29c9..8a46534173 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,7 @@ ENV PATH="${PY_PATH}/bin:${PATH}" # dependency RUN dnf install -y ninja-build cmake git ccache gcc-c++ lld zip binutils patchelf chrpath && dnf clean all -RUN pip install --no-cache-dir numpy 'pybind11<3' nanobind setuptools wheel auditwheel +RUN pip install --no-cache-dir numpy 'pybind11<3' nanobind 'scikit-build-core>=0.12.2,<2' auditwheel COPY cmake/LinuxHardeningCache.cmake /tmp/LinuxHardeningCache.cmake @@ -31,7 +31,6 @@ ENV LLVM_BUILD_DIR=$LLVM_SOURCE_DIR/build-release ENV PTO_SOURCE_DIR=$WORKSPACE_DIR/PTOAS ENV PTO_BUILD_DIR=$PTO_SOURCE_DIR/build-release -ENV PTO_INSTALL_DIR=$PTO_SOURCE_DIR/install-release # build LLVM WORKDIR $WORKSPACE_DIR @@ -52,33 +51,19 @@ RUN cmake -C /tmp/LinuxHardeningCache.cmake -G Ninja -S llvm -B $LLVM_BUILD_DIR RUN ninja -C $LLVM_BUILD_DIR -# build ptoas +# build the ptoas wheel through the standard PEP 517 backend WORKDIR $WORKSPACE_DIR RUN git clone https://github.com/zhangstevenunity/PTOAS.git # TODO: tag git commit of PTOAS repo WORKDIR $PTO_SOURCE_DIR -RUN cmake -C /tmp/LinuxHardeningCache.cmake -G Ninja \ - -S . \ - -B $PTO_BUILD_DIR \ - -DLLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm \ - -DMLIR_DIR=$LLVM_BUILD_DIR/lib/cmake/mlir \ - -DPython3_ROOT_DIR=${PY_PATH} \ - -DPython3_EXECUTABLE=${PY_PATH}/bin/python \ - -DPython3_FIND_STRATEGY=LOCATION \ - -Dpybind11_DIR=$(python -m pybind11 --cmakedir) \ - -DMLIR_PYTHON_PACKAGE_DIR=${LLVM_BUILD_DIR}/tools/mlir/python_packages/mlir_core \ - -DCMAKE_INSTALL_PREFIX=${PTO_INSTALL_DIR} \ - -DCMAKE_BUILD_TYPE=Release - -RUN ninja -C $PTO_BUILD_DIR && ninja -C $PTO_BUILD_DIR install - -# create python wheel ENV PTO_WHEEL_DIST_DIR=$PTO_SOURCE_DIR/build/wheel-dist -RUN bash "$PTO_SOURCE_DIR/docker/create_wheel.sh" +RUN SKBUILD_BUILD_DIR=$PTO_BUILD_DIR \ + python -m pip wheel . --no-build-isolation --no-deps \ + --wheel-dir "$PTO_WHEEL_DIST_DIR" # fix missing so -RUN export LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:$LD_LIBRARY_PATH \ +RUN export LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$LD_LIBRARY_PATH \ && auditwheel repair --plat manylinux_2_34_${ARCH} "$PTO_WHEEL_DIST_DIR"/ptoas*.whl -w /wheelhouse RUN pip install /wheelhouse/ptoas*.whl diff --git a/docker/collect_ptoas_dist.sh b/docker/collect_ptoas_dist.sh index 83f9f7299f..798260e036 100755 --- a/docker/collect_ptoas_dist.sh +++ b/docker/collect_ptoas_dist.sh @@ -13,18 +13,14 @@ # # Required environment variables: # LLVM_BUILD_DIR - Path to LLVM build directory -# PTO_BUILD_DIR - Path to PTO build directory (optional, defaults to PTO_SOURCE_DIR/build) # PTO_INSTALL_DIR - Path to PTO install directory # PTO_SOURCE_DIR - Path to PTO source directory # # Output structure: # / -# bin/ptoas - Python wrapper entrypoint -# ptoas/ - Launcher package used by the wrapper -# ptoas_wheel_bootstrap.py - Compatibility bootstrap module -# lib/ptoas.so - Shared runtime loaded by the launcher +# bin/ptoas - Python console wrapper +# ptoas/ - CLI package and native extension # lib/*.so* - Required shared library dependencies -# share/ptoas/TileOps - TileLang template library # tilelang_dsl/ - TileLang DSL Python package # # This compiler-oriented binary artifact intentionally does not bundle the @@ -49,37 +45,21 @@ done export LD_LIBRARY_PATH="${LLVM_BUILD_DIR}/lib:${PTO_INSTALL_DIR}/lib:${LD_LIBRARY_PATH:-}" -PTO_BUILD_DIR="${PTO_BUILD_DIR:-${PTO_SOURCE_DIR}/build}" -PTOAS_BIN="${PTO_BUILD_DIR}/tools/ptoas/ptoas" -PTOAS_SHARED_MODULE="${PTO_INSTALL_DIR}/lib/ptoas.so" +PTOAS_BIN="${PTO_INSTALL_DIR}/bin/ptoas" +PTOAS_PACKAGE_SRC_DIR="${PTO_INSTALL_DIR}/ptoas" +PTOAS_PACKAGE_DIST_DIR="${PTOAS_DIST_DIR}/ptoas" PTOAS_DEPS_DIR="${PTOAS_DIST_DIR}/lib" -PTOAS_SHARED_MODULE_DIST_PATH="${PTOAS_DEPS_DIR}/ptoas.so" -PTOAS_TILEOPS_SRC_DIR="${PTO_INSTALL_DIR}/share/ptoas/TileOps" -PTOAS_TILEOPS_DIST_DIR="${PTOAS_DIST_DIR}/share/ptoas/TileOps" PTOAS_TILELANG_DSL_SRC_DIR="${PTO_INSTALL_DIR}/tilelang_dsl" PTOAS_TILELANG_DSL_DIST_DIR="${PTOAS_DIST_DIR}/tilelang_dsl" -PTOAS_WRAPPER_PKG_SRC_DIR="${PTO_INSTALL_DIR}/ptoas" -PTOAS_WRAPPER_PKG_DIST_DIR="${PTOAS_DIST_DIR}/ptoas" -PTOAS_WHEEL_BOOTSTRAP_SRC="${PTO_INSTALL_DIR}/ptoas_wheel_bootstrap.py" -PTOAS_WHEEL_BOOTSTRAP_DIST_PATH="${PTOAS_DIST_DIR}/ptoas_wheel_bootstrap.py" if [ ! -f "$PTOAS_BIN" ]; then - echo "Error: ptoas wrapper not found at $PTOAS_BIN" >&2 + echo "Error: installed ptoas wrapper not found at $PTOAS_BIN" >&2 exit 1 fi -if [ ! -f "$PTOAS_SHARED_MODULE" ]; then - echo "Error: shared launcher module not found at $PTOAS_SHARED_MODULE" >&2 +if [ ! -d "$PTOAS_PACKAGE_SRC_DIR" ]; then + echo "Error: installed ptoas package not found at $PTOAS_PACKAGE_SRC_DIR" >&2 exit 1 fi -if [ ! -d "$PTOAS_WRAPPER_PKG_SRC_DIR" ]; then - echo "Error: ptoas Python package not found at $PTOAS_WRAPPER_PKG_SRC_DIR" >&2 - exit 1 -fi -if [ ! -f "$PTOAS_WHEEL_BOOTSTRAP_SRC" ]; then - echo "Error: ptoas wheel bootstrap module not found at $PTOAS_WHEEL_BOOTSTRAP_SRC" >&2 - exit 1 -fi - remove_rpath() { local path="$1" if ! has_rpath "$path"; then @@ -163,21 +143,19 @@ harden_elf() { # Create output directories mkdir -p \ "${PTOAS_DIST_DIR}/bin" \ - "${PTOAS_DEPS_DIR}" \ - "$(dirname "${PTOAS_TILEOPS_DIST_DIR}")" -rm -rf "${PTOAS_WRAPPER_PKG_DIST_DIR}" "${PTOAS_WHEEL_BOOTSTRAP_DIST_PATH}" -cp -R "${PTOAS_WRAPPER_PKG_SRC_DIR}" "${PTOAS_WRAPPER_PKG_DIST_DIR}" -cp "${PTOAS_WHEEL_BOOTSTRAP_SRC}" "${PTOAS_WHEEL_BOOTSTRAP_DIST_PATH}" - -# Copy ptoas binary -echo "Copying ptoas wrapper..." -cp "$PTOAS_BIN" "${PTOAS_DIST_DIR}/bin/" + "${PTOAS_DEPS_DIR}" +rm -rf "${PTOAS_PACKAGE_DIST_DIR}" +cp -R "${PTOAS_PACKAGE_SRC_DIR}" "${PTOAS_PACKAGE_DIST_DIR}" +cp "$PTOAS_BIN" "${PTOAS_DIST_DIR}/bin/ptoas" chmod +x "${PTOAS_DIST_DIR}/bin/ptoas" -echo "Copying ptoas shared runtime..." -cp -fL "$PTOAS_SHARED_MODULE" "${PTOAS_SHARED_MODULE_DIST_PATH}" +PTOAS_NATIVE_MODULE="$(find "${PTOAS_PACKAGE_DIST_DIR}" -maxdepth 1 -type f -name '_native*.so' -print -quit)" +if [ -z "${PTOAS_NATIVE_MODULE}" ]; then + echo "Error: packaged ptoas._native extension not found" >&2 + exit 1 +fi -# Collect non-system *.so dependencies needed by the packaged shared runtime. +# Collect non-system dependencies needed by the Python extension. echo "Collecting shared library dependencies..." linux_runtime_dep_paths() { local path="$1" @@ -216,28 +194,28 @@ while read -r res; do [[ -n "$res" ]] || continue should_bundle_linux_dep "$res" || continue copy_so "$res" -done < <(linux_runtime_dep_paths "$PTOAS_BIN") -while read -r res; do - [[ -n "$res" ]] || continue - should_bundle_linux_dep "$res" || continue - copy_so "$res" -done < <(linux_runtime_dep_paths "$PTOAS_SHARED_MODULE") +done < <(linux_runtime_dep_paths "${PTOAS_NATIVE_MODULE}") while read -r packaged; do harden_elf "$packaged" done < <(find "${PTOAS_DEPS_DIR}" -type f | sort) -echo "Copying TileLang runtime resources..." -if [ ! -d "${PTOAS_TILEOPS_SRC_DIR}" ]; then - echo "Error: TileOps resource directory not found at ${PTOAS_TILEOPS_SRC_DIR}" >&2 +if ! command -v patchelf >/dev/null 2>&1; then + echo "Error: patchelf is required to make the ptoas archive relocatable" >&2 exit 1 fi +while read -r packaged; do + patchelf --set-rpath '$ORIGIN' "$packaged" +done < <(find "${PTOAS_DEPS_DIR}" -type f | sort) +harden_elf "${PTOAS_NATIVE_MODULE}" +patchelf --set-rpath '$ORIGIN/../lib' "${PTOAS_NATIVE_MODULE}" + +echo "Copying TileLang runtime resources..." if [ ! -d "${PTOAS_TILELANG_DSL_SRC_DIR}" ]; then echo "Error: tilelang_dsl package directory not found at ${PTOAS_TILELANG_DSL_SRC_DIR}" >&2 exit 1 fi -rm -rf "${PTOAS_TILEOPS_DIST_DIR}" "${PTOAS_TILELANG_DSL_DIST_DIR}" -cp -R "${PTOAS_TILEOPS_SRC_DIR}" "${PTOAS_TILEOPS_DIST_DIR}" +rm -rf "${PTOAS_TILELANG_DSL_DIST_DIR}" cp -R "${PTOAS_TILELANG_DSL_SRC_DIR}" "${PTOAS_TILELANG_DSL_DIST_DIR}" echo "Smoke testing packaged ptoas dist..." @@ -255,11 +233,9 @@ else echo "$VERSION_OUTPUT" | grep -Eq '^ptoas [0-9]+\.[0-9]+$' fi -test -d "${PTOAS_TILEOPS_DIST_DIR}" +test -f "${PTOAS_PACKAGE_DIST_DIR}/_cli.py" +test -d "${PTOAS_PACKAGE_DIST_DIR}/_runtime/share/ptoas/TileOps" test -f "${PTOAS_TILELANG_DSL_DIST_DIR}/__init__.py" -test -f "${PTOAS_SHARED_MODULE_DIST_PATH}" -test -f "${PTOAS_WRAPPER_PKG_DIST_DIR}/_runtime_entry.py" -test -f "${PTOAS_WHEEL_BOOTSTRAP_DIST_PATH}" if [ -e "${PTOAS_DIST_DIR}/ptodsl" ]; then echo "Error: compiler-oriented ptoas dist must not bundle PTODSL" >&2 exit 1 @@ -270,7 +246,7 @@ echo "" echo "=== ptoas distribution contents ===" ls -la "${PTOAS_DIST_DIR}/" ls -la "${PTOAS_DIST_DIR}/bin/" -ls -la "${PTOAS_DIST_DIR}/share/ptoas/" +ls -la "${PTOAS_PACKAGE_DIST_DIR}/" ls -la "${PTOAS_TILELANG_DSL_DIST_DIR}" SO_COUNT=$(find "${PTOAS_DEPS_DIR}" -name "*.so*" 2>/dev/null | wc -l) echo "=== Collected .so dependencies (${SO_COUNT} files) ===" diff --git a/docker/collect_ptoas_dist_mac.sh b/docker/collect_ptoas_dist_mac.sh index ee4a1f1f7e..a3fb2c13d2 100644 --- a/docker/collect_ptoas_dist_mac.sh +++ b/docker/collect_ptoas_dist_mac.sh @@ -13,18 +13,14 @@ # # Required environment variables: # LLVM_BUILD_DIR - Path to LLVM build directory -# PTO_BUILD_DIR - Path to PTO build directory (optional, defaults to PTO_SOURCE_DIR/build) # PTO_INSTALL_DIR - Path to PTO install directory # PTO_SOURCE_DIR - Path to PTO source directory # # Output structure: # / -# bin/ptoas - Python wrapper entrypoint -# ptoas/ - Launcher package used by the wrapper -# ptoas_wheel_bootstrap.py - Compatibility bootstrap module -# lib/ptoas.so - Shared runtime loaded by the launcher +# bin/ptoas - Python console wrapper +# ptoas/ - CLI package and native extension # lib/*.dylib - Required shared library dependencies -# share/ptoas/TileOps - TileLang template library # tilelang_dsl/ - TileLang DSL Python package # # This compiler-oriented binary artifact intentionally does not bundle the @@ -47,48 +43,35 @@ for var in LLVM_BUILD_DIR PTO_INSTALL_DIR PTO_SOURCE_DIR; do fi done -PTO_BUILD_DIR="${PTO_BUILD_DIR:-${PTO_SOURCE_DIR}/build}" -PTOAS_BIN="${PTO_BUILD_DIR}/tools/ptoas/ptoas" -PTOAS_SHARED_MODULE="${PTO_INSTALL_DIR}/lib/ptoas.so" +PTOAS_BIN="${PTO_INSTALL_DIR}/bin/ptoas" +PTOAS_PACKAGE_SRC_DIR="${PTO_INSTALL_DIR}/ptoas" +PTOAS_PACKAGE_DIST_DIR="${PTOAS_DIST_DIR}/ptoas" PTOAS_DEPS_DIR="${PTOAS_DIST_DIR}/lib" -PTOAS_SHARED_MODULE_DIST_PATH="${PTOAS_DEPS_DIR}/ptoas.so" -PTOAS_TILEOPS_SRC_DIR="${PTO_INSTALL_DIR}/share/ptoas/TileOps" -PTOAS_TILEOPS_DIST_DIR="${PTOAS_DIST_DIR}/share/ptoas/TileOps" PTOAS_TILELANG_DSL_SRC_DIR="${PTO_INSTALL_DIR}/tilelang_dsl" PTOAS_TILELANG_DSL_DIST_DIR="${PTOAS_DIST_DIR}/tilelang_dsl" -PTOAS_WRAPPER_PKG_SRC_DIR="${PTO_INSTALL_DIR}/ptoas" -PTOAS_WRAPPER_PKG_DIST_DIR="${PTOAS_DIST_DIR}/ptoas" -PTOAS_WHEEL_BOOTSTRAP_SRC="${PTO_INSTALL_DIR}/ptoas_wheel_bootstrap.py" -PTOAS_WHEEL_BOOTSTRAP_DIST_PATH="${PTOAS_DIST_DIR}/ptoas_wheel_bootstrap.py" UNRESOLVED_NON_SYSTEM_COUNT=0 if [ ! -f "$PTOAS_BIN" ]; then - echo "Error: ptoas wrapper not found at $PTOAS_BIN" >&2 + echo "Error: installed ptoas wrapper not found at $PTOAS_BIN" >&2 exit 1 fi -if [ ! -f "$PTOAS_SHARED_MODULE" ]; then - echo "Error: shared launcher module not found at $PTOAS_SHARED_MODULE" >&2 +if [ ! -d "$PTOAS_PACKAGE_SRC_DIR" ]; then + echo "Error: installed ptoas package not found at $PTOAS_PACKAGE_SRC_DIR" >&2 exit 1 fi -if [ ! -d "$PTOAS_WRAPPER_PKG_SRC_DIR" ]; then - echo "Error: ptoas Python package not found at $PTOAS_WRAPPER_PKG_SRC_DIR" >&2 - exit 1 -fi -if [ ! -f "$PTOAS_WHEEL_BOOTSTRAP_SRC" ]; then - echo "Error: ptoas wheel bootstrap module not found at $PTOAS_WHEEL_BOOTSTRAP_SRC" >&2 - exit 1 -fi - mkdir -p \ "${PTOAS_DIST_DIR}/bin" \ - "${PTOAS_DEPS_DIR}" \ - "$(dirname "${PTOAS_TILEOPS_DIST_DIR}")" -rm -rf "${PTOAS_WRAPPER_PKG_DIST_DIR}" "${PTOAS_WHEEL_BOOTSTRAP_DIST_PATH}" -cp -R "${PTOAS_WRAPPER_PKG_SRC_DIR}" "${PTOAS_WRAPPER_PKG_DIST_DIR}" -cp "${PTOAS_WHEEL_BOOTSTRAP_SRC}" "${PTOAS_WHEEL_BOOTSTRAP_DIST_PATH}" -cp -fL "$PTOAS_BIN" "${PTOAS_DIST_DIR}/bin/" + "${PTOAS_DEPS_DIR}" +rm -rf "${PTOAS_PACKAGE_DIST_DIR}" +cp -R "${PTOAS_PACKAGE_SRC_DIR}" "${PTOAS_PACKAGE_DIST_DIR}" +cp -fL "$PTOAS_BIN" "${PTOAS_DIST_DIR}/bin/ptoas" chmod +x "${PTOAS_DIST_DIR}/bin/ptoas" -cp -fL "$PTOAS_SHARED_MODULE" "${PTOAS_SHARED_MODULE_DIST_PATH}" + +PTOAS_NATIVE_MODULE="$(find "${PTOAS_PACKAGE_DIST_DIR}" -maxdepth 1 -type f -name '_native*.so' -print -quit)" +if [ -z "${PTOAS_NATIVE_MODULE}" ]; then + echo "Error: packaged ptoas._native extension not found" >&2 + exit 1 +fi # Resolve @rpath / @loader_path / @executable_path / absolute install names. resolve_dep_path() { @@ -189,6 +172,7 @@ from pathlib import Path dist_dir = Path(sys.argv[1]).resolve() deps_dir = Path(sys.argv[2]).resolve() bin_dir = (dist_dir / "bin").resolve() +package_dir = (dist_dir / "ptoas").resolve() allowed_prefixes = ( "@loader_path/", "@rpath/", @@ -209,13 +193,15 @@ def is_under(path: Path, root: Path) -> bool: def packaged_dep_ref(owner: Path, dep_base: str) -> str: if is_under(owner, bin_dir): return f"@loader_path/../lib/{dep_base}" + if is_under(owner, package_dir): + return f"@loader_path/../lib/{dep_base}" if is_under(owner, deps_dir): return f"@loader_path/{dep_base}" return f"@loader_path/{dep_base}" def iter_targets(): - for root in (bin_dir, deps_dir): + for root in (package_dir, deps_dir): if not root.exists(): continue for base, _, files in os.walk(root): @@ -272,19 +258,14 @@ PY } echo "Collecting dylib dependencies..." -collect_dylibs "${PTOAS_SHARED_MODULE_DIST_PATH}" +collect_dylibs "${PTOAS_NATIVE_MODULE}" echo "Copying TileLang runtime resources..." -if [[ ! -d "${PTOAS_TILEOPS_SRC_DIR}" ]]; then - echo "Error: TileOps resource directory not found at ${PTOAS_TILEOPS_SRC_DIR}" >&2 - exit 1 -fi if [[ ! -d "${PTOAS_TILELANG_DSL_SRC_DIR}" ]]; then echo "Error: tilelang_dsl package directory not found at ${PTOAS_TILELANG_DSL_SRC_DIR}" >&2 exit 1 fi -rm -rf "${PTOAS_TILEOPS_DIST_DIR}" "${PTOAS_TILELANG_DSL_DIST_DIR}" -cp -R "${PTOAS_TILEOPS_SRC_DIR}" "${PTOAS_TILEOPS_DIST_DIR}" +rm -rf "${PTOAS_TILELANG_DSL_DIST_DIR}" cp -R "${PTOAS_TILELANG_DSL_SRC_DIR}" "${PTOAS_TILELANG_DSL_DIST_DIR}" if [[ -e "${PTOAS_DIST_DIR}/ptodsl" ]]; then echo "Error: compiler-oriented ptoas dist must not bundle PTODSL" >&2 @@ -350,7 +331,7 @@ fi echo "Ad-hoc signing packaged binaries and dylibs..." shopt -s nullglob -SIGN_TARGETS=("${PTOAS_DEPS_DIR}"/*.so "${PTOAS_DEPS_DIR}"/*.dylib) +SIGN_TARGETS=("${PTOAS_NATIVE_MODULE}" "${PTOAS_DEPS_DIR}"/*.so "${PTOAS_DEPS_DIR}"/*.dylib) for target in "${SIGN_TARGETS[@]}"; do codesign --force --sign - --timestamp=none "$target" done @@ -375,11 +356,9 @@ if [ -n "${EXPECTED_PTOAS_CLI_VERSION}" ]; then else echo "$VERSION_OUTPUT" | grep -Eq '^ptoas [0-9]+\.[0-9]+$' fi -test -d "${PTOAS_TILEOPS_DIST_DIR}" +test -f "${PTOAS_PACKAGE_DIST_DIR}/_cli.py" +test -d "${PTOAS_PACKAGE_DIST_DIR}/_runtime/share/ptoas/TileOps" test -f "${PTOAS_TILELANG_DSL_DIST_DIR}/__init__.py" -test -f "${PTOAS_SHARED_MODULE_DIST_PATH}" -test -f "${PTOAS_WRAPPER_PKG_DIST_DIR}/_runtime_entry.py" -test -f "${PTOAS_WHEEL_BOOTSTRAP_DIST_PATH}" env -u DYLD_LIBRARY_PATH -u LD_LIBRARY_PATH \ "${PTOAS_DIST_DIR}/bin/ptoas" \ "${PTO_SOURCE_DIR}/test/lit/pto/kernel_kind_vector_scf_while_emitc.pto" \ @@ -389,7 +368,7 @@ echo "" echo "=== ptoas distribution contents ===" ls -la "${PTOAS_DIST_DIR}/" ls -la "${PTOAS_DIST_DIR}/bin/" -ls -la "${PTOAS_DIST_DIR}/share/ptoas/" +ls -la "${PTOAS_PACKAGE_DIST_DIR}/" ls -la "${PTOAS_TILELANG_DSL_DIST_DIR}" DYLIB_COUNT=$(find "${PTOAS_DEPS_DIR}" -name "*.dylib" 2>/dev/null | wc -l) echo "=== Collected .dylib dependencies (${DYLIB_COUNT} files) ===" diff --git a/docker/copy_ptoas_deps.sh b/docker/copy_ptoas_deps.sh deleted file mode 100644 index 6ddc187138..0000000000 --- a/docker/copy_ptoas_deps.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -# Collect only non-system *.so actually needed by the packaged ptoas shared -# runtime. The `ptoas` wrapper is a Python script, so dependency discovery must -# start from `ptoas.so` instead of running `ldd` on the wrapper itself. -# -# Expects: LLVM_BUILD_DIR, PTO_INSTALL_DIR, PTOAS_DEPS_DIR - -set -euo pipefail - -export LD_LIBRARY_PATH="${LLVM_BUILD_DIR}/lib:${PTO_INSTALL_DIR}/lib:${LD_LIBRARY_PATH:-}" -PTOAS_SHARED_MODULE="${PTO_INSTALL_DIR}/lib/ptoas.so" - -if [[ ! -f "${PTOAS_SHARED_MODULE}" ]]; then - echo "Error: shared launcher module not found at ${PTOAS_SHARED_MODULE}" >&2 - exit 1 -fi - -remove_rpath() { - local path="$1" - if ! has_rpath "$path"; then - return - fi - if ! can_scrub_rpath; then - echo "WARN: skipping RPATH/RUNPATH scrub for ${path}; install patchelf or chrpath to harden local dist artifacts" >&2 - return - fi - if command -v patchelf >/dev/null 2>&1; then - patchelf --remove-rpath "$path" - fi - if has_rpath "$path" && command -v chrpath >/dev/null 2>&1; then - chrpath -d "$path" - fi - if has_rpath "$path"; then - echo "Error: failed to scrub RPATH/RUNPATH from ${path}" >&2 - exit 1 - fi -} - -strip_symbols() { - local path="$1" - strip --strip-unneeded "$path" -} - -has_rpath() { - local path="$1" - if command -v patchelf >/dev/null 2>&1; then - local rpath_value - rpath_value="$(patchelf --print-rpath "$path" 2>/dev/null || true)" - [[ -n "$rpath_value" ]] - return - fi - readelf -d "$path" 2>/dev/null | grep -Eq '(RPATH|RUNPATH)' -} - -can_scrub_rpath() { - command -v patchelf >/dev/null 2>&1 || command -v chrpath >/dev/null 2>&1 -} - -assert_relro() { - local path="$1" - if ! readelf -l "$path" 2>/dev/null | grep -q 'GNU_RELRO'; then - echo "WARN: RELRO segment missing in ${path}" >&2 - return - fi - if ! readelf -d "$path" 2>/dev/null | grep -Eq '(BIND_NOW|FLAGS.*NOW|FLAGS_1.*NOW)'; then - echo "WARN: NOW binding missing in ${path}" >&2 - fi -} - -assert_no_symtab() { - local path="$1" - if readelf -S "$path" 2>/dev/null | grep -Eq '[[:space:]]\\.symtab[[:space:]]'; then - echo "Error: symbol table still present in ${path}" >&2 - exit 1 - fi -} - -assert_no_rpath() { - local path="$1" - if ! can_scrub_rpath; then - return - fi - if has_rpath "$path"; then - echo "Error: runtime search path still present in ${path}" >&2 - exit 1 - fi -} - -harden_elf() { - local path="$1" - remove_rpath "$path" - strip_symbols "$path" - assert_relro "$path" - assert_no_symtab "$path" - assert_no_rpath "$path" -} - -copy_so() { - local f="$1" - [[ -f "$f" ]] || return 0 - local name - name=$(basename "$f") - [[ -f "${PTOAS_DEPS_DIR}/${name}" ]] && return 0 - cp -L -n "$f" "${PTOAS_DEPS_DIR}/" 2>/dev/null || true - harden_elf "${PTOAS_DEPS_DIR}/${name}" - while read -r res; do - [[ -n "$res" ]] || continue - should_bundle_linux_dep "$res" || continue - copy_so "$res" - done < <(linux_runtime_dep_paths "$f") -} - -linux_runtime_dep_paths() { - local path="$1" - ldd "$path" 2>/dev/null | awk ' - /=> \// { print $3 } - /^\// { print $1 } - ' -} - -should_bundle_linux_dep() { - local path="$1" - case "$path" in - /lib/*|/lib64/*|/usr/lib/*|/usr/lib64/*) - return 1 - ;; - esac - return 0 -} - -mkdir -p "$PTOAS_DEPS_DIR" -while read -r res; do - [[ -n "$res" ]] || continue - should_bundle_linux_dep "$res" || continue - copy_so "$res" -done < <(linux_runtime_dep_paths "$PTOAS_SHARED_MODULE") - -while read -r packaged; do - harden_elf "$packaged" -done < <(find "$PTOAS_DEPS_DIR" -type f | sort) diff --git a/docker/create_wheel.sh b/docker/create_wheel.sh deleted file mode 100755 index 36a9f8fa6d..0000000000 --- a/docker/create_wheel.sh +++ /dev/null @@ -1,309 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -set -euo pipefail - -for var in PTO_SOURCE_DIR PTO_INSTALL_DIR LLVM_BUILD_DIR; do - if [[ -z "${!var:-}" ]]; then - echo "Error: $var environment variable is not set" >&2 - exit 1 - fi -done - -WHEEL_STAGING_DIR="${PTO_WHEEL_STAGING_DIR:-${PTO_SOURCE_DIR}/build/wheel-staging}" -WHEEL_DIST_DIR="${PTO_WHEEL_DIST_DIR:-${PTO_SOURCE_DIR}/build/wheel-dist}" -RUNTIME_STAGING_DIR="${PTO_RUNTIME_STAGING_DIR:-${PTO_SOURCE_DIR}/build/runtime-staging}" -PTO_BUILD_DIR="${PTO_BUILD_DIR:-${PTO_SOURCE_DIR}/build}" -PYTHON_BIN="${PYTHON:-python3}" -PTOAS_PYTHON_PACKAGE_NAME="${PTOAS_PYTHON_PACKAGE_NAME:-ptoas}" -PTOAS_PYTHON_PACKAGE_VERSION="${PTOAS_PYTHON_PACKAGE_VERSION:-${PTOAS_VERSION:-}}" -PTOAS_CLI_VERSION="${PTOAS_CLI_VERSION:-${PTOAS_VERSION:-}}" -if [[ ! "${PTOAS_PYTHON_PACKAGE_NAME}" =~ ^[A-Za-z0-9]+([-_.][A-Za-z0-9]+)*$ ]]; then - echo "Error: invalid PTOAS_PYTHON_PACKAGE_NAME '${PTOAS_PYTHON_PACKAGE_NAME}'; expected a normalized project name" >&2 - exit 1 -fi -PTOAS_WHEEL_DISTRIBUTION_NAME="$(printf '%s' "${PTOAS_PYTHON_PACKAGE_NAME}" | sed -E 's/[-_.]+/_/g')" -PTOAS_WRAPPER_PKG_DIR="${PTO_SOURCE_DIR}/ptodsl/ptoas" -PTOAS_WHEEL_BOOTSTRAP="${PTO_SOURCE_DIR}/ptodsl/ptoas_wheel_bootstrap.py" -PTODSL_INSTALL_DIR="${PTO_INSTALL_DIR}/ptodsl" -MLIR_PYTHON_PACKAGE_DIR="${LLVM_BUILD_DIR}/tools/mlir/python_packages/mlir_core" - -if [[ -z "${PTOAS_PYTHON_PACKAGE_VERSION}" ]]; then - PTOAS_PYTHON_PACKAGE_VERSION="$("${PYTHON_BIN}" "${PTO_SOURCE_DIR}/.github/scripts/compute_ptoas_version.py" \ - --cmake-file "${PTO_SOURCE_DIR}/CMakeLists.txt" --mode dev)" -fi -export PTOAS_PYTHON_PACKAGE_VERSION - -linux_runtime_dep_paths() { - local path="$1" - local library_path="${2:-}" - if [[ -n "${library_path}" ]]; then - env LD_LIBRARY_PATH="${library_path}" ldd "$path" 2>/dev/null | awk ' - /=> \// { print $3 } - /^\// { print $1 } - ' - return - fi - ldd "$path" 2>/dev/null | awk ' - /=> \// { print $3 } - /^\// { print $1 } - ' -} - -should_bundle_linux_dep() { - local path="$1" - case "$path" in - /lib/*|/lib64/*|/usr/lib/*|/usr/lib64/*) - return 1 - ;; - esac - return 0 -} - -assemble_linux_wheel_runtime() { - local ptoas_wrapper="${PTO_INSTALL_DIR}/bin/ptoas" - if [[ ! -f "${ptoas_wrapper}" ]]; then - ptoas_wrapper="${PTO_BUILD_DIR}/tools/ptoas/ptoas" - fi - if [[ ! -f "${ptoas_wrapper}" ]]; then - echo "Error: ptoas wrapper not found in build tree or install tree" >&2 - exit 1 - fi - if [[ ! -d "${PTO_INSTALL_DIR}/share/ptoas/TileOps" ]]; then - echo "Error: TileOps resource directory not found at ${PTO_INSTALL_DIR}/share/ptoas/TileOps" >&2 - exit 1 - fi - - mkdir -p "${RUNTIME_STAGING_DIR}/lib" "${RUNTIME_STAGING_DIR}/share/ptoas" - cp -R "${PTO_INSTALL_DIR}/share/ptoas/TileOps" "${RUNTIME_STAGING_DIR}/share/ptoas/TileOps" - cp "${PTO_INSTALL_DIR}/lib/ptoas.so" "${RUNTIME_STAGING_DIR}/lib/ptoas.so" - - # Resolve transitive MLIR/LLVM dependencies through the build tree so - # ldd can discover non-installed libs such as libMLIRMlirOptMain.so.*. - local dep_ld_library_path="${LLVM_BUILD_DIR}/lib:${PTO_INSTALL_DIR}/lib:${LD_LIBRARY_PATH:-}" - while read -r dep_path; do - [[ -n "${dep_path}" ]] || continue - should_bundle_linux_dep "${dep_path}" || continue - cp -L -n "${dep_path}" "${RUNTIME_STAGING_DIR}/lib/" - done < <(linux_runtime_dep_paths "${PTO_INSTALL_DIR}/lib/ptoas.so" "${dep_ld_library_path}" | sort -u) - - local version_output - local version_ld_library_path="${LLVM_BUILD_DIR}/lib:${RUNTIME_STAGING_DIR}/lib:${PTO_INSTALL_DIR}/lib:${LD_LIBRARY_PATH:-}" - version_output="$( - env -u PYTHONPATH -u DYLD_LIBRARY_PATH \ - LD_LIBRARY_PATH="${version_ld_library_path}" \ - "${ptoas_wrapper}" --version | tr -d '\r' - )" - echo "${version_output}" - if [[ -n "${PTOAS_CLI_VERSION:-}" ]]; then - local expected_version_output="ptoas ${PTOAS_CLI_VERSION}" - if [[ "${version_output}" != "${expected_version_output}" ]]; then - echo "Error: expected '${expected_version_output}', got '${version_output}'" >&2 - exit 1 - fi - else - echo "${version_output}" | grep -Eq '^ptoas [0-9]+\.[0-9]+$' - fi -} - -echo "Creating Python wheel..." -echo "Wheel package version: ${PTOAS_PYTHON_PACKAGE_VERSION}" - -rm -rf "${WHEEL_STAGING_DIR}" "${WHEEL_DIST_DIR}" "${RUNTIME_STAGING_DIR}" -mkdir -p "${WHEEL_STAGING_DIR}" "${WHEEL_DIST_DIR}" - -echo "Assembling unified runtime staging tree..." -case "$(uname -s)" in - Darwin) - bash "${PTO_SOURCE_DIR}/docker/collect_ptoas_dist_mac.sh" "${RUNTIME_STAGING_DIR}" - ;; - *) - assemble_linux_wheel_runtime - ;; -esac - -echo "Copying MLIR Python package into wheel staging..." -cp -a "${MLIR_PYTHON_PACKAGE_DIR}/." "${WHEEL_STAGING_DIR}/" - -echo "Overlaying PTO dialect files..." -mkdir -p "${WHEEL_STAGING_DIR}/mlir/dialects" -find "${PTO_INSTALL_DIR}/mlir/dialects" -maxdepth 1 -type f -name '*.py' -exec cp {} "${WHEEL_STAGING_DIR}/mlir/dialects/" \; - -echo "Overlaying PTO native extension..." -mkdir -p "${WHEEL_STAGING_DIR}/mlir/_mlir_libs" -find "${PTO_INSTALL_DIR}/mlir/_mlir_libs" -maxdepth 1 -type f -name '_pto*' -exec cp {} "${WHEEL_STAGING_DIR}/mlir/_mlir_libs/" \; - -echo "Copying TileLang resources..." -rm -rf "${WHEEL_STAGING_DIR}/tilelang_dsl" "${WHEEL_STAGING_DIR}/TileOps" -cp -R "${PTO_INSTALL_DIR}/tilelang_dsl" "${WHEEL_STAGING_DIR}/tilelang_dsl" -cp -R "${PTO_INSTALL_DIR}/share/ptoas/TileOps" "${WHEEL_STAGING_DIR}/TileOps" - -echo "Copying ptodsl package..." -if [[ ! -d "${PTODSL_INSTALL_DIR}" ]]; then - echo "Error: ptodsl package directory not found at ${PTODSL_INSTALL_DIR}" >&2 - exit 1 -fi -if [[ ! -f "${PTODSL_INSTALL_DIR}/__init__.py" ]]; then - echo "Error: ptodsl package is missing ${PTODSL_INSTALL_DIR}/__init__.py" >&2 - exit 1 -fi -rm -rf "${WHEEL_STAGING_DIR}/ptodsl" -cp -R "${PTODSL_INSTALL_DIR}" "${WHEEL_STAGING_DIR}/ptodsl" - -echo "Copying ptoas wheel wrapper package..." -if [[ ! -d "${PTOAS_WRAPPER_PKG_DIR}" ]]; then - echo "Error: ptoas wrapper package directory not found at ${PTOAS_WRAPPER_PKG_DIR}" >&2 - exit 1 -fi -if [[ ! -f "${PTOAS_WRAPPER_PKG_DIR}/__init__.py" ]]; then - echo "Error: ptoas wrapper package is missing ${PTOAS_WRAPPER_PKG_DIR}/__init__.py" >&2 - exit 1 -fi -cp -R "${PTOAS_WRAPPER_PKG_DIR}" "${WHEEL_STAGING_DIR}/ptoas" -if [[ ! -f "${PTOAS_WHEEL_BOOTSTRAP}" ]]; then - echo "Error: ptoas wheel bootstrap module not found at ${PTOAS_WHEEL_BOOTSTRAP}" >&2 - exit 1 -fi -cp "${PTOAS_WHEEL_BOOTSTRAP}" "${WHEEL_STAGING_DIR}/ptoas_wheel_bootstrap.py" - -echo "Embedding unified runtime payload for wheel-side ptoas launcher..." -mkdir -p "${WHEEL_STAGING_DIR}/ptoas/_runtime" -cp -R "${RUNTIME_STAGING_DIR}/share" "${WHEEL_STAGING_DIR}/ptoas/_runtime/share" -cp -R "${RUNTIME_STAGING_DIR}/lib" "${WHEEL_STAGING_DIR}/ptoas/_runtime/lib" - -echo "Removing packaging residue..." -find "${WHEEL_STAGING_DIR}" \( -name '*.egg-info' -o -name '*.dist-info' \) -prune -exec rm -rf {} + -find "${WHEEL_STAGING_DIR}" -name '__pycache__' -prune -exec rm -rf {} + - -export PTO_WHEEL_STAGING_DIR="${WHEEL_STAGING_DIR}" -export PTO_WHEEL_DIST_DIR="${WHEEL_DIST_DIR}" -export PTOAS_PYTHON_PACKAGE_NAME -export PTOAS_PYTHON_PACKAGE_VERSION -export PTOAS_WHEEL_DISTRIBUTION_NAME - -echo "Building wheel archive directly..." -"${PYTHON_BIN}" - <<'PY' -import base64 -import csv -import hashlib -import os -import platform -import re -import sys -import zipfile -from pathlib import Path - -staging = Path(os.environ["PTO_WHEEL_STAGING_DIR"]) -dist = Path(os.environ["PTO_WHEEL_DIST_DIR"]) -project_name = os.environ["PTOAS_PYTHON_PACKAGE_NAME"] -version = os.environ["PTOAS_PYTHON_PACKAGE_VERSION"] -dist_name = os.environ.get("PTOAS_WHEEL_DISTRIBUTION_NAME") or re.sub( - r"[-_.]+", "_", project_name -) - -py_tag = f"cp{sys.version_info.major}{sys.version_info.minor}" -if sys.platform == "darwin": - platform_tag = os.environ.get("WHEEL_PLAT_NAME") or "macosx_11_0_arm64" -else: - arch = platform.machine().lower().replace("-", "_") - platform_tag = os.environ.get("WHEEL_PLAT_NAME") or f"linux_{arch}" - -wheel_name = f"{dist_name}-{version}-{py_tag}-{py_tag}-{platform_tag}.whl" -wheel_path = dist / wheel_name -dist_info = f"{dist_name}-{version}.dist-info" - -metadata = "\n".join([ - "Metadata-Version: 2.1", - f"Name: {project_name}", - f"Version: {version}", - "Summary: PTO Assembler & Optimizer", - "Requires-Python: >=3.9", - "License: Apache-2.0", - "Requires-Dist: numpy", - "", -]).encode("utf-8") - -wheel = "\n".join([ - "Wheel-Version: 1.0", - "Generator: create_wheel.sh", - "Root-Is-Purelib: false", - f"Tag: {py_tag}-{py_tag}-{platform_tag}", - "", -]).encode("utf-8") - -record_rows = [] -has_ptodsl_init = False -has_ptoas_shared_module = False - -def hash_bytes(data: bytes) -> str: - digest = hashlib.sha256(data).digest() - return "sha256=" + base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") - -with zipfile.ZipFile(wheel_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: - for path in sorted(staging.rglob("*")): - if not path.is_file(): - continue - rel = path.relative_to(staging).as_posix() - data = path.read_bytes() - zf.write(path, rel) - record_rows.append((rel, hash_bytes(data), str(len(data)))) - if rel == "ptodsl/__init__.py": - has_ptodsl_init = True - if rel == "ptoas/_runtime/lib/ptoas.so": - has_ptoas_shared_module = True - - entry_points = "\n".join([ - "[console_scripts]", - "ptoas=ptoas_wheel_bootstrap:main", - "", - ]).encode("utf-8") - - for rel, data in [ - (f"{dist_info}/METADATA", metadata), - (f"{dist_info}/WHEEL", wheel), - (f"{dist_info}/entry_points.txt", entry_points), - ]: - zf.writestr(rel, data) - record_rows.append((rel, hash_bytes(data), str(len(data)))) - - record_rel = f"{dist_info}/RECORD" - from io import StringIO - buf = StringIO() - writer = csv.writer(buf, lineterminator="\n") - for row in record_rows: - writer.writerow(row) - writer.writerow((record_rel, "", "")) - record_bytes = buf.getvalue().encode("utf-8") - zf.writestr(record_rel, record_bytes) - -if not has_ptodsl_init: - raise SystemExit("Wheel staging payload is missing ptodsl/__init__.py") -if not has_ptoas_shared_module: - raise SystemExit("Wheel staging payload is missing ptoas/_runtime/lib/ptoas.so") - -with zipfile.ZipFile(wheel_path) as zf: - if "ptodsl/__init__.py" not in zf.namelist(): - raise SystemExit("Built wheel is missing ptodsl/__init__.py") - if "ptoas/_runtime/lib/ptoas.so" not in zf.namelist(): - raise SystemExit("Built wheel is missing ptoas/_runtime/lib/ptoas.so") - if "ptoas/_runtime/bin/ptoas" in zf.namelist(): - raise SystemExit("Built wheel unexpectedly contains ptoas/_runtime/bin/ptoas") - -print(f"Wheel created at {wheel_path}") -PY - -echo "Wheel package name: ${PTOAS_PYTHON_PACKAGE_NAME}" -echo "Wheel created at ${WHEEL_DIST_DIR}/" -ls -la "${WHEEL_DIST_DIR}/"*.whl - -EXPECTED_WHEEL_GLOB="${WHEEL_DIST_DIR}/${PTOAS_WHEEL_DISTRIBUTION_NAME}-${PTOAS_PYTHON_PACKAGE_VERSION}-"*.whl -if ! compgen -G "${EXPECTED_WHEEL_GLOB}" >/dev/null 2>&1; then - echo "Error: expected wheel matching ${EXPECTED_WHEEL_GLOB}" >&2 - exit 1 -fi diff --git a/docker/setup.py b/docker/setup.py deleted file mode 100644 index e6820d5faf..0000000000 --- a/docker/setup.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -import os -import pathlib -import re - -from setuptools import setup, find_namespace_packages - - -_PROJECT_VERSION_RE = re.compile( - r"project\s*\(\s*ptoas\s+VERSION\s+([0-9]+\.[0-9]+)\s*\)" -) - - -def read_package_name() -> str: - package_name = os.environ.get("PTOAS_PYTHON_PACKAGE_NAME", "").strip() - return package_name or "ptoas" - - -def read_package_version() -> str: - version = os.environ.get("PTOAS_PYTHON_PACKAGE_VERSION", "").strip() - if version: - return version - cmake_file = pathlib.Path(__file__).resolve().parents[1] / "CMakeLists.txt" - match = _PROJECT_VERSION_RE.search(cmake_file.read_text(encoding="utf-8")) - if not match: - raise RuntimeError(f"could not find PTOAS version in {cmake_file}") - return match.group(1) - -setup( - name=read_package_name(), - version=read_package_version(), - description="PTO Assembler & Optimizer", - # NOTE: find_namespace_packages detects folders even without __init__.py - packages=find_namespace_packages(), - # NOTE: The * at the end captures .so.22, .so.22.1, etc. - package_data={ - "mlir": [ - "**/*.so*", - "**/*.pyd", - "**/*.py", - "_mlir_libs/*.so*", - ], - }, - include_package_data=True, - zip_safe=False, - python_requires=">=3.9", -) diff --git a/docker/setup_mac.py b/docker/setup_mac.py deleted file mode 100644 index c61ad3e1a5..0000000000 --- a/docker/setup_mac.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -import os -import pathlib -import re - -from setuptools import find_namespace_packages, setup -from setuptools.dist import Distribution - - -class BinaryDistribution(Distribution): - """Force a platform-tagged wheel for macOS artifacts.""" - - def has_ext_modules(self): - return True - - -def read_package_name() -> str: - package_name = os.environ.get("PTOAS_PYTHON_PACKAGE_NAME", "").strip() - return package_name or "ptoas" - - -def read_package_version() -> str: - version = os.environ.get("PTOAS_PYTHON_PACKAGE_VERSION", "").strip() - if version: - return version - cmake_file = pathlib.Path(__file__).resolve().parents[1] / "CMakeLists.txt" - project_version_re = re.compile( - r"project\s*\(\s*ptoas\s+VERSION\s+([0-9]+\.[0-9]+)\s*\)" - ) - match = project_version_re.search(cmake_file.read_text(encoding="utf-8")) - if not match: - raise RuntimeError(f"could not find PTOAS version in {cmake_file}") - return match.group(1) - -setup( - name=read_package_name(), - version=read_package_version(), - description="PTO Assembler & Optimizer", - # NOTE: find_namespace_packages detects folders even without __init__.py - packages=find_namespace_packages(), - # Include native libraries used by macOS wheels (.dylib), while keeping - # existing patterns so this file remains robust across build layouts. - package_data={ - "mlir": [ - "**/*.dylib", - "_mlir_libs/*.dylib", - "**/*.so*", - "**/*.pyd", - "**/*.py", - "_mlir_libs/*.so*", - ], - }, - include_package_data=True, - zip_safe=False, - python_requires=">=3.9", - distclass=BinaryDistribution, -) diff --git a/docker/test_wheel_imports.sh b/docker/test_wheel_imports.sh index 50d20694a0..f06b6938b4 100755 --- a/docker/test_wheel_imports.sh +++ b/docker/test_wheel_imports.sh @@ -89,6 +89,7 @@ echo "Testing ptodsl public imports..." "$PYTHON_BIN" -c "from ptodsl import pto, scalar; print('ptodsl public imports imported successfully')" echo "Testing installed ptoas console entry..." +"$PYTHON_BIN" -c "from ptoas import _native; print(f'ptoas native extension imported successfully from {_native.__file__}')" PTOAS_VERSION_OUTPUT="$(ptoas --version | tr -d '\r')" echo "${PTOAS_VERSION_OUTPUT}" EXPECTED_PTOAS_CLI_VERSION="${PTOAS_CLI_VERSION:-${PTOAS_VERSION:-}}" @@ -102,33 +103,6 @@ else echo "${PTOAS_VERSION_OUTPUT}" | grep -Eq '^ptoas [0-9]+\.[0-9]+$' fi -echo "Testing installed ptoas console entry under polluted Python and LLVM paths..." -POLLUTED_ENV_DIR="${TEST_TMPDIR}/polluted-env" -mkdir -p "${POLLUTED_ENV_DIR}/ptoas" -cat > "${POLLUTED_ENV_DIR}/ptoas/__init__.py" <<'PY' -raise RuntimeError("shadow ptoas package must not be imported") -PY -cat > "${POLLUTED_ENV_DIR}/ptoas/_runtime_entry.py" <<'PY' -raise RuntimeError("shadow runtime entry must not be imported") -PY -POLLUTED_PTOAS_VERSION_OUTPUT="$( - env \ - PYTHONPATH="${POLLUTED_ENV_DIR}" \ - LD_LIBRARY_PATH="/tmp/polluted-llvm" \ - DYLD_LIBRARY_PATH="/tmp/polluted-dylib" \ - "${PTOAS_ENTRYPOINT}" --version | tr -d '\r' -)" -echo "${POLLUTED_PTOAS_VERSION_OUTPUT}" -if [[ -n "${EXPECTED_PTOAS_CLI_VERSION}" ]]; then - EXPECTED_VERSION_OUTPUT="ptoas ${EXPECTED_PTOAS_CLI_VERSION}" - if [[ "${POLLUTED_PTOAS_VERSION_OUTPUT}" != "${EXPECTED_VERSION_OUTPUT}" ]]; then - echo "Error: expected '${EXPECTED_VERSION_OUTPUT}', got '${POLLUTED_PTOAS_VERSION_OUTPUT}'" >&2 - exit 1 - fi -else - echo "${POLLUTED_PTOAS_VERSION_OUTPUT}" | grep -Eq '^ptoas [0-9]+\.[0-9]+$' -fi - echo "Testing installed ptoas console entry in a clean environment..." CLEAN_ENV_DIR="${TEST_TMPDIR}/clean-env" CLEAN_ENV_PTO="${CLEAN_ENV_DIR}/wheel-clean-env-probe.pto" diff --git a/docker/validate_wheel_payload.py b/docker/validate_wheel_payload.py index 15da0c69cc..2d4879d0d6 100644 --- a/docker/validate_wheel_payload.py +++ b/docker/validate_wheel_payload.py @@ -12,22 +12,25 @@ from __future__ import annotations import argparse +import importlib.machinery import zipfile from pathlib import Path REQUIRED_FILES = { - "ptoas_wheel_bootstrap.py", "ptoas/__init__.py", - "ptoas/_launcher.py", - "ptoas/_runtime_entry.py", - "ptoas/_runtime/lib/ptoas.so", + "ptoas/_cli.py", + "ptoas/_runtime/share/ptoas/TileOps/__init__.py", } FORBIDDEN_FILES = { "ptoas/_runtime/bin/ptoas", } -PTOAS_ENTRYPOINT_TARGET = "ptoas_wheel_bootstrap:main" +PTOAS_ENTRYPOINT_TARGET = "ptoas._cli:main" WHEEL_GLOB = "ptoas*.whl" +NATIVE_MODULE_PATHS = { + f"ptoas/_native{suffix}" + for suffix in importlib.machinery.EXTENSION_SUFFIXES +} def _resolve_wheel(candidate: str) -> Path: @@ -83,6 +86,13 @@ def validate_wheel_payload(wheel: Path) -> None: if missing: raise SystemExit(f"wheel is missing required payload files: {missing}") + native_modules = sorted(NATIVE_MODULE_PATHS & names) + if len(native_modules) != 1: + raise SystemExit( + "wheel must contain exactly one importable ptoas._native extension, " + f"found {native_modules}" + ) + present_forbidden = sorted(FORBIDDEN_FILES & names) if present_forbidden: raise SystemExit( @@ -107,7 +117,7 @@ def validate_wheel_payload(wheel: Path) -> None: if console_scripts.get("ptoas") != PTOAS_ENTRYPOINT_TARGET: raise SystemExit( "wheel entry points do not route ptoas through " - "ptoas_wheel_bootstrap:main" + f"{PTOAS_ENTRYPOINT_TARGET}" ) diff --git a/docs/designs/ptoas-python-launcher-layout.md b/docs/designs/ptoas-python-launcher-layout.md index 3cf5a59d42..382cf1cd4e 100644 --- a/docs/designs/ptoas-python-launcher-layout.md +++ b/docs/designs/ptoas-python-launcher-layout.md @@ -12,122 +12,102 @@ This document records the internal launcher contract for the Python-backed `ptoas` command. The user-facing README should describe how to install and run -PTOAS, but it should not expose this implementation detail. - -The wheel launcher intentionally uses a Python wrapper instead of packaging the -native `ptoas` executable as the console entry. This avoids auditwheel / -manylinux handling issues for native executables while still letting the wheel -ship the native compiler implementation as a shared library. - -## Goals - -- Keep wheel, build-tree, and install-tree startup deterministic. -- Resolve Python packages and runtime payloads from the current distribution - layout only. -- Avoid falling back to unrelated source checkouts, editable installs, or - ambient `PYTHONPATH` entries. -- Avoid loading duplicated LLVM/MLIR shared libraries inside one Python process. -- Keep `ptoas._launcher` importable temporarily as a compatibility shim, but do - not treat it as a stable public API. +PTOAS without exposing these implementation details. ## Entry Model -The launcher has three layers: +The wheel, build-tree, and install-tree launchers use the standard Python +console-script and native-extension model: ```text -distribution-specific entry - -> layout resolver - -> ptoas._runtime_entry.launch() - -> ctypes loads ptoas.so - -> calls ptoas_entrypoint(argc, argv) +console script or CMake tree wrapper + -> ptoas._cli.main() + -> import ptoas._native + -> ptoas._native.main(argv) ``` -`ptoas._runtime_entry` is the shared execution layer. It should not guess which -distribution layout is active; callers pass a resolved `PTOASRuntimeLayout`. +`ptoas._native` is built with `pybind11_add_module`. CMake and Python therefore +own the platform and ABI-specific filename. Launcher and packaging code refer to +the import name and never construct `.so`, `.dylib`, or `.pyd` paths. -## Wheel Layout - -Wheel console scripts point to the top-level `ptoas_wheel_bootstrap:main` -module, not to `ptoas._launcher:main`. +The LLVM-based driver implementation is compiled as an object library so it +can use LLVM's RTTI and exception settings independently from pybind11. Those +objects are linked into the Python extension; no separate native executable is +produced. Wheel and standalone-archive entrypoints therefore use the same +extension and CLI module. -The bootstrap module lives outside the `ptoas` package so that the console entry -can identify the installed wheel root before importing `ptoas`. This prevents a -polluted `PYTHONPATH` from resolving a shadow `ptoas` package from a source -checkout or another environment. +## Wheel Layout -Wheel startup is a two-stage process: +Wheel console scripts point directly to `ptoas._cli:main`. The CLI imports the +native extension through Python's normal package machinery and resolves TileOps +package data relative to the installed extension. It does not re-execute itself, +load modules by file path, rewrite `PYTHONPATH`, or override Python's standard +package precedence rules. -1. Stage 1 resolves the wheel-owned package root and runtime root. -2. Stage 1 `execve`s the same console script with a sanitized environment. -3. Stage 2 imports the wheel-owned `ptoas._runtime_entry`. -4. Stage 2 calls the shared `ptoas_entrypoint` from - `ptoas/_runtime/lib/ptoas.so`. +Editable installs use scikit-build-core's redirect mode. The backend maps the +Python sources to the checkout and the CMake-installed native extension to the +editable build output without package-local path manipulation. -In stage 2, the environment is isolated: +Auditwheel and delocate discover the native extension through the standard +wheel binary scan, bundle its dependencies, and rewrite its runtime search +paths. The launcher does not preload or enumerate those libraries. -- `PYTHONPATH` is set to the wheel site-packages root. -- `LD_LIBRARY_PATH` is set to `ptoas/_runtime/lib`. -- `DYLD_LIBRARY_PATH` is set to `ptoas/_runtime/lib`. +Wheel and editable builds use `scikit-build-core` directly as the PEP 517/660 +backend. Project metadata and the console entry point live in `pyproject.toml`; +wheel tags, metadata, RECORD generation, CMake configure/build/install, and +editable redirects are owned by the backend rather than repository scripts. -Wheel mode must not preload every shared library under `ptoas/_runtime/lib` -before loading `ptoas.so`. Auditwheel may rewrite `ptoas.so` dependencies to -hashed libraries under `.libs`. If the launcher preloads the -unhashed runtime copies first and `ptoas.so` then loads the auditwheel-rewritten -copies, LLVM/MLIR command-line options can be registered twice in the same -process. +The main `pyproject.toml` has the static distribution name `ptoas`. +`packaging/ptoas-vmi/pyproject.toml` is a second static PEP 621 project for the +mutually exclusive `ptoas-vmi` distribution. Both projects build the same CMake +source and install the same `ptoas` import package. This keeps project names +standards-compliant because PEP 621 does not permit a dynamic `project.name`. -Therefore wheel mode lets the dynamic loader resolve `ptoas.so` dependencies -from the already-sanitized process environment and the auditwheel RPATH. +CMake's `PTOAS_Python` install component contains only the generated/native +wheel payload. Python source packages are declared through +`tool.scikit-build.wheel.packages`, which also lets editable installs redirect +imports to the source tree without namespace-package or custom `.pth` logic. ## Build-Tree Layout -The build-tree wrapper is generated for one build directory and resolves only -that tree's staged payload: +The build-tree wrapper resolves only its own generated outputs: - wrapper: `/tools/ptoas/ptoas` - Python root: `/python` -- runtime root: `/runtime-staging` -- shared entry: `/runtime-staging/lib/ptoas.so` +- native module: importable as `ptoas._native` from the Python root +- TileOps: `/python/ptoas/_runtime/share/ptoas/TileOps` -If the staged Python package or runtime payload is missing, the wrapper fails -with a layout error. It must not repair startup by searching unrelated Python -roots or source checkouts. - -Build-tree mode prepends the tree-owned Python and library paths to the current -environment. Because changing `LD_LIBRARY_PATH` after process startup is not a -reliable way to affect subsequent `dlopen` resolution on glibc, build-tree mode -may preload runtime libraries before loading `ptoas.so`. +Missing Python packages or TileOps resources are hard layout errors. The +wrapper only adds `/python` to `sys.path`; `ptoas._cli` owns the common +runtime-resource resolution and native invocation. ## Install-Tree Layout -The install-tree wrapper resolves only files installed under the same prefix: +The install-tree wrapper resolves only files under the same prefix: - wrapper: `/bin/ptoas` -- Python root: the installed PTOAS Python package root under that prefix -- runtime root: the installed PTOAS runtime payload under that prefix -- shared entry: `/lib/ptoas.so` - -As with build-tree mode, missing installed payloads are hard layout errors, not -signals to search another checkout or wheel. - -Install-tree mode follows the same dynamic-library policy as build-tree mode: -it may preload runtime libraries before loading `ptoas.so` because the process -was not re-execed with a fully isolated library environment. +- Python root: `` +- native module: importable as `ptoas._native` from the Python root +- TileOps: `/ptoas/_runtime/share/ptoas/TileOps` -## Compatibility Shim +The install-tree wrapper only adds `` to `sys.path`, then delegates to +the same `ptoas._cli` module used by wheels. -`ptoas._launcher` remains importable for a transition period and forwards to the -new launcher path. New code should invoke the `ptoas` command instead of -importing `ptoas._launcher` directly. +## Standalone Archive Layout -The shim exists only to keep older internal scripts from breaking immediately. -It should not grow new layout-resolution logic. +Standalone compiler archives contain the installed Python wrapper and package: -## Non-Goals +```text +bin/ptoas +ptoas/_cli.py +ptoas/_native..so +ptoas/_runtime/share/ptoas/TileOps +lib/ +tilelang_dsl/ +``` -- Support installing `ptoas` and `ptoas-vmi` into the same Python environment. - Those distributions share the same import package and console script and are - intentionally mutually exclusive. -- Support arbitrary `PTOAS_PYTHON_ROOT` fallback search in normal launcher - operation. -- Replace the wheel Python wrapper with a native executable console entry. +The archive requires a Python interpreter compatible with the packaged +extension ABI. `bin/ptoas` adds the archive root to `sys.path`, then uses the +same `ptoas._cli -> ptoas._native` path as the install tree. Linux archives set +the extension runtime path to `$ORIGIN/../lib`; macOS archives rewrite the +extension's install names relative to its package directory. diff --git a/lib/Bindings/Python/CMakeLists.txt b/lib/Bindings/Python/CMakeLists.txt index 4e484668bd..215f385a8e 100644 --- a/lib/Bindings/Python/CMakeLists.txt +++ b/lib/Bindings/Python/CMakeLists.txt @@ -71,6 +71,7 @@ endif() install(TARGETS _pto LIBRARY DESTINATION "mlir/_mlir_libs" + COMPONENT PTOAS_Python ) # ---- 2) Generate ODS Python op bindings: _pto_ops_gen.py ---- @@ -103,6 +104,7 @@ install(FILES "${PTO_PY_SRC}" "${CMAKE_CURRENT_BINARY_DIR}/_pto_ops_gen.py" DESTINATION mlir/dialects + COMPONENT PTOAS_Python ) # Keep the MLIR build-tree Python package self-consistent as well. When users diff --git a/packaging/ptoas-vmi/pyproject.toml b/packaging/ptoas-vmi/pyproject.toml new file mode 100644 index 0000000000..d9c28bd5e4 --- /dev/null +++ b/packaging/ptoas-vmi/pyproject.toml @@ -0,0 +1,60 @@ +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +# The VMI distribution intentionally installs the same `ptoas` import package +# and console script as the main distribution. The two wheels are mutually +# exclusive; this separate static PEP 621 project supplies the distribution +# name required by package indexes without a custom build backend. + +[build-system] +requires = ["scikit-build-core>=0.12.2,<2", "pybind11<3"] +build-backend = "scikit_build_core.build" + +[project] +name = "ptoas-vmi" +dynamic = ["version"] +description = "PTO Assembler & Optimizer with VMI support" +requires-python = ">=3.9" +license = "Apache-2.0" +dependencies = [ + "numpy", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] + +[project.scripts] +ptoas = "ptoas._cli:main" + +[tool.scikit-build] +minimum-version = "build-system.requires" +cmake.source-dir = "../.." +cmake.version = "CMakeLists.txt" +ninja.version = ">=1.10" +build.verbose = false +editable.mode = "redirect" +install.components = ["PTOAS_Python"] + +[tool.scikit-build.wheel.packages] +ptodsl = "../../ptodsl/ptodsl" +ptoas = "../../ptodsl/ptoas" +tilelang_dsl = "../../tilelang-dsl/python/tilelang_dsl" + +[tool.scikit-build.cmake.define] +PTOAS_RELEASE_VERSION_OVERRIDE = { env = "PTOAS_RELEASE_VERSION_OVERRIDE", default = "" } + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "../../docs/release/VMI_VERSION" +regex = '^(?P[0-9]+\.[0-9]+\.[0-9]+)\s*$' diff --git a/ptodsl/CMakeLists.txt b/ptodsl/CMakeLists.txt index 4123b99996..ef602d320f 100644 --- a/ptodsl/CMakeLists.txt +++ b/ptodsl/CMakeLists.txt @@ -12,29 +12,16 @@ set(PTODSL_PACKAGE_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ptodsl") -set(PTOAS_PY_PACKAGE_SRC_DIR - "${CMAKE_CURRENT_SOURCE_DIR}/ptoas") -set(PTOAS_WHEEL_BOOTSTRAP_SOURCE - "${CMAKE_CURRENT_SOURCE_DIR}/ptoas_wheel_bootstrap.py") set(PTODSL_BUILD_ROOT "${CMAKE_BINARY_DIR}/python") set(PTODSL_BUILD_PACKAGE_DIR "${PTODSL_BUILD_ROOT}/ptodsl") -set(PTOAS_PY_BUILD_PACKAGE_DIR - "${PTODSL_BUILD_ROOT}/ptoas") add_custom_target(PTODSLPackage ALL COMMAND ${CMAKE_COMMAND} -E make_directory "${PTODSL_BUILD_ROOT}" COMMAND ${CMAKE_COMMAND} -E remove_directory "${PTODSL_BUILD_PACKAGE_DIR}" - COMMAND ${CMAKE_COMMAND} -E remove_directory "${PTOAS_PY_BUILD_PACKAGE_DIR}" COMMAND ${CMAKE_COMMAND} -E copy_directory "${PTODSL_PACKAGE_SRC_DIR}" "${PTODSL_BUILD_PACKAGE_DIR}" - COMMAND ${CMAKE_COMMAND} -E copy_directory - "${PTOAS_PY_PACKAGE_SRC_DIR}" - "${PTOAS_PY_BUILD_PACKAGE_DIR}" - COMMAND ${CMAKE_COMMAND} -E copy - "${PTOAS_WHEEL_BOOTSTRAP_SOURCE}" - "${PTODSL_BUILD_ROOT}/ptoas_wheel_bootstrap.py" COMMENT "Staging ptodsl package into build/python" VERBATIM ) @@ -48,15 +35,9 @@ install( ) install( - DIRECTORY "${PTOAS_PY_PACKAGE_SRC_DIR}" + DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/ptoas" DESTINATION "." COMPONENT PTOAS_Runtime PATTERN "__pycache__" EXCLUDE PATTERN "*.pyc" EXCLUDE ) - -install( - FILES "${PTOAS_WHEEL_BOOTSTRAP_SOURCE}" - DESTINATION "." - COMPONENT PTOAS_Runtime -) diff --git a/ptodsl/ptoas/__init__.py b/ptodsl/ptoas/__init__.py index 20c98a7808..e7a40405e2 100644 --- a/ptodsl/ptoas/__init__.py +++ b/ptodsl/ptoas/__init__.py @@ -6,5 +6,4 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -"""PTOAS wheel runtime helpers.""" - +"""Python package for the PTOAS command-line interface.""" diff --git a/ptodsl/ptoas/_cli.py b/ptodsl/ptoas/_cli.py new file mode 100644 index 0000000000..dbc2ce2468 --- /dev/null +++ b/ptodsl/ptoas/_cli.py @@ -0,0 +1,100 @@ +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +"""Standard Python console entry point for PTOAS.""" + +from __future__ import annotations + +import os +import shutil +import sys +from pathlib import Path +from typing import Sequence + + +def _resolve_wrapper_path(argv0: str | None = None) -> Path: + candidate = argv0 if argv0 is not None else sys.argv[0] + wrapper = Path(candidate) + if wrapper.exists(): + return wrapper.resolve() + + found = shutil.which(wrapper.name or "ptoas") + if found: + return Path(found).resolve() + + raise SystemExit(f"unable to locate the active ptoas entry point: {candidate}") + + +def _load_native_module(): + from ptoas import _native + + return _native + + +def _resolve_runtime_paths(native_module) -> tuple[Path, Path]: + module_file = getattr(native_module, "__file__", None) + if not module_file: + raise SystemExit("ptoas._native does not expose a module file") + + package_root = Path(module_file).resolve().parent + python_root = package_root.parent + runtime_root = package_root / "_runtime" + tileops_dir = runtime_root / "share" / "ptoas" / "TileOps" + if not tileops_dir.is_dir(): + raise SystemExit( + "unable to locate packaged PTOAS TileOps resources: expected " + f"{tileops_dir}" + ) + return python_root, tileops_dir.resolve() + + +def _has_cli_option(arguments: Sequence[str], option: str) -> bool: + option_with_value = f"{option}=" + return any( + argument == option or argument.startswith(option_with_value) + for argument in arguments + ) + + +def _prepend_path(name: str, value: Path) -> None: + current = os.environ.get(name, "") + parts = [part for part in current.split(os.pathsep) if part] + rendered = str(value) + if rendered in parts: + parts.remove(rendered) + parts.insert(0, rendered) + os.environ[name] = os.pathsep.join(parts) + + +def launch(user_args: Sequence[str], *, wrapper: Path | None = None) -> int: + native_module = _load_native_module() + python_root, tileops_dir = _resolve_runtime_paths(native_module) + wrapper = wrapper.resolve() if wrapper is not None else _resolve_wrapper_path() + + os.environ["PTOAS_BIN"] = str(wrapper) + os.environ["PTOAS_PYTHON_EXE"] = sys.executable + _prepend_path("PATH", wrapper.parent) + + argv = [str(wrapper)] + if not _has_cli_option(user_args, "--tilelang-path"): + argv.extend(["--tilelang-path", str(tileops_dir)]) + if not _has_cli_option(user_args, "--tilelang-pkg-path"): + argv.extend(["--tilelang-pkg-path", str(python_root)]) + if not _has_cli_option(user_args, "--ptodsl-pkg-path"): + argv.extend(["--ptodsl-pkg-path", str(python_root)]) + argv.extend(user_args) + + return int(native_module.main(argv)) + + +def main() -> int: + return launch(sys.argv[1:]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ptodsl/ptoas/_launcher.py b/ptodsl/ptoas/_launcher.py deleted file mode 100644 index a830bbf4ac..0000000000 --- a/ptodsl/ptoas/_launcher.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -"""Compatibility shim for legacy `ptoas._launcher` imports.""" - -from __future__ import annotations - -from ptoas_wheel_bootstrap import main as _wheel_bootstrap_main - - -def main() -> None: - _wheel_bootstrap_main() - - -if __name__ == "__main__": - main() diff --git a/ptodsl/ptoas/_runtime_entry.py b/ptodsl/ptoas/_runtime_entry.py deleted file mode 100644 index 2fa032727d..0000000000 --- a/ptodsl/ptoas/_runtime_entry.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -"""Shared runtime entry helpers for `ptoas` Python launchers. - -This module owns the common in-process ctypes launch contract. Thin entry -wrappers such as the wheel console entry and the build/install-tree wrapper -should resolve their own filesystem layouts, then delegate here for -environment setup and `ptoas_entrypoint` execution. -""" - -from __future__ import annotations - -import ctypes -import os -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import MutableMapping, Sequence - - -@dataclass(frozen=True) -class PTOASRuntimeLayout: - wrapper: Path - runtime_root: Path - python_root: Path - shared_module: Path - tileops_dir: Path - isolated_env: bool - - -def _prepend_env_path(env: MutableMapping[str, str], name: str, value: Path) -> None: - if not value.exists(): - return - current = env.get(name, "") - rendered = str(value) - parts = [part for part in current.split(os.pathsep) if part] - if rendered in parts: - parts.remove(rendered) - parts.insert(0, rendered) - env[name] = os.pathsep.join(parts) - - -def _has_cli_option(argv: Sequence[str], option: str) -> bool: - option_with_value = f"{option}=" - for arg in argv: - if arg == option or arg.startswith(option_with_value): - return True - return False - - -def _iter_runtime_library_files(runtime_lib_dir: Path) -> list[Path]: - if not runtime_lib_dir.is_dir(): - return [] - - libraries: dict[Path, None] = {} - for pattern in ("*.so", "*.so.*", "*.dylib", "*.dylib.*"): - for candidate in sorted(runtime_lib_dir.glob(pattern)): - if candidate.is_file(): - libraries[candidate.resolve()] = None - return list(libraries) - - -def _preload_runtime_libraries(shared_module: Path, runtime_lib_dir: Path) -> None: - runtime_lib_dir = runtime_lib_dir.resolve() - if not runtime_lib_dir.is_dir(): - return - - shared_module = shared_module.resolve() - pending = [ - path for path in _iter_runtime_library_files(runtime_lib_dir) - if path != shared_module and path.name != shared_module.name - ] - if not pending: - return - - mode = getattr(ctypes, "RTLD_GLOBAL", 0) - last_error: OSError | None = None - while pending: - next_pending: list[Path] = [] - made_progress = False - for candidate in pending: - try: - ctypes.CDLL(str(candidate), mode=mode) - made_progress = True - except OSError as exc: - last_error = exc - next_pending.append(candidate) - if not next_pending: - return - if not made_progress: - if last_error is not None: - raise last_error - raise OSError(f"unable to preload runtime libraries from {runtime_lib_dir}") - pending = next_pending - - -def _load_shared_entrypoint(shared_module: Path, runtime_lib_dir: Path, *, preload_runtime_libraries: bool): - if preload_runtime_libraries: - _preload_runtime_libraries(shared_module, runtime_lib_dir) - library = ctypes.CDLL(str(shared_module), mode=getattr(ctypes, "RTLD_GLOBAL", 0)) - entrypoint = library.ptoas_entrypoint - entrypoint.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_char_p)] - entrypoint.restype = ctypes.c_int - return entrypoint - - -def launch(layout: PTOASRuntimeLayout, user_args: Sequence[str]) -> int: - env = os.environ - env["PTOAS_HOME"] = str(layout.runtime_root) - env["PTOAS_BIN"] = str(layout.wrapper) - env["PTOAS_TILEOPS_DIR"] = str(layout.tileops_dir) - env["PTOAS_PYTHON_EXE"] = sys.executable - - _prepend_env_path(env, "PATH", layout.wrapper.parent) - runtime_lib_dir = layout.runtime_root / "lib" - # Keep the wheel-owned packages and libraries first, while preserving - # host paths needed by injected runtimes such as CANN/MSProf. - _prepend_env_path(env, "PYTHONPATH", layout.python_root) - _prepend_env_path(env, "LD_LIBRARY_PATH", runtime_lib_dir) - _prepend_env_path(env, "DYLD_LIBRARY_PATH", runtime_lib_dir) - - argv = [str(layout.wrapper)] - if not _has_cli_option(user_args, "--tilelang-path"): - argv.extend(["--tilelang-path", str(layout.tileops_dir)]) - if not _has_cli_option(user_args, "--tilelang-pkg-path"): - argv.extend(["--tilelang-pkg-path", str(layout.python_root)]) - if not _has_cli_option(user_args, "--ptodsl-pkg-path"): - argv.extend(["--ptodsl-pkg-path", str(layout.python_root)]) - argv.extend(user_args) - - entrypoint = _load_shared_entrypoint( - layout.shared_module, - runtime_lib_dir, - preload_runtime_libraries=not layout.isolated_env, - ) - argv_bytes = [os.fsencode(arg) for arg in argv] - c_argv = (ctypes.c_char_p * len(argv_bytes))(*argv_bytes) - return int(entrypoint(len(argv_bytes), c_argv)) diff --git a/ptodsl/ptoas_wheel_bootstrap.py b/ptodsl/ptoas_wheel_bootstrap.py deleted file mode 100644 index d9da235a63..0000000000 --- a/ptodsl/ptoas_wheel_bootstrap.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -"""Shadow-safe wheel bootstrap for the `ptoas` console entry. - -The console script must not import `ptoas._launcher` directly because caller -environments may prepend unrelated `ptoas` packages via `PYTHONPATH`, editable -checkouts, or build-tree staging directories. This bootstrap has a unique -top-level module name, resolves the active wrapper's sibling site-packages -payload, then loads the wheel-owned `ptoas._runtime_entry` helper by file path. -""" - -from __future__ import annotations - -import importlib.util -import os -import shutil -import sys -import sysconfig -from pathlib import Path -from types import ModuleType -from typing import NoReturn - - -def _resolve_wrapper_path(argv0: str | None = None) -> Path: - candidate = argv0 if argv0 is not None else sys.argv[0] - wrapper = Path(candidate) - if wrapper.exists(): - return wrapper.resolve() - - found = shutil.which(wrapper.name or "ptoas") - if found: - return Path(found).resolve() - - raise SystemExit(f"unable to locate the installed ptoas wrapper: {candidate}") - - -def _unique_paths(paths: list[Path]) -> list[Path]: - unique: list[Path] = [] - seen: set[Path] = set() - for path in paths: - candidate = path.resolve() - if candidate in seen: - continue - seen.add(candidate) - unique.append(candidate) - return unique - - -def _prepend_env_path(env: dict[str, str], name: str, value: Path) -> None: - if not value.exists(): - return - rendered = str(value.resolve()) - parts = [part for part in env.get(name, "").split(os.pathsep) if part] - if rendered in parts: - parts.remove(rendered) - parts.insert(0, rendered) - env[name] = os.pathsep.join(parts) - - -def _candidate_wheel_python_roots(wrapper: Path, module_file: str | None = None) -> list[Path]: - candidates: list[Path] = [] - - if module_file: - candidates.append(Path(module_file).resolve().parent) - - for scheme_name in ("purelib", "platlib"): - scheme_path = sysconfig.get_path(scheme_name) - if scheme_path: - candidates.append(Path(scheme_path)) - - if len(wrapper.parents) >= 2: - prefix = wrapper.parents[1] - py_version = f"python{sys.version_info.major}.{sys.version_info.minor}" - candidates.extend( - [ - prefix / "lib" / py_version / "site-packages", - prefix / "lib" / py_version / "dist-packages", - prefix / "lib64" / py_version / "site-packages", - prefix / "Lib" / "site-packages", - ] - ) - - return _unique_paths(candidates) - - -def _resolve_wheel_python_root(wrapper: Path, module_file: str | None = None) -> tuple[Path, Path]: - searched: list[str] = [] - for python_root in _candidate_wheel_python_roots(wrapper, module_file): - package_root = python_root / "ptoas" - runtime_entry = package_root / "_runtime_entry.py" - runtime_root = package_root / "_runtime" - searched.append(str(package_root)) - if runtime_entry.is_file() and runtime_root.is_dir(): - return python_root, package_root - - rendered = ", ".join(searched) if searched else "" - raise SystemExit( - "unable to locate the wheel-installed ptoas package next to the active wrapper; " - "expected /ptoas/_runtime_entry.py and /ptoas/_runtime. " - f"Searched: {rendered}. This usually means an external PYTHONPATH is shadowing the installed wheel." - ) - - -def _resolve_wheel_layout(runtime_entry: ModuleType, package_root: Path, wrapper: Path): - package_root = package_root.resolve() - wrapper = wrapper.resolve() - runtime_root = package_root / "_runtime" - if not runtime_root.is_dir(): - raise SystemExit( - "wheel-installed ptoas is missing the packaged runtime root: " - "expected ptoas/_runtime" - ) - - shared_module = runtime_root / "lib" / "ptoas.so" - if not shared_module.is_file() or shared_module.stat().st_size <= 0: - raise SystemExit( - "wheel-installed ptoas is missing the packaged shared module: " - "expected ptoas/_runtime/lib/ptoas.so" - ) - - tileops_dir = runtime_root / "share" / "ptoas" / "TileOps" - if not tileops_dir.is_dir(): - raise SystemExit( - "wheel-installed ptoas is missing the packaged TileOps directory: " - "expected ptoas/_runtime/share/ptoas/TileOps" - ) - - return runtime_entry.PTOASRuntimeLayout( - wrapper=wrapper, - runtime_root=runtime_root.resolve(), - python_root=package_root.parent.resolve(), - shared_module=shared_module.resolve(), - tileops_dir=tileops_dir.resolve(), - isolated_env=True, - ) - - -def _load_runtime_entry(python_root: Path, package_root: Path) -> ModuleType: - python_root_text = str(python_root) - if python_root_text in sys.path: - sys.path.remove(python_root_text) - sys.path.insert(0, python_root_text) - - runtime_entry_path = package_root / "_runtime_entry.py" - module_name = "_ptoas_wheel_runtime_entry" - spec = importlib.util.spec_from_file_location(module_name, runtime_entry_path) - if spec is None or spec.loader is None: - raise SystemExit(f"unable to load ptoas runtime entry helper from {runtime_entry_path}") - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) - return module - - -def _prepare_stage2_env(python_root: Path, package_root: Path) -> dict[str, str]: - runtime_lib_dir = package_root / "_runtime" / "lib" - env = os.environ.copy() - env["PTOAS_WHEEL_STAGE2"] = "1" - # Put wheel paths first without discarding host paths required by native - # integrations such as CANN/MSProf. - _prepend_env_path(env, "PYTHONPATH", python_root) - _prepend_env_path(env, "LD_LIBRARY_PATH", runtime_lib_dir) - _prepend_env_path(env, "DYLD_LIBRARY_PATH", runtime_lib_dir) - return env - - -def _maybe_reexec_isolated_process(wrapper: Path, python_root: Path, package_root: Path) -> None: - if os.environ.get("PTOAS_WHEEL_STAGE2") == "1": - return - env = _prepare_stage2_env(python_root, package_root) - os.execve(str(wrapper), [str(wrapper), *sys.argv[1:]], env) - - -def main() -> NoReturn: - wrapper = _resolve_wrapper_path() - python_root, package_root = _resolve_wheel_python_root(wrapper, __file__) - _maybe_reexec_isolated_process(wrapper, python_root, package_root) - runtime_entry = _load_runtime_entry(python_root, package_root) - layout = _resolve_wheel_layout(runtime_entry, package_root, wrapper) - raise SystemExit(runtime_entry.launch(layout, sys.argv[1:])) - - -if __name__ == "__main__": - main() diff --git a/ptodsl/tests/CMakeLists.txt b/ptodsl/tests/CMakeLists.txt index 3f4542216f..1722bf5d08 100644 --- a/ptodsl/tests/CMakeLists.txt +++ b/ptodsl/tests/CMakeLists.txt @@ -19,6 +19,7 @@ if(NOT DEFINED _pto_python_test_env) "PYTHONPATH=${_ptodsl_test_pythonpath}" "LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:${_llvm_root}/lib:${CMAKE_INSTALL_PREFIX}/lib:$ENV{LD_LIBRARY_PATH}" "PATH=${CMAKE_BINARY_DIR}/tools/ptoas:${CMAKE_INSTALL_PREFIX}/bin:${_pto_python_bin_dir}:$ENV{PATH}" + "PTOAS_BIN=${CMAKE_BINARY_DIR}/tools/ptoas/ptoas" ) endif() diff --git a/ptodsl/tests/test_docs_as_test.py b/ptodsl/tests/test_docs_as_test.py index 6520d2f296..dccd1e5746 100644 --- a/ptodsl/tests/test_docs_as_test.py +++ b/ptodsl/tests/test_docs_as_test.py @@ -14,7 +14,6 @@ import json import linecache import re -import shutil import subprocess import sys import tempfile @@ -26,6 +25,7 @@ from ptodsl import pto, scalar from ptodsl._bootstrap import make_context from ptodsl._runtime.launch import LaunchHandle, _marshal_launch_args +from ptodsl._runtime.toolchain import resolve_ptoas_binary from mlir.ir import Module from support.docs_fragment_fixtures import FRAGMENT_FIXTURES, render_fragment_fixture @@ -131,22 +131,6 @@ def block_label(block: MarkdownCodeBlock, symbol: Optional[str] = None) -> str: return format_doc_context(block.path, block.start_line, symbol) -def resolve_ptoas_binary() -> Path: - candidates = [ - REPO_ROOT / "build" / "tools" / "ptoas" / "ptoas", - REPO_ROOT / "install" / "bin" / "ptoas", - ] - for candidate in candidates: - if candidate.is_file(): - return candidate - - from_path = shutil.which("ptoas") - if from_path: - return Path(from_path) - - raise FileNotFoundError("unable to locate a ptoas binary under build/, install/, or PATH") - - def expect_parse_roundtrip_and_verify(text: str, label: str) -> None: with make_context() as ctx: parsed = Module.parse(text, ctx) diff --git a/ptodsl/tests/test_pipe_surface_sample_compile.py b/ptodsl/tests/test_pipe_surface_sample_compile.py index 9ae6391a5a..88541a3635 100644 --- a/ptodsl/tests/test_pipe_surface_sample_compile.py +++ b/ptodsl/tests/test_pipe_surface_sample_compile.py @@ -11,11 +11,12 @@ from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path -import shutil import subprocess import sys import tempfile +from ptodsl._runtime.toolchain import resolve_ptoas_binary as resolve_runtime_ptoas_binary + REPO_ROOT = Path(__file__).resolve().parents[2] SAMPLE = REPO_ROOT / "test" / "samples" / "TPushTPop" / "ptodsl" / "local_c2v" / "kernel.py" @@ -36,18 +37,10 @@ def load_sample_module(): def resolve_ptoas_binary() -> Path | None: - candidates = [ - REPO_ROOT / "build" / "tools" / "ptoas" / "ptoas", - REPO_ROOT / "install" / "bin" / "ptoas", - ] - for candidate in candidates: - if candidate.is_file(): - return candidate - - from_path = shutil.which("ptoas") - if from_path: - return Path(from_path) - return None + try: + return resolve_runtime_ptoas_binary() + except FileNotFoundError: + return None def run_ptoas_frontend(ptoas_bin: Path, mlir_text: str) -> str: diff --git a/ptodsl/tests/test_ptoas_cli.py b/ptodsl/tests/test_ptoas_cli.py new file mode 100644 index 0000000000..4e4f9dfa50 --- /dev/null +++ b/ptodsl/tests/test_ptoas_cli.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import os +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from ptoas import _cli + + +class PTOASCLITests(unittest.TestCase): + def _make_native_module(self, package_root: Path): + return SimpleNamespace( + __file__=str(package_root / "_native.fake.so"), + main=mock.Mock(return_value=0), + ) + + def test_launch_uses_standard_native_module_and_packaged_resources(self): + with tempfile.TemporaryDirectory() as temp_dir: + package_root = Path(temp_dir) / "site-packages" / "ptoas" + tileops_dir = package_root / "_runtime" / "share" / "ptoas" / "TileOps" + wrapper = Path(temp_dir) / "bin" / "ptoas" + tileops_dir.mkdir(parents=True) + wrapper.parent.mkdir(parents=True) + wrapper.write_text("", encoding="utf-8") + native_module = self._make_native_module(package_root) + + with mock.patch.object( + _cli, "_load_native_module", return_value=native_module + ), mock.patch.dict( + _cli.os.environ, + {"PATH": "/usr/bin"}, + clear=True, + ): + exit_code = _cli.launch(["--version"], wrapper=wrapper) + environment = dict(_cli.os.environ) + + self.assertEqual(exit_code, 0) + native_module.main.assert_called_once_with( + [ + str(wrapper.resolve()), + "--tilelang-path", + str(tileops_dir.resolve()), + "--tilelang-pkg-path", + str(package_root.parent.resolve()), + "--ptodsl-pkg-path", + str(package_root.parent.resolve()), + "--version", + ] + ) + self.assertEqual(environment["PTOAS_BIN"], str(wrapper.resolve())) + self.assertEqual(environment["PTOAS_PYTHON_EXE"], _cli.sys.executable) + self.assertEqual( + environment["PATH"], + str(wrapper.parent.resolve()) + os.pathsep + "/usr/bin", + ) + + def test_explicit_resource_options_are_not_overridden(self): + with tempfile.TemporaryDirectory() as temp_dir: + package_root = Path(temp_dir) / "install" / "ptoas" + (package_root / "_runtime" / "share" / "ptoas" / "TileOps").mkdir( + parents=True + ) + wrapper = Path(temp_dir) / "install" / "bin" / "ptoas" + wrapper.parent.mkdir(parents=True) + wrapper.write_text("", encoding="utf-8") + native_module = self._make_native_module(package_root) + arguments = [ + "--tilelang-path=/custom/tileops", + "--tilelang-pkg-path", + "/custom/tilelang", + "--ptodsl-pkg-path=/custom/ptodsl", + "--version", + ] + + with mock.patch.object( + _cli, "_load_native_module", return_value=native_module + ): + _cli.launch(arguments, wrapper=wrapper) + + native_module.main.assert_called_once_with( + [str(wrapper.resolve()), *arguments] + ) + + def test_build_tree_uses_the_same_packaged_resource_layout(self): + with tempfile.TemporaryDirectory() as temp_dir: + build_root = Path(temp_dir) / "build" + package_root = build_root / "python" / "ptoas" + tileops_dir = package_root / "_runtime" / "share" / "ptoas" / "TileOps" + tileops_dir.mkdir(parents=True) + native_module = self._make_native_module(package_root) + + python_root, resolved_tileops = _cli._resolve_runtime_paths( + native_module + ) + + self.assertEqual(python_root, package_root.parent.resolve()) + self.assertEqual(resolved_tileops, tileops_dir.resolve()) + + def test_missing_tileops_resources_is_an_error(self): + with tempfile.TemporaryDirectory() as temp_dir: + package_root = Path(temp_dir) / "site-packages" / "ptoas" + native_module = self._make_native_module(package_root) + + with self.assertRaisesRegex(SystemExit, "TileOps"): + _cli._resolve_runtime_paths(native_module) + + +if __name__ == "__main__": + unittest.main() diff --git a/ptodsl/tests/test_ptoas_frontend_verify.py b/ptodsl/tests/test_ptoas_frontend_verify.py index 18661544bc..083ef4daa5 100644 --- a/ptodsl/tests/test_ptoas_frontend_verify.py +++ b/ptodsl/tests/test_ptoas_frontend_verify.py @@ -8,7 +8,6 @@ # See LICENSE in the root of the software repository for the full text of the License. from pathlib import Path -import shutil import subprocess import sys import tempfile @@ -19,6 +18,7 @@ from ptodsl import pto from ptodsl import scalar from ptodsl._bootstrap import make_context +from ptodsl._runtime.toolchain import resolve_ptoas_binary from mlir.ir import Module @@ -27,22 +27,6 @@ def expect(condition: bool, message: str) -> None: raise AssertionError(message) -def resolve_ptoas_binary() -> Path: - candidates = [ - REPO_ROOT / "build" / "tools" / "ptoas" / "ptoas", - REPO_ROOT / "install" / "bin" / "ptoas", - ] - for candidate in candidates: - if candidate.is_file(): - return candidate - - from_path = shutil.which("ptoas") - if from_path: - return Path(from_path) - - raise FileNotFoundError("unable to locate a ptoas binary under build/, install/, or PATH") - - def emit_example_mlir(example_path: Path) -> str: result = subprocess.run( [sys.executable, str(example_path), "--emit-mlir"], diff --git a/ptodsl/tests/test_ptoas_launcher_shim.py b/ptodsl/tests/test_ptoas_launcher_shim.py deleted file mode 100644 index b56d2fc8e7..0000000000 --- a/ptodsl/tests/test_ptoas_launcher_shim.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -import unittest -import importlib.util -import sys -from pathlib import Path -from unittest import mock - -REPO_ROOT = Path(__file__).resolve().parents[2] -LAUNCHER_SOURCE = REPO_ROOT / "ptodsl" / "ptoas" / "_launcher.py" - - -def _load_source_launcher(): - spec = importlib.util.spec_from_file_location("test_ptoas_launcher_shim_source", LAUNCHER_SOURCE) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - saved_sys_path = list(sys.path) - try: - sys.path.insert(0, str(REPO_ROOT / "ptodsl")) - spec.loader.exec_module(module) - finally: - sys.path = saved_sys_path - return module - - -class LauncherShimTests(unittest.TestCase): - def test_main_forwards_directly_to_wheel_bootstrap(self): - launcher = _load_source_launcher() - - with mock.patch.object(launcher, "_wheel_bootstrap_main") as wheel_main: - launcher.main() - - wheel_main.assert_called_once_with() - - -if __name__ == "__main__": - unittest.main() diff --git a/ptodsl/tests/test_ptoas_runtime_entry.py b/ptodsl/tests/test_ptoas_runtime_entry.py deleted file mode 100644 index 11def2f490..0000000000 --- a/ptodsl/tests/test_ptoas_runtime_entry.py +++ /dev/null @@ -1,249 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -import os -import tempfile -import unittest -from pathlib import Path -from unittest import mock - -from ptoas import _runtime_entry - - -class RuntimeEntryTests(unittest.TestCase): - def test_launch_injects_runtime_environment_and_calls_shared_entrypoint(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - wrapper = temp_root / "bin" / "ptoas" - runtime_root = temp_root / "runtime" - python_root = temp_root / "python" - tileops_dir = runtime_root / "share" / "ptoas" / "TileOps" - shared_module = runtime_root / "lib" / "ptoas.so" - wrapper.parent.mkdir(parents=True, exist_ok=True) - runtime_root.mkdir(parents=True, exist_ok=True) - python_root.mkdir(parents=True, exist_ok=True) - tileops_dir.mkdir(parents=True, exist_ok=True) - shared_module.parent.mkdir(parents=True, exist_ok=True) - shared_module.write_text("fake shared module", encoding="utf-8") - - layout = _runtime_entry.PTOASRuntimeLayout( - wrapper=wrapper, - runtime_root=runtime_root, - python_root=python_root, - shared_module=shared_module, - tileops_dir=tileops_dir, - isolated_env=False, - ) - entrypoint = mock.Mock(return_value=0) - - with mock.patch.dict( - _runtime_entry.os.environ, - { - "PYTHONPATH": "/tmp/external-python", - "LD_LIBRARY_PATH": "/tmp/external-lib", - "DYLD_LIBRARY_PATH": "/tmp/external-dylib", - }, - clear=True, - ), mock.patch.object( - _runtime_entry, - "_load_shared_entrypoint", - return_value=entrypoint, - ) as load_shared_entrypoint: - exit_code = _runtime_entry.launch(layout, ["--version"]) - env = dict(_runtime_entry.os.environ) - - self.assertEqual(exit_code, 0) - load_shared_entrypoint.assert_called_once_with( - layout.shared_module, - layout.runtime_root / "lib", - preload_runtime_libraries=True, - ) - argc, c_argv = entrypoint.call_args.args - self.assertEqual( - [c_argv[i].decode("utf-8") for i in range(argc)], - [ - str(layout.wrapper), - "--tilelang-path", - str(layout.tileops_dir), - "--tilelang-pkg-path", - str(layout.python_root), - "--ptodsl-pkg-path", - str(layout.python_root), - "--version", - ], - ) - self.assertEqual(env["PTOAS_HOME"], str(layout.runtime_root)) - self.assertEqual(env["PTOAS_BIN"], str(layout.wrapper)) - self.assertEqual(env["PTOAS_TILEOPS_DIR"], str(layout.tileops_dir)) - self.assertEqual(env["PYTHONPATH"], str(layout.python_root) + os.pathsep + "/tmp/external-python") - self.assertEqual(env["LD_LIBRARY_PATH"], str(layout.runtime_root / "lib") + os.pathsep + "/tmp/external-lib") - self.assertEqual(env["DYLD_LIBRARY_PATH"], str(layout.runtime_root / "lib") + os.pathsep + "/tmp/external-dylib") - - def test_launch_prioritizes_runtime_environment_for_wheel_layout(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - wrapper = temp_root / "bin" / "ptoas" - runtime_root = temp_root / "runtime" - python_root = temp_root / "python" - tileops_dir = runtime_root / "share" / "ptoas" / "TileOps" - shared_module = runtime_root / "lib" / "ptoas.so" - wrapper.parent.mkdir(parents=True, exist_ok=True) - runtime_root.mkdir(parents=True, exist_ok=True) - python_root.mkdir(parents=True, exist_ok=True) - tileops_dir.mkdir(parents=True, exist_ok=True) - shared_module.parent.mkdir(parents=True, exist_ok=True) - shared_module.write_text("fake shared module", encoding="utf-8") - - layout = _runtime_entry.PTOASRuntimeLayout( - wrapper=wrapper, - runtime_root=runtime_root, - python_root=python_root, - shared_module=shared_module, - tileops_dir=tileops_dir, - isolated_env=True, - ) - entrypoint = mock.Mock(return_value=0) - - with mock.patch.dict( - _runtime_entry.os.environ, - { - "PYTHONPATH": "/tmp/external-python", - "LD_LIBRARY_PATH": "/tmp/external-lib", - "DYLD_LIBRARY_PATH": "/tmp/external-dylib", - }, - clear=True, - ), mock.patch.object( - _runtime_entry, - "_load_shared_entrypoint", - return_value=entrypoint, - ) as load_shared_entrypoint: - exit_code = _runtime_entry.launch(layout, ["--help"]) - env = dict(_runtime_entry.os.environ) - - self.assertEqual(exit_code, 0) - load_shared_entrypoint.assert_called_once_with( - layout.shared_module, - layout.runtime_root / "lib", - preload_runtime_libraries=False, - ) - self.assertEqual( - env["PYTHONPATH"], - str(layout.python_root) + os.pathsep + "/tmp/external-python", - ) - self.assertEqual( - env["LD_LIBRARY_PATH"], - str(layout.runtime_root / "lib") + os.pathsep + "/tmp/external-lib", - ) - self.assertEqual( - env["DYLD_LIBRARY_PATH"], - str(layout.runtime_root / "lib") + os.pathsep + "/tmp/external-dylib", - ) - - def test_load_shared_entrypoint_configures_in_process_ctypes_call(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - runtime_lib_dir = temp_root / "runtime" / "lib" - runtime_lib_dir.mkdir(parents=True, exist_ok=True) - dep = runtime_lib_dir / "libMLIRMlirOptMain.so.21.1" - dep.write_text("fake dep", encoding="utf-8") - shared_module = temp_root / "runtime" / "pto" / "ptoas.so" - shared_module.parent.mkdir(parents=True, exist_ok=True) - shared_module.write_text("fake shared module", encoding="utf-8") - - dep_library = mock.Mock() - shared_library = mock.Mock() - with mock.patch.object( - _runtime_entry.ctypes, "CDLL", side_effect=[dep_library, shared_library] - ) as load_library: - entrypoint = _runtime_entry._load_shared_entrypoint( - shared_module, - runtime_lib_dir, - preload_runtime_libraries=True, - ) - - self.assertEqual( - [call.args[0] for call in load_library.call_args_list], - [str(dep.resolve()), str(shared_module.resolve())], - ) - self.assertTrue( - all( - call.kwargs["mode"] == getattr(_runtime_entry.ctypes, "RTLD_GLOBAL", 0) - for call in load_library.call_args_list - ) - ) - self.assertIs(entrypoint, shared_library.ptoas_entrypoint) - self.assertEqual( - entrypoint.argtypes, - [_runtime_entry.ctypes.c_int, _runtime_entry.ctypes.POINTER(_runtime_entry.ctypes.c_char_p)], - ) - self.assertIs(entrypoint.restype, _runtime_entry.ctypes.c_int) - - def test_load_shared_entrypoint_can_skip_preloading_runtime_libraries(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - runtime_lib_dir = temp_root / "runtime" / "lib" - runtime_lib_dir.mkdir(parents=True, exist_ok=True) - dep = runtime_lib_dir / "libLLVMOption.so.21.1" - dep.write_text("fake dep", encoding="utf-8") - shared_module = runtime_lib_dir / "ptoas.so" - shared_module.write_text("fake shared module", encoding="utf-8") - - shared_library = mock.Mock() - with mock.patch.object(_runtime_entry.ctypes, "CDLL", return_value=shared_library) as load_library: - entrypoint = _runtime_entry._load_shared_entrypoint( - shared_module, - runtime_lib_dir, - preload_runtime_libraries=False, - ) - - load_library.assert_called_once_with( - str(shared_module), - mode=getattr(_runtime_entry.ctypes, "RTLD_GLOBAL", 0), - ) - self.assertIs(entrypoint, shared_library.ptoas_entrypoint) - - def test_preload_runtime_libraries_retries_until_dependencies_resolve(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - runtime_lib_dir = temp_root / "runtime" / "lib" - runtime_lib_dir.mkdir(parents=True, exist_ok=True) - dep_a = runtime_lib_dir / "libA.so.1" - dep_b = runtime_lib_dir / "libB.so.1" - dep_a.write_text("fake dep a", encoding="utf-8") - dep_b.write_text("fake dep b", encoding="utf-8") - shared_module = temp_root / "runtime" / "pto" / "ptoas.so" - shared_module.parent.mkdir(parents=True, exist_ok=True) - shared_module.write_text("fake shared module", encoding="utf-8") - - loaded: list[str] = [] - dep_b_attempts = 0 - - def fake_cdll(path: str, *, mode: int): - nonlocal dep_b_attempts - loaded.append(path) - if Path(path).resolve() == dep_b.resolve() and dep_b_attempts == 0: - dep_b_attempts += 1 - raise OSError("missing libA dependency") - return mock.Mock() - - with mock.patch.object(_runtime_entry.ctypes, "CDLL", side_effect=fake_cdll): - _runtime_entry._preload_runtime_libraries(shared_module, runtime_lib_dir) - - self.assertEqual( - loaded, - [ - str(dep_a.resolve()), - str(dep_b.resolve()), - str(dep_b.resolve()), - ], - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/ptodsl/tests/test_ptoas_tree_wrapper.py b/ptodsl/tests/test_ptoas_tree_wrapper.py index acf983e353..59f5545873 100644 --- a/ptodsl/tests/test_ptoas_tree_wrapper.py +++ b/ptodsl/tests/test_ptoas_tree_wrapper.py @@ -17,411 +17,90 @@ REPO_ROOT = Path(__file__).resolve().parents[2] WRAPPER_SOURCE = REPO_ROOT / "tools" / "ptoas" / "ptoas_wrapper.py" - -class TreeWrapperTests(unittest.TestCase): - def test_build_tree_wrapper_bootstraps_runtime_entry_and_delegates_shared_launcher(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - build_root = temp_root / "build" - build_python_root = build_root / "python" - package_root = build_python_root / "ptoas" - package_root.mkdir(parents=True, exist_ok=True) - (package_root / "__init__.py").write_text("", encoding="utf-8") - (package_root / "_runtime_entry.py").write_text( - """ -from dataclasses import dataclass - +FAKE_CLI = """ calls = [] - -@dataclass(frozen=True) -class PTOASRuntimeLayout: - wrapper: object - runtime_root: object - python_root: object - shared_module: object - tileops_dir: object - isolated_env: bool - - -def launch(layout, user_args): - calls.append(("launch", layout, list(user_args))) +def launch(user_args, *, wrapper=None): + calls.append((list(user_args), wrapper)) return 17 -""".strip() - + "\n", - encoding="utf-8", - ) - - runtime_root = build_root / "runtime-staging" - (runtime_root / "lib").mkdir(parents=True, exist_ok=True) - (runtime_root / "share" / "ptoas" / "TileOps").mkdir(parents=True, exist_ok=True) - (runtime_root / "lib" / "ptoas.so").write_text("fake shared module", encoding="utf-8") - install_root = temp_root / "install" - (install_root / "lib").mkdir(parents=True, exist_ok=True) - (install_root / "share" / "ptoas" / "TileOps").mkdir(parents=True, exist_ok=True) - (install_root / "ptoas").mkdir(parents=True, exist_ok=True) - (install_root / "lib" / "ptoas.so").write_text("install shared module", encoding="utf-8") - (install_root / "ptoas" / "_runtime_entry.py").write_text("", encoding="utf-8") - wrapper_path = temp_root / "build" / "tools" / "ptoas" / "ptoas" - wrapper_path.parent.mkdir(parents=True, exist_ok=True) - wrapper_path.write_text("", encoding="utf-8") - - spec = importlib.util.spec_from_file_location("test_ptoas_wrapper", WRAPPER_SOURCE) - self.assertIsNotNone(spec) - self.assertIsNotNone(spec.loader) - wrapper_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(wrapper_module) - - wrapper_module.__file__ = str(wrapper_path) - - saved_sys_path = list(sys.path) - saved_argv = list(sys.argv) - saved_ptoas = sys.modules.pop("ptoas", None) - saved_runtime_entry = sys.modules.pop("ptoas._runtime_entry", None) - try: - sys.path = [entry for entry in sys.path if entry != str(build_python_root)] - sys.argv = [str(wrapper_path), "--version"] - - with self.assertRaises(SystemExit) as exc: - wrapper_module.main() - - self.assertEqual(exc.exception.code, 17) - self.assertEqual(sys.path[0], str(build_python_root)) - - runtime_entry = sys.modules["ptoas._runtime_entry"] - self.assertEqual( - runtime_entry.calls, - [ - ( - "launch", - runtime_entry.PTOASRuntimeLayout( - wrapper=wrapper_path.resolve(), - runtime_root=runtime_root.resolve(), - python_root=build_python_root.resolve(), - shared_module=(runtime_root / "lib" / "ptoas.so").resolve(), - tileops_dir=(runtime_root / "share" / "ptoas" / "TileOps").resolve(), - isolated_env=False, - ), - ["--version"], - ), - ], - ) - finally: - sys.path = saved_sys_path - sys.argv = saved_argv - sys.modules.pop("ptoas", None) - sys.modules.pop("ptoas._runtime_entry", None) - if saved_ptoas is not None: - sys.modules["ptoas"] = saved_ptoas - if saved_runtime_entry is not None: - sys.modules["ptoas._runtime_entry"] = saved_runtime_entry - - def test_install_tree_wrapper_bootstraps_install_prefix_and_delegates_shared_launcher(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - build_root = temp_root / "build" - build_python_root = build_root / "python" - (build_python_root / "ptoas").mkdir(parents=True, exist_ok=True) - (build_python_root / "ptoas" / "_runtime_entry.py").write_text("", encoding="utf-8") - (build_root / "runtime-staging" / "lib").mkdir(parents=True, exist_ok=True) - (build_root / "runtime-staging" / "share" / "ptoas" / "TileOps").mkdir(parents=True, exist_ok=True) - (build_root / "runtime-staging" / "lib" / "ptoas.so").write_text("build shared module", encoding="utf-8") - - install_root = temp_root / "install" - install_python_root = install_root - package_root = install_python_root / "ptoas" - package_root.mkdir(parents=True, exist_ok=True) - (package_root / "__init__.py").write_text("", encoding="utf-8") - (package_root / "_runtime_entry.py").write_text( - """ -from dataclasses import dataclass - -calls = [] - - -@dataclass(frozen=True) -class PTOASRuntimeLayout: - wrapper: object - runtime_root: object - python_root: object - shared_module: object - tileops_dir: object - isolated_env: bool - - -def launch(layout, user_args): - calls.append(("launch", layout, list(user_args))) - return 23 -""".strip() - + "\n", - encoding="utf-8", - ) - (install_root / "lib").mkdir(parents=True, exist_ok=True) - (install_root / "share" / "ptoas" / "TileOps").mkdir(parents=True, exist_ok=True) - (install_root / "lib" / "ptoas.so").write_text("install shared module", encoding="utf-8") - wrapper_path = install_root / "bin" / "ptoas" - wrapper_path.parent.mkdir(parents=True, exist_ok=True) - wrapper_path.write_text("", encoding="utf-8") - - spec = importlib.util.spec_from_file_location("test_ptoas_wrapper_install", WRAPPER_SOURCE) - self.assertIsNotNone(spec) - self.assertIsNotNone(spec.loader) - wrapper_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(wrapper_module) - - wrapper_module.__file__ = str(wrapper_path) - - saved_sys_path = list(sys.path) - saved_argv = list(sys.argv) - saved_ptoas = sys.modules.pop("ptoas", None) - saved_runtime_entry = sys.modules.pop("ptoas._runtime_entry", None) - try: - sys.path = [entry for entry in sys.path if entry != str(install_python_root)] - sys.argv = [str(wrapper_path), "--help"] - - with self.assertRaises(SystemExit) as exc: - wrapper_module.main() +""".lstrip() - self.assertEqual(exc.exception.code, 23) - self.assertEqual(sys.path[0], str(install_python_root)) - runtime_entry = sys.modules["ptoas._runtime_entry"] - self.assertEqual( - runtime_entry.calls, - [ - ( - "launch", - runtime_entry.PTOASRuntimeLayout( - wrapper=wrapper_path.resolve(), - runtime_root=install_root.resolve(), - python_root=install_python_root.resolve(), - shared_module=(install_root / "lib" / "ptoas.so").resolve(), - tileops_dir=(install_root / "share" / "ptoas" / "TileOps").resolve(), - isolated_env=False, - ), - ["--help"], - ), - ], - ) - finally: - sys.path = saved_sys_path - sys.argv = saved_argv - sys.modules.pop("ptoas", None) - sys.modules.pop("ptoas._runtime_entry", None) - if saved_ptoas is not None: - sys.modules["ptoas"] = saved_ptoas - if saved_runtime_entry is not None: - sys.modules["ptoas._runtime_entry"] = saved_runtime_entry - - def test_build_tree_wrapper_does_not_fall_back_to_install_prefix_python_root(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - build_root = temp_root / "build" - (build_root / "runtime-staging" / "lib").mkdir(parents=True, exist_ok=True) - (build_root / "runtime-staging" / "share" / "ptoas" / "TileOps").mkdir(parents=True, exist_ok=True) - (build_root / "runtime-staging" / "lib" / "ptoas.so").write_text("fake shared module", encoding="utf-8") - - install_root = temp_root / "install" - (install_root / "lib").mkdir(parents=True, exist_ok=True) - (install_root / "share" / "ptoas" / "TileOps").mkdir(parents=True, exist_ok=True) - (install_root / "ptoas").mkdir(parents=True, exist_ok=True) - (install_root / "lib" / "ptoas.so").write_text("install shared module", encoding="utf-8") - (install_root / "ptoas" / "_runtime_entry.py").write_text("", encoding="utf-8") - wrapper_path = temp_root / "build" / "tools" / "ptoas" / "ptoas" - wrapper_path.parent.mkdir(parents=True, exist_ok=True) - wrapper_path.write_text("", encoding="utf-8") - - spec = importlib.util.spec_from_file_location("test_ptoas_wrapper_build_fallback", WRAPPER_SOURCE) - self.assertIsNotNone(spec) - self.assertIsNotNone(spec.loader) - wrapper_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(wrapper_module) - wrapper_module.__file__ = str(wrapper_path) - - saved_sys_path = list(sys.path) - saved_argv = list(sys.argv) - try: - sys.path = [entry for entry in sys.path if entry != str(install_root)] - sys.argv = [str(wrapper_path), "--version"] - - with self.assertRaises(SystemExit) as exc: - wrapper_module.main() - - self.assertIn("build-tree ptoas Python package root", str(exc.exception)) - finally: - sys.path = saved_sys_path - sys.argv = saved_argv - - def test_build_tree_wrapper_fails_when_build_runtime_layout_is_missing(self): +class TreeWrapperTests(unittest.TestCase): + def _load_wrapper(self, wrapper_path: Path, module_name: str): + spec = importlib.util.spec_from_file_location(module_name, WRAPPER_SOURCE) + self.assertIsNotNone(spec) + self.assertIsNotNone(spec.loader) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.__file__ = str(wrapper_path) + return module + + def _run_wrapper(self, wrapper_module, python_root: Path, wrapper: Path): + saved_sys_path = list(sys.path) + saved_argv = list(sys.argv) + saved_modules = { + name: sys.modules.pop(name, None) + for name in ("ptoas", "ptoas._cli") + } + try: + sys.path = [entry for entry in sys.path if entry != str(python_root)] + sys.argv = [str(wrapper), "--version"] + with self.assertRaises(SystemExit) as exc: + wrapper_module.main() + self.assertEqual(exc.exception.code, 17) + return sys.modules["ptoas._cli"] + finally: + sys.path = saved_sys_path + sys.argv = saved_argv + for name in ("ptoas._cli", "ptoas"): + sys.modules.pop(name, None) + if saved_modules[name] is not None: + sys.modules[name] = saved_modules[name] + + def _make_package(self, python_root: Path) -> None: + package_root = python_root / "ptoas" + package_root.mkdir(parents=True) + (package_root / "__init__.py").write_text("", encoding="utf-8") + (package_root / "_cli.py").write_text(FAKE_CLI, encoding="utf-8") + + def test_build_tree_wrapper_delegates_to_cli_module(self): with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - build_root = temp_root / "build" - build_python_root = build_root / "python" - package_root = build_python_root / "ptoas" - package_root.mkdir(parents=True, exist_ok=True) - (package_root / "__init__.py").write_text("", encoding="utf-8") - (package_root / "_runtime_entry.py").write_text( - """ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class PTOASRuntimeLayout: - wrapper: object - runtime_root: object - python_root: object - shared_module: object - tileops_dir: object - isolated_env: bool - - -def launch(layout, user_args): - raise AssertionError("missing runtime layout must fail before launch") -""".strip() - + "\n", - encoding="utf-8", - ) - install_root = temp_root / "install" - (install_root / "lib").mkdir(parents=True, exist_ok=True) - (install_root / "share" / "ptoas" / "TileOps").mkdir(parents=True, exist_ok=True) - (install_root / "lib" / "ptoas.so").write_text("install shared module", encoding="utf-8") - wrapper_path = build_root / "tools" / "ptoas" / "ptoas" - wrapper_path.parent.mkdir(parents=True, exist_ok=True) - wrapper_path.write_text("", encoding="utf-8") + build_root = Path(temp_dir) / "build" + python_root = build_root / "python" + wrapper = build_root / "tools" / "ptoas" / "ptoas" + self._make_package(python_root) + wrapper.parent.mkdir(parents=True) + wrapper.write_text("", encoding="utf-8") - spec = importlib.util.spec_from_file_location("test_ptoas_wrapper_build_missing_runtime", WRAPPER_SOURCE) - self.assertIsNotNone(spec) - self.assertIsNotNone(spec.loader) - wrapper_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(wrapper_module) - wrapper_module.__file__ = str(wrapper_path) + module = self._load_wrapper(wrapper, "test_ptoas_build_wrapper") + cli = self._run_wrapper(module, python_root, wrapper) - saved_sys_path = list(sys.path) - saved_argv = list(sys.argv) - saved_ptoas = sys.modules.pop("ptoas", None) - saved_runtime_entry = sys.modules.pop("ptoas._runtime_entry", None) - try: - sys.path = [entry for entry in sys.path if entry != str(build_python_root)] - sys.argv = [str(wrapper_path), "--version"] + self.assertEqual(cli.calls, [(["--version"], wrapper.resolve())]) - with self.assertRaises(SystemExit) as exc: - wrapper_module.main() - - self.assertIn("ptoas runtime tree", str(exc.exception)) - self.assertIn(str(build_root / "runtime-staging"), str(exc.exception)) - finally: - sys.path = saved_sys_path - sys.argv = saved_argv - sys.modules.pop("ptoas", None) - sys.modules.pop("ptoas._runtime_entry", None) - if saved_ptoas is not None: - sys.modules["ptoas"] = saved_ptoas - if saved_runtime_entry is not None: - sys.modules["ptoas._runtime_entry"] = saved_runtime_entry - - def test_install_tree_wrapper_does_not_fall_back_to_build_tree_python_root(self): + def test_install_tree_wrapper_delegates_to_cli_module(self): with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - build_root = temp_root / "build" - build_python_root = build_root / "python" - (build_python_root / "ptoas").mkdir(parents=True, exist_ok=True) - (build_python_root / "ptoas" / "_runtime_entry.py").write_text("", encoding="utf-8") - (build_root / "runtime-staging" / "lib").mkdir(parents=True, exist_ok=True) - (build_root / "runtime-staging" / "share" / "ptoas" / "TileOps").mkdir(parents=True, exist_ok=True) - (build_root / "runtime-staging" / "lib" / "ptoas.so").write_text("build shared module", encoding="utf-8") - - install_root = temp_root / "install" - wrapper_path = install_root / "bin" / "ptoas" - wrapper_path.parent.mkdir(parents=True, exist_ok=True) - wrapper_path.write_text("", encoding="utf-8") - - spec = importlib.util.spec_from_file_location("test_ptoas_wrapper_install_fallback", WRAPPER_SOURCE) - self.assertIsNotNone(spec) - self.assertIsNotNone(spec.loader) - wrapper_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(wrapper_module) - wrapper_module.__file__ = str(wrapper_path) + install_root = Path(temp_dir) / "install" + wrapper = install_root / "bin" / "ptoas" + self._make_package(install_root) + wrapper.parent.mkdir(parents=True) + wrapper.write_text("", encoding="utf-8") - saved_sys_path = list(sys.path) - saved_argv = list(sys.argv) - try: - sys.path = [entry for entry in sys.path if entry != str(build_python_root)] - sys.argv = [str(wrapper_path), "--help"] + module = self._load_wrapper(wrapper, "test_ptoas_install_wrapper") + cli = self._run_wrapper(module, install_root, wrapper) - with self.assertRaises(SystemExit) as exc: - wrapper_module.main() + self.assertEqual(cli.calls, [(["--version"], wrapper.resolve())]) - self.assertIn("install-tree ptoas Python package root", str(exc.exception)) - finally: - sys.path = saved_sys_path - sys.argv = saved_argv - - def test_install_tree_wrapper_fails_when_install_runtime_layout_is_missing(self): + def test_wrapper_requires_cli_module(self): with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - build_root = temp_root / "build" - (build_root / "runtime-staging" / "lib").mkdir(parents=True, exist_ok=True) - (build_root / "runtime-staging" / "share" / "ptoas" / "TileOps").mkdir(parents=True, exist_ok=True) - (build_root / "runtime-staging" / "lib" / "ptoas.so").write_text("build shared module", encoding="utf-8") - - install_root = temp_root / "install" - package_root = install_root / "ptoas" - package_root.mkdir(parents=True, exist_ok=True) - (package_root / "__init__.py").write_text("", encoding="utf-8") - (package_root / "_runtime_entry.py").write_text( - """ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class PTOASRuntimeLayout: - wrapper: object - runtime_root: object - python_root: object - shared_module: object - tileops_dir: object - isolated_env: bool - - -def launch(layout, user_args): - raise AssertionError("missing runtime layout must fail before launch") -""".strip() - + "\n", - encoding="utf-8", + python_root = Path(temp_dir) / "python" + python_root.mkdir() + module = self._load_wrapper( + Path(temp_dir) / "build" / "tools" / "ptoas" / "ptoas", + "test_ptoas_missing_cli", ) - wrapper_path = install_root / "bin" / "ptoas" - wrapper_path.parent.mkdir(parents=True, exist_ok=True) - wrapper_path.write_text("", encoding="utf-8") - - spec = importlib.util.spec_from_file_location("test_ptoas_wrapper_install_missing_runtime", WRAPPER_SOURCE) - self.assertIsNotNone(spec) - self.assertIsNotNone(spec.loader) - wrapper_module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(wrapper_module) - wrapper_module.__file__ = str(wrapper_path) - - saved_sys_path = list(sys.path) - saved_argv = list(sys.argv) - saved_ptoas = sys.modules.pop("ptoas", None) - saved_runtime_entry = sys.modules.pop("ptoas._runtime_entry", None) - try: - sys.path = [entry for entry in sys.path if entry != str(install_root)] - sys.argv = [str(wrapper_path), "--help"] - - with self.assertRaises(SystemExit) as exc: - wrapper_module.main() - - self.assertIn("ptoas runtime tree", str(exc.exception)) - self.assertIn(str(install_root), str(exc.exception)) - finally: - sys.path = saved_sys_path - sys.argv = saved_argv - sys.modules.pop("ptoas", None) - sys.modules.pop("ptoas._runtime_entry", None) - if saved_ptoas is not None: - sys.modules["ptoas"] = saved_ptoas - if saved_runtime_entry is not None: - sys.modules["ptoas._runtime_entry"] = saved_runtime_entry + with self.assertRaisesRegex(SystemExit, "ptoas/_cli.py"): + module._require_python_root(python_root, context="test") if __name__ == "__main__": diff --git a/ptodsl/tests/test_ptoas_wheel_bootstrap.py b/ptodsl/tests/test_ptoas_wheel_bootstrap.py deleted file mode 100644 index cd04d23bff..0000000000 --- a/ptodsl/tests/test_ptoas_wheel_bootstrap.py +++ /dev/null @@ -1,319 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -import json -import os -import subprocess -import sys -import tempfile -import textwrap -import unittest -from dataclasses import dataclass -from pathlib import Path -from types import SimpleNamespace -from unittest import mock - -import ptoas_wheel_bootstrap as wheel_bootstrap - - -class WheelBootstrapTests(unittest.TestCase): - def _make_runtime_tree(self, temp_root: Path) -> tuple[Path, Path]: - python_root = temp_root / "site-packages" - package_root = python_root / "ptoas" - runtime_root = package_root / "_runtime" - package_root.mkdir(parents=True, exist_ok=True) - (package_root / "_runtime_entry.py").write_text("", encoding="utf-8") - (runtime_root / "lib").mkdir(parents=True, exist_ok=True) - (runtime_root / "share" / "ptoas" / "TileOps").mkdir(parents=True, exist_ok=True) - (runtime_root / "lib" / "ptoas.so").write_text("fake shared module", encoding="utf-8") - bootstrap = python_root / "ptoas_wheel_bootstrap.py" - bootstrap.write_text("", encoding="utf-8") - wrapper = temp_root / "bin" / "ptoas" - wrapper.parent.mkdir(parents=True, exist_ok=True) - wrapper.write_text("", encoding="utf-8") - return bootstrap, wrapper - - def test_stage1_reexecs_with_prioritized_python_and_library_paths(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - bootstrap, wrapper = self._make_runtime_tree(temp_root) - - with mock.patch.dict( - wheel_bootstrap.os.environ, - { - "PYTHONPATH": "/tmp/external-python", - "LD_LIBRARY_PATH": "/tmp/external-lib", - "DYLD_LIBRARY_PATH": "/tmp/external-dylib", - }, - clear=True, - ), mock.patch.object( - wheel_bootstrap, - "__file__", - str(bootstrap), - ), mock.patch.object( - wheel_bootstrap.sys, - "argv", - [str(wrapper), "--version"], - ), mock.patch.object( - wheel_bootstrap.os, - "execve", - side_effect=SystemExit(19), - ) as execve: - with self.assertRaises(SystemExit) as exc: - wheel_bootstrap.main() - - self.assertEqual(exc.exception.code, 19) - execve.assert_called_once() - invoked_path, invoked_argv, invoked_env = execve.call_args.args - self.assertEqual(invoked_path, str(wrapper.resolve())) - self.assertEqual(invoked_argv, [str(wrapper.resolve()), "--version"]) - self.assertEqual(invoked_env["PTOAS_WHEEL_STAGE2"], "1") - self.assertEqual( - invoked_env["PYTHONPATH"], - str(bootstrap.parent.resolve()) + os.pathsep + "/tmp/external-python", - ) - self.assertEqual( - invoked_env["LD_LIBRARY_PATH"], - str((bootstrap.parent / "ptoas" / "_runtime" / "lib").resolve()) - + os.pathsep - + "/tmp/external-lib", - ) - self.assertEqual( - invoked_env["DYLD_LIBRARY_PATH"], - str((bootstrap.parent / "ptoas" / "_runtime" / "lib").resolve()) - + os.pathsep - + "/tmp/external-dylib", - ) - - def test_stage2_loads_runtime_entry_without_reexec(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - bootstrap, wrapper = self._make_runtime_tree(temp_root) - @dataclass(frozen=True) - class FakeLayout: - wrapper: Path - runtime_root: Path - python_root: Path - shared_module: Path - tileops_dir: Path - isolated_env: bool - - runtime_entry = SimpleNamespace( - PTOASRuntimeLayout=FakeLayout, - launch=mock.Mock(return_value=7), - ) - - with mock.patch.dict( - wheel_bootstrap.os.environ, - {"PTOAS_WHEEL_STAGE2": "1"}, - clear=True, - ), mock.patch.object( - wheel_bootstrap, - "__file__", - str(bootstrap), - ), mock.patch.object( - wheel_bootstrap.sys, - "argv", - [str(wrapper), "--help"], - ), mock.patch.object( - wheel_bootstrap, - "_load_runtime_entry", - return_value=runtime_entry, - ) as load_runtime_entry, mock.patch.object( - wheel_bootstrap.os, - "execve", - ) as execve: - with self.assertRaises(SystemExit) as exc: - wheel_bootstrap.main() - - self.assertEqual(exc.exception.code, 7) - execve.assert_not_called() - load_runtime_entry.assert_called_once_with(bootstrap.parent.resolve(), (bootstrap.parent / "ptoas").resolve()) - runtime_entry.launch.assert_called_once_with( - FakeLayout( - wrapper=wrapper.resolve(), - runtime_root=(bootstrap.parent / "ptoas" / "_runtime").resolve(), - python_root=bootstrap.parent.resolve(), - shared_module=(bootstrap.parent / "ptoas" / "_runtime" / "lib" / "ptoas.so").resolve(), - tileops_dir=(bootstrap.parent / "ptoas" / "_runtime" / "share" / "ptoas" / "TileOps").resolve(), - isolated_env=True, - ), - ["--help"], - ) - - def test_resolve_wheel_python_root_prefers_installed_package_when_shadowed(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - bootstrap, wrapper = self._make_runtime_tree(temp_root) - package_root = bootstrap.parent / "ptoas" - shadow_launcher = temp_root / "shadow" / "ptoas" / "_launcher.py" - shadow_launcher.parent.mkdir(parents=True, exist_ok=True) - shadow_launcher.write_text("", encoding="utf-8") - - def fake_get_path(name: str) -> str: - if name in ("purelib", "platlib"): - return str(package_root.parent) - return "" - - with mock.patch.object( - wheel_bootstrap.sysconfig, - "get_path", - side_effect=fake_get_path, - ): - python_root, resolved_package_root = wheel_bootstrap._resolve_wheel_python_root( - wrapper, - str(shadow_launcher), - ) - - self.assertEqual(python_root, package_root.parent.resolve()) - self.assertEqual(resolved_package_root, package_root.resolve()) - - def test_console_entry_reexec_prioritizes_wheel_paths(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - wheel_root = temp_root / "wheel-site" - package_root = wheel_root / "ptoas" - runtime_root = package_root / "_runtime" - runtime_lib_dir = runtime_root / "lib" - tileops_dir = runtime_root / "share" / "ptoas" / "TileOps" - result_path = temp_root / "launch-result.json" - wrapper = temp_root / "venv" / "bin" / "ptoas" - - package_root.mkdir(parents=True, exist_ok=True) - runtime_lib_dir.mkdir(parents=True, exist_ok=True) - tileops_dir.mkdir(parents=True, exist_ok=True) - (runtime_lib_dir / "ptoas.so").write_text("fake shared module", encoding="utf-8") - (wheel_root / "ptoas_wheel_bootstrap.py").write_text( - Path(wheel_bootstrap.__file__).read_text(encoding="utf-8"), - encoding="utf-8", - ) - (package_root / "_runtime_entry.py").write_text( - textwrap.dedent( - """ - import json - import os - from dataclasses import dataclass - from pathlib import Path - - @dataclass(frozen=True) - class PTOASRuntimeLayout: - wrapper: Path - runtime_root: Path - python_root: Path - shared_module: Path - tileops_dir: Path - isolated_env: bool - - def launch(layout, user_args): - result = { - "user_args": list(user_args), - "env": { - "PTOAS_WHEEL_STAGE2": os.environ.get("PTOAS_WHEEL_STAGE2"), - "PYTHONPATH": os.environ.get("PYTHONPATH"), - "LD_LIBRARY_PATH": os.environ.get("LD_LIBRARY_PATH"), - "DYLD_LIBRARY_PATH": os.environ.get("DYLD_LIBRARY_PATH"), - }, - "layout": { - "wrapper": str(layout.wrapper), - "runtime_root": str(layout.runtime_root), - "python_root": str(layout.python_root), - "shared_module": str(layout.shared_module), - "tileops_dir": str(layout.tileops_dir), - "isolated_env": layout.isolated_env, - }, - } - Path(os.environ["PTOAS_TEST_RESULT"]).write_text( - json.dumps(result, sort_keys=True), - encoding="utf-8", - ) - return 23 - """ - ).lstrip(), - encoding="utf-8", - ) - - shadow_root = temp_root / "shadow-site" - shadow_package = shadow_root / "ptoas" - shadow_package.mkdir(parents=True, exist_ok=True) - (shadow_package / "__init__.py").write_text( - "raise RuntimeError('shadow ptoas package must not be imported')\n", - encoding="utf-8", - ) - (shadow_package / "_runtime_entry.py").write_text( - "raise RuntimeError('shadow runtime entry must not be imported')\n", - encoding="utf-8", - ) - - wrapper.parent.mkdir(parents=True, exist_ok=True) - wrapper.write_text( - textwrap.dedent( - f""" - #!{sys.executable} - import sys - sys.path.insert(0, {str(wheel_root)!r}) - import ptoas_wheel_bootstrap - ptoas_wheel_bootstrap.main() - """ - ).lstrip(), - encoding="utf-8", - ) - wrapper.chmod(0o755) - - env = os.environ.copy() - env.pop("PTOAS_WHEEL_STAGE2", None) - env["PYTHONPATH"] = os.pathsep.join([str(shadow_root), str(wheel_root)]) - env["LD_LIBRARY_PATH"] = "/tmp/polluted-llvm" - env["DYLD_LIBRARY_PATH"] = "/tmp/polluted-dylib" - env["PTOAS_TEST_RESULT"] = str(result_path) - - completed = subprocess.run( - [str(wrapper), "--version"], - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - check=False, - ) - - self.assertEqual( - completed.returncode, - 23, - msg=f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}", - ) - result = json.loads(result_path.read_text(encoding="utf-8")) - - self.assertEqual(result["user_args"], ["--version"]) - self.assertEqual(result["env"]["PTOAS_WHEEL_STAGE2"], "1") - self.assertEqual( - result["env"]["PYTHONPATH"], - str(wheel_root.resolve()) + os.pathsep + str(shadow_root), - ) - self.assertEqual( - result["env"]["LD_LIBRARY_PATH"], - str(runtime_lib_dir.resolve()) + os.pathsep + "/tmp/polluted-llvm", - ) - self.assertEqual( - result["env"]["DYLD_LIBRARY_PATH"], - str(runtime_lib_dir.resolve()) + os.pathsep + "/tmp/polluted-dylib", - ) - self.assertEqual( - result["layout"], - { - "wrapper": str(wrapper.resolve()), - "runtime_root": str(runtime_root.resolve()), - "python_root": str(wheel_root.resolve()), - "shared_module": str((runtime_lib_dir / "ptoas.so").resolve()), - "tileops_dir": str(tileops_dir.resolve()), - "isolated_env": True, - }, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/pyproject.toml b/pyproject.toml index b34908352a..100b364b39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,29 +1,22 @@ -# pyproject.toml - Project metadata -# -# Build flow (requires LLVM already built): -# pip install . -# -# This will: -# 1. CMake configure + Ninja build + install -# 2. Package Python bindings into a wheel via docker/create_wheel.sh -# -# Environment variables (all optional): -# LLVM_BUILD_DIR Path to LLVM build dir -# (default: /llvm-workspace/llvm-project/build-shared) -# PTO_BUILD_DIR Path to PTOAS build dir (default: /build) -# PTO_INSTALL_DIR Install prefix (default: /install) -# CMAKE_C_COMPILER Optional C compiler passed through to CMake -# CMAKE_CXX_COMPILER Optional C++ compiler passed through to CMake -# PTOAS_PYTHON_PACKAGE_VERSION Wheel version override +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +# Standard PEP 517 configuration for the `ptoas` distribution. Set +# LLVM_BUILD_DIR to an existing LLVM/MLIR build. SKBUILD_BUILD_DIR can be used +# when a persistent CMake build tree is required by tests or development tools. [build-system] -requires = ["setuptools>=68", "wheel"] -build-backend = "_ptoas_build_backend" -backend-path = ["."] +requires = ["scikit-build-core>=0.12.2,<2", "pybind11<3"] +build-backend = "scikit_build_core.build" [project] name = "ptoas" -version = "0.51" +dynamic = ["version"] description = "PTO Assembler & Optimizer" readme = "README.md" requires-python = ">=3.9" @@ -40,3 +33,27 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ] + +[project.scripts] +ptoas = "ptoas._cli:main" + +[tool.scikit-build] +minimum-version = "build-system.requires" +cmake.version = "CMakeLists.txt" +ninja.version = ">=1.10" +build.verbose = false +editable.mode = "redirect" +install.components = ["PTOAS_Python"] + +[tool.scikit-build.wheel.packages] +ptodsl = "ptodsl/ptodsl" +ptoas = "ptodsl/ptoas" +tilelang_dsl = "tilelang-dsl/python/tilelang_dsl" + +[tool.scikit-build.cmake.define] +PTOAS_RELEASE_VERSION_OVERRIDE = { env = "PTOAS_RELEASE_VERSION_OVERRIDE", default = "" } + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "CMakeLists.txt" +regex = 'project\s*\(\s*ptoas\s+VERSION\s+(?P[0-9]+\.[0-9]+)\s*\)' diff --git a/quick_install.sh b/quick_install.sh index 38509b61d5..ec72e2830e 100755 --- a/quick_install.sh +++ b/quick_install.sh @@ -9,7 +9,7 @@ # For quick development, build and install ptoas and its python bindings # on top of Docker image https://github.com/learning-chip/agent_docker_npu/pull/8 -# assume MLIR is already installed to save time, takes <3min to finish the build of pto extension +# assume MLIR is already installed to save time # # Optional env: # LLVM_BUILD_DIR - default: ${LLVM_SOURCE_DIR:-/llvm-workspace/llvm-project}/build-shared @@ -23,13 +23,10 @@ PTO_INSTALL_DIR="${PTO_INSTALL_DIR:-${PTO_SOURCE_DIR}/install}" LLVM_SOURCE_DIR="${LLVM_SOURCE_DIR:-/llvm-workspace/llvm-project}" LLVM_BUILD_DIR="${LLVM_BUILD_DIR:-${LLVM_SOURCE_DIR}/build-shared}" -PY_ROOT="$(python -c 'import sys; print(sys.prefix)')" - for d in "$LLVM_BUILD_DIR/lib/cmake/llvm" "$LLVM_BUILD_DIR/lib/cmake/mlir"; do test -d "$d" || { echo "error: missing $d (set LLVM_BUILD_DIR?)" >&2; exit 1; } done -PYBIND11_DIR="$(python -m pybind11 --cmakedir)" MLIR_PY_PKG="${LLVM_BUILD_DIR}/tools/mlir/python_packages/mlir_core" test -d "$MLIR_PY_PKG" || { echo "error: MLIR python package dir missing: $MLIR_PY_PKG" >&2; exit 1; } @@ -37,30 +34,20 @@ PTOAS_VERSION="${PTOAS_VERSION:-$(python "${PTO_SOURCE_DIR}/.github/scripts/comp cd "$PTO_SOURCE_DIR" -cmake -C "${PTO_SOURCE_DIR}/cmake/LinuxHardeningCache.cmake" -G Ninja \ - -S . \ - -B build \ - -DLLVM_DIR="${LLVM_BUILD_DIR}/lib/cmake/llvm" \ - -DMLIR_DIR="${LLVM_BUILD_DIR}/lib/cmake/mlir" \ - -DPython3_ROOT_DIR="${PY_ROOT}" \ - -DPython3_EXECUTABLE=python \ - -DPython3_FIND_STRATEGY=LOCATION \ - -Dpybind11_DIR="${PYBIND11_DIR}" \ - -DMLIR_PYTHON_PACKAGE_DIR="${MLIR_PY_PKG}" \ - -DPTOAS_RELEASE_VERSION_OVERRIDE="${PTOAS_VERSION}" \ - -DCMAKE_INSTALL_PREFIX="${PTO_INSTALL_DIR}" - -ninja -C build -ninja -C build install - -export PTO_SOURCE_DIR PTO_INSTALL_DIR LLVM_BUILD_DIR -export PTOAS_PYTHON_PACKAGE_VERSION="${PTOAS_PYTHON_PACKAGE_VERSION:-${PTOAS_VERSION}}" -bash "${PTO_SOURCE_DIR}/docker/create_wheel.sh" +PTO_WHEEL_DIR="${PTO_SOURCE_DIR}/build/wheel-dist" +rm -rf "${PTO_WHEEL_DIR}" +mkdir -p "${PTO_WHEEL_DIR}" +LLVM_BUILD_DIR="${LLVM_BUILD_DIR}" \ +PTOAS_RELEASE_VERSION_OVERRIDE="${PTOAS_VERSION}" \ +SKBUILD_BUILD_DIR="${PTO_SOURCE_DIR}/build" \ + python -m pip wheel . --no-deps --wheel-dir "${PTO_WHEEL_DIR}" + +cmake --install "${PTO_SOURCE_DIR}/build" --prefix "${PTO_INSTALL_DIR}" shopt -s nullglob -wheels=("${MLIR_PY_PKG}/dist/ptoas-"*.whl) +wheels=("${PTO_WHEEL_DIR}/ptoas-"*.whl) shopt -u nullglob -((${#wheels[@]} > 0)) || { echo "error: no ptoas-*.whl under ${MLIR_PY_PKG}/dist" >&2; exit 1; } +((${#wheels[@]} > 0)) || { echo "error: no ptoas-*.whl under ${PTO_WHEEL_DIR}" >&2; exit 1; } pip install --force-reinstall "${wheels[0]}" export PATH="${PTO_SOURCE_DIR}/build/tools/ptoas:${PATH}" diff --git a/test/python/test_docker_runtime_packaging.py b/test/python/test_docker_runtime_packaging.py index 646d621db3..a3b1768163 100644 --- a/test/python/test_docker_runtime_packaging.py +++ b/test/python/test_docker_runtime_packaging.py @@ -13,14 +13,25 @@ REPO_ROOT = Path(__file__).resolve().parents[2] DOCKERFILE = REPO_ROOT / "docker" / "Dockerfile" -COPY_DEPS_SCRIPT = REPO_ROOT / "docker" / "copy_ptoas_deps.sh" COLLECT_DIST_SCRIPT = REPO_ROOT / "docker" / "collect_ptoas_dist.sh" COLLECT_DIST_MAC_SCRIPT = REPO_ROOT / "docker" / "collect_ptoas_dist_mac.sh" +PTOAS_CMAKE = REPO_ROOT / "tools" / "ptoas" / "CMakeLists.txt" BUILD_WHEEL_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "build_wheel.yml" BUILD_WHEEL_MAC_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "build_wheel_mac.yml" class DockerRuntimePackagingTests(unittest.TestCase): + def test_runtime_uses_standard_python_extension_without_native_cli(self): + cmake = PTOAS_CMAKE.read_text(encoding="utf-8") + + self.assertIn( + "pybind11_add_module(ptoas_runtime MODULE", + cmake, + ) + self.assertIn('OUTPUT_NAME "_native"', cmake) + self.assertNotIn("add_executable(ptoas_native_cli", cmake) + self.assertNotIn('OUTPUT_NAME "ptoas-native"', cmake) + def test_runtime_image_uses_wheel_entrypoint_instead_of_copied_wrapper(self): dockerfile = DOCKERFILE.read_text(encoding="utf-8") @@ -35,41 +46,35 @@ def test_runtime_image_uses_wheel_entrypoint_instead_of_copied_wrapper(self): ) self.assertNotIn("/usr/local/lib/ptoas", dockerfile) - def test_dependency_collection_roots_at_shared_module(self): - script = COPY_DEPS_SCRIPT.read_text(encoding="utf-8") + def test_docker_builds_wheel_through_standard_pep517_backend(self): + dockerfile = DOCKERFILE.read_text(encoding="utf-8") - self.assertIn('PTOAS_SHARED_MODULE="${PTO_INSTALL_DIR}/lib/ptoas.so"', script) - self.assertIn('done < <(linux_runtime_dep_paths "$PTOAS_SHARED_MODULE")', script) - self.assertNotIn('PTOAS_BIN="${PTO_BUILD_DIR}/tools/ptoas/ptoas"', script) - self.assertNotIn('done < <(linux_runtime_dep_paths "$PTOAS_BIN")', script) + self.assertIn("python -m pip wheel .", dockerfile) + self.assertIn("SKBUILD_BUILD_DIR=$PTO_BUILD_DIR", dockerfile) + self.assertNotIn("create_wheel.sh", dockerfile) + self.assertNotIn("_ptoas_build_backend", dockerfile) - def test_linux_dist_stages_install_tree_python_root(self): + def test_linux_dist_packages_python_wrapper_and_native_extension(self): script = COLLECT_DIST_SCRIPT.read_text(encoding="utf-8") - self.assertIn('PTOAS_WRAPPER_PKG_SRC_DIR="${PTO_INSTALL_DIR}/ptoas"', script) - self.assertIn('PTOAS_WRAPPER_PKG_DIST_DIR="${PTOAS_DIST_DIR}/ptoas"', script) - self.assertIn('PTOAS_WHEEL_BOOTSTRAP_SRC="${PTO_INSTALL_DIR}/ptoas_wheel_bootstrap.py"', script) - self.assertIn('cp "${PTOAS_WHEEL_BOOTSTRAP_SRC}" "${PTOAS_WHEEL_BOOTSTRAP_DIST_PATH}"', script) - self.assertIn('test -f "${PTOAS_WRAPPER_PKG_DIST_DIR}/_runtime_entry.py"', script) - self.assertIn('test -f "${PTOAS_WHEEL_BOOTSTRAP_DIST_PATH}"', script) + self.assertIn('PTOAS_BIN="${PTO_INSTALL_DIR}/bin/ptoas"', script) + self.assertIn('PTOAS_PACKAGE_SRC_DIR="${PTO_INSTALL_DIR}/ptoas"', script) + self.assertIn('PTOAS_NATIVE_MODULE="$(find "${PTOAS_PACKAGE_DIST_DIR}"', script) + self.assertIn("patchelf --set-rpath '$ORIGIN/../lib' \"${PTOAS_NATIVE_MODULE}\"", script) self.assertIn('"${PTOAS_DIST_DIR}/bin/ptoas" --version', script) - self.assertNotIn("PTOAS_PYTHON_ROOT_DIST_DIR", script) - self.assertNotIn("export PTOAS_PYTHON_ROOT", script) - self.assertNotIn('cat > "${PTOAS_DIST_DIR}/ptoas"', script) + self.assertNotIn("ptoas-native", script) + self.assertNotIn("ptoas.so", script) - def test_macos_dist_stages_install_tree_python_root(self): + def test_macos_dist_packages_python_wrapper_and_native_extension(self): script = COLLECT_DIST_MAC_SCRIPT.read_text(encoding="utf-8") - self.assertIn('PTOAS_SHARED_MODULE="${PTO_INSTALL_DIR}/lib/ptoas.so"', script) - self.assertIn('PTOAS_WRAPPER_PKG_SRC_DIR="${PTO_INSTALL_DIR}/ptoas"', script) - self.assertIn('PTOAS_WRAPPER_PKG_DIST_DIR="${PTOAS_DIST_DIR}/ptoas"', script) - self.assertIn('PTOAS_WHEEL_BOOTSTRAP_SRC="${PTO_INSTALL_DIR}/ptoas_wheel_bootstrap.py"', script) - self.assertIn('cp "${PTOAS_WHEEL_BOOTSTRAP_SRC}" "${PTOAS_WHEEL_BOOTSTRAP_DIST_PATH}"', script) - self.assertIn('collect_dylibs "${PTOAS_SHARED_MODULE_DIST_PATH}"', script) - self.assertIn('test -f "${PTOAS_WRAPPER_PKG_DIST_DIR}/_runtime_entry.py"', script) - self.assertIn('test -f "${PTOAS_WHEEL_BOOTSTRAP_DIST_PATH}"', script) + self.assertIn('PTOAS_BIN="${PTO_INSTALL_DIR}/bin/ptoas"', script) + self.assertIn('PTOAS_PACKAGE_SRC_DIR="${PTO_INSTALL_DIR}/ptoas"', script) + self.assertIn('PTOAS_NATIVE_MODULE="$(find "${PTOAS_PACKAGE_DIST_DIR}"', script) + self.assertIn('collect_dylibs "${PTOAS_NATIVE_MODULE}"', script) self.assertIn('"${PTOAS_DIST_DIR}/bin/ptoas" --version', script) - self.assertNotIn('cat > "${PTOAS_DIST_DIR}/ptoas"', script) + self.assertNotIn("ptoas-native", script) + self.assertNotIn("ptoas.so", script) def test_dist_archives_use_bin_entrypoint(self): linux_workflow = BUILD_WHEEL_WORKFLOW.read_text(encoding="utf-8") @@ -82,15 +87,11 @@ def test_dist_archives_use_bin_entrypoint(self): self.assertIn('"$TEST_DIR/extracted/bin/ptoas" --version', mac_workflow) self.assertNotIn('"$TEST_DIR/extracted/ptoas" --version', mac_workflow) - def test_wheel_import_smoke_covers_polluted_ptoas_version(self): + def test_wheel_import_smoke_imports_native_extension(self): script = (REPO_ROOT / "docker" / "test_wheel_imports.sh").read_text(encoding="utf-8") - self.assertIn("Testing installed ptoas console entry under polluted Python and LLVM paths...", script) - self.assertIn("POLLUTED_ENV_DIR=\"${TEST_TMPDIR}/polluted-env\"", script) - self.assertIn('PYTHONPATH="${POLLUTED_ENV_DIR}"', script) - self.assertIn("POLLUTED_PTOAS_VERSION_OUTPUT", script) - self.assertIn('"/tmp/polluted-llvm"', script) - self.assertIn('"/tmp/polluted-dylib"', script) + self.assertIn("from ptoas import _native", script) + self.assertNotIn("POLLUTED_PTOAS_VERSION_OUTPUT", script) if __name__ == "__main__": diff --git a/test/python/test_ptoas_build_backend.py b/test/python/test_ptoas_build_backend.py deleted file mode 100644 index 09ea739f14..0000000000 --- a/test/python/test_ptoas_build_backend.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -import tempfile -import unittest -import zipfile -from pathlib import Path -from unittest import mock - -import _ptoas_build_backend as build_backend - - -class PtoasBuildBackendTests(unittest.TestCase): - def test_prepare_metadata_honors_package_name_override(self): - with tempfile.TemporaryDirectory() as temp_dir, mock.patch.dict( - build_backend.os.environ, - { - "PTOAS_PYTHON_PACKAGE_NAME": "ptoas-vmi", - "PTOAS_PYTHON_PACKAGE_VERSION": "0.1.0", - }, - clear=False, - ): - dist_info_name = build_backend.prepare_metadata_for_build_wheel(temp_dir) - - dist_info = Path(temp_dir) / dist_info_name - metadata = (dist_info / "METADATA").read_text(encoding="utf-8") - - self.assertEqual(dist_info_name, "ptoas_vmi-0.1.0.dist-info") - self.assertIn("Name: ptoas-vmi", metadata) - self.assertIn("Version: 0.1.0", metadata) - - def test_cmake_build_driver_is_used_for_build_and_install(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - repo = temp_root / "repo" - build_dir = temp_root / "build" - llvm_build = temp_root / "llvm-build" - install_dir = temp_root / "install" - (repo / "cmake").mkdir(parents=True) - (install_dir / "lib").mkdir(parents=True) - (install_dir / "lib" / "ptoas.so").write_bytes(b"fake shared module") - - check_call_args = [] - - def fake_check_call(args): - check_call_args.append(args) - - with mock.patch.object(build_backend, "_REPO", repo), mock.patch.object( - build_backend, "_BUILD_DIR", build_dir - ), mock.patch.object( - build_backend, "_LLVM_BUILD_DIR", llvm_build - ), mock.patch.object( - build_backend, "_PTO_INSTALL_DIR", install_dir - ), mock.patch.object( - build_backend, - "_should_use_linux_hardening_cache", - return_value=False, - ), mock.patch.object( - build_backend.subprocess, - "check_output", - return_value="/tmp/pybind11-cmake", - ), mock.patch.object( - build_backend.subprocess, - "check_call", - side_effect=fake_check_call, - ), mock.patch.object( - build_backend, - "_assert_installed_ptodsl_payload", - ): - build_backend._cmake_configure_and_build() - - self.assertEqual( - check_call_args[-2], - ["cmake", "--build", str(build_dir)], - ) - self.assertEqual( - check_call_args[-1], - ["cmake", "--build", str(build_dir), "--target", "install"], - ) - - def test_installed_shared_module_must_be_non_empty(self): - with tempfile.TemporaryDirectory() as temp_dir: - install_dir = Path(temp_dir) / "install" - shared_module = install_dir / "lib" / "ptoas.so" - shared_module.parent.mkdir(parents=True) - shared_module.write_bytes(b"") - - with mock.patch.object(build_backend, "_PTO_INSTALL_DIR", install_dir): - with self.assertRaisesRegex(RuntimeError, "missing or empty"): - build_backend._assert_installed_ptoas_shared_module() - - def test_build_editable_routes_console_entry_through_wheel_bootstrap(self): - with tempfile.TemporaryDirectory() as temp_dir, mock.patch.dict( - build_backend.os.environ, - { - "PTOAS_PYTHON_PACKAGE_NAME": "ptoas", - "PTOAS_PYTHON_PACKAGE_VERSION": "0.1.0", - }, - clear=False, - ), mock.patch.object( - build_backend, - "_cmake_configure_and_build", - ) as cmake_build: - wheel_dir = Path(temp_dir) / "wheel" - wheel_dir.mkdir(parents=True, exist_ok=True) - - wheel_name = build_backend.build_editable(str(wheel_dir)) - wheel_path = wheel_dir / wheel_name - - with zipfile.ZipFile(wheel_path) as zf: - entry_points = zf.read("ptoas-0.1.0.dist-info/entry_points.txt").decode("utf-8") - - cmake_build.assert_called_once_with(skip_install=True) - - self.assertIn("ptoas=ptoas_wheel_bootstrap:main", entry_points) - self.assertNotIn("ptoas._launcher", entry_points) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/python/test_scikit_build_config.py b/test/python/test_scikit_build_config.py new file mode 100644 index 0000000000..3f013e71dc --- /dev/null +++ b/test/python/test_scikit_build_config.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import json +import subprocess +import sys +import tomllib +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PTOAS_PYPROJECT = REPO_ROOT / "pyproject.toml" +VMI_PROJECT_DIR = REPO_ROOT / "packaging" / "ptoas-vmi" +VMI_PYPROJECT = VMI_PROJECT_DIR / "pyproject.toml" +PTOAS_PACKAGE_INIT = REPO_ROOT / "ptodsl" / "ptoas" / "__init__.py" + + +def _read_pyproject(path: Path) -> dict: + with path.open("rb") as file: + return tomllib.load(file) + + +def _project_table(project_dir: Path) -> dict: + result = subprocess.run( + [ + sys.executable, + "-m", + "scikit_build_core.build", + "project-table", + ], + cwd=project_dir, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise AssertionError(result.stderr or result.stdout) + return json.loads(result.stdout) + + +class ScikitBuildConfigTests(unittest.TestCase): + def test_root_project_uses_standard_backend_and_console_script(self): + config = _read_pyproject(PTOAS_PYPROJECT) + + self.assertEqual( + config["build-system"]["build-backend"], + "scikit_build_core.build", + ) + self.assertIn("scikit-build-core>=0.12.2,<2", config["build-system"]["requires"]) + self.assertEqual(config["project"]["name"], "ptoas") + self.assertEqual(config["project"]["scripts"]["ptoas"], "ptoas._cli:main") + self.assertEqual( + config["tool"]["scikit-build"]["install"]["components"], + ["PTOAS_Python"], + ) + + def test_static_projects_resolve_expected_distribution_metadata(self): + ptoas = _project_table(REPO_ROOT) + vmi = _project_table(VMI_PROJECT_DIR) + + self.assertEqual((ptoas["name"], ptoas["version"]), ("ptoas", "0.51")) + self.assertEqual((vmi["name"], vmi["version"]), ("ptoas-vmi", "0.1.3")) + self.assertEqual(ptoas["scripts"]["ptoas"], "ptoas._cli:main") + self.assertEqual(vmi["scripts"]["ptoas"], "ptoas._cli:main") + + def test_projects_share_python_package_sources(self): + ptoas = _read_pyproject(PTOAS_PYPROJECT) + vmi = _read_pyproject(VMI_PYPROJECT) + + self.assertEqual( + ptoas["tool"]["scikit-build"]["wheel"]["packages"], + { + "ptodsl": "ptodsl/ptodsl", + "ptoas": "ptodsl/ptoas", + "tilelang_dsl": "tilelang-dsl/python/tilelang_dsl", + }, + ) + self.assertEqual( + vmi["tool"]["scikit-build"]["cmake"]["source-dir"], + "../..", + ) + + def test_editable_install_uses_backend_redirect_without_package_path_hacks(self): + ptoas = _read_pyproject(PTOAS_PYPROJECT) + package_init = PTOAS_PACKAGE_INIT.read_text(encoding="utf-8") + + self.assertEqual( + ptoas["tool"]["scikit-build"]["editable"]["mode"], + "redirect", + ) + self.assertNotIn("extend_path", package_init) + + def test_cmake_python_payload_uses_dedicated_install_component(self): + cmake_files = [ + REPO_ROOT / "CMakeLists.txt", + REPO_ROOT / "lib" / "Bindings" / "Python" / "CMakeLists.txt", + REPO_ROOT / "tools" / "ptoas" / "CMakeLists.txt", + ] + combined = "\n".join(path.read_text(encoding="utf-8") for path in cmake_files) + + self.assertIn("COMPONENT PTOAS_Python", combined) + self.assertIn("if(SKBUILD)", combined) + self.assertIn('DIRECTORY "${MLIR_PYTHON_PACKAGE_DIR}/mlir"', combined) + + def test_legacy_wheel_builders_are_removed(self): + self.assertFalse((REPO_ROOT / "_ptoas_build_backend.py").exists()) + self.assertFalse((REPO_ROOT / "docker" / "create_wheel.sh").exists()) + self.assertFalse((REPO_ROOT / "docker" / "setup.py").exists()) + self.assertFalse((REPO_ROOT / "docker" / "setup_mac.py").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/python/test_validate_wheel_payload.py b/test/python/test_validate_wheel_payload.py index 5b02a6e95e..318d1a6fde 100644 --- a/test/python/test_validate_wheel_payload.py +++ b/test/python/test_validate_wheel_payload.py @@ -11,12 +11,12 @@ import tempfile import unittest import zipfile +import importlib.machinery from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] VALIDATOR = REPO_ROOT / "docker" / "validate_wheel_payload.py" -CREATE_WHEEL = REPO_ROOT / "docker" / "create_wheel.sh" LINUX_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "build_wheel.yml" MAC_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "build_wheel_mac.yml" WHEEL_IMPORTS = REPO_ROOT / "docker" / "test_wheel_imports.sh" @@ -29,23 +29,25 @@ def _make_wheel( self, root: Path, *, - include_runtime_so: bool, - include_runtime_entry: bool = True, - include_bootstrap: bool = True, - entry_points_text: str = "[console_scripts]\nptoas=ptoas_wheel_bootstrap:main\n", + include_native_module: bool, + include_cli: bool = True, + include_tileops: bool = True, + entry_points_text: str = "[console_scripts]\nptoas=ptoas._cli:main\n", wheel_stem: str = "ptoas", dist_info_stem: str = "ptoas", ) -> Path: wheel = root / f"{wheel_stem}-1.2.3-cp311-cp311-linux_x86_64.whl" with zipfile.ZipFile(wheel, "w") as zf: - if include_bootstrap: - zf.writestr("ptoas_wheel_bootstrap.py", "") zf.writestr("ptoas/__init__.py", "") - zf.writestr("ptoas/_launcher.py", "") - if include_runtime_entry: - zf.writestr("ptoas/_runtime_entry.py", "") - if include_runtime_so: - zf.writestr("ptoas/_runtime/lib/ptoas.so", "fake") + if include_cli: + zf.writestr("ptoas/_cli.py", "") + if include_tileops: + zf.writestr( + "ptoas/_runtime/share/ptoas/TileOps/__init__.py", "" + ) + if include_native_module: + suffix = importlib.machinery.EXTENSION_SUFFIXES[0] + zf.writestr(f"ptoas/_native{suffix}", "fake") zf.writestr( f"{dist_info_stem}-1.2.3.dist-info/entry_points.txt", entry_points_text, @@ -54,7 +56,7 @@ def _make_wheel( def test_validator_accepts_current_runtime_payload_layout(self): with tempfile.TemporaryDirectory() as temp_dir: - wheel = self._make_wheel(Path(temp_dir), include_runtime_so=True) + wheel = self._make_wheel(Path(temp_dir), include_native_module=True) result = subprocess.run( ["python3", str(VALIDATOR), str(wheel)], capture_output=True, @@ -65,9 +67,9 @@ def test_validator_accepts_current_runtime_payload_layout(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("validated wheel payload and launcher contract", result.stdout) - def test_validator_rejects_missing_runtime_shared_module(self): + def test_validator_rejects_missing_native_module(self): with tempfile.TemporaryDirectory() as temp_dir: - wheel = self._make_wheel(Path(temp_dir), include_runtime_so=False) + wheel = self._make_wheel(Path(temp_dir), include_native_module=False) result = subprocess.run( ["python3", str(VALIDATOR), str(wheel)], capture_output=True, @@ -76,14 +78,14 @@ def test_validator_rejects_missing_runtime_shared_module(self): ) self.assertNotEqual(result.returncode, 0) - self.assertIn("ptoas/_runtime/lib/ptoas.so", result.stderr) + self.assertIn("ptoas._native", result.stderr) - def test_validator_rejects_missing_runtime_entry_module(self): + def test_validator_rejects_missing_cli_module(self): with tempfile.TemporaryDirectory() as temp_dir: wheel = self._make_wheel( Path(temp_dir), - include_runtime_so=True, - include_runtime_entry=False, + include_native_module=True, + include_cli=False, ) result = subprocess.run( ["python3", str(VALIDATOR), str(wheel)], @@ -93,14 +95,14 @@ def test_validator_rejects_missing_runtime_entry_module(self): ) self.assertNotEqual(result.returncode, 0) - self.assertIn("ptoas/_runtime_entry.py", result.stderr) + self.assertIn("ptoas/_cli.py", result.stderr) - def test_validator_rejects_missing_wheel_bootstrap_module(self): + def test_validator_rejects_missing_packaged_tileops(self): with tempfile.TemporaryDirectory() as temp_dir: wheel = self._make_wheel( Path(temp_dir), - include_runtime_so=True, - include_bootstrap=False, + include_native_module=True, + include_tileops=False, ) result = subprocess.run( ["python3", str(VALIDATOR), str(wheel)], @@ -110,14 +112,14 @@ def test_validator_rejects_missing_wheel_bootstrap_module(self): ) self.assertNotEqual(result.returncode, 0) - self.assertIn("ptoas_wheel_bootstrap.py", result.stderr) + self.assertIn("TileOps/__init__.py", result.stderr) - def test_validator_rejects_legacy_ptoas_launcher_entrypoint(self): + def test_validator_rejects_legacy_bootstrap_entrypoint(self): with tempfile.TemporaryDirectory() as temp_dir: wheel = self._make_wheel( Path(temp_dir), - include_runtime_so=True, - entry_points_text="[console_scripts]\nptoas=ptoas._launcher:main\n", + include_native_module=True, + entry_points_text="[console_scripts]\nptoas=ptoas_wheel_bootstrap:main\n", ) result = subprocess.run( ["python3", str(VALIDATOR), str(wheel)], @@ -127,14 +129,14 @@ def test_validator_rejects_legacy_ptoas_launcher_entrypoint(self): ) self.assertNotEqual(result.returncode, 0) - self.assertIn("ptoas_wheel_bootstrap:main", result.stderr) + self.assertIn("ptoas._cli:main", result.stderr) def test_validator_accepts_normalized_entrypoint_spacing(self): with tempfile.TemporaryDirectory() as temp_dir: wheel = self._make_wheel( Path(temp_dir), - include_runtime_so=True, - entry_points_text="[console_scripts]\nptoas = ptoas_wheel_bootstrap:main\n", + include_native_module=True, + entry_points_text="[console_scripts]\nptoas = ptoas._cli:main\n", ) result = subprocess.run( ["python3", str(VALIDATOR), str(wheel)], @@ -149,7 +151,7 @@ def test_validator_accepts_vmi_distribution_name(self): with tempfile.TemporaryDirectory() as temp_dir: wheel = self._make_wheel( Path(temp_dir), - include_runtime_so=True, + include_native_module=True, wheel_stem="ptoas_vmi", dist_info_stem="ptoas_vmi", ) @@ -196,7 +198,8 @@ def test_release_workflows_separate_cli_and_wheel_versions(self): ) self.assertIn('PTOAS_CLI_VERSION="vmi ${PTOAS_VERSION}"', linux_workflow) self.assertIn('PTOAS_RELEASE_VERSION_OVERRIDE="${PTOAS_CLI_VERSION}"', linux_workflow) - self.assertIn('export PTOAS_PYTHON_PACKAGE_VERSION="${PTOAS_VERSION}"', linux_workflow) + self.assertIn('PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}/packaging/ptoas-vmi"', linux_workflow) + self.assertIn('SKBUILD_BUILD_DIR="${PTO_BUILD_DIR}"', linux_workflow) self.assertIn("workflow_dispatch:", mac_workflow) self.assertIn("release_kind:", mac_workflow) @@ -212,19 +215,8 @@ def test_release_workflows_separate_cli_and_wheel_versions(self): ) self.assertIn('PTOAS_CLI_VERSION="vmi ${PTOAS_VERSION}"', mac_workflow) self.assertIn('PTOAS_RELEASE_VERSION_OVERRIDE="${PTOAS_CLI_VERSION}"', mac_workflow) - self.assertIn('export PTOAS_PYTHON_PACKAGE_VERSION="${PTOAS_VERSION}"', mac_workflow) - - def test_create_wheel_script_validates_package_name(self): - script = CREATE_WHEEL.read_text(encoding="utf-8") - - self.assertIn( - "if [[ ! \"${PTOAS_PYTHON_PACKAGE_NAME}\" =~ ^[A-Za-z0-9]+([-_.][A-Za-z0-9]+)*$ ]]; then", - script, - ) - self.assertIn( - "Error: invalid PTOAS_PYTHON_PACKAGE_NAME", - script, - ) + self.assertIn('PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}/packaging/ptoas-vmi"', mac_workflow) + self.assertIn('SKBUILD_BUILD_DIR="${PTO_BUILD_DIR}"', mac_workflow) def test_wheel_imports_script_keeps_clean_env_ptoas_smoke(self): script = WHEEL_IMPORTS.read_text(encoding="utf-8") diff --git a/tools/ptoas/CMakeLists.txt b/tools/ptoas/CMakeLists.txt index 1e2918bd5a..0d4ede0c40 100644 --- a/tools/ptoas/CMakeLists.txt +++ b/tools/ptoas/CMakeLists.txt @@ -6,11 +6,6 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - set(PTOAS_RUNTIME_SOURCES ptoas.cpp driver.cpp @@ -45,18 +40,12 @@ function(ptoas_configure_runtime_target target_name) PTOAS_DEFAULT_TILELANG_PATH="${CMAKE_SOURCE_DIR}/lib/TileOps" PTOAS_DEFAULT_TILELANG_PKG_PATH="${CMAKE_SOURCE_DIR}/tilelang-dsl/python" PTOAS_DEFAULT_PTODSL_PKG_PATH="${CMAKE_SOURCE_DIR}/ptodsl" - ${ARGN} ) - target_link_libraries(${target_name} PRIVATE - # === 你的本地库 (请确认 lib/PTO/*/CMakeLists.txt 里的定义名) === - PTOIR # 替代 MLIRPTODialect - PTOTransforms # 替代 MLIRPTOTransforms + target_link_libraries(${target_name} PUBLIC + PTOIR + PTOTransforms ptobc_lib - - # === MLIR 基础设施 === - MLIRMlirOptMain # [重要] 包含了 main 函数入口逻辑，通常必须链接 - - # === MLIR 标准库 === + MLIRMlirOptMain MLIRBufferizationTransforms MLIRArithTransforms MLIRTensorTransforms @@ -86,43 +75,54 @@ function(ptoas_configure_runtime_target target_name) ) endfunction() -add_library(ptoas_runtime SHARED ${PTOAS_RUNTIME_SOURCES}) -# Keep the shared runtime's compile mode aligned with LLVM/MLIR so RTTI, -# exceptions, and visibility expectations match the libraries it links to. -llvm_update_compile_flags(ptoas_runtime) +# Keep the LLVM-based driver sources under LLVM's RTTI/exception compile mode. +# The pybind11 translation unit must retain the Python extension's normal RTTI +# and exception mode, so it is compiled by the module target separately. +add_library(ptoas_runtime_objects OBJECT ${PTOAS_RUNTIME_SOURCES}) +llvm_update_compile_flags(ptoas_runtime_objects) +ptoas_configure_runtime_target(ptoas_runtime_objects) + +# Build the normal Python extension used by wheel, build-tree, install-tree, +# and Python-based standalone distributions. Linking the object library pulls +# in the driver implementation and propagates its native link dependencies. +pybind11_add_module(ptoas_runtime MODULE NativeModule.cpp) +target_link_libraries(ptoas_runtime PRIVATE ptoas_runtime_objects) set_target_properties(ptoas_runtime PROPERTIES - OUTPUT_NAME "ptoas" - PREFIX "" -) -ptoas_configure_runtime_target(ptoas_runtime - PTOAS_BUILD_SHARED_LIBRARY=1 + OUTPUT_NAME "_native" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/python-module" ) -set(PTOAS_BUILD_RUNTIME_STAGING_DIR "${CMAKE_BINARY_DIR}/runtime-staging") -add_custom_target(ptoas_runtime_staging ALL - COMMAND ${CMAKE_COMMAND} -E make_directory - "${PTOAS_BUILD_RUNTIME_STAGING_DIR}/lib" - "${PTOAS_BUILD_RUNTIME_STAGING_DIR}/share/ptoas" +set(PTOAS_BUILD_PYTHON_PACKAGE_DIR "${CMAKE_BINARY_DIR}/python/ptoas") +add_custom_target(PTOASPythonPackage ALL + COMMAND ${CMAKE_COMMAND} -E remove_directory + "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}" + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${CMAKE_SOURCE_DIR}/ptodsl/ptoas" + "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}" COMMAND ${CMAKE_COMMAND} -E copy_if_different "$" - "${PTOAS_BUILD_RUNTIME_STAGING_DIR}/lib/ptoas.so" - COMMAND ${CMAKE_COMMAND} -E remove_directory - "${PTOAS_BUILD_RUNTIME_STAGING_DIR}/share/ptoas/TileOps" + "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}/$" + COMMAND ${CMAKE_COMMAND} -E make_directory + "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}/_runtime/share/ptoas" COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_SOURCE_DIR}/lib/TileOps" - "${PTOAS_BUILD_RUNTIME_STAGING_DIR}/share/ptoas/TileOps" + "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}/_runtime/share/ptoas/TileOps" + COMMAND ${CMAKE_COMMAND} -E remove_directory + "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}/_runtime/share/ptoas/TileOps/__pycache__" DEPENDS ptoas_runtime - COMMENT "Staging ptoas runtime into build/runtime-staging" + COMMENT "Staging the ptoas Python package" VERBATIM ) - if(TARGET PTODSLPackage) - # The build-tree launcher imports ptoas._runtime_entry from build/python. - # Keep that package staging coupled to every runtime staging entrypoint, - # including partial builds such as tools/ptoas/all. - add_dependencies(ptoas_runtime_staging PTODSLPackage) + add_dependencies(PTOASPythonPackage PTODSLPackage) endif() +# Keep the historical aggregate target name for existing developer commands; +# the actual runtime payload now lives in the staged Python package. +add_custom_target(ptoas_runtime_staging ALL + DEPENDS PTOASPythonPackage +) + if(TARGET ptoas_runtime_deps) add_dependencies(ptoas_runtime_deps ptoas_runtime_staging) endif() @@ -133,6 +133,14 @@ install(PROGRAMS "${PTOAS_WRAPPER_BINARY}" ) install(TARGETS ptoas_runtime - LIBRARY DESTINATION lib - COMPONENT PTOAS_Runtime + LIBRARY DESTINATION ptoas + COMPONENT PTOAS_Python +) + +install( + DIRECTORY "${CMAKE_SOURCE_DIR}/lib/TileOps" + DESTINATION "ptoas/_runtime/share/ptoas" + COMPONENT PTOAS_Python + PATTERN "__pycache__" EXCLUDE + PATTERN "*.pyc" EXCLUDE ) diff --git a/tools/ptoas/NativeModule.cpp b/tools/ptoas/NativeModule.cpp new file mode 100644 index 0000000000..54bfc26e41 --- /dev/null +++ b/tools/ptoas/NativeModule.cpp @@ -0,0 +1,37 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#include "ptoas.h" + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +#include +#include + +namespace py = pybind11; + +namespace { + +int runPTOASFromPython(const std::vector &arguments) { + std::vector storage = arguments; + std::vector argv; + argv.reserve(storage.size()); + for (std::string &argument : storage) + argv.push_back(argument.data()); + + py::gil_scoped_release release; + return mlir::pto::runPTOAS(static_cast(argv.size()), argv.data()); +} + +} // namespace + +PYBIND11_MODULE(_native, module) { + module.doc() = "Native PTOAS command-line entrypoint"; + module.def("main", &runPTOASFromPython, py::arg("argv")); +} diff --git a/tools/ptoas/driver.cpp b/tools/ptoas/driver.cpp index 6caa23ce33..5c6e81b79e 100644 --- a/tools/ptoas/driver.cpp +++ b/tools/ptoas/driver.cpp @@ -1284,12 +1284,6 @@ static int runPTOASDriver(int argc, char **argv) { return 1; } -extern "C" int ptoas_entrypoint(int argc, char **argv) { +int mlir::pto::runPTOAS(int argc, char **argv) { return runPTOASDriver(argc, argv); } - -#ifndef PTOAS_BUILD_SHARED_LIBRARY -int main(int argc, char **argv) { - return runPTOASDriver(argc, argv); -} -#endif diff --git a/tools/ptoas/ptoas.h b/tools/ptoas/ptoas.h index 92ce03ecee..ba422628a8 100644 --- a/tools/ptoas/ptoas.h +++ b/tools/ptoas/ptoas.h @@ -125,8 +125,8 @@ void registerPTOASDialects(DialectRegistry ®istry); void registerPTOASPassesAndCLOptions(); void loadPTOASDialects(MLIRContext &context); -// Shared-library entrypoint for wheel-installed in-process launching. -extern "C" int ptoas_entrypoint(int argc, char **argv); +// Reusable driver entry shared by the Python extension and standalone CLI. +int runPTOAS(int argc, char **argv); // Attach textual-.pto SSA name hints (function args, block args, op results) // to the parsed module's Locations as debug metadata. Called by the driver diff --git a/tools/ptoas/ptoas_wrapper.py b/tools/ptoas/ptoas_wrapper.py index 3dadc46d16..fb07afaf9b 100644 --- a/tools/ptoas/ptoas_wrapper.py +++ b/tools/ptoas/ptoas_wrapper.py @@ -7,12 +7,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -"""Build/install-tree wrapper for the `ptoas` entrypoint. - -This script is configured into the build tree and installed as `bin/ptoas`. -It bootstraps the Python package root for tree layouts, then delegates to the -shared in-process runtime entry contract in `ptoas._runtime_entry`. -""" +"""Add a CMake tree's Python root, then run the regular PTOAS CLI module.""" from __future__ import annotations @@ -34,115 +29,46 @@ def _resolve_wrapper_path(argv0: str | None = None) -> Path: raise SystemExit(f"unable to locate the installed ptoas wrapper: {candidate}") -def _build_tree_root(wrapper: Path) -> Path | None: - if len(wrapper.parents) < 3: - return None - return wrapper.parents[2] - - -def _install_tree_root(wrapper: Path) -> Path | None: - if len(wrapper.parents) < 2: - return None - return wrapper.parents[1] - - def _is_build_tree_wrapper(wrapper: Path) -> bool: return len(wrapper.parents) >= 2 and wrapper.parents[1].name == "tools" def _require_python_root(python_root: Path, *, context: str) -> Path: - helper = python_root / "ptoas" / "_runtime_entry.py" + helper = python_root / "ptoas" / "_cli.py" if helper.is_file(): return python_root.resolve() raise SystemExit( f"unable to locate the {context} ptoas Python package root: " - f"expected {python_root}/ptoas/_runtime_entry.py" + f"expected {python_root}/ptoas/_cli.py" ) -def _require_runtime_root(runtime_root: Path) -> Path: - shared_module = runtime_root / "lib" / "ptoas.so" - tileops_dir = runtime_root / "share" / "ptoas" / "TileOps" - if shared_module.is_file() and shared_module.stat().st_size > 0 and tileops_dir.is_dir(): - return runtime_root.resolve() - raise SystemExit( - "unable to locate the ptoas runtime tree: expected " - f"{runtime_root}/lib/ptoas.so and {runtime_root}/share/ptoas/TileOps" - ) - - -def _resolve_build_tree_layout(wrapper: Path): - from ptoas import _runtime_entry - - build_root = _build_tree_root(wrapper) - if build_root is None: - raise SystemExit("unable to locate the build-tree layout for ptoas") - - python_root = _require_python_root(build_root / "python", context="build-tree") - runtime_root = _require_runtime_root(build_root / "runtime-staging") - - return _runtime_entry.PTOASRuntimeLayout( - wrapper=wrapper, - runtime_root=runtime_root, - python_root=python_root, - shared_module=(runtime_root / "lib" / "ptoas.so").resolve(), - tileops_dir=(runtime_root / "share" / "ptoas" / "TileOps").resolve(), - isolated_env=False, - ) - - -def _resolve_install_tree_layout(wrapper: Path): - from ptoas import _runtime_entry - - install_root = _install_tree_root(wrapper) - if install_root is None: - raise SystemExit("unable to locate the install-tree layout for ptoas") - - python_root = _require_python_root(install_root, context="install-tree") - runtime_root = _require_runtime_root(install_root) - - return _runtime_entry.PTOASRuntimeLayout( - wrapper=wrapper, - runtime_root=runtime_root, - python_root=python_root, - shared_module=(runtime_root / "lib" / "ptoas.so").resolve(), - tileops_dir=(runtime_root / "share" / "ptoas" / "TileOps").resolve(), - isolated_env=False, - ) - - -def _bootstrap_python_path(wrapper: Path) -> None: +def _bootstrap_python_path(wrapper: Path) -> Path: if _is_build_tree_wrapper(wrapper): - build_root = _build_tree_root(wrapper) - if build_root is not None: - root = build_root / "python" - _require_python_root(root, context="build-tree") - root_text = str(root) - if root_text not in sys.path: - sys.path.insert(0, root_text) - return - raise SystemExit("unable to locate the build-tree layout for ptoas") - - install_root = _install_tree_root(wrapper) - if install_root is not None: - _require_python_root(install_root, context="install-tree") - root_text = str(install_root) - if root_text not in sys.path: - sys.path.insert(0, root_text) - return - raise SystemExit("unable to locate the ptoas Python package root for the build/install-tree wrapper") + if len(wrapper.parents) < 3: + raise SystemExit("unable to locate the build-tree root for ptoas") + python_root = _require_python_root( + wrapper.parents[2] / "python", context="build-tree" + ) + else: + if len(wrapper.parents) < 2: + raise SystemExit("unable to locate the install-tree root for ptoas") + python_root = _require_python_root( + wrapper.parents[1], context="install-tree" + ) + + python_root_text = str(python_root) + if python_root_text not in sys.path: + sys.path.insert(0, python_root_text) + return python_root def main() -> None: wrapper = _resolve_wrapper_path() _bootstrap_python_path(wrapper) - from ptoas import _runtime_entry + from ptoas import _cli - if _is_build_tree_wrapper(wrapper): - layout = _resolve_build_tree_layout(wrapper) - else: - layout = _resolve_install_tree_layout(wrapper) - raise SystemExit(_runtime_entry.launch(layout, sys.argv[1:])) + raise SystemExit(_cli.launch(sys.argv[1:], wrapper=wrapper)) if __name__ == "__main__": From 4a80bc39c8e438b4a4f26fa1ee0df648c30f28ce Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Wed, 22 Jul 2026 17:57:45 +0800 Subject: [PATCH 02/32] Remove runtime path bootstrap fallbacks --- docs/designs/ptoas-python-launcher-layout.md | 26 +++- ptodsl/README.md | 18 ++- ptodsl/ptoas/_cli.py | 17 -- ptodsl/ptodsl/_bootstrap.py | 145 ------------------ ptodsl/ptodsl/_builtin_vector.py | 1 - ptodsl/ptodsl/_context.py | 30 ++++ ptodsl/ptodsl/_control_flow.py | 1 - ptodsl/ptodsl/_ops.py | 1 - ptodsl/ptodsl/_runtime/codegen.py | 2 +- ptodsl/ptodsl/_runtime/toolchain.py | 26 +--- ptodsl/ptodsl/_source_loader.py | 2 +- ptodsl/ptodsl/_surface_types.py | 2 - ptodsl/ptodsl/_tracing/runtime.py | 2 +- ptodsl/ptodsl/_types.py | 2 - ptodsl/ptodsl/scalar.py | 1 - ptodsl/ptodsl/tilelib/_render_runtime.py | 2 +- ptodsl/tests/test_ast_rewrite_example_ir.py | 2 +- ptodsl/tests/test_bootstrap_runtime_env.py | 92 ----------- ptodsl/tests/test_context.py | 23 +++ ptodsl/tests/test_docs_as_test.py | 2 +- .../test_flash_attention_demo_compile.py | 2 +- ptodsl/tests/test_jit_compile.py | 2 +- ptodsl/tests/test_ptoas_cli.py | 6 +- ptodsl/tests/test_ptoas_frontend_verify.py | 2 +- ptodsl/tests/test_runtime_toolchain.py | 36 +++-- ptodsl/tests/test_tileop.py | 2 +- ptodsl/tests/test_vector_cube_ops.py | 2 +- tools/ptoas/ptoas_wrapper.py | 11 +- 28 files changed, 124 insertions(+), 336 deletions(-) delete mode 100644 ptodsl/ptodsl/_bootstrap.py create mode 100644 ptodsl/ptodsl/_context.py delete mode 100644 ptodsl/tests/test_bootstrap_runtime_env.py create mode 100644 ptodsl/tests/test_context.py diff --git a/docs/designs/ptoas-python-launcher-layout.md b/docs/designs/ptoas-python-launcher-layout.md index 382cf1cd4e..171b5b7187 100644 --- a/docs/designs/ptoas-python-launcher-layout.md +++ b/docs/designs/ptoas-python-launcher-layout.md @@ -20,10 +20,9 @@ The wheel, build-tree, and install-tree launchers use the standard Python console-script and native-extension model: ```text -console script or CMake tree wrapper - -> ptoas._cli.main() - -> import ptoas._native - -> ptoas._native.main(argv) +wheel console script -> ptoas._cli.main() +CMake tree wrapper -> add its own Python root -> ptoas._cli.launch() +both -> import ptoas._native -> ptoas._native.main(argv) ``` `ptoas._native` is built with `pybind11_add_module`. CMake and Python therefore @@ -93,6 +92,11 @@ The install-tree wrapper resolves only files under the same prefix: The install-tree wrapper only adds `` to `sys.path`, then delegates to the same `ptoas._cli` module used by wheels. +Both tree wrappers are CMake packaging adapters, not general runtime discovery +helpers. Each wrapper adds exactly the Python root belonging to its own tree; +it never scans repositories, neighboring build directories, or environment +variables for another installation. + ## Standalone Archive Layout Standalone compiler archives contain the installed Python wrapper and package: @@ -111,3 +115,17 @@ extension ABI. `bin/ptoas` adds the archive root to `sys.path`, then uses the same `ptoas._cli -> ptoas._native` path as the install tree. Linux archives set the extension runtime path to `$ORIGIN/../lib`; macOS archives rewrite the extension's install names relative to its package directory. + +## PTODSL and TileLib Runtime + +PTODSL imports MLIR and the PTO dialect through normal Python package +resolution. Wheel and editable installs provide those packages through their +declared installation layout. CTest and direct developer-tree runs must set an +explicit matching `PYTHONPATH`; PTODSL does not guess repository, LLVM build, +or PTOAS install paths at import time. + +TileOp expansion remains a lazy, separate daemon process. The PTOAS CLI passes +the packaged PTODSL root and the active Python executable to the native driver, +which starts the daemon only when expansion is required. Keeping the daemon out +of the compiler process also prevents independently packaged MLIR/LLVM Python +bindings from registering runtime state in the native PTOAS process. diff --git a/ptodsl/README.md b/ptodsl/README.md index db05638333..7a5ecaf333 100644 --- a/ptodsl/README.md +++ b/ptodsl/README.md @@ -18,7 +18,7 @@ ptodsl/ │ ├── __init__.py # exports: pto, scalar │ ├── pto.py # main PTO DSL namespace │ ├── scalar.py # top-level scalar.* helper namespace -│ ├── _bootstrap.py # MLIR path setup + context factory +│ ├── _context.py # MLIR context factory │ ├── _types.py # lazy dtype descriptors and type constructors │ ├── _ops.py # PTO operation wrappers │ ├── _control_flow.py # for_, if_, yield_ context managers @@ -62,7 +62,11 @@ python3 -c "import ptodsl; from ptodsl import pto, scalar; print(ptodsl.__file__ Not supported: - `cd ptodsl && pip install -e .` -- repo-walk / `PYTHONPATH` / `sys.path` repair just to locate the `ptodsl` package +- runtime repository walks or `sys.path` repair to locate MLIR/PTO bindings + +Direct CTest/developer-tree execution must provide the matching MLIR and PTO +Python roots explicitly through the test environment or `PYTHONPATH`; PTODSL +does not search nearby build and install directories at import time. Artifact boundary: @@ -89,11 +93,11 @@ ptoas --pto-arch=a5 --pto-backend=vpto --emit-vpto \ input.pto -o - ``` -The source-tree build bakes in `$PTOAS_REPO_ROOT/ptodsl` as the package root. -The `PTODSL_PYTHON_ROOT` environment variable from `scripts/ptoas_env.sh` -overrides that default. Use `--ptodsl-pkg-path=/path/to/package/root` for an -explicit command-line override. PTODSL daemon failures are reported as errors -and never fall back to the TileLang implementation. Use +Wheel and CMake-tree launchers pass the Python root containing their own +installed or staged `ptodsl` package to the native driver. Use +`--ptodsl-pkg-path=/path/to/package/root` for an explicit command-line +override. PTODSL daemon failures are reported as errors and never fall back to +the TileLang implementation. Use `--tile-lib-backend=tilelang` only when explicitly comparing against the legacy TileLangDSL template library. diff --git a/ptodsl/ptoas/_cli.py b/ptodsl/ptoas/_cli.py index dbc2ce2468..7cb3132679 100644 --- a/ptodsl/ptoas/_cli.py +++ b/ptodsl/ptoas/_cli.py @@ -11,7 +11,6 @@ from __future__ import annotations import os -import shutil import sys from pathlib import Path from typing import Sequence @@ -23,10 +22,6 @@ def _resolve_wrapper_path(argv0: str | None = None) -> Path: if wrapper.exists(): return wrapper.resolve() - found = shutil.which(wrapper.name or "ptoas") - if found: - return Path(found).resolve() - raise SystemExit(f"unable to locate the active ptoas entry point: {candidate}") @@ -61,16 +56,6 @@ def _has_cli_option(arguments: Sequence[str], option: str) -> bool: ) -def _prepend_path(name: str, value: Path) -> None: - current = os.environ.get(name, "") - parts = [part for part in current.split(os.pathsep) if part] - rendered = str(value) - if rendered in parts: - parts.remove(rendered) - parts.insert(0, rendered) - os.environ[name] = os.pathsep.join(parts) - - def launch(user_args: Sequence[str], *, wrapper: Path | None = None) -> int: native_module = _load_native_module() python_root, tileops_dir = _resolve_runtime_paths(native_module) @@ -78,8 +63,6 @@ def launch(user_args: Sequence[str], *, wrapper: Path | None = None) -> int: os.environ["PTOAS_BIN"] = str(wrapper) os.environ["PTOAS_PYTHON_EXE"] = sys.executable - _prepend_path("PATH", wrapper.parent) - argv = [str(wrapper)] if not _has_cli_option(user_args, "--tilelang-path"): argv.extend(["--tilelang-path", str(tileops_dir)]) diff --git a/ptodsl/ptodsl/_bootstrap.py b/ptodsl/ptodsl/_bootstrap.py deleted file mode 100644 index 86d8954d48..0000000000 --- a/ptodsl/ptodsl/_bootstrap.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. -""" -MLIR path bootstrap and context factory. - -Discovers local LLVM MLIR Python bindings plus PTO Python dialect artifacts so -that ``ptodsl`` can import ``mlir`` / ``mlir.dialects.pto`` directly from a -developer workspace without requiring the caller to pre-seed ``PYTHONPATH``. -When the current interpreter already has a complete installed MLIR/PTO stack, -leave ``sys.path`` untouched so wheel installs do not accidentally mix in a -developer checkout from environment variables or nearby directories. -""" - -import importlib.util -import os -import sys -from pathlib import Path - - -def _path_has(path: Path, relative: str) -> bool: - try: - return (path / relative).exists() - except OSError: - return False - - -def _pythonpath_already_provides_mlir_and_pto() -> bool: - """Return true when the caller has already configured compatible MLIR/PTO roots. - - Lit preserves ``PYTHONPATH`` but not the convenience environment variables - used by ``ptoas_env.sh``. In that case we should not prepend repository - default fallbacks like ``build/python`` / ``install`` because they may belong - to an older build tree and can mix Python/native MLIR bindings. - """ - has_mlir_core = False - has_pto_dialect = False - for entry in sys.path: - if not entry: - continue - path = Path(entry) - has_mlir_core = has_mlir_core or _path_has(path, "mlir/ir.py") - has_pto_dialect = has_pto_dialect or _path_has( - path, "mlir/dialects/pto.py" - ) - if has_mlir_core and has_pto_dialect: - return True - return False - - -def _candidate_python_roots() -> list[Path]: - here = Path(__file__).resolve() - repo_root = here.parents[2] - owner_root = repo_root.parent - github_root = owner_root.parent - env_roots = [] - has_explicit_pto_root = False - for env_name in ( - "MLIR_PYTHON_ROOT", - "PTO_PYTHON_BUILD_ROOT", - "PTO_PYTHON_ROOT", - "PTO_INSTALL_DIR", - ): - raw = os.environ.get(env_name) - if raw: - env_roots.append(Path(raw)) - if env_name != "MLIR_PYTHON_ROOT": - has_explicit_pto_root = True - - if not env_roots and _pythonpath_already_provides_mlir_and_pto(): - return [] - - fallback_roots = [ - github_root / "llvm" / "llvm-project" / "build" / "tools" / "mlir" / "python_packages" / "mlir_core", - github_root / "llvm" / "llvm-project" / "install" / "python_packages" / "mlir_core", - github_root / "llvm" / "llvm-project" / "build-shared" / "tools" / "mlir" / "python_packages" / "mlir_core", - ] - if not has_explicit_pto_root: - fallback_roots = [ - repo_root / "build" / "python", - repo_root / "install", - *fallback_roots, - ] - - return [ - *env_roots, - *fallback_roots, - ] - - -def _bootstrap_python_paths() -> None: - ordered_roots: list[str] = [] - seen = set() - for root in _candidate_python_roots(): - if not root or not root.is_dir(): - continue - if not (root / "mlir").exists(): - continue - root_text = str(root) - if root_text in seen: - continue - ordered_roots.append(root_text) - seen.add(root_text) - for root_text in reversed(ordered_roots): - if root_text in sys.path: - sys.path.remove(root_text) - sys.path.insert(0, root_text) - - -def _can_import_active_python_mlir() -> bool: - required_modules = ("mlir.ir", "mlir.dialects.pto") - for module_name in required_modules: - try: - if importlib.util.find_spec(module_name) is None: - return False - except (ImportError, ValueError, ModuleNotFoundError): - return False - return True - - -if not _can_import_active_python_mlir(): - _bootstrap_python_paths() - -from mlir.dialects import pto as _pto_dialect # noqa: E402 -try: - from mlir.dialects import llvm as _llvm_dialect # noqa: E402 -except Exception: # pragma: no cover - depends on the installed MLIR package. - _llvm_dialect = None -from mlir.ir import Context, Location # noqa: E402 - - -def make_context() -> Context: - """Create a fresh MLIR Context with the PTO dialect loaded.""" - ctx = Context() - _pto_dialect.register_dialect(ctx, load=True) - if _llvm_dialect is not None and hasattr(_llvm_dialect, "register_dialect"): - _llvm_dialect.register_dialect(ctx, load=True) - return ctx - - -__all__ = ["make_context"] diff --git a/ptodsl/ptodsl/_builtin_vector.py b/ptodsl/ptodsl/_builtin_vector.py index 6daf049a8d..544b66470b 100644 --- a/ptodsl/ptodsl/_builtin_vector.py +++ b/ptodsl/ptodsl/_builtin_vector.py @@ -7,7 +7,6 @@ # See LICENSE in the root of the software repository for the full text of the License. """Builtin MLIR vector helpers for PTODSL scalar-contiguous access.""" -from ._bootstrap import make_context # ensure MLIR is on sys.path # noqa: F401 from ._scalar_coercion import coerce_scalar_to_type from ._surface_values import VecValue, unwrap_surface_value from ._types import _resolve, _validate_vec_size, vec_type diff --git a/ptodsl/ptodsl/_context.py b/ptodsl/ptodsl/_context.py new file mode 100644 index 0000000000..efdc47af59 --- /dev/null +++ b/ptodsl/ptodsl/_context.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +"""MLIR context construction for an installed or explicitly configured PTODSL.""" + +from mlir.dialects import pto as _pto_dialect + +try: + from mlir.dialects import llvm as _llvm_dialect +except Exception: # pragma: no cover - depends on the installed MLIR package. + _llvm_dialect = None + +from mlir.ir import Context + + +def make_context() -> Context: + """Create a fresh MLIR context with the PTO dialect loaded.""" + ctx = Context() + _pto_dialect.register_dialect(ctx, load=True) + if _llvm_dialect is not None and hasattr(_llvm_dialect, "register_dialect"): + _llvm_dialect.register_dialect(ctx, load=True) + return ctx + + +__all__ = ["make_context"] diff --git a/ptodsl/ptodsl/_control_flow.py b/ptodsl/ptodsl/_control_flow.py index 55f389840c..e412d96b58 100644 --- a/ptodsl/ptodsl/_control_flow.py +++ b/ptodsl/ptodsl/_control_flow.py @@ -21,7 +21,6 @@ ``const_expr(value)`` – trace-time ``if`` escape hatch for AST rewrite """ -from ._bootstrap import make_context # noqa: F401 from ._diagnostics import explicit_mode_required_with_context_error from ._runtime_index_ops import coerce_runtime_index from ._scalar_coercion import coerce_scalar_to_type diff --git a/ptodsl/ptodsl/_ops.py b/ptodsl/ptodsl/_ops.py index c3349f5459..df07382fbe 100644 --- a/ptodsl/ptodsl/_ops.py +++ b/ptodsl/ptodsl/_ops.py @@ -24,7 +24,6 @@ from functools import wraps -from ._bootstrap import make_context # noqa: F401 – ensure MLIR on sys.path from ._diagnostics import ( explicit_mode_required_with_context_error, make_tensor_view_invalid_layout_error, diff --git a/ptodsl/ptodsl/_runtime/codegen.py b/ptodsl/ptodsl/_runtime/codegen.py index ba93106c25..768a11155b 100644 --- a/ptodsl/ptodsl/_runtime/codegen.py +++ b/ptodsl/ptodsl/_runtime/codegen.py @@ -9,7 +9,7 @@ from __future__ import annotations -from .._bootstrap import make_context +from .._context import make_context from mlir.ir import BF16Type, F16Type, F32Type, IndexType, IntegerType from .._kernel_signature import DeviceParameterSpec, RuntimeScalarParameterSpec diff --git a/ptodsl/ptodsl/_runtime/toolchain.py b/ptodsl/ptodsl/_runtime/toolchain.py index b41ef53a54..5e774a14de 100644 --- a/ptodsl/ptodsl/_runtime/toolchain.py +++ b/ptodsl/ptodsl/_runtime/toolchain.py @@ -27,36 +27,12 @@ def resolve_ptoas_binary() -> Path: f"PTOAS_BIN is set but does not resolve to an existing executable: {env_override}" ) - for var in ("PTO_BUILD_DIR", "PTO_INSTALL_DIR"): - if var in os.environ: - cand = Path(os.environ[var]) / "tools" / "ptoas" / "ptoas" - if cand.is_file(): - return cand - cand2 = Path(os.environ[var]) / "bin" / "ptoas" - if cand2.is_file(): - return cand2 - - repo_root = Path(__file__).resolve().parents[4] - candidates = [ - repo_root / "build" / "tools" / "ptoas" / "ptoas", - repo_root / "install" / "bin" / "ptoas", - ] - for candidate in candidates: - if candidate.is_file(): - return candidate - - for start in [Path.cwd()] + list(Path.cwd().parents): - for pattern in [start / "build" / "tools" / "ptoas" / "ptoas", - start / "install" / "bin" / "ptoas"]: - if pattern.is_file(): - return pattern - from_path = shutil.which("ptoas") if from_path: return Path(from_path) raise FileNotFoundError( - "unable to locate ptoas; build ptoas or add it to PATH after sourcing scripts/ptoas_env.sh" + "unable to locate ptoas; install it, add it to PATH, or set PTOAS_BIN" ) diff --git a/ptodsl/ptodsl/_source_loader.py b/ptodsl/ptodsl/_source_loader.py index b936937ddf..f4bac6f730 100644 --- a/ptodsl/ptodsl/_source_loader.py +++ b/ptodsl/ptodsl/_source_loader.py @@ -13,7 +13,7 @@ from dataclasses import dataclass from pathlib import Path -from ._bootstrap import make_context +from ._context import make_context from ._diagnostics import ( jit_source_abi_error, jit_source_entry_error, diff --git a/ptodsl/ptodsl/_surface_types.py b/ptodsl/ptodsl/_surface_types.py index 7a79fcded1..9ec4d95142 100644 --- a/ptodsl/ptodsl/_surface_types.py +++ b/ptodsl/ptodsl/_surface_types.py @@ -9,8 +9,6 @@ from enum import Enum -from ._bootstrap import make_context # noqa: F401 - from mlir.dialects import pto as _pto diff --git a/ptodsl/ptodsl/_tracing/runtime.py b/ptodsl/ptodsl/_tracing/runtime.py index e876c25f05..4a22c6b462 100644 --- a/ptodsl/ptodsl/_tracing/runtime.py +++ b/ptodsl/ptodsl/_tracing/runtime.py @@ -13,7 +13,7 @@ from .module_builder import create_kernel_module from .session import TraceSession from .._diagnostics import kernel_module_return_value_error -from .._bootstrap import make_context +from .._context import make_context from .._types import _resolve from mlir.dialects import func diff --git a/ptodsl/ptodsl/_types.py b/ptodsl/ptodsl/_types.py index 269769b56e..b01ec8d056 100644 --- a/ptodsl/ptodsl/_types.py +++ b/ptodsl/ptodsl/_types.py @@ -19,8 +19,6 @@ def softmax(arg0: pto.ptr(pto.float32, "GM"), ...): the actual type is materialised later by the ``@pto.jit`` decorator. """ -from ._bootstrap import make_context # ensure MLIR is on sys.path - from mlir.dialects import pto as _pto from mlir.dialects import arith from mlir.dialects.builtin import UnrealizedConversionCastOp diff --git a/ptodsl/ptodsl/scalar.py b/ptodsl/ptodsl/scalar.py index 10b2d3f58e..3e18343b8e 100644 --- a/ptodsl/ptodsl/scalar.py +++ b/ptodsl/ptodsl/scalar.py @@ -15,7 +15,6 @@ address views such as `tile[row, col]` and `tile.as_ptr() + offset`. """ -from ._bootstrap import make_context # ensure MLIR is on sys.path # noqa: F401 from ._scalar_coercion import coerce_scalar_to_type from ._scalar_adaptation import classify_runtime_scalar_type from ._runtime_scalar_ops import ( diff --git a/ptodsl/ptodsl/tilelib/_render_runtime.py b/ptodsl/ptodsl/tilelib/_render_runtime.py index fc9bd8aa8e..b87dafe8ae 100644 --- a/ptodsl/ptodsl/tilelib/_render_runtime.py +++ b/ptodsl/ptodsl/tilelib/_render_runtime.py @@ -28,7 +28,7 @@ from .metadata import ScalarSpec, TileSpec, VectorSpec, ViewSpec, scalar_descriptor from .._ast_rewrite import rewrite_jit_function -from .._bootstrap import make_context +from .._context import make_context from .._surface_types import Tile from .._surface_values import PartitionTensorViewValue, TensorViewValue, TileValue from .._tracing import KernelModuleSpec, ModuleStyle, TracingRuntime diff --git a/ptodsl/tests/test_ast_rewrite_example_ir.py b/ptodsl/tests/test_ast_rewrite_example_ir.py index e30a094eaf..bc6d6242cc 100644 --- a/ptodsl/tests/test_ast_rewrite_example_ir.py +++ b/ptodsl/tests/test_ast_rewrite_example_ir.py @@ -14,7 +14,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] from ptodsl import pto -from ptodsl._bootstrap import make_context +from ptodsl._context import make_context from mlir.ir import Module diff --git a/ptodsl/tests/test_bootstrap_runtime_env.py b/ptodsl/tests/test_bootstrap_runtime_env.py deleted file mode 100644 index ffcf1f9eb2..0000000000 --- a/ptodsl/tests/test_bootstrap_runtime_env.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -import importlib.util -import sys -import types -import unittest -from pathlib import Path -from unittest import mock - - -class BootstrapRuntimeEnvTests(unittest.TestCase): - def test_bootstrap_keeps_installed_mlir_environment_unchanged(self): - repo_root = Path(__file__).resolve().parents[2] - bootstrap_path = repo_root / "ptodsl" / "ptodsl" / "_bootstrap.py" - module_name = "_test_ptodsl_bootstrap_runtime_env" - spec = importlib.util.spec_from_file_location(module_name, bootstrap_path) - self.assertIsNotNone(spec) - self.assertIsNotNone(spec.loader) - - mlir_module = types.ModuleType("mlir") - mlir_module.__path__ = [] - mlir_dialects_module = types.ModuleType("mlir.dialects") - mlir_dialects_module.__path__ = [] - mlir_pto_module = types.ModuleType("mlir.dialects.pto") - mlir_llvm_module = types.ModuleType("mlir.dialects.llvm") - mlir_ir_module = types.ModuleType("mlir.ir") - - class FakeContext: - pass - - class FakeLocation: - pass - - def register_pto_dialect(ctx, load=False): - ctx.pto_loaded = load - - def register_llvm_dialect(ctx, load=False): - ctx.llvm_loaded = load - - mlir_ir_module.Context = FakeContext - mlir_ir_module.Location = FakeLocation - mlir_pto_module.register_dialect = register_pto_dialect - mlir_llvm_module.register_dialect = register_llvm_dialect - mlir_dialects_module.pto = mlir_pto_module - mlir_dialects_module.llvm = mlir_llvm_module - mlir_module.dialects = mlir_dialects_module - mlir_module.ir = mlir_ir_module - - fake_sys_modules = { - "mlir": mlir_module, - "mlir.dialects": mlir_dialects_module, - "mlir.dialects.pto": mlir_pto_module, - "mlir.dialects.llvm": mlir_llvm_module, - "mlir.ir": mlir_ir_module, - } - - required_specs = { - "mlir.ir": object(), - "mlir.dialects.pto": object(), - } - original_find_spec = importlib.util.find_spec - original_sys_path = list(sys.path) - - def fake_find_spec(name, package=None): - if name in required_specs: - return required_specs[name] - return original_find_spec(name, package) - - with mock.patch.object(sys, "path", list(original_sys_path)), mock.patch.dict( - sys.modules, fake_sys_modules, clear=False - ), mock.patch("importlib.util.find_spec", side_effect=fake_find_spec): - module = importlib.util.module_from_spec(spec) - sys.modules.pop(module_name, None) - try: - spec.loader.exec_module(module) - self.assertEqual(sys.path, original_sys_path) - ctx = module.make_context() - self.assertTrue(getattr(ctx, "pto_loaded", False)) - self.assertTrue(getattr(ctx, "llvm_loaded", False)) - finally: - sys.modules.pop(module_name, None) - - -if __name__ == "__main__": - unittest.main() diff --git a/ptodsl/tests/test_context.py b/ptodsl/tests/test_context.py new file mode 100644 index 0000000000..b7d9b71800 --- /dev/null +++ b/ptodsl/tests/test_context.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import unittest + +from ptodsl._context import make_context + + +class ContextTests(unittest.TestCase): + def test_make_context_loads_pto_dialect(self): + context = make_context() + + self.assertIsNotNone(context) + + +if __name__ == "__main__": + unittest.main() diff --git a/ptodsl/tests/test_docs_as_test.py b/ptodsl/tests/test_docs_as_test.py index dccd1e5746..7e03b8debe 100644 --- a/ptodsl/tests/test_docs_as_test.py +++ b/ptodsl/tests/test_docs_as_test.py @@ -23,7 +23,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] USER_GUIDE_ROOT = REPO_ROOT / "ptodsl" / "docs" / "user_guide" from ptodsl import pto, scalar -from ptodsl._bootstrap import make_context +from ptodsl._context import make_context from ptodsl._runtime.launch import LaunchHandle, _marshal_launch_args from ptodsl._runtime.toolchain import resolve_ptoas_binary from mlir.ir import Module diff --git a/ptodsl/tests/test_flash_attention_demo_compile.py b/ptodsl/tests/test_flash_attention_demo_compile.py index ef4d5ee234..00bb6eadf1 100644 --- a/ptodsl/tests/test_flash_attention_demo_compile.py +++ b/ptodsl/tests/test_flash_attention_demo_compile.py @@ -11,7 +11,7 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] -from ptodsl._bootstrap import make_context +from ptodsl._context import make_context from mlir.ir import Module diff --git a/ptodsl/tests/test_jit_compile.py b/ptodsl/tests/test_jit_compile.py index 7047c31b11..a4f93dc90e 100644 --- a/ptodsl/tests/test_jit_compile.py +++ b/ptodsl/tests/test_jit_compile.py @@ -22,7 +22,7 @@ from ptodsl import _types as pto_types import ptodsl._vmi_namespace as vmi_namespace from ptodsl._ast_rewrite import PTODSLAstRewriteError -from ptodsl._bootstrap import make_context +from ptodsl._context import make_context from ptodsl._kernel_signature import DeviceParameterSpec, HelperMarkerParameterSpec, RuntimeScalarParameterSpec from ptodsl._tracing.runtime import SignatureTracingRuntime from ptodsl._runtime import native_build as native_build_runtime diff --git a/ptodsl/tests/test_ptoas_cli.py b/ptodsl/tests/test_ptoas_cli.py index 4e4f9dfa50..9afd21e42c 100644 --- a/ptodsl/tests/test_ptoas_cli.py +++ b/ptodsl/tests/test_ptoas_cli.py @@ -7,7 +7,6 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -import os import tempfile import unittest from pathlib import Path @@ -59,10 +58,7 @@ def test_launch_uses_standard_native_module_and_packaged_resources(self): ) self.assertEqual(environment["PTOAS_BIN"], str(wrapper.resolve())) self.assertEqual(environment["PTOAS_PYTHON_EXE"], _cli.sys.executable) - self.assertEqual( - environment["PATH"], - str(wrapper.parent.resolve()) + os.pathsep + "/usr/bin", - ) + self.assertEqual(environment["PATH"], "/usr/bin") def test_explicit_resource_options_are_not_overridden(self): with tempfile.TemporaryDirectory() as temp_dir: diff --git a/ptodsl/tests/test_ptoas_frontend_verify.py b/ptodsl/tests/test_ptoas_frontend_verify.py index 083ef4daa5..b3e80a4687 100644 --- a/ptodsl/tests/test_ptoas_frontend_verify.py +++ b/ptodsl/tests/test_ptoas_frontend_verify.py @@ -17,7 +17,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] from ptodsl import pto from ptodsl import scalar -from ptodsl._bootstrap import make_context +from ptodsl._context import make_context from ptodsl._runtime.toolchain import resolve_ptoas_binary from mlir.ir import Module diff --git a/ptodsl/tests/test_runtime_toolchain.py b/ptodsl/tests/test_runtime_toolchain.py index bea4a0fcc3..f9c1797899 100644 --- a/ptodsl/tests/test_runtime_toolchain.py +++ b/ptodsl/tests/test_runtime_toolchain.py @@ -49,33 +49,41 @@ def test_native_launch_flags_use_target_arch(self): class ResolvePtoasBinaryTests(unittest.TestCase): - def test_env_override_wins_over_repo_default(self): + def test_env_override_wins_over_path(self): with tempfile.TemporaryDirectory() as temp_dir: temp_root = Path(temp_dir) env_ptoas = temp_root / "env-ptoas" env_ptoas.write_text("", encoding="utf-8") - repo_build_ptoas = temp_root / "build" / "tools" / "ptoas" / "ptoas" - repo_build_ptoas.parent.mkdir(parents=True, exist_ok=True) - repo_build_ptoas.write_text("", encoding="utf-8") + path_ptoas = temp_root / "path-ptoas" - fake_toolchain_file = ( - temp_root / "repo" / "ptodsl" / "ptodsl" / "_runtime" / "toolchain.py" - ) - fake_toolchain_file.parent.mkdir(parents=True, exist_ok=True) - fake_toolchain_file.write_text("", encoding="utf-8") - - with mock.patch.dict(os.environ, {"PTOAS_BIN": str(env_ptoas)}, clear=False), mock.patch.object( - toolchain, "__file__", str(fake_toolchain_file) - ): + with mock.patch.dict( + os.environ, {"PTOAS_BIN": str(env_ptoas)}, clear=True + ), mock.patch.object( + toolchain.shutil, "which", return_value=str(path_ptoas) + ) as which: resolved = toolchain.resolve_ptoas_binary() self.assertEqual(resolved, env_ptoas) + which.assert_not_called() + + def test_path_is_used_without_env_override(self): + with tempfile.TemporaryDirectory() as temp_dir: + path_ptoas = Path(temp_dir) / "path-ptoas" + with mock.patch.dict(os.environ, {}, clear=True), mock.patch.object( + toolchain.shutil, "which", return_value=str(path_ptoas) + ) as which: + resolved = toolchain.resolve_ptoas_binary() + + self.assertEqual(resolved, path_ptoas) + which.assert_called_once_with("ptoas") def test_invalid_env_override_raises(self): with tempfile.TemporaryDirectory() as temp_dir: missing = Path(temp_dir) / "missing-ptoas" - with mock.patch.dict(os.environ, {"PTOAS_BIN": str(missing)}, clear=False): + with mock.patch.dict( + os.environ, {"PTOAS_BIN": str(missing)}, clear=True + ), mock.patch.object(toolchain.shutil, "which", return_value=None): with self.assertRaises(FileNotFoundError): toolchain.resolve_ptoas_binary() diff --git a/ptodsl/tests/test_tileop.py b/ptodsl/tests/test_tileop.py index 0c55ddc671..6b30cb88fc 100644 --- a/ptodsl/tests/test_tileop.py +++ b/ptodsl/tests/test_tileop.py @@ -10,7 +10,7 @@ """Focused tracing coverage for the public ``@pto.tileop`` surface.""" from ptodsl import pto -from ptodsl._bootstrap import make_context +from ptodsl._context import make_context from mlir.ir import Module diff --git a/ptodsl/tests/test_vector_cube_ops.py b/ptodsl/tests/test_vector_cube_ops.py index 71b975ab34..ae49695047 100644 --- a/ptodsl/tests/test_vector_cube_ops.py +++ b/ptodsl/tests/test_vector_cube_ops.py @@ -13,7 +13,7 @@ import ptodsl._ops as _ops import ptodsl._pipe_namespace as _pipe_namespace -from ptodsl._bootstrap import make_context +from ptodsl._context import make_context from ptodsl import pto from mlir.ir import F32Type def _identity(value): diff --git a/tools/ptoas/ptoas_wrapper.py b/tools/ptoas/ptoas_wrapper.py index fb07afaf9b..07a8d423d7 100644 --- a/tools/ptoas/ptoas_wrapper.py +++ b/tools/ptoas/ptoas_wrapper.py @@ -11,7 +11,6 @@ from __future__ import annotations -import shutil import sys from pathlib import Path @@ -22,11 +21,7 @@ def _resolve_wrapper_path(argv0: str | None = None) -> Path: if wrapper.exists(): return wrapper.resolve() - found = shutil.which(wrapper.name or "ptoas") - if found: - return Path(found).resolve() - - raise SystemExit(f"unable to locate the installed ptoas wrapper: {candidate}") + raise SystemExit(f"unable to locate the ptoas tree wrapper: {candidate}") def _is_build_tree_wrapper(wrapper: Path) -> bool: @@ -43,7 +38,7 @@ def _require_python_root(python_root: Path, *, context: str) -> Path: ) -def _bootstrap_python_path(wrapper: Path) -> Path: +def _add_tree_python_root(wrapper: Path) -> Path: if _is_build_tree_wrapper(wrapper): if len(wrapper.parents) < 3: raise SystemExit("unable to locate the build-tree root for ptoas") @@ -65,7 +60,7 @@ def _bootstrap_python_path(wrapper: Path) -> Path: def main() -> None: wrapper = _resolve_wrapper_path() - _bootstrap_python_path(wrapper) + _add_tree_python_root(wrapper) from ptoas import _cli raise SystemExit(_cli.launch(sys.argv[1:], wrapper=wrapper)) From c8bb48813006818dc117a5330a106608370b61cb Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Thu, 23 Jul 2026 18:31:55 +0800 Subject: [PATCH 03/32] Harden Python wheel developer workflow --- .github/workflows/build_wheel.yml | 13 +++++ .github/workflows/build_wheel_mac.yml | 13 ++++- .github/workflows/ci.yml | 4 +- docker/test_ptoas_cli.sh | 27 +++------ test/python/test_docker_runtime_packaging.py | 26 ++++----- test/python/test_validate_wheel_payload.py | 60 +------------------- tools/ptoas/CMakeLists.txt | 6 ++ 7 files changed, 54 insertions(+), 95 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 02e776ac42..0a4505e789 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -230,6 +230,18 @@ jobs: --wheel-dir "${PTO_SOURCE_DIR}/build/wheel-dist" cmake --install "${PTO_BUILD_DIR}" --prefix "${PTO_INSTALL_DIR}" + - name: Install built wheel for Python sample generation + run: | + export PATH="${PY_PATH}/bin:$PATH" + shopt -s nullglob + wheels=("$PTO_SOURCE_DIR"/build/wheel-dist/ptoas*.whl) + if [ "${#wheels[@]}" -ne 1 ]; then + echo "Expected exactly one wheel in $PTO_SOURCE_DIR/build/wheel-dist, found ${#wheels[@]}" >&2 + printf ' %s\n' "${wheels[@]}" >&2 + exit 1 + fi + python -m pip install --no-deps --force-reinstall "${wheels[0]}" + - name: Validate wheel payload and launcher contract if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: | @@ -256,6 +268,7 @@ jobs: - name: Test ptoas CLI run: | export PATH="${PY_PATH}/bin:$PATH" + export PTOAS_BIN="${PTO_BUILD_DIR}/tools/ptoas/ptoas" bash $PTO_SOURCE_DIR/docker/test_ptoas_cli.sh - name: Upload wheel artifact diff --git a/.github/workflows/build_wheel_mac.yml b/.github/workflows/build_wheel_mac.yml index a473cc9c4c..b78213a34f 100644 --- a/.github/workflows/build_wheel_mac.yml +++ b/.github/workflows/build_wheel_mac.yml @@ -234,6 +234,17 @@ jobs: printf 'Built wheel file: %s\n' "$(basename "${built_wheels[0]}")" fi + - name: Install built wheel for Python sample generation + run: | + shopt -s nullglob + wheels=("$PTO_SOURCE_DIR"/build/wheel-dist/ptoas*.whl) + if [ "${#wheels[@]}" -ne 1 ]; then + echo "Expected exactly one wheel in $PTO_SOURCE_DIR/build/wheel-dist, found ${#wheels[@]}" >&2 + printf ' %s\n' "${wheels[@]}" >&2 + exit 1 + fi + python -m pip install --no-deps --force-reinstall "${wheels[0]}" + - name: Validate wheel payload and launcher contract if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: | @@ -324,7 +335,7 @@ jobs: - name: Test ptoas CLI run: | - export DYLD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:${DYLD_LIBRARY_PATH} + export PTOAS_BIN="${PTO_BUILD_DIR}/tools/ptoas/ptoas" bash $PTO_SOURCE_DIR/docker/test_ptoas_cli.sh - name: Upload wheel artifact diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e3eb1f92a..d1544fcd06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -262,7 +262,7 @@ jobs: -DCMAKE_CXX_COMPILER="${PTOAS_CMAKE_CXX_COMPILER}" \ -Werror=dev - - name: Create PTOAS test venv + - name: Create PTOAS developer environment shell: bash run: | set -euo pipefail @@ -298,7 +298,7 @@ jobs: exit 1 fi - "${PTOAS_VENV}/bin/pip" install --force-reinstall "${ptoas_wheels[0]}" + "${PTOAS_VENV}/bin/pip" install --no-deps --force-reinstall "${ptoas_wheels[0]}" PATH="${PTOAS_VENV}/bin:${PATH}" bash docker/test_wheel_imports.sh - name: Run PTODSL Python tests diff --git a/docker/test_ptoas_cli.sh b/docker/test_ptoas_cli.sh index 6902d679ae..8ce29e1b87 100755 --- a/docker/test_ptoas_cli.sh +++ b/docker/test_ptoas_cli.sh @@ -13,35 +13,22 @@ # # Required environment variables: # PTO_SOURCE_DIR - Path to PTO source directory -# PTO_BUILD_DIR - Path to PTO build directory -# LLVM_BUILD_DIR - Path to LLVM build directory -# PTO_INSTALL_DIR - Path to PTO install directory -# PTOAS_BIN - Path to the Python ptoas wrapper (optional) +# PTOAS_BIN - Exact ptoas entrypoint under test -set -e +set -euo pipefail # Validate required environment variables -for var in PTO_SOURCE_DIR PTO_BUILD_DIR LLVM_BUILD_DIR PTO_INSTALL_DIR; do - if [ -z "${!var}" ]; then +for var in PTO_SOURCE_DIR PTOAS_BIN; do + if [ -z "${!var:-}" ]; then echo "Error: $var environment variable is not set" >&2 exit 1 fi done -# Setup environment -export PATH="${PTO_BUILD_DIR}/tools/ptoas:${PATH}" -export LD_LIBRARY_PATH="${LLVM_BUILD_DIR}/lib:${PTO_INSTALL_DIR}/lib:${LD_LIBRARY_PATH}" -export DYLD_LIBRARY_PATH="${LLVM_BUILD_DIR}/lib:${PTO_INSTALL_DIR}/lib:${DYLD_LIBRARY_PATH}" - -if [ -n "${PTOAS_BIN:-}" ]; then - if [ ! -x "${PTOAS_BIN}" ]; then - echo "Error: PTOAS_BIN is not executable: ${PTOAS_BIN}" >&2 - exit 1 - fi -else - PTOAS_BIN="$(command -v ptoas)" +if [ ! -x "${PTOAS_BIN}" ]; then + echo "Error: PTOAS_BIN is not executable: ${PTOAS_BIN}" >&2 + exit 1 fi - echo "Testing ptoas CLI..." echo "${PTOAS_BIN}" diff --git a/test/python/test_docker_runtime_packaging.py b/test/python/test_docker_runtime_packaging.py index a3b1768163..a56475ae74 100644 --- a/test/python/test_docker_runtime_packaging.py +++ b/test/python/test_docker_runtime_packaging.py @@ -16,8 +16,7 @@ COLLECT_DIST_SCRIPT = REPO_ROOT / "docker" / "collect_ptoas_dist.sh" COLLECT_DIST_MAC_SCRIPT = REPO_ROOT / "docker" / "collect_ptoas_dist_mac.sh" PTOAS_CMAKE = REPO_ROOT / "tools" / "ptoas" / "CMakeLists.txt" -BUILD_WHEEL_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "build_wheel.yml" -BUILD_WHEEL_MAC_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "build_wheel_mac.yml" +PTOAS_CLI_TEST = REPO_ROOT / "docker" / "test_ptoas_cli.sh" class DockerRuntimePackagingTests(unittest.TestCase): @@ -29,6 +28,8 @@ def test_runtime_uses_standard_python_extension_without_native_cli(self): cmake, ) self.assertIn('OUTPUT_NAME "_native"', cmake) + self.assertIn('BUILD_RPATH "${PTO_LLVM_BUILD_LIBRARY_DIR}"', cmake) + self.assertIn('INSTALL_RPATH "${PTO_LLVM_BUILD_LIBRARY_DIR}"', cmake) self.assertNotIn("add_executable(ptoas_native_cli", cmake) self.assertNotIn('OUTPUT_NAME "ptoas-native"', cmake) @@ -76,23 +77,22 @@ def test_macos_dist_packages_python_wrapper_and_native_extension(self): self.assertNotIn("ptoas-native", script) self.assertNotIn("ptoas.so", script) - def test_dist_archives_use_bin_entrypoint(self): - linux_workflow = BUILD_WHEEL_WORKFLOW.read_text(encoding="utf-8") - mac_workflow = BUILD_WHEEL_MAC_WORKFLOW.read_text(encoding="utf-8") - - self.assertIn('chmod +x "$GITHUB_WORKSPACE/ptoas-dist/bin/ptoas"', linux_workflow) - self.assertNotIn('chmod +x "$GITHUB_WORKSPACE/ptoas-dist/ptoas"', linux_workflow) - self.assertIn('chmod +x "$GITHUB_WORKSPACE/ptoas-dist/bin/ptoas"', mac_workflow) - self.assertNotIn('chmod +x "$GITHUB_WORKSPACE/ptoas-dist/ptoas"', mac_workflow) - self.assertIn('"$TEST_DIR/extracted/bin/ptoas" --version', mac_workflow) - self.assertNotIn('"$TEST_DIR/extracted/ptoas" --version', mac_workflow) - def test_wheel_import_smoke_imports_native_extension(self): script = (REPO_ROOT / "docker" / "test_wheel_imports.sh").read_text(encoding="utf-8") self.assertIn("from ptoas import _native", script) self.assertNotIn("POLLUTED_PTOAS_VERSION_OUTPUT", script) + def test_build_tree_cli_test_receives_explicit_test_environment(self): + script = PTOAS_CLI_TEST.read_text(encoding="utf-8") + + self.assertIn("for var in PTO_SOURCE_DIR PTOAS_BIN", script) + self.assertIn("python ./tmatmulk.py", script) + self.assertIn("python ./abs.py", script) + self.assertNotIn("PYTHON_BIN", script) + self.assertNotIn('command -v ptoas', script) + self.assertNotIn('export PATH="${PTO_BUILD_DIR}', script) + self.assertNotIn('export LD_LIBRARY_PATH="${LLVM_BUILD_DIR}', script) if __name__ == "__main__": unittest.main() diff --git a/test/python/test_validate_wheel_payload.py b/test/python/test_validate_wheel_payload.py index 318d1a6fde..5b99367026 100644 --- a/test/python/test_validate_wheel_payload.py +++ b/test/python/test_validate_wheel_payload.py @@ -17,11 +17,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] VALIDATOR = REPO_ROOT / "docker" / "validate_wheel_payload.py" -LINUX_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "build_wheel.yml" -MAC_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "build_wheel_mac.yml" WHEEL_IMPORTS = REPO_ROOT / "docker" / "test_wheel_imports.sh" -CI_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "ci.yml" -CI_SIM_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "ci_sim.yml" class ValidateWheelPayloadTests(unittest.TestCase): @@ -165,59 +161,12 @@ def test_validator_accepts_vmi_distribution_name(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("validated wheel payload and launcher contract", result.stdout) - def test_workflows_and_shell_probe_reuse_shared_validator(self): - validator_call = 'python "$PTO_SOURCE_DIR/docker/validate_wheel_payload.py" "$PTO_SOURCE_DIR/build/wheel-dist"' - self.assertIn( - validator_call, - LINUX_WORKFLOW.read_text(encoding="utf-8"), - ) - self.assertIn( - validator_call, - MAC_WORKFLOW.read_text(encoding="utf-8"), - ) + def test_shell_probe_reuses_shared_validator(self): self.assertIn( '"${PYTHON_BIN}" "${REPO_ROOT}/docker/validate_wheel_payload.py" "${TEST_WHEEL}"', WHEEL_IMPORTS.read_text(encoding="utf-8"), ) - def test_release_workflows_separate_cli_and_wheel_versions(self): - linux_workflow = LINUX_WORKFLOW.read_text(encoding="utf-8") - mac_workflow = MAC_WORKFLOW.read_text(encoding="utf-8") - - self.assertIn("workflow_dispatch:", linux_workflow) - self.assertIn("release_kind:", linux_workflow) - self.assertIn('type: choice', linux_workflow) - self.assertIn('--mode release)', linux_workflow) - self.assertIn( - "if: github.event_name != 'release' || startsWith(github.ref_name, 'v') || startsWith(github.ref_name, 'ptoas-v')", - linux_workflow, - ) - self.assertIn( - "if: (github.event_name == 'release' && (startsWith(github.ref_name, 'v') || startsWith(github.ref_name, 'ptoas-v'))) || github.event_name == 'schedule'", - linux_workflow, - ) - self.assertIn('PTOAS_CLI_VERSION="vmi ${PTOAS_VERSION}"', linux_workflow) - self.assertIn('PTOAS_RELEASE_VERSION_OVERRIDE="${PTOAS_CLI_VERSION}"', linux_workflow) - self.assertIn('PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}/packaging/ptoas-vmi"', linux_workflow) - self.assertIn('SKBUILD_BUILD_DIR="${PTO_BUILD_DIR}"', linux_workflow) - - self.assertIn("workflow_dispatch:", mac_workflow) - self.assertIn("release_kind:", mac_workflow) - self.assertIn('type: choice', mac_workflow) - self.assertIn('--mode release)', mac_workflow) - self.assertIn( - "if: github.event_name != 'release' || startsWith(github.ref_name, 'v') || startsWith(github.ref_name, 'ptoas-v')", - mac_workflow, - ) - self.assertIn( - "if: (github.event_name == 'release' && (startsWith(github.ref_name, 'v') || startsWith(github.ref_name, 'ptoas-v'))) || github.event_name == 'schedule'", - mac_workflow, - ) - self.assertIn('PTOAS_CLI_VERSION="vmi ${PTOAS_VERSION}"', mac_workflow) - self.assertIn('PTOAS_RELEASE_VERSION_OVERRIDE="${PTOAS_CLI_VERSION}"', mac_workflow) - self.assertIn('PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}/packaging/ptoas-vmi"', mac_workflow) - self.assertIn('SKBUILD_BUILD_DIR="${PTO_BUILD_DIR}"', mac_workflow) - def test_wheel_imports_script_keeps_clean_env_ptoas_smoke(self): script = WHEEL_IMPORTS.read_text(encoding="utf-8") @@ -242,12 +191,5 @@ def test_wheel_imports_script_keeps_clean_env_ptoas_smoke(self): self.assertIn('grep -q "TileLib daemon started successfully" "${CLEAN_ENV_LOG}"', script) self.assertIn('grep -q "TileLib daemon stopped" "${CLEAN_ENV_LOG}"', script) - def test_ci_workflows_accept_generic_ptoas_wheel_glob(self): - self.assertIn("name 'ptoas*.whl'", CI_WORKFLOW.read_text(encoding="utf-8")) - ci_sim_text = CI_SIM_WORKFLOW.read_text(encoding="utf-8") - self.assertIn("name 'ptoas*.whl'", ci_sim_text) - self.assertNotIn("name 'ptoas-*.whl'", ci_sim_text) - - if __name__ == "__main__": unittest.main() diff --git a/tools/ptoas/CMakeLists.txt b/tools/ptoas/CMakeLists.txt index 0d4ede0c40..75db33c55a 100644 --- a/tools/ptoas/CMakeLists.txt +++ b/tools/ptoas/CMakeLists.txt @@ -90,6 +90,12 @@ target_link_libraries(ptoas_runtime PRIVATE ptoas_runtime_objects) set_target_properties(ptoas_runtime PROPERTIES OUTPUT_NAME "_native" LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/python-module" + # Keep the external LLVM/MLIR build discoverable for build-tree and + # editable Python installs. Release wheels are subsequently repaired by + # auditwheel, which replaces this machine-local path with wheel-local + # dependencies and a relative RPATH. + BUILD_RPATH "${PTO_LLVM_BUILD_LIBRARY_DIR}" + INSTALL_RPATH "${PTO_LLVM_BUILD_LIBRARY_DIR}" ) set(PTOAS_BUILD_PYTHON_PACKAGE_DIR "${CMAKE_BINARY_DIR}/python/ptoas") From 9a7467345228a4149c5f0591a9d6ce3929144d91 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Thu, 23 Jul 2026 19:33:10 +0800 Subject: [PATCH 04/32] Fix compiler archive Python selection --- .github/workflows/build_wheel.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 0a4505e789..a670aba41d 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -281,6 +281,7 @@ jobs: - name: Collect ptoas binary and dependencies if: matrix.python == '3.11' run: | + export PATH="${PY_PATH}/bin:$PATH" bash $PTO_SOURCE_DIR/docker/collect_ptoas_dist.sh $GITHUB_WORKSPACE/ptoas-dist - name: Archive ptoas binary artifact From f66e9dd8afbe460aa918a158dbc88c5449f22f41 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Thu, 23 Jul 2026 19:37:49 +0800 Subject: [PATCH 05/32] Persist selected manylinux Python on PATH --- .github/workflows/build_wheel.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index a670aba41d..8f6be18831 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -67,6 +67,7 @@ jobs: esac echo "PY_VER=$PY_VER" >> $GITHUB_ENV echo "PY_PATH=/opt/python/$PY_VER" >> $GITHUB_ENV + echo "/opt/python/$PY_VER/bin" >> $GITHUB_PATH - name: Compute PTOAS CLI version run: | @@ -127,14 +128,12 @@ jobs: - name: Install Python dependencies run: | - export PATH="${PY_PATH}/bin:$PATH" # LLVM/MLIR Python bindings are not yet compatible with pybind11 3.x # (see pybind11 static_assert on def_property + keep_alive). pip install --no-cache-dir numpy 'pybind11<3' nanobind 'scikit-build-core>=0.12.2,<2' auditwheel - name: Validate PEP 517 project metadata run: | - export PATH="${PY_PATH}/bin:$PATH" python -m scikit_build_core.build project-table (cd packaging/ptoas-vmi && python -m scikit_build_core.build project-table) @@ -187,7 +186,6 @@ jobs: - name: Build LLVM/MLIR if: steps.cache-llvm.outputs.cache-hit != 'true' run: | - export PATH="${PY_PATH}/bin:$PATH" cd $LLVM_SOURCE_DIR cmake -C "$PTO_SOURCE_DIR/cmake/LinuxHardeningCache.cmake" -G Ninja -S llvm -B $LLVM_BUILD_DIR \ -DLLVM_ENABLE_PROJECTS="mlir;clang" \ @@ -215,7 +213,6 @@ jobs: env: CMAKE_BUILD_PARALLEL_LEVEL: "2" run: | - export PATH="${PY_PATH}/bin:$PATH" if [ "${PTOAS_PYTHON_PACKAGE_NAME}" = "ptoas-vmi" ]; then PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}/packaging/ptoas-vmi" else @@ -232,7 +229,6 @@ jobs: - name: Install built wheel for Python sample generation run: | - export PATH="${PY_PATH}/bin:$PATH" shopt -s nullglob wheels=("$PTO_SOURCE_DIR"/build/wheel-dist/ptoas*.whl) if [ "${#wheels[@]}" -ne 1 ]; then @@ -245,13 +241,11 @@ jobs: - name: Validate wheel payload and launcher contract if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: | - export PATH="${PY_PATH}/bin:$PATH" python "$PTO_SOURCE_DIR/docker/validate_wheel_payload.py" "$PTO_SOURCE_DIR/build/wheel-dist" - name: Repair wheel with auditwheel if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: | - export PATH="${PY_PATH}/bin:$PATH" export PTO_WHEEL_DIST_DIR=$PTO_SOURCE_DIR/build/wheel-dist export PTO_WHEELHOUSE=$GITHUB_WORKSPACE/wheelhouse export LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:$LD_LIBRARY_PATH @@ -261,13 +255,11 @@ jobs: - name: Test wheel installation if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: | - export PATH="${PY_PATH}/bin:$PATH" pip install $GITHUB_WORKSPACE/wheelhouse/ptoas*.whl bash $PTO_SOURCE_DIR/docker/test_wheel_imports.sh - name: Test ptoas CLI run: | - export PATH="${PY_PATH}/bin:$PATH" export PTOAS_BIN="${PTO_BUILD_DIR}/tools/ptoas/ptoas" bash $PTO_SOURCE_DIR/docker/test_ptoas_cli.sh @@ -281,7 +273,6 @@ jobs: - name: Collect ptoas binary and dependencies if: matrix.python == '3.11' run: | - export PATH="${PY_PATH}/bin:$PATH" bash $PTO_SOURCE_DIR/docker/collect_ptoas_dist.sh $GITHUB_WORKSPACE/ptoas-dist - name: Archive ptoas binary artifact From a5bb0ad2f77dca03c2f3d31895436056cc816a4c Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Thu, 23 Jul 2026 20:57:41 +0800 Subject: [PATCH 06/32] Make lit tests self-contained for MLIR Python --- test/lit/lit.cfg.py | 11 +++++++++++ test/lit/lit.site.cfg.py.in | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/test/lit/lit.cfg.py b/test/lit/lit.cfg.py index 94e6df594a..4b9054c7db 100644 --- a/test/lit/lit.cfg.py +++ b/test/lit/lit.cfg.py @@ -54,6 +54,17 @@ 'ASCEND_HOME_PATH', 'ASCEND_OPP_PATH', 'ASCEND_AICPU_PATH', 'ASCEND_TOOLKIT_HOME', 'LD_LIBRARY_PATH', 'PYTHONPATH']) +# Keep build-tree lit tests self-contained. PTOAS contributes an overlay +# under the build tree (generated dialect Python plus the PTO extension), +# while the base ``mlir.ir`` package comes from the LLVM/MLIR build. MLIR's +# own lit setup wires these paths from its CMake-generated site configuration; +# do the same here instead of requiring developers to export PYTHONPATH. +if getattr(config, 'enable_bindings_python', False): + python_paths = [config.ptoas_python_dir] + if config.mlir_python_package_dir: + python_paths.append(config.mlir_python_package_dir) + llvm_config.with_environment('PYTHONPATH', python_paths, append_path=True) + llvm_config.use_default_substitutions() # excludes: A list of directories to exclude from the testsuite. The 'Inputs' diff --git a/test/lit/lit.site.cfg.py.in b/test/lit/lit.site.cfg.py.in index 47f7ccbc59..b820f33fd8 100644 --- a/test/lit/lit.site.cfg.py.in +++ b/test/lit/lit.site.cfg.py.in @@ -19,7 +19,10 @@ config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@" config.mlir_obj_dir = "@MLIR_BINARY_DIR@" config.mlir_lib_dir = "@MLIR_LIB_DIR@" config.python_executable = "@Python3_EXECUTABLE@" -# config.enable_bindings_python = @MLIR_ENABLE_BINDINGS_PYTHON@ +config.enable_bindings_python = "@PTO_ENABLE_PYTHON_BINDING@" == "ON" +config.ptoas_python_dir = path(r"@CMAKE_BINARY_DIR@/python") +config.mlir_python_package_dir = lit_config.substitute( + path(r"@MLIR_PYTHON_PACKAGE_DIR@")) config.ptoir_src_root = "@CMAKE_SOURCE_DIR@" config.ptoir_obj_root = "@CMAKE_BINARY_DIR@" config.pto_enable_vfsim_costmodel = "@PTO_ENABLE_VFSIM_COSTMODEL@" == "ON" From dc9ea1c762c23e7f3eb3d2a36ec4f1fb02ae63fa Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Thu, 23 Jul 2026 22:21:34 +0800 Subject: [PATCH 07/32] Streamline editable developer builds --- .github/workflows/ci_sim.yml | 28 ++++++++++++-- lib/Bindings/Python/CMakeLists.txt | 32 ---------------- pyproject.toml | 5 ++- quick_install.sh | 59 ++++++++---------------------- 4 files changed, 42 insertions(+), 82 deletions(-) diff --git a/.github/workflows/ci_sim.yml b/.github/workflows/ci_sim.yml index 000446d8af..e31aa760c8 100644 --- a/.github/workflows/ci_sim.yml +++ b/.github/workflows/ci_sim.yml @@ -201,6 +201,24 @@ jobs: python3 -m pip install 'scikit-build-core>=0.12.2,<2' 'pybind11<3' nanobind numpy ml-dtypes fi + - name: Configure ccache + shell: bash + run: | + set -euo pipefail + if ! command -v ccache >/dev/null 2>&1; then + if command -v sudo >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y ccache + else + echo "ccache is unavailable; continuing without a compiler launcher" + exit 0 + fi + fi + + ccache --version + echo "CMAKE_C_COMPILER_LAUNCHER=ccache" >> "${GITHUB_ENV}" + echo "CMAKE_CXX_COMPILER_LAUNCHER=ccache" >> "${GITHUB_ENV}" + - name: Resolve LLVM directories shell: bash run: | @@ -324,8 +342,9 @@ jobs: export CC=gcc export CXX=g++ LLVM_BUILD_DIR="${LLVM_DIR}" \ - SKBUILD_BUILD_DIR="${GITHUB_WORKSPACE}/build" \ - python3 -m pip wheel . --no-build-isolation --no-deps --wheel-dir "${PTO_WHEELHOUSE}" + python3 -m pip wheel . --no-build-isolation --no-deps \ + --config-settings="build-dir=${GITHUB_WORKSPACE}/build" \ + --wheel-dir "${PTO_WHEELHOUSE}" cmake --install "${GITHUB_WORKSPACE}/build" --prefix "${PTO_INSTALL_DIR}" - name: Create PTOAS simulator venv @@ -795,8 +814,9 @@ jobs: export CC=gcc export CXX=g++ LLVM_BUILD_DIR="${PTO_DSL_ST_LLVM_DIR}" \ - SKBUILD_BUILD_DIR="${PTO_DSL_ST_BUILD_DIR}" \ - "${PTO_DSL_ST_PYTHON_BIN}" -m pip wheel . --no-build-isolation --no-deps --wheel-dir "${PTO_DSL_ST_WHEELHOUSE}" + "${PTO_DSL_ST_PYTHON_BIN}" -m pip wheel . --no-build-isolation --no-deps \ + --config-settings="build-dir=${PTO_DSL_ST_BUILD_DIR}" \ + --wheel-dir "${PTO_DSL_ST_WHEELHOUSE}" cmake --install "${PTO_DSL_ST_BUILD_DIR}" --prefix "${PTO_DSL_ST_INSTALL_DIR}" - name: Install PTODSL PTOAS wheel diff --git a/lib/Bindings/Python/CMakeLists.txt b/lib/Bindings/Python/CMakeLists.txt index 215f385a8e..06b3829b44 100644 --- a/lib/Bindings/Python/CMakeLists.txt +++ b/lib/Bindings/Python/CMakeLists.txt @@ -106,35 +106,3 @@ install(FILES DESTINATION mlir/dialects COMPONENT PTOAS_Python ) - -# Keep the MLIR build-tree Python package self-consistent as well. When users -# import `mlir.dialects.pto` directly from MLIR_PYTHON_PACKAGE_DIR, the wrapper -# modules must be updated alongside the native `_pto` extension during normal -# builds, not just during `cmake --install`. -add_custom_command(TARGET _pto POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - "${MLIR_PYTHON_PACKAGE_DIR}/mlir/dialects" - COMMAND ${CMAKE_COMMAND} -E make_directory - "${MLIR_PYTHON_PACKAGE_DIR}/mlir/_mlir_libs" - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "${PTO_PY_SRC}" - "${MLIR_PYTHON_PACKAGE_DIR}/mlir/dialects/pto.py" - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "${CMAKE_CURRENT_BINARY_DIR}/_pto_ops_gen.py" - "${MLIR_PYTHON_PACKAGE_DIR}/mlir/dialects/_pto_ops_gen.py" - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "$" - "${MLIR_PYTHON_PACKAGE_DIR}/mlir/_mlir_libs/$" - VERBATIM -) - -add_custom_command(TARGET _pto POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - "${CMAKE_BINARY_DIR}/python/mlir/_mlir_libs" - COMMAND ${CMAKE_COMMAND} -E make_directory - "${MLIR_PYTHON_PACKAGE_DIR}/mlir/_mlir_libs" - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "$" - "${MLIR_PYTHON_PACKAGE_DIR}/mlir/_mlir_libs/$" - VERBATIM -) diff --git a/pyproject.toml b/pyproject.toml index 100b364b39..31a339e328 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,8 +7,9 @@ # See LICENSE in the root of the software repository for the full text of the License. # Standard PEP 517 configuration for the `ptoas` distribution. Set -# LLVM_BUILD_DIR to an existing LLVM/MLIR build. SKBUILD_BUILD_DIR can be used -# when a persistent CMake build tree is required by tests or development tools. +# LLVM_BUILD_DIR to an existing LLVM/MLIR build. Pass +# `--config-settings=build-dir=` when tests or development tools require +# a persistent CMake build tree. [build-system] requires = ["scikit-build-core>=0.12.2,<2", "pybind11<3"] diff --git a/quick_install.sh b/quick_install.sh index ec72e2830e..78a05bbe24 100755 --- a/quick_install.sh +++ b/quick_install.sh @@ -7,60 +7,31 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -# For quick development, build and install ptoas and its python bindings -# on top of Docker image https://github.com/learning-chip/agent_docker_npu/pull/8 -# assume MLIR is already installed to save time +# Build and install PTOAS in editable mode against an existing LLVM/MLIR +# build. The persistent build directory can subsequently be used with Ninja, +# for example: ninja -C build check-pto. # # Optional env: # LLVM_BUILD_DIR - default: ${LLVM_SOURCE_DIR:-/llvm-workspace/llvm-project}/build-shared -# PTO_INSTALL_DIR - default: /install +# PTO_BUILD_DIR - default: /build +# PYTHON_BIN - default: python set -euo pipefail PTO_SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PTO_INSTALL_DIR="${PTO_INSTALL_DIR:-${PTO_SOURCE_DIR}/install}" - LLVM_SOURCE_DIR="${LLVM_SOURCE_DIR:-/llvm-workspace/llvm-project}" LLVM_BUILD_DIR="${LLVM_BUILD_DIR:-${LLVM_SOURCE_DIR}/build-shared}" +PTO_BUILD_DIR="${PTO_BUILD_DIR:-${PTO_SOURCE_DIR}/build}" +PYTHON_BIN="${PYTHON_BIN:-python}" -for d in "$LLVM_BUILD_DIR/lib/cmake/llvm" "$LLVM_BUILD_DIR/lib/cmake/mlir"; do - test -d "$d" || { echo "error: missing $d (set LLVM_BUILD_DIR?)" >&2; exit 1; } -done - -MLIR_PY_PKG="${LLVM_BUILD_DIR}/tools/mlir/python_packages/mlir_core" -test -d "$MLIR_PY_PKG" || { echo "error: MLIR python package dir missing: $MLIR_PY_PKG" >&2; exit 1; } - -PTOAS_VERSION="${PTOAS_VERSION:-$(python "${PTO_SOURCE_DIR}/.github/scripts/compute_ptoas_version.py" --cmake-file "${PTO_SOURCE_DIR}/CMakeLists.txt" --mode dev)}" +if command -v ccache >/dev/null 2>&1; then + export CMAKE_C_COMPILER_LAUNCHER="${CMAKE_C_COMPILER_LAUNCHER:-ccache}" + export CMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER:-ccache}" +fi -cd "$PTO_SOURCE_DIR" - -PTO_WHEEL_DIR="${PTO_SOURCE_DIR}/build/wheel-dist" -rm -rf "${PTO_WHEEL_DIR}" -mkdir -p "${PTO_WHEEL_DIR}" LLVM_BUILD_DIR="${LLVM_BUILD_DIR}" \ -PTOAS_RELEASE_VERSION_OVERRIDE="${PTOAS_VERSION}" \ -SKBUILD_BUILD_DIR="${PTO_SOURCE_DIR}/build" \ - python -m pip wheel . --no-deps --wheel-dir "${PTO_WHEEL_DIR}" - -cmake --install "${PTO_SOURCE_DIR}/build" --prefix "${PTO_INSTALL_DIR}" - -shopt -s nullglob -wheels=("${PTO_WHEEL_DIR}/ptoas-"*.whl) -shopt -u nullglob -((${#wheels[@]} > 0)) || { echo "error: no ptoas-*.whl under ${PTO_WHEEL_DIR}" >&2; exit 1; } -pip install --force-reinstall "${wheels[0]}" - -export PATH="${PTO_SOURCE_DIR}/build/tools/ptoas:${PATH}" -export LD_LIBRARY_PATH="${LLVM_BUILD_DIR}/lib:${PTO_INSTALL_DIR}/lib:${LD_LIBRARY_PATH:-}" - -python -c "import mlir.ir" -python -c "from mlir.dialects import pto" - -which ptoas - -PTOAS_ENV_TMP="${PTO_SOURCE_DIR}/tmp/set_ptoas_env" -mkdir -p "${PTOAS_ENV_TMP}/MatMul" "${PTOAS_ENV_TMP}/Abs" -(cd "${PTO_SOURCE_DIR}/test/samples/MatMul" && python ./tmatmulk.py > "${PTOAS_ENV_TMP}/MatMul/tmatmulk.pto" && ptoas "${PTOAS_ENV_TMP}/MatMul/tmatmulk.pto" -o "${PTOAS_ENV_TMP}/MatMul/tmatmulk.cpp") -(cd "${PTO_SOURCE_DIR}/test/samples/Abs" && python ./abs.py > "${PTOAS_ENV_TMP}/Abs/abs.pto" && ptoas --enable-insert-sync "${PTOAS_ENV_TMP}/Abs/abs.pto" -o "${PTOAS_ENV_TMP}/Abs/abs.cpp") + "${PYTHON_BIN}" -m pip install --editable "${PTO_SOURCE_DIR}" \ + --config-settings="build-dir=${PTO_BUILD_DIR}" -echo "quick_install.sh: OK" +echo "PTOAS editable install complete." +echo "Build directory: ${PTO_BUILD_DIR}" From 089f100c7984f57089aacdf0e21d263eaad20e25 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Thu, 23 Jul 2026 23:18:41 +0800 Subject: [PATCH 08/32] Use editable installs in CI workflows --- .github/workflows/ci.yml | 28 ++++++---------- .github/workflows/ci_sim.yml | 64 ++++++++++-------------------------- 2 files changed, 27 insertions(+), 65 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1544fcd06..8861b79783 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,7 +103,6 @@ jobs: LLVM_DIR: ${{ github.workspace }}/llvm-project/llvm/build-assert PTO_BUILD_DIR: ${{ github.workspace }}/build-assert PTO_INSTALL_DIR: ${{ github.workspace }}/install-assert - PTO_WHEELHOUSE: ${{ github.workspace }}/wheelhouse-assert PTOAS_VENV: ${{ github.workspace }}/.venv-ptoas-assert MLIR_PYTHONPATH: ${{ github.workspace }}/llvm-project/llvm/build-assert/tools/mlir/python_packages/mlir_core LLVM_CACHE_FLAVOR: llvm21-assert-shared-mlirpy-hardening-v1 @@ -271,35 +270,28 @@ jobs: "${PTOAS_VENV}/bin/python" -m pip install --upgrade pip "${PTOAS_VENV}/bin/pip" install 'scikit-build-core>=0.12.2,<2' 'pybind11<3' nanobind numpy ml-dtypes - - name: Build PTOAS wheel + - name: Install PTOAS editable run: | set -euo pipefail rm -rf "${PTO_BUILD_DIR}" rm -rf "${PTO_INSTALL_DIR}" - rm -rf "${PTO_WHEELHOUSE}" - mkdir -p "${PTO_WHEELHOUSE}" - # scikit-build-core reuses this build tree so the following CTest and - # lit steps exercise the exact sources used to produce the wheel. + # Keep a persistent CMake build tree for CTest and lit while exposing + # the current sources and Python extension through the developer venv. LLVM_BUILD_DIR="${LLVM_DIR}" \ - SKBUILD_BUILD_DIR="${PTO_BUILD_DIR}" \ CC="${PTOAS_CMAKE_C_COMPILER}" \ CXX="${PTOAS_CMAKE_CXX_COMPILER}" \ - "${PTOAS_VENV}/bin/python" -m pip wheel . --no-build-isolation --no-deps --wheel-dir "${PTO_WHEELHOUSE}" + "${PTOAS_VENV}/bin/python" -m pip install --editable . \ + --no-build-isolation --no-deps \ + --config-settings="build-dir=${PTO_BUILD_DIR}" cmake --install "${PTO_BUILD_DIR}" --prefix "${PTO_INSTALL_DIR}" - - name: Install PTOAS wheel into test venv + - name: Test PTOAS editable install shell: bash run: | set -euo pipefail - readarray -t ptoas_wheels < <(find "${PTO_WHEELHOUSE}" -maxdepth 1 -type f -name 'ptoas*.whl' | sort) - if [[ "${#ptoas_wheels[@]}" -ne 1 ]]; then - echo "ERROR: expected exactly one PTOAS wheel in ${PTO_WHEELHOUSE}, found ${#ptoas_wheels[@]}" >&2 - printf ' %s\n' "${ptoas_wheels[@]}" >&2 - exit 1 - fi - - "${PTOAS_VENV}/bin/pip" install --no-deps --force-reinstall "${ptoas_wheels[0]}" - PATH="${PTOAS_VENV}/bin:${PATH}" bash docker/test_wheel_imports.sh + PATH="${PTOAS_VENV}/bin:${PATH}" \ + PYTHON="${PTOAS_VENV}/bin/python" \ + bash docker/test_wheel_imports.sh - name: Run PTODSL Python tests shell: bash diff --git a/.github/workflows/ci_sim.yml b/.github/workflows/ci_sim.yml index e31aa760c8..5f6ef605cf 100644 --- a/.github/workflows/ci_sim.yml +++ b/.github/workflows/ci_sim.yml @@ -112,7 +112,6 @@ jobs: LLVM_REPO: https://github.com/vpto-dev/llvm-project.git LLVM_REF: feature-vpto-llvm21 PTO_INSTALL_DIR: ${{ github.workspace }}/install - PTO_WHEELHOUSE: ${{ github.workspace }}/.work/ptoas-wheelhouse PTOAS_VENV: ${{ github.workspace }}/.work/ptoas-sim-venv VPTO_SIM_WORKSPACE: ${{ github.workspace }}/.work/vpto-sim-ci TILELANG_DSL_WORKSPACE: ${{ github.workspace }}/.work/tilelang-dsl-ci @@ -233,7 +232,6 @@ jobs: set -euo pipefail echo "PTO_DSL_ST_BUILD_DIR=${GITHUB_WORKSPACE}/build-ptodsl" >> "${GITHUB_ENV}" echo "PTO_DSL_ST_INSTALL_DIR=${GITHUB_WORKSPACE}/install-ptodsl" >> "${GITHUB_ENV}" - echo "PTO_DSL_ST_WHEELHOUSE=${GITHUB_WORKSPACE}/.work/ptodsl-wheelhouse" >> "${GITHUB_ENV}" - name: Resolve LLVM cache key id: llvm-cache-key @@ -267,8 +265,6 @@ jobs: rm -rf "${GITHUB_WORKSPACE}/build-ptodsl" rm -rf "${PTO_INSTALL_DIR}" rm -rf "${PTO_DSL_ST_INSTALL_DIR}" - rm -rf "${PTO_DSL_ST_WHEELHOUSE}" - rm -rf "${PTO_WHEELHOUSE}" rm -rf "${PTOAS_VENV}" rm -rf "${VPTO_SIM_WORKSPACE}" rm -rf "${TILELANG_DSL_WORKSPACE}" @@ -331,39 +327,28 @@ jobs: path: ${{ env.LLVM_DIR }} key: ${{ steps.llvm-cache-key.outputs.key }} - - name: Build PTOAS wheel + - name: Install PTOAS editable in simulator venv shell: bash run: | set -euo pipefail rm -rf "${PTO_INSTALL_DIR}" - rm -rf "${PTO_WHEELHOUSE}" - mkdir -p "${PTO_WHEELHOUSE}" + rm -rf "${PTOAS_VENV}" + python3 -m venv "${PTOAS_VENV}" + "${PTOAS_VENV}/bin/python" -m pip install --upgrade pip + "${PTOAS_VENV}/bin/pip" install \ + 'scikit-build-core>=0.12.2,<2' 'pybind11<3' \ + nanobind numpy ml-dtypes # Pin system compilers for the same reason as LLVM above. export CC=gcc export CXX=g++ LLVM_BUILD_DIR="${LLVM_DIR}" \ - python3 -m pip wheel . --no-build-isolation --no-deps \ - --config-settings="build-dir=${GITHUB_WORKSPACE}/build" \ - --wheel-dir "${PTO_WHEELHOUSE}" + "${PTOAS_VENV}/bin/python" -m pip install --editable . \ + --no-build-isolation --no-deps \ + --config-settings="build-dir=${GITHUB_WORKSPACE}/build" cmake --install "${GITHUB_WORKSPACE}/build" --prefix "${PTO_INSTALL_DIR}" - - - name: Create PTOAS simulator venv - shell: bash - run: | - set -euo pipefail - rm -rf "${PTOAS_VENV}" - python3 -m venv "${PTOAS_VENV}" - "${PTOAS_VENV}/bin/python" -m pip install --upgrade pip - - readarray -t ptoas_wheels < <(find "${PTO_WHEELHOUSE}" -maxdepth 1 -type f -name 'ptoas*.whl' | sort) - if [[ "${#ptoas_wheels[@]}" -ne 1 ]]; then - echo "ERROR: expected exactly one PTOAS wheel in ${PTO_WHEELHOUSE}, found ${#ptoas_wheels[@]}" >&2 - printf ' %s\n' "${ptoas_wheels[@]}" >&2 - exit 1 - fi - - "${PTOAS_VENV}/bin/pip" install numpy ml-dtypes "${ptoas_wheels[0]}" - PATH="${PTOAS_VENV}/bin:${PATH}" bash docker/test_wheel_imports.sh + PATH="${PTOAS_VENV}/bin:${PATH}" \ + PYTHON="${PTOAS_VENV}/bin/python" \ + bash docker/test_wheel_imports.sh - name: Checkout PyPTO uses: actions/checkout@v4 @@ -804,34 +789,19 @@ jobs: path: ${{ env.PTO_DSL_ST_LLVM_DIR }} key: llvm-build-${{ steps.llvm-cache-key.outputs.sha }}-assert-${{ env.PTO_DSL_ST_PYTHON_TAG }}-v2 - - name: Build PTODSL PTOAS wheel + - name: Install PTODSL PTOAS editable shell: bash run: | set -euo pipefail rm -rf "${PTO_DSL_ST_BUILD_DIR}" "${PTO_DSL_ST_INSTALL_DIR}" - rm -rf "${PTO_DSL_ST_WHEELHOUSE}" - mkdir -p "${PTO_DSL_ST_WHEELHOUSE}" export CC=gcc export CXX=g++ LLVM_BUILD_DIR="${PTO_DSL_ST_LLVM_DIR}" \ - "${PTO_DSL_ST_PYTHON_BIN}" -m pip wheel . --no-build-isolation --no-deps \ - --config-settings="build-dir=${PTO_DSL_ST_BUILD_DIR}" \ - --wheel-dir "${PTO_DSL_ST_WHEELHOUSE}" + "${PTO_DSL_ST_PYTHON_BIN}" -m pip install --editable . \ + --no-build-isolation --no-deps \ + --config-settings="build-dir=${PTO_DSL_ST_BUILD_DIR}" cmake --install "${PTO_DSL_ST_BUILD_DIR}" --prefix "${PTO_DSL_ST_INSTALL_DIR}" - - name: Install PTODSL PTOAS wheel - shell: bash - run: | - set -euo pipefail - readarray -t ptodsl_wheels < <(find "${PTO_DSL_ST_WHEELHOUSE}" -maxdepth 1 -type f -name 'ptoas*.whl' | sort) - if [[ "${#ptodsl_wheels[@]}" -ne 1 ]]; then - echo "ERROR: expected exactly one PTOAS wheel in ${PTO_DSL_ST_WHEELHOUSE}, found ${#ptodsl_wheels[@]}" >&2 - printf ' %s\n' "${ptodsl_wheels[@]}" >&2 - exit 1 - fi - - "${PTO_DSL_ST_PYTHON_BIN}" -m pip install --force-reinstall "${ptodsl_wheels[0]}" - - name: Probe PTODSL runtime Python shell: bash run: | From d239d5b56252c2b1eda1a4b7c66fcaa9722d2488 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Thu, 23 Jul 2026 23:28:42 +0800 Subject: [PATCH 09/32] Run wheel repair in pull request CI --- .github/workflows/build_wheel.yml | 18 ++---------------- .github/workflows/build_wheel_mac.yml | 19 +++---------------- 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 8f6be18831..2730cd6634 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -227,24 +227,11 @@ jobs: --wheel-dir "${PTO_SOURCE_DIR}/build/wheel-dist" cmake --install "${PTO_BUILD_DIR}" --prefix "${PTO_INSTALL_DIR}" - - name: Install built wheel for Python sample generation - run: | - shopt -s nullglob - wheels=("$PTO_SOURCE_DIR"/build/wheel-dist/ptoas*.whl) - if [ "${#wheels[@]}" -ne 1 ]; then - echo "Expected exactly one wheel in $PTO_SOURCE_DIR/build/wheel-dist, found ${#wheels[@]}" >&2 - printf ' %s\n' "${wheels[@]}" >&2 - exit 1 - fi - python -m pip install --no-deps --force-reinstall "${wheels[0]}" - - name: Validate wheel payload and launcher contract - if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: | python "$PTO_SOURCE_DIR/docker/validate_wheel_payload.py" "$PTO_SOURCE_DIR/build/wheel-dist" - name: Repair wheel with auditwheel - if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: | export PTO_WHEEL_DIST_DIR=$PTO_SOURCE_DIR/build/wheel-dist export PTO_WHEELHOUSE=$GITHUB_WORKSPACE/wheelhouse @@ -253,10 +240,9 @@ jobs: auditwheel repair --plat manylinux_2_34_${{ matrix.arch }} "$PTO_WHEEL_DIST_DIR"/ptoas*.whl -w "$PTO_WHEELHOUSE" - name: Test wheel installation - if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: | - pip install $GITHUB_WORKSPACE/wheelhouse/ptoas*.whl - bash $PTO_SOURCE_DIR/docker/test_wheel_imports.sh + PTO_TEST_WHEEL_PATH="$GITHUB_WORKSPACE/wheelhouse/ptoas*.whl" \ + bash $PTO_SOURCE_DIR/docker/test_wheel_imports.sh - name: Test ptoas CLI run: | diff --git a/.github/workflows/build_wheel_mac.yml b/.github/workflows/build_wheel_mac.yml index b78213a34f..0e2ffc4058 100644 --- a/.github/workflows/build_wheel_mac.yml +++ b/.github/workflows/build_wheel_mac.yml @@ -12,6 +12,7 @@ on: push: tags-ignore: - '**' + pull_request: # Nightly build & publish (UTC time, offset from Linux workflow). schedule: - cron: "40 17 * * *" @@ -234,24 +235,11 @@ jobs: printf 'Built wheel file: %s\n' "$(basename "${built_wheels[0]}")" fi - - name: Install built wheel for Python sample generation - run: | - shopt -s nullglob - wheels=("$PTO_SOURCE_DIR"/build/wheel-dist/ptoas*.whl) - if [ "${#wheels[@]}" -ne 1 ]; then - echo "Expected exactly one wheel in $PTO_SOURCE_DIR/build/wheel-dist, found ${#wheels[@]}" >&2 - printf ' %s\n' "${wheels[@]}" >&2 - exit 1 - fi - python -m pip install --no-deps --force-reinstall "${wheels[0]}" - - name: Validate wheel payload and launcher contract - if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: | python "$PTO_SOURCE_DIR/docker/validate_wheel_payload.py" "$PTO_SOURCE_DIR/build/wheel-dist" - name: Repair wheel with delocate - if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: | set -euo pipefail export PTO_WHEEL_DIST_DIR=$PTO_SOURCE_DIR/build/wheel-dist @@ -327,11 +315,10 @@ jobs: ls -lh "$PTO_WHEELHOUSE" - name: Test wheel installation - if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' run: | - pip install $GITHUB_WORKSPACE/wheelhouse/ptoas*.whl export DYLD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:${DYLD_LIBRARY_PATH} - bash $PTO_SOURCE_DIR/docker/test_wheel_imports.sh + PTO_TEST_WHEEL_PATH="$GITHUB_WORKSPACE/wheelhouse/ptoas*.whl" \ + bash $PTO_SOURCE_DIR/docker/test_wheel_imports.sh - name: Test ptoas CLI run: | From 3f1efa3460dbd7bc38bfc28e07b9d84bbd2814d3 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Fri, 24 Jul 2026 02:00:35 +0800 Subject: [PATCH 10/32] ci: streamline wheel packaging checks --- .github/workflows/build_wheel.yml | 33 ++-- .github/workflows/build_wheel_mac.yml | 67 ++----- docker/Dockerfile | 2 +- packaging/ptoas-vmi/pyproject.toml | 3 +- pyproject.toml | 3 +- test/python/test_docker_runtime_packaging.py | 98 ---------- test/python/test_scikit_build_config.py | 120 ------------ test/python/test_validate_wheel_payload.py | 195 ------------------- 8 files changed, 40 insertions(+), 481 deletions(-) delete mode 100644 test/python/test_docker_runtime_packaging.py delete mode 100644 test/python/test_scikit_build_config.py delete mode 100644 test/python/test_validate_wheel_payload.py diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 2730cd6634..8952245bde 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -10,8 +10,8 @@ name: Build Wheel on: push: - tags-ignore: - - '**' + branches: + - main pull_request: # Nightly build & publish (UTC time). schedule: @@ -29,7 +29,11 @@ on: types: [prereleased, released] permissions: - contents: write + contents: read + +concurrency: + group: build-wheel-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }} env: LLVM_REPO: https://github.com/vpto-dev/llvm-project.git @@ -41,7 +45,7 @@ jobs: if: github.event_name != 'release' || startsWith(github.ref_name, 'v') || startsWith(github.ref_name, 'ptoas-v') # Non-release runs keep a single-Python matrix for faster gates; # release/nightly runs fan out to publish the full wheel set plus binary artifacts. - name: Build ptoas-bin (${{ matrix.arch }}, py${{ matrix.python }}) + name: Build ptoas wheel (${{ matrix.arch }}, py${{ matrix.python }}) runs-on: ${{ matrix.arch == 'x86_64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }} strategy: fail-fast: false @@ -59,11 +63,13 @@ jobs: - name: Set Python version run: | case "${{ matrix.python }}" in - "3.9") PY_VER="cp39-cp39" ;; "3.10") PY_VER="cp310-cp310" ;; "3.11") PY_VER="cp311-cp311" ;; "3.12") PY_VER="cp312-cp312" ;; - *) PY_VER="cp311-cp311" ;; + *) + echo "Unsupported Python version: ${{ matrix.python }}" >&2 + exit 1 + ;; esac echo "PY_VER=$PY_VER" >> $GITHUB_ENV echo "PY_PATH=/opt/python/$PY_VER" >> $GITHUB_ENV @@ -123,7 +129,7 @@ jobs: - name: Install system dependencies run: | - dnf install -y ninja-build cmake git ccache gcc-c++ lld zip binutils patchelf chrpath + dnf install -y ninja-build cmake git gcc-c++ lld zip binutils patchelf chrpath dnf clean all - name: Install Python dependencies @@ -172,8 +178,8 @@ jobs: git remote add origin "${LLVM_REPO}" fi git remote set-url origin "${LLVM_REPO}" - git fetch --depth 1 origin "${LLVM_REF}" - git checkout --force FETCH_HEAD + git fetch --depth 1 origin "${{ steps.llvm-source.outputs.sha }}" + git checkout --force "${{ steps.llvm-source.outputs.sha }}" - name: Warn when default-branch cache is missing for PR/release runs if: steps.cache-llvm.outputs.cache-hit != 'true' && (github.event_name == 'pull_request' || github.event_name == 'release') @@ -244,11 +250,6 @@ jobs: PTO_TEST_WHEEL_PATH="$GITHUB_WORKSPACE/wheelhouse/ptoas*.whl" \ bash $PTO_SOURCE_DIR/docker/test_wheel_imports.sh - - name: Test ptoas CLI - run: | - export PTOAS_BIN="${PTO_BUILD_DIR}/tools/ptoas/ptoas" - bash $PTO_SOURCE_DIR/docker/test_ptoas_cli.sh - - name: Upload wheel artifact if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@v4 @@ -280,6 +281,8 @@ jobs: if: (github.event_name == 'release' && (startsWith(github.ref_name, 'v') || startsWith(github.ref_name, 'ptoas-v'))) || github.event_name == 'schedule' needs: build_wheel runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Compute release metadata @@ -370,6 +373,8 @@ jobs: if: github.event_name == 'release' && github.event.action == 'released' && (startsWith(github.ref_name, 'ptoas-v') || (startsWith(github.ref_name, 'v') && !startsWith(github.ref_name, 'vmi-v'))) needs: upload_release_assets runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout default branch diff --git a/.github/workflows/build_wheel_mac.yml b/.github/workflows/build_wheel_mac.yml index 0e2ffc4058..5ac014e9dc 100644 --- a/.github/workflows/build_wheel_mac.yml +++ b/.github/workflows/build_wheel_mac.yml @@ -10,8 +10,8 @@ name: Build Wheel (macOS) on: push: - tags-ignore: - - '**' + branches: + - main pull_request: # Nightly build & publish (UTC time, offset from Linux workflow). schedule: @@ -29,7 +29,11 @@ on: types: [prereleased, released] permissions: - contents: write + contents: read + +concurrency: + group: build-wheel-macos-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }} env: LLVM_REPO: https://github.com/vpto-dev/llvm-project.git @@ -41,16 +45,13 @@ jobs: if: github.event_name != 'release' || startsWith(github.ref_name, 'v') || startsWith(github.ref_name, 'ptoas-v') # Non-release runs keep a single-Python matrix for faster gates; # release/nightly runs fan out to publish the full wheel set plus binary artifacts. - name: Build ptoas-bin (macOS ${{ matrix.arch }}, py${{ matrix.python }}) + name: Build ptoas wheel (macOS ${{ matrix.arch }}, py${{ matrix.python }}) runs-on: ${{ matrix.arch == 'x86_64' && 'macos-15-intel' || 'macos-26' }} strategy: fail-fast: false matrix: python: ${{ fromJson((github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '["3.10","3.11","3.12"]' || '["3.11"]') }} arch: ["x86_64", "aarch64"] - exclude: - - python: "3.10" - arch: "x86_64" steps: - name: Checkout repository @@ -128,7 +129,7 @@ jobs: - name: Install system dependencies run: | - brew install ninja ccache + brew install ninja - name: Install Python dependencies run: | @@ -136,6 +137,11 @@ jobs: # (see pybind11 static_assert on def_property + keep_alive). pip install --no-cache-dir numpy 'pybind11<3' nanobind 'scikit-build-core>=0.12.2,<2' delocate + - name: Validate PEP 517 project metadata + run: | + python -m scikit_build_core.build project-table + (cd packaging/ptoas-vmi && python -m scikit_build_core.build project-table) + - name: Set build directories run: | echo "WORKSPACE_DIR=${RUNNER_TEMP}/llvm-workspace" >> $GITHUB_ENV @@ -171,8 +177,8 @@ jobs: git remote add origin "${LLVM_REPO}" fi git remote set-url origin "${LLVM_REPO}" - git fetch --depth 1 origin "${LLVM_REF}" - git checkout --force FETCH_HEAD + git fetch --depth 1 origin "${{ steps.llvm-source.outputs.sha }}" + git checkout --force "${{ steps.llvm-source.outputs.sha }}" - name: Warn when default-branch cache is missing for PR/release runs if: steps.cache-llvm.outputs.cache-hit != 'true' && (github.event_name == 'pull_request' || github.event_name == 'release') @@ -316,15 +322,9 @@ jobs: - name: Test wheel installation run: | - export DYLD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:${DYLD_LIBRARY_PATH} PTO_TEST_WHEEL_PATH="$GITHUB_WORKSPACE/wheelhouse/ptoas*.whl" \ bash $PTO_SOURCE_DIR/docker/test_wheel_imports.sh - - name: Test ptoas CLI - run: | - export PTOAS_BIN="${PTO_BUILD_DIR}/tools/ptoas/ptoas" - bash $PTO_SOURCE_DIR/docker/test_ptoas_cli.sh - - name: Upload wheel artifact if: github.event_name == 'release' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' uses: actions/upload-artifact@v4 @@ -363,39 +363,6 @@ jobs: "$PTO_SOURCE_DIR/test/lit/pto/kernel_kind_vector_scf_while_emitc.pto" \ >/dev/null - - name: Smoke test wheel imports after collecting artifacts - if: (github.event_name == 'release' || github.event_name == 'schedule') && matrix.python == '3.11' - run: | - # Test the copied wheel artifact from an isolated env with no build-tree - # paths, so post-collect/release regressions (e.g. ARM cs_invalid_page) - # are caught before upload. - brew install uv - uv python install 3.11 - TEST_DIR="$RUNNER_TEMP/wheel-smoke-test" - VENV_DIR="$TEST_DIR/.venv" - mkdir -p "$TEST_DIR" - cd "$TEST_DIR" - uv venv --python 3.11 "$VENV_DIR" - source "$VENV_DIR/bin/activate" - - unset PYTHONPATH - unset DYLD_LIBRARY_PATH - unset LD_LIBRARY_PATH - - shopt -s nullglob - wheels=("$GITHUB_WORKSPACE"/wheelhouse/ptoas*.whl) - if [ "${#wheels[@]}" -ne 1 ]; then - echo "Expected exactly one wheel in \$GITHUB_WORKSPACE/wheelhouse, found ${#wheels[@]}:" - printf ' %s\n' "${wheels[@]}" - exit 1 - fi - uv pip install "${wheels[0]}" - - if [ "${{ matrix.arch }}" = "aarch64" ]; then - export DYLD_PRINT_LIBRARIES=1 - fi - bash "$GITHUB_WORKSPACE/docker/test_wheel_imports.sh" - - name: Upload ptoas binary artifact if: matrix.python == '3.11' uses: actions/upload-artifact@v4 @@ -408,6 +375,8 @@ jobs: if: (github.event_name == 'release' && (startsWith(github.ref_name, 'v') || startsWith(github.ref_name, 'ptoas-v'))) || github.event_name == 'schedule' needs: build_wheel runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Compute release metadata diff --git a/docker/Dockerfile b/docker/Dockerfile index 8a46534173..3bdd056ba4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,7 +19,7 @@ ENV PY_PATH="/opt/python/${PY_VER}" ENV PATH="${PY_PATH}/bin:${PATH}" # dependency -RUN dnf install -y ninja-build cmake git ccache gcc-c++ lld zip binutils patchelf chrpath && dnf clean all +RUN dnf install -y ninja-build cmake git gcc-c++ lld zip binutils patchelf chrpath && dnf clean all RUN pip install --no-cache-dir numpy 'pybind11<3' nanobind 'scikit-build-core>=0.12.2,<2' auditwheel COPY cmake/LinuxHardeningCache.cmake /tmp/LinuxHardeningCache.cmake diff --git a/packaging/ptoas-vmi/pyproject.toml b/packaging/ptoas-vmi/pyproject.toml index d9c28bd5e4..c5bf63ca99 100644 --- a/packaging/ptoas-vmi/pyproject.toml +++ b/packaging/ptoas-vmi/pyproject.toml @@ -19,7 +19,7 @@ build-backend = "scikit_build_core.build" name = "ptoas-vmi" dynamic = ["version"] description = "PTO Assembler & Optimizer with VMI support" -requires-python = ">=3.9" +requires-python = ">=3.10" license = "Apache-2.0" dependencies = [ "numpy", @@ -28,7 +28,6 @@ classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", diff --git a/pyproject.toml b/pyproject.toml index 31a339e328..072d2fb95f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ name = "ptoas" dynamic = ["version"] description = "PTO Assembler & Optimizer" readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" license = "Apache-2.0" dependencies = [ "numpy", @@ -29,7 +29,6 @@ classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", diff --git a/test/python/test_docker_runtime_packaging.py b/test/python/test_docker_runtime_packaging.py deleted file mode 100644 index a56475ae74..0000000000 --- a/test/python/test_docker_runtime_packaging.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -import unittest -from pathlib import Path - - -REPO_ROOT = Path(__file__).resolve().parents[2] -DOCKERFILE = REPO_ROOT / "docker" / "Dockerfile" -COLLECT_DIST_SCRIPT = REPO_ROOT / "docker" / "collect_ptoas_dist.sh" -COLLECT_DIST_MAC_SCRIPT = REPO_ROOT / "docker" / "collect_ptoas_dist_mac.sh" -PTOAS_CMAKE = REPO_ROOT / "tools" / "ptoas" / "CMakeLists.txt" -PTOAS_CLI_TEST = REPO_ROOT / "docker" / "test_ptoas_cli.sh" - - -class DockerRuntimePackagingTests(unittest.TestCase): - def test_runtime_uses_standard_python_extension_without_native_cli(self): - cmake = PTOAS_CMAKE.read_text(encoding="utf-8") - - self.assertIn( - "pybind11_add_module(ptoas_runtime MODULE", - cmake, - ) - self.assertIn('OUTPUT_NAME "_native"', cmake) - self.assertIn('BUILD_RPATH "${PTO_LLVM_BUILD_LIBRARY_DIR}"', cmake) - self.assertIn('INSTALL_RPATH "${PTO_LLVM_BUILD_LIBRARY_DIR}"', cmake) - self.assertNotIn("add_executable(ptoas_native_cli", cmake) - self.assertNotIn('OUTPUT_NAME "ptoas-native"', cmake) - - def test_runtime_image_uses_wheel_entrypoint_instead_of_copied_wrapper(self): - dockerfile = DOCKERFILE.read_text(encoding="utf-8") - - self.assertIn("COPY --from=builder /wheelhouse/ptoas*.whl /tmp/", dockerfile) - self.assertIn( - "RUN pip install --no-cache-dir /tmp/ptoas*.whl && rm /tmp/ptoas*.whl", - dockerfile, - ) - self.assertNotIn( - "COPY --from=builder /llvm-workspace/PTOAS/build-release/tools/ptoas/ptoas /usr/local/bin/ptoas", - dockerfile, - ) - self.assertNotIn("/usr/local/lib/ptoas", dockerfile) - - def test_docker_builds_wheel_through_standard_pep517_backend(self): - dockerfile = DOCKERFILE.read_text(encoding="utf-8") - - self.assertIn("python -m pip wheel .", dockerfile) - self.assertIn("SKBUILD_BUILD_DIR=$PTO_BUILD_DIR", dockerfile) - self.assertNotIn("create_wheel.sh", dockerfile) - self.assertNotIn("_ptoas_build_backend", dockerfile) - - def test_linux_dist_packages_python_wrapper_and_native_extension(self): - script = COLLECT_DIST_SCRIPT.read_text(encoding="utf-8") - - self.assertIn('PTOAS_BIN="${PTO_INSTALL_DIR}/bin/ptoas"', script) - self.assertIn('PTOAS_PACKAGE_SRC_DIR="${PTO_INSTALL_DIR}/ptoas"', script) - self.assertIn('PTOAS_NATIVE_MODULE="$(find "${PTOAS_PACKAGE_DIST_DIR}"', script) - self.assertIn("patchelf --set-rpath '$ORIGIN/../lib' \"${PTOAS_NATIVE_MODULE}\"", script) - self.assertIn('"${PTOAS_DIST_DIR}/bin/ptoas" --version', script) - self.assertNotIn("ptoas-native", script) - self.assertNotIn("ptoas.so", script) - - def test_macos_dist_packages_python_wrapper_and_native_extension(self): - script = COLLECT_DIST_MAC_SCRIPT.read_text(encoding="utf-8") - - self.assertIn('PTOAS_BIN="${PTO_INSTALL_DIR}/bin/ptoas"', script) - self.assertIn('PTOAS_PACKAGE_SRC_DIR="${PTO_INSTALL_DIR}/ptoas"', script) - self.assertIn('PTOAS_NATIVE_MODULE="$(find "${PTOAS_PACKAGE_DIST_DIR}"', script) - self.assertIn('collect_dylibs "${PTOAS_NATIVE_MODULE}"', script) - self.assertIn('"${PTOAS_DIST_DIR}/bin/ptoas" --version', script) - self.assertNotIn("ptoas-native", script) - self.assertNotIn("ptoas.so", script) - - def test_wheel_import_smoke_imports_native_extension(self): - script = (REPO_ROOT / "docker" / "test_wheel_imports.sh").read_text(encoding="utf-8") - - self.assertIn("from ptoas import _native", script) - self.assertNotIn("POLLUTED_PTOAS_VERSION_OUTPUT", script) - - def test_build_tree_cli_test_receives_explicit_test_environment(self): - script = PTOAS_CLI_TEST.read_text(encoding="utf-8") - - self.assertIn("for var in PTO_SOURCE_DIR PTOAS_BIN", script) - self.assertIn("python ./tmatmulk.py", script) - self.assertIn("python ./abs.py", script) - self.assertNotIn("PYTHON_BIN", script) - self.assertNotIn('command -v ptoas', script) - self.assertNotIn('export PATH="${PTO_BUILD_DIR}', script) - self.assertNotIn('export LD_LIBRARY_PATH="${LLVM_BUILD_DIR}', script) - -if __name__ == "__main__": - unittest.main() diff --git a/test/python/test_scikit_build_config.py b/test/python/test_scikit_build_config.py deleted file mode 100644 index 3f013e71dc..0000000000 --- a/test/python/test_scikit_build_config.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -import json -import subprocess -import sys -import tomllib -import unittest -from pathlib import Path - - -REPO_ROOT = Path(__file__).resolve().parents[2] -PTOAS_PYPROJECT = REPO_ROOT / "pyproject.toml" -VMI_PROJECT_DIR = REPO_ROOT / "packaging" / "ptoas-vmi" -VMI_PYPROJECT = VMI_PROJECT_DIR / "pyproject.toml" -PTOAS_PACKAGE_INIT = REPO_ROOT / "ptodsl" / "ptoas" / "__init__.py" - - -def _read_pyproject(path: Path) -> dict: - with path.open("rb") as file: - return tomllib.load(file) - - -def _project_table(project_dir: Path) -> dict: - result = subprocess.run( - [ - sys.executable, - "-m", - "scikit_build_core.build", - "project-table", - ], - cwd=project_dir, - capture_output=True, - text=True, - check=False, - ) - if result.returncode != 0: - raise AssertionError(result.stderr or result.stdout) - return json.loads(result.stdout) - - -class ScikitBuildConfigTests(unittest.TestCase): - def test_root_project_uses_standard_backend_and_console_script(self): - config = _read_pyproject(PTOAS_PYPROJECT) - - self.assertEqual( - config["build-system"]["build-backend"], - "scikit_build_core.build", - ) - self.assertIn("scikit-build-core>=0.12.2,<2", config["build-system"]["requires"]) - self.assertEqual(config["project"]["name"], "ptoas") - self.assertEqual(config["project"]["scripts"]["ptoas"], "ptoas._cli:main") - self.assertEqual( - config["tool"]["scikit-build"]["install"]["components"], - ["PTOAS_Python"], - ) - - def test_static_projects_resolve_expected_distribution_metadata(self): - ptoas = _project_table(REPO_ROOT) - vmi = _project_table(VMI_PROJECT_DIR) - - self.assertEqual((ptoas["name"], ptoas["version"]), ("ptoas", "0.51")) - self.assertEqual((vmi["name"], vmi["version"]), ("ptoas-vmi", "0.1.3")) - self.assertEqual(ptoas["scripts"]["ptoas"], "ptoas._cli:main") - self.assertEqual(vmi["scripts"]["ptoas"], "ptoas._cli:main") - - def test_projects_share_python_package_sources(self): - ptoas = _read_pyproject(PTOAS_PYPROJECT) - vmi = _read_pyproject(VMI_PYPROJECT) - - self.assertEqual( - ptoas["tool"]["scikit-build"]["wheel"]["packages"], - { - "ptodsl": "ptodsl/ptodsl", - "ptoas": "ptodsl/ptoas", - "tilelang_dsl": "tilelang-dsl/python/tilelang_dsl", - }, - ) - self.assertEqual( - vmi["tool"]["scikit-build"]["cmake"]["source-dir"], - "../..", - ) - - def test_editable_install_uses_backend_redirect_without_package_path_hacks(self): - ptoas = _read_pyproject(PTOAS_PYPROJECT) - package_init = PTOAS_PACKAGE_INIT.read_text(encoding="utf-8") - - self.assertEqual( - ptoas["tool"]["scikit-build"]["editable"]["mode"], - "redirect", - ) - self.assertNotIn("extend_path", package_init) - - def test_cmake_python_payload_uses_dedicated_install_component(self): - cmake_files = [ - REPO_ROOT / "CMakeLists.txt", - REPO_ROOT / "lib" / "Bindings" / "Python" / "CMakeLists.txt", - REPO_ROOT / "tools" / "ptoas" / "CMakeLists.txt", - ] - combined = "\n".join(path.read_text(encoding="utf-8") for path in cmake_files) - - self.assertIn("COMPONENT PTOAS_Python", combined) - self.assertIn("if(SKBUILD)", combined) - self.assertIn('DIRECTORY "${MLIR_PYTHON_PACKAGE_DIR}/mlir"', combined) - - def test_legacy_wheel_builders_are_removed(self): - self.assertFalse((REPO_ROOT / "_ptoas_build_backend.py").exists()) - self.assertFalse((REPO_ROOT / "docker" / "create_wheel.sh").exists()) - self.assertFalse((REPO_ROOT / "docker" / "setup.py").exists()) - self.assertFalse((REPO_ROOT / "docker" / "setup_mac.py").exists()) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/python/test_validate_wheel_payload.py b/test/python/test_validate_wheel_payload.py deleted file mode 100644 index 5b99367026..0000000000 --- a/test/python/test_validate_wheel_payload.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -import subprocess -import tempfile -import unittest -import zipfile -import importlib.machinery -from pathlib import Path - - -REPO_ROOT = Path(__file__).resolve().parents[2] -VALIDATOR = REPO_ROOT / "docker" / "validate_wheel_payload.py" -WHEEL_IMPORTS = REPO_ROOT / "docker" / "test_wheel_imports.sh" - - -class ValidateWheelPayloadTests(unittest.TestCase): - def _make_wheel( - self, - root: Path, - *, - include_native_module: bool, - include_cli: bool = True, - include_tileops: bool = True, - entry_points_text: str = "[console_scripts]\nptoas=ptoas._cli:main\n", - wheel_stem: str = "ptoas", - dist_info_stem: str = "ptoas", - ) -> Path: - wheel = root / f"{wheel_stem}-1.2.3-cp311-cp311-linux_x86_64.whl" - with zipfile.ZipFile(wheel, "w") as zf: - zf.writestr("ptoas/__init__.py", "") - if include_cli: - zf.writestr("ptoas/_cli.py", "") - if include_tileops: - zf.writestr( - "ptoas/_runtime/share/ptoas/TileOps/__init__.py", "" - ) - if include_native_module: - suffix = importlib.machinery.EXTENSION_SUFFIXES[0] - zf.writestr(f"ptoas/_native{suffix}", "fake") - zf.writestr( - f"{dist_info_stem}-1.2.3.dist-info/entry_points.txt", - entry_points_text, - ) - return wheel - - def test_validator_accepts_current_runtime_payload_layout(self): - with tempfile.TemporaryDirectory() as temp_dir: - wheel = self._make_wheel(Path(temp_dir), include_native_module=True) - result = subprocess.run( - ["python3", str(VALIDATOR), str(wheel)], - capture_output=True, - text=True, - check=False, - ) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("validated wheel payload and launcher contract", result.stdout) - - def test_validator_rejects_missing_native_module(self): - with tempfile.TemporaryDirectory() as temp_dir: - wheel = self._make_wheel(Path(temp_dir), include_native_module=False) - result = subprocess.run( - ["python3", str(VALIDATOR), str(wheel)], - capture_output=True, - text=True, - check=False, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("ptoas._native", result.stderr) - - def test_validator_rejects_missing_cli_module(self): - with tempfile.TemporaryDirectory() as temp_dir: - wheel = self._make_wheel( - Path(temp_dir), - include_native_module=True, - include_cli=False, - ) - result = subprocess.run( - ["python3", str(VALIDATOR), str(wheel)], - capture_output=True, - text=True, - check=False, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("ptoas/_cli.py", result.stderr) - - def test_validator_rejects_missing_packaged_tileops(self): - with tempfile.TemporaryDirectory() as temp_dir: - wheel = self._make_wheel( - Path(temp_dir), - include_native_module=True, - include_tileops=False, - ) - result = subprocess.run( - ["python3", str(VALIDATOR), str(wheel)], - capture_output=True, - text=True, - check=False, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("TileOps/__init__.py", result.stderr) - - def test_validator_rejects_legacy_bootstrap_entrypoint(self): - with tempfile.TemporaryDirectory() as temp_dir: - wheel = self._make_wheel( - Path(temp_dir), - include_native_module=True, - entry_points_text="[console_scripts]\nptoas=ptoas_wheel_bootstrap:main\n", - ) - result = subprocess.run( - ["python3", str(VALIDATOR), str(wheel)], - capture_output=True, - text=True, - check=False, - ) - - self.assertNotEqual(result.returncode, 0) - self.assertIn("ptoas._cli:main", result.stderr) - - def test_validator_accepts_normalized_entrypoint_spacing(self): - with tempfile.TemporaryDirectory() as temp_dir: - wheel = self._make_wheel( - Path(temp_dir), - include_native_module=True, - entry_points_text="[console_scripts]\nptoas = ptoas._cli:main\n", - ) - result = subprocess.run( - ["python3", str(VALIDATOR), str(wheel)], - capture_output=True, - text=True, - check=False, - ) - - self.assertEqual(result.returncode, 0, result.stderr) - - def test_validator_accepts_vmi_distribution_name(self): - with tempfile.TemporaryDirectory() as temp_dir: - wheel = self._make_wheel( - Path(temp_dir), - include_native_module=True, - wheel_stem="ptoas_vmi", - dist_info_stem="ptoas_vmi", - ) - result = subprocess.run( - ["python3", str(VALIDATOR), str(wheel)], - capture_output=True, - text=True, - check=False, - ) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("validated wheel payload and launcher contract", result.stdout) - - def test_shell_probe_reuses_shared_validator(self): - self.assertIn( - '"${PYTHON_BIN}" "${REPO_ROOT}/docker/validate_wheel_payload.py" "${TEST_WHEEL}"', - WHEEL_IMPORTS.read_text(encoding="utf-8"), - ) - - def test_wheel_imports_script_keeps_clean_env_ptoas_smoke(self): - script = WHEEL_IMPORTS.read_text(encoding="utf-8") - - self.assertIn('EXPECTED_PTOAS_CLI_VERSION="${PTOAS_CLI_VERSION:-${PTOAS_VERSION:-}}"', script) - self.assertIn('env -i \\', script) - self.assertIn('CLEAN_ENV_PTO="${CLEAN_ENV_PTO}" \\', script) - self.assertIn('CLEAN_ENV_LOG="${CLEAN_ENV_DIR}/wheel-clean-env-probe.log"', script) - self.assertIn('CLEAN_ENV_PTO_IR="${CLEAN_ENV_DIR}/wheel-clean-env-probe.pto.ir"', script) - self.assertIn('def wheel_clean_env_probe():', script) - self.assertIn('pto.alloc_tile(shape=[1, 16], dtype=pto.f32, addr=0)', script) - self.assertIn('pto.castptr(pto.const(0, dtype=pto.ui64), pto.ptr(pto.f32, "gm"))', script) - self.assertIn('pto.tile.load(a_view, a_tile)', script) - self.assertIn('pto.tile.store(o_tile, o_view)', script) - self.assertIn('--emit-pto-ir "${CLEAN_ENV_PTO}" -o "${CLEAN_ENV_PTO_IR}"', script) - self.assertIn('def wheel_clean_env_probe(', script) - self.assertIn('"${PTOAS_ENTRYPOINT}" --pto-arch=a5 --pto-backend=vpto --pto-level=level3 --enable-tile-op-expand --emit-pto-ir "${CLEAN_ENV_PTO}" -o "${CLEAN_ENV_PTO_IR}"', script) - self.assertIn('>"${CLEAN_ENV_LOG}" 2>&1', script) - self.assertIn('grep -q "wheel_clean_env_probe" "${CLEAN_ENV_PTO_IR}"', script) - self.assertIn('grep -q "pto.tload" "${CLEAN_ENV_PTO_IR}"', script) - self.assertIn('grep -q "pto.tstore" "${CLEAN_ENV_PTO_IR}"', script) - self.assertIn('grep -q "candidates = " "${CLEAN_ENV_PTO_IR}"', script) - self.assertIn('grep -q "TileLib daemon started successfully" "${CLEAN_ENV_LOG}"', script) - self.assertIn('grep -q "TileLib daemon stopped" "${CLEAN_ENV_LOG}"', script) - -if __name__ == "__main__": - unittest.main() From d0ec236959bc6049926eca5de1583578e31274d8 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Fri, 24 Jul 2026 02:51:33 +0800 Subject: [PATCH 11/32] build: streamline Python development workflow --- .github/workflows/ci.yml | 4 +- .github/workflows/ci_sim.yml | 6 +-- CMakeLists.txt | 1 - README.md | 59 +++++++----------------------- README_en.md | 42 ++++++++------------- lib/Bindings/Python/CMakeLists.txt | 6 +++ python/CMakeLists.txt | 27 -------------- quick_install.sh | 5 +++ 8 files changed, 46 insertions(+), 104 deletions(-) delete mode 100644 python/CMakeLists.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8861b79783..5f1af77ba7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -197,8 +197,8 @@ jobs: fi git remote set-url origin "${LLVM_REPO}" - git fetch --depth 1 origin "${LLVM_REF}" - git checkout --force FETCH_HEAD + git fetch --depth 1 origin "${LLVM_SOURCE_SHA}" + git checkout --force "${LLVM_SOURCE_SHA}" - name: Build LLVM/MLIR (only if cache miss) if: steps.cache-llvm.outputs.cache-hit != 'true' diff --git a/.github/workflows/ci_sim.yml b/.github/workflows/ci_sim.yml index 5f6ef605cf..5832f6039c 100644 --- a/.github/workflows/ci_sim.yml +++ b/.github/workflows/ci_sim.yml @@ -286,9 +286,9 @@ jobs: fi git remote set-url origin "${LLVM_REPO}" - ref="${LLVM_TAG:-${LLVM_REF}}" - git fetch --depth 1 origin "${ref}" - git checkout --force FETCH_HEAD + sha="${{ steps.llvm-cache-key.outputs.sha }}" + git fetch --depth 1 origin "${sha}" + git checkout --force "${sha}" - name: Build LLVM/MLIR if: steps.llvm-cache.outputs.cache-hit != 'true' diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a195bb99c..ccbc3cec05 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -297,7 +297,6 @@ if(PTO_ENABLE_PYTHON_BINDING) # 设置 Python 扩展的输出目录，方便调试 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/python/pto) - add_subdirectory(python) add_subdirectory(ptodsl) add_subdirectory(tilelang-dsl) endif() diff --git a/README.md b/README.md index 608bf263cd..b1d834813a 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,10 @@ mkdir -p $WORKSPACE_DIR * **OS**: Linux (Ubuntu 20.04+ 推荐) * **Compiler**: GCC >= 9 或 Clang (支持 C++17) * **Build System**: CMake >= 3.20, Ninja -* **Python**: 3.8+ -* **Python Packages**: `pybind11<3`, `nanobind`, `numpy` +* **Python**: 3.10+ +* **Python Packages**: `scikit-build-core`, `pybind11<3`, `nanobind`, `numpy` ```bash -python3 -m pip install 'pybind11<3' nanobind numpy +python3 -m pip install 'scikit-build-core>=0.12.2,<2' 'pybind11<3' nanobind numpy ``` @@ -123,52 +123,21 @@ cd $WORKSPACE_DIR git clone https://gitcode.com/cann/pto-as.git PTOAS cd $PTO_SOURCE_DIR -# 2. 获取 pybind11 的 CMake 路径 -export PYBIND11_CMAKE_DIR=$(python3 -m pybind11 --cmakedir) +# 2. 安装到当前 Python 环境，并保留可增量构建的 build tree +PYTHON_BIN=python3 \ +LLVM_BUILD_DIR="$LLVM_BUILD_DIR" \ +PTO_BUILD_DIR="$PTO_SOURCE_DIR/build" \ + ./quick_install.sh -# 3. 配置 CMake -# 注意：此处直接使用了 3.0 章节中定义的变量，无需手动修改 -cmake -G Ninja \ - -S . \ - -B build \ - -DLLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm \ - -DMLIR_DIR=$LLVM_BUILD_DIR/lib/cmake/mlir \ - -DPython3_EXECUTABLE=$(which python3) \ - -DPython3_FIND_STRATEGY=LOCATION \ - -Dpybind11_DIR="${PYBIND11_CMAKE_DIR}" \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DMLIR_PYTHON_PACKAGE_DIR=$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core \ - -DCMAKE_INSTALL_PREFIX="$PTO_INSTALL_DIR" - -# 4. 编译并安装 -ninja -C build-llvm21 -ninja -C build-llvm21 install - -# 5. 检查构建产物 -# build 输出（便于本地开发/调试） -$PTO_SOURCE_DIR/build-llvm21/python/ -├── mlir -│ ├── _mlir_libs -│ │ └── _pto.cpython-*.so -│ └── dialects -│ ├── pto.py -│ └── _pto_ops_gen.py - -# install 输出（Python 方言文件和原生扩展） -$PTO_INSTALL_DIR/ -└── mlir - ├── dialects - │ ├── pto.py - │ └── _pto_ops_gen.py - └── _mlir_libs - └── _pto.cpython-*.so - -# CLI 工具 -$PTO_SOURCE_DIR/build-llvm21/tools/ptoas/ptoas -$PTO_SOURCE_DIR/build-llvm21/tools/ptobc/ptobc +# 3. 后续可直接复用同一个 build tree +ninja -C "$PTO_SOURCE_DIR/build" check-pto ``` +`quick_install.sh` 使用 editable install，并关闭 build isolation，避免把临时 +构建环境中的 pybind11 路径写入持久化 `CMakeCache.txt`。`ptoas` 会直接安装到 +`PYTHON_BIN` 对应的当前环境中。 + ### 3.4 Python 安装合同 (Python Distribution Contract) 如果你要使用 Python 绑定、PTODSL资源，推荐使用仓库根目录 diff --git a/README_en.md b/README_en.md index 81d2fb8ab8..2dc0de52fe 100644 --- a/README_en.md +++ b/README_en.md @@ -66,11 +66,11 @@ mkdir -p $WORKSPACE_DIR * **OS**: Linux (Ubuntu 20.04+ recommended) * **Compiler**: GCC >= 9 or Clang (C++17 support required) * **Build System**: CMake >= 3.20, Ninja -* **Python**: 3.8+ -* **Python Packages**: `pybind11<3`, `nanobind`, `numpy` +* **Python**: 3.10+ +* **Python Packages**: `scikit-build-core`, `pybind11<3`, `nanobind`, `numpy` ```bash -python3 -m pip install "pybind11<3" nanobind numpy +python3 -m pip install "scikit-build-core>=0.12.2,<2" "pybind11<3" nanobind numpy ``` > **Note**: The current LLVM/MLIR Python bindings are not compatible with `pybind11` 3.x. @@ -113,32 +113,22 @@ cd $WORKSPACE_DIR git clone https://gitcode.com/cann/pto-as.git PTOAS cd $PTO_SOURCE_DIR -# 2. Build and install via pip -# The build backend (pyproject.toml) drives CMake + Ninja automatically. -pip install . --no-build-isolation -``` - -This produces the same artifacts as a manual CMake build: +# 2. Install into the current Python environment while keeping a persistent, +# incrementally reusable build tree. +PYTHON_BIN=python3 \ +LLVM_BUILD_DIR="$LLVM_BUILD_DIR" \ +PTO_BUILD_DIR="$PTO_SOURCE_DIR/build" \ + ./quick_install.sh -```text -# CLI tools -$PTO_SOURCE_DIR/build/tools/ptoas/ptoas -$PTO_SOURCE_DIR/build/tools/ptobc/ptobc - -# Native extension installed into the MLIR Python package -$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core/ -└── mlir - └── _mlir_libs - └── _pto.cpython-*.so - -# Python dialect files -$PTO_INSTALL_DIR/ -└── mlir - └── dialects - ├── pto.py - └── _pto_ops_gen.py +# 3. Reuse the same build tree for subsequent test or development builds. +ninja -C "$PTO_SOURCE_DIR/build" check-pto ``` +`quick_install.sh` uses an editable install with build isolation disabled so a +temporary build environment's pybind11 path is not persisted in +`CMakeCache.txt`. The `ptoas` command is installed into the environment that +owns `PYTHON_BIN`. + ### 3.4 Step 3: Supported Python Install Flows If you want to use Python bindings or PTODSL, prefer the repository-root diff --git a/lib/Bindings/Python/CMakeLists.txt b/lib/Bindings/Python/CMakeLists.txt index 06b3829b44..f167cc3791 100644 --- a/lib/Bindings/Python/CMakeLists.txt +++ b/lib/Bindings/Python/CMakeLists.txt @@ -86,6 +86,12 @@ mlir_tablegen(_pto_ops_gen.py add_public_tablegen_target(PTOPythonGen) add_dependencies(_pto PTOPythonGen) +# Stable aggregate target used by CTest and developer workflows. Keep the +# dialect binding generation owned by this directory so there is only one +# _pto_ops_gen.py producer. +add_custom_target(PTOPythonModules) +add_dependencies(PTOPythonModules _pto) + # ---- 3) Copy generated python + handwritten pto.py into build/python ---- set(PTO_PY_SRC "${CMAKE_SOURCE_DIR}/python/pto/dialects/pto.py") diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt deleted file mode 100644 index a044a0049b..0000000000 --- a/python/CMakeLists.txt +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -# ========================================================= -# Python Package Configuration -# ========================================================= - -add_custom_target(PTOPythonModules) - -# [修改] 依赖必须指向 lib/Bindings/Python 中定义的 Target (_pto) -add_dependencies(PTOPythonModules _pto) - -# 生成 TableGen 绑定代码 -set(LLVM_TARGET_DEFINITIONS pto/dialects/PTOOps.td) -mlir_tablegen(pto/dialects/_pto_ops_gen.py - -gen-python-op-bindings - -bind-dialect=pto - -I${PROJECT_SOURCE_DIR}/include -) -add_public_tablegen_target(PTOOpsPyGen) - -add_dependencies(PTOPythonModules PTOOpsPyGen) diff --git a/quick_install.sh b/quick_install.sh index 78a05bbe24..9c3767a28b 100755 --- a/quick_install.sh +++ b/quick_install.sh @@ -29,8 +29,13 @@ if command -v ccache >/dev/null 2>&1; then export CMAKE_CXX_COMPILER_LAUNCHER="${CMAKE_CXX_COMPILER_LAUNCHER:-ccache}" fi +"${PYTHON_BIN}" -m pip install \ + 'scikit-build-core>=0.12.2,<2' \ + 'pybind11<3' + LLVM_BUILD_DIR="${LLVM_BUILD_DIR}" \ "${PYTHON_BIN}" -m pip install --editable "${PTO_SOURCE_DIR}" \ + --no-build-isolation \ --config-settings="build-dir=${PTO_BUILD_DIR}" echo "PTOAS editable install complete." From e092a7bebfcb65b3df9f1c62999b3060dbf48813 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Mon, 27 Jul 2026 09:20:56 +0800 Subject: [PATCH 12/32] build: standardize MLIR Python package integration --- .github/workflows/build_wheel_mac.yml | 2 +- .github/workflows/ci.yml | 5 +- .github/workflows/ci_sim.yml | 16 +- CMakeLists.txt | 176 ++-------------- README.md | 10 +- docker/collect_ptoas_dist.sh | 4 +- docker/collect_ptoas_dist_mac.sh | 4 +- docker/test_wheel_imports.sh | 2 +- docker/validate_wheel_payload.py | 4 +- docs/build_with_installed_llvm.md | 23 +-- docs/designs/ci-board-validation-guide.md | 7 +- docs/designs/ptoas-python-launcher-layout.md | 23 ++- docs/no_npu_compile_only_guide_zh.md | 2 +- lib/Bindings/Python/CMakeLists.txt | 194 +++++++++--------- lib/Bindings/Python/PTOModule.cpp | 16 +- lib/Bindings/Python/PTOModule.h | 22 ++ lib/Bindings/Python/PythonRegistration.cpp | 28 +++ packaging/ptoas-vmi/pyproject.toml | 2 +- ptodsl/CMakeLists.txt | 20 +- .../tilelib-debugging-playbook.md | 2 +- ptodsl/ptoas/_cli.py | 6 +- ptodsl/tests/CMakeLists.txt | 10 +- ptodsl/tests/test_allreduce.py | 5 - ptodsl/tests/test_ptoas_cli.py | 2 +- ptodsl/tests/test_ptoas_tree_wrapper.py | 27 ++- ptodsl/tests/test_vmi_vmull.py | 6 - ptodsl/tests/test_vmi_vshr_signedness.py | 6 - pyproject.toml | 2 +- python/pto/dialects/pto.py | 77 +------ quick_install.sh | 3 +- scripts/ptoas_env.sh | 9 +- test/lit/lit.cfg.py | 14 +- test/lit/lit.site.cfg.py.in | 2 - test/tilelib-st/README.md | 16 +- tilelang-dsl/CMakeLists.txt | 20 +- .../python/tilelang_dsl/env_config.py | 23 +-- .../python/tilelang_dsl/pybind_renderer.py | 13 +- .../tests/backend/test_e2e_pybind_backend.py | 4 +- tools/ptoas/CMakeLists.txt | 135 +++++++----- tools/ptoas/NativeModule.cpp | 8 +- tools/ptoas/ptoas_wrapper.py | 37 ++-- 41 files changed, 418 insertions(+), 569 deletions(-) create mode 100644 lib/Bindings/Python/PTOModule.h create mode 100644 lib/Bindings/Python/PythonRegistration.cpp diff --git a/.github/workflows/build_wheel_mac.yml b/.github/workflows/build_wheel_mac.yml index 5ac014e9dc..d422cf433e 100644 --- a/.github/workflows/build_wheel_mac.yml +++ b/.github/workflows/build_wheel_mac.yml @@ -354,7 +354,7 @@ jobs: test -x "$TEST_DIR/extracted/bin/ptoas" test -f "$TEST_DIR/extracted/ptoas/_cli.py" - compgen -G "$TEST_DIR/extracted/ptoas/_native*.so" >/dev/null + compgen -G "$TEST_DIR/extracted/ptoas/_core*.so" >/dev/null env -u PYTHONPATH -u DYLD_LIBRARY_PATH -u LD_LIBRARY_PATH \ "$TEST_DIR/extracted/bin/ptoas" --version diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f1af77ba7..849a727468 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,6 @@ jobs: PTO_BUILD_DIR: ${{ github.workspace }}/build-assert PTO_INSTALL_DIR: ${{ github.workspace }}/install-assert PTOAS_VENV: ${{ github.workspace }}/.venv-ptoas-assert - MLIR_PYTHONPATH: ${{ github.workspace }}/llvm-project/llvm/build-assert/tools/mlir/python_packages/mlir_core LLVM_CACHE_FLAVOR: llvm21-assert-shared-mlirpy-hardening-v1 steps: - name: Checkout @@ -251,11 +250,9 @@ jobs: -DPython3_EXECUTABLE="${cmake4_venv}/bin/python" \ -DPython3_FIND_STRATEGY=LOCATION \ -Dpybind11_DIR="$("${cmake4_venv}/bin/python" -m pybind11 --cmakedir)" \ + -Dnanobind_DIR="$("${cmake4_venv}/bin/python" -m nanobind --cmake_dir)" \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DMLIR_PYTHON_PACKAGE_DIR="${MLIR_PYTHONPATH}" \ -DPTO_ENABLE_PYTHON_BINDING=ON \ - -DPTOAS_VALIDATE_CMAKE4_FETCHCONTENT_COMPAT=ON \ - -DPTOAS_ENABLE_WERROR=OFF \ -DBUILD_TESTING=ON \ -DCMAKE_C_COMPILER="${PTOAS_CMAKE_C_COMPILER}" \ -DCMAKE_CXX_COMPILER="${PTOAS_CMAKE_CXX_COMPILER}" \ diff --git a/.github/workflows/ci_sim.yml b/.github/workflows/ci_sim.yml index 5832f6039c..32a2eab474 100644 --- a/.github/workflows/ci_sim.yml +++ b/.github/workflows/ci_sim.yml @@ -224,7 +224,6 @@ jobs: set -euo pipefail echo "LLVM_ROOT=${RUNNER_TOOL_CACHE}/llvm-project" >> "${GITHUB_ENV}" echo "LLVM_DIR=${RUNNER_TOOL_CACHE}/llvm-project/llvm/build-assert" >> "${GITHUB_ENV}" - echo "MLIR_PYTHONPATH=${RUNNER_TOOL_CACHE}/llvm-project/llvm/build-assert/tools/mlir/python_packages/mlir_core" >> "${GITHUB_ENV}" - name: Resolve PTODSL build directories shell: bash @@ -715,10 +714,17 @@ jobs: mkdir -p "${TILELANG_DSL_WORKSPACE}" export LLVM_BUILD_DIR="${LLVM_DIR}" export MLIR_PYTHON_ROOT="${MLIR_PYTHONPATH}" - ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \ - PTOAS_BIN="${PTOAS_BIN}" \ - bash test/tilelang_st/script/run_ci.sh -r sim -v a5 --jobs 64 --smoke \ - 2>&1 | tee "${TILELANG_DSL_WORKSPACE}/run_ci.log" + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \ + PTOAS_BIN="${PTOAS_BIN}" \ + bash test/tilelang_st/script/run_ci.sh -r sim -v a5 --jobs 64 --smoke \ + 2>&1 | tee "${TILELANG_DSL_WORKSPACE}/run_ci.log" + else + ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \ + PTOAS_BIN="${PTOAS_BIN}" \ + bash test/tilelang_st/script/run_ci.sh -r sim -v a5 --jobs 64 \ + 2>&1 | tee "${TILELANG_DSL_WORKSPACE}/run_ci.log" + fi - name: Restore PTODSL LLVM build cache id: ptodsl-llvm-cache diff --git a/CMakeLists.txt b/CMakeLists.txt index ccbc3cec05..156ca5d23f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,64 +18,6 @@ if(POLICY CMP0169) set(CMAKE_POLICY_DEFAULT_CMP0169 OLD) endif() -function(ptoas_get_cmake_cache_value cache_file variable out_var) - if(NOT EXISTS "${cache_file}") - set(${out_var} "" PARENT_SCOPE) - return() - endif() - - file(STRINGS "${cache_file}" cache_lines - REGEX "^${variable}:[^=]*=") - if(cache_lines) - list(GET cache_lines 0 cache_line) - string(REGEX REPLACE "^[^=]*=" "" cache_value "${cache_line}") - set(${out_var} "${cache_value}" PARENT_SCOPE) - else() - set(${out_var} "" PARENT_SCOPE) - endif() -endfunction() - -function(ptoas_get_llvm_build_dir_from_llvm_dir out_var) - if(NOT DEFINED LLVM_DIR OR LLVM_DIR STREQUAL "") - set(${out_var} "" PARENT_SCOPE) - return() - endif() - - get_filename_component(llvm_cmake_dir "${LLVM_DIR}" REALPATH) - get_filename_component(llvm_build_dir "${llvm_cmake_dir}/../../.." REALPATH) - if(EXISTS "${llvm_build_dir}/CMakeCache.txt") - set(${out_var} "${llvm_build_dir}" PARENT_SCOPE) - else() - set(${out_var} "" PARENT_SCOPE) - endif() -endfunction() - -function(ptoas_preseed_compilers_from_llvm_cache) - ptoas_get_llvm_build_dir_from_llvm_dir(llvm_build_dir) - if(llvm_build_dir STREQUAL "") - return() - endif() - - set(llvm_cache "${llvm_build_dir}/CMakeCache.txt") - if(NOT DEFINED CMAKE_C_COMPILER AND "$ENV{CC}" STREQUAL "") - ptoas_get_cmake_cache_value("${llvm_cache}" "CMAKE_C_COMPILER" - llvm_c_compiler) - if(llvm_c_compiler AND EXISTS "${llvm_c_compiler}") - set(CMAKE_C_COMPILER "${llvm_c_compiler}" CACHE FILEPATH - "C compiler inherited from the configured LLVM build" FORCE) - endif() - endif() - - if(NOT DEFINED CMAKE_CXX_COMPILER AND "$ENV{CXX}" STREQUAL "") - ptoas_get_cmake_cache_value("${llvm_cache}" "CMAKE_CXX_COMPILER" - llvm_cxx_compiler) - if(llvm_cxx_compiler AND EXISTS "${llvm_cxx_compiler}") - set(CMAKE_CXX_COMPILER "${llvm_cxx_compiler}" CACHE FILEPATH - "C++ compiler inherited from the configured LLVM build" FORCE) - endif() - endif() -endfunction() - # Standard Python build backends pass environment variables through to CMake # but do not synthesize project-specific cache arguments. Accept the existing # LLVM_BUILD_DIR convention directly so `pip wheel .` and `pip install .` can @@ -92,16 +34,6 @@ if((NOT DEFINED MLIR_DIR OR MLIR_DIR STREQUAL "") set(MLIR_DIR "$ENV{LLVM_BUILD_DIR}/lib/cmake/mlir" CACHE PATH "MLIR package directory derived from LLVM_BUILD_DIR") endif() -if((NOT DEFINED MLIR_PYTHON_PACKAGE_DIR - OR MLIR_PYTHON_PACKAGE_DIR STREQUAL "") - AND DEFINED ENV{LLVM_BUILD_DIR} - AND NOT "$ENV{LLVM_BUILD_DIR}" STREQUAL "") - set(MLIR_PYTHON_PACKAGE_DIR - "$ENV{LLVM_BUILD_DIR}/tools/mlir/python_packages/mlir_core" - CACHE PATH "MLIR Python package derived from LLVM_BUILD_DIR") -endif() - -ptoas_preseed_compilers_from_llvm_cache() project(ptoas VERSION 0.55) set(PTOAS_VMI_VERSION "0.1.4") @@ -112,28 +44,6 @@ if(SKBUILD AND CMAKE_SYSTEM_NAME STREQUAL "Linux") include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/LinuxHardeningCache.cmake") endif() -option(PTOAS_VALIDATE_CMAKE4_FETCHCONTENT_COMPAT - "Validate CMake 4 compatibility for legacy FetchContent users" OFF) -if(PTOAS_VALIDATE_CMAKE4_FETCHCONTENT_COMPAT) - set(_ptoas_cmp0169_check_dir "${CMAKE_BINARY_DIR}/cmake-cmp0169-compat") - file(MAKE_DIRECTORY "${_ptoas_cmp0169_check_dir}/nested/cann-cmake") - file(WRITE "${_ptoas_cmp0169_check_dir}/nested/cann-cmake/CMakeLists.txt" - "cmake_minimum_required(VERSION 3.20)\n" - "project(ptoas_cmp0169_cann_cmake_fixture NONE)\n") - file(WRITE "${_ptoas_cmp0169_check_dir}/nested/CMakeLists.txt" [=[ -cmake_minimum_required(VERSION 3.20) -project(ptoas_cmp0169_nested_fixture NONE) -include(FetchContent) -FetchContent_Declare(cann-cmake SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/cann-cmake") -FetchContent_GetProperties(cann-cmake) -if(NOT cann-cmake_POPULATED) - FetchContent_Populate(cann-cmake) -endif() -]=]) - add_subdirectory("${_ptoas_cmp0169_check_dir}/nested" - "${_ptoas_cmp0169_check_dir}/nested-build") -endif() - set(PTOAS_RELEASE_VERSION_OVERRIDE "" CACHE STRING "Override the version printed by ptoas --version") set(PTOAS_EXACT_GIT_TAG_VERSION "") @@ -186,33 +96,6 @@ if(NOT LLVM_VERSION_MAJOR STREQUAL "21") "LLVM ${LLVM_PACKAGE_VERSION}. Please point LLVM_DIR/MLIR_DIR to an " "LLVM 21 build, for example vpto-dev/llvm-project:feature-vpto-llvm21.") endif() -get_filename_component(PTO_LLVM_BUILD_LIBRARY_DIR - "${LLVM_BUILD_LIBRARY_DIR}" REALPATH) -ptoas_get_llvm_build_dir_from_llvm_dir(PTO_LLVM_BUILD_DIR) -if(PTO_LLVM_BUILD_DIR) - ptoas_get_cmake_cache_value("${PTO_LLVM_BUILD_DIR}/CMakeCache.txt" - "CMAKE_CXX_COMPILER" - PTO_LLVM_CXX_COMPILER) -endif() -if(PTO_LLVM_CXX_COMPILER AND EXISTS "${PTO_LLVM_CXX_COMPILER}") - execute_process( - COMMAND "${PTO_LLVM_CXX_COMPILER}" --version - OUTPUT_VARIABLE PTO_LLVM_CXX_VERSION_TEXT - ERROR_VARIABLE PTO_LLVM_CXX_VERSION_TEXT - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_STRIP_TRAILING_WHITESPACE) - if(PTO_LLVM_CXX_VERSION_TEXT MATCHES "[Cc]lang" - AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - message(FATAL_ERROR - "PTOAS is being configured with GNU C++ (${CMAKE_CXX_COMPILER}), " - "but the selected LLVM build was compiled with a Clang-compatible " - "compiler (${PTO_LLVM_CXX_COMPILER}). MLIR TypeID fallback strings are " - "compiler-family sensitive across shared libraries; use " - "-DCMAKE_CXX_COMPILER=${PTO_LLVM_CXX_COMPILER} or leave the compiler " - "unset so PTOAS can inherit it from LLVM_DIR.") - endif() -endif() - # 将 LLVM 模块路径加入 CMake 搜索路径 list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}") @@ -235,13 +118,14 @@ include(AddMLIR) # 3. 配置构建环境 # ========================================================= # Keep warning policy in CMake so CI, wheel, and external build entrypoints -# all enforce the same diagnostics contract. +# all enforce the same diagnostics contract. Warnings are errors by default; +# directories that compile only third-party sources may opt out narrowly. option(PTOAS_ENABLE_WERROR "Treat PTOAS warnings as errors" ON) if(PTOAS_ENABLE_WERROR) if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|AppleClang") - add_compile_options($<$:-Werror>) + add_compile_options("$<$:-Werror>") elseif(MSVC) - add_compile_options($<$:/WX>) + add_compile_options("$<$:/WX>") endif() endif() @@ -253,12 +137,12 @@ if(APPLE) endforeach() endif() -include_directories(${LLVM_INCLUDE_DIRS}) -include_directories(${MLIR_INCLUDE_DIRS}) +include_directories(SYSTEM ${LLVM_INCLUDE_DIRS}) +include_directories(SYSTEM ${MLIR_INCLUDE_DIRS}) include_directories(${PROJECT_SOURCE_DIR}/include) include_directories(${PROJECT_BINARY_DIR}/include) -link_directories(${PTO_LLVM_BUILD_LIBRARY_DIR}) +link_directories(${LLVM_BUILD_LIBRARY_DIR}) add_definitions(${LLVM_DEFINITIONS}) # ========================================================= @@ -272,30 +156,9 @@ option(PTO_ENABLE_PYTHON_BINDING "Enable Python bindings" ON) include(cmake/VfSimulator.cmake) if(PTO_ENABLE_PYTHON_BINDING) - find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) - find_package(pybind11 CONFIG REQUIRED) - - # A wheel must include the base MLIR Python package in addition to PTOAS's - # generated dialect modules. Normal CMake installs continue to use the LLVM - # installation supplied by the user and therefore do not duplicate it. - if(SKBUILD) - if(NOT MLIR_PYTHON_PACKAGE_DIR - OR NOT EXISTS "${MLIR_PYTHON_PACKAGE_DIR}/mlir") - message(FATAL_ERROR - "MLIR_PYTHON_PACKAGE_DIR must contain the MLIR Python package when " - "building a PTOAS wheel") - endif() - install( - DIRECTORY "${MLIR_PYTHON_PACKAGE_DIR}/mlir" - DESTINATION "." - COMPONENT PTOAS_Python - PATTERN "__pycache__" EXCLUDE - PATTERN "*.pyc" EXCLUDE - ) - endif() - - # 设置 Python 扩展的输出目录，方便调试 - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/python/pto) + include(MLIRDetectPythonEnv) + include(AddMLIRPython) + mlir_configure_python_dev_packages() add_subdirectory(ptodsl) add_subdirectory(tilelang-dsl) @@ -305,8 +168,8 @@ endif() # 4. 添加子目录 # ========================================================= if(BUILD_TESTING AND NOT TARGET ptoas_runtime_deps) - # Define this aggregate target before tools/ptoas runs so it can attach - # ptoas_runtime as a dependency during configure. + # Define this aggregate target before tools/ptoas runs so it can attach the + # staged ptoas._core package as a dependency during configure. add_custom_target(ptoas_runtime_deps) endif() add_subdirectory(include) @@ -319,17 +182,13 @@ add_subdirectory(tools) if(BUILD_TESTING) enable_testing() if(PTO_ENABLE_PYTHON_BINDING) - get_filename_component(_llvm_root "${LLVM_DIR}/../../.." ABSOLUTE) get_filename_component(_pto_python_bin_dir "${Python3_EXECUTABLE}" DIRECTORY) set(_pto_python_test_pythonpath - "${CMAKE_BINARY_DIR}/python:${MLIR_PYTHON_PACKAGE_DIR}:${CMAKE_INSTALL_PREFIX}:$ENV{PYTHONPATH}" + "${CMAKE_BINARY_DIR}/python:$ENV{PYTHONPATH}" ) - if(DEFINED ENV{PTO_INSTALL_DIR} AND NOT "$ENV{PTO_INSTALL_DIR}" STREQUAL "") - set(_pto_python_test_pythonpath "$ENV{PTO_INSTALL_DIR}:${_pto_python_test_pythonpath}") - endif() set(_pto_python_test_env "PYTHONPATH=${_pto_python_test_pythonpath}" - "LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:${_llvm_root}/lib:${CMAKE_INSTALL_PREFIX}/lib:$ENV{LD_LIBRARY_PATH}" + "LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:${LLVM_BUILD_LIBRARY_DIR}:${CMAKE_INSTALL_PREFIX}/lib:$ENV{LD_LIBRARY_PATH}" "PATH=${CMAKE_BINARY_DIR}/tools/ptoas:${CMAKE_INSTALL_PREFIX}/bin:${_pto_python_bin_dir}:$ENV{PATH}" "PTOAS_BIN=${CMAKE_BINARY_DIR}/tools/ptoas/ptoas" ) @@ -374,7 +233,12 @@ if(BUILD_TESTING) add_subdirectory(test/lit) set(_pto_check_ctest_depends) - foreach(_target IN ITEMS PTOPythonModules ptoas_runtime_deps ptobc) + foreach(_target IN ITEMS + PTOAS_Python + PTODSLPackage + TileLangDSLPackage + ptoas_runtime_deps + ptobc) if(TARGET ${_target}) list(APPEND _pto_check_ctest_depends ${_target}) endif() diff --git a/README.md b/README.md index b1d834813a..e17ca864b3 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ from mlir.dialects import pto as mlir_pto 下面这组环境变量主要用于**直接消费 build/install tree** 的场景，例如： - 不走 pip 安装，直接调试 CMake install 输出 -- 调试 `ptoas` CLI、动态库搜索路径或 MLIR Python overlay +- 调试 `ptoas` CLI、动态库搜索路径或 build-tree Python package - 复用仓库脚本做 compile-only / simulator / sample 生成 您可以将以下命令添加到 `.bashrc` 或启动脚本中。 @@ -198,11 +198,9 @@ from mlir.dialects import pto as mlir_pto ```bash # --- 运行时变量配置 (基于之前定义的路径) --- -# 1. Python Path: 拼接 MLIR Core 和 PTO Core -# 这样在 python 中 import mlir.dialects.pto 时能正确找到 -export MLIR_PYTHON_ROOT=$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core -export PTO_PYTHON_ROOT=$PTO_INSTALL_DIR/ -export PYTHONPATH=$PTO_PYTHON_ROOT:$MLIR_PYTHON_ROOT:$PYTHONPATH +# 1. Python Path: PTOAS 的 build/install tree 已包含统一的 MLIR + PTO package +export PTO_PYTHON_ROOT=$PTO_INSTALL_DIR +export PYTHONPATH=$PTO_PYTHON_ROOT:$PYTHONPATH # 2. Library Path: 确保能加载 LLVM 和 PTO 的动态库 export LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:$LD_LIBRARY_PATH diff --git a/docker/collect_ptoas_dist.sh b/docker/collect_ptoas_dist.sh index 798260e036..d682f842bd 100755 --- a/docker/collect_ptoas_dist.sh +++ b/docker/collect_ptoas_dist.sh @@ -149,9 +149,9 @@ cp -R "${PTOAS_PACKAGE_SRC_DIR}" "${PTOAS_PACKAGE_DIST_DIR}" cp "$PTOAS_BIN" "${PTOAS_DIST_DIR}/bin/ptoas" chmod +x "${PTOAS_DIST_DIR}/bin/ptoas" -PTOAS_NATIVE_MODULE="$(find "${PTOAS_PACKAGE_DIST_DIR}" -maxdepth 1 -type f -name '_native*.so' -print -quit)" +PTOAS_NATIVE_MODULE="$(find "${PTOAS_PACKAGE_DIST_DIR}" -maxdepth 1 -type f -name '_core*.so' -print -quit)" if [ -z "${PTOAS_NATIVE_MODULE}" ]; then - echo "Error: packaged ptoas._native extension not found" >&2 + echo "Error: packaged ptoas._core extension not found" >&2 exit 1 fi diff --git a/docker/collect_ptoas_dist_mac.sh b/docker/collect_ptoas_dist_mac.sh index a3fb2c13d2..6685930f5f 100644 --- a/docker/collect_ptoas_dist_mac.sh +++ b/docker/collect_ptoas_dist_mac.sh @@ -67,9 +67,9 @@ cp -R "${PTOAS_PACKAGE_SRC_DIR}" "${PTOAS_PACKAGE_DIST_DIR}" cp -fL "$PTOAS_BIN" "${PTOAS_DIST_DIR}/bin/ptoas" chmod +x "${PTOAS_DIST_DIR}/bin/ptoas" -PTOAS_NATIVE_MODULE="$(find "${PTOAS_PACKAGE_DIST_DIR}" -maxdepth 1 -type f -name '_native*.so' -print -quit)" +PTOAS_NATIVE_MODULE="$(find "${PTOAS_PACKAGE_DIST_DIR}" -maxdepth 1 -type f -name '_core*.so' -print -quit)" if [ -z "${PTOAS_NATIVE_MODULE}" ]; then - echo "Error: packaged ptoas._native extension not found" >&2 + echo "Error: packaged ptoas._core extension not found" >&2 exit 1 fi diff --git a/docker/test_wheel_imports.sh b/docker/test_wheel_imports.sh index f06b6938b4..df2a30487c 100755 --- a/docker/test_wheel_imports.sh +++ b/docker/test_wheel_imports.sh @@ -89,7 +89,7 @@ echo "Testing ptodsl public imports..." "$PYTHON_BIN" -c "from ptodsl import pto, scalar; print('ptodsl public imports imported successfully')" echo "Testing installed ptoas console entry..." -"$PYTHON_BIN" -c "from ptoas import _native; print(f'ptoas native extension imported successfully from {_native.__file__}')" +"$PYTHON_BIN" -c "from ptoas import _core; print(f'ptoas native extension imported successfully from {_core.__file__}')" PTOAS_VERSION_OUTPUT="$(ptoas --version | tr -d '\r')" echo "${PTOAS_VERSION_OUTPUT}" EXPECTED_PTOAS_CLI_VERSION="${PTOAS_CLI_VERSION:-${PTOAS_VERSION:-}}" diff --git a/docker/validate_wheel_payload.py b/docker/validate_wheel_payload.py index 2d4879d0d6..0ddcda8a3d 100644 --- a/docker/validate_wheel_payload.py +++ b/docker/validate_wheel_payload.py @@ -28,7 +28,7 @@ PTOAS_ENTRYPOINT_TARGET = "ptoas._cli:main" WHEEL_GLOB = "ptoas*.whl" NATIVE_MODULE_PATHS = { - f"ptoas/_native{suffix}" + f"ptoas/_core{suffix}" for suffix in importlib.machinery.EXTENSION_SUFFIXES } @@ -89,7 +89,7 @@ def validate_wheel_payload(wheel: Path) -> None: native_modules = sorted(NATIVE_MODULE_PATHS & names) if len(native_modules) != 1: raise SystemExit( - "wheel must contain exactly one importable ptoas._native extension, " + "wheel must contain exactly one importable ptoas._core extension, " f"found {native_modules}" ) diff --git a/docs/build_with_installed_llvm.md b/docs/build_with_installed_llvm.md index 7b09173b27..807485d898 100644 --- a/docs/build_with_installed_llvm.md +++ b/docs/build_with_installed_llvm.md @@ -73,15 +73,15 @@ README 第 3.2 节是 LLVM/MLIR 的下载和编译步骤。当前场景下 LLVM 这里沿用 README 第 3.3 节的流程，但 `LLVM_DIR` 和 `MLIR_DIR` 需要改为 `/opt/llvm/lib/cmake/...`。 -`MLIR_PYTHON_PACKAGE_DIR` 仍然指向 LLVM 的 MLIR Python package。PTOAS 的 -`pto.py`、`_pto_ops_gen.py` 和 `_pto.cpython-*.so` 会安装到 -`CMAKE_INSTALL_PREFIX`，不会写入共享的 LLVM 安装目录。 +PTOAS 使用 MLIR 导出的 Python source-set 在自己的 build/install tree 中 +组装完整的 `mlir` package，不再复制或 overlay LLVM build tree 中已经生成的 +Python package。 ```bash cd "$PTO_SOURCE_DIR" -# 1. 获取 pybind11 的 CMake 路径 -export PYBIND11_CMAKE_DIR=$(python3 -m pybind11 --cmakedir) +# 1. 安装 MLIR Python binding 的构建依赖 +python3 -m pip install 'pybind11<3' 'nanobind>=2.4' # 2. 配置 CMake cmake -G Ninja \ @@ -91,9 +91,7 @@ cmake -G Ninja \ -DMLIR_DIR=$LLVM_INSTALL_DIR/lib/cmake/mlir \ -DPython3_EXECUTABLE=$(which python3) \ -DPython3_FIND_STRATEGY=LOCATION \ - -Dpybind11_DIR="${PYBIND11_CMAKE_DIR}" \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DMLIR_PYTHON_PACKAGE_DIR="$LLVM_INSTALL_DIR/python_packages/mlir_core" \ -DCMAKE_INSTALL_PREFIX="$PTO_INSTALL_DIR" # 3. 编译并安装 @@ -108,11 +106,11 @@ cmake --install build - build 目录： - `$PTO_SOURCE_DIR/build/tools/ptoas/ptoas` - `$PTO_SOURCE_DIR/build/tools/ptobc/ptobc` - - `$PTO_SOURCE_DIR/build/python/mlir/_mlir_libs/_pto.cpython-*.so` + - `$PTO_SOURCE_DIR/build/python/ptoas/_core.cpython-*.so` - `$PTO_SOURCE_DIR/build/python/mlir/dialects/pto.py` - install 目录： - `$PTO_INSTALL_DIR/bin/ptoas` - - `$PTO_INSTALL_DIR/mlir/_mlir_libs/_pto.cpython-*.so` + - `$PTO_INSTALL_DIR/ptoas/_core.cpython-*.so` - `$PTO_INSTALL_DIR/mlir/dialects/pto.py` - `$PTO_INSTALL_DIR/share/ptoas/oplib/level3` @@ -122,7 +120,7 @@ cmake --install build ```bash export PATH=$PTO_SOURCE_DIR/build/tools/ptoas:$PATH -export PYTHONPATH=$LLVM_INSTALL_DIR/python_packages/mlir_core:$PTO_SOURCE_DIR/build/python:$PYTHONPATH +export PYTHONPATH=$PTO_SOURCE_DIR/build/python:$PYTHONPATH export LD_LIBRARY_PATH=$LLVM_INSTALL_DIR/lib:$PTO_SOURCE_DIR/build/lib:$LD_LIBRARY_PATH ``` @@ -130,7 +128,7 @@ export LD_LIBRARY_PATH=$LLVM_INSTALL_DIR/lib:$PTO_SOURCE_DIR/build/lib:$LD_LIBRA ```bash export PATH=$PTO_INSTALL_DIR/bin:$PATH -export PYTHONPATH=$LLVM_INSTALL_DIR/python_packages/mlir_core:$PTO_INSTALL_DIR:$PYTHONPATH +export PYTHONPATH=$PTO_INSTALL_DIR:$PYTHONPATH export LD_LIBRARY_PATH=$LLVM_INSTALL_DIR/lib:$PTO_INSTALL_DIR/lib:$LD_LIBRARY_PATH ``` @@ -145,12 +143,11 @@ export LD_LIBRARY_PATH=$LLVM_INSTALL_DIR/lib:$PTO_INSTALL_DIR/lib:$LD_LIBRARY_PA - `LLVM_DIR=/opt/llvm/lib/cmake/llvm` - `MLIR_DIR=/opt/llvm/lib/cmake/mlir` -- `MLIR_PYTHON_PACKAGE_DIR=/opt/llvm/python_packages/mlir_core` - `CMAKE_INSTALL_PREFIX=$PTO_INSTALL_DIR` 最小验证结果： - build 版 `ptoas --version` 输出 `ptoas 0.22` - build 版 `ptoas` 可成功处理 `test/lit/pto/empty_func.pto` -- install 版 Python 绑定可在 `PYTHONPATH=/opt/llvm/python_packages/mlir_core:$PTO_INSTALL_DIR` 下正常导入 +- install 版 Python 绑定可在 `PYTHONPATH=$PTO_INSTALL_DIR` 下正常导入 - 若 install 版 `ptoas` 配合 `LD_LIBRARY_PATH=/opt/llvm/lib:$PTO_INSTALL_DIR/lib`，可正常执行 diff --git a/docs/designs/ci-board-validation-guide.md b/docs/designs/ci-board-validation-guide.md index bb963841d9..ea02051bfc 100644 --- a/docs/designs/ci-board-validation-guide.md +++ b/docs/designs/ci-board-validation-guide.md @@ -103,16 +103,14 @@ ninja -C llvm/build-shared ```bash export LLVM_DIR=$PWD/llvm-project/llvm/build-shared export PTO_INSTALL_DIR=$PWD/install -export PYBIND11_CMAKE_DIR="$(python3 -m pybind11 --cmakedir)" +python3 -m pip install 'pybind11<3' 'nanobind>=2.4' cmake -G Ninja -S . -B build \ -DLLVM_DIR="$LLVM_DIR/lib/cmake/llvm" \ -DMLIR_DIR="$LLVM_DIR/lib/cmake/mlir" \ -DPython3_EXECUTABLE=python3 \ -DPython3_FIND_STRATEGY=LOCATION \ - -Dpybind11_DIR="$PYBIND11_CMAKE_DIR" \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DMLIR_PYTHON_PACKAGE_DIR="$LLVM_DIR/tools/mlir/python_packages/mlir_core" \ -DCMAKE_INSTALL_PREFIX="$PTO_INSTALL_DIR" \ -DCMAKE_BUILD_TYPE=Release @@ -124,9 +122,8 @@ ninja -C build install ### 3.3 跑样例生成链路 ```bash -export MLIR_PYTHON_ROOT=$PWD/llvm-project/llvm/build-shared/tools/mlir/python_packages/mlir_core export PTO_PYTHON_ROOT=$PWD/install -export PYTHONPATH="$MLIR_PYTHON_ROOT:$PTO_PYTHON_ROOT:${PYTHONPATH:-}" +export PYTHONPATH="$PTO_PYTHON_ROOT:${PYTHONPATH:-}" export LD_LIBRARY_PATH="$LLVM_DIR/lib:$PTO_INSTALL_DIR/lib:${LD_LIBRARY_PATH:-}" export PTOAS_BIN=$PWD/build/tools/ptoas/ptoas diff --git a/docs/designs/ptoas-python-launcher-layout.md b/docs/designs/ptoas-python-launcher-layout.md index 171b5b7187..f85b9014e4 100644 --- a/docs/designs/ptoas-python-launcher-layout.md +++ b/docs/designs/ptoas-python-launcher-layout.md @@ -16,18 +16,21 @@ PTOAS without exposing these implementation details. ## Entry Model -The wheel, build-tree, and install-tree launchers use the standard Python -console-script and native-extension model: +The wheel launcher uses the standard Python console-script and native-extension +model. Build-tree and install-tree entrypoints are narrow CMake adapters around +the same Python CLI and native extension: ```text wheel console script -> ptoas._cli.main() CMake tree wrapper -> add its own Python root -> ptoas._cli.launch() -both -> import ptoas._native -> ptoas._native.main(argv) +both -> import ptoas._core -> ptoas._core.main(argv) ``` -`ptoas._native` is built with `pybind11_add_module`. CMake and Python therefore -own the platform and ABI-specific filename. Launcher and packaging code refer to -the import name and never construct `.so`, `.dylib`, or `.pyd` paths. +`ptoas._core` is the single PTOAS-owned native extension. It provides the +compiler entry point and the native PTO dialect bindings used by the public +`mlir.dialects.pto` facade. CMake and Python own the platform and ABI-specific +filename. Launcher and packaging code refer to the import name and never +construct `.so`, `.dylib`, or `.pyd` paths. The LLVM-based driver implementation is compiled as an object library so it can use LLVM's RTTI and exception settings independently from pybind11. Those @@ -73,7 +76,7 @@ The build-tree wrapper resolves only its own generated outputs: - wrapper: `/tools/ptoas/ptoas` - Python root: `/python` -- native module: importable as `ptoas._native` from the Python root +- native module: importable as `ptoas._core` from the Python root - TileOps: `/python/ptoas/_runtime/share/ptoas/TileOps` Missing Python packages or TileOps resources are hard layout errors. The @@ -86,7 +89,7 @@ The install-tree wrapper resolves only files under the same prefix: - wrapper: `/bin/ptoas` - Python root: `` -- native module: importable as `ptoas._native` from the Python root +- native module: importable as `ptoas._core` from the Python root - TileOps: `/ptoas/_runtime/share/ptoas/TileOps` The install-tree wrapper only adds `` to `sys.path`, then delegates to @@ -104,7 +107,7 @@ Standalone compiler archives contain the installed Python wrapper and package: ```text bin/ptoas ptoas/_cli.py -ptoas/_native..so +ptoas/_core..so ptoas/_runtime/share/ptoas/TileOps lib/ tilelang_dsl/ @@ -112,7 +115,7 @@ tilelang_dsl/ The archive requires a Python interpreter compatible with the packaged extension ABI. `bin/ptoas` adds the archive root to `sys.path`, then uses the -same `ptoas._cli -> ptoas._native` path as the install tree. Linux archives set +same `ptoas._cli -> ptoas._core` path as the install tree. Linux archives set the extension runtime path to `$ORIGIN/../lib`; macOS archives rewrite the extension's install names relative to its package directory. diff --git a/docs/no_npu_compile_only_guide_zh.md b/docs/no_npu_compile_only_guide_zh.md index 7c9b96ca99..c96b7a1b35 100644 --- a/docs/no_npu_compile_only_guide_zh.md +++ b/docs/no_npu_compile_only_guide_zh.md @@ -187,7 +187,7 @@ export PTOAS_BIN=$PWD/build/tools/ptoas/ptoas export PTOBC_BIN=$PWD/build/tools/ptobc/ptobc export PYTHON_BIN=/usr/bin/python3 export PTOAS_OUT_DIR="$PAYLOAD_ROOT/test/samples" -export PYTHONPATH="$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core:$PTO_INSTALL_DIR:${PYTHONPATH:-}" +export PYTHONPATH="$PTO_INSTALL_DIR:${PYTHONPATH:-}" export LD_LIBRARY_PATH="$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:${LD_LIBRARY_PATH:-}" export SOC_VERSION="$TARGET_SOC_VERSION" diff --git a/lib/Bindings/Python/CMakeLists.txt b/lib/Bindings/Python/CMakeLists.txt index f167cc3791..c0bd2b19d4 100644 --- a/lib/Bindings/Python/CMakeLists.txt +++ b/lib/Bindings/Python/CMakeLists.txt @@ -6,109 +6,117 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -# lib/Bindings/Python/CMakeLists.txt - -find_package(Python3 COMPONENTS Interpreter Development.Module REQUIRED) -find_package(pybind11 CONFIG REQUIRED) - -include(TableGen) -include(AddMLIR) - -get_filename_component(PTO_LLVM_BUILD_LIBRARY_DIR - "${LLVM_BUILD_LIBRARY_DIR}" REALPATH) - -# ---- 1) Python native extension: mlir._mlir_libs._pto ---- -pybind11_add_module(_pto MODULE - PTOModule.cpp -) - -target_include_directories(_pto PRIVATE - ${CMAKE_SOURCE_DIR}/include -) +include(AddMLIRPython) + +# This directory materializes C++ sources exported by upstream MLIR. PTOAS +# keeps -Werror enabled globally, but upstream sources are not part of the +# project warning contract. Scope the exception to this directory; the +# project-owned ptoas._core extension is built in tools/ptoas and therefore +# remains under -Werror. +if(PTOAS_ENABLE_WERROR) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|AppleClang") + add_compile_options("$<$:-Wno-error>") + elseif(MSVC) + add_compile_options("$<$:/WX->") + endif() +endif() -target_link_options(_pto PRIVATE) +# Build the `mlir` Python package from MLIR's exported source declarations and +# add PTOAS's dialect to the same package. This is the composition mechanism +# supported by AddMLIRPython and avoids copying or overlaying another build's +# package tree. +set(PTOAS_MLIR_PYTHON_PACKAGE_DIR "${CMAKE_BINARY_DIR}/python/mlir") -target_link_libraries(_pto PRIVATE +declare_mlir_python_sources(PTOASPythonSources) - PTOCAPI - PTOIR - MLIRIR - MLIRCAPIIR - MLIRSupport - MLIRArithDialect - MLIRMemRefDialect - MLIRSCFDialect - MLIRDestinationStyleOpInterface - MLIRInferTypeOpInterface - MLIRSideEffectInterfaces - MLIRCallInterfaces - MLIRControlFlowInterfaces - MLIRLoopLikeInterface - MLIRViewLikeInterface - MLIRFunctionInterfaces - MLIRLLVMDialect +declare_mlir_dialect_python_bindings( + ADD_TO_PARENT PTOASPythonSources + ROOT_DIR "${CMAKE_SOURCE_DIR}/python/pto" + TD_FILE dialects/PTOOps.td + SOURCES dialects/pto.py + DIALECT_NAME pto ) -# 关键：放到 mlir/_mlir_libs 下（匹配 MLIR dialect python 的 import 习惯） -set_target_properties(_pto PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python/mlir/_mlir_libs" - BUILD_RPATH "${PTO_LLVM_BUILD_LIBRARY_DIR}" - INSTALL_RPATH "${PTO_LLVM_BUILD_LIBRARY_DIR}" +# Use MLIR's standard site-initializer hook to seed each Python Context with +# the selected upstream dialects shipped by PTOAS. Declaring the extension in +# the Python source graph lets AddMLIRPython own its build, installation, and +# common-CAPI dependency collection. +set(PTOAS_PYTHON_REGISTRATION_SOURCE + "${CMAKE_CURRENT_SOURCE_DIR}/PythonRegistration.cpp") +declare_mlir_python_extension(PTOASPythonSources.Registration + MODULE_NAME _site_initialize_0 + ADD_TO_PARENT PTOASPythonSources + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}" + PYTHON_BINDINGS_LIBRARY nanobind + SOURCES PythonRegistration.cpp + EMBED_CAPI_LINK_LIBS + MLIRCAPIArith + MLIRCAPIFunc + MLIRCAPILLVM + MLIRCAPIMath + MLIRCAPIMemRef + MLIRCAPISCF ) -# macOS: 避免 ld 警告 "-undefined suppress is deprecated"，仅保留 -undefined dynamic_lookup -if(APPLE) - target_link_options(_pto PRIVATE "LINKER:-undefined,dynamic_lookup") -endif() -if(NOT MLIR_PYTHON_PACKAGE_DIR) - message(FATAL_ERROR "MLIR_PYTHON_PACKAGE_DIR must be set when PTO_ENABLE_PYTHON_BINDING=ON") +# This directory otherwise materializes only upstream MLIR/nanobind C++ +# sources under -Wno-error. Keep the project-owned registration source under +# PTOAS's default strict warning policy while retaining one declarative Python +# package graph. +if(PTOAS_ENABLE_WERROR) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang|AppleClang") + set_property(SOURCE "${PTOAS_PYTHON_REGISTRATION_SOURCE}" APPEND PROPERTY + COMPILE_OPTIONS -Werror) + elseif(MSVC) + set_property(SOURCE "${PTOAS_PYTHON_REGISTRATION_SOURCE}" APPEND PROPERTY + COMPILE_OPTIONS /WX) + endif() endif() -install(TARGETS _pto - LIBRARY DESTINATION "mlir/_mlir_libs" - COMPONENT PTOAS_Python -) - -# ---- 2) Generate ODS Python op bindings: _pto_ops_gen.py ---- -set(PTO_TD_DIR "${CMAKE_SOURCE_DIR}/include/PTO/IR") -set(LLVM_TARGET_DEFINITIONS ${PTO_TD_DIR}/PTOOps.td) - -mlir_tablegen(_pto_ops_gen.py - -gen-python-op-bindings - --bind-dialect=pto +# Package only the upstream Python APIs used by PTOAS and PTODSL. In +# particular, do not instantiate MLIRPythonSources or RegisterEverything: +# downstream MLIR packages are expected to select their public dialect set +# explicitly rather than linking every upstream dialect, translation and pass. +set(PTOAS_MLIR_PYTHON_SOURCE_SETS + MLIRPythonSources.Core + MLIRPythonSources.Dialects.arith + MLIRPythonSources.Dialects.builtin + MLIRPythonSources.Dialects.func + MLIRPythonSources.Dialects.llvm + MLIRPythonSources.Dialects.math + MLIRPythonSources.Dialects.memref + MLIRPythonSources.Dialects.scf ) -add_public_tablegen_target(PTOPythonGen) -add_dependencies(_pto PTOPythonGen) - -# Stable aggregate target used by CTest and developer workflows. Keep the -# dialect binding generation owned by this directory so there is only one -# _pto_ops_gen.py producer. -add_custom_target(PTOPythonModules) -add_dependencies(PTOPythonModules _pto) - -# ---- 3) Copy generated python + handwritten pto.py into build/python ---- -set(PTO_PY_SRC "${CMAKE_SOURCE_DIR}/python/pto/dialects/pto.py") - -add_custom_command(TARGET _pto POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/python/mlir/dialects" - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "${PTO_PY_SRC}" - "${CMAKE_BINARY_DIR}/python/mlir/dialects/pto.py" - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "${CMAKE_CURRENT_BINARY_DIR}/_pto_ops_gen.py" - "${CMAKE_BINARY_DIR}/python/mlir/dialects/_pto_ops_gen.py" - VERBATIM +# Keep one common CAPI DSO for the upstream MLIR Python package. PTOAS owns a +# single ptoas._core extension, built in tools/ptoas, which links this common +# CAPI and contains both the PTO dialect implementation and compiler driver. +add_mlir_python_common_capi_library(PTOASPythonCAPI + INSTALL_COMPONENT PTOAS_Python + INSTALL_DESTINATION "mlir/_mlir_libs" + OUTPUT_DIRECTORY "${PTOAS_MLIR_PYTHON_PACKAGE_DIR}/_mlir_libs" + RELATIVE_INSTALL_ROOT "../../../.." + DECLARED_HEADERS + MLIRPythonCAPI.HeaderSources + DECLARED_SOURCES + ${PTOAS_MLIR_PYTHON_SOURCE_SETS} + PTOASPythonSources + # MLIRPythonExtension.Core currently obtains its rewrite CAPI through + # RegisterEverything. PTOAS deliberately omits RegisterEverything, so retain + # the missing direct dependency until upstream Core declares it itself. + EMBED_LIBS + MLIRCAPITransforms ) - -install(FILES - "${PTO_PY_SRC}" - "${CMAKE_CURRENT_BINARY_DIR}/_pto_ops_gen.py" - DESTINATION mlir/dialects - COMPONENT PTOAS_Python +# Imported MLIR aggregate objects do not expose a source language from which +# CMake can infer the linker driver. The common CAPI is C++, matching MLIR's +# own in-tree target. +set_property(TARGET PTOASPythonCAPI PROPERTY LINKER_LANGUAGE CXX) + +add_mlir_python_modules(PTOAS_Python + ROOT_PREFIX "${PTOAS_MLIR_PYTHON_PACKAGE_DIR}" + INSTALL_PREFIX mlir + DECLARED_SOURCES + ${PTOAS_MLIR_PYTHON_SOURCE_SETS} + PTOASPythonSources + COMMON_CAPI_LINK_LIBS + PTOASPythonCAPI ) diff --git a/lib/Bindings/Python/PTOModule.cpp b/lib/Bindings/Python/PTOModule.cpp index 432d08e262..8d6d061b29 100644 --- a/lib/Bindings/Python/PTOModule.cpp +++ b/lib/Bindings/Python/PTOModule.cpp @@ -8,14 +8,13 @@ //===- DialectPTO.cpp -----------------------------------------------------===// // -// Python bindings for the PTO dialect types (pybind11 version). -// -// This file is intended to be built via declare_mlir_python_extension(...) -// with PYTHON_BINDINGS_LIBRARY pybind11, and linked with MLIRCAPIPTO. +// Python bindings for the PTO dialect types in the project-owned ptoas._core +// extension. The public Python facade remains mlir.dialects.pto. // //===----------------------------------------------------------------------===// -#include "pybind11/pybind11.h" +#include "PTOModule.h" + #include "pybind11/stl.h" #include "mlir/Bindings/Python/PybindAdaptors.h" #include "mlir/CAPI/IR.h" @@ -119,8 +118,7 @@ void populatePTODialectSubmodule(pybind11::module &m) { (void)m; } -static void bindPTOModule(pybind11::module &m) { - m.doc() = "PTO dialect Python bindings (pybind11)."; +void mlir::pto::python::populatePTODialectBindings(pybind11::module_ &m) { // -------------------------------------------------------------------------- // Dialect registration helper @@ -1440,7 +1438,3 @@ static void bindPTOModule(pybind11::module &m) { populatePTODialectSubmodule(m); } - -PYBIND11_MODULE(_pto, m) { - bindPTOModule(m); -} diff --git a/lib/Bindings/Python/PTOModule.h b/lib/Bindings/Python/PTOModule.h new file mode 100644 index 0000000000..6fdf04a2b4 --- /dev/null +++ b/lib/Bindings/Python/PTOModule.h @@ -0,0 +1,22 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#ifndef PTOAS_LIB_BINDINGS_PYTHON_PTOMODULE_H +#define PTOAS_LIB_BINDINGS_PYTHON_PTOMODULE_H + +#include "pybind11/pybind11.h" + +namespace mlir::pto::python { + +/// Adds PTO dialect types, attributes, enums, and registration helpers to the +/// project-owned ptoas._core extension module. +void populatePTODialectBindings(pybind11::module_ &module); + +} // namespace mlir::pto::python + +#endif // PTOAS_LIB_BINDINGS_PYTHON_PTOMODULE_H diff --git a/lib/Bindings/Python/PythonRegistration.cpp b/lib/Bindings/Python/PythonRegistration.cpp new file mode 100644 index 0000000000..5fec0b0e7e --- /dev/null +++ b/lib/Bindings/Python/PythonRegistration.cpp @@ -0,0 +1,28 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +#include "mlir-c/Dialect/Arith.h" +#include "mlir-c/Dialect/Func.h" +#include "mlir-c/Dialect/LLVM.h" +#include "mlir-c/Dialect/Math.h" +#include "mlir-c/Dialect/MemRef.h" +#include "mlir-c/Dialect/SCF.h" +#include "mlir-c/IR.h" +#include "mlir/Bindings/Python/NanobindAdaptors.h" + +NB_MODULE(_site_initialize_0, module) { + module.doc() = "PTOAS MLIR dialect registration"; + module.def("register_dialects", [](MlirDialectRegistry registry) { + mlirDialectHandleInsertDialect(mlirGetDialectHandle__arith__(), registry); + mlirDialectHandleInsertDialect(mlirGetDialectHandle__func__(), registry); + mlirDialectHandleInsertDialect(mlirGetDialectHandle__llvm__(), registry); + mlirDialectHandleInsertDialect(mlirGetDialectHandle__math__(), registry); + mlirDialectHandleInsertDialect(mlirGetDialectHandle__memref__(), registry); + mlirDialectHandleInsertDialect(mlirGetDialectHandle__scf__(), registry); + }); +} diff --git a/packaging/ptoas-vmi/pyproject.toml b/packaging/ptoas-vmi/pyproject.toml index c5bf63ca99..c5de369d66 100644 --- a/packaging/ptoas-vmi/pyproject.toml +++ b/packaging/ptoas-vmi/pyproject.toml @@ -12,7 +12,7 @@ # name required by package indexes without a custom build backend. [build-system] -requires = ["scikit-build-core>=0.12.2,<2", "pybind11<3"] +requires = ["scikit-build-core>=0.12.2,<2", "pybind11<3", "nanobind>=2.4"] build-backend = "scikit_build_core.build" [project] diff --git a/ptodsl/CMakeLists.txt b/ptodsl/CMakeLists.txt index ef602d320f..4be1601179 100644 --- a/ptodsl/CMakeLists.txt +++ b/ptodsl/CMakeLists.txt @@ -12,19 +12,19 @@ set(PTODSL_PACKAGE_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ptodsl") -set(PTODSL_BUILD_ROOT "${CMAKE_BINARY_DIR}/python") set(PTODSL_BUILD_PACKAGE_DIR - "${PTODSL_BUILD_ROOT}/ptodsl") + "${CMAKE_BINARY_DIR}/python/ptodsl") -add_custom_target(PTODSLPackage ALL - COMMAND ${CMAKE_COMMAND} -E make_directory "${PTODSL_BUILD_ROOT}" - COMMAND ${CMAKE_COMMAND} -E remove_directory "${PTODSL_BUILD_PACKAGE_DIR}" - COMMAND ${CMAKE_COMMAND} -E copy_directory - "${PTODSL_PACKAGE_SRC_DIR}" - "${PTODSL_BUILD_PACKAGE_DIR}" - COMMENT "Staging ptodsl package into build/python" - VERBATIM +declare_mlir_python_sources(PTODSLPythonSources + ROOT_DIR "${PTODSL_PACKAGE_SRC_DIR}" + SOURCES_GLOB "*.py" ) +add_mlir_python_sources_target(PTODSLPackageStage + OUTPUT_DIRECTORY "${PTODSL_BUILD_PACKAGE_DIR}" + SOURCES_TARGETS PTODSLPythonSources +) +add_custom_target(PTODSLPackage ALL) +add_dependencies(PTODSLPackage PTODSLPackageStage) install( DIRECTORY "${PTODSL_PACKAGE_SRC_DIR}" diff --git a/ptodsl/docs/developer_guide/tilelib-debugging-playbook.md b/ptodsl/docs/developer_guide/tilelib-debugging-playbook.md index db519c5165..324764b93d 100644 --- a/ptodsl/docs/developer_guide/tilelib-debugging-playbook.md +++ b/ptodsl/docs/developer_guide/tilelib-debugging-playbook.md @@ -12,7 +12,7 @@ usually live in different layers. Build PTOAS after C++ or TableGen changes: ```bash -cmake --build build-llvm21 --target ptoas_runtime +cmake --build build-llvm21 --target PTOASPythonCore ``` Stage PTODSL after Python package changes: diff --git a/ptodsl/ptoas/_cli.py b/ptodsl/ptoas/_cli.py index 7cb3132679..7798275f28 100644 --- a/ptodsl/ptoas/_cli.py +++ b/ptodsl/ptoas/_cli.py @@ -26,15 +26,15 @@ def _resolve_wrapper_path(argv0: str | None = None) -> Path: def _load_native_module(): - from ptoas import _native + from ptoas import _core - return _native + return _core def _resolve_runtime_paths(native_module) -> tuple[Path, Path]: module_file = getattr(native_module, "__file__", None) if not module_file: - raise SystemExit("ptoas._native does not expose a module file") + raise SystemExit("ptoas._core does not expose a module file") package_root = Path(module_file).resolve().parent python_root = package_root.parent diff --git a/ptodsl/tests/CMakeLists.txt b/ptodsl/tests/CMakeLists.txt index 1722bf5d08..1bbb4076d2 100644 --- a/ptodsl/tests/CMakeLists.txt +++ b/ptodsl/tests/CMakeLists.txt @@ -7,17 +7,13 @@ # See LICENSE in the root of the software repository for the full text of the License. if(NOT DEFINED _pto_python_test_env) - get_filename_component(_llvm_root "${LLVM_DIR}/../../.." ABSOLUTE) get_filename_component(_pto_python_bin_dir "${Python3_EXECUTABLE}" DIRECTORY) set(_ptodsl_test_pythonpath - "${CMAKE_BINARY_DIR}/python:${MLIR_PYTHON_PACKAGE_DIR}:${CMAKE_INSTALL_PREFIX}:$ENV{PYTHONPATH}" + "${CMAKE_BINARY_DIR}/python:$ENV{PYTHONPATH}" ) - if(DEFINED ENV{PTO_INSTALL_DIR} AND NOT "$ENV{PTO_INSTALL_DIR}" STREQUAL "") - set(_ptodsl_test_pythonpath "$ENV{PTO_INSTALL_DIR}:${_ptodsl_test_pythonpath}") - endif() set(_pto_python_test_env "PYTHONPATH=${_ptodsl_test_pythonpath}" - "LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:${_llvm_root}/lib:${CMAKE_INSTALL_PREFIX}/lib:$ENV{LD_LIBRARY_PATH}" + "LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/lib:${LLVM_BUILD_LIBRARY_DIR}:${CMAKE_INSTALL_PREFIX}/lib:$ENV{LD_LIBRARY_PATH}" "PATH=${CMAKE_BINARY_DIR}/tools/ptoas:${CMAKE_INSTALL_PREFIX}/bin:${_pto_python_bin_dir}:$ENV{PATH}" "PTOAS_BIN=${CMAKE_BINARY_DIR}/tools/ptoas/ptoas" ) @@ -60,7 +56,7 @@ endforeach() if(NOT TARGET check-dsl) set(_ptodsl_check_depends) - foreach(_target IN ITEMS PTODSLPackage PTOPythonModules ptoas_runtime_deps) + foreach(_target IN ITEMS PTODSLPackage PTOAS_Python ptoas_runtime_deps) if(TARGET ${_target}) list(APPEND _ptodsl_check_depends ${_target}) endif() diff --git a/ptodsl/tests/test_allreduce.py b/ptodsl/tests/test_allreduce.py index 533ba62750..78bac49288 100644 --- a/ptodsl/tests/test_allreduce.py +++ b/ptodsl/tests/test_allreduce.py @@ -7,11 +7,6 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from pathlib import Path -import sys - -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "ptodsl")) - from ptodsl import pto diff --git a/ptodsl/tests/test_ptoas_cli.py b/ptodsl/tests/test_ptoas_cli.py index 9afd21e42c..7917a2ccb3 100644 --- a/ptodsl/tests/test_ptoas_cli.py +++ b/ptodsl/tests/test_ptoas_cli.py @@ -19,7 +19,7 @@ class PTOASCLITests(unittest.TestCase): def _make_native_module(self, package_root: Path): return SimpleNamespace( - __file__=str(package_root / "_native.fake.so"), + __file__=str(package_root / "_core.fake.so"), main=mock.Mock(return_value=0), ) diff --git a/ptodsl/tests/test_ptoas_tree_wrapper.py b/ptodsl/tests/test_ptoas_tree_wrapper.py index 59f5545873..5bea72f01c 100644 --- a/ptodsl/tests/test_ptoas_tree_wrapper.py +++ b/ptodsl/tests/test_ptoas_tree_wrapper.py @@ -27,13 +27,22 @@ def launch(user_args, *, wrapper=None): class TreeWrapperTests(unittest.TestCase): - def _load_wrapper(self, wrapper_path: Path, module_name: str): + def _load_wrapper( + self, + wrapper_path: Path, + module_name: str, + *, + python_root_mode: str, + python_root: Path, + ): spec = importlib.util.spec_from_file_location(module_name, WRAPPER_SOURCE) self.assertIsNotNone(spec) self.assertIsNotNone(spec.loader) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) module.__file__ = str(wrapper_path) + module._PYTHON_ROOT_MODE = python_root_mode + module._PYTHON_ROOT = python_root return module def _run_wrapper(self, wrapper_module, python_root: Path, wrapper: Path): @@ -73,7 +82,12 @@ def test_build_tree_wrapper_delegates_to_cli_module(self): wrapper.parent.mkdir(parents=True) wrapper.write_text("", encoding="utf-8") - module = self._load_wrapper(wrapper, "test_ptoas_build_wrapper") + module = self._load_wrapper( + wrapper, + "test_ptoas_build_wrapper", + python_root_mode="absolute", + python_root=python_root, + ) cli = self._run_wrapper(module, python_root, wrapper) self.assertEqual(cli.calls, [(["--version"], wrapper.resolve())]) @@ -86,7 +100,12 @@ def test_install_tree_wrapper_delegates_to_cli_module(self): wrapper.parent.mkdir(parents=True) wrapper.write_text("", encoding="utf-8") - module = self._load_wrapper(wrapper, "test_ptoas_install_wrapper") + module = self._load_wrapper( + wrapper, + "test_ptoas_install_wrapper", + python_root_mode="wrapper-relative", + python_root=Path(".."), + ) cli = self._run_wrapper(module, install_root, wrapper) self.assertEqual(cli.calls, [(["--version"], wrapper.resolve())]) @@ -98,6 +117,8 @@ def test_wrapper_requires_cli_module(self): module = self._load_wrapper( Path(temp_dir) / "build" / "tools" / "ptoas" / "ptoas", "test_ptoas_missing_cli", + python_root_mode="absolute", + python_root=python_root, ) with self.assertRaisesRegex(SystemExit, "ptoas/_cli.py"): module._require_python_root(python_root, context="test") diff --git a/ptodsl/tests/test_vmi_vmull.py b/ptodsl/tests/test_vmi_vmull.py index 3fbf2972a4..fad01a208c 100644 --- a/ptodsl/tests/test_vmi_vmull.py +++ b/ptodsl/tests/test_vmi_vmull.py @@ -7,12 +7,6 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from pathlib import Path -import sys - - -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "ptodsl")) - from ptodsl import pto diff --git a/ptodsl/tests/test_vmi_vshr_signedness.py b/ptodsl/tests/test_vmi_vshr_signedness.py index f4dfe09340..848ddec81a 100644 --- a/ptodsl/tests/test_vmi_vshr_signedness.py +++ b/ptodsl/tests/test_vmi_vshr_signedness.py @@ -7,12 +7,6 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from pathlib import Path -import sys - - -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "ptodsl")) - from ptodsl import pto diff --git a/pyproject.toml b/pyproject.toml index 072d2fb95f..beaed1b33b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ # a persistent CMake build tree. [build-system] -requires = ["scikit-build-core>=0.12.2,<2", "pybind11<3"] +requires = ["scikit-build-core>=0.12.2,<2", "pybind11<3", "nanobind>=2.4"] build-backend = "scikit_build_core.build" [project] diff --git a/python/pto/dialects/pto.py b/python/pto/dialects/pto.py index 3e66f0bdb1..f2038d060a 100644 --- a/python/pto/dialects/pto.py +++ b/python/pto/dialects/pto.py @@ -6,12 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -import importlib -import importlib.util import functools -import os -from pathlib import Path -import sys from typing import Optional from mlir import ir as _ods_ir @@ -22,77 +17,7 @@ get_op_result_or_value as _get_op_result_or_value, get_op_results_or_values as _get_op_results_or_values, ) - - -def _candidate_pto_ext_dirs(): - # Prefer the extension shipped next to the active wrapper so installed - # wheels stay self-consistent even if the caller also exported a developer - # PTO_INSTALL_DIR / PTO_PYTHON_ROOT in the environment. - candidates = [Path(__file__).resolve().parent.parent / "_mlir_libs"] - env_roots = ( - os.environ.get("PTO_PYTHON_BUILD_ROOT"), - os.environ.get("PTO_PYTHON_ROOT"), - os.environ.get("PTO_INSTALL_DIR"), - ) - for root in env_roots: - if not root: - continue - candidates.append(Path(root) / "mlir" / "_mlir_libs") - - seen = set() - ordered = [] - for candidate in candidates: - candidate = candidate.resolve() - candidate_text = str(candidate) - if candidate_text in seen: - continue - seen.add(candidate_text) - ordered.append(candidate) - return ordered - - -def _load_local_pto_ext(): - module_name = "mlir._mlir_libs._pto" - cached = sys.modules.get(module_name) - if cached is not None: - return cached - - candidates = [] - for lib_dir in _candidate_pto_ext_dirs(): - for suffix in ("*.so", "*.pyd", "*.dll", "*.dylib"): - candidates.extend(lib_dir.glob(f"_pto{suffix}")) - if not candidates: - raise FileNotFoundError("cannot locate local _pto extension in candidate _mlir_libs directories") - - first_error = None - for so_path in candidates: - try: - spec = importlib.util.spec_from_file_location( - module_name, so_path - ) - if spec and spec.loader: - mod = importlib.util.module_from_spec(spec) - previous = sys.modules.get(module_name) - sys.modules[module_name] = mod - try: - spec.loader.exec_module(mod) - except Exception: - if previous is None: - sys.modules.pop(module_name, None) - else: - sys.modules[module_name] = previous - raise - return mod - except ImportError as exc: - if first_error is None: - first_error = exc - raise ImportError("failed to load local _pto extension from candidate _mlir_libs directories") from first_error - - -try: - _pto_mod = _load_local_pto_ext() -except FileNotFoundError: - _pto_mod = importlib.import_module(".._mlir_libs._pto", __package__) +from ptoas import _core as _pto_mod def _export_generated_symbols(): diff --git a/quick_install.sh b/quick_install.sh index 9c3767a28b..253f5713ab 100755 --- a/quick_install.sh +++ b/quick_install.sh @@ -31,7 +31,8 @@ fi "${PYTHON_BIN}" -m pip install \ 'scikit-build-core>=0.12.2,<2' \ - 'pybind11<3' + 'pybind11<3' \ + 'nanobind>=2.4' LLVM_BUILD_DIR="${LLVM_BUILD_DIR}" \ "${PYTHON_BIN}" -m pip install --editable "${PTO_SOURCE_DIR}" \ diff --git a/scripts/ptoas_env.sh b/scripts/ptoas_env.sh index b5e814ccda..1686ba1bc0 100644 --- a/scripts/ptoas_env.sh +++ b/scripts/ptoas_env.sh @@ -41,7 +41,6 @@ export PTO_INSTALL_DIR="${PTO_INSTALL_DIR:-${PTO_SOURCE_DIR}/install}" export PTO_ISA_PATH="${PTO_ISA_PATH:-${WORKSPACE_DIR}/pto-isa}" export ASCEND_HOME_PATH="${ASCEND_HOME_PATH:-${HOME}/cann}" -export MLIR_PYTHON_ROOT="${MLIR_PYTHON_ROOT:-${LLVM_BUILD_DIR}/tools/mlir/python_packages/mlir_core}" if [[ -z "${PTOAS_PYTHON_SITE:-}" ]]; then PTOAS_PYTHON_SITE="$( PTO_INSTALL_DIR="${PTO_INSTALL_DIR}" python3 - <<'PY' 2>/dev/null || true @@ -57,7 +56,6 @@ export PTOAS_PYTHON_SITE export PTO_PYTHON_ROOT="${PTO_PYTHON_ROOT:-${PTO_INSTALL_DIR}}" export PTO_PYTHON_BUILD_ROOT="${PTO_PYTHON_BUILD_ROOT:-${PTO_SOURCE_DIR}/build/python}" export PTODSL_PYTHON_ROOT="${PTODSL_PYTHON_ROOT:-${PTO_SOURCE_DIR}/ptodsl}" -export PYBIND11_CMAKE_DIR=$(python3 -m pybind11 --cmakedir) export PTOAS_FLAGS="${PTOAS_FLAGS:-}" export PTOAS_OUT_DIR="${PTOAS_OUT_DIR:-${PTO_SOURCE_DIR}/build/output}" @@ -110,13 +108,12 @@ _ptoas_run_legacy_smoke_test() { echo "test set_env: OK" } -# Prefer the in-tree PTO Python overlay plus LLVM's full MLIR package first. -# The install prefix may only contain the PTO overlay fragments, and when it is -# placed ahead of mlir_core it can shadow the real MLIR Python bindings. +# PTOAS stages a complete MLIR + PTO package in its build and install trees. +# Prefer the build tree for active development, while retaining install-tree +# fallbacks for scripts that consume a direct CMake installation. _ptoas_prepend_path PYTHONPATH "${PTO_INSTALL_DIR}" _ptoas_prepend_path PYTHONPATH "${PTO_PYTHON_ROOT}" _ptoas_prepend_path PYTHONPATH "${PTOAS_PYTHON_SITE}" -_ptoas_prepend_path PYTHONPATH "${MLIR_PYTHON_ROOT}" _ptoas_prepend_path PYTHONPATH "${PTO_PYTHON_BUILD_ROOT}" _ptoas_prepend_path PYTHONPATH "${PTODSL_PYTHON_ROOT}" diff --git a/test/lit/lit.cfg.py b/test/lit/lit.cfg.py index 4b9054c7db..cb8d4a408b 100644 --- a/test/lit/lit.cfg.py +++ b/test/lit/lit.cfg.py @@ -54,16 +54,12 @@ 'ASCEND_HOME_PATH', 'ASCEND_OPP_PATH', 'ASCEND_AICPU_PATH', 'ASCEND_TOOLKIT_HOME', 'LD_LIBRARY_PATH', 'PYTHONPATH']) -# Keep build-tree lit tests self-contained. PTOAS contributes an overlay -# under the build tree (generated dialect Python plus the PTO extension), -# while the base ``mlir.ir`` package comes from the LLVM/MLIR build. MLIR's -# own lit setup wires these paths from its CMake-generated site configuration; -# do the same here instead of requiring developers to export PYTHONPATH. +# Keep build-tree lit tests self-contained. The PTOAS build stages the complete +# MLIR Python runtime together with generated PTO dialect modules and the PTO +# extension under one build-tree Python root. if getattr(config, 'enable_bindings_python', False): - python_paths = [config.ptoas_python_dir] - if config.mlir_python_package_dir: - python_paths.append(config.mlir_python_package_dir) - llvm_config.with_environment('PYTHONPATH', python_paths, append_path=True) + llvm_config.with_environment( + 'PYTHONPATH', [config.ptoas_python_dir], append_path=True) llvm_config.use_default_substitutions() diff --git a/test/lit/lit.site.cfg.py.in b/test/lit/lit.site.cfg.py.in index b820f33fd8..3972bbf620 100644 --- a/test/lit/lit.site.cfg.py.in +++ b/test/lit/lit.site.cfg.py.in @@ -21,8 +21,6 @@ config.mlir_lib_dir = "@MLIR_LIB_DIR@" config.python_executable = "@Python3_EXECUTABLE@" config.enable_bindings_python = "@PTO_ENABLE_PYTHON_BINDING@" == "ON" config.ptoas_python_dir = path(r"@CMAKE_BINARY_DIR@/python") -config.mlir_python_package_dir = lit_config.substitute( - path(r"@MLIR_PYTHON_PACKAGE_DIR@")) config.ptoir_src_root = "@CMAKE_SOURCE_DIR@" config.ptoir_obj_root = "@CMAKE_BINARY_DIR@" config.pto_enable_vfsim_costmodel = "@PTO_ENABLE_VFSIM_COSTMODEL@" == "ON" diff --git a/test/tilelib-st/README.md b/test/tilelib-st/README.md index eb3aa26b34..cb87dee72d 100644 --- a/test/tilelib-st/README.md +++ b/test/tilelib-st/README.md @@ -226,7 +226,6 @@ export PTOAS_REPO="$PWD" export PTOAS_BUILD="$PTOAS_REPO/build-sim" export ASCEND_HOME_PATH=/path/to/cann export LLVM_BUILD_DIR=/path/to/vpto-llvm/build -export MLIR_PYTHON_ROOT="$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core" export PYTHON_BIN=/path/to/python-with-torch-npu export PTO_PYTHON_BIN="$PYTHON_BIN" export PTOAS_ENV_SKIP_SMOKE_TEST=1 @@ -246,7 +245,6 @@ Check the simulator, LLVM, and Python prerequisites: command -v msprof test -d "$LLVM_BUILD_DIR/lib/cmake/llvm" test -d "$LLVM_BUILD_DIR/lib/cmake/mlir" -test -d "$MLIR_PYTHON_ROOT" "$PYTHON_BIN" - <<'PY' import nanobind @@ -277,27 +275,20 @@ PTOAS Python bindings must have the same Python ABI as the runtime Python that will run the ST cases: ```bash -PYBIND11_CMAKE_DIR="$("$PYTHON_BIN" -m pybind11 --cmakedir)" -NANOBIND_CMAKE_DIR="$("$PYTHON_BIN" -m nanobind --cmake_dir)" - cmake -S "$PTOAS_REPO" -B "$PTOAS_BUILD" -G Ninja \ -DLLVM_DIR="$LLVM_BUILD_DIR/lib/cmake/llvm" \ -DMLIR_DIR="$LLVM_BUILD_DIR/lib/cmake/mlir" \ -DPython3_EXECUTABLE="$PYTHON_BIN" \ -DPython3_FIND_STRATEGY=LOCATION \ - -Dpybind11_DIR="$PYBIND11_CMAKE_DIR" \ - -Dnanobind_DIR="$NANOBIND_CMAKE_DIR" \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DMLIR_PYTHON_PACKAGE_DIR="$MLIR_PYTHON_ROOT" \ -DPTO_ENABLE_PYTHON_BINDING=ON \ - -DPTOAS_ENABLE_WERROR=OFF \ -DBUILD_TESTING=ON ``` Build the compiler and PTO Python bindings: ```bash -ninja -C "$PTOAS_BUILD" ptoas PTOPythonModules +ninja -C "$PTOAS_BUILD" ptoas PTOAS_Python ``` Export the runtime lookup paths. `PTOAS_BIN` is useful, but PTODSL native build @@ -306,7 +297,7 @@ also resolves `ptoas` through `PATH`, so keep the freshly built binary first: ```bash export PTOAS_BIN="$PTOAS_BUILD/tools/ptoas/ptoas" export PATH="$PTOAS_BUILD/tools/ptoas:$PATH" -export PYTHONPATH="$PTOAS_BUILD/python:$PTOAS_REPO/ptodsl:$MLIR_PYTHON_ROOT:${PYTHONPATH:-}" +export PYTHONPATH="$PTOAS_BUILD/python:$PTOAS_REPO/ptodsl:${PYTHONPATH:-}" export LD_LIBRARY_PATH="$LLVM_BUILD_DIR/lib:$PTOAS_BUILD/lib:${LD_LIBRARY_PATH:-}" ``` @@ -365,7 +356,6 @@ Example path layout: ```bash export ASCEND_HOME_PATH=/opt/ascend/cann export LLVM_BUILD_DIR=/opt/vpto-llvm/build -export MLIR_PYTHON_ROOT="$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core" export PYTHON_BIN=/opt/ptodsl-runtime/bin/python ``` @@ -379,5 +369,5 @@ Common failures: - `No TileLib ST cases discovered`: pass the case root through `scripts/sim_dsl.sh ... run_tilelib_st.py -- test/tilelib-st/a5`, keeping the `--` separator. -- Stale compiler behavior: rebuild `ptoas PTOPythonModules`, put +- Stale compiler behavior: rebuild `ptoas PTOAS_Python`, put `$PTOAS_BUILD/tools/ptoas` first in `PATH`, and verify `$PTOAS_BIN --version`. diff --git a/tilelang-dsl/CMakeLists.txt b/tilelang-dsl/CMakeLists.txt index 2bbca7c3dc..db11784d6f 100644 --- a/tilelang-dsl/CMakeLists.txt +++ b/tilelang-dsl/CMakeLists.txt @@ -12,19 +12,19 @@ set(TILELANG_DSL_PACKAGE_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/python/tilelang_dsl") -set(TILELANG_DSL_BUILD_ROOT "${CMAKE_BINARY_DIR}/python") set(TILELANG_DSL_BUILD_PACKAGE_DIR - "${TILELANG_DSL_BUILD_ROOT}/tilelang_dsl") + "${CMAKE_BINARY_DIR}/python/tilelang_dsl") -add_custom_target(TileLangDSLPackage ALL - COMMAND ${CMAKE_COMMAND} -E make_directory "${TILELANG_DSL_BUILD_ROOT}" - COMMAND ${CMAKE_COMMAND} -E remove_directory "${TILELANG_DSL_BUILD_PACKAGE_DIR}" - COMMAND ${CMAKE_COMMAND} -E copy_directory - "${TILELANG_DSL_PACKAGE_SRC_DIR}" - "${TILELANG_DSL_BUILD_PACKAGE_DIR}" - COMMENT "Staging tilelang_dsl package into build/python" - VERBATIM +declare_mlir_python_sources(TileLangDSLPythonSources + ROOT_DIR "${TILELANG_DSL_PACKAGE_SRC_DIR}" + SOURCES_GLOB "*.py" ) +add_mlir_python_sources_target(TileLangDSLPackageStage + OUTPUT_DIRECTORY "${TILELANG_DSL_BUILD_PACKAGE_DIR}" + SOURCES_TARGETS TileLangDSLPythonSources +) +add_custom_target(TileLangDSLPackage ALL) +add_dependencies(TileLangDSLPackage TileLangDSLPackageStage) install( DIRECTORY "${TILELANG_DSL_PACKAGE_SRC_DIR}" diff --git a/tilelang-dsl/python/tilelang_dsl/env_config.py b/tilelang-dsl/python/tilelang_dsl/env_config.py index 00dc0e66e5..47fdd9b7ef 100644 --- a/tilelang-dsl/python/tilelang_dsl/env_config.py +++ b/tilelang-dsl/python/tilelang_dsl/env_config.py @@ -34,10 +34,10 @@ def check_pto_dialect_available() -> bool: """Check if PTO dialect bindings are available. Returns: - True if pto.dialects.pto module can be imported, False otherwise. + True if mlir.dialects.pto can be imported, False otherwise. """ try: - from pto.dialects import pto + from mlir.dialects import pto return True except ImportError: return False @@ -64,28 +64,23 @@ def print_environment_help() -> None: print("PybindBackend Environment Configuration Guide") print("=" * 60) print() - print("PybindBackend requires MLIR Python bindings from LLVM build.") + print("PybindBackend requires the MLIR Python package built by PTOAS.") print() print("To set up the environment, add the following to PYTHONPATH:") print() - print("1. LLVM MLIR Python package:") - print(" $LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core") - print() - print("2. PTO dialect bindings (optional, for PTO ops):") - print(" $PTOAS_BUILD_DIR/python/mlir") + print("1. PTOAS build-tree Python package:") + print(" $PTOAS_BUILD_DIR/python") print(" OR") - print(" $PTO_INSTALL_DIR/mlir") + print("2. PTOAS install tree:") + print(" $PTO_INSTALL_DIR") print() print("Example setup:") print() - print(" # Assuming LLVM is built at ~/llvm-workspace/llvm-project/build-shared") - print(" export LLVM_BUILD_DIR=~/llvm-workspace/llvm-project/build-shared") - print() print(" # Assuming PTOAS is at ~/llvm-workspace/pto-as") print(" export PTOAS_BUILD_DIR=~/llvm-workspace/pto-as/build") print() print(" # Set PYTHONPATH") - print(" export PYTHONPATH=$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core:$PTOAS_BUILD_DIR/python") + print(" export PYTHONPATH=$PTOAS_BUILD_DIR/python") print() print("After setup, verify with:") print(" python3 -c 'from mlir import ir; print(\"MLIR bindings OK\")'") @@ -128,4 +123,4 @@ def verify_environment() -> bool: "get_environment_status", "print_environment_help", "verify_environment", -] \ No newline at end of file +] diff --git a/tilelang-dsl/python/tilelang_dsl/pybind_renderer.py b/tilelang-dsl/python/tilelang_dsl/pybind_renderer.py index 8b9ad249c0..a817cec884 100644 --- a/tilelang-dsl/python/tilelang_dsl/pybind_renderer.py +++ b/tilelang-dsl/python/tilelang_dsl/pybind_renderer.py @@ -14,16 +14,15 @@ Phase 2 implementation: core framework with basic stmt/expr rendering. Dependencies: - - mlir.ir: MLIR Python bindings from LLVM build + - mlir.ir: MLIR Python bindings assembled by the PTOAS build - mlir.dialects.func, arith, scf: Standard MLIR dialects - mlir.dialects.pto: PTO dialect bindings from PTOAS build -To use this module, ensure PYTHONPATH includes: - 1. LLVM MLIR Python package: $LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core - 2. PTOAS Python bindings: $PTOAS_BUILD_DIR/python +To use this module from a build tree, ensure PYTHONPATH includes: + $PTOAS_BUILD_DIR/python Example: - export PYTHONPATH="$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core:$PTOAS_BUILD_DIR/python" + export PYTHONPATH="$PTOAS_BUILD_DIR/python" """ from __future__ import annotations @@ -67,8 +66,8 @@ def _ensure_mlir_bindings() -> None: except ImportError as exc: raise ImportError( "MLIR Python bindings not found. Please ensure PYTHONPATH includes " - "the MLIR Python package from your LLVM build. " - "Example: export PYTHONPATH=$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core " + "the Python package assembled by your PTOAS build. " + "Example: export PYTHONPATH=$PTOAS_BUILD_DIR/python " "See README.md for build instructions." ) from exc diff --git a/tilelang-dsl/tests/backend/test_e2e_pybind_backend.py b/tilelang-dsl/tests/backend/test_e2e_pybind_backend.py index c660cb64ee..a3b6088b93 100644 --- a/tilelang-dsl/tests/backend/test_e2e_pybind_backend.py +++ b/tilelang-dsl/tests/backend/test_e2e_pybind_backend.py @@ -18,7 +18,7 @@ python3 tests/backend/test_e2e_pybind_backend.py Environment setup (optional for PybindBackend): - export PYTHONPATH=$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core:$PTOAS_BUILD_DIR/python + export PYTHONPATH=$PTOAS_BUILD_DIR/python """ import sys @@ -309,4 +309,4 @@ def main(): if __name__ == "__main__": success = main() - sys.exit(0 if success else 1) \ No newline at end of file + sys.exit(0 if success else 1) diff --git a/tools/ptoas/CMakeLists.txt b/tools/ptoas/CMakeLists.txt index 75db33c55a..0ec5afe766 100644 --- a/tools/ptoas/CMakeLists.txt +++ b/tools/ptoas/CMakeLists.txt @@ -16,19 +16,38 @@ set(PTOAS_RUNTIME_SOURCES set(PTOAS_WRAPPER_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/ptoas_wrapper.py") set(PTOAS_WRAPPER_BINARY "${CMAKE_CURRENT_BINARY_DIR}/ptoas") +set(PTOAS_INSTALL_WRAPPER_BINARY "${CMAKE_CURRENT_BINARY_DIR}/ptoas-install") +set(PTOAS_WRAPPER_PYTHON_ROOT_MODE "absolute") +set(PTOAS_WRAPPER_PYTHON_ROOT "${CMAKE_BINARY_DIR}/python") configure_file( "${PTOAS_WRAPPER_SOURCE}" "${PTOAS_WRAPPER_BINARY}" @ONLY ) -file(CHMOD "${PTOAS_WRAPPER_BINARY}" - PERMISSIONS - OWNER_READ OWNER_WRITE OWNER_EXECUTE - GROUP_READ GROUP_EXECUTE - WORLD_READ WORLD_EXECUTE + +# CMake install prefixes may be overridden at install time. Resolve the +# installed package relative to bin/ptoas instead of baking in the configure +# prefix. +set(PTOAS_WRAPPER_PYTHON_ROOT_MODE "wrapper-relative") +set(PTOAS_WRAPPER_PYTHON_ROOT "..") +configure_file( + "${PTOAS_WRAPPER_SOURCE}" + "${PTOAS_INSTALL_WRAPPER_BINARY}" + @ONLY ) +foreach(_ptoas_wrapper IN ITEMS + "${PTOAS_WRAPPER_BINARY}" + "${PTOAS_INSTALL_WRAPPER_BINARY}") + file(CHMOD "${_ptoas_wrapper}" + PERMISSIONS + OWNER_READ OWNER_WRITE OWNER_EXECUTE + GROUP_READ GROUP_EXECUTE + WORLD_READ WORLD_EXECUTE + ) +endforeach() + function(ptoas_configure_runtime_target target_name) target_compile_definitions(${target_name} PRIVATE PTOAS_RELEASE_VERSION="${PTOAS_CLI_VERSION}" @@ -82,67 +101,79 @@ add_library(ptoas_runtime_objects OBJECT ${PTOAS_RUNTIME_SOURCES}) llvm_update_compile_flags(ptoas_runtime_objects) ptoas_configure_runtime_target(ptoas_runtime_objects) -# Build the normal Python extension used by wheel, build-tree, install-tree, -# and Python-based standalone distributions. Linking the object library pulls -# in the driver implementation and propagates its native link dependencies. -pybind11_add_module(ptoas_runtime MODULE NativeModule.cpp) -target_link_libraries(ptoas_runtime PRIVATE ptoas_runtime_objects) -set_target_properties(ptoas_runtime PROPERTIES - OUTPUT_NAME "_native" - LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/python-module" - # Keep the external LLVM/MLIR build discoverable for build-tree and - # editable Python installs. Release wheels are subsequently repaired by - # auditwheel, which replaces this machine-local path with wheel-local - # dependencies and a relative RPATH. - BUILD_RPATH "${PTO_LLVM_BUILD_LIBRARY_DIR}" - INSTALL_RPATH "${PTO_LLVM_BUILD_LIBRARY_DIR}" +# Build the single PTOAS-owned Python extension. It contains both the compiler +# driver and PTO dialect bindings, while linking MLIR's shared common CAPI so +# Python-created MLIR objects and the compiler use the same MLIR runtime. +set(PTOAS_BUILD_PYTHON_PACKAGE_DIR "${CMAKE_BINARY_DIR}/python/ptoas") +add_mlir_python_extension(PTOASPythonCore _core + INSTALL_COMPONENT PTOAS_Python + INSTALL_DIR ptoas + OUTPUT_DIRECTORY "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}" + PYTHON_BINDINGS_LIBRARY pybind11 + SOURCES + NativeModule.cpp + "${CMAKE_SOURCE_DIR}/lib/Bindings/Python/PTOModule.cpp" + LINK_LIBS + ptoas_runtime_objects + PTOCAPI + PTOASPythonCAPI +) +target_include_directories(PTOASPythonCore PRIVATE + "${CMAKE_SOURCE_DIR}/lib/Bindings/Python" ) +mlir_python_setup_extension_rpath(PTOASPythonCore) -set(PTOAS_BUILD_PYTHON_PACKAGE_DIR "${CMAKE_BINARY_DIR}/python/ptoas") -add_custom_target(PTOASPythonPackage ALL - COMMAND ${CMAKE_COMMAND} -E remove_directory - "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}" - COMMAND ${CMAKE_COMMAND} -E copy_directory - "${CMAKE_SOURCE_DIR}/ptodsl/ptoas" - "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}" - COMMAND ${CMAKE_COMMAND} -E copy_if_different - "$" - "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}/$" - COMMAND ${CMAKE_COMMAND} -E make_directory - "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}/_runtime/share/ptoas" - COMMAND ${CMAKE_COMMAND} -E copy_directory - "${CMAKE_SOURCE_DIR}/lib/TileOps" - "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}/_runtime/share/ptoas/TileOps" - COMMAND ${CMAKE_COMMAND} -E remove_directory - "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}/_runtime/share/ptoas/TileOps/__pycache__" - DEPENDS ptoas_runtime - COMMENT "Staging the ptoas Python package" - VERBATIM +if(APPLE) + set_property(TARGET PTOASPythonCore PROPERTY INSTALL_RPATH + "@loader_path;@loader_path/../mlir/_mlir_libs") +elseif(UNIX) + set_property(TARGET PTOASPythonCore PROPERTY INSTALL_RPATH + "$ORIGIN;$ORIGIN/../mlir/_mlir_libs") +endif() + +declare_mlir_python_sources(PTOASRuntimePythonSources + ROOT_DIR "${CMAKE_SOURCE_DIR}/ptodsl/ptoas" + SOURCES_GLOB "*.py" +) +add_mlir_python_sources_target(PTOASRuntimePythonSourcesStage + OUTPUT_DIRECTORY "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}" + SOURCES_TARGETS PTOASRuntimePythonSources +) + +declare_mlir_python_sources(PTOASTileOpsPythonSources + ROOT_DIR "${CMAKE_SOURCE_DIR}/lib/TileOps" + SOURCES_GLOB "*.py" +) +add_mlir_python_sources_target(PTOASTileOpsPythonSourcesStage + OUTPUT_DIRECTORY + "${PTOAS_BUILD_PYTHON_PACKAGE_DIR}/_runtime/share/ptoas/TileOps" + SOURCES_TARGETS PTOASTileOpsPythonSources +) + +add_custom_target(PTOASPythonPackage ALL) +add_dependencies(PTOASPythonPackage + PTOASPythonCore + PTOAS_Python + PTOASRuntimePythonSourcesStage + PTOASTileOpsPythonSourcesStage ) if(TARGET PTODSLPackage) add_dependencies(PTOASPythonPackage PTODSLPackage) endif() - -# Keep the historical aggregate target name for existing developer commands; -# the actual runtime payload now lives in the staged Python package. -add_custom_target(ptoas_runtime_staging ALL - DEPENDS PTOASPythonPackage -) +if(TARGET TileLangDSLPackage) + add_dependencies(PTOASPythonPackage TileLangDSLPackage) +endif() if(TARGET ptoas_runtime_deps) - add_dependencies(ptoas_runtime_deps ptoas_runtime_staging) + add_dependencies(ptoas_runtime_deps PTOASPythonPackage) endif() -install(PROGRAMS "${PTOAS_WRAPPER_BINARY}" +install(PROGRAMS "${PTOAS_INSTALL_WRAPPER_BINARY}" DESTINATION bin + RENAME ptoas COMPONENT PTOAS_Runtime ) -install(TARGETS ptoas_runtime - LIBRARY DESTINATION ptoas - COMPONENT PTOAS_Python -) - install( DIRECTORY "${CMAKE_SOURCE_DIR}/lib/TileOps" DESTINATION "ptoas/_runtime/share/ptoas" diff --git a/tools/ptoas/NativeModule.cpp b/tools/ptoas/NativeModule.cpp index 54bfc26e41..e0097e0641 100644 --- a/tools/ptoas/NativeModule.cpp +++ b/tools/ptoas/NativeModule.cpp @@ -8,6 +8,8 @@ #include "ptoas.h" +#include "PTOModule.h" + #include "pybind11/pybind11.h" #include "pybind11/stl.h" @@ -31,7 +33,9 @@ int runPTOASFromPython(const std::vector &arguments) { } // namespace -PYBIND11_MODULE(_native, module) { - module.doc() = "Native PTOAS command-line entrypoint"; +PYBIND11_MODULE(_core, module) { + module.doc() = "PTOAS compiler and PTO dialect native bindings"; + py::module_::import("mlir.ir"); + mlir::pto::python::populatePTODialectBindings(module); module.def("main", &runPTOASFromPython, py::arg("argv")); } diff --git a/tools/ptoas/ptoas_wrapper.py b/tools/ptoas/ptoas_wrapper.py index 07a8d423d7..3b1e96ad25 100644 --- a/tools/ptoas/ptoas_wrapper.py +++ b/tools/ptoas/ptoas_wrapper.py @@ -7,7 +7,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -"""Add a CMake tree's Python root, then run the regular PTOAS CLI module.""" +"""Run the regular PTOAS CLI from a configured CMake package tree.""" from __future__ import annotations @@ -15,6 +15,10 @@ from pathlib import Path +_PYTHON_ROOT_MODE = "@PTOAS_WRAPPER_PYTHON_ROOT_MODE@" +_PYTHON_ROOT = Path(r"@PTOAS_WRAPPER_PYTHON_ROOT@") + + def _resolve_wrapper_path(argv0: str | None = None) -> Path: candidate = argv0 if argv0 is not None else sys.argv[0] wrapper = Path(candidate) @@ -24,10 +28,6 @@ def _resolve_wrapper_path(argv0: str | None = None) -> Path: raise SystemExit(f"unable to locate the ptoas tree wrapper: {candidate}") -def _is_build_tree_wrapper(wrapper: Path) -> bool: - return len(wrapper.parents) >= 2 and wrapper.parents[1].name == "tools" - - def _require_python_root(python_root: Path, *, context: str) -> Path: helper = python_root / "ptoas" / "_cli.py" if helper.is_file(): @@ -38,19 +38,18 @@ def _require_python_root(python_root: Path, *, context: str) -> Path: ) -def _add_tree_python_root(wrapper: Path) -> Path: - if _is_build_tree_wrapper(wrapper): - if len(wrapper.parents) < 3: - raise SystemExit("unable to locate the build-tree root for ptoas") - python_root = _require_python_root( - wrapper.parents[2] / "python", context="build-tree" - ) - else: - if len(wrapper.parents) < 2: - raise SystemExit("unable to locate the install-tree root for ptoas") - python_root = _require_python_root( - wrapper.parents[1], context="install-tree" - ) +def _configured_python_root(wrapper: Path) -> Path: + if _PYTHON_ROOT_MODE == "absolute": + return _PYTHON_ROOT + if _PYTHON_ROOT_MODE == "wrapper-relative": + return wrapper.parent / _PYTHON_ROOT + raise SystemExit(f"unsupported PTOAS wrapper mode: {_PYTHON_ROOT_MODE}") + + +def _add_configured_python_root(wrapper: Path) -> Path: + python_root = _require_python_root( + _configured_python_root(wrapper), context="configured" + ) python_root_text = str(python_root) if python_root_text not in sys.path: @@ -60,7 +59,7 @@ def _add_tree_python_root(wrapper: Path) -> Path: def main() -> None: wrapper = _resolve_wrapper_path() - _add_tree_python_root(wrapper) + _add_configured_python_root(wrapper) from ptoas import _cli raise SystemExit(_cli.launch(sys.argv[1:], wrapper=wrapper)) From 5b0551f7bc1addb5c396d9ed3f82da3fb6b440d5 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Mon, 27 Jul 2026 11:43:52 +0800 Subject: [PATCH 13/32] build: clean up editable CI and wheel RPATHs --- .github/workflows/build_wheel.yml | 7 +++- .github/workflows/build_wheel_mac.yml | 6 ++- .github/workflows/ci.yml | 46 +++++++------------- .github/workflows/ci_sim.yml | 60 ++++++++++----------------- lib/Bindings/Python/CMakeLists.txt | 12 ++++++ tools/ptoas/CMakeLists.txt | 12 ++++++ 6 files changed, 72 insertions(+), 71 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 8952245bde..c309932bc5 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -231,7 +231,6 @@ jobs: python -m pip wheel "${PTOAS_WHEEL_SOURCE_DIR}" \ --no-build-isolation --no-deps \ --wheel-dir "${PTO_SOURCE_DIR}/build/wheel-dist" - cmake --install "${PTO_BUILD_DIR}" --prefix "${PTO_INSTALL_DIR}" - name: Validate wheel payload and launcher contract run: | @@ -241,8 +240,11 @@ jobs: run: | export PTO_WHEEL_DIST_DIR=$PTO_SOURCE_DIR/build/wheel-dist export PTO_WHEELHOUSE=$GITHUB_WORKSPACE/wheelhouse - export LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:$LD_LIBRARY_PATH mkdir -p "$PTO_WHEELHOUSE" + # The raw wheel intentionally keeps package-relative RPATHs. Give + # auditwheel the external LLVM build only while resolving and + # bundling its shared dependencies. + export LD_LIBRARY_PATH="${LLVM_BUILD_DIR}/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" auditwheel repair --plat manylinux_2_34_${{ matrix.arch }} "$PTO_WHEEL_DIST_DIR"/ptoas*.whl -w "$PTO_WHEELHOUSE" - name: Test wheel installation @@ -260,6 +262,7 @@ jobs: - name: Collect ptoas binary and dependencies if: matrix.python == '3.11' run: | + cmake --install "${PTO_BUILD_DIR}" --prefix "${PTO_INSTALL_DIR}" bash $PTO_SOURCE_DIR/docker/collect_ptoas_dist.sh $GITHUB_WORKSPACE/ptoas-dist - name: Archive ptoas binary artifact diff --git a/.github/workflows/build_wheel_mac.yml b/.github/workflows/build_wheel_mac.yml index d422cf433e..b3e96f8ff5 100644 --- a/.github/workflows/build_wheel_mac.yml +++ b/.github/workflows/build_wheel_mac.yml @@ -234,7 +234,6 @@ jobs: python -m pip wheel "${PTOAS_WHEEL_SOURCE_DIR}" \ --no-build-isolation --no-deps \ --wheel-dir "${PTO_SOURCE_DIR}/build/wheel-dist" - cmake --install "${PTO_BUILD_DIR}" --prefix "${PTO_INSTALL_DIR}" shopt -s nullglob built_wheels=("$PTO_SOURCE_DIR"/build/wheel-dist/ptoas*.whl) if [ "${#built_wheels[@]}" -gt 0 ]; then @@ -272,6 +271,10 @@ jobs: echo "delocate dependency summary:" delocate-listdeps "${wheels[0]}" || true + # The raw wheel intentionally keeps package-relative RPATHs. Give + # delocate the external LLVM build only while resolving and bundling + # its shared dependencies. + export DYLD_LIBRARY_PATH="${LLVM_BUILD_DIR}/lib${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}}" delocate-wheel \ --require-archs "${REQUIRED_ARCH}" \ --wheel-dir "$PTO_WHEELHOUSE" \ @@ -335,6 +338,7 @@ jobs: - name: Collect ptoas binary and dependencies if: matrix.python == '3.11' run: | + cmake --install "${PTO_BUILD_DIR}" --prefix "${PTO_INSTALL_DIR}" bash "$PTO_SOURCE_DIR/docker/collect_ptoas_dist_mac.sh" "$GITHUB_WORKSPACE/ptoas-dist" - name: Archive ptoas binary artifact diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 849a727468..f741ab16da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,9 +102,8 @@ jobs: LLVM_BUILD_DIR: ${{ github.workspace }}/llvm-project/llvm/build-assert LLVM_DIR: ${{ github.workspace }}/llvm-project/llvm/build-assert PTO_BUILD_DIR: ${{ github.workspace }}/build-assert - PTO_INSTALL_DIR: ${{ github.workspace }}/install-assert PTOAS_VENV: ${{ github.workspace }}/.venv-ptoas-assert - LLVM_CACHE_FLAVOR: llvm21-assert-shared-mlirpy-hardening-v1 + LLVM_CACHE_FLAVOR: llvm21-assert-shared-mlirpy-hardening-v2 steps: - name: Checkout uses: actions/checkout@v4 @@ -112,7 +111,7 @@ jobs: fetch-depth: 1 persist-credentials: false - - name: Install dependencies + - name: Install dependencies and create developer environment run: | # GitHub-hosted runners may include Microsoft apt sources unrelated to # PTOAS; these can intermittently return 403 and break apt-get update. @@ -129,9 +128,13 @@ jobs: python3 python3-pip python3-venv \ clang-${PTOAS_CLANG_MAJOR} lld-${PTOAS_CLANG_MAJOR} \ libedit-dev zlib1g-dev libxml2-dev libzstd-dev - python3 -m pip install --upgrade pip + rm -rf "${PTOAS_VENV}" + python3 -m venv "${PTOAS_VENV}" + "${PTOAS_VENV}/bin/python" -m pip install --upgrade pip # LLVM/MLIR Python bindings are not yet compatible with pybind11 3.x. - python3 -m pip install 'scikit-build-core>=0.12.2,<2' 'pybind11<3' nanobind numpy + "${PTOAS_VENV}/bin/pip" install \ + 'scikit-build-core>=0.12.2,<2' 'pybind11<3' \ + nanobind numpy ml-dtypes - name: Define payload paths shell: bash @@ -208,10 +211,10 @@ jobs: -DBUILD_SHARED_LIBS=ON \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DPython3_EXECUTABLE=python3 \ - -DPython_EXECUTABLE=python3 \ - -Dpybind11_DIR="$(python3 -m pybind11 --cmakedir)" \ - -Dnanobind_DIR="$(python3 -m nanobind --cmake_dir)" \ + -DPython3_EXECUTABLE="${PTOAS_VENV}/bin/python" \ + -DPython_EXECUTABLE="${PTOAS_VENV}/bin/python" \ + -Dpybind11_DIR="$("${PTOAS_VENV}/bin/python" -m pybind11 --cmakedir)" \ + -Dnanobind_DIR="$("${PTOAS_VENV}/bin/python" -m nanobind --cmake_dir)" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_COMPILER="${PTOAS_CMAKE_C_COMPILER}" \ -DCMAKE_CXX_COMPILER="${PTOAS_CMAKE_CXX_COMPILER}" \ @@ -258,20 +261,10 @@ jobs: -DCMAKE_CXX_COMPILER="${PTOAS_CMAKE_CXX_COMPILER}" \ -Werror=dev - - name: Create PTOAS developer environment - shell: bash - run: | - set -euo pipefail - rm -rf "${PTOAS_VENV}" - python3 -m venv "${PTOAS_VENV}" - "${PTOAS_VENV}/bin/python" -m pip install --upgrade pip - "${PTOAS_VENV}/bin/pip" install 'scikit-build-core>=0.12.2,<2' 'pybind11<3' nanobind numpy ml-dtypes - - name: Install PTOAS editable run: | set -euo pipefail rm -rf "${PTO_BUILD_DIR}" - rm -rf "${PTO_INSTALL_DIR}" # Keep a persistent CMake build tree for CTest and lit while exposing # the current sources and Python extension through the developer venv. LLVM_BUILD_DIR="${LLVM_DIR}" \ @@ -280,7 +273,6 @@ jobs: "${PTOAS_VENV}/bin/python" -m pip install --editable . \ --no-build-isolation --no-deps \ --config-settings="build-dir=${PTO_BUILD_DIR}" - cmake --install "${PTO_BUILD_DIR}" --prefix "${PTO_INSTALL_DIR}" - name: Test PTOAS editable install shell: bash @@ -294,20 +286,16 @@ jobs: shell: bash env: LLVM_DIR: ${{ env.LLVM_DIR }} - PTO_INSTALL_DIR: ${{ env.PTO_INSTALL_DIR }} run: | - export PATH="${PTOAS_VENV}/bin:${LLVM_DIR}/bin:${PTO_INSTALL_DIR}/bin:${PATH}" - export LD_LIBRARY_PATH="${LLVM_DIR}/lib:${PTO_INSTALL_DIR}/lib:${LD_LIBRARY_PATH:-}" + export PATH="${PTOAS_VENV}/bin:${LLVM_DIR}/bin:${PATH}" ctest --test-dir "${PTO_BUILD_DIR}" --output-on-failure -L PTODSL - name: Run lit tests shell: bash env: LLVM_DIR: ${{ env.LLVM_DIR }} - PTO_INSTALL_DIR: ${{ env.PTO_INSTALL_DIR }} run: | - export PATH="${PTOAS_VENV}/bin:${LLVM_DIR}/bin:${PTO_INSTALL_DIR}/bin:${PATH}" - export LD_LIBRARY_PATH="${LLVM_DIR}/lib:${PTO_INSTALL_DIR}/lib:${LD_LIBRARY_PATH:-}" + export PATH="${PTOAS_VENV}/bin:${LLVM_DIR}/bin:${PATH}" ninja -C "${PTO_BUILD_DIR}" check-pto - name: Run sample tests (py -> pto -> cpp) @@ -315,15 +303,13 @@ jobs: env: CI_EVENT_NAME: ${{ github.event_name }} WORKFLOW_SOC_VERSION: ${{ github.event.inputs.soc_version || 'Ascend910' }} - PTOAS_BIN: ${{ env.PTO_INSTALL_DIR }}/bin/ptoas + PTOAS_BIN: ${{ env.PTOAS_VENV }}/bin/ptoas PTOBC_BIN: ${{ env.PTO_BUILD_DIR }}/tools/ptobc/ptobc PTO_BUILD_DIR: ${{ env.PTO_BUILD_DIR }} PYTHON_BIN: ${{ env.PTOAS_VENV }}/bin/python - PTO_INSTALL_DIR: ${{ env.PTO_INSTALL_DIR }} run: | set -euo pipefail - export PATH="${PTOAS_VENV}/bin:${PTO_INSTALL_DIR}/bin:${PATH}" - export LD_LIBRARY_PATH="${LLVM_DIR}/lib:${PTO_INSTALL_DIR}/lib:${LD_LIBRARY_PATH:-}" + export PATH="${PTOAS_VENV}/bin:${PATH}" export PTOAS_OUT_DIR="${PAYLOAD_DIR}/test/samples" if [[ "${CI_EVENT_NAME}" == "workflow_dispatch" || "${CI_EVENT_NAME}" == "schedule" ]]; then # Board-validation payloads must only contain the arch-matching diff --git a/.github/workflows/ci_sim.yml b/.github/workflows/ci_sim.yml index 32a2eab474..b87084d438 100644 --- a/.github/workflows/ci_sim.yml +++ b/.github/workflows/ci_sim.yml @@ -111,7 +111,6 @@ jobs: env: LLVM_REPO: https://github.com/vpto-dev/llvm-project.git LLVM_REF: feature-vpto-llvm21 - PTO_INSTALL_DIR: ${{ github.workspace }}/install PTOAS_VENV: ${{ github.workspace }}/.work/ptoas-sim-venv VPTO_SIM_WORKSPACE: ${{ github.workspace }}/.work/vpto-sim-ci TILELANG_DSL_WORKSPACE: ${{ github.workspace }}/.work/tilelang-dsl-ci @@ -154,7 +153,7 @@ jobs: fi echo "ASCEND_HOME_PATH=${ASCEND_HOME_PATH_DETECTED}" >> "${GITHUB_ENV}" - echo "PTOAS_BIN=${GITHUB_WORKSPACE}/install/bin/ptoas" >> "${GITHUB_ENV}" + echo "PTOAS_BIN=${PTOAS_VENV}/bin/ptoas" >> "${GITHUB_ENV}" - name: Ensure runner dependencies shell: bash @@ -170,7 +169,7 @@ jobs: if [[ "${#missing_tools[@]}" -gt 0 ]]; then if command -v sudo >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then sudo apt-get update - sudo apt-get install -y python3 python3-pip git cmake ninja-build make + sudo apt-get install -y python3 python3-pip python3-venv git cmake ninja-build make else echo "ERROR: missing required tools on self-hosted runner: ${missing_tools[*]}" >&2 echo "ERROR: automatic installation requires sudo + apt-get" >&2 @@ -178,26 +177,14 @@ jobs: fi fi - python3 -m pip --version >/dev/null 2>&1 || { + if ! python3 -m ensurepip --version >/dev/null 2>&1; then if command -v sudo >/dev/null 2>&1 && command -v apt-get >/dev/null 2>&1; then sudo apt-get update - sudo apt-get install -y python3-pip + sudo apt-get install -y python3-pip python3-venv else - echo "ERROR: python3-pip is required on self-hosted runner" >&2 + echo "ERROR: python3-venv is required on self-hosted runner" >&2 exit 1 fi - } - - need_pip_install=0 - python3 -c "import numpy" >/dev/null 2>&1 || need_pip_install=1 - python3 -c "import scikit_build_core" >/dev/null 2>&1 || need_pip_install=1 - python3 -m pybind11 --cmakedir >/dev/null 2>&1 || need_pip_install=1 - python3 -m nanobind --cmake_dir >/dev/null 2>&1 || need_pip_install=1 - python3 -c "import ml_dtypes" >/dev/null 2>&1 || need_pip_install=1 - - if [[ "${need_pip_install}" -eq 1 ]]; then - python3 -m pip install --upgrade pip - python3 -m pip install 'scikit-build-core>=0.12.2,<2' 'pybind11<3' nanobind numpy ml-dtypes fi - name: Configure ccache @@ -230,7 +217,6 @@ jobs: run: | set -euo pipefail echo "PTO_DSL_ST_BUILD_DIR=${GITHUB_WORKSPACE}/build-ptodsl" >> "${GITHUB_ENV}" - echo "PTO_DSL_ST_INSTALL_DIR=${GITHUB_WORKSPACE}/install-ptodsl" >> "${GITHUB_ENV}" - name: Resolve LLVM cache key id: llvm-cache-key @@ -246,7 +232,7 @@ jobs: exit 1 fi echo "sha=${sha}" >> "${GITHUB_OUTPUT}" - echo "key=llvm-build-${sha}-assert-v1" >> "${GITHUB_OUTPUT}" + echo "key=llvm-build-${sha}-assert-v2" >> "${GITHUB_OUTPUT}" - name: Restore LLVM build cache id: llvm-cache @@ -262,8 +248,6 @@ jobs: set -euo pipefail rm -rf "${GITHUB_WORKSPACE}/build" rm -rf "${GITHUB_WORKSPACE}/build-ptodsl" - rm -rf "${PTO_INSTALL_DIR}" - rm -rf "${PTO_DSL_ST_INSTALL_DIR}" rm -rf "${PTOAS_VENV}" rm -rf "${VPTO_SIM_WORKSPACE}" rm -rf "${TILELANG_DSL_WORKSPACE}" @@ -272,6 +256,16 @@ jobs: rm -rf "${PTO_ISA_ROOT}" rm -rf "${GITHUB_WORKSPACE}/.work/camodel" + - name: Create PTOAS simulator environment + shell: bash + run: | + set -euo pipefail + python3 -m venv "${PTOAS_VENV}" + "${PTOAS_VENV}/bin/python" -m pip install --upgrade pip + "${PTOAS_VENV}/bin/pip" install \ + 'scikit-build-core>=0.12.2,<2' 'pybind11<3' \ + nanobind numpy ml-dtypes + - name: Prepare LLVM source shell: bash run: | @@ -309,10 +303,10 @@ jobs: -DBUILD_SHARED_LIBS=ON \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DPython3_EXECUTABLE=python3 \ - -DPython_EXECUTABLE=python3 \ - -Dpybind11_DIR="$(python3 -m pybind11 --cmakedir)" \ - -Dnanobind_DIR="$(python3 -m nanobind --cmake_dir)" \ + -DPython3_EXECUTABLE="${PTOAS_VENV}/bin/python" \ + -DPython_EXECUTABLE="${PTOAS_VENV}/bin/python" \ + -Dpybind11_DIR="$("${PTOAS_VENV}/bin/python" -m pybind11 --cmakedir)" \ + -Dnanobind_DIR="$("${PTOAS_VENV}/bin/python" -m nanobind --cmake_dir)" \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_TARGETS_TO_BUILD="host" @@ -326,17 +320,10 @@ jobs: path: ${{ env.LLVM_DIR }} key: ${{ steps.llvm-cache-key.outputs.key }} - - name: Install PTOAS editable in simulator venv + - name: Install PTOAS editable shell: bash run: | set -euo pipefail - rm -rf "${PTO_INSTALL_DIR}" - rm -rf "${PTOAS_VENV}" - python3 -m venv "${PTOAS_VENV}" - "${PTOAS_VENV}/bin/python" -m pip install --upgrade pip - "${PTOAS_VENV}/bin/pip" install \ - 'scikit-build-core>=0.12.2,<2' 'pybind11<3' \ - nanobind numpy ml-dtypes # Pin system compilers for the same reason as LLVM above. export CC=gcc export CXX=g++ @@ -344,7 +331,6 @@ jobs: "${PTOAS_VENV}/bin/python" -m pip install --editable . \ --no-build-isolation --no-deps \ --config-settings="build-dir=${GITHUB_WORKSPACE}/build" - cmake --install "${GITHUB_WORKSPACE}/build" --prefix "${PTO_INSTALL_DIR}" PATH="${PTOAS_VENV}/bin:${PATH}" \ PYTHON="${PTOAS_VENV}/bin/python" \ bash docker/test_wheel_imports.sh @@ -799,14 +785,13 @@ jobs: shell: bash run: | set -euo pipefail - rm -rf "${PTO_DSL_ST_BUILD_DIR}" "${PTO_DSL_ST_INSTALL_DIR}" + rm -rf "${PTO_DSL_ST_BUILD_DIR}" export CC=gcc export CXX=g++ LLVM_BUILD_DIR="${PTO_DSL_ST_LLVM_DIR}" \ "${PTO_DSL_ST_PYTHON_BIN}" -m pip install --editable . \ --no-build-isolation --no-deps \ --config-settings="build-dir=${PTO_DSL_ST_BUILD_DIR}" - cmake --install "${PTO_DSL_ST_BUILD_DIR}" --prefix "${PTO_DSL_ST_INSTALL_DIR}" - name: Probe PTODSL runtime Python shell: bash @@ -935,7 +920,6 @@ jobs: set -euo pipefail mkdir -p "${TILELANG_DSL_UT_WORKSPACE}" export PATH="${PTOAS_VENV}/bin:${PATH}" - export LD_LIBRARY_PATH="${LLVM_DIR}/lib:${PTO_INSTALL_DIR}/lib:${GITHUB_WORKSPACE}/build/lib:${LD_LIBRARY_PATH:-}" cd tilelang-dsl/python python3 -m unittest discover -s ../tests -p 'test_*.py' \ 2>&1 | tee "${TILELANG_DSL_UT_WORKSPACE}/unittest.log" diff --git a/lib/Bindings/Python/CMakeLists.txt b/lib/Bindings/Python/CMakeLists.txt index c0bd2b19d4..ba950c649b 100644 --- a/lib/Bindings/Python/CMakeLists.txt +++ b/lib/Bindings/Python/CMakeLists.txt @@ -8,6 +8,18 @@ include(AddMLIRPython) +# Do not carry toolchain-injected paths, such as a Conda compiler RPATH, into +# installed Python modules. Distribution builds keep only MLIR's relative +# package RPATHs; editable builds add their external link paths below. +set(CMAKE_INSTALL_REMOVE_ENVIRONMENT_RPATH ON) + +# scikit-build-core editable installs use the external LLVM build directly. +# Keep this developer-only fallback local to the MLIR Python targets; wheel +# builds use package-relative RPATHs and let auditwheel/delocate collect LLVM. +if(SKBUILD_STATE STREQUAL "editable") + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON) +endif() + # This directory materializes C++ sources exported by upstream MLIR. PTOAS # keeps -Werror enabled globally, but upstream sources are not part of the # project warning contract. Scope the exception to this directory; the diff --git a/tools/ptoas/CMakeLists.txt b/tools/ptoas/CMakeLists.txt index 0ec5afe766..c0fb306764 100644 --- a/tools/ptoas/CMakeLists.txt +++ b/tools/ptoas/CMakeLists.txt @@ -6,6 +6,18 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. +# Do not carry toolchain-injected paths, such as a Conda compiler RPATH, into +# installed Python modules. Distribution builds keep only MLIR's relative +# package RPATHs; editable builds add their external link paths below. +set(CMAKE_INSTALL_REMOVE_ENVIRONMENT_RPATH ON) + +# scikit-build-core editable installs use the external LLVM build directly. +# Keep this developer-only fallback local to the PTOAS Python extension; wheel +# builds use package-relative RPATHs and let auditwheel/delocate collect LLVM. +if(SKBUILD_STATE STREQUAL "editable") + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON) +endif() + set(PTOAS_RUNTIME_SOURCES ptoas.cpp driver.cpp From 0258273d00424cd58110db7b02a1819da95b7423 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Mon, 27 Jul 2026 12:48:33 +0800 Subject: [PATCH 14/32] build: retain ccache in wheel environments --- .github/workflows/build_wheel.yml | 2 +- .github/workflows/build_wheel_mac.yml | 2 +- docker/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index c309932bc5..11ff6ee9ef 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -129,7 +129,7 @@ jobs: - name: Install system dependencies run: | - dnf install -y ninja-build cmake git gcc-c++ lld zip binutils patchelf chrpath + dnf install -y ninja-build cmake git ccache gcc-c++ lld zip binutils patchelf chrpath dnf clean all - name: Install Python dependencies diff --git a/.github/workflows/build_wheel_mac.yml b/.github/workflows/build_wheel_mac.yml index b3e96f8ff5..b047c1b280 100644 --- a/.github/workflows/build_wheel_mac.yml +++ b/.github/workflows/build_wheel_mac.yml @@ -129,7 +129,7 @@ jobs: - name: Install system dependencies run: | - brew install ninja + brew install ninja ccache - name: Install Python dependencies run: | diff --git a/docker/Dockerfile b/docker/Dockerfile index 3bdd056ba4..8a46534173 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,7 +19,7 @@ ENV PY_PATH="/opt/python/${PY_VER}" ENV PATH="${PY_PATH}/bin:${PATH}" # dependency -RUN dnf install -y ninja-build cmake git gcc-c++ lld zip binutils patchelf chrpath && dnf clean all +RUN dnf install -y ninja-build cmake git ccache gcc-c++ lld zip binutils patchelf chrpath && dnf clean all RUN pip install --no-cache-dir numpy 'pybind11<3' nanobind 'scikit-build-core>=0.12.2,<2' auditwheel COPY cmake/LinuxHardeningCache.cmake /tmp/LinuxHardeningCache.cmake From aa1d92e6f82efd27a1a06c87e286cf0153b5df15 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Mon, 27 Jul 2026 20:59:28 +0800 Subject: [PATCH 15/32] build: isolate MLIR Python packaging --- .codex/skills/build-ptoas-wsl/SKILL.md | 6 +- .github/workflows/build_wheel.yml | 35 +- .github/workflows/build_wheel_mac.yml | 15 +- README.md | 10 +- README_en.md | 8 +- cmake/RelocatePTOASCompilerArchive.cmake.in | 190 +++++ docker/collect_ptoas_dist.sh | 255 ------- docker/collect_ptoas_dist_mac.sh | 378 ---------- docker/test_wheel_imports.sh | 20 +- docker/validate_wheel_payload.py | 33 + docs/build_with_installed_llvm.md | 2 +- ...oas-mlir-namespace-and-native-isolation.md | 692 ++++++++++++++++++ docs/designs/ptoas-python-launcher-layout.md | 29 +- lib/Bindings/Python/CMakeLists.txt | 17 +- lib/Bindings/Python/PTOModule.cpp | 2 +- ptodsl/CMakeLists.txt | 4 +- ptodsl/examples/softmax_lowlevel.py | 4 +- ptodsl/examples/tadd_lowlevel.py | 4 +- ptodsl/ptodsl/_allreduce.py | 4 +- ptodsl/ptodsl/_builtin_vector.py | 6 +- ptodsl/ptodsl/_context.py | 6 +- ptodsl/ptodsl/_control_flow.py | 4 +- ptodsl/ptodsl/_jit.py | 2 +- ptodsl/ptodsl/_ops.py | 6 +- ptodsl/ptodsl/_runtime/codegen.py | 2 +- ptodsl/ptodsl/_runtime/launch.py | 2 +- ptodsl/ptodsl/_runtime_scalar_ops.py | 4 +- ptodsl/ptodsl/_scalar_adaptation.py | 4 +- ptodsl/ptodsl/_source_loader.py | 2 +- ptodsl/ptodsl/_surface_types.py | 2 +- ptodsl/ptodsl/_surface_values.py | 8 +- ptodsl/ptodsl/_tile_template_tracing.py | 4 +- ptodsl/ptodsl/_tracing/artifacts.py | 4 +- ptodsl/ptodsl/_tracing/control_flow.py | 6 +- ptodsl/ptodsl/_tracing/module_builder.py | 4 +- ptodsl/ptodsl/_tracing/runtime.py | 4 +- ptodsl/ptodsl/_tracing/session.py | 6 +- ptodsl/ptodsl/_types.py | 14 +- ptodsl/ptodsl/_vmi_namespace.py | 4 +- ptodsl/ptodsl/pto.py | 2 +- ptodsl/ptodsl/scalar.py | 12 +- ptodsl/ptodsl/tilelib/_render_runtime.py | 4 +- ptodsl/ptodsl/tilelib/metadata.py | 2 +- ptodsl/ptodsl/tilelib/templates/a5/tcvt.py | 2 +- .../tests/support/docs_fragment_fixtures.py | 8 +- ptodsl/tests/test_ast_rewrite_example_ir.py | 2 +- ptodsl/tests/test_docs_as_test.py | 2 +- .../test_flash_attention_demo_compile.py | 2 +- ptodsl/tests/test_jit_compile.py | 2 +- ptodsl/tests/test_ptoas_frontend_verify.py | 2 +- ptodsl/tests/test_ptoas_tree_wrapper.py | 25 + ptodsl/tests/test_tileop.py | 2 +- ptodsl/tests/test_vector_cube_ops.py | 2 +- python/pto/dialects/pto.py | 8 +- scripts/sim_dsl.sh | 4 +- test/dsl/abs.py | 2 +- test/dsl/strict_vecscope.py | 2 +- test/dsl/template_abs.py | 2 +- test/python/compact_tile_buf_asm.py | 4 +- test/python/low_precision_types.py | 4 +- test/python/memory_consistency_bindings.py | 4 +- test/samples/Abs/abs.py | 6 +- test/samples/AddPtr/addptr.py | 6 +- test/samples/AddPtr/addptr_chain.py | 6 +- test/samples/AddPtr/addptr_dynamic.py | 6 +- test/samples/AddPtr/addptr_f16.py | 6 +- test/samples/Addc/addc.py | 6 +- test/samples/Adds/adds.py | 6 +- test/samples/Addsc/addsc.py | 6 +- test/samples/AllocTile/alloc_tile_addr.py | 6 +- test/samples/And/and.py | 6 +- test/samples/Ands/ands.py | 6 +- test/samples/AsyncComm/async_comm.py | 4 +- .../AsyncComm/tget_async_kernel_impl_like.py | 4 +- .../AsyncComm/tput_async_kernel_impl_like.py | 4 +- test/samples/Bf16/bf16_tile.py | 4 +- test/samples/Ci/ci.py | 6 +- test/samples/Cmp/cmp.py | 6 +- test/samples/Cmps/cmps.py | 6 +- test/samples/Colexpand/colexpand.py | 6 +- test/samples/Colexpandadd/colexpandadd.py | 6 +- test/samples/Colexpanddiv/colexpanddiv.py | 6 +- ...colexpanddiv_src1_vcol_mismatch_invalid.py | 6 +- .../Colexpandexpdif/colexpandexpdif.py | 6 +- .../colexpandexpdif_dtype_invalid.py | 6 +- test/samples/Colexpandmax/colexpandmax.py | 6 +- test/samples/Colexpandmin/colexpandmin.py | 6 +- test/samples/Colexpandmul/colexpandmul.py | 6 +- test/samples/Colexpandsub/colexpandsub.py | 6 +- .../colexpandsub_src0_layout_invalid.py | 6 +- test/samples/Colmax/colmax.py | 6 +- test/samples/Colmin/colmin.py | 6 +- test/samples/Colprod/colprod.py | 6 +- test/samples/Colsum/colsum.py | 6 +- test/samples/CommSync/comm_collective.py | 4 +- .../comm_collective_binding_variants.py | 4 +- test/samples/CommSync/comm_p2p.py | 4 +- .../CommSync/comm_p2p_binding_variants.py | 4 +- .../CommSync/tbroadcast_root_binding.py | 4 +- test/samples/CommSync/tgather_root_binding.py | 4 +- .../CommSync/tnotify_atomic_add_binding.py | 4 +- test/samples/CommSync/treduce_root_binding.py | 4 +- .../samples/CommSync/tscatter_root_binding.py | 4 +- test/samples/CommSync/twait_atomic_binding.py | 4 +- test/samples/Complex/cv_region.py | 4 +- test/samples/Complex/mix_kernel.py | 6 +- test/samples/Concat/concat.py | 6 +- test/samples/Cvt/cvt_f32_f16.py | 4 +- test/samples/Cvt/cvt_f32_f32.py | 6 +- test/samples/Cvt/tcvt.py | 4 +- test/samples/Dequant/dequant.py | 4 +- test/samples/Dequant/dequant_i8.py | 4 +- test/samples/Div/div.py | 6 +- test/samples/Divs/divs.py | 6 +- test/samples/Divs2/divs2.py | 6 +- .../DynamicTailMatmul/dynamic_tail_matmul.py | 4 +- test/samples/Exp/exp.py | 6 +- test/samples/Expands/expand.py | 6 +- test/samples/Extract/extract.py | 6 +- test/samples/Extract/extract_fp.py | 6 +- test/samples/Fillpad/fillpad.py | 6 +- test/samples/Fillpad/fillpad_expand.py | 6 +- .../samples/Fillpad/fillpad_expand_invalid.py | 6 +- .../fillpad_expand_pad_null_invalid.py | 6 +- test/samples/Fillpad/fillpad_inplace.py | 6 +- test/samples/Fmod/fmod.py | 6 +- test/samples/Fmods/fmods.py | 6 +- .../Fmods/tfmods_scalar_type_invalid.py | 6 +- test/samples/Gather/gather.py | 6 +- test/samples/Gather/gather_legacy.py | 6 +- test/samples/Gatherb/gatherb.py | 4 +- test/samples/Gemv/gemv.py | 6 +- test/samples/Gemv/gemvacc.py | 6 +- test/samples/Gemv/gemvbias.py | 6 +- test/samples/Gemvmx/gemvmx.py | 6 +- .../Layout/tensor_view_infer_layout_dn.py | 6 +- test/samples/Layout/tensor_view_layout_dn.py | 4 +- .../LayoutInference/layout_inference.py | 4 +- test/samples/Log/log.py | 6 +- test/samples/Lrelu/lrelu.py | 6 +- test/samples/MatMul/tmatmulk.py | 8 +- .../matmul_mx_low_precision.py | 4 +- .../Matmul_transpose/Matmul_transpose.py | 6 +- test/samples/Max/max.py | 6 +- test/samples/Maxs/maxs.py | 6 +- test/samples/Mgather/mgather.py | 6 +- test/samples/Min/min.py | 6 +- test/samples/Mins/mins.py | 6 +- test/samples/Movfp/movfp.py | 6 +- test/samples/Mrgsort/mrgsort.py | 6 +- test/samples/Mrgsort/mrgsort_a5.py | 6 +- test/samples/Mrgsort/mrgsort_format2.py | 8 +- test/samples/Mscatter/mscatter.py | 6 +- test/samples/Mul/mul.py | 6 +- test/samples/Muls/muls.py | 6 +- test/samples/Neg/neg.py | 6 +- test/samples/Not/not.py | 6 +- test/samples/Or/or.py | 6 +- test/samples/Ors/ors.py | 6 +- test/samples/Partadd/partadd.py | 6 +- test/samples/Partarg/partarg.py | 4 +- test/samples/Partition5D/partition5d.py | 4 +- test/samples/Partition5D/partition5d_a5.py | 4 +- test/samples/Partmax/partmax.py | 6 +- test/samples/Partmin/partmin.py | 6 +- test/samples/Partmul/partmul.py | 6 +- test/samples/Prelu/prelu.py | 6 +- test/samples/Print/print.py | 6 +- test/samples/Print/print_scalar.py | 6 +- test/samples/Quant/quant.py | 4 +- test/samples/Quant/quant_asym.py | 4 +- test/samples/Recip/recip.py | 6 +- test/samples/Relu/relu.py | 6 +- test/samples/Rem/rem.py | 6 +- test/samples/Rems/rems.py | 6 +- test/samples/Reshape/bitcast_dtype_alias.py | 6 +- test/samples/Reshape/bitcast_inplace_cvt.py | 6 +- .../Reshape/bitcast_same_dtype_invalid.py | 6 +- test/samples/Reshape/reshape.py | 6 +- test/samples/Rowexpand/rowexpand.py | 6 +- test/samples/Rowexpandadd/rowexpandadd.py | 6 +- .../rowexpandadd_src1_width_invalid.py | 6 +- test/samples/Rowexpanddiv/rowexpanddiv.py | 6 +- .../Rowexpandexpdif/rowexpandexpdif.py | 6 +- .../rowexpandexpdif_dtype_invalid.py | 6 +- test/samples/Rowexpandmax/rowexpandmax.py | 6 +- .../rowexpandmax_a5_tmp_invalid.py | 6 +- test/samples/Rowexpandmin/rowexpandmin.py | 6 +- test/samples/Rowexpandmul/rowexpandmul.py | 6 +- test/samples/Rowexpandsub/rowexpandsub.py | 6 +- test/samples/Rowmax/rowmax.py | 6 +- test/samples/Rowmin/rowmin.py | 6 +- test/samples/Rowprod/rowprod.py | 6 +- test/samples/Rowsum/rowsum.py | 6 +- test/samples/Rsqrt/rsqrt.py | 6 +- test/samples/ScalarPtr/scalar_ptr.py | 6 +- test/samples/Scatter/scatter.py | 4 +- test/samples/Sel/sel.py | 6 +- test/samples/Sels/sels.py | 6 +- test/samples/SetValidShape/set_validshape.py | 6 +- test/samples/Shl/shl.py | 6 +- test/samples/Shls/shls.py | 6 +- test/samples/Shr/shr.py | 6 +- test/samples/Shrs/shrs.py | 6 +- test/samples/Sort32/sort32.py | 6 +- test/samples/Sqrt/sqrt.py | 6 +- test/samples/Storefp/storefp.py | 6 +- test/samples/Storefp/storefp_invalid.py | 6 +- test/samples/Sub/sub.py | 6 +- test/samples/SubView/subview.py | 6 +- test/samples/SubView/subview_boxed_dynamic.py | 6 +- test/samples/SubView/subview_boxed_invalid.py | 6 +- test/samples/SubView/subview_tsubs.py | 6 +- test/samples/SubView/vadd_pto_pingpong.py | 6 +- test/samples/Subc/subc.py | 6 +- test/samples/Subs/subs.py | 6 +- test/samples/Subsc/subsc.py | 6 +- test/samples/Sync/sync.py | 6 +- test/samples/Sync/syncHigh.py | 6 +- test/samples/Sync/test_a5_buf_sync.py | 4 +- test/samples/Sync/test_inject_sync_if.py | 4 +- test/samples/Sync/test_inject_sync_if_else.py | 4 +- .../test_inject_sync_intra_pipe_barrier.py | 6 +- test/samples/Sync/test_inject_sync_loop.py | 6 +- .../Sync/test_inject_sync_loop_nest.py | 6 +- .../Sync/test_inject_sync_two_event_id.py | 4 +- test/samples/Sync/test_intercore_sync_a3.py | 4 +- .../Sync/test_intercore_sync_a3_dyn.py | 4 +- .../test_intercore_sync_a3_missing_setffts.py | 4 +- .../Sync/test_intercore_sync_a3_modes.py | 4 +- test/samples/Sync/test_intercore_sync_a5.py | 4 +- .../Sync/test_intercore_sync_a5_dyn.py | 4 +- .../Sync/test_intercore_sync_a5_functional.py | 4 +- .../Sync/test_intercore_sync_a5_ptoisa_vec.py | 4 +- .../Sync/test_mem_inject_sync_basic.py | 4 +- .../samples/Sync/test_set_wait_unified_api.py | 6 +- test/samples/Sync/tmatmulk_autosync.py | 6 +- test/samples/Sync/tmatmulk_autosync_a5.py | 6 +- test/samples/SyncAll/syncall_binding.py | 4 +- test/samples/TInsert/tinsert.py | 6 +- test/samples/TInsert/tinsert_fp.py | 6 +- test/samples/TPrefetch/tprefetch.py | 4 +- .../TPrefetchAsync/tprefetch_async_binding.py | 4 +- test/samples/TTri/ttri.py | 4 +- .../TileSetGetValue/tileSetGetValue.py | 6 +- .../tile_getval_mat_invalid.py | 6 +- test/samples/Tpow/tpow.py | 6 +- test/samples/Tpows/tpows.py | 6 +- test/samples/Trans/trans.py | 6 +- test/samples/Trap/trap.py | 4 +- test/samples/VectorAddition/vadd_pto_ir.py | 6 +- .../samples/VectorAddition/vadd_validshape.py | 6 +- .../VectorAddition/vadd_validshape_dynamic.py | 6 +- .../VectorAddition/vadd_validshape_hyper.py | 6 +- test/samples/Xor/xor.py | 6 +- test/samples/Xors/xors.py | 6 +- test/tilelib-st/README.md | 2 +- tilelang-dsl/CMakeLists.txt | 4 +- .../python/tilelang_dsl/env_config.py | 6 +- .../python/tilelang_dsl/lowering_backend.py | 12 +- .../python/tilelang_dsl/pybind_renderer.py | 18 +- .../tests/backend/test_e2e_pybind_backend.py | 2 +- tools/ptoas/CMakeLists.txt | 36 +- tools/ptoas/NativeModule.cpp | 2 +- tools/ptoas/ptoas_wrapper.py | 13 + 265 files changed, 1748 insertions(+), 1326 deletions(-) create mode 100644 cmake/RelocatePTOASCompilerArchive.cmake.in delete mode 100755 docker/collect_ptoas_dist.sh delete mode 100644 docker/collect_ptoas_dist_mac.sh create mode 100644 docs/designs/ptoas-mlir-namespace-and-native-isolation.md diff --git a/.codex/skills/build-ptoas-wsl/SKILL.md b/.codex/skills/build-ptoas-wsl/SKILL.md index dabd88045d..a35773f03c 100644 --- a/.codex/skills/build-ptoas-wsl/SKILL.md +++ b/.codex/skills/build-ptoas-wsl/SKILL.md @@ -160,8 +160,8 @@ Validate Python dialect loading: ```bash python3 - <<'PY' -from mlir.ir import Context, Module, Location -from mlir.dialects import pto +from ptoas.mlir.ir import Context, Module, Location +from ptoas.mlir.dialects import pto with Context() as ctx, Location.unknown(): pto.register_dialect(ctx, load=True) @@ -174,6 +174,6 @@ PY - `def_property family does not currently support keep_alive`: reinstall `pybind11==2.12.0`, clear the affected CMake cache if needed, and reconfigure. - CMake cannot find LLVM or MLIR: confirm `LLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm` and `MLIR_DIR=$LLVM_BUILD_DIR/lib/cmake/mlir` exist and were produced by the `llvmorg-19.1.7` build. -- Python cannot import `mlir.dialects.pto`: re-export `PYTHONPATH` with MLIR core first and PTO install second, and confirm `_pto.cpython-*.so` exists under MLIR's `_mlir_libs`. +- Python cannot import `ptoas.mlir.dialects.pto`: make sure the PTOAS build/install Python root is on `PYTHONPATH`, and confirm `_core.cpython-*.so` and `ptoas/mlir/_mlir_libs/_mlir*.so` exist in the same package tree. - Runtime linker errors for LLVM or PTO libraries: re-export `LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:$LD_LIBRARY_PATH`. - Builds are very slow under `/mnt/c`: move the workspace into the WSL Linux filesystem, for example `$HOME/llvm-workspace`. diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 11ff6ee9ef..a047da35d6 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -150,7 +150,6 @@ jobs: echo "LLVM_BUILD_DIR=/llvm-workspace/llvm-project/build-release" >> $GITHUB_ENV echo "PTO_SOURCE_DIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV echo "PTO_BUILD_DIR=$GITHUB_WORKSPACE/build-release" >> $GITHUB_ENV - echo "PTO_INSTALL_DIR=$GITHUB_WORKSPACE/install-release" >> $GITHUB_ENV - name: Resolve LLVM source SHA id: llvm-source @@ -262,8 +261,13 @@ jobs: - name: Collect ptoas binary and dependencies if: matrix.python == '3.11' run: | - cmake --install "${PTO_BUILD_DIR}" --prefix "${PTO_INSTALL_DIR}" - bash $PTO_SOURCE_DIR/docker/collect_ptoas_dist.sh $GITHUB_WORKSPACE/ptoas-dist + rm -rf "$GITHUB_WORKSPACE/ptoas-dist" + cmake --install "${PTO_BUILD_DIR}" \ + --prefix "$GITHUB_WORKSPACE/ptoas-dist" \ + --component PTOAS_Python + cmake --install "${PTO_BUILD_DIR}" \ + --prefix "$GITHUB_WORKSPACE/ptoas-dist" \ + --component PTOAS_CompilerArchive - name: Archive ptoas binary artifact if: matrix.python == '3.11' @@ -272,6 +276,31 @@ jobs: tar -czf "$GITHUB_WORKSPACE/ptoas-bin-${{ matrix.arch }}.tar.gz" \ -C "$GITHUB_WORKSPACE/ptoas-dist" . + - name: Smoke test archived ptoas binary artifact + if: matrix.python == '3.11' + run: | + TEST_DIR="$RUNNER_TEMP/ptoas-dist-smoke-${{ matrix.arch }}" + rm -rf "$TEST_DIR" + mkdir -p "$TEST_DIR/extracted" + tar -xzf "$GITHUB_WORKSPACE/ptoas-bin-${{ matrix.arch }}.tar.gz" \ + -C "$TEST_DIR/extracted" + + test -x "$TEST_DIR/extracted/bin/ptoas" + test -f "$TEST_DIR/extracted/ptoas/_cli.py" + test -f "$TEST_DIR/extracted/ptoas/mlir/ir.py" + test -f "$TEST_DIR/extracted/ptodsl/__init__.py" + test -f "$TEST_DIR/extracted/tilelang_dsl/__init__.py" + test -d "$TEST_DIR/extracted/share/ptoas/TileOps" + test "$(cat "$TEST_DIR/extracted/.ptoas-python-version")" = "3.11" + compgen -G "$TEST_DIR/extracted/ptoas/_core*.so" >/dev/null + + env -u PYTHONPATH -u DYLD_LIBRARY_PATH -u LD_LIBRARY_PATH \ + "$TEST_DIR/extracted/bin/ptoas" --version + env -u PYTHONPATH -u DYLD_LIBRARY_PATH -u LD_LIBRARY_PATH \ + "$TEST_DIR/extracted/bin/ptoas" \ + "$PTO_SOURCE_DIR/test/lit/pto/kernel_kind_vector_scf_while_emitc.pto" \ + >/dev/null + - name: Upload ptoas binary artifact if: matrix.python == '3.11' uses: actions/upload-artifact@v4 diff --git a/.github/workflows/build_wheel_mac.yml b/.github/workflows/build_wheel_mac.yml index b047c1b280..1158508314 100644 --- a/.github/workflows/build_wheel_mac.yml +++ b/.github/workflows/build_wheel_mac.yml @@ -149,7 +149,6 @@ jobs: echo "LLVM_BUILD_DIR=${RUNNER_TEMP}/llvm-workspace/llvm-project/build-release" >> $GITHUB_ENV echo "PTO_SOURCE_DIR=$GITHUB_WORKSPACE" >> $GITHUB_ENV echo "PTO_BUILD_DIR=$GITHUB_WORKSPACE/build-release" >> $GITHUB_ENV - echo "PTO_INSTALL_DIR=$GITHUB_WORKSPACE/install-release" >> $GITHUB_ENV - name: Resolve LLVM source SHA id: llvm-source @@ -338,8 +337,13 @@ jobs: - name: Collect ptoas binary and dependencies if: matrix.python == '3.11' run: | - cmake --install "${PTO_BUILD_DIR}" --prefix "${PTO_INSTALL_DIR}" - bash "$PTO_SOURCE_DIR/docker/collect_ptoas_dist_mac.sh" "$GITHUB_WORKSPACE/ptoas-dist" + rm -rf "$GITHUB_WORKSPACE/ptoas-dist" + cmake --install "${PTO_BUILD_DIR}" \ + --prefix "$GITHUB_WORKSPACE/ptoas-dist" \ + --component PTOAS_Python + cmake --install "${PTO_BUILD_DIR}" \ + --prefix "$GITHUB_WORKSPACE/ptoas-dist" \ + --component PTOAS_CompilerArchive - name: Archive ptoas binary artifact if: matrix.python == '3.11' @@ -358,6 +362,11 @@ jobs: test -x "$TEST_DIR/extracted/bin/ptoas" test -f "$TEST_DIR/extracted/ptoas/_cli.py" + test -f "$TEST_DIR/extracted/ptoas/mlir/ir.py" + test -f "$TEST_DIR/extracted/ptodsl/__init__.py" + test -f "$TEST_DIR/extracted/tilelang_dsl/__init__.py" + test -d "$TEST_DIR/extracted/share/ptoas/TileOps" + test "$(cat "$TEST_DIR/extracted/.ptoas-python-version")" = "3.11" compgen -G "$TEST_DIR/extracted/ptoas/_core*.so" >/dev/null env -u PYTHONPATH -u DYLD_LIBRARY_PATH -u LD_LIBRARY_PATH \ diff --git a/README.md b/README.md index e17ca864b3..64f3858ed5 100644 --- a/README.md +++ b/README.md @@ -164,7 +164,7 @@ pip install /path/to/ptoas*.whl ```python import ptodsl from ptodsl import pto, scalar -from mlir.dialects import pto as mlir_pto +from ptoas.mlir.dialects import pto as mlir_pto ``` > 说明： @@ -184,7 +184,7 @@ from mlir.dialects import pto as mlir_pto ## 4. 运行环境配置 (Runtime Environment) 如果你已经通过 `pip install .`、`pip install -e .` 或 `pip install ptoas*.whl` -完成安装，那么 `import ptodsl` / `from mlir.dialects import pto` / `ptoas` +完成安装，那么 `import ptodsl` / `from ptoas.mlir.dialects import pto` / `ptoas` 都不应再依赖手动设置 `PYTHONPATH`。 下面这组环境变量主要用于**直接消费 build/install tree** 的场景，例如： @@ -243,9 +243,9 @@ ptoas --version 在支持的 `ptoas` 安装环境中，PTO Dialect 与 PTODSL 都可以直接导入。 ```python -from mlir.ir import Context, Module, Location -# [关键] 从 mlir.dialects 导入 pto，这是 Out-of-tree 绑定的标准用法 -from mlir.dialects import pto +from ptoas.mlir.ir import Context, Module, Location +# PTOAS 自带的 MLIR Python API 位于 ptoas.mlir 命名空间。 +from ptoas.mlir.dialects import pto from ptodsl import pto as jit_pto, scalar with Context() as ctx, Location.unknown(): diff --git a/README_en.md b/README_en.md index 2dc0de52fe..f582003bb2 100644 --- a/README_en.md +++ b/README_en.md @@ -152,7 +152,7 @@ After installation, the following imports should work directly: ```python import ptodsl from ptodsl import pto, scalar -from mlir.dialects import pto as mlir_pto +from ptoas.mlir.dialects import pto as mlir_pto ``` > Notes: @@ -198,9 +198,9 @@ In a supported `ptoas` install environment, both the PTO Dialect and PTODSL can be imported directly. ```python -from mlir.ir import Context, Module, Location -# [Key] Import pto from mlir.dialects — the standard pattern for out-of-tree bindings -from mlir.dialects import pto +from ptoas.mlir.ir import Context, Module, Location +# PTOAS ships its MLIR Python API in the ptoas.mlir namespace. +from ptoas.mlir.dialects import pto from ptodsl import pto as jit_pto, scalar with Context() as ctx, Location.unknown(): diff --git a/cmake/RelocatePTOASCompilerArchive.cmake.in b/cmake/RelocatePTOASCompilerArchive.cmake.in new file mode 100644 index 0000000000..2acb48de7f --- /dev/null +++ b/cmake/RelocatePTOASCompilerArchive.cmake.in @@ -0,0 +1,190 @@ +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +# Relocate a staged PTOAS compiler archive. This script intentionally runs at +# install time: CMake inspects the binaries that are present in the explicit +# staging prefix, computes their runtime dependency closure, and copies only +# external dependencies into /lib. + +# A full developer `cmake --install` keeps the normal install-tree contract. +# Relocation is an explicit archive-staging operation selected with +# `--component PTOAS_CompilerArchive`. +if(NOT CMAKE_INSTALL_COMPONENT STREQUAL "PTOAS_CompilerArchive") + return() +endif() + +set(_ptoas_archive_root "${CMAKE_INSTALL_PREFIX}") +set(_ptoas_package_root "${_ptoas_archive_root}/ptoas") +set(_ptoas_mlir_lib_root "${_ptoas_package_root}/mlir/_mlir_libs") +set(_ptoas_external_lib_root "${_ptoas_archive_root}/lib") + +if(NOT EXISTS "${_ptoas_package_root}/_cli.py") + message(FATAL_ERROR + "PTOAS compiler archive is missing ptoas/_cli.py under ${_ptoas_archive_root}; " + "install component PTOAS_Python before PTOAS_CompilerArchive") +endif() + +set(_ptoas_known_native_files + "${_ptoas_package_root}/$" + "${_ptoas_mlir_lib_root}/$" +) +foreach(_ptoas_native IN LISTS _ptoas_known_native_files) + if(NOT EXISTS "${_ptoas_native}") + message(FATAL_ERROR + "PTOAS compiler archive is missing native entry point ${_ptoas_native}; " + "install component PTOAS_Python before PTOAS_CompilerArchive") + endif() +endforeach() + +# AddMLIRPython owns the concrete module target list. Treat its installed +# extension directory as the plugin boundary, as opposed to scanning the +# complete archive tree for arbitrary shared libraries. +file(GLOB _ptoas_generated_native_files LIST_DIRECTORIES FALSE + "${_ptoas_mlir_lib_root}/*.so" + "${_ptoas_mlir_lib_root}/*.so.*" + "${_ptoas_mlir_lib_root}/*.dylib") +set(_ptoas_archive_native_files + ${_ptoas_known_native_files} + ${_ptoas_generated_native_files}) +list(REMOVE_DUPLICATES _ptoas_archive_native_files) +if(NOT _ptoas_generated_native_files) + message(FATAL_ERROR + "PTOAS compiler archive contains no MLIR native modules under " + "${_ptoas_mlir_lib_root}; " + "install component PTOAS_Python before PTOAS_CompilerArchive") +endif() + +file(MAKE_DIRECTORY "${_ptoas_external_lib_root}") +set(_ptoas_dependency_search_dirs + "${_ptoas_mlir_lib_root}" + "${_ptoas_archive_root}/lib" + "@LLVM_BUILD_LIBRARY_DIR@") + +file(GET_RUNTIME_DEPENDENCIES + LIBRARIES ${_ptoas_archive_native_files} + DIRECTORIES ${_ptoas_dependency_search_dirs} + RESOLVED_DEPENDENCIES_VAR _ptoas_resolved_dependencies + UNRESOLVED_DEPENDENCIES_VAR _ptoas_unresolved_dependencies + CONFLICTING_DEPENDENCIES_PREFIX _ptoas_conflicting_dependencies + POST_EXCLUDE_REGEXES + "^/lib/.*" + "^/lib64/.*" + "^/usr/lib/.*" + "^/usr/lib64/.*" + "^/System/Library/.*") + +if(_ptoas_unresolved_dependencies) + list(JOIN _ptoas_unresolved_dependencies "\n " _ptoas_unresolved_text) + message(FATAL_ERROR + "Unresolved PTOAS compiler archive dependencies:\n ${_ptoas_unresolved_text}") +endif() + +# A basename collision is harmless only when every candidate has identical +# contents. Otherwise flattening the dependency closure into lib/ would make +# the selected ABI depend on search order. +foreach(_ptoas_conflicting_name IN LISTS + _ptoas_conflicting_dependencies_FILENAMES) + set(_ptoas_conflicting_paths_var + "_ptoas_conflicting_dependencies_${_ptoas_conflicting_name}") + set(_ptoas_conflicting_paths ${${_ptoas_conflicting_paths_var}}) + list(GET _ptoas_conflicting_paths 0 _ptoas_selected_dependency) + file(SHA256 "${_ptoas_selected_dependency}" _ptoas_selected_sha256) + set(_ptoas_conflict_is_identical TRUE) + foreach(_ptoas_conflicting_path IN LISTS _ptoas_conflicting_paths) + file(SHA256 "${_ptoas_conflicting_path}" _ptoas_conflicting_sha256) + if(NOT _ptoas_conflicting_sha256 STREQUAL _ptoas_selected_sha256) + set(_ptoas_conflict_is_identical FALSE) + break() + endif() + endforeach() + + if(NOT _ptoas_conflict_is_identical) + list(JOIN _ptoas_conflicting_paths "\n " _ptoas_conflicting_text) + message(FATAL_ERROR + "Conflicting PTOAS compiler archive dependency " + "${_ptoas_conflicting_name}:\n ${_ptoas_conflicting_text}") + endif() + list(APPEND _ptoas_resolved_dependencies "${_ptoas_selected_dependency}") +endforeach() +list(REMOVE_DUPLICATES _ptoas_resolved_dependencies) + +foreach(_ptoas_dependency IN LISTS _ptoas_resolved_dependencies) + file(REAL_PATH "${_ptoas_dependency}" _ptoas_dependency_real) + string(FIND "${_ptoas_dependency_real}" "${_ptoas_archive_root}/" _ptoas_is_staged) + if(_ptoas_is_staged EQUAL 0) + continue() + endif() + + get_filename_component(_ptoas_dependency_name "${_ptoas_dependency_real}" NAME) + if(_ptoas_dependency_name MATCHES "^libPTOASPythonCAPI") + continue() + endif() + file(INSTALL "${_ptoas_dependency}" + DESTINATION "${_ptoas_external_lib_root}" + TYPE SHARED_LIBRARY + FOLLOW_SYMLINK_CHAIN) +endforeach() + +if(APPLE) + find_program(_ptoas_delocate_path NAMES delocate-path REQUIRED) + execute_process( + COMMAND "${_ptoas_delocate_path}" + --dylibs-only + --lib-path "${_ptoas_external_lib_root}" + "${_ptoas_package_root}" + RESULT_VARIABLE _ptoas_delocate_result + COMMAND_ECHO STDOUT) + if(NOT _ptoas_delocate_result EQUAL 0) + message(FATAL_ERROR "delocate-path failed for PTOAS compiler archive") + endif() +elseif(UNIX) + find_program(_ptoas_patchelf NAMES patchelf REQUIRED) + + function(_ptoas_origin_relative_rpath _ptoas_output _ptoas_from _ptoas_to) + file(RELATIVE_PATH _ptoas_relative_path "${_ptoas_from}" "${_ptoas_to}") + if(_ptoas_relative_path STREQUAL "") + set(_ptoas_rpath "$ORIGIN") + else() + set(_ptoas_rpath "$ORIGIN/${_ptoas_relative_path}") + endif() + set(${_ptoas_output} "${_ptoas_rpath}" PARENT_SCOPE) + endfunction() + + foreach(_ptoas_native IN LISTS _ptoas_archive_native_files) + if(IS_SYMLINK "${_ptoas_native}") + continue() + endif() + get_filename_component(_ptoas_native_dir "${_ptoas_native}" DIRECTORY) + _ptoas_origin_relative_rpath( + _ptoas_mlir_lib_rpath "${_ptoas_native_dir}" "${_ptoas_mlir_lib_root}") + _ptoas_origin_relative_rpath( + _ptoas_external_lib_rpath "${_ptoas_native_dir}" "${_ptoas_external_lib_root}") + set(_ptoas_native_rpaths + "$ORIGIN" + "${_ptoas_mlir_lib_rpath}" + "${_ptoas_external_lib_rpath}") + list(REMOVE_DUPLICATES _ptoas_native_rpaths) + list(JOIN _ptoas_native_rpaths ":" _ptoas_native_rpath) + execute_process( + COMMAND "${_ptoas_patchelf}" --set-rpath "${_ptoas_native_rpath}" + "${_ptoas_native}" + COMMAND_ERROR_IS_FATAL ANY) + endforeach() + + file(GLOB _ptoas_external_libraries LIST_DIRECTORIES FALSE + "${_ptoas_external_lib_root}/*.so" + "${_ptoas_external_lib_root}/*.so.*") + foreach(_ptoas_library IN LISTS _ptoas_external_libraries) + if(IS_SYMLINK "${_ptoas_library}") + continue() + endif() + execute_process( + COMMAND "${_ptoas_patchelf}" --set-rpath "$ORIGIN" "${_ptoas_library}" + COMMAND_ERROR_IS_FATAL ANY) + endforeach() +endif() diff --git a/docker/collect_ptoas_dist.sh b/docker/collect_ptoas_dist.sh deleted file mode 100755 index d682f842bd..0000000000 --- a/docker/collect_ptoas_dist.sh +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -# Collect ptoas binary and its dependencies into a self-contained distribution. -# -# Usage: ./collect_ptoas_dist.sh -# -# Required environment variables: -# LLVM_BUILD_DIR - Path to LLVM build directory -# PTO_INSTALL_DIR - Path to PTO install directory -# PTO_SOURCE_DIR - Path to PTO source directory -# -# Output structure: -# / -# bin/ptoas - Python console wrapper -# ptoas/ - CLI package and native extension -# lib/*.so* - Required shared library dependencies -# tilelang_dsl/ - TileLang DSL Python package -# -# This compiler-oriented binary artifact intentionally does not bundle the -# PTODSL Python package. PTODSL is provided only by the `ptoas` wheel. - -set -euo pipefail - -if [ $# -lt 1 ]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -PTOAS_DIST_DIR="$1" - -# Validate required environment variables -for var in LLVM_BUILD_DIR PTO_INSTALL_DIR PTO_SOURCE_DIR; do - if [ -z "${!var}" ]; then - echo "Error: $var environment variable is not set" >&2 - exit 1 - fi -done - -export LD_LIBRARY_PATH="${LLVM_BUILD_DIR}/lib:${PTO_INSTALL_DIR}/lib:${LD_LIBRARY_PATH:-}" - -PTOAS_BIN="${PTO_INSTALL_DIR}/bin/ptoas" -PTOAS_PACKAGE_SRC_DIR="${PTO_INSTALL_DIR}/ptoas" -PTOAS_PACKAGE_DIST_DIR="${PTOAS_DIST_DIR}/ptoas" -PTOAS_DEPS_DIR="${PTOAS_DIST_DIR}/lib" -PTOAS_TILELANG_DSL_SRC_DIR="${PTO_INSTALL_DIR}/tilelang_dsl" -PTOAS_TILELANG_DSL_DIST_DIR="${PTOAS_DIST_DIR}/tilelang_dsl" - -if [ ! -f "$PTOAS_BIN" ]; then - echo "Error: installed ptoas wrapper not found at $PTOAS_BIN" >&2 - exit 1 -fi -if [ ! -d "$PTOAS_PACKAGE_SRC_DIR" ]; then - echo "Error: installed ptoas package not found at $PTOAS_PACKAGE_SRC_DIR" >&2 - exit 1 -fi -remove_rpath() { - local path="$1" - if ! has_rpath "$path"; then - return - fi - if ! can_scrub_rpath; then - echo "WARN: skipping RPATH/RUNPATH scrub for ${path}; install patchelf or chrpath to harden local dist artifacts" >&2 - return - fi - if command -v patchelf >/dev/null 2>&1; then - patchelf --remove-rpath "$path" - fi - if has_rpath "$path" && command -v chrpath >/dev/null 2>&1; then - chrpath -d "$path" - fi - if has_rpath "$path"; then - echo "Error: failed to scrub RPATH/RUNPATH from ${path}" >&2 - exit 1 - fi -} - -strip_symbols() { - local path="$1" - strip --strip-unneeded "$path" -} - -has_rpath() { - local path="$1" - if command -v patchelf >/dev/null 2>&1; then - local rpath_value - rpath_value="$(patchelf --print-rpath "$path" 2>/dev/null || true)" - [[ -n "$rpath_value" ]] - return - fi - readelf -d "$path" 2>/dev/null | grep -Eq '(RPATH|RUNPATH)' -} - -can_scrub_rpath() { - command -v patchelf >/dev/null 2>&1 || command -v chrpath >/dev/null 2>&1 -} - -assert_relro() { - local path="$1" - if ! readelf -l "$path" 2>/dev/null | grep -q 'GNU_RELRO'; then - echo "WARN: RELRO segment missing in ${path}" >&2 - return - fi - if ! readelf -d "$path" 2>/dev/null | grep -Eq '(BIND_NOW|FLAGS.*NOW|FLAGS_1.*NOW)'; then - echo "WARN: NOW binding missing in ${path}" >&2 - fi -} - -assert_no_symtab() { - local path="$1" - if readelf -S "$path" 2>/dev/null | grep -Eq '[[:space:]]\\.symtab[[:space:]]'; then - echo "Error: symbol table still present in ${path}" >&2 - exit 1 - fi -} - -assert_no_rpath() { - local path="$1" - if ! can_scrub_rpath; then - return - fi - if has_rpath "$path"; then - echo "Error: runtime search path still present in ${path}" >&2 - exit 1 - fi -} - -harden_elf() { - local path="$1" - remove_rpath "$path" - strip_symbols "$path" - assert_relro "$path" - assert_no_symtab "$path" - assert_no_rpath "$path" -} - -# Create output directories -mkdir -p \ - "${PTOAS_DIST_DIR}/bin" \ - "${PTOAS_DEPS_DIR}" -rm -rf "${PTOAS_PACKAGE_DIST_DIR}" -cp -R "${PTOAS_PACKAGE_SRC_DIR}" "${PTOAS_PACKAGE_DIST_DIR}" -cp "$PTOAS_BIN" "${PTOAS_DIST_DIR}/bin/ptoas" -chmod +x "${PTOAS_DIST_DIR}/bin/ptoas" - -PTOAS_NATIVE_MODULE="$(find "${PTOAS_PACKAGE_DIST_DIR}" -maxdepth 1 -type f -name '_core*.so' -print -quit)" -if [ -z "${PTOAS_NATIVE_MODULE}" ]; then - echo "Error: packaged ptoas._core extension not found" >&2 - exit 1 -fi - -# Collect non-system dependencies needed by the Python extension. -echo "Collecting shared library dependencies..." -linux_runtime_dep_paths() { - local path="$1" - ldd "$path" 2>/dev/null | awk ' - /=> \// { print $3 } - /^\// { print $1 } - ' -} - -should_bundle_linux_dep() { - local path="$1" - case "$path" in - /lib/*|/lib64/*|/usr/lib/*|/usr/lib64/*) - return 1 - ;; - esac - return 0 -} - -copy_so() { - local f="$1" - [[ -f "$f" ]] || return 0 - local name - name=$(basename "$f") - [[ -f "${PTOAS_DEPS_DIR}/${name}" ]] && return 0 - cp -L -n "$f" "${PTOAS_DEPS_DIR}/" 2>/dev/null || true - harden_elf "${PTOAS_DEPS_DIR}/${name}" - while read -r res; do - [[ -n "$res" ]] || continue - should_bundle_linux_dep "$res" || continue - copy_so "$res" - done < <(linux_runtime_dep_paths "$f") -} - -while read -r res; do - [[ -n "$res" ]] || continue - should_bundle_linux_dep "$res" || continue - copy_so "$res" -done < <(linux_runtime_dep_paths "${PTOAS_NATIVE_MODULE}") - -while read -r packaged; do - harden_elf "$packaged" -done < <(find "${PTOAS_DEPS_DIR}" -type f | sort) - -if ! command -v patchelf >/dev/null 2>&1; then - echo "Error: patchelf is required to make the ptoas archive relocatable" >&2 - exit 1 -fi -while read -r packaged; do - patchelf --set-rpath '$ORIGIN' "$packaged" -done < <(find "${PTOAS_DEPS_DIR}" -type f | sort) -harden_elf "${PTOAS_NATIVE_MODULE}" -patchelf --set-rpath '$ORIGIN/../lib' "${PTOAS_NATIVE_MODULE}" - -echo "Copying TileLang runtime resources..." -if [ ! -d "${PTOAS_TILELANG_DSL_SRC_DIR}" ]; then - echo "Error: tilelang_dsl package directory not found at ${PTOAS_TILELANG_DSL_SRC_DIR}" >&2 - exit 1 -fi -rm -rf "${PTOAS_TILELANG_DSL_DIST_DIR}" -cp -R "${PTOAS_TILELANG_DSL_SRC_DIR}" "${PTOAS_TILELANG_DSL_DIST_DIR}" - -echo "Smoke testing packaged ptoas dist..." -VERSION_OUTPUT="$(env -u PYTHONPATH -u DYLD_LIBRARY_PATH -u LD_LIBRARY_PATH \ - "${PTOAS_DIST_DIR}/bin/ptoas" --version | tr -d '\r')" -echo "$VERSION_OUTPUT" -EXPECTED_PTOAS_CLI_VERSION="${PTOAS_CLI_VERSION:-${PTOAS_VERSION:-}}" -if [ -n "${EXPECTED_PTOAS_CLI_VERSION}" ]; then - EXPECTED_VERSION_OUTPUT="ptoas ${EXPECTED_PTOAS_CLI_VERSION}" - if [ "${VERSION_OUTPUT}" != "${EXPECTED_VERSION_OUTPUT}" ]; then - echo "Error: expected '${EXPECTED_VERSION_OUTPUT}', got '${VERSION_OUTPUT}'" >&2 - exit 1 - fi -else - echo "$VERSION_OUTPUT" | grep -Eq '^ptoas [0-9]+\.[0-9]+$' -fi - -test -f "${PTOAS_PACKAGE_DIST_DIR}/_cli.py" -test -d "${PTOAS_PACKAGE_DIST_DIR}/_runtime/share/ptoas/TileOps" -test -f "${PTOAS_TILELANG_DSL_DIST_DIR}/__init__.py" -if [ -e "${PTOAS_DIST_DIR}/ptodsl" ]; then - echo "Error: compiler-oriented ptoas dist must not bundle PTODSL" >&2 - exit 1 -fi - -# Show collected files -echo "" -echo "=== ptoas distribution contents ===" -ls -la "${PTOAS_DIST_DIR}/" -ls -la "${PTOAS_DIST_DIR}/bin/" -ls -la "${PTOAS_PACKAGE_DIST_DIR}/" -ls -la "${PTOAS_TILELANG_DSL_DIST_DIR}" -SO_COUNT=$(find "${PTOAS_DEPS_DIR}" -name "*.so*" 2>/dev/null | wc -l) -echo "=== Collected .so dependencies (${SO_COUNT} files) ===" -du -sh "${PTOAS_DEPS_DIR}/" -echo "" -echo "Distribution created at: ${PTOAS_DIST_DIR}" diff --git a/docker/collect_ptoas_dist_mac.sh b/docker/collect_ptoas_dist_mac.sh deleted file mode 100644 index 6685930f5f..0000000000 --- a/docker/collect_ptoas_dist_mac.sh +++ /dev/null @@ -1,378 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -# Collect ptoas binary and macOS dylib dependencies into a self-contained distribution. -# -# Usage: ./collect_ptoas_dist_mac.sh -# -# Required environment variables: -# LLVM_BUILD_DIR - Path to LLVM build directory -# PTO_INSTALL_DIR - Path to PTO install directory -# PTO_SOURCE_DIR - Path to PTO source directory -# -# Output structure: -# / -# bin/ptoas - Python console wrapper -# ptoas/ - CLI package and native extension -# lib/*.dylib - Required shared library dependencies -# tilelang_dsl/ - TileLang DSL Python package -# -# This compiler-oriented binary artifact intentionally does not bundle the -# PTODSL Python package. PTODSL is provided only by the `ptoas` wheel. - -set -euo pipefail - -if [ $# -lt 1 ]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -PTOAS_DIST_DIR="$1" - -# Validate required environment variables -for var in LLVM_BUILD_DIR PTO_INSTALL_DIR PTO_SOURCE_DIR; do - if [ -z "${!var:-}" ]; then - echo "Error: $var environment variable is not set" >&2 - exit 1 - fi -done - -PTOAS_BIN="${PTO_INSTALL_DIR}/bin/ptoas" -PTOAS_PACKAGE_SRC_DIR="${PTO_INSTALL_DIR}/ptoas" -PTOAS_PACKAGE_DIST_DIR="${PTOAS_DIST_DIR}/ptoas" -PTOAS_DEPS_DIR="${PTOAS_DIST_DIR}/lib" -PTOAS_TILELANG_DSL_SRC_DIR="${PTO_INSTALL_DIR}/tilelang_dsl" -PTOAS_TILELANG_DSL_DIST_DIR="${PTOAS_DIST_DIR}/tilelang_dsl" -UNRESOLVED_NON_SYSTEM_COUNT=0 - -if [ ! -f "$PTOAS_BIN" ]; then - echo "Error: installed ptoas wrapper not found at $PTOAS_BIN" >&2 - exit 1 -fi -if [ ! -d "$PTOAS_PACKAGE_SRC_DIR" ]; then - echo "Error: installed ptoas package not found at $PTOAS_PACKAGE_SRC_DIR" >&2 - exit 1 -fi -mkdir -p \ - "${PTOAS_DIST_DIR}/bin" \ - "${PTOAS_DEPS_DIR}" -rm -rf "${PTOAS_PACKAGE_DIST_DIR}" -cp -R "${PTOAS_PACKAGE_SRC_DIR}" "${PTOAS_PACKAGE_DIST_DIR}" -cp -fL "$PTOAS_BIN" "${PTOAS_DIST_DIR}/bin/ptoas" -chmod +x "${PTOAS_DIST_DIR}/bin/ptoas" - -PTOAS_NATIVE_MODULE="$(find "${PTOAS_PACKAGE_DIST_DIR}" -maxdepth 1 -type f -name '_core*.so' -print -quit)" -if [ -z "${PTOAS_NATIVE_MODULE}" ]; then - echo "Error: packaged ptoas._core extension not found" >&2 - exit 1 -fi - -# Resolve @rpath / @loader_path / @executable_path / absolute install names. -resolve_dep_path() { - local owner="$1" - local dep="$2" - local owner_dir - owner_dir="$(dirname "$owner")" - - # macOS GitHub runners use bash 3.2; avoid mapfile for compatibility. - local owner_rpaths=() - local rp_line - while IFS= read -r rp_line; do - [ -n "$rp_line" ] && owner_rpaths+=("$rp_line") - done < <( - otool -l "$owner" | awk ' - $1=="cmd" && $2=="LC_RPATH" {want=1; next} - want && $1=="path" {print $2; want=0} - ' - ) - - local dep_tail="$dep" - if [[ "$dep" == @rpath/* ]]; then - dep_tail="${dep#@rpath/}" - fi - - local candidates=() - if [[ "$dep" = /* ]]; then - candidates+=("$dep") - fi - if [[ "$dep" == @loader_path/* ]]; then - candidates+=("${owner_dir}/${dep#@loader_path/}") - fi - if [[ "$dep" == @executable_path/* ]]; then - candidates+=("${PTOAS_DIST_DIR}/bin/${dep#@executable_path/}") - fi - if [[ "$dep" == @rpath/* ]]; then - for rp in "${owner_rpaths[@]:-}"; do - case "$rp" in - @loader_path/*) rp="${owner_dir}/${rp#@loader_path/}" ;; - @executable_path/*) rp="${PTOAS_DIST_DIR}/bin/${rp#@executable_path/}" ;; - esac - candidates+=("${rp}/${dep_tail}") - done - candidates+=( - "${LLVM_BUILD_DIR}/lib/${dep_tail}" - "${PTO_INSTALL_DIR}/lib/${dep_tail}" - "${owner_dir}/${dep_tail}" - ) - fi - - local c - for c in "${candidates[@]}"; do - if [[ -f "$c" ]]; then - echo "$c" - return 0 - fi - done - return 1 -} - -collect_dylibs() { - local bin="$1" - while read -r dep; do - [ -n "$dep" ] || continue - local resolved - resolved="$(resolve_dep_path "$bin" "$dep" || true)" - if [ -z "$resolved" ]; then - case "$dep" in - /usr/lib/*|/System/Library/*) - # Expected on macOS: system libs are provided by the host. - ;; - *) - echo "WARN: unresolved non-system dep for $bin -> $dep" - UNRESOLVED_NON_SYSTEM_COUNT=$((UNRESOLVED_NON_SYSTEM_COUNT + 1)) - ;; - esac - continue - fi - - local base - base="$(basename "$resolved")" - if [ ! -f "${PTOAS_DEPS_DIR}/${base}" ]; then - cp -fL "$resolved" "${PTOAS_DEPS_DIR}/${base}" - install_name_tool -id "@loader_path/${base}" "${PTOAS_DEPS_DIR}/${base}" || true - collect_dylibs "${PTOAS_DEPS_DIR}/${base}" - fi - install_name_tool -change "$dep" "@loader_path/../lib/${base}" "$bin" || true - done < <(otool -L "$bin" | awk 'NR>1 {print $1}') -} - -rewrite_packaged_install_names() { - python3 - "${PTOAS_DIST_DIR}" "${PTOAS_DEPS_DIR}" <<'PY' -import os -import subprocess -import sys -from pathlib import Path - -dist_dir = Path(sys.argv[1]).resolve() -deps_dir = Path(sys.argv[2]).resolve() -bin_dir = (dist_dir / "bin").resolve() -package_dir = (dist_dir / "ptoas").resolve() -allowed_prefixes = ( - "@loader_path/", - "@rpath/", - "@executable_path/", - "/usr/lib/", - "/System/Library/", -) - - -def is_under(path: Path, root: Path) -> bool: - try: - path.relative_to(root) - return True - except ValueError: - return False - - -def packaged_dep_ref(owner: Path, dep_base: str) -> str: - if is_under(owner, bin_dir): - return f"@loader_path/../lib/{dep_base}" - if is_under(owner, package_dir): - return f"@loader_path/../lib/{dep_base}" - if is_under(owner, deps_dir): - return f"@loader_path/{dep_base}" - return f"@loader_path/{dep_base}" - - -def iter_targets(): - for root in (package_dir, deps_dir): - if not root.exists(): - continue - for base, _, files in os.walk(root): - for name in sorted(files): - if name.endswith(".dylib") or name.endswith(".so"): - yield Path(base, name).resolve() - - -def iter_deps(target: Path): - try: - output = subprocess.check_output( - ["otool", "-L", str(target)], - stderr=subprocess.STDOUT, - text=True, - ) - except subprocess.CalledProcessError as exc: - sys.stderr.write( - f"ERROR: failed to inspect install names for {target}: " - f"{exc.output.strip()}\n" - ) - raise SystemExit(exc.returncode or 1) - - for line in output.splitlines()[1:]: - dep = line.strip().split(" ", 1)[0] - if dep: - yield dep - - -for target in iter_targets(): - for dep in iter_deps(target): - if dep.startswith(allowed_prefixes): - continue - - dep_base = os.path.basename(dep) - if not (deps_dir / dep_base).is_file(): - continue - - replacement = packaged_dep_ref(target, dep_base) - if dep == replacement: - continue - - print(f"rewrite install name: {target} :: {dep} -> {replacement}") - try: - subprocess.check_call( - ["install_name_tool", "-change", dep, replacement, str(target)] - ) - except subprocess.CalledProcessError as exc: - sys.stderr.write( - f"ERROR: failed to rewrite install name for {target}: {dep} -> " - f"{replacement} (exit {exc.returncode})\n" - ) - raise SystemExit(exc.returncode or 1) -PY -} - -echo "Collecting dylib dependencies..." -collect_dylibs "${PTOAS_NATIVE_MODULE}" - -echo "Copying TileLang runtime resources..." -if [[ ! -d "${PTOAS_TILELANG_DSL_SRC_DIR}" ]]; then - echo "Error: tilelang_dsl package directory not found at ${PTOAS_TILELANG_DSL_SRC_DIR}" >&2 - exit 1 -fi -rm -rf "${PTOAS_TILELANG_DSL_DIST_DIR}" -cp -R "${PTOAS_TILELANG_DSL_SRC_DIR}" "${PTOAS_TILELANG_DSL_DIST_DIR}" -if [[ -e "${PTOAS_DIST_DIR}/ptodsl" ]]; then - echo "Error: compiler-oriented ptoas dist must not bundle PTODSL" >&2 - exit 1 -fi - -echo "Rewriting packaged install names..." -rewrite_packaged_install_names - -echo "Validating packaged dependency install names..." -if ! python3 - "${PTOAS_DIST_DIR}" <<'PY' -import os -import subprocess -import sys - -root = sys.argv[1] -allowed_prefixes = ( - "@loader_path/", - "@rpath/", - "@executable_path/", - "/usr/lib/", - "/System/Library/", -) - -bad = [] -for base, _, files in os.walk(root): - for name in files: - if not name.endswith(".dylib") and not name.endswith(".so"): - continue - path = os.path.join(base, name) - try: - output = subprocess.check_output( - ["otool", "-L", path], - stderr=subprocess.STDOUT, - text=True, - ) - except subprocess.CalledProcessError as exc: - print(f"ERROR: failed to inspect {path}: {exc.output.strip()}", - file=sys.stderr) - sys.exit(2) - - for line in output.splitlines()[1:]: - dep = line.strip().split(" ", 1)[0] - if dep.startswith(allowed_prefixes): - continue - bad.append((path, dep)) - -for path, dep in bad: - print(f"ERROR: non-portable dependency in {path} -> {dep}", file=sys.stderr) - -print(f"portable dependency scan checked {root} ({len(bad)} offending deps)") -sys.exit(1 if bad else 0) -PY -then - echo "Error: found non-portable dependency install names" >&2 - exit 1 -fi - -if ! command -v codesign >/dev/null 2>&1; then - echo "Error: codesign is required on macOS to sign packaged artifacts" >&2 - exit 1 -fi - -echo "Ad-hoc signing packaged binaries and dylibs..." -shopt -s nullglob -SIGN_TARGETS=("${PTOAS_NATIVE_MODULE}" "${PTOAS_DEPS_DIR}"/*.so "${PTOAS_DEPS_DIR}"/*.dylib) -for target in "${SIGN_TARGETS[@]}"; do - codesign --force --sign - --timestamp=none "$target" -done - -echo "Verifying code signatures..." -for target in "${SIGN_TARGETS[@]}"; do - codesign --verify --strict --verbose=2 "$target" -done -shopt -u nullglob - -echo "Smoke testing packaged ptoas dist..." -VERSION_OUTPUT="$(env -u PYTHONPATH -u DYLD_LIBRARY_PATH -u LD_LIBRARY_PATH \ - "${PTOAS_DIST_DIR}/bin/ptoas" --version | tr -d '\r')" -echo "$VERSION_OUTPUT" -EXPECTED_PTOAS_CLI_VERSION="${PTOAS_CLI_VERSION:-${PTOAS_VERSION:-}}" -if [ -n "${EXPECTED_PTOAS_CLI_VERSION}" ]; then - EXPECTED_VERSION_OUTPUT="ptoas ${EXPECTED_PTOAS_CLI_VERSION}" - if [ "${VERSION_OUTPUT}" != "${EXPECTED_VERSION_OUTPUT}" ]; then - echo "Error: expected '${EXPECTED_VERSION_OUTPUT}', got '${VERSION_OUTPUT}'" >&2 - exit 1 - fi -else - echo "$VERSION_OUTPUT" | grep -Eq '^ptoas [0-9]+\.[0-9]+$' -fi -test -f "${PTOAS_PACKAGE_DIST_DIR}/_cli.py" -test -d "${PTOAS_PACKAGE_DIST_DIR}/_runtime/share/ptoas/TileOps" -test -f "${PTOAS_TILELANG_DSL_DIST_DIR}/__init__.py" -env -u DYLD_LIBRARY_PATH -u LD_LIBRARY_PATH \ - "${PTOAS_DIST_DIR}/bin/ptoas" \ - "${PTO_SOURCE_DIR}/test/lit/pto/kernel_kind_vector_scf_while_emitc.pto" \ - >/dev/null - -echo "" -echo "=== ptoas distribution contents ===" -ls -la "${PTOAS_DIST_DIR}/" -ls -la "${PTOAS_DIST_DIR}/bin/" -ls -la "${PTOAS_PACKAGE_DIST_DIR}/" -ls -la "${PTOAS_TILELANG_DSL_DIST_DIR}" -DYLIB_COUNT=$(find "${PTOAS_DEPS_DIR}" -name "*.dylib" 2>/dev/null | wc -l) -echo "=== Collected .dylib dependencies (${DYLIB_COUNT} files) ===" -du -sh "${PTOAS_DEPS_DIR}/" -echo "=== Unresolved non-system deps: ${UNRESOLVED_NON_SYSTEM_COUNT} ===" -echo "" -echo "Distribution created at: ${PTOAS_DIST_DIR}" diff --git a/docker/test_wheel_imports.sh b/docker/test_wheel_imports.sh index df2a30487c..aff59bcc2a 100755 --- a/docker/test_wheel_imports.sh +++ b/docker/test_wheel_imports.sh @@ -12,8 +12,8 @@ # Usage: ./test_wheel_imports.sh # # This script tests that the installed wheel can import: -# - mlir.ir -# - mlir.dialects.pto +# - ptoas.mlir.ir +# - ptoas.mlir.dialects.pto # - ptodsl # - from ptodsl import pto, scalar # - ptoas CLI entry @@ -76,11 +76,21 @@ cd /tmp PTOAS_ENTRYPOINT="$(command -v ptoas)" PYTHON_ENTRYPOINT="$(command -v "${PYTHON_BIN}")" -echo "Testing mlir.ir import..." -"$PYTHON_BIN" -c "import mlir.ir; print('mlir.ir imported successfully')" +echo "Testing ptoas.mlir.ir import..." +"$PYTHON_BIN" -c "from ptoas.mlir import ir; print('ptoas.mlir.ir imported successfully')" echo "Testing pto dialect import..." -"$PYTHON_BIN" -c "from mlir.dialects import pto; print('pto dialect imported successfully')" +"$PYTHON_BIN" -c "from ptoas.mlir.dialects import pto; print('pto dialect imported successfully')" + +echo "Testing that PTOAS does not claim the top-level mlir namespace..." +"$PYTHON_BIN" - <<'PY' +try: + import mlir # noqa: F401 +except ImportError: + pass +else: + raise SystemExit("the ptoas wheel unexpectedly provides a top-level mlir package") +PY echo "Testing ptodsl import..." "$PYTHON_BIN" -c "import ptodsl; print(f'ptodsl imported successfully from {ptodsl.__file__}')" diff --git a/docker/validate_wheel_payload.py b/docker/validate_wheel_payload.py index 0ddcda8a3d..a96f677fa0 100644 --- a/docker/validate_wheel_payload.py +++ b/docker/validate_wheel_payload.py @@ -21,6 +21,9 @@ "ptoas/__init__.py", "ptoas/_cli.py", "ptoas/_runtime/share/ptoas/TileOps/__init__.py", + "ptoas/mlir/ir.py", + "ptoas/mlir/dialects/pto.py", + "ptoas/mlir/_mlir_libs/libPTOASPythonCAPI.so", } FORBIDDEN_FILES = { "ptoas/_runtime/bin/ptoas", @@ -31,6 +34,15 @@ f"ptoas/_core{suffix}" for suffix in importlib.machinery.EXTENSION_SUFFIXES } +MLIR_NATIVE_MODULE_PREFIX = "ptoas/mlir/_mlir_libs/" +MLIR_NATIVE_MODULE_PATHS = { + f"{MLIR_NATIVE_MODULE_PREFIX}_mlir{suffix}" + for suffix in importlib.machinery.EXTENSION_SUFFIXES +} +SITE_INITIALIZER_PATHS = { + f"{MLIR_NATIVE_MODULE_PREFIX}_site_initialize_0{suffix}" + for suffix in importlib.machinery.EXTENSION_SUFFIXES +} def _resolve_wheel(candidate: str) -> Path: @@ -93,6 +105,27 @@ def validate_wheel_payload(wheel: Path) -> None: f"found {native_modules}" ) + mlir_native_modules = sorted(MLIR_NATIVE_MODULE_PATHS & names) + if len(mlir_native_modules) != 1: + raise SystemExit( + "wheel must contain exactly one ptoas-owned MLIR native module, " + f"found {mlir_native_modules}" + ) + + site_initializers = sorted(SITE_INITIALIZER_PATHS & names) + if len(site_initializers) != 1: + raise SystemExit( + "wheel must contain exactly one PTOAS MLIR site initializer, " + f"found {site_initializers}" + ) + + top_level_mlir = sorted(name for name in names if name.startswith("mlir/")) + if top_level_mlir: + raise SystemExit( + "wheel must not install a top-level mlir package; " + f"found {top_level_mlir[:10]}" + ) + present_forbidden = sorted(FORBIDDEN_FILES & names) if present_forbidden: raise SystemExit( diff --git a/docs/build_with_installed_llvm.md b/docs/build_with_installed_llvm.md index 807485d898..06e751dfc4 100644 --- a/docs/build_with_installed_llvm.md +++ b/docs/build_with_installed_llvm.md @@ -111,7 +111,7 @@ cmake --install build - install 目录： - `$PTO_INSTALL_DIR/bin/ptoas` - `$PTO_INSTALL_DIR/ptoas/_core.cpython-*.so` - - `$PTO_INSTALL_DIR/mlir/dialects/pto.py` + - `$PTO_INSTALL_DIR/ptoas/mlir/dialects/pto.py` - `$PTO_INSTALL_DIR/share/ptoas/oplib/level3` ## 补充：运行环境 diff --git a/docs/designs/ptoas-mlir-namespace-and-native-isolation.md b/docs/designs/ptoas-mlir-namespace-and-native-isolation.md new file mode 100644 index 0000000000..40eecde609 --- /dev/null +++ b/docs/designs/ptoas-mlir-namespace-and-native-isolation.md @@ -0,0 +1,692 @@ +// Copyright (c) 2026 Huawei Technologies Co., Ltd. +// This program is free software, you can redistribute it and/or modify it under the terms and conditions of +// CANN Open Software License Agreement Version 2.0 (the "License"). +// Please refer to the License for details. You may not use this file except in compliance with the License. +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +// INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +// See LICENSE in the root of the software repository for the full text of the License. + +# PTOAS MLIR Python Namespace and Native Isolation Design + +## Status + +This document records the agreed migration direction and its constraints. The +design is **proposed and not yet implemented**. Until the migration lands, the +current public imports remain `mlir.ir` and `mlir.dialects.pto`. + +The intended public API is: + +```python +from ptoas.mlir import ir +from ptoas.mlir.ir import Context, Module +from ptoas.mlir.dialects import pto +``` + +The migration must remove the PTOAS-owned top-level `mlir` package rather than +installing a compatibility alias that would recreate the namespace collision. + +## Goals + +1. Place the complete PTOAS-owned MLIR Python runtime under `ptoas.mlir`. +2. Allow PTOAS and another self-contained MLIR distribution, such as IREE, + CIRCT, torch-mlir, or JAX, to coexist in one Python environment. +3. Give all PTOAS MLIR extensions one project-specific nanobind domain. +4. Keep one native MLIR runtime inside PTOAS so that `ptoas.mlir`, the PTO + dialect facade, PTODSL, and `ptoas._core` share object identity. +5. Use MLIR's exported CMake source graph and standard wheel repair tools + instead of copying package trees or implementing a custom loader. +6. Define and test the supported boundary between independently packaged MLIR + runtimes. +7. Keep the repaired wheel as the official Python distribution, while retaining + `ptoas-bin` as a CI and board-validation compatibility artifact. Generate + that artifact from a deterministic CMake install-tree staging directory; + do not make it a second Python package distribution. + +## Non-Goals + +- Do not make `ptoas.mlir.ir.Module` interchangeable with an IREE, CIRCT, + torch-mlir, JAX, or system-MLIR `Module`. +- Do not install or populate a top-level `mlir` compatibility package. +- Do not scan repositories, neighboring build trees, `PYTHONPATH`, or library + directories at Python import time. +- Do not preload LLVM/MLIR libraries with `RTLD_GLOBAL` or a custom bootstrap + loader. +- Do not switch the LLVM SDK from shared to static libraries as part of the + namespace migration. +- Do not modify LLVM/MLIR sources to implement the package layout. +- Do not make the compiler archive a second PyPI-style distribution with its + own metadata, Python dependency resolver, or import bootstrap. +- Do not scan the source tree or arbitrary build directories while assembling + the archive. Its contents must come from an explicit install prefix. +- Do not pretend that `auditwheel repair` or `delocate` is a generic tarball + relocation tool. They remain the standard repair tools for wheels; any + archive-native relocation must be a small, explicit packaging step. + +## Industry and Upstream References + +The design separates mechanisms with direct upstream/industry precedent from +the PTOAS-specific compatibility artifact required by the current board-test +workflow. + +### Directly Referenced Mechanisms + +- LLVM's standalone example uses `MLIR_PYTHON_PACKAGE_PREFIX`, + `add_mlir_python_common_capi_library`, and `add_mlir_python_modules` to embed + MLIR bindings in a downstream namespace. +- torch-mlir explicitly vendors MLIR in the `torch_mlir` namespace and defines + `MLIR_PYTHON_PACKAGE_PREFIX=torch_mlir.`. +- IREE defines `MLIR_PYTHON_PACKAGE_PREFIX=iree.compiler.`, installs its MLIR + tree below `iree/compiler`, and makes the Python bindings consume one + project-owned shared compiler implementation. Its compiler tools are found + relative to the installed Python package rather than by scanning build or + source trees. +- CIRCT defines `MLIR_PYTHON_PACKAGE_PREFIX=circt.`, installs below `circt`, and + uses a project-owned `CIRCTBindingsPythonCAPI` library. +- JAX packages its MLIR bindings below `jaxlib.mlir` with a private + `_mlir_libs` directory. +- Linux native wheels conventionally use `auditwheel repair`; auditwheel copies + external libraries, gives them content-derived names, rewrites `DT_NEEDED`, + and installs package-relative RPATH entries. +- macOS native wheels conventionally use `delocate`, which copies dependent + dylibs and rewrites install names and RPATH entries. +- CMake install components and CPack archive an explicit install graph into a + deterministic prefix. They are the standard boundary for a native install + tree, although they do not replace wheel-specific dependency repair. + +These references directly support the `ptoas.mlir` namespace, project-owned +common CAPI, project-specific nanobind domain, and repaired-wheel architecture. + +### Distribution and Archive Boundary + +There is no universal MLIR or Python packaging standard for publishing a +compiler-only tarball alongside a Python wheel. Comparable projects either +publish wheels only or maintain a project-specific compiler/binary package. +The standard Python distribution remains the repaired wheel; `auditwheel` and +`delocate` are intentionally wheel-oriented tools. + +PTOAS must currently retain a `ptoas-bin` archive because the PR board-test +monitor downloads the Linux artifact before running compiler cases. This is a +compatibility artifact, not a second public Python distribution. The archive +will be assembled from a dedicated CMake install-tree staging prefix, using +the same `ptoas` package layout and runtime contract as the wheel. CPack/TGZ or +an equivalent archive of that install prefix is the appropriate packaging +boundary; a script must not reconstruct a package by scanning the repository. + +The archive and wheel have different native packaging constraints. Wheel +repair can be delegated to `auditwheel`/`delocate`. A relocatable install-tree +archive may still need a narrowly scoped platform step to ensure that bundled +non-system libraries use relative RPATHs or install names. That step must not +also copy Python sources, invent a second package layout, or run a second +general-purpose dependency-discovery pipeline hidden inside the launcher. + +Relevant upstream sources: + +- `llvm-project/mlir/examples/standalone/python/CMakeLists.txt` +- `llvm-project/mlir/cmake/modules/AddMLIRPython.cmake` +- `llvm/torch-mlir/python/CMakeLists.txt` +- `iree-org/iree/compiler/bindings/python/CMakeLists.txt` +- `iree-org/iree/compiler/bindings/python/iree/compiler/tools/binaries.py` +- `llvm/circt/lib/Bindings/Python/CMakeLists.txt` +- `cmake/Help/module/CPack.rst` +- `pypa/auditwheel` +- `matthew-brett/delocate` + +## Target Package Layout + +```text +ptoas/ +├── __init__.py +├── _cli.py +├── _core..so +├── _runtime/ +└── mlir/ + ├── __init__.py + ├── ir.py + ├── passmanager.py + ├── dialects/ + │ ├── arith.py + │ ├── func.py + │ ├── llvm.py + │ ├── math.py + │ ├── memref.py + │ ├── pto.py + │ └── scf.py + └── _mlir_libs/ + ├── _mlir..so + ├── _site_initialize_0..so + └── libPTOASPythonCAPI.so +``` + +The corresponding native ownership graph is: + +```text +ptoas.mlir.ir + │ + ▼ +ptoas/mlir/_mlir_libs/_mlir.so + │ + ├──────────────────────┐ + ▼ ▼ +libPTOASPythonCAPI.so ◄── ptoas/_core.so + │ + ▼ +repaired, content-named LLVM/MLIR shared libraries +``` + +All PTOAS binding modules must resolve MLIR objects through the same +`PTOASPythonCAPI` and the same underlying LLVM/MLIR shared libraries. + +## Python Namespace Configuration + +Moving files under `ptoas/mlir` is not sufficient. Native MLIR bindings also +use the package prefix to locate Python classes and capsules. The binding +targets must therefore be compiled with: + +```cmake +add_compile_definitions( + "MLIR_PYTHON_PACKAGE_PREFIX=ptoas.mlir." +) +``` + +The definition must apply to the MLIR-generated extension targets and to +`PTOASPythonCore`. Since `_core` is built in a separate CMake directory, it may +need an explicit target-scoped compile definition in addition to the binding +directory definition. + +The staged and installed package roots should become: + +```cmake +set(PTOAS_MLIR_PYTHON_PACKAGE_DIR + "${CMAKE_BINARY_DIR}/python/ptoas/mlir") + +add_mlir_python_common_capi_library(PTOASPythonCAPI + INSTALL_DESTINATION "ptoas/mlir/_mlir_libs" + OUTPUT_DIRECTORY "${PTOAS_MLIR_PYTHON_PACKAGE_DIR}/_mlir_libs" + ... +) + +add_mlir_python_modules(PTOAS_Python + ROOT_PREFIX "${PTOAS_MLIR_PYTHON_PACKAGE_DIR}" + INSTALL_PREFIX "ptoas/mlir" + ... +) +``` + +The generated package remains owned by the `PTOAS_Python` CMake install +component. It should not be copied from an LLVM build tree or added as a second +`tool.scikit-build.wheel.packages` source tree. + +## Nanobind Domain Isolation + +MLIR's nanobind domain determines which extension modules share native Python +type registrations. All modules inside one PTOAS MLIR runtime must use one +domain, while other MLIR distributions must use different domains. + +Use: + +```cmake +set(MLIR_BINDINGS_PYTHON_NB_DOMAIN "ptoas_mlir") +``` + +### Current LLVM Fork Constraint + +The LLVM fork currently used by PTOAS has an older `AddMLIRPython.cmake` API. +Its `add_mlir_python_modules` function accepts `ROOT_PREFIX` and +`INSTALL_PREFIX`, but does not accept the newer per-call +`MLIR_BINDINGS_PYTHON_NB_DOMAIN` argument. It reads the CMake variable when it +creates nanobind extension targets. + +Therefore the current implementation must set the variable before calling +`add_mlir_python_modules`; passing an extra argument to that function is not a +valid implementation for the current SDK. When the LLVM fork adopts the newer +upstream API, this can be converted to a per-package argument without changing +the public layout. + +`ptoas._core` currently uses pybind11, so the nanobind domain does not directly +configure `_core`. Its interoperability with MLIR objects instead depends on +the package prefix, MLIR's adaptor/C API boundary, and the shared common CAPI. + +## Common CAPI and Runtime Identity + +`PTOASPythonCAPI` is already a project-owned common CAPI name and should remain +the single native binding runtime for PTOAS. The migration must preserve these +relationships: + +- `_mlir` links `PTOASPythonCAPI`. +- `_site_initialize_0` links the same common CAPI. +- `ptoas._core` links the same common CAPI. +- `ptoas._core` initializes the matching Python runtime with: + + ```cpp + py::module_::import("ptoas.mlir.ir"); + ``` + +- `ptoas.mlir.dialects.pto` imports `ptoas._core` for project-owned PTO types + and enums. +- No PTOAS component imports or links a second system `MLIRPythonCAPI`. + +`ptoas.__init__` should stay minimal and must not eagerly import `_core` or +`ptoas.mlir`, avoiding an import cycle between `_core`, `ir`, and the PTO +dialect facade. + +## Dynamic Library Isolation + +### Linux Repaired Wheels + +PTOAS currently builds its LLVM SDK with `BUILD_SHARED_LIBS=ON`. This produces +many `libLLVM*.so` and `libMLIR*.so` dependencies. The release wheel must +continue through `auditwheel repair`. + +Auditwheel provides the distribution isolation for those external libraries: + +1. Copy each non-system dependency into the wheel-owned `ptoas.libs` tree. +2. Rename it with a content-derived hash. +3. Rewrite its ELF SONAME. +4. Rewrite consumers' `DT_NEEDED` entries. +5. Install package-relative RPATH entries. + +PTOAS's repaired wheel already demonstrates this pattern with names such as: + +```text +ptoas.libs/libMLIRArithDialect-00278e1e.so.21.1 +ptoas.libs/libLLVMTransformUtils-2cda0d9b.so.21.1 +``` + +The project should not manually rename or enumerate all LLVM/MLIR libraries. +The internal common CAPI name is already project-specific; auditwheel owns the +external dependency names. + +### macOS Repaired Wheels + +The macOS wheel must continue through `delocate`. Delocate owns copying dylibs +and rewriting install names and RPATH entries. PTOAS should not add a parallel +`install_name_tool` implementation. + +### `_core` Package-Local RPATH + +After nesting MLIR under `ptoas`, `_core` and its common CAPI are laid out as: + +```text +ptoas/_core.so +ptoas/mlir/_mlir_libs/libPTOASPythonCAPI.so +``` + +The package-local install RPATH must therefore be: + +```text +Linux: $ORIGIN;$ORIGIN/mlir/_mlir_libs +macOS: @loader_path;@loader_path/mlir/_mlir_libs +``` + +The old `$ORIGIN/../mlir/_mlir_libs` path belongs to the current sibling +packages and must not survive the migration. + +For the common CAPI under `ptoas/mlir/_mlir_libs`, the +`RELATIVE_INSTALL_ROOT` value must be recomputed from the new location rather +than copied from the old configuration. From this location back to the Python +install prefix is normally three levels (`../../..`). The final value must be +verified from installed ELF/Mach-O metadata because auditwheel and delocate +will subsequently replace distribution-wheel dependency paths. + +### Editable and Build-Tree Installs + +Editable and direct build-tree installations do not pass through wheel repair. +They may use RPATH entries that refer to the selected developer LLVM build. +This is sufficient for a convenient PTOAS development environment, but it does +not guarantee same-process coexistence with an unrelated developer MLIR build. + +The supported isolation contract applies most strongly to repaired release +wheels. Editable multi-MLIR coexistence is best effort unless both projects +were intentionally built against one compatible runtime. + +### Compiler Compatibility Archive + +The `ptoas-bin` archive is compiler-oriented and includes PTODSL as an internal +compiler runtime dependency. PTOAS defaults to the PTODSL TileLib backend, so +excluding `ptodsl/` would make an extracted archive unable to compile `.pto` +input containing unexpanded TileOps with the normal CLI defaults. The archive +must therefore contain both PTODSL and the installed nested MLIR package: + +```text +ptodsl/ +ptoas/_core.so +ptoas/mlir/ +ptoas/mlir/_mlir_libs/_mlir.so +ptoas/mlir/_mlir_libs/libPTOASPythonCAPI.so +``` + +This does not make the archive a general PTODSL development distribution. +`ptodsl/` and `ptoas.mlir` are present so the compiler's default TileLib path +works without changing CLI semantics. The repaired wheel remains the supported +Python installation for authoring and importing PTODSL applications. + +The repaired wheel remains the canonical Python distribution. Wheel and +archive packaging share the same CMake target graph but use different install +contracts. The archive is produced from an explicit compiler-runtime install +component so that its contents are owned by CMake rather than reconstructed +from source paths: + +```text +configured CMake build + │ + ├── wheel install components + │ └── raw wheel + │ └── auditwheel / delocate + │ └── repaired Python wheel + │ + └── PTOAS_Python + compiler-runtime install components + └── archive staging prefix + └── native relocation, if required + └── tar/CPack board artifact +``` + +The staging prefix must contain only the compiler archive contract: + +```text +bin/ptoas +ptodsl/ +ptoas/ +ptoas/mlir/ +ptoas/_runtime/share/ptoas/TileOps/ +tilelang_dsl/ +lib/ # only if native dependencies are bundled +``` + +The archive exposes PTODSL only as part of the compiler runtime contract. It +does not include wheel metadata or claim to satisfy normal `pip` dependency +resolution. Board validation invokes the packaged compiler rather than using +the archive as an installable Python distribution. + +Archive assembly is limited to install-tree and relocation operations: + +- install `PTOAS_Python` and then `PTOAS_CompilerArchive` with an explicit, + shared staging prefix; +- preserve the package-relative `ptoas/`, `ptoas/mlir/`, and `ptodsl/` layout; +- place any bundled non-system native dependencies below the archive prefix + and use `$ORIGIN`/`@loader_path`-relative linkage; +- create the tarball from that prefix with CPack/TGZ or an equivalent simple + archive command; +- smoke test the extracted archive with a clean environment. + +The archive's `lib/` directory, when needed, is an install-tree convention and +is distinct from auditwheel's wheel-local `*.libs` sidecar. We must not copy a +wheel sidecar into an unrelated layout without also preserving the native +linkage that refers to it. + +Native dependency discovery for the archive should use CMake's install-time +`file(GET_RUNTIME_DEPENDENCIES)`, which is available within PTOAS's existing +CMake 3.20 minimum, rather than handwritten recursive `ldd` or `otool` +parsers. Package-owned files such as `PTOASPythonCAPI`, `_mlir`, and site +initializers remain under `ptoas/mlir/_mlir_libs` and must be excluded from the +external dependency copy into `lib/`. + +The archive relocation contract is: + +```text +Linux ptoas/_core.so: + $ORIGIN;$ORIGIN/mlir/_mlir_libs;$ORIGIN/../lib +Linux ptoas/mlir/_mlir_libs/*: + $ORIGIN;$ORIGIN/../../../lib +Linux lib/*.so*: + $ORIGIN + +macOS ptoas/_core.so: + @loader_path;@loader_path/mlir/_mlir_libs;@loader_path/../lib +macOS ptoas/mlir/_mlir_libs/*: + @loader_path;@loader_path/../../../lib +macOS lib/*.dylib: + @loader_path +``` + +Linux uses `patchelf` only to apply this staging-tree RPATH contract after +CMake resolves and copies dependencies. macOS uses `delocate-path` for the +equivalent directory relocation instead of maintaining a project-owned Mach-O +dependency parser. Neither platform searches for libraries at Python import +time. + +The current compatibility archive is built from the CPython 3.11 workflow and +contains CPython-ABI-specific extension modules. It therefore requires CPython +3.11 at runtime. The existing `ptoas-bin-` artifact names remain +unchanged for the external board-test monitor; the build and extracted-archive +tests must validate the interpreter version explicitly. Moving to abi3 or +publishing per-Python archive names is outside this migration. + +The single-runtime invariant is inherited from the shared CMake target graph: + +- exactly one package-owned `PTOASPythonCAPI` is retained; +- `_core`, `_mlir`, and site initializers link the same project-owned common + CAPI before the wheel and archive take their platform-specific packaging + paths; +- LLVM/MLIR dependencies in the archive use relative install-tree RPATHs or + install names and contain no build-tree absolute paths; +- any platform relocation helper is limited to the archive staging prefix and + is separately validated from the wheel repair path. + +The standalone archive is intended to launch a fresh compiler process. Unlike +the repaired wheel, it does not promise arbitrary same-process import +coexistence with another MLIR distribution. Its required invariant is narrower +but strict: one archive invocation loads one internally consistent PTOAS native +runtime built from the same CMake targets as the wheel and relocated only for +the archive's own install-tree layout. + +## Why Not Switch Directly to Static LLVM + +Changing only `BUILD_SHARED_LIBS=OFF` can make `_mlir.so` and `_core.so` +statically embed separate copies of MLIR. That would duplicate registries, +TypeIDs, command-line options, and native global state inside one process and +could break PTOAS's own cross-module object identity. + +A future single-DSO design must instead follow an IREE-style ownership model: + +```text +libPTOASCompiler.so +├── LLVM/MLIR implementation +├── PTO dialect and passes +├── compiler driver +└── exported C API + +_mlir.so ──► libPTOASCompiler.so +_core.so ──► libPTOASCompiler.so +``` + +That is a separate architectural change requiring a deliberate exported API +boundary and symbol-visibility policy. It is not a prerequisite for the +namespace migration because repaired wheels already isolate external shared +libraries with package-local, content-derived names. + +## Public API Migration + +Production imports, tests, samples, and documentation must move together: + +```python +# Old +from mlir import ir +from mlir.ir import Module +from mlir.dialects import pto + +# New +from ptoas.mlir import ir +from ptoas.mlir.ir import Module +from ptoas.mlir.dialects import pto +``` + +The affected surfaces include: + +- PTO dialect Python facade and generated source graph; +- PTODSL implementation and public MLIR object documentation; +- TileLang DSL pybind backend; +- PTOAS Python tests; +- sample programs; +- README installation and Python API examples; +- wheel payload validation and import smoke tests; +- native `mlir.ir` import strings. + +Because README and wheel tests currently promise top-level `mlir`, removing it +is a public API break and requires an explicit release note. A default alias is +not an acceptable compatibility mechanism because it defeats package +isolation. + +## Supported Coexistence Boundary + +| Scenario | Contract | +|---|---| +| Install PTOAS and another namespaced MLIR wheel in one environment | Supported target | +| Import both repaired wheels in one interpreter | Must be validated | +| Create independent contexts and modules in each runtime | Supported target | +| Pass a PTOAS `Module`, `Type`, or `Value` into another runtime | Unsupported | +| Exchange textual MLIR, bytecode, or files | Supported integration boundary | +| Mix editable builds backed by unrelated LLVM trees | Best effort, not guaranteed | + +No public function should imply that independently packaged MLIR native objects +are interchangeable. Cross-project integration must serialize the IR or use a +subprocess unless the projects deliberately share one compatible runtime. + +## Validation Plan + +### Package Contract + +- The wheel contains `ptoas/mlir/ir.py` and + `ptoas/mlir/dialects/pto.py`. +- The wheel does not contain a top-level `mlir/` tree. +- `ptoas._core` and exactly one PTOAS `_mlir` core extension are present. +- Importing PTOAS does not create `sys.modules["mlir"]`. + +### Functional Imports + +```python +from ptoas.mlir import ir +from ptoas.mlir.dialects import pto +import ptodsl +from ptodsl import pto as dsl_pto, scalar +``` + +The test must create a context and module, register the PTO dialect, compile a +minimal PTODSL kernel, and invoke the installed `ptoas` console script. + +### Runtime Identity + +- PTO types created through `ptoas._core` work with + `ptoas.mlir.ir.Context`. +- PTODSL returns `ptoas.mlir.ir.Module` objects. +- PTOAS binding modules all resolve the same `PTOASPythonCAPI`. +- Import order between `ptoas._core`, `ptoas.mlir.ir`, and + `ptoas.mlir.dialects.pto` is deterministic and cycle-free. + +### Linux Wheel Dependencies + +- `auditwheel show` reports an accepted manylinux policy. +- Repaired LLVM/MLIR dependencies have content-derived names. +- `readelf -d` contains no absolute LLVM build path. +- No non-system `libLLVM*` or `libMLIR*` dependency remains outside the wheel. + +### macOS Wheel Dependencies + +- `delocate-listdeps` reports no external developer LLVM path. +- `otool -L` contains only system or wheel-owned dylibs. +- No absolute LLVM build or install path remains. + +### Compiler Archive Dependencies + +- The archive contains `ptoas/mlir` and `ptodsl` so the default PTODSL TileLib + backend works without extra CLI options. +- The archive is produced by an explicit CMake compiler-runtime install + component, not by scanning the source tree or an arbitrary install tree. +- Package-relative paths are preserved and the archive contains no source, + build, or install-prefix absolute paths. +- Exactly one package-owned `PTOASPythonCAPI` exists. +- `_core`, `_mlir`, site initializers, and the common CAPI resolve through the + archive's own relative native paths. +- Any native dependency relocation is performed only on the staging prefix and + is not implemented by Python import-time path scanning. +- Runtime dependency discovery uses CMake + `file(GET_RUNTIME_DEPENDENCIES)` rather than recursive shell parsing. +- Package-owned MLIR extensions and `PTOASPythonCAPI` remain under + `ptoas/mlir/_mlir_libs`; only external native dependencies are copied to + the archive `lib/` directory. +- The archive reports a clear error unless it is launched with CPython 3.11. +- The extracted archive runs `ptoas --version` and compiles a representative + `.pto` input containing an unexpanded TileOp through the default PTODSL + backend with `PYTHONPATH`, `LD_LIBRARY_PATH`, and `DYLD_LIBRARY_PATH` unset. +- PR workflows continue to upload the artifact name consumed by the external + board-test monitor until that consumer is explicitly migrated. + +### Coexistence + +A packaging or release integration job should install one representative +namespaced MLIR distribution, preferably IREE, and test both import orders: + +```python +import ptoas.mlir.ir +import iree.compiler.ir +``` + +and: + +```python +import iree.compiler.ir +import ptoas.mlir.ir +``` + +Each runtime should create and verify its own context/module. The test must not +pass objects between runtimes. This heavier check belongs in wheel packaging or +release validation rather than every source-only PR job. + +### Repository Regression + +- Python binding and PTODSL tests; +- TileLang DSL backend tests; +- PTOAS lit tests; +- VMI lit and simulator regression where Python samples are involved; +- Linux and macOS wheel build, repair, clean-install, and CLI smoke tests. + +## Implementation Sequence + +1. Add `ptoas.mlir` package prefix and staging/install destinations. +2. Set the current LLVM fork's global nanobind domain to `ptoas_mlir`. +3. Keep and relocate the single `PTOASPythonCAPI`. +4. Update `_core` package import and package-local RPATH. +5. Add a dedicated compiler-runtime install component and explicit archive + staging prefix; remove source-tree and arbitrary-prefix discovery from the + collectors. +6. Include PTODSL in the compiler-runtime component so the default TileLib + backend remains functional. +7. Replace recursive native dependency parsing with install-time CMake + `file(GET_RUNTIME_DEPENDENCIES)`, then apply the documented Linux/macOS + staging-tree relocation contract. +8. Update the PTO dialect facade and all production Python imports. +9. Update samples, tests, README, workflow artifacts, and wheel/archive + contract validation without renaming the board-test artifact consumed by + the external monitor. +10. Remove top-level `mlir` from the wheel without a default alias. +11. Run build-tree, editable, lit, simulator, repaired-wheel, and extracted + standalone-archive regressions. +12. Validate dynamic dependencies and representative same-process wheel + coexistence. +13. Consider a monolithic `libPTOASCompiler` only as a later, separately + justified optimization for wheel size, startup time, or stronger symbol + isolation. + +## Decision Summary + +- Public namespace: `ptoas.mlir`. +- Public PTO dialect: `ptoas.mlir.dialects.pto`. +- Native compiler module: `ptoas._core`. +- MLIR Python package prefix: `ptoas.mlir.`. +- Nanobind domain: `ptoas_mlir`. +- Common native runtime: one `PTOASPythonCAPI`. +- Linux distribution isolation: `auditwheel repair`. +- macOS distribution isolation: `delocate`. +- Official Python distribution: repaired wheel with the `ptoas` console script. +- Board-validation artifact: compiler-only tarball from the CMake + compiler-runtime install component, preserving the nested `ptoas.mlir` + package layout, including PTODSL for the default TileLib backend, and using + relative native paths. +- Compiler archive Python ABI: CPython 3.11 until a separate abi3 or + per-Python archive design is adopted. +- Archive dependency discovery: CMake `file(GET_RUNTIME_DEPENDENCIES)`; + platform tools perform only staging-tree relocation. +- Board-test compatibility: retain the current `ptoas-bin` artifact contract + until the external monitor is migrated deliberately. +- Cross-runtime native objects: unsupported. +- Top-level `mlir` compatibility alias: not installed. +- Static or monolithic LLVM redesign: deferred to a separate proposal. diff --git a/docs/designs/ptoas-python-launcher-layout.md b/docs/designs/ptoas-python-launcher-layout.md index f85b9014e4..9d197029cd 100644 --- a/docs/designs/ptoas-python-launcher-layout.md +++ b/docs/designs/ptoas-python-launcher-layout.md @@ -14,6 +14,10 @@ This document records the internal launcher contract for the Python-backed `ptoas` command. The user-facing README should describe how to install and run PTOAS without exposing these implementation details. +The PTOAS-owned MLIR Python runtime lives under `ptoas.mlir`; its namespace and +native-library isolation requirements are recorded separately in +[`ptoas-mlir-namespace-and-native-isolation.md`](ptoas-mlir-namespace-and-native-isolation.md). + ## Entry Model The wheel launcher uses the standard Python console-script and native-extension @@ -28,7 +32,7 @@ both -> import ptoas._core -> ptoas._core.main(argv) `ptoas._core` is the single PTOAS-owned native extension. It provides the compiler entry point and the native PTO dialect bindings used by the public -`mlir.dialects.pto` facade. CMake and Python own the platform and ABI-specific +`ptoas.mlir.dialects.pto` facade. CMake and Python own the platform and ABI-specific filename. Launcher and packaging code refer to the import name and never construct `.so`, `.dylib`, or `.pyd` paths. @@ -109,15 +113,28 @@ bin/ptoas ptoas/_cli.py ptoas/_core..so ptoas/_runtime/share/ptoas/TileOps +ptoas/mlir/ +ptodsl/ lib/ tilelang_dsl/ ``` -The archive requires a Python interpreter compatible with the packaged -extension ABI. `bin/ptoas` adds the archive root to `sys.path`, then uses the -same `ptoas._cli -> ptoas._core` path as the install tree. Linux archives set -the extension runtime path to `$ORIGIN/../lib`; macOS archives rewrite the -extension's install names relative to its package directory. +The archive is assembled by installing `PTOAS_Python` and then +`PTOAS_CompilerArchive` into one staging prefix. The second component owns the +wrapper and compiler-time Python resources and performs relocation against the +already staged native payload. Packaging code does not scan source trees or +assemble Python packages from unrelated build directories. + +The current archive is built against CPython 3.11 and requires a CPython 3.11 +interpreter. `bin/ptoas` adds the archive root to `sys.path`, then uses the same +`ptoas._cli -> ptoas._core` path as the install tree. The packaged `ptodsl/` +tree supports the compiler's default PTODSL TileLib backend; it does not turn +the archive into a normal pip-installable PTODSL distribution. + +Linux archives use package-relative and archive-relative `$ORIGIN` RPATHs; +macOS archives use the equivalent `@loader_path` install names. Package-owned +MLIR extensions remain under `ptoas/mlir/_mlir_libs`, while only external +native dependencies are collected under the archive `lib/` directory. ## PTODSL and TileLib Runtime diff --git a/lib/Bindings/Python/CMakeLists.txt b/lib/Bindings/Python/CMakeLists.txt index ba950c649b..0c56fef22e 100644 --- a/lib/Bindings/Python/CMakeLists.txt +++ b/lib/Bindings/Python/CMakeLists.txt @@ -8,6 +8,13 @@ include(AddMLIRPython) +# Keep the complete PTOAS-owned MLIR Python runtime below the project namespace +# so it can coexist with independently packaged MLIR distributions. This is a +# package-prefix contract as well as a Python import path: MLIR's generated +# capsules and nanobind registration code use the same value. +add_compile_definitions("MLIR_PYTHON_PACKAGE_PREFIX=ptoas.mlir.") +set(MLIR_BINDINGS_PYTHON_NB_DOMAIN "ptoas_mlir") + # Do not carry toolchain-injected paths, such as a Conda compiler RPATH, into # installed Python modules. Distribution builds keep only MLIR's relative # package RPATHs; editable builds add their external link paths below. @@ -33,11 +40,11 @@ if(PTOAS_ENABLE_WERROR) endif() endif() -# Build the `mlir` Python package from MLIR's exported source declarations and +# Build the `ptoas.mlir` Python package from MLIR's exported source declarations and # add PTOAS's dialect to the same package. This is the composition mechanism # supported by AddMLIRPython and avoids copying or overlaying another build's # package tree. -set(PTOAS_MLIR_PYTHON_PACKAGE_DIR "${CMAKE_BINARY_DIR}/python/mlir") +set(PTOAS_MLIR_PYTHON_PACKAGE_DIR "${CMAKE_BINARY_DIR}/python/ptoas/mlir") declare_mlir_python_sources(PTOASPythonSources) @@ -104,9 +111,9 @@ set(PTOAS_MLIR_PYTHON_SOURCE_SETS # CAPI and contains both the PTO dialect implementation and compiler driver. add_mlir_python_common_capi_library(PTOASPythonCAPI INSTALL_COMPONENT PTOAS_Python - INSTALL_DESTINATION "mlir/_mlir_libs" + INSTALL_DESTINATION "ptoas/mlir/_mlir_libs" OUTPUT_DIRECTORY "${PTOAS_MLIR_PYTHON_PACKAGE_DIR}/_mlir_libs" - RELATIVE_INSTALL_ROOT "../../../.." + RELATIVE_INSTALL_ROOT "../../../" DECLARED_HEADERS MLIRPythonCAPI.HeaderSources DECLARED_SOURCES @@ -125,7 +132,7 @@ set_property(TARGET PTOASPythonCAPI PROPERTY LINKER_LANGUAGE CXX) add_mlir_python_modules(PTOAS_Python ROOT_PREFIX "${PTOAS_MLIR_PYTHON_PACKAGE_DIR}" - INSTALL_PREFIX mlir + INSTALL_PREFIX ptoas/mlir DECLARED_SOURCES ${PTOAS_MLIR_PYTHON_SOURCE_SETS} PTOASPythonSources diff --git a/lib/Bindings/Python/PTOModule.cpp b/lib/Bindings/Python/PTOModule.cpp index 8d6d061b29..b9abb1cc30 100644 --- a/lib/Bindings/Python/PTOModule.cpp +++ b/lib/Bindings/Python/PTOModule.cpp @@ -9,7 +9,7 @@ //===- DialectPTO.cpp -----------------------------------------------------===// // // Python bindings for the PTO dialect types in the project-owned ptoas._core -// extension. The public Python facade remains mlir.dialects.pto. +// extension. The public Python facade remains ptoas.mlir.dialects.pto. // //===----------------------------------------------------------------------===// diff --git a/ptodsl/CMakeLists.txt b/ptodsl/CMakeLists.txt index 4be1601179..c8d1e9ba25 100644 --- a/ptodsl/CMakeLists.txt +++ b/ptodsl/CMakeLists.txt @@ -29,7 +29,7 @@ add_dependencies(PTODSLPackage PTODSLPackageStage) install( DIRECTORY "${PTODSL_PACKAGE_SRC_DIR}" DESTINATION "." - COMPONENT PTOAS_Runtime + COMPONENT PTOAS_CompilerArchive PATTERN "__pycache__" EXCLUDE PATTERN "*.pyc" EXCLUDE ) @@ -37,7 +37,7 @@ install( install( DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/ptoas" DESTINATION "." - COMPONENT PTOAS_Runtime + COMPONENT PTOAS_CompilerArchive PATTERN "__pycache__" EXCLUDE PATTERN "*.pyc" EXCLUDE ) diff --git a/ptodsl/examples/softmax_lowlevel.py b/ptodsl/examples/softmax_lowlevel.py index b962551fab..4d673ca93d 100644 --- a/ptodsl/examples/softmax_lowlevel.py +++ b/ptodsl/examples/softmax_lowlevel.py @@ -14,7 +14,7 @@ using raw MLIR Python binding calls, with no additional abstraction layer. """ -from mlir.ir import ( +from ptoas.mlir.ir import ( Attribute, Context, F32Type, @@ -28,7 +28,7 @@ Type, UnitAttr, ) -from mlir.dialects import arith, func, pto, scf +from ptoas.mlir.dialects import arith, func, pto, scf def build(): diff --git a/ptodsl/examples/tadd_lowlevel.py b/ptodsl/examples/tadd_lowlevel.py index 8dd98c77dc..fbae54c7b8 100644 --- a/ptodsl/examples/tadd_lowlevel.py +++ b/ptodsl/examples/tadd_lowlevel.py @@ -41,7 +41,7 @@ } """ -from mlir.ir import ( +from ptoas.mlir.ir import ( Attribute, Context, F32Type, @@ -54,7 +54,7 @@ StringAttr, Type, ) -from mlir.dialects import arith, func, pto, scf +from ptoas.mlir.dialects import arith, func, pto, scf def build(): diff --git a/ptodsl/ptodsl/_allreduce.py b/ptodsl/ptodsl/_allreduce.py index 4d14b540c0..24080cb72d 100644 --- a/ptodsl/ptodsl/_allreduce.py +++ b/ptodsl/ptodsl/_allreduce.py @@ -28,8 +28,8 @@ from ._surface_values import unwrap_surface_value from ._types import _resolve, float16 as _f16_dtype, float32 as _f32_dtype, si32 as _si32_dtype, ui32 as _ui32_dtype -from mlir.dialects import pto as _pto -from mlir.ir import F16Type, F32Type, IntegerType +from ptoas.mlir.dialects import pto as _pto +from ptoas.mlir.ir import F16Type, F32Type, IntegerType # ── helpers ──────────────────────────────────────────────────────────────────── diff --git a/ptodsl/ptodsl/_builtin_vector.py b/ptodsl/ptodsl/_builtin_vector.py index 544b66470b..d1cfa5bac2 100644 --- a/ptodsl/ptodsl/_builtin_vector.py +++ b/ptodsl/ptodsl/_builtin_vector.py @@ -11,9 +11,9 @@ from ._surface_values import VecValue, unwrap_surface_value from ._types import _resolve, _validate_vec_size, vec_type -from mlir.dialects import arith -from mlir.dialects import llvm -from mlir.ir import IntegerType, VectorType +from ptoas.mlir.dialects import arith +from ptoas.mlir.dialects import llvm +from ptoas.mlir.ir import IntegerType, VectorType def Vec(dtype, size: int, *, init=None): diff --git a/ptodsl/ptodsl/_context.py b/ptodsl/ptodsl/_context.py index efdc47af59..a3f2fd7be4 100644 --- a/ptodsl/ptodsl/_context.py +++ b/ptodsl/ptodsl/_context.py @@ -8,14 +8,14 @@ """MLIR context construction for an installed or explicitly configured PTODSL.""" -from mlir.dialects import pto as _pto_dialect +from ptoas.mlir.dialects import pto as _pto_dialect try: - from mlir.dialects import llvm as _llvm_dialect + from ptoas.mlir.dialects import llvm as _llvm_dialect except Exception: # pragma: no cover - depends on the installed MLIR package. _llvm_dialect = None -from mlir.ir import Context +from ptoas.mlir.ir import Context def make_context() -> Context: diff --git a/ptodsl/ptodsl/_control_flow.py b/ptodsl/ptodsl/_control_flow.py index e412d96b58..33d862deb8 100644 --- a/ptodsl/ptodsl/_control_flow.py +++ b/ptodsl/ptodsl/_control_flow.py @@ -28,8 +28,8 @@ from ._tracing.active import current_session from ._surface_values import unwrap_surface_value, wrap_like_surface_value, wrap_surface_value -from mlir.dialects import pto as _pto, scf -from mlir.ir import InsertionPoint +from ptoas.mlir.dialects import pto as _pto, scf +from ptoas.mlir.ir import InsertionPoint # ── vecscope ────────────────────────────────────────────────────────────────── diff --git a/ptodsl/ptodsl/_jit.py b/ptodsl/ptodsl/_jit.py index 26fb8311e5..f90f7809ac 100644 --- a/ptodsl/ptodsl/_jit.py +++ b/ptodsl/ptodsl/_jit.py @@ -28,7 +28,7 @@ current_runtime, ) -from mlir.ir import InsertionPoint +from ptoas.mlir.ir import InsertionPoint _MODULE_ATTRS = ("pto.target_arch",) diff --git a/ptodsl/ptodsl/_ops.py b/ptodsl/ptodsl/_ops.py index df07382fbe..403d3c8ef9 100644 --- a/ptodsl/ptodsl/_ops.py +++ b/ptodsl/ptodsl/_ops.py @@ -72,8 +72,8 @@ vreg_type, ) -from mlir.dialects import arith, pto as _pto -from mlir.ir import ( +from ptoas.mlir.dialects import arith, pto as _pto +from ptoas.mlir.ir import ( Attribute, BF16Type, F16Type, @@ -235,7 +235,7 @@ def const(value: int, *, dtype=None): """ Emit an ``arith.constant``. - ``dtype`` is a ``_DType`` descriptor or a concrete ``mlir.ir.Type``. + ``dtype`` is a ``_DType`` descriptor or a concrete ``ptoas.mlir.ir.Type``. Defaults to ``index`` when omitted. """ from ._types import index as _idx_dtype diff --git a/ptodsl/ptodsl/_runtime/codegen.py b/ptodsl/ptodsl/_runtime/codegen.py index 768a11155b..f0fa1a4e83 100644 --- a/ptodsl/ptodsl/_runtime/codegen.py +++ b/ptodsl/ptodsl/_runtime/codegen.py @@ -10,7 +10,7 @@ from __future__ import annotations from .._context import make_context -from mlir.ir import BF16Type, F16Type, F32Type, IndexType, IntegerType +from ptoas.mlir.ir import BF16Type, F16Type, F32Type, IndexType, IntegerType from .._kernel_signature import DeviceParameterSpec, RuntimeScalarParameterSpec from .._types import _PtrDescriptor, _resolve diff --git a/ptodsl/ptodsl/_runtime/launch.py b/ptodsl/ptodsl/_runtime/launch.py index 5780fe1a93..d6e5c26402 100644 --- a/ptodsl/ptodsl/_runtime/launch.py +++ b/ptodsl/ptodsl/_runtime/launch.py @@ -16,7 +16,7 @@ from .._types import _resolve from .native_build import build_native_library -from mlir.ir import BF16Type, Context, F16Type, F32Type, IndexType, IntegerType +from ptoas.mlir.ir import BF16Type, Context, F16Type, F32Type, IndexType, IntegerType if TYPE_CHECKING: from .._kernel_compilation import CompiledKernelHandle diff --git a/ptodsl/ptodsl/_runtime_scalar_ops.py b/ptodsl/ptodsl/_runtime_scalar_ops.py index fa0acf0a7b..0fef5843a1 100644 --- a/ptodsl/ptodsl/_runtime_scalar_ops.py +++ b/ptodsl/ptodsl/_runtime_scalar_ops.py @@ -19,8 +19,8 @@ _strip_integer_signedness, ) -from mlir.dialects import arith, math -from mlir.ir import IndexType, IntegerType +from ptoas.mlir.dialects import arith, math +from ptoas.mlir.ir import IndexType, IntegerType _FLOAT_BINARY_OPS = { diff --git a/ptodsl/ptodsl/_scalar_adaptation.py b/ptodsl/ptodsl/_scalar_adaptation.py index faa1fdb8b4..4457c693db 100644 --- a/ptodsl/ptodsl/_scalar_adaptation.py +++ b/ptodsl/ptodsl/_scalar_adaptation.py @@ -17,8 +17,8 @@ _strip_integer_signedness, ) -from mlir.dialects import arith -from mlir.ir import BF16Type, F16Type, F32Type, FloatAttr, IndexType, IntegerType, VectorType +from ptoas.mlir.dialects import arith +from ptoas.mlir.ir import BF16Type, F16Type, F32Type, FloatAttr, IndexType, IntegerType, VectorType def classify_runtime_scalar_type(type_obj): diff --git a/ptodsl/ptodsl/_source_loader.py b/ptodsl/ptodsl/_source_loader.py index f4bac6f730..1603d0fc0f 100644 --- a/ptodsl/ptodsl/_source_loader.py +++ b/ptodsl/ptodsl/_source_loader.py @@ -20,7 +20,7 @@ jit_source_file_error, ) -from mlir.ir import Location, Module +from ptoas.mlir.ir import Location, Module @dataclass(frozen=True) diff --git a/ptodsl/ptodsl/_surface_types.py b/ptodsl/ptodsl/_surface_types.py index 9ec4d95142..0074c10e62 100644 --- a/ptodsl/ptodsl/_surface_types.py +++ b/ptodsl/ptodsl/_surface_types.py @@ -9,7 +9,7 @@ from enum import Enum -from mlir.dialects import pto as _pto +from ptoas.mlir.dialects import pto as _pto class _ConstExprHelper: diff --git a/ptodsl/ptodsl/_surface_values.py b/ptodsl/ptodsl/_surface_values.py index d541e776de..d8fa1fb0e3 100644 --- a/ptodsl/ptodsl/_surface_values.py +++ b/ptodsl/ptodsl/_surface_values.py @@ -22,10 +22,10 @@ from ._surface_types import PartitionTensorView, TensorView, Tile from ._types import _normalize_address_space, _resolve, ptr -from mlir.dialects import arith -from mlir.dialects import memref -from mlir.dialects import pto as _pto -from mlir.ir import IndexType, IntegerAttr, IntegerType, MemRefType, ShapedType, StridedLayoutAttr, Type, VectorType +from ptoas.mlir.dialects import arith +from ptoas.mlir.dialects import memref +from ptoas.mlir.dialects import pto as _pto +from ptoas.mlir.ir import IndexType, IntegerAttr, IntegerType, MemRefType, ShapedType, StridedLayoutAttr, Type, VectorType def _validate_surface_value_access(value): diff --git a/ptodsl/ptodsl/_tile_template_tracing.py b/ptodsl/ptodsl/_tile_template_tracing.py index 571239717d..ea10cf2582 100644 --- a/ptodsl/ptodsl/_tile_template_tracing.py +++ b/ptodsl/ptodsl/_tile_template_tracing.py @@ -58,8 +58,8 @@ vreg_type as _vreg_type, ) -from mlir.dialects import arith, pto as _pto, scf -from mlir.ir import InsertionPoint, IntegerType, Type +from ptoas.mlir.dialects import arith, pto as _pto, scf +from ptoas.mlir.ir import InsertionPoint, IntegerType, Type @dataclass(frozen=True) diff --git a/ptodsl/ptodsl/_tracing/artifacts.py b/ptodsl/ptodsl/_tracing/artifacts.py index 5dc51e3feb..975efe05b4 100644 --- a/ptodsl/ptodsl/_tracing/artifacts.py +++ b/ptodsl/ptodsl/_tracing/artifacts.py @@ -27,7 +27,7 @@ def __init__(self, py_name: str, *, module=None, module_factory=None, mlir_text: self._build_metadata = {} def build(self): - """Return the cached ``mlir.ir.Module``.""" + """Return the cached ``ptoas.mlir.ir.Module``.""" if self._cached_module is None: if self._module_factory is None: raise RuntimeError(f"{self._py_name} has no module factory") @@ -43,7 +43,7 @@ def build(self): return self._cached_module def mlir_module(self): - """Return the cached ``mlir.ir.Module``.""" + """Return the cached ``ptoas.mlir.ir.Module``.""" return self.build() def mlir_text(self) -> str: diff --git a/ptodsl/ptodsl/_tracing/control_flow.py b/ptodsl/ptodsl/_tracing/control_flow.py index 60a2c4ec84..f878e9198e 100644 --- a/ptodsl/ptodsl/_tracing/control_flow.py +++ b/ptodsl/ptodsl/_tracing/control_flow.py @@ -14,9 +14,9 @@ from .._runtime_index_ops import coerce_runtime_index from .._surface_values import unwrap_surface_value -from mlir.dialects import arith -from mlir.dialects import scf -from mlir.ir import InsertionPoint, IntegerType +from ptoas.mlir.dialects import arith +from ptoas.mlir.dialects import scf +from ptoas.mlir.ir import InsertionPoint, IntegerType @dataclass diff --git a/ptodsl/ptodsl/_tracing/module_builder.py b/ptodsl/ptodsl/_tracing/module_builder.py index e5fb99859a..9894ceb62e 100644 --- a/ptodsl/ptodsl/_tracing/module_builder.py +++ b/ptodsl/ptodsl/_tracing/module_builder.py @@ -12,8 +12,8 @@ from dataclasses import dataclass from enum import Enum -from mlir.dialects import func -from mlir.ir import Attribute, InsertionPoint, Module, Operation, StringAttr, UnitAttr +from ptoas.mlir.dialects import func +from ptoas.mlir.ir import Attribute, InsertionPoint, Module, Operation, StringAttr, UnitAttr class ModuleStyle(str, Enum): diff --git a/ptodsl/ptodsl/_tracing/runtime.py b/ptodsl/ptodsl/_tracing/runtime.py index 4a22c6b462..d9172d95e2 100644 --- a/ptodsl/ptodsl/_tracing/runtime.py +++ b/ptodsl/ptodsl/_tracing/runtime.py @@ -16,8 +16,8 @@ from .._context import make_context from .._types import _resolve -from mlir.dialects import func -from mlir.ir import InsertionPoint, Location +from ptoas.mlir.dialects import func +from ptoas.mlir.ir import InsertionPoint, Location class TracingRuntime: diff --git a/ptodsl/ptodsl/_tracing/session.py b/ptodsl/ptodsl/_tracing/session.py index 617fc05578..e5aa3a2ca0 100644 --- a/ptodsl/ptodsl/_tracing/session.py +++ b/ptodsl/ptodsl/_tracing/session.py @@ -36,9 +36,9 @@ from .._types import _strip_integer_signedness from .module_builder import create_container_child_module -from mlir.dialects import arith, func -from mlir.dialects import pto as _pto -from mlir.ir import ( +from ptoas.mlir.dialects import arith, func +from ptoas.mlir.dialects import pto as _pto +from ptoas.mlir.ir import ( Attribute, FlatSymbolRefAttr, IndexType, diff --git a/ptodsl/ptodsl/_types.py b/ptodsl/ptodsl/_types.py index b01ec8d056..4cec4393cd 100644 --- a/ptodsl/ptodsl/_types.py +++ b/ptodsl/ptodsl/_types.py @@ -9,7 +9,7 @@ Lazy MLIR type descriptors and eager type constructors. Type descriptors (``_DType`` subclasses) can be created *before* any MLIR -Context exists – they only resolve to concrete ``mlir.ir.Type`` objects when +Context exists – they only resolve to concrete ``ptoas.mlir.ir.Type`` objects when ``_resolve()`` is called inside an active context. This lets users write:: def softmax(arg0: pto.ptr(pto.float32, "GM"), ...): @@ -19,10 +19,10 @@ def softmax(arg0: pto.ptr(pto.float32, "GM"), ...): the actual type is materialised later by the ``@pto.jit`` decorator. """ -from mlir.dialects import pto as _pto -from mlir.dialects import arith -from mlir.dialects.builtin import UnrealizedConversionCastOp -from mlir.ir import ( +from ptoas.mlir.dialects import pto as _pto +from ptoas.mlir.dialects import arith +from ptoas.mlir.dialects.builtin import UnrealizedConversionCastOp +from ptoas.mlir.ir import ( BF16Type, F16Type, F32Type, @@ -216,10 +216,10 @@ def _validate_vec_size(size: int, *, context: str) -> int: def _resolve(dtype) -> Type: - """Coerce a ``_DType`` descriptor or a concrete ``mlir.ir.Type`` to a Type.""" + """Coerce a ``_DType`` descriptor or a concrete ``ptoas.mlir.ir.Type`` to a Type.""" if isinstance(dtype, _DType): return dtype.resolve() - return dtype # already an mlir.ir.Type + return dtype # already a ptoas.mlir.ir.Type def _classify_scalar_type(type_obj): diff --git a/ptodsl/ptodsl/_vmi_namespace.py b/ptodsl/ptodsl/_vmi_namespace.py index b22287b23c..aef05c8d82 100644 --- a/ptodsl/ptodsl/_vmi_namespace.py +++ b/ptodsl/ptodsl/_vmi_namespace.py @@ -11,8 +11,8 @@ from collections.abc import Sequence -from mlir.dialects import pto as _pto -from mlir.ir import BF16Type, F16Type, F32Type, Float8E4M3FNType, Float8E5M2Type, IntegerType, MemRefType +from ptoas.mlir.dialects import pto as _pto +from ptoas.mlir.ir import BF16Type, F16Type, F32Type, Float8E4M3FNType, Float8E5M2Type, IntegerType, MemRefType from ._scalar_coercion import coerce_scalar_to_type from ._surface_values import _coerce_index_value, _try_get_constant_index, unwrap_surface_value, wrap_surface_value diff --git a/ptodsl/ptodsl/pto.py b/ptodsl/ptodsl/pto.py index 9c6279d7c7..c9a3156edf 100644 --- a/ptodsl/ptodsl/pto.py +++ b/ptodsl/ptodsl/pto.py @@ -17,7 +17,7 @@ from ptodsl import pto All user-facing symbols live here. Low-level MLIR bindings are accessed -internally as ``_pto`` (``from mlir.dialects import pto as _pto``). +internally as ``_pto`` (``from ptoas.mlir.dialects import pto as _pto``). """ from ._diagnostics import unsupported_public_surface_error diff --git a/ptodsl/ptodsl/scalar.py b/ptodsl/ptodsl/scalar.py index 3e18343b8e..a646fe728f 100644 --- a/ptodsl/ptodsl/scalar.py +++ b/ptodsl/ptodsl/scalar.py @@ -9,7 +9,7 @@ Scalar arithmetic helpers – exposed as top-level ``scalar.*`` from the ``ptodsl`` package (for example ``from ptodsl import scalar``). -Arithmetic helpers operate on raw ``mlir.ir.Value`` objects and emit the +Arithmetic helpers operate on raw ``ptoas.mlir.ir.Value`` objects and emit the corresponding arith dialect operations at the active insertion point. Scalar memory helpers (`load` / `store`) also accept PTODSL surface-level address views such as `tile[row, col]` and `tile.as_ptr() + offset`. @@ -33,11 +33,11 @@ ) from ._types import _resolve -from mlir.dialects import arith -from mlir.dialects import llvm -from mlir.dialects import math -from mlir.ir import IndexType, IntegerType, MemRefType, Operation, VectorType -from mlir.dialects import pto as _pto +from ptoas.mlir.dialects import arith +from ptoas.mlir.dialects import llvm +from ptoas.mlir.dialects import math +from ptoas.mlir.ir import IndexType, IntegerType, MemRefType, Operation, VectorType +from ptoas.mlir.dialects import pto as _pto def muli(lhs, rhs): diff --git a/ptodsl/ptodsl/tilelib/_render_runtime.py b/ptodsl/ptodsl/tilelib/_render_runtime.py index b87dafe8ae..3aa6fed25b 100644 --- a/ptodsl/ptodsl/tilelib/_render_runtime.py +++ b/ptodsl/ptodsl/tilelib/_render_runtime.py @@ -36,8 +36,8 @@ from .._surface_values import wrap_surface_value from .._types import _DType, _resolve -from mlir.dialects import func -from mlir.ir import Attribute, InsertionPoint, Location, Module, StringAttr, UnitAttr +from ptoas.mlir.dialects import func +from ptoas.mlir.ir import Attribute, InsertionPoint, Location, Module, StringAttr, UnitAttr # ── tile handle handed to the template body ──────────────────────────────────────── diff --git a/ptodsl/ptodsl/tilelib/metadata.py b/ptodsl/ptodsl/tilelib/metadata.py index 9ebaef8877..94ed62903b 100644 --- a/ptodsl/ptodsl/tilelib/metadata.py +++ b/ptodsl/ptodsl/tilelib/metadata.py @@ -40,7 +40,7 @@ _resolve, ) -from mlir.ir import Type +from ptoas.mlir.ir import Type @dataclass(frozen=True) diff --git a/ptodsl/ptodsl/tilelib/templates/a5/tcvt.py b/ptodsl/ptodsl/tilelib/templates/a5/tcvt.py index 17f65cc02d..e54a360d79 100644 --- a/ptodsl/ptodsl/tilelib/templates/a5/tcvt.py +++ b/ptodsl/ptodsl/tilelib/templates/a5/tcvt.py @@ -10,7 +10,7 @@ from ptodsl import pto from ptodsl._surface_values import unwrap_surface_value, wrap_surface_value import ptodsl.tilelib as tilelib -from mlir.dialects import pto as _pto +from ptoas.mlir.dialects import pto as _pto def _rowwise(src_shape, src_valid_shape, dst_shape, dst_valid_shape, src_config, dst_config, **_): diff --git a/ptodsl/tests/support/docs_fragment_fixtures.py b/ptodsl/tests/support/docs_fragment_fixtures.py index 9aed2ff2ea..85ad735759 100644 --- a/ptodsl/tests/support/docs_fragment_fixtures.py +++ b/ptodsl/tests/support/docs_fragment_fixtures.py @@ -2111,8 +2111,8 @@ def pipe_communication_c2v_local_declaration_probe(): ), "pipe_communication.c2v_local_import": _fixture( f""" - from mlir.dialects import func - from mlir.ir import InsertionPoint + from ptoas.mlir.dialects import func + from ptoas.mlir.ir import InsertionPoint from ptodsl._tracing import current_session @@ -2135,8 +2135,8 @@ def pipe_communication_c2v_local_import_probe(): ), "pipe_communication.c2v_local_producer": _fixture( f""" - from mlir.dialects import func - from mlir.ir import InsertionPoint + from ptoas.mlir.dialects import func + from ptoas.mlir.ir import InsertionPoint from ptodsl._tracing import current_session diff --git a/ptodsl/tests/test_ast_rewrite_example_ir.py b/ptodsl/tests/test_ast_rewrite_example_ir.py index bc6d6242cc..b8e3de51b9 100644 --- a/ptodsl/tests/test_ast_rewrite_example_ir.py +++ b/ptodsl/tests/test_ast_rewrite_example_ir.py @@ -15,7 +15,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] from ptodsl import pto from ptodsl._context import make_context -from mlir.ir import Module +from ptoas.mlir.ir import Module def expect(condition: bool, message: str) -> None: diff --git a/ptodsl/tests/test_docs_as_test.py b/ptodsl/tests/test_docs_as_test.py index 7e03b8debe..5f3f70dcd6 100644 --- a/ptodsl/tests/test_docs_as_test.py +++ b/ptodsl/tests/test_docs_as_test.py @@ -26,7 +26,7 @@ from ptodsl._context import make_context from ptodsl._runtime.launch import LaunchHandle, _marshal_launch_args from ptodsl._runtime.toolchain import resolve_ptoas_binary -from mlir.ir import Module +from ptoas.mlir.ir import Module from support.docs_fragment_fixtures import FRAGMENT_FIXTURES, render_fragment_fixture FENCE_RE = re.compile(r"^```(?P[A-Za-z0-9_+-]*)\s*$") diff --git a/ptodsl/tests/test_flash_attention_demo_compile.py b/ptodsl/tests/test_flash_attention_demo_compile.py index 00bb6eadf1..d022103a8d 100644 --- a/ptodsl/tests/test_flash_attention_demo_compile.py +++ b/ptodsl/tests/test_flash_attention_demo_compile.py @@ -12,7 +12,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] from ptodsl._context import make_context -from mlir.ir import Module +from ptoas.mlir.ir import Module def expect(condition: bool, message: str) -> None: diff --git a/ptodsl/tests/test_jit_compile.py b/ptodsl/tests/test_jit_compile.py index a4f93dc90e..05b0982b95 100644 --- a/ptodsl/tests/test_jit_compile.py +++ b/ptodsl/tests/test_jit_compile.py @@ -30,7 +30,7 @@ from ptodsl._runtime.codegen import generate_launch_cpp from ptodsl._runtime.launch import _marshal_launch_args from ptodsl._tracing import current_session -from mlir.ir import InsertionPoint, Location, Module +from ptoas.mlir.ir import InsertionPoint, Location, Module def expect(condition: bool, message: str) -> None: diff --git a/ptodsl/tests/test_ptoas_frontend_verify.py b/ptodsl/tests/test_ptoas_frontend_verify.py index b3e80a4687..c8403171df 100644 --- a/ptodsl/tests/test_ptoas_frontend_verify.py +++ b/ptodsl/tests/test_ptoas_frontend_verify.py @@ -19,7 +19,7 @@ from ptodsl import scalar from ptodsl._context import make_context from ptodsl._runtime.toolchain import resolve_ptoas_binary -from mlir.ir import Module +from ptoas.mlir.ir import Module def expect(condition: bool, message: str) -> None: diff --git a/ptodsl/tests/test_ptoas_tree_wrapper.py b/ptodsl/tests/test_ptoas_tree_wrapper.py index 5bea72f01c..e289172a11 100644 --- a/ptodsl/tests/test_ptoas_tree_wrapper.py +++ b/ptodsl/tests/test_ptoas_tree_wrapper.py @@ -123,6 +123,31 @@ def test_wrapper_requires_cli_module(self): with self.assertRaisesRegex(SystemExit, "ptoas/_cli.py"): module._require_python_root(python_root, context="test") + def test_archive_wrapper_rejects_a_different_python_minor(self): + with tempfile.TemporaryDirectory() as temp_dir: + python_root = Path(temp_dir) / "archive" + wrapper = python_root / "bin" / "ptoas" + self._make_package(python_root) + wrapper.parent.mkdir(parents=True) + wrapper.write_text("", encoding="utf-8") + (python_root / ".ptoas-python-version").write_text( + "0.0\n", encoding="utf-8" + ) + + module = self._load_wrapper( + wrapper, + "test_ptoas_archive_python_version", + python_root_mode="wrapper-relative", + python_root=Path(".."), + ) + saved_argv = list(sys.argv) + try: + sys.argv = [str(wrapper)] + with self.assertRaisesRegex(SystemExit, "requires CPython 0.0"): + module.main() + finally: + sys.argv = saved_argv + if __name__ == "__main__": unittest.main() diff --git a/ptodsl/tests/test_tileop.py b/ptodsl/tests/test_tileop.py index 6b30cb88fc..1496439638 100644 --- a/ptodsl/tests/test_tileop.py +++ b/ptodsl/tests/test_tileop.py @@ -11,7 +11,7 @@ from ptodsl import pto from ptodsl._context import make_context -from mlir.ir import Module +from ptoas.mlir.ir import Module @pto.simt(max_threads=64, ast_rewrite=False) diff --git a/ptodsl/tests/test_vector_cube_ops.py b/ptodsl/tests/test_vector_cube_ops.py index ae49695047..ddfa884d0a 100644 --- a/ptodsl/tests/test_vector_cube_ops.py +++ b/ptodsl/tests/test_vector_cube_ops.py @@ -15,7 +15,7 @@ import ptodsl._pipe_namespace as _pipe_namespace from ptodsl._context import make_context from ptodsl import pto -from mlir.ir import F32Type +from ptoas.mlir.ir import F32Type def _identity(value): return value diff --git a/python/pto/dialects/pto.py b/python/pto/dialects/pto.py index f2038d060a..83709ba68a 100644 --- a/python/pto/dialects/pto.py +++ b/python/pto/dialects/pto.py @@ -9,7 +9,7 @@ import functools from typing import Optional -from mlir import ir as _ods_ir +from ptoas.mlir import ir as _ods_ir from . import _pto_ops_gen as _pto_ops_gen from ._ods_common import ( @@ -1138,9 +1138,9 @@ def _project_result(group, index, ty): def _load_standard_dialects(): try: - from mlir.dialects import arith as _mlir_arith # noqa: F401 - from mlir.dialects import func as _mlir_func # noqa: F401 - from mlir.dialects import scf as _mlir_scf # noqa: F401 + from ptoas.mlir.dialects import arith as _mlir_arith # noqa: F401 + from ptoas.mlir.dialects import func as _mlir_func # noqa: F401 + from ptoas.mlir.dialects import scf as _mlir_scf # noqa: F401 except ImportError as exc: raise RuntimeError("mlir standard dialect python bindings are required for vkernel parsing") from exc diff --git a/scripts/sim_dsl.sh b/scripts/sim_dsl.sh index 63dbf0b8ad..33d2cbec5e 100755 --- a/scripts/sim_dsl.sh +++ b/scripts/sim_dsl.sh @@ -129,7 +129,7 @@ print_msprof_log() { python_can_import_ptodsl() { local python_bin="$1" "${python_bin}" - <<'PY' >/dev/null 2>&1 -import mlir.ir # noqa: F401 +import ptoas.mlir.ir # noqa: F401 from ptodsl import pto # noqa: F401 PY } @@ -232,7 +232,7 @@ if [[ -n "${RESOLVED_PTOAS_BIN:-}" ]]; then export PTOAS_BIN="${RESOLVED_PTOAS_BIN}" fi if ! python_can_import_ptodsl "${RESOLVED_PYTHON_BIN}"; then - die "active Python environment cannot import ptodsl/mlir.ir; install the PTOAS wheel or use a preconfigured environment" + die "active Python environment cannot import ptodsl/ptoas.mlir.ir; install the PTOAS wheel or use a preconfigured environment" fi if ! command -v ptoas >/dev/null 2>&1; then die "ptoas is not available on PATH; install the PTOAS wheel or export PTOAS_BIN" diff --git a/test/dsl/abs.py b/test/dsl/abs.py index 1917a57250..807eca7d0c 100644 --- a/test/dsl/abs.py +++ b/test/dsl/abs.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -import mlir.dialects.pto as pto +import ptoas.mlir.dialects.pto as pto @pto.vkernel(target="a5", name="abs_kernel_2d") diff --git a/test/dsl/strict_vecscope.py b/test/dsl/strict_vecscope.py index 7badac2ae4..03bf99a801 100644 --- a/test/dsl/strict_vecscope.py +++ b/test/dsl/strict_vecscope.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -import mlir.dialects.pto as pto +import ptoas.mlir.dialects.pto as pto @pto.vkernel(target="a5", name="abs_strict_vecscope_kernel_2d") diff --git a/test/dsl/template_abs.py b/test/dsl/template_abs.py index dc674adee0..512ad54f08 100644 --- a/test/dsl/template_abs.py +++ b/test/dsl/template_abs.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -import mlir.dialects.pto as pto +import ptoas.mlir.dialects.pto as pto @pto.vkernel(target="a5", name="template_abs_kernel") diff --git a/test/python/compact_tile_buf_asm.py b/test/python/compact_tile_buf_asm.py index 8c07e945ce..cdf74879e7 100644 --- a/test/python/compact_tile_buf_asm.py +++ b/test/python/compact_tile_buf_asm.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -from mlir.ir import Context, F32Type, MLIRError, Module -from mlir.dialects import pto +from ptoas.mlir.ir import Context, F32Type, MLIRError, Module +from ptoas.mlir.dialects import pto def expect_equal(actual: str, expected: str, label: str) -> None: diff --git a/test/python/low_precision_types.py b/test/python/low_precision_types.py index dde603f191..aac0e2d976 100644 --- a/test/python/low_precision_types.py +++ b/test/python/low_precision_types.py @@ -7,8 +7,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context -from mlir.dialects import pto +from ptoas.mlir.ir import Context +from ptoas.mlir.dialects import pto def assert_contains(text: str, needle: str) -> None: diff --git a/test/python/memory_consistency_bindings.py b/test/python/memory_consistency_bindings.py index e52d26bae5..a227332d3b 100644 --- a/test/python/memory_consistency_bindings.py +++ b/test/python/memory_consistency_bindings.py @@ -7,8 +7,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, InsertionPoint, Location, Module -from mlir.dialects import pto +from ptoas.mlir.ir import Context, InsertionPoint, Location, Module +from ptoas.mlir.dialects import pto def assert_contains(text: str, needle: str) -> None: diff --git a/test/samples/Abs/abs.py b/test/samples/Abs/abs.py index 5d01854e83..7e3a862254 100644 --- a/test/samples/Abs/abs.py +++ b/test/samples/Abs/abs.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/AddPtr/addptr.py b/test/samples/AddPtr/addptr.py index 603162e98e..2a5cddd3bc 100644 --- a/test/samples/AddPtr/addptr.py +++ b/test/samples/AddPtr/addptr.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/AddPtr/addptr_chain.py b/test/samples/AddPtr/addptr_chain.py index cc9a3da26d..7d460cbe8e 100644 --- a/test/samples/AddPtr/addptr_chain.py +++ b/test/samples/AddPtr/addptr_chain.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/AddPtr/addptr_dynamic.py b/test/samples/AddPtr/addptr_dynamic.py index ce25d91178..a6c81e5e55 100644 --- a/test/samples/AddPtr/addptr_dynamic.py +++ b/test/samples/AddPtr/addptr_dynamic.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/AddPtr/addptr_f16.py b/test/samples/AddPtr/addptr_f16.py index 16ef95e3bc..0db5337049 100644 --- a/test/samples/AddPtr/addptr_f16.py +++ b/test/samples/AddPtr/addptr_f16.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F16Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F16Type, IndexType def build(): diff --git a/test/samples/Addc/addc.py b/test/samples/Addc/addc.py index 460e194c07..89209b44d8 100644 --- a/test/samples/Addc/addc.py +++ b/test/samples/Addc/addc.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Adds/adds.py b/test/samples/Adds/adds.py index 1a2c02b070..0cb7e53081 100644 --- a/test/samples/Adds/adds.py +++ b/test/samples/Adds/adds.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Addsc/addsc.py b/test/samples/Addsc/addsc.py index 1d96ed7dff..a15e6cc3eb 100644 --- a/test/samples/Addsc/addsc.py +++ b/test/samples/Addsc/addsc.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/AllocTile/alloc_tile_addr.py b/test/samples/AllocTile/alloc_tile_addr.py index 6072a6098c..452b408e63 100644 --- a/test/samples/AllocTile/alloc_tile_addr.py +++ b/test/samples/AllocTile/alloc_tile_addr.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IntegerType, IntegerAttr, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IntegerType, IntegerAttr, IndexType def build(): diff --git a/test/samples/And/and.py b/test/samples/And/and.py index 0bb8cb7a03..b138cc5041 100644 --- a/test/samples/And/and.py +++ b/test/samples/And/and.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IntegerType, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IntegerType, IndexType def build(): diff --git a/test/samples/Ands/ands.py b/test/samples/Ands/ands.py index a34372db67..1bdb4ef328 100644 --- a/test/samples/Ands/ands.py +++ b/test/samples/Ands/ands.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IntegerType, IndexType, IntegerAttr +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IntegerType, IndexType, IntegerAttr def build(): diff --git a/test/samples/AsyncComm/async_comm.py b/test/samples/AsyncComm/async_comm.py index f8e4177f63..3efa0b39ce 100644 --- a/test/samples/AsyncComm/async_comm.py +++ b/test/samples/AsyncComm/async_comm.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, F32Type, IntegerType, IntegerAttr, IndexType, Operation, UnitAttr -from mlir.dialects import arith, func, pto, scf +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, F32Type, IntegerType, IntegerAttr, IndexType, Operation, UnitAttr +from ptoas.mlir.dialects import arith, func, pto, scf def _build_async_session(scratch, workspace, i32, ctx, sync_id=0): diff --git a/test/samples/AsyncComm/tget_async_kernel_impl_like.py b/test/samples/AsyncComm/tget_async_kernel_impl_like.py index 691f1cb160..ca93990421 100644 --- a/test/samples/AsyncComm/tget_async_kernel_impl_like.py +++ b/test/samples/AsyncComm/tget_async_kernel_impl_like.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, F32Type, @@ -19,7 +19,7 @@ Operation, Type, ) -from mlir.dialects import arith, func, pto, scf +from ptoas.mlir.dialects import arith, func, pto, scf def _build_async_session(scratch, workspace, i32, ctx, sync_id=0): diff --git a/test/samples/AsyncComm/tput_async_kernel_impl_like.py b/test/samples/AsyncComm/tput_async_kernel_impl_like.py index 63551fb208..73a3af2e67 100644 --- a/test/samples/AsyncComm/tput_async_kernel_impl_like.py +++ b/test/samples/AsyncComm/tput_async_kernel_impl_like.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, F32Type, @@ -19,7 +19,7 @@ Operation, Type, ) -from mlir.dialects import arith, func, pto, scf +from ptoas.mlir.dialects import arith, func, pto, scf def _build_async_session(scratch, workspace, i32, ctx, sync_id=0): diff --git a/test/samples/Bf16/bf16_tile.py b/test/samples/Bf16/bf16_tile.py index 54b08ece08..f08e6fd68a 100644 --- a/test/samples/Bf16/bf16_tile.py +++ b/test/samples/Bf16/bf16_tile.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, Location, @@ -15,7 +15,7 @@ BF16Type, StringAttr, ) -from mlir.dialects import func, arith, pto, builtin +from ptoas.mlir.dialects import func, arith, pto, builtin def _idx_const(v: int): diff --git a/test/samples/Ci/ci.py b/test/samples/Ci/ci.py index d7d374930c..ad5d337231 100644 --- a/test/samples/Ci/ci.py +++ b/test/samples/Ci/ci.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IntegerType, IndexType, IntegerAttr, BoolAttr +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IntegerType, IndexType, IntegerAttr, BoolAttr def build(): diff --git a/test/samples/Cmp/cmp.py b/test/samples/Cmp/cmp.py index b71552638d..fbbada8478 100644 --- a/test/samples/Cmp/cmp.py +++ b/test/samples/Cmp/cmp.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType, IntegerType def build(): diff --git a/test/samples/Cmps/cmps.py b/test/samples/Cmps/cmps.py index 17eae6b9be..2b8f0bb7ff 100644 --- a/test/samples/Cmps/cmps.py +++ b/test/samples/Cmps/cmps.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType, IntegerType def build(): diff --git a/test/samples/Colexpand/colexpand.py b/test/samples/Colexpand/colexpand.py index 044dfcb225..e2f9deed8a 100644 --- a/test/samples/Colexpand/colexpand.py +++ b/test/samples/Colexpand/colexpand.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Colexpandadd/colexpandadd.py b/test/samples/Colexpandadd/colexpandadd.py index 94df4271b2..5043948b7d 100644 --- a/test/samples/Colexpandadd/colexpandadd.py +++ b/test/samples/Colexpandadd/colexpandadd.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Colexpanddiv/colexpanddiv.py b/test/samples/Colexpanddiv/colexpanddiv.py index b3e6679608..f6faf69413 100644 --- a/test/samples/Colexpanddiv/colexpanddiv.py +++ b/test/samples/Colexpanddiv/colexpanddiv.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Colexpanddiv/colexpanddiv_src1_vcol_mismatch_invalid.py b/test/samples/Colexpanddiv/colexpanddiv_src1_vcol_mismatch_invalid.py index aa2e612c1c..ea67e95bf3 100644 --- a/test/samples/Colexpanddiv/colexpanddiv_src1_vcol_mismatch_invalid.py +++ b/test/samples/Colexpanddiv/colexpanddiv_src1_vcol_mismatch_invalid.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr -from mlir.dialects import func, pto -from mlir.ir import F32Type +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr +from ptoas.mlir.dialects import func, pto +from ptoas.mlir.ir import F32Type def build(): diff --git a/test/samples/Colexpandexpdif/colexpandexpdif.py b/test/samples/Colexpandexpdif/colexpandexpdif.py index 7ec49106fd..bb207c8ebc 100644 --- a/test/samples/Colexpandexpdif/colexpandexpdif.py +++ b/test/samples/Colexpandexpdif/colexpandexpdif.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Colexpandexpdif/colexpandexpdif_dtype_invalid.py b/test/samples/Colexpandexpdif/colexpandexpdif_dtype_invalid.py index d507da8518..429a2f8b50 100644 --- a/test/samples/Colexpandexpdif/colexpandexpdif_dtype_invalid.py +++ b/test/samples/Colexpandexpdif/colexpandexpdif_dtype_invalid.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr -from mlir.dialects import func, pto -from mlir.ir import IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr +from ptoas.mlir.dialects import func, pto +from ptoas.mlir.ir import IntegerType def build(): diff --git a/test/samples/Colexpandmax/colexpandmax.py b/test/samples/Colexpandmax/colexpandmax.py index f7bbf72c69..681661c8f0 100644 --- a/test/samples/Colexpandmax/colexpandmax.py +++ b/test/samples/Colexpandmax/colexpandmax.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Colexpandmin/colexpandmin.py b/test/samples/Colexpandmin/colexpandmin.py index 8680dfc3b1..28b8deaf10 100644 --- a/test/samples/Colexpandmin/colexpandmin.py +++ b/test/samples/Colexpandmin/colexpandmin.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Colexpandmul/colexpandmul.py b/test/samples/Colexpandmul/colexpandmul.py index e8a02938c3..3e748fe3ae 100644 --- a/test/samples/Colexpandmul/colexpandmul.py +++ b/test/samples/Colexpandmul/colexpandmul.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Colexpandsub/colexpandsub.py b/test/samples/Colexpandsub/colexpandsub.py index dd1c3ef8ef..287bd35eab 100644 --- a/test/samples/Colexpandsub/colexpandsub.py +++ b/test/samples/Colexpandsub/colexpandsub.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Colexpandsub/colexpandsub_src0_layout_invalid.py b/test/samples/Colexpandsub/colexpandsub_src0_layout_invalid.py index d811ca37f5..097a979265 100644 --- a/test/samples/Colexpandsub/colexpandsub_src0_layout_invalid.py +++ b/test/samples/Colexpandsub/colexpandsub_src0_layout_invalid.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr -from mlir.dialects import func, pto -from mlir.ir import F32Type +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr +from ptoas.mlir.dialects import func, pto +from ptoas.mlir.ir import F32Type def build(): diff --git a/test/samples/Colmax/colmax.py b/test/samples/Colmax/colmax.py index fba23dfe26..250188a5b7 100644 --- a/test/samples/Colmax/colmax.py +++ b/test/samples/Colmax/colmax.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Colmin/colmin.py b/test/samples/Colmin/colmin.py index acdea6e5de..f98683bc4f 100644 --- a/test/samples/Colmin/colmin.py +++ b/test/samples/Colmin/colmin.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Colprod/colprod.py b/test/samples/Colprod/colprod.py index 5d946c5043..9e40d974cf 100644 --- a/test/samples/Colprod/colprod.py +++ b/test/samples/Colprod/colprod.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Colsum/colsum.py b/test/samples/Colsum/colsum.py index d5fca8ca96..0c3c32d044 100644 --- a/test/samples/Colsum/colsum.py +++ b/test/samples/Colsum/colsum.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType, BoolAttr +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType, BoolAttr def build(): diff --git a/test/samples/CommSync/comm_collective.py b/test/samples/CommSync/comm_collective.py index d198c41df4..142bbf16bd 100644 --- a/test/samples/CommSync/comm_collective.py +++ b/test/samples/CommSync/comm_collective.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto def build(): with Context() as ctx: diff --git a/test/samples/CommSync/comm_collective_binding_variants.py b/test/samples/CommSync/comm_collective_binding_variants.py index 0c5ad90b77..f8350d6377 100644 --- a/test/samples/CommSync/comm_collective_binding_variants.py +++ b/test/samples/CommSync/comm_collective_binding_variants.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto def build(): diff --git a/test/samples/CommSync/comm_p2p.py b/test/samples/CommSync/comm_p2p.py index a7325df7d2..e26a0f1f83 100644 --- a/test/samples/CommSync/comm_p2p.py +++ b/test/samples/CommSync/comm_p2p.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto def build(): diff --git a/test/samples/CommSync/comm_p2p_binding_variants.py b/test/samples/CommSync/comm_p2p_binding_variants.py index 3df46d762b..a302ab2eb2 100644 --- a/test/samples/CommSync/comm_p2p_binding_variants.py +++ b/test/samples/CommSync/comm_p2p_binding_variants.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto def build(): diff --git a/test/samples/CommSync/tbroadcast_root_binding.py b/test/samples/CommSync/tbroadcast_root_binding.py index 4ec5565004..b078b29463 100644 --- a/test/samples/CommSync/tbroadcast_root_binding.py +++ b/test/samples/CommSync/tbroadcast_root_binding.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto, scf +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto, scf def _make_part(ptr, tv_ty, pv_ty, c0, c1, count): diff --git a/test/samples/CommSync/tgather_root_binding.py b/test/samples/CommSync/tgather_root_binding.py index b5e75c8ed9..51aeb7a1e9 100644 --- a/test/samples/CommSync/tgather_root_binding.py +++ b/test/samples/CommSync/tgather_root_binding.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto, scf +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto, scf def _make_part(ptr, tv_ty, pv_ty, c0, c1, count): diff --git a/test/samples/CommSync/tnotify_atomic_add_binding.py b/test/samples/CommSync/tnotify_atomic_add_binding.py index bb93ab4193..c7d0655727 100644 --- a/test/samples/CommSync/tnotify_atomic_add_binding.py +++ b/test/samples/CommSync/tnotify_atomic_add_binding.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Context, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto def build(): diff --git a/test/samples/CommSync/treduce_root_binding.py b/test/samples/CommSync/treduce_root_binding.py index 780b36e0c8..00571c3041 100644 --- a/test/samples/CommSync/treduce_root_binding.py +++ b/test/samples/CommSync/treduce_root_binding.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto, scf +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto, scf def _make_part(ptr, tv_ty, pv_ty, c0, c1, count): diff --git a/test/samples/CommSync/tscatter_root_binding.py b/test/samples/CommSync/tscatter_root_binding.py index bfd1b3d8bb..c9bdf71c5d 100644 --- a/test/samples/CommSync/tscatter_root_binding.py +++ b/test/samples/CommSync/tscatter_root_binding.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto, scf +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto, scf def _make_part(ptr, tv_ty, pv_ty, c0, c1, count): diff --git a/test/samples/CommSync/twait_atomic_binding.py b/test/samples/CommSync/twait_atomic_binding.py index 639e2a1e32..bed112f013 100644 --- a/test/samples/CommSync/twait_atomic_binding.py +++ b/test/samples/CommSync/twait_atomic_binding.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto, scf +from ptoas.mlir.ir import Context, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto, scf def build(): diff --git a/test/samples/Complex/cv_region.py b/test/samples/Complex/cv_region.py index 05d7ea37d0..b000ff7cef 100644 --- a/test/samples/Complex/cv_region.py +++ b/test/samples/Complex/cv_region.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, IndexType, F32Type, UnitAttr -from mlir.dialects import func, arith, pto +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, IndexType, F32Type, UnitAttr +from ptoas.mlir.dialects import func, arith, pto def build_sections_test(): with Context() as ctx: diff --git a/test/samples/Complex/mix_kernel.py b/test/samples/Complex/mix_kernel.py index a5ed3312ea..a7f4e8f6ba 100644 --- a/test/samples/Complex/mix_kernel.py +++ b/test/samples/Complex/mix_kernel.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, Attribute, IntegerType, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, Attribute, IntegerType, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(M=32, N=32, K=32, TM=32, TN=32, TK=32): diff --git a/test/samples/Concat/concat.py b/test/samples/Concat/concat.py index 4cb4c5264f..c740481b70 100644 --- a/test/samples/Concat/concat.py +++ b/test/samples/Concat/concat.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IntegerType, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IntegerType, IndexType def build(): diff --git a/test/samples/Cvt/cvt_f32_f16.py b/test/samples/Cvt/cvt_f32_f16.py index fe00d1547f..7f68e6ed4a 100644 --- a/test/samples/Cvt/cvt_f32_f16.py +++ b/test/samples/Cvt/cvt_f32_f16.py @@ -16,8 +16,8 @@ make_tensor_view -> partition_view -> alloc_tile -> tload -> tcvt -> tstore """ -from mlir.ir import Attribute, Context, F16Type, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Attribute, Context, F16Type, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto def build(): diff --git a/test/samples/Cvt/cvt_f32_f32.py b/test/samples/Cvt/cvt_f32_f32.py index 2fa89a0df2..b8ebf0742d 100644 --- a/test/samples/Cvt/cvt_f32_f32.py +++ b/test/samples/Cvt/cvt_f32_f32.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType, BoolAttr +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType, BoolAttr def build(): diff --git a/test/samples/Cvt/tcvt.py b/test/samples/Cvt/tcvt.py index 2ab189faef..b8a2a66c1b 100644 --- a/test/samples/Cvt/tcvt.py +++ b/test/samples/Cvt/tcvt.py @@ -17,7 +17,7 @@ make_tensor_view -> partition_view -> alloc_tile -> tload -> tcvt -> tstore """ -from mlir.ir import ( +from ptoas.mlir.ir import ( Attribute, Context, F32Type, @@ -28,7 +28,7 @@ Module, UnitAttr, ) -from mlir.dialects import arith, func, pto +from ptoas.mlir.dialects import arith, func, pto def build(): diff --git a/test/samples/Dequant/dequant.py b/test/samples/Dequant/dequant.py index 3829aed870..93bc8375dd 100644 --- a/test/samples/Dequant/dequant.py +++ b/test/samples/Dequant/dequant.py @@ -23,7 +23,7 @@ f32 parameter tiles: Cols*4 >= 32 means Cols >= 8. """ -from mlir.ir import ( +from ptoas.mlir.ir import ( Attribute, Context, Location, @@ -34,7 +34,7 @@ IntegerType, UnitAttr, ) -from mlir.dialects import func, arith, pto +from ptoas.mlir.dialects import func, arith, pto # Tile shapes diff --git a/test/samples/Dequant/dequant_i8.py b/test/samples/Dequant/dequant_i8.py index a21672fdbf..134159dd90 100644 --- a/test/samples/Dequant/dequant_i8.py +++ b/test/samples/Dequant/dequant_i8.py @@ -23,7 +23,7 @@ f32 parameter tiles: Cols*4 >= 32 means Cols >= 8. """ -from mlir.ir import ( +from ptoas.mlir.ir import ( Attribute, Context, Location, @@ -34,7 +34,7 @@ IntegerType, UnitAttr, ) -from mlir.dialects import func, arith, pto +from ptoas.mlir.dialects import func, arith, pto # Tile shapes diff --git a/test/samples/Div/div.py b/test/samples/Div/div.py index eddd50b339..d36196e52d 100644 --- a/test/samples/Div/div.py +++ b/test/samples/Div/div.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, IndexType, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, IndexType, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type def build(): diff --git a/test/samples/Divs/divs.py b/test/samples/Divs/divs.py index e37ce94435..c285efc195 100644 --- a/test/samples/Divs/divs.py +++ b/test/samples/Divs/divs.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Divs2/divs2.py b/test/samples/Divs2/divs2.py index 6d52e659dc..248e8aac71 100644 --- a/test/samples/Divs2/divs2.py +++ b/test/samples/Divs2/divs2.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/DynamicTailMatmul/dynamic_tail_matmul.py b/test/samples/DynamicTailMatmul/dynamic_tail_matmul.py index 762ac5b3ac..d86c136808 100644 --- a/test/samples/DynamicTailMatmul/dynamic_tail_matmul.py +++ b/test/samples/DynamicTailMatmul/dynamic_tail_matmul.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, IndexType, IntegerType, F16Type, F32Type, UnitAttr -from mlir.dialects import func, arith, pto +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, IndexType, IntegerType, F16Type, F32Type, UnitAttr +from ptoas.mlir.dialects import func, arith, pto def build(): diff --git a/test/samples/Exp/exp.py b/test/samples/Exp/exp.py index aeddc984e0..979acdacd6 100644 --- a/test/samples/Exp/exp.py +++ b/test/samples/Exp/exp.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Expands/expand.py b/test/samples/Expands/expand.py index 77c30d59b2..f5f7e55efa 100644 --- a/test/samples/Expands/expand.py +++ b/test/samples/Expands/expand.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Extract/extract.py b/test/samples/Extract/extract.py index 2618f43f72..d649fe44ed 100644 --- a/test/samples/Extract/extract.py +++ b/test/samples/Extract/extract.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F16Type, F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F16Type, F32Type, IndexType def build(): diff --git a/test/samples/Extract/extract_fp.py b/test/samples/Extract/extract_fp.py index 085db33f01..594e40c971 100644 --- a/test/samples/Extract/extract_fp.py +++ b/test/samples/Extract/extract_fp.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType, IntegerType def build(): diff --git a/test/samples/Fillpad/fillpad.py b/test/samples/Fillpad/fillpad.py index 07908652ae..e221c9accd 100644 --- a/test/samples/Fillpad/fillpad.py +++ b/test/samples/Fillpad/fillpad.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Fillpad/fillpad_expand.py b/test/samples/Fillpad/fillpad_expand.py index 5f06ab054a..1bfd10011e 100644 --- a/test/samples/Fillpad/fillpad_expand.py +++ b/test/samples/Fillpad/fillpad_expand.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, InsertionPoint, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, InsertionPoint, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Fillpad/fillpad_expand_invalid.py b/test/samples/Fillpad/fillpad_expand_invalid.py index 1cb14c144a..98fdf7206b 100644 --- a/test/samples/Fillpad/fillpad_expand_invalid.py +++ b/test/samples/Fillpad/fillpad_expand_invalid.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, InsertionPoint, Location, Module, MLIRError, UnitAttr -from mlir.dialects import func, pto -from mlir.ir import F32Type +from ptoas.mlir.ir import Context, InsertionPoint, Location, Module, MLIRError, UnitAttr +from ptoas.mlir.dialects import func, pto +from ptoas.mlir.ir import F32Type def build(): diff --git a/test/samples/Fillpad/fillpad_expand_pad_null_invalid.py b/test/samples/Fillpad/fillpad_expand_pad_null_invalid.py index 8926dc905a..a3f250b95f 100644 --- a/test/samples/Fillpad/fillpad_expand_pad_null_invalid.py +++ b/test/samples/Fillpad/fillpad_expand_pad_null_invalid.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, InsertionPoint, Location, Module, MLIRError, UnitAttr -from mlir.dialects import func, pto -from mlir.ir import F32Type +from ptoas.mlir.ir import Context, InsertionPoint, Location, Module, MLIRError, UnitAttr +from ptoas.mlir.dialects import func, pto +from ptoas.mlir.ir import F32Type def build(): diff --git a/test/samples/Fillpad/fillpad_inplace.py b/test/samples/Fillpad/fillpad_inplace.py index 25f16eddd7..b4159ef842 100644 --- a/test/samples/Fillpad/fillpad_inplace.py +++ b/test/samples/Fillpad/fillpad_inplace.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Fmod/fmod.py b/test/samples/Fmod/fmod.py index 87c58f33b7..32a1a79ff0 100644 --- a/test/samples/Fmod/fmod.py +++ b/test/samples/Fmod/fmod.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Fmods/fmods.py b/test/samples/Fmods/fmods.py index e852a67fe3..77c43e8193 100644 --- a/test/samples/Fmods/fmods.py +++ b/test/samples/Fmods/fmods.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Fmods/tfmods_scalar_type_invalid.py b/test/samples/Fmods/tfmods_scalar_type_invalid.py index f478435ae1..634ad763ae 100644 --- a/test/samples/Fmods/tfmods_scalar_type_invalid.py +++ b/test/samples/Fmods/tfmods_scalar_type_invalid.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IntegerType def build(): diff --git a/test/samples/Gather/gather.py b/test/samples/Gather/gather.py index bcb9c3dbf7..08ec15b6f8 100644 --- a/test/samples/Gather/gather.py +++ b/test/samples/Gather/gather.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto -from mlir.ir import IntegerType +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto +from ptoas.mlir.ir import IntegerType def build(): diff --git a/test/samples/Gather/gather_legacy.py b/test/samples/Gather/gather_legacy.py index eecfab05e0..dcf33bcdce 100644 --- a/test/samples/Gather/gather_legacy.py +++ b/test/samples/Gather/gather_legacy.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Gatherb/gatherb.py b/test/samples/Gatherb/gatherb.py index 4b1a0e435d..cf76034bbe 100644 --- a/test/samples/Gatherb/gatherb.py +++ b/test/samples/Gatherb/gatherb.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.dialects import arith, func, pto -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr def build(): diff --git a/test/samples/Gemv/gemv.py b/test/samples/Gemv/gemv.py index 416c2b0076..5870784948 100644 --- a/test/samples/Gemv/gemv.py +++ b/test/samples/Gemv/gemv.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F16Type, F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F16Type, F32Type, IndexType def build(): diff --git a/test/samples/Gemv/gemvacc.py b/test/samples/Gemv/gemvacc.py index d70916cfeb..27c2b8c276 100644 --- a/test/samples/Gemv/gemvacc.py +++ b/test/samples/Gemv/gemvacc.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F16Type, F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F16Type, F32Type, IndexType def build(): diff --git a/test/samples/Gemv/gemvbias.py b/test/samples/Gemv/gemvbias.py index 5beb088c7d..6ca847542e 100644 --- a/test/samples/Gemv/gemvbias.py +++ b/test/samples/Gemv/gemvbias.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F16Type, F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F16Type, F32Type, IndexType def build(): diff --git a/test/samples/Gemvmx/gemvmx.py b/test/samples/Gemvmx/gemvmx.py index b667873a7b..c624a5d452 100644 --- a/test/samples/Gemvmx/gemvmx.py +++ b/test/samples/Gemvmx/gemvmx.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, StringAttr, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F16Type, F32Type, Float8E4M3FNType, Float8E5M2Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, StringAttr, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F16Type, F32Type, Float8E4M3FNType, Float8E5M2Type, IndexType def build(): diff --git a/test/samples/Layout/tensor_view_infer_layout_dn.py b/test/samples/Layout/tensor_view_infer_layout_dn.py index 05b29b8fd1..488cb3eb75 100644 --- a/test/samples/Layout/tensor_view_infer_layout_dn.py +++ b/test/samples/Layout/tensor_view_infer_layout_dn.py @@ -16,9 +16,9 @@ The expected emitted C++ should use `pto::Layout::DN` in GlobalTensor<>. """ -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Layout/tensor_view_layout_dn.py b/test/samples/Layout/tensor_view_layout_dn.py index 8d311920a4..984599cba1 100644 --- a/test/samples/Layout/tensor_view_layout_dn.py +++ b/test/samples/Layout/tensor_view_layout_dn.py @@ -15,8 +15,8 @@ - The generated C++ GlobalTensor uses pto::Layout::DN. """ -from mlir.ir import Context, Location, InsertionPoint, Module, IndexType -from mlir.dialects import arith, func, pto, builtin +from ptoas.mlir.ir import Context, Location, InsertionPoint, Module, IndexType +from ptoas.mlir.dialects import arith, func, pto, builtin def idx(val: int): diff --git a/test/samples/LayoutInference/layout_inference.py b/test/samples/LayoutInference/layout_inference.py index 8b72d15646..2944d3b969 100644 --- a/test/samples/LayoutInference/layout_inference.py +++ b/test/samples/LayoutInference/layout_inference.py @@ -20,7 +20,7 @@ on each make_tensor_view op. """ -from mlir.ir import ( +from ptoas.mlir.ir import ( Context, InsertionPoint, Location, @@ -29,7 +29,7 @@ IndexType, UnitAttr, ) -from mlir.dialects import func, arith, pto +from ptoas.mlir.dialects import func, arith, pto from typing import Optional import sys diff --git a/test/samples/Log/log.py b/test/samples/Log/log.py index 0edf56293c..ac93b89b4d 100644 --- a/test/samples/Log/log.py +++ b/test/samples/Log/log.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Lrelu/lrelu.py b/test/samples/Lrelu/lrelu.py index 877a0a74ae..56e5736a99 100644 --- a/test/samples/Lrelu/lrelu.py +++ b/test/samples/Lrelu/lrelu.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/MatMul/tmatmulk.py b/test/samples/MatMul/tmatmulk.py index 1a75231e86..06bc905a4f 100644 --- a/test/samples/MatMul/tmatmulk.py +++ b/test/samples/MatMul/tmatmulk.py @@ -6,17 +6,17 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, Location, InsertionPoint, IndexType, IntegerType, F16Type, F32Type, StringAttr ) -from mlir.dialects import func, arith, scf, pto, builtin -from mlir.dialects.pto import ( +from ptoas.mlir.dialects import func, arith, scf, pto, builtin +from ptoas.mlir.dialects.pto import ( TLOAD, TMOV_M2L, TMATMUL, TSTORE_ACC, EVENT_ID0 ) -from mlir.dialects.arith import CmpIPredicate +from ptoas.mlir.dialects.arith import CmpIPredicate def _idx_const(v: int): diff --git a/test/samples/MatmulMxLowPrecision/matmul_mx_low_precision.py b/test/samples/MatmulMxLowPrecision/matmul_mx_low_precision.py index 14a30dd803..3661b60692 100644 --- a/test/samples/MatmulMxLowPrecision/matmul_mx_low_precision.py +++ b/test/samples/MatmulMxLowPrecision/matmul_mx_low_precision.py @@ -7,7 +7,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( Attribute, Context, F16Type, @@ -20,7 +20,7 @@ StringAttr, UnitAttr, ) -from mlir.dialects import func, pto +from ptoas.mlir.dialects import func, pto def build(): diff --git a/test/samples/Matmul_transpose/Matmul_transpose.py b/test/samples/Matmul_transpose/Matmul_transpose.py index 46b2ea916c..9e1f8d9aa4 100644 --- a/test/samples/Matmul_transpose/Matmul_transpose.py +++ b/test/samples/Matmul_transpose/Matmul_transpose.py @@ -15,9 +15,9 @@ - pto-as/test/samples/MatMul/tmatmulk.py """ -from mlir.ir import Context, Location, InsertionPoint, IndexType, IntegerType, F32Type, StringAttr, UnitAttr -from mlir.dialects import func, arith, scf, pto, builtin -from mlir.dialects.pto import TLOAD, TMOV_M2L, TMATMUL, TSTORE_ACC, EVENT_ID0 +from ptoas.mlir.ir import Context, Location, InsertionPoint, IndexType, IntegerType, F32Type, StringAttr, UnitAttr +from ptoas.mlir.dialects import func, arith, scf, pto, builtin +from ptoas.mlir.dialects.pto import TLOAD, TMOV_M2L, TMATMUL, TSTORE_ACC, EVENT_ID0 def _idx_const(v: int): diff --git a/test/samples/Max/max.py b/test/samples/Max/max.py index f6c74c3c1d..a62b1c7107 100644 --- a/test/samples/Max/max.py +++ b/test/samples/Max/max.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Maxs/maxs.py b/test/samples/Maxs/maxs.py index a51b0b3139..cd0e85cb6c 100644 --- a/test/samples/Maxs/maxs.py +++ b/test/samples/Maxs/maxs.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Mgather/mgather.py b/test/samples/Mgather/mgather.py index 543d1211ae..9269e3072e 100644 --- a/test/samples/Mgather/mgather.py +++ b/test/samples/Mgather/mgather.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Attribute, Context, Location, Module, InsertionPoint, StringAttr, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IndexType, IntegerType +from ptoas.mlir.ir import Attribute, Context, Location, Module, InsertionPoint, StringAttr, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IndexType, IntegerType def build(): diff --git a/test/samples/Min/min.py b/test/samples/Min/min.py index 4b19f65e91..6ffe24fdb9 100644 --- a/test/samples/Min/min.py +++ b/test/samples/Min/min.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Mins/mins.py b/test/samples/Mins/mins.py index 86cfd556fa..928db3d38c 100644 --- a/test/samples/Mins/mins.py +++ b/test/samples/Mins/mins.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Movfp/movfp.py b/test/samples/Movfp/movfp.py index dda90fe81e..47e75e88ad 100644 --- a/test/samples/Movfp/movfp.py +++ b/test/samples/Movfp/movfp.py @@ -16,9 +16,9 @@ - src must be ACC tile; dst must be MAT tile. """ -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F16Type, F32Type, IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F16Type, F32Type, IndexType, IntegerType def build(): diff --git a/test/samples/Mrgsort/mrgsort.py b/test/samples/Mrgsort/mrgsort.py index 4d58443f0b..4c5b000911 100644 --- a/test/samples/Mrgsort/mrgsort.py +++ b/test/samples/Mrgsort/mrgsort.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType, IntegerType def build(): diff --git a/test/samples/Mrgsort/mrgsort_a5.py b/test/samples/Mrgsort/mrgsort_a5.py index 40251dd2fb..b3d2ac9970 100644 --- a/test/samples/Mrgsort/mrgsort_a5.py +++ b/test/samples/Mrgsort/mrgsort_a5.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType, IntegerType def build(): diff --git a/test/samples/Mrgsort/mrgsort_format2.py b/test/samples/Mrgsort/mrgsort_format2.py index 4179f6e3eb..6c43018901 100644 --- a/test/samples/Mrgsort/mrgsort_format2.py +++ b/test/samples/Mrgsort/mrgsort_format2.py @@ -15,11 +15,11 @@ - This testcase covers all three forms in a single function entry so it fits the remote validation flow. """ -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType, IntegerType -from mlir.ir import VectorType +from ptoas.mlir.ir import VectorType def build(): diff --git a/test/samples/Mscatter/mscatter.py b/test/samples/Mscatter/mscatter.py index d3dca96d53..a314705e92 100644 --- a/test/samples/Mscatter/mscatter.py +++ b/test/samples/Mscatter/mscatter.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, StringAttr, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, StringAttr, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IndexType, IntegerType def build(): diff --git a/test/samples/Mul/mul.py b/test/samples/Mul/mul.py index 3b1be61534..05f80fdea8 100644 --- a/test/samples/Mul/mul.py +++ b/test/samples/Mul/mul.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Muls/muls.py b/test/samples/Muls/muls.py index dd1573dace..fa72991760 100644 --- a/test/samples/Muls/muls.py +++ b/test/samples/Muls/muls.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Neg/neg.py b/test/samples/Neg/neg.py index 5cb5f8f10b..a64a264947 100644 --- a/test/samples/Neg/neg.py +++ b/test/samples/Neg/neg.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Not/not.py b/test/samples/Not/not.py index 400fcbcedb..369481fddc 100644 --- a/test/samples/Not/not.py +++ b/test/samples/Not/not.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IntegerType, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IntegerType, IndexType def build(): diff --git a/test/samples/Or/or.py b/test/samples/Or/or.py index e958119129..76a1676669 100644 --- a/test/samples/Or/or.py +++ b/test/samples/Or/or.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IntegerType, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IntegerType, IndexType def build(): diff --git a/test/samples/Ors/ors.py b/test/samples/Ors/ors.py index 3ed43d47fd..afa0e8f8a4 100644 --- a/test/samples/Ors/ors.py +++ b/test/samples/Ors/ors.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IntegerType, IndexType, IntegerAttr +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IntegerType, IndexType, IntegerAttr def build(): diff --git a/test/samples/Partadd/partadd.py b/test/samples/Partadd/partadd.py index 85cfe40adc..3a54a054af 100644 --- a/test/samples/Partadd/partadd.py +++ b/test/samples/Partadd/partadd.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Partarg/partarg.py b/test/samples/Partarg/partarg.py index 0c156e36a6..01b9459a91 100644 --- a/test/samples/Partarg/partarg.py +++ b/test/samples/Partarg/partarg.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.dialects import arith, func, pto -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr def build(): diff --git a/test/samples/Partition5D/partition5d.py b/test/samples/Partition5D/partition5d.py index 0e9d11c67e..25a684f34c 100644 --- a/test/samples/Partition5D/partition5d.py +++ b/test/samples/Partition5D/partition5d.py @@ -16,14 +16,14 @@ 4) tstore writes back to the destination tensor_view """ -from mlir.ir import ( +from ptoas.mlir.ir import ( Context, Location, InsertionPoint, Module, IndexType, ) -from mlir.dialects import arith, func, pto, builtin +from ptoas.mlir.dialects import arith, func, pto, builtin def idx(val: int): diff --git a/test/samples/Partition5D/partition5d_a5.py b/test/samples/Partition5D/partition5d_a5.py index 91da2458e8..021693ee52 100644 --- a/test/samples/Partition5D/partition5d_a5.py +++ b/test/samples/Partition5D/partition5d_a5.py @@ -16,14 +16,14 @@ 4) tstore writes back to the destination tensor_view """ -from mlir.ir import ( +from ptoas.mlir.ir import ( Context, Location, InsertionPoint, Module, IndexType, ) -from mlir.dialects import arith, func, pto, builtin +from ptoas.mlir.dialects import arith, func, pto, builtin def idx(val: int): diff --git a/test/samples/Partmax/partmax.py b/test/samples/Partmax/partmax.py index 9c91b5df56..b5a2a5e347 100644 --- a/test/samples/Partmax/partmax.py +++ b/test/samples/Partmax/partmax.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Partmin/partmin.py b/test/samples/Partmin/partmin.py index 32baa46d83..d4a6cd5a16 100644 --- a/test/samples/Partmin/partmin.py +++ b/test/samples/Partmin/partmin.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Partmul/partmul.py b/test/samples/Partmul/partmul.py index 0dcde4f4eb..d7dd68e626 100644 --- a/test/samples/Partmul/partmul.py +++ b/test/samples/Partmul/partmul.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Prelu/prelu.py b/test/samples/Prelu/prelu.py index e01f0fc94e..ec508cf027 100644 --- a/test/samples/Prelu/prelu.py +++ b/test/samples/Prelu/prelu.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType, IntegerType def build(): diff --git a/test/samples/Print/print.py b/test/samples/Print/print.py index 33f2942391..5fd0562496 100644 --- a/test/samples/Print/print.py +++ b/test/samples/Print/print.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): with Context() as ctx: diff --git a/test/samples/Print/print_scalar.py b/test/samples/Print/print_scalar.py index 8478319114..58c3013879 100644 --- a/test/samples/Print/print_scalar.py +++ b/test/samples/Print/print_scalar.py @@ -7,9 +7,9 @@ # See LICENSE in the root of the software repository for the full text of the License. """Test pto.print: format is string attribute, scalar is operand. Generates PRINTF("format", scalar) in C++.""" -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, pto -from mlir.ir import F32Type +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, pto +from ptoas.mlir.ir import F32Type def build(): diff --git a/test/samples/Quant/quant.py b/test/samples/Quant/quant.py index 7e802bbc6f..cdfa3d5952 100644 --- a/test/samples/Quant/quant.py +++ b/test/samples/Quant/quant.py @@ -17,7 +17,7 @@ (the NPU aligned-size). At 1 byte/element that means Cols >= 32. """ -from mlir.ir import ( +from ptoas.mlir.ir import ( Attribute, Context, Location, @@ -28,7 +28,7 @@ IntegerType, UnitAttr, ) -from mlir.dialects import func, arith, pto +from ptoas.mlir.dialects import func, arith, pto # Tile shape used throughout the sample. diff --git a/test/samples/Quant/quant_asym.py b/test/samples/Quant/quant_asym.py index 67490c96e7..aab8e02ed2 100644 --- a/test/samples/Quant/quant_asym.py +++ b/test/samples/Quant/quant_asym.py @@ -18,7 +18,7 @@ (the NPU aligned-size). At 1 byte/element that means Cols >= 32. """ -from mlir.ir import ( +from ptoas.mlir.ir import ( Attribute, Context, Location, @@ -29,7 +29,7 @@ IntegerType, UnitAttr, ) -from mlir.dialects import func, arith, pto +from ptoas.mlir.dialects import func, arith, pto # Tile shape used throughout the sample. diff --git a/test/samples/Recip/recip.py b/test/samples/Recip/recip.py index 132295f5e2..410e8ab7f2 100644 --- a/test/samples/Recip/recip.py +++ b/test/samples/Recip/recip.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Relu/relu.py b/test/samples/Relu/relu.py index a4b6078f1f..073eb19393 100644 --- a/test/samples/Relu/relu.py +++ b/test/samples/Relu/relu.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rem/rem.py b/test/samples/Rem/rem.py index 7a8f50ef9d..245b16d73b 100644 --- a/test/samples/Rem/rem.py +++ b/test/samples/Rem/rem.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rems/rems.py b/test/samples/Rems/rems.py index 3f5b0cefd6..fd55edeba9 100644 --- a/test/samples/Rems/rems.py +++ b/test/samples/Rems/rems.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Reshape/bitcast_dtype_alias.py b/test/samples/Reshape/bitcast_dtype_alias.py index a107ecf1e2..ff2826d645 100644 --- a/test/samples/Reshape/bitcast_dtype_alias.py +++ b/test/samples/Reshape/bitcast_dtype_alias.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType, IntegerType def build(): diff --git a/test/samples/Reshape/bitcast_inplace_cvt.py b/test/samples/Reshape/bitcast_inplace_cvt.py index 7ba321e293..0f504f7590 100644 --- a/test/samples/Reshape/bitcast_inplace_cvt.py +++ b/test/samples/Reshape/bitcast_inplace_cvt.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F16Type, F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F16Type, F32Type, IndexType def build(): diff --git a/test/samples/Reshape/bitcast_same_dtype_invalid.py b/test/samples/Reshape/bitcast_same_dtype_invalid.py index 44f3bd3c46..ea0271ffd5 100644 --- a/test/samples/Reshape/bitcast_same_dtype_invalid.py +++ b/test/samples/Reshape/bitcast_same_dtype_invalid.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr -from mlir.dialects import func, pto -from mlir.ir import F32Type +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr +from ptoas.mlir.dialects import func, pto +from ptoas.mlir.ir import F32Type def build(): diff --git a/test/samples/Reshape/reshape.py b/test/samples/Reshape/reshape.py index 244b3a540d..ce265c0ccb 100644 --- a/test/samples/Reshape/reshape.py +++ b/test/samples/Reshape/reshape.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rowexpand/rowexpand.py b/test/samples/Rowexpand/rowexpand.py index 6f0622cbec..54c62361dc 100644 --- a/test/samples/Rowexpand/rowexpand.py +++ b/test/samples/Rowexpand/rowexpand.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rowexpandadd/rowexpandadd.py b/test/samples/Rowexpandadd/rowexpandadd.py index cf4faa7f16..74d8414337 100644 --- a/test/samples/Rowexpandadd/rowexpandadd.py +++ b/test/samples/Rowexpandadd/rowexpandadd.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rowexpandadd/rowexpandadd_src1_width_invalid.py b/test/samples/Rowexpandadd/rowexpandadd_src1_width_invalid.py index a47feb3b80..19a68eeb5f 100644 --- a/test/samples/Rowexpandadd/rowexpandadd_src1_width_invalid.py +++ b/test/samples/Rowexpandadd/rowexpandadd_src1_width_invalid.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr -from mlir.dialects import func, pto -from mlir.ir import F32Type +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr +from ptoas.mlir.dialects import func, pto +from ptoas.mlir.ir import F32Type def build(): diff --git a/test/samples/Rowexpanddiv/rowexpanddiv.py b/test/samples/Rowexpanddiv/rowexpanddiv.py index aa6c2e46fe..cff800754f 100644 --- a/test/samples/Rowexpanddiv/rowexpanddiv.py +++ b/test/samples/Rowexpanddiv/rowexpanddiv.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rowexpandexpdif/rowexpandexpdif.py b/test/samples/Rowexpandexpdif/rowexpandexpdif.py index 6955a0d411..ca0c22d032 100644 --- a/test/samples/Rowexpandexpdif/rowexpandexpdif.py +++ b/test/samples/Rowexpandexpdif/rowexpandexpdif.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rowexpandexpdif/rowexpandexpdif_dtype_invalid.py b/test/samples/Rowexpandexpdif/rowexpandexpdif_dtype_invalid.py index 7027715436..772b2d5cf5 100644 --- a/test/samples/Rowexpandexpdif/rowexpandexpdif_dtype_invalid.py +++ b/test/samples/Rowexpandexpdif/rowexpandexpdif_dtype_invalid.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr -from mlir.dialects import func, pto -from mlir.ir import IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr +from ptoas.mlir.dialects import func, pto +from ptoas.mlir.ir import IntegerType def build(): diff --git a/test/samples/Rowexpandmax/rowexpandmax.py b/test/samples/Rowexpandmax/rowexpandmax.py index 26b2d5d2dd..fc6e695974 100644 --- a/test/samples/Rowexpandmax/rowexpandmax.py +++ b/test/samples/Rowexpandmax/rowexpandmax.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rowexpandmax/rowexpandmax_a5_tmp_invalid.py b/test/samples/Rowexpandmax/rowexpandmax_a5_tmp_invalid.py index 48d832a527..1c3ac39955 100644 --- a/test/samples/Rowexpandmax/rowexpandmax_a5_tmp_invalid.py +++ b/test/samples/Rowexpandmax/rowexpandmax_a5_tmp_invalid.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, StringAttr, MLIRError, UnitAttr -from mlir.dialects import func, pto -from mlir.ir import F32Type, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, StringAttr, MLIRError, UnitAttr +from ptoas.mlir.dialects import func, pto +from ptoas.mlir.ir import F32Type, IntegerType def build(): diff --git a/test/samples/Rowexpandmin/rowexpandmin.py b/test/samples/Rowexpandmin/rowexpandmin.py index b708716550..9ea8728d0a 100644 --- a/test/samples/Rowexpandmin/rowexpandmin.py +++ b/test/samples/Rowexpandmin/rowexpandmin.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rowexpandmul/rowexpandmul.py b/test/samples/Rowexpandmul/rowexpandmul.py index e90dfce604..066eb3817e 100644 --- a/test/samples/Rowexpandmul/rowexpandmul.py +++ b/test/samples/Rowexpandmul/rowexpandmul.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rowexpandsub/rowexpandsub.py b/test/samples/Rowexpandsub/rowexpandsub.py index 15768203c6..04a84f89a9 100644 --- a/test/samples/Rowexpandsub/rowexpandsub.py +++ b/test/samples/Rowexpandsub/rowexpandsub.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rowmax/rowmax.py b/test/samples/Rowmax/rowmax.py index 8e7798d8ac..30819eca8a 100644 --- a/test/samples/Rowmax/rowmax.py +++ b/test/samples/Rowmax/rowmax.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rowmin/rowmin.py b/test/samples/Rowmin/rowmin.py index 4c7c3fde81..a51efdf81c 100644 --- a/test/samples/Rowmin/rowmin.py +++ b/test/samples/Rowmin/rowmin.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rowprod/rowprod.py b/test/samples/Rowprod/rowprod.py index 7246a7407a..15e7c55734 100644 --- a/test/samples/Rowprod/rowprod.py +++ b/test/samples/Rowprod/rowprod.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rowsum/rowsum.py b/test/samples/Rowsum/rowsum.py index 452fb36c32..d7c5842f82 100644 --- a/test/samples/Rowsum/rowsum.py +++ b/test/samples/Rowsum/rowsum.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Rsqrt/rsqrt.py b/test/samples/Rsqrt/rsqrt.py index a43331cc41..bc04f8a13d 100644 --- a/test/samples/Rsqrt/rsqrt.py +++ b/test/samples/Rsqrt/rsqrt.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/ScalarPtr/scalar_ptr.py b/test/samples/ScalarPtr/scalar_ptr.py index baec3bd58c..f4f5e20bff 100644 --- a/test/samples/ScalarPtr/scalar_ptr.py +++ b/test/samples/ScalarPtr/scalar_ptr.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Scatter/scatter.py b/test/samples/Scatter/scatter.py index 569f612c82..6ef873b06d 100644 --- a/test/samples/Scatter/scatter.py +++ b/test/samples/Scatter/scatter.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.dialects import arith, func, pto -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr def build(): diff --git a/test/samples/Sel/sel.py b/test/samples/Sel/sel.py index ffbd047026..f4ae49b5ac 100644 --- a/test/samples/Sel/sel.py +++ b/test/samples/Sel/sel.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType, IntegerType def build(): diff --git a/test/samples/Sels/sels.py b/test/samples/Sels/sels.py index 40411f708a..1e10484e6b 100644 --- a/test/samples/Sels/sels.py +++ b/test/samples/Sels/sels.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, IntegerType, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, IntegerType, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): with Context() as ctx: diff --git a/test/samples/SetValidShape/set_validshape.py b/test/samples/SetValidShape/set_validshape.py index 93a9edb7ee..dc003b07d2 100644 --- a/test/samples/SetValidShape/set_validshape.py +++ b/test/samples/SetValidShape/set_validshape.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Shl/shl.py b/test/samples/Shl/shl.py index 2c65bdab16..3a45069f6f 100644 --- a/test/samples/Shl/shl.py +++ b/test/samples/Shl/shl.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IntegerType, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IntegerType, IndexType def build(): diff --git a/test/samples/Shls/shls.py b/test/samples/Shls/shls.py index 3359f4f47e..19886789e0 100644 --- a/test/samples/Shls/shls.py +++ b/test/samples/Shls/shls.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IndexType, IntegerType def build(): diff --git a/test/samples/Shr/shr.py b/test/samples/Shr/shr.py index ca500102dd..5040781517 100644 --- a/test/samples/Shr/shr.py +++ b/test/samples/Shr/shr.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IntegerType, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IntegerType, IndexType def build(): diff --git a/test/samples/Shrs/shrs.py b/test/samples/Shrs/shrs.py index a1640c0e08..448d4049c1 100644 --- a/test/samples/Shrs/shrs.py +++ b/test/samples/Shrs/shrs.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IndexType, IntegerType def build(): diff --git a/test/samples/Sort32/sort32.py b/test/samples/Sort32/sort32.py index 79c12c1c7d..d32567ea13 100644 --- a/test/samples/Sort32/sort32.py +++ b/test/samples/Sort32/sort32.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IntegerType, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IntegerType, IndexType def build(): diff --git a/test/samples/Sqrt/sqrt.py b/test/samples/Sqrt/sqrt.py index d2f34653ab..fc9cfaa525 100644 --- a/test/samples/Sqrt/sqrt.py +++ b/test/samples/Sqrt/sqrt.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Storefp/storefp.py b/test/samples/Storefp/storefp.py index 0d8edd5dcb..a37f689366 100644 --- a/test/samples/Storefp/storefp.py +++ b/test/samples/Storefp/storefp.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IntegerType, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IntegerType, IndexType def build(): with Context() as ctx: diff --git a/test/samples/Storefp/storefp_invalid.py b/test/samples/Storefp/storefp_invalid.py index 92906076ab..aeed23011e 100644 --- a/test/samples/Storefp/storefp_invalid.py +++ b/test/samples/Storefp/storefp_invalid.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr -from mlir.dialects import func, pto -from mlir.ir import F16Type, IntegerType, MemRefType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr +from ptoas.mlir.dialects import func, pto +from ptoas.mlir.ir import F16Type, IntegerType, MemRefType def build(): diff --git a/test/samples/Sub/sub.py b/test/samples/Sub/sub.py index 4f199b1a16..375b9ded5c 100644 --- a/test/samples/Sub/sub.py +++ b/test/samples/Sub/sub.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/SubView/subview.py b/test/samples/SubView/subview.py index 7b33c74fa1..ecdc4cb940 100644 --- a/test/samples/SubView/subview.py +++ b/test/samples/SubView/subview.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/SubView/subview_boxed_dynamic.py b/test/samples/SubView/subview_boxed_dynamic.py index 57ce85d657..6a69817376 100644 --- a/test/samples/SubView/subview_boxed_dynamic.py +++ b/test/samples/SubView/subview_boxed_dynamic.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F16Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F16Type, IndexType def build(): diff --git a/test/samples/SubView/subview_boxed_invalid.py b/test/samples/SubView/subview_boxed_invalid.py index 6099705119..03a76ba229 100644 --- a/test/samples/SubView/subview_boxed_invalid.py +++ b/test/samples/SubView/subview_boxed_invalid.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F16Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F16Type, IndexType def build(): diff --git a/test/samples/SubView/subview_tsubs.py b/test/samples/SubView/subview_tsubs.py index 7081d0fb6f..60676aed00 100644 --- a/test/samples/SubView/subview_tsubs.py +++ b/test/samples/SubView/subview_tsubs.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/SubView/vadd_pto_pingpong.py b/test/samples/SubView/vadd_pto_pingpong.py index f6e106c26d..7153f50ebe 100644 --- a/test/samples/SubView/vadd_pto_pingpong.py +++ b/test/samples/SubView/vadd_pto_pingpong.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build_pingpong(): with Context() as ctx: diff --git a/test/samples/Subc/subc.py b/test/samples/Subc/subc.py index d220f4c378..f06057ded5 100644 --- a/test/samples/Subc/subc.py +++ b/test/samples/Subc/subc.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Subs/subs.py b/test/samples/Subs/subs.py index e0e60988ac..a59742a8ac 100644 --- a/test/samples/Subs/subs.py +++ b/test/samples/Subs/subs.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Subsc/subsc.py b/test/samples/Subsc/subsc.py index 510d913f63..f0005a1b0a 100644 --- a/test/samples/Subsc/subsc.py +++ b/test/samples/Subsc/subsc.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Sync/sync.py b/test/samples/Sync/sync.py index 5b404f618a..b7b72e09e4 100644 --- a/test/samples/Sync/sync.py +++ b/test/samples/Sync/sync.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Sync/syncHigh.py b/test/samples/Sync/syncHigh.py index 61e797e15b..ff4dd26e65 100755 --- a/test/samples/Sync/syncHigh.py +++ b/test/samples/Sync/syncHigh.py @@ -7,9 +7,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Sync/test_a5_buf_sync.py b/test/samples/Sync/test_a5_buf_sync.py index 62be895521..9a5676c370 100644 --- a/test/samples/Sync/test_a5_buf_sync.py +++ b/test/samples/Sync/test_a5_buf_sync.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, pto +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, pto def build(): diff --git a/test/samples/Sync/test_inject_sync_if.py b/test/samples/Sync/test_inject_sync_if.py index c0ffd0ea81..29478d54b6 100644 --- a/test/samples/Sync/test_inject_sync_if.py +++ b/test/samples/Sync/test_inject_sync_if.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, Location, @@ -16,7 +16,7 @@ IndexType, IntegerType, ) -from mlir.dialects import func, arith, scf, pto +from ptoas.mlir.dialects import func, arith, scf, pto def _idx_const(v: int): diff --git a/test/samples/Sync/test_inject_sync_if_else.py b/test/samples/Sync/test_inject_sync_if_else.py index 4dc1d20d0c..c175f15ed0 100644 --- a/test/samples/Sync/test_inject_sync_if_else.py +++ b/test/samples/Sync/test_inject_sync_if_else.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, Location, @@ -16,7 +16,7 @@ IndexType, IntegerType, ) -from mlir.dialects import func, arith, scf, pto +from ptoas.mlir.dialects import func, arith, scf, pto def _idx_const(v: int): diff --git a/test/samples/Sync/test_inject_sync_intra_pipe_barrier.py b/test/samples/Sync/test_inject_sync_intra_pipe_barrier.py index 0f63d00657..60059ae3ef 100644 --- a/test/samples/Sync/test_inject_sync_intra_pipe_barrier.py +++ b/test/samples/Sync/test_inject_sync_intra_pipe_barrier.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.ir import F16Type -from mlir.dialects import func, pto +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.ir import F16Type +from ptoas.mlir.dialects import func, pto def build(): diff --git a/test/samples/Sync/test_inject_sync_loop.py b/test/samples/Sync/test_inject_sync_loop.py index 3b12958241..e2221dea8c 100644 --- a/test/samples/Sync/test_inject_sync_loop.py +++ b/test/samples/Sync/test_inject_sync_loop.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, Location, @@ -15,8 +15,8 @@ F16Type, IndexType, ) -from mlir.dialects import func, arith, scf, pto -from mlir.dialects.arith import CmpIPredicate +from ptoas.mlir.dialects import func, arith, scf, pto +from ptoas.mlir.dialects.arith import CmpIPredicate def _idx_const(v: int): diff --git a/test/samples/Sync/test_inject_sync_loop_nest.py b/test/samples/Sync/test_inject_sync_loop_nest.py index 3e25acc909..b561f0846f 100644 --- a/test/samples/Sync/test_inject_sync_loop_nest.py +++ b/test/samples/Sync/test_inject_sync_loop_nest.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, Location, @@ -15,8 +15,8 @@ F16Type, IndexType, ) -from mlir.dialects import func, arith, scf, pto -from mlir.dialects.arith import CmpIPredicate +from ptoas.mlir.dialects import func, arith, scf, pto +from ptoas.mlir.dialects.arith import CmpIPredicate def _idx_const(v: int): diff --git a/test/samples/Sync/test_inject_sync_two_event_id.py b/test/samples/Sync/test_inject_sync_two_event_id.py index 0b42083c95..e277fe5d95 100644 --- a/test/samples/Sync/test_inject_sync_two_event_id.py +++ b/test/samples/Sync/test_inject_sync_two_event_id.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, Location, @@ -15,7 +15,7 @@ F16Type, IndexType, ) -from mlir.dialects import func, arith, pto +from ptoas.mlir.dialects import func, arith, pto def _idx_const(v: int): diff --git a/test/samples/Sync/test_intercore_sync_a3.py b/test/samples/Sync/test_intercore_sync_a3.py index 5ce20add1d..59fb54d326 100644 --- a/test/samples/Sync/test_intercore_sync_a3.py +++ b/test/samples/Sync/test_intercore_sync_a3.py @@ -7,7 +7,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, F32Type, @@ -18,7 +18,7 @@ MemRefType, Module, ) -from mlir.dialects import arith, func, pto, scf +from ptoas.mlir.dialects import arith, func, pto, scf def build(): diff --git a/test/samples/Sync/test_intercore_sync_a3_dyn.py b/test/samples/Sync/test_intercore_sync_a3_dyn.py index 302554847b..375929788f 100644 --- a/test/samples/Sync/test_intercore_sync_a3_dyn.py +++ b/test/samples/Sync/test_intercore_sync_a3_dyn.py @@ -7,7 +7,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, F32Type, @@ -18,7 +18,7 @@ MemRefType, Module, ) -from mlir.dialects import arith, func, pto, scf +from ptoas.mlir.dialects import arith, func, pto, scf def build(): diff --git a/test/samples/Sync/test_intercore_sync_a3_missing_setffts.py b/test/samples/Sync/test_intercore_sync_a3_missing_setffts.py index 847e29c9a9..223b4b421c 100644 --- a/test/samples/Sync/test_intercore_sync_a3_missing_setffts.py +++ b/test/samples/Sync/test_intercore_sync_a3_missing_setffts.py @@ -7,8 +7,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, InsertionPoint, Location, Module, UnitAttr -from mlir.dialects import func, pto +from ptoas.mlir.ir import Context, InsertionPoint, Location, Module, UnitAttr +from ptoas.mlir.dialects import func, pto def build(): diff --git a/test/samples/Sync/test_intercore_sync_a3_modes.py b/test/samples/Sync/test_intercore_sync_a3_modes.py index ff7614b488..f3c9d4c5e3 100644 --- a/test/samples/Sync/test_intercore_sync_a3_modes.py +++ b/test/samples/Sync/test_intercore_sync_a3_modes.py @@ -7,8 +7,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, IndexType, InsertionPoint, IntegerType, Location, MemRefType, Module, UnitAttr -from mlir.dialects import func, pto +from ptoas.mlir.ir import Context, IndexType, InsertionPoint, IntegerType, Location, MemRefType, Module, UnitAttr +from ptoas.mlir.dialects import func, pto def build(): diff --git a/test/samples/Sync/test_intercore_sync_a5.py b/test/samples/Sync/test_intercore_sync_a5.py index c3f9607ed4..d7089fc697 100644 --- a/test/samples/Sync/test_intercore_sync_a5.py +++ b/test/samples/Sync/test_intercore_sync_a5.py @@ -7,7 +7,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, F32Type, @@ -16,7 +16,7 @@ Location, Module, ) -from mlir.dialects import arith, func, pto +from ptoas.mlir.dialects import arith, func, pto def build(): diff --git a/test/samples/Sync/test_intercore_sync_a5_dyn.py b/test/samples/Sync/test_intercore_sync_a5_dyn.py index cc7b9d7535..b53e759143 100644 --- a/test/samples/Sync/test_intercore_sync_a5_dyn.py +++ b/test/samples/Sync/test_intercore_sync_a5_dyn.py @@ -7,8 +7,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto def build(): diff --git a/test/samples/Sync/test_intercore_sync_a5_functional.py b/test/samples/Sync/test_intercore_sync_a5_functional.py index 77d37495e5..4049393f51 100644 --- a/test/samples/Sync/test_intercore_sync_a5_functional.py +++ b/test/samples/Sync/test_intercore_sync_a5_functional.py @@ -7,8 +7,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto def build(): diff --git a/test/samples/Sync/test_intercore_sync_a5_ptoisa_vec.py b/test/samples/Sync/test_intercore_sync_a5_ptoisa_vec.py index 68919478f1..04beb6fe3a 100644 --- a/test/samples/Sync/test_intercore_sync_a5_ptoisa_vec.py +++ b/test/samples/Sync/test_intercore_sync_a5_ptoisa_vec.py @@ -7,8 +7,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto def build(): diff --git a/test/samples/Sync/test_mem_inject_sync_basic.py b/test/samples/Sync/test_mem_inject_sync_basic.py index 01d29aa6dc..878523918f 100644 --- a/test/samples/Sync/test_mem_inject_sync_basic.py +++ b/test/samples/Sync/test_mem_inject_sync_basic.py @@ -6,7 +6,7 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, Location, @@ -16,7 +16,7 @@ IndexType, IntegerType, ) -from mlir.dialects import func, arith, scf, pto +from ptoas.mlir.dialects import func, arith, scf, pto def _idx_const(v: int): diff --git a/test/samples/Sync/test_set_wait_unified_api.py b/test/samples/Sync/test_set_wait_unified_api.py index 1f3172ded7..5df69762d9 100644 --- a/test/samples/Sync/test_set_wait_unified_api.py +++ b/test/samples/Sync/test_set_wait_unified_api.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import arith, func, pto -from mlir.ir import IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import arith, func, pto +from ptoas.mlir.ir import IndexType def build(): diff --git a/test/samples/Sync/tmatmulk_autosync.py b/test/samples/Sync/tmatmulk_autosync.py index d89717c1fb..6a2f3756dd 100644 --- a/test/samples/Sync/tmatmulk_autosync.py +++ b/test/samples/Sync/tmatmulk_autosync.py @@ -6,13 +6,13 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, Location, InsertionPoint, IndexType, IntegerType, F16Type, F32Type, StringAttr ) -from mlir.dialects import func, arith, scf, pto, builtin -from mlir.dialects.arith import CmpIPredicate +from ptoas.mlir.dialects import func, arith, scf, pto, builtin +from ptoas.mlir.dialects.arith import CmpIPredicate def _idx_const(v: int): diff --git a/test/samples/Sync/tmatmulk_autosync_a5.py b/test/samples/Sync/tmatmulk_autosync_a5.py index e3f3b012c9..106ebc34a5 100644 --- a/test/samples/Sync/tmatmulk_autosync_a5.py +++ b/test/samples/Sync/tmatmulk_autosync_a5.py @@ -6,13 +6,13 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import ( +from ptoas.mlir.ir import ( UnitAttr, Context, Location, InsertionPoint, IndexType, IntegerType, F16Type, F32Type, StringAttr ) -from mlir.dialects import func, arith, scf, pto, builtin -from mlir.dialects.arith import CmpIPredicate +from ptoas.mlir.dialects import func, arith, scf, pto, builtin +from ptoas.mlir.dialects.arith import CmpIPredicate def _idx_const(v: int): diff --git a/test/samples/SyncAll/syncall_binding.py b/test/samples/SyncAll/syncall_binding.py index 1ac3e1472b..578093c131 100644 --- a/test/samples/SyncAll/syncall_binding.py +++ b/test/samples/SyncAll/syncall_binding.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Attribute, Context, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Attribute, Context, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto def _mode(name): diff --git a/test/samples/TInsert/tinsert.py b/test/samples/TInsert/tinsert.py index 7180061e30..dbb367c0e4 100644 --- a/test/samples/TInsert/tinsert.py +++ b/test/samples/TInsert/tinsert.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F16Type, F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F16Type, F32Type, IndexType def build(): diff --git a/test/samples/TInsert/tinsert_fp.py b/test/samples/TInsert/tinsert_fp.py index dfc99bb116..470dbf7beb 100644 --- a/test/samples/TInsert/tinsert_fp.py +++ b/test/samples/TInsert/tinsert_fp.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType, IntegerType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType, IntegerType def build(): diff --git a/test/samples/TPrefetch/tprefetch.py b/test/samples/TPrefetch/tprefetch.py index f08919bdd0..9338203f02 100644 --- a/test/samples/TPrefetch/tprefetch.py +++ b/test/samples/TPrefetch/tprefetch.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, InsertionPoint, Location, Module, IndexType, F16Type, UnitAttr -from mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Context, InsertionPoint, Location, Module, IndexType, F16Type, UnitAttr +from ptoas.mlir.dialects import arith, func, pto def build(): diff --git a/test/samples/TPrefetchAsync/tprefetch_async_binding.py b/test/samples/TPrefetchAsync/tprefetch_async_binding.py index d28bcd212a..b21f50ff83 100644 --- a/test/samples/TPrefetchAsync/tprefetch_async_binding.py +++ b/test/samples/TPrefetchAsync/tprefetch_async_binding.py @@ -6,8 +6,8 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr -from mlir.dialects import arith, func, pto +from ptoas.mlir.ir import Context, F32Type, IndexType, InsertionPoint, IntegerType, Location, Module, UnitAttr +from ptoas.mlir.dialects import arith, func, pto def build(): diff --git a/test/samples/TTri/ttri.py b/test/samples/TTri/ttri.py index 7000b10645..401e288f17 100644 --- a/test/samples/TTri/ttri.py +++ b/test/samples/TTri/ttri.py @@ -12,8 +12,8 @@ stores them into a single output buffer as two consecutive 32x32 slices. """ -from mlir.ir import Context, Location, Module, InsertionPoint, IndexType, IntegerType, UnitAttr -from mlir.dialects import func, arith, pto +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, IndexType, IntegerType, UnitAttr +from ptoas.mlir.dialects import func, arith, pto def build(): diff --git a/test/samples/TileSetGetValue/tileSetGetValue.py b/test/samples/TileSetGetValue/tileSetGetValue.py index 46ee475e89..49d5529f24 100644 --- a/test/samples/TileSetGetValue/tileSetGetValue.py +++ b/test/samples/TileSetGetValue/tileSetGetValue.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/TileSetGetValue/tile_getval_mat_invalid.py b/test/samples/TileSetGetValue/tile_getval_mat_invalid.py index 18065bd6e3..917f2c2376 100644 --- a/test/samples/TileSetGetValue/tile_getval_mat_invalid.py +++ b/test/samples/TileSetGetValue/tile_getval_mat_invalid.py @@ -10,9 +10,9 @@ Test that TGetValOp rejects MAT tile_buf (Ascend hardware does not support reading from MAT tile_buf to scalar). Verification must fail. """ -from mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, MLIRError, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Tpow/tpow.py b/test/samples/Tpow/tpow.py index e58442d06b..a648654461 100644 --- a/test/samples/Tpow/tpow.py +++ b/test/samples/Tpow/tpow.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Tpows/tpows.py b/test/samples/Tpows/tpows.py index 1c4ad7acc1..8bcc8cc00a 100644 --- a/test/samples/Tpows/tpows.py +++ b/test/samples/Tpows/tpows.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Trans/trans.py b/test/samples/Trans/trans.py index a7b6b2e5b8..9b17b31b1e 100644 --- a/test/samples/Trans/trans.py +++ b/test/samples/Trans/trans.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/Trap/trap.py b/test/samples/Trap/trap.py index b4376af13a..005b559f1d 100644 --- a/test/samples/Trap/trap.py +++ b/test/samples/Trap/trap.py @@ -7,8 +7,8 @@ # See LICENSE in the root of the software repository for the full text of the License. """Test pto.trap: generates TRAP() in C++.""" -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, pto +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, pto def build(): diff --git a/test/samples/VectorAddition/vadd_pto_ir.py b/test/samples/VectorAddition/vadd_pto_ir.py index c06bd018f9..4e43773bd8 100644 --- a/test/samples/VectorAddition/vadd_pto_ir.py +++ b/test/samples/VectorAddition/vadd_pto_ir.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): diff --git a/test/samples/VectorAddition/vadd_validshape.py b/test/samples/VectorAddition/vadd_validshape.py index 269c7604a1..3239998bb2 100644 --- a/test/samples/VectorAddition/vadd_validshape.py +++ b/test/samples/VectorAddition/vadd_validshape.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, IntegerType, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, IntegerType, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): with Context() as ctx: diff --git a/test/samples/VectorAddition/vadd_validshape_dynamic.py b/test/samples/VectorAddition/vadd_validshape_dynamic.py index 639716c890..01e6818fe8 100644 --- a/test/samples/VectorAddition/vadd_validshape_dynamic.py +++ b/test/samples/VectorAddition/vadd_validshape_dynamic.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, IntegerType, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, IntegerType, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build(): with Context() as ctx: diff --git a/test/samples/VectorAddition/vadd_validshape_hyper.py b/test/samples/VectorAddition/vadd_validshape_hyper.py index 45b1415de6..5172f5192a 100644 --- a/test/samples/VectorAddition/vadd_validshape_hyper.py +++ b/test/samples/VectorAddition/vadd_validshape_hyper.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, IntegerType, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import F32Type, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, IntegerType, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import F32Type, IndexType def build_mixed_shape(): with Context() as ctx: diff --git a/test/samples/Xor/xor.py b/test/samples/Xor/xor.py index 0d25112001..86acc61a46 100644 --- a/test/samples/Xor/xor.py +++ b/test/samples/Xor/xor.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IntegerType, IndexType +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IntegerType, IndexType def build(): diff --git a/test/samples/Xors/xors.py b/test/samples/Xors/xors.py index 02f2c8c8e6..3ef70a27ea 100644 --- a/test/samples/Xors/xors.py +++ b/test/samples/Xors/xors.py @@ -6,9 +6,9 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. -from mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr -from mlir.dialects import func, arith, pto -from mlir.ir import IntegerType, IndexType, IntegerAttr +from ptoas.mlir.ir import Context, Location, Module, InsertionPoint, UnitAttr +from ptoas.mlir.dialects import func, arith, pto +from ptoas.mlir.ir import IntegerType, IndexType, IntegerAttr def build(): diff --git a/test/tilelib-st/README.md b/test/tilelib-st/README.md index cb87dee72d..4dce3784ca 100644 --- a/test/tilelib-st/README.md +++ b/test/tilelib-st/README.md @@ -307,7 +307,7 @@ build: ```bash "$PYTHON_BIN" - <<'PY' from ptodsl import pto # noqa: F401 -from mlir.dialects import pto as _pto # noqa: F401 +from ptoas.mlir.dialects import pto as _pto # noqa: F401 print("ptodsl imports ok") PY diff --git a/tilelang-dsl/CMakeLists.txt b/tilelang-dsl/CMakeLists.txt index db11784d6f..3c00670513 100644 --- a/tilelang-dsl/CMakeLists.txt +++ b/tilelang-dsl/CMakeLists.txt @@ -29,7 +29,7 @@ add_dependencies(TileLangDSLPackage TileLangDSLPackageStage) install( DIRECTORY "${TILELANG_DSL_PACKAGE_SRC_DIR}" DESTINATION "." - COMPONENT PTOAS_Runtime + COMPONENT PTOAS_CompilerArchive PATTERN "__pycache__" EXCLUDE PATTERN "*.pyc" EXCLUDE ) @@ -37,7 +37,7 @@ install( install( DIRECTORY "${CMAKE_SOURCE_DIR}/lib/TileOps" DESTINATION "share/ptoas" - COMPONENT PTOAS_Runtime + COMPONENT PTOAS_CompilerArchive PATTERN "__pycache__" EXCLUDE PATTERN "*.pyc" EXCLUDE ) diff --git a/tilelang-dsl/python/tilelang_dsl/env_config.py b/tilelang-dsl/python/tilelang_dsl/env_config.py index 47fdd9b7ef..a6967e869d 100644 --- a/tilelang-dsl/python/tilelang_dsl/env_config.py +++ b/tilelang-dsl/python/tilelang_dsl/env_config.py @@ -24,7 +24,7 @@ def check_mlir_bindings_available() -> bool: True if mlir.ir module can be imported, False otherwise. """ try: - from mlir import ir + from ptoas.mlir import ir return True except ImportError: return False @@ -37,7 +37,7 @@ def check_pto_dialect_available() -> bool: True if mlir.dialects.pto can be imported, False otherwise. """ try: - from mlir.dialects import pto + from ptoas.mlir.dialects import pto return True except ImportError: return False @@ -83,7 +83,7 @@ def print_environment_help() -> None: print(" export PYTHONPATH=$PTOAS_BUILD_DIR/python") print() print("After setup, verify with:") - print(" python3 -c 'from mlir import ir; print(\"MLIR bindings OK\")'") + print(" python3 -c 'from ptoas.mlir import ir; print(\"MLIR bindings OK\")'") print(" python3 -c 'from pto.dialects import pto; print(\"PTO dialect OK\")'") print() print("=" * 60) diff --git a/tilelang-dsl/python/tilelang_dsl/lowering_backend.py b/tilelang-dsl/python/tilelang_dsl/lowering_backend.py index 80bdb4e972..9d1c5cd406 100644 --- a/tilelang-dsl/python/tilelang_dsl/lowering_backend.py +++ b/tilelang-dsl/python/tilelang_dsl/lowering_backend.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: from .semantic import SemanticKernel - from mlir import ir as _ods_ir + from ptoas.mlir import ir as _ods_ir @dataclass(frozen=True) @@ -70,10 +70,10 @@ def _parse_text_to_module( Registers all required dialects (func, arith, scf, pto) before parsing. """ - from mlir import ir as _ods_ir - from mlir.dialects import func as _func_dialect - from mlir.dialects import arith as _arith_dialect - from mlir.dialects import scf as _scf_dialect + from ptoas.mlir import ir as _ods_ir + from ptoas.mlir.dialects import func as _func_dialect + from ptoas.mlir.dialects import arith as _arith_dialect + from ptoas.mlir.dialects import scf as _scf_dialect from pto.dialects import pto as _pto_dialect ctx = context if context is not None else _ods_ir.Context() @@ -180,7 +180,7 @@ def _check_mlir_available(self) -> bool: return self._mlir_available try: - from mlir import ir + from ptoas.mlir import ir self._mlir_available = True except ImportError: self._mlir_available = False diff --git a/tilelang-dsl/python/tilelang_dsl/pybind_renderer.py b/tilelang-dsl/python/tilelang_dsl/pybind_renderer.py index a817cec884..b0962408af 100644 --- a/tilelang-dsl/python/tilelang_dsl/pybind_renderer.py +++ b/tilelang-dsl/python/tilelang_dsl/pybind_renderer.py @@ -14,9 +14,9 @@ Phase 2 implementation: core framework with basic stmt/expr rendering. Dependencies: - - mlir.ir: MLIR Python bindings assembled by the PTOAS build - - mlir.dialects.func, arith, scf: Standard MLIR dialects - - mlir.dialects.pto: PTO dialect bindings from PTOAS build + - ptoas.mlir.ir: MLIR Python bindings assembled by the PTOAS build + - ptoas.mlir.dialects.func, arith, scf: Standard MLIR dialects + - ptoas.mlir.dialects.pto: PTO dialect bindings from PTOAS build To use this module from a build tree, ensure PYTHONPATH includes: $PTOAS_BUILD_DIR/python @@ -31,7 +31,7 @@ from typing import Any, TYPE_CHECKING if TYPE_CHECKING: - from mlir import ir as _ods_ir + from ptoas.mlir import ir as _ods_ir # Lazy imports - only load when actually needed _mlir_ir: Any = None @@ -54,10 +54,10 @@ def _ensure_mlir_bindings() -> None: return # Already loaded try: - from mlir import ir as mlir_ir - from mlir.dialects import func as func_dialect - from mlir.dialects import arith as arith_dialect - from mlir.dialects import scf as scf_dialect + from ptoas.mlir import ir as mlir_ir + from ptoas.mlir.dialects import func as func_dialect + from ptoas.mlir.dialects import arith as arith_dialect + from ptoas.mlir.dialects import scf as scf_dialect _mlir_ir = mlir_ir _func_dialect = func_dialect @@ -72,7 +72,7 @@ def _ensure_mlir_bindings() -> None: ) from exc try: - from mlir.dialects import pto as pto_dialect + from ptoas.mlir.dialects import pto as pto_dialect _pto_dialect = pto_dialect except ImportError: # PTO dialect is optional for some operations diff --git a/tilelang-dsl/tests/backend/test_e2e_pybind_backend.py b/tilelang-dsl/tests/backend/test_e2e_pybind_backend.py index a3b6088b93..5ca6630957 100644 --- a/tilelang-dsl/tests/backend/test_e2e_pybind_backend.py +++ b/tilelang-dsl/tests/backend/test_e2e_pybind_backend.py @@ -46,7 +46,7 @@ def _check_pto_dialect_available(): """Check if PTO dialect bindings are available.""" try: - from mlir.dialects import pto + from ptoas.mlir.dialects import pto return True except ImportError: return False diff --git a/tools/ptoas/CMakeLists.txt b/tools/ptoas/CMakeLists.txt index c0fb306764..18652a8ce3 100644 --- a/tools/ptoas/CMakeLists.txt +++ b/tools/ptoas/CMakeLists.txt @@ -130,6 +130,9 @@ add_mlir_python_extension(PTOASPythonCore _core PTOCAPI PTOASPythonCAPI ) +target_compile_definitions(PTOASPythonCore PRIVATE + "MLIR_PYTHON_PACKAGE_PREFIX=ptoas.mlir." +) target_include_directories(PTOASPythonCore PRIVATE "${CMAKE_SOURCE_DIR}/lib/Bindings/Python" ) @@ -137,10 +140,10 @@ mlir_python_setup_extension_rpath(PTOASPythonCore) if(APPLE) set_property(TARGET PTOASPythonCore PROPERTY INSTALL_RPATH - "@loader_path;@loader_path/../mlir/_mlir_libs") + "@loader_path;@loader_path/mlir/_mlir_libs") elseif(UNIX) set_property(TARGET PTOASPythonCore PROPERTY INSTALL_RPATH - "$ORIGIN;$ORIGIN/../mlir/_mlir_libs") + "$ORIGIN;$ORIGIN/mlir/_mlir_libs") endif() declare_mlir_python_sources(PTOASRuntimePythonSources @@ -183,7 +186,34 @@ endif() install(PROGRAMS "${PTOAS_INSTALL_WRAPPER_BINARY}" DESTINATION bin RENAME ptoas - COMPONENT PTOAS_Runtime + COMPONENT PTOAS_CompilerArchive +) + +set(PTOAS_ARCHIVE_PYTHON_VERSION_FILE + "${CMAKE_CURRENT_BINARY_DIR}/.ptoas-python-version") +file(WRITE "${PTOAS_ARCHIVE_PYTHON_VERSION_FILE}" + "${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}\n") +install(FILES "${PTOAS_ARCHIVE_PYTHON_VERSION_FILE}" + DESTINATION "." + COMPONENT PTOAS_CompilerArchive +) + +# The standalone compiler archive is staged by installing PTOAS_Python first +# and PTOAS_CompilerArchive second into the same prefix. Generate the deploy +# script so known native entry points come from CMake targets; the script scans +# only AddMLIRPython's generated extension directory for the remaining modules. +configure_file( + "${CMAKE_SOURCE_DIR}/cmake/RelocatePTOASCompilerArchive.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/RelocatePTOASCompilerArchive.cmake.in" + @ONLY +) +file(GENERATE + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/RelocatePTOASCompilerArchive.cmake" + INPUT "${CMAKE_CURRENT_BINARY_DIR}/RelocatePTOASCompilerArchive.cmake.in" +) +install(SCRIPT + "${CMAKE_CURRENT_BINARY_DIR}/RelocatePTOASCompilerArchive.cmake" + COMPONENT PTOAS_CompilerArchive ) install( diff --git a/tools/ptoas/NativeModule.cpp b/tools/ptoas/NativeModule.cpp index e0097e0641..5a474cbd27 100644 --- a/tools/ptoas/NativeModule.cpp +++ b/tools/ptoas/NativeModule.cpp @@ -35,7 +35,7 @@ int runPTOASFromPython(const std::vector &arguments) { PYBIND11_MODULE(_core, module) { module.doc() = "PTOAS compiler and PTO dialect native bindings"; - py::module_::import("mlir.ir"); + py::module_::import("ptoas.mlir.ir"); mlir::pto::python::populatePTODialectBindings(module); module.def("main", &runPTOASFromPython, py::arg("argv")); } diff --git a/tools/ptoas/ptoas_wrapper.py b/tools/ptoas/ptoas_wrapper.py index 3b1e96ad25..0328d9733f 100644 --- a/tools/ptoas/ptoas_wrapper.py +++ b/tools/ptoas/ptoas_wrapper.py @@ -17,6 +17,7 @@ _PYTHON_ROOT_MODE = "@PTOAS_WRAPPER_PYTHON_ROOT_MODE@" _PYTHON_ROOT = Path(r"@PTOAS_WRAPPER_PYTHON_ROOT@") +_ARCHIVE_PYTHON_REQUIREMENT_FILE = ".ptoas-python-version" def _resolve_wrapper_path(argv0: str | None = None) -> Path: @@ -59,6 +60,18 @@ def _add_configured_python_root(wrapper: Path) -> Path: def main() -> None: wrapper = _resolve_wrapper_path() + requirement_file = wrapper.parent.parent / _ARCHIVE_PYTHON_REQUIREMENT_FILE + if requirement_file.is_file(): + required_python_major_minor = requirement_file.read_text( + encoding="utf-8" + ).strip() + actual = f"{sys.version_info.major}.{sys.version_info.minor}" + if actual != required_python_major_minor: + raise SystemExit( + "this ptoas compiler archive requires CPython " + f"{required_python_major_minor}, but the active interpreter is " + f"{actual} ({sys.executable})" + ) _add_configured_python_root(wrapper) from ptoas import _cli From bf9435606f3586d7b5e2a729a202b86a7233471e Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Mon, 27 Jul 2026 22:24:41 +0800 Subject: [PATCH 16/32] build: fix packaging gate regressions --- packaging/ptoas-vmi/pyproject.toml | 4 ++-- test/python/compact_tile_buf_asm.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packaging/ptoas-vmi/pyproject.toml b/packaging/ptoas-vmi/pyproject.toml index c5de369d66..185a135e66 100644 --- a/packaging/ptoas-vmi/pyproject.toml +++ b/packaging/ptoas-vmi/pyproject.toml @@ -55,5 +55,5 @@ PTOAS_RELEASE_VERSION_OVERRIDE = { env = "PTOAS_RELEASE_VERSION_OVERRIDE", defau [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" -input = "../../docs/release/VMI_VERSION" -regex = '^(?P[0-9]+\.[0-9]+\.[0-9]+)\s*$' +input = "../../CMakeLists.txt" +regex = 'set\s*\(\s*PTOAS_VMI_VERSION\s+"(?P[0-9]+\.[0-9]+\.[0-9]+)"\s*\)' diff --git a/test/python/compact_tile_buf_asm.py b/test/python/compact_tile_buf_asm.py index cdf74879e7..f73be451f5 100644 --- a/test/python/compact_tile_buf_asm.py +++ b/test/python/compact_tile_buf_asm.py @@ -1,4 +1,11 @@ #!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. from ptoas.mlir.ir import Context, F32Type, MLIRError, Module from ptoas.mlir.dialects import pto From 99b0d0865d97297e3c349a2df9b7714f63e71d85 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Mon, 27 Jul 2026 23:11:25 +0800 Subject: [PATCH 17/32] build: source VMI version from package metadata --- .github/scripts/compute_vmi_version.py | 73 --------- .github/scripts/update_vmi_version.py | 84 ----------- .github/workflows/build_wheel.yml | 75 +++------- .github/workflows/build_wheel_mac.yml | 32 ++-- CMakeLists.txt | 13 +- README.md | 4 +- README_en.md | 1 + lib/Bindings/Python/CMakeLists.txt | 35 ++++- packaging/ptoas-vmi/pyproject.toml | 12 +- test/python/test_release_version_scripts.py | 156 -------------------- 10 files changed, 85 insertions(+), 400 deletions(-) delete mode 100644 .github/scripts/compute_vmi_version.py delete mode 100644 .github/scripts/update_vmi_version.py delete mode 100644 test/python/test_release_version_scripts.py diff --git a/.github/scripts/compute_vmi_version.py b/.github/scripts/compute_vmi_version.py deleted file mode 100644 index 3d37f87217..0000000000 --- a/.github/scripts/compute_vmi_version.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -import argparse -import pathlib -import re -import sys - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Compute the VMI release version from the top-level CMakeLists.txt." - ) - parser.add_argument( - "--cmake-file", - default="CMakeLists.txt", - help="Path to the top-level CMakeLists.txt file.", - ) - parser.add_argument( - "--check-tag", - help="Optional release tag to validate, e.g. vmi-v0.1.0.", - ) - return parser.parse_args() - - -def read_version(cmake_file: pathlib.Path) -> str: - content = cmake_file.read_text(encoding="utf-8") - match = re.search( - r'set\s*\(\s*PTOAS_VMI_VERSION\s+"([0-9]+\.[0-9]+\.[0-9]+)"\s*\)', - content, - ) - if not match: - raise ValueError( - f'could not find \'set(PTOAS_VMI_VERSION "x.y.z")\' in {cmake_file}' - ) - return match.group(1) - - -def normalize_tag(tag: str) -> str: - normalized = tag.strip() - if normalized.startswith("vmi-"): - normalized = normalized[len("vmi-"):] - if normalized.startswith("v"): - normalized = normalized[1:] - return normalized - - -def main() -> int: - args = parse_args() - cmake_file = pathlib.Path(args.cmake_file) - version = read_version(cmake_file) - - if args.check_tag is not None: - normalized_tag = normalize_tag(args.check_tag) - if normalized_tag != version: - print( - f"release tag '{args.check_tag}' does not match computed version '{version}'", - file=sys.stderr, - ) - return 1 - - print(version) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/scripts/update_vmi_version.py b/.github/scripts/update_vmi_version.py deleted file mode 100644 index 8c029009d6..0000000000 --- a/.github/scripts/update_vmi_version.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -import argparse -import pathlib -import re -import sys - - -VMI_VERSION_RE = re.compile( - r'(set\s*\(\s*PTOAS_VMI_VERSION\s+")([0-9]+\.[0-9]+\.[0-9]+)("\s*\))' -) -TAG_VERSION_RE = re.compile(r"^vmi-v([0-9]+)\.([0-9]+)\.([0-9]+)$") - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Update the VMI version in the top-level CMakeLists.txt." - ) - parser.add_argument( - "--cmake-file", - default="CMakeLists.txt", - help="Path to the top-level CMakeLists.txt file.", - ) - parser.add_argument( - "--version", - required=True, - help="Released VMI version, for example vmi-v0.1.3.", - ) - parser.add_argument( - "--next", - action="store_true", - help="Advance the resolved version by one patch step.", - ) - return parser.parse_args() - - -def normalize_version(version: str) -> str: - match = TAG_VERSION_RE.fullmatch(version.strip()) - if not match: - raise ValueError(f"invalid VMI release tag '{version}'") - return ".".join(match.groups()) - - -def bump_version(version: str) -> str: - major, minor, patch = (int(part) for part in version.split(".")) - return f"{major}.{minor}.{patch + 1}" - - -def update_version(cmake_file: pathlib.Path, version: str) -> bool: - content = cmake_file.read_text(encoding="utf-8") - updated, count = VMI_VERSION_RE.subn( - lambda match: f"{match.group(1)}{version}{match.group(3)}", - content, - count=1, - ) - if count != 1: - raise ValueError( - f'could not find \'set(PTOAS_VMI_VERSION "x.y.z")\' in {cmake_file}' - ) - if updated == content: - return False - cmake_file.write_text(updated, encoding="utf-8") - return True - - -def main() -> int: - args = parse_args() - version = normalize_version(args.version) - if args.next: - version = bump_version(version) - update_version(pathlib.Path(args.cmake_file), version) - print(version) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index a047da35d6..1ffafa50cc 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -75,25 +75,37 @@ jobs: echo "PY_PATH=/opt/python/$PY_VER" >> $GITHUB_ENV echo "/opt/python/$PY_VER/bin" >> $GITHUB_PATH - - name: Compute PTOAS CLI version + - name: Install system dependencies + run: | + dnf install -y ninja-build cmake git ccache gcc-c++ lld zip binutils patchelf chrpath + dnf clean all + + - name: Install Python dependencies run: | + # LLVM/MLIR Python bindings are not yet compatible with pybind11 3.x + # (see pybind11 static_assert on def_property + keep_alive). + pip install --no-cache-dir numpy 'pybind11<3' nanobind 'scikit-build-core>=0.12.2,<2' auditwheel + + - name: Resolve PTOAS distribution version + run: | + VMI_PROJECT_METADATA="$(cd packaging/ptoas-vmi && python -m scikit_build_core.build project-table)" + VMI_PROJECT_VERSION="$(printf '%s\n' "${VMI_PROJECT_METADATA}" | python -c 'import json, sys; print(json.load(sys.stdin)["version"])')" if [ "${GITHUB_EVENT_NAME}" = "release" ]; then case "${GITHUB_REF_NAME}" in vmi-v*) - EXPECTED_RELEASE_VERSION="$(${PY_PATH}/bin/python .github/scripts/compute_vmi_version.py \ - --check-tag "${GITHUB_REF_NAME}")" + EXPECTED_RELEASE_VERSION="${VMI_PROJECT_VERSION}" PTOAS_VERSION="${GITHUB_REF_NAME#vmi-v}" PTOAS_CLI_VERSION="vmi ${PTOAS_VERSION}" PTOAS_PYTHON_PACKAGE_NAME="ptoas-vmi" ;; ptoas-v*) - EXPECTED_RELEASE_VERSION="$(${PY_PATH}/bin/python .github/scripts/compute_ptoas_version.py --mode release)" + EXPECTED_RELEASE_VERSION="$(python .github/scripts/compute_ptoas_version.py --mode release)" PTOAS_VERSION="${GITHUB_REF_NAME#ptoas-v}" PTOAS_CLI_VERSION="${PTOAS_VERSION}" PTOAS_PYTHON_PACKAGE_NAME="ptoas" ;; v*) - EXPECTED_RELEASE_VERSION="$(${PY_PATH}/bin/python .github/scripts/compute_ptoas_version.py --mode release)" + EXPECTED_RELEASE_VERSION="$(python .github/scripts/compute_ptoas_version.py --mode release)" PTOAS_VERSION="${GITHUB_REF_NAME#v}" PTOAS_CLI_VERSION="${PTOAS_VERSION}" PTOAS_PYTHON_PACKAGE_NAME="ptoas" @@ -104,19 +116,19 @@ jobs: ;; esac if [ "${PTOAS_VERSION}" != "${EXPECTED_RELEASE_VERSION}" ]; then - echo "release tag '${GITHUB_REF_NAME}' does not match computed version '${EXPECTED_RELEASE_VERSION}'" >&2 + echo "release tag '${GITHUB_REF_NAME}' does not match project version '${EXPECTED_RELEASE_VERSION}'" >&2 exit 1 fi elif [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ] && [ "${{ inputs.release_kind }}" = "vmi" ]; then - PTOAS_VERSION="$(${PY_PATH}/bin/python .github/scripts/compute_vmi_version.py)" + PTOAS_VERSION="${VMI_PROJECT_VERSION}" PTOAS_CLI_VERSION="vmi ${PTOAS_VERSION}" PTOAS_PYTHON_PACKAGE_NAME="ptoas-vmi" elif [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then - PTOAS_VERSION="$(${PY_PATH}/bin/python .github/scripts/compute_ptoas_version.py --mode release)" + PTOAS_VERSION="$(python .github/scripts/compute_ptoas_version.py --mode release)" PTOAS_CLI_VERSION="${PTOAS_VERSION}" PTOAS_PYTHON_PACKAGE_NAME="ptoas" else - PTOAS_VERSION="$(${PY_PATH}/bin/python .github/scripts/compute_ptoas_version.py --mode dev)" + PTOAS_VERSION="$(python .github/scripts/compute_ptoas_version.py --mode dev)" PTOAS_CLI_VERSION="${PTOAS_VERSION}" PTOAS_PYTHON_PACKAGE_NAME="ptoas" fi @@ -127,17 +139,6 @@ jobs: echo "Using PTOAS wheel version: ${PTOAS_VERSION}" echo "Using PTOAS Python package name: ${PTOAS_PYTHON_PACKAGE_NAME}" - - name: Install system dependencies - run: | - dnf install -y ninja-build cmake git ccache gcc-c++ lld zip binutils patchelf chrpath - dnf clean all - - - name: Install Python dependencies - run: | - # LLVM/MLIR Python bindings are not yet compatible with pybind11 3.x - # (see pybind11 static_assert on def_property + keep_alive). - pip install --no-cache-dir numpy 'pybind11<3' nanobind 'scikit-build-core>=0.12.2,<2' auditwheel - - name: Validate PEP 517 project metadata run: | python -m scikit_build_core.build project-table @@ -225,7 +226,6 @@ jobs: fi rm -rf "${PTO_SOURCE_DIR}/build/wheel-dist" mkdir -p "${PTO_SOURCE_DIR}/build/wheel-dist" - PTOAS_RELEASE_VERSION_OVERRIDE="${PTOAS_CLI_VERSION}" \ SKBUILD_BUILD_DIR="${PTO_BUILD_DIR}" \ python -m pip wheel "${PTOAS_WHEEL_SOURCE_DIR}" \ --no-build-isolation --no-deps \ @@ -434,36 +434,3 @@ jobs: git add CMakeLists.txt git commit -m "chore(release): bump base version to v${{ steps.bump.outputs.next_base_version }}" git push origin HEAD:${{ github.event.repository.default_branch }} - - bump_vmi_version: - name: Bump VMI version after release - if: github.event_name == 'release' && github.event.action == 'released' && startsWith(github.ref_name, 'vmi-v') - needs: upload_release_assets - runs-on: ubuntu-latest - - steps: - - name: Checkout default branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.repository.default_branch }} - - - name: Update VMI version - id: bump - run: | - next_vmi_version="$(python3 .github/scripts/update_vmi_version.py \ - --version "${GITHUB_REF_NAME}" \ - --next)" - echo "next_vmi_version=${next_vmi_version}" >> "$GITHUB_OUTPUT" - - - name: Commit version bump - run: | - if git diff --quiet -- CMakeLists.txt; then - echo "CMakeLists.txt already matches the next VMI version." - exit 0 - fi - - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add CMakeLists.txt - git commit -m "chore(release): bump VMI version to v${{ steps.bump.outputs.next_vmi_version }}" - git push origin HEAD:${{ github.event.repository.default_branch }} diff --git a/.github/workflows/build_wheel_mac.yml b/.github/workflows/build_wheel_mac.yml index 1158508314..de79399e3f 100644 --- a/.github/workflows/build_wheel_mac.yml +++ b/.github/workflows/build_wheel_mac.yml @@ -75,13 +75,24 @@ jobs: run: | echo "PY_PATH=$(python -c 'import sys; print(sys.prefix)')" >> $GITHUB_ENV - - name: Compute PTOAS CLI version + - name: Install system dependencies + run: | + brew install ninja ccache + + - name: Install Python dependencies + run: | + # LLVM/MLIR Python bindings are not yet compatible with pybind11 3.x + # (see pybind11 static_assert on def_property + keep_alive). + pip install --no-cache-dir numpy 'pybind11<3' nanobind 'scikit-build-core>=0.12.2,<2' delocate + + - name: Resolve PTOAS distribution version run: | + VMI_PROJECT_METADATA="$(cd packaging/ptoas-vmi && python -m scikit_build_core.build project-table)" + VMI_PROJECT_VERSION="$(printf '%s\n' "${VMI_PROJECT_METADATA}" | python -c 'import json, sys; print(json.load(sys.stdin)["version"])')" if [ "${GITHUB_EVENT_NAME}" = "release" ]; then case "${GITHUB_REF_NAME}" in vmi-v*) - EXPECTED_RELEASE_VERSION="$(python .github/scripts/compute_vmi_version.py \ - --check-tag "${GITHUB_REF_NAME}")" + EXPECTED_RELEASE_VERSION="${VMI_PROJECT_VERSION}" PTOAS_VERSION="${GITHUB_REF_NAME#vmi-v}" PTOAS_CLI_VERSION="vmi ${PTOAS_VERSION}" PTOAS_PYTHON_PACKAGE_NAME="ptoas-vmi" @@ -104,11 +115,11 @@ jobs: ;; esac if [ "${PTOAS_VERSION}" != "${EXPECTED_RELEASE_VERSION}" ]; then - echo "release tag '${GITHUB_REF_NAME}' does not match computed version '${EXPECTED_RELEASE_VERSION}'" >&2 + echo "release tag '${GITHUB_REF_NAME}' does not match project version '${EXPECTED_RELEASE_VERSION}'" >&2 exit 1 fi elif [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ] && [ "${{ inputs.release_kind }}" = "vmi" ]; then - PTOAS_VERSION="$(python .github/scripts/compute_vmi_version.py)" + PTOAS_VERSION="${VMI_PROJECT_VERSION}" PTOAS_CLI_VERSION="vmi ${PTOAS_VERSION}" PTOAS_PYTHON_PACKAGE_NAME="ptoas-vmi" elif [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then @@ -127,16 +138,6 @@ jobs: echo "Using PTOAS wheel version: ${PTOAS_VERSION}" echo "Using PTOAS Python package name: ${PTOAS_PYTHON_PACKAGE_NAME}" - - name: Install system dependencies - run: | - brew install ninja ccache - - - name: Install Python dependencies - run: | - # LLVM/MLIR Python bindings are not yet compatible with pybind11 3.x - # (see pybind11 static_assert on def_property + keep_alive). - pip install --no-cache-dir numpy 'pybind11<3' nanobind 'scikit-build-core>=0.12.2,<2' delocate - - name: Validate PEP 517 project metadata run: | python -m scikit_build_core.build project-table @@ -228,7 +229,6 @@ jobs: export _PYTHON_HOST_PLATFORM="$(python "$PTO_SOURCE_DIR/docker/get_macos_wheel_plat_name.py" "$TARGET_ARCH")" rm -rf "${PTO_SOURCE_DIR}/build/wheel-dist" mkdir -p "${PTO_SOURCE_DIR}/build/wheel-dist" - PTOAS_RELEASE_VERSION_OVERRIDE="${PTOAS_CLI_VERSION}" \ SKBUILD_BUILD_DIR="${PTO_BUILD_DIR}" \ python -m pip wheel "${PTOAS_WHEEL_SOURCE_DIR}" \ --no-build-isolation --no-deps \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 156ca5d23f..b3d8e3657a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,7 +36,6 @@ if((NOT DEFINED MLIR_DIR OR MLIR_DIR STREQUAL "") endif() project(ptoas VERSION 0.55) -set(PTOAS_VMI_VERSION "0.1.4") # Wheel builds carry the same Linux hardening policy as release CMake builds. # Including the cache settings here keeps the policy backend-independent. @@ -46,6 +45,11 @@ endif() set(PTOAS_RELEASE_VERSION_OVERRIDE "" CACHE STRING "Override the version printed by ptoas --version") +# scikit-build-core owns wheel metadata and supplies its resolved PEP 621 +# version here. The optional label distinguishes distributions that install the +# same CLI without making CMake a second source of package-version truth. +set(PTOAS_CLI_VERSION_LABEL "" + CACHE STRING "Optional label prepended to the packaged ptoas version") set(PTOAS_EXACT_GIT_TAG_VERSION "") if(EXISTS "${PROJECT_SOURCE_DIR}/.git") find_package(Git QUIET) @@ -68,6 +72,13 @@ set(PTOAS_CLI_VERSION "${PROJECT_VERSION}") if(PTOAS_EXACT_GIT_TAG_VERSION) set(PTOAS_CLI_VERSION "${PTOAS_EXACT_GIT_TAG_VERSION}") endif() +if(SKBUILD AND SKBUILD_PROJECT_VERSION_FULL) + set(PTOAS_CLI_VERSION "${SKBUILD_PROJECT_VERSION_FULL}") +endif() +if(PTOAS_CLI_VERSION_LABEL) + set(PTOAS_CLI_VERSION + "${PTOAS_CLI_VERSION_LABEL} ${PTOAS_CLI_VERSION}") +endif() if(PTOAS_RELEASE_VERSION_OVERRIDE) set(PTOAS_CLI_VERSION "${PTOAS_RELEASE_VERSION_OVERRIDE}") endif() diff --git a/README.md b/README.md index 64f3858ed5..e51767fea0 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,9 @@ from ptoas.mlir.dialects import pto as mlir_pto > - `ptoas-bin-*.tar.gz` 这类 compiler-only 二进制 tarball 只提供 CLI/toolchain， > **不是** PTODSL-capable Python distribution；仅解压 tarball 不能保证 > `import ptodsl` 可用。 -> - release tag 约定：`ptoas-vX.Y` 发布主工具链，`vmi-vA.B.C` 发布 VMI 文档/规范。 +> - release tag 约定：`ptoas-vX.Y` 发布主工具链，`vmi-vA.B.C` 发布 +> `ptoas-vmi` distribution。创建 VMI release tag 前，应通过发布 PR 将 +> `packaging/ptoas-vmi/pyproject.toml` 中的版本更新为相同的 `A.B.C`。 --- diff --git a/README_en.md b/README_en.md index f582003bb2..efde0e8c48 100644 --- a/README_en.md +++ b/README_en.md @@ -161,6 +161,7 @@ from ptoas.mlir.dialects import pto as mlir_pto > - The VMI release line keeps the `ptoas` CLI name; `ptoas --version` prints `ptoas vmi A.B.C`. > - The `ptoas` and `ptoas-vmi` release wheels are **mutually exclusive**. They both install the same top-level `ptoas` Python package and `ptoas` console script, so do **not** install them into the same Python environment. Mixing them will overwrite files, and uninstalling one can break the other. > - `ptoas-bin-*.tar.gz` compiler-only tarballs provide CLI/toolchain bits, not a PTODSL-capable Python distribution. +> - Release tags use `ptoas-vX.Y` for the main toolchain and `vmi-vA.B.C` for the `ptoas-vmi` distribution. Before creating a VMI release tag, update the version in `packaging/ptoas-vmi/pyproject.toml` to the matching `A.B.C` through the release PR. > - `--no-build-isolation` keeps pip from baking a temporary pybind11 path into `CMakeCache.txt`, which would break later `ninja` reconfigure runs after the temporary virtual environment is removed. If you previously ran `pip install -e .` without the flag and your build is now broken, fix the existing `CMakeCache.txt` with: diff --git a/lib/Bindings/Python/CMakeLists.txt b/lib/Bindings/Python/CMakeLists.txt index 0c56fef22e..6be7441b47 100644 --- a/lib/Bindings/Python/CMakeLists.txt +++ b/lib/Bindings/Python/CMakeLists.txt @@ -15,16 +15,15 @@ include(AddMLIRPython) add_compile_definitions("MLIR_PYTHON_PACKAGE_PREFIX=ptoas.mlir.") set(MLIR_BINDINGS_PYTHON_NB_DOMAIN "ptoas_mlir") -# Do not carry toolchain-injected paths, such as a Conda compiler RPATH, into -# installed Python modules. Distribution builds keep only MLIR's relative -# package RPATHs; editable builds add their external link paths below. -set(CMAKE_INSTALL_REMOVE_ENVIRONMENT_RPATH ON) - # scikit-build-core editable installs use the external LLVM build directly. -# Keep this developer-only fallback local to the MLIR Python targets; wheel -# builds use package-relative RPATHs and let auditwheel/delocate collect LLVM. +# Wheel builds retain AddMLIRPython's package-relative RPATHs for +# auditwheel/delocate to repair. if(SKBUILD_STATE STREQUAL "editable") - set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON) + set(CMAKE_INSTALL_REMOVE_ENVIRONMENT_RPATH OFF) +else() + # Do not carry toolchain-injected paths, such as a Conda compiler RPATH, + # into distribution modules. + set(CMAKE_INSTALL_REMOVE_ENVIRONMENT_RPATH ON) endif() # This directory materializes C++ sources exported by upstream MLIR. PTOAS @@ -139,3 +138,23 @@ add_mlir_python_modules(PTOAS_Python COMMON_CAPI_LINK_LIBS PTOASPythonCAPI ) + +# AddMLIRPython deliberately resets each generated target's install RPATH to +# its package-relative layout. Append the external LLVM directory afterwards +# for editable installs, where those dependencies are not bundled into the +# Python environment. Discover the native targets by CMake type so additional +# generated MLIR extensions inherit the same contract automatically. +if(SKBUILD_STATE STREQUAL "editable") + get_filename_component(_ptoas_editable_llvm_library_dir + "${LLVM_BUILD_LIBRARY_DIR}" REALPATH) + get_property(_ptoas_python_directory_targets + DIRECTORY PROPERTY BUILDSYSTEM_TARGETS) + foreach(_ptoas_python_target IN LISTS _ptoas_python_directory_targets) + get_target_property(_ptoas_python_target_type + "${_ptoas_python_target}" TYPE) + if(_ptoas_python_target_type MATCHES "^(MODULE|SHARED)_LIBRARY$") + set_property(TARGET "${_ptoas_python_target}" APPEND PROPERTY + INSTALL_RPATH "${_ptoas_editable_llvm_library_dir}") + endif() + endforeach() +endif() diff --git a/packaging/ptoas-vmi/pyproject.toml b/packaging/ptoas-vmi/pyproject.toml index 185a135e66..d655e523ff 100644 --- a/packaging/ptoas-vmi/pyproject.toml +++ b/packaging/ptoas-vmi/pyproject.toml @@ -9,7 +9,9 @@ # The VMI distribution intentionally installs the same `ptoas` import package # and console script as the main distribution. The two wheels are mutually # exclusive; this separate static PEP 621 project supplies the distribution -# name required by package indexes without a custom build backend. +# name and version required by package indexes without a custom build backend. +# scikit-build-core forwards that version to CMake as +# `SKBUILD_PROJECT_VERSION_FULL`. [build-system] requires = ["scikit-build-core>=0.12.2,<2", "pybind11<3", "nanobind>=2.4"] @@ -17,7 +19,7 @@ build-backend = "scikit_build_core.build" [project] name = "ptoas-vmi" -dynamic = ["version"] +version = "0.1.4" description = "PTO Assembler & Optimizer with VMI support" requires-python = ">=3.10" license = "Apache-2.0" @@ -52,8 +54,4 @@ tilelang_dsl = "../../tilelang-dsl/python/tilelang_dsl" [tool.scikit-build.cmake.define] PTOAS_RELEASE_VERSION_OVERRIDE = { env = "PTOAS_RELEASE_VERSION_OVERRIDE", default = "" } - -[tool.scikit-build.metadata.version] -provider = "scikit_build_core.metadata.regex" -input = "../../CMakeLists.txt" -regex = 'set\s*\(\s*PTOAS_VMI_VERSION\s+"(?P[0-9]+\.[0-9]+\.[0-9]+)"\s*\)' +PTOAS_CLI_VERSION_LABEL = "vmi" diff --git a/test/python/test_release_version_scripts.py b/test/python/test_release_version_scripts.py deleted file mode 100644 index 235face241..0000000000 --- a/test/python/test_release_version_scripts.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -import subprocess -import sys -import tempfile -from pathlib import Path -import unittest - - -REPO_ROOT = Path(__file__).resolve().parents[2] - - -class ReleaseVersionScriptTests(unittest.TestCase): - def test_ptoas_version_script_accepts_plain_v_tag(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - cmake_file = temp_root / "CMakeLists.txt" - cmake_file.write_text("project(ptoas VERSION 0.51)\n", encoding="utf-8") - result = subprocess.run( - [ - sys.executable, - str(REPO_ROOT / ".github/scripts/compute_ptoas_version.py"), - "--cmake-file", - str(cmake_file), - "--check-tag", - "v0.51", - ], - check=True, - capture_output=True, - text=True, - ) - self.assertEqual(result.stdout.strip(), "0.51") - - def test_ptoas_version_script_accepts_ptoas_tag_prefix(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - cmake_file = temp_root / "CMakeLists.txt" - cmake_file.write_text("project(ptoas VERSION 0.51)\n", encoding="utf-8") - result = subprocess.run( - [ - sys.executable, - str(REPO_ROOT / ".github/scripts/compute_ptoas_version.py"), - "--cmake-file", - str(cmake_file), - "--check-tag", - "ptoas-v0.51", - ], - check=True, - capture_output=True, - text=True, - ) - self.assertEqual(result.stdout.strip(), "0.51") - - def test_vmi_version_script_accepts_vmi_tag_prefix(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - cmake_file = temp_root / "CMakeLists.txt" - cmake_file.write_text( - 'project(ptoas VERSION 0.51)\nset(PTOAS_VMI_VERSION "0.1.0")\n', - encoding="utf-8", - ) - result = subprocess.run( - [ - sys.executable, - str(REPO_ROOT / ".github/scripts/compute_vmi_version.py"), - "--cmake-file", - str(cmake_file), - "--check-tag", - "vmi-v0.1.0", - ], - check=True, - capture_output=True, - text=True, - ) - self.assertEqual(result.stdout.strip(), "0.1.0") - - def test_vmi_version_bump_updates_only_vmi_version(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - cmake_file = temp_root / "CMakeLists.txt" - cmake_file.write_text( - 'project(ptoas VERSION 0.51)\nset(PTOAS_VMI_VERSION "0.1.9")\n', - encoding="utf-8", - ) - result = subprocess.run( - [ - sys.executable, - str(REPO_ROOT / ".github/scripts/update_vmi_version.py"), - "--cmake-file", - str(cmake_file), - "--version", - "vmi-v0.1.9", - "--next", - ], - check=True, - capture_output=True, - text=True, - ) - self.assertEqual(result.stdout.strip(), "0.1.10") - self.assertEqual( - cmake_file.read_text(encoding="utf-8"), - 'project(ptoas VERSION 0.51)\nset(PTOAS_VMI_VERSION "0.1.10")\n', - ) - - def test_vmi_version_bump_rejects_ptoas_tag(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - cmake_file = temp_root / "CMakeLists.txt" - cmake_file.write_text( - 'project(ptoas VERSION 0.51)\nset(PTOAS_VMI_VERSION "0.1.0")\n', - encoding="utf-8", - ) - result = subprocess.run( - [ - sys.executable, - str(REPO_ROOT / ".github/scripts/update_vmi_version.py"), - "--cmake-file", - str(cmake_file), - "--version", - "v0.51", - "--next", - ], - capture_output=True, - text=True, - ) - self.assertNotEqual(result.returncode, 0) - - def test_ptoas_version_bump_rejects_vmi_tag(self): - with tempfile.TemporaryDirectory() as temp_dir: - temp_root = Path(temp_dir) - cmake_file = temp_root / "CMakeLists.txt" - cmake_file.write_text("project(ptoas VERSION 0.51)\n", encoding="utf-8") - result = subprocess.run( - [ - sys.executable, - str(REPO_ROOT / ".github/scripts/update_ptoas_base_version.py"), - "--cmake-file", - str(cmake_file), - "--version", - "vmi-v0.1.0", - ], - capture_output=True, - text=True, - ) - self.assertNotEqual(result.returncode, 0) - - -if __name__ == "__main__": - unittest.main() From a86ce4045e4bd2f37b7ca3b60994f4ad77882ff5 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Mon, 27 Jul 2026 23:33:59 +0800 Subject: [PATCH 18/32] build: fix editable core LLVM rpath --- CMakeLists.txt | 15 +++++++++++++++ lib/Bindings/Python/CMakeLists.txt | 19 +++++-------------- tools/ptoas/CMakeLists.txt | 12 +++--------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b3d8e3657a..a8b0d3b3ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,21 @@ set(PTOAS_RELEASE_VERSION_OVERRIDE "" # same CLI without making CMake a second source of package-version truth. set(PTOAS_CLI_VERSION_LABEL "" CACHE STRING "Optional label prepended to the packaged ptoas version") + +# Editable installs intentionally link against the developer's external LLVM +# build instead of bundling it. Apply that contract explicitly to every native +# Python target; generated and project-owned extensions live in different +# CMake directories and must use the same install RPATH policy. +function(ptoas_append_editable_llvm_rpath target_name) + if(NOT SKBUILD_STATE STREQUAL "editable") + return() + endif() + get_filename_component(_ptoas_editable_llvm_library_dir + "${LLVM_BUILD_LIBRARY_DIR}" REALPATH) + set_property(TARGET "${target_name}" APPEND PROPERTY + INSTALL_RPATH "${_ptoas_editable_llvm_library_dir}") +endfunction() + set(PTOAS_EXACT_GIT_TAG_VERSION "") if(EXISTS "${PROJECT_SOURCE_DIR}/.git") find_package(Git QUIET) diff --git a/lib/Bindings/Python/CMakeLists.txt b/lib/Bindings/Python/CMakeLists.txt index 6be7441b47..04ed2a183b 100644 --- a/lib/Bindings/Python/CMakeLists.txt +++ b/lib/Bindings/Python/CMakeLists.txt @@ -15,16 +15,10 @@ include(AddMLIRPython) add_compile_definitions("MLIR_PYTHON_PACKAGE_PREFIX=ptoas.mlir.") set(MLIR_BINDINGS_PYTHON_NB_DOMAIN "ptoas_mlir") -# scikit-build-core editable installs use the external LLVM build directly. -# Wheel builds retain AddMLIRPython's package-relative RPATHs for -# auditwheel/delocate to repair. -if(SKBUILD_STATE STREQUAL "editable") - set(CMAKE_INSTALL_REMOVE_ENVIRONMENT_RPATH OFF) -else() - # Do not carry toolchain-injected paths, such as a Conda compiler RPATH, - # into distribution modules. - set(CMAKE_INSTALL_REMOVE_ENVIRONMENT_RPATH ON) -endif() +# Do not carry toolchain-injected paths, such as a Conda compiler RPATH, into +# installed Python modules. Wheel builds retain AddMLIRPython's package-relative +# RPATHs for repair; editable builds append only the external LLVM directory. +set(CMAKE_INSTALL_REMOVE_ENVIRONMENT_RPATH ON) # This directory materializes C++ sources exported by upstream MLIR. PTOAS # keeps -Werror enabled globally, but upstream sources are not part of the @@ -145,16 +139,13 @@ add_mlir_python_modules(PTOAS_Python # Python environment. Discover the native targets by CMake type so additional # generated MLIR extensions inherit the same contract automatically. if(SKBUILD_STATE STREQUAL "editable") - get_filename_component(_ptoas_editable_llvm_library_dir - "${LLVM_BUILD_LIBRARY_DIR}" REALPATH) get_property(_ptoas_python_directory_targets DIRECTORY PROPERTY BUILDSYSTEM_TARGETS) foreach(_ptoas_python_target IN LISTS _ptoas_python_directory_targets) get_target_property(_ptoas_python_target_type "${_ptoas_python_target}" TYPE) if(_ptoas_python_target_type MATCHES "^(MODULE|SHARED)_LIBRARY$") - set_property(TARGET "${_ptoas_python_target}" APPEND PROPERTY - INSTALL_RPATH "${_ptoas_editable_llvm_library_dir}") + ptoas_append_editable_llvm_rpath("${_ptoas_python_target}") endif() endforeach() endif() diff --git a/tools/ptoas/CMakeLists.txt b/tools/ptoas/CMakeLists.txt index 18652a8ce3..7a8575f854 100644 --- a/tools/ptoas/CMakeLists.txt +++ b/tools/ptoas/CMakeLists.txt @@ -7,17 +7,10 @@ # See LICENSE in the root of the software repository for the full text of the License. # Do not carry toolchain-injected paths, such as a Conda compiler RPATH, into -# installed Python modules. Distribution builds keep only MLIR's relative -# package RPATHs; editable builds add their external link paths below. +# installed Python modules. Distribution builds keep only package-relative +# RPATHs; editable builds add the external LLVM directory explicitly below. set(CMAKE_INSTALL_REMOVE_ENVIRONMENT_RPATH ON) -# scikit-build-core editable installs use the external LLVM build directly. -# Keep this developer-only fallback local to the PTOAS Python extension; wheel -# builds use package-relative RPATHs and let auditwheel/delocate collect LLVM. -if(SKBUILD_STATE STREQUAL "editable") - set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON) -endif() - set(PTOAS_RUNTIME_SOURCES ptoas.cpp driver.cpp @@ -145,6 +138,7 @@ elseif(UNIX) set_property(TARGET PTOASPythonCore PROPERTY INSTALL_RPATH "$ORIGIN;$ORIGIN/mlir/_mlir_libs") endif() +ptoas_append_editable_llvm_rpath(PTOASPythonCore) declare_mlir_python_sources(PTOASRuntimePythonSources ROOT_DIR "${CMAKE_SOURCE_DIR}/ptodsl/ptoas" From 5241b74feda543eed422717d51991f712ffd3322 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Tue, 28 Jul 2026 00:36:56 +0800 Subject: [PATCH 19/32] ci: validate macOS common CAPI wheel payload --- docker/validate_wheel_payload.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docker/validate_wheel_payload.py b/docker/validate_wheel_payload.py index a96f677fa0..0b67082e39 100644 --- a/docker/validate_wheel_payload.py +++ b/docker/validate_wheel_payload.py @@ -23,7 +23,6 @@ "ptoas/_runtime/share/ptoas/TileOps/__init__.py", "ptoas/mlir/ir.py", "ptoas/mlir/dialects/pto.py", - "ptoas/mlir/_mlir_libs/libPTOASPythonCAPI.so", } FORBIDDEN_FILES = { "ptoas/_runtime/bin/ptoas", @@ -35,6 +34,10 @@ for suffix in importlib.machinery.EXTENSION_SUFFIXES } MLIR_NATIVE_MODULE_PREFIX = "ptoas/mlir/_mlir_libs/" +COMMON_CAPI_LIBRARY_PATHS = { + f"{MLIR_NATIVE_MODULE_PREFIX}libPTOASPythonCAPI{suffix}" + for suffix in (".so", ".dylib") +} MLIR_NATIVE_MODULE_PATHS = { f"{MLIR_NATIVE_MODULE_PREFIX}_mlir{suffix}" for suffix in importlib.machinery.EXTENSION_SUFFIXES @@ -112,6 +115,13 @@ def validate_wheel_payload(wheel: Path) -> None: f"found {mlir_native_modules}" ) + common_capi_libraries = sorted(COMMON_CAPI_LIBRARY_PATHS & names) + if len(common_capi_libraries) != 1: + raise SystemExit( + "wheel must contain exactly one PTOAS MLIR common CAPI library, " + f"found {common_capi_libraries}" + ) + site_initializers = sorted(SITE_INITIALIZER_PATHS & names) if len(site_initializers) != 1: raise SystemExit( From 16d6fef8c5bceec242e02776459c5f16bf7c28c0 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Tue, 28 Jul 2026 09:52:17 +0800 Subject: [PATCH 20/32] ci: probe namespaced MLIR Python bindings --- .github/workflows/ci_sim.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_sim.yml b/.github/workflows/ci_sim.yml index b87084d438..5c6d9e2e38 100644 --- a/.github/workflows/ci_sim.yml +++ b/.github/workflows/ci_sim.yml @@ -804,12 +804,12 @@ jobs: probe_ptodsl_python() { "${PTO_DSL_ST_PYTHON_BIN}" - <<'PY' import sys - import mlir.ir # noqa: F401 + import ptoas.mlir.ir # noqa: F401 from ptodsl import pto # noqa: F401 - from mlir.dialects import pto as _pto # noqa: F401 + from ptoas.mlir.dialects import pto as _pto # noqa: F401 print(sys.executable) - print("ptodsl/mlir import ok") + print("ptodsl/ptoas.mlir import ok") PY } From 2693a693f684f1014cebff6ca599b0be5f0df573 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Tue, 28 Jul 2026 11:36:54 +0800 Subject: [PATCH 21/32] build: fix macOS compiler archive relocation --- cmake/RelocatePTOASCompilerArchive.cmake.in | 41 ++++++++++++++------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/cmake/RelocatePTOASCompilerArchive.cmake.in b/cmake/RelocatePTOASCompilerArchive.cmake.in index 2acb48de7f..843207adf1 100644 --- a/cmake/RelocatePTOASCompilerArchive.cmake.in +++ b/cmake/RelocatePTOASCompilerArchive.cmake.in @@ -60,6 +60,33 @@ if(NOT _ptoas_generated_native_files) endif() file(MAKE_DIRECTORY "${_ptoas_external_lib_root}") + +# CMake's DIRECTORIES argument to GET_RUNTIME_DEPENDENCIES has no effect on +# Apple platforms, so it cannot resolve LLVM's @rpath dependencies from an +# external build tree. Let delocate perform both dependency discovery and +# Mach-O install-name relocation, just as it does for the macOS wheel. +if(APPLE) + find_program(_ptoas_delocate_path NAMES delocate-path REQUIRED) + set(_ptoas_dyld_library_path "@LLVM_BUILD_LIBRARY_DIR@") + if(DEFINED ENV{DYLD_LIBRARY_PATH} + AND NOT "$ENV{DYLD_LIBRARY_PATH}" STREQUAL "") + string(APPEND _ptoas_dyld_library_path ":$ENV{DYLD_LIBRARY_PATH}") + endif() + execute_process( + COMMAND "${CMAKE_COMMAND}" -E env + "DYLD_LIBRARY_PATH=${_ptoas_dyld_library_path}" + "${_ptoas_delocate_path}" + --dylibs-only + --lib-path "${_ptoas_external_lib_root}" + "${_ptoas_package_root}" + RESULT_VARIABLE _ptoas_delocate_result + COMMAND_ECHO STDOUT) + if(NOT _ptoas_delocate_result EQUAL 0) + message(FATAL_ERROR "delocate-path failed for PTOAS compiler archive") + endif() + return() +endif() + set(_ptoas_dependency_search_dirs "${_ptoas_mlir_lib_root}" "${_ptoas_archive_root}/lib" @@ -130,19 +157,7 @@ foreach(_ptoas_dependency IN LISTS _ptoas_resolved_dependencies) FOLLOW_SYMLINK_CHAIN) endforeach() -if(APPLE) - find_program(_ptoas_delocate_path NAMES delocate-path REQUIRED) - execute_process( - COMMAND "${_ptoas_delocate_path}" - --dylibs-only - --lib-path "${_ptoas_external_lib_root}" - "${_ptoas_package_root}" - RESULT_VARIABLE _ptoas_delocate_result - COMMAND_ECHO STDOUT) - if(NOT _ptoas_delocate_result EQUAL 0) - message(FATAL_ERROR "delocate-path failed for PTOAS compiler archive") - endif() -elseif(UNIX) +if(UNIX) find_program(_ptoas_patchelf NAMES patchelf REQUIRED) function(_ptoas_origin_relative_rpath _ptoas_output _ptoas_from _ptoas_to) From 9aeb3c71bc992abcddcde3c2581bbabc9102f19b Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Wed, 29 Jul 2026 00:08:18 +0800 Subject: [PATCH 22/32] fix: address Python packaging review findings --- .codex/skills/build-ptoas-wsl/SKILL.md | 75 ++++--- .github/workflows/build_wheel.yml | 11 +- .github/workflows/build_wheel_mac.yml | 11 +- README.md | 5 +- README_en.md | 2 +- docs/designs/ptoas-python-launcher-layout.md | 21 +- packaging/ptoas-vmi/prepare_source.py | 194 ++++++++++++++++++ packaging/ptoas-vmi/pyproject.toml | 57 ----- packaging/ptoas-vmi/pyproject.toml.patch | 29 +++ .../python/tilelang_dsl/env_config.py | 5 +- .../python/tilelang_dsl/lowering_backend.py | 13 +- .../tests/test_lowering_backend_conversion.py | 23 +++ 12 files changed, 327 insertions(+), 119 deletions(-) create mode 100644 packaging/ptoas-vmi/prepare_source.py delete mode 100644 packaging/ptoas-vmi/pyproject.toml create mode 100644 packaging/ptoas-vmi/pyproject.toml.patch create mode 100644 tilelang-dsl/tests/test_lowering_backend_conversion.py diff --git a/.codex/skills/build-ptoas-wsl/SKILL.md b/.codex/skills/build-ptoas-wsl/SKILL.md index a35773f03c..e99749963f 100644 --- a/.codex/skills/build-ptoas-wsl/SKILL.md +++ b/.codex/skills/build-ptoas-wsl/SKILL.md @@ -1,13 +1,13 @@ --- name: build-ptoas-wsl -description: Build PTOAS from source inside WSL using the repository README workflow. Use when Codex is asked to build, configure, install, test, or troubleshoot ptoas/PTOAS in WSL or Ubuntu, including LLVM/MLIR llvmorg-19.1.7 setup, CMake/Ninja out-of-tree builds, pybind11 Python bindings, runtime environment variables, CLI smoke tests, or Python dialect import validation. +description: Build PTOAS from source inside WSL using the repository README workflow. Use when Codex is asked to build, configure, install, test, or troubleshoot ptoas/PTOAS in WSL or Ubuntu, including the VPTO LLVM 21 setup, editable scikit-build-core installs, CMake/Ninja incremental builds, namespaced MLIR Python bindings, CLI smoke tests, or Python dialect import validation. --- # Build PTOAS in WSL ## Overview -Use this skill to build PTOAS in a Linux environment under WSL. Keep the build aligned with the README: LLVM/MLIR must be `llvmorg-19.1.7`, LLVM must be built with shared libraries and MLIR Python bindings, and PTOAS is built out-of-tree against that LLVM build. +Use this skill to build PTOAS in a Linux environment under WSL. Keep the build aligned with the README: use the VPTO-enabled LLVM branch `feature-vpto-llvm21`, build shared LLVM/MLIR libraries with Python bindings, and install PTOAS through the repository's standard editable PEP 517 workflow. Prefer running Linux build commands through WSL, not Windows PowerShell/CMake. If the user is already in a Windows checkout, either convert the path to `/mnt/c/...` for a quick build or clone/copy into the WSL ext4 filesystem for better performance. @@ -31,10 +31,11 @@ wsl.exe -- bash -lc 'uname -a; cat /etc/os-release | head' ```bash sudo apt-get update sudo apt-get install -y git cmake ninja-build build-essential python3 python3-pip python3-dev -python3 -m pip install --user pybind11==2.12.0 numpy +python3 -m pip install --user \ + 'scikit-build-core>=0.12.2,<2' 'pybind11<3' 'nanobind>=2.4' numpy ``` -Pin `pybind11==2.12.0`; LLVM/MLIR Python bindings are not compatible with pybind11 3.x in this workflow. +Keep `pybind11<3`; LLVM/MLIR Python bindings are not compatible with pybind11 3.x in this workflow. ## Environment Variables @@ -45,7 +46,6 @@ export WORKSPACE_DIR=$HOME/llvm-workspace export LLVM_SOURCE_DIR=$WORKSPACE_DIR/llvm-project export LLVM_BUILD_DIR=$LLVM_SOURCE_DIR/build-shared export PTO_SOURCE_DIR=$WORKSPACE_DIR/PTOAS -export PTO_INSTALL_DIR=$PTO_SOURCE_DIR/install mkdir -p "$WORKSPACE_DIR" ``` @@ -65,18 +65,22 @@ Build LLVM only once per WSL workspace unless the user requests a clean rebuild. ```bash cd "$WORKSPACE_DIR" if [ ! -d "$LLVM_SOURCE_DIR/.git" ]; then - git clone https://github.com/llvm/llvm-project.git "$LLVM_SOURCE_DIR" + git clone https://github.com/vpto-dev/llvm-project.git "$LLVM_SOURCE_DIR" fi cd "$LLVM_SOURCE_DIR" git fetch --tags -git checkout llvmorg-19.1.7 +git checkout feature-vpto-llvm21 cmake -G Ninja -S llvm -B "$LLVM_BUILD_DIR" \ -DLLVM_ENABLE_PROJECTS="mlir;clang" \ -DBUILD_SHARED_LIBS=ON \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ + -DLLVM_ENABLE_ASSERTIONS=ON \ -DPython3_EXECUTABLE="$(which python3)" \ + -DPython_EXECUTABLE="$(which python3)" \ + -Dpybind11_DIR="$(python3 -m pybind11 --cmakedir)" \ + -Dnanobind_DIR="$(python3 -m nanobind --cmake_dir)" \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_TARGETS_TO_BUILD="host" @@ -102,22 +106,13 @@ if [ ! -d "$PTO_SOURCE_DIR/.git" ]; then fi cd "$PTO_SOURCE_DIR" -export PYBIND11_CMAKE_DIR="$(python3 -m pybind11 --cmakedir)" +PYTHON_BIN=python3 \ +LLVM_BUILD_DIR="$LLVM_BUILD_DIR" \ +PTO_BUILD_DIR="$PTO_SOURCE_DIR/build" \ + ./quick_install.sh -cmake -G Ninja \ - -S . \ - -B build \ - -DLLVM_DIR="$LLVM_BUILD_DIR/lib/cmake/llvm" \ - -DMLIR_DIR="$LLVM_BUILD_DIR/lib/cmake/mlir" \ - -DPython3_EXECUTABLE="$(which python3)" \ - -DPython3_FIND_STRATEGY=LOCATION \ - -Dpybind11_DIR="$PYBIND11_CMAKE_DIR" \ - -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ - -DMLIR_PYTHON_PACKAGE_DIR="$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core" \ - -DCMAKE_INSTALL_PREFIX="$PTO_INSTALL_DIR" - -ninja -C build -ninja -C build install +# Reuse the persistent CMake tree for incremental builds and checks. +ninja -C "$PTO_SOURCE_DIR/build" check-pto ``` Expected outputs: @@ -125,19 +120,25 @@ Expected outputs: ```bash test -x "$PTO_SOURCE_DIR/build/tools/ptoas/ptoas" test -x "$PTO_SOURCE_DIR/build/tools/ptobc/ptobc" -find "$PTO_SOURCE_DIR/build/python" -name '_pto.cpython-*.so' -print -test -f "$PTO_INSTALL_DIR/mlir/dialects/pto.py" +test -n "$(find "$PTO_SOURCE_DIR/build/python/ptoas" \ + -name '_core.cpython-*.so' -print -quit)" +test -f "$PTO_SOURCE_DIR/build/python/ptoas/mlir/dialects/pto.py" ``` ## Runtime Environment -After installation, set the runtime paths before using `ptoas`, `ptobc`, or Python imports. +`quick_install.sh` performs an editable install into the selected Python environment. +Use the installed `ptoas` console script and import `ptoas.mlir` directly. Do not add +LLVM's top-level `mlir_core` package or the deleted top-level `pto` package to +`PYTHONPATH`. + +Only when invoking artifacts directly from the CMake build tree, expose that single +tree explicitly: ```bash -export MLIR_PYTHON_ROOT=$LLVM_BUILD_DIR/tools/mlir/python_packages/mlir_core -export PTO_PYTHON_ROOT=$PTO_INSTALL_DIR -export PYTHONPATH=$MLIR_PYTHON_ROOT:$PTO_PYTHON_ROOT:$PYTHONPATH -export LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:$LD_LIBRARY_PATH +export PTOAS_BUILD_PYTHON_ROOT=$PTO_SOURCE_DIR/build/python +export PYTHONPATH=$PTOAS_BUILD_PYTHON_ROOT:$PYTHONPATH +export LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_SOURCE_DIR/build/lib:$LD_LIBRARY_PATH export PATH=$PTO_SOURCE_DIR/build/tools/ptoas:$PTO_SOURCE_DIR/build/tools/ptobc:$PATH ``` @@ -172,8 +173,16 @@ PY ## Troubleshooting -- `def_property family does not currently support keep_alive`: reinstall `pybind11==2.12.0`, clear the affected CMake cache if needed, and reconfigure. -- CMake cannot find LLVM or MLIR: confirm `LLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm` and `MLIR_DIR=$LLVM_BUILD_DIR/lib/cmake/mlir` exist and were produced by the `llvmorg-19.1.7` build. -- Python cannot import `ptoas.mlir.dialects.pto`: make sure the PTOAS build/install Python root is on `PYTHONPATH`, and confirm `_core.cpython-*.so` and `ptoas/mlir/_mlir_libs/_mlir*.so` exist in the same package tree. -- Runtime linker errors for LLVM or PTO libraries: re-export `LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:$LD_LIBRARY_PATH`. +- `def_property family does not currently support keep_alive`: reinstall a supported + `pybind11<3`, clear the affected CMake cache if needed, and reconfigure. +- CMake cannot find LLVM or MLIR: confirm + `LLVM_DIR=$LLVM_BUILD_DIR/lib/cmake/llvm` and + `MLIR_DIR=$LLVM_BUILD_DIR/lib/cmake/mlir` exist in the + `feature-vpto-llvm21` build. +- Python cannot import `ptoas.mlir.dialects.pto`: confirm the editable install is + active, then check `build/python/ptoas/_core.cpython-*.so` and + `build/python/ptoas/mlir/_mlir_libs/_mlir*.so` in the same build tree. For + direct build-tree imports, prepend only `build/python` to `PYTHONPATH`. +- Runtime linker errors for LLVM or PTO libraries: re-export + `LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_SOURCE_DIR/build/lib:$LD_LIBRARY_PATH`. - Builds are very slow under `/mnt/c`: move the workspace into the WSL Linux filesystem, for example `$HOME/llvm-workspace`. diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 1ffafa50cc..1bbec889a4 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -88,12 +88,10 @@ jobs: - name: Resolve PTOAS distribution version run: | - VMI_PROJECT_METADATA="$(cd packaging/ptoas-vmi && python -m scikit_build_core.build project-table)" - VMI_PROJECT_VERSION="$(printf '%s\n' "${VMI_PROJECT_METADATA}" | python -c 'import json, sys; print(json.load(sys.stdin)["version"])')" if [ "${GITHUB_EVENT_NAME}" = "release" ]; then case "${GITHUB_REF_NAME}" in vmi-v*) - EXPECTED_RELEASE_VERSION="${VMI_PROJECT_VERSION}" + EXPECTED_RELEASE_VERSION="$(python packaging/ptoas-vmi/prepare_source.py --print-version)" PTOAS_VERSION="${GITHUB_REF_NAME#vmi-v}" PTOAS_CLI_VERSION="vmi ${PTOAS_VERSION}" PTOAS_PYTHON_PACKAGE_NAME="ptoas-vmi" @@ -120,7 +118,7 @@ jobs: exit 1 fi elif [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ] && [ "${{ inputs.release_kind }}" = "vmi" ]; then - PTOAS_VERSION="${VMI_PROJECT_VERSION}" + PTOAS_VERSION="$(python packaging/ptoas-vmi/prepare_source.py --print-version)" PTOAS_CLI_VERSION="vmi ${PTOAS_VERSION}" PTOAS_PYTHON_PACKAGE_NAME="ptoas-vmi" elif [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then @@ -142,7 +140,6 @@ jobs: - name: Validate PEP 517 project metadata run: | python -m scikit_build_core.build project-table - (cd packaging/ptoas-vmi && python -m scikit_build_core.build project-table) - name: Set build directories run: | @@ -220,7 +217,9 @@ jobs: CMAKE_BUILD_PARALLEL_LEVEL: "2" run: | if [ "${PTOAS_PYTHON_PACKAGE_NAME}" = "ptoas-vmi" ]; then - PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}/packaging/ptoas-vmi" + PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}/.work/ptoas-vmi-source" + python packaging/ptoas-vmi/prepare_source.py \ + --output-dir "${PTOAS_WHEEL_SOURCE_DIR}" else PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}" fi diff --git a/.github/workflows/build_wheel_mac.yml b/.github/workflows/build_wheel_mac.yml index de79399e3f..3453822943 100644 --- a/.github/workflows/build_wheel_mac.yml +++ b/.github/workflows/build_wheel_mac.yml @@ -87,12 +87,10 @@ jobs: - name: Resolve PTOAS distribution version run: | - VMI_PROJECT_METADATA="$(cd packaging/ptoas-vmi && python -m scikit_build_core.build project-table)" - VMI_PROJECT_VERSION="$(printf '%s\n' "${VMI_PROJECT_METADATA}" | python -c 'import json, sys; print(json.load(sys.stdin)["version"])')" if [ "${GITHUB_EVENT_NAME}" = "release" ]; then case "${GITHUB_REF_NAME}" in vmi-v*) - EXPECTED_RELEASE_VERSION="${VMI_PROJECT_VERSION}" + EXPECTED_RELEASE_VERSION="$(python packaging/ptoas-vmi/prepare_source.py --print-version)" PTOAS_VERSION="${GITHUB_REF_NAME#vmi-v}" PTOAS_CLI_VERSION="vmi ${PTOAS_VERSION}" PTOAS_PYTHON_PACKAGE_NAME="ptoas-vmi" @@ -119,7 +117,7 @@ jobs: exit 1 fi elif [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ] && [ "${{ inputs.release_kind }}" = "vmi" ]; then - PTOAS_VERSION="${VMI_PROJECT_VERSION}" + PTOAS_VERSION="$(python packaging/ptoas-vmi/prepare_source.py --print-version)" PTOAS_CLI_VERSION="vmi ${PTOAS_VERSION}" PTOAS_PYTHON_PACKAGE_NAME="ptoas-vmi" elif [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then @@ -141,7 +139,6 @@ jobs: - name: Validate PEP 517 project metadata run: | python -m scikit_build_core.build project-table - (cd packaging/ptoas-vmi && python -m scikit_build_core.build project-table) - name: Set build directories run: | @@ -217,7 +214,9 @@ jobs: - name: Build PTOAS run: | if [ "${PTOAS_PYTHON_PACKAGE_NAME}" = "ptoas-vmi" ]; then - PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}/packaging/ptoas-vmi" + PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}/.work/ptoas-vmi-source" + python packaging/ptoas-vmi/prepare_source.py \ + --output-dir "${PTOAS_WHEEL_SOURCE_DIR}" else PTOAS_WHEEL_SOURCE_DIR="${PTO_SOURCE_DIR}" fi diff --git a/README.md b/README.md index e51767fea0..f3d2cd47d2 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,10 @@ from ptoas.mlir.dialects import pto as mlir_pto > `import ptodsl` 可用。 > - release tag 约定：`ptoas-vX.Y` 发布主工具链，`vmi-vA.B.C` 发布 > `ptoas-vmi` distribution。创建 VMI release tag 前，应通过发布 PR 将 -> `packaging/ptoas-vmi/pyproject.toml` 中的版本更新为相同的 `A.B.C`。 +> `packaging/ptoas-vmi/pyproject.toml.patch` 中的版本更新为相同的 `A.B.C`。 +> VMI 发布流程会从当前 Git revision 导出完整源码快照，只在 staging tree +> 中应用该 metadata patch，再直接从 staging tree 构建 wheel；不会修改工作区 +> 根目录的 `pyproject.toml`，也不会在普通门禁或 release 中生成、发布 sdist。 --- diff --git a/README_en.md b/README_en.md index efde0e8c48..73fc6a794a 100644 --- a/README_en.md +++ b/README_en.md @@ -161,7 +161,7 @@ from ptoas.mlir.dialects import pto as mlir_pto > - The VMI release line keeps the `ptoas` CLI name; `ptoas --version` prints `ptoas vmi A.B.C`. > - The `ptoas` and `ptoas-vmi` release wheels are **mutually exclusive**. They both install the same top-level `ptoas` Python package and `ptoas` console script, so do **not** install them into the same Python environment. Mixing them will overwrite files, and uninstalling one can break the other. > - `ptoas-bin-*.tar.gz` compiler-only tarballs provide CLI/toolchain bits, not a PTODSL-capable Python distribution. -> - Release tags use `ptoas-vX.Y` for the main toolchain and `vmi-vA.B.C` for the `ptoas-vmi` distribution. Before creating a VMI release tag, update the version in `packaging/ptoas-vmi/pyproject.toml` to the matching `A.B.C` through the release PR. +> - Release tags use `ptoas-vX.Y` for the main toolchain and `vmi-vA.B.C` for the `ptoas-vmi` distribution. Before creating a VMI release tag, update the version in `packaging/ptoas-vmi/pyproject.toml.patch` to the matching `A.B.C` through the release PR. A VMI build exports the current Git revision, applies that metadata patch only in a staging tree, and builds the wheel directly from that tree. It neither modifies the checkout's top-level `pyproject.toml` nor generates or publishes an sdist in ordinary gates or releases. > - `--no-build-isolation` keeps pip from baking a temporary pybind11 path into `CMakeCache.txt`, which would break later `ninja` reconfigure runs after the temporary virtual environment is removed. If you previously ran `pip install -e .` without the flag and your build is now broken, fix the existing `CMakeCache.txt` with: diff --git a/docs/designs/ptoas-python-launcher-layout.md b/docs/designs/ptoas-python-launcher-layout.md index 9d197029cd..1a9d3d2d29 100644 --- a/docs/designs/ptoas-python-launcher-layout.md +++ b/docs/designs/ptoas-python-launcher-layout.md @@ -63,11 +63,22 @@ backend. Project metadata and the console entry point live in `pyproject.toml`; wheel tags, metadata, RECORD generation, CMake configure/build/install, and editable redirects are owned by the backend rather than repository scripts. -The main `pyproject.toml` has the static distribution name `ptoas`. -`packaging/ptoas-vmi/pyproject.toml` is a second static PEP 621 project for the -mutually exclusive `ptoas-vmi` distribution. Both projects build the same CMake -source and install the same `ptoas` import package. This keeps project names -standards-compliant because PEP 621 does not permit a dynamic `project.name`. +The top-level `pyproject.toml` is the single canonical project configuration. +For the mutually exclusive `ptoas-vmi` distribution, +`packaging/ptoas-vmi/prepare_source.py` is the single staging entry point. It +exports every tracked file from one Git revision and applies +`pyproject.toml.patch`; the wheel is built directly from that tree. The patch +changes only the distribution name, static VMI version, description, CLI label, +and sdist inclusion mode; all CMake and Python package paths remain rooted in +the staged source tree. The checkout's top-level metadata is never modified. + +Git owns source selection instead of a hand-maintained directory list. +scikit-build-core then owns PEP 517 metadata and wheel assembly. If an sdist is +requested independently, manual inclusion makes the backend include the complete +tracked snapshot without applying `.gitignore` a second time. +Submodule contents and untracked/generated working-tree files are intentionally +outside this source-distribution contract. Ordinary gates and releases do not +generate or publish a VMI sdist. CMake's `PTOAS_Python` install component contains only the generated/native wheel payload. Python source packages are declared through diff --git a/packaging/ptoas-vmi/prepare_source.py b/packaging/ptoas-vmi/prepare_source.py new file mode 100644 index 0000000000..a4a5752bc0 --- /dev/null +++ b/packaging/ptoas-vmi/prepare_source.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +"""Prepare the complete source tree used to build the ``ptoas-vmi`` wheel.""" + +from __future__ import annotations + +import argparse +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +from pathlib import Path + + +SCRIPT_PATH = Path(__file__).resolve() +REPOSITORY_ROOT = SCRIPT_PATH.parents[2] +METADATA_PATCH = SCRIPT_PATH.with_name("pyproject.toml.patch") +DEFAULT_OUTPUT_DIR = REPOSITORY_ROOT / ".work" / "ptoas-vmi-source" +VERSION_PATTERN = re.compile(r'^\+version = "([0-9]+\.[0-9]+\.[0-9]+)"$', re.M) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output-dir", + type=Path, + default=DEFAULT_OUTPUT_DIR, + help="Staging directory to replace (default: .work/ptoas-vmi-source).", + ) + parser.add_argument( + "--revision", + default="HEAD", + help="Git revision to archive (default: HEAD).", + ) + parser.add_argument( + "--print-version", + action="store_true", + help="Print the configured VMI version without preparing a source tree.", + ) + return parser.parse_args() + + +def read_vmi_version() -> str: + versions = VERSION_PATTERN.findall(METADATA_PATCH.read_text(encoding="utf-8")) + if len(versions) != 1: + raise ValueError("VMI metadata patch does not set one static X.Y.Z version") + return versions[0] + + +def _check_output_dir(output_dir: Path) -> None: + try: + relative = output_dir.relative_to(REPOSITORY_ROOT) + except ValueError: + if output_dir.exists() and ( + not output_dir.is_dir() or any(output_dir.iterdir()) + ): + raise ValueError( + f"external output directory must be absent or empty: {output_dir}" + ) + return + if len(relative.parts) < 2 or relative.parts[0] != ".work": + raise ValueError( + f"repository-local output directory must be below .work: {output_dir}" + ) + + +def _remove_path(path: Path) -> None: + if path.is_symlink() or path.is_file(): + path.unlink() + elif path.exists(): + shutil.rmtree(path) + + +def prepare_source(output_dir: Path, revision: str) -> dict[str, str]: + output_dir = output_dir.expanduser().resolve() + _check_output_dir(output_dir) + output_dir.parent.mkdir(parents=True, exist_ok=True) + temporary_dir = Path( + tempfile.mkdtemp(prefix=f".{output_dir.name}-", dir=output_dir.parent) + ) + archive_path: Path | None = None + try: + resolved_revision = subprocess.run( + [ + "git", + "-C", + str(REPOSITORY_ROOT), + "rev-parse", + "--verify", + f"{revision}^{{commit}}", + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + with tempfile.NamedTemporaryFile( + prefix="ptoas-vmi-source-", + suffix=".tar", + dir=output_dir.parent, + delete=False, + ) as archive_file: + archive_path = Path(archive_file.name) + + subprocess.run( + [ + "git", + "-C", + str(REPOSITORY_ROOT), + "archive", + "--format=tar", + f"--output={archive_path}", + resolved_revision, + ], + check=True, + ) + with tarfile.open(archive_path, mode="r:") as archive: + for member in archive.getmembers(): + member_path = Path(member.name) + if member_path.is_absolute() or ".." in member_path.parts: + raise ValueError( + f"git archive contains an unsafe path: {member.name}" + ) + if member.issym() or member.islnk(): + raise ValueError( + f"git archive contains an unsupported link: {member.name}" + ) + archive.extractall(temporary_dir) + + apply_environment = os.environ.copy() + apply_environment.update( + { + "GIT_DIR": str(temporary_dir / ".git-not-present"), + "GIT_WORK_TREE": str(temporary_dir), + } + ) + subprocess.run( + ["git", "apply", str(METADATA_PATCH)], + cwd=temporary_dir, + env=apply_environment, + check=True, + ) + + version = read_vmi_version() + staged_text = (temporary_dir / "pyproject.toml").read_text(encoding="utf-8") + required_fragments = ( + 'name = "ptoas-vmi"', + f'version = "{version}"', + 'PTOAS_CLI_VERSION_LABEL = "vmi"', + 'sdist.inclusion-mode = "manual"', + ) + missing = [item for item in required_fragments if item not in staged_text] + if missing: + raise ValueError(f"VMI metadata patch is incomplete: {missing}") + if "../" in staged_text: + raise ValueError("staged pyproject.toml contains an external relative path") + + _remove_path(output_dir) + temporary_dir.replace(output_dir) + except BaseException: + _remove_path(temporary_dir) + raise + finally: + if archive_path is not None: + archive_path.unlink(missing_ok=True) + + return {"version": version, "source": str(output_dir)} + + +def main() -> int: + args = parse_args() + try: + if args.print_version: + print(read_vmi_version()) + return 0 + outputs = prepare_source(args.output_dir, args.revision) + print(outputs["source"]) + except (OSError, RuntimeError, subprocess.CalledProcessError, ValueError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packaging/ptoas-vmi/pyproject.toml b/packaging/ptoas-vmi/pyproject.toml deleted file mode 100644 index d655e523ff..0000000000 --- a/packaging/ptoas-vmi/pyproject.toml +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -# The VMI distribution intentionally installs the same `ptoas` import package -# and console script as the main distribution. The two wheels are mutually -# exclusive; this separate static PEP 621 project supplies the distribution -# name and version required by package indexes without a custom build backend. -# scikit-build-core forwards that version to CMake as -# `SKBUILD_PROJECT_VERSION_FULL`. - -[build-system] -requires = ["scikit-build-core>=0.12.2,<2", "pybind11<3", "nanobind>=2.4"] -build-backend = "scikit_build_core.build" - -[project] -name = "ptoas-vmi" -version = "0.1.4" -description = "PTO Assembler & Optimizer with VMI support" -requires-python = ">=3.10" -license = "Apache-2.0" -dependencies = [ - "numpy", -] -classifiers = [ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", -] - -[project.scripts] -ptoas = "ptoas._cli:main" - -[tool.scikit-build] -minimum-version = "build-system.requires" -cmake.source-dir = "../.." -cmake.version = "CMakeLists.txt" -ninja.version = ">=1.10" -build.verbose = false -editable.mode = "redirect" -install.components = ["PTOAS_Python"] - -[tool.scikit-build.wheel.packages] -ptodsl = "../../ptodsl/ptodsl" -ptoas = "../../ptodsl/ptoas" -tilelang_dsl = "../../tilelang-dsl/python/tilelang_dsl" - -[tool.scikit-build.cmake.define] -PTOAS_RELEASE_VERSION_OVERRIDE = { env = "PTOAS_RELEASE_VERSION_OVERRIDE", default = "" } -PTOAS_CLI_VERSION_LABEL = "vmi" diff --git a/packaging/ptoas-vmi/pyproject.toml.patch b/packaging/ptoas-vmi/pyproject.toml.patch new file mode 100644 index 0000000000..97cf810a8f --- /dev/null +++ b/packaging/ptoas-vmi/pyproject.toml.patch @@ -0,0 +1,29 @@ +diff --git a/pyproject.toml b/pyproject.toml +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -18,7 +18,7 @@ build-backend = "scikit_build_core.build" + [project] +-name = "ptoas" +-dynamic = ["version"] +-description = "PTO Assembler & Optimizer" ++name = "ptoas-vmi" ++version = "0.1.4" ++description = "PTO Assembler & Optimizer with VMI support" + readme = "README.md" + requires-python = ">=3.10" + license = "Apache-2.0" +@@ -43,4 +43,5 @@ cmake.version = "CMakeLists.txt" + ninja.version = ">=1.10" + build.verbose = false + editable.mode = "redirect" ++sdist.inclusion-mode = "manual" + install.components = ["PTOAS_Python"] +@@ -53,7 +54,3 @@ tilelang_dsl = "tilelang-dsl/python/tilelang_dsl" + [tool.scikit-build.cmake.define] + PTOAS_RELEASE_VERSION_OVERRIDE = { env = "PTOAS_RELEASE_VERSION_OVERRIDE", default = "" } +- +-[tool.scikit-build.metadata.version] +-provider = "scikit_build_core.metadata.regex" +-input = "CMakeLists.txt" +-regex = 'project\s*\(\s*ptoas\s+VERSION\s+(?P[0-9]+\.[0-9]+)\s*\)' ++PTOAS_CLI_VERSION_LABEL = "vmi" diff --git a/tilelang-dsl/python/tilelang_dsl/env_config.py b/tilelang-dsl/python/tilelang_dsl/env_config.py index a6967e869d..f45b256b94 100644 --- a/tilelang-dsl/python/tilelang_dsl/env_config.py +++ b/tilelang-dsl/python/tilelang_dsl/env_config.py @@ -84,7 +84,10 @@ def print_environment_help() -> None: print() print("After setup, verify with:") print(" python3 -c 'from ptoas.mlir import ir; print(\"MLIR bindings OK\")'") - print(" python3 -c 'from pto.dialects import pto; print(\"PTO dialect OK\")'") + print( + " python3 -c 'from ptoas.mlir.dialects import pto; " + "print(\"PTO dialect OK\")'" + ) print() print("=" * 60) diff --git a/tilelang-dsl/python/tilelang_dsl/lowering_backend.py b/tilelang-dsl/python/tilelang_dsl/lowering_backend.py index 9d1c5cd406..ed13f9a120 100644 --- a/tilelang-dsl/python/tilelang_dsl/lowering_backend.py +++ b/tilelang-dsl/python/tilelang_dsl/lowering_backend.py @@ -68,18 +68,13 @@ def _parse_text_to_module( ) -> "_ods_ir.Module": """Parse MLIR text into a structured Module. - Registers all required dialects (func, arith, scf, pto) before parsing. + The PTOAS MLIR site initializer registers the bundled upstream dialects. + Register the project-owned PTO dialect explicitly before parsing. """ from ptoas.mlir import ir as _ods_ir - from ptoas.mlir.dialects import func as _func_dialect - from ptoas.mlir.dialects import arith as _arith_dialect - from ptoas.mlir.dialects import scf as _scf_dialect - from pto.dialects import pto as _pto_dialect + from ptoas.mlir.dialects import pto as _pto_dialect ctx = context if context is not None else _ods_ir.Context() - _func_dialect.register_dialect(ctx) - _arith_dialect.register_dialect(ctx) - _scf_dialect.register_dialect(ctx) _pto_dialect.register_dialect(ctx, load=True) return _ods_ir.Module.parse(text, ctx) @@ -278,4 +273,4 @@ def lower_with_backend( "PybindBackend", "get_backend", "lower_with_backend", -] \ No newline at end of file +] diff --git a/tilelang-dsl/tests/test_lowering_backend_conversion.py b/tilelang-dsl/tests/test_lowering_backend_conversion.py new file mode 100644 index 0000000000..35d724b84d --- /dev/null +++ b/tilelang-dsl/tests/test_lowering_backend_conversion.py @@ -0,0 +1,23 @@ +# Copyright (c) 2026 Huawei Technologies Co., Ltd. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. + +import unittest + +from ptoas.mlir import ir +from tilelang_dsl.lowering_backend import LoweringResult + + +class LoweringBackendConversionTests(unittest.TestCase): + def test_text_result_parses_with_namespaced_pto_dialect(self): + module = LoweringResult(text="module {}").as_module() + + self.assertIsInstance(module, ir.Module) + + +if __name__ == "__main__": + unittest.main() From 4969eab5011ca7abf485a02c993283e2a1367c23 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Wed, 29 Jul 2026 09:39:14 +0800 Subject: [PATCH 23/32] ci: allow pull requests to save LLVM caches --- .github/workflows/build_wheel.yml | 12 ++++++------ .github/workflows/build_wheel_mac.yml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 1bbec889a4..7eba510489 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -178,11 +178,11 @@ jobs: git fetch --depth 1 origin "${{ steps.llvm-source.outputs.sha }}" git checkout --force "${{ steps.llvm-source.outputs.sha }}" - - name: Warn when default-branch cache is missing for PR/release runs - if: steps.cache-llvm.outputs.cache-hit != 'true' && (github.event_name == 'pull_request' || github.event_name == 'release') + - name: Warn when default-branch cache is missing for release runs + if: steps.cache-llvm.outputs.cache-hit != 'true' && github.event_name == 'release' run: | - echo "LLVM cache miss while running on PR/release." - echo "PR/release runs are cache-consumers only and should reuse cache generated on the default branch." + echo "LLVM cache miss while running on release." + echo "Release runs are cache-consumers only and should reuse cache generated on the default branch." echo "Please run this workflow on the default branch first to populate cache key:" echo "llvm-${{ steps.llvm-source.outputs.sha }}-manylinux_2_34-${{ matrix.arch }}-${{ matrix.python }}-${{ env.LLVM_CACHE_FLAVOR }}" @@ -205,8 +205,8 @@ jobs: # Upload cache immediately after the build step to avoid later steps polluting the release build tree. - name: Save LLVM cache - # Allow cache writes except for PR/release runs to avoid duplicated tag/merge caches. - if: steps.cache-llvm.outputs.cache-hit != 'true' && github.event_name != 'pull_request' && github.event_name != 'release' + # PR caches accelerate later commits and reruns, while release runs remain cache consumers only. + if: steps.cache-llvm.outputs.cache-hit != 'true' && github.event_name != 'release' uses: actions/cache/save@v4 with: path: /llvm-workspace/llvm-project/build-release diff --git a/.github/workflows/build_wheel_mac.yml b/.github/workflows/build_wheel_mac.yml index 3453822943..2b6fd0271b 100644 --- a/.github/workflows/build_wheel_mac.yml +++ b/.github/workflows/build_wheel_mac.yml @@ -177,11 +177,11 @@ jobs: git fetch --depth 1 origin "${{ steps.llvm-source.outputs.sha }}" git checkout --force "${{ steps.llvm-source.outputs.sha }}" - - name: Warn when default-branch cache is missing for PR/release runs - if: steps.cache-llvm.outputs.cache-hit != 'true' && (github.event_name == 'pull_request' || github.event_name == 'release') + - name: Warn when default-branch cache is missing for release runs + if: steps.cache-llvm.outputs.cache-hit != 'true' && github.event_name == 'release' run: | - echo "LLVM cache miss while running on PR/release." - echo "PR/release runs are cache-consumers only and should reuse cache generated on the default branch." + echo "LLVM cache miss while running on release." + echo "Release runs are cache-consumers only and should reuse cache generated on the default branch." echo "Please run this workflow on the default branch first to populate cache key:" echo "llvm-${{ steps.llvm-source.outputs.sha }}-${{ steps.runner-meta.outputs.label }}-${{ matrix.arch }}-${{ matrix.python }}-${{ env.LLVM_CACHE_FLAVOR }}" @@ -204,8 +204,8 @@ jobs: # Upload cache immediately after the build step to avoid later steps polluting the release build tree. - name: Save LLVM cache - # Allow cache writes except for PR/release runs to avoid duplicated tag/merge caches. - if: steps.cache-llvm.outputs.cache-hit != 'true' && github.event_name != 'pull_request' && github.event_name != 'release' + # PR caches accelerate later commits and reruns, while release runs remain cache consumers only. + if: steps.cache-llvm.outputs.cache-hit != 'true' && github.event_name != 'release' uses: actions/cache/save@v4 with: path: ${{ runner.temp }}/llvm-workspace/llvm-project/build-release From 6438e7b04d975ca4e14c090f7c8c9aa65a4edce6 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Wed, 29 Jul 2026 10:03:54 +0800 Subject: [PATCH 24/32] ci: cache LLVM builds for release runs --- .github/workflows/build_wheel.yml | 11 +---------- .github/workflows/build_wheel_mac.yml | 11 +---------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build_wheel.yml b/.github/workflows/build_wheel.yml index 7eba510489..3107aac34b 100644 --- a/.github/workflows/build_wheel.yml +++ b/.github/workflows/build_wheel.yml @@ -178,14 +178,6 @@ jobs: git fetch --depth 1 origin "${{ steps.llvm-source.outputs.sha }}" git checkout --force "${{ steps.llvm-source.outputs.sha }}" - - name: Warn when default-branch cache is missing for release runs - if: steps.cache-llvm.outputs.cache-hit != 'true' && github.event_name == 'release' - run: | - echo "LLVM cache miss while running on release." - echo "Release runs are cache-consumers only and should reuse cache generated on the default branch." - echo "Please run this workflow on the default branch first to populate cache key:" - echo "llvm-${{ steps.llvm-source.outputs.sha }}-manylinux_2_34-${{ matrix.arch }}-${{ matrix.python }}-${{ env.LLVM_CACHE_FLAVOR }}" - - name: Build LLVM/MLIR if: steps.cache-llvm.outputs.cache-hit != 'true' run: | @@ -205,8 +197,7 @@ jobs: # Upload cache immediately after the build step to avoid later steps polluting the release build tree. - name: Save LLVM cache - # PR caches accelerate later commits and reruns, while release runs remain cache consumers only. - if: steps.cache-llvm.outputs.cache-hit != 'true' && github.event_name != 'release' + if: steps.cache-llvm.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: path: /llvm-workspace/llvm-project/build-release diff --git a/.github/workflows/build_wheel_mac.yml b/.github/workflows/build_wheel_mac.yml index 2b6fd0271b..cac6d60841 100644 --- a/.github/workflows/build_wheel_mac.yml +++ b/.github/workflows/build_wheel_mac.yml @@ -177,14 +177,6 @@ jobs: git fetch --depth 1 origin "${{ steps.llvm-source.outputs.sha }}" git checkout --force "${{ steps.llvm-source.outputs.sha }}" - - name: Warn when default-branch cache is missing for release runs - if: steps.cache-llvm.outputs.cache-hit != 'true' && github.event_name == 'release' - run: | - echo "LLVM cache miss while running on release." - echo "Release runs are cache-consumers only and should reuse cache generated on the default branch." - echo "Please run this workflow on the default branch first to populate cache key:" - echo "llvm-${{ steps.llvm-source.outputs.sha }}-${{ steps.runner-meta.outputs.label }}-${{ matrix.arch }}-${{ matrix.python }}-${{ env.LLVM_CACHE_FLAVOR }}" - - name: Build LLVM/MLIR if: steps.cache-llvm.outputs.cache-hit != 'true' run: | @@ -204,8 +196,7 @@ jobs: # Upload cache immediately after the build step to avoid later steps polluting the release build tree. - name: Save LLVM cache - # PR caches accelerate later commits and reruns, while release runs remain cache consumers only. - if: steps.cache-llvm.outputs.cache-hit != 'true' && github.event_name != 'release' + if: steps.cache-llvm.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: path: ${{ runner.temp }}/llvm-workspace/llvm-project/build-release From 0110561032b8e9535705d6e73457d4cff318b70c Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Wed, 29 Jul 2026 12:42:32 +0800 Subject: [PATCH 25/32] ci: restore macOS wheel trigger policy --- .github/workflows/build_wheel_mac.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_wheel_mac.yml b/.github/workflows/build_wheel_mac.yml index cac6d60841..855742ddba 100644 --- a/.github/workflows/build_wheel_mac.yml +++ b/.github/workflows/build_wheel_mac.yml @@ -10,9 +10,8 @@ name: Build Wheel (macOS) on: push: - branches: - - main - pull_request: + tags-ignore: + - '**' # Nightly build & publish (UTC time, offset from Linux workflow). schedule: - cron: "40 17 * * *" From 59396a98ede070338d5983b847f2c2b1b7fdf1ff Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Wed, 29 Jul 2026 12:56:15 +0800 Subject: [PATCH 26/32] ci: preserve TileLang smoke policy after rebase --- .github/workflows/ci_sim.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci_sim.yml b/.github/workflows/ci_sim.yml index 5c6d9e2e38..dffc2e7ea7 100644 --- a/.github/workflows/ci_sim.yml +++ b/.github/workflows/ci_sim.yml @@ -700,17 +700,10 @@ jobs: mkdir -p "${TILELANG_DSL_WORKSPACE}" export LLVM_BUILD_DIR="${LLVM_DIR}" export MLIR_PYTHON_ROOT="${MLIR_PYTHONPATH}" - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \ - PTOAS_BIN="${PTOAS_BIN}" \ - bash test/tilelang_st/script/run_ci.sh -r sim -v a5 --jobs 64 --smoke \ - 2>&1 | tee "${TILELANG_DSL_WORKSPACE}/run_ci.log" - else - ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \ - PTOAS_BIN="${PTOAS_BIN}" \ - bash test/tilelang_st/script/run_ci.sh -r sim -v a5 --jobs 64 \ - 2>&1 | tee "${TILELANG_DSL_WORKSPACE}/run_ci.log" - fi + ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \ + PTOAS_BIN="${PTOAS_BIN}" \ + bash test/tilelang_st/script/run_ci.sh -r sim -v a5 --jobs 64 --smoke \ + 2>&1 | tee "${TILELANG_DSL_WORKSPACE}/run_ci.log" - name: Restore PTODSL LLVM build cache id: ptodsl-llvm-cache From df6aee2d4b8d855c5bc085dc67dc6b7c96b4d9a6 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Wed, 29 Jul 2026 15:15:43 +0800 Subject: [PATCH 27/32] docs: remove obsolete Python root variables --- .github/workflows/ci_sim.yml | 1 - README.md | 3 +-- docker/Dockerfile.dev | 1 - docs/designs/ci-board-validation-guide.md | 3 +-- scripts/ptoas_env.sh | 5 +---- 5 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci_sim.yml b/.github/workflows/ci_sim.yml index dffc2e7ea7..a3dcbdd151 100644 --- a/.github/workflows/ci_sim.yml +++ b/.github/workflows/ci_sim.yml @@ -699,7 +699,6 @@ jobs: set -euo pipefail mkdir -p "${TILELANG_DSL_WORKSPACE}" export LLVM_BUILD_DIR="${LLVM_DIR}" - export MLIR_PYTHON_ROOT="${MLIR_PYTHONPATH}" ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \ PTOAS_BIN="${PTOAS_BIN}" \ bash test/tilelang_st/script/run_ci.sh -r sim -v a5 --jobs 64 --smoke \ diff --git a/README.md b/README.md index f3d2cd47d2..a00c62660c 100644 --- a/README.md +++ b/README.md @@ -204,8 +204,7 @@ from ptoas.mlir.dialects import pto as mlir_pto # --- 运行时变量配置 (基于之前定义的路径) --- # 1. Python Path: PTOAS 的 build/install tree 已包含统一的 MLIR + PTO package -export PTO_PYTHON_ROOT=$PTO_INSTALL_DIR -export PYTHONPATH=$PTO_PYTHON_ROOT:$PYTHONPATH +export PYTHONPATH=$PTO_INSTALL_DIR:$PTO_SOURCE_DIR/build/python:$PYTHONPATH # 2. Library Path: 确保能加载 LLVM 和 PTO 的动态库 export LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:$LD_LIBRARY_PATH diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 193815e853..71a8dce2b9 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -75,7 +75,6 @@ RUN pip install --no-cache-dir torch==2.9.0 --index-url https://download.pytorch # ========================================================================= ENV PATH="${LLVM_INSTALL_DIR}/bin:${ASCEND_HOME_PATH}/compiler/bin:${PATH}" ENV LD_LIBRARY_PATH="${LLVM_INSTALL_DIR}/lib:${ASCEND_HOME_PATH}/lib64:${ASCEND_HOME_PATH}/lib64/plugin/opskernel:${ASCEND_HOME_PATH}/lib64/plugin/nnengine:${ASCEND_HOME_PATH}/opp/built-in/op_impl/ai_core/tbe/op_tiling/lib/linux/aarch64:${LD_LIBRARY_PATH}" -ENV MLIR_PYTHON_ROOT="${LLVM_INSTALL_DIR}/python_packages/mlir_core" ENV BISHENG_OPTS="-I${PTO_LIB_PATH}/include/pto -fPIC -shared -O2 -std=c++17 --npu-arch=dav-2201 -DMEMORY_BASE" diff --git a/docs/designs/ci-board-validation-guide.md b/docs/designs/ci-board-validation-guide.md index ea02051bfc..88b97a4884 100644 --- a/docs/designs/ci-board-validation-guide.md +++ b/docs/designs/ci-board-validation-guide.md @@ -122,8 +122,7 @@ ninja -C build install ### 3.3 跑样例生成链路 ```bash -export PTO_PYTHON_ROOT=$PWD/install -export PYTHONPATH="$PTO_PYTHON_ROOT:${PYTHONPATH:-}" +export PYTHONPATH="$PWD/install:$PWD/build/python:${PYTHONPATH:-}" export LD_LIBRARY_PATH="$LLVM_DIR/lib:$PTO_INSTALL_DIR/lib:${LD_LIBRARY_PATH:-}" export PTOAS_BIN=$PWD/build/tools/ptoas/ptoas diff --git a/scripts/ptoas_env.sh b/scripts/ptoas_env.sh index 1686ba1bc0..27f20cfbf5 100644 --- a/scripts/ptoas_env.sh +++ b/scripts/ptoas_env.sh @@ -53,8 +53,6 @@ PY )" fi export PTOAS_PYTHON_SITE -export PTO_PYTHON_ROOT="${PTO_PYTHON_ROOT:-${PTO_INSTALL_DIR}}" -export PTO_PYTHON_BUILD_ROOT="${PTO_PYTHON_BUILD_ROOT:-${PTO_SOURCE_DIR}/build/python}" export PTODSL_PYTHON_ROOT="${PTODSL_PYTHON_ROOT:-${PTO_SOURCE_DIR}/ptodsl}" export PTOAS_FLAGS="${PTOAS_FLAGS:-}" export PTOAS_OUT_DIR="${PTOAS_OUT_DIR:-${PTO_SOURCE_DIR}/build/output}" @@ -112,9 +110,8 @@ _ptoas_run_legacy_smoke_test() { # Prefer the build tree for active development, while retaining install-tree # fallbacks for scripts that consume a direct CMake installation. _ptoas_prepend_path PYTHONPATH "${PTO_INSTALL_DIR}" -_ptoas_prepend_path PYTHONPATH "${PTO_PYTHON_ROOT}" _ptoas_prepend_path PYTHONPATH "${PTOAS_PYTHON_SITE}" -_ptoas_prepend_path PYTHONPATH "${PTO_PYTHON_BUILD_ROOT}" +_ptoas_prepend_path PYTHONPATH "${PTO_SOURCE_DIR}/build/python" _ptoas_prepend_path PYTHONPATH "${PTODSL_PYTHON_ROOT}" _ptoas_prepend_path LD_LIBRARY_PATH "${LLVM_BUILD_DIR}/lib" From a4f3efda94a8a44f2d7cc30c9e86f2c739ff554a Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Wed, 29 Jul 2026 15:57:48 +0800 Subject: [PATCH 28/32] fix: validate repaired wheel in Docker builder --- docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8a46534173..59b92eb174 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -70,10 +70,10 @@ RUN pip install /wheelhouse/ptoas*.whl # test the installed wheel contract without adjusting $PYTHONPATH WORKDIR /test -RUN bash "$PTO_SOURCE_DIR/docker/test_wheel_imports.sh" +RUN PTO_TEST_WHEEL_PATH="/wheelhouse/ptoas*.whl" \ + bash "$PTO_SOURCE_DIR/docker/test_wheel_imports.sh" -# test ptoas cli -ENV PATH=$PTO_BUILD_DIR/tools/ptoas:$PATH +# test the wheel-provided ptoas cli RUN which ptoas WORKDIR $PTO_SOURCE_DIR/test/samples/MatMul/ From 56e34cf091b7d6fee300281ec332d5981ca9aad2 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Wed, 29 Jul 2026 15:57:48 +0800 Subject: [PATCH 29/32] build: use installed ptoas in repository workflows --- .claude/skill/pto_debug_skill.md | 17 +-- README.md | 25 +--- README_en.md | 2 +- docs/designs/ci-board-validation-guide.md | 5 +- docs/designs/ptoas-tileop-expand-design.md | 5 +- docs/designs/tilelang-st-framework.md | 10 +- ptodsl/README.md | 9 +- scripts/batch_compile_output_cpp.sh | 5 +- scripts/compile_pto_to_vpto_llvm.sh | 16 ++- scripts/ptoas_env.sh | 148 --------------------- test/dsl-st/README.md | 1 - test/samples/runop.sh | 29 ++-- test/tilelang_st/script/run_ci.sh | 6 - test/tilelang_st/script/run_st.py | 25 ++-- test/tilelib-st/README.md | 1 - tilelang-dsl/README.md | 4 +- tilelang-dsl/docs/v1-lowering.md | 2 +- 17 files changed, 65 insertions(+), 245 deletions(-) delete mode 100644 scripts/ptoas_env.sh diff --git a/.claude/skill/pto_debug_skill.md b/.claude/skill/pto_debug_skill.md index 99d04aca0c..389b3c9611 100644 --- a/.claude/skill/pto_debug_skill.md +++ b/.claude/skill/pto_debug_skill.md @@ -20,7 +20,7 @@ PTOAS 编译器调试工作流。给定一个 `.pto` 文件，AI 自动生成配 3. 全局替换：tadd→, TADD→ 4. 按 cases.py 调整 shape/dtype/eps 5. 修改 gen_data.py 的 golden 公式 -6. 运行: python3 test/tilelang_st/script/run_st.py -r sim -v a5 -t -p build/tools/ptoas/ptoas +6. 运行: python3 test/tilelang_st/script/run_st.py -r sim -v a5 -t 7. 如果比对失败 → 阶段 2 TPrint 调试 ``` @@ -29,7 +29,8 @@ PTOAS 编译器调试工作流。给定一个 `.pto` 文件，AI 自动生成配 ## 前置 ```bash -source scripts/ptoas_env.sh +LLVM_BUILD_DIR=/path/to/llvm/build ./quick_install.sh +source "${ASCEND_HOME_PATH}/bin/setenv.bash" ``` 所有命令从 repo root 执行。 @@ -45,7 +46,7 @@ source scripts/ptoas_env.sh | PTO IR 样本/测试 | `test/samples//` | | ST testcase | `test/tilelang_st/npu/a5/src/st/testcase/` | | ST 运行脚本 | `test/tilelang_st/script/run_st.py` | -| ptoas 二进制 | `build/tools/ptoas/ptoas` | +| ptoas 命令 | 当前 Python 环境安装的 `ptoas`（可用 `PTOAS_BIN` 覆盖） | | ODS 定义 | `include/PTO/IR/PTOOps.td` | | VPTO ODS 定义 | `include/PTO/IR/VPTOOps.td` | | C++ IR/Verifier | `lib/PTO/IR/PTO.cpp` | @@ -668,22 +669,22 @@ TPrint 不匹配时，按以下顺序排查： **完整验证**： ```bash -ninja -C build ptoas && \ +ninja -C build ptoas && pip install -e . --no-build-isolation && \ rm -rf test/tilelang_st/npu/a5/src/st/build && \ - python3 test/tilelang_st/script/run_st.py -r sim -v a5 -t -p build/tools/ptoas/ptoas + python3 test/tilelang_st/script/run_st.py -r sim -v a5 -t ``` ### run_st.py 常用参数 ```bash # 运行单个 op 的所有 case -python3 test/tilelang_st/script/run_st.py -r sim -v a5 -t tadd -p build/tools/ptoas/ptoas +python3 test/tilelang_st/script/run_st.py -r sim -v a5 -t tadd # 只运行特定 case -python3 test/tilelang_st/script/run_st.py -r sim -v a5 -t tadd -c f32_16x64 -p build/tools/ptoas/ptoas +python3 test/tilelang_st/script/run_st.py -r sim -v a5 -t tadd -c f32_16x64 # 跳过构建（使用已有 build） -python3 test/tilelang_st/script/run_st.py -r sim -v a5 -t tadd -p build/tools/ptoas/ptoas --without-build +python3 test/tilelang_st/script/run_st.py -r sim -v a5 -t tadd --without-build ``` --- diff --git a/README.md b/README.md index a00c62660c..ac3c4b41f8 100644 --- a/README.md +++ b/README.md @@ -192,26 +192,13 @@ from ptoas.mlir.dialects import pto as mlir_pto 完成安装，那么 `import ptodsl` / `from ptoas.mlir.dialects import pto` / `ptoas` 都不应再依赖手动设置 `PYTHONPATH`。 -下面这组环境变量主要用于**直接消费 build/install tree** 的场景，例如： - -- 不走 pip 安装，直接调试 CMake install 输出 -- 调试 `ptoas` CLI、动态库搜索路径或 build-tree Python package -- 复用仓库脚本做 compile-only / simulator / sample 生成 - -您可以将以下命令添加到 `.bashrc` 或启动脚本中。 +需要 CANN、Bisheng、simulator 或 NPU 时，在完成 PTOAS 安装后再加载 +CANN 自带的环境脚本。PTOAS 的 Python 包和 CLI 仍由当前 pip 环境提供， +不需要手工添加 build/install tree 到 `PYTHONPATH` 或 `PATH`。 ```bash -# --- 运行时变量配置 (基于之前定义的路径) --- - -# 1. Python Path: PTOAS 的 build/install tree 已包含统一的 MLIR + PTO package -export PYTHONPATH=$PTO_INSTALL_DIR:$PTO_SOURCE_DIR/build/python:$PYTHONPATH - -# 2. Library Path: 确保能加载 LLVM 和 PTO 的动态库 -export LD_LIBRARY_PATH=$LLVM_BUILD_DIR/lib:$PTO_INSTALL_DIR/lib:$LD_LIBRARY_PATH - -# 3. PATH: 将 ptoas / ptobc 添加到命令行路径 -export PATH=$PTO_SOURCE_DIR/build-llvm21/tools/ptoas:$PTO_SOURCE_DIR/build-llvm21/tools/ptobc:$PATH - +source "${ASCEND_HOME_PATH}/bin/setenv.bash" +ptoas --version ``` --- @@ -272,7 +259,7 @@ cd $PTO_SOURCE_DIR/test/samples/MatMul/ python3 ./tmatmulk.py > ./tmatmulk.pto # 运行ptoas 测试 -$PTO_SOURCE_DIR/build/tools/ptoas/ptoas ./tmatmulk.pto -o ./tmatmulk.cpp +ptoas ./tmatmulk.pto -o ./tmatmulk.cpp ``` ### 5.4 上板验证 diff --git a/README_en.md b/README_en.md index 73fc6a794a..046ad24e4d 100644 --- a/README_en.md +++ b/README_en.md @@ -223,7 +223,7 @@ cd $PTO_SOURCE_DIR/test/samples/MatMul/ python3 ./tmatmulk.py > ./tmatmulk.pto # Run ptoas tests -$PTO_SOURCE_DIR/build/tools/ptoas/ptoas ./tmatmulk.pto -o ./tmatmulk.cpp +ptoas ./tmatmulk.pto -o ./tmatmulk.cpp ``` ### 4.4 On-Board Validation diff --git a/docs/designs/ci-board-validation-guide.md b/docs/designs/ci-board-validation-guide.md index 88b97a4884..a9506fc8f4 100644 --- a/docs/designs/ci-board-validation-guide.md +++ b/docs/designs/ci-board-validation-guide.md @@ -122,9 +122,8 @@ ninja -C build install ### 3.3 跑样例生成链路 ```bash -export PYTHONPATH="$PWD/install:$PWD/build/python:${PYTHONPATH:-}" -export LD_LIBRARY_PATH="$LLVM_DIR/lib:$PTO_INSTALL_DIR/lib:${LD_LIBRARY_PATH:-}" -export PTOAS_BIN=$PWD/build/tools/ptoas/ptoas +LLVM_BUILD_DIR="$LLVM_DIR" ./quick_install.sh +export PTOAS_BIN="$(command -v ptoas)" bash test/samples/runop.sh --enablebc all ``` diff --git a/docs/designs/ptoas-tileop-expand-design.md b/docs/designs/ptoas-tileop-expand-design.md index 9c5c49c577..56fffc09fc 100644 --- a/docs/designs/ptoas-tileop-expand-design.md +++ b/docs/designs/ptoas-tileop-expand-design.md @@ -1248,9 +1248,10 @@ Python 侧的 case 名、dtype、shape、valid_shape、eps（以及必要时的 ##### 运行方式 统一入口为 `test/tilelang_st/script/run_st.py`。前置条件： -- `ptoas` 已编译（默认路径 `build/tools/ptoas/ptoas`，也可通过 `-p` 指定或 `PTOAS_BIN` 环境变量） +- `ptoas` 已通过 `./quick_install.sh` 或 editable install 安装到当前 + Python 环境（也可通过 `-p` 指定或 `PTOAS_BIN` 环境变量覆盖） - `ASCEND_HOME_PATH` 已设置 -- 建议先执行 `source scripts/ptoas_env.sh` +- simulator / NPU 运行前已加载 CANN `setenv.bash` ```bash # simulator 上跑 tsub 全部 case diff --git a/docs/designs/tilelang-st-framework.md b/docs/designs/tilelang-st-framework.md index 60a48bb051..189cafc83a 100644 --- a/docs/designs/tilelang-st-framework.md +++ b/docs/designs/tilelang-st-framework.md @@ -140,15 +140,17 @@ test/tilelang_st/ 运行 TileLang ST 之前，建议先确认下面几件事： -- 仓库里的 `ptoas` 已经编出来，默认路径是 `build/tools/ptoas/ptoas` +- `ptoas` 已经通过 `./quick_install.sh` 或 `pip install -e . --no-build-isolation` + 安装到当前 Python 环境 - `ASCEND_HOME_PATH` 已经设置正确 -- 如果需要手工跑 `ptoas`、`bisheng` 或 lit，优先先执行： +- 如果需要 simulator、NPU 或 `bisheng`，再加载 CANN 环境： ```bash -source scripts/ptoas_env.sh +source "${ASCEND_HOME_PATH}/bin/setenv.bash" ``` -`run_st.py` 会在运行时补充 simulator / NPU 相关环境，但它不会替你构建 `ptoas`。 +`run_st.py` 会优先使用 `PTOAS_BIN`，否则使用当前 `PATH` 中由 pip +安装的 `ptoas`。它不会替你构建或安装 PTOAS。 ### 5.1 运行已有 testcase diff --git a/ptodsl/README.md b/ptodsl/README.md index 7a5ecaf333..f0023f325f 100644 --- a/ptodsl/README.md +++ b/ptodsl/README.md @@ -73,13 +73,13 @@ Artifact boundary: - `ptoas` wheel: PTODSL-capable Python distribution - `ptoas-bin-*.tar.gz`: compiler-only artifact, does **not** imply `import ptodsl` -If you are working from a source checkout and want the repository helper -scripts (`scripts/sim_dsl.sh`, sample runners, direct CLI debugging) to pick up -the local build/install tree, you may still source: +For source development, install the checkout into the active Python environment. +The editable install retains an incremental CMake build tree without requiring +manual `PYTHONPATH` or build-tree wrapper setup: ```bash cd $PTOAS_REPO_ROOT -source scripts/ptoas_env.sh +LLVM_BUILD_DIR=/path/to/llvm/build ./quick_install.sh ``` --- @@ -135,7 +135,6 @@ Set up the environment in each new shell: ```bash cd $PTOAS_REPO_ROOT pip install -e . --no-build-isolation -source scripts/ptoas_env.sh source "${ASCEND_HOME_PATH}/bin/setenv.bash" ``` diff --git a/scripts/batch_compile_output_cpp.sh b/scripts/batch_compile_output_cpp.sh index 8de0fb1141..093cf5bed7 100755 --- a/scripts/batch_compile_output_cpp.sh +++ b/scripts/batch_compile_output_cpp.sh @@ -64,8 +64,7 @@ print_usage() { --log-dir 编译日志目录，默认: /logs --help, -h 显示帮助 -推荐先执行: - source scripts/ptoas_env.sh +运行前请通过 CANN setenv 或显式参数准备 bisheng 和 PTO-ISA 路径。 默认编译参数来源: 由 test/npu_validation/scripts/generate_testcase.py 中 @@ -155,7 +154,7 @@ elif [[ "${COMPILER}" != */* ]] && command -v "${COMPILER}" >/dev/null 2>&1; the COMPILER="$(command -v "${COMPILER}")" fi -[[ -n "${COMPILER}" ]] || die "未找到编译器，请先 source scripts/ptoas_env.sh，或通过 --compiler/COMPILER 指定 bisheng 路径" +[[ -n "${COMPILER}" ]] || die "未找到编译器，请通过 CANN setenv、--compiler 或 COMPILER 指定 bisheng 路径" [[ -n "${PTO_ISA_PATH}" ]] || die "未找到 PTO-ISA 路径，请通过 --pto-isa-path、PTO_ISA_PATH 或 PTO_ISA_ROOT 指定" [[ -x "${COMPILER}" ]] || die "编译器不可执行: ${COMPILER}" [[ -d "${SRC_ROOT}" ]] || die "源码目录不存在: ${SRC_ROOT}" diff --git a/scripts/compile_pto_to_vpto_llvm.sh b/scripts/compile_pto_to_vpto_llvm.sh index 4b6c066c94..0dd0e123c2 100755 --- a/scripts/compile_pto_to_vpto_llvm.sh +++ b/scripts/compile_pto_to_vpto_llvm.sh @@ -15,12 +15,12 @@ ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" PTO_FILE="${1:-}" OUT_DIR_ARG="${2:-}" -PTOAS_BIN="${PTOAS_BIN:-${ROOT_DIR}/build/tools/ptoas/ptoas}" +PTOAS_BIN="${PTOAS_BIN:-}" PTOAS_FLAGS="${PTOAS_FLAGS:---pto-arch a5}" VPTO_FLAGS="${VPTO_FLAGS:---pto-backend=vpto --vpto-emit-hivm-llvm}" AICORE_ARCH="${AICORE_ARCH:-dav-c310-vec}" ASCEND_HOME_PATH="${ASCEND_HOME_PATH:-${HOME}/cann}" -BISHENG_BIN="" +BISHENG_BIN="${BISHENG_BIN:-}" BISHENG_FLAGS="${BISHENG_FLAGS:-}" LLVM_IR="" DEVICE_OBJ="" @@ -74,19 +74,21 @@ EOF [[ "${PTO_FILE}" == *.pto ]] || die "input must be a .pto file: ${PTO_FILE}" [[ -f "${PTO_FILE}" ]] || die "missing input file: ${PTO_FILE}" -set +u -source "${ROOT_DIR}/scripts/ptoas_env.sh" -set -u - if [[ -n "${ASCEND_HOME_PATH}" && -f "${ASCEND_HOME_PATH}/set_env.sh" ]]; then set +u source "${ASCEND_HOME_PATH}/set_env.sh" >/dev/null 2>&1 set -u fi +if [[ -z "${PTOAS_BIN}" ]]; then + PTOAS_BIN="$(command -v ptoas || true)" +elif [[ "${PTOAS_BIN}" != */* ]]; then + PTOAS_BIN="$(command -v "${PTOAS_BIN}" || true)" +fi BISHENG_BIN="${BISHENG_BIN:-${ASCEND_HOME_PATH}/bin/bisheng}" -[[ -x "${PTOAS_BIN}" ]] || die "PTOAS_BIN is not executable: ${PTOAS_BIN}" +[[ -n "${PTOAS_BIN}" && -x "${PTOAS_BIN}" ]] || \ + die "ptoas not found; run ./quick_install.sh or set PTOAS_BIN" command -v "${BISHENG_BIN}" >/dev/null 2>&1 || die "bisheng not found: ${BISHENG_BIN}" pto_abs="$(cd "$(dirname "${PTO_FILE}")" && pwd)/$(basename "${PTO_FILE}")" diff --git a/scripts/ptoas_env.sh b/scripts/ptoas_env.sh deleted file mode 100644 index 27f20cfbf5..0000000000 --- a/scripts/ptoas_env.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026 Huawei Technologies Co., Ltd. -# This program is free software, you can redistribute it and/or modify it under the terms and conditions of -# CANN Open Software License Agreement Version 2.0 (the "License"). -# Please refer to the License for details. You may not use this file except in compliance with the License. -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. -# See LICENSE in the root of the software repository for the full text of the License. - -# PTOAS runtime environment bootstrap. -# Usage: -# source scripts/ptoas_env.sh -# -# Optional overrides before sourcing: -# export WORKSPACE_DIR=/path/to/workspace -# export LLVM_BUILD_DIR=/path/to/llvm-project/build-shared -# export PTO_SOURCE_DIR=/path/to/PTOAS -# export PTODSL_PYTHON_ROOT=/path/to/PTOAS/ptodsl -# export PTO_INSTALL_DIR=/path/to/PTOAS/install -# export PTO_PYTHON_BIN=/path/to/python3 -# export PTOAS_ENV_SKIP_SMOKE_TEST=1 # skip legacy MatMul/Abs sample checks -# CI shells skip the legacy smoke test by default. - -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - echo "This script must be sourced: source scripts/ptoas_env.sh" - exit 1 -fi - -_PTOAS_ENV_SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -_PTOAS_REPO_DIR="$(cd -- "${_PTOAS_ENV_SCRIPT_DIR}/.." && pwd)" - -# Default layout: -# / -# ├── PTOAS/ -# └── llvm-project/ -export PTO_SOURCE_DIR="${PTO_SOURCE_DIR:-${_PTOAS_REPO_DIR}}" -export WORKSPACE_DIR="${WORKSPACE_DIR:-$(cd -- "${PTO_SOURCE_DIR}/.." && pwd)}" -export LLVM_SOURCE_DIR="${LLVM_SOURCE_DIR:-${WORKSPACE_DIR}/llvm-project}" -export LLVM_BUILD_DIR="${LLVM_BUILD_DIR:-${LLVM_SOURCE_DIR}/build-shared}" -export PTO_INSTALL_DIR="${PTO_INSTALL_DIR:-${PTO_SOURCE_DIR}/install}" -export PTO_ISA_PATH="${PTO_ISA_PATH:-${WORKSPACE_DIR}/pto-isa}" -export ASCEND_HOME_PATH="${ASCEND_HOME_PATH:-${HOME}/cann}" - -if [[ -z "${PTOAS_PYTHON_SITE:-}" ]]; then - PTOAS_PYTHON_SITE="$( - PTO_INSTALL_DIR="${PTO_INSTALL_DIR}" python3 - <<'PY' 2>/dev/null || true -import os -import sysconfig - -prefix = os.environ["PTO_INSTALL_DIR"] -print(sysconfig.get_path("purelib", vars={"base": prefix, "platbase": prefix})) -PY - )" -fi -export PTOAS_PYTHON_SITE -export PTODSL_PYTHON_ROOT="${PTODSL_PYTHON_ROOT:-${PTO_SOURCE_DIR}/ptodsl}" -export PTOAS_FLAGS="${PTOAS_FLAGS:-}" -export PTOAS_OUT_DIR="${PTOAS_OUT_DIR:-${PTO_SOURCE_DIR}/build/output}" - -_ptoas_prepend_path() { - local var_name="$1" - local value="$2" - local current="${!var_name:-}" - if [[ -z "${value}" ]]; then - return 0 - fi - if [[ ! -e "${value}" ]]; then - return 0 - fi - if [[ ":${current}:" == *":${value}:"* ]]; then - return 0 - fi - if [[ -z "${current}" ]]; then - printf -v "${var_name}" '%s' "${value}" - else - printf -v "${var_name}" '%s:%s' "${value}" "${current}" - fi - export "${var_name}" -} - -_ptoas_run_legacy_smoke_test() { - local python_bin="${PTO_PYTHON_BIN:-${PYTHON_BIN:-$(command -v python3 || command -v python || true)}}" - if [[ -z "${python_bin}" ]]; then - echo "error: unable to locate python3 or python for scripts/ptoas_env.sh" >&2 - return 1 - fi - - if ! command -v ptoas >/dev/null 2>&1; then - echo "error: ptoas is not on PATH after sourcing scripts/ptoas_env.sh" >&2 - return 1 - fi - - local tmp_root="${PTOAS_ENV_TMP:-${PTO_SOURCE_DIR}/tmp/set_ptoas_env}" - mkdir -p "${tmp_root}/MatMul" "${tmp_root}/Abs" - ( - cd "${PTO_SOURCE_DIR}/test/samples/MatMul" && - "${python_bin}" ./tmatmulk.py > "${tmp_root}/MatMul/tmatmulk.pto" && - ptoas "${tmp_root}/MatMul/tmatmulk.pto" -o "${tmp_root}/MatMul/tmatmulk.cpp" - ) - ( - cd "${PTO_SOURCE_DIR}/test/samples/Abs" && - "${python_bin}" ./abs.py > "${tmp_root}/Abs/abs.pto" && - ptoas --enable-insert-sync "${tmp_root}/Abs/abs.pto" -o "${tmp_root}/Abs/abs.cpp" - ) - - echo "test set_env: OK" -} - -# PTOAS stages a complete MLIR + PTO package in its build and install trees. -# Prefer the build tree for active development, while retaining install-tree -# fallbacks for scripts that consume a direct CMake installation. -_ptoas_prepend_path PYTHONPATH "${PTO_INSTALL_DIR}" -_ptoas_prepend_path PYTHONPATH "${PTOAS_PYTHON_SITE}" -_ptoas_prepend_path PYTHONPATH "${PTO_SOURCE_DIR}/build/python" -_ptoas_prepend_path PYTHONPATH "${PTODSL_PYTHON_ROOT}" - -_ptoas_prepend_path LD_LIBRARY_PATH "${LLVM_BUILD_DIR}/lib" -_ptoas_prepend_path LD_LIBRARY_PATH "${PTO_INSTALL_DIR}/lib" -_ptoas_prepend_path LD_LIBRARY_PATH "${PTO_SOURCE_DIR}/build/lib" - -_ptoas_prepend_path PATH "${PTO_SOURCE_DIR}/build/tools/ptoas" - -if [[ -n "${PTO_PYTHON_BIN:-}" && -x "${PTO_PYTHON_BIN}" ]]; then - alias ptoas-python="${PTO_PYTHON_BIN}" -fi - -echo "[ptoas_env] PTO_SOURCE_DIR=${PTO_SOURCE_DIR}" -echo "[ptoas_env] LLVM_BUILD_DIR=${LLVM_BUILD_DIR}" -echo "[ptoas_env] PTO_INSTALL_DIR=${PTO_INSTALL_DIR}" -echo "[ptoas_env] PTO_ISA_PATH=${PTO_ISA_PATH}" -echo "[ptoas_env] ASCEND_HOME_PATH=${ASCEND_HOME_PATH}" -echo "[ptoas_env] PATH/LD_LIBRARY_PATH updated" - -_ptoas_env_skip_smoke="${PTOAS_ENV_SKIP_SMOKE_TEST:-0}" -if [[ -z "${PTOAS_ENV_SKIP_SMOKE_TEST:-}" && - ("${CI:-}" == "1" || "${CI:-}" == "true") ]]; then - _ptoas_env_skip_smoke=1 -fi - -if [[ "${_ptoas_env_skip_smoke}" != "1" ]]; then - _ptoas_run_legacy_smoke_test -fi - -unset -f _ptoas_prepend_path -unset -f _ptoas_run_legacy_smoke_test -unset _PTOAS_ENV_SCRIPT_DIR -unset _PTOAS_REPO_DIR -unset _ptoas_env_skip_smoke diff --git a/test/dsl-st/README.md b/test/dsl-st/README.md index 94505052f5..442ba361d0 100644 --- a/test/dsl-st/README.md +++ b/test/dsl-st/README.md @@ -20,7 +20,6 @@ Python 环境，而不是在运行时自己修 `PYTHONPATH` 或 `sys.path`。 ```bash cd $PTOAS_REPO_ROOT pip install -e . --no-build-isolation -source scripts/ptoas_env.sh ``` 如果你在验证 release/CI wheel，也可以直接： diff --git a/test/samples/runop.sh b/test/samples/runop.sh index a529574a83..a9a632c9da 100755 --- a/test/samples/runop.sh +++ b/test/samples/runop.sh @@ -35,7 +35,7 @@ Env: PTOBC_BIN # path to ptobc executable (optional) PYTHON_BIN # python executable to run samples (optional) PTOAS_OUT_DIR # where generated *.mlir/*.cpp go (optional; defaults to a temp dir) - PTO_BUILD_DIR # build directory root that contains tools/ptoas and tools/ptobc (optional) + PTO_BUILD_DIR # build directory root that contains tools/ptobc (optional) PTOAS_FLAGS # extra flags passed to ptoas (e.g. --enable-insert-sync) PTOAS_ENABLE_INSERT_SYNC # 1 to append --enable-insert-sync to PTOAS_FLAGS (default: 1) PTO_PTO_DIRS # space-separated dirs to run .pto directly (default: Sync Qwen3DecodeA3 Qwen3DecodeA5 DeepseekV4DecodeA3 DeepseekV4DecodeA5 CommSync Prelu Rem Rems Gemvmx MatmulMxLowPrecision TquantMx Movfp) @@ -62,27 +62,16 @@ lcfirst() { resolve_ptoas_bin() { if [[ -n "${PTOAS_BIN}" ]]; then - echo "${PTOAS_BIN}" - return 0 + local override + override="$(command -v "${PTOAS_BIN}" 2>/dev/null || true)" + if [[ -n "${override}" && -x "${override}" ]]; then + echo "${override}" + return 0 + fi + return 1 fi - # Common locations: - # - installed wrapper in repo: PTOAS/install/bin/ptoas - # - out-of-tree build in repo: PTOAS/build/tools/ptoas/ptoas - # - legacy layout: build/bin/ptoas local cand - if [[ -n "${PTO_BUILD_DIR}" ]]; then - cand="${PTO_BUILD_DIR%/build}/install/bin/ptoas" - [[ -x "$cand" ]] && { echo "$cand"; return 0; } - cand="${PTO_BUILD_DIR}/tools/ptoas/ptoas" - [[ -x "$cand" ]] && { echo "$cand"; return 0; } - cand="${PTO_BUILD_DIR}/bin/ptoas" - [[ -x "$cand" ]] && { echo "$cand"; return 0; } - fi - cand="${BASE_DIR}/../../build/tools/ptoas/ptoas" - [[ -x "$cand" ]] && { echo "$cand"; return 0; } - cand="${BASE_DIR}/../../../../build/bin/ptoas" - [[ -x "$cand" ]] && { echo "$cand"; return 0; } cand="$(command -v ptoas 2>/dev/null || true)" [[ -n "$cand" && -x "$cand" ]] && { echo "$cand"; return 0; } @@ -240,7 +229,7 @@ process_one_dir() { fi if [[ -z "$ptoas" || ! -x "$ptoas" ]]; then - echo -e "${A}\tFAIL\tMissing executable: PTOAS_BIN (searched common paths)" + echo -e "${A}\tFAIL\tMissing ptoas command (install PTOAS or set PTOAS_BIN)" return 0 fi if [[ -z "$python" || ! -x "$python" ]]; then diff --git a/test/tilelang_st/script/run_ci.sh b/test/tilelang_st/script/run_ci.sh index 385a7bda9f..f6a4fe5eb6 100755 --- a/test/tilelang_st/script/run_ci.sh +++ b/test/tilelang_st/script/run_ci.sh @@ -10,12 +10,6 @@ set -euo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../../.." && pwd)" - -if [[ -f "${REPO_ROOT}/scripts/ptoas_env.sh" ]]; then - # shellcheck source=/dev/null - source "${REPO_ROOT}/scripts/ptoas_env.sh" -fi export PYTHONUNBUFFERED=1 diff --git a/test/tilelang_st/script/run_st.py b/test/tilelang_st/script/run_st.py index 45826a6929..ce6ed478c3 100755 --- a/test/tilelang_st/script/run_st.py +++ b/test/tilelang_st/script/run_st.py @@ -49,21 +49,18 @@ def _normalize_case_filters(case_filters): def find_ptoas_bin(): - """Locate the ptoas binary by walking up from this script to the repo root.""" + """Locate the ptoas entry point from an override or the active environment.""" env_bin = os.environ.get("PTOAS_BIN") - if env_bin and os.path.isfile(env_bin): - return os.path.abspath(env_bin) - - search_dir = os.path.dirname(os.path.abspath(__file__)) - for _ in range(8): - candidate = os.path.join(search_dir, "build", "tools", "ptoas", "ptoas") - if os.path.isfile(candidate): - return os.path.abspath(candidate) - parent = os.path.dirname(search_dir) - if parent == search_dir: - break - search_dir = parent - return None + if env_bin: + resolved = shutil.which(env_bin) + if resolved: + return os.path.abspath(resolved) + if os.path.isfile(env_bin) and os.access(env_bin, os.X_OK): + return os.path.abspath(env_bin) + return None + + resolved = shutil.which("ptoas") + return os.path.abspath(resolved) if resolved else None def set_env_variables(run_mode, soc_version): diff --git a/test/tilelib-st/README.md b/test/tilelib-st/README.md index 4dce3784ca..51d4e63bd1 100644 --- a/test/tilelib-st/README.md +++ b/test/tilelib-st/README.md @@ -228,7 +228,6 @@ export ASCEND_HOME_PATH=/path/to/cann export LLVM_BUILD_DIR=/path/to/vpto-llvm/build export PYTHON_BIN=/path/to/python-with-torch-npu export PTO_PYTHON_BIN="$PYTHON_BIN" -export PTOAS_ENV_SKIP_SMOKE_TEST=1 export TORCH_DEVICE_BACKEND_AUTOLOAD=0 ``` diff --git a/tilelang-dsl/README.md b/tilelang-dsl/README.md index 37d13f0158..38095a856c 100644 --- a/tilelang-dsl/README.md +++ b/tilelang-dsl/README.md @@ -111,9 +111,9 @@ To check that the generated MLIR passes the current repo VPTO authoring-stage legality path: ```bash -source scripts/ptoas_env.sh +LLVM_BUILD_DIR=/path/to/llvm/build ./quick_install.sh PYTHONPATH=$PWD/tilelang-dsl/python python3 tilelang-dsl/examples/v1_verify_smoke.py /tmp/tilelang_v1_verify.mlir -build/tools/ptoas/ptoas --pto-arch a5 --pto-backend=vpto --emit-vpto \ +ptoas --pto-arch a5 --pto-backend=vpto --emit-vpto \ /tmp/tilelang_v1_verify.mlir -o /tmp/tilelang_v1_verify.checked.mlir ``` diff --git a/tilelang-dsl/docs/v1-lowering.md b/tilelang-dsl/docs/v1-lowering.md index 8568fbc4ae..0355e9e19b 100644 --- a/tilelang-dsl/docs/v1-lowering.md +++ b/tilelang-dsl/docs/v1-lowering.md @@ -164,7 +164,7 @@ PYTHONPATH=$PWD/tilelang-dsl/python \ PYTHONPATH=$PWD/tilelang-dsl/python \ python3 tilelang-dsl/examples/v1_verify_smoke.py /tmp/tilelang_v1_verify.mlir -build/tools/ptoas/ptoas --pto-arch a5 --pto-backend=vpto --emit-vpto \ +ptoas --pto-arch a5 --pto-backend=vpto --emit-vpto \ /tmp/tilelang_v1_verify.mlir -o /tmp/tilelang_v1_verify.checked.mlir ``` From 98ac5bc51f477d1159d53bb1e92712a4a919266e Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Wed, 29 Jul 2026 15:57:48 +0800 Subject: [PATCH 30/32] fix: export remaining low-precision PTO types --- CMakeLists.txt | 9 +++++++++ python/pto/dialects/pto.py | 4 ++++ test/python/low_precision_types.py | 4 ++++ 3 files changed, 17 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index a8b0d3b3ed..720ad96cbc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -248,6 +248,15 @@ if(BUILD_TESTING) LABELS "PTO" ENVIRONMENT "${_pto_python_test_env}" ) + add_test( + NAME pto_low_precision_types + COMMAND "${Python3_EXECUTABLE}" -S + "${CMAKE_CURRENT_SOURCE_DIR}/test/python/low_precision_types.py" + ) + set_tests_properties(pto_low_precision_types PROPERTIES + LABELS "PTO" + ENVIRONMENT "${_pto_python_test_env}" + ) endif() add_subdirectory(tools/ptobc/tests) endif() diff --git a/python/pto/dialects/pto.py b/python/pto/dialects/pto.py index 83709ba68a..eb7731e7af 100644 --- a/python/pto/dialects/pto.py +++ b/python/pto/dialects/pto.py @@ -51,6 +51,8 @@ def _export_optional_cext_symbol(name): AsyncEventType = _pto_mod.AsyncEventType PrefetchAsyncContextType = _export_optional_cext_symbol("PrefetchAsyncContextType") HiF8Type = _pto_mod.HiF8Type +HiF8x2Type = _pto_mod.HiF8x2Type +F8E8M0Type = _pto_mod.F8E8M0Type F4E1M2x2Type = _pto_mod.F4E1M2x2Type F4E2M1x2Type = _pto_mod.F4E2M1x2Type TensorViewType = _pto_mod.TensorViewType @@ -229,6 +231,8 @@ def fence_scope_attr_builder(value, context=None): "AsyncSessionType", "AsyncEventType", "HiF8Type", + "HiF8x2Type", + "F8E8M0Type", "F4E1M2x2Type", "F4E2M1x2Type", "TensorViewType", diff --git a/test/python/low_precision_types.py b/test/python/low_precision_types.py index aac0e2d976..38e9999de7 100644 --- a/test/python/low_precision_types.py +++ b/test/python/low_precision_types.py @@ -21,10 +21,14 @@ def main() -> None: pto.register_dialect(ctx) hif8 = pto.HiF8Type.get(ctx) + hif8x2 = pto.HiF8x2Type.get(ctx) + f8e8m0 = pto.F8E8M0Type.get(ctx) f4e1 = pto.F4E1M2x2Type.get(ctx) f4e2 = pto.F4E2M1x2Type.get(ctx) assert_contains(str(hif8), "hif8") + assert_contains(str(hif8x2), "hif8x2") + assert_contains(str(f8e8m0), "f8E8M0") assert_contains(str(f4e1), "f4E1M2x2") assert_contains(str(f4e2), "f4E2M1x2") From aa3c436b3f3d31a01c2890b4a3c646b5cc727b07 Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Wed, 29 Jul 2026 16:53:33 +0800 Subject: [PATCH 31/32] ci: limit TileLang shared build concurrency --- .github/workflows/ci_sim.yml | 3 ++- test/tilelang_st/script/run_all_st.py | 16 +++++++++++++++- .../script/run_ptodsl_st_parallel.py | 1 + test/tilelang_st/script/run_st.py | 19 +++++++++++++++---- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_sim.yml b/.github/workflows/ci_sim.yml index a3dcbdd151..27faf767f1 100644 --- a/.github/workflows/ci_sim.yml +++ b/.github/workflows/ci_sim.yml @@ -701,7 +701,8 @@ jobs: export LLVM_BUILD_DIR="${LLVM_DIR}" ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \ PTOAS_BIN="${PTOAS_BIN}" \ - bash test/tilelang_st/script/run_ci.sh -r sim -v a5 --jobs 64 --smoke \ + bash test/tilelang_st/script/run_ci.sh -r sim -v a5 \ + --build-jobs 8 --jobs 64 --smoke \ 2>&1 | tee "${TILELANG_DSL_WORKSPACE}/run_ci.log" - name: Restore PTODSL LLVM build cache diff --git a/test/tilelang_st/script/run_all_st.py b/test/tilelang_st/script/run_all_st.py index 557482548d..34f55d27aa 100755 --- a/test/tilelang_st/script/run_all_st.py +++ b/test/tilelang_st/script/run_all_st.py @@ -101,6 +101,10 @@ def parse_args(): "-j", "--jobs", type=int, default=1, help="Number of testcases to run in parallel after the shared build (default: 1).", ) + parser.add_argument( + "--build-jobs", type=int, default=None, + help="Maximum parallel jobs for the shared CMake build (default: host CPU count).", + ) parser.add_argument( "--smoke", action="store_true", help="Run only a representative smoke subset of cases for each testcase.", @@ -168,6 +172,9 @@ def main(): if args.jobs < 1: print("[ERROR] --jobs must be >= 1", file=sys.stderr) sys.exit(1) + if args.build_jobs is not None and args.build_jobs < 1: + print("[ERROR] --build-jobs must be >= 1", file=sys.stderr) + sys.exit(1) batch_script_path = os.path.abspath(__file__) run_st_script_path = os.path.abspath(run_st.__file__) @@ -219,6 +226,7 @@ def main(): print(f"[INFO] selected_testcases={', '.join(selected_testcases)}") print(f"[INFO] smoke={args.smoke}") print(f"[INFO] jobs={args.jobs}") + print(f"[INFO] build_jobs={args.build_jobs or os.cpu_count() or 4}") original_dir = os.getcwd() failures = [] @@ -232,7 +240,13 @@ def main(): else: build_target = "all" print(f"[INFO] build requested for {build_target}") - run_st.build_project(args.run_mode, default_soc_version, build_target, ptoas_bin) + run_st.build_project( + args.run_mode, + default_soc_version, + build_target, + ptoas_bin, + args.build_jobs, + ) total = len(selected_testcases) if args.jobs == 1: diff --git a/test/tilelang_st/script/run_ptodsl_st_parallel.py b/test/tilelang_st/script/run_ptodsl_st_parallel.py index f446992997..4c1e4a0ac7 100755 --- a/test/tilelang_st/script/run_ptodsl_st_parallel.py +++ b/test/tilelang_st/script/run_ptodsl_st_parallel.py @@ -83,6 +83,7 @@ def _run_build(args, default_soc_version, target_dir, log_dir, ptoas_bin): default_soc_version, "all", str(ptoas_bin), + args.jobs, ) finally: os.chdir(original_dir) diff --git a/test/tilelang_st/script/run_st.py b/test/tilelang_st/script/run_st.py index ce6ed478c3..db5fe52eff 100755 --- a/test/tilelang_st/script/run_st.py +++ b/test/tilelang_st/script/run_st.py @@ -115,7 +115,7 @@ def get_testcase_work_dir(testcase): return os.path.join("build", "testcase", testcase) -def build_project(run_mode, soc_version, testcase, ptoas_bin): +def build_project(run_mode, soc_version, testcase, ptoas_bin, build_jobs=None): build_dir = "build" if os.path.exists(build_dir): print(f"clean build: {build_dir}") @@ -140,8 +140,8 @@ def build_project(run_mode, soc_version, testcase, ptoas_bin): text=True, ) - cpu_count = os.cpu_count() or 4 - make_cmd = ["make", "VERBOSE=1", "-j", str(cpu_count)] + build_jobs = build_jobs or os.cpu_count() or 4 + make_cmd = ["make", "VERBOSE=1", "-j", str(build_jobs)] result = subprocess.run( make_cmd, cwd=build_dir, @@ -272,9 +272,14 @@ def main(): help="Skip build (requires prior build)") parser.add_argument("--target-dir", required=False, help="TileLang ST target directory. Defaults to npu//src/st.") + parser.add_argument("--build-jobs", type=int, default=None, + help="Maximum parallel jobs for the shared CMake build.") args = parser.parse_args() + if args.build_jobs is not None and args.build_jobs < 1: + parser.error("--build-jobs must be >= 1") + if args.soc_version == "a5": default_soc_version = "Ascend950PR_9599" else: @@ -312,7 +317,13 @@ def main(): set_env_variables(args.run_mode, default_soc_version) if not args.without_build: - build_project(args.run_mode, default_soc_version, testcase, ptoas_bin) + build_project( + args.run_mode, + default_soc_version, + testcase, + ptoas_bin, + args.build_jobs, + ) # gen golden → run binary → compare run_gen_data(testcase, args.case) From 07ddb454240b4989d13849fc16fb62bcecf148ab Mon Sep 17 00:00:00 2001 From: mouliangyu Date: Wed, 29 Jul 2026 16:59:50 +0800 Subject: [PATCH 32/32] ci: raise TileLang shared build concurrency --- .github/workflows/ci_sim.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_sim.yml b/.github/workflows/ci_sim.yml index 27faf767f1..54ddedf6ef 100644 --- a/.github/workflows/ci_sim.yml +++ b/.github/workflows/ci_sim.yml @@ -702,7 +702,7 @@ jobs: ASCEND_HOME_PATH="${ASCEND_HOME_PATH}" \ PTOAS_BIN="${PTOAS_BIN}" \ bash test/tilelang_st/script/run_ci.sh -r sim -v a5 \ - --build-jobs 8 --jobs 64 --smoke \ + --build-jobs 32 --jobs 64 --smoke \ 2>&1 | tee "${TILELANG_DSL_WORKSPACE}/run_ci.log" - name: Restore PTODSL LLVM build cache