From 37ff115395f79426e32d48431da8d283ffe23e4a Mon Sep 17 00:00:00 2001 From: temp-deepshard <304551954+temp-deepshard@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:43:31 -0700 Subject: [PATCH 1/4] Enforce complete Truffile release packages --- README.md | 16 ++++-- scripts/__init__.py | 1 + scripts/build_package.py | 103 +++++++++++++++++++++++++++++++++++++++ scripts/verify_wheel.py | 103 +++++++++++++++++++++++++++++++++++++++ setup.py | 38 +++++++++++++++ tests/test_packaging.py | 77 +++++++++++++++++++++++++++++ 6 files changed, 335 insertions(+), 3 deletions(-) create mode 100644 scripts/__init__.py create mode 100644 scripts/build_package.py create mode 100644 scripts/verify_wheel.py create mode 100644 setup.py create mode 100644 tests/test_packaging.py diff --git a/README.md b/README.md index 728369d..db5ec39 100644 --- a/README.md +++ b/README.md @@ -117,14 +117,24 @@ CLI wrappers: - `truffile models` - `truffile chat` (streaming by default) -## Proto Sync +## Release Packaging -Refresh vendored protos from firmware repo: +The app runtime and generated proto packages are owned by the private `pyfw` +source tree. They are intentionally ignored here so generated release inputs do +not become a second source of truth. + +From a clean checkout, stage both packages and verify the finished wheel: ```bash -./scripts/sync_protos.sh +python3.12 scripts/build_package.py --pyfw-path /path/to/pyfw +python3.12 -m build +python3.12 scripts/verify_wheel.py dist/*.whl ``` +The build fails if either package was not staged. `verify_wheel.py` then installs +the wheel in a fresh virtual environment, imports `truffile.sdk` and +`truffile.app_runtime`, and imports a newly generated foreground app. + ## Development Loop The supported app development loop is CLI-first: diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..9167594 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Maintainer packaging helpers.""" diff --git a/scripts/build_package.py b/scripts/build_package.py new file mode 100644 index 0000000..391344d --- /dev/null +++ b/scripts/build_package.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Stage Truffile's release-only Python packages from the pyfw source tree.""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +REQUIRED_PACKAGE_FILES = ( + Path("truffile/app_runtime/__init__.py"), + Path("truffle/app/app_runtime_pb2.py"), +) + + +def missing_package_inputs(repo_root: Path = REPO_ROOT) -> list[Path]: + return [path for path in REQUIRED_PACKAGE_FILES if not (repo_root / path).is_file()] + + +def require_package_inputs(repo_root: Path = REPO_ROOT) -> None: + missing = missing_package_inputs(repo_root) + if not missing: + return + paths = ", ".join(str(path) for path in missing) + raise RuntimeError( + f"release package inputs are missing: {paths}. " + "Run `python3.12 scripts/build_package.py --pyfw-path /path/to/pyfw` first." + ) + + +def resolve_pyfw_path(raw_path: str | None) -> Path: + value = raw_path or os.environ.get("PYFW_PATH", "") + if not value: + raise RuntimeError("provide --pyfw-path or export PYFW_PATH") + pyfw_path = Path(value).expanduser().resolve() + runtime_init = pyfw_path / "python" / "app_runtime" / "__init__.py" + if not runtime_init.is_file(): + raise RuntimeError(f"pyfw checkout is missing: {runtime_init}") + return pyfw_path + + +def build_protos(pyfw_path: Path) -> None: + build_script = pyfw_path / "python" / "tools" / "build_protos.py" + if not build_script.is_file(): + raise RuntimeError(f"pyfw proto build script is missing: {build_script}") + subprocess.run([sys.executable, str(build_script)], cwd=pyfw_path, check=True) + + +def _replace_tree(source: Path, destination: Path) -> None: + if destination.exists(): + shutil.rmtree(destination) + shutil.copytree(source, destination, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + + +def stage_package_inputs(pyfw_path: Path, repo_root: Path = REPO_ROOT) -> None: + app_runtime = pyfw_path / "python" / "app_runtime" + truffle_protos = pyfw_path / "python" / "truffle" + required_sources = ( + app_runtime / "__init__.py", + truffle_protos / "app" / "app_runtime_pb2.py", + ) + missing = [path for path in required_sources if not path.is_file()] + if missing: + raise RuntimeError(f"pyfw checkout is missing: {', '.join(str(path) for path in missing)}") + + _replace_tree(app_runtime, repo_root / "truffile" / "app_runtime") + _replace_tree(truffle_protos, repo_root / "truffle") + require_package_inputs(repo_root) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Stage app_runtime and generated Truffle protos from their pyfw source-of-truth" + ) + parser.add_argument("--pyfw-path", default=None, help="path to a pyfw checkout") + parser.add_argument( + "--skip-proto-build", + action="store_true", + dest="skip_proto_build", + help="use the protos already generated in pyfw", + ) + args = parser.parse_args(argv) + + try: + pyfw_path = resolve_pyfw_path(args.pyfw_path) + if not args.skip_proto_build: + build_protos(pyfw_path) + stage_package_inputs(pyfw_path) + except (OSError, RuntimeError, subprocess.CalledProcessError) as exc: + parser.exit(1, f"error: {exc}\n") + + print(f"Staged release packages from {pyfw_path}") + print("Next: python3.12 -m build && python3.12 scripts/verify_wheel.py dist/*.whl") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_wheel.py b/scripts/verify_wheel.py new file mode 100644 index 0000000..db004b7 --- /dev/null +++ b/scripts/verify_wheel.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Verify the packages and imports required by a released Truffile wheel.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path + + +REQUIRED_WHEEL_FILES = ( + "truffile/sdk.py", + "truffile/app_runtime/__init__.py", + "truffile/app_runtime/foreground.py", + "truffile/app_runtime/background/runtime.py", + "truffle/app/app_pb2.py", + "truffle/app/app_runtime_pb2.py", + "truffle/app/background_pb2.py", + "truffle/os/app_queries_pb2.py", + "truffle/os/builder_pb2.py", + "truffle/os/client_session_pb2.py", + "truffle/os/truffleos_pb2_grpc.py", +) + + +def missing_wheel_files(wheel_path: Path) -> list[str]: + with zipfile.ZipFile(wheel_path) as archive: + names = set(archive.namelist()) + return [path for path in REQUIRED_WHEEL_FILES if path not in names] + + +def _venv_python(venv_dir: Path) -> Path: + return venv_dir / ("Scripts/python.exe" if sys.platform == "win32" else "bin/python") + + +def verify_installed_wheel(wheel_path: Path) -> None: + wheel_path = wheel_path.resolve() + with tempfile.TemporaryDirectory(prefix="truffile-wheel-") as temp_dir: + temp = Path(temp_dir) + venv_dir = temp / ".venv" + subprocess.run([sys.executable, "-m", "venv", str(venv_dir)], check=True) + python = _venv_python(venv_dir) + subprocess.run( + [ + str(python), + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--no-input", + str(wheel_path), + ], + check=True, + ) + smoke_test = """ +import importlib.util +import tempfile +from pathlib import Path +from types import SimpleNamespace + +from truffile.app_runtime import BackgroundWorkerApp, ForegroundApp +from truffile.client import TruffleClient +from truffile.cli.create import cmd_create +from truffile.schedule import parse_runtime_policy +from truffile.sdk import AppHarness + +with tempfile.TemporaryDirectory() as root: + result = cmd_create(SimpleNamespace(name="wheel-smoke", path=root)) + if result != 0: + raise SystemExit(result) + entrypoint = Path(root) / "wheel-smoke" / "wheel_smoke_foreground.py" + spec = importlib.util.spec_from_file_location("wheel_smoke_foreground", entrypoint) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + background = Path(root) / "wheel-smoke" / "wheel_smoke_background.py" + spec = importlib.util.spec_from_file_location("wheel_smoke_background", background) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) +""" + subprocess.run([str(python), "-c", smoke_test], check=True) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="verify a built Truffile wheel in a fresh environment") + parser.add_argument("wheel", type=Path) + args = parser.parse_args(argv) + wheel_path = args.wheel.expanduser().resolve() + if not wheel_path.is_file(): + parser.error(f"wheel does not exist: {wheel_path}") + + missing = missing_wheel_files(wheel_path) + if missing: + parser.exit(1, f"error: wheel is missing: {', '.join(missing)}\n") + verify_installed_wheel(wheel_path) + print(f"Verified {wheel_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..38e9c83 --- /dev/null +++ b/setup.py @@ -0,0 +1,38 @@ +from pathlib import Path + +from setuptools import setup +from setuptools.command.build_py import build_py +from setuptools.command.sdist import sdist +from setuptools.errors import SetupError + + +REPO_ROOT = Path(__file__).resolve().parent +REQUIRED_PACKAGE_FILES = ( + Path("truffile/app_runtime/__init__.py"), + Path("truffle/app/app_runtime_pb2.py"), +) + + +def require_package_inputs() -> None: + missing = [path for path in REQUIRED_PACKAGE_FILES if not (REPO_ROOT / path).is_file()] + if missing: + paths = ", ".join(str(path) for path in missing) + raise SetupError( + f"release package inputs are missing: {paths}. " + "Run `python3.12 scripts/build_package.py --pyfw-path /path/to/pyfw` first." + ) + + +class VerifiedBuildPy(build_py): + def run(self) -> None: + require_package_inputs() + super().run() + + +class VerifiedSdist(sdist): + def run(self) -> None: + require_package_inputs() + super().run() + + +setup(cmdclass={"build_py": VerifiedBuildPy, "sdist": VerifiedSdist}) diff --git a/tests/test_packaging.py b/tests/test_packaging.py new file mode 100644 index 0000000..332beaf --- /dev/null +++ b/tests/test_packaging.py @@ -0,0 +1,77 @@ +import tempfile +import unittest +import zipfile +from pathlib import Path +from runpy import run_path +from unittest.mock import patch + +from setuptools import Distribution +from setuptools.errors import SetupError + +from scripts.build_package import missing_package_inputs, require_package_inputs, stage_package_inputs +from scripts.verify_wheel import REQUIRED_WHEEL_FILES, missing_wheel_files + + +class TestPackageInputs(unittest.TestCase): + def test_missing_inputs_are_reported(self): + with tempfile.TemporaryDirectory() as tmp: + repo_root = Path(tmp) + self.assertEqual(len(missing_package_inputs(repo_root)), 2) + with self.assertRaisesRegex(RuntimeError, "build_package.py"): + require_package_inputs(repo_root) + + def test_inputs_are_staged_from_pyfw(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + pyfw = root / "pyfw" + repo = root / "truffile-repo" + runtime_init = pyfw / "python" / "app_runtime" / "__init__.py" + proto = pyfw / "python" / "truffle" / "app" / "app_runtime_pb2.py" + runtime_init.parent.mkdir(parents=True) + proto.parent.mkdir(parents=True) + runtime_init.write_text("RUNTIME = True\n", encoding="utf-8") + proto.write_text("PROTO = True\n", encoding="utf-8") + + stage_package_inputs(pyfw, repo) + + self.assertEqual(missing_package_inputs(repo), []) + self.assertEqual( + (repo / "truffile" / "app_runtime" / "__init__.py").read_text(encoding="utf-8"), + "RUNTIME = True\n", + ) + self.assertEqual( + (repo / "truffle" / "app" / "app_runtime_pb2.py").read_text(encoding="utf-8"), + "PROTO = True\n", + ) + + def test_setuptools_build_rejects_missing_inputs(self): + setup_path = Path(__file__).resolve().parents[1] / "setup.py" + with patch("setuptools.setup") as setup, tempfile.TemporaryDirectory() as tmp: + run_path(str(setup_path)) + verified_build = setup.call_args.kwargs["cmdclass"]["build_py"] + verified_build.run.__globals__["REPO_ROOT"] = Path(tmp) + with self.assertRaisesRegex(SetupError, "build_package.py"): + verified_build(Distribution()).run() + + +class TestWheelContract(unittest.TestCase): + def test_missing_wheel_packages_are_reported(self): + with tempfile.TemporaryDirectory() as tmp: + wheel = Path(tmp) / "truffile.whl" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("truffile/sdk.py", "") + missing = missing_wheel_files(wheel) + self.assertIn("truffile/app_runtime/__init__.py", missing) + self.assertIn("truffle/app/app_runtime_pb2.py", missing) + + def test_archive_contract_accepts_all_required_paths(self): + with tempfile.TemporaryDirectory() as tmp: + wheel = Path(tmp) / "truffile.whl" + with zipfile.ZipFile(wheel, "w") as archive: + for path in REQUIRED_WHEEL_FILES: + archive.writestr(path, "") + self.assertEqual(missing_wheel_files(wheel), []) + + +if __name__ == "__main__": + unittest.main() From 09a2eff97ae9e76c24a3d4f8cda8ec34b123ba67 Mon Sep 17 00:00:00 2001 From: temp-deepshard <304551954+temp-deepshard@users.noreply.github.com> Date: Wed, 15 Jul 2026 22:27:53 -0700 Subject: [PATCH 2/4] Run packaging contract in CI --- .github/workflows/packaging-contract.yml | 27 ++++++++++++++++++++++++ tests/test_packaging.py | 14 ++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 .github/workflows/packaging-contract.yml diff --git a/.github/workflows/packaging-contract.yml b/.github/workflows/packaging-contract.yml new file mode 100644 index 0000000..9ccde11 --- /dev/null +++ b/.github/workflows/packaging-contract.yml @@ -0,0 +1,27 @@ +name: Packaging contract + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + required-wheel-packages: + runs-on: ubuntu-latest + steps: + - name: Check out source + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + + - name: Set up Python 3.12 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + + - name: Install contract-test dependencies + run: python -m pip install "pytest>=9.0.2" "setuptools>=70" wheel + + - name: Enforce required wheel packages + run: python -m pytest tests/test_packaging.py diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 332beaf..f400163 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -64,6 +64,20 @@ def test_missing_wheel_packages_are_reported(self): self.assertIn("truffile/app_runtime/__init__.py", missing) self.assertIn("truffle/app/app_runtime_pb2.py", missing) + def test_each_required_package_is_enforced_independently(self): + package_roots = ( + "truffile/app_runtime/__init__.py", + "truffle/app/app_runtime_pb2.py", + ) + for omitted in package_roots: + with self.subTest(omitted=omitted), tempfile.TemporaryDirectory() as tmp: + wheel = Path(tmp) / "truffile.whl" + with zipfile.ZipFile(wheel, "w") as archive: + for path in REQUIRED_WHEEL_FILES: + if path != omitted: + archive.writestr(path, "") + self.assertEqual(missing_wheel_files(wheel), [omitted]) + def test_archive_contract_accepts_all_required_paths(self): with tempfile.TemporaryDirectory() as tmp: wheel = Path(tmp) / "truffile.whl" From 37ea1a2f6239c4d659ebb6a14ac1b144e56eb79c Mon Sep 17 00:00:00 2001 From: temp-deepshard <304551954+temp-deepshard@users.noreply.github.com> Date: Wed, 15 Jul 2026 22:43:45 -0700 Subject: [PATCH 3/4] Document the release build bootstrap --- .github/workflows/packaging-contract.yml | 2 +- README.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/packaging-contract.yml b/.github/workflows/packaging-contract.yml index 9ccde11..d7ccdff 100644 --- a/.github/workflows/packaging-contract.yml +++ b/.github/workflows/packaging-contract.yml @@ -21,7 +21,7 @@ jobs: python-version: "3.12" - name: Install contract-test dependencies - run: python -m pip install "pytest>=9.0.2" "setuptools>=70" wheel + run: python -m pip install "pytest>=9.0.2" "setuptools>=70" wheel build - name: Enforce required wheel packages run: python -m pytest tests/test_packaging.py diff --git a/README.md b/README.md index db5ec39..f5e974d 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,7 @@ not become a second source of truth. From a clean checkout, stage both packages and verify the finished wheel: ```bash +python3.12 -m pip install --upgrade build python3.12 scripts/build_package.py --pyfw-path /path/to/pyfw python3.12 -m build python3.12 scripts/verify_wheel.py dist/*.whl From 2e8a0875e81145d52bbafb55ea3a027d1a2bf1c3 Mon Sep 17 00:00:00 2001 From: temp-deepshard <304551954+temp-deepshard@users.noreply.github.com> Date: Wed, 15 Jul 2026 23:24:13 -0700 Subject: [PATCH 4/4] Pin the release protobuf toolchain --- README.md | 2 +- pyproject.toml | 6 +++--- scripts/build_package.py | 23 +++++++++++++++++++++++ tests/test_packaging.py | 24 +++++++++++++++++++++++- 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f5e974d..863e849 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ not become a second source of truth. From a clean checkout, stage both packages and verify the finished wheel: ```bash -python3.12 -m pip install --upgrade build +python3.12 -m pip install --upgrade build "grpcio-tools==1.82.1" "protobuf==7.35.1" python3.12 scripts/build_package.py --pyfw-path /path/to/pyfw python3.12 -m build python3.12 scripts/verify_wheel.py dist/*.whl diff --git a/pyproject.toml b/pyproject.toml index 310ae49..742e69a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,10 +12,10 @@ license = {text = "MIT"} dependencies = [ "aiohttp>=3.9.0", - "protobuf>=6.30.0", + "protobuf>=7.35.1,<8.0.0", "googleapis-common-protos>=1.63.2", - "grpcio==1.76.0", - "grpcio-reflection==1.76.0", + "grpcio==1.82.1", + "grpcio-reflection==1.82.1", "httpx>=0.27.0", "pyyaml>=6.0", "platformdirs>=3.10.0", diff --git a/scripts/build_package.py b/scripts/build_package.py index 391344d..c6edad9 100644 --- a/scripts/build_package.py +++ b/scripts/build_package.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +from importlib import metadata import os import shutil import subprocess @@ -16,6 +17,10 @@ Path("truffile/app_runtime/__init__.py"), Path("truffle/app/app_runtime_pb2.py"), ) +REQUIRED_PROTO_TOOLCHAIN = ( + ("grpcio-tools", "1.82.1"), + ("protobuf", "7.35.1"), +) def missing_package_inputs(repo_root: Path = REPO_ROOT) -> list[Path]: @@ -33,6 +38,23 @@ def require_package_inputs(repo_root: Path = REPO_ROOT) -> None: ) +def require_proto_toolchain() -> None: + mismatches: list[str] = [] + for package, expected in REQUIRED_PROTO_TOOLCHAIN: + try: + actual = metadata.version(package) + except metadata.PackageNotFoundError: + actual = "missing" + if actual != expected: + mismatches.append(f"{package}=={expected} (found {actual})") + if mismatches: + requirements = " ".join(f'"{package}=={version}"' for package, version in REQUIRED_PROTO_TOOLCHAIN) + raise RuntimeError( + f"release protobuf toolchain mismatch: {', '.join(mismatches)}. " + f"Run `python3.12 -m pip install --upgrade {requirements}`." + ) + + def resolve_pyfw_path(raw_path: str | None) -> Path: value = raw_path or os.environ.get("PYFW_PATH", "") if not value: @@ -89,6 +111,7 @@ def main(argv: list[str] | None = None) -> int: try: pyfw_path = resolve_pyfw_path(args.pyfw_path) if not args.skip_proto_build: + require_proto_toolchain() build_protos(pyfw_path) stage_package_inputs(pyfw_path) except (OSError, RuntimeError, subprocess.CalledProcessError) as exc: diff --git a/tests/test_packaging.py b/tests/test_packaging.py index f400163..3f3df94 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -8,11 +8,33 @@ from setuptools import Distribution from setuptools.errors import SetupError -from scripts.build_package import missing_package_inputs, require_package_inputs, stage_package_inputs +from scripts.build_package import ( + missing_package_inputs, + require_package_inputs, + require_proto_toolchain, + stage_package_inputs, +) from scripts.verify_wheel import REQUIRED_WHEEL_FILES, missing_wheel_files class TestPackageInputs(unittest.TestCase): + @patch("scripts.build_package.metadata.version") + def test_release_proto_toolchain_is_pinned(self, version): + version.side_effect = lambda package: { + "grpcio-tools": "1.82.1", + "protobuf": "7.35.1", + }[package] + require_proto_toolchain() + + @patch("scripts.build_package.metadata.version") + def test_release_proto_toolchain_rejects_drift(self, version): + version.side_effect = lambda package: { + "grpcio-tools": "1.82.1", + "protobuf": "6.33.6", + }[package] + with self.assertRaisesRegex(RuntimeError, "protobuf==7.35.1"): + require_proto_toolchain() + def test_missing_inputs_are_reported(self): with tempfile.TemporaryDirectory() as tmp: repo_root = Path(tmp)