diff --git a/.github/workflows/quality.yaml b/.github/workflows/quality.yaml index 6b9806ec..229c9e88 100644 --- a/.github/workflows/quality.yaml +++ b/.github/workflows/quality.yaml @@ -40,7 +40,23 @@ jobs: install-args: --only main,lint,test - name: Run quality hooks - run: poetry run pre-commit run --all-files + run: poetry run -- pre-commit run --all-files - name: Run tests - run: xvfb-run -a -s "-screen 0 1920x1080x24 -nolisten tcp" poetry run python -m pytest tests + run: xvfb-run -a -s "-screen 0 1920x1080x24 -dpi 96 -nolisten tcp" poetry run -- python3 -m pytest tests + + - name: Run visual regression tests + run: | + visual_jobs=$(( $(nproc) - 1 )) + visual_jobs=$(( visual_jobs > 0 ? visual_jobs : 1 )) + poetry run -- python3 scripts/visual_tests.py reference-compare --jobs "$visual_jobs" --xvfb + + - name: Upload visual regression output + if: failure() + uses: actions/upload-artifact@v4 + with: + name: visual-regression-output + path: | + tests/integration/output/**/*.png + tests/integration/reference/**/*.png + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index b6f9843f..deb99a27 100644 --- a/.gitignore +++ b/.gitignore @@ -51,7 +51,7 @@ coverage.xml .pytest_cache/ cover/ tests/integration/output/ -tests/integration/reference/ +tests/integration/local-reference/ # Translations *.mo @@ -94,4 +94,4 @@ assets/packaging/ios/carveracontroller-ios/carveracontrollerpkg/* # Libraries for hidapi support that would normally be packaged carveracontroller/hidapi.dll -carveracontroller/libhidapi.dylib \ No newline at end of file +carveracontroller/libhidapi.dylib diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8f0ec344..847b1652 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,13 +31,18 @@ Thank you for your interest in contributing to our project! We welcome contribut Run the configured hooks before opening a pull request: ```bash -poetry run pre-commit run --all-files +poetry run -- pre-commit run --all-files ``` Run the test suite separately: ```bash -poetry run python -m pytest tests -q +poetry run -- python3 -m pytest tests -q ``` -For targeted checks, run the individual Poetry commands listed in the [Quality Checks section in the README](README.md#quality-checks). +For screenshot visual tests, use local ignored references while iterating and committed references for Linux checks. +See [README visual tests](README.md#visual-regression-tests) for commands, including the CI-like container helper for +macOS and Windows. + +For targeted checks, run the individual Poetry commands listed in the +[Quality Checks section in the README](README.md#quality-checks). diff --git a/README.md b/README.md index 9fbca286..edb8c7e5 100644 --- a/README.md +++ b/README.md @@ -90,14 +90,14 @@ Once you have Poetry installed, setting up the development environment is straig poetry env activate ``` - This step is usually not necessary since `poetry run ` automatically uses the virtual environment, but it can be helpful if you want to run multiple commands without prefixing `poetry run`. + This step is usually not necessary since `poetry run -- ` automatically uses the virtual environment, but it can be helpful if you want to run multiple commands without prefixing `poetry run --`. ### Running the Project You can run the Controller software using Poetry's run command without installation. This is handy for iterative development. ```bash -poetry run python -m carveracontroller +poetry run -- python3 -m carveracontroller ``` To run the iOS app, you first need to build its dependencies using the Local Packaging instructions below. The build script will open Xcode for you, or you can open the project manually by finding it in `assets/packaging/ios/carveracontroller-ios`. @@ -109,38 +109,92 @@ The project uses [Ruff](https://docs.astral.sh/ruff/) (linting), [Mypy](https:// Run all configured hooks with pre-commit: ```bash -poetry run pre-commit run --all-files +poetry run -- pre-commit run --all-files ``` To run the same checks automatically on every `git commit`, install the hooks once: ```bash -poetry run pre-commit install +poetry run -- pre-commit install ``` Run individual checks with Poetry: ```bash -poetry run ruff check carveracontroller tests scripts -poetry run ruff format --check carveracontroller tests scripts -poetry run mypy carveracontroller --config-file pyproject.toml -poetry run mypy carveracontroller/machine --config-file pyproject.toml --strict -poetry run lint-imports --config pyproject.toml -poetry run python -m pytest tests -q +poetry run -- ruff check carveracontroller tests scripts +poetry run -- ruff format --check carveracontroller tests scripts +poetry run -- mypy carveracontroller --config-file pyproject.toml +poetry run -- mypy carveracontroller/machine --config-file pyproject.toml --strict +poetry run -- lint-imports --config pyproject.toml +poetry run -- python3 -m pytest tests -q ``` To format checked Python files, run: ```bash -poetry run ruff format carveracontroller tests scripts +poetry run -- ruff format carveracontroller tests scripts ``` +### Visual Regression Tests + +The screenshot tests support two reference modes: + +* Local mode uses ignored host-local references in `tests/integration/local-reference/`, grouped by test file. +* Committed mode uses tracked Linux references in `tests/integration/reference/`, grouped by test file. + +For fast local UI iteration, create host-local references before changing the UI: + +```bash +poetry run -- python3 scripts/visual_tests.py local-update +``` + +Then compare against those local references: + +```bash +poetry run -- python3 scripts/visual_tests.py local-compare +``` + +Local comparisons skip any screenshot that is missing from `tests/integration/local-reference/`; they do not fall back +to committed Linux references. Extra pytest arguments can be passed after the script command, for example: + +```bash +poetry run -- python3 scripts/visual_tests.py local-compare \ + tests/integration/test_visual_regression.py::TestDisconnectedState::test_control_page +``` + +On Linux, update or check the committed references directly: + +```bash +poetry run -- python3 scripts/visual_tests.py reference-update --xvfb +poetry run -- python3 scripts/visual_tests.py reference-compare --xvfb +``` + +Reference compare/update can run isolated screenshot tests in parallel: + +```bash +visual_jobs=$(( $(nproc) - 1 )) +visual_jobs=$(( visual_jobs > 0 ? visual_jobs : 1 )) +poetry run -- python3 scripts/visual_tests.py reference-compare --jobs "$visual_jobs" --xvfb +``` + +On macOS or Windows, run the same command inside the CI-like dependency container: + +```bash +poetry run -- python3 scripts/run_in_container.py -- \ + sh -c 'visual_jobs=$(( $(nproc) - 1 )); visual_jobs=$(( visual_jobs > 0 ? visual_jobs : 1 )); poetry run -- python3 scripts/visual_tests.py reference-compare --jobs "$visual_jobs" --xvfb' +``` + +The container helper uses Podman if available, then Docker. Override that with `CARVERA_CI_ENGINE=docker` or +`--engine docker`. The image tag can be overridden with `CARVERA_CI_IMAGE` or `--image`. The helper passes a native +container platform by default, such as `linux/arm64` on Apple Silicon and `linux/amd64` on typical CI runners; +override it with `CARVERA_CI_PLATFORM` or `--platform` if your container engine uses a remote architecture. + ### Local Packaging The application is packaged using PyInstaller (except for iOS). This tool converts Python applications into a standalone executable, so it can be run on systems without requiring management of a installed Python interpreter or dependent libraries. An build helper script is configured with Poetry and can be run with: ```bash -poetry run python scripts/build.py --os os --version version [--no-appimage] +poetry run -- python3 scripts/build.py --os os --version version [--no-appimage] ``` The options for `os` are windows, macos, linux, pypi, ios or android. If selecting `linux`, an appimage is built by default unless --no-appimage is specified. @@ -157,7 +211,7 @@ If you add or modify any UI text strings you need to update the messages.pot fil Updating the .pot and .po strings, as well as compiling to .mo can be performed by running the following command: ``` bash -poetry run python scripts/update_translations.py +poetry run -- python3 scripts/update_translations.py ``` This utility scans the python and kivvy code for new strings and updates the mapping files. It does not clear/overwrite previous translations. diff --git a/carveracontroller/Objloader.py b/carveracontroller/Objloader.py index 4df3a72c..d1471e31 100755 --- a/carveracontroller/Objloader.py +++ b/carveracontroller/Objloader.py @@ -103,49 +103,50 @@ def __init__(self, filename, swapyz=False): self._current_object = None material = None - for line in open(filename): - if line.startswith("#"): - continue - if line.startswith("s"): - continue - values = line.split() - if not values: - continue - if values[0] == "o": - self.finish_object() - self._current_object = values[1] - # elif values[0] == 'mtllib': - # self.mtl = MTL(values[1]) - # elif values[0] in ('usemtl', 'usemat'): - # material = values[1] - if values[0] == "v": - v = list(map(float, values[1:4])) - if swapyz: - v = v[0], v[2], v[1] - self.vertices.append(v) - elif values[0] == "vn": - v = list(map(float, values[1:4])) - if swapyz: - v = v[0], v[2], v[1] - self.normals.append(v) - elif values[0] == "vt": - self.texcoords.append(list(map(float, values[1:3]))) - elif values[0] == "f": - face = [] - texcoords = [] - norms = [] - for v in values[1:]: - w = v.split("/") - face.append(int(w[0])) - if len(w) >= 2 and len(w[1]) > 0: - texcoords.append(int(w[1])) - else: - texcoords.append(-1) - if len(w) >= 3 and len(w[2]) > 0: - norms.append(int(w[2])) - else: - norms.append(-1) - self.faces.append((face, norms, texcoords, material)) + with open(filename, encoding="utf-8") as obj_file: + for line in obj_file: + if line.startswith("#"): + continue + if line.startswith("s"): + continue + values = line.split() + if not values: + continue + if values[0] == "o": + self.finish_object() + self._current_object = values[1] + # elif values[0] == 'mtllib': + # self.mtl = MTL(values[1]) + # elif values[0] in ('usemtl', 'usemat'): + # material = values[1] + if values[0] == "v": + v = list(map(float, values[1:4])) + if swapyz: + v = v[0], v[2], v[1] + self.vertices.append(v) + elif values[0] == "vn": + v = list(map(float, values[1:4])) + if swapyz: + v = v[0], v[2], v[1] + self.normals.append(v) + elif values[0] == "vt": + self.texcoords.append(list(map(float, values[1:3]))) + elif values[0] == "f": + face = [] + texcoords = [] + norms = [] + for v in values[1:]: + w = v.split("/") + face.append(int(w[0])) + if len(w) >= 2 and len(w[1]) > 0: + texcoords.append(int(w[1])) + else: + texcoords.append(-1) + if len(w) >= 3 and len(w[2]) > 0: + norms.append(int(w[2])) + else: + norms.append(-1) + self.faces.append((face, norms, texcoords, material)) self.finish_object() diff --git a/carveracontroller/translation.py b/carveracontroller/translation.py index 21dd2e5f..85ceb198 100644 --- a/carveracontroller/translation.py +++ b/carveracontroller/translation.py @@ -78,10 +78,10 @@ def __getattr__(self, name): def init(langname: str | None = None): if langname is None or langname not in LANGS: try: - default_locale = locale.getdefaultlocale() - if default_locale is not None and default_locale[0] is not None: + locale_name = locale.getlocale()[0] + if locale_name is not None: for lang_key in LANGS: - if default_locale[0][0:2] in lang_key: + if locale_name[0:2] in lang_key: langname = lang_key break except: diff --git a/scripts/run.sh b/scripts/run.sh index 7287184a..4194b83e 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -5,4 +5,4 @@ cd "$(dirname "$0")" cd .. poetry sync -poetry run python -m carveracontroller +poetry run -- python3 -m carveracontroller diff --git a/scripts/run_in_container.py b/scripts/run_in_container.py new file mode 100644 index 00000000..729b0ffa --- /dev/null +++ b/scripts/run_in_container.py @@ -0,0 +1,119 @@ +"""Run a repository command inside the CI-like dependency container.""" + +from __future__ import annotations + +import argparse +import os +import platform as platform_module +import shutil +import subprocess +from collections.abc import Sequence +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_IMAGE = "carvera-controller-ci:latest" +CONTAINERFILE_PATH = REPO_ROOT / "tests" / "integration" / "visual" / "Containerfile" + + +def default_container_platform() -> str | None: + machine = platform_module.machine().lower() + if machine in {"arm64", "aarch64"}: + return "linux/arm64" + if machine in {"x86_64", "amd64"}: + return "linux/amd64" + return None + + +def detect_engine(explicit_engine: str | None) -> str: + if explicit_engine: + return explicit_engine + env_engine = os.environ.get("CARVERA_CI_ENGINE") + if env_engine: + return env_engine + for candidate in ("podman", "docker"): + if shutil.which(candidate): + return candidate + raise SystemExit("No container engine found. Install Docker or Podman, or set CARVERA_CI_ENGINE.") + + +def container_build_command(engine: str, image: str, platform: str | None) -> list[str]: + command = [engine, "build"] + if platform: + command.extend(["--platform", platform]) + command.extend(["-f", str(CONTAINERFILE_PATH), "-t", image, str(REPO_ROOT)]) + return command + + +def container_run_command( + engine: str, + image: str, + repo_root: Path, + platform: str | None, + command_args: Sequence[str], + use_host_user: bool, +) -> list[str]: + command = [ + engine, + "run", + "--rm", + "-e", + "HOME=/tmp", + "-e", + "POETRY_VIRTUALENVS_IN_PROJECT=false", + ] + if platform: + command.extend(["--platform", platform]) + command.extend(["-v", f"{repo_root}:/workspace", "-w", "/workspace"]) + if use_host_user and hasattr(os, "getuid") and hasattr(os, "getgid"): + command.extend(["--user", f"{os.getuid()}:{os.getgid()}"]) + command.append(image) + command.extend(command_args) + return command + + +def run_command(command: Sequence[str]) -> int: + return subprocess.run(command, cwd=REPO_ROOT, check=False).returncode + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--engine", choices=("docker", "podman"), default=None) + parser.add_argument("--image", default=None) + parser.add_argument("--platform", default=None, help="Container platform, for example linux/arm64.") + parser.add_argument("--no-build", action="store_true", help="Reuse an already-built CI-like container image.") + parser.add_argument("command", nargs=argparse.REMAINDER, help="Command to run in the container, after --.") + return parser + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + args = build_parser().parse_args(argv) + if args.command and args.command[0] == "--": + args.command = args.command[1:] + if not args.command: + raise SystemExit("Pass the command to run after --.") + return args + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + engine = detect_engine(args.engine) + image = args.image or os.environ.get("CARVERA_CI_IMAGE", DEFAULT_IMAGE) + platform = args.platform or os.environ.get("CARVERA_CI_PLATFORM") or default_container_platform() + if not args.no_build: + build_result = run_command(container_build_command(engine=engine, image=image, platform=platform)) + if build_result != 0: + return build_result + return run_command( + container_run_command( + engine=engine, + image=image, + repo_root=REPO_ROOT, + platform=platform, + command_args=args.command, + use_host_user=os.name != "nt", + ) + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/visual_tests.py b/scripts/visual_tests.py new file mode 100644 index 00000000..38e6c0b0 --- /dev/null +++ b/scripts/visual_tests.py @@ -0,0 +1,268 @@ +"""Run screenshot-based visual regression tests.""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +VISUAL_TEST_GLOB = "test_visual_regression*.py" +XVFB_SERVER_ARGS = "-screen 0 1920x1080x24 -dpi 96 -nolisten tcp" +VISUAL_SUBPROCESS_ENV = { + "KIVY_DPI": "96", + "KIVY_METRICS_DENSITY": "1", + "KIVY_METRICS_FONTSCALE": "1", + "LIBGL_ALWAYS_SOFTWARE": "1", +} + + +def visual_subprocess_env() -> dict[str, str]: + env = os.environ.copy() + env.update(VISUAL_SUBPROCESS_ENV) + return env + + +def visual_test_paths() -> list[str]: + return [ + str(path.relative_to(REPO_ROOT)) + for path in sorted((REPO_ROOT / "tests" / "integration").glob(VISUAL_TEST_GLOB)) + ] + + +def visual_pytest_command(mode: str, update: bool, target: str, extra_pytest_args: Sequence[str]) -> list[str]: + command = [ + sys.executable, + "-m", + "pytest", + target, + "--visual-run", + f"--visual-reference-mode={mode}", + ] + if update: + command.append("--update-references") + command.extend(extra_pytest_args) + return command + + +def split_pytest_selection_and_options(extra_pytest_args: Sequence[str]) -> tuple[list[str], list[str]]: + selection_args: list[str] = [] + option_args: list[str] = [] + value_expected = False + options_with_values = { + "-k", + "-m", + "-W", + "--color", + "--log-level", + "--maxfail", + "--tb", + "--timeout", + } + for arg in extra_pytest_args: + if value_expected: + option_args.append(arg) + value_expected = False + continue + if not arg.startswith("-"): + selection_args.append(arg) + continue + option_args.append(arg) + option_name = arg.split("=", 1)[0] + if option_name in options_with_values and "=" not in arg: + value_expected = True + return selection_args, option_args + + +def collect_visual_tests_command(mode: str, extra_pytest_args: Sequence[str]) -> list[str]: + selection_args, option_args = split_pytest_selection_and_options(extra_pytest_args) + targets = selection_args if selection_args else visual_test_paths() + return [ + sys.executable, + "-m", + "pytest", + *targets, + "--visual-run", + f"--visual-reference-mode={mode}", + "--collect-only", + "-q", + *option_args, + ] + + +def pytest_option_args(extra_pytest_args: Sequence[str]) -> list[str]: + _selection_args, option_args = split_pytest_selection_and_options(extra_pytest_args) + return option_args + + +def collect_visual_nodeids(mode: str, extra_pytest_args: Sequence[str]) -> tuple[int, list[str]]: + result = subprocess.run( + collect_visual_tests_command(mode=mode, extra_pytest_args=extra_pytest_args), + cwd=REPO_ROOT, + env=visual_subprocess_env(), + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + if result.returncode != 0: + print(result.stdout, end="") + return result.returncode, [] + nodeids = [line.strip() for line in result.stdout.splitlines() if "::" in line] + return 0, nodeids + + +def split_pytest_args(argv: Sequence[str]) -> tuple[list[str], list[str]]: + if "--" not in argv: + return list(argv), [] + separator = argv.index("--") + return list(argv[:separator]), list(argv[separator + 1 :]) + + +def run_command(command: Sequence[str]) -> int: + return subprocess.run(command, cwd=REPO_ROOT, env=visual_subprocess_env(), check=False).returncode + + +def isolated_visual_test_command( + mode: str, + update: bool, + nodeid: str, + pytest_args: Sequence[str], + xvfb: bool, +) -> list[str]: + command = visual_pytest_command( + mode=mode, + update=update, + target=nodeid, + extra_pytest_args=pytest_option_args(pytest_args), + ) + if not xvfb: + return command + return ["xvfb-run", "-a", "-s", XVFB_SERVER_ARGS, *command] + + +def run_isolated_visual_nodeid( + mode: str, + update: bool, + nodeid: str, + pytest_args: Sequence[str], + xvfb: bool, +) -> tuple[int, str]: + result = subprocess.run( + isolated_visual_test_command(mode=mode, update=update, nodeid=nodeid, pytest_args=pytest_args, xvfb=xvfb), + cwd=REPO_ROOT, + env=visual_subprocess_env(), + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + return result.returncode, result.stdout + + +def run_visual_tests(mode: str, update: bool, pytest_args: Sequence[str], jobs: int, xvfb: bool) -> int: + collect_result, nodeids = collect_visual_nodeids(mode=mode, extra_pytest_args=pytest_args) + if collect_result != 0: + return collect_result + if not nodeids: + print("No visual regression tests selected.") + return 5 + + jobs = max(1, jobs) + if jobs > 1 and not xvfb: + raise SystemExit("Parallel visual jobs require --xvfb so each worker gets an isolated display.") + + if jobs == 1: + for nodeid in nodeids: + print(f"\n===== {nodeid} =====", flush=True) + result = run_command( + isolated_visual_test_command( + mode=mode, + update=update, + nodeid=nodeid, + pytest_args=pytest_args, + xvfb=xvfb, + ) + ) + if result != 0: + return result + return 0 + + failed = 0 + with ThreadPoolExecutor(max_workers=jobs) as executor: + futures = { + executor.submit(run_isolated_visual_nodeid, mode, update, nodeid, pytest_args, xvfb): nodeid + for nodeid in nodeids + } + for future in as_completed(futures): + nodeid = futures[future] + result, output = future.result() + print(f"\n===== {nodeid} =====", flush=True) + print(output, end="") + if result != 0 and failed == 0: + failed = result + return failed + + +def add_visual_options(subparser: argparse.ArgumentParser) -> None: + subparser.add_argument( + "--jobs", + type=int, + default=1, + help="Number of isolated visual pytest subprocesses to run at once.", + ) + subparser.add_argument( + "--xvfb", + action="store_true", + help="Run each isolated visual pytest subprocess under xvfb-run.", + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + for name, help_text in ( + ("local-update", "Generate ignored host-local references."), + ("local-compare", "Compare against ignored host-local references."), + ("reference-update", "Generate committed references in the current environment."), + ("reference-compare", "Compare against committed references in the current environment."), + ): + command = subparsers.add_parser(name, help=help_text) + add_visual_options(command) + + return parser + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + raw_argv = list(sys.argv[1:] if argv is None else argv) + parser_args, pytest_args = split_pytest_args(raw_argv) + args, unknown_args = build_parser().parse_known_args(parser_args) + args.pytest_args = pytest_args + unknown_args + return args + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + command_config = { + "local-update": ("local", True), + "local-compare": ("local", False), + "reference-update": ("committed", True), + "reference-compare": ("committed", False), + } + mode, update = command_config[args.command] + return run_visual_tests( + mode=mode, + update=update, + pytest_args=args.pytest_args, + jobs=args.jobs, + xvfb=args.xvfb, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8543861b..bca0061e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -9,16 +9,22 @@ import tempfile import threading import time +from contextlib import suppress # Isolate Kivy config to a temp directory so tests don't mutate the user's # real Kivy config. Must be set BEFORE any Kivy import. _kivy_home = tempfile.mkdtemp(prefix="kivy_test_") os.environ["KIVY_HOME"] = _kivy_home +os.environ.setdefault("KIVY_DPI", "96") +os.environ.setdefault("KIVY_METRICS_DENSITY", "1") +os.environ.setdefault("KIVY_METRICS_FONTSCALE", "1") os.environ.setdefault("KIVY_NO_FILELOG", "1") os.environ.setdefault("KIVY_LOG_MODE", "MIXED") os.environ.setdefault("KIVY_NO_CONSOLELOG", "0") +os.environ.setdefault("LIBGL_ALWAYS_SOFTWARE", "1") import pytest +from kivy.animation import Animation from kivy.config import Config from PIL import Image, ImageChops @@ -30,11 +36,49 @@ Config.set("kivy", "pause_on_minimize", "0") Config.set("input", "mouse", "mouse,multitouch_on_demand") +import kivy.uix.widget as kivy_widget from kivy.base import EventLoop from kivy.clock import Clock +from kivy.lang import Builder + +from tests.integration.visual_references import create_visual_reference_config -REFERENCE_DIR = os.path.join(os.path.dirname(__file__), "reference") OUTPUT_DIR = os.path.join(os.path.dirname(__file__), "output") +VISUAL_MAX_CHANNEL_DELTA = 4 +VISUAL_MAX_CHANGED_PIXEL_RATIO = 0.005 + + +def _start_animation_at_end(self, widget): + for prop_name, value in self.animated_properties.items(): + with suppress(Exception): + setattr(widget, prop_name, value) + with suppress(Exception): + self.dispatch("on_complete", widget) + + +Animation.start = _start_animation_at_end + + +def _disable_scrolling_label_marquee(): + from carveracontroller.custom_widgets import ScrollingLabel + + for method_name in ("_start_marquee", "_start_marquee_anim", "_evaluate_fit", "_check_after_shrink"): + + def _noop(self, *_args, **_kwargs): + return None + + _noop.__name__ = method_name + setattr(ScrollingLabel, method_name, _noop) + + +def _safe_widget_destructor(uid, _ref): + if uid not in kivy_widget._widget_destructors: + return + del kivy_widget._widget_destructors[uid] + Builder.unbind_widget(uid) + + +kivy_widget._widget_destructor = _safe_widget_destructor def pump_frames(count=10, sleep=0): @@ -52,6 +96,17 @@ def pump_frames(count=10, sleep=0): Clock.tick() +def force_render_to_backbuffer(): + """Render the current Kivy frame into the back buffer for screenshot capture.""" + from kivy.core.window import Window + + Window.canvas.ask_update() + Builder.sync() + Clock.tick_draw() + Builder.sync() + Window.dispatch("on_draw") + + def apply_machine_state(app): """Push current CNC.vars into the UI widgets and let the UI settle. @@ -60,7 +115,82 @@ def apply_machine_state(app): """ app.root.config_loaded = True app.root.updateStatus() - pump_frames(20, sleep=0.05) # 20 frames * 50ms = ~1 second + freeze_animated_status(app) + pump_frames(5) + freeze_animated_status(app) + + +def show_content_page(app, page_name): + """Switch the main content page and wait for the visual tree to settle.""" + app.root.content.transition.direction = "right" if page_name == "File" else "left" + app.root.content.current = page_name + pump_frames(5) + + +def stabilize_gcode_viewer(app): + """Render loaded gcode from the app's default full-toolpath view.""" + viewer = app.root.gcode_viewer + viewer.set_display_offset(app.root.content.x, app.root.content.y) + app.root.gcode_play_to_end() + viewer.restore_default_view() + viewer._on_frame_tick(0) + viewer.canvas.ask_update() + pump_frames(5) + + +def freeze_animated_status(app): + """Pin cyclic/blinking status widgets to deterministic display states.""" + Clock.unschedule(app.root.blink_state) + Clock.unschedule(app.root.switch_status) + app.root.heartbeat_time = time.time() + app.root.status_index = 0 + app.root.holding = 0 + app.root.pausing = 0 + app.root.waiting = 0 + app.root.tooling = 0 + for slot in getattr(app.root, "control_list", {}).values(): + if isinstance(slot, list) and slot: + slot[0] = 0 + app.root.updateStatus() + + +def stop_residual_marquees(app): + """Reset ScrollingLabel offsets that may have been scheduled before the freeze.""" + from kivy.core.window import Window + + from carveracontroller.custom_widgets import ScrollingLabel + + def reset_in_tree(widget): + if isinstance(widget, ScrollingLabel): + if widget._marquee_anim is not None: + with suppress(Exception): + widget._marquee_anim.cancel(widget) + widget._marquee_anim = None + widget.scroll_x = 0 + for child in widget.children: + reset_in_tree(child) + + reset_in_tree(app.root) + for child in list(Window.children): + reset_in_tree(child) + + +def redraw_gcode_viewer(app): + """Push the loaded gcode viewer to a settled frame before screenshot capture.""" + viewer = getattr(app.root, "gcode_viewer", None) + if viewer is None or viewer.lengths is None or len(viewer.lengths) <= 1: + return + with suppress(Exception): + viewer.display_count = viewer.get_total_distance() + viewer.dynamic_display = False + viewer._scene_dirty = True + viewer._proj_dirty = True + with suppress(Exception): + viewer._on_frame_tick(None) + + +def pytest_sessionfinish(session, exitstatus): + shutil.rmtree(_kivy_home, ignore_errors=True) def load_gcode_file(app, filepath): @@ -98,8 +228,14 @@ def capture_screenshot(app, name): """ from kivy.core.window import Window - os.makedirs(OUTPUT_DIR, exist_ok=True) + freeze_animated_status(app) + stop_residual_marquees(app) + redraw_gcode_viewer(app) + EventLoop.idle() + Clock.tick() + force_render_to_backbuffer() filepath = os.path.join(OUTPUT_DIR, f"{name}.png") + os.makedirs(os.path.dirname(filepath), exist_ok=True) # Window.screenshot inserts a counter: "name.png" -> "name0001.png" actual_path = Window.screenshot(name=filepath) @@ -113,17 +249,22 @@ def capture_screenshot(app, name): return filepath -def compare_screenshots(name): +def save_or_compare_reference(name, visual_reference_config, update_references): + if update_references: + save_reference(name, visual_reference_config) + else: + compare_screenshots(name, visual_reference_config) + + +def compare_screenshots(name, visual_reference_config): """Compare an output screenshot against its reference baseline. Saves a diff image on failure for debugging. """ - ref_path = os.path.join(REFERENCE_DIR, f"{name}.png") + visual_reference_config.skip_if_missing(name) + ref_path = visual_reference_config.reference_path(name) out_path = os.path.join(OUTPUT_DIR, f"{name}.png") - if not os.path.exists(ref_path): - pytest.skip(f"No reference screenshot for '{name}'. Run with --update-references to create one.") - ref = Image.open(ref_path).convert("RGB") out = Image.open(out_path).convert("RGB") @@ -133,16 +274,29 @@ def compare_screenshots(name): bbox = diff.getbbox() if bbox is not None: + max_channel_delta = max(channel_max for _channel_min, channel_max in diff.getextrema()) + changed_pixels = sum(1 for pixel in diff.getdata() if pixel != (0, 0, 0)) + changed_pixel_ratio = changed_pixels / (ref.size[0] * ref.size[1]) + if max_channel_delta <= VISUAL_MAX_CHANNEL_DELTA: + return + diff_path = os.path.join(OUTPUT_DIR, f"{name}_DIFF.png") diff.save(diff_path) - pytest.fail(f"Visual difference detected in '{name}'. Diff region: {bbox}. See {diff_path}") + amplified_diff_path = os.path.join(OUTPUT_DIR, f"{name}_DIFF_X16.png") + diff.point(lambda value: min(value * 16, 255)).save(amplified_diff_path) + changed_percent = changed_pixel_ratio * 100 + pytest.fail( + f"Visual difference detected in '{name}'. Diff region: {bbox}. " + f"Changed pixels: {changed_pixels} ({changed_percent:.3f}%), " + f"max channel delta: {max_channel_delta}. See {diff_path}." + ) -def save_reference(name): +def save_reference(name, visual_reference_config): """Copy an output screenshot to become the new reference baseline.""" - os.makedirs(REFERENCE_DIR, exist_ok=True) src = os.path.join(OUTPUT_DIR, f"{name}.png") - dst = os.path.join(REFERENCE_DIR, f"{name}.png") + dst = visual_reference_config.reference_path(name) + dst.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src, dst) @@ -151,8 +305,53 @@ def pytest_addoption(parser): "--update-references", action="store_true", default=False, - help="Update reference screenshots instead of comparing against them.", + help="Update reference screenshots in the selected visual reference mode.", ) + parser.addoption( + "--visual-reference-mode", + choices=("local", "committed"), + default="local", + help=( + "Reference screenshot source: local uses the ignored host-local directory, " + "committed uses tracked Linux container baselines." + ), + ) + parser.addoption( + "--visual-run", + action="store_true", + default=False, + help="Run visual regression tests. The visual runner sets this while isolating each test process.", + ) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "visual_reference(name): screenshot reference name used for early missing-reference checks.", + ) + + +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_setup(item): + marker = item.get_closest_marker("visual_reference") + if marker is None or not item.config.getoption("--visual-run"): + return + + reference_config = create_visual_reference_config( + item.config.getoption("--visual-reference-mode"), + item.config.getoption("--update-references"), + ) + reference_config.skip_if_missing(marker.args[0]) + + +def pytest_collection_modifyitems(items): + run_visual = items[0].config.getoption("--visual-run") if items else False + skip_visual = pytest.mark.skip(reason="run visual regression tests with scripts/visual_tests.py") + for item in items: + path = item.path + if run_visual or not (path.name.startswith("test_visual_regression") and path.suffix == ".py"): + continue + item.add_marker(skip_visual) @pytest.fixture(scope="session") @@ -160,14 +359,42 @@ def update_references(request): return request.config.getoption("--update-references") +@pytest.fixture(scope="session") +def visual_reference_config(request): + return create_visual_reference_config( + request.config.getoption("--visual-reference-mode"), + request.config.getoption("--update-references"), + ) + + +@pytest.fixture(autouse=True) +def disable_modal_animations(monkeypatch): + """Make popup screenshots deterministic by disabling ModalView fades.""" + from kivy.uix.modalview import ModalView + + original_open = ModalView.open + original_dismiss = ModalView.dismiss + + def open_without_animation(self, *_args, **kwargs): + kwargs["animation"] = False + return original_open(self, *_args, **kwargs) + + def dismiss_without_animation(self, *_args, **kwargs): + kwargs["animation"] = False + return original_dismiss(self, *_args, **kwargs) + + monkeypatch.setattr(ModalView, "open", open_without_animation) + monkeypatch.setattr(ModalView, "dismiss", dismiss_without_animation) + + @pytest.fixture(scope="session") def kivy_app(): """Boot the full MakeraApp with mocked hardware. - This mirrors the startup sequence in main.py:main() but avoids calling - app.run(), which would block forever in the Kivy event loop. Instead we - use _run_prepare() to build the widget tree and manually pump frames. + The visual runner invokes pytest once per screenshot test, so this + session-scoped fixture creates one app instance per isolated process. """ + import carveracontroller.main as main_module from carveracontroller import translation from carveracontroller.main import ( @@ -199,27 +426,21 @@ def kivy_app(): register_fonts(base_path) register_images(base_path) - # Create the app and build its widget tree without entering the event loop EventLoop.ensure_window() + _disable_scrolling_label_marquee() app = MakeraApp() app._run_prepare() - # Disable screen transition animations so page switches are instant from kivy.uix.screenmanager import NoTransition app.root.content.transition = NoTransition() app.root.cmd_manager.transition = NoTransition() - # Let the UI settle with real elapsed time so timed callbacks complete - # (blink_state @ 0.5s, viewport updates @ 0.25s, etc.) - pump_frames(60, sleep=0.05) # 60 frames * 50ms = ~3 seconds + pump_frames(10) + freeze_animated_status(app) yield app - # Teardown: stop background threads and close the event loop - app.root.stop.set() # signals monitorSerial to exit + app.root.stop.set() app.stop() EventLoop.close() - - # Clean up temp Kivy home - shutil.rmtree(_kivy_home, ignore_errors=True) diff --git a/tests/integration/reference/test_visual_regression/alarm_control_page.png b/tests/integration/reference/test_visual_regression/alarm_control_page.png new file mode 100644 index 00000000..91b4f5e6 Binary files /dev/null and b/tests/integration/reference/test_visual_regression/alarm_control_page.png differ diff --git a/tests/integration/reference/test_visual_regression/change_tool_popup.png b/tests/integration/reference/test_visual_regression/change_tool_popup.png new file mode 100644 index 00000000..44c8217f Binary files /dev/null and b/tests/integration/reference/test_visual_regression/change_tool_popup.png differ diff --git a/tests/integration/reference/test_visual_regression/connected_idle_control_page.png b/tests/integration/reference/test_visual_regression/connected_idle_control_page.png new file mode 100644 index 00000000..3de44994 Binary files /dev/null and b/tests/integration/reference/test_visual_regression/connected_idle_control_page.png differ diff --git a/tests/integration/reference/test_visual_regression/connected_idle_file_page.png b/tests/integration/reference/test_visual_regression/connected_idle_file_page.png new file mode 100644 index 00000000..f06aadf5 Binary files /dev/null and b/tests/integration/reference/test_visual_regression/connected_idle_file_page.png differ diff --git a/tests/integration/reference/test_visual_regression/connected_idle_gcode_loaded.png b/tests/integration/reference/test_visual_regression/connected_idle_gcode_loaded.png new file mode 100644 index 00000000..fd71c6be Binary files /dev/null and b/tests/integration/reference/test_visual_regression/connected_idle_gcode_loaded.png differ diff --git a/tests/integration/reference/test_visual_regression/connected_idle_settings_popup.png b/tests/integration/reference/test_visual_regression/connected_idle_settings_popup.png new file mode 100644 index 00000000..e8a02585 Binary files /dev/null and b/tests/integration/reference/test_visual_regression/connected_idle_settings_popup.png differ diff --git a/tests/integration/reference/test_visual_regression/disconnected_control_page.png b/tests/integration/reference/test_visual_regression/disconnected_control_page.png new file mode 100644 index 00000000..7ba1891b Binary files /dev/null and b/tests/integration/reference/test_visual_regression/disconnected_control_page.png differ diff --git a/tests/integration/reference/test_visual_regression/disconnected_file_page.png b/tests/integration/reference/test_visual_regression/disconnected_file_page.png new file mode 100644 index 00000000..f1abefde Binary files /dev/null and b/tests/integration/reference/test_visual_regression/disconnected_file_page.png differ diff --git a/tests/integration/reference/test_visual_regression/disconnected_settings_popup.png b/tests/integration/reference/test_visual_regression/disconnected_settings_popup.png new file mode 100644 index 00000000..1adb255d Binary files /dev/null and b/tests/integration/reference/test_visual_regression/disconnected_settings_popup.png differ diff --git a/tests/integration/reference/test_visual_regression/move_a_popup.png b/tests/integration/reference/test_visual_regression/move_a_popup.png new file mode 100644 index 00000000..5fc6b4a3 Binary files /dev/null and b/tests/integration/reference/test_visual_regression/move_a_popup.png differ diff --git a/tests/integration/reference/test_visual_regression/set_a_popup.png b/tests/integration/reference/test_visual_regression/set_a_popup.png new file mode 100644 index 00000000..8fa5867d Binary files /dev/null and b/tests/integration/reference/test_visual_regression/set_a_popup.png differ diff --git a/tests/integration/reference/test_visual_regression/set_tool_popup.png b/tests/integration/reference/test_visual_regression/set_tool_popup.png new file mode 100644 index 00000000..20dcddcf Binary files /dev/null and b/tests/integration/reference/test_visual_regression/set_tool_popup.png differ diff --git a/tests/integration/reference/test_visual_regression/set_x_popup.png b/tests/integration/reference/test_visual_regression/set_x_popup.png new file mode 100644 index 00000000..2f13ad5e Binary files /dev/null and b/tests/integration/reference/test_visual_regression/set_x_popup.png differ diff --git a/tests/integration/reference/test_visual_regression/set_y_popup.png b/tests/integration/reference/test_visual_regression/set_y_popup.png new file mode 100644 index 00000000..040cc5a0 Binary files /dev/null and b/tests/integration/reference/test_visual_regression/set_y_popup.png differ diff --git a/tests/integration/reference/test_visual_regression/set_z_popup.png b/tests/integration/reference/test_visual_regression/set_z_popup.png new file mode 100644 index 00000000..ef2eca71 Binary files /dev/null and b/tests/integration/reference/test_visual_regression/set_z_popup.png differ diff --git a/tests/integration/test_visual_regression.py b/tests/integration/test_visual_regression.py index b5ae3b99..c611d413 100644 --- a/tests/integration/test_visual_regression.py +++ b/tests/integration/test_visual_regression.py @@ -5,16 +5,17 @@ changed something visible. Usage: - # First run: capture reference baselines - poetry run python -m pytest tests/integration/test_visual_regression.py --update-references + # Local developer baselines. + poetry run -- python3 scripts/visual_tests.py local-update - # Subsequent runs: compare against baselines - poetry run python -m pytest tests/integration/test_visual_regression.py + # Local comparisons use local baselines and skip tests with missing local references. + poetry run -- python3 scripts/visual_tests.py local-compare """ import json import os +import pytest from kivy.app import App from tests.integration.conftest import ( @@ -24,89 +25,101 @@ load_gcode_file, pump_frames, save_reference, + show_content_page, + stabilize_gcode_viewer, ) _TESTS_DIR = os.path.join(os.path.dirname(__file__), "..") GCODE_FILE = os.path.join(_TESTS_DIR, "resources", "Face 4x4 stock.cnc") CONFIG_C1_PATH = os.path.join(_TESTS_DIR, "..", "carveracontroller", "config_c1.json") +REFERENCE_GROUP = os.path.splitext(os.path.basename(__file__))[0] + + +def reference_name(name): + return f"{REFERENCE_GROUP}/{name}" class TestDisconnectedState: """Screenshots of the app with no machine connected (default boot state).""" - def test_control_page(self, kivy_app, update_references): - name = "disconnected_control_page" - kivy_app.root.content.current = "Control" - pump_frames(10) + @pytest.mark.visual_reference(reference_name("disconnected_control_page")) + def test_control_page(self, kivy_app, update_references, visual_reference_config): + name = reference_name("disconnected_control_page") + show_content_page(kivy_app, "Control") capture_screenshot(kivy_app, name) if update_references: - save_reference(name) + save_reference(name, visual_reference_config) else: - compare_screenshots(name) + compare_screenshots(name, visual_reference_config) - def test_file_page(self, kivy_app, update_references): - name = "disconnected_file_page" - kivy_app.root.content.current = "File" - pump_frames(10) + @pytest.mark.visual_reference(reference_name("disconnected_file_page")) + def test_file_page(self, kivy_app, update_references, visual_reference_config): + name = reference_name("disconnected_file_page") + show_content_page(kivy_app, "File") capture_screenshot(kivy_app, name) if update_references: - save_reference(name) + save_reference(name, visual_reference_config) else: - compare_screenshots(name) + compare_screenshots(name, visual_reference_config) - def test_settings_popup(self, kivy_app, update_references): - name = "disconnected_settings_popup" - kivy_app.root.content.current = "Control" + @pytest.mark.visual_reference(reference_name("disconnected_settings_popup")) + def test_settings_popup(self, kivy_app, update_references, visual_reference_config): + name = reference_name("disconnected_settings_popup") + show_content_page(kivy_app, "Control") kivy_app.root.config_popup.open() pump_frames(20, sleep=0.05) capture_screenshot(kivy_app, name) kivy_app.root.config_popup.dismiss() pump_frames(10, sleep=0.05) if update_references: - save_reference(name) + save_reference(name, visual_reference_config) else: - compare_screenshots(name) + compare_screenshots(name, visual_reference_config) class TestConnectedIdleState: """Screenshots with a simulated connected, idle machine.""" - def test_control_page(self, kivy_app, connected_idle_state, update_references): - name = "connected_idle_control_page" - kivy_app.root.content.current = "Control" + @pytest.mark.visual_reference(reference_name("connected_idle_control_page")) + def test_control_page(self, kivy_app, connected_idle_state, update_references, visual_reference_config): + name = reference_name("connected_idle_control_page") + show_content_page(kivy_app, "Control") apply_machine_state(kivy_app) capture_screenshot(kivy_app, name) if update_references: - save_reference(name) + save_reference(name, visual_reference_config) else: - compare_screenshots(name) + compare_screenshots(name, visual_reference_config) - def test_file_page(self, kivy_app, connected_idle_state, update_references): - name = "connected_idle_file_page" - kivy_app.root.content.current = "File" + @pytest.mark.visual_reference(reference_name("connected_idle_file_page")) + def test_file_page(self, kivy_app, connected_idle_state, update_references, visual_reference_config): + name = reference_name("connected_idle_file_page") + show_content_page(kivy_app, "File") apply_machine_state(kivy_app) capture_screenshot(kivy_app, name) if update_references: - save_reference(name) + save_reference(name, visual_reference_config) else: - compare_screenshots(name) + compare_screenshots(name, visual_reference_config) - def test_gcode_loaded(self, kivy_app, connected_idle_state, update_references): - name = "connected_idle_gcode_loaded" + @pytest.mark.visual_reference(reference_name("connected_idle_gcode_loaded")) + def test_gcode_loaded(self, kivy_app, connected_idle_state, update_references, visual_reference_config): + name = reference_name("connected_idle_gcode_loaded") apply_machine_state(kivy_app) load_gcode_file(kivy_app, GCODE_FILE) - kivy_app.root.content.current = "File" + show_content_page(kivy_app, "File") kivy_app.root.cmd_manager.current = "gcode_cmd_page" - pump_frames(10, sleep=0.05) + stabilize_gcode_viewer(kivy_app) capture_screenshot(kivy_app, name) if update_references: - save_reference(name) + save_reference(name, visual_reference_config) else: - compare_screenshots(name) + compare_screenshots(name, visual_reference_config) - def test_settings_popup(self, kivy_app, connected_idle_state, update_references): - name = "connected_idle_settings_popup" - kivy_app.root.content.current = "Control" + @pytest.mark.visual_reference(reference_name("connected_idle_settings_popup")) + def test_settings_popup(self, kivy_app, connected_idle_state, update_references, visual_reference_config): + name = reference_name("connected_idle_settings_popup") + show_content_page(kivy_app, "Control") App.get_running_app().model = "C1" kivy_app.root.config_loaded = True # Pre-populate setting_list with defaults from config JSON so @@ -124,59 +137,107 @@ def test_settings_popup(self, kivy_app, connected_idle_state, update_references) kivy_app.root.config_popup.dismiss() pump_frames(10, sleep=0.05) if update_references: - save_reference(name) + save_reference(name, visual_reference_config) else: - compare_screenshots(name) + compare_screenshots(name, visual_reference_config) class TestAlarmState: """Screenshots with the machine in alarm state.""" - def test_alarm_control_page(self, kivy_app, alarm_state, update_references): - name = "alarm_control_page" - kivy_app.root.content.current = "Control" + @pytest.mark.visual_reference(reference_name("alarm_control_page")) + def test_alarm_control_page(self, kivy_app, alarm_state, update_references, visual_reference_config): + name = reference_name("alarm_control_page") + show_content_page(kivy_app, "Control") apply_machine_state(kivy_app) capture_screenshot(kivy_app, name) if update_references: - save_reference(name) + save_reference(name, visual_reference_config) else: - compare_screenshots(name) + compare_screenshots(name, visual_reference_config) class TestSetPositionPopups: """Screenshots of the SetX/Y/Z/A, SetTool, ChangeTool, and MoveA popups.""" - def _capture_popup(self, kivy_app, popup, name, update_references): - kivy_app.root.content.current = "Control" + def _capture_popup(self, kivy_app, popup, name, update_references, visual_reference_config): + show_content_page(kivy_app, "Control") popup.open() - pump_frames(20, sleep=0.05) + pump_frames(5) capture_screenshot(kivy_app, name) popup.dismiss() - pump_frames(10, sleep=0.05) + pump_frames(2) if update_references: - save_reference(name) + save_reference(name, visual_reference_config) else: - compare_screenshots(name) + compare_screenshots(name, visual_reference_config) - def test_set_x_popup(self, kivy_app, update_references): - self._capture_popup(kivy_app, kivy_app.root.coord_popup.setx_popup, "set_x_popup", update_references) + @pytest.mark.visual_reference(reference_name("set_x_popup")) + def test_set_x_popup(self, kivy_app, update_references, visual_reference_config): + self._capture_popup( + kivy_app, + kivy_app.root.coord_popup.setx_popup, + reference_name("set_x_popup"), + update_references, + visual_reference_config, + ) - def test_set_y_popup(self, kivy_app, update_references): - self._capture_popup(kivy_app, kivy_app.root.coord_popup.sety_popup, "set_y_popup", update_references) + @pytest.mark.visual_reference(reference_name("set_y_popup")) + def test_set_y_popup(self, kivy_app, update_references, visual_reference_config): + self._capture_popup( + kivy_app, + kivy_app.root.coord_popup.sety_popup, + reference_name("set_y_popup"), + update_references, + visual_reference_config, + ) - def test_set_z_popup(self, kivy_app, update_references): - self._capture_popup(kivy_app, kivy_app.root.coord_popup.setz_popup, "set_z_popup", update_references) + @pytest.mark.visual_reference(reference_name("set_z_popup")) + def test_set_z_popup(self, kivy_app, update_references, visual_reference_config): + self._capture_popup( + kivy_app, + kivy_app.root.coord_popup.setz_popup, + reference_name("set_z_popup"), + update_references, + visual_reference_config, + ) - def test_set_a_popup(self, kivy_app, update_references): - self._capture_popup(kivy_app, kivy_app.root.coord_popup.seta_popup, "set_a_popup", update_references) + @pytest.mark.visual_reference(reference_name("set_a_popup")) + def test_set_a_popup(self, kivy_app, update_references, visual_reference_config): + self._capture_popup( + kivy_app, + kivy_app.root.coord_popup.seta_popup, + reference_name("set_a_popup"), + update_references, + visual_reference_config, + ) - def test_set_tool_popup(self, kivy_app, update_references): - self._capture_popup(kivy_app, kivy_app.root.coord_popup.settool_popup, "set_tool_popup", update_references) + @pytest.mark.visual_reference(reference_name("set_tool_popup")) + def test_set_tool_popup(self, kivy_app, update_references, visual_reference_config): + self._capture_popup( + kivy_app, + kivy_app.root.coord_popup.settool_popup, + reference_name("set_tool_popup"), + update_references, + visual_reference_config, + ) - def test_change_tool_popup(self, kivy_app, update_references): + @pytest.mark.visual_reference(reference_name("change_tool_popup")) + def test_change_tool_popup(self, kivy_app, update_references, visual_reference_config): self._capture_popup( - kivy_app, kivy_app.root.coord_popup.change_tool_popup, "change_tool_popup", update_references + kivy_app, + kivy_app.root.coord_popup.change_tool_popup, + reference_name("change_tool_popup"), + update_references, + visual_reference_config, ) - def test_move_a_popup(self, kivy_app, update_references): - self._capture_popup(kivy_app, kivy_app.root.coord_popup.MoveA_popup, "move_a_popup", update_references) + @pytest.mark.visual_reference(reference_name("move_a_popup")) + def test_move_a_popup(self, kivy_app, update_references, visual_reference_config): + self._capture_popup( + kivy_app, + kivy_app.root.coord_popup.MoveA_popup, + reference_name("move_a_popup"), + update_references, + visual_reference_config, + ) diff --git a/tests/integration/visual/Containerfile b/tests/integration/visual/Containerfile new file mode 100644 index 00000000..994e67ff --- /dev/null +++ b/tests/integration/visual/Containerfile @@ -0,0 +1,36 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive \ + POETRY_HOME=/opt/poetry \ + POETRY_VIRTUALENVS_PATH=/opt/poetry-venvs \ + POETRY_VIRTUALENVS_IN_PROJECT=false \ + PATH="/opt/poetry/bin:${PATH}" \ + PYTHONUNBUFFERED=1 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + fonts-dejavu-core \ + libgl1 \ + libgles2 \ + libmtdev1 \ + libsm6 \ + libxext6 \ + libxrender1 \ + python3.12 \ + python3.12-dev \ + python3.12-venv \ + xvfb \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /tmp/.X11-unix && chmod 1777 /tmp/.X11-unix + +RUN curl -sSL https://install.python-poetry.org | python3.12 - + +WORKDIR /workspace + +COPY pyproject.toml poetry.lock ./ +RUN poetry install --only main,test --no-root + +CMD ["poetry", "run", "--", "python3", "scripts/visual_tests.py", "reference-compare", "--xvfb"] diff --git a/tests/integration/visual_references.py b/tests/integration/visual_references.py new file mode 100644 index 00000000..aa668e79 --- /dev/null +++ b/tests/integration/visual_references.py @@ -0,0 +1,46 @@ +"""Reference screenshot paths and policies for visual regression tests.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +import pytest + +VISUAL_TEST_DIR = Path(__file__).resolve().parent +COMMITTED_REFERENCE_DIR = VISUAL_TEST_DIR / "reference" +LOCAL_REFERENCE_DIR = VISUAL_TEST_DIR / "local-reference" + +VisualReferenceMode = Literal["local", "committed"] + + +@dataclass(frozen=True) +class VisualReferenceConfig: + mode: VisualReferenceMode + reference_dir: Path + update: bool + + def reference_path(self, name: str) -> Path: + return self.reference_dir / f"{name}.png" + + def skip_if_missing(self, name: str) -> None: + if self.update or self.reference_path(name).exists(): + return + if self.mode == "local": + pytest.skip( + f"No local reference screenshot for '{name}'. " + "Run with --update-references --visual-reference-mode=local to create one." + ) + raise FileNotFoundError( + f"No committed reference screenshot for '{name}'. " + "Run the container update command to create committed references." + ) + + +def create_visual_reference_config(mode: str, update: bool) -> VisualReferenceConfig: + if mode == "local": + return VisualReferenceConfig(mode="local", reference_dir=LOCAL_REFERENCE_DIR, update=update) + if mode == "committed": + return VisualReferenceConfig(mode="committed", reference_dir=COMMITTED_REFERENCE_DIR, update=update) + raise ValueError(f"Unknown visual reference mode: {mode}")