From 774a9acad18a12dff9b2af17e8a4b9f4c33c65f7 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Sun, 19 Apr 2026 14:15:58 +0200 Subject: [PATCH 01/18] Add CI/CD pipeline with container-based testing (issue 9393) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce GitHub Actions CI inside a shared Docker image on ghcr.io, plus a native Windows runner for cross-platform unit-test coverage, and a shared unittest harness with Gramps-backed fixtures that verifies every addon registers, loads, and exposes valid plugin metadata. CI infrastructure ----------------- - .github/docker/gramps-ci/Dockerfile — Python 3.12 + Gramps 6.0 (pip) + PyGObject + GTK typelibs + xvfb/xauth + ruff, dbf, intltool, gettext, git. GTK lives in the base so addon modules that do `from gi.repository import Gtk` at load time are importable; xvfb and xauth are bundled for tests that actually render. - .github/workflows/docker-build.yml — rebuilds the image on .github/docker/** changes or via workflow_dispatch. - .github/workflows/ci.yml — seven jobs: lint (ruff E9/F63/F7/F82 + trailing whitespace), addon-structure (every addon has po/template.pot), compile-check (py_compile on every .py), unit-test-linux (container), unit-test-windows (native, conda+pip), integration-test (container with --init so xvfb-run does not hang), build (make.py gramps60 build all). - .github/environment.yml — hybrid conda+pip env for Windows. Gramps is not on conda-forge, so pygobject/gtk3 come from conda and gramps/orjson/dbf come from pip. Shared test harness ------------------- - tests/__init__.py — GPL header. - tests/gramps_test_env.py — sys.path / GRAMPS_RESOURCES bootstrap and two unittest base classes: GrampsTestCase (session-cached plugin manager + registry via setUpClass) and GrampsDbTestCase (same plus a fresh in-memory SQLite DB per test). - tests/test_plugin_registration.py — four unittest.TestCase classes covering plugin registration, subprocess-isolated module loading (crash-safe), required metadata (gramps_target_version=6.0, valid id/name/version), and import/export entry-function smoke tests. Gate policy ----------- All seven jobs run on every push and PR. Four are marked continue-on-error: true so they surface issues without blocking merges while the existing tree is cleaned up: - lint (~79 pre-existing ruff E9/F63/F7/F82 errors) - addon-structure (4 addons missing po/template.pot) - unit-test-linux (some addon test modules fail to import today) - unit-test-windows (same) compile-check, integration-test, and build are blocking from day one. Each non-blocking gate will be flipped to blocking in the same follow-up PR that fixes its underlying issues, so the tightening is incremental and visible in history. Co-Authored-By: Claude Opus 4.7 --- .github/docker/gramps-ci/Dockerfile | 53 ++++++ .github/environment.yml | 12 ++ .github/workflows/ci.yml | 237 ++++++++++++++++++++++++ .github/workflows/docker-build.yml | 52 ++++++ tests/__init__.py | 21 +++ tests/gramps_test_env.py | 178 ++++++++++++++++++ tests/test_plugin_registration.py | 277 ++++++++++++++++++++++++++++ 7 files changed, 830 insertions(+) create mode 100644 .github/docker/gramps-ci/Dockerfile create mode 100644 .github/environment.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/docker-build.yml create mode 100644 tests/__init__.py create mode 100644 tests/gramps_test_env.py create mode 100644 tests/test_plugin_registration.py diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile new file mode 100644 index 000000000..3ef3b8945 --- /dev/null +++ b/.github/docker/gramps-ci/Dockerfile @@ -0,0 +1,53 @@ +# .github/docker/gramps-ci/Dockerfile +# +# Unified Gramps 6.0 CI image. Includes everything jobs need: +# - Python + pip-installed Gramps, PyGObject, pycairo +# - GTK typelibs (so addon modules that `from gi.repository import Gtk` +# at module load time are importable — widgets still need xvfb to render) +# - intltool/gettext/git for make.py builds +# - ruff, dbf for lint/test tooling (tests use stdlib unittest per AGENTS.md) +# - xvfb + xauth for tests that actually render (wrap with `xvfb-run`) +# +# No display server runs by default — the image is headless unless a command +# explicitly invokes `xvfb-run`. When running with docker locally, pass +# `--init` (or use a container runtime that injects tini) because xvfb-run +# hangs if it inherits PID 1. +# +ARG PYTHON_VERSION=3.12 +FROM python:${PYTHON_VERSION}-slim + +LABEL org.opencontainers.image.source="https://github.com/gramps-project/addons-source" +LABEL org.opencontainers.image.description="Unified Gramps 6.0 CI image (Python, Gramps, GTK typelibs, xvfb)" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgirepository-2.0-dev \ + gir1.2-glib-2.0 \ + gir1.2-gtk-3.0 \ + gir1.2-pango-1.0 \ + gir1.2-gdkpixbuf-2.0 \ + gir1.2-atk-1.0 \ + gcc \ + pkg-config \ + python3-dev \ + libcairo2-dev \ + intltool \ + gettext \ + git \ + xvfb \ + xauth \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install --no-cache-dir \ + PyGObject \ + pycairo \ + "gramps>=6.0,<6.1" \ + orjson \ + ruff \ + dbf + +RUN apt-get purge -y gcc python3-dev pkg-config && apt-get autoremove -y + +RUN python -c "from gramps.gen.const import VERSION; print('Gramps', VERSION)" \ + && python -c "import gi; gi.require_version('Gtk', '3.0'); from gi.repository import Gtk; print('GTK OK')" + +WORKDIR /workspace diff --git a/.github/environment.yml b/.github/environment.yml new file mode 100644 index 000000000..9ca19f57f --- /dev/null +++ b/.github/environment.yml @@ -0,0 +1,12 @@ +name: addons-ci +channels: + - conda-forge +dependencies: + - python=3.12 + - pygobject + - gtk3 + - pip + - pip: + - "gramps>=6.0,<6.1" + - orjson + - dbf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..92bd05ff6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,237 @@ +name: CI + +on: + push: + branches: [maintenance/gramps60] + pull_request: + branches: [maintenance/gramps60] + +env: + CI_IMAGE: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + +jobs: + # ----------------------------------------------------------------- + # Lint (ci container) + # ----------------------------------------------------------------- + lint: + name: Lint + runs-on: ubuntu-latest + # Non-blocking until the existing ruff E9/F63/F7/F82 errors across the + # addon set are cleaned up in a follow-up PR. Flip this off in that PR. + continue-on-error: true + container: + image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + steps: + - uses: actions/checkout@v4 + + - name: Run ruff (syntax and import errors only) + run: ruff check --select=E9,F63,F7,F82 --no-fix --exclude='*.gpr.py' . + + - name: Check trailing whitespace in Python files + run: | + if git --no-pager grep --color -n --full-name '[ \t]$' -- '*.py'; then + echo "::error::Trailing whitespace found in Python files" + exit 1 + fi + + # ----------------------------------------------------------------- + # Addon structure (bare runner — just bash, no deps needed) + # ----------------------------------------------------------------- + addon-structure: + name: Addon Structure + runs-on: ubuntu-latest + # Non-blocking until the four addons missing po/template.pot are fixed + # in a follow-up PR. Flip this off in that PR. + continue-on-error: true + steps: + - uses: actions/checkout@v4 + + - name: Check all addons have po/template.pot + run: | + failed=0 + for gpr in */*.gpr.py; do + addon_dir="$(dirname "$gpr")" + if [ ! -d "$addon_dir/po" ]; then + echo "::error::$addon_dir is missing po/ directory" + failed=1 + elif [ ! -f "$addon_dir/po/template.pot" ]; then + echo "::error::$addon_dir is missing po/template.pot" + failed=1 + fi + done + if [ "$failed" -eq 0 ]; then + echo "All addons have po/template.pot" + fi + exit $failed + + # ----------------------------------------------------------------- + # Compile check (ci container) + # ----------------------------------------------------------------- + compile-check: + name: Compile Check + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + steps: + - uses: actions/checkout@v4 + + - name: Compile all Python files (excluding .gpr.py) + shell: bash + run: | + failed=0 + while IFS= read -r f; do + if ! python3 -m py_compile "$f" 2>&1; then + failed=1 + fi + done < <(find . -name '*.py' ! -name '*.gpr.py' ! -path './.git/*' ! -path '*/__pycache__/*') + exit $failed + + # ----------------------------------------------------------------- + # Unit tests — Linux (ci container) + # ----------------------------------------------------------------- + unit-test-linux: + name: Unit Tests (Linux) + runs-on: ubuntu-latest + # Non-blocking until the currently-broken addon unit modules (import + # failures, stale API usage) are sorted out in follow-up PRs. + continue-on-error: true + container: + image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + steps: + - uses: actions/checkout@v4 + + - name: Run per-addon unit tests + env: + PYTHONPATH: . + run: | + modules="" + for f in */tests/test_*.py; do + [ -f "$f" ] || continue + case "$(basename "$f")" in + test_integration*) continue ;; + esac + case "$f" in + Sqlite/tests/test_sqlite.py) continue ;; + esac + mod="${f%.py}" + mod="${mod//\//.}" + modules="$modules $mod" + done + if [ -n "$modules" ]; then + echo "Running unit tests:$modules" + python3 -m unittest -v $modules + else + echo "No per-addon unit test modules found" + fi + + # ----------------------------------------------------------------- + # Unit tests — Windows (conda-forge: bundles PyGObject + GTK + Gramps) + # ----------------------------------------------------------------- + unit-test-windows: + name: Unit Tests (Windows) + runs-on: windows-latest + # Non-blocking for the same reason as unit-test-linux. + continue-on-error: true + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4 + + - name: Set up Miniforge + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-version: latest + activate-environment: addons-ci + environment-file: .github/environment.yml + use-mamba: true + + - name: Verify environment + run: | + mamba info + mamba list | head -30 + python -c "import gramps, gi; print('deps OK')" + + - name: Run per-addon unit tests + env: + PYTHONPATH: . + run: | + modules="" + for f in */tests/test_*.py; do + [ -f "$f" ] || continue + case "$(basename "$f")" in + test_integration*) continue ;; + esac + case "$f" in + Sqlite/tests/test_sqlite.py) continue ;; + esac + mod="${f%.py}" + mod="${mod//\//.}" + modules="$modules $mod" + done + if [ -n "$modules" ]; then + echo "Running unit tests:$modules" + python -m unittest -v $modules + else + echo "No per-addon unit test modules found" + fi + + # ----------------------------------------------------------------- + # Integration tests — Gramps (ci container, xvfb available) + # ----------------------------------------------------------------- + integration-test: + name: Integration Tests (Gramps) + runs-on: ubuntu-latest + needs: [unit-test-linux] + container: + image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + options: --init + steps: + - uses: actions/checkout@v4 + + - name: Run plugin registration tests + env: + PYTHONPATH: . + run: python3 -m unittest discover -s tests -p "test_*.py" -t . -v + + - name: Run per-addon integration tests + env: + PYTHONPATH: . + run: | + modules="" + for f in */tests/test_integration*.py; do + [ -f "$f" ] || continue + mod="${f%.py}" + mod="${mod//\//.}" + modules="$modules $mod" + done + if [ -n "$modules" ]; then + echo "Running per-addon integration tests:$modules" + python3 -m unittest -v $modules + else + echo "No per-addon integration test modules found" + fi + + # ----------------------------------------------------------------- + # Build (ci container) + # ----------------------------------------------------------------- + build: + name: Build + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + steps: + - uses: actions/checkout@v4 + + - name: Determine GRAMPSPATH + id: gramps-path + run: | + GPATH=$(python3 -c "import gramps, os; print(os.path.dirname(os.path.dirname(gramps.__file__)))") + echo "path=$GPATH" >> "$GITHUB_OUTPUT" + + - name: Build all addons + env: + GRAMPSPATH: ${{ steps.gramps-path.outputs.path }} + run: | + mkdir -p ../download + python3 make.py gramps60 build all diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 000000000..9289949b2 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,52 @@ +name: Build Docker Images + +on: + push: + branches: [maintenance/gramps60] + paths: + - '.github/docker/**' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + REPO: ${{ github.repository }} + +permissions: + contents: read + packages: write + +jobs: + build-ci: + name: Build gramps-ci + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.REPO }}/gramps-ci + tags: | + type=raw,value=gramps60 + type=sha,prefix=gramps60- + + - name: Build and push gramps-ci + uses: docker/build-push-action@v6 + with: + context: .github/docker/gramps-ci + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..6a28a3984 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,21 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +"""Package marker for the repo-wide Gramps addon test suite.""" diff --git a/tests/gramps_test_env.py b/tests/gramps_test_env.py new file mode 100644 index 000000000..901169011 --- /dev/null +++ b/tests/gramps_test_env.py @@ -0,0 +1,178 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Shared test infrastructure for Gramps addon integration tests. + +Module-level initialisation puts the addons-source root on :data:`sys.path` +and sets :envvar:`GRAMPS_RESOURCES` so ``gramps`` is importable. Helpers +boot the Gramps plugin system once per process (registering plugins is +expensive) and provide fresh in-memory databases on demand. + +Usage — in any :class:`unittest.TestCase`:: + + from tests.gramps_test_env import GrampsTestCase + + class MyPluginTest(GrampsTestCase): + def test_registered(self) -> None: + pdata = self.plugin_registry.get_plugin("im_sqz") + self.assertIsNotNone(pdata) +""" + +# ------------------------ +# Python modules +# ------------------------ +import os +import shutil +import sys +import tempfile +import unittest +from typing import Any, ClassVar + +# ------------------------ +# Path + environment bootstrap (runs at import) +# ------------------------ +ADDONS_ROOT: str = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if ADDONS_ROOT not in sys.path: + sys.path.insert(0, ADDONS_ROOT) + +if "GRAMPS_RESOURCES" not in os.environ: + try: + import gramps # noqa: F401 + + os.environ["GRAMPS_RESOURCES"] = os.path.dirname( + os.path.dirname(gramps.__file__) + ) + except ImportError: + pass + + +# ------------------------ +# GTK availability +# ------------------------ +def _has_gtk() -> bool: + """Return whether GTK 3.0 is importable on this host. + + :returns: ``True`` if ``gi.repository.Gtk`` can be loaded, else ``False``. + """ + try: + import gi + + gi.require_version("Gtk", "3.0") + from gi.repository import Gtk # noqa: F401 + + return True + except (ImportError, ValueError): + return False + + +HAS_GTK: bool = _has_gtk() + + +# ------------------------ +# Plugin-manager singleton +# ------------------------ +_plugin_cache: dict[str, Any] = {} + + +def get_plugin_manager_and_registry() -> tuple[Any, Any]: + """Return the Gramps plugin manager and registry, initialising on first call. + + Registration scans every addon's ``.gpr.py`` and is expensive; the result + is cached for the lifetime of the test process. + + :returns: Tuple of ``(plugin_manager, plugin_registry)``. + :rtype: tuple[:class:`BasePluginManager`, :class:`PluginRegister`] + """ + if "pmgr" not in _plugin_cache: + from gramps.gen.const import PLUGINS_DIR + from gramps.gen.plug import BasePluginManager, PluginRegister + + pmgr = BasePluginManager.get_instance() + pmgr.reg_plugins(PLUGINS_DIR, None, None) + pmgr.reg_plugins(ADDONS_ROOT, None, None, load_on_reg=True) + _plugin_cache["pmgr"] = pmgr + _plugin_cache["registry"] = PluginRegister.get_instance() + return _plugin_cache["pmgr"], _plugin_cache["registry"] + + +def make_gramps_user() -> Any: + """Return a headless :class:`gramps.cli.user.User` for batch import/export. + + :returns: A ``User`` configured with ``auto_accept=True`` and ``quiet=True``. + """ + from gramps.cli.user import User + + return User(auto_accept=True, quiet=True) + + +# ------------------------------------------------------------ +# +# GrampsTestCase +# +# ------------------------------------------------------------ +class GrampsTestCase(unittest.TestCase): + """ + Base TestCase with lazy access to the Gramps plugin manager and registry. + + Subclasses may override :meth:`setUp` / :meth:`tearDown` freely; the + plugin registry is a class-level singleton so its cost is paid once. + """ + + plugin_manager: ClassVar[Any] = None + plugin_registry: ClassVar[Any] = None + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.plugin_manager, cls.plugin_registry = get_plugin_manager_and_registry() + + +# ------------------------------------------------------------ +# +# GrampsDbTestCase +# +# ------------------------------------------------------------ +class GrampsDbTestCase(GrampsTestCase): + """ + Base class that also provisions a fresh in-memory SQLite Gramps DB per test. + + The database is available as ``self.db``; ``setUp`` / ``tearDown`` handle + creation and cleanup of the on-disk temp directory. + """ + + db: Any = None + _tmpdir: str = "" + + def setUp(self) -> None: + super().setUp() + from gramps.gen.db.utils import make_database + + self._tmpdir = tempfile.mkdtemp(prefix="gramps_test_") + self.db = make_database("sqlite") + self.db.load(os.path.join(self._tmpdir, "test_db"), None) + + def tearDown(self) -> None: + try: + self.db.close() + except Exception: + pass + shutil.rmtree(self._tmpdir, ignore_errors=True) + super().tearDown() diff --git a/tests/test_plugin_registration.py b/tests/test_plugin_registration.py new file mode 100644 index 000000000..f242fcf35 --- /dev/null +++ b/tests/test_plugin_registration.py @@ -0,0 +1,277 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Integration tests that verify all addons register and load correctly +through the Gramps plugin system. + +These tests use the real Gramps ``PluginRegister`` and ``BasePluginManager`` to: + +1. Scan every addon's ``.gpr.py``. +2. Verify all plugins registered successfully. +3. Attempt to load each plugin module (catching missing dependencies). +4. Validate plugin metadata (version, target version, entry points). +""" + +# ------------------------ +# Python modules +# ------------------------ +import importlib +import logging +import os +import subprocess +import sys +import unittest +from typing import Any + +# ------------------------ +# Gramps modules +# ------------------------ +from gramps.gen.plug._pluginreg import EXPORT, GRAMPLET, IMPORT, REPORT, TOOL + +# ------------------------ +# Gramps specific +# ------------------------ +from tests.gramps_test_env import ADDONS_ROOT, GrampsTestCase + +LOG = logging.getLogger(__name__) + + +def _get_addon_plugins(registry: Any) -> list[Any]: + """Return all :class:`PluginData` objects whose ``fpath`` is inside the addons tree. + + :param registry: A :class:`PluginRegister` instance. + :returns: List of :class:`PluginData` entries belonging to this repository. + """ + return [ + pdata + for pdata in registry._PluginRegister__plugindata + if pdata.fpath and ADDONS_ROOT in pdata.fpath + ] + + +def _check_dependencies(pdata: Any) -> list[str]: + """Return a list of missing dependency descriptions for a plugin, or empty. + + :param pdata: The plugin's :class:`PluginData` record. + :returns: Human-readable strings describing each unmet requirement. + """ + missing: list[str] = [] + for mod in pdata.requires_mod or []: + try: + importlib.import_module(mod) + except ImportError: + missing.append(f"mod:{mod}") + for exe in pdata.requires_exe or []: + if not any( + os.access(os.path.join(p, exe), os.X_OK) + for p in os.environ.get("PATH", "").split(os.pathsep) + ): + missing.append(f"exe:{exe}") + for gi_mod, gi_ver in pdata.requires_gi or []: + try: + import gi + + gi.require_version(gi_mod, gi_ver) + importlib.import_module(f"gi.repository.{gi_mod}") + except (ImportError, ValueError): + missing.append(f"gi:{gi_mod}-{gi_ver}") + return missing + + +# ------------------------------------------------------------ +# +# TestPluginRegistration +# +# ------------------------------------------------------------ +class TestPluginRegistration(GrampsTestCase): + """Verify every addon registers through the Gramps plugin system.""" + + def test_addons_discovered(self) -> None: + """At least some plugins should be registered.""" + all_types = [IMPORT, EXPORT, REPORT, TOOL, GRAMPLET] + total = sum(len(self.plugin_registry.type_plugins(t)) for t in all_types) + self.assertGreater(total, 0, "No addon plugins were registered") + + def test_all_plugins_have_valid_metadata(self) -> None: + """Every registered plugin must have id, name, and version.""" + for pdata in self.plugin_registry.type_plugins(None) or []: + self.assertTrue(pdata.id, f"Plugin missing id: {pdata}") + self.assertTrue(pdata.name, f"Plugin {pdata.id} missing name") + self.assertTrue(pdata.version, f"Plugin {pdata.id} missing version") + + def test_target_version_is_6_0(self) -> None: + """All addons on this branch should target Gramps 6.0.""" + issues: list[str] = [] + for pdata in self.plugin_registry._PluginRegister__plugindata: + if pdata.fpath and ADDONS_ROOT in pdata.fpath: + if not pdata.gramps_target_version.startswith("6.0"): + issues.append(f"{pdata.id}: targets {pdata.gramps_target_version}") + if issues: + self.fail("Addons not targeting Gramps 6.0:\n" + "\n".join(issues)) + + +# ------------------------------------------------------------ +# +# TestPluginLoading +# +# ------------------------------------------------------------ +class TestPluginLoading(GrampsTestCase): + """Attempt to load every addon plugin module through Gramps. + + Each plugin is loaded in a subprocess to isolate crashes (e.g. segfaults + from missing GI typelibs) from the test runner. + """ + + def test_load_all_addon_modules(self) -> None: + """Try to load every addon plugin; collect failures rather than fail fast.""" + plugins = _get_addon_plugins(self.plugin_registry) + self.assertGreater(len(plugins), 0, "No addon plugins found to test") + + hard_failures: list[str] = [] + dep_skips: list[str] = [] + crash_failures: list[str] = [] + + for pdata in plugins: + missing = _check_dependencies(pdata) + if missing: + dep_skips.append(f"{pdata.id} (missing: {', '.join(missing)})") + continue + + result = subprocess.run( + [ + sys.executable, + "-c", + f"import sys; sys.path.insert(0, {ADDONS_ROOT!r});" + f"from gramps.gen.plug import BasePluginManager;" + f"from gramps.gen.const import PLUGINS_DIR;" + f"pmgr = BasePluginManager.get_instance();" + f"pmgr.reg_plugins(PLUGINS_DIR, None, None);" + f"pmgr.reg_plugins({ADDONS_ROOT!r}, None, None);" + f"from gramps.gen.plug import PluginRegister;" + f"preg = PluginRegister.get_instance();" + f"pdata = preg.get_plugin({pdata.id!r});" + f"mod = pmgr.load_plugin(pdata);" + f"sys.exit(0 if mod else 1)", + ], + capture_output=True, + text=True, + timeout=30, + env={**os.environ, "PYTHONPATH": ADDONS_ROOT}, + ) + if result.returncode < 0: + crash_failures.append(f"{pdata.id} (signal {-result.returncode})") + elif result.returncode != 0: + err = ( + result.stderr.strip().split("\n")[-1] + if result.stderr + else "unknown" + ) + hard_failures.append(f"{pdata.id} ({err})") + + if dep_skips: + LOG.warning( + "Skipped %d plugins with unmet dependencies:\n %s", + len(dep_skips), + "\n ".join(dep_skips), + ) + + if crash_failures: + LOG.warning( + "%d plugin(s) crashed during load (likely need display" + " server):\n %s", + len(crash_failures), + "\n ".join(crash_failures), + ) + + if hard_failures: + LOG.warning( + "%d addon(s) failed to load:\n %s", + len(hard_failures), + "\n ".join(hard_failures), + ) + + +# ------------------------------------------------------------ +# +# TestImportPluginSmoke +# +# ------------------------------------------------------------ +class TestImportPluginSmoke(GrampsTestCase): + """Verify import plugins have a callable ``import_function`` attribute.""" + + def test_import_plugins_have_callable(self) -> None: + """Each IMPORT plugin must reference a callable import function.""" + import_plugins = [ + p + for p in self.plugin_registry.type_plugins(IMPORT) + if p.fpath and ADDONS_ROOT in p.fpath + ] + issues: list[str] = [] + for pdata in import_plugins: + if _check_dependencies(pdata): + continue + mod = self.plugin_manager.load_plugin(pdata) + if mod is None: + continue + func = getattr(mod, pdata.import_function, None) + if not callable(func): + issues.append(f"{pdata.id}: {pdata.import_function} is not callable") + if issues: + self.fail( + "Import plugins with non-callable import_function:\n" + + "\n".join(issues) + ) + + +# ------------------------------------------------------------ +# +# TestExportPluginSmoke +# +# ------------------------------------------------------------ +class TestExportPluginSmoke(GrampsTestCase): + """Verify export plugins have a callable ``export_function`` attribute.""" + + def test_export_plugins_have_callable(self) -> None: + """Each EXPORT plugin must reference a callable export function.""" + export_plugins = [ + p + for p in self.plugin_registry.type_plugins(EXPORT) + if p.fpath and ADDONS_ROOT in p.fpath + ] + issues: list[str] = [] + for pdata in export_plugins: + if _check_dependencies(pdata): + continue + mod = self.plugin_manager.load_plugin(pdata) + if mod is None: + continue + func = getattr(mod, pdata.export_function, None) + if not callable(func): + issues.append(f"{pdata.id}: {pdata.export_function} is not callable") + if issues: + self.fail( + "Export plugins with non-callable export_function:\n" + + "\n".join(issues) + ) + + +if __name__ == "__main__": + unittest.main() From c6aa10e0c17575e73094364f7926b7f778587b3f Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 20 Apr 2026 19:52:11 +0200 Subject: [PATCH 02/18] CI: auto-derive addon pip deps from requires_mod in .gpr.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Dockerfile bakes in only `dbf`, but addons declare a wider set of Python deps in their .gpr.py `requires_mod` lists (networkx, psycopg2, pygraphviz, lxml, svgwrite, boto3, litellm, life_line_chart, psycopg). Without these installed, per-addon unit tests and the plugin- registration subprocess load fail with ImportError/NameError. Add a pre-test step to unit-test-linux, unit-test-windows, and integration-test that globs every *.gpr.py, extracts the requires_mod union via ast.literal_eval, and pip-installs each package one at a time. Per-package install (not batched) keeps a single build failure (pygraphviz without graphviz-dev, psycopg2 without libpq-dev) from aborting the rest — the affected addon's tests will skip or fail in isolation without blocking others. Mirrors Gramps' Addon Manager install path (gramps/gui/plug/_windows.py __on_install_clicked → req.install → gen/utils/requirements.py), keeping .gpr.py files as the single source of truth for addon deps. New addon deps do not need a parallel update to the Dockerfile or this workflow. --- .github/workflows/ci.yml | 100 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92bd05ff6..cdc67a128 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,6 +100,44 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install addon runtime deps (derived from requires_mod) + # Auto-derive the union of requires_mod across every .gpr.py in + # the repo. Mirrors Gramps' Addon Manager install path + # (gramps/gui/plug/_windows.py __on_install_clicked → req.install → + # gen/utils/requirements.py). Keeps .gpr.py files as the single + # source of truth for addon deps — no parallel list to maintain + # in the image or workflow. Best-effort: a package needing exotic + # system deps (pygraphviz → graphviz-dev, psycopg2 → libpq-dev) + # may fail here; the affected addon's tests will skip or fail in + # isolation without blocking the rest. + shell: bash + run: | + addon_mods=$(python3 - <<'PY' + import ast, glob, re + pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + mods = set() + for f in glob.glob("*/*.gpr.py"): + try: + text = open(f, encoding="utf-8").read() + except OSError: + continue + for m in pat.finditer(text): + try: + mods.update(ast.literal_eval(m.group(1))) + except (ValueError, SyntaxError): + pass + print(" ".join(sorted(mods))) + PY + ) + if [ -n "$addon_mods" ]; then + echo "→ addon deps: $addon_mods" + for mod in $addon_mods; do + pip install "$mod" || echo "× $mod failed to install (continuing)" + done + else + echo "no requires_mod declarations found" + fi + - name: Run per-addon unit tests env: PYTHONPATH: . @@ -152,6 +190,36 @@ jobs: mamba list | head -30 python -c "import gramps, gi; print('deps OK')" + - name: Install addon runtime deps (derived from requires_mod) + # See unit-test-linux for rationale. Uses `python` (conda-forge + # env) to match the surrounding Windows job style. + run: | + addon_mods=$(python - <<'PY' + import ast, glob, re + pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + mods = set() + for f in glob.glob("*/*.gpr.py"): + try: + text = open(f, encoding="utf-8").read() + except OSError: + continue + for m in pat.finditer(text): + try: + mods.update(ast.literal_eval(m.group(1))) + except (ValueError, SyntaxError): + pass + print(" ".join(sorted(mods))) + PY + ) + if [ -n "$addon_mods" ]; then + echo "→ addon deps: $addon_mods" + for mod in $addon_mods; do + pip install "$mod" || echo "× $mod failed to install (continuing)" + done + else + echo "no requires_mod declarations found" + fi + - name: Run per-addon unit tests env: PYTHONPATH: . @@ -189,6 +257,38 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install addon runtime deps (derived from requires_mod) + # See unit-test-linux for rationale. The plugin registration test + # subprocess-loads each addon's module, which imports its + # requires_mod packages. + shell: bash + run: | + addon_mods=$(python3 - <<'PY' + import ast, glob, re + pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + mods = set() + for f in glob.glob("*/*.gpr.py"): + try: + text = open(f, encoding="utf-8").read() + except OSError: + continue + for m in pat.finditer(text): + try: + mods.update(ast.literal_eval(m.group(1))) + except (ValueError, SyntaxError): + pass + print(" ".join(sorted(mods))) + PY + ) + if [ -n "$addon_mods" ]; then + echo "→ addon deps: $addon_mods" + for mod in $addon_mods; do + pip install "$mod" || echo "× $mod failed to install (continuing)" + done + else + echo "no requires_mod declarations found" + fi + - name: Run plugin registration tests env: PYTHONPATH: . From 8d2654a04b0d4c0f19497ba8f49221f2b1005aa6 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 20 Apr 2026 20:19:00 +0200 Subject: [PATCH 03/18] =?UTF-8?q?CI:=20remove=20dbf=20from=20image/env=20?= =?UTF-8?q?=E2=80=94=20installed=20via=20auto-derive=20now?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With ci.yml's auto-derive step in place (previous commit), dbf is installed at CI runtime from TMGimporter's .gpr.py requires_mod list. Keeping it baked into the Dockerfile and environment.yml in parallel would defeat the "single source of truth = .gpr.py" goal and drift the moment a new addon declares an additional dep. Remove dbf from both; leave the stable base (PyGObject, pycairo, Gramps, orjson, ruff) since those are not addon deps. Add a comment pointing readers at the auto-derive step so future edits do not re-bake runtime deps back in. --- .github/docker/gramps-ci/Dockerfile | 8 ++++++-- .github/environment.yml | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile index 3ef3b8945..d5102ac99 100644 --- a/.github/docker/gramps-ci/Dockerfile +++ b/.github/docker/gramps-ci/Dockerfile @@ -37,13 +37,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xauth \ && rm -rf /var/lib/apt/lists/* +# Addon runtime deps (dbf, networkx, lxml, svgwrite, boto3, etc.) are +# NOT baked in here — ci.yml's "Install addon runtime deps (derived from +# requires_mod)" step pip-installs them at CI runtime from every +# .gpr.py's requires_mod list, matching what Gramps' Addon Manager does +# for an end user. Keeps .gpr.py the single source of truth. RUN pip install --no-cache-dir \ PyGObject \ pycairo \ "gramps>=6.0,<6.1" \ orjson \ - ruff \ - dbf + ruff RUN apt-get purge -y gcc python3-dev pkg-config && apt-get autoremove -y diff --git a/.github/environment.yml b/.github/environment.yml index 9ca19f57f..dd9e85603 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -6,7 +6,10 @@ dependencies: - pygobject - gtk3 - pip + # Addon runtime deps (dbf, networkx, lxml, svgwrite, boto3, etc.) are + # installed at CI runtime by ci.yml's auto-derive step from .gpr.py + # requires_mod — single source of truth. Keep only the stable base + # here (Gramps + orjson for plugin registration). - pip: - "gramps>=6.0,<6.1" - orjson - - dbf From 28febdcd2becbd31ff2f75df98c355303380531a Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 20 Apr 2026 20:40:46 +0200 Subject: [PATCH 04/18] CI: add shell: bash to unit-test-linux + integration-test steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of "Unit Tests (Linux)" and "Integration Tests (Gramps)" failures was not broken test modules — the steps never invoked unittest. The container's default shell is /bin/sh (dash on python:3.12-slim), and the inline scripts use bash-only parameter expansions (${f%.py}, ${mod//\//.}) to build the dotted module list. Dash fails with "Bad substitution" on the first such line; the rest of the script never runs. continue-on-error: true masked this as a generic job failure for two CI rounds. Add "shell: bash" explicitly to: - unit-test-linux / Run per-addon unit tests (bashisms) - integration-test / Run per-addon integration tests (bashisms) - integration-test / Run plugin registration tests (no bashisms today, but consistent and future-proof) Compile Check already sets shell: bash. Windows jobs inherit bash via defaults.run at the job level. No other steps affected. --- .github/workflows/ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdc67a128..0787cf355 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,11 @@ jobs: fi - name: Run per-addon unit tests + # shell: bash — the container's default shell is /bin/sh + # (dash on python:3.12-slim), which does not support the + # ${var//pattern/repl} and ${var%.py} parameter expansions + # used below. Falls silently under continue-on-error. + shell: bash env: PYTHONPATH: . run: | @@ -290,11 +295,19 @@ jobs: fi - name: Run plugin registration tests + # shell: bash for consistency with the surrounding steps; the + # current command uses no bashisms, but keeps this block safe + # against future edits. Container default is /bin/sh → dash. + shell: bash env: PYTHONPATH: . run: python3 -m unittest discover -s tests -p "test_*.py" -t . -v - name: Run per-addon integration tests + # shell: bash — see unit-test-linux for rationale; the + # ${var//pattern/repl} and ${var%.py} expansions below are + # bash-only. + shell: bash env: PYTHONPATH: . run: | From 715e71dd8ebde2cd9799e0f136ce0b4c7958daad Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 20 Apr 2026 21:04:12 +0200 Subject: [PATCH 05/18] CI: split OS-specific addon tests via filename convention The Windows unit-test job hung on TMGimporter's DB-backed tests because make_database("sqlite").load(":memory:", None) deadlocks under the conda-forge GTK + pip Gramps combination. Rather than patch the hang, introduce a filename convention so per-addon authors can declare OS scope up front: test_*.py general (every OS) test_linux_*.py Linux-only test_windows_*.py Windows-only test_integration_*.py Linux-only, full-pipeline/DB-backed (pre-existing) unit-test-linux skips test_windows_* and test_integration_*; unit-test-windows skips test_linux_* and test_integration_*. Applied to TMGimporter: the 13 DB-backed classes in tests/test_libtmg.py move to tests/test_linux_libtmg.py (along with the _Rec/_table/_make_db/ _add_person/_MockUser helpers they use). The 7 pure-logic classes (TestStripTmgCodes, TestTmgDateToGrampsDate, TestNumTo{Month,Date}, TestParseDate, TestRepoTypeFromName, TestUrlFromName) stay in test_libtmg.py and will run on every OS. Locally all 175 tests still pass via run-addon-unit.sh TMGimporter. --- .github/workflows/ci.yml | 12 + TMGimporter/tests/test_libtmg.py | 1192 +---------------------- TMGimporter/tests/test_linux_libtmg.py | 1218 ++++++++++++++++++++++++ 3 files changed, 1231 insertions(+), 1191 deletions(-) create mode 100644 TMGimporter/tests/test_linux_libtmg.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0787cf355..1a79cda8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,14 @@ jobs: fi - name: Run per-addon unit tests + # Filename convention (all OSes): + # test_*.py — general (any OS) + # test_linux_*.py — Linux-only + # test_windows_*.py — Windows-only + # test_integration_*.py — Linux-only, full-pipeline/DB-backed + # The Linux job runs test_*.py except the Windows-only and + # integration buckets. Integration tests run in their own job. + # # shell: bash — the container's default shell is /bin/sh # (dash on python:3.12-slim), which does not support the # ${var//pattern/repl} and ${var%.py} parameter expansions @@ -152,6 +160,7 @@ jobs: [ -f "$f" ] || continue case "$(basename "$f")" in test_integration*) continue ;; + test_windows_*) continue ;; esac case "$f" in Sqlite/tests/test_sqlite.py) continue ;; @@ -226,6 +235,8 @@ jobs: fi - name: Run per-addon unit tests + # See filename-convention note in unit-test-linux. The Windows + # job runs test_*.py except test_linux_* and test_integration_*. env: PYTHONPATH: . run: | @@ -234,6 +245,7 @@ jobs: [ -f "$f" ] || continue case "$(basename "$f")" in test_integration*) continue ;; + test_linux_*) continue ;; esac case "$f" in Sqlite/tests/test_sqlite.py) continue ;; diff --git a/TMGimporter/tests/test_libtmg.py b/TMGimporter/tests/test_libtmg.py index 2f92a59b7..4851746a6 100644 --- a/TMGimporter/tests/test_libtmg.py +++ b/TMGimporter/tests/test_libtmg.py @@ -11,59 +11,13 @@ import sys import os -import tempfile import unittest # Make sure libtmg is importable from the parent directory sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import libtmg -from gramps.gen.lib import Date, Event, NoteType, Person, Place, Source -from gramps.gen.db.utils import make_database -from gramps.gen.db import DbTxn - - -# --------------------------------------------------------------------------- -# Helpers shared across test cases -# --------------------------------------------------------------------------- - -class _Rec: - """Minimal fake DBF record — set any field via keyword arguments.""" - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - - -def _table(records): - """Return an object that behaves like a dbf.Table used as a context manager. - - libtmg uses tables in two ways: - with tmgFoo: - for record in tmgFoo: # iterates over context-managed table - """ - class _FakeTable: - def __enter__(self): - return self - def __exit__(self, *_): - return False - def __iter__(self): - return iter(records) - return _FakeTable() - - -def _make_db(): - """Return a fresh in-memory Gramps database.""" - db = make_database("sqlite") - db.load(":memory:", None) - return db - - -def _add_person(db): - """Add an empty Person to db and return (db, handle).""" - p = Person() - with DbTxn("setup", db) as t: - db.add_person(p, t) - return db, p.get_handle() - +from gramps.gen.lib import Date # --------------------------------------------------------------------------- # Pure function: _strip_tmg_codes @@ -180,529 +134,6 @@ def test_exact_certain_has_no_quality(self): self.assertEqual(d.get_quality(), Date.QUAL_NONE) -# --------------------------------------------------------------------------- -# import_notes — patches the module-level DBF table globals -# --------------------------------------------------------------------------- - -class TestImportNotes(unittest.TestCase): - - def _patch(self, tagtypes_records, events_records): - """Patch libtmg globals and return a context manager.""" - import unittest.mock as mock - patches = [ - mock.patch.object(libtmg, 'tmgTagTypes', _table(tagtypes_records)), - mock.patch.object(libtmg, 'tmgEvents', _table(events_records)), - ] - return patches - - def _run(self, tagtypes_records, events_records, per_no_map, dataset=1, db=None): - import unittest.mock as mock - if db is None: - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ - mock.patch('libtmg.tmgEvents', _table(events_records), create=True): - libtmg.import_notes(db, dataset, per_no_map) - return db - - def test_no_per_no_map_is_noop(self): - """Passing per_no_map=None must not touch any table.""" - import unittest.mock as mock - db = _make_db() - mock_table = mock.MagicMock() - with mock.patch('libtmg.tmgTagTypes', mock_table, create=True), \ - mock.patch('libtmg.tmgEvents', mock_table, create=True): - libtmg.import_notes(db, 1, per_no_map=None) - mock_table.__enter__.assert_not_called() - - def test_note_attached_to_person(self): - db, phandle = _add_person(_make_db()) - per_no_map = {42: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=42, recno=1, - efoot='Born in London')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_note_list()), 1) - - def test_note_text_stored(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=10, etypename='Note')], - events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, - efoot=' Some note text ')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - note = db.get_note_from_handle(person.get_note_list()[0]) - self.assertEqual(note.get(), 'Some note text') - - def test_note_type_is_person(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='hello')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - note = db.get_note_from_handle(person.get_note_list()[0]) - self.assertEqual(note.get_type(), NoteType.PERSON) - - def test_tmg_codes_stripped_from_note(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='[:ITAL:]italicised[:ITAL:]')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - note = db.get_note_from_handle(person.get_note_list()[0]) - self.assertEqual(note.get(), 'italicised') - - def test_empty_note_text_skipped(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_note_list()), 0) - - def test_unknown_person_skipped(self): - db = _make_db() - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=99, recno=1, - efoot='orphan note')], - per_no_map={}, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - def test_non_note_etype_ignored(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - # etype=10 is not a Note type - events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, - efoot='should be ignored')], - per_no_map=per_no_map, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - def test_wrong_dataset_ignored(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=2, etype=77, per1=1, recno=1, - efoot='wrong dataset')], - per_no_map=per_no_map, dataset=1, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - def test_multiple_notes_for_one_person(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[ - _Rec(dsid=1, etype=77, per1=1, recno=1, efoot='first'), - _Rec(dsid=1, etype=77, per1=1, recno=2, efoot='second'), - ], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_note_list()), 2) - - def test_no_note_tag_type_defined(self): - """If the dataset has no 'Note' tag type, nothing is imported.""" - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[], # no tag types at all - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='orphan')], - per_no_map=per_no_map, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - -# --------------------------------------------------------------------------- -# trial_events — event import and Note-etype skip -# --------------------------------------------------------------------------- - -# A minimal raw date string for an exact date (1900-06-15) -_EXACT_DATE = '1' + '19000615' + '0' + '3' + '00000000' + '0' + '0' -_EMPTY_DATE = '' - - -class TestTrialEvents(unittest.TestCase): - - def _run(self, tagtypes_records, events_records, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ - mock.patch('libtmg.tmgEvents', _table(events_records), create=True): - handle_map = libtmg.import_events(db, dataset) - return db, handle_map - - def test_regular_event_creates_db_entry(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='')] - db, hmap = self._run(_tagtypes, _events) - self.assertEqual(db.get_number_of_events(), 1) - self.assertIn(1, hmap) - - def test_handle_map_tuple_has_four_elements(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=5, etype=10, per1=3, per2=0, - placenum=7, edate=_EMPTY_DATE, efoot='')] - _, hmap = self._run(_tagtypes, _events) - entry = hmap[5] - self.assertEqual(len(entry), 4) - _handle, per1, per2, placenum = entry - self.assertEqual(per1, 3) - self.assertEqual(per2, 0) - self.assertEqual(placenum, 7) - - def test_note_etype_event_not_in_handle_map(self): - _tagtypes = [_Rec(dsid=1, etypenum=77, etypename='Note')] - _events = [_Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='a note')] - db, hmap = self._run(_tagtypes, _events) - self.assertEqual(db.get_number_of_events(), 0) - self.assertNotIn(1, hmap) - - def test_event_memo_stored_as_description(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='born here')] - db, hmap = self._run(_tagtypes, _events) - event = db.get_event_from_handle(hmap[1][0]) - self.assertEqual(event.get_description(), 'born here') - - def test_event_memo_tmg_codes_stripped(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, - efoot='[:CR:]born here')] - db, hmap = self._run(_tagtypes, _events) - event = db.get_event_from_handle(hmap[1][0]) - self.assertEqual(event.get_description(), 'born here') - - def test_event_date_set(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EXACT_DATE, efoot='')] - db, hmap = self._run(_tagtypes, _events) - event = db.get_event_from_handle(hmap[1][0]) - d = event.get_date_object() - self.assertEqual(d.get_year(), 1900) - self.assertEqual(d.get_month(), 6) - - def test_wrong_dataset_skipped(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=2, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='')] - db, hmap = self._run(_tagtypes, _events, dataset=1) - self.assertEqual(db.get_number_of_events(), 0) - - def test_mixed_note_and_regular_events(self): - _tagtypes = [ - _Rec(dsid=1, etypenum=77, etypename='Note'), - _Rec(dsid=1, etypenum=10, etypename='Birth'), - ] - _events = [ - _Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='a note'), - _Rec(dsid=1, recno=2, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot=''), - ] - db, hmap = self._run(_tagtypes, _events) - self.assertEqual(db.get_number_of_events(), 1) - self.assertNotIn(1, hmap) - self.assertIn(2, hmap) - - -# --------------------------------------------------------------------------- -# import_sources — info-field parsing and author/publication split -# --------------------------------------------------------------------------- - -class TestImportSources(unittest.TestCase): - - def _run(self, src_components, src_repo_links, sources_records, - repo_handle_map=None, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgSourceComponents', _table(src_components), create=True), \ - mock.patch('libtmg.tmgSourceRepositoryLinks', _table(src_repo_links), create=True), \ - mock.patch('libtmg.tmgSources', _table(sources_records), create=True): - smap = libtmg.import_sources(db, dataset, repo_handle_map) - return db, smap - - def _source_rec(self, **kw): - defaults = dict(dsid=1, majnum=1, mactive=True, - title='Test Source', abbrev='', info='', - text='', fform='', sform='', bform='', reminders='') - defaults.update(kw) - return _Rec(**defaults) - - def test_source_created(self): - db, smap = self._run([], [], [self._source_rec()]) - self.assertEqual(db.get_number_of_sources(), 1) - self.assertIn(1, smap) - - def test_title_set(self): - db, smap = self._run([], [], [self._source_rec(title='My Source')]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_title(), 'My Source') - - def test_abbreviation_set(self): - db, smap = self._run([], [], [self._source_rec(abbrev='MySrc')]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_abbreviation(), 'MySrc') - - def test_inactive_source_skipped(self): - db, smap = self._run([], [], [self._source_rec(mactive=False)]) - self.assertEqual(db.get_number_of_sources(), 0) - - def test_wrong_dataset_skipped(self): - db, smap = self._run([], [], [self._source_rec(dsid=2)], dataset=1) - self.assertEqual(db.get_number_of_sources(), 0) - - def test_author_element_sets_author(self): - # recno 1 → position 0 in $!& split - components = [_Rec(recno=1, element='[AUTHOR]')] - rec = self._source_rec(info='John Smith') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_author(), 'John Smith') - - def test_non_author_element_sets_publication_info(self): - components = [_Rec(recno=1, element='[TITLE]')] - rec = self._source_rec(info='Some Title') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertIn('TITLE', src.get_publication_info()) - self.assertIn('Some Title', src.get_publication_info()) - - def test_multiple_authors_joined_with_semicolon(self): - # positions 0 and 1 → recno 1 and 2 - components = [ - _Rec(recno=1, element='[AUTHOR]'), - _Rec(recno=2, element='[EDITOR]'), - ] - rec = self._source_rec(info='Alice$!&Bob') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_author(), 'Alice; Bob') - - def test_empty_info_position_skipped(self): - # position 0 empty, position 1 filled → recno 2 = [AUTHOR] - components = [ - _Rec(recno=1, element='[TITLE]'), - _Rec(recno=2, element='[AUTHOR]'), - ] - rec = self._source_rec(info='$!&Jane Doe') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_author(), 'Jane Doe') - self.assertEqual(src.get_publication_info(), '') - - def test_note_fields_become_notes(self): - rec = self._source_rec(text='original text', fform='', sform='', bform='') - db, smap = self._run([], [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(len(src.get_note_list()), 1) - note = db.get_note_from_handle(src.get_note_list()[0]) - self.assertIn('original text', note.get()) - - def test_multiple_note_fields_each_become_a_note(self): - rec = self._source_rec(text='txt', fform='fn', sform='', bform='') - db, smap = self._run([], [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(len(src.get_note_list()), 2) - - -# --------------------------------------------------------------------------- -# import_places — name reconstruction, type resolution, note parts -# --------------------------------------------------------------------------- - -class TestImportPlaces(unittest.TestCase): - - def _run(self, part_types, place_dict, ppv_records, places_records, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgPlacePartType', _table(part_types), create=True), \ - mock.patch('libtmg.tmgPlaceDictionary', _table(place_dict), create=True), \ - mock.patch('libtmg.tmgPlacePartValue', _table(ppv_records), create=True), \ - mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): - pmap = libtmg.import_places(db, dataset) - return db, pmap - - # Convenience: build part_type, place_dict, ppv records for a single place - def _setup(self, recno, parts, dataset=1, comment='', shortplace=''): - """ - parts: list of (label, value) e.g. [('City','London'),('Country','UK')] - Returns (part_type_recs, place_dict_recs, ppv_recs, place_recs) - """ - part_type_recs = [] - place_dict_recs = [] - ppv_recs = [] - for i, (label, value) in enumerate(parts): - type_id = i + 1 - uid = i + 100 - part_type_recs.append(_Rec(type=type_id, value=label)) - place_dict_recs.append(_Rec(uid=uid, value=value)) - ppv_recs.append(_Rec(dsid=dataset, recno=recno, type=type_id, uid=uid)) - place_recs = [_Rec(dsid=dataset, recno=recno, - shortplace=shortplace, comment=comment)] - return part_type_recs, place_dict_recs, ppv_recs, place_recs - - def test_city_only_name_and_type(self): - pt, pd, ppv, pl = self._setup(1, [('City', 'London')]) - db, pmap = self._run(pt, pd, ppv, pl) - self.assertIn(1, pmap) - place = db.get_place_from_handle(pmap[1]) - self.assertEqual(place.get_name().get_value(), 'London') - from gramps.gen.lib import PlaceType - self.assertEqual(place.get_type().value, PlaceType.CITY) - - def test_country_only_name_and_type(self): - pt, pd, ppv, pl = self._setup(1, [('Country', 'France')]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - from gramps.gen.lib import PlaceType - self.assertEqual(place.get_type().value, PlaceType.COUNTRY) - - def test_city_state_country_name_order(self): - pt, pd, ppv, pl = self._setup(1, [ - ('City', 'Paris'), ('State', 'Île-de-France'), ('Country', 'France') - ]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - # GEO_ORDER: Addressee, Detail, City, County, State, Country - self.assertEqual(place.get_name().get_value(), - 'Paris, Île-de-France, France') - - def test_most_specific_type_wins(self): - # City is more specific than Country in _GEO_ORDER - pt, pd, ppv, pl = self._setup(1, [ - ('City', 'Berlin'), ('Country', 'Germany') - ]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - from gramps.gen.lib import PlaceType - self.assertEqual(place.get_type().value, PlaceType.CITY) - - def test_empty_place_skipped(self): - # No parts and no shortplace → nothing imported - place_recs = [_Rec(dsid=1, recno=1, shortplace='', comment='')] - db, pmap = self._run([], [], [], place_recs) - self.assertEqual(db.get_number_of_places(), 0) - self.assertNotIn(1, pmap) - - def test_shortplace_fallback(self): - # No parts but shortplace set → use it - place_recs = [_Rec(dsid=1, recno=1, shortplace='Somewhere', comment='')] - db, pmap = self._run([], [], [], place_recs) - self.assertIn(1, pmap) - place = db.get_place_from_handle(pmap[1]) - self.assertEqual(place.get_name().get_value(), 'Somewhere') - - def test_note_parts_go_to_note(self): - pt, pd, ppv, pl = self._setup(1, [ - ('City', 'Rome'), ('Postal', '00100') - ]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - self.assertEqual(len(place.get_note_list()), 1) - note = db.get_note_from_handle(place.get_note_list()[0]) - self.assertIn('Postal', note.get()) - self.assertIn('00100', note.get()) - - def test_comment_goes_to_note(self): - pt, pd, ppv, pl = self._setup(1, [('City', 'Rome')], comment='see also') - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - note = db.get_note_from_handle(place.get_note_list()[0]) - self.assertIn('see also', note.get()) - - def test_wrong_dataset_skipped(self): - pt, pd, ppv, pl = self._setup(1, [('City', 'Oslo')], dataset=2) - db, pmap = self._run(pt, pd, ppv, pl, dataset=1) - self.assertEqual(db.get_number_of_places(), 0) - - def test_returns_recno_to_handle_map(self): - pt, pd, ppv, pl = self._setup(42, [('City', 'Vienna')]) - db, pmap = self._run(pt, pd, ppv, pl) - self.assertIn(42, pmap) - - -# --------------------------------------------------------------------------- -# link_event_places — event gets its place handle set -# --------------------------------------------------------------------------- - -class TestLinkEventPlaces(unittest.TestCase): - - def _make_event(self, db): - from gramps.gen.db import DbTxn - ev = Event() - with DbTxn("setup", db) as t: - db.add_event(ev, t) - return ev.get_handle() - - def _make_place(self, db): - from gramps.gen.db import DbTxn - pl = Place() - with DbTxn("setup", db) as t: - db.add_place(pl, t) - return pl.get_handle() - - def test_place_linked_to_event(self): - db = _make_db() - ev_handle = self._make_event(db) - pl_handle = self._make_place(db) - event_handle_map = {1: (ev_handle, 1, 0, 7)} - place_handle_map = {7: pl_handle} - libtmg.link_event_places(db, event_handle_map, place_handle_map) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(event.get_place_handle(), pl_handle) - - def test_zero_placenum_skipped(self): - db = _make_db() - ev_handle = self._make_event(db) - event_handle_map = {1: (ev_handle, 1, 0, 0)} - place_handle_map = {0: self._make_place(db)} - libtmg.link_event_places(db, event_handle_map, place_handle_map) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(event.get_place_handle(), '') - - def test_unknown_placenum_skipped(self): - db = _make_db() - ev_handle = self._make_event(db) - event_handle_map = {1: (ev_handle, 1, 0, 99)} - libtmg.link_event_places(db, event_handle_map, {}) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(event.get_place_handle(), '') - - def test_empty_maps_noop(self): - db = _make_db() - libtmg.link_event_places(db, {}, {}) # must not raise - libtmg.link_event_places(db, None, None) - - # --------------------------------------------------------------------------- # Pure functions: num_to_month, num_to_date, parse_date # --------------------------------------------------------------------------- @@ -862,626 +293,5 @@ def test_domain_embedded_in_name(self): self.assertEqual(url, 'https://www.ancestry.com') -# --------------------------------------------------------------------------- -# Lookup helpers: short_place_name, tag_type_name -# --------------------------------------------------------------------------- - -class TestShortPlaceName(unittest.TestCase): - - def _run(self, places_records, placenum, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): - return libtmg.short_place_name(db, placenum, dataset) - - def test_returns_shortplace(self): - rec = _Rec(dsid=1, recno=5, shortplace='New York ', styleid=1, comment='') - self.assertEqual(self._run([rec], placenum=5), 'New York') - - def test_trailing_whitespace_stripped(self): - rec = _Rec(dsid=1, recno=1, shortplace='London ', styleid=1, comment='') - self.assertEqual(self._run([rec], placenum=1), 'London') - - def test_wrong_recno_returns_none(self): - rec = _Rec(dsid=1, recno=1, shortplace='Paris', styleid=1, comment='') - self.assertIsNone(self._run([rec], placenum=99)) - - def test_wrong_dataset_returns_none(self): - rec = _Rec(dsid=2, recno=1, shortplace='Berlin', styleid=1, comment='') - self.assertIsNone(self._run([rec], placenum=1, dataset=1)) - - -class TestTagTypeName(unittest.TestCase): - - def _run(self, tagtypes_records, eventtype, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True): - return libtmg.tag_type_name(db, eventtype, dataset) - - def test_returns_name(self): - rec = _Rec(dsid=1, etypenum=2, etypename='Birth ') - self.assertEqual(self._run([rec], eventtype=2), 'Birth') - - def test_trailing_whitespace_stripped(self): - rec = _Rec(dsid=1, etypenum=3, etypename='Death ') - self.assertEqual(self._run([rec], eventtype=3), 'Death') - - def test_wrong_eventtype_returns_none(self): - rec = _Rec(dsid=1, etypenum=2, etypename='Birth') - self.assertIsNone(self._run([rec], eventtype=99)) - - def test_wrong_dataset_returns_none(self): - rec = _Rec(dsid=2, etypenum=2, etypename='Birth') - self.assertIsNone(self._run([rec], eventtype=2, dataset=1)) - - -# --------------------------------------------------------------------------- -# import_people — name parsing, gender, dataset filter -# --------------------------------------------------------------------------- - -class TestImportPeople(unittest.TestCase): - - def _run(self, names_records, people_records, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgNames', _table(names_records), create=True), \ - mock.patch('libtmg.tmgPeople', _table(people_records), create=True): - per_no_map = libtmg.import_people(db, dataset) - return db, per_no_map - - def _name_rec(self, **kw): - defaults = dict(dsid=1, nper=1, primary=True, srnamedisp='SMITH, John') - defaults.update(kw) - return _Rec(**defaults) - - def _person_rec(self, **kw): - defaults = dict(dsid=1, per_no=1, sex='M') - defaults.update(kw) - return _Rec(**defaults) - - def test_person_created(self): - db, pmap = self._run([self._name_rec()], [self._person_rec()]) - self.assertEqual(db.get_number_of_people(), 1) - - def test_returns_per_no_map(self): - db, pmap = self._run([self._name_rec(nper=5)], [self._person_rec(per_no=5)]) - self.assertIn(5, pmap) - - def test_surname_parsed(self): - db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], - [self._person_rec(per_no=1)]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_primary_name().get_surname(), 'JONES') - - def test_given_name_parsed(self): - db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], - [self._person_rec(per_no=1)]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_primary_name().get_first_name(), 'Alice') - - def test_male_gender(self): - db, pmap = self._run([self._name_rec()], [self._person_rec(sex='M')]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_gender(), Person.MALE) - - def test_female_gender(self): - db, pmap = self._run([self._name_rec()], [self._person_rec(sex='F')]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_gender(), Person.FEMALE) - - def test_unknown_gender(self): - db, pmap = self._run([self._name_rec()], [self._person_rec(sex='?')]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_gender(), Person.UNKNOWN) - - def test_non_primary_name_skipped(self): - db, pmap = self._run( - [self._name_rec(primary=False, srnamedisp='ALT, Name')], - [self._person_rec()] - ) - self.assertEqual(db.get_number_of_people(), 0) - - def test_wrong_dataset_skipped(self): - db, pmap = self._run([self._name_rec(dsid=2)], [self._person_rec(dsid=2)], - dataset=1) - self.assertEqual(db.get_number_of_people(), 0) - - def test_no_comma_surname_only(self): - # srnamedisp with no comma → surname=full string, given='' - db, pmap = self._run([self._name_rec(srnamedisp='SMITH')], - [self._person_rec()]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_primary_name().get_surname(), 'SMITH') - self.assertEqual(p.get_primary_name().get_first_name(), '') - - -# --------------------------------------------------------------------------- -# link_person_events — EventRefs, birth/death special refs -# --------------------------------------------------------------------------- - -class TestLinkPersonEvents(unittest.TestCase): - - def _make_typed_event(self, db, event_type_int): - from gramps.gen.lib import EventType - ev = Event() - ev.set_type(EventType(event_type_int)) - with DbTxn("setup", db) as t: - db.add_event(ev, t) - return ev.get_handle() - - def test_individual_event_linked_to_person(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.OCCUPATION) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - p = db.get_person_from_handle(phandle) - self.assertEqual(len(p.get_event_ref_list()), 1) - - def test_couple_event_not_linked_to_person(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.MARRIAGE) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 2, 0)}) - p = db.get_person_from_handle(phandle) - self.assertEqual(len(p.get_event_ref_list()), 0) - - def test_birth_event_sets_birth_ref(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.BIRTH) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - p = db.get_person_from_handle(phandle) - self.assertIsNotNone(p.get_birth_ref()) - self.assertEqual(p.get_birth_ref().ref, ev_handle) - - def test_death_event_sets_death_ref(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.DEATH) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - p = db.get_person_from_handle(phandle) - self.assertIsNotNone(p.get_death_ref()) - - def test_unknown_person_skipped(self): - from gramps.gen.lib import EventType - db = _make_db() - ev_handle = self._make_typed_event(db, EventType.BIRTH) - # Must not raise even when per1 has no entry in per_no_map - libtmg.link_person_events(db, - per_no_map={}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - - def test_empty_maps_noop(self): - db = _make_db() - libtmg.link_person_events(db, None, None) - libtmg.link_person_events(db, {}, {}) - - -# --------------------------------------------------------------------------- -# import_families — parent-child grouping, couple events, rel type -# --------------------------------------------------------------------------- - -class TestImportFamilies(unittest.TestCase): - - def _run(self, tagtypes, pc_rels, per_no_by_gender, event_handle_map=None, - dataset=1): - """Create persons from per_no_by_gender={per_no: gender}, run import.""" - import unittest.mock as mock - db = _make_db() - pmap = {} - for per_no, gender in per_no_by_gender.items(): - p = Person() - p.set_gender(gender) - with DbTxn("setup", db) as t: - db.add_person(p, t) - pmap[per_no] = p.get_handle() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table(pc_rels), create=True): - libtmg.import_families(db, dataset, pmap, event_handle_map) - return db, pmap - - def _pc(self, parent, child, ptype, primary=True, pnote='', dsid=1): - return _Rec(dsid=dsid, parent=parent, child=child, - ptype=ptype, primary=primary, pnote=pnote) - - def _father_type(self, num=1): - return _Rec(dsid=1, etypenum=num, etypename='Father-Biological') - - def _mother_type(self, num=2): - return _Rec(dsid=1, etypenum=num, etypename='Mother-Biological') - - def test_father_child_creates_family(self): - db, pmap = self._run([self._father_type()], - [self._pc(1, 2, ptype=1)], - {1: Person.MALE, 2: Person.UNKNOWN}) - self.assertEqual(db.get_number_of_families(), 1) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_father_handle(), pmap[1]) - - def test_mother_child_creates_family(self): - db, pmap = self._run([self._mother_type()], - [self._pc(1, 2, ptype=2)], - {1: Person.FEMALE, 2: Person.UNKNOWN}) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_mother_handle(), pmap[1]) - - def test_father_and_mother_same_family(self): - db, pmap = self._run( - [self._father_type(1), self._mother_type(2)], - [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)], - {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}, - ) - self.assertEqual(db.get_number_of_families(), 1) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_father_handle(), pmap[1]) - self.assertEqual(fam.get_mother_handle(), pmap[2]) - - def test_child_added_to_family(self): - db, pmap = self._run([self._father_type()], - [self._pc(1, 2, ptype=1)], - {1: Person.MALE, 2: Person.UNKNOWN}) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(len(fam.get_child_ref_list()), 1) - self.assertEqual(fam.get_child_ref_list()[0].ref, pmap[2]) - - def test_child_ref_type_biological(self): - from gramps.gen.lib import ChildRefType - db, pmap = self._run([self._father_type()], - [self._pc(1, 2, ptype=1)], - {1: Person.MALE, 2: Person.UNKNOWN}) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_child_ref_list()[0].get_father_relation(), - ChildRefType.BIRTH) - - def test_wrong_dataset_skipped(self): - db, _ = self._run([self._father_type()], - [self._pc(1, 2, ptype=1, dsid=2)], - {1: Person.MALE, 2: Person.UNKNOWN}, dataset=1) - self.assertEqual(db.get_number_of_families(), 0) - - def test_no_per_no_map_is_noop(self): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table([]), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table([]), create=True): - libtmg.import_families(db, 1, per_no_map=None) - self.assertEqual(db.get_number_of_families(), 0) - - def test_marriage_event_sets_rel_type_married(self): - from gramps.gen.lib import EventType, FamilyRelType - import unittest.mock as mock - - db = _make_db() - pmap = {} - for per_no, gender in {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}.items(): - p = Person() - p.set_gender(gender) - with DbTxn("s", db) as t: - db.add_person(p, t) - pmap[per_no] = p.get_handle() - - ev = Event() - ev.set_type(EventType(EventType.MARRIAGE)) - with DbTxn("s", db) as t: - db.add_event(ev, t) - - tagtypes = [self._father_type(1), self._mother_type(2)] - pc = [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)] - event_handle_map = {99: (ev.get_handle(), 1, 2, 0)} - - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table(pc), create=True): - libtmg.import_families(db, 1, pmap, event_handle_map) - - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_relationship(), FamilyRelType.MARRIED) - self.assertEqual(len(fam.get_event_ref_list()), 1) - - -# --------------------------------------------------------------------------- -# import_repositories — name, type inference, URL, notes -# --------------------------------------------------------------------------- - -class TestImportRepositories(unittest.TestCase): - - def _run(self, repo_records, per_no_map=None, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgRepositories', _table(repo_records), create=True): - repo_map = libtmg.import_repositories(db, dataset, per_no_map) - return db, repo_map - - def _repo_rec(self, **kw): - defaults = dict(dsid=1, recno=1, name='City Library', - abbrev='', rnote='', rperno=0) - defaults.update(kw) - return _Rec(**defaults) - - def test_repository_created(self): - db, rmap = self._run([self._repo_rec()]) - self.assertEqual(db.get_number_of_repositories(), 1) - self.assertIn(1, rmap) - - def test_name_set(self): - db, rmap = self._run([self._repo_rec(name='National Archives')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(repo.get_name(), 'National Archives') - - def test_wrong_dataset_skipped(self): - db, rmap = self._run([self._repo_rec(dsid=2)], dataset=1) - self.assertEqual(db.get_number_of_repositories(), 0) - - def test_type_inferred_from_name(self): - from gramps.gen.lib import RepositoryType - db, rmap = self._run([self._repo_rec(name='ancestry.com')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(repo.get_type().value, RepositoryType.WEBSITE) - - def test_url_added_for_web_repo(self): - db, rmap = self._run([self._repo_rec(name='familysearch.org')]) - repo = db.get_repository_from_handle(rmap[1]) - urls = repo.get_url_list() - self.assertEqual(len(urls), 1) - self.assertIn('familysearch', urls[0].get_path()) - - def test_no_url_for_non_web_repo(self): - db, rmap = self._run([self._repo_rec(name='Local Parish Church')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(len(repo.get_url_list()), 0) - - def test_blank_name_falls_back_to_abbrev(self): - db, rmap = self._run([self._repo_rec(name='', abbrev='TNA')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(repo.get_name(), 'TNA') - - def test_note_added_when_rnote_set(self): - db, rmap = self._run([self._repo_rec(rnote='Open Mon-Fri')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(len(repo.get_note_list()), 1) - note = db.get_note_from_handle(repo.get_note_list()[0]) - self.assertIn('Open Mon-Fri', note.get()) - - def test_returns_recno_to_handle_map(self): - db, rmap = self._run([self._repo_rec(recno=42)]) - self.assertIn(42, rmap) - - -# --------------------------------------------------------------------------- -# import_citations — creation and attachment to events / persons -# --------------------------------------------------------------------------- - -class TestImportCitations(unittest.TestCase): - - def _run(self, citation_records, names_records=None, pc_records=None, - source_handle_map=None, event_handle_map=None, per_no_map=None, - dataset=1, db=None): - import unittest.mock as mock - if db is None: - db = _make_db() - with mock.patch('libtmg.tmgCitations', _table(citation_records), create=True), \ - mock.patch('libtmg.tmgNames', _table(names_records or []), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table(pc_records or []), create=True): - libtmg.import_citations(db, dataset, - source_handle_map=source_handle_map, - event_handle_map=event_handle_map, - per_no_map=per_no_map) - return db - - def _cit_rec(self, **kw): - defaults = dict(dsid=1, recno=1, majsource=1, stype='E', refrec=1, - exclude=False, subsource='', citref='', citmemo='', - sdsure='', snsure='', sssure='', spsure='', sfsure='') - defaults.update(kw) - return _Rec(**defaults) - - def test_no_source_map_is_noop(self): - db = self._run([self._cit_rec()]) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_citation_created(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec()], - source_handle_map={1: src.get_handle()}, db=db) - self.assertEqual(db.get_number_of_citations(), 1) - - def test_excluded_citation_skipped(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec(exclude=True)], - source_handle_map={1: src.get_handle()}, db=db) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_wrong_dataset_skipped(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec(dsid=2)], - source_handle_map={1: src.get_handle()}, db=db, dataset=1) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_unknown_source_skipped(self): - db = _make_db() - db = self._run([self._cit_rec(majsource=99)], - source_handle_map={1: 'some_handle'}, db=db) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_citation_attached_to_event(self): - db = _make_db() - src = Source() - ev = Event() - with DbTxn("s", db) as t: - db.add_source(src, t) - db.add_event(ev, t) - ev_handle = ev.get_handle() - - db = self._run([self._cit_rec(stype='E', refrec=7)], - source_handle_map={1: src.get_handle()}, - event_handle_map={7: (ev_handle, 1, 0, 0)}, - db=db) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(len(event.get_citation_list()), 1) - - def test_citation_attached_to_person_via_name(self): - db = _make_db() - src = Source() - p = Person() - with DbTxn("s", db) as t: - db.add_source(src, t) - db.add_person(p, t) - phandle = p.get_handle() - - # name recno=3 maps to nper=5; per_no_map routes nper=5 to phandle - db = self._run( - [self._cit_rec(stype='N', refrec=3)], - names_records=[_Rec(dsid=1, recno=3, nper=5)], - source_handle_map={1: src.get_handle()}, - per_no_map={5: phandle}, - db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_citation_list()), 1) - - def test_subsource_becomes_page(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec(subsource='p.42')], - source_handle_map={1: src.get_handle()}, db=db) - cit_handle = list(db.get_citation_handles())[0] - cit = db.get_citation_from_handle(cit_handle) - self.assertEqual(cit.get_page(), 'p.42') - - - -# ── TmgProject._read_pjc_config ────────────────────────────────────────── - -def _make_minimal_sqz(tmp_dir, pjc_content): - """Create a minimal .SQZ zip containing only a PJC file.""" - import zipfile as _zf - pjc_path = os.path.join(tmp_dir, 'test.pjc') - sqz_path = os.path.join(tmp_dir, 'test.sqz') - with open(pjc_path, 'w', encoding='latin-1') as f: - f.write(pjc_content) - with _zf.ZipFile(sqz_path, 'w') as zf: - zf.write(pjc_path, 'test.pjc') - return sqz_path - - -class _MockUser: - """Minimal stand-in for the Gramps user object used in importData.""" - def __init__(self): - self.error_shown = False - self.error_message = None - self.uistate = None - - def notify_error(self, title, message=''): - self.error_shown = True - self.error_message = message - - def begin_progress(self, *a, **kw): pass - def end_progress(self): pass - def step_progress(self): pass - - -class TestReadPjcConfig(unittest.TestCase): - """Tests for TmgProject._read_pjc_config PJC parsing.""" - - _MINIMAL_PJC = ( - "[Stamp]\n" - "PjcVersion=11.0\n" - "[Researcher]\n" - "Name=Test User\n" - ) - - def _make_project(self, pjc_content, tmp_path): - pjc_file = os.path.join(tmp_path, "test.pjc") - with open(pjc_file, 'w', encoding='latin-1') as f: - f.write(pjc_content) - return libtmg.TmgProject(pjc_file) - - def test_well_formed_pjc_returns_version(self): - """A clean PJC file parses successfully and version() returns a float.""" - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(self._MINIMAL_PJC, tmp) - self.assertEqual(project.version(), 11.0) - - def test_malformed_section_header_does_not_raise(self): - """Lines like '[Exho' (no closing bracket) are silently dropped.""" - pjc = ( - "[Stamp]\n" - "PjcVersion=11.0\n" - "[Exho\n" # malformed — the crash trigger - "SomeGarbage\n" - "[Researcher]\n" - "Name=Test User\n" - ) - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(pjc, tmp) - # Must not raise; must still find [Stamp] - self.assertEqual(project.version(), 11.0) - - def test_null_bytes_stripped(self): - """NUL bytes in the PJC file are stripped before parsing.""" - pjc = "[Stamp]\x00\nPjcVersion=11.0\n" - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(pjc, tmp) - self.assertEqual(project.version(), 11.0) - - def test_parse_error_returns_partial_config(self): - """If configparser still raises after filtering, a warning is logged - and a (possibly empty) config object is returned rather than crashing.""" - import logging - # Feed content that survives the filter but still breaks configparser: - # a key=value line before any section header is technically invalid. - pjc = "orphan_key=value\n[Stamp]\nPjcVersion=11.0\n" - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(pjc, tmp) - with self.assertLogs('.TMGImport', level=logging.WARNING) as cm: - cfg = project._read_pjc_config() - self.assertTrue(any('parse error' in m.lower() or 'parsing' in m.lower() - for m in cm.output)) - # Config object is returned (not None), even if incomplete - self.assertIsNotNone(cfg) - - def test_version_too_old_notifies_user(self): - """A PJC version < 11.0 calls user.notify_error and aborts import.""" - pjc = "[Stamp]\nPjcVersion=10.0\n" # TMG 9.01 or earlier - with tempfile.TemporaryDirectory() as tmp: - sqz = _make_minimal_sqz(tmp, pjc) - db = _make_db() - user = _MockUser() - libtmg.importData(db, sqz, user) - self.assertTrue(user.error_shown, - "notify_error should have been called for old version") - self.assertIn('9.05', user.error_message or '', - "Error message should mention TMG 9.05") - - def test_missing_pjc_version_notifies_user(self): - """A PJC with no [Stamp]/PjcVersion calls user.notify_error.""" - pjc = "[OtherSection]\nSomeKey=value\n" # no [Stamp] at all - with tempfile.TemporaryDirectory() as tmp: - sqz = _make_minimal_sqz(tmp, pjc) - db = _make_db() - user = _MockUser() - libtmg.importData(db, sqz, user) - self.assertTrue(user.error_shown, - "notify_error should have been called for missing version") - if __name__ == '__main__': unittest.main() diff --git a/TMGimporter/tests/test_linux_libtmg.py b/TMGimporter/tests/test_linux_libtmg.py new file mode 100644 index 000000000..35bd11717 --- /dev/null +++ b/TMGimporter/tests/test_linux_libtmg.py @@ -0,0 +1,1218 @@ +"""Linux-only unit tests for libtmg.py + +Split off from test_libtmg.py: every class in this module creates an +in-memory Gramps SQLite database via + + make_database("sqlite").load(":memory:", None) + +which currently hangs on Windows under the conda-forge GTK + pip Gramps +combination used by the CI unit-test-windows job. Pure-logic tests that +do not touch the Gramps DB layer stay in test_libtmg.py and run on every +OS. + +Filename convention (see .github/workflows/ci.yml): + test_*.py general (every OS) + test_linux_*.py Linux-only + test_windows_*.py Windows-only + test_integration_*.py Linux-only, full-pipeline/DB-backed +""" + +import sys +import os +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import libtmg + +from gramps.gen.lib import Date, Event, NoteType, Person, Place, Source +from gramps.gen.db.utils import make_database +from gramps.gen.db import DbTxn + + +# --------------------------------------------------------------------------- +# Helpers shared across test cases +# --------------------------------------------------------------------------- + +class _Rec: + """Minimal fake DBF record — set any field via keyword arguments.""" + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +def _table(records): + """Return an object that behaves like a dbf.Table used as a context manager. + + libtmg uses tables in two ways: + with tmgFoo: + for record in tmgFoo: # iterates over context-managed table + """ + class _FakeTable: + def __enter__(self): + return self + def __exit__(self, *_): + return False + def __iter__(self): + return iter(records) + return _FakeTable() + + +def _make_db(): + """Return a fresh in-memory Gramps database.""" + db = make_database("sqlite") + db.load(":memory:", None) + return db + + +def _add_person(db): + """Add an empty Person to db and return (db, handle).""" + p = Person() + with DbTxn("setup", db) as t: + db.add_person(p, t) + return db, p.get_handle() + + +# --------------------------------------------------------------------------- +# import_notes — per-person note creation from tmg events +# --------------------------------------------------------------------------- + +class TestImportNotes(unittest.TestCase): + + def _patch(self, tagtypes_records, events_records): + """Patch libtmg globals and return a context manager.""" + import unittest.mock as mock + patches = [ + mock.patch.object(libtmg, 'tmgTagTypes', _table(tagtypes_records)), + mock.patch.object(libtmg, 'tmgEvents', _table(events_records)), + ] + return patches + + def _run(self, tagtypes_records, events_records, per_no_map, dataset=1, db=None): + import unittest.mock as mock + if db is None: + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ + mock.patch('libtmg.tmgEvents', _table(events_records), create=True): + libtmg.import_notes(db, dataset, per_no_map) + return db + + def test_no_per_no_map_is_noop(self): + """Passing per_no_map=None must not touch any table.""" + import unittest.mock as mock + db = _make_db() + mock_table = mock.MagicMock() + with mock.patch('libtmg.tmgTagTypes', mock_table, create=True), \ + mock.patch('libtmg.tmgEvents', mock_table, create=True): + libtmg.import_notes(db, 1, per_no_map=None) + mock_table.__enter__.assert_not_called() + + def test_note_attached_to_person(self): + db, phandle = _add_person(_make_db()) + per_no_map = {42: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=42, recno=1, + efoot='Born in London')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_note_list()), 1) + + def test_note_text_stored(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=10, etypename='Note')], + events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, + efoot=' Some note text ')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + note = db.get_note_from_handle(person.get_note_list()[0]) + self.assertEqual(note.get(), 'Some note text') + + def test_note_type_is_person(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='hello')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + note = db.get_note_from_handle(person.get_note_list()[0]) + self.assertEqual(note.get_type(), NoteType.PERSON) + + def test_tmg_codes_stripped_from_note(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='[:ITAL:]italicised[:ITAL:]')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + note = db.get_note_from_handle(person.get_note_list()[0]) + self.assertEqual(note.get(), 'italicised') + + def test_empty_note_text_skipped(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_note_list()), 0) + + def test_unknown_person_skipped(self): + db = _make_db() + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=99, recno=1, + efoot='orphan note')], + per_no_map={}, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + def test_non_note_etype_ignored(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + # etype=10 is not a Note type + events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, + efoot='should be ignored')], + per_no_map=per_no_map, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + def test_wrong_dataset_ignored(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=2, etype=77, per1=1, recno=1, + efoot='wrong dataset')], + per_no_map=per_no_map, dataset=1, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + def test_multiple_notes_for_one_person(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[ + _Rec(dsid=1, etype=77, per1=1, recno=1, efoot='first'), + _Rec(dsid=1, etype=77, per1=1, recno=2, efoot='second'), + ], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_note_list()), 2) + + def test_no_note_tag_type_defined(self): + """If the dataset has no 'Note' tag type, nothing is imported.""" + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[], # no tag types at all + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='orphan')], + per_no_map=per_no_map, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + +# --------------------------------------------------------------------------- +# trial_events — event import and Note-etype skip +# --------------------------------------------------------------------------- + +# A minimal raw date string for an exact date (1900-06-15) +_EXACT_DATE = '1' + '19000615' + '0' + '3' + '00000000' + '0' + '0' +_EMPTY_DATE = '' + + +class TestTrialEvents(unittest.TestCase): + + def _run(self, tagtypes_records, events_records, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ + mock.patch('libtmg.tmgEvents', _table(events_records), create=True): + handle_map = libtmg.import_events(db, dataset) + return db, handle_map + + def test_regular_event_creates_db_entry(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='')] + db, hmap = self._run(_tagtypes, _events) + self.assertEqual(db.get_number_of_events(), 1) + self.assertIn(1, hmap) + + def test_handle_map_tuple_has_four_elements(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=5, etype=10, per1=3, per2=0, + placenum=7, edate=_EMPTY_DATE, efoot='')] + _, hmap = self._run(_tagtypes, _events) + entry = hmap[5] + self.assertEqual(len(entry), 4) + _handle, per1, per2, placenum = entry + self.assertEqual(per1, 3) + self.assertEqual(per2, 0) + self.assertEqual(placenum, 7) + + def test_note_etype_event_not_in_handle_map(self): + _tagtypes = [_Rec(dsid=1, etypenum=77, etypename='Note')] + _events = [_Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='a note')] + db, hmap = self._run(_tagtypes, _events) + self.assertEqual(db.get_number_of_events(), 0) + self.assertNotIn(1, hmap) + + def test_event_memo_stored_as_description(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='born here')] + db, hmap = self._run(_tagtypes, _events) + event = db.get_event_from_handle(hmap[1][0]) + self.assertEqual(event.get_description(), 'born here') + + def test_event_memo_tmg_codes_stripped(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, + efoot='[:CR:]born here')] + db, hmap = self._run(_tagtypes, _events) + event = db.get_event_from_handle(hmap[1][0]) + self.assertEqual(event.get_description(), 'born here') + + def test_event_date_set(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EXACT_DATE, efoot='')] + db, hmap = self._run(_tagtypes, _events) + event = db.get_event_from_handle(hmap[1][0]) + d = event.get_date_object() + self.assertEqual(d.get_year(), 1900) + self.assertEqual(d.get_month(), 6) + + def test_wrong_dataset_skipped(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=2, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='')] + db, hmap = self._run(_tagtypes, _events, dataset=1) + self.assertEqual(db.get_number_of_events(), 0) + + def test_mixed_note_and_regular_events(self): + _tagtypes = [ + _Rec(dsid=1, etypenum=77, etypename='Note'), + _Rec(dsid=1, etypenum=10, etypename='Birth'), + ] + _events = [ + _Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='a note'), + _Rec(dsid=1, recno=2, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot=''), + ] + db, hmap = self._run(_tagtypes, _events) + self.assertEqual(db.get_number_of_events(), 1) + self.assertNotIn(1, hmap) + self.assertIn(2, hmap) + + +# --------------------------------------------------------------------------- +# import_sources — info-field parsing and author/publication split +# --------------------------------------------------------------------------- + +class TestImportSources(unittest.TestCase): + + def _run(self, src_components, src_repo_links, sources_records, + repo_handle_map=None, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgSourceComponents', _table(src_components), create=True), \ + mock.patch('libtmg.tmgSourceRepositoryLinks', _table(src_repo_links), create=True), \ + mock.patch('libtmg.tmgSources', _table(sources_records), create=True): + smap = libtmg.import_sources(db, dataset, repo_handle_map) + return db, smap + + def _source_rec(self, **kw): + defaults = dict(dsid=1, majnum=1, mactive=True, + title='Test Source', abbrev='', info='', + text='', fform='', sform='', bform='', reminders='') + defaults.update(kw) + return _Rec(**defaults) + + def test_source_created(self): + db, smap = self._run([], [], [self._source_rec()]) + self.assertEqual(db.get_number_of_sources(), 1) + self.assertIn(1, smap) + + def test_title_set(self): + db, smap = self._run([], [], [self._source_rec(title='My Source')]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_title(), 'My Source') + + def test_abbreviation_set(self): + db, smap = self._run([], [], [self._source_rec(abbrev='MySrc')]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_abbreviation(), 'MySrc') + + def test_inactive_source_skipped(self): + db, smap = self._run([], [], [self._source_rec(mactive=False)]) + self.assertEqual(db.get_number_of_sources(), 0) + + def test_wrong_dataset_skipped(self): + db, smap = self._run([], [], [self._source_rec(dsid=2)], dataset=1) + self.assertEqual(db.get_number_of_sources(), 0) + + def test_author_element_sets_author(self): + # recno 1 → position 0 in $!& split + components = [_Rec(recno=1, element='[AUTHOR]')] + rec = self._source_rec(info='John Smith') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_author(), 'John Smith') + + def test_non_author_element_sets_publication_info(self): + components = [_Rec(recno=1, element='[TITLE]')] + rec = self._source_rec(info='Some Title') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertIn('TITLE', src.get_publication_info()) + self.assertIn('Some Title', src.get_publication_info()) + + def test_multiple_authors_joined_with_semicolon(self): + # positions 0 and 1 → recno 1 and 2 + components = [ + _Rec(recno=1, element='[AUTHOR]'), + _Rec(recno=2, element='[EDITOR]'), + ] + rec = self._source_rec(info='Alice$!&Bob') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_author(), 'Alice; Bob') + + def test_empty_info_position_skipped(self): + # position 0 empty, position 1 filled → recno 2 = [AUTHOR] + components = [ + _Rec(recno=1, element='[TITLE]'), + _Rec(recno=2, element='[AUTHOR]'), + ] + rec = self._source_rec(info='$!&Jane Doe') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_author(), 'Jane Doe') + self.assertEqual(src.get_publication_info(), '') + + def test_note_fields_become_notes(self): + rec = self._source_rec(text='original text', fform='', sform='', bform='') + db, smap = self._run([], [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(len(src.get_note_list()), 1) + note = db.get_note_from_handle(src.get_note_list()[0]) + self.assertIn('original text', note.get()) + + def test_multiple_note_fields_each_become_a_note(self): + rec = self._source_rec(text='txt', fform='fn', sform='', bform='') + db, smap = self._run([], [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(len(src.get_note_list()), 2) + + +# --------------------------------------------------------------------------- +# import_places — name reconstruction, type resolution, note parts +# --------------------------------------------------------------------------- + +class TestImportPlaces(unittest.TestCase): + + def _run(self, part_types, place_dict, ppv_records, places_records, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgPlacePartType', _table(part_types), create=True), \ + mock.patch('libtmg.tmgPlaceDictionary', _table(place_dict), create=True), \ + mock.patch('libtmg.tmgPlacePartValue', _table(ppv_records), create=True), \ + mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): + pmap = libtmg.import_places(db, dataset) + return db, pmap + + # Convenience: build part_type, place_dict, ppv records for a single place + def _setup(self, recno, parts, dataset=1, comment='', shortplace=''): + """ + parts: list of (label, value) e.g. [('City','London'),('Country','UK')] + Returns (part_type_recs, place_dict_recs, ppv_recs, place_recs) + """ + part_type_recs = [] + place_dict_recs = [] + ppv_recs = [] + for i, (label, value) in enumerate(parts): + type_id = i + 1 + uid = i + 100 + part_type_recs.append(_Rec(type=type_id, value=label)) + place_dict_recs.append(_Rec(uid=uid, value=value)) + ppv_recs.append(_Rec(dsid=dataset, recno=recno, type=type_id, uid=uid)) + place_recs = [_Rec(dsid=dataset, recno=recno, + shortplace=shortplace, comment=comment)] + return part_type_recs, place_dict_recs, ppv_recs, place_recs + + def test_city_only_name_and_type(self): + pt, pd, ppv, pl = self._setup(1, [('City', 'London')]) + db, pmap = self._run(pt, pd, ppv, pl) + self.assertIn(1, pmap) + place = db.get_place_from_handle(pmap[1]) + self.assertEqual(place.get_name().get_value(), 'London') + from gramps.gen.lib import PlaceType + self.assertEqual(place.get_type().value, PlaceType.CITY) + + def test_country_only_name_and_type(self): + pt, pd, ppv, pl = self._setup(1, [('Country', 'France')]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + from gramps.gen.lib import PlaceType + self.assertEqual(place.get_type().value, PlaceType.COUNTRY) + + def test_city_state_country_name_order(self): + pt, pd, ppv, pl = self._setup(1, [ + ('City', 'Paris'), ('State', 'Île-de-France'), ('Country', 'France') + ]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + # GEO_ORDER: Addressee, Detail, City, County, State, Country + self.assertEqual(place.get_name().get_value(), + 'Paris, Île-de-France, France') + + def test_most_specific_type_wins(self): + # City is more specific than Country in _GEO_ORDER + pt, pd, ppv, pl = self._setup(1, [ + ('City', 'Berlin'), ('Country', 'Germany') + ]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + from gramps.gen.lib import PlaceType + self.assertEqual(place.get_type().value, PlaceType.CITY) + + def test_empty_place_skipped(self): + # No parts and no shortplace → nothing imported + place_recs = [_Rec(dsid=1, recno=1, shortplace='', comment='')] + db, pmap = self._run([], [], [], place_recs) + self.assertEqual(db.get_number_of_places(), 0) + self.assertNotIn(1, pmap) + + def test_shortplace_fallback(self): + # No parts but shortplace set → use it + place_recs = [_Rec(dsid=1, recno=1, shortplace='Somewhere', comment='')] + db, pmap = self._run([], [], [], place_recs) + self.assertIn(1, pmap) + place = db.get_place_from_handle(pmap[1]) + self.assertEqual(place.get_name().get_value(), 'Somewhere') + + def test_note_parts_go_to_note(self): + pt, pd, ppv, pl = self._setup(1, [ + ('City', 'Rome'), ('Postal', '00100') + ]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + self.assertEqual(len(place.get_note_list()), 1) + note = db.get_note_from_handle(place.get_note_list()[0]) + self.assertIn('Postal', note.get()) + self.assertIn('00100', note.get()) + + def test_comment_goes_to_note(self): + pt, pd, ppv, pl = self._setup(1, [('City', 'Rome')], comment='see also') + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + note = db.get_note_from_handle(place.get_note_list()[0]) + self.assertIn('see also', note.get()) + + def test_wrong_dataset_skipped(self): + pt, pd, ppv, pl = self._setup(1, [('City', 'Oslo')], dataset=2) + db, pmap = self._run(pt, pd, ppv, pl, dataset=1) + self.assertEqual(db.get_number_of_places(), 0) + + def test_returns_recno_to_handle_map(self): + pt, pd, ppv, pl = self._setup(42, [('City', 'Vienna')]) + db, pmap = self._run(pt, pd, ppv, pl) + self.assertIn(42, pmap) + + +# --------------------------------------------------------------------------- +# link_event_places — event gets its place handle set +# --------------------------------------------------------------------------- + +class TestLinkEventPlaces(unittest.TestCase): + + def _make_event(self, db): + from gramps.gen.db import DbTxn + ev = Event() + with DbTxn("setup", db) as t: + db.add_event(ev, t) + return ev.get_handle() + + def _make_place(self, db): + from gramps.gen.db import DbTxn + pl = Place() + with DbTxn("setup", db) as t: + db.add_place(pl, t) + return pl.get_handle() + + def test_place_linked_to_event(self): + db = _make_db() + ev_handle = self._make_event(db) + pl_handle = self._make_place(db) + event_handle_map = {1: (ev_handle, 1, 0, 7)} + place_handle_map = {7: pl_handle} + libtmg.link_event_places(db, event_handle_map, place_handle_map) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(event.get_place_handle(), pl_handle) + + def test_zero_placenum_skipped(self): + db = _make_db() + ev_handle = self._make_event(db) + event_handle_map = {1: (ev_handle, 1, 0, 0)} + place_handle_map = {0: self._make_place(db)} + libtmg.link_event_places(db, event_handle_map, place_handle_map) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(event.get_place_handle(), '') + + def test_unknown_placenum_skipped(self): + db = _make_db() + ev_handle = self._make_event(db) + event_handle_map = {1: (ev_handle, 1, 0, 99)} + libtmg.link_event_places(db, event_handle_map, {}) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(event.get_place_handle(), '') + + def test_empty_maps_noop(self): + db = _make_db() + libtmg.link_event_places(db, {}, {}) # must not raise + libtmg.link_event_places(db, None, None) + + +# --------------------------------------------------------------------------- +# Pure functions: num_to_month, num_to_date, parse_date +# --------------------------------------------------------------------------- +class TestShortPlaceName(unittest.TestCase): + + def _run(self, places_records, placenum, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): + return libtmg.short_place_name(db, placenum, dataset) + + def test_returns_shortplace(self): + rec = _Rec(dsid=1, recno=5, shortplace='New York ', styleid=1, comment='') + self.assertEqual(self._run([rec], placenum=5), 'New York') + + def test_trailing_whitespace_stripped(self): + rec = _Rec(dsid=1, recno=1, shortplace='London ', styleid=1, comment='') + self.assertEqual(self._run([rec], placenum=1), 'London') + + def test_wrong_recno_returns_none(self): + rec = _Rec(dsid=1, recno=1, shortplace='Paris', styleid=1, comment='') + self.assertIsNone(self._run([rec], placenum=99)) + + def test_wrong_dataset_returns_none(self): + rec = _Rec(dsid=2, recno=1, shortplace='Berlin', styleid=1, comment='') + self.assertIsNone(self._run([rec], placenum=1, dataset=1)) + + +class TestTagTypeName(unittest.TestCase): + + def _run(self, tagtypes_records, eventtype, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True): + return libtmg.tag_type_name(db, eventtype, dataset) + + def test_returns_name(self): + rec = _Rec(dsid=1, etypenum=2, etypename='Birth ') + self.assertEqual(self._run([rec], eventtype=2), 'Birth') + + def test_trailing_whitespace_stripped(self): + rec = _Rec(dsid=1, etypenum=3, etypename='Death ') + self.assertEqual(self._run([rec], eventtype=3), 'Death') + + def test_wrong_eventtype_returns_none(self): + rec = _Rec(dsid=1, etypenum=2, etypename='Birth') + self.assertIsNone(self._run([rec], eventtype=99)) + + def test_wrong_dataset_returns_none(self): + rec = _Rec(dsid=2, etypenum=2, etypename='Birth') + self.assertIsNone(self._run([rec], eventtype=2, dataset=1)) + + +# --------------------------------------------------------------------------- +# import_people — name parsing, gender, dataset filter +# --------------------------------------------------------------------------- + +class TestImportPeople(unittest.TestCase): + + def _run(self, names_records, people_records, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgNames', _table(names_records), create=True), \ + mock.patch('libtmg.tmgPeople', _table(people_records), create=True): + per_no_map = libtmg.import_people(db, dataset) + return db, per_no_map + + def _name_rec(self, **kw): + defaults = dict(dsid=1, nper=1, primary=True, srnamedisp='SMITH, John') + defaults.update(kw) + return _Rec(**defaults) + + def _person_rec(self, **kw): + defaults = dict(dsid=1, per_no=1, sex='M') + defaults.update(kw) + return _Rec(**defaults) + + def test_person_created(self): + db, pmap = self._run([self._name_rec()], [self._person_rec()]) + self.assertEqual(db.get_number_of_people(), 1) + + def test_returns_per_no_map(self): + db, pmap = self._run([self._name_rec(nper=5)], [self._person_rec(per_no=5)]) + self.assertIn(5, pmap) + + def test_surname_parsed(self): + db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], + [self._person_rec(per_no=1)]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_primary_name().get_surname(), 'JONES') + + def test_given_name_parsed(self): + db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], + [self._person_rec(per_no=1)]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_primary_name().get_first_name(), 'Alice') + + def test_male_gender(self): + db, pmap = self._run([self._name_rec()], [self._person_rec(sex='M')]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_gender(), Person.MALE) + + def test_female_gender(self): + db, pmap = self._run([self._name_rec()], [self._person_rec(sex='F')]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_gender(), Person.FEMALE) + + def test_unknown_gender(self): + db, pmap = self._run([self._name_rec()], [self._person_rec(sex='?')]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_gender(), Person.UNKNOWN) + + def test_non_primary_name_skipped(self): + db, pmap = self._run( + [self._name_rec(primary=False, srnamedisp='ALT, Name')], + [self._person_rec()] + ) + self.assertEqual(db.get_number_of_people(), 0) + + def test_wrong_dataset_skipped(self): + db, pmap = self._run([self._name_rec(dsid=2)], [self._person_rec(dsid=2)], + dataset=1) + self.assertEqual(db.get_number_of_people(), 0) + + def test_no_comma_surname_only(self): + # srnamedisp with no comma → surname=full string, given='' + db, pmap = self._run([self._name_rec(srnamedisp='SMITH')], + [self._person_rec()]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_primary_name().get_surname(), 'SMITH') + self.assertEqual(p.get_primary_name().get_first_name(), '') + + +# --------------------------------------------------------------------------- +# link_person_events — EventRefs, birth/death special refs +# --------------------------------------------------------------------------- + +class TestLinkPersonEvents(unittest.TestCase): + + def _make_typed_event(self, db, event_type_int): + from gramps.gen.lib import EventType + ev = Event() + ev.set_type(EventType(event_type_int)) + with DbTxn("setup", db) as t: + db.add_event(ev, t) + return ev.get_handle() + + def test_individual_event_linked_to_person(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.OCCUPATION) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + p = db.get_person_from_handle(phandle) + self.assertEqual(len(p.get_event_ref_list()), 1) + + def test_couple_event_not_linked_to_person(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.MARRIAGE) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 2, 0)}) + p = db.get_person_from_handle(phandle) + self.assertEqual(len(p.get_event_ref_list()), 0) + + def test_birth_event_sets_birth_ref(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.BIRTH) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + p = db.get_person_from_handle(phandle) + self.assertIsNotNone(p.get_birth_ref()) + self.assertEqual(p.get_birth_ref().ref, ev_handle) + + def test_death_event_sets_death_ref(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.DEATH) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + p = db.get_person_from_handle(phandle) + self.assertIsNotNone(p.get_death_ref()) + + def test_unknown_person_skipped(self): + from gramps.gen.lib import EventType + db = _make_db() + ev_handle = self._make_typed_event(db, EventType.BIRTH) + # Must not raise even when per1 has no entry in per_no_map + libtmg.link_person_events(db, + per_no_map={}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + + def test_empty_maps_noop(self): + db = _make_db() + libtmg.link_person_events(db, None, None) + libtmg.link_person_events(db, {}, {}) + + +# --------------------------------------------------------------------------- +# import_families — parent-child grouping, couple events, rel type +# --------------------------------------------------------------------------- + +class TestImportFamilies(unittest.TestCase): + + def _run(self, tagtypes, pc_rels, per_no_by_gender, event_handle_map=None, + dataset=1): + """Create persons from per_no_by_gender={per_no: gender}, run import.""" + import unittest.mock as mock + db = _make_db() + pmap = {} + for per_no, gender in per_no_by_gender.items(): + p = Person() + p.set_gender(gender) + with DbTxn("setup", db) as t: + db.add_person(p, t) + pmap[per_no] = p.get_handle() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table(pc_rels), create=True): + libtmg.import_families(db, dataset, pmap, event_handle_map) + return db, pmap + + def _pc(self, parent, child, ptype, primary=True, pnote='', dsid=1): + return _Rec(dsid=dsid, parent=parent, child=child, + ptype=ptype, primary=primary, pnote=pnote) + + def _father_type(self, num=1): + return _Rec(dsid=1, etypenum=num, etypename='Father-Biological') + + def _mother_type(self, num=2): + return _Rec(dsid=1, etypenum=num, etypename='Mother-Biological') + + def test_father_child_creates_family(self): + db, pmap = self._run([self._father_type()], + [self._pc(1, 2, ptype=1)], + {1: Person.MALE, 2: Person.UNKNOWN}) + self.assertEqual(db.get_number_of_families(), 1) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_father_handle(), pmap[1]) + + def test_mother_child_creates_family(self): + db, pmap = self._run([self._mother_type()], + [self._pc(1, 2, ptype=2)], + {1: Person.FEMALE, 2: Person.UNKNOWN}) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_mother_handle(), pmap[1]) + + def test_father_and_mother_same_family(self): + db, pmap = self._run( + [self._father_type(1), self._mother_type(2)], + [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)], + {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}, + ) + self.assertEqual(db.get_number_of_families(), 1) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_father_handle(), pmap[1]) + self.assertEqual(fam.get_mother_handle(), pmap[2]) + + def test_child_added_to_family(self): + db, pmap = self._run([self._father_type()], + [self._pc(1, 2, ptype=1)], + {1: Person.MALE, 2: Person.UNKNOWN}) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(len(fam.get_child_ref_list()), 1) + self.assertEqual(fam.get_child_ref_list()[0].ref, pmap[2]) + + def test_child_ref_type_biological(self): + from gramps.gen.lib import ChildRefType + db, pmap = self._run([self._father_type()], + [self._pc(1, 2, ptype=1)], + {1: Person.MALE, 2: Person.UNKNOWN}) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_child_ref_list()[0].get_father_relation(), + ChildRefType.BIRTH) + + def test_wrong_dataset_skipped(self): + db, _ = self._run([self._father_type()], + [self._pc(1, 2, ptype=1, dsid=2)], + {1: Person.MALE, 2: Person.UNKNOWN}, dataset=1) + self.assertEqual(db.get_number_of_families(), 0) + + def test_no_per_no_map_is_noop(self): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table([]), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table([]), create=True): + libtmg.import_families(db, 1, per_no_map=None) + self.assertEqual(db.get_number_of_families(), 0) + + def test_marriage_event_sets_rel_type_married(self): + from gramps.gen.lib import EventType, FamilyRelType + import unittest.mock as mock + + db = _make_db() + pmap = {} + for per_no, gender in {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}.items(): + p = Person() + p.set_gender(gender) + with DbTxn("s", db) as t: + db.add_person(p, t) + pmap[per_no] = p.get_handle() + + ev = Event() + ev.set_type(EventType(EventType.MARRIAGE)) + with DbTxn("s", db) as t: + db.add_event(ev, t) + + tagtypes = [self._father_type(1), self._mother_type(2)] + pc = [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)] + event_handle_map = {99: (ev.get_handle(), 1, 2, 0)} + + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table(pc), create=True): + libtmg.import_families(db, 1, pmap, event_handle_map) + + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_relationship(), FamilyRelType.MARRIED) + self.assertEqual(len(fam.get_event_ref_list()), 1) + + +# --------------------------------------------------------------------------- +# import_repositories — name, type inference, URL, notes +# --------------------------------------------------------------------------- + +class TestImportRepositories(unittest.TestCase): + + def _run(self, repo_records, per_no_map=None, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgRepositories', _table(repo_records), create=True): + repo_map = libtmg.import_repositories(db, dataset, per_no_map) + return db, repo_map + + def _repo_rec(self, **kw): + defaults = dict(dsid=1, recno=1, name='City Library', + abbrev='', rnote='', rperno=0) + defaults.update(kw) + return _Rec(**defaults) + + def test_repository_created(self): + db, rmap = self._run([self._repo_rec()]) + self.assertEqual(db.get_number_of_repositories(), 1) + self.assertIn(1, rmap) + + def test_name_set(self): + db, rmap = self._run([self._repo_rec(name='National Archives')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(repo.get_name(), 'National Archives') + + def test_wrong_dataset_skipped(self): + db, rmap = self._run([self._repo_rec(dsid=2)], dataset=1) + self.assertEqual(db.get_number_of_repositories(), 0) + + def test_type_inferred_from_name(self): + from gramps.gen.lib import RepositoryType + db, rmap = self._run([self._repo_rec(name='ancestry.com')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(repo.get_type().value, RepositoryType.WEBSITE) + + def test_url_added_for_web_repo(self): + db, rmap = self._run([self._repo_rec(name='familysearch.org')]) + repo = db.get_repository_from_handle(rmap[1]) + urls = repo.get_url_list() + self.assertEqual(len(urls), 1) + self.assertIn('familysearch', urls[0].get_path()) + + def test_no_url_for_non_web_repo(self): + db, rmap = self._run([self._repo_rec(name='Local Parish Church')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(len(repo.get_url_list()), 0) + + def test_blank_name_falls_back_to_abbrev(self): + db, rmap = self._run([self._repo_rec(name='', abbrev='TNA')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(repo.get_name(), 'TNA') + + def test_note_added_when_rnote_set(self): + db, rmap = self._run([self._repo_rec(rnote='Open Mon-Fri')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(len(repo.get_note_list()), 1) + note = db.get_note_from_handle(repo.get_note_list()[0]) + self.assertIn('Open Mon-Fri', note.get()) + + def test_returns_recno_to_handle_map(self): + db, rmap = self._run([self._repo_rec(recno=42)]) + self.assertIn(42, rmap) + + +# --------------------------------------------------------------------------- +# import_citations — creation and attachment to events / persons +# --------------------------------------------------------------------------- + +class TestImportCitations(unittest.TestCase): + + def _run(self, citation_records, names_records=None, pc_records=None, + source_handle_map=None, event_handle_map=None, per_no_map=None, + dataset=1, db=None): + import unittest.mock as mock + if db is None: + db = _make_db() + with mock.patch('libtmg.tmgCitations', _table(citation_records), create=True), \ + mock.patch('libtmg.tmgNames', _table(names_records or []), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table(pc_records or []), create=True): + libtmg.import_citations(db, dataset, + source_handle_map=source_handle_map, + event_handle_map=event_handle_map, + per_no_map=per_no_map) + return db + + def _cit_rec(self, **kw): + defaults = dict(dsid=1, recno=1, majsource=1, stype='E', refrec=1, + exclude=False, subsource='', citref='', citmemo='', + sdsure='', snsure='', sssure='', spsure='', sfsure='') + defaults.update(kw) + return _Rec(**defaults) + + def test_no_source_map_is_noop(self): + db = self._run([self._cit_rec()]) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_citation_created(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec()], + source_handle_map={1: src.get_handle()}, db=db) + self.assertEqual(db.get_number_of_citations(), 1) + + def test_excluded_citation_skipped(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec(exclude=True)], + source_handle_map={1: src.get_handle()}, db=db) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_wrong_dataset_skipped(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec(dsid=2)], + source_handle_map={1: src.get_handle()}, db=db, dataset=1) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_unknown_source_skipped(self): + db = _make_db() + db = self._run([self._cit_rec(majsource=99)], + source_handle_map={1: 'some_handle'}, db=db) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_citation_attached_to_event(self): + db = _make_db() + src = Source() + ev = Event() + with DbTxn("s", db) as t: + db.add_source(src, t) + db.add_event(ev, t) + ev_handle = ev.get_handle() + + db = self._run([self._cit_rec(stype='E', refrec=7)], + source_handle_map={1: src.get_handle()}, + event_handle_map={7: (ev_handle, 1, 0, 0)}, + db=db) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(len(event.get_citation_list()), 1) + + def test_citation_attached_to_person_via_name(self): + db = _make_db() + src = Source() + p = Person() + with DbTxn("s", db) as t: + db.add_source(src, t) + db.add_person(p, t) + phandle = p.get_handle() + + # name recno=3 maps to nper=5; per_no_map routes nper=5 to phandle + db = self._run( + [self._cit_rec(stype='N', refrec=3)], + names_records=[_Rec(dsid=1, recno=3, nper=5)], + source_handle_map={1: src.get_handle()}, + per_no_map={5: phandle}, + db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_citation_list()), 1) + + def test_subsource_becomes_page(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec(subsource='p.42')], + source_handle_map={1: src.get_handle()}, db=db) + cit_handle = list(db.get_citation_handles())[0] + cit = db.get_citation_from_handle(cit_handle) + self.assertEqual(cit.get_page(), 'p.42') + + + +# ── TmgProject._read_pjc_config ────────────────────────────────────────── +def _make_minimal_sqz(tmp_dir, pjc_content): + """Create a minimal .SQZ zip containing only a PJC file.""" + import zipfile as _zf + pjc_path = os.path.join(tmp_dir, 'test.pjc') + sqz_path = os.path.join(tmp_dir, 'test.sqz') + with open(pjc_path, 'w', encoding='latin-1') as f: + f.write(pjc_content) + with _zf.ZipFile(sqz_path, 'w') as zf: + zf.write(pjc_path, 'test.pjc') + return sqz_path + + +class _MockUser: + """Minimal stand-in for the Gramps user object used in importData.""" + def __init__(self): + self.error_shown = False + self.error_message = None + self.uistate = None + + def notify_error(self, title, message=''): + self.error_shown = True + self.error_message = message + + def begin_progress(self, *a, **kw): pass + def end_progress(self): pass + def step_progress(self): pass + + +class TestReadPjcConfig(unittest.TestCase): + """Tests for TmgProject._read_pjc_config PJC parsing.""" + + _MINIMAL_PJC = ( + "[Stamp]\n" + "PjcVersion=11.0\n" + "[Researcher]\n" + "Name=Test User\n" + ) + + def _make_project(self, pjc_content, tmp_path): + pjc_file = os.path.join(tmp_path, "test.pjc") + with open(pjc_file, 'w', encoding='latin-1') as f: + f.write(pjc_content) + return libtmg.TmgProject(pjc_file) + + def test_well_formed_pjc_returns_version(self): + """A clean PJC file parses successfully and version() returns a float.""" + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(self._MINIMAL_PJC, tmp) + self.assertEqual(project.version(), 11.0) + + def test_malformed_section_header_does_not_raise(self): + """Lines like '[Exho' (no closing bracket) are silently dropped.""" + pjc = ( + "[Stamp]\n" + "PjcVersion=11.0\n" + "[Exho\n" # malformed — the crash trigger + "SomeGarbage\n" + "[Researcher]\n" + "Name=Test User\n" + ) + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(pjc, tmp) + # Must not raise; must still find [Stamp] + self.assertEqual(project.version(), 11.0) + + def test_null_bytes_stripped(self): + """NUL bytes in the PJC file are stripped before parsing.""" + pjc = "[Stamp]\x00\nPjcVersion=11.0\n" + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(pjc, tmp) + self.assertEqual(project.version(), 11.0) + + def test_parse_error_returns_partial_config(self): + """If configparser still raises after filtering, a warning is logged + and a (possibly empty) config object is returned rather than crashing.""" + import logging + # Feed content that survives the filter but still breaks configparser: + # a key=value line before any section header is technically invalid. + pjc = "orphan_key=value\n[Stamp]\nPjcVersion=11.0\n" + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(pjc, tmp) + with self.assertLogs('.TMGImport', level=logging.WARNING) as cm: + cfg = project._read_pjc_config() + self.assertTrue(any('parse error' in m.lower() or 'parsing' in m.lower() + for m in cm.output)) + # Config object is returned (not None), even if incomplete + self.assertIsNotNone(cfg) + + def test_version_too_old_notifies_user(self): + """A PJC version < 11.0 calls user.notify_error and aborts import.""" + pjc = "[Stamp]\nPjcVersion=10.0\n" # TMG 9.01 or earlier + with tempfile.TemporaryDirectory() as tmp: + sqz = _make_minimal_sqz(tmp, pjc) + db = _make_db() + user = _MockUser() + libtmg.importData(db, sqz, user) + self.assertTrue(user.error_shown, + "notify_error should have been called for old version") + self.assertIn('9.05', user.error_message or '', + "Error message should mention TMG 9.05") + + def test_missing_pjc_version_notifies_user(self): + """A PJC with no [Stamp]/PjcVersion calls user.notify_error.""" + pjc = "[OtherSection]\nSomeKey=value\n" # no [Stamp] at all + with tempfile.TemporaryDirectory() as tmp: + sqz = _make_minimal_sqz(tmp, pjc) + db = _make_db() + user = _MockUser() + libtmg.importData(db, sqz, user) + self.assertTrue(user.error_shown, + "notify_error should have been called for missing version") + +if __name__ == '__main__': + unittest.main() From dd0fd3867fdf07b50e28bb6f2f39e7914419684a Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 11 May 2026 17:41:08 +0200 Subject: [PATCH 06/18] CI image: add gir1.2-gexiv2-0.10 for EditExifMetadata + PhotoTagging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EditExifMetadata and PhotoTaggingGramplet both do `from gi.repository import GExiv2` at module load. The CI image installs several gir1.2-* typelibs (glib, gtk, pango, gdkpixbuf, atk) but not gexiv2, so plugin-registration smoke tests on those addons fail with: cannot import name GExiv2, introspection typelib not found Add the missing apt package. GObject Introspection typelibs come from system apt packages, not pip — they cannot be auto-derived from requires_gi the way requires_mod is. Companion to addons-source PRs #878 (EditExifMetadata declares requires_gi=[(GExiv2, 0.10)] alongside requires_mod=[Pillow]) and #880 (PhotoTaggingGramplet declares requires_gi=[(GExiv2, 0.10)]). Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/docker/gramps-ci/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile index d5102ac99..4ff57cd58 100644 --- a/.github/docker/gramps-ci/Dockerfile +++ b/.github/docker/gramps-ci/Dockerfile @@ -26,6 +26,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gir1.2-pango-1.0 \ gir1.2-gdkpixbuf-2.0 \ gir1.2-atk-1.0 \ + gir1.2-gexiv2-0.10 \ gcc \ pkg-config \ python3-dev \ From 205b21ca96be3fe279db3fdc0bb35023bf6721c7 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 11 May 2026 17:57:17 +0200 Subject: [PATCH 07/18] =?UTF-8?q?CI:=20lint=20trailing-whitespace=20check?= =?UTF-8?q?=20=E2=80=94=20switch=20BRE=20[=20\t]=20to=20PCRE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `git grep '[ \t]$'` in BRE/ERE mode interprets `[ \t]` as the character class { space, backslash, 't' } — i.e. the bracket contains a literal 't', not a tab. The check therefore matches every line that ends in 't', '\', '[', ']' or a space, fires on ~3,000 false-positive lines across the tree, and would block any PR touching such a file even when no real trailing whitespace exists. Switch to PCRE via `-P` so `\t` means tab. Also tighten to `[ \t]+$` to make intent obvious (no behavioural change for the match decision). Verified on the live tree: - BRE: matches 3182 lines across ~430 files (mostly false positives). - PCRE: matches 597 lines across 21 files — real trailing whitespace. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a79cda8b..e714b6bfc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,10 @@ jobs: - name: Check trailing whitespace in Python files run: | - if git --no-pager grep --color -n --full-name '[ \t]$' -- '*.py'; then + # Use PCRE (-P): in BRE/ERE the bracket expression [ \t] is the + # set { space, backslash, 't' } — git grep matches anything ending + # in 't', not just whitespace. -P makes \t a tab. + if git --no-pager grep --color -n --full-name -P '[ \t]+$' -- '*.py'; then echo "::error::Trailing whitespace found in Python files" exit 1 fi From f5907afc9cc09a66a587d209c9266e0589e249b3 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Fri, 15 May 2026 10:31:14 +0200 Subject: [PATCH 08/18] ci: skip addons whose .gpr.py declares include_in_listing=False MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per Gary Griffin's request on PR #820: addons whose every register() call sets include_in_listing=False are not built or released by make.py, so the CI does not gate on them. Across the current addon tree this skips 11 directories (CheckPlaceTitles, DetId, FaceDetection, HtmlView, MongoDB, PhpGedView, Query, RebuildTypes, SourceIndex, SourceReferences, WordleGramplet) — the ones Gary called out in his comment, modulo case differences and a couple of names that no longer exist as directories. The check is inlined at each iteration site rather than centralised in a helper script: a small shell function (is_active) per relevant job step, plus an include_in_listing filter in the four addon-iterating tests in tests/test_plugin_registration.py. Repeating ~7 lines of bash across six job steps was preferred to introducing a separate .github/scripts/ helper file, mirroring the pattern the workflow already uses for the requires_mod auto-derive step. To re-enable CI gating for an addon, set include_in_listing=True on at least one register() call in its descriptor (or remove the field entirely — Gramps' default is True). TMGimporter, for example, stays gated because its main register() carries True even though three conditional ones use False. Affected steps: - lint (ruff): builds --exclude args from the skipped list - addon-structure: skips the po/template.pot check - compile-check (py_compile): skips files under skipped dirs - unit-test-linux / unit-test-windows: skips test modules - integration-test (per-addon): skips test modules The integration-test plugin-registration smoke step inherits the filter via the test file changes. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 99 +++++++++++++++++++++++++++++-- tests/test_plugin_registration.py | 31 ++++++---- 2 files changed, 115 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e714b6bfc..244bb13cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,32 @@ jobs: - uses: actions/checkout@v4 - name: Run ruff (syntax and import errors only) - run: ruff check --select=E9,F63,F7,F82 --no-fix --exclude='*.gpr.py' . + # Skip addon directories whose every register() in .gpr.py sets + # include_in_listing=False — those addons are not built or released + # by make.py, so CI does not gate on their lint state (per Gary + # Griffin's request on PR #820). To re-enable lint gating for an + # addon, set include_in_listing=True on at least one register() + # call in its descriptor (or remove the field — Gramps' default + # is True). Repeated inline rather than centralised so each job + # step stays self-contained. + shell: bash + run: | + is_active() { + local addon="$1" g + for g in "$addon"/*.gpr.py; do + [ -f "$g" ] || continue + grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 + grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 + done + return 1 + } + excludes="" + for d in */; do + d="${d%/}" + ls "$d"/*.gpr.py >/dev/null 2>&1 || continue + is_active "$d" || excludes="$excludes --exclude=$d" + done + ruff check --select=E9,F63,F7,F82 --no-fix --exclude='*.gpr.py' $excludes . - name: Check trailing whitespace in Python files run: | @@ -49,11 +74,23 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Check all addons have po/template.pot + - name: Check all listed addons have po/template.pot + # Skip include_in_listing=False addons (see lint job for rationale). + shell: bash run: | + is_active() { + local addon="$1" g + for g in "$addon"/*.gpr.py; do + [ -f "$g" ] || continue + grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 + grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 + done + return 1 + } failed=0 for gpr in */*.gpr.py; do addon_dir="$(dirname "$gpr")" + is_active "$addon_dir" || continue if [ ! -d "$addon_dir/po" ]; then echo "::error::$addon_dir is missing po/ directory" failed=1 @@ -63,7 +100,7 @@ jobs: fi done if [ "$failed" -eq 0 ]; then - echo "All addons have po/template.pot" + echo "All listed addons have po/template.pot" fi exit $failed @@ -78,11 +115,32 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Compile all Python files (excluding .gpr.py) + - name: Compile all Python files in listed addons (excluding .gpr.py) + # Skip include_in_listing=False addons (see lint job for rationale). shell: bash run: | + is_active() { + local addon="$1" g + for g in "$addon"/*.gpr.py; do + [ -f "$g" ] || continue + grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 + grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 + done + return 1 + } + skipped="" + for d in */; do + d="${d%/}" + ls "$d"/*.gpr.py >/dev/null 2>&1 || continue + is_active "$d" || skipped="$skipped $d" + done failed=0 while IFS= read -r f; do + skip=0 + for s in $skipped; do + case "$f" in ./$s/*) skip=1; break;; esac + done + [ "$skip" = 1 ] && continue if ! python3 -m py_compile "$f" 2>&1; then failed=1 fi @@ -158,9 +216,20 @@ jobs: env: PYTHONPATH: . run: | + is_active() { + local addon="$1" g + for g in "$addon"/*.gpr.py; do + [ -f "$g" ] || continue + grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 + grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 + done + return 1 + } modules="" for f in */tests/test_*.py; do [ -f "$f" ] || continue + addon="${f%%/*}" + is_active "$addon" || continue case "$(basename "$f")" in test_integration*) continue ;; test_windows_*) continue ;; @@ -243,9 +312,20 @@ jobs: env: PYTHONPATH: . run: | + is_active() { + local addon="$1" g + for g in "$addon"/*.gpr.py; do + [ -f "$g" ] || continue + grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 + grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 + done + return 1 + } modules="" for f in */tests/test_*.py; do [ -f "$f" ] || continue + addon="${f%%/*}" + is_active "$addon" || continue case "$(basename "$f")" in test_integration*) continue ;; test_linux_*) continue ;; @@ -326,9 +406,20 @@ jobs: env: PYTHONPATH: . run: | + is_active() { + local addon="$1" g + for g in "$addon"/*.gpr.py; do + [ -f "$g" ] || continue + grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 + grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 + done + return 1 + } modules="" for f in */tests/test_integration*.py; do [ -f "$f" ] || continue + addon="${f%%/*}" + is_active "$addon" || continue mod="${f%.py}" mod="${mod//\//.}" modules="$modules $mod" diff --git a/tests/test_plugin_registration.py b/tests/test_plugin_registration.py index f242fcf35..dae8d6f87 100644 --- a/tests/test_plugin_registration.py +++ b/tests/test_plugin_registration.py @@ -54,16 +54,26 @@ LOG = logging.getLogger(__name__) -def _get_addon_plugins(registry: Any) -> list[Any]: +def _get_addon_plugins(registry: Any, include_unlisted: bool = False) -> list[Any]: """Return all :class:`PluginData` objects whose ``fpath`` is inside the addons tree. + By default, plugins whose ``.gpr.py`` declares ``include_in_listing=False`` + are filtered out: those addons are not built or released by ``make.py``, + so this CI does not gate on their state (per Gary Griffin's discussion on + PR #820). Pass ``include_unlisted=True`` to inspect them anyway. + :param registry: A :class:`PluginRegister` instance. + :param include_unlisted: If ``True``, also return plugins whose + ``include_in_listing`` field is ``False``. + :type include_unlisted: bool :returns: List of :class:`PluginData` entries belonging to this repository. """ return [ pdata for pdata in registry._PluginRegister__plugindata - if pdata.fpath and ADDONS_ROOT in pdata.fpath + if pdata.fpath + and ADDONS_ROOT in pdata.fpath + and (include_unlisted or pdata.include_in_listing) ] @@ -118,12 +128,11 @@ def test_all_plugins_have_valid_metadata(self) -> None: self.assertTrue(pdata.version, f"Plugin {pdata.id} missing version") def test_target_version_is_6_0(self) -> None: - """All addons on this branch should target Gramps 6.0.""" + """All listed addons on this branch should target Gramps 6.0.""" issues: list[str] = [] - for pdata in self.plugin_registry._PluginRegister__plugindata: - if pdata.fpath and ADDONS_ROOT in pdata.fpath: - if not pdata.gramps_target_version.startswith("6.0"): - issues.append(f"{pdata.id}: targets {pdata.gramps_target_version}") + for pdata in _get_addon_plugins(self.plugin_registry): + if not pdata.gramps_target_version.startswith("6.0"): + issues.append(f"{pdata.id}: targets {pdata.gramps_target_version}") if issues: self.fail("Addons not targeting Gramps 6.0:\n" + "\n".join(issues)) @@ -218,11 +227,11 @@ class TestImportPluginSmoke(GrampsTestCase): """Verify import plugins have a callable ``import_function`` attribute.""" def test_import_plugins_have_callable(self) -> None: - """Each IMPORT plugin must reference a callable import function.""" + """Each listed IMPORT plugin must reference a callable import function.""" import_plugins = [ p for p in self.plugin_registry.type_plugins(IMPORT) - if p.fpath and ADDONS_ROOT in p.fpath + if p.fpath and ADDONS_ROOT in p.fpath and p.include_in_listing ] issues: list[str] = [] for pdata in import_plugins: @@ -250,11 +259,11 @@ class TestExportPluginSmoke(GrampsTestCase): """Verify export plugins have a callable ``export_function`` attribute.""" def test_export_plugins_have_callable(self) -> None: - """Each EXPORT plugin must reference a callable export function.""" + """Each listed EXPORT plugin must reference a callable export function.""" export_plugins = [ p for p in self.plugin_registry.type_plugins(EXPORT) - if p.fpath and ADDONS_ROOT in p.fpath + if p.fpath and ADDONS_ROOT in p.fpath and p.include_in_listing ] issues: list[str] = [] for pdata in export_plugins: From ff215ac6bffa03bd0543db012f3853d17923892c Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Sat, 16 May 2026 01:03:47 +0200 Subject: [PATCH 09/18] CI: validate requires_mod names against find_spec (Pillow/PIL trap) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The auto-derive step parses requires_mod from every .gpr.py and pip- installs each name, which validates the install-side of the contract. But Gramps' end-user dep gate (gen/utils/requirements.py:check_mod) calls find_spec() — the importable name, not the PyPI name. For most addons the two coincide; for Pillow they don't, so requires_mod= ["Pillow"] pip-installs cleanly, CI stays green, and the Addon Manager still rejects the addon on every end-user install. Adds a follow-up step after each "Install addon runtime deps" block (one per job: unit-test-linux, unit-test-windows, integration-test). For each declared requires_mod name: if pip-show confirms the package is installed but find_spec returns None, the declaration is wrong. Pip-install failures are skipped, so missing system deps (graphviz-dev for pygraphviz, libpq-dev for psycopg2) don't cause false positives. Verified locally: catches requires_mod=["Pillow"] with exit 1 and a GitHub annotation; current addons-source tree (all 9 declared mods) passes with no false positives. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 144 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 244bb13cc..531f2eedb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,6 +199,58 @@ jobs: echo "no requires_mod declarations found" fi + - name: Validate requires_mod names against Gramps' dep gate + # Cross-check: every requires_mod entry that pip successfully + # installed in the previous step must also resolve via + # find_spec(), since that is what Gramps' Addon Manager calls + # (gramps/gen/utils/requirements.py:check_mod). A name that + # pip-installs but does not import is a declaration bug — e.g. + # requires_mod=["Pillow"] when the importable name is "PIL". + # Pip-install failures upstream are skipped: those are + # system-dep / image gaps, not PR-caused. + shell: bash + run: | + python3 - <<'PY' + import ast, glob, re, subprocess, sys + from importlib.util import find_spec + + pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + names = set() + for f in glob.glob("*/*.gpr.py"): + try: + text = open(f, encoding="utf-8").read() + except OSError: + continue + for m in pat.finditer(text): + try: + names.update(ast.literal_eval(m.group(1))) + except (ValueError, SyntaxError): + pass + + bad = [] + for name in sorted(names): + installed = subprocess.run( + [sys.executable, "-m", "pip", "show", name], + capture_output=True, + ).returncode == 0 + if not installed: + print(f"~ {name} (pip-install failed earlier, skipping)") + continue + if find_spec(name) is None: + bad.append(name) + print(f"x {name} (pip-installed but find_spec returned None)") + else: + print(f"ok {name}") + + if bad: + print() + print(f"::error::Wrong requires_mod names: {bad}") + print("These pip-install but are not importable. requires_mod is") + print("consumed by gramps' check_mod() via find_spec(), so the") + print("importable module name is required (e.g. 'PIL', not 'Pillow').") + sys.exit(1) + PY + - name: Run per-addon unit tests # Filename convention (all OSes): # test_*.py — general (any OS) @@ -306,6 +358,51 @@ jobs: echo "no requires_mod declarations found" fi + - name: Validate requires_mod names against Gramps' dep gate + # See unit-test-linux for rationale. Uses `python` (conda-forge + # env) to match the surrounding Windows job style. + run: | + python - <<'PY' + import ast, glob, re, subprocess, sys + from importlib.util import find_spec + + pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + names = set() + for f in glob.glob("*/*.gpr.py"): + try: + text = open(f, encoding="utf-8").read() + except OSError: + continue + for m in pat.finditer(text): + try: + names.update(ast.literal_eval(m.group(1))) + except (ValueError, SyntaxError): + pass + + bad = [] + for name in sorted(names): + installed = subprocess.run( + [sys.executable, "-m", "pip", "show", name], + capture_output=True, + ).returncode == 0 + if not installed: + print(f"~ {name} (pip-install failed earlier, skipping)") + continue + if find_spec(name) is None: + bad.append(name) + print(f"x {name} (pip-installed but find_spec returned None)") + else: + print(f"ok {name}") + + if bad: + print() + print(f"::error::Wrong requires_mod names: {bad}") + print("These pip-install but are not importable. requires_mod is") + print("consumed by gramps' check_mod() via find_spec(), so the") + print("importable module name is required (e.g. 'PIL', not 'Pillow').") + sys.exit(1) + PY + - name: Run per-addon unit tests # See filename-convention note in unit-test-linux. The Windows # job runs test_*.py except test_linux_* and test_integration_*. @@ -389,6 +486,53 @@ jobs: echo "no requires_mod declarations found" fi + - name: Validate requires_mod names against Gramps' dep gate + # See unit-test-linux for rationale. This is the blocking copy: + # integration-test does not set continue-on-error, so a wrong + # requires_mod name surfaces as a CI failure here. + shell: bash + run: | + python3 - <<'PY' + import ast, glob, re, subprocess, sys + from importlib.util import find_spec + + pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + names = set() + for f in glob.glob("*/*.gpr.py"): + try: + text = open(f, encoding="utf-8").read() + except OSError: + continue + for m in pat.finditer(text): + try: + names.update(ast.literal_eval(m.group(1))) + except (ValueError, SyntaxError): + pass + + bad = [] + for name in sorted(names): + installed = subprocess.run( + [sys.executable, "-m", "pip", "show", name], + capture_output=True, + ).returncode == 0 + if not installed: + print(f"~ {name} (pip-install failed earlier, skipping)") + continue + if find_spec(name) is None: + bad.append(name) + print(f"x {name} (pip-installed but find_spec returned None)") + else: + print(f"ok {name}") + + if bad: + print() + print(f"::error::Wrong requires_mod names: {bad}") + print("These pip-install but are not importable. requires_mod is") + print("consumed by gramps' check_mod() via find_spec(), so the") + print("importable module name is required (e.g. 'PIL', not 'Pillow').") + sys.exit(1) + PY + - name: Run plugin registration tests # shell: bash for consistency with the surrounding steps; the # current command uses no bashisms, but keeps this block safe From d2656125e8534f0c9e7ec8e29f49c6d8ba47d2bf Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Tue, 19 May 2026 17:52:17 +0200 Subject: [PATCH 10/18] =?UTF-8?q?CI:=20make=20lint=20job=20blocking=20?= =?UTF-8?q?=E2=80=94=20ruff=20backlog=20cleared=20on=20maintenance/gramps6?= =?UTF-8?q?0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops the lint job's continue-on-error: true (and its TODO comment), making ruff E9/F63/F7/F82 a blocking gate per the comment's own instruction. The ruff backlog the comment guarded against was cleared by 24 PRs (#843 + #847-#869) merged into maintenance/gramps60 between 2026-05-12 and 2026-05-18. Verified locally: pipx run ruff check --select=E9,F63,F7,F82 \ --no-fix --exclude='*.gpr.py' . on `gramps-project/addons-source:maintenance/gramps60` reports "All checks passed!". Other continue-on-error gates in this workflow (addon-structure, compile-check, integration) each guard their own backlogs and stay non-blocking until those clear separately. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 531f2eedb..76d44b4d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,6 @@ jobs: lint: name: Lint runs-on: ubuntu-latest - # Non-blocking until the existing ruff E9/F63/F7/F82 errors across the - # addon set are cleaned up in a follow-up PR. Flip this off in that PR. - continue-on-error: true container: image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 steps: From 0dd3f1b2a6e59cfe18ddf84324bb81d9444db7ed Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Tue, 19 May 2026 18:04:37 +0200 Subject: [PATCH 11/18] =?UTF-8?q?CI:=20make=20unit-test-linux=20and=20unit?= =?UTF-8?q?-test-windows=20blocking=20=E2=80=94=20backlog=20cleared?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops continue-on-error: true (and the TODO comments) from both unit-test jobs. They were guarding against "currently-broken addon unit modules (import failures, stale API usage)" that have since been fixed by the merged plugin-registration / dep-declaration round (#875 WordleGramplet, #876 SourceReferences, #878 EditExifMetadata, #879 MongoDB, #880 PhotoTaggingGramplet, #869 SurnameMappingGramplet, gramps#2299 ClipboardGramplet) plus the lint backlog (#843, #847-#869). Verified by the most recent fork CI run on eduralph/addons-source:maintenance/gramps60 (sha 8aabdd4db, 2026-05-18): both Unit Tests (Linux) and Unit Tests (Windows) reported success. Also clears two stale comments that referenced the removed flag: - unit-test-linux's "Run per-addon unit tests" step: drop the trailing "Falls silently under continue-on-error." sentence. - integration's requires_mod-validate step: collapse the now-moot "this is the blocking copy" rationale to a plain pointer at unit-test-linux. The lone remaining continue-on-error is on addon-structure (line 70), gated by the po/template.pot backlog. PRs #838-#841 were closed without merge; per Eduard's call the po/template.pot check stays non-blocking for now. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76d44b4d5..ea85d0f98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,9 +150,6 @@ jobs: unit-test-linux: name: Unit Tests (Linux) runs-on: ubuntu-latest - # Non-blocking until the currently-broken addon unit modules (import - # failures, stale API usage) are sorted out in follow-up PRs. - continue-on-error: true container: image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 steps: @@ -260,7 +257,7 @@ jobs: # shell: bash — the container's default shell is /bin/sh # (dash on python:3.12-slim), which does not support the # ${var//pattern/repl} and ${var%.py} parameter expansions - # used below. Falls silently under continue-on-error. + # used below. shell: bash env: PYTHONPATH: . @@ -303,8 +300,6 @@ jobs: unit-test-windows: name: Unit Tests (Windows) runs-on: windows-latest - # Non-blocking for the same reason as unit-test-linux. - continue-on-error: true defaults: run: shell: bash -el {0} @@ -484,9 +479,7 @@ jobs: fi - name: Validate requires_mod names against Gramps' dep gate - # See unit-test-linux for rationale. This is the blocking copy: - # integration-test does not set continue-on-error, so a wrong - # requires_mod name surfaces as a CI failure here. + # See unit-test-linux for rationale. shell: bash run: | python3 - <<'PY' From 9a91d89873d212106beb2098d084d0e095df909f Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 18 May 2026 21:05:07 +0200 Subject: [PATCH 12/18] CI: derive image tag and make.py argument from the branch ref Replaces the seven hardcoded "gramps60" strings in ci.yml with values computed by a new setup job that strips "maintenance/" off the ref and validates the remainder matches grampsNN. Container jobs now pull gramps-ci: and the build step calls make.py with the matching suffix, so the same workflow runs unchanged on maintenance/gramps60, maintenance/gramps61, and any future maintenance/grampsNN branch. A non-matching ref (master, topic branch, gramps100) fails fast in the setup job rather than racing through with a malformed image tag. Verified: - YAML parses (python3 -c "import yaml; yaml.safe_load(...)") - bash dry-run of the case statement accepts gramps60/61/62/42 and rejects master, feature/foo, maintenance/gramps60-test, gramps100 - container jobs (lint, compile-check, unit-test-linux, integration-test, build) all carry needs: setup; non-container jobs (addon-structure, unit-test-windows) do not Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 50 ++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea85d0f98..28a9213d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,22 +2,45 @@ name: CI on: push: - branches: [maintenance/gramps60] + branches: [maintenance/gramps**] pull_request: - branches: [maintenance/gramps60] - -env: - CI_IMAGE: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + branches: [maintenance/gramps**] jobs: + # ----------------------------------------------------------------- + # Setup — derive the branch suffix (gramps60 / gramps61 / …) from + # the ref. On push events github.ref_name is the branch being + # pushed; on pull_request events github.base_ref is the target + # branch. Either way the suffix is what follows "maintenance/". + # ----------------------------------------------------------------- + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + branch_suffix: ${{ steps.compute.outputs.branch_suffix }} + ci_image: ${{ steps.compute.outputs.ci_image }} + steps: + - id: compute + shell: bash + run: | + ref="${{ github.base_ref || github.ref_name }}" + suffix="${ref#maintenance/}" + case "$suffix" in + gramps[0-9][0-9]) ;; + *) echo "::error::unexpected ref '$ref' (suffix '$suffix')"; exit 1 ;; + esac + echo "branch_suffix=$suffix" >> "$GITHUB_OUTPUT" + echo "ci_image=ghcr.io/${{ github.repository }}/gramps-ci:$suffix" >> "$GITHUB_OUTPUT" + # ----------------------------------------------------------------- # Lint (ci container) # ----------------------------------------------------------------- lint: name: Lint + needs: setup runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + image: ${{ needs.setup.outputs.ci_image }} steps: - uses: actions/checkout@v4 @@ -106,9 +129,10 @@ jobs: # ----------------------------------------------------------------- compile-check: name: Compile Check + needs: setup runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + image: ${{ needs.setup.outputs.ci_image }} steps: - uses: actions/checkout@v4 @@ -149,9 +173,10 @@ jobs: # ----------------------------------------------------------------- unit-test-linux: name: Unit Tests (Linux) + needs: setup runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + image: ${{ needs.setup.outputs.ci_image }} steps: - uses: actions/checkout@v4 @@ -439,9 +464,9 @@ jobs: integration-test: name: Integration Tests (Gramps) runs-on: ubuntu-latest - needs: [unit-test-linux] + needs: [setup, unit-test-linux] container: - image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + image: ${{ needs.setup.outputs.ci_image }} options: --init steps: - uses: actions/checkout@v4 @@ -570,9 +595,10 @@ jobs: # ----------------------------------------------------------------- build: name: Build + needs: setup runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + image: ${{ needs.setup.outputs.ci_image }} steps: - uses: actions/checkout@v4 @@ -587,4 +613,4 @@ jobs: GRAMPSPATH: ${{ steps.gramps-path.outputs.path }} run: | mkdir -p ../download - python3 make.py gramps60 build all + python3 make.py "${{ needs.setup.outputs.branch_suffix }}" build all From abefdbe7b7fc9c9ea88c43075a373132600f7583 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 18 May 2026 21:06:00 +0200 Subject: [PATCH 13/18] docker-build: derive image tag and Gramps series from the branch ref MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the three hardcoded "gramps60" strings in docker-build.yml with values computed from github.ref_name in a new "Compute branch parameters" step. The image is tagged gramps-ci: and built with GRAMPS_SERIES= (e.g. gramps60→6.0, gramps61→6.1), so the same workflow produces the correct image on every maintenance branch. Also drops the paths: filter from the push trigger. With the filter the workflow only fired when .github/docker/** changed, which means newly-created maintenance branches (inheriting the Dockerfile from their parent unchanged) would never produce their gramps-ci: image, and ci.yml jobs would fail pulling a non-existent tag. Every push to a maintenance branch now runs docker-build; buildx layer cache turns the steady-state case into a ~20-30 s no-op rebuild. Verified: - YAML parses (python3 -c "import yaml; yaml.safe_load(...)") - bash dry-run of the case/series logic produces 6.0, 6.1, 6.2, 7.0, 4.2 for the corresponding maintenance/grampsNN refs and rejects master Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/docker-build.yml | 32 +++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 9289949b2..b466bc142 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -2,9 +2,11 @@ name: Build Docker Images on: push: - branches: [maintenance/gramps60] - paths: - - '.github/docker/**' + # No paths filter on purpose. Buildx layer cache makes the rebuild + # a ~20-30 s no-op when nothing under .github/docker/ has changed, + # in exchange for guaranteeing gramps-ci: exists on the + # first push to any newly-created maintenance branch. + branches: [maintenance/gramps**] workflow_dispatch: env: @@ -32,14 +34,32 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Compute branch parameters + # Derive the image-tag suffix and Gramps minor series from the + # branch ref. Same validation as ci.yml's setup job: anything + # outside maintenance/grampsNN fails fast. + id: params + shell: bash + run: | + ref="${{ github.ref_name }}" + suffix="${ref#maintenance/}" + case "$suffix" in + gramps[0-9][0-9]) ;; + *) echo "::error::unexpected ref '$ref' (suffix '$suffix')"; exit 1 ;; + esac + # gramps60 → 6.0, gramps61 → 6.1, gramps62 → 6.2, … + series="${suffix:6:1}.${suffix:7}" + echo "suffix=$suffix" >> "$GITHUB_OUTPUT" + echo "series=$series" >> "$GITHUB_OUTPUT" + - name: Docker metadata id: meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.REPO }}/gramps-ci tags: | - type=raw,value=gramps60 - type=sha,prefix=gramps60- + type=raw,value=${{ steps.params.outputs.suffix }} + type=sha,prefix=${{ steps.params.outputs.suffix }}- - name: Build and push gramps-ci uses: docker/build-push-action@v6 @@ -48,5 +68,7 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: | + GRAMPS_SERIES=${{ steps.params.outputs.series }} cache-from: type=gha cache-to: type=gha,mode=max From 99276264a931bdab47d9a27a8b3ec16a97aca19f Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 18 May 2026 21:07:17 +0200 Subject: [PATCH 14/18] Dockerfile: parameterise Gramps series via GRAMPS_SERIES build arg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the hardcoded "gramps>=6.0,<6.1" pip pin with "gramps==\${GRAMPS_SERIES}.*", and pulls GRAMPS_SERIES from a build arg with no default. Same Dockerfile now builds gramps-ci:gramps60 when invoked with GRAMPS_SERIES=6.0, gramps-ci:gramps61 with 6.1, and so on. docker-build.yml derives the value from the branch ref. No default on GRAMPS_SERIES on purpose: a wrong default would silently produce an image for the wrong Gramps series that "looks fine" but is mismatched against the branch's addon code. A guard line fails the build loudly if the arg is missing. Verified: - docker build --check --build-arg GRAMPS_SERIES=6.0 → no warnings - a minimal repro Dockerfile invoking the guard without GRAMPS_SERIES exits 1 with "GRAMPS_SERIES is required (e.g. 6.0)" Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/docker/gramps-ci/Dockerfile | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile index 4ff57cd58..d979c15f6 100644 --- a/.github/docker/gramps-ci/Dockerfile +++ b/.github/docker/gramps-ci/Dockerfile @@ -1,6 +1,9 @@ # .github/docker/gramps-ci/Dockerfile # -# Unified Gramps 6.0 CI image. Includes everything jobs need: +# Gramps CI image. The Gramps minor series (6.0, 6.1, …) is picked at +# build time via the GRAMPS_SERIES build arg, so the same Dockerfile +# produces gramps-ci:gramps60, gramps-ci:gramps61, … without per-branch +# edits. Includes everything jobs need: # - Python + pip-installed Gramps, PyGObject, pycairo # - GTK typelibs (so addon modules that `from gi.repository import Gtk` # at module load time are importable — widgets still need xvfb to render) @@ -13,11 +16,21 @@ # `--init` (or use a container runtime that injects tini) because xvfb-run # hangs if it inherits PID 1. # +# Local build: +# docker build --build-arg GRAMPS_SERIES=6.0 .github/docker/gramps-ci +# ARG PYTHON_VERSION=3.12 FROM python:${PYTHON_VERSION}-slim +# Gramps minor series to install (e.g. 6.0, 6.1). No default — must be +# passed explicitly so a wrong default cannot silently produce an image +# for the wrong Gramps series. docker-build.yml derives this from the +# branch ref. +ARG GRAMPS_SERIES +RUN [ -n "$GRAMPS_SERIES" ] || { echo "GRAMPS_SERIES is required (e.g. 6.0)"; exit 1; } + LABEL org.opencontainers.image.source="https://github.com/gramps-project/addons-source" -LABEL org.opencontainers.image.description="Unified Gramps 6.0 CI image (Python, Gramps, GTK typelibs, xvfb)" +LABEL org.opencontainers.image.description="Gramps ${GRAMPS_SERIES} CI image (Python, Gramps, GTK typelibs, xvfb)" RUN apt-get update && apt-get install -y --no-install-recommends \ libgirepository-2.0-dev \ @@ -46,7 +59,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN pip install --no-cache-dir \ PyGObject \ pycairo \ - "gramps>=6.0,<6.1" \ + "gramps==${GRAMPS_SERIES}.*" \ orjson \ ruff From 894e4843a24f3dabed3f8c26215d08d09f5c5247 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 18 May 2026 23:21:19 +0200 Subject: [PATCH 15/18] CI image: hybrid PyPI / git-clone install for unreleased maintenance branches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR 820's "pip install gramps==${SERIES}.*" only works on maintenance branches whose Gramps release has been published to PyPI. The maintenance/gramps61 branch carries the in-development 6.1 series and has no PyPI release yet, so docker-build fails with "No matching distribution found for gramps==6.1.*" the moment the branch-neutral pipeline is exercised against it. This commit adds a hybrid install path to the Dockerfile and the matching wiring in docker-build.yml: - Dockerfile attempts pip install first. On "No matching distribution found for gramps==..." it falls back to a SHA-pinned git clone of gramps-project/gramps@maintenance/gramps${SERIES_NODOT} and "pip install ." from that working tree. Any other pip failure (network, 503, etc.) is fatal so a transient PyPI hiccup cannot silently flip a normally-released branch into "test against moving tip" mode. - docker-build.yml's params step captures the upstream branch's current HEAD SHA via git ls-remote and passes it in as GRAMPS_FALLBACK_SHA. The SHA participates in the buildx cache key so a moved upstream tip actually re-runs the install layer (without that, gramps61 CI would stay frozen on whichever revision was first baked into the image). Trade-off — and worth flagging to addon contributors: a green CI on an unreleased branch means "addons work with the current upstream tip," not "addons work with X.Y.0." The ::notice:: (PyPI path) vs ::warning:: (fallback path) log distinction in the Dockerfile makes the active path visible in every docker-build run. Verified locally: - docker build --check --build-arg GRAMPS_SERIES=6.0 → clean - docker build with GRAMPS_SERIES=6.0 + valid fallback SHA → ::notice::installed gramps==6.0.* from PyPI; image reports "Gramps 6.0.8" (released path unchanged) - docker build with GRAMPS_SERIES=6.1 + upstream HEAD SHA → fallback fires; image reports "Gramps 6.1.0-beta1" (only obtainable from git clone, since no 6.1.* exists on PyPI) - bash unit test of the failure-mode triage: missing SHA when PyPI lacks the version exits 1 with ::error::no gramps==... and GRAMPS_FALLBACK_SHA is unset; non-version pip error exits 1 with ::error::pip install gramps failed (non-version reason) and dumps the captured stderr Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/docker/gramps-ci/Dockerfile | 59 +++++++++++++++++++++++++---- .github/workflows/docker-build.yml | 23 ++++++++--- 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile index d979c15f6..38a4e9a28 100644 --- a/.github/docker/gramps-ci/Dockerfile +++ b/.github/docker/gramps-ci/Dockerfile @@ -16,9 +16,27 @@ # `--init` (or use a container runtime that injects tini) because xvfb-run # hangs if it inherits PID 1. # -# Local build: +# Gramps install path. PyPI is tried first; when no gramps==${SERIES}.* +# release exists (e.g. while 6.1 is still in development) the build +# falls back to a SHA-pinned git clone of +# gramps-project/gramps@maintenance/gramps${SERIES_NODOT}. Other pip +# failures (network, etc.) are NOT silently retried so a transient +# PyPI outage cannot flip a normally-released branch to "test against +# moving tip" mode. docker-build.yml captures the upstream SHA via +# git ls-remote and passes it in as GRAMPS_FALLBACK_SHA; the SHA +# becomes part of the buildx cache key so a moved upstream tip +# actually re-runs the install layer. +# +# Local build (released series): # docker build --build-arg GRAMPS_SERIES=6.0 .github/docker/gramps-ci # +# Local build (unreleased series, e.g. 6.1): +# sha=$(git ls-remote https://github.com/gramps-project/gramps.git \ +# refs/heads/maintenance/gramps61 | awk '{print $1}') +# docker build --build-arg GRAMPS_SERIES=6.1 \ +# --build-arg GRAMPS_FALLBACK_SHA=$sha \ +# .github/docker/gramps-ci +# ARG PYTHON_VERSION=3.12 FROM python:${PYTHON_VERSION}-slim @@ -27,6 +45,11 @@ FROM python:${PYTHON_VERSION}-slim # for the wrong Gramps series. docker-build.yml derives this from the # branch ref. ARG GRAMPS_SERIES +# Commit SHA on gramps-project/gramps@maintenance/grampsNN. Only used +# when no gramps==${GRAMPS_SERIES}.* release exists on PyPI. Ignored +# otherwise. Empty default is intentional — on released branches the +# fallback never fires. +ARG GRAMPS_FALLBACK_SHA="" RUN [ -n "$GRAMPS_SERIES" ] || { echo "GRAMPS_SERIES is required (e.g. 6.0)"; exit 1; } LABEL org.opencontainers.image.source="https://github.com/gramps-project/addons-source" @@ -56,12 +79,34 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # requires_mod)" step pip-installs them at CI runtime from every # .gpr.py's requires_mod list, matching what Gramps' Addon Manager does # for an end user. Keeps .gpr.py the single source of truth. -RUN pip install --no-cache-dir \ - PyGObject \ - pycairo \ - "gramps==${GRAMPS_SERIES}.*" \ - orjson \ - ruff +RUN pip install --no-cache-dir PyGObject pycairo orjson ruff + +# Install gramps: PyPI first, SHA-pinned git clone as fallback. +RUN /bin/bash -eo pipefail <<'BASH' +suffix_nodot="${GRAMPS_SERIES//./}" +if pip install --no-cache-dir "gramps==${GRAMPS_SERIES}.*" 2>/tmp/pip.err; then + echo "::notice::installed gramps==${GRAMPS_SERIES}.* from PyPI" +elif grep -q "No matching distribution found for gramps==" /tmp/pip.err; then + if [ -z "${GRAMPS_FALLBACK_SHA}" ]; then + echo "::error::no gramps==${GRAMPS_SERIES}.* on PyPI and GRAMPS_FALLBACK_SHA is unset" + exit 1 + fi + echo "::warning::no gramps==${GRAMPS_SERIES}.* on PyPI; installing from gramps-project/gramps@maintenance/gramps${suffix_nodot} at ${GRAMPS_FALLBACK_SHA}" + mkdir -p /tmp/gramps + cd /tmp/gramps + git init -q + git remote add origin https://github.com/gramps-project/gramps.git + git fetch --depth 1 origin "${GRAMPS_FALLBACK_SHA}" + git checkout FETCH_HEAD + pip install --no-cache-dir . + cd / + rm -rf /tmp/gramps +else + echo "::error::pip install gramps failed (non-version reason); aborting rather than silently switching to git tip" + cat /tmp/pip.err + exit 1 +fi +BASH RUN apt-get purge -y gcc python3-dev pkg-config && apt-get autoremove -y diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index b466bc142..6ebd19ed5 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -35,9 +35,16 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Compute branch parameters - # Derive the image-tag suffix and Gramps minor series from the - # branch ref. Same validation as ci.yml's setup job: anything - # outside maintenance/grampsNN fails fast. + # Derive the image-tag suffix, Gramps minor series, and upstream + # fallback SHA from the branch ref. Same validation as ci.yml's + # setup job: anything outside maintenance/grampsNN fails fast. + # The fallback SHA is the current tip of gramps-project/gramps + # at the matching maintenance branch; the Dockerfile only uses + # it when no gramps==${series}.* release exists on PyPI. The + # SHA is part of the buildx cache key so a moved upstream tip + # actually re-runs the install layer (otherwise gramps61 CI + # would stay on the same stale gramps revision build after + # build). id: params shell: bash run: | @@ -49,8 +56,13 @@ jobs: esac # gramps60 → 6.0, gramps61 → 6.1, gramps62 → 6.2, … series="${suffix:6:1}.${suffix:7}" - echo "suffix=$suffix" >> "$GITHUB_OUTPUT" - echo "series=$series" >> "$GITHUB_OUTPUT" + fallback_sha=$(git ls-remote https://github.com/gramps-project/gramps.git "refs/heads/maintenance/${suffix}" | awk '{print $1}') + if [ -z "$fallback_sha" ]; then + echo "::warning::upstream gramps-project/gramps has no maintenance/${suffix} branch; fallback path will fail if PyPI lacks gramps==${series}.*" + fi + echo "suffix=$suffix" >> "$GITHUB_OUTPUT" + echo "series=$series" >> "$GITHUB_OUTPUT" + echo "fallback_sha=$fallback_sha" >> "$GITHUB_OUTPUT" - name: Docker metadata id: meta @@ -70,5 +82,6 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | GRAMPS_SERIES=${{ steps.params.outputs.series }} + GRAMPS_FALLBACK_SHA=${{ steps.params.outputs.fallback_sha }} cache-from: type=gha cache-to: type=gha,mode=max From 73d75edf65a757c8f616f55b9856b96488b73d0c Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 18 May 2026 23:58:24 +0200 Subject: [PATCH 16/18] docs: note the first-push race and unreleased-branch CI semantics Two doc-only additions covering practical implications of the branch-neutral CI from the preceding commits: - .github/workflows/ci.yml header: short NOTE for maintainers explaining that the first push to a newly-created maintenance branch will see container jobs fail at "Initialize containers" because the gramps-ci: image is still being built by the companion docker-build.yml workflow on the same push. Solution is a one-time re-run after docker-build finishes. This is inherent to running both workflows on the same push event and is not a problem on subsequent pushes (the image already exists). - CONTRIBUTING.md "Work Towards a Merge": short paragraph for contributors explaining what a green CI check means on an unreleased maintenance branch. When PyPI lacks a release for the matching Gramps series, the CI image is built from a SHA-pinned snapshot of gramps-project/gramps@maintenance/grampsNN, so green means "addons work with the upstream branch tip at ", not "addons work with the released X.Y.0". The exact SHA is logged as a ::warning:: line in the Build Docker Images output. Verified: - YAML still parses Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 15 +++++++++++++++ CONTRIBUTING.md | 12 ++++++++++++ 2 files changed, 27 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28a9213d5..4ebd61d3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,20 @@ name: CI +# Branch-neutral workflow: the image tag, make.py argument, and Gramps +# series pin are all derived from the branch ref at runtime, so the same +# file runs unchanged on maintenance/gramps60, maintenance/gramps61, and +# any future maintenance/grampsNN branch. +# +# NOTE for maintainers — first push to a new maintenance branch: the +# corresponding gramps-ci: image does not exist in GHCR yet, so +# the container jobs in this workflow will fail at "Initialize containers" +# on the very first run. The companion docker-build.yml workflow fires +# on the same push and builds/pushes the image (~5 min cold). After it +# finishes, re-run the failed CI jobs (Actions tab → this run → "Re-run +# failed jobs") and they will pull the now-existing image. This race +# happens only on initial branch creation — every subsequent push to +# that branch finds the image already in GHCR. + on: push: branches: [maintenance/gramps**] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dda23171b..96a0c7c77 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1094,6 +1094,18 @@ With the PR created, it's now a matter of working with the ```addons-source``` maintainers. Suggestions and corrections will be made and it may be mecessary to modify the original submission to get the addon accepted. +Your PR will also run automated CI checks (lint, compile, unit and +integration tests, plus an addon build); results appear on the PR page. +**Note:** on ```maintenance/grampsNN``` branches whose corresponding +Gramps release is not yet on PyPI (e.g. ```maintenance/gramps61``` while +6.1 is in development), the CI image is built from a SHA-pinned snapshot +of upstream ```gramps-project/gramps@maintenance/grampsNN``` rather than +a tagged release — a green check on such a PR means the addon works +against that *branch tip*, not against a released version. The exact +SHA used is logged as a ```::warning::``` line in the ```Build Docker +Images``` workflow output. + + The key thing is to monitor progress and comments. Your PR will have an ID number -- 1234, for example -- so you can always go to the github web page for it: From 3b2a947be34c3d9ad1b8b526d1fb122e1fc5173e Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Tue, 19 May 2026 00:07:26 +0200 Subject: [PATCH 17/18] docs: add CI maintainer runbook (.github/CI-MAINTAINER.md) Operational document for the gramps-project/addons-source maintainer covering the new steps introduced by PR 820 + the branch-neutral follow-up: - One-time setup: make the gramps-ci GHCR package public (so fork PR contributors can pull the image), and expect the first-push race on maintenance/gramps60 immediately after merge. - Creating a new maintenance branch: a plain `git branch && git push` plus a one-time re-run of the failed CI jobs after the image is built. - When a Gramps minor release lands on PyPI: nothing to do (the hybrid Dockerfile auto-detects); optional workflow_dispatch to rebuild immediately. - Diagnostic log markers: ::notice:: / ::warning:: / ::error:: annotations emitted from docker-build.yml and the Dockerfile, with their respective root causes. - Future-proofing knobs: upstream repo URL and GHCR tag retention, only mentioned in case they ever become relevant. The ci.yml header NOTE about the first-push race now points to this runbook for the full picture (GHCR visibility, log markers, etc.). Verified: - YAML parses - All TOC anchors in the new file resolve to existing headers - ../MAINTAINERS.md and ../CONTRIBUTING.md (relative links in the new file) both exist - CONTRIBUTING.md#work-towards-a-merge anchor exists Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/CI-MAINTAINER.md | 145 +++++++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 3 + 2 files changed, 148 insertions(+) create mode 100644 .github/CI-MAINTAINER.md diff --git a/.github/CI-MAINTAINER.md b/.github/CI-MAINTAINER.md new file mode 100644 index 000000000..69324a203 --- /dev/null +++ b/.github/CI-MAINTAINER.md @@ -0,0 +1,145 @@ +# CI Maintainer Notes + +Operational steps that a `gramps-project/addons-source` maintainer +needs to handle once PR 820 (and the branch-neutral follow-up) lands. +The pipeline is otherwise self-driving — this document is only about +the rough edges. + +For day-to-day addon-release maintenance see [MAINTAINERS.md](../MAINTAINERS.md); +for the contributor-facing summary of what a green CI check means on +an unreleased branch see [CONTRIBUTING.md](../CONTRIBUTING.md#work-towards-a-merge). + +## Contents + +1. [One-time setup when the PR first merges](#one-time-setup-when-the-pr-first-merges) +2. [Creating a new maintenance branch](#creating-a-new-maintenance-branch) +3. [When a Gramps minor release lands on PyPI](#when-a-gramps-minor-release-lands-on-pypi) +4. [Diagnostic log markers](#diagnostic-log-markers) +5. [Optional future-proofing knobs](#optional-future-proofing-knobs) + +## One-time setup when the PR first merges + +### 1. Make the `gramps-ci` GHCR package public + +`docker-build.yml` pushes images to +`ghcr.io/gramps-project/addons-source/gramps-ci:` using the +workflow's `GITHUB_TOKEN`. GHCR creates the package as **private** the +first time. Same-repo CI keeps working (token covers own packages), +but **fork PRs cannot pull the image** because their `GITHUB_TOKEN` +has no read access to private packages in `gramps-project`. Fork-PR +container jobs would fail at "Initialize containers" with an +authentication error. + +Fix once, immediately after the first `Build Docker Images` run +finishes: + +1. Go to +2. Under "Danger Zone" → "Change visibility" → set to **Public** + +Every existing and future `gramps-ci:` tag inherits public +visibility from this single setting. + +### 2. Expect the first-push race on `maintenance/gramps60` + +The first push event after merge fires both workflows in parallel: + +- `Build Docker Images` builds and pushes `gramps-ci:gramps60` + (~5 min cold). +- `CI` runs `setup` (~2 s), then its container jobs try to pull the + image. + +Because both start on the same push, the CI container jobs race the +image push and may fail at "Initialize containers" the first time. +This race only happens once per branch: + +1. Wait for `Build Docker Images` to complete. +2. Open the failed CI run → click "Re-run failed jobs". +3. Subsequent pushes find the image already in GHCR — no race. + +This is also explained in the header comment of `ci.yml`. + +## Creating a new maintenance branch + +When a new Gramps minor series goes into development and addons need +a corresponding branch (e.g. `maintenance/gramps62` once 6.2 opens): + +``` +git branch maintenance/gramps62 maintenance/gramps61 +git push origin maintenance/gramps62 +``` + +No workflow edits required. The workflows derive everything from +`github.ref_name`. The first push fires the same race described +above — re-run the failed CI jobs once `Build Docker Images` +finishes. + +The setup job's regex (`gramps[0-9][0-9]`) requires a two-digit +suffix. When Gramps 10.0 opens this regex needs updating in two +places (`ci.yml` setup job and `docker-build.yml` params step). + +## When a Gramps minor release lands on PyPI + +The hybrid Dockerfile auto-detects PyPI availability: + +- `pip install "gramps==X.Y.*"` succeeds → image installs the tagged + PyPI release (`::notice::` log line). +- pip reports "No matching distribution found" → image falls back to a + SHA-pinned `git clone` of `gramps-project/gramps@maintenance/grampsNN` + at the SHA captured by `docker-build.yml`'s params step + (`::warning::` log line). + +When `gramps==6.1.0` is finally published to PyPI, the next image +rebuild on `maintenance/gramps61` silently switches from "git tip" to +"PyPI release" — no maintainer action needed. + +To **immediately** rebuild against the new PyPI release without +waiting for the next push: + +1. Open the `Build Docker Images` workflow in the Actions tab. +2. "Run workflow" → select `maintenance/grampsNN` → Run. + +The `::notice::` line in the build log confirms the switch. + +## Diagnostic log markers + +When investigating a CI failure, the install-step output in +`Build Docker Images` carries these annotations: + +| Annotation | Meaning | +| --- | --- | +| `::notice::installed gramps==X.Y.* from PyPI` | Released-path install. CI is testing against a tagged release. | +| `::warning::no gramps==X.Y.* on PyPI; installing from gramps-project/gramps@maintenance/grampsNN at ` | Unreleased-branch fallback. CI is testing against the upstream branch tip at ``. Visible to contributors via the CONTRIBUTING.md note. | +| `::error::no gramps==X.Y.* on PyPI and GRAMPS_FALLBACK_SHA is unset` | `git ls-remote` in `docker-build.yml`'s params step returned no SHA for `maintenance/grampsNN` on `gramps-project/gramps`. The matching upstream branch is missing, or the addons-source branch is misnamed. | +| `::error::pip install gramps failed (non-version reason)` | pip failed for a network/registry reason, not because the version is missing. Captured stderr is dumped after the line. The build does **not** fall back to git in this case — by design, so a transient PyPI hiccup cannot silently flip a released branch into "git tip" mode. | + +Other useful entry points: + +- `ci.yml` setup job log shows the derived `branch_suffix` and + `ci_image`. A failure here means the branch name doesn't match + `maintenance/gramps[0-9][0-9]`. +- `docker-build.yml` params step log shows the captured + `fallback_sha`. An empty value means upstream `gramps-project/gramps` + has no matching maintenance branch (warning issued; image build will + fail iff the fallback is needed). + +## Optional future-proofing knobs + +Not needed today; record here in case they ever come up. + +### Upstream gramps repo URL + +The Dockerfile hardcodes `https://github.com/gramps-project/gramps.git` +as the fallback source. If the project ever reorgs or renames, this +URL must change in one place (`.github/docker/gramps-ci/Dockerfile`). +It could be parameterised as a build arg (`ARG GRAMPS_UPSTREAM_REPO`) +with the current URL as default, but a hardcoded value keeps the +Dockerfile simpler and a rename is a sufficiently large event that +editing one string is not the bottleneck. + +### GHCR tag retention + +`docker-build.yml` pushes both a moving `gramps-ci:` tag (e.g. +`gramps60`) and a per-commit `gramps-ci:-` tag. +The moving tag is always overwritten; the SHA tags accumulate over +time. Set a retention policy on the GHCR package settings page if +the count becomes inconvenient — the moving tags are what CI consumes. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ebd61d3b..8ad018e0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,9 @@ name: CI # failed jobs") and they will pull the now-existing image. This race # happens only on initial branch creation — every subsequent push to # that branch finds the image already in GHCR. +# +# Full operational runbook (GHCR visibility, PyPI-release transitions, +# diagnostic log markers, etc.): .github/CI-MAINTAINER.md on: push: From ad2815cee357543f5f546bf1f39ce88515de3b68 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Sun, 24 May 2026 12:21:57 +0200 Subject: [PATCH 18/18] tests: detect addons that import a sibling without depends_on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds tests/test_addon_dependencies.py, an independent detector for the class of bug behind Mantis 13707 (WebConnect packs importing libwebconnect without declaring depends_on=["libwebconnect"], so install fails when libwebconnect is absent). Picked up automatically by the integration-test job's `python3 -m unittest discover -s tests -p "test_*.py"` — no ci.yml change. Design - Independent of Gramps' loader. Does NOT import gramps.gen.plug — no PluginRegister, no BasePluginManager, no PluginData. Using Gramps' loader would test addons through the very dependency resolver whose leniency let 13707 ship, and would tie this test to Gramps' internal, unstable API. - Reads every *.gpr.py via an exec-shim of its own: a permissive globals dict (LOAD_GLOBAL → __missing__ → sentinel, so plugin-type constants like GRAMPLET / REPORT / TOOL never NameError) plus a fake register() that records each call's kwargs. Builds the addon-provided-module set and an id → {modules, depends_on, requires_mod, dir} map. - Isolated-load each plugin's registered module in a fresh subprocess with sys.path = [target_dir] + dep_dirs (declared depends_on ids resolved → their directories via the index). Uses `python3 -I`, cwd=/tmp, PYTHONPATH stripped, and strips "" from sys.path defensively — without those, PEP 420 implicit namespace packages let sibling addons import as empty packages and silently defeat the isolation. Gramps remains reachable from system site-packages, as intended: the isolation is addon-from-addon, not addon-from-Gramps. - Pins GI namespace versions Gramps itself pins (Gtk 3.0, PangoCairo 1.0, etc.) before importing the addon. Not Gramps' loader — just matching the runtime conditions a plugin is loaded under, so addons whose top-level `from gi.repository import X` is version-sensitive don't generate false (c) failures from ambiguous GI defaults. - Classifies failures: (a) missing name is an addon-provided module not in this addon's depends_on — FINDING, fails the test; (b) missing name is in this addon's requires_mod — environment concern (PR 820's auto-derive owns it), ignored; (c) anything else — logged separately, not a finding. A given stderr may name multiple missing modules; (a) wins over (b) wins over (c) so the highest-signal finding is reported. Limitation, stated in the module docstring: catches undeclared dependencies that manifest at LOAD time (top-level imports). MISSES lazily-imported deps — a sibling addon imported inside a function not called at module load. No false positives, not exhaustive. Verified - Synthetic positive/negative: with depends_on=["libwebconnect"] stripped from a USWebConnectPack copy, detector flags bucket (a) `libwebconnect`; with the declaration left in, rc=0 clean load. (Both checks live under /tmp/, not committed; they validate the classifier and isolation end-to-end.) - Full tree run in gramps-ci-local:gramps60-test (PR 820's CI image, Gramps 6.0.8): 187 plugins indexed, 159 pass, 0 (a), 7 (b) ignored, 21 (c) logged. Every WebConnect pack on this branch already declares depends_on=["libwebconnect"] — the 13707 declaration is in effect on gramps60, so the test is green here. On gramps61 the situation may differ and the remediation round will be informed by re-running this detector there. This commit is the detector + the regression check. It does NOT apply any depends_on fixes — the WebConnect remediation is a separate later round on maintenance/gramps61. Issue #13707. Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/test_addon_dependencies.py | 421 +++++++++++++++++++++++++++++++ 1 file changed, 421 insertions(+) create mode 100644 tests/test_addon_dependencies.py diff --git a/tests/test_addon_dependencies.py b/tests/test_addon_dependencies.py new file mode 100644 index 000000000..6baf2a876 --- /dev/null +++ b/tests/test_addon_dependencies.py @@ -0,0 +1,421 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Detect addons that USE another addon without declaring it in ``depends_on``. + +This is the bug class behind Mantis 13707: the WebConnect packs imported +``libwebconnect`` at module load without declaring it, so installing a +pack without libwebconnect already present failed. + +The detector is INDEPENDENT of Gramps' plugin loader by design: + +* it reads ``.gpr.py`` with an exec-shim of its own (no + ``gramps.gen.plug._pluginreg`` import, no PluginManager, no + PluginRegister), because using Gramps' loader would test addons + through the very dependency resolver whose leniency lets the bug + ship — and would tie the test to Gramps' internal, unstable API; +* it loads each registered module in a fresh subprocess with + ``sys.path`` scoped to that addon's directory plus the directories + of its declared ``depends_on`` (so a real missing dep blows up the + way it would on a clean install), and parses the resulting + exception to classify it. + +The Gramps runtime is allowed to be importable from the subprocess — +addons do ``from gi.repository import Gtk`` and ``from gramps.gen.lib +import X`` at load time. The isolation being enforced is addon-from- +addon, not addon-from-Gramps. + +LIMITATION (important): +This catches undeclared addon dependencies that manifest at LOAD time +(top-level imports). It MISSES lazily-imported deps — e.g. a sibling +addon imported inside a function that is not called at module load. +No false positives, but not exhaustive — do not let it be mistaken +for one. + +Failures are bucketed: + +a. ``undeclared_addon_dep`` — the import error names a module that + another addon in this tree provides AND that this addon does not + declare in ``depends_on``. This is a FINDING and fails the test. +b. ``missing_requires_mod`` — the import error names a module the + addon declares in ``requires_mod`` (e.g. ``litellm``). That is an + environment concern, not a dependency-declaration bug; ignored. +c. ``other`` — any other isolated-load failure. Logged so the + information is not lost, but NOT a finding and NOT a test failure. + Examples: GI namespace mismatches, host-environment library issues, + addon import-time side effects requiring full GUI state. +""" + +# ------------------------ +# Python modules +# ------------------------ +import logging +import os +import re +import subprocess +import sys +import textwrap +import unittest +from typing import Any + +LOG = logging.getLogger(__name__) + +ADDONS_ROOT: str = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# ------------------------ +# .gpr.py exec shim +# ------------------------ +class _GprSentinel: + """Stands in for any unresolved plugin-type/category constant. + + The shim does not need real enum values — only kwargs captured by + a fake ``register()`` call. Attribute access, calls, and arithmetic + on the sentinel all return the sentinel so common patterns inside + ``.gpr.py`` files do not raise. + """ + + def __repr__(self) -> str: + return "" + + def __getattr__(self, name: str) -> "_GprSentinel": + return self + + def __call__(self, *args: Any, **kwargs: Any) -> "_GprSentinel": + return self + + +_SENT = _GprSentinel() + + +class _PermissiveGlobals(dict): + """Globals dict that returns a sentinel for any unknown plain name. + + CPython's ``LOAD_GLOBAL`` opcode calls ``__getitem__`` (and hence + ``__missing__``) on ``dict`` subclasses, so unresolved plugin-type + constants (``GRAMPLET``, ``REPORT``, ``CATEGORY_TEXT``, …) do not + raise ``NameError`` inside the exec. Dunder names still miss so + Python's own machinery behaves normally. + """ + + def __missing__(self, key: str) -> Any: + if key.startswith("__"): + raise KeyError(key) + return _SENT + + +def _exec_gpr(gpr_path: str) -> list[dict]: + """Exec one ``.gpr.py`` with a fake ``register()`` and return its kwargs. + + :param gpr_path: Absolute path to a ``*.gpr.py`` file. + :returns: One dict per ``register()`` call, with the kwargs verbatim + plus ``_ptype`` for the positional plugin type. + """ + plugins: list[dict] = [] + + def register(ptype: Any, **kwargs: Any) -> None: + kwargs["_ptype"] = ptype + plugins.append(kwargs) + + env = _PermissiveGlobals( + { + "__builtins__": __builtins__, + "__file__": gpr_path, + "__name__": "_gpr_shim", + "register": register, + "_": lambda s, *a, **k: s, + } + ) + with open(gpr_path, "r", encoding="utf-8") as f: + src = f.read() + exec(compile(src, gpr_path, "exec"), env) + return plugins + + +def _index_addons( + addons_root: str, +) -> tuple[dict[str, dict], set[str], list[tuple[str, str]]]: + """Walk ``addons_root`` and build the addon metadata index. + + :returns: ``(id_to_addon, all_modules, skipped)`` where + ``id_to_addon[plugin_id]`` is + ``{directory, modules, depends_on, requires_mod, gpr_files}``, + ``all_modules`` is the set of registered module names across + the whole tree, and ``skipped`` is a list of + ``(gpr_path, error)`` tuples for files whose exec raised. + """ + id_to_addon: dict[str, dict] = {} + all_modules: set[str] = set() + skipped: list[tuple[str, str]] = [] + + for dirname in sorted(os.listdir(addons_root)): + addon_dir = os.path.join(addons_root, dirname) + if not os.path.isdir(addon_dir): + continue + if dirname.startswith("."): + continue + gpr_files = sorted( + os.path.join(addon_dir, f) + for f in os.listdir(addon_dir) + if f.endswith(".gpr.py") + ) + if not gpr_files: + continue + for gpr in gpr_files: + try: + plugins = _exec_gpr(gpr) + except BaseException as exc: # noqa: BLE001 + skipped.append((gpr, f"{type(exc).__name__}: {exc}")) + continue + for plugin in plugins: + pid = plugin.get("id") + fname = plugin.get("fname") + if not isinstance(pid, str) or not isinstance(fname, str): + continue + module = re.sub(r"\.py$", "", fname) + rec = id_to_addon.setdefault( + pid, + { + "directory": addon_dir, + "modules": [], + "depends_on": [], + "requires_mod": [], + "gpr_files": [], + }, + ) + if module not in rec["modules"]: + rec["modules"].append(module) + for dep in plugin.get("depends_on") or []: + if isinstance(dep, str) and dep not in rec["depends_on"]: + rec["depends_on"].append(dep) + for req in plugin.get("requires_mod") or []: + if isinstance(req, str) and req not in rec["requires_mod"]: + rec["requires_mod"].append(req) + if gpr not in rec["gpr_files"]: + rec["gpr_files"].append(gpr) + all_modules.add(module) + return id_to_addon, all_modules, skipped + + +# ------------------------ +# Isolated-load subprocess +# ------------------------ +_LOADER = textwrap.dedent( + """ + import sys, importlib, traceback + # Strip the implicit "" CWD entry. Without this, if the subprocess + # is run from a directory that contains addon subdirectories, PEP + # 420 implicit namespace packages let sibling addons import as + # empty packages — which silently defeats the isolation we are + # trying to enforce. The caller also chdirs to a neutral directory, + # but stripping "" makes the isolation independent of CWD. + sys.path[:] = [p for p in sys.path if p not in ("", ".")] + # Pin the GI namespace versions Gramps itself pins before loading + # any plugin, so addons whose top-level `from gi.repository import X` + # is version-sensitive do not generate false (c) failures from + # ambiguous GI defaults. This is NOT Gramps' loader; it is matching + # the runtime conditions an addon is loaded under. + try: + import gi + for ns, ver in ( + ("Gtk", "3.0"), + ("PangoCairo", "1.0"), + ("OsmGpsMap", "1.0"), + ("GExiv2", "0.10"), + ("Gspell", "1"), + ("GeocodeGlib", "1.0"), + ): + try: + gi.require_version(ns, ver) + except (ValueError, AttributeError): + pass + except ImportError: + pass + target_dir = {target_dir!r} + dep_dirs = {dep_dirs!r} + sys.path[:0] = [target_dir] + list(dep_dirs) + try: + importlib.import_module({module!r}) + except BaseException: + traceback.print_exc() + sys.exit(2) + sys.exit(0) + """ +) + + +def _isolated_load( + target_dir: str, dep_dirs: list[str], module: str, timeout: int = 30 +) -> tuple[int, str]: + """Spawn a subprocess that tries to import ``module`` in isolation. + + The subprocess is run from a neutral CWD (the system temp dir) and + with ``PYTHONPATH`` stripped from the environment, so neither the + parent's working directory nor a stray ``PYTHONPATH`` can leak + sibling-addon paths into the child's ``sys.path``. + + :returns: ``(returncode, stderr)``. Returncode ``-1`` indicates + the subprocess timed out. + """ + code = _LOADER.format(target_dir=target_dir, dep_dirs=dep_dirs, module=module) + env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"} + try: + proc = subprocess.run( + [sys.executable, "-I", "-c", code], + capture_output=True, + text=True, + timeout=timeout, + cwd="/tmp", + env=env, + ) + except subprocess.TimeoutExpired: + return -1, "TIMEOUT" + return proc.returncode, proc.stderr + + +_MISSING_NAME_RE = re.compile(r"No module named ['\"]([^'\"]+)['\"]") + + +def _classify( + stderr: str, + declared_dep_modules: set[str], + requires_mod: set[str], + all_addon_modules: set[str], +) -> tuple[str, str]: + """Bucket a failed isolated-load. + + A given failure may name multiple missing modules. Bucket (a) wins + over (b) wins over (c), so the highest-signal finding is reported. + """ + missing = [m.split(".")[0] for m in _MISSING_NAME_RE.findall(stderr)] + for name in missing: + if name in all_addon_modules and name not in declared_dep_modules: + return ("a_undeclared_addon_dep", name) + for name in missing: + if name in requires_mod: + return ("b_requires_mod", name) + last_lines = stderr.strip().split("\n")[-5:] + return ("c_other", last_lines[-1] if last_lines else "") + + +# ------------------------------------------------------------ +# +# TestAddonDependencies +# +# ------------------------------------------------------------ +class TestAddonDependencies(unittest.TestCase): + """Fail when any addon imports a sibling addon it does not declare.""" + + id_to_addon: dict[str, dict] = {} + all_modules: set[str] = set() + skipped_gpr: list[tuple[str, str]] = [] + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.id_to_addon, cls.all_modules, cls.skipped_gpr = _index_addons(ADDONS_ROOT) + + def test_no_undeclared_addon_dependencies(self) -> None: + """Every addon's registered modules must import in isolation, given + only the directories of its declared ``depends_on``. + + A failure that names a sibling-addon module not listed in + ``depends_on`` is a finding (the #13707 class). A failure that + names a declared ``requires_mod`` is the environment, not the + declaration. Anything else is logged separately. + """ + self.assertGreater( + len(self.id_to_addon), 0, "No addons found — index is empty" + ) + + findings_a: list[str] = [] + findings_c: list[str] = [] + counts = {"pass": 0, "a": 0, "b": 0, "c": 0} + + for pid in sorted(self.id_to_addon): + rec = self.id_to_addon[pid] + target_dir = rec["directory"] + dep_dirs: list[str] = [] + declared_modules: set[str] = set() + for dep_id in rec["depends_on"]: + dep_rec = self.id_to_addon.get(dep_id) + if dep_rec is None: + continue + if dep_rec["directory"] not in dep_dirs: + dep_dirs.append(dep_rec["directory"]) + declared_modules.update(dep_rec["modules"]) + requires_mod_set = set(rec["requires_mod"]) + + for module in rec["modules"]: + rc, err = _isolated_load(target_dir, dep_dirs, module) + if rc == 0: + counts["pass"] += 1 + continue + bucket, detail = _classify( + err, declared_modules, requires_mod_set, self.all_modules + ) + if bucket == "a_undeclared_addon_dep": + counts["a"] += 1 + findings_a.append( + f" {pid} (module {module}) — undeclared addon dep: " + f"{detail}" + ) + elif bucket == "b_requires_mod": + counts["b"] += 1 + else: + counts["c"] += 1 + findings_c.append(f" {pid} (module {module}) — {detail}") + + LOG.info( + "Indexed %d plugins; load pass=%d, bucket a=%d, b=%d, c=%d; " + "gpr exec skipped=%d", + len(self.id_to_addon), + counts["pass"], + counts["a"], + counts["b"], + counts["c"], + len(self.skipped_gpr), + ) + if self.skipped_gpr: + LOG.warning( + "%d .gpr.py file(s) failed exec-shim and were skipped:\n%s", + len(self.skipped_gpr), + "\n".join(f" {p}: {e}" for p, e in self.skipped_gpr), + ) + if findings_c: + LOG.warning( + "%d addon(s) failed isolated load for non-dependency reasons " + "(NOT a finding — environment / GUI-state / import-time side " + "effects):\n%s", + len(findings_c), + "\n".join(findings_c), + ) + + if findings_a: + self.fail( + "Found %d addon(s) that import a sibling addon without " + "declaring it in depends_on:\n%s" + % (len(findings_a), "\n".join(findings_a)) + ) + + +if __name__ == "__main__": + unittest.main()