From 9c86c6bee0c69b3d91a361717a205e3c329c7da9 Mon Sep 17 00:00:00 2001 From: DaveSch-gramps Date: Wed, 3 Jun 2026 09:17:09 -0400 Subject: [PATCH 01/20] Add Integrations to the Themes panels Adds the new FS Integrations preferences Tab to the Themes' overriding panel list. --- Themes/themes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Themes/themes.py b/Themes/themes.py index f2e0a17e5..5438fec68 100644 --- a/Themes/themes.py +++ b/Themes/themes.py @@ -102,6 +102,7 @@ def __init__(self, uistate, dbstate): self.add_text_panel, self.add_warnings_panel, self.add_researcher_panel, + self.add_integrations_panel, self.add_ptypes_panel, self.add_themes_panel) ConfigureDialog.__init__(self, uistate, dbstate, page_funcs, @@ -120,6 +121,7 @@ def __init__(self, uistate, dbstate): self.add_text_panel, self.add_warnings_panel, self.add_researcher_panel, + self.add_integrations_panel, self.add_themes_panel) ConfigureDialog.__init__(self, uistate, dbstate, page_funcs, GrampsPreferences, config, From 26e645d0d1abb54f98e32fed1270a6a5f61ee7d6 Mon Sep 17 00:00:00 2001 From: GaryGriffin Date: Sun, 7 Jun 2026 09:31:56 -0700 Subject: [PATCH 02/20] Merge Add Integrations to the Themes panels #937 --- Themes/themes.gpr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Themes/themes.gpr.py b/Themes/themes.gpr.py index b2215f0d1..24818b0fd 100644 --- a/Themes/themes.gpr.py +++ b/Themes/themes.gpr.py @@ -31,7 +31,7 @@ "An addition to Preferences for simple Theme and Font" " adjustment. Especially useful for Windows users." ), - version = '0.0.19', + version = '0.0.20', gramps_target_version="6.1", fname="themes_load.py", authors=["Paul Culley"], From 0852c1ef795a60befcd4cc080bbd8c9503b4018f Mon Sep 17 00:00:00 2001 From: Brian Caudill Date: Thu, 4 Jun 2026 15:15:39 -0400 Subject: [PATCH 03/20] PlaceCoordinateGramplet: cache forward-geocode results per session Re-searching the same place name now returns instantly from an in-session cache instead of re-querying the geocoding service, which is faster and kinder to the Nominatim usage policy. Applies to both the geopy/Nominatim and GeocodeGlib code paths. --- .../PlaceCoordinateGramplet.py | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/PlaceCoordinateGramplet/PlaceCoordinateGramplet.py b/PlaceCoordinateGramplet/PlaceCoordinateGramplet.py index 80532412c..85ba5a011 100644 --- a/PlaceCoordinateGramplet/PlaceCoordinateGramplet.py +++ b/PlaceCoordinateGramplet/PlaceCoordinateGramplet.py @@ -202,14 +202,30 @@ def on_searchButton_clicked(self, widget): # lat = config.get("geography.center-lat") # lon = config.get("geography.center-lon") # self.osm.grab_focus() + query = self.entry_name.get_text().strip() + # Cache forward-geocode results for this session so re-searching the same + # place name is instant and does not query the geocoding service again + # (faster, and kinder to the Nominatim usage policy). + cache = getattr(self, "_geocode_cache", None) + if cache is None: + cache = self._geocode_cache = {} + if query in cache: + lat, lon, found = cache[query] + self.entry_lat.set_text(lat) + self.entry_long.set_text(lon) + self.entry_foundName.set_text(found) + return if use_geopy: geolocator = Nominatim(user_agent="GrampsPlaceCoordinateGramplet") try : - location = geolocator.geocode(self.entry_name.get_text()) + location = geolocator.geocode(query) if location: - self.entry_lat.set_text("%.10f" % location.latitude) - self.entry_long.set_text("%.10f" % location.longitude) + lat = "%.10f" % location.latitude + lon = "%.10f" % location.longitude + self.entry_lat.set_text(lat) + self.entry_long.set_text(lon) self.entry_foundName.set_text(location.address) + cache[query] = (lat, lon, location.address) else: self.entry_foundName.set_text( _("The place was not found. " @@ -220,8 +236,7 @@ def on_searchButton_clicked(self, widget): "some unexpected error.") + ' ' + str(e)) else: try: - location_ = GeocodeGlib.Forward.new_for_string( - self.entry_name.get_text()) + location_ = GeocodeGlib.Forward.new_for_string(query) try: result = location_.search() error_message = "You may clarify the search keywords." @@ -235,10 +250,14 @@ def on_searchButton_clicked(self, widget): if result.get_property(p.name)) geo_loc = location_information['location'] - self.entry_lat.set_text("%.10f" % geo_loc.get_latitude()) - self.entry_long.set_text("%.10f" % geo_loc.get_longitude()) - self.entry_foundName.set_text( - generate_address_string(location_information, STR_ADDRESS_CONFIG)) + lat = "%.10f" % geo_loc.get_latitude() + lon = "%.10f" % geo_loc.get_longitude() + found = generate_address_string( + location_information, STR_ADDRESS_CONFIG) + self.entry_lat.set_text(lat) + self.entry_long.set_text(lon) + self.entry_foundName.set_text(found) + cache[query] = (lat, lon, found) else: self.entry_foundName.set_text( _("The place was not found.") + ' ' + error_message) From 91390ab1bfa38cb9102a435f05d0cd538f7546e1 Mon Sep 17 00:00:00 2001 From: GaryGriffin Date: Sun, 7 Jun 2026 09:49:31 -0700 Subject: [PATCH 04/20] Merge PlaceCoordinateGramplet: cache forward-geocode results per session#939 --- PlaceCoordinateGramplet/PlaceCoordinateGramplet.gpr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PlaceCoordinateGramplet/PlaceCoordinateGramplet.gpr.py b/PlaceCoordinateGramplet/PlaceCoordinateGramplet.gpr.py index dc80dd288..15eb2c874 100644 --- a/PlaceCoordinateGramplet/PlaceCoordinateGramplet.gpr.py +++ b/PlaceCoordinateGramplet/PlaceCoordinateGramplet.gpr.py @@ -23,7 +23,7 @@ id="geoIDplaceCoordinateGramplet", name=_("Place Coordinate Gramplet view"), description=_("View for the place coordinate gramplet."), - version = '1.1.28', + version = '1.1.29', gramps_target_version="6.1", status=STABLE, fname="PlaceCoordinateGeoView.py", @@ -42,7 +42,7 @@ id="Place Coordinates", name=_("Place and Coordinates"), description=_("Gramplet that simplifies setting the coordinates of a place"), - version = '1.1.28', + version = '1.1.29', gramps_target_version="6.1", status=STABLE, fname="PlaceCoordinateGramplet.py", From d136b942f5b979282474c35da4f6d375f3e63a96 Mon Sep 17 00:00:00 2001 From: megabyte0x <79016290+megabyte0x@users.noreply.github.com> Date: Fri, 5 Jun 2026 10:29:49 +0530 Subject: [PATCH 05/20] Add FinalNotes obituary research link --- WebSearch/assets/csv/static-links.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/WebSearch/assets/csv/static-links.csv b/WebSearch/assets/csv/static-links.csv index 825db4d7b..019939206 100644 --- a/WebSearch/assets/csv/static-links.csv +++ b/WebSearch/assets/csv/static-links.csv @@ -1,3 +1,4 @@ Navigation type,Title,Is enabled,URL,Comment +People,FinalNotes Obituary Research Guide,1,https://www.finalnotes.page/obituary-research-guide/,Guide for obituary research and turning records into life-story source material "People,Places",Static example 1,1,https://www.google.com/search?q=I+like+this+link1,Shown only in People and Places tabs *,Static example 2,1,https://www.google.com/search?q=I+like+this+link2,Shown in all tabs except People and Places From af8af6634ca0945e9e6c5d3ebfe0f3ba5988de90 Mon Sep 17 00:00:00 2001 From: GaryGriffin Date: Sun, 7 Jun 2026 12:30:53 -0700 Subject: [PATCH 06/20] Merge WebSearch: add FinalNotes obituary research guide#940 --- WebSearch/WebSearch.gpr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WebSearch/WebSearch.gpr.py b/WebSearch/WebSearch.gpr.py index ecf6a5850..eb183bfe4 100644 --- a/WebSearch/WebSearch.gpr.py +++ b/WebSearch/WebSearch.gpr.py @@ -37,7 +37,7 @@ "Person, Place, Family, or Source record" ), status=STABLE, - version = '1.10.19', + version = '1.10.20', fname="WebSearch.py", height=20, detached_width=400, From d8e352d9e4d5a9db9c899dae367fb0c14a083926 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Fri, 5 Jun 2026 09:28:58 -0700 Subject: [PATCH 07/20] Fix PostgreSQL collation check to support versions before 12. The previous code used DROP+CREATE on every connection startup. Replace with a pg_collation existence check so the collation is only created when absent. This also preserves column_exists and drop_column methods added for schema migration support. Also apply black formatting throughout the file. Fixes #65803. Co-Authored-By: Claude Sonnet 4.6 --- PostgreSQL/postgresql.py | 92 ++++++++++++++++++++++------------------ 1 file changed, 51 insertions(+), 41 deletions(-) diff --git a/PostgreSQL/postgresql.py b/PostgreSQL/postgresql.py index 8ffa75dab..867b7bf45 100644 --- a/PostgreSQL/postgresql.py +++ b/PostgreSQL/postgresql.py @@ -23,39 +23,41 @@ Backend for PostgreSQL database. """ -#------------------------------------------------------------------------- +# ------------------------------------------------------------------------- # # Standard python modules # -#------------------------------------------------------------------------- +# ------------------------------------------------------------------------- import psycopg2 import os import re -#------------------------------------------------------------------------- +# ------------------------------------------------------------------------- # # Gramps modules # -#------------------------------------------------------------------------- +# ------------------------------------------------------------------------- from gramps.plugins.db.dbapi.dbapi import DBAPI from gramps.gen.utils.configmanager import ConfigManager from gramps.gen.config import config from gramps.gen.db.dbconst import ARRAYSIZE from gramps.gen.db.exceptions import DbConnectionError from gramps.gen.const import GRAMPS_LOCALE as glocale + try: _trans = glocale.get_addon_translator(__file__) except ValueError: _trans = glocale.translation _ = _trans.gettext -psycopg2.paramstyle = 'format' +psycopg2.paramstyle = "format" + -#------------------------------------------------------------------------- +# ------------------------------------------------------------------------- # # PostgreSQL class # -#------------------------------------------------------------------------- +# ------------------------------------------------------------------------- class PostgreSQL(DBAPI): def get_summary(self): @@ -64,42 +66,44 @@ def get_summary(self): backend. """ summary = super().get_summary() - summary.update({ - _("Database version"): psycopg2.__version__, - _("Database module location"): psycopg2.__file__, - }) + summary.update( + { + _("Database version"): psycopg2.__version__, + _("Database module location"): psycopg2.__file__, + } + ) return summary def requires_login(self): return True def _initialize(self, directory, username, password): - config_file = os.path.join(directory, 'settings.ini') + config_file = os.path.join(directory, "settings.ini") config_mgr = ConfigManager(config_file) - config_mgr.register('database.dbname', '') - config_mgr.register('database.host', '') - config_mgr.register('database.port', '') + config_mgr.register("database.dbname", "") + config_mgr.register("database.host", "") + config_mgr.register("database.port", "") if not os.path.exists(config_file): - name_file = os.path.join(directory, 'name.txt') - with open(name_file, 'r', encoding='utf8') as file: + name_file = os.path.join(directory, "name.txt") + with open(name_file, "r", encoding="utf8") as file: dbname = file.readline().strip() - config_mgr.set('database.dbname', dbname) - config_mgr.set('database.host', config.get('database.host')) - config_mgr.set('database.port', config.get('database.port')) + config_mgr.set("database.dbname", dbname) + config_mgr.set("database.host", config.get("database.host")) + config_mgr.set("database.port", config.get("database.port")) config_mgr.save() config_mgr.load() dbkwargs = {} - for key in config_mgr.get_section_settings('database'): - value = config_mgr.get('database.' + key) + for key in config_mgr.get_section_settings("database"): + value = config_mgr.get("database." + key) if value: dbkwargs[key] = value if username: - dbkwargs['user'] = username + dbkwargs["user"] = username if password: - dbkwargs['password'] = password + dbkwargs["password"] = password try: self.dbapi = Connection(**dbkwargs) @@ -107,11 +111,11 @@ def _initialize(self, directory, username, password): raise DbConnectionError(str(msg), config_file) -#------------------------------------------------------------------------- +# ------------------------------------------------------------------------- # # Connection class # -#------------------------------------------------------------------------- +# ------------------------------------------------------------------------- class Connection: def __init__(self, *args, **kwargs): @@ -125,13 +129,17 @@ def check_collation(self, locale): Checks that a collation exists and if not creates it. :param locale: Locale to be checked. - :param type: A GrampsLocale object. + :type locale: A GrampsLocale object. """ - # Duplicating system collations works, but to delete them the schema - # must be specified, so get the current schema collation = locale.get_collation() - self.execute('CREATE COLLATION IF NOT EXISTS "%s"' - "(LOCALE = '%s')" % (collation, locale.collation)) + # Use pg_collation to check existence rather than IF NOT EXISTS, which + # requires PostgreSQL 12+. + self.execute("SELECT 1 FROM pg_collation WHERE collname = %s", [collation]) + if not self.fetchone(): + self.execute( + "CREATE COLLATION \"%s\" (LOCALE = '%s')" + % (collation, locale.collation) + ) def _hack_query(self, query): query = query.replace("?", "%s") @@ -142,16 +150,17 @@ def _hack_query(self, query): ## count can be -1, for all ## LIMIT -1 ## LIMIT offset, -1 - query = query.replace("LIMIT -1", - "LIMIT all") ## + query = query.replace("LIMIT -1", "LIMIT all") ## match = re.match(".* LIMIT (.*), (.*) ", query) if match and match.groups(): offset, count = match.groups() if count == "-1": count = "all" - query = re.sub("(.*) LIMIT (.*), (.*) ", - "\\1 LIMIT %s OFFSET %s " % (count, offset), - query) + query = re.sub( + "(.*) LIMIT (.*), (.*) ", + "\\1 LIMIT %s OFFSET %s " % (count, offset), + query, + ) return query def execute(self, *args, **kwargs): @@ -185,9 +194,10 @@ def rollback(self): self.__connection.rollback() def table_exists(self, table): - self.__cursor.execute("SELECT COUNT(*) " - "FROM information_schema.tables " - "WHERE table_name=%s;", [table]) + self.__cursor.execute( + "SELECT COUNT(*) " "FROM information_schema.tables " "WHERE table_name=%s;", + [table], + ) return self.fetchone()[0] != 0 @@ -225,11 +235,11 @@ def cursor(self): return Cursor(self.__connection) -#------------------------------------------------------------------------- +# ------------------------------------------------------------------------- # # Cursor class # -#------------------------------------------------------------------------- +# ------------------------------------------------------------------------- class Cursor: def __init__(self, connection): self.__connection = connection From cdca6f9d33278428b519743b3bd098c79eb346a3 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Fri, 5 Jun 2026 11:34:21 -0700 Subject: [PATCH 08/20] Replace _hack_query() with proper dialect methods in PostgreSQL addon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the monolithic _hack_query() string-substitution function and replace it with dialect-aware overrides on the PostgreSQL class: - _sql_type() returns 'bytea' instead of 'BLOB' - _quote_column() properly quotes PostgreSQL reserved words (e.g. 'desc') rather than blindly renaming them; marked for removal once gramps PR #2178 (which adds _quote_column to the DBAPI base) is merged - Connection.execute() handles only the two remaining SQL syntax differences: qmark→format paramstyle and REGEXP→~ operator - Drop dead LIMIT rewriting code (not generated by current dbapi.py) - Remove now-unused 'import re' Co-Authored-By: Claude Sonnet 4.6 --- PostgreSQL/postgresql.py | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/PostgreSQL/postgresql.py b/PostgreSQL/postgresql.py index 867b7bf45..1338d4493 100644 --- a/PostgreSQL/postgresql.py +++ b/PostgreSQL/postgresql.py @@ -30,7 +30,6 @@ # ------------------------------------------------------------------------- import psycopg2 import os -import re # ------------------------------------------------------------------------- # @@ -60,6 +59,17 @@ # ------------------------------------------------------------------------- class PostgreSQL(DBAPI): + dialect = "postgresql" + + def _sql_type(self, schema_type, max_length): + result = super()._sql_type(schema_type, max_length) + return "bytea" if result == "BLOB" else result + + def _quote_column(self, col): + # Remove this method when gramps PR #2178 (dbapi _quote_column) is merged. + _RESERVED = {"desc", "order", "where", "select"} + return f'"{col}"' if col in _RESERVED else col + def get_summary(self): """ Return a diction of information about this database @@ -141,30 +151,9 @@ def check_collation(self, locale): % (collation, locale.collation) ) - def _hack_query(self, query): - query = query.replace("?", "%s") - query = query.replace("REGEXP", "~") - query = query.replace("desc", "desc_") - query = query.replace("BLOB", "bytea") - ## LIMIT offset, count - ## count can be -1, for all - ## LIMIT -1 - ## LIMIT offset, -1 - query = query.replace("LIMIT -1", "LIMIT all") ## - match = re.match(".* LIMIT (.*), (.*) ", query) - if match and match.groups(): - offset, count = match.groups() - if count == "-1": - count = "all" - query = re.sub( - "(.*) LIMIT (.*), (.*) ", - "\\1 LIMIT %s OFFSET %s " % (count, offset), - query, - ) - return query - def execute(self, *args, **kwargs): - sql = self._hack_query(args[0]) + sql = args[0].replace("?", "%s") # qmark → format paramstyle + sql = sql.replace(" REGEXP ", " ~ ") # SQLite REGEXP → PostgreSQL ~ if len(args) > 1: args = args[1] else: From 3772331b546e60d6b6e7108e4e58c72703c3e84e Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Fri, 5 Jun 2026 11:41:43 -0700 Subject: [PATCH 09/20] Use desc_ (not "desc") to stay compatible with existing PostgreSQL databases Quoting the reserved word as "desc" would require a migration for every existing PostgreSQL database where the column was already created as desc_ by the old _hack_query. Using desc_ keeps backward compatibility. Co-Authored-By: Claude Sonnet 4.6 --- PostgreSQL/postgresql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PostgreSQL/postgresql.py b/PostgreSQL/postgresql.py index 1338d4493..2bcd6a439 100644 --- a/PostgreSQL/postgresql.py +++ b/PostgreSQL/postgresql.py @@ -68,7 +68,7 @@ def _sql_type(self, schema_type, max_length): def _quote_column(self, col): # Remove this method when gramps PR #2178 (dbapi _quote_column) is merged. _RESERVED = {"desc", "order", "where", "select"} - return f'"{col}"' if col in _RESERVED else col + return f"{col}_" if col in _RESERVED else col def get_summary(self): """ From 87511b55c8a72b6aa366edef92a53ba8acb41d5c Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Mon, 8 Jun 2026 06:35:05 -0700 Subject: [PATCH 10/20] PostgreSQL: restore forward-compatible LIMIT dialect translations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add back LIMIT offset,count → LIMIT count OFFSET offset and LIMIT -1 → LIMIT ALL rewrites in Connection.execute(), using re.sub with a staticmethod replacer to match the style of the other dialect translations. Co-Authored-By: Claude Sonnet 4.6 --- PostgreSQL/postgresql.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/PostgreSQL/postgresql.py b/PostgreSQL/postgresql.py index 2bcd6a439..800a5ca1f 100644 --- a/PostgreSQL/postgresql.py +++ b/PostgreSQL/postgresql.py @@ -30,6 +30,7 @@ # ------------------------------------------------------------------------- import psycopg2 import os +import re # ------------------------------------------------------------------------- # @@ -151,9 +152,17 @@ def check_collation(self, locale): % (collation, locale.collation) ) + @staticmethod + def _limit_repl(m): + offset, count = m.group(1), m.group(2) + count = 'ALL' if count == '-1' else count + return f'LIMIT {count} OFFSET {offset}' + def execute(self, *args, **kwargs): sql = args[0].replace("?", "%s") # qmark → format paramstyle sql = sql.replace(" REGEXP ", " ~ ") # SQLite REGEXP → PostgreSQL ~ + sql = re.sub(r'\bLIMIT\s+(-?\d+)\s*,\s*(-?\d+)', self._limit_repl, sql, flags=re.IGNORECASE) # LIMIT offset, count → LIMIT count OFFSET offset + sql = re.sub(r'\bLIMIT\s+-1\b', 'LIMIT ALL', sql, flags=re.IGNORECASE) # LIMIT -1 → LIMIT ALL if len(args) > 1: args = args[1] else: From 42f02f9ce2ee9b10ef663521c6a3ea10138e8621 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Mon, 8 Jun 2026 06:37:49 -0700 Subject: [PATCH 11/20] PostgreSQL: add unit tests for SQL dialect translations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 23 tests covering all four rewrite rules in Connection.execute() (qmark→%s, REGEXP→~, LIMIT offset,count, LIMIT -1→ALL), plus _sql_type BLOB→bytea and _quote_column reserved-word handling. psycopg2 is stubbed so no real database is required. Co-Authored-By: Claude Sonnet 4.6 --- PostgreSQL/tests/__init__.py | 0 PostgreSQL/tests/test_sql_translations.py | 277 ++++++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 PostgreSQL/tests/__init__.py create mode 100644 PostgreSQL/tests/test_sql_translations.py diff --git a/PostgreSQL/tests/__init__.py b/PostgreSQL/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/PostgreSQL/tests/test_sql_translations.py b/PostgreSQL/tests/test_sql_translations.py new file mode 100644 index 000000000..24574db43 --- /dev/null +++ b/PostgreSQL/tests/test_sql_translations.py @@ -0,0 +1,277 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2015-2016 Douglas S. Blank +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Unit tests for the PostgreSQL SQL dialect translations in Connection.execute(). + +These tests cover every rewrite rule applied before a query reaches psycopg2: + - qmark → format paramstyle (? → %s) + - REGEXP operator (REGEXP → ~) + - two-arg LIMIT (LIMIT offset, count → LIMIT count OFFSET offset) + - unlimited LIMIT (LIMIT -1 → LIMIT ALL) + +psycopg2 is stubbed so no real database is required. gramps core is +required for the import chain; the whole module is skipped cleanly if +it is not present. + +Run with:: + + python3 -m unittest PostgreSQL.tests.test_sql_translations -v +""" + +# ------------------------------------------------------------------------- +# +# Standard python modules +# +# ------------------------------------------------------------------------- +import os +import sys +import unittest +from unittest import mock + +# ------------------------------------------------------------------------- +# +# Stub psycopg2 before the addon is imported so no real DB driver is needed +# +# ------------------------------------------------------------------------- +ADDON_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if ADDON_DIR not in sys.path: + sys.path.insert(0, ADDON_DIR) + +_mock_psycopg2 = mock.MagicMock() +_mock_psycopg2.paramstyle = "format" +_mock_psycopg2.OperationalError = Exception +sys.modules.setdefault("psycopg2", _mock_psycopg2) + +# ------------------------------------------------------------------------- +# +# Gramps modules (required by the addon's import chain) +# +# ------------------------------------------------------------------------- +try: + import gramps +except ImportError as _err: + raise unittest.SkipTest("gramps package not available: %s" % _err) + +if "GRAMPS_RESOURCES" not in os.environ: + os.environ["GRAMPS_RESOURCES"] = os.path.dirname( + os.path.dirname(gramps.__file__) + ) + +try: + from PostgreSQL.postgresql import Connection, PostgreSQL +except Exception as _err: + raise unittest.SkipTest("PostgreSQL module unavailable: %s" % _err) + + +# ------------------------------------------------------------------------- +# +# Helpers +# +# ------------------------------------------------------------------------- + +def _make_connection(): + """Return a (Connection, mock_cursor) pair without touching psycopg2.""" + conn = Connection.__new__(Connection) + cursor = mock.MagicMock() + conn._Connection__cursor = cursor + return conn, cursor + + +def _translated(sql): + """Return the SQL string that Connection.execute() would pass to psycopg2.""" + conn, cursor = _make_connection() + conn.execute(sql) + return cursor.execute.call_args[0][0] + + +# ------------------------------------------------------------------------- +# +# TestLimitReplStaticmethod +# +# ------------------------------------------------------------------------- +class TestLimitReplStaticmethod(unittest.TestCase): + """Direct tests for Connection._limit_repl via re.sub.""" + + def _apply(self, sql): + import re + return re.sub( + r'\bLIMIT\s+(-?\d+)\s*,\s*(-?\d+)', + Connection._limit_repl, + sql, + flags=re.IGNORECASE, + ) + + def test_offset_and_count(self): + self.assertEqual( + self._apply("SELECT * FROM t LIMIT 5, 10"), + "SELECT * FROM t LIMIT 10 OFFSET 5", + ) + + def test_offset_and_unlimited(self): + self.assertEqual( + self._apply("SELECT * FROM t LIMIT 5, -1"), + "SELECT * FROM t LIMIT ALL OFFSET 5", + ) + + def test_zero_offset(self): + self.assertEqual( + self._apply("SELECT * FROM t LIMIT 0, 20"), + "SELECT * FROM t LIMIT 20 OFFSET 0", + ) + + +# ------------------------------------------------------------------------- +# +# TestExecuteQmarkParamstyle +# +# ------------------------------------------------------------------------- +class TestExecuteQmarkParamstyle(unittest.TestCase): + """? → %s substitution.""" + + def test_single_placeholder(self): + self.assertEqual( + _translated("SELECT * FROM person WHERE gramps_id = ?"), + "SELECT * FROM person WHERE gramps_id = %s", + ) + + def test_multiple_placeholders(self): + result = _translated("INSERT INTO t (a, b) VALUES (?, ?)") + self.assertEqual(result.count("%s"), 2) + self.assertNotIn("?", result) + + def test_no_placeholders_unchanged(self): + sql = "SELECT * FROM person" + self.assertEqual(_translated(sql), sql) + + +# ------------------------------------------------------------------------- +# +# TestExecuteRegexpOperator +# +# ------------------------------------------------------------------------- +class TestExecuteRegexpOperator(unittest.TestCase): + """REGEXP → ~ substitution.""" + + def test_regexp_replaced(self): + result = _translated("SELECT * FROM person WHERE name REGEXP 'foo'") + self.assertIn(" ~ ", result) + self.assertNotIn("REGEXP", result) + + def test_no_regexp_unchanged(self): + sql = "SELECT * FROM person WHERE name = 'foo'" + self.assertEqual(_translated(sql), sql) + + +# ------------------------------------------------------------------------- +# +# TestExecuteLimitTranslations +# +# ------------------------------------------------------------------------- +class TestExecuteLimitTranslations(unittest.TestCase): + """LIMIT dialect translations.""" + + def test_limit_minus_one_becomes_all(self): + result = _translated("SELECT * FROM person LIMIT -1") + self.assertIn("LIMIT ALL", result) + self.assertNotIn("-1", result) + + def test_limit_offset_comma_count(self): + result = _translated("SELECT * FROM person LIMIT 5, 10") + self.assertIn("LIMIT 10 OFFSET 5", result) + + def test_limit_offset_comma_minus_one(self): + result = _translated("SELECT * FROM person LIMIT 5, -1") + self.assertIn("LIMIT ALL OFFSET 5", result) + + def test_plain_limit_unchanged(self): + result = _translated("SELECT * FROM person LIMIT 10") + self.assertEqual(result, "SELECT * FROM person LIMIT 10") + + def test_limit_with_offset_clause_unchanged(self): + result = _translated("SELECT * FROM person LIMIT 10 OFFSET 5") + self.assertEqual(result, "SELECT * FROM person LIMIT 10 OFFSET 5") + + +# ------------------------------------------------------------------------- +# +# TestPostgreSQLSqlType +# +# ------------------------------------------------------------------------- +class TestPostgreSQLSqlType(unittest.TestCase): + """PostgreSQL._sql_type maps BLOB → bytea; other types pass through.""" + + def setUp(self): + self.pg = PostgreSQL.__new__(PostgreSQL) + + def test_blob_becomes_bytea(self): + with mock.patch.object( + PostgreSQL.__bases__[0], "_sql_type", return_value="BLOB" + ): + self.assertEqual(self.pg._sql_type("blob_field", 0), "bytea") + + def test_text_unchanged(self): + with mock.patch.object( + PostgreSQL.__bases__[0], "_sql_type", return_value="TEXT" + ): + self.assertEqual(self.pg._sql_type("text_field", 255), "TEXT") + + def test_integer_unchanged(self): + with mock.patch.object( + PostgreSQL.__bases__[0], "_sql_type", return_value="INTEGER" + ): + self.assertEqual(self.pg._sql_type("int_field", 0), "INTEGER") + + +# ------------------------------------------------------------------------- +# +# TestPostgreSQLQuoteColumn +# +# ------------------------------------------------------------------------- +class TestPostgreSQLQuoteColumn(unittest.TestCase): + """PostgreSQL._quote_column appends _ to reserved words only.""" + + def setUp(self): + self.pg = PostgreSQL.__new__(PostgreSQL) + + def test_desc_reserved(self): + self.assertEqual(self.pg._quote_column("desc"), "desc_") + + def test_order_reserved(self): + self.assertEqual(self.pg._quote_column("order"), "order_") + + def test_where_reserved(self): + self.assertEqual(self.pg._quote_column("where"), "where_") + + def test_select_reserved(self): + self.assertEqual(self.pg._quote_column("select"), "select_") + + def test_normal_column_unchanged(self): + self.assertEqual(self.pg._quote_column("gramps_id"), "gramps_id") + + def test_handle_unchanged(self): + self.assertEqual(self.pg._quote_column("handle"), "handle") + + def test_change_unchanged(self): + self.assertEqual(self.pg._quote_column("change"), "change") + + +if __name__ == "__main__": + unittest.main() From 6fdbb13aa9d7a1cf1bd2ab9d514431f4cff6d5e7 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Mon, 8 Jun 2026 09:45:35 -0700 Subject: [PATCH 12/20] PostgreSQL: wire _quote_column into schema and query methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Override _create_secondary_columns, _update_secondary_values, and get_media_handles in the PostgreSQL class to call self._quote_column() so reserved words (e.g. desc → desc_) are handled at the right sites rather than via post-hoc string replacement. Add a targeted Connection.execute() fix for the one CREATE INDEX site that has no override hook (CREATE INDEX media_desc ON media(desc)). All four overrides are marked for removal once gramps PR #2178 (_quote_column hook in DBAPI base) is merged into core. Also inline the LIMIT rewrite lambda (removing _limit_repl staticmethod) and drop the redundant TestLimitReplStaticmethod test class. Co-Authored-By: Claude Sonnet 4.6 --- PostgreSQL/postgresql.py | 68 ++++++++++++++++++++--- PostgreSQL/tests/test_sql_translations.py | 36 ------------ 2 files changed, 60 insertions(+), 44 deletions(-) diff --git a/PostgreSQL/postgresql.py b/PostgreSQL/postgresql.py index 800a5ca1f..4d153dfaf 100644 --- a/PostgreSQL/postgresql.py +++ b/PostgreSQL/postgresql.py @@ -43,7 +43,9 @@ from gramps.gen.db.dbconst import ARRAYSIZE from gramps.gen.db.exceptions import DbConnectionError from gramps.gen.const import GRAMPS_LOCALE as glocale - +from gramps.gen.lib import ( + Citation, Event, Family, Media, Note, Person, Place, Repository, Source, Tag +) try: _trans = glocale.get_addon_translator(__file__) except ValueError: @@ -71,6 +73,58 @@ def _quote_column(self, col): _RESERVED = {"desc", "order", "where", "select"} return f"{col}_" if col in _RESERVED else col + def _create_secondary_columns(self): + # Remove override when gramps PR #2178 (_quote_column) is merged into core. + for cls in (Person, Family, Event, Place, Repository, Source, Citation, Media, Note, Tag): + table_name = cls.__name__.lower() + for field, schema_type, max_length in cls.get_secondary_fields(): + if field != "handle": + sql_type = self._sql_type(schema_type, max_length) + self.dbapi.execute( + f"ALTER TABLE {table_name} ADD COLUMN" + f" {self._quote_column(field)} {sql_type}" + ) + + def _update_secondary_values(self, obj): + # Remove override when gramps PR #2178 (_quote_column) is merged into core. + table = obj.__class__.__name__ + fields = [field[0] for field in obj.get_secondary_fields()] + sets = [] + values = [] + for field in fields: + sets.append(f"{self._quote_column(field)} = ?") + values.append(getattr(obj, field)) + + if table == "Person": + given_name, surname = self._get_person_data(obj) + sets.append("given_name = ?") + values.append(given_name) + sets.append("surname = ?") + values.append(surname) + if table == "Place": + handle = self._get_place_data(obj) + sets.append("enclosed_by = ?") + values.append(handle) + + if len(values) > 0: + table_name = table.lower() + self.dbapi.execute( + f'UPDATE {table_name} SET {", ".join(sets)} where handle = ?', + self._sql_cast_list(values) + [obj.handle], + ) + + def get_media_handles(self, sort_handles=False, locale=glocale): + # Remove override when gramps PR #2178 (_quote_column) is merged into core. + if sort_handles: + self.dbapi.execute( + "SELECT handle FROM media " + f"ORDER BY {self._quote_column('desc')} " + f'COLLATE "{self._collation(locale)}"' + ) + else: + self.dbapi.execute("SELECT handle FROM media") + return [row[0] for row in self.dbapi.fetchall()] + def get_summary(self): """ Return a diction of information about this database @@ -152,16 +206,14 @@ def check_collation(self, locale): % (collation, locale.collation) ) - @staticmethod - def _limit_repl(m): - offset, count = m.group(1), m.group(2) - count = 'ALL' if count == '-1' else count - return f'LIMIT {count} OFFSET {offset}' - def execute(self, *args, **kwargs): sql = args[0].replace("?", "%s") # qmark → format paramstyle sql = sql.replace(" REGEXP ", " ~ ") # SQLite REGEXP → PostgreSQL ~ - sql = re.sub(r'\bLIMIT\s+(-?\d+)\s*,\s*(-?\d+)', self._limit_repl, sql, flags=re.IGNORECASE) # LIMIT offset, count → LIMIT count OFFSET offset + # TODO: remove when gramps PR #2178 (_quote_column) is merged into core + sql = sql.replace("ON media(desc)", "ON media(desc_)") + sql = re.sub(r'\bLIMIT\s+(-?\d+)\s*,\s*(-?\d+)', + lambda m: f'LIMIT {"ALL" if m.group(2) == "-1" else m.group(2)} OFFSET {m.group(1)}', + sql, flags=re.IGNORECASE) # LIMIT offset, count → LIMIT count OFFSET offset sql = re.sub(r'\bLIMIT\s+-1\b', 'LIMIT ALL', sql, flags=re.IGNORECASE) # LIMIT -1 → LIMIT ALL if len(args) > 1: args = args[1] diff --git a/PostgreSQL/tests/test_sql_translations.py b/PostgreSQL/tests/test_sql_translations.py index 24574db43..499a22406 100644 --- a/PostgreSQL/tests/test_sql_translations.py +++ b/PostgreSQL/tests/test_sql_translations.py @@ -102,42 +102,6 @@ def _translated(sql): return cursor.execute.call_args[0][0] -# ------------------------------------------------------------------------- -# -# TestLimitReplStaticmethod -# -# ------------------------------------------------------------------------- -class TestLimitReplStaticmethod(unittest.TestCase): - """Direct tests for Connection._limit_repl via re.sub.""" - - def _apply(self, sql): - import re - return re.sub( - r'\bLIMIT\s+(-?\d+)\s*,\s*(-?\d+)', - Connection._limit_repl, - sql, - flags=re.IGNORECASE, - ) - - def test_offset_and_count(self): - self.assertEqual( - self._apply("SELECT * FROM t LIMIT 5, 10"), - "SELECT * FROM t LIMIT 10 OFFSET 5", - ) - - def test_offset_and_unlimited(self): - self.assertEqual( - self._apply("SELECT * FROM t LIMIT 5, -1"), - "SELECT * FROM t LIMIT ALL OFFSET 5", - ) - - def test_zero_offset(self): - self.assertEqual( - self._apply("SELECT * FROM t LIMIT 0, 20"), - "SELECT * FROM t LIMIT 20 OFFSET 0", - ) - - # ------------------------------------------------------------------------- # # TestExecuteQmarkParamstyle From 12724fe37ecaa0502ad4911925ce14884a656979 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Wed, 10 Jun 2026 11:52:12 -0700 Subject: [PATCH 13/20] =?UTF-8?q?PostgreSQL:=20fix=20BLOB=E2=86=92BYTEA=20?= =?UTF-8?q?for=20schema=20creation;=20add=20full=20translation=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PostgreSQL has no BLOB type; the core _create_schema() hardcodes "BLOB" in CREATE TABLE statements that bypass _sql_type(). Add a word-boundary re.sub in execute() to translate BLOB→BYTEA for all SQL passing through the adapter. Also add TestExecuteMediaDescIndex and TestExecuteBlobType so every rewrite rule in execute() is covered by its own test class. Co-Authored-By: Claude Sonnet 4.6 --- PostgreSQL/postgresql.py | 1 + PostgreSQL/tests/test_sql_translations.py | 72 +++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/PostgreSQL/postgresql.py b/PostgreSQL/postgresql.py index 4d153dfaf..c7f91b18b 100644 --- a/PostgreSQL/postgresql.py +++ b/PostgreSQL/postgresql.py @@ -211,6 +211,7 @@ def execute(self, *args, **kwargs): sql = sql.replace(" REGEXP ", " ~ ") # SQLite REGEXP → PostgreSQL ~ # TODO: remove when gramps PR #2178 (_quote_column) is merged into core sql = sql.replace("ON media(desc)", "ON media(desc_)") + sql = re.sub(r'\bBLOB\b', 'BYTEA', sql) # SQLite BLOB → PostgreSQL BYTEA sql = re.sub(r'\bLIMIT\s+(-?\d+)\s*,\s*(-?\d+)', lambda m: f'LIMIT {"ALL" if m.group(2) == "-1" else m.group(2)} OFFSET {m.group(1)}', sql, flags=re.IGNORECASE) # LIMIT offset, count → LIMIT count OFFSET offset diff --git a/PostgreSQL/tests/test_sql_translations.py b/PostgreSQL/tests/test_sql_translations.py index 499a22406..3b4f3976b 100644 --- a/PostgreSQL/tests/test_sql_translations.py +++ b/PostgreSQL/tests/test_sql_translations.py @@ -174,6 +174,78 @@ def test_limit_with_offset_clause_unchanged(self): self.assertEqual(result, "SELECT * FROM person LIMIT 10 OFFSET 5") +# ------------------------------------------------------------------------- +# +# TestExecuteMediaDescIndex +# +# ------------------------------------------------------------------------- +class TestExecuteMediaDescIndex(unittest.TestCase): + """ON media(desc) → ON media(desc_) substitution. + + PostgreSQL reserves DESC as a keyword; the column was renamed desc_ + by _quote_column, so any index referencing it needs the same rename. + This literal replacement is a temporary workaround until gramps PR #2178 + (_quote_column) is merged into core. + """ + + def test_media_desc_index_renamed(self): + result = _translated("CREATE INDEX ON media(desc)") + self.assertIn("ON media(desc_)", result) + self.assertNotIn("ON media(desc)", result.replace("ON media(desc_)", "")) + + def test_other_table_desc_unchanged(self): + sql = "CREATE INDEX ON person(desc)" + self.assertEqual(_translated(sql), sql) + + def test_non_index_sql_unchanged(self): + sql = "SELECT * FROM media WHERE desc_ = %s" + self.assertEqual(_translated(sql), sql) + + +# ------------------------------------------------------------------------- +# +# TestExecuteBlobType +# +# ------------------------------------------------------------------------- +class TestExecuteBlobType(unittest.TestCase): + """BLOB → BYTEA substitution in execute().""" + + def test_metadata_table_blob_replaced(self): + """Reproduces the schema creation error: metadata value BLOB.""" + result = _translated( + "CREATE TABLE metadata " + "(setting VARCHAR(50) PRIMARY KEY NOT NULL, json_data TEXT, value BLOB)" + ) + self.assertIn("BYTEA", result) + self.assertNotIn("BLOB", result) + + def test_blob_data_column_replaced(self): + """Schema tables using blob_data BLOB column.""" + result = _translated( + "CREATE TABLE person " + "(handle VARCHAR(50) PRIMARY KEY NOT NULL, blob_data BLOB)" + ) + self.assertIn("BYTEA", result) + self.assertNotIn("BLOB", result) + + def test_blob_word_boundary_not_in_identifier(self): + """BLOB as part of a longer identifier is not replaced.""" + result = _translated("SELECT blobfield FROM person") + self.assertNotIn("BYTEA", result) + self.assertEqual(result, "SELECT blobfield FROM person") + + def test_multiple_blob_columns_all_replaced(self): + result = _translated( + "CREATE TABLE t (a BLOB, b TEXT, c BLOB)" + ) + self.assertEqual(result.count("BYTEA"), 2) + self.assertNotIn("BLOB", result) + + def test_non_blob_sql_unchanged(self): + sql = "SELECT * FROM person WHERE handle = %s" + self.assertEqual(_translated(sql), sql) + + # ------------------------------------------------------------------------- # # TestPostgreSQLSqlType From efd9bea88d49e879b979ff2ef824701786e5519a Mon Sep 17 00:00:00 2001 From: GaryGriffin Date: Wed, 10 Jun 2026 13:59:22 -0700 Subject: [PATCH 14/20] Merge PostgreSQL: replace _hack_query() with proper dialect methods#943 --- PostgreSQL/postgresql.gpr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PostgreSQL/postgresql.gpr.py b/PostgreSQL/postgresql.gpr.py index 99ad65e0d..a5e6fd21d 100644 --- a/PostgreSQL/postgresql.gpr.py +++ b/PostgreSQL/postgresql.gpr.py @@ -23,7 +23,7 @@ name=_("PostgreSQL"), name_accell=_("_PostgreSQL Database"), description=_("PostgreSQL Database"), - version = '1.0.22', + version = '1.0.23', gramps_target_version="6.1", status=STABLE, audience=EXPERT, From 1474572f5b62228f8ac40ccca943a6447e4ac298 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Sat, 13 Jun 2026 21:19:03 +0200 Subject: [PATCH 15/20] Let GraphView import when GooCanvas or GraphViz is absent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GraphView's module body raised a bare Exception while being imported when the GooCanvas library or the GraphViz `dot` binary was missing. Because the raise fired at top level, any code importing the module crashed before Gramps had a chance to decide whether the plugin should load — including the addon's own test suite, whose collection imports the module at the top level and aborted with a traceback (0 tests run) on hosts that have no `dot`. Gramps already declines to load the view when those dependencies are absent: the requires_gi / requires_exe declarations in graphview.gpr.py drive the plugin manager's requirement check, which removes the plugin before its module is ever imported. The import-time raises duplicated that gate while breaking every direct importer, so this removes them. The missing-dependency experience for end users is unchanged, because the registration-level gate remains the sole arbiter of whether the plugin loads. A new regression test imports the production GraphView.graphview module in a child interpreter with `dot` made unavailable, and fails if the import raises, so GraphView/tests/ now collects and runs on a host lacking `dot`. --- GraphView/graphview.py | 7 -- GraphView/tests/test_graphview_import.py | 101 +++++++++++++++++++++++ 2 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 GraphView/tests/test_graphview_import.py diff --git a/GraphView/graphview.py b/GraphView/graphview.py index 07bd876ac..afec53951 100644 --- a/GraphView/graphview.py +++ b/GraphView/graphview.py @@ -108,19 +108,12 @@ break except (ImportError, ValueError): _GOO = False -if not _GOO: - raise Exception("Goocanvas 2 or 3 (http://live.gnome.org/GooCanvas) is " - "required for this view to work") if os.sys.platform == "win32": _DOT_FOUND = search_for("dot.exe") else: _DOT_FOUND = search_for("dot") -if not _DOT_FOUND: - raise Exception("GraphViz (http://www.graphviz.org) is " - "required for this view to work") - SPLINE = {0: 'false', 1: 'true', 2: 'ortho'} WIKI_PAGE = 'https://gramps-project.org/wiki/index.php?title=Graph_View' diff --git a/GraphView/tests/test_graphview_import.py b/GraphView/tests/test_graphview_import.py new file mode 100644 index 000000000..5510853bd --- /dev/null +++ b/GraphView/tests/test_graphview_import.py @@ -0,0 +1,101 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . +# + +""" +Regression test for issue 46: GraphView/graphview.py raised a bare Exception +at *module import* when the GraphViz ``dot`` binary (or GooCanvas) was +unavailable (graphview.py:111-113 / :120-122). The module-level raise +duplicates the registration-layer gate (``requires_exe=["dot"]`` / +``requires_gi=[("GooCanvas", ...)]`` in graphview.gpr.py) that Gramps' plugin +manager already owns, and breaks every direct importer -- including this +addon's own test collection, which crashes during import on a host lacking +``dot``. + +The module must instead import without a side-effecting failure when a +declared dependency is absent; whether the plugin loads stays the sole +responsibility of the ``requires_*`` gate at registration time. + +This test drives the *production* ``GraphView.graphview`` import in a fresh +child interpreter with the ``dot`` binary made unfindable (an empty ``PATH``, +so ``gramps.gen.utils.file.search_for("dot")`` returns 0 exactly as it does on +a host without GraphViz). Pre-fix the child import raises and exits non-zero; +post-fix it imports cleanly and exits zero. + +The Gtk/GooCanvas-bound production module is imported only *inside the child +process*, never at this test module's top level, so the headless test +collector never executes a GUI import during collection -- this module imports +only the stdlib. +""" + +import os +import subprocess +import sys +import unittest +from pathlib import Path + +# addons-source root = GraphView/tests/ -> parents[2]. The child +# interpreter imports the real GraphView.graphview namespace package from here. +_ADDONS_ROOT = Path(__file__).resolve().parents[2] + +# Import the production module and the class the brief's success criterion names. +# `import GraphView.graphview` runs the whole module body (the lines that, pre-fix, +# raise when GooCanvas/dot are absent); the DotSvgGenerator import proves the +# success criterion's second clause. A non-zero exit means the import raised. +_CHILD = ( + "import GraphView.graphview\n" + "from GraphView.graphview import DotSvgGenerator\n" + "assert DotSvgGenerator is not None\n" +) + + +class GraphViewImportTest(unittest.TestCase): + """The plugin module must import even when its declared external deps are absent.""" + + def test_module_imports_with_dot_unavailable(self): + # Simulate the GraphViz `dot` binary being unavailable: an empty PATH + # makes search_for("dot") (gramps.gen.utils.file:236-240) return 0, so + # _DOT_FOUND is falsy -- pre-fix that triggers the module-level + # `raise Exception("GraphViz ... required")` at graphview.py:120-122. + env = dict(os.environ) + env["PATH"] = "" + # Keep the inherited PYTHONPATH (carries the gi_bootstrap GI version pin), + # and put the addons-source root first so `import GraphView` resolves. + env["PYTHONPATH"] = os.pathsep.join( + [str(_ADDONS_ROOT), env.get("PYTHONPATH", "")] + ).rstrip(os.pathsep) + + result = subprocess.run( + [sys.executable, "-c", _CHILD], + cwd=str(_ADDONS_ROOT), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertEqual( + result.returncode, + 0, + "importing GraphView.graphview with `dot` unavailable must not raise; " + "child exited %s\n--- stderr ---\n%s" % (result.returncode, result.stderr), + ) + + +if __name__ == "__main__": + unittest.main() From def50c2c2db821f255c8d44a88903ca35d864591 Mon Sep 17 00:00:00 2001 From: GaryGriffin Date: Mon, 15 Jun 2026 09:46:07 -0700 Subject: [PATCH 16/20] Merge Let GraphView import when GooCanvas or GraphViz is absent#951 --- GraphView/graphview.gpr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GraphView/graphview.gpr.py b/GraphView/graphview.gpr.py index 731bdb7f7..b5b5cfbb3 100644 --- a/GraphView/graphview.gpr.py +++ b/GraphView/graphview.gpr.py @@ -4,7 +4,7 @@ name=_("Graph View"), category=("Ancestry", _("Charts")), description=_("Dynamic and interactive graph of relations"), - version = '1.0.148', + version = '1.0.149', gramps_target_version="6.1", status=STABLE, fname="graphview.py", From fc46e926eeee43a916933ba820d2a485223ca7a7 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 8 Jun 2026 23:38:38 +0200 Subject: [PATCH 17/20] Keep the Home person out of Deep Connections path interiors The Deep Connections Gramplet searches outward from the Home person and reports how the active person connects back to them. When the active person reached Home through one of Home's own direct relatives -- for example a niece or nephew, who is the child of Home's sibling -- the reported path named the Home person as an intermediate relationship step ("... / sibling of / ...") on top of its terminal root. The Home person is the search origin, so it should appear in a path only as that root, never as an interior anchor. The search now keeps the Home person out of the body of every produced path: a relative that is the Home person is dropped, and when the Home person is the node being expanded its relatives attach directly to the root rather than through a redundant "of " step. Paths still terminate at the Home person and still connect the active person to them; only the spurious interior mention is removed. The renderer and the rest of the search loop are unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) Fixes #8653 --- .../DeepConnectionsGramplet.py | 28 +- DeepConnectionsGramplet/tests/__init__.py | 0 .../tests/test_deep_connections.py | 327 ++++++++++++++++++ 3 files changed, 353 insertions(+), 2 deletions(-) create mode 100644 DeepConnectionsGramplet/tests/__init__.py create mode 100644 DeepConnectionsGramplet/tests/test_deep_connections.py diff --git a/DeepConnectionsGramplet/DeepConnectionsGramplet.py b/DeepConnectionsGramplet/DeepConnectionsGramplet.py index 73f2bf37b..cbcdce44d 100644 --- a/DeepConnectionsGramplet/DeepConnectionsGramplet.py +++ b/DeepConnectionsGramplet/DeepConnectionsGramplet.py @@ -218,9 +218,15 @@ def get_links_from_notes(self, obj, path, relation, person_handle): retval += [(link[3], (path, (relation, person_handle)))] return retval - def get_relatives(self, person_handle, path): + def get_relatives(self, person_handle, path, start_handle=None): """ Gets all of the relations of person_handle. + + ``start_handle`` is the Home/start person's handle, passed in by the + caller (:meth:`main`) *after* its "No Home Person set" guard so this + method never has to re-dereference ``self.default_person``. When it is + given, the Home person is kept out of the body of every produced path + (issue 8653) -- see the post-processing at the end of this method. """ retval = [] person = self.dbstate.db.get_person_from_handle(person_handle) @@ -276,6 +282,22 @@ def get_relatives(self, person_handle, path): retval += self.get_links_from_notes( person, path, _("Note on Person"), person_handle ) + + # Issue 8653: the Home (start) person is the search origin, so it may + # appear in a produced path only as its terminal "self" root -- never + # as, nor as the anchor of, an intermediate relationship step. Two + # cases, applied only once the caller has supplied the start handle: + # * drop a relative that *is* the start person, so the origin is never + # re-entered as an interior node of a path; and + # * when the start person itself is being expanded, attach its + # relatives straight to the root node (``path`` is that root here) + # instead of through a redundant "(relation, start)" step that would + # render the Home person mid-path (the reporter's + # "... / sibling of " chain). + if start_handle is not None: + retval = [item for item in retval if item[0] != start_handle] + if person_handle == start_handle: + retval = [(item[0], path) for item in retval] return retval def active_changed(self, handle): @@ -436,7 +458,9 @@ def main(self): continue self.cache.add(current_handle) - relatives = self.get_relatives(current_handle, current_path) + relatives = self.get_relatives( + current_handle, current_path, self.default_person.handle + ) # Track search depth if current_path[0] is not None: diff --git a/DeepConnectionsGramplet/tests/__init__.py b/DeepConnectionsGramplet/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/DeepConnectionsGramplet/tests/test_deep_connections.py b/DeepConnectionsGramplet/tests/test_deep_connections.py new file mode 100644 index 000000000..0c11beb76 --- /dev/null +++ b/DeepConnectionsGramplet/tests/test_deep_connections.py @@ -0,0 +1,327 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 The Gramps Development Team +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Regression test for Mantis issue 8653 -- the Deep Connections Gramplet must not +route a connection path back through the Home (start) person. The Home person +is the search origin and may appear in a produced path only as its terminal +"self" root; it must never be named as the anchor of an intermediate +relationship step (the reporter's "... / sibling of " chain). + +PRODUCTION-PATH NOTE (brief.md PRODUCTION-PATH REQUIREMENT / principles §3.4): +this test drives ``DeepConnectionsGramplet.main()`` itself -- the *actual* +breadth-first search generator the gramplet runs in production -- not a copy of +its loop. A lightweight harness subclass overrides only the GUI surface +(``append_text``/``link``/``pretty_print``/progress widgets/``pause``...), so +``main`` runs the real queue/cache/``get_relatives`` path-construction code +against a tiny in-memory database. The produced path is captured at the point +``main`` hands it to ``pretty_print``. Because the harness drives ``main`` +*outside* the broad ``try/except`` in production, any error in the search +surfaces as a test failure rather than being swallowed into "Error during +search". + +The gramplet module imports ``gi``/Gtk at load time; the addon C4 gate runs +under ``xvfb`` with the GI-version bootstrap, so importing it here is safe (the +same pattern SurnameMappingGramplet's import test uses). No display-bound +widget is ever constructed: the harness skips ``Gramplet.__init__``. +""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# The addon directory and its implementation module share a name; use the +# explicit submodule path (same trap noted in libaccess / gramps bug 0012691). +from DeepConnectionsGramplet import DeepConnectionsGramplet as dcg_mod + + +# --------------------------------------------------------------------------- +# Minimal in-memory stand-ins for the Gramps person/family/db objects, exposing +# only the accessors get_relatives / main touch. +# --------------------------------------------------------------------------- +class _ChildRef: + def __init__(self, ref): + self.ref = ref + + +class _Person: + def __init__(self, handle, families=(), parent_families=()): + self.handle = handle + self._families = list(families) + self._parent_families = list(parent_families) + + def get_family_handle_list(self): + return self._families + + def get_parent_family_handle_list(self): + return self._parent_families + + def get_person_ref_list(self): + return [] + + def get_note_list(self): + return [] + + def get_primary_name(self): + return self.handle + + +class _Family: + def __init__(self, handle, father=None, mother=None, children=()): + self.handle = handle + self._father = father + self._mother = mother + self._children = list(children) + + def get_child_ref_list(self): + return [_ChildRef(c) for c in self._children] + + def get_father_handle(self): + return self._father + + def get_mother_handle(self): + return self._mother + + def get_note_list(self): + return [] + + +class _DB: + def __init__(self, people, families, default): + self._people = people + self._families = families + self._default = default + + def get_default_person(self): + return self._people.get(self._default) + + def get_person_from_handle(self, handle): + return self._people.get(handle) + + def get_family_from_handle(self, handle): + return self._families.get(handle) + + def get_note_from_handle(self, handle): # no notes in these fixtures + return None + + +class _DBState: + def __init__(self, db): + self.db = db + + +class _StubRelCalc: + """Relationship calculator stub; main() only needs a falsy/str result.""" + + def get_one_relationship(self, db, person1, person2): + return "" + + +class _StubNameDisplayer: + """Avoid needing a real gramps Name object for display.""" + + def display_name(self, name): + return str(name) + + +class _Widget: + """No-op stand-in for the Gtk widgets main() toggles.""" + + def set_visible(self, *args, **kwargs): + pass + + def set_sensitive(self, *args, **kwargs): + pass + + +class _Harness(dcg_mod.DeepConnectionsGramplet): + """ + Drives the real ``DeepConnectionsGramplet.main`` generator without a GTK + context. Only the GUI surface is overridden; ``main``, ``get_relatives``, + ``get_links_from_notes`` and ``_calculate_path_depth`` are the inherited + production implementations. + """ + + def __init__(self, db, active_handle): + # Deliberately do NOT call Gramplet.__init__ (no GUI / GTK). + self.dbstate = _DBState(db) + self._active_handle = active_handle + self.selected_handles = set() + self.captured_paths = [] + self.relationship_calc = _StubRelCalc() + self.progress_bar = _Widget() + self.pause_button = _Widget() + self.continue_button = _Widget() + self.copy_button = _Widget() + + # -- GUI surface stubbed out ------------------------------------------- + def get_active_object(self, _kind): + return self.dbstate.db.get_person_from_handle(self._active_handle) + + def set_text(self, *args, **kwargs): + pass + + def render_text(self, *args, **kwargs): + pass + + def update_status(self, *args, **kwargs): + pass + + def update_progress(self, *args, **kwargs): + pass + + def update_search_info(self, *args, **kwargs): + pass + + def append_text(self, *args, **kwargs): + pass + + def link(self, *args, **kwargs): + pass + + def pause(self, *args, **kwargs): + pass + + # -- capture the produced path instead of rendering it ----------------- + def pretty_print(self, path): + self.captured_paths.append(path) + + +def _flatten(path): + """ + Flatten a connection path's linked list into ``[(relation, anchor), ...]`` + ordered from the outermost step down to the terminal "self" root. + + Node shape (unchanged from the gramplet):: + + (more_path, (relation_text, anchor_handle, [parents...])) + """ + steps = [] + node = path + while node is not None: + steps.append((node[1][0], node[1][1])) + node = node[0] + return steps + + +def _nibling_db(): + """ + Home person ``D`` and sibling ``S`` are children of family ``Fp`` (parents + ``P1``/``P2``). ``S`` has their own family ``Fs`` with spouse ``W`` and + child ``A`` (the active person). So ``A`` connects to Home ``D`` as + "child of D's sibling S" -- a two-hop path that, before the fix, named + ``D`` (Home) as an intermediate "sibling of " step. + """ + families = { + "Fp": _Family("Fp", father="P1", mother="P2", children=["D", "S"]), + "Fs": _Family("Fs", father="S", mother="W", children=["A"]), + } + people = { + "D": _Person("D", parent_families=["Fp"]), + "S": _Person("S", families=["Fs"], parent_families=["Fp"]), + "A": _Person("A", parent_families=["Fs"]), + "W": _Person("W", families=["Fs"]), + "P1": _Person("P1", families=["Fp"]), + "P2": _Person("P2", families=["Fp"]), + } + return _DB(people, families, default="D") + + +class TestStartPersonNotIntermediate(unittest.TestCase): + """The Home (start) person must never be an intermediate path step (8653).""" + + def _first_path_steps(self, db, active_handle): + """ + Run the production ``main`` generator until it produces the first + connection path; return its flattened steps (outermost -> root). + """ + harness = _Harness(db, active_handle) + # Patch the module-level name displayer so main() needs no real Name. + original_displayer = dcg_mod.name_displayer + dcg_mod.name_displayer = _StubNameDisplayer() + try: + for _signal in harness.main(): + if harness.captured_paths: + break + finally: + dcg_mod.name_displayer = original_displayer + self.assertTrue( + harness.captured_paths, + "main() produced no connection path to %r" % (active_handle,), + ) + return _flatten(harness.captured_paths[0]) + + def test_home_not_an_intermediate_step_on_multi_hop_path(self): + """A two-hop connection must not name the Home person mid-path.""" + steps = self._first_path_steps(_nibling_db(), "A") + + root = steps[-1] + non_root = steps[:-1] + intermediate_handles = [anchor for (_relation, anchor) in non_root] + + # Not vacuous: the connection genuinely routes through the intermediate + # relative S (active A is "child of S") -- the multi-hop case the bug + # needs, not a direct relative of Home. + self.assertIn( + "S", + intermediate_handles, + "scenario does not route through the intermediate S; steps=%r" % (steps,), + ) + + # The bug (issue 8653): D (Home) appears as a non-root anchor. + self.assertNotIn( + "D", + intermediate_handles, + "Home person re-entered as an intermediate step; steps=%r" % (steps,), + ) + + # The chain is not broken: it still terminates at the Home person as the + # "self" root, so A is connected to Home (A -> child of S -> root D). + self.assertEqual( + root[1], + "D", + "path must terminate at the Home person as its root; steps=%r" % (steps,), + ) + self.assertEqual( + [anchor for (_relation, anchor) in steps].count("D"), + 1, + "Home person must appear exactly once (as the root); steps=%r" % (steps,), + ) + + def test_direct_relative_of_home_only_at_root(self): + """A direct sibling of Home still connects, with Home only as root.""" + steps = self._first_path_steps(_nibling_db(), "S") # S is Home's sibling + intermediate_handles = [anchor for (_relation, anchor) in steps[:-1]] + self.assertNotIn( + "D", + intermediate_handles, + "Home person re-entered as an intermediate step; steps=%r" % (steps,), + ) + self.assertEqual( + steps[-1][1], + "D", + "path must terminate at the Home person as its root; steps=%r" % (steps,), + ) + + +if __name__ == "__main__": + unittest.main() From b41c93049ba5778467dda8bca494f31cef18a9b5 Mon Sep 17 00:00:00 2001 From: GaryGriffin Date: Mon, 15 Jun 2026 09:55:53 -0700 Subject: [PATCH 18/20] Merge Keep the Home person out of Deep Connections path interiors#946 --- DeepConnectionsGramplet/DeepConnectionsGramplet.gpr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DeepConnectionsGramplet/DeepConnectionsGramplet.gpr.py b/DeepConnectionsGramplet/DeepConnectionsGramplet.gpr.py index 5a83742a2..5a94e37fb 100644 --- a/DeepConnectionsGramplet/DeepConnectionsGramplet.gpr.py +++ b/DeepConnectionsGramplet/DeepConnectionsGramplet.gpr.py @@ -13,7 +13,7 @@ gramplet_title=_("Deep Connections"), detached_width=510, detached_height=480, - version = '1.0.49', + version = '1.0.50', gramps_target_version="6.1", help_url="Deep_Connections_Gramplet", navtypes=["Person"], From 1ca31387a59f101f12369d2709d60e8a94b97c42 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Sat, 13 Jun 2026 16:38:40 +0200 Subject: [PATCH 19/20] Pin GTK and GDK to 3.0 for the repo-root test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tests/ suite (run by `python3 -m unittest discover -s tests`) imports gramps.gui.* modules directly, which bypasses the Gramps GUI launcher's own gi.require_version. On a host where GTK 4 is the default GI resolution, Gtk/Gdk then load unpinned — emitting PyGIWarning and risking the wrong stack (and the GUI-touching tests silently skipping). Pin both in tests/__init__.py, which `unittest discover -s tests` imports before any test module, so the whole repo-root suite runs on the GTK 3 / GDK 3 stack a real Gramps session uses. tests/test_gtk_version_pin.py guards it (skips where the 3.0 typelibs are absent). Verified: `python3 -m unittest discover -s tests -p "test_*.py" -t .` pins both (get_required_version == "3.0"); the guard is red without the pin. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/__init__.py | 19 +++++++++++++++++++ tests/test_gtk_version_pin.py | 18 ++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/test_gtk_version_pin.py diff --git a/tests/__init__.py b/tests/__init__.py index e69de29bb..ff68cee43 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,19 @@ +"""Pin the GTK / GDK introspection versions for the repo-root test suite. + +``python3 -m unittest discover -s tests`` imports this package before any test +module under ``tests/``, so requiring the versions here pins the whole suite to +the GTK 3 / GDK 3 stack a real Gramps GUI session uses. A test that imports a +``gramps.gui.*`` module directly never runs the launcher's own +``require_version``; without this the GI stack can resolve to GTK 4 on a host +where that is the default — emitting ``PyGIWarning`` and risking the wrong stack. +""" + +try: + import gi + + gi.require_version("Gdk", "3.0") + gi.require_version("Gtk", "3.0") +except Exception: + # No PyGObject, or the 3.0 typelibs are unavailable — leave the environment + # untouched; this only fixes the version when it can. + pass diff --git a/tests/test_gtk_version_pin.py b/tests/test_gtk_version_pin.py new file mode 100644 index 000000000..3d9c32633 --- /dev/null +++ b/tests/test_gtk_version_pin.py @@ -0,0 +1,18 @@ +"""Guard: the repo-root test suite pins GTK/GDK to 3 (via tests/__init__.py).""" + +import unittest + + +class GtkGdkVersionPin(unittest.TestCase): + def test_suite_pins_gtk_and_gdk_to_3(self): + try: + import gi + except ImportError: + self.skipTest("PyGObject not available") + repo = gi.Repository.get_default() + if "3.0" not in repo.enumerate_versions("Gtk") or \ + "3.0" not in repo.enumerate_versions("Gdk"): + self.skipTest("GTK/GDK 3.0 introspection typelibs not installed") + # Importing the tests package ran tests/__init__.py, which must pin both. + self.assertEqual(gi.get_required_version("Gtk"), "3.0") + self.assertEqual(gi.get_required_version("Gdk"), "3.0") From 594fe49a1d52f77ead07b75d684bd4612f8c1429 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Tue, 16 Jun 2026 10:46:53 +0200 Subject: [PATCH 20/20] Move DB-backed TMGimporter tests into a Linux-only test module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On maintenance/gramps61, TMGimporter/tests/test_libtmg.py carried the 13 real-database import test classes alongside the pure-function tests. Those DB-backed tests drive an in-memory Gramps database, and on the Windows CI lane the only available Gramps comes from conda-forge (no 6.1 yet, resolving to 6.0.8); run against that mismatched core the import path hangs, so the file had grown two stopgaps to cope: an inline gi.require_version GTK/Gdk pin block at the top and a sys.platform == "win32" SkipTest inside _make_db(). This separates an addon's OS-environment-dependent tests from its portable ones by file placement, following the test__* convention: a test a lane cannot run is no longer placed in a file that lane executes. The 13 DB-backed classes and their shared helpers (_Rec, _table, _make_db, _add_person) move verbatim into a new Linux-only TMGimporter/tests/ test_linux_libtmg.py; test_libtmg.py keeps only the pure-function tests and its reduced "from gramps.gen.lib import Date" import. As a result the two runtime stopgaps disappear — the Linux lane runs both modules against the matching Gramps 6.1, while the Windows lane excludes test_linux_* and no longer hangs. This brings maintenance/gramps61 in line with the equivalent split already merged on maintenance/gramps60 by upstream PR 949, which addressed the Windows unit-test hang against a mismatched conda-forge Gramps. The cherry-pick of that split forward had failed because gramps61's test_libtmg.py had independently diverged in the regions the split rewrites. gramps60 is untouched. No production code or test logic changes. --- TMGimporter/tests/test_libtmg.py | 1223 +----------------------- TMGimporter/tests/test_linux_libtmg.py | 1218 +++++++++++++++++++++++ 2 files changed, 1219 insertions(+), 1222 deletions(-) create mode 100644 TMGimporter/tests/test_linux_libtmg.py diff --git a/TMGimporter/tests/test_libtmg.py b/TMGimporter/tests/test_libtmg.py index 44cab9913..4851746a6 100644 --- a/TMGimporter/tests/test_libtmg.py +++ b/TMGimporter/tests/test_libtmg.py @@ -11,90 +11,13 @@ import sys import os -import tempfile import unittest -# The TMGimporter module imports Gtk at module load — skip the whole file if -# gi/Gtk aren't available (headless-without-GTK environments). On systems -# where both GTK3 and GTK4 are present, pin Gtk to 3.0 before any gramps -# import (mirrors what gramps.grampsapp does at startup); otherwise -# PyGObject loads GTK4 and the gramps.gui import chain crashes on -# Gtk.IconSize.MENU (a GTK3-only enum). -try: - import gi - - gi.require_version("Gtk", "3.0") - gi.require_version("Gdk", "3.0") -except (ImportError, ValueError, AttributeError) as err: - raise unittest.SkipTest("GTK 3.0 / PyGObject not available: %s" % err) - - # Make sure libtmg is importable from the parent directory sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import libtmg -from gramps.gen.lib import Date, Event, NoteType, Person, Place, Source -from gramps.gen.db.utils import make_database -from gramps.gen.db import DbTxn - - -# --------------------------------------------------------------------------- -# Helpers shared across test cases -# --------------------------------------------------------------------------- - -class _Rec: - """Minimal fake DBF record — set any field via keyword arguments.""" - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - - -def _table(records): - """Return an object that behaves like a dbf.Table used as a context manager. - - libtmg uses tables in two ways: - with tmgFoo: - for record in tmgFoo: # iterates over context-managed table - """ - class _FakeTable: - def __enter__(self): - return self - def __exit__(self, *_): - return False - def __iter__(self): - return iter(records) - return _FakeTable() - - -def _make_db(): - """Return a fresh in-memory Gramps database. - - Skips on Windows: these import tests drive a real in-memory Gramps - database, and on the Windows CI lane the only available Gramps comes - from conda-forge, which has no 6.1 yet (and Gramps 6.1 does not build - in the conda env -- its Windows build targets MSYS2 UCRT64, not conda). - Run against that mismatched Gramps the real-DB import path hangs, so - these tests are exercised on the Linux lane (which runs the Gramps - matching this branch) instead. The pure-function tests in this module - do not call this helper and still run on Windows. - """ - if sys.platform == "win32": - raise unittest.SkipTest( - "real-DB TMG import tests run on Linux; the Windows CI lane has no " - "Gramps matching this branch (6.1 is not on conda-forge), and the " - "import path hangs against a mismatched Gramps" - ) - db = make_database("sqlite") - db.load(":memory:", None) - return db - - -def _add_person(db): - """Add an empty Person to db and return (db, handle).""" - p = Person() - with DbTxn("setup", db) as t: - db.add_person(p, t) - return db, p.get_handle() - +from gramps.gen.lib import Date # --------------------------------------------------------------------------- # Pure function: _strip_tmg_codes @@ -211,529 +134,6 @@ def test_exact_certain_has_no_quality(self): self.assertEqual(d.get_quality(), Date.QUAL_NONE) -# --------------------------------------------------------------------------- -# import_notes — patches the module-level DBF table globals -# --------------------------------------------------------------------------- - -class TestImportNotes(unittest.TestCase): - - def _patch(self, tagtypes_records, events_records): - """Patch libtmg globals and return a context manager.""" - import unittest.mock as mock - patches = [ - mock.patch.object(libtmg, 'tmgTagTypes', _table(tagtypes_records)), - mock.patch.object(libtmg, 'tmgEvents', _table(events_records)), - ] - return patches - - def _run(self, tagtypes_records, events_records, per_no_map, dataset=1, db=None): - import unittest.mock as mock - if db is None: - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ - mock.patch('libtmg.tmgEvents', _table(events_records), create=True): - libtmg.import_notes(db, dataset, per_no_map) - return db - - def test_no_per_no_map_is_noop(self): - """Passing per_no_map=None must not touch any table.""" - import unittest.mock as mock - db = _make_db() - mock_table = mock.MagicMock() - with mock.patch('libtmg.tmgTagTypes', mock_table, create=True), \ - mock.patch('libtmg.tmgEvents', mock_table, create=True): - libtmg.import_notes(db, 1, per_no_map=None) - mock_table.__enter__.assert_not_called() - - def test_note_attached_to_person(self): - db, phandle = _add_person(_make_db()) - per_no_map = {42: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=42, recno=1, - efoot='Born in London')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_note_list()), 1) - - def test_note_text_stored(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=10, etypename='Note')], - events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, - efoot=' Some note text ')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - note = db.get_note_from_handle(person.get_note_list()[0]) - self.assertEqual(note.get(), 'Some note text') - - def test_note_type_is_person(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='hello')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - note = db.get_note_from_handle(person.get_note_list()[0]) - self.assertEqual(note.get_type(), NoteType.PERSON) - - def test_tmg_codes_stripped_from_note(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='[:ITAL:]italicised[:ITAL:]')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - note = db.get_note_from_handle(person.get_note_list()[0]) - self.assertEqual(note.get(), 'italicised') - - def test_empty_note_text_skipped(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_note_list()), 0) - - def test_unknown_person_skipped(self): - db = _make_db() - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=99, recno=1, - efoot='orphan note')], - per_no_map={}, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - def test_non_note_etype_ignored(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - # etype=10 is not a Note type - events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, - efoot='should be ignored')], - per_no_map=per_no_map, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - def test_wrong_dataset_ignored(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=2, etype=77, per1=1, recno=1, - efoot='wrong dataset')], - per_no_map=per_no_map, dataset=1, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - def test_multiple_notes_for_one_person(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[ - _Rec(dsid=1, etype=77, per1=1, recno=1, efoot='first'), - _Rec(dsid=1, etype=77, per1=1, recno=2, efoot='second'), - ], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_note_list()), 2) - - def test_no_note_tag_type_defined(self): - """If the dataset has no 'Note' tag type, nothing is imported.""" - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[], # no tag types at all - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='orphan')], - per_no_map=per_no_map, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - -# --------------------------------------------------------------------------- -# trial_events — event import and Note-etype skip -# --------------------------------------------------------------------------- - -# A minimal raw date string for an exact date (1900-06-15) -_EXACT_DATE = '1' + '19000615' + '0' + '3' + '00000000' + '0' + '0' -_EMPTY_DATE = '' - - -class TestTrialEvents(unittest.TestCase): - - def _run(self, tagtypes_records, events_records, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ - mock.patch('libtmg.tmgEvents', _table(events_records), create=True): - handle_map = libtmg.import_events(db, dataset) - return db, handle_map - - def test_regular_event_creates_db_entry(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='')] - db, hmap = self._run(_tagtypes, _events) - self.assertEqual(db.get_number_of_events(), 1) - self.assertIn(1, hmap) - - def test_handle_map_tuple_has_four_elements(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=5, etype=10, per1=3, per2=0, - placenum=7, edate=_EMPTY_DATE, efoot='')] - _, hmap = self._run(_tagtypes, _events) - entry = hmap[5] - self.assertEqual(len(entry), 4) - _handle, per1, per2, placenum = entry - self.assertEqual(per1, 3) - self.assertEqual(per2, 0) - self.assertEqual(placenum, 7) - - def test_note_etype_event_not_in_handle_map(self): - _tagtypes = [_Rec(dsid=1, etypenum=77, etypename='Note')] - _events = [_Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='a note')] - db, hmap = self._run(_tagtypes, _events) - self.assertEqual(db.get_number_of_events(), 0) - self.assertNotIn(1, hmap) - - def test_event_memo_stored_as_description(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='born here')] - db, hmap = self._run(_tagtypes, _events) - event = db.get_event_from_handle(hmap[1][0]) - self.assertEqual(event.get_description(), 'born here') - - def test_event_memo_tmg_codes_stripped(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, - efoot='[:CR:]born here')] - db, hmap = self._run(_tagtypes, _events) - event = db.get_event_from_handle(hmap[1][0]) - self.assertEqual(event.get_description(), 'born here') - - def test_event_date_set(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EXACT_DATE, efoot='')] - db, hmap = self._run(_tagtypes, _events) - event = db.get_event_from_handle(hmap[1][0]) - d = event.get_date_object() - self.assertEqual(d.get_year(), 1900) - self.assertEqual(d.get_month(), 6) - - def test_wrong_dataset_skipped(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=2, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='')] - db, hmap = self._run(_tagtypes, _events, dataset=1) - self.assertEqual(db.get_number_of_events(), 0) - - def test_mixed_note_and_regular_events(self): - _tagtypes = [ - _Rec(dsid=1, etypenum=77, etypename='Note'), - _Rec(dsid=1, etypenum=10, etypename='Birth'), - ] - _events = [ - _Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='a note'), - _Rec(dsid=1, recno=2, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot=''), - ] - db, hmap = self._run(_tagtypes, _events) - self.assertEqual(db.get_number_of_events(), 1) - self.assertNotIn(1, hmap) - self.assertIn(2, hmap) - - -# --------------------------------------------------------------------------- -# import_sources — info-field parsing and author/publication split -# --------------------------------------------------------------------------- - -class TestImportSources(unittest.TestCase): - - def _run(self, src_components, src_repo_links, sources_records, - repo_handle_map=None, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgSourceComponents', _table(src_components), create=True), \ - mock.patch('libtmg.tmgSourceRepositoryLinks', _table(src_repo_links), create=True), \ - mock.patch('libtmg.tmgSources', _table(sources_records), create=True): - smap = libtmg.import_sources(db, dataset, repo_handle_map) - return db, smap - - def _source_rec(self, **kw): - defaults = dict(dsid=1, majnum=1, mactive=True, - title='Test Source', abbrev='', info='', - text='', fform='', sform='', bform='', reminders='') - defaults.update(kw) - return _Rec(**defaults) - - def test_source_created(self): - db, smap = self._run([], [], [self._source_rec()]) - self.assertEqual(db.get_number_of_sources(), 1) - self.assertIn(1, smap) - - def test_title_set(self): - db, smap = self._run([], [], [self._source_rec(title='My Source')]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_title(), 'My Source') - - def test_abbreviation_set(self): - db, smap = self._run([], [], [self._source_rec(abbrev='MySrc')]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_abbreviation(), 'MySrc') - - def test_inactive_source_skipped(self): - db, smap = self._run([], [], [self._source_rec(mactive=False)]) - self.assertEqual(db.get_number_of_sources(), 0) - - def test_wrong_dataset_skipped(self): - db, smap = self._run([], [], [self._source_rec(dsid=2)], dataset=1) - self.assertEqual(db.get_number_of_sources(), 0) - - def test_author_element_sets_author(self): - # recno 1 → position 0 in $!& split - components = [_Rec(recno=1, element='[AUTHOR]')] - rec = self._source_rec(info='John Smith') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_author(), 'John Smith') - - def test_non_author_element_sets_publication_info(self): - components = [_Rec(recno=1, element='[TITLE]')] - rec = self._source_rec(info='Some Title') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertIn('TITLE', src.get_publication_info()) - self.assertIn('Some Title', src.get_publication_info()) - - def test_multiple_authors_joined_with_semicolon(self): - # positions 0 and 1 → recno 1 and 2 - components = [ - _Rec(recno=1, element='[AUTHOR]'), - _Rec(recno=2, element='[EDITOR]'), - ] - rec = self._source_rec(info='Alice$!&Bob') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_author(), 'Alice; Bob') - - def test_empty_info_position_skipped(self): - # position 0 empty, position 1 filled → recno 2 = [AUTHOR] - components = [ - _Rec(recno=1, element='[TITLE]'), - _Rec(recno=2, element='[AUTHOR]'), - ] - rec = self._source_rec(info='$!&Jane Doe') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_author(), 'Jane Doe') - self.assertEqual(src.get_publication_info(), '') - - def test_note_fields_become_notes(self): - rec = self._source_rec(text='original text', fform='', sform='', bform='') - db, smap = self._run([], [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(len(src.get_note_list()), 1) - note = db.get_note_from_handle(src.get_note_list()[0]) - self.assertIn('original text', note.get()) - - def test_multiple_note_fields_each_become_a_note(self): - rec = self._source_rec(text='txt', fform='fn', sform='', bform='') - db, smap = self._run([], [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(len(src.get_note_list()), 2) - - -# --------------------------------------------------------------------------- -# import_places — name reconstruction, type resolution, note parts -# --------------------------------------------------------------------------- - -class TestImportPlaces(unittest.TestCase): - - def _run(self, part_types, place_dict, ppv_records, places_records, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgPlacePartType', _table(part_types), create=True), \ - mock.patch('libtmg.tmgPlaceDictionary', _table(place_dict), create=True), \ - mock.patch('libtmg.tmgPlacePartValue', _table(ppv_records), create=True), \ - mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): - pmap = libtmg.import_places(db, dataset) - return db, pmap - - # Convenience: build part_type, place_dict, ppv records for a single place - def _setup(self, recno, parts, dataset=1, comment='', shortplace=''): - """ - parts: list of (label, value) e.g. [('City','London'),('Country','UK')] - Returns (part_type_recs, place_dict_recs, ppv_recs, place_recs) - """ - part_type_recs = [] - place_dict_recs = [] - ppv_recs = [] - for i, (label, value) in enumerate(parts): - type_id = i + 1 - uid = i + 100 - part_type_recs.append(_Rec(type=type_id, value=label)) - place_dict_recs.append(_Rec(uid=uid, value=value)) - ppv_recs.append(_Rec(dsid=dataset, recno=recno, type=type_id, uid=uid)) - place_recs = [_Rec(dsid=dataset, recno=recno, - shortplace=shortplace, comment=comment)] - return part_type_recs, place_dict_recs, ppv_recs, place_recs - - def test_city_only_name_and_type(self): - pt, pd, ppv, pl = self._setup(1, [('City', 'London')]) - db, pmap = self._run(pt, pd, ppv, pl) - self.assertIn(1, pmap) - place = db.get_place_from_handle(pmap[1]) - self.assertEqual(place.get_name().get_value(), 'London') - from gramps.gen.lib import PlaceType - self.assertEqual(place.get_type().value, PlaceType.CITY) - - def test_country_only_name_and_type(self): - pt, pd, ppv, pl = self._setup(1, [('Country', 'France')]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - from gramps.gen.lib import PlaceType - self.assertEqual(place.get_type().value, PlaceType.COUNTRY) - - def test_city_state_country_name_order(self): - pt, pd, ppv, pl = self._setup(1, [ - ('City', 'Paris'), ('State', 'Île-de-France'), ('Country', 'France') - ]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - # GEO_ORDER: Addressee, Detail, City, County, State, Country - self.assertEqual(place.get_name().get_value(), - 'Paris, Île-de-France, France') - - def test_most_specific_type_wins(self): - # City is more specific than Country in _GEO_ORDER - pt, pd, ppv, pl = self._setup(1, [ - ('City', 'Berlin'), ('Country', 'Germany') - ]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - from gramps.gen.lib import PlaceType - self.assertEqual(place.get_type().value, PlaceType.CITY) - - def test_empty_place_skipped(self): - # No parts and no shortplace → nothing imported - place_recs = [_Rec(dsid=1, recno=1, shortplace='', comment='')] - db, pmap = self._run([], [], [], place_recs) - self.assertEqual(db.get_number_of_places(), 0) - self.assertNotIn(1, pmap) - - def test_shortplace_fallback(self): - # No parts but shortplace set → use it - place_recs = [_Rec(dsid=1, recno=1, shortplace='Somewhere', comment='')] - db, pmap = self._run([], [], [], place_recs) - self.assertIn(1, pmap) - place = db.get_place_from_handle(pmap[1]) - self.assertEqual(place.get_name().get_value(), 'Somewhere') - - def test_note_parts_go_to_note(self): - pt, pd, ppv, pl = self._setup(1, [ - ('City', 'Rome'), ('Postal', '00100') - ]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - self.assertEqual(len(place.get_note_list()), 1) - note = db.get_note_from_handle(place.get_note_list()[0]) - self.assertIn('Postal', note.get()) - self.assertIn('00100', note.get()) - - def test_comment_goes_to_note(self): - pt, pd, ppv, pl = self._setup(1, [('City', 'Rome')], comment='see also') - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - note = db.get_note_from_handle(place.get_note_list()[0]) - self.assertIn('see also', note.get()) - - def test_wrong_dataset_skipped(self): - pt, pd, ppv, pl = self._setup(1, [('City', 'Oslo')], dataset=2) - db, pmap = self._run(pt, pd, ppv, pl, dataset=1) - self.assertEqual(db.get_number_of_places(), 0) - - def test_returns_recno_to_handle_map(self): - pt, pd, ppv, pl = self._setup(42, [('City', 'Vienna')]) - db, pmap = self._run(pt, pd, ppv, pl) - self.assertIn(42, pmap) - - -# --------------------------------------------------------------------------- -# link_event_places — event gets its place handle set -# --------------------------------------------------------------------------- - -class TestLinkEventPlaces(unittest.TestCase): - - def _make_event(self, db): - from gramps.gen.db import DbTxn - ev = Event() - with DbTxn("setup", db) as t: - db.add_event(ev, t) - return ev.get_handle() - - def _make_place(self, db): - from gramps.gen.db import DbTxn - pl = Place() - with DbTxn("setup", db) as t: - db.add_place(pl, t) - return pl.get_handle() - - def test_place_linked_to_event(self): - db = _make_db() - ev_handle = self._make_event(db) - pl_handle = self._make_place(db) - event_handle_map = {1: (ev_handle, 1, 0, 7)} - place_handle_map = {7: pl_handle} - libtmg.link_event_places(db, event_handle_map, place_handle_map) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(event.get_place_handle(), pl_handle) - - def test_zero_placenum_skipped(self): - db = _make_db() - ev_handle = self._make_event(db) - event_handle_map = {1: (ev_handle, 1, 0, 0)} - place_handle_map = {0: self._make_place(db)} - libtmg.link_event_places(db, event_handle_map, place_handle_map) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(event.get_place_handle(), '') - - def test_unknown_placenum_skipped(self): - db = _make_db() - ev_handle = self._make_event(db) - event_handle_map = {1: (ev_handle, 1, 0, 99)} - libtmg.link_event_places(db, event_handle_map, {}) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(event.get_place_handle(), '') - - def test_empty_maps_noop(self): - db = _make_db() - libtmg.link_event_places(db, {}, {}) # must not raise - libtmg.link_event_places(db, None, None) - - # --------------------------------------------------------------------------- # Pure functions: num_to_month, num_to_date, parse_date # --------------------------------------------------------------------------- @@ -893,626 +293,5 @@ def test_domain_embedded_in_name(self): self.assertEqual(url, 'https://www.ancestry.com') -# --------------------------------------------------------------------------- -# Lookup helpers: short_place_name, tag_type_name -# --------------------------------------------------------------------------- - -class TestShortPlaceName(unittest.TestCase): - - def _run(self, places_records, placenum, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): - return libtmg.short_place_name(db, placenum, dataset) - - def test_returns_shortplace(self): - rec = _Rec(dsid=1, recno=5, shortplace='New York ', styleid=1, comment='') - self.assertEqual(self._run([rec], placenum=5), 'New York') - - def test_trailing_whitespace_stripped(self): - rec = _Rec(dsid=1, recno=1, shortplace='London ', styleid=1, comment='') - self.assertEqual(self._run([rec], placenum=1), 'London') - - def test_wrong_recno_returns_none(self): - rec = _Rec(dsid=1, recno=1, shortplace='Paris', styleid=1, comment='') - self.assertIsNone(self._run([rec], placenum=99)) - - def test_wrong_dataset_returns_none(self): - rec = _Rec(dsid=2, recno=1, shortplace='Berlin', styleid=1, comment='') - self.assertIsNone(self._run([rec], placenum=1, dataset=1)) - - -class TestTagTypeName(unittest.TestCase): - - def _run(self, tagtypes_records, eventtype, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True): - return libtmg.tag_type_name(db, eventtype, dataset) - - def test_returns_name(self): - rec = _Rec(dsid=1, etypenum=2, etypename='Birth ') - self.assertEqual(self._run([rec], eventtype=2), 'Birth') - - def test_trailing_whitespace_stripped(self): - rec = _Rec(dsid=1, etypenum=3, etypename='Death ') - self.assertEqual(self._run([rec], eventtype=3), 'Death') - - def test_wrong_eventtype_returns_none(self): - rec = _Rec(dsid=1, etypenum=2, etypename='Birth') - self.assertIsNone(self._run([rec], eventtype=99)) - - def test_wrong_dataset_returns_none(self): - rec = _Rec(dsid=2, etypenum=2, etypename='Birth') - self.assertIsNone(self._run([rec], eventtype=2, dataset=1)) - - -# --------------------------------------------------------------------------- -# import_people — name parsing, gender, dataset filter -# --------------------------------------------------------------------------- - -class TestImportPeople(unittest.TestCase): - - def _run(self, names_records, people_records, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgNames', _table(names_records), create=True), \ - mock.patch('libtmg.tmgPeople', _table(people_records), create=True): - per_no_map = libtmg.import_people(db, dataset) - return db, per_no_map - - def _name_rec(self, **kw): - defaults = dict(dsid=1, nper=1, primary=True, srnamedisp='SMITH, John') - defaults.update(kw) - return _Rec(**defaults) - - def _person_rec(self, **kw): - defaults = dict(dsid=1, per_no=1, sex='M') - defaults.update(kw) - return _Rec(**defaults) - - def test_person_created(self): - db, pmap = self._run([self._name_rec()], [self._person_rec()]) - self.assertEqual(db.get_number_of_people(), 1) - - def test_returns_per_no_map(self): - db, pmap = self._run([self._name_rec(nper=5)], [self._person_rec(per_no=5)]) - self.assertIn(5, pmap) - - def test_surname_parsed(self): - db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], - [self._person_rec(per_no=1)]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_primary_name().get_surname(), 'JONES') - - def test_given_name_parsed(self): - db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], - [self._person_rec(per_no=1)]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_primary_name().get_first_name(), 'Alice') - - def test_male_gender(self): - db, pmap = self._run([self._name_rec()], [self._person_rec(sex='M')]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_gender(), Person.MALE) - - def test_female_gender(self): - db, pmap = self._run([self._name_rec()], [self._person_rec(sex='F')]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_gender(), Person.FEMALE) - - def test_unknown_gender(self): - db, pmap = self._run([self._name_rec()], [self._person_rec(sex='?')]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_gender(), Person.UNKNOWN) - - def test_non_primary_name_skipped(self): - db, pmap = self._run( - [self._name_rec(primary=False, srnamedisp='ALT, Name')], - [self._person_rec()] - ) - self.assertEqual(db.get_number_of_people(), 0) - - def test_wrong_dataset_skipped(self): - db, pmap = self._run([self._name_rec(dsid=2)], [self._person_rec(dsid=2)], - dataset=1) - self.assertEqual(db.get_number_of_people(), 0) - - def test_no_comma_surname_only(self): - # srnamedisp with no comma → surname=full string, given='' - db, pmap = self._run([self._name_rec(srnamedisp='SMITH')], - [self._person_rec()]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_primary_name().get_surname(), 'SMITH') - self.assertEqual(p.get_primary_name().get_first_name(), '') - - -# --------------------------------------------------------------------------- -# link_person_events — EventRefs, birth/death special refs -# --------------------------------------------------------------------------- - -class TestLinkPersonEvents(unittest.TestCase): - - def _make_typed_event(self, db, event_type_int): - from gramps.gen.lib import EventType - ev = Event() - ev.set_type(EventType(event_type_int)) - with DbTxn("setup", db) as t: - db.add_event(ev, t) - return ev.get_handle() - - def test_individual_event_linked_to_person(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.OCCUPATION) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - p = db.get_person_from_handle(phandle) - self.assertEqual(len(p.get_event_ref_list()), 1) - - def test_couple_event_not_linked_to_person(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.MARRIAGE) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 2, 0)}) - p = db.get_person_from_handle(phandle) - self.assertEqual(len(p.get_event_ref_list()), 0) - - def test_birth_event_sets_birth_ref(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.BIRTH) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - p = db.get_person_from_handle(phandle) - self.assertIsNotNone(p.get_birth_ref()) - self.assertEqual(p.get_birth_ref().ref, ev_handle) - - def test_death_event_sets_death_ref(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.DEATH) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - p = db.get_person_from_handle(phandle) - self.assertIsNotNone(p.get_death_ref()) - - def test_unknown_person_skipped(self): - from gramps.gen.lib import EventType - db = _make_db() - ev_handle = self._make_typed_event(db, EventType.BIRTH) - # Must not raise even when per1 has no entry in per_no_map - libtmg.link_person_events(db, - per_no_map={}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - - def test_empty_maps_noop(self): - db = _make_db() - libtmg.link_person_events(db, None, None) - libtmg.link_person_events(db, {}, {}) - - -# --------------------------------------------------------------------------- -# import_families — parent-child grouping, couple events, rel type -# --------------------------------------------------------------------------- - -class TestImportFamilies(unittest.TestCase): - - def _run(self, tagtypes, pc_rels, per_no_by_gender, event_handle_map=None, - dataset=1): - """Create persons from per_no_by_gender={per_no: gender}, run import.""" - import unittest.mock as mock - db = _make_db() - pmap = {} - for per_no, gender in per_no_by_gender.items(): - p = Person() - p.set_gender(gender) - with DbTxn("setup", db) as t: - db.add_person(p, t) - pmap[per_no] = p.get_handle() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table(pc_rels), create=True): - libtmg.import_families(db, dataset, pmap, event_handle_map) - return db, pmap - - def _pc(self, parent, child, ptype, primary=True, pnote='', dsid=1): - return _Rec(dsid=dsid, parent=parent, child=child, - ptype=ptype, primary=primary, pnote=pnote) - - def _father_type(self, num=1): - return _Rec(dsid=1, etypenum=num, etypename='Father-Biological') - - def _mother_type(self, num=2): - return _Rec(dsid=1, etypenum=num, etypename='Mother-Biological') - - def test_father_child_creates_family(self): - db, pmap = self._run([self._father_type()], - [self._pc(1, 2, ptype=1)], - {1: Person.MALE, 2: Person.UNKNOWN}) - self.assertEqual(db.get_number_of_families(), 1) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_father_handle(), pmap[1]) - - def test_mother_child_creates_family(self): - db, pmap = self._run([self._mother_type()], - [self._pc(1, 2, ptype=2)], - {1: Person.FEMALE, 2: Person.UNKNOWN}) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_mother_handle(), pmap[1]) - - def test_father_and_mother_same_family(self): - db, pmap = self._run( - [self._father_type(1), self._mother_type(2)], - [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)], - {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}, - ) - self.assertEqual(db.get_number_of_families(), 1) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_father_handle(), pmap[1]) - self.assertEqual(fam.get_mother_handle(), pmap[2]) - - def test_child_added_to_family(self): - db, pmap = self._run([self._father_type()], - [self._pc(1, 2, ptype=1)], - {1: Person.MALE, 2: Person.UNKNOWN}) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(len(fam.get_child_ref_list()), 1) - self.assertEqual(fam.get_child_ref_list()[0].ref, pmap[2]) - - def test_child_ref_type_biological(self): - from gramps.gen.lib import ChildRefType - db, pmap = self._run([self._father_type()], - [self._pc(1, 2, ptype=1)], - {1: Person.MALE, 2: Person.UNKNOWN}) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_child_ref_list()[0].get_father_relation(), - ChildRefType.BIRTH) - - def test_wrong_dataset_skipped(self): - db, _ = self._run([self._father_type()], - [self._pc(1, 2, ptype=1, dsid=2)], - {1: Person.MALE, 2: Person.UNKNOWN}, dataset=1) - self.assertEqual(db.get_number_of_families(), 0) - - def test_no_per_no_map_is_noop(self): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table([]), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table([]), create=True): - libtmg.import_families(db, 1, per_no_map=None) - self.assertEqual(db.get_number_of_families(), 0) - - def test_marriage_event_sets_rel_type_married(self): - from gramps.gen.lib import EventType, FamilyRelType - import unittest.mock as mock - - db = _make_db() - pmap = {} - for per_no, gender in {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}.items(): - p = Person() - p.set_gender(gender) - with DbTxn("s", db) as t: - db.add_person(p, t) - pmap[per_no] = p.get_handle() - - ev = Event() - ev.set_type(EventType(EventType.MARRIAGE)) - with DbTxn("s", db) as t: - db.add_event(ev, t) - - tagtypes = [self._father_type(1), self._mother_type(2)] - pc = [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)] - event_handle_map = {99: (ev.get_handle(), 1, 2, 0)} - - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table(pc), create=True): - libtmg.import_families(db, 1, pmap, event_handle_map) - - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_relationship(), FamilyRelType.MARRIED) - self.assertEqual(len(fam.get_event_ref_list()), 1) - - -# --------------------------------------------------------------------------- -# import_repositories — name, type inference, URL, notes -# --------------------------------------------------------------------------- - -class TestImportRepositories(unittest.TestCase): - - def _run(self, repo_records, per_no_map=None, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgRepositories', _table(repo_records), create=True): - repo_map = libtmg.import_repositories(db, dataset, per_no_map) - return db, repo_map - - def _repo_rec(self, **kw): - defaults = dict(dsid=1, recno=1, name='City Library', - abbrev='', rnote='', rperno=0) - defaults.update(kw) - return _Rec(**defaults) - - def test_repository_created(self): - db, rmap = self._run([self._repo_rec()]) - self.assertEqual(db.get_number_of_repositories(), 1) - self.assertIn(1, rmap) - - def test_name_set(self): - db, rmap = self._run([self._repo_rec(name='National Archives')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(repo.get_name(), 'National Archives') - - def test_wrong_dataset_skipped(self): - db, rmap = self._run([self._repo_rec(dsid=2)], dataset=1) - self.assertEqual(db.get_number_of_repositories(), 0) - - def test_type_inferred_from_name(self): - from gramps.gen.lib import RepositoryType - db, rmap = self._run([self._repo_rec(name='ancestry.com')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(repo.get_type().value, RepositoryType.WEBSITE) - - def test_url_added_for_web_repo(self): - db, rmap = self._run([self._repo_rec(name='familysearch.org')]) - repo = db.get_repository_from_handle(rmap[1]) - urls = repo.get_url_list() - self.assertEqual(len(urls), 1) - self.assertIn('familysearch', urls[0].get_path()) - - def test_no_url_for_non_web_repo(self): - db, rmap = self._run([self._repo_rec(name='Local Parish Church')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(len(repo.get_url_list()), 0) - - def test_blank_name_falls_back_to_abbrev(self): - db, rmap = self._run([self._repo_rec(name='', abbrev='TNA')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(repo.get_name(), 'TNA') - - def test_note_added_when_rnote_set(self): - db, rmap = self._run([self._repo_rec(rnote='Open Mon-Fri')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(len(repo.get_note_list()), 1) - note = db.get_note_from_handle(repo.get_note_list()[0]) - self.assertIn('Open Mon-Fri', note.get()) - - def test_returns_recno_to_handle_map(self): - db, rmap = self._run([self._repo_rec(recno=42)]) - self.assertIn(42, rmap) - - -# --------------------------------------------------------------------------- -# import_citations — creation and attachment to events / persons -# --------------------------------------------------------------------------- - -class TestImportCitations(unittest.TestCase): - - def _run(self, citation_records, names_records=None, pc_records=None, - source_handle_map=None, event_handle_map=None, per_no_map=None, - dataset=1, db=None): - import unittest.mock as mock - if db is None: - db = _make_db() - with mock.patch('libtmg.tmgCitations', _table(citation_records), create=True), \ - mock.patch('libtmg.tmgNames', _table(names_records or []), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table(pc_records or []), create=True): - libtmg.import_citations(db, dataset, - source_handle_map=source_handle_map, - event_handle_map=event_handle_map, - per_no_map=per_no_map) - return db - - def _cit_rec(self, **kw): - defaults = dict(dsid=1, recno=1, majsource=1, stype='E', refrec=1, - exclude=False, subsource='', citref='', citmemo='', - sdsure='', snsure='', sssure='', spsure='', sfsure='') - defaults.update(kw) - return _Rec(**defaults) - - def test_no_source_map_is_noop(self): - db = self._run([self._cit_rec()]) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_citation_created(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec()], - source_handle_map={1: src.get_handle()}, db=db) - self.assertEqual(db.get_number_of_citations(), 1) - - def test_excluded_citation_skipped(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec(exclude=True)], - source_handle_map={1: src.get_handle()}, db=db) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_wrong_dataset_skipped(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec(dsid=2)], - source_handle_map={1: src.get_handle()}, db=db, dataset=1) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_unknown_source_skipped(self): - db = _make_db() - db = self._run([self._cit_rec(majsource=99)], - source_handle_map={1: 'some_handle'}, db=db) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_citation_attached_to_event(self): - db = _make_db() - src = Source() - ev = Event() - with DbTxn("s", db) as t: - db.add_source(src, t) - db.add_event(ev, t) - ev_handle = ev.get_handle() - - db = self._run([self._cit_rec(stype='E', refrec=7)], - source_handle_map={1: src.get_handle()}, - event_handle_map={7: (ev_handle, 1, 0, 0)}, - db=db) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(len(event.get_citation_list()), 1) - - def test_citation_attached_to_person_via_name(self): - db = _make_db() - src = Source() - p = Person() - with DbTxn("s", db) as t: - db.add_source(src, t) - db.add_person(p, t) - phandle = p.get_handle() - - # name recno=3 maps to nper=5; per_no_map routes nper=5 to phandle - db = self._run( - [self._cit_rec(stype='N', refrec=3)], - names_records=[_Rec(dsid=1, recno=3, nper=5)], - source_handle_map={1: src.get_handle()}, - per_no_map={5: phandle}, - db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_citation_list()), 1) - - def test_subsource_becomes_page(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec(subsource='p.42')], - source_handle_map={1: src.get_handle()}, db=db) - cit_handle = list(db.get_citation_handles())[0] - cit = db.get_citation_from_handle(cit_handle) - self.assertEqual(cit.get_page(), 'p.42') - - - -# ── TmgProject._read_pjc_config ────────────────────────────────────────── - -def _make_minimal_sqz(tmp_dir, pjc_content): - """Create a minimal .SQZ zip containing only a PJC file.""" - import zipfile as _zf - pjc_path = os.path.join(tmp_dir, 'test.pjc') - sqz_path = os.path.join(tmp_dir, 'test.sqz') - with open(pjc_path, 'w', encoding='latin-1') as f: - f.write(pjc_content) - with _zf.ZipFile(sqz_path, 'w') as zf: - zf.write(pjc_path, 'test.pjc') - return sqz_path - - -class _MockUser: - """Minimal stand-in for the Gramps user object used in importData.""" - def __init__(self): - self.error_shown = False - self.error_message = None - self.uistate = None - - def notify_error(self, title, message=''): - self.error_shown = True - self.error_message = message - - def begin_progress(self, *a, **kw): pass - def end_progress(self): pass - def step_progress(self): pass - - -class TestReadPjcConfig(unittest.TestCase): - """Tests for TmgProject._read_pjc_config PJC parsing.""" - - _MINIMAL_PJC = ( - "[Stamp]\n" - "PjcVersion=11.0\n" - "[Researcher]\n" - "Name=Test User\n" - ) - - def _make_project(self, pjc_content, tmp_path): - pjc_file = os.path.join(tmp_path, "test.pjc") - with open(pjc_file, 'w', encoding='latin-1') as f: - f.write(pjc_content) - return libtmg.TmgProject(pjc_file) - - def test_well_formed_pjc_returns_version(self): - """A clean PJC file parses successfully and version() returns a float.""" - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(self._MINIMAL_PJC, tmp) - self.assertEqual(project.version(), 11.0) - - def test_malformed_section_header_does_not_raise(self): - """Lines like '[Exho' (no closing bracket) are silently dropped.""" - pjc = ( - "[Stamp]\n" - "PjcVersion=11.0\n" - "[Exho\n" # malformed — the crash trigger - "SomeGarbage\n" - "[Researcher]\n" - "Name=Test User\n" - ) - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(pjc, tmp) - # Must not raise; must still find [Stamp] - self.assertEqual(project.version(), 11.0) - - def test_null_bytes_stripped(self): - """NUL bytes in the PJC file are stripped before parsing.""" - pjc = "[Stamp]\x00\nPjcVersion=11.0\n" - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(pjc, tmp) - self.assertEqual(project.version(), 11.0) - - def test_parse_error_returns_partial_config(self): - """If configparser still raises after filtering, a warning is logged - and a (possibly empty) config object is returned rather than crashing.""" - import logging - # Feed content that survives the filter but still breaks configparser: - # a key=value line before any section header is technically invalid. - pjc = "orphan_key=value\n[Stamp]\nPjcVersion=11.0\n" - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(pjc, tmp) - with self.assertLogs('.TMGImport', level=logging.WARNING) as cm: - cfg = project._read_pjc_config() - self.assertTrue(any('parse error' in m.lower() or 'parsing' in m.lower() - for m in cm.output)) - # Config object is returned (not None), even if incomplete - self.assertIsNotNone(cfg) - - def test_version_too_old_notifies_user(self): - """A PJC version < 11.0 calls user.notify_error and aborts import.""" - pjc = "[Stamp]\nPjcVersion=10.0\n" # TMG 9.01 or earlier - with tempfile.TemporaryDirectory() as tmp: - sqz = _make_minimal_sqz(tmp, pjc) - db = _make_db() - user = _MockUser() - libtmg.importData(db, sqz, user) - self.assertTrue(user.error_shown, - "notify_error should have been called for old version") - self.assertIn('9.05', user.error_message or '', - "Error message should mention TMG 9.05") - - def test_missing_pjc_version_notifies_user(self): - """A PJC with no [Stamp]/PjcVersion calls user.notify_error.""" - pjc = "[OtherSection]\nSomeKey=value\n" # no [Stamp] at all - with tempfile.TemporaryDirectory() as tmp: - sqz = _make_minimal_sqz(tmp, pjc) - db = _make_db() - user = _MockUser() - libtmg.importData(db, sqz, user) - self.assertTrue(user.error_shown, - "notify_error should have been called for missing version") - if __name__ == '__main__': unittest.main() diff --git a/TMGimporter/tests/test_linux_libtmg.py b/TMGimporter/tests/test_linux_libtmg.py new file mode 100644 index 000000000..35bd11717 --- /dev/null +++ b/TMGimporter/tests/test_linux_libtmg.py @@ -0,0 +1,1218 @@ +"""Linux-only unit tests for libtmg.py + +Split off from test_libtmg.py: every class in this module creates an +in-memory Gramps SQLite database via + + make_database("sqlite").load(":memory:", None) + +which currently hangs on Windows under the conda-forge GTK + pip Gramps +combination used by the CI unit-test-windows job. Pure-logic tests that +do not touch the Gramps DB layer stay in test_libtmg.py and run on every +OS. + +Filename convention (see .github/workflows/ci.yml): + test_*.py general (every OS) + test_linux_*.py Linux-only + test_windows_*.py Windows-only + test_integration_*.py Linux-only, full-pipeline/DB-backed +""" + +import sys +import os +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import libtmg + +from gramps.gen.lib import Date, Event, NoteType, Person, Place, Source +from gramps.gen.db.utils import make_database +from gramps.gen.db import DbTxn + + +# --------------------------------------------------------------------------- +# Helpers shared across test cases +# --------------------------------------------------------------------------- + +class _Rec: + """Minimal fake DBF record — set any field via keyword arguments.""" + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +def _table(records): + """Return an object that behaves like a dbf.Table used as a context manager. + + libtmg uses tables in two ways: + with tmgFoo: + for record in tmgFoo: # iterates over context-managed table + """ + class _FakeTable: + def __enter__(self): + return self + def __exit__(self, *_): + return False + def __iter__(self): + return iter(records) + return _FakeTable() + + +def _make_db(): + """Return a fresh in-memory Gramps database.""" + db = make_database("sqlite") + db.load(":memory:", None) + return db + + +def _add_person(db): + """Add an empty Person to db and return (db, handle).""" + p = Person() + with DbTxn("setup", db) as t: + db.add_person(p, t) + return db, p.get_handle() + + +# --------------------------------------------------------------------------- +# import_notes — per-person note creation from tmg events +# --------------------------------------------------------------------------- + +class TestImportNotes(unittest.TestCase): + + def _patch(self, tagtypes_records, events_records): + """Patch libtmg globals and return a context manager.""" + import unittest.mock as mock + patches = [ + mock.patch.object(libtmg, 'tmgTagTypes', _table(tagtypes_records)), + mock.patch.object(libtmg, 'tmgEvents', _table(events_records)), + ] + return patches + + def _run(self, tagtypes_records, events_records, per_no_map, dataset=1, db=None): + import unittest.mock as mock + if db is None: + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ + mock.patch('libtmg.tmgEvents', _table(events_records), create=True): + libtmg.import_notes(db, dataset, per_no_map) + return db + + def test_no_per_no_map_is_noop(self): + """Passing per_no_map=None must not touch any table.""" + import unittest.mock as mock + db = _make_db() + mock_table = mock.MagicMock() + with mock.patch('libtmg.tmgTagTypes', mock_table, create=True), \ + mock.patch('libtmg.tmgEvents', mock_table, create=True): + libtmg.import_notes(db, 1, per_no_map=None) + mock_table.__enter__.assert_not_called() + + def test_note_attached_to_person(self): + db, phandle = _add_person(_make_db()) + per_no_map = {42: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=42, recno=1, + efoot='Born in London')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_note_list()), 1) + + def test_note_text_stored(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=10, etypename='Note')], + events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, + efoot=' Some note text ')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + note = db.get_note_from_handle(person.get_note_list()[0]) + self.assertEqual(note.get(), 'Some note text') + + def test_note_type_is_person(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='hello')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + note = db.get_note_from_handle(person.get_note_list()[0]) + self.assertEqual(note.get_type(), NoteType.PERSON) + + def test_tmg_codes_stripped_from_note(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='[:ITAL:]italicised[:ITAL:]')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + note = db.get_note_from_handle(person.get_note_list()[0]) + self.assertEqual(note.get(), 'italicised') + + def test_empty_note_text_skipped(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_note_list()), 0) + + def test_unknown_person_skipped(self): + db = _make_db() + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=99, recno=1, + efoot='orphan note')], + per_no_map={}, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + def test_non_note_etype_ignored(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + # etype=10 is not a Note type + events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, + efoot='should be ignored')], + per_no_map=per_no_map, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + def test_wrong_dataset_ignored(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=2, etype=77, per1=1, recno=1, + efoot='wrong dataset')], + per_no_map=per_no_map, dataset=1, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + def test_multiple_notes_for_one_person(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[ + _Rec(dsid=1, etype=77, per1=1, recno=1, efoot='first'), + _Rec(dsid=1, etype=77, per1=1, recno=2, efoot='second'), + ], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_note_list()), 2) + + def test_no_note_tag_type_defined(self): + """If the dataset has no 'Note' tag type, nothing is imported.""" + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[], # no tag types at all + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='orphan')], + per_no_map=per_no_map, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + +# --------------------------------------------------------------------------- +# trial_events — event import and Note-etype skip +# --------------------------------------------------------------------------- + +# A minimal raw date string for an exact date (1900-06-15) +_EXACT_DATE = '1' + '19000615' + '0' + '3' + '00000000' + '0' + '0' +_EMPTY_DATE = '' + + +class TestTrialEvents(unittest.TestCase): + + def _run(self, tagtypes_records, events_records, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ + mock.patch('libtmg.tmgEvents', _table(events_records), create=True): + handle_map = libtmg.import_events(db, dataset) + return db, handle_map + + def test_regular_event_creates_db_entry(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='')] + db, hmap = self._run(_tagtypes, _events) + self.assertEqual(db.get_number_of_events(), 1) + self.assertIn(1, hmap) + + def test_handle_map_tuple_has_four_elements(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=5, etype=10, per1=3, per2=0, + placenum=7, edate=_EMPTY_DATE, efoot='')] + _, hmap = self._run(_tagtypes, _events) + entry = hmap[5] + self.assertEqual(len(entry), 4) + _handle, per1, per2, placenum = entry + self.assertEqual(per1, 3) + self.assertEqual(per2, 0) + self.assertEqual(placenum, 7) + + def test_note_etype_event_not_in_handle_map(self): + _tagtypes = [_Rec(dsid=1, etypenum=77, etypename='Note')] + _events = [_Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='a note')] + db, hmap = self._run(_tagtypes, _events) + self.assertEqual(db.get_number_of_events(), 0) + self.assertNotIn(1, hmap) + + def test_event_memo_stored_as_description(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='born here')] + db, hmap = self._run(_tagtypes, _events) + event = db.get_event_from_handle(hmap[1][0]) + self.assertEqual(event.get_description(), 'born here') + + def test_event_memo_tmg_codes_stripped(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, + efoot='[:CR:]born here')] + db, hmap = self._run(_tagtypes, _events) + event = db.get_event_from_handle(hmap[1][0]) + self.assertEqual(event.get_description(), 'born here') + + def test_event_date_set(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EXACT_DATE, efoot='')] + db, hmap = self._run(_tagtypes, _events) + event = db.get_event_from_handle(hmap[1][0]) + d = event.get_date_object() + self.assertEqual(d.get_year(), 1900) + self.assertEqual(d.get_month(), 6) + + def test_wrong_dataset_skipped(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=2, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='')] + db, hmap = self._run(_tagtypes, _events, dataset=1) + self.assertEqual(db.get_number_of_events(), 0) + + def test_mixed_note_and_regular_events(self): + _tagtypes = [ + _Rec(dsid=1, etypenum=77, etypename='Note'), + _Rec(dsid=1, etypenum=10, etypename='Birth'), + ] + _events = [ + _Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='a note'), + _Rec(dsid=1, recno=2, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot=''), + ] + db, hmap = self._run(_tagtypes, _events) + self.assertEqual(db.get_number_of_events(), 1) + self.assertNotIn(1, hmap) + self.assertIn(2, hmap) + + +# --------------------------------------------------------------------------- +# import_sources — info-field parsing and author/publication split +# --------------------------------------------------------------------------- + +class TestImportSources(unittest.TestCase): + + def _run(self, src_components, src_repo_links, sources_records, + repo_handle_map=None, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgSourceComponents', _table(src_components), create=True), \ + mock.patch('libtmg.tmgSourceRepositoryLinks', _table(src_repo_links), create=True), \ + mock.patch('libtmg.tmgSources', _table(sources_records), create=True): + smap = libtmg.import_sources(db, dataset, repo_handle_map) + return db, smap + + def _source_rec(self, **kw): + defaults = dict(dsid=1, majnum=1, mactive=True, + title='Test Source', abbrev='', info='', + text='', fform='', sform='', bform='', reminders='') + defaults.update(kw) + return _Rec(**defaults) + + def test_source_created(self): + db, smap = self._run([], [], [self._source_rec()]) + self.assertEqual(db.get_number_of_sources(), 1) + self.assertIn(1, smap) + + def test_title_set(self): + db, smap = self._run([], [], [self._source_rec(title='My Source')]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_title(), 'My Source') + + def test_abbreviation_set(self): + db, smap = self._run([], [], [self._source_rec(abbrev='MySrc')]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_abbreviation(), 'MySrc') + + def test_inactive_source_skipped(self): + db, smap = self._run([], [], [self._source_rec(mactive=False)]) + self.assertEqual(db.get_number_of_sources(), 0) + + def test_wrong_dataset_skipped(self): + db, smap = self._run([], [], [self._source_rec(dsid=2)], dataset=1) + self.assertEqual(db.get_number_of_sources(), 0) + + def test_author_element_sets_author(self): + # recno 1 → position 0 in $!& split + components = [_Rec(recno=1, element='[AUTHOR]')] + rec = self._source_rec(info='John Smith') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_author(), 'John Smith') + + def test_non_author_element_sets_publication_info(self): + components = [_Rec(recno=1, element='[TITLE]')] + rec = self._source_rec(info='Some Title') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertIn('TITLE', src.get_publication_info()) + self.assertIn('Some Title', src.get_publication_info()) + + def test_multiple_authors_joined_with_semicolon(self): + # positions 0 and 1 → recno 1 and 2 + components = [ + _Rec(recno=1, element='[AUTHOR]'), + _Rec(recno=2, element='[EDITOR]'), + ] + rec = self._source_rec(info='Alice$!&Bob') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_author(), 'Alice; Bob') + + def test_empty_info_position_skipped(self): + # position 0 empty, position 1 filled → recno 2 = [AUTHOR] + components = [ + _Rec(recno=1, element='[TITLE]'), + _Rec(recno=2, element='[AUTHOR]'), + ] + rec = self._source_rec(info='$!&Jane Doe') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_author(), 'Jane Doe') + self.assertEqual(src.get_publication_info(), '') + + def test_note_fields_become_notes(self): + rec = self._source_rec(text='original text', fform='', sform='', bform='') + db, smap = self._run([], [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(len(src.get_note_list()), 1) + note = db.get_note_from_handle(src.get_note_list()[0]) + self.assertIn('original text', note.get()) + + def test_multiple_note_fields_each_become_a_note(self): + rec = self._source_rec(text='txt', fform='fn', sform='', bform='') + db, smap = self._run([], [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(len(src.get_note_list()), 2) + + +# --------------------------------------------------------------------------- +# import_places — name reconstruction, type resolution, note parts +# --------------------------------------------------------------------------- + +class TestImportPlaces(unittest.TestCase): + + def _run(self, part_types, place_dict, ppv_records, places_records, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgPlacePartType', _table(part_types), create=True), \ + mock.patch('libtmg.tmgPlaceDictionary', _table(place_dict), create=True), \ + mock.patch('libtmg.tmgPlacePartValue', _table(ppv_records), create=True), \ + mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): + pmap = libtmg.import_places(db, dataset) + return db, pmap + + # Convenience: build part_type, place_dict, ppv records for a single place + def _setup(self, recno, parts, dataset=1, comment='', shortplace=''): + """ + parts: list of (label, value) e.g. [('City','London'),('Country','UK')] + Returns (part_type_recs, place_dict_recs, ppv_recs, place_recs) + """ + part_type_recs = [] + place_dict_recs = [] + ppv_recs = [] + for i, (label, value) in enumerate(parts): + type_id = i + 1 + uid = i + 100 + part_type_recs.append(_Rec(type=type_id, value=label)) + place_dict_recs.append(_Rec(uid=uid, value=value)) + ppv_recs.append(_Rec(dsid=dataset, recno=recno, type=type_id, uid=uid)) + place_recs = [_Rec(dsid=dataset, recno=recno, + shortplace=shortplace, comment=comment)] + return part_type_recs, place_dict_recs, ppv_recs, place_recs + + def test_city_only_name_and_type(self): + pt, pd, ppv, pl = self._setup(1, [('City', 'London')]) + db, pmap = self._run(pt, pd, ppv, pl) + self.assertIn(1, pmap) + place = db.get_place_from_handle(pmap[1]) + self.assertEqual(place.get_name().get_value(), 'London') + from gramps.gen.lib import PlaceType + self.assertEqual(place.get_type().value, PlaceType.CITY) + + def test_country_only_name_and_type(self): + pt, pd, ppv, pl = self._setup(1, [('Country', 'France')]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + from gramps.gen.lib import PlaceType + self.assertEqual(place.get_type().value, PlaceType.COUNTRY) + + def test_city_state_country_name_order(self): + pt, pd, ppv, pl = self._setup(1, [ + ('City', 'Paris'), ('State', 'Île-de-France'), ('Country', 'France') + ]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + # GEO_ORDER: Addressee, Detail, City, County, State, Country + self.assertEqual(place.get_name().get_value(), + 'Paris, Île-de-France, France') + + def test_most_specific_type_wins(self): + # City is more specific than Country in _GEO_ORDER + pt, pd, ppv, pl = self._setup(1, [ + ('City', 'Berlin'), ('Country', 'Germany') + ]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + from gramps.gen.lib import PlaceType + self.assertEqual(place.get_type().value, PlaceType.CITY) + + def test_empty_place_skipped(self): + # No parts and no shortplace → nothing imported + place_recs = [_Rec(dsid=1, recno=1, shortplace='', comment='')] + db, pmap = self._run([], [], [], place_recs) + self.assertEqual(db.get_number_of_places(), 0) + self.assertNotIn(1, pmap) + + def test_shortplace_fallback(self): + # No parts but shortplace set → use it + place_recs = [_Rec(dsid=1, recno=1, shortplace='Somewhere', comment='')] + db, pmap = self._run([], [], [], place_recs) + self.assertIn(1, pmap) + place = db.get_place_from_handle(pmap[1]) + self.assertEqual(place.get_name().get_value(), 'Somewhere') + + def test_note_parts_go_to_note(self): + pt, pd, ppv, pl = self._setup(1, [ + ('City', 'Rome'), ('Postal', '00100') + ]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + self.assertEqual(len(place.get_note_list()), 1) + note = db.get_note_from_handle(place.get_note_list()[0]) + self.assertIn('Postal', note.get()) + self.assertIn('00100', note.get()) + + def test_comment_goes_to_note(self): + pt, pd, ppv, pl = self._setup(1, [('City', 'Rome')], comment='see also') + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + note = db.get_note_from_handle(place.get_note_list()[0]) + self.assertIn('see also', note.get()) + + def test_wrong_dataset_skipped(self): + pt, pd, ppv, pl = self._setup(1, [('City', 'Oslo')], dataset=2) + db, pmap = self._run(pt, pd, ppv, pl, dataset=1) + self.assertEqual(db.get_number_of_places(), 0) + + def test_returns_recno_to_handle_map(self): + pt, pd, ppv, pl = self._setup(42, [('City', 'Vienna')]) + db, pmap = self._run(pt, pd, ppv, pl) + self.assertIn(42, pmap) + + +# --------------------------------------------------------------------------- +# link_event_places — event gets its place handle set +# --------------------------------------------------------------------------- + +class TestLinkEventPlaces(unittest.TestCase): + + def _make_event(self, db): + from gramps.gen.db import DbTxn + ev = Event() + with DbTxn("setup", db) as t: + db.add_event(ev, t) + return ev.get_handle() + + def _make_place(self, db): + from gramps.gen.db import DbTxn + pl = Place() + with DbTxn("setup", db) as t: + db.add_place(pl, t) + return pl.get_handle() + + def test_place_linked_to_event(self): + db = _make_db() + ev_handle = self._make_event(db) + pl_handle = self._make_place(db) + event_handle_map = {1: (ev_handle, 1, 0, 7)} + place_handle_map = {7: pl_handle} + libtmg.link_event_places(db, event_handle_map, place_handle_map) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(event.get_place_handle(), pl_handle) + + def test_zero_placenum_skipped(self): + db = _make_db() + ev_handle = self._make_event(db) + event_handle_map = {1: (ev_handle, 1, 0, 0)} + place_handle_map = {0: self._make_place(db)} + libtmg.link_event_places(db, event_handle_map, place_handle_map) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(event.get_place_handle(), '') + + def test_unknown_placenum_skipped(self): + db = _make_db() + ev_handle = self._make_event(db) + event_handle_map = {1: (ev_handle, 1, 0, 99)} + libtmg.link_event_places(db, event_handle_map, {}) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(event.get_place_handle(), '') + + def test_empty_maps_noop(self): + db = _make_db() + libtmg.link_event_places(db, {}, {}) # must not raise + libtmg.link_event_places(db, None, None) + + +# --------------------------------------------------------------------------- +# Pure functions: num_to_month, num_to_date, parse_date +# --------------------------------------------------------------------------- +class TestShortPlaceName(unittest.TestCase): + + def _run(self, places_records, placenum, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): + return libtmg.short_place_name(db, placenum, dataset) + + def test_returns_shortplace(self): + rec = _Rec(dsid=1, recno=5, shortplace='New York ', styleid=1, comment='') + self.assertEqual(self._run([rec], placenum=5), 'New York') + + def test_trailing_whitespace_stripped(self): + rec = _Rec(dsid=1, recno=1, shortplace='London ', styleid=1, comment='') + self.assertEqual(self._run([rec], placenum=1), 'London') + + def test_wrong_recno_returns_none(self): + rec = _Rec(dsid=1, recno=1, shortplace='Paris', styleid=1, comment='') + self.assertIsNone(self._run([rec], placenum=99)) + + def test_wrong_dataset_returns_none(self): + rec = _Rec(dsid=2, recno=1, shortplace='Berlin', styleid=1, comment='') + self.assertIsNone(self._run([rec], placenum=1, dataset=1)) + + +class TestTagTypeName(unittest.TestCase): + + def _run(self, tagtypes_records, eventtype, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True): + return libtmg.tag_type_name(db, eventtype, dataset) + + def test_returns_name(self): + rec = _Rec(dsid=1, etypenum=2, etypename='Birth ') + self.assertEqual(self._run([rec], eventtype=2), 'Birth') + + def test_trailing_whitespace_stripped(self): + rec = _Rec(dsid=1, etypenum=3, etypename='Death ') + self.assertEqual(self._run([rec], eventtype=3), 'Death') + + def test_wrong_eventtype_returns_none(self): + rec = _Rec(dsid=1, etypenum=2, etypename='Birth') + self.assertIsNone(self._run([rec], eventtype=99)) + + def test_wrong_dataset_returns_none(self): + rec = _Rec(dsid=2, etypenum=2, etypename='Birth') + self.assertIsNone(self._run([rec], eventtype=2, dataset=1)) + + +# --------------------------------------------------------------------------- +# import_people — name parsing, gender, dataset filter +# --------------------------------------------------------------------------- + +class TestImportPeople(unittest.TestCase): + + def _run(self, names_records, people_records, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgNames', _table(names_records), create=True), \ + mock.patch('libtmg.tmgPeople', _table(people_records), create=True): + per_no_map = libtmg.import_people(db, dataset) + return db, per_no_map + + def _name_rec(self, **kw): + defaults = dict(dsid=1, nper=1, primary=True, srnamedisp='SMITH, John') + defaults.update(kw) + return _Rec(**defaults) + + def _person_rec(self, **kw): + defaults = dict(dsid=1, per_no=1, sex='M') + defaults.update(kw) + return _Rec(**defaults) + + def test_person_created(self): + db, pmap = self._run([self._name_rec()], [self._person_rec()]) + self.assertEqual(db.get_number_of_people(), 1) + + def test_returns_per_no_map(self): + db, pmap = self._run([self._name_rec(nper=5)], [self._person_rec(per_no=5)]) + self.assertIn(5, pmap) + + def test_surname_parsed(self): + db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], + [self._person_rec(per_no=1)]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_primary_name().get_surname(), 'JONES') + + def test_given_name_parsed(self): + db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], + [self._person_rec(per_no=1)]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_primary_name().get_first_name(), 'Alice') + + def test_male_gender(self): + db, pmap = self._run([self._name_rec()], [self._person_rec(sex='M')]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_gender(), Person.MALE) + + def test_female_gender(self): + db, pmap = self._run([self._name_rec()], [self._person_rec(sex='F')]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_gender(), Person.FEMALE) + + def test_unknown_gender(self): + db, pmap = self._run([self._name_rec()], [self._person_rec(sex='?')]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_gender(), Person.UNKNOWN) + + def test_non_primary_name_skipped(self): + db, pmap = self._run( + [self._name_rec(primary=False, srnamedisp='ALT, Name')], + [self._person_rec()] + ) + self.assertEqual(db.get_number_of_people(), 0) + + def test_wrong_dataset_skipped(self): + db, pmap = self._run([self._name_rec(dsid=2)], [self._person_rec(dsid=2)], + dataset=1) + self.assertEqual(db.get_number_of_people(), 0) + + def test_no_comma_surname_only(self): + # srnamedisp with no comma → surname=full string, given='' + db, pmap = self._run([self._name_rec(srnamedisp='SMITH')], + [self._person_rec()]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_primary_name().get_surname(), 'SMITH') + self.assertEqual(p.get_primary_name().get_first_name(), '') + + +# --------------------------------------------------------------------------- +# link_person_events — EventRefs, birth/death special refs +# --------------------------------------------------------------------------- + +class TestLinkPersonEvents(unittest.TestCase): + + def _make_typed_event(self, db, event_type_int): + from gramps.gen.lib import EventType + ev = Event() + ev.set_type(EventType(event_type_int)) + with DbTxn("setup", db) as t: + db.add_event(ev, t) + return ev.get_handle() + + def test_individual_event_linked_to_person(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.OCCUPATION) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + p = db.get_person_from_handle(phandle) + self.assertEqual(len(p.get_event_ref_list()), 1) + + def test_couple_event_not_linked_to_person(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.MARRIAGE) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 2, 0)}) + p = db.get_person_from_handle(phandle) + self.assertEqual(len(p.get_event_ref_list()), 0) + + def test_birth_event_sets_birth_ref(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.BIRTH) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + p = db.get_person_from_handle(phandle) + self.assertIsNotNone(p.get_birth_ref()) + self.assertEqual(p.get_birth_ref().ref, ev_handle) + + def test_death_event_sets_death_ref(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.DEATH) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + p = db.get_person_from_handle(phandle) + self.assertIsNotNone(p.get_death_ref()) + + def test_unknown_person_skipped(self): + from gramps.gen.lib import EventType + db = _make_db() + ev_handle = self._make_typed_event(db, EventType.BIRTH) + # Must not raise even when per1 has no entry in per_no_map + libtmg.link_person_events(db, + per_no_map={}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + + def test_empty_maps_noop(self): + db = _make_db() + libtmg.link_person_events(db, None, None) + libtmg.link_person_events(db, {}, {}) + + +# --------------------------------------------------------------------------- +# import_families — parent-child grouping, couple events, rel type +# --------------------------------------------------------------------------- + +class TestImportFamilies(unittest.TestCase): + + def _run(self, tagtypes, pc_rels, per_no_by_gender, event_handle_map=None, + dataset=1): + """Create persons from per_no_by_gender={per_no: gender}, run import.""" + import unittest.mock as mock + db = _make_db() + pmap = {} + for per_no, gender in per_no_by_gender.items(): + p = Person() + p.set_gender(gender) + with DbTxn("setup", db) as t: + db.add_person(p, t) + pmap[per_no] = p.get_handle() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table(pc_rels), create=True): + libtmg.import_families(db, dataset, pmap, event_handle_map) + return db, pmap + + def _pc(self, parent, child, ptype, primary=True, pnote='', dsid=1): + return _Rec(dsid=dsid, parent=parent, child=child, + ptype=ptype, primary=primary, pnote=pnote) + + def _father_type(self, num=1): + return _Rec(dsid=1, etypenum=num, etypename='Father-Biological') + + def _mother_type(self, num=2): + return _Rec(dsid=1, etypenum=num, etypename='Mother-Biological') + + def test_father_child_creates_family(self): + db, pmap = self._run([self._father_type()], + [self._pc(1, 2, ptype=1)], + {1: Person.MALE, 2: Person.UNKNOWN}) + self.assertEqual(db.get_number_of_families(), 1) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_father_handle(), pmap[1]) + + def test_mother_child_creates_family(self): + db, pmap = self._run([self._mother_type()], + [self._pc(1, 2, ptype=2)], + {1: Person.FEMALE, 2: Person.UNKNOWN}) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_mother_handle(), pmap[1]) + + def test_father_and_mother_same_family(self): + db, pmap = self._run( + [self._father_type(1), self._mother_type(2)], + [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)], + {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}, + ) + self.assertEqual(db.get_number_of_families(), 1) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_father_handle(), pmap[1]) + self.assertEqual(fam.get_mother_handle(), pmap[2]) + + def test_child_added_to_family(self): + db, pmap = self._run([self._father_type()], + [self._pc(1, 2, ptype=1)], + {1: Person.MALE, 2: Person.UNKNOWN}) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(len(fam.get_child_ref_list()), 1) + self.assertEqual(fam.get_child_ref_list()[0].ref, pmap[2]) + + def test_child_ref_type_biological(self): + from gramps.gen.lib import ChildRefType + db, pmap = self._run([self._father_type()], + [self._pc(1, 2, ptype=1)], + {1: Person.MALE, 2: Person.UNKNOWN}) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_child_ref_list()[0].get_father_relation(), + ChildRefType.BIRTH) + + def test_wrong_dataset_skipped(self): + db, _ = self._run([self._father_type()], + [self._pc(1, 2, ptype=1, dsid=2)], + {1: Person.MALE, 2: Person.UNKNOWN}, dataset=1) + self.assertEqual(db.get_number_of_families(), 0) + + def test_no_per_no_map_is_noop(self): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table([]), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table([]), create=True): + libtmg.import_families(db, 1, per_no_map=None) + self.assertEqual(db.get_number_of_families(), 0) + + def test_marriage_event_sets_rel_type_married(self): + from gramps.gen.lib import EventType, FamilyRelType + import unittest.mock as mock + + db = _make_db() + pmap = {} + for per_no, gender in {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}.items(): + p = Person() + p.set_gender(gender) + with DbTxn("s", db) as t: + db.add_person(p, t) + pmap[per_no] = p.get_handle() + + ev = Event() + ev.set_type(EventType(EventType.MARRIAGE)) + with DbTxn("s", db) as t: + db.add_event(ev, t) + + tagtypes = [self._father_type(1), self._mother_type(2)] + pc = [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)] + event_handle_map = {99: (ev.get_handle(), 1, 2, 0)} + + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table(pc), create=True): + libtmg.import_families(db, 1, pmap, event_handle_map) + + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_relationship(), FamilyRelType.MARRIED) + self.assertEqual(len(fam.get_event_ref_list()), 1) + + +# --------------------------------------------------------------------------- +# import_repositories — name, type inference, URL, notes +# --------------------------------------------------------------------------- + +class TestImportRepositories(unittest.TestCase): + + def _run(self, repo_records, per_no_map=None, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgRepositories', _table(repo_records), create=True): + repo_map = libtmg.import_repositories(db, dataset, per_no_map) + return db, repo_map + + def _repo_rec(self, **kw): + defaults = dict(dsid=1, recno=1, name='City Library', + abbrev='', rnote='', rperno=0) + defaults.update(kw) + return _Rec(**defaults) + + def test_repository_created(self): + db, rmap = self._run([self._repo_rec()]) + self.assertEqual(db.get_number_of_repositories(), 1) + self.assertIn(1, rmap) + + def test_name_set(self): + db, rmap = self._run([self._repo_rec(name='National Archives')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(repo.get_name(), 'National Archives') + + def test_wrong_dataset_skipped(self): + db, rmap = self._run([self._repo_rec(dsid=2)], dataset=1) + self.assertEqual(db.get_number_of_repositories(), 0) + + def test_type_inferred_from_name(self): + from gramps.gen.lib import RepositoryType + db, rmap = self._run([self._repo_rec(name='ancestry.com')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(repo.get_type().value, RepositoryType.WEBSITE) + + def test_url_added_for_web_repo(self): + db, rmap = self._run([self._repo_rec(name='familysearch.org')]) + repo = db.get_repository_from_handle(rmap[1]) + urls = repo.get_url_list() + self.assertEqual(len(urls), 1) + self.assertIn('familysearch', urls[0].get_path()) + + def test_no_url_for_non_web_repo(self): + db, rmap = self._run([self._repo_rec(name='Local Parish Church')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(len(repo.get_url_list()), 0) + + def test_blank_name_falls_back_to_abbrev(self): + db, rmap = self._run([self._repo_rec(name='', abbrev='TNA')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(repo.get_name(), 'TNA') + + def test_note_added_when_rnote_set(self): + db, rmap = self._run([self._repo_rec(rnote='Open Mon-Fri')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(len(repo.get_note_list()), 1) + note = db.get_note_from_handle(repo.get_note_list()[0]) + self.assertIn('Open Mon-Fri', note.get()) + + def test_returns_recno_to_handle_map(self): + db, rmap = self._run([self._repo_rec(recno=42)]) + self.assertIn(42, rmap) + + +# --------------------------------------------------------------------------- +# import_citations — creation and attachment to events / persons +# --------------------------------------------------------------------------- + +class TestImportCitations(unittest.TestCase): + + def _run(self, citation_records, names_records=None, pc_records=None, + source_handle_map=None, event_handle_map=None, per_no_map=None, + dataset=1, db=None): + import unittest.mock as mock + if db is None: + db = _make_db() + with mock.patch('libtmg.tmgCitations', _table(citation_records), create=True), \ + mock.patch('libtmg.tmgNames', _table(names_records or []), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table(pc_records or []), create=True): + libtmg.import_citations(db, dataset, + source_handle_map=source_handle_map, + event_handle_map=event_handle_map, + per_no_map=per_no_map) + return db + + def _cit_rec(self, **kw): + defaults = dict(dsid=1, recno=1, majsource=1, stype='E', refrec=1, + exclude=False, subsource='', citref='', citmemo='', + sdsure='', snsure='', sssure='', spsure='', sfsure='') + defaults.update(kw) + return _Rec(**defaults) + + def test_no_source_map_is_noop(self): + db = self._run([self._cit_rec()]) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_citation_created(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec()], + source_handle_map={1: src.get_handle()}, db=db) + self.assertEqual(db.get_number_of_citations(), 1) + + def test_excluded_citation_skipped(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec(exclude=True)], + source_handle_map={1: src.get_handle()}, db=db) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_wrong_dataset_skipped(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec(dsid=2)], + source_handle_map={1: src.get_handle()}, db=db, dataset=1) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_unknown_source_skipped(self): + db = _make_db() + db = self._run([self._cit_rec(majsource=99)], + source_handle_map={1: 'some_handle'}, db=db) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_citation_attached_to_event(self): + db = _make_db() + src = Source() + ev = Event() + with DbTxn("s", db) as t: + db.add_source(src, t) + db.add_event(ev, t) + ev_handle = ev.get_handle() + + db = self._run([self._cit_rec(stype='E', refrec=7)], + source_handle_map={1: src.get_handle()}, + event_handle_map={7: (ev_handle, 1, 0, 0)}, + db=db) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(len(event.get_citation_list()), 1) + + def test_citation_attached_to_person_via_name(self): + db = _make_db() + src = Source() + p = Person() + with DbTxn("s", db) as t: + db.add_source(src, t) + db.add_person(p, t) + phandle = p.get_handle() + + # name recno=3 maps to nper=5; per_no_map routes nper=5 to phandle + db = self._run( + [self._cit_rec(stype='N', refrec=3)], + names_records=[_Rec(dsid=1, recno=3, nper=5)], + source_handle_map={1: src.get_handle()}, + per_no_map={5: phandle}, + db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_citation_list()), 1) + + def test_subsource_becomes_page(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec(subsource='p.42')], + source_handle_map={1: src.get_handle()}, db=db) + cit_handle = list(db.get_citation_handles())[0] + cit = db.get_citation_from_handle(cit_handle) + self.assertEqual(cit.get_page(), 'p.42') + + + +# ── TmgProject._read_pjc_config ────────────────────────────────────────── +def _make_minimal_sqz(tmp_dir, pjc_content): + """Create a minimal .SQZ zip containing only a PJC file.""" + import zipfile as _zf + pjc_path = os.path.join(tmp_dir, 'test.pjc') + sqz_path = os.path.join(tmp_dir, 'test.sqz') + with open(pjc_path, 'w', encoding='latin-1') as f: + f.write(pjc_content) + with _zf.ZipFile(sqz_path, 'w') as zf: + zf.write(pjc_path, 'test.pjc') + return sqz_path + + +class _MockUser: + """Minimal stand-in for the Gramps user object used in importData.""" + def __init__(self): + self.error_shown = False + self.error_message = None + self.uistate = None + + def notify_error(self, title, message=''): + self.error_shown = True + self.error_message = message + + def begin_progress(self, *a, **kw): pass + def end_progress(self): pass + def step_progress(self): pass + + +class TestReadPjcConfig(unittest.TestCase): + """Tests for TmgProject._read_pjc_config PJC parsing.""" + + _MINIMAL_PJC = ( + "[Stamp]\n" + "PjcVersion=11.0\n" + "[Researcher]\n" + "Name=Test User\n" + ) + + def _make_project(self, pjc_content, tmp_path): + pjc_file = os.path.join(tmp_path, "test.pjc") + with open(pjc_file, 'w', encoding='latin-1') as f: + f.write(pjc_content) + return libtmg.TmgProject(pjc_file) + + def test_well_formed_pjc_returns_version(self): + """A clean PJC file parses successfully and version() returns a float.""" + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(self._MINIMAL_PJC, tmp) + self.assertEqual(project.version(), 11.0) + + def test_malformed_section_header_does_not_raise(self): + """Lines like '[Exho' (no closing bracket) are silently dropped.""" + pjc = ( + "[Stamp]\n" + "PjcVersion=11.0\n" + "[Exho\n" # malformed — the crash trigger + "SomeGarbage\n" + "[Researcher]\n" + "Name=Test User\n" + ) + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(pjc, tmp) + # Must not raise; must still find [Stamp] + self.assertEqual(project.version(), 11.0) + + def test_null_bytes_stripped(self): + """NUL bytes in the PJC file are stripped before parsing.""" + pjc = "[Stamp]\x00\nPjcVersion=11.0\n" + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(pjc, tmp) + self.assertEqual(project.version(), 11.0) + + def test_parse_error_returns_partial_config(self): + """If configparser still raises after filtering, a warning is logged + and a (possibly empty) config object is returned rather than crashing.""" + import logging + # Feed content that survives the filter but still breaks configparser: + # a key=value line before any section header is technically invalid. + pjc = "orphan_key=value\n[Stamp]\nPjcVersion=11.0\n" + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(pjc, tmp) + with self.assertLogs('.TMGImport', level=logging.WARNING) as cm: + cfg = project._read_pjc_config() + self.assertTrue(any('parse error' in m.lower() or 'parsing' in m.lower() + for m in cm.output)) + # Config object is returned (not None), even if incomplete + self.assertIsNotNone(cfg) + + def test_version_too_old_notifies_user(self): + """A PJC version < 11.0 calls user.notify_error and aborts import.""" + pjc = "[Stamp]\nPjcVersion=10.0\n" # TMG 9.01 or earlier + with tempfile.TemporaryDirectory() as tmp: + sqz = _make_minimal_sqz(tmp, pjc) + db = _make_db() + user = _MockUser() + libtmg.importData(db, sqz, user) + self.assertTrue(user.error_shown, + "notify_error should have been called for old version") + self.assertIn('9.05', user.error_message or '', + "Error message should mention TMG 9.05") + + def test_missing_pjc_version_notifies_user(self): + """A PJC with no [Stamp]/PjcVersion calls user.notify_error.""" + pjc = "[OtherSection]\nSomeKey=value\n" # no [Stamp] at all + with tempfile.TemporaryDirectory() as tmp: + sqz = _make_minimal_sqz(tmp, pjc) + db = _make_db() + user = _MockUser() + libtmg.importData(db, sqz, user) + self.assertTrue(user.error_shown, + "notify_error should have been called for missing version") + +if __name__ == '__main__': + unittest.main()