diff --git a/app/distributions/export_models.py b/app/distributions/export_models.py index e60ff99..c759717 100644 --- a/app/distributions/export_models.py +++ b/app/distributions/export_models.py @@ -1,5 +1,6 @@ from typing import Any +from boto3.dynamodb.types import TypeSerializer from pydantic import BaseModel @@ -12,22 +13,9 @@ def as_dynamodb_item(self) -> dict[str, Any]: Returns: dict[str, Any]: The dataset represented as a DynamoDB item. """ - - def serialize(value: Any) -> dict[str, Any]: - if value is None: - return {"NULL": True} - if isinstance(value, int): - return {"N": str(value)} - if isinstance(value, str): - return {"S": value} - if isinstance(value, list): - return {"L": [serialize(i) for i in value]} - if isinstance(value, dict): - return {"M": {k: serialize(v) for k, v in value.items()}} - raise ValueError(f"Unexpected type {type(value)}") - - item = self.model_dump(mode="json") - return {key: serialize(value) for key, value in item.items()} + serializer = TypeSerializer() + item = self.model_dump() + return {key: serializer.serialize(value) for key, value in item.items()} class ExportDataset(BaseModelWithDynamoDBSerialization): diff --git a/app/distributions/management/commands/geocat_harvest.py b/app/distributions/management/commands/geocat_harvest.py index 64ada76..9dcd0c5 100644 --- a/app/distributions/management/commands/geocat_harvest.py +++ b/app/distributions/management/commands/geocat_harvest.py @@ -27,11 +27,39 @@ GEOCAT_URL = "https://www.geocat.ch/geonetwork/srv/api/records/{}/formatters/xml?approved=true" NS = { - "che": "http://www.geocat.ch/2008/che", - "gco": "http://www.isotc211.org/2005/gco", - "gmd": "http://www.isotc211.org/2005/gmd", - "gmx": "http://www.isotc211.org/2005/gmx", + "cat": "http://standards.iso.org/iso/19115/-3/cat/1.0", + "che": "http://geocat.ch/che", + "cit": "http://standards.iso.org/iso/19115/-3/cit/2.0", + "dqm": "http://standards.iso.org/iso/19157/-2/dqm/1.0", + "gco": "http://standards.iso.org/iso/19115/-3/gco/1.0", + "gcx": "http://standards.iso.org/iso/19115/-3/gcx/1.0", + "gex": "http://standards.iso.org/iso/19115/-3/gex/1.0", + "gfc": "http://standards.iso.org/iso/19110/gfc/1.1", + "gml": "http://www.opengis.net/gml/3.2", + "lan": "http://standards.iso.org/iso/19115/-3/lan/1.0", + "mac": "http://standards.iso.org/iso/19115/-3/mac/2.0", + "mas": "http://standards.iso.org/iso/19115/-3/mas/1.0", + "mcc": "http://standards.iso.org/iso/19115/-3/mcc/1.0", + "mco": "http://standards.iso.org/iso/19115/-3/mco/1.0", + "md1": "http://standards.iso.org/iso/19115/-3/md1/2.0", + "md2": "http://standards.iso.org/iso/19115/-3/md2/2.0", + "mda": "http://standards.iso.org/iso/19115/-3/mda/2.0", + "mdb": "http://standards.iso.org/iso/19115/-3/mdb/2.0", + "mdq": "http://standards.iso.org/iso/19157/-2/mdq/1.0", + "mds": "http://standards.iso.org/iso/19115/-3/mds/2.0", + "mdt": "http://standards.iso.org/iso/19115/-3/mdt/2.0", + "mex": "http://standards.iso.org/iso/19115/-3/mex/1.0", + "mmi": "http://standards.iso.org/iso/19115/-3/mmi/1.0", + "mpc": "http://standards.iso.org/iso/19115/-3/mpc/1.0", + "mrc": "http://standards.iso.org/iso/19115/-3/mrc/2.0", + "mrd": "http://standards.iso.org/iso/19115/-3/mrd/1.0", + "mri": "http://standards.iso.org/iso/19115/-3/mri/1.0", + "mrl": "http://standards.iso.org/iso/19115/-3/mrl/2.0", + "mrs": "http://standards.iso.org/iso/19115/-3/mrs/1.0", + "msr": "http://standards.iso.org/iso/19115/-3/msr/2.0", + "srv": "http://standards.iso.org/iso/19115/-3/srv/2.0", "xlink": "http://www.w3.org/1999/xlink", + "xsi": "http://www.w3.org/2001/XMLSchema-instance", } TIMEOUT = 30 @@ -253,31 +281,54 @@ def harvest_keywords( # pylint: disable=too-many-positional-arguments,too-many- self.print(f"Getting keywords for dataset {dataset_id} from geocat") keywords = [] - for block in root.findall(".//gmd:MD_Keywords", NS): + for block in root.findall(".//mri:MD_Keywords", NS): keyword_type = None - if (element := block.find(".//gmd:MD_KeywordTypeCode", NS)) is not None: + if (element := block.find(".//mri:MD_KeywordTypeCode", NS)) is not None: keyword_type = element.get("codeListValue") + # Note: eCH defines the thesaurus id as mcc:code only very vaguely as "Zeichenfolge". + # In reality this is typically encapsulated in an gcx:Anchor. But other examples + # (outside of thesauri) of mcc:code encapsulate this in a gco:CharacterString. thesaurus_id = None thesaurus_url = None - if (element := block.find(".//gmd:thesaurusName//gmx:Anchor", NS)) is not None: + if ( + element := + block.find(".//mri:thesaurusName//cit:identifier//mcc:code//gcx:Anchor", NS) + ) is not None: thesaurus_id = element.text thesaurus_url = next( (element.get(key) for key in element.keys() if "href" in str(key)), None, ) + elif ( + element := block.find( + ".//mri:thesaurusName//cit:identifier//mcc:code//gco:CharacterString", NS + ) + ) is not None: + thesaurus_id = element.text + elif ( + element := block.find(".//mri:thesaurusName//cit:identifier//mcc:code", NS) + ) is not None: + thesaurus_id = element.text + + if thesaurus_id: + thesaurus_id = thesaurus_id.strip() - thesaurus_date = None - if (element := block.find(".//gmd:thesaurusName//gco:Date", NS)) is not None: - thesaurus_date = element.text + # Note: eCH defines different types of dates (publication, update etc.), we only care + # about the last of these + thesaurus_dates = sorted({ + str(element.text) + for element in block.findall(".//mri:thesaurusName//gco:Date", NS) + }) + thesaurus_date = thesaurus_dates[-1] if thesaurus_dates else None - for element in block.findall("gmd:keyword", NS): + for element in block.findall("mri:keyword", NS): if (text := element.find("gco:CharacterString", NS)) is None or not text.text: continue translations = { translation.get("locale", "").lstrip("#").lower(): translation.text - for translation in element.findall(".//gmd:LocalisedCharacterString", NS) + for translation in element.findall(".//lan:LocalisedCharacterString", NS) if translation.text } @@ -326,78 +377,77 @@ def find_code(element: etree._Element, path: str) -> str | None: return None contacts: list[Contact] = [] - for block in root.findall(".//gmd:pointOfContact", NS): + for block in root.findall(".//mri:pointOfContact", NS): - name = find(block, './/gmd:organisationName/gco:CharacterString') + name = find(block, './/che:CHE_CI_Organisation/cit:name/gco:CharacterString') if not name: continue - role = find_code(block, ".//gmd:CI_RoleCode") + role = find_code(block, ".//cit:CI_RoleCode") online_resources = [ OnlineResource( - url=find(resource, ".//gmd:linkage/gmd:URL"), - url_de=find(resource, './/che:LocalisedURL[@locale="#DE"]'), - url_fr=find(resource, './/che:LocalisedURL[@locale="#FR"]'), - url_en=find(resource, './/che:LocalisedURL[@locale="#EN"]'), - url_it=find(resource, './/che:LocalisedURL[@locale="#IT"]'), - url_rm=find(resource, './/che:LocalisedURL[@locale="#RM"]'), - protocol=find(resource, ".//gmd:protocol/*"), - name_de=find(resource, './/gmd:name//*[@locale="#DE"]'), - name_fr=find(resource, './/gmd:name//*[@locale="#FR"]'), - name_en=find(resource, './/gmd:name//*[@locale="#EN"]'), - name_it=find(resource, './/gmd:name//*[@locale="#IT"]'), - name_rm=find(resource, './/gmd:name//*[@locale="#RM"]'), - description_de=find(resource, './/gmd:description//*[@locale="#DE"]'), - description_fr=find(resource, './/gmd:description//*[@locale="#FR"]'), - description_en=find(resource, './/gmd:description//*[@locale="#EN"]'), - description_it=find(resource, './/gmd:description//*[@locale="#IT"]'), - description_rm=find(resource, './/gmd:description//*[@locale="#RM"]'), - function=find_code(resource, ".//gmd:function/*"), - ) for resource in block.findall(".//gmd:CI_OnlineResource", NS) + url=find(resource, ".//cit:linkage/gco:CharacterString"), + url_de=find(resource, './/cit:linkage//*[@locale="#DE"]'), + url_fr=find(resource, './/cit:linkage//*[@locale="#FR"]'), + url_en=find(resource, './/cit:linkage//*[@locale="#EN"]'), + url_it=find(resource, './/cit:linkage//*[@locale="#IT"]'), + url_rm=find(resource, './/cit:linkage//*[@locale="#RM"]'), + protocol=find(resource, ".//cit:protocol/*"), + name_de=find(resource, './/cit:name//*[@locale="#DE"]'), + name_fr=find(resource, './/cit:name//*[@locale="#FR"]'), + name_en=find(resource, './/cit:name//*[@locale="#EN"]'), + name_it=find(resource, './/cit:name//*[@locale="#IT"]'), + name_rm=find(resource, './/cit:name//*[@locale="#RM"]'), + description_de=find(resource, './/cit:description//*[@locale="#DE"]'), + description_fr=find(resource, './/cit:description//*[@locale="#FR"]'), + description_en=find(resource, './/cit:description//*[@locale="#EN"]'), + description_it=find(resource, './/cit:description//*[@locale="#IT"]'), + description_rm=find(resource, './/cit:description//*[@locale="#RM"]'), + function=find_code(resource, ".//cit:function/*"), + ) for resource in block.findall(".//cit:CI_OnlineResource", NS) ] online_resources.sort(key=lambda r: r.url or "") - parts = ( - find(block, ".//che:streetName/*"), - find(block, ".//che:streetNumber/*"), - find(block, ".//che:postBox/*") - ) - delivery_point = " ".join(part for part in parts if part) or None + phone_numbers = { + find(resource, ".//cit:CI_TelephoneTypeCode"): + find(resource, ".//cit:number/gco:CharacterString") + for resource in block.findall(".//cit:CI_Telephone", NS) + } emails = [ - email for element in block.findall(".//gmd:electronicMailAddress//*", NS) + email for element in block.findall(".//cit:electronicMailAddress//*", NS) if (email := getattr(element, "text", None)) ] contact = Contact( role=role, org_name=name, - org_name_de=find(block, './/gmd:organisationName//*[@locale="#DE"]'), - org_name_fr=find(block, './/gmd:organisationName//*[@locale="#FR"]'), - org_name_en=find(block, './/gmd:organisationName//*[@locale="#EN"]'), - org_name_it=find(block, './/gmd:organisationName//*[@locale="#IT"]'), - org_name_rm=find(block, './/gmd:organisationName//*[@locale="#RM"]'), + org_name_de=find(block, './/che:CHE_CI_Organisation/cit:name//*[@locale="#DE"]'), + org_name_fr=find(block, './/che:CHE_CI_Organisation/cit:name//*[@locale="#FR"]'), + org_name_en=find(block, './/che:CHE_CI_Organisation/cit:name//*[@locale="#EN"]'), + org_name_it=find(block, './/che:CHE_CI_Organisation/cit:name//*[@locale="#IT"]'), + org_name_rm=find(block, './/che:CHE_CI_Organisation/cit:name//*[@locale="#RM"]'), org_acronym=find(block, './/che:organisationAcronym/gco:CharacterString'), org_acronym_de=find(block, './/che:organisationAcronym//*[@locale="#DE"]'), org_acronym_fr=find(block, './/che:organisationAcronym//*[@locale="#FR"]'), org_acronym_en=find(block, './/che:organisationAcronym//*[@locale="#EN"]'), org_acronym_it=find(block, './/che:organisationAcronym//*[@locale="#IT"]'), org_acronym_rm=find(block, './/che:organisationAcronym//*[@locale="#RM"]'), - position_name_de=find(block, './/gmd:positionName//*[@locale="#DE"]'), - position_name_fr=find(block, './/gmd:positionName//*[@locale="#FR"]'), - position_name_en=find(block, './/gmd:positionName//*[@locale="#EN"]'), - position_name_it=find(block, './/gmd:positionName//*[@locale="#IT"]'), - position_name_rm=find(block, './/gmd:positionName//*[@locale="#RM"]'), - contact_voice=find(block, ".//gmd:voice/*"), - contact_facsimile=find(block, ".//gmd:facsimile/*"), - contact_sms=None, - contact_city=find(block, ".//gmd:city/*"), - contact_administrative_area=find(block, ".//gmd:administrativeArea/*"), - contact_postal_code=find(block, ".//gmd:postalCode/*"), - contact_country=find(block, ".//gmd:country/*"), + position_name_de=find(block, './/cit:CI_Individual//*[@locale="#DE"]'), + position_name_fr=find(block, './/cit:CI_Individual//*[@locale="#FR"]'), + position_name_en=find(block, './/cit:CI_Individual//*[@locale="#EN"]'), + position_name_it=find(block, './/cit:CI_Individual//*[@locale="#IT"]'), + position_name_rm=find(block, './/cit:CI_Individual//*[@locale="#RM"]'), + contact_voice=phone_numbers.get('voice'), + contact_facsimile=phone_numbers.get('facsimile'), + contact_sms=phone_numbers.get('sms'), + contact_city=find(block, ".//cit:city/*"), + contact_administrative_area=find(block, ".//cit:administrativeArea/*"), + contact_postal_code=find(block, ".//cit:postalCode/*"), + contact_country=find(block, ".//cit:country/*"), contact_electronic_mail_addresses=emails, - contact_delivery_point=delivery_point, + contact_delivery_point=find(block, ".//cit:deliveryPoint/*"), online_resources=online_resources ) diff --git a/app/tests/distributions/data/geocat_entry.zip b/app/tests/distributions/data/geocat_entry.zip index 52d6744..66aff3d 100644 Binary files a/app/tests/distributions/data/geocat_entry.zip and b/app/tests/distributions/data/geocat_entry.zip differ diff --git a/app/tests/distributions/test_export_models.py b/app/tests/distributions/test_export_models.py new file mode 100644 index 0000000..c17a997 --- /dev/null +++ b/app/tests/distributions/test_export_models.py @@ -0,0 +1,61 @@ +from decimal import Decimal + +from distributions.export_models import BaseModelWithDynamoDBSerialization + + +def test_base_model_with_dynamo_db_serialization(): + + class MyModel(BaseModelWithDynamoDBSerialization): + string: str + number_int: Decimal + number_float: Decimal + list: list + map: dict + null: None + bool_true: bool + bool_false: bool + + model = MyModel( + string="value", + number_int=Decimal(10), + number_float=Decimal("10.1"), + list=["value"], + map={"key": "value"}, + null=None, + bool_true=True, + bool_false=False, + ) + item = model.as_dynamodb_item() + + assert item == { + "string": { + "S": "value" + }, + "number_int": { + "N": "10" + }, + "number_float": { + "N": "10.1" + }, + "list": { + "L": [{ + "S": "value" + }] + }, + "map": { + "M": { + "key": { + "S": "value" + } + } + }, + "null": { + "NULL": True + }, + "bool_true": { + "BOOL": True + }, + "bool_false": { + "BOOL": False + }, + } diff --git a/app/tests/distributions/test_geocat_harvest_command.py b/app/tests/distributions/test_geocat_harvest_command.py index 606c2ef..7f1112b 100644 --- a/app/tests/distributions/test_geocat_harvest_command.py +++ b/app/tests/distributions/test_geocat_harvest_command.py @@ -32,7 +32,10 @@ def not_found_response(): @patch('distributions.management.commands.geocat_harvest.Session') def test_command_harvests(dynamo_session_mock, requests_get_mock, db): requests_get_mock.side_effect = [ - xml_response("geocat_entry"), xml_response("geocat_thesaurus"), not_found_response() + xml_response("geocat_entry"), + xml_response("geocat_thesaurus"), + not_found_response(), + not_found_response() ] BodDataset.objects.create( @@ -44,7 +47,6 @@ def test_command_harvests(dynamo_session_mock, requests_get_mock, db): # Note: These are the actual data in geocat at the time of writing this test, not parsing # errors ;) - assert dynamo_session_mock.mock_calls == [ call(), call().client('dynamodb', region_name='eu-central-1'), @@ -69,11 +71,11 @@ def test_command_harvests(dynamo_session_mock, requests_get_mock, db): }, 'thesaurus_url': { 'S': - 'https://geocat-int.dev.bgdi.ch/geonetwork/srv/api/registries/vocabularies/local.theme.geocat.ch' + 'https://geocat-dev.dev.bgdi.ch/geonetwork/srv/api/registries/vocabularies/local.theme.geocat.ch' }, 'thesaurus_date': { - 'S': '2024-06-13' + 'S': '2026-03-13' }, 'concept': { @@ -112,7 +114,7 @@ def test_command_harvests(dynamo_session_mock, requests_get_mock, db): }, 'thesaurus_url': { 'S': - 'https://geocat-int.dev.bgdi.ch/geonetwork/srv/api/registries/vocabularies/local.theme.geocat.ch' + 'https://geocat-dev.dev.bgdi.ch/geonetwork/srv/api/registries/vocabularies/local.theme.geocat.ch' }, 'thesaurus_date': { 'S': '2024-06-13' @@ -148,10 +150,10 @@ def test_command_harvests(dynamo_session_mock, requests_get_mock, db): }, 'thesaurus_url': { 'S': - 'https://geocat-int.dev.bgdi.ch/geonetwork/srv/api/registries/vocabularies/local.theme.geocat.ch' + 'https://geocat-dev.dev.bgdi.ch/geonetwork/srv/api/registries/vocabularies/local.theme.geocat.ch' }, 'thesaurus_date': { - 'S': '2024-06-13' + 'S': '2026-03-13' }, 'concept': { 'S': @@ -186,10 +188,10 @@ def test_command_harvests(dynamo_session_mock, requests_get_mock, db): }, 'thesaurus_url': { 'S': - 'https://geocat-int.dev.bgdi.ch/geonetwork/srv/api/registries/vocabularies/local.theme.geocat.ch' + 'https://geocat-dev.dev.bgdi.ch/geonetwork/srv/api/registries/vocabularies/local.theme.geocat.ch' }, 'thesaurus_date': { - 'S': '2024-06-13' + 'S': '2026-03-13' }, 'concept': { 'S': @@ -315,7 +317,7 @@ def test_command_harvests(dynamo_session_mock, requests_get_mock, db): 'L': [{ 'M': { 'url': { - 'NULL': True + 'S': 'https://www.bafu.admin.ch/bafu/de/home.html' }, 'url_de': { 'S': 'https://www.bafu.admin.ch/bafu/de/home.html' @@ -438,7 +440,7 @@ def test_command_harvests(dynamo_session_mock, requests_get_mock, db): 'S': '+41 58 462 93 89' }, 'contact_facsimile': { - 'S': '+41 58 463 89 74' + 'NULL': True }, 'contact_sms': { 'NULL': True @@ -467,23 +469,24 @@ def test_command_harvests(dynamo_session_mock, requests_get_mock, db): 'L': [{ 'M': { 'url': { - 'NULL': True + 'S': + 'https://www.bafu.admin.ch/bafu/de/home/amt/geschaeftsleitung-des-bafu/direktionsbereich-biologische-vielfalt/abteilung-biodiversitaet-und-landschaft.html' }, 'url_de': { 'S': - 'https://www.bafu.admin.ch/bafu/de/home/amt/abteilungen-sektionen/abteilung-biodiversitaet-und-landschaft.html' + 'https://www.bafu.admin.ch/bafu/de/home/amt/geschaeftsleitung-des-bafu/direktionsbereich-biologische-vielfalt/abteilung-biodiversitaet-und-landschaft.html' }, 'url_fr': { 'S': - 'https://www.bafu.admin.ch/bafu/fr/home/office/divisions-sections/division-biodiversite-et-paysage.html' + 'https://www.bafu.admin.ch/bafu/fr/home/office/direction/domaine-de-direction-biodiversite/division-biodiversite-et-paysage.html' }, 'url_en': { 'S': - 'https://www.bafu.admin.ch/bafu/en/home/office/divisions-sections/biodiversity-and-landscape-division.html' + 'https://www.bafu.admin.ch/bafu/en/home/office/management-board/sector-biological-diversity/biodiversity-and-landscape-division.html' }, 'url_it': { 'S': - 'https://www.bafu.admin.ch/bafu/it/home/ufficio/divisioni-sezioni/divisione-biodiversita-e-paesaggio.html' + 'https://www.bafu.admin.ch/bafu/it/home/ufficio/direzione/unita-direzione-biodiversita/divisione-biodiversita-e-paesaggio.html' }, 'url_rm': { 'NULL': True